diff --git a/.gitattributes b/.gitattributes index 744578b54..e5c5394e6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -53,6 +53,7 @@ *.engine filter=lfs diff=lfs merge=lfs -text *.sbsar filter=lfs diff=lfs merge=lfs -text *.msgpack filter=lfs diff=lfs merge=lfs -text +GameData/*.cc filter=lfs diff=lfs merge=lfs -text *.a filter=lfs diff=lfs merge=lfs -text *AkSoundEngine filter=lfs diff=lfs merge=lfs -text *AkWaapiClient filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore index f43ec6271..7fb0d0527 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,9 @@ ExportedObj/ /*.sln /*.csproj +# CultCache single-file store locks +GameData/*.lock + # Per-user editor state /[Uu]ser[Ss]ettings/ diff --git a/Assets/Scripts/ServerShared/AetheriaStores.cs b/Assets/Scripts/ServerShared/AetheriaStores.cs index 4394e89d6..924777f41 100644 --- a/Assets/Scripts/ServerShared/AetheriaStores.cs +++ b/Assets/Scripts/ServerShared/AetheriaStores.cs @@ -6,9 +6,9 @@ using GameCult.Caching.MessagePack; // Aetheria's stores: one home store per type, composed here and nowhere else. public static class AetheriaStores { - public static readonly Type[] CatalogTypes = { typeof(ItemData), typeof(Faction), typeof(FactionProductData), typeof(PersonalityAttribute), typeof(NameFile) }; + public static readonly Type[] CatalogTypes = { typeof(ItemData), typeof(Faction), typeof(FactionProductData), typeof(PersonalityAttribute), typeof(NameFile), typeof(InputLayout) }; public static readonly Type[] RunTypes = { typeof(OrbitData), typeof(BodyData), typeof(SavedZone), typeof(SavedGame) }; - public static readonly Type[] PlayerTypes = { typeof(PlayerSettings), typeof(InputLayout) }; + public static readonly Type[] PlayerTypes = { typeof(PlayerSettings) }; // Attaches (hydrates) the catalog read-only unless catalogWritable, then the run and player stores when given. // A read-only catalog throws when a [CultGlobal] type routed to it has no record. A writable catalog is being diff --git a/GameData/AetherDB.msgpack b/GameData/AetherDB.msgpack deleted file mode 100644 index c61b7845e..000000000 --- a/GameData/AetherDB.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8963a68670ae8e8e0ac411c96de7a39322802293b91f53f4d3f50717b06429aa -size 46150 diff --git a/GameData/Aetheria.cc b/GameData/Aetheria.cc new file mode 100644 index 000000000..a8eb4780c --- /dev/null +++ b/GameData/Aetheria.cc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e65d8ea6af07d0a3a037b4e66cbf0d57ff0b0cfbb24804f89c2fa9f6dd2b4a75 +size 12686407 diff --git a/GameData/KeyboardLayouts/ansi104.json b/GameData/KeyboardLayouts/ansi104.json deleted file mode 100644 index 44a2ba7e6..000000000 --- a/GameData/KeyboardLayouts/ansi104.json +++ /dev/null @@ -1 +0,0 @@ -{"rows":[{"columns":[{"mainLabel":"Esc","altLabel":"","path":"escape","width":1.0,"$type":"InputLayoutBindableKey"},{"width":1.0,"$type":"InputLayoutColumnSpacer"},{"mainLabel":"F1","altLabel":"","path":"f1","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"F2","altLabel":"","path":"f2","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"F3","altLabel":"","path":"f3","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"F4","altLabel":"","path":"f4","width":1.0,"$type":"InputLayoutBindableKey"},{"width":0.5,"$type":"InputLayoutColumnSpacer"},{"mainLabel":"F5","altLabel":"","path":"f5","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"F6","altLabel":"","path":"f6","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"F7","altLabel":"","path":"f7","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"F8","altLabel":"","path":"f8","width":1.0,"$type":"InputLayoutBindableKey"},{"width":0.5,"$type":"InputLayoutColumnSpacer"},{"mainLabel":"F9","altLabel":"","path":"f9","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"F10","altLabel":"","path":"f10","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"F11","altLabel":"","path":"f11","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"F12","altLabel":"","path":"f12","width":1.0,"$type":"InputLayoutBindableKey"},{"width":0.25,"$type":"InputLayoutColumnSpacer"},{"width":1.0,"$type":"InputLayoutKey"},{"width":1.0,"$type":"InputLayoutKey"},{"width":1.0,"$type":"InputLayoutKey"}],"$type":"InputLayoutKeyRow"},{"height":0.5,"$type":"InputLayoutRowSpacer"},{"columns":[{"mainLabel":"`","altLabel":"~","path":"backquote","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"1","altLabel":"!","path":"1","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"2","altLabel":"@","path":"2","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"3","altLabel":"#","path":"3","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"4","altLabel":"$","path":"4","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"5","altLabel":"%","path":"5","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"6","altLabel":"^","path":"6","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"7","altLabel":"&","path":"7","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"8","altLabel":"*","path":"8","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"9","altLabel":"(","path":"9","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"0","altLabel":")","path":"0","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"-","altLabel":"_","path":"minus","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"=","altLabel":"+","path":"equals","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"Backspace","altLabel":"","path":"backspace","width":2.0,"$type":"InputLayoutBindableKey"},{"width":0.25,"$type":"InputLayoutColumnSpacer"},{"mainLabel":"Insert","altLabel":"","path":"insert","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"Home","altLabel":"","path":"home","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"PgUp","altLabel":"","path":"pageUp","width":1.0,"$type":"InputLayoutBindableKey"},{"width":0.25,"$type":"InputLayoutColumnSpacer"},{"width":1.0,"$type":"InputLayoutKey"},{"mainLabel":"/","altLabel":"","path":"numpadDivide","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"*","altLabel":"","path":"numpadMultiply","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"-","altLabel":"","path":"numpadMinus","width":1.0,"$type":"InputLayoutBindableKey"}],"$type":"InputLayoutKeyRow"},{"columns":[{"mainLabel":"Tab","altLabel":"","path":"tab","width":1.5,"$type":"InputLayoutBindableKey"},{"mainLabel":"Q","altLabel":"","path":"q","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"W","altLabel":"","path":"w","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"E","altLabel":"","path":"e","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"R","altLabel":"","path":"r","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"T","altLabel":"","path":"t","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"Y","altLabel":"","path":"y","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"U","altLabel":"","path":"u","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"I","altLabel":"","path":"i","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"O","altLabel":"","path":"o","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"P","altLabel":"","path":"p","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"[","altLabel":"{","path":"leftBracket","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"]","altLabel":"}","path":"rightBracket","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"\\","altLabel":"|","path":"backslash","width":1.5,"$type":"InputLayoutBindableKey"},{"width":0.25,"$type":"InputLayoutColumnSpacer"},{"mainLabel":"Delete","altLabel":"","path":"delete","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"End","altLabel":"","path":"end","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"PgDn","altLabel":"","path":"pageDown","width":1.0,"$type":"InputLayoutBindableKey"},{"width":0.25,"$type":"InputLayoutColumnSpacer"},{"mainLabel":"7","altLabel":"Home","path":"numpad7","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"8","altLabel":"↑","path":"numpad8","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"9","altLabel":"PgUp","path":"numpad9","width":1.0,"$type":"InputLayoutBindableKey"},{"height":2,"mainLabel":"+","altLabel":"","path":"numpadPlus","width":1.0,"$type":"InputLayoutMultiRowKey"}],"$type":"InputLayoutKeyRow"},{"columns":[{"mainLabel":"Caps Lock","altLabel":"","path":"capsLock","width":1.75,"$type":"InputLayoutBindableKey"},{"mainLabel":"A","altLabel":"","path":"a","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"S","altLabel":"","path":"s","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"D","altLabel":"","path":"d","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"F","altLabel":"","path":"f","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"G","altLabel":"","path":"g","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"H","altLabel":"","path":"h","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"J","altLabel":"","path":"j","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"K","altLabel":"","path":"k","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"L","altLabel":"","path":"l","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":";","altLabel":":","path":"semicolon","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"'","altLabel":"\"","path":"quote","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"Enter","altLabel":"","path":"enter","width":2.25,"$type":"InputLayoutBindableKey"},{"width":3.5,"$type":"InputLayoutColumnSpacer"},{"mainLabel":"4","altLabel":"←","path":"numpad4","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"5","altLabel":"","path":"numpad5","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"6","altLabel":"→","path":"numpad6","width":1.0,"$type":"InputLayoutBindableKey"}],"$type":"InputLayoutKeyRow"},{"columns":[{"mainLabel":"Shift","altLabel":"","path":"leftShift","width":2.25,"$type":"InputLayoutBindableKey"},{"mainLabel":"Z","altLabel":"","path":"z","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"X","altLabel":"","path":"x","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"C","altLabel":"","path":"c","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"V","altLabel":"","path":"v","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"B","altLabel":"","path":"b","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"N","altLabel":"","path":"n","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"M","altLabel":"","path":"m","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":",","altLabel":"<","path":"comma","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":".","altLabel":">","path":"period","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"/","altLabel":"?","path":"slash","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"Shift","altLabel":"","path":"rightShift","width":2.75,"$type":"InputLayoutBindableKey"},{"width":1.25,"$type":"InputLayoutColumnSpacer"},{"mainLabel":"↑","altLabel":"","path":"upArrow","width":1.0,"$type":"InputLayoutBindableKey"},{"width":1.25,"$type":"InputLayoutColumnSpacer"},{"mainLabel":"1","altLabel":"End","path":"numpad1","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"2","altLabel":"↓","path":"numpad2","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"3","altLabel":"PgDn","path":"numpad3","width":1.0,"$type":"InputLayoutBindableKey"},{"height":2,"mainLabel":"Enter","altLabel":"","path":"numpadEnter","width":1.0,"$type":"InputLayoutMultiRowKey"}],"$type":"InputLayoutKeyRow"},{"columns":[{"mainLabel":"Ctrl","altLabel":"","path":"leftCtrl","width":1.25,"$type":"InputLayoutBindableKey"},{"width":1.25,"$type":"InputLayoutKey"},{"mainLabel":"Alt","altLabel":"","path":"leftAlt","width":1.25,"$type":"InputLayoutBindableKey"},{"mainLabel":"Spacebar","altLabel":"","path":"space","width":6.25,"$type":"InputLayoutBindableKey"},{"mainLabel":"Alt","altLabel":"","path":"rightAlt","width":1.25,"$type":"InputLayoutBindableKey"},{"width":1.25,"$type":"InputLayoutKey"},{"width":1.25,"$type":"InputLayoutKey"},{"mainLabel":"Ctrl","altLabel":"","path":"rightCtrl","width":1.25,"$type":"InputLayoutBindableKey"},{"width":0.25,"$type":"InputLayoutColumnSpacer"},{"mainLabel":"←","altLabel":"","path":"leftArrow","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"↓","altLabel":"","path":"downArrow","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"→","altLabel":"","path":"rightArrow","width":1.0,"$type":"InputLayoutBindableKey"},{"width":0.25,"$type":"InputLayoutColumnSpacer"},{"mainLabel":"0","altLabel":"Ins","path":"numpad0","width":2.0,"$type":"InputLayoutBindableKey"},{"mainLabel":".","altLabel":"Del","path":"numpadPeriod","width":1.0,"$type":"InputLayoutBindableKey"}],"$type":"InputLayoutKeyRow"}]} \ No newline at end of file diff --git a/GameData/KeyboardLayouts/ansi104.msgpack b/GameData/KeyboardLayouts/ansi104.msgpack deleted file mode 100644 index 839a6dcc1..000000000 --- a/GameData/KeyboardLayouts/ansi104.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:77a2be69d0ba3fc3d7b89183c8cac69137728b7f298547fc2263cfdd69862ce7 -size 1961 diff --git a/GameData/NameFile/Australia.msgpack b/GameData/NameFile/Australia.msgpack deleted file mode 100644 index a898a8e8d..000000000 --- a/GameData/NameFile/Australia.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b65a87570c0bbaecef45b047b5673cc83fbff5354ea0de68ca530cee290d30d6 -size 442830 diff --git a/GameData/NameFile/Congo.msgpack b/GameData/NameFile/Congo.msgpack deleted file mode 100644 index 895ec9238..000000000 --- a/GameData/NameFile/Congo.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:bdaa83a28f9448061b043115bdcbe09448f1eaf46c1cf44a78fe0d9044db9fd0 -size 506753 diff --git a/GameData/NameFile/Germany.msgpack b/GameData/NameFile/Germany.msgpack deleted file mode 100644 index 6241d8117..000000000 --- a/GameData/NameFile/Germany.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1f6fff7465869365a5dbb9aece1e6f2f3c72617808e66a942915d371ca22cd05 -size 1284248 diff --git a/GameData/NameFile/India.msgpack b/GameData/NameFile/India.msgpack deleted file mode 100644 index 9a79b7e17..000000000 --- a/GameData/NameFile/India.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:09cc5aa83fb1d97461846f86a0976dc58fb4730d7e27cc6e4f832f77b755f81a -size 3229371 diff --git a/GameData/NameFile/Iran.msgpack b/GameData/NameFile/Iran.msgpack deleted file mode 100644 index 0d29bb504..000000000 --- a/GameData/NameFile/Iran.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:275965b57ba3e75f1fd7b36339b843992a9a9a1b318d3bf08d940d458e7c7b79 -size 1073151 diff --git a/GameData/NameFile/Japan.msgpack b/GameData/NameFile/Japan.msgpack deleted file mode 100644 index 7ed8c9b37..000000000 --- a/GameData/NameFile/Japan.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f3d7c59066055e76cc2f199ca8d47c6ca14ac20ad4cab66a273b72a0eb63804a -size 1031188 diff --git a/GameData/NameFile/Mexico.msgpack b/GameData/NameFile/Mexico.msgpack deleted file mode 100644 index 19cf62fba..000000000 --- a/GameData/NameFile/Mexico.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f12af0d8bb159f41b6618a8a2f5ac7ddf2c0a2b1dc449dd21188416f2ccc4427 -size 633464 diff --git a/GameData/NameFile/Russia.msgpack b/GameData/NameFile/Russia.msgpack deleted file mode 100644 index ae5a5f51c..000000000 --- a/GameData/NameFile/Russia.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2f3b5155c9003056d1fd33ad4a665d7f9330bed35bb45ab01bbdd05ef0cb006d -size 2375362 diff --git a/GameData/NameFile/Swahili.msgpack b/GameData/NameFile/Swahili.msgpack deleted file mode 100644 index 3d7b45f19..000000000 --- a/GameData/NameFile/Swahili.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5227513492cd33f502506dba741b99d5c7e9140318a44a5d84579258397aa227 -size 501035 diff --git a/GameData/NameFile/Thailand.msgpack b/GameData/NameFile/Thailand.msgpack deleted file mode 100644 index 8c4f33b4b..000000000 --- a/GameData/NameFile/Thailand.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8352d97cd61b83b07dd9ce549358436800c588263147764df6bd5b556124f2cb -size 143118 diff --git a/GameData/NameFile/USA.msgpack b/GameData/NameFile/USA.msgpack deleted file mode 100644 index f7585adc6..000000000 --- a/GameData/NameFile/USA.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:085dbd6a8c2cf491d1344ce6de0ac024f5fbeb5f838c06ccbeb1506674a9b4d0 -size 954959 diff --git a/GameData/NameFile/pleiades.msgpack b/GameData/NameFile/pleiades.msgpack deleted file mode 100644 index c9cb7a85f..000000000 --- a/GameData/NameFile/pleiades.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7f489a299b1a4d663a6b9be4790ff459ead7b36806783b759e6e24c0a26175d1 -size 400978 diff --git a/tests/Aetheria.Shared.Tests/AetheriaStoresTests.cs b/tests/Aetheria.Shared.Tests/AetheriaStoresTests.cs index 28c6b2ce7..023d04877 100644 --- a/tests/Aetheria.Shared.Tests/AetheriaStoresTests.cs +++ b/tests/Aetheria.Shared.Tests/AetheriaStoresTests.cs @@ -30,6 +30,7 @@ public sealed class AetheriaStoresTests : IDisposable faction.Allegiance[cache.Upsert(faction)] = 1; cache.Upsert(faction); cache.Upsert(new WeaponItemData { Name = "Lance", Manufacturer = cache.RefOf(faction) }); + cache.Upsert(new InputLayout { Rows = new InputLayoutRow[] { new InputLayoutRowSpacer { Height = 1 } } }); cache.FlushAsync().Wait(); } @@ -43,7 +44,6 @@ public sealed class AetheriaStoresTests : IDisposable var zone = cache.Upsert(new SavedZone { Name = "Entrance", Position = new float2(1, 2), Contents = new ZonePack() }); cache.Upsert(new SavedGame { Zones = new[] { zone }, Factions = new[] { cache.RefOf(cache.GetAll().Single()) } }); cache.Upsert(new PlayerSettings()); - cache.Upsert(new InputLayout { Rows = new InputLayoutRow[] { new InputLayoutRowSpacer { Height = 1 } } }); cache.FlushAsync().Wait(); } diff --git a/tools/AetherDb/Import.cs b/tools/AetherDb/Import.cs new file mode 100644 index 000000000..3e9142da3 --- /dev/null +++ b/tools/AetherDb/Import.cs @@ -0,0 +1,245 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using MessagePack; + +// One-shot import of the legacy GameData msgpack files into GameData/Aetheria.cc. Cut 10 deletes it. +public static class Import +{ + // The legacy DatabaseEntry union's catalog tags. Any other tag is not catalog state and throws. + private static readonly (int Tag, Type Type)[] Tags = + { + (0, typeof(SimpleCommodityData)), (1, typeof(CompoundCommodityData)), (2, typeof(GearData)), (3, typeof(HullData)), + (9, typeof(NameFile)), (13, typeof(Faction)), (17, typeof(PersonalityAttribute)), (29, typeof(CargoBayData)), + (30, typeof(DockingBayData)), (31, typeof(WeaponItemData)), (32, typeof(FactionProductData)) + }; + + private static string GameData => Path.Combine(AetherDb.FindRoot(), "GameData"); + private static string[] NameFiles => Directory.GetFiles(Path.Combine(GameData, "NameFile"), "*.msgpack"); + private static string[] Layouts => Directory.GetFiles(Path.Combine(GameData, "KeyboardLayouts"), "*.msgpack"); + + public static int LegacyCensus() + { + var records = Read(Path.Combine(GameData, "AetherDB.msgpack"), true); + Console.WriteLine("tags " + string.Join(", ", records.GroupBy(r => r.Tag).OrderBy(g => g.Key).Select(g => $"{g.Key}:{g.Count()}"))); + Console.WriteLine($"{NameFiles.Length} name files, {Layouts.Length} layouts"); + return 0; + } + + public static int Run() + { + var path = Path.Combine(GameData, "Aetheria.cc"); + if (File.Exists(path)) + { + Console.WriteLine($"{path} exists. The import is one-shot and never merges."); + return 1; + } + + // Every Guid-slotted type, which is every tagged one; InputLayout is imported bare and keeps its Key(0). + var keyed = Tags.Select(t => t.Type).FirstOrDefault(type => Members(type).ContainsKey(0)); + if (keyed != null) throw new InvalidOperationException($"{keyed.Name} declares [Key(0)], the slot the legacy Guid occupies."); + + var records = Read(Path.Combine(GameData, "AetherDB.msgpack"), true).Concat(NameFiles.SelectMany(file => Read(file, false))).ToArray(); + var context = new Context(); + using (var cache = AetheriaStores.Open(path, catalogWritable: true)) + { + var documents = new List<(Type Type, object Document, CultRecordKey Key)>(); + for (var index = 0; index < records.Length; index++) + { + var record = records[index]; + context.Key = null; + try + { + var type = Tags.FirstOrDefault(t => t.Tag == record.Tag).Type ?? throw new InvalidOperationException("not a catalog tag"); + var buffer = new ArrayBufferWriter(); + var writer = new MessagePackWriter(buffer); + var reader = new MessagePackReader(record.Payload); + Rewrite(ref reader, ref writer, type, context, document: true); + writer.Flush(); + documents.Add((type, CultDocumentMessagePackSerialization.DeserializeUntyped(type, buffer.WrittenSpan.ToArray(), cache.Registry), new CultRecordKey(context.Key))); + } + catch (Exception e) + { + throw new InvalidOperationException($"tag {record.Tag}, record {index} ({record.Source}), legacy key {context.Key ?? "unread"}: {e.Message}", e); + } + } + + foreach (var layout in Layouts) + documents.Add((typeof(InputLayout), CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(InputLayout), File.ReadAllBytes(layout), cache.Registry), + new CultRecordKey(Path.GetFileNameWithoutExtension(layout)))); + + if (!cache.Commit(batch => + { + foreach (var (type, document, key) in documents) batch.Upsert(type, document, key); + })) + throw new InvalidOperationException("The catalog commit was refused."); + } + + // Reopened read-only, so the catalog-global check runs against what landed. + using (var cache = AetheriaStores.Open(path)) + { + var order = Tags.Select(t => t.Type).Append(typeof(InputLayout)).ToList(); + var stored = cache.AllStoredDocuments.ToArray(); + Console.WriteLine(string.Join(", ", stored.GroupBy(s => s.Document.GetType()).OrderBy(g => order.IndexOf(g.Key)) + .Select(g => $"{g.Key.Name} {g.Count()}")) + $", {stored.Length} records in all"); + + var dangling = context.Refs.Where(r => !r.Target.IsInstanceOfType(cache.Get(new CultRecordKey(r.Key)))).ToArray(); + Console.WriteLine($"{dangling.Length} refs resolve to nothing:"); + foreach (var r in dangling) + { + var owner = cache.Get(new CultRecordKey(r.Record)); + Console.WriteLine($" {owner?.GetType().GetField("Name")?.GetValue(owner)} {r.Member} -> {r.Key}"); + } + } + + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + Console.WriteLine("schemas " + string.Join(", ", snapshot.Records + .Select(record => snapshot.SchemaCatalog.Single(entry => entry.SchemaId == record.SchemaId).SchemaName).Distinct().OrderBy(name => name))); + return 0; + } + + private sealed class Context + { + public string Key; + public string Member; + public readonly List<(string Record, string Member, Type Target, string Key)> Refs = new List<(string, string, Type, string)>(); + } + + // Legacy records are [tag, payload] pairs: many in AetherDB.msgpack, one per name file. + private static List<(int Tag, byte[] Payload, string Source)> Read(string path, bool many) + { + var records = new List<(int, byte[], string)>(); + var reader = new MessagePackReader(File.ReadAllBytes(path)); + var count = many ? reader.ReadArrayHeader() : 1; + for (var i = 0; i < count; i++) + { + if (reader.ReadArrayHeader() != 2) throw new InvalidOperationException($"{path} record {i} is not [tag, payload]."); + var tag = reader.ReadInt32(); + records.Add((tag, reader.ReadRaw().ToArray(), $"{Path.GetFileName(path)}[{i}]")); + } + return records; + } + + // Copies one legacy value, rewriting Guids into record keys by the target type's shape. + private static void Rewrite(ref MessagePackReader reader, ref MessagePackWriter writer, Type type, Context context, bool document = false) + { + if (reader.TryReadNil()) + { + writer.WriteNil(); + return; + } + + if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(CultRecordRef<>)) + { + var key = ReadGuid(ref reader); + if (key == null) writer.WriteNil(); + else + { + writer.Write(key); + context.Refs.Add((context.Key, context.Member, type.GetGenericArguments()[0], key)); + } + return; + } + + var unions = type.GetCustomAttributes(false).ToArray(); + if (unions.Length > 0) + { + if (reader.ReadArrayHeader() != 2) throw new InvalidOperationException($"{type.Name} is not [tag, payload]."); + var tag = reader.ReadInt32(); + var subtype = unions.FirstOrDefault(u => u.Key == tag)?.SubType ?? throw new InvalidOperationException($"{type.Name} has no union tag {tag}."); + writer.WriteArrayHeader(2); + writer.Write(tag); + Rewrite(ref reader, ref writer, subtype, context); + return; + } + + if (document && (type.GetCustomAttribute(false) == null || reader.NextMessagePackType != MessagePackType.Array)) + throw new InvalidOperationException($"{type.Name} is not an integer-keyed MessagePack object."); + + if (type.GetCustomAttribute(false) != null && reader.NextMessagePackType == MessagePackType.Array) + { + var members = Members(type); + var count = reader.ReadArrayHeader(); + writer.WriteArrayHeader(count); + for (var slot = 0; slot < count; slot++) + { + if (document && slot == 0) + { + context.Key = ReadGuid(ref reader) ?? throw new InvalidOperationException("slot 0 holds no Guid"); + writer.WriteNil(); + } + else if (!members.TryGetValue(slot, out var member)) writer.WriteRaw(reader.ReadRaw()); + else if (member.DeclaringType == typeof(StatModifierData) && member.Name == nameof(StatModifierData.RequireBehavior)) + { + // A legacy assembly-qualified type name; the game compares simple names. + var name = reader.ReadString(); + var head = name?.Split(',')[0]; + writer.Write(head?.Substring(head.LastIndexOf('.') + 1)); + } + else + { + var outer = context.Member; + if (document) context.Member = $"{type.Name}.{member.Name}"; + Rewrite(ref reader, ref writer, member is FieldInfo field ? field.FieldType : ((PropertyInfo)member).PropertyType, context); + context.Member = outer; + } + } + return; + } + + var list = type.IsArray && type.GetArrayRank() == 1 && type != typeof(byte[]) ? type.GetElementType() + : type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>) ? type.GetGenericArguments()[0] : null; + if (list != null && reader.NextMessagePackType == MessagePackType.Array) + { + var count = reader.ReadArrayHeader(); + writer.WriteArrayHeader(count); + for (var i = 0; i < count; i++) Rewrite(ref reader, ref writer, list, context); + return; + } + + if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Dictionary<,>) && reader.NextMessagePackType == MessagePackType.Map) + { + var arguments = type.GetGenericArguments(); + var count = reader.ReadMapHeader(); + writer.WriteMapHeader(count); + for (var i = 0; i < count; i++) + { + Rewrite(ref reader, ref writer, arguments[0], context); + Rewrite(ref reader, ref writer, arguments[1], context); + } + return; + } + + writer.WriteRaw(reader.ReadRaw()); + } + + // A legacy Guid: a 16-byte bin, a DatabaseLink [bin16], or nil. The all-zero Guid is unset, so it becomes nil. + private static string ReadGuid(ref MessagePackReader reader) + { + if (reader.TryReadNil()) return null; + if (reader.NextMessagePackType == MessagePackType.Array) + { + if (reader.ReadArrayHeader() != 1) throw new InvalidOperationException("a legacy DatabaseLink is not [guid]"); + return ReadGuid(ref reader); + } + var bytes = reader.ReadBytes()?.ToArray(); + if (bytes == null || bytes.Length != 16) throw new InvalidOperationException("a legacy Guid is not a 16-byte bin"); + var guid = new Guid(bytes); + return guid == Guid.Empty ? null : guid.ToString("D"); + } + + private static Dictionary Members(Type type) + { + var members = new Dictionary(); + for (var t = type; t != null; t = t.BaseType) + foreach (var member in t.GetMembers(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)) + if ((member is FieldInfo || member is PropertyInfo) && member.GetCustomAttribute()?.IntKey is int slot && !members.ContainsKey(slot)) + members[slot] = member; + return members; + } +} diff --git a/tools/AetherDb/Program.cs b/tools/AetherDb/Program.cs index bd5398d34..c15f35b2a 100644 --- a/tools/AetherDb/Program.cs +++ b/tools/AetherDb/Program.cs @@ -26,8 +26,10 @@ public static class Program case "clear-boss-hulls": return ClearBossHulls(args.Contains("apply")); case "settings": return Settings(); case "settings-dump": return SettingsDump(); + case "legacy-census": return Import.LegacyCensus(); + case "import": return Import.Run(); default: - Console.WriteLine("commands: census, factions, station-fit, hardpoint-fit, loadout [seed], save, settings, settings-dump, clear-boss-hulls [apply]"); + Console.WriteLine("commands: census, factions, station-fit, hardpoint-fit, loadout [seed], save, settings, settings-dump, clear-boss-hulls [apply], legacy-census, import"); return 1; } }