From 70fbaca1ed29bebc0d76e50d986113674e90bc15 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 14 Sep 2026 21:11:06 +0200 Subject: [PATCH] Import the legacy catalog into GameData/Aetheria.cc (Cut 9) AetherDb gains legacy-census and a one-shot import (tools/AetherDb/Import.cs, deleted by Cut 10). The import rewrites each legacy [tag, payload] record by the target type's [Key] shape: slot 0 Guid becomes the record key and nil, CultRecordRef members and DatabaseLink wrappers become D strings, the all-zero Guid becomes nil, StatModifierData.RequireBehavior keeps the simple type name. All 180 records land in one Commit; a second import refuses. InputLayout moves to the catalog (Q9-1 A), with the test fixture seeding it while the catalog is writable. GameData/*.cc is LFS, GameData/*.lock ignored. The legacy msgpack and ansi104.json sources are removed. Imported: SimpleCommodityData 13, CompoundCommodityData 51, GearData 25, HullData 3, NameFile 12, Faction 12, PersonalityAttribute 3, CargoBayData 4, DockingBayData 1, WeaponItemData 18, FactionProductData 37, InputLayout 1. Six refs dangled already in the legacy data and are preserved as authored. The Key(0) precondition checks only the Guid-slotted tagged types; InputLayout is imported bare and keeps its Key(0). Co-Authored-By: Claude Opus 5 --- .gitattributes | 1 + .gitignore | 3 + Assets/Scripts/ServerShared/AetheriaStores.cs | 4 +- GameData/AetherDB.msgpack | 3 - GameData/Aetheria.cc | 3 + GameData/KeyboardLayouts/ansi104.json | 1 - GameData/KeyboardLayouts/ansi104.msgpack | 3 - GameData/NameFile/Australia.msgpack | 3 - GameData/NameFile/Congo.msgpack | 3 - GameData/NameFile/Germany.msgpack | 3 - GameData/NameFile/India.msgpack | 3 - GameData/NameFile/Iran.msgpack | 3 - GameData/NameFile/Japan.msgpack | 3 - GameData/NameFile/Mexico.msgpack | 3 - GameData/NameFile/Russia.msgpack | 3 - GameData/NameFile/Swahili.msgpack | 3 - GameData/NameFile/Thailand.msgpack | 3 - GameData/NameFile/USA.msgpack | 3 - GameData/NameFile/pleiades.msgpack | 3 - .../AetheriaStoresTests.cs | 2 +- tools/AetherDb/Import.cs | 245 ++++++++++++++++++ tools/AetherDb/Program.cs | 4 +- 22 files changed, 258 insertions(+), 47 deletions(-) delete mode 100644 GameData/AetherDB.msgpack create mode 100644 GameData/Aetheria.cc delete mode 100644 GameData/KeyboardLayouts/ansi104.json delete mode 100644 GameData/KeyboardLayouts/ansi104.msgpack delete mode 100644 GameData/NameFile/Australia.msgpack delete mode 100644 GameData/NameFile/Congo.msgpack delete mode 100644 GameData/NameFile/Germany.msgpack delete mode 100644 GameData/NameFile/India.msgpack delete mode 100644 GameData/NameFile/Iran.msgpack delete mode 100644 GameData/NameFile/Japan.msgpack delete mode 100644 GameData/NameFile/Mexico.msgpack delete mode 100644 GameData/NameFile/Russia.msgpack delete mode 100644 GameData/NameFile/Swahili.msgpack delete mode 100644 GameData/NameFile/Thailand.msgpack delete mode 100644 GameData/NameFile/USA.msgpack delete mode 100644 GameData/NameFile/pleiades.msgpack create mode 100644 tools/AetherDb/Import.cs diff --git a/.gitattributes b/.gitattributes index 744578b54..e5c5394e6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -53,6 +53,7 @@ *.engine filter=lfs diff=lfs merge=lfs -text *.sbsar filter=lfs diff=lfs merge=lfs -text *.msgpack filter=lfs diff=lfs merge=lfs -text +GameData/*.cc filter=lfs diff=lfs merge=lfs -text *.a filter=lfs diff=lfs merge=lfs -text *AkSoundEngine filter=lfs diff=lfs merge=lfs -text *AkWaapiClient filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore index f43ec6271..7fb0d0527 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,9 @@ ExportedObj/ /*.sln /*.csproj +# CultCache single-file store locks +GameData/*.lock + # Per-user editor state /[Uu]ser[Ss]ettings/ diff --git a/Assets/Scripts/ServerShared/AetheriaStores.cs b/Assets/Scripts/ServerShared/AetheriaStores.cs index 4394e89d6..924777f41 100644 --- a/Assets/Scripts/ServerShared/AetheriaStores.cs +++ b/Assets/Scripts/ServerShared/AetheriaStores.cs @@ -6,9 +6,9 @@ using GameCult.Caching.MessagePack; // Aetheria's stores: one home store per type, composed here and nowhere else. public static class AetheriaStores { - public static readonly Type[] CatalogTypes = { typeof(ItemData), typeof(Faction), typeof(FactionProductData), typeof(PersonalityAttribute), typeof(NameFile) }; + public static readonly Type[] CatalogTypes = { typeof(ItemData), typeof(Faction), typeof(FactionProductData), typeof(PersonalityAttribute), typeof(NameFile), typeof(InputLayout) }; public static readonly Type[] RunTypes = { typeof(OrbitData), typeof(BodyData), typeof(SavedZone), typeof(SavedGame) }; - public static readonly Type[] PlayerTypes = { typeof(PlayerSettings), typeof(InputLayout) }; + public static readonly Type[] PlayerTypes = { typeof(PlayerSettings) }; // Attaches (hydrates) the catalog read-only unless catalogWritable, then the run and player stores when given. // A read-only catalog throws when a [CultGlobal] type routed to it has no record. A writable catalog is being diff --git a/GameData/AetherDB.msgpack b/GameData/AetherDB.msgpack deleted file mode 100644 index c61b7845e..000000000 --- a/GameData/AetherDB.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8963a68670ae8e8e0ac411c96de7a39322802293b91f53f4d3f50717b06429aa -size 46150 diff --git a/GameData/Aetheria.cc b/GameData/Aetheria.cc new file mode 100644 index 000000000..a8eb4780c --- /dev/null +++ b/GameData/Aetheria.cc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e65d8ea6af07d0a3a037b4e66cbf0d57ff0b0cfbb24804f89c2fa9f6dd2b4a75 +size 12686407 diff --git a/GameData/KeyboardLayouts/ansi104.json b/GameData/KeyboardLayouts/ansi104.json deleted file mode 100644 index 44a2ba7e6..000000000 --- a/GameData/KeyboardLayouts/ansi104.json +++ /dev/null @@ -1 +0,0 @@ -{"rows":[{"columns":[{"mainLabel":"Esc","altLabel":"","path":"escape","width":1.0,"$type":"InputLayoutBindableKey"},{"width":1.0,"$type":"InputLayoutColumnSpacer"},{"mainLabel":"F1","altLabel":"","path":"f1","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"F2","altLabel":"","path":"f2","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"F3","altLabel":"","path":"f3","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"F4","altLabel":"","path":"f4","width":1.0,"$type":"InputLayoutBindableKey"},{"width":0.5,"$type":"InputLayoutColumnSpacer"},{"mainLabel":"F5","altLabel":"","path":"f5","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"F6","altLabel":"","path":"f6","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"F7","altLabel":"","path":"f7","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"F8","altLabel":"","path":"f8","width":1.0,"$type":"InputLayoutBindableKey"},{"width":0.5,"$type":"InputLayoutColumnSpacer"},{"mainLabel":"F9","altLabel":"","path":"f9","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"F10","altLabel":"","path":"f10","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"F11","altLabel":"","path":"f11","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"F12","altLabel":"","path":"f12","width":1.0,"$type":"InputLayoutBindableKey"},{"width":0.25,"$type":"InputLayoutColumnSpacer"},{"width":1.0,"$type":"InputLayoutKey"},{"width":1.0,"$type":"InputLayoutKey"},{"width":1.0,"$type":"InputLayoutKey"}],"$type":"InputLayoutKeyRow"},{"height":0.5,"$type":"InputLayoutRowSpacer"},{"columns":[{"mainLabel":"`","altLabel":"~","path":"backquote","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"1","altLabel":"!","path":"1","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"2","altLabel":"@","path":"2","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"3","altLabel":"#","path":"3","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"4","altLabel":"$","path":"4","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"5","altLabel":"%","path":"5","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"6","altLabel":"^","path":"6","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"7","altLabel":"&","path":"7","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"8","altLabel":"*","path":"8","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"9","altLabel":"(","path":"9","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"0","altLabel":")","path":"0","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"-","altLabel":"_","path":"minus","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"=","altLabel":"+","path":"equals","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"Backspace","altLabel":"","path":"backspace","width":2.0,"$type":"InputLayoutBindableKey"},{"width":0.25,"$type":"InputLayoutColumnSpacer"},{"mainLabel":"Insert","altLabel":"","path":"insert","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"Home","altLabel":"","path":"home","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"PgUp","altLabel":"","path":"pageUp","width":1.0,"$type":"InputLayoutBindableKey"},{"width":0.25,"$type":"InputLayoutColumnSpacer"},{"width":1.0,"$type":"InputLayoutKey"},{"mainLabel":"/","altLabel":"","path":"numpadDivide","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"*","altLabel":"","path":"numpadMultiply","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"-","altLabel":"","path":"numpadMinus","width":1.0,"$type":"InputLayoutBindableKey"}],"$type":"InputLayoutKeyRow"},{"columns":[{"mainLabel":"Tab","altLabel":"","path":"tab","width":1.5,"$type":"InputLayoutBindableKey"},{"mainLabel":"Q","altLabel":"","path":"q","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"W","altLabel":"","path":"w","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"E","altLabel":"","path":"e","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"R","altLabel":"","path":"r","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"T","altLabel":"","path":"t","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"Y","altLabel":"","path":"y","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"U","altLabel":"","path":"u","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"I","altLabel":"","path":"i","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"O","altLabel":"","path":"o","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"P","altLabel":"","path":"p","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"[","altLabel":"{","path":"leftBracket","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"]","altLabel":"}","path":"rightBracket","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"\\","altLabel":"|","path":"backslash","width":1.5,"$type":"InputLayoutBindableKey"},{"width":0.25,"$type":"InputLayoutColumnSpacer"},{"mainLabel":"Delete","altLabel":"","path":"delete","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"End","altLabel":"","path":"end","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"PgDn","altLabel":"","path":"pageDown","width":1.0,"$type":"InputLayoutBindableKey"},{"width":0.25,"$type":"InputLayoutColumnSpacer"},{"mainLabel":"7","altLabel":"Home","path":"numpad7","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"8","altLabel":"↑","path":"numpad8","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"9","altLabel":"PgUp","path":"numpad9","width":1.0,"$type":"InputLayoutBindableKey"},{"height":2,"mainLabel":"+","altLabel":"","path":"numpadPlus","width":1.0,"$type":"InputLayoutMultiRowKey"}],"$type":"InputLayoutKeyRow"},{"columns":[{"mainLabel":"Caps Lock","altLabel":"","path":"capsLock","width":1.75,"$type":"InputLayoutBindableKey"},{"mainLabel":"A","altLabel":"","path":"a","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"S","altLabel":"","path":"s","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"D","altLabel":"","path":"d","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"F","altLabel":"","path":"f","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"G","altLabel":"","path":"g","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"H","altLabel":"","path":"h","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"J","altLabel":"","path":"j","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"K","altLabel":"","path":"k","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"L","altLabel":"","path":"l","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":";","altLabel":":","path":"semicolon","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"'","altLabel":"\"","path":"quote","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"Enter","altLabel":"","path":"enter","width":2.25,"$type":"InputLayoutBindableKey"},{"width":3.5,"$type":"InputLayoutColumnSpacer"},{"mainLabel":"4","altLabel":"←","path":"numpad4","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"5","altLabel":"","path":"numpad5","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"6","altLabel":"→","path":"numpad6","width":1.0,"$type":"InputLayoutBindableKey"}],"$type":"InputLayoutKeyRow"},{"columns":[{"mainLabel":"Shift","altLabel":"","path":"leftShift","width":2.25,"$type":"InputLayoutBindableKey"},{"mainLabel":"Z","altLabel":"","path":"z","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"X","altLabel":"","path":"x","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"C","altLabel":"","path":"c","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"V","altLabel":"","path":"v","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"B","altLabel":"","path":"b","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"N","altLabel":"","path":"n","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"M","altLabel":"","path":"m","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":",","altLabel":"<","path":"comma","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":".","altLabel":">","path":"period","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"/","altLabel":"?","path":"slash","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"Shift","altLabel":"","path":"rightShift","width":2.75,"$type":"InputLayoutBindableKey"},{"width":1.25,"$type":"InputLayoutColumnSpacer"},{"mainLabel":"↑","altLabel":"","path":"upArrow","width":1.0,"$type":"InputLayoutBindableKey"},{"width":1.25,"$type":"InputLayoutColumnSpacer"},{"mainLabel":"1","altLabel":"End","path":"numpad1","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"2","altLabel":"↓","path":"numpad2","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"3","altLabel":"PgDn","path":"numpad3","width":1.0,"$type":"InputLayoutBindableKey"},{"height":2,"mainLabel":"Enter","altLabel":"","path":"numpadEnter","width":1.0,"$type":"InputLayoutMultiRowKey"}],"$type":"InputLayoutKeyRow"},{"columns":[{"mainLabel":"Ctrl","altLabel":"","path":"leftCtrl","width":1.25,"$type":"InputLayoutBindableKey"},{"width":1.25,"$type":"InputLayoutKey"},{"mainLabel":"Alt","altLabel":"","path":"leftAlt","width":1.25,"$type":"InputLayoutBindableKey"},{"mainLabel":"Spacebar","altLabel":"","path":"space","width":6.25,"$type":"InputLayoutBindableKey"},{"mainLabel":"Alt","altLabel":"","path":"rightAlt","width":1.25,"$type":"InputLayoutBindableKey"},{"width":1.25,"$type":"InputLayoutKey"},{"width":1.25,"$type":"InputLayoutKey"},{"mainLabel":"Ctrl","altLabel":"","path":"rightCtrl","width":1.25,"$type":"InputLayoutBindableKey"},{"width":0.25,"$type":"InputLayoutColumnSpacer"},{"mainLabel":"←","altLabel":"","path":"leftArrow","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"↓","altLabel":"","path":"downArrow","width":1.0,"$type":"InputLayoutBindableKey"},{"mainLabel":"→","altLabel":"","path":"rightArrow","width":1.0,"$type":"InputLayoutBindableKey"},{"width":0.25,"$type":"InputLayoutColumnSpacer"},{"mainLabel":"0","altLabel":"Ins","path":"numpad0","width":2.0,"$type":"InputLayoutBindableKey"},{"mainLabel":".","altLabel":"Del","path":"numpadPeriod","width":1.0,"$type":"InputLayoutBindableKey"}],"$type":"InputLayoutKeyRow"}]} \ No newline at end of file diff --git a/GameData/KeyboardLayouts/ansi104.msgpack b/GameData/KeyboardLayouts/ansi104.msgpack deleted file mode 100644 index 839a6dcc1..000000000 --- a/GameData/KeyboardLayouts/ansi104.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:77a2be69d0ba3fc3d7b89183c8cac69137728b7f298547fc2263cfdd69862ce7 -size 1961 diff --git a/GameData/NameFile/Australia.msgpack b/GameData/NameFile/Australia.msgpack deleted file mode 100644 index a898a8e8d..000000000 --- a/GameData/NameFile/Australia.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b65a87570c0bbaecef45b047b5673cc83fbff5354ea0de68ca530cee290d30d6 -size 442830 diff --git a/GameData/NameFile/Congo.msgpack b/GameData/NameFile/Congo.msgpack deleted file mode 100644 index 895ec9238..000000000 --- a/GameData/NameFile/Congo.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:bdaa83a28f9448061b043115bdcbe09448f1eaf46c1cf44a78fe0d9044db9fd0 -size 506753 diff --git a/GameData/NameFile/Germany.msgpack b/GameData/NameFile/Germany.msgpack deleted file mode 100644 index 6241d8117..000000000 --- a/GameData/NameFile/Germany.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1f6fff7465869365a5dbb9aece1e6f2f3c72617808e66a942915d371ca22cd05 -size 1284248 diff --git a/GameData/NameFile/India.msgpack b/GameData/NameFile/India.msgpack deleted file mode 100644 index 9a79b7e17..000000000 --- a/GameData/NameFile/India.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:09cc5aa83fb1d97461846f86a0976dc58fb4730d7e27cc6e4f832f77b755f81a -size 3229371 diff --git a/GameData/NameFile/Iran.msgpack b/GameData/NameFile/Iran.msgpack deleted file mode 100644 index 0d29bb504..000000000 --- a/GameData/NameFile/Iran.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:275965b57ba3e75f1fd7b36339b843992a9a9a1b318d3bf08d940d458e7c7b79 -size 1073151 diff --git a/GameData/NameFile/Japan.msgpack b/GameData/NameFile/Japan.msgpack deleted file mode 100644 index 7ed8c9b37..000000000 --- a/GameData/NameFile/Japan.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f3d7c59066055e76cc2f199ca8d47c6ca14ac20ad4cab66a273b72a0eb63804a -size 1031188 diff --git a/GameData/NameFile/Mexico.msgpack b/GameData/NameFile/Mexico.msgpack deleted file mode 100644 index 19cf62fba..000000000 --- a/GameData/NameFile/Mexico.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f12af0d8bb159f41b6618a8a2f5ac7ddf2c0a2b1dc449dd21188416f2ccc4427 -size 633464 diff --git a/GameData/NameFile/Russia.msgpack b/GameData/NameFile/Russia.msgpack deleted file mode 100644 index ae5a5f51c..000000000 --- a/GameData/NameFile/Russia.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2f3b5155c9003056d1fd33ad4a665d7f9330bed35bb45ab01bbdd05ef0cb006d -size 2375362 diff --git a/GameData/NameFile/Swahili.msgpack b/GameData/NameFile/Swahili.msgpack deleted file mode 100644 index 3d7b45f19..000000000 --- a/GameData/NameFile/Swahili.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5227513492cd33f502506dba741b99d5c7e9140318a44a5d84579258397aa227 -size 501035 diff --git a/GameData/NameFile/Thailand.msgpack b/GameData/NameFile/Thailand.msgpack deleted file mode 100644 index 8c4f33b4b..000000000 --- a/GameData/NameFile/Thailand.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8352d97cd61b83b07dd9ce549358436800c588263147764df6bd5b556124f2cb -size 143118 diff --git a/GameData/NameFile/USA.msgpack b/GameData/NameFile/USA.msgpack deleted file mode 100644 index f7585adc6..000000000 --- a/GameData/NameFile/USA.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:085dbd6a8c2cf491d1344ce6de0ac024f5fbeb5f838c06ccbeb1506674a9b4d0 -size 954959 diff --git a/GameData/NameFile/pleiades.msgpack b/GameData/NameFile/pleiades.msgpack deleted file mode 100644 index c9cb7a85f..000000000 --- a/GameData/NameFile/pleiades.msgpack +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7f489a299b1a4d663a6b9be4790ff459ead7b36806783b759e6e24c0a26175d1 -size 400978 diff --git a/tests/Aetheria.Shared.Tests/AetheriaStoresTests.cs b/tests/Aetheria.Shared.Tests/AetheriaStoresTests.cs index 28c6b2ce7..023d04877 100644 --- a/tests/Aetheria.Shared.Tests/AetheriaStoresTests.cs +++ b/tests/Aetheria.Shared.Tests/AetheriaStoresTests.cs @@ -30,6 +30,7 @@ public sealed class AetheriaStoresTests : IDisposable faction.Allegiance[cache.Upsert(faction)] = 1; cache.Upsert(faction); cache.Upsert(new WeaponItemData { Name = "Lance", Manufacturer = cache.RefOf(faction) }); + cache.Upsert(new InputLayout { Rows = new InputLayoutRow[] { new InputLayoutRowSpacer { Height = 1 } } }); cache.FlushAsync().Wait(); } @@ -43,7 +44,6 @@ public sealed class AetheriaStoresTests : IDisposable var zone = cache.Upsert(new SavedZone { Name = "Entrance", Position = new float2(1, 2), Contents = new ZonePack() }); cache.Upsert(new SavedGame { Zones = new[] { zone }, Factions = new[] { cache.RefOf(cache.GetAll().Single()) } }); cache.Upsert(new PlayerSettings()); - cache.Upsert(new InputLayout { Rows = new InputLayoutRow[] { new InputLayoutRowSpacer { Height = 1 } } }); cache.FlushAsync().Wait(); } diff --git a/tools/AetherDb/Import.cs b/tools/AetherDb/Import.cs new file mode 100644 index 000000000..3e9142da3 --- /dev/null +++ b/tools/AetherDb/Import.cs @@ -0,0 +1,245 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using MessagePack; + +// One-shot import of the legacy GameData msgpack files into GameData/Aetheria.cc. Cut 10 deletes it. +public static class Import +{ + // The legacy DatabaseEntry union's catalog tags. Any other tag is not catalog state and throws. + private static readonly (int Tag, Type Type)[] Tags = + { + (0, typeof(SimpleCommodityData)), (1, typeof(CompoundCommodityData)), (2, typeof(GearData)), (3, typeof(HullData)), + (9, typeof(NameFile)), (13, typeof(Faction)), (17, typeof(PersonalityAttribute)), (29, typeof(CargoBayData)), + (30, typeof(DockingBayData)), (31, typeof(WeaponItemData)), (32, typeof(FactionProductData)) + }; + + private static string GameData => Path.Combine(AetherDb.FindRoot(), "GameData"); + private static string[] NameFiles => Directory.GetFiles(Path.Combine(GameData, "NameFile"), "*.msgpack"); + private static string[] Layouts => Directory.GetFiles(Path.Combine(GameData, "KeyboardLayouts"), "*.msgpack"); + + public static int LegacyCensus() + { + var records = Read(Path.Combine(GameData, "AetherDB.msgpack"), true); + Console.WriteLine("tags " + string.Join(", ", records.GroupBy(r => r.Tag).OrderBy(g => g.Key).Select(g => $"{g.Key}:{g.Count()}"))); + Console.WriteLine($"{NameFiles.Length} name files, {Layouts.Length} layouts"); + return 0; + } + + public static int Run() + { + var path = Path.Combine(GameData, "Aetheria.cc"); + if (File.Exists(path)) + { + Console.WriteLine($"{path} exists. The import is one-shot and never merges."); + return 1; + } + + // Every Guid-slotted type, which is every tagged one; InputLayout is imported bare and keeps its Key(0). + var keyed = Tags.Select(t => t.Type).FirstOrDefault(type => Members(type).ContainsKey(0)); + if (keyed != null) throw new InvalidOperationException($"{keyed.Name} declares [Key(0)], the slot the legacy Guid occupies."); + + var records = Read(Path.Combine(GameData, "AetherDB.msgpack"), true).Concat(NameFiles.SelectMany(file => Read(file, false))).ToArray(); + var context = new Context(); + using (var cache = AetheriaStores.Open(path, catalogWritable: true)) + { + var documents = new List<(Type Type, object Document, CultRecordKey Key)>(); + for (var index = 0; index < records.Length; index++) + { + var record = records[index]; + context.Key = null; + try + { + var type = Tags.FirstOrDefault(t => t.Tag == record.Tag).Type ?? throw new InvalidOperationException("not a catalog tag"); + var buffer = new ArrayBufferWriter(); + var writer = new MessagePackWriter(buffer); + var reader = new MessagePackReader(record.Payload); + Rewrite(ref reader, ref writer, type, context, document: true); + writer.Flush(); + documents.Add((type, CultDocumentMessagePackSerialization.DeserializeUntyped(type, buffer.WrittenSpan.ToArray(), cache.Registry), new CultRecordKey(context.Key))); + } + catch (Exception e) + { + throw new InvalidOperationException($"tag {record.Tag}, record {index} ({record.Source}), legacy key {context.Key ?? "unread"}: {e.Message}", e); + } + } + + foreach (var layout in Layouts) + documents.Add((typeof(InputLayout), CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(InputLayout), File.ReadAllBytes(layout), cache.Registry), + new CultRecordKey(Path.GetFileNameWithoutExtension(layout)))); + + if (!cache.Commit(batch => + { + foreach (var (type, document, key) in documents) batch.Upsert(type, document, key); + })) + throw new InvalidOperationException("The catalog commit was refused."); + } + + // Reopened read-only, so the catalog-global check runs against what landed. + using (var cache = AetheriaStores.Open(path)) + { + var order = Tags.Select(t => t.Type).Append(typeof(InputLayout)).ToList(); + var stored = cache.AllStoredDocuments.ToArray(); + Console.WriteLine(string.Join(", ", stored.GroupBy(s => s.Document.GetType()).OrderBy(g => order.IndexOf(g.Key)) + .Select(g => $"{g.Key.Name} {g.Count()}")) + $", {stored.Length} records in all"); + + var dangling = context.Refs.Where(r => !r.Target.IsInstanceOfType(cache.Get(new CultRecordKey(r.Key)))).ToArray(); + Console.WriteLine($"{dangling.Length} refs resolve to nothing:"); + foreach (var r in dangling) + { + var owner = cache.Get(new CultRecordKey(r.Record)); + Console.WriteLine($" {owner?.GetType().GetField("Name")?.GetValue(owner)} {r.Member} -> {r.Key}"); + } + } + + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + Console.WriteLine("schemas " + string.Join(", ", snapshot.Records + .Select(record => snapshot.SchemaCatalog.Single(entry => entry.SchemaId == record.SchemaId).SchemaName).Distinct().OrderBy(name => name))); + return 0; + } + + private sealed class Context + { + public string Key; + public string Member; + public readonly List<(string Record, string Member, Type Target, string Key)> Refs = new List<(string, string, Type, string)>(); + } + + // Legacy records are [tag, payload] pairs: many in AetherDB.msgpack, one per name file. + private static List<(int Tag, byte[] Payload, string Source)> Read(string path, bool many) + { + var records = new List<(int, byte[], string)>(); + var reader = new MessagePackReader(File.ReadAllBytes(path)); + var count = many ? reader.ReadArrayHeader() : 1; + for (var i = 0; i < count; i++) + { + if (reader.ReadArrayHeader() != 2) throw new InvalidOperationException($"{path} record {i} is not [tag, payload]."); + var tag = reader.ReadInt32(); + records.Add((tag, reader.ReadRaw().ToArray(), $"{Path.GetFileName(path)}[{i}]")); + } + return records; + } + + // Copies one legacy value, rewriting Guids into record keys by the target type's shape. + private static void Rewrite(ref MessagePackReader reader, ref MessagePackWriter writer, Type type, Context context, bool document = false) + { + if (reader.TryReadNil()) + { + writer.WriteNil(); + return; + } + + if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(CultRecordRef<>)) + { + var key = ReadGuid(ref reader); + if (key == null) writer.WriteNil(); + else + { + writer.Write(key); + context.Refs.Add((context.Key, context.Member, type.GetGenericArguments()[0], key)); + } + return; + } + + var unions = type.GetCustomAttributes(false).ToArray(); + if (unions.Length > 0) + { + if (reader.ReadArrayHeader() != 2) throw new InvalidOperationException($"{type.Name} is not [tag, payload]."); + var tag = reader.ReadInt32(); + var subtype = unions.FirstOrDefault(u => u.Key == tag)?.SubType ?? throw new InvalidOperationException($"{type.Name} has no union tag {tag}."); + writer.WriteArrayHeader(2); + writer.Write(tag); + Rewrite(ref reader, ref writer, subtype, context); + return; + } + + if (document && (type.GetCustomAttribute(false) == null || reader.NextMessagePackType != MessagePackType.Array)) + throw new InvalidOperationException($"{type.Name} is not an integer-keyed MessagePack object."); + + if (type.GetCustomAttribute(false) != null && reader.NextMessagePackType == MessagePackType.Array) + { + var members = Members(type); + var count = reader.ReadArrayHeader(); + writer.WriteArrayHeader(count); + for (var slot = 0; slot < count; slot++) + { + if (document && slot == 0) + { + context.Key = ReadGuid(ref reader) ?? throw new InvalidOperationException("slot 0 holds no Guid"); + writer.WriteNil(); + } + else if (!members.TryGetValue(slot, out var member)) writer.WriteRaw(reader.ReadRaw()); + else if (member.DeclaringType == typeof(StatModifierData) && member.Name == nameof(StatModifierData.RequireBehavior)) + { + // A legacy assembly-qualified type name; the game compares simple names. + var name = reader.ReadString(); + var head = name?.Split(',')[0]; + writer.Write(head?.Substring(head.LastIndexOf('.') + 1)); + } + else + { + var outer = context.Member; + if (document) context.Member = $"{type.Name}.{member.Name}"; + Rewrite(ref reader, ref writer, member is FieldInfo field ? field.FieldType : ((PropertyInfo)member).PropertyType, context); + context.Member = outer; + } + } + return; + } + + var list = type.IsArray && type.GetArrayRank() == 1 && type != typeof(byte[]) ? type.GetElementType() + : type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>) ? type.GetGenericArguments()[0] : null; + if (list != null && reader.NextMessagePackType == MessagePackType.Array) + { + var count = reader.ReadArrayHeader(); + writer.WriteArrayHeader(count); + for (var i = 0; i < count; i++) Rewrite(ref reader, ref writer, list, context); + return; + } + + if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Dictionary<,>) && reader.NextMessagePackType == MessagePackType.Map) + { + var arguments = type.GetGenericArguments(); + var count = reader.ReadMapHeader(); + writer.WriteMapHeader(count); + for (var i = 0; i < count; i++) + { + Rewrite(ref reader, ref writer, arguments[0], context); + Rewrite(ref reader, ref writer, arguments[1], context); + } + return; + } + + writer.WriteRaw(reader.ReadRaw()); + } + + // A legacy Guid: a 16-byte bin, a DatabaseLink [bin16], or nil. The all-zero Guid is unset, so it becomes nil. + private static string ReadGuid(ref MessagePackReader reader) + { + if (reader.TryReadNil()) return null; + if (reader.NextMessagePackType == MessagePackType.Array) + { + if (reader.ReadArrayHeader() != 1) throw new InvalidOperationException("a legacy DatabaseLink is not [guid]"); + return ReadGuid(ref reader); + } + var bytes = reader.ReadBytes()?.ToArray(); + if (bytes == null || bytes.Length != 16) throw new InvalidOperationException("a legacy Guid is not a 16-byte bin"); + var guid = new Guid(bytes); + return guid == Guid.Empty ? null : guid.ToString("D"); + } + + private static Dictionary Members(Type type) + { + var members = new Dictionary(); + for (var t = type; t != null; t = t.BaseType) + foreach (var member in t.GetMembers(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly)) + if ((member is FieldInfo || member is PropertyInfo) && member.GetCustomAttribute()?.IntKey is int slot && !members.ContainsKey(slot)) + members[slot] = member; + return members; + } +} diff --git a/tools/AetherDb/Program.cs b/tools/AetherDb/Program.cs index bd5398d34..c15f35b2a 100644 --- a/tools/AetherDb/Program.cs +++ b/tools/AetherDb/Program.cs @@ -26,8 +26,10 @@ public static class Program case "clear-boss-hulls": return ClearBossHulls(args.Contains("apply")); case "settings": return Settings(); case "settings-dump": return SettingsDump(); + case "legacy-census": return Import.LegacyCensus(); + case "import": return Import.Run(); default: - Console.WriteLine("commands: census, factions, station-fit, hardpoint-fit, loadout [seed], save, settings, settings-dump, clear-boss-hulls [apply]"); + Console.WriteLine("commands: census, factions, station-fit, hardpoint-fit, loadout [seed], save, settings, settings-dump, clear-boss-hulls [apply], legacy-census, import"); return 1; } }