Import the legacy catalog into GameData/Aetheria.cc (Cut 9)

AetherDb gains legacy-census and a one-shot import (tools/AetherDb/Import.cs,
deleted by Cut 10). The import rewrites each legacy [tag, payload] record by
the target type's [Key] shape: slot 0 Guid becomes the record key and nil,
CultRecordRef members and DatabaseLink wrappers become D strings, the all-zero
Guid becomes nil, StatModifierData.RequireBehavior keeps the simple type name.
All 180 records land in one Commit; a second import refuses.

InputLayout moves to the catalog (Q9-1 A), with the test fixture seeding it
while the catalog is writable. GameData/*.cc is LFS, GameData/*.lock ignored.
The legacy msgpack and ansi104.json sources are removed.

Imported: SimpleCommodityData 13, CompoundCommodityData 51, GearData 25,
HullData 3, NameFile 12, Faction 12, PersonalityAttribute 3, CargoBayData 4,
DockingBayData 1, WeaponItemData 18, FactionProductData 37, InputLayout 1.
Six refs dangled already in the legacy data and are preserved as authored.

The Key(0) precondition checks only the Guid-slotted tagged types; InputLayout
is imported bare and keeps its Key(0).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

.gitattributes

@@ -53,6 +53,7 @@
 *.engine filter=lfs diff=lfs merge=lfs -text
 *.sbsar filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
+GameData/*.cc filter=lfs diff=lfs merge=lfs -text
 *.a filter=lfs diff=lfs merge=lfs -text
 *AkSoundEngine filter=lfs diff=lfs merge=lfs -text
 *AkWaapiClient filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -55,6 +55,9 @@ ExportedObj/
 /*.sln
 /*.csproj
 
+# CultCache single-file store locks
+GameData/*.lock
+
 # Per-user editor state
 /[Uu]ser[Ss]ettings/
 

Assets/Scripts/ServerShared/AetheriaStores.cs

@@ -6,9 +6,9 @@ using GameCult.Caching.MessagePack;
 // Aetheria's stores: one home store per type, composed here and nowhere else.
 public static class AetheriaStores
 {
-    public static readonly Type[] CatalogTypes = { typeof(ItemData), typeof(Faction), typeof(FactionProductData), typeof(PersonalityAttribute), typeof(NameFile) };
+    public static readonly Type[] CatalogTypes = { typeof(ItemData), typeof(Faction), typeof(FactionProductData), typeof(PersonalityAttribute), typeof(NameFile), typeof(InputLayout) };
     public static readonly Type[] RunTypes = { typeof(OrbitData), typeof(BodyData), typeof(SavedZone), typeof(SavedGame) };
-    public static readonly Type[] PlayerTypes = { typeof(PlayerSettings), typeof(InputLayout) };
+    public static readonly Type[] PlayerTypes = { typeof(PlayerSettings) };
 
     // Attaches (hydrates) the catalog read-only unless catalogWritable, then the run and player stores when given.
     // A read-only catalog throws when a [CultGlobal] type routed to it has no record. A writable catalog is being

tests/Aetheria.Shared.Tests/AetheriaStoresTests.cs

@@ -30,6 +30,7 @@ public sealed class AetheriaStoresTests : IDisposable
         faction.Allegiance[cache.Upsert(faction)] = 1;
         cache.Upsert(faction);
         cache.Upsert(new WeaponItemData { Name = "Lance", Manufacturer = cache.RefOf(faction) });
+        cache.Upsert(new InputLayout { Rows = new InputLayoutRow[] { new InputLayoutRowSpacer { Height = 1 } } });
         cache.FlushAsync().Wait();
     }
 
@@ -43,7 +44,6 @@ public sealed class AetheriaStoresTests : IDisposable
             var zone = cache.Upsert(new SavedZone { Name = "Entrance", Position = new float2(1, 2), Contents = new ZonePack() });
             cache.Upsert(new SavedGame { Zones = new[] { zone }, Factions = new[] { cache.RefOf(cache.GetAll<Faction>().Single()) } });
             cache.Upsert(new PlayerSettings());
-            cache.Upsert(new InputLayout { Rows = new InputLayoutRow[] { new InputLayoutRowSpacer { Height = 1 } } });
             cache.FlushAsync().Wait();
         }
 

tools/AetherDb/Import.cs

@@ -0,0 +1,245 @@
+using System;
+using System.Buffers;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using GameCult.Caching;
+using GameCult.Caching.MessagePack;
+using MessagePack;
+
+// One-shot import of the legacy GameData msgpack files into GameData/Aetheria.cc. Cut 10 deletes it.
+public static class Import
+{
+    // The legacy DatabaseEntry union's catalog tags. Any other tag is not catalog state and throws.
+    private static readonly (int Tag, Type Type)[] Tags =
+    {
+        (0, typeof(SimpleCommodityData)), (1, typeof(CompoundCommodityData)), (2, typeof(GearData)), (3, typeof(HullData)),
+        (9, typeof(NameFile)), (13, typeof(Faction)), (17, typeof(PersonalityAttribute)), (29, typeof(CargoBayData)),
+        (30, typeof(DockingBayData)), (31, typeof(WeaponItemData)), (32, typeof(FactionProductData))
+    };
+
+    private static string GameData => Path.Combine(AetherDb.FindRoot(), "GameData");
+    private static string[] NameFiles => Directory.GetFiles(Path.Combine(GameData, "NameFile"), "*.msgpack");
+    private static string[] Layouts => Directory.GetFiles(Path.Combine(GameData, "KeyboardLayouts"), "*.msgpack");
+
+    public static int LegacyCensus()
+    {
+        var records = Read(Path.Combine(GameData, "AetherDB.msgpack"), true);
+        Console.WriteLine("tags " + string.Join(", ", records.GroupBy(r => r.Tag).OrderBy(g => g.Key).Select(g => $"{g.Key}:{g.Count()}")));
+        Console.WriteLine($"{NameFiles.Length} name files, {Layouts.Length} layouts");
+        return 0;
+    }
+
+    public static int Run()
+    {
+        var path = Path.Combine(GameData, "Aetheria.cc");
+        if (File.Exists(path))
+        {
+            Console.WriteLine($"{path} exists. The import is one-shot and never merges.");
+            return 1;
+        }
+
+        // Every Guid-slotted type, which is every tagged one; InputLayout is imported bare and keeps its Key(0).
+        var keyed = Tags.Select(t => t.Type).FirstOrDefault(type => Members(type).ContainsKey(0));
+        if (keyed != null) throw new InvalidOperationException($"{keyed.Name} declares [Key(0)], the slot the legacy Guid occupies.");
+
+        var records = Read(Path.Combine(GameData, "AetherDB.msgpack"), true).Concat(NameFiles.SelectMany(file => Read(file, false))).ToArray();
+        var context = new Context();
+        using (var cache = AetheriaStores.Open(path, catalogWritable: true))
+        {
+            var documents = new List<(Type Type, object Document, CultRecordKey Key)>();
+            for (var index = 0; index < records.Length; index++)
+            {
+                var record = records[index];
+                context.Key = null;
+                try
+                {
+                    var type = Tags.FirstOrDefault(t => t.Tag == record.Tag).Type ?? throw new InvalidOperationException("not a catalog tag");
+                    var buffer = new ArrayBufferWriter<byte>();
+                    var writer = new MessagePackWriter(buffer);
+                    var reader = new MessagePackReader(record.Payload);
+                    Rewrite(ref reader, ref writer, type, context, document: true);
+                    writer.Flush();
+                    documents.Add((type, CultDocumentMessagePackSerialization.DeserializeUntyped(type, buffer.WrittenSpan.ToArray(), cache.Registry), new CultRecordKey(context.Key)));
+                }
+                catch (Exception e)
+                {
+                    throw new InvalidOperationException($"tag {record.Tag}, record {index} ({record.Source}), legacy key {context.Key ?? "unread"}: {e.Message}", e);
+                }
+            }
+
+            foreach (var layout in Layouts)
+                documents.Add((typeof(InputLayout), CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(InputLayout), File.ReadAllBytes(layout), cache.Registry),
+                    new CultRecordKey(Path.GetFileNameWithoutExtension(layout))));
+
+            if (!cache.Commit(batch =>
+                {
+                    foreach (var (type, document, key) in documents) batch.Upsert(type, document, key);
+                }))
+                throw new InvalidOperationException("The catalog commit was refused.");
+        }
+
+        // Reopened read-only, so the catalog-global check runs against what landed.
+        using (var cache = AetheriaStores.Open(path))
+        {
+            var order = Tags.Select(t => t.Type).Append(typeof(InputLayout)).ToList();
+            var stored = cache.AllStoredDocuments.ToArray();
+            Console.WriteLine(string.Join(", ", stored.GroupBy(s => s.Document.GetType()).OrderBy(g => order.IndexOf(g.Key))
+                .Select(g => $"{g.Key.Name} {g.Count()}")) + $", {stored.Length} records in all");
+
+            var dangling = context.Refs.Where(r => !r.Target.IsInstanceOfType(cache.Get(new CultRecordKey(r.Key)))).ToArray();
+            Console.WriteLine($"{dangling.Length} refs resolve to nothing:");
+            foreach (var r in dangling)
+            {
+                var owner = cache.Get(new CultRecordKey(r.Record));
+                Console.WriteLine($"  {owner?.GetType().GetField("Name")?.GetValue(owner)} {r.Member} -> {r.Key}");
+            }
+        }
+
+        var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path));
+        Console.WriteLine("schemas " + string.Join(", ", snapshot.Records
+            .Select(record => snapshot.SchemaCatalog.Single(entry => entry.SchemaId == record.SchemaId).SchemaName).Distinct().OrderBy(name => name)));
+        return 0;
+    }
+
+    private sealed class Context
+    {
+        public string Key;
+        public string Member;
+        public readonly List<(string Record, string Member, Type Target, string Key)> Refs = new List<(string, string, Type, string)>();
+    }
+
+    // Legacy records are [tag, payload] pairs: many in AetherDB.msgpack, one per name file.
+    private static List<(int Tag, byte[] Payload, string Source)> Read(string path, bool many)
+    {
+        var records = new List<(int, byte[], string)>();
+        var reader = new MessagePackReader(File.ReadAllBytes(path));
+        var count = many ? reader.ReadArrayHeader() : 1;
+        for (var i = 0; i < count; i++)
+        {
+            if (reader.ReadArrayHeader() != 2) throw new InvalidOperationException($"{path} record {i} is not [tag, payload].");
+            var tag = reader.ReadInt32();
+            records.Add((tag, reader.ReadRaw().ToArray(), $"{Path.GetFileName(path)}[{i}]"));
+        }
+        return records;
+    }
+
+    // Copies one legacy value, rewriting Guids into record keys by the target type's shape.
+    private static void Rewrite(ref MessagePackReader reader, ref MessagePackWriter writer, Type type, Context context, bool document = false)
+    {
+        if (reader.TryReadNil())
+        {
+            writer.WriteNil();
+            return;
+        }
+
+        if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(CultRecordRef<>))
+        {
+            var key = ReadGuid(ref reader);
+            if (key == null) writer.WriteNil();
+            else
+            {
+                writer.Write(key);
+                context.Refs.Add((context.Key, context.Member, type.GetGenericArguments()[0], key));
+            }
+            return;
+        }
+
+        var unions = type.GetCustomAttributes<UnionAttribute>(false).ToArray();
+        if (unions.Length > 0)
+        {
+            if (reader.ReadArrayHeader() != 2) throw new InvalidOperationException($"{type.Name} is not [tag, payload].");
+            var tag = reader.ReadInt32();
+            var subtype = unions.FirstOrDefault(u => u.Key == tag)?.SubType ?? throw new InvalidOperationException($"{type.Name} has no union tag {tag}.");
+            writer.WriteArrayHeader(2);
+            writer.Write(tag);
+            Rewrite(ref reader, ref writer, subtype, context);
+            return;
+        }
+
+        if (document && (type.GetCustomAttribute<MessagePackObjectAttribute>(false) == null || reader.NextMessagePackType != MessagePackType.Array))
+            throw new InvalidOperationException($"{type.Name} is not an integer-keyed MessagePack object.");
+
+        if (type.GetCustomAttribute<MessagePackObjectAttribute>(false) != null && reader.NextMessagePackType == MessagePackType.Array)
+        {
+            var members = Members(type);
+            var count = reader.ReadArrayHeader();
+            writer.WriteArrayHeader(count);
+            for (var slot = 0; slot < count; slot++)
+            {
+                if (document && slot == 0)
+                {
+                    context.Key = ReadGuid(ref reader) ?? throw new InvalidOperationException("slot 0 holds no Guid");
+                    writer.WriteNil();
+                }
+                else if (!members.TryGetValue(slot, out var member)) writer.WriteRaw(reader.ReadRaw());
+                else if (member.DeclaringType == typeof(StatModifierData) && member.Name == nameof(StatModifierData.RequireBehavior))
+                {
+                    // A legacy assembly-qualified type name; the game compares simple names.
+                    var name = reader.ReadString();
+                    var head = name?.Split(',')[0];
+                    writer.Write(head?.Substring(head.LastIndexOf('.') + 1));
+                }
+                else
+                {
+                    var outer = context.Member;
+                    if (document) context.Member = $"{type.Name}.{member.Name}";
+                    Rewrite(ref reader, ref writer, member is FieldInfo field ? field.FieldType : ((PropertyInfo)member).PropertyType, context);
+                    context.Member = outer;
+                }
+            }
+            return;
+        }
+
+        var list = type.IsArray && type.GetArrayRank() == 1 && type != typeof(byte[]) ? type.GetElementType()
+            : type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>) ? type.GetGenericArguments()[0] : null;
+        if (list != null && reader.NextMessagePackType == MessagePackType.Array)
+        {
+            var count = reader.ReadArrayHeader();
+            writer.WriteArrayHeader(count);
+            for (var i = 0; i < count; i++) Rewrite(ref reader, ref writer, list, context);
+            return;
+        }
+
+        if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Dictionary<,>) && reader.NextMessagePackType == MessagePackType.Map)
+        {
+            var arguments = type.GetGenericArguments();
+            var count = reader.ReadMapHeader();
+            writer.WriteMapHeader(count);
+            for (var i = 0; i < count; i++)
+            {
+                Rewrite(ref reader, ref writer, arguments[0], context);
+                Rewrite(ref reader, ref writer, arguments[1], context);
+            }
+            return;
+        }
+
+        writer.WriteRaw(reader.ReadRaw());
+    }
+
+    // A legacy Guid: a 16-byte bin, a DatabaseLink [bin16], or nil. The all-zero Guid is unset, so it becomes nil.
+    private static string ReadGuid(ref MessagePackReader reader)
+    {
+        if (reader.TryReadNil()) return null;
+        if (reader.NextMessagePackType == MessagePackType.Array)
+        {
+            if (reader.ReadArrayHeader() != 1) throw new InvalidOperationException("a legacy DatabaseLink is not [guid]");
+            return ReadGuid(ref reader);
+        }
+        var bytes = reader.ReadBytes()?.ToArray();
+        if (bytes == null || bytes.Length != 16) throw new InvalidOperationException("a legacy Guid is not a 16-byte bin");
+        var guid = new Guid(bytes);
+        return guid == Guid.Empty ? null : guid.ToString("D");
+    }
+
+    private static Dictionary<int, MemberInfo> Members(Type type)
+    {
+        var members = new Dictionary<int, MemberInfo>();
+        for (var t = type; t != null; t = t.BaseType)
+            foreach (var member in t.GetMembers(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly))
+                if ((member is FieldInfo || member is PropertyInfo) && member.GetCustomAttribute<KeyAttribute>()?.IntKey is int slot && !members.ContainsKey(slot))
+                    members[slot] = member;
+        return members;
+    }
+}

tools/AetherDb/Program.cs

@@ -26,8 +26,10 @@ public static class Program
             case "clear-boss-hulls": return ClearBossHulls(args.Contains("apply"));
             case "settings": return Settings();
             case "settings-dump": return SettingsDump();
+            case "legacy-census": return Import.LegacyCensus();
+            case "import": return Import.Run();
             default:
-                Console.WriteLine("commands: census, factions, station-fit, hardpoint-fit, loadout [seed], save, settings, settings-dump, clear-boss-hulls [apply]");
+                Console.WriteLine("commands: census, factions, station-fit, hardpoint-fit, loadout [seed], save, settings, settings-dump, clear-boss-hulls [apply], legacy-census, import");
                 return 1;
         }
     }