From ef5ad0dce4165e44525c88688dca99185a07c206 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sat, 13 Jun 2026 06:15:55 +0100 Subject: [PATCH] Map legacy catalog payloads --- Aetheria.State.Import/Program.cs | 312 +++++++++++++++++- Aetheria.State.Smoke/Program.cs | 29 ++ Aetheria.State/AetheriaDocumentRegistry.cs | 2 + Aetheria.State/AetheriaStateNode.cs | 20 ++ .../Documents/AetheriaMigrationLedger.cs | 22 ++ Aetheria.State/Documents/AetheriaNameFile.cs | 23 ++ Aetheria.State/README.md | 11 +- docs/aetheria-perfect-machine-map.md | 21 +- 8 files changed, 424 insertions(+), 16 deletions(-) create mode 100644 Aetheria.State/Documents/AetheriaNameFile.cs diff --git a/Aetheria.State.Import/Program.cs b/Aetheria.State.Import/Program.cs index 8f48b7c9f..655207b8a 100644 --- a/Aetheria.State.Import/Program.cs +++ b/Aetheria.State.Import/Program.cs @@ -2,6 +2,8 @@ using System.Security.Cryptography; using Aetheria.State; using Aetheria.State.Documents; using Aetheria.State.Migration; +using GameCult.Caching; +using MessagePack; var root = args.Length > 0 ? Path.GetFullPath(args[0]) : Directory.GetCurrentDirectory(); var statePath = args.Length > 1 @@ -20,6 +22,25 @@ var nameFiles = Directory.Exists(nameFilesRoot) .Select(path => CaptureFile(root, path)) .ToArray() : []; +var entries = LegacyCatalogReader.Read(catalogPath); +var nameFileEntries = Directory.Exists(nameFilesRoot) + ? Directory.EnumerateFiles(nameFilesRoot, "*.msgpack", SearchOption.TopDirectoryOnly) + .OrderBy(path => path, StringComparer.OrdinalIgnoreCase) + .Select(LegacyCatalogReader.ReadSingle) + .ToArray() + : []; +var itemDefinitions = entries + .Where(entry => entry.ItemDefinition != null) + .Select(entry => entry.ItemDefinition!) + .ToArray(); +var corporations = entries + .Where(entry => entry.Corporation != null) + .Select(entry => entry.Corporation!) + .ToArray(); +var parsedNameFiles = entries.Concat(nameFileEntries) + .Where(entry => entry.NameFile != null) + .Select(entry => entry.NameFile!) + .ToArray(); await using var node = await AetheriaStateNode.OpenAsync(statePath, "aetheria-legacy-catalog-import"); @@ -33,8 +54,8 @@ await node.PutLegacyCatalogQuarantineAsync(new AetheriaLegacyCatalogQuarantine NameFiles = nameFiles, Notes = [ - "This document quarantines legacy catalog file facts only. It does not deserialize old DatabaseEntry payloads or grant old MessagePack files runtime state authority.", - "AetherDB.msgpack and NameFile/*.msgpack remain migration inputs until a bounded catalog mapper emits typed Aetheria item/faction/name documents." + "This document quarantines legacy catalog file facts and records the bounded raw payload mapping pass. It does not grant old MessagePack files runtime state authority.", + "AetherDB.msgpack and NameFile/*.msgpack remain migration inputs until Unity runtime reads typed Aetheria catalog documents directly." ] }); @@ -59,20 +80,55 @@ await node.PutMigrationLedgerAsync(new AetheriaMigrationLedger { DocumentType = "legacy.name_file", Count = nameFiles.Length + }, + new AetheriaMigrationCount + { + DocumentType = "aetheria.item_definition.v1", + Count = itemDefinitions.Length + }, + new AetheriaMigrationCount + { + DocumentType = "aetheria.corporation.v1", + Count = corporations.Length + }, + new AetheriaMigrationCount + { + DocumentType = "aetheria.name_file.v1", + Count = parsedNameFiles.Length } ], + LegacyCatalogEntries = entries.Select(entry => entry.Summary).ToArray(), Notes = [ - "Legacy catalog quarantine captured file fingerprints into typed CultCache state without reading old DatabaseEntry payloads.", + "Legacy catalog quarantine captured file fingerprints and mapped stable old MessagePack union payload fields into typed CultCache documents.", + "The mapper intentionally reads only stable scalar/catalog fields. Runtime object graphs, behavior payloads, and Unity-specific shapes remain legacy until dedicated typed documents exist.", $"State path: {statePath}" ] }); +foreach (var item in itemDefinitions) +{ + await node.PutItemDefinitionAsync(new CultRecordKey($"legacy:item:{item.LegacyId}"), item); +} + +foreach (var corporation in corporations) +{ + await node.PutCorporationAsync(new CultRecordKey($"legacy:faction:{corporation.LegacyId}"), corporation); +} + +foreach (var nameFile in parsedNameFiles) +{ + await node.PutNameFileAsync(new CultRecordKey($"legacy:name-file:{nameFile.LegacyId}"), nameFile); +} + await node.FlushAsync(); -Console.WriteLine($"Aetheria legacy catalog quarantine captured: {statePath}"); +Console.WriteLine($"Aetheria legacy catalog mapped into typed state: {statePath}"); Console.WriteLine($"Catalog: {catalog.RelativePath} {catalog.Bytes} bytes {catalog.Fingerprint}"); Console.WriteLine($"Name files: {nameFiles.Length}"); +Console.WriteLine($"Mapped items: {itemDefinitions.Length}"); +Console.WriteLine($"Mapped factions: {corporations.Length}"); +Console.WriteLine($"Mapped name files: {parsedNameFiles.Length}"); static AetheriaLegacyCatalogFile CaptureFile(string root, string path) { @@ -95,3 +151,251 @@ static AetheriaLegacyCatalogFile CaptureFile(string root, string path) Bytes = stream.Length }; } + +internal static class LegacyCatalogReader +{ + private static readonly Dictionary UnionNames = new() + { + [0] = "SimpleCommodityData", + [1] = "CompoundCommodityData", + [2] = "GearData", + [3] = "HullData", + [8] = "GalaxyMapLayerData", + [9] = "NameFile", + [11] = "PlayerData", + [13] = "Faction", + [15] = "OrbitData", + [16] = "BodyData", + [17] = "PersonalityAttribute", + [25] = "AsteroidBeltData", + [26] = "GasGiantData", + [27] = "SunData", + [28] = "PlanetData", + [29] = "CargoBayData", + [30] = "DockingBayData", + [31] = "WeaponItemData" + }; + + private static readonly HashSet ItemUnionKeys = [0, 1, 2, 3, 29, 30, 31]; + + public static IReadOnlyList Read(string catalogPath) + { + if (!File.Exists(catalogPath)) + { + return []; + } + + var bytes = File.ReadAllBytes(catalogPath); + var reader = new MessagePackReader(bytes); + var count = reader.ReadArrayHeader(); + var entries = new List(count); + for (var i = 0; i < count; i++) + { + entries.Add(ReadEntry(ref reader)); + } + + return entries; + } + + public static LegacyCatalogEntry ReadSingle(string path) + { + var bytes = File.ReadAllBytes(path); + var reader = new MessagePackReader(bytes); + return ReadEntry(ref reader); + } + + private static LegacyCatalogEntry ReadEntry(ref MessagePackReader reader) + { + var unionHeader = reader.ReadArrayHeader(); + if (unionHeader != 2) + { + throw new InvalidDataException($"Unexpected legacy catalog union header length: {unionHeader}."); + } + + var unionKey = reader.ReadInt32(); + var unionName = UnionNames.TryGetValue(unionKey, out var knownName) ? knownName : $"union:{unionKey}"; + var payload = ReadArrayPayload(ref reader); + var legacyId = GetGuid(payload, 0); + var name = GetString(payload, 1); + var description = GetString(payload, 2); + var summary = new AetheriaLegacyCatalogEntrySummary + { + LegacyId = legacyId, + UnionKey = unionKey, + DocumentKind = unionName, + Name = name, + Note = "Mapped from stable legacy MessagePack array fields." + }; + + return new LegacyCatalogEntry( + summary, + ItemUnionKeys.Contains(unionKey) + ? new AetheriaItemDefinition + { + Name = name, + Category = unionName, + LegacyId = legacyId, + Description = description, + Mass = GetDouble(payload, 4), + Volume = GetArrayLength(payload, 5), + Tags = [unionName, "legacy-catalog"] + } + : null, + unionKey == 13 + ? new AetheriaCorporation + { + Name = name, + LegacyId = legacyId, + Description = description + } + : null, + unionKey == 9 + ? new AetheriaNameFile + { + Name = name, + LegacyId = legacyId, + NameCount = GetArrayLength(payload, 2), + SampleNames = GetStringArraySample(payload, 2, 8) + } + : null); + } + + private static Dictionary ReadArrayPayload(ref MessagePackReader reader) + { + var length = reader.ReadArrayHeader(); + var values = new Dictionary(length); + for (var index = 0; index < length; index++) + { + values[index] = ReadValue(ref reader); + } + + return values; + } + + private static object? ReadValue(ref MessagePackReader reader) + { + if (reader.NextMessagePackType == MessagePackType.Nil) + { + reader.ReadNil(); + return null; + } + + return reader.NextMessagePackType switch + { + MessagePackType.Integer => ReadInteger(ref reader), + MessagePackType.Boolean => reader.ReadBoolean(), + MessagePackType.Float => reader.ReadDouble(), + MessagePackType.String => reader.ReadString(), + MessagePackType.Binary => ReadBinary(ref reader), + MessagePackType.Array => ReadNestedArray(ref reader), + MessagePackType.Map => ReadNestedMap(ref reader), + _ => SkipValue(ref reader) + }; + } + + private static long ReadInteger(ref MessagePackReader reader) + { + return reader.NextCode >= MessagePackCode.MinNegativeFixInt || reader.NextCode == MessagePackCode.Int8 || + reader.NextCode == MessagePackCode.Int16 || reader.NextCode == MessagePackCode.Int32 || + reader.NextCode == MessagePackCode.Int64 + ? reader.ReadInt64() + : (long) reader.ReadUInt64(); + } + + private static object?[] ReadNestedArray(ref MessagePackReader reader) + { + var length = reader.ReadArrayHeader(); + var values = new object?[length]; + for (var index = 0; index < length; index++) + { + values[index] = ReadValue(ref reader); + } + + return values; + } + + private static byte[]? ReadBinary(ref MessagePackReader reader) + { + var bytes = reader.ReadBytes(); + if (bytes == null) + { + return null; + } + + var result = new byte[bytes.Value.Length]; + var offset = 0; + foreach (var segment in bytes.Value) + { + segment.Span.CopyTo(result.AsSpan(offset)); + offset += segment.Length; + } + + return result; + } + + private static Dictionary ReadNestedMap(ref MessagePackReader reader) + { + var length = reader.ReadMapHeader(); + var values = new Dictionary(length); + for (var index = 0; index < length; index++) + { + var key = ReadValue(ref reader); + values[key?.ToString() ?? ""] = ReadValue(ref reader); + } + + return values; + } + + private static object? SkipValue(ref MessagePackReader reader) + { + reader.Skip(); + return null; + } + + private static string GetGuid(IReadOnlyDictionary payload, int key) + { + return payload.TryGetValue(key, out var value) && value is byte[] bytes && bytes.Length == 16 + ? new Guid(bytes).ToString("D") + : ""; + } + + private static string GetString(IReadOnlyDictionary payload, int key) + { + return payload.TryGetValue(key, out var value) ? value as string ?? "" : ""; + } + + private static double GetDouble(IReadOnlyDictionary payload, int key) + { + if (!payload.TryGetValue(key, out var value) || value == null) + { + return 0; + } + + return value switch + { + double doubleValue => doubleValue, + float floatValue => floatValue, + long integerValue => integerValue, + int integerValue => integerValue, + _ => 0 + }; + } + + private static int GetArrayLength(IReadOnlyDictionary payload, int key) + { + return payload.TryGetValue(key, out var value) && value is object?[] array ? array.Length : 0; + } + + private static string[] GetStringArraySample(IReadOnlyDictionary payload, int key, int max) + { + return payload.TryGetValue(key, out var value) && value is object?[] array + ? array.OfType().Take(max).ToArray() + : []; + } +} + +internal sealed record LegacyCatalogEntry( + AetheriaLegacyCatalogEntrySummary Summary, + AetheriaItemDefinition? ItemDefinition, + AetheriaCorporation? Corporation, + AetheriaNameFile? NameFile); diff --git a/Aetheria.State.Smoke/Program.cs b/Aetheria.State.Smoke/Program.cs index ec30b06d8..9477c29da 100644 --- a/Aetheria.State.Smoke/Program.cs +++ b/Aetheria.State.Smoke/Program.cs @@ -7,6 +7,8 @@ var root = args.Length > 0 ? args[0] : Directory.GetCurrentDirectory(); var statePath = AetheriaStatePaths.ResolveDefaultStatePath(root); var now = DateTimeOffset.UtcNow.ToString("O"); var itemKey = new CultRecordKey("item:smoke-aether-drive"); +var factionKey = new CultRecordKey("faction:smoke"); +var nameFileKey = new CultRecordKey("name-file:smoke"); var runKey = new CultRecordKey("run:smoke"); await using (var node = await AetheriaStateNode.OpenAsync(statePath, "aetheria-state-smoke")) @@ -68,6 +70,21 @@ await using (var node = await AetheriaStateNode.OpenAsync(statePath, "aetheria-s Notes = ["Smoke proves legacy catalog quarantine state is typed and durable."] }); + await node.PutCorporationAsync(factionKey, new AetheriaCorporation + { + Name = "Smoke Faction", + LegacyId = "smoke:faction", + Description = "Typed faction/corporation document for legacy catalog migration smoke." + }); + + await node.PutNameFileAsync(nameFileKey, new AetheriaNameFile + { + Name = "Smoke Names", + LegacyId = "smoke:name-file", + NameCount = 2, + SampleNames = ["Ada", "Grace"] + }); + await node.PutSavedRunAsync(runKey, new AetheriaSavedRun { RunId = "smoke", @@ -101,6 +118,8 @@ await using (var reopened = await AetheriaStateNode.OpenAsync(statePath, "aether { var world = await reopened.GetWorldAsync(); var item = await reopened.GetItemDefinitionAsync(itemKey); + var faction = await reopened.GetCorporationAsync(factionKey); + var nameFile = await reopened.GetNameFileAsync(nameFileKey); var quarantine = await reopened.GetLegacyCatalogQuarantineAsync(); var playerSettings = await reopened.GetPlayerSettingsAsync(); var savedRun = await reopened.GetSavedRunAsync(runKey); @@ -115,6 +134,16 @@ await using (var reopened = await AetheriaStateNode.OpenAsync(statePath, "aether throw new InvalidOperationException("Item definition did not survive flush/reopen."); } + if (faction?.LegacyId != "smoke:faction") + { + throw new InvalidOperationException("Faction/corporation document did not survive flush/reopen."); + } + + if (nameFile?.NameCount != 2) + { + throw new InvalidOperationException("Name file document did not survive flush/reopen."); + } + if (quarantine?.CatalogFingerprint != "smoke" || quarantine.NameFiles.Length != 1) { throw new InvalidOperationException("Legacy catalog quarantine did not survive flush/reopen."); diff --git a/Aetheria.State/AetheriaDocumentRegistry.cs b/Aetheria.State/AetheriaDocumentRegistry.cs index 79d640b47..714de18af 100644 --- a/Aetheria.State/AetheriaDocumentRegistry.cs +++ b/Aetheria.State/AetheriaDocumentRegistry.cs @@ -27,6 +27,7 @@ public static class AetheriaDocumentRegistry CultNetDocumentBinding.ForDocument(registry), CultNetDocumentBinding.ForDocument(registry), CultNetDocumentBinding.ForDocument(registry), + CultNetDocumentBinding.ForDocument(registry), CultNetDocumentBinding.ForDocument(registry), CultNetDocumentBinding.ForDocument(registry), CultNetDocumentBinding.ForDocument(registry), @@ -43,6 +44,7 @@ public static class AetheriaDocumentRegistry typeof(AetheriaWorldState), typeof(AetheriaItemDefinition), typeof(AetheriaCorporation), + typeof(AetheriaNameFile), typeof(AetheriaPlayerProfile), typeof(AetheriaRuntimeSession), typeof(AetheriaMigrationLedger), diff --git a/Aetheria.State/AetheriaStateNode.cs b/Aetheria.State/AetheriaStateNode.cs index 3aba40243..2f66e982f 100644 --- a/Aetheria.State/AetheriaStateNode.cs +++ b/Aetheria.State/AetheriaStateNode.cs @@ -100,6 +100,26 @@ public sealed class AetheriaStateNode : IAsyncDisposable, IDisposable return Database.GetAsync(key); } + public Task> PutCorporationAsync(CultRecordKey key, AetheriaCorporation corporation) + { + return Database.PutAsync(key, corporation); + } + + public Task GetCorporationAsync(CultRecordKey key) + { + return Database.GetAsync(key); + } + + public Task> PutNameFileAsync(CultRecordKey key, AetheriaNameFile nameFile) + { + return Database.PutAsync(key, nameFile); + } + + public Task GetNameFileAsync(CultRecordKey key) + { + return Database.GetAsync(key); + } + public Task> PutPlayerSettingsAsync(AetheriaPlayerSettings settings) { return Database.PutAsync(new CultRecordKey("global:aetheria.player_settings.v1"), settings); diff --git a/Aetheria.State/Documents/AetheriaMigrationLedger.cs b/Aetheria.State/Documents/AetheriaMigrationLedger.cs index 9fa1a5973..bfda94079 100644 --- a/Aetheria.State/Documents/AetheriaMigrationLedger.cs +++ b/Aetheria.State/Documents/AetheriaMigrationLedger.cs @@ -26,6 +26,9 @@ public sealed class AetheriaMigrationLedger [Key(5)] public string[] Notes { get; set; } = []; + + [Key(6)] + public AetheriaLegacyCatalogEntrySummary[] LegacyCatalogEntries { get; set; } = []; } [MessagePackObject] @@ -37,3 +40,22 @@ public sealed class AetheriaMigrationCount [Key(1)] public int Count { get; set; } } + +[MessagePackObject] +public sealed class AetheriaLegacyCatalogEntrySummary +{ + [Key(0)] + public string LegacyId { get; set; } = ""; + + [Key(1)] + public int UnionKey { get; set; } + + [Key(2)] + public string DocumentKind { get; set; } = ""; + + [Key(3)] + public string Name { get; set; } = ""; + + [Key(4)] + public string Note { get; set; } = ""; +} diff --git a/Aetheria.State/Documents/AetheriaNameFile.cs b/Aetheria.State/Documents/AetheriaNameFile.cs new file mode 100644 index 000000000..6ef605841 --- /dev/null +++ b/Aetheria.State/Documents/AetheriaNameFile.cs @@ -0,0 +1,23 @@ +using GameCult.Caching; +using MessagePack; + +namespace Aetheria.State.Documents; + +[CultDocument("aetheria.name_file", "aetheria.name_file.v1")] +[MessagePackObject] +public sealed class AetheriaNameFile +{ + [Key(0)] + [CultName] + public string Name { get; set; } = ""; + + [Key(1)] + [CultIndex("legacyId")] + public string LegacyId { get; set; } = ""; + + [Key(2)] + public int NameCount { get; set; } + + [Key(3)] + public string[] SampleNames { get; set; } = []; +} diff --git a/Aetheria.State/README.md b/Aetheria.State/README.md index 0bdd7d74c..4b98d546f 100644 --- a/Aetheria.State/README.md +++ b/Aetheria.State/README.md @@ -35,10 +35,13 @@ move. `Aetheria.State.Import` captures the legacy catalog files into typed quarantine state: path, size, and SHA-256 fingerprint for `AetherDB.msgpack` plus -`NameFile/*.msgpack`, and a migration ledger update. It intentionally does not -deserialize old `DatabaseEntry` payloads. The next catalog migration step must -map the old union payloads into typed item/faction/name documents without making -`Aetheria.State` depend on Unity's legacy model. +`NameFile/*.msgpack`, and a migration ledger update. It also performs a bounded +raw MessagePack union pass that maps stable catalog fields into typed +item/faction/name-file documents without compiling Unity's legacy domain model +into `Aetheria.State`. The current checked-in catalog maps to 115 item +definitions, 12 factions, and 12 name files. Runtime object graphs, behaviors, +Unity shapes, and simulation state remain legacy until dedicated typed +documents exist. Current rebuild notes: diff --git a/docs/aetheria-perfect-machine-map.md b/docs/aetheria-perfect-machine-map.md index 2b9b806f9..37a289dbd 100644 --- a/docs/aetheria-perfect-machine-map.md +++ b/docs/aetheria-perfect-machine-map.md @@ -34,9 +34,11 @@ local CultCache, and legacy UI paths should be migration-only or deleted. - `Economy.Server` now starts the modern `Aetheria.State` CultMesh node and no longer owns RethinkDB/LiteNetLib state. - `Aetheria.State.Import` writes a typed quarantine manifest and migration - ledger for the legacy catalog files. It records file facts only; it does not - deserialize old `DatabaseEntry` payloads or grant the old files runtime - authority. + ledger for the legacy catalog files. It also raw-decodes stable old + MessagePack union fields into typed item/faction/name-file documents without + compiling the old Unity domain model into `Aetheria.State`. The current + checked-in catalog maps to 115 item definitions, 12 factions, and 12 name + files. - The old IMGUI DB inspector under `Assets/Scripts/CultCache/Editor/` has been deleted. `NameTools` can still clean/generate names, but legacy NameFile `.msgpack` export is disabled. @@ -163,9 +165,12 @@ First Aetheria surfaces to publish: typed quarantine manifest without granting them runtime authority. - Done: emit typed CultCache records plus a migration ledger for the catalog quarantine preflight. - - Remaining: implement a bounded payload mapper that converts old - `DatabaseEntry` union payloads into typed catalog documents without making - `Aetheria.State` depend on Unity's legacy model. + - Done: implement a bounded raw payload mapper that converts stable old + `DatabaseEntry` union fields into typed item/faction/name-file catalog + documents without making `Aetheria.State` depend on Unity's legacy model. + - Remaining: add typed documents/mappers for runtime object graphs, + behaviors, Unity shapes, simulation state, and any catalog fields not + covered by the stable scalar pass. 4. Runtime cutover - Replace `ActionGameManager` cache bootstrap with the new state runtime. @@ -213,8 +218,8 @@ First Aetheria surfaces to publish: reopen. - Quarantine import smoke proves old catalog file facts can be captured into typed state without old readers remaining on the live runtime path. -- Migration smoke proves old catalog payloads can be read once and converted - without old readers remaining on the live runtime path. +- Migration smoke proves stable old catalog payload fields can be read once and + converted without old readers remaining on the live runtime path. - Unity play smoke proves runtime UI reads from Eve surfaces and sends commands through the shared state service. - UI Toolkit lowering parity compares Aetheria surfaces against the Eve browser