Map legacy catalog payloads

This commit is contained in:
Meta Vens 2026-06-13 06:15:55 +01:00
commit ef5ad0dce4
8 changed files with 424 additions and 16 deletions

View file

@ -2,6 +2,8 @@ using System.Security.Cryptography;
using Aetheria.State;
using Aetheria.State.Documents;
using Aetheria.State.Migration;
using GameCult.Caching;
using MessagePack;
var root = args.Length > 0 ? Path.GetFullPath(args[0]) : Directory.GetCurrentDirectory();
var statePath = args.Length > 1
@ -20,6 +22,25 @@ var nameFiles = Directory.Exists(nameFilesRoot)
.Select(path => CaptureFile(root, path))
.ToArray()
: [];
var entries = LegacyCatalogReader.Read(catalogPath);
var nameFileEntries = Directory.Exists(nameFilesRoot)
? Directory.EnumerateFiles(nameFilesRoot, "*.msgpack", SearchOption.TopDirectoryOnly)
.OrderBy(path => path, StringComparer.OrdinalIgnoreCase)
.Select(LegacyCatalogReader.ReadSingle)
.ToArray()
: [];
var itemDefinitions = entries
.Where(entry => entry.ItemDefinition != null)
.Select(entry => entry.ItemDefinition!)
.ToArray();
var corporations = entries
.Where(entry => entry.Corporation != null)
.Select(entry => entry.Corporation!)
.ToArray();
var parsedNameFiles = entries.Concat(nameFileEntries)
.Where(entry => entry.NameFile != null)
.Select(entry => entry.NameFile!)
.ToArray();
await using var node = await AetheriaStateNode.OpenAsync(statePath, "aetheria-legacy-catalog-import");
@ -33,8 +54,8 @@ await node.PutLegacyCatalogQuarantineAsync(new AetheriaLegacyCatalogQuarantine
NameFiles = nameFiles,
Notes =
[
"This document quarantines legacy catalog file facts only. It does not deserialize old DatabaseEntry payloads or grant old MessagePack files runtime state authority.",
"AetherDB.msgpack and NameFile/*.msgpack remain migration inputs until a bounded catalog mapper emits typed Aetheria item/faction/name documents."
"This document quarantines legacy catalog file facts and records the bounded raw payload mapping pass. It does not grant old MessagePack files runtime state authority.",
"AetherDB.msgpack and NameFile/*.msgpack remain migration inputs until Unity runtime reads typed Aetheria catalog documents directly."
]
});
@ -59,20 +80,55 @@ await node.PutMigrationLedgerAsync(new AetheriaMigrationLedger
{
DocumentType = "legacy.name_file",
Count = nameFiles.Length
},
new AetheriaMigrationCount
{
DocumentType = "aetheria.item_definition.v1",
Count = itemDefinitions.Length
},
new AetheriaMigrationCount
{
DocumentType = "aetheria.corporation.v1",
Count = corporations.Length
},
new AetheriaMigrationCount
{
DocumentType = "aetheria.name_file.v1",
Count = parsedNameFiles.Length
}
],
LegacyCatalogEntries = entries.Select(entry => entry.Summary).ToArray(),
Notes =
[
"Legacy catalog quarantine captured file fingerprints into typed CultCache state without reading old DatabaseEntry payloads.",
"Legacy catalog quarantine captured file fingerprints and mapped stable old MessagePack union payload fields into typed CultCache documents.",
"The mapper intentionally reads only stable scalar/catalog fields. Runtime object graphs, behavior payloads, and Unity-specific shapes remain legacy until dedicated typed documents exist.",
$"State path: {statePath}"
]
});
foreach (var item in itemDefinitions)
{
await node.PutItemDefinitionAsync(new CultRecordKey($"legacy:item:{item.LegacyId}"), item);
}
foreach (var corporation in corporations)
{
await node.PutCorporationAsync(new CultRecordKey($"legacy:faction:{corporation.LegacyId}"), corporation);
}
foreach (var nameFile in parsedNameFiles)
{
await node.PutNameFileAsync(new CultRecordKey($"legacy:name-file:{nameFile.LegacyId}"), nameFile);
}
await node.FlushAsync();
Console.WriteLine($"Aetheria legacy catalog quarantine captured: {statePath}");
Console.WriteLine($"Aetheria legacy catalog mapped into typed state: {statePath}");
Console.WriteLine($"Catalog: {catalog.RelativePath} {catalog.Bytes} bytes {catalog.Fingerprint}");
Console.WriteLine($"Name files: {nameFiles.Length}");
Console.WriteLine($"Mapped items: {itemDefinitions.Length}");
Console.WriteLine($"Mapped factions: {corporations.Length}");
Console.WriteLine($"Mapped name files: {parsedNameFiles.Length}");
static AetheriaLegacyCatalogFile CaptureFile(string root, string path)
{
@ -95,3 +151,251 @@ static AetheriaLegacyCatalogFile CaptureFile(string root, string path)
Bytes = stream.Length
};
}
internal static class LegacyCatalogReader
{
private static readonly Dictionary<int, string> UnionNames = new()
{
[0] = "SimpleCommodityData",
[1] = "CompoundCommodityData",
[2] = "GearData",
[3] = "HullData",
[8] = "GalaxyMapLayerData",
[9] = "NameFile",
[11] = "PlayerData",
[13] = "Faction",
[15] = "OrbitData",
[16] = "BodyData",
[17] = "PersonalityAttribute",
[25] = "AsteroidBeltData",
[26] = "GasGiantData",
[27] = "SunData",
[28] = "PlanetData",
[29] = "CargoBayData",
[30] = "DockingBayData",
[31] = "WeaponItemData"
};
private static readonly HashSet<int> ItemUnionKeys = [0, 1, 2, 3, 29, 30, 31];
public static IReadOnlyList<LegacyCatalogEntry> Read(string catalogPath)
{
if (!File.Exists(catalogPath))
{
return [];
}
var bytes = File.ReadAllBytes(catalogPath);
var reader = new MessagePackReader(bytes);
var count = reader.ReadArrayHeader();
var entries = new List<LegacyCatalogEntry>(count);
for (var i = 0; i < count; i++)
{
entries.Add(ReadEntry(ref reader));
}
return entries;
}
public static LegacyCatalogEntry ReadSingle(string path)
{
var bytes = File.ReadAllBytes(path);
var reader = new MessagePackReader(bytes);
return ReadEntry(ref reader);
}
private static LegacyCatalogEntry ReadEntry(ref MessagePackReader reader)
{
var unionHeader = reader.ReadArrayHeader();
if (unionHeader != 2)
{
throw new InvalidDataException($"Unexpected legacy catalog union header length: {unionHeader}.");
}
var unionKey = reader.ReadInt32();
var unionName = UnionNames.TryGetValue(unionKey, out var knownName) ? knownName : $"union:{unionKey}";
var payload = ReadArrayPayload(ref reader);
var legacyId = GetGuid(payload, 0);
var name = GetString(payload, 1);
var description = GetString(payload, 2);
var summary = new AetheriaLegacyCatalogEntrySummary
{
LegacyId = legacyId,
UnionKey = unionKey,
DocumentKind = unionName,
Name = name,
Note = "Mapped from stable legacy MessagePack array fields."
};
return new LegacyCatalogEntry(
summary,
ItemUnionKeys.Contains(unionKey)
? new AetheriaItemDefinition
{
Name = name,
Category = unionName,
LegacyId = legacyId,
Description = description,
Mass = GetDouble(payload, 4),
Volume = GetArrayLength(payload, 5),
Tags = [unionName, "legacy-catalog"]
}
: null,
unionKey == 13
? new AetheriaCorporation
{
Name = name,
LegacyId = legacyId,
Description = description
}
: null,
unionKey == 9
? new AetheriaNameFile
{
Name = name,
LegacyId = legacyId,
NameCount = GetArrayLength(payload, 2),
SampleNames = GetStringArraySample(payload, 2, 8)
}
: null);
}
private static Dictionary<int, object?> ReadArrayPayload(ref MessagePackReader reader)
{
var length = reader.ReadArrayHeader();
var values = new Dictionary<int, object?>(length);
for (var index = 0; index < length; index++)
{
values[index] = ReadValue(ref reader);
}
return values;
}
private static object? ReadValue(ref MessagePackReader reader)
{
if (reader.NextMessagePackType == MessagePackType.Nil)
{
reader.ReadNil();
return null;
}
return reader.NextMessagePackType switch
{
MessagePackType.Integer => ReadInteger(ref reader),
MessagePackType.Boolean => reader.ReadBoolean(),
MessagePackType.Float => reader.ReadDouble(),
MessagePackType.String => reader.ReadString(),
MessagePackType.Binary => ReadBinary(ref reader),
MessagePackType.Array => ReadNestedArray(ref reader),
MessagePackType.Map => ReadNestedMap(ref reader),
_ => SkipValue(ref reader)
};
}
private static long ReadInteger(ref MessagePackReader reader)
{
return reader.NextCode >= MessagePackCode.MinNegativeFixInt || reader.NextCode == MessagePackCode.Int8 ||
reader.NextCode == MessagePackCode.Int16 || reader.NextCode == MessagePackCode.Int32 ||
reader.NextCode == MessagePackCode.Int64
? reader.ReadInt64()
: (long) reader.ReadUInt64();
}
private static object?[] ReadNestedArray(ref MessagePackReader reader)
{
var length = reader.ReadArrayHeader();
var values = new object?[length];
for (var index = 0; index < length; index++)
{
values[index] = ReadValue(ref reader);
}
return values;
}
private static byte[]? ReadBinary(ref MessagePackReader reader)
{
var bytes = reader.ReadBytes();
if (bytes == null)
{
return null;
}
var result = new byte[bytes.Value.Length];
var offset = 0;
foreach (var segment in bytes.Value)
{
segment.Span.CopyTo(result.AsSpan(offset));
offset += segment.Length;
}
return result;
}
private static Dictionary<string, object?> ReadNestedMap(ref MessagePackReader reader)
{
var length = reader.ReadMapHeader();
var values = new Dictionary<string, object?>(length);
for (var index = 0; index < length; index++)
{
var key = ReadValue(ref reader);
values[key?.ToString() ?? ""] = ReadValue(ref reader);
}
return values;
}
private static object? SkipValue(ref MessagePackReader reader)
{
reader.Skip();
return null;
}
private static string GetGuid(IReadOnlyDictionary<int, object?> payload, int key)
{
return payload.TryGetValue(key, out var value) && value is byte[] bytes && bytes.Length == 16
? new Guid(bytes).ToString("D")
: "";
}
private static string GetString(IReadOnlyDictionary<int, object?> payload, int key)
{
return payload.TryGetValue(key, out var value) ? value as string ?? "" : "";
}
private static double GetDouble(IReadOnlyDictionary<int, object?> payload, int key)
{
if (!payload.TryGetValue(key, out var value) || value == null)
{
return 0;
}
return value switch
{
double doubleValue => doubleValue,
float floatValue => floatValue,
long integerValue => integerValue,
int integerValue => integerValue,
_ => 0
};
}
private static int GetArrayLength(IReadOnlyDictionary<int, object?> payload, int key)
{
return payload.TryGetValue(key, out var value) && value is object?[] array ? array.Length : 0;
}
private static string[] GetStringArraySample(IReadOnlyDictionary<int, object?> payload, int key, int max)
{
return payload.TryGetValue(key, out var value) && value is object?[] array
? array.OfType<string>().Take(max).ToArray()
: [];
}
}
internal sealed record LegacyCatalogEntry(
AetheriaLegacyCatalogEntrySummary Summary,
AetheriaItemDefinition? ItemDefinition,
AetheriaCorporation? Corporation,
AetheriaNameFile? NameFile);

View file

@ -7,6 +7,8 @@ var root = args.Length > 0 ? args[0] : Directory.GetCurrentDirectory();
var statePath = AetheriaStatePaths.ResolveDefaultStatePath(root);
var now = DateTimeOffset.UtcNow.ToString("O");
var itemKey = new CultRecordKey("item:smoke-aether-drive");
var factionKey = new CultRecordKey("faction:smoke");
var nameFileKey = new CultRecordKey("name-file:smoke");
var runKey = new CultRecordKey("run:smoke");
await using (var node = await AetheriaStateNode.OpenAsync(statePath, "aetheria-state-smoke"))
@ -68,6 +70,21 @@ await using (var node = await AetheriaStateNode.OpenAsync(statePath, "aetheria-s
Notes = ["Smoke proves legacy catalog quarantine state is typed and durable."]
});
await node.PutCorporationAsync(factionKey, new AetheriaCorporation
{
Name = "Smoke Faction",
LegacyId = "smoke:faction",
Description = "Typed faction/corporation document for legacy catalog migration smoke."
});
await node.PutNameFileAsync(nameFileKey, new AetheriaNameFile
{
Name = "Smoke Names",
LegacyId = "smoke:name-file",
NameCount = 2,
SampleNames = ["Ada", "Grace"]
});
await node.PutSavedRunAsync(runKey, new AetheriaSavedRun
{
RunId = "smoke",
@ -101,6 +118,8 @@ await using (var reopened = await AetheriaStateNode.OpenAsync(statePath, "aether
{
var world = await reopened.GetWorldAsync();
var item = await reopened.GetItemDefinitionAsync(itemKey);
var faction = await reopened.GetCorporationAsync(factionKey);
var nameFile = await reopened.GetNameFileAsync(nameFileKey);
var quarantine = await reopened.GetLegacyCatalogQuarantineAsync();
var playerSettings = await reopened.GetPlayerSettingsAsync();
var savedRun = await reopened.GetSavedRunAsync(runKey);
@ -115,6 +134,16 @@ await using (var reopened = await AetheriaStateNode.OpenAsync(statePath, "aether
throw new InvalidOperationException("Item definition did not survive flush/reopen.");
}
if (faction?.LegacyId != "smoke:faction")
{
throw new InvalidOperationException("Faction/corporation document did not survive flush/reopen.");
}
if (nameFile?.NameCount != 2)
{
throw new InvalidOperationException("Name file document did not survive flush/reopen.");
}
if (quarantine?.CatalogFingerprint != "smoke" || quarantine.NameFiles.Length != 1)
{
throw new InvalidOperationException("Legacy catalog quarantine did not survive flush/reopen.");

View file

@ -27,6 +27,7 @@ public static class AetheriaDocumentRegistry
CultNetDocumentBinding.ForDocument<AetheriaWorldState>(registry),
CultNetDocumentBinding.ForDocument<AetheriaItemDefinition>(registry),
CultNetDocumentBinding.ForDocument<AetheriaCorporation>(registry),
CultNetDocumentBinding.ForDocument<AetheriaNameFile>(registry),
CultNetDocumentBinding.ForDocument<AetheriaPlayerProfile>(registry),
CultNetDocumentBinding.ForDocument<AetheriaRuntimeSession>(registry),
CultNetDocumentBinding.ForDocument<AetheriaMigrationLedger>(registry),
@ -43,6 +44,7 @@ public static class AetheriaDocumentRegistry
typeof(AetheriaWorldState),
typeof(AetheriaItemDefinition),
typeof(AetheriaCorporation),
typeof(AetheriaNameFile),
typeof(AetheriaPlayerProfile),
typeof(AetheriaRuntimeSession),
typeof(AetheriaMigrationLedger),

View file

@ -100,6 +100,26 @@ public sealed class AetheriaStateNode : IAsyncDisposable, IDisposable
return Database.GetAsync<AetheriaItemDefinition>(key);
}
public Task<CultRecordHandle<AetheriaCorporation>> PutCorporationAsync(CultRecordKey key, AetheriaCorporation corporation)
{
return Database.PutAsync(key, corporation);
}
public Task<AetheriaCorporation?> GetCorporationAsync(CultRecordKey key)
{
return Database.GetAsync<AetheriaCorporation>(key);
}
public Task<CultRecordHandle<AetheriaNameFile>> PutNameFileAsync(CultRecordKey key, AetheriaNameFile nameFile)
{
return Database.PutAsync(key, nameFile);
}
public Task<AetheriaNameFile?> GetNameFileAsync(CultRecordKey key)
{
return Database.GetAsync<AetheriaNameFile>(key);
}
public Task<CultRecordHandle<AetheriaPlayerSettings>> PutPlayerSettingsAsync(AetheriaPlayerSettings settings)
{
return Database.PutAsync(new CultRecordKey("global:aetheria.player_settings.v1"), settings);

View file

@ -26,6 +26,9 @@ public sealed class AetheriaMigrationLedger
[Key(5)]
public string[] Notes { get; set; } = [];
[Key(6)]
public AetheriaLegacyCatalogEntrySummary[] LegacyCatalogEntries { get; set; } = [];
}
[MessagePackObject]
@ -37,3 +40,22 @@ public sealed class AetheriaMigrationCount
[Key(1)]
public int Count { get; set; }
}
[MessagePackObject]
public sealed class AetheriaLegacyCatalogEntrySummary
{
[Key(0)]
public string LegacyId { get; set; } = "";
[Key(1)]
public int UnionKey { get; set; }
[Key(2)]
public string DocumentKind { get; set; } = "";
[Key(3)]
public string Name { get; set; } = "";
[Key(4)]
public string Note { get; set; } = "";
}

View file

@ -0,0 +1,23 @@
using GameCult.Caching;
using MessagePack;
namespace Aetheria.State.Documents;
[CultDocument("aetheria.name_file", "aetheria.name_file.v1")]
[MessagePackObject]
public sealed class AetheriaNameFile
{
[Key(0)]
[CultName]
public string Name { get; set; } = "";
[Key(1)]
[CultIndex("legacyId")]
public string LegacyId { get; set; } = "";
[Key(2)]
public int NameCount { get; set; }
[Key(3)]
public string[] SampleNames { get; set; } = [];
}

View file

@ -35,10 +35,13 @@ move.
`Aetheria.State.Import` captures the legacy catalog files into typed quarantine
state: path, size, and SHA-256 fingerprint for `AetherDB.msgpack` plus
`NameFile/*.msgpack`, and a migration ledger update. It intentionally does not
deserialize old `DatabaseEntry` payloads. The next catalog migration step must
map the old union payloads into typed item/faction/name documents without making
`Aetheria.State` depend on Unity's legacy model.
`NameFile/*.msgpack`, and a migration ledger update. It also performs a bounded
raw MessagePack union pass that maps stable catalog fields into typed
item/faction/name-file documents without compiling Unity's legacy domain model
into `Aetheria.State`. The current checked-in catalog maps to 115 item
definitions, 12 factions, and 12 name files. Runtime object graphs, behaviors,
Unity shapes, and simulation state remain legacy until dedicated typed
documents exist.
Current rebuild notes:

View file

@ -34,9 +34,11 @@ local CultCache, and legacy UI paths should be migration-only or deleted.
- `Economy.Server` now starts the modern `Aetheria.State` CultMesh node and no
longer owns RethinkDB/LiteNetLib state.
- `Aetheria.State.Import` writes a typed quarantine manifest and migration
ledger for the legacy catalog files. It records file facts only; it does not
deserialize old `DatabaseEntry` payloads or grant the old files runtime
authority.
ledger for the legacy catalog files. It also raw-decodes stable old
MessagePack union fields into typed item/faction/name-file documents without
compiling the old Unity domain model into `Aetheria.State`. The current
checked-in catalog maps to 115 item definitions, 12 factions, and 12 name
files.
- The old IMGUI DB inspector under `Assets/Scripts/CultCache/Editor/` has been
deleted. `NameTools` can still clean/generate names, but legacy NameFile
`.msgpack` export is disabled.
@ -163,9 +165,12 @@ First Aetheria surfaces to publish:
typed quarantine manifest without granting them runtime authority.
- Done: emit typed CultCache records plus a migration ledger for the catalog
quarantine preflight.
- Remaining: implement a bounded payload mapper that converts old
`DatabaseEntry` union payloads into typed catalog documents without making
`Aetheria.State` depend on Unity's legacy model.
- Done: implement a bounded raw payload mapper that converts stable old
`DatabaseEntry` union fields into typed item/faction/name-file catalog
documents without making `Aetheria.State` depend on Unity's legacy model.
- Remaining: add typed documents/mappers for runtime object graphs,
behaviors, Unity shapes, simulation state, and any catalog fields not
covered by the stable scalar pass.
4. Runtime cutover
- Replace `ActionGameManager` cache bootstrap with the new state runtime.
@ -213,8 +218,8 @@ First Aetheria surfaces to publish:
reopen.
- Quarantine import smoke proves old catalog file facts can be captured into
typed state without old readers remaining on the live runtime path.
- Migration smoke proves old catalog payloads can be read once and converted
without old readers remaining on the live runtime path.
- Migration smoke proves stable old catalog payload fields can be read once and
converted without old readers remaining on the live runtime path.
- Unity play smoke proves runtime UI reads from Eve surfaces and sends commands
through the shared state service.
- UI Toolkit lowering parity compares Aetheria surfaces against the Eve browser