CultLib/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs

using System;
using System.IO;
using System.Buffers;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using GameCult.Caching;
using MessagePack;
using MessagePack.Formatters;
using MessagePack.Resolvers;

namespace GameCult.Caching.MessagePack;

public sealed class CultDocumentResolver : IFormatterResolver
{
    public static readonly CultDocumentResolver Instance = new();
    private CultDocumentResolver() { }

    public IMessagePackFormatter<T>? GetFormatter<T>()
    {
        var type = typeof(T);
        if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(CultRecordRef<>))
        {
            var formatterType = typeof(global::GameCult.Caching.CultRecordRefFormatter<>).MakeGenericType(type.GetGenericArguments()[0]);
            return (IMessagePackFormatter<T>)Activator.CreateInstance(formatterType)!;
        }

        return null;
    }
}

public static class CultDocumentMessagePackSerialization
{
    private const int PersistedRecordFieldCount = 4;
    private const int VariantRecordFieldCount = 5;
    private const int SchemaCatalogEntryFieldCount = 7;
    private const int SchemaCatalogMemberFieldCount = 8;
    private const int StoreSnapshotFieldCount = 3;

    private static readonly (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) Plain = Compose(Array.Empty<IFormatterResolver>());

    public static readonly MessagePackSerializerOptions Options = Plain.Options;

    // Per assembly: the options a document type's bytes are written with, and the resolvers composed ahead of
    // StandardResolver, the ones that can claim a type before its [Key] layout does.
    private static readonly ConcurrentDictionary<Assembly, (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead)> AssemblyOptions = new();

    // Per document type: why its element ids cannot reach disk under the composed resolvers, or null. Types and resolvers
    // are fixed for the process, so this is a cache of a pure answer.
    private static readonly ConcurrentDictionary<Type, string?> DocumentProblems = new();

    private static readonly MethodInfo GetFormatterMethod = typeof(IFormatterResolver).GetMethod(nameof(IFormatterResolver.GetFormatter))!;

    public static MessagePackSerializerOptions OptionsFor(Assembly documentAssembly) => ComposedFor(documentAssembly).Options;

    private static (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) ComposedFor(Assembly documentAssembly)
    {
        if (documentAssembly == null) throw new ArgumentNullException(nameof(documentAssembly));
        return AssemblyOptions.GetOrAdd(documentAssembly, static assembly =>
        {
            var resolvers = assembly.GetCustomAttributes<CultCacheFormatterResolverAttribute>()
                .Select(attribute => CreateResolver(attribute.ResolverType))
                .ToArray();
            return resolvers.Length == 0 ? Plain : Compose(resolvers);
        });
    }

    // The options a document's bytes are written and read with. A document's list elements carry their ids by their [Key]
    // layout, so a resolver ahead of StandardResolver that formats a type on the way to those lists would write them
    // without the id: refused here, once per document type, on every encode and decode path.
    internal static MessagePackSerializerOptions OptionsForDocument(Type documentType)
    {
        if (documentType == null) throw new ArgumentNullException(nameof(documentType));
        var (options, ahead) = ComposedFor(documentType.Assembly);
        var problem = DocumentProblems.GetOrAdd(documentType, type => ResolverClaim(type, ahead));
        return problem == null ? options : throw new InvalidOperationException(problem);
    }

    private static string? ResolverClaim(Type documentType, IFormatterResolver[] ahead)
    {
        foreach (var type in CultElementIds.Inspect(documentType).IdPath)
        {
            var generic = GetFormatterMethod.MakeGenericMethod(type);
            // A resolver that hands back MessagePack's own formatter for the type writes its [Key] layout: that is no claim, and the
            // composite stops there, so no later resolver is asked. StandardResolverAllowPrivate's formatter writes the same layout
            // plus non-public [Key] members; those extra members are whole values the walk does not own, so it claims nothing
            // either.
            object? standard;
            try
            {
                standard = generic.Invoke(StandardResolver.Instance, null);
            }
            catch (Exception exception)
            {
                return CannotBuild(documentType, type, (exception.InnerException ?? exception).Message);
            }

            if (standard == null)
                return CannotBuild(documentType, type, "StandardResolver has no formatter for it");

            if (WrittenAsCollection(type, generic, standard))
                return $"Cult document {documentType.Name}: MessagePack writes {type.Name} as a collection, but the element-id walk reads it by its [Key] layout, so element ids would be lost or invented. A [MessagePackObject] type that is also a collection (ICollection<T>, IList, IDictionary) is written as a collection; make it one or the other. Or declare [MessagePackFormatter] on the type or member, which makes it one whole value with no element ids.";

            var allowPrivateAsked = false;
            object? allowPrivate = null;
            foreach (var resolver in ahead)
            {
                object? formatter;
                try
                {
                    formatter = generic.Invoke(resolver, null);
                }
                catch (Exception exception)
                {
                    return $"Cult document {documentType.Name}: formatter resolver {resolver.GetType().FullName} failed to build a formatter for {type.Name}, which is on the path to a list whose elements carry ids ({(exception.InnerException ?? exception).Message}); a resolver may not throw for such a type.";
                }

                if (formatter == null)
                    continue;
                if (ReferenceEquals(formatter, standard))
                    break;
                if (!allowPrivateAsked)
                {
                    allowPrivateAsked = true;
                    try
                    {
                        allowPrivate = generic.Invoke(StandardResolverAllowPrivate.Instance, null);
                    }
                    catch (Exception)
                    {
                        allowPrivate = null;
                    }
                }

                if (allowPrivate != null && ReferenceEquals(formatter, allowPrivate))
                    break;
                return $"Cult document {documentType.Name}: formatter resolver {resolver.GetType().FullName} formats {type.Name}, which is on the path to a list whose elements carry ids, so its element ids would never reach disk. A resolver may not claim such a type; write it by its [Key] layout, keep it out of the document's lists, or declare [MessagePackFormatter] on the type or member, which makes it one whole value with no element ids.";
            }
        }

        return null;
    }

    private static string CannotBuild(Type documentType, Type type, string reason) =>
        $"Cult document {documentType.Name}: MessagePack cannot build a formatter for {type.Name}, which is on the path to a list whose elements carry ids ({reason}).";

    // The door verifies the walk and never decides ownership. It refuses a proven mismatch: a type the walk reads by key
    // (an owner) whose StandardResolver formatter is the very instance the generic resolver's singleton hands out (a keyed
    // ICollection<T> or dictionary). Union types walk as owners and the immutable and builtin resolvers claim only framework
    // types, so neither can contradict. Any other formatter, a source-generated one among them, passes.
    private static bool WrittenAsCollection(Type type, MethodInfo generic, object? standard) =>
        CultElementIds.WalksAsOwner(type) && ReferenceEquals(Formatter(DynamicGenericResolver.Instance, generic), standard);

    private static object? Formatter(IFormatterResolver resolver, MethodInfo generic)
    {
        try
        {
            return generic.Invoke(resolver, null);
        }
        catch (Exception)
        {
            return null;
        }
    }

    private static (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) Compose(IFormatterResolver[] consumerResolvers)
    {
        // The guard goes first and wraps whatever formatter the rest of the composite would hand out (a union, or a concrete int-keyed object for its slot count), so a consumer
        // resolver still encodes an arm while the key check runs.
        var ahead = consumerResolvers.Append(CultDocumentResolver.Instance).Append(CultMathResolver.Instance).Append(CultMathExprResolver.Instance).ToArray();
        var inner = CompositeResolver.Create(ahead.Append(StandardResolver.Instance).ToArray());
        var options = MessagePackSerializerOptions.Standard
            .WithResolver(CompositeResolver.Create(new CultUnionGuard(inner), inner))
            .WithSecurity(CultMessagePackSecurity.Instance);
        return (options, ahead);
    }

    private static IFormatterResolver CreateResolver(Type resolverType)
    {
        var instance = resolverType.GetField("Instance", BindingFlags.Public | BindingFlags.Static)?.GetValue(null)
                       ?? resolverType.GetProperty("Instance", BindingFlags.Public | BindingFlags.Static)?.GetValue(null)
                       ?? (resolverType.GetConstructor(Type.EmptyTypes) != null ? Activator.CreateInstance(resolverType) : null);
        return instance as IFormatterResolver
               ?? throw new InvalidOperationException(
                   $"Formatter resolver {resolverType.FullName} needs a public static Instance or a public parameterless constructor and must implement {nameof(IFormatterResolver)}.");
    }

    public static byte[] SerializeUntyped(object value, Type type)
    {
        return SerializeUntyped(value, type, CultDocumentRegistry.Shared);
    }

    public static byte[] SerializeUntyped(object value, Type type, CultDocumentRegistry registry)
    {
        if (registry == null) throw new ArgumentNullException(nameof(registry));
        if (value != null) registry.GetRequired(type);
        var payload = MessagePackSerializer.Serialize(type, value, OptionsForDocument(type));
        CultPayloadDepth.Check(payload, type, PayloadDirection.Write);
        return payload;
    }

    public static object DeserializeUntyped(Type type, byte[] payload)
    {
        return DeserializeUntyped(type, payload, CultDocumentRegistry.Shared);
    }

    public static object DeserializeUntyped(Type type, byte[] payload, CultDocumentRegistry registry)
    {
        if (registry == null) throw new ArgumentNullException(nameof(registry));
        registry.GetRequired(type);
        CultPayloadDepth.Check(payload, type, PayloadDirection.Read);
        return MessagePackSerializer.Deserialize(type, payload, OptionsForDocument(type))
            ?? throw new InvalidOperationException($"MessagePack returned null for Cult document type {type.FullName}.");
    }

    // The same decode, with the slots it read past noted (CultSlotScope). Opened around the one decode and after the depth check,
    // so a payload past the nesting limit is refused, never marked.
    internal static CultDecoded DeserializeMarked(Type type, byte[] payload, CultDocumentRegistry registry)
    {
        if (registry == null) throw new ArgumentNullException(nameof(registry));
        registry.GetRequired(type);
        CultPayloadDepth.Check(payload, type, PayloadDirection.Read);
        using var scope = CultSlotScope.Open();
        var document = MessagePackSerializer.Deserialize(type, payload, OptionsForDocument(type))
            ?? throw new InvalidOperationException($"MessagePack returned null for Cult document type {type.FullName}.");
        return new CultDecoded(document, scope.Finding());
    }

    /// <summary>
    /// Replaces slots of a serialized document payload (a MessagePack array indexed by slot) with already-encoded member
    /// values, leaving every other slot byte for byte as it was. A slot past the end extends the array with nil.
    /// </summary>
    public static byte[] OverlaySlots(byte[] payload, IReadOnlyList<KeyValuePair<int, byte[]>> slots)
    {
        var reader = new MessagePackReader(payload);
        if (reader.NextMessagePackType != MessagePackType.Array)
        {
            throw new InvalidOperationException("A document payload is a MessagePack array; this one is not, so its slots cannot be overridden.");
        }

        var count = reader.ReadArrayHeader();
        var elements = new List<byte[]>(count);
        for (var index = 0; index < count; index++)
        {
            var start = reader.Position;
            reader.Skip();
            elements.Add(reader.Sequence.Slice(start, reader.Position).ToArray());
        }

        foreach (var (slot, value) in slots)
        {
            while (elements.Count <= slot)
            {
                elements.Add(new byte[] { 0xc0 });
            }

            elements[slot] = value;
        }

        var buffer = new global::System.Buffers.ArrayBufferWriter<byte>();
        var writer = new MessagePackWriter(buffer);
        writer.WriteArrayHeader(elements.Count);
        foreach (var element in elements)
        {
            writer.WriteRaw(element);
        }

        writer.Flush();
        return buffer.WrittenSpan.ToArray();
    }

    public static byte[] SerializePersistedRecord(CultPersistedRecord record)
    {
        var buffer = new global::System.Buffers.ArrayBufferWriter<byte>();
        var writer = new MessagePackWriter(buffer);
        WritePersistedRecord(ref writer, record);
        writer.Flush();
        return buffer.WrittenSpan.ToArray();
    }

    public static CultPersistedRecord DeserializePersistedRecord(byte[] payload)
    {
        var reader = new MessagePackReader(payload);
        return ReadPersistedRecord(ref reader);
    }

    /// <summary>
    /// Encodes the snapshot: every catalog entry and record from its fields, so what a caller edited is what is written.
    /// </summary>
    public static byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot) =>
        SerializeStore(
            snapshot.FormatVersion,
            snapshot.SchemaCatalog.Select(SerializeCatalogEntry).ToArray(),
            snapshot.Records.Select(SerializePersistedRecord).ToArray());

    // A store file from its parts, each already encoded: what a store carries forward is the bytes its file held, what it writes is
    // an encoding of the object.
    internal static byte[] SerializeStore(string formatVersion, IReadOnlyList<byte[]> catalogEntries, IReadOnlyList<byte[]> records)
    {
        if (formatVersion == null)
            throw new ArgumentException("A store is written under a format version; the snapshot carries none.", nameof(formatVersion));
        var buffer = new global::System.Buffers.ArrayBufferWriter<byte>();
        var writer = new MessagePackWriter(buffer);
        writer.WriteArrayHeader(StoreSnapshotFieldCount);
        writer.Write(formatVersion);
        writer.WriteArrayHeader(catalogEntries.Count);
        foreach (var entry in catalogEntries)
        {
            writer.WriteRaw(new ReadOnlySpan<byte>(entry));
        }

        writer.WriteArrayHeader(records.Count);
        foreach (var record in records)
        {
            writer.WriteRaw(new ReadOnlySpan<byte>(record));
        }

        writer.Flush();
        return buffer.WrittenSpan.ToArray();
    }

    /// <summary>
    /// Reads a store file: exactly one complete MessagePack array of three slots (header, schema catalog, records), or an empty
    /// array, which is an empty store. A truncated file, bytes after the array, a value that is not an array, a first slot that
    /// is not a header string, a store with fewer or more slots, a record whose schema the catalog does not publish, two records under one key, or a body
    /// that does not decode are refused with
    /// <see cref="CultStoreUnreadableException"/>, so a reader never takes part of a file for the whole and never skips a slot
    /// it does not understand. The rewriting paths use this same reader as their verdict on whether a file may be replaced.
    /// </summary>
    public static CultPersistedStoreSnapshot DeserializeSnapshot(byte[] payload) => ReadStore(payload).Snapshot;

    // The reader behind DeserializeSnapshot, and beside the snapshot the exact bytes the file held for each catalog entry and each
    // record (parallel to the snapshot's arrays). The snapshot's objects hold no bytes: a store that carries entries forward keeps
    // these slices itself. The stamp decides readability and the body never does: requireReadable, when a store passes its rule,
    // runs on slot 0 before the catalog or any record is parsed; the public parser passes none and reads any stamp.
    internal static (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(
        byte[] payload,
        Action<string>? requireReadable = null)
    {
        try
        {
            // The stamp is judged from the leading bytes alone, so a body that will not parse can never make a newer store look
            // like damage.
            JudgeStamp(payload, requireReadable);
            RequireOneArray(payload);
            var reader = new MessagePackReader(payload);
            var fieldCount = reader.ReadArrayHeader();
            if (fieldCount != 0 && fieldCount != StoreSnapshotFieldCount)
            {
                throw new CultStoreUnreadableException(
                    $"The store has {fieldCount} top-level slots; this runtime reads {StoreSnapshotFieldCount} (header, schema catalog, records) or none.");
            }

            var snapshot = new CultPersistedStoreSnapshot();
            if (fieldCount == 0)
                return (snapshot, Array.Empty<byte[]>(), Array.Empty<byte[]>());

            snapshot.FormatVersion = reader.ReadString()!;

            var catalogCount = reader.ReadArrayHeader();
            snapshot.SchemaCatalog = new CultSchemaCatalogEntry[catalogCount];
            var catalogBytes = new byte[catalogCount][];
            for (var index = 0; index < catalogCount; index++)
            {
                var start = reader.Position;
                snapshot.SchemaCatalog[index] = ReadSchemaCatalogEntry(ref reader);
                catalogBytes[index] = reader.Sequence.Slice(start, reader.Position).ToArray();
            }

            var recordCount = reader.ReadArrayHeader();
            snapshot.Records = new CultPersistedRecord[recordCount];
            var recordBytes = new byte[recordCount][];
            for (var index = 0; index < recordCount; index++)
            {
                var start = reader.Position;
                snapshot.Records[index] = ReadPersistedRecord(ref reader);
                recordBytes[index] = reader.Sequence.Slice(start, reader.Position).ToArray();
            }

            RequirePublishedSchemas(snapshot);
            RequireUniqueKeys(snapshot);
            return (snapshot, catalogBytes, recordBytes);
        }
        catch (CultStoreUnreadableException)
        {
            throw;
        }
        catch (Exception ex) when (ex is not OutOfMemoryException)
        {
            throw new CultStoreUnreadableException(ex.Message, innerException: ex);
        }
    }

    // A record whose schema the store's catalog does not publish cannot be read by any runtime that has only the file.
    private static void RequirePublishedSchemas(CultPersistedStoreSnapshot snapshot)
    {
        var published = new HashSet<string>(StringComparer.Ordinal);
        foreach (var entry in snapshot.SchemaCatalog)
        {
            published.Add(entry.SchemaId);
            foreach (var compatible in entry.CompatibleSchemaIds)
                published.Add(compatible);
        }

        var unpublished = snapshot.Records.FirstOrDefault(record => !published.Contains(record.SchemaId));
        if (unpublished != null)
        {
            throw new CultStoreUnreadableException(
                $"Record '{unpublished.Key}' names schema '{unpublished.SchemaId}', which the store's schema catalog does not publish.");
        }
    }

    // A key names one record: identity is the key string alone, compared by ordinal code units (no case folding, no normalisation).
    private static void RequireUniqueKeys(CultPersistedStoreSnapshot snapshot)
    {
        var seen = new HashSet<string>(StringComparer.Ordinal);
        foreach (var record in snapshot.Records)
        {
            if (!seen.Add(record.Key))
                throw new CultStoreUnreadableException($"Record '{record.Key}' appears more than once.");
        }
    }

    // Slot 0 of a non-empty store, read before anything after it: a binary or nil stamp is refused as such, and a string stamp is
    // judged by the caller's rule when it passes one. Bytes that do not open with an array and a string are left to the full read.
    private static void JudgeStamp(byte[] payload, Action<string>? requireReadable)
    {
        string? stamp;
        try
        {
            var reader = new MessagePackReader(payload);
            if (reader.NextMessagePackType != MessagePackType.Array || reader.ReadArrayHeader() == 0)
                return;
            if (reader.NextMessagePackType == MessagePackType.Binary)
                throw new CultStoreUnreadableException("Store snapshot declares its format version as binary; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + ", " + CultPersistedStoreSnapshot.FormatV2 + " and " + CultPersistedStoreSnapshot.FormatV3 + " as strings.");
            if (reader.TryReadNil())
                throw new CultStoreUnreadableException("Store snapshot declares no format version; this runtime reads " + CultPersistedStoreSnapshot.FormatV1 + ", " + CultPersistedStoreSnapshot.FormatV2 + " and " + CultPersistedStoreSnapshot.FormatV3 + ".");
            stamp = reader.NextMessagePackType == MessagePackType.String ? reader.ReadString() : null;
        }
        catch (Exception ex) when (ex is EndOfStreamException or MessagePackSerializationException or InvalidOperationException)
        {
            return;
        }

        if (stamp != null)
            requireReadable?.Invoke(stamp);
    }

    private static void RequireOneArray(byte[] bytes)
    {
        var whole = new MessagePackReader(bytes);
        try
        {
            if (whole.NextMessagePackType != MessagePackType.Array)
                throw new CultStoreUnreadableException($"The store is a MessagePack {whole.NextMessagePackType}, not an array.");
            whole.Skip();
        }
        catch (Exception ex) when (ex is EndOfStreamException or MessagePackSerializationException or InvalidOperationException)
        {
            throw new CultStoreUnreadableException("The store is not one complete MessagePack array.", innerException: ex);
        }

        if (!whole.End)
            throw new CultStoreUnreadableException("The store has bytes after its MessagePack array.");
    }

    /// <summary>Refuses a single-file snapshot this runtime cannot read, naming the format or record it found.</summary>
    public static void RequireSingleFileFormat(CultPersistedStoreSnapshot snapshot)
    {
        var version = snapshot.FormatVersion;
        RequireReadableSingleFileStamp(version);

        // Only a v2 or v3 store may hold a variant: a v1 header over a variant record is a store no reader could trust.
        if (string.Equals(version, CultPersistedStoreSnapshot.FormatV1, StringComparison.Ordinal))
        {
            var variant = snapshot.Records.FirstOrDefault(record => record.Variant != null);
            if (variant != null)
            {
                throw new CultStoreUnreadableException(
                    $"Record '{variant.Key}' (schema '{variant.SchemaId}') is a variant but the store declares {version}; variants need {CultPersistedStoreSnapshot.FormatV2} or {CultPersistedStoreSnapshot.FormatV3}.");
            }
        }
    }

    // The one verdict on a single-file store's stamp, judged on slot 0 before the body is parsed: this runtime reads v1, v2 and v3.
    internal static void RequireReadableSingleFileStamp(string version)
    {
        if (!string.Equals(version, CultPersistedStoreSnapshot.FormatV1, StringComparison.Ordinal) &&
            !string.Equals(version, CultPersistedStoreSnapshot.FormatV2, StringComparison.Ordinal) &&
            !string.Equals(version, CultPersistedStoreSnapshot.FormatV3, StringComparison.Ordinal))
        {
            throw new CultStoreUnreadableException(
                $"Store format {DescribeHeader(version)} is not readable; this runtime reads {CultPersistedStoreSnapshot.FormatV1}, {CultPersistedStoreSnapshot.FormatV2} and {CultPersistedStoreSnapshot.FormatV3}.");
        }
    }

    // A store header as a refusal may show it: echoed in the shape cultcache.store.v<digits>, and otherwise described
    // only by its length, since the bytes are the store's.
    internal static string DescribeHeader(string header)
    {
        const string prefix = "cultcache.store.v";
        if (header.StartsWith(prefix, StringComparison.Ordinal))
        {
            var end = prefix.Length;
            while (end < header.Length && header[end] >= '0' && header[end] <= '9')
                end++;
            var digits = end - prefix.Length;
            if (digits is > 0 and <= 6 && (end == header.Length || header[end] == '.'))
            {
                var named = header.Substring(0, end);
                return end == header.Length ? named : $"{named} with a suffix of {System.Text.Encoding.UTF8.GetByteCount(header) - end} bytes";
            }
        }

        return $"an unrecognised cultcache.store.* header of {System.Text.Encoding.UTF8.GetByteCount(header)} bytes";
    }

    private static void WritePersistedRecord(ref MessagePackWriter writer, CultPersistedRecord record)
    {
        writer.WriteArrayHeader(record.Variant == null ? PersistedRecordFieldCount : VariantRecordFieldCount);
        writer.Write(record.Key);
        writer.Write(record.SchemaId);
        writer.Write(record.StoredAt);
        writer.Write(record.Payload);
        if (record.Variant != null)
        {
            // Slot 4: [baseKey, overrides[]]; an override is [op, path[], id, value], a path step [slot, elementId],
            // and the value is the member's own MessagePack encoding, inline.
            writer.WriteArrayHeader(2);
            writer.Write(record.Variant.BaseKey);
            writer.WriteArrayHeader(record.Variant.Overrides.Count);
            foreach (var entry in record.Variant.Overrides)
            {
                writer.WriteArrayHeader(4);
                writer.Write((int)entry.Op);
                writer.WriteArrayHeader(entry.Path.Count);
                foreach (var step in entry.Path)
                {
                    writer.WriteArrayHeader(2);
                    writer.Write(step.Slot);
                    writer.Write(step.ElementId);
                }

                writer.Write(entry.Id);
                writer.WriteRaw(entry.Value);
            }
        }
    }

    private static CultVariantDelta ReadVariant(ref MessagePackReader reader, string key, string schemaId)
    {
        var slots = reader.ReadArrayHeader();
        if (slots != 2)
        {
            throw new NotSupportedException(
                $"Record '{key}' (schema '{schemaId}') has a variant slot of {slots} entries; this runtime reads 2 (base key, overrides).");
        }

        var baseKey = reader.ReadString() ?? string.Empty;
        var count = reader.ReadArrayHeader();
        var overrides = new CultVariantOverride[count];
        for (var index = 0; index < count; index++)
        {
            var fields = reader.ReadArrayHeader();
            if (fields != 4)
            {
                throw new NotSupportedException(
                    $"Record '{key}' (schema '{schemaId}') has an override of {fields} entries; this runtime reads 4 (op, path, id, value).");
            }

            var op = reader.ReadInt32();
            if (op is < 0 or > 2)
            {
                throw new NotSupportedException($"Record '{key}' (schema '{schemaId}') has an override with unknown op {op}.");
            }

            var stepCount = reader.ReadArrayHeader();
            var path = new CultPathStep[stepCount];
            for (var step = 0; step < stepCount; step++)
            {
                var stepFields = reader.ReadArrayHeader();
                if (stepFields != 2)
                {
                    throw new NotSupportedException(
                        $"Record '{key}' (schema '{schemaId}') has a path step of {stepFields} entries; this runtime reads 2 (slot, elementId).");
                }

                path[step] = new CultPathStep(reader.ReadInt32(), reader.ReadString() ?? string.Empty);
            }

            var id = reader.ReadString() ?? string.Empty;
            var start = reader.Position;
            reader.Skip();
            var value = reader.Sequence.Slice(start, reader.Position).ToArray();
            overrides[index] = new CultVariantOverride((CultOverrideOp)op, path, id, value);
        }

        return new CultVariantDelta(baseKey, overrides);
    }

    public static CultPersistedRecord ReadPersistedRecord(ref MessagePackReader reader)
    {
        var fieldCount = reader.ReadArrayHeader();
        var record = new CultPersistedRecord();

        if (fieldCount > 0)
        {
            record.Key = reader.ReadString() ?? string.Empty;
        }

        if (fieldCount > 1)
        {
            record.SchemaId = reader.ReadString() ?? string.Empty;
        }

        if (fieldCount > 2)
        {
            record.StoredAt = reader.ReadString() ?? string.Empty;
        }

        if (fieldCount > 3)
        {
            record.Payload = reader.ReadBytes()?.ToArray() ?? Array.Empty<byte>();
        }

        if (fieldCount > VariantRecordFieldCount)
        {
            throw new NotSupportedException(
                $"Record '{record.Key}' (schema '{record.SchemaId}') has {fieldCount} slots; this runtime reads {PersistedRecordFieldCount} " +
                $"(or {VariantRecordFieldCount} for a variant).");
        }

        if (fieldCount == VariantRecordFieldCount)
        {
            record.Variant = ReadVariant(ref reader, record.Key, record.SchemaId);
            if (record.Payload.Length != 0)
            {
                throw new NotSupportedException(
                    $"Record '{record.Key}' (schema '{record.SchemaId}') is a variant and must carry an empty payload.");
            }
        }

        return record;
    }

    internal static byte[] SerializeCatalogEntry(CultSchemaCatalogEntry entry)
    {
        var buffer = new global::System.Buffers.ArrayBufferWriter<byte>();
        var writer = new MessagePackWriter(buffer);
        WriteSchemaCatalogEntry(ref writer, entry);
        writer.Flush();
        return buffer.WrittenSpan.ToArray();
    }

    private static void WriteSchemaCatalogEntry(ref MessagePackWriter writer, CultSchemaCatalogEntry entry)
    {
        writer.WriteArrayHeader(SchemaCatalogEntryFieldCount);
        writer.Write(entry.SchemaId);
        writer.Write(entry.SchemaName);
        writer.Write(entry.SchemaVersion);
        writer.Write(entry.ContentHash);
        writer.Write(entry.CanonicalSchemaJson);
        writer.WriteArrayHeader(entry.CompatibleSchemaIds.Length);
        foreach (var schemaId in entry.CompatibleSchemaIds)
        {
            writer.Write(schemaId);
        }

        writer.WriteArrayHeader(entry.Members.Length);
        foreach (var member in entry.Members)
        {
            WriteSchemaCatalogMember(ref writer, member);
        }
    }

    private static CultSchemaCatalogEntry ReadSchemaCatalogEntry(ref MessagePackReader reader)
    {
        var fieldCount = reader.ReadArrayHeader();
        var entry = new CultSchemaCatalogEntry();

        if (fieldCount > 0)
        {
            entry.SchemaId = reader.ReadString() ?? string.Empty;
        }

        if (fieldCount > 1)
        {
            entry.SchemaName = reader.ReadString() ?? string.Empty;
        }

        if (fieldCount > 2)
        {
            entry.SchemaVersion = reader.ReadString() ?? string.Empty;
        }

        if (fieldCount > 3)
        {
            entry.ContentHash = reader.ReadString() ?? string.Empty;
        }

        if (fieldCount > 4)
        {
            entry.CanonicalSchemaJson = reader.ReadString() ?? string.Empty;
        }

        if (fieldCount > 5)
        {
            var compatibleCount = reader.ReadArrayHeader();
            entry.CompatibleSchemaIds = new string[compatibleCount];
            for (var index = 0; index < compatibleCount; index++)
            {
                entry.CompatibleSchemaIds[index] = reader.ReadString() ?? string.Empty;
            }
        }

        if (fieldCount > 6)
        {
            var memberCount = reader.ReadArrayHeader();
            entry.Members = new CultSchemaMemberCatalogEntry[memberCount];
            for (var index = 0; index < memberCount; index++)
            {
                entry.Members[index] = ReadSchemaCatalogMember(ref reader);
            }
        }

        for (var index = SchemaCatalogEntryFieldCount; index < fieldCount; index++)
        {
            reader.Skip();
        }

        return entry;
    }

    private static void WriteSchemaCatalogMember(ref MessagePackWriter writer, CultSchemaMemberCatalogEntry member)
    {
        writer.WriteArrayHeader(SchemaCatalogMemberFieldCount);
        writer.Write(member.Slot);
        writer.Write(member.MemberName);
        writer.Write(member.TypeName);
        writer.Write(member.IsReference);
        writer.Write(member.IsMany);
        writer.Write(member.TargetSchemaName);
        writer.Write(member.IsName);
        writer.Write(member.IndexAlias);
    }

    private static CultSchemaMemberCatalogEntry ReadSchemaCatalogMember(ref MessagePackReader reader)
    {
        var fieldCount = reader.ReadArrayHeader();
        var member = new CultSchemaMemberCatalogEntry();

        if (fieldCount > 0)
        {
            member.Slot = reader.ReadInt32();
        }

        if (fieldCount > 1)
        {
            member.MemberName = reader.ReadString() ?? string.Empty;
        }

        if (fieldCount > 2)
        {
            member.TypeName = reader.ReadString() ?? string.Empty;
        }

        if (fieldCount > 3)
        {
            member.IsReference = reader.ReadBoolean();
        }

        if (fieldCount > 4)
        {
            member.IsMany = reader.ReadBoolean();
        }

        if (fieldCount > 5)
        {
            member.TargetSchemaName = reader.ReadString();
        }

        if (fieldCount > 6)
        {
            member.IsName = reader.ReadBoolean();
        }

        if (fieldCount > 7)
        {
            member.IndexAlias = reader.ReadString();
        }

        for (var index = SchemaCatalogMemberFieldCount; index < fieldCount; index++)
        {
            reader.Skip();
        }

        return member;
    }
}

public class SingleFileMessagePackBackingStore : SingleFileBackingStore
{
    public SingleFileMessagePackBackingStore(string filePath, bool readOnly = false) : base(filePath, readOnly)
    {
    }

    protected override byte[] SerializeStore(string formatVersion, IReadOnlyList<byte[]> catalogEntries, IReadOnlyList<byte[]> records)
    {
        return CultDocumentMessagePackSerialization.SerializeStore(formatVersion, catalogEntries, records);
    }

    protected override byte[] SerializeRecord(CultPersistedRecord record)
    {
        return CultDocumentMessagePackSerialization.SerializePersistedRecord(record);
    }

    protected override byte[] SerializeCatalogEntry(CultSchemaCatalogEntry entry)
    {
        return CultDocumentMessagePackSerialization.SerializeCatalogEntry(entry);
    }

    protected override (CultPersistedStoreSnapshot Snapshot, byte[][] CatalogBytes, byte[][] RecordBytes) ReadStore(byte[] data)
    {
        var store = CultDocumentMessagePackSerialization.ReadStore(data, CultDocumentMessagePackSerialization.RequireReadableSingleFileStamp);
        CultDocumentMessagePackSerialization.RequireSingleFileFormat(store.Snapshot);
        return store;
    }

    protected override byte[] SerializePayload(object document, Type documentType)
    {
        return CultDocumentMessagePackSerialization.SerializeUntyped(document, documentType, Registry);
    }

    protected override object DeserializePayload(Type documentType, byte[] payload)
    {
        return CultDocumentMessagePackSerialization.DeserializeUntyped(documentType, payload, Registry);
    }

    internal override CultDecoded DeserializeMarked(Type documentType, byte[] payload) =>
        CultDocumentMessagePackSerialization.DeserializeMarked(documentType, payload, Registry);
}