Use paged CultCache trial stores

src/AquaSynth.Core/IpaTrialResults.cs

@@ -87,12 +87,12 @@ public static class IpaTrialResultCultCacheStore
 
     public static async Task<IReadOnlyList<IpaTrialResult>> ReadResultsAsync(string filePath)
     {
-        if (!File.Exists(filePath))
+        if (!StoreExists(filePath))
         {
             return [];
         }
 
-        using var cache = await CultCacheMessagePack.OpenAsync(filePath).ConfigureAwait(false);
+        using var cache = await OpenTrialStoreAsync(filePath, pullOnOpen: true).ConfigureAwait(false);
         return cache.GetAll<IpaTrialResult>()
             .OrderBy(result => result.TrialId, StringComparer.Ordinal)
             .ToArray();
@@ -100,12 +100,12 @@ public static class IpaTrialResultCultCacheStore
 
     public static async Task<IReadOnlyList<SongChallengeEvidenceDocument>> ReadSongChallengeEvidenceAsync(string filePath)
     {
-        if (!File.Exists(filePath))
+        if (!StoreExists(filePath))
         {
             return [];
         }
 
-        using var cache = await CultCacheMessagePack.OpenAsync(filePath).ConfigureAwait(false);
+        using var cache = await OpenTrialStoreAsync(filePath, pullOnOpen: true).ConfigureAwait(false);
         return cache.GetAll<SongChallengeEvidenceDocument>()
             .OrderBy(document => document.EvidenceId, StringComparer.Ordinal)
             .ToArray();
@@ -113,12 +113,12 @@ public static class IpaTrialResultCultCacheStore
 
     public static async Task<IReadOnlyList<SongTrialDistillationDocument>> ReadSongTrialDistillationsAsync(string filePath)
     {
-        if (!File.Exists(filePath))
+        if (!StoreExists(filePath))
         {
             return [];
         }
 
-        using var cache = await CultCacheMessagePack.OpenAsync(filePath).ConfigureAwait(false);
+        using var cache = await OpenTrialStoreAsync(filePath, pullOnOpen: true).ConfigureAwait(false);
         return cache.GetAll<SongTrialDistillationDocument>()
             .OrderBy(document => document.DistillationId, StringComparer.Ordinal)
             .ToArray();
@@ -136,9 +136,7 @@ public static class IpaTrialResultCultCacheStore
             Directory.CreateDirectory(directory);
         }
 
-        using var cache = await CultCacheMessagePack.OpenAsync(
-            filePath,
-            new CultCacheOpenOptions { PullOnOpen = File.Exists(filePath) }).ConfigureAwait(false);
+        using var cache = await OpenTrialStoreAsync(filePath, pullOnOpen: StoreExists(filePath)).ConfigureAwait(false);
         foreach (var document in documents)
         {
             await cache.UpsertAsync(document, new CultRecordHandle<T>(new CultRecordKey(key(document)))).ConfigureAwait(false);
@@ -146,4 +144,17 @@ public static class IpaTrialResultCultCacheStore
 
         await cache.FlushAsync().ConfigureAwait(false);
     }
+
+    private static Task<CultCache> OpenTrialStoreAsync(string filePath, bool pullOnOpen) =>
+        CultCacheMessagePack.OpenAsync(
+            filePath,
+            new CultCacheOpenOptions
+            {
+                PullOnOpen = pullOnOpen,
+                UseDirectoryStore = true
+            });
+
+    private static bool StoreExists(string filePath) =>
+        File.Exists(filePath) ||
+        Directory.Exists(DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(filePath));
 }

state/evidence.jsonl

@@ -258,3 +258,4 @@
 {"date":"2026-05-31","area":"song-curriculum-playlists-and-invalid-candidates","lesson":"Song curriculum passes now publish a playable top-candidate playlist per pass and keep invalid agent syntax as negative evidence instead of aborting the swarm. A two-pass paired run produced pass-001 best cosine `0.386098` with 8 playable candidates and 2 `render-failed` syntax casualties, then pass-002 improved to best cosine `0.502435`, mean cosine `0.352907`, 10 playable candidates, and 8 pressure verdicts after reading distilled pass-1 evidence. Playlist reports filter to candidates with actual rendered WAVs and list skipped non-playable failures separately.","evidence":"tools/run-song-snippet-trial-swarm.ps1; tools/IpaTrialWorker/Program.cs; artifacts/parity/song-snippet-swarms/20260531T084455803/pass-001/top-scoring-candidates.m3u; artifacts/parity/song-snippet-swarms/20260531T084455803/pass-002/top-scoring-candidates.m3u; artifacts/parity/song-snippet-swarms/20260531T084455803/pass-002/distillation-report.md"}
 {"date":"2026-05-31","area":"song-instrument-owner-scoring","lesson":"The song curriculum harness must teach instrument ownership, not only spectrogram proximity. `song-score` now records syrinx-voice, subtractive-drum, additive/PAD, musical-instrument-score, and chip-distress-risk metrics for every candidate; swarm prompts require voice-like leads to prefer syrinx/acoustic sources, percussion to prefer subtractive pitched body plus filtered noise skin, and pads/beds to prefer additive/PAD banks. Distillation ranking now rewards owned roles and demotes chip-distress-risk candidates unless they are explicit contrast evidence.","evidence":"tools/IpaTrialWorker/Program.cs; tools/run-song-snippet-trial-swarm.ps1; docs/audio-doctrine.md; patches/advanced/syrinx-voice.aqua; patches/advanced/subtractive-drums.aqua; patches/advanced/additive-pad.aqua; artifacts/parity/song-instrument-metric-smoke/smoke/summary.csv"}
 {"date":"2026-05-31","area":"song-full-iteration-loop","lesson":"Song agents need a studio loop, not a one-shot submission. `song-prepare --duration-seconds 0` freezes full decoded songs; the swarm exposes `-FullSongs`; song challenges and candidate renders now include whitened spectral autocorrelation over band-whitened log-mel flux; and the prompt requires five local score/revise attempts per target before publishing a final patch. Candidate-side analysis artifacts let agents compare their Faust render against the target using the same feature vocabulary instead of guessing from the final verdict.","evidence":"tools/IpaTrialWorker/Program.cs; tools/run-song-snippet-trial-swarm.ps1; artifacts/parity/song-candidate-feature-smoke-2; artifacts/parity/song-snippet-swarms/20260531T120800752"}
+{"date":"2026-05-31","area":"cultcache-storage","summary":"Full-song song-trial stores can exceed 2 GB; single-file MessagePack snapshot flushing is the wrong owner for hot trial writes because it rewrites every cold record and can OOM. CultLib now has DirectoryMessagePackBackingStore: .cc is a small schema manifest, records live under <store>.records/*.msgpack, and AquaSynth trial-result stores open with UseDirectoryStore=true.","verification":"dotnet test E:\\Projects\\CultLib\\tests\\GameCult.Caching.Tests\\GameCult.Caching.Tests.csproj --no-restore --filter BackingStoreTests: 14 passed; dotnet build E:\\Projects\\AquaSynth\\tools\\IpaTrialWorker\\IpaTrialWorker.csproj --no-restore: passed; dotnet test E:\\Projects\\AquaSynth\\tests\\AquaSynth.Dsl.Tests\\AquaSynth.Dsl.Tests.csproj --no-restore --filter CultCache: 7 passed"}

state/scratch.md

@@ -1855,3 +1855,25 @@ Next likely slice:
   passed; zero-pass full-song swarm syntax smoke passed; candidate feature
   smoke wrote the new candidate autocorrelation and analysis artifacts under
   `artifacts/parity/song-candidate-feature-smoke-2`.
+
+2026-05-31 CultCache paged trial-store slice:
+
+- Full-song iterative swarm exposed the storage owner fault: the shared
+  `song-trial-results.cc` single-file MessagePack store grew to ~2.13 GB and
+  failed at the end of pass 3 with `System.OutOfMemoryException` while trying
+  to allocate another whole-store buffer. The issue was not the song agents;
+  it was the backing store treating cold old records as part of every hot write.
+- CultLib now has `DirectoryMessagePackBackingStore`. In directory mode, the
+  `.cc` path is a small schema manifest and each record is an individual
+  MessagePack page under `<store>.records/`. `PushAll` rewrites changed/deleted
+  records and the manifest only; cold record files keep their timestamps and
+  are not serialized into one giant byte array.
+- `CultCacheOpenOptions.UseDirectoryStore` selects the paged store, and
+  `CultCacheMessagePack.Create/OpenAsync` automatically reopen a paged store
+  when the `.records` directory exists. AquaSynth trial-result stores now
+  always use directory mode through `IpaTrialResultCultCacheStore`, so future
+  song/IPA curriculum runs stop dragging the whole archive through RAM on each
+  upsert.
+- Verification: CultLib `BackingStoreTests` pass 14/14, including a cold-record
+  timestamp test; AquaSynth `dotnet build tools/IpaTrialWorker/IpaTrialWorker.csproj
+  --no-restore` passes; AquaSynth `--filter CultCache` tests pass 7/7.