diff --git a/src/AquaSynth.Core/IpaTrialResults.cs b/src/AquaSynth.Core/IpaTrialResults.cs index e67661f..fad97e7 100644 --- a/src/AquaSynth.Core/IpaTrialResults.cs +++ b/src/AquaSynth.Core/IpaTrialResults.cs @@ -87,12 +87,12 @@ public static class IpaTrialResultCultCacheStore public static async Task> ReadResultsAsync(string filePath) { - if (!File.Exists(filePath)) + if (!StoreExists(filePath)) { return []; } - using var cache = await CultCacheMessagePack.OpenAsync(filePath).ConfigureAwait(false); + using var cache = await OpenTrialStoreAsync(filePath, pullOnOpen: true).ConfigureAwait(false); return cache.GetAll() .OrderBy(result => result.TrialId, StringComparer.Ordinal) .ToArray(); @@ -100,12 +100,12 @@ public static class IpaTrialResultCultCacheStore public static async Task> ReadSongChallengeEvidenceAsync(string filePath) { - if (!File.Exists(filePath)) + if (!StoreExists(filePath)) { return []; } - using var cache = await CultCacheMessagePack.OpenAsync(filePath).ConfigureAwait(false); + using var cache = await OpenTrialStoreAsync(filePath, pullOnOpen: true).ConfigureAwait(false); return cache.GetAll() .OrderBy(document => document.EvidenceId, StringComparer.Ordinal) .ToArray(); @@ -113,12 +113,12 @@ public static class IpaTrialResultCultCacheStore public static async Task> ReadSongTrialDistillationsAsync(string filePath) { - if (!File.Exists(filePath)) + if (!StoreExists(filePath)) { return []; } - using var cache = await CultCacheMessagePack.OpenAsync(filePath).ConfigureAwait(false); + using var cache = await OpenTrialStoreAsync(filePath, pullOnOpen: true).ConfigureAwait(false); return cache.GetAll() .OrderBy(document => document.DistillationId, StringComparer.Ordinal) .ToArray(); @@ -136,9 +136,7 @@ public static class IpaTrialResultCultCacheStore Directory.CreateDirectory(directory); } - using var cache = await CultCacheMessagePack.OpenAsync( - filePath, - new CultCacheOpenOptions { PullOnOpen = File.Exists(filePath) }).ConfigureAwait(false); + using var cache = await OpenTrialStoreAsync(filePath, pullOnOpen: StoreExists(filePath)).ConfigureAwait(false); foreach (var document in documents) { await cache.UpsertAsync(document, new CultRecordHandle(new CultRecordKey(key(document)))).ConfigureAwait(false); @@ -146,4 +144,17 @@ public static class IpaTrialResultCultCacheStore await cache.FlushAsync().ConfigureAwait(false); } + + private static Task OpenTrialStoreAsync(string filePath, bool pullOnOpen) => + CultCacheMessagePack.OpenAsync( + filePath, + new CultCacheOpenOptions + { + PullOnOpen = pullOnOpen, + UseDirectoryStore = true + }); + + private static bool StoreExists(string filePath) => + File.Exists(filePath) || + Directory.Exists(DirectoryMessagePackBackingStore.DefaultRecordDirectoryPath(filePath)); } diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 37cf474..b90129a 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -258,3 +258,4 @@ {"date":"2026-05-31","area":"song-curriculum-playlists-and-invalid-candidates","lesson":"Song curriculum passes now publish a playable top-candidate playlist per pass and keep invalid agent syntax as negative evidence instead of aborting the swarm. A two-pass paired run produced pass-001 best cosine `0.386098` with 8 playable candidates and 2 `render-failed` syntax casualties, then pass-002 improved to best cosine `0.502435`, mean cosine `0.352907`, 10 playable candidates, and 8 pressure verdicts after reading distilled pass-1 evidence. Playlist reports filter to candidates with actual rendered WAVs and list skipped non-playable failures separately.","evidence":"tools/run-song-snippet-trial-swarm.ps1; tools/IpaTrialWorker/Program.cs; artifacts/parity/song-snippet-swarms/20260531T084455803/pass-001/top-scoring-candidates.m3u; artifacts/parity/song-snippet-swarms/20260531T084455803/pass-002/top-scoring-candidates.m3u; artifacts/parity/song-snippet-swarms/20260531T084455803/pass-002/distillation-report.md"} {"date":"2026-05-31","area":"song-instrument-owner-scoring","lesson":"The song curriculum harness must teach instrument ownership, not only spectrogram proximity. `song-score` now records syrinx-voice, subtractive-drum, additive/PAD, musical-instrument-score, and chip-distress-risk metrics for every candidate; swarm prompts require voice-like leads to prefer syrinx/acoustic sources, percussion to prefer subtractive pitched body plus filtered noise skin, and pads/beds to prefer additive/PAD banks. Distillation ranking now rewards owned roles and demotes chip-distress-risk candidates unless they are explicit contrast evidence.","evidence":"tools/IpaTrialWorker/Program.cs; tools/run-song-snippet-trial-swarm.ps1; docs/audio-doctrine.md; patches/advanced/syrinx-voice.aqua; patches/advanced/subtractive-drums.aqua; patches/advanced/additive-pad.aqua; artifacts/parity/song-instrument-metric-smoke/smoke/summary.csv"} {"date":"2026-05-31","area":"song-full-iteration-loop","lesson":"Song agents need a studio loop, not a one-shot submission. `song-prepare --duration-seconds 0` freezes full decoded songs; the swarm exposes `-FullSongs`; song challenges and candidate renders now include whitened spectral autocorrelation over band-whitened log-mel flux; and the prompt requires five local score/revise attempts per target before publishing a final patch. Candidate-side analysis artifacts let agents compare their Faust render against the target using the same feature vocabulary instead of guessing from the final verdict.","evidence":"tools/IpaTrialWorker/Program.cs; tools/run-song-snippet-trial-swarm.ps1; artifacts/parity/song-candidate-feature-smoke-2; artifacts/parity/song-snippet-swarms/20260531T120800752"} +{"date":"2026-05-31","area":"cultcache-storage","summary":"Full-song song-trial stores can exceed 2 GB; single-file MessagePack snapshot flushing is the wrong owner for hot trial writes because it rewrites every cold record and can OOM. CultLib now has DirectoryMessagePackBackingStore: .cc is a small schema manifest, records live under .records/*.msgpack, and AquaSynth trial-result stores open with UseDirectoryStore=true.","verification":"dotnet test E:\\Projects\\CultLib\\tests\\GameCult.Caching.Tests\\GameCult.Caching.Tests.csproj --no-restore --filter BackingStoreTests: 14 passed; dotnet build E:\\Projects\\AquaSynth\\tools\\IpaTrialWorker\\IpaTrialWorker.csproj --no-restore: passed; dotnet test E:\\Projects\\AquaSynth\\tests\\AquaSynth.Dsl.Tests\\AquaSynth.Dsl.Tests.csproj --no-restore --filter CultCache: 7 passed"} diff --git a/state/scratch.md b/state/scratch.md index d1152e9..01269e9 100644 --- a/state/scratch.md +++ b/state/scratch.md @@ -1855,3 +1855,25 @@ Next likely slice: passed; zero-pass full-song swarm syntax smoke passed; candidate feature smoke wrote the new candidate autocorrelation and analysis artifacts under `artifacts/parity/song-candidate-feature-smoke-2`. + +2026-05-31 CultCache paged trial-store slice: + +- Full-song iterative swarm exposed the storage owner fault: the shared + `song-trial-results.cc` single-file MessagePack store grew to ~2.13 GB and + failed at the end of pass 3 with `System.OutOfMemoryException` while trying + to allocate another whole-store buffer. The issue was not the song agents; + it was the backing store treating cold old records as part of every hot write. +- CultLib now has `DirectoryMessagePackBackingStore`. In directory mode, the + `.cc` path is a small schema manifest and each record is an individual + MessagePack page under `.records/`. `PushAll` rewrites changed/deleted + records and the manifest only; cold record files keep their timestamps and + are not serialized into one giant byte array. +- `CultCacheOpenOptions.UseDirectoryStore` selects the paged store, and + `CultCacheMessagePack.Create/OpenAsync` automatically reopen a paged store + when the `.records` directory exists. AquaSynth trial-result stores now + always use directory mode through `IpaTrialResultCultCacheStore`, so future + song/IPA curriculum runs stop dragging the whole archive through RAM on each + upsert. +- Verification: CultLib `BackingStoreTests` pass 14/14, including a cold-record + timestamp test; AquaSynth `dotnet build tools/IpaTrialWorker/IpaTrialWorker.csproj + --no-restore` passes; AquaSynth `--filter CultCache` tests pass 7/7.