diff --git a/docs/fensalir-rendering-rebuild-migration.md b/docs/fensalir-rendering-rebuild-migration.md index fd0168e..da81be9 100644 --- a/docs/fensalir-rendering-rebuild-migration.md +++ b/docs/fensalir-rendering-rebuild-migration.md @@ -228,6 +228,9 @@ TubeSpline metadata whose claim is not Tube-encoded or whose resource differs from the claim payload. Mimir's typed surface-intent lowering now emits the matching TubeSpline metadata for audio spectrum/waveform Tube claims. That is engine ownership, not a reason for Mimir to create a parallel renderer. +Mimir also uploads normalized spectral frames as Float32 data into a +Fensalir-owned structured buffer resource, so the TubeField path consumes live +GPU buffer contents rather than an empty declaration. ### 3. Calibration Constraint diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 14b0dcc..42074f6 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -137,8 +137,11 @@ a named invariant that the native runtime cannot protect yet. spectrum dashboard. Live spectrum visualization authority is the `AquariumFieldEvidenceFrame`: Mimir declares the rolling resource, emits a Tube claim and `AquariumFieldTubeSplineLowering`, and Fensalir plans/expands - the TubeField generated mesh. The old `AquariumBufferFieldFrame` / - `ReservoirSplats` dashboard path is also silent. + the TubeField generated mesh. Mimir now normalizes current spectral frames + into a row-major Float32 matrix, attaches that as a field resource upload, and + Fensalir copies it into the resolved GPU structured buffer before TubeField + compute reads. The old `AquariumBufferFieldFrame` / `ReservoirSplats` + dashboard path is also silent. - `MimirFensalirFieldLowering` now emits `AquariumFieldResourceDeclaration` rows for live native/GPU payload views. Observation claims reference `mimir:resource:*` keys, and Fensalir validation/planning can reject or defer diff --git a/notes/current-system-map.md b/notes/current-system-map.md index d5d4657..942bed6 100644 --- a/notes/current-system-map.md +++ b/notes/current-system-map.md @@ -377,7 +377,10 @@ submits direct `AquariumSplineFrame` trails for the spectrum dashboard and does not submit `AquariumBufferFieldFrame` / fractal `ReservoirSplats` for it either. Mimir declares the rolling resource, publishes a Tube claim plus `AquariumFieldTubeSplineLowering`, and Fensalir owns planning, generated mesh -expansion, and TubeField material rendering. +expansion, and TubeField material rendering. The spectrum resource is not just +a handle: Mimir uploads the current spectral amplitudes as row-major Float32 +data into a Fensalir-owned GPU structured buffer before TubeField compute reads +it. ## Known Risks diff --git a/src/Mimir.Runtime/MimirRuntime.cs b/src/Mimir.Runtime/MimirRuntime.cs index 934477b..473a94c 100644 --- a/src/Mimir.Runtime/MimirRuntime.cs +++ b/src/Mimir.Runtime/MimirRuntime.cs @@ -190,10 +190,16 @@ public sealed class MimirRuntime : IAquariumRuntime continue; } + if (window.SourceKind == MimirStreamKind.Audio) + { + continue; + } + intents.Add(BuildSurfaceIntent(window, observation)); } - return fieldLowering.BuildFieldEvidenceFrame(windows, observations, constraints, intents); + var frame = fieldLowering.BuildFieldEvidenceFrame(windows, observations, constraints, intents); + return AddSpectrumFieldEvidence(frame); } private static MimirSurfaceIntent BuildSurfaceIntent(MimirRollingStreamWindow window, MimirObservation observation) @@ -214,6 +220,172 @@ public sealed class MimirRuntime : IAquariumRuntime return MimirFensalirBridgeMapper.MapDefaultSurfaceIntent(window, observation); } + private AquariumFieldEvidenceFrame AddSpectrumFieldEvidence(AquariumFieldEvidenceFrame frame) + { + if (lastAudioSpectra.Count == 0) + { + return frame; + } + + var spectra = lastAudioSpectra + .Where(static spectrum => spectrum.BandDecibels.Count >= 2) + .OrderBy(static spectrum => spectrum.SourceId, StringComparer.Ordinal) + .ToArray(); + if (spectra.Length == 0) + { + return frame; + } + + var width = Math.Max(2, spectra.Max(static spectrum => spectrum.BandDecibels.Count)); + var height = spectra.Length; + var samples = new float[width * height]; + for (var row = 0; row < spectra.Length; row++) + { + WriteNormalizedSpectrum(spectra[row], samples.AsSpan(row * width, width)); + } + + const string resourceKey = "mimir:resource:spectrum:field-upload"; + const string domainKey = "mimir:domain:spectrum:field-upload"; + const string claimKey = "intent:mimir:spectrum:field-upload"; + var version = unchecked((ulong)Math.Max(0, spectrumHistorySequence)); + var confidence = (float)Math.Clamp(spectra.Average(static spectrum => spectrum.Rms > 0.0 ? 1.0 : 0.65), 0.0, 1.0); + var support = new AquariumFieldSupport( + Center: Vector3.Zero, + Radius: new Vector3(5.0f, SpectrumAmplitudeHeight, Math.Max(0.1f, height * 0.1f)), + LocalFrame: Matrix4x4.Identity, + ConservativeRadius: 5.0f, + ProjectedError: 0.0f, + Curvature: 0.0f, + TemporalUncertainty: (float)Math.Max(0.0, spectrumUpdateIntervalSeconds)); + var proposal = new AquariumFieldProposalPolicy( + AquariumFieldProposalKind.DebugIntent, + SourcePdf: 1.0f, + TargetContribution: confidence, + RepresentedCandidateCount: height, + Seed: 0x5EC7_0001u); + var resource = new AquariumFieldResourceDeclaration( + ResourceKey: resourceKey, + Kind: AquariumFieldResourceKind.StructuredBuffer, + Residency: AquariumFieldResourceResidency.GpuResident, + Access: AquariumFieldShaderAccess.ShaderResource, + Format: "Float32", + Width: width, + Height: height, + DepthOrCount: samples.Length, + StrideBytes: sizeof(float), + ValidFromNs: 0, + ValidUntilNs: 0, + Version: version, + NativeHandle: IntPtr.Zero, + NativeHandleKind: "fensalir-owned-spectrum-upload"); + var claim = new AquariumFieldClaim( + ClaimKey: claimKey, + DomainKey: domainKey, + ProducerKey: "Mimir.Runtime", + Layer: AquariumFieldLayer.Form, + Encoding: AquariumFieldEncoding.Tube, + Support: support, + Proposal: proposal, + PayloadHandle: resourceKey, + ObservedTimeNs: 0, + Confidence: confidence); + var axisStepX = width > 1 ? 10.0f / (width - 1) : 10.0f; + + return new AquariumFieldEvidenceFrame + { + Domains = + [ + .. frame.Domains, + new AquariumFieldDomain( + domainKey, + "", + AquariumFieldDomainKind.RollingBuffer, + Matrix4x4.Identity, + Matrix4x4.Identity, + new Vector3(-5.0f, 0.0f, 0.0f), + new Vector3(5.0f, SpectrumAmplitudeHeight, Math.Max(0.1f, height * 0.1f)), + Vector3.Zero, + "Mimir.Runtime"), + ], + Claims = [.. frame.Claims, claim], + Candidates = + [ + .. frame.Candidates, + new AquariumFieldCandidate( + $"{claimKey}:candidate", + claimKey, + AquariumFieldLayer.Form, + AquariumFieldEncoding.Tube, + proposal, + AquariumFieldGuide.Valid(confidence, spectrumUpdateIntervalSeconds)), + ], + BackendPackets = frame.BackendPackets, + Resources = [.. frame.Resources, resource], + ResourceUploads = + [ + .. frame.ResourceUploads, + new AquariumFieldResourceUpload + { + ResourceKey = resourceKey, + Version = version, + Float32Data = samples, + }, + ], + TubeSplineLowerings = + [ + .. frame.TubeSplineLowerings, + new AquariumFieldTubeSplineLowering( + LoweringKey: "tube-spline:mimir:spectrum:field-upload", + ClaimKey: claimKey, + ResourceKey: resourceKey, + Width: width, + Height: height, + StrideBytes: sizeof(float), + FirstColumn: 0, + ColumnCount: height, + ColumnStride: 1, + RollingModulo: height, + RollingOffset: 0, + Origin: new Vector3(-5.0f, 0.0f, 0.0f), + AxisStep: new Vector3(axisStepX, 0.0f, 0.0f), + ColumnStep: new Vector3(0.0f, 0.0f, 0.1f), + AmplitudePower: 2.0f, + AmplitudeScale: SpectrumAmplitudeHeight, + NormalizeMin: 0.0f, + NormalizeMax: 1.0f, + BaseRadius: 0.012f, + RadiusScale: 0.030f, + Alpha: 0.92f, + Feather: 0.20f, + RampTexturePath: "", + RampResourceKey: "", + EmissionScale: 10.0f, + CatmullRomSubdivisions: 4).Normalized(), + ], + AccumulationWindowSeconds = frame.AccumulationWindowSeconds, + PresentationDelaySeconds = frame.PresentationDelaySeconds, + }; + } + + private static void WriteNormalizedSpectrum(MimirAudioSpectrumSnapshot spectrum, Span destination) + { + destination.Clear(); + var bands = spectrum.BandDecibels; + if (bands.Count == 0) + { + return; + } + + var floor = Math.Min(spectrum.NoiseFloorDb, bands.Min()); + var ceiling = Math.Max(-24.0, bands.Max()); + var span = Math.Max(18.0, ceiling - floor); + var count = Math.Min(destination.Length, bands.Count); + for (var index = 0; index < count; index++) + { + destination[index] = (float)Math.Clamp((bands[index] - floor) / span, 0.0, 1.0); + } + } + private static Vector3 SpectrumCameraPosition( int spectrumCount, int windowCount, diff --git a/state/evidence.jsonl b/state/evidence.jsonl index f954704..28c6129 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -157,3 +157,4 @@ {"ts": "2026-05-26T22:58:00+01:00", "type": "implementation", "status": "ok", "note": "Fensalir validation now rejects TubeSpline lowering metadata if the referenced claim is not Tube-encoded or if the lowering resource differs from the claim payload resource. TubeField split authority is caught at validation, before renderer packet gating. Fensalir build/tests/headless reload passed."} {"ts": "2026-05-26T23:03:00+01:00", "type": "implementation", "status": "ok", "note": "Mimir's typed Fensalir surface-intent lowering now emits AquariumFieldTubeSplineLowering metadata for audio spectrum/waveform Tube claims. The hardware-free field evidence smoke now proves one resource, three claims, one TubeSpline lowering, one planned TubeField packet, and two deferred non-backend claims; the DSL smoke still proves resource-bound TubeSpline lowering with ramp binding."} {"ts": "2026-05-26T23:18:00+01:00", "type": "correction", "status": "ok", "note": "Silenced the legacy direct spectrum spline path. MimirRuntime now submits AquariumSplineFrame.Empty and the old BuildSpectrumSplineFrame/direct spline styling helpers were deleted, so live spectrum visualization authority is FieldEvidence TubeField only. Mimir build and both Fensalir field smokes passed."} +{"ts": "2026-05-26T23:36:00+01:00", "type": "implementation", "status": "ok", "note": "Mimir now feeds spectral frames into the GPU buffer path. Current audio spectra are normalized into a row-major Float32 matrix, declared as mimir:resource:spectrum:field-upload, attached as a field ResourceUpload, and consumed by the TubeField claim/lowering. Fensalir now uploads ResourceUpload Float32 data into resolved structured buffers before TubeField compute runs. Mimir build, Fensalir build/tests/headless reload, and both field smokes passed."} diff --git a/state/map.yaml b/state/map.yaml index e5cddaf..ccbf330 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -352,7 +352,10 @@ accepted_design: generated mesh path is fed by real Mimir evidence rather than only the DSL smoke. The legacy direct `AquariumSplineFrame` spectrum dashboard is silent: `MimirRuntime` submits an empty spline frame and the old direct spectrum - spline builder has been deleted. + spline builder has been deleted. Live spectrum samples are now normalized + into a row-major Float32 matrix, declared as + `mimir:resource:spectrum:field-upload`, uploaded into a Fensalir-owned + GPU-resident structured buffer, and consumed by the TubeField lowering. - `config/perfect-machine.example.json` declares live owners, sensors, reservoir views, and runtime policy. - `src/Mimir.Runtime/Synchronization` now includes the Perfect Machine module