diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index ab57e93..75fa8e4 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -154,8 +154,11 @@ a named invariant that the native runtime cannot protect yet. using the bank-assigned Faust `sourceN` slot as channel authority. Fensalir now publishes processed DSP output as `AquariumAudioStemFrame` records through an engine-owned `IAquariumAudioStemBus`, and Mimir declares deterministic - aligned-source stem names with the actuator program. OBS-facing publication is - still the next cut before those stems become separately mixable program + aligned-source stem names with the actuator program. + `MimirObsStemPublicationState` consumes the Fensalir stem bus without copying + audio and validates ready/missing/unconfigured lanes against the + alignment-actuator OBS stem profile. Actual OBS device/plugin routing remains + the next output cut before those stems become separately mixable program outputs. - `MimirRuntime` no longer submits the legacy direct `AquariumSplineFrame` spectrum dashboard. Live spectrum visualization authority is the diff --git a/notes/current-system-map.md b/notes/current-system-map.md index 416868c..5a56075 100644 --- a/notes/current-system-map.md +++ b/notes/current-system-map.md @@ -315,8 +315,11 @@ path as channel authority. Fensalir can compile/host/process that persistent Faust graph now. The engine now publishes processed output lanes as `AquariumAudioStemFrame` records through `IAquariumAudioStemBus`, with Mimir's actuator program declaring deterministic aligned-source stem names. OBS-facing -publication still has to consume those stems as separately mixable program -audio. +publication now has a first readiness surface: +`MimirObsStemPublicationState` consumes those frames without copying audio and +reports ready, missing, and unconfigured stems against the alignment actuator +stem-bus profile. Actual device/plugin routing for OBS still has to consume the +same validated stem stream as separately mixable program audio. ## Visual Fusion diff --git a/src/Mimir.BufferSmoke/Program.cs b/src/Mimir.BufferSmoke/Program.cs index 00ceaca..922ab99 100644 --- a/src/Mimir.BufferSmoke/Program.cs +++ b/src/Mimir.BufferSmoke/Program.cs @@ -74,6 +74,11 @@ if (args.Any(arg => string.Equals(arg, "--audio-actuator-bank-smoke", StringComp return RunAudioActuatorBankSmoke(); } +if (args.Any(arg => string.Equals(arg, "--audio-stem-publication-smoke", StringComparison.OrdinalIgnoreCase))) +{ + return RunAudioStemPublicationSmoke(); +} + if (args.Any(arg => string.Equals(arg, "--complex-contour-tracker-self-test", StringComparison.OrdinalIgnoreCase))) { return RunComplexContourTrackerSelfTest( @@ -1882,6 +1887,36 @@ static int RunAudioActuatorBankSmoke() : 1; } +static int RunAudioStemPublicationSmoke() +{ + var publication = new MimirObsStemPublicationState(MimirObsPublicationConfigurations.AlignmentActuatorStemBus); + publication.Consume(new AquariumAudioStemFrame( + MimirAlignmentActuatorProfile.SixSourceFaust.Id, + [ + new AquariumAudioStemChannel(0, "aligned_source_0", "Aligned source 0", "mic-0", [0.1f, 0.2f]), + new AquariumAudioStemChannel(1, "aligned_source_1", "Aligned source 1", "mic-1", [0.3f, 0.4f]) + ], + FrameCount: 2, + SampleRate: 48_000, + Sequence: 9)); + + var snapshot = publication.Capture(); + Console.WriteLine( + $"audio-stem-publication-smoke profile={snapshot.ConfigurationId} ready={snapshot.ReadyStems.Count} missing={snapshot.MissingStemIds.Count} unconfigured={snapshot.UnconfiguredStemIds.Count} seq={snapshot.LatestSequence}"); + + return string.Equals(snapshot.ConfigurationId, MimirObsPublicationConfigurations.AlignmentActuatorStemBus.Id, StringComparison.Ordinal) && + snapshot.ReadyStems.Count == 2 && + snapshot.MissingStemIds.Count == MimirAlignmentActuatorProfile.SixSourceFaust.SourceCount - 2 && + snapshot.UnconfiguredStemIds.Count == 0 && + snapshot.ReadyStems.Any(stem => + string.Equals(stem.StemId, "aligned_source_0", StringComparison.Ordinal) && + string.Equals(stem.SourceId, "mic-0", StringComparison.Ordinal) && + stem.Configured) && + snapshot.LatestSequence == 9 + ? 0 + : 1; +} + static MimirAudioSynchronizationState NewSyncState(string sourceId, double delaySamples, double confidence) => new( "loopback-scarlett-speakers", diff --git a/src/Mimir.Runtime/MimirRuntime.cs b/src/Mimir.Runtime/MimirRuntime.cs index 77ee36e..773cc5b 100644 --- a/src/Mimir.Runtime/MimirRuntime.cs +++ b/src/Mimir.Runtime/MimirRuntime.cs @@ -40,6 +40,7 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec private readonly MimirFensalirFieldLowering fieldLowering; private readonly MimirAudioSpectrumAnalyzer spectrumAnalyzer; private readonly MimirAlignmentActuatorBank audioActuatorBank = new(); + private readonly MimirObsStemPublicationState obsStemPublication = new(MimirObsPublicationConfigurations.AlignmentActuatorStemBus); private readonly IReadOnlyList sourceFactories; private readonly AquariumUiDocument ui; private readonly AquariumAudioDocument audio = new(); @@ -73,6 +74,7 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec private (Vector3 Min, Vector3 Max) lastSpectrumAabb = (Vector3.Zero, Vector3.One); private IReadOnlyList lastAudioSynchronizationReports = []; private IReadOnlyList lastAudioSpectra = []; + private MimirObsStemPublicationSnapshot lastObsStemPublication = MimirObsStemPublicationSnapshot.Empty; private long spectrumHistorySequence; private long audioActuatorFrameSequence; private bool audioActuatorProgramQueued; @@ -654,6 +656,7 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec runtimeSeconds += Math.Max(deltaSeconds, 0.0f); lastPollCount = synchronization.PollSources(); + UpdateObsStemPublication(); QueueCalibrationTimeline(); UpdateAudioSpectra(); UpdateAudioSynchronization(); @@ -1036,6 +1039,21 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec sequence)); } + private void UpdateObsStemPublication() + { + var consumed = false; + foreach (var stemFrame in runtimeServices.AudioStems.DrainPublishedFrames()) + { + obsStemPublication.Consume(stemFrame); + consumed = true; + } + + if (consumed) + { + lastObsStemPublication = obsStemPublication.Capture(); + } + } + private void UpdateAudioSpectra() { if (runtimeSeconds < nextSpectrumSeconds) @@ -1092,6 +1110,12 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec var states = synchronization.AudioSynchronizationStates; Console.WriteLine( $"mimir-sync-telemetry t={runtimeSeconds:0.00}s audioSync={audioSyncSettings.Mode} loopbackCount={loopback?.Count ?? 0} loopbackEdgeNs={loopback?.EdgeNs ?? 0} reports={lastAudioSynchronizationReports.Count} states={states.Count} analyzeMs={lastAudioSyncAnalysisMilliseconds:0.0} aligned={DescribeAlignedAudio()}"); + if (lastObsStemPublication.ReadyStems.Count > 0 || lastObsStemPublication.MissingStemIds.Count > 0) + { + Console.WriteLine( + $"mimir-stem-telemetry profile={lastObsStemPublication.ConfigurationId} ready={lastObsStemPublication.ReadyStems.Count} missing={lastObsStemPublication.MissingStemIds.Count} unconfigured={lastObsStemPublication.UnconfiguredStemIds.Count} seq={lastObsStemPublication.LatestSequence}"); + } + Console.WriteLine($"mimir-sync-buffers {DescribeAudioBuffers()}"); Console.WriteLine($"mimir-spectrum-frustum {DescribeSpectrumFrustum()}"); foreach (var spectrum in lastAudioSpectra.OrderBy(snapshot => snapshot.SourceId, StringComparer.Ordinal)) diff --git a/src/Mimir.Runtime/Synchronization/MimirObsPublicationConfiguration.cs b/src/Mimir.Runtime/Synchronization/MimirObsPublicationConfiguration.cs index f18ad49..f4b632e 100644 --- a/src/Mimir.Runtime/Synchronization/MimirObsPublicationConfiguration.cs +++ b/src/Mimir.Runtime/Synchronization/MimirObsPublicationConfiguration.cs @@ -35,6 +35,82 @@ public sealed record MimirObsPublicationConfiguration( double TargetPresentationDelaySeconds, bool DiagnosticOnly); +public sealed record MimirObsPublishedAudioStem( + string StemId, + string DisplayName, + string SourceId, + int ChannelIndex, + int FrameCount, + int SampleRate, + long Sequence, + bool Configured); + +public sealed record MimirObsStemPublicationSnapshot( + string ConfigurationId, + MimirObsAudioPublicationKind AudioKind, + IReadOnlyList ReadyStems, + IReadOnlyList MissingStemIds, + IReadOnlyList UnconfiguredStemIds, + long LatestSequence) +{ + public static MimirObsStemPublicationSnapshot Empty { get; } = new( + "", + MimirObsAudioPublicationKind.FaustStemBus, + [], + [], + [], + 0); +} + +public sealed class MimirObsStemPublicationState(MimirObsPublicationConfiguration configuration) +{ + private readonly Dictionary latestByStemId = new(StringComparer.Ordinal); + + public void Consume(Aquarium.Engine.Audio.AquariumAudioStemFrame frame) + { + var configured = configuration.AudioStems.ToDictionary(stem => stem.Id, StringComparer.Ordinal); + foreach (var channel in frame.Channels) + { + var isConfigured = configured.TryGetValue(channel.StemId, out var configuredStem); + latestByStemId[channel.StemId] = new MimirObsPublishedAudioStem( + channel.StemId, + isConfigured ? configuredStem!.DisplayName : channel.DisplayName, + channel.SourceId, + channel.ChannelIndex, + frame.FrameCount, + frame.SampleRate, + frame.Sequence, + isConfigured); + } + } + + public MimirObsStemPublicationSnapshot Capture() + { + var configuredIds = configuration.AudioStems + .Select(stem => stem.Id) + .ToHashSet(StringComparer.Ordinal); + var ready = latestByStemId.Values + .OrderBy(stem => stem.StemId, StringComparer.Ordinal) + .ToArray(); + var missing = configuration.AudioStems + .Where(stem => !latestByStemId.ContainsKey(stem.Id)) + .Select(stem => stem.Id) + .Order(StringComparer.Ordinal) + .ToArray(); + var unconfigured = latestByStemId.Keys + .Where(stemId => !configuredIds.Contains(stemId)) + .Order(StringComparer.Ordinal) + .ToArray(); + return new MimirObsStemPublicationSnapshot( + configuration.Id, + configuration.AudioKind, + ready, + missing, + unconfigured, + ready.Length == 0 ? 0 : ready.Max(stem => stem.Sequence)); + } +} + public static class MimirObsPublicationConfigurations { public static IReadOnlyList ProgramStems { get; } = @@ -48,6 +124,17 @@ public static class MimirObsPublicationConfigurations new("spatial_bed", "Spatial bed", 4, UserMixable: true, CarriesTimingWitness: false, "Encoded volumetric ambience or ambisonic fold.") ]; + public static IReadOnlyList AlignmentSourceStems { get; } = + Enumerable.Range(0, MimirAlignmentActuatorProfile.SixSourceFaust.SourceCount) + .Select(index => new MimirObsAudioStem( + $"aligned_source_{index}", + $"Aligned source {index}", + 1, + UserMixable: true, + CarriesTimingWitness: false, + "Post-actuator source lane before final separation/spatial mixing.")) + .ToArray(); + public static MimirObsPublicationConfiguration NativeProgram { get; } = new( "native-program-spout-faust", "Production target: Fensalir publishes synchronized video, Faust publishes aligned stems.", @@ -58,6 +145,16 @@ public static class MimirObsPublicationConfigurations TargetPresentationDelaySeconds: 5.0, DiagnosticOnly: false); + public static MimirObsPublicationConfiguration AlignmentActuatorStemBus { get; } = new( + "alignment-actuator-stem-bus", + "Program-stage Faust stem bus: aligned source lanes before final mix/separation.", + MimirObsVideoPublicationKind.Spout2, + "Mimir Point Cloud", + MimirObsAudioPublicationKind.FaustStemBus, + AlignmentSourceStems, + TargetPresentationDelaySeconds: 5.0, + DiagnosticOnly: false); + public static MimirObsPublicationConfiguration TextureInteropProgram { get; } = NativeProgram with { Id = "native-program-d3d12-shared-texture", @@ -87,6 +184,7 @@ public static class MimirObsPublicationConfigurations public static IReadOnlyList BuiltIn { get; } = [ NativeProgram, + AlignmentActuatorStemBus, TextureInteropProgram, EveNativeProgram, DiagnosticSrtBridge diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 423336d..fe9bb6d 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -188,3 +188,4 @@ {"ts": "2026-05-27T18:54:28+01:00", "type": "decision", "status": "ok", "note": "Mimir now declares faust/mimir_alignment_actuator.dsp as an AquariumStreamingDspProgram before queuing actuator controls, giving Fensalir both the persistent Faust DSP graph and the control frames. Live sample alignment still waits on the next device I/O/stem-routing cut."} {"ts": "2026-05-27T20:18:50+01:00", "type": "verification", "status": "ok", "note": "MimirRuntime now queues sample-bearing AquariumStreamingAudioBlock lanes for actuator source buffers after publishing the control frame. Channel indices are derived from the bank-assigned Faust sourceN control path, so source disappearance does not reshuffle lanes. Mimir build passed; Fensalir tests prove declared blocks enter the streaming Faust DSP host."} {"ts": "2026-05-27T20:28:43+01:00", "type": "verification", "status": "ok", "note": "Fensalir now publishes processed streaming Faust DSP output lanes as AquariumAudioStemFrame records through an engine-owned IAquariumAudioStemBus. Mimir declares deterministic aligned-source output stems when queuing faust/mimir_alignment_actuator.dsp, so processed actuator lanes have stable stem names before OBS-facing publication consumes them. Fensalir build and Aquarium.Engine.Tests passed; Mimir build and --audio-actuator-bank-smoke passed."} +{"ts": "2026-05-27T20:33:41+01:00", "type": "verification", "status": "ok", "note": "Mimir now has an OBS-facing stem publication state for the Fensalir Faust stem bus. It consumes AquariumAudioStemFrame records without copying audio, validates them against the alignment-actuator stem profile, reports ready/missing/unconfigured stems, and telemetry can print publication readiness. --audio-stem-publication-smoke passed; Mimir build passed."} diff --git a/state/map.yaml b/state/map.yaml index 7bd40ef..6eafc7e 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -192,7 +192,10 @@ accepted_design: path. Fensalir now publishes processed DSP output lanes as `AquariumAudioStemFrame` records on an engine-owned audio stem bus, and Mimir declares deterministic aligned-source stem names with the actuator - program; OBS-facing stem publication is the remaining program-audio cut. The synthetic + program. Mimir now consumes that bus through `MimirObsStemPublicationState`, + validating ready/missing/unconfigured lanes against the alignment actuator + stem-bus profile without copying audio; actual OBS device/plugin routing is + the remaining program-audio cut. The synthetic codebook proof is `dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --bioacoustic-self-test`. `Mimir.BufferSmoke --render-chirp-bin-f32` plus