Track OBS stem publication readiness

This commit is contained in:
Meta Vens 2026-05-27 20:34:09 +01:00
commit 0c39921f96
7 changed files with 172 additions and 5 deletions

View file

@ -154,8 +154,11 @@ a named invariant that the native runtime cannot protect yet.
using the bank-assigned Faust `sourceN` slot as channel authority. Fensalir
now publishes processed DSP output as `AquariumAudioStemFrame` records through
an engine-owned `IAquariumAudioStemBus`, and Mimir declares deterministic
aligned-source stem names with the actuator program. OBS-facing publication is
still the next cut before those stems become separately mixable program
aligned-source stem names with the actuator program.
`MimirObsStemPublicationState` consumes the Fensalir stem bus without copying
audio and validates ready/missing/unconfigured lanes against the
alignment-actuator OBS stem profile. Actual OBS device/plugin routing remains
the next output cut before those stems become separately mixable program
outputs.
- `MimirRuntime` no longer submits the legacy direct `AquariumSplineFrame`
spectrum dashboard. Live spectrum visualization authority is the

View file

@ -315,8 +315,11 @@ path as channel authority. Fensalir can compile/host/process that persistent
Faust graph now. The engine now publishes processed output lanes as
`AquariumAudioStemFrame` records through `IAquariumAudioStemBus`, with Mimir's
actuator program declaring deterministic aligned-source stem names. OBS-facing
publication still has to consume those stems as separately mixable program
audio.
publication now has a first readiness surface:
`MimirObsStemPublicationState` consumes those frames without copying audio and
reports ready, missing, and unconfigured stems against the alignment actuator
stem-bus profile. Actual device/plugin routing for OBS still has to consume the
same validated stem stream as separately mixable program audio.
## Visual Fusion

View file

@ -74,6 +74,11 @@ if (args.Any(arg => string.Equals(arg, "--audio-actuator-bank-smoke", StringComp
return RunAudioActuatorBankSmoke();
}
if (args.Any(arg => string.Equals(arg, "--audio-stem-publication-smoke", StringComparison.OrdinalIgnoreCase)))
{
return RunAudioStemPublicationSmoke();
}
if (args.Any(arg => string.Equals(arg, "--complex-contour-tracker-self-test", StringComparison.OrdinalIgnoreCase)))
{
return RunComplexContourTrackerSelfTest(
@ -1882,6 +1887,36 @@ static int RunAudioActuatorBankSmoke()
: 1;
}
static int RunAudioStemPublicationSmoke()
{
var publication = new MimirObsStemPublicationState(MimirObsPublicationConfigurations.AlignmentActuatorStemBus);
publication.Consume(new AquariumAudioStemFrame(
MimirAlignmentActuatorProfile.SixSourceFaust.Id,
[
new AquariumAudioStemChannel(0, "aligned_source_0", "Aligned source 0", "mic-0", [0.1f, 0.2f]),
new AquariumAudioStemChannel(1, "aligned_source_1", "Aligned source 1", "mic-1", [0.3f, 0.4f])
],
FrameCount: 2,
SampleRate: 48_000,
Sequence: 9));
var snapshot = publication.Capture();
Console.WriteLine(
$"audio-stem-publication-smoke profile={snapshot.ConfigurationId} ready={snapshot.ReadyStems.Count} missing={snapshot.MissingStemIds.Count} unconfigured={snapshot.UnconfiguredStemIds.Count} seq={snapshot.LatestSequence}");
return string.Equals(snapshot.ConfigurationId, MimirObsPublicationConfigurations.AlignmentActuatorStemBus.Id, StringComparison.Ordinal) &&
snapshot.ReadyStems.Count == 2 &&
snapshot.MissingStemIds.Count == MimirAlignmentActuatorProfile.SixSourceFaust.SourceCount - 2 &&
snapshot.UnconfiguredStemIds.Count == 0 &&
snapshot.ReadyStems.Any(stem =>
string.Equals(stem.StemId, "aligned_source_0", StringComparison.Ordinal) &&
string.Equals(stem.SourceId, "mic-0", StringComparison.Ordinal) &&
stem.Configured) &&
snapshot.LatestSequence == 9
? 0
: 1;
}
static MimirAudioSynchronizationState NewSyncState(string sourceId, double delaySamples, double confidence) =>
new(
"loopback-scarlett-speakers",

View file

@ -40,6 +40,7 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec
private readonly MimirFensalirFieldLowering fieldLowering;
private readonly MimirAudioSpectrumAnalyzer spectrumAnalyzer;
private readonly MimirAlignmentActuatorBank audioActuatorBank = new();
private readonly MimirObsStemPublicationState obsStemPublication = new(MimirObsPublicationConfigurations.AlignmentActuatorStemBus);
private readonly IReadOnlyList<MimirStreamSourceFactory> sourceFactories;
private readonly AquariumUiDocument ui;
private readonly AquariumAudioDocument audio = new();
@ -73,6 +74,7 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec
private (Vector3 Min, Vector3 Max) lastSpectrumAabb = (Vector3.Zero, Vector3.One);
private IReadOnlyList<MimirAudioSynchronizationReport> lastAudioSynchronizationReports = [];
private IReadOnlyList<MimirAudioSpectrumSnapshot> lastAudioSpectra = [];
private MimirObsStemPublicationSnapshot lastObsStemPublication = MimirObsStemPublicationSnapshot.Empty;
private long spectrumHistorySequence;
private long audioActuatorFrameSequence;
private bool audioActuatorProgramQueued;
@ -654,6 +656,7 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec
runtimeSeconds += Math.Max(deltaSeconds, 0.0f);
lastPollCount = synchronization.PollSources();
UpdateObsStemPublication();
QueueCalibrationTimeline();
UpdateAudioSpectra();
UpdateAudioSynchronization();
@ -1036,6 +1039,21 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec
sequence));
}
private void UpdateObsStemPublication()
{
var consumed = false;
foreach (var stemFrame in runtimeServices.AudioStems.DrainPublishedFrames())
{
obsStemPublication.Consume(stemFrame);
consumed = true;
}
if (consumed)
{
lastObsStemPublication = obsStemPublication.Capture();
}
}
private void UpdateAudioSpectra()
{
if (runtimeSeconds < nextSpectrumSeconds)
@ -1092,6 +1110,12 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec
var states = synchronization.AudioSynchronizationStates;
Console.WriteLine(
$"mimir-sync-telemetry t={runtimeSeconds:0.00}s audioSync={audioSyncSettings.Mode} loopbackCount={loopback?.Count ?? 0} loopbackEdgeNs={loopback?.EdgeNs ?? 0} reports={lastAudioSynchronizationReports.Count} states={states.Count} analyzeMs={lastAudioSyncAnalysisMilliseconds:0.0} aligned={DescribeAlignedAudio()}");
if (lastObsStemPublication.ReadyStems.Count > 0 || lastObsStemPublication.MissingStemIds.Count > 0)
{
Console.WriteLine(
$"mimir-stem-telemetry profile={lastObsStemPublication.ConfigurationId} ready={lastObsStemPublication.ReadyStems.Count} missing={lastObsStemPublication.MissingStemIds.Count} unconfigured={lastObsStemPublication.UnconfiguredStemIds.Count} seq={lastObsStemPublication.LatestSequence}");
}
Console.WriteLine($"mimir-sync-buffers {DescribeAudioBuffers()}");
Console.WriteLine($"mimir-spectrum-frustum {DescribeSpectrumFrustum()}");
foreach (var spectrum in lastAudioSpectra.OrderBy(snapshot => snapshot.SourceId, StringComparer.Ordinal))

View file

@ -35,6 +35,82 @@ public sealed record MimirObsPublicationConfiguration(
double TargetPresentationDelaySeconds,
bool DiagnosticOnly);
public sealed record MimirObsPublishedAudioStem(
string StemId,
string DisplayName,
string SourceId,
int ChannelIndex,
int FrameCount,
int SampleRate,
long Sequence,
bool Configured);
public sealed record MimirObsStemPublicationSnapshot(
string ConfigurationId,
MimirObsAudioPublicationKind AudioKind,
IReadOnlyList<MimirObsPublishedAudioStem> ReadyStems,
IReadOnlyList<string> MissingStemIds,
IReadOnlyList<string> UnconfiguredStemIds,
long LatestSequence)
{
public static MimirObsStemPublicationSnapshot Empty { get; } = new(
"",
MimirObsAudioPublicationKind.FaustStemBus,
[],
[],
[],
0);
}
public sealed class MimirObsStemPublicationState(MimirObsPublicationConfiguration configuration)
{
private readonly Dictionary<string, MimirObsPublishedAudioStem> latestByStemId = new(StringComparer.Ordinal);
public void Consume(Aquarium.Engine.Audio.AquariumAudioStemFrame frame)
{
var configured = configuration.AudioStems.ToDictionary(stem => stem.Id, StringComparer.Ordinal);
foreach (var channel in frame.Channels)
{
var isConfigured = configured.TryGetValue(channel.StemId, out var configuredStem);
latestByStemId[channel.StemId] = new MimirObsPublishedAudioStem(
channel.StemId,
isConfigured ? configuredStem!.DisplayName : channel.DisplayName,
channel.SourceId,
channel.ChannelIndex,
frame.FrameCount,
frame.SampleRate,
frame.Sequence,
isConfigured);
}
}
public MimirObsStemPublicationSnapshot Capture()
{
var configuredIds = configuration.AudioStems
.Select(stem => stem.Id)
.ToHashSet(StringComparer.Ordinal);
var ready = latestByStemId.Values
.OrderBy(stem => stem.StemId, StringComparer.Ordinal)
.ToArray();
var missing = configuration.AudioStems
.Where(stem => !latestByStemId.ContainsKey(stem.Id))
.Select(stem => stem.Id)
.Order(StringComparer.Ordinal)
.ToArray();
var unconfigured = latestByStemId.Keys
.Where(stemId => !configuredIds.Contains(stemId))
.Order(StringComparer.Ordinal)
.ToArray();
return new MimirObsStemPublicationSnapshot(
configuration.Id,
configuration.AudioKind,
ready,
missing,
unconfigured,
ready.Length == 0 ? 0 : ready.Max(stem => stem.Sequence));
}
}
public static class MimirObsPublicationConfigurations
{
public static IReadOnlyList<MimirObsAudioStem> ProgramStems { get; } =
@ -48,6 +124,17 @@ public static class MimirObsPublicationConfigurations
new("spatial_bed", "Spatial bed", 4, UserMixable: true, CarriesTimingWitness: false, "Encoded volumetric ambience or ambisonic fold.")
];
public static IReadOnlyList<MimirObsAudioStem> AlignmentSourceStems { get; } =
Enumerable.Range(0, MimirAlignmentActuatorProfile.SixSourceFaust.SourceCount)
.Select(index => new MimirObsAudioStem(
$"aligned_source_{index}",
$"Aligned source {index}",
1,
UserMixable: true,
CarriesTimingWitness: false,
"Post-actuator source lane before final separation/spatial mixing."))
.ToArray();
public static MimirObsPublicationConfiguration NativeProgram { get; } = new(
"native-program-spout-faust",
"Production target: Fensalir publishes synchronized video, Faust publishes aligned stems.",
@ -58,6 +145,16 @@ public static class MimirObsPublicationConfigurations
TargetPresentationDelaySeconds: 5.0,
DiagnosticOnly: false);
public static MimirObsPublicationConfiguration AlignmentActuatorStemBus { get; } = new(
"alignment-actuator-stem-bus",
"Program-stage Faust stem bus: aligned source lanes before final mix/separation.",
MimirObsVideoPublicationKind.Spout2,
"Mimir Point Cloud",
MimirObsAudioPublicationKind.FaustStemBus,
AlignmentSourceStems,
TargetPresentationDelaySeconds: 5.0,
DiagnosticOnly: false);
public static MimirObsPublicationConfiguration TextureInteropProgram { get; } = NativeProgram with
{
Id = "native-program-d3d12-shared-texture",
@ -87,6 +184,7 @@ public static class MimirObsPublicationConfigurations
public static IReadOnlyList<MimirObsPublicationConfiguration> BuiltIn { get; } =
[
NativeProgram,
AlignmentActuatorStemBus,
TextureInteropProgram,
EveNativeProgram,
DiagnosticSrtBridge

View file

@ -188,3 +188,4 @@
{"ts": "2026-05-27T18:54:28+01:00", "type": "decision", "status": "ok", "note": "Mimir now declares faust/mimir_alignment_actuator.dsp as an AquariumStreamingDspProgram before queuing actuator controls, giving Fensalir both the persistent Faust DSP graph and the control frames. Live sample alignment still waits on the next device I/O/stem-routing cut."}
{"ts": "2026-05-27T20:18:50+01:00", "type": "verification", "status": "ok", "note": "MimirRuntime now queues sample-bearing AquariumStreamingAudioBlock lanes for actuator source buffers after publishing the control frame. Channel indices are derived from the bank-assigned Faust sourceN control path, so source disappearance does not reshuffle lanes. Mimir build passed; Fensalir tests prove declared blocks enter the streaming Faust DSP host."}
{"ts": "2026-05-27T20:28:43+01:00", "type": "verification", "status": "ok", "note": "Fensalir now publishes processed streaming Faust DSP output lanes as AquariumAudioStemFrame records through an engine-owned IAquariumAudioStemBus. Mimir declares deterministic aligned-source output stems when queuing faust/mimir_alignment_actuator.dsp, so processed actuator lanes have stable stem names before OBS-facing publication consumes them. Fensalir build and Aquarium.Engine.Tests passed; Mimir build and --audio-actuator-bank-smoke passed."}
{"ts": "2026-05-27T20:33:41+01:00", "type": "verification", "status": "ok", "note": "Mimir now has an OBS-facing stem publication state for the Fensalir Faust stem bus. It consumes AquariumAudioStemFrame records without copying audio, validates them against the alignment-actuator stem profile, reports ready/missing/unconfigured stems, and telemetry can print publication readiness. --audio-stem-publication-smoke passed; Mimir build passed."}

View file

@ -192,7 +192,10 @@ accepted_design:
path. Fensalir now publishes processed DSP output lanes as
`AquariumAudioStemFrame` records on an engine-owned audio stem bus, and
Mimir declares deterministic aligned-source stem names with the actuator
program; OBS-facing stem publication is the remaining program-audio cut. The synthetic
program. Mimir now consumes that bus through `MimirObsStemPublicationState`,
validating ready/missing/unconfigured lanes against the alignment actuator
stem-bus profile without copying audio; actual OBS device/plugin routing is
the remaining program-audio cut. The synthetic
codebook proof is `dotnet run --project
.\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --bioacoustic-self-test`.
`Mimir.BufferSmoke --render-chirp-bin-f32` plus