Track OBS stem publication readiness
This commit is contained in:
parent
51c70f0b49
commit
0c39921f96
7 changed files with 172 additions and 5 deletions
|
|
@ -154,8 +154,11 @@ a named invariant that the native runtime cannot protect yet.
|
|||
using the bank-assigned Faust `sourceN` slot as channel authority. Fensalir
|
||||
now publishes processed DSP output as `AquariumAudioStemFrame` records through
|
||||
an engine-owned `IAquariumAudioStemBus`, and Mimir declares deterministic
|
||||
aligned-source stem names with the actuator program. OBS-facing publication is
|
||||
still the next cut before those stems become separately mixable program
|
||||
aligned-source stem names with the actuator program.
|
||||
`MimirObsStemPublicationState` consumes the Fensalir stem bus without copying
|
||||
audio and validates ready/missing/unconfigured lanes against the
|
||||
alignment-actuator OBS stem profile. Actual OBS device/plugin routing remains
|
||||
the next output cut before those stems become separately mixable program
|
||||
outputs.
|
||||
- `MimirRuntime` no longer submits the legacy direct `AquariumSplineFrame`
|
||||
spectrum dashboard. Live spectrum visualization authority is the
|
||||
|
|
|
|||
|
|
@ -315,8 +315,11 @@ path as channel authority. Fensalir can compile/host/process that persistent
|
|||
Faust graph now. The engine now publishes processed output lanes as
|
||||
`AquariumAudioStemFrame` records through `IAquariumAudioStemBus`, with Mimir's
|
||||
actuator program declaring deterministic aligned-source stem names. OBS-facing
|
||||
publication still has to consume those stems as separately mixable program
|
||||
audio.
|
||||
publication now has a first readiness surface:
|
||||
`MimirObsStemPublicationState` consumes those frames without copying audio and
|
||||
reports ready, missing, and unconfigured stems against the alignment actuator
|
||||
stem-bus profile. Actual device/plugin routing for OBS still has to consume the
|
||||
same validated stem stream as separately mixable program audio.
|
||||
|
||||
## Visual Fusion
|
||||
|
||||
|
|
|
|||
|
|
@ -74,6 +74,11 @@ if (args.Any(arg => string.Equals(arg, "--audio-actuator-bank-smoke", StringComp
|
|||
return RunAudioActuatorBankSmoke();
|
||||
}
|
||||
|
||||
if (args.Any(arg => string.Equals(arg, "--audio-stem-publication-smoke", StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
return RunAudioStemPublicationSmoke();
|
||||
}
|
||||
|
||||
if (args.Any(arg => string.Equals(arg, "--complex-contour-tracker-self-test", StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
return RunComplexContourTrackerSelfTest(
|
||||
|
|
@ -1882,6 +1887,36 @@ static int RunAudioActuatorBankSmoke()
|
|||
: 1;
|
||||
}
|
||||
|
||||
static int RunAudioStemPublicationSmoke()
|
||||
{
|
||||
var publication = new MimirObsStemPublicationState(MimirObsPublicationConfigurations.AlignmentActuatorStemBus);
|
||||
publication.Consume(new AquariumAudioStemFrame(
|
||||
MimirAlignmentActuatorProfile.SixSourceFaust.Id,
|
||||
[
|
||||
new AquariumAudioStemChannel(0, "aligned_source_0", "Aligned source 0", "mic-0", [0.1f, 0.2f]),
|
||||
new AquariumAudioStemChannel(1, "aligned_source_1", "Aligned source 1", "mic-1", [0.3f, 0.4f])
|
||||
],
|
||||
FrameCount: 2,
|
||||
SampleRate: 48_000,
|
||||
Sequence: 9));
|
||||
|
||||
var snapshot = publication.Capture();
|
||||
Console.WriteLine(
|
||||
$"audio-stem-publication-smoke profile={snapshot.ConfigurationId} ready={snapshot.ReadyStems.Count} missing={snapshot.MissingStemIds.Count} unconfigured={snapshot.UnconfiguredStemIds.Count} seq={snapshot.LatestSequence}");
|
||||
|
||||
return string.Equals(snapshot.ConfigurationId, MimirObsPublicationConfigurations.AlignmentActuatorStemBus.Id, StringComparison.Ordinal) &&
|
||||
snapshot.ReadyStems.Count == 2 &&
|
||||
snapshot.MissingStemIds.Count == MimirAlignmentActuatorProfile.SixSourceFaust.SourceCount - 2 &&
|
||||
snapshot.UnconfiguredStemIds.Count == 0 &&
|
||||
snapshot.ReadyStems.Any(stem =>
|
||||
string.Equals(stem.StemId, "aligned_source_0", StringComparison.Ordinal) &&
|
||||
string.Equals(stem.SourceId, "mic-0", StringComparison.Ordinal) &&
|
||||
stem.Configured) &&
|
||||
snapshot.LatestSequence == 9
|
||||
? 0
|
||||
: 1;
|
||||
}
|
||||
|
||||
static MimirAudioSynchronizationState NewSyncState(string sourceId, double delaySamples, double confidence) =>
|
||||
new(
|
||||
"loopback-scarlett-speakers",
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec
|
|||
private readonly MimirFensalirFieldLowering fieldLowering;
|
||||
private readonly MimirAudioSpectrumAnalyzer spectrumAnalyzer;
|
||||
private readonly MimirAlignmentActuatorBank audioActuatorBank = new();
|
||||
private readonly MimirObsStemPublicationState obsStemPublication = new(MimirObsPublicationConfigurations.AlignmentActuatorStemBus);
|
||||
private readonly IReadOnlyList<MimirStreamSourceFactory> sourceFactories;
|
||||
private readonly AquariumUiDocument ui;
|
||||
private readonly AquariumAudioDocument audio = new();
|
||||
|
|
@ -73,6 +74,7 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec
|
|||
private (Vector3 Min, Vector3 Max) lastSpectrumAabb = (Vector3.Zero, Vector3.One);
|
||||
private IReadOnlyList<MimirAudioSynchronizationReport> lastAudioSynchronizationReports = [];
|
||||
private IReadOnlyList<MimirAudioSpectrumSnapshot> lastAudioSpectra = [];
|
||||
private MimirObsStemPublicationSnapshot lastObsStemPublication = MimirObsStemPublicationSnapshot.Empty;
|
||||
private long spectrumHistorySequence;
|
||||
private long audioActuatorFrameSequence;
|
||||
private bool audioActuatorProgramQueued;
|
||||
|
|
@ -654,6 +656,7 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec
|
|||
|
||||
runtimeSeconds += Math.Max(deltaSeconds, 0.0f);
|
||||
lastPollCount = synchronization.PollSources();
|
||||
UpdateObsStemPublication();
|
||||
QueueCalibrationTimeline();
|
||||
UpdateAudioSpectra();
|
||||
UpdateAudioSynchronization();
|
||||
|
|
@ -1036,6 +1039,21 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec
|
|||
sequence));
|
||||
}
|
||||
|
||||
private void UpdateObsStemPublication()
|
||||
{
|
||||
var consumed = false;
|
||||
foreach (var stemFrame in runtimeServices.AudioStems.DrainPublishedFrames())
|
||||
{
|
||||
obsStemPublication.Consume(stemFrame);
|
||||
consumed = true;
|
||||
}
|
||||
|
||||
if (consumed)
|
||||
{
|
||||
lastObsStemPublication = obsStemPublication.Capture();
|
||||
}
|
||||
}
|
||||
|
||||
private void UpdateAudioSpectra()
|
||||
{
|
||||
if (runtimeSeconds < nextSpectrumSeconds)
|
||||
|
|
@ -1092,6 +1110,12 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec
|
|||
var states = synchronization.AudioSynchronizationStates;
|
||||
Console.WriteLine(
|
||||
$"mimir-sync-telemetry t={runtimeSeconds:0.00}s audioSync={audioSyncSettings.Mode} loopbackCount={loopback?.Count ?? 0} loopbackEdgeNs={loopback?.EdgeNs ?? 0} reports={lastAudioSynchronizationReports.Count} states={states.Count} analyzeMs={lastAudioSyncAnalysisMilliseconds:0.0} aligned={DescribeAlignedAudio()}");
|
||||
if (lastObsStemPublication.ReadyStems.Count > 0 || lastObsStemPublication.MissingStemIds.Count > 0)
|
||||
{
|
||||
Console.WriteLine(
|
||||
$"mimir-stem-telemetry profile={lastObsStemPublication.ConfigurationId} ready={lastObsStemPublication.ReadyStems.Count} missing={lastObsStemPublication.MissingStemIds.Count} unconfigured={lastObsStemPublication.UnconfiguredStemIds.Count} seq={lastObsStemPublication.LatestSequence}");
|
||||
}
|
||||
|
||||
Console.WriteLine($"mimir-sync-buffers {DescribeAudioBuffers()}");
|
||||
Console.WriteLine($"mimir-spectrum-frustum {DescribeSpectrumFrustum()}");
|
||||
foreach (var spectrum in lastAudioSpectra.OrderBy(snapshot => snapshot.SourceId, StringComparer.Ordinal))
|
||||
|
|
|
|||
|
|
@ -35,6 +35,82 @@ public sealed record MimirObsPublicationConfiguration(
|
|||
double TargetPresentationDelaySeconds,
|
||||
bool DiagnosticOnly);
|
||||
|
||||
public sealed record MimirObsPublishedAudioStem(
|
||||
string StemId,
|
||||
string DisplayName,
|
||||
string SourceId,
|
||||
int ChannelIndex,
|
||||
int FrameCount,
|
||||
int SampleRate,
|
||||
long Sequence,
|
||||
bool Configured);
|
||||
|
||||
public sealed record MimirObsStemPublicationSnapshot(
|
||||
string ConfigurationId,
|
||||
MimirObsAudioPublicationKind AudioKind,
|
||||
IReadOnlyList<MimirObsPublishedAudioStem> ReadyStems,
|
||||
IReadOnlyList<string> MissingStemIds,
|
||||
IReadOnlyList<string> UnconfiguredStemIds,
|
||||
long LatestSequence)
|
||||
{
|
||||
public static MimirObsStemPublicationSnapshot Empty { get; } = new(
|
||||
"",
|
||||
MimirObsAudioPublicationKind.FaustStemBus,
|
||||
[],
|
||||
[],
|
||||
[],
|
||||
0);
|
||||
}
|
||||
|
||||
public sealed class MimirObsStemPublicationState(MimirObsPublicationConfiguration configuration)
|
||||
{
|
||||
private readonly Dictionary<string, MimirObsPublishedAudioStem> latestByStemId = new(StringComparer.Ordinal);
|
||||
|
||||
public void Consume(Aquarium.Engine.Audio.AquariumAudioStemFrame frame)
|
||||
{
|
||||
var configured = configuration.AudioStems.ToDictionary(stem => stem.Id, StringComparer.Ordinal);
|
||||
foreach (var channel in frame.Channels)
|
||||
{
|
||||
var isConfigured = configured.TryGetValue(channel.StemId, out var configuredStem);
|
||||
latestByStemId[channel.StemId] = new MimirObsPublishedAudioStem(
|
||||
channel.StemId,
|
||||
isConfigured ? configuredStem!.DisplayName : channel.DisplayName,
|
||||
channel.SourceId,
|
||||
channel.ChannelIndex,
|
||||
frame.FrameCount,
|
||||
frame.SampleRate,
|
||||
frame.Sequence,
|
||||
isConfigured);
|
||||
}
|
||||
}
|
||||
|
||||
public MimirObsStemPublicationSnapshot Capture()
|
||||
{
|
||||
var configuredIds = configuration.AudioStems
|
||||
.Select(stem => stem.Id)
|
||||
.ToHashSet(StringComparer.Ordinal);
|
||||
var ready = latestByStemId.Values
|
||||
.OrderBy(stem => stem.StemId, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
var missing = configuration.AudioStems
|
||||
.Where(stem => !latestByStemId.ContainsKey(stem.Id))
|
||||
.Select(stem => stem.Id)
|
||||
.Order(StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
var unconfigured = latestByStemId.Keys
|
||||
.Where(stemId => !configuredIds.Contains(stemId))
|
||||
.Order(StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
return new MimirObsStemPublicationSnapshot(
|
||||
configuration.Id,
|
||||
configuration.AudioKind,
|
||||
ready,
|
||||
missing,
|
||||
unconfigured,
|
||||
ready.Length == 0 ? 0 : ready.Max(stem => stem.Sequence));
|
||||
}
|
||||
}
|
||||
|
||||
public static class MimirObsPublicationConfigurations
|
||||
{
|
||||
public static IReadOnlyList<MimirObsAudioStem> ProgramStems { get; } =
|
||||
|
|
@ -48,6 +124,17 @@ public static class MimirObsPublicationConfigurations
|
|||
new("spatial_bed", "Spatial bed", 4, UserMixable: true, CarriesTimingWitness: false, "Encoded volumetric ambience or ambisonic fold.")
|
||||
];
|
||||
|
||||
public static IReadOnlyList<MimirObsAudioStem> AlignmentSourceStems { get; } =
|
||||
Enumerable.Range(0, MimirAlignmentActuatorProfile.SixSourceFaust.SourceCount)
|
||||
.Select(index => new MimirObsAudioStem(
|
||||
$"aligned_source_{index}",
|
||||
$"Aligned source {index}",
|
||||
1,
|
||||
UserMixable: true,
|
||||
CarriesTimingWitness: false,
|
||||
"Post-actuator source lane before final separation/spatial mixing."))
|
||||
.ToArray();
|
||||
|
||||
public static MimirObsPublicationConfiguration NativeProgram { get; } = new(
|
||||
"native-program-spout-faust",
|
||||
"Production target: Fensalir publishes synchronized video, Faust publishes aligned stems.",
|
||||
|
|
@ -58,6 +145,16 @@ public static class MimirObsPublicationConfigurations
|
|||
TargetPresentationDelaySeconds: 5.0,
|
||||
DiagnosticOnly: false);
|
||||
|
||||
public static MimirObsPublicationConfiguration AlignmentActuatorStemBus { get; } = new(
|
||||
"alignment-actuator-stem-bus",
|
||||
"Program-stage Faust stem bus: aligned source lanes before final mix/separation.",
|
||||
MimirObsVideoPublicationKind.Spout2,
|
||||
"Mimir Point Cloud",
|
||||
MimirObsAudioPublicationKind.FaustStemBus,
|
||||
AlignmentSourceStems,
|
||||
TargetPresentationDelaySeconds: 5.0,
|
||||
DiagnosticOnly: false);
|
||||
|
||||
public static MimirObsPublicationConfiguration TextureInteropProgram { get; } = NativeProgram with
|
||||
{
|
||||
Id = "native-program-d3d12-shared-texture",
|
||||
|
|
@ -87,6 +184,7 @@ public static class MimirObsPublicationConfigurations
|
|||
public static IReadOnlyList<MimirObsPublicationConfiguration> BuiltIn { get; } =
|
||||
[
|
||||
NativeProgram,
|
||||
AlignmentActuatorStemBus,
|
||||
TextureInteropProgram,
|
||||
EveNativeProgram,
|
||||
DiagnosticSrtBridge
|
||||
|
|
|
|||
|
|
@ -188,3 +188,4 @@
|
|||
{"ts": "2026-05-27T18:54:28+01:00", "type": "decision", "status": "ok", "note": "Mimir now declares faust/mimir_alignment_actuator.dsp as an AquariumStreamingDspProgram before queuing actuator controls, giving Fensalir both the persistent Faust DSP graph and the control frames. Live sample alignment still waits on the next device I/O/stem-routing cut."}
|
||||
{"ts": "2026-05-27T20:18:50+01:00", "type": "verification", "status": "ok", "note": "MimirRuntime now queues sample-bearing AquariumStreamingAudioBlock lanes for actuator source buffers after publishing the control frame. Channel indices are derived from the bank-assigned Faust sourceN control path, so source disappearance does not reshuffle lanes. Mimir build passed; Fensalir tests prove declared blocks enter the streaming Faust DSP host."}
|
||||
{"ts": "2026-05-27T20:28:43+01:00", "type": "verification", "status": "ok", "note": "Fensalir now publishes processed streaming Faust DSP output lanes as AquariumAudioStemFrame records through an engine-owned IAquariumAudioStemBus. Mimir declares deterministic aligned-source output stems when queuing faust/mimir_alignment_actuator.dsp, so processed actuator lanes have stable stem names before OBS-facing publication consumes them. Fensalir build and Aquarium.Engine.Tests passed; Mimir build and --audio-actuator-bank-smoke passed."}
|
||||
{"ts": "2026-05-27T20:33:41+01:00", "type": "verification", "status": "ok", "note": "Mimir now has an OBS-facing stem publication state for the Fensalir Faust stem bus. It consumes AquariumAudioStemFrame records without copying audio, validates them against the alignment-actuator stem profile, reports ready/missing/unconfigured stems, and telemetry can print publication readiness. --audio-stem-publication-smoke passed; Mimir build passed."}
|
||||
|
|
|
|||
|
|
@ -192,7 +192,10 @@ accepted_design:
|
|||
path. Fensalir now publishes processed DSP output lanes as
|
||||
`AquariumAudioStemFrame` records on an engine-owned audio stem bus, and
|
||||
Mimir declares deterministic aligned-source stem names with the actuator
|
||||
program; OBS-facing stem publication is the remaining program-audio cut. The synthetic
|
||||
program. Mimir now consumes that bus through `MimirObsStemPublicationState`,
|
||||
validating ready/missing/unconfigured lanes against the alignment actuator
|
||||
stem-bus profile without copying audio; actual OBS device/plugin routing is
|
||||
the remaining program-audio cut. The synthetic
|
||||
codebook proof is `dotnet run --project
|
||||
.\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --bioacoustic-self-test`.
|
||||
`Mimir.BufferSmoke --render-chirp-bin-f32` plus
|
||||
|
|
|
|||
Loading…
Reference in a new issue