599 lines
24 KiB
C#
599 lines
24 KiB
C#
using Aquarium.Engine;
|
|
using Aquarium.Engine.Audio;
|
|
using Aquarium.Engine.Input;
|
|
using Aquarium.Engine.Render;
|
|
using Aquarium.Engine.Ui;
|
|
using Mimir.Runtime.Synchronization;
|
|
using System.Diagnostics;
|
|
using System.Numerics;
|
|
using System.Text;
|
|
|
|
namespace Mimir.Runtime;
|
|
|
|
public sealed class MimirRuntime : IAquariumRuntime
|
|
{
|
|
private const float DefaultAudioSyncUpdateIntervalSeconds = 0.1f;
|
|
private const double HybridPassiveConfidenceThreshold = 0.12;
|
|
private const double CalibrationStartSeconds = 0.5;
|
|
private const int CalibrationBatchSegments = 4;
|
|
private const int HybridWatermarkIntervalSegments = 4;
|
|
private const float DefaultSpectrumUpdateIntervalSeconds = 0.2f;
|
|
private readonly MimirSynchronizationHub synchronization;
|
|
private readonly MimirAudioSpectrumAnalyzer spectrumAnalyzer;
|
|
private readonly AquariumUiDocument ui;
|
|
private readonly AquariumAudioDocument audio = new();
|
|
private readonly MimirAudioSynchronizationSettings audioSyncSettings;
|
|
private readonly float telemetryIntervalSeconds;
|
|
private readonly float audioSyncUpdateIntervalSeconds;
|
|
private readonly float spectrumUpdateIntervalSeconds;
|
|
private readonly float calibrationGain;
|
|
private readonly float watermarkGain;
|
|
private readonly MimirBioacousticContestantRenderer? complexContourWitness;
|
|
private int lastPollCount;
|
|
private float runtimeSeconds;
|
|
private float nextAudioSyncSeconds;
|
|
private float nextSpectrumSeconds;
|
|
private float nextTelemetrySeconds;
|
|
private bool sceneReady;
|
|
private ulong calibrationSegmentIndex;
|
|
private long lastHybridWatermarkSegment = -1;
|
|
private double lastAudioSyncAnalysisMilliseconds;
|
|
private double lastSpectrumAnalysisMilliseconds;
|
|
private double lastPassiveSynchronizationConfidence;
|
|
private IReadOnlyList<MimirAudioSynchronizationReport> lastAudioSynchronizationReports = [];
|
|
private IReadOnlyList<MimirAudioSpectrumSnapshot> lastAudioSpectra = [];
|
|
|
|
public MimirRuntime(AquariumRuntimeOptions options)
|
|
: this(options, MimirRuntimeConfiguration.Load())
|
|
{
|
|
}
|
|
|
|
public MimirRuntime(AquariumRuntimeOptions options, MimirRuntimeConfiguration configuration)
|
|
: this(options, configuration.Settings, configuration.Sources)
|
|
{
|
|
}
|
|
|
|
public MimirRuntime(AquariumRuntimeOptions options, MimirSynchronizationSettings settings)
|
|
: this(options, settings, [])
|
|
{
|
|
}
|
|
|
|
public MimirRuntime(
|
|
AquariumRuntimeOptions options,
|
|
MimirSynchronizationSettings settings,
|
|
IEnumerable<IMimirStreamSource> streamSources)
|
|
{
|
|
Options = options;
|
|
synchronization = new MimirSynchronizationHub(settings);
|
|
spectrumAnalyzer = new MimirAudioSpectrumAnalyzer(ParseSpectrumFftSize());
|
|
audioSyncSettings = settings.Audio;
|
|
telemetryIntervalSeconds = ParseTelemetryIntervalSeconds();
|
|
audioSyncUpdateIntervalSeconds = ParseAudioSyncIntervalSeconds();
|
|
spectrumUpdateIntervalSeconds = ParseSpectrumUpdateIntervalSeconds();
|
|
calibrationGain = settings.Audio.CalibrationGain;
|
|
watermarkGain = settings.Audio.WatermarkGain;
|
|
complexContourWitness = settings.Audio.EnableComplexContourRuntime
|
|
? new MimirBioacousticContestantRenderer(
|
|
MimirBioacousticContestants.BuiltIn.FirstOrDefault(profile =>
|
|
string.Equals(profile.Id, settings.Audio.BioacousticWitnessProfileId, StringComparison.OrdinalIgnoreCase))
|
|
?? MimirBioacousticContestants.CanaryPacketTrill)
|
|
: null;
|
|
nextAudioSyncSeconds = audioSyncUpdateIntervalSeconds;
|
|
nextTelemetrySeconds = telemetryIntervalSeconds;
|
|
foreach (var source in streamSources)
|
|
{
|
|
synchronization.AddSource(source);
|
|
}
|
|
|
|
ui = CreateUi();
|
|
}
|
|
|
|
public AquariumRuntimeOptions Options { get; }
|
|
|
|
public AquariumFrame Frame => new(
|
|
new ViewFrame(Vector2.Zero, 24.0f),
|
|
new Vector3(0.0f, -10.0f, 7.0f),
|
|
runtimeSeconds,
|
|
Vector2.Zero,
|
|
new AquariumSceneState
|
|
{
|
|
TraceHeightFieldSurface = false,
|
|
UseStarfieldBackground = false,
|
|
});
|
|
|
|
public GraphicsSettings GraphicsSettings { get; set; } = GraphicsSettings.Default;
|
|
|
|
public AquariumRenderPlan RenderPlan { get; } = CreateRenderPlan();
|
|
|
|
public AquariumUiDocument Ui => ui;
|
|
|
|
public AquariumAudioDocument Audio => audio;
|
|
|
|
public AquariumSynthDocument Synth => AquariumSynthDocument.Empty;
|
|
|
|
public void RegisterStreamSource(IMimirStreamSource source)
|
|
{
|
|
synchronization.AddSource(source);
|
|
}
|
|
|
|
public void Start()
|
|
{
|
|
Console.WriteLine($"Mimir runtime sync buffers: {synchronization.Summary()} @ {synchronization.Settings.BufferDuration.TotalSeconds:0.###}s audioSync={audioSyncSettings.Mode} reference={audioSyncSettings.ReferenceSourceId}");
|
|
Console.WriteLine("Mimir empty scene runtime booted.");
|
|
}
|
|
|
|
public void Update(float deltaSeconds, InputState input)
|
|
{
|
|
runtimeSeconds += Math.Max(deltaSeconds, 0.0f);
|
|
lastPollCount = synchronization.PollSources();
|
|
if (sceneReady)
|
|
{
|
|
QueueCalibrationTimeline();
|
|
UpdateAudioSpectra();
|
|
UpdateAudioSynchronization();
|
|
}
|
|
|
|
EmitTelemetry();
|
|
}
|
|
|
|
public void OnSceneReady()
|
|
{
|
|
if (sceneReady)
|
|
{
|
|
return;
|
|
}
|
|
|
|
sceneReady = true;
|
|
nextAudioSyncSeconds = runtimeSeconds + audioSyncUpdateIntervalSeconds;
|
|
nextSpectrumSeconds = runtimeSeconds;
|
|
Console.WriteLine($"Mimir Fensalir scene ready at {runtimeSeconds:0.000}s; runtime audio tests and spectra enabled.");
|
|
}
|
|
|
|
public AquariumFrame ComposeFrame(AquariumFrame frame, AquariumFrameInput input)
|
|
{
|
|
return frame;
|
|
}
|
|
|
|
public void FlushState()
|
|
{
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
synchronization.Dispose();
|
|
}
|
|
|
|
private AquariumUiDocument CreateUi()
|
|
{
|
|
return new AquariumUiDocument()
|
|
.Panel("Mimir Sync", 18.0f, 82.0f, 390.0f, panel =>
|
|
{
|
|
panel.Section("Rolling Buffers");
|
|
panel.Readout("Window", () => $"{synchronization.Settings.BufferDuration.TotalSeconds:0.###}s");
|
|
panel.Readout("Streams", synchronization.Summary);
|
|
panel.Readout("Sources", () => $"{synchronization.SourceCount}");
|
|
panel.Readout("Last poll", () => $"{lastPollCount} samples");
|
|
panel.Readout("Ingested", () => $"{synchronization.IngestedSamples}");
|
|
panel.Readout("Buffer details", DescribeBuffers);
|
|
panel.Readout("Audio sync", DescribeAudioSync);
|
|
panel.Readout("Audio sync state", DescribeAudioSyncState);
|
|
panel.Readout("Aligned audio", DescribeAlignedAudio);
|
|
panel.Readout("Chirplet reference", DescribeChirpletReference);
|
|
panel.Section("Live FFT");
|
|
panel.Readout("Spectrum cadence", () => sceneReady
|
|
? $"{lastAudioSpectra.Count} spectra fft={lastAudioSpectra.FirstOrDefault()?.FftSize ?? 0} analyze={lastSpectrumAnalysisMilliseconds:0.00}ms"
|
|
: "waiting for Fensalir scene ready");
|
|
panel.TextBox(
|
|
"Spectra",
|
|
DescribeSpectra,
|
|
_ => { },
|
|
lines: 12,
|
|
acceptsReturn: false,
|
|
monospace: true,
|
|
tooltip: "Cached FFT spectra from the runtime rolling audio buffers.");
|
|
});
|
|
}
|
|
|
|
private static AquariumRenderPlan CreateRenderPlan()
|
|
{
|
|
var app = new AquariumApp();
|
|
var scene = app.RenderTargets.Hdr("scene");
|
|
app.Cameras.Perspective("main");
|
|
app.Graph.Pass("scene").Fullscreen();
|
|
app.Features.Presentation(scene.Color);
|
|
app.Features.DirectWriteOverlay();
|
|
app.Debug.View("Scene", scene.Color);
|
|
return app.Plan;
|
|
}
|
|
|
|
private void QueueCalibrationTimeline()
|
|
{
|
|
if (!ShouldEmitCalibrationTimeline())
|
|
{
|
|
return;
|
|
}
|
|
|
|
var currentSegmentIndex = CurrentCalibrationSegmentIndex();
|
|
if (audioSyncSettings.Mode == MimirAudioSyncMode.Hybrid)
|
|
{
|
|
calibrationSegmentIndex = currentSegmentIndex;
|
|
if (calibrationSegmentIndex % HybridWatermarkIntervalSegments != 0UL ||
|
|
(long)calibrationSegmentIndex == lastHybridWatermarkSegment)
|
|
{
|
|
return;
|
|
}
|
|
}
|
|
else if (calibrationSegmentIndex + 1UL < currentSegmentIndex)
|
|
{
|
|
calibrationSegmentIndex = currentSegmentIndex;
|
|
}
|
|
|
|
var nextSegmentSeconds = CalibrationStartSeconds + calibrationSegmentIndex * MimirBioacousticTimeline.SegmentSeconds;
|
|
if (runtimeSeconds < nextSegmentSeconds)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var segmentCount = audioSyncSettings.Mode == MimirAudioSyncMode.Hybrid ? 1 : CalibrationBatchSegments;
|
|
var outputGain = audioSyncSettings.Mode == MimirAudioSyncMode.Hybrid ? watermarkGain : calibrationGain;
|
|
var leftBatch = RenderCalibrationBatchPcm16Base64(
|
|
calibrationSegmentIndex,
|
|
segmentCount,
|
|
MimirBioacousticSpeaker.Left,
|
|
out var peak);
|
|
var rightBatch = RenderCalibrationBatchPcm16Base64(
|
|
calibrationSegmentIndex,
|
|
segmentCount,
|
|
MimirBioacousticSpeaker.Right,
|
|
out var rightPeak);
|
|
peak = Math.Max(peak, rightPeak);
|
|
audio.EnqueuePcm16Base64(
|
|
leftBatch,
|
|
MimirBioacousticTimeline.SampleRate,
|
|
channels: 1,
|
|
gain: outputGain,
|
|
pan: -1.0f);
|
|
audio.EnqueuePcm16Base64(
|
|
rightBatch,
|
|
MimirBioacousticTimeline.SampleRate,
|
|
channels: 1,
|
|
gain: outputGain,
|
|
pan: 1.0f);
|
|
Console.WriteLine(
|
|
$"mimir-bioacoustic-batch mode={DescribeAudioSyncMode()} firstSegment={calibrationSegmentIndex} segments={segmentCount} seconds={segmentCount * MimirBioacousticTimeline.SegmentSeconds:0.00} peak={peak:0.000000} gain={outputGain:0.###} leftBase64Bytes={leftBatch.Length} rightBase64Bytes={rightBatch.Length}");
|
|
if (audioSyncSettings.Mode == MimirAudioSyncMode.Hybrid)
|
|
{
|
|
lastHybridWatermarkSegment = (long)calibrationSegmentIndex;
|
|
}
|
|
else
|
|
{
|
|
calibrationSegmentIndex += (ulong)segmentCount;
|
|
}
|
|
}
|
|
|
|
private ulong CurrentCalibrationSegmentIndex()
|
|
{
|
|
if (runtimeSeconds <= CalibrationStartSeconds)
|
|
{
|
|
return 0UL;
|
|
}
|
|
|
|
return (ulong)Math.Floor((runtimeSeconds - CalibrationStartSeconds) / MimirBioacousticTimeline.SegmentSeconds);
|
|
}
|
|
|
|
private string DescribeChirpletReference()
|
|
{
|
|
if (!ShouldEmitCalibrationTimeline())
|
|
{
|
|
return $"{audioSyncSettings.ReferenceSourceId} passive mode; calibration emission disabled";
|
|
}
|
|
|
|
var emittedUntilSeconds = calibrationSegmentIndex * MimirBioacousticTimeline.SegmentSeconds;
|
|
return $"{audioSyncSettings.ReferenceSourceId} {DescribeAudioSyncMode()} bioacoustic timeline {MimirBioacousticTimeline.SegmentSeconds:0.00}s segments emitted to {emittedUntilSeconds:0.00}s passiveConfidence={lastPassiveSynchronizationConfidence:0.000}";
|
|
}
|
|
|
|
private string RenderCalibrationBatchPcm16Base64(
|
|
ulong firstSegment,
|
|
int segmentCount,
|
|
MimirBioacousticSpeaker speaker,
|
|
out float peak)
|
|
{
|
|
var samplesPerSegment = (int)Math.Round(MimirBioacousticTimeline.SegmentSeconds * MimirBioacousticTimeline.SampleRate);
|
|
var bytes = new byte[samplesPerSegment * Math.Max(1, segmentCount) * sizeof(short)];
|
|
var byteIndex = 0;
|
|
peak = 0.0f;
|
|
for (var segment = 0; segment < segmentCount; segment++)
|
|
{
|
|
var samples = complexContourWitness == null
|
|
? MimirBioacousticTimeline.Default.RenderSegmentMonoFloat(
|
|
firstSegment + (ulong)segment,
|
|
MimirBioacousticTimeline.SampleRate,
|
|
speaker)
|
|
: complexContourWitness.RenderSegmentMonoFloat(
|
|
firstSegment + (ulong)segment,
|
|
MimirBioacousticTimeline.SegmentSeconds,
|
|
MimirBioacousticTimeline.SampleRate,
|
|
speaker);
|
|
for (var index = 0; index < samples.Length; index++)
|
|
{
|
|
peak = Math.Max(peak, Math.Abs(samples[index]));
|
|
var sample = (short)Math.Round(Math.Clamp(samples[index], -1.0f, 1.0f) * short.MaxValue);
|
|
bytes[byteIndex++] = (byte)(sample & 0xff);
|
|
bytes[byteIndex++] = (byte)((sample >> 8) & 0xff);
|
|
}
|
|
}
|
|
|
|
return Convert.ToBase64String(bytes);
|
|
}
|
|
|
|
private string DescribeBuffers()
|
|
{
|
|
return string.Join(" | ", synchronization.Buffers.Buffers.Select(DescribeBuffer));
|
|
}
|
|
|
|
private void UpdateAudioSynchronization()
|
|
{
|
|
if (runtimeSeconds < nextAudioSyncSeconds)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var stopwatch = Stopwatch.StartNew();
|
|
synchronization.AnalyzeAudioSynchronizationStep(audioSyncSettings.ReferenceSourceId, audioSyncSettings.Mode);
|
|
if (synchronization.ComplexContourRuntimeEnabled)
|
|
{
|
|
synchronization.AnalyzeComplexContourSynchronizationStep(audioSyncSettings.ReferenceSourceId, runtimeSeconds);
|
|
}
|
|
stopwatch.Stop();
|
|
lastAudioSyncAnalysisMilliseconds = stopwatch.Elapsed.TotalMilliseconds;
|
|
lastAudioSynchronizationReports = synchronization.AudioSynchronizationReports;
|
|
UpdatePassiveSynchronizationConfidence(lastAudioSynchronizationReports);
|
|
nextAudioSyncSeconds = runtimeSeconds + audioSyncUpdateIntervalSeconds;
|
|
}
|
|
|
|
private void UpdateAudioSpectra()
|
|
{
|
|
if (runtimeSeconds < nextSpectrumSeconds)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var stopwatch = Stopwatch.StartNew();
|
|
lastAudioSpectra = spectrumAnalyzer.Analyze(synchronization.Buffers.Buffers, audioSyncSettings.ReferenceSourceId);
|
|
stopwatch.Stop();
|
|
lastSpectrumAnalysisMilliseconds = stopwatch.Elapsed.TotalMilliseconds;
|
|
nextSpectrumSeconds = runtimeSeconds + spectrumUpdateIntervalSeconds;
|
|
}
|
|
|
|
private void EmitTelemetry()
|
|
{
|
|
if (telemetryIntervalSeconds <= 0.0f || runtimeSeconds < nextTelemetrySeconds)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var loopback = synchronization.Buffers.Buffers.FirstOrDefault(buffer =>
|
|
string.Equals(buffer.Descriptor.SourceId, audioSyncSettings.ReferenceSourceId, StringComparison.Ordinal));
|
|
var states = synchronization.AudioSynchronizationStates;
|
|
Console.WriteLine(
|
|
$"mimir-sync-telemetry t={runtimeSeconds:0.00}s audioSync={audioSyncSettings.Mode} loopbackCount={loopback?.Count ?? 0} loopbackEdgeNs={loopback?.EdgeNs ?? 0} reports={lastAudioSynchronizationReports.Count} states={states.Count} analyzeMs={lastAudioSyncAnalysisMilliseconds:0.0} aligned={DescribeAlignedAudio()}");
|
|
Console.WriteLine($"mimir-sync-buffers {DescribeAudioBuffers()}");
|
|
foreach (var spectrum in lastAudioSpectra.OrderBy(snapshot => snapshot.SourceId, StringComparer.Ordinal))
|
|
{
|
|
Console.WriteLine(
|
|
$"mimir-spectrum {spectrum.SourceId} rate={spectrum.SampleRate} fft={spectrum.FftSize} rms={spectrum.Rms:0.000000} peak={spectrum.Peak:0.000000} floorDb={spectrum.NoiseFloorDb:0.0} peaks={DescribeSpectrumPeaks(spectrum)}");
|
|
}
|
|
|
|
foreach (var report in lastAudioSynchronizationReports.OrderBy(report => report.SourceId, StringComparer.Ordinal))
|
|
{
|
|
Console.WriteLine(
|
|
$"mimir-sync-report {report.ReferenceSourceId}->{report.SourceId} evidence={report.EvidenceKind} delaySamples={report.FractionalDelaySamples:0.000000} delayUs={report.DelayMicroseconds:0.000} delayMs={report.DelayMilliseconds:0.000} confidence={report.Confidence:0.000} timelineEvents={report.TimelineMatchedEvents} timelineConfidence={report.TimelineConfidence:0.000}");
|
|
}
|
|
|
|
foreach (var report in synchronization.ComplexContourSynchronizationReports.OrderBy(report => report.SourceId, StringComparer.Ordinal))
|
|
{
|
|
Console.WriteLine(
|
|
$"mimir-complex-contour-report {report.ReferenceSourceId}->{report.SourceId} delaySamples={report.FractionalDelaySamples:0.000000} delayUs={report.DelayMicroseconds:0.000} confidence={report.Confidence:0.000} directHits={report.TimelineMatchedEvents}");
|
|
}
|
|
|
|
foreach (var state in states)
|
|
{
|
|
Console.WriteLine(
|
|
$"mimir-sync-state {state.ReferenceSourceId}->{state.SourceId} delaySamples={state.SmoothedDelaySamples:0.000000} delayUs={state.DelayMicroseconds:0.000} delayMs={state.DelayMilliseconds:0.000} sroPpm={state.SamplingRateOffsetPpm:0.000} confidence={state.Confidence:0.000}");
|
|
}
|
|
|
|
foreach (var trace in synchronization.AudioSynchronizationDecodeTraces.OrderBy(trace => trace.SourceId, StringComparer.Ordinal))
|
|
{
|
|
Console.WriteLine(
|
|
$"mimir-sync-decode {trace.ReferenceSourceId}->{trace.SourceId} status={trace.Status} compared={trace.ComparedSamples} rate={trace.SampleRate} refFrames={trace.ReferenceFrames} refAnchors={trace.ReferenceAnchors} refClock={trace.ReferenceClockConfidence:0.000} refEnergy={trace.ReferenceBestEnergy:0.000} candFrames={trace.CandidateFrames} candAnchors={trace.CandidateAnchors} candClock={trace.CandidateClockConfidence:0.000} candEnergy={trace.CandidateBestEnergy:0.000} matched={trace.MatchedEvents} confidence={trace.Confidence:0.000}");
|
|
}
|
|
|
|
nextTelemetrySeconds += telemetryIntervalSeconds;
|
|
}
|
|
|
|
private static float ParseTelemetryIntervalSeconds()
|
|
{
|
|
return float.TryParse(Environment.GetEnvironmentVariable("MIMIR_SYNC_TELEMETRY_SECONDS"), out var seconds)
|
|
? Math.Clamp(seconds, 0.0f, 60.0f)
|
|
: 0.0f;
|
|
}
|
|
|
|
private static float ParseAudioSyncIntervalSeconds()
|
|
{
|
|
return float.TryParse(Environment.GetEnvironmentVariable("MIMIR_AUDIO_SYNC_INTERVAL_SECONDS"), out var seconds)
|
|
? Math.Clamp(seconds, 0.1f, 10.0f)
|
|
: DefaultAudioSyncUpdateIntervalSeconds;
|
|
}
|
|
|
|
private static float ParseSpectrumUpdateIntervalSeconds()
|
|
{
|
|
return float.TryParse(Environment.GetEnvironmentVariable("MIMIR_SPECTRUM_INTERVAL_SECONDS"), out var seconds)
|
|
? Math.Clamp(seconds, 0.05f, 10.0f)
|
|
: DefaultSpectrumUpdateIntervalSeconds;
|
|
}
|
|
|
|
private static int ParseSpectrumFftSize()
|
|
{
|
|
return int.TryParse(Environment.GetEnvironmentVariable("MIMIR_SPECTRUM_FFT_SIZE"), out var value)
|
|
? Math.Clamp(value, 1024, 32768)
|
|
: 8192;
|
|
}
|
|
|
|
private bool ShouldEmitCalibrationTimeline()
|
|
{
|
|
if (!HasReferenceAudioBuffer())
|
|
{
|
|
return false;
|
|
}
|
|
|
|
return audioSyncSettings.Mode switch
|
|
{
|
|
MimirAudioSyncMode.ChirpOnly => true,
|
|
MimirAudioSyncMode.Passive => false,
|
|
MimirAudioSyncMode.Hybrid => lastPassiveSynchronizationConfidence < HybridPassiveConfidenceThreshold,
|
|
_ => true,
|
|
};
|
|
}
|
|
|
|
private bool HasReferenceAudioBuffer()
|
|
{
|
|
return synchronization.Buffers.Buffers.Any(buffer =>
|
|
buffer.Descriptor.Kind == MimirStreamKind.Audio &&
|
|
string.Equals(buffer.Descriptor.SourceId, audioSyncSettings.ReferenceSourceId, StringComparison.Ordinal));
|
|
}
|
|
|
|
private string DescribeAudioSyncMode()
|
|
{
|
|
return audioSyncSettings.Mode switch
|
|
{
|
|
MimirAudioSyncMode.ChirpOnly => "chirp-only",
|
|
MimirAudioSyncMode.Passive => "passive",
|
|
MimirAudioSyncMode.Hybrid => "hybrid",
|
|
_ => audioSyncSettings.Mode.ToString(),
|
|
};
|
|
}
|
|
|
|
private string DescribeAudioBuffers()
|
|
{
|
|
return string.Join(" | ", synchronization.Buffers.Buffers
|
|
.Where(buffer => buffer.Descriptor.Kind == MimirStreamKind.Audio)
|
|
.OrderBy(buffer => buffer.Descriptor.SourceId, StringComparer.Ordinal)
|
|
.Select(buffer => $"{buffer.Descriptor.SourceId}:{buffer.Count}@{buffer.EdgeNs}"));
|
|
}
|
|
|
|
private string DescribeSpectra()
|
|
{
|
|
if (!sceneReady)
|
|
{
|
|
return "Fensalir scene not ready.";
|
|
}
|
|
|
|
if (lastAudioSpectra.Count == 0)
|
|
{
|
|
return "No sample-bearing audio buffers yet.";
|
|
}
|
|
|
|
var builder = new StringBuilder();
|
|
foreach (var spectrum in lastAudioSpectra)
|
|
{
|
|
builder
|
|
.Append(spectrum.SourceId)
|
|
.Append(" rms=")
|
|
.Append(spectrum.Rms.ToString("0.000000"))
|
|
.Append(" peak=")
|
|
.Append(spectrum.Peak.ToString("0.000000"))
|
|
.Append(" ")
|
|
.Append(SpectrumBars(spectrum))
|
|
.AppendLine();
|
|
builder
|
|
.Append(" ")
|
|
.Append(DescribeSpectrumPeaks(spectrum))
|
|
.AppendLine();
|
|
}
|
|
|
|
return builder.ToString().TrimEnd();
|
|
}
|
|
|
|
private static string SpectrumBars(MimirAudioSpectrumSnapshot spectrum)
|
|
{
|
|
const string ramp = " .:-=+*#%@";
|
|
if (spectrum.BandDecibels.Count == 0)
|
|
{
|
|
return "";
|
|
}
|
|
|
|
var floor = Math.Min(spectrum.NoiseFloorDb, spectrum.BandDecibels.Min());
|
|
var ceiling = Math.Max(-24.0, spectrum.BandDecibels.Max());
|
|
var span = Math.Max(12.0, ceiling - floor);
|
|
var chars = spectrum.BandDecibels.Select(value =>
|
|
{
|
|
var normalized = Math.Clamp((value - floor) / span, 0.0, 1.0);
|
|
return ramp[(int)Math.Round(normalized * (ramp.Length - 1))];
|
|
});
|
|
return new string(chars.ToArray());
|
|
}
|
|
|
|
private static string DescribeSpectrumPeaks(MimirAudioSpectrumSnapshot spectrum)
|
|
{
|
|
return spectrum.Peaks.Count == 0
|
|
? "no FFT peaks"
|
|
: string.Join(", ", spectrum.Peaks.Select(peak => $"{peak.FrequencyHz / 1000.0:0.00}kHz {peak.Decibels:0.0}dB"));
|
|
}
|
|
|
|
private void UpdatePassiveSynchronizationConfidence(IReadOnlyList<MimirAudioSynchronizationReport> reports)
|
|
{
|
|
var passiveConfidence = reports
|
|
.Where(report => string.Equals(report.EvidenceKind, "passive", StringComparison.Ordinal))
|
|
.Select(report => report.Confidence)
|
|
.DefaultIfEmpty(0.0)
|
|
.Max();
|
|
lastPassiveSynchronizationConfidence = passiveConfidence > 0.0
|
|
? passiveConfidence
|
|
: lastPassiveSynchronizationConfidence * 0.95;
|
|
}
|
|
|
|
private static string DescribeBuffer(MimirRollingStreamBuffer buffer)
|
|
{
|
|
var latest = buffer.Latest;
|
|
if (latest?.VideoFrame is { } frame)
|
|
{
|
|
return $"{buffer.Descriptor.SourceId}: {buffer.Count} {frame.Width}x{frame.Height} {frame.PixelFormat} bytes {latest.Value.ByteLength} edge {buffer.EdgeNs}";
|
|
}
|
|
|
|
if (latest?.AudioBlock is { } block)
|
|
{
|
|
return $"{buffer.Descriptor.SourceId}: {buffer.Count} {block.Channels}ch {block.SampleRate}Hz {block.SampleFormat} frames {block.FrameCount} bytes {latest.Value.ByteLength} edge {buffer.EdgeNs}";
|
|
}
|
|
|
|
return $"{buffer.Descriptor.SourceId}: {buffer.Count} edge {buffer.EdgeNs}";
|
|
}
|
|
|
|
private string DescribeAudioSync()
|
|
{
|
|
if (audioSyncSettings.Mode == MimirAudioSyncMode.Passive)
|
|
{
|
|
return "passive mode; waiting for program-audio coherence";
|
|
}
|
|
|
|
var reports = synchronization.AudioSynchronizationReports;
|
|
return reports.Count == 0
|
|
? "no payload windows"
|
|
: string.Join(" | ", reports.Select(report => $"{report.SourceId}: {report.FractionalDelaySamples:0.000} samples {report.DelayMicroseconds:0.0}us c={report.Confidence:0.00} {report.EvidenceKind} events={report.TimelineMatchedEvents}"));
|
|
}
|
|
|
|
private string DescribeAlignedAudio()
|
|
{
|
|
var states = synchronization.AudioSynchronizationStates;
|
|
return states.Count == 0
|
|
? "no aligned state"
|
|
: $"{states.Count + 1}ch state-ready";
|
|
}
|
|
|
|
private string DescribeAudioSyncState()
|
|
{
|
|
var states = synchronization.AudioSynchronizationStates;
|
|
return states.Count == 0
|
|
? "no sync state"
|
|
: string.Join(" | ", states.Select(state => $"{state.SourceId}: {state.SmoothedDelaySamples:0.000} samples {state.DelayMicroseconds:0.0}us sro={state.SamplingRateOffsetPpm:0.0}ppm c={state.Confidence:0.00}"));
|
|
}
|
|
}
|