diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 31b4f5a..71ee1b3 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -788,6 +788,11 @@ a named invariant that the native runtime cannot protect yet. states and measured band responses, emits a corrected composite plus scores, and is proved by `--calibrated-audio-composite-smoke`. Treat that as the contract and offline scorer for the native/Faust implementation, not as a - parallel live mixer authority. + parallel live mixer authority. The next matching probe loop also exists as + `MimirTargetedBioacousticProbePlanner`: residuals and mic pathology notes + select gain-bounded shaped probe bands for the bioacoustic renderer. Start + with the 192 kHz shotgun/cardioid pair; only widen to every connected mic + once those two paths have credible direct-path and frequency-response + confidence. 10. Keep the OBS bridge witness ledger as evidence before expanding receiver machinery. diff --git a/notes/current-system-map.md b/notes/current-system-map.md index e6ea6a7..5817178 100644 --- a/notes/current-system-map.md +++ b/notes/current-system-map.md @@ -306,6 +306,19 @@ owns the production alignment, suppression, separation, and stem lane. synthetic delayed, colored, noisy mic paths: the uncalibrated mix measures -3.171 dB SNR, the calibrated composite measures 15.135 dB SNR, and response flatness improves from 0.548 to 1.000. +`MimirTargetedBioacousticProbePlanner` is the matching "ask the room a better +question" owner. It consumes per-source spectral residuals and optional source +pathology notes, then emits gain-bounded shaped probe bands for the +bioacoustic renderer to spend energy on. The first smoke models the two +professional 192 kHz Scarlett mics only: `asio-ch0-shotgun` is explicitly +noisy and low-heavy, `asio-ch1-cardioid` is cleaner, and the planner prioritizes +the shotgun's weak/unmeasured high-band evidence before lower residuals. +`Mimir.BufferSmoke --targeted-bioacoustic-probe-smoke` selects four probe +bands, led by about 18.6 kHz as an unmeasured band at the max bounded gain, +then about 9.9 kHz weak response, 3.6 kHz phase instability, and 910 Hz +magnitude residual. This is the intended first physical strategy: improve +shotgun/cardioid confidence at 192 kHz before asking every connected mic to +help solve the room field. That real-world proof now has a first receipt: `calibration/bioacoustic/complex-contour-live-20260525-063229.md`. A freshly rendered canary-packet witness was played through Focusrite ASIO and captured diff --git a/src/Mimir.BufferSmoke/Program.cs b/src/Mimir.BufferSmoke/Program.cs index 1d56d02..4dc1a9f 100644 --- a/src/Mimir.BufferSmoke/Program.cs +++ b/src/Mimir.BufferSmoke/Program.cs @@ -80,6 +80,11 @@ if (args.Any(arg => string.Equals(arg, "--calibrated-audio-composite-smoke", Str return RunCalibratedAudioCompositeSmoke(); } +if (args.Any(arg => string.Equals(arg, "--targeted-bioacoustic-probe-smoke", StringComparison.OrdinalIgnoreCase))) +{ + return RunTargetedBioacousticProbeSmoke(); +} + if (args.Any(arg => string.Equals(arg, "--synchronized-buffer-planner-smoke", StringComparison.OrdinalIgnoreCase))) { return RunSynchronizedBufferPlannerSmoke(); @@ -3365,6 +3370,57 @@ static int RunCalibratedAudioCompositeSmoke() : 1; } +static int RunTargetedBioacousticProbeSmoke() +{ + var residuals = new[] + { + new MimirAudioCalibrationBandResidual("asio-ch0-shotgun", 240.0, 0.91, 0.4, ResponseEnergy: 0.62), + new MimirAudioCalibrationBandResidual("asio-ch1-cardioid", 240.0, 0.87, 0.6, ResponseEnergy: 0.58), + new MimirAudioCalibrationBandResidual("asio-ch0-shotgun", 890.0, 0.52, 4.8, DelayResidualMicroseconds: 11.0, ResponseEnergy: 0.30), + new MimirAudioCalibrationBandResidual("asio-ch1-cardioid", 910.0, 0.47, 5.2, DelayResidualMicroseconds: 14.0, ResponseEnergy: 0.27), + new MimirAudioCalibrationBandResidual("asio-ch0-shotgun", 3_600.0, 0.44, -6.4, PhaseResidualRadians: 1.4, ResponseEnergy: 0.11), + new MimirAudioCalibrationBandResidual("asio-ch1-cardioid", 3_690.0, 0.41, -5.8, PhaseResidualRadians: 1.2, ResponseEnergy: 0.09), + new MimirAudioCalibrationBandResidual("asio-ch0-shotgun", 9_800.0, 0.28, -9.5, PhaseResidualRadians: 2.0, ResponseEnergy: 0.035), + new MimirAudioCalibrationBandResidual("asio-ch1-cardioid", 9_950.0, 0.22, -10.2, PhaseResidualRadians: 1.8, ResponseEnergy: 0.030), + new MimirAudioCalibrationBandResidual("asio-ch0-shotgun", 18_500.0, 0.03, -18.0, DelayResidualMicroseconds: 34.0, PhaseResidualRadians: 2.7, ResponseEnergy: 0.0), + new MimirAudioCalibrationBandResidual("asio-ch1-cardioid", 18_700.0, 0.02, -16.0, DelayResidualMicroseconds: 28.0, PhaseResidualRadians: 2.4, ResponseEnergy: 0.0), + }; + var plan = new MimirTargetedBioacousticProbePlanner().Plan( + residuals, + ["asio-ch0-shotgun", "asio-ch1-cardioid"], + [ + new MimirAudioCalibrationSourcePathology( + "asio-ch0-shotgun", + NoiseFloorRms: 0.055, + LowHighTiltDb: 17.0, + HighBandConfidenceBias: 0.35), + new MimirAudioCalibrationSourcePathology( + "asio-ch1-cardioid", + NoiseFloorRms: 0.018, + LowHighTiltDb: 4.0) + ]); + var top = plan.Bands.FirstOrDefault(); + var hasLowBand = plan.Bands.Any(band => band.CenterHz is > 800.0 and < 1_000.0); + var hasMidBand = plan.Bands.Any(band => band.CenterHz is > 3_300.0 and < 3_900.0); + var hasHighBand = plan.Bands.Any(band => band.CenterHz is > 17_000.0 and < 20_000.0); + Console.WriteLine( + $"targeted-bioacoustic-probe-smoke sampleRate={plan.SampleRate} sources={string.Join(",", plan.MeasurementSourceIds)} bands={plan.Bands.Count} " + + $"duration={plan.EstimatedDurationSeconds:0.000} top={top?.CenterHz:0}Hz reason={top?.Reason} gain={top?.Gain:0.000} priority={top?.Priority:0.000} " + + $"selected={string.Join(",", plan.Bands.Select(band => $"{band.CenterHz:0}Hz/{band.Reason}/{band.Gain:0.000}"))}"); + + return plan.SampleRate == 192_000 && + plan.MeasurementSourceIds.SequenceEqual(new[] { "asio-ch0-shotgun", "asio-ch1-cardioid" }) && + plan.Bands.Count >= 3 && + hasLowBand && + hasMidBand && + hasHighBand && + top is { CenterHz: > 17_000.0 and < 20_000.0 } && + top.Reason is MimirBioacousticProbeReason.UnmeasuredBand or MimirBioacousticProbeReason.WeakResponse or MimirBioacousticProbeReason.PhaseUnstable && + plan.Bands.All(band => band.Gain is >= 0.018 and <= 0.085) + ? 0 + : 1; +} + static int RunSynchronizedBufferPlannerSmoke() { const long t0 = 1_000_000_000; diff --git a/src/Mimir.Runtime/Synchronization/MimirTargetedBioacousticProbePlanner.cs b/src/Mimir.Runtime/Synchronization/MimirTargetedBioacousticProbePlanner.cs new file mode 100644 index 0000000..9ba149b --- /dev/null +++ b/src/Mimir.Runtime/Synchronization/MimirTargetedBioacousticProbePlanner.cs @@ -0,0 +1,217 @@ +namespace Mimir.Runtime.Synchronization; + +public enum MimirBioacousticProbeReason +{ + LowConfidence, + HighResidual, + WeakResponse, + PhaseUnstable, + UnmeasuredBand +} + +public sealed record MimirAudioCalibrationBandResidual( + string SourceId, + double CenterHz, + double Confidence, + double ResidualDb, + double DelayResidualMicroseconds = 0.0, + double PhaseResidualRadians = 0.0, + double ResponseEnergy = 0.0); + +public sealed record MimirAudioCalibrationSourcePathology( + string SourceId, + double NoiseFloorRms = 0.0, + double LowHighTiltDb = 0.0, + double HighBandConfidenceBias = 0.0); + +public sealed record MimirTargetedBioacousticProbeOptions( + int SampleRate = 192_000, + int MaxProbeBands = 8, + double MinFrequencyHz = 120.0, + double MaxFrequencyHz = 42_000.0, + double TargetConfidence = 0.82, + double ResidualDbBudget = 1.5, + double WeakResponseThreshold = 0.08, + double PhaseResidualBudgetRadians = 0.80, + double MinimumBandSpacingFraction = 0.055, + double BaseGain = 0.018, + double MaxGain = 0.085); + +public sealed record MimirTargetedBioacousticProbeBand( + double CenterHz, + double StartHz, + double EndHz, + double Gain, + double Priority, + MimirBioacousticProbeReason Reason, + IReadOnlyList SourceIds); + +public sealed record MimirTargetedBioacousticProbePlan( + string Id, + int SampleRate, + IReadOnlyList MeasurementSourceIds, + IReadOnlyList Bands, + double EstimatedDurationSeconds, + string Notes); + +public sealed class MimirTargetedBioacousticProbePlanner(MimirTargetedBioacousticProbeOptions? options = null) +{ + private readonly MimirTargetedBioacousticProbeOptions options = options ?? new(); + + public MimirTargetedBioacousticProbePlan Plan( + IReadOnlyList residuals, + IReadOnlyList? preferredSourceIds = null, + IReadOnlyList? sourcePathologies = null) + { + var pathologyBySource = (sourcePathologies ?? []) + .GroupBy(pathology => pathology.SourceId, StringComparer.Ordinal) + .ToDictionary(group => group.Key, group => group.First(), StringComparer.Ordinal); + var sourceFilter = preferredSourceIds is { Count: > 0 } + ? preferredSourceIds.ToHashSet(StringComparer.Ordinal) + : null; + var usableResiduals = residuals + .Where(residual => residual.CenterHz >= options.MinFrequencyHz && + residual.CenterHz <= Math.Min(options.MaxFrequencyHz, options.SampleRate * 0.45) && + (sourceFilter == null || sourceFilter.Contains(residual.SourceId))) + .ToArray(); + var sourceIds = (sourceFilter ?? usableResiduals.Select(residual => residual.SourceId).ToHashSet(StringComparer.Ordinal)) + .Order(StringComparer.Ordinal) + .ToArray(); + var candidates = usableResiduals + .GroupBy(residual => QuantizeBand(residual.CenterHz)) + .Select(group => BuildCandidate(group.ToArray(), pathologyBySource)) + .Where(candidate => candidate.Priority > 0.0) + .OrderByDescending(candidate => candidate.Priority) + .ToArray(); + var selected = new List(options.MaxProbeBands); + foreach (var candidate in candidates) + { + if (selected.Any(existing => BandsTooClose(existing.CenterHz, candidate.CenterHz))) + { + continue; + } + + selected.Add(candidate); + if (selected.Count >= options.MaxProbeBands) + { + break; + } + } + + return new MimirTargetedBioacousticProbePlan( + "targeted-bioacoustic-probe.v1", + options.SampleRate, + sourceIds, + selected, + selected.Count == 0 ? 0.0 : 0.105 * selected.Count + 0.080, + "Probe bands are selected from residual uncertainty. The renderer may shape them as birdcall syllables, but the planner only owns target bands and gain budget."); + } + + private MimirTargetedBioacousticProbeBand BuildCandidate( + IReadOnlyList group, + IReadOnlyDictionary pathologyBySource) + { + var centerHz = WeightedAverage(group, residual => residual.CenterHz, residual => BandPriority(residual, pathologyBySource)); + var priority = group.Sum(residual => BandPriority(residual, pathologyBySource)) / Math.Max(1, group.Count); + var reason = group + .OrderByDescending(residual => BandPriority(residual, pathologyBySource)) + .Select(ReasonFor) + .FirstOrDefault(); + var responseEnergy = group + .Where(residual => residual.ResponseEnergy > 0.0) + .Select(residual => residual.ResponseEnergy) + .DefaultIfEmpty(options.WeakResponseThreshold) + .Average(); + var confidenceGap = group + .Select(residual => Math.Max(0.0, options.TargetConfidence - residual.Confidence)) + .DefaultIfEmpty(0.0) + .Average(); + var weakResponseBoost = responseEnergy < options.WeakResponseThreshold + ? (options.WeakResponseThreshold - responseEnergy) / options.WeakResponseThreshold + : 0.0; + var gain = Math.Clamp( + options.BaseGain * (1.0 + 1.6 * priority + 0.8 * confidenceGap + 0.9 * weakResponseBoost), + options.BaseGain, + options.MaxGain); + var halfWidth = Math.Clamp(centerHz * 0.045, 40.0, 1200.0); + return new MimirTargetedBioacousticProbeBand( + centerHz, + Math.Max(options.MinFrequencyHz, centerHz - halfWidth), + Math.Min(Math.Min(options.MaxFrequencyHz, options.SampleRate * 0.45), centerHz + halfWidth), + gain, + priority, + reason, + group.Select(residual => residual.SourceId).Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToArray()); + } + + private double QuantizeBand(double centerHz) + { + var semitone = Math.Round(12.0 * Math.Log2(centerHz / 110.0)); + return 110.0 * Math.Pow(2.0, semitone / 12.0); + } + + private bool BandsTooClose(double leftHz, double rightHz) + { + var distance = Math.Abs(Math.Log(leftHz / rightHz)); + return distance < options.MinimumBandSpacingFraction; + } + + private double BandPriority( + MimirAudioCalibrationBandResidual residual, + IReadOnlyDictionary pathologyBySource) + { + var confidenceGap = Math.Max(0.0, options.TargetConfidence - residual.Confidence) / Math.Max(0.001, options.TargetConfidence); + var residualPressure = Math.Max(0.0, Math.Abs(residual.ResidualDb) - options.ResidualDbBudget) / 12.0; + var weakResponse = residual.ResponseEnergy <= 0.0 + ? 0.35 + : Math.Max(0.0, options.WeakResponseThreshold - residual.ResponseEnergy) / options.WeakResponseThreshold; + var phasePressure = Math.Max(0.0, Math.Abs(residual.PhaseResidualRadians) - options.PhaseResidualBudgetRadians) / Math.PI; + var delayPressure = Math.Min(1.0, Math.Abs(residual.DelayResidualMicroseconds) / 25.0); + var pathologyPressure = 0.0; + if (pathologyBySource.TryGetValue(residual.SourceId, out var pathology)) + { + var highBand = residual.CenterHz >= 3_000.0 + ? Math.Clamp(Math.Log2(residual.CenterHz / 3_000.0) / 3.0, 0.0, 1.0) + : 0.0; + var noisePressure = Math.Clamp(pathology.NoiseFloorRms / 0.050, 0.0, 1.0); + var tiltPressure = Math.Clamp(pathology.LowHighTiltDb / 18.0, 0.0, 1.0); + pathologyPressure = highBand * (noisePressure * 0.35 + tiltPressure * 0.75 + Math.Clamp(pathology.HighBandConfidenceBias, 0.0, 1.0)); + } + + return confidenceGap * 1.20 + residualPressure + weakResponse * 0.85 + phasePressure * 0.75 + delayPressure * 0.40 + pathologyPressure; + } + + private MimirBioacousticProbeReason ReasonFor(MimirAudioCalibrationBandResidual residual) + { + if (residual.Confidence < 0.05 && residual.ResponseEnergy <= 0.0) + { + return MimirBioacousticProbeReason.UnmeasuredBand; + } + + if (residual.ResponseEnergy > 0.0 && residual.ResponseEnergy < options.WeakResponseThreshold) + { + return MimirBioacousticProbeReason.WeakResponse; + } + + if (Math.Abs(residual.PhaseResidualRadians) > options.PhaseResidualBudgetRadians) + { + return MimirBioacousticProbeReason.PhaseUnstable; + } + + if (Math.Abs(residual.ResidualDb) > options.ResidualDbBudget) + { + return MimirBioacousticProbeReason.HighResidual; + } + + return MimirBioacousticProbeReason.LowConfidence; + } + + private static double WeightedAverage( + IReadOnlyList group, + Func value, + Func weight) + { + var totalWeight = group.Sum(item => Math.Max(1.0e-6, weight(item))); + return group.Sum(item => value(item) * Math.Max(1.0e-6, weight(item))) / totalWeight; + } +} diff --git a/state/evidence.jsonl b/state/evidence.jsonl index fad0611..935d2b0 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -308,3 +308,4 @@ {"ts": "2026-06-01T02:42:04+01:00", "type": "decision", "status": "ok", "note": "Raven video bottleneck was not NVENC. BtbN FFmpeg ddagrab through Raven's interactive desktop session records Desktop Duplication API frames into h264_nvenc at ~56.8 fps for 1080p60 with Mimir.WasapiLoopback AAC audio in the same local MP4. The remaining sparse-frame direct proof is the TCP/SSH proof-capture path, not Raven capture/encode. start-raven-av-sender now defaults to ddagrab and keeps gdigrab only as an explicit fallback."} {"ts": "2026-06-01T02:45:51+01:00", "type": "verification", "status": "ok", "note": "Direct LAN SRT Raven proof is healthy after removing the SSH tunnel. Starfire listened on srt://0.0.0.0:5204 while Raven called 192.168.1.66:5204 with ddagrab + h264_nvenc + Mimir.WasapiLoopback AAC; artifact artifacts/runtime/stream-proof/raven-nvenc-ddagrab-lan-srt-20260601-024525.mp4 contains 564 video frames over 10.005 s (~56.5 fps) plus 48 kHz stereo AAC. The earlier sparse-frame recordings were the TCP/SSH proof path, not Raven GPU/capture/encode or LAN SRT."} {"ts": "2026-06-01T02:54:52+01:00", "type": "verification", "status": "ok", "note": "Calibrated audio composite contract now exists as MimirCalibratedAudioCompositeBuilder. The smoke consumes synthetic delayed/noisy/colored mic windows plus sync states and band responses, applies fractional alignment plus sparse measured-band equalization and confidence/noise weighting, and improves SNR from -3.171 dB uncalibrated to 15.135 dB calibrated while response flatness rises from 0.548 to 1.000. This is an offline/diagnostic contract for the future Faust/native hot lane, not a claim that live room nulls are perfectly flattened."} +{"ts": "2026-06-01T03:04:44+01:00", "type": "decision", "status": "ok", "note": "Targeted bioacoustic probing is now residual-driven instead of generic sweep-driven. MimirTargetedBioacousticProbePlanner consumes per-source spectral residuals and optional mic pathology notes such as shotgun noise floor and low-heavy tilt, then emits gain-bounded probe bands. The first smoke scopes authority to the 192 kHz Scarlett shotgun/cardioid pair and selects about 18.6 kHz unmeasured high-band evidence first, then about 9.9 kHz weak response, 3.6 kHz phase instability, and 910 Hz magnitude residual. Wider room-field use of every connected mic remains later evidence, not the initial calibration owner."} diff --git a/state/map.yaml b/state/map.yaml index efb8a14..bd6a6a2 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -644,6 +644,18 @@ accepted_design: to 15.135 dB calibrated and response flatness from 0.548 to 1.000. Faust/native DSP still owns the production hot lane; this C# path is a contract, scorer, and offline diagnostic, not a competing live mixer. + `MimirTargetedBioacousticProbePlanner` is now the residual-driven probe + planner for improving that calibration instead of blindly EQing harder. It + consumes per-source spectral residuals plus optional source pathology notes + such as high noise floor and low-heavy tilt, then emits gain-bounded probe + bands for the bioacoustic renderer. The first smoke is intentionally scoped + to the two professional 192 kHz Scarlett mics: `asio-ch0-shotgun` is + modeled as noisy and low-heavy, `asio-ch1-cardioid` as cleaner. The planner + selects about 18.6 kHz first as unmeasured high-band evidence, then about + 9.9 kHz weak response, 3.6 kHz phase instability, and 910 Hz magnitude + residual. Use the two good mics first; every connected mic can become room + field evidence later, but it should not dilute the initial direct-path + calibration loop. A fresh live Scarlett proof is preserved at `calibration/bioacoustic/complex-contour-live-20260525-063229.md`: Focusrite ASIO playback/capture measured Loopback 1 -> Loopback 2 at