Plan targeted bioacoustic probes from residuals

This commit is contained in:
Meta Vens 2026-06-01 03:05:06 +01:00
commit 8bb854859b
6 changed files with 305 additions and 1 deletions

View file

@ -788,6 +788,11 @@ a named invariant that the native runtime cannot protect yet.
states and measured band responses, emits a corrected composite plus scores,
and is proved by `--calibrated-audio-composite-smoke`. Treat that as the
contract and offline scorer for the native/Faust implementation, not as a
parallel live mixer authority.
parallel live mixer authority. The next matching probe loop also exists as
`MimirTargetedBioacousticProbePlanner`: residuals and mic pathology notes
select gain-bounded shaped probe bands for the bioacoustic renderer. Start
with the 192 kHz shotgun/cardioid pair; only widen to every connected mic
once those two paths have credible direct-path and frequency-response
confidence.
10. Keep the OBS bridge witness ledger as evidence before expanding receiver
machinery.

View file

@ -306,6 +306,19 @@ owns the production alignment, suppression, separation, and stem lane.
synthetic delayed, colored, noisy mic paths: the uncalibrated mix measures
-3.171 dB SNR, the calibrated composite measures 15.135 dB SNR, and response
flatness improves from 0.548 to 1.000.
`MimirTargetedBioacousticProbePlanner` is the matching "ask the room a better
question" owner. It consumes per-source spectral residuals and optional source
pathology notes, then emits gain-bounded shaped probe bands for the
bioacoustic renderer to spend energy on. The first smoke models the two
professional 192 kHz Scarlett mics only: `asio-ch0-shotgun` is explicitly
noisy and low-heavy, `asio-ch1-cardioid` is cleaner, and the planner prioritizes
the shotgun's weak/unmeasured high-band evidence before lower residuals.
`Mimir.BufferSmoke --targeted-bioacoustic-probe-smoke` selects four probe
bands, led by about 18.6 kHz as an unmeasured band at the max bounded gain,
then about 9.9 kHz weak response, 3.6 kHz phase instability, and 910 Hz
magnitude residual. This is the intended first physical strategy: improve
shotgun/cardioid confidence at 192 kHz before asking every connected mic to
help solve the room field.
That real-world proof now has a first receipt:
`calibration/bioacoustic/complex-contour-live-20260525-063229.md`. A freshly
rendered canary-packet witness was played through Focusrite ASIO and captured

View file

@ -80,6 +80,11 @@ if (args.Any(arg => string.Equals(arg, "--calibrated-audio-composite-smoke", Str
return RunCalibratedAudioCompositeSmoke();
}
if (args.Any(arg => string.Equals(arg, "--targeted-bioacoustic-probe-smoke", StringComparison.OrdinalIgnoreCase)))
{
return RunTargetedBioacousticProbeSmoke();
}
if (args.Any(arg => string.Equals(arg, "--synchronized-buffer-planner-smoke", StringComparison.OrdinalIgnoreCase)))
{
return RunSynchronizedBufferPlannerSmoke();
@ -3365,6 +3370,57 @@ static int RunCalibratedAudioCompositeSmoke()
: 1;
}
static int RunTargetedBioacousticProbeSmoke()
{
var residuals = new[]
{
new MimirAudioCalibrationBandResidual("asio-ch0-shotgun", 240.0, 0.91, 0.4, ResponseEnergy: 0.62),
new MimirAudioCalibrationBandResidual("asio-ch1-cardioid", 240.0, 0.87, 0.6, ResponseEnergy: 0.58),
new MimirAudioCalibrationBandResidual("asio-ch0-shotgun", 890.0, 0.52, 4.8, DelayResidualMicroseconds: 11.0, ResponseEnergy: 0.30),
new MimirAudioCalibrationBandResidual("asio-ch1-cardioid", 910.0, 0.47, 5.2, DelayResidualMicroseconds: 14.0, ResponseEnergy: 0.27),
new MimirAudioCalibrationBandResidual("asio-ch0-shotgun", 3_600.0, 0.44, -6.4, PhaseResidualRadians: 1.4, ResponseEnergy: 0.11),
new MimirAudioCalibrationBandResidual("asio-ch1-cardioid", 3_690.0, 0.41, -5.8, PhaseResidualRadians: 1.2, ResponseEnergy: 0.09),
new MimirAudioCalibrationBandResidual("asio-ch0-shotgun", 9_800.0, 0.28, -9.5, PhaseResidualRadians: 2.0, ResponseEnergy: 0.035),
new MimirAudioCalibrationBandResidual("asio-ch1-cardioid", 9_950.0, 0.22, -10.2, PhaseResidualRadians: 1.8, ResponseEnergy: 0.030),
new MimirAudioCalibrationBandResidual("asio-ch0-shotgun", 18_500.0, 0.03, -18.0, DelayResidualMicroseconds: 34.0, PhaseResidualRadians: 2.7, ResponseEnergy: 0.0),
new MimirAudioCalibrationBandResidual("asio-ch1-cardioid", 18_700.0, 0.02, -16.0, DelayResidualMicroseconds: 28.0, PhaseResidualRadians: 2.4, ResponseEnergy: 0.0),
};
var plan = new MimirTargetedBioacousticProbePlanner().Plan(
residuals,
["asio-ch0-shotgun", "asio-ch1-cardioid"],
[
new MimirAudioCalibrationSourcePathology(
"asio-ch0-shotgun",
NoiseFloorRms: 0.055,
LowHighTiltDb: 17.0,
HighBandConfidenceBias: 0.35),
new MimirAudioCalibrationSourcePathology(
"asio-ch1-cardioid",
NoiseFloorRms: 0.018,
LowHighTiltDb: 4.0)
]);
var top = plan.Bands.FirstOrDefault();
var hasLowBand = plan.Bands.Any(band => band.CenterHz is > 800.0 and < 1_000.0);
var hasMidBand = plan.Bands.Any(band => band.CenterHz is > 3_300.0 and < 3_900.0);
var hasHighBand = plan.Bands.Any(band => band.CenterHz is > 17_000.0 and < 20_000.0);
Console.WriteLine(
$"targeted-bioacoustic-probe-smoke sampleRate={plan.SampleRate} sources={string.Join(",", plan.MeasurementSourceIds)} bands={plan.Bands.Count} " +
$"duration={plan.EstimatedDurationSeconds:0.000} top={top?.CenterHz:0}Hz reason={top?.Reason} gain={top?.Gain:0.000} priority={top?.Priority:0.000} " +
$"selected={string.Join(",", plan.Bands.Select(band => $"{band.CenterHz:0}Hz/{band.Reason}/{band.Gain:0.000}"))}");
return plan.SampleRate == 192_000 &&
plan.MeasurementSourceIds.SequenceEqual(new[] { "asio-ch0-shotgun", "asio-ch1-cardioid" }) &&
plan.Bands.Count >= 3 &&
hasLowBand &&
hasMidBand &&
hasHighBand &&
top is { CenterHz: > 17_000.0 and < 20_000.0 } &&
top.Reason is MimirBioacousticProbeReason.UnmeasuredBand or MimirBioacousticProbeReason.WeakResponse or MimirBioacousticProbeReason.PhaseUnstable &&
plan.Bands.All(band => band.Gain is >= 0.018 and <= 0.085)
? 0
: 1;
}
static int RunSynchronizedBufferPlannerSmoke()
{
const long t0 = 1_000_000_000;

View file

@ -0,0 +1,217 @@
namespace Mimir.Runtime.Synchronization;
public enum MimirBioacousticProbeReason
{
LowConfidence,
HighResidual,
WeakResponse,
PhaseUnstable,
UnmeasuredBand
}
public sealed record MimirAudioCalibrationBandResidual(
string SourceId,
double CenterHz,
double Confidence,
double ResidualDb,
double DelayResidualMicroseconds = 0.0,
double PhaseResidualRadians = 0.0,
double ResponseEnergy = 0.0);
public sealed record MimirAudioCalibrationSourcePathology(
string SourceId,
double NoiseFloorRms = 0.0,
double LowHighTiltDb = 0.0,
double HighBandConfidenceBias = 0.0);
public sealed record MimirTargetedBioacousticProbeOptions(
int SampleRate = 192_000,
int MaxProbeBands = 8,
double MinFrequencyHz = 120.0,
double MaxFrequencyHz = 42_000.0,
double TargetConfidence = 0.82,
double ResidualDbBudget = 1.5,
double WeakResponseThreshold = 0.08,
double PhaseResidualBudgetRadians = 0.80,
double MinimumBandSpacingFraction = 0.055,
double BaseGain = 0.018,
double MaxGain = 0.085);
public sealed record MimirTargetedBioacousticProbeBand(
double CenterHz,
double StartHz,
double EndHz,
double Gain,
double Priority,
MimirBioacousticProbeReason Reason,
IReadOnlyList<string> SourceIds);
public sealed record MimirTargetedBioacousticProbePlan(
string Id,
int SampleRate,
IReadOnlyList<string> MeasurementSourceIds,
IReadOnlyList<MimirTargetedBioacousticProbeBand> Bands,
double EstimatedDurationSeconds,
string Notes);
public sealed class MimirTargetedBioacousticProbePlanner(MimirTargetedBioacousticProbeOptions? options = null)
{
private readonly MimirTargetedBioacousticProbeOptions options = options ?? new();
public MimirTargetedBioacousticProbePlan Plan(
IReadOnlyList<MimirAudioCalibrationBandResidual> residuals,
IReadOnlyList<string>? preferredSourceIds = null,
IReadOnlyList<MimirAudioCalibrationSourcePathology>? sourcePathologies = null)
{
var pathologyBySource = (sourcePathologies ?? [])
.GroupBy(pathology => pathology.SourceId, StringComparer.Ordinal)
.ToDictionary(group => group.Key, group => group.First(), StringComparer.Ordinal);
var sourceFilter = preferredSourceIds is { Count: > 0 }
? preferredSourceIds.ToHashSet(StringComparer.Ordinal)
: null;
var usableResiduals = residuals
.Where(residual => residual.CenterHz >= options.MinFrequencyHz &&
residual.CenterHz <= Math.Min(options.MaxFrequencyHz, options.SampleRate * 0.45) &&
(sourceFilter == null || sourceFilter.Contains(residual.SourceId)))
.ToArray();
var sourceIds = (sourceFilter ?? usableResiduals.Select(residual => residual.SourceId).ToHashSet(StringComparer.Ordinal))
.Order(StringComparer.Ordinal)
.ToArray();
var candidates = usableResiduals
.GroupBy(residual => QuantizeBand(residual.CenterHz))
.Select(group => BuildCandidate(group.ToArray(), pathologyBySource))
.Where(candidate => candidate.Priority > 0.0)
.OrderByDescending(candidate => candidate.Priority)
.ToArray();
var selected = new List<MimirTargetedBioacousticProbeBand>(options.MaxProbeBands);
foreach (var candidate in candidates)
{
if (selected.Any(existing => BandsTooClose(existing.CenterHz, candidate.CenterHz)))
{
continue;
}
selected.Add(candidate);
if (selected.Count >= options.MaxProbeBands)
{
break;
}
}
return new MimirTargetedBioacousticProbePlan(
"targeted-bioacoustic-probe.v1",
options.SampleRate,
sourceIds,
selected,
selected.Count == 0 ? 0.0 : 0.105 * selected.Count + 0.080,
"Probe bands are selected from residual uncertainty. The renderer may shape them as birdcall syllables, but the planner only owns target bands and gain budget.");
}
private MimirTargetedBioacousticProbeBand BuildCandidate(
IReadOnlyList<MimirAudioCalibrationBandResidual> group,
IReadOnlyDictionary<string, MimirAudioCalibrationSourcePathology> pathologyBySource)
{
var centerHz = WeightedAverage(group, residual => residual.CenterHz, residual => BandPriority(residual, pathologyBySource));
var priority = group.Sum(residual => BandPriority(residual, pathologyBySource)) / Math.Max(1, group.Count);
var reason = group
.OrderByDescending(residual => BandPriority(residual, pathologyBySource))
.Select(ReasonFor)
.FirstOrDefault();
var responseEnergy = group
.Where(residual => residual.ResponseEnergy > 0.0)
.Select(residual => residual.ResponseEnergy)
.DefaultIfEmpty(options.WeakResponseThreshold)
.Average();
var confidenceGap = group
.Select(residual => Math.Max(0.0, options.TargetConfidence - residual.Confidence))
.DefaultIfEmpty(0.0)
.Average();
var weakResponseBoost = responseEnergy < options.WeakResponseThreshold
? (options.WeakResponseThreshold - responseEnergy) / options.WeakResponseThreshold
: 0.0;
var gain = Math.Clamp(
options.BaseGain * (1.0 + 1.6 * priority + 0.8 * confidenceGap + 0.9 * weakResponseBoost),
options.BaseGain,
options.MaxGain);
var halfWidth = Math.Clamp(centerHz * 0.045, 40.0, 1200.0);
return new MimirTargetedBioacousticProbeBand(
centerHz,
Math.Max(options.MinFrequencyHz, centerHz - halfWidth),
Math.Min(Math.Min(options.MaxFrequencyHz, options.SampleRate * 0.45), centerHz + halfWidth),
gain,
priority,
reason,
group.Select(residual => residual.SourceId).Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToArray());
}
private double QuantizeBand(double centerHz)
{
var semitone = Math.Round(12.0 * Math.Log2(centerHz / 110.0));
return 110.0 * Math.Pow(2.0, semitone / 12.0);
}
private bool BandsTooClose(double leftHz, double rightHz)
{
var distance = Math.Abs(Math.Log(leftHz / rightHz));
return distance < options.MinimumBandSpacingFraction;
}
private double BandPriority(
MimirAudioCalibrationBandResidual residual,
IReadOnlyDictionary<string, MimirAudioCalibrationSourcePathology> pathologyBySource)
{
var confidenceGap = Math.Max(0.0, options.TargetConfidence - residual.Confidence) / Math.Max(0.001, options.TargetConfidence);
var residualPressure = Math.Max(0.0, Math.Abs(residual.ResidualDb) - options.ResidualDbBudget) / 12.0;
var weakResponse = residual.ResponseEnergy <= 0.0
? 0.35
: Math.Max(0.0, options.WeakResponseThreshold - residual.ResponseEnergy) / options.WeakResponseThreshold;
var phasePressure = Math.Max(0.0, Math.Abs(residual.PhaseResidualRadians) - options.PhaseResidualBudgetRadians) / Math.PI;
var delayPressure = Math.Min(1.0, Math.Abs(residual.DelayResidualMicroseconds) / 25.0);
var pathologyPressure = 0.0;
if (pathologyBySource.TryGetValue(residual.SourceId, out var pathology))
{
var highBand = residual.CenterHz >= 3_000.0
? Math.Clamp(Math.Log2(residual.CenterHz / 3_000.0) / 3.0, 0.0, 1.0)
: 0.0;
var noisePressure = Math.Clamp(pathology.NoiseFloorRms / 0.050, 0.0, 1.0);
var tiltPressure = Math.Clamp(pathology.LowHighTiltDb / 18.0, 0.0, 1.0);
pathologyPressure = highBand * (noisePressure * 0.35 + tiltPressure * 0.75 + Math.Clamp(pathology.HighBandConfidenceBias, 0.0, 1.0));
}
return confidenceGap * 1.20 + residualPressure + weakResponse * 0.85 + phasePressure * 0.75 + delayPressure * 0.40 + pathologyPressure;
}
private MimirBioacousticProbeReason ReasonFor(MimirAudioCalibrationBandResidual residual)
{
if (residual.Confidence < 0.05 && residual.ResponseEnergy <= 0.0)
{
return MimirBioacousticProbeReason.UnmeasuredBand;
}
if (residual.ResponseEnergy > 0.0 && residual.ResponseEnergy < options.WeakResponseThreshold)
{
return MimirBioacousticProbeReason.WeakResponse;
}
if (Math.Abs(residual.PhaseResidualRadians) > options.PhaseResidualBudgetRadians)
{
return MimirBioacousticProbeReason.PhaseUnstable;
}
if (Math.Abs(residual.ResidualDb) > options.ResidualDbBudget)
{
return MimirBioacousticProbeReason.HighResidual;
}
return MimirBioacousticProbeReason.LowConfidence;
}
private static double WeightedAverage(
IReadOnlyList<MimirAudioCalibrationBandResidual> group,
Func<MimirAudioCalibrationBandResidual, double> value,
Func<MimirAudioCalibrationBandResidual, double> weight)
{
var totalWeight = group.Sum(item => Math.Max(1.0e-6, weight(item)));
return group.Sum(item => value(item) * Math.Max(1.0e-6, weight(item))) / totalWeight;
}
}

View file

@ -308,3 +308,4 @@
{"ts": "2026-06-01T02:42:04+01:00", "type": "decision", "status": "ok", "note": "Raven video bottleneck was not NVENC. BtbN FFmpeg ddagrab through Raven's interactive desktop session records Desktop Duplication API frames into h264_nvenc at ~56.8 fps for 1080p60 with Mimir.WasapiLoopback AAC audio in the same local MP4. The remaining sparse-frame direct proof is the TCP/SSH proof-capture path, not Raven capture/encode. start-raven-av-sender now defaults to ddagrab and keeps gdigrab only as an explicit fallback."}
{"ts": "2026-06-01T02:45:51+01:00", "type": "verification", "status": "ok", "note": "Direct LAN SRT Raven proof is healthy after removing the SSH tunnel. Starfire listened on srt://0.0.0.0:5204 while Raven called 192.168.1.66:5204 with ddagrab + h264_nvenc + Mimir.WasapiLoopback AAC; artifact artifacts/runtime/stream-proof/raven-nvenc-ddagrab-lan-srt-20260601-024525.mp4 contains 564 video frames over 10.005 s (~56.5 fps) plus 48 kHz stereo AAC. The earlier sparse-frame recordings were the TCP/SSH proof path, not Raven GPU/capture/encode or LAN SRT."}
{"ts": "2026-06-01T02:54:52+01:00", "type": "verification", "status": "ok", "note": "Calibrated audio composite contract now exists as MimirCalibratedAudioCompositeBuilder. The smoke consumes synthetic delayed/noisy/colored mic windows plus sync states and band responses, applies fractional alignment plus sparse measured-band equalization and confidence/noise weighting, and improves SNR from -3.171 dB uncalibrated to 15.135 dB calibrated while response flatness rises from 0.548 to 1.000. This is an offline/diagnostic contract for the future Faust/native hot lane, not a claim that live room nulls are perfectly flattened."}
{"ts": "2026-06-01T03:04:44+01:00", "type": "decision", "status": "ok", "note": "Targeted bioacoustic probing is now residual-driven instead of generic sweep-driven. MimirTargetedBioacousticProbePlanner consumes per-source spectral residuals and optional mic pathology notes such as shotgun noise floor and low-heavy tilt, then emits gain-bounded probe bands. The first smoke scopes authority to the 192 kHz Scarlett shotgun/cardioid pair and selects about 18.6 kHz unmeasured high-band evidence first, then about 9.9 kHz weak response, 3.6 kHz phase instability, and 910 Hz magnitude residual. Wider room-field use of every connected mic remains later evidence, not the initial calibration owner."}

View file

@ -644,6 +644,18 @@ accepted_design:
to 15.135 dB calibrated and response flatness from 0.548 to 1.000.
Faust/native DSP still owns the production hot lane; this C# path is a
contract, scorer, and offline diagnostic, not a competing live mixer.
`MimirTargetedBioacousticProbePlanner` is now the residual-driven probe
planner for improving that calibration instead of blindly EQing harder. It
consumes per-source spectral residuals plus optional source pathology notes
such as high noise floor and low-heavy tilt, then emits gain-bounded probe
bands for the bioacoustic renderer. The first smoke is intentionally scoped
to the two professional 192 kHz Scarlett mics: `asio-ch0-shotgun` is
modeled as noisy and low-heavy, `asio-ch1-cardioid` as cleaner. The planner
selects about 18.6 kHz first as unmeasured high-band evidence, then about
9.9 kHz weak response, 3.6 kHz phase instability, and 910 Hz magnitude
residual. Use the two good mics first; every connected mic can become room
field evidence later, but it should not dilute the initial direct-path
calibration loop.
A fresh live Scarlett proof is preserved at
`calibration/bioacoustic/complex-contour-live-20260525-063229.md`:
Focusrite ASIO playback/capture measured Loopback 1 -> Loopback 2 at