Plan targeted bioacoustic probes from residuals
This commit is contained in:
parent
8df851850c
commit
8bb854859b
6 changed files with 305 additions and 1 deletions
|
|
@ -788,6 +788,11 @@ a named invariant that the native runtime cannot protect yet.
|
|||
states and measured band responses, emits a corrected composite plus scores,
|
||||
and is proved by `--calibrated-audio-composite-smoke`. Treat that as the
|
||||
contract and offline scorer for the native/Faust implementation, not as a
|
||||
parallel live mixer authority.
|
||||
parallel live mixer authority. The next matching probe loop also exists as
|
||||
`MimirTargetedBioacousticProbePlanner`: residuals and mic pathology notes
|
||||
select gain-bounded shaped probe bands for the bioacoustic renderer. Start
|
||||
with the 192 kHz shotgun/cardioid pair; only widen to every connected mic
|
||||
once those two paths have credible direct-path and frequency-response
|
||||
confidence.
|
||||
10. Keep the OBS bridge witness ledger as evidence before expanding receiver
|
||||
machinery.
|
||||
|
|
|
|||
|
|
@ -306,6 +306,19 @@ owns the production alignment, suppression, separation, and stem lane.
|
|||
synthetic delayed, colored, noisy mic paths: the uncalibrated mix measures
|
||||
-3.171 dB SNR, the calibrated composite measures 15.135 dB SNR, and response
|
||||
flatness improves from 0.548 to 1.000.
|
||||
`MimirTargetedBioacousticProbePlanner` is the matching "ask the room a better
|
||||
question" owner. It consumes per-source spectral residuals and optional source
|
||||
pathology notes, then emits gain-bounded shaped probe bands for the
|
||||
bioacoustic renderer to spend energy on. The first smoke models the two
|
||||
professional 192 kHz Scarlett mics only: `asio-ch0-shotgun` is explicitly
|
||||
noisy and low-heavy, `asio-ch1-cardioid` is cleaner, and the planner prioritizes
|
||||
the shotgun's weak/unmeasured high-band evidence before lower residuals.
|
||||
`Mimir.BufferSmoke --targeted-bioacoustic-probe-smoke` selects four probe
|
||||
bands, led by about 18.6 kHz as an unmeasured band at the max bounded gain,
|
||||
then about 9.9 kHz weak response, 3.6 kHz phase instability, and 910 Hz
|
||||
magnitude residual. This is the intended first physical strategy: improve
|
||||
shotgun/cardioid confidence at 192 kHz before asking every connected mic to
|
||||
help solve the room field.
|
||||
That real-world proof now has a first receipt:
|
||||
`calibration/bioacoustic/complex-contour-live-20260525-063229.md`. A freshly
|
||||
rendered canary-packet witness was played through Focusrite ASIO and captured
|
||||
|
|
|
|||
|
|
@ -80,6 +80,11 @@ if (args.Any(arg => string.Equals(arg, "--calibrated-audio-composite-smoke", Str
|
|||
return RunCalibratedAudioCompositeSmoke();
|
||||
}
|
||||
|
||||
if (args.Any(arg => string.Equals(arg, "--targeted-bioacoustic-probe-smoke", StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
return RunTargetedBioacousticProbeSmoke();
|
||||
}
|
||||
|
||||
if (args.Any(arg => string.Equals(arg, "--synchronized-buffer-planner-smoke", StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
return RunSynchronizedBufferPlannerSmoke();
|
||||
|
|
@ -3365,6 +3370,57 @@ static int RunCalibratedAudioCompositeSmoke()
|
|||
: 1;
|
||||
}
|
||||
|
||||
static int RunTargetedBioacousticProbeSmoke()
|
||||
{
|
||||
var residuals = new[]
|
||||
{
|
||||
new MimirAudioCalibrationBandResidual("asio-ch0-shotgun", 240.0, 0.91, 0.4, ResponseEnergy: 0.62),
|
||||
new MimirAudioCalibrationBandResidual("asio-ch1-cardioid", 240.0, 0.87, 0.6, ResponseEnergy: 0.58),
|
||||
new MimirAudioCalibrationBandResidual("asio-ch0-shotgun", 890.0, 0.52, 4.8, DelayResidualMicroseconds: 11.0, ResponseEnergy: 0.30),
|
||||
new MimirAudioCalibrationBandResidual("asio-ch1-cardioid", 910.0, 0.47, 5.2, DelayResidualMicroseconds: 14.0, ResponseEnergy: 0.27),
|
||||
new MimirAudioCalibrationBandResidual("asio-ch0-shotgun", 3_600.0, 0.44, -6.4, PhaseResidualRadians: 1.4, ResponseEnergy: 0.11),
|
||||
new MimirAudioCalibrationBandResidual("asio-ch1-cardioid", 3_690.0, 0.41, -5.8, PhaseResidualRadians: 1.2, ResponseEnergy: 0.09),
|
||||
new MimirAudioCalibrationBandResidual("asio-ch0-shotgun", 9_800.0, 0.28, -9.5, PhaseResidualRadians: 2.0, ResponseEnergy: 0.035),
|
||||
new MimirAudioCalibrationBandResidual("asio-ch1-cardioid", 9_950.0, 0.22, -10.2, PhaseResidualRadians: 1.8, ResponseEnergy: 0.030),
|
||||
new MimirAudioCalibrationBandResidual("asio-ch0-shotgun", 18_500.0, 0.03, -18.0, DelayResidualMicroseconds: 34.0, PhaseResidualRadians: 2.7, ResponseEnergy: 0.0),
|
||||
new MimirAudioCalibrationBandResidual("asio-ch1-cardioid", 18_700.0, 0.02, -16.0, DelayResidualMicroseconds: 28.0, PhaseResidualRadians: 2.4, ResponseEnergy: 0.0),
|
||||
};
|
||||
var plan = new MimirTargetedBioacousticProbePlanner().Plan(
|
||||
residuals,
|
||||
["asio-ch0-shotgun", "asio-ch1-cardioid"],
|
||||
[
|
||||
new MimirAudioCalibrationSourcePathology(
|
||||
"asio-ch0-shotgun",
|
||||
NoiseFloorRms: 0.055,
|
||||
LowHighTiltDb: 17.0,
|
||||
HighBandConfidenceBias: 0.35),
|
||||
new MimirAudioCalibrationSourcePathology(
|
||||
"asio-ch1-cardioid",
|
||||
NoiseFloorRms: 0.018,
|
||||
LowHighTiltDb: 4.0)
|
||||
]);
|
||||
var top = plan.Bands.FirstOrDefault();
|
||||
var hasLowBand = plan.Bands.Any(band => band.CenterHz is > 800.0 and < 1_000.0);
|
||||
var hasMidBand = plan.Bands.Any(band => band.CenterHz is > 3_300.0 and < 3_900.0);
|
||||
var hasHighBand = plan.Bands.Any(band => band.CenterHz is > 17_000.0 and < 20_000.0);
|
||||
Console.WriteLine(
|
||||
$"targeted-bioacoustic-probe-smoke sampleRate={plan.SampleRate} sources={string.Join(",", plan.MeasurementSourceIds)} bands={plan.Bands.Count} " +
|
||||
$"duration={plan.EstimatedDurationSeconds:0.000} top={top?.CenterHz:0}Hz reason={top?.Reason} gain={top?.Gain:0.000} priority={top?.Priority:0.000} " +
|
||||
$"selected={string.Join(",", plan.Bands.Select(band => $"{band.CenterHz:0}Hz/{band.Reason}/{band.Gain:0.000}"))}");
|
||||
|
||||
return plan.SampleRate == 192_000 &&
|
||||
plan.MeasurementSourceIds.SequenceEqual(new[] { "asio-ch0-shotgun", "asio-ch1-cardioid" }) &&
|
||||
plan.Bands.Count >= 3 &&
|
||||
hasLowBand &&
|
||||
hasMidBand &&
|
||||
hasHighBand &&
|
||||
top is { CenterHz: > 17_000.0 and < 20_000.0 } &&
|
||||
top.Reason is MimirBioacousticProbeReason.UnmeasuredBand or MimirBioacousticProbeReason.WeakResponse or MimirBioacousticProbeReason.PhaseUnstable &&
|
||||
plan.Bands.All(band => band.Gain is >= 0.018 and <= 0.085)
|
||||
? 0
|
||||
: 1;
|
||||
}
|
||||
|
||||
static int RunSynchronizedBufferPlannerSmoke()
|
||||
{
|
||||
const long t0 = 1_000_000_000;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,217 @@
|
|||
namespace Mimir.Runtime.Synchronization;
|
||||
|
||||
public enum MimirBioacousticProbeReason
|
||||
{
|
||||
LowConfidence,
|
||||
HighResidual,
|
||||
WeakResponse,
|
||||
PhaseUnstable,
|
||||
UnmeasuredBand
|
||||
}
|
||||
|
||||
public sealed record MimirAudioCalibrationBandResidual(
|
||||
string SourceId,
|
||||
double CenterHz,
|
||||
double Confidence,
|
||||
double ResidualDb,
|
||||
double DelayResidualMicroseconds = 0.0,
|
||||
double PhaseResidualRadians = 0.0,
|
||||
double ResponseEnergy = 0.0);
|
||||
|
||||
public sealed record MimirAudioCalibrationSourcePathology(
|
||||
string SourceId,
|
||||
double NoiseFloorRms = 0.0,
|
||||
double LowHighTiltDb = 0.0,
|
||||
double HighBandConfidenceBias = 0.0);
|
||||
|
||||
public sealed record MimirTargetedBioacousticProbeOptions(
|
||||
int SampleRate = 192_000,
|
||||
int MaxProbeBands = 8,
|
||||
double MinFrequencyHz = 120.0,
|
||||
double MaxFrequencyHz = 42_000.0,
|
||||
double TargetConfidence = 0.82,
|
||||
double ResidualDbBudget = 1.5,
|
||||
double WeakResponseThreshold = 0.08,
|
||||
double PhaseResidualBudgetRadians = 0.80,
|
||||
double MinimumBandSpacingFraction = 0.055,
|
||||
double BaseGain = 0.018,
|
||||
double MaxGain = 0.085);
|
||||
|
||||
public sealed record MimirTargetedBioacousticProbeBand(
|
||||
double CenterHz,
|
||||
double StartHz,
|
||||
double EndHz,
|
||||
double Gain,
|
||||
double Priority,
|
||||
MimirBioacousticProbeReason Reason,
|
||||
IReadOnlyList<string> SourceIds);
|
||||
|
||||
public sealed record MimirTargetedBioacousticProbePlan(
|
||||
string Id,
|
||||
int SampleRate,
|
||||
IReadOnlyList<string> MeasurementSourceIds,
|
||||
IReadOnlyList<MimirTargetedBioacousticProbeBand> Bands,
|
||||
double EstimatedDurationSeconds,
|
||||
string Notes);
|
||||
|
||||
public sealed class MimirTargetedBioacousticProbePlanner(MimirTargetedBioacousticProbeOptions? options = null)
|
||||
{
|
||||
private readonly MimirTargetedBioacousticProbeOptions options = options ?? new();
|
||||
|
||||
public MimirTargetedBioacousticProbePlan Plan(
|
||||
IReadOnlyList<MimirAudioCalibrationBandResidual> residuals,
|
||||
IReadOnlyList<string>? preferredSourceIds = null,
|
||||
IReadOnlyList<MimirAudioCalibrationSourcePathology>? sourcePathologies = null)
|
||||
{
|
||||
var pathologyBySource = (sourcePathologies ?? [])
|
||||
.GroupBy(pathology => pathology.SourceId, StringComparer.Ordinal)
|
||||
.ToDictionary(group => group.Key, group => group.First(), StringComparer.Ordinal);
|
||||
var sourceFilter = preferredSourceIds is { Count: > 0 }
|
||||
? preferredSourceIds.ToHashSet(StringComparer.Ordinal)
|
||||
: null;
|
||||
var usableResiduals = residuals
|
||||
.Where(residual => residual.CenterHz >= options.MinFrequencyHz &&
|
||||
residual.CenterHz <= Math.Min(options.MaxFrequencyHz, options.SampleRate * 0.45) &&
|
||||
(sourceFilter == null || sourceFilter.Contains(residual.SourceId)))
|
||||
.ToArray();
|
||||
var sourceIds = (sourceFilter ?? usableResiduals.Select(residual => residual.SourceId).ToHashSet(StringComparer.Ordinal))
|
||||
.Order(StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
var candidates = usableResiduals
|
||||
.GroupBy(residual => QuantizeBand(residual.CenterHz))
|
||||
.Select(group => BuildCandidate(group.ToArray(), pathologyBySource))
|
||||
.Where(candidate => candidate.Priority > 0.0)
|
||||
.OrderByDescending(candidate => candidate.Priority)
|
||||
.ToArray();
|
||||
var selected = new List<MimirTargetedBioacousticProbeBand>(options.MaxProbeBands);
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
if (selected.Any(existing => BandsTooClose(existing.CenterHz, candidate.CenterHz)))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
selected.Add(candidate);
|
||||
if (selected.Count >= options.MaxProbeBands)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return new MimirTargetedBioacousticProbePlan(
|
||||
"targeted-bioacoustic-probe.v1",
|
||||
options.SampleRate,
|
||||
sourceIds,
|
||||
selected,
|
||||
selected.Count == 0 ? 0.0 : 0.105 * selected.Count + 0.080,
|
||||
"Probe bands are selected from residual uncertainty. The renderer may shape them as birdcall syllables, but the planner only owns target bands and gain budget.");
|
||||
}
|
||||
|
||||
private MimirTargetedBioacousticProbeBand BuildCandidate(
|
||||
IReadOnlyList<MimirAudioCalibrationBandResidual> group,
|
||||
IReadOnlyDictionary<string, MimirAudioCalibrationSourcePathology> pathologyBySource)
|
||||
{
|
||||
var centerHz = WeightedAverage(group, residual => residual.CenterHz, residual => BandPriority(residual, pathologyBySource));
|
||||
var priority = group.Sum(residual => BandPriority(residual, pathologyBySource)) / Math.Max(1, group.Count);
|
||||
var reason = group
|
||||
.OrderByDescending(residual => BandPriority(residual, pathologyBySource))
|
||||
.Select(ReasonFor)
|
||||
.FirstOrDefault();
|
||||
var responseEnergy = group
|
||||
.Where(residual => residual.ResponseEnergy > 0.0)
|
||||
.Select(residual => residual.ResponseEnergy)
|
||||
.DefaultIfEmpty(options.WeakResponseThreshold)
|
||||
.Average();
|
||||
var confidenceGap = group
|
||||
.Select(residual => Math.Max(0.0, options.TargetConfidence - residual.Confidence))
|
||||
.DefaultIfEmpty(0.0)
|
||||
.Average();
|
||||
var weakResponseBoost = responseEnergy < options.WeakResponseThreshold
|
||||
? (options.WeakResponseThreshold - responseEnergy) / options.WeakResponseThreshold
|
||||
: 0.0;
|
||||
var gain = Math.Clamp(
|
||||
options.BaseGain * (1.0 + 1.6 * priority + 0.8 * confidenceGap + 0.9 * weakResponseBoost),
|
||||
options.BaseGain,
|
||||
options.MaxGain);
|
||||
var halfWidth = Math.Clamp(centerHz * 0.045, 40.0, 1200.0);
|
||||
return new MimirTargetedBioacousticProbeBand(
|
||||
centerHz,
|
||||
Math.Max(options.MinFrequencyHz, centerHz - halfWidth),
|
||||
Math.Min(Math.Min(options.MaxFrequencyHz, options.SampleRate * 0.45), centerHz + halfWidth),
|
||||
gain,
|
||||
priority,
|
||||
reason,
|
||||
group.Select(residual => residual.SourceId).Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToArray());
|
||||
}
|
||||
|
||||
private double QuantizeBand(double centerHz)
|
||||
{
|
||||
var semitone = Math.Round(12.0 * Math.Log2(centerHz / 110.0));
|
||||
return 110.0 * Math.Pow(2.0, semitone / 12.0);
|
||||
}
|
||||
|
||||
private bool BandsTooClose(double leftHz, double rightHz)
|
||||
{
|
||||
var distance = Math.Abs(Math.Log(leftHz / rightHz));
|
||||
return distance < options.MinimumBandSpacingFraction;
|
||||
}
|
||||
|
||||
private double BandPriority(
|
||||
MimirAudioCalibrationBandResidual residual,
|
||||
IReadOnlyDictionary<string, MimirAudioCalibrationSourcePathology> pathologyBySource)
|
||||
{
|
||||
var confidenceGap = Math.Max(0.0, options.TargetConfidence - residual.Confidence) / Math.Max(0.001, options.TargetConfidence);
|
||||
var residualPressure = Math.Max(0.0, Math.Abs(residual.ResidualDb) - options.ResidualDbBudget) / 12.0;
|
||||
var weakResponse = residual.ResponseEnergy <= 0.0
|
||||
? 0.35
|
||||
: Math.Max(0.0, options.WeakResponseThreshold - residual.ResponseEnergy) / options.WeakResponseThreshold;
|
||||
var phasePressure = Math.Max(0.0, Math.Abs(residual.PhaseResidualRadians) - options.PhaseResidualBudgetRadians) / Math.PI;
|
||||
var delayPressure = Math.Min(1.0, Math.Abs(residual.DelayResidualMicroseconds) / 25.0);
|
||||
var pathologyPressure = 0.0;
|
||||
if (pathologyBySource.TryGetValue(residual.SourceId, out var pathology))
|
||||
{
|
||||
var highBand = residual.CenterHz >= 3_000.0
|
||||
? Math.Clamp(Math.Log2(residual.CenterHz / 3_000.0) / 3.0, 0.0, 1.0)
|
||||
: 0.0;
|
||||
var noisePressure = Math.Clamp(pathology.NoiseFloorRms / 0.050, 0.0, 1.0);
|
||||
var tiltPressure = Math.Clamp(pathology.LowHighTiltDb / 18.0, 0.0, 1.0);
|
||||
pathologyPressure = highBand * (noisePressure * 0.35 + tiltPressure * 0.75 + Math.Clamp(pathology.HighBandConfidenceBias, 0.0, 1.0));
|
||||
}
|
||||
|
||||
return confidenceGap * 1.20 + residualPressure + weakResponse * 0.85 + phasePressure * 0.75 + delayPressure * 0.40 + pathologyPressure;
|
||||
}
|
||||
|
||||
private MimirBioacousticProbeReason ReasonFor(MimirAudioCalibrationBandResidual residual)
|
||||
{
|
||||
if (residual.Confidence < 0.05 && residual.ResponseEnergy <= 0.0)
|
||||
{
|
||||
return MimirBioacousticProbeReason.UnmeasuredBand;
|
||||
}
|
||||
|
||||
if (residual.ResponseEnergy > 0.0 && residual.ResponseEnergy < options.WeakResponseThreshold)
|
||||
{
|
||||
return MimirBioacousticProbeReason.WeakResponse;
|
||||
}
|
||||
|
||||
if (Math.Abs(residual.PhaseResidualRadians) > options.PhaseResidualBudgetRadians)
|
||||
{
|
||||
return MimirBioacousticProbeReason.PhaseUnstable;
|
||||
}
|
||||
|
||||
if (Math.Abs(residual.ResidualDb) > options.ResidualDbBudget)
|
||||
{
|
||||
return MimirBioacousticProbeReason.HighResidual;
|
||||
}
|
||||
|
||||
return MimirBioacousticProbeReason.LowConfidence;
|
||||
}
|
||||
|
||||
private static double WeightedAverage(
|
||||
IReadOnlyList<MimirAudioCalibrationBandResidual> group,
|
||||
Func<MimirAudioCalibrationBandResidual, double> value,
|
||||
Func<MimirAudioCalibrationBandResidual, double> weight)
|
||||
{
|
||||
var totalWeight = group.Sum(item => Math.Max(1.0e-6, weight(item)));
|
||||
return group.Sum(item => value(item) * Math.Max(1.0e-6, weight(item))) / totalWeight;
|
||||
}
|
||||
}
|
||||
|
|
@ -308,3 +308,4 @@
|
|||
{"ts": "2026-06-01T02:42:04+01:00", "type": "decision", "status": "ok", "note": "Raven video bottleneck was not NVENC. BtbN FFmpeg ddagrab through Raven's interactive desktop session records Desktop Duplication API frames into h264_nvenc at ~56.8 fps for 1080p60 with Mimir.WasapiLoopback AAC audio in the same local MP4. The remaining sparse-frame direct proof is the TCP/SSH proof-capture path, not Raven capture/encode. start-raven-av-sender now defaults to ddagrab and keeps gdigrab only as an explicit fallback."}
|
||||
{"ts": "2026-06-01T02:45:51+01:00", "type": "verification", "status": "ok", "note": "Direct LAN SRT Raven proof is healthy after removing the SSH tunnel. Starfire listened on srt://0.0.0.0:5204 while Raven called 192.168.1.66:5204 with ddagrab + h264_nvenc + Mimir.WasapiLoopback AAC; artifact artifacts/runtime/stream-proof/raven-nvenc-ddagrab-lan-srt-20260601-024525.mp4 contains 564 video frames over 10.005 s (~56.5 fps) plus 48 kHz stereo AAC. The earlier sparse-frame recordings were the TCP/SSH proof path, not Raven GPU/capture/encode or LAN SRT."}
|
||||
{"ts": "2026-06-01T02:54:52+01:00", "type": "verification", "status": "ok", "note": "Calibrated audio composite contract now exists as MimirCalibratedAudioCompositeBuilder. The smoke consumes synthetic delayed/noisy/colored mic windows plus sync states and band responses, applies fractional alignment plus sparse measured-band equalization and confidence/noise weighting, and improves SNR from -3.171 dB uncalibrated to 15.135 dB calibrated while response flatness rises from 0.548 to 1.000. This is an offline/diagnostic contract for the future Faust/native hot lane, not a claim that live room nulls are perfectly flattened."}
|
||||
{"ts": "2026-06-01T03:04:44+01:00", "type": "decision", "status": "ok", "note": "Targeted bioacoustic probing is now residual-driven instead of generic sweep-driven. MimirTargetedBioacousticProbePlanner consumes per-source spectral residuals and optional mic pathology notes such as shotgun noise floor and low-heavy tilt, then emits gain-bounded probe bands. The first smoke scopes authority to the 192 kHz Scarlett shotgun/cardioid pair and selects about 18.6 kHz unmeasured high-band evidence first, then about 9.9 kHz weak response, 3.6 kHz phase instability, and 910 Hz magnitude residual. Wider room-field use of every connected mic remains later evidence, not the initial calibration owner."}
|
||||
|
|
|
|||
|
|
@ -644,6 +644,18 @@ accepted_design:
|
|||
to 15.135 dB calibrated and response flatness from 0.548 to 1.000.
|
||||
Faust/native DSP still owns the production hot lane; this C# path is a
|
||||
contract, scorer, and offline diagnostic, not a competing live mixer.
|
||||
`MimirTargetedBioacousticProbePlanner` is now the residual-driven probe
|
||||
planner for improving that calibration instead of blindly EQing harder. It
|
||||
consumes per-source spectral residuals plus optional source pathology notes
|
||||
such as high noise floor and low-heavy tilt, then emits gain-bounded probe
|
||||
bands for the bioacoustic renderer. The first smoke is intentionally scoped
|
||||
to the two professional 192 kHz Scarlett mics: `asio-ch0-shotgun` is
|
||||
modeled as noisy and low-heavy, `asio-ch1-cardioid` as cleaner. The planner
|
||||
selects about 18.6 kHz first as unmeasured high-band evidence, then about
|
||||
9.9 kHz weak response, 3.6 kHz phase instability, and 910 Hz magnitude
|
||||
residual. Use the two good mics first; every connected mic can become room
|
||||
field evidence later, but it should not dilute the initial direct-path
|
||||
calibration loop.
|
||||
A fresh live Scarlett proof is preserved at
|
||||
`calibration/bioacoustic/complex-contour-live-20260525-063229.md`:
|
||||
Focusrite ASIO playback/capture measured Loopback 1 -> Loopback 2 at
|
||||
|
|
|
|||
Loading…
Reference in a new issue