Make hybrid watermark unobtrusive
This commit is contained in:
Meta Vens
parent
aae1ec7bca
commit
8ee6b42b62
10 changed files with 82 additions and 19 deletions
|
|
@ -3,7 +3,8 @@
|
|||
"audioSync": {
|
||||
"mode": "hybrid",
|
||||
"referenceSourceId": "loopback-scarlett-speakers",
|
||||
"calibrationGain": 2.0
|
||||
"calibrationGain": 2.0,
|
||||
"watermarkGain": 0.035
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -3,7 +3,8 @@
|
|||
"audioSync": {
|
||||
"mode": "hybrid",
|
||||
"referenceSourceId": "loopback-scarlett-speakers",
|
||||
"calibrationGain": 2.0
|
||||
"calibrationGain": 2.0,
|
||||
"watermarkGain": 0.035
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
|
|
@ -34,7 +35,7 @@
|
|||
"320",
|
||||
"240",
|
||||
"187",
|
||||
"30",
|
||||
"3600",
|
||||
"--emit-json-frames"
|
||||
],
|
||||
"acceptSourceIds": [
|
||||
|
|
@ -55,7 +56,7 @@
|
|||
"--endpoint",
|
||||
"{0.0.1.00000000}.{16af9455-4539-4de2-8172-563d991d9e34}",
|
||||
"--seconds",
|
||||
"30",
|
||||
"3600",
|
||||
"--emit-json-blocks",
|
||||
"--include-samples"
|
||||
]
|
||||
|
|
@ -73,7 +74,7 @@
|
|||
"--endpoint",
|
||||
"{0.0.1.00000000}.{abc7fd11-3b7d-4cfa-82ee-8767dae13fac}",
|
||||
"--seconds",
|
||||
"30",
|
||||
"3600",
|
||||
"--emit-json-blocks",
|
||||
"--include-samples"
|
||||
]
|
||||
|
|
@ -91,7 +92,7 @@
|
|||
"--endpoint",
|
||||
"{0.0.1.00000000}.{ef1fdd6f-c721-4d5a-b183-5bd81dc57c1f}",
|
||||
"--seconds",
|
||||
"30",
|
||||
"3600",
|
||||
"--emit-json-blocks",
|
||||
"--include-samples"
|
||||
]
|
||||
|
|
@ -109,7 +110,7 @@
|
|||
"--endpoint",
|
||||
"{0.0.1.00000000}.{57befef3-599c-4c73-a0cd-9539ffc2f54f}",
|
||||
"--seconds",
|
||||
"30",
|
||||
"3600",
|
||||
"--emit-json-blocks",
|
||||
"--include-samples"
|
||||
]
|
||||
|
|
@ -127,7 +128,7 @@
|
|||
"--endpoint",
|
||||
"{0.0.1.00000000}.{f26c55af-5018-438a-8cc1-6dbead205a3c}",
|
||||
"--seconds",
|
||||
"30",
|
||||
"3600",
|
||||
"--emit-json-blocks",
|
||||
"--include-samples"
|
||||
]
|
||||
|
|
@ -146,7 +147,7 @@
|
|||
"--endpoint",
|
||||
"{0.0.0.00000000}.{defdb2ab-ea1a-4eba-ab59-57038754f387}",
|
||||
"--seconds",
|
||||
"30",
|
||||
"3600",
|
||||
"--emit-json-blocks",
|
||||
"--include-samples"
|
||||
]
|
||||
|
|
|
|||
|
|
@ -3,7 +3,8 @@
|
|||
"audioSync": {
|
||||
"mode": "chirp-only",
|
||||
"referenceSourceId": "loopback-scarlett-speakers",
|
||||
"calibrationGain": 2.0
|
||||
"calibrationGain": 2.0,
|
||||
"watermarkGain": 0.035
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
|
|
@ -19,7 +20,7 @@
|
|||
"--endpoint",
|
||||
"{0.0.1.00000000}.{16af9455-4539-4de2-8172-563d991d9e34}",
|
||||
"--seconds",
|
||||
"30",
|
||||
"3600",
|
||||
"--emit-json-blocks",
|
||||
"--include-samples"
|
||||
]
|
||||
|
|
@ -37,7 +38,7 @@
|
|||
"--endpoint",
|
||||
"{0.0.1.00000000}.{57befef3-599c-4c73-a0cd-9539ffc2f54f}",
|
||||
"--seconds",
|
||||
"30",
|
||||
"3600",
|
||||
"--emit-json-blocks",
|
||||
"--include-samples"
|
||||
]
|
||||
|
|
|
|||
|
|
@ -55,7 +55,8 @@ timing evidence Mimir is allowed to emit:
|
|||
GCC-PHAT-style phase correlation. The active fallback now uses
|
||||
`MimirChirpBinTimeline`: a fixed-slope chirp-bin watermark whose symbols are
|
||||
decoded by one dechirp and a fixed Goertzel bin bank instead of dense sliding
|
||||
matched filters.
|
||||
matched filters. Hybrid emission is pulsed, not continuous: when passive
|
||||
confidence is weak, Mimir emits one half-second coded burst every two seconds.
|
||||
|
||||
The mode belongs to the runtime, not the decoder. The decoder should consume
|
||||
known timing evidence; it should not decide whether Mimir is allowed to make
|
||||
|
|
@ -75,7 +76,9 @@ uses a common chirp duration, chirp slope, and Hann window for every symbol.
|
|||
Symbol identity is the dechirped frequency bin. The synthetic invariant is
|
||||
`dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --chirp-bin-self-test`;
|
||||
it renders the chirp-bin timeline, decodes by dechirp plus Goertzel bins, and
|
||||
requires code-valid triplet anchors plus a stable source clock.
|
||||
requires code-valid triplet anchors plus a stable source clock. Default hybrid
|
||||
watermark gain is intentionally low (`watermarkGain`, or
|
||||
`MIMIR_WATERMARK_GAIN`) and separate from the louder chirp-only lab gain.
|
||||
|
||||
## Next Cut
|
||||
|
||||
|
|
|
|||
|
|
@ -82,7 +82,9 @@ a named invariant that the native runtime cannot protect yet.
|
|||
- `MimirChirpBinTimeline` is the active hybrid watermark path. It renders a
|
||||
fixed-slope chirp-bin codebook and decodes symbols with one dechirp plus
|
||||
fixed Goertzel bins, then feeds the same de Bruijn triplet timeline-anchor
|
||||
machine. Use `--chirp-bin-self-test` to prove this path without hardware.
|
||||
machine. Hybrid emits this as low-gain half-second bursts every two seconds
|
||||
only while passive confidence is weak. Use `--chirp-bin-self-test` to prove
|
||||
this path without hardware.
|
||||
- `MimirVideoFrameDescriptor` for dimensions, pixel format, stride, device
|
||||
timestamp, and native/GPU handle metadata.
|
||||
- `IMimirVideoCaptureDriver` and `MimirVideoCaptureDriverSource` as the live
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ public sealed class MimirRuntime : IAquariumRuntime
|
|||
private const double HybridPassiveConfidenceThreshold = 0.12;
|
||||
private const double CalibrationStartSeconds = 0.5;
|
||||
private const int CalibrationBatchSegments = 120;
|
||||
private const int HybridWatermarkIntervalSegments = 4;
|
||||
private readonly LocalCastRuntime visualRuntime;
|
||||
private readonly MimirSynchronizationHub synchronization;
|
||||
private readonly AquariumUiDocument ui;
|
||||
|
|
@ -22,11 +23,13 @@ public sealed class MimirRuntime : IAquariumRuntime
|
|||
private readonly float telemetryIntervalSeconds;
|
||||
private readonly float audioSyncUpdateIntervalSeconds;
|
||||
private readonly float calibrationGain;
|
||||
private readonly float watermarkGain;
|
||||
private int lastPollCount;
|
||||
private float runtimeSeconds;
|
||||
private float nextAudioSyncSeconds;
|
||||
private float nextTelemetrySeconds;
|
||||
private ulong calibrationSegmentIndex;
|
||||
private long lastHybridWatermarkSegment = -1;
|
||||
private double lastAudioSyncAnalysisMilliseconds;
|
||||
private double lastPassiveSynchronizationConfidence;
|
||||
private IReadOnlyList<MimirAudioSynchronizationReport> lastAudioSynchronizationReports = [];
|
||||
|
|
@ -58,6 +61,7 @@ public sealed class MimirRuntime : IAquariumRuntime
|
|||
telemetryIntervalSeconds = ParseTelemetryIntervalSeconds();
|
||||
audioSyncUpdateIntervalSeconds = ParseAudioSyncIntervalSeconds();
|
||||
calibrationGain = settings.Audio.CalibrationGain;
|
||||
watermarkGain = settings.Audio.WatermarkGain;
|
||||
nextAudioSyncSeconds = audioSyncUpdateIntervalSeconds;
|
||||
nextTelemetrySeconds = telemetryIntervalSeconds;
|
||||
foreach (var source in streamSources)
|
||||
|
|
@ -149,6 +153,21 @@ public sealed class MimirRuntime : IAquariumRuntime
|
|||
return;
|
||||
}
|
||||
|
||||
var currentSegmentIndex = CurrentCalibrationSegmentIndex();
|
||||
if (audioSyncSettings.Mode == MimirAudioSyncMode.Hybrid)
|
||||
{
|
||||
calibrationSegmentIndex = currentSegmentIndex;
|
||||
if (calibrationSegmentIndex % HybridWatermarkIntervalSegments != 0UL ||
|
||||
(long)calibrationSegmentIndex == lastHybridWatermarkSegment)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
else if (calibrationSegmentIndex + 1UL < currentSegmentIndex)
|
||||
{
|
||||
calibrationSegmentIndex = currentSegmentIndex;
|
||||
}
|
||||
|
||||
var nextSegmentSeconds = CalibrationStartSeconds + calibrationSegmentIndex * MimirChirpletTimeline.SegmentSeconds;
|
||||
if (runtimeSeconds < nextSegmentSeconds)
|
||||
{
|
||||
|
|
@ -156,6 +175,7 @@ public sealed class MimirRuntime : IAquariumRuntime
|
|||
}
|
||||
|
||||
var segmentCount = audioSyncSettings.Mode == MimirAudioSyncMode.Hybrid ? 1 : CalibrationBatchSegments;
|
||||
var outputGain = audioSyncSettings.Mode == MimirAudioSyncMode.Hybrid ? watermarkGain : calibrationGain;
|
||||
var batch = RenderCalibrationBatchPcm16Base64(
|
||||
calibrationSegmentIndex,
|
||||
segmentCount,
|
||||
|
|
@ -165,10 +185,27 @@ public sealed class MimirRuntime : IAquariumRuntime
|
|||
batch,
|
||||
MimirChirpletTimeline.SampleRate,
|
||||
channels: 1,
|
||||
gain: calibrationGain);
|
||||
gain: outputGain);
|
||||
Console.WriteLine(
|
||||
$"mimir-chirplet-batch mode={DescribeAudioSyncMode()} firstSegment={calibrationSegmentIndex} segments={segmentCount} seconds={segmentCount * MimirChirpletTimeline.SegmentSeconds:0.00} peak={peak:0.000000} gain={calibrationGain:0.###} base64Bytes={batch.Length}");
|
||||
calibrationSegmentIndex += (ulong)segmentCount;
|
||||
$"mimir-chirplet-batch mode={DescribeAudioSyncMode()} firstSegment={calibrationSegmentIndex} segments={segmentCount} seconds={segmentCount * MimirChirpletTimeline.SegmentSeconds:0.00} peak={peak:0.000000} gain={outputGain:0.###} base64Bytes={batch.Length}");
|
||||
if (audioSyncSettings.Mode == MimirAudioSyncMode.Hybrid)
|
||||
{
|
||||
lastHybridWatermarkSegment = (long)calibrationSegmentIndex;
|
||||
}
|
||||
else
|
||||
{
|
||||
calibrationSegmentIndex += (ulong)segmentCount;
|
||||
}
|
||||
}
|
||||
|
||||
private ulong CurrentCalibrationSegmentIndex()
|
||||
{
|
||||
if (runtimeSeconds <= CalibrationStartSeconds)
|
||||
{
|
||||
return 0UL;
|
||||
}
|
||||
|
||||
return (ulong)Math.Floor((runtimeSeconds - CalibrationStartSeconds) / MimirChirpletTimeline.SegmentSeconds);
|
||||
}
|
||||
|
||||
private string DescribeChirpletReference()
|
||||
|
|
|
|||
|
|
@ -199,6 +199,8 @@ public sealed class MimirAudioSyncConfig
|
|||
|
||||
public float CalibrationGain { get; set; } = float.NaN;
|
||||
|
||||
public float WatermarkGain { get; set; } = float.NaN;
|
||||
|
||||
public MimirAudioSynchronizationSettings ToSettings(MimirAudioSynchronizationSettings fallback)
|
||||
{
|
||||
return new MimirAudioSynchronizationSettings
|
||||
|
|
@ -210,6 +212,9 @@ public sealed class MimirAudioSyncConfig
|
|||
CalibrationGain = float.IsFinite(CalibrationGain)
|
||||
? Math.Clamp(CalibrationGain, 0.0f, 4.0f)
|
||||
: fallback.CalibrationGain,
|
||||
WatermarkGain = float.IsFinite(WatermarkGain)
|
||||
? Math.Clamp(WatermarkGain, 0.0f, 0.25f)
|
||||
: fallback.WatermarkGain,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ public sealed class MimirAudioSynchronizationSettings
|
|||
{
|
||||
public const string DefaultReferenceSourceId = "loopback-scarlett-speakers";
|
||||
public const float DefaultCalibrationGain = 2.0f;
|
||||
public const float DefaultWatermarkGain = 0.035f;
|
||||
|
||||
public MimirAudioSyncMode Mode { get; init; } = MimirAudioSyncMode.Hybrid;
|
||||
|
||||
|
|
@ -63,6 +64,8 @@ public sealed class MimirAudioSynchronizationSettings
|
|||
|
||||
public float CalibrationGain { get; init; } = DefaultCalibrationGain;
|
||||
|
||||
public float WatermarkGain { get; init; } = DefaultWatermarkGain;
|
||||
|
||||
public static MimirAudioSynchronizationSettings FromEnvironment()
|
||||
{
|
||||
return new MimirAudioSynchronizationSettings().WithEnvironmentOverrides();
|
||||
|
|
@ -75,6 +78,7 @@ public sealed class MimirAudioSynchronizationSettings
|
|||
Mode = ParseMode(Environment.GetEnvironmentVariable("MIMIR_AUDIO_SYNC_MODE"), Mode),
|
||||
ReferenceSourceId = ReadReferenceSourceId(ReferenceSourceId),
|
||||
CalibrationGain = ReadCalibrationGain(CalibrationGain),
|
||||
WatermarkGain = ReadWatermarkGain(WatermarkGain),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -108,6 +112,13 @@ public sealed class MimirAudioSynchronizationSettings
|
|||
? Math.Clamp(gain, 0.0f, 4.0f)
|
||||
: fallback;
|
||||
}
|
||||
|
||||
private static float ReadWatermarkGain(float fallback)
|
||||
{
|
||||
return float.TryParse(Environment.GetEnvironmentVariable("MIMIR_WATERMARK_GAIN"), out var gain)
|
||||
? Math.Clamp(gain, 0.0f, 0.25f)
|
||||
: fallback;
|
||||
}
|
||||
}
|
||||
|
||||
public enum MimirAudioSyncMode
|
||||
|
|
|
|||
|
|
@ -57,3 +57,4 @@
|
|||
{"ts": "2026-05-22T19:08:56+01:00", "type": "verification", "status": "partial", "note": "Passive live smoke with MIMIR_AUDIO_SYNC_MODE=passive and config/mimir-runtime.probes.example.json emitted no calibration audio and produced loopback/program-audio reports: Focusrite delaySamples=6653 confidence=0.805, Kiyo Pro delaySamples=6804 confidence=0.227, USB Camera 0 delaySamples=-882 confidence=1.000 over 88200 compared 44.1k samples. This proves the passive path can lock on real loopback/music, but the negative Eye delay needs timestamp/window-sign inspection before driving an actuator."}
|
||||
{"ts": "2026-05-22T19:33:21+01:00", "type": "verification", "status": "ok", "note": "Fixed passive sync confidence bug. Negative passive lags now produce status=passive-negative-lag with confidence=0, single-window passive confidence is capped at 0.85, and hub invalidates cached reports/state for negative-lag traces. Passive self-test now proves both positive delay recovery and reversed-input zero confidence. A 14s live passive smoke showed negative Focusrite evidence at confidence 0 without leaving a report/state, while positive passive reports stayed capped at 0.85."}
|
||||
{"ts": "2026-05-22T19:49:43+01:00", "type": "implementation", "status": "partial", "note": "Built the first chirp-bin active watermark machine. MimirChirpBinTimeline renders a fixed-slope/window 32-bin chirp codebook and decodes with one dechirp plus fixed Goertzel bin scoring, then feeds de Bruijn triplet anchors and clock fitting. Hybrid fallback now renders/decodes chirp-bin evidence instead of the old dense chirplet matcher. Synthetic --chirp-bin-self-test passed with 23 frames, 21 anchors, 48000 Hz clock, and zero MAE; old --chirplet-self-test still passed. A real hybrid app run emitted chirp-bin batches, but the current probe config's 30s capture limit left loopback/mic windows stale before acoustic chirp-bin reports could be proven."}
|
||||
{"ts": "2026-05-22T19:58:44+01:00", "type": "implementation", "status": "ok", "note": "Fixed hybrid watermark being obtrusive. Root causes: Aquarium applied EnqueuePcm16Base64 gain twice, Mimir treated immediate audio chunks like scheduled future segments and caught up missed hybrid segments as audible backlog, and probe configs stopped WASAPI capture after 30s. Aquarium now applies queued PCM gain once; Mimir hybrid uses separate low watermarkGain default 0.035, emits one 0.5s chirp-bin burst every 2s only while passive confidence is weak, and skips missed segments instead of catch-up dumping. Probe audio durations are now 3600s. Short hybrid app run showed batches at firstSegment 0,4,8,12 with gain 0.035."}
|
||||
|
|
|
|||
|
|
@ -105,7 +105,8 @@ accepted_design:
|
|||
`MimirChirpBinTimeline` is the active hybrid watermark path: one fixed chirp
|
||||
slope/window, 32 symbol frequency bins, dechirp plus fixed Goertzel bin
|
||||
scoring, then the existing de Bruijn triplet anchor and clock-fit machine.
|
||||
The synthetic proof is `dotnet run --project
|
||||
Hybrid emits it as low-gain half-second bursts every two seconds while
|
||||
passive confidence is weak, never as catch-up backlog. The synthetic proof is `dotnet run --project
|
||||
.\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --chirp-bin-self-test`.
|
||||
- `IMimirVideoCaptureDriver` and `MimirVideoCaptureDriverSource` are the live
|
||||
seam for Leap, Media Foundation, DirectShow, libusb, LeapC, or shared
|
||||
|
|
|
|||
Loading…
Reference in a new issue