From 82f2c7cf9671ecbdc8dedd256811c97b855a31fc Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 1 Jun 2026 23:52:17 +0100 Subject: [PATCH] Let Mimir Well tune camera exposure --- scripts/start-online-verse-daemons.ps1 | 8 +- .../MimirCameraExposureControl.cs | 358 ++++++++++++++++++ .../MimirSynchronizationHub.cs | 3 + .../MimirVideoCaptureDriverSource.cs | 13 +- src/Mimir.Well/Program.cs | 77 +++- state/evidence.jsonl | 1 + state/map.yaml | 10 + 7 files changed, 467 insertions(+), 3 deletions(-) create mode 100644 src/Mimir.Runtime/Synchronization/MimirCameraExposureControl.cs diff --git a/scripts/start-online-verse-daemons.ps1 b/scripts/start-online-verse-daemons.ps1 index 01f60c8..5a6f4a3 100644 --- a/scripts/start-online-verse-daemons.ps1 +++ b/scripts/start-online-verse-daemons.ps1 @@ -268,7 +268,13 @@ if (-not $SkipRuntime) { "--publish-ms", "250", "--sync-ms", "250", "--presentation-delay-ms", "2500", - "--max-samples-per-source", "4" + "--max-samples-per-source", "4", + "--visual-calibration", "true", + "--visual-calibration-ms", "250", + "--visual-expected-leds", "38", + "--visual-minimum-luma", "0.55", + "--visual-setting-seconds", "0.75", + "--visual-resweep-seconds", "12.0" ) ` -Environment @{ "MIMIR_RUNTIME_CONFIG" = $ConfigPath diff --git a/src/Mimir.Runtime/Synchronization/MimirCameraExposureControl.cs b/src/Mimir.Runtime/Synchronization/MimirCameraExposureControl.cs new file mode 100644 index 0000000..a4f8b0b --- /dev/null +++ b/src/Mimir.Runtime/Synchronization/MimirCameraExposureControl.cs @@ -0,0 +1,358 @@ +namespace Mimir.Runtime.Synchronization; + +public interface IMimirCameraExposureGainActuator +{ + string SourceId { get; } + + string ExposureControlKind { get; } + + bool SupportsExposureGain { get; } + + bool TrySetExposureGain(int? exposure, int? gain); +} + +public sealed record MimirOnlineExposureGainSetting(string Id, int? Exposure, int? Gain); + +public sealed record MimirCameraExposureControlOptions( + bool Enabled = true, + int ExpectedLedCount = 38, + double MinimumLuma = 0.55, + double SettingSeconds = 0.75, + double ResweepSeconds = 12.0); + +public sealed record MimirCameraExposureControlStatus( + string SourceId, + string ControlKind, + bool SupportsExposureGain, + string State, + string CurrentSettingId, + int? CurrentExposure, + int? CurrentGain, + string BestSettingId, + int? BestExposure, + int? BestGain, + double BestScore, + int BestDetectedLedCount, + bool BestUsableForCalibration, + int FramesScored, + bool LastApplySucceeded, + string Reason); + +public sealed class MimirCameraExposureController(MimirCameraExposureControlOptions options) +{ + private readonly Dictionary states = new(StringComparer.Ordinal); + private readonly IReadOnlyList settings = + [ + new("exp-10_gain-0", -10, 0), + new("exp-10_gain-8", -10, 8), + new("exp-9_gain-0", -9, 0), + new("exp-9_gain-8", -9, 8), + new("exp-8_gain-0", -8, 0), + new("exp-8_gain-8", -8, 8), + new("exp-7_gain-0", -7, 0), + new("exp-7_gain-8", -7, 8), + new("exp-6_gain-0", -6, 0), + new("exp-6_gain-8", -6, 8), + ]; + + public IReadOnlyList Statuses => + states.Values + .OrderBy(state => state.SourceId, StringComparer.Ordinal) + .Select(state => state.ToStatus()) + .ToArray(); + + public void Update( + DateTimeOffset now, + IEnumerable buffers, + IEnumerable actuators) + { + if (!options.Enabled) + { + return; + } + + var actuatorBySource = actuators.ToDictionary(actuator => actuator.SourceId, StringComparer.Ordinal); + foreach (var buffer in buffers.Where(static buffer => buffer.Descriptor.Kind == MimirStreamKind.Video)) + { + if (!actuatorBySource.TryGetValue(buffer.Descriptor.SourceId, out var actuator)) + { + continue; + } + + var state = StateFor(actuator); + if (!actuator.SupportsExposureGain) + { + state.MarkUnsupported(); + continue; + } + + if (state.State == ExposureControlState.Idle || now >= state.NextTransitionUtc) + { + AdvanceState(state, actuator, now); + } + + if (buffer.Latest is { } sample) + { + ScoreSample(state, sample); + } + } + } + + private SourceState StateFor(IMimirCameraExposureGainActuator actuator) + { + if (states.TryGetValue(actuator.SourceId, out var state)) + { + return state; + } + + state = new SourceState(actuator.SourceId, actuator.ExposureControlKind); + states.Add(actuator.SourceId, state); + return state; + } + + private void AdvanceState(SourceState state, IMimirCameraExposureGainActuator actuator, DateTimeOffset now) + { + if (state.State == ExposureControlState.Locked && now < state.NextTransitionUtc) + { + return; + } + + if (state.State == ExposureControlState.Scanning) + { + state.SettingIndex++; + } + else + { + state.BeginSweep(); + } + + if (state.SettingIndex >= settings.Count) + { + LockBest(state, actuator, now); + return; + } + + var setting = settings[state.SettingIndex]; + state.CurrentSetting = setting; + state.CurrentScore = new Accumulator(); + state.LastApplySucceeded = actuator.TrySetExposureGain(setting.Exposure, setting.Gain); + state.State = ExposureControlState.Scanning; + state.Reason = state.LastApplySucceeded ? "scoring-setting" : "control-apply-failed"; + state.NextTransitionUtc = now + TimeSpan.FromSeconds(Math.Max(0.10, options.SettingSeconds)); + } + + private void LockBest(SourceState state, IMimirCameraExposureGainActuator actuator, DateTimeOffset now) + { + var best = state.BestSetting ?? state.CurrentSetting ?? settings[0]; + state.CurrentSetting = best; + state.LastApplySucceeded = actuator.TrySetExposureGain(best.Exposure, best.Gain); + state.State = ExposureControlState.Locked; + state.Reason = state.LastApplySucceeded ? "locked-best-spline-score" : "best-control-apply-failed"; + state.NextTransitionUtc = now + TimeSpan.FromSeconds(Math.Max(options.ResweepSeconds, settings.Count * options.SettingSeconds)); + state.SettingIndex = settings.Count; + } + + private void ScoreSample(SourceState state, MimirStreamSample sample) + { + if (state.State != ExposureControlState.Scanning || + sample.Sequence == state.LastScoredSequence || + sample.VideoFrame is not { } frame || + sample.Data.IsEmpty || + !TryExtractLuma(frame, sample.Data.Span, out var luma, out var width, out var height)) + { + return; + } + + state.LastScoredSequence = sample.Sequence; + var analyzer = new MimirLedSplineFrameAnalyzer(new MimirLedSplineFrameAnalyzerOptions( + options.ExpectedLedCount, + options.MinimumLuma, + MinimumComponentPixels: 2, + MaximumComponentPixels: 8192)); + var analysis = analyzer.AnalyzeLumaFrame( + sample.SourceId, + $"{sample.SourceId}:well:{sample.Sequence}", + width, + height, + luma, + frame.DeviceTimestampNs); + state.CurrentScore.Add(analysis.Quality); + if (state.CurrentScore.BestQuality is { } best && + (state.BestQuality == null || + best.Score > state.BestQuality.Score || + (Math.Abs(best.Score - state.BestQuality.Score) <= 1.0e-9 && + best.DetectedLedCount > state.BestQuality.DetectedLedCount))) + { + state.BestQuality = best; + state.BestSetting = state.CurrentSetting; + } + } + + private static bool TryExtractLuma( + MimirVideoFrameDescriptor frame, + ReadOnlySpan data, + out byte[] luma, + out int width, + out int height) + { + width = frame.Width; + height = frame.Height; + if (width <= 0 || height <= 0) + { + luma = []; + return false; + } + + luma = new byte[checked(width * height)]; + var stride = frame.StrideBytes <= 0 ? DefaultStride(frame) : frame.StrideBytes; + switch (frame.PixelFormat) + { + case MimirVideoPixelFormat.Gray8: + case MimirVideoPixelFormat.R8: + case MimirVideoPixelFormat.Bayer8: + for (var y = 0; y < height; y++) + { + var row = y * stride; + if (row + width > data.Length) + { + return false; + } + + data.Slice(row, width).CopyTo(luma.AsSpan(y * width, width)); + } + + return true; + case MimirVideoPixelFormat.Yuy2: + case MimirVideoPixelFormat.LeapStereoIr: + case MimirVideoPixelFormat.Rg8: + for (var y = 0; y < height; y++) + { + var row = y * stride; + if (row + width * 2 > data.Length) + { + return false; + } + + for (var x = 0; x < width; x++) + { + luma[y * width + x] = data[row + x * 2]; + } + } + + return true; + case MimirVideoPixelFormat.Bgra8: + for (var y = 0; y < height; y++) + { + var row = y * stride; + if (row + width * 4 > data.Length) + { + return false; + } + + for (var x = 0; x < width; x++) + { + var offset = row + x * 4; + luma[y * width + x] = (byte)Math.Clamp( + (int)Math.Round(data[offset] * 0.114 + data[offset + 1] * 0.587 + data[offset + 2] * 0.299), + 0, + 255); + } + } + + return true; + default: + luma = []; + return false; + } + } + + private static int DefaultStride(MimirVideoFrameDescriptor frame) => + frame.PixelFormat switch + { + MimirVideoPixelFormat.Yuy2 or MimirVideoPixelFormat.LeapStereoIr or MimirVideoPixelFormat.Rg8 => frame.Width * 2, + MimirVideoPixelFormat.Bgra8 => frame.Width * 4, + _ => frame.Width, + }; + + private sealed class SourceState(string sourceId, string controlKind) + { + public string SourceId { get; } = sourceId; + public string ControlKind { get; } = controlKind; + public ExposureControlState State { get; set; } = ExposureControlState.Idle; + public int SettingIndex { get; set; } = -1; + public MimirOnlineExposureGainSetting? CurrentSetting { get; set; } + public MimirOnlineExposureGainSetting? BestSetting { get; set; } + public MimirLedSplineQualityReport? BestQuality { get; set; } + public Accumulator CurrentScore { get; set; } = new(); + public ulong LastScoredSequence { get; set; } = ulong.MaxValue; + public DateTimeOffset NextTransitionUtc { get; set; } = DateTimeOffset.MinValue; + public bool LastApplySucceeded { get; set; } + public string Reason { get; set; } = "not-started"; + + public void BeginSweep() + { + State = ExposureControlState.Scanning; + SettingIndex = 0; + BestSetting = null; + BestQuality = null; + CurrentScore = new Accumulator(); + LastScoredSequence = ulong.MaxValue; + Reason = "begin-sweep"; + } + + public void MarkUnsupported() + { + State = ExposureControlState.Unsupported; + Reason = "exposure-gain-not-supported"; + } + + public MimirCameraExposureControlStatus ToStatus() + { + var best = BestSetting; + var quality = BestQuality; + return new MimirCameraExposureControlStatus( + SourceId, + ControlKind, + State != ExposureControlState.Unsupported, + State.ToString(), + CurrentSetting?.Id ?? "", + CurrentSetting?.Exposure, + CurrentSetting?.Gain, + best?.Id ?? "", + best?.Exposure, + best?.Gain, + quality?.Score ?? 0.0, + quality?.DetectedLedCount ?? 0, + quality?.UsableForCalibration ?? false, + CurrentScore.FramesScored, + LastApplySucceeded, + Reason); + } + } + + private sealed class Accumulator + { + public int FramesScored { get; private set; } + public MimirLedSplineQualityReport? BestQuality { get; private set; } + + public void Add(MimirLedSplineQualityReport quality) + { + FramesScored++; + if (BestQuality == null || + quality.Score > BestQuality.Score || + (Math.Abs(quality.Score - BestQuality.Score) <= 1.0e-9 && + quality.DetectedLedCount > BestQuality.DetectedLedCount)) + { + BestQuality = quality; + } + } + } + + private enum ExposureControlState + { + Idle, + Scanning, + Locked, + Unsupported, + } +} diff --git a/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs b/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs index 35a9a30..d04bd76 100644 --- a/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs +++ b/src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs @@ -38,6 +38,9 @@ public sealed class MimirSynchronizationHub : IDisposable public int SourceCount => sources.Count; + public IReadOnlyList CameraExposureGainActuators => + sources.OfType().ToArray(); + public IReadOnlyList AudioSynchronizationStates => audioSynchronizationState.States; diff --git a/src/Mimir.Runtime/Synchronization/MimirVideoCaptureDriverSource.cs b/src/Mimir.Runtime/Synchronization/MimirVideoCaptureDriverSource.cs index 77b3f91..7f52830 100644 --- a/src/Mimir.Runtime/Synchronization/MimirVideoCaptureDriverSource.cs +++ b/src/Mimir.Runtime/Synchronization/MimirVideoCaptureDriverSource.cs @@ -1,6 +1,6 @@ namespace Mimir.Runtime.Synchronization; -public sealed class MimirVideoCaptureDriverSource : IMimirStreamSource, IMimirFensalirTextureLeaseReceiver +public sealed class MimirVideoCaptureDriverSource : IMimirStreamSource, IMimirFensalirTextureLeaseReceiver, IMimirCameraExposureGainActuator { private readonly IMimirVideoCaptureDriver driver; private readonly MimirNativeIngestStreamSource nativeSource; @@ -24,6 +24,13 @@ public sealed class MimirVideoCaptureDriverSource : IMimirStreamSource, IMimirFe public MimirStreamDescriptor Descriptor { get; } + public string SourceId => Descriptor.SourceId; + + public string ExposureControlKind => + driver is MimirKsVideoCaptureDriver ? "ks-exposure-gain" : "fixed-control"; + + public bool SupportsExposureGain => driver is MimirKsVideoCaptureDriver; + public Func ReadArrivalNs { get; } public int LastUploadedCopyCount { get; private set; } @@ -39,6 +46,10 @@ public sealed class MimirVideoCaptureDriverSource : IMimirStreamSource, IMimirFe } } + public bool TrySetExposureGain(int? exposure, int? gain) => + driver is MimirKsVideoCaptureDriver ksDriver && + ksDriver.TrySetExposureGain(exposure, gain); + public bool TryRead(out MimirStreamSample sample) { if (nativeSource.TryRead(out sample)) diff --git a/src/Mimir.Well/Program.cs b/src/Mimir.Well/Program.cs index c23f39e..31143b4 100644 --- a/src/Mimir.Well/Program.cs +++ b/src/Mimir.Well/Program.cs @@ -51,7 +51,14 @@ var sequence = 0L; var startedAt = DateTimeOffset.UtcNow; var nextPublish = DateTimeOffset.MinValue; var nextSync = DateTimeOffset.MinValue; +var nextVisualCalibration = DateTimeOffset.MinValue; var presentation = new MimirPresentationControlState(); +var exposureController = new MimirCameraExposureController(new MimirCameraExposureControlOptions( + options.VisualCalibrationEnabled, + options.VisualExpectedLedCount, + options.VisualMinimumLuma, + options.VisualSettingSeconds, + options.VisualResweepSeconds)); Console.Error.WriteLine($"Mimir Well publishing to {options.PublishUrl}"); Console.Error.WriteLine($"Mimir Well sources={runtimeConfig.SourceFactories.Count} buffers={hub.Buffers.Buffers.Count}"); @@ -77,6 +84,12 @@ while (!stopping.IsCancellationRequested) nextSync = now + TimeSpan.FromMilliseconds(options.SyncIntervalMs); } + if (now >= nextVisualCalibration) + { + exposureController.Update(now, hub.Buffers.Buffers, hub.CameraExposureGainActuators); + nextVisualCalibration = now + TimeSpan.FromMilliseconds(options.VisualCalibrationIntervalMs); + } + if (now >= nextPublish) { presentation.SyncFromBuffers(hub.Buffers.Buffers); @@ -87,6 +100,7 @@ while (!stopping.IsCancellationRequested) hub, presentation, frame, + exposureController.Statuses, sourceErrors, ++sequence, startedAt); @@ -142,6 +156,12 @@ internal sealed record MimirWellOptions( int PresentationDelayMs, int MaxSamplesPerSource, int SyncCandidatesPerStep, + bool VisualCalibrationEnabled, + int VisualCalibrationIntervalMs, + int VisualExpectedLedCount, + double VisualMinimumLuma, + double VisualSettingSeconds, + double VisualResweepSeconds, int MeterEvery) { public static MimirWellOptions Parse(string[] args) => new( @@ -154,6 +174,12 @@ internal sealed record MimirWellOptions( ParseInt(args, "--presentation-delay-ms", 2500), ParseInt(args, "--max-samples-per-source", 4), ParseInt(args, "--sync-candidates", 1), + ParseBool(args, "--visual-calibration", true), + ParseInt(args, "--visual-calibration-ms", 250), + ParseInt(args, "--visual-expected-leds", 38), + ParseDouble(args, "--visual-minimum-luma", 0.55), + ParseDouble(args, "--visual-setting-seconds", 0.75), + ParseDouble(args, "--visual-resweep-seconds", 12.0), ParseInt(args, "--meter-every", 20)); private static string ParseString(IReadOnlyList args, string name, string fallback) @@ -174,6 +200,27 @@ internal sealed record MimirWellOptions( private static double ParseDouble(IReadOnlyList args, string name, double fallback) => double.TryParse(ParseString(args, name, ""), out var value) ? value : fallback; + + private static bool ParseBool(IReadOnlyList args, string name, bool fallback) + { + for (var index = 0; index < args.Count; index++) + { + if (!string.Equals(args[index], name, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (index == args.Count - 1 || + args[index + 1].StartsWith("--", StringComparison.Ordinal)) + { + return true; + } + + return bool.TryParse(args[index + 1], out var value) ? value : fallback; + } + + return fallback; + } } internal sealed class MimirWellPublisher(Uri url) : IAsyncDisposable @@ -218,6 +265,7 @@ internal static class MimirWellSnapshot MimirSynchronizationHub hub, MimirPresentationControlState presentation, MimirSynchronizedBufferFrame frame, + IReadOnlyList visualCalibration, IReadOnlyList sourceErrors, long sequence, DateTimeOffset startedAt) => new @@ -279,7 +327,34 @@ internal static class MimirWellSnapshot hub.LastBioacousticProbeSchedule.AggregateSyncConfidence, hub.LastBioacousticProbeSchedule.AggregateFrequencyResponseConfidence, reason = hub.LastBioacousticProbeSchedule.Reason.ToString(), - }, + }, + }, + visualCalibration = new + { + enabled = options.VisualCalibrationEnabled, + expectedLedCount = options.VisualExpectedLedCount, + minimumLuma = options.VisualMinimumLuma, + settingSeconds = options.VisualSettingSeconds, + resweepSeconds = options.VisualResweepSeconds, + cameras = visualCalibration.Select(camera => new + { + camera.SourceId, + camera.ControlKind, + camera.SupportsExposureGain, + camera.State, + camera.CurrentSettingId, + camera.CurrentExposure, + camera.CurrentGain, + camera.BestSettingId, + camera.BestExposure, + camera.BestGain, + camera.BestScore, + camera.BestDetectedLedCount, + camera.BestUsableForCalibration, + camera.FramesScored, + camera.LastApplySucceeded, + camera.Reason, + }).ToArray(), }, composite = new { diff --git a/state/evidence.jsonl b/state/evidence.jsonl index baacb6a..ac989e6 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -343,3 +343,4 @@ {"ts": "2026-06-01T22:49:16+01:00", "type": "implementation", "status": "ok", "note": "2026-06-01: Corrected the Move/Syrinx witness invariant. Online Move music sync no longer loops an independent Realtek Syrinx bed; ScoreGestureScheduler now owns sparse score strikes with note target, glide, vibrato, harmonic lane, and intensity, and BioacousticRealtekTrigger emits a short Realtek AquaSynth Syrinx call only when that same score strike drives a Move pulse. Trace records now include score_voice_contours, score_voice_events, and emitted_audio_event_count for Mimir Well/offline timeline correlation. Live run artifacts/runtime/move-syrinx-witness-20260601-224705 produced 79 voice events, 69 emitted Realtek calls, and all three Moves participating in a 1764-frame poll."} {"ts": "2026-06-01T23:24:26+01:00", "type": "implementation", "status": "ok", "note": "2026-06-01: Added live score detection to the Move/Syrinx worker. OnlineAnalyzer now emits tempo_candidates and a 12-bin cyclic_tempogram from onset autocorrelation. LiveScoreEstimator publishes mimir.live_score.v1 with detected voices, note names, durations, score confidence, target-confidence deficit, gesture density, and per-Move note targets. ScoreGestureScheduler consumes that score document so Move/Syrinx calibration gestures are spread across detected voices/spectral lanes according to confidence deficit. Live run artifacts/runtime/move-live-score-20260601-232156 sampled 350/350 records with live_score, maxVoices=3, maxTempoCandidates=8, 21 score event frames, 20 emitted Syrinx calls, and Move targets assigned to shotgun-mic, realtek-loopback, and cardioid-mic."} {"ts": "2026-06-01T23:37:14+01:00", "type": "implementation", "status": "ok", "note": "2026-06-01: Move/Syrinx score gestures now carry explicit articulation roles. ScoreGestureScheduler assigns Moves to violin-syrinx, harp-arpeggio, and peck-syrinx by default; events include articulation, gesture note sequence, attack shape, and duration hint. Move RGB lowering and Realtek AquaSynth playback consume the same event so each pulse remains one audiovisual calibration voice. Verified with a synthetic scheduler probe plus py_compile and launcher dry-run."} +{"ts": "2026-06-01T23:50:56+01:00", "type": "implementation", "status": "ok", "note": "2026-06-01: Mimir.Well now owns online camera exposure/gain optimization for visual sync. MimirCameraExposureController samples live Well video buffers, scores LED spline quality with MimirLedSplineFrameAnalyzer, cycles KS exposure/gain candidates, locks the best score, periodically resweeps, and publishes visualCalibration status in mimir.well_snapshot.v1. Verified with dotnet build src/Mimir.Well and supervisor dry-run options."} diff --git a/state/map.yaml b/state/map.yaml index 81e79b1..16d7847 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -193,6 +193,16 @@ invariants: for both Kiyo Pro and Kiyo Basic. LeapUVC rejected the generic KS exposure/gain writes and PS3 Eye capture remains fixed-control until its native wrapper owns gain/exposure. + `Mimir.Well` now owns the online version of that visual-sync actuator: + `MimirCameraExposureController` samples latest video frames from live Well + buffers, scores the LED spline with `MimirLedSplineFrameAnalyzer`, cycles + controllable KS camera exposure/gain candidates, locks the best spline + score, and periodically resweeps as room lighting and performer pose + change. `MimirVideoCaptureDriverSource` exposes only the narrow + `IMimirCameraExposureGainActuator` command surface, so the Well does not + know KS internals. `mimir.well_snapshot.v1` now publishes + `visualCalibration` status per camera: current setting, best setting, + score, detected LED count, usability, frame count, apply status, and reason. - PS Move is usable as an active marker/IMU witness, not as an optical depth owner. `Mimir.PsMoveProbe` now enumerates Sony `VID_054C` / `PID_03D5` through Windows HID without third-party packages. The connected controller