Let Mimir Well tune camera exposure

scripts/start-online-verse-daemons.ps1

@@ -268,7 +268,13 @@ if (-not $SkipRuntime) {
             "--publish-ms", "250",
             "--sync-ms", "250",
             "--presentation-delay-ms", "2500",
-            "--max-samples-per-source", "4"
+            "--max-samples-per-source", "4",
+            "--visual-calibration", "true",
+            "--visual-calibration-ms", "250",
+            "--visual-expected-leds", "38",
+            "--visual-minimum-luma", "0.55",
+            "--visual-setting-seconds", "0.75",
+            "--visual-resweep-seconds", "12.0"
         ) `
         -Environment @{
             "MIMIR_RUNTIME_CONFIG" = $ConfigPath

src/Mimir.Runtime/Synchronization/MimirCameraExposureControl.cs

@@ -0,0 +1,358 @@
+namespace Mimir.Runtime.Synchronization;
+
+public interface IMimirCameraExposureGainActuator
+{
+    string SourceId { get; }
+
+    string ExposureControlKind { get; }
+
+    bool SupportsExposureGain { get; }
+
+    bool TrySetExposureGain(int? exposure, int? gain);
+}
+
+public sealed record MimirOnlineExposureGainSetting(string Id, int? Exposure, int? Gain);
+
+public sealed record MimirCameraExposureControlOptions(
+    bool Enabled = true,
+    int ExpectedLedCount = 38,
+    double MinimumLuma = 0.55,
+    double SettingSeconds = 0.75,
+    double ResweepSeconds = 12.0);
+
+public sealed record MimirCameraExposureControlStatus(
+    string SourceId,
+    string ControlKind,
+    bool SupportsExposureGain,
+    string State,
+    string CurrentSettingId,
+    int? CurrentExposure,
+    int? CurrentGain,
+    string BestSettingId,
+    int? BestExposure,
+    int? BestGain,
+    double BestScore,
+    int BestDetectedLedCount,
+    bool BestUsableForCalibration,
+    int FramesScored,
+    bool LastApplySucceeded,
+    string Reason);
+
+public sealed class MimirCameraExposureController(MimirCameraExposureControlOptions options)
+{
+    private readonly Dictionary<string, SourceState> states = new(StringComparer.Ordinal);
+    private readonly IReadOnlyList<MimirOnlineExposureGainSetting> settings =
+    [
+        new("exp-10_gain-0", -10, 0),
+        new("exp-10_gain-8", -10, 8),
+        new("exp-9_gain-0", -9, 0),
+        new("exp-9_gain-8", -9, 8),
+        new("exp-8_gain-0", -8, 0),
+        new("exp-8_gain-8", -8, 8),
+        new("exp-7_gain-0", -7, 0),
+        new("exp-7_gain-8", -7, 8),
+        new("exp-6_gain-0", -6, 0),
+        new("exp-6_gain-8", -6, 8),
+    ];
+
+    public IReadOnlyList<MimirCameraExposureControlStatus> Statuses =>
+        states.Values
+            .OrderBy(state => state.SourceId, StringComparer.Ordinal)
+            .Select(state => state.ToStatus())
+            .ToArray();
+
+    public void Update(
+        DateTimeOffset now,
+        IEnumerable<MimirRollingStreamBuffer> buffers,
+        IEnumerable<IMimirCameraExposureGainActuator> actuators)
+    {
+        if (!options.Enabled)
+        {
+            return;
+        }
+
+        var actuatorBySource = actuators.ToDictionary(actuator => actuator.SourceId, StringComparer.Ordinal);
+        foreach (var buffer in buffers.Where(static buffer => buffer.Descriptor.Kind == MimirStreamKind.Video))
+        {
+            if (!actuatorBySource.TryGetValue(buffer.Descriptor.SourceId, out var actuator))
+            {
+                continue;
+            }
+
+            var state = StateFor(actuator);
+            if (!actuator.SupportsExposureGain)
+            {
+                state.MarkUnsupported();
+                continue;
+            }
+
+            if (state.State == ExposureControlState.Idle || now >= state.NextTransitionUtc)
+            {
+                AdvanceState(state, actuator, now);
+            }
+
+            if (buffer.Latest is { } sample)
+            {
+                ScoreSample(state, sample);
+            }
+        }
+    }
+
+    private SourceState StateFor(IMimirCameraExposureGainActuator actuator)
+    {
+        if (states.TryGetValue(actuator.SourceId, out var state))
+        {
+            return state;
+        }
+
+        state = new SourceState(actuator.SourceId, actuator.ExposureControlKind);
+        states.Add(actuator.SourceId, state);
+        return state;
+    }
+
+    private void AdvanceState(SourceState state, IMimirCameraExposureGainActuator actuator, DateTimeOffset now)
+    {
+        if (state.State == ExposureControlState.Locked && now < state.NextTransitionUtc)
+        {
+            return;
+        }
+
+        if (state.State == ExposureControlState.Scanning)
+        {
+            state.SettingIndex++;
+        }
+        else
+        {
+            state.BeginSweep();
+        }
+
+        if (state.SettingIndex >= settings.Count)
+        {
+            LockBest(state, actuator, now);
+            return;
+        }
+
+        var setting = settings[state.SettingIndex];
+        state.CurrentSetting = setting;
+        state.CurrentScore = new Accumulator();
+        state.LastApplySucceeded = actuator.TrySetExposureGain(setting.Exposure, setting.Gain);
+        state.State = ExposureControlState.Scanning;
+        state.Reason = state.LastApplySucceeded ? "scoring-setting" : "control-apply-failed";
+        state.NextTransitionUtc = now + TimeSpan.FromSeconds(Math.Max(0.10, options.SettingSeconds));
+    }
+
+    private void LockBest(SourceState state, IMimirCameraExposureGainActuator actuator, DateTimeOffset now)
+    {
+        var best = state.BestSetting ?? state.CurrentSetting ?? settings[0];
+        state.CurrentSetting = best;
+        state.LastApplySucceeded = actuator.TrySetExposureGain(best.Exposure, best.Gain);
+        state.State = ExposureControlState.Locked;
+        state.Reason = state.LastApplySucceeded ? "locked-best-spline-score" : "best-control-apply-failed";
+        state.NextTransitionUtc = now + TimeSpan.FromSeconds(Math.Max(options.ResweepSeconds, settings.Count * options.SettingSeconds));
+        state.SettingIndex = settings.Count;
+    }
+
+    private void ScoreSample(SourceState state, MimirStreamSample sample)
+    {
+        if (state.State != ExposureControlState.Scanning ||
+            sample.Sequence == state.LastScoredSequence ||
+            sample.VideoFrame is not { } frame ||
+            sample.Data.IsEmpty ||
+            !TryExtractLuma(frame, sample.Data.Span, out var luma, out var width, out var height))
+        {
+            return;
+        }
+
+        state.LastScoredSequence = sample.Sequence;
+        var analyzer = new MimirLedSplineFrameAnalyzer(new MimirLedSplineFrameAnalyzerOptions(
+            options.ExpectedLedCount,
+            options.MinimumLuma,
+            MinimumComponentPixels: 2,
+            MaximumComponentPixels: 8192));
+        var analysis = analyzer.AnalyzeLumaFrame(
+            sample.SourceId,
+            $"{sample.SourceId}:well:{sample.Sequence}",
+            width,
+            height,
+            luma,
+            frame.DeviceTimestampNs);
+        state.CurrentScore.Add(analysis.Quality);
+        if (state.CurrentScore.BestQuality is { } best &&
+            (state.BestQuality == null ||
+                best.Score > state.BestQuality.Score ||
+                (Math.Abs(best.Score - state.BestQuality.Score) <= 1.0e-9 &&
+                    best.DetectedLedCount > state.BestQuality.DetectedLedCount)))
+        {
+            state.BestQuality = best;
+            state.BestSetting = state.CurrentSetting;
+        }
+    }
+
+    private static bool TryExtractLuma(
+        MimirVideoFrameDescriptor frame,
+        ReadOnlySpan<byte> data,
+        out byte[] luma,
+        out int width,
+        out int height)
+    {
+        width = frame.Width;
+        height = frame.Height;
+        if (width <= 0 || height <= 0)
+        {
+            luma = [];
+            return false;
+        }
+
+        luma = new byte[checked(width * height)];
+        var stride = frame.StrideBytes <= 0 ? DefaultStride(frame) : frame.StrideBytes;
+        switch (frame.PixelFormat)
+        {
+            case MimirVideoPixelFormat.Gray8:
+            case MimirVideoPixelFormat.R8:
+            case MimirVideoPixelFormat.Bayer8:
+                for (var y = 0; y < height; y++)
+                {
+                    var row = y * stride;
+                    if (row + width > data.Length)
+                    {
+                        return false;
+                    }
+
+                    data.Slice(row, width).CopyTo(luma.AsSpan(y * width, width));
+                }
+
+                return true;
+            case MimirVideoPixelFormat.Yuy2:
+            case MimirVideoPixelFormat.LeapStereoIr:
+            case MimirVideoPixelFormat.Rg8:
+                for (var y = 0; y < height; y++)
+                {
+                    var row = y * stride;
+                    if (row + width * 2 > data.Length)
+                    {
+                        return false;
+                    }
+
+                    for (var x = 0; x < width; x++)
+                    {
+                        luma[y * width + x] = data[row + x * 2];
+                    }
+                }
+
+                return true;
+            case MimirVideoPixelFormat.Bgra8:
+                for (var y = 0; y < height; y++)
+                {
+                    var row = y * stride;
+                    if (row + width * 4 > data.Length)
+                    {
+                        return false;
+                    }
+
+                    for (var x = 0; x < width; x++)
+                    {
+                        var offset = row + x * 4;
+                        luma[y * width + x] = (byte)Math.Clamp(
+                            (int)Math.Round(data[offset] * 0.114 + data[offset + 1] * 0.587 + data[offset + 2] * 0.299),
+                            0,
+                            255);
+                    }
+                }
+
+                return true;
+            default:
+                luma = [];
+                return false;
+        }
+    }
+
+    private static int DefaultStride(MimirVideoFrameDescriptor frame) =>
+        frame.PixelFormat switch
+        {
+            MimirVideoPixelFormat.Yuy2 or MimirVideoPixelFormat.LeapStereoIr or MimirVideoPixelFormat.Rg8 => frame.Width * 2,
+            MimirVideoPixelFormat.Bgra8 => frame.Width * 4,
+            _ => frame.Width,
+        };
+
+    private sealed class SourceState(string sourceId, string controlKind)
+    {
+        public string SourceId { get; } = sourceId;
+        public string ControlKind { get; } = controlKind;
+        public ExposureControlState State { get; set; } = ExposureControlState.Idle;
+        public int SettingIndex { get; set; } = -1;
+        public MimirOnlineExposureGainSetting? CurrentSetting { get; set; }
+        public MimirOnlineExposureGainSetting? BestSetting { get; set; }
+        public MimirLedSplineQualityReport? BestQuality { get; set; }
+        public Accumulator CurrentScore { get; set; } = new();
+        public ulong LastScoredSequence { get; set; } = ulong.MaxValue;
+        public DateTimeOffset NextTransitionUtc { get; set; } = DateTimeOffset.MinValue;
+        public bool LastApplySucceeded { get; set; }
+        public string Reason { get; set; } = "not-started";
+
+        public void BeginSweep()
+        {
+            State = ExposureControlState.Scanning;
+            SettingIndex = 0;
+            BestSetting = null;
+            BestQuality = null;
+            CurrentScore = new Accumulator();
+            LastScoredSequence = ulong.MaxValue;
+            Reason = "begin-sweep";
+        }
+
+        public void MarkUnsupported()
+        {
+            State = ExposureControlState.Unsupported;
+            Reason = "exposure-gain-not-supported";
+        }
+
+        public MimirCameraExposureControlStatus ToStatus()
+        {
+            var best = BestSetting;
+            var quality = BestQuality;
+            return new MimirCameraExposureControlStatus(
+                SourceId,
+                ControlKind,
+                State != ExposureControlState.Unsupported,
+                State.ToString(),
+                CurrentSetting?.Id ?? "",
+                CurrentSetting?.Exposure,
+                CurrentSetting?.Gain,
+                best?.Id ?? "",
+                best?.Exposure,
+                best?.Gain,
+                quality?.Score ?? 0.0,
+                quality?.DetectedLedCount ?? 0,
+                quality?.UsableForCalibration ?? false,
+                CurrentScore.FramesScored,
+                LastApplySucceeded,
+                Reason);
+        }
+    }
+
+    private sealed class Accumulator
+    {
+        public int FramesScored { get; private set; }
+        public MimirLedSplineQualityReport? BestQuality { get; private set; }
+
+        public void Add(MimirLedSplineQualityReport quality)
+        {
+            FramesScored++;
+            if (BestQuality == null ||
+                quality.Score > BestQuality.Score ||
+                (Math.Abs(quality.Score - BestQuality.Score) <= 1.0e-9 &&
+                    quality.DetectedLedCount > BestQuality.DetectedLedCount))
+            {
+                BestQuality = quality;
+            }
+        }
+    }
+
+    private enum ExposureControlState
+    {
+        Idle,
+        Scanning,
+        Locked,
+        Unsupported,
+    }
+}

src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs

@@ -38,6 +38,9 @@ public sealed class MimirSynchronizationHub : IDisposable
 
     public int SourceCount => sources.Count;
 
+    public IReadOnlyList<IMimirCameraExposureGainActuator> CameraExposureGainActuators =>
+        sources.OfType<IMimirCameraExposureGainActuator>().ToArray();
+
     public IReadOnlyList<MimirAudioSynchronizationState> AudioSynchronizationStates =>
         audioSynchronizationState.States;
 

src/Mimir.Runtime/Synchronization/MimirVideoCaptureDriverSource.cs

@@ -1,6 +1,6 @@
 namespace Mimir.Runtime.Synchronization;
 
-public sealed class MimirVideoCaptureDriverSource : IMimirStreamSource, IMimirFensalirTextureLeaseReceiver
+public sealed class MimirVideoCaptureDriverSource : IMimirStreamSource, IMimirFensalirTextureLeaseReceiver, IMimirCameraExposureGainActuator
 {
     private readonly IMimirVideoCaptureDriver driver;
     private readonly MimirNativeIngestStreamSource nativeSource;
@@ -24,6 +24,13 @@ public sealed class MimirVideoCaptureDriverSource : IMimirStreamSource, IMimirFe
 
     public MimirStreamDescriptor Descriptor { get; }
 
+    public string SourceId => Descriptor.SourceId;
+
+    public string ExposureControlKind =>
+        driver is MimirKsVideoCaptureDriver ? "ks-exposure-gain" : "fixed-control";
+
+    public bool SupportsExposureGain => driver is MimirKsVideoCaptureDriver;
+
     public Func<long> ReadArrivalNs { get; }
 
     public int LastUploadedCopyCount { get; private set; }
@@ -39,6 +46,10 @@ public sealed class MimirVideoCaptureDriverSource : IMimirStreamSource, IMimirFe
         }
     }
 
+    public bool TrySetExposureGain(int? exposure, int? gain) =>
+        driver is MimirKsVideoCaptureDriver ksDriver &&
+        ksDriver.TrySetExposureGain(exposure, gain);
+
     public bool TryRead(out MimirStreamSample sample)
     {
         if (nativeSource.TryRead(out sample))

src/Mimir.Well/Program.cs

@@ -51,7 +51,14 @@ var sequence = 0L;
 var startedAt = DateTimeOffset.UtcNow;
 var nextPublish = DateTimeOffset.MinValue;
 var nextSync = DateTimeOffset.MinValue;
+var nextVisualCalibration = DateTimeOffset.MinValue;
 var presentation = new MimirPresentationControlState();
+var exposureController = new MimirCameraExposureController(new MimirCameraExposureControlOptions(
+    options.VisualCalibrationEnabled,
+    options.VisualExpectedLedCount,
+    options.VisualMinimumLuma,
+    options.VisualSettingSeconds,
+    options.VisualResweepSeconds));
 Console.Error.WriteLine($"Mimir Well publishing to {options.PublishUrl}");
 Console.Error.WriteLine($"Mimir Well sources={runtimeConfig.SourceFactories.Count} buffers={hub.Buffers.Buffers.Count}");
 
@@ -77,6 +84,12 @@ while (!stopping.IsCancellationRequested)
         nextSync = now + TimeSpan.FromMilliseconds(options.SyncIntervalMs);
     }
 
+    if (now >= nextVisualCalibration)
+    {
+        exposureController.Update(now, hub.Buffers.Buffers, hub.CameraExposureGainActuators);
+        nextVisualCalibration = now + TimeSpan.FromMilliseconds(options.VisualCalibrationIntervalMs);
+    }
+
     if (now >= nextPublish)
     {
         presentation.SyncFromBuffers(hub.Buffers.Buffers);
@@ -87,6 +100,7 @@ while (!stopping.IsCancellationRequested)
             hub,
             presentation,
             frame,
+            exposureController.Statuses,
             sourceErrors,
             ++sequence,
             startedAt);
@@ -142,6 +156,12 @@ internal sealed record MimirWellOptions(
     int PresentationDelayMs,
     int MaxSamplesPerSource,
     int SyncCandidatesPerStep,
+    bool VisualCalibrationEnabled,
+    int VisualCalibrationIntervalMs,
+    int VisualExpectedLedCount,
+    double VisualMinimumLuma,
+    double VisualSettingSeconds,
+    double VisualResweepSeconds,
     int MeterEvery)
 {
     public static MimirWellOptions Parse(string[] args) => new(
@@ -154,6 +174,12 @@ internal sealed record MimirWellOptions(
         ParseInt(args, "--presentation-delay-ms", 2500),
         ParseInt(args, "--max-samples-per-source", 4),
         ParseInt(args, "--sync-candidates", 1),
+        ParseBool(args, "--visual-calibration", true),
+        ParseInt(args, "--visual-calibration-ms", 250),
+        ParseInt(args, "--visual-expected-leds", 38),
+        ParseDouble(args, "--visual-minimum-luma", 0.55),
+        ParseDouble(args, "--visual-setting-seconds", 0.75),
+        ParseDouble(args, "--visual-resweep-seconds", 12.0),
         ParseInt(args, "--meter-every", 20));
 
     private static string ParseString(IReadOnlyList<string> args, string name, string fallback)
@@ -174,6 +200,27 @@ internal sealed record MimirWellOptions(
 
     private static double ParseDouble(IReadOnlyList<string> args, string name, double fallback) =>
         double.TryParse(ParseString(args, name, ""), out var value) ? value : fallback;
+
+    private static bool ParseBool(IReadOnlyList<string> args, string name, bool fallback)
+    {
+        for (var index = 0; index < args.Count; index++)
+        {
+            if (!string.Equals(args[index], name, StringComparison.OrdinalIgnoreCase))
+            {
+                continue;
+            }
+
+            if (index == args.Count - 1 ||
+                args[index + 1].StartsWith("--", StringComparison.Ordinal))
+            {
+                return true;
+            }
+
+            return bool.TryParse(args[index + 1], out var value) ? value : fallback;
+        }
+
+        return fallback;
+    }
 }
 
 internal sealed class MimirWellPublisher(Uri url) : IAsyncDisposable
@@ -218,6 +265,7 @@ internal static class MimirWellSnapshot
         MimirSynchronizationHub hub,
         MimirPresentationControlState presentation,
         MimirSynchronizedBufferFrame frame,
+        IReadOnlyList<MimirCameraExposureControlStatus> visualCalibration,
         IReadOnlyList<object> sourceErrors,
         long sequence,
         DateTimeOffset startedAt) => new
@@ -279,7 +327,34 @@ internal static class MimirWellSnapshot
                     hub.LastBioacousticProbeSchedule.AggregateSyncConfidence,
                     hub.LastBioacousticProbeSchedule.AggregateFrequencyResponseConfidence,
                     reason = hub.LastBioacousticProbeSchedule.Reason.ToString(),
-                },
+            },
+        },
+        visualCalibration = new
+        {
+            enabled = options.VisualCalibrationEnabled,
+            expectedLedCount = options.VisualExpectedLedCount,
+            minimumLuma = options.VisualMinimumLuma,
+            settingSeconds = options.VisualSettingSeconds,
+            resweepSeconds = options.VisualResweepSeconds,
+            cameras = visualCalibration.Select(camera => new
+            {
+                camera.SourceId,
+                camera.ControlKind,
+                camera.SupportsExposureGain,
+                camera.State,
+                camera.CurrentSettingId,
+                camera.CurrentExposure,
+                camera.CurrentGain,
+                camera.BestSettingId,
+                camera.BestExposure,
+                camera.BestGain,
+                camera.BestScore,
+                camera.BestDetectedLedCount,
+                camera.BestUsableForCalibration,
+                camera.FramesScored,
+                camera.LastApplySucceeded,
+                camera.Reason,
+            }).ToArray(),
         },
         composite = new
         {

state/evidence.jsonl

@@ -343,3 +343,4 @@
 {"ts": "2026-06-01T22:49:16+01:00", "type": "implementation", "status": "ok", "note": "2026-06-01: Corrected the Move/Syrinx witness invariant. Online Move music sync no longer loops an independent Realtek Syrinx bed; ScoreGestureScheduler now owns sparse score strikes with note target, glide, vibrato, harmonic lane, and intensity, and BioacousticRealtekTrigger emits a short Realtek AquaSynth Syrinx call only when that same score strike drives a Move pulse. Trace records now include score_voice_contours, score_voice_events, and emitted_audio_event_count for Mimir Well/offline timeline correlation. Live run artifacts/runtime/move-syrinx-witness-20260601-224705 produced 79 voice events, 69 emitted Realtek calls, and all three Moves participating in a 1764-frame poll."}
 {"ts": "2026-06-01T23:24:26+01:00", "type": "implementation", "status": "ok", "note": "2026-06-01: Added live score detection to the Move/Syrinx worker. OnlineAnalyzer now emits tempo_candidates and a 12-bin cyclic_tempogram from onset autocorrelation. LiveScoreEstimator publishes mimir.live_score.v1 with detected voices, note names, durations, score confidence, target-confidence deficit, gesture density, and per-Move note targets. ScoreGestureScheduler consumes that score document so Move/Syrinx calibration gestures are spread across detected voices/spectral lanes according to confidence deficit. Live run artifacts/runtime/move-live-score-20260601-232156 sampled 350/350 records with live_score, maxVoices=3, maxTempoCandidates=8, 21 score event frames, 20 emitted Syrinx calls, and Move targets assigned to shotgun-mic, realtek-loopback, and cardioid-mic."}
 {"ts": "2026-06-01T23:37:14+01:00", "type": "implementation", "status": "ok", "note": "2026-06-01: Move/Syrinx score gestures now carry explicit articulation roles. ScoreGestureScheduler assigns Moves to violin-syrinx, harp-arpeggio, and peck-syrinx by default; events include articulation, gesture note sequence, attack shape, and duration hint. Move RGB lowering and Realtek AquaSynth playback consume the same event so each pulse remains one audiovisual calibration voice. Verified with a synthetic scheduler probe plus py_compile and launcher dry-run."}
+{"ts": "2026-06-01T23:50:56+01:00", "type": "implementation", "status": "ok", "note": "2026-06-01: Mimir.Well now owns online camera exposure/gain optimization for visual sync. MimirCameraExposureController samples live Well video buffers, scores LED spline quality with MimirLedSplineFrameAnalyzer, cycles KS exposure/gain candidates, locks the best score, periodically resweeps, and publishes visualCalibration status in mimir.well_snapshot.v1. Verified with dotnet build src/Mimir.Well and supervisor dry-run options."}

state/map.yaml

@@ -193,6 +193,16 @@ invariants:
     for both Kiyo Pro and Kiyo Basic. LeapUVC rejected the generic KS
     exposure/gain writes and PS3 Eye capture remains fixed-control until its
     native wrapper owns gain/exposure.
+    `Mimir.Well` now owns the online version of that visual-sync actuator:
+    `MimirCameraExposureController` samples latest video frames from live Well
+    buffers, scores the LED spline with `MimirLedSplineFrameAnalyzer`, cycles
+    controllable KS camera exposure/gain candidates, locks the best spline
+    score, and periodically resweeps as room lighting and performer pose
+    change. `MimirVideoCaptureDriverSource` exposes only the narrow
+    `IMimirCameraExposureGainActuator` command surface, so the Well does not
+    know KS internals. `mimir.well_snapshot.v1` now publishes
+    `visualCalibration` status per camera: current setting, best setting,
+    score, detected LED count, usability, frame count, apply status, and reason.
   - PS Move is usable as an active marker/IMU witness, not as an optical depth
     owner. `Mimir.PsMoveProbe` now enumerates Sony `VID_054C` / `PID_03D5`
     through Windows HID without third-party packages. The connected controller