From c7f4acbb78c5478fcea362aa0f028ba27f2d6dfb Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 31 May 2026 23:52:23 +0100 Subject: [PATCH] Add Leap Eyes Kiyo MVP launcher --- ...imir-runtime.mvp-leap-eyes-kiyo.local.json | 67 +++++++++++++ docs/move-controller-stream.md | 21 ++++ scripts/start-mvp-leap-eyes-kiyo.ps1 | 98 +++++++++++++++++++ state/evidence.jsonl | 1 + state/map.yaml | 8 ++ tools/nightwing_typed_witness_publisher.py | 92 +++++++++++++++++ tools/nw_eye_cap.py | 29 ++++-- 7 files changed, 308 insertions(+), 8 deletions(-) create mode 100644 config/mimir-runtime.mvp-leap-eyes-kiyo.local.json create mode 100644 scripts/start-mvp-leap-eyes-kiyo.ps1 diff --git a/config/mimir-runtime.mvp-leap-eyes-kiyo.local.json b/config/mimir-runtime.mvp-leap-eyes-kiyo.local.json new file mode 100644 index 0000000..f5bdf7d --- /dev/null +++ b/config/mimir-runtime.mvp-leap-eyes-kiyo.local.json @@ -0,0 +1,67 @@ +{ + "bufferSeconds": 5.0, + "audioSync": { + "mode": "passive", + "referenceSourceId": "asio-ch2", + "calibrationGain": 0.0, + "watermarkGain": 0.0, + "complexContourChannelModelPath": "../calibration/bioacoustic/complex-contour-channel-model.json", + "enableComplexContourRuntime": true + }, + "streams": [ + { + "sourceId": "focusrite-asio", + "displayName": "Starfire Scarlett ASIO", + "kind": "Audio", + "origin": "LocalDevice", + "enabled": true, + "adapter": "asio", + "command": "../native/asio_capture/build/Release/mimir_asio_capture.dll", + "sampleRate": 192000, + "acceptSourceIds": [ + "asio-ch0", + "asio-ch1", + "asio-ch2", + "asio-ch3" + ], + "sourceLabels": { + "asio-ch0": "Shotgun hero mic", + "asio-ch1": "Cardioid hero mic", + "asio-ch2": "Raven/program loopback L", + "asio-ch3": "Raven/program loopback R" + } + }, + { + "sourceId": "leap-stereo-ir", + "displayName": "LeapUVC stereo IR / depth root", + "kind": "Video", + "origin": "LocalDevice", + "enabled": true, + "adapter": "ks-camera", + "command": "../native/camera_capture/build/Release/mimir_camera_capture.dll", + "pathNeedle": "vid_f182&pid_0003&mi_00", + "width": 640, + "height": 240, + "inputFormat": "YUY2", + "pixelFormat": "LeapStereoIr", + "minimumFramesPerSecond": 100.0, + "queueDepth": 8 + }, + { + "sourceId": "kiyo-pro-rgb", + "displayName": "Kiyo Pro RGB program view", + "kind": "Video", + "origin": "LocalDevice", + "enabled": true, + "adapter": "ks-camera", + "command": "../native/camera_capture/build/Release/mimir_camera_capture.dll", + "pathNeedle": "vid_1532&pid_0e05&mi_00", + "width": 640, + "height": 480, + "inputFormat": "YUY2", + "pixelFormat": "YUY2", + "minimumFramesPerSecond": 25.0, + "queueDepth": 8 + } + ] +} diff --git a/docs/move-controller-stream.md b/docs/move-controller-stream.md index 92affa9..cf4fe1c 100644 --- a/docs/move-controller-stream.md +++ b/docs/move-controller-stream.md @@ -81,6 +81,27 @@ correctly drops the result to `InsufficientWitnesses`. ## Runtime Profile +For the ASAP MVP stream, use the narrower Leap/Eyes/Kiyo Pro profile: + +```powershell +E:\Projects\Mimir\scripts\start-mvp-leap-eyes-kiyo.ps1 +``` + +This launches Starfire with only local Scarlett ASIO, local Leap stereo IR, and +local Kiyo Pro RGB in the Mimir rolling buffers. Nightwing does not stream PS3 +Eye pixels to Starfire. The launcher stages `nw_eye_cap.py`, +`nw_move_hint.py`, and `nightwing_typed_witness_publisher.py` on Nightwing, +then starts the typed witness publisher with `--track-eyes`. Nightwing reads +`/dev/video2` and `/dev/video3`, extracts compact Move-sphere observations, +and sends `mimir.move_controller_observation_state.v1` claims to the +`/eve/periwinkle` CultMesh observation receiver. Starfire/Fensalir can consume +those claims for overlays and calibration without inheriting Nightwing's raw +camera bandwidth or local tracking authority. + +The MVP profile refuses to start when LeapUVC is not enumerated. That is +intentional: the Leap is the dense close-range geometry root, not an optional +checkbox pretending to be present. + Use: ```powershell diff --git a/scripts/start-mvp-leap-eyes-kiyo.ps1 b/scripts/start-mvp-leap-eyes-kiyo.ps1 new file mode 100644 index 0000000..ebd8e76 --- /dev/null +++ b/scripts/start-mvp-leap-eyes-kiyo.ps1 @@ -0,0 +1,98 @@ +param( + [switch]$Headless, + [int]$HeadlessWidth = 1280, + [int]$HeadlessHeight = 720, + [string]$CaptureFrame = "", + [string]$NightwingHost = "nightwing", + [string]$WitnessUrl = "ws://192.168.1.66:8796/eve/periwinkle", + [string]$ConfigPath = "E:\Projects\Mimir\config\mimir-runtime.mvp-leap-eyes-kiyo.local.json" +) + +$ErrorActionPreference = "Stop" +$repo = Split-Path -Parent $PSScriptRoot +$logDir = Join-Path $repo "artifacts\runtime\mvp-leap-eyes-kiyo" +New-Item -ItemType Directory -Force -Path $logDir | Out-Null + +function Test-CommandOk { + param([string[]]$Command, [string]$Name) + $process = Start-Process -FilePath $Command[0] -ArgumentList $Command[1..($Command.Length - 1)] -NoNewWindow -Wait -PassThru -RedirectStandardOutput (Join-Path $logDir "$Name.out.log") -RedirectStandardError (Join-Path $logDir "$Name.err.log") + return $process.ExitCode -eq 0 +} + +$kiyo = Get-PnpDevice -PresentOnly | Where-Object { $_.InstanceId -match "VID_1532&PID_0E05&MI_00" } | Select-Object -First 1 +$leap = Get-PnpDevice -PresentOnly | Where-Object { $_.InstanceId -match "VID_F182&PID_0003&MI_00" } | Select-Object -First 1 + +if (-not $kiyo) { + throw "Kiyo Pro camera interface is not present. MVP needs Kiyo Pro RGB." +} + +if (-not $leap) { + throw "LeapUVC camera interface is not present. Replug/wake the Leap before launching this MVP profile." +} + +$nightwingCheck = ssh $NightwingHost "test -e /dev/video2 -a -e /dev/video3 && command -v python3 >/dev/null && echo ok" 2>$null +if ($LASTEXITCODE -ne 0 -or $nightwingCheck -notmatch "ok") { + throw "Nightwing does not expose /dev/video2 + /dev/video3 with python3 available." +} + +scp (Join-Path $repo "tools\nw_eye_cap.py") "${NightwingHost}:/tmp/nw_eye_cap.py" | Out-Null +if ($LASTEXITCODE -ne 0) { + throw "Could not stage nw_eye_cap.py on Nightwing." +} +scp (Join-Path $repo "tools\nw_move_hint.py") "${NightwingHost}:/tmp/nw_move_hint.py" | Out-Null +if ($LASTEXITCODE -ne 0) { + throw "Could not stage nw_move_hint.py on Nightwing." +} +scp (Join-Path $repo "tools\nightwing_typed_witness_publisher.py") "${NightwingHost}:/tmp/nightwing_typed_witness_publisher.py" | Out-Null +if ($LASTEXITCODE -ne 0) { + throw "Could not stage nightwing_typed_witness_publisher.py on Nightwing." +} + +$receiverRunning = Get-CimInstance Win32_Process | + Where-Object { $_.CommandLine -match "Mimir.EveSensorReceiver" -and $_.CommandLine -match "--port 8796" } | + Select-Object -First 1 +if (-not $receiverRunning) { + $receiverOut = Join-Path $logDir "nightwing-witness-receiver.out.log" + $receiverErr = Join-Path $logDir "nightwing-witness-receiver.err.log" + Start-Process -WindowStyle Hidden -FilePath "dotnet" -ArgumentList @( + "run", + "--no-build", + "--project", + (Join-Path $repo "src\Mimir.EveSensorReceiver\Mimir.EveSensorReceiver.csproj"), + "--", + "--port", + "8796", + "--path", + "/eve/periwinkle", + "--source-id", + "nightwing-witness", + "--type", + "cultmesh-observation" + ) -RedirectStandardOutput $receiverOut -RedirectStandardError $receiverErr + Start-Sleep -Milliseconds 800 +} + +$remotePublisher = "nohup python3 /tmp/nightwing_typed_witness_publisher.py --url '$WitnessUrl' --track-eyes --interval 0.12 > /tmp/mimir-nightwing-mvp-witness.log 2>&1 &" +ssh $NightwingHost $remotePublisher | Out-Null +if ($LASTEXITCODE -ne 0) { + throw "Could not start Nightwing typed witness publisher." +} + +$env:MIMIR_RUNTIME_CONFIG = $ConfigPath +$args = @("run", "--project", (Join-Path $repo "src\Mimir.App\Mimir.App.csproj")) +if ($Headless) { + $args += "--" + $args += "--headless" + $args += "--headless-width" + $args += "$HeadlessWidth" + $args += "--headless-height" + $args += "$HeadlessHeight" + if ($CaptureFrame) { + $args += "--capture-frame" + $args += $CaptureFrame + } +} + +Write-Host "MIMIR_RUNTIME_CONFIG=$ConfigPath" +Write-Host "Starting MVP: Starfire Leap + Kiyo Pro, Nightwing Eye/Move witness claims" +dotnet @args diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 31d6a94..a9861e3 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -297,3 +297,4 @@ {"ts": "2026-05-31T23:31:00+01:00", "type": "implementation", "status": "ok", "note": "sync_sweep.py now accepts a mimir.music_keyed_move_chirp_plan.v1 as the shared gesture authority for articulated chirps and chirp-flashes. The sweep copies/renders the plan WAV, sends exact Move visual contour samples over HID, attaches RGB visual_contour decoder scores to video receipts, and runs Mimir perfect-machine live receipts for local Leap/Kiyos plus Raven/Eve configs unless disabled. This fixes the earlier DirectShow/Nightwing-only sweep omission path: Leap/Eve are explicit receipt surfaces instead of assumed present."} {"ts": "2026-05-31T23:48:00+01:00", "type": "verification", "status": "partial", "note": "Ran an articulated all-sensor sweep from a live Scarlett-derived music plan at artifacts/runtime/all-sensor-articulated-20260531-232349. ASIO loopback channels 2/3 matched 77 chirp events with 0.083/0.292 ms median residuals; Nightwing builtin cam and nw-eye-1 each emitted 100/100 Move observations; Kiyo Pro/Kiyo captured RGB video and offline crunch extracted 47/51 marker observations. tools/crunch_sync_sweep.py wrote 298 total marker observations to calibration-observations.json/csv. Missing witnesses are explicit: nw-eye-0 timed out, Nightwing mic only matched one chirp, Focusrite DShow was unavailable while ASIO succeeded, LeapUVC KS open failed, and Raven/Eve receiver runtime saw no camera textures."} {"ts": "2026-05-31T23:54:00+01:00", "type": "verification", "status": "ok", "note": "The articulated sweep crunch now reports pairwise shared marker events for calibration solve selection. In the 20260531-232349 run: Nightwing builtin cam and nw-eye-1 share 100 events; Kiyo shares 51 with each Nightwing view; Kiyo Pro shares 47 with each Nightwing view; Kiyo and Kiyo Pro share 28. This is enough correspondence substrate for initial multi-view frustum fitting on the sensors that actually saw the Move spheres."} +{"ts": "2026-06-01T00:12:00+01:00", "type": "decision", "status": "ok", "note": "Corrected the MVP stream split after user pushback: Starfire should not ingest Nightwing PS3 Eye pixels. The MVP profile now keeps Starfire buffers to local Scarlett ASIO, local Leap stereo IR, and local Kiyo Pro RGB; Nightwing owns PS3 Eye reads and emits compact mimir.move_controller_observation_state.v1 typed witness claims through the /eve/periwinkle CultMesh observation receiver. The launcher refuses to start without LeapUVC enumerated, because Leap is the MVP geometry root."} diff --git a/state/map.yaml b/state/map.yaml index 0b25511..212a36d 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -211,6 +211,14 @@ invariants: rather than silently inferred. `tools/crunch_sync_sweep.py` turns a sweep directory into `calibration-observations.json/csv` by merging Nightwing's typed Move observations with local Kiyo frame-derived marker centroids. + The ASAP streaming MVP is narrower and authority-correct: + `config/mimir-runtime.mvp-leap-eyes-kiyo.local.json` keeps Starfire's live + rolling buffers to local Scarlett ASIO, local Leap stereo IR, and local + Kiyo Pro RGB only. Nightwing's PS3 Eyes do not cross as raw video; the + `start-mvp-leap-eyes-kiyo.ps1` launcher stages Nightwing tools and starts + `nightwing_typed_witness_publisher.py --track-eyes`, so Nightwing publishes + compact `mimir.move_controller_observation_state.v1` claims through the + `/eve/periwinkle` CultMesh observation receiver. `MimirMoveControllerTracker` is the first Mimir-native overlay owner for multiple PS Move controllers: it tracks bright sphere observations from PS3 Eye luma frames, keeps a bounded history per `controllerId`, preserves diff --git a/tools/nightwing_typed_witness_publisher.py b/tools/nightwing_typed_witness_publisher.py index 4b8e238..41b41a8 100644 --- a/tools/nightwing_typed_witness_publisher.py +++ b/tools/nightwing_typed_witness_publisher.py @@ -20,6 +20,13 @@ import struct import time import urllib.parse +try: + import nw_eye_cap + import nw_move_hint +except Exception: + nw_eye_cap = None + nw_move_hint = None + def pack_nil() -> bytes: return b"\xc0" @@ -206,6 +213,80 @@ def observations(sequence: int) -> list[bytes]: return docs +def eye_tracking_observations(args: argparse.Namespace, sequence: int) -> list[bytes]: + if nw_eye_cap is None or nw_move_hint is None: + return [] + docs: list[bytes] = [] + devices = [part for part in args.eye_devices.split(",") if part] + for eye_index, device in enumerate(devices): + source_id = f"ps3-eye-{eye_index}" + stats_path = f"/tmp/mimir-mvp-{source_id}.json" + cap_args = argparse.Namespace( + device=device, + width=args.eye_width, + height=args.eye_height, + format="YUYV", + fps=args.eye_fps, + seconds=args.eye_window_seconds, + buffers=8, + out=stats_path, + stdout=False, + ) + try: + stats = nw_eye_cap.capture(cap_args) + raw_path = stats["frames_path"] + width = int(stats["width"]) + height = int(stats["height"]) + frame_bytes = width * height + with open(raw_path, "rb") as handle: + raw = handle.read() + frame_count = min(len(raw) // frame_bytes, len(stats.get("timestamps_s", []))) + stride = max(1, int(args.tracking_stride)) + for frame_index in range(0, frame_count, stride): + frame = raw[frame_index * frame_bytes:(frame_index + 1) * frame_bytes] + blob = nw_move_hint.best_blob(frame, width, height) + if blob is None or float(blob["confidence"]) < args.min_confidence: + continue + values = [ + float(blob["clip_x"]), + float(blob["clip_y"]), + float(blob["area_px"]), + float(blob["radius_px"]), + float(blob["mean_luma"]), + float(blob["peak_luma"]), + float(blob["confidence"]), + float(frame_index), + float(width), + float(height), + ] + docs.append(sensor_document( + observation_id=f"nightwing:{source_id}:move-sphere:{sequence}:{frame_index}", + device_id="nightwing", + stream_id=source_id, + kind="mimir.move_controller_observation_state.v1", + sequence=sequence * 100000 + frame_index, + values=values, + accuracy=3, + )) + except Exception as ex: + docs.append(sensor_document( + observation_id=f"nightwing:{source_id}:tracking-error:{sequence}", + device_id="nightwing", + stream_id=source_id, + kind=f"tracking-error:{type(ex).__name__}", + sequence=sequence, + values=[0.0], + accuracy=0, + )) + finally: + try: + if "raw_path" in locals(): + os.unlink(raw_path) + except OSError: + pass + return docs + + def run(args: argparse.Namespace) -> None: sequence = 0 while True: @@ -215,6 +296,9 @@ def run(args: argparse.Namespace) -> None: while True: for payload in observations(sequence): ws.send_binary(payload) + if args.track_eyes: + for payload in eye_tracking_observations(args, sequence): + ws.send_binary(payload) sequence += 1 if args.once: return @@ -232,6 +316,14 @@ def main() -> int: parser.add_argument("--interval", type=float, default=0.25) parser.add_argument("--reconnect-seconds", type=float, default=2.0) parser.add_argument("--once", action="store_true") + parser.add_argument("--track-eyes", action="store_true", help="Publish compact Nightwing-local PS3 Eye Move-sphere observations.") + parser.add_argument("--eye-devices", default="/dev/video2,/dev/video3") + parser.add_argument("--eye-width", type=int, default=320) + parser.add_argument("--eye-height", type=int, default=240) + parser.add_argument("--eye-fps", type=int, default=187) + parser.add_argument("--eye-window-seconds", type=float, default=0.12) + parser.add_argument("--tracking-stride", type=int, default=6) + parser.add_argument("--min-confidence", type=float, default=0.25) args = parser.parse_args() run(args) return 0 diff --git a/tools/nw_eye_cap.py b/tools/nw_eye_cap.py index 0891b17..a1fd768 100644 --- a/tools/nw_eye_cap.py +++ b/tools/nw_eye_cap.py @@ -11,6 +11,7 @@ import json import mmap import os import select +import sys import time from pathlib import Path @@ -166,9 +167,10 @@ def yuyv_to_luma(raw: bytes, width: int, height: int, stride: int) -> bytes: def capture(args: argparse.Namespace) -> dict: fd = os.open(args.device, os.O_RDWR | os.O_NONBLOCK) maps: list[mmap.mmap] = [] - out = Path(args.out) - out.parent.mkdir(parents=True, exist_ok=True) - frames_path = out.with_suffix(".y8") + out = Path(args.out) if args.out else None + if out is not None: + out.parent.mkdir(parents=True, exist_ok=True) + frames_path = out.with_suffix(".y8") if out is not None else None try: set_frame_interval(fd, args.fps) fmt = V4L2Format() @@ -201,7 +203,8 @@ def capture(args: argparse.Namespace) -> dict: deadline = start + args.seconds timestamps = [] sequences = [] - with frames_path.open("wb") as frames: + frames = sys.stdout.buffer if args.stdout else frames_path.open("wb") # type: ignore[union-attr] + try: while time.monotonic() < deadline: ready, _, _ = select.select([fd], [], [], 0.25) if not ready: @@ -218,6 +221,9 @@ def capture(args: argparse.Namespace) -> dict: timestamps.append(float(buf.timestamp.tv_sec) + float(buf.timestamp.tv_usec) / 1_000_000.0) sequences.append(int(buf.sequence)) xioctl(fd, VIDIOC_QBUF, buf) + finally: + if not args.stdout: + frames.close() try: fcntl.ioctl(fd, VIDIOC_STREAMOFF, stream_type) except OSError: @@ -227,7 +233,7 @@ def capture(args: argparse.Namespace) -> dict: gap = max(0, (sequences[-1] - sequences[0] + 1) - len(sequences)) if len(sequences) >= 2 else 0 stats = { "device": args.device, - "frames_path": str(frames_path), + "frames_path": str(frames_path) if frames_path is not None else "stdout", "width": int(actual.width), "height": int(actual.height), "stride": int(actual.bytesperline), @@ -238,7 +244,10 @@ def capture(args: argparse.Namespace) -> dict: "sequence_gap": gap, "duration_s": (rel[-1] - rel[0]) if len(rel) >= 2 else 0.0, } - out.write_text(json.dumps(stats, indent=2, sort_keys=True), encoding="utf-8") + if out is not None: + out.write_text(json.dumps(stats, indent=2, sort_keys=True), encoding="utf-8") + else: + print(json.dumps(stats, sort_keys=True), file=sys.stderr) return stats finally: for mapping in maps: @@ -255,10 +264,14 @@ def main() -> int: parser.add_argument("--fps", type=int, default=187) parser.add_argument("--seconds", type=float, default=8.0) parser.add_argument("--buffers", type=int, default=8) - parser.add_argument("--out", required=True) + parser.add_argument("--out", default=None) + parser.add_argument("--stdout", action="store_true", help="Stream raw luma frames to stdout and write stats to stderr.") args = parser.parse_args() + if not args.stdout and not args.out: + parser.error("--out is required unless --stdout is set") stats = capture(args) - print(json.dumps(stats, sort_keys=True)) + if not args.stdout: + print(json.dumps(stats, sort_keys=True)) return 0 if stats["frames"] else 1