GameCult/Mimir
Watch
9
0
Fork
You've already forked Mimir
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add Leap Eyes Kiyo MVP launcher

Browse source
This commit is contained in:
Meta Vens 2026-05-31 23:52:23 +01:00
commit c7f4acbb78
7 changed files with 308 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

67
config/mimir-runtime.mvp-leap-eyes-kiyo.local.json Normal file
View file

@ -0,0 +1,67 @@
{
"bufferSeconds": 5.0,
"audioSync": {
"mode": "passive",
"referenceSourceId": "asio-ch2",
"calibrationGain": 0.0,
"watermarkGain": 0.0,
"complexContourChannelModelPath": "../calibration/bioacoustic/complex-contour-channel-model.json",
"enableComplexContourRuntime": true
},
"streams": [
{
"sourceId": "focusrite-asio",
"displayName": "Starfire Scarlett ASIO",
"kind": "Audio",
"origin": "LocalDevice",
"enabled": true,
"adapter": "asio",
"command": "../native/asio_capture/build/Release/mimir_asio_capture.dll",
"sampleRate": 192000,
"acceptSourceIds": [
"asio-ch0",
"asio-ch1",
"asio-ch2",
"asio-ch3"
],
"sourceLabels": {
"asio-ch0": "Shotgun hero mic",
"asio-ch1": "Cardioid hero mic",
"asio-ch2": "Raven/program loopback L",
"asio-ch3": "Raven/program loopback R"
}
},
{
"sourceId": "leap-stereo-ir",
"displayName": "LeapUVC stereo IR / depth root",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_f182&pid_0003&mi_00",
"width": 640,
"height": 240,
"inputFormat": "YUY2",
"pixelFormat": "LeapStereoIr",
"minimumFramesPerSecond": 100.0,
"queueDepth": 8
},
{
"sourceId": "kiyo-pro-rgb",
"displayName": "Kiyo Pro RGB program view",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_1532&pid_0e05&mi_00",
"width": 640,
"height": 480,
"inputFormat": "YUY2",
"pixelFormat": "YUY2",
"minimumFramesPerSecond": 25.0,
"queueDepth": 8
}
]
}

21
docs/move-controller-stream.md
View file

@ -81,6 +81,27 @@ correctly drops the result to `InsufficientWitnesses`.
## Runtime Profile
For the ASAP MVP stream, use the narrower Leap/Eyes/Kiyo Pro profile:
```powershell
E:\Projects\Mimir\scripts\start-mvp-leap-eyes-kiyo.ps1
```
This launches Starfire with only local Scarlett ASIO, local Leap stereo IR, and
local Kiyo Pro RGB in the Mimir rolling buffers. Nightwing does not stream PS3
Eye pixels to Starfire. The launcher stages `nw_eye_cap.py`,
`nw_move_hint.py`, and `nightwing_typed_witness_publisher.py` on Nightwing,
then starts the typed witness publisher with `--track-eyes`. Nightwing reads
`/dev/video2` and `/dev/video3`, extracts compact Move-sphere observations,
and sends `mimir.move_controller_observation_state.v1` claims to the
`/eve/periwinkle` CultMesh observation receiver. Starfire/Fensalir can consume
those claims for overlays and calibration without inheriting Nightwing's raw
camera bandwidth or local tracking authority.
The MVP profile refuses to start when LeapUVC is not enumerated. That is
intentional: the Leap is the dense close-range geometry root, not an optional
checkbox pretending to be present.
Use:
```powershell

98
scripts/start-mvp-leap-eyes-kiyo.ps1 Normal file
View file

@ -0,0 +1,98 @@
param(
[switch]$Headless,
[int]$HeadlessWidth = 1280,
[int]$HeadlessHeight = 720,
[string]$CaptureFrame = "",
[string]$NightwingHost = "nightwing",
[string]$WitnessUrl = "ws://192.168.1.66:8796/eve/periwinkle",
[string]$ConfigPath = "E:\Projects\Mimir\config\mimir-runtime.mvp-leap-eyes-kiyo.local.json"
)
$ErrorActionPreference = "Stop"
$repo = Split-Path -Parent $PSScriptRoot
$logDir = Join-Path $repo "artifacts\runtime\mvp-leap-eyes-kiyo"
New-Item -ItemType Directory -Force -Path $logDir | Out-Null
function Test-CommandOk {
param([string[]]$Command, [string]$Name)
$process = Start-Process -FilePath $Command[0] -ArgumentList $Command[1..($Command.Length - 1)] -NoNewWindow -Wait -PassThru -RedirectStandardOutput (Join-Path $logDir "$Name.out.log") -RedirectStandardError (Join-Path $logDir "$Name.err.log")
return $process.ExitCode -eq 0
}
$kiyo = Get-PnpDevice -PresentOnly | Where-Object { $_.InstanceId -match "VID_1532&PID_0E05&MI_00" } | Select-Object -First 1
$leap = Get-PnpDevice -PresentOnly | Where-Object { $_.InstanceId -match "VID_F182&PID_0003&MI_00" } | Select-Object -First 1
if (-not $kiyo) {
throw "Kiyo Pro camera interface is not present. MVP needs Kiyo Pro RGB."
}
if (-not $leap) {
throw "LeapUVC camera interface is not present. Replug/wake the Leap before launching this MVP profile."
}
$nightwingCheck = ssh $NightwingHost "test -e /dev/video2 -a -e /dev/video3 && command -v python3 >/dev/null && echo ok" 2>$null
if ($LASTEXITCODE -ne 0 -or $nightwingCheck -notmatch "ok") {
throw "Nightwing does not expose /dev/video2 + /dev/video3 with python3 available."
}
scp (Join-Path $repo "tools\nw_eye_cap.py") "${NightwingHost}:/tmp/nw_eye_cap.py" | Out-Null
if ($LASTEXITCODE -ne 0) {
throw "Could not stage nw_eye_cap.py on Nightwing."
}
scp (Join-Path $repo "tools\nw_move_hint.py") "${NightwingHost}:/tmp/nw_move_hint.py" | Out-Null
if ($LASTEXITCODE -ne 0) {
throw "Could not stage nw_move_hint.py on Nightwing."
}
scp (Join-Path $repo "tools\nightwing_typed_witness_publisher.py") "${NightwingHost}:/tmp/nightwing_typed_witness_publisher.py" | Out-Null
if ($LASTEXITCODE -ne 0) {
throw "Could not stage nightwing_typed_witness_publisher.py on Nightwing."
}
$receiverRunning = Get-CimInstance Win32_Process |
Where-Object { $_.CommandLine -match "Mimir.EveSensorReceiver" -and $_.CommandLine -match "--port 8796" } |
Select-Object -First 1
if (-not $receiverRunning) {
$receiverOut = Join-Path $logDir "nightwing-witness-receiver.out.log"
$receiverErr = Join-Path $logDir "nightwing-witness-receiver.err.log"
Start-Process -WindowStyle Hidden -FilePath "dotnet" -ArgumentList @(
"run",
"--no-build",
"--project",
(Join-Path $repo "src\Mimir.EveSensorReceiver\Mimir.EveSensorReceiver.csproj"),
"--",
"--port",
"8796",
"--path",
"/eve/periwinkle",
"--source-id",
"nightwing-witness",
"--type",
"cultmesh-observation"
) -RedirectStandardOutput $receiverOut -RedirectStandardError $receiverErr
Start-Sleep -Milliseconds 800
}
$remotePublisher = "nohup python3 /tmp/nightwing_typed_witness_publisher.py --url '$WitnessUrl' --track-eyes --interval 0.12 > /tmp/mimir-nightwing-mvp-witness.log 2>&1 &"
ssh $NightwingHost $remotePublisher | Out-Null
if ($LASTEXITCODE -ne 0) {
throw "Could not start Nightwing typed witness publisher."
}
$env:MIMIR_RUNTIME_CONFIG = $ConfigPath
$args = @("run", "--project", (Join-Path $repo "src\Mimir.App\Mimir.App.csproj"))
if ($Headless) {
$args += "--"
$args += "--headless"
$args += "--headless-width"
$args += "$HeadlessWidth"
$args += "--headless-height"
$args += "$HeadlessHeight"
if ($CaptureFrame) {
$args += "--capture-frame"
$args += $CaptureFrame
}
}
Write-Host "MIMIR_RUNTIME_CONFIG=$ConfigPath"
Write-Host "Starting MVP: Starfire Leap + Kiyo Pro, Nightwing Eye/Move witness claims"
dotnet @args

1
state/evidence.jsonl
View file

@ -297,3 +297,4 @@
{"ts": "2026-05-31T23:31:00+01:00", "type": "implementation", "status": "ok", "note": "sync_sweep.py now accepts a mimir.music_keyed_move_chirp_plan.v1 as the shared gesture authority for articulated chirps and chirp-flashes. The sweep copies/renders the plan WAV, sends exact Move visual contour samples over HID, attaches RGB visual_contour decoder scores to video receipts, and runs Mimir perfect-machine live receipts for local Leap/Kiyos plus Raven/Eve configs unless disabled. This fixes the earlier DirectShow/Nightwing-only sweep omission path: Leap/Eve are explicit receipt surfaces instead of assumed present."}
{"ts": "2026-05-31T23:48:00+01:00", "type": "verification", "status": "partial", "note": "Ran an articulated all-sensor sweep from a live Scarlett-derived music plan at artifacts/runtime/all-sensor-articulated-20260531-232349. ASIO loopback channels 2/3 matched 77 chirp events with 0.083/0.292 ms median residuals; Nightwing builtin cam and nw-eye-1 each emitted 100/100 Move observations; Kiyo Pro/Kiyo captured RGB video and offline crunch extracted 47/51 marker observations. tools/crunch_sync_sweep.py wrote 298 total marker observations to calibration-observations.json/csv. Missing witnesses are explicit: nw-eye-0 timed out, Nightwing mic only matched one chirp, Focusrite DShow was unavailable while ASIO succeeded, LeapUVC KS open failed, and Raven/Eve receiver runtime saw no camera textures."}
{"ts": "2026-05-31T23:54:00+01:00", "type": "verification", "status": "ok", "note": "The articulated sweep crunch now reports pairwise shared marker events for calibration solve selection. In the 20260531-232349 run: Nightwing builtin cam and nw-eye-1 share 100 events; Kiyo shares 51 with each Nightwing view; Kiyo Pro shares 47 with each Nightwing view; Kiyo and Kiyo Pro share 28. This is enough correspondence substrate for initial multi-view frustum fitting on the sensors that actually saw the Move spheres."}
{"ts": "2026-06-01T00:12:00+01:00", "type": "decision", "status": "ok", "note": "Corrected the MVP stream split after user pushback: Starfire should not ingest Nightwing PS3 Eye pixels. The MVP profile now keeps Starfire buffers to local Scarlett ASIO, local Leap stereo IR, and local Kiyo Pro RGB; Nightwing owns PS3 Eye reads and emits compact mimir.move_controller_observation_state.v1 typed witness claims through the /eve/periwinkle CultMesh observation receiver. The launcher refuses to start without LeapUVC enumerated, because Leap is the MVP geometry root."}

8
state/map.yaml
View file

@ -211,6 +211,14 @@ invariants:
rather than silently inferred. `tools/crunch_sync_sweep.py` turns a sweep
directory into `calibration-observations.json/csv` by merging Nightwing's
typed Move observations with local Kiyo frame-derived marker centroids.
The ASAP streaming MVP is narrower and authority-correct:
`config/mimir-runtime.mvp-leap-eyes-kiyo.local.json` keeps Starfire's live
rolling buffers to local Scarlett ASIO, local Leap stereo IR, and local
Kiyo Pro RGB only. Nightwing's PS3 Eyes do not cross as raw video; the
`start-mvp-leap-eyes-kiyo.ps1` launcher stages Nightwing tools and starts
`nightwing_typed_witness_publisher.py --track-eyes`, so Nightwing publishes
compact `mimir.move_controller_observation_state.v1` claims through the
`/eve/periwinkle` CultMesh observation receiver.
`MimirMoveControllerTracker` is the first Mimir-native overlay owner for
multiple PS Move controllers: it tracks bright sphere observations from
PS3 Eye luma frames, keeps a bounded history per `controllerId`, preserves

92
tools/nightwing_typed_witness_publisher.py
View file

@ -20,6 +20,13 @@ import struct
import time
import urllib.parse
try:
import nw_eye_cap
import nw_move_hint
except Exception:
nw_eye_cap = None
nw_move_hint = None
def pack_nil() -> bytes:
return b"\xc0"
@ -206,6 +213,80 @@ def observations(sequence: int) -> list[bytes]:
return docs
def eye_tracking_observations(args: argparse.Namespace, sequence: int) -> list[bytes]:
if nw_eye_cap is None or nw_move_hint is None:
return []
docs: list[bytes] = []
devices = [part for part in args.eye_devices.split(",") if part]
for eye_index, device in enumerate(devices):
source_id = f"ps3-eye-{eye_index}"
stats_path = f"/tmp/mimir-mvp-{source_id}.json"
cap_args = argparse.Namespace(
device=device,
width=args.eye_width,
height=args.eye_height,
format="YUYV",
fps=args.eye_fps,
seconds=args.eye_window_seconds,
buffers=8,
out=stats_path,
stdout=False,
)
try:
stats = nw_eye_cap.capture(cap_args)
raw_path = stats["frames_path"]
width = int(stats["width"])
height = int(stats["height"])
frame_bytes = width * height
with open(raw_path, "rb") as handle:
raw = handle.read()
frame_count = min(len(raw) // frame_bytes, len(stats.get("timestamps_s", [])))
stride = max(1, int(args.tracking_stride))
for frame_index in range(0, frame_count, stride):
frame = raw[frame_index * frame_bytes:(frame_index + 1) * frame_bytes]
blob = nw_move_hint.best_blob(frame, width, height)
if blob is None or float(blob["confidence"]) < args.min_confidence:
continue
values = [
float(blob["clip_x"]),
float(blob["clip_y"]),
float(blob["area_px"]),
float(blob["radius_px"]),
float(blob["mean_luma"]),
float(blob["peak_luma"]),
float(blob["confidence"]),
float(frame_index),
float(width),
float(height),
]
docs.append(sensor_document(
observation_id=f"nightwing:{source_id}:move-sphere:{sequence}:{frame_index}",
device_id="nightwing",
stream_id=source_id,
kind="mimir.move_controller_observation_state.v1",
sequence=sequence * 100000 + frame_index,
values=values,
accuracy=3,
))
except Exception as ex:
docs.append(sensor_document(
observation_id=f"nightwing:{source_id}:tracking-error:{sequence}",
device_id="nightwing",
stream_id=source_id,
kind=f"tracking-error:{type(ex).__name__}",
sequence=sequence,
values=[0.0],
accuracy=0,
))
finally:
try:
if "raw_path" in locals():
os.unlink(raw_path)
except OSError:
pass
return docs
def run(args: argparse.Namespace) -> None:
sequence = 0
while True:
@ -215,6 +296,9 @@ def run(args: argparse.Namespace) -> None:
while True:
for payload in observations(sequence):
ws.send_binary(payload)
if args.track_eyes:
for payload in eye_tracking_observations(args, sequence):
ws.send_binary(payload)
sequence += 1
if args.once:
return
@ -232,6 +316,14 @@ def main() -> int:
parser.add_argument("--interval", type=float, default=0.25)
parser.add_argument("--reconnect-seconds", type=float, default=2.0)
parser.add_argument("--once", action="store_true")
parser.add_argument("--track-eyes", action="store_true", help="Publish compact Nightwing-local PS3 Eye Move-sphere observations.")
parser.add_argument("--eye-devices", default="/dev/video2,/dev/video3")
parser.add_argument("--eye-width", type=int, default=320)
parser.add_argument("--eye-height", type=int, default=240)
parser.add_argument("--eye-fps", type=int, default=187)
parser.add_argument("--eye-window-seconds", type=float, default=0.12)
parser.add_argument("--tracking-stride", type=int, default=6)
parser.add_argument("--min-confidence", type=float, default=0.25)
args = parser.parse_args()
run(args)
return 0

29
tools/nw_eye_cap.py
View file

@ -11,6 +11,7 @@ import json
import mmap
import os
import select
import sys
import time
from pathlib import Path
@ -166,9 +167,10 @@ def yuyv_to_luma(raw: bytes, width: int, height: int, stride: int) -> bytes:
def capture(args: argparse.Namespace) -> dict:
fd = os.open(args.device, os.O_RDWR | os.O_NONBLOCK)
maps: list[mmap.mmap] = []
out = Path(args.out)
out.parent.mkdir(parents=True, exist_ok=True)
frames_path = out.with_suffix(".y8")
out = Path(args.out) if args.out else None
if out is not None:
out.parent.mkdir(parents=True, exist_ok=True)
frames_path = out.with_suffix(".y8") if out is not None else None
try:
set_frame_interval(fd, args.fps)
fmt = V4L2Format()
@ -201,7 +203,8 @@ def capture(args: argparse.Namespace) -> dict:
deadline = start + args.seconds
timestamps = []
sequences = []
with frames_path.open("wb") as frames:
frames = sys.stdout.buffer if args.stdout else frames_path.open("wb") # type: ignore[union-attr]
try:
while time.monotonic() < deadline:
ready, _, _ = select.select([fd], [], [], 0.25)
if not ready:
@ -218,6 +221,9 @@ def capture(args: argparse.Namespace) -> dict:
timestamps.append(float(buf.timestamp.tv_sec) + float(buf.timestamp.tv_usec) / 1_000_000.0)
sequences.append(int(buf.sequence))
xioctl(fd, VIDIOC_QBUF, buf)
finally:
if not args.stdout:
frames.close()
try:
fcntl.ioctl(fd, VIDIOC_STREAMOFF, stream_type)
except OSError:
@ -227,7 +233,7 @@ def capture(args: argparse.Namespace) -> dict:
gap = max(0, (sequences[-1] - sequences[0] + 1) - len(sequences)) if len(sequences) >= 2 else 0
stats = {
"device": args.device,
"frames_path": str(frames_path),
"frames_path": str(frames_path) if frames_path is not None else "stdout",
"width": int(actual.width),
"height": int(actual.height),
"stride": int(actual.bytesperline),
@ -238,7 +244,10 @@ def capture(args: argparse.Namespace) -> dict:
"sequence_gap": gap,
"duration_s": (rel[-1] - rel[0]) if len(rel) >= 2 else 0.0,
}
out.write_text(json.dumps(stats, indent=2, sort_keys=True), encoding="utf-8")
if out is not None:
out.write_text(json.dumps(stats, indent=2, sort_keys=True), encoding="utf-8")
else:
print(json.dumps(stats, sort_keys=True), file=sys.stderr)
return stats
finally:
for mapping in maps:
@ -255,10 +264,14 @@ def main() -> int:
parser.add_argument("--fps", type=int, default=187)
parser.add_argument("--seconds", type=float, default=8.0)
parser.add_argument("--buffers", type=int, default=8)
parser.add_argument("--out", required=True)
parser.add_argument("--out", default=None)
parser.add_argument("--stdout", action="store_true", help="Stream raw luma frames to stdout and write stats to stderr.")
args = parser.parse_args()
if not args.stdout and not args.out:
parser.error("--out is required unless --stdout is set")
stats = capture(args)
print(json.dumps(stats, sort_keys=True))
if not args.stdout:
print(json.dumps(stats, sort_keys=True))
return 0 if stats["frames"] else 1