diff --git a/docs/move-controller-stream.md b/docs/move-controller-stream.md index 1dd7920..c6a7478 100644 --- a/docs/move-controller-stream.md +++ b/docs/move-controller-stream.md @@ -148,7 +148,10 @@ best-fit fundamentals, then writes one shared plan: - beat-aligned chirps quantized to the estimated minor-pentatonic scale; - de Bruijn-coded event symbols so adjacent timing/frequency/color words remain distinguishable; -- per-Move RGB pulses using stable controller identities and unique colors; +- per-Move visual words using stable controller identities and unique colors; + the default is no longer a square flash, but a sampled RGB contour with + attack/release envelope, symbol-dependent hue glide, tremolo articulation, + and an emphasized controller lane; - a JSON receipt shaped like AquaSynth song analysis vocabulary: `tempo_bpm`, `beat_seconds`, `tempo_confidence`, `root_note`, `suggested_scale`, `scale_frequencies_hz`, and @@ -185,6 +188,15 @@ key confidence `0.703` after widening the search ceiling with `--tempo-max-bpm 260`. The receipt records `tempo_family` so later decoders can see the selected grid alongside half-time/double-time alternatives. +Visual calibration words are deliberately shaped like the audio words: one +gesture carries more than one bit. `--visual-gesture contour` records a +`mimir.move_visual_contour_word.v1` per Move per event, with `sample_rate_hz`, +duration, base identity color, symbol emphasis, and the exact RGB samples sent +over HID. Start conservatively around `--visual-gesture-hz 80`; the HID writer +accepted a short 80 Hz contour test on Nightwing, but the useful rate is the +rate cameras can recover cleanly, not merely the rate the controller accepts. +`--visual-gesture square` remains available as a baseline comparator. + The non-dry run targets the two wireless Nightwing Moves by default: `00:07:04:a6:be:5f` on `/dev/hidraw2` and `00:06:f5:23:e2:d1` on `/dev/hidraw3`. Nightwing now carries `/etc/udev/rules.d/70-move.rules`, which diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 28d48c9..3dc6ced 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -293,3 +293,4 @@ {"ts":"2026-05-31T23:09:31+01:00","type":"implementation","status":"ok","note":"Audio Mixer is now authored as a horizontal CultUI channel-strip bank beneath the compositor preview instead of a selected-stem inspector form. Each strip reads a live presentation audio feed slot, owns M/S/gain controls for that feed, and displays compact sync/chirp response status from MimirSynchronizationHub while the right rail remains the selected video-layer Inspector. The compositor preview is weighted ahead of the mixer dock. This is UI/control-surface wiring; DSP neutralization and sample-perfect alignment remain runtime/audio authority, not inspector-owned truth.","files":["src/Mimir.Runtime/MimirRuntime.cs"],"source":"dotnet build src/Mimir.App/Mimir.App.csproj --no-restore; dotnet run --no-build --project src/Mimir.BufferSmoke/Mimir.BufferSmoke.csproj -- --presentation-control-smoke; dotnet run --no-build --project src/Mimir.BufferSmoke/Mimir.BufferSmoke.csproj -- --scene-editor-smoke; dotnet run --no-build --project src/Mimir.App/Mimir.App.csproj -- --headless --headless-width 1280 --headless-height 720 --capture-frame artifacts/runtime/mimir-editor-horizontal-audio-dock-current-headless.png"} {"ts": "2026-05-31T22:52:47+01:00", "type": "implementation", "status": "ok", "note": "Extended MimirCameraFrustumCalibrationSolver beyond LED strips: it now also consumes MimirFeatureTrackFieldCandidate plus scene-space TrackId anchors, so PS Move sphere histories, sparse Eye tracks, and Eve-local marker tracks enter the same Starfire-owned frustum authority. BufferSmoke --frustum-calibration-smoke now proves five LED-spline frustums and five feature-track frustums for ps3-eye-0, ps3-eye-1, kiyo-pro-rgb, leap-stereo-ir, and eve-camera with mean clip error 0.000000."} {"ts": "2026-05-31T22:55:38+01:00", "type": "implementation", "status": "ok", "note": "Added mimir.local_frustum_hint_state.v1 as the CultMesh document for local witness frustum solves. It carries producer node, calibration id, candidate key, marker set id, per-camera pose/frustum snapshots, residuals, point counts, and confidence, so Eve/Nightwing can publish compact solved hints while Starfire remains canonical global placement authority."} +{"ts": "2026-05-31T23:14:03+01:00", "type": "implementation", "status": "ok", "note": "Changed music_keyed_move_chirp_sync.py visual pulses from square on/off flashes into sampled Move visual words. The default mimir.move_visual_contour_word.v1 uses identity color bias, attack/release envelope, symbol-dependent hue glide, tremolo articulation, emphasized Move lane, and an explicit visual_gesture_hz rate knob. Synthetic dry smoke wrote contour samples; a short non-dry Nightwing HID test at 80 Hz completed without writer failure. The camera-decodable rate still needs offline recovery scoring from captured frames."} diff --git a/tools/music_keyed_move_chirp_sync.py b/tools/music_keyed_move_chirp_sync.py index 5d7049b..7d6781c 100644 --- a/tools/music_keyed_move_chirp_sync.py +++ b/tools/music_keyed_move_chirp_sync.py @@ -26,6 +26,7 @@ import base64 import json import math import os +import colorsys import subprocess import sys import time @@ -117,6 +118,8 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--lead-seconds", type=float, default=0.55) parser.add_argument("--chirp-ms", type=float, default=55.0) parser.add_argument("--pulse-ms", type=float, default=90.0) + parser.add_argument("--visual-gesture", choices=("square", "contour"), default="contour") + parser.add_argument("--visual-gesture-hz", type=float, default=80.0) parser.add_argument("--sample-rate", type=int, default=48000) parser.add_argument("--fps", type=float, default=60.0) parser.add_argument("--fft-size", type=int, default=512) @@ -310,6 +313,52 @@ def midi_to_hz(midi: int) -> float: return 440.0 * (2.0 ** ((midi - 69) / 12.0)) +def rgb_to_hsv(rgb: tuple[int, int, int]) -> tuple[float, float, float]: + return colorsys.rgb_to_hsv(*(channel / 255.0 for channel in rgb)) + + +def hsv_to_rgb255(hue: float, saturation: float, value: float) -> tuple[int, int, int]: + r, g, b = colorsys.hsv_to_rgb(hue % 1.0, max(0.0, min(1.0, saturation)), max(0.0, min(1.0, value))) + return tuple(int(round(channel * 255.0)) for channel in (r, g, b)) # type: ignore[return-value] + + +def visual_contour_samples( + symbol: int, + move: MoveTarget, + move_index: int, + move_count: int, + pulse_seconds: float, + gesture_hz: float, + emphasized: bool, +) -> list[dict]: + if pulse_seconds <= 0.0: + return [] + step = 1.0 / max(1.0, gesture_hz) + sample_count = max(3, int(math.ceil(pulse_seconds / step)) + 1) + base_hue, base_sat, base_value = rgb_to_hsv(move.color) + symbol_phase = ((symbol * 0.071428571) + move_index / max(1, move_count)) % 1.0 + hue_span = 0.07 + 0.015 * (symbol % 5) + tremolo_cycles = 1 + (symbol % 3) + tremolo_depth = 0.16 + 0.05 * ((symbol // 3) % 3) + identity_floor = 0.22 if emphasized else 0.13 + peak = 1.0 if emphasized else 0.58 + samples: list[dict] = [] + for sample_index in range(sample_count): + offset = min(pulse_seconds, sample_index * step) + t = offset / pulse_seconds + attack = min(1.0, t / 0.22) + release = min(1.0, (1.0 - t) / 0.34) + envelope = max(0.0, min(1.0, attack, release)) + articulation = 1.0 - tremolo_depth * (0.5 - 0.5 * math.cos(2.0 * math.pi * tremolo_cycles * t + symbol_phase * math.tau)) + value = max(identity_floor, peak * envelope * articulation) + hue = base_hue + hue_span * math.sin(math.pi * (t - 0.5)) + symbol_phase * 0.035 + saturation = min(1.0, max(0.55, base_sat + 0.10 * math.sin(math.tau * t + symbol_phase * math.tau))) + rgb = hsv_to_rgb255(hue, saturation, value * base_value) + samples.append({"offset_seconds": offset, "rgb": rgb, "envelope": envelope, "value": value, "hue": hue % 1.0}) + samples.append({"offset_seconds": pulse_seconds, "rgb": (0, 0, 0), "envelope": 0.0, "value": 0.0, "hue": base_hue}) + return samples + + def render_chirp(start_hz: float, end_hz: float, sample_rate: int, chirp_ms: float) -> np.ndarray: length = max(16, int(sample_rate * chirp_ms * 0.001)) t = np.arange(length, dtype=np.float32) / sample_rate @@ -352,6 +401,7 @@ def build_plan(args: argparse.Namespace, moves: list[MoveTarget], bpm: float, be symbols = rotate_to_distinct_opening(build_debruijn(max(2, len(MINOR_PENTATONIC)), 3), 3) scale = [midi_to_hz(root_midi + interval) for interval in MINOR_PENTATONIC] events = [] + pulse_seconds = args.pulse_ms * 0.001 for index in range(args.events): symbol = symbols[index % len(symbols)] note_hz = scale[symbol % len(scale)] @@ -359,9 +409,30 @@ def build_plan(args: argparse.Namespace, moves: list[MoveTarget], bpm: float, be offset = args.lead_seconds + index * beat_seconds move_events = [] for move_index, move in enumerate(moves): - intensity = 1.0 if move_index == symbol % max(1, len(moves)) else 0.55 + emphasized = move_index == symbol % max(1, len(moves)) + intensity = 1.0 if emphasized else 0.55 color = tuple(int(channel * intensity) for channel in move.color) - move_events.append({"name": move.name, "hidraw": move.hidraw, "rgb": color}) + contour = ( + visual_contour_samples(symbol, move, move_index, len(moves), pulse_seconds, args.visual_gesture_hz, emphasized) + if args.visual_gesture == "contour" + else [ + {"offset_seconds": 0.0, "rgb": color, "envelope": 1.0, "value": intensity, "hue": rgb_to_hsv(color)[0]}, + {"offset_seconds": pulse_seconds, "rgb": (0, 0, 0), "envelope": 0.0, "value": 0.0, "hue": rgb_to_hsv(color)[0]}, + ] + ) + move_events.append({ + "name": move.name, + "hidraw": move.hidraw, + "base_rgb": color, + "visual_word": { + "kind": f"mimir.move_visual_{args.visual_gesture}_word.v1", + "sample_rate_hz": args.visual_gesture_hz, + "duration_seconds": pulse_seconds, + "symbol": symbol, + "emphasized": emphasized, + "samples": contour, + }, + }) events.append( { "index": index, @@ -385,6 +456,12 @@ def build_plan(args: argparse.Namespace, moves: list[MoveTarget], bpm: float, be "key_confidence": key_confidence, "suggested_scale": "minor-pentatonic", "scale_frequencies_hz": scale, + "visual_gesture": { + "mode": args.visual_gesture, + "sample_rate_hz": args.visual_gesture_hz, + "pulse_seconds": pulse_seconds, + "encoding": "identity color bias plus symbol-dependent hue glide, envelope, and tremolo articulation", + }, "moves": [move.__dict__ for move in moves], "events": events, } @@ -412,12 +489,15 @@ def emit(plan: dict, args: argparse.Namespace, moves: list[MoveTarget], wav_path ) ssh = subprocess.Popen(["ssh", "-o", "BatchMode=yes", args.ssh_target, remote_cmd], stdin=subprocess.PIPE, text=True) assert ssh.stdin is not None - pulse_seconds = args.pulse_ms * 0.001 for event in plan["events"]: for pulse in event["move_pulses"]: - r, g, b = pulse["rgb"] - ssh.stdin.write(f"{event['offset_seconds']:.6f} {pulse['name']} {r} {g} {b}\n") - ssh.stdin.write(f"{event['offset_seconds'] + pulse_seconds:.6f} {pulse['name']} 0 0 0\n") + samples = pulse.get("visual_word", {}).get("samples") or [ + {"offset_seconds": 0.0, "rgb": pulse.get("base_rgb", (0, 0, 0))}, + {"offset_seconds": args.pulse_ms * 0.001, "rgb": (0, 0, 0)}, + ] + for sample in samples: + r, g, b = sample["rgb"] + ssh.stdin.write(f"{event['offset_seconds'] + sample['offset_seconds']:.6f} {pulse['name']} {int(r)} {int(g)} {int(b)}\n") ssh.stdin.write("go\n") ssh.stdin.flush() audio = subprocess.Popen([args.ffplay, "-nodisp", "-autoexit", "-loglevel", "quiet", str(wav_path)])