Prioritize Leap evidence in LOD cache

This commit is contained in:
Meta Vens 2026-05-18 22:55:13 +01:00
commit a4b4473499
7 changed files with 183 additions and 24 deletions

View file

@ -28,6 +28,7 @@
- `localcast.sensor_fusion.cultcache_docs` stores live visual state as typed CultCache MessagePack documents.
- `localcast.sensor_fusion.spout_output` renders render-frame packets into a GPU texture and publishes it as a named Spout sender for OBS.
- `scripts/live_sensor_fusion.py` writes fused render frames into `calibration/runs/visual-state.msgpack`.
- The live visual producer writes a multi-LOD scene cache with source kind and priority. Real Leap frames are promoted as the highest-priority visual timing/spatial evidence; Leap fallback frames remain lower-priority diagnostics.
- `scripts/stream_spout.py` runs the deadline Spout sender loop from the typed cache with typed and JSON heartbeat status.
- `docs/obs-spout-streaming.md` documents OBS setup and the Aquarium replacement boundary.
- `docs/typed-visual-state.md` documents the CultCache/CultNet visual boundary.
@ -70,6 +71,7 @@
11. Move the render-frame consumer into Aquarium Engine so dense brush/splat rendering replaces the deadline OpenGL point sink behind the same OBS Spout boundary.
12. Capture fixed-board ChArUco observations for every camera, solve `config/sensor-fusion.json`, and verify cross-view feature triangulation with `scripts/triangulate_surface_features.py`.
13. Replace synthetic live-fusion observations with PS3 Eye detector observations and calibrated surface-feature tracks.
14. Keep the neighbor loopback leg running through the scheduled interactive WASAPI capture path and use it as the timing witness once actual neighbor app audio is present.
15. Let the co-streamer surface delay drive the shared presentation buffer horizon for audio stems, AmbiX, and remote video.
16. Wire `stream_phase_field.py` and `stream_faust_mic_field.py` to the real live aligned-field producer instead of replaying WAV artifacts, then feed emitted active chirps back through loopback/mic capture so confidence maintenance closes against fresh observations instead of dry-run probe manifests.
14. Fix the live Leap capture backend so actual Leap frames, not fallback frames, populate `leap-ground-truth` cells in the multi-LOD cache.
15. Keep the neighbor loopback leg running through the scheduled interactive WASAPI capture path and use it as the timing witness once actual neighbor app audio is present.
16. Let the co-streamer surface delay drive the shared presentation buffer horizon for audio stems, AmbiX, and remote video.
17. Wire `stream_phase_field.py` and `stream_faust_mic_field.py` to the real live aligned-field producer instead of replaying WAV artifacts, then feed emitted active chirps back through loopback/mic capture so confidence maintenance closes against fresh observations instead of dry-run probe manifests.

View file

@ -40,11 +40,15 @@ from .surface_features import (
from .stochastic_mapping import (
ImageRayBiasMap,
LODCell,
LODEvidencePoint,
MultiLODSceneCache,
RayBiasSample,
SceneRay,
TransientMatchHypothesis,
evidence_from_fusion_items,
evidence_from_render_points,
multilod_cache_from_points,
multilod_cache_from_evidence,
pixel_scene_ray,
stochastic_transient_matches,
)
@ -103,6 +107,7 @@ __all__ = [
"SurfaceFeatureTrack",
"ImageRayBiasMap",
"LODCell",
"LODEvidencePoint",
"MultiLODSceneCache",
"RayBiasSample",
"SceneRay",
@ -134,7 +139,10 @@ __all__ = [
"match_surface_features",
"measure_frame_quality",
"orb_surface_observations",
"evidence_from_fusion_items",
"evidence_from_render_points",
"multilod_cache_from_points",
"multilod_cache_from_evidence",
"overlay_audio_events",
"pixel_scene_ray",
"stochastic_transient_matches",

View file

@ -95,6 +95,8 @@ class LODCell:
count: int
source_time_min_ns: int
source_time_max_ns: int
source_priority: float = 1.0
source_kind: str = "generic"
@dataclass(frozen=True)
@ -119,6 +121,8 @@ class MultiLODSceneCache:
"count": cell.count,
"sourceTimeMinNs": cell.source_time_min_ns,
"sourceTimeMaxNs": cell.source_time_max_ns,
"sourcePriority": cell.source_priority,
"sourceKind": cell.source_kind,
}
for cell in self.cells
],
@ -304,6 +308,94 @@ def multilod_cache_from_points(
)
@dataclass(frozen=True)
class LODEvidencePoint:
stable_key: str
xyz: np.ndarray
confidence: float
timestamp_ns: int
source_priority: float
source_kind: str
def multilod_cache_from_evidence(
evidence: Iterable[LODEvidencePoint],
*,
levels: tuple[float, ...] = (0.02, 0.08, 0.32),
created_monotonic_ns: int = 0,
) -> MultiLODSceneCache:
items = list(evidence)
cells: list[LODCell] = []
for level_index, cell_size in enumerate(levels):
buckets: dict[tuple[int, int, int], list[LODEvidencePoint]] = {}
for item in items:
key = tuple(int(np.floor(float(value) / float(cell_size))) for value in np.asarray(item.xyz, dtype=np.float64))
buckets.setdefault(key, []).append(item)
for key, bucket in buckets.items():
weights = np.asarray([max(1.0e-6, item.confidence * item.source_priority) for item in bucket], dtype=np.float64)
positions = np.asarray([np.asarray(item.xyz, dtype=np.float64) for item in bucket], dtype=np.float64)
center = np.average(positions, axis=0, weights=weights)
priority = float(max(item.source_priority for item in bucket))
source_kind = max(bucket, key=lambda item: item.source_priority).source_kind
cells.append(
LODCell(
level=level_index,
cell=key,
center=tuple(float(value) for value in center),
radius_m=float(cell_size) * 0.5,
confidence=float(min(1.0, np.mean(weights))),
count=len(bucket),
source_time_min_ns=min(item.timestamp_ns for item in bucket),
source_time_max_ns=max(item.timestamp_ns for item in bucket),
source_priority=priority,
source_kind=source_kind,
)
)
return MultiLODSceneCache(
schema="localcast.sensor_fusion.multilod_scene_cache.v1",
created_monotonic_ns=int(created_monotonic_ns),
levels=tuple(float(level) for level in levels),
cells=tuple(cells),
)
def evidence_from_render_points(points: Iterable, *, source_kind: str, source_priority: float) -> tuple[LODEvidencePoint, ...]:
rows: list[LODEvidencePoint] = []
for point in points:
rows.append(
LODEvidencePoint(
stable_key=str(point.stable_key),
xyz=np.asarray(point.xyz, dtype=np.float64),
confidence=float(point.confidence),
timestamp_ns=int(point.source_timestamp_ns),
source_priority=float(source_priority),
source_kind=str(source_kind),
)
)
return tuple(rows)
def evidence_from_fusion_items(
items: Iterable[TriangulatedPoint | TransientMatchHypothesis],
*,
source_kind: str,
source_priority: float,
) -> tuple[LODEvidencePoint, ...]:
rows: list[LODEvidencePoint] = []
for item in items:
rows.append(
LODEvidencePoint(
stable_key=str(getattr(item, "marker_id", getattr(item, "stable_key", "scene-claim"))),
xyz=np.asarray(item.xyz, dtype=np.float64),
confidence=float(item.confidence),
timestamp_ns=item_timestamp_ns(item),
source_priority=float(source_priority),
source_kind=str(source_kind),
)
)
return tuple(rows)
def item_timestamp_ns(item: TriangulatedPoint | TransientMatchHypothesis) -> int:
if hasattr(item, "timestamp_ns"):
return int(getattr(item, "timestamp_ns"))

View file

@ -91,16 +91,20 @@ flowchart TD
E --> F["ScreenBrushPacket lowering"]
F --> G["Spout sender"]
G --> H["OBS Spout2 Capture"]
I["Leap motion evidence"] --> J["multi-LOD scene cache"]
K["RGB/stochastic/synthetic evidence"] --> J
J --> L["Aquarium compute reconciliation"]
```
Ownership:
- `localcast.sensor_fusion.cultcache_docs` owns typed visual state documents.
- `scripts/live_sensor_fusion.py` currently owns the live producer and writes `localcast.visual.render_frame` into CultCache.
- The live producer also writes a multi-LOD scene cache with source kind and priority. Real Leap frames are the highest-priority timing/spatial evidence; Leap fallback frames are tagged as fallback and do not become ground truth.
- `scripts/stream_spout.py` consumes typed CultCache state and publishes Spout.
- CultNet document replication is the intended API boundary for other producers/consumers.
Invariant: JSON is not the visual-state authority. The current renderer can still write a JSON heartbeat for human inspection, but live visual state lives in typed CultCache MessagePack documents.
Invariant: JSON is not the visual-state authority. The current renderer can still write a JSON heartbeat for human inspection, but live visual state lives in typed CultCache MessagePack documents. Leap outranks camera surfaces only when the frame source is actual Leap capture; diagnostics must stay labeled as diagnostics.
## OBS Program Surface

View file

@ -21,8 +21,10 @@ from localcast.sensor_fusion import (
SensorRig,
SurfaceFeatureObservation,
dense_stereo_points,
evidence_from_fusion_items,
evidence_from_render_points,
measure_frame_quality,
multilod_cache_from_points,
multilod_cache_from_evidence,
put_live_render_frame,
lower_points_to_render_frame,
stochastic_transient_matches,
@ -599,11 +601,17 @@ class LeapPackedMotionSampler:
self._capture = None
self._previous_channels: dict[str, np.ndarray] = {}
self._last_timestamp_ns: int | None = None
self._last_frame_kind: str | None = None
self._fallback_index = 0
@property
def last_timestamp_ns(self) -> int | None:
return self._last_timestamp_ns
@property
def last_frame_kind(self) -> str | None:
return self._last_frame_kind
def close(self) -> None:
if self._capture is not None:
try:
@ -646,23 +654,36 @@ class LeapPackedMotionSampler:
return self._fallback_frame()
ok, frame = self._capture.read()
if ok and frame is not None:
self._last_frame_kind = "capture"
return frame
return self._fallback_frame()
def _fallback_frame(self) -> np.ndarray | None:
import cv2
for path in (
self.fallback_dir / f"rgb-probe-{self.api}-{self.index}.png",
self.fallback_dir / f"rgb-probe-dshow-{self.index}.png",
self.fallback_dir / f"rgb-probe-msmf-{self.index}.png",
self.fallback_dir / "leap-probe.png",
):
if path.exists():
frame = cv2.imread(str(path))
if frame is not None:
return frame
return None
path = self.fallback_dir / "leap-probe.png"
if path.exists():
frame = cv2.imread(str(path))
if frame is not None:
self._last_frame_kind = "fallback-file"
return frame
return self._synthetic_motion_frame()
def _synthetic_motion_frame(self) -> np.ndarray:
self._last_frame_kind = "fallback-synthetic"
frame = np.zeros((self.height, self.width, 3), dtype=np.uint8)
t = self._fallback_index
self._fallback_index += 1
cx = int((self.width * 0.30) + (self.width * 0.40) * ((np.sin(t * 0.37) + 1.0) * 0.5))
cy = int((self.height * 0.30) + (self.height * 0.34) * ((np.cos(t * 0.41) + 1.0) * 0.5))
y0 = max(0, cy - 4)
y1 = min(self.height, cy + 5)
x0 = max(0, cx - 4)
x1 = min(self.width, cx + 5)
frame[y0:y1, x0:x1, 1] = 220
frame[max(0, y0 - 8) : min(self.height, y1 + 8), max(0, x0 - 8) : min(self.width, x1 + 8), 2] = 80
frame[max(0, y0 - 12) : min(self.height, y1 + 12), max(0, x0 - 12) : min(self.width, x1 + 12), 0] = 55
return frame
def unpack_leap_packed_channels(frame_bgr: np.ndarray) -> dict[str, np.ndarray]:
@ -804,11 +825,6 @@ def main() -> None:
samples_per_observation=2,
seed=frame_id,
)
multilod_cache_from_points(
tuple(result.points) + tuple(transient_matches),
levels=(0.02, 0.08, 0.32),
created_monotonic_ns=time.monotonic_ns(),
).write_json(lod_cache_path)
frame = lower_points_to_render_frame(
result.points,
render_config,
@ -820,8 +836,22 @@ def main() -> None:
leap_points = () if leap_sampler is None else leap_sampler.motion_splats(timestamp_ns)
if leap_sampler is not None and leap_sampler.last_timestamp_ns is not None:
timestamp_ns = max(timestamp_ns, leap_sampler.last_timestamp_ns)
leap_source_kind = (
"leap-ground-truth"
if leap_sampler is not None and leap_sampler.last_frame_kind == "capture"
else "leap-fallback"
)
leap_source_priority = 3.0 if leap_source_kind == "leap-ground-truth" else 1.1
support_points = dim_fusion_points(tuple(frame.points)) if not rgb_points else ()
stochastic_points = transient_match_render_points(transient_matches, timestamp_ns)
multilod_cache_from_evidence(
evidence_from_render_points(leap_points, source_kind=leap_source_kind, source_priority=leap_source_priority)
+ evidence_from_render_points(rgb_points, source_kind="rgb-surface", source_priority=1.4)
+ evidence_from_fusion_items(transient_matches, source_kind="stochastic-transient", source_priority=1.2)
+ evidence_from_fusion_items(result.points, source_kind="synthetic-fusion", source_priority=0.8),
levels=(0.01, 0.04, 0.16, 0.64),
created_monotonic_ns=time.monotonic_ns(),
).write_json(lod_cache_path)
frame = RenderFramePacket(
schema=frame.schema,
frame_id=frame.frame_id,

View file

@ -158,7 +158,10 @@ research:
cache rule: semantic claims first, cached backend lowerings second.
Stochastic transient matching now lowers descriptor-consistent image
observations into scene-ray hypotheses and a multi-LOD world-space cache
for Aquarium compute-shader reconciliation.
for Aquarium compute-shader reconciliation. Leap evidence is the
highest-priority temporal/spatial source when live Leap frames are present;
fallback Leap frames stay tagged as fallback evidence and must not be
treated as ground truth.
typed_visual_state:
summary: docs/typed-visual-state.md
key_decision: >
@ -267,7 +270,10 @@ current_status:
observations can be descriptor-matched and triangulated into world-space
tracks. Unit tests inject
synthetic cameras and frames so the core does not depend on live camera
drivers.
drivers. The multi-LOD visual cache now carries source kind and source
priority so Aquarium can prefer real Leap evidence over RGB surfaces,
stochastic transient hypotheses, and synthetic fusion support while still
seeing lower-priority fallback diagnostics when the Leap capture path stalls.
next_action: >
Capture real ChArUco intrinsics for the PS3 Eyes and Kiyos, capture
fixed-board observations from every camera, solve a local
@ -280,7 +286,9 @@ current_status:
Then replace the synthetic live producer with a detector adapter that
turns FFmpeg PS3 Eye frames into marker `Observation2D` records for the
fusion core and publishes the same typed CultCache/CultNet render-frame
document. In parallel, move the render-frame consumer into Aquarium Engine so
document. Also fix the live Leap capture backend so the cache emits
`leap-ground-truth` rather than `leap-fallback`; current MSMF probing can
still stall on frame reads. In parallel, move the render-frame consumer into Aquarium Engine so
Aquarium renders cached points/brushes into its GPU render target and
publishes or hands off the same Spout sender `LocalCastBridge Point Cloud`
for OBS, replacing the deadline OpenGL point sink without changing packet

View file

@ -7,7 +7,9 @@ import numpy as np
from localcast.sensor_fusion.core import CameraModel, SensorRig, Observation2D
from localcast.sensor_fusion.stochastic_mapping import (
ImageRayBiasMap,
LODEvidencePoint,
RayBiasSample,
multilod_cache_from_evidence,
multilod_cache_from_points,
pixel_scene_ray,
stochastic_transient_matches,
@ -95,6 +97,19 @@ class StochasticMappingTests(unittest.TestCase):
cache.write_json(path)
self.assertIn("multilod_scene_cache", path.read_text(encoding="utf-8"))
def test_multilod_evidence_prefers_leap_priority(self):
evidence = [
LODEvidencePoint("rgb", np.array([0.01, 0.0, 1.0]), 0.7, 100, 1.0, "rgb"),
LODEvidencePoint("leap", np.array([0.03, 0.0, 1.0]), 0.8, 120, 3.0, "leap-ground-truth"),
]
cache = multilod_cache_from_evidence(evidence, levels=(0.10,), created_monotonic_ns=5)
cell = cache.cells[0]
self.assertEqual("leap-ground-truth", cell.source_kind)
self.assertEqual(3.0, cell.source_priority)
self.assertGreater(cell.center[0], 0.02)
if __name__ == "__main__":
unittest.main()