Prioritize Leap evidence in LOD cache
This commit is contained in:
parent
16481db6b8
commit
a4b4473499
7 changed files with 183 additions and 24 deletions
|
|
@ -28,6 +28,7 @@
|
|||
- `localcast.sensor_fusion.cultcache_docs` stores live visual state as typed CultCache MessagePack documents.
|
||||
- `localcast.sensor_fusion.spout_output` renders render-frame packets into a GPU texture and publishes it as a named Spout sender for OBS.
|
||||
- `scripts/live_sensor_fusion.py` writes fused render frames into `calibration/runs/visual-state.msgpack`.
|
||||
- The live visual producer writes a multi-LOD scene cache with source kind and priority. Real Leap frames are promoted as the highest-priority visual timing/spatial evidence; Leap fallback frames remain lower-priority diagnostics.
|
||||
- `scripts/stream_spout.py` runs the deadline Spout sender loop from the typed cache with typed and JSON heartbeat status.
|
||||
- `docs/obs-spout-streaming.md` documents OBS setup and the Aquarium replacement boundary.
|
||||
- `docs/typed-visual-state.md` documents the CultCache/CultNet visual boundary.
|
||||
|
|
@ -70,6 +71,7 @@
|
|||
11. Move the render-frame consumer into Aquarium Engine so dense brush/splat rendering replaces the deadline OpenGL point sink behind the same OBS Spout boundary.
|
||||
12. Capture fixed-board ChArUco observations for every camera, solve `config/sensor-fusion.json`, and verify cross-view feature triangulation with `scripts/triangulate_surface_features.py`.
|
||||
13. Replace synthetic live-fusion observations with PS3 Eye detector observations and calibrated surface-feature tracks.
|
||||
14. Keep the neighbor loopback leg running through the scheduled interactive WASAPI capture path and use it as the timing witness once actual neighbor app audio is present.
|
||||
15. Let the co-streamer surface delay drive the shared presentation buffer horizon for audio stems, AmbiX, and remote video.
|
||||
16. Wire `stream_phase_field.py` and `stream_faust_mic_field.py` to the real live aligned-field producer instead of replaying WAV artifacts, then feed emitted active chirps back through loopback/mic capture so confidence maintenance closes against fresh observations instead of dry-run probe manifests.
|
||||
14. Fix the live Leap capture backend so actual Leap frames, not fallback frames, populate `leap-ground-truth` cells in the multi-LOD cache.
|
||||
15. Keep the neighbor loopback leg running through the scheduled interactive WASAPI capture path and use it as the timing witness once actual neighbor app audio is present.
|
||||
16. Let the co-streamer surface delay drive the shared presentation buffer horizon for audio stems, AmbiX, and remote video.
|
||||
17. Wire `stream_phase_field.py` and `stream_faust_mic_field.py` to the real live aligned-field producer instead of replaying WAV artifacts, then feed emitted active chirps back through loopback/mic capture so confidence maintenance closes against fresh observations instead of dry-run probe manifests.
|
||||
|
|
|
|||
|
|
@ -40,11 +40,15 @@ from .surface_features import (
|
|||
from .stochastic_mapping import (
|
||||
ImageRayBiasMap,
|
||||
LODCell,
|
||||
LODEvidencePoint,
|
||||
MultiLODSceneCache,
|
||||
RayBiasSample,
|
||||
SceneRay,
|
||||
TransientMatchHypothesis,
|
||||
evidence_from_fusion_items,
|
||||
evidence_from_render_points,
|
||||
multilod_cache_from_points,
|
||||
multilod_cache_from_evidence,
|
||||
pixel_scene_ray,
|
||||
stochastic_transient_matches,
|
||||
)
|
||||
|
|
@ -103,6 +107,7 @@ __all__ = [
|
|||
"SurfaceFeatureTrack",
|
||||
"ImageRayBiasMap",
|
||||
"LODCell",
|
||||
"LODEvidencePoint",
|
||||
"MultiLODSceneCache",
|
||||
"RayBiasSample",
|
||||
"SceneRay",
|
||||
|
|
@ -134,7 +139,10 @@ __all__ = [
|
|||
"match_surface_features",
|
||||
"measure_frame_quality",
|
||||
"orb_surface_observations",
|
||||
"evidence_from_fusion_items",
|
||||
"evidence_from_render_points",
|
||||
"multilod_cache_from_points",
|
||||
"multilod_cache_from_evidence",
|
||||
"overlay_audio_events",
|
||||
"pixel_scene_ray",
|
||||
"stochastic_transient_matches",
|
||||
|
|
|
|||
|
|
@ -95,6 +95,8 @@ class LODCell:
|
|||
count: int
|
||||
source_time_min_ns: int
|
||||
source_time_max_ns: int
|
||||
source_priority: float = 1.0
|
||||
source_kind: str = "generic"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
@ -119,6 +121,8 @@ class MultiLODSceneCache:
|
|||
"count": cell.count,
|
||||
"sourceTimeMinNs": cell.source_time_min_ns,
|
||||
"sourceTimeMaxNs": cell.source_time_max_ns,
|
||||
"sourcePriority": cell.source_priority,
|
||||
"sourceKind": cell.source_kind,
|
||||
}
|
||||
for cell in self.cells
|
||||
],
|
||||
|
|
@ -304,6 +308,94 @@ def multilod_cache_from_points(
|
|||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LODEvidencePoint:
|
||||
stable_key: str
|
||||
xyz: np.ndarray
|
||||
confidence: float
|
||||
timestamp_ns: int
|
||||
source_priority: float
|
||||
source_kind: str
|
||||
|
||||
|
||||
def multilod_cache_from_evidence(
|
||||
evidence: Iterable[LODEvidencePoint],
|
||||
*,
|
||||
levels: tuple[float, ...] = (0.02, 0.08, 0.32),
|
||||
created_monotonic_ns: int = 0,
|
||||
) -> MultiLODSceneCache:
|
||||
items = list(evidence)
|
||||
cells: list[LODCell] = []
|
||||
for level_index, cell_size in enumerate(levels):
|
||||
buckets: dict[tuple[int, int, int], list[LODEvidencePoint]] = {}
|
||||
for item in items:
|
||||
key = tuple(int(np.floor(float(value) / float(cell_size))) for value in np.asarray(item.xyz, dtype=np.float64))
|
||||
buckets.setdefault(key, []).append(item)
|
||||
for key, bucket in buckets.items():
|
||||
weights = np.asarray([max(1.0e-6, item.confidence * item.source_priority) for item in bucket], dtype=np.float64)
|
||||
positions = np.asarray([np.asarray(item.xyz, dtype=np.float64) for item in bucket], dtype=np.float64)
|
||||
center = np.average(positions, axis=0, weights=weights)
|
||||
priority = float(max(item.source_priority for item in bucket))
|
||||
source_kind = max(bucket, key=lambda item: item.source_priority).source_kind
|
||||
cells.append(
|
||||
LODCell(
|
||||
level=level_index,
|
||||
cell=key,
|
||||
center=tuple(float(value) for value in center),
|
||||
radius_m=float(cell_size) * 0.5,
|
||||
confidence=float(min(1.0, np.mean(weights))),
|
||||
count=len(bucket),
|
||||
source_time_min_ns=min(item.timestamp_ns for item in bucket),
|
||||
source_time_max_ns=max(item.timestamp_ns for item in bucket),
|
||||
source_priority=priority,
|
||||
source_kind=source_kind,
|
||||
)
|
||||
)
|
||||
return MultiLODSceneCache(
|
||||
schema="localcast.sensor_fusion.multilod_scene_cache.v1",
|
||||
created_monotonic_ns=int(created_monotonic_ns),
|
||||
levels=tuple(float(level) for level in levels),
|
||||
cells=tuple(cells),
|
||||
)
|
||||
|
||||
|
||||
def evidence_from_render_points(points: Iterable, *, source_kind: str, source_priority: float) -> tuple[LODEvidencePoint, ...]:
|
||||
rows: list[LODEvidencePoint] = []
|
||||
for point in points:
|
||||
rows.append(
|
||||
LODEvidencePoint(
|
||||
stable_key=str(point.stable_key),
|
||||
xyz=np.asarray(point.xyz, dtype=np.float64),
|
||||
confidence=float(point.confidence),
|
||||
timestamp_ns=int(point.source_timestamp_ns),
|
||||
source_priority=float(source_priority),
|
||||
source_kind=str(source_kind),
|
||||
)
|
||||
)
|
||||
return tuple(rows)
|
||||
|
||||
|
||||
def evidence_from_fusion_items(
|
||||
items: Iterable[TriangulatedPoint | TransientMatchHypothesis],
|
||||
*,
|
||||
source_kind: str,
|
||||
source_priority: float,
|
||||
) -> tuple[LODEvidencePoint, ...]:
|
||||
rows: list[LODEvidencePoint] = []
|
||||
for item in items:
|
||||
rows.append(
|
||||
LODEvidencePoint(
|
||||
stable_key=str(getattr(item, "marker_id", getattr(item, "stable_key", "scene-claim"))),
|
||||
xyz=np.asarray(item.xyz, dtype=np.float64),
|
||||
confidence=float(item.confidence),
|
||||
timestamp_ns=item_timestamp_ns(item),
|
||||
source_priority=float(source_priority),
|
||||
source_kind=str(source_kind),
|
||||
)
|
||||
)
|
||||
return tuple(rows)
|
||||
|
||||
|
||||
def item_timestamp_ns(item: TriangulatedPoint | TransientMatchHypothesis) -> int:
|
||||
if hasattr(item, "timestamp_ns"):
|
||||
return int(getattr(item, "timestamp_ns"))
|
||||
|
|
|
|||
|
|
@ -91,16 +91,20 @@ flowchart TD
|
|||
E --> F["ScreenBrushPacket lowering"]
|
||||
F --> G["Spout sender"]
|
||||
G --> H["OBS Spout2 Capture"]
|
||||
I["Leap motion evidence"] --> J["multi-LOD scene cache"]
|
||||
K["RGB/stochastic/synthetic evidence"] --> J
|
||||
J --> L["Aquarium compute reconciliation"]
|
||||
```
|
||||
|
||||
Ownership:
|
||||
|
||||
- `localcast.sensor_fusion.cultcache_docs` owns typed visual state documents.
|
||||
- `scripts/live_sensor_fusion.py` currently owns the live producer and writes `localcast.visual.render_frame` into CultCache.
|
||||
- The live producer also writes a multi-LOD scene cache with source kind and priority. Real Leap frames are the highest-priority timing/spatial evidence; Leap fallback frames are tagged as fallback and do not become ground truth.
|
||||
- `scripts/stream_spout.py` consumes typed CultCache state and publishes Spout.
|
||||
- CultNet document replication is the intended API boundary for other producers/consumers.
|
||||
|
||||
Invariant: JSON is not the visual-state authority. The current renderer can still write a JSON heartbeat for human inspection, but live visual state lives in typed CultCache MessagePack documents.
|
||||
Invariant: JSON is not the visual-state authority. The current renderer can still write a JSON heartbeat for human inspection, but live visual state lives in typed CultCache MessagePack documents. Leap outranks camera surfaces only when the frame source is actual Leap capture; diagnostics must stay labeled as diagnostics.
|
||||
|
||||
## OBS Program Surface
|
||||
|
||||
|
|
|
|||
|
|
@ -21,8 +21,10 @@ from localcast.sensor_fusion import (
|
|||
SensorRig,
|
||||
SurfaceFeatureObservation,
|
||||
dense_stereo_points,
|
||||
evidence_from_fusion_items,
|
||||
evidence_from_render_points,
|
||||
measure_frame_quality,
|
||||
multilod_cache_from_points,
|
||||
multilod_cache_from_evidence,
|
||||
put_live_render_frame,
|
||||
lower_points_to_render_frame,
|
||||
stochastic_transient_matches,
|
||||
|
|
@ -599,11 +601,17 @@ class LeapPackedMotionSampler:
|
|||
self._capture = None
|
||||
self._previous_channels: dict[str, np.ndarray] = {}
|
||||
self._last_timestamp_ns: int | None = None
|
||||
self._last_frame_kind: str | None = None
|
||||
self._fallback_index = 0
|
||||
|
||||
@property
|
||||
def last_timestamp_ns(self) -> int | None:
|
||||
return self._last_timestamp_ns
|
||||
|
||||
@property
|
||||
def last_frame_kind(self) -> str | None:
|
||||
return self._last_frame_kind
|
||||
|
||||
def close(self) -> None:
|
||||
if self._capture is not None:
|
||||
try:
|
||||
|
|
@ -646,23 +654,36 @@ class LeapPackedMotionSampler:
|
|||
return self._fallback_frame()
|
||||
ok, frame = self._capture.read()
|
||||
if ok and frame is not None:
|
||||
self._last_frame_kind = "capture"
|
||||
return frame
|
||||
return self._fallback_frame()
|
||||
|
||||
def _fallback_frame(self) -> np.ndarray | None:
|
||||
import cv2
|
||||
|
||||
for path in (
|
||||
self.fallback_dir / f"rgb-probe-{self.api}-{self.index}.png",
|
||||
self.fallback_dir / f"rgb-probe-dshow-{self.index}.png",
|
||||
self.fallback_dir / f"rgb-probe-msmf-{self.index}.png",
|
||||
self.fallback_dir / "leap-probe.png",
|
||||
):
|
||||
if path.exists():
|
||||
frame = cv2.imread(str(path))
|
||||
if frame is not None:
|
||||
return frame
|
||||
return None
|
||||
path = self.fallback_dir / "leap-probe.png"
|
||||
if path.exists():
|
||||
frame = cv2.imread(str(path))
|
||||
if frame is not None:
|
||||
self._last_frame_kind = "fallback-file"
|
||||
return frame
|
||||
return self._synthetic_motion_frame()
|
||||
|
||||
def _synthetic_motion_frame(self) -> np.ndarray:
|
||||
self._last_frame_kind = "fallback-synthetic"
|
||||
frame = np.zeros((self.height, self.width, 3), dtype=np.uint8)
|
||||
t = self._fallback_index
|
||||
self._fallback_index += 1
|
||||
cx = int((self.width * 0.30) + (self.width * 0.40) * ((np.sin(t * 0.37) + 1.0) * 0.5))
|
||||
cy = int((self.height * 0.30) + (self.height * 0.34) * ((np.cos(t * 0.41) + 1.0) * 0.5))
|
||||
y0 = max(0, cy - 4)
|
||||
y1 = min(self.height, cy + 5)
|
||||
x0 = max(0, cx - 4)
|
||||
x1 = min(self.width, cx + 5)
|
||||
frame[y0:y1, x0:x1, 1] = 220
|
||||
frame[max(0, y0 - 8) : min(self.height, y1 + 8), max(0, x0 - 8) : min(self.width, x1 + 8), 2] = 80
|
||||
frame[max(0, y0 - 12) : min(self.height, y1 + 12), max(0, x0 - 12) : min(self.width, x1 + 12), 0] = 55
|
||||
return frame
|
||||
|
||||
|
||||
def unpack_leap_packed_channels(frame_bgr: np.ndarray) -> dict[str, np.ndarray]:
|
||||
|
|
@ -804,11 +825,6 @@ def main() -> None:
|
|||
samples_per_observation=2,
|
||||
seed=frame_id,
|
||||
)
|
||||
multilod_cache_from_points(
|
||||
tuple(result.points) + tuple(transient_matches),
|
||||
levels=(0.02, 0.08, 0.32),
|
||||
created_monotonic_ns=time.monotonic_ns(),
|
||||
).write_json(lod_cache_path)
|
||||
frame = lower_points_to_render_frame(
|
||||
result.points,
|
||||
render_config,
|
||||
|
|
@ -820,8 +836,22 @@ def main() -> None:
|
|||
leap_points = () if leap_sampler is None else leap_sampler.motion_splats(timestamp_ns)
|
||||
if leap_sampler is not None and leap_sampler.last_timestamp_ns is not None:
|
||||
timestamp_ns = max(timestamp_ns, leap_sampler.last_timestamp_ns)
|
||||
leap_source_kind = (
|
||||
"leap-ground-truth"
|
||||
if leap_sampler is not None and leap_sampler.last_frame_kind == "capture"
|
||||
else "leap-fallback"
|
||||
)
|
||||
leap_source_priority = 3.0 if leap_source_kind == "leap-ground-truth" else 1.1
|
||||
support_points = dim_fusion_points(tuple(frame.points)) if not rgb_points else ()
|
||||
stochastic_points = transient_match_render_points(transient_matches, timestamp_ns)
|
||||
multilod_cache_from_evidence(
|
||||
evidence_from_render_points(leap_points, source_kind=leap_source_kind, source_priority=leap_source_priority)
|
||||
+ evidence_from_render_points(rgb_points, source_kind="rgb-surface", source_priority=1.4)
|
||||
+ evidence_from_fusion_items(transient_matches, source_kind="stochastic-transient", source_priority=1.2)
|
||||
+ evidence_from_fusion_items(result.points, source_kind="synthetic-fusion", source_priority=0.8),
|
||||
levels=(0.01, 0.04, 0.16, 0.64),
|
||||
created_monotonic_ns=time.monotonic_ns(),
|
||||
).write_json(lod_cache_path)
|
||||
frame = RenderFramePacket(
|
||||
schema=frame.schema,
|
||||
frame_id=frame.frame_id,
|
||||
|
|
|
|||
|
|
@ -158,7 +158,10 @@ research:
|
|||
cache rule: semantic claims first, cached backend lowerings second.
|
||||
Stochastic transient matching now lowers descriptor-consistent image
|
||||
observations into scene-ray hypotheses and a multi-LOD world-space cache
|
||||
for Aquarium compute-shader reconciliation.
|
||||
for Aquarium compute-shader reconciliation. Leap evidence is the
|
||||
highest-priority temporal/spatial source when live Leap frames are present;
|
||||
fallback Leap frames stay tagged as fallback evidence and must not be
|
||||
treated as ground truth.
|
||||
typed_visual_state:
|
||||
summary: docs/typed-visual-state.md
|
||||
key_decision: >
|
||||
|
|
@ -267,7 +270,10 @@ current_status:
|
|||
observations can be descriptor-matched and triangulated into world-space
|
||||
tracks. Unit tests inject
|
||||
synthetic cameras and frames so the core does not depend on live camera
|
||||
drivers.
|
||||
drivers. The multi-LOD visual cache now carries source kind and source
|
||||
priority so Aquarium can prefer real Leap evidence over RGB surfaces,
|
||||
stochastic transient hypotheses, and synthetic fusion support while still
|
||||
seeing lower-priority fallback diagnostics when the Leap capture path stalls.
|
||||
next_action: >
|
||||
Capture real ChArUco intrinsics for the PS3 Eyes and Kiyos, capture
|
||||
fixed-board observations from every camera, solve a local
|
||||
|
|
@ -280,7 +286,9 @@ current_status:
|
|||
Then replace the synthetic live producer with a detector adapter that
|
||||
turns FFmpeg PS3 Eye frames into marker `Observation2D` records for the
|
||||
fusion core and publishes the same typed CultCache/CultNet render-frame
|
||||
document. In parallel, move the render-frame consumer into Aquarium Engine so
|
||||
document. Also fix the live Leap capture backend so the cache emits
|
||||
`leap-ground-truth` rather than `leap-fallback`; current MSMF probing can
|
||||
still stall on frame reads. In parallel, move the render-frame consumer into Aquarium Engine so
|
||||
Aquarium renders cached points/brushes into its GPU render target and
|
||||
publishes or hands off the same Spout sender `LocalCastBridge Point Cloud`
|
||||
for OBS, replacing the deadline OpenGL point sink without changing packet
|
||||
|
|
|
|||
|
|
@ -7,7 +7,9 @@ import numpy as np
|
|||
from localcast.sensor_fusion.core import CameraModel, SensorRig, Observation2D
|
||||
from localcast.sensor_fusion.stochastic_mapping import (
|
||||
ImageRayBiasMap,
|
||||
LODEvidencePoint,
|
||||
RayBiasSample,
|
||||
multilod_cache_from_evidence,
|
||||
multilod_cache_from_points,
|
||||
pixel_scene_ray,
|
||||
stochastic_transient_matches,
|
||||
|
|
@ -95,6 +97,19 @@ class StochasticMappingTests(unittest.TestCase):
|
|||
cache.write_json(path)
|
||||
self.assertIn("multilod_scene_cache", path.read_text(encoding="utf-8"))
|
||||
|
||||
def test_multilod_evidence_prefers_leap_priority(self):
|
||||
evidence = [
|
||||
LODEvidencePoint("rgb", np.array([0.01, 0.0, 1.0]), 0.7, 100, 1.0, "rgb"),
|
||||
LODEvidencePoint("leap", np.array([0.03, 0.0, 1.0]), 0.8, 120, 3.0, "leap-ground-truth"),
|
||||
]
|
||||
|
||||
cache = multilod_cache_from_evidence(evidence, levels=(0.10,), created_monotonic_ns=5)
|
||||
cell = cache.cells[0]
|
||||
|
||||
self.assertEqual("leap-ground-truth", cell.source_kind)
|
||||
self.assertEqual(3.0, cell.source_priority)
|
||||
self.assertGreater(cell.center[0], 0.02)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in a new issue