diff --git a/config/sensor-fusion.example.json b/config/sensor-fusion.example.json index 8dc8352..813119a 100644 --- a/config/sensor-fusion.example.json +++ b/config/sensor-fusion.example.json @@ -5,6 +5,14 @@ "min_confidence": 0.1, "cache_ttl_ns": 500000000 }, + "render_bridge": { + "spout_sender_name": "LocalCastBridge Point Cloud", + "target_width": 1920, + "target_height": 1080, + "visual_delay_ns": 250000000, + "audio_alignment_delay_ns": 250000000, + "default_point_radius_m": 0.025 + }, "cameras": [ { "id": "ps3eye_left", diff --git a/docs/sensor-fusion-architecture.md b/docs/sensor-fusion-architecture.md index f832c2c..266792f 100644 --- a/docs/sensor-fusion-architecture.md +++ b/docs/sensor-fusion-architecture.md @@ -36,13 +36,17 @@ flowchart TD D --> F F --> G["TrackCache"] G --> H["PointCloud PLY"] - G --> I["Future brush/splat packets"] + G --> I["RenderFramePacket"] + I --> J["Aquarium GPU renderer"] + J --> K["Spout2 sender texture"] + K --> L["OBS Spout2 source"] ``` ## Ownership - `localcast.sensor_fusion.core` owns calibrated camera models, 2D observations, DLT triangulation, reprojection gating, confidence scoring, point-cloud export, and track cache expiry. - `localcast.sensor_fusion.adapters` owns driver-facing frame ingress. Its first concrete adapter is an FFmpeg raw BGR reader for the PS3 Eye DirectShow path. +- `localcast.sensor_fusion.render_bridge` owns the render-frame ABI: cached point claims plus target dimensions, Spout sender name, source timestamp range, intended visual presentation time, and ambisonic/audio alignment time. - `scripts/sensor_fusion.py` owns CLI composition for offline observation files. - `config/sensor-fusion.example.json` owns the declarative rig shape: capture hints, intrinsics, extrinsics, latency, and fusion thresholds. @@ -57,8 +61,10 @@ sensor observation -> calibrated world ray -> triangulated marker claim -> cached track --> point cloud / brush packet / splat packet --> renderer +-> render frame packet +-> Aquarium brush/splat packet +-> Spout sender texture +-> OBS source ``` The cache is not a bucket. `TrackCache` has a TTL and confidence update rule so the system can buffer and converge without pretending stale points are fresh. Later GPU buffers should mirror this shape: observation SoA, track SoA, brush packet SoA. @@ -74,17 +80,51 @@ The core can be tested without cameras: Those tests matter because camera drivers are not unit-test dependencies. They are weather. +## OBS And Spout Boundary + +OBS should see a named Spout sender, not a camera window, file tail, or desktop capture. Current references confirm the ordinary Windows path: the Off World Live OBS Spout2 plugin receives Spout shared textures, while the Spout2 SDK is the app-side texture sharing surface. + +The intended production handoff is: + +```text +LocalCastBridge TrackCache +-> RenderFramePacket JSON/binary stream +-> Aquarium Engine typed runtime state +-> GPU point/brush/splat buffer +-> D3D render target +-> Spout2 sender named "LocalCastBridge Point Cloud" +-> OBS Spout2 Capture source +``` + +`RenderFramePacket` is a scaffold ABI, not the final hot path. JSON is useful for inspection, tests, and handoff. Once Aquarium consumes the shape, switch the transport to a binary or shared-memory ring with the same fields. The invariant is the packet contract, not JSON. + +## Latency Contract + +Latency is allowed. Drift is not. + +Each render frame carries: + +- `source_time_min_ns` and `source_time_max_ns`: the observation window that produced the cached points. +- `present_time_ns`: when the visual frame wants to be presented after visual buffering. +- `audio_alignment_time_ns`: the corresponding ambisonic output time. + +That lets the final stream compositor buffer point-cloud visuals until they align with the bounded audio-field cache. The renderer may show a slightly older world if the packet is coherent and labeled. It may not quietly mix fresh audio with stale visual geometry and call that sync. + ## Next Cut 1. Add detector adapters that turn PS3 Eye frames into `Observation2D` marker detections. 2. Load real ChArUco intrinsics/extrinsics into `config/sensor-fusion.json`. 3. Record an observation JSONL/JSON run from the two PS3 Eyes. 4. Generate a live sparse PLY stream or debug preview. -5. Lower cached tracks into brush/splat packets once the point cloud is stable. +5. Feed `RenderFramePacket` into Aquarium Engine. +6. Add Aquarium-side GPU point/brush rendering to a D3D render target. +7. Publish the render target as a Spout2 sender and receive it in OBS. +8. Replace JSON packet transport with a binary/shared-memory ring once the contract is stable. ## References - OpenCV multi-camera calibration and triangulation are the geometry spine. See `research/visual-spatial-map/summary.md`. - Jacob and Haeb-Umbach 2015 motivates audio-visual coordinate alignment. See `research/visual-spatial-map/mirrors/krekovic-2015-audio-visual-geometry-calibration.pdf`. - Kerbl et al. 2023 and RTG-SLAM motivate splats as render packets, not state ownership. See `research/visual-spatial-map/summary.md` and `research/gpu-fusion-splatting/summary.md`. +- Spout2 and the OBS Spout2 plugin are the Windows/OBS texture-sharing bridge. See `research/spout-obs-bridge/summary.md`. - `E:\Projects\gamecult-site\GameCult\Blog\fractal-domains-cache-that-bites.md` supplies the cache discipline: semantic authority first, backend packet lowering second. diff --git a/localcast/sensor_fusion/__init__.py b/localcast/sensor_fusion/__init__.py index 0ab2026..3716c0a 100644 --- a/localcast/sensor_fusion/__init__.py +++ b/localcast/sensor_fusion/__init__.py @@ -11,6 +11,7 @@ from .core import ( TriangulatedPoint, load_fusion_config, ) +from .render_bridge import RenderBridgeConfig, RenderFramePacket, RenderPointPacket, lower_points_to_render_frame __all__ = [ "CameraModel", @@ -22,4 +23,8 @@ __all__ = [ "TrackCache", "TriangulatedPoint", "load_fusion_config", + "RenderBridgeConfig", + "RenderFramePacket", + "RenderPointPacket", + "lower_points_to_render_frame", ] diff --git a/localcast/sensor_fusion/render_bridge.py b/localcast/sensor_fusion/render_bridge.py new file mode 100644 index 0000000..b59f50e --- /dev/null +++ b/localcast/sensor_fusion/render_bridge.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +from dataclasses import dataclass +import json +from pathlib import Path + +import numpy as np + +from .core import TriangulatedPoint + + +@dataclass(frozen=True) +class RenderBridgeConfig: + spout_sender_name: str = "LocalCastBridge Point Cloud" + target_width: int = 1920 + target_height: int = 1080 + visual_delay_ns: int = 250_000_000 + audio_alignment_delay_ns: int = 250_000_000 + default_point_radius_m: float = 0.025 + + @staticmethod + def from_dict(data: dict) -> "RenderBridgeConfig": + return RenderBridgeConfig( + spout_sender_name=data.get("spout_sender_name", "LocalCastBridge Point Cloud"), + target_width=int(data.get("target_width", 1920)), + target_height=int(data.get("target_height", 1080)), + visual_delay_ns=int(data.get("visual_delay_ns", 250_000_000)), + audio_alignment_delay_ns=int(data.get("audio_alignment_delay_ns", 250_000_000)), + default_point_radius_m=float(data.get("default_point_radius_m", 0.025)), + ) + + +@dataclass(frozen=True) +class RenderPointPacket: + stable_key: str + xyz: np.ndarray + radius_m: float + color_rgba: tuple[float, float, float, float] + confidence: float + source_timestamp_ns: int + + def to_dict(self) -> dict: + return { + "stable_key": self.stable_key, + "xyz": [float(v) for v in self.xyz], + "radius_m": float(self.radius_m), + "color_rgba": [float(v) for v in self.color_rgba], + "confidence": float(self.confidence), + "source_timestamp_ns": int(self.source_timestamp_ns), + } + + +@dataclass(frozen=True) +class RenderFramePacket: + schema: str + frame_id: int + created_monotonic_ns: int + source_time_min_ns: int + source_time_max_ns: int + present_time_ns: int + audio_alignment_time_ns: int + spout_sender_name: str + target_width: int + target_height: int + points: tuple[RenderPointPacket, ...] + + def to_dict(self) -> dict: + return { + "schema": self.schema, + "frame_id": self.frame_id, + "created_monotonic_ns": self.created_monotonic_ns, + "source_time_min_ns": self.source_time_min_ns, + "source_time_max_ns": self.source_time_max_ns, + "present_time_ns": self.present_time_ns, + "audio_alignment_time_ns": self.audio_alignment_time_ns, + "spout_sender_name": self.spout_sender_name, + "target_width": self.target_width, + "target_height": self.target_height, + "points": [point.to_dict() for point in self.points], + } + + def write_json(self, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(self.to_dict(), indent=2), encoding="utf-8") + + +def lower_points_to_render_frame( + points: tuple[TriangulatedPoint, ...], + config: RenderBridgeConfig, + frame_id: int, + created_monotonic_ns: int, +) -> RenderFramePacket: + if points: + source_min = min(point.timestamp_ns for point in points) + source_max = max(point.timestamp_ns for point in points) + else: + source_min = created_monotonic_ns + source_max = created_monotonic_ns + render_points = tuple(point_to_render_packet(point, config) for point in points) + present_time = source_max + config.visual_delay_ns + return RenderFramePacket( + schema="localcast.sensor_fusion.render_frame.v1", + frame_id=frame_id, + created_monotonic_ns=created_monotonic_ns, + source_time_min_ns=source_min, + source_time_max_ns=source_max, + present_time_ns=present_time, + audio_alignment_time_ns=source_max + config.audio_alignment_delay_ns, + spout_sender_name=config.spout_sender_name, + target_width=config.target_width, + target_height=config.target_height, + points=render_points, + ) + + +def point_to_render_packet(point: TriangulatedPoint, config: RenderBridgeConfig) -> RenderPointPacket: + confidence = max(0.0, min(1.0, float(point.confidence))) + return RenderPointPacket( + stable_key=point.marker_id, + xyz=point.xyz.astype(np.float64), + radius_m=config.default_point_radius_m, + color_rgba=(0.25 + 0.75 * confidence, 0.65, 1.0, confidence), + confidence=confidence, + source_timestamp_ns=point.timestamp_ns, + ) diff --git a/research/spout-obs-bridge/bibliography.md b/research/spout-obs-bridge/bibliography.md new file mode 100644 index 0000000..ea95b60 --- /dev/null +++ b/research/spout-obs-bridge/bibliography.md @@ -0,0 +1,5 @@ +# Spout OBS Bridge Bibliography + +- Off World Live, `obs-spout2-plugin`, OBS plugin implementing Spout2 input/output. Mirror: `mirrors/obs-spout2-plugin-github.html`. +- leadedge, `Spout2`, Windows video frame sharing system and SDK. Mirror: `mirrors/leadedge-spout2-github.html`. +- leadedge, Spout project home. Mirror: `mirrors/spout-home.html`. diff --git a/research/spout-obs-bridge/mirrors/leadedge-spout2-github.html b/research/spout-obs-bridge/mirrors/leadedge-spout2-github.html new file mode 100644 index 0000000..b808bd2 --- /dev/null +++ b/research/spout-obs-bridge/mirrors/leadedge-spout2-github.html @@ -0,0 +1,1822 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + GitHub - leadedge/Spout2: A video frame sharing system for Microsoft Windows · GitHub + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+ Skip to content + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + + + +
+ + + + + +
+ + + + + + + + + +
+
+
+ + + + + + + + + + +
+ +
+ +
+ +
+ + + + / + + Spout2 + + + Public +
+ + +
+ +
+ + +
+
+ +
+
+ + + + +
+ + + + + +
+ + + + + + + + + + + + + + + + + + +
+ + + + +

leadedge/Spout2

+ + + +
BranchesTags

Repository files navigation

Spout is a video frame sharing system for Microsoft Windows, which allows applications to share OpenGL textures in a similar way to Syphon for the Mac.

+

Spout supports DirectX 9, DirectX 11, DirectX 12 and OpenGL textures and includes a developer SDK and sample applications.

+

For questions and suggestions please contact the Spout Discourse Group. Please reserve repository Issues for code development.

+

Documentation

+ +

Further pdf documents and examples can be found in the relevant folders.
+For general questions please visit the Spout Discourse group

+

Pull requests

+

Please make all pull requests from the beta branch.

+

Donations

+

Your support will help the project to continue and develop.
+Become a GitHub Sponsor and show your support.
+Or make a donation :
+

+
+ +
+
+
+
+

About

+ +

+ A video frame sharing system for Microsoft Windows +

+ +

Topics

+
+
+ + texture-sharing + +
+ +
+ +

Resources

+
+ + + Readme +
+ + +

License

+
+ + + BSD-2-Clause license + +
+ + + + + + + + + + + +
+ + Activity
+ + +

Stars

+
+ + 969 + stars
+ +

Watchers

+
+ + 55 + watching
+ +

Forks

+
+ + 161 + forks
+ + +
+ + Report repository +
+
+ +
+
+ + +
+
+

+ Releases + 22

+ + + +
+
+ Spout 2.007.017 + + Latest +
+
Oct 22, 2025
+
+
+ + 21 releases
+
+
+ + + + +
+
+ + + +

+ Packages +

+ + +
+
+
 
+
+
+
+
 
+
+
+
+
 
+
+ + + + + +
+
+ + + + + +
+
+ + +

+ Contributors

+ +
    +
  • +
    +
    • +
  • +
    +
    • +
  • +
    +
    • +
+ + + +
+
+ + +
+
+

Languages

+
+ + + + + +
+ + +
+
+ + +
+
+ + + + + +
+ + + +
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + +
+
+
+ + + diff --git a/research/spout-obs-bridge/mirrors/obs-spout2-plugin-github.html b/research/spout-obs-bridge/mirrors/obs-spout2-plugin-github.html new file mode 100644 index 0000000..6f88289 --- /dev/null +++ b/research/spout-obs-bridge/mirrors/obs-spout2-plugin-github.html @@ -0,0 +1,1821 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + GitHub - Off-World-Live/obs-spout2-plugin: A Plugin for OBS Studio to enable Spout2 (https://github.com/leadedge/Spout2) input / output · GitHub + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+ Skip to content + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + + + +
+ + + + + +
+ + + + + + + + + +
+
+
+ + + + + + + + + + + + + +
+ +
+ +
+ +
+ + + + / + + obs-spout2-plugin + + + Public +
+ + +
+ +
+ + +
+
+ +
+
+ + + + +
+ + + + + +
+ + + + + + + + + + + + + + + + + + +
+ + + + +

Off-World-Live/obs-spout2-plugin

+ + + +
BranchesTags

Repository files navigation

Spout2 Plugin for OBS Studio (64bit)

+

This plugin enables the import and export of shared textures at high resolution to and from SPOUT2 compatible +programs.

+

Why

+

Previously the only way to import shared textures from SPOUT was via the DirectShow SpoutCam interface or by screen-capturing +the full-screen output of the SpoutReceiver program.

+

The SpoutCam is limited to standard webcam resolutions and capped at 1920x1080 and capturing the SpoutReceiver is both +inefficient and limited by your current screen resolution.

+

Previously, there was no way of outputting Spout video textures from OBS.

+

This plugin implements the SPOUT2 SDK, creates an OBS Source from the SPOUT shared texture and a Spout output which sends the content of the OBS canvas to Spout.

+

Please see installation and usage guide here

+

Acknowledgements

+

Thanks to the developer of OBS-OpenVR-Input-Plugin whose source +helped greatly in getting my head around the OBS API.

+

Thanks to the OBS team and their discord channel

+

Thanks to the authors of SPOUT for the library and clear documentation

+

Installation

+
    +
  • Go to the Releases Page
    • +
  • Download the windows installer: OBS_Spout2_Plugin_Installer.exe
    • +
  • Run the installer (accepting installation from untrusted source)
    • +
  • Select the OBS directory if not the default install location
    • +
+
+

N.B there are no current plans for 32bit builds, although theoretically this should be possible

+
+

Contributing / Building

+
    +
  • Clone this repo recursively
    • +
+
git clone --recursive git@github.com:off-world-live/obs-spout2-plugin
+
+
    +
  • Install CMAKE min version 3.28
    • +
  • Either configure and generate through the CMAKE Gui or through the command line for the windows-x64 architecture
    • +
  • Run Configure, Generate and then Open Project in the CMake Gui
    • +
+

Building a release locally

+
    +
  • Open git bash or similar bash terminal interpreter
    • +
  • Run ./scripts/Release.sh <version number>
    • +
  • You should find the executable (installer) and zip file in the main win-spout directory
    • +
+

Building the windows installer

+ +

Pull Requests welcome!

+

Contributors

+

Thanks to everybody that submitted bug tickets and in particular the code contributors:

+ +

Roadmap

+
    +
  • Improve CMakeLists.txt to copy Spout.dll automatically (thanks to @shugen002)
    • +
  • Spout Output
    • +
  • Spout Filter Output
    • +
+

License

+

This plugin authored by Campbell Morgan is Copyright Off World Live Ltd, 2019-2021 and licenced under the GPL V.2.

+
+ +
+
+
+
+

About

+ +

+ A Plugin for OBS Studio to enable Spout2 (https://github.com/leadedge/Spout2) input / output +

+
+ + + knowledge.offworld.live/en/spout-plugin-for-obs-studio + +
+ +

Topics

+
+
+ + spout + + + obs + + + obs-studio + +
+ +
+ +

Resources

+
+ + + Readme +
+ + +

License

+
+ + + GPL-2.0 license + +
+ + + + + + + + + + + +
+ + Activity
+ +
+ + Custom properties
+ +

Stars

+
+ + 970 + stars
+ +

Watchers

+
+ + 29 + watching
+ +

Forks

+
+ + 65 + forks
+ + +
+ + Report repository +
+
+ +
+
+ + +
+
+

+ Releases + 16

+ + + +
+
+ Spout2 Plugin for OBS 1.9.0 + + Latest +
+
Apr 1, 2025
+
+
+ + 15 releases
+
+
+ + + + + + + + +
+
+ + +

+ Contributors

+ +
    +
  • +
    +
    • +
  • +
    +
    • +
  • +
    +
    • +
+ + + +
+
+ + +
+
+

Languages

+
+ + + + + +
+ + +
+
+ + +
+
+ + + + + +
+ + + +
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + +
+
+
+ + + diff --git a/research/spout-obs-bridge/mirrors/spout-home.html b/research/spout-obs-bridge/mirrors/spout-home.html new file mode 100644 index 0000000..050268d --- /dev/null +++ b/research/spout-obs-bridge/mirrors/spout-home.html @@ -0,0 +1,102 @@ + + + + + Spout + + + + + + +
+ + + +
+ +
+

Spout

+

+ Spout + is a system for Microsoft Windows for GPU accelerated sharing
+ of graphics between applications in a similar way to Syphon for OSX.
+ Spout supports DirectX 9, DirectX 11 and OpenGL textures.
+ A comprehensive SDK can be used for creating applications.
+ Explore the other Spout repositories for more detail.
+ Spout setup or the Spout SDK can be downloaded below. +

+ +

Download

+ +


+ +

Open source projects

+

+ As a convenient start, either for development or for practical use
+ explore a sorted list of open source projects that use Spout. +


+ +

SpoutCam

+ +

"SpoutCam" is a virtual webcam that is also a Spout receiver. A setup program provides options for frame rate and resolution and can be used to register for both 32 bit and 64 bit host programs. +

+ Download the latest release from the repository Releases page below.
+ Refer to the instructions and "readme.txt" in the archive for further details. +

+ +

Download

+
+ +

Spout To NDI

+

+ Spout to NDI is a set of programs that allow Spout senders and receivers to share video and audio by way of a network using the NDI (“Network Device Interface”) protocol, a video connectivity standard that enables multimedia systems to communicate with one another over IP. Options are available for Spout to NDI to capture system audio and send it together with image data and for NDI to Spout to receive audio from the NDI source as well as the graphics and play it through the speakers. Conversion from GPU textures is done using Openframeworks shaders by way of the ofxNDI addon. Unzip the distribution file to any convenient folder.
+ +

Download

+
+ +

Spout To OMT

+

+ Spout to OMT is a set of programs that allow Spout senders and receivers to share video by way of a network using the OMT (“Open Media Transport”) protocol. This is an open-source network protocol for high performance, low latency video over a local area network. Conversion from GPU textures is done using high speed compute shaders for minimum overhead. Unzip the distribution file to any convenient folder.
+ +

Download

+ +


+ +

Contact and support

+

+ Please visit the Spout Discourse group. +


+ +

Donations

+

Spout is open source software and there is no company or income to assist ongoing development, maintenance and support. So please consider becoming a supporter to help keep the project going. All contributions are most welcome.

+ + Show your support and become a Sponsor through GitHub

+ or

+ + Make a donation through PayPal
+ + +
+ + + + +
+ +


+ +



+ +
+ +
+ + + diff --git a/research/spout-obs-bridge/summary.md b/research/spout-obs-bridge/summary.md new file mode 100644 index 0000000..b2faa99 --- /dev/null +++ b/research/spout-obs-bridge/summary.md @@ -0,0 +1,7 @@ +# Spout OBS Bridge Summary + +Spout is the Windows-local GPU texture sharing bridge for this rig. The OBS path is the Off World Live OBS Spout2 plugin, which creates an OBS source from a Spout shared texture and can also send the OBS canvas back to Spout. The Spout2 SDK is the application-side sender/receiver surface; its modern path centers on DirectX shared textures and has native DirectX/OpenGL integration options. + +Decision: LocalCastBridge should not render final pixels itself. It should publish typed render-frame packets from the fusion cache. Aquarium Engine should consume those packets, render them into a GPU render target, and publish that texture as a named Spout sender. OBS then receives that sender through the OBS Spout2 plugin. + +Latency decision: the render packet carries both source timestamps and intended presentation time. The stream compositor can buffer visual frames until they align with the ambisonic audio field's presentation timeline. Latency is allowed; unbounded drift is not. diff --git a/scripts/sensor_fusion.py b/scripts/sensor_fusion.py index 614ac8c..eec5dbf 100644 --- a/scripts/sensor_fusion.py +++ b/scripts/sensor_fusion.py @@ -7,7 +7,14 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) -from localcast.sensor_fusion import Observation2D, PointCloud, TrackCache, load_fusion_config +from localcast.sensor_fusion import ( + Observation2D, + PointCloud, + RenderBridgeConfig, + TrackCache, + load_fusion_config, + lower_points_to_render_frame, +) @@ -19,7 +26,9 @@ def load_observations(path: Path) -> list[Observation2D]: def fuse_file(args) -> None: - rig = load_fusion_config(Path(args.config)) + config_path = Path(args.config) + config_data = json.loads(config_path.read_text(encoding="utf-8")) + rig = load_fusion_config(config_path) observations = load_observations(Path(args.observations)) result = rig.fuse(observations) cache = TrackCache(ttl_ns=rig.config.cache_ttl_ns) @@ -30,8 +39,20 @@ def fuse_file(args) -> None: cloud = PointCloud(cache.points()) output = Path(args.output) cloud.write_ply(output) + render_frame_output = None + if args.render_frame_json: + render_config = RenderBridgeConfig.from_dict(config_data.get("render_bridge", {})) + frame = lower_points_to_render_frame( + cloud.points, + render_config, + frame_id=args.frame_id, + created_monotonic_ns=args.created_ns, + ) + render_frame_output = Path(args.render_frame_json) + frame.write_json(render_frame_output) summary = { "output": str(output), + "render_frame_json": None if render_frame_output is None else str(render_frame_output), "points": len(cloud.points), "dropped": list(result.dropped), } @@ -47,6 +68,9 @@ def main() -> None: p.add_argument("--observations", required=True) p.add_argument("--output", required=True) p.add_argument("--now-ns", type=int) + p.add_argument("--render-frame-json") + p.add_argument("--frame-id", type=int, default=0) + p.add_argument("--created-ns", type=int, default=0) p.set_defaults(func=fuse_file) args = parser.parse_args() diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 3c61954..373e182 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -30,3 +30,4 @@ {"ts": "2026-05-18T14:50:58+00:00", "type": "verification", "status": "partial", "note": "Audio profile updated for actual distributed rig: local Focusrite shielded cardioid and neighbor Focusrite shotgun are dialogue anchors; Kiyo/PS Eye mics are spatial/context sources. Local PortAudio/WASAPI probe: Scarlett input accepts 48 kHz and rejects 96 kHz; Scarlett output accepts 44.1 kHz and rejects 48/96 kHz, so ASIO/native Focusrite access must be checked before high-rate calibration."} {"ts": "2026-05-18T14:55:13+00:00", "type": "decision", "status": "ok", "note": "Audio field live output may use bounded buffering: latency is acceptable while delay/SRO estimates settle, but cache depth must converge toward real-time. Core audio code now has testable seams for buffers, latency policy, injectable capture/alignment/encoder/sink ports, and pipeline orchestration."} {"ts": "2026-05-18T14:57:32+00:00", "type": "decision", "status": "ok", "note": "Sensor-fusion sidecar starts with semantic observation authority: calibrated cameras plus timestamped 2D marker observations triangulate into cached 3D point claims, then lower to PLY/brush/splat packets. Capture adapters and render lowerings stay outside the pure fusion core so unit tests can inject synthetic cameras, frames, and observations."} +{"ts": "2026-05-18T15:07:24+00:00", "type": "decision", "status": "ok", "note": "OBS visualization boundary: LocalCastBridge should emit typed render-frame packets from the sensor-fusion cache; Aquarium Engine should consume them, render points/brushes/splats to a GPU render target, and publish that texture as Spout sender 'LocalCastBridge Point Cloud'. Render packets carry visual presentation and audio-alignment timestamps so latency can converge with the ambisonic output timeline."} diff --git a/state/map.yaml b/state/map.yaml index 171fe58..6289d43 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -97,7 +97,7 @@ research: before adding RGB projection or Gaussian splat rendering. For a Dreams-like volumetric brushstroke look, keep the world model as truth and lower semantic scene claims into cached brush/splat packets under budget. - gpu_fusion_splatting: + gpu_fusion_splatting: summary: research/gpu-fusion-splatting/summary.md bibliography: research/gpu-fusion-splatting/bibliography.md key_decision: > @@ -106,6 +106,15 @@ research: GPU-shaped observation and brush packet buffers, and CUDA/Vulkan/D3D splat rendering with minimal readback. CPU owns orchestration and semantics; kernels do not own calibration truth. + spout_obs_bridge: + summary: research/spout-obs-bridge/summary.md + bibliography: research/spout-obs-bridge/bibliography.md + key_decision: > + OBS should receive a named Spout2 shared texture from Aquarium Engine, + not desktop capture or file polling. LocalCastBridge emits typed + render-frame packets with source timestamps, intended presentation time, + audio-alignment time, target size, Spout sender name, and point/brush + claims. Aquarium owns GPU rendering and Spout publication. sensor_fusion_architecture: summary: docs/sensor-fusion-architecture.md key_decision: > @@ -156,14 +165,21 @@ current_status: timestamped 2D observations, DLT triangulation, reprojection/confidence gating, a TTL track cache, and PLY point-cloud export; adapters own FFmpeg/raw-frame ingress; `scripts/sensor_fusion.py fuse-file` can turn an - observation file into a sparse PLY. Unit tests inject synthetic cameras and - frames so the core does not depend on live camera drivers. + observation file into a sparse PLY. The render bridge now lowers cached + points into `localcast.sensor_fusion.render_frame.v1` packets carrying + source time ranges, intended visual presentation time, audio-alignment + time, target render size, Spout sender name, and point draw metadata for + Aquarium/Spout. Unit tests inject synthetic cameras and frames so the core + does not depend on live camera drivers. next_action: > Capture real ChArUco intrinsics/extrinsics for the PS3 Eyes and Kiyos, replace `config/sensor-fusion.example.json` placeholder transforms with a local `config/sensor-fusion.json`, then add a detector adapter that turns FFmpeg PS3 Eye frames into marker `Observation2D` records for the fusion - core. In parallel, create local `config/audio-field.json`, confirm local + core. Then add an Aquarium Engine consumer for render-frame packets that + renders cached points/brushes into a GPU render target and publishes it as + Spout sender `LocalCastBridge Point Cloud` for OBS. In parallel, create local + `config/audio-field.json`, confirm local Kiyo/PS Eye/Focusrite device matches, confirm the neighbor Focusrite shotgun capture/transport path, replace placeholder mic/speaker geometry with measured world coordinates, then build the delay/SRO alignment stage diff --git a/tests/test_sensor_fusion.py b/tests/test_sensor_fusion.py index 769bbe4..e4afdd8 100644 --- a/tests/test_sensor_fusion.py +++ b/tests/test_sensor_fusion.py @@ -3,7 +3,16 @@ import unittest import numpy as np -from localcast.sensor_fusion import CameraModel, FusionConfig, Observation2D, PointCloud, SensorRig, TrackCache +from localcast.sensor_fusion import ( + CameraModel, + FusionConfig, + Observation2D, + PointCloud, + RenderBridgeConfig, + SensorRig, + TrackCache, + lower_points_to_render_frame, +) from localcast.sensor_fusion.adapters import read_raw_bgr_frames @@ -112,6 +121,31 @@ class SensorFusionTests(unittest.TestCase): self.assertEqual("eye", frames[0].sensor_id) self.assertEqual((2, 2, 3), frames[0].image_bgr.shape) + def test_render_frame_packet_carries_spout_and_audio_alignment_metadata(self): + left = camera("left", -0.25) + right = camera("right", 0.25) + point = SensorRig(cameras={"left": left, "right": right}).fuse( + [ + Observation2D("left", "ball", 1_000, left.project_world(np.array([0.0, 0.0, 2.0])), 1.0), + Observation2D("right", "ball", 2_000, right.project_world(np.array([0.0, 0.0, 2.0])), 0.8), + ] + ).points[0] + config = RenderBridgeConfig( + spout_sender_name="test-spout", + visual_delay_ns=100, + audio_alignment_delay_ns=200, + default_point_radius_m=0.05, + ) + + frame = lower_points_to_render_frame((point,), config, frame_id=7, created_monotonic_ns=3_000) + data = frame.to_dict() + + self.assertEqual("localcast.sensor_fusion.render_frame.v1", data["schema"]) + self.assertEqual("test-spout", data["spout_sender_name"]) + self.assertEqual(2_100, data["present_time_ns"]) + self.assertEqual(2_200, data["audio_alignment_time_ns"]) + self.assertEqual(0.05, data["points"][0]["radius_m"]) + if __name__ == "__main__": unittest.main()