GameCult/Mimir
Watch
9
0
Fork
You've already forked Mimir
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Bias Spout budget toward focused surfaces

Browse source
This commit is contained in:
Meta Vens 2026-05-18 23:18:07 +01:00
commit 883b07574e
2 changed files with 80 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

61
localcast/sensor_fusion/spout_output.py
View file

@ -225,20 +225,24 @@ def frame_to_vertex_array(frame: RenderFramePacket, point_scale: float = 1.0) ->
return vertices
def frame_with_point_budget(frame: RenderFramePacket, max_points: int) -> RenderFramePacket:
def frame_with_point_budget(
frame: RenderFramePacket,
max_points: int,
camera_preset_name: str = "kiyo-mid-deru",
) -> RenderFramePacket:
if max_points <= 0 or len(frame.points) <= max_points:
return frame
priority_count = max(1, int(max_points * 0.55))
temporal_count = max(0, max_points - priority_count)
priority = sorted(
frame.points,
key=lambda point: (-float(point.confidence), stable_hash(point.stable_key, 0)),
key=lambda point: (-render_priority(point, camera_preset_name), stable_hash(point.stable_key, 0)),
)[:priority_count]
selected_keys = {point.stable_key for point in priority}
remainder = [point for point in frame.points if point.stable_key not in selected_keys]
temporal = sorted(
remainder,
key=lambda point: stable_hash(point.stable_key, frame.frame_id),
key=lambda point: stable_hash(point.stable_key, frame.frame_id) / max(0.05, render_priority(point, camera_preset_name)),
)[:temporal_count]
points = priority + temporal
points = sorted(points, key=lambda point: point.stable_key)
@ -262,6 +266,35 @@ def stable_hash(key: str, salt: int) -> int:
return int.from_bytes(digest, "little")
def render_priority(point: RenderPointPacket, camera_preset_name: str) -> float:
key = point.stable_key
confidence = max(0.0, min(1.0, float(point.confidence)))
semantic = 1.0
if key.startswith("dense-rgb:"):
semantic = 2.0
elif key.startswith("room-rgb:"):
semantic = 1.15
elif key.startswith("leap-motion:"):
semantic = 1.0
elif key.startswith("stochastic:"):
semantic = 0.36
elif key.startswith("audio-event-"):
semantic = 0.24
elif key.startswith(("sensor:", "frustum:")):
semantic = 0.12
focus = render_focus_weight(point, camera_preset_name)
return max(0.001, confidence * semantic * focus)
def render_focus_weight(point: RenderPointPacket, camera_preset_name: str) -> float:
if camera_preset_name != "kiyo-mid-deru":
return 1.0
xyz = np.asarray(point.xyz, dtype=np.float64)
deru = np.array([0.48, 0.28, 1.24], dtype=np.float64)
distance = float(np.linalg.norm((xyz - deru) * np.array([1.0, 1.0, 0.75], dtype=np.float64)))
return 0.22 + 1.78 / (1.0 + distance * 1.85)
def lower_frame_to_screen_brushes(
frame: RenderFramePacket,
width: int,
@ -276,7 +309,8 @@ def lower_frame_to_screen_brushes(
if projected is None:
continue
px, py, depth, clip_w = projected
radius = max(2, int(round(float(point.radius_m) * point_scale * height * 1.45 / max(0.42, clip_w))))
radius_scale = semantic_radius_scale(point.stable_key)
radius = max(2, int(round(float(point.radius_m) * point_scale * height * 1.45 * radius_scale / max(0.42, clip_w))))
if px < -radius * 3 or px >= width + radius * 3 or py < -radius * 3 or py >= height + radius * 3:
continue
stable = hashlib.blake2s(point.stable_key.encode("utf-8"), digest_size=4).digest()
@ -286,11 +320,28 @@ def lower_frame_to_screen_brushes(
radii = (max(2, int(round(radius * stretch))), radius)
rgba = np.clip(np.asarray(point.color_rgba, dtype=np.float32), 0.0, 1.0)
rgba[3] *= max(0.0, min(1.0, float(point.confidence)))
rgba[3] *= semantic_alpha_scale(point.stable_key)
color = tuple(int(v) for v in (rgba * 255.0)[:4])
brushes.append(ScreenBrushPacket((px, py), radii, angle, color, depth))
return tuple(sorted(brushes, key=lambda brush: brush.depth, reverse=True))
def semantic_radius_scale(stable_key: str) -> float:
if stable_key.startswith("audio-event-"):
return 0.42
if stable_key.startswith("stochastic:"):
return 0.50
return 1.0
def semantic_alpha_scale(stable_key: str) -> float:
if stable_key.startswith("audio-event-"):
return 0.36
if stable_key.startswith("stochastic:"):
return 0.46
return 1.0
def rasterize_frame_rgba(
frame: RenderFramePacket,
width: int,
@ -639,7 +690,7 @@ class OpenGLSpoutPointRenderer:
assert self.vbo is not None
assert self.vao is not None
render_frame = frame_with_point_budget(frame, self.config.max_render_points)
render_frame = frame_with_point_budget(frame, self.config.max_render_points, self.config.camera_preset)
self.last_render_point_count = len(render_frame.points)
vertices = frame_to_vertex_array(render_frame, self.config.point_scale)
glBindFramebuffer(GL_FRAMEBUFFER, self.fbo)

35
tests/test_sensor_fusion.py
View file

@ -231,7 +231,7 @@ class SensorFusionTests(unittest.TestCase):
self.assertAlmostEqual(160, brushes[0].center_px[0], delta=8)
self.assertAlmostEqual(90, brushes[0].center_px[1], delta=8)
def test_render_point_budget_preserves_high_confidence_claims_and_rotates_remainder(self):
def test_render_point_budget_prefers_focus_surface_claims_and_rotates_remainder(self):
def make_frame(frame_id):
return RenderFramePacket(
schema="localcast.sensor_fusion.render_frame.v1",
@ -246,24 +246,37 @@ class SensorFusionTests(unittest.TestCase):
target_height=180,
points=tuple(
RenderPointPacket(
stable_key=f"point-{index}",
xyz=np.array([0.0, 0.0, 1.0 + index * 0.01]),
stable_key=f"room-rgb:deru:{index}",
xyz=np.array([0.48 + index * 0.005, 0.28, 1.1 + index * 0.01]),
radius_m=0.02,
color_rgba=(1.0, 1.0, 1.0, 1.0),
confidence=index / 10.0,
confidence=0.55,
source_timestamp_ns=1,
)
for index in range(10)
for index in range(8)
)
+ tuple(
RenderPointPacket(
stable_key=f"stochastic:debug:{index}",
xyz=np.array([-1.0 + index * 0.05, -0.8, 1.0]),
radius_m=0.04,
color_rgba=(1.0, 0.88, 0.32, 1.0),
confidence=0.95,
source_timestamp_ns=1,
)
for index in range(8)
),
)
first = frame_with_point_budget(make_frame(1), 4)
second = frame_with_point_budget(make_frame(2), 4)
first = frame_with_point_budget(make_frame(1), 6, "kiyo-mid-deru")
second = frame_with_point_budget(make_frame(2), 6, "kiyo-mid-deru")
self.assertEqual(4, len(first.points))
self.assertEqual(4, len(second.points))
self.assertTrue({"point-8", "point-9"}.issubset({point.stable_key for point in first.points}))
self.assertTrue({"point-8", "point-9"}.issubset({point.stable_key for point in second.points}))
self.assertEqual(6, len(first.points))
self.assertEqual(6, len(second.points))
self.assertGreater(
sum(point.stable_key.startswith("room-rgb:deru:") for point in first.points),
sum(point.stable_key.startswith("stochastic:debug:") for point in first.points),
)
self.assertNotEqual(
{point.stable_key for point in first.points},
{point.stable_key for point in second.points},