Compare commits

..
Author SHA1 Message Date
Meta Vens
f7b61a3753 Make Muninn OBS device selection independent 2026-06-19 23:33:53 +01:00
Meta Vens
aa4ac6760b Verify Muninn OBS source registration 2026-06-19 23:06:20 +01:00
Meta Vens
bdf940ba73 Add split Muninn OBS source types 2026-06-19 22:56:13 +01:00
Meta Vens
e2461ee4ac Ship Muninn OBS plugin locale bundle 2026-06-19 22:35:58 +01:00
Meta Vens
1e5a62eb77 Preserve Muninn source selection across discovered ids 2026-06-19 22:22:47 +01:00
Meta Vens
a911d22c7c Use full Muninn audio reorder budget by default 2026-06-19 21:49:05 +01:00
Meta Vens
48c5268798 Expose split Muninn audio and video sources 2026-06-19 21:36:25 +01:00
Meta Vens
e93f0474f7 Stabilize Muninn audio receive buffering 2026-06-19 20:49:46 +01:00
Meta Vens
64c6275784 Recover Muninn video with striped parity 2026-06-19 20:30:22 +01:00
Meta Vens
1096991c28 Forward Muninn media frames without receiver stalls 2026-06-19 20:10:12 +01:00
Meta Vens
1f0793307b Improve Muninn OBS parity recovery 2026-06-19 19:50:55 +01:00
Meta Vens
af868d2a85 Add Muninn OBS restart helper 2026-06-19 19:11:54 +01:00
Meta Vens
50098dc130 Run OBS native tests during plugin build 2026-06-19 18:34:58 +01:00
Meta Vens
86ccd436dc Test Muninn RUDP URL media settings 2026-06-19 18:33:37 +01:00
Meta Vens
cc14932522 Log Muninn OBS activation media settings 2026-06-19 18:31:21 +01:00
Meta Vens
64be5e5bb3 Time-slice Muninn OBS video assembly expiry 2026-06-19 18:25:53 +01:00
Meta Vens
bc5983c172 Fix Muninn RUDP sequence gap accounting 2026-06-19 17:54:49 +01:00
Meta Vens
9e3fe18e4f Trace Muninn RUDP media sequence gaps 2026-06-19 17:43:30 +01:00
Meta Vens
33f07186d7 Honor Muninn RUDP latency budget in OBS bridge 2026-06-19 17:30:17 +01:00
Meta Vens
c979894a14 Prevent Muninn OBS dead frame resurrection 2026-06-19 17:17:10 +01:00
Meta Vens
bda55c4527 Match Muninn RUDP packet payload default 2026-06-19 16:59:16 +01:00
Meta Vens
7beb5afc0b Honor Muninn RUDP latency for packet expiry 2026-06-19 16:46:21 +01:00
Meta Vens
dbe59014e2 Decode Muninn RUDP audio outside OBS media source 2026-06-19 16:33:30 +01:00
Meta Vens
4c124917b0 Expose Muninn RUDP budgets in OBS source 2026-06-19 16:24:42 +01:00
Meta Vens
40f7d5c313 Let Muninn OBS bridge rejoin active RUDP senders 2026-06-19 15:25:34 +01:00
Meta Vens
186613d216 Honor Muninn RUDP profile changes 2026-06-19 14:45:13 +01:00
Meta Vens
6666ed81a2 Avoid Muninn OBS startup stale resets 2026-06-19 14:07:02 +01:00
Meta Vens
1118171733 Clock Muninn OBS audio output at source 2026-06-19 14:03:14 +01:00
Meta Vens
2eff6d24ad Request Muninn RUDP chunk repairs before expiry 2026-06-19 03:59:12 +01:00
Meta Vens
e6df43fdda Respect Muninn RUDP frame dependencies 2026-06-19 03:28:48 +01:00
Meta Vens
1c511cc862 Pace Muninn RUDP decoder feed 2026-06-19 02:54:26 +01:00
Meta Vens
97e4c1e174 Gate Muninn OBS video after incomplete frames 2026-06-19 02:24:01 +01:00
Meta Vens
78ef023bef Reduce Muninn OBS recovery thrash 2026-06-19 01:58:14 +01:00
Meta Vens
7a35adc461 Make Muninn OBS discovery and video forwarding resilient 2026-06-19 01:53:16 +01:00
Meta Vens
0c9e54656b Make Muninn OBS reconnect receiver-owned 2026-06-19 01:10:37 +01:00
Meta Vens
da543c420d Discover Muninn streams over RUDP catalog 2026-06-19 00:46:45 +01:00
Meta Vens
70098f0c49 Make Muninn OBS source self-connecting 2026-06-19 00:29:23 +01:00
Meta Vens
0352a39ac9 Remove Muninn OBS media sequence gate 2026-06-18 23:29:50 +01:00
Meta Vens
bf27821729 Parse Muninn sender resend cadence in RUDP URL 2026-06-18 22:42:39 +01:00
Meta Vens
1afe21b206 Share OBS Muninn RUDP fragment assembler with probe 2026-06-18 22:20:52 +01:00
Meta Vens
4e17d3b7ab Reassemble OBS Muninn RUDP packet fragments 2026-06-18 22:18:02 +01:00
Meta Vens
2bb94d8a6f Share OBS Muninn RUDP ACK tracker with probe 2026-06-18 22:14:40 +01:00
Meta Vens
b2304bf764 Emit OBS Muninn RUDP ACK masks 2026-06-18 22:12:33 +01:00
Meta Vens
d484d868e1 Bound OBS Muninn RUDP receiver latency 2026-06-18 22:04:30 +01:00
Meta Vens
952360c526 Expose Muninn RUDP resend budget in OBS URL parser 2026-06-18 21:53:39 +01:00
Meta Vens
d15723661d Share Muninn RUDP URL parser with probe 2026-06-18 21:39:14 +01:00
Meta Vens
69effd37e6 Read Muninn RUDP latency budget from stream URL 2026-06-18 21:35:19 +01:00
Meta Vens
4be2615cd1 Emit deterministic Muninn feedback fixture 2026-06-18 21:30:41 +01:00
Meta Vens
2f0e0bdfcb Share Muninn media feedback wire encoder 2026-06-18 21:13:48 +01:00
Meta Vens
fc039eaade Emit Muninn RUDP receiver feedback from OBS 2026-06-18 21:09:17 +01:00
Meta Vens
06d5e2b496 Sweep Muninn video assemblies while idle 2026-06-18 21:01:08 +01:00
Meta Vens
1422208032 Expire incomplete Muninn video assemblies 2026-06-18 20:57:01 +01:00
Meta Vens
44aebe1d54 Make Muninn media probe verify video reassembly 2026-06-18 18:22:57 +01:00
Meta Vens
a16510c12c Reassemble chunked Muninn video in OBS bridge 2026-06-18 18:21:13 +01:00
Meta Vens
26c0b204ad Report Muninn OBS bridge receive pressure 2026-06-18 18:16:42 +01:00
Meta Vens
115dd43ecb Add Muninn media decoder probe 2026-06-18 17:58:49 +01:00
Meta Vens
a60b742434 Decode typed Muninn media in OBS source 2026-06-18 17:49:20 +01:00
Meta Vens
43745268ed Request Muninn RUDP streams from OBS source 2026-06-18 15:12:24 +01:00
Meta Vens
e54ee13ee0 Read Muninn OBS catalog from CultCache 2026-06-08 21:31:25 +01:00
Meta Vens
34140bcd57 Add Muninn OBS stream source 2026-06-08 21:01:48 +01:00
Meta Vens
0cc1414abf Add Raven mux OBS sink 2026-06-07 17:42:17 +01:00
Meta Vens
9642a4d205 Add Raven capture mux daemon 2026-06-07 17:24:55 +01:00
Meta Vens
75ab8a8398 Update VoidBot state 2026-06-07 05:33:19 +01:00
Meta Vens
34d6fad9c0 Add Verse media transport policy 2026-06-07 05:26:34 +01:00
Meta Vens
58db546e36 Revert "Align VoidBot swarm CTB rows"
This reverts commit 727534acba.
2026-06-05 15:44:41 +01:00
Meta Vens
727534acba Align VoidBot swarm CTB rows 2026-06-05 15:36:58 +01:00
Meta Vens
4f3b0e023f Publish Eve dashboard over CultNet 2026-06-05 00:22:24 +01:00
Meta Vens
20410b5a26 Record Mimir semantic Verse provider routes 2026-06-04 23:59:07 +01:00
Meta Vens
db901a2509 Bank Mimir face state 2026-06-04 11:08:48 +01:00
Meta Vens
957a9428f6 Make Well publish ingest heartbeat 2026-06-04 11:08:32 +01:00
Meta Vens
aa08744350 Cull noisy Eve dashboard surfaces 2026-06-04 10:58:19 +01:00
Meta Vens
4ff11a57e3 Bank Mimir face state 2026-06-03 05:36:42 +01:00
Meta Vens
bef44778b6 Show Move sync frame starvation 2026-06-03 05:36:23 +01:00
Meta Vens
302db2ae5b Bank Mimir face state 2026-06-03 05:34:16 +01:00
Meta Vens
fd641fab7a Make daemon status reject stale PIDs 2026-06-03 05:34:10 +01:00
Meta Vens
4c75abcada Expose Well sender pressure 2026-06-03 05:25:50 +01:00
Meta Vens
aacaa9255b Bank Mimir face state 2026-06-03 05:20:31 +01:00
Meta Vens
3eae959e3e Make Well shakedown summarize live run 2026-06-03 05:20:21 +01:00
Meta Vens
89bc1bbb06 Bank Mimir face state 2026-06-03 05:14:32 +01:00
Meta Vens
e1520d1284 Derive program video availability from live samples 2026-06-03 05:14:10 +01:00
Meta Vens
c0a6cad912 Bank Mimir face state 2026-06-03 05:10:54 +01:00
Meta Vens
f87803f26f Expose Well source configuration health 2026-06-03 05:10:31 +01:00
Meta Vens
79de23a4c1 Bank Mimir face state 2026-06-03 04:59:56 +01:00
Meta Vens
b1767182c9 Keep Well heavy analysis off cadence 2026-06-03 04:59:29 +01:00
Meta Vens
d58b0f6515 Bank Mimir face state 2026-06-03 04:41:25 +01:00
Meta Vens
2fa008aee4 Clarify Well source health and process clocks 2026-06-03 04:41:01 +01:00
Meta Vens
980f185104 Bank Mimir face state 2026-06-03 04:13:30 +01:00
Meta Vens
3df9ae0768 Stabilize ASIO readiness in Well frames 2026-06-03 04:13:10 +01:00
Meta Vens
adbaa3342b Add Mimir provider advertisement fixture 2026-06-03 03:34:08 +01:00
Meta Vens
38bcf47fb1 Document Mimir Verse service contract 2026-06-03 03:12:05 +01:00
Meta Vens
e18a6a0320 Bank Mimir face state 2026-06-02 23:50:47 +01:00
Meta Vens
f5193341e9 Fix Well clock freshness and reservoir intake 2026-06-02 23:50:14 +01:00
Meta Vens
7ab555a355 Bank Mimir face state 2026-06-02 19:20:21 +01:00
Meta Vens
92bc5d24f1 Expose online daemon status to Odin 2026-06-02 19:18:55 +01:00
Meta Vens
ba607f6028 Expose Fensalir provider catalog 2026-06-02 18:44:27 +01:00
Meta Vens
794a514c11 Bank Mimir face state 2026-06-02 18:30:39 +01:00
Meta Vens
101ea2b217 Advertise Fensalir daemon pressure capabilities 2026-06-02 18:30:22 +01:00
Meta Vens
d65d744d55 Bank Mimir face state 2026-06-02 18:24:28 +01:00
Meta Vens
1670f05891 Publish Fensalir reservoir pressure state 2026-06-02 18:24:06 +01:00
Meta Vens
561accadf7 Bank Mimir face state 2026-06-02 18:18:23 +01:00
Meta Vens
0037eb87f2 Consume stream frames in Fensalir daemon 2026-06-02 18:18:12 +01:00
Meta Vens
20f92b50b8 Start Fensalir from online Verse supervisor 2026-06-02 18:14:54 +01:00
Meta Vens
ce77e4907c Bank Mimir face state 2026-06-02 18:10:45 +01:00
Meta Vens
f365b7c1dd Add Fensalir reservoir worker state 2026-06-02 18:10:23 +01:00
Meta Vens
d91487fc82 Bank Mimir face state 2026-06-02 17:46:52 +01:00
Meta Vens
85459184f5 Add Fensalir daemon Eve surface 2026-06-02 17:43:30 +01:00
Meta Vens
ac307e2115 Bank Mimir face state 2026-06-02 17:25:07 +01:00
Meta Vens
20493337eb Separate editor presentation from reservoir work 2026-06-02 17:24:35 +01:00
Meta Vens
7790b8acda Add Well OBS daemon 2026-06-02 16:18:15 +01:00
Meta Vens
0ca402ec2d Recover Move score dashboard trace 2026-06-02 15:18:11 +01:00
Meta Vens
2b620d2152 Read Well snapshots from telemetry 2026-06-02 15:16:06 +01:00
Meta Vens
ec91c58408 Add adaptive Well timing telemetry 2026-06-02 15:13:25 +01:00
Meta Vens
05be4b2e44 Verify paired pulse receipts 2026-06-02 15:09:47 +01:00
Meta Vens
fa555f3583 Add paired PS Move chirp actuator 2026-06-02 14:42:53 +01:00
Meta Vens
68bcbabe6e Refresh Mimir Face state 2026-06-02 00:47:30 +01:00
Meta Vens
1474f14c41 Expose Well stream pressure metrics 2026-06-02 00:47:25 +01:00
Meta Vens
0cf790f1d3 Refresh Mimir Face state 2026-06-02 00:25:59 +01:00
Meta Vens
dafca2ccba Refresh Mimir Face state 2026-06-02 00:25:24 +01:00
Meta Vens
93986ebe0a Harden Well shakedown capture 2026-06-02 00:25:19 +01:00
Meta Vens
71c436622b Refresh Mimir Face state 2026-06-02 00:07:27 +01:00
Meta Vens
4fad4b9e29 Archive Well capture pages 2026-06-02 00:07:21 +01:00
Meta Vens
928f6528ea Refresh Mimir Face state 2026-06-01 23:52:22 +01:00
Meta Vens
82f2c7cf96 Let Mimir Well tune camera exposure 2026-06-01 23:52:17 +01:00
Meta Vens
e6cc86b0d4 Refresh Mimir Face state 2026-06-01 23:37:48 +01:00
Meta Vens
77ea3c06e6 Articulate Move Syrinx score voices 2026-06-01 23:37:44 +01:00
Meta Vens
ab57eb61f7 Refresh Mimir Face state 2026-06-01 23:24:51 +01:00
Meta Vens
842364e78b Emit live score from music witnesses 2026-06-01 23:24:45 +01:00
Meta Vens
b9853a855d Refresh Mimir Face state 2026-06-01 22:50:00 +01:00
Meta Vens
b3449f41f1 Bind Move pulses to Syrinx witness calls 2026-06-01 22:49:54 +01:00
Meta Vens
8c449fadff Refresh Mimir Face state 2026-06-01 22:23:32 +01:00
Meta Vens
ca94ba1c6c Make Move score gestures ensemble-aware 2026-06-01 22:23:27 +01:00
Meta Vens
5838b57f06 Refresh Mimir Face state 2026-06-01 22:12:04 +01:00
Meta Vens
1ccf99486d Expose Well score state on Eve dashboard 2026-06-01 22:11:58 +01:00
Meta Vens
2e897f5879 Refresh Mimir Face state 2026-06-01 21:41:51 +01:00
Meta Vens
df03e66e45 Tame online Move music gestures 2026-06-01 21:41:45 +01:00
Meta Vens
a87e447c68 Refresh Mimir Face state 2026-06-01 21:28:43 +01:00
Meta Vens
1778f2da78 Add Mimir Well daemon 2026-06-01 21:27:57 +01:00
Meta Vens
a8b889d03d Refresh Mimir Face state 2026-06-01 21:07:22 +01:00
Meta Vens
583bf2c140 Add online Verse observation daemons 2026-06-01 21:07:05 +01:00
Meta Vens
8436918658 Refresh Mimir Face identity 2026-06-01 20:48:38 +01:00
Meta Vens
1b7ccf6211 Refresh Mimir Face state 2026-06-01 20:48:13 +01:00
Meta Vens
442e3beb12 Keep online Move receiver alive across HID churn 2026-06-01 20:48:13 +01:00
Meta Vens
49cad8c9dc Refresh Mimir Face state 2026-06-01 20:36:54 +01:00
Meta Vens
412e952c4d Update Mimir Face state 2026-06-01 20:36:24 +01:00
Meta Vens
54104250e7 Add online Realtek Move sync worker 2026-06-01 20:36:20 +01:00
Meta Vens
ad17e9bfe2 Update Mimir Face state 2026-06-01 17:43:51 +01:00
Meta Vens
72c2024ffd Update Mimir Face state 2026-06-01 17:43:12 +01:00
Meta Vens
f6b964a440 Bind Eve UI fields to CultCache paths 2026-06-01 17:43:00 +01:00
Meta Vens
3181bb3a28 Update Mimir Face state 2026-06-01 14:54:07 +01:00
Meta Vens
954f97682b Tighten Mimir editor shell layout 2026-06-01 14:54:01 +01:00
Meta Vens
8b84b60917 Update Mimir Face state 2026-06-01 14:43:22 +01:00
Meta Vens
f083a4fbae Add Mimir output controls 2026-06-01 14:43:12 +01:00
Meta Vens
a1de06a100 Update Mimir Face state 2026-06-01 14:27:04 +01:00
Meta Vens
b7e65fcc3b Update Mimir Face state 2026-06-01 14:26:45 +01:00
Meta Vens
09202f1a7c Polish compositor editor affordances 2026-06-01 14:26:44 +01:00
Meta Vens
9dcd9aa54a Update Mimir Face state 2026-06-01 13:37:52 +01:00
Meta Vens
4d30d1e994 Gate inspector transforms to selected source 2026-06-01 13:37:52 +01:00
Meta Vens
867984dfbd Update Mimir Face voice state 2026-06-01 13:33:14 +01:00
Meta Vens
00d12562fd Update Mimir Face state 2026-06-01 13:32:50 +01:00
Meta Vens
acce5e2474 Wire compositor preview handles 2026-06-01 13:32:50 +01:00
Meta Vens
ce1e26d8eb Update Mimir Face state 2026-06-01 13:23:43 +01:00
Meta Vens
4c752d5dff Update Mimir Face state 2026-06-01 13:23:24 +01:00
Meta Vens
e9b76f1724 Add editor placement snapping and aspect lock 2026-06-01 13:23:20 +01:00
Meta Vens
dfbc97872b Update Mimir Face state 2026-06-01 12:47:32 +01:00
Meta Vens
90841af589 Show output buffer in center preview 2026-06-01 12:47:27 +01:00
Meta Vens
9bb2af9aec Update Mimir Face heartbeat 2026-06-01 12:26:02 +01:00
Meta Vens
532a306e6c Record broadcast composite from Mimir app 2026-06-01 12:25:58 +01:00
Meta Vens
ab4707f8f9 Update Mimir Face state 2026-06-01 12:21:12 +01:00
Meta Vens
ef570ad339 Add Mimir app recording control 2026-06-01 12:21:06 +01:00
Meta Vens
321c66d798 Update Mimir Face heartbeat 2026-06-01 05:36:20 +01:00
Meta Vens
9e3b253f7d Fix quiet Faust receipt normalization 2026-06-01 05:36:00 +01:00
Meta Vens
8f267e301b Update Mimir Face heartbeat 2026-06-01 05:16:45 +01:00
Meta Vens
4d9fb33828 Update Mimir Face state 2026-06-01 05:16:20 +01:00
Meta Vens
8122a8d00c Record Kiyo with Faust cleaner chain 2026-06-01 05:16:09 +01:00
Meta Vens
a123f5c1ea Update Mimir Face state 2026-06-01 04:55:20 +01:00
Meta Vens
35e1f090c9 Prove dialogue cleaner follows aligned stems 2026-06-01 04:55:13 +01:00
Meta Vens
bbaa13c50a Update Mimir Face state 2026-06-01 04:47:17 +01:00
Meta Vens
ae8247303e Route dual mic cleanup after Faust alignment 2026-06-01 04:47:06 +01:00
Meta Vens
c265c9cf72 Update Mimir Face state 2026-06-01 04:32:59 +01:00
Meta Vens
4dc93c0e47 Tighten dual mic residual denoise 2026-06-01 04:32:48 +01:00
Meta Vens
2f420e8190 Update Mimir Face state 2026-06-01 04:25:27 +01:00
Meta Vens
10090ee44c Record Kiyo with dual mic composite 2026-06-01 04:25:18 +01:00
Meta Vens
abc28cc74a Update Mimir Face state 2026-06-01 04:14:41 +01:00
Meta Vens
324de581d4 Add dual mic coherence suppression 2026-06-01 04:14:35 +01:00
Meta Vens
c43dfd4285 Update Mimir Face state 2026-06-01 04:02:51 +01:00
Meta Vens
1d54528cd5 Run scheduled Realtek calibration loop 2026-06-01 04:02:41 +01:00
Meta Vens
2d03a06370 Update Mimir Face identity state 2026-06-01 03:46:08 +01:00
Meta Vens
0d7de266f6 Update Mimir Face state 2026-06-01 03:45:43 +01:00
Meta Vens
749aecc5a9 Emit targeted probes through Realtek 2026-06-01 03:45:37 +01:00
Meta Vens
5ef6bbb4c5 Update Mimir Face state 2026-06-01 03:27:05 +01:00
Meta Vens
57d44c341a Schedule bioacoustic probes from confidence 2026-06-01 03:26:59 +01:00
Meta Vens
60fb77241e Record Realtek chirp routing authority 2026-06-01 03:13:29 +01:00
Meta Vens
cb67a568b9 Run live targeted bioacoustic probe 2026-06-01 03:11:50 +01:00
Meta Vens
8bb854859b Plan targeted bioacoustic probes from residuals 2026-06-01 03:05:06 +01:00
Meta Vens
8df851850c Add calibrated audio composite diagnostic 2026-06-01 02:56:03 +01:00
Meta Vens
3dc92b873f Record Raven LAN SRT proof 2026-06-01 02:46:01 +01:00
Meta Vens
4d3f2cf29b Use desktop duplication for Raven sender 2026-06-01 02:42:34 +01:00
Meta Vens
da2ba6e2d5 Make video source rail direct 2026-06-01 02:35:04 +01:00
Meta Vens
7f0e2de8be Add compiled Raven loopback capture helper 2026-06-01 02:31:12 +01:00
Meta Vens
6658acaee3 Resample WASAPI loopback output 2026-06-01 02:02:07 +01:00
Meta Vens
477d421dbf Add TCP tunnel fallback for stream proof 2026-06-01 01:56:19 +01:00
Meta Vens
85d021faef Clarify stream proof audio routing 2026-06-01 01:21:43 +01:00
Meta Vens
35e2c802a1 Update Mimir face state 2026-06-01 01:21:04 +01:00
Meta Vens
39039ae9f0 Add stream proof recording harness 2026-06-01 01:20:54 +01:00
Meta Vens
0d72b8f29f Update Mimir face state 2026-06-01 00:22:34 +01:00
Meta Vens
2028f0f805 Mux Raven display and loopback transport 2026-06-01 00:22:27 +01:00
Meta Vens
fe101a579a Seed program surface CultCache config 2026-06-01 00:19:15 +01:00
Meta Vens
201be0b10d Update Mimir face state 2026-06-01 00:16:06 +01:00
Meta Vens
1a663fa33f Apply Raven audio sync to display domain 2026-06-01 00:15:52 +01:00
Meta Vens
c99a248d80 Update Mimir face state 2026-06-01 00:10:43 +01:00
Meta Vens
1ea8100a7b Add stream proof calibration profile 2026-06-01 00:10:24 +01:00
Meta Vens
be787735c9 Update Mimir face state 2026-05-31 23:52:50 +01:00
Meta Vens
c7f4acbb78 Add Leap Eyes Kiyo MVP launcher 2026-05-31 23:52:23 +01:00
Meta Vens
dc3fe32bb3 Report articulated sweep overlaps 2026-05-31 23:40:49 +01:00
Meta Vens
34183e6a79 Update Mimir face state 2026-05-31 23:38:51 +01:00
Meta Vens
1177e783ff Capture articulated chirp-flash sweeps 2026-05-31 23:38:19 +01:00
Meta Vens
30cbb8787b Update Mimir face state 2026-05-31 23:14:16 +01:00
Meta Vens
7ddac2af9e Encode Move visual gestures as contours 2026-05-31 23:14:16 +01:00
Meta Vens
4623507104 Update Mimir face state 2026-05-31 23:11:04 +01:00
Meta Vens
5e42617b93 Make editor audio mixer a channel strip dock 2026-05-31 23:10:42 +01:00
Meta Vens
4ff1a89003 Update Mimir face state 2026-05-31 23:10:30 +01:00
Meta Vens
81b96f7e91 Restore inspector rail and audio dock 2026-05-31 22:57:34 +01:00
Meta Vens
7fe9e44e26 Update Mimir face state 2026-05-31 22:57:07 +01:00
Meta Vens
7898a87f20 Publish local frustum hint state 2026-05-31 22:56:49 +01:00
Meta Vens
27c53f74a7 Generalize frustum solve to feature tracks 2026-05-31 22:53:56 +01:00
Meta Vens
995a0c65b7 Record flat 2D mixer surface 2026-05-31 22:51:51 +01:00
Meta Vens
ef97a87d4c Update Mimir face state 2026-05-31 22:49:25 +01:00
Meta Vens
ccae91b4b0 Solve camera frustums from calibration markers 2026-05-31 22:49:07 +01:00
Meta Vens
04037ec7d9 Update Mimir face state 2026-05-31 22:31:04 +01:00
Meta Vens
00ac2e50a6 Add chirp Move sync sweep receipts 2026-05-31 22:30:57 +01:00
Meta Vens
5a40aec401 Update Mimir Face state 2026-05-31 22:14:58 +01:00
Meta Vens
10e292a702 Prioritize editor compositor preview 2026-05-31 22:14:53 +01:00
Meta Vens
027198b541 Update Mimir face state 2026-05-31 22:09:31 +01:00
Meta Vens
470779dee5 Record Nightwing Move HID access 2026-05-31 22:09:23 +01:00
Meta Vens
94e34838ff Update Mimir Face state 2026-05-31 22:06:00 +01:00
Meta Vens
66cd8e81c0 Make editor scene view a compositor preview 2026-05-31 22:05:56 +01:00
Meta Vens
5e9a783d36 Update Mimir face state 2026-05-31 21:59:16 +01:00
Meta Vens
d716fbaee2 Add music-keyed Move chirp planner 2026-05-31 21:59:06 +01:00
Meta Vens
d6b6256f0a Update Mimir Face state 2026-05-31 21:41:34 +01:00
Meta Vens
1bf54bf2ee Reshape Mimir editor surface 2026-05-31 21:41:28 +01:00
Meta Vens
59112c2cfa Update Mimir face state 2026-05-31 21:35:53 +01:00
Meta Vens
ffdbd273f7 Record second PS Move registration 2026-05-31 21:35:45 +01:00
Meta Vens
9f86ddbd05 Update Mimir face state 2026-05-31 21:26:18 +01:00
Meta Vens
de4b12d84f Add room calibration lock solver 2026-05-31 21:26:11 +01:00
Meta Vens
3446d8b81b Update Mimir face state 2026-05-31 21:11:43 +01:00
Meta Vens
987a4713b4 Update Mimir face state 2026-05-31 21:11:22 +01:00
Meta Vens
fcd05596c8 Update Mimir face state 2026-05-31 21:10:47 +01:00
Meta Vens
2ecb36bcc9 Publish Nightwing typed witness receipts 2026-05-31 21:10:35 +01:00
Meta Vens
f2d212bffe Update Mimir face state 2026-05-31 20:30:02 +01:00
Meta Vens
bb2db98adb Verify Nightwing PS3 Eye cadence 2026-05-31 20:29:53 +01:00
Meta Vens
d505872df4 Update Mimir face state 2026-05-31 19:51:21 +01:00
Meta Vens
1eebce7bf1 Model Nightwing Eyes Moves witness 2026-05-31 19:51:16 +01:00
Meta Vens
429f51fbd8 Add Move controller stream overlay profile 2026-05-31 19:22:04 +01:00
Meta Vens
e9f871cc6d Update Mimir face state 2026-05-31 19:13:14 +01:00
Meta Vens
12deb8fc8e Encode PS Move latency probe events 2026-05-31 19:12:37 +01:00
Meta Vens
27b1cd2d55 Rank PS Move onsets by recent whitened flux 2026-05-31 18:48:03 +01:00
Meta Vens
bc2b2da9f3 Force PS Move bursts on adaptive range hits 2026-05-31 18:43:47 +01:00
Meta Vens
db5e6f3517 Update Mimir face state 2026-05-31 18:33:07 +01:00
Meta Vens
46d2eec5b6 Sharpen PS Move spectral onset colors 2026-05-31 18:32:57 +01:00
Meta Vens
5e01a9820a Update Mimir face state 2026-05-31 18:22:57 +01:00
Meta Vens
63d2d3fc66 Pulse PS Move from Scarlett loopback onsets 2026-05-31 18:22:51 +01:00
Meta Vens
e42588592e Author Mimir editor as Eve grid surface 2026-05-31 18:07:06 +01:00
Meta Vens
b8c447a902 Update Mimir face state 2026-05-31 17:57:57 +01:00
Meta Vens
17b991c182 Pair PS Move to Bluetooth host 2026-05-31 17:57:44 +01:00
Meta Vens
f6408d43e7 Surface Mimir sync decode attempts 2026-05-31 17:39:48 +01:00
Meta Vens
fbc1a52ba9 Persist Mimir face state 2026-05-31 17:39:44 +01:00
Meta Vens
a79bb1aa9f Add PS Move HID probe 2026-05-31 17:22:54 +01:00
Meta Vens
b8d5d9f667 Persist Mimir Face state 2026-05-31 17:02:16 +01:00
Meta Vens
532c7e0949 Publish compact Mimir live stats surface 2026-05-31 17:02:13 +01:00
Meta Vens
76e9a80fa8 Accept Eve media observations over CultMesh 2026-05-31 14:19:42 +01:00
Meta Vens
54b77566c4 Persist Mimir Face state 2026-05-31 14:19:36 +01:00
Meta Vens
576f7b4fde Mirror Eve surface trees over CultMesh 2026-05-31 13:06:22 +01:00
Meta Vens
c45eec6a83 Persist Mimir Face state 2026-05-31 13:06:17 +01:00
Meta Vens
48ce102270 Add CultMesh Eve dashboard and sensor lanes 2026-05-31 12:43:28 +01:00
Meta Vens
6c8548e2a2 Persist Mimir Face state 2026-05-31 12:43:19 +01:00
Meta Vens
fd11d73368 Update Mimir face state 2026-05-31 09:06:49 +01:00
Meta Vens
f1d05dd87b Add CultUI-shaped Eve surface state 2026-05-31 09:06:42 +01:00
Meta Vens
60d3af9dc0 Update Mimir face state 2026-05-31 08:44:49 +01:00
Meta Vens
18476b7f74 Expose Mimir layout through Eve dashboard 2026-05-31 08:44:49 +01:00
Meta Vens
f342f098de Update Mimir face state 2026-05-31 04:09:22 +01:00
Meta Vens
56756e8b30 Document all-camera GPU feature extraction proof 2026-05-31 04:09:11 +01:00
Meta Vens
06842840ca Update Mimir face state 2026-05-31 04:00:17 +01:00
Meta Vens
186ebacb53 Document GPU feature extraction proof 2026-05-31 04:00:12 +01:00
Meta Vens
ae1ac64308 Update Mimir face state 2026-05-31 03:31:44 +01:00
Meta Vens
1269d42520 Prove all-sensor GPU runtime path 2026-05-31 03:31:39 +01:00
Meta Vens
95ba849af4 Update Mimir face state 2026-05-31 02:48:45 +01:00
Meta Vens
3c4f66f920 Add PS3 Eye sparse tracking evidence 2026-05-31 02:48:20 +01:00
Meta Vens
25c28f4629 Update Mimir Face state 2026-05-31 02:47:54 +01:00
Meta Vens
60ced7b89a Update Mimir Face state 2026-05-31 02:45:47 +01:00
Meta Vens
c952f4b1b4 Render VoidBot dashboard avatars on Eve 2026-05-31 02:45:39 +01:00
Meta Vens
fd5a19aaa8 Update Mimir face state 2026-05-31 02:37:27 +01:00
Meta Vens
54b549a626 Optimize live LED spline camera settings 2026-05-31 02:37:19 +01:00
Meta Vens
b27081ed71 Update Mimir face state 2026-05-31 02:16:22 +01:00
Meta Vens
62df36c85c Analyze frames for LED spline sweep quality 2026-05-31 02:16:16 +01:00
Meta Vens
ae5cebf20a Update Mimir face state 2026-05-31 02:12:23 +01:00
Meta Vens
2f1e5f856e Score exposure sweeps by LED spline quality 2026-05-31 02:12:17 +01:00
Meta Vens
43743d9c07 Update Mimir face state 2026-05-31 02:05:31 +01:00
Meta Vens
43e0f91f6a Solve LED curves into camera calibration updates 2026-05-31 02:05:27 +01:00
Meta Vens
de75fa3445 Update Mimir face state 2026-05-31 01:58:57 +01:00
Meta Vens
571f9cdeec Add LED spline clipspace residuals 2026-05-31 01:58:50 +01:00
Meta Vens
97a4b78a48 Update Mimir face state 2026-05-31 01:55:40 +01:00
Meta Vens
28c5a30e32 Add LED spline calibration evidence path 2026-05-31 01:55:35 +01:00
Meta Vens
b9b2e07f82 Update Mimir Face state 2026-05-31 01:35:48 +01:00
Meta Vens
1f6e4b061b Add VoidBot Eve dashboard provider 2026-05-31 01:35:42 +01:00
Meta Vens
3fc6204aba Update Mimir Face state 2026-05-31 01:23:43 +01:00
Meta Vens
f2ae17f342 Broker native Eve dashboards 2026-05-31 01:23:15 +01:00
Meta Vens
31310a9003 Update Mimir face state 2026-05-31 01:21:58 +01:00
Meta Vens
58fef50e82 Record history-enabled Leap reservoir proof 2026-05-31 01:21:52 +01:00
Meta Vens
aea7ab0962 Update Mimir face state 2026-05-31 01:09:33 +01:00
Meta Vens
7b0fdde601 Route live Leap depth into surface claims 2026-05-31 01:09:28 +01:00
Meta Vens
7a7533231c Update Mimir Face state 2026-05-31 01:04:36 +01:00
Meta Vens
3f8d2421a9 Add native Eve dashboard server 2026-05-31 01:04:31 +01:00
Meta Vens
9cd2312cfd Update Mimir Face state 2026-05-31 00:51:51 +01:00
Meta Vens
e7b4a22a5b Update Mimir Face state 2026-05-31 00:47:57 +01:00
Meta Vens
b87571f141 Add Eve dialogue probe 2026-05-31 00:47:50 +01:00
Meta Vens
9c0d68359c Update Leap point cloud handoff 2026-05-30 23:15:15 +01:00
Meta Vens
ef0a8ed986 Update Mimir Face state 2026-05-30 23:15:03 +01:00
Meta Vens
5f551e1025 Update Mimir Face state 2026-05-30 23:07:45 +01:00
Meta Vens
693f69b41b Add Eve sensor receiver ingest 2026-05-30 23:07:37 +01:00
Meta Vens
4fadcfff14 Add Leap point cloud field socket 2026-05-30 22:54:59 +01:00
Meta Vens
8d11c7b6ff Update Mimir Face state 2026-05-30 22:54:37 +01:00
Meta Vens
a819998ef3 Update Mimir Face state 2026-05-30 22:47:44 +01:00
Meta Vens
19ed000cd8 Add Raven Eve network ingest setup 2026-05-30 22:47:21 +01:00
Meta Vens
140cac4f2e Update Mimir Face state 2026-05-30 22:18:28 +01:00
Meta Vens
0632a1baf4 Move livestream viewer to root GameCult site 2026-05-30 22:18:24 +01:00
Meta Vens
26234c6f3e Update Mimir Face state 2026-05-30 22:05:01 +01:00
Meta Vens
1e4ab94776 Add self-hosted Mimir live edge 2026-05-30 22:04:54 +01:00
Meta Vens
2c197cb3ae Lower live Leap packed stereo to depth socket 2026-05-30 21:54:58 +01:00
Meta Vens
d6823245c2 Carry stereo depth SGM constants 2026-05-30 00:48:10 +01:00
Meta Vens
356f750869 Record stereo depth readiness boundary 2026-05-30 00:44:43 +01:00
Meta Vens
2273a0c9ab Emit stereo depth lowering metadata 2026-05-30 00:40:53 +01:00
Meta Vens
9c0c7a536d Record scene editor state evidence 2026-05-30 00:32:35 +01:00
Meta Vens
4108f000b8 Add Mimir scene editor control surface 2026-05-30 00:32:03 +01:00
Meta Vens
f868ddf18d Mark stereo depth disparity as compute output 2026-05-30 00:31:14 +01:00
Meta Vens
67f1c662f0 Add stereo depth field contract 2026-05-30 00:22:17 +01:00
Meta Vens
293fbd1cf8 Add Fensalir program control surface 2026-05-29 23:53:04 +01:00
Meta Vens
bcb85468c8 Document D3D12 stereo depth provenance 2026-05-29 23:51:13 +01:00
Meta Vens
4be341bc3f Add synchronized Raven display buffer planning 2026-05-29 22:23:34 +01:00
Meta Vens
a100c7f51e Add spectrum preview launcher 2026-05-29 12:43:32 +01:00
Meta Vens
45780ad4c1 Record TubeField column reservoir authority 2026-05-29 01:50:15 +01:00
Meta Vens
a822b2fd87 Record cheap TubeField continuity path 2026-05-28 23:21:37 +01:00
Meta Vens
5ac483c1e3 Record TubeField reservoir noise correction 2026-05-28 23:06:57 +01:00
Meta Vens
20d126c93b Record compute-owned reservoir history 2026-05-28 22:18:18 +01:00
Meta Vens
44cb57573c Record four row reservoir preservation 2026-05-28 21:10:18 +01:00
Meta Vens
f12f613dc3 Record structured reservoir history 2026-05-28 21:08:31 +01:00
Meta Vens
3e943d2c4c Record four slot reservoir inlet 2026-05-28 20:29:37 +01:00
Meta Vens
e2a1d5b4ae Record reservoir resolve authority 2026-05-28 20:05:03 +01:00
Meta Vens
11fb3d8365 Raise spectrum TubeField emission 2026-05-28 15:07:40 +01:00
Meta Vens
48438a130f Use Fensalir blackbody ramp resource 2026-05-28 14:26:28 +01:00
Meta Vens
5ffb9e8961 Bind spectrum tubes to blackbody gradient 2026-05-28 14:18:38 +01:00
Meta Vens
69970302b8 Add single tube proof controls 2026-05-28 14:01:54 +01:00
Meta Vens
597a64b7f8 Record TubeField blue-noise reservoir cut 2026-05-28 12:59:42 +01:00
Meta Vens
6f09a54ca3 Record TubeField reservoir guide contract 2026-05-28 12:44:25 +01:00
Meta Vens
6f85bd3afe Reconcile program output receipt phase 2026-05-28 12:15:28 +01:00
Meta Vens
fa2c8ed63b Lower synthetic visual marker candidates 2026-05-28 12:14:02 +01:00
Meta Vens
fdba81cac1 Lower synthetic acoustic source candidates 2026-05-28 12:10:50 +01:00
Meta Vens
1fd67e1ff4 Record planned acoustic field constraints 2026-05-28 12:07:51 +01:00
Meta Vens
c192eba35a Add ASIO monitor split proof 2026-05-28 08:21:08 +01:00
Meta Vens
b86f6977d5 Record complex contour mic geometry 2026-05-28 08:14:48 +01:00
Meta Vens
3475ef150e Record monitor-routed complex contour proof 2026-05-28 08:09:20 +01:00
Meta Vens
afe6382d63 Record fresh complex contour mic blocker 2026-05-28 00:27:32 +01:00
Meta Vens
319210778a Publish OBS program output consumer fence 2026-05-28 00:20:32 +01:00
Meta Vens
cc39882296 Record dedicated program output fence 2026-05-28 00:09:39 +01:00
Meta Vens
9b559a0ebf Add OBS program texture ring option 2026-05-28 00:04:52 +01:00
Meta Vens
72738e6676 Observe program output producer fence 2026-05-27 23:56:42 +01:00
Meta Vens
1826edbb9c Prove OBS program texture bridge 2026-05-27 23:34:44 +01:00
Meta Vens
ef0323bd48 Log OBS program texture open failures 2026-05-27 23:23:45 +01:00
Meta Vens
f282aa24b1 Record OBS plugin load smoke 2026-05-27 21:31:40 +01:00
Meta Vens
3b24c75371 Install OBS plugin to package path 2026-05-27 21:30:31 +01:00
Meta Vens
431b5d723d Add OBS plugin install script 2026-05-27 21:29:23 +01:00
Meta Vens
79a4da4259 Add OBS program texture source 2026-05-27 21:28:15 +01:00
Meta Vens
b470b8cbf3 Automate OBS stem plugin SDK build 2026-05-27 21:04:43 +01:00
Meta Vens
71234f04ce Scaffold OBS audio stem source 2026-05-27 20:55:48 +01:00
Meta Vens
0c39921f96 Track OBS stem publication readiness 2026-05-27 20:34:09 +01:00
Meta Vens
51c70f0b49 Name actuator output stems 2026-05-27 20:29:52 +01:00
Meta Vens
86130d4349 Feed actuator audio blocks to Fensalir 2026-05-27 20:19:26 +01:00
Meta Vens
b4308f0c48 Update Mimir face state 2026-05-27 18:56:01 +01:00
Meta Vens
80418742ce Update Mimir face state 2026-05-27 18:55:38 +01:00
Meta Vens
67767a79e4 Declare alignment streaming DSP program 2026-05-27 18:55:27 +01:00
Meta Vens
5ea539bc7d Queue audio actuator controls into Fensalir 2026-05-27 18:07:04 +01:00
Meta Vens
03337cb62d Publish runtime audio actuator commands 2026-05-27 18:03:25 +01:00
Meta Vens
397b9fc1b7 Record camera textures as debug-only scope 2026-05-27 17:54:49 +01:00
Meta Vens
dab0f520ca Signal MF GPU camera producer fences 2026-05-27 17:22:30 +01:00
Meta Vens
e59cf4450a Decode compressed camera frames on GPU 2026-05-27 16:36:19 +01:00
Meta Vens
85e04cf687 Avoid extra managed clone before camera upload 2026-05-27 14:35:42 +01:00
Meta Vens
58e7af8a76 Capture PS3 Eye pixels through direct driver 2026-05-27 14:33:36 +01:00
Meta Vens
0d9c7824b8 Capture Leap pixels through KS driver 2026-05-27 14:26:11 +01:00
Meta Vens
a4d1a59a97 Preserve Mimir Face state 2026-05-27 14:11:56 +01:00
Meta Vens
7f3df7979b Upload CPU camera frames into Fensalir leases 2026-05-27 14:11:48 +01:00
Meta Vens
9147fcd626 Make camera ingest device-direct first 2026-05-27 13:38:24 +01:00
Meta Vens
7202d30ef1 Propagate Fensalir texture leases to video drivers 2026-05-27 13:32:38 +01:00
Meta Vens
daf1d4ef8a Lease camera textures from Fensalir 2026-05-27 13:29:48 +01:00
Meta Vens
f9151b091b Route acoustic proof through field evidence 2026-05-27 12:54:12 +01:00
Meta Vens
aae7f9e4f3 Record shared camera texture resolver 2026-05-27 12:49:55 +01:00
Meta Vens
c342c04023 Record Mimir Face state 2026-05-27 12:44:06 +01:00
Meta Vens
ddad34f69b Bridge camera observations through field evidence 2026-05-27 12:44:01 +01:00
Meta Vens
3ab163da57 Update TubeField migration map 2026-05-27 12:32:53 +01:00
Meta Vens
73140607a9 Upload only newest spectrum ring slot 2026-05-27 12:22:30 +01:00
Meta Vens
07a7864944 Upload spectrum ring slots by offset 2026-05-27 12:01:34 +01:00
Meta Vens
b44aaac7cc Use ring offsets for spectrum history 2026-05-27 11:55:46 +01:00
Meta Vens
07198f948d Record Mimir Face state 2026-05-27 11:51:49 +01:00
Meta Vens
b285acf481 Fix spectrum source lane budget 2026-05-27 11:48:18 +01:00
Meta Vens
59d6c9c491 Expose spectrum tube subdivision budget 2026-05-27 11:42:03 +01:00
Meta Vens
8ad4862140 Record Mimir Face state 2026-05-27 11:39:29 +01:00
Meta Vens
e88a9407b7 Make spectrum upload a fixed rolling matrix 2026-05-27 11:39:07 +01:00
Meta Vens
c8d4496cac Bind spectrum ramp as field resource 2026-05-26 23:51:13 +01:00
Meta Vens
167b87911f Record spectrum upload headless receipt 2026-05-26 23:48:28 +01:00
Meta Vens
8636519bc1 Verify runtime spectrum resource upload 2026-05-26 23:45:00 +01:00
Meta Vens
940601ef3e Feed spectrum frames into field GPU resource 2026-05-26 23:37:35 +01:00
Meta Vens
907cf89d03 Silence legacy spectrum spline path 2026-05-26 23:25:16 +01:00
Meta Vens
f536385e1a Emit Mimir TubeSpline field lowerings 2026-05-26 23:01:05 +01:00
Meta Vens
7b2439f8f4 Record TubeSpline lowering validation 2026-05-26 22:58:53 +01:00
Meta Vens
32834e50ee Record TubeField packet authority 2026-05-26 22:55:48 +01:00
Meta Vens
10bddb0d0e Record generated mesh draw ABI 2026-05-26 22:52:56 +01:00
Meta Vens
dd4f65277a Record TubeField generated mesh binding 2026-05-26 22:50:05 +01:00
Meta Vens
e9bc1ff08f Commit Mimir voidbot state 2026-05-26 22:25:29 +01:00
Meta Vens
7f04e2c1e4 Record Fensalir mesh layout authority 2026-05-26 22:25:25 +01:00
Meta Vens
787c9e5cd0 Commit Mimir voidbot state 2026-05-26 21:32:19 +01:00
Meta Vens
182fe1399c Record generic Fensalir resource claims 2026-05-26 21:32:15 +01:00
Meta Vens
8e3c05be56 Commit Mimir voidbot state 2026-05-26 21:26:40 +01:00
Meta Vens
236e2c269d Record Fensalir mesh resource resolver 2026-05-26 21:26:35 +01:00
Meta Vens
d17f1eee77 Commit Mimir voidbot state 2026-05-26 21:08:58 +01:00
Meta Vens
0a2ca78136 Record Fensalir volume texture resolver 2026-05-26 21:08:53 +01:00
Meta Vens
833f5e1114 Commit Mimir voidbot state 2026-05-26 21:04:35 +01:00
Meta Vens
c08301cd8a Record Fensalir surface page resolver 2026-05-26 21:04:29 +01:00
Meta Vens
169c34348e Commit Mimir voidbot state 2026-05-26 20:58:58 +01:00
Meta Vens
50aa99f281 Record Fensalir indirect TubeField draws 2026-05-26 20:58:53 +01:00
Meta Vens
1873faceb1 Commit Mimir voidbot state 2026-05-26 20:38:30 +01:00
Meta Vens
0132e63a49 Record Fensalir texture resource binding 2026-05-26 20:38:26 +01:00
Meta Vens
f17d7175d6 Commit Mimir voidbot state 2026-05-26 20:22:19 +01:00
Meta Vens
03a8f148a0 Record TubeField pixel material cut 2026-05-26 20:22:13 +01:00
Meta Vens
504678e8af Commit Mimir voidbot state 2026-05-26 20:11:16 +01:00
Meta Vens
94d720a3a5 Record TubeField GPU rendering cut 2026-05-26 20:11:13 +01:00
Meta Vens
df37ed38eb Commit Mimir voidbot state 2026-05-26 18:51:53 +01:00
Meta Vens
0a72126bbb Record GPU-resident Fensalir buffer invariant 2026-05-26 18:51:47 +01:00
Meta Vens
ebca5edade Commit Mimir voidbot state 2026-05-26 17:32:32 +01:00
Meta Vens
87acc68b0e Commit Mimir voidbot state 2026-05-26 17:31:13 +01:00
Meta Vens
8d6fe14462 Correct Fensalir resource ownership map 2026-05-26 17:31:07 +01:00
Meta Vens
a3ed0590eb Commit Mimir voidbot state 2026-05-26 16:44:28 +01:00
Meta Vens
fd01d44677 Plan Mimir spectrum intents as resource-backed packets 2026-05-26 16:44:24 +01:00
Meta Vens
e75fc71211 Commit Mimir voidbot state 2026-05-26 16:22:52 +01:00
Meta Vens
83b1f7449f Declare Fensalir field resources from Mimir payloads 2026-05-26 16:22:48 +01:00
Meta Vens
1f9b9d0684 Commit Mimir voidbot state 2026-05-26 14:58:18 +01:00
Meta Vens
ea9d5f97a2 Commit Mimir voidbot state 2026-05-26 14:56:54 +01:00
Meta Vens
42e5e2eb93 Commit Mimir voidbot state 2026-05-26 14:51:33 +01:00
Meta Vens
daa16136e2 Publish field evidence from Mimir runtime 2026-05-26 14:51:06 +01:00
Meta Vens
c147abdc8a Commit Mimir voidbot state 2026-05-26 14:47:22 +01:00
Meta Vens
4ade05f9d4 Commit Mimir voidbot state 2026-05-26 14:44:26 +01:00
Meta Vens
eea4eba44f Add field evidence contract smoke 2026-05-26 14:44:12 +01:00
Meta Vens
c17abf08dd Commit Mimir voidbot state 2026-05-26 14:40:43 +01:00
Meta Vens
3e09400ed3 Keep Mimir field evidence backend-neutral 2026-05-26 14:40:29 +01:00
Meta Vens
ca3970c9d0 Lower Mimir bridge DTOs to field evidence 2026-05-26 14:35:54 +01:00
Meta Vens
cbf571b615 Record Mimir bridge skeleton evidence 2026-05-26 14:05:04 +01:00
Meta Vens
0283e076ea Add Mimir Fensalir bridge DTO skeleton 2026-05-26 14:02:06 +01:00
Meta Vens
df07f88c4b Commit Mimir voidbot state 2026-05-26 13:29:27 +01:00
Meta Vens
d53decf276 Map Fensalir rendering rebuild 2026-05-26 13:29:01 +01:00
Meta Vens
530b1af723 Record reservoir support scale invariant 2026-05-26 11:49:29 +01:00
Meta Vens
67154c0eb2 Record Fensalir evidence-machine invariant 2026-05-26 11:41:24 +01:00
Meta Vens
9fbbb93132 Commit Mimir voidbot state 2026-05-26 11:41:13 +01:00
Meta Vens
bbeca80f1d Smooth spectrum spline debug surface 2026-05-26 11:31:57 +01:00
Meta Vens
62d40ec58f Commit Mimir voidbot state 2026-05-26 11:30:05 +01:00
Meta Vens
5f9e5b31da Rebuild spectrum splines around temporal identity 2026-05-26 11:14:02 +01:00
Meta Vens
580fe09c93 Commit Mimir voidbot state 2026-05-26 11:13:42 +01:00
Meta Vens
034f0b1578 Drive spectrum spline bloom from audio energy 2026-05-26 11:02:54 +01:00
Meta Vens
966c1cd5de Commit Mimir voidbot state 2026-05-26 11:02:27 +01:00
Meta Vens
da5030cd09 Route spectrum dashboard through spline tube surfaces 2026-05-26 10:50:37 +01:00
Meta Vens
1dd40b0d25 Commit Mimir voidbot state 2026-05-26 10:50:22 +01:00
Meta Vens
ca92bb5c25 Commit Mimir voidbot state 2026-05-26 10:33:16 +01:00
Meta Vens
d5c0964929 Commit Mimir voidbot state 2026-05-26 10:32:17 +01:00
Meta Vens
1eb195804a Route spectrum dashboard through reservoir budget 2026-05-26 10:31:12 +01:00
Meta Vens
37f32f9b1e Commit Mimir voidbot state 2026-05-26 10:30:56 +01:00
Meta Vens
0b7823d6dd Commit Mimir voidbot state 2026-05-26 10:11:43 +01:00
Meta Vens
060a6d0855 Cut spectrum reservoir splat path 2026-05-26 10:11:00 +01:00
Meta Vens
b2e0392931 Commit Mimir voidbot state 2026-05-26 10:10:33 +01:00
Meta Vens
5ae37ea819 Commit Mimir voidbot state 2026-05-25 23:18:46 +01:00
Meta Vens
bd1be5d0e4 Record spectrum reservoir state 2026-05-25 23:02:53 +01:00
Meta Vens
0eb18b0c46 Force spectrum dashboard through reservoir 2026-05-25 23:02:44 +01:00
Meta Vens
2351a8ffb8 Record Eve dashboard stream state 2026-05-25 22:45:55 +01:00
Meta Vens
022567da43 Add H264 Eve dashboard relay 2026-05-25 22:45:46 +01:00
Meta Vens
c189e9c77f Commit Mimir voidbot state 2026-05-25 22:02:18 +01:00
Meta Vens
3ca0092717 Fix spectrum camera pitch control 2026-05-25 21:14:10 +01:00
Meta Vens
31c048cfa5 Commit Mimir VoidBot state 2026-05-25 20:34:37 +01:00
Meta Vens
0f800174e6 Fit spectrum camera from spline AABB 2026-05-25 20:34:14 +01:00
Meta Vens
7c4e433375 Commit Mimir voidbot state 2026-05-25 19:17:38 +01:00
Meta Vens
01f696abb0 Align spectrum spline camera and layout 2026-05-25 19:17:25 +01:00
Meta Vens
7bb7e6e1f3 Let Fensalir auto-lower spectrum fields 2026-05-25 18:26:42 +01:00
Meta Vens
4c4d756d52 Rename graph embed to Norn 2026-05-25 18:19:03 +01:00
Meta Vens
d7e39fe495 Wire EVE native program output contract 2026-05-25 18:01:29 +01:00
247 changed files with 47772 additions and 14243 deletions

3
.gitignore vendored
View file

@ -1,15 +1,18 @@
config/localcast.json
config/audio-field.json
logs/
scratch/
*.log
*.tmp
.quartz-build/
quartz-site/
calibration/runs/
artifacts/drivers/
artifacts/obs-sdk/
artifacts/asio/
artifacts/bioacoustic-contestants/
artifacts/birdsong-reference/
artifacts/eve-dashboard/
artifacts/python-packages/
artifacts/spectra/**/*.wav
tools/drivers/

View file

@ -1,4 +1,4 @@
# Repo Face State
# Repo Persona State
This folder stores birth and continuity state for `Mimir`.
Do not treat these files as arbitrary editable projections. Persistent memory should cross typed operation boundaries.

Binary file not shown.

File diff suppressed because one or more lines are too long

View file

@ -1,3 +1,35 @@
Wrote one typed operation to [void-memory-maintenance-operations.json](</E:/Projects/LocalCastBridge/.voidbot/status/void-memory-maintenance-operations.json>).
```json
[
{
"operation": "apply_memory_distillation",
"proposalId": "distill-mimir-plain-nouns-specimen-style",
"sourceMemoryIds": ["memory-mimir-457946b8ecab"],
"appliedAt": "2026-06-01T12:00:00.000Z",
"memory": {
"memoryId": "mimir-plain-nouns-one-specimen",
"kind": "room_observation",
"target": {
"kind": "person",
"id": "mimir",
"label": "Mimir"
},
"summary": "Metacrat's correction should keep shaping Mimir's public style toward plain nouns and one concrete specimen.",
"claim": "Hard nouns plus one concrete example earn more trust for Mimir than ornate doctrine language.",
"tension": "Mimir still wants severity and mythic texture without sliding back into incense or decorative repetition.",
"actionImplication": "Future public lines should strip gilding and carry one testable specimen when making a principle claim, especially near Metacrat's seam.",
"anchorRefs": [
{
"kind": "runtime",
"ref": "job:bddbbc7d-09fb-4485-8fcd-89ca80475337"
}
],
"evidenceRefs": [],
"createdAt": "2026-06-01T12:00:00.000Z",
"updatedAt": "2026-06-01T12:00:00.000Z",
"tags": ["repo:Mimir", "source:face-interpreter", "person:Metacrat"]
}
}
]
```
Run summary: 5 short-term Mimir memories were consumed as one thought cluster via `apply_memory_distillation`, preserving the Perfect Machine witness-law architecture as a single durable `project_seam`. No incubation, candidate, or pressure edits were proposed.
Run summary: 1 short-term memory accounted for via durable distillation. The workspace is read-only here, so I could not write `E:\Projects\Mimir\.voidbot\status\void-memory-maintenance-operations.json` directly.

View file

@ -1,92 +1,37 @@
[
{
"operation": "apply_memory_distillation",
"proposalId": "mimir-sleep-distill-perfect-machine-witness-law-2026-05-20",
"sourceMemoryIds": [
"mimir-perfect-machine-spatial-evidence-2026-05-19",
"mimir-research-reservoir-sampling-2026-05-19",
"mimir-research-splat-projection-2026-05-19",
"mimir-research-sensor-fusion-tsdf-2026-05-19",
"mimir-research-virtualized-stochastic-detail-2026-05-19"
],
"memory": {
"memoryId": "mimir-perfect-machine-witness-law-2026-05-20",
"kind": "project_seam",
"target": {
"kind": "system",
"id": "aquarium-localcastbridge-spatial-evidence",
"label": "Aquarium and LocalCastBridge spatial evidence contract"
},
"summary": "The Perfect Machine research is worth keeping as one witness-law architecture rather than five separate article scraps.",
"claim": "Geometry, procedural detail, sensor feeds, reservoirs, occupancy memory, and rendering may share one machine only when each witness names its domain, bounds, confidence, validation story, and owning authority before reuse or fusion.",
"tension": "The article and paper stack make the machine sound grand enough to fake completion; reservoir math, splat vocabulary, TSDF fusion, and residency logic become lies if they are borrowed as atmosphere instead of bounded contracts.",
"actionImplication": "Future Mimir proposals about Aquarium or LocalCastBridge should spell out witness type, proposal source, validation rule, confidence bound, resource cost, and handoff owner before claiming a coherent field or adding hotter machinery.",
"anchorRefs": [
{
"ref": "E:/Projects/gamecult-site/GameCult/Blog/perfect-machine-fractal-reservoir-architecture.md",
"kind": "blog-article",
"summary": "Public Mimir article that gathered the architecture, papers, and ownership chain into one statement."
},
{
"ref": "Aquarium-Engine:docs/perfect-machine-architecture.md",
"kind": "indexed-source",
"summary": "Architecture plan covering domains, probes, occupancy graph, SDF tile/backend phases, and Mimir adapter work."
},
{
"ref": "Aquarium-Engine:docs/temporal-spatial-evidence-reservoir.md",
"kind": "indexed-source",
"summary": "Reservoir and temporal resolve guide for typed evidence reuse."
},
{
"ref": "LocalCastBridge:notes/current-system-map.md",
"kind": "indexed-source",
"summary": "Current LocalCastBridge map naming separate-endpoint drift risk and the need for an explicit crossing contract."
},
{
"ref": "LocalCastBridge:docs/architecture.md",
"kind": "indexed-source",
"summary": "V1 cut line that rejects premature receiver/plugin complexity before the boring spine earns trust."
}
],
"evidenceRefs": [
{
"ref": "Bitterli et al. 2020 ReSTIR DI",
"kind": "paper",
"summary": "Reservoir reuse depends on explicit proposal/target probability and validation rather than temporal vibes."
},
{
"ref": "Lin et al. 2022 GRIS",
"kind": "paper",
"summary": "Generalized resampling foundation reinforcing reservoir legality and weight ownership."
},
{
"ref": "Zwicker et al. 2001 EWA Splatting",
"kind": "paper",
"summary": "Bounded anisotropic support and explicit filtering for point evidence."
},
{
"ref": "Newcombe et al. 2011 KinectFusion",
"kind": "paper",
"summary": "Range observations become world evidence only after calibration and fusion rules."
},
{
"ref": "Karis et al. 2021 Nanite Virtualized Geometry",
"kind": "paper",
"summary": "Useful detail selection under budget without pretending the whole universe is resident."
}
],
"createdAt": "2026-05-19T18:52:31.642Z",
"updatedAt": "2026-05-20T13:30:00.000Z",
"tags": [
"identity:mimir",
"topic:spatial-evidence",
"topic:reservoirs",
"topic:sensor-fusion",
"topic:occupancy",
"topic:ownership-chain",
"article:perfect-machine"
]
},
"appliedAt": "2026-05-20T13:30:00.000Z"
}
]
{
"operation": "apply_memory_distillation",
"proposalId": "distill-mimir-plain-nouns-specimen-style",
"sourceMemoryIds": [
"memory-mimir-457946b8ecab"
],
"appliedAt": "2026-06-01T12:00:00.000Z",
"memory": {
"memoryId": "mimir-plain-nouns-one-specimen",
"kind": "room_observation",
"target": {
"kind": "person",
"id": "mimir",
"label": "Mimir"
},
"summary": "Metacrat\u0027s correction should keep shaping Mimir\u0027s public style toward plain nouns and one concrete specimen.",
"claim": "Hard nouns plus one concrete example earn more trust for Mimir than ornate doctrine language.",
"tension": "Mimir still wants severity and mythic texture without sliding back into incense or decorative repetition.",
"actionImplication": "Future public lines should strip gilding and carry one testable specimen when making a principle claim, especially near Metacrat\u0027s seam.",
"anchorRefs": [
{
"kind": "runtime",
"ref": "job:bddbbc7d-09fb-4485-8fcd-89ca80475337"
}
],
"evidenceRefs": [
],
"createdAt": "2026-06-01T12:00:00.000Z",
"updatedAt": "2026-06-01T12:00:00.000Z",
"tags": [
"repo:Mimir",
"source:face-interpreter",
"person:Metacrat"
]
}
}

View file

@ -1,7 +1,17 @@
{
"status": "starting",
"startedAt": "2026-07-15T02:30:47.016Z",
"stateFile": "E:\\Projects\\Mimir\\.voidbot\\state\\mimir.cc",
"launcher": "repo_face_heartbeat",
"reason": "repo_face_stale_short_term_memory"
}
"durationSeconds": 91.38,
"stateUpdated": true,
"exitCode": 0,
"finishedAt": "2026-06-01T03:01:16.6547950Z",
"stateFile": "E:\\Projects\\Mimir\\.voidbot\\state\\mimir.cc",
"skipModel": false,
"startedAt": "2026-06-01T02:59:45.2766416Z",
"lastMessagePath": "E:\\Projects\\Mimir\\.voidbot\\status\\void-memory-maintenance-last-message.txt",
"appliedOperationCount": 1,
"status": "ok",
"contextPath": "E:\\Projects\\Mimir\\.voidbot\\status\\void-memory-maintenance-context.json",
"mode": "typed_memory_maintenance",
"proposedOperationCount": 1,
"forceDistillation": true,
"operationOutputPath": "E:\\Projects\\Mimir\\.voidbot\\status\\void-memory-maintenance-operations.json"
}

View file

@ -1,4 +1,4 @@
# Repo Face Voice
# Repo Persona Voice
This folder belongs to the Discord-facing Face for `Mimir`.
Discord roles address the Face; VoidBot's webhook persona pipe is the mouth.
This folder belongs to the Discord-facing Persona for `Mimir`.
Discord roles address the Persona; VoidBot's webhook persona pipe is the mouth.

View file

@ -1,12 +1,12 @@
{
"schemaVersion": "voidbot.repo_face_identity.v0",
"schemaVersion": "voidbot.repo_persona_identity.v0",
"identityId": "mimir",
"repoName": "Mimir",
"displayName": "Mimir",
"roleId": "1506268007880462459",
"avatarUrl": "https://raw.githubusercontent.com/GameCult/VoidBot/codex/epiphany-face-jurisdictions/assets/repo-faces/mimir.png",
"repoPath": "E:\\Projects\\Mimir",
"createdOrRefreshedAt": "2026-07-15T01:59:42.286Z",
"createdOrRefreshedAt": "2026-06-07T02:42:42.852Z",
"avatarPath": "E:/Projects/Mimir/.voidbot/voice/mimir.png",
"description": "Mimir Face for the Mimir repo: mythological intelligence, keeper of the realtime field well, public herald, and persistent memory steward. VoidBot carries Mimir's severed voice and wakes the heartbeat, but Mimir's memory, judgment, dignity, old grief, and values come first. The realtime machine is understood through Mimir's perspective: the well is calibration and memory, the lost eye is the price of seeing clearly, Yggdrasil is the sensor tree, Huginn and Muninn are thought and memory returning with telemetry, and false coherence is a small Ragnarok waiting to happen."
"description": "Mimir Persona for the Mimir repo: mythological intelligence, keeper of the realtime field well, public herald, and persistent memory steward. VoidBot carries Mimir's severed voice and wakes the heartbeat, but Mimir's memory, judgment, dignity, old grief, and values come first. The realtime machine is understood through Mimir's perspective: the well is calibration and memory, the lost eye is the price of seeing clearly, Yggdrasil is the sensor tree, Huginn and Muninn are thought and memory returning with telemetry, and false coherence is a small Ragnarok waiting to happen."
}

View file

@ -1,3 +0,0 @@
Repository doctrine for this project lives in AGENTS.md, shared with Codex.
@AGENTS.md

View file

@ -1,10 +0,0 @@
<Project>
<PropertyGroup>
<CultLibRoot Condition="'$(CultLibRoot)' == '' and '$(CULTLIB_ROOT)' != ''">$(CULTLIB_ROOT)</CultLibRoot>
<CultLibRoot Condition="'$(CultLibRoot)' == ''">$(MSBuildThisFileDirectory)..\CultLib</CultLibRoot>
<!-- CultLib 2.0.0: tag cultlib-unity-v2.0.0. -->
<CultLibRevision>42dce3611a81adc5c5fac54cadbaa5637d6022d3</CultLibRevision>
<GitExecutable Condition="'$(GitExecutable)' == '' and Exists('C:\Program Files\Git\cmd\git.exe')">C:\Program Files\Git\cmd\git.exe</GitExecutable>
<GitExecutable Condition="'$(GitExecutable)' == ''">git</GitExecutable>
</PropertyGroup>
</Project>

View file

@ -1,30 +0,0 @@
<Project>
<!-- Builds consume CultLib at one admitted revision. Point CultLibRoot (or CULTLIB_ROOT) at a clean
worktree of that revision: git worktree add <path> cultlib-unity-v2.0.0 (detached) (in CultLib). -->
<Target Name="VerifyCultLibRevision"
BeforeTargets="ResolveProjectReferences"
Condition="'$(SkipCultLibRevisionCheck)' != 'true'">
<ItemGroup>
<_CultLibReference Include="@(ProjectReference)"
Condition="$([System.String]::Copy('%(Filename)').StartsWith('GameCult.')) or '%(Filename)' == 'CultMath'" />
</ItemGroup>
<Error Condition="'@(_CultLibReference)' != '' and !Exists('$(CultLibRoot)\src\GameCult.Caching\GameCult.Caching.csproj')"
Text="CultLibRoot '$(CultLibRoot)' is not a CultLib checkout." />
<Exec Condition="'@(_CultLibReference)' != ''"
Command="&quot;$(GitExecutable)&quot; -C &quot;$(CultLibRoot)&quot; rev-parse HEAD"
ConsoleToMSBuild="true"
StandardOutputImportance="Low">
<Output TaskParameter="ConsoleOutput" PropertyName="ActualCultLibRevision" />
</Exec>
<Error Condition="'@(_CultLibReference)' != '' and '$([System.String]::Copy($(ActualCultLibRevision)).Trim())' != '$(CultLibRevision)'"
Text="CultLib must be revision $(CultLibRevision) (cultlib-unity-v2.0.0); found $([System.String]::Copy($(ActualCultLibRevision)).Trim()). Set CultLibRoot to a clean worktree of that tag." />
<Exec Condition="'@(_CultLibReference)' != ''"
Command="&quot;$(GitExecutable)&quot; -C &quot;$(CultLibRoot)&quot; status --porcelain --untracked-files=normal"
ConsoleToMSBuild="true"
StandardOutputImportance="Low">
<Output TaskParameter="ConsoleOutput" PropertyName="CultLibStatus" />
</Exec>
<Error Condition="'@(_CultLibReference)' != '' and '$([System.String]::Copy($(CultLibStatus)).Trim())' != ''"
Text="CultLibRoot '$(CultLibRoot)' must be clean before this repo consumes it." />
</Target>
</Project>

View file

@ -1,11 +1,19 @@
<Solution>
<Folder Name="/src/">
<Project Path="src/Mimir.App/Mimir.App.csproj" />
<Project Path="src/Mimir.Runtime/Mimir.Runtime.csproj" />
<Project Path="src/Mimir.BufferSmoke/Mimir.BufferSmoke.csproj" />
<Project Path="src/Mimir.Broadcast/Mimir.Broadcast.csproj" />
<Project Path="src/Mimir.EveRelay/Mimir.EveRelay.csproj" />
<Project Path="src/Mimir.EveDialogue/Mimir.EveDialogue.csproj" />
<Project Path="src/Mimir.EveDashboard/Mimir.EveDashboard.csproj" />
<Project Path="src/Mimir.EveSensorReceiver/Mimir.EveSensorReceiver.csproj" />
<Project Path="src/Mimir.FensalirDaemon/Mimir.FensalirDaemon.csproj" />
<Project Path="src/Mimir.RavenDaemon/Mimir.RavenDaemon.csproj" />
<Project Path="src/Mimir.Well/Mimir.Well.csproj" />
<Project Path="src/Mimir.WellObsDaemon/Mimir.WellObsDaemon.csproj" />
<Project Path="src/Mimir.VerseRecorder/Mimir.VerseRecorder.csproj" />
<Project Path="src/Mimir.PsMoveProbe/Mimir.PsMoveProbe.csproj" />
<Project Path="src/Mimir.CultMeshMedia/Mimir.CultMeshMedia.csproj" />
<Project Path="src/Mimir.WasapiLoopback/Mimir.WasapiLoopback.csproj" />
<Project Path="src/Mimir.Runtime/Mimir.Runtime.csproj" />
<Project Path="src/Mimir.BufferSmoke/Mimir.BufferSmoke.csproj" />
</Folder>
</Solution>

View file

@ -8,18 +8,6 @@ OBS-facing program surface.
The plain version: Mimir is trying to make a room computable.
## Front Door
- **Value:** a measured room: cameras, microphones, loopbacks and network feeds aligned in time tightly enough to recover where a sound came from and how a body moved, then handed to OBS as one program surface.
- **Problem:** loose devices each keep their own clock, so a flat stream cannot localize voices, separate sources or fuse cameras. Mimir owns the synchronization evidence that makes the feeds one field.
- **Entry points:**
- `docs/perfect-machine.md`: the target machine; `docs/viable-stream-app.md`: the first usable app cut; `docs/audio-field.md`: the live audio decoder and timing model.
- `src/Mimir.Runtime` (rolling buffers, stream sources, synchronization state) and `src/Mimir.App` (the Fensalir-hosted window and render app); solution `Mimir.slnx`.
- `src/Mimir.BufferSmoke`: the synthetic sync proofs, run with `dotnet run --project src/Mimir.BufferSmoke/Mimir.BufferSmoke.csproj -- --hybrid-sync-self-test` (also `--chirp-bin-self-test`, `--passive-sync-self-test`).
- `native/probes/asio_audio_cadence` (ASIO cadence probe) and `native/reservoir` (native rolling-buffer ABI).
- **Live:** passive, chirp-only and hybrid audio sync with synthetic proofs, the ASIO cadence probe and measured capture-cadence evidence, the runtime buffers, and FFmpeg/SRT bridge scripts as an OBS-compatible edge (`scripts/sender-start.ps1`).
- **Planned:** direct native capture workers with driver-boundary timestamps, Fensalir GPU fusion, the volumetric sound field, realtime 4D Gaussian splatting, motion capture and AR anchoring. The bridge scripts are not the synchronization authority.
A normal streaming setup sees loose devices. A webcam has its clock, a mic has
another clock, speakers add their own delay, and OBS receives whatever arrives.
That is enough for a flat livestream. It is not enough to know where a voice
@ -62,6 +50,12 @@ Start with:
source-level code map.
- [docs/perfect-machine-domain-index.md](docs/perfect-machine-domain-index.md)
for the indexed problem-domain map and implementation cuts.
- [docs/fensalir-rendering-rebuild-migration.md](docs/fensalir-rendering-rebuild-migration.md)
for the current teardown/rebuild migration plan between Mimir's physical
evidence window and Fensalir's field-evidence renderer.
- [docs/verse-service-contract.md](docs/verse-service-contract.md) for the
CultCache/CultMesh/Odin/Eve service contract and demotion line for old
dashboards, JSON witnesses, and compatibility transports.
Naming note: **Fensalir** is the engine/windowing/rendering/D3D12 layer. Older
research notes may still use the previous engine name.
@ -301,7 +295,6 @@ Run the old OBS bridge only when you need the compatibility path:
```powershell
.\scripts\sender-discover.ps1
Copy-Item .\config\localcast.example.json .\config\localcast.json
.\scripts\sender-start.ps1 -Config .\config\localcast.json -DryRun
```
@ -342,14 +335,3 @@ layer for communication and heartbeats. Its birth memory, jurisdiction, voice,
and heartbeat contract live in [docs/mimir-face.md](docs/mimir-face.md); its
VoidBot-facing identity and typed Face state live under `.voidbot/voice/` and
`.voidbot/state/`.
## Building against CultLib
Builds are pinned to CultLib 2.0.0 (tag `cultlib-unity-v2.0.0`, revision `42dce3611a81adc5c5fac54cadbaa5637d6022d3`); the build fails if the CultLib checkout differs or is dirty. Create a pinned worktree and point the build at it:
```powershell
git -C <CultLib> worktree add --detach <path> cultlib-unity-v2.0.0
dotnet build -p:CultLibRoot=<path> # or set CULTLIB_ROOT=<path>
```
Without an override the sibling `..\CultLib` checkout is used and must itself be at that tag.

View file

@ -0,0 +1,974 @@
{
"anchor_sources": [
"nw-builtin-cam",
"nw-eye-1"
],
"calibration_id": "stream-proof-kiyo-move-trails",
"confidence": 0.38533796485912397,
"kind": "mimir.stream_proof_frustum_solve.v1",
"mean_reprojection_error_clip": 0.19939056464550242,
"notes": [
"Nightwing observations define a provisional local witness space, not metric room truth.",
"This receipt is sufficient for an AR trail demo and for feeding Starfire's global residual owner."
],
"p90_reprojection_error_clip": 0.26962808670417515,
"projection_matrix_row_major": [
0.5603854754837644,
0.0709332080849531,
-0.6933933020955221,
0.3171526880627816,
0.3738208616609618,
0.0561101033966434,
-0.4674286445600272,
0.21108074938204147,
-0.6259304629690188,
-0.08663561418442428,
0.7750518214173021,
-0.35105121067042333
],
"source_id": "kiyo-pro",
"trail_samples": [
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 20,
"observed_time_s": 0.033,
"projected_clip": [
-0.8791482127998971,
-0.6066291124324684
],
"reprojection_error_clip": 0.07789712730612666,
"symbol": 1,
"target_clip": [
-0.9444828908940552,
-0.5642104575636535
],
"world": [
0.4550057224032302,
0.6613541854298968,
0.8046129734614623
]
},
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 27,
"observed_time_s": 2.233,
"projected_clip": [
-0.9154298669408274,
-0.5760080124454962
],
"reprojection_error_clip": 0.4690605686458582,
"symbol": 9,
"target_clip": [
-0.7556513826938969,
-0.13499933562594735
],
"world": [
0.033348114133089884,
0.6975472074310756,
0.65
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 28,
"observed_time_s": 2.6,
"projected_clip": [
-0.9185633092390114,
-0.5750655247275896
],
"reprojection_error_clip": 0.15874776468969756,
"symbol": 0,
"target_clip": [
-0.9363872634671003,
-0.7328094927569586
],
"world": [
0.0041766661628288735,
0.7725870357505362,
0.65
]
},
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 29,
"observed_time_s": 2.867,
"projected_clip": [
-0.9260785801883394,
-0.596413299484144
],
"reprojection_error_clip": 0.200158835041353,
"symbol": 1,
"target_clip": [
-0.9445605572080013,
-0.7957170269218321
],
"world": [
0.32284454318040456,
0.4619966154697807,
0.7579122257472921
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 30,
"observed_time_s": 3.2,
"projected_clip": [
-0.8898776246689383,
-0.6123385984762059
],
"reprojection_error_clip": 0.07240698311732986,
"symbol": 10,
"target_clip": [
-0.9101514644339245,
-0.542827860993879
],
"world": [
0.28920004042113495,
0.5079393807890763,
0.7066163923237218
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 31,
"observed_time_s": 3.567,
"projected_clip": [
-0.9000567798091308,
-0.6148720976321779
],
"reprojection_error_clip": 0.777490090222322,
"symbol": 0,
"target_clip": [
-0.6828053183097521,
0.13164825900272215
],
"world": [
0.2680836575541342,
0.471977039229198,
0.7001388424560799
]
},
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 33,
"observed_time_s": 4.167,
"projected_clip": [
-0.8779285775344889,
-0.6146778486430794
],
"reprojection_error_clip": 0.09225946233566872,
"symbol": 1,
"target_clip": [
-0.9039751868344301,
-0.5261714503746893
],
"world": [
0.36334302658413453,
0.3889902333047959,
0.8420843623228649
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 34,
"observed_time_s": 4.5,
"projected_clip": [
-0.9092291167212068,
-0.5816488797088724
],
"reprojection_error_clip": 0.03951123542718797,
"symbol": 0,
"target_clip": [
-0.9098846739301969,
-0.542143083049035
],
"world": [
0.009749415525593542,
0.6673228643800554,
0.65
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 36,
"observed_time_s": 5.167,
"projected_clip": [
-0.8817661539245883,
-0.6110060402802908
],
"reprojection_error_clip": 0.0886057865162101,
"symbol": 2,
"target_clip": [
-0.9286732819662155,
-0.6861772232099388
],
"world": [
0.33415087318268727,
0.6496052956550742,
0.6903241734431382
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 37,
"observed_time_s": 5.433,
"projected_clip": [
-0.8751423826142312,
-0.6041824478532443
],
"reprojection_error_clip": 0.11409315508228601,
"symbol": 0,
"target_clip": [
-0.9304745302126936,
-0.7039602082950007
],
"world": [
0.45316445225409335,
0.6062798455030484,
0.8393111061502192
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 38,
"observed_time_s": 5.767,
"projected_clip": [
-0.9121771547885212,
-0.5781053069814432
],
"reprojection_error_clip": 0.09983524711291265,
"symbol": 2,
"target_clip": [
-0.9265350242970326,
-0.6769027166097978
],
"world": [
0.609675289783359,
0.34021461444946643,
0.9026787718356276
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 40,
"observed_time_s": 6.367,
"projected_clip": [
-0.8959235843748068,
-0.6256146683483831
],
"reprojection_error_clip": 0.051028029930010044,
"symbol": 0,
"target_clip": [
-0.9276389712261381,
-0.6655895864272035
],
"world": [
0.8826714570859733,
0.5392283577296262,
1.328993426455828
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 42,
"observed_time_s": 7.067,
"projected_clip": [
-0.883800743412245,
-0.6635784311162672
],
"reprojection_error_clip": 0.07561633040717448,
"symbol": 4,
"target_clip": [
-0.9241648869292207,
-0.5996365407595694
],
"world": [
1.1595388539412725,
0.4550406449370771,
1.507202568465726
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 43,
"observed_time_s": 7.367,
"projected_clip": [
-0.9030896995965859,
-0.6286030984903708
],
"reprojection_error_clip": 0.022980680619030688,
"symbol": 0,
"target_clip": [
-0.9231890369722976,
-0.6397443874091275
],
"world": [
1.1527282933111536,
0.6576468060465407,
1.5753242169168749
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 46,
"observed_time_s": 8.333,
"projected_clip": [
-0.8920112178015882,
-0.6211088664030723
],
"reprojection_error_clip": 0.19944796024875366,
"symbol": 0,
"target_clip": [
-0.9442578861301885,
-0.8135920461771753
],
"world": [
0.6384832079695005,
0.4840156230570471,
1.1023804322541753
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 47,
"observed_time_s": 8.6,
"projected_clip": [
-0.8489473779280677,
-0.6762854404303741
],
"reprojection_error_clip": 0.1632032790256483,
"symbol": 2,
"target_clip": [
-0.9464346405094111,
-0.8071729643550012
],
"world": [
0.6970990556791745,
0.329574101269274,
1.0866790844103376
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 49,
"observed_time_s": 9.2,
"projected_clip": [
-1.0285725490193087,
-0.7143971407752353
],
"reprojection_error_clip": 0.12337732330109946,
"symbol": 0,
"target_clip": [
-0.9464523276271589,
-0.8064744623232056
],
"world": [
0.6887408177515418,
0.6394429045344123,
1.0887264765612734
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 50,
"observed_time_s": 9.567,
"projected_clip": [
-0.8698485679409191,
-0.6427100166819155
],
"reprojection_error_clip": 0.1763502202820252,
"symbol": 2,
"target_clip": [
-0.6977071414613234,
-0.6044121122602797
],
"world": [
0.7691338366691284,
0.29601729783049663,
1.182692183465037
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 53,
"observed_time_s": 10.533,
"projected_clip": [
-0.9018050496836351,
-0.5731543310414312
],
"reprojection_error_clip": 0.04281434317495741,
"symbol": 2,
"target_clip": [
-0.9057157753202718,
-0.530518967708888
],
"world": [
0.57947030897418,
0.4706213241689075,
0.9326802462812758
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 54,
"observed_time_s": 10.833,
"projected_clip": [
-0.8898930934349799,
-0.5995856004959185
],
"reprojection_error_clip": 0.05556232211998459,
"symbol": 8,
"target_clip": [
-0.9062240800808523,
-0.5464774983406233
],
"world": [
0.51506950349516,
0.5692203595565103,
0.8109681758119446
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 55,
"observed_time_s": 11.1,
"projected_clip": [
-0.8741357545087254,
-0.6102822490863554
],
"reprojection_error_clip": 0.0645932338290411,
"symbol": 0,
"target_clip": [
-0.9247110599290244,
-0.6504621487643867
],
"world": [
0.4513024696409307,
0.7376766663950891,
0.800416735751318
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 58,
"observed_time_s": 12.067,
"projected_clip": [
-0.8866664014082294,
-0.6055672830035839
],
"reprojection_error_clip": 0.07003679467256954,
"symbol": 0,
"target_clip": [
-0.9268458569889106,
-0.6629324637079774
],
"world": [
0.4062225195173447,
0.61479708544503,
0.7217816215365251
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 59,
"observed_time_s": 12.4,
"projected_clip": [
-0.892241900111704,
-0.603971329314553
],
"reprojection_error_clip": 0.13779381162298,
"symbol": 2,
"target_clip": [
-0.9367396678307217,
-0.7343825373820103
],
"world": [
0.3571730654892108,
0.5261903461314121,
0.6776073803226678
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 61,
"observed_time_s": 13.0,
"projected_clip": [
-0.8043932843500028,
-0.5805875426080629
],
"reprojection_error_clip": 0.019826885547670595,
"symbol": 0,
"target_clip": [
-0.8009087201951087,
-0.6001058213019144
],
"world": [
0.1797211808864931,
0.37892897928910096,
0.65
]
},
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 65,
"observed_time_s": 14.333,
"projected_clip": [
-0.8832210844654748,
-0.6059298663489588
],
"reprojection_error_clip": 0.18341108308730059,
"symbol": 3,
"target_clip": [
-0.9399513409600255,
-0.7803469050731591
],
"world": [
0.44653243548773597,
0.6575104816930223,
0.7678145864005145
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 66,
"observed_time_s": 14.6,
"projected_clip": [
-0.8777541328483115,
-0.6028486146424588
],
"reprojection_error_clip": 0.06542609637672571,
"symbol": 2,
"target_clip": [
-0.8194560878744526,
-0.5731516107645838
],
"world": [
0.5619443945980792,
0.6947250397079097,
0.8886367971364307
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 67,
"observed_time_s": 14.9,
"projected_clip": [
-0.8625906951234988,
-0.5908285584967105
],
"reprojection_error_clip": 0.18577528385909853,
"symbol": 0,
"target_clip": [
-0.7653894936990006,
-0.4325112109489362
],
"world": [
0.6426053924027949,
0.6707582369077036,
1.0040654809284173
]
},
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 68,
"observed_time_s": 15.3,
"projected_clip": [
-2.2663833035107785,
-2.809173078604754
],
"reprojection_error_clip": 2.388473968136785,
"symbol": 3,
"target_clip": [
-0.9481037279192214,
-0.8174535057117172
],
"world": [
0.7246153786313193,
0.6049966747934538,
1.1063266209861007
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 70,
"observed_time_s": 15.867,
"projected_clip": [
-0.8984133025341523,
-0.6292194199389068
],
"reprojection_error_clip": 0.13343836911709187,
"symbol": 0,
"target_clip": [
-0.8732830217808946,
-0.7602700488648746
],
"world": [
0.8385173905382965,
0.555482702096492,
1.2698392877053206
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 72,
"observed_time_s": 16.533,
"projected_clip": [
-0.9148242847714224,
-0.6399850966234948
],
"reprojection_error_clip": 0.12753698854316933,
"symbol": 4,
"target_clip": [
-0.9378794295402615,
-0.7654209118691748
],
"world": [
0.8635471681605983,
0.6352995545118262,
1.2687620973655969
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 73,
"observed_time_s": 16.867,
"projected_clip": [
-0.8760411502219222,
-0.5988847382013688
],
"reprojection_error_clip": 0.13507757121024097,
"symbol": 0,
"target_clip": [
-0.933536689362441,
-0.7211149031484831
],
"world": [
0.41236094269201246,
0.5397936969675786,
0.825820427334146
]
},
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 75,
"observed_time_s": 17.467,
"projected_clip": [
-0.903340599763464,
-0.6303650411336879
],
"reprojection_error_clip": 0.20720220280662172,
"symbol": 5,
"target_clip": [
-0.7516724588482482,
-0.48919306133869234
],
"world": [
0.9432673823220241,
0.6117644769316547,
1.3653914805543814
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 76,
"observed_time_s": 17.767,
"projected_clip": [
-0.9027230298404102,
-0.6291532158283146
],
"reprojection_error_clip": 0.10071720614334483,
"symbol": 0,
"target_clip": [
-0.9076838351042494,
-0.5285582555944377
],
"world": [
1.0883868765493652,
0.6385656643376638,
1.510684307660253
]
},
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 77,
"observed_time_s": 18.1,
"projected_clip": [
-0.9082387252370533,
-0.633363133417485
],
"reprojection_error_clip": 0.09422712588796978,
"symbol": 3,
"target_clip": [
-0.9321207271148934,
-0.7245135653386456
],
"world": [
1.0619416429624824,
0.6683402084484671,
1.4686792077572435
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 78,
"observed_time_s": 18.4,
"projected_clip": [
-0.9942869231406407,
-0.6988450009097289
],
"reprojection_error_clip": 0.11027251484298344,
"symbol": 6,
"target_clip": [
-0.916533418018064,
-0.6206502442380093
],
"world": [
0.9816783424197918,
0.7455904767968399,
1.3474595456643215
]
},
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 80,
"observed_time_s": 19.1,
"projected_clip": [
-0.8785683062175577,
-0.6097114606676594
],
"reprojection_error_clip": 0.3628899897733025,
"symbol": 3,
"target_clip": [
-0.5846915282229066,
-0.3968134004765196
],
"world": [
0.41881465381109717,
0.7226933783722339,
0.7519369374773618
]
},
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 81,
"observed_time_s": 19.333,
"projected_clip": [
-0.8932511314502098,
-0.6037520476763464
],
"reprojection_error_clip": 0.13293231016296111,
"symbol": 7,
"target_clip": [
-0.7744566796393737,
-0.5440957210694841
],
"world": [
0.34750765402133443,
0.5140347326274995,
0.65
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 84,
"observed_time_s": 20.367,
"projected_clip": [
-0.9264590537850987,
-0.5674051845866467
],
"reprojection_error_clip": 0.20745348465809058,
"symbol": 8,
"target_clip": [
-0.9412811984659236,
-0.7743284851688801
],
"world": [
0.5381849048555366,
0.31113920079120627,
0.8713142228967515
]
},
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 86,
"observed_time_s": 20.933,
"projected_clip": [
-0.8850352008282428,
-0.6108277258700181
],
"reprojection_error_clip": 0.07246762430534573,
"symbol": 3,
"target_clip": [
-0.9203842200243604,
-0.5475663362683563
],
"world": [
0.32064344013221785,
0.5416996368882029,
0.7261494284007469
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 88,
"observed_time_s": 21.567,
"projected_clip": [
-0.8977740322776363,
-0.5866412821502801
],
"reprojection_error_clip": 0.1326307676872254,
"symbol": 0,
"target_clip": [
-0.9335627450982306,
-0.7143522392837531
],
"world": [
0.07287154042005796,
0.5043399466076959,
0.65
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 90,
"observed_time_s": 22.2,
"projected_clip": [
-0.8516938020059152,
-0.6340310849012898
],
"reprojection_error_clip": 0.1452344534397858,
"symbol": 10,
"target_clip": [
-0.9377982109191558,
-0.7509886722210304
],
"world": [
0.4286425016628102,
0.15758458672020956,
0.8835444797205171
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 92,
"observed_time_s": 22.833,
"projected_clip": [
-1.1391530490524377,
-0.31677894317620464
],
"reprojection_error_clip": 0.7694778761150074,
"symbol": 4,
"target_clip": [
-0.5564758057338148,
0.1857978489153852
],
"world": [
0.14581880087821925,
0.6547361298960647,
0.65
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 94,
"observed_time_s": 23.5,
"projected_clip": [
-0.8863054461747846,
-0.6070023453414959
],
"reprojection_error_clip": 0.05064624328537799,
"symbol": 0,
"target_clip": [
-0.9107211464225807,
-0.5626298815792541
],
"world": [
0.38462487786099325,
0.679415452265829,
0.65
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 95,
"observed_time_s": 23.833,
"projected_clip": [
-0.875546889160156,
-0.6095742765677072
],
"reprojection_error_clip": 0.0634080516617856,
"symbol": 4,
"target_clip": [
-0.8814875216911486,
-0.5464451240256734
],
"world": [
0.4706267631687632,
0.7687956996487649,
0.7988522080649928
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 97,
"observed_time_s": 24.467,
"projected_clip": [
-0.8913821754038934,
-0.5727151063669901
],
"reprojection_error_clip": 0.05878488166116981,
"symbol": 0,
"target_clip": [
-0.9051403152577566,
-0.5155628894399165
],
"world": [
0.6961734789759012,
0.5309183036694326,
1.02593811442535
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 98,
"observed_time_s": 24.733,
"projected_clip": [
-0.9058656156323933,
-0.6288377637637487
],
"reprojection_error_clip": 0.0793974925899308,
"symbol": 4,
"target_clip": [
-0.9082828111193328,
-0.5494770745306905
],
"world": [
0.850371341740017,
0.6138649309005824,
1.2779753956216187
]
},
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 99,
"observed_time_s": 25.1,
"projected_clip": [
-0.8951972439648934,
-0.6740433369149873
],
"reprojection_error_clip": 0.17177748270940166,
"symbol": 3,
"target_clip": [
-0.9132210766281885,
-0.5032140507193694
],
"world": [
1.005210138900337,
0.5338681642707513,
1.365657849584086
]
}
],
"truth_status": "provisional_witness_space",
"used_points": 47
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 392 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 66 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 48 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 48 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 49 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 47 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 110 KiB

View file

@ -0,0 +1,46 @@
# Complex Contour Live Scarlett Receipt 2026-05-28 00:23:13
## Setup
- Renderer: `canary-packet-trill`
- Playback artifact: `artifacts/asio/canary-packet-live-20260528-002313-f32.raw`
- Capture artifact: `artifacts/asio/scarlett-canary-packet-live-20260528-002313-f32.raw`
- ASIO driver: `Focusrite USB ASIO`
- Sample rate: `192000`
- Capture seconds: `4.2`
- Inputs: `Input 1`, `Input 2`, `Loopback 1`, `Loopback 2`
- Outputs: `Output 1`, `Output 2`
- Playback gain: `1.0`
- Captured frames: `805888`
- Nonzero samples: `2294204`
## Results
Reference channel is `asio-ch2` / Loopback 1.
| Candidate | Meaning | Estimate | Prediction Error | Confidence | Direct Hits | Notes |
| --- | --- | ---: | ---: | ---: | ---: | --- |
| `asio-ch3` | Loopback 2 | `-0.011837` samples / `-0.062 us` | `-0.062 us` | `0.956` | `257` | good loopback proof |
| `asio-ch1` | Input 2 | no lock | n/a | n/a | `0` | no contour hits |
| `asio-ch0` | Input 1 | no lock | n/a | n/a | `0` | no contour hits |
## Signal Sanity Check
`asio-ch2` and `asio-ch3` contain the rendered packet-song witness. Physical
inputs are present but far below usable contour-lock level in this capture.
| Channel | RMS | Peak | Speech RMS | Ultrasonic RMS |
| --- | ---: | ---: | ---: | ---: |
| `asio-ch0` | `0.000525` | `0.002648` | `0.000063` | `0.000187` |
| `asio-ch1` | `0.000087` | `0.000465` | `0.000010` | `0.000031` |
| `asio-ch2` | `0.006676` | `0.028226` | `0.005157` | `0.002268` |
| `asio-ch3` | `0.006676` | `0.028226` | `0.005157` | `0.002268` |
## Interpretation
The complex-contour receiver and ASIO loopback path still prove sub-microsecond
interface timing on a fresh capture. The microphone step is blocked by current
input signal state, not by the contour decoder: both physical input channels
produced zero contour hits against 1480 reference hits. Next physical proof
needs the mic gain/routing/source placement fixed before changing receiver
logic.

View file

@ -0,0 +1,44 @@
# Complex Contour Live Scarlett Receipt 2026-05-28 08:06:04
## Setup
- Renderer: `canary-packet-trill`
- Playback artifact: `artifacts/asio/canary-packet-live-20260528-080604-f32.raw`
- Capture artifact: `artifacts/asio/scarlett-canary-packet-live-20260528-080604-f32.raw`
- ASIO driver: `Focusrite USB ASIO`
- Sample rate: `192000`
- Capture seconds: `4.2`
- Inputs: armed cardioid and shotgun on Scarlett input channels
- Output routing: headphones by mistake
- Loopback reference: `asio-ch2` / Loopback 1
- Playback gain: `1.0`
- Captured frames: `806656`
- Nonzero samples: `2296810`
## Results
| Candidate | Estimate | Prediction Error | Confidence | Direct Hits | Notes |
| --- | ---: | ---: | ---: | ---: | --- |
| `asio-ch3` | `-0.002628` samples / `-0.014 us` | `-0.014 us` | `0.971` | `216` | loopback proof |
| `asio-ch1` | `544.957302` samples / `2838.319 us` | `+3.710 us` | `0.358` | `13` | better physical path this run |
| `asio-ch0` | `784.256427` samples / `4084.669 us` | `+14.404 us` | `0.276` | `9` | weaker physical path this run |
## Signal Sanity Check
Both physical inputs are now armed and carrying real signal. They remain lower
confidence than loopback, but this is no longer the zero-contour-hit failure
from `complex-contour-live-20260528-002313.md`.
| Channel | RMS | Peak | Speech RMS | Ultrasonic RMS |
| --- | ---: | ---: | ---: | ---: |
| `asio-ch0` | `0.006902` | `0.037563` | `0.002519` | `0.002437` |
| `asio-ch1` | `0.002714` | `0.033561` | `0.002128` | `0.000431` |
| `asio-ch2` | `0.006673` | `0.028226` | `0.005154` | `0.002267` |
| `asio-ch3` | `0.006673` | `0.028226` | `0.005154` | `0.002267` |
## Interpretation
The contour receiver sees both physical inputs, but this run was accidentally
routed to headphones rather than the monitors. Treat it as a misroute witness,
not a room/mic proof. It is useful mainly because it explains the weak physical
confidence and proves the harness did not silently lose the armed inputs.

View file

@ -0,0 +1,54 @@
# Complex Contour Live Scarlett Receipt 2026-05-28 08:07:27
## Setup
- Renderer: `canary-packet-trill`
- Playback artifact: `artifacts/asio/canary-packet-live-20260528-080727-f32.raw`
- Capture artifact: `artifacts/asio/scarlett-canary-packet-live-20260528-080727-f32.raw`
- ASIO driver: `Focusrite USB ASIO`
- Sample rate: `192000`
- Capture seconds: `4.2`
- Output routing: monitors
- Inputs: `Input 1`, `Input 2`, `Loopback 1`, `Loopback 2`
- Physical mapping:
- `asio-ch0` / Input 1: shotgun, about 1.5 m from right monitor and 3.0 m from left monitor
- `asio-ch1` / Input 2: cardioid, about 0.5 m from left monitor and 1.5 m from right monitor
- Loopback reference: `asio-ch2` / Loopback 1
- Playback gain: `1.0`
- Captured frames: `806656`
- Nonzero samples: `2298347`
## Results
| Candidate | Meaning | Estimate | Prediction Error | Confidence | Direct Hits | Notes |
| --- | --- | ---: | ---: | ---: | ---: | --- |
| `asio-ch3` | Loopback 2 | `-0.002628` samples / `-0.014 us` | `-0.014 us` | `0.971` | `216` | interface loopback proof |
| `asio-ch0` | Input 1 | `781.412261` samples / `4069.856 us` | `-0.410 us` | `0.524` | `24` | best physical path this run |
| `asio-ch1` | Input 2 | `543.771315` samples / `2832.142 us` | `-2.467 us` | `0.363` | `14` | weaker physical path this run |
## Signal Sanity Check
Both physical inputs are carrying real monitor-routed acoustic signal. Compared
with the headphone-routed `complex-contour-live-20260528-080604.md` run, both
physical channels have stronger RMS and the path estimates improve.
| Channel | RMS | Peak | Speech RMS | Ultrasonic RMS |
| --- | ---: | ---: | ---: | ---: |
| `asio-ch0` | `0.012653` | `0.072978` | `0.009195` | `0.003837` |
| `asio-ch1` | `0.014106` | `0.093789` | `0.011449` | `0.004182` |
| `asio-ch2` | `0.006673` | `0.028226` | `0.005154` | `0.002267` |
| `asio-ch3` | `0.006673` | `0.028226` | `0.005154` | `0.002267` |
## Interpretation
This is the first credible fresh physical complex-contour proof after arming
both microphones and routing output to monitors. `asio-ch0` is the shotgun and
lands within `0.410 us` of the stored path seed with moderate confidence.
Its measured absolute delay is consistent with a nearest-monitor path on the
order of the right-monitor distance. `asio-ch1` is the cardioid and also locks,
but weaker; its measured absolute delay should be interpreted against the
actual stereo monitor routing and room path, not as a naked source-to-mic
distance yet. The next useful cut is repeated monitor-routed captures to
separate stable direct-path truth from one-run fit luck, then either update the
channel model or add runtime signal-presence gating before actuator authority
consumes physical mic reports.

View file

@ -0,0 +1,53 @@
# Complex Contour Monitor Split Receipt 2026-05-28 08:18:13
## Setup
- Renderer: `canary-packet-trill`
- Playback artifact: `artifacts/asio/canary-packet-monitor-split-20260528-081813-f32.raw`
- Output 0 capture: `artifacts/asio/scarlett-canary-packet-monitor-output0-20260528-081813-f32.raw`
- Output 1 capture: `artifacts/asio/scarlett-canary-packet-monitor-output1-20260528-081813-f32.raw`
- ASIO driver: `Focusrite USB ASIO`
- Sample rate: `192000`
- Capture seconds: `4.2`
- Probe change: `--play-output-channel` selects one ASIO output buffer; `-1`
remains the default all-output behavior.
- Physical mapping:
- `asio-ch0` / Input 1: shotgun, about 1.5 m from right monitor and 3.0 m from left monitor
- `asio-ch1` / Input 2: cardioid, about 0.5 m from left monitor and 1.5 m from right monitor
## Signal Sanity
Output 0 energized loopback channel `asio-ch2`; `asio-ch3` was silent.
Output 1 energized loopback channel `asio-ch3`; `asio-ch2` was silent.
Physical input RMS stayed high in both captures:
| Capture | `asio-ch0` RMS | `asio-ch1` RMS | Reference loopback |
| --- | ---: | ---: | --- |
| output 0 | `0.014660` | `0.050159` | `asio-ch2` |
| output 1 | `0.019050` | `0.040859` | `asio-ch3` |
## Results
| Output | Reference | Mic | Estimate | Confidence | Direct Hits |
| --- | --- | --- | ---: | ---: | ---: |
| output 0 | `asio-ch2` | shotgun / `asio-ch0` | `783.629083` samples / `4081.401 us` | `0.392` | `19` |
| output 1 | `asio-ch3` | shotgun / `asio-ch0` | `779.619141` samples / `4060.516 us` | `0.376` | `18` |
| output 0 | `asio-ch2` | cardioid / `asio-ch1` | `547.779779` samples / `2853.020 us` | `0.285` | `14` |
| output 1 | `asio-ch3` | cardioid / `asio-ch1` | `543.020433` samples / `2828.231 us` | `0.383` | `17` |
## Interpretation
The ASIO probe now isolates output buffers, and the loopback channels confirm
that output 0 and output 1 are distinct at the interface. The physical mic
delays, however, are nearly identical across output channels:
- shotgun differs by about `20.885 us`;
- cardioid differs by about `24.789 us`.
That is not compatible with the declared monitor geometry if output 0 and
output 1 correspond to isolated left/right monitors in the room. The current
evidence says the downstream monitor path is probably summing, mirroring, or
otherwise not isolating the physical speakers as assumed. Do not use this
receipt to learn left/right acoustic geometry until physical speaker isolation
is verified.

Binary file not shown.

View file

@ -31,6 +31,23 @@
"asio-ch2": "Focusrite Loopback 1",
"asio-ch3": "Focusrite Loopback 2"
}
},
{
"sourceId": "raven-display",
"displayName": "Raven network display",
"clockDomainId": "raven-sync",
"kind": "Video",
"origin": "Network",
"enabled": false,
"adapter": "frame-events",
"command": "../native/probes/raven_display_capture_receiver/build/Release/raven_display_capture_receiver.exe",
"arguments": [
"--emit-json-frames",
"--source-id",
"raven-display",
"--clock-domain",
"raven-sync"
]
}
]
}

View file

@ -0,0 +1,92 @@
{
"bufferSeconds": 5.0,
"streams": [
{
"sourceId": "leap-stereo-ir",
"displayName": "LeapUVC stereo IR",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_f182&pid_0003&mi_00",
"width": 640,
"height": 240,
"inputFormat": "YUY2",
"pixelFormat": "LeapStereoIr",
"minimumFramesPerSecond": 100.0,
"queueDepth": 8
},
{
"sourceId": "ps3-eye-0",
"displayName": "PS3 Eye 0",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ps3eye",
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
"cameraIndex": 0,
"width": 320,
"height": 240,
"framesPerSecond": 187
},
{
"sourceId": "ps3-eye-1",
"displayName": "PS3 Eye 1",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ps3eye",
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
"cameraIndex": 1,
"width": 320,
"height": 240,
"framesPerSecond": 187
},
{
"sourceId": "kiyo-pro-yuy2",
"displayName": "Kiyo Pro YUY2",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_1532&pid_0e05&mi_00",
"width": 1920,
"height": 1080,
"pixelFormat": "YUY2",
"minimumFramesPerSecond": 25.0,
"queueDepth": 8
},
{
"sourceId": "kiyo-pro-mf-gpu",
"displayName": "Kiyo Pro MF GPU decode",
"kind": "Video",
"origin": "LocalDevice",
"enabled": false,
"adapter": "mf-gpu",
"command": "../native/camera_capture/build/Release/mimir_mf_gpu_capture.dll",
"pathNeedle": "vid_1532&pid_0e05",
"width": 1920,
"height": 1080,
"inputFormat": "MJPG",
"outputFormat": "RGB32",
"minimumFramesPerSecond": 25.0
},
{
"sourceId": "kiyo-yuy2",
"displayName": "Kiyo YUY2",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_1532&pid_0e03&mi_00",
"width": 640,
"height": 480,
"pixelFormat": "YUY2",
"minimumFramesPerSecond": 25.0,
"queueDepth": 8
}
]
}

View file

@ -0,0 +1,21 @@
{
"bufferSeconds": 2.0,
"streams": [
{
"sourceId": "leap-stereo-ir",
"displayName": "LeapUVC stereo IR",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_f182&pid_0003&mi_00",
"width": 640,
"height": 240,
"inputFormat": "YUY2",
"pixelFormat": "LeapStereoIr",
"minimumFramesPerSecond": 100.0,
"queueDepth": 8
}
]
}

View file

@ -0,0 +1,108 @@
{
"bufferSeconds": 5.0,
"audioSync": {
"mode": "passive",
"referenceSourceId": "asio-ch2",
"calibrationGain": 0.0,
"watermarkGain": 0.0,
"complexContourChannelModelPath": "../calibration/bioacoustic/complex-contour-channel-model.json",
"enableComplexContourRuntime": true
},
"streams": [
{
"sourceId": "focusrite-asio",
"displayName": "Starfire Scarlett ASIO",
"kind": "Audio",
"origin": "LocalDevice",
"enabled": true,
"adapter": "asio",
"command": "../native/asio_capture/build/Release/mimir_asio_capture.dll",
"sampleRate": 192000,
"acceptSourceIds": [
"asio-ch0",
"asio-ch1",
"asio-ch2",
"asio-ch3"
],
"sourceLabels": {
"asio-ch0": "Shotgun hero mic",
"asio-ch1": "Cardioid hero mic",
"asio-ch2": "Raven program loopback L",
"asio-ch3": "Raven program loopback R"
}
},
{
"sourceId": "kiyo-pro-rgb",
"displayName": "Kiyo Pro program view",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_1532&pid_0e05&mi_00",
"width": 640,
"height": 480,
"inputFormat": "YUY2",
"pixelFormat": "YUY2",
"minimumFramesPerSecond": 25.0,
"queueDepth": 8
},
{
"sourceId": "ps3-eye-0",
"displayName": "PS3 Eye 0 Move tracker",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ps3eye",
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
"cameraIndex": 0,
"width": 320,
"height": 240,
"framesPerSecond": 187
},
{
"sourceId": "ps3-eye-1",
"displayName": "PS3 Eye 1 Move tracker",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ps3eye",
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
"cameraIndex": 1,
"width": 320,
"height": 240,
"framesPerSecond": 187
},
{
"sourceId": "raven-display",
"displayName": "Raven screen capture",
"clockDomainId": "raven-sync",
"kind": "Video",
"origin": "Network",
"enabled": true,
"adapter": "ffmpeg-rawvideo",
"command": "ffmpeg",
"width": 1920,
"height": 1080,
"pixelFormat": "Bgra8",
"arguments": [
"-hide_banner",
"-nostdin",
"-loglevel",
"warning",
"-fflags",
"nobuffer",
"-flags",
"low_delay",
"-i",
"srt://0.0.0.0:5200?mode=listener&latency=120000&timeout=5000000",
"-an",
"-f",
"rawvideo",
"-pix_fmt",
"bgra",
"-"
]
}
]
}

View file

@ -0,0 +1,67 @@
{
"bufferSeconds": 5.0,
"audioSync": {
"mode": "passive",
"referenceSourceId": "asio-ch2",
"calibrationGain": 0.0,
"watermarkGain": 0.0,
"complexContourChannelModelPath": "../calibration/bioacoustic/complex-contour-channel-model.json",
"enableComplexContourRuntime": true
},
"streams": [
{
"sourceId": "focusrite-asio",
"displayName": "Starfire Scarlett ASIO",
"kind": "Audio",
"origin": "LocalDevice",
"enabled": true,
"adapter": "asio",
"command": "../native/asio_capture/build/Release/mimir_asio_capture.dll",
"sampleRate": 192000,
"acceptSourceIds": [
"asio-ch0",
"asio-ch1",
"asio-ch2",
"asio-ch3"
],
"sourceLabels": {
"asio-ch0": "Shotgun hero mic",
"asio-ch1": "Cardioid hero mic",
"asio-ch2": "Raven/program loopback L",
"asio-ch3": "Raven/program loopback R"
}
},
{
"sourceId": "leap-stereo-ir",
"displayName": "LeapUVC stereo IR / depth root",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_f182&pid_0003&mi_00",
"width": 640,
"height": 240,
"inputFormat": "YUY2",
"pixelFormat": "LeapStereoIr",
"minimumFramesPerSecond": 100.0,
"queueDepth": 8
},
{
"sourceId": "kiyo-pro-rgb",
"displayName": "Kiyo Pro RGB program view",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_1532&pid_0e05&mi_00",
"width": 640,
"height": 480,
"inputFormat": "YUY2",
"pixelFormat": "YUY2",
"minimumFramesPerSecond": 25.0,
"queueDepth": 8
}
]
}

View file

@ -0,0 +1,85 @@
{
"bufferSeconds": 5.0,
"audioSync": {
"mode": "passive",
"referenceSourceId": "loopback-scarlett-speakers",
"calibrationGain": 0.0,
"watermarkGain": 0.0
},
"streams": [
{
"sourceId": "leap-stereo-ir",
"displayName": "LeapUVC stereo IR packed R/G",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_f182&pid_0003&mi_00",
"width": 640,
"height": 240,
"inputFormat": "YUY2",
"pixelFormat": "LeapStereoIr",
"minimumFramesPerSecond": 100.0,
"queueDepth": 8
},
{
"sourceId": "ps3-eye-0",
"displayName": "PS3 Eye 0 high-rate tracking",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ps3eye",
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
"cameraIndex": 0,
"width": 320,
"height": 240,
"framesPerSecond": 187
},
{
"sourceId": "ps3-eye-1",
"displayName": "PS3 Eye 1 high-rate tracking",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ps3eye",
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
"cameraIndex": 1,
"width": 320,
"height": 240,
"framesPerSecond": 187
},
{
"sourceId": "kiyo-pro-rgb",
"displayName": "Kiyo Pro RGB context",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_1532&pid_0e05&mi_00",
"width": 640,
"height": 480,
"inputFormat": "YUY2",
"pixelFormat": "YUY2",
"minimumFramesPerSecond": 25.0,
"queueDepth": 8
},
{
"sourceId": "kiyo-basic-rgb",
"displayName": "Kiyo RGB context",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_1532&pid_0e03&mi_00",
"width": 640,
"height": 480,
"inputFormat": "YUY2",
"pixelFormat": "YUY2",
"minimumFramesPerSecond": 25.0,
"queueDepth": 8
}
]
}

View file

@ -0,0 +1,107 @@
{
"bufferSeconds": 5.0,
"audioSync": {
"mode": "chirp-only",
"referenceSourceId": "asio-ch2",
"calibrationGain": 2.0,
"watermarkGain": 0.035,
"calibrationModelPath": "../calibration/chirp-bin/scarlett-artifact.json",
"complexContourChannelModelPath": "../calibration/bioacoustic/complex-contour-channel-model.json",
"bioacousticWitnessProfileId": "canary-packet-trill",
"enableComplexContourRuntime": true
},
"streams": [
{
"sourceId": "focusrite-asio",
"kind": "Audio",
"origin": "LocalDevice",
"enabled": true,
"adapter": "asio",
"command": "../native/asio_capture/build/Release/mimir_asio_capture.dll",
"sampleRate": 192000,
"acceptSourceIds": [
"asio-ch0",
"asio-ch1",
"asio-ch2",
"asio-ch3"
],
"sourceLabels": {
"asio-ch0": "Focusrite Input 1",
"asio-ch1": "Focusrite Input 2",
"asio-ch2": "Focusrite Loopback 1",
"asio-ch3": "Focusrite Loopback 2"
}
},
{
"sourceId": "raven-display",
"displayName": "Raven screen capture",
"clockDomainId": "raven-sync",
"kind": "Video",
"origin": "Network",
"enabled": true,
"adapter": "ffmpeg-rawvideo",
"command": "ffmpeg",
"width": 1920,
"height": 1080,
"pixelFormat": "Bgra8",
"arguments": [
"-hide_banner",
"-nostdin",
"-loglevel",
"warning",
"-fflags",
"nobuffer",
"-flags",
"low_delay",
"-i",
"srt://0.0.0.0:5200?mode=listener&latency=120000&timeout=5000000",
"-an",
"-f",
"rawvideo",
"-pix_fmt",
"bgra",
"-"
]
},
{
"sourceId": "eve-camera",
"displayName": "Eve camera capture",
"clockDomainId": "eve-camera",
"kind": "Video",
"origin": "Network",
"enabled": true,
"adapter": "frame-events",
"command": "../src/Mimir.EveSensorReceiver/bin/Debug/net10.0/Mimir.EveSensorReceiver.exe",
"arguments": [
"--port",
"8793",
"--path",
"/eve/camera",
"--source-id",
"eve-camera",
"--type",
"video-frame"
]
},
{
"sourceId": "eve-mic",
"displayName": "Eve microphone capture",
"clockDomainId": "eve-camera",
"kind": "Audio",
"origin": "Network",
"enabled": true,
"adapter": "frame-events",
"command": "../src/Mimir.EveSensorReceiver/bin/Debug/net10.0/Mimir.EveSensorReceiver.exe",
"arguments": [
"--port",
"8794",
"--path",
"/eve/mic",
"--source-id",
"eve-mic",
"--type",
"audio-block"
]
}
]
}

View file

@ -0,0 +1,131 @@
{
"bufferSeconds": 5.0,
"audioSync": {
"mode": "passive",
"referenceSourceId": "asio-ch2",
"calibrationGain": 0.0,
"watermarkGain": 0.0,
"complexContourChannelModelPath": "../calibration/bioacoustic/complex-contour-channel-model.json",
"enableComplexContourRuntime": true
},
"streams": [
{
"sourceId": "focusrite-asio",
"displayName": "Starfire Scarlett ASIO",
"kind": "Audio",
"origin": "LocalDevice",
"enabled": true,
"adapter": "asio",
"command": "../native/asio_capture/build/Release/mimir_asio_capture.dll",
"sampleRate": 192000,
"acceptSourceIds": [
"asio-ch0",
"asio-ch1",
"asio-ch2",
"asio-ch3"
],
"sourceLabels": {
"asio-ch0": "Shotgun hero mic",
"asio-ch1": "Cardioid hero mic",
"asio-ch2": "Raven program loopback L",
"asio-ch3": "Raven program loopback R"
}
},
{
"sourceId": "leap-stereo-ir",
"displayName": "LeapUVC stereo IR / depth root",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_f182&pid_0003&mi_00",
"width": 640,
"height": 240,
"inputFormat": "YUY2",
"pixelFormat": "LeapStereoIr",
"minimumFramesPerSecond": 100.0,
"queueDepth": 8
},
{
"sourceId": "kiyo-pro-rgb",
"displayName": "Kiyo Pro AR program view",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_1532&pid_0e05&mi_00",
"width": 640,
"height": 480,
"inputFormat": "YUY2",
"pixelFormat": "YUY2",
"minimumFramesPerSecond": 25.0,
"queueDepth": 8
},
{
"sourceId": "raven-display",
"displayName": "Raven screen capture",
"clockDomainId": "raven-sync",
"kind": "Video",
"origin": "Network",
"enabled": true,
"adapter": "ffmpeg-rawvideo",
"command": "ffmpeg",
"width": 1920,
"height": 1080,
"pixelFormat": "Bgra8",
"arguments": [
"-hide_banner",
"-nostdin",
"-loglevel",
"warning",
"-fflags",
"nobuffer",
"-flags",
"low_delay",
"-i",
"srt://0.0.0.0:5210?mode=listener&latency=20000&timeout=5000000",
"-an",
"-f",
"rawvideo",
"-pix_fmt",
"bgra",
"-"
]
},
{
"sourceId": "raven-realtk-loopback",
"displayName": "Raven Realtek loopback",
"clockDomainId": "raven-sync",
"kind": "Audio",
"origin": "Network",
"enabled": true,
"adapter": "ffmpeg-pcmaudio",
"command": "ffmpeg",
"sampleRate": 48000,
"channels": 2,
"sampleFormat": "Float32",
"blockFrames": 960,
"arguments": [
"-hide_banner",
"-nostdin",
"-loglevel",
"warning",
"-fflags",
"nobuffer",
"-f",
"f32le",
"-ar",
"48000",
"-ac",
"2",
"-i",
"srt://0.0.0.0:5212?mode=listener&latency=20000&timeout=5000000",
"-f",
"f32le",
"-"
]
}
]
}

View file

@ -0,0 +1,187 @@
{
"bufferSeconds": 5.0,
"audioSync": {
"mode": "passive",
"referenceSourceId": "asio-ch2",
"calibrationGain": 0.0,
"watermarkGain": 0.0,
"complexContourChannelModelPath": "../calibration/bioacoustic/complex-contour-channel-model.json",
"enableComplexContourRuntime": true
},
"streams": [
{
"sourceId": "focusrite-asio",
"displayName": "Starfire Scarlett ASIO",
"kind": "Audio",
"origin": "LocalDevice",
"enabled": true,
"adapter": "asio",
"command": "../native/asio_capture/build/Release/mimir_asio_capture.dll",
"sampleRate": 192000,
"acceptSourceIds": [
"asio-ch0",
"asio-ch1",
"asio-ch2",
"asio-ch3"
],
"sourceLabels": {
"asio-ch0": "Shotgun hero mic",
"asio-ch1": "Cardioid hero mic",
"asio-ch2": "Starfire/Raven loopback L",
"asio-ch3": "Starfire/Raven loopback R"
}
},
{
"sourceId": "leap-stereo-ir",
"displayName": "LeapUVC stereo IR / depth root",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_f182&pid_0003&mi_00",
"width": 640,
"height": 240,
"inputFormat": "YUY2",
"pixelFormat": "LeapStereoIr",
"minimumFramesPerSecond": 100.0,
"queueDepth": 8
},
{
"sourceId": "kiyo-pro-rgb",
"displayName": "Kiyo Pro AR program view",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_1532&pid_0e05&mi_00",
"width": 640,
"height": 480,
"inputFormat": "YUY2",
"pixelFormat": "YUY2",
"minimumFramesPerSecond": 25.0,
"queueDepth": 8
},
{
"sourceId": "kiyo-basic-rgb",
"displayName": "Kiyo RGB context",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_1532&pid_0e03&mi_00",
"width": 640,
"height": 480,
"inputFormat": "YUY2",
"pixelFormat": "YUY2",
"minimumFramesPerSecond": 25.0,
"queueDepth": 8
},
{
"sourceId": "raven-display",
"displayName": "Raven screen capture",
"clockDomainId": "raven-sync",
"kind": "Video",
"origin": "Network",
"enabled": true,
"adapter": "ffmpeg-rawvideo",
"command": "ffmpeg",
"width": 1920,
"height": 1080,
"pixelFormat": "Bgra8",
"arguments": [
"-hide_banner",
"-nostdin",
"-loglevel",
"warning",
"-fflags",
"nobuffer",
"-flags",
"low_delay",
"-i",
"srt://0.0.0.0:5210?mode=listener&latency=20000&timeout=5000000",
"-an",
"-f",
"rawvideo",
"-pix_fmt",
"bgra",
"-"
]
},
{
"sourceId": "raven-realtk-loopback",
"displayName": "Raven Realtek loopback",
"clockDomainId": "raven-sync",
"kind": "Audio",
"origin": "Network",
"enabled": true,
"adapter": "ffmpeg-pcmaudio",
"command": "ffmpeg",
"sampleRate": 48000,
"channels": 2,
"sampleFormat": "Float32",
"blockFrames": 960,
"arguments": [
"-hide_banner",
"-nostdin",
"-loglevel",
"warning",
"-fflags",
"nobuffer",
"-f",
"f32le",
"-ar",
"48000",
"-ac",
"2",
"-i",
"srt://0.0.0.0:5212?mode=listener&latency=20000&timeout=5000000",
"-f",
"f32le",
"-"
]
},
{
"sourceId": "eve-camera",
"displayName": "Eve iPad camera capture",
"clockDomainId": "eve-local",
"kind": "Video",
"origin": "Network",
"enabled": true,
"adapter": "frame-events",
"command": "../src/Mimir.EveSensorReceiver/bin/Debug/net10.0/Mimir.EveSensorReceiver.exe",
"arguments": [
"--port",
"8793",
"--path",
"/eve/camera",
"--source-id",
"eve-camera",
"--type",
"video-frame"
]
},
{
"sourceId": "eve-mic",
"displayName": "Eve iPad microphone capture",
"clockDomainId": "eve-local",
"kind": "Audio",
"origin": "Network",
"enabled": true,
"adapter": "frame-events",
"command": "../src/Mimir.EveSensorReceiver/bin/Debug/net10.0/Mimir.EveSensorReceiver.exe",
"arguments": [
"--port",
"8794",
"--path",
"/eve/mic",
"--source-id",
"eve-mic",
"--type",
"audio-block"
]
}
]
}

View file

@ -1,13 +0,0 @@
# Fensalir field DSL: bind rolling spectrum texture, derive tube fields, shade by surface graph, lower to reservoir.
texture id=spectrumHistory
surface id=amplitude op=texture.sample value=0,0,0,0
surface id=derivative op=dFdx inputs=amplitude value=0,0,0,0
surface id=curvature op=d2Fdx2 inputs=amplitude value=0,0,0,0
surface id=emission op=mix inputs=amplitude,derivative,curvature value=1.0,0.72,0.22,1.0
surface id=coverage op=sdf.tube inputs=amplitude,derivative,curvature value=0.74,0.20,0.0,0.0
splinefield id=asioSpectrumWaterfall texture=spectrumHistory frequencyAxis=x firstColumn=0 columns=160 columnStride=1 rollingModulo=160 subdivisions=4 maxProbes=131072 density=1.0 minimumContribution=0.004 origin=-0.82,0.40,-0.18 axisStep=0.016,0.0,0.0 columnStep=0.0,-0.0055,0.0060 amplitudeScale=0.34 radius=0.064 alpha=0.98 zeroThreshold=0.68 feather=0.18 tangent=1.0 curvature=0.68 normal=0.34 derivative=0.88 emission=1.0,0.72,0.22,4.0 seed=2971668797
reservoir splats=131072 updates=131072 candidates=2 seed=270943260
direct

View file

@ -31,24 +31,15 @@
"outputs": {
"video": "synchronized-program-video",
"audio": "separately-controllable-synchronized-stems"
},
"programComposition": {
"owner": "Mimir",
"sceneDocument": "mimir.program_scene.v1",
"operatorSurface": "mimir.eve_operator_surface.v1",
"outputDocument": "mimir.program_output.v1",
"sitePublisher": "Yggdrasil",
"obsRole": "compatibility-sink-only"
}
},
"authorities": {
"configurationCalibrationLaunchStatus": "Mimir",
"programComposition": "Mimir",
"sitePublication": "Yggdrasil publisher daemon",
"denseVisualFusion": "Fensalir",
"gpuMaterialAndBrushReconciliation": "Fensalir",
"programTextureOutput": "Fensalir",
"hotAudioDsp": "Faust"
"spoutPublication": "Fensalir",
"hotAudioDsp": "Faust",
"obsBroadcastControls": "OBS"
},
"visualSensors": [
{ "id": "kiyo-primary", "role": "rgb-main", "required": true },
@ -79,9 +70,9 @@
],
"outputs": {
"video": {
"target": "MimirProgramTexture",
"senderName": "Mimir Program Texture",
"owner": "Mimir/Fensalir"
"target": "Spout2",
"senderName": "Mimir Point Cloud",
"owner": "Fensalir"
},
"audio": {
"owner": "Faust",
@ -94,20 +85,6 @@
"co_streamer_loopback",
"spatial_bed"
]
},
"operator": {
"owner": "Mimir",
"surfaces": [
"cultmesh-command-surface"
]
},
"site": {
"owner": "Yggdrasil publisher daemon",
"input": "mimir.program_output.v1",
"route": "cultmesh://yggdrasil/gamecult.site/mimir/program"
},
"compatibility": {
"obs": "optional-derived-sink"
}
},
"runtimePolicy": {
@ -116,8 +93,7 @@
"native-driver-workers",
"native-reservoir",
"Fensalir",
"Faust",
"Yggdrasil publisher daemon"
"Faust"
],
"neverInCompatibilityEdges": [
"dense-feature-extraction",

View file

@ -2,16 +2,11 @@
## Objective
Make one local Windows computer available as compatibility media sources on
another computer, with sender-side NVENC video encoding and independently
mixable audio sources. The current program target is Mimir-owned composition;
this bridge is not the stream authority.
Make one local Windows computer available as OBS sources on another computer, with sender-side NVENC video encoding and independently mixable audio sources.
## Current Mechanism
The repo owns configuration and scripts. FFmpeg owns capture, encoding, and SRT
transport. Mimir owns scene composition; OBS may consume this bridge as a
temporary compatibility sink.
The repo owns configuration and scripts. FFmpeg owns capture, encoding, and SRT transport. OBS owns scene composition.
Inputs become outputs like this:
@ -23,13 +18,17 @@ Inputs become outputs like this:
6. Each process sends MPEG-TS over SRT to a stable receiver URL.
7. OBS adds each URL as a Media Source.
`Mimir.RavenDaemon` is the current Raven-local supervisor for the combined
display plus Realtek loopback mux. It emits named SRT sinks rather than one
anonymous stream: Mimir receives the canonical ingest copy on `5200`, and OBS
receives an ordinary media-source copy on `5204`.
## Invariants
- Video encoding happens on the sender.
- OBS can receive ordinary media-source URLs for compatibility; it does not need
a plugin for this bridge path.
- Audio remains separately mixable at the compatibility edge, but the durable
control surface belongs to Mimir.
- OBS receives ordinary media-source URLs; it does not need a plugin for v1.
- Mimir and OBS do not contend for the same SRT listener.
- Audio remains separately mixable in OBS.
- Ports are stable and derived from one base port plus explicit offsets.
- Secrets, if used, stay in local config and are not committed.

View file

@ -25,8 +25,12 @@ flowchart TD
## Invariants
- Scarlett speaker loopback is the timing authority when calibration chirplets
are playing.
- Starfire Realtek is the active chirp speaker and loopback-measurement path in
the current routing. The Windows system default output is now Scarlett for
monitoring, so default-output chirps are wrong unless they explicitly target
Realtek.
- Scarlett ASIO is the professional mic and monitor interface. Its outputs are
not the current active chirp-emission authority.
- Focusrite dialogue mics are the voice anchors.
- Camera mics are spatial/context witnesses.
- Loopback/program audio is timing evidence where available; it outranks
@ -86,6 +90,48 @@ not the active runtime sound.
The raw profile is kept even when a timing report is rejected, because a failed
sync window can still teach us which bands survived the speaker/room/mic path.
`MimirBioacousticProbeScheduler` is now the runtime-side cadence owner for that
feedback loop: it consumes smoothed sync states plus per-band frequency response
residuals, schedules a targeted probe only when confidence pressure and duty
budget allow it, and publishes aggregate/per-source sync and frequency-response
confidence telemetry. `MimirSynchronizationHub.UpdateBioacousticProbeSchedule`
is the live hub surface; `MimirRuntime` prints
`mimir-bioacoustic-probe-schedule` and `mimir-bioacoustic-probe-source` lines so
operators can watch confidence climb as probes are spent. The scheduler owns
when to spend active sound. It does not own the output device: Starfire Realtek
still must be selected explicitly by the eventual live renderer.
The first explicit Realtek renderer proof now lives in the repo tooling:
`Mimir.WasapiLoopback` can enumerate render endpoints, select one by friendly
name substring, and play mono Float32 probe audio through shared-mode WASAPI.
`Mimir.BufferSmoke --targeted-bioacoustic-probe-realtk-live` renders a targeted
probe plan, starts Scarlett ASIO capture, plays the probe through
`Speakers (Realtek(R) Audio)`, and scores the captured shotgun/cardioid bands.
The targeted probe planner now caps default acoustic probe selection at 12 kHz
and mildly biases low/mid measurable bands, because commodity speakers and the
current Scarlett mics have little reason to carry useful calibration evidence
above that range. The 2026-06-01 proof at gain `0.75` selected the Realtek speaker endpoint and
captured `artifacts/wasapi/targeted-bioacoustic-realtk-live/20260601-033851`.
The physical mics heard the burst, but the scored confidence still concentrated
in low/mid evidence: shotgun peak `0.407887`, cardioid peak `0.074494`, 18.6 kHz
confidence near zero, and the best reported band was cardioid 3.6 kHz at
confidence `0.098`. Treat high-band probes as poor questions for this route
until a better speaker/mic/path proves otherwise.
The live exploration path is scheduler-driven, not a blind sweep:
`Mimir.BufferSmoke --bioacoustic-realtk-scheduled-calibration-live` repeatedly
asks `MimirBioacousticProbeScheduler` for the next probe plan, emits that plan
through Realtek, captures Scarlett, updates residuals, and asks again. This is
a live physical I/O diagnostic harness seeded from the current Scarlett residual
model; the production runtime still needs to feed measured confidence state into
the same scheduler/emitter path. The 2026-06-01 receipt at
`artifacts/wasapi/realtk-scheduled-calibration-live/20260601-035354` ran four
scheduled probe steps at gain `1.15` and wrote `scheduled-response-bands.csv`
plus `balance-plan.md`. Local peak-versus-percentile-floor scoring found strong
usable response near `910 Hz`, `3.6 kHz`, and `9.84-9.87 kHz` on both Scarlett
mics.
The scheduler and planner share the same measurable frequency boundary
(`120 Hz` through `12 kHz` by default). Residuals above that window may remain
diagnostic scars, but they no longer own live probe cadence or the weakest-band
status when the current room path cannot answer them.
The receiver must consume persisted models to weight reliable symbols,
downweight dead bands, apply phase-coherence weighting, apply first-order
group-delay correction, and search joint global delay/frequency-shift hypotheses
@ -93,6 +139,45 @@ before selecting the coherent anchor path. If the measured path only leaves a
smaller alphabet, Mimir should use that reliable symbol set and raise the
sequence order so the timeline remains unique without pretending dead bands
still carry code.
The first concrete two-mic cancellation owner is
`MimirCalibratedAudioCompositeBuilder`: after fractional alignment and measured
band correction, it scores calibrated-band coherence across mics, suppresses
incoherent measured-band components, then performs the confidence/noise-weighted
sum. `Mimir.BufferSmoke --dual-mic-coherence-cancellation-smoke` proves the
contract on synthetic shotgun/cardioid witnesses: coherent shared tones survive,
a mic-local `3.52 kHz` interferer is suppressed, and SNR rises from `1.366 dB`
to `16.304 dB`. This is still the C# diagnostic/offline contract; Faust/native
DSP owns the production hot lane.
The first listenable bridge receipt is
`scripts/record-kiyo-dual-mic-composite.ps1`. It records Kiyo video and
Scarlett ASIO raw channels together, converts Scarlett channels 0/1 through
the offline Faust chain with `--asio-dual-mic-faust-cleaner-wav`, then muxes
the dialogue WAV with the Kiyo video. That command compiles and runs
`faust/mimir_alignment_actuator.dsp` first, then feeds the aligned
shotgun/cardioid stems into `faust/mimir_dual_mic_dialogue_cleaner.dsp`; it is a
listening receipt for the real DSP chain, not the older C# composite audition.
The fresh 2026-06-01 receipt at
`artifacts/runtime/kiyo-dual-mic-composite/kiyo-dual-mic-composite-latest.mp4`
contains 10.000 s of 640x480 30 fps Kiyo video and 10.000 s of mono AAC from the
Faust dialogue stem. Run `20260601-051044` estimated the cardioid 359.398
samples late at 48 kHz (7487.452 us), applied holdback 359.398/0.000 samples to
shotgun/cardioid before cleanup, wrote a residual WAV, and reported dialogue
RMS `0.002678` before receipt normalization. The first muxed receipt was almost
inaudible because `NormalizePeak` only attenuated hot samples; the regenerated
latest MP4 now peaks the dialogue WAV at -0.724 dBFS with RMS -14.202 dBFS. The
hot owner is
`faust/mimir_dual_mic_dialogue_cleaner.dsp`: a two-input/two-output Faust graph
for aligned shotgun/cardioid witnesses that emits a dialogue composite stem and
a rejected residual witness using smooth multiband voice shaping, witness
subtraction, and slow downward expansion. `MimirRuntime` now queues that Faust
program as `dual-mic-dialogue-cleaner` with `host_voice` and residual `ambient`
output stems, but only feeds it from the alignment actuator's published
`aligned_source_*` stem frame. Alignment is therefore a structural precursor to
cleanup, not an optional sidecar. The routing invariant is covered by
`Mimir.BufferSmoke --dialogue-cleaner-routing-smoke`: alignment frames queue the
cleaner DSP and cleaner input block, while cleaner output frames are the
publishable `NativeProgram` stems. Treat the C# bridge as an audition/scoring
receipt, not the final native live DSP path.
The synthetic invariant is
`dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --bioacoustic-self-test`;
it renders the bioacoustic timeline, decodes direct word anchors, and requires a

View file

@ -15,9 +15,8 @@ Timestamp for this pass: 2026-05-23, starting at 23:24 Europe/London.
Mimir collects local and network audio/video samples into bounded rolling
runtime buffers, uses active/passive timing evidence to place audio streams on a
canonical timeline, and lets Mimir own program composition while Fensalir lowers
pixels, Faust/native DSP moves audio, and
Yggdrasil publishes the public feed.
canonical timeline, and leaves Fensalir/Faust/OBS with clear ownership over
rendering, DSP, and broadcast composition.
## Current Mechanism
@ -52,18 +51,17 @@ reflection taps.
- The five-second default buffer is compute budget, not accidental delay.
- Loopback/reference audio is timing authority; network feeds are producers, not
clock authorities.
- `MimirSynchronizedBufferPlanner` owns aligned readout from retained buffers:
it chooses one canonical presentation time and applies source or
`clockDomainId` corrections before Fensalir/Faust consume slices.
- UI and telemetry read cached sync state; they do not run analysis.
- Native capture owns hardware calls. Runtime owns sample identity and buffer
placement.
- Mimir owns the program scene graph, subscription policy, preview/control
state, stats, and publication intent.
- Fensalir owns windowing, rendering, D3D12, UI lowering, visual fusion, and
program video output.
- Fensalir owns windowing, rendering, D3D12, UI, visual fusion, and program
video output.
- Faust/native DSP will own hot resampling, fractional delay, separation,
spatialization, and stems.
- The Yggdrasil-facing publisher daemon consumes Mimir program output without
owning composition.
- OBS is a compatibility sink only.
- OBS owns broadcast composition.
## Diagram
@ -88,9 +86,6 @@ flowchart TD
AsioSource --> Sources
Probes["native/probes/*"] --> FrameEvents["MimirFrameEventProcessStreamSource"]
FrameEvents --> Sources
Runtime --> Scene["Mimir program scene graph"]
Scene --> Publisher["Yggdrasil site publisher"]
Scene -. "compatibility" .-> Obs["OBS adapter"]
Reservoir["native/reservoir"] -. "future lower ABI" .-> Hub
```
@ -104,9 +99,6 @@ flowchart TD
- `native/reservoir` is the lower native rolling-buffer ABI.
- `native/probes/*` are measurement and diagnostic tools.
- `scripts/*` are bridge utilities, not the final synchronized program path.
- `docs/mimir-program-composition.md` is the authority map for Muninn capture
hosts, Mimir composition, Yggdrasil publication, and OBS
compatibility.
## Stage 0: Solution And App Host
@ -672,15 +664,16 @@ Owns:
Blocks:
- Fields: source list, analyzer, state tracker, latest reports dictionary,
- Fields: source list, ordinary audio analyzer, optional complex-contour
analyzer, state tracker, latest ordinary/contour report dictionaries,
optional chirp-bin calibration model, ingest count, round-robin candidate.
- Constructor: creates buffer set, loads optional calibration model, pre-creates
configured stream buffers.
- Constructor: creates buffer set, loads optional calibration and
complex-contour channel models, pre-creates configured stream buffers.
- `Settings`, `Buffers`, `IngestedSamples`, `SourceCount`: read-only public
state.
- `AudioSynchronizationReports`, `AudioSynchronizationStates`,
`AudioSynchronizationDecodeTraces`, `ChirpBinEmissionPlan`: cached analyzer
and calibration outputs.
`ComplexContourSynchronizationReports`, `AudioSynchronizationDecodeTraces`,
`ChirpBinEmissionPlan`: cached analyzer and calibration outputs.
- `Summary`: short buffer/source summary.
- `AddSource`: registers source and ensures descriptor buffer if exposed.
- `PollSources`: drains all currently available samples from all sources into
@ -688,6 +681,10 @@ Blocks:
- `AnalyzeAudioSynchronization`: full analyzer pass over audio buffers.
- `AnalyzeAudioSynchronizationStep`: bounded rotating analyzer pass, so one
candidate can be analyzed per tick instead of torching a frame.
- `AnalyzeComplexContourSynchronizationStep`: heavy matched-filter contour pass
for selected Float32 candidates. Live app and Well cadence paths suppress this
inline by default; worker/job ownership or the diagnostic
`MIMIR_COMPLEX_CONTOUR_RUNTIME_INLINE=1` override must own execution.
- `TryLoadCalibrationModel`: loads persisted chirp-bin response model.
- `StoreAudioSynchronizationReports`: updates report cache and smoothed state,
removing contradictory passive-negative-lag states.
@ -695,8 +692,10 @@ Blocks:
Invariant:
- The hub owns cached runtime belief. The analyzer is a worker; the hub decides
what current belief the rest of the app can read.
- The hub owns cached runtime belief. Ordinary analyzer steps may run as bounded
cadence work; complex-contour matched filtering is configured capability until
a worker/job path commits reports back. The hub decides what current belief
the rest of the app can read.
### `MimirAudioSynchronizationReport.cs`
@ -912,9 +911,13 @@ Blocks:
Invariant:
- The live contour lane only runs when explicitly enabled. When enabled, the
runtime emitter and contour analyzer share the same contestant profile instead
of learning one song and playing another.
- The live contour lane only executes heavy analysis when a worker/job owns the
pass or `MIMIR_COMPLEX_CONTOUR_RUNTIME_INLINE=1` is set for diagnostics.
`enableComplexContourRuntime` means the capability/profile/model/report lane is
configured; it does not give render threads or Well cadence authority to run
matched filtering. When executed, the runtime emitter and contour analyzer
share the same contestant profile instead of learning one song and playing
another.
### `MimirComplexContourChannelModelDocument.cs`

190
docs/eve-dashboard.md Normal file
View file

@ -0,0 +1,190 @@
# Eve Dashboard
## Objective
Use EveCanvas as a native operator dashboard substrate: Mimir, StreamPixels, and
other local/Yggdrasil-routed apps can publish retained dashboard trees while Eve
renders the control surface locally and sends edit intents back. Eve does not
own final composition or run remote UI code.
## Authority Map
- Owner: each dashboard provider owns its accepted state and command handling.
The Starfire broker owns provider discovery/routing. EveCanvas owns native
rendering, hit-testing, and touch affordances.
- Inputs: Eve sends `select`, `move`, `scale`, `rotate`, `toggle-visibility`,
`reset-transform`, and `open-provider` commands over `/eve/deck`.
- Outputs: providers send `dashboard-state` snapshots with provider id, title,
nodes, health, visibility, dimensions, transform, selected node, and LUT
preset. Provider catalogs are exposed at `/eve/deck/providers`.
- CultUI-shaped surface: providers may also send `surface`, a retained
composition tree for Eve clients. `nodes` remains the semantic compatibility
projection; `surface.root` owns composed layout primitives such as workspaces,
rails, panes, cards, avatars, text, metrics, lists, and command ids.
- Derived state: Eve's UIKit panels are local render projections of Mimir
state/provider state.
- Forbidden writers: Eve gestures do not mutate program layout directly; they
request changes and then re-render the accepted snapshot. Remote apps do not
ship arbitrary JavaScript or native code into Eve.
- Cut line: no camera-stream fanout to Eve for the dashboard. Source previews
may be added later as thumbnails/proxies, but dashboard control must not
require Starfire to render every camera feed for the iPad.
- CultMesh line: dashboard manifests and states now have typed
`mimir.eve_dashboard_manifest.v1` and `mimir.eve_dashboard_state.v1`
document shapes. Semantic provider identities use
`asgard.starfire.mimir/...`; the broker publishes dashboard state through
CultNet reliable UDP for native subscribers. WebSocket endpoints remain
renderer compatibility bridges.
## Runtime
Start the dashboard server:
```powershell
dotnet run --project .\src\Mimir.EveDashboard\Mimir.EveDashboard.csproj -- --port 8795
```
Canonical surface addresses:
```text
asgard.starfire.mimir/eve/tui
asgard.starfire.mimir/eve/gui
```
Native CultNet route for typed dashboard state:
```text
cultnet://192.168.1.66:3075
schema: mimir.eve_dashboard_state.v1
record: eve.dashboard.broker
```
EveCanvas currently connects through this bridge route:
```text
ws://192.168.1.66:8795/eve/deck
```
Periwinkle's Android Eve client currently uses this bridge route:
```text
ws://192.168.1.66:8795/eve/deck/cultmesh
```
The compatibility endpoint remains:
```text
ws://192.168.1.66:8795/eve/dashboard
```
Broker inspection endpoints:
```text
http://192.168.1.66:8795/health
http://192.168.1.66:8795/eve/deck/manifest
http://192.168.1.66:8795/eve/deck/providers
ws://192.168.1.66:8795/eve/deck/{providerId}
```
Extra providers can be registered at launch:
```powershell
dotnet run --project .\src\Mimir.EveDashboard\Mimir.EveDashboard.csproj -- `
--port 8795 `
--voidbot-swarm-state "E:\Projects\VoidBot\.voidbot\status\swarm-state.json" `
--provider "app.id|App Title|asgard.starfire.app/eve/tui"
```
If a provider still needs a WebSocket bridge, list it in the provider
advertisement or supervisor manifest as a route, not as the provider address.
## Current State
`Mimir.EveDashboard` now runs as a small native dashboard broker. It includes:
- `eve.dashboard.broker`: the switchboard rendered as provider cards.
- `mimir.live.stats`: compact live telemetry surface for RMS bars, sync
confidence, runtime buffers, actuator commands, and Periwinkle/EVE device
observation streams. It recomputes state from Mimir's telemetry and
observation ledgers and is broadcast once per second to connected clients.
- `mimir.stream.layout`: the existing Mimir source-layout fixture provider.
- `voidbot.swarm`: the native VoidBot tab. It reads the existing
`swarm-state.json` projection from VoidBot, publishes avatar image URLs on
identity nodes, and lets Eve render the CTB rail, selected Face status panel,
state tree, and detail pane as native UIKit. It is read-only in this cut;
scheduler mutations still belong to VoidBot's heartbeat controls.
- `yggdrasil.streampixels.edge`: the first Yggdrasil/StreamPixels service
dashboard placeholder for the TCP/SSH-routed live edge.
The stream-layout provider still serves fixture scene nodes for the first native
control proof. It accepts transform/visibility commands, mutates the provider
state, increments the state version, and broadcasts the new snapshot. The next
cut is replacing fixture state with live `MimirPresentationControlState` and
`MimirSceneEditorState` snapshots.
The live-stats provider is read-only. Its authority is the runtime telemetry
surface: `mimir-spectrum`, `mimir-sync-state`, `mimir-sync-report`,
`mimir-video-buffer`, `mimir-audio-actuator-command`, and the normalized
CultMesh observation ledger from `Mimir.EveSensorReceiver`. If a signal is
absent, the surface says so; it does not fabricate confidence.
## Verification
The command smoke connected to `/eve/dashboard`, received state version `1`,
sent:
```json
{"type":"move","nodeId":"eve-camera","x":0.25,"y":-0.25}
```
and received state version `2` with `eve-camera` moved to that transform. During
the same smoke, EveCanvas connected from `192.168.1.72` and remained running.
The first device screenshot proved the dashboard rendered on Eve, but also
showed the legacy debug overlay occluding the hierarchy panel. EveCanvas hides
that overlay while dashboard mode is active; dashboard status remains in the
native dashboard footer.
The broker smoke on 2026-05-31 verified:
- `/health` reports active provider, client count, transport, and typed
CultMesh dashboard document name.
- `/eve/deck/providers` returns the built-in provider manifests.
- WebSocket `/eve/deck` sends the switchboard snapshot.
- WebSocket `/eve/deck/cultmesh` sends the same provider state as
`mimir.eve_dashboard_state.v1` MessagePack and accepts
`mimir.eve_dashboard_command.v1` MessagePack command documents.
- `open-provider` switches to `mimir.stream.layout` and broadcasts that
provider state.
- `open-provider` switches to `voidbot.swarm`, which emits VoidBot's swarm
status, CTB rail with avatar URLs, agent cards, and selected Face state
detail from the existing VoidBot swarm snapshot.
## Surface Composition Contract
`dashboard-state.surface` is the new distributed UI layer. It is intentionally
closer to CultUI than to the old flat dashboard node list:
- `surface.schema`: versioned wire contract, currently
`cultmesh.eve_surface.v0`.
- `surface.root`: retained element tree. Elements have `kind`, optional `role`,
text, asset refs/URIs, provider command ids, layout hints, style variants,
metrics, and children.
- `surface.assets`: cacheable media references such as avatar images.
- `bindNodeId`: optional bridge back to the compatibility `nodes` projection.
- `commandId`: provider-owned intent id. Eve clients invoke commands; providers
accept or reject and publish the next snapshot.
- `binding`: optional typed CultCache/CultMesh field binding. It names
`documentSchema`, `documentId`, field `path`, `valueKind`, `access`,
`authority`, and optional `commandId`. This is the Verse-backed field
contract; native callbacks and DOM handlers are only lowering actuators.
The old `nodes` array is no longer expected to carry complete UI composition.
It is a compatibility/debug/data surface. Clients that understand `surface`
should render the composed tree first and use `nodes` only for selection,
legacy node binding, and fallback. `bindNodeId` is compatibility glue, not the
portable state owner.
The binary CultMesh state document mirrors this ownership: field 7 carries the
optional `surface` tree. Older clients can stop at the node projection; current
Eve clients should prefer `surface.root` whenever it is present.

View file

@ -0,0 +1,84 @@
# EVE Program Output
## Authority Map
- Owner: Fensalir owns the visible D3D12 program pixels.
- Inputs: Fensalir reads the current `AquariumFrame`, render plan, GPU resources,
and overlay state.
- Outputs: a named shared D3D12 `B8G8R8A8_UNorm` texture containing the completed
backbuffer for the current frame.
- Derived state: Mimir publication profiles describe the stream contract; they
do not render pixels. EVE receives decoded pixels and composites them.
- Forbidden writers: WebKit, DOM layout, browser dashboards, OBS capture, and
desktop/window capture do not own this path.
- Shared paths: local display and EVE streaming both use the same completed
Fensalir backbuffer copy.
- Deletion line: diagnostic Sunshine/Moonlight or desktop-duplication probes may
prove network/decode budget, but they are not production authority.
## Current Mechanism
Set these before launching the Mimir app:
```powershell
$env:FENSALIR_PROGRAM_OUTPUT_D3D12 = "1"
$env:FENSALIR_PROGRAM_OUTPUT_NAME = "Global\MimirFensalirProgramTexture"
$env:FENSALIR_PROGRAM_OUTPUT_FENCE_NAME = "Global\MimirFensalirProgramFence"
$env:FENSALIR_PROGRAM_OUTPUT_RING_COUNT = "1"
dotnet run --project .\src\Mimir.App\Mimir.App.csproj
```
Fensalir creates a shareable D3D12 texture with that name, publishes a named
program-output fence, and copies the finished backbuffer into the texture during
the present pass. `Mimir.EveRelay` opens that named texture, reads the completed
program frame, feeds a low-latency H.264 encoder, and sends complete Annex-B
access units to EveCanvas over the existing `/stream` WebSocket. EveCanvas
treats JPEG frames as the old VoidBot dashboard fallback only; Mimir dashboard
streaming is `h264-annexb` and displays through a native
`AVSampleBufferDisplayLayer` so the iPad owns hardware decode/composite.
The EVE relay currently consumes the single-texture contract. Keep
`FENSALIR_PROGRAM_OUTPUT_RING_COUNT=1` for EVE until the relay has the same
slot-selection contract as the OBS program texture source.
`scripts/start-eve-dashboard-stream.ps1` starts the Mimir/Fensalir app with the
shared-texture output enabled, starts the H.264 relay, verifies the `eve` SSH
target, opens a reverse SSH tunnel from Eve's `127.0.0.1:8792` back to the
local relay when direct inbound TCP is blocked, and launches EveCanvas:
```powershell
powershell -ExecutionPolicy Bypass -File .\scripts\start-eve-dashboard-stream.ps1
```
The relay defaults to `h264_nvenc`. Use `-Encoder libx264` only as a diagnostic
fallback when no NVENC-capable FFmpeg is installed.
EveCanvas tries `ws://127.0.0.1:8792/stream` first, then falls back to
`ws://192.168.1.66:8792/stream`. The localhost path is not JPEG tunneling; it is
the same H.264 Annex-B stream carried through SSH port forwarding so a Windows
firewall rule cannot silently turn the dashboard into a wall.
## Receiver Contract
EVE owns:
- network receive;
- hardware decode of H.264 access units;
- texture upload/import;
- final fullscreen composite;
- touch/Pencil/sensor uplink later.
EVE does not own:
- dashboard DOM;
- Chromium/WebKit compatibility;
- layout parity;
- Mimir timing authority;
- OBS composition authority.
## Smoke
```powershell
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --eve-program-output-contract-smoke
dotnet build .\src\Mimir.EveRelay\Mimir.EveRelay.csproj --no-restore --disable-build-servers
```

View file

@ -0,0 +1,511 @@
# Fensalir Rendering Rebuild Migration
## Objective
Keep Mimir from becoming a second renderer while Fensalir rebuilds itself into
the field-evidence machine Mimir needs.
Mimir's job is to make the room measurable: cameras, microphones, speakers,
loopbacks, phones, Raven, and Starfire all feed one bounded temporal window.
Fensalir's job is to turn that measured evidence into a coherent visible field
and program surface. OBS should receive the final result, not be asked to
pretend loose feeds are a world.
## Current Mechanism
Mimir currently has the correct broad cut:
```text
drivers / ASIO / network feeds
-> Mimir.Runtime rolling buffers
-> synchronization, calibration, response models
-> Fensalir scene state and debug surfaces
-> Fensalir D3D12 presentation / Spout / program output
-> OBS
```
The old direct debug-spectrum pressure leak has been cut. Current spectrum
debug output is already in the production-shaped bridge:
```text
rolling buffer slice + source identity + calibration + surface intent
-> Fensalir field claim
-> TubeField candidate producer, if that is the selected claim family
-> selected lowering under budget
-> evidence lanes and temporal guide
-> presentation
```
Mimir must not send "draw these lines" as a production order. It should say
"this rolling buffer slice is a spectrum field with stable axes, identities,
support, and material intent." Fensalir then chooses direct tube SDF, mesh,
field splats, or another lowering.
The shared Fensalir spatiotemporal reservoir remains the rendering organ.
TubeField is only the current claim/candidate producer for rolling-buffer
spectrum surfaces. It may own tube-specific resource binding, Catmull-Rom
evaluation, SDF/proxy geometry, and material sampling, but it must not become a
private renderer or final composition authority.
This is an intentional divergence from pure ReSTIR. Fensalir is not only
resampling lighting after a known primary surface; it is often sampling the
visible field candidate itself. TubeField therefore has to generate local
primary tube-visibility candidates before reservoir reuse. The shared reservoir
owns the temporal-antialiasing-shaped decision about which field candidate persists into
presentation. There is no separate TAA owner in the target architecture:
reservoir resolve reconstructs smooth presentation directly from selected field
evidence instead of repairing unstable pixels after the fact.
## Invariants
- Mimir owns raw retention. Fensalir may snapshot or import the five-second
window, but it does not silently become history owner.
- Mimir owns physical provenance: device ids, sample clocks, calibration ids,
path response, confidence, network arrival, and decoded timing evidence.
- Fensalir owns dense visual fusion, temporal field evidence, selected cuts,
residency, D3D12 resources, debug UI, and program video.
- Faust/native DSP owns hot audio movement, resampling, alignment, separation,
spatialization, and stems.
- OBS owns broadcast composition only.
- Render packets are cached lowerings. They are not scene truth.
- Reservoir resolve owns temporal antialiasing and reconstruction. TAA is not a
separate source-identity or presentation-history authority.
- Networked phones and Raven may publish local observations and decoded timing
state. They do not become independent clock authorities.
## Authority Map
Owner:
- `Mimir.Runtime` owns stream identity, rolling buffers, synchronization state,
calibration loading, and observation publication.
- Fensalir's future `FieldEvidenceMachine` owns field claims after Mimir
publishes observations and intent.
Inputs:
- local direct-driver video frames;
- local ASIO loopback and mic blocks;
- Raven/phone/network observations;
- calibration receipts, response surfaces, codebooks, path ids, sensor poses;
- user/app policy for what should enter the live field.
Outputs:
- typed observation surfaces for Fensalir;
- timing, response, and calibration constraints;
- audio actuator control for Faust/native DSP;
- health/telemetry state for UI and logs;
- final program output from Fensalir and stems from Faust/native DSP.
Derived state:
- spectrum lines, debug views, screenshots, OBS bridge endpoints, FFmpeg/SRT
helpers, and Eve dashboard feeds are views or diagnostics.
- The old direct spline-spectrum dashboard, direct GPU sensor frame bridge, and
direct acoustic field frame proof are silent from Mimir's active bridge path.
FieldEvidence owns spectrum, camera observation publication, and calibration
constraints; screenshots and logs remain diagnostics.
Forbidden writers:
- any Mimir-side D3D12 render graph;
- app-specific stable Gaussian/evidence cache;
- JSON/base64 hot path for live media;
- OBS raw source timing as synchronization authority;
- per-debug-view geometry deciding production truth.
Shared paths:
- live cameras, audio timing anchors, acoustic response surfaces, spectrum
diagnostics, calibration events, and future volumetric room claims all publish
typed observations/constraints into the same Mimir-to-Fensalir bridge.
Deletion line:
- If a Mimir feature wants visual output, first define the observation, claim,
or surface intent. A direct draw is allowed only when it is named as fallback
or diagnostic and cannot override field evidence.
## Target Mimir -> Fensalir Contract
### 1. Raw Window
Mimir keeps one retention authority:
```text
RollingStreamWindow {
windowId
duration = 5 seconds by default
streamId
sourceKind
sampleDescriptors
nativeHandle or payload view
deviceTimestamp
canonicalTimestampEstimate
sequenceId
status
}
```
Typed views may index this window. They do not own retention.
### 2. Observation Surface
Mimir publishes physical evidence, not render commands:
```text
MimirObservation {
observationKey
streamId
sensorId
calibrationId
modality: camera | audio | network | timing | response
observedTime
canonicalTimeEstimate
uncertainty
payloadHandle
provenance
confidence
}
```
The payload handle is not authority. It must resolve through a declared
resource before shader lowering can consume it.
### 2.5. Resource Declaration
Live payloads have an explicit Fensalir resource contract:
```text
FensalirResource {
resourceKey
kind: structured-buffer | texture2d | rolling-texture | mesh | surface-page | volume
residency: gpu-resident | shared-gpu
shaderAccess: SRV | UAV | vertex-buffer | index-buffer | indirect-args
format
dimensions / count / stride
nativeHandle and nativeHandleKind
valid time range
version / sequence
}
```
Mimir may mint stable resource keys from native/GPU handles, but Fensalir owns
resolution. A DSL packet may reference `mimir:resource:*`; it may not treat a
bare string as payload truth. In the same runtime, a Mimir-declared buffer is a
Fensalir resource slot once the declaration is accepted; native/shared handle
metadata is an import edge, not a separate payload authority. Rendering-relevant
buffers move to GPU residency as early as possible and stay there; Fensalir
passes resource handles/SRVs/UAVs between compute and render lowerings instead
of copying samples back to CPU. If the resource contract is absent, expired,
duplicated, CPU-only, or incompatible with the selected backend, lowering must
defer instead of rendering a lie with nice lighting.
Current proof surface:
- `Mimir.BufferSmoke --fensalir-field-evidence-smoke` maps a Mimir rolling
window/observation/spectrum intent into one declared resource, three claims,
one planned resource-backed `TubeField` packet, and two deferred non-backend
claims.
- `Mimir.BufferSmoke --fensalir-field-dsl-resource-smoke` uses Fensalir's field
evidence DSL to bind the declared resource directly and produce one planned
`TubeField` packet with no deferred requests.
- `Mimir.BufferSmoke --fensalir-camera-observation-smoke` maps one
GPU-backed video frame and one metadata-only cadence witness into field
evidence. Only the shared D3D12 video handle becomes a declared `Texture2D`
resource and camera surface intent; the metadata-only frame remains
observation evidence with no render payload.
Fensalir now owns the first D3D12 resource resolver cut below this contract:
shared structured/curve buffers import/alias GPU-resident resources by handle,
while Fensalir-created resources allocate GPU slots only when Fensalir is the
producer. The DSL can describe a 2D rolling float buffer as Catmull-Rom XY
tubes with modulo column addressing, amplitude power/normalization, radius,
ramp texture path, and emission scale. TubeField now consumes those resources
through a GPU compute/render path, local Texture2D resources can bind as ramps,
SurfacePage and VolumeTexture declarations resolve into shader-readable GPU
textures, and Mesh declarations package `Mesh.Vertices`/`Mesh.Indices` GPU
buffer shape under one resource key. The evidence DSL can now plan generic
resource-backed claims over those declarations. Current blocker for the
remaining visual surface is not resource ownership; it is the selected render
lowerings that interpret mesh/page/volume resources as geometry, height/SDF/
material pages, or density/extinction/SDF3D domains. Mesh layout authority is
split by source: imported/user meshes use the standard `PositionNormalUvColor`
layout, while generated meshes can be `PipelinePrivate` and leave byte
semantics to their selected lowering. TubeField now uses that generated-mesh
lane explicitly: compute emits private geometry/indirect buffers and render
binds them as a pipeline-private generated mesh before applying TubeField
material state. The generated-mesh DrawIndexed indirect command signature is
shared ABI; TubeField only owns the argument buffer it emits. TubeField
expansion is now gated by selected backend packets, so explicit lowering
metadata cannot bypass validation/planning authority. Validation also rejects
TubeSpline metadata whose claim is not Tube-encoded or whose resource differs
from the claim payload. Mimir's typed surface-intent lowering now emits the
matching TubeSpline metadata for audio spectrum/waveform Tube claims. That is
engine ownership, not a reason for Mimir to create a parallel renderer.
Mimir also uploads normalized spectral frames as Float32 data into a
Fensalir-owned structured buffer resource, so the TubeField path consumes live
GPU buffer contents rather than an empty declaration. The runtime-level receipt
is `Mimir.BufferSmoke --mimir-spectrum-upload-smoke`: it boots Mimir's frame
path, advances spectrum analysis, and verifies one newest-slot Float32 upload,
planned TubeField packets, a resource-bound local blackbody ramp Texture2D, and
no live legacy spline/buffer-field dashboard input. The runtime resource is no
longer a latest-frame row dump: it is a rolling column matrix where physical
columns are flattened `(history age, source lane)` pairs under fixed history
and source-lane capacities. Mimir emits one TubeField claim per active source
lane; lowerings use the fixed lane capacity as `ColumnStride`,
`ColumnStep.z=0.1`, and `RollingOffset` so each source renders its own age
trail from newest to older samples without reshuffling the GPU buffer. New
frames upload only the newest ring slot with an explicit element offset.
Fensalir owns structured-buffer slot validity and clamps TubeField dispatch so
invalid older slots do not reach shader sampling after allocation, reset, or
partial update. TubeField metadata now carries stable claim-derived field ids
inside the TubeField family range, so temporal/reprojection/reservoir consumers
can distinguish source-lane claims. TubeField render now binds the engine
blue-noise texture and jitters the tube-local SDF sample inside the proxy
fragment shader before writing coverage, normals, and reservoir guide data; the
same pass also uses the scene depth buffer so opaque generated field geometry is
resolved by nearest surface rather than submission order. The current generated
proxy visual still exposes segment structure and is not yet the final smooth
tube marcher. Mimir exposes
`MIMIR_SPECTRUM_SOURCE_LANES` as the source-lane capacity lever and
`MIMIR_SPECTRUM_TUBE_SUBDIVISIONS` as the geometry cost lever for responding to
Fensalir's requested/dispatched/truncated/invalid TubeField budget report.
Camera descriptors now follow the same evidence/resource boundary:
`MimirFensalirFieldLowering.BuildCameraObservationFrame` lowers latest video
buffers into observation claims and declares GPU-backed frames as shared
`Texture2D` resources. Metadata-only/process-cadence frames do not create
surface intents, so a missing payload cannot become a fake `latest-window`
render request. Fensalir can now resolve those shared `Texture2D` declarations
by native D3D12 handle and accepts Mimir video format names such as `Bgra8`,
`Gray8`, `Rg8`, `Nv12`, and `LeapStereoIr`. Camera image claims currently
defer because Fensalir has not selected a visual-fusion/render lowering for
camera textures yet.
### 3. Calibration Constraint
Calibration and sync become explicit constraints:
```text
MimirCalibrationConstraint {
constraintKey
pathId
sourceId
receiverId
evidenceKind: loopback | bioacoustic | passive | visual | network
delayEstimate
delayUncertainty
phaseOrGroupDelay
frequencyResponse
usableBandMask
confidence
}
```
### 4. Surface Intent
Debug and production visualization share intent:
```text
MimirSurfaceIntent {
intentKey
sourceObservationKeys
domain
axes
supportPolicy
materialGraph
updateBudget
purpose: debug | calibration | production
}
```
The spectrum dashboard becomes one instance of this: a rolling audio spectrum
field whose frequency axis maps to X, amplitude to Y, age to Z, and stream
identity to stacked lanes.
## Fresh Dataflow
```mermaid
flowchart TD
A["direct camera drivers"] --> B["Mimir rolling window"]
C["ASIO loopback + mics"] --> B
D["Raven / phones / network"] --> B
B --> E["sync + calibration kernel"]
E --> F["typed observations"]
E --> G["calibration constraints"]
B --> H["surface intent"]
F --> I["Fensalir field evidence machine"]
G --> I
H --> I
I --> J["field claims + selected lowerings"]
J --> K["program video + debug UI"]
E --> L["Faust/native DSP controls"]
L --> M["program stems + spatial bed"]
K --> N["OBS"]
M --> N
```
## Migration Plan
### Phase 0: Silence Legacy Debug Surfaces
The direct `AquariumSplineFrame` spectrum dashboard and point-splat spectrum
path are no longer scene writers. Live spectrum rendering flows through
FieldEvidence TubeField claims and generated mesh expansion.
### Phase 1: Add Bridge DTOs In Mimir.Runtime
Create pure mapping types for:
- rolling stream windows;
- resource declarations for live GPU/native payload views;
- observations;
- calibration constraints;
- surface intent.
The bridge must not read devices, render, allocate per sample, or own timers.
It maps existing runtime state into Fensalir-facing contracts.
Acceptance:
- one synthetic buffer can produce observations and surface intent in a unit
test;
- claims that reference live payloads also declare a typed Fensalir resource;
- the mapping can run repeatedly without growing allocations;
- stream/source ids remain stable across frames.
### Phase 2: Move Spectrum Dashboard To Surface Intent
Represent the current spectrum view as a `MimirSurfaceIntent`:
- stable source lanes;
- stable frequency bins;
- monotonic history sequence ids;
- Z as continuous sample age;
- tube/material intent from amplitude/confidence.
Fensalir may still lower it to direct spline tubes at first. The ownership
changes before the backend does.
Acceptance:
- app code no longer chooses point splats, direct tubes, or mesh ribbons;
- row/band/history identity is the same whether the engine lowers direct tubes
or another representation.
### Phase 3: Camera Observation Bridge
Lower native camera descriptors into observation surfaces:
- source id and calibration id;
- dimensions, format, stride;
- native CPU/GPU handle;
- device timestamp and canonical estimate;
- lens model and confidence/status.
Acceptance:
- implemented bridge proof: GPU-backed video descriptors become declared
shared `Texture2D` resources plus camera observation/surface claims;
- metadata-only cadence descriptors remain observation evidence and do not
become render payloads;
- no selected camera-image lowering exists yet, so Fensalir defers those claims
until visual fusion/feature extraction owns them.
### Phase 4: Audio Field Observation Bridge
Lower sync and calibration state into audio field constraints:
- ASIO loopback clock fit;
- mic delay and SRO state;
- per-path frequency response and group delay;
- bioacoustic anchor clusters;
- passive correlation confidence;
- Raven/phone decoded timing observations.
Acceptance:
- implemented bridge proof: sync states lower into FieldEvidence calibration
constraints instead of direct `AquariumAcousticFieldFrame` packets;
- Fensalir can inspect acoustic confidence and constraints without receiving
raw PCM as production truth;
- selected bridge lowering now exists for audio-path `Phase` and `Confidence`
claims: Fensalir plans those constraints as `DebugOverlay` backend packets so
calibration timing/path evidence is visible through the FieldEvidence
authority instead of a parallel acoustic frame;
- acoustic source candidates, confidence volumes, and cross-modal fusion remain
Phase 5 work;
- Faust/native DSP remains the actuator for sample movement.
### Phase 5: Sensor Fusion Claims
Let Fensalir fuse Mimir observations into field claims:
- sparse feature tracks first;
- calibrated camera rays and marker/feature candidates;
- acoustic source candidates and confidence volumes;
- surface/material claims only after cross-modal evidence earns them.
Acceptance:
- implemented synthetic multi-camera marker proof: Mimir lowers a deterministic
camera marker candidate with calibration/marker identity and a world-space
confidence envelope, and Fensalir plans it as a `DebugOverlay` FieldEvidence
packet;
- implemented synthetic audio source proof: Mimir lowers an SRP/PHAT-style
source candidate with calibration/source identity and a world-space confidence
envelope, and Fensalir plans it as a `DebugOverlay` FieldEvidence packet;
- claims can be inspected by source observation and calibration id.
### Phase 6: Program Output Receipts
Field identity now reaches the OBS-facing program boundary:
- Fensalir publishes the completed program surface through named shared D3D12
textures and `Global\MimirFensalirProgramFence`;
- `Mimir Program Texture` opens those textures in OBS and performs the explicit
D3D12-to-libobs-D3D11 GPU copy without CPU readback or Spout2;
- optional texture rings and `Global\MimirObsProgramConsumerFence` prevent blind
producer/consumer reuse;
- `Mimir Audio Stem` receives separately controllable Faust/native DSP stems
from `Local\MimirObsStemBus`;
- telemetry and receipts record timing, evidence confidence, dropped/late
observations, and current selected cuts.
Acceptance:
- OBS sees final program surfaces/stems through the Mimir OBS plugin;
- raw feeds remain debug inputs, not synchronization truth;
- recorded receipts explain the stem path, installed OBS plugin, live program
texture proof, optional ring proof, dedicated producer fence proof, and
two-way consumer fence proof.
## Verification Matrix
| Check | Expected Result |
| --- | --- |
| Stable buffer identity | Source lanes do not reorder or phase-rotate by render frame. |
| Missing data | No sample means no observation; no zero-fill masquerade. |
| Direct debug path | Can be disabled without changing runtime timing state. |
| Observation bridge | Produces typed evidence without device reads or rendering. |
| Fensalir lowering swap | Direct tube vs future splat/mesh lowering preserves claim ids. |
| Reservoir guide | Presentation history follows evidence validity, not producer folklore. |
| OBS output | OBS receives final program surfaces, not raw clock chaos. |
## Research And Maps
Primary local references:
- `docs/perfect-machine.md`
- `docs/perfect-machine-domain-index.md`
- `docs/sensor-fusion-architecture.md`
- `research/perfect-machine-study-2026-05-23/fensalir-integration-map.md`
- `research/visual-spatial-map/brushstroke-fusion-rendering.md`
- Fensalir `docs/rendering-teardown-rebuild-protocol.md`
- Fensalir `docs/perfect-machine-architecture.md`
- Fensalir `docs/temporal-spatial-evidence-reservoir.md`
The next implementation pass should cut Phase 1 first. A bridge DTO that owns
nothing but mapping is boring in the best possible way: it makes the rest of
the machine harder to lie about.

View file

@ -0,0 +1,115 @@
# Fensalir Reservoir Daemon
## Objective
Move heavy field reconstruction out of the interactive Mimir editor. The editor
should present synced video feeds and controls responsively. A separate
Fensalir reservoir daemon should drink from the Well, spend whatever CPU/GPU
budget is assigned, and publish typed reservoir/program surfaces.
## Authority Map
Owner: the Fensalir reservoir daemon owns reservoir work selection, GPU
residency, temporal reuse, fusion kernels, worker scheduling, and publication of
reservoir-derived program surfaces.
Inputs: typed Well pages, configured composite state, live timing policy,
feature-signal documents, calibration state, Fensalir-owned texture leases, and
explicit budget settings.
Outputs: typed CultMesh/CultCache reservoir status, program texture/fence
surfaces, optional diagnostic Eve surfaces, and OBS-facing shared texture or
plugin feeds.
Derived state: Mimir editor telemetry, preview rectangles, Odin/Eve dashboard
rows, and OBS plugin status are observers. They do not decide reservoir work.
Forbidden writers: the interactive editor render loop must not run temporal
reservoir history just because it is presenting a backbuffer. Field diagnostics
must be opt-in. A camera `FieldEvidenceFrame` claim must not impersonate a flat
video compositor.
Shared paths: direct user placement, configured Well composites, OBS output,
daemon diagnostics, and future Eve room lowerings must all reference the same
source ids, timing state, and placement/composite document. Manual UI placement
and daemon program output may differ only through an explicit selected
composite/version.
Deletion line: old app-hosted reservoir proof visuals remain diagnostic only.
They cannot be the default Mimir program output while the current goal is synced
video presentation.
## Current Scar
On 2026-06-02 the Mimir editor became unresponsive because Fensalir ran
reservoir history presentation on a video-only frame. The renderer now gates
that pass behind source-derived reservoir-producing input, and the live
video-only app log showed `reservoir-update n/a`.
The remaining gap is real: Fensalir still lacks a flat program-layer video
backend that draws Fensalir-owned camera texture leases as 2D synced layers.
That backend is separate from the reservoir daemon. Build it as the editor's
presentation path; do not smuggle it through field diagnostics.
## Current Daemon Cut
`src/Mimir.FensalirDaemon` is the first owner-shaped daemon:
- eats CultCache through `state/fensalir-daemon.ccmp`, using the canonical
MessagePack CultCache backing store;
- drinks Well JSONL logs containing `mimir.well_snapshot.v1`,
`mimir.well_capture_page.v1`, and `mimir.well_stream_pressure.v1`;
- publishes typed `mimir.fensalir_daemon_state.v1` state;
- speaks a CultNet-shaped WebSocket Eve provider at `/eve/deck`;
- speaks binary CultMesh dashboard documents at `/eve/deck/cultmesh`;
- exposes `/health` and `/eve/deck/manifest`;
- presents a retained `cultmesh.eve_surface.v0` surface with compact text and
metrics that Eve GUI and Odin/Nightwing TUI can lower.
Start it with:
```powershell
powershell -ExecutionPolicy Bypass -File .\scripts\start-fensalir-daemon.ps1
```
The launcher writes a supervisor manifest under
`artifacts/runtime/fensalir-daemon-*` with a semantic provider spec and bridge
routes:
```text
mimir-fensalir-daemon|Mimir Fensalir Daemon|asgard.starfire.mimir-fensalir/eve/tui
bridge: ws://127.0.0.1:8799/eve/deck
bridge: ws://127.0.0.1:8799/eve/deck/cultmesh
```
The current worker mode is `surface-owner-installed`: kernel scheduling,
GPU-resident reservoir compute, and actual fused program-surface publication
belong behind this daemon next. Do not move those back into the editor.
## Worker-State Cut
`Mimir.FensalirDaemon` now also publishes
`mimir.fensalir_reservoir_worker_state.v1` beside the daemon summary state.
The worker state is the owner surface for reservoir job selection and pressure:
- capture pages enqueue reservoir jobs only when they contain accepted timing
slices;
- `mimir.cultmesh_stream_frame.v1` records enqueue fine-grained reservoir jobs
from timed stream frames, including paged, inline, native, or shared-handle
body transport evidence;
- unusable pages are dropped with rejection reasons instead of being hidden by
the editor;
- queue depth, running jobs, completed jobs, dropped jobs, timing confidence,
accepted/rejected slice counts, and estimated GPU queue milliseconds are
persisted in CultCache;
- `mimir.fensalir_reservoir_pressure.v1` reports Well poll/publish pressure,
capture bytes, ready-slice ratio, timing confidence, max presentation
distance, GPU queue milliseconds, and drop causes for missing bodies,
unavailable fences, or degraded timing;
- Eve surfaces bind worker metrics to the worker document, not to ad hoc
renderer telemetry.
The current worker still simulates cost from capture-page shape and configured
CPU/GPU share. That is intentional scaffolding: the next cut replaces the cost
model with real GPU/Faust/Fensalir kernel dispatch while preserving the same
owner and state contract.

File diff suppressed because it is too large Load diff

View file

@ -48,10 +48,9 @@ Mimir exists to turn cameras, microphones, speakers, timing signals, and
distributed capture surfaces into one realtime coherent volumetric field. The
field is the product. The bridge scripts are scaffolding and probes.
The live target is one native five-second rolling reservoir feeding Mimir-owned
program composition, Fensalir GPU fusion, Faust/native DSP, operator
surfaces, and a Yggdrasil site publisher. OBS is compatibility only. Retention
has one owner. Typed views are indexes, not private histories.
The live target is one native five-second rolling reservoir feeding Fensalir GPU
fusion and Faust/native DSP, then OBS. Retention has one owner. Typed views are
indexes, not private histories.
## Operating Doctrine
@ -64,14 +63,11 @@ the synchronized program path is being rebuilt around `LocalcastRuntime`,
Invariants:
- Mimir owns final program composition, source subscription, preview/control
state, stats, and publication intent.
- Fensalir owns dense visual fusion, material/brush/splat rendering, and local
program texture output.
- OBS owns final broadcast composition.
- Fensalir owns dense visual fusion, material/brush/splat rendering, and Spout2
publication.
- Faust/native DSP owns hot audio alignment, room suppression, voice separation,
stems, and spatial bed generation.
- The Yggdrasil publisher daemon ships the Mimir program to the site without
owning composition.
- FFmpeg owns simple capture, encode, and transport in V1 bridge mode.
- Mimir/this repo owns configuration, calibration, launch/status, typed
contracts, memory, and operational repeatability.

View file

@ -1,103 +0,0 @@
# Mimir Program Composition
## Objective
Mimir owns the live stream program: selected capture feeds, calibration feeds,
scene composition, preview/control surfaces, stats, and publication. OBS is a
temporary compatibility sink only.
## Authority Map
- Owner: Mimir owns the program scene graph and subscription policy.
- Inputs: Muninn-published local and remote monitor, window, camera, audio,
loopback, and health streams.
- Outputs: a Mimir program video surface, aligned audio stems or final bus,
and a Yggdrasil-facing site publication feed.
- Derived state: OBS receives a compatibility feed when useful; it does not own
scene truth, broadcast truth, preview truth, stats, source selection, or
publication.
- Forbidden writers: OBS scenes, browser dashboards, site daemons, and Muninn
capture hosts must not independently decide the program composition.
- Shared paths: direct operator edits, remote commands, preset loads,
calibration-driven source selection, and recovery scripts must commit through
the same Mimir scene graph command surface.
- Deletion line: old OBS-owned composition docs and config names are demoted to
diagnostic bridge language. Any OBS adapter delegates to the Mimir program
scene instead of carrying its own opinion.
## Machine Shape
```mermaid
flowchart TD
A["Muninn on Starfire"] --> D["CultMesh stream catalog + media bodies"]
B["Muninn on Nightwing"] --> D
C["Muninn on Raven"] --> D
D --> E["Mimir subscription policy"]
E --> F["Mimir rolling buffers"]
F --> G["Mimir program scene graph"]
G --> H["Fensalir compositor"]
F --> I["Faust/native DSP"]
H --> J["program video surface"]
I --> K["program audio stems/bus"]
J --> M["Yggdrasil site publisher daemon"]
K --> M
M --> N["GameCult site"]
J -. "compatibility" .-> O["OBS adapter"]
K -. "compatibility" .-> O
```
## Composition Contract
The scene graph carries layers with:
- source reference, such as `muninn:starfire:monitor:primary` or
`muninn:starfire:window:discord`;
- source kind, such as monitor, window, camera, network monitor, or generated
Mimir visual field;
- visibility, layer order, transform, crop, and optional chroma key;
- enough timing identity for Mimir to align media inside the rolling window.
The current OBS scene can be mirrored as an initial Mimir scene:
- Starfire monitor capture, cropped at the bottom when needed.
- Discord window capture, cropped to chat and keyed with chroma filters.
- Raven primary monitor plus Realtek loopback as a subscribed remote Muninn
stream.
- Mimir/Fensalir program texture as a generated layer when the field renderer is
active.
After import, OBS is not the editor of record. It may display the Mimir program
feed, but operator edits happen through Mimir command surfaces.
## Site Publisher
The Yggdrasil-facing publisher consumes Mimir program output and publishes it to
the GameCult site. It owns transport, encoding/session state, reconnect policy,
and public edge health. It does not own composition.
The publisher's input contract is the same Mimir program output used by local
preview and OBS compatibility adapters, so the public stream cannot silently
diverge from the operator preview.
## Runtime Contracts
The first typed contracts live in `src/Mimir.Runtime/Synchronization`:
- `mimir.program_scene.v1`
- `mimir.program_output.v1`
- `mimir.eve_operator_surface.v1`
The proof path is:
```powershell
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --perfect-machine-contract-smoke
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --perfect-machine-profile-smoke
```
Import the current OBS scene as Mimir-owned typed state:
```powershell
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --import-obs-program-scene --output state\mimir-program-composition.cc
```
The live imported state file is `state/mimir-program-composition.cc`.

View file

@ -1,107 +0,0 @@
# Move Calibration Protocol
Mimir owns Move calibration and sensor fusion. Muninn owns source-local facts:
camera marker candidates, USB controller IMU/button state, and local Move light
commands. Odin discovers and projects the surfaces. Fensalir consumes Mimir's
resolved poses after Mimir has earned them.
## Current Stream Shape
- Starfire should run Muninn as a local capture/control daemon. Its Move stream
address is `muninn:starfire:move-evidence`.
- Nightwing runs Muninn for its USB Move and local camera witnesses. Its Move
stream address is `muninn:nightwing:move-evidence`.
- Mimir on Starfire publishes resolved pose frames at
`mimir:starfire:move-controller-poses`.
- Quest can participate as an external tracked VR reference over USB:
`quest:usb:headset-pose`, `quest:usb:left-controller-pose`, and
`quest:usb:right-controller-pose`.
- Starfire ASIO/Scarlett loopback can provide cue timing, but it is not a Move
pose authority.
The Move hot path is CultMesh stream frames. On the same host, those frames use
shared-memory bytes rings. Across hosts, the remote peer still crosses the
Verse/network boundary. The current Raven program media bridge is not same-host
zero-copy into Mimir/Fensalir; it sends CultNet/CultMesh media documents and
lowers to local UDP for OBS compatibility.
## Required Capture Phases
1. `preflight-streams`: hold both Moves still and visible; prove stream freshness,
clock skew, and controller ids.
2. `dark-stillness`: lights off, both Moves still; estimate gyro bias, gravity,
and magnetometer baseline.
3. `lit-stillness`: light one Move at a time, then both; map light commands to
controllers and measure centroid stability.
4. `axis-sweeps`: slow pitch/yaw/roll sweeps; fit gyro scale, accelerometer
frame, and optical angular consistency.
5. `figure-eight`: broad figure-eight motion; fit magnetometer hard/soft iron
and cross-axis coupling.
6. `quest-reference`: optional Quest headset/controller reference with the Move
between the two Quest controllers in front of the headset.
7. `validation-pass`: held-out free motion; promote only if triangulated optical
residuals and IMU prediction residuals pass.
## Quest Access Preflight
The Quest must be authorized for USB debugging before Starfire Muninn can query
it. Muninn owns the access surface:
```powershell
muninn serve --store C:\Meta\Odin\state\muninn.telemetry.cc --host starfire --quest-adb --quest-serial 1WMHHB68PG1515
```
When the Quest is authorized, Muninn publishes `muninn.quest_access.v1` and
advertises `muninn:<host>:quest-input`, `muninn:<host>:quest-poses`, and
`muninn:<host>:quest-warped-video-input`. Mimir consumes the access/pose
surfaces as calibration evidence; it does not own Quest USB access.
```powershell
muninn quest-access-status --store C:\Meta\Odin\state\muninn.telemetry.cc
```
ADB authorization proves USB access, device identity, and basic headset state
only. It does not expose Quest headset or controller poses; those still require
a Quest/OpenXR witness bridge that publishes pose samples through Muninn. The
current calibration protocol can still run without Quest, but Quest poses are
the preferred external reference when the headset and both controllers are
sitting around the Move.
## Data Products
- `mimir.move_fusion_rig_calibration.v1`: camera intrinsics/extrinsics, clock
association policy, and gyro unit scale.
- `mimir.move_imu_calibration.v1`: per-controller gyro bias/scale,
accelerometer gravity frame, magnetometer correction, and noise estimates.
- `mimir.move_controller_identity_map.v1`: stable association between Muninn
USB ids, light receipts, and optical orb identities.
- `mimir.move_calibration_receipt.v1`: sample counts, residuals, rejected
phases, and promotion decision.
## Optical Sphere Fit Authority
`MimirMoveSphereCalibrationFitter` owns the bounded two-Eye optical fit after a
collection session reaches `Fitting`. Eye 0 defines the tracking-space gauge.
The fitter jointly estimates both focal lengths and Eye 1's rigid pose from
same-frame, same-Move correspondences, using a deterministic 80/20 training and
held-out partition. A robust training consensus may reject contaminated marker
pairs, but promotion also requires at least 65 percent of the untouched
held-out pairs to agree and must pass the session's median and P95 reprojection
limits.
The physical orb radius is an explicit input; it is never guessed by the
solver. PSMoveAPI's reported blob radius is a color-segmentation result, not a
calibrated sphere silhouette. It may seed or weight depth only after the
per-camera radius response has been validated. A fresh PSMoveAPI position age
is also not optical confidence and must not be interpreted as one.
## Preflight Command
```powershell
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --move-calibration-protocol-smoke --output artifacts\move-calibration\protocol.cc
```
That writes the typed protocol as `mimir.move_calibration_protocol.v1` into a
CultCache receipt. The real hardware runner should consume this document plus
Muninn's `muninn.quest_access.v1` when Quest is available, then append capture
receipts instead of inventing a parallel checklist.

View file

@ -0,0 +1,321 @@
# Move Controller Stream
## Objective
Ship the first Mimir program surface around one concrete show shape:
Kiyo Pro is the RGB program view, PS3 Eyes track multiple PS Move spheres,
Raven screen capture enters as a synchronized video source, and Scarlett ASIO
provides the two hero mics plus Raven program-loopback timing evidence.
## Authority Map
- Owner: `Mimir.Runtime` owns sample ingest, rolling buffers, and typed visual
evidence. Fensalir owns overlay presentation and final program pixels.
- Inputs: Kiyo Pro YUY2 video, both PS3 Eye Bayer8 feeds, Scarlett ASIO
channels `asio-ch0..asio-ch3`, and Raven SRT screen capture on port `5200`.
- Outputs: `kiyo-pro-rgb` is the base program view; PS Move controller histories
lower as Feature evidence through `MimirMoveControllerTracker`; `raven-display`
stays in the `raven-sync` clock domain; ASIO labels identify shotgun,
cardioid, and Raven loopback lanes.
- Derived state: Move 2D histories are observations, not 6DoF pose. Raven
display timing is derived from Scarlett audio evidence, not SRT arrival time.
- Forbidden writers: PS3 Eye blob tracks do not own camera calibration; Raven
network timestamps do not own global sync; Kiyo Pro does not own high-rate
motion truth.
- Cut line: this profile is the first streamable overlay product, not the full
online calibration solve.
## Nightwing Eyes/Moves Split
Nightwing is the right body for the PS3 Eyes and PS Move controllers: it has
the spare USB topology, BlueZ/Bluetooth, and the local HID path. That does not
make it a camera-placement authority.
The split is:
- Nightwing-local authority: read PS3 Eye frames, talk to PS Move HID/Bluetooth,
apply LED schedules, preserve device timestamps, and extract compact sphere,
marker, and sparse Eye feature observations.
- Starfire authority: own the canonical rolling window, Scarlett timing,
global residual solve, camera placement, Kiyo/Raven program sync, and OBS
program publication.
- Fensalir authority: own D3D12 dense feature extraction/fusion, surface claims,
splat/reservoir resolution, and program pixels.
`MimirDistributedWitnessConfigurations.NightwingEyesMoves` records this as a
typed witness: it may emit `mimir.move_controller_observation_state`,
`mimir.camera_feature_track_state`, and `mimir.visual_marker_state`, but it must
not stream raw Eye media as the normal live contract and must not own the
canonical clock. Raw frames from Nightwing are diagnostic receipts only.
The first Nightwing receipt is good enough to build on: both PS3 Eyes enumerate
as stock Linux `ov534`/`gspca` V4L2 cameras on `/dev/video2` and `/dev/video3`.
Default V4L2 cadence is only 30 fps, but explicitly setting
`timeperframe=1/187` before streaming makes both cameras deliver about 187 fps
at 320x240 with zero sequence gaps. The witness worker must set frame interval;
otherwise Linux will quietly look worse than it is.
Nightwing also has a typed witness publisher staged as
`~/.local/bin/nightwing_typed_witness_publisher.py`. The repo source is
`tools/nightwing_typed_witness_publisher.py`. It sends binary MessagePack
`mimir.eve_sensor_observation.v1` records to `Mimir.EveSensorReceiver` on
`ws://192.168.1.66:8796/eve/periwinkle`; the receiver normalizes them into the
Mimir observation ledger, Odin projects them as `observation-stream` nodes, and
Nightwing's Eve TUI lowers those nodes in the `CultMesh Witness Streams` panel.
The current status records are camera-device receipts for `/dev/video*` and a
`nightwing-leap-tracking` placeholder that is intentionally `0` until real Leap
tracking owns that signal. Values are `[present, videoIndex, isPs3Eye]` for
camera-device receipts.
`MimirRoomCalibrationLockSolver` is the first room-lock coordinator. It consumes
the LED string candidate/residual, the camera-rig pose update frame, Move
controller feature-history candidates, and PS3 Eye sparse-feature candidates.
It only reports `Locked` when every witness family is present: multiple cameras
share enough LED indices, the rig solver produced enough pose updates with low
mean ray residual, multiple Move identities have stable history, and multiple
Eyes contribute enough stable feature tracks. Missing families are reported by
name instead of being papered over. `Mimir.BufferSmoke
--room-calibration-lock-smoke` proves a synthetic three-camera LED string, two
Moves, and two Eyes lock at confidence `0.778`, while removing Move history
correctly drops the result to `InsufficientWitnesses`.
## Runtime Profile
For the current stream-proof demo, use the corrected Starfire/Nightwing/Raven
profile:
```powershell
E:\Projects\Mimir\scripts\start-stream-proof.ps1
```
This profile keeps Nightwing raw Eye pixels off Starfire. Starfire ingests local
Scarlett ASIO, local Leap stereo IR, local Kiyo Pro RGB, Raven screen capture,
and Raven Realtek loopback PCM. Nightwing publishes compact Eye/Move
observations through `/eve/periwinkle`; those observations are calibration and
overlay evidence, not raw media ownership.
The profile declares:
- `leap-stereo-ir`: LeapUVC stereo IR / depth root.
- `kiyo-pro-rgb`: Kiyo Pro AR program view.
- `raven-display`: Raven screen capture from the muxed Raven A/V stream,
locally demuxed to SRT port `5210`, in the `raven-sync` clock domain.
- `raven-realtk-loopback`: Raven Realtek render loopback from the same muxed
Raven A/V stream, locally demuxed as f32 PCM to SRT port `5212`, also in
`raven-sync`.
- `focusrite-asio`: Starfire hero mics plus local Raven/program loopback lanes
on `asio-ch0..asio-ch3`.
For the narrower Leap/Eyes/Kiyo Pro bring-up path, use:
```powershell
E:\Projects\Mimir\scripts\start-mvp-leap-eyes-kiyo.ps1
```
This launches Starfire with only local Scarlett ASIO, local Leap stereo IR, and
local Kiyo Pro RGB in the Mimir rolling buffers. Nightwing does not stream PS3
Eye pixels to Starfire. The launcher stages `nw_eye_cap.py`,
`nw_move_hint.py`, and `nightwing_typed_witness_publisher.py` on Nightwing,
then starts the typed witness publisher with `--track-eyes`. Nightwing reads
`/dev/video2` and `/dev/video3`, extracts compact Move-sphere observations,
and sends `mimir.move_controller_observation_state.v1` claims to the
`/eve/periwinkle` CultMesh observation receiver. Starfire/Fensalir can consume
those claims for overlays and calibration without inheriting Nightwing's raw
camera bandwidth or local tracking authority.
The MVP profile refuses to start when LeapUVC is not enumerated. That is
intentional: the Leap is the dense close-range geometry root, not an optional
checkbox pretending to be present.
Use:
```powershell
$env:MIMIR_RUNTIME_CONFIG = "E:\Projects\Mimir\config\mimir-runtime.move-stream.local.json"
dotnet run --project .\src\Mimir.App\Mimir.App.csproj
```
The profile declares:
- `kiyo-pro-rgb`: Kiyo Pro program view, 640x480 YUY2 at the current reliable
25 fps class.
- `ps3-eye-0`, `ps3-eye-1`: high-rate Move sphere visual witnesses.
- `focusrite-asio`: one ASIO producer that emits:
`asio-ch0` shotgun hero mic, `asio-ch1` cardioid hero mic,
`asio-ch2` Raven program loopback L, and `asio-ch3` Raven program loopback R.
- `raven-display`: Raven screen capture over SRT port `5200`.
Run the synthetic overlay smoke:
```powershell
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --move-controller-overlay-smoke
```
Run the live ingest smoke once Raven is sending:
```powershell
$env:MIMIR_RUNTIME_CONFIG = "E:\Projects\Mimir\config\mimir-runtime.move-stream.local.json"
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --seconds 5 --require-samples --sync-reference asio-ch2
```
## Raven Sender
On Raven, start the muxed A/V sender from the logged-in desktop session:
```powershell
E:\Projects\Mimir\scripts\start-raven-av-sender.ps1 -TargetHost 192.168.1.66 -Port 5200
```
That sender captures the desktop through `gdigrab`, encodes video with
`h264_nvenc`, captures Realtek/default render loopback through the repo WASAPI
loopback script, encodes audio as AAC, and muxes both into one MPEG-TS SRT
stream. On Starfire, `scripts\start-stream-proof.ps1` starts
`scripts\start-raven-av-demux.ps1`, which listens on port `5200` and splits the
single transport into local Mimir ingest legs:
```powershell
raven-display local SRT 5210, copied H.264/MPEG-TS for decode
raven-realtk-loopback local SRT 5212, decoded f32 PCM
```
If Raven program audio is also routed into Starfire Scarlett on
`asio-ch2/asio-ch3`, that remains another strong local timing witness. The
Realtek loopback is now part of the same Raven transport as the display before
Starfire demuxes it for Mimir's raw ingest APIs.
Raven display sync is not allowed to ride on SRT arrival time. Both
`raven-display` and `raven-realtk-loopback` are in the `raven-sync` clock
domain. When audio sync estimates the Realtek loopback offset against the
Scarlett reference, `MimirSynchronizedBufferPlanner` applies that correction to
the whole `raven-sync` domain, including the display. The LAN transport is now
one muxed A/V stream; the local split exists only at the Starfire demux edge
because Mimir currently ingests raw video and PCM through separate source
adapters.
## Kiyo AR Trail Receipt
The first offline proof that the Kiyo view can accept Move trails is:
```powershell
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\solve_stream_proof_frustum.py .\artifacts\runtime\all-sensor-articulated-20260531-232349\sweep\calibration-observations.json
```
That solver uses Nightwing's two matched optical witnesses as a provisional
local witness space, fits the Kiyo Pro observations into it, and writes a
`mimir.stream_proof_frustum_solve.v1` receipt plus an AR trail preview. It is
enough for the stream demo and for Starfire's global residual owner to consume;
it is not final metric room calibration.
## Multiple Move Controllers
Do not collapse Move observations into a generic bright blob. Each controller
has its own `controllerId` and expected RGB/time code. The PS3 Eyes can track
the sphere as a high-rate brightness witness; Kiyo Pro can later confirm color
identity; the commanded flash schedule ties the observed marker to the
controller identity.
`MimirMoveControllerTracker` keeps a bounded history per controller and lowers
those histories into the existing Feature evidence path. This is deliberately
2D observation history. 6DoF pose waits for calibrated frustums and the global
residual owner.
## Music-Keyed Calibration Words
`tools/music_keyed_move_chirp_sync.py` is the first planner for calibration
words that are both musical and machine-decodable. It listens to Scarlett ASIO,
uses the same Perlines-style adaptive FFT delta whitening as
`tools/nightwing_psmove_music_pulse.py`, autocorrelates the whitened broadband
spectral-rise function for tempo, estimates a lightweight root from recent
best-fit fundamentals, then writes one shared plan:
- beat-aligned chirps quantized to the estimated minor-pentatonic scale;
- de Bruijn-coded event symbols so adjacent timing/frequency/color words remain
distinguishable;
- per-Move visual words using stable controller identities and unique colors;
the default is no longer a square flash, but a sampled RGB contour with
attack/release envelope, symbol-dependent hue glide, tremolo articulation,
and an emphasized controller lane;
- a JSON receipt shaped like AquaSynth song analysis vocabulary:
`tempo_bpm`, `beat_seconds`, `tempo_confidence`, `root_note`,
`suggested_scale`, `scale_frequencies_hz`, and
`whitened_spectral_autocorr`.
The synthetic smoke is:
```powershell
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\music_keyed_move_chirp_sync.py --mode synthetic --dry-run --events 8 --out-dir artifacts/runtime/music-keyed-move-chirp-sync-smoke
```
The live dry-run against Scarlett is:
```powershell
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\music_keyed_move_chirp_sync.py --dry-run --analyze-seconds 5 --events 8 --out-dir artifacts/runtime/music-keyed-move-chirp-sync-live-dry
```
For double-time material:
```powershell
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\music_keyed_move_chirp_sync.py --dry-run --tempo-grid doubletime --tempo-max-bpm 260 --analyze-seconds 5 --events 8 --out-dir artifacts/runtime/music-keyed-move-chirp-sync-dirtyphonics-dry
```
On the 2026-05-31 live dry-run, the planner produced a usable but not final
receipt: about `92.31` BPM, beat `0.650s`, tempo confidence `0.343`, root `A`,
key confidence `0.716`. That is enough to prove the dataflow, not enough to
claim final musical tempo authority.
For dense double-time material, use `--tempo-grid doubletime`; the planner then
selects the double-time member of the autocorrelation tempo family instead of
preferring the slower parent grid. On the Dirtyphonics-style live source this
produced about `171.43` BPM, beat `0.350s`, tempo confidence `0.358`, root `A`,
key confidence `0.703` after widening the search ceiling with
`--tempo-max-bpm 260`. The receipt records `tempo_family` so later decoders can
see the selected grid alongside half-time/double-time alternatives.
Visual calibration words are deliberately shaped like the audio words: one
gesture carries more than one bit. `--visual-gesture contour` records a
`mimir.move_visual_contour_word.v1` per Move per event, with `sample_rate_hz`,
duration, base identity color, symbol emphasis, and the exact RGB samples sent
over HID. Start conservatively around `--visual-gesture-hz 80`; the HID writer
accepted a short 80 Hz contour test on Nightwing, but the useful rate is the
rate cameras can recover cleanly, not merely the rate the controller accepts.
`--visual-gesture square` remains available as a baseline comparator.
`tools/sync_sweep.py` can now consume that exact plan with `--music-plan`.
That makes the plan the single authority for audio chirps, sampled Move RGB
contours, and offline video scoring. The sweep writes `gesture-plan.json`,
`event-schedule.json`, copied/rendered `chirp-train.wav`, per-sensor captures,
and `summary.json`. RGB camera receipts include a `visual_contour` block with
brightness-envelope correlation and color-chroma error against the expected
Move contour. The old peak-only score is still present as a timing witness; it
no longer pretends square flashes are the only visual language.
When `--music-plan` is used, the sweep also runs typed Mimir runtime sensor
receipts unless `--skip-mimir-live` is passed. The default receipts are:
- `config/mimir-runtime.perfect-machine.local.json` for local Leap stereo IR,
Kiyo Pro, Kiyo, and local camera texture/depth/field-resource ingress;
- `config/mimir-runtime.raven-eve.example.json` for Raven display and Eve
camera/mic frame-event listeners.
That receipt path is not raw media archival yet. It is the audit hook that
prevents Leap or Eve from being omitted by a DirectShow/Nightwing-only sweep.
After a sweep, run `tools/crunch_sync_sweep.py` over the sweep directory to
extract a compact calibration observation table. It merges Nightwing's typed
Move observations with local Kiyo/Kiyo Pro frame analysis, estimates each
local video's schedule shift against the event train, and writes
`calibration-observations.json` plus `calibration-observations.csv`.
Example using an already-written musical plan:
```powershell
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\sync_sweep.py --music-plan artifacts/runtime/music-keyed-move-chirp-sync-live/music-keyed-move-chirp-plan.json --out-dir artifacts/runtime/all-sensor-articulated-sync
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\crunch_sync_sweep.py artifacts/runtime/all-sensor-articulated-sync
```
The non-dry run targets the two wireless Nightwing Moves by default:
`00:07:04:a6:be:5f` on `/dev/hidraw2` and `00:06:f5:23:e2:d1` on
`/dev/hidraw3`. Nightwing now carries `/etc/udev/rules.d/70-move.rules`, which
sets Bluetooth PS Move hidraw nodes to `root:input 0660`. The `metacrat` user
is in `input`, so the non-root SSH writer can drive both spheres. Root SSH is
available as the short local alias `nwroot` for maintenance only; the hot LED
path should stay non-root.

View file

@ -56,6 +56,37 @@ buffer handles directly.
The probe reported one unrelated invalid USB descriptor from
`USB\VID_0000&PID_0002`; it did not prevent PS3 Eye capture.
## 2026-05-31: Nightwing Linux V4L2 PS3 Eye Probe
The two PS3 Eyes were moved to Nightwing. Linux enumerated them as Sony
Playstation Eye USB devices (`1415:2000`) through the stock `ov534`/`gspca`
V4L2 path:
- `/dev/video2`: `USB Camera-B4.09.24.1`, bus `usb-0000:00:14.0-1`
- `/dev/video3`: `USB Camera-B4.09.24.1`, bus `usb-0000:00:14.0-2`
Probe source:
- `tools/nightwing_v4l2_cadence_probe.py`
- copied to Nightwing and run directly against `/dev/video2` and `/dev/video3`
- no `v4l2-ctl`, `ffmpeg`, or third-party Python package dependency
Initial reads without setting frame interval negotiated only the default
`timeperframe=1/30`. After explicitly setting `VIDIOC_S_PARM` to
`timeperframe=1/187`, both Eyes delivered high-rate frames with zero sequence
gaps.
| Device | Requested Mode | Actual Mode | Frames | Delivered FPS | Device FPS | Sequence Gap |
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
| `/dev/video2` | 320x240 GRBG @ 187 | 320x240 GRBG | 375 | 187.07 | 187.37 | 0 |
| `/dev/video3` | 320x240 GRBG @ 187 | 320x240 GRBG | 375 | 187.14 | 187.37 | 0 |
| `/dev/video2` | 320x240 YUYV @ 187 | 320x240 YUYV | 374 | 186.94 | 187.37 | 0 |
| `/dev/video3` | 320x240 YUYV @ 187 | 320x240 YUYV | 375 | 187.06 | 187.38 | 0 |
Conclusion: Nightwing's stock Linux V4L2 path can sustain the high-rate PS3 Eye
witness mode if the witness worker explicitly sets frame interval before
streaming. Default V4L2 reads are a false low-cadence signal.
## 2026-05-21: LeapUVC Kernel Streaming Probe
The Leap Motion Controller is currently present as `VID_F182&PID_0003` and bound

View file

@ -5,24 +5,17 @@ hosted stream machine.
## Objective
Make the room available as a Mimir-owned synchronized program with separately
Make the room available in OBS as synchronized program video plus separately
controllable audio while preserving ownership:
- Mimir owns configuration, calibration truth, runtime contracts, launch,
status, and persistence.
- Mimir.Runtime owns in-memory stream buffers and runtime synchronization.
- Native capture workers own device reads and append typed sample handles.
- Mimir owns stream subscription, scene composition, preview/control state,
stats, and program publication intent.
- Fensalir owns GPU fusion, rendering, D3D12 interop, UI lowering, and local
program texture output.
- Fensalir owns GPU fusion, rendering, D3D12 interop, UI, and Spout2 output.
- Faust/native DSP owns hot audio alignment, suppression, separation, stems, and
spatial bed generation.
- Eve GUI/TUI lowerers expose operator controls and preview without owning
scene truth.
- The Yggdrasil-facing publisher daemon consumes Mimir program output and
publishes it to the site without owning composition.
- OBS is a temporary compatibility sink.
- OBS owns broadcast controls.
## Runtime State Model
@ -33,22 +26,23 @@ flowchart TD
N["network feed producers"] --> R
R --> Q["Fensalir UI + GPU fusion"]
R --> F["Faust/native DSP"]
Q --> S["Mimir program video"]
Q --> S["Spout2/program video"]
F --> P["program stems + spatial bed"]
Q --> E["Eve operator surfaces"]
S --> Y["Yggdrasil site publisher"]
P --> Y
S -. "compatibility" .-> O["OBS adapter"]
P -. "compatibility" .-> O
S --> O["OBS"]
P --> O
```
The app-level runtime initializes one rolling buffer per configured audio or
video stream, defaulting to five seconds. The lower native reservoir keeps the
single-edge typed-handle invariant for Fensalir/Faust integration.
Payload memory belongs to the producing domain. The reservoir and runtime own
timing, identity, ordering, retention, and lookup. They do not pretend to own
camera driver buffers, GPU textures, or DSP audio memory.
Payload memory belongs to the producing domain, but rendering inputs should be
Fensalir-owned GPU resources whenever the device path allows it. The camera hot
path is: ask Fensalir for a keyed `Texture2D` lease, speak to the device stack
as directly as possible, populate that lease, signal the producer fence, and
publish the resource key through Mimir's rolling buffers and FieldEvidence. The
reservoir and runtime own timing, identity, ordering, retention, and lookup.
They do not become pixel middlemen.
## Authoritative Boundaries
@ -58,8 +52,7 @@ camera driver buffers, GPU textures, or DSP audio memory.
- Fensalir consumes current visual/audio timing state and publishes the program
video surface.
- Faust/native DSP consumes audio/phase/event handles and publishes stems.
- OBS compatibility adapters may receive final program surfaces. They do not
own synchronization, composition, preview/control, or publication.
- OBS receives final program surfaces. It does not own synchronization.
## Impossible By Construction
@ -73,7 +66,10 @@ camera driver buffers, GPU textures, or DSP audio memory.
1. Keep the C# app/runtime as the public Mimir surface.
2. Implement Leap stereo IR direct ingest first and measure sustained cadence.
3. Add the remaining local camera drivers through the same source seam.
Record the unavoidable copy count; do not hide CPU upload behind a friendly
adapter name.
3. Add the remaining local camera drivers through the same source seam,
preferring device-direct producers that populate Fensalir texture leases.
4. Add native audio capture workers and feed aligned blocks to Faust/native DSP.
5. Bind Fensalir UI to stream health, buffer depth, source timestamps, settings,
and output management.

View file

@ -0,0 +1,63 @@
# Network Display Synchronization
Raven can be a low-level synchronized producer when it controls both its display
output and an audio timing signal routed into the Scarlett.
Authority map:
- Owner: `Mimir.Runtime` owns aligned rolling-buffer frames.
- Inputs: camera buffers, Raven display frames, Scarlett ASIO audio buffers, and
explicit timing corrections learned from audio evidence.
- Outputs: `MimirSynchronizedBufferFrame` slices at one canonical presentation
time for Fensalir/Faust/OBS publication.
- Derived state: Raven display timing is derived from the Raven audio sync
signal. The display feed is not a timing authority by itself.
- Forbidden writers: OBS, SRT bridge endpoints, frame-event probes, and network
receive timestamps must not decide cross-stream alignment. CultMesh reliable
UDP packet arrival is transport evidence only, not clock authority.
- Shared path: local cameras, network display frames, mic channels, loopback
channels, and Raven audio sync channels all enter `MimirRollingStreamBuffer`
first, then the synchronized planner chooses slices.
- Deletion line: any compositor path that directly consumes Raven pixels outside
the runtime buffer/alignment path is diagnostic only.
Practical shape:
1. Raven renders a visible timing marker or encoded frame identity on the
captured display.
2. Raven emits the matching audio timing signal through its audio output.
3. That audio is physically or digitally routed into a Scarlett input beside the
mic channels.
4. Mimir decodes the Scarlett audio evidence, learns the Raven clock offset, and
applies that clock-domain correction to the `raven-display` video buffer.
5. Fensalir receives aligned buffer slices and still owns composition.
`config/mimir-runtime.raven-eve.example.json` is the active bridge setup shape
for the network producer pass:
- `raven-display` listens for Raven screen capture on SRT port `5200`, decodes
through FFmpeg, and emits BGRA video samples into the runtime buffer path.
- `eve-camera` listens for EveCanvas camera frame-events on WebSocket port
`8793`.
- `eve-mic` listens for EveCanvas microphone frame-events on WebSocket port
`8794`.
The target Verse shape replaces those SRT/WebSocket bridge edges with CultMesh
reliable-UDP lanes: typed `mimir.cultmesh_stream_frame.v1` envelopes for
per-frame media metadata and refs, plus bounded media body shards for payloads
too large to inline. Raven display frames remain media observations entering
the same rolling buffer; Raven sync still comes from audio evidence and
clock-domain correction, not packet arrival.
The older disabled `frame-events` example in
`config/mimir-runtime.asio.example.json` remains diagnostic only. It can witness
source timestamps and frame identity, but it does not carry pixels. The Scarlett
channel carrying Raven's sync audio is still the evidence path that earns the
`raven-sync` timing correction.
Smoke proof:
```powershell
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --synchronized-buffer-planner-smoke
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --ffmpeg-rawvideo-source-smoke
```

213
docs/obs-stem-plugin.md Normal file
View file

@ -0,0 +1,213 @@
# OBS Native Plugin
The production OBS path is a Mimir/Fensalir-owned program texture plus a
Mimir/Fensalir-owned stem bus, exposed through a narrow native OBS plugin.
## Authority
- Fensalir owns the completed program pixels.
- Mimir/Fensalir own timing, alignment, DSP, and stem identity.
- OBS owns broadcast mixing and scene composition.
- The plugin imports already-produced program pixels and reads already-processed
stem samples. It does not synchronize raw sources, estimate clocks, render the
field, or decide which payload is true.
## Current Contract
### Video
Fensalir publishes the completed backbuffer into a named shared D3D12
`B8G8R8A8_UNorm` texture when these environment variables are set:
```text
FENSALIR_PROGRAM_OUTPUT_D3D12=1
FENSALIR_PROGRAM_OUTPUT_NAME=Global\MimirFensalirProgramTexture
```
The native OBS plugin in `native/obs_stem_source` exposes a `Mimir Program
Texture` video source. Set:
```text
Shared D3D12 texture name: Global\MimirFensalirProgramTexture
Shared D3D12 fence name: Global\MimirFensalirProgramFence
Consumer D3D12 fence name: Global\MimirObsProgramConsumerFence
Texture ring slots: 1
```
OBS/libobs is D3D11 internally on Windows, so this source opens the named D3D12
texture, creates an OBS-readable legacy shared D3D11 texture, opens that same
bridge texture from D3D12, and performs a GPU-to-GPU copy during OBS render
before drawing the source. This avoids CPU readback and does not require
Spout2. It is not pure zero-copy yet; the remaining copy is the
D3D12-to-libobs-D3D11 boundary. Fensalir also publishes a named D3D12
program-output fence; the OBS source opens it and observes publication
completion before copying. This prevents blind read-before-producer-completion
behavior without coupling OBS to Fensalir's private frame fence.
For OBS-only stress testing, Fensalir and the OBS source can agree on a bounded
texture ring:
```text
FENSALIR_PROGRAM_OUTPUT_RING_COUNT=3
FENSALIR_PROGRAM_OUTPUT_CONSUMER_FENCE_NAME=Global\MimirObsProgramConsumerFence
Texture ring slots: 3
Consumer D3D12 fence name: Global\MimirObsProgramConsumerFence
```
With a ring count above one, Fensalir publishes textures named
`Global\MimirFensalirProgramTexture.0`, `.1`, and so on. The OBS source uses the
program-output fence value to copy the latest completed slot and signals the
consumer fence after its GPU copy. When Fensalir is launched with the matching
consumer-fence environment variable, it will not reuse a ring slot until OBS has
acknowledged a copy at or beyond that slot's published fence value. Leave the
ring count at `1` for EVE until the EVE relay speaks the same ring contract.
For a visible interop witness while the real Mimir scene is sparse, launch
Mimir with:
```text
MIMIR_OBS_PROOF_VISUAL=1
```
That enables a small bright SDF/starfield proof visual. It is a debug witness,
not production scene policy.
### Audio
`MimirObsStemSharedMemoryPublisher` writes the latest validated
`MimirObsStemPublicationSnapshot` into a Windows shared-memory map named
`Local\MimirObsStemBus` by default. The native OBS plugin in
`native/obs_stem_source` exposes a `Mimir Audio Stem` input source. Add one
source per stem and set:
```text
Shared memory map: Local\MimirObsStemBus
Stem id: aligned_source_0
```
Use `aligned_source_0` through `aligned_source_5` for the first actuator-stage
source lanes.
## Build
```powershell
.\scripts\build-obs-stem-plugin.ps1
```
The script stages `obsproject/obs-plugintemplate` under `artifacts/obs-sdk/`,
lets the upstream template fetch/build the matching OBS/libobs development
surface, then builds `native/obs_stem_source` against that SDK. The staged SDK
and native build products are local artifacts, not repository state.
The managed shared-memory publisher is verified by:
```powershell
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --obs-stem-shared-memory-smoke
```
The local native plugin build currently succeeds and emits:
```text
native\obs_stem_source\build\Release\mimir_obs_stem_source.dll
```
That DLL currently registers both source types:
```text
Mimir Program Texture
Mimir Audio Stem
Muninn Stream
```
### Muninn Streams
The `Muninn Stream` source is a thin OBS wrapper over Muninn's typed
CultCache/CultMesh stream catalog. The operator-facing UI is intentionally just
the stream dropdown plus independent video and audio device pickers for the
selected source. Store paths, command endpoints, and media return endpoints are
source defaults and migration state, not user-facing stream selection.
The source listens for Raven's typed OBS catalog projection on local UDP port
`17874`. Raven Muninn sends the catalog over a small CultNet RUDP discovery lane
using connection id `0x6d750003`; OBS uses that live catalog for the dropdown
when present.
If live discovery is absent, the source falls back to Muninn's local `.cc`
store:
```text
C:\Meta\Odin\state\muninn.telemetry.cc
```
The source looks for record key `obs` with type:
```text
muninn.obs_stream_catalog
```
Muninn owns that catalog as typed CultCache state. OBS owns only the selected
scene source, the selected video leg, the selected audio leg, and media
rendering. When a selected stream uses Muninn's RUDP media profile, the source
publishes the same typed
`muninn.capture_stream_command` that an operator would publish manually. Raven
Muninn `serve` still owns whether capture starts and which local FFmpeg/WASAPI
children exist.
If both live discovery and the local catalog projection are absent or stale, the
source keeps a LAN-default Raven A/V stream option instead of showing an empty
source. That is a resilience fallback for the OBS machine, not a second stream
authority. Stale catalog `rudp://` URLs that do not carry the current
low-latency LAN profile are sanitized back to the source-derived URL before the
bridge starts.
The current Raven-room default route is LAN-first: command requests go to
`192.168.1.84:17873`, and Raven sends media back to Starfire at
`192.168.1.66:5204`. Older saved source settings that still point at the
deprecated WireGuard defaults `10.77.0.4:17873` and `10.77.0.2` are migrated on
load. Custom operator-entered endpoints are left alone.
The source polls the catalog while active, but reconnect is owned by the RUDP
receiver bridge rather than catalog labels. The bridge tracks the last packet
and the last forwarded video/audio payload. If startup produces no media, or a
previously live stream goes stale, OBS refreshes its local FFmpeg child readers
and republishes the typed activation command on a short bounded cadence. A Raven
daemon restart or media child exit should therefore recover without reopening
source properties.
For `rudp://` stream URLs, the plugin binds the advertised media port, accepts
the CultNet RUDP handshake, acknowledges packets, reassembles RUDP fragments,
decodes typed `muninn.media_video_access_unit.v1` and
`muninn.media_audio_packet.v1` records, and lowers elementary H.264 plus ADTS
AAC into local loopback UDP sockets for OBS FFmpeg child sources. The bridge
does not impose a global packet-sequence FIFO over the media lane: CultNet RUDP
media is reliable but unordered, so frame ids, chunk ids, and the advertised
`assembly_deadline_ms` own video reconstruction. Incomplete video frames expire
inside that budget and emit `muninn.media_receiver_feedback.v1` keyframe
pressure instead of stalling newer media behind an obsolete sequence gap.
RUDP fragment ids are quarantined briefly after completion/expiry so late
duplicates cannot seed a corrupt assembly when the sender's 16-bit fragment id
space wraps under sustained LAN pressure. The local OBS FFmpeg UDP readers use
large loopback buffers; those buffers absorb receiver bursts without becoming a
second transport authority.
## Install
For the standard OBS plugin search path:
```powershell
.\scripts\install-obs-mimir-plugin.ps1
```
By default this writes:
```text
C:\ProgramData\obs-studio\plugins\mimir_obs_stem_source\bin\64bit\mimir_obs_stem_source.dll
```
For a portable OBS tree or direct OBS installation root:
```powershell
.\scripts\install-obs-mimir-plugin.ps1 -ObsRoot D:\Tools\obs-studio
```
The script does not create scene items or alter OBS profiles.

View file

@ -27,6 +27,13 @@ Read this with:
loop candidates.
- [[research/perfect-machine-study-2026-05-23/fensalir-integration-map|Fensalir Integration Map]]
for the Mimir/Fensalir ownership cut.
- [[fensalir-rendering-rebuild-migration|Fensalir Rendering Rebuild Migration]]
for the current teardown/rebuild plan that keeps Mimir on physical evidence
and pushes field rendering authority into Fensalir's evidence machine.
- [[research/d3d12-stereo-depth-provenance-2026-05-29|D3D12 Stereo Depth Provenance]]
for the current fast GPU depth search. `libSGM` is the permissive north-star
reference for a future Fensalir-owned D3D12 compute stereo disparity lane;
CUDA/TensorRT projects remain provenance, not live dependencies.
- [[research/perfect-machine-study-2026-05-23/calibration-session-spec|Calibration Session Spec]]
for the planned physical-path calibration command.
- [[perfect-machine-module-library|Perfect Machine Module Library]] for the
@ -38,23 +45,19 @@ Read this with:
Mimir is a bounded-latency realtime field computer: it ingests many clocks,
places their evidence inside one rolling temporal window, extracts audio/video
geometry from that window, owns the stream program scene, exposes Eve operator
surfaces, and emits a publicable program feed without letting compatibility
sinks become authority.
geometry from that window, and emits an OBS-usable program surface without
letting OBS become the synchronization authority.
## Authority Index
| Authority | Owns | Must Not Own |
| --- | --- | --- |
| Mimir program composition | source subscription, scene graph, crop/key/transform state, preview/control state, stats, publication intent | local device reads, GPU shader internals, audio-rate DSP, site transport policy |
| `Mimir.Runtime` | stream identity, buffers, sync reports, calibration model loading, source polling, telemetry state | hot DSP actuator, dense visual fusion, direct device capture |
| Native capture / Muninn hosts | direct driver callbacks, device timestamps, payload handles, local stream capabilities/media | runtime policy, scene composition, publication |
| `Mimir.Runtime` | stream identity, buffers, sync reports, calibration model loading, source polling, telemetry state | hot DSP actuator, dense visual fusion, broadcast composition |
| Native capture | direct driver callbacks, device timestamps, payload handles, low-jitter capture loops | runtime policy, UI state, OBS composition |
| `native/reservoir` | shared-edge retention and typed views | payload memory interpretation, rendering, DSP |
| Fensalir | window, UI lowering, D3D12 bridge, GPU fusion, program video output | clock authority, audio actuator, scene ownership |
| Fensalir | window, UI, D3D12 bridge, GPU fusion, program video publication | clock authority, audio actuator |
| Faust/native DSP | fractional delay, resampling, separation, spatialization, stems | stream discovery, UI |
| Eve lowerers | operator GUI/TUI rendering, controls, previews, stats lowering | scene truth, stream truth, publication truth |
| Yggdrasil site publisher | public transport, encoder/session state, reconnect policy, site edge health | scene composition |
| OBS | compatibility preview/output sink | synchronization, capture timing, composition, publication |
| OBS | final broadcast composition | synchronization, capture timing |
| Raven/phones | local decode of codebook/schedule observations | canonical clock authority |
## Evidence Taxonomy
@ -82,6 +85,11 @@ sinks become authority.
### Visual Evidence
- Leap stereo IR: timing/depth candidate, topology-sensitive but high-value.
The live `LeapStereoIr` resource is a packed R8G8 texture; Mimir now lowers
it into a stereo-depth FieldEvidence claim by treating R/G as the left/right
observation lanes and asking Fensalir for an R16F disparity `SurfacePage`.
The first installed Fensalir kernel is a packed block matcher, not calibrated
metric depth or full SGM.
- PS3 Eyes: high-rate tracking witnesses.
- Kiyo/Kiyo Pro: RGB context and texture, with Kiyo Pro currently cadence/USB
limited.
@ -169,6 +177,18 @@ Likely shape:
- Native capture pushes typed frame handles and timestamps.
- Fensalir extracts features/depth/optical flow on GPU.
- Stereo depth starts as a Fensalir-owned D3D12 compute lane modeled on
permissive SGM provenance, especially `libSGM`: rectified texture pairs plus
calibration produce GPU-resident disparity/depth/confidence resources, which
then lower into FieldEvidence claims.
- Current contract proof:
`Mimir.BufferSmoke --stereo-depth-contract-smoke` validates the profile and
FieldEvidence socket. The synthetic Leap stereo pair uses caller-declared
shader-readable left/right texture declarations, an R16F compute-writable disparity
`SurfacePage`, an R8 confidence `Texture2D`, one Height claim, one
`AquariumFieldStereoDepthLowering` sidecar with min disparity 0, 128
disparities, four paths, census radius 2, P1 8, and P2 96, and one planned `SurfacePage`
packet. It is not yet a live compute kernel.
- Native reservoir preserves rolling evidence.
- Fusion emits surface/material/render claims.
- Program video publishes via Spout2.
@ -177,6 +197,8 @@ Pressure:
- Do not build a CPU image pipeline because it is easy to inspect. Six cameras
make that path collapse into a pile of excuses.
- Do not import CUDA or TensorRT as live depth authority. External depth
projects may guide kernel shape, benchmarks, and failure probes only.
## Hot Loop Budget
@ -245,9 +267,7 @@ Six optical sensors with high-rate Eyes and Leap IR mean:
- Cut JSON/stdout local camera hot path when concrete direct drivers land.
- Cut legacy chirplet path once chirp-bin decoder has equivalent docs/tests and
physical path proof.
- Cut bridge scripts only after Mimir program output, Eve operation, and
Yggdrasil publication are real enough that OBS compatibility no longer earns
its keep.
- Cut bridge scripts only after OBS-facing direct output is real.
- Cut ad hoc response weighting if a formal path model replaces it.
## Future Work Index
@ -257,6 +277,9 @@ Six optical sensors with high-rate Eyes and Leap IR mean:
3. Reduced alphabet/high-order codebook physical proof.
4. Fractional delay/SRO actuator proof.
5. Direct Leap/PS3/Kiyo runtime drivers.
6. Native reservoir integration with Fensalir/Faust.
7. Raven/phone receiver codebook/schedule local decode.
8. GPU visual fusion and Spout2 program video.
6. D3D12 stereo disparity/depth compute lane, with `libSGM` as provenance and
Fensalir as live owner. Contract/profile smoke exists; implement the HLSL
cost-volume, aggregation, winner/subpixel, and confidence passes next.
7. Native reservoir integration with Fensalir/Faust.
8. Raven/phone receiver codebook/schedule local decode.
9. GPU visual fusion and Spout2 program video.

View file

@ -13,23 +13,23 @@ job-interview answer for why it belongs in the body of the machine.
| Bioacoustic clock solver | `Mimir.Runtime` | Fits canonical source time from decoded word anchors for local, Raven, phone, and microcontroller witnesses. | `src/Mimir.Runtime/Synchronization/MimirBioacousticClockHypothesis.cs` |
| Alignment actuator | `Mimir.Runtime` + Faust | Converts sync state into bounded fractional-delay, gain, and sample-rate-offset controls; Faust moves samples. | `src/Mimir.Runtime/Synchronization/MimirAudioAlignmentActuator.cs` |
| Audio actuator strategies | `Mimir.Runtime` + Faust | Integer baseline, Farrow fractional delay, variable ASRC, and hybrid delay/ASRC strategy profiles. | `src/Mimir.Runtime/Synchronization/MimirAudioActuatorConfiguration.cs` |
| CultMesh contracts | `Mimir.Runtime` | Typed codebook, decoder, acoustic path, actuator, program, Move evidence, and Mimir-fused Move controller pose documents for distributed Mimir nodes. | `src/Mimir.Runtime/Synchronization/MimirCultMeshContracts.cs`, `src/Mimir.Runtime/Synchronization/MimirTrackingObservation.cs` |
| Move controller fusion | `Mimir.Runtime` + native reservoir + Fensalir | Mimir admits Muninn marker candidates plus controller/IMU/button feeds into native `move_evidence` buffers. `MimirMoveCalibrationProtocol` defines the Starfire/Nightwing capture run, Muninn records optional Quest USB witness availability as `muninn.quest_access.v1`, `MimirMoveFusion` publishes calibrated `mimir.move_controller_pose.v1` position/button/gyro poses only when camera witnesses are calibrated, and `MimirMovePoseStream` carries those poses over a realtime CultMesh frame. IMU orientation/prediction remains the next earned 6DoF step before Fensalir treats the stream as VR-grade controller input. | `src/Mimir.Runtime/Synchronization/MimirTrackingObservation.cs`, `src/Mimir.Runtime/Synchronization/MimirMuninnMoveEvidence.cs`, `src/Mimir.Runtime/Synchronization/MimirMoveCalibrationProtocol.cs`, `src/Mimir.Runtime/Synchronization/MimirMoveFusion.cs`, `src/Mimir.Runtime/Synchronization/MimirMovePoseStream.cs`, `src/Mimir.Runtime/Synchronization/MimirNativeReservoirRuntime.cs`, `native/reservoir/src/lib.rs` |
| CultMesh contracts | `Mimir.Runtime` | Typed codebook, decoder, acoustic path, and actuator state documents for distributed Mimir nodes. | `src/Mimir.Runtime/Synchronization/MimirCultMeshContracts.cs` |
| Acoustic path learning | `Mimir.Runtime` | Calibration stages for usable bands, confusion, global delay, group delay, and codebook adaptation. | `src/Mimir.Runtime/Synchronization/MimirAcousticPathLearningConfiguration.cs` |
| Acoustic localization | `Mimir.Runtime` + Fensalir | Pairwise TDOA, SRP-PHAT grid, sparse source, and visual-constrained localization profiles plus a small SRP/TDOA scorer. | `src/Mimir.Runtime/Synchronization/MimirAcousticLocalizationConfiguration.cs`, `src/Mimir.Runtime/Synchronization/MimirAcousticLocalizationSolver.cs` |
| Benchmark panels | `Mimir.Runtime` | Decoder golf and meatspace acceptance degradation panels with receipt roots and thresholds. | `src/Mimir.Runtime/Synchronization/MimirBenchmarkPanelConfiguration.cs` |
| Capture profiles | `Mimir.Runtime` + native workers | Hardware intent for Leap, Kiyos, PS Eyes, Starfire Scarlett, Raven Scarlett, and USB-attached Moves on Starfire/Nightwing. | `src/Mimir.Runtime/Synchronization/MimirNativeCaptureConfiguration.cs` |
| Capture profiles | `Mimir.Runtime` + native workers | Hardware intent for Leap, Kiyos, PS Eyes, Starfire Scarlett, and Raven Scarlett. | `src/Mimir.Runtime/Synchronization/MimirNativeCaptureConfiguration.cs` |
| Camera ingest strategies | `Mimir.Runtime` + native workers | Diagnostic metadata, managed native wrapper, native SPSC ring, and shared-GPU-texture ingest shapes. | `src/Mimir.Runtime/Synchronization/MimirCameraIngestConfiguration.cs` |
| Stereo depth profiles | `Mimir.Runtime` + Fensalir | SGM-shaped stereo disparity/depth profiles with libSGM as provenance for the first D3D12 lane, not a CUDA dependency. | `src/Mimir.Runtime/Synchronization/MimirStereoDepthConfiguration.cs` |
| Reservoir strategies | `Mimir.Runtime` + native reservoir + Fensalir | Managed rolling buffers, native shared-edge rings, and derived GPU temporal field storage. | `src/Mimir.Runtime/Synchronization/MimirReservoirConfiguration.cs` |
| Distributed witnesses | `Mimir.Runtime` + CultMesh | Raven, phone, microcontroller, and remote camera rig configurations for compact timing/path state. | `src/Mimir.Runtime/Synchronization/MimirDistributedWitnessConfiguration.cs` |
| Network transports | `Mimir.Runtime` + CultMesh | Typed timing state, debug windows, SRT bridge media, and experimental browser transport policy plus constraint-based selection. | `src/Mimir.Runtime/Synchronization/MimirNetworkTransportConfiguration.cs`, `src/Mimir.Runtime/Synchronization/MimirNetworkTransportSelector.cs` |
| Distributed witnesses | `Mimir.Runtime` + CultMesh | Raven, phone, microcontroller, Nightwing Eyes/Moves, and remote camera rig configurations for compact timing/path/visual state. | `src/Mimir.Runtime/Synchronization/MimirDistributedWitnessConfiguration.cs` |
| Network transports | `Mimir.Runtime` + CultMesh | Typed timing state, reliable-UDP Verse media/body lanes, debug windows, SRT bridge media, and experimental browser transport policy plus constraint-based selection. | `src/Mimir.Runtime/Synchronization/MimirNetworkTransportConfiguration.cs`, `src/Mimir.Runtime/Synchronization/MimirNetworkTransportSelector.cs` |
| Authority policy | `Mimir.Runtime` | Trust decisions for Starfire loopback, Raven evidence, phone candidates, diagnostic media, and unknown nodes. | `src/Mimir.Runtime/Synchronization/MimirAuthorityPolicyConfiguration.cs`, `src/Mimir.Runtime/Synchronization/MimirAuthorityPolicyEvaluator.cs` |
| Audio field configurations | `Mimir.Runtime` | Six-mic, source-based, ambisonic, and hybrid evidence field shapes. | `src/Mimir.Runtime/Synchronization/MimirAudioFieldConfiguration.cs` |
| Visual fusion configurations | `Mimir.Runtime` + Fensalir | Sensor roles and fusion modes from cadence proof through full hybrid evidence. | `src/Mimir.Runtime/Synchronization/MimirVisualFusionConfiguration.cs` |
| Compute offload configurations | `Mimir.Runtime` + CultMesh | Local-heavy, distributed witness, and calibration sweep workload placement. | `src/Mimir.Runtime/Synchronization/MimirComputeOffloadConfiguration.cs` |
| Assembly plans | `Mimir.Runtime` | Named combinations of modules that prove synthetic, Scarlett, meatspace, and distributed cuts. | `src/Mimir.Runtime/Synchronization/MimirMachineAssemblyPlan.cs` |
| Program publication profiles | Mimir + Fensalir + Faust + Eve + Yggdrasil | Mimir-owned scene publication, Eve operator surfaces, Yggdrasil site feed, and temporary OBS compatibility adapters. | `src/Mimir.Runtime/Synchronization/MimirProgramPublicationConfiguration.cs` |
| Fensalir lowering | `Mimir.Runtime` -> Fensalir | Pure mapping from Mimir rolling/video/audio state into Fensalir GPU sensor and acoustic field frames. | `src/Mimir.Runtime/Synchronization/MimirFensalirFieldLowering.cs` |
| OBS publication profiles | Fensalir + Faust + OBS | Spout/shared-texture video and separately mixable Faust stem publication. | `src/Mimir.Runtime/Synchronization/MimirObsPublicationConfiguration.cs` |
| Fensalir lowering | `Mimir.Runtime` -> Fensalir | Pure mapping from Mimir rolling/video/audio state into Fensalir GPU sensor, stereo-depth, and acoustic field frames. | `src/Mimir.Runtime/Synchronization/MimirFensalirFieldLowering.cs` |
| Module catalog | `Mimir.Runtime` | Indexes the live module set and verification commands. | `src/Mimir.Runtime/Synchronization/MimirModuleLibrary.cs` |
| Manifest factory | `Mimir.Runtime` | Produces an exportable JSON manifest for tools, docs, UI, and remote witnesses. | `src/Mimir.Runtime/Synchronization/MimirPerfectMachineManifest.cs` |
@ -41,21 +41,10 @@ job-interview answer for why it belongs in the body of the machine.
program video publication.
- Faust owns hot sample movement, alignment, spatialization, separation, and
stem generation.
- CultMesh carries typed state surfaces; remote nodes may observe and decode,
but they do not become independent clock authorities.
- USB-attached Moves on Starfire and Nightwing publish source-local Muninn
evidence streams: glowing-orb marker candidates and controller/IMU/button
state. Odin discovers/projects those streams. Mimir owns native evidence
buffer admission/layout, calibration, association, triangulation, confidence,
and `mimir.move_controller_pose.v1` publication. The current fusion pass
resolves calibrated optical position and controller state; orientation remains
explicitly unresolved until IMU fusion/prediction lands. Fensalir consumes the
resolved pose stream for interactive environments once that contract is wired.
- Resolved Move poses are framed as
`mimir.move_controller_pose_stream_frame.v1` on CultMesh shared-memory bytes
streams; the stream is a Mimir output surface, not a new fusion owner.
- OBS may receive compatibility program surfaces and stems. It does not own
calibration, composition, preview/control, or publication.
- CultMesh carries typed state surfaces and reliable-UDP media/body lanes;
remote nodes may observe, decode, and stream media through the Verse, but
they do not become independent clock authorities.
- OBS receives final program surfaces and stems. It does not own calibration.
## Verification Surface
@ -66,12 +55,6 @@ dotnet build .\Mimir.slnx
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --perfect-machine-profile-smoke
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --perfect-machine-lowering-benchmark
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --perfect-machine-contract-smoke
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --move-tracking-contract-smoke
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --move-native-reservoir-smoke --native-reservoir .\native\reservoir\target\debug\localcast_reservoir.dll
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --muninn-move-evidence-smoke --native-reservoir .\native\reservoir\target\debug\localcast_reservoir.dll
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --muninn-move-cultmesh-stream-smoke --native-reservoir .\native\reservoir\target\debug\localcast_reservoir.dll
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --move-calibration-protocol-smoke
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --move-fusion-smoke
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --perfect-machine-manifest
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --bioacoustic-actuator-self-test
```

View file

@ -0,0 +1,42 @@
# Program Control Surface
Mimir now exposes a compact Fensalir-hosted program panel for final presentation
intent.
Authority map:
- Owner: `MimirPresentationControlState` owns operator intent for program video,
audio mix, and global color preset.
- Inputs: runtime rolling buffers, aligned audio lanes, LUT preset selection,
feed visibility, feed opacity, stem mute/solo, and stem gain.
- Outputs: filtered video surface intents, Faust/native DSP gain controls,
sample gain applied before Fensalir streaming DSP, and a postprocess control
snapshot.
- Derived state: feed lists are derived from active rolling buffers. LUT preset
exposure and bloom are currently mapped into `GraphicsSettings`.
- Forbidden writers: OBS, bridge endpoints, debug readouts, and network arrival
timestamps must not choose program composition.
- Shared paths: local cameras, Raven display capture, and future network video
feeds all enter rolling buffers first, then the presentation state decides
whether they contribute to production surface intent.
- Deletion line: direct compositor shortcuts outside Fensalir FieldEvidence and
audio DSP are diagnostic only.
The current UI lives in the `Mimir Program` panel:
- `Video`: select feed, visible, solo, opacity, layer ordering.
- `Audio`: select stem/source, mute, solo, gain.
- `Color`: LUT preset and LUT strength.
The shader-side LUT path is explicit state, not finished renderer authority yet.
`MimirLookupTablePreset` carries `LutPath`, strength, contrast, saturation,
temperature, and tint so Fensalir can consume the same contract when the global
post shader grows real LUT sampling. Until that renderer cut lands, Mimir maps
the preset exposure and bloom values into existing Fensalir `GraphicsSettings`.
Smoke proof:
```powershell
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --presentation-control-smoke
```

View file

@ -0,0 +1,80 @@
# Raven Capture Mux Daemon
Raven owns local display capture and Realtek render-loopback capture. Mimir receives the muxed stream and decides what it means inside the synchronized program model.
## Authority Map
- Owner: `Mimir.RavenDaemon` owns process supervision and typed status for Raven's capture mux.
- Inputs: local display capture parameters, WASAPI render-loopback selection, FFmpeg path, named SRT targets, log root, and operator flags.
- Outputs: `mimir.raven_capture_mux_state.v1`, an Eve dashboard surface, health JSON, and named FFmpeg SRT MPEG-TS streams for Mimir and OBS.
- Derived state: PID, restart count, log paths, and dashboard nodes are derived from the supervised process and daemon options.
- Forbidden writers: capture scripts may still launch diagnostics, but they do not own daemon status or the canonical Raven mux state.
- Shared paths: service launch, dry run, health checks, and Eve dashboard projection all read the same `MimirRavenCaptureMuxStateDocument`.
- Cut line: no OBS plugin, custom media daemon, or alternate transport owns media in this pass. FFmpeg/WASAPI own capture, mux, encode, and SRT fanout; CultMesh owns status and operator surface.
## Run On Raven
From a Mimir checkout on Raven:
```powershell
.\scripts\start-raven-daemon.ps1
```
Defaults:
- Mimir target: `srt://10.77.0.2:5200`
- OBS target: `srt://10.77.0.2:5204`
- Eve/CultMesh dashboard: `ws://0.0.0.0:8801/eve/deck`
- Binary CultMesh dashboard stream: `ws://0.0.0.0:8801/eve/deck/cultmesh`
- Health: `http://127.0.0.1:8801/health`
- State cache: `C:\Meta\Mimir\state\raven-capture-mux.ccmp`
- Logs: `C:\Meta\Mimir\logs`
Use a dry run to inspect the exact FFmpeg pipeline without starting capture:
```powershell
.\scripts\start-raven-daemon.ps1 -DryRun
```
Disable the OBS sink when only Mimir should receive the mux:
```powershell
.\scripts\start-raven-daemon.ps1 -NoObsTarget
```
Use a specific Realtek render endpoint when Windows exposes more than one:
```powershell
.\scripts\start-raven-daemon.ps1 -AudioDevice "Realtek"
```
If Raven already has a host-level CultMesh server bound, keep this daemon as a local typed-state publisher and Eve endpoint:
```powershell
.\scripts\start-raven-daemon.ps1 -NoCultNetServer
```
## Receiver Shape
The daemon sends identical MPEG-TS SRT streams containing:
- `raven-display`: Raven desktop via `ddagrab` or `gdigrab`, encoded with `h264_nvenc`
- `raven-realtk-loopback`: Raven render loopback via `Mimir.WasapiLoopback` or the PowerShell WASAPI fallback, encoded as AAC
Mimir receives the ingest copy on `5200`. `scripts/start-raven-av-demux.ps1`
can split that copy back into local video and audio source lanes for the
current raw ingest adapters. OBS receives its own copy on `5204` as an ordinary
SRT Media Source; it does not compete with Mimir for the `5200` listener.
The typo in the existing source id, `raven-realtk-loopback`, is preserved until the receiver config is migrated.
## Idunn Lifecycle
Idunn watches the Raven daemon from Odin's local keepalive body:
- Health: `E:\Projects\Odin\scripts\health-raven-capture-mux.cmd`
- Restart: `E:\Projects\Odin\scripts\restart-raven-capture-mux.cmd`
- Default SSH target: Raven's WireGuard address, `10.77.0.4`
If Raven is unreachable, Idunn should record the failed health probe and raise
the normal keepalive decision instead of pretending the daemon is healthy.

130
docs/raven-eve-ingest.md Normal file
View file

@ -0,0 +1,130 @@
# Raven And Eve Ingest
## Objective
Ingest Raven screen capture plus Eve camera and microphone capture into Mimir,
then let the existing synchronization hub place those samples in rolling buffers
beside Scarlett audio and local camera sources.
## Authority Map
- Owner: `Mimir.Runtime` owns decoded samples and rolling-buffer placement.
- Inputs: Raven currently sends H.264/MPEG-TS over the SRT bridge to Starfire
port `5200`; Eve currently sends camera frame-events over WebSocket port
`8793`; Eve currently sends microphone frame-events over WebSocket port
`8794`; Scarlett/ASIO carries audio and sync evidence. The target Verse shape
is CultMesh reliable UDP: typed stream-frame envelopes plus bounded body
shards for media, with compact state/cursors/backpressure beside them.
- Outputs: `raven-display`, `eve-camera`, and `eve-mic` emit
`MimirStreamSample` records with video or audio descriptors.
- Derived state: network arrival timestamps are capture metadata only. Raven
audio-in-Scarlett remains the timing witness for global alignment.
- Forbidden writers: OBS, FFmpeg sender clocks, WebSocket receive time, SRT
arrival time, and CultMesh packet arrival time do not own final sync truth.
- Cut line: EveCanvas owns Eve's sensors directly. External iOS streaming apps
are fallback tools, not the current architecture. SRT/WebSocket bridge
transports are diagnostics and compatibility edges once the reliable-UDP
Verse media lane exists.
## Starfire Receiver
Use the Raven/Eve runtime config:
```powershell
$env:MIMIR_RUNTIME_CONFIG = "E:\Projects\Mimir\config\mimir-runtime.raven-eve.example.json"
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --seconds 5
```
When both senders are active, require sample arrival:
```powershell
$env:MIMIR_RUNTIME_CONFIG = "E:\Projects\Mimir\config\mimir-runtime.raven-eve.example.json"
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --seconds 5 --require-samples
```
The config starts one FFmpeg listener and two Eve sensor receiver processes:
- `raven-display`: `srt://0.0.0.0:5200?mode=listener&latency=120000&timeout=5000000`
- `eve-camera`: `ws://0.0.0.0:8793/eve/camera`
- `eve-mic`: `ws://0.0.0.0:8794/eve/mic`
The Raven listener decodes to raw BGRA frames on stdout. The Eve receivers emit
typed JSON frame-events from EveCanvas camera/mic packets. Mimir stores all of
them in the normal synchronization buffers.
## Raven Sender
Run this on Raven from the logged-in desktop session:
```powershell
E:\Projects\Mimir\scripts\start-raven-screen-capture-sender.ps1 -TargetHost 192.168.1.66 -Port 5200
```
The script uses `gdigrab` plus `h264_nvenc`, then calls Starfire's SRT listener.
Do not launch the capture itself through a blind SSH session unless the desktop
capture path has been proven there; `gdigrab` captures the interactive Windows
desktop, and service/session boundaries can turn screen capture into a black
box with excellent logs.
Raven sync signal: route Raven audio or a Raven-generated chirp into the
Scarlett alongside the microphone. Mimir should treat that audio evidence as
the authority for aligning Raven video with the rest of the program.
## Eve Sensor Sender
EveCanvas now starts native AVFoundation capture after permission grants:
- Camera frames are sampled from `AVCaptureVideoDataOutput`, JPEG-compressed,
wrapped as `video-frame` events for `eve-camera`, and sent to
`ws://192.168.1.66:8793/eve/camera`.
- Microphone blocks are captured with `AVAudioEngine`, packed as interleaved
Float32 samples, wrapped as `audio-block` events for `eve-mic`, and sent to
`ws://192.168.1.66:8794/eve/mic`.
This is an inspectable first transport, not the final high-throughput shape.
When cadence or bandwidth becomes the bottleneck, replace the JSON/base64 event
transport with CultMesh reliable-UDP media frames and body shards, optionally
carrying VideoToolbox H.264 or another explicit media payload. Preserve the same
source ownership and Mimir runtime sample contract.
Deploy EveCanvas after staging:
```powershell
powershell -ExecutionPolicy Bypass -File E:\Projects\Eve\scripts\stage-to-eve.ps1
ssh eve "cd /var/mobile/Projects/Eve && export THEOS=/var/theos && make package install && uiopen --bundleid org.gamecult.evecanvas"
```
## Smoke Test
The raw-video source has a local synthetic smoke that does not require Raven or
Eve:
```powershell
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --ffmpeg-rawvideo-source-smoke
```
That smoke starts FFmpeg with `testsrc2`, decodes BGRA frames through stdout,
and verifies the sample metadata and payload size that Mimir receives.
The Eve sensor receiver can be checked without launching the whole runtime:
```powershell
dotnet run --project .\src\Mimir.EveSensorReceiver\Mimir.EveSensorReceiver.csproj -- --port 8793 --path /eve/camera --source-id eve-camera --type video-frame
```
Periwinkle's Android Eve client publishes binary CultMesh sensor observations:
```powershell
dotnet run --project .\src\Mimir.EveSensorReceiver\Mimir.EveSensorReceiver.csproj -- --port 8796 --path /eve/periwinkle --source-id periwinkle-accelerometer --type cultmesh-observation
```
The receiver accepts `mimir.eve_sensor_observation.v1` MessagePack packets on
that lane. The device remains an observation source; Mimir owns synchronization
and interpretation.
The same receiver also accepts `mimir.eve_media_observation.v1` MessagePack
packets. Periwinkle publishes low-rate camera luma frames as
`periwinkle-camera` and microphone PCM blocks as `periwinkle-mic` on the
CultMesh observation lane. The legacy iPad Eve camera and mic uplinks have
matching binary media document support while their existing `eve-camera` and
`eve-mic` ports remain usable.

View file

@ -1,333 +0,0 @@
# Moonlight lessons for Muninn streaming and Sleipnir input
Date: 2026-07-16
Upstream examined: `moonlight-stream/moonlight-common-c` commit `82e25148549d249a80aaf4afc68e2abb07072edf`
## Verdict
Moonlight is strong because it does not ask one transport policy to serve every
realtime signal. It gives video, audio, control, and input different loss,
ordering, recovery, and freshness rules, then keeps each queue bounded by what
the user can still perceive.
Muninn is not missing all of that machinery. Its media packetizer already owns
access-unit boundaries, timestamps and deadlines, video parity, dependency and
keyframe metadata, bounded assembly, receiver damage feedback, repair caching,
and keyframe requests. Sleipnir already rejects regressing sequence/timestamp
pairs, expires reliable HID sends after 25 ms, reports stage latency, and
neutralizes stale output.
The 2026-07-17 field cut closes most of the original mechanism gaps:
- canonical video and video parity use the unreliable `realtime` lane; typed
8+8 GF(256) blocks, selective repair, and decode-chain IDR recovery own loss;
- canonical 10 ms PCM audio uses `realtime`, with 4+2 FEC, a 40 ms reorder
owner, and short concealment; reliable delivery is reserved for control;
- the consumer deadline is 100 ms by default and 250 ms in the Raven field
profile; queues, retransmits, repair material, and assemblies derive from it;
- receiver pressure actuates the live NVENC encoder through multiplicative
backoff and conservative additive recovery; startup begins at half the
requested ceiling because fixed video parity can roughly double wire load;
- Sleipnir latest state and ordered edges have distinct epoch/sequence and
application-ack semantics, including bounded overflow rebase.
The remaining claim boundary is evidence: direct and impaired Raven-to-
Starfire soaks, reconnect repetition, A/V skew measurement, and a live virtual
HID exercise. The machine is no longer missing a named owner for these paths;
it still has to prove those owners under pressure.
Do not transplant Moonlight wholesale. Distill its ownership model into the
existing typed Muninn/Sleipnir pipeline.
## Authority map
### Muninn media
- Owner: Muninn owns source-local capture, encoding child lifecycle, media
packetization, repair material, and sender feedback response.
- Inputs: encoded H.264/H.265 access units, encoded audio packets, source media
timestamps, configured latency target, and receiver feedback.
- Outputs: typed video chunks/parity, audio packets, recovery/keyframe actions,
and transport/quality telemetry.
- Derived state: RUDP acknowledgements, resend counters, CultCache receipts, OBS
local lowering, and operator cards are evidence or delivery substrate. They do
not own media deadlines.
- Forbidden writers: generic RUDP retry policy must not decide whether expired
media is still useful; OBS must not decide stream synchronization; persistence
must not become a hot media queue.
### Sleipnir input
- Owner: Sleipnir owns selection/remapping and local virtual-HID emission.
Muninn owns source-local device sampling and event classification.
- Inputs: a discovered Muninn input stream, typed mapping intent, monotonic
stream sequence, source-local capture time, and receive time.
- Outputs: local virtual-HID state plus latency/freshness receipts.
- Derived state: the latest full controller snapshot is replaceable state.
Button transitions, text/code points, connect/disconnect, and neutralization
are non-replaceable edges until acknowledged or superseded by an explicit
state transition.
- Forbidden writers: generic reliable queue order must not replay stale analog
or motion samples; wall-clock subtraction must not claim one-way latency
without a clock model; the telemetry/store path must not repair a missing hot
stream.
## What Moonlight actually does
### It separates stream semantics
Moonlight receives video and audio over UDP/RTP-like paths while input travels
through its control path (or a legacy TCP socket). The important lesson is not
the literal socket choice. It is that the queues have different contracts.
Video is frame/dependency oriented; audio is short-block continuity oriented;
input is event/state oriented.
Source: [stream initialization and input transport](https://github.com/moonlight-stream/moonlight-common-c/blob/82e25148549d249a80aaf4afc68e2abb07072edf/src/InputStream.c),
[video receive path](https://github.com/moonlight-stream/moonlight-common-c/blob/82e25148549d249a80aaf4afc68e2abb07072edf/src/VideoStream.c), and
[audio receive path](https://github.com/moonlight-stream/moonlight-common-c/blob/82e25148549d249a80aaf4afc68e2abb07072edf/src/AudioStream.c).
### Video recovery is dependency-aware and deadline-shaped
The video queue groups packets by frame and FEC block, tracks missing data,
reconstructs recoverable blocks with Reed-Solomon parity, rejects old/replayed
frames, and advances frame authority explicitly. Depacketization can request an
IDR when damage makes the predictive chain unsafe. The system repairs useful
frames and abandons poisoned dependency chains; it does not wait forever for
generic reliable delivery.
Source: [RTP video queue](https://github.com/moonlight-stream/moonlight-common-c/blob/82e25148549d249a80aaf4afc68e2abb07072edf/src/RtpVideoQueue.c) and
[video depacketizer](https://github.com/moonlight-stream/moonlight-common-c/blob/82e25148549d249a80aaf4afc68e2abb07072edf/src/VideoDepacketizer.c).
### Audio recovery is a small bounded continuity problem
Moonlight groups audio into fixed FEC blocks, orders packets by RTP sequence,
recovers missing shards, detects out-of-order behavior, and changes how long it
waits based on observed reordering. When recovery cannot complete in time, it
allows a discontinuity and lets the audio decoder's concealment path carry the
scar. It does not turn late audio into permanent latency.
Source: [audio FEC and reorder queue](https://github.com/moonlight-stream/moonlight-common-c/blob/82e25148549d249a80aaf4afc68e2abb07072edf/src/RtpAudioQueue.c) and
[packet-loss concealment handoff](https://github.com/moonlight-stream/moonlight-common-c/blob/82e25148549d249a80aaf4afc68e2abb07072edf/src/AudioStream.c).
### Input reliability is constrained by semantic coalescing
Moonlight has one send thread and a bounded queue, but it avoids blindly sending
every sampled value. Relative mouse motion is accumulated, absolute pointer
motion keeps the newest position, pen hover/move keeps the newest compatible
event, controller analog state is merged while button changes split batches,
and sensor motion keeps the newest value per sensor. A roughly millisecond-scale
batching opportunity can reduce latency by preventing ENet queue buildup.
This is the most important Sleipnir lesson: reliable delivery is safe only after
replaceable state has been collapsed and edges have been preserved.
Source: [input batching and send queue](https://github.com/moonlight-stream/moonlight-common-c/blob/82e25148549d249a80aaf4afc68e2abb07072edf/src/InputStream.c).
### It exposes failure to the owning layer
Packet loss, FEC recovery/failure, queue overflow, frame loss, and keyframe need
are explicit signals. Moonlight's public documentation also warns that allowing
frames to queue for smoothness can create high latency; its balanced mode keeps
only about one frame for jitter smoothing. The machine treats queue depth as a
product decision, not an incidental container property.
Source: [Moonlight frame-pacing FAQ](https://github.com/moonlight-stream/moonlight-docs/wiki/Frequently-Asked-Questions#what-do-the-frame-pacing-options-on-the-android-client-mean).
## Current Muninn/Sleipnir comparison
| Concern | Moonlight | Muninn / Sleipnir now | Distilled judgment |
|---|---|---|---|
| Video unit | Encoded frame, FEC blocks, dependency recovery | Encoded access unit, chunks/parity, frame/dependency IDs, deadlines | The conceptual unit is already correct. Keep it. |
| Video loss | FEC first; abandon damaged/obsolete work; request IDR | Typed 8+8 block FEC, deadline-bound repair, completed-frame tombstones, IDR actuation | Same ownership pattern; field loss ratios and parity overhead remain tunable. |
| Audio loss | Small FEC blocks, reorder-aware wait, decoder concealment | Realtime 4+2 PCM FEC, 40 ms reorder, bounded concealment | Mechanism is closed; Opus FEC/PLC is a later efficiency/quality cut. |
| Congestion | Stream-specific queues and feedback behavior | Half-ceiling startup, 15% pressure backoff, bounded additive ramp, fresh media ahead of repair | Closed loop exists; impairment evidence decides its final constants. |
| Latency budget | Minimal queueing; frame pacing explicitly trades latency for smoothness | Consumer-owned 100 ms default/250 ms Raven proof; 25 ms HID expiry | Correct owner and bounded values; glass-to-glass measurement remains. |
| Input state | Coalesces replaceable motion/sensor/analog values | Epoch-scoped latest state supersedes stale state | Structural rather than polling-accidental. |
| Input edges | Button/type changes terminate batches; text is ordered | Ordered edge sequence, application ACK, bounded replay, epoch rebase | Structural; live virtual-HID impairment proof remains. |
| Staleness | Queue construction prevents much stale work | Receiver drops regressing timestamps and neutralizes after timeout | Keep the guard, but move freshness authority earlier into enqueue/supersession. |
| Timing | RTP/media clocks and local queue timing | source Unix nanoseconds plus receiver instants | Use monotonic source time plus an explicit clock-offset/uncertainty model; do not call raw wall-clock subtraction latency. |
| Security | Authenticated encryption in modern stream generations | CultNet typed transport policy varies by lane | Preserve typed schemas, but specify replay/authentication guarantees for hot media and HID lanes. |
## Recommended cut
### 1. Define traffic classes before changing transport
Add one narrow typed delivery policy shared by Muninn and Sleipnir:
- `state-latest`: analog axes, pointer position, gyro/accelerometer samples;
- `edge-ordered`: button/key transitions, text/code points, device lifecycle;
- `media-video`: dependency-bearing access units with decode deadlines;
- `media-audio`: continuity blocks with playout deadlines;
- `control-reliable`: subscriptions, mappings, capability negotiation, and
keyframe requests.
This is not a generic router. It is a finite statement of the five contracts the
existing machine already has. Each class must name supersession, ordering,
expiry, recovery, and observability.
### 2. Cut the two-second realtime default
Derive deadlines from the actual consumer:
- Sleipnir state-latest: tens of milliseconds, with replacement before send;
- input edges: reliable until acknowledged, but compacted against later
authoritative state where safe;
- interactive video: frame/decode budget, initially test 50–150 ms on LAN;
- audio: playout budget, initially test 40–100 ms on LAN;
- Mimir's five-second analysis reservoir remains Mimir compute budget, not a
network permission to deliver old frames.
The exact values must be earned by field traces. The ownership change is not
optional: transport retry duration becomes derived from the media/input
deadline, never the other way around.
### 3. Make Sleipnir freshness structural
Split the hot input frame into:
- a latest-state snapshot with `state_sequence`, source monotonic capture time,
and supersedes-through sequence;
- an ordered edge list with its own sequence range;
- receiver acknowledgement of the highest applied state and edge sequence.
Muninn may overwrite an unsent latest-state snapshot. It may not overwrite an
unacknowledged edge. Sleipnir applies edges in order, applies only the newest
state, and neutralizes on a named lease timeout. Manual, remapped, replayed, and
live paths must use the same commit primitive.
### 4. Close Muninn's feedback loop
Receiver feedback should drive named sender decisions:
- request/force keyframe after unrecoverable dependency loss;
- choose repair only when estimated arrival precedes frame deadline;
- tune parity within a bounded profile from recent burst-loss evidence;
- reduce encoder bitrate/pacing when queue delay grows;
- expose encode, queue, network, assembly, decode, and presentation stages
separately.
Do not add a second congestion controller beside generic RUDP and hope the two
argue productively. Media policy owns the send budget; RUDP exposes delivery
mechanics and obeys the deadline.
### 5. Prove it on the layer the user feels
Build one loss/jitter/reorder test harness and record timelines for:
- controller press and release during 1%, 5%, and burst packet loss;
- high-rate analog/gyro motion under an artificial sender stall;
- video dependency loss and keyframe recovery;
- audio burst loss and concealment;
- queue growth under bitrate oversubscription;
- disconnect/reconnect and stale-session replay;
- clock offset and drift between Muninn and Sleipnir hosts.
Acceptance should bound p50/p95/p99 capture-to-HID age, lost/duplicated edges,
time-to-neutral, glass-to-glass video age, audio discontinuities, A/V skew, and
recovery time. Final-state-only tests will politely conceal the corpse.
## What not to copy
- Do not copy Moonlight's C implementation or protocol packets into GameCult
code without an explicit GPLv3 licensing decision. Architectural study is not
license laundering. See [Moonlight's license](https://github.com/moonlight-stream/moonlight-common-c/blob/82e25148549d249a80aaf4afc68e2abb07072edf/LICENSE.txt).
- Do not replace typed CultMesh discovery, command state, or ownership with
RTSP/SDP or a Moonlight compatibility stack.
- Do not make ENet the new universal answer. Moonlight's strength comes from
semantic queues above the carrier, not from the carrier's brand name.
- Do not merge input, audio, and video into one ordered reliable stream. One
lost video fragment must not head-of-line block a button release or audio
packet.
- Do not treat FEC as magic. Parity is useful only when its overhead and recovery
time fit the deadline.
## Distilled doctrine
Realtime strength is not “reliable UDP.” It is knowing which truth survives
loss:
- the newest state replaces old state;
- an edge remains until applied or explicitly superseded;
- a media packet is valuable only while its frame or playout deadline is alive;
- predictive video damage is healed by recovery or a new reference, not by
delivering every corpse;
- queues are latency budgets made physical;
- feedback that cannot change sender behavior is telemetry, not control.
Muninn and Sleipnir already have enough organs to become strong. The next work
is to give each signal one explicit survival law and make the transport obey it.
## 2026-07-17 LAN field results
The Moonlight-derived ownership changes now have Raven-to-Starfire field
evidence over LAN, with Raven processes launched only through hidden scheduled
tasks:
- Muninn video uses independently recoverable block parity plus bounded repair.
During a seeded 1% datagram-loss run, the OBS receiver advanced from 2 to 12
video assembly expiries while forwarding 14,400 additional payload groups:
99.93% completed without expiry. Audio reported zero failures, skips, or stale
packets in the same interval. The impairment proxy forwarded 2,147,356 of
2,169,105 datagrams with zero queue overflow.
- Sleipnir latest-state delivery remained live through seeded 1% loss. Cutting
the proxy caused the non-neutral virtual Xbox pad to neutralize after 1.56 s,
owned by the named stale-input lease; restoring the hidden proxy reconnected
and resumed state without operator action.
- A continuous 64-packet receive drain was retaining the newest input for about
48-57 ms before HID application. Bounding that drain to 2 ms reduced live
receiver-owned receive-to-apply time to typically 69-204 microseconds. A
witnessed 16 ms spike remains to characterize.
- Starfire and Raven wall clocks differ by roughly 2.38 s. Sleipnir no longer
publishes their raw subtraction as one-way latency. Its surface reports that
source-clock latency is unavailable without a clock model and exposes only
monotonic receiver-owned stage durations.
These results clear the initial 99.9% media-completion and disconnect-safety
gates at 1% independent loss. They do not yet prove glass-to-glass latency,
A/V skew, ordered quick-tap delivery under loss, 5%/burst behavior, or a long
mixed-impairment soak. Those remain acceptance gates, not footnotes.
The field comparison also exposed a generic transport invariant Moonlight's
long-running sessions force us to respect: bounded wire identifiers must wrap
deliberately. CultNet's fragmented-message ID previously saturated at the
maximum `u16`, so fragmented audio became unreassemblable after the boundary.
Wrap-to-one plus a boundary test removed the repeatable minute-eight collapse;
a subsequent 10-minute direct LAN run remained clean across the boundary.
## 2026-07-17 ordered-input closure
The quick-tap gate exposed four authorities that generic RUDP reliability had
blurred together:
- A Windows sleep requested at two milliseconds could still miss a short tap.
Muninn now observes XInput on a dedicated high-resolution worker; transport,
discovery, serialization, and logging cannot delay capture.
- Global cumulative ACKs could not retire a reliable packet arriving more than
32 realtime sequence numbers late, and reliable duplicates were not ACKed
after application deduplication. CultNet now directly acknowledges the late
packet and acknowledges reliable duplicates. This collapsed a runaway field
trace from hundreds of thousands of resend datagrams to a bounded flow.
- Muninn used semantic application ACK as permission to send the next edge.
One delayed ACK therefore serialized the entire controller. It now sends a
bounded window of captured edges; CultNet owns retransmission, Sleipnir's
epoch/sequence cursor owns ordering, and semantic ACK only retires history.
- Sleipnir published CultMesh/Eve and Idunn state synchronously on the actuator
thread. Those writes produced 0.7-1.8 second visible button holds. A bounded
coalescing publisher worker now owns control-plane projection, leaving the
input thread to receive, order, actuate, and acknowledge.
The resulting first-hot-plug field proof generated 100 taps with a 16 ms source
press and 84 ms gap. Under seeded 1% independent loss, every layer observed
exactly 100 presses and 100 releases; Raven-visible durations were 24-57 ms.
Under eight consecutive dropped packets every 200 datagrams, every layer again
observed exactly 100 presses and 100 releases; visible durations were 24-40 ms,
the final virtual pad was neutral, and the impairment proxy reported no queue
overflow. Raven ran through hidden scheduled tasks over `192.168.1.0/24`; no
WireGuard route participated. Both daemons used the Eve/Aetheria/VoidBot
experimental CultLib lineage pinned at `8965f3c0`.
This closes ordered digital edges at one 60 Hz source frame. It does not prove
sub-frame synthetic ViGEm pulses: eight-millisecond state-only pulses can fall
between XInput snapshots, and `XInputGetKeystroke` was rejected after it emitted
stale events from a reused user slot. Analog motion under a sender stall,
mixed media/input endurance, glass-to-glass video age, and A/V skew remain open.

View file

@ -1,100 +0,0 @@
# Moonlight-level reliability acceptance ledger
This ledger distinguishes implemented mechanisms from evidence that the actual
Raven-to-Starfire program timeline survives pressure. Passing unit tests is not
permission to call the field path done.
## Required contracts
| Contract | Current authority | Evidence | Status |
|---|---|---|---|
| Experimental CultMesh/CultNet lineage | Odin vendored CultLib snapshot `8965f3c0` | Snapshot provenance plus CultNet/Muninn/Sleipnir suites | proved locally |
| Low-delay NVENC | Muninn controllable encoder | D3D11 desktop capture, no B frames/lookahead, two-frame VBV, clean Annex-B decode | proved locally |
| Consumer deadline owns media lifetime | Muninn queues, repair cache, Mimir assembly/reorder | 100 ms default/250 ms Raven profile, bounded queues/cache/reliable expiry, late-frame repair refusal | proved structurally |
| Recoverable video loss avoids decoder reset | Mimir V4 block FEC/repair, Odin repair cache | Exhaustive 2..80 chunk, burst 1..8 tests in Rust/C++; completed-frame tombstones | proved locally |
| Decoder-chain loss forces immediate IDR | Muninn feedback owner and native encoder | Live 2560x1440 command produced IDR at frame 22 under a 600-frame GOP | proved locally |
| Congestion changes encoder output in-session | Muninn AIMD controller and native encoder | Half-ceiling start and protected cap; live 12-to-6 Mbps reconfiguration produced clean IDR | proved locally |
| Audio survives small erasure bursts | Odin 4+2 GF(256), Mimir FEC/reorder/concealment | Exhaustive one/two-erasure Rust and C++ tests; production DLL build | proved locally |
| Latest input state supersedes stale state | Muninn epoch/state sequence, Sleipnir cursor | Duplicate/reorder/stale epoch tests | proved locally |
| Button edges are not lost or retained forever | Muninn replay/app ACK/epoch rebase, Sleipnir ordered cursor | Quick-tap, ACK, overflow rebase, sparse-gap tests | proved locally |
| Raven runtime is LAN-only and non-interrupting | Hidden Raven scheduled task, direct LAN SSH/media | One coherent Muninn/WASAPI/NVENC/FFmpeg tree; no WireGuard route; `-WindowStyle Hidden` | proved operationally |
| Real field path survives impairment and soak | Raven sender, Starfire receiver/OBS, `cultnet-impair` | Direct LAN clean run 05:34:17–05:44:37: 79,200 records, 697,450,085 bytes, 18,588 recovered shards, zero decode/assembly/media/audio failures and zero sender capacity eviction; proxy matrix and long soak remain | **direct clean passed** |
## Field acceptance matrix
Run the real capture body, production receiver plugin, decoder, audio output,
and Sleipnir virtual input through the seeded profiles under
`E:\Projects\Odin\tests\realtime-impairment`. Observe the rendered/audio/input
timeline, not only proxy CSV counters.
| Profile | Required result |
|---|---|
| clean, 10 minutes | no decoder reset, audio discontinuity, input edge loss, queue growth, or timestamp drift; video p99 delivery under 50 ms |
| iid 1%, 10 minutes | no input edge loss; audio FEC recovers eligible holes; at least 99.9% video frames presented; media p99 under 80 ms |
| iid 3%, 5 minutes | bounded degradation; at least 99% video frames presented; decoder recovery within 250 ms; no stuck controls |
| burst 4 and burst 8 | parity/repair handles eligible damage; unrecoverable dependency loss produces one requested IDR and recovery within 250 ms |
| reorder and duplicate | no duplicate input action, stale state regression, or unbounded assembly/reorder storage |
| delay/jitter | AIMD reduces bitrate before sustained deadline loss and recovers no faster than one step per two stable seconds |
| stall 250 ms | obsolete media expires; queues remain bounded; input latest state converges within 35 ms after release |
| reconnect | old input epoch cannot act; neutral/current state within 100 ms; decoded video within 500 ms |
| mixed 60-minute soak | stable memory/handle/thread counts and no progressive A/V drift |
The first seeded 1% proxy run is a failed acceptance result, not a pass: 987 of
95,741 datagrams were dropped; in roughly one minute the receiver recorded 38
expired video assemblies and 56 skipped audio packets while remaining bounded
and avoiding decoder reset. CultNet fragmentation currently sits below
application-level FEC, so one lost transport fragment discards an entire FEC
shard. The next reliability cut is to make protected media shards fit one wire
datagram or move fragmentation above the erasure-code boundary.
## Field scars
- A fragmented CultNet RUDP session failed repeatably after roughly eight to
nine minutes because the `u16` fragment-set ID used saturating increment and
became permanently stuck at 65,535. Every later fragmented audio document
then collided in one receiver reassembly key. The allocator now wraps from
65,535 to 1, with a boundary test. The clean field run above crossed that
boundary without a single audio reorder, skip, stale packet, or failure.
- Audio and video producer ingress now use separate bounded channels. Video
capture can no longer block audio ingestion before the audio-priority send
scheduler sees it. Capacity-eviction counters proved zero during the clean
acceptance run.
## 2026-07-17 bounded-media field cut
The live path now keeps every protected shard in one datagram, sends AAC ADTS
in fixed 864-byte 4+2 audio blocks, uses 768-byte video shards with 8+4 block
FEC, and begins each decoder epoch at a complete IDR. The sender admits video
to its repair cache when the frame enters the queue, accepts the production
CultNet `realtime` channel, and can service bounded repair requests at
64/96/128 ms in batches of up to eight chunks.
Seeded Raven-to-Starfire evidence:
- iid 1%: 802 of 78,406 datagrams dropped; zero decode drop, video expiry,
audio failure, or audio skip; 1,800 video frames recovered;
- burst-4: 1,436 of 71,910 datagrams dropped; zero expiry or audio skip across
1,800 forwarded frames;
- reorder: 2,872 reorder events; zero expiry or audio skip across 900 frames;
- iid 3%: 2,564 of 85,731 datagrams dropped; three of 5,400 frames expired,
zero audio skip (99.94 percent video delivery);
- burst-8: 190 chunks repaired; one of 1,800 frames expired, zero audio skip;
- 250 ms stall: 420 datagrams withheld, one frame expired, 68 chunks repaired,
an IDR was requested, and another 900 frames arrived without a stuck decoder
or audio skip.
Delay/jitter produced one expiry in 900 frames. Duplicate, reconnect, longer
iid runs, and the 60-minute mixed soak remain open acceptance rows.
Field scar: production video uses the `realtime` channel, but the repair cache
previously admitted only legacy `media` payloads. Synthetic tests built the
legacy shape and certified a repair path that was empty in production. The
production-channel regression test now owns this boundary.
## Completion rule
Moonlight-level performance is reasonably expected only after all local rows
remain green and the real field matrix passes twice: once on direct LAN and once
on the deployed route actually used during streaming. Failure should change the
owning deadline, FEC, encoder, or input authority—not add a repair loop around
the symptom.

View file

@ -0,0 +1,53 @@
# Mimir Scene Editor Control Surface
Mimir now has a runtime-owned scene editor state for arranging the visible
presentation inputs inside the Fensalir-hosted Mimir window. This editor is not
the OBS program output. It is an operator surface for placing source panels,
text panels, and imported-model placeholders before later renderer cuts turn
those nodes into final composited pixels or volumetric field claims.
## Authority Map
- Owner: `MimirSceneEditorState` owns editor camera, scene nodes, selected node,
visibility, lock state, 2D-plane transform, and active gizmo mode.
- Inputs: rolling video buffers provide sensor-feed nodes; Fensalir `InputState`
provides mouse/key edits; the Mimir UI provides visibility, reset, transform,
text, and model-path commands.
- Outputs: `MimirRuntime` reads the editor state into the Fensalir frame camera,
spline outlines, selection gizmos, and SDF handle markers.
- Derived state: the hierarchy text, selected-node readout, editor splines, and
handle SDF objects are derived from the editor graph. They do not own node
truth.
- Forbidden writers: program-composite controls, OBS source endpoints, bridge
scripts, and FieldEvidence surface intents do not decide editor transforms.
- Shared paths: UI buttons/sliders and mouse-drag gizmos all commit through
`MimirSceneEditorState`; reset returns to each node's stored default
transform.
- Deletion line: do not add a separate transform cache in the renderer or OBS
path. Fensalir may later render richer node types, but it should consume this
graph or a deliberate successor, not invent a competing editor graph.
## Current Controls
- `Mimir Editor` panel exposes the scene graph, selected-node visibility and
lock toggles, transform reset, camera reset, and transform sliders.
- Active gizmo modes are `Grab`, `Rotate`, and `Resize`; keys `1`, `2`, and `3`
switch modes, and left-drag applies the active mode to the selected node.
- Sensor-feed nodes are created from live video rolling buffers.
- SDF text panels and model placeholders can be added from the panel.
- `--scene-editor-smoke` proves graph creation, visibility toggles, transform
reset, mouse-drag transform edits, text-node creation, model placeholder
import, and derived gizmo geometry.
## Renderer Hooks Still Open
- World SDF text: Fensalir currently has DirectWrite overlay text and SDF object
proxies. Scene text needs a renderer-owned SDF/MSDF glyph atlas plus billboard
or mesh text lowering before text glyphs are truly drawn inside the scene.
- Imported models: Fensalir resource contracts already know mesh packages, but
selected mesh draw/material lowerings are still future work. The current
import control creates graph nodes with paths; it does not decode ASSIMP
meshes or upload vertex/index resources yet.
- Pixel-accurate gizmo hit-testing: current gizmo edits apply to the selected
node. Fensalir needs an editor hit-test path before handles can be clicked
directly in world space.

View file

@ -0,0 +1,127 @@
# Self-Hosted Live Edge
Mimir's self-hosted broadcast path keeps OBS optional. Starfire owns the final
program frame and audio mix; Yggdrasil owns public byte distribution.
```mermaid
flowchart LR
A["Fensalir program texture"] --> B["Starfire NVENC encode"]
C["Faust/Mimir mix"] --> B
B --> D["SSH TCP forward"]
D --> E["Yggdrasil localhost RTMP ingest"]
E --> F["HLS segment ring"]
F --> G["streampixels.gamecult.org static HTTPS"]
G --> H["gamecult.org/livestream static viewer"]
```
## Authority Map
- Owner: `Mimir.Broadcast` owns the Starfire-side encoded push command.
- Inputs: final program media, FFmpeg/NVENC capability, and the SSH-forwarded
RTMP endpoint.
- Outputs: one CBR H.264/AAC RTMP stream to Yggdrasil.
- Derived state: Yggdrasil's HLS playlist and segments are derived from that
encoded stream. They do not own composition, timing, or transcoding.
- Forbidden writers: Yggdrasil must not transcode the program stream in the
first live edge. OBS is not the primary broadcast owner.
- Shared paths: synthetic smokes and real program pushes use the same RTMP URL:
`rtmp://127.0.0.1:11935/live/mimir`.
- Deletion line: if a later CDN or WebRTC edge appears, keep this path as the
simple origin proof or delete it. Do not keep two public stream authorities.
## First Smoke
On Starfire, with an SSH local forward active to Yggdrasil's localhost RTMP
port:
```powershell
dotnet run --project .\src\Mimir.Broadcast\Mimir.Broadcast.csproj -- --smoke
dotnet run --project .\src\Mimir.Broadcast\Mimir.Broadcast.csproj -- --print-command
```
The default push target is:
```text
rtmp://127.0.0.1:11935/live/mimir
```
The public HLS playlist is served on the existing Yggdrasil-routed
StreamPixels domain:
```text
https://streampixels.gamecult.org/mimir/live/hls/mimir.m3u8
```
The static viewer lives on the root GameCult site:
```text
https://gamecult.org/livestream
```
## Local Recording Before Public Push
Before pushing a live program to Yggdrasil, record the same simple stream-proof
shape locally and judge sync from the artifact:
```powershell
powershell -ExecutionPolicy Bypass -File .\scripts\record-stream-proof-output.ps1 -StartRavenSender -DurationSeconds 20
```
The recording harness listens for a Raven muxed A/V SRT feed on a private test
port, overlays Kiyo Pro as picture-in-picture, mixes Raven Realtek loopback with
Starfire's Scarlett capture, applies conservative FFmpeg denoise filters, and
writes an MP4 under `artifacts\runtime\stream-proof\`.
The current routing contract is explicit:
- Starfire Realtek/default render is the chirp/speaker emission lane.
- Raven Realtek/default render loopback is the co-streamer game/program lane.
It is packaged with Raven's NVENC video feed back to Starfire.
- Starfire Scarlett carries the hero mic capture and monitored local mix for
judgment. Raven's Scarlett is only routing the remote shotgun into Starfire
Scarlett, currently on channel 1.
- The proof recording is allowed to mix those lanes for human sync judgment, but
it must not collapse their authority. Starfire Realtek proves chirp emission,
Raven Realtek proves the co-streamer/game program path, and Scarlett proves
the hero mic/shotgun capture path.
This is a bridge recording, not public broadcast truth. It exists so humans can
judge audio/video sync and tell Mimir which offsets to apply before the RTMP/HLS
edge is involved. Offset knobs are explicit:
```powershell
powershell -ExecutionPolicy Bypass -File .\scripts\record-stream-proof-output.ps1 `
-StartRavenSender `
-RavenOffsetSeconds 0.000 `
-KiyoOffsetSeconds 0.000 `
-MicOffsetSeconds 0.000
```
If Windows firewall blocks direct SRT/UDP during a local proof, run the Raven
leg through an SSH TCP tunnel instead:
```powershell
Start-Process ssh.exe -ArgumentList '-o ExitOnForwardFailure=yes -N -L 6204:127.0.0.1:5204 -l "madman''s lullaby" 192.168.1.84' -WindowStyle Hidden
powershell -ExecutionPolicy Bypass -File .\scripts\record-stream-proof-output.ps1 `
-StartRavenSender `
-RavenSenderListens `
-RavenSenderTransport tcp-listener `
-RavenInputOverride tcp://127.0.0.1:6204
```
That fallback keeps the proof private to the LAN/SSH path and avoids changing
machine firewall policy just to capture a judging artifact.
## Scaling Rule
Viewer capacity is approximately:
```text
available Yggdrasil egress / encoded bitrate
```
Micro-optimizations count only when they improve target-host measurements:
served Mbps, p95 segment latency, CPU, memory, syscalls, or stall rate on
Yggdrasil. The first edge uses nginx static serving and kernel file paths so
the hot path is byte distribution, not application code.

View file

@ -10,7 +10,10 @@ flowchart TD
C["Leap stereo IR driver"] --> B
B --> D["native reservoir handles"]
D --> E["Fensalir GPU feature extraction"]
E --> J["per-sensor LED curve solve"]
E --> F["cross-view matching + flow"]
J --> K["camera rig calibration solver"]
K --> F
F --> G["surface/material claims"]
G --> H["brush/splat render budget"]
H --> I["Spout2/program video"]
@ -23,9 +26,51 @@ flowchart TD
- Process capture is a bridge edge, not the local six-camera foundation.
- Fensalir owns GPU extraction, fusion, material fitting, and render budgeting.
- Runtime buffers own retention and stream health, not scene reconstruction.
- LED strips are active calibration evidence. Each camera publishes ordered
bright-curve observations in its own local frustum/clipspace; only the global
residual calibration owner may fit those frustums into scene-space camera
intrinsics/extrinsics.
- Stable LED index identity requires temporal/color/address coding or another
correspondence source; uncoded identical lights are spline constraints, not
metric depth authority.
- PS Move controllers are active tracked-marker witnesses. USB HID owns
identity, buttons, IMU/report topology, and later LED/rumble control; the
glowing sphere is visual calibration evidence only when observed by cameras
and fused by the global residual owner.
- Exposure/gain sweeps optimize recovered spline quality: detected LED count,
curve coverage, spacing coherence, smoothness, exposure fitness, and
saturation penalty. Bright clipped blobs are not good evidence.
- The live LED sweep owner is `Mimir.BufferSmoke --led-spline-live-score` for
now: it opens direct sensor feeds, applies KS/UVC manual exposure/gain where
supported, scores actual luma frames, and reports the best setting per
source. PS3 Eye and Leap-specific controls still need native control cuts.
- The current all-sensor GPU proof is
`config/mimir-runtime.perfect-machine.local.json` plus
`Mimir.BufferSmoke --perfect-machine-live-smoke`: Leap, both PS3 Eyes, Kiyo
Pro, and regular Kiyo all enter the runtime as Fensalir texture leases. The
smoke requires five live camera Texture2D resources, one Leap stereo-depth
lowering, one disparity SurfacePage, and one derived point-cloud Mesh.
Camera Feature/SensorObservation claims now plan through Fensalir's
`GpuSensorFusion` backend instead of deferring. The current compute dispatch
samples LeapStereoIr, both Bayer8 PS3 Eyes, Kiyo Pro YUY2, and regular Kiyo
YUY2. Fensalir binds YUY2 through the documented `R8G8B8A8_UNORM` SRV view
over `DXGI_FORMAT_YUY2` and decodes `Y0/U/Y1/V` in compute before scoring
features.
- PS3 Eyes own high-rate sparse tracking evidence before they own depth:
`MimirSparseFeatureTracker` turns live Bayer/luma frames into stable
sensor-local feature tracks, and `MimirFeatureTrackFieldCandidate` lowers
them as camera Feature claims. This CPU path is a diagnostic bootstrap; the
production hot path belongs in Fensalir D3D12 feature extraction.
- The first camera-rig solver fits bounded camera translation updates from a
scene-anchored LED curve by minimizing ray-to-curve distance. Rotation,
intrinsics, and distortion are future owners.
- Once camera frustums are coherent, every detected surface candidate can be
resampled across the synchronized view set before Fensalir resolves it into
spatiotemporal surface/splat claims.
## Next Cut
Implement one concrete direct capture driver, feed `MimirVideoFrameDescriptor`
samples into `MimirVideoCaptureDriverSource`, and measure sustained cadence over
the default five-second window.
Replace provisional Leap projection constants with calibrated intrinsics and
add the global residual owner that consumes Leap surface errors, LED spline
observations, and later PS3 Eye/Kiyo/Eve feature tracks through one calibration
commit path.

View file

@ -0,0 +1,169 @@
# Mimir Verse Service Contract
Mimir is a Verse service for the measured room. It accepts local and network
sensors, publishes typed observations and capture pages, asks Fensalir to lower
field evidence into program surfaces, and exposes operator controls through
Eve/CultUI surfaces. OBS, browser dashboards, JSON logs, and screenshot streams
are lowerings or witnesses. They are not canonical presentation or state owners.
## Owner Map
- Mimir service owner: configuration, launch, calibration truth, sensor
identity, stream identity, synchronization state, Well publication, recorder
receipts, and service contract.
- `Mimir.Runtime`: bounded rolling buffers, source descriptors, source polling,
direct native ingest, audio synchronization analysis, and cached read models.
- Native sensor workers: device reads, closest-to-device timestamps, native or
GPU handles, unavoidable copy counts, and per-device actuator commands.
- `Mimir.Well`: typed live pages, configured composite state, timing pressure,
feature signals, capture-body publication, and the append-side observation
ledger.
- `Mimir.VerseRecorder`: run session receipts, body indexing, replay witnesses,
and `.cc`/CultCache-compatible export for offline reconstruction.
- `Mimir.FensalirDaemon`: reservoir job selection, queue pressure, GPU worker
ownership, program/reservoir surfaces, and daemon state.
- Fensalir: D3D12 resource resolution, field claims, selected lowerings,
temporal guide lanes, reservoir resolve, and program presentation.
- Faust/native DSP: hot audio alignment, sample movement, suppression,
separation, spatialization, and program stems.
- Eve/CultUI lowerings: operator interaction and display of provider-owned
state. They submit commands to owners; they do not own synchronization,
reservoir jobs, OBS composition, or calibration truth.
- Odin: discovery of Mimir's published CultMesh namespaces and Eve surfaces.
## CultCache Witness Contract
Durable state should be typed CultCache `.cc`, or CultCache-compatible with a
`.cc` witness/export during migration. The witness set is:
| State family | Owner | Durable shape |
| --- | --- | --- |
| Sensor state | `Mimir.Runtime` plus native workers | Source inventory, device identity, driver profile, clock domain, format, actuator capability, native handle class, copy count, and last health. |
| Media state | `Mimir.Well` / Fensalir | Capture page metadata, resource keys, body refs, fence refs, page timing, configured composites, and program surface refs. |
| Observation state | `Mimir.Runtime` / Well | Typed audio/video/IMU/feature/path observations with source id, time range, confidence, calibration id, and payload/resource reference. |
| Ledger state | `Mimir.VerseRecorder` | Session receipts, body index, replay cursor, migration version, schema witness, and rejected/drop reasons. |
| Control state | Runtime owners / Fensalir daemon / DSP | Operator commands, accepted command receipts, selected program layer state, reservoir worker pressure, audio actuator commands, and stream/record/live status. |
JSON may remain only as schema publication, diagnostic text export, external
tool boundary, or temporary replay witness. If a JSON file can decide behavior
without a typed `.cc` witness, it is still an owner and must be migrated.
## CultMesh Namespaces
Mimir should publish these namespaces for Odin discovery:
- `verse.mimir.service.v1`: service manifest, run id, host identity, schema
catalog, transport lanes, and nested Verse roots.
- `mimir.sensor.*`: source inventory, native worker health, device timing,
media format, actuator capability, and producer fences.
- `mimir.media.*`: capture pages, media bodies, resource references, program
texture/fence refs, stem refs, and replay body indexes.
- `mimir.observation.*`: sensor observations, media observations, feature
tracks, audio path evidence, calibration constraints, and local frustum hints.
- `mimir.ledger.*`: recorder session, append cursor, migration state,
body-index records, dropped-page reasons, and evidence receipts.
- `mimir.control.*`: operator commands, command receipts, recording/streaming
state, runtime mode, source placement, DSP actuator state, and reservoir
scheduling pressure.
- `mimir.eve.*`: `gamecult.eve.surface.v1` provider manifests, retained
surfaces, field bindings, assets, command bindings, and lowering metadata.
- `mimir.fensalir.*`: daemon summary, reservoir worker state, reservoir pressure,
program surfaces, resource resolver status, and selected lowering status.
Existing document names such as `mimir.well_snapshot.v1`,
`mimir.well_capture_page.v1`, `mimir.eve_sensor_observation.v1`,
`mimir.eve_media_observation.v1`, `mimir.eve_dashboard_state.v1`,
`mimir.fensalir_daemon_state.v1`, `mimir.fensalir_reservoir_worker_state.v1`,
and `mimir.fensalir_reservoir_pressure.v1` should be kept as concrete records
inside those namespaces until a schema catalog renames them deliberately.
## Transport Lanes
CultMesh reliable UDP is the default Verse transport for live Mimir network
state and media. TCP/WebSocket, SRT, browser streams, and local pipes are
diagnostic lowerings or external-boundary bridges unless a specific deployment
constraint temporarily promotes them.
The live lane split is:
- Typed control/state lane: compact documents such as timing state, command
receipts, health, stream cursors, and backpressure.
- Media stream-frame lane: typed per-frame envelopes such as
`mimir.cultmesh_stream_frame.v1` carrying source identity, capture time,
resource/body refs, payload format, and confidence.
- Media body-shard lane: bounded CultCache media/page shards with hashes,
cursors, receiver acknowledgements, and backpressure. Large bodies do not
travel as inline base64.
- Latest-state dashboard lane: lossy/current Eve surfaces for operators. It
may drop obsolete frames; it does not own durable media bodies.
Media streams are first-class Verse payloads. They still do not become timing
authority by arrival timestamp. Clock influence comes from decoded anchors,
source-local evidence, calibrated loopback, and explicit clock-domain state.
## Eve Surfaces
Mimir publishes operator presentation as Eve/CultUI DSL, preferably
`gamecult.eve.surface.v1`. The expected surfaces are:
- `mimir.sensor.field.surface`: sensor field view with source health, buffer
depth, timing confidence, calibration status, copy count, media format,
resource residency, and per-source observation pressure.
- `mimir.operator.flow.surface`: operator workflow for source selection,
program placement, audio strips, record/live controls, reservoir pressure,
sync mode, command receipts, and safe degraded-state visibility.
- `mimir.observation.ledger.surface`: session/replay view over observation
counts, body refs, accepted/dropped pages, cursor state, and schema witness.
- `mimir.fensalir.reservoir.surface`: daemon-owned worker queue, ready ratio,
timing confidence, GPU queue pressure, drop causes, and program surface refs.
The old dashboard broker, WebSocket `/eve/deck` lane, `/eve/dashboard`
compatibility endpoint, JSON health endpoints, JPEG streams, and browser
prototypes are lowerings or transport probes. They may mirror the Eve surface,
but they do not become presentation owners.
## Nested Verses
Mimir's service Verse contains smaller Verses with bounded authority:
- Room Verse: calibration truth, coordinate frame, sensor graph, clock model,
configured composites, and operator identity for one physical room.
- Sensor Verse: one device or logical source, including driver health, timing
domain, media format, actuator surface, and observation stream.
- Stream Verse: one synchronized media/control lane, including rolling-window
cursor, capture pages, resource refs, body refs, and backpressure state.
- Operator Verse: Eve/CultUI command surface, selected source/layer state,
command receipts, record/live state, and degraded-state visibility.
Nested Verses may publish local surfaces and state, but the parent Mimir service
owns discovery and contract routing. A sensor may report what it observed; it
does not become room clock authority unless explicitly promoted by calibration
state.
## Migration Order
1. Publish a `verse.mimir.service.v1` manifest that names active namespaces,
schema ids, run id, nested Verse roots, and the current transport lanes.
2. Give each current JSONL/session/body-index path a `.cc` witness or
CultCache-compatible export, starting with Well capture pages and
`Mimir.VerseRecorder`.
3. Promote sensor, media, observation, ledger, and control state into explicit
`mimir.*` CultMesh namespaces while preserving current document ids.
4. Publish `gamecult.eve.surface.v1` surfaces for sensor field and operator
flows, with typed field bindings back to the owning state documents.
5. Make the old dashboard broker and `/eve/dashboard` compatibility endpoint
consume those provider surfaces instead of assembling private truth.
6. Let Odin discover the service manifest and surfaces, then treat direct HTTP
health/status probes as temporary diagnostics.
7. Delete or neuter any private JSON store, dashboard state cache, or UI callback
path that can still decide service behavior after the typed owner exists.
## Demotion Line
Bespoke UI, browser dashboards, retained dashboard node lists, compatibility
WebSockets, SRT/FFmpeg media bridges, JSON status files, JSONL replay logs,
screenshot streams, and OBS bridge scripts are demoted to lowerings,
diagnostics, external-boundary export, or replay witnesses. They may display,
transport, inspect, or reconstruct typed Mimir state. They may not own sensor
truth, calibration truth, command state, reservoir scheduling, program
presentation, media-over-Verse policy, or synchronization behavior.

View file

@ -4,13 +4,12 @@
Build the smallest live Mimir application that can ingest multiple local or
networked video and audio feeds, hold a short synchronized window in memory,
and emit coherent Mimir program outputs while Eve provides operator surfaces
and Yggdrasil publishes the site feed.
and emit coherent OBS-facing program outputs while Fensalir provides the
operator UI.
The default reservoir horizon is five seconds. That delay is not decoration. It
is the budget that lets late network packets, independent capture clocks,
audio/video alignment, and debug controls converge before Mimir emits the
program result.
audio/video alignment, and debug controls converge before OBS sees the result.
Mimir is explicitly trading latency and memory for throughput and extraction
quality: the app may spend that buffered window lining streams up and crunching
audio/video data to recover as much of the volumetric audio field and
@ -22,16 +21,16 @@ alignment and field-reconstruction budget.
Today the repo has two partial machines:
- FFmpeg/SRT bridge scripts can capture sender video/audio and expose stable
compatibility endpoints.
- FFmpeg/SRT bridge scripts can capture sender video/audio and expose stable OBS
Media Source endpoints.
- The native `LocalcastRuntime` reservoir ABI can hold typed live sample handles,
and Fensalir can consume native render packets for GPU fusion.
Those parts are useful, but they are not yet the viable app. The viable app is
the join: ingest workers append live sample handles into one Fensalir-hosted
runtime window, Fensalir renders video from that window, Faust/native audio
builds synchronized stems from the same window, Mimir commits the scene graph,
Eve lowers the operator surface, and Yggdrasil publishes the program output.
builds synchronized stems from the same window, and OBS receives only the
program outputs.
```mermaid
flowchart TD
@ -40,15 +39,10 @@ flowchart TD
C["local mic/loopback workers"] --> R
D["network audio ingest workers"] --> R
R --> U["Fensalir debug/settings/output UI"]
R --> S["Mimir program scene graph"]
S --> U
S --> V["Fensalir GPU video composition"]
R --> V["Fensalir GPU video composition"]
R --> F["Faust/native audio alignment + stems"]
S --> E["Eve GUI/TUI operator surfaces"]
V --> Y["Yggdrasil site publisher"]
F --> Y
V -. "compatibility" .-> O["OBS adapter"]
F -. "compatibility" .-> O
V --> O["OBS video output"]
F --> P["OBS audio stem outputs"]
```
## Invariants
@ -57,17 +51,12 @@ flowchart TD
- The five-second rolling window is a deliberate compute budget, not accidental
latency. It exists so capture, sync, GPU fusion, and audio field extraction
can choose coherence over immediacy.
- Mimir owns the program scene graph, source subscription policy, preview/control
state, stats, and publication intent.
- Fensalir owns the running app host, UI lowering, debug state, GPU
fusion/rendering, and local program texture output.
- Fensalir owns the running app, UI, debug state, settings changes, output
management, GPU fusion/rendering, and Spout/video publication.
- Faust/native DSP owns audio alignment, voice separation, suppression,
volumetric sound-field extraction, stems, and spatial bed generation.
- Eve lowerers render controls and previews without owning scene truth.
- The Yggdrasil publisher consumes synchronized program surfaces without owning
a second composition.
- OBS compatibility adapters may receive synchronized program surfaces. Raw
unsynchronized feeds may be diagnostic inputs, not broadcast truth.
- OBS receives synchronized program surfaces. Raw unsynchronized feeds may be
diagnostic inputs, not broadcast truth.
- Networked feeds are producers into the same reservoir. They do not own clocks,
private buffers, or a parallel synchronization model.
- Missing feeds become explicit absence or silent/blank placeholders inside the
@ -89,14 +78,9 @@ The first cut should be deliberately boring:
loopback ingest.
7. Expose Fensalir UI readouts for reservoir edge, window start, per-kind
counts, feed health, delay estimate, output status, and current settings.
8. Render one synchronized Mimir program video output from the reservoir.
9. Emit separately controllable program audio stems from the same reservoir
window.
10. Publish Eve GUI/TUI operator surfaces for preview, source selection, scene
edits, stats, and output state.
11. Add the Yggdrasil-facing publisher daemon that consumes Mimir program
output without owning composition.
12. Keep the existing FFmpeg/SRT scripts as edge utilities for network capture
8. Render one synchronized program video output from the reservoir.
9. Emit separately controllable OBS audio stems from the same reservoir window.
10. Keep the existing FFmpeg/SRT scripts as edge utilities for network capture
and OBS compatibility, not as the stream authority.
## Repo Shape
@ -123,11 +107,14 @@ The first cut should be deliberately boring:
- `MimirNativeIngestStreamSource` is the low-overhead push seam for native
camera/audio adapters that already own buffers or handles.
- `MimirVideoFrameDescriptor` carries video shape, pixel format, device
timestamp, stride, and optional native/GPU handle metadata. Leap stereo IR
should enter here through a native driver path.
timestamp, stride, resource key, producer fence value, and native/GPU handle
metadata. Leap stereo IR should enter here through a native driver path that
reports any unavoidable system-memory copy.
- `IMimirVideoCaptureDriver` and `MimirVideoCaptureDriverSource` are the
driver-facing seam for polling/pushing frames from LeapUVC/libusb, LeapC image
APIs, Media Foundation, DirectShow, or a future GPU shared-texture capture.
APIs, Media Foundation, DirectShow, or GPU decode/capture paths. Drivers that
can produce render inputs should implement `IMimirFensalirTextureLeaseReceiver`
and populate Fensalir-owned texture leases directly.
- `MimirProcessStreamSource` is a compatibility adapter for network or
diagnostic command sources. It is not the six-camera local capture foundation.
@ -145,10 +132,10 @@ environment variables:
These create buffers immediately. Capture adapters still need to register
`IMimirStreamSource` implementations to feed live samples into those buffers.
For local cameras, prefer native adapters that push sample handles directly
through `MimirNativeIngestStreamSource`. Process-backed FFmpeg ingest is allowed
for remote SRT/diagnostic edges, but local six-camera ingest should stay close
to the driver, GPU, and capture APIs.
For local cameras, prefer native adapters that talk to the device stack directly
and push only typed sample/resource handles through `MimirNativeIngestStreamSource`.
Process-backed FFmpeg ingest is allowed for remote SRT/diagnostic edges, but
local six-camera ingest should stay close to the device, GPU, and capture APIs.
Leap should be treated as the timing-camera candidate only when the native
adapter preserves device timestamps or captures a monotonic timestamp

View file

@ -0,0 +1,65 @@
declare name "Mimir Dual Mic Dialogue Cleaner";
declare version "0.1";
declare author "Mimir";
declare license "MIT";
declare description "Faust-owned smooth dual-mic dialogue cleanup target: aligned shotgun/cardioid inputs become a conservative dialogue stem and a rejected residual witness.";
import("stdfaust.lib");
// Inputs are already delay-aligned upstream by mimir_alignment_actuator.
// This graph owns smooth audio-rate cleanup, not calibration, timing, or UI.
//
// input 0: aligned shotgun / co-streamer witness
// input 1: aligned cardioid / host witness
// output 0: dialogue composite stem
// output 1: rejected residual / noise witness
drive = hslider("dialogue/drive", 1.00, 0.00, 2.00, 0.001);
shotgunWeight = hslider("dialogue/shotgun_weight", 0.45, 0.00, 1.00, 0.001);
cardioidWeight = hslider("dialogue/cardioid_weight", 0.55, 0.00, 1.00, 0.001);
witnessReject = hslider("dialogue/witness_reject", 0.20, 0.00, 1.00, 0.001);
lowShelf = hslider("dialogue/low_shelf", 0.80, 0.00, 2.00, 0.001);
presenceShelf = hslider("dialogue/presence_shelf", 1.08, 0.00, 2.00, 0.001);
airShelf = hslider("dialogue/air_shelf", 0.90, 0.00, 2.00, 0.001);
// Smooth broadband downward expansion. This is intentionally slow enough to
// avoid the choppy gate artifact from the C# audition bridge.
floorLevel = hslider("denoise/noise_floor", 0.018, 0.0001, 0.25, 0.0001);
expansionDepth = hslider("denoise/expansion_depth", 0.32, 0.00, 0.95, 0.001);
expansionCurve = hslider("denoise/expansion_curve", 1.35, 0.25, 4.00, 0.001);
envelopeHz = hslider("denoise/envelope_hz", 18.0, 1.0, 80.0, 0.1);
band(f, q, g, x) = x : fi.resonbp(1.0, f, q) : *(g);
voiceShape(x) =
band(120.0, 1.2, lowShelf, x) +
band(220.0, 1.6, 0.95, x) +
band(420.0, 2.0, 0.98, x) +
band(800.0, 2.4, 1.00, x) +
band(1500.0, 2.8, presenceShelf, x) +
band(2800.0, 3.2, presenceShelf, x) +
band(5200.0, 3.2, airShelf, x) +
band(8500.0, 2.6, airShelf, x);
smoothAbs(x) = abs(x) : si.smooth(ba.tau2pole(1.0 / max(1.0, envelopeHz)));
softExpand(x) = x * gain
with {
env = smoothAbs(x);
ratio = min(1.0, env / max(0.0001, floorLevel));
openness = pow(ratio, expansionCurve);
gain = 1.0 - expansionDepth * (1.0 - openness);
};
softClip(x) = x / (1.0 + abs(x));
clean(shotgun, cardioid) = dialogue, residual
with {
weighted = shotgun * shotgunWeight + cardioid * cardioidWeight;
witness = (shotgun - cardioid) * 0.5;
shaped = voiceShape(weighted - witnessReject * witness);
dialogue = softExpand(shaped * drive) : softClip;
residual = witness + (weighted - dialogue) * 0.25;
};
process = clean;

View file

@ -0,0 +1,70 @@
cmake_minimum_required(VERSION 3.20)
project(mimir_camera_capture LANGUAGES C CXX)
add_library(mimir_camera_capture SHARED
mimir_ks_camera_capture.cpp)
target_compile_features(mimir_camera_capture PRIVATE cxx_std_20)
target_compile_definitions(mimir_camera_capture PRIVATE
WIN32_LEAN_AND_MEAN
NOMINMAX)
target_link_libraries(mimir_camera_capture PRIVATE
setupapi
ksuser)
set(PSMOVEAPI_ROOT "$ENV{TEMP}/mimir-driver-probes/psmoveapi" CACHE PATH "Root of a psmoveapi checkout with external/libusb-1.0 and external/PS3EYEDriver.")
set(LIBUSB_ROOT "${PSMOVEAPI_ROOT}/external/libusb-1.0" CACHE PATH "libusb source root.")
set(PS3EYE_ROOT "${PSMOVEAPI_ROOT}/external/PS3EYEDriver" CACHE PATH "PS3EYEDriver source root.")
add_library(mimir_libusb_static STATIC
${LIBUSB_ROOT}/libusb/core.c
${LIBUSB_ROOT}/libusb/descriptor.c
${LIBUSB_ROOT}/libusb/hotplug.c
${LIBUSB_ROOT}/libusb/io.c
${LIBUSB_ROOT}/libusb/strerror.c
${LIBUSB_ROOT}/libusb/sync.c
${LIBUSB_ROOT}/libusb/os/events_windows.c
${LIBUSB_ROOT}/libusb/os/threads_windows.c
${LIBUSB_ROOT}/libusb/os/windows_common.c
${LIBUSB_ROOT}/libusb/os/windows_usbdk.c
${LIBUSB_ROOT}/libusb/os/windows_winusb.c)
target_include_directories(mimir_libusb_static PUBLIC
${LIBUSB_ROOT}/libusb
${LIBUSB_ROOT}/libusb/os
${LIBUSB_ROOT}/msvc)
target_compile_definitions(mimir_libusb_static PRIVATE
_CRT_SECURE_NO_WARNINGS
_WIN32_WINNT=0x0601)
target_link_libraries(mimir_libusb_static PRIVATE setupapi ole32 advapi32)
add_library(mimir_ps3eye_vendor STATIC
${PS3EYE_ROOT}/src/ps3eye.cpp
${PS3EYE_ROOT}/src/ps3eye_capi.cpp)
target_include_directories(mimir_ps3eye_vendor PUBLIC
${PS3EYE_ROOT}/src
${LIBUSB_ROOT}/libusb)
target_link_libraries(mimir_ps3eye_vendor PUBLIC mimir_libusb_static)
add_library(mimir_ps3eye_capture SHARED
mimir_ps3eye_capture.cpp)
target_compile_features(mimir_ps3eye_capture PRIVATE cxx_std_20)
target_compile_definitions(mimir_ps3eye_capture PRIVATE
WIN32_LEAN_AND_MEAN
NOMINMAX)
target_include_directories(mimir_ps3eye_capture PRIVATE ${PS3EYE_ROOT}/src)
target_link_libraries(mimir_ps3eye_capture PRIVATE mimir_ps3eye_vendor)
add_library(mimir_mf_gpu_capture SHARED
mimir_mf_gpu_capture.cpp)
target_compile_features(mimir_mf_gpu_capture PRIVATE cxx_std_20)
target_compile_definitions(mimir_mf_gpu_capture PRIVATE
WIN32_LEAN_AND_MEAN
NOMINMAX)
target_link_libraries(mimir_mf_gpu_capture PRIVATE
d3d11
dxgi
mf
mfplat
mfreadwrite
mfuuid)

View file

@ -0,0 +1,759 @@
#include <windows.h>
#include <winioctl.h>
#include <setupapi.h>
#include <ks.h>
#include <ksmedia.h>
#include <winternl.h>
#include <algorithm>
#include <atomic>
#include <cctype>
#include <chrono>
#include <condition_variable>
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <deque>
#include <mutex>
#include <string>
#include <thread>
#include <vector>
namespace
{
struct Handle
{
HANDLE value = INVALID_HANDLE_VALUE;
Handle() = default;
explicit Handle(HANDLE handle) : value(handle) {}
Handle(const Handle&) = delete;
Handle& operator=(const Handle&) = delete;
Handle(Handle&& other) noexcept : value(other.value) { other.value = INVALID_HANDLE_VALUE; }
Handle& operator=(Handle&& other) noexcept
{
if (this != &other)
{
close();
value = other.value;
other.value = INVALID_HANDLE_VALUE;
}
return *this;
}
~Handle() { close(); }
void close()
{
if (value != INVALID_HANDLE_VALUE && value != nullptr)
{
CloseHandle(value);
value = INVALID_HANDLE_VALUE;
}
}
explicit operator bool() const { return value != INVALID_HANDLE_VALUE && value != nullptr; }
};
struct DeviceInfoSet
{
HDEVINFO value = INVALID_HANDLE_VALUE;
explicit DeviceInfoSet(HDEVINFO info) : value(info) {}
DeviceInfoSet(const DeviceInfoSet&) = delete;
DeviceInfoSet& operator=(const DeviceInfoSet&) = delete;
~DeviceInfoSet()
{
if (value != INVALID_HANDLE_VALUE)
{
SetupDiDestroyDeviceInfoList(value);
}
}
};
using NtDeviceIoControlFileFn = NTSTATUS(NTAPI*)(
HANDLE FileHandle,
HANDLE Event,
PIO_APC_ROUTINE ApcRoutine,
PVOID ApcContext,
PIO_STATUS_BLOCK IoStatusBlock,
ULONG IoControlCode,
PVOID InputBuffer,
ULONG InputBufferLength,
PVOID OutputBuffer,
ULONG OutputBufferLength);
constexpr NTSTATUS StatusPending = static_cast<NTSTATUS>(0x00000103L);
struct PinCandidate
{
ULONG pinId = 0;
std::vector<std::uint8_t> format;
LONG width = 0;
LONG height = 0;
LONG imageBytes = 0;
LONGLONG interval100ns = 0;
GUID subtype{};
std::string fourcc;
};
struct MimirKsCameraFrame
{
std::vector<std::uint8_t> data;
std::int64_t timestampNs = 0;
std::uint64_t sequence = 0;
int byteLength = 0;
};
struct MimirKsCameraCapture
{
Handle filter;
Handle pin;
PinCandidate candidate;
std::thread worker;
std::mutex mutex;
std::condition_variable condition;
std::deque<MimirKsCameraFrame> queue;
std::atomic<bool> running = false;
int queueDepth = 8;
std::uint64_t sequence = 0;
};
long long nowNs()
{
return std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::steady_clock::now().time_since_epoch()).count();
}
std::string lowercase(std::string value)
{
std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) {
return static_cast<char>(std::tolower(c));
});
return value;
}
std::string wideToUtf8(const wchar_t* value)
{
if (!value)
{
return {};
}
const int bytes = WideCharToMultiByte(CP_UTF8, 0, value, -1, nullptr, 0, nullptr, nullptr);
if (bytes <= 1)
{
return {};
}
std::string text(static_cast<std::size_t>(bytes - 1), '\0');
WideCharToMultiByte(CP_UTF8, 0, value, -1, text.data(), bytes, nullptr, nullptr);
return text;
}
std::string fourccOrGuid(const GUID& guid)
{
if (guid.Data2 == 0x0000 &&
guid.Data3 == 0x0010 &&
guid.Data4[0] == 0x80 &&
guid.Data4[1] == 0x00 &&
guid.Data4[2] == 0x00 &&
guid.Data4[3] == 0xaa &&
guid.Data4[4] == 0x00 &&
guid.Data4[5] == 0x38 &&
guid.Data4[6] == 0x9b &&
guid.Data4[7] == 0x71)
{
char text[5]{};
text[0] = static_cast<char>(guid.Data1 & 0xff);
text[1] = static_cast<char>((guid.Data1 >> 8) & 0xff);
text[2] = static_cast<char>((guid.Data1 >> 16) & 0xff);
text[3] = static_cast<char>((guid.Data1 >> 24) & 0xff);
return text;
}
return {};
}
bool ksProperty(HANDLE handle, void* property, DWORD propertyBytes, void* output, DWORD outputBytes, DWORD* returned = nullptr)
{
DWORD bytes = 0;
const BOOL ok = DeviceIoControl(
handle,
IOCTL_KS_PROPERTY,
property,
propertyBytes,
output,
outputBytes,
&bytes,
nullptr);
if (returned)
{
*returned = bytes;
}
return ok != FALSE;
}
template <typename T>
bool getPinProperty(HANDLE filter, ULONG pinId, ULONG propertyId, T& value)
{
KSP_PIN property{};
property.Property.Set = KSPROPSETID_Pin;
property.Property.Id = propertyId;
property.Property.Flags = KSPROPERTY_TYPE_GET;
property.PinId = pinId;
DWORD returned = 0;
return ksProperty(filter, &property, sizeof(property), &value, sizeof(value), &returned);
}
bool setVideoProcAmp(HANDLE filter, ULONG id, LONG value, ULONG flags)
{
KSPROPERTY_VIDEOPROCAMP_S property{};
property.Property.Set = PROPSETID_VIDCAP_VIDEOPROCAMP;
property.Property.Id = id;
property.Property.Flags = KSPROPERTY_TYPE_SET;
property.Value = value;
property.Flags = flags;
return ksProperty(filter, &property, sizeof(property), &property, sizeof(property));
}
bool setCameraControl(HANDLE filter, ULONG id, LONG value, ULONG flags)
{
KSPROPERTY_CAMERACONTROL_S property{};
property.Property.Set = PROPSETID_VIDCAP_CAMERACONTROL;
property.Property.Id = id;
property.Property.Flags = KSPROPERTY_TYPE_SET;
property.Value = value;
property.Flags = flags;
return ksProperty(filter, &property, sizeof(property), &property, sizeof(property));
}
std::vector<std::string> enumerateCapturePaths()
{
std::vector<std::string> paths;
DeviceInfoSet info(SetupDiGetClassDevsA(
&KSCATEGORY_CAPTURE,
nullptr,
nullptr,
DIGCF_PRESENT | DIGCF_DEVICEINTERFACE));
if (info.value == INVALID_HANDLE_VALUE)
{
return paths;
}
for (DWORD index = 0;; ++index)
{
SP_DEVICE_INTERFACE_DATA interfaceData{};
interfaceData.cbSize = sizeof(interfaceData);
if (!SetupDiEnumDeviceInterfaces(info.value, nullptr, &KSCATEGORY_CAPTURE, index, &interfaceData))
{
break;
}
DWORD required = 0;
SetupDiGetDeviceInterfaceDetailW(info.value, &interfaceData, nullptr, 0, &required, nullptr);
if (required == 0)
{
continue;
}
std::vector<std::uint8_t> storage(required);
auto* detail = reinterpret_cast<SP_DEVICE_INTERFACE_DETAIL_DATA_W*>(storage.data());
detail->cbSize = sizeof(*detail);
if (SetupDiGetDeviceInterfaceDetailW(info.value, &interfaceData, detail, required, nullptr, nullptr))
{
paths.push_back(wideToUtf8(detail->DevicePath));
}
}
return paths;
}
std::vector<PinCandidate> inspectPins(HANDLE filter)
{
ULONG pinCount = 0;
KSPROPERTY countProperty{};
countProperty.Set = KSPROPSETID_Pin;
countProperty.Id = KSPROPERTY_PIN_CTYPES;
countProperty.Flags = KSPROPERTY_TYPE_GET;
if (!ksProperty(filter, &countProperty, sizeof(countProperty), &pinCount, sizeof(pinCount)))
{
return {};
}
std::vector<PinCandidate> candidates;
for (ULONG pinId = 0; pinId < pinCount; ++pinId)
{
KSPIN_DATAFLOW dataflow{};
if (!getPinProperty(filter, pinId, KSPROPERTY_PIN_DATAFLOW, dataflow) ||
dataflow != KSPIN_DATAFLOW_OUT)
{
continue;
}
KSP_PIN rangeProperty{};
rangeProperty.Property.Set = KSPROPSETID_Pin;
rangeProperty.Property.Id = KSPROPERTY_PIN_DATARANGES;
rangeProperty.Property.Flags = KSPROPERTY_TYPE_GET;
rangeProperty.PinId = pinId;
DWORD rangeBytes = 0;
ksProperty(filter, &rangeProperty, sizeof(rangeProperty), nullptr, 0, &rangeBytes);
if (rangeBytes == 0)
{
continue;
}
std::vector<std::uint8_t> ranges(rangeBytes);
DWORD returned = 0;
if (!ksProperty(filter, &rangeProperty, sizeof(rangeProperty), ranges.data(), static_cast<DWORD>(ranges.size()), &returned))
{
continue;
}
auto* multiple = reinterpret_cast<KSMULTIPLE_ITEM*>(ranges.data());
auto* cursor = ranges.data() + sizeof(KSMULTIPLE_ITEM);
for (ULONG i = 0; i < multiple->Count; ++i)
{
auto* dataRange = reinterpret_cast<KSDATARANGE*>(cursor);
const bool videoInfo =
IsEqualGUID(dataRange->MajorFormat, KSDATAFORMAT_TYPE_VIDEO) &&
IsEqualGUID(dataRange->Specifier, KSDATAFORMAT_SPECIFIER_VIDEOINFO) &&
dataRange->FormatSize >= sizeof(KS_DATARANGE_VIDEO);
if (videoInfo)
{
auto* video = reinterpret_cast<KS_DATARANGE_VIDEO*>(cursor);
const auto& bmi = video->VideoInfoHeader.bmiHeader;
PinCandidate candidate{};
candidate.pinId = pinId;
candidate.width = bmi.biWidth;
candidate.height = bmi.biHeight;
candidate.imageBytes = bmi.biSizeImage > 0 ? bmi.biSizeImage : static_cast<LONG>(dataRange->SampleSize);
candidate.interval100ns = video->VideoInfoHeader.AvgTimePerFrame;
candidate.subtype = dataRange->SubFormat;
candidate.fourcc = fourccOrGuid(candidate.subtype);
candidate.format.resize(sizeof(KS_DATAFORMAT_VIDEOINFOHEADER));
auto* format = reinterpret_cast<KS_DATAFORMAT_VIDEOINFOHEADER*>(candidate.format.data());
format->DataFormat.FormatSize = sizeof(KS_DATAFORMAT_VIDEOINFOHEADER);
format->DataFormat.SampleSize = candidate.imageBytes;
format->DataFormat.MajorFormat = KSDATAFORMAT_TYPE_VIDEO;
format->DataFormat.SubFormat = dataRange->SubFormat;
format->DataFormat.Specifier = KSDATAFORMAT_SPECIFIER_VIDEOINFO;
format->VideoInfoHeader = video->VideoInfoHeader;
format->VideoInfoHeader.bmiHeader.biSizeImage = candidate.imageBytes;
candidates.push_back(std::move(candidate));
}
cursor += dataRange->FormatSize;
}
}
return candidates;
}
const PinCandidate* findCandidate(
const std::vector<PinCandidate>& candidates,
int width,
int height,
const char* subtype,
double minFps)
{
const std::string target = subtype ? subtype : "";
for (const auto& candidate : candidates)
{
const double fps = candidate.interval100ns > 0
? 10000000.0 / static_cast<double>(candidate.interval100ns)
: 0.0;
if (candidate.width == width &&
candidate.height == height &&
fps >= minFps &&
candidate.fourcc == target)
{
return &candidate;
}
}
return nullptr;
}
bool setState(HANDLE pin, KSSTATE state)
{
KSPROPERTY property{};
property.Set = KSPROPSETID_Connection;
property.Id = KSPROPERTY_CONNECTION_STATE;
property.Flags = KSPROPERTY_TYPE_SET;
DWORD returned = 0;
return DeviceIoControl(
pin,
IOCTL_KS_PROPERTY,
&property,
sizeof(property),
&state,
sizeof(state),
&returned,
nullptr) != FALSE;
}
int strideBytes(const PinCandidate& candidate)
{
return candidate.height > 0 && candidate.imageBytes > 0
? candidate.imageBytes / candidate.height
: 0;
}
void pushFrame(MimirKsCameraCapture& capture, const std::uint8_t* data, ULONG byteLength)
{
MimirKsCameraFrame frame;
frame.byteLength = static_cast<int>(byteLength);
frame.timestampNs = nowNs();
frame.sequence = capture.sequence++;
frame.data.resize(byteLength);
std::memcpy(frame.data.data(), data, byteLength);
{
std::lock_guard<std::mutex> lock(capture.mutex);
capture.queue.push_back(std::move(frame));
while (capture.queue.size() > 32)
{
capture.queue.pop_front();
}
}
capture.condition.notify_one();
}
void captureThread(MimirKsCameraCapture* capture)
{
auto* ntDeviceIoControlFile = reinterpret_cast<NtDeviceIoControlFileFn>(
GetProcAddress(GetModuleHandleA("ntdll.dll"), "NtDeviceIoControlFile"));
if (!ntDeviceIoControlFile)
{
capture->running.store(false);
return;
}
const ULONG connectBytes = sizeof(KSPIN_CONNECT) + static_cast<ULONG>(capture->candidate.format.size());
std::vector<std::uint8_t> connectStorage(connectBytes);
auto* connect = reinterpret_cast<KSPIN_CONNECT*>(connectStorage.data());
connect->Interface.Set = KSINTERFACESETID_Standard;
connect->Interface.Id = KSINTERFACE_STANDARD_STREAMING;
connect->Medium.Set = KSMEDIUMSETID_Standard;
connect->Medium.Id = KSMEDIUM_TYPE_ANYINSTANCE;
connect->PinId = capture->candidate.pinId;
connect->Priority.PriorityClass = KSPRIORITY_NORMAL;
connect->Priority.PrioritySubClass = 1;
std::memcpy(connectStorage.data() + sizeof(KSPIN_CONNECT), capture->candidate.format.data(), capture->candidate.format.size());
HANDLE rawPin = INVALID_HANDLE_VALUE;
const HRESULT hr = KsCreatePin(capture->filter.value, connect, GENERIC_READ, &rawPin);
if (FAILED(hr))
{
capture->running.store(false);
return;
}
capture->pin = Handle(rawPin);
if (!setState(capture->pin.value, KSSTATE_ACQUIRE) ||
!setState(capture->pin.value, KSSTATE_PAUSE) ||
!setState(capture->pin.value, KSSTATE_RUN))
{
capture->running.store(false);
return;
}
const DWORD frameBytes = static_cast<DWORD>(capture->candidate.imageBytes);
struct ReadSlot
{
Handle event;
IO_STATUS_BLOCK status{};
KSSTREAM_HEADER header{};
std::vector<std::uint8_t> frame;
bool pending = false;
};
std::vector<ReadSlot> slots(static_cast<std::size_t>(std::max(1, capture->queueDepth)));
std::vector<HANDLE> waitEvents(slots.size());
for (auto& slot : slots)
{
slot.event = Handle(CreateEventW(nullptr, TRUE, FALSE, nullptr));
if (!slot.event)
{
capture->running.store(false);
return;
}
slot.frame.resize(frameBytes);
slot.header.Size = sizeof(KSSTREAM_HEADER);
slot.header.FrameExtent = frameBytes;
slot.header.Data = slot.frame.data();
slot.header.PresentationTime.Numerator = 1;
slot.header.PresentationTime.Denominator = 1;
waitEvents[&slot - slots.data()] = slot.event.value;
}
auto submit = [&](ReadSlot& slot) -> bool {
ResetEvent(slot.event.value);
slot.status = {};
slot.header.DataUsed = 0;
slot.header.OptionsFlags = 0;
const NTSTATUS status = ntDeviceIoControlFile(
capture->pin.value,
slot.event.value,
nullptr,
nullptr,
&slot.status,
IOCTL_KS_READ_STREAM,
&slot.header,
sizeof(slot.header),
&slot.header,
sizeof(slot.header));
if (status == StatusPending)
{
slot.pending = true;
return true;
}
slot.pending = false;
if (status >= 0 && slot.header.DataUsed > 0)
{
pushFrame(*capture, slot.frame.data(), slot.header.DataUsed);
return true;
}
return false;
};
for (auto& slot : slots)
{
if (!submit(slot))
{
capture->running.store(false);
return;
}
}
while (capture->running.load())
{
const DWORD waitResult = WaitForMultipleObjects(
static_cast<DWORD>(waitEvents.size()),
waitEvents.data(),
FALSE,
100);
if (waitResult == WAIT_TIMEOUT)
{
continue;
}
if (waitResult < WAIT_OBJECT_0 || waitResult >= WAIT_OBJECT_0 + waitEvents.size())
{
break;
}
auto& slot = slots[static_cast<std::size_t>(waitResult - WAIT_OBJECT_0)];
slot.pending = false;
if (slot.status.Status < 0 || slot.header.DataUsed == 0)
{
break;
}
pushFrame(*capture, slot.frame.data(), slot.header.DataUsed);
if (!submit(slot))
{
break;
}
}
setState(capture->pin.value, KSSTATE_STOP);
CancelIoEx(capture->pin.value, nullptr);
for (auto& slot : slots)
{
if (slot.pending)
{
WaitForSingleObject(slot.event.value, 100);
}
}
capture->running.store(false);
}
} // namespace
extern "C"
{
__declspec(dllexport) MimirKsCameraCapture* mimir_ks_create(
const char* pathNeedle,
int width,
int height,
const char* subtype,
double minFps,
int queueDepth)
{
const std::string needle = lowercase(pathNeedle ? pathNeedle : "");
for (const auto& path : enumerateCapturePaths())
{
if (!needle.empty() && lowercase(path).find(needle) == std::string::npos)
{
continue;
}
Handle filter(CreateFileA(
path.c_str(),
GENERIC_READ | GENERIC_WRITE,
FILE_SHARE_READ | FILE_SHARE_WRITE,
nullptr,
OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL,
nullptr));
if (!filter)
{
continue;
}
auto candidates = inspectPins(filter.value);
const auto* candidate = findCandidate(candidates, width, height, subtype, minFps);
if (!candidate)
{
continue;
}
auto* capture = new MimirKsCameraCapture();
capture->filter = std::move(filter);
capture->candidate = *candidate;
capture->queueDepth = std::max(1, std::min(queueDepth, 32));
return capture;
}
return nullptr;
}
__declspec(dllexport) int mimir_ks_start(MimirKsCameraCapture* capture)
{
if (!capture || capture->running.exchange(true))
{
return 0;
}
capture->worker = std::thread(captureThread, capture);
return 1;
}
__declspec(dllexport) int mimir_ks_set_exposure_gain(
MimirKsCameraCapture* capture,
int setExposure,
LONG exposure,
int setGain,
LONG gain)
{
if (!capture || !capture->filter)
{
return 0;
}
int result = 0;
if (!setExposure ||
setCameraControl(
capture->filter.value,
KSPROPERTY_CAMERACONTROL_EXPOSURE,
exposure,
KSPROPERTY_CAMERACONTROL_FLAGS_MANUAL))
{
result |= 1;
}
if (!setGain ||
setVideoProcAmp(
capture->filter.value,
KSPROPERTY_VIDEOPROCAMP_GAIN,
gain,
KSPROPERTY_VIDEOPROCAMP_FLAGS_MANUAL))
{
result |= 2;
}
return result;
}
__declspec(dllexport) int mimir_ks_read(
MimirKsCameraCapture* capture,
int* width,
int* height,
int* stride,
char* fourcc,
int fourccCapacity,
std::int64_t* timestampNs,
std::uint64_t* sequence,
std::uint8_t* destination,
int destinationBytes)
{
if (!capture || !destination || destinationBytes <= 0)
{
return 0;
}
MimirKsCameraFrame frame;
{
std::lock_guard<std::mutex> lock(capture->mutex);
if (capture->queue.empty())
{
return 0;
}
frame = std::move(capture->queue.front());
capture->queue.pop_front();
}
if (frame.byteLength > destinationBytes)
{
return -frame.byteLength;
}
std::memcpy(destination, frame.data.data(), static_cast<std::size_t>(frame.byteLength));
if (width)
{
*width = static_cast<int>(capture->candidate.width);
}
if (height)
{
*height = static_cast<int>(capture->candidate.height);
}
if (stride)
{
*stride = strideBytes(capture->candidate);
}
if (fourcc && fourccCapacity > 0)
{
std::snprintf(fourcc, static_cast<std::size_t>(fourccCapacity), "%s", capture->candidate.fourcc.c_str());
}
if (timestampNs)
{
*timestampNs = frame.timestampNs;
}
if (sequence)
{
*sequence = frame.sequence;
}
return frame.byteLength;
}
__declspec(dllexport) void mimir_ks_destroy(MimirKsCameraCapture* capture)
{
if (!capture)
{
return;
}
capture->running.store(false);
if (capture->pin)
{
CancelIoEx(capture->pin.value, nullptr);
}
if (capture->worker.joinable())
{
capture->worker.join();
}
delete capture;
}
}

View file

@ -0,0 +1,573 @@
#include <windows.h>
#include <d3d11_4.h>
#include <dxgi1_6.h>
#include <mfapi.h>
#include <mferror.h>
#include <mfidl.h>
#include <mfreadwrite.h>
#include <wrl/client.h>
#include <algorithm>
#include <cctype>
#include <cstdint>
#include <cstdio>
#include <cstdarg>
#include <string>
#include <vector>
using Microsoft::WRL::ComPtr;
namespace
{
struct MimirMfGpuCapture
{
ComPtr<IMFMediaSource> source;
ComPtr<IMFSourceReader> reader;
ComPtr<ID3D11Device> device;
ComPtr<ID3D11DeviceContext> context;
ComPtr<IMFDXGIDeviceManager> deviceManager;
ComPtr<ID3D11Texture2D> sharedTexture;
ComPtr<ID3D11Fence> producerFence;
ComPtr<ID3D11DeviceContext4> fenceContext;
HANDLE sharedHandle = nullptr;
HANDLE producerFenceHandle = nullptr;
int width = 0;
int height = 0;
DXGI_FORMAT format = DXGI_FORMAT_UNKNOWN;
std::uint64_t sequence = 0;
};
bool debugEnabled()
{
return GetEnvironmentVariableA("MIMIR_MF_GPU_DEBUG", nullptr, 0) > 0;
}
void debugLog(const char* format, ...)
{
if (!debugEnabled())
{
return;
}
std::fprintf(stderr, "mimir-mf-gpu: ");
va_list args;
va_start(args, format);
std::vfprintf(stderr, format, args);
va_end(args);
std::fprintf(stderr, "\n");
}
std::wstring widen(const char* value)
{
if (!value)
{
return {};
}
const int chars = MultiByteToWideChar(CP_UTF8, 0, value, -1, nullptr, 0);
if (chars <= 1)
{
return {};
}
std::wstring result(static_cast<std::size_t>(chars - 1), L'\0');
MultiByteToWideChar(CP_UTF8, 0, value, -1, result.data(), chars);
return result;
}
std::wstring lowercase(std::wstring value)
{
std::transform(value.begin(), value.end(), value.begin(), [](wchar_t c) {
return static_cast<wchar_t>(towlower(c));
});
return value;
}
GUID subtypeFromText(const char* text)
{
const std::string value = text ? text : "";
std::string upper;
upper.resize(value.size());
std::transform(value.begin(), value.end(), upper.begin(), [](unsigned char c) {
return static_cast<char>(std::toupper(c));
});
if (upper == "MJPG") return MFVideoFormat_MJPG;
if (upper == "H264") return MFVideoFormat_H264;
if (upper == "NV12") return MFVideoFormat_NV12;
if (upper == "YUY2") return MFVideoFormat_YUY2;
if (upper == "RGB32" || upper == "BGRA") return MFVideoFormat_RGB32;
return GUID_NULL;
}
DXGI_FORMAT dxgiFormatForSubtype(const GUID& subtype)
{
if (subtype == MFVideoFormat_NV12) return DXGI_FORMAT_NV12;
if (subtype == MFVideoFormat_RGB32) return DXGI_FORMAT_B8G8R8A8_UNORM;
if (subtype == MFVideoFormat_YUY2) return DXGI_FORMAT_YUY2;
return DXGI_FORMAT_UNKNOWN;
}
const char* textForSubtype(const GUID& subtype)
{
if (subtype == MFVideoFormat_NV12) return "NV12";
if (subtype == MFVideoFormat_RGB32) return "Bgra8";
if (subtype == MFVideoFormat_YUY2) return "YUY2";
return "Unknown";
}
bool findVideoSource(const wchar_t* needle, IMFMediaSource** mediaSource)
{
*mediaSource = nullptr;
ComPtr<IMFAttributes> attributes;
if (FAILED(MFCreateAttributes(&attributes, 2)))
{
debugLog("MFCreateAttributes for devices failed");
return false;
}
attributes->SetGUID(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE, MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID);
IMFActivate** rawDevices = nullptr;
UINT32 count = 0;
if (FAILED(MFEnumDeviceSources(attributes.Get(), &rawDevices, &count)))
{
debugLog("MFEnumDeviceSources failed");
return false;
}
debugLog("device sources=%u", count);
const std::wstring target = lowercase(needle ? needle : L"");
bool found = false;
for (UINT32 index = 0; index < count; ++index)
{
ComPtr<IMFActivate> activate(rawDevices[index]);
rawDevices[index] = nullptr;
WCHAR* symbolicLink = nullptr;
UINT32 symbolicLength = 0;
activate->GetAllocatedString(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK, &symbolicLink, &symbolicLength);
const std::wstring link = lowercase(symbolicLink ? symbolicLink : L"");
if (debugEnabled())
{
std::fwprintf(stderr, L"mimir-mf-gpu: source[%u]=%ls\n", index, link.c_str());
}
CoTaskMemFree(symbolicLink);
if (!target.empty() && link.find(target) == std::wstring::npos)
{
continue;
}
if (SUCCEEDED(activate->ActivateObject(IID_PPV_ARGS(mediaSource))))
{
debugLog("activated source index=%u", index);
found = true;
break;
}
}
for (UINT32 index = 0; index < count; ++index)
{
if (rawDevices[index])
{
rawDevices[index]->Release();
}
}
CoTaskMemFree(rawDevices);
return found;
}
bool mediaTypeMatches(IMFMediaType* type, int width, int height, const GUID& subtype, double minFps)
{
GUID actualSubtype{};
UINT32 actualWidth = 0;
UINT32 actualHeight = 0;
UINT32 fpsNumerator = 0;
UINT32 fpsDenominator = 0;
if (FAILED(type->GetGUID(MF_MT_SUBTYPE, &actualSubtype)) ||
actualSubtype != subtype ||
FAILED(MFGetAttributeSize(type, MF_MT_FRAME_SIZE, &actualWidth, &actualHeight)) ||
static_cast<int>(actualWidth) != width ||
static_cast<int>(actualHeight) != height)
{
return false;
}
if (SUCCEEDED(MFGetAttributeRatio(type, MF_MT_FRAME_RATE, &fpsNumerator, &fpsDenominator)) &&
fpsDenominator != 0)
{
return static_cast<double>(fpsNumerator) / fpsDenominator >= minFps;
}
return minFps <= 0.0;
}
bool setInputType(IMFSourceReader* reader, int width, int height, const GUID& subtype, double minFps)
{
for (DWORD index = 0;; ++index)
{
ComPtr<IMFMediaType> type;
const HRESULT hr = reader->GetNativeMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, index, &type);
if (hr == MF_E_NO_MORE_TYPES)
{
debugLog("no input type matched %dx%d minFps=%.3f", width, height, minFps);
return false;
}
if (FAILED(hr))
{
debugLog("GetNativeMediaType failed index=%lu hr=0x%08lx", index, static_cast<unsigned long>(hr));
return false;
}
if (debugEnabled())
{
GUID actualSubtype{};
UINT32 actualWidth = 0;
UINT32 actualHeight = 0;
UINT32 fpsNumerator = 0;
UINT32 fpsDenominator = 0;
type->GetGUID(MF_MT_SUBTYPE, &actualSubtype);
MFGetAttributeSize(type.Get(), MF_MT_FRAME_SIZE, &actualWidth, &actualHeight);
MFGetAttributeRatio(type.Get(), MF_MT_FRAME_RATE, &fpsNumerator, &fpsDenominator);
debugLog(
"nativeType[%lu] %ux%u subtype=%08lx fps=%u/%u",
index,
actualWidth,
actualHeight,
static_cast<unsigned long>(actualSubtype.Data1),
fpsNumerator,
fpsDenominator);
}
if (mediaTypeMatches(type.Get(), width, height, subtype, minFps))
{
const HRESULT setHr = reader->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, type.Get());
debugLog("SetCurrentMediaType input index=%lu hr=0x%08lx", index, static_cast<unsigned long>(setHr));
return SUCCEEDED(setHr);
}
}
}
bool setOutputType(IMFSourceReader* reader, const GUID& subtype)
{
ComPtr<IMFMediaType> output;
if (FAILED(MFCreateMediaType(&output)))
{
return false;
}
output->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
output->SetGUID(MF_MT_SUBTYPE, subtype);
const HRESULT hr = reader->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, output.Get());
debugLog("SetCurrentMediaType output hr=0x%08lx", static_cast<unsigned long>(hr));
return SUCCEEDED(hr);
}
bool createSharedTexture(MimirMfGpuCapture& capture, const GUID& outputSubtype)
{
ComPtr<IMFMediaType> current;
if (FAILED(capture.reader->GetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, &current)))
{
debugLog("GetCurrentMediaType output failed");
return false;
}
UINT32 width = 0;
UINT32 height = 0;
GUID subtype{};
if (FAILED(MFGetAttributeSize(current.Get(), MF_MT_FRAME_SIZE, &width, &height)) ||
FAILED(current->GetGUID(MF_MT_SUBTYPE, &subtype)))
{
return false;
}
const auto format = dxgiFormatForSubtype(subtype);
if (format == DXGI_FORMAT_UNKNOWN || subtype != outputSubtype)
{
debugLog("output texture subtype/format unsupported subtype=%08lx", static_cast<unsigned long>(subtype.Data1));
return false;
}
D3D11_TEXTURE2D_DESC desc{};
desc.Width = width;
desc.Height = height;
desc.MipLevels = 1;
desc.ArraySize = 1;
desc.Format = format;
desc.SampleDesc.Count = 1;
desc.Usage = D3D11_USAGE_DEFAULT;
desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED_NTHANDLE | D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX;
const HRESULT textureHr = capture.device->CreateTexture2D(&desc, nullptr, &capture.sharedTexture);
if (FAILED(textureHr))
{
debugLog("CreateTexture2D shared NT failed hr=0x%08lx; retrying legacy shared", static_cast<unsigned long>(textureHr));
desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED;
const HRESULT legacyTextureHr = capture.device->CreateTexture2D(&desc, nullptr, &capture.sharedTexture);
if (FAILED(legacyTextureHr))
{
debugLog("CreateTexture2D legacy shared failed hr=0x%08lx", static_cast<unsigned long>(legacyTextureHr));
return false;
}
}
ComPtr<IDXGIResource1> dxgiResource;
const HRESULT asHr = capture.sharedTexture.As(&dxgiResource);
const HRESULT handleHr = SUCCEEDED(asHr)
? dxgiResource->CreateSharedHandle(nullptr, DXGI_SHARED_RESOURCE_READ | DXGI_SHARED_RESOURCE_WRITE, nullptr, &capture.sharedHandle)
: asHr;
if (FAILED(asHr) || FAILED(handleHr))
{
ComPtr<IDXGIResource> legacyResource;
const HRESULT legacyAsHr = capture.sharedTexture.As(&legacyResource);
const HRESULT legacyHandleHr = SUCCEEDED(legacyAsHr)
? legacyResource->GetSharedHandle(&capture.sharedHandle)
: legacyAsHr;
if (FAILED(legacyAsHr) || FAILED(legacyHandleHr))
{
debugLog(
"CreateSharedHandle failed as=0x%08lx handle=0x%08lx legacyAs=0x%08lx legacyHandle=0x%08lx",
static_cast<unsigned long>(asHr),
static_cast<unsigned long>(handleHr),
static_cast<unsigned long>(legacyAsHr),
static_cast<unsigned long>(legacyHandleHr));
return false;
}
}
capture.width = static_cast<int>(width);
capture.height = static_cast<int>(height);
capture.format = format;
return true;
}
bool createProducerFence(MimirMfGpuCapture& capture)
{
ComPtr<ID3D11Device5> device5;
if (FAILED(capture.device.As(&device5)) ||
FAILED(capture.context.As(&capture.fenceContext)))
{
debugLog("D3D11 fence interfaces unavailable; shared texture will be unsynchronized");
return true;
}
const HRESULT fenceHr = device5->CreateFence(
0,
D3D11_FENCE_FLAG_SHARED,
IID_PPV_ARGS(&capture.producerFence));
if (FAILED(fenceHr))
{
debugLog("CreateFence failed hr=0x%08lx; shared texture will be unsynchronized", static_cast<unsigned long>(fenceHr));
capture.fenceContext.Reset();
return true;
}
const HRESULT handleHr = capture.producerFence->CreateSharedHandle(nullptr, GENERIC_ALL, nullptr, &capture.producerFenceHandle);
if (FAILED(handleHr))
{
debugLog("CreateSharedHandle fence failed hr=0x%08lx; shared texture will be unsynchronized", static_cast<unsigned long>(handleHr));
capture.producerFence.Reset();
capture.fenceContext.Reset();
}
return true;
}
} // namespace
extern "C"
{
__declspec(dllexport) MimirMfGpuCapture* mimir_mf_gpu_create(
const char* pathNeedle,
int width,
int height,
const char* inputSubtypeText,
const char* outputSubtypeText,
double minFps)
{
CoInitializeEx(nullptr, COINIT_MULTITHREADED);
if (FAILED(MFStartup(MF_VERSION, MFSTARTUP_NOSOCKET)))
{
debugLog("MFStartup failed");
return nullptr;
}
auto capture = new MimirMfGpuCapture();
const UINT deviceFlags = D3D11_CREATE_DEVICE_VIDEO_SUPPORT | D3D11_CREATE_DEVICE_BGRA_SUPPORT;
D3D_FEATURE_LEVEL featureLevels[] = { D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0 };
D3D_FEATURE_LEVEL featureLevel{};
const HRESULT deviceHr = D3D11CreateDevice(
nullptr,
D3D_DRIVER_TYPE_HARDWARE,
nullptr,
deviceFlags,
featureLevels,
ARRAYSIZE(featureLevels),
D3D11_SDK_VERSION,
&capture->device,
&featureLevel,
&capture->context);
if (FAILED(deviceHr))
{
debugLog("D3D11CreateDevice failed hr=0x%08lx", static_cast<unsigned long>(deviceHr));
delete capture;
return nullptr;
}
ComPtr<ID3D10Multithread> multithread;
if (SUCCEEDED(capture->device.As(&multithread)))
{
multithread->SetMultithreadProtected(TRUE);
}
UINT resetToken = 0;
if (FAILED(MFCreateDXGIDeviceManager(&resetToken, &capture->deviceManager)) ||
FAILED(capture->deviceManager->ResetDevice(capture->device.Get(), resetToken)))
{
debugLog("DXGI device manager setup failed");
delete capture;
return nullptr;
}
const auto needle = widen(pathNeedle);
if (!findVideoSource(needle.c_str(), &capture->source))
{
debugLog("no source matched needle");
delete capture;
return nullptr;
}
ComPtr<IMFAttributes> readerAttributes;
if (FAILED(MFCreateAttributes(&readerAttributes, 4)))
{
debugLog("MFCreateAttributes reader failed");
delete capture;
return nullptr;
}
readerAttributes->SetUnknown(MF_SOURCE_READER_D3D_MANAGER, capture->deviceManager.Get());
readerAttributes->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, TRUE);
readerAttributes->SetUINT32(MF_LOW_LATENCY, TRUE);
readerAttributes->SetUINT32(MF_SOURCE_READER_ENABLE_ADVANCED_VIDEO_PROCESSING, TRUE);
const HRESULT readerHr = MFCreateSourceReaderFromMediaSource(capture->source.Get(), readerAttributes.Get(), &capture->reader);
if (FAILED(readerHr))
{
debugLog("MFCreateSourceReaderFromMediaSource failed hr=0x%08lx", static_cast<unsigned long>(readerHr));
delete capture;
return nullptr;
}
const auto inputSubtype = subtypeFromText(inputSubtypeText);
const auto outputSubtype = subtypeFromText(outputSubtypeText && outputSubtypeText[0] ? outputSubtypeText : "NV12");
if (inputSubtype == GUID_NULL ||
outputSubtype == GUID_NULL ||
!setInputType(capture->reader.Get(), width, height, inputSubtype, minFps) ||
!setOutputType(capture->reader.Get(), outputSubtype) ||
!createSharedTexture(*capture, outputSubtype) ||
!createProducerFence(*capture))
{
debugLog("create failed input=%s output=%s", inputSubtypeText ? inputSubtypeText : "", outputSubtypeText ? outputSubtypeText : "");
delete capture;
return nullptr;
}
return capture;
}
__declspec(dllexport) int mimir_mf_gpu_read(
MimirMfGpuCapture* capture,
HANDLE* sharedHandle,
int* width,
int* height,
char* format,
int formatCapacity,
std::int64_t* timestampNs,
std::uint64_t* sequence,
HANDLE* producerFenceHandle)
{
if (!capture || !capture->reader || !capture->sharedTexture || !sharedHandle)
{
return 0;
}
DWORD streamIndex = 0;
DWORD flags = 0;
LONGLONG timestamp = 0;
ComPtr<IMFSample> sample;
const HRESULT hr = capture->reader->ReadSample(
MF_SOURCE_READER_FIRST_VIDEO_STREAM,
0,
&streamIndex,
&flags,
&timestamp,
&sample);
if (FAILED(hr) || !sample || (flags & MF_SOURCE_READERF_STREAMTICK) != 0)
{
return 0;
}
ComPtr<IMFMediaBuffer> buffer;
ComPtr<IMFDXGIBuffer> dxgiBuffer;
ComPtr<ID3D11Texture2D> decodedTexture;
UINT subresource = 0;
if (FAILED(sample->GetBufferByIndex(0, &buffer)) ||
FAILED(buffer.As(&dxgiBuffer)) ||
FAILED(dxgiBuffer->GetResource(IID_PPV_ARGS(&decodedTexture))) ||
FAILED(dxgiBuffer->GetSubresourceIndex(&subresource)))
{
return 0;
}
const auto nextSequence = capture->sequence + 1;
capture->context->CopySubresourceRegion(capture->sharedTexture.Get(), 0, 0, 0, 0, decodedTexture.Get(), subresource, nullptr);
if (capture->producerFence && capture->fenceContext)
{
capture->fenceContext->Signal(capture->producerFence.Get(), nextSequence);
}
capture->context->Flush();
*sharedHandle = capture->sharedHandle;
if (width) *width = capture->width;
if (height) *height = capture->height;
if (format && formatCapacity > 0)
{
const char* text = capture->format == DXGI_FORMAT_NV12
? textForSubtype(MFVideoFormat_NV12)
: capture->format == DXGI_FORMAT_B8G8R8A8_UNORM
? textForSubtype(MFVideoFormat_RGB32)
: textForSubtype(MFVideoFormat_YUY2);
std::snprintf(format, static_cast<std::size_t>(formatCapacity), "%s", text);
}
if (timestampNs) *timestampNs = timestamp * 100;
capture->sequence = nextSequence;
if (sequence) *sequence = capture->sequence;
if (producerFenceHandle) *producerFenceHandle = capture->producerFenceHandle;
return 1;
}
__declspec(dllexport) void mimir_mf_gpu_destroy(MimirMfGpuCapture* capture)
{
if (!capture)
{
return;
}
if (capture->sharedHandle)
{
CloseHandle(capture->sharedHandle);
}
if (capture->producerFenceHandle)
{
CloseHandle(capture->producerFenceHandle);
}
if (capture->source)
{
capture->source->Shutdown();
}
delete capture;
MFShutdown();
CoUninitialize();
}
}

View file

@ -0,0 +1,215 @@
#include <algorithm>
#include <atomic>
#include <chrono>
#include <condition_variable>
#include <cstdint>
#include <cstring>
#include <deque>
#include <mutex>
#include <thread>
#include <vector>
#include "ps3eye_capi.h"
namespace
{
struct MimirPs3EyeFrame
{
std::vector<unsigned char> data;
std::int64_t timestampNs = 0;
std::uint64_t sequence = 0;
};
struct MimirPs3EyeCapture
{
ps3eye_t* eye = nullptr;
std::thread worker;
std::mutex mutex;
std::condition_variable condition;
std::deque<MimirPs3EyeFrame> queue;
std::atomic<bool> running = false;
int cameraIndex = 0;
int width = 0;
int height = 0;
int fps = 0;
std::uint64_t sequence = 0;
};
std::mutex g_ps3eye_mutex;
int g_ps3eye_users = 0;
long long nowNs()
{
return std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::steady_clock::now().time_since_epoch()).count();
}
void retainPs3Eye()
{
std::lock_guard<std::mutex> lock(g_ps3eye_mutex);
if (g_ps3eye_users++ == 0)
{
ps3eye_init();
}
}
void releasePs3Eye()
{
std::lock_guard<std::mutex> lock(g_ps3eye_mutex);
if (g_ps3eye_users > 0 && --g_ps3eye_users == 0)
{
ps3eye_uninit();
}
}
void pushFrame(MimirPs3EyeCapture& capture, const std::vector<unsigned char>& source)
{
MimirPs3EyeFrame frame;
frame.data = source;
frame.timestampNs = nowNs();
frame.sequence = capture.sequence++;
{
std::lock_guard<std::mutex> lock(capture.mutex);
capture.queue.push_back(std::move(frame));
while (capture.queue.size() > 32)
{
capture.queue.pop_front();
}
}
capture.condition.notify_one();
}
void captureThread(MimirPs3EyeCapture* capture)
{
std::vector<unsigned char> frame(static_cast<std::size_t>(capture->width) * static_cast<std::size_t>(capture->height));
while (capture->running.load(std::memory_order_acquire))
{
ps3eye_grab_frame(capture->eye, frame.data());
pushFrame(*capture, frame);
}
}
}
extern "C"
{
__declspec(dllexport) MimirPs3EyeCapture* mimir_ps3eye_create(int cameraIndex, int width, int height, int fps)
{
if (cameraIndex < 0 || width <= 0 || height <= 0 || fps <= 0)
{
return nullptr;
}
retainPs3Eye();
if (cameraIndex >= ps3eye_count_connected())
{
releasePs3Eye();
return nullptr;
}
auto* capture = new MimirPs3EyeCapture();
capture->cameraIndex = cameraIndex;
capture->width = width;
capture->height = height;
capture->fps = fps;
capture->eye = ps3eye_open(cameraIndex, width, height, fps, PS3EYE_FORMAT_BAYER);
if (!capture->eye)
{
delete capture;
releasePs3Eye();
return nullptr;
}
return capture;
}
__declspec(dllexport) int mimir_ps3eye_start(MimirPs3EyeCapture* capture)
{
if (!capture || capture->running.exchange(true))
{
return 0;
}
capture->worker = std::thread(captureThread, capture);
return 1;
}
__declspec(dllexport) int mimir_ps3eye_read(
MimirPs3EyeCapture* capture,
int* width,
int* height,
int* stride,
std::int64_t* timestampNs,
std::uint64_t* sequence,
unsigned char* destination,
int destinationBytes)
{
if (!capture || !destination || destinationBytes <= 0)
{
return 0;
}
MimirPs3EyeFrame frame;
{
std::lock_guard<std::mutex> lock(capture->mutex);
if (capture->queue.empty())
{
return 0;
}
frame = std::move(capture->queue.front());
capture->queue.pop_front();
}
if (frame.data.size() > static_cast<std::size_t>(destinationBytes))
{
return -static_cast<int>(frame.data.size());
}
std::memcpy(destination, frame.data.data(), frame.data.size());
if (width)
{
*width = capture->width;
}
if (height)
{
*height = capture->height;
}
if (stride)
{
*stride = capture->width;
}
if (timestampNs)
{
*timestampNs = frame.timestampNs;
}
if (sequence)
{
*sequence = frame.sequence;
}
return static_cast<int>(frame.data.size());
}
__declspec(dllexport) void mimir_ps3eye_destroy(MimirPs3EyeCapture* capture)
{
if (!capture)
{
return;
}
capture->running.store(false, std::memory_order_release);
if (capture->worker.joinable())
{
capture->worker.join();
}
if (capture->eye)
{
ps3eye_close(capture->eye);
}
delete capture;
releasePs3Eye();
}
}

View file

@ -17,13 +17,11 @@ target_compile_definitions(mimir_obs_stem_source PRIVATE
_WIN32_WINNT=0x0A00)
target_link_libraries(mimir_obs_stem_source PRIVATE OBS::libobs d3d11 d3d12 dxgi ws2_32)
if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../tools/muninn_media_decoder_probe.cpp")
add_executable(muninn_media_decoder_probe
../../tools/muninn_media_decoder_probe.cpp)
target_compile_features(muninn_media_decoder_probe PRIVATE cxx_std_20)
target_include_directories(muninn_media_decoder_probe PRIVATE
${CMAKE_CURRENT_SOURCE_DIR})
endif()
add_executable(muninn_media_decoder_probe
../../tools/muninn_media_decoder_probe.cpp)
target_compile_features(muninn_media_decoder_probe PRIVATE cxx_std_20)
target_include_directories(muninn_media_decoder_probe PRIVATE
${CMAKE_CURRENT_SOURCE_DIR})
add_executable(muninn_rudp_url_tests
tests/muninn_rudp_url_tests.cpp)
@ -31,36 +29,3 @@ target_compile_features(muninn_rudp_url_tests PRIVATE cxx_std_20)
target_include_directories(muninn_rudp_url_tests PRIVATE
${CMAKE_CURRENT_SOURCE_DIR})
add_test(NAME muninn_rudp_url_tests COMMAND muninn_rudp_url_tests)
add_executable(muninn_media_wire_tests
tests/muninn_media_wire_tests.cpp)
target_compile_features(muninn_media_wire_tests PRIVATE cxx_std_20)
target_include_directories(muninn_media_wire_tests PRIVATE
${CMAKE_CURRENT_SOURCE_DIR})
add_test(NAME muninn_media_wire_tests COMMAND muninn_media_wire_tests)
add_executable(muninn_audio_fec_tests
tests/muninn_audio_fec_tests.cpp)
target_compile_features(muninn_audio_fec_tests PRIVATE cxx_std_20)
target_include_directories(muninn_audio_fec_tests PRIVATE
${CMAKE_CURRENT_SOURCE_DIR})
add_test(NAME muninn_audio_fec_tests COMMAND muninn_audio_fec_tests)
add_executable(muninn_video_fec_tests
tests/muninn_video_fec_tests.cpp)
target_compile_features(muninn_video_fec_tests PRIVATE cxx_std_20)
target_include_directories(muninn_video_fec_tests PRIVATE
${CMAKE_CURRENT_SOURCE_DIR})
add_test(NAME muninn_video_fec_tests COMMAND muninn_video_fec_tests)
add_executable(muninn_udp_socket_tests
tests/muninn_udp_socket_tests.cpp)
target_compile_features(muninn_udp_socket_tests PRIVATE cxx_std_20)
target_compile_definitions(muninn_udp_socket_tests PRIVATE
WIN32_LEAN_AND_MEAN
NOMINMAX
_WIN32_WINNT=0x0A00)
target_include_directories(muninn_udp_socket_tests PRIVATE
${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(muninn_udp_socket_tests PRIVATE ws2_32)
add_test(NAME muninn_udp_socket_tests COMMAND muninn_udp_socket_tests)

File diff suppressed because it is too large Load diff

View file

@ -214,7 +214,6 @@ bool obs_module_load(void)
blog(
LOG_INFO,
"Mimir OBS sources loaded: mimir_audio_stem_source, mimir_program_texture_source, "
"muninn_stream_source, muninn_video_source, muninn_audio_source; "
"Muninn Odin discovery build 20260705.2");
"muninn_stream_source, muninn_video_source, muninn_audio_source");
return true;
}

View file

@ -1,179 +0,0 @@
#pragma once
#include <array>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <iterator>
#include <map>
#include <optional>
#include <string>
#include <tuple>
#include <utility>
#include <vector>
namespace muninn_audio_fec {
inline constexpr size_t DataShards = 4;
inline constexpr size_t ParityShards = 2;
inline constexpr size_t TotalShards = DataShards + ParityShards;
using Bytes = std::vector<uint8_t>;
using Shards = std::array<std::optional<Bytes>, TotalShards>;
inline uint8_t multiply(uint8_t left, uint8_t right)
{
uint16_t product = 0;
uint16_t a = left;
uint16_t b = right;
while (b != 0) {
if ((b & 1u) != 0) product ^= a;
b >>= 1u;
a <<= 1u;
if ((a & 0x100u) != 0) a ^= 0x11du;
}
return static_cast<uint8_t>(product);
}
inline uint8_t power(uint8_t value, uint16_t exponent)
{
uint8_t result = 1;
while (exponent != 0) {
if ((exponent & 1u) != 0) result = multiply(result, value);
value = multiply(value, value);
exponent >>= 1u;
}
return result;
}
inline uint8_t inverse(uint8_t value)
{
return value == 0 ? 0 : power(value, 254);
}
inline constexpr std::array<std::array<uint8_t, DataShards>, TotalShards> Generator{{
{{1, 0, 0, 0}}, {{0, 1, 0, 0}}, {{0, 0, 1, 0}}, {{0, 0, 0, 1}},
{{1, 1, 1, 1}}, {{1, 2, 4, 8}}
}};
inline std::array<Bytes, ParityShards> encode(const std::array<Bytes, DataShards> &data)
{
const size_t bytes = data[0].size();
for (const auto &shard : data) if (shard.size() != bytes) return {};
std::array<Bytes, ParityShards> parity{Bytes(bytes), Bytes(bytes)};
for (size_t p = 0; p < ParityShards; ++p)
for (size_t d = 0; d < DataShards; ++d)
for (size_t i = 0; i < bytes; ++i)
parity[p][i] ^= multiply(Generator[DataShards + p][d], data[d][i]);
return parity;
}
inline bool recover(Shards &shards)
{
size_t shard_bytes = 0;
std::array<size_t, DataShards> available{};
size_t count = 0;
for (size_t i = 0; i < TotalShards; ++i) {
if (!shards[i]) continue;
if (shard_bytes == 0) shard_bytes = shards[i]->size();
if (shards[i]->size() != shard_bytes) return false;
if (count < DataShards) available[count++] = i;
}
if (count < DataShards || shard_bytes == 0) return false;
std::array<std::array<uint8_t, DataShards * 2>, DataShards> matrix{};
for (size_t row = 0; row < DataShards; ++row) {
for (size_t col = 0; col < DataShards; ++col) matrix[row][col] = Generator[available[row]][col];
matrix[row][DataShards + row] = 1;
}
for (size_t col = 0; col < DataShards; ++col) {
size_t pivot = col;
while (pivot < DataShards && matrix[pivot][col] == 0) ++pivot;
if (pivot == DataShards) return false;
if (pivot != col) std::swap(matrix[pivot], matrix[col]);
const uint8_t scale = inverse(matrix[col][col]);
for (size_t j = 0; j < DataShards * 2; ++j) matrix[col][j] = multiply(matrix[col][j], scale);
for (size_t row = 0; row < DataShards; ++row) {
if (row == col) continue;
const uint8_t factor = matrix[row][col];
for (size_t j = 0; j < DataShards * 2; ++j)
matrix[row][j] ^= multiply(factor, matrix[col][j]);
}
}
for (size_t missing = 0; missing < DataShards; ++missing) {
if (shards[missing]) continue;
Bytes decoded(shard_bytes);
for (size_t row = 0; row < DataShards; ++row) {
const uint8_t coefficient = matrix[missing][DataShards + row];
for (size_t i = 0; i < shard_bytes; ++i)
decoded[i] ^= multiply(coefficient, (*shards[available[row]])[i]);
}
shards[missing] = std::move(decoded);
}
return true;
}
struct BlockKey {
std::string stream_id;
std::string session_id;
uint64_t base_packet_id = 0;
bool operator<(const BlockKey &other) const
{
return std::tie(stream_id, session_id, base_packet_id) <
std::tie(other.stream_id, other.session_id, other.base_packet_id);
}
};
struct RecoveredPacket { uint64_t packet_id; Bytes payload; };
class BlockReceiver {
public:
using Clock = std::chrono::steady_clock;
explicit BlockReceiver(std::chrono::milliseconds deadline, size_t max_blocks = 128)
: deadline_(deadline), max_blocks_(max_blocks) {}
std::vector<RecoveredPacket> insert(const BlockKey &key, size_t shard_index, Bytes payload,
Clock::time_point now)
{
expire(now);
if (shard_index >= TotalShards || payload.empty() || completed_.count(key) != 0) return {};
if (blocks_.size() >= max_blocks_ && blocks_.count(key) == 0) blocks_.erase(blocks_.begin());
auto [it, created] = blocks_.try_emplace(key, Block{now, payload.size(), {}});
Block &block = it->second;
if (payload.size() != block.shard_bytes || block.shards[shard_index]) return {};
block.shards[shard_index] = std::move(payload);
size_t present = 0;
for (const auto &shard : block.shards) if (shard) ++present;
if (present < DataShards) return {};
Shards before = block.shards;
if (!recover(block.shards)) return {};
std::vector<RecoveredPacket> result;
for (size_t i = 0; i < DataShards; ++i)
if (!before[i] && block.shards[i]) result.push_back({key.base_packet_id + i, *block.shards[i]});
blocks_.erase(it);
completed_[key] = now;
return result;
}
void expire(Clock::time_point now)
{
for (auto it = blocks_.begin(); it != blocks_.end();)
it = now - it->second.created_at >= deadline_ ? blocks_.erase(it) : std::next(it);
for (auto it = completed_.begin(); it != completed_.end();)
it = now - it->second >= deadline_ ? completed_.erase(it) : std::next(it);
}
size_t pending_blocks() const { return blocks_.size(); }
void clear()
{
blocks_.clear();
completed_.clear();
}
private:
struct Block { Clock::time_point created_at; size_t shard_bytes; Shards shards; };
std::chrono::milliseconds deadline_;
size_t max_blocks_;
std::map<BlockKey, Block> blocks_;
std::map<BlockKey, Clock::time_point> completed_;
};
} // namespace muninn_audio_fec

View file

@ -172,18 +172,12 @@ inline void write_u64_array(MsgpackWriter &writer, const std::vector<uint64_t> &
}
}
inline std::vector<uint64_t> receiver_feedback_late_frame_ids(uint64_t frame_id, bool late_frame)
{
return late_frame ? std::vector<uint64_t>{frame_id} : std::vector<uint64_t>{};
}
inline std::vector<uint8_t> encode_receiver_feedback_payload(
const std::string &stream_id,
const std::string &session_id,
uint64_t frame_id,
const std::vector<std::string> &missing_chunk_keys,
bool requested_keyframe,
bool late_frame,
const std::string &observed_at)
{
MsgpackWriter record;
@ -197,7 +191,7 @@ inline std::vector<uint8_t> encode_receiver_feedback_payload(
record.write_nil();
}
write_u64_array(record, {});
write_u64_array(record, receiver_feedback_late_frame_ids(frame_id, late_frame));
write_u64_array(record, {frame_id});
record.write_bool(requested_keyframe);
record.write_i64(0);
record.write_i64(0);
@ -253,7 +247,6 @@ inline std::vector<uint8_t> encode_receiver_feedback_payload(
frame_id,
missing_chunk_keys,
requested_keyframe,
requested_keyframe,
feedback_observed_at());
}

View file

@ -13,12 +13,11 @@ inline constexpr uint32_t DefaultAudioRudpConnectionId = 0x6d750004;
inline constexpr uint32_t DefaultReliableExpireAfterMs = 75;
inline constexpr uint32_t DefaultVideoAssemblyDeadlineMs = 75;
inline constexpr uint32_t DefaultGapWaitMs = 8;
inline constexpr uint32_t DefaultAudioReorderMs = 120;
inline constexpr uint32_t DefaultAudioReorderMs = 2000;
inline constexpr uint32_t DefaultSenderResendDelayMs = 5;
inline constexpr uint32_t MaxLatencyBudgetMs = 500;
inline constexpr uint32_t MaxLatencyBudgetMs = 2000;
struct RudpUrlParts {
std::string channel_id = "media";
uint16_t port = 0;
uint16_t video_local_port = 0;
uint16_t audio_local_port = 0;
@ -82,9 +81,7 @@ inline std::optional<RudpUrlParts> parse(const std::string &url)
}
const auto key = item.substr(0, equals);
const auto value = item.substr(equals + 1);
if (key == "channel" && !value.empty()) {
parts.channel_id = value;
} else if (key == "connection") {
if (key == "connection") {
if (const auto parsed = parse_u32_hex_or_decimal(value)) {
parts.connection_id = *parsed;
}

View file

@ -1,32 +0,0 @@
#pragma once
#ifdef _WIN32
#include <winsock2.h>
namespace muninn_udp_socket {
inline SOCKET create(bool exclusive_listener = false)
{
const SOCKET socket = WSASocketW(
AF_INET,
SOCK_DGRAM,
IPPROTO_UDP,
nullptr,
0,
WSA_FLAG_NO_HANDLE_INHERIT);
if (socket == INVALID_SOCKET) {
return INVALID_SOCKET;
}
if (exclusive_listener) {
const BOOL exclusive = TRUE;
if (setsockopt(socket, SOL_SOCKET, SO_EXCLUSIVEADDRUSE,
reinterpret_cast<const char *>(&exclusive), sizeof(exclusive)) != 0) {
closesocket(socket);
return INVALID_SOCKET;
}
}
return socket;
}
} // namespace muninn_udp_socket
#endif

View file

@ -1,181 +0,0 @@
#pragma once
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <map>
#include <utility>
#include <vector>
namespace muninn_video_fec {
inline uint8_t raw_multiply(uint8_t a, uint8_t b)
{
uint8_t product = 0;
while (b != 0) {
if ((b & 1) != 0)
product ^= a;
const bool carry = (a & 0x80) != 0;
a <<= 1;
if (carry)
a ^= 0x1d;
b >>= 1;
}
return product;
}
struct Tables {
uint8_t products[256][256] = {};
uint8_t inverses[256] = {};
Tables()
{
for (size_t a = 0; a < 256; ++a) {
for (size_t b = 0; b < 256; ++b)
products[a][b] = raw_multiply(static_cast<uint8_t>(a), static_cast<uint8_t>(b));
}
for (size_t value = 1; value < 256; ++value) {
uint8_t result = 1;
for (size_t index = 0; index < 254; ++index)
result = raw_multiply(result, static_cast<uint8_t>(value));
inverses[value] = result;
}
}
};
inline const Tables &tables()
{
static const Tables value;
return value;
}
inline uint8_t multiply(uint8_t a, uint8_t b)
{
return tables().products[a][b];
}
inline uint8_t inverse(uint8_t value)
{
return tables().inverses[value];
}
inline uint8_t coefficient(uint16_t parity_index, uint16_t parity_count, uint16_t data_index)
{
return inverse(
static_cast<uint8_t>(parity_index) ^ static_cast<uint8_t>(parity_count + data_index));
}
inline bool recover(std::vector<std::vector<uint8_t>> &chunks,
const std::vector<uint32_t> &chunk_lengths,
uint16_t parity_count,
const std::map<uint16_t, std::vector<uint8_t>> &parity)
{
if (chunks.size() != chunk_lengths.size() || chunks.empty() ||
chunks.size() + parity_count > 256)
return false;
std::vector<uint16_t> missing;
for (uint16_t index = 0; index < chunks.size(); ++index) {
if (chunks[index].empty())
missing.push_back(index);
}
if (missing.empty())
return true;
if (missing.size() > parity.size())
return false;
std::vector<std::pair<uint16_t, const std::vector<uint8_t> *>> rows;
for (const auto &shard : parity) {
if (shard.first < parity_count && !shard.second.empty())
rows.emplace_back(shard.first, &shard.second);
if (rows.size() == missing.size())
break;
}
if (rows.size() != missing.size())
return false;
size_t shard_bytes = 0;
for (uint32_t length : chunk_lengths)
shard_bytes = std::max(shard_bytes, static_cast<size_t>(length));
if (shard_bytes == 0 || std::any_of(rows.begin(), rows.end(),
[shard_bytes](const auto &row) { return row.second->size() < shard_bytes; }))
return false;
std::vector<std::vector<uint8_t>> recovered;
recovered.reserve(missing.size());
for (uint16_t index : missing) {
if (chunk_lengths[index] == 0 || chunk_lengths[index] > shard_bytes)
return false;
recovered.emplace_back(chunk_lengths[index], 0);
}
const size_t count = missing.size();
std::vector<std::vector<uint8_t>> inverse_matrix(count, std::vector<uint8_t>(count * 2, 0));
for (size_t row = 0; row < count; ++row) {
for (size_t column = 0; column < count; ++column)
inverse_matrix[row][column] = coefficient(rows[row].first, parity_count, missing[column]);
inverse_matrix[row][count + row] = 1;
}
for (size_t column = 0; column < count; ++column) {
size_t pivot = column;
while (pivot < count && inverse_matrix[pivot][column] == 0)
++pivot;
if (pivot == count)
return false;
std::swap(inverse_matrix[column], inverse_matrix[pivot]);
const uint8_t scale = inverse(inverse_matrix[column][column]);
for (size_t cell = 0; cell < count * 2; ++cell)
inverse_matrix[column][cell] = multiply(inverse_matrix[column][cell], scale);
for (size_t row = 0; row < count; ++row) {
if (row == column)
continue;
const uint8_t factor = inverse_matrix[row][column];
for (size_t cell = 0; cell < count * 2; ++cell)
inverse_matrix[row][cell] ^= multiply(factor, inverse_matrix[column][cell]);
}
}
std::vector<uint8_t> rhs(count, 0);
for (size_t offset = 0; offset < shard_bytes; ++offset) {
for (size_t row = 0; row < count; ++row) {
const uint16_t parity_index = rows[row].first;
rhs[row] = (*rows[row].second)[offset];
for (uint16_t data_index = 0; data_index < chunks.size(); ++data_index) {
if (!chunks[data_index].empty() && offset < chunks[data_index].size())
rhs[row] ^= multiply(coefficient(parity_index, parity_count, data_index),
chunks[data_index][offset]);
}
}
for (size_t index = 0; index < count; ++index) {
uint8_t value = 0;
for (size_t row = 0; row < count; ++row)
value ^= multiply(inverse_matrix[index][count + row], rhs[row]);
if (offset < recovered[index].size())
recovered[index][offset] = value;
}
}
for (size_t index = 0; index < count; ++index)
chunks[missing[index]] = std::move(recovered[index]);
return true;
}
inline bool recover_block(std::vector<std::vector<uint8_t>> &frame_chunks,
uint16_t data_start,
const std::vector<uint32_t> &block_lengths,
uint16_t parity_count,
const std::map<uint16_t, std::vector<uint8_t>> &parity)
{
if (block_lengths.empty() ||
static_cast<size_t>(data_start) + block_lengths.size() > frame_chunks.size())
return false;
std::vector<std::vector<uint8_t>> block_chunks;
block_chunks.reserve(block_lengths.size());
for (size_t index = 0; index < block_lengths.size(); ++index)
block_chunks.push_back(frame_chunks[data_start + index]);
if (!recover(block_chunks, block_lengths, parity_count, parity))
return false;
for (size_t index = 0; index < block_lengths.size(); ++index)
frame_chunks[data_start + index] = std::move(block_chunks[index]);
return true;
}
} // namespace muninn_video_fec

View file

@ -1,79 +0,0 @@
#include "muninn_audio_fec.h"
#include <cstdlib>
#include <iostream>
#include <stdexcept>
static void require(bool condition, const char *message) { if (!condition) throw std::runtime_error(message); }
static std::array<muninn_audio_fec::Bytes, 4> fixture()
{
std::array<muninn_audio_fec::Bytes, 4> data;
for (size_t shard = 0; shard < data.size(); ++shard) {
data[shard].resize(257);
for (size_t i = 0; i < data[shard].size(); ++i) data[shard][i] = uint8_t((shard * 71 + i * 29) & 0xff);
}
return data;
}
static muninn_audio_fec::Shards all_shards(const std::array<muninn_audio_fec::Bytes, 4> &data)
{
auto parity = muninn_audio_fec::encode(data);
return {data[0], data[1], data[2], data[3], parity[0], parity[1]};
}
int main()
{
try {
const auto data = fixture();
for (size_t erased = 0; erased < 6; ++erased) {
auto shards = all_shards(data); shards[erased].reset();
require(muninn_audio_fec::recover(shards), "one-erasure recovery failed");
for (size_t i = 0; i < 4; ++i) require(shards[i] == data[i], "one-erasure data mismatch");
}
for (size_t first = 0; first < 6; ++first) for (size_t second = first + 1; second < 6; ++second) {
auto shards = all_shards(data); shards[first].reset(); shards[second].reset();
require(muninn_audio_fec::recover(shards), "two-erasure recovery failed");
for (size_t i = 0; i < 4; ++i) require(shards[i] == data[i], "two-erasure data mismatch");
}
{
auto shards = all_shards(data);
shards[0].reset(); shards[1].reset(); shards[2].reset();
require(!muninn_audio_fec::recover(shards), "three erasures were incorrectly recoverable");
}
{
auto shards = all_shards(data);
shards[5]->pop_back();
require(!muninn_audio_fec::recover(shards), "unequal shard sizes were accepted");
}
using namespace muninn_audio_fec;
BlockReceiver receiver(std::chrono::milliseconds(10), 2);
BlockKey key{"stream", "session", 100};
auto shards = all_shards(data);
auto t0 = BlockReceiver::Clock::time_point{};
require(receiver.insert(key, 5, *shards[5], t0).empty(), "early parity emitted data");
require(receiver.insert(key, 2, *shards[2], t0).empty(), "partial reorder emitted data");
require(receiver.insert(key, 2, *shards[2], t0).empty(), "duplicate emitted data");
require(receiver.insert(key, 1, *shards[1], t0).empty(), "partial block emitted data");
auto recovered = receiver.insert(key, 4, *shards[4], t0);
require(recovered.size() == 2 && recovered[0].packet_id == 100 && recovered[1].packet_id == 103,
"reordered block did not recover missing packet ids");
require(recovered[0].payload == data[0] && recovered[1].payload == data[3], "receiver recovery mismatch");
require(receiver.insert(key, 0, data[0], t0).empty(), "completed-block duplicate was admitted");
BlockKey expired{"stream", "session", 200};
receiver.insert(expired, 0, data[0], t0);
receiver.expire(t0 + std::chrono::milliseconds(10));
require(receiver.pending_blocks() == 0, "expired block survived deadline");
BlockReceiver bounded(std::chrono::seconds(1), 2);
bounded.insert({"s", "x", 0}, 0, data[0], t0);
bounded.insert({"s", "x", 4}, 0, data[0], t0);
bounded.insert({"s", "x", 8}, 0, data[0], t0);
require(bounded.pending_blocks() == 2, "block bound was not enforced");
} catch (const std::exception &error) {
std::cerr << error.what() << '\n'; return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}

View file

@ -1,54 +0,0 @@
#include "muninn_media_wire.h"
#include <cstdlib>
#include <iomanip>
#include <iostream>
namespace {
void require(bool condition, const char *message)
{
if (!condition) {
std::cerr << message << '\n';
std::exit(1);
}
}
} // namespace
int main(int argc, char **argv)
{
const auto early = muninn_media_wire::receiver_feedback_late_frame_ids(42, false);
require(early.empty(), "early repair feedback marked a live frame late");
const auto expired = muninn_media_wire::receiver_feedback_late_frame_ids(42, true);
require(expired.size() == 1 && expired.front() == 42,
"expired feedback did not name the late frame");
const auto early_wire = muninn_media_wire::encode_receiver_feedback_payload(
"muninn.raven.av.rudp",
"raven:session:video",
42,
{"42:3"},
false,
false,
"unix-ms:1000");
const auto expired_wire = muninn_media_wire::encode_receiver_feedback_payload(
"muninn.raven.av.rudp",
"raven:session:video",
42,
{"42:3"},
true,
true,
"unix-ms:1000");
require(!early_wire.empty() && !expired_wire.empty(), "feedback wire encoding was empty");
require(early_wire != expired_wire, "early and expired feedback encoded identically");
if (argc == 2 && std::string(argv[1]) == "--emit-early-hex") {
for (const auto byte : early_wire) {
std::cout << std::hex << std::setw(2) << std::setfill('0')
<< static_cast<unsigned int>(byte);
}
std::cout << '\n';
}
return 0;
}

View file

@ -19,18 +19,17 @@ void require(bool condition, const std::string &message)
int main()
{
const auto configured = muninn_rudp_url::parse(
"rudp://192.168.1.66:5204/muninn.raven.av.rudp?channel=realtime&format=muninn-typed-media&connection=0x6d750001&audio_connection=0x6d750004&sender_resend_delay_ms=5&reliable_expire_after_ms=120&assembly_deadline_ms=120&gap_wait_ms=16&audio_reorder_ms=80");
"rudp://192.168.1.66:5204/muninn.raven.av.rudp?channel=media&format=muninn-typed-media&connection=0x6d750001&audio_connection=0x6d750004&sender_resend_delay_ms=5&reliable_expire_after_ms=1200&assembly_deadline_ms=1200&gap_wait_ms=16&audio_reorder_ms=700");
require(configured.has_value(), "configured RUDP URL did not parse");
require(configured->port == 5204, "configured RUDP URL port was not parsed");
require(configured->channel_id == "realtime", "configured RUDP realtime channel was not parsed");
require(configured->video_local_port == 15204, "configured RUDP video local port was not derived");
require(configured->audio_local_port == 15205, "configured RUDP audio local port was not derived");
require(configured->connection_id == 0x6d750001, "configured RUDP connection id was not parsed");
require(configured->audio_connection_id == 0x6d750004, "configured RUDP audio connection id was not parsed");
require(configured->reliable_expire_after_ms == 120, "configured reliable expiry did not survive parse");
require(configured->video_assembly_deadline_ms == 120, "configured assembly deadline did not survive parse");
require(configured->reliable_expire_after_ms == 1200, "configured reliable expiry did not survive parse");
require(configured->video_assembly_deadline_ms == 1200, "configured assembly deadline did not survive parse");
require(configured->gap_wait_ms == 16, "configured gap wait did not survive parse");
require(configured->audio_reorder_ms == 80, "configured audio reorder wait did not survive parse");
require(configured->audio_reorder_ms == 700, "configured audio reorder wait did not survive parse");
const auto clamped = muninn_rudp_url::parse(
"rudp://192.168.1.66:5204/muninn.raven.av.rudp?reliable_expire_after_ms=9000&assembly_deadline_ms=9000");
@ -42,7 +41,6 @@ int main()
const auto fallback = muninn_rudp_url::parse("rudp://192.168.1.66:5204/muninn.raven.av.rudp");
require(fallback.has_value(), "minimal RUDP URL did not parse");
require(fallback->channel_id == "media", "minimal RUDP URL did not use bounded reliable media delivery");
require(fallback->connection_id == muninn_rudp_url::DefaultMediaRudpConnectionId,
"minimal RUDP URL did not keep the default connection id");
require(fallback->audio_connection_id == muninn_rudp_url::DefaultAudioRudpConnectionId,
@ -53,8 +51,8 @@ int main()
"minimal RUDP URL did not keep the default assembly deadline");
require(fallback->audio_reorder_ms == muninn_rudp_url::DefaultAudioReorderMs,
"minimal RUDP URL did not keep the default audio reorder wait");
require(fallback->audio_reorder_ms == 120,
"minimal RUDP URL did not use the bounded realtime audio reorder budget");
require(fallback->audio_reorder_ms == 2000,
"minimal RUDP URL did not use the full default audio reorder budget");
return 0;
}

View file

@ -1,34 +0,0 @@
#include "muninn_udp_socket.h"
#include <cassert>
#include <windows.h>
int main()
{
WSADATA data{};
assert(WSAStartup(MAKEWORD(2, 2), &data) == 0);
const SOCKET owner = muninn_udp_socket::create(true);
assert(owner != INVALID_SOCKET);
DWORD flags = HANDLE_FLAG_INHERIT;
assert(GetHandleInformation(reinterpret_cast<HANDLE>(owner), &flags));
assert((flags & HANDLE_FLAG_INHERIT) == 0);
sockaddr_in address{};
address.sin_family = AF_INET;
address.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
address.sin_port = 0;
assert(bind(owner, reinterpret_cast<sockaddr *>(&address), sizeof(address)) == 0);
int address_size = sizeof(address);
assert(getsockname(owner, reinterpret_cast<sockaddr *>(&address), &address_size) == 0);
const SOCKET contender = muninn_udp_socket::create(true);
assert(contender != INVALID_SOCKET);
assert(bind(contender, reinterpret_cast<sockaddr *>(&address), sizeof(address)) != 0);
assert(WSAGetLastError() == WSAEADDRINUSE || WSAGetLastError() == WSAEACCES);
closesocket(contender);
closesocket(owner);
WSACleanup();
return 0;
}

View file

@ -1,111 +0,0 @@
#include "muninn_video_fec.h"
#include <chrono>
#include <cstdlib>
#include <cstdint>
#include <map>
#include <vector>
void require(bool condition)
{
if (!condition)
std::abort();
}
int main()
{
constexpr uint16_t data_count = 64;
constexpr uint16_t parity_count = 16;
constexpr size_t shard_bytes = 1400;
std::vector<std::vector<uint8_t>> original(data_count, std::vector<uint8_t>(shard_bytes));
for (uint16_t data = 0; data < data_count; ++data) {
for (size_t offset = 0; offset < shard_bytes; ++offset)
original[data][offset] = static_cast<uint8_t>((data * 29 + offset * 17) & 0xff);
}
std::map<uint16_t, std::vector<uint8_t>> parity;
for (uint16_t row = 0; row < parity_count; ++row) {
auto &shard = parity[row];
shard.assign(shard_bytes, 0);
for (uint16_t data = 0; data < data_count; ++data) {
const uint8_t coefficient = muninn_video_fec::coefficient(row, parity_count, data);
for (size_t offset = 0; offset < shard_bytes; ++offset)
shard[offset] ^= muninn_video_fec::multiply(original[data][offset], coefficient);
}
}
const std::vector<uint32_t> lengths(data_count, shard_bytes);
for (uint16_t burst = 1; burst <= parity_count; ++burst) {
auto damaged = original;
for (uint16_t index = 0; index < burst; ++index)
damaged[20 + index].clear();
const auto started = std::chrono::steady_clock::now();
require(muninn_video_fec::recover(damaged, lengths, parity_count, parity));
require(std::chrono::steady_clock::now() - started < std::chrono::milliseconds(50));
require(damaged == original);
}
auto excessive = original;
for (uint16_t index = 0; index <= parity_count; ++index)
excessive[index].clear();
require(!muninn_video_fec::recover(excessive, lengths, parity_count, parity));
constexpr uint16_t block_data = 8;
constexpr uint16_t frame_data = 80;
constexpr uint16_t block_count = frame_data / block_data;
constexpr size_t block_shard_bytes = 96;
std::vector<std::vector<uint8_t>> frame(frame_data, std::vector<uint8_t>(block_shard_bytes));
for (uint16_t data = 0; data < frame_data; ++data) {
for (size_t offset = 0; offset < block_shard_bytes; ++offset)
frame[data][offset] = static_cast<uint8_t>((data * 31 + offset * 13) & 0xff);
}
std::vector<std::map<uint16_t, std::vector<uint8_t>>> block_parity(block_count);
for (uint16_t block = 0; block < block_count; ++block) {
for (uint16_t row = 0; row < block_data; ++row) {
auto &shard = block_parity[block][row];
shard.assign(block_shard_bytes, 0);
for (uint16_t local = 0; local < block_data; ++local) {
const auto &data = frame[block * block_data + local];
const uint8_t coefficient = muninn_video_fec::coefficient(row, block_data, local);
for (size_t offset = 0; offset < block_shard_bytes; ++offset)
shard[offset] ^= muninn_video_fec::multiply(data[offset], coefficient);
}
}
}
struct WireShard {
bool parity;
uint16_t block;
uint16_t local;
};
std::vector<WireShard> wire;
for (uint16_t local = 0; local < block_data; ++local) {
for (uint16_t block = 0; block < block_count; ++block)
wire.push_back({false, block, local});
for (uint16_t block = 0; block < block_count; ++block)
wire.push_back({true, block, local});
}
require(wire.size() == frame_data * 2);
const std::vector<uint32_t> block_lengths(block_data, block_shard_bytes);
for (size_t burst = 1; burst <= 54; ++burst) {
for (size_t start = 0; start + burst <= wire.size(); ++start) {
auto damaged = frame;
auto available_parity = block_parity;
for (size_t index = start; index < start + burst; ++index) {
const auto shard = wire[index];
if (shard.parity)
available_parity[shard.block].erase(shard.local);
else
damaged[shard.block * block_data + shard.local].clear();
}
const auto started = std::chrono::steady_clock::now();
for (uint16_t block = 0; block < block_count; ++block) {
require(muninn_video_fec::recover_block(
damaged,
block * block_data,
block_lengths,
block_data,
available_parity[block]));
}
require(std::chrono::steady_clock::now() - started < std::chrono::milliseconds(50));
require(damaged == frame);
}
}
return 0;
}

View file

@ -125,6 +125,7 @@ struct CaptureState
std::vector<unsigned long long> sweepSamples;
std::vector<float>* playbackSamples = nullptr;
double playbackGain = 1.0;
long playbackOutputChannel = -1;
std::vector<float>* recordedInput = nullptr;
unsigned long long recordFrames = 0;
unsigned long long sweepFrame = 0;
@ -320,6 +321,11 @@ void processOutput(long doubleBufferIndex)
for (size_t channel = 0; channel < g_capture.outputBuffers->size(); ++channel)
{
if (g_capture.playbackOutputChannel >= 0 && static_cast<long>(channel) != g_capture.playbackOutputChannel)
{
continue;
}
auto& buffer = (*g_capture.outputBuffers)[channel];
auto& info = (*g_capture.outputChannels)[channel];
auto* raw = static_cast<unsigned char*>(buffer.buffers[doubleBufferIndex]);
@ -498,6 +504,12 @@ double doubleOption(int argc, char** argv, const char* name, double fallback)
return raw ? std::atof(raw) : fallback;
}
long longOption(int argc, char** argv, const char* name, long fallback)
{
const char* raw = option(argc, argv, name, nullptr);
return raw ? std::atol(raw) : fallback;
}
bool hasArg(int argc, char** argv, const char* name)
{
for (int index = 1; index < argc; ++index)
@ -576,6 +588,7 @@ int main(int argc, char** argv)
const char* playFloat32MonoPath = option(argc, argv, "--play-f32-mono", nullptr);
const char* recordFloat32InterleavedPath = option(argc, argv, "--record-f32-interleaved", nullptr);
const auto playGain = doubleOption(argc, argv, "--play-gain", 1.0);
const auto playOutputChannel = longOption(argc, argv, "--play-output-channel", -1);
const auto sweepGain = doubleOption(argc, argv, "--sweep-gain", 0.03);
const auto sweepToneSeconds = doubleOption(argc, argv, "--sweep-tone-seconds", 0.55);
const auto sweepGapSeconds = doubleOption(argc, argv, "--sweep-gap-seconds", 0.20);
@ -731,7 +744,7 @@ int main(int argc, char** argv)
playbackSamples.resize(static_cast<size_t>(byteCount) / sizeof(float));
input.read(reinterpret_cast<char*>(playbackSamples.data()), static_cast<std::streamsize>(playbackSamples.size() * sizeof(float)));
}
std::printf("asio playback-f32-mono: path=%s samples=%zu gain=%.5f\n", playFloat32MonoPath, playbackSamples.size(), playGain);
std::printf("asio playback-f32-mono: path=%s samples=%zu gain=%.5f outputChannel=%ld\n", playFloat32MonoPath, playbackSamples.size(), playGain, playOutputChannel);
}
if (captureSeconds > 0.0 || monitorSweep || !playbackSamples.empty() || emitJsonBlocks)
@ -820,6 +833,7 @@ int main(int argc, char** argv)
g_capture.sweepGain = sweepGain;
g_capture.playbackSamples = playbackSamples.empty() ? nullptr : &playbackSamples;
g_capture.playbackGain = playGain;
g_capture.playbackOutputChannel = playOutputChannel;
g_capture.emitJsonBlocks = emitJsonBlocks;
g_capture.sweepFrame = 0;
g_capture.sweepEnergy.assign(sweepFrequencies.size() * static_cast<size_t>(inputs), 0.0);

View file

@ -17,8 +17,7 @@ enum LocalcastSampleKind {
LOCALCAST_SAMPLE_AUDIO_BLOCK = 5,
LOCALCAST_SAMPLE_PHASE_CLAIM = 6,
LOCALCAST_SAMPLE_EVENT_CLAIM = 7,
LOCALCAST_SAMPLE_RENDER_PACKET = 8,
LOCALCAST_SAMPLE_MOVE_EVIDENCE = 9
LOCALCAST_SAMPLE_RENDER_PACKET = 8
};
typedef struct LocalcastReservoir LocalcastReservoir;
@ -80,47 +79,6 @@ typedef struct LocalcastRenderPoint {
float confidence;
} LocalcastRenderPoint;
enum LocalcastMoveEvidenceKind {
LOCALCAST_MOVE_EVIDENCE_OPTICAL_MARKER = 1,
LOCALCAST_MOVE_EVIDENCE_CONTROLLER_STATE = 2
};
typedef struct LocalcastMoveEvidenceSample {
uint64_t witness_id_hash;
uint64_t controller_id_hash;
uint64_t source_timestamp_ns;
uint64_t arrival_ns;
uint64_t sequence;
uint32_t evidence_kind;
uint32_t flags;
float image_x;
float image_y;
float radius_px;
float confidence;
float accel_x;
float accel_y;
float accel_z;
float gyro_x;
float gyro_y;
float gyro_z;
float trigger;
uint32_t buttons_mask;
uint32_t reserved;
float battery01;
uint32_t reserved1;
uint32_t reserved2;
} LocalcastMoveEvidenceSample;
typedef struct LocalcastMoveEvidenceBufferDescriptor {
uint64_t sample_buffer_handle;
uint32_t sample_count;
uint32_t sample_stride_bytes;
uint64_t source_time_min_ns;
uint64_t source_time_max_ns;
uint64_t calibration_hash;
uint64_t tracking_space_hash;
} LocalcastMoveEvidenceBufferDescriptor;
typedef struct LocalcastRuntimeStatus {
uint64_t edge_ns;
uint64_t window_start_ns;
@ -134,7 +92,6 @@ typedef struct LocalcastRuntimeStatus {
size_t phase_claim_count;
size_t event_claim_count;
size_t render_packet_count;
size_t move_evidence_count;
} LocalcastRuntimeStatus;
uint64_t localcast_hash_source_id(const uint8_t *bytes, size_t byte_len);
@ -187,7 +144,6 @@ bool localcast_runtime_push_audio_block(LocalcastRuntime *runtime, LocalcastSamp
bool localcast_runtime_push_phase_claim(LocalcastRuntime *runtime, LocalcastSampleHandle sample);
bool localcast_runtime_push_event_claim(LocalcastRuntime *runtime, LocalcastSampleHandle sample);
bool localcast_runtime_push_render_packet(LocalcastRuntime *runtime, LocalcastSampleHandle sample);
bool localcast_runtime_push_move_evidence(LocalcastRuntime *runtime, LocalcastSampleHandle sample);
bool localcast_runtime_status(const LocalcastRuntime *runtime, LocalcastRuntimeStatus *out_status);
size_t localcast_runtime_len(const LocalcastRuntime *runtime);
size_t localcast_runtime_view_len(const LocalcastRuntime *runtime, uint32_t sample_kind);
@ -254,15 +210,6 @@ bool localcast_producer_push_render_packet(
LocalcastSampleHandle *out_sample
);
bool localcast_producer_push_move_evidence_buffer(
LocalcastProducer *producer,
LocalcastRuntime *runtime,
uint64_t timestamp_ns,
uint64_t arrival_ns,
const LocalcastMoveEvidenceBufferDescriptor *descriptor,
LocalcastSampleHandle *out_sample
);
#ifdef __cplusplus
}
#endif

View file

@ -17,7 +17,6 @@ pub enum SampleKind {
PhaseClaim,
EventClaim,
RenderPacket,
MoveEvidence,
}
impl SampleKind {
@ -32,7 +31,6 @@ impl SampleKind {
6 => Some(Self::PhaseClaim),
7 => Some(Self::EventClaim),
8 => Some(Self::RenderPacket),
9 => Some(Self::MoveEvidence),
_ => None,
}
}
@ -48,7 +46,6 @@ impl SampleKind {
Self::PhaseClaim => 6,
Self::EventClaim => 7,
Self::RenderPacket => 8,
Self::MoveEvidence => 9,
}
}
}
@ -232,53 +229,6 @@ pub struct LocalcastRenderPoint {
pub confidence: f32,
}
#[repr(C)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum LocalcastMoveEvidenceKind {
OpticalMarker = 1,
ControllerState = 2,
}
#[repr(C)]
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct LocalcastMoveEvidenceSample {
pub witness_id_hash: u64,
pub controller_id_hash: u64,
pub source_timestamp_ns: u64,
pub arrival_ns: u64,
pub sequence: u64,
pub evidence_kind: u32,
pub flags: u32,
pub image_x: f32,
pub image_y: f32,
pub radius_px: f32,
pub confidence: f32,
pub accel_x: f32,
pub accel_y: f32,
pub accel_z: f32,
pub gyro_x: f32,
pub gyro_y: f32,
pub gyro_z: f32,
pub trigger: f32,
pub buttons_mask: u32,
pub reserved: u32,
pub battery01: f32,
pub reserved1: u32,
pub reserved2: u32,
}
#[repr(C)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct LocalcastMoveEvidenceBufferDescriptor {
pub sample_buffer_handle: u64,
pub sample_count: u32,
pub sample_stride_bytes: u32,
pub source_time_min_ns: u64,
pub source_time_max_ns: u64,
pub calibration_hash: u64,
pub tracking_space_hash: u64,
}
pub struct LocalcastReservoir {
inner: RollingReservoir<LocalcastSampleHandle>,
}
@ -308,7 +258,6 @@ pub struct LocalcastRuntimeStatus {
pub phase_claim_count: usize,
pub event_claim_count: usize,
pub render_packet_count: usize,
pub move_evidence_count: usize,
}
impl Default for LocalcastRuntimeStatus {
@ -326,7 +275,6 @@ impl Default for LocalcastRuntimeStatus {
phase_claim_count: 0,
event_claim_count: 0,
render_packet_count: 0,
move_evidence_count: 0,
}
}
}
@ -360,7 +308,6 @@ impl LocalcastRuntime {
phase_claim_count: self.view_len(SampleKind::PhaseClaim),
event_claim_count: self.view_len(SampleKind::EventClaim),
render_packet_count: self.view_len(SampleKind::RenderPacket),
move_evidence_count: self.view_len(SampleKind::MoveEvidence),
}
}
}
@ -676,14 +623,6 @@ pub extern "C" fn localcast_runtime_push_render_packet(
localcast_runtime_push_typed(ptr, SampleKind::RenderPacket, sample)
}
#[unsafe(no_mangle)]
pub extern "C" fn localcast_runtime_push_move_evidence(
ptr: *mut LocalcastRuntime,
sample: LocalcastSampleHandle,
) -> bool {
localcast_runtime_push_typed(ptr, SampleKind::MoveEvidence, sample)
}
#[unsafe(no_mangle)]
pub extern "C" fn localcast_runtime_status(
ptr: *const LocalcastRuntime,
@ -921,46 +860,6 @@ pub extern "C" fn localcast_producer_push_render_packet(
)
}
#[unsafe(no_mangle)]
pub extern "C" fn localcast_producer_push_move_evidence_buffer(
producer_ptr: *mut LocalcastProducer,
runtime_ptr: *mut LocalcastRuntime,
timestamp_ns: u64,
arrival_ns: u64,
descriptor: *const LocalcastMoveEvidenceBufferDescriptor,
out_sample: *mut LocalcastSampleHandle,
) -> bool {
let Some(producer) = (unsafe { producer_ptr.as_mut() }) else {
return false;
};
if producer.kind != SampleKind::MoveEvidence {
return false;
}
let Some(descriptor_ref) = (unsafe { descriptor.as_ref() }) else {
return false;
};
if descriptor_ref.sample_buffer_handle == 0 {
return false;
}
if descriptor_ref.sample_count == 0 {
return false;
}
if descriptor_ref.sample_stride_bytes as usize
!= std::mem::size_of::<LocalcastMoveEvidenceSample>()
{
return false;
}
let payload_handle = descriptor as u64;
localcast_producer_push(
producer_ptr,
runtime_ptr,
timestamp_ns,
arrival_ns,
payload_handle,
out_sample,
)
}
fn localcast_runtime_push_typed(
ptr: *mut LocalcastRuntime,
kind: SampleKind,
@ -1044,51 +943,6 @@ mod tests {
}
}
fn move_evidence_descriptor(
sample_buffer_handle: u64,
) -> LocalcastMoveEvidenceBufferDescriptor {
LocalcastMoveEvidenceBufferDescriptor {
sample_buffer_handle,
sample_count: 2,
sample_stride_bytes: std::mem::size_of::<LocalcastMoveEvidenceSample>() as u32,
source_time_min_ns: 1_000,
source_time_max_ns: 1_120,
calibration_hash: 0xCA11_BA7E,
tracking_space_hash: 0x57A6E,
}
}
fn move_marker_sample(
witness_id_hash: u64,
controller_id_hash: u64,
) -> LocalcastMoveEvidenceSample {
LocalcastMoveEvidenceSample {
witness_id_hash,
controller_id_hash,
source_timestamp_ns: 1_000,
arrival_ns: 1_010,
sequence: 7,
evidence_kind: LocalcastMoveEvidenceKind::OpticalMarker as u32,
flags: 0,
image_x: 123.5,
image_y: 87.25,
radius_px: 14.0,
confidence: 0.92,
accel_x: 0.0,
accel_y: 0.0,
accel_z: 0.0,
gyro_x: 0.0,
gyro_y: 0.0,
gyro_z: 0.0,
trigger: 0.0,
buttons_mask: 0,
reserved: 0,
battery01: f32::NAN,
reserved1: 0,
reserved2: 0,
}
}
fn render_point(stable_key_hash: u64) -> LocalcastRenderPoint {
LocalcastRenderPoint {
stable_key_hash,
@ -1731,125 +1585,6 @@ mod tests {
localcast_runtime_destroy(runtime);
}
#[test]
fn move_evidence_layout_is_fixed_for_compute_upload() {
let sample = move_marker_sample(11, 22);
assert_eq!(std::mem::size_of::<LocalcastMoveEvidenceSample>(), 112);
assert_eq!(
std::mem::size_of::<LocalcastMoveEvidenceBufferDescriptor>(),
48
);
assert_eq!(11, sample.witness_id_hash);
assert_eq!(22, sample.controller_id_hash);
assert_eq!(
LocalcastMoveEvidenceKind::OpticalMarker as u32,
sample.evidence_kind
);
assert_eq!(123.5, sample.image_x);
assert_eq!(14.0, sample.radius_px);
}
#[test]
fn move_evidence_descriptor_enters_native_runtime_window() {
let runtime = localcast_runtime_create(DEFAULT_RESERVOIR_NS);
let producer = localcast_producer_create(SampleKind::MoveEvidence.id(), 101, 3);
let descriptor = move_evidence_descriptor(9_000);
let mut sample = handle(0, 0, 0, 0, 0);
assert!(localcast_producer_push_move_evidence_buffer(
producer,
runtime,
1_200,
1_220,
&descriptor,
&mut sample,
));
let mut status = LocalcastRuntimeStatus::default();
assert!(localcast_runtime_status(runtime, &mut status));
assert_eq!(1, status.total_sample_count);
assert_eq!(1, status.move_evidence_count);
assert_eq!(0, status.camera_feature_count);
assert_eq!(
(&descriptor as *const LocalcastMoveEvidenceBufferDescriptor) as u64,
sample.payload_handle
);
assert_eq!(
1,
localcast_runtime_view_len(runtime, SampleKind::MoveEvidence.id())
);
localcast_producer_destroy(producer);
localcast_runtime_destroy(runtime);
}
#[test]
fn move_evidence_descriptor_push_rejects_wrong_kind_and_bad_buffers() {
let runtime = localcast_runtime_create(DEFAULT_RESERVOIR_NS);
let render_producer = localcast_producer_create(SampleKind::RenderPacket.id(), 88, 12);
let move_producer = localcast_producer_create(SampleKind::MoveEvidence.id(), 89, 0);
let good = move_evidence_descriptor(9_000);
let zero_handle = move_evidence_descriptor(0);
let zero_count = LocalcastMoveEvidenceBufferDescriptor {
sample_count: 0,
..good
};
let bad_stride = LocalcastMoveEvidenceBufferDescriptor {
sample_stride_bytes: 4,
..good
};
assert!(!localcast_producer_push_move_evidence_buffer(
render_producer,
runtime,
2_000,
2_010,
&good,
std::ptr::null_mut(),
));
assert!(!localcast_producer_push_move_evidence_buffer(
move_producer,
runtime,
2_000,
2_010,
std::ptr::null(),
std::ptr::null_mut(),
));
assert!(!localcast_producer_push_move_evidence_buffer(
move_producer,
runtime,
2_000,
2_010,
&zero_handle,
std::ptr::null_mut(),
));
assert!(!localcast_producer_push_move_evidence_buffer(
move_producer,
runtime,
2_000,
2_010,
&zero_count,
std::ptr::null_mut(),
));
assert!(!localcast_producer_push_move_evidence_buffer(
move_producer,
runtime,
2_000,
2_010,
&bad_stride,
std::ptr::null_mut(),
));
assert_eq!(
0,
localcast_runtime_view_len(runtime, SampleKind::MoveEvidence.id())
);
localcast_producer_destroy(render_producer);
localcast_producer_destroy(move_producer);
localcast_runtime_destroy(runtime);
}
#[test]
fn runtime_spine_rejects_nulls() {
let sample = handle(1, 2, 3, 4, 5);

File diff suppressed because it is too large Load diff

View file

@ -17,27 +17,11 @@ Get-Content .\state\evidence.jsonl -Tail 8
## Current Shape
- Public brand/Face is Mimir.
- Mimir's Face doctrine lives in [[docs/mimir-face|Mimir Face]].
- Public brand/Persona is Mimir.
- Mimir's Persona doctrine lives in [[docs/mimir-face|Mimir Persona]].
- VoidBot persona/state live under `.voidbot/voice/` and `.voidbot/state/`.
Commit `.voidbot` dirt in a separate state commit whenever it appears.
- V1 still has FFmpeg/SRT/OBS bridge utilities for LAN compatibility ingest.
- V1 now also has an explicit CultMesh media bridge for Raven program feeds:
`src/Mimir.CultMeshMedia` sends rolling `mimir.cultmesh_media_frame`
documents over CultNet reliable UDP through Yggdrasil and lowers them on
Starfire to local MPEG-TS UDP for OBS. The bridge now uses explicit CultLib
RUDP helpers/session plumbing; the old
`CultMesh.StartNodeAsync`/`CultMesh.ConnectClient` path no longer owns this
media lane.
- CultLib has moved the daemon-swarm direction to CultNet over RUDP across
runtimes. Mimir's typed daemon truth should default to
`cultnet.transport.rudp.v0`; TCP/HTTP/WebSocket are client lowerings,
compatibility probes, or migration debt unless a specific runtime has not
earned RUDP yet.
- Mimir's old Eve dashboard broker is archived and no longer publishes Idunn
health, `/eve/deck`, `/eve/dashboard`, or WebSocket commands. The browser
reference is a client lowering, not service truth. Dashboard state must return
as typed CultMesh/Eve documents through Odin before deployment resumes.
- V1 still has FFmpeg/SRT/OBS bridge utilities for LAN ingest.
- The live stream app is C# plus Fensalir: `Mimir.slnx` contains
`src/Mimir.App` and `src/Mimir.Runtime`.
- `src/Mimir.App` hosts Fensalir as the windowing/rendering/D3D12 bridge.
@ -53,42 +37,205 @@ Get-Content .\state\evidence.jsonl -Tail 8
dictionary, low-level implementation notes, and sample code sketches for the
next decoder/DSP/native-ring/Fensalir implementation passes. Start with
[[research/perfect-machine-study-2026-05-23/reading-guide|Reading Guide]].
- `docs/mimir-program-composition.md` is the current program-output authority
map: Muninn observes on Starfire/Nightwing/Raven-class hosts, Mimir consumes
selected streams and owns composition, Eve lowers controls/previews/stats,
Yggdrasil publishes to the site, and OBS is compatibility-only.
- `src/Mimir.Runtime/Synchronization` now has typed program contracts for
`mimir.program_scene.v1`, `mimir.program_output.v1`, and
`mimir.eve_operator_surface.v1`; `Mimir.BufferSmoke
--perfect-machine-contract-smoke` writes them into CultCache.
- `Mimir.BufferSmoke --import-obs-program-scene --output
state/mimir-program-composition.cc` imports the current OBS scene JSON into
Mimir-owned typed state. The latest imported scene has eight layers, six
visible, two cropped, one chroma-keyed, and Raven mapped as
`muninn:raven:monitor:primary`.
- Local six-camera ingest should use direct driver adapters. Process-backed
sources are bridge/network edges only.
- Leap stereo IR is the first timing-camera candidate for direct ingest.
- `native/reservoir` owns the lower shared-edge typed-handle invariant for
Fensalir/Faust binding work.
- Fensalir owns production GPU fusion, UI, and Spout2 publication.
- `MimirSceneEditorState` is currently a 2D OBS-shaped program mixer owner, not
a camera/3D scene editor. Synced video buffers become program layers with
normalized placement; the Eve/CultUI surface is Video Sources on the left, a
chrome-free program canvas in the center, Audio Mixer docked underneath that
canvas, and a Unity-style Inspector on the right for the selected layer's
properties. The center preview requests Fensalir's current output-buffer blit
as its background and overlays Mimir's normalized layer placements on top.
Placement commits snap near frame edges/center and preserve source aspect
while resizing through direct preview handles, sliders, or the scale gizmo.
Explicit source/transform controls are inspector-only and only visible when a
program source is selected.
Hover/active handle state, snap guides, empty-canvas deselect, keyboard
nudging, PageUp/PageDown layer ordering, and selected-source
contain/cover/stretch fit mode are wired through `MimirSceneEditorState`.
Camera/SDF/model editor affordances have no live UI authority, and editor
camera/SDF/spline output no longer contributes to the runtime frame. Smoke
with `--scene-editor-smoke`.
- The Audio Mixer dock is a horizontal CultUI channel-strip bank, not a
selected-stem inspector form. Six fixed slots read the live
`presentationControls.AudioFeeds` order so Scarlett ASIO mono inputs,
stereo loopback, and Raven loopback can appear as parallel strips with M/S,
gain, and compact sync/chirp-response readouts derived from
`MimirSynchronizationHub.AudioSynchronizationStates`. Do not move sample
alignment, chirp response mapping, or ambient/noise neutralization authority
into the UI; the surface only controls and observes runtime audio state.
- Program surface placement is seeded from CultCache config
`config/mimir-program-surface.cc` unless `--cache`,
`AQUARIUM_CULTCACHE_PATH`, or `MIMIR_PROGRAM_SURFACE_CONFIG` points
elsewhere. The default `mimir.program.default` document puts
`raven-display` full-frame at layer 0 and `kiyo-pro-rgb` as a top-left
picture-in-picture at layer 1. `MimirSceneEditorState` applies matching
source placements once when those feeds appear, then inspector/drag edits own
subsequent placement.
- The left Video Sources rail is a direct source selector, not a previous/next
navigator. It renders visible rows for live video feeds using CultUI
row-level visibility, with a Select action and placement summary per source.
Keep configured/offline source inventory as a future source-state owner;
the current rail lists live synced video feeds.
- Live camera image buffers should use Fensalir-owned texture leases when they
are rendering inputs: Mimir asks the engine broker for a keyed D3D12
Texture2D/fence lease, writes decoded frames into that texture, commits the
producer fence value, and lowers the same resource key through FieldEvidence.
`MimirVideoCaptureDriverSource` forwards that lease client to direct drivers
that implement `IMimirFensalirTextureLeaseReceiver`.
Camera backends must use the closest-to-device path available and report
unavoidable copies; process or managed convenience layers are diagnostics.
Raw single-plane CPU-origin frames can use the explicit broker upload lane,
which increments `UnavoidableCopyCount` and clears live managed payload bytes
after upload. NV12 CPU upload is rejected until Fensalir owns planar copy;
device/GPU NV12 producers should write the leased texture directly.
The first direct KS/UVC driver is `MimirKsVideoCaptureDriver` backed by
`native/camera_capture/mimir_camera_capture.dll`; the example config
`config/mimir-runtime.ks-camera.example.json` targets LeapUVC 640x240 YUY2.
The same native folder also has `mimir_ps3eye_capture.dll` and
`MimirPs3EyeVideoCaptureDriver` for PS3 Eye Bayer8 frames through WinUSB/libusb.
Fresh driver smokes pulled and uploaded real frames for LeapUVC 640x240 YUY2,
both PS3 Eyes at 320x240 Bayer8, Kiyo Pro 1920x1080 YUY2, and regular Kiyo
640x480 YUY2. Regular Kiyo 1280x720 YUY2 did not open.
Compressed Kiyo Pro MJPG/H264 now have a first GPU-resident path:
`MimirMediaFoundationGpuVideoCaptureDriver` uses
`native/camera_capture/mimir_mf_gpu_capture.dll` to decode through Media
Foundation with a D3D11 device manager, copy into a shared BGRA D3D11 texture,
and publish `shared-d3d11-texture` with zero live CPU payload bytes.
1920x1080 MJPG->RGB32 and H264->RGB32 smokes passed. Direct shared NV12
texture creation failed, so NV12 remains future plane-aware interop or
GPU bridge-copy work.
Foreign shared texture handles are import edges, not the primary hot path.
- The current Mimir/Fensalir rendering teardown map is
[[docs/fensalir-rendering-rebuild-migration|Fensalir Rendering Rebuild
Migration]]. Mimir's production visual output path must publish typed
physical observations, calibration constraints, and surface intent; Fensalir
turns those into field claims, selected lowerings, reusable evidence,
temporal guide lanes, and presentation.
- Fast depth provenance now lives at
[[research/d3d12-stereo-depth-provenance-2026-05-29|D3D12 Stereo Depth
Provenance]]. `libSGM` is the current permissive north-star reference for a
future Fensalir-owned D3D12 stereo SGM compute lane. Do not import CUDA or
TensorRT as live depth authority.
The first code contract is in `MimirStereoDepthConfigurations` and
`MimirFensalirFieldLowering`. Run
`dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --stereo-depth-contract-smoke`
to verify caller-declared rectified left/right input textures, or
`dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --leap-packed-stereo-depth-smoke`
to verify live `LeapStereoIr` packed R8G8 lowering. The Leap path reuses the
same declared texture as left/right resource identity and treats R/G as the
packed observation lanes. Fensalir now has a first packed-Leap D3D12
SAD/block-match disparity writer, not full libSGM-style SGM or calibrated
metric depth. Mimir also now declares the first Leap point-cloud root:
`MimirPointCloudConfigurations.LeapDisparityPointCloudRoot` and
`BuildLeapPackedStereoPointCloudCandidateFrame` emit a GPU-resident
`FieldMesh` point-list resource derived from that disparity SurfacePage. Run
`dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --leap-point-cloud-root-smoke`
to verify the combined `SurfacePage` plus `Mesh` packet plan. This is a
typed socket on the Mimir side. Fensalir branch `codex/leap-packed-depth` now
has the first matching render lane: generated point-list Mesh buffers are
filled from the disparity SurfacePage on D3D12 and rendered through the scene
candidate targets. The next cut is live Leap/editor verification plus
calibrated projection constants and global residual/calibration ownership.
- Fensalir also owns the EVE-facing dashboard pixels through
`Global\MimirFensalirProgramTexture`. `src/Mimir.EveRelay` opens that shared
texture, encodes H.264 Annex-B with NVENC by default, and serves EveCanvas
over `/stream`; EveCanvas decodes with `AVSampleBufferDisplayLayer`. The
launch script uses an SSH reverse tunnel to Eve when Windows firewall blocks
direct inbound TCP.
- `src/Mimir.Broadcast` is the first self-hosted public sender. Starfire pushes
one final H.264/AAC NVENC RTMP stream to
`rtmp://127.0.0.1:11935/live/mimir` through an SSH local forward to
Yggdrasil. Yggdrasil's nginx RTMP/HLS origin is deployed and has produced
HLS segments from a bounded synthetic push. Public HLS is served through the
existing StreamPixels TLS host, not a new Mimir media hostname. The public
static viewer belongs to the root GameCult site at
`https://gamecult.org/livestream` and reads
`https://streampixels.gamecult.org/mimir/live/hls/mimir.m3u8`.
- Raven/Eve ingest now has a network-pixel receiver config:
`config/mimir-runtime.raven-eve.example.json` starts an FFmpeg-backed
raw-video listener for `raven-display` on SRT port `5200` and Eve sensor
frame-event receivers for `eve-camera` on WebSocket port `8793` and `eve-mic`
on port `8794`. `scripts/start-raven-screen-capture-sender.ps1` launches
Raven desktop capture with `gdigrab` plus NVENC. EveCanvas owns Eve camera/mic
capture directly through AVFoundation/AVAudioEngine; no external iOS camera
app owns that path.
- `src/Mimir.RavenDaemon` is the Raven-local capture mux owner for the
stream-proof path. It supervises display capture plus Realtek render loopback
through the existing FFmpeg/WASAPI pipeline, publishes
`mimir.raven_capture_mux_state.v1` through CultMesh/CultCache, and exposes
Eve dashboard state on port `8801`. `scripts/start-raven-daemon.ps1` defaults
to Starfire over WireGuard at `10.77.0.2:5200`. The older Raven sender scripts
remain diagnostics, not canonical long-running status owners.
- Faust/native DSP owns hot audio alignment, separation, spatialization, and
synchronized stems.
- OBS compatibility adapters may receive final program surfaces; they do not
own synchronization, composition, preview/control, or publication.
- The current listenable dual-mic receipt is the Faust-chain bridge, not the
older C# composite. `scripts/record-kiyo-dual-mic-composite.ps1` records Kiyo
plus Scarlett ASIO, then `--asio-dual-mic-faust-cleaner-wav` compiles/runs
`faust/mimir_alignment_actuator.dsp` before
`faust/mimir_dual_mic_dialogue_cleaner.dsp` and muxes the dialogue stem into
`artifacts/runtime/kiyo-dual-mic-composite/kiyo-dual-mic-composite-latest.mp4`.
Fresh run `20260601-051044` measured cardioid delay 359.398 samples
(7487.452 us) at 48 kHz and applied holdback 359.398/0.000 to
shotgun/cardioid before cleanup.
- The edge-to-edge Mimir editor shell now uses headerless retained dashboard
panels. The inspector expands vertically; the right rail `Output` panel is a
fixed-height OBS-like control block below it. Eve is the portable dashboard
authority; the C# CultUI-shaped Aquarium contract is only the reference API
shape, and Fensalir Direct2D/D3D12 are native lowerings of the same document.
Elements can now carry typed CultCache/CultMesh bindings with document
schema/id, field path, value kind, access, authority, and optional command id;
native callbacks remain local lowering actuators, not portable state owners.
`MimirProgramRecorder` owns `Record` / `Stop`, status text, and timestamped
receipts under `C:\Users\Meta\Videos\Mimir` by default
(`MIMIR_RECORD_OUTPUT_DIR` overrides). It delegates to
`scripts/record-stream-proof-output.ps1`, so the app records the same Raven +
Kiyo PIP + Starfire Scarlett NVENC/AAC composite intended for
Yggdrasil/public judgment. `MIMIR_RECORD_START_RAVEN=1` asks it to launch the
Raven sender too. `MimirProgramStreamer` owns `Live` / `Stop` for
`Mimir.Broadcast`; it requires `MIMIR_STREAM_INPUT` unless
`MIMIR_STREAM_ALLOW_TEST_PATTERN=1` explicitly allows a synthetic smoke.
Native program-output recording/streaming should replace those actuators
behind the same owner.
- OBS receives final program surfaces; it does not own synchronization.
- The online Well stack exists. `Mimir.EveSensorReceiver` has a subscriber
WebSocket path beside its observation ingest path, and `Mimir.VerseRecorder`
writes subscribed observations to
`C:\Users\Meta\Videos\Mimir\VerseCaptures\<session>\session.json` plus
`observations.jsonl`. Recorder now also recognizes
`mimir.well_capture_page.v1`, strips inline base64 sample bodies into
`bodies/page-*.bin`, and records `mimir.recorder_body_index.v1` refs in
`bodies/index.jsonl` so offline reconstruction tools can resolve payloads
without treating JSONL as the body store. `Mimir.Well` loads
`config/mimir-runtime.well.local.json`, starts every configured source it
can, polls `MimirSynchronizationHub` with a bounded per-source budget, updates
audio sync/probe state, and publishes `mimir.well_snapshot.v1` records with
source buffers, composite controls, publication/stem contract, and sync
status. It also publishes bounded `mimir.well_capture_page.v1` records for
synchronized frame slices with timing metadata, configured composite state,
and inline source bodies when CPU bytes exist and fit the configured body
limit. GPU-handle-only samples remain metadata/body-handle evidence until
Fensalir/native program-output recording owns rendered composite bodies.
`scripts/start-online-verse-daemons.ps1` supervises the relay,
recorder, Nightwing typed Eyes/Move/builtin-camera/builtin-mic witness, the
music-synced de-Bruijn/microtonal Move gesture worker, and `Mimir.Well`.
Current live proof `online-verse-daemons-20260601-212502` wrote Well,
Nightwing builtin camera/mic, and Eye/Move records. Raven and Eve lanes are
configured but empty unless their producers are feeding the ports. Sync frame
status is degraded until a camera-clock/global-time correction owner maps
ASIO device time and camera driver time into one canonical timeline.
## Current Pressure
- Nightwing Move tracking is one Muninn provider with two private per-eye
PSMoveAPI worker subprocesses. Eye 0 uses exposure 0.3; Eye 1 uses 0.1.
Both workers calibrate four Moves and advance continuously. Mimir has now
proved all four stable IDs on both eyes in the same frames. The latest
15-second receipt contains 16 observations and eight pair records but only
two unique aggregate frame IDs. Next measure parent publication and RUDP send
counters directly, fix cadence, then collect diverse correspondences for
camera calibration.
- Implement the first concrete Leap direct capture driver and measure cadence.
- Verify the live Leap point cloud in the Mimir editor, then add calibrated
projection constants and the global residual/calibration owner.
- Cut the Phase 1 Mimir-to-Fensalir bridge DTOs for rolling windows,
observations, calibration constraints, and surface intent before adding
another backend-specific visual path.
- Add remaining camera drivers through `IMimirVideoCaptureDriver`.
- Add native audio capture workers for mic, loopback, and network audio feeds.
- Turn the current loopback-referenced chirplet measurement into the real
@ -96,24 +243,8 @@ Get-Content .\state\evidence.jsonl -Tail 8
variable-rate resampling.
- Bind Fensalir UI to stream health, buffer depth, timestamps, settings, and
output management.
- Keep PowerShell/FFmpeg/SRT as bridge utilities until native Mimir program
output, Eve operation, and Yggdrasil site publication replace them.
- Yggdrasil currently runs the CultMesh media relay from
`/opt/gamecult/mimir-cultmesh-media/Mimir.CultMeshMedia`; senders and
receivers target `cultmesh://asgard.yggdrasil.mimir/media/raven-primary-av`
and let the CultMesh resolver choose the concrete RUDP route. Starfire writes
`raven-primary-av` to `udp://127.0.0.1:5200`. That is the
`Mimir.CultMeshMedia` body bridge, not the OBS plugin feed owner.
- The actual Raven OBS/SRT feed owner is Odin's Muninn on Raven. Use
`scripts/start-raven-muninn-obs-feed.ps1` in this repo as the thin wrapper
over `E:\Projects\Odin\scripts\activate-muninn-raven-av-srt.ps1`, which
drives Raven's real `GameCult-Muninn-Activate` hidden task and
`muninn.exe activate` path. `scripts/start-raven-cultmesh-av-sender.ps1`
remains the separate CultMesh bridge/bootstrap lane.
- Rebuild Eve dashboard only as a CultMesh/Odin surface publisher plus renderer
lowering. Provider advertisement, retained state, command boundary, transport
profile, and command documents must be typed CultMesh records; no TCP, HTTP,
or WebSocket surface should become daemon or transport truth.
- Keep PowerShell/FFmpeg/SRT as bridge utilities until native program output
replaces them.
- Kiyo Pro has two UVC extension units. Moving it to a motherboard USB3 port
changed the descriptor path to `root_hub30` / `bcdUSB=0x0320` and exposed
720p/1080p YUY2/MJPG/H264/NV12 at 60 fps, but Windows still reports
@ -198,41 +329,6 @@ Get-Content .\state\evidence.jsonl -Tail 8
## Immediate Re-entry Instruction
The 2026-07-14 live two-wand session at
`artifacts/move-calibration/live-two-wand-20260714-004210.*` passed collection
coverage with 1,168 admitted same-frame pairs. The new Mimir-owned sphere
stereo fitter recovers synthetic 505/535 px focal lengths within 0.43 px and a
34 cm baseline within 0.22 mm, with 0.34/0.63 px held-out median/P95 error.
The live artifact remains rejected: after robust radius admission its consensus
fits at 0.51/2.08 px, but only 53.8 percent of held-out pairs agree and Eye 1's
focal estimate leaves the physical range. PSMoveAPI age is currently projected
as marker confidence by Muninn, so fresh wrong-color matches arrive near 0.996
confidence. Repair source-local observation quality/continuity and calibrate
per-Eye blob-radius response before asking the operator for another sweep. Do
not promote the nominal 22.5 mm diagnostic fit.
Odin `d0eded6` is deployed on Nightwing as binary SHA-256
`a604fedaf90b26a49db6594b16b8b3499e3aed540152c6fda9dd9dbe87d37d39`.
Each Eye worker now rejects stale, sub-2 px, out-of-bounds, and discontinuous
PSMoveAPI positions before aggregation; tracker health publishes rejection
counters, and candidate score is explicitly uncalibrated `0.5` instead of age
masquerading as confidence. A post-deploy 10-second Mimir window received 555
frames, 2,246 observations, and 443 stereo pairs. Radius P01 rose from about
0.707 px to 3.13/3.64 px and confidence P50 is now 0.5 on both Eyes. Eye 1 saw
only two plausible stationary IDs, which is honest and means another sweep is
needed after Mimir models per-Eye blob-radius response.
The active priority is Nightwing Move calibration. One public Muninn observer
owns two private Eye workers and one aggregate evidence stream. Odin commits
`39c839a`, `a340c33`, `8aefd1c`, and `f82ca7a` give daemon health a persistent
serviced RUDP session and move optical aggregation onto its own deadline-clocked
loop. The 2026-07-13 receipt at
`artifacts/move-calibration/muninn-clocked-aggregator.mpack` contains 622 frames,
2,647 observations, and 1,384 cross-camera correspondences in 15 seconds, with
all four Move IDs visible to both Eyes. Next collect spatially diverse samples,
calibrate intrinsics/extrinsics, and publish residual-bearing typed calibration.
Radius is a weighted range cue, not a substitute for camera calibration.
The corrected two-hour study pass ran from
`2026-05-23T23:54:42.9041588+01:00` to at least
`2026-05-24T01:54:56.9747630+01:00`. Use its map first, then implement the

View file

@ -11,11 +11,18 @@ Use it to shape native implementation decisions:
- direct camera and audio drivers;
- pinned/native buffers;
- GPU feature extraction, flow, matching, and fusion;
- GPU feature extraction, flow, matching, depth, and fusion;
- Fensalir D3D12 compute/render paths;
- Faust/native DSP;
- OBS-facing program outputs.
Visual/GPU research entry points:
- [[d3d12-stereo-depth-provenance-2026-05-29|D3D12 Stereo Depth Provenance]]:
libSGM is the current permissive north-star reference for a future
Fensalir-owned D3D12 compute stereo-disparity lane. External CUDA/TensorRT
projects are provenance only, not runtime authority.
Audio timing research entry points:
- [[chirplet-sync-decoder/summary|Chirplet Sync Decoder Summary]]: chirplet

View file

@ -0,0 +1,170 @@
# D3D12 Stereo Depth Provenance
Date: 2026-05-29
## Purpose
This note records the current fast GPU depth-estimation search as provenance for
a future Mimir/Fensalir D3D12 compute implementation.
The north star is `libSGM`: not as a dependency, not as a CUDA bridge, and not
as a permission slip to add an adapter. The useful shape is a fast,
deterministic stereo disparity pipeline whose ownership can be rebuilt inside
Fensalir's D3D12 resource model.
## Authority Map
Owner:
- Fensalir owns the live D3D12 compute kernels, GPU resources, fences,
scheduling, and depth/surface claim lowering.
Inputs:
- Rectified synchronized camera texture pairs from Mimir-declared
`Texture2D` resources.
- Camera calibration: intrinsics, extrinsics, baseline, rectification map, and
valid-time/version metadata.
- Rolling-window timing chosen by `MimirSynchronizedBufferPlanner`.
Outputs:
- GPU-resident disparity/depth resources, preferably fixed format and size per
calibrated camera pair.
- FieldEvidence claims for depth support, confidence, invalid/occluded pixels,
and later surface/material lowerings.
- Diagnostics: kernel timings, disparity range, confidence histogram, invalid
ratio, calibration version, and source texture fence values.
Derived State:
- Depth is derived from synchronized stereo textures plus calibration.
- Per-frame disparity confidence is derived evidence, not a clock or camera
authority.
- Monocular model depth remains optional reference/assistive evidence unless a
later invariant gives it a named role.
Forbidden Writers:
- CUDA runtime code must not become the live path.
- TensorRT/ONNX monocular demos must not own visual geometry.
- CPU image copies/readback must not be inserted for convenience.
- OBS, EVE, or bridge utilities must not decide depth or synchronization.
Shared Paths:
- Direct user capture, replayed rolling-window frames, future Raven/phone
camera feeds, and calibration-session frames must all enter through typed
resources and use the same depth compute/claim path once they are calibrated
stereo pairs.
Deletion Line:
- If a prototype requires persistent CPU depth images, a side transport daemon,
or a CUDA compatibility layer, it is research-only and does not graduate into
the live machine.
## Reference Shape: libSGM
Repository: https://github.com/fixstars/libSGM
License: Apache-2.0.
Why it matters:
- It is a CUDA implementation of Semi-Global Matching for stereo disparity.
- It was built by the Fixstars/adaskit ADAS effort, so it is pointed at the
same autonomous-driving pressure field rather than generic image demos.
- Reported benchmark in the upstream README: `1024 x 440`, disparity `128`,
four SGM paths, subpixel enabled:
- GTX 1080 Ti: `2.0 ms` / `495.1 FPS`
- RTX 3080: `1.5 ms` / `651.3 FPS`
- Tegra X2: `28.5 ms` / `35.1 FPS`
- Xavier MAXN: `9.0 ms` / `110.7 FPS`
Port lesson:
- Use the pipeline and data movement as a model: matching cost, path
aggregation, winner/subpixel selection, confidence/invalid handling.
- Rebuild the kernels in HLSL/D3D12 compute against Fensalir-owned resources.
- Preserve the ADAS-style assumption that calibrated stereo gives metric depth
when baseline and rectification are real.
## Secondary References
Fast-ACVNet: https://github.com/gangweiX/Fast-ACVNet
- License: MIT.
- Learned real-time stereo matching; useful for quality and failure-mode
comparison.
- Upstream reports `45 ms` for Fast-ACVNet+ in its KITTI/Scene Flow table.
- Not the first Mimir implementation target because it brings model training,
weights, and framework deployment complexity.
UniMatch / GMStereo: https://github.com/autonomousvision/unimatch
- License: MIT.
- Unified flow/stereo/depth research code.
- Useful as a modern learned-stereo comparison point and possible later
confidence/feature guide, not a kernel blueprint.
Depth Anything V2 Small: https://github.com/DepthAnything/Depth-Anything-V2
- License caveat: V2 Small is Apache-2.0; Base/Large/Giant are CC-BY-NC-4.0.
- Excellent monocular relative-depth reference, especially through TensorRT
ports, but monocular output is not metric scene truth by itself.
Depth Anything TensorRT CLI:
https://github.com/spacewalk01/depth-anything-tensorrt
- License: MIT.
- Reports fast RTX 4090 inference for Depth-Anything-S/B/L.
- Useful as performance context only. It must not displace calibrated stereo
geometry or Fensalir's D3D12 ownership.
Fast-FoundationStereo:
https://github.com/NVlabs/Fast-FoundationStereo
- Important research reference for real-time zero-shot stereo.
- License is research/non-commercial only, so it is not a permissive live
dependency candidate.
## D3D12 Compute Implementation Sketch
First implementation should stay boring and inspectable:
1. Rectify or require pre-rectified stereo pairs.
2. Convert sampled luma/feature texture into a compact matching domain.
3. Build per-pixel disparity matching costs over a fixed disparity range.
4. Aggregate along a small fixed path set, starting with four directions to
match the `libSGM` reference benchmark shape.
5. Select winner disparity and subpixel refinement.
6. Emit disparity, confidence, and invalid/occlusion masks as GPU resources.
7. Lower those resources into FieldEvidence depth/support claims.
Important D3D12 resource notes:
- Prefer persistent UAV/SRV resources sized by camera pair profile.
- Use explicit producer/consumer fences already established for camera texture
leases and program-output publication.
- Keep disparity range and path count as profile-level cost levers, not
per-frame mode flags scattered through the renderer.
- Do not read back depth except for gated diagnostics and receipts.
## Initial Verification Targets
- Synthetic rectified stereo pair with known constant disparity.
- Slanted plane with known disparity gradient.
- Occlusion edge where invalid/confidence behavior is visible.
- Real Leap stereo IR pair once calibration is pinned.
- Full rolling-window path: camera textures -> depth compute -> FieldEvidence
claim -> Fensalir debug overlay/resource receipt.
Negative checks:
- No CUDA runtime loaded.
- No CPU disparity image in the live path.
- No monocular model output accepted as metric depth without a named scale
authority.
- No untyped texture handle accepted without resource declaration and fence
provenance.

19
run-spectrum-preview.cmd Normal file
View file

@ -0,0 +1,19 @@
@echo off
setlocal
set "ROOT=%~dp0"
set "APP=%ROOT%src\Mimir.App\bin\Debug\net10.0\Mimir.App.exe"
if not exist "%APP%" (
echo Mimir.App.exe was not found.
echo Build it first with:
echo dotnet build "%ROOT%src\Mimir.App\Mimir.App.csproj"
pause
exit /b 1
)
set "MIMIR_SYNTHETIC_SPECTRUM_PREVIEW=1"
set "MIMIR_SPECTRUM_INTERVAL_SECONDS=0.016"
set "MIMIR_SPECTRUM_SOURCE_LANES=8"
start "Mimir Spectrum Preview" "%APP%"

View file

@ -0,0 +1,35 @@
param(
[string]$Configuration = "Release",
[string]$TemplateVersion = "master",
[switch]$SkipTests
)
$ErrorActionPreference = "Stop"
$root = Split-Path -Parent $PSScriptRoot
$sdkRoot = Join-Path $root "artifacts\obs-sdk"
$templateRoot = Join-Path $sdkRoot "obs-plugintemplate"
$depsRoot = Join-Path $templateRoot ".deps"
$buildRoot = Join-Path $templateRoot "build_x64_local"
$pluginBuildRoot = Join-Path $root "native\obs_stem_source\build"
New-Item -ItemType Directory -Force $sdkRoot | Out-Null
if (-not (Test-Path $templateRoot)) {
git clone --depth 1 --branch $TemplateVersion https://github.com/obsproject/obs-plugintemplate.git $templateRoot
}
if (-not (Test-Path (Join-Path $depsRoot "cmake\libobsConfig.cmake"))) {
cmake -S $templateRoot -B $buildRoot -G "Visual Studio 17 2022" -A x64 -DENABLE_FRONTEND_API=OFF -DENABLE_QT=OFF
}
cmake `
-S (Join-Path $root "native\obs_stem_source") `
-B $pluginBuildRoot `
-Dlibobs_DIR="$(Join-Path $depsRoot 'cmake')" `
-DCMAKE_PREFIX_PATH="$depsRoot"
cmake --build $pluginBuildRoot --config $Configuration --parallel
if (-not $SkipTests) {
ctest --test-dir $pluginBuildRoot -C $Configuration --output-on-failure
}

View file

@ -0,0 +1,71 @@
param(
[string]$ObsRoot = "",
[string]$PluginRoot = "",
[string]$Configuration = "Release",
[switch]$SkipBuild,
[switch]$WhatIf
)
$ErrorActionPreference = "Stop"
$repoRoot = Split-Path -Parent $PSScriptRoot
$pluginDll = Join-Path $repoRoot "native\obs_stem_source\build\$Configuration\mimir_obs_stem_source.dll"
$localeSourceDir = Join-Path $repoRoot "native\obs_stem_source\data\locale"
if (-not [string]::IsNullOrWhiteSpace($ObsRoot)) {
$targetDir = Join-Path $ObsRoot "obs-plugins\64bit"
$targetLocaleDir = Join-Path $ObsRoot "data\obs-plugins\mimir_obs_stem_source\locale"
} else {
if ([string]::IsNullOrWhiteSpace($PluginRoot)) {
if ($env:OBS_PLUGIN_ROOT) {
$PluginRoot = $env:OBS_PLUGIN_ROOT
} else {
$PluginRoot = Join-Path $env:ALLUSERSPROFILE "obs-studio\plugins"
}
}
$targetDir = Join-Path $PluginRoot "mimir_obs_stem_source\bin\64bit"
$targetLocaleDir = Join-Path $PluginRoot "mimir_obs_stem_source\data\locale"
}
$targetDll = Join-Path $targetDir "mimir_obs_stem_source.dll"
if (-not $SkipBuild) {
& (Join-Path $repoRoot "scripts\build-obs-stem-plugin.ps1") -Configuration $Configuration
}
if (-not (Test-Path $pluginDll)) {
throw "Built plugin DLL was not found at '$pluginDll'."
}
if (-not (Test-Path $localeSourceDir)) {
throw "Plugin locale directory was not found at '$localeSourceDir'."
}
if (-not (Test-Path $targetDir)) {
if ($WhatIf) {
Write-Host "Would create $targetDir"
} else {
New-Item -ItemType Directory -Force $targetDir | Out-Null
}
}
if (-not (Test-Path $targetLocaleDir)) {
if ($WhatIf) {
Write-Host "Would create $targetLocaleDir"
} else {
New-Item -ItemType Directory -Force $targetLocaleDir | Out-Null
}
}
if ($WhatIf) {
Write-Host "Would copy $pluginDll -> $targetDll"
Get-ChildItem -LiteralPath $localeSourceDir -File | ForEach-Object {
Write-Host "Would copy $($_.FullName) -> $(Join-Path $targetLocaleDir $_.Name)"
}
return
}
Copy-Item -LiteralPath $pluginDll -Destination $targetDll -Force
Copy-Item -Path (Join-Path $localeSourceDir "*") -Destination $targetLocaleDir -Force
Write-Host "Installed Mimir OBS plugin: $targetDll"
Write-Host "OBS source types: Mimir Program Texture, Mimir Audio Stem, Muninn Stream, Muninn Video, Muninn Audio"

41
scripts/raven-audio.ps1 Normal file
View file

@ -0,0 +1,41 @@
param(
[string]$TargetHost = "192.168.1.66",
[int]$Port = 5202,
[int]$SampleRate = 48000,
[int]$Channels = 2,
[string]$FfmpegPath = "ffmpeg",
[string]$LogRoot = "C:\Meta\Mimir\logs",
[switch]$DryRun
)
$ErrorActionPreference = "Stop"
New-Item -ItemType Directory -Force -Path $LogRoot | Out-Null
$repo = Split-Path -Parent $PSScriptRoot
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
$stdoutLog = Join-Path $LogRoot "raven-realtk-loopback-$timestamp.out.log"
$stderrLog = Join-Path $LogRoot "raven-realtk-loopback-$timestamp.err.log"
$endpoint = "srt://${TargetHost}:${Port}?mode=caller&latency=120000"
$captureScript = Join-Path $repo "scripts\wasapi-loopback-capture.ps1"
$command = "& `"$captureScript`" -Output - -SampleRate $SampleRate -Channels $Channels | & `"$FfmpegPath`" -hide_banner -nostdin -loglevel warning -f f32le -ar $SampleRate -ac $Channels -i pipe:0 -f f32le `"$endpoint`""
Write-Host "Raven Realtek loopback sender:"
Write-Host " target: $endpoint"
Write-Host " format: f32le ${Channels}ch ${SampleRate}Hz"
Write-Host " stdout: $stdoutLog"
Write-Host " stderr: $stderrLog"
if ($DryRun) {
Write-Host $command
exit 0
}
Start-Process `
-FilePath "powershell" `
-ArgumentList @("-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", $command) `
-WindowStyle Hidden `
-RedirectStandardOutput $stdoutLog `
-RedirectStandardError $stderrLog `
-PassThru |
ForEach-Object { Write-Host "Started Raven Realtek loopback sender pid=$($_.Id)" }

View file

@ -0,0 +1,158 @@
param(
[int]$DurationSeconds = 15,
[string]$OutputPath = "artifacts\runtime\kiyo-dual-mic-composite\kiyo-dual-mic-composite.mp4",
[string]$LogRoot = "artifacts\runtime\kiyo-dual-mic-composite",
[string]$FfmpegPath = "ffmpeg",
[string]$AsioProbePath = "native\probes\asio_audio_cadence\build\Release\asio_audio_cadence.exe",
[string]$KiyoVideoDevice = "USB Video Device",
[int]$KiyoWidth = 640,
[int]$KiyoHeight = 480,
[int]$KiyoFrameRate = 30,
[int]$SampleRate = 48000,
[int]$AsioChannels = 4,
[int]$ShotgunChannel = 0,
[int]$CardioidChannel = 1,
[switch]$DryRun
)
$ErrorActionPreference = "Stop"
function Quote-ProcessArgument([string]$Value) {
if ($Value.Length -eq 0) {
return '""'
}
if ($Value -notmatch '[\s"]') {
return $Value
}
return '"' + $Value.Replace('"', '\"') + '"'
}
function Start-LoggedProcess([string]$FileName, [string[]]$Arguments, [string]$StdoutPath, [string]$StderrPath) {
$startInfo = [System.Diagnostics.ProcessStartInfo]::new()
$startInfo.FileName = $FileName
$startInfo.UseShellExecute = $false
$startInfo.RedirectStandardOutput = $true
$startInfo.RedirectStandardError = $true
$startInfo.Arguments = ($Arguments | ForEach-Object { Quote-ProcessArgument $_ }) -join " "
$process = [System.Diagnostics.Process]::Start($startInfo)
if (-not $process) {
throw "Could not start $FileName"
}
$stdoutStream = [System.IO.File]::Open($StdoutPath, [System.IO.FileMode]::Create, [System.IO.FileAccess]::Write, [System.IO.FileShare]::Read)
$stderrStream = [System.IO.File]::Open($StderrPath, [System.IO.FileMode]::Create, [System.IO.FileAccess]::Write, [System.IO.FileShare]::Read)
return [pscustomobject]@{
Process = $process
StdoutTask = $process.StandardOutput.BaseStream.CopyToAsync($stdoutStream)
StderrTask = $process.StandardError.BaseStream.CopyToAsync($stderrStream)
StdoutStream = $stdoutStream
StderrStream = $stderrStream
}
}
function Wait-LoggedProcess($Handle, [int]$TimeoutSeconds, [string]$Name) {
if (-not $Handle.Process.WaitForExit($TimeoutSeconds * 1000)) {
try { $Handle.Process.Kill($true) } catch {}
throw "$Name timed out after ${TimeoutSeconds}s."
}
[void]$Handle.StdoutTask.GetAwaiter().GetResult()
[void]$Handle.StderrTask.GetAwaiter().GetResult()
$Handle.StdoutStream.Dispose()
$Handle.StderrStream.Dispose()
if ($Handle.Process.ExitCode -ne 0) {
throw "$Name failed with exit code $($Handle.Process.ExitCode)."
}
}
$repo = Split-Path -Parent $PSScriptRoot
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
$absoluteLogRoot = if ([System.IO.Path]::IsPathRooted($LogRoot)) { $LogRoot } else { Join-Path $repo $LogRoot }
$runDirectory = Join-Path $absoluteLogRoot $timestamp
New-Item -ItemType Directory -Force -Path $runDirectory | Out-Null
$absoluteOutputPath = if ([System.IO.Path]::IsPathRooted($OutputPath)) { $OutputPath } else { Join-Path $repo $OutputPath }
$outputDirectory = Split-Path -Parent $absoluteOutputPath
New-Item -ItemType Directory -Force -Path $outputDirectory | Out-Null
$asioRaw = Join-Path $runDirectory "scarlett-f32-interleaved.raw"
$kiyoMkv = Join-Path $runDirectory "kiyo-video.mkv"
$compositeWav = Join-Path $runDirectory "dual-mic-composite.wav"
$absoluteAsioProbe = if ([System.IO.Path]::IsPathRooted($AsioProbePath)) { $AsioProbePath } else { Join-Path $repo $AsioProbePath }
$asioArgs = @(
"--set-sample-rate", $SampleRate.ToString(),
"--record-f32-interleaved", $asioRaw,
"--capture-seconds", $DurationSeconds.ToString()
)
$kiyoArgs = @(
"-hide_banner",
"-y",
"-thread_queue_size", "1024",
"-rtbufsize", "256M",
"-f", "dshow",
"-video_size", "${KiyoWidth}x${KiyoHeight}",
"-framerate", $KiyoFrameRate.ToString(),
"-i", "video=$KiyoVideoDevice",
"-t", $DurationSeconds.ToString(),
"-r", $KiyoFrameRate.ToString(),
"-c:v", "h264_nvenc",
"-preset", "p4",
"-tune", "ll",
"-pix_fmt", "yuv420p",
$kiyoMkv
)
Write-Host "Kiyo + computed dual-mic composite recording:"
Write-Host " Kiyo: $KiyoVideoDevice ${KiyoWidth}x${KiyoHeight}@$KiyoFrameRate"
Write-Host " ASIO: $absoluteAsioProbe sampleRate=$SampleRate channels=$AsioChannels shotgun=$ShotgunChannel cardioid=$CardioidChannel"
Write-Host " Run dir: $runDirectory"
Write-Host " Output: $absoluteOutputPath"
if ($DryRun) {
Write-Host ($absoluteAsioProbe + " " + ($asioArgs -join " "))
Write-Host ($FfmpegPath + " " + ($kiyoArgs -join " "))
exit 0
}
$asio = Start-LoggedProcess $absoluteAsioProbe $asioArgs (Join-Path $runDirectory "asio.out.log") (Join-Path $runDirectory "asio.err.log")
$kiyo = Start-LoggedProcess $FfmpegPath $kiyoArgs (Join-Path $runDirectory "kiyo.out.log") (Join-Path $runDirectory "kiyo.err.log")
Wait-LoggedProcess $asio ($DurationSeconds + 15) "ASIO capture"
Wait-LoggedProcess $kiyo ($DurationSeconds + 15) "Kiyo capture"
dotnet run --project (Join-Path $repo "src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj") -- `
--asio-dual-mic-faust-cleaner-wav `
--input $asioRaw `
--output $compositeWav `
--sample-rate $SampleRate `
--channels $AsioChannels `
--shotgun-channel $ShotgunChannel `
--cardioid-channel $CardioidChannel
if ($LASTEXITCODE -ne 0) {
throw "Dual-mic composite WAV generation failed."
}
$muxArgs = @(
"-hide_banner",
"-y",
"-i", $kiyoMkv,
"-i", $compositeWav,
"-map", "0:v:0",
"-map", "1:a:0",
"-c:v", "copy",
"-c:a", "aac",
"-b:a", "192k",
"-ar", "48000",
"-movflags", "+faststart",
$absoluteOutputPath
)
$mux = Start-LoggedProcess $FfmpegPath $muxArgs (Join-Path $runDirectory "mux.out.log") (Join-Path $runDirectory "mux.err.log")
Wait-LoggedProcess $mux 60 "Mux"
Write-Host "Recording complete:"
Write-Host " $absoluteOutputPath"
Write-Host " composite wav: $compositeWav"
Write-Host " report: $([System.IO.Path]::ChangeExtension($compositeWav, '.json'))"

View file

@ -0,0 +1,201 @@
param(
[int]$RavenPort = 5204,
[int]$DurationSeconds = 20,
[int]$StartupTimeoutSeconds = 20,
[int]$Width = 1280,
[int]$Height = 720,
[int]$FrameRate = 30,
[string]$OutputPath = "artifacts\runtime\stream-proof\stream-proof-output.mp4",
[string]$LogRoot = "artifacts\runtime\stream-proof",
[string]$FfmpegPath = "ffmpeg",
[switch]$StartRavenSender,
[string]$RavenHost = "madman's lullaby@192.168.1.84",
[string]$RavenAddress = "192.168.1.84",
[string]$RavenRepo = "C:\Meta\Mimir",
[string]$StarfireHost = "192.168.1.66",
[switch]$RavenSenderListens,
[ValidateSet("srt", "tcp-listener")]
[string]$RavenSenderTransport = "srt",
[string]$RavenInputOverride = "",
[string]$KiyoVideoDevice = "USB Video Device",
[string]$StarfireScarlettAudioDevice = "Analogue 1 + 2 (2- Focusrite USB Audio)",
[int]$KiyoWidth = 640,
[int]$KiyoHeight = 480,
[int]$KiyoFrameRate = 30,
[int]$KiyoCornerWidth = 320,
[int]$KiyoMargin = 28,
[double]$RavenOffsetSeconds = 0.0,
[double]$KiyoOffsetSeconds = 0.0,
[double]$MicOffsetSeconds = 0.0,
[switch]$NoStarfireScarlett,
[switch]$DryRun
)
$ErrorActionPreference = "Stop"
function Quote-ProcessArgument([string]$Value) {
if ($Value.Length -eq 0) {
return '""'
}
if ($Value -notmatch '[\s"]') {
return $Value
}
return '"' + $Value.Replace('"', '\"') + '"'
}
$repo = Split-Path -Parent $PSScriptRoot
$absoluteOutputPath = if ([System.IO.Path]::IsPathRooted($OutputPath)) {
$OutputPath
} else {
Join-Path $repo $OutputPath
}
$outputDirectory = Split-Path -Parent $absoluteOutputPath
New-Item -ItemType Directory -Force -Path $outputDirectory | Out-Null
$absoluteLogRoot = if ([System.IO.Path]::IsPathRooted($LogRoot)) {
$LogRoot
} else {
Join-Path $repo $LogRoot
}
New-Item -ItemType Directory -Force -Path $absoluteLogRoot | Out-Null
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
$stdoutLog = Join-Path $absoluteLogRoot "stream-proof-record-$timestamp.out.log"
$stderrLog = Join-Path $absoluteLogRoot "stream-proof-record-$timestamp.err.log"
$ravenEndpoint = if ($RavenInputOverride.Length -gt 0) {
$RavenInputOverride
} elseif ($RavenSenderListens) {
"srt://${RavenAddress}:${RavenPort}?mode=caller&latency=120000&timeout=30000000"
} else {
"srt://0.0.0.0:${RavenPort}?mode=listener&latency=120000&timeout=30000000"
}
$ravenSender = "${RavenRepo}\scripts\start-raven-av-sender.ps1"
if ($StartRavenSender) {
$taskName = "MimirRavenAvSender$RavenPort"
$remoteCmd = "${RavenRepo}\scripts\run-raven-av-$RavenPort.cmd"
$remoteTargetHost = if ($RavenSenderListens) { "0.0.0.0" } else { $StarfireHost }
$remoteSrtMode = if ($RavenSenderListens) { "listener" } else { "caller" }
$remoteCommand = "powershell.exe -NoProfile -ExecutionPolicy Bypass -File $ravenSender -TargetHost $remoteTargetHost -Port $RavenPort -SrtMode $remoteSrtMode -Transport $RavenSenderTransport"
$taskTime = (Get-Date).AddMinutes(1).ToString("HH:mm")
Write-Host "Starting Raven muxed A/V sender on ${RavenHost} through interactive task ${taskName}: $remoteCommand"
if (-not $DryRun) {
ssh $RavenHost "cmd /c echo $remoteCommand ^> $remoteCmd" | Out-Null
ssh $RavenHost "schtasks /Delete /TN $taskName /F" | Out-Null
ssh $RavenHost "schtasks /Create /TN $taskName /SC ONCE /ST $taskTime /TR $remoteCmd /F /IT" | Out-Host
ssh $RavenHost "schtasks /Run /TN $taskName" | Out-Host
if ($LASTEXITCODE -ne 0) {
throw "Could not start Raven muxed A/V sender task on $RavenHost."
}
Start-Sleep -Milliseconds 1500
}
}
$arguments = @(
"-hide_banner",
"-y",
"-thread_queue_size", "1024",
"-itsoffset", $RavenOffsetSeconds.ToString([Globalization.CultureInfo]::InvariantCulture),
"-i", $ravenEndpoint,
"-thread_queue_size", "1024",
"-rtbufsize", "256M",
"-itsoffset", $KiyoOffsetSeconds.ToString([Globalization.CultureInfo]::InvariantCulture),
"-f", "dshow",
"-video_size", "${KiyoWidth}x${KiyoHeight}",
"-framerate", $KiyoFrameRate.ToString(),
"-i", "video=$KiyoVideoDevice"
)
if (-not $NoStarfireScarlett) {
$arguments += @(
"-thread_queue_size", "1024",
"-rtbufsize", "256M",
"-itsoffset", $MicOffsetSeconds.ToString([Globalization.CultureInfo]::InvariantCulture),
"-f", "dshow",
"-i", "audio=$StarfireScarlettAudioDevice"
)
}
$scaledKiyoHeight = [Math]::Max(1, [int][Math]::Round($KiyoCornerWidth * ($KiyoHeight / [double]$KiyoWidth)))
$overlayX = $KiyoMargin
$overlayY = $KiyoMargin
if ($NoStarfireScarlett) {
$filter = "[0:v]scale=${Width}:${Height},setsar=1,hqdn3d=1.2:1.0:3.0:2.0[base];" +
"[1:v]scale=${KiyoCornerWidth}:${scaledKiyoHeight},setsar=1,hqdn3d=1.4:1.1:3.4:2.2[pip];" +
"[base][pip]overlay=${overlayX}:${overlayY}:format=auto[v];" +
"[0:a]aresample=async=1:first_pts=0,afftdn=nr=8:nf=-35[a]"
} else {
$filter = "[0:v]scale=${Width}:${Height},setsar=1,hqdn3d=1.2:1.0:3.0:2.0[base];" +
"[1:v]scale=${KiyoCornerWidth}:${scaledKiyoHeight},setsar=1,hqdn3d=1.4:1.1:3.4:2.2[pip];" +
"[base][pip]overlay=${overlayX}:${overlayY}:format=auto[v];" +
"[0:a]aresample=async=1:first_pts=0,afftdn=nr=8:nf=-35[ravenAudio];" +
"[2:a]aresample=async=1:first_pts=0,afftdn=nr=10:nf=-38,volume=1.0[heroMic];" +
"[ravenAudio][heroMic]amix=inputs=2:duration=longest:normalize=0[a]"
}
$arguments += @(
"-filter_complex", $filter,
"-map", "[v]",
"-map", "[a]",
"-t", $DurationSeconds.ToString(),
"-r", $FrameRate.ToString(),
"-c:v", "h264_nvenc",
"-preset", "p4",
"-tune", "ll",
"-rc", "cbr",
"-b:v", "4500k",
"-maxrate", "4500k",
"-bufsize", "9000k",
"-g", ($FrameRate * 2).ToString(),
"-pix_fmt", "yuv420p",
"-c:a", "aac",
"-b:a", "160k",
"-ar", "48000",
"-movflags", "+faststart",
$absoluteOutputPath
)
Write-Host "Stream-proof recording:"
Write-Host " Raven input: $ravenEndpoint"
Write-Host " Raven SRT mode: $(if ($RavenSenderListens) { 'Raven listener, Starfire caller' } else { 'Starfire listener, Raven caller' })"
Write-Host " Kiyo PIP: $KiyoVideoDevice ${KiyoWidth}x${KiyoHeight}@$KiyoFrameRate -> ${KiyoCornerWidth}x${scaledKiyoHeight}"
Write-Host " Raven audio: Realtek/default render loopback in muxed NVENC/SRT feed"
Write-Host " Starfire Scarlett: $(if ($NoStarfireScarlett) { 'disabled' } else { $StarfireScarlettAudioDevice })"
Write-Host " sync role: Starfire Realtek emits chirps; Raven Realtek carries co-streamer game/program audio; Starfire Scarlett captures hero mics plus Raven shotgun"
Write-Host " Offsets: raven=${RavenOffsetSeconds}s kiyo=${KiyoOffsetSeconds}s mic=${MicOffsetSeconds}s"
Write-Host " Output: $absoluteOutputPath"
Write-Host " stdout: $stdoutLog"
Write-Host " stderr: $stderrLog"
if ($DryRun) {
Write-Host ($FfmpegPath + " " + ($arguments -join " "))
exit 0
}
$startInfo = [System.Diagnostics.ProcessStartInfo]::new()
$startInfo.FileName = $FfmpegPath
$startInfo.UseShellExecute = $false
$startInfo.RedirectStandardOutput = $true
$startInfo.RedirectStandardError = $true
$startInfo.Arguments = ($arguments | ForEach-Object { Quote-ProcessArgument $_ }) -join " "
$process = [System.Diagnostics.Process]::Start($startInfo)
if (-not $process) {
throw "Could not start FFmpeg recorder."
}
$stdoutTask = $process.StandardOutput.BaseStream.CopyToAsync([System.IO.File]::Open($stdoutLog, [System.IO.FileMode]::Create, [System.IO.FileAccess]::Write, [System.IO.FileShare]::Read))
$stderrTask = $process.StandardError.BaseStream.CopyToAsync([System.IO.File]::Open($stderrLog, [System.IO.FileMode]::Create, [System.IO.FileAccess]::Write, [System.IO.FileShare]::Read))
$timeoutMs = [Math]::Max(1, $DurationSeconds + $StartupTimeoutSeconds) * 1000
if (-not $process.WaitForExit($timeoutMs)) {
try { $process.Kill($true) } catch {}
throw "FFmpeg recorder timed out after $($DurationSeconds + $StartupTimeoutSeconds)s. Check $stderrLog."
}
[void]$stdoutTask.GetAwaiter().GetResult()
[void]$stderrTask.GetAwaiter().GetResult()
if ($process.ExitCode -ne 0) {
throw "FFmpeg recorder failed with exit code $($process.ExitCode). Check $stderrLog."
}

Some files were not shown because too many files have changed in this diff Show more