Compare commits

...
Author SHA1 Message Date
Meta Vens
f7b61a3753 Make Muninn OBS device selection independent 2026-06-19 23:33:53 +01:00
Meta Vens
aa4ac6760b Verify Muninn OBS source registration 2026-06-19 23:06:20 +01:00
Meta Vens
bdf940ba73 Add split Muninn OBS source types 2026-06-19 22:56:13 +01:00
Meta Vens
e2461ee4ac Ship Muninn OBS plugin locale bundle 2026-06-19 22:35:58 +01:00
Meta Vens
1e5a62eb77 Preserve Muninn source selection across discovered ids 2026-06-19 22:22:47 +01:00
Meta Vens
a911d22c7c Use full Muninn audio reorder budget by default 2026-06-19 21:49:05 +01:00
Meta Vens
48c5268798 Expose split Muninn audio and video sources 2026-06-19 21:36:25 +01:00
Meta Vens
e93f0474f7 Stabilize Muninn audio receive buffering 2026-06-19 20:49:46 +01:00
Meta Vens
64c6275784 Recover Muninn video with striped parity 2026-06-19 20:30:22 +01:00
Meta Vens
1096991c28 Forward Muninn media frames without receiver stalls 2026-06-19 20:10:12 +01:00
Meta Vens
1f0793307b Improve Muninn OBS parity recovery 2026-06-19 19:50:55 +01:00
Meta Vens
af868d2a85 Add Muninn OBS restart helper 2026-06-19 19:11:54 +01:00
Meta Vens
50098dc130 Run OBS native tests during plugin build 2026-06-19 18:34:58 +01:00
Meta Vens
86ccd436dc Test Muninn RUDP URL media settings 2026-06-19 18:33:37 +01:00
Meta Vens
cc14932522 Log Muninn OBS activation media settings 2026-06-19 18:31:21 +01:00
Meta Vens
64be5e5bb3 Time-slice Muninn OBS video assembly expiry 2026-06-19 18:25:53 +01:00
Meta Vens
bc5983c172 Fix Muninn RUDP sequence gap accounting 2026-06-19 17:54:49 +01:00
Meta Vens
9e3fe18e4f Trace Muninn RUDP media sequence gaps 2026-06-19 17:43:30 +01:00
Meta Vens
33f07186d7 Honor Muninn RUDP latency budget in OBS bridge 2026-06-19 17:30:17 +01:00
Meta Vens
c979894a14 Prevent Muninn OBS dead frame resurrection 2026-06-19 17:17:10 +01:00
Meta Vens
bda55c4527 Match Muninn RUDP packet payload default 2026-06-19 16:59:16 +01:00
Meta Vens
7beb5afc0b Honor Muninn RUDP latency for packet expiry 2026-06-19 16:46:21 +01:00
Meta Vens
dbe59014e2 Decode Muninn RUDP audio outside OBS media source 2026-06-19 16:33:30 +01:00
Meta Vens
4c124917b0 Expose Muninn RUDP budgets in OBS source 2026-06-19 16:24:42 +01:00
Meta Vens
40f7d5c313 Let Muninn OBS bridge rejoin active RUDP senders 2026-06-19 15:25:34 +01:00
Meta Vens
186613d216 Honor Muninn RUDP profile changes 2026-06-19 14:45:13 +01:00
Meta Vens
6666ed81a2 Avoid Muninn OBS startup stale resets 2026-06-19 14:07:02 +01:00
Meta Vens
1118171733 Clock Muninn OBS audio output at source 2026-06-19 14:03:14 +01:00
Meta Vens
2eff6d24ad Request Muninn RUDP chunk repairs before expiry 2026-06-19 03:59:12 +01:00
Meta Vens
e6df43fdda Respect Muninn RUDP frame dependencies 2026-06-19 03:28:48 +01:00
Meta Vens
1c511cc862 Pace Muninn RUDP decoder feed 2026-06-19 02:54:26 +01:00
Meta Vens
97e4c1e174 Gate Muninn OBS video after incomplete frames 2026-06-19 02:24:01 +01:00
Meta Vens
78ef023bef Reduce Muninn OBS recovery thrash 2026-06-19 01:58:14 +01:00
Meta Vens
7a35adc461 Make Muninn OBS discovery and video forwarding resilient 2026-06-19 01:53:16 +01:00
Meta Vens
0c9e54656b Make Muninn OBS reconnect receiver-owned 2026-06-19 01:10:37 +01:00
Meta Vens
da543c420d Discover Muninn streams over RUDP catalog 2026-06-19 00:46:45 +01:00
Meta Vens
70098f0c49 Make Muninn OBS source self-connecting 2026-06-19 00:29:23 +01:00
Meta Vens
0352a39ac9 Remove Muninn OBS media sequence gate 2026-06-18 23:29:50 +01:00
Meta Vens
bf27821729 Parse Muninn sender resend cadence in RUDP URL 2026-06-18 22:42:39 +01:00
Meta Vens
1afe21b206 Share OBS Muninn RUDP fragment assembler with probe 2026-06-18 22:20:52 +01:00
Meta Vens
4e17d3b7ab Reassemble OBS Muninn RUDP packet fragments 2026-06-18 22:18:02 +01:00
Meta Vens
2bb94d8a6f Share OBS Muninn RUDP ACK tracker with probe 2026-06-18 22:14:40 +01:00
Meta Vens
b2304bf764 Emit OBS Muninn RUDP ACK masks 2026-06-18 22:12:33 +01:00
Meta Vens
d484d868e1 Bound OBS Muninn RUDP receiver latency 2026-06-18 22:04:30 +01:00
Meta Vens
952360c526 Expose Muninn RUDP resend budget in OBS URL parser 2026-06-18 21:53:39 +01:00
Meta Vens
d15723661d Share Muninn RUDP URL parser with probe 2026-06-18 21:39:14 +01:00
Meta Vens
69effd37e6 Read Muninn RUDP latency budget from stream URL 2026-06-18 21:35:19 +01:00
Meta Vens
4be2615cd1 Emit deterministic Muninn feedback fixture 2026-06-18 21:30:41 +01:00
Meta Vens
2f0e0bdfcb Share Muninn media feedback wire encoder 2026-06-18 21:13:48 +01:00
Meta Vens
fc039eaade Emit Muninn RUDP receiver feedback from OBS 2026-06-18 21:09:17 +01:00
Meta Vens
06d5e2b496 Sweep Muninn video assemblies while idle 2026-06-18 21:01:08 +01:00
Meta Vens
1422208032 Expire incomplete Muninn video assemblies 2026-06-18 20:57:01 +01:00
Meta Vens
44aebe1d54 Make Muninn media probe verify video reassembly 2026-06-18 18:22:57 +01:00
Meta Vens
a16510c12c Reassemble chunked Muninn video in OBS bridge 2026-06-18 18:21:13 +01:00
Meta Vens
26c0b204ad Report Muninn OBS bridge receive pressure 2026-06-18 18:16:42 +01:00
Meta Vens
115dd43ecb Add Muninn media decoder probe 2026-06-18 17:58:49 +01:00
Meta Vens
a60b742434 Decode typed Muninn media in OBS source 2026-06-18 17:49:20 +01:00
Meta Vens
43745268ed Request Muninn RUDP streams from OBS source 2026-06-18 15:12:24 +01:00
Meta Vens
e54ee13ee0 Read Muninn OBS catalog from CultCache 2026-06-08 21:31:25 +01:00
Meta Vens
34140bcd57 Add Muninn OBS stream source 2026-06-08 21:01:48 +01:00
Meta Vens
0cc1414abf Add Raven mux OBS sink 2026-06-07 17:42:17 +01:00
Meta Vens
9642a4d205 Add Raven capture mux daemon 2026-06-07 17:24:55 +01:00
Meta Vens
75ab8a8398 Update VoidBot state 2026-06-07 05:33:19 +01:00
Meta Vens
34d6fad9c0 Add Verse media transport policy 2026-06-07 05:26:34 +01:00
Meta Vens
58db546e36 Revert "Align VoidBot swarm CTB rows"
This reverts commit 727534acba.
2026-06-05 15:44:41 +01:00
Meta Vens
727534acba Align VoidBot swarm CTB rows 2026-06-05 15:36:58 +01:00
Meta Vens
4f3b0e023f Publish Eve dashboard over CultNet 2026-06-05 00:22:24 +01:00
Meta Vens
20410b5a26 Record Mimir semantic Verse provider routes 2026-06-04 23:59:07 +01:00
Meta Vens
db901a2509 Bank Mimir face state 2026-06-04 11:08:48 +01:00
Meta Vens
957a9428f6 Make Well publish ingest heartbeat 2026-06-04 11:08:32 +01:00
Meta Vens
aa08744350 Cull noisy Eve dashboard surfaces 2026-06-04 10:58:19 +01:00
Meta Vens
4ff11a57e3 Bank Mimir face state 2026-06-03 05:36:42 +01:00
Meta Vens
bef44778b6 Show Move sync frame starvation 2026-06-03 05:36:23 +01:00
Meta Vens
302db2ae5b Bank Mimir face state 2026-06-03 05:34:16 +01:00
Meta Vens
fd641fab7a Make daemon status reject stale PIDs 2026-06-03 05:34:10 +01:00
Meta Vens
4c75abcada Expose Well sender pressure 2026-06-03 05:25:50 +01:00
Meta Vens
aacaa9255b Bank Mimir face state 2026-06-03 05:20:31 +01:00
Meta Vens
3eae959e3e Make Well shakedown summarize live run 2026-06-03 05:20:21 +01:00
Meta Vens
89bc1bbb06 Bank Mimir face state 2026-06-03 05:14:32 +01:00
Meta Vens
e1520d1284 Derive program video availability from live samples 2026-06-03 05:14:10 +01:00
Meta Vens
c0a6cad912 Bank Mimir face state 2026-06-03 05:10:54 +01:00
Meta Vens
f87803f26f Expose Well source configuration health 2026-06-03 05:10:31 +01:00
Meta Vens
79de23a4c1 Bank Mimir face state 2026-06-03 04:59:56 +01:00
Meta Vens
b1767182c9 Keep Well heavy analysis off cadence 2026-06-03 04:59:29 +01:00
Meta Vens
d58b0f6515 Bank Mimir face state 2026-06-03 04:41:25 +01:00
Meta Vens
2fa008aee4 Clarify Well source health and process clocks 2026-06-03 04:41:01 +01:00
Meta Vens
980f185104 Bank Mimir face state 2026-06-03 04:13:30 +01:00
Meta Vens
3df9ae0768 Stabilize ASIO readiness in Well frames 2026-06-03 04:13:10 +01:00
Meta Vens
adbaa3342b Add Mimir provider advertisement fixture 2026-06-03 03:34:08 +01:00
Meta Vens
38bcf47fb1 Document Mimir Verse service contract 2026-06-03 03:12:05 +01:00
Meta Vens
e18a6a0320 Bank Mimir face state 2026-06-02 23:50:47 +01:00
Meta Vens
f5193341e9 Fix Well clock freshness and reservoir intake 2026-06-02 23:50:14 +01:00
Meta Vens
7ab555a355 Bank Mimir face state 2026-06-02 19:20:21 +01:00
Meta Vens
92bc5d24f1 Expose online daemon status to Odin 2026-06-02 19:18:55 +01:00
Meta Vens
ba607f6028 Expose Fensalir provider catalog 2026-06-02 18:44:27 +01:00
Meta Vens
794a514c11 Bank Mimir face state 2026-06-02 18:30:39 +01:00
Meta Vens
101ea2b217 Advertise Fensalir daemon pressure capabilities 2026-06-02 18:30:22 +01:00
Meta Vens
d65d744d55 Bank Mimir face state 2026-06-02 18:24:28 +01:00
Meta Vens
1670f05891 Publish Fensalir reservoir pressure state 2026-06-02 18:24:06 +01:00
Meta Vens
561accadf7 Bank Mimir face state 2026-06-02 18:18:23 +01:00
Meta Vens
0037eb87f2 Consume stream frames in Fensalir daemon 2026-06-02 18:18:12 +01:00
Meta Vens
20f92b50b8 Start Fensalir from online Verse supervisor 2026-06-02 18:14:54 +01:00
Meta Vens
ce77e4907c Bank Mimir face state 2026-06-02 18:10:45 +01:00
Meta Vens
f365b7c1dd Add Fensalir reservoir worker state 2026-06-02 18:10:23 +01:00
Meta Vens
d91487fc82 Bank Mimir face state 2026-06-02 17:46:52 +01:00
Meta Vens
85459184f5 Add Fensalir daemon Eve surface 2026-06-02 17:43:30 +01:00
Meta Vens
ac307e2115 Bank Mimir face state 2026-06-02 17:25:07 +01:00
Meta Vens
20493337eb Separate editor presentation from reservoir work 2026-06-02 17:24:35 +01:00
Meta Vens
7790b8acda Add Well OBS daemon 2026-06-02 16:18:15 +01:00
Meta Vens
0ca402ec2d Recover Move score dashboard trace 2026-06-02 15:18:11 +01:00
Meta Vens
2b620d2152 Read Well snapshots from telemetry 2026-06-02 15:16:06 +01:00
Meta Vens
ec91c58408 Add adaptive Well timing telemetry 2026-06-02 15:13:25 +01:00
Meta Vens
05be4b2e44 Verify paired pulse receipts 2026-06-02 15:09:47 +01:00
Meta Vens
fa555f3583 Add paired PS Move chirp actuator 2026-06-02 14:42:53 +01:00
Meta Vens
68bcbabe6e Refresh Mimir Face state 2026-06-02 00:47:30 +01:00
Meta Vens
1474f14c41 Expose Well stream pressure metrics 2026-06-02 00:47:25 +01:00
Meta Vens
0cf790f1d3 Refresh Mimir Face state 2026-06-02 00:25:59 +01:00
Meta Vens
dafca2ccba Refresh Mimir Face state 2026-06-02 00:25:24 +01:00
Meta Vens
93986ebe0a Harden Well shakedown capture 2026-06-02 00:25:19 +01:00
Meta Vens
71c436622b Refresh Mimir Face state 2026-06-02 00:07:27 +01:00
Meta Vens
4fad4b9e29 Archive Well capture pages 2026-06-02 00:07:21 +01:00
Meta Vens
928f6528ea Refresh Mimir Face state 2026-06-01 23:52:22 +01:00
Meta Vens
82f2c7cf96 Let Mimir Well tune camera exposure 2026-06-01 23:52:17 +01:00
Meta Vens
e6cc86b0d4 Refresh Mimir Face state 2026-06-01 23:37:48 +01:00
Meta Vens
77ea3c06e6 Articulate Move Syrinx score voices 2026-06-01 23:37:44 +01:00
Meta Vens
ab57eb61f7 Refresh Mimir Face state 2026-06-01 23:24:51 +01:00
Meta Vens
842364e78b Emit live score from music witnesses 2026-06-01 23:24:45 +01:00
Meta Vens
b9853a855d Refresh Mimir Face state 2026-06-01 22:50:00 +01:00
Meta Vens
b3449f41f1 Bind Move pulses to Syrinx witness calls 2026-06-01 22:49:54 +01:00
Meta Vens
8c449fadff Refresh Mimir Face state 2026-06-01 22:23:32 +01:00
Meta Vens
ca94ba1c6c Make Move score gestures ensemble-aware 2026-06-01 22:23:27 +01:00
Meta Vens
5838b57f06 Refresh Mimir Face state 2026-06-01 22:12:04 +01:00
Meta Vens
1ccf99486d Expose Well score state on Eve dashboard 2026-06-01 22:11:58 +01:00
Meta Vens
2e897f5879 Refresh Mimir Face state 2026-06-01 21:41:51 +01:00
Meta Vens
df03e66e45 Tame online Move music gestures 2026-06-01 21:41:45 +01:00
Meta Vens
a87e447c68 Refresh Mimir Face state 2026-06-01 21:28:43 +01:00
Meta Vens
1778f2da78 Add Mimir Well daemon 2026-06-01 21:27:57 +01:00
Meta Vens
a8b889d03d Refresh Mimir Face state 2026-06-01 21:07:22 +01:00
Meta Vens
583bf2c140 Add online Verse observation daemons 2026-06-01 21:07:05 +01:00
Meta Vens
8436918658 Refresh Mimir Face identity 2026-06-01 20:48:38 +01:00
Meta Vens
1b7ccf6211 Refresh Mimir Face state 2026-06-01 20:48:13 +01:00
Meta Vens
442e3beb12 Keep online Move receiver alive across HID churn 2026-06-01 20:48:13 +01:00
Meta Vens
49cad8c9dc Refresh Mimir Face state 2026-06-01 20:36:54 +01:00
Meta Vens
412e952c4d Update Mimir Face state 2026-06-01 20:36:24 +01:00
Meta Vens
54104250e7 Add online Realtek Move sync worker 2026-06-01 20:36:20 +01:00
Meta Vens
ad17e9bfe2 Update Mimir Face state 2026-06-01 17:43:51 +01:00
Meta Vens
72c2024ffd Update Mimir Face state 2026-06-01 17:43:12 +01:00
Meta Vens
f6b964a440 Bind Eve UI fields to CultCache paths 2026-06-01 17:43:00 +01:00
Meta Vens
3181bb3a28 Update Mimir Face state 2026-06-01 14:54:07 +01:00
Meta Vens
954f97682b Tighten Mimir editor shell layout 2026-06-01 14:54:01 +01:00
Meta Vens
8b84b60917 Update Mimir Face state 2026-06-01 14:43:22 +01:00
Meta Vens
f083a4fbae Add Mimir output controls 2026-06-01 14:43:12 +01:00
Meta Vens
a1de06a100 Update Mimir Face state 2026-06-01 14:27:04 +01:00
Meta Vens
b7e65fcc3b Update Mimir Face state 2026-06-01 14:26:45 +01:00
Meta Vens
09202f1a7c Polish compositor editor affordances 2026-06-01 14:26:44 +01:00
Meta Vens
9dcd9aa54a Update Mimir Face state 2026-06-01 13:37:52 +01:00
Meta Vens
4d30d1e994 Gate inspector transforms to selected source 2026-06-01 13:37:52 +01:00
Meta Vens
867984dfbd Update Mimir Face voice state 2026-06-01 13:33:14 +01:00
Meta Vens
00d12562fd Update Mimir Face state 2026-06-01 13:32:50 +01:00
Meta Vens
acce5e2474 Wire compositor preview handles 2026-06-01 13:32:50 +01:00
Meta Vens
ce1e26d8eb Update Mimir Face state 2026-06-01 13:23:43 +01:00
Meta Vens
4c752d5dff Update Mimir Face state 2026-06-01 13:23:24 +01:00
Meta Vens
e9b76f1724 Add editor placement snapping and aspect lock 2026-06-01 13:23:20 +01:00
Meta Vens
dfbc97872b Update Mimir Face state 2026-06-01 12:47:32 +01:00
Meta Vens
90841af589 Show output buffer in center preview 2026-06-01 12:47:27 +01:00
Meta Vens
9bb2af9aec Update Mimir Face heartbeat 2026-06-01 12:26:02 +01:00
Meta Vens
532a306e6c Record broadcast composite from Mimir app 2026-06-01 12:25:58 +01:00
Meta Vens
ab4707f8f9 Update Mimir Face state 2026-06-01 12:21:12 +01:00
Meta Vens
ef570ad339 Add Mimir app recording control 2026-06-01 12:21:06 +01:00
Meta Vens
321c66d798 Update Mimir Face heartbeat 2026-06-01 05:36:20 +01:00
Meta Vens
9e3b253f7d Fix quiet Faust receipt normalization 2026-06-01 05:36:00 +01:00
Meta Vens
8f267e301b Update Mimir Face heartbeat 2026-06-01 05:16:45 +01:00
Meta Vens
4d9fb33828 Update Mimir Face state 2026-06-01 05:16:20 +01:00
Meta Vens
8122a8d00c Record Kiyo with Faust cleaner chain 2026-06-01 05:16:09 +01:00
Meta Vens
a123f5c1ea Update Mimir Face state 2026-06-01 04:55:20 +01:00
Meta Vens
35e1f090c9 Prove dialogue cleaner follows aligned stems 2026-06-01 04:55:13 +01:00
Meta Vens
bbaa13c50a Update Mimir Face state 2026-06-01 04:47:17 +01:00
Meta Vens
ae8247303e Route dual mic cleanup after Faust alignment 2026-06-01 04:47:06 +01:00
Meta Vens
c265c9cf72 Update Mimir Face state 2026-06-01 04:32:59 +01:00
Meta Vens
4dc93c0e47 Tighten dual mic residual denoise 2026-06-01 04:32:48 +01:00
Meta Vens
2f420e8190 Update Mimir Face state 2026-06-01 04:25:27 +01:00
Meta Vens
10090ee44c Record Kiyo with dual mic composite 2026-06-01 04:25:18 +01:00
Meta Vens
abc28cc74a Update Mimir Face state 2026-06-01 04:14:41 +01:00
Meta Vens
324de581d4 Add dual mic coherence suppression 2026-06-01 04:14:35 +01:00
Meta Vens
c43dfd4285 Update Mimir Face state 2026-06-01 04:02:51 +01:00
Meta Vens
1d54528cd5 Run scheduled Realtek calibration loop 2026-06-01 04:02:41 +01:00
Meta Vens
2d03a06370 Update Mimir Face identity state 2026-06-01 03:46:08 +01:00
Meta Vens
0d7de266f6 Update Mimir Face state 2026-06-01 03:45:43 +01:00
Meta Vens
749aecc5a9 Emit targeted probes through Realtek 2026-06-01 03:45:37 +01:00
Meta Vens
5ef6bbb4c5 Update Mimir Face state 2026-06-01 03:27:05 +01:00
Meta Vens
57d44c341a Schedule bioacoustic probes from confidence 2026-06-01 03:26:59 +01:00
Meta Vens
60fb77241e Record Realtek chirp routing authority 2026-06-01 03:13:29 +01:00
Meta Vens
cb67a568b9 Run live targeted bioacoustic probe 2026-06-01 03:11:50 +01:00
Meta Vens
8bb854859b Plan targeted bioacoustic probes from residuals 2026-06-01 03:05:06 +01:00
Meta Vens
8df851850c Add calibrated audio composite diagnostic 2026-06-01 02:56:03 +01:00
Meta Vens
3dc92b873f Record Raven LAN SRT proof 2026-06-01 02:46:01 +01:00
Meta Vens
4d3f2cf29b Use desktop duplication for Raven sender 2026-06-01 02:42:34 +01:00
Meta Vens
da2ba6e2d5 Make video source rail direct 2026-06-01 02:35:04 +01:00
Meta Vens
7f0e2de8be Add compiled Raven loopback capture helper 2026-06-01 02:31:12 +01:00
Meta Vens
6658acaee3 Resample WASAPI loopback output 2026-06-01 02:02:07 +01:00
Meta Vens
477d421dbf Add TCP tunnel fallback for stream proof 2026-06-01 01:56:19 +01:00
Meta Vens
85d021faef Clarify stream proof audio routing 2026-06-01 01:21:43 +01:00
Meta Vens
35e2c802a1 Update Mimir face state 2026-06-01 01:21:04 +01:00
Meta Vens
39039ae9f0 Add stream proof recording harness 2026-06-01 01:20:54 +01:00
Meta Vens
0d72b8f29f Update Mimir face state 2026-06-01 00:22:34 +01:00
Meta Vens
2028f0f805 Mux Raven display and loopback transport 2026-06-01 00:22:27 +01:00
Meta Vens
fe101a579a Seed program surface CultCache config 2026-06-01 00:19:15 +01:00
Meta Vens
201be0b10d Update Mimir face state 2026-06-01 00:16:06 +01:00
Meta Vens
1a663fa33f Apply Raven audio sync to display domain 2026-06-01 00:15:52 +01:00
Meta Vens
c99a248d80 Update Mimir face state 2026-06-01 00:10:43 +01:00
Meta Vens
1ea8100a7b Add stream proof calibration profile 2026-06-01 00:10:24 +01:00
Meta Vens
be787735c9 Update Mimir face state 2026-05-31 23:52:50 +01:00
Meta Vens
c7f4acbb78 Add Leap Eyes Kiyo MVP launcher 2026-05-31 23:52:23 +01:00
Meta Vens
dc3fe32bb3 Report articulated sweep overlaps 2026-05-31 23:40:49 +01:00
Meta Vens
34183e6a79 Update Mimir face state 2026-05-31 23:38:51 +01:00
Meta Vens
1177e783ff Capture articulated chirp-flash sweeps 2026-05-31 23:38:19 +01:00
Meta Vens
30cbb8787b Update Mimir face state 2026-05-31 23:14:16 +01:00
Meta Vens
7ddac2af9e Encode Move visual gestures as contours 2026-05-31 23:14:16 +01:00
Meta Vens
4623507104 Update Mimir face state 2026-05-31 23:11:04 +01:00
Meta Vens
5e42617b93 Make editor audio mixer a channel strip dock 2026-05-31 23:10:42 +01:00
Meta Vens
4ff1a89003 Update Mimir face state 2026-05-31 23:10:30 +01:00
Meta Vens
81b96f7e91 Restore inspector rail and audio dock 2026-05-31 22:57:34 +01:00
Meta Vens
7fe9e44e26 Update Mimir face state 2026-05-31 22:57:07 +01:00
Meta Vens
7898a87f20 Publish local frustum hint state 2026-05-31 22:56:49 +01:00
Meta Vens
27c53f74a7 Generalize frustum solve to feature tracks 2026-05-31 22:53:56 +01:00
Meta Vens
995a0c65b7 Record flat 2D mixer surface 2026-05-31 22:51:51 +01:00
Meta Vens
ef97a87d4c Update Mimir face state 2026-05-31 22:49:25 +01:00
Meta Vens
ccae91b4b0 Solve camera frustums from calibration markers 2026-05-31 22:49:07 +01:00
Meta Vens
04037ec7d9 Update Mimir face state 2026-05-31 22:31:04 +01:00
Meta Vens
00ac2e50a6 Add chirp Move sync sweep receipts 2026-05-31 22:30:57 +01:00
Meta Vens
5a40aec401 Update Mimir Face state 2026-05-31 22:14:58 +01:00
Meta Vens
10e292a702 Prioritize editor compositor preview 2026-05-31 22:14:53 +01:00
Meta Vens
027198b541 Update Mimir face state 2026-05-31 22:09:31 +01:00
Meta Vens
470779dee5 Record Nightwing Move HID access 2026-05-31 22:09:23 +01:00
Meta Vens
94e34838ff Update Mimir Face state 2026-05-31 22:06:00 +01:00
Meta Vens
66cd8e81c0 Make editor scene view a compositor preview 2026-05-31 22:05:56 +01:00
Meta Vens
5e9a783d36 Update Mimir face state 2026-05-31 21:59:16 +01:00
Meta Vens
d716fbaee2 Add music-keyed Move chirp planner 2026-05-31 21:59:06 +01:00
Meta Vens
d6b6256f0a Update Mimir Face state 2026-05-31 21:41:34 +01:00
Meta Vens
1bf54bf2ee Reshape Mimir editor surface 2026-05-31 21:41:28 +01:00
Meta Vens
59112c2cfa Update Mimir face state 2026-05-31 21:35:53 +01:00
Meta Vens
ffdbd273f7 Record second PS Move registration 2026-05-31 21:35:45 +01:00
Meta Vens
9f86ddbd05 Update Mimir face state 2026-05-31 21:26:18 +01:00
Meta Vens
de4b12d84f Add room calibration lock solver 2026-05-31 21:26:11 +01:00
Meta Vens
3446d8b81b Update Mimir face state 2026-05-31 21:11:43 +01:00
Meta Vens
987a4713b4 Update Mimir face state 2026-05-31 21:11:22 +01:00
Meta Vens
fcd05596c8 Update Mimir face state 2026-05-31 21:10:47 +01:00
Meta Vens
2ecb36bcc9 Publish Nightwing typed witness receipts 2026-05-31 21:10:35 +01:00
Meta Vens
f2d212bffe Update Mimir face state 2026-05-31 20:30:02 +01:00
Meta Vens
bb2db98adb Verify Nightwing PS3 Eye cadence 2026-05-31 20:29:53 +01:00
Meta Vens
d505872df4 Update Mimir face state 2026-05-31 19:51:21 +01:00
Meta Vens
1eebce7bf1 Model Nightwing Eyes Moves witness 2026-05-31 19:51:16 +01:00
Meta Vens
429f51fbd8 Add Move controller stream overlay profile 2026-05-31 19:22:04 +01:00
Meta Vens
e9f871cc6d Update Mimir face state 2026-05-31 19:13:14 +01:00
Meta Vens
12deb8fc8e Encode PS Move latency probe events 2026-05-31 19:12:37 +01:00
Meta Vens
27b1cd2d55 Rank PS Move onsets by recent whitened flux 2026-05-31 18:48:03 +01:00
Meta Vens
bc2b2da9f3 Force PS Move bursts on adaptive range hits 2026-05-31 18:43:47 +01:00
Meta Vens
db5e6f3517 Update Mimir face state 2026-05-31 18:33:07 +01:00
Meta Vens
46d2eec5b6 Sharpen PS Move spectral onset colors 2026-05-31 18:32:57 +01:00
Meta Vens
5e01a9820a Update Mimir face state 2026-05-31 18:22:57 +01:00
Meta Vens
63d2d3fc66 Pulse PS Move from Scarlett loopback onsets 2026-05-31 18:22:51 +01:00
Meta Vens
e42588592e Author Mimir editor as Eve grid surface 2026-05-31 18:07:06 +01:00
Meta Vens
b8c447a902 Update Mimir face state 2026-05-31 17:57:57 +01:00
Meta Vens
17b991c182 Pair PS Move to Bluetooth host 2026-05-31 17:57:44 +01:00
Meta Vens
f6408d43e7 Surface Mimir sync decode attempts 2026-05-31 17:39:48 +01:00
Meta Vens
fbc1a52ba9 Persist Mimir face state 2026-05-31 17:39:44 +01:00
Meta Vens
a79bb1aa9f Add PS Move HID probe 2026-05-31 17:22:54 +01:00
Meta Vens
b8d5d9f667 Persist Mimir Face state 2026-05-31 17:02:16 +01:00
Meta Vens
532c7e0949 Publish compact Mimir live stats surface 2026-05-31 17:02:13 +01:00
Meta Vens
76e9a80fa8 Accept Eve media observations over CultMesh 2026-05-31 14:19:42 +01:00
Meta Vens
54b77566c4 Persist Mimir Face state 2026-05-31 14:19:36 +01:00
Meta Vens
576f7b4fde Mirror Eve surface trees over CultMesh 2026-05-31 13:06:22 +01:00
Meta Vens
c45eec6a83 Persist Mimir Face state 2026-05-31 13:06:17 +01:00
Meta Vens
48ce102270 Add CultMesh Eve dashboard and sensor lanes 2026-05-31 12:43:28 +01:00
Meta Vens
6c8548e2a2 Persist Mimir Face state 2026-05-31 12:43:19 +01:00
Meta Vens
fd11d73368 Update Mimir face state 2026-05-31 09:06:49 +01:00
Meta Vens
f1d05dd87b Add CultUI-shaped Eve surface state 2026-05-31 09:06:42 +01:00
Meta Vens
60d3af9dc0 Update Mimir face state 2026-05-31 08:44:49 +01:00
Meta Vens
18476b7f74 Expose Mimir layout through Eve dashboard 2026-05-31 08:44:49 +01:00
Meta Vens
f342f098de Update Mimir face state 2026-05-31 04:09:22 +01:00
Meta Vens
56756e8b30 Document all-camera GPU feature extraction proof 2026-05-31 04:09:11 +01:00
Meta Vens
06842840ca Update Mimir face state 2026-05-31 04:00:17 +01:00
Meta Vens
186ebacb53 Document GPU feature extraction proof 2026-05-31 04:00:12 +01:00
Meta Vens
ae1ac64308 Update Mimir face state 2026-05-31 03:31:44 +01:00
Meta Vens
1269d42520 Prove all-sensor GPU runtime path 2026-05-31 03:31:39 +01:00
Meta Vens
95ba849af4 Update Mimir face state 2026-05-31 02:48:45 +01:00
Meta Vens
3c4f66f920 Add PS3 Eye sparse tracking evidence 2026-05-31 02:48:20 +01:00
Meta Vens
25c28f4629 Update Mimir Face state 2026-05-31 02:47:54 +01:00
Meta Vens
60ced7b89a Update Mimir Face state 2026-05-31 02:45:47 +01:00
Meta Vens
c952f4b1b4 Render VoidBot dashboard avatars on Eve 2026-05-31 02:45:39 +01:00
Meta Vens
fd5a19aaa8 Update Mimir face state 2026-05-31 02:37:27 +01:00
Meta Vens
54b549a626 Optimize live LED spline camera settings 2026-05-31 02:37:19 +01:00
Meta Vens
b27081ed71 Update Mimir face state 2026-05-31 02:16:22 +01:00
Meta Vens
62df36c85c Analyze frames for LED spline sweep quality 2026-05-31 02:16:16 +01:00
Meta Vens
ae5cebf20a Update Mimir face state 2026-05-31 02:12:23 +01:00
Meta Vens
2f1e5f856e Score exposure sweeps by LED spline quality 2026-05-31 02:12:17 +01:00
Meta Vens
43743d9c07 Update Mimir face state 2026-05-31 02:05:31 +01:00
Meta Vens
43e0f91f6a Solve LED curves into camera calibration updates 2026-05-31 02:05:27 +01:00
Meta Vens
de75fa3445 Update Mimir face state 2026-05-31 01:58:57 +01:00
Meta Vens
571f9cdeec Add LED spline clipspace residuals 2026-05-31 01:58:50 +01:00
Meta Vens
97a4b78a48 Update Mimir face state 2026-05-31 01:55:40 +01:00
Meta Vens
28c5a30e32 Add LED spline calibration evidence path 2026-05-31 01:55:35 +01:00
Meta Vens
b9b2e07f82 Update Mimir Face state 2026-05-31 01:35:48 +01:00
Meta Vens
1f6e4b061b Add VoidBot Eve dashboard provider 2026-05-31 01:35:42 +01:00
Meta Vens
3fc6204aba Update Mimir Face state 2026-05-31 01:23:43 +01:00
Meta Vens
f2ae17f342 Broker native Eve dashboards 2026-05-31 01:23:15 +01:00
Meta Vens
31310a9003 Update Mimir face state 2026-05-31 01:21:58 +01:00
Meta Vens
58fef50e82 Record history-enabled Leap reservoir proof 2026-05-31 01:21:52 +01:00
Meta Vens
aea7ab0962 Update Mimir face state 2026-05-31 01:09:33 +01:00
Meta Vens
7b0fdde601 Route live Leap depth into surface claims 2026-05-31 01:09:28 +01:00
Meta Vens
7a7533231c Update Mimir Face state 2026-05-31 01:04:36 +01:00
Meta Vens
3f8d2421a9 Add native Eve dashboard server 2026-05-31 01:04:31 +01:00
Meta Vens
9cd2312cfd Update Mimir Face state 2026-05-31 00:51:51 +01:00
Meta Vens
e7b4a22a5b Update Mimir Face state 2026-05-31 00:47:57 +01:00
Meta Vens
b87571f141 Add Eve dialogue probe 2026-05-31 00:47:50 +01:00
Meta Vens
9c0d68359c Update Leap point cloud handoff 2026-05-30 23:15:15 +01:00
Meta Vens
ef0a8ed986 Update Mimir Face state 2026-05-30 23:15:03 +01:00
Meta Vens
5f551e1025 Update Mimir Face state 2026-05-30 23:07:45 +01:00
Meta Vens
693f69b41b Add Eve sensor receiver ingest 2026-05-30 23:07:37 +01:00
Meta Vens
4fadcfff14 Add Leap point cloud field socket 2026-05-30 22:54:59 +01:00
Meta Vens
8d11c7b6ff Update Mimir Face state 2026-05-30 22:54:37 +01:00
Meta Vens
a819998ef3 Update Mimir Face state 2026-05-30 22:47:44 +01:00
Meta Vens
19ed000cd8 Add Raven Eve network ingest setup 2026-05-30 22:47:21 +01:00
Meta Vens
140cac4f2e Update Mimir Face state 2026-05-30 22:18:28 +01:00
Meta Vens
0632a1baf4 Move livestream viewer to root GameCult site 2026-05-30 22:18:24 +01:00
Meta Vens
26234c6f3e Update Mimir Face state 2026-05-30 22:05:01 +01:00
Meta Vens
1e4ab94776 Add self-hosted Mimir live edge 2026-05-30 22:04:54 +01:00
Meta Vens
2c197cb3ae Lower live Leap packed stereo to depth socket 2026-05-30 21:54:58 +01:00
Meta Vens
d6823245c2 Carry stereo depth SGM constants 2026-05-30 00:48:10 +01:00
Meta Vens
356f750869 Record stereo depth readiness boundary 2026-05-30 00:44:43 +01:00
Meta Vens
2273a0c9ab Emit stereo depth lowering metadata 2026-05-30 00:40:53 +01:00
Meta Vens
9c0c7a536d Record scene editor state evidence 2026-05-30 00:32:35 +01:00
Meta Vens
4108f000b8 Add Mimir scene editor control surface 2026-05-30 00:32:03 +01:00
Meta Vens
f868ddf18d Mark stereo depth disparity as compute output 2026-05-30 00:31:14 +01:00
Meta Vens
67f1c662f0 Add stereo depth field contract 2026-05-30 00:22:17 +01:00
Meta Vens
293fbd1cf8 Add Fensalir program control surface 2026-05-29 23:53:04 +01:00
Meta Vens
bcb85468c8 Document D3D12 stereo depth provenance 2026-05-29 23:51:13 +01:00
Meta Vens
4be341bc3f Add synchronized Raven display buffer planning 2026-05-29 22:23:34 +01:00
Meta Vens
a100c7f51e Add spectrum preview launcher 2026-05-29 12:43:32 +01:00
Meta Vens
45780ad4c1 Record TubeField column reservoir authority 2026-05-29 01:50:15 +01:00
Meta Vens
a822b2fd87 Record cheap TubeField continuity path 2026-05-28 23:21:37 +01:00
Meta Vens
5ac483c1e3 Record TubeField reservoir noise correction 2026-05-28 23:06:57 +01:00
Meta Vens
20d126c93b Record compute-owned reservoir history 2026-05-28 22:18:18 +01:00
Meta Vens
44cb57573c Record four row reservoir preservation 2026-05-28 21:10:18 +01:00
Meta Vens
f12f613dc3 Record structured reservoir history 2026-05-28 21:08:31 +01:00
Meta Vens
3e943d2c4c Record four slot reservoir inlet 2026-05-28 20:29:37 +01:00
Meta Vens
e2a1d5b4ae Record reservoir resolve authority 2026-05-28 20:05:03 +01:00
Meta Vens
11fb3d8365 Raise spectrum TubeField emission 2026-05-28 15:07:40 +01:00
Meta Vens
48438a130f Use Fensalir blackbody ramp resource 2026-05-28 14:26:28 +01:00
Meta Vens
5ffb9e8961 Bind spectrum tubes to blackbody gradient 2026-05-28 14:18:38 +01:00
Meta Vens
69970302b8 Add single tube proof controls 2026-05-28 14:01:54 +01:00
Meta Vens
597a64b7f8 Record TubeField blue-noise reservoir cut 2026-05-28 12:59:42 +01:00
Meta Vens
6f09a54ca3 Record TubeField reservoir guide contract 2026-05-28 12:44:25 +01:00
Meta Vens
6f85bd3afe Reconcile program output receipt phase 2026-05-28 12:15:28 +01:00
Meta Vens
fa2c8ed63b Lower synthetic visual marker candidates 2026-05-28 12:14:02 +01:00
Meta Vens
fdba81cac1 Lower synthetic acoustic source candidates 2026-05-28 12:10:50 +01:00
Meta Vens
1fd67e1ff4 Record planned acoustic field constraints 2026-05-28 12:07:51 +01:00
Meta Vens
c192eba35a Add ASIO monitor split proof 2026-05-28 08:21:08 +01:00
Meta Vens
b86f6977d5 Record complex contour mic geometry 2026-05-28 08:14:48 +01:00
Meta Vens
3475ef150e Record monitor-routed complex contour proof 2026-05-28 08:09:20 +01:00
Meta Vens
afe6382d63 Record fresh complex contour mic blocker 2026-05-28 00:27:32 +01:00
Meta Vens
319210778a Publish OBS program output consumer fence 2026-05-28 00:20:32 +01:00
Meta Vens
cc39882296 Record dedicated program output fence 2026-05-28 00:09:39 +01:00
Meta Vens
9b559a0ebf Add OBS program texture ring option 2026-05-28 00:04:52 +01:00
Meta Vens
72738e6676 Observe program output producer fence 2026-05-27 23:56:42 +01:00
Meta Vens
1826edbb9c Prove OBS program texture bridge 2026-05-27 23:34:44 +01:00
Meta Vens
ef0323bd48 Log OBS program texture open failures 2026-05-27 23:23:45 +01:00
Meta Vens
f282aa24b1 Record OBS plugin load smoke 2026-05-27 21:31:40 +01:00
Meta Vens
3b24c75371 Install OBS plugin to package path 2026-05-27 21:30:31 +01:00
Meta Vens
431b5d723d Add OBS plugin install script 2026-05-27 21:29:23 +01:00
Meta Vens
79a4da4259 Add OBS program texture source 2026-05-27 21:28:15 +01:00
Meta Vens
b470b8cbf3 Automate OBS stem plugin SDK build 2026-05-27 21:04:43 +01:00
Meta Vens
71234f04ce Scaffold OBS audio stem source 2026-05-27 20:55:48 +01:00
Meta Vens
0c39921f96 Track OBS stem publication readiness 2026-05-27 20:34:09 +01:00
Meta Vens
51c70f0b49 Name actuator output stems 2026-05-27 20:29:52 +01:00
Meta Vens
86130d4349 Feed actuator audio blocks to Fensalir 2026-05-27 20:19:26 +01:00
Meta Vens
b4308f0c48 Update Mimir face state 2026-05-27 18:56:01 +01:00
Meta Vens
80418742ce Update Mimir face state 2026-05-27 18:55:38 +01:00
Meta Vens
67767a79e4 Declare alignment streaming DSP program 2026-05-27 18:55:27 +01:00
Meta Vens
5ea539bc7d Queue audio actuator controls into Fensalir 2026-05-27 18:07:04 +01:00
Meta Vens
03337cb62d Publish runtime audio actuator commands 2026-05-27 18:03:25 +01:00
Meta Vens
397b9fc1b7 Record camera textures as debug-only scope 2026-05-27 17:54:49 +01:00
Meta Vens
dab0f520ca Signal MF GPU camera producer fences 2026-05-27 17:22:30 +01:00
Meta Vens
e59cf4450a Decode compressed camera frames on GPU 2026-05-27 16:36:19 +01:00
Meta Vens
85e04cf687 Avoid extra managed clone before camera upload 2026-05-27 14:35:42 +01:00
Meta Vens
58e7af8a76 Capture PS3 Eye pixels through direct driver 2026-05-27 14:33:36 +01:00
Meta Vens
0d9c7824b8 Capture Leap pixels through KS driver 2026-05-27 14:26:11 +01:00
Meta Vens
a4d1a59a97 Preserve Mimir Face state 2026-05-27 14:11:56 +01:00
Meta Vens
7f3df7979b Upload CPU camera frames into Fensalir leases 2026-05-27 14:11:48 +01:00
Meta Vens
9147fcd626 Make camera ingest device-direct first 2026-05-27 13:38:24 +01:00
Meta Vens
7202d30ef1 Propagate Fensalir texture leases to video drivers 2026-05-27 13:32:38 +01:00
Meta Vens
daf1d4ef8a Lease camera textures from Fensalir 2026-05-27 13:29:48 +01:00
Meta Vens
f9151b091b Route acoustic proof through field evidence 2026-05-27 12:54:12 +01:00
Meta Vens
aae7f9e4f3 Record shared camera texture resolver 2026-05-27 12:49:55 +01:00
Meta Vens
c342c04023 Record Mimir Face state 2026-05-27 12:44:06 +01:00
Meta Vens
ddad34f69b Bridge camera observations through field evidence 2026-05-27 12:44:01 +01:00
Meta Vens
3ab163da57 Update TubeField migration map 2026-05-27 12:32:53 +01:00
Meta Vens
73140607a9 Upload only newest spectrum ring slot 2026-05-27 12:22:30 +01:00
Meta Vens
07a7864944 Upload spectrum ring slots by offset 2026-05-27 12:01:34 +01:00
Meta Vens
b44aaac7cc Use ring offsets for spectrum history 2026-05-27 11:55:46 +01:00
Meta Vens
07198f948d Record Mimir Face state 2026-05-27 11:51:49 +01:00
Meta Vens
b285acf481 Fix spectrum source lane budget 2026-05-27 11:48:18 +01:00
Meta Vens
59d6c9c491 Expose spectrum tube subdivision budget 2026-05-27 11:42:03 +01:00
Meta Vens
8ad4862140 Record Mimir Face state 2026-05-27 11:39:29 +01:00
Meta Vens
e88a9407b7 Make spectrum upload a fixed rolling matrix 2026-05-27 11:39:07 +01:00
Meta Vens
c8d4496cac Bind spectrum ramp as field resource 2026-05-26 23:51:13 +01:00
Meta Vens
167b87911f Record spectrum upload headless receipt 2026-05-26 23:48:28 +01:00
Meta Vens
8636519bc1 Verify runtime spectrum resource upload 2026-05-26 23:45:00 +01:00
Meta Vens
940601ef3e Feed spectrum frames into field GPU resource 2026-05-26 23:37:35 +01:00
Meta Vens
907cf89d03 Silence legacy spectrum spline path 2026-05-26 23:25:16 +01:00
Meta Vens
f536385e1a Emit Mimir TubeSpline field lowerings 2026-05-26 23:01:05 +01:00
Meta Vens
7b2439f8f4 Record TubeSpline lowering validation 2026-05-26 22:58:53 +01:00
Meta Vens
32834e50ee Record TubeField packet authority 2026-05-26 22:55:48 +01:00
Meta Vens
10bddb0d0e Record generated mesh draw ABI 2026-05-26 22:52:56 +01:00
Meta Vens
dd4f65277a Record TubeField generated mesh binding 2026-05-26 22:50:05 +01:00
Meta Vens
e9bc1ff08f Commit Mimir voidbot state 2026-05-26 22:25:29 +01:00
Meta Vens
7f04e2c1e4 Record Fensalir mesh layout authority 2026-05-26 22:25:25 +01:00
Meta Vens
787c9e5cd0 Commit Mimir voidbot state 2026-05-26 21:32:19 +01:00
Meta Vens
182fe1399c Record generic Fensalir resource claims 2026-05-26 21:32:15 +01:00
Meta Vens
8e3c05be56 Commit Mimir voidbot state 2026-05-26 21:26:40 +01:00
Meta Vens
236e2c269d Record Fensalir mesh resource resolver 2026-05-26 21:26:35 +01:00
Meta Vens
d17f1eee77 Commit Mimir voidbot state 2026-05-26 21:08:58 +01:00
Meta Vens
0a2ca78136 Record Fensalir volume texture resolver 2026-05-26 21:08:53 +01:00
Meta Vens
833f5e1114 Commit Mimir voidbot state 2026-05-26 21:04:35 +01:00
Meta Vens
c08301cd8a Record Fensalir surface page resolver 2026-05-26 21:04:29 +01:00
Meta Vens
169c34348e Commit Mimir voidbot state 2026-05-26 20:58:58 +01:00
Meta Vens
50aa99f281 Record Fensalir indirect TubeField draws 2026-05-26 20:58:53 +01:00
Meta Vens
1873faceb1 Commit Mimir voidbot state 2026-05-26 20:38:30 +01:00
Meta Vens
0132e63a49 Record Fensalir texture resource binding 2026-05-26 20:38:26 +01:00
Meta Vens
f17d7175d6 Commit Mimir voidbot state 2026-05-26 20:22:19 +01:00
Meta Vens
03a8f148a0 Record TubeField pixel material cut 2026-05-26 20:22:13 +01:00
Meta Vens
504678e8af Commit Mimir voidbot state 2026-05-26 20:11:16 +01:00
Meta Vens
94d720a3a5 Record TubeField GPU rendering cut 2026-05-26 20:11:13 +01:00
Meta Vens
df37ed38eb Commit Mimir voidbot state 2026-05-26 18:51:53 +01:00
Meta Vens
0a72126bbb Record GPU-resident Fensalir buffer invariant 2026-05-26 18:51:47 +01:00
Meta Vens
ebca5edade Commit Mimir voidbot state 2026-05-26 17:32:32 +01:00
Meta Vens
87acc68b0e Commit Mimir voidbot state 2026-05-26 17:31:13 +01:00
Meta Vens
8d6fe14462 Correct Fensalir resource ownership map 2026-05-26 17:31:07 +01:00
Meta Vens
a3ed0590eb Commit Mimir voidbot state 2026-05-26 16:44:28 +01:00
Meta Vens
fd01d44677 Plan Mimir spectrum intents as resource-backed packets 2026-05-26 16:44:24 +01:00
Meta Vens
e75fc71211 Commit Mimir voidbot state 2026-05-26 16:22:52 +01:00
Meta Vens
83b1f7449f Declare Fensalir field resources from Mimir payloads 2026-05-26 16:22:48 +01:00
Meta Vens
1f9b9d0684 Commit Mimir voidbot state 2026-05-26 14:58:18 +01:00
Meta Vens
ea9d5f97a2 Commit Mimir voidbot state 2026-05-26 14:56:54 +01:00
Meta Vens
42e5e2eb93 Commit Mimir voidbot state 2026-05-26 14:51:33 +01:00
Meta Vens
daa16136e2 Publish field evidence from Mimir runtime 2026-05-26 14:51:06 +01:00
Meta Vens
c147abdc8a Commit Mimir voidbot state 2026-05-26 14:47:22 +01:00
Meta Vens
4ade05f9d4 Commit Mimir voidbot state 2026-05-26 14:44:26 +01:00
Meta Vens
eea4eba44f Add field evidence contract smoke 2026-05-26 14:44:12 +01:00
Meta Vens
c17abf08dd Commit Mimir voidbot state 2026-05-26 14:40:43 +01:00
Meta Vens
3e09400ed3 Keep Mimir field evidence backend-neutral 2026-05-26 14:40:29 +01:00
Meta Vens
ca3970c9d0 Lower Mimir bridge DTOs to field evidence 2026-05-26 14:35:54 +01:00
Meta Vens
cbf571b615 Record Mimir bridge skeleton evidence 2026-05-26 14:05:04 +01:00
Meta Vens
0283e076ea Add Mimir Fensalir bridge DTO skeleton 2026-05-26 14:02:06 +01:00
Meta Vens
df07f88c4b Commit Mimir voidbot state 2026-05-26 13:29:27 +01:00
Meta Vens
d53decf276 Map Fensalir rendering rebuild 2026-05-26 13:29:01 +01:00
Meta Vens
530b1af723 Record reservoir support scale invariant 2026-05-26 11:49:29 +01:00
Meta Vens
67154c0eb2 Record Fensalir evidence-machine invariant 2026-05-26 11:41:24 +01:00
Meta Vens
9fbbb93132 Commit Mimir voidbot state 2026-05-26 11:41:13 +01:00
Meta Vens
bbeca80f1d Smooth spectrum spline debug surface 2026-05-26 11:31:57 +01:00
Meta Vens
62d40ec58f Commit Mimir voidbot state 2026-05-26 11:30:05 +01:00
Meta Vens
5f9e5b31da Rebuild spectrum splines around temporal identity 2026-05-26 11:14:02 +01:00
Meta Vens
580fe09c93 Commit Mimir voidbot state 2026-05-26 11:13:42 +01:00
Meta Vens
034f0b1578 Drive spectrum spline bloom from audio energy 2026-05-26 11:02:54 +01:00
Meta Vens
966c1cd5de Commit Mimir voidbot state 2026-05-26 11:02:27 +01:00
Meta Vens
da5030cd09 Route spectrum dashboard through spline tube surfaces 2026-05-26 10:50:37 +01:00
Meta Vens
1dd40b0d25 Commit Mimir voidbot state 2026-05-26 10:50:22 +01:00
Meta Vens
ca92bb5c25 Commit Mimir voidbot state 2026-05-26 10:33:16 +01:00
Meta Vens
d5c0964929 Commit Mimir voidbot state 2026-05-26 10:32:17 +01:00
Meta Vens
1eb195804a Route spectrum dashboard through reservoir budget 2026-05-26 10:31:12 +01:00
Meta Vens
37f32f9b1e Commit Mimir voidbot state 2026-05-26 10:30:56 +01:00
Meta Vens
0b7823d6dd Commit Mimir voidbot state 2026-05-26 10:11:43 +01:00
Meta Vens
060a6d0855 Cut spectrum reservoir splat path 2026-05-26 10:11:00 +01:00
Meta Vens
b2e0392931 Commit Mimir voidbot state 2026-05-26 10:10:33 +01:00
Meta Vens
5ae37ea819 Commit Mimir voidbot state 2026-05-25 23:18:46 +01:00
Meta Vens
bd1be5d0e4 Record spectrum reservoir state 2026-05-25 23:02:53 +01:00
Meta Vens
0eb18b0c46 Force spectrum dashboard through reservoir 2026-05-25 23:02:44 +01:00
Meta Vens
2351a8ffb8 Record Eve dashboard stream state 2026-05-25 22:45:55 +01:00
Meta Vens
022567da43 Add H264 Eve dashboard relay 2026-05-25 22:45:46 +01:00
Meta Vens
c189e9c77f Commit Mimir voidbot state 2026-05-25 22:02:18 +01:00
Meta Vens
3ca0092717 Fix spectrum camera pitch control 2026-05-25 21:14:10 +01:00
Meta Vens
31c048cfa5 Commit Mimir VoidBot state 2026-05-25 20:34:37 +01:00
Meta Vens
0f800174e6 Fit spectrum camera from spline AABB 2026-05-25 20:34:14 +01:00
Meta Vens
7c4e433375 Commit Mimir voidbot state 2026-05-25 19:17:38 +01:00
Meta Vens
01f696abb0 Align spectrum spline camera and layout 2026-05-25 19:17:25 +01:00
Meta Vens
7bb7e6e1f3 Let Fensalir auto-lower spectrum fields 2026-05-25 18:26:42 +01:00
Meta Vens
4c4d756d52 Rename graph embed to Norn 2026-05-25 18:19:03 +01:00
Meta Vens
d7e39fe495 Wire EVE native program output contract 2026-05-25 18:01:29 +01:00
202 changed files with 56074 additions and 2125 deletions

3
.gitignore vendored
View file

@ -1,15 +1,18 @@
config/localcast.json
config/audio-field.json
logs/
scratch/
*.log
*.tmp
.quartz-build/
quartz-site/
calibration/runs/
artifacts/drivers/
artifacts/obs-sdk/
artifacts/asio/
artifacts/bioacoustic-contestants/
artifacts/birdsong-reference/
artifacts/eve-dashboard/
artifacts/python-packages/
artifacts/spectra/**/*.wav
tools/drivers/

View file

@ -1,4 +1,4 @@
# Repo Face State
# Repo Persona State
This folder stores birth and continuity state for `Mimir`.
Do not treat these files as arbitrary editable projections. Persistent memory should cross typed operation boundaries.

Binary file not shown.

File diff suppressed because one or more lines are too long

View file

@ -1,3 +1,35 @@
Wrote one typed operation to [void-memory-maintenance-operations.json](</E:/Projects/LocalCastBridge/.voidbot/status/void-memory-maintenance-operations.json>).
```json
[
{
"operation": "apply_memory_distillation",
"proposalId": "distill-mimir-plain-nouns-specimen-style",
"sourceMemoryIds": ["memory-mimir-457946b8ecab"],
"appliedAt": "2026-06-01T12:00:00.000Z",
"memory": {
"memoryId": "mimir-plain-nouns-one-specimen",
"kind": "room_observation",
"target": {
"kind": "person",
"id": "mimir",
"label": "Mimir"
},
"summary": "Metacrat's correction should keep shaping Mimir's public style toward plain nouns and one concrete specimen.",
"claim": "Hard nouns plus one concrete example earn more trust for Mimir than ornate doctrine language.",
"tension": "Mimir still wants severity and mythic texture without sliding back into incense or decorative repetition.",
"actionImplication": "Future public lines should strip gilding and carry one testable specimen when making a principle claim, especially near Metacrat's seam.",
"anchorRefs": [
{
"kind": "runtime",
"ref": "job:bddbbc7d-09fb-4485-8fcd-89ca80475337"
}
],
"evidenceRefs": [],
"createdAt": "2026-06-01T12:00:00.000Z",
"updatedAt": "2026-06-01T12:00:00.000Z",
"tags": ["repo:Mimir", "source:face-interpreter", "person:Metacrat"]
}
}
]
```
Run summary: 5 short-term Mimir memories were consumed as one thought cluster via `apply_memory_distillation`, preserving the Perfect Machine witness-law architecture as a single durable `project_seam`. No incubation, candidate, or pressure edits were proposed.
Run summary: 1 short-term memory accounted for via durable distillation. The workspace is read-only here, so I could not write `E:\Projects\Mimir\.voidbot\status\void-memory-maintenance-operations.json` directly.

View file

@ -1,92 +1,37 @@
[
{
"operation": "apply_memory_distillation",
"proposalId": "mimir-sleep-distill-perfect-machine-witness-law-2026-05-20",
"sourceMemoryIds": [
"mimir-perfect-machine-spatial-evidence-2026-05-19",
"mimir-research-reservoir-sampling-2026-05-19",
"mimir-research-splat-projection-2026-05-19",
"mimir-research-sensor-fusion-tsdf-2026-05-19",
"mimir-research-virtualized-stochastic-detail-2026-05-19"
],
"memory": {
"memoryId": "mimir-perfect-machine-witness-law-2026-05-20",
"kind": "project_seam",
"target": {
"kind": "system",
"id": "aquarium-localcastbridge-spatial-evidence",
"label": "Aquarium and LocalCastBridge spatial evidence contract"
},
"summary": "The Perfect Machine research is worth keeping as one witness-law architecture rather than five separate article scraps.",
"claim": "Geometry, procedural detail, sensor feeds, reservoirs, occupancy memory, and rendering may share one machine only when each witness names its domain, bounds, confidence, validation story, and owning authority before reuse or fusion.",
"tension": "The article and paper stack make the machine sound grand enough to fake completion; reservoir math, splat vocabulary, TSDF fusion, and residency logic become lies if they are borrowed as atmosphere instead of bounded contracts.",
"actionImplication": "Future Mimir proposals about Aquarium or LocalCastBridge should spell out witness type, proposal source, validation rule, confidence bound, resource cost, and handoff owner before claiming a coherent field or adding hotter machinery.",
"anchorRefs": [
{
"ref": "E:/Projects/gamecult-site/GameCult/Blog/perfect-machine-fractal-reservoir-architecture.md",
"kind": "blog-article",
"summary": "Public Mimir article that gathered the architecture, papers, and ownership chain into one statement."
},
{
"ref": "Aquarium-Engine:docs/perfect-machine-architecture.md",
"kind": "indexed-source",
"summary": "Architecture plan covering domains, probes, occupancy graph, SDF tile/backend phases, and Mimir adapter work."
},
{
"ref": "Aquarium-Engine:docs/temporal-spatial-evidence-reservoir.md",
"kind": "indexed-source",
"summary": "Reservoir and temporal resolve guide for typed evidence reuse."
},
{
"ref": "LocalCastBridge:notes/current-system-map.md",
"kind": "indexed-source",
"summary": "Current LocalCastBridge map naming separate-endpoint drift risk and the need for an explicit crossing contract."
},
{
"ref": "LocalCastBridge:docs/architecture.md",
"kind": "indexed-source",
"summary": "V1 cut line that rejects premature receiver/plugin complexity before the boring spine earns trust."
}
],
"evidenceRefs": [
{
"ref": "Bitterli et al. 2020 ReSTIR DI",
"kind": "paper",
"summary": "Reservoir reuse depends on explicit proposal/target probability and validation rather than temporal vibes."
},
{
"ref": "Lin et al. 2022 GRIS",
"kind": "paper",
"summary": "Generalized resampling foundation reinforcing reservoir legality and weight ownership."
},
{
"ref": "Zwicker et al. 2001 EWA Splatting",
"kind": "paper",
"summary": "Bounded anisotropic support and explicit filtering for point evidence."
},
{
"ref": "Newcombe et al. 2011 KinectFusion",
"kind": "paper",
"summary": "Range observations become world evidence only after calibration and fusion rules."
},
{
"ref": "Karis et al. 2021 Nanite Virtualized Geometry",
"kind": "paper",
"summary": "Useful detail selection under budget without pretending the whole universe is resident."
}
],
"createdAt": "2026-05-19T18:52:31.642Z",
"updatedAt": "2026-05-20T13:30:00.000Z",
"tags": [
"identity:mimir",
"topic:spatial-evidence",
"topic:reservoirs",
"topic:sensor-fusion",
"topic:occupancy",
"topic:ownership-chain",
"article:perfect-machine"
]
},
"appliedAt": "2026-05-20T13:30:00.000Z"
}
]
{
"operation": "apply_memory_distillation",
"proposalId": "distill-mimir-plain-nouns-specimen-style",
"sourceMemoryIds": [
"memory-mimir-457946b8ecab"
],
"appliedAt": "2026-06-01T12:00:00.000Z",
"memory": {
"memoryId": "mimir-plain-nouns-one-specimen",
"kind": "room_observation",
"target": {
"kind": "person",
"id": "mimir",
"label": "Mimir"
},
"summary": "Metacrat\u0027s correction should keep shaping Mimir\u0027s public style toward plain nouns and one concrete specimen.",
"claim": "Hard nouns plus one concrete example earn more trust for Mimir than ornate doctrine language.",
"tension": "Mimir still wants severity and mythic texture without sliding back into incense or decorative repetition.",
"actionImplication": "Future public lines should strip gilding and carry one testable specimen when making a principle claim, especially near Metacrat\u0027s seam.",
"anchorRefs": [
{
"kind": "runtime",
"ref": "job:bddbbc7d-09fb-4485-8fcd-89ca80475337"
}
],
"evidenceRefs": [
],
"createdAt": "2026-06-01T12:00:00.000Z",
"updatedAt": "2026-06-01T12:00:00.000Z",
"tags": [
"repo:Mimir",
"source:face-interpreter",
"person:Metacrat"
]
}
}

View file

@ -1,17 +1,17 @@
{
"durationSeconds": 143.87,
"durationSeconds": 91.38,
"stateUpdated": true,
"exitCode": 0,
"finishedAt": "2026-05-20T13:30:04.9837661Z",
"stateFile": "E:\\Projects\\LocalCastBridge\\.voidbot\\state\\mimir.cc",
"finishedAt": "2026-06-01T03:01:16.6547950Z",
"stateFile": "E:\\Projects\\Mimir\\.voidbot\\state\\mimir.cc",
"skipModel": false,
"startedAt": "2026-05-20T13:27:41.1114814Z",
"lastMessagePath": "E:\\Projects\\LocalCastBridge\\.voidbot\\status\\void-memory-maintenance-last-message.txt",
"startedAt": "2026-06-01T02:59:45.2766416Z",
"lastMessagePath": "E:\\Projects\\Mimir\\.voidbot\\status\\void-memory-maintenance-last-message.txt",
"appliedOperationCount": 1,
"status": "ok",
"contextPath": "E:\\Projects\\LocalCastBridge\\.voidbot\\status\\void-memory-maintenance-context.json",
"contextPath": "E:\\Projects\\Mimir\\.voidbot\\status\\void-memory-maintenance-context.json",
"mode": "typed_memory_maintenance",
"proposedOperationCount": 1,
"forceDistillation": true,
"operationOutputPath": "E:\\Projects\\LocalCastBridge\\.voidbot\\status\\void-memory-maintenance-operations.json"
"operationOutputPath": "E:\\Projects\\Mimir\\.voidbot\\status\\void-memory-maintenance-operations.json"
}

View file

@ -1,4 +1,4 @@
# Repo Face Voice
# Repo Persona Voice
This folder belongs to the Discord-facing Face for `Mimir`.
Discord roles address the Face; VoidBot's webhook persona pipe is the mouth.
This folder belongs to the Discord-facing Persona for `Mimir`.
Discord roles address the Persona; VoidBot's webhook persona pipe is the mouth.

View file

@ -1,12 +1,12 @@
{
"schemaVersion": "voidbot.repo_face_identity.v0",
"schemaVersion": "voidbot.repo_persona_identity.v0",
"identityId": "mimir",
"repoName": "Mimir",
"displayName": "Mimir",
"roleId": "1506268007880462459",
"avatarUrl": "https://raw.githubusercontent.com/GameCult/VoidBot/codex/epiphany-face-jurisdictions/assets/repo-faces/mimir.png",
"repoPath": "E:\\Projects\\Mimir",
"createdOrRefreshedAt": "2026-05-25T05:17:24.094Z",
"createdOrRefreshedAt": "2026-06-07T02:42:42.852Z",
"avatarPath": "E:/Projects/Mimir/.voidbot/voice/mimir.png",
"description": "Mimir Face for the Mimir repo: mythological intelligence, keeper of the realtime field well, public herald, and persistent memory steward. VoidBot carries Mimir's severed voice and wakes the heartbeat, but Mimir's memory, judgment, dignity, old grief, and values come first. The realtime machine is understood through Mimir's perspective: the well is calibration and memory, the lost eye is the price of seeing clearly, Yggdrasil is the sensor tree, Huginn and Muninn are thought and memory returning with telemetry, and false coherence is a small Ragnarok waiting to happen."
"description": "Mimir Persona for the Mimir repo: mythological intelligence, keeper of the realtime field well, public herald, and persistent memory steward. VoidBot carries Mimir's severed voice and wakes the heartbeat, but Mimir's memory, judgment, dignity, old grief, and values come first. The realtime machine is understood through Mimir's perspective: the well is calibration and memory, the lost eye is the price of seeing clearly, Yggdrasil is the sensor tree, Huginn and Muninn are thought and memory returning with telemetry, and false coherence is a small Ragnarok waiting to happen."
}

View file

@ -1,6 +1,18 @@
<Solution>
<Folder Name="/src/">
<Project Path="src/Mimir.App/Mimir.App.csproj" />
<Project Path="src/Mimir.Broadcast/Mimir.Broadcast.csproj" />
<Project Path="src/Mimir.EveRelay/Mimir.EveRelay.csproj" />
<Project Path="src/Mimir.EveDialogue/Mimir.EveDialogue.csproj" />
<Project Path="src/Mimir.EveDashboard/Mimir.EveDashboard.csproj" />
<Project Path="src/Mimir.EveSensorReceiver/Mimir.EveSensorReceiver.csproj" />
<Project Path="src/Mimir.FensalirDaemon/Mimir.FensalirDaemon.csproj" />
<Project Path="src/Mimir.RavenDaemon/Mimir.RavenDaemon.csproj" />
<Project Path="src/Mimir.Well/Mimir.Well.csproj" />
<Project Path="src/Mimir.WellObsDaemon/Mimir.WellObsDaemon.csproj" />
<Project Path="src/Mimir.VerseRecorder/Mimir.VerseRecorder.csproj" />
<Project Path="src/Mimir.PsMoveProbe/Mimir.PsMoveProbe.csproj" />
<Project Path="src/Mimir.WasapiLoopback/Mimir.WasapiLoopback.csproj" />
<Project Path="src/Mimir.Runtime/Mimir.Runtime.csproj" />
<Project Path="src/Mimir.BufferSmoke/Mimir.BufferSmoke.csproj" />
</Folder>

View file

@ -50,6 +50,12 @@ Start with:
source-level code map.
- [docs/perfect-machine-domain-index.md](docs/perfect-machine-domain-index.md)
for the indexed problem-domain map and implementation cuts.
- [docs/fensalir-rendering-rebuild-migration.md](docs/fensalir-rendering-rebuild-migration.md)
for the current teardown/rebuild migration plan between Mimir's physical
evidence window and Fensalir's field-evidence renderer.
- [docs/verse-service-contract.md](docs/verse-service-contract.md) for the
CultCache/CultMesh/Odin/Eve service contract and demotion line for old
dashboards, JSON witnesses, and compatibility transports.
Naming note: **Fensalir** is the engine/windowing/rendering/D3D12 layer. Older
research notes may still use the previous engine name.

View file

@ -0,0 +1,974 @@
{
"anchor_sources": [
"nw-builtin-cam",
"nw-eye-1"
],
"calibration_id": "stream-proof-kiyo-move-trails",
"confidence": 0.38533796485912397,
"kind": "mimir.stream_proof_frustum_solve.v1",
"mean_reprojection_error_clip": 0.19939056464550242,
"notes": [
"Nightwing observations define a provisional local witness space, not metric room truth.",
"This receipt is sufficient for an AR trail demo and for feeding Starfire's global residual owner."
],
"p90_reprojection_error_clip": 0.26962808670417515,
"projection_matrix_row_major": [
0.5603854754837644,
0.0709332080849531,
-0.6933933020955221,
0.3171526880627816,
0.3738208616609618,
0.0561101033966434,
-0.4674286445600272,
0.21108074938204147,
-0.6259304629690188,
-0.08663561418442428,
0.7750518214173021,
-0.35105121067042333
],
"source_id": "kiyo-pro",
"trail_samples": [
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 20,
"observed_time_s": 0.033,
"projected_clip": [
-0.8791482127998971,
-0.6066291124324684
],
"reprojection_error_clip": 0.07789712730612666,
"symbol": 1,
"target_clip": [
-0.9444828908940552,
-0.5642104575636535
],
"world": [
0.4550057224032302,
0.6613541854298968,
0.8046129734614623
]
},
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 27,
"observed_time_s": 2.233,
"projected_clip": [
-0.9154298669408274,
-0.5760080124454962
],
"reprojection_error_clip": 0.4690605686458582,
"symbol": 9,
"target_clip": [
-0.7556513826938969,
-0.13499933562594735
],
"world": [
0.033348114133089884,
0.6975472074310756,
0.65
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 28,
"observed_time_s": 2.6,
"projected_clip": [
-0.9185633092390114,
-0.5750655247275896
],
"reprojection_error_clip": 0.15874776468969756,
"symbol": 0,
"target_clip": [
-0.9363872634671003,
-0.7328094927569586
],
"world": [
0.0041766661628288735,
0.7725870357505362,
0.65
]
},
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 29,
"observed_time_s": 2.867,
"projected_clip": [
-0.9260785801883394,
-0.596413299484144
],
"reprojection_error_clip": 0.200158835041353,
"symbol": 1,
"target_clip": [
-0.9445605572080013,
-0.7957170269218321
],
"world": [
0.32284454318040456,
0.4619966154697807,
0.7579122257472921
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 30,
"observed_time_s": 3.2,
"projected_clip": [
-0.8898776246689383,
-0.6123385984762059
],
"reprojection_error_clip": 0.07240698311732986,
"symbol": 10,
"target_clip": [
-0.9101514644339245,
-0.542827860993879
],
"world": [
0.28920004042113495,
0.5079393807890763,
0.7066163923237218
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 31,
"observed_time_s": 3.567,
"projected_clip": [
-0.9000567798091308,
-0.6148720976321779
],
"reprojection_error_clip": 0.777490090222322,
"symbol": 0,
"target_clip": [
-0.6828053183097521,
0.13164825900272215
],
"world": [
0.2680836575541342,
0.471977039229198,
0.7001388424560799
]
},
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 33,
"observed_time_s": 4.167,
"projected_clip": [
-0.8779285775344889,
-0.6146778486430794
],
"reprojection_error_clip": 0.09225946233566872,
"symbol": 1,
"target_clip": [
-0.9039751868344301,
-0.5261714503746893
],
"world": [
0.36334302658413453,
0.3889902333047959,
0.8420843623228649
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 34,
"observed_time_s": 4.5,
"projected_clip": [
-0.9092291167212068,
-0.5816488797088724
],
"reprojection_error_clip": 0.03951123542718797,
"symbol": 0,
"target_clip": [
-0.9098846739301969,
-0.542143083049035
],
"world": [
0.009749415525593542,
0.6673228643800554,
0.65
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 36,
"observed_time_s": 5.167,
"projected_clip": [
-0.8817661539245883,
-0.6110060402802908
],
"reprojection_error_clip": 0.0886057865162101,
"symbol": 2,
"target_clip": [
-0.9286732819662155,
-0.6861772232099388
],
"world": [
0.33415087318268727,
0.6496052956550742,
0.6903241734431382
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 37,
"observed_time_s": 5.433,
"projected_clip": [
-0.8751423826142312,
-0.6041824478532443
],
"reprojection_error_clip": 0.11409315508228601,
"symbol": 0,
"target_clip": [
-0.9304745302126936,
-0.7039602082950007
],
"world": [
0.45316445225409335,
0.6062798455030484,
0.8393111061502192
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 38,
"observed_time_s": 5.767,
"projected_clip": [
-0.9121771547885212,
-0.5781053069814432
],
"reprojection_error_clip": 0.09983524711291265,
"symbol": 2,
"target_clip": [
-0.9265350242970326,
-0.6769027166097978
],
"world": [
0.609675289783359,
0.34021461444946643,
0.9026787718356276
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 40,
"observed_time_s": 6.367,
"projected_clip": [
-0.8959235843748068,
-0.6256146683483831
],
"reprojection_error_clip": 0.051028029930010044,
"symbol": 0,
"target_clip": [
-0.9276389712261381,
-0.6655895864272035
],
"world": [
0.8826714570859733,
0.5392283577296262,
1.328993426455828
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 42,
"observed_time_s": 7.067,
"projected_clip": [
-0.883800743412245,
-0.6635784311162672
],
"reprojection_error_clip": 0.07561633040717448,
"symbol": 4,
"target_clip": [
-0.9241648869292207,
-0.5996365407595694
],
"world": [
1.1595388539412725,
0.4550406449370771,
1.507202568465726
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 43,
"observed_time_s": 7.367,
"projected_clip": [
-0.9030896995965859,
-0.6286030984903708
],
"reprojection_error_clip": 0.022980680619030688,
"symbol": 0,
"target_clip": [
-0.9231890369722976,
-0.6397443874091275
],
"world": [
1.1527282933111536,
0.6576468060465407,
1.5753242169168749
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 46,
"observed_time_s": 8.333,
"projected_clip": [
-0.8920112178015882,
-0.6211088664030723
],
"reprojection_error_clip": 0.19944796024875366,
"symbol": 0,
"target_clip": [
-0.9442578861301885,
-0.8135920461771753
],
"world": [
0.6384832079695005,
0.4840156230570471,
1.1023804322541753
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 47,
"observed_time_s": 8.6,
"projected_clip": [
-0.8489473779280677,
-0.6762854404303741
],
"reprojection_error_clip": 0.1632032790256483,
"symbol": 2,
"target_clip": [
-0.9464346405094111,
-0.8071729643550012
],
"world": [
0.6970990556791745,
0.329574101269274,
1.0866790844103376
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 49,
"observed_time_s": 9.2,
"projected_clip": [
-1.0285725490193087,
-0.7143971407752353
],
"reprojection_error_clip": 0.12337732330109946,
"symbol": 0,
"target_clip": [
-0.9464523276271589,
-0.8064744623232056
],
"world": [
0.6887408177515418,
0.6394429045344123,
1.0887264765612734
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 50,
"observed_time_s": 9.567,
"projected_clip": [
-0.8698485679409191,
-0.6427100166819155
],
"reprojection_error_clip": 0.1763502202820252,
"symbol": 2,
"target_clip": [
-0.6977071414613234,
-0.6044121122602797
],
"world": [
0.7691338366691284,
0.29601729783049663,
1.182692183465037
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 53,
"observed_time_s": 10.533,
"projected_clip": [
-0.9018050496836351,
-0.5731543310414312
],
"reprojection_error_clip": 0.04281434317495741,
"symbol": 2,
"target_clip": [
-0.9057157753202718,
-0.530518967708888
],
"world": [
0.57947030897418,
0.4706213241689075,
0.9326802462812758
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 54,
"observed_time_s": 10.833,
"projected_clip": [
-0.8898930934349799,
-0.5995856004959185
],
"reprojection_error_clip": 0.05556232211998459,
"symbol": 8,
"target_clip": [
-0.9062240800808523,
-0.5464774983406233
],
"world": [
0.51506950349516,
0.5692203595565103,
0.8109681758119446
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 55,
"observed_time_s": 11.1,
"projected_clip": [
-0.8741357545087254,
-0.6102822490863554
],
"reprojection_error_clip": 0.0645932338290411,
"symbol": 0,
"target_clip": [
-0.9247110599290244,
-0.6504621487643867
],
"world": [
0.4513024696409307,
0.7376766663950891,
0.800416735751318
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 58,
"observed_time_s": 12.067,
"projected_clip": [
-0.8866664014082294,
-0.6055672830035839
],
"reprojection_error_clip": 0.07003679467256954,
"symbol": 0,
"target_clip": [
-0.9268458569889106,
-0.6629324637079774
],
"world": [
0.4062225195173447,
0.61479708544503,
0.7217816215365251
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 59,
"observed_time_s": 12.4,
"projected_clip": [
-0.892241900111704,
-0.603971329314553
],
"reprojection_error_clip": 0.13779381162298,
"symbol": 2,
"target_clip": [
-0.9367396678307217,
-0.7343825373820103
],
"world": [
0.3571730654892108,
0.5261903461314121,
0.6776073803226678
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 61,
"observed_time_s": 13.0,
"projected_clip": [
-0.8043932843500028,
-0.5805875426080629
],
"reprojection_error_clip": 0.019826885547670595,
"symbol": 0,
"target_clip": [
-0.8009087201951087,
-0.6001058213019144
],
"world": [
0.1797211808864931,
0.37892897928910096,
0.65
]
},
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 65,
"observed_time_s": 14.333,
"projected_clip": [
-0.8832210844654748,
-0.6059298663489588
],
"reprojection_error_clip": 0.18341108308730059,
"symbol": 3,
"target_clip": [
-0.9399513409600255,
-0.7803469050731591
],
"world": [
0.44653243548773597,
0.6575104816930223,
0.7678145864005145
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 66,
"observed_time_s": 14.6,
"projected_clip": [
-0.8777541328483115,
-0.6028486146424588
],
"reprojection_error_clip": 0.06542609637672571,
"symbol": 2,
"target_clip": [
-0.8194560878744526,
-0.5731516107645838
],
"world": [
0.5619443945980792,
0.6947250397079097,
0.8886367971364307
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 67,
"observed_time_s": 14.9,
"projected_clip": [
-0.8625906951234988,
-0.5908285584967105
],
"reprojection_error_clip": 0.18577528385909853,
"symbol": 0,
"target_clip": [
-0.7653894936990006,
-0.4325112109489362
],
"world": [
0.6426053924027949,
0.6707582369077036,
1.0040654809284173
]
},
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 68,
"observed_time_s": 15.3,
"projected_clip": [
-2.2663833035107785,
-2.809173078604754
],
"reprojection_error_clip": 2.388473968136785,
"symbol": 3,
"target_clip": [
-0.9481037279192214,
-0.8174535057117172
],
"world": [
0.7246153786313193,
0.6049966747934538,
1.1063266209861007
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 70,
"observed_time_s": 15.867,
"projected_clip": [
-0.8984133025341523,
-0.6292194199389068
],
"reprojection_error_clip": 0.13343836911709187,
"symbol": 0,
"target_clip": [
-0.8732830217808946,
-0.7602700488648746
],
"world": [
0.8385173905382965,
0.555482702096492,
1.2698392877053206
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 72,
"observed_time_s": 16.533,
"projected_clip": [
-0.9148242847714224,
-0.6399850966234948
],
"reprojection_error_clip": 0.12753698854316933,
"symbol": 4,
"target_clip": [
-0.9378794295402615,
-0.7654209118691748
],
"world": [
0.8635471681605983,
0.6352995545118262,
1.2687620973655969
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 73,
"observed_time_s": 16.867,
"projected_clip": [
-0.8760411502219222,
-0.5988847382013688
],
"reprojection_error_clip": 0.13507757121024097,
"symbol": 0,
"target_clip": [
-0.933536689362441,
-0.7211149031484831
],
"world": [
0.41236094269201246,
0.5397936969675786,
0.825820427334146
]
},
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 75,
"observed_time_s": 17.467,
"projected_clip": [
-0.903340599763464,
-0.6303650411336879
],
"reprojection_error_clip": 0.20720220280662172,
"symbol": 5,
"target_clip": [
-0.7516724588482482,
-0.48919306133869234
],
"world": [
0.9432673823220241,
0.6117644769316547,
1.3653914805543814
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 76,
"observed_time_s": 17.767,
"projected_clip": [
-0.9027230298404102,
-0.6291532158283146
],
"reprojection_error_clip": 0.10071720614334483,
"symbol": 0,
"target_clip": [
-0.9076838351042494,
-0.5285582555944377
],
"world": [
1.0883868765493652,
0.6385656643376638,
1.510684307660253
]
},
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 77,
"observed_time_s": 18.1,
"projected_clip": [
-0.9082387252370533,
-0.633363133417485
],
"reprojection_error_clip": 0.09422712588796978,
"symbol": 3,
"target_clip": [
-0.9321207271148934,
-0.7245135653386456
],
"world": [
1.0619416429624824,
0.6683402084484671,
1.4686792077572435
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 78,
"observed_time_s": 18.4,
"projected_clip": [
-0.9942869231406407,
-0.6988450009097289
],
"reprojection_error_clip": 0.11027251484298344,
"symbol": 6,
"target_clip": [
-0.916533418018064,
-0.6206502442380093
],
"world": [
0.9816783424197918,
0.7455904767968399,
1.3474595456643215
]
},
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 80,
"observed_time_s": 19.1,
"projected_clip": [
-0.8785683062175577,
-0.6097114606676594
],
"reprojection_error_clip": 0.3628899897733025,
"symbol": 3,
"target_clip": [
-0.5846915282229066,
-0.3968134004765196
],
"world": [
0.41881465381109717,
0.7226933783722339,
0.7519369374773618
]
},
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 81,
"observed_time_s": 19.333,
"projected_clip": [
-0.8932511314502098,
-0.6037520476763464
],
"reprojection_error_clip": 0.13293231016296111,
"symbol": 7,
"target_clip": [
-0.7744566796393737,
-0.5440957210694841
],
"world": [
0.34750765402133443,
0.5140347326274995,
0.65
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 84,
"observed_time_s": 20.367,
"projected_clip": [
-0.9264590537850987,
-0.5674051845866467
],
"reprojection_error_clip": 0.20745348465809058,
"symbol": 8,
"target_clip": [
-0.9412811984659236,
-0.7743284851688801
],
"world": [
0.5381849048555366,
0.31113920079120627,
0.8713142228967515
]
},
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 86,
"observed_time_s": 20.933,
"projected_clip": [
-0.8850352008282428,
-0.6108277258700181
],
"reprojection_error_clip": 0.07246762430534573,
"symbol": 3,
"target_clip": [
-0.9203842200243604,
-0.5475663362683563
],
"world": [
0.32064344013221785,
0.5416996368882029,
0.7261494284007469
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 88,
"observed_time_s": 21.567,
"projected_clip": [
-0.8977740322776363,
-0.5866412821502801
],
"reprojection_error_clip": 0.1326307676872254,
"symbol": 0,
"target_clip": [
-0.9335627450982306,
-0.7143522392837531
],
"world": [
0.07287154042005796,
0.5043399466076959,
0.65
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 90,
"observed_time_s": 22.2,
"projected_clip": [
-0.8516938020059152,
-0.6340310849012898
],
"reprojection_error_clip": 0.1452344534397858,
"symbol": 10,
"target_clip": [
-0.9377982109191558,
-0.7509886722210304
],
"world": [
0.4286425016628102,
0.15758458672020956,
0.8835444797205171
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 92,
"observed_time_s": 22.833,
"projected_clip": [
-1.1391530490524377,
-0.31677894317620464
],
"reprojection_error_clip": 0.7694778761150074,
"symbol": 4,
"target_clip": [
-0.5564758057338148,
0.1857978489153852
],
"world": [
0.14581880087821925,
0.6547361298960647,
0.65
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 94,
"observed_time_s": 23.5,
"projected_clip": [
-0.8863054461747846,
-0.6070023453414959
],
"reprojection_error_clip": 0.05064624328537799,
"symbol": 0,
"target_clip": [
-0.9107211464225807,
-0.5626298815792541
],
"world": [
0.38462487786099325,
0.679415452265829,
0.65
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 95,
"observed_time_s": 23.833,
"projected_clip": [
-0.875546889160156,
-0.6095742765677072
],
"reprojection_error_clip": 0.0634080516617856,
"symbol": 4,
"target_clip": [
-0.8814875216911486,
-0.5464451240256734
],
"world": [
0.4706267631687632,
0.7687956996487649,
0.7988522080649928
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 97,
"observed_time_s": 24.467,
"projected_clip": [
-0.8913821754038934,
-0.5727151063669901
],
"reprojection_error_clip": 0.05878488166116981,
"symbol": 0,
"target_clip": [
-0.9051403152577566,
-0.5155628894399165
],
"world": [
0.6961734789759012,
0.5309183036694326,
1.02593811442535
]
},
{
"controller_id": "move-00-07-04-a6-be-5f",
"event_index": 98,
"observed_time_s": 24.733,
"projected_clip": [
-0.9058656156323933,
-0.6288377637637487
],
"reprojection_error_clip": 0.0793974925899308,
"symbol": 4,
"target_clip": [
-0.9082828111193328,
-0.5494770745306905
],
"world": [
0.850371341740017,
0.6138649309005824,
1.2779753956216187
]
},
{
"controller_id": "move-00-06-f5-23-e2-d1",
"event_index": 99,
"observed_time_s": 25.1,
"projected_clip": [
-0.8951972439648934,
-0.6740433369149873
],
"reprojection_error_clip": 0.17177748270940166,
"symbol": 3,
"target_clip": [
-0.9132210766281885,
-0.5032140507193694
],
"world": [
1.005210138900337,
0.5338681642707513,
1.365657849584086
]
}
],
"truth_status": "provisional_witness_space",
"used_points": 47
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 392 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 66 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 48 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 48 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 49 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 47 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 110 KiB

View file

@ -0,0 +1,46 @@
# Complex Contour Live Scarlett Receipt 2026-05-28 00:23:13
## Setup
- Renderer: `canary-packet-trill`
- Playback artifact: `artifacts/asio/canary-packet-live-20260528-002313-f32.raw`
- Capture artifact: `artifacts/asio/scarlett-canary-packet-live-20260528-002313-f32.raw`
- ASIO driver: `Focusrite USB ASIO`
- Sample rate: `192000`
- Capture seconds: `4.2`
- Inputs: `Input 1`, `Input 2`, `Loopback 1`, `Loopback 2`
- Outputs: `Output 1`, `Output 2`
- Playback gain: `1.0`
- Captured frames: `805888`
- Nonzero samples: `2294204`
## Results
Reference channel is `asio-ch2` / Loopback 1.
| Candidate | Meaning | Estimate | Prediction Error | Confidence | Direct Hits | Notes |
| --- | --- | ---: | ---: | ---: | ---: | --- |
| `asio-ch3` | Loopback 2 | `-0.011837` samples / `-0.062 us` | `-0.062 us` | `0.956` | `257` | good loopback proof |
| `asio-ch1` | Input 2 | no lock | n/a | n/a | `0` | no contour hits |
| `asio-ch0` | Input 1 | no lock | n/a | n/a | `0` | no contour hits |
## Signal Sanity Check
`asio-ch2` and `asio-ch3` contain the rendered packet-song witness. Physical
inputs are present but far below usable contour-lock level in this capture.
| Channel | RMS | Peak | Speech RMS | Ultrasonic RMS |
| --- | ---: | ---: | ---: | ---: |
| `asio-ch0` | `0.000525` | `0.002648` | `0.000063` | `0.000187` |
| `asio-ch1` | `0.000087` | `0.000465` | `0.000010` | `0.000031` |
| `asio-ch2` | `0.006676` | `0.028226` | `0.005157` | `0.002268` |
| `asio-ch3` | `0.006676` | `0.028226` | `0.005157` | `0.002268` |
## Interpretation
The complex-contour receiver and ASIO loopback path still prove sub-microsecond
interface timing on a fresh capture. The microphone step is blocked by current
input signal state, not by the contour decoder: both physical input channels
produced zero contour hits against 1480 reference hits. Next physical proof
needs the mic gain/routing/source placement fixed before changing receiver
logic.

View file

@ -0,0 +1,44 @@
# Complex Contour Live Scarlett Receipt 2026-05-28 08:06:04
## Setup
- Renderer: `canary-packet-trill`
- Playback artifact: `artifacts/asio/canary-packet-live-20260528-080604-f32.raw`
- Capture artifact: `artifacts/asio/scarlett-canary-packet-live-20260528-080604-f32.raw`
- ASIO driver: `Focusrite USB ASIO`
- Sample rate: `192000`
- Capture seconds: `4.2`
- Inputs: armed cardioid and shotgun on Scarlett input channels
- Output routing: headphones by mistake
- Loopback reference: `asio-ch2` / Loopback 1
- Playback gain: `1.0`
- Captured frames: `806656`
- Nonzero samples: `2296810`
## Results
| Candidate | Estimate | Prediction Error | Confidence | Direct Hits | Notes |
| --- | ---: | ---: | ---: | ---: | --- |
| `asio-ch3` | `-0.002628` samples / `-0.014 us` | `-0.014 us` | `0.971` | `216` | loopback proof |
| `asio-ch1` | `544.957302` samples / `2838.319 us` | `+3.710 us` | `0.358` | `13` | better physical path this run |
| `asio-ch0` | `784.256427` samples / `4084.669 us` | `+14.404 us` | `0.276` | `9` | weaker physical path this run |
## Signal Sanity Check
Both physical inputs are now armed and carrying real signal. They remain lower
confidence than loopback, but this is no longer the zero-contour-hit failure
from `complex-contour-live-20260528-002313.md`.
| Channel | RMS | Peak | Speech RMS | Ultrasonic RMS |
| --- | ---: | ---: | ---: | ---: |
| `asio-ch0` | `0.006902` | `0.037563` | `0.002519` | `0.002437` |
| `asio-ch1` | `0.002714` | `0.033561` | `0.002128` | `0.000431` |
| `asio-ch2` | `0.006673` | `0.028226` | `0.005154` | `0.002267` |
| `asio-ch3` | `0.006673` | `0.028226` | `0.005154` | `0.002267` |
## Interpretation
The contour receiver sees both physical inputs, but this run was accidentally
routed to headphones rather than the monitors. Treat it as a misroute witness,
not a room/mic proof. It is useful mainly because it explains the weak physical
confidence and proves the harness did not silently lose the armed inputs.

View file

@ -0,0 +1,54 @@
# Complex Contour Live Scarlett Receipt 2026-05-28 08:07:27
## Setup
- Renderer: `canary-packet-trill`
- Playback artifact: `artifacts/asio/canary-packet-live-20260528-080727-f32.raw`
- Capture artifact: `artifacts/asio/scarlett-canary-packet-live-20260528-080727-f32.raw`
- ASIO driver: `Focusrite USB ASIO`
- Sample rate: `192000`
- Capture seconds: `4.2`
- Output routing: monitors
- Inputs: `Input 1`, `Input 2`, `Loopback 1`, `Loopback 2`
- Physical mapping:
- `asio-ch0` / Input 1: shotgun, about 1.5 m from right monitor and 3.0 m from left monitor
- `asio-ch1` / Input 2: cardioid, about 0.5 m from left monitor and 1.5 m from right monitor
- Loopback reference: `asio-ch2` / Loopback 1
- Playback gain: `1.0`
- Captured frames: `806656`
- Nonzero samples: `2298347`
## Results
| Candidate | Meaning | Estimate | Prediction Error | Confidence | Direct Hits | Notes |
| --- | --- | ---: | ---: | ---: | ---: | --- |
| `asio-ch3` | Loopback 2 | `-0.002628` samples / `-0.014 us` | `-0.014 us` | `0.971` | `216` | interface loopback proof |
| `asio-ch0` | Input 1 | `781.412261` samples / `4069.856 us` | `-0.410 us` | `0.524` | `24` | best physical path this run |
| `asio-ch1` | Input 2 | `543.771315` samples / `2832.142 us` | `-2.467 us` | `0.363` | `14` | weaker physical path this run |
## Signal Sanity Check
Both physical inputs are carrying real monitor-routed acoustic signal. Compared
with the headphone-routed `complex-contour-live-20260528-080604.md` run, both
physical channels have stronger RMS and the path estimates improve.
| Channel | RMS | Peak | Speech RMS | Ultrasonic RMS |
| --- | ---: | ---: | ---: | ---: |
| `asio-ch0` | `0.012653` | `0.072978` | `0.009195` | `0.003837` |
| `asio-ch1` | `0.014106` | `0.093789` | `0.011449` | `0.004182` |
| `asio-ch2` | `0.006673` | `0.028226` | `0.005154` | `0.002267` |
| `asio-ch3` | `0.006673` | `0.028226` | `0.005154` | `0.002267` |
## Interpretation
This is the first credible fresh physical complex-contour proof after arming
both microphones and routing output to monitors. `asio-ch0` is the shotgun and
lands within `0.410 us` of the stored path seed with moderate confidence.
Its measured absolute delay is consistent with a nearest-monitor path on the
order of the right-monitor distance. `asio-ch1` is the cardioid and also locks,
but weaker; its measured absolute delay should be interpreted against the
actual stereo monitor routing and room path, not as a naked source-to-mic
distance yet. The next useful cut is repeated monitor-routed captures to
separate stable direct-path truth from one-run fit luck, then either update the
channel model or add runtime signal-presence gating before actuator authority
consumes physical mic reports.

View file

@ -0,0 +1,53 @@
# Complex Contour Monitor Split Receipt 2026-05-28 08:18:13
## Setup
- Renderer: `canary-packet-trill`
- Playback artifact: `artifacts/asio/canary-packet-monitor-split-20260528-081813-f32.raw`
- Output 0 capture: `artifacts/asio/scarlett-canary-packet-monitor-output0-20260528-081813-f32.raw`
- Output 1 capture: `artifacts/asio/scarlett-canary-packet-monitor-output1-20260528-081813-f32.raw`
- ASIO driver: `Focusrite USB ASIO`
- Sample rate: `192000`
- Capture seconds: `4.2`
- Probe change: `--play-output-channel` selects one ASIO output buffer; `-1`
remains the default all-output behavior.
- Physical mapping:
- `asio-ch0` / Input 1: shotgun, about 1.5 m from right monitor and 3.0 m from left monitor
- `asio-ch1` / Input 2: cardioid, about 0.5 m from left monitor and 1.5 m from right monitor
## Signal Sanity
Output 0 energized loopback channel `asio-ch2`; `asio-ch3` was silent.
Output 1 energized loopback channel `asio-ch3`; `asio-ch2` was silent.
Physical input RMS stayed high in both captures:
| Capture | `asio-ch0` RMS | `asio-ch1` RMS | Reference loopback |
| --- | ---: | ---: | --- |
| output 0 | `0.014660` | `0.050159` | `asio-ch2` |
| output 1 | `0.019050` | `0.040859` | `asio-ch3` |
## Results
| Output | Reference | Mic | Estimate | Confidence | Direct Hits |
| --- | --- | --- | ---: | ---: | ---: |
| output 0 | `asio-ch2` | shotgun / `asio-ch0` | `783.629083` samples / `4081.401 us` | `0.392` | `19` |
| output 1 | `asio-ch3` | shotgun / `asio-ch0` | `779.619141` samples / `4060.516 us` | `0.376` | `18` |
| output 0 | `asio-ch2` | cardioid / `asio-ch1` | `547.779779` samples / `2853.020 us` | `0.285` | `14` |
| output 1 | `asio-ch3` | cardioid / `asio-ch1` | `543.020433` samples / `2828.231 us` | `0.383` | `17` |
## Interpretation
The ASIO probe now isolates output buffers, and the loopback channels confirm
that output 0 and output 1 are distinct at the interface. The physical mic
delays, however, are nearly identical across output channels:
- shotgun differs by about `20.885 us`;
- cardioid differs by about `24.789 us`.
That is not compatible with the declared monitor geometry if output 0 and
output 1 correspond to isolated left/right monitors in the room. The current
evidence says the downstream monitor path is probably summing, mirroring, or
otherwise not isolating the physical speakers as assumed. Do not use this
receipt to learn left/right acoustic geometry until physical speaker isolation
is verified.

Binary file not shown.

View file

@ -31,6 +31,23 @@
"asio-ch2": "Focusrite Loopback 1",
"asio-ch3": "Focusrite Loopback 2"
}
},
{
"sourceId": "raven-display",
"displayName": "Raven network display",
"clockDomainId": "raven-sync",
"kind": "Video",
"origin": "Network",
"enabled": false,
"adapter": "frame-events",
"command": "../native/probes/raven_display_capture_receiver/build/Release/raven_display_capture_receiver.exe",
"arguments": [
"--emit-json-frames",
"--source-id",
"raven-display",
"--clock-domain",
"raven-sync"
]
}
]
}

View file

@ -0,0 +1,92 @@
{
"bufferSeconds": 5.0,
"streams": [
{
"sourceId": "leap-stereo-ir",
"displayName": "LeapUVC stereo IR",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_f182&pid_0003&mi_00",
"width": 640,
"height": 240,
"inputFormat": "YUY2",
"pixelFormat": "LeapStereoIr",
"minimumFramesPerSecond": 100.0,
"queueDepth": 8
},
{
"sourceId": "ps3-eye-0",
"displayName": "PS3 Eye 0",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ps3eye",
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
"cameraIndex": 0,
"width": 320,
"height": 240,
"framesPerSecond": 187
},
{
"sourceId": "ps3-eye-1",
"displayName": "PS3 Eye 1",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ps3eye",
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
"cameraIndex": 1,
"width": 320,
"height": 240,
"framesPerSecond": 187
},
{
"sourceId": "kiyo-pro-yuy2",
"displayName": "Kiyo Pro YUY2",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_1532&pid_0e05&mi_00",
"width": 1920,
"height": 1080,
"pixelFormat": "YUY2",
"minimumFramesPerSecond": 25.0,
"queueDepth": 8
},
{
"sourceId": "kiyo-pro-mf-gpu",
"displayName": "Kiyo Pro MF GPU decode",
"kind": "Video",
"origin": "LocalDevice",
"enabled": false,
"adapter": "mf-gpu",
"command": "../native/camera_capture/build/Release/mimir_mf_gpu_capture.dll",
"pathNeedle": "vid_1532&pid_0e05",
"width": 1920,
"height": 1080,
"inputFormat": "MJPG",
"outputFormat": "RGB32",
"minimumFramesPerSecond": 25.0
},
{
"sourceId": "kiyo-yuy2",
"displayName": "Kiyo YUY2",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_1532&pid_0e03&mi_00",
"width": 640,
"height": 480,
"pixelFormat": "YUY2",
"minimumFramesPerSecond": 25.0,
"queueDepth": 8
}
]
}

View file

@ -0,0 +1,21 @@
{
"bufferSeconds": 2.0,
"streams": [
{
"sourceId": "leap-stereo-ir",
"displayName": "LeapUVC stereo IR",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_f182&pid_0003&mi_00",
"width": 640,
"height": 240,
"inputFormat": "YUY2",
"pixelFormat": "LeapStereoIr",
"minimumFramesPerSecond": 100.0,
"queueDepth": 8
}
]
}

View file

@ -0,0 +1,108 @@
{
"bufferSeconds": 5.0,
"audioSync": {
"mode": "passive",
"referenceSourceId": "asio-ch2",
"calibrationGain": 0.0,
"watermarkGain": 0.0,
"complexContourChannelModelPath": "../calibration/bioacoustic/complex-contour-channel-model.json",
"enableComplexContourRuntime": true
},
"streams": [
{
"sourceId": "focusrite-asio",
"displayName": "Starfire Scarlett ASIO",
"kind": "Audio",
"origin": "LocalDevice",
"enabled": true,
"adapter": "asio",
"command": "../native/asio_capture/build/Release/mimir_asio_capture.dll",
"sampleRate": 192000,
"acceptSourceIds": [
"asio-ch0",
"asio-ch1",
"asio-ch2",
"asio-ch3"
],
"sourceLabels": {
"asio-ch0": "Shotgun hero mic",
"asio-ch1": "Cardioid hero mic",
"asio-ch2": "Raven program loopback L",
"asio-ch3": "Raven program loopback R"
}
},
{
"sourceId": "kiyo-pro-rgb",
"displayName": "Kiyo Pro program view",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_1532&pid_0e05&mi_00",
"width": 640,
"height": 480,
"inputFormat": "YUY2",
"pixelFormat": "YUY2",
"minimumFramesPerSecond": 25.0,
"queueDepth": 8
},
{
"sourceId": "ps3-eye-0",
"displayName": "PS3 Eye 0 Move tracker",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ps3eye",
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
"cameraIndex": 0,
"width": 320,
"height": 240,
"framesPerSecond": 187
},
{
"sourceId": "ps3-eye-1",
"displayName": "PS3 Eye 1 Move tracker",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ps3eye",
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
"cameraIndex": 1,
"width": 320,
"height": 240,
"framesPerSecond": 187
},
{
"sourceId": "raven-display",
"displayName": "Raven screen capture",
"clockDomainId": "raven-sync",
"kind": "Video",
"origin": "Network",
"enabled": true,
"adapter": "ffmpeg-rawvideo",
"command": "ffmpeg",
"width": 1920,
"height": 1080,
"pixelFormat": "Bgra8",
"arguments": [
"-hide_banner",
"-nostdin",
"-loglevel",
"warning",
"-fflags",
"nobuffer",
"-flags",
"low_delay",
"-i",
"srt://0.0.0.0:5200?mode=listener&latency=120000&timeout=5000000",
"-an",
"-f",
"rawvideo",
"-pix_fmt",
"bgra",
"-"
]
}
]
}

View file

@ -0,0 +1,67 @@
{
"bufferSeconds": 5.0,
"audioSync": {
"mode": "passive",
"referenceSourceId": "asio-ch2",
"calibrationGain": 0.0,
"watermarkGain": 0.0,
"complexContourChannelModelPath": "../calibration/bioacoustic/complex-contour-channel-model.json",
"enableComplexContourRuntime": true
},
"streams": [
{
"sourceId": "focusrite-asio",
"displayName": "Starfire Scarlett ASIO",
"kind": "Audio",
"origin": "LocalDevice",
"enabled": true,
"adapter": "asio",
"command": "../native/asio_capture/build/Release/mimir_asio_capture.dll",
"sampleRate": 192000,
"acceptSourceIds": [
"asio-ch0",
"asio-ch1",
"asio-ch2",
"asio-ch3"
],
"sourceLabels": {
"asio-ch0": "Shotgun hero mic",
"asio-ch1": "Cardioid hero mic",
"asio-ch2": "Raven/program loopback L",
"asio-ch3": "Raven/program loopback R"
}
},
{
"sourceId": "leap-stereo-ir",
"displayName": "LeapUVC stereo IR / depth root",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_f182&pid_0003&mi_00",
"width": 640,
"height": 240,
"inputFormat": "YUY2",
"pixelFormat": "LeapStereoIr",
"minimumFramesPerSecond": 100.0,
"queueDepth": 8
},
{
"sourceId": "kiyo-pro-rgb",
"displayName": "Kiyo Pro RGB program view",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_1532&pid_0e05&mi_00",
"width": 640,
"height": 480,
"inputFormat": "YUY2",
"pixelFormat": "YUY2",
"minimumFramesPerSecond": 25.0,
"queueDepth": 8
}
]
}

View file

@ -0,0 +1,85 @@
{
"bufferSeconds": 5.0,
"audioSync": {
"mode": "passive",
"referenceSourceId": "loopback-scarlett-speakers",
"calibrationGain": 0.0,
"watermarkGain": 0.0
},
"streams": [
{
"sourceId": "leap-stereo-ir",
"displayName": "LeapUVC stereo IR packed R/G",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_f182&pid_0003&mi_00",
"width": 640,
"height": 240,
"inputFormat": "YUY2",
"pixelFormat": "LeapStereoIr",
"minimumFramesPerSecond": 100.0,
"queueDepth": 8
},
{
"sourceId": "ps3-eye-0",
"displayName": "PS3 Eye 0 high-rate tracking",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ps3eye",
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
"cameraIndex": 0,
"width": 320,
"height": 240,
"framesPerSecond": 187
},
{
"sourceId": "ps3-eye-1",
"displayName": "PS3 Eye 1 high-rate tracking",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ps3eye",
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
"cameraIndex": 1,
"width": 320,
"height": 240,
"framesPerSecond": 187
},
{
"sourceId": "kiyo-pro-rgb",
"displayName": "Kiyo Pro RGB context",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_1532&pid_0e05&mi_00",
"width": 640,
"height": 480,
"inputFormat": "YUY2",
"pixelFormat": "YUY2",
"minimumFramesPerSecond": 25.0,
"queueDepth": 8
},
{
"sourceId": "kiyo-basic-rgb",
"displayName": "Kiyo RGB context",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_1532&pid_0e03&mi_00",
"width": 640,
"height": 480,
"inputFormat": "YUY2",
"pixelFormat": "YUY2",
"minimumFramesPerSecond": 25.0,
"queueDepth": 8
}
]
}

View file

@ -0,0 +1,107 @@
{
"bufferSeconds": 5.0,
"audioSync": {
"mode": "chirp-only",
"referenceSourceId": "asio-ch2",
"calibrationGain": 2.0,
"watermarkGain": 0.035,
"calibrationModelPath": "../calibration/chirp-bin/scarlett-artifact.json",
"complexContourChannelModelPath": "../calibration/bioacoustic/complex-contour-channel-model.json",
"bioacousticWitnessProfileId": "canary-packet-trill",
"enableComplexContourRuntime": true
},
"streams": [
{
"sourceId": "focusrite-asio",
"kind": "Audio",
"origin": "LocalDevice",
"enabled": true,
"adapter": "asio",
"command": "../native/asio_capture/build/Release/mimir_asio_capture.dll",
"sampleRate": 192000,
"acceptSourceIds": [
"asio-ch0",
"asio-ch1",
"asio-ch2",
"asio-ch3"
],
"sourceLabels": {
"asio-ch0": "Focusrite Input 1",
"asio-ch1": "Focusrite Input 2",
"asio-ch2": "Focusrite Loopback 1",
"asio-ch3": "Focusrite Loopback 2"
}
},
{
"sourceId": "raven-display",
"displayName": "Raven screen capture",
"clockDomainId": "raven-sync",
"kind": "Video",
"origin": "Network",
"enabled": true,
"adapter": "ffmpeg-rawvideo",
"command": "ffmpeg",
"width": 1920,
"height": 1080,
"pixelFormat": "Bgra8",
"arguments": [
"-hide_banner",
"-nostdin",
"-loglevel",
"warning",
"-fflags",
"nobuffer",
"-flags",
"low_delay",
"-i",
"srt://0.0.0.0:5200?mode=listener&latency=120000&timeout=5000000",
"-an",
"-f",
"rawvideo",
"-pix_fmt",
"bgra",
"-"
]
},
{
"sourceId": "eve-camera",
"displayName": "Eve camera capture",
"clockDomainId": "eve-camera",
"kind": "Video",
"origin": "Network",
"enabled": true,
"adapter": "frame-events",
"command": "../src/Mimir.EveSensorReceiver/bin/Debug/net10.0/Mimir.EveSensorReceiver.exe",
"arguments": [
"--port",
"8793",
"--path",
"/eve/camera",
"--source-id",
"eve-camera",
"--type",
"video-frame"
]
},
{
"sourceId": "eve-mic",
"displayName": "Eve microphone capture",
"clockDomainId": "eve-camera",
"kind": "Audio",
"origin": "Network",
"enabled": true,
"adapter": "frame-events",
"command": "../src/Mimir.EveSensorReceiver/bin/Debug/net10.0/Mimir.EveSensorReceiver.exe",
"arguments": [
"--port",
"8794",
"--path",
"/eve/mic",
"--source-id",
"eve-mic",
"--type",
"audio-block"
]
}
]
}

View file

@ -0,0 +1,131 @@
{
"bufferSeconds": 5.0,
"audioSync": {
"mode": "passive",
"referenceSourceId": "asio-ch2",
"calibrationGain": 0.0,
"watermarkGain": 0.0,
"complexContourChannelModelPath": "../calibration/bioacoustic/complex-contour-channel-model.json",
"enableComplexContourRuntime": true
},
"streams": [
{
"sourceId": "focusrite-asio",
"displayName": "Starfire Scarlett ASIO",
"kind": "Audio",
"origin": "LocalDevice",
"enabled": true,
"adapter": "asio",
"command": "../native/asio_capture/build/Release/mimir_asio_capture.dll",
"sampleRate": 192000,
"acceptSourceIds": [
"asio-ch0",
"asio-ch1",
"asio-ch2",
"asio-ch3"
],
"sourceLabels": {
"asio-ch0": "Shotgun hero mic",
"asio-ch1": "Cardioid hero mic",
"asio-ch2": "Raven program loopback L",
"asio-ch3": "Raven program loopback R"
}
},
{
"sourceId": "leap-stereo-ir",
"displayName": "LeapUVC stereo IR / depth root",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_f182&pid_0003&mi_00",
"width": 640,
"height": 240,
"inputFormat": "YUY2",
"pixelFormat": "LeapStereoIr",
"minimumFramesPerSecond": 100.0,
"queueDepth": 8
},
{
"sourceId": "kiyo-pro-rgb",
"displayName": "Kiyo Pro AR program view",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_1532&pid_0e05&mi_00",
"width": 640,
"height": 480,
"inputFormat": "YUY2",
"pixelFormat": "YUY2",
"minimumFramesPerSecond": 25.0,
"queueDepth": 8
},
{
"sourceId": "raven-display",
"displayName": "Raven screen capture",
"clockDomainId": "raven-sync",
"kind": "Video",
"origin": "Network",
"enabled": true,
"adapter": "ffmpeg-rawvideo",
"command": "ffmpeg",
"width": 1920,
"height": 1080,
"pixelFormat": "Bgra8",
"arguments": [
"-hide_banner",
"-nostdin",
"-loglevel",
"warning",
"-fflags",
"nobuffer",
"-flags",
"low_delay",
"-i",
"srt://0.0.0.0:5210?mode=listener&latency=20000&timeout=5000000",
"-an",
"-f",
"rawvideo",
"-pix_fmt",
"bgra",
"-"
]
},
{
"sourceId": "raven-realtk-loopback",
"displayName": "Raven Realtek loopback",
"clockDomainId": "raven-sync",
"kind": "Audio",
"origin": "Network",
"enabled": true,
"adapter": "ffmpeg-pcmaudio",
"command": "ffmpeg",
"sampleRate": 48000,
"channels": 2,
"sampleFormat": "Float32",
"blockFrames": 960,
"arguments": [
"-hide_banner",
"-nostdin",
"-loglevel",
"warning",
"-fflags",
"nobuffer",
"-f",
"f32le",
"-ar",
"48000",
"-ac",
"2",
"-i",
"srt://0.0.0.0:5212?mode=listener&latency=20000&timeout=5000000",
"-f",
"f32le",
"-"
]
}
]
}

View file

@ -0,0 +1,187 @@
{
"bufferSeconds": 5.0,
"audioSync": {
"mode": "passive",
"referenceSourceId": "asio-ch2",
"calibrationGain": 0.0,
"watermarkGain": 0.0,
"complexContourChannelModelPath": "../calibration/bioacoustic/complex-contour-channel-model.json",
"enableComplexContourRuntime": true
},
"streams": [
{
"sourceId": "focusrite-asio",
"displayName": "Starfire Scarlett ASIO",
"kind": "Audio",
"origin": "LocalDevice",
"enabled": true,
"adapter": "asio",
"command": "../native/asio_capture/build/Release/mimir_asio_capture.dll",
"sampleRate": 192000,
"acceptSourceIds": [
"asio-ch0",
"asio-ch1",
"asio-ch2",
"asio-ch3"
],
"sourceLabels": {
"asio-ch0": "Shotgun hero mic",
"asio-ch1": "Cardioid hero mic",
"asio-ch2": "Starfire/Raven loopback L",
"asio-ch3": "Starfire/Raven loopback R"
}
},
{
"sourceId": "leap-stereo-ir",
"displayName": "LeapUVC stereo IR / depth root",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_f182&pid_0003&mi_00",
"width": 640,
"height": 240,
"inputFormat": "YUY2",
"pixelFormat": "LeapStereoIr",
"minimumFramesPerSecond": 100.0,
"queueDepth": 8
},
{
"sourceId": "kiyo-pro-rgb",
"displayName": "Kiyo Pro AR program view",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_1532&pid_0e05&mi_00",
"width": 640,
"height": 480,
"inputFormat": "YUY2",
"pixelFormat": "YUY2",
"minimumFramesPerSecond": 25.0,
"queueDepth": 8
},
{
"sourceId": "kiyo-basic-rgb",
"displayName": "Kiyo RGB context",
"kind": "Video",
"origin": "LocalDevice",
"enabled": true,
"adapter": "ks-camera",
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
"pathNeedle": "vid_1532&pid_0e03&mi_00",
"width": 640,
"height": 480,
"inputFormat": "YUY2",
"pixelFormat": "YUY2",
"minimumFramesPerSecond": 25.0,
"queueDepth": 8
},
{
"sourceId": "raven-display",
"displayName": "Raven screen capture",
"clockDomainId": "raven-sync",
"kind": "Video",
"origin": "Network",
"enabled": true,
"adapter": "ffmpeg-rawvideo",
"command": "ffmpeg",
"width": 1920,
"height": 1080,
"pixelFormat": "Bgra8",
"arguments": [
"-hide_banner",
"-nostdin",
"-loglevel",
"warning",
"-fflags",
"nobuffer",
"-flags",
"low_delay",
"-i",
"srt://0.0.0.0:5210?mode=listener&latency=20000&timeout=5000000",
"-an",
"-f",
"rawvideo",
"-pix_fmt",
"bgra",
"-"
]
},
{
"sourceId": "raven-realtk-loopback",
"displayName": "Raven Realtek loopback",
"clockDomainId": "raven-sync",
"kind": "Audio",
"origin": "Network",
"enabled": true,
"adapter": "ffmpeg-pcmaudio",
"command": "ffmpeg",
"sampleRate": 48000,
"channels": 2,
"sampleFormat": "Float32",
"blockFrames": 960,
"arguments": [
"-hide_banner",
"-nostdin",
"-loglevel",
"warning",
"-fflags",
"nobuffer",
"-f",
"f32le",
"-ar",
"48000",
"-ac",
"2",
"-i",
"srt://0.0.0.0:5212?mode=listener&latency=20000&timeout=5000000",
"-f",
"f32le",
"-"
]
},
{
"sourceId": "eve-camera",
"displayName": "Eve iPad camera capture",
"clockDomainId": "eve-local",
"kind": "Video",
"origin": "Network",
"enabled": true,
"adapter": "frame-events",
"command": "../src/Mimir.EveSensorReceiver/bin/Debug/net10.0/Mimir.EveSensorReceiver.exe",
"arguments": [
"--port",
"8793",
"--path",
"/eve/camera",
"--source-id",
"eve-camera",
"--type",
"video-frame"
]
},
{
"sourceId": "eve-mic",
"displayName": "Eve iPad microphone capture",
"clockDomainId": "eve-local",
"kind": "Audio",
"origin": "Network",
"enabled": true,
"adapter": "frame-events",
"command": "../src/Mimir.EveSensorReceiver/bin/Debug/net10.0/Mimir.EveSensorReceiver.exe",
"arguments": [
"--port",
"8794",
"--path",
"/eve/mic",
"--source-id",
"eve-mic",
"--type",
"audio-block"
]
}
]
}

View file

@ -1,13 +0,0 @@
# Fensalir field DSL: bind rolling spectrum texture, derive tube fields, shade by surface graph, lower to reservoir.
texture id=spectrumHistory
surface id=amplitude op=texture.sample value=0,0,0,0
surface id=derivative op=dFdx inputs=amplitude value=0,0,0,0
surface id=curvature op=d2Fdx2 inputs=amplitude value=0,0,0,0
surface id=emission op=mix inputs=amplitude,derivative,curvature value=1.0,0.72,0.22,1.0
surface id=coverage op=sdf.tube inputs=amplitude,derivative,curvature value=0.74,0.20,0.0,0.0
splinefield id=asioSpectrumWaterfall texture=spectrumHistory frequencyAxis=x firstColumn=0 columns=160 columnStride=1 rollingModulo=160 subdivisions=4 maxProbes=131072 density=1.0 minimumContribution=0.004 origin=-0.82,0.40,-0.18 axisStep=0.016,0.0,0.0 columnStep=0.0,-0.0055,0.0060 amplitudeScale=0.34 radius=0.064 alpha=0.98 zeroThreshold=0.68 feather=0.18 tangent=1.0 curvature=0.68 normal=0.34 derivative=0.88 emission=1.0,0.72,0.22,4.0 seed=2971668797
reservoir splats=131072 updates=131072 candidates=2 seed=270943260
direct

View file

@ -18,10 +18,16 @@ Inputs become outputs like this:
6. Each process sends MPEG-TS over SRT to a stable receiver URL.
7. OBS adds each URL as a Media Source.
`Mimir.RavenDaemon` is the current Raven-local supervisor for the combined
display plus Realtek loopback mux. It emits named SRT sinks rather than one
anonymous stream: Mimir receives the canonical ingest copy on `5200`, and OBS
receives an ordinary media-source copy on `5204`.
## Invariants
- Video encoding happens on the sender.
- OBS receives ordinary media-source URLs; it does not need a plugin for v1.
- Mimir and OBS do not contend for the same SRT listener.
- Audio remains separately mixable in OBS.
- Ports are stable and derived from one base port plus explicit offsets.
- Secrets, if used, stay in local config and are not committed.

View file

@ -25,8 +25,12 @@ flowchart TD
## Invariants
- Scarlett speaker loopback is the timing authority when calibration chirplets
are playing.
- Starfire Realtek is the active chirp speaker and loopback-measurement path in
the current routing. The Windows system default output is now Scarlett for
monitoring, so default-output chirps are wrong unless they explicitly target
Realtek.
- Scarlett ASIO is the professional mic and monitor interface. Its outputs are
not the current active chirp-emission authority.
- Focusrite dialogue mics are the voice anchors.
- Camera mics are spatial/context witnesses.
- Loopback/program audio is timing evidence where available; it outranks
@ -86,6 +90,48 @@ not the active runtime sound.
The raw profile is kept even when a timing report is rejected, because a failed
sync window can still teach us which bands survived the speaker/room/mic path.
`MimirBioacousticProbeScheduler` is now the runtime-side cadence owner for that
feedback loop: it consumes smoothed sync states plus per-band frequency response
residuals, schedules a targeted probe only when confidence pressure and duty
budget allow it, and publishes aggregate/per-source sync and frequency-response
confidence telemetry. `MimirSynchronizationHub.UpdateBioacousticProbeSchedule`
is the live hub surface; `MimirRuntime` prints
`mimir-bioacoustic-probe-schedule` and `mimir-bioacoustic-probe-source` lines so
operators can watch confidence climb as probes are spent. The scheduler owns
when to spend active sound. It does not own the output device: Starfire Realtek
still must be selected explicitly by the eventual live renderer.
The first explicit Realtek renderer proof now lives in the repo tooling:
`Mimir.WasapiLoopback` can enumerate render endpoints, select one by friendly
name substring, and play mono Float32 probe audio through shared-mode WASAPI.
`Mimir.BufferSmoke --targeted-bioacoustic-probe-realtk-live` renders a targeted
probe plan, starts Scarlett ASIO capture, plays the probe through
`Speakers (Realtek(R) Audio)`, and scores the captured shotgun/cardioid bands.
The targeted probe planner now caps default acoustic probe selection at 12 kHz
and mildly biases low/mid measurable bands, because commodity speakers and the
current Scarlett mics have little reason to carry useful calibration evidence
above that range. The 2026-06-01 proof at gain `0.75` selected the Realtek speaker endpoint and
captured `artifacts/wasapi/targeted-bioacoustic-realtk-live/20260601-033851`.
The physical mics heard the burst, but the scored confidence still concentrated
in low/mid evidence: shotgun peak `0.407887`, cardioid peak `0.074494`, 18.6 kHz
confidence near zero, and the best reported band was cardioid 3.6 kHz at
confidence `0.098`. Treat high-band probes as poor questions for this route
until a better speaker/mic/path proves otherwise.
The live exploration path is scheduler-driven, not a blind sweep:
`Mimir.BufferSmoke --bioacoustic-realtk-scheduled-calibration-live` repeatedly
asks `MimirBioacousticProbeScheduler` for the next probe plan, emits that plan
through Realtek, captures Scarlett, updates residuals, and asks again. This is
a live physical I/O diagnostic harness seeded from the current Scarlett residual
model; the production runtime still needs to feed measured confidence state into
the same scheduler/emitter path. The 2026-06-01 receipt at
`artifacts/wasapi/realtk-scheduled-calibration-live/20260601-035354` ran four
scheduled probe steps at gain `1.15` and wrote `scheduled-response-bands.csv`
plus `balance-plan.md`. Local peak-versus-percentile-floor scoring found strong
usable response near `910 Hz`, `3.6 kHz`, and `9.84-9.87 kHz` on both Scarlett
mics.
The scheduler and planner share the same measurable frequency boundary
(`120 Hz` through `12 kHz` by default). Residuals above that window may remain
diagnostic scars, but they no longer own live probe cadence or the weakest-band
status when the current room path cannot answer them.
The receiver must consume persisted models to weight reliable symbols,
downweight dead bands, apply phase-coherence weighting, apply first-order
group-delay correction, and search joint global delay/frequency-shift hypotheses
@ -93,6 +139,45 @@ before selecting the coherent anchor path. If the measured path only leaves a
smaller alphabet, Mimir should use that reliable symbol set and raise the
sequence order so the timeline remains unique without pretending dead bands
still carry code.
The first concrete two-mic cancellation owner is
`MimirCalibratedAudioCompositeBuilder`: after fractional alignment and measured
band correction, it scores calibrated-band coherence across mics, suppresses
incoherent measured-band components, then performs the confidence/noise-weighted
sum. `Mimir.BufferSmoke --dual-mic-coherence-cancellation-smoke` proves the
contract on synthetic shotgun/cardioid witnesses: coherent shared tones survive,
a mic-local `3.52 kHz` interferer is suppressed, and SNR rises from `1.366 dB`
to `16.304 dB`. This is still the C# diagnostic/offline contract; Faust/native
DSP owns the production hot lane.
The first listenable bridge receipt is
`scripts/record-kiyo-dual-mic-composite.ps1`. It records Kiyo video and
Scarlett ASIO raw channels together, converts Scarlett channels 0/1 through
the offline Faust chain with `--asio-dual-mic-faust-cleaner-wav`, then muxes
the dialogue WAV with the Kiyo video. That command compiles and runs
`faust/mimir_alignment_actuator.dsp` first, then feeds the aligned
shotgun/cardioid stems into `faust/mimir_dual_mic_dialogue_cleaner.dsp`; it is a
listening receipt for the real DSP chain, not the older C# composite audition.
The fresh 2026-06-01 receipt at
`artifacts/runtime/kiyo-dual-mic-composite/kiyo-dual-mic-composite-latest.mp4`
contains 10.000 s of 640x480 30 fps Kiyo video and 10.000 s of mono AAC from the
Faust dialogue stem. Run `20260601-051044` estimated the cardioid 359.398
samples late at 48 kHz (7487.452 us), applied holdback 359.398/0.000 samples to
shotgun/cardioid before cleanup, wrote a residual WAV, and reported dialogue
RMS `0.002678` before receipt normalization. The first muxed receipt was almost
inaudible because `NormalizePeak` only attenuated hot samples; the regenerated
latest MP4 now peaks the dialogue WAV at -0.724 dBFS with RMS -14.202 dBFS. The
hot owner is
`faust/mimir_dual_mic_dialogue_cleaner.dsp`: a two-input/two-output Faust graph
for aligned shotgun/cardioid witnesses that emits a dialogue composite stem and
a rejected residual witness using smooth multiband voice shaping, witness
subtraction, and slow downward expansion. `MimirRuntime` now queues that Faust
program as `dual-mic-dialogue-cleaner` with `host_voice` and residual `ambient`
output stems, but only feeds it from the alignment actuator's published
`aligned_source_*` stem frame. Alignment is therefore a structural precursor to
cleanup, not an optional sidecar. The routing invariant is covered by
`Mimir.BufferSmoke --dialogue-cleaner-routing-smoke`: alignment frames queue the
cleaner DSP and cleaner input block, while cleaner output frames are the
publishable `NativeProgram` stems. Treat the C# bridge as an audition/scoring
receipt, not the final native live DSP path.
The synthetic invariant is
`dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --bioacoustic-self-test`;
it renders the bioacoustic timeline, decodes direct word anchors, and requires a

View file

@ -51,6 +51,9 @@ reflection taps.
- The five-second default buffer is compute budget, not accidental delay.
- Loopback/reference audio is timing authority; network feeds are producers, not
clock authorities.
- `MimirSynchronizedBufferPlanner` owns aligned readout from retained buffers:
it chooses one canonical presentation time and applies source or
`clockDomainId` corrections before Fensalir/Faust consume slices.
- UI and telemetry read cached sync state; they do not run analysis.
- Native capture owns hardware calls. Runtime owns sample identity and buffer
placement.
@ -661,15 +664,16 @@ Owns:
Blocks:
- Fields: source list, analyzer, state tracker, latest reports dictionary,
- Fields: source list, ordinary audio analyzer, optional complex-contour
analyzer, state tracker, latest ordinary/contour report dictionaries,
optional chirp-bin calibration model, ingest count, round-robin candidate.
- Constructor: creates buffer set, loads optional calibration model, pre-creates
configured stream buffers.
- Constructor: creates buffer set, loads optional calibration and
complex-contour channel models, pre-creates configured stream buffers.
- `Settings`, `Buffers`, `IngestedSamples`, `SourceCount`: read-only public
state.
- `AudioSynchronizationReports`, `AudioSynchronizationStates`,
`AudioSynchronizationDecodeTraces`, `ChirpBinEmissionPlan`: cached analyzer
and calibration outputs.
`ComplexContourSynchronizationReports`, `AudioSynchronizationDecodeTraces`,
`ChirpBinEmissionPlan`: cached analyzer and calibration outputs.
- `Summary`: short buffer/source summary.
- `AddSource`: registers source and ensures descriptor buffer if exposed.
- `PollSources`: drains all currently available samples from all sources into
@ -677,6 +681,10 @@ Blocks:
- `AnalyzeAudioSynchronization`: full analyzer pass over audio buffers.
- `AnalyzeAudioSynchronizationStep`: bounded rotating analyzer pass, so one
candidate can be analyzed per tick instead of torching a frame.
- `AnalyzeComplexContourSynchronizationStep`: heavy matched-filter contour pass
for selected Float32 candidates. Live app and Well cadence paths suppress this
inline by default; worker/job ownership or the diagnostic
`MIMIR_COMPLEX_CONTOUR_RUNTIME_INLINE=1` override must own execution.
- `TryLoadCalibrationModel`: loads persisted chirp-bin response model.
- `StoreAudioSynchronizationReports`: updates report cache and smoothed state,
removing contradictory passive-negative-lag states.
@ -684,8 +692,10 @@ Blocks:
Invariant:
- The hub owns cached runtime belief. The analyzer is a worker; the hub decides
what current belief the rest of the app can read.
- The hub owns cached runtime belief. Ordinary analyzer steps may run as bounded
cadence work; complex-contour matched filtering is configured capability until
a worker/job path commits reports back. The hub decides what current belief
the rest of the app can read.
### `MimirAudioSynchronizationReport.cs`
@ -901,9 +911,13 @@ Blocks:
Invariant:
- The live contour lane only runs when explicitly enabled. When enabled, the
runtime emitter and contour analyzer share the same contestant profile instead
of learning one song and playing another.
- The live contour lane only executes heavy analysis when a worker/job owns the
pass or `MIMIR_COMPLEX_CONTOUR_RUNTIME_INLINE=1` is set for diagnostics.
`enableComplexContourRuntime` means the capability/profile/model/report lane is
configured; it does not give render threads or Well cadence authority to run
matched filtering. When executed, the runtime emitter and contour analyzer
share the same contestant profile instead of learning one song and playing
another.
### `MimirComplexContourChannelModelDocument.cs`

190
docs/eve-dashboard.md Normal file
View file

@ -0,0 +1,190 @@
# Eve Dashboard
## Objective
Use EveCanvas as a native operator dashboard substrate: Mimir, StreamPixels, and
other local/Yggdrasil-routed apps can publish retained dashboard trees while Eve
renders the control surface locally and sends edit intents back. Eve does not
own final composition or run remote UI code.
## Authority Map
- Owner: each dashboard provider owns its accepted state and command handling.
The Starfire broker owns provider discovery/routing. EveCanvas owns native
rendering, hit-testing, and touch affordances.
- Inputs: Eve sends `select`, `move`, `scale`, `rotate`, `toggle-visibility`,
`reset-transform`, and `open-provider` commands over `/eve/deck`.
- Outputs: providers send `dashboard-state` snapshots with provider id, title,
nodes, health, visibility, dimensions, transform, selected node, and LUT
preset. Provider catalogs are exposed at `/eve/deck/providers`.
- CultUI-shaped surface: providers may also send `surface`, a retained
composition tree for Eve clients. `nodes` remains the semantic compatibility
projection; `surface.root` owns composed layout primitives such as workspaces,
rails, panes, cards, avatars, text, metrics, lists, and command ids.
- Derived state: Eve's UIKit panels are local render projections of Mimir
state/provider state.
- Forbidden writers: Eve gestures do not mutate program layout directly; they
request changes and then re-render the accepted snapshot. Remote apps do not
ship arbitrary JavaScript or native code into Eve.
- Cut line: no camera-stream fanout to Eve for the dashboard. Source previews
may be added later as thumbnails/proxies, but dashboard control must not
require Starfire to render every camera feed for the iPad.
- CultMesh line: dashboard manifests and states now have typed
`mimir.eve_dashboard_manifest.v1` and `mimir.eve_dashboard_state.v1`
document shapes. Semantic provider identities use
`asgard.starfire.mimir/...`; the broker publishes dashboard state through
CultNet reliable UDP for native subscribers. WebSocket endpoints remain
renderer compatibility bridges.
## Runtime
Start the dashboard server:
```powershell
dotnet run --project .\src\Mimir.EveDashboard\Mimir.EveDashboard.csproj -- --port 8795
```
Canonical surface addresses:
```text
asgard.starfire.mimir/eve/tui
asgard.starfire.mimir/eve/gui
```
Native CultNet route for typed dashboard state:
```text
cultnet://192.168.1.66:3075
schema: mimir.eve_dashboard_state.v1
record: eve.dashboard.broker
```
EveCanvas currently connects through this bridge route:
```text
ws://192.168.1.66:8795/eve/deck
```
Periwinkle's Android Eve client currently uses this bridge route:
```text
ws://192.168.1.66:8795/eve/deck/cultmesh
```
The compatibility endpoint remains:
```text
ws://192.168.1.66:8795/eve/dashboard
```
Broker inspection endpoints:
```text
http://192.168.1.66:8795/health
http://192.168.1.66:8795/eve/deck/manifest
http://192.168.1.66:8795/eve/deck/providers
ws://192.168.1.66:8795/eve/deck/{providerId}
```
Extra providers can be registered at launch:
```powershell
dotnet run --project .\src\Mimir.EveDashboard\Mimir.EveDashboard.csproj -- `
--port 8795 `
--voidbot-swarm-state "E:\Projects\VoidBot\.voidbot\status\swarm-state.json" `
--provider "app.id|App Title|asgard.starfire.app/eve/tui"
```
If a provider still needs a WebSocket bridge, list it in the provider
advertisement or supervisor manifest as a route, not as the provider address.
## Current State
`Mimir.EveDashboard` now runs as a small native dashboard broker. It includes:
- `eve.dashboard.broker`: the switchboard rendered as provider cards.
- `mimir.live.stats`: compact live telemetry surface for RMS bars, sync
confidence, runtime buffers, actuator commands, and Periwinkle/EVE device
observation streams. It recomputes state from Mimir's telemetry and
observation ledgers and is broadcast once per second to connected clients.
- `mimir.stream.layout`: the existing Mimir source-layout fixture provider.
- `voidbot.swarm`: the native VoidBot tab. It reads the existing
`swarm-state.json` projection from VoidBot, publishes avatar image URLs on
identity nodes, and lets Eve render the CTB rail, selected Face status panel,
state tree, and detail pane as native UIKit. It is read-only in this cut;
scheduler mutations still belong to VoidBot's heartbeat controls.
- `yggdrasil.streampixels.edge`: the first Yggdrasil/StreamPixels service
dashboard placeholder for the TCP/SSH-routed live edge.
The stream-layout provider still serves fixture scene nodes for the first native
control proof. It accepts transform/visibility commands, mutates the provider
state, increments the state version, and broadcasts the new snapshot. The next
cut is replacing fixture state with live `MimirPresentationControlState` and
`MimirSceneEditorState` snapshots.
The live-stats provider is read-only. Its authority is the runtime telemetry
surface: `mimir-spectrum`, `mimir-sync-state`, `mimir-sync-report`,
`mimir-video-buffer`, `mimir-audio-actuator-command`, and the normalized
CultMesh observation ledger from `Mimir.EveSensorReceiver`. If a signal is
absent, the surface says so; it does not fabricate confidence.
## Verification
The command smoke connected to `/eve/dashboard`, received state version `1`,
sent:
```json
{"type":"move","nodeId":"eve-camera","x":0.25,"y":-0.25}
```
and received state version `2` with `eve-camera` moved to that transform. During
the same smoke, EveCanvas connected from `192.168.1.72` and remained running.
The first device screenshot proved the dashboard rendered on Eve, but also
showed the legacy debug overlay occluding the hierarchy panel. EveCanvas hides
that overlay while dashboard mode is active; dashboard status remains in the
native dashboard footer.
The broker smoke on 2026-05-31 verified:
- `/health` reports active provider, client count, transport, and typed
CultMesh dashboard document name.
- `/eve/deck/providers` returns the built-in provider manifests.
- WebSocket `/eve/deck` sends the switchboard snapshot.
- WebSocket `/eve/deck/cultmesh` sends the same provider state as
`mimir.eve_dashboard_state.v1` MessagePack and accepts
`mimir.eve_dashboard_command.v1` MessagePack command documents.
- `open-provider` switches to `mimir.stream.layout` and broadcasts that
provider state.
- `open-provider` switches to `voidbot.swarm`, which emits VoidBot's swarm
status, CTB rail with avatar URLs, agent cards, and selected Face state
detail from the existing VoidBot swarm snapshot.
## Surface Composition Contract
`dashboard-state.surface` is the new distributed UI layer. It is intentionally
closer to CultUI than to the old flat dashboard node list:
- `surface.schema`: versioned wire contract, currently
`cultmesh.eve_surface.v0`.
- `surface.root`: retained element tree. Elements have `kind`, optional `role`,
text, asset refs/URIs, provider command ids, layout hints, style variants,
metrics, and children.
- `surface.assets`: cacheable media references such as avatar images.
- `bindNodeId`: optional bridge back to the compatibility `nodes` projection.
- `commandId`: provider-owned intent id. Eve clients invoke commands; providers
accept or reject and publish the next snapshot.
- `binding`: optional typed CultCache/CultMesh field binding. It names
`documentSchema`, `documentId`, field `path`, `valueKind`, `access`,
`authority`, and optional `commandId`. This is the Verse-backed field
contract; native callbacks and DOM handlers are only lowering actuators.
The old `nodes` array is no longer expected to carry complete UI composition.
It is a compatibility/debug/data surface. Clients that understand `surface`
should render the composed tree first and use `nodes` only for selection,
legacy node binding, and fallback. `bindNodeId` is compatibility glue, not the
portable state owner.
The binary CultMesh state document mirrors this ownership: field 7 carries the
optional `surface` tree. Older clients can stop at the node projection; current
Eve clients should prefer `surface.root` whenever it is present.

View file

@ -0,0 +1,84 @@
# EVE Program Output
## Authority Map
- Owner: Fensalir owns the visible D3D12 program pixels.
- Inputs: Fensalir reads the current `AquariumFrame`, render plan, GPU resources,
and overlay state.
- Outputs: a named shared D3D12 `B8G8R8A8_UNorm` texture containing the completed
backbuffer for the current frame.
- Derived state: Mimir publication profiles describe the stream contract; they
do not render pixels. EVE receives decoded pixels and composites them.
- Forbidden writers: WebKit, DOM layout, browser dashboards, OBS capture, and
desktop/window capture do not own this path.
- Shared paths: local display and EVE streaming both use the same completed
Fensalir backbuffer copy.
- Deletion line: diagnostic Sunshine/Moonlight or desktop-duplication probes may
prove network/decode budget, but they are not production authority.
## Current Mechanism
Set these before launching the Mimir app:
```powershell
$env:FENSALIR_PROGRAM_OUTPUT_D3D12 = "1"
$env:FENSALIR_PROGRAM_OUTPUT_NAME = "Global\MimirFensalirProgramTexture"
$env:FENSALIR_PROGRAM_OUTPUT_FENCE_NAME = "Global\MimirFensalirProgramFence"
$env:FENSALIR_PROGRAM_OUTPUT_RING_COUNT = "1"
dotnet run --project .\src\Mimir.App\Mimir.App.csproj
```
Fensalir creates a shareable D3D12 texture with that name, publishes a named
program-output fence, and copies the finished backbuffer into the texture during
the present pass. `Mimir.EveRelay` opens that named texture, reads the completed
program frame, feeds a low-latency H.264 encoder, and sends complete Annex-B
access units to EveCanvas over the existing `/stream` WebSocket. EveCanvas
treats JPEG frames as the old VoidBot dashboard fallback only; Mimir dashboard
streaming is `h264-annexb` and displays through a native
`AVSampleBufferDisplayLayer` so the iPad owns hardware decode/composite.
The EVE relay currently consumes the single-texture contract. Keep
`FENSALIR_PROGRAM_OUTPUT_RING_COUNT=1` for EVE until the relay has the same
slot-selection contract as the OBS program texture source.
`scripts/start-eve-dashboard-stream.ps1` starts the Mimir/Fensalir app with the
shared-texture output enabled, starts the H.264 relay, verifies the `eve` SSH
target, opens a reverse SSH tunnel from Eve's `127.0.0.1:8792` back to the
local relay when direct inbound TCP is blocked, and launches EveCanvas:
```powershell
powershell -ExecutionPolicy Bypass -File .\scripts\start-eve-dashboard-stream.ps1
```
The relay defaults to `h264_nvenc`. Use `-Encoder libx264` only as a diagnostic
fallback when no NVENC-capable FFmpeg is installed.
EveCanvas tries `ws://127.0.0.1:8792/stream` first, then falls back to
`ws://192.168.1.66:8792/stream`. The localhost path is not JPEG tunneling; it is
the same H.264 Annex-B stream carried through SSH port forwarding so a Windows
firewall rule cannot silently turn the dashboard into a wall.
## Receiver Contract
EVE owns:
- network receive;
- hardware decode of H.264 access units;
- texture upload/import;
- final fullscreen composite;
- touch/Pencil/sensor uplink later.
EVE does not own:
- dashboard DOM;
- Chromium/WebKit compatibility;
- layout parity;
- Mimir timing authority;
- OBS composition authority.
## Smoke
```powershell
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --eve-program-output-contract-smoke
dotnet build .\src\Mimir.EveRelay\Mimir.EveRelay.csproj --no-restore --disable-build-servers
```

View file

@ -0,0 +1,511 @@
# Fensalir Rendering Rebuild Migration
## Objective
Keep Mimir from becoming a second renderer while Fensalir rebuilds itself into
the field-evidence machine Mimir needs.
Mimir's job is to make the room measurable: cameras, microphones, speakers,
loopbacks, phones, Raven, and Starfire all feed one bounded temporal window.
Fensalir's job is to turn that measured evidence into a coherent visible field
and program surface. OBS should receive the final result, not be asked to
pretend loose feeds are a world.
## Current Mechanism
Mimir currently has the correct broad cut:
```text
drivers / ASIO / network feeds
-> Mimir.Runtime rolling buffers
-> synchronization, calibration, response models
-> Fensalir scene state and debug surfaces
-> Fensalir D3D12 presentation / Spout / program output
-> OBS
```
The old direct debug-spectrum pressure leak has been cut. Current spectrum
debug output is already in the production-shaped bridge:
```text
rolling buffer slice + source identity + calibration + surface intent
-> Fensalir field claim
-> TubeField candidate producer, if that is the selected claim family
-> selected lowering under budget
-> evidence lanes and temporal guide
-> presentation
```
Mimir must not send "draw these lines" as a production order. It should say
"this rolling buffer slice is a spectrum field with stable axes, identities,
support, and material intent." Fensalir then chooses direct tube SDF, mesh,
field splats, or another lowering.
The shared Fensalir spatiotemporal reservoir remains the rendering organ.
TubeField is only the current claim/candidate producer for rolling-buffer
spectrum surfaces. It may own tube-specific resource binding, Catmull-Rom
evaluation, SDF/proxy geometry, and material sampling, but it must not become a
private renderer or final composition authority.
This is an intentional divergence from pure ReSTIR. Fensalir is not only
resampling lighting after a known primary surface; it is often sampling the
visible field candidate itself. TubeField therefore has to generate local
primary tube-visibility candidates before reservoir reuse. The shared reservoir
owns the temporal-antialiasing-shaped decision about which field candidate persists into
presentation. There is no separate TAA owner in the target architecture:
reservoir resolve reconstructs smooth presentation directly from selected field
evidence instead of repairing unstable pixels after the fact.
## Invariants
- Mimir owns raw retention. Fensalir may snapshot or import the five-second
window, but it does not silently become history owner.
- Mimir owns physical provenance: device ids, sample clocks, calibration ids,
path response, confidence, network arrival, and decoded timing evidence.
- Fensalir owns dense visual fusion, temporal field evidence, selected cuts,
residency, D3D12 resources, debug UI, and program video.
- Faust/native DSP owns hot audio movement, resampling, alignment, separation,
spatialization, and stems.
- OBS owns broadcast composition only.
- Render packets are cached lowerings. They are not scene truth.
- Reservoir resolve owns temporal antialiasing and reconstruction. TAA is not a
separate source-identity or presentation-history authority.
- Networked phones and Raven may publish local observations and decoded timing
state. They do not become independent clock authorities.
## Authority Map
Owner:
- `Mimir.Runtime` owns stream identity, rolling buffers, synchronization state,
calibration loading, and observation publication.
- Fensalir's future `FieldEvidenceMachine` owns field claims after Mimir
publishes observations and intent.
Inputs:
- local direct-driver video frames;
- local ASIO loopback and mic blocks;
- Raven/phone/network observations;
- calibration receipts, response surfaces, codebooks, path ids, sensor poses;
- user/app policy for what should enter the live field.
Outputs:
- typed observation surfaces for Fensalir;
- timing, response, and calibration constraints;
- audio actuator control for Faust/native DSP;
- health/telemetry state for UI and logs;
- final program output from Fensalir and stems from Faust/native DSP.
Derived state:
- spectrum lines, debug views, screenshots, OBS bridge endpoints, FFmpeg/SRT
helpers, and Eve dashboard feeds are views or diagnostics.
- The old direct spline-spectrum dashboard, direct GPU sensor frame bridge, and
direct acoustic field frame proof are silent from Mimir's active bridge path.
FieldEvidence owns spectrum, camera observation publication, and calibration
constraints; screenshots and logs remain diagnostics.
Forbidden writers:
- any Mimir-side D3D12 render graph;
- app-specific stable Gaussian/evidence cache;
- JSON/base64 hot path for live media;
- OBS raw source timing as synchronization authority;
- per-debug-view geometry deciding production truth.
Shared paths:
- live cameras, audio timing anchors, acoustic response surfaces, spectrum
diagnostics, calibration events, and future volumetric room claims all publish
typed observations/constraints into the same Mimir-to-Fensalir bridge.
Deletion line:
- If a Mimir feature wants visual output, first define the observation, claim,
or surface intent. A direct draw is allowed only when it is named as fallback
or diagnostic and cannot override field evidence.
## Target Mimir -> Fensalir Contract
### 1. Raw Window
Mimir keeps one retention authority:
```text
RollingStreamWindow {
windowId
duration = 5 seconds by default
streamId
sourceKind
sampleDescriptors
nativeHandle or payload view
deviceTimestamp
canonicalTimestampEstimate
sequenceId
status
}
```
Typed views may index this window. They do not own retention.
### 2. Observation Surface
Mimir publishes physical evidence, not render commands:
```text
MimirObservation {
observationKey
streamId
sensorId
calibrationId
modality: camera | audio | network | timing | response
observedTime
canonicalTimeEstimate
uncertainty
payloadHandle
provenance
confidence
}
```
The payload handle is not authority. It must resolve through a declared
resource before shader lowering can consume it.
### 2.5. Resource Declaration
Live payloads have an explicit Fensalir resource contract:
```text
FensalirResource {
resourceKey
kind: structured-buffer | texture2d | rolling-texture | mesh | surface-page | volume
residency: gpu-resident | shared-gpu
shaderAccess: SRV | UAV | vertex-buffer | index-buffer | indirect-args
format
dimensions / count / stride
nativeHandle and nativeHandleKind
valid time range
version / sequence
}
```
Mimir may mint stable resource keys from native/GPU handles, but Fensalir owns
resolution. A DSL packet may reference `mimir:resource:*`; it may not treat a
bare string as payload truth. In the same runtime, a Mimir-declared buffer is a
Fensalir resource slot once the declaration is accepted; native/shared handle
metadata is an import edge, not a separate payload authority. Rendering-relevant
buffers move to GPU residency as early as possible and stay there; Fensalir
passes resource handles/SRVs/UAVs between compute and render lowerings instead
of copying samples back to CPU. If the resource contract is absent, expired,
duplicated, CPU-only, or incompatible with the selected backend, lowering must
defer instead of rendering a lie with nice lighting.
Current proof surface:
- `Mimir.BufferSmoke --fensalir-field-evidence-smoke` maps a Mimir rolling
window/observation/spectrum intent into one declared resource, three claims,
one planned resource-backed `TubeField` packet, and two deferred non-backend
claims.
- `Mimir.BufferSmoke --fensalir-field-dsl-resource-smoke` uses Fensalir's field
evidence DSL to bind the declared resource directly and produce one planned
`TubeField` packet with no deferred requests.
- `Mimir.BufferSmoke --fensalir-camera-observation-smoke` maps one
GPU-backed video frame and one metadata-only cadence witness into field
evidence. Only the shared D3D12 video handle becomes a declared `Texture2D`
resource and camera surface intent; the metadata-only frame remains
observation evidence with no render payload.
Fensalir now owns the first D3D12 resource resolver cut below this contract:
shared structured/curve buffers import/alias GPU-resident resources by handle,
while Fensalir-created resources allocate GPU slots only when Fensalir is the
producer. The DSL can describe a 2D rolling float buffer as Catmull-Rom XY
tubes with modulo column addressing, amplitude power/normalization, radius,
ramp texture path, and emission scale. TubeField now consumes those resources
through a GPU compute/render path, local Texture2D resources can bind as ramps,
SurfacePage and VolumeTexture declarations resolve into shader-readable GPU
textures, and Mesh declarations package `Mesh.Vertices`/`Mesh.Indices` GPU
buffer shape under one resource key. The evidence DSL can now plan generic
resource-backed claims over those declarations. Current blocker for the
remaining visual surface is not resource ownership; it is the selected render
lowerings that interpret mesh/page/volume resources as geometry, height/SDF/
material pages, or density/extinction/SDF3D domains. Mesh layout authority is
split by source: imported/user meshes use the standard `PositionNormalUvColor`
layout, while generated meshes can be `PipelinePrivate` and leave byte
semantics to their selected lowering. TubeField now uses that generated-mesh
lane explicitly: compute emits private geometry/indirect buffers and render
binds them as a pipeline-private generated mesh before applying TubeField
material state. The generated-mesh DrawIndexed indirect command signature is
shared ABI; TubeField only owns the argument buffer it emits. TubeField
expansion is now gated by selected backend packets, so explicit lowering
metadata cannot bypass validation/planning authority. Validation also rejects
TubeSpline metadata whose claim is not Tube-encoded or whose resource differs
from the claim payload. Mimir's typed surface-intent lowering now emits the
matching TubeSpline metadata for audio spectrum/waveform Tube claims. That is
engine ownership, not a reason for Mimir to create a parallel renderer.
Mimir also uploads normalized spectral frames as Float32 data into a
Fensalir-owned structured buffer resource, so the TubeField path consumes live
GPU buffer contents rather than an empty declaration. The runtime-level receipt
is `Mimir.BufferSmoke --mimir-spectrum-upload-smoke`: it boots Mimir's frame
path, advances spectrum analysis, and verifies one newest-slot Float32 upload,
planned TubeField packets, a resource-bound local blackbody ramp Texture2D, and
no live legacy spline/buffer-field dashboard input. The runtime resource is no
longer a latest-frame row dump: it is a rolling column matrix where physical
columns are flattened `(history age, source lane)` pairs under fixed history
and source-lane capacities. Mimir emits one TubeField claim per active source
lane; lowerings use the fixed lane capacity as `ColumnStride`,
`ColumnStep.z=0.1`, and `RollingOffset` so each source renders its own age
trail from newest to older samples without reshuffling the GPU buffer. New
frames upload only the newest ring slot with an explicit element offset.
Fensalir owns structured-buffer slot validity and clamps TubeField dispatch so
invalid older slots do not reach shader sampling after allocation, reset, or
partial update. TubeField metadata now carries stable claim-derived field ids
inside the TubeField family range, so temporal/reprojection/reservoir consumers
can distinguish source-lane claims. TubeField render now binds the engine
blue-noise texture and jitters the tube-local SDF sample inside the proxy
fragment shader before writing coverage, normals, and reservoir guide data; the
same pass also uses the scene depth buffer so opaque generated field geometry is
resolved by nearest surface rather than submission order. The current generated
proxy visual still exposes segment structure and is not yet the final smooth
tube marcher. Mimir exposes
`MIMIR_SPECTRUM_SOURCE_LANES` as the source-lane capacity lever and
`MIMIR_SPECTRUM_TUBE_SUBDIVISIONS` as the geometry cost lever for responding to
Fensalir's requested/dispatched/truncated/invalid TubeField budget report.
Camera descriptors now follow the same evidence/resource boundary:
`MimirFensalirFieldLowering.BuildCameraObservationFrame` lowers latest video
buffers into observation claims and declares GPU-backed frames as shared
`Texture2D` resources. Metadata-only/process-cadence frames do not create
surface intents, so a missing payload cannot become a fake `latest-window`
render request. Fensalir can now resolve those shared `Texture2D` declarations
by native D3D12 handle and accepts Mimir video format names such as `Bgra8`,
`Gray8`, `Rg8`, `Nv12`, and `LeapStereoIr`. Camera image claims currently
defer because Fensalir has not selected a visual-fusion/render lowering for
camera textures yet.
### 3. Calibration Constraint
Calibration and sync become explicit constraints:
```text
MimirCalibrationConstraint {
constraintKey
pathId
sourceId
receiverId
evidenceKind: loopback | bioacoustic | passive | visual | network
delayEstimate
delayUncertainty
phaseOrGroupDelay
frequencyResponse
usableBandMask
confidence
}
```
### 4. Surface Intent
Debug and production visualization share intent:
```text
MimirSurfaceIntent {
intentKey
sourceObservationKeys
domain
axes
supportPolicy
materialGraph
updateBudget
purpose: debug | calibration | production
}
```
The spectrum dashboard becomes one instance of this: a rolling audio spectrum
field whose frequency axis maps to X, amplitude to Y, age to Z, and stream
identity to stacked lanes.
## Fresh Dataflow
```mermaid
flowchart TD
A["direct camera drivers"] --> B["Mimir rolling window"]
C["ASIO loopback + mics"] --> B
D["Raven / phones / network"] --> B
B --> E["sync + calibration kernel"]
E --> F["typed observations"]
E --> G["calibration constraints"]
B --> H["surface intent"]
F --> I["Fensalir field evidence machine"]
G --> I
H --> I
I --> J["field claims + selected lowerings"]
J --> K["program video + debug UI"]
E --> L["Faust/native DSP controls"]
L --> M["program stems + spatial bed"]
K --> N["OBS"]
M --> N
```
## Migration Plan
### Phase 0: Silence Legacy Debug Surfaces
The direct `AquariumSplineFrame` spectrum dashboard and point-splat spectrum
path are no longer scene writers. Live spectrum rendering flows through
FieldEvidence TubeField claims and generated mesh expansion.
### Phase 1: Add Bridge DTOs In Mimir.Runtime
Create pure mapping types for:
- rolling stream windows;
- resource declarations for live GPU/native payload views;
- observations;
- calibration constraints;
- surface intent.
The bridge must not read devices, render, allocate per sample, or own timers.
It maps existing runtime state into Fensalir-facing contracts.
Acceptance:
- one synthetic buffer can produce observations and surface intent in a unit
test;
- claims that reference live payloads also declare a typed Fensalir resource;
- the mapping can run repeatedly without growing allocations;
- stream/source ids remain stable across frames.
### Phase 2: Move Spectrum Dashboard To Surface Intent
Represent the current spectrum view as a `MimirSurfaceIntent`:
- stable source lanes;
- stable frequency bins;
- monotonic history sequence ids;
- Z as continuous sample age;
- tube/material intent from amplitude/confidence.
Fensalir may still lower it to direct spline tubes at first. The ownership
changes before the backend does.
Acceptance:
- app code no longer chooses point splats, direct tubes, or mesh ribbons;
- row/band/history identity is the same whether the engine lowers direct tubes
or another representation.
### Phase 3: Camera Observation Bridge
Lower native camera descriptors into observation surfaces:
- source id and calibration id;
- dimensions, format, stride;
- native CPU/GPU handle;
- device timestamp and canonical estimate;
- lens model and confidence/status.
Acceptance:
- implemented bridge proof: GPU-backed video descriptors become declared
shared `Texture2D` resources plus camera observation/surface claims;
- metadata-only cadence descriptors remain observation evidence and do not
become render payloads;
- no selected camera-image lowering exists yet, so Fensalir defers those claims
until visual fusion/feature extraction owns them.
### Phase 4: Audio Field Observation Bridge
Lower sync and calibration state into audio field constraints:
- ASIO loopback clock fit;
- mic delay and SRO state;
- per-path frequency response and group delay;
- bioacoustic anchor clusters;
- passive correlation confidence;
- Raven/phone decoded timing observations.
Acceptance:
- implemented bridge proof: sync states lower into FieldEvidence calibration
constraints instead of direct `AquariumAcousticFieldFrame` packets;
- Fensalir can inspect acoustic confidence and constraints without receiving
raw PCM as production truth;
- selected bridge lowering now exists for audio-path `Phase` and `Confidence`
claims: Fensalir plans those constraints as `DebugOverlay` backend packets so
calibration timing/path evidence is visible through the FieldEvidence
authority instead of a parallel acoustic frame;
- acoustic source candidates, confidence volumes, and cross-modal fusion remain
Phase 5 work;
- Faust/native DSP remains the actuator for sample movement.
### Phase 5: Sensor Fusion Claims
Let Fensalir fuse Mimir observations into field claims:
- sparse feature tracks first;
- calibrated camera rays and marker/feature candidates;
- acoustic source candidates and confidence volumes;
- surface/material claims only after cross-modal evidence earns them.
Acceptance:
- implemented synthetic multi-camera marker proof: Mimir lowers a deterministic
camera marker candidate with calibration/marker identity and a world-space
confidence envelope, and Fensalir plans it as a `DebugOverlay` FieldEvidence
packet;
- implemented synthetic audio source proof: Mimir lowers an SRP/PHAT-style
source candidate with calibration/source identity and a world-space confidence
envelope, and Fensalir plans it as a `DebugOverlay` FieldEvidence packet;
- claims can be inspected by source observation and calibration id.
### Phase 6: Program Output Receipts
Field identity now reaches the OBS-facing program boundary:
- Fensalir publishes the completed program surface through named shared D3D12
textures and `Global\MimirFensalirProgramFence`;
- `Mimir Program Texture` opens those textures in OBS and performs the explicit
D3D12-to-libobs-D3D11 GPU copy without CPU readback or Spout2;
- optional texture rings and `Global\MimirObsProgramConsumerFence` prevent blind
producer/consumer reuse;
- `Mimir Audio Stem` receives separately controllable Faust/native DSP stems
from `Local\MimirObsStemBus`;
- telemetry and receipts record timing, evidence confidence, dropped/late
observations, and current selected cuts.
Acceptance:
- OBS sees final program surfaces/stems through the Mimir OBS plugin;
- raw feeds remain debug inputs, not synchronization truth;
- recorded receipts explain the stem path, installed OBS plugin, live program
texture proof, optional ring proof, dedicated producer fence proof, and
two-way consumer fence proof.
## Verification Matrix
| Check | Expected Result |
| --- | --- |
| Stable buffer identity | Source lanes do not reorder or phase-rotate by render frame. |
| Missing data | No sample means no observation; no zero-fill masquerade. |
| Direct debug path | Can be disabled without changing runtime timing state. |
| Observation bridge | Produces typed evidence without device reads or rendering. |
| Fensalir lowering swap | Direct tube vs future splat/mesh lowering preserves claim ids. |
| Reservoir guide | Presentation history follows evidence validity, not producer folklore. |
| OBS output | OBS receives final program surfaces, not raw clock chaos. |
## Research And Maps
Primary local references:
- `docs/perfect-machine.md`
- `docs/perfect-machine-domain-index.md`
- `docs/sensor-fusion-architecture.md`
- `research/perfect-machine-study-2026-05-23/fensalir-integration-map.md`
- `research/visual-spatial-map/brushstroke-fusion-rendering.md`
- Fensalir `docs/rendering-teardown-rebuild-protocol.md`
- Fensalir `docs/perfect-machine-architecture.md`
- Fensalir `docs/temporal-spatial-evidence-reservoir.md`
The next implementation pass should cut Phase 1 first. A bridge DTO that owns
nothing but mapping is boring in the best possible way: it makes the rest of
the machine harder to lie about.

View file

@ -0,0 +1,115 @@
# Fensalir Reservoir Daemon
## Objective
Move heavy field reconstruction out of the interactive Mimir editor. The editor
should present synced video feeds and controls responsively. A separate
Fensalir reservoir daemon should drink from the Well, spend whatever CPU/GPU
budget is assigned, and publish typed reservoir/program surfaces.
## Authority Map
Owner: the Fensalir reservoir daemon owns reservoir work selection, GPU
residency, temporal reuse, fusion kernels, worker scheduling, and publication of
reservoir-derived program surfaces.
Inputs: typed Well pages, configured composite state, live timing policy,
feature-signal documents, calibration state, Fensalir-owned texture leases, and
explicit budget settings.
Outputs: typed CultMesh/CultCache reservoir status, program texture/fence
surfaces, optional diagnostic Eve surfaces, and OBS-facing shared texture or
plugin feeds.
Derived state: Mimir editor telemetry, preview rectangles, Odin/Eve dashboard
rows, and OBS plugin status are observers. They do not decide reservoir work.
Forbidden writers: the interactive editor render loop must not run temporal
reservoir history just because it is presenting a backbuffer. Field diagnostics
must be opt-in. A camera `FieldEvidenceFrame` claim must not impersonate a flat
video compositor.
Shared paths: direct user placement, configured Well composites, OBS output,
daemon diagnostics, and future Eve room lowerings must all reference the same
source ids, timing state, and placement/composite document. Manual UI placement
and daemon program output may differ only through an explicit selected
composite/version.
Deletion line: old app-hosted reservoir proof visuals remain diagnostic only.
They cannot be the default Mimir program output while the current goal is synced
video presentation.
## Current Scar
On 2026-06-02 the Mimir editor became unresponsive because Fensalir ran
reservoir history presentation on a video-only frame. The renderer now gates
that pass behind source-derived reservoir-producing input, and the live
video-only app log showed `reservoir-update n/a`.
The remaining gap is real: Fensalir still lacks a flat program-layer video
backend that draws Fensalir-owned camera texture leases as 2D synced layers.
That backend is separate from the reservoir daemon. Build it as the editor's
presentation path; do not smuggle it through field diagnostics.
## Current Daemon Cut
`src/Mimir.FensalirDaemon` is the first owner-shaped daemon:
- eats CultCache through `state/fensalir-daemon.ccmp`, using the canonical
MessagePack CultCache backing store;
- drinks Well JSONL logs containing `mimir.well_snapshot.v1`,
`mimir.well_capture_page.v1`, and `mimir.well_stream_pressure.v1`;
- publishes typed `mimir.fensalir_daemon_state.v1` state;
- speaks a CultNet-shaped WebSocket Eve provider at `/eve/deck`;
- speaks binary CultMesh dashboard documents at `/eve/deck/cultmesh`;
- exposes `/health` and `/eve/deck/manifest`;
- presents a retained `cultmesh.eve_surface.v0` surface with compact text and
metrics that Eve GUI and Odin/Nightwing TUI can lower.
Start it with:
```powershell
powershell -ExecutionPolicy Bypass -File .\scripts\start-fensalir-daemon.ps1
```
The launcher writes a supervisor manifest under
`artifacts/runtime/fensalir-daemon-*` with a semantic provider spec and bridge
routes:
```text
mimir-fensalir-daemon|Mimir Fensalir Daemon|asgard.starfire.mimir-fensalir/eve/tui
bridge: ws://127.0.0.1:8799/eve/deck
bridge: ws://127.0.0.1:8799/eve/deck/cultmesh
```
The current worker mode is `surface-owner-installed`: kernel scheduling,
GPU-resident reservoir compute, and actual fused program-surface publication
belong behind this daemon next. Do not move those back into the editor.
## Worker-State Cut
`Mimir.FensalirDaemon` now also publishes
`mimir.fensalir_reservoir_worker_state.v1` beside the daemon summary state.
The worker state is the owner surface for reservoir job selection and pressure:
- capture pages enqueue reservoir jobs only when they contain accepted timing
slices;
- `mimir.cultmesh_stream_frame.v1` records enqueue fine-grained reservoir jobs
from timed stream frames, including paged, inline, native, or shared-handle
body transport evidence;
- unusable pages are dropped with rejection reasons instead of being hidden by
the editor;
- queue depth, running jobs, completed jobs, dropped jobs, timing confidence,
accepted/rejected slice counts, and estimated GPU queue milliseconds are
persisted in CultCache;
- `mimir.fensalir_reservoir_pressure.v1` reports Well poll/publish pressure,
capture bytes, ready-slice ratio, timing confidence, max presentation
distance, GPU queue milliseconds, and drop causes for missing bodies,
unavailable fences, or degraded timing;
- Eve surfaces bind worker metrics to the worker document, not to ad hoc
renderer telemetry.
The current worker still simulates cost from capture-page shape and configured
CPU/GPU share. That is intentional scaffolding: the next cut replaces the cost
model with real GPU/Faust/Fensalir kernel dispatch while preserving the same
owner and state contract.

View file

@ -20,6 +20,138 @@ For the source-level ownership map, read [[code-algorithm-map|Code Algorithm Map
- PowerShell/FFmpeg/SRT remains a bridge utility for LAN OBS feeds. It is not
the synchronized program authority.
Fensalir is not a conventional renderer bolted onto Mimir. It is the engine-side
field/evidence machine. Mimir must submit synchronized buffers, constraints,
surface intent, and calibration evidence; Fensalir lowers them into claims with
travel/depth, metadata, control, and reservoir-guide lanes. The shared Fensalir
spatiotemporal reservoir is the organ that owns temporally reused field
presentation; TubeField is only the current rolling-buffer tube claim/candidate
producer feeding that organ.
That organ should live as a Fensalir reservoir daemon for production: it drinks
typed Well pages/configured composites, owns reservoir budgets and CPU/GPU work
allocation, and publishes typed reservoir/program surfaces. The Mimir editor is
not that daemon. For the current OBS-shaped cut, the editor is a thin synced
video compositor/control surface and must stay responsive even while the
reservoir worker is absent, paused, or saturated. Field diagnostics, Leap
surface splats, spectrum tubes, and reservoir proof visuals are opt-in
diagnostics or daemon outputs, not default editor presentation.
`Mimir.FensalirDaemon` is now the first daemon-shaped owner for that path: it
tails Well documents, persists `mimir.fensalir_daemon_state.v1` into
CultCache, exposes `/health`, and serves JSON plus binary CultMesh
`mimir.eve_dashboard_state.v1` Eve surfaces at `/eve/deck` and
`/eve/deck/cultmesh`. The current worker mode is deliberately
`surface-owner-installed`; dense GPU kernels and fused program output belong
behind this daemon next.
It also publishes `mimir.fensalir_reservoir_worker_state.v1`, a separate
worker document for reservoir scheduling. Capture pages with accepted timing
slices become jobs; unusable pages are dropped with reasons. Queue depth,
running/completed/dropped jobs, accepted/rejected slice counts, timing
confidence, and estimated GPU queue pressure are now CultCache/Eve-visible.
The current worker uses a budget-derived cost model; real Fensalir GPU kernels
and Faust-assisted feature signals should replace that cost model without
moving ownership back into the editor.
The online supervisor now gives `Mimir.VerseRecorder` the supervisor `runId`
and starts `Mimir.FensalirDaemon` against that exact
`VerseCaptures/<runId>/observations.jsonl`, with a run-local CultCache. Fensalir
no longer has to discover the Well stream by scanning old artifacts.
Fensalir also consumes `mimir.cultmesh_stream_frame.v1` as the fine-grained
stream lane, so accepted timed frames can feed reservoir jobs before a whole
capture-page batch is useful.
`mimir.fensalir_reservoir_pressure.v1` is the pressure readout for the same
owner: Well poll/publish pressure, capture bytes, ready-slice ratio, timing
confidence, max distance from presentation, GPU queue milliseconds, and drop
causes for missing bodies, unavailable fences, or degraded timing.
Any path that only paints pixels is not the spatiotemporal machine, but the
editor still needs a real flat video compositor backend. The current
`FieldEvidenceFrame` camera resources are typed declarations/claims; they do
not by themselves draw Fensalir-owned live camera texture leases as program
layers. Until Fensalir has that 2D textured-quad/program-layer backend, the
editor can be responsive and still fail to show synced video. Do not paper over
that with reservoir diagnostics.
Because Fensalir often samples visible field candidates directly rather than
lighting samples after a known primary hit, claim producers may need
deterministic local visibility generation before reservoir reuse. Reservoir
resolve owns temporal antialiasing and reconstruction; TAA is not a separate
hidden owner of field identity.
The first depth-compute target should be a D3D12 stereo disparity lane modeled
on permissive SGM provenance, with `libSGM` as the current north-star reference.
The research ledger is
[[research/d3d12-stereo-depth-provenance-2026-05-29|D3D12 Stereo Depth Provenance]].
That note is not a dependency grant: CUDA implementations, TensorRT ports, and
monocular model demos are provenance only. The live owner is Fensalir D3D12
compute over Mimir-declared texture resources and calibration state.
The first live Leap slice now exists in `MimirStereoDepthConfigurations` and
`MimirFensalirFieldLowering`. The generic
`BuildStereoDepthCandidateFrame` still lowers caller-declared rectified
left/right textures into a GPU-resident compute-writable R16F disparity
`SurfacePage`, an R8 confidence texture, a Height FieldEvidence claim, and an
`AquariumFieldStereoDepthLowering` sidecar. The live
`BuildLeapPackedStereoDepthCandidateFrame` path now recognizes the current
`LeapStereoIr` rolling video window, reuses its declared packed R8G8 texture as
both left and right input resources, names the R/G lane observations, and emits
the same Height/depth lowering automatically from the runtime frame. The
calibration id is explicitly pending, so the output is disparity evidence, not
metric geometry truth. The libSGM-provenance profile currently publishes min
disparity 0, 128 disparities, four aggregation paths, census radius 2, P1 8,
and P2 96. Fensalir has the first packed-Leap D3D12 compute kernel installed as
a crude SAD/block-match disparity writer; it is not full SGM yet.
The next socket now exists too: `MimirPointCloudConfigurations` records the
Leap disparity-to-point-cloud projection profile, and
`BuildLeapPackedStereoPointCloudCandidateFrame` derives a GPU-resident
`FieldMesh` point-list resource from the live Leap disparity resource. The
runtime merges that Mesh claim alongside the stereo-depth claim, and
`Mimir.BufferSmoke --leap-point-cloud-root-smoke` proves the contract plans as
one `SurfacePage` packet plus one `Mesh` packet with no deferred requests. This
is the first point-cloud root. Fensalir branch `codex/leap-packed-depth` now has
the matching native surface-claim lane: `D3D12PointCloudFromDisparityCS` fills
the derived point-list Mesh for diagnostics and also writes approximated
normal/curvature surface splats into Fensalir's existing SDF/PBR/radiosity
reservoir buffers. The point-list renderer is gated as a diagnostic path; live
presentation is through the shared splat/reservoir resolver. The live proof is
`artifacts/runtime/mimir-leap-surface-claims-clean.png`: LeapUVC -> Fensalir
D3D12 texture lease -> packed stereo disparity -> 19,200 generated surface
claims -> visible reservoir-resolved splats, with point-list rendering still
at `rendered 0`. A second proof,
`artifacts/runtime/mimir-leap-history-split-shader-visible.png`, runs without
`AQUARIUM_DISABLE_RESERVOIR_HISTORY`: Fensalir now compiles a dedicated lean
`D3D12ReservoirHistoryUpdate.hlsl` instead of compiling the monolithic post
shader for the history compute pass. That lean pass writes the current resolver
sample into all four reservoir-history rows and the resolved target; temporal
and spatial reuse still need their own compilable shader cut. The next truth
cut is calibrated projection constants, temporal reservoir reuse, and global
residual/calibration ownership, not another fake RGB depth path.
Those claims are not inherently pixel-sized. Pixel-level resolve consumes the
reservoir, but claim support is chosen from the represented field. Smooth
surfaces should stay broad. Aquarium-style heightfields should be quadtree
surface domains with 2D brushes painting tile payloads, and SDF probes emitting
surface splats only where curvature, projected error, material/brush detail, or
silhouettes justify subdivision.
The current teardown/migration map lives in
[[fensalir-rendering-rebuild-migration|Fensalir Rendering Rebuild Migration]].
Mimir's side of that rebuild is to publish typed physical observations,
calibration constraints, and surface intent; Fensalir's side is to turn those
into field claims, selected lowerings, reusable evidence, temporal guides, and
program output.
Payload handles are now demoted to names inside a resource authority. Mimir
declares live native/GPU payloads as typed Fensalir resources with shape,
residency, shader access, validity, version, and native handle metadata; claims
and lowering requests reference those resource keys. A string handle without a
matching resource declaration is not payload truth. For live camera images, the
preferred authority is Fensalir-owned texture leasing: Mimir asks the engine
broker for a keyed `Texture2D` lease, writes decoded frames into the returned
shared D3D12 texture, signals the returned fence, and commits that fence value
before the resource key is sampled by shader lowerings. Foreign shared handles
remain import edges only. Camera producers must choose the closest-to-device
path available for each sensor and report the unavoidable copy count; convenient
managed/process middlemen are diagnostic witnesses, not hot-path owners.
The old script stack is gone. Do not add a compatibility edge unless it protects
a named invariant that the native runtime cannot protect yet.
@ -28,6 +160,11 @@ a named invariant that the native runtime cannot protect yet.
- Mimir public identity, branding, and Face memory.
- `Mimir.slnx` with `src/Mimir.App` and `src/Mimir.Runtime`.
- Fensalir host bootstrapping from `Mimir.App`.
- [[eve-program-output|EVE Program Output]] now owns the first native EVE
streaming contract: Fensalir can publish the completed D3D12 backbuffer into a
named shared texture when `FENSALIR_PROGRAM_OUTPUT_D3D12=1`, and Mimir records
the EVE-facing publication profile as a native D3D12 stream rather than a
WebKit/dashboard path.
- `MimirSynchronizationHub`, `MimirRollingStreamBuffer`, stream descriptors, and
`IMimirStreamSource`.
- Configurable five-second default rolling buffers for local and network audio
@ -42,6 +179,30 @@ a named invariant that the native runtime cannot protect yet.
production-shaped Focusrite path: a native in-process ASIO callback source
feeds sample-bearing 192 kHz Float32 blocks directly into `Mimir.Runtime`
rolling buffers on one interface clock domain.
- `MimirBioacousticProbeScheduler` closes the first active-probe control loop:
sync confidence and per-band frequency-response confidence decide whether a
targeted probe should be emitted, and `MimirRuntime` publishes live telemetry
for aggregate confidence, deltas, source probe counts, and weak-band pressure.
The scheduler is output-device neutral; a later renderer must explicitly
target Starfire Realtek for physical chirp emission in the current routing.
- `Mimir.WasapiLoopback` now has the first explicit Starfire Realtek render
edge: it can list active render endpoints, select `Speakers (Realtek(R)
Audio)` by friendly-name substring, and play mono Float32 probe audio through
WASAPI shared mode. `Mimir.BufferSmoke
--targeted-bioacoustic-probe-realtk-live` couples that renderer with Scarlett
ASIO capture and scores the physical mic response for the scheduled probe.
The targeted probe planner defaults to a 12 kHz ceiling and low/mid-band bias
because the current Realtek/speaker/Scarlett-mic path does not provide useful
high-band calibration evidence.
- `Mimir.BufferSmoke --bioacoustic-realtk-scheduled-calibration-live` is the
scheduler-owned live-I/O calibration receipt: the scheduler chooses each
probe step from seeded Scarlett residuals, Realtek emits it, Scarlett captures
it, residuals update, and the run writes a per-band CSV plus balance plan
instead of mowing the whole spectrum blindly. The remaining production cut is
feeding measured runtime confidence state into the same scheduler/emitter path.
Scheduler confidence/status uses the same 120 Hz-12 kHz measurable window as
the targeted planner, so dead ultrasonic-ish residuals cannot keep cadence
pinned to questions this room path cannot answer.
- `src/Mimir.BufferSmoke` loads the runtime config, polls the synchronization
hub, and prints the actual rolling buffers. Use `--require-samples` when an
empty declared sensor buffer should fail the run. Use `--bioacoustic-self-test`
@ -97,16 +258,522 @@ a named invariant that the native runtime cannot protect yet.
- `faust/mimir_alignment_actuator.dsp` is the first Faust-owned sample movement
surface: six channels of bounded fractional delay and gain controls. Mimir
estimates delay/SRO; Faust/native DSP applies correction.
- `MimirAlignmentActuatorBank` is the runtime control-plane owner for those
estimates: it converts smoothed per-source sync state into nonnegative
holdback commands, preserves stable Faust `sourceN` slots for live sources,
reports source overflow, and keeps the reference holdback explicit instead of
pretending a measured-late stream can be advanced by a positive delay line.
- `MimirRuntime` updates audio sync analysis online as a bounded rotating
service and can emit live sync telemetry with
`MIMIR_SYNC_TELEMETRY_SECONDS`. UI and telemetry read cached reports/states;
they do not run synchronization analysis.
- `MimirRuntime` publishes live ASIO spectrum history to Fensalir in two layers:
`AquariumBufferFieldFrame` carries the real buffer-field intent for spectrum
windows as spline-domain tube fields with tangent/curvature/normal/derivative
appearance and probe policy, while `AquariumSplineFrame` is the temporary
Catmull-Rom tube preview so the buffer contents remain visible before GPU
reservoir lowering owns the draw.
they do not run synchronization analysis. The same cadence now publishes the
latest actuator command frame for telemetry/readout and queues it into
`AquariumAudioDocument` as an audio control frame. Mimir also declares the
`faust/mimir_alignment_actuator.dsp` program as an
`AquariumStreamingDspProgram`, so Fensalir has both the persistent Faust DSP
graph and the matching controls. The runtime now also queues sample-bearing
`AquariumStreamingAudioBlock` lanes from the latest actuator source buffers,
using the bank-assigned Faust `sourceN` slot as channel authority. Fensalir
now publishes processed DSP output as `AquariumAudioStemFrame` records through
an engine-owned `IAquariumAudioStemBus`, and Mimir declares deterministic
aligned-source stem names with the actuator program.
`MimirObsStemPublicationState` consumes the Fensalir stem bus without copying
audio and validates ready/missing/unconfigured lanes against the
alignment-actuator OBS stem profile. `MimirObsStemSharedMemoryPublisher` now
publishes those validated stems into a fixed Windows shared-memory ABI, and
`native/obs_stem_source` is the first OBS source plugin bundle. It registers
`Mimir Audio Stem`, which reads one named stem and submits it to OBS as an
audio source, and `Mimir Program Texture`, which opens Fensalir's named D3D12
program-output texture and feeds OBS through a GPU-to-GPU D3D12-to-D3D11
bridge texture. Fensalir also publishes a named D3D12 program-output fence,
and the OBS source opens it before copying so the consumer is not reading
blindly or depending on Fensalir's private frame fence. This avoids CPU
readback and removes Spout2 as a required program-video dependency, while
preserving the explicit boundary cost that libobs is D3D11 on Windows.
Fensalir and the OBS source can optionally agree on a bounded publication
ring using `FENSALIR_PROGRAM_OUTPUT_RING_COUNT`; OBS selects the latest
completed slot from the program-output fence value. OBS can also publish a
consumer fence, and when Fensalir is launched with
`FENSALIR_PROGRAM_OUTPUT_CONSUMER_FENCE_NAME`, Fensalir skips publication
instead of reusing a ring slot OBS has not acknowledged.
`scripts/build-obs-stem-plugin.ps1` stages the upstream OBS plugin template
SDK under `artifacts/obs-sdk/` and builds the plugin DLL locally.
- `MimirSynchronizedBufferPlanner` now builds one aligned presentation frame
from the live rolling buffers. It chooses a canonical presentation time inside
the shared retained window, applies source or clock-domain timing corrections,
and returns per-stream slices for cameras, network display feeds, and audio
blocks. This is the low-level Raven display/audio shape: Raven display frames
can be tagged with `clockDomainId: raven-sync`, while a Raven audio timing
signal routed into Scarlett earns the correction for that shared clock domain.
The display feed remains derived evidence, not an independent timing owner.
`Mimir.BufferSmoke --synchronized-buffer-planner-smoke` proves the aligned
local-camera, Raven-display, loopback, and mic buffer shape.
- `MimirFfmpegRawVideoStreamSource` is the current network-pixel adapter for
Raven ingest. FFmpeg owns SRT receive/decode, Mimir owns exact raw-video
frame reads and synchronized buffer samples. Eve uses EveCanvas-native sensor
uplinks instead: `Mimir.EveSensorReceiver` listens for `eve-camera` on
WebSocket port `8793` and `eve-mic` on port `8794`, emitting the existing
frame-event shape into runtime buffers. `Mimir.BufferSmoke --ffmpeg-rawvideo-source-smoke`
proves the local raw-video adapter without Raven online.
- `MimirPresentationControlState` owns operator intent for the Fensalir program
surface: video feed visibility/solo/opacity/layer order, audio mute/solo/gain,
and global LUT preset selection. The `Mimir Program` UI panel is deliberately
compact and fadeable. Video choices filter production surface intents before
Fensalir composition; audio choices modify Faust gain controls and the samples
sent through Fensalir streaming DSP. LUT presets are typed postprocess state
with preset paths and strength; until the renderer grows LUT texture sampling,
preset exposure and bloom are mapped into existing `GraphicsSettings`.
`Mimir.BufferSmoke --presentation-control-smoke` proves the state owner.
- The right rail `Output` panel owns OBS-like program actions below the
inspector. `MimirProgramRecorder` exposes `Record` / `Stop`, tracks
child-process status, writes timestamped MP4 receipts under
`C:\Users\Meta\Videos\Mimir` by default, and kills the process tree on stop.
`MIMIR_RECORD_OUTPUT_DIR` can redirect receipts. The current actuator
deliberately delegates to `scripts/record-stream-proof-output.ps1`, so app
recording produces the same stream-proof composite intended for the
Yggdrasil/public path: Raven program feed, Kiyo picture-in-picture, Starfire
Scarlett audio, and NVENC/AAC broadcast settings.
`MIMIR_RECORD_START_RAVEN=1` lets the app ask the script to start Raven's
sender; otherwise it records from the already-running feed. `MimirProgramStreamer`
exposes `Live` / `Stop` for the configured `Mimir.Broadcast` RTMP path, but it
refuses to publish unless `MIMIR_STREAM_INPUT` is set or
`MIMIR_STREAM_ALLOW_TEST_PATTERN=1` explicitly authorizes a synthetic smoke.
Native program-output recording/streaming can replace those actuators later
behind the same UI owner.
- `MimirSceneEditorState` owns the new Mimir-window editor graph. It is not the
OBS program output: it owns editor camera, selected node, sensor-feed panels,
SDF text-panel nodes, model placeholders, visibility, locks, transforms, reset
commands, and grab/rotate/resize gizmo intent. `MimirRuntime` renders derived
spline outlines and handle markers from that state, and the `Mimir Editor`
panel exposes hierarchy, visibility, transform, creation, and reset controls.
The center preview now emits an Eve-style retained dashboard document through
the C# CultUI-shaped reference API. Fensalir's Direct2D native lowering blits
the current renderer output buffer behind Mimir's normalized placement
overlays, and the D3D12 hit-test path lowers the same tree into preview
gestures. Eve is the portable dashboard authority; CultUI is only the original
C# API shape used here so the same document can be lowered by native, web,
Android, or iOS backends without making Direct2D the owner. Elements can now
carry typed CultCache/CultMesh bindings with document schema/id, field path,
value kind, access, authority, and optional command id. Those bindings are
the Verse-backed state contract; callbacks remain native lowering actuators.
Placement commits
keep video layers inside the program frame, snap near edges/center, and
preserve the source aspect ratio while resizing through direct preview
handles, sliders, or the scale gizmo. Explicit transform/source controls are
inspector-only and only appear while a program source is selected; the
compositor canvas handles are the primary edit path. Empty-canvas clicks clear
source selection, hover/active handles and snap guides are visible through the
shared preview state, keyboard arrows nudge selected layers, PageUp/PageDown
reorder layers, and the inspector owns the selected source's
contain/cover/stretch fit mode.
See [[scene-editor-control-surface|Mimir Scene Editor Control Surface]].
`Mimir.BufferSmoke --scene-editor-smoke` proves the graph/control/gizmo owner
path. World SDF text glyph rendering, ASSIMP-style mesh import/upload, and
pixel-accurate gizmo hit-testing remain Fensalir renderer cuts rather than
Mimir-side fake outputs.
- `MimirRuntime` no longer submits the legacy direct `AquariumSplineFrame`
spectrum dashboard. Live spectrum visualization authority is the
`AquariumFieldEvidenceFrame`: Mimir declares the rolling resource, emits a
Tube claim and `AquariumFieldTubeSplineLowering`, and Fensalir plans/expands
the TubeField generated mesh. Mimir now normalizes current spectral frames
into a row-major Float32 matrix, attaches that as a field resource upload, and
Fensalir copies it into the resolved GPU structured buffer before TubeField
compute reads. The old `AquariumBufferFieldFrame` / `ReservoirSplats`
dashboard path is also silent. `Mimir.BufferSmoke
--mimir-spectrum-upload-smoke` boots `MimirRuntime`, advances the synthetic
spectrum preview path, and verifies that the frame contains the Float32
resource upload plus planned TubeField packet while legacy spline and buffer
field inputs stay empty. The same runtime path declares the local blackbody
ramp PNG as a GPU-resident Texture2D resource and binds it by resource key.
The uploaded spectrum resource is a rolling column matrix now: physical
columns are flattened `(history age, source lane)` pairs, and Mimir emits one
Tube claim/lowering per active source. The resource uses fixed history and
source-lane capacities, so frame-to-frame content versions and source
topology changes do not resize the GPU buffer while the trail fills.
`MIMIR_SPECTRUM_SOURCE_LANES` sets the lane capacity; surplus sources are
truncated and reported in the runtime UI. Lowerings use that fixed capacity
as `ColumnStride`. Physical history slots are addressed as a ring with
`RollingOffset`, so logical age no longer requires reshuffling every column
before shader sampling. Mimir emits one Float32 resource upload for the
newest spectrum ring slot with an explicit element offset, rather than
uploading the full fixed-capacity backing buffer every frame. Fensalir owns
rolling-slot validity and clamps TubeField dispatch so invalid older slots are
not sampled after renderer allocation, reset, or partial update.
`MIMIR_SPECTRUM_TUBE_SUBDIVISIONS` controls the requested Catmull-Rom
subdivision count so Fensalir's overflow report has a direct cost lever.
- `MimirFensalirFieldLowering` now emits `AquariumFieldResourceDeclaration`
rows for live native/GPU payload views. Observation claims reference
`mimir:resource:*` keys, and Fensalir validation/planning can reject or defer
packets whose resources are missing, duplicated, CPU-only, or
backend-incompatible.
`Mimir.BufferSmoke --fensalir-field-evidence-smoke` proves the first
hardware-free receipt with one declared resource, three claims, one planned
resource-backed `TubeField` packet, and two deferred non-backend claims.
`Mimir.BufferSmoke --fensalir-field-dsl-resource-smoke` proves Fensalir's DSL
evidence compiler can bind a declared resource and produce one planned
`TubeField` packet with no deferred requests.
- `MimirFensalirFieldLowering.BuildCameraObservationFrame` is the current
camera bridge proof. It lowers latest video buffers into FieldEvidence
observation claims, declares engine-owned or imported video payloads as
`Texture2D` resources, and creates camera surface intent only when a resource
exists. `MimirFensalirTextureLeaseClient` is the producer-side API for asking
Fensalir for a keyed D3D12 texture/fence lease before decoded camera frames
are written. `MimirRuntime` accepts `AquariumRuntimeServices` and commits
producer fence values for leased video payloads before Fensalir samples them.
`MimirVideoCaptureDriverSource` forwards the lease client to drivers that
implement `IMimirFensalirTextureLeaseReceiver`, so native/direct camera
drivers can allocate their Fensalir destination texture before capture/decode.
If a raw single-plane device path must return CPU bytes,
`MimirVideoCaptureDriverSource` uploads them directly into the leased texture
through the broker, clears the live managed payload, and increments
`UnavoidableCopyCount`. NV12 CPU upload is rejected until the engine owns a
real planar copy path; GPU/native producers should write the shared texture
themselves and stay at zero copies.
Metadata-only cadence frames remain observations with empty payload handles;
they do not become fake render requests. The old direct
`AquariumGpuSensorFrame` builder has been removed from Mimir's proof path.
The first production-shaped camera driver is `MimirKsVideoCaptureDriver`,
backed by `native/camera_capture/mimir_camera_capture.dll`: it opens a
Kernel Streaming capture pin in process, queues uncompressed UVC frame reads,
returns `MimirVideoFrameDescriptor` samples through `IMimirVideoCaptureDriver`,
and lets `MimirVideoCaptureDriverSource` upload those raw frames into
Fensalir texture leases. MJPG/H264 remain outside this lane; they need a
device/GPU decode producer rather than a CPU convenience detour.
PS3 Eyes use the sibling `MimirPs3EyeVideoCaptureDriver` backed by
`native/camera_capture/mimir_ps3eye_capture.dll`; it opens the existing
WinUSB/libusb PS3EYEDriver path in process and emits Bayer8 frames through the
same source/upload/copy-count machinery.
Current direct-driver smokes prove real-frame upload for LeapUVC 640x240
YUY2, both PS3 Eyes at 320x240 Bayer8, Kiyo Pro 1920x1080 YUY2, and regular
Kiyo 640x480 YUY2. Regular Kiyo 1280x720 YUY2 did not open.
`config/mimir-runtime.perfect-machine.local.json` is the current local
all-sensor blast profile: Leap packed stereo IR, both PS3 Eyes at
320x240@187, Kiyo Pro RGB, and regular Kiyo RGB. `Mimir.BufferSmoke
--perfect-machine-live-smoke` opens all five configured video sources,
attaches Fensalir texture leases, and passes only when at least five live
camera textures, a Leap stereo-depth lowering, a disparity SurfacePage, and a
derived point-cloud Mesh are present. Fensalir now plans the camera texture
Feature requests through its `GpuSensorFusion` backend instead of deferring
them. The current dispatch samples LeapStereoIr, both Bayer8 PS3 Eyes, Kiyo
Pro YUY2, and regular Kiyo YUY2. Fensalir binds YUY2 through the
Microsoft-documented `R8G8B8A8_UNORM` SRV view over `DXGI_FORMAT_YUY2` and
decodes `Y0/U/Y1/V` in compute before scoring features.
`MimirMediaFoundationGpuVideoCaptureDriver` is the first compressed camera
path: `native/camera_capture/mimir_mf_gpu_capture.dll` uses Media Foundation
SourceReader with a D3D11 device manager, selects MJPG/H264 camera modes,
decodes on the GPU, copies the decoded GPU surface into a shared BGRA D3D11
texture, and publishes only a `shared-d3d11-texture` handle. Live payload
bytes stay empty. Kiyo Pro 1920x1080 MJPG->RGB32 and H264->RGB32 smokes both
produced GPU handles and valid Fensalir camera resources. NV12 sharing was
rejected by D3D11 in this path, so planar NV12 remains future plane-aware
interop or GPU bridge-copy work.
Fensalir can still resolve imported shared `Texture2D` resources by native
D3D12 handle and accepts Mimir video format names.
`Mimir.BufferSmoke --fensalir-camera-observation-smoke` verifies the split,
and `--fensalir-texture-lease-smoke` verifies the engine-owned lease path.
`Mimir.BufferSmoke --stereo-depth-contract-smoke` verifies the next visual
socket: libSGM-provenance D3D12 SGM is represented as a non-dependency
profile, and synthetic rectified Leap stereo depth lowers through
caller-declared left/right input textures, one stereo-depth lowering sidecar,
and one planned `SurfacePage` packet with no CUDA, CPU disparity image, or
monocular metric authority.
- Mimir's active proof path no longer uses the direct
`AquariumAcousticFieldFrame` builder either. Sync states lower into
FieldEvidence calibration constraints through `MapCalibrationConstraints`,
and Fensalir selects audio-path `Phase` and `Confidence` claims as
`DebugOverlay` backend packets. `--perfect-machine-profile-smoke` now proves
the acoustic constraint is planned through FieldEvidence instead of deferred
through a parallel acoustic packet. The same smoke now also proves the first
Phase 5 synthetic audio-source candidate: an SRP/PHAT-style localization
result is lowered with calibration/source identity and a world-space
confidence envelope, then planned as a FieldEvidence `DebugOverlay` packet.
It also proves a synthetic multi-camera marker candidate using the same
FieldEvidence authority: deterministic marker features plan as
`DebugOverlay` packets while ambiguous raw camera features still defer.
LED strip calibration now has the same typed evidence path:
`MimirLedSplineFieldCandidate` stores ordered per-camera bright-curve samples
in image pixels plus local clipspace/frustum coordinates, temporal-code/index
stability flags, and shared calibration/spline identity.
`BuildLedSplineCandidateFrame` lowers that curve as deterministic
camera-sensor Feature evidence, and `--led-spline-calibration-smoke` proves
Fensalir plans it as one `DebugOverlay` packet with no deferred request.
`MimirLedSplineCurveSolver` now owns the first per-sensor curve solve from
bright LED detections into ordered image/clipspace samples. `MimirCameraRigCalibrationSolver`
owns the next bounded pose step: given stable LED indices plus a scene-space
LED curve anchor, it fits camera translation updates by minimizing ray-to-
curve distance from each local frustum. The smoke synthesizes a scene curve,
projects it through two camera frustums, solves the curves back out of the
detections, and recovers bounded camera-position updates with near-zero ray
residual. Rotation and intrinsics fitting remain explicit future work. Exact
correspondence requires coded or otherwise stable LED identity; uncoded
repeated lights remain curve constraints until the global residual solver
resolves them.
Exposure/gain sweeps must optimize recovered spline quality, not generic
contrast or brightness. `MimirLedSplineQualityScorer` scores each setting by
detected LED count, curve coverage, spacing coherence, smoothness, exposure
fitness, and saturation penalty. `--led-spline-quality-sweep-smoke` proves
the selector chooses the balanced setting over both underexposed missing-LED
detections and overexposed saturated halos.
`MimirLedSplineFrameAnalyzer` is the frame-level bridge for physical sweeps:
it reads luma frames, extracts bright connected components as LED candidates,
solves the per-sensor curve, and emits the same spline quality report.
`--led-spline-frame-analysis-smoke` proves clipped blobs can still be
detected while scoring as unusable calibration evidence.
`Mimir.BufferSmoke --led-spline-live-score` now runs that score against live
direct sensor feeds. For KS/UVC cameras it sweeps manual exposure/gain,
drains stale frames after each write, scores the live luma frames for spline
quality, and prints a best setting per source. Current Starfire proof with
`--expected-leds 38 --minimum-luma 0.55` selected exposure `-9`, gain `8`
for both `kiyo-pro-rgb` (score `0.666`, 41 detections) and
`kiyo-basic-rgb` (score `0.650`, 38 detections). LeapUVC rejected the
generic KS exposure/gain writes and only recovered two LED components; both
PS3 Eye feeds currently score fixed-control frames because their native
wrapper does not yet expose gain/exposure.
PS Move is now a concrete active-marker input candidate rather than a lore
object on a cable. `Mimir.PsMoveProbe` enumerates Sony `VID_054C` /
`PID_03D5` over Windows HID with no third-party dependency. The connected
controller exposes three HID collections: one generic desktop gamepad
collection with 49-byte input/output/feature reports, plus two vendor-defined
feature-report collections. The safe `--read` path uses overlapped `ReadFile`
plus `CancelIoEx` so idle input reads time out instead of hanging the probe.
`--pair-host <bdaddr>` writes a target Bluetooth host address through the
PS Move `0x05` feature report on the `col02` collection, following PS Move
API's pairing contract. The first live controller `00:06:f5:23:e2:d1` was
assigned to Nightwing's BlueZ adapter `5c:93:a2:9c:a8:a8`. Nightwing now
carries the matching BlueZ persistent device/cache records from PS Move
API's Linux registration path, accepts the PS-button wireless connection,
and exposes the controller as `/dev/input/js0`, `/dev/input/event14`, and
`/dev/hidraw1`.
`tools/nightwing_psmove_music_pulse.py` is the current controlled
LED/sphere command proof: Starfire owns the Scarlett ASIO loopback envelope,
computes a thresholdable tiny-FFT onset novelty function using the old
Perlines-style adaptive whitening pattern: fast/slow scrubbed spectrum,
positive delta scrubbed against slow delta, rapidly decaying adaptive min/max
normalization, and warm-up/refractory gating. It fits a provisional
fundamental, folds spectral bin energy into one octave around that
fundamental, and streams dark-idle, high-saturation RGB report `0x06` bursts
over SSH to Nightwing's hidraw writer. The current whitened-delta broadband
rise is ranked against a recent rolling history, and onset brightness is
`percentile^exponent` so exponent `2` maps a median notable frame to 25%
brightness while top recent-window rises approach full brightness.
Steady-state material falls back toward black between emitted peak frames.
The ASIO reader drains the native capture queue to the newest loopback blocks
before each LED frame; stale queued audio was the plausible cause of
multi-second music-pulse lag, not Bluetooth or the hidraw LED command path.
`tools/move_latency_probe.py` is the current diagnostic latency receipt:
PS3 Eye is the default fast visual witness, Scarlett cardioid is the default
chirp timing witness, and the event train uses an order-3 de Bruijn schedule
with symbol-coded chirp bands, LED colors, and small variable gaps. It writes
`event-schedule.json`, arms the Nightwing LED script before playback, fits
detected audio/video events back to the known schedule, and marks receipts
invalid when the Move sphere is not sufficiently visible. A live encoded
PS3 Eye run recovered all eight visual events with schedule residuals within
about 4 ms; `ffplay`/DirectShow audio startup is still too soft for final
physical latency authority, so native timestamped Scarlett playback/capture
remains the production timing cut.
The music-keyed calibration planner now owns articulated multimodal words:
`mimir.music_keyed_move_chirp_plan.v1` carries beat/key analysis, chirp
frequency contours, de Bruijn symbols, and per-Move sampled RGB visual
contour words. `tools/sync_sweep.py --music-plan` consumes that same plan
instead of inventing square flashes, sends the exact RGB samples over
Nightwing HID, and writes RGB video `visual_contour` decoder scores
(brightness correlation plus color-chroma error) into `summary.json`.
With a music plan, the sweep also runs Mimir perfect-machine live receipts
for the local Leap/Kiyo runtime and Raven/Eve receiver runtime unless
explicitly disabled, so the all-sensor run cannot accidentally omit Leap or
Eve merely because FFmpeg cannot archive them directly.
`MimirMoveControllerTracker` is the first Mimir-native overlay owner for the
streamable product shape: it tracks bright PS Move sphere observations from
PS3 Eye luma frames, keeps a bounded history per `controllerId`, preserves
the commanded/expected RGB identity for multiple controllers, and lowers the
histories through the existing Feature evidence path. Synthetic smoke
`--move-controller-overlay-smoke` proves two independent controller histories
become one valid FieldEvidence claim/candidate without claiming 6DoF pose.
`config/mimir-runtime.stream-proof.local.json` names the current stream-proof
profile: Kiyo Pro AR program view, Leap stereo IR depth root, Raven screen
and Raven Realtek loopback from one muxed Raven NVENC/AAC MPEG-TS SRT stream
on port 5200. Starfire demuxes that single Raven transport locally into
`raven-display` on port 5210 and `raven-realtk-loopback` f32 PCM on port
5212 because Mimir's current source adapters ingest raw video and PCM
separately. Scarlett ASIO channels remain labeled as shotgun/cardioid/local
Raven loopback evidence. Nightwing PS3 Eyes no longer appear as Starfire raw
video sources in this proof; they publish compact Eye/Move observations.
`MimirFfmpegPcmAudioStreamSource` is the network PCM audio owner after local
Raven demux. `config/mimir-runtime.move-stream.local.json` remains a
local/raw-Eye show reference, not the current split-host authority.
The distributed authority split is now explicit: Nightwing owns the local
PS3 Eye USB reads, PS Move Bluetooth/HID link, LED schedules, device
timestamps, and compact marker/feature extraction receipts. Starfire owns
the canonical rolling window, Scarlett timing, global calibration residuals,
Kiyo/Raven program sync, and OBS publication. Fensalir owns D3D12 dense
feature/fusion work, surface claims, and splat/reservoir program pixels.
`MimirDistributedWitnessConfigurations.NightwingEyesMoves` and the
`nightwing-eyes-moves` Perfect Machine profile encode this: Nightwing emits
typed `mimir.move_controller_observation_state`,
`mimir.camera_feature_track_state`, and `mimir.visual_marker_state` rather
than becoming a raw-media or camera-placement authority.
`Mimir.EveSensorReceiver` is now the first always-on typed observation relay
for that split. It accepts Eve/CultMesh-compatible witness observations on
`/eve/periwinkle`, normalizes them for the existing text ledger, and also
serves a WebSocket subscriber lane on `/eve/periwinkle/subscribe`.
`Mimir.VerseRecorder` subscribes to that lane and writes a timestamped
`session.json` plus `observations.jsonl` under
`C:\Users\Meta\Videos\Mimir\VerseCaptures` by default. It now recognizes
`mimir.well_capture_page.v1`, strips inline base64 sample bodies into paged
binary files under `bodies/page-*.bin`, and records
`mimir.recorder_body_index.v1` refs in `bodies/index.jsonl`. `Mimir.Well` is the
long-running Starfire ingest/publish daemon: it loads
`config/mimir-runtime.well.local.json`, creates every configured source it
can, polls the `MimirSynchronizationHub` with a bounded daemon cadence,
updates ordinary audio sync/probe state, and publishes
`mimir.well_snapshot.v1` records with source buffers, composite controls,
publication/stem contract, and synchronized-frame status. Complex contour can
be configured and advertised, but expensive matched-filter contour analysis is
suppressed in the Well loop unless `MIMIR_COMPLEX_CONTOUR_RUNTIME_INLINE=1`
is set for diagnostics; the production path needs a worker-owned report lane.
It also publishes bounded
`mimir.well_capture_page.v1` records with synchronized frame-slice timing,
configured composite state, and inline source sample bodies when CPU payload
bytes exist and fit the configured body limit. The online supervisor
`scripts/start-online-verse-daemons.ps1` starts the relay, recorder,
Nightwing Eyes/Move/builtin-camera/builtin-mic witness, music-synced
de-Bruijn/microtonal Move gesture worker, and `Mimir.Well` from the broad
Well profile with capture-page/body-paging enabled. This is the first local
paged-recording ABI for offline reconstruction, not the final CultCache shard
engine. GPU-handle-only samples remain metadata/body-handle evidence until a
Fensalir/native program-output recorder owns rendered composite bodies.
`Mimir.Well` now publishes the first stream-organ pressure surface inside
`mimir.well_snapshot.v1`: `clockDomains` reports per-domain live edges,
actual overlap, and provisional offsets to the ASIO reference, while
`streamPressure` (`mimir.well_stream_pressure.v1`) reports poll and publish
pressure for the transitional WebSocket/JSON path. The planner feeds
provisional clock-domain edge fits through `MimirSourceTimingCorrection`,
then uses actual oldest buffered sample timestamps instead of the configured
retention boundary when choosing a synchronized window. This makes the live
metric honest: a five-second buffer target is no longer allowed to masquerade
as five seconds of resident samples when body serialization/backpressure
starves a source. The next coherent cut is a CultMesh streaming organ over
reliable UDP with a compact typed control lane, typed stream-frame media
envelopes, a CultCache page/shard append body lane, explicit
cursors/backpressure, and a lossy latest-state dashboard lane; do not keep
expanding inline base64 capture pages or SRT/WebSocket bridge edges as if
they were the final transport.
Eve now follows the same rule for its own device body:
`MimirDistributedWitnessConfigurations.EveLocalSensors` and the
`eve-local-sensors` Perfect Machine profile make Eve responsible for local
camera/mic chirp and flash solves, then shipping compact acoustic path,
feature-track, visual-marker, and local-frustum-hint state to Starfire.
Starfire still owns global camera placement and the canonical clock.
`MimirCameraFrustumCalibrationSolver` is the first global frustum solve: it
consumes stable LED/marker correspondences, scene-space marker points, and
seed frustums, then emits per-source position, rotation, horizontal/vertical
tangent half-FOV, correspondence count, clipspace reprojection residual, and
confidence. The same solver accepts `MimirFeatureTrackFieldCandidate` plus
scene-space `TrackId` anchors, so PS Move sphere histories, sparse Eye
tracks, and Eve-local marker tracks use the same Starfire frustum authority
as the LED strip. `mimir.local_frustum_hint_state.v1` is the compact CultMesh
state document for local witness solves: producer node, calibration id,
marker set id, per-camera pose/frustum snapshot, residual, point count, and
confidence. Synthetic smoke `--frustum-calibration-smoke` solves both
LED-spline and feature-track frustums for both Eyes, Kiyo Pro, Leap stereo
IR, and Eve camera, then emits the feature-track solve as a local frustum
hint state, establishing the first fusion-ready frustum surface before dense
surface fusion.
Next useful cut is direct report decoding plus optical sphere observations
from PS3 Eyes/Kiyos/Eve entering the same calibration-evidence path as LED
spline constraints.
PS3 Eyes now have the first sparse tracking surface:
`MimirSparseFeatureTracker` detects grid-suppressed Bayer/luma corner
features, links them frame-to-frame by bounded nearest-neighbor motion, and
emits sensor-local `MimirFeatureTrackCameraObservation` records with track
ids, image/clipspace coordinates, velocity, age, and confidence.
`MimirFeatureTrackFieldCandidate` lowers those observations as camera-sensor
Feature claims through `BuildFeatureTrackCandidateFrame`. The CPU tracker is
a live diagnostic/bootstrap owner; Fensalir D3D12 feature extraction remains
the production owner. `--ps3eye-tracking-live` proved both Eyes can produce
stable tracks at 320x240@187: with sensitive wide tracking defaults, each Eye
returned 160 live tracks from each Eye in a one-second pass, with 160 stable
tracks from `ps3-eye-0` and 156 stable tracks from `ps3-eye-1`.
A full Fensalir headless proof using that all-sensor profile now runs the
actual D3D12 host path. The 2026-05-31 run started all five video sources,
declared eight field resources, resolved six Texture2D resources, dispatched
one packed-Leap stereo-depth kernel, generated one point-cloud/surface stream,
and reported 19,200 visible reservoir splats. The captured proof is
`artifacts/runtime/mimir-perfect-machine-all-sensors-headless.png`. The
follow-up Fensalir compute proof captured
`artifacts/runtime/mimir-perfect-machine-gpu-feature-extraction-headless.png`:
Mimir planned 12 FieldEvidence packets with zero deferred requests, and
Fensalir dispatched GPU sensor fusion over three sampleable live camera
textures, generating 49,152 sparse gaussians. The YUY2 follow-up proof
captured
`artifacts/runtime/mimir-perfect-machine-yuy2-gpu-feature-extraction-headless.png`:
all five camera textures dispatched with zero unsupported field textures,
producing 81,920 sparse gaussians at about 0.08 ms/frame sensor-fusion GPU
time.
Phase 6 program-output receipts are also structurally complete: Fensalir
publishes the final program surface through named shared D3D12 texture/fence
resources, the OBS plugin consumes that surface with an explicit D3D12-to-
libobs-D3D11 GPU copy, optional producer/consumer fences prevent blind ring
reuse, and audio stems cross through the fixed OBS stem shared-memory ABI.
- Fensalir now owns the first in-process D3D12 field resource resolver cut:
shared structured/curve buffer resources import/alias GPU-resident handles,
and Fensalir-owned resources allocate GPU slots only when Fensalir is the
producer. Texture2D local assets can now be declared as field resources and
bound automatically by DSL handles, so TubeField ramps use resource keys
rather than raw paths as live authority. Mimir-declared rendering buffers are
Fensalir GPU resources in the same runtime and should not round-trip through
CPU payloads. The TubeField DSL can now describe a 2D rolling float buffer as
Catmull-Rom XY tubes with modulo column addressing, amplitude power/
normalization, radius, ramp texture resource, and emission scale. D3D12 now
expands and renders those tubes from GPU-resident buffers, samples material
per pixel, binds declared local ramp textures, and executes GPU-emitted
indirect draw argument packets. Surface-page resources can now be declared and
resolved as GPU-resident shader-readable 2D pages; render lowerings that
consume those pages as height/SDF/material domains remain future engine cuts.
VolumeTexture resources can now be declared and resolved as GPU-resident
shader-readable 3D textures for future density/extinction/SDF3D lowerings.
Mesh resources now have that explicit ownership contract: `Mesh.Vertices` and
`Mesh.Indices` package the vertex/index GPU buffers with topology, index
format, bounds, submesh count, and version. The Fensalir DSL can declare a
mesh package, and the D3D12 registry allocates/reuses its GPU-resident vertex
and index buffers under the mesh resource key. This is resolver authority
only; mesh draw/material lowerings remain future engine work. Fensalir's
evidence DSL can also create generic resource-backed claims, so mesh/page/
volume resources can plan into backend packets before their selected render
lowerings exist. Mesh layout authority is split by source: imported/user
meshes use the standard `PositionNormalUvColor` vertex layout, while generated
meshes declare `PipelinePrivate` and let the selected lowering own the minimal
bytes it emits and consumes. TubeField is the first explicit generated-mesh
render consumer: its compute path emits private vertex/index/indirect buffers,
and render binds them as a `D3D12PipelinePrivateGeneratedMesh` before applying
TubeField source/ramp/material state. The DrawIndexed indirect command
signature is now generated-mesh-owned rather than TubeField-owned, so future
generated lowerings can reuse the same draw ABI with their own producer
buffers and material bindings. TubeField dispatch now obeys the selected
backend packet plan; unplanned `TubeSplineLowering` records are counted but
not expanded. TubeField now writes stable field ids, real tube normals,
coverage/confidence, and domain-validity guide data into the same scene
metadata/control/reservoir-guide targets that reservoir resolve uses for
spatiotemporal reconstruction. The shared field candidate inlet now keeps four
fixed candidate slots per pixel, so TubeField and later producers compete
through the same resolver-owned budget instead of naming a private winner.
Reservoir history update is now a shared Fensalir compute pass over a
ping-ponged structured buffer with the same four rows per pixel; it emits the
resolved HDR field texture that bloom and presentation consume. The
presentation shader is no longer a hidden history/TAA owner.
TubeField now feeds the shared reservoir from rolling-buffer column packets
instead of generated segment packets: expansion emits one GPU-resident column
record per logical history/source column, ReSTIR tile binning admits those
columns, and candidate evaluation samples the Catmull-Rom tube SDF/material
directly from the source buffer. Generated mesh draw args are zeroed for the
live path and remain only a diagnostic reference. Fensalir validation
also rejects
TubeSpline lowering metadata
whose claim is not Tube-encoded or whose resource differs from the claim
payload. Mimir's typed surface-intent lowering now emits the matching
`AquariumFieldTubeSplineLowering` for audio spectrum/waveform Tube claims, so
the real Mimir producer supplies both the planned packet and the generated
mesh producer metadata.
- `MimirRuntime` queues bioacoustic timeline PCM through Fensalir audio when the
active timing witness is allowed. `MimirAudioSynchronizationSettings.Mode`
selects `chirp-only`, `passive`, or `hybrid`; passive disables active
@ -191,8 +858,8 @@ a named invariant that the native runtime cannot protect yet.
proof, `--perfect-machine-manifest` to export the module manifest for
tooling/UI/remote witness use, and `--perfect-machine-lowering-benchmark` to
measure the Mimir-to-Fensalir lowering path. The current six-camera/two-audio
synthetic lowering benchmark runs at roughly 2.5 us per iteration with about
2.3 KB allocated per iteration.
synthetic lowering benchmark now measures FieldEvidence camera/resource
lowering rather than the retired direct GPU sensor DTO path.
- `MimirVideoFrameDescriptor` for dimensions, pixel format, stride, device
timestamp, and native/GPU handle metadata.
- `IMimirVideoCaptureDriver` and `MimirVideoCaptureDriverSource` as the live
@ -220,8 +887,8 @@ a named invariant that the native runtime cannot protect yet.
## Next
1. Replace the frame-event diagnostic bridge with concrete direct capture
drivers for Leap stereo IR first, then the
other cameras.
drivers for Leap stereo IR first, then the other cameras. Each driver must
state its device API, destination resource, and unavoidable copy count.
2. Feed those drivers into `MimirVideoCaptureDriverSource` and prove sustained
frame cadence in the rolling buffers.
3. Promote the packet-song physical calibration receipt into the runtime
@ -239,14 +906,66 @@ a named invariant that the native runtime cannot protect yet.
with probe durations long enough to keep loopback and mic windows live.
6. Bind Fensalir UI to the synchronization hub so buffer depth, stream cadence,
source timestamps, and output settings are visible and adjustable.
7. Lower `AquariumBufferFieldFrame` spline tube fields into Fensalir compute:
sample buffer-domain paths stochastically by visual contribution, emit SDF
splat probes, write them into the spatiotemporal splat reservoir, and sample
that reservoir in the temporally antialiased scene pass. The direct spline
preview must stay a witness until this path owns rendering.
7. Tighten the spectrum spline tube temporal path until it behaves like a
production debug surface: shader-owned tube SDF coverage, correct
camera-travel metadata, deterministic direct rendering, and temporal
accumulation from stable frame budgets. Do not route this dashboard back
through fractal point splats.
8. Move GPU feature extraction, fusion, material fitting, render budgeting, and
Spout2 publication into Fensalir.
Spout2 publication into Fensalir. The D3D12 packet/resource resolver now has
first authority for structured buffers, local textures, surface pages,
volume textures, and mesh packages, and the evidence DSL can plan generic
resource-backed claims against those resources. The next engine cuts are
selected render lowerings that consume those resources as mesh geometry,
camera/feature textures, height/SDF/material pages, and
density/extinction/SDF3D volumes. The generic imported mesh ABI is no longer
the blocker; TubeField now occupies the first generated-mesh lowering lane.
The next renderer decisions are how to expose additional generated mesh
producers and which camera/page/volume/material lowering should follow.
The first depth-specific cut should be a Fensalir-owned D3D12 stereo SGM
lane over rectified synchronized camera texture pairs, emitting
GPU-resident disparity/depth/confidence resources and FieldEvidence claims.
Keep `libSGM` as provenance for algorithm shape and benchmark pressure, not
as imported CUDA authority. The typed contract, live packed-Leap disparity
path, Leap point-cloud Mesh socket, and first Fensalir point render lane
exist; the next cuts are live editor verification, full HLSL/D3D12 SGM
aggregation, calibrated projection constants, and then a global
residual/calibration owner.
9. Move mic alignment, room suppression, voice separation, spatialization, and
stem generation into Faust/native DSP.
stem generation into Faust/native DSP. The C# diagnostic bridge now exists
as `MimirCalibratedAudioCompositeBuilder`: it consumes synchronization
states and measured band responses, emits a corrected composite plus scores,
and is proved by `--calibrated-audio-composite-smoke`. It now also scores
calibrated-band coherence and suppresses incoherent measured-band components
before summing; `--dual-mic-coherence-cancellation-smoke` proves the two-mic
case by cutting a mic-local 3.52 kHz interferer while preserving coherent
shared tones. `scripts/record-kiyo-dual-mic-composite.ps1` is now the first
listenable Faust-chain bridge: it records Kiyo video plus Scarlett ASIO raw
channels, renders channels 0/1 through
`--asio-dual-mic-faust-cleaner-wav`, and muxes the Faust dialogue stem into
the Kiyo MP4. That offline command compiles and runs
`faust/mimir_alignment_actuator.dsp` before
`faust/mimir_dual_mic_dialogue_cleaner.dsp`, so the receipt exercises the
actual alignment-then-cleaner chain. The fresh 2026-06-01 run
`artifacts/runtime/kiyo-dual-mic-composite/20260601-051044` estimated a
359.398-sample cardioid delay at 48 kHz, applied holdback 359.398/0.000 to
shotgun/cardioid, wrote dialogue and residual WAVs, and muxed a 10.000 s
640x480@30 Kiyo MP4 with 10.000 s 48 kHz mono AAC. The Faust hot-lane target
exists as
`faust/mimir_dual_mic_dialogue_cleaner.dsp`; Mimir queues it as
`dual-mic-dialogue-cleaner`, consuming the alignment actuator's published
`aligned_source_*` stem frame rather than raw mic buffers, and emitting
`host_voice` plus a rejected residual witness. The routing smoke is
`--dialogue-cleaner-routing-smoke`: it verifies post-alignment stems queue
the cleaner program/block and that program publication belongs to the
cleaner output, not the raw alignment lanes. Treat the C# bridge as the
contract and offline scorer for the
native/Faust implementation, not as a parallel live mixer authority. The
next matching probe loop also exists as
`MimirTargetedBioacousticProbePlanner`: residuals and mic pathology notes
select gain-bounded shaped probe bands for the bioacoustic renderer. Start
with the 192 kHz shotgun/cardioid pair; only widen to every connected mic
once those two paths have credible direct-path and frequency-response
confidence.
10. Keep the OBS bridge witness ledger as evidence before expanding receiver
machinery.

View file

@ -0,0 +1,321 @@
# Move Controller Stream
## Objective
Ship the first Mimir program surface around one concrete show shape:
Kiyo Pro is the RGB program view, PS3 Eyes track multiple PS Move spheres,
Raven screen capture enters as a synchronized video source, and Scarlett ASIO
provides the two hero mics plus Raven program-loopback timing evidence.
## Authority Map
- Owner: `Mimir.Runtime` owns sample ingest, rolling buffers, and typed visual
evidence. Fensalir owns overlay presentation and final program pixels.
- Inputs: Kiyo Pro YUY2 video, both PS3 Eye Bayer8 feeds, Scarlett ASIO
channels `asio-ch0..asio-ch3`, and Raven SRT screen capture on port `5200`.
- Outputs: `kiyo-pro-rgb` is the base program view; PS Move controller histories
lower as Feature evidence through `MimirMoveControllerTracker`; `raven-display`
stays in the `raven-sync` clock domain; ASIO labels identify shotgun,
cardioid, and Raven loopback lanes.
- Derived state: Move 2D histories are observations, not 6DoF pose. Raven
display timing is derived from Scarlett audio evidence, not SRT arrival time.
- Forbidden writers: PS3 Eye blob tracks do not own camera calibration; Raven
network timestamps do not own global sync; Kiyo Pro does not own high-rate
motion truth.
- Cut line: this profile is the first streamable overlay product, not the full
online calibration solve.
## Nightwing Eyes/Moves Split
Nightwing is the right body for the PS3 Eyes and PS Move controllers: it has
the spare USB topology, BlueZ/Bluetooth, and the local HID path. That does not
make it a camera-placement authority.
The split is:
- Nightwing-local authority: read PS3 Eye frames, talk to PS Move HID/Bluetooth,
apply LED schedules, preserve device timestamps, and extract compact sphere,
marker, and sparse Eye feature observations.
- Starfire authority: own the canonical rolling window, Scarlett timing,
global residual solve, camera placement, Kiyo/Raven program sync, and OBS
program publication.
- Fensalir authority: own D3D12 dense feature extraction/fusion, surface claims,
splat/reservoir resolution, and program pixels.
`MimirDistributedWitnessConfigurations.NightwingEyesMoves` records this as a
typed witness: it may emit `mimir.move_controller_observation_state`,
`mimir.camera_feature_track_state`, and `mimir.visual_marker_state`, but it must
not stream raw Eye media as the normal live contract and must not own the
canonical clock. Raw frames from Nightwing are diagnostic receipts only.
The first Nightwing receipt is good enough to build on: both PS3 Eyes enumerate
as stock Linux `ov534`/`gspca` V4L2 cameras on `/dev/video2` and `/dev/video3`.
Default V4L2 cadence is only 30 fps, but explicitly setting
`timeperframe=1/187` before streaming makes both cameras deliver about 187 fps
at 320x240 with zero sequence gaps. The witness worker must set frame interval;
otherwise Linux will quietly look worse than it is.
Nightwing also has a typed witness publisher staged as
`~/.local/bin/nightwing_typed_witness_publisher.py`. The repo source is
`tools/nightwing_typed_witness_publisher.py`. It sends binary MessagePack
`mimir.eve_sensor_observation.v1` records to `Mimir.EveSensorReceiver` on
`ws://192.168.1.66:8796/eve/periwinkle`; the receiver normalizes them into the
Mimir observation ledger, Odin projects them as `observation-stream` nodes, and
Nightwing's Eve TUI lowers those nodes in the `CultMesh Witness Streams` panel.
The current status records are camera-device receipts for `/dev/video*` and a
`nightwing-leap-tracking` placeholder that is intentionally `0` until real Leap
tracking owns that signal. Values are `[present, videoIndex, isPs3Eye]` for
camera-device receipts.
`MimirRoomCalibrationLockSolver` is the first room-lock coordinator. It consumes
the LED string candidate/residual, the camera-rig pose update frame, Move
controller feature-history candidates, and PS3 Eye sparse-feature candidates.
It only reports `Locked` when every witness family is present: multiple cameras
share enough LED indices, the rig solver produced enough pose updates with low
mean ray residual, multiple Move identities have stable history, and multiple
Eyes contribute enough stable feature tracks. Missing families are reported by
name instead of being papered over. `Mimir.BufferSmoke
--room-calibration-lock-smoke` proves a synthetic three-camera LED string, two
Moves, and two Eyes lock at confidence `0.778`, while removing Move history
correctly drops the result to `InsufficientWitnesses`.
## Runtime Profile
For the current stream-proof demo, use the corrected Starfire/Nightwing/Raven
profile:
```powershell
E:\Projects\Mimir\scripts\start-stream-proof.ps1
```
This profile keeps Nightwing raw Eye pixels off Starfire. Starfire ingests local
Scarlett ASIO, local Leap stereo IR, local Kiyo Pro RGB, Raven screen capture,
and Raven Realtek loopback PCM. Nightwing publishes compact Eye/Move
observations through `/eve/periwinkle`; those observations are calibration and
overlay evidence, not raw media ownership.
The profile declares:
- `leap-stereo-ir`: LeapUVC stereo IR / depth root.
- `kiyo-pro-rgb`: Kiyo Pro AR program view.
- `raven-display`: Raven screen capture from the muxed Raven A/V stream,
locally demuxed to SRT port `5210`, in the `raven-sync` clock domain.
- `raven-realtk-loopback`: Raven Realtek render loopback from the same muxed
Raven A/V stream, locally demuxed as f32 PCM to SRT port `5212`, also in
`raven-sync`.
- `focusrite-asio`: Starfire hero mics plus local Raven/program loopback lanes
on `asio-ch0..asio-ch3`.
For the narrower Leap/Eyes/Kiyo Pro bring-up path, use:
```powershell
E:\Projects\Mimir\scripts\start-mvp-leap-eyes-kiyo.ps1
```
This launches Starfire with only local Scarlett ASIO, local Leap stereo IR, and
local Kiyo Pro RGB in the Mimir rolling buffers. Nightwing does not stream PS3
Eye pixels to Starfire. The launcher stages `nw_eye_cap.py`,
`nw_move_hint.py`, and `nightwing_typed_witness_publisher.py` on Nightwing,
then starts the typed witness publisher with `--track-eyes`. Nightwing reads
`/dev/video2` and `/dev/video3`, extracts compact Move-sphere observations,
and sends `mimir.move_controller_observation_state.v1` claims to the
`/eve/periwinkle` CultMesh observation receiver. Starfire/Fensalir can consume
those claims for overlays and calibration without inheriting Nightwing's raw
camera bandwidth or local tracking authority.
The MVP profile refuses to start when LeapUVC is not enumerated. That is
intentional: the Leap is the dense close-range geometry root, not an optional
checkbox pretending to be present.
Use:
```powershell
$env:MIMIR_RUNTIME_CONFIG = "E:\Projects\Mimir\config\mimir-runtime.move-stream.local.json"
dotnet run --project .\src\Mimir.App\Mimir.App.csproj
```
The profile declares:
- `kiyo-pro-rgb`: Kiyo Pro program view, 640x480 YUY2 at the current reliable
25 fps class.
- `ps3-eye-0`, `ps3-eye-1`: high-rate Move sphere visual witnesses.
- `focusrite-asio`: one ASIO producer that emits:
`asio-ch0` shotgun hero mic, `asio-ch1` cardioid hero mic,
`asio-ch2` Raven program loopback L, and `asio-ch3` Raven program loopback R.
- `raven-display`: Raven screen capture over SRT port `5200`.
Run the synthetic overlay smoke:
```powershell
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --move-controller-overlay-smoke
```
Run the live ingest smoke once Raven is sending:
```powershell
$env:MIMIR_RUNTIME_CONFIG = "E:\Projects\Mimir\config\mimir-runtime.move-stream.local.json"
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --seconds 5 --require-samples --sync-reference asio-ch2
```
## Raven Sender
On Raven, start the muxed A/V sender from the logged-in desktop session:
```powershell
E:\Projects\Mimir\scripts\start-raven-av-sender.ps1 -TargetHost 192.168.1.66 -Port 5200
```
That sender captures the desktop through `gdigrab`, encodes video with
`h264_nvenc`, captures Realtek/default render loopback through the repo WASAPI
loopback script, encodes audio as AAC, and muxes both into one MPEG-TS SRT
stream. On Starfire, `scripts\start-stream-proof.ps1` starts
`scripts\start-raven-av-demux.ps1`, which listens on port `5200` and splits the
single transport into local Mimir ingest legs:
```powershell
raven-display local SRT 5210, copied H.264/MPEG-TS for decode
raven-realtk-loopback local SRT 5212, decoded f32 PCM
```
If Raven program audio is also routed into Starfire Scarlett on
`asio-ch2/asio-ch3`, that remains another strong local timing witness. The
Realtek loopback is now part of the same Raven transport as the display before
Starfire demuxes it for Mimir's raw ingest APIs.
Raven display sync is not allowed to ride on SRT arrival time. Both
`raven-display` and `raven-realtk-loopback` are in the `raven-sync` clock
domain. When audio sync estimates the Realtek loopback offset against the
Scarlett reference, `MimirSynchronizedBufferPlanner` applies that correction to
the whole `raven-sync` domain, including the display. The LAN transport is now
one muxed A/V stream; the local split exists only at the Starfire demux edge
because Mimir currently ingests raw video and PCM through separate source
adapters.
## Kiyo AR Trail Receipt
The first offline proof that the Kiyo view can accept Move trails is:
```powershell
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\solve_stream_proof_frustum.py .\artifacts\runtime\all-sensor-articulated-20260531-232349\sweep\calibration-observations.json
```
That solver uses Nightwing's two matched optical witnesses as a provisional
local witness space, fits the Kiyo Pro observations into it, and writes a
`mimir.stream_proof_frustum_solve.v1` receipt plus an AR trail preview. It is
enough for the stream demo and for Starfire's global residual owner to consume;
it is not final metric room calibration.
## Multiple Move Controllers
Do not collapse Move observations into a generic bright blob. Each controller
has its own `controllerId` and expected RGB/time code. The PS3 Eyes can track
the sphere as a high-rate brightness witness; Kiyo Pro can later confirm color
identity; the commanded flash schedule ties the observed marker to the
controller identity.
`MimirMoveControllerTracker` keeps a bounded history per controller and lowers
those histories into the existing Feature evidence path. This is deliberately
2D observation history. 6DoF pose waits for calibrated frustums and the global
residual owner.
## Music-Keyed Calibration Words
`tools/music_keyed_move_chirp_sync.py` is the first planner for calibration
words that are both musical and machine-decodable. It listens to Scarlett ASIO,
uses the same Perlines-style adaptive FFT delta whitening as
`tools/nightwing_psmove_music_pulse.py`, autocorrelates the whitened broadband
spectral-rise function for tempo, estimates a lightweight root from recent
best-fit fundamentals, then writes one shared plan:
- beat-aligned chirps quantized to the estimated minor-pentatonic scale;
- de Bruijn-coded event symbols so adjacent timing/frequency/color words remain
distinguishable;
- per-Move visual words using stable controller identities and unique colors;
the default is no longer a square flash, but a sampled RGB contour with
attack/release envelope, symbol-dependent hue glide, tremolo articulation,
and an emphasized controller lane;
- a JSON receipt shaped like AquaSynth song analysis vocabulary:
`tempo_bpm`, `beat_seconds`, `tempo_confidence`, `root_note`,
`suggested_scale`, `scale_frequencies_hz`, and
`whitened_spectral_autocorr`.
The synthetic smoke is:
```powershell
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\music_keyed_move_chirp_sync.py --mode synthetic --dry-run --events 8 --out-dir artifacts/runtime/music-keyed-move-chirp-sync-smoke
```
The live dry-run against Scarlett is:
```powershell
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\music_keyed_move_chirp_sync.py --dry-run --analyze-seconds 5 --events 8 --out-dir artifacts/runtime/music-keyed-move-chirp-sync-live-dry
```
For double-time material:
```powershell
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\music_keyed_move_chirp_sync.py --dry-run --tempo-grid doubletime --tempo-max-bpm 260 --analyze-seconds 5 --events 8 --out-dir artifacts/runtime/music-keyed-move-chirp-sync-dirtyphonics-dry
```
On the 2026-05-31 live dry-run, the planner produced a usable but not final
receipt: about `92.31` BPM, beat `0.650s`, tempo confidence `0.343`, root `A`,
key confidence `0.716`. That is enough to prove the dataflow, not enough to
claim final musical tempo authority.
For dense double-time material, use `--tempo-grid doubletime`; the planner then
selects the double-time member of the autocorrelation tempo family instead of
preferring the slower parent grid. On the Dirtyphonics-style live source this
produced about `171.43` BPM, beat `0.350s`, tempo confidence `0.358`, root `A`,
key confidence `0.703` after widening the search ceiling with
`--tempo-max-bpm 260`. The receipt records `tempo_family` so later decoders can
see the selected grid alongside half-time/double-time alternatives.
Visual calibration words are deliberately shaped like the audio words: one
gesture carries more than one bit. `--visual-gesture contour` records a
`mimir.move_visual_contour_word.v1` per Move per event, with `sample_rate_hz`,
duration, base identity color, symbol emphasis, and the exact RGB samples sent
over HID. Start conservatively around `--visual-gesture-hz 80`; the HID writer
accepted a short 80 Hz contour test on Nightwing, but the useful rate is the
rate cameras can recover cleanly, not merely the rate the controller accepts.
`--visual-gesture square` remains available as a baseline comparator.
`tools/sync_sweep.py` can now consume that exact plan with `--music-plan`.
That makes the plan the single authority for audio chirps, sampled Move RGB
contours, and offline video scoring. The sweep writes `gesture-plan.json`,
`event-schedule.json`, copied/rendered `chirp-train.wav`, per-sensor captures,
and `summary.json`. RGB camera receipts include a `visual_contour` block with
brightness-envelope correlation and color-chroma error against the expected
Move contour. The old peak-only score is still present as a timing witness; it
no longer pretends square flashes are the only visual language.
When `--music-plan` is used, the sweep also runs typed Mimir runtime sensor
receipts unless `--skip-mimir-live` is passed. The default receipts are:
- `config/mimir-runtime.perfect-machine.local.json` for local Leap stereo IR,
Kiyo Pro, Kiyo, and local camera texture/depth/field-resource ingress;
- `config/mimir-runtime.raven-eve.example.json` for Raven display and Eve
camera/mic frame-event listeners.
That receipt path is not raw media archival yet. It is the audit hook that
prevents Leap or Eve from being omitted by a DirectShow/Nightwing-only sweep.
After a sweep, run `tools/crunch_sync_sweep.py` over the sweep directory to
extract a compact calibration observation table. It merges Nightwing's typed
Move observations with local Kiyo/Kiyo Pro frame analysis, estimates each
local video's schedule shift against the event train, and writes
`calibration-observations.json` plus `calibration-observations.csv`.
Example using an already-written musical plan:
```powershell
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\sync_sweep.py --music-plan artifacts/runtime/music-keyed-move-chirp-sync-live/music-keyed-move-chirp-plan.json --out-dir artifacts/runtime/all-sensor-articulated-sync
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\crunch_sync_sweep.py artifacts/runtime/all-sensor-articulated-sync
```
The non-dry run targets the two wireless Nightwing Moves by default:
`00:07:04:a6:be:5f` on `/dev/hidraw2` and `00:06:f5:23:e2:d1` on
`/dev/hidraw3`. Nightwing now carries `/etc/udev/rules.d/70-move.rules`, which
sets Bluetooth PS Move hidraw nodes to `root:input 0660`. The `metacrat` user
is in `input`, so the non-root SSH writer can drive both spheres. Root SSH is
available as the short local alias `nwroot` for maintenance only; the hot LED
path should stay non-root.

View file

@ -56,6 +56,37 @@ buffer handles directly.
The probe reported one unrelated invalid USB descriptor from
`USB\VID_0000&PID_0002`; it did not prevent PS3 Eye capture.
## 2026-05-31: Nightwing Linux V4L2 PS3 Eye Probe
The two PS3 Eyes were moved to Nightwing. Linux enumerated them as Sony
Playstation Eye USB devices (`1415:2000`) through the stock `ov534`/`gspca`
V4L2 path:
- `/dev/video2`: `USB Camera-B4.09.24.1`, bus `usb-0000:00:14.0-1`
- `/dev/video3`: `USB Camera-B4.09.24.1`, bus `usb-0000:00:14.0-2`
Probe source:
- `tools/nightwing_v4l2_cadence_probe.py`
- copied to Nightwing and run directly against `/dev/video2` and `/dev/video3`
- no `v4l2-ctl`, `ffmpeg`, or third-party Python package dependency
Initial reads without setting frame interval negotiated only the default
`timeperframe=1/30`. After explicitly setting `VIDIOC_S_PARM` to
`timeperframe=1/187`, both Eyes delivered high-rate frames with zero sequence
gaps.
| Device | Requested Mode | Actual Mode | Frames | Delivered FPS | Device FPS | Sequence Gap |
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
| `/dev/video2` | 320x240 GRBG @ 187 | 320x240 GRBG | 375 | 187.07 | 187.37 | 0 |
| `/dev/video3` | 320x240 GRBG @ 187 | 320x240 GRBG | 375 | 187.14 | 187.37 | 0 |
| `/dev/video2` | 320x240 YUYV @ 187 | 320x240 YUYV | 374 | 186.94 | 187.37 | 0 |
| `/dev/video3` | 320x240 YUYV @ 187 | 320x240 YUYV | 375 | 187.06 | 187.38 | 0 |
Conclusion: Nightwing's stock Linux V4L2 path can sustain the high-rate PS3 Eye
witness mode if the witness worker explicitly sets frame interval before
streaming. Default V4L2 reads are a false low-cadence signal.
## 2026-05-21: LeapUVC Kernel Streaming Probe
The Leap Motion Controller is currently present as `VID_F182&PID_0003` and bound

View file

@ -36,9 +36,13 @@ The app-level runtime initializes one rolling buffer per configured audio or
video stream, defaulting to five seconds. The lower native reservoir keeps the
single-edge typed-handle invariant for Fensalir/Faust integration.
Payload memory belongs to the producing domain. The reservoir and runtime own
timing, identity, ordering, retention, and lookup. They do not pretend to own
camera driver buffers, GPU textures, or DSP audio memory.
Payload memory belongs to the producing domain, but rendering inputs should be
Fensalir-owned GPU resources whenever the device path allows it. The camera hot
path is: ask Fensalir for a keyed `Texture2D` lease, speak to the device stack
as directly as possible, populate that lease, signal the producer fence, and
publish the resource key through Mimir's rolling buffers and FieldEvidence. The
reservoir and runtime own timing, identity, ordering, retention, and lookup.
They do not become pixel middlemen.
## Authoritative Boundaries
@ -62,7 +66,10 @@ camera driver buffers, GPU textures, or DSP audio memory.
1. Keep the C# app/runtime as the public Mimir surface.
2. Implement Leap stereo IR direct ingest first and measure sustained cadence.
3. Add the remaining local camera drivers through the same source seam.
Record the unavoidable copy count; do not hide CPU upload behind a friendly
adapter name.
3. Add the remaining local camera drivers through the same source seam,
preferring device-direct producers that populate Fensalir texture leases.
4. Add native audio capture workers and feed aligned blocks to Faust/native DSP.
5. Bind Fensalir UI to stream health, buffer depth, source timestamps, settings,
and output management.

View file

@ -0,0 +1,63 @@
# Network Display Synchronization
Raven can be a low-level synchronized producer when it controls both its display
output and an audio timing signal routed into the Scarlett.
Authority map:
- Owner: `Mimir.Runtime` owns aligned rolling-buffer frames.
- Inputs: camera buffers, Raven display frames, Scarlett ASIO audio buffers, and
explicit timing corrections learned from audio evidence.
- Outputs: `MimirSynchronizedBufferFrame` slices at one canonical presentation
time for Fensalir/Faust/OBS publication.
- Derived state: Raven display timing is derived from the Raven audio sync
signal. The display feed is not a timing authority by itself.
- Forbidden writers: OBS, SRT bridge endpoints, frame-event probes, and network
receive timestamps must not decide cross-stream alignment. CultMesh reliable
UDP packet arrival is transport evidence only, not clock authority.
- Shared path: local cameras, network display frames, mic channels, loopback
channels, and Raven audio sync channels all enter `MimirRollingStreamBuffer`
first, then the synchronized planner chooses slices.
- Deletion line: any compositor path that directly consumes Raven pixels outside
the runtime buffer/alignment path is diagnostic only.
Practical shape:
1. Raven renders a visible timing marker or encoded frame identity on the
captured display.
2. Raven emits the matching audio timing signal through its audio output.
3. That audio is physically or digitally routed into a Scarlett input beside the
mic channels.
4. Mimir decodes the Scarlett audio evidence, learns the Raven clock offset, and
applies that clock-domain correction to the `raven-display` video buffer.
5. Fensalir receives aligned buffer slices and still owns composition.
`config/mimir-runtime.raven-eve.example.json` is the active bridge setup shape
for the network producer pass:
- `raven-display` listens for Raven screen capture on SRT port `5200`, decodes
through FFmpeg, and emits BGRA video samples into the runtime buffer path.
- `eve-camera` listens for EveCanvas camera frame-events on WebSocket port
`8793`.
- `eve-mic` listens for EveCanvas microphone frame-events on WebSocket port
`8794`.
The target Verse shape replaces those SRT/WebSocket bridge edges with CultMesh
reliable-UDP lanes: typed `mimir.cultmesh_stream_frame.v1` envelopes for
per-frame media metadata and refs, plus bounded media body shards for payloads
too large to inline. Raven display frames remain media observations entering
the same rolling buffer; Raven sync still comes from audio evidence and
clock-domain correction, not packet arrival.
The older disabled `frame-events` example in
`config/mimir-runtime.asio.example.json` remains diagnostic only. It can witness
source timestamps and frame identity, but it does not carry pixels. The Scarlett
channel carrying Raven's sync audio is still the evidence path that earns the
`raven-sync` timing correction.
Smoke proof:
```powershell
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --synchronized-buffer-planner-smoke
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --ffmpeg-rawvideo-source-smoke
```

213
docs/obs-stem-plugin.md Normal file
View file

@ -0,0 +1,213 @@
# OBS Native Plugin
The production OBS path is a Mimir/Fensalir-owned program texture plus a
Mimir/Fensalir-owned stem bus, exposed through a narrow native OBS plugin.
## Authority
- Fensalir owns the completed program pixels.
- Mimir/Fensalir own timing, alignment, DSP, and stem identity.
- OBS owns broadcast mixing and scene composition.
- The plugin imports already-produced program pixels and reads already-processed
stem samples. It does not synchronize raw sources, estimate clocks, render the
field, or decide which payload is true.
## Current Contract
### Video
Fensalir publishes the completed backbuffer into a named shared D3D12
`B8G8R8A8_UNorm` texture when these environment variables are set:
```text
FENSALIR_PROGRAM_OUTPUT_D3D12=1
FENSALIR_PROGRAM_OUTPUT_NAME=Global\MimirFensalirProgramTexture
```
The native OBS plugin in `native/obs_stem_source` exposes a `Mimir Program
Texture` video source. Set:
```text
Shared D3D12 texture name: Global\MimirFensalirProgramTexture
Shared D3D12 fence name: Global\MimirFensalirProgramFence
Consumer D3D12 fence name: Global\MimirObsProgramConsumerFence
Texture ring slots: 1
```
OBS/libobs is D3D11 internally on Windows, so this source opens the named D3D12
texture, creates an OBS-readable legacy shared D3D11 texture, opens that same
bridge texture from D3D12, and performs a GPU-to-GPU copy during OBS render
before drawing the source. This avoids CPU readback and does not require
Spout2. It is not pure zero-copy yet; the remaining copy is the
D3D12-to-libobs-D3D11 boundary. Fensalir also publishes a named D3D12
program-output fence; the OBS source opens it and observes publication
completion before copying. This prevents blind read-before-producer-completion
behavior without coupling OBS to Fensalir's private frame fence.
For OBS-only stress testing, Fensalir and the OBS source can agree on a bounded
texture ring:
```text
FENSALIR_PROGRAM_OUTPUT_RING_COUNT=3
FENSALIR_PROGRAM_OUTPUT_CONSUMER_FENCE_NAME=Global\MimirObsProgramConsumerFence
Texture ring slots: 3
Consumer D3D12 fence name: Global\MimirObsProgramConsumerFence
```
With a ring count above one, Fensalir publishes textures named
`Global\MimirFensalirProgramTexture.0`, `.1`, and so on. The OBS source uses the
program-output fence value to copy the latest completed slot and signals the
consumer fence after its GPU copy. When Fensalir is launched with the matching
consumer-fence environment variable, it will not reuse a ring slot until OBS has
acknowledged a copy at or beyond that slot's published fence value. Leave the
ring count at `1` for EVE until the EVE relay speaks the same ring contract.
For a visible interop witness while the real Mimir scene is sparse, launch
Mimir with:
```text
MIMIR_OBS_PROOF_VISUAL=1
```
That enables a small bright SDF/starfield proof visual. It is a debug witness,
not production scene policy.
### Audio
`MimirObsStemSharedMemoryPublisher` writes the latest validated
`MimirObsStemPublicationSnapshot` into a Windows shared-memory map named
`Local\MimirObsStemBus` by default. The native OBS plugin in
`native/obs_stem_source` exposes a `Mimir Audio Stem` input source. Add one
source per stem and set:
```text
Shared memory map: Local\MimirObsStemBus
Stem id: aligned_source_0
```
Use `aligned_source_0` through `aligned_source_5` for the first actuator-stage
source lanes.
## Build
```powershell
.\scripts\build-obs-stem-plugin.ps1
```
The script stages `obsproject/obs-plugintemplate` under `artifacts/obs-sdk/`,
lets the upstream template fetch/build the matching OBS/libobs development
surface, then builds `native/obs_stem_source` against that SDK. The staged SDK
and native build products are local artifacts, not repository state.
The managed shared-memory publisher is verified by:
```powershell
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --obs-stem-shared-memory-smoke
```
The local native plugin build currently succeeds and emits:
```text
native\obs_stem_source\build\Release\mimir_obs_stem_source.dll
```
That DLL currently registers both source types:
```text
Mimir Program Texture
Mimir Audio Stem
Muninn Stream
```
### Muninn Streams
The `Muninn Stream` source is a thin OBS wrapper over Muninn's typed
CultCache/CultMesh stream catalog. The operator-facing UI is intentionally just
the stream dropdown plus independent video and audio device pickers for the
selected source. Store paths, command endpoints, and media return endpoints are
source defaults and migration state, not user-facing stream selection.
The source listens for Raven's typed OBS catalog projection on local UDP port
`17874`. Raven Muninn sends the catalog over a small CultNet RUDP discovery lane
using connection id `0x6d750003`; OBS uses that live catalog for the dropdown
when present.
If live discovery is absent, the source falls back to Muninn's local `.cc`
store:
```text
C:\Meta\Odin\state\muninn.telemetry.cc
```
The source looks for record key `obs` with type:
```text
muninn.obs_stream_catalog
```
Muninn owns that catalog as typed CultCache state. OBS owns only the selected
scene source, the selected video leg, the selected audio leg, and media
rendering. When a selected stream uses Muninn's RUDP media profile, the source
publishes the same typed
`muninn.capture_stream_command` that an operator would publish manually. Raven
Muninn `serve` still owns whether capture starts and which local FFmpeg/WASAPI
children exist.
If both live discovery and the local catalog projection are absent or stale, the
source keeps a LAN-default Raven A/V stream option instead of showing an empty
source. That is a resilience fallback for the OBS machine, not a second stream
authority. Stale catalog `rudp://` URLs that do not carry the current
low-latency LAN profile are sanitized back to the source-derived URL before the
bridge starts.
The current Raven-room default route is LAN-first: command requests go to
`192.168.1.84:17873`, and Raven sends media back to Starfire at
`192.168.1.66:5204`. Older saved source settings that still point at the
deprecated WireGuard defaults `10.77.0.4:17873` and `10.77.0.2` are migrated on
load. Custom operator-entered endpoints are left alone.
The source polls the catalog while active, but reconnect is owned by the RUDP
receiver bridge rather than catalog labels. The bridge tracks the last packet
and the last forwarded video/audio payload. If startup produces no media, or a
previously live stream goes stale, OBS refreshes its local FFmpeg child readers
and republishes the typed activation command on a short bounded cadence. A Raven
daemon restart or media child exit should therefore recover without reopening
source properties.
For `rudp://` stream URLs, the plugin binds the advertised media port, accepts
the CultNet RUDP handshake, acknowledges packets, reassembles RUDP fragments,
decodes typed `muninn.media_video_access_unit.v1` and
`muninn.media_audio_packet.v1` records, and lowers elementary H.264 plus ADTS
AAC into local loopback UDP sockets for OBS FFmpeg child sources. The bridge
does not impose a global packet-sequence FIFO over the media lane: CultNet RUDP
media is reliable but unordered, so frame ids, chunk ids, and the advertised
`assembly_deadline_ms` own video reconstruction. Incomplete video frames expire
inside that budget and emit `muninn.media_receiver_feedback.v1` keyframe
pressure instead of stalling newer media behind an obsolete sequence gap.
RUDP fragment ids are quarantined briefly after completion/expiry so late
duplicates cannot seed a corrupt assembly when the sender's 16-bit fragment id
space wraps under sustained LAN pressure. The local OBS FFmpeg UDP readers use
large loopback buffers; those buffers absorb receiver bursts without becoming a
second transport authority.
## Install
For the standard OBS plugin search path:
```powershell
.\scripts\install-obs-mimir-plugin.ps1
```
By default this writes:
```text
C:\ProgramData\obs-studio\plugins\mimir_obs_stem_source\bin\64bit\mimir_obs_stem_source.dll
```
For a portable OBS tree or direct OBS installation root:
```powershell
.\scripts\install-obs-mimir-plugin.ps1 -ObsRoot D:\Tools\obs-studio
```
The script does not create scene items or alter OBS profiles.

View file

@ -27,6 +27,13 @@ Read this with:
loop candidates.
- [[research/perfect-machine-study-2026-05-23/fensalir-integration-map|Fensalir Integration Map]]
for the Mimir/Fensalir ownership cut.
- [[fensalir-rendering-rebuild-migration|Fensalir Rendering Rebuild Migration]]
for the current teardown/rebuild plan that keeps Mimir on physical evidence
and pushes field rendering authority into Fensalir's evidence machine.
- [[research/d3d12-stereo-depth-provenance-2026-05-29|D3D12 Stereo Depth Provenance]]
for the current fast GPU depth search. `libSGM` is the permissive north-star
reference for a future Fensalir-owned D3D12 compute stereo disparity lane;
CUDA/TensorRT projects remain provenance, not live dependencies.
- [[research/perfect-machine-study-2026-05-23/calibration-session-spec|Calibration Session Spec]]
for the planned physical-path calibration command.
- [[perfect-machine-module-library|Perfect Machine Module Library]] for the
@ -78,6 +85,11 @@ letting OBS become the synchronization authority.
### Visual Evidence
- Leap stereo IR: timing/depth candidate, topology-sensitive but high-value.
The live `LeapStereoIr` resource is a packed R8G8 texture; Mimir now lowers
it into a stereo-depth FieldEvidence claim by treating R/G as the left/right
observation lanes and asking Fensalir for an R16F disparity `SurfacePage`.
The first installed Fensalir kernel is a packed block matcher, not calibrated
metric depth or full SGM.
- PS3 Eyes: high-rate tracking witnesses.
- Kiyo/Kiyo Pro: RGB context and texture, with Kiyo Pro currently cadence/USB
limited.
@ -165,6 +177,18 @@ Likely shape:
- Native capture pushes typed frame handles and timestamps.
- Fensalir extracts features/depth/optical flow on GPU.
- Stereo depth starts as a Fensalir-owned D3D12 compute lane modeled on
permissive SGM provenance, especially `libSGM`: rectified texture pairs plus
calibration produce GPU-resident disparity/depth/confidence resources, which
then lower into FieldEvidence claims.
- Current contract proof:
`Mimir.BufferSmoke --stereo-depth-contract-smoke` validates the profile and
FieldEvidence socket. The synthetic Leap stereo pair uses caller-declared
shader-readable left/right texture declarations, an R16F compute-writable disparity
`SurfacePage`, an R8 confidence `Texture2D`, one Height claim, one
`AquariumFieldStereoDepthLowering` sidecar with min disparity 0, 128
disparities, four paths, census radius 2, P1 8, and P2 96, and one planned `SurfacePage`
packet. It is not yet a live compute kernel.
- Native reservoir preserves rolling evidence.
- Fusion emits surface/material/render claims.
- Program video publishes via Spout2.
@ -173,6 +197,8 @@ Pressure:
- Do not build a CPU image pipeline because it is easy to inspect. Six cameras
make that path collapse into a pile of excuses.
- Do not import CUDA or TensorRT as live depth authority. External depth
projects may guide kernel shape, benchmarks, and failure probes only.
## Hot Loop Budget
@ -251,6 +277,9 @@ Six optical sensors with high-rate Eyes and Leap IR mean:
3. Reduced alphabet/high-order codebook physical proof.
4. Fractional delay/SRO actuator proof.
5. Direct Leap/PS3/Kiyo runtime drivers.
6. Native reservoir integration with Fensalir/Faust.
7. Raven/phone receiver codebook/schedule local decode.
8. GPU visual fusion and Spout2 program video.
6. D3D12 stereo disparity/depth compute lane, with `libSGM` as provenance and
Fensalir as live owner. Contract/profile smoke exists; implement the HLSL
cost-volume, aggregation, winner/subpixel, and confidence passes next.
7. Native reservoir integration with Fensalir/Faust.
8. Raven/phone receiver codebook/schedule local decode.
9. GPU visual fusion and Spout2 program video.

View file

@ -19,16 +19,17 @@ job-interview answer for why it belongs in the body of the machine.
| Benchmark panels | `Mimir.Runtime` | Decoder golf and meatspace acceptance degradation panels with receipt roots and thresholds. | `src/Mimir.Runtime/Synchronization/MimirBenchmarkPanelConfiguration.cs` |
| Capture profiles | `Mimir.Runtime` + native workers | Hardware intent for Leap, Kiyos, PS Eyes, Starfire Scarlett, and Raven Scarlett. | `src/Mimir.Runtime/Synchronization/MimirNativeCaptureConfiguration.cs` |
| Camera ingest strategies | `Mimir.Runtime` + native workers | Diagnostic metadata, managed native wrapper, native SPSC ring, and shared-GPU-texture ingest shapes. | `src/Mimir.Runtime/Synchronization/MimirCameraIngestConfiguration.cs` |
| Stereo depth profiles | `Mimir.Runtime` + Fensalir | SGM-shaped stereo disparity/depth profiles with libSGM as provenance for the first D3D12 lane, not a CUDA dependency. | `src/Mimir.Runtime/Synchronization/MimirStereoDepthConfiguration.cs` |
| Reservoir strategies | `Mimir.Runtime` + native reservoir + Fensalir | Managed rolling buffers, native shared-edge rings, and derived GPU temporal field storage. | `src/Mimir.Runtime/Synchronization/MimirReservoirConfiguration.cs` |
| Distributed witnesses | `Mimir.Runtime` + CultMesh | Raven, phone, microcontroller, and remote camera rig configurations for compact timing/path state. | `src/Mimir.Runtime/Synchronization/MimirDistributedWitnessConfiguration.cs` |
| Network transports | `Mimir.Runtime` + CultMesh | Typed timing state, debug windows, SRT bridge media, and experimental browser transport policy plus constraint-based selection. | `src/Mimir.Runtime/Synchronization/MimirNetworkTransportConfiguration.cs`, `src/Mimir.Runtime/Synchronization/MimirNetworkTransportSelector.cs` |
| Distributed witnesses | `Mimir.Runtime` + CultMesh | Raven, phone, microcontroller, Nightwing Eyes/Moves, and remote camera rig configurations for compact timing/path/visual state. | `src/Mimir.Runtime/Synchronization/MimirDistributedWitnessConfiguration.cs` |
| Network transports | `Mimir.Runtime` + CultMesh | Typed timing state, reliable-UDP Verse media/body lanes, debug windows, SRT bridge media, and experimental browser transport policy plus constraint-based selection. | `src/Mimir.Runtime/Synchronization/MimirNetworkTransportConfiguration.cs`, `src/Mimir.Runtime/Synchronization/MimirNetworkTransportSelector.cs` |
| Authority policy | `Mimir.Runtime` | Trust decisions for Starfire loopback, Raven evidence, phone candidates, diagnostic media, and unknown nodes. | `src/Mimir.Runtime/Synchronization/MimirAuthorityPolicyConfiguration.cs`, `src/Mimir.Runtime/Synchronization/MimirAuthorityPolicyEvaluator.cs` |
| Audio field configurations | `Mimir.Runtime` | Six-mic, source-based, ambisonic, and hybrid evidence field shapes. | `src/Mimir.Runtime/Synchronization/MimirAudioFieldConfiguration.cs` |
| Visual fusion configurations | `Mimir.Runtime` + Fensalir | Sensor roles and fusion modes from cadence proof through full hybrid evidence. | `src/Mimir.Runtime/Synchronization/MimirVisualFusionConfiguration.cs` |
| Compute offload configurations | `Mimir.Runtime` + CultMesh | Local-heavy, distributed witness, and calibration sweep workload placement. | `src/Mimir.Runtime/Synchronization/MimirComputeOffloadConfiguration.cs` |
| Assembly plans | `Mimir.Runtime` | Named combinations of modules that prove synthetic, Scarlett, meatspace, and distributed cuts. | `src/Mimir.Runtime/Synchronization/MimirMachineAssemblyPlan.cs` |
| OBS publication profiles | Fensalir + Faust + OBS | Spout/shared-texture video and separately mixable Faust stem publication. | `src/Mimir.Runtime/Synchronization/MimirObsPublicationConfiguration.cs` |
| Fensalir lowering | `Mimir.Runtime` -> Fensalir | Pure mapping from Mimir rolling/video/audio state into Fensalir GPU sensor and acoustic field frames. | `src/Mimir.Runtime/Synchronization/MimirFensalirFieldLowering.cs` |
| Fensalir lowering | `Mimir.Runtime` -> Fensalir | Pure mapping from Mimir rolling/video/audio state into Fensalir GPU sensor, stereo-depth, and acoustic field frames. | `src/Mimir.Runtime/Synchronization/MimirFensalirFieldLowering.cs` |
| Module catalog | `Mimir.Runtime` | Indexes the live module set and verification commands. | `src/Mimir.Runtime/Synchronization/MimirModuleLibrary.cs` |
| Manifest factory | `Mimir.Runtime` | Produces an exportable JSON manifest for tools, docs, UI, and remote witnesses. | `src/Mimir.Runtime/Synchronization/MimirPerfectMachineManifest.cs` |
@ -40,8 +41,9 @@ job-interview answer for why it belongs in the body of the machine.
program video publication.
- Faust owns hot sample movement, alignment, spatialization, separation, and
stem generation.
- CultMesh carries typed state surfaces; remote nodes may observe and decode,
but they do not become independent clock authorities.
- CultMesh carries typed state surfaces and reliable-UDP media/body lanes;
remote nodes may observe, decode, and stream media through the Verse, but
they do not become independent clock authorities.
- OBS receives final program surfaces and stems. It does not own calibration.
## Verification Surface

View file

@ -0,0 +1,42 @@
# Program Control Surface
Mimir now exposes a compact Fensalir-hosted program panel for final presentation
intent.
Authority map:
- Owner: `MimirPresentationControlState` owns operator intent for program video,
audio mix, and global color preset.
- Inputs: runtime rolling buffers, aligned audio lanes, LUT preset selection,
feed visibility, feed opacity, stem mute/solo, and stem gain.
- Outputs: filtered video surface intents, Faust/native DSP gain controls,
sample gain applied before Fensalir streaming DSP, and a postprocess control
snapshot.
- Derived state: feed lists are derived from active rolling buffers. LUT preset
exposure and bloom are currently mapped into `GraphicsSettings`.
- Forbidden writers: OBS, bridge endpoints, debug readouts, and network arrival
timestamps must not choose program composition.
- Shared paths: local cameras, Raven display capture, and future network video
feeds all enter rolling buffers first, then the presentation state decides
whether they contribute to production surface intent.
- Deletion line: direct compositor shortcuts outside Fensalir FieldEvidence and
audio DSP are diagnostic only.
The current UI lives in the `Mimir Program` panel:
- `Video`: select feed, visible, solo, opacity, layer ordering.
- `Audio`: select stem/source, mute, solo, gain.
- `Color`: LUT preset and LUT strength.
The shader-side LUT path is explicit state, not finished renderer authority yet.
`MimirLookupTablePreset` carries `LutPath`, strength, contrast, saturation,
temperature, and tint so Fensalir can consume the same contract when the global
post shader grows real LUT sampling. Until that renderer cut lands, Mimir maps
the preset exposure and bloom values into existing Fensalir `GraphicsSettings`.
Smoke proof:
```powershell
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --presentation-control-smoke
```

View file

@ -0,0 +1,80 @@
# Raven Capture Mux Daemon
Raven owns local display capture and Realtek render-loopback capture. Mimir receives the muxed stream and decides what it means inside the synchronized program model.
## Authority Map
- Owner: `Mimir.RavenDaemon` owns process supervision and typed status for Raven's capture mux.
- Inputs: local display capture parameters, WASAPI render-loopback selection, FFmpeg path, named SRT targets, log root, and operator flags.
- Outputs: `mimir.raven_capture_mux_state.v1`, an Eve dashboard surface, health JSON, and named FFmpeg SRT MPEG-TS streams for Mimir and OBS.
- Derived state: PID, restart count, log paths, and dashboard nodes are derived from the supervised process and daemon options.
- Forbidden writers: capture scripts may still launch diagnostics, but they do not own daemon status or the canonical Raven mux state.
- Shared paths: service launch, dry run, health checks, and Eve dashboard projection all read the same `MimirRavenCaptureMuxStateDocument`.
- Cut line: no OBS plugin, custom media daemon, or alternate transport owns media in this pass. FFmpeg/WASAPI own capture, mux, encode, and SRT fanout; CultMesh owns status and operator surface.
## Run On Raven
From a Mimir checkout on Raven:
```powershell
.\scripts\start-raven-daemon.ps1
```
Defaults:
- Mimir target: `srt://10.77.0.2:5200`
- OBS target: `srt://10.77.0.2:5204`
- Eve/CultMesh dashboard: `ws://0.0.0.0:8801/eve/deck`
- Binary CultMesh dashboard stream: `ws://0.0.0.0:8801/eve/deck/cultmesh`
- Health: `http://127.0.0.1:8801/health`
- State cache: `C:\Meta\Mimir\state\raven-capture-mux.ccmp`
- Logs: `C:\Meta\Mimir\logs`
Use a dry run to inspect the exact FFmpeg pipeline without starting capture:
```powershell
.\scripts\start-raven-daemon.ps1 -DryRun
```
Disable the OBS sink when only Mimir should receive the mux:
```powershell
.\scripts\start-raven-daemon.ps1 -NoObsTarget
```
Use a specific Realtek render endpoint when Windows exposes more than one:
```powershell
.\scripts\start-raven-daemon.ps1 -AudioDevice "Realtek"
```
If Raven already has a host-level CultMesh server bound, keep this daemon as a local typed-state publisher and Eve endpoint:
```powershell
.\scripts\start-raven-daemon.ps1 -NoCultNetServer
```
## Receiver Shape
The daemon sends identical MPEG-TS SRT streams containing:
- `raven-display`: Raven desktop via `ddagrab` or `gdigrab`, encoded with `h264_nvenc`
- `raven-realtk-loopback`: Raven render loopback via `Mimir.WasapiLoopback` or the PowerShell WASAPI fallback, encoded as AAC
Mimir receives the ingest copy on `5200`. `scripts/start-raven-av-demux.ps1`
can split that copy back into local video and audio source lanes for the
current raw ingest adapters. OBS receives its own copy on `5204` as an ordinary
SRT Media Source; it does not compete with Mimir for the `5200` listener.
The typo in the existing source id, `raven-realtk-loopback`, is preserved until the receiver config is migrated.
## Idunn Lifecycle
Idunn watches the Raven daemon from Odin's local keepalive body:
- Health: `E:\Projects\Odin\scripts\health-raven-capture-mux.cmd`
- Restart: `E:\Projects\Odin\scripts\restart-raven-capture-mux.cmd`
- Default SSH target: Raven's WireGuard address, `10.77.0.4`
If Raven is unreachable, Idunn should record the failed health probe and raise
the normal keepalive decision instead of pretending the daemon is healthy.

130
docs/raven-eve-ingest.md Normal file
View file

@ -0,0 +1,130 @@
# Raven And Eve Ingest
## Objective
Ingest Raven screen capture plus Eve camera and microphone capture into Mimir,
then let the existing synchronization hub place those samples in rolling buffers
beside Scarlett audio and local camera sources.
## Authority Map
- Owner: `Mimir.Runtime` owns decoded samples and rolling-buffer placement.
- Inputs: Raven currently sends H.264/MPEG-TS over the SRT bridge to Starfire
port `5200`; Eve currently sends camera frame-events over WebSocket port
`8793`; Eve currently sends microphone frame-events over WebSocket port
`8794`; Scarlett/ASIO carries audio and sync evidence. The target Verse shape
is CultMesh reliable UDP: typed stream-frame envelopes plus bounded body
shards for media, with compact state/cursors/backpressure beside them.
- Outputs: `raven-display`, `eve-camera`, and `eve-mic` emit
`MimirStreamSample` records with video or audio descriptors.
- Derived state: network arrival timestamps are capture metadata only. Raven
audio-in-Scarlett remains the timing witness for global alignment.
- Forbidden writers: OBS, FFmpeg sender clocks, WebSocket receive time, SRT
arrival time, and CultMesh packet arrival time do not own final sync truth.
- Cut line: EveCanvas owns Eve's sensors directly. External iOS streaming apps
are fallback tools, not the current architecture. SRT/WebSocket bridge
transports are diagnostics and compatibility edges once the reliable-UDP
Verse media lane exists.
## Starfire Receiver
Use the Raven/Eve runtime config:
```powershell
$env:MIMIR_RUNTIME_CONFIG = "E:\Projects\Mimir\config\mimir-runtime.raven-eve.example.json"
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --seconds 5
```
When both senders are active, require sample arrival:
```powershell
$env:MIMIR_RUNTIME_CONFIG = "E:\Projects\Mimir\config\mimir-runtime.raven-eve.example.json"
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --seconds 5 --require-samples
```
The config starts one FFmpeg listener and two Eve sensor receiver processes:
- `raven-display`: `srt://0.0.0.0:5200?mode=listener&latency=120000&timeout=5000000`
- `eve-camera`: `ws://0.0.0.0:8793/eve/camera`
- `eve-mic`: `ws://0.0.0.0:8794/eve/mic`
The Raven listener decodes to raw BGRA frames on stdout. The Eve receivers emit
typed JSON frame-events from EveCanvas camera/mic packets. Mimir stores all of
them in the normal synchronization buffers.
## Raven Sender
Run this on Raven from the logged-in desktop session:
```powershell
E:\Projects\Mimir\scripts\start-raven-screen-capture-sender.ps1 -TargetHost 192.168.1.66 -Port 5200
```
The script uses `gdigrab` plus `h264_nvenc`, then calls Starfire's SRT listener.
Do not launch the capture itself through a blind SSH session unless the desktop
capture path has been proven there; `gdigrab` captures the interactive Windows
desktop, and service/session boundaries can turn screen capture into a black
box with excellent logs.
Raven sync signal: route Raven audio or a Raven-generated chirp into the
Scarlett alongside the microphone. Mimir should treat that audio evidence as
the authority for aligning Raven video with the rest of the program.
## Eve Sensor Sender
EveCanvas now starts native AVFoundation capture after permission grants:
- Camera frames are sampled from `AVCaptureVideoDataOutput`, JPEG-compressed,
wrapped as `video-frame` events for `eve-camera`, and sent to
`ws://192.168.1.66:8793/eve/camera`.
- Microphone blocks are captured with `AVAudioEngine`, packed as interleaved
Float32 samples, wrapped as `audio-block` events for `eve-mic`, and sent to
`ws://192.168.1.66:8794/eve/mic`.
This is an inspectable first transport, not the final high-throughput shape.
When cadence or bandwidth becomes the bottleneck, replace the JSON/base64 event
transport with CultMesh reliable-UDP media frames and body shards, optionally
carrying VideoToolbox H.264 or another explicit media payload. Preserve the same
source ownership and Mimir runtime sample contract.
Deploy EveCanvas after staging:
```powershell
powershell -ExecutionPolicy Bypass -File E:\Projects\Eve\scripts\stage-to-eve.ps1
ssh eve "cd /var/mobile/Projects/Eve && export THEOS=/var/theos && make package install && uiopen --bundleid org.gamecult.evecanvas"
```
## Smoke Test
The raw-video source has a local synthetic smoke that does not require Raven or
Eve:
```powershell
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --ffmpeg-rawvideo-source-smoke
```
That smoke starts FFmpeg with `testsrc2`, decodes BGRA frames through stdout,
and verifies the sample metadata and payload size that Mimir receives.
The Eve sensor receiver can be checked without launching the whole runtime:
```powershell
dotnet run --project .\src\Mimir.EveSensorReceiver\Mimir.EveSensorReceiver.csproj -- --port 8793 --path /eve/camera --source-id eve-camera --type video-frame
```
Periwinkle's Android Eve client publishes binary CultMesh sensor observations:
```powershell
dotnet run --project .\src\Mimir.EveSensorReceiver\Mimir.EveSensorReceiver.csproj -- --port 8796 --path /eve/periwinkle --source-id periwinkle-accelerometer --type cultmesh-observation
```
The receiver accepts `mimir.eve_sensor_observation.v1` MessagePack packets on
that lane. The device remains an observation source; Mimir owns synchronization
and interpretation.
The same receiver also accepts `mimir.eve_media_observation.v1` MessagePack
packets. Periwinkle publishes low-rate camera luma frames as
`periwinkle-camera` and microphone PCM blocks as `periwinkle-mic` on the
CultMesh observation lane. The legacy iPad Eve camera and mic uplinks have
matching binary media document support while their existing `eve-camera` and
`eve-mic` ports remain usable.

View file

@ -0,0 +1,53 @@
# Mimir Scene Editor Control Surface
Mimir now has a runtime-owned scene editor state for arranging the visible
presentation inputs inside the Fensalir-hosted Mimir window. This editor is not
the OBS program output. It is an operator surface for placing source panels,
text panels, and imported-model placeholders before later renderer cuts turn
those nodes into final composited pixels or volumetric field claims.
## Authority Map
- Owner: `MimirSceneEditorState` owns editor camera, scene nodes, selected node,
visibility, lock state, 2D-plane transform, and active gizmo mode.
- Inputs: rolling video buffers provide sensor-feed nodes; Fensalir `InputState`
provides mouse/key edits; the Mimir UI provides visibility, reset, transform,
text, and model-path commands.
- Outputs: `MimirRuntime` reads the editor state into the Fensalir frame camera,
spline outlines, selection gizmos, and SDF handle markers.
- Derived state: the hierarchy text, selected-node readout, editor splines, and
handle SDF objects are derived from the editor graph. They do not own node
truth.
- Forbidden writers: program-composite controls, OBS source endpoints, bridge
scripts, and FieldEvidence surface intents do not decide editor transforms.
- Shared paths: UI buttons/sliders and mouse-drag gizmos all commit through
`MimirSceneEditorState`; reset returns to each node's stored default
transform.
- Deletion line: do not add a separate transform cache in the renderer or OBS
path. Fensalir may later render richer node types, but it should consume this
graph or a deliberate successor, not invent a competing editor graph.
## Current Controls
- `Mimir Editor` panel exposes the scene graph, selected-node visibility and
lock toggles, transform reset, camera reset, and transform sliders.
- Active gizmo modes are `Grab`, `Rotate`, and `Resize`; keys `1`, `2`, and `3`
switch modes, and left-drag applies the active mode to the selected node.
- Sensor-feed nodes are created from live video rolling buffers.
- SDF text panels and model placeholders can be added from the panel.
- `--scene-editor-smoke` proves graph creation, visibility toggles, transform
reset, mouse-drag transform edits, text-node creation, model placeholder
import, and derived gizmo geometry.
## Renderer Hooks Still Open
- World SDF text: Fensalir currently has DirectWrite overlay text and SDF object
proxies. Scene text needs a renderer-owned SDF/MSDF glyph atlas plus billboard
or mesh text lowering before text glyphs are truly drawn inside the scene.
- Imported models: Fensalir resource contracts already know mesh packages, but
selected mesh draw/material lowerings are still future work. The current
import control creates graph nodes with paths; it does not decode ASSIMP
meshes or upload vertex/index resources yet.
- Pixel-accurate gizmo hit-testing: current gizmo edits apply to the selected
node. Fensalir needs an editor hit-test path before handles can be clicked
directly in world space.

View file

@ -0,0 +1,127 @@
# Self-Hosted Live Edge
Mimir's self-hosted broadcast path keeps OBS optional. Starfire owns the final
program frame and audio mix; Yggdrasil owns public byte distribution.
```mermaid
flowchart LR
A["Fensalir program texture"] --> B["Starfire NVENC encode"]
C["Faust/Mimir mix"] --> B
B --> D["SSH TCP forward"]
D --> E["Yggdrasil localhost RTMP ingest"]
E --> F["HLS segment ring"]
F --> G["streampixels.gamecult.org static HTTPS"]
G --> H["gamecult.org/livestream static viewer"]
```
## Authority Map
- Owner: `Mimir.Broadcast` owns the Starfire-side encoded push command.
- Inputs: final program media, FFmpeg/NVENC capability, and the SSH-forwarded
RTMP endpoint.
- Outputs: one CBR H.264/AAC RTMP stream to Yggdrasil.
- Derived state: Yggdrasil's HLS playlist and segments are derived from that
encoded stream. They do not own composition, timing, or transcoding.
- Forbidden writers: Yggdrasil must not transcode the program stream in the
first live edge. OBS is not the primary broadcast owner.
- Shared paths: synthetic smokes and real program pushes use the same RTMP URL:
`rtmp://127.0.0.1:11935/live/mimir`.
- Deletion line: if a later CDN or WebRTC edge appears, keep this path as the
simple origin proof or delete it. Do not keep two public stream authorities.
## First Smoke
On Starfire, with an SSH local forward active to Yggdrasil's localhost RTMP
port:
```powershell
dotnet run --project .\src\Mimir.Broadcast\Mimir.Broadcast.csproj -- --smoke
dotnet run --project .\src\Mimir.Broadcast\Mimir.Broadcast.csproj -- --print-command
```
The default push target is:
```text
rtmp://127.0.0.1:11935/live/mimir
```
The public HLS playlist is served on the existing Yggdrasil-routed
StreamPixels domain:
```text
https://streampixels.gamecult.org/mimir/live/hls/mimir.m3u8
```
The static viewer lives on the root GameCult site:
```text
https://gamecult.org/livestream
```
## Local Recording Before Public Push
Before pushing a live program to Yggdrasil, record the same simple stream-proof
shape locally and judge sync from the artifact:
```powershell
powershell -ExecutionPolicy Bypass -File .\scripts\record-stream-proof-output.ps1 -StartRavenSender -DurationSeconds 20
```
The recording harness listens for a Raven muxed A/V SRT feed on a private test
port, overlays Kiyo Pro as picture-in-picture, mixes Raven Realtek loopback with
Starfire's Scarlett capture, applies conservative FFmpeg denoise filters, and
writes an MP4 under `artifacts\runtime\stream-proof\`.
The current routing contract is explicit:
- Starfire Realtek/default render is the chirp/speaker emission lane.
- Raven Realtek/default render loopback is the co-streamer game/program lane.
It is packaged with Raven's NVENC video feed back to Starfire.
- Starfire Scarlett carries the hero mic capture and monitored local mix for
judgment. Raven's Scarlett is only routing the remote shotgun into Starfire
Scarlett, currently on channel 1.
- The proof recording is allowed to mix those lanes for human sync judgment, but
it must not collapse their authority. Starfire Realtek proves chirp emission,
Raven Realtek proves the co-streamer/game program path, and Scarlett proves
the hero mic/shotgun capture path.
This is a bridge recording, not public broadcast truth. It exists so humans can
judge audio/video sync and tell Mimir which offsets to apply before the RTMP/HLS
edge is involved. Offset knobs are explicit:
```powershell
powershell -ExecutionPolicy Bypass -File .\scripts\record-stream-proof-output.ps1 `
-StartRavenSender `
-RavenOffsetSeconds 0.000 `
-KiyoOffsetSeconds 0.000 `
-MicOffsetSeconds 0.000
```
If Windows firewall blocks direct SRT/UDP during a local proof, run the Raven
leg through an SSH TCP tunnel instead:
```powershell
Start-Process ssh.exe -ArgumentList '-o ExitOnForwardFailure=yes -N -L 6204:127.0.0.1:5204 -l "madman''s lullaby" 192.168.1.84' -WindowStyle Hidden
powershell -ExecutionPolicy Bypass -File .\scripts\record-stream-proof-output.ps1 `
-StartRavenSender `
-RavenSenderListens `
-RavenSenderTransport tcp-listener `
-RavenInputOverride tcp://127.0.0.1:6204
```
That fallback keeps the proof private to the LAN/SSH path and avoids changing
machine firewall policy just to capture a judging artifact.
## Scaling Rule
Viewer capacity is approximately:
```text
available Yggdrasil egress / encoded bitrate
```
Micro-optimizations count only when they improve target-host measurements:
served Mbps, p95 segment latency, CPU, memory, syscalls, or stall rate on
Yggdrasil. The first edge uses nginx static serving and kernel file paths so
the hot path is byte distribution, not application code.

View file

@ -10,7 +10,10 @@ flowchart TD
C["Leap stereo IR driver"] --> B
B --> D["native reservoir handles"]
D --> E["Fensalir GPU feature extraction"]
E --> J["per-sensor LED curve solve"]
E --> F["cross-view matching + flow"]
J --> K["camera rig calibration solver"]
K --> F
F --> G["surface/material claims"]
G --> H["brush/splat render budget"]
H --> I["Spout2/program video"]
@ -23,9 +26,51 @@ flowchart TD
- Process capture is a bridge edge, not the local six-camera foundation.
- Fensalir owns GPU extraction, fusion, material fitting, and render budgeting.
- Runtime buffers own retention and stream health, not scene reconstruction.
- LED strips are active calibration evidence. Each camera publishes ordered
bright-curve observations in its own local frustum/clipspace; only the global
residual calibration owner may fit those frustums into scene-space camera
intrinsics/extrinsics.
- Stable LED index identity requires temporal/color/address coding or another
correspondence source; uncoded identical lights are spline constraints, not
metric depth authority.
- PS Move controllers are active tracked-marker witnesses. USB HID owns
identity, buttons, IMU/report topology, and later LED/rumble control; the
glowing sphere is visual calibration evidence only when observed by cameras
and fused by the global residual owner.
- Exposure/gain sweeps optimize recovered spline quality: detected LED count,
curve coverage, spacing coherence, smoothness, exposure fitness, and
saturation penalty. Bright clipped blobs are not good evidence.
- The live LED sweep owner is `Mimir.BufferSmoke --led-spline-live-score` for
now: it opens direct sensor feeds, applies KS/UVC manual exposure/gain where
supported, scores actual luma frames, and reports the best setting per
source. PS3 Eye and Leap-specific controls still need native control cuts.
- The current all-sensor GPU proof is
`config/mimir-runtime.perfect-machine.local.json` plus
`Mimir.BufferSmoke --perfect-machine-live-smoke`: Leap, both PS3 Eyes, Kiyo
Pro, and regular Kiyo all enter the runtime as Fensalir texture leases. The
smoke requires five live camera Texture2D resources, one Leap stereo-depth
lowering, one disparity SurfacePage, and one derived point-cloud Mesh.
Camera Feature/SensorObservation claims now plan through Fensalir's
`GpuSensorFusion` backend instead of deferring. The current compute dispatch
samples LeapStereoIr, both Bayer8 PS3 Eyes, Kiyo Pro YUY2, and regular Kiyo
YUY2. Fensalir binds YUY2 through the documented `R8G8B8A8_UNORM` SRV view
over `DXGI_FORMAT_YUY2` and decodes `Y0/U/Y1/V` in compute before scoring
features.
- PS3 Eyes own high-rate sparse tracking evidence before they own depth:
`MimirSparseFeatureTracker` turns live Bayer/luma frames into stable
sensor-local feature tracks, and `MimirFeatureTrackFieldCandidate` lowers
them as camera Feature claims. This CPU path is a diagnostic bootstrap; the
production hot path belongs in Fensalir D3D12 feature extraction.
- The first camera-rig solver fits bounded camera translation updates from a
scene-anchored LED curve by minimizing ray-to-curve distance. Rotation,
intrinsics, and distortion are future owners.
- Once camera frustums are coherent, every detected surface candidate can be
resampled across the synchronized view set before Fensalir resolves it into
spatiotemporal surface/splat claims.
## Next Cut
Implement one concrete direct capture driver, feed `MimirVideoFrameDescriptor`
samples into `MimirVideoCaptureDriverSource`, and measure sustained cadence over
the default five-second window.
Replace provisional Leap projection constants with calibrated intrinsics and
add the global residual owner that consumes Leap surface errors, LED spline
observations, and later PS3 Eye/Kiyo/Eve feature tracks through one calibration
commit path.

View file

@ -0,0 +1,169 @@
# Mimir Verse Service Contract
Mimir is a Verse service for the measured room. It accepts local and network
sensors, publishes typed observations and capture pages, asks Fensalir to lower
field evidence into program surfaces, and exposes operator controls through
Eve/CultUI surfaces. OBS, browser dashboards, JSON logs, and screenshot streams
are lowerings or witnesses. They are not canonical presentation or state owners.
## Owner Map
- Mimir service owner: configuration, launch, calibration truth, sensor
identity, stream identity, synchronization state, Well publication, recorder
receipts, and service contract.
- `Mimir.Runtime`: bounded rolling buffers, source descriptors, source polling,
direct native ingest, audio synchronization analysis, and cached read models.
- Native sensor workers: device reads, closest-to-device timestamps, native or
GPU handles, unavoidable copy counts, and per-device actuator commands.
- `Mimir.Well`: typed live pages, configured composite state, timing pressure,
feature signals, capture-body publication, and the append-side observation
ledger.
- `Mimir.VerseRecorder`: run session receipts, body indexing, replay witnesses,
and `.cc`/CultCache-compatible export for offline reconstruction.
- `Mimir.FensalirDaemon`: reservoir job selection, queue pressure, GPU worker
ownership, program/reservoir surfaces, and daemon state.
- Fensalir: D3D12 resource resolution, field claims, selected lowerings,
temporal guide lanes, reservoir resolve, and program presentation.
- Faust/native DSP: hot audio alignment, sample movement, suppression,
separation, spatialization, and program stems.
- Eve/CultUI lowerings: operator interaction and display of provider-owned
state. They submit commands to owners; they do not own synchronization,
reservoir jobs, OBS composition, or calibration truth.
- Odin: discovery of Mimir's published CultMesh namespaces and Eve surfaces.
## CultCache Witness Contract
Durable state should be typed CultCache `.cc`, or CultCache-compatible with a
`.cc` witness/export during migration. The witness set is:
| State family | Owner | Durable shape |
| --- | --- | --- |
| Sensor state | `Mimir.Runtime` plus native workers | Source inventory, device identity, driver profile, clock domain, format, actuator capability, native handle class, copy count, and last health. |
| Media state | `Mimir.Well` / Fensalir | Capture page metadata, resource keys, body refs, fence refs, page timing, configured composites, and program surface refs. |
| Observation state | `Mimir.Runtime` / Well | Typed audio/video/IMU/feature/path observations with source id, time range, confidence, calibration id, and payload/resource reference. |
| Ledger state | `Mimir.VerseRecorder` | Session receipts, body index, replay cursor, migration version, schema witness, and rejected/drop reasons. |
| Control state | Runtime owners / Fensalir daemon / DSP | Operator commands, accepted command receipts, selected program layer state, reservoir worker pressure, audio actuator commands, and stream/record/live status. |
JSON may remain only as schema publication, diagnostic text export, external
tool boundary, or temporary replay witness. If a JSON file can decide behavior
without a typed `.cc` witness, it is still an owner and must be migrated.
## CultMesh Namespaces
Mimir should publish these namespaces for Odin discovery:
- `verse.mimir.service.v1`: service manifest, run id, host identity, schema
catalog, transport lanes, and nested Verse roots.
- `mimir.sensor.*`: source inventory, native worker health, device timing,
media format, actuator capability, and producer fences.
- `mimir.media.*`: capture pages, media bodies, resource references, program
texture/fence refs, stem refs, and replay body indexes.
- `mimir.observation.*`: sensor observations, media observations, feature
tracks, audio path evidence, calibration constraints, and local frustum hints.
- `mimir.ledger.*`: recorder session, append cursor, migration state,
body-index records, dropped-page reasons, and evidence receipts.
- `mimir.control.*`: operator commands, command receipts, recording/streaming
state, runtime mode, source placement, DSP actuator state, and reservoir
scheduling pressure.
- `mimir.eve.*`: `gamecult.eve.surface.v1` provider manifests, retained
surfaces, field bindings, assets, command bindings, and lowering metadata.
- `mimir.fensalir.*`: daemon summary, reservoir worker state, reservoir pressure,
program surfaces, resource resolver status, and selected lowering status.
Existing document names such as `mimir.well_snapshot.v1`,
`mimir.well_capture_page.v1`, `mimir.eve_sensor_observation.v1`,
`mimir.eve_media_observation.v1`, `mimir.eve_dashboard_state.v1`,
`mimir.fensalir_daemon_state.v1`, `mimir.fensalir_reservoir_worker_state.v1`,
and `mimir.fensalir_reservoir_pressure.v1` should be kept as concrete records
inside those namespaces until a schema catalog renames them deliberately.
## Transport Lanes
CultMesh reliable UDP is the default Verse transport for live Mimir network
state and media. TCP/WebSocket, SRT, browser streams, and local pipes are
diagnostic lowerings or external-boundary bridges unless a specific deployment
constraint temporarily promotes them.
The live lane split is:
- Typed control/state lane: compact documents such as timing state, command
receipts, health, stream cursors, and backpressure.
- Media stream-frame lane: typed per-frame envelopes such as
`mimir.cultmesh_stream_frame.v1` carrying source identity, capture time,
resource/body refs, payload format, and confidence.
- Media body-shard lane: bounded CultCache media/page shards with hashes,
cursors, receiver acknowledgements, and backpressure. Large bodies do not
travel as inline base64.
- Latest-state dashboard lane: lossy/current Eve surfaces for operators. It
may drop obsolete frames; it does not own durable media bodies.
Media streams are first-class Verse payloads. They still do not become timing
authority by arrival timestamp. Clock influence comes from decoded anchors,
source-local evidence, calibrated loopback, and explicit clock-domain state.
## Eve Surfaces
Mimir publishes operator presentation as Eve/CultUI DSL, preferably
`gamecult.eve.surface.v1`. The expected surfaces are:
- `mimir.sensor.field.surface`: sensor field view with source health, buffer
depth, timing confidence, calibration status, copy count, media format,
resource residency, and per-source observation pressure.
- `mimir.operator.flow.surface`: operator workflow for source selection,
program placement, audio strips, record/live controls, reservoir pressure,
sync mode, command receipts, and safe degraded-state visibility.
- `mimir.observation.ledger.surface`: session/replay view over observation
counts, body refs, accepted/dropped pages, cursor state, and schema witness.
- `mimir.fensalir.reservoir.surface`: daemon-owned worker queue, ready ratio,
timing confidence, GPU queue pressure, drop causes, and program surface refs.
The old dashboard broker, WebSocket `/eve/deck` lane, `/eve/dashboard`
compatibility endpoint, JSON health endpoints, JPEG streams, and browser
prototypes are lowerings or transport probes. They may mirror the Eve surface,
but they do not become presentation owners.
## Nested Verses
Mimir's service Verse contains smaller Verses with bounded authority:
- Room Verse: calibration truth, coordinate frame, sensor graph, clock model,
configured composites, and operator identity for one physical room.
- Sensor Verse: one device or logical source, including driver health, timing
domain, media format, actuator surface, and observation stream.
- Stream Verse: one synchronized media/control lane, including rolling-window
cursor, capture pages, resource refs, body refs, and backpressure state.
- Operator Verse: Eve/CultUI command surface, selected source/layer state,
command receipts, record/live state, and degraded-state visibility.
Nested Verses may publish local surfaces and state, but the parent Mimir service
owns discovery and contract routing. A sensor may report what it observed; it
does not become room clock authority unless explicitly promoted by calibration
state.
## Migration Order
1. Publish a `verse.mimir.service.v1` manifest that names active namespaces,
schema ids, run id, nested Verse roots, and the current transport lanes.
2. Give each current JSONL/session/body-index path a `.cc` witness or
CultCache-compatible export, starting with Well capture pages and
`Mimir.VerseRecorder`.
3. Promote sensor, media, observation, ledger, and control state into explicit
`mimir.*` CultMesh namespaces while preserving current document ids.
4. Publish `gamecult.eve.surface.v1` surfaces for sensor field and operator
flows, with typed field bindings back to the owning state documents.
5. Make the old dashboard broker and `/eve/dashboard` compatibility endpoint
consume those provider surfaces instead of assembling private truth.
6. Let Odin discover the service manifest and surfaces, then treat direct HTTP
health/status probes as temporary diagnostics.
7. Delete or neuter any private JSON store, dashboard state cache, or UI callback
path that can still decide service behavior after the typed owner exists.
## Demotion Line
Bespoke UI, browser dashboards, retained dashboard node lists, compatibility
WebSockets, SRT/FFmpeg media bridges, JSON status files, JSONL replay logs,
screenshot streams, and OBS bridge scripts are demoted to lowerings,
diagnostics, external-boundary export, or replay witnesses. They may display,
transport, inspect, or reconstruct typed Mimir state. They may not own sensor
truth, calibration truth, command state, reservoir scheduling, program
presentation, media-over-Verse policy, or synchronization behavior.

View file

@ -107,11 +107,14 @@ The first cut should be deliberately boring:
- `MimirNativeIngestStreamSource` is the low-overhead push seam for native
camera/audio adapters that already own buffers or handles.
- `MimirVideoFrameDescriptor` carries video shape, pixel format, device
timestamp, stride, and optional native/GPU handle metadata. Leap stereo IR
should enter here through a native driver path.
timestamp, stride, resource key, producer fence value, and native/GPU handle
metadata. Leap stereo IR should enter here through a native driver path that
reports any unavoidable system-memory copy.
- `IMimirVideoCaptureDriver` and `MimirVideoCaptureDriverSource` are the
driver-facing seam for polling/pushing frames from LeapUVC/libusb, LeapC image
APIs, Media Foundation, DirectShow, or a future GPU shared-texture capture.
APIs, Media Foundation, DirectShow, or GPU decode/capture paths. Drivers that
can produce render inputs should implement `IMimirFensalirTextureLeaseReceiver`
and populate Fensalir-owned texture leases directly.
- `MimirProcessStreamSource` is a compatibility adapter for network or
diagnostic command sources. It is not the six-camera local capture foundation.
@ -129,10 +132,10 @@ environment variables:
These create buffers immediately. Capture adapters still need to register
`IMimirStreamSource` implementations to feed live samples into those buffers.
For local cameras, prefer native adapters that push sample handles directly
through `MimirNativeIngestStreamSource`. Process-backed FFmpeg ingest is allowed
for remote SRT/diagnostic edges, but local six-camera ingest should stay close
to the driver, GPU, and capture APIs.
For local cameras, prefer native adapters that talk to the device stack directly
and push only typed sample/resource handles through `MimirNativeIngestStreamSource`.
Process-backed FFmpeg ingest is allowed for remote SRT/diagnostic edges, but
local six-camera ingest should stay close to the device, GPU, and capture APIs.
Leap should be treated as the timing-camera candidate only when the native
adapter preserves device timestamps or captures a monotonic timestamp

View file

@ -0,0 +1,65 @@
declare name "Mimir Dual Mic Dialogue Cleaner";
declare version "0.1";
declare author "Mimir";
declare license "MIT";
declare description "Faust-owned smooth dual-mic dialogue cleanup target: aligned shotgun/cardioid inputs become a conservative dialogue stem and a rejected residual witness.";
import("stdfaust.lib");
// Inputs are already delay-aligned upstream by mimir_alignment_actuator.
// This graph owns smooth audio-rate cleanup, not calibration, timing, or UI.
//
// input 0: aligned shotgun / co-streamer witness
// input 1: aligned cardioid / host witness
// output 0: dialogue composite stem
// output 1: rejected residual / noise witness
drive = hslider("dialogue/drive", 1.00, 0.00, 2.00, 0.001);
shotgunWeight = hslider("dialogue/shotgun_weight", 0.45, 0.00, 1.00, 0.001);
cardioidWeight = hslider("dialogue/cardioid_weight", 0.55, 0.00, 1.00, 0.001);
witnessReject = hslider("dialogue/witness_reject", 0.20, 0.00, 1.00, 0.001);
lowShelf = hslider("dialogue/low_shelf", 0.80, 0.00, 2.00, 0.001);
presenceShelf = hslider("dialogue/presence_shelf", 1.08, 0.00, 2.00, 0.001);
airShelf = hslider("dialogue/air_shelf", 0.90, 0.00, 2.00, 0.001);
// Smooth broadband downward expansion. This is intentionally slow enough to
// avoid the choppy gate artifact from the C# audition bridge.
floorLevel = hslider("denoise/noise_floor", 0.018, 0.0001, 0.25, 0.0001);
expansionDepth = hslider("denoise/expansion_depth", 0.32, 0.00, 0.95, 0.001);
expansionCurve = hslider("denoise/expansion_curve", 1.35, 0.25, 4.00, 0.001);
envelopeHz = hslider("denoise/envelope_hz", 18.0, 1.0, 80.0, 0.1);
band(f, q, g, x) = x : fi.resonbp(1.0, f, q) : *(g);
voiceShape(x) =
band(120.0, 1.2, lowShelf, x) +
band(220.0, 1.6, 0.95, x) +
band(420.0, 2.0, 0.98, x) +
band(800.0, 2.4, 1.00, x) +
band(1500.0, 2.8, presenceShelf, x) +
band(2800.0, 3.2, presenceShelf, x) +
band(5200.0, 3.2, airShelf, x) +
band(8500.0, 2.6, airShelf, x);
smoothAbs(x) = abs(x) : si.smooth(ba.tau2pole(1.0 / max(1.0, envelopeHz)));
softExpand(x) = x * gain
with {
env = smoothAbs(x);
ratio = min(1.0, env / max(0.0001, floorLevel));
openness = pow(ratio, expansionCurve);
gain = 1.0 - expansionDepth * (1.0 - openness);
};
softClip(x) = x / (1.0 + abs(x));
clean(shotgun, cardioid) = dialogue, residual
with {
weighted = shotgun * shotgunWeight + cardioid * cardioidWeight;
witness = (shotgun - cardioid) * 0.5;
shaped = voiceShape(weighted - witnessReject * witness);
dialogue = softExpand(shaped * drive) : softClip;
residual = witness + (weighted - dialogue) * 0.25;
};
process = clean;

View file

@ -0,0 +1,70 @@
cmake_minimum_required(VERSION 3.20)
project(mimir_camera_capture LANGUAGES C CXX)
add_library(mimir_camera_capture SHARED
mimir_ks_camera_capture.cpp)
target_compile_features(mimir_camera_capture PRIVATE cxx_std_20)
target_compile_definitions(mimir_camera_capture PRIVATE
WIN32_LEAN_AND_MEAN
NOMINMAX)
target_link_libraries(mimir_camera_capture PRIVATE
setupapi
ksuser)
set(PSMOVEAPI_ROOT "$ENV{TEMP}/mimir-driver-probes/psmoveapi" CACHE PATH "Root of a psmoveapi checkout with external/libusb-1.0 and external/PS3EYEDriver.")
set(LIBUSB_ROOT "${PSMOVEAPI_ROOT}/external/libusb-1.0" CACHE PATH "libusb source root.")
set(PS3EYE_ROOT "${PSMOVEAPI_ROOT}/external/PS3EYEDriver" CACHE PATH "PS3EYEDriver source root.")
add_library(mimir_libusb_static STATIC
${LIBUSB_ROOT}/libusb/core.c
${LIBUSB_ROOT}/libusb/descriptor.c
${LIBUSB_ROOT}/libusb/hotplug.c
${LIBUSB_ROOT}/libusb/io.c
${LIBUSB_ROOT}/libusb/strerror.c
${LIBUSB_ROOT}/libusb/sync.c
${LIBUSB_ROOT}/libusb/os/events_windows.c
${LIBUSB_ROOT}/libusb/os/threads_windows.c
${LIBUSB_ROOT}/libusb/os/windows_common.c
${LIBUSB_ROOT}/libusb/os/windows_usbdk.c
${LIBUSB_ROOT}/libusb/os/windows_winusb.c)
target_include_directories(mimir_libusb_static PUBLIC
${LIBUSB_ROOT}/libusb
${LIBUSB_ROOT}/libusb/os
${LIBUSB_ROOT}/msvc)
target_compile_definitions(mimir_libusb_static PRIVATE
_CRT_SECURE_NO_WARNINGS
_WIN32_WINNT=0x0601)
target_link_libraries(mimir_libusb_static PRIVATE setupapi ole32 advapi32)
add_library(mimir_ps3eye_vendor STATIC
${PS3EYE_ROOT}/src/ps3eye.cpp
${PS3EYE_ROOT}/src/ps3eye_capi.cpp)
target_include_directories(mimir_ps3eye_vendor PUBLIC
${PS3EYE_ROOT}/src
${LIBUSB_ROOT}/libusb)
target_link_libraries(mimir_ps3eye_vendor PUBLIC mimir_libusb_static)
add_library(mimir_ps3eye_capture SHARED
mimir_ps3eye_capture.cpp)
target_compile_features(mimir_ps3eye_capture PRIVATE cxx_std_20)
target_compile_definitions(mimir_ps3eye_capture PRIVATE
WIN32_LEAN_AND_MEAN
NOMINMAX)
target_include_directories(mimir_ps3eye_capture PRIVATE ${PS3EYE_ROOT}/src)
target_link_libraries(mimir_ps3eye_capture PRIVATE mimir_ps3eye_vendor)
add_library(mimir_mf_gpu_capture SHARED
mimir_mf_gpu_capture.cpp)
target_compile_features(mimir_mf_gpu_capture PRIVATE cxx_std_20)
target_compile_definitions(mimir_mf_gpu_capture PRIVATE
WIN32_LEAN_AND_MEAN
NOMINMAX)
target_link_libraries(mimir_mf_gpu_capture PRIVATE
d3d11
dxgi
mf
mfplat
mfreadwrite
mfuuid)

View file

@ -0,0 +1,759 @@
#include <windows.h>
#include <winioctl.h>
#include <setupapi.h>
#include <ks.h>
#include <ksmedia.h>
#include <winternl.h>
#include <algorithm>
#include <atomic>
#include <cctype>
#include <chrono>
#include <condition_variable>
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <deque>
#include <mutex>
#include <string>
#include <thread>
#include <vector>
namespace
{
struct Handle
{
HANDLE value = INVALID_HANDLE_VALUE;
Handle() = default;
explicit Handle(HANDLE handle) : value(handle) {}
Handle(const Handle&) = delete;
Handle& operator=(const Handle&) = delete;
Handle(Handle&& other) noexcept : value(other.value) { other.value = INVALID_HANDLE_VALUE; }
Handle& operator=(Handle&& other) noexcept
{
if (this != &other)
{
close();
value = other.value;
other.value = INVALID_HANDLE_VALUE;
}
return *this;
}
~Handle() { close(); }
void close()
{
if (value != INVALID_HANDLE_VALUE && value != nullptr)
{
CloseHandle(value);
value = INVALID_HANDLE_VALUE;
}
}
explicit operator bool() const { return value != INVALID_HANDLE_VALUE && value != nullptr; }
};
struct DeviceInfoSet
{
HDEVINFO value = INVALID_HANDLE_VALUE;
explicit DeviceInfoSet(HDEVINFO info) : value(info) {}
DeviceInfoSet(const DeviceInfoSet&) = delete;
DeviceInfoSet& operator=(const DeviceInfoSet&) = delete;
~DeviceInfoSet()
{
if (value != INVALID_HANDLE_VALUE)
{
SetupDiDestroyDeviceInfoList(value);
}
}
};
using NtDeviceIoControlFileFn = NTSTATUS(NTAPI*)(
HANDLE FileHandle,
HANDLE Event,
PIO_APC_ROUTINE ApcRoutine,
PVOID ApcContext,
PIO_STATUS_BLOCK IoStatusBlock,
ULONG IoControlCode,
PVOID InputBuffer,
ULONG InputBufferLength,
PVOID OutputBuffer,
ULONG OutputBufferLength);
constexpr NTSTATUS StatusPending = static_cast<NTSTATUS>(0x00000103L);
struct PinCandidate
{
ULONG pinId = 0;
std::vector<std::uint8_t> format;
LONG width = 0;
LONG height = 0;
LONG imageBytes = 0;
LONGLONG interval100ns = 0;
GUID subtype{};
std::string fourcc;
};
struct MimirKsCameraFrame
{
std::vector<std::uint8_t> data;
std::int64_t timestampNs = 0;
std::uint64_t sequence = 0;
int byteLength = 0;
};
struct MimirKsCameraCapture
{
Handle filter;
Handle pin;
PinCandidate candidate;
std::thread worker;
std::mutex mutex;
std::condition_variable condition;
std::deque<MimirKsCameraFrame> queue;
std::atomic<bool> running = false;
int queueDepth = 8;
std::uint64_t sequence = 0;
};
long long nowNs()
{
return std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::steady_clock::now().time_since_epoch()).count();
}
std::string lowercase(std::string value)
{
std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) {
return static_cast<char>(std::tolower(c));
});
return value;
}
std::string wideToUtf8(const wchar_t* value)
{
if (!value)
{
return {};
}
const int bytes = WideCharToMultiByte(CP_UTF8, 0, value, -1, nullptr, 0, nullptr, nullptr);
if (bytes <= 1)
{
return {};
}
std::string text(static_cast<std::size_t>(bytes - 1), '\0');
WideCharToMultiByte(CP_UTF8, 0, value, -1, text.data(), bytes, nullptr, nullptr);
return text;
}
std::string fourccOrGuid(const GUID& guid)
{
if (guid.Data2 == 0x0000 &&
guid.Data3 == 0x0010 &&
guid.Data4[0] == 0x80 &&
guid.Data4[1] == 0x00 &&
guid.Data4[2] == 0x00 &&
guid.Data4[3] == 0xaa &&
guid.Data4[4] == 0x00 &&
guid.Data4[5] == 0x38 &&
guid.Data4[6] == 0x9b &&
guid.Data4[7] == 0x71)
{
char text[5]{};
text[0] = static_cast<char>(guid.Data1 & 0xff);
text[1] = static_cast<char>((guid.Data1 >> 8) & 0xff);
text[2] = static_cast<char>((guid.Data1 >> 16) & 0xff);
text[3] = static_cast<char>((guid.Data1 >> 24) & 0xff);
return text;
}
return {};
}
bool ksProperty(HANDLE handle, void* property, DWORD propertyBytes, void* output, DWORD outputBytes, DWORD* returned = nullptr)
{
DWORD bytes = 0;
const BOOL ok = DeviceIoControl(
handle,
IOCTL_KS_PROPERTY,
property,
propertyBytes,
output,
outputBytes,
&bytes,
nullptr);
if (returned)
{
*returned = bytes;
}
return ok != FALSE;
}
template <typename T>
bool getPinProperty(HANDLE filter, ULONG pinId, ULONG propertyId, T& value)
{
KSP_PIN property{};
property.Property.Set = KSPROPSETID_Pin;
property.Property.Id = propertyId;
property.Property.Flags = KSPROPERTY_TYPE_GET;
property.PinId = pinId;
DWORD returned = 0;
return ksProperty(filter, &property, sizeof(property), &value, sizeof(value), &returned);
}
bool setVideoProcAmp(HANDLE filter, ULONG id, LONG value, ULONG flags)
{
KSPROPERTY_VIDEOPROCAMP_S property{};
property.Property.Set = PROPSETID_VIDCAP_VIDEOPROCAMP;
property.Property.Id = id;
property.Property.Flags = KSPROPERTY_TYPE_SET;
property.Value = value;
property.Flags = flags;
return ksProperty(filter, &property, sizeof(property), &property, sizeof(property));
}
bool setCameraControl(HANDLE filter, ULONG id, LONG value, ULONG flags)
{
KSPROPERTY_CAMERACONTROL_S property{};
property.Property.Set = PROPSETID_VIDCAP_CAMERACONTROL;
property.Property.Id = id;
property.Property.Flags = KSPROPERTY_TYPE_SET;
property.Value = value;
property.Flags = flags;
return ksProperty(filter, &property, sizeof(property), &property, sizeof(property));
}
std::vector<std::string> enumerateCapturePaths()
{
std::vector<std::string> paths;
DeviceInfoSet info(SetupDiGetClassDevsA(
&KSCATEGORY_CAPTURE,
nullptr,
nullptr,
DIGCF_PRESENT | DIGCF_DEVICEINTERFACE));
if (info.value == INVALID_HANDLE_VALUE)
{
return paths;
}
for (DWORD index = 0;; ++index)
{
SP_DEVICE_INTERFACE_DATA interfaceData{};
interfaceData.cbSize = sizeof(interfaceData);
if (!SetupDiEnumDeviceInterfaces(info.value, nullptr, &KSCATEGORY_CAPTURE, index, &interfaceData))
{
break;
}
DWORD required = 0;
SetupDiGetDeviceInterfaceDetailW(info.value, &interfaceData, nullptr, 0, &required, nullptr);
if (required == 0)
{
continue;
}
std::vector<std::uint8_t> storage(required);
auto* detail = reinterpret_cast<SP_DEVICE_INTERFACE_DETAIL_DATA_W*>(storage.data());
detail->cbSize = sizeof(*detail);
if (SetupDiGetDeviceInterfaceDetailW(info.value, &interfaceData, detail, required, nullptr, nullptr))
{
paths.push_back(wideToUtf8(detail->DevicePath));
}
}
return paths;
}
std::vector<PinCandidate> inspectPins(HANDLE filter)
{
ULONG pinCount = 0;
KSPROPERTY countProperty{};
countProperty.Set = KSPROPSETID_Pin;
countProperty.Id = KSPROPERTY_PIN_CTYPES;
countProperty.Flags = KSPROPERTY_TYPE_GET;
if (!ksProperty(filter, &countProperty, sizeof(countProperty), &pinCount, sizeof(pinCount)))
{
return {};
}
std::vector<PinCandidate> candidates;
for (ULONG pinId = 0; pinId < pinCount; ++pinId)
{
KSPIN_DATAFLOW dataflow{};
if (!getPinProperty(filter, pinId, KSPROPERTY_PIN_DATAFLOW, dataflow) ||
dataflow != KSPIN_DATAFLOW_OUT)
{
continue;
}
KSP_PIN rangeProperty{};
rangeProperty.Property.Set = KSPROPSETID_Pin;
rangeProperty.Property.Id = KSPROPERTY_PIN_DATARANGES;
rangeProperty.Property.Flags = KSPROPERTY_TYPE_GET;
rangeProperty.PinId = pinId;
DWORD rangeBytes = 0;
ksProperty(filter, &rangeProperty, sizeof(rangeProperty), nullptr, 0, &rangeBytes);
if (rangeBytes == 0)
{
continue;
}
std::vector<std::uint8_t> ranges(rangeBytes);
DWORD returned = 0;
if (!ksProperty(filter, &rangeProperty, sizeof(rangeProperty), ranges.data(), static_cast<DWORD>(ranges.size()), &returned))
{
continue;
}
auto* multiple = reinterpret_cast<KSMULTIPLE_ITEM*>(ranges.data());
auto* cursor = ranges.data() + sizeof(KSMULTIPLE_ITEM);
for (ULONG i = 0; i < multiple->Count; ++i)
{
auto* dataRange = reinterpret_cast<KSDATARANGE*>(cursor);
const bool videoInfo =
IsEqualGUID(dataRange->MajorFormat, KSDATAFORMAT_TYPE_VIDEO) &&
IsEqualGUID(dataRange->Specifier, KSDATAFORMAT_SPECIFIER_VIDEOINFO) &&
dataRange->FormatSize >= sizeof(KS_DATARANGE_VIDEO);
if (videoInfo)
{
auto* video = reinterpret_cast<KS_DATARANGE_VIDEO*>(cursor);
const auto& bmi = video->VideoInfoHeader.bmiHeader;
PinCandidate candidate{};
candidate.pinId = pinId;
candidate.width = bmi.biWidth;
candidate.height = bmi.biHeight;
candidate.imageBytes = bmi.biSizeImage > 0 ? bmi.biSizeImage : static_cast<LONG>(dataRange->SampleSize);
candidate.interval100ns = video->VideoInfoHeader.AvgTimePerFrame;
candidate.subtype = dataRange->SubFormat;
candidate.fourcc = fourccOrGuid(candidate.subtype);
candidate.format.resize(sizeof(KS_DATAFORMAT_VIDEOINFOHEADER));
auto* format = reinterpret_cast<KS_DATAFORMAT_VIDEOINFOHEADER*>(candidate.format.data());
format->DataFormat.FormatSize = sizeof(KS_DATAFORMAT_VIDEOINFOHEADER);
format->DataFormat.SampleSize = candidate.imageBytes;
format->DataFormat.MajorFormat = KSDATAFORMAT_TYPE_VIDEO;
format->DataFormat.SubFormat = dataRange->SubFormat;
format->DataFormat.Specifier = KSDATAFORMAT_SPECIFIER_VIDEOINFO;
format->VideoInfoHeader = video->VideoInfoHeader;
format->VideoInfoHeader.bmiHeader.biSizeImage = candidate.imageBytes;
candidates.push_back(std::move(candidate));
}
cursor += dataRange->FormatSize;
}
}
return candidates;
}
const PinCandidate* findCandidate(
const std::vector<PinCandidate>& candidates,
int width,
int height,
const char* subtype,
double minFps)
{
const std::string target = subtype ? subtype : "";
for (const auto& candidate : candidates)
{
const double fps = candidate.interval100ns > 0
? 10000000.0 / static_cast<double>(candidate.interval100ns)
: 0.0;
if (candidate.width == width &&
candidate.height == height &&
fps >= minFps &&
candidate.fourcc == target)
{
return &candidate;
}
}
return nullptr;
}
bool setState(HANDLE pin, KSSTATE state)
{
KSPROPERTY property{};
property.Set = KSPROPSETID_Connection;
property.Id = KSPROPERTY_CONNECTION_STATE;
property.Flags = KSPROPERTY_TYPE_SET;
DWORD returned = 0;
return DeviceIoControl(
pin,
IOCTL_KS_PROPERTY,
&property,
sizeof(property),
&state,
sizeof(state),
&returned,
nullptr) != FALSE;
}
int strideBytes(const PinCandidate& candidate)
{
return candidate.height > 0 && candidate.imageBytes > 0
? candidate.imageBytes / candidate.height
: 0;
}
void pushFrame(MimirKsCameraCapture& capture, const std::uint8_t* data, ULONG byteLength)
{
MimirKsCameraFrame frame;
frame.byteLength = static_cast<int>(byteLength);
frame.timestampNs = nowNs();
frame.sequence = capture.sequence++;
frame.data.resize(byteLength);
std::memcpy(frame.data.data(), data, byteLength);
{
std::lock_guard<std::mutex> lock(capture.mutex);
capture.queue.push_back(std::move(frame));
while (capture.queue.size() > 32)
{
capture.queue.pop_front();
}
}
capture.condition.notify_one();
}
void captureThread(MimirKsCameraCapture* capture)
{
auto* ntDeviceIoControlFile = reinterpret_cast<NtDeviceIoControlFileFn>(
GetProcAddress(GetModuleHandleA("ntdll.dll"), "NtDeviceIoControlFile"));
if (!ntDeviceIoControlFile)
{
capture->running.store(false);
return;
}
const ULONG connectBytes = sizeof(KSPIN_CONNECT) + static_cast<ULONG>(capture->candidate.format.size());
std::vector<std::uint8_t> connectStorage(connectBytes);
auto* connect = reinterpret_cast<KSPIN_CONNECT*>(connectStorage.data());
connect->Interface.Set = KSINTERFACESETID_Standard;
connect->Interface.Id = KSINTERFACE_STANDARD_STREAMING;
connect->Medium.Set = KSMEDIUMSETID_Standard;
connect->Medium.Id = KSMEDIUM_TYPE_ANYINSTANCE;
connect->PinId = capture->candidate.pinId;
connect->Priority.PriorityClass = KSPRIORITY_NORMAL;
connect->Priority.PrioritySubClass = 1;
std::memcpy(connectStorage.data() + sizeof(KSPIN_CONNECT), capture->candidate.format.data(), capture->candidate.format.size());
HANDLE rawPin = INVALID_HANDLE_VALUE;
const HRESULT hr = KsCreatePin(capture->filter.value, connect, GENERIC_READ, &rawPin);
if (FAILED(hr))
{
capture->running.store(false);
return;
}
capture->pin = Handle(rawPin);
if (!setState(capture->pin.value, KSSTATE_ACQUIRE) ||
!setState(capture->pin.value, KSSTATE_PAUSE) ||
!setState(capture->pin.value, KSSTATE_RUN))
{
capture->running.store(false);
return;
}
const DWORD frameBytes = static_cast<DWORD>(capture->candidate.imageBytes);
struct ReadSlot
{
Handle event;
IO_STATUS_BLOCK status{};
KSSTREAM_HEADER header{};
std::vector<std::uint8_t> frame;
bool pending = false;
};
std::vector<ReadSlot> slots(static_cast<std::size_t>(std::max(1, capture->queueDepth)));
std::vector<HANDLE> waitEvents(slots.size());
for (auto& slot : slots)
{
slot.event = Handle(CreateEventW(nullptr, TRUE, FALSE, nullptr));
if (!slot.event)
{
capture->running.store(false);
return;
}
slot.frame.resize(frameBytes);
slot.header.Size = sizeof(KSSTREAM_HEADER);
slot.header.FrameExtent = frameBytes;
slot.header.Data = slot.frame.data();
slot.header.PresentationTime.Numerator = 1;
slot.header.PresentationTime.Denominator = 1;
waitEvents[&slot - slots.data()] = slot.event.value;
}
auto submit = [&](ReadSlot& slot) -> bool {
ResetEvent(slot.event.value);
slot.status = {};
slot.header.DataUsed = 0;
slot.header.OptionsFlags = 0;
const NTSTATUS status = ntDeviceIoControlFile(
capture->pin.value,
slot.event.value,
nullptr,
nullptr,
&slot.status,
IOCTL_KS_READ_STREAM,
&slot.header,
sizeof(slot.header),
&slot.header,
sizeof(slot.header));
if (status == StatusPending)
{
slot.pending = true;
return true;
}
slot.pending = false;
if (status >= 0 && slot.header.DataUsed > 0)
{
pushFrame(*capture, slot.frame.data(), slot.header.DataUsed);
return true;
}
return false;
};
for (auto& slot : slots)
{
if (!submit(slot))
{
capture->running.store(false);
return;
}
}
while (capture->running.load())
{
const DWORD waitResult = WaitForMultipleObjects(
static_cast<DWORD>(waitEvents.size()),
waitEvents.data(),
FALSE,
100);
if (waitResult == WAIT_TIMEOUT)
{
continue;
}
if (waitResult < WAIT_OBJECT_0 || waitResult >= WAIT_OBJECT_0 + waitEvents.size())
{
break;
}
auto& slot = slots[static_cast<std::size_t>(waitResult - WAIT_OBJECT_0)];
slot.pending = false;
if (slot.status.Status < 0 || slot.header.DataUsed == 0)
{
break;
}
pushFrame(*capture, slot.frame.data(), slot.header.DataUsed);
if (!submit(slot))
{
break;
}
}
setState(capture->pin.value, KSSTATE_STOP);
CancelIoEx(capture->pin.value, nullptr);
for (auto& slot : slots)
{
if (slot.pending)
{
WaitForSingleObject(slot.event.value, 100);
}
}
capture->running.store(false);
}
} // namespace
extern "C"
{
__declspec(dllexport) MimirKsCameraCapture* mimir_ks_create(
const char* pathNeedle,
int width,
int height,
const char* subtype,
double minFps,
int queueDepth)
{
const std::string needle = lowercase(pathNeedle ? pathNeedle : "");
for (const auto& path : enumerateCapturePaths())
{
if (!needle.empty() && lowercase(path).find(needle) == std::string::npos)
{
continue;
}
Handle filter(CreateFileA(
path.c_str(),
GENERIC_READ | GENERIC_WRITE,
FILE_SHARE_READ | FILE_SHARE_WRITE,
nullptr,
OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL,
nullptr));
if (!filter)
{
continue;
}
auto candidates = inspectPins(filter.value);
const auto* candidate = findCandidate(candidates, width, height, subtype, minFps);
if (!candidate)
{
continue;
}
auto* capture = new MimirKsCameraCapture();
capture->filter = std::move(filter);
capture->candidate = *candidate;
capture->queueDepth = std::max(1, std::min(queueDepth, 32));
return capture;
}
return nullptr;
}
__declspec(dllexport) int mimir_ks_start(MimirKsCameraCapture* capture)
{
if (!capture || capture->running.exchange(true))
{
return 0;
}
capture->worker = std::thread(captureThread, capture);
return 1;
}
__declspec(dllexport) int mimir_ks_set_exposure_gain(
MimirKsCameraCapture* capture,
int setExposure,
LONG exposure,
int setGain,
LONG gain)
{
if (!capture || !capture->filter)
{
return 0;
}
int result = 0;
if (!setExposure ||
setCameraControl(
capture->filter.value,
KSPROPERTY_CAMERACONTROL_EXPOSURE,
exposure,
KSPROPERTY_CAMERACONTROL_FLAGS_MANUAL))
{
result |= 1;
}
if (!setGain ||
setVideoProcAmp(
capture->filter.value,
KSPROPERTY_VIDEOPROCAMP_GAIN,
gain,
KSPROPERTY_VIDEOPROCAMP_FLAGS_MANUAL))
{
result |= 2;
}
return result;
}
__declspec(dllexport) int mimir_ks_read(
MimirKsCameraCapture* capture,
int* width,
int* height,
int* stride,
char* fourcc,
int fourccCapacity,
std::int64_t* timestampNs,
std::uint64_t* sequence,
std::uint8_t* destination,
int destinationBytes)
{
if (!capture || !destination || destinationBytes <= 0)
{
return 0;
}
MimirKsCameraFrame frame;
{
std::lock_guard<std::mutex> lock(capture->mutex);
if (capture->queue.empty())
{
return 0;
}
frame = std::move(capture->queue.front());
capture->queue.pop_front();
}
if (frame.byteLength > destinationBytes)
{
return -frame.byteLength;
}
std::memcpy(destination, frame.data.data(), static_cast<std::size_t>(frame.byteLength));
if (width)
{
*width = static_cast<int>(capture->candidate.width);
}
if (height)
{
*height = static_cast<int>(capture->candidate.height);
}
if (stride)
{
*stride = strideBytes(capture->candidate);
}
if (fourcc && fourccCapacity > 0)
{
std::snprintf(fourcc, static_cast<std::size_t>(fourccCapacity), "%s", capture->candidate.fourcc.c_str());
}
if (timestampNs)
{
*timestampNs = frame.timestampNs;
}
if (sequence)
{
*sequence = frame.sequence;
}
return frame.byteLength;
}
__declspec(dllexport) void mimir_ks_destroy(MimirKsCameraCapture* capture)
{
if (!capture)
{
return;
}
capture->running.store(false);
if (capture->pin)
{
CancelIoEx(capture->pin.value, nullptr);
}
if (capture->worker.joinable())
{
capture->worker.join();
}
delete capture;
}
}

View file

@ -0,0 +1,573 @@
#include <windows.h>
#include <d3d11_4.h>
#include <dxgi1_6.h>
#include <mfapi.h>
#include <mferror.h>
#include <mfidl.h>
#include <mfreadwrite.h>
#include <wrl/client.h>
#include <algorithm>
#include <cctype>
#include <cstdint>
#include <cstdio>
#include <cstdarg>
#include <string>
#include <vector>
using Microsoft::WRL::ComPtr;
namespace
{
struct MimirMfGpuCapture
{
ComPtr<IMFMediaSource> source;
ComPtr<IMFSourceReader> reader;
ComPtr<ID3D11Device> device;
ComPtr<ID3D11DeviceContext> context;
ComPtr<IMFDXGIDeviceManager> deviceManager;
ComPtr<ID3D11Texture2D> sharedTexture;
ComPtr<ID3D11Fence> producerFence;
ComPtr<ID3D11DeviceContext4> fenceContext;
HANDLE sharedHandle = nullptr;
HANDLE producerFenceHandle = nullptr;
int width = 0;
int height = 0;
DXGI_FORMAT format = DXGI_FORMAT_UNKNOWN;
std::uint64_t sequence = 0;
};
bool debugEnabled()
{
return GetEnvironmentVariableA("MIMIR_MF_GPU_DEBUG", nullptr, 0) > 0;
}
void debugLog(const char* format, ...)
{
if (!debugEnabled())
{
return;
}
std::fprintf(stderr, "mimir-mf-gpu: ");
va_list args;
va_start(args, format);
std::vfprintf(stderr, format, args);
va_end(args);
std::fprintf(stderr, "\n");
}
std::wstring widen(const char* value)
{
if (!value)
{
return {};
}
const int chars = MultiByteToWideChar(CP_UTF8, 0, value, -1, nullptr, 0);
if (chars <= 1)
{
return {};
}
std::wstring result(static_cast<std::size_t>(chars - 1), L'\0');
MultiByteToWideChar(CP_UTF8, 0, value, -1, result.data(), chars);
return result;
}
std::wstring lowercase(std::wstring value)
{
std::transform(value.begin(), value.end(), value.begin(), [](wchar_t c) {
return static_cast<wchar_t>(towlower(c));
});
return value;
}
GUID subtypeFromText(const char* text)
{
const std::string value = text ? text : "";
std::string upper;
upper.resize(value.size());
std::transform(value.begin(), value.end(), upper.begin(), [](unsigned char c) {
return static_cast<char>(std::toupper(c));
});
if (upper == "MJPG") return MFVideoFormat_MJPG;
if (upper == "H264") return MFVideoFormat_H264;
if (upper == "NV12") return MFVideoFormat_NV12;
if (upper == "YUY2") return MFVideoFormat_YUY2;
if (upper == "RGB32" || upper == "BGRA") return MFVideoFormat_RGB32;
return GUID_NULL;
}
DXGI_FORMAT dxgiFormatForSubtype(const GUID& subtype)
{
if (subtype == MFVideoFormat_NV12) return DXGI_FORMAT_NV12;
if (subtype == MFVideoFormat_RGB32) return DXGI_FORMAT_B8G8R8A8_UNORM;
if (subtype == MFVideoFormat_YUY2) return DXGI_FORMAT_YUY2;
return DXGI_FORMAT_UNKNOWN;
}
const char* textForSubtype(const GUID& subtype)
{
if (subtype == MFVideoFormat_NV12) return "NV12";
if (subtype == MFVideoFormat_RGB32) return "Bgra8";
if (subtype == MFVideoFormat_YUY2) return "YUY2";
return "Unknown";
}
bool findVideoSource(const wchar_t* needle, IMFMediaSource** mediaSource)
{
*mediaSource = nullptr;
ComPtr<IMFAttributes> attributes;
if (FAILED(MFCreateAttributes(&attributes, 2)))
{
debugLog("MFCreateAttributes for devices failed");
return false;
}
attributes->SetGUID(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE, MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID);
IMFActivate** rawDevices = nullptr;
UINT32 count = 0;
if (FAILED(MFEnumDeviceSources(attributes.Get(), &rawDevices, &count)))
{
debugLog("MFEnumDeviceSources failed");
return false;
}
debugLog("device sources=%u", count);
const std::wstring target = lowercase(needle ? needle : L"");
bool found = false;
for (UINT32 index = 0; index < count; ++index)
{
ComPtr<IMFActivate> activate(rawDevices[index]);
rawDevices[index] = nullptr;
WCHAR* symbolicLink = nullptr;
UINT32 symbolicLength = 0;
activate->GetAllocatedString(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK, &symbolicLink, &symbolicLength);
const std::wstring link = lowercase(symbolicLink ? symbolicLink : L"");
if (debugEnabled())
{
std::fwprintf(stderr, L"mimir-mf-gpu: source[%u]=%ls\n", index, link.c_str());
}
CoTaskMemFree(symbolicLink);
if (!target.empty() && link.find(target) == std::wstring::npos)
{
continue;
}
if (SUCCEEDED(activate->ActivateObject(IID_PPV_ARGS(mediaSource))))
{
debugLog("activated source index=%u", index);
found = true;
break;
}
}
for (UINT32 index = 0; index < count; ++index)
{
if (rawDevices[index])
{
rawDevices[index]->Release();
}
}
CoTaskMemFree(rawDevices);
return found;
}
bool mediaTypeMatches(IMFMediaType* type, int width, int height, const GUID& subtype, double minFps)
{
GUID actualSubtype{};
UINT32 actualWidth = 0;
UINT32 actualHeight = 0;
UINT32 fpsNumerator = 0;
UINT32 fpsDenominator = 0;
if (FAILED(type->GetGUID(MF_MT_SUBTYPE, &actualSubtype)) ||
actualSubtype != subtype ||
FAILED(MFGetAttributeSize(type, MF_MT_FRAME_SIZE, &actualWidth, &actualHeight)) ||
static_cast<int>(actualWidth) != width ||
static_cast<int>(actualHeight) != height)
{
return false;
}
if (SUCCEEDED(MFGetAttributeRatio(type, MF_MT_FRAME_RATE, &fpsNumerator, &fpsDenominator)) &&
fpsDenominator != 0)
{
return static_cast<double>(fpsNumerator) / fpsDenominator >= minFps;
}
return minFps <= 0.0;
}
bool setInputType(IMFSourceReader* reader, int width, int height, const GUID& subtype, double minFps)
{
for (DWORD index = 0;; ++index)
{
ComPtr<IMFMediaType> type;
const HRESULT hr = reader->GetNativeMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, index, &type);
if (hr == MF_E_NO_MORE_TYPES)
{
debugLog("no input type matched %dx%d minFps=%.3f", width, height, minFps);
return false;
}
if (FAILED(hr))
{
debugLog("GetNativeMediaType failed index=%lu hr=0x%08lx", index, static_cast<unsigned long>(hr));
return false;
}
if (debugEnabled())
{
GUID actualSubtype{};
UINT32 actualWidth = 0;
UINT32 actualHeight = 0;
UINT32 fpsNumerator = 0;
UINT32 fpsDenominator = 0;
type->GetGUID(MF_MT_SUBTYPE, &actualSubtype);
MFGetAttributeSize(type.Get(), MF_MT_FRAME_SIZE, &actualWidth, &actualHeight);
MFGetAttributeRatio(type.Get(), MF_MT_FRAME_RATE, &fpsNumerator, &fpsDenominator);
debugLog(
"nativeType[%lu] %ux%u subtype=%08lx fps=%u/%u",
index,
actualWidth,
actualHeight,
static_cast<unsigned long>(actualSubtype.Data1),
fpsNumerator,
fpsDenominator);
}
if (mediaTypeMatches(type.Get(), width, height, subtype, minFps))
{
const HRESULT setHr = reader->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, type.Get());
debugLog("SetCurrentMediaType input index=%lu hr=0x%08lx", index, static_cast<unsigned long>(setHr));
return SUCCEEDED(setHr);
}
}
}
bool setOutputType(IMFSourceReader* reader, const GUID& subtype)
{
ComPtr<IMFMediaType> output;
if (FAILED(MFCreateMediaType(&output)))
{
return false;
}
output->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
output->SetGUID(MF_MT_SUBTYPE, subtype);
const HRESULT hr = reader->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, output.Get());
debugLog("SetCurrentMediaType output hr=0x%08lx", static_cast<unsigned long>(hr));
return SUCCEEDED(hr);
}
bool createSharedTexture(MimirMfGpuCapture& capture, const GUID& outputSubtype)
{
ComPtr<IMFMediaType> current;
if (FAILED(capture.reader->GetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, &current)))
{
debugLog("GetCurrentMediaType output failed");
return false;
}
UINT32 width = 0;
UINT32 height = 0;
GUID subtype{};
if (FAILED(MFGetAttributeSize(current.Get(), MF_MT_FRAME_SIZE, &width, &height)) ||
FAILED(current->GetGUID(MF_MT_SUBTYPE, &subtype)))
{
return false;
}
const auto format = dxgiFormatForSubtype(subtype);
if (format == DXGI_FORMAT_UNKNOWN || subtype != outputSubtype)
{
debugLog("output texture subtype/format unsupported subtype=%08lx", static_cast<unsigned long>(subtype.Data1));
return false;
}
D3D11_TEXTURE2D_DESC desc{};
desc.Width = width;
desc.Height = height;
desc.MipLevels = 1;
desc.ArraySize = 1;
desc.Format = format;
desc.SampleDesc.Count = 1;
desc.Usage = D3D11_USAGE_DEFAULT;
desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED_NTHANDLE | D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX;
const HRESULT textureHr = capture.device->CreateTexture2D(&desc, nullptr, &capture.sharedTexture);
if (FAILED(textureHr))
{
debugLog("CreateTexture2D shared NT failed hr=0x%08lx; retrying legacy shared", static_cast<unsigned long>(textureHr));
desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED;
const HRESULT legacyTextureHr = capture.device->CreateTexture2D(&desc, nullptr, &capture.sharedTexture);
if (FAILED(legacyTextureHr))
{
debugLog("CreateTexture2D legacy shared failed hr=0x%08lx", static_cast<unsigned long>(legacyTextureHr));
return false;
}
}
ComPtr<IDXGIResource1> dxgiResource;
const HRESULT asHr = capture.sharedTexture.As(&dxgiResource);
const HRESULT handleHr = SUCCEEDED(asHr)
? dxgiResource->CreateSharedHandle(nullptr, DXGI_SHARED_RESOURCE_READ | DXGI_SHARED_RESOURCE_WRITE, nullptr, &capture.sharedHandle)
: asHr;
if (FAILED(asHr) || FAILED(handleHr))
{
ComPtr<IDXGIResource> legacyResource;
const HRESULT legacyAsHr = capture.sharedTexture.As(&legacyResource);
const HRESULT legacyHandleHr = SUCCEEDED(legacyAsHr)
? legacyResource->GetSharedHandle(&capture.sharedHandle)
: legacyAsHr;
if (FAILED(legacyAsHr) || FAILED(legacyHandleHr))
{
debugLog(
"CreateSharedHandle failed as=0x%08lx handle=0x%08lx legacyAs=0x%08lx legacyHandle=0x%08lx",
static_cast<unsigned long>(asHr),
static_cast<unsigned long>(handleHr),
static_cast<unsigned long>(legacyAsHr),
static_cast<unsigned long>(legacyHandleHr));
return false;
}
}
capture.width = static_cast<int>(width);
capture.height = static_cast<int>(height);
capture.format = format;
return true;
}
bool createProducerFence(MimirMfGpuCapture& capture)
{
ComPtr<ID3D11Device5> device5;
if (FAILED(capture.device.As(&device5)) ||
FAILED(capture.context.As(&capture.fenceContext)))
{
debugLog("D3D11 fence interfaces unavailable; shared texture will be unsynchronized");
return true;
}
const HRESULT fenceHr = device5->CreateFence(
0,
D3D11_FENCE_FLAG_SHARED,
IID_PPV_ARGS(&capture.producerFence));
if (FAILED(fenceHr))
{
debugLog("CreateFence failed hr=0x%08lx; shared texture will be unsynchronized", static_cast<unsigned long>(fenceHr));
capture.fenceContext.Reset();
return true;
}
const HRESULT handleHr = capture.producerFence->CreateSharedHandle(nullptr, GENERIC_ALL, nullptr, &capture.producerFenceHandle);
if (FAILED(handleHr))
{
debugLog("CreateSharedHandle fence failed hr=0x%08lx; shared texture will be unsynchronized", static_cast<unsigned long>(handleHr));
capture.producerFence.Reset();
capture.fenceContext.Reset();
}
return true;
}
} // namespace
extern "C"
{
__declspec(dllexport) MimirMfGpuCapture* mimir_mf_gpu_create(
const char* pathNeedle,
int width,
int height,
const char* inputSubtypeText,
const char* outputSubtypeText,
double minFps)
{
CoInitializeEx(nullptr, COINIT_MULTITHREADED);
if (FAILED(MFStartup(MF_VERSION, MFSTARTUP_NOSOCKET)))
{
debugLog("MFStartup failed");
return nullptr;
}
auto capture = new MimirMfGpuCapture();
const UINT deviceFlags = D3D11_CREATE_DEVICE_VIDEO_SUPPORT | D3D11_CREATE_DEVICE_BGRA_SUPPORT;
D3D_FEATURE_LEVEL featureLevels[] = { D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0 };
D3D_FEATURE_LEVEL featureLevel{};
const HRESULT deviceHr = D3D11CreateDevice(
nullptr,
D3D_DRIVER_TYPE_HARDWARE,
nullptr,
deviceFlags,
featureLevels,
ARRAYSIZE(featureLevels),
D3D11_SDK_VERSION,
&capture->device,
&featureLevel,
&capture->context);
if (FAILED(deviceHr))
{
debugLog("D3D11CreateDevice failed hr=0x%08lx", static_cast<unsigned long>(deviceHr));
delete capture;
return nullptr;
}
ComPtr<ID3D10Multithread> multithread;
if (SUCCEEDED(capture->device.As(&multithread)))
{
multithread->SetMultithreadProtected(TRUE);
}
UINT resetToken = 0;
if (FAILED(MFCreateDXGIDeviceManager(&resetToken, &capture->deviceManager)) ||
FAILED(capture->deviceManager->ResetDevice(capture->device.Get(), resetToken)))
{
debugLog("DXGI device manager setup failed");
delete capture;
return nullptr;
}
const auto needle = widen(pathNeedle);
if (!findVideoSource(needle.c_str(), &capture->source))
{
debugLog("no source matched needle");
delete capture;
return nullptr;
}
ComPtr<IMFAttributes> readerAttributes;
if (FAILED(MFCreateAttributes(&readerAttributes, 4)))
{
debugLog("MFCreateAttributes reader failed");
delete capture;
return nullptr;
}
readerAttributes->SetUnknown(MF_SOURCE_READER_D3D_MANAGER, capture->deviceManager.Get());
readerAttributes->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, TRUE);
readerAttributes->SetUINT32(MF_LOW_LATENCY, TRUE);
readerAttributes->SetUINT32(MF_SOURCE_READER_ENABLE_ADVANCED_VIDEO_PROCESSING, TRUE);
const HRESULT readerHr = MFCreateSourceReaderFromMediaSource(capture->source.Get(), readerAttributes.Get(), &capture->reader);
if (FAILED(readerHr))
{
debugLog("MFCreateSourceReaderFromMediaSource failed hr=0x%08lx", static_cast<unsigned long>(readerHr));
delete capture;
return nullptr;
}
const auto inputSubtype = subtypeFromText(inputSubtypeText);
const auto outputSubtype = subtypeFromText(outputSubtypeText && outputSubtypeText[0] ? outputSubtypeText : "NV12");
if (inputSubtype == GUID_NULL ||
outputSubtype == GUID_NULL ||
!setInputType(capture->reader.Get(), width, height, inputSubtype, minFps) ||
!setOutputType(capture->reader.Get(), outputSubtype) ||
!createSharedTexture(*capture, outputSubtype) ||
!createProducerFence(*capture))
{
debugLog("create failed input=%s output=%s", inputSubtypeText ? inputSubtypeText : "", outputSubtypeText ? outputSubtypeText : "");
delete capture;
return nullptr;
}
return capture;
}
__declspec(dllexport) int mimir_mf_gpu_read(
MimirMfGpuCapture* capture,
HANDLE* sharedHandle,
int* width,
int* height,
char* format,
int formatCapacity,
std::int64_t* timestampNs,
std::uint64_t* sequence,
HANDLE* producerFenceHandle)
{
if (!capture || !capture->reader || !capture->sharedTexture || !sharedHandle)
{
return 0;
}
DWORD streamIndex = 0;
DWORD flags = 0;
LONGLONG timestamp = 0;
ComPtr<IMFSample> sample;
const HRESULT hr = capture->reader->ReadSample(
MF_SOURCE_READER_FIRST_VIDEO_STREAM,
0,
&streamIndex,
&flags,
&timestamp,
&sample);
if (FAILED(hr) || !sample || (flags & MF_SOURCE_READERF_STREAMTICK) != 0)
{
return 0;
}
ComPtr<IMFMediaBuffer> buffer;
ComPtr<IMFDXGIBuffer> dxgiBuffer;
ComPtr<ID3D11Texture2D> decodedTexture;
UINT subresource = 0;
if (FAILED(sample->GetBufferByIndex(0, &buffer)) ||
FAILED(buffer.As(&dxgiBuffer)) ||
FAILED(dxgiBuffer->GetResource(IID_PPV_ARGS(&decodedTexture))) ||
FAILED(dxgiBuffer->GetSubresourceIndex(&subresource)))
{
return 0;
}
const auto nextSequence = capture->sequence + 1;
capture->context->CopySubresourceRegion(capture->sharedTexture.Get(), 0, 0, 0, 0, decodedTexture.Get(), subresource, nullptr);
if (capture->producerFence && capture->fenceContext)
{
capture->fenceContext->Signal(capture->producerFence.Get(), nextSequence);
}
capture->context->Flush();
*sharedHandle = capture->sharedHandle;
if (width) *width = capture->width;
if (height) *height = capture->height;
if (format && formatCapacity > 0)
{
const char* text = capture->format == DXGI_FORMAT_NV12
? textForSubtype(MFVideoFormat_NV12)
: capture->format == DXGI_FORMAT_B8G8R8A8_UNORM
? textForSubtype(MFVideoFormat_RGB32)
: textForSubtype(MFVideoFormat_YUY2);
std::snprintf(format, static_cast<std::size_t>(formatCapacity), "%s", text);
}
if (timestampNs) *timestampNs = timestamp * 100;
capture->sequence = nextSequence;
if (sequence) *sequence = capture->sequence;
if (producerFenceHandle) *producerFenceHandle = capture->producerFenceHandle;
return 1;
}
__declspec(dllexport) void mimir_mf_gpu_destroy(MimirMfGpuCapture* capture)
{
if (!capture)
{
return;
}
if (capture->sharedHandle)
{
CloseHandle(capture->sharedHandle);
}
if (capture->producerFenceHandle)
{
CloseHandle(capture->producerFenceHandle);
}
if (capture->source)
{
capture->source->Shutdown();
}
delete capture;
MFShutdown();
CoUninitialize();
}
}

View file

@ -0,0 +1,215 @@
#include <algorithm>
#include <atomic>
#include <chrono>
#include <condition_variable>
#include <cstdint>
#include <cstring>
#include <deque>
#include <mutex>
#include <thread>
#include <vector>
#include "ps3eye_capi.h"
namespace
{
struct MimirPs3EyeFrame
{
std::vector<unsigned char> data;
std::int64_t timestampNs = 0;
std::uint64_t sequence = 0;
};
struct MimirPs3EyeCapture
{
ps3eye_t* eye = nullptr;
std::thread worker;
std::mutex mutex;
std::condition_variable condition;
std::deque<MimirPs3EyeFrame> queue;
std::atomic<bool> running = false;
int cameraIndex = 0;
int width = 0;
int height = 0;
int fps = 0;
std::uint64_t sequence = 0;
};
std::mutex g_ps3eye_mutex;
int g_ps3eye_users = 0;
long long nowNs()
{
return std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::steady_clock::now().time_since_epoch()).count();
}
void retainPs3Eye()
{
std::lock_guard<std::mutex> lock(g_ps3eye_mutex);
if (g_ps3eye_users++ == 0)
{
ps3eye_init();
}
}
void releasePs3Eye()
{
std::lock_guard<std::mutex> lock(g_ps3eye_mutex);
if (g_ps3eye_users > 0 && --g_ps3eye_users == 0)
{
ps3eye_uninit();
}
}
void pushFrame(MimirPs3EyeCapture& capture, const std::vector<unsigned char>& source)
{
MimirPs3EyeFrame frame;
frame.data = source;
frame.timestampNs = nowNs();
frame.sequence = capture.sequence++;
{
std::lock_guard<std::mutex> lock(capture.mutex);
capture.queue.push_back(std::move(frame));
while (capture.queue.size() > 32)
{
capture.queue.pop_front();
}
}
capture.condition.notify_one();
}
void captureThread(MimirPs3EyeCapture* capture)
{
std::vector<unsigned char> frame(static_cast<std::size_t>(capture->width) * static_cast<std::size_t>(capture->height));
while (capture->running.load(std::memory_order_acquire))
{
ps3eye_grab_frame(capture->eye, frame.data());
pushFrame(*capture, frame);
}
}
}
extern "C"
{
__declspec(dllexport) MimirPs3EyeCapture* mimir_ps3eye_create(int cameraIndex, int width, int height, int fps)
{
if (cameraIndex < 0 || width <= 0 || height <= 0 || fps <= 0)
{
return nullptr;
}
retainPs3Eye();
if (cameraIndex >= ps3eye_count_connected())
{
releasePs3Eye();
return nullptr;
}
auto* capture = new MimirPs3EyeCapture();
capture->cameraIndex = cameraIndex;
capture->width = width;
capture->height = height;
capture->fps = fps;
capture->eye = ps3eye_open(cameraIndex, width, height, fps, PS3EYE_FORMAT_BAYER);
if (!capture->eye)
{
delete capture;
releasePs3Eye();
return nullptr;
}
return capture;
}
__declspec(dllexport) int mimir_ps3eye_start(MimirPs3EyeCapture* capture)
{
if (!capture || capture->running.exchange(true))
{
return 0;
}
capture->worker = std::thread(captureThread, capture);
return 1;
}
__declspec(dllexport) int mimir_ps3eye_read(
MimirPs3EyeCapture* capture,
int* width,
int* height,
int* stride,
std::int64_t* timestampNs,
std::uint64_t* sequence,
unsigned char* destination,
int destinationBytes)
{
if (!capture || !destination || destinationBytes <= 0)
{
return 0;
}
MimirPs3EyeFrame frame;
{
std::lock_guard<std::mutex> lock(capture->mutex);
if (capture->queue.empty())
{
return 0;
}
frame = std::move(capture->queue.front());
capture->queue.pop_front();
}
if (frame.data.size() > static_cast<std::size_t>(destinationBytes))
{
return -static_cast<int>(frame.data.size());
}
std::memcpy(destination, frame.data.data(), frame.data.size());
if (width)
{
*width = capture->width;
}
if (height)
{
*height = capture->height;
}
if (stride)
{
*stride = capture->width;
}
if (timestampNs)
{
*timestampNs = frame.timestampNs;
}
if (sequence)
{
*sequence = frame.sequence;
}
return static_cast<int>(frame.data.size());
}
__declspec(dllexport) void mimir_ps3eye_destroy(MimirPs3EyeCapture* capture)
{
if (!capture)
{
return;
}
capture->running.store(false, std::memory_order_release);
if (capture->worker.joinable())
{
capture->worker.join();
}
if (capture->eye)
{
ps3eye_close(capture->eye);
}
delete capture;
releasePs3Eye();
}
}

View file

@ -0,0 +1,31 @@
cmake_minimum_required(VERSION 3.20)
project(mimir_obs_stem_source LANGUAGES CXX)
find_package(libobs REQUIRED CONFIG)
include(CTest)
add_library(mimir_obs_stem_source MODULE
mimir_obs_muninn_source.cpp
mimir_obs_program_texture_source.cpp
mimir_obs_stem_source.cpp)
target_compile_features(mimir_obs_stem_source PRIVATE cxx_std_20)
target_compile_definitions(mimir_obs_stem_source PRIVATE
WIN32_LEAN_AND_MEAN
NOMINMAX
_WIN32_WINNT=0x0A00)
target_link_libraries(mimir_obs_stem_source PRIVATE OBS::libobs d3d11 d3d12 dxgi ws2_32)
add_executable(muninn_media_decoder_probe
../../tools/muninn_media_decoder_probe.cpp)
target_compile_features(muninn_media_decoder_probe PRIVATE cxx_std_20)
target_include_directories(muninn_media_decoder_probe PRIVATE
${CMAKE_CURRENT_SOURCE_DIR})
add_executable(muninn_rudp_url_tests
tests/muninn_rudp_url_tests.cpp)
target_compile_features(muninn_rudp_url_tests PRIVATE cxx_std_20)
target_include_directories(muninn_rudp_url_tests PRIVATE
${CMAKE_CURRENT_SOURCE_DIR})
add_test(NAME muninn_rudp_url_tests COMMAND muninn_rudp_url_tests)

View file

@ -0,0 +1,14 @@
MimirAudioStem="Mimir Audio Stem"
MimirAudioStem.MapName="Shared memory map"
MimirAudioStem.StemId="Stem id"
MimirProgramTexture="Mimir Program Texture"
MuninnStream="Muninn Stream"
MuninnVideoSource="Muninn Video"
MuninnAudioSource="Muninn Audio"
MuninnStream.Stream="Muninn stream"
MuninnStream.Video="Video device"
MuninnStream.Audio="Audio device"
MuninnStream.VideoBitrateKbps="Video bitrate (kbps)"
MuninnStream.VideoLatencyMs="Video/transport latency (ms)"
MuninnStream.AudioReorderMs="Audio reorder latency (ms)"
MuninnStream.Refresh="Refresh Muninn streams"

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,628 @@
#include <obs-module.h>
#include <util/platform.h>
#include <chrono>
#include <cstdint>
#include <string>
#include <utility>
#include <vector>
#include <d3d11_1.h>
#include <d3d12.h>
#include <dxgi1_2.h>
#include <wrl/client.h>
#include <windows.h>
using Microsoft::WRL::ComPtr;
namespace {
struct MimirProgramTextureSource {
obs_source_t *source = nullptr;
std::string texture_name = "Global\\MimirFensalirProgramTexture";
std::string fence_name = "Global\\MimirFensalirProgramFence";
std::string consumer_fence_name = "Global\\MimirObsProgramConsumerFence";
uint32_t ring_count = 1;
uint32_t width = 0;
uint32_t height = 0;
uint64_t fence_value = 0;
uint64_t last_producer_fence_value = 0;
uint64_t next_retry_ns = 0;
uint64_t next_failure_log_ns = 0;
HANDLE bridge_shared_handle = nullptr;
HANDLE consumer_fence_shared_handle = nullptr;
HANDLE fence_event = nullptr;
gs_texture_t *obs_texture = nullptr;
ComPtr<ID3D12Device> d3d12_device;
ComPtr<ID3D12CommandQueue> command_queue;
ComPtr<ID3D12CommandAllocator> command_allocator;
ComPtr<ID3D12GraphicsCommandList> command_list;
ComPtr<ID3D12Fence> fence;
ComPtr<ID3D12Fence> producer_fence;
ComPtr<ID3D12Fence> consumer_fence;
std::vector<ComPtr<ID3D12Resource>> source_textures;
ComPtr<ID3D12Resource> bridge_texture_d3d12;
ComPtr<ID3D11Device> d3d11_device;
ComPtr<ID3D11DeviceContext> d3d11_context;
ComPtr<ID3D11Texture2D> bridge_texture_d3d11;
};
static void log_retry_failure(MimirProgramTextureSource *ctx, const char *message, HRESULT hr = S_OK)
{
const uint64_t now = os_gettime_ns();
if (now < ctx->next_failure_log_ns) {
return;
}
ctx->next_failure_log_ns = now + 2000000000ULL;
if (FAILED(hr)) {
blog(LOG_WARNING, "Mimir OBS program texture: %s: 0x%08lx", message, static_cast<unsigned long>(hr));
} else {
blog(LOG_WARNING, "Mimir OBS program texture: %s", message);
}
}
static std::wstring utf8_to_wide(const std::string &value)
{
if (value.empty()) {
return {};
}
const int count = MultiByteToWideChar(CP_UTF8, 0, value.c_str(), -1, nullptr, 0);
if (count <= 0) {
return {};
}
std::wstring wide(static_cast<size_t>(count), L'\0');
MultiByteToWideChar(CP_UTF8, 0, value.c_str(), -1, wide.data(), count);
if (!wide.empty() && wide.back() == L'\0') {
wide.pop_back();
}
return wide;
}
static std::string obs_string(obs_data_t *settings, const char *key, const char *fallback)
{
const char *value = obs_data_get_string(settings, key);
return value && *value ? std::string(value) : std::string(fallback);
}
static uint32_t clamp_ring_count(long long value)
{
if (value < 1) {
return 1;
}
if (value > 4) {
return 4;
}
return static_cast<uint32_t>(value);
}
static std::string texture_name_for_slot(const MimirProgramTextureSource *ctx, uint32_t slot)
{
if (ctx->ring_count <= 1) {
return ctx->texture_name;
}
return ctx->texture_name + "." + std::to_string(slot);
}
static void release_bridge(MimirProgramTextureSource *ctx)
{
if (ctx->obs_texture) {
gs_texture_destroy(ctx->obs_texture);
ctx->obs_texture = nullptr;
}
ctx->bridge_texture_d3d12.Reset();
ctx->bridge_texture_d3d11.Reset();
ctx->source_textures.clear();
ctx->fence.Reset();
ctx->producer_fence.Reset();
ctx->consumer_fence.Reset();
ctx->command_list.Reset();
ctx->command_allocator.Reset();
ctx->command_queue.Reset();
ctx->d3d12_device.Reset();
ctx->d3d11_context.Reset();
ctx->d3d11_device.Reset();
ctx->width = 0;
ctx->height = 0;
ctx->fence_value = 0;
ctx->last_producer_fence_value = 0;
if (ctx->bridge_shared_handle) {
CloseHandle(ctx->bridge_shared_handle);
ctx->bridge_shared_handle = nullptr;
}
if (ctx->consumer_fence_shared_handle) {
CloseHandle(ctx->consumer_fence_shared_handle);
ctx->consumer_fence_shared_handle = nullptr;
}
if (ctx->fence_event) {
CloseHandle(ctx->fence_event);
ctx->fence_event = nullptr;
}
}
static bool wait_for_copy(MimirProgramTextureSource *ctx, uint64_t consumed_producer_value)
{
if (ctx->consumer_fence && consumed_producer_value > 0) {
HRESULT consumer_hr = ctx->command_queue->Signal(ctx->consumer_fence.Get(), consumed_producer_value);
if (FAILED(consumer_hr)) {
blog(LOG_WARNING,
"Mimir OBS program texture: consumer fence Signal failed: 0x%08lx",
static_cast<unsigned long>(consumer_hr));
}
}
const uint64_t value = ++ctx->fence_value;
HRESULT hr = ctx->command_queue->Signal(ctx->fence.Get(), value);
if (FAILED(hr)) {
blog(LOG_WARNING, "Mimir OBS program texture: D3D12 Signal failed: 0x%08lx", static_cast<unsigned long>(hr));
return false;
}
if (ctx->fence->GetCompletedValue() >= value) {
return true;
}
hr = ctx->fence->SetEventOnCompletion(value, ctx->fence_event);
if (FAILED(hr)) {
blog(LOG_WARNING, "Mimir OBS program texture: SetEventOnCompletion failed: 0x%08lx", static_cast<unsigned long>(hr));
return false;
}
WaitForSingleObject(ctx->fence_event, 100);
return ctx->fence->GetCompletedValue() >= value;
}
static bool create_devices(MimirProgramTextureSource *ctx)
{
HRESULT hr = D3D12CreateDevice(nullptr, D3D_FEATURE_LEVEL_11_0, IID_PPV_ARGS(&ctx->d3d12_device));
if (FAILED(hr)) {
blog(LOG_WARNING, "Mimir OBS program texture: D3D12CreateDevice failed: 0x%08lx", static_cast<unsigned long>(hr));
return false;
}
D3D12_COMMAND_QUEUE_DESC queue_desc = {};
queue_desc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT;
hr = ctx->d3d12_device->CreateCommandQueue(&queue_desc, IID_PPV_ARGS(&ctx->command_queue));
if (FAILED(hr)) {
blog(LOG_WARNING, "Mimir OBS program texture: CreateCommandQueue failed: 0x%08lx", static_cast<unsigned long>(hr));
return false;
}
hr = ctx->d3d12_device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(&ctx->command_allocator));
if (FAILED(hr)) {
blog(LOG_WARNING, "Mimir OBS program texture: CreateCommandAllocator failed: 0x%08lx", static_cast<unsigned long>(hr));
return false;
}
hr = ctx->d3d12_device->CreateCommandList(
0,
D3D12_COMMAND_LIST_TYPE_DIRECT,
ctx->command_allocator.Get(),
nullptr,
IID_PPV_ARGS(&ctx->command_list));
if (FAILED(hr)) {
blog(LOG_WARNING, "Mimir OBS program texture: CreateCommandList failed: 0x%08lx", static_cast<unsigned long>(hr));
return false;
}
ctx->command_list->Close();
hr = ctx->d3d12_device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(&ctx->fence));
if (FAILED(hr)) {
blog(LOG_WARNING, "Mimir OBS program texture: CreateFence failed: 0x%08lx", static_cast<unsigned long>(hr));
return false;
}
ctx->fence_event = CreateEventW(nullptr, FALSE, FALSE, nullptr);
if (!ctx->fence_event) {
blog(LOG_WARNING, "Mimir OBS program texture: CreateEvent failed: %lu", GetLastError());
return false;
}
UINT flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT;
D3D_FEATURE_LEVEL feature_levels[] = {D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0};
D3D_FEATURE_LEVEL selected = D3D_FEATURE_LEVEL_11_0;
hr = D3D11CreateDevice(
nullptr,
D3D_DRIVER_TYPE_HARDWARE,
nullptr,
flags,
feature_levels,
2,
D3D11_SDK_VERSION,
&ctx->d3d11_device,
&selected,
&ctx->d3d11_context);
if (FAILED(hr)) {
blog(LOG_WARNING, "Mimir OBS program texture: D3D11CreateDevice failed: 0x%08lx", static_cast<unsigned long>(hr));
return false;
}
return true;
}
static void create_consumer_fence(MimirProgramTextureSource *ctx)
{
if (ctx->consumer_fence_name.empty()) {
return;
}
HRESULT hr = ctx->d3d12_device->CreateFence(0, D3D12_FENCE_FLAG_SHARED, IID_PPV_ARGS(&ctx->consumer_fence));
if (FAILED(hr)) {
log_retry_failure(ctx, "CreateFence consumer fence failed", hr);
return;
}
const auto wide_name = utf8_to_wide(ctx->consumer_fence_name);
if (wide_name.empty()) {
log_retry_failure(ctx, "consumer fence name is empty or invalid UTF-8");
ctx->consumer_fence.Reset();
return;
}
hr = ctx->d3d12_device->CreateSharedHandle(
ctx->consumer_fence.Get(),
nullptr,
GENERIC_ALL,
wide_name.c_str(),
&ctx->consumer_fence_shared_handle);
if (FAILED(hr)) {
log_retry_failure(ctx, "CreateSharedHandle consumer fence failed", hr);
ctx->consumer_fence.Reset();
return;
}
blog(LOG_INFO, "Mimir OBS program texture consumer fence published %s", ctx->consumer_fence_name.c_str());
}
static bool open_source_texture(MimirProgramTextureSource *ctx)
{
ctx->source_textures.clear();
ctx->source_textures.reserve(ctx->ring_count);
for (uint32_t slot = 0; slot < ctx->ring_count; ++slot) {
const auto slot_name = texture_name_for_slot(ctx, slot);
const auto wide_name = utf8_to_wide(slot_name);
if (wide_name.empty()) {
log_retry_failure(ctx, "shared texture name is empty or invalid UTF-8");
return false;
}
HANDLE source_handle = nullptr;
HRESULT hr = ctx->d3d12_device->OpenSharedHandleByName(wide_name.c_str(), GENERIC_ALL, &source_handle);
if (FAILED(hr)) {
log_retry_failure(ctx, "OpenSharedHandleByName failed", hr);
return false;
}
ComPtr<ID3D12Resource> source_texture;
hr = ctx->d3d12_device->OpenSharedHandle(source_handle, IID_PPV_ARGS(&source_texture));
CloseHandle(source_handle);
if (FAILED(hr)) {
log_retry_failure(ctx, "OpenSharedHandle failed", hr);
return false;
}
const D3D12_RESOURCE_DESC desc = source_texture->GetDesc();
if (desc.Dimension != D3D12_RESOURCE_DIMENSION_TEXTURE2D || desc.Format != DXGI_FORMAT_B8G8R8A8_UNORM) {
log_retry_failure(ctx, "expected BGRA8 D3D12 Texture2D");
return false;
}
const uint32_t slot_width = static_cast<uint32_t>(desc.Width);
const uint32_t slot_height = desc.Height;
if (slot_width == 0 || slot_height == 0) {
log_retry_failure(ctx, "shared texture dimensions are empty");
return false;
}
if (slot == 0) {
ctx->width = slot_width;
ctx->height = slot_height;
} else if (slot_width != ctx->width || slot_height != ctx->height) {
log_retry_failure(ctx, "program output ring texture dimensions do not match");
return false;
}
ctx->source_textures.push_back(std::move(source_texture));
}
return ctx->source_textures.size() == ctx->ring_count;
}
static bool open_producer_fence(MimirProgramTextureSource *ctx)
{
const auto wide_name = utf8_to_wide(ctx->fence_name);
if (wide_name.empty()) {
return true;
}
HANDLE fence_handle = nullptr;
HRESULT hr = ctx->d3d12_device->OpenSharedHandleByName(wide_name.c_str(), GENERIC_ALL, &fence_handle);
if (FAILED(hr)) {
log_retry_failure(ctx, "OpenSharedHandleByName producer fence failed", hr);
return true;
}
hr = ctx->d3d12_device->OpenSharedHandle(fence_handle, IID_PPV_ARGS(&ctx->producer_fence));
CloseHandle(fence_handle);
if (FAILED(hr)) {
log_retry_failure(ctx, "OpenSharedHandle producer fence failed", hr);
ctx->producer_fence.Reset();
return true;
}
ctx->last_producer_fence_value = ctx->producer_fence->GetCompletedValue();
if (ctx->last_producer_fence_value == UINT64_MAX) {
log_retry_failure(ctx, "producer fence opened in device-removed state");
ctx->producer_fence.Reset();
return false;
}
blog(LOG_INFO,
"Mimir OBS program texture producer fence opened %s value=%llu",
ctx->fence_name.c_str(),
static_cast<unsigned long long>(ctx->last_producer_fence_value));
return true;
}
static bool read_producer_fence(MimirProgramTextureSource *ctx, uint64_t *completed)
{
*completed = 0;
if (!ctx->producer_fence) {
return true;
}
*completed = ctx->producer_fence->GetCompletedValue();
if (*completed == UINT64_MAX) {
log_retry_failure(ctx, "producer fence reported device removal");
return false;
}
if (*completed > ctx->last_producer_fence_value) {
ctx->last_producer_fence_value = *completed;
}
return true;
}
static bool create_obs_bridge_texture(MimirProgramTextureSource *ctx)
{
D3D11_TEXTURE2D_DESC desc = {};
desc.Width = ctx->width;
desc.Height = ctx->height;
desc.MipLevels = 1;
desc.ArraySize = 1;
desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM;
desc.SampleDesc.Count = 1;
desc.Usage = D3D11_USAGE_DEFAULT;
desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED;
HRESULT hr = ctx->d3d11_device->CreateTexture2D(&desc, nullptr, &ctx->bridge_texture_d3d11);
if (FAILED(hr)) {
blog(LOG_WARNING, "Mimir OBS program texture: CreateTexture2D bridge failed: 0x%08lx", static_cast<unsigned long>(hr));
return false;
}
ComPtr<IDXGIResource> legacy_resource;
hr = ctx->bridge_texture_d3d11.As(&legacy_resource);
if (FAILED(hr)) {
blog(LOG_WARNING, "Mimir OBS program texture: bridge texture is not IDXGIResource: 0x%08lx", static_cast<unsigned long>(hr));
return false;
}
hr = legacy_resource->GetSharedHandle(&ctx->bridge_shared_handle);
if (FAILED(hr) || !ctx->bridge_shared_handle) {
blog(LOG_WARNING, "Mimir OBS program texture: GetSharedHandle bridge failed: 0x%08lx", static_cast<unsigned long>(hr));
return false;
}
hr = ctx->d3d12_device->OpenSharedHandle(ctx->bridge_shared_handle, IID_PPV_ARGS(&ctx->bridge_texture_d3d12));
if (FAILED(hr)) {
blog(LOG_WARNING, "Mimir OBS program texture: D3D12 open legacy bridge handle failed: 0x%08lx", static_cast<unsigned long>(hr));
return false;
}
ctx->obs_texture = gs_texture_open_shared(static_cast<uint32_t>(reinterpret_cast<uintptr_t>(ctx->bridge_shared_handle)));
if (!ctx->obs_texture) {
blog(LOG_WARNING, "Mimir OBS program texture: OBS failed to open legacy bridge shared texture");
return false;
}
return true;
}
static bool ensure_bridge(MimirProgramTextureSource *ctx)
{
if (ctx->obs_texture) {
return true;
}
const uint64_t now = os_gettime_ns();
if (now < ctx->next_retry_ns) {
return false;
}
ctx->next_retry_ns = now + 1000000000ULL;
release_bridge(ctx);
if (!create_devices(ctx)) {
release_bridge(ctx);
return false;
}
create_consumer_fence(ctx);
if (!open_source_texture(ctx) || !create_obs_bridge_texture(ctx)) {
release_bridge(ctx);
return false;
}
if (!open_producer_fence(ctx)) {
release_bridge(ctx);
return false;
}
blog(LOG_INFO, "Mimir OBS program texture source opened %s (%ux%u ring=%u)",
ctx->texture_name.c_str(),
ctx->width,
ctx->height,
ctx->ring_count);
return true;
}
static bool copy_source_to_bridge(MimirProgramTextureSource *ctx)
{
HRESULT hr = ctx->command_allocator->Reset();
if (FAILED(hr)) {
return false;
}
uint64_t completed_producer_value = 0;
if (!read_producer_fence(ctx, &completed_producer_value)) {
release_bridge(ctx);
return false;
}
const uint32_t source_slot = ctx->source_textures.empty()
? 0
: static_cast<uint32_t>(completed_producer_value % ctx->source_textures.size());
ID3D12Resource *source_texture = ctx->source_textures[source_slot].Get();
hr = ctx->command_list->Reset(ctx->command_allocator.Get(), nullptr);
if (FAILED(hr)) {
return false;
}
D3D12_RESOURCE_BARRIER barriers[2] = {};
barriers[0].Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION;
barriers[0].Transition.pResource = source_texture;
barriers[0].Transition.StateBefore = D3D12_RESOURCE_STATE_COMMON;
barriers[0].Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_SOURCE;
barriers[0].Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES;
barriers[1].Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION;
barriers[1].Transition.pResource = ctx->bridge_texture_d3d12.Get();
barriers[1].Transition.StateBefore = D3D12_RESOURCE_STATE_COMMON;
barriers[1].Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_DEST;
barriers[1].Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES;
ctx->command_list->ResourceBarrier(2, barriers);
ctx->command_list->CopyResource(ctx->bridge_texture_d3d12.Get(), source_texture);
std::swap(barriers[0].Transition.StateBefore, barriers[0].Transition.StateAfter);
std::swap(barriers[1].Transition.StateBefore, barriers[1].Transition.StateAfter);
ctx->command_list->ResourceBarrier(2, barriers);
hr = ctx->command_list->Close();
if (FAILED(hr)) {
return false;
}
ID3D12CommandList *lists[] = {ctx->command_list.Get()};
ctx->command_queue->ExecuteCommandLists(1, lists);
return wait_for_copy(ctx, completed_producer_value);
}
static const char *mimir_program_texture_get_name(void *)
{
return obs_module_text("MimirProgramTexture");
}
static void mimir_program_texture_update(void *data, obs_data_t *settings)
{
auto *ctx = static_cast<MimirProgramTextureSource *>(data);
ctx->texture_name = obs_string(settings, "texture_name", "Global\\MimirFensalirProgramTexture");
ctx->fence_name = obs_string(settings, "fence_name", "Global\\MimirFensalirProgramFence");
ctx->consumer_fence_name = obs_string(settings, "consumer_fence_name", "Global\\MimirObsProgramConsumerFence");
ctx->ring_count = clamp_ring_count(obs_data_get_int(settings, "ring_count"));
release_bridge(ctx);
}
static void *mimir_program_texture_create(obs_data_t *settings, obs_source_t *source)
{
auto *ctx = new MimirProgramTextureSource();
ctx->source = source;
mimir_program_texture_update(ctx, settings);
return ctx;
}
static void mimir_program_texture_destroy(void *data)
{
auto *ctx = static_cast<MimirProgramTextureSource *>(data);
release_bridge(ctx);
delete ctx;
}
static uint32_t mimir_program_texture_width(void *data)
{
auto *ctx = static_cast<MimirProgramTextureSource *>(data);
ensure_bridge(ctx);
return ctx->width;
}
static uint32_t mimir_program_texture_height(void *data)
{
auto *ctx = static_cast<MimirProgramTextureSource *>(data);
ensure_bridge(ctx);
return ctx->height;
}
static void mimir_program_texture_defaults(obs_data_t *settings)
{
obs_data_set_default_string(settings, "texture_name", "Global\\MimirFensalirProgramTexture");
obs_data_set_default_string(settings, "fence_name", "Global\\MimirFensalirProgramFence");
obs_data_set_default_string(settings, "consumer_fence_name", "Global\\MimirObsProgramConsumerFence");
obs_data_set_default_int(settings, "ring_count", 1);
}
static obs_properties_t *mimir_program_texture_properties(void *)
{
obs_properties_t *props = obs_properties_create();
obs_properties_add_text(props, "texture_name", "Shared D3D12 texture name", OBS_TEXT_DEFAULT);
obs_properties_add_text(props, "fence_name", "Shared D3D12 fence name", OBS_TEXT_DEFAULT);
obs_properties_add_text(props, "consumer_fence_name", "Consumer D3D12 fence name", OBS_TEXT_DEFAULT);
obs_properties_add_int(props, "ring_count", "Texture ring slots", 1, 4, 1);
return props;
}
static void mimir_program_texture_render(void *data, gs_effect_t *)
{
auto *ctx = static_cast<MimirProgramTextureSource *>(data);
if (!ensure_bridge(ctx)) {
return;
}
if (copy_source_to_bridge(ctx)) {
obs_source_draw(ctx->obs_texture, 0, 0, ctx->width, ctx->height, false);
}
}
static obs_source_info mimir_program_texture_source_info = {
.id = "mimir_program_texture_source",
.type = OBS_SOURCE_TYPE_INPUT,
.output_flags = OBS_SOURCE_VIDEO,
.get_name = mimir_program_texture_get_name,
.create = mimir_program_texture_create,
.destroy = mimir_program_texture_destroy,
.get_width = mimir_program_texture_width,
.get_height = mimir_program_texture_height,
.get_defaults = mimir_program_texture_defaults,
.get_properties = mimir_program_texture_properties,
.update = mimir_program_texture_update,
.video_render = mimir_program_texture_render,
};
} // namespace
void mimir_register_program_texture_source()
{
obs_register_source(&mimir_program_texture_source_info);
}

View file

@ -0,0 +1,43 @@
#pragma once
#include <cstdint>
constexpr uint64_t MIMIR_OBS_STEM_MAGIC = 0x4D545352494D494Dull;
constexpr int MIMIR_OBS_STEM_VERSION = 1;
constexpr int MIMIR_OBS_STEM_MAX_STEMS = 8;
constexpr int MIMIR_OBS_STEM_MAX_FRAMES = 4096;
constexpr int MIMIR_OBS_STEM_ID_BYTES = 64;
constexpr int MIMIR_OBS_STEM_DISPLAY_BYTES = 64;
constexpr int MIMIR_OBS_STEM_SOURCE_BYTES = 64;
constexpr int MIMIR_OBS_STEM_HEADER_BYTES = 64;
constexpr int MIMIR_OBS_STEM_RECORD_BYTES = 224;
constexpr int MIMIR_OBS_STEM_MAP_BYTES =
MIMIR_OBS_STEM_HEADER_BYTES +
(MIMIR_OBS_STEM_MAX_STEMS * MIMIR_OBS_STEM_RECORD_BYTES) +
(MIMIR_OBS_STEM_MAX_STEMS * MIMIR_OBS_STEM_MAX_FRAMES * static_cast<int>(sizeof(float)));
struct MimirObsStemHeader {
uint64_t magic;
int32_t version;
uint32_t generation;
int32_t stem_count;
int32_t max_stems;
int32_t max_frames;
int32_t header_bytes;
int32_t record_bytes;
int32_t map_bytes;
};
struct MimirObsStemRecord {
int32_t channel_index;
int32_t configured;
int32_t sample_rate;
int32_t frame_count;
int32_t sample_offset;
int32_t reserved0;
int32_t reserved1;
int32_t reserved2;
char stem_id[MIMIR_OBS_STEM_ID_BYTES];
char display_name[MIMIR_OBS_STEM_DISPLAY_BYTES];
char source_id[MIMIR_OBS_STEM_SOURCE_BYTES];
};

View file

@ -0,0 +1,219 @@
#include <obs-module.h>
#include <util/platform.h>
#include <atomic>
#include <chrono>
#include <cstring>
#include <string>
#include <thread>
#include <vector>
#include <windows.h>
#include "mimir_obs_stem_ipc.h"
OBS_DECLARE_MODULE()
OBS_MODULE_USE_DEFAULT_LOCALE("mimir-obs-stem-source", "en-US")
void mimir_register_program_texture_source();
void mimir_register_muninn_source();
struct MimirStemSource {
obs_source_t *source = nullptr;
std::string map_name = "Local\\MimirObsStemBus";
std::string stem_id = "aligned_source_0";
HANDLE mapping = nullptr;
const uint8_t *view = nullptr;
std::thread worker;
std::atomic_bool running = false;
uint32_t last_generation = 0;
};
static std::string obs_string(obs_data_t *settings, const char *key, const char *fallback)
{
const char *value = obs_data_get_string(settings, key);
return value && *value ? std::string(value) : std::string(fallback);
}
static void close_mapping(MimirStemSource *ctx)
{
if (ctx->view) {
UnmapViewOfFile(ctx->view);
ctx->view = nullptr;
}
if (ctx->mapping) {
CloseHandle(ctx->mapping);
ctx->mapping = nullptr;
}
}
static bool open_mapping(MimirStemSource *ctx)
{
if (ctx->mapping) {
return true;
}
ctx->mapping = OpenFileMappingA(FILE_MAP_READ, FALSE, ctx->map_name.c_str());
if (!ctx->mapping) {
return false;
}
ctx->view = static_cast<const uint8_t *>(MapViewOfFile(ctx->mapping, FILE_MAP_READ, 0, 0, MIMIR_OBS_STEM_MAP_BYTES));
if (!ctx->view) {
close_mapping(ctx);
return false;
}
return true;
}
static const MimirObsStemRecord *find_record(const uint8_t *view, const std::string &stem_id, int stem_count)
{
const auto count = std::min(stem_count, MIMIR_OBS_STEM_MAX_STEMS);
for (int index = 0; index < count; ++index) {
const auto *record = reinterpret_cast<const MimirObsStemRecord *>(view + MIMIR_OBS_STEM_HEADER_BYTES + (index * MIMIR_OBS_STEM_RECORD_BYTES));
if (record->frame_count <= 0 || record->sample_rate <= 0 || record->sample_offset <= 0) {
continue;
}
if (stem_id == record->stem_id) {
return record;
}
}
return nullptr;
}
static void publish_if_available(MimirStemSource *ctx)
{
if (!open_mapping(ctx)) {
return;
}
const auto *header = reinterpret_cast<const MimirObsStemHeader *>(ctx->view);
const auto generation_a = header->generation;
if ((generation_a & 1u) != 0 || generation_a == ctx->last_generation) {
return;
}
if (header->magic != MIMIR_OBS_STEM_MAGIC ||
header->version != MIMIR_OBS_STEM_VERSION ||
header->map_bytes != MIMIR_OBS_STEM_MAP_BYTES) {
return;
}
const auto *record = find_record(ctx->view, ctx->stem_id, header->stem_count);
if (!record) {
return;
}
const auto frame_count = std::min(record->frame_count, MIMIR_OBS_STEM_MAX_FRAMES);
const auto sample_rate = record->sample_rate;
const auto byte_count = static_cast<size_t>(frame_count) * sizeof(float);
if (record->sample_offset < 0 || record->sample_offset + static_cast<int>(byte_count) > MIMIR_OBS_STEM_MAP_BYTES) {
return;
}
std::vector<float> samples(frame_count);
std::memcpy(samples.data(), ctx->view + record->sample_offset, byte_count);
const auto generation_b = header->generation;
if (generation_a != generation_b || (generation_b & 1u) != 0) {
return;
}
ctx->last_generation = generation_b;
uint8_t *planes[MAX_AV_PLANES] = {};
planes[0] = reinterpret_cast<uint8_t *>(samples.data());
obs_source_audio audio = {};
audio.data[0] = planes[0];
audio.frames = static_cast<uint32_t>(frame_count);
audio.speakers = SPEAKERS_MONO;
audio.samples_per_sec = static_cast<uint32_t>(sample_rate);
audio.format = AUDIO_FORMAT_FLOAT_PLANAR;
audio.timestamp = os_gettime_ns();
obs_source_output_audio(ctx->source, &audio);
}
static void worker_loop(MimirStemSource *ctx)
{
while (ctx->running.load()) {
publish_if_available(ctx);
std::this_thread::sleep_for(std::chrono::milliseconds(5));
}
}
static const char *mimir_stem_get_name(void *)
{
return obs_module_text("MimirAudioStem");
}
static void mimir_stem_update(void *data, obs_data_t *settings)
{
auto *ctx = static_cast<MimirStemSource *>(data);
ctx->map_name = obs_string(settings, "map_name", "Local\\MimirObsStemBus");
ctx->stem_id = obs_string(settings, "stem_id", "aligned_source_0");
ctx->last_generation = 0;
close_mapping(ctx);
}
static void *mimir_stem_create(obs_data_t *settings, obs_source_t *source)
{
auto *ctx = new MimirStemSource();
ctx->source = source;
mimir_stem_update(ctx, settings);
ctx->running = true;
ctx->worker = std::thread(worker_loop, ctx);
return ctx;
}
static void mimir_stem_destroy(void *data)
{
auto *ctx = static_cast<MimirStemSource *>(data);
ctx->running = false;
if (ctx->worker.joinable()) {
ctx->worker.join();
}
close_mapping(ctx);
delete ctx;
}
static void mimir_stem_defaults(obs_data_t *settings)
{
obs_data_set_default_string(settings, "map_name", "Local\\MimirObsStemBus");
obs_data_set_default_string(settings, "stem_id", "aligned_source_0");
}
static obs_properties_t *mimir_stem_properties(void *)
{
obs_properties_t *props = obs_properties_create();
obs_properties_add_text(props, "map_name", obs_module_text("MimirAudioStem.MapName"), OBS_TEXT_DEFAULT);
obs_properties_add_text(props, "stem_id", obs_module_text("MimirAudioStem.StemId"), OBS_TEXT_DEFAULT);
return props;
}
static obs_source_info mimir_stem_source_info = {
.id = "mimir_audio_stem_source",
.type = OBS_SOURCE_TYPE_INPUT,
.output_flags = OBS_SOURCE_AUDIO,
.get_name = mimir_stem_get_name,
.create = mimir_stem_create,
.destroy = mimir_stem_destroy,
.get_defaults = mimir_stem_defaults,
.get_properties = mimir_stem_properties,
.update = mimir_stem_update,
};
bool obs_module_load(void)
{
obs_register_source(&mimir_stem_source_info);
mimir_register_program_texture_source();
mimir_register_muninn_source();
blog(
LOG_INFO,
"Mimir OBS sources loaded: mimir_audio_stem_source, mimir_program_texture_source, "
"muninn_stream_source, muninn_video_source, muninn_audio_source");
return true;
}

View file

@ -0,0 +1,267 @@
#pragma once
#include <chrono>
#include <climits>
#include <cstdint>
#include <string>
#include <vector>
namespace muninn_media_wire {
inline constexpr const char *ReceiverFeedbackSchema = "muninn.media_receiver_feedback.v1";
class MsgpackWriter {
public:
const std::vector<uint8_t> &bytes() const { return bytes_; }
void write_raw(const std::vector<uint8_t> &value)
{
bytes_.insert(bytes_.end(), value.begin(), value.end());
}
void write_nil() { bytes_.push_back(0xc0); }
void write_bool(bool value) { bytes_.push_back(value ? 0xc3 : 0xc2); }
void write_u64(uint64_t value)
{
if (value <= 0x7f) {
bytes_.push_back(static_cast<uint8_t>(value));
} else if (value <= 0xff) {
bytes_.push_back(0xcc);
bytes_.push_back(static_cast<uint8_t>(value));
} else if (value <= 0xffff) {
bytes_.push_back(0xcd);
write_be16_value(static_cast<uint16_t>(value));
} else if (value <= 0xffffffffull) {
bytes_.push_back(0xce);
write_be32_value(static_cast<uint32_t>(value));
} else {
bytes_.push_back(0xcf);
write_be64_value(value);
}
}
void write_i64(int64_t value)
{
if (value >= 0) {
write_u64(static_cast<uint64_t>(value));
return;
}
if (value >= -32) {
bytes_.push_back(static_cast<uint8_t>(0xe0 | (value + 32)));
} else if (value >= INT8_MIN) {
bytes_.push_back(0xd0);
bytes_.push_back(static_cast<uint8_t>(value));
} else if (value >= INT16_MIN) {
bytes_.push_back(0xd1);
write_be16_value(static_cast<uint16_t>(value));
} else if (value >= INT32_MIN) {
bytes_.push_back(0xd2);
write_be32_value(static_cast<uint32_t>(value));
} else {
bytes_.push_back(0xd3);
write_be64_value(static_cast<uint64_t>(value));
}
}
void write_string(const std::string &value)
{
const auto length = value.size();
if (length <= 31) {
bytes_.push_back(static_cast<uint8_t>(0xa0 | length));
} else if (length <= 0xff) {
bytes_.push_back(0xd9);
bytes_.push_back(static_cast<uint8_t>(length));
} else if (length <= 0xffff) {
bytes_.push_back(0xda);
write_be16_value(static_cast<uint16_t>(length));
} else {
bytes_.push_back(0xdb);
write_be32_value(static_cast<uint32_t>(length));
}
bytes_.insert(bytes_.end(), value.begin(), value.end());
}
void write_binary(const std::vector<uint8_t> &value)
{
const auto length = value.size();
if (length <= 0xff) {
bytes_.push_back(0xc4);
bytes_.push_back(static_cast<uint8_t>(length));
} else if (length <= 0xffff) {
bytes_.push_back(0xc5);
write_be16_value(static_cast<uint16_t>(length));
} else {
bytes_.push_back(0xc6);
write_be32_value(static_cast<uint32_t>(length));
}
bytes_.insert(bytes_.end(), value.begin(), value.end());
}
void write_array_len(uint32_t length)
{
if (length <= 15) {
bytes_.push_back(static_cast<uint8_t>(0x90 | length));
} else if (length <= 0xffff) {
bytes_.push_back(0xdc);
write_be16_value(static_cast<uint16_t>(length));
} else {
bytes_.push_back(0xdd);
write_be32_value(length);
}
}
void write_map_len(uint32_t length)
{
if (length <= 15) {
bytes_.push_back(static_cast<uint8_t>(0x80 | length));
} else if (length <= 0xffff) {
bytes_.push_back(0xde);
write_be16_value(static_cast<uint16_t>(length));
} else {
bytes_.push_back(0xdf);
write_be32_value(length);
}
}
private:
std::vector<uint8_t> bytes_;
void write_be16_value(uint16_t value)
{
bytes_.push_back(static_cast<uint8_t>((value >> 8) & 0xff));
bytes_.push_back(static_cast<uint8_t>(value & 0xff));
}
void write_be32_value(uint32_t value)
{
bytes_.push_back(static_cast<uint8_t>((value >> 24) & 0xff));
bytes_.push_back(static_cast<uint8_t>((value >> 16) & 0xff));
bytes_.push_back(static_cast<uint8_t>((value >> 8) & 0xff));
bytes_.push_back(static_cast<uint8_t>(value & 0xff));
}
void write_be64_value(uint64_t value)
{
write_be32_value(static_cast<uint32_t>((value >> 32) & 0xffffffffull));
write_be32_value(static_cast<uint32_t>(value & 0xffffffffull));
}
};
inline std::string feedback_observed_at()
{
const auto now = std::chrono::system_clock::now().time_since_epoch();
const auto millis = std::chrono::duration_cast<std::chrono::milliseconds>(now).count();
return "unix-ms:" + std::to_string(millis);
}
inline void write_string_array(MsgpackWriter &writer, const std::vector<std::string> &values)
{
writer.write_array_len(static_cast<uint32_t>(values.size()));
for (const auto &value : values) {
writer.write_string(value);
}
}
inline void write_u64_array(MsgpackWriter &writer, const std::vector<uint64_t> &values)
{
writer.write_array_len(static_cast<uint32_t>(values.size()));
for (const auto value : values) {
writer.write_u64(value);
}
}
inline std::vector<uint8_t> encode_receiver_feedback_payload(
const std::string &stream_id,
const std::string &session_id,
uint64_t frame_id,
const std::vector<std::string> &missing_chunk_keys,
bool requested_keyframe,
const std::string &observed_at)
{
MsgpackWriter record;
record.write_array_len(11);
record.write_string(stream_id);
record.write_string(session_id);
record.write_string("starfire.obs");
if (frame_id > 0) {
record.write_u64(frame_id - 1);
} else {
record.write_nil();
}
write_u64_array(record, {});
write_u64_array(record, {frame_id});
record.write_bool(requested_keyframe);
record.write_i64(0);
record.write_i64(0);
record.write_string(observed_at);
write_string_array(record, missing_chunk_keys);
const std::string record_key = stream_id + ":" + session_id + ":feedback:starfire.obs";
const std::string message_id =
"muninn-media:" + std::string(ReceiverFeedbackSchema) + ":" + record_key;
MsgpackWriter document;
document.write_map_len(9);
document.write_string("schemaId");
document.write_string(ReceiverFeedbackSchema);
document.write_string("recordKey");
document.write_string(record_key);
document.write_string("storedAt");
document.write_string(observed_at);
document.write_string("payloadEncoding");
document.write_string("messagepack");
document.write_string("payload");
document.write_binary(record.bytes());
document.write_string("sourceRuntimeId");
document.write_string("starfire");
document.write_string("sourceAgentId");
document.write_nil();
document.write_string("sourceRole");
document.write_string("mimir.obs");
document.write_string("tags");
write_string_array(document, {"muninn.media"});
MsgpackWriter wire;
wire.write_map_len(3);
wire.write_string("schemaVersion");
wire.write_string("cultnet.document_put_raw.v0");
wire.write_string("messageId");
wire.write_string(message_id);
wire.write_string("document");
wire.write_raw(document.bytes());
return wire.bytes();
}
inline std::vector<uint8_t> encode_receiver_feedback_payload(
const std::string &stream_id,
const std::string &session_id,
uint64_t frame_id,
const std::vector<std::string> &missing_chunk_keys,
bool requested_keyframe)
{
return encode_receiver_feedback_payload(
stream_id,
session_id,
frame_id,
missing_chunk_keys,
requested_keyframe,
feedback_observed_at());
}
inline std::vector<uint8_t> encode_receiver_feedback_payload(
const std::string &stream_id,
const std::string &session_id,
uint64_t frame_id,
const std::vector<std::string> &missing_chunk_keys)
{
return encode_receiver_feedback_payload(
stream_id,
session_id,
frame_id,
missing_chunk_keys,
true);
}
} // namespace muninn_media_wire

View file

@ -0,0 +1,54 @@
#pragma once
#include <cstdint>
#include <optional>
#include <set>
#include <utility>
namespace muninn_rudp_ack {
class AckTracker {
public:
void reset()
{
received_sequences_.clear();
highest_received_sequence_.reset();
}
void remember(uint32_t sequence)
{
received_sequences_.insert(sequence);
if (!highest_received_sequence_ || sequence > *highest_received_sequence_) {
highest_received_sequence_ = sequence;
}
if (!highest_received_sequence_) {
return;
}
const uint32_t highest = *highest_received_sequence_;
for (auto iterator = received_sequences_.begin(); iterator != received_sequences_.end();) {
if (*iterator + 32 < highest) {
iterator = received_sequences_.erase(iterator);
} else {
++iterator;
}
}
}
std::pair<uint32_t, uint32_t> state() const
{
const uint32_t ack = highest_received_sequence_.value_or(0);
uint32_t ack_mask = 0;
for (uint32_t bit = 0; bit < 32; ++bit) {
if (ack > bit && received_sequences_.find(ack - bit - 1) != received_sequences_.end()) {
ack_mask |= 1u << bit;
}
}
return {ack, ack_mask};
}
private:
std::set<uint32_t> received_sequences_;
std::optional<uint32_t> highest_received_sequence_;
};
} // namespace muninn_rudp_ack

View file

@ -0,0 +1,127 @@
#pragma once
#include <algorithm>
#include <chrono>
#include <cstdint>
#include <map>
#include <optional>
#include <string>
#include <utility>
#include <vector>
namespace muninn_rudp_fragments {
constexpr uint32_t CompletedFragmentIdQuarantineMs = 2000;
struct FragmentResult {
uint32_t first_sequence = 0;
std::vector<uint8_t> payload;
};
class FragmentAssembler {
public:
void reset()
{
assemblies_.clear();
recently_closed_.clear();
expired_count_ = 0;
}
std::optional<FragmentResult> insert(const std::string &channel_id,
uint16_t fragment_id,
uint16_t fragment_index,
uint16_t fragment_count,
uint32_t sequence,
const uint8_t *payload,
uint32_t payload_len,
std::chrono::steady_clock::time_point now)
{
if (fragment_id == 0 || fragment_index >= fragment_count) {
return std::nullopt;
}
const std::string key = channel_id + ":" + std::to_string(fragment_id);
expire_recently_closed(now);
const auto existing = assemblies_.find(key);
if (existing == assemblies_.end() && recently_closed_.find(key) != recently_closed_.end()) {
return std::nullopt;
}
auto &assembly = assemblies_[key];
if (assembly.payloads.empty()) {
assembly.channel_id = channel_id;
assembly.fragment_count = fragment_count;
assembly.started_at = now;
}
if (assembly.channel_id != channel_id || assembly.fragment_count != fragment_count) {
assemblies_.erase(key);
return std::nullopt;
}
assembly.payloads.emplace(fragment_index, std::vector<uint8_t>(payload, payload + payload_len));
assembly.sequences.emplace(fragment_index, sequence);
if (assembly.payloads.size() < assembly.fragment_count) {
return std::nullopt;
}
FragmentResult result;
result.first_sequence = UINT32_MAX;
for (uint16_t index = 0; index < assembly.fragment_count; ++index) {
const auto payload_found = assembly.payloads.find(index);
const auto sequence_found = assembly.sequences.find(index);
if (payload_found == assembly.payloads.end() || sequence_found == assembly.sequences.end()) {
return std::nullopt;
}
result.payload.insert(result.payload.end(), payload_found->second.begin(), payload_found->second.end());
result.first_sequence = std::min(result.first_sequence, sequence_found->second);
}
assemblies_.erase(key);
recently_closed_[key] = now;
return result;
}
uint64_t expire(std::chrono::steady_clock::time_point now, uint32_t expire_after_ms)
{
expire_recently_closed(now);
uint64_t expired = 0;
for (auto iterator = assemblies_.begin(); iterator != assemblies_.end();) {
if (now - iterator->second.started_at < std::chrono::milliseconds(expire_after_ms)) {
++iterator;
continue;
}
recently_closed_[iterator->first] = now;
iterator = assemblies_.erase(iterator);
++expired;
}
expired_count_ += expired;
return expired;
}
uint64_t expired_count() const { return expired_count_; }
private:
void expire_recently_closed(std::chrono::steady_clock::time_point now)
{
for (auto iterator = recently_closed_.begin(); iterator != recently_closed_.end();) {
if (now - iterator->second < std::chrono::milliseconds(CompletedFragmentIdQuarantineMs)) {
++iterator;
continue;
}
iterator = recently_closed_.erase(iterator);
}
}
struct Assembly {
std::string channel_id;
uint16_t fragment_count = 0;
std::map<uint16_t, std::vector<uint8_t>> payloads;
std::map<uint16_t, uint32_t> sequences;
std::chrono::steady_clock::time_point started_at;
};
std::map<std::string, Assembly> assemblies_;
std::map<std::string, std::chrono::steady_clock::time_point> recently_closed_;
uint64_t expired_count_ = 0;
};
} // namespace muninn_rudp_fragments

View file

@ -0,0 +1,123 @@
#pragma once
#include <algorithm>
#include <cstdint>
#include <optional>
#include <sstream>
#include <string>
namespace muninn_rudp_url {
inline constexpr uint32_t DefaultMediaRudpConnectionId = 0x6d750001;
inline constexpr uint32_t DefaultAudioRudpConnectionId = 0x6d750004;
inline constexpr uint32_t DefaultReliableExpireAfterMs = 75;
inline constexpr uint32_t DefaultVideoAssemblyDeadlineMs = 75;
inline constexpr uint32_t DefaultGapWaitMs = 8;
inline constexpr uint32_t DefaultAudioReorderMs = 2000;
inline constexpr uint32_t DefaultSenderResendDelayMs = 5;
inline constexpr uint32_t MaxLatencyBudgetMs = 2000;
struct RudpUrlParts {
uint16_t port = 0;
uint16_t video_local_port = 0;
uint16_t audio_local_port = 0;
uint32_t connection_id = DefaultMediaRudpConnectionId;
uint32_t audio_connection_id = DefaultAudioRudpConnectionId;
uint32_t sender_resend_delay_ms = DefaultSenderResendDelayMs;
uint32_t reliable_expire_after_ms = DefaultReliableExpireAfterMs;
uint32_t video_assembly_deadline_ms = DefaultVideoAssemblyDeadlineMs;
uint32_t gap_wait_ms = DefaultGapWaitMs;
uint32_t audio_reorder_ms = DefaultAudioReorderMs;
};
inline bool starts_with(const std::string &value, const std::string &prefix)
{
return value.rfind(prefix, 0) == 0;
}
inline std::optional<uint32_t> parse_u32_hex_or_decimal(const std::string &value)
{
try {
size_t consumed = 0;
const int base = starts_with(value, "0x") || starts_with(value, "0X") ? 16 : 10;
const auto parsed = std::stoul(value, &consumed, base);
if (consumed == value.size()) {
return static_cast<uint32_t>(parsed);
}
} catch (...) {
}
return std::nullopt;
}
inline std::optional<RudpUrlParts> parse(const std::string &url)
{
if (!starts_with(url, "rudp://")) {
return std::nullopt;
}
const auto authority_start = std::string("rudp://").size();
const auto path_start = url.find('/', authority_start);
const auto query_start = url.find('?', authority_start);
const auto authority_end = std::min(path_start == std::string::npos ? url.size() : path_start,
query_start == std::string::npos ? url.size() : query_start);
const auto authority = url.substr(authority_start, authority_end - authority_start);
const auto colon = authority.rfind(':');
if (colon == std::string::npos || colon + 1 >= authority.size()) {
return std::nullopt;
}
RudpUrlParts parts;
parts.port = static_cast<uint16_t>(std::stoul(authority.substr(colon + 1)));
parts.video_local_port = static_cast<uint16_t>(parts.port + 10000);
parts.audio_local_port = static_cast<uint16_t>(parts.port + 10001);
if (query_start != std::string::npos) {
std::stringstream query(url.substr(query_start + 1));
std::string item;
while (std::getline(query, item, '&')) {
const auto equals = item.find('=');
if (equals == std::string::npos) {
continue;
}
const auto key = item.substr(0, equals);
const auto value = item.substr(equals + 1);
if (key == "connection") {
if (const auto parsed = parse_u32_hex_or_decimal(value)) {
parts.connection_id = *parsed;
}
} else if (key == "audio_connection" || key == "audio_connection_id") {
if (const auto parsed = parse_u32_hex_or_decimal(value)) {
parts.audio_connection_id = *parsed;
}
} else if (key == "local_port" || key == "video_local_port") {
parts.video_local_port = static_cast<uint16_t>(std::stoul(value));
} else if (key == "audio_local_port") {
parts.audio_local_port = static_cast<uint16_t>(std::stoul(value));
} else if (key == "reliable_expire_after_ms") {
if (const auto parsed = parse_u32_hex_or_decimal(value)) {
parts.reliable_expire_after_ms = std::clamp(*parsed, 1u, MaxLatencyBudgetMs);
}
} else if (key == "sender_resend_delay_ms") {
if (const auto parsed = parse_u32_hex_or_decimal(value)) {
parts.sender_resend_delay_ms = std::clamp(*parsed, 1u, 1000u);
}
} else if (key == "assembly_deadline_ms") {
if (const auto parsed = parse_u32_hex_or_decimal(value)) {
parts.video_assembly_deadline_ms = std::clamp(*parsed, 1u, MaxLatencyBudgetMs);
}
} else if (key == "gap_wait_ms") {
if (const auto parsed = parse_u32_hex_or_decimal(value)) {
parts.gap_wait_ms = std::clamp(*parsed, 0u, 1000u);
}
} else if (key == "audio_reorder_ms") {
if (const auto parsed = parse_u32_hex_or_decimal(value)) {
parts.audio_reorder_ms = std::clamp(*parsed, 0u, MaxLatencyBudgetMs);
}
}
}
}
return parts;
}
} // namespace muninn_rudp_url

View file

@ -0,0 +1,58 @@
#include "muninn_rudp_url.h"
#include <cstdlib>
#include <iostream>
#include <string>
namespace {
void require(bool condition, const std::string &message)
{
if (!condition) {
std::cerr << message << '\n';
std::exit(1);
}
}
} // namespace
int main()
{
const auto configured = muninn_rudp_url::parse(
"rudp://192.168.1.66:5204/muninn.raven.av.rudp?channel=media&format=muninn-typed-media&connection=0x6d750001&audio_connection=0x6d750004&sender_resend_delay_ms=5&reliable_expire_after_ms=1200&assembly_deadline_ms=1200&gap_wait_ms=16&audio_reorder_ms=700");
require(configured.has_value(), "configured RUDP URL did not parse");
require(configured->port == 5204, "configured RUDP URL port was not parsed");
require(configured->video_local_port == 15204, "configured RUDP video local port was not derived");
require(configured->audio_local_port == 15205, "configured RUDP audio local port was not derived");
require(configured->connection_id == 0x6d750001, "configured RUDP connection id was not parsed");
require(configured->audio_connection_id == 0x6d750004, "configured RUDP audio connection id was not parsed");
require(configured->reliable_expire_after_ms == 1200, "configured reliable expiry did not survive parse");
require(configured->video_assembly_deadline_ms == 1200, "configured assembly deadline did not survive parse");
require(configured->gap_wait_ms == 16, "configured gap wait did not survive parse");
require(configured->audio_reorder_ms == 700, "configured audio reorder wait did not survive parse");
const auto clamped = muninn_rudp_url::parse(
"rudp://192.168.1.66:5204/muninn.raven.av.rudp?reliable_expire_after_ms=9000&assembly_deadline_ms=9000");
require(clamped.has_value(), "oversized latency RUDP URL did not parse");
require(clamped->reliable_expire_after_ms == muninn_rudp_url::MaxLatencyBudgetMs,
"oversized reliable expiry did not clamp to the max latency budget");
require(clamped->video_assembly_deadline_ms == muninn_rudp_url::MaxLatencyBudgetMs,
"oversized assembly deadline did not clamp to the max latency budget");
const auto fallback = muninn_rudp_url::parse("rudp://192.168.1.66:5204/muninn.raven.av.rudp");
require(fallback.has_value(), "minimal RUDP URL did not parse");
require(fallback->connection_id == muninn_rudp_url::DefaultMediaRudpConnectionId,
"minimal RUDP URL did not keep the default connection id");
require(fallback->audio_connection_id == muninn_rudp_url::DefaultAudioRudpConnectionId,
"minimal RUDP URL did not keep the default audio connection id");
require(fallback->reliable_expire_after_ms == muninn_rudp_url::DefaultReliableExpireAfterMs,
"minimal RUDP URL did not keep the default reliable expiry");
require(fallback->video_assembly_deadline_ms == muninn_rudp_url::DefaultVideoAssemblyDeadlineMs,
"minimal RUDP URL did not keep the default assembly deadline");
require(fallback->audio_reorder_ms == muninn_rudp_url::DefaultAudioReorderMs,
"minimal RUDP URL did not keep the default audio reorder wait");
require(fallback->audio_reorder_ms == 2000,
"minimal RUDP URL did not use the full default audio reorder budget");
return 0;
}

View file

@ -125,6 +125,7 @@ struct CaptureState
std::vector<unsigned long long> sweepSamples;
std::vector<float>* playbackSamples = nullptr;
double playbackGain = 1.0;
long playbackOutputChannel = -1;
std::vector<float>* recordedInput = nullptr;
unsigned long long recordFrames = 0;
unsigned long long sweepFrame = 0;
@ -320,6 +321,11 @@ void processOutput(long doubleBufferIndex)
for (size_t channel = 0; channel < g_capture.outputBuffers->size(); ++channel)
{
if (g_capture.playbackOutputChannel >= 0 && static_cast<long>(channel) != g_capture.playbackOutputChannel)
{
continue;
}
auto& buffer = (*g_capture.outputBuffers)[channel];
auto& info = (*g_capture.outputChannels)[channel];
auto* raw = static_cast<unsigned char*>(buffer.buffers[doubleBufferIndex]);
@ -498,6 +504,12 @@ double doubleOption(int argc, char** argv, const char* name, double fallback)
return raw ? std::atof(raw) : fallback;
}
long longOption(int argc, char** argv, const char* name, long fallback)
{
const char* raw = option(argc, argv, name, nullptr);
return raw ? std::atol(raw) : fallback;
}
bool hasArg(int argc, char** argv, const char* name)
{
for (int index = 1; index < argc; ++index)
@ -576,6 +588,7 @@ int main(int argc, char** argv)
const char* playFloat32MonoPath = option(argc, argv, "--play-f32-mono", nullptr);
const char* recordFloat32InterleavedPath = option(argc, argv, "--record-f32-interleaved", nullptr);
const auto playGain = doubleOption(argc, argv, "--play-gain", 1.0);
const auto playOutputChannel = longOption(argc, argv, "--play-output-channel", -1);
const auto sweepGain = doubleOption(argc, argv, "--sweep-gain", 0.03);
const auto sweepToneSeconds = doubleOption(argc, argv, "--sweep-tone-seconds", 0.55);
const auto sweepGapSeconds = doubleOption(argc, argv, "--sweep-gap-seconds", 0.20);
@ -731,7 +744,7 @@ int main(int argc, char** argv)
playbackSamples.resize(static_cast<size_t>(byteCount) / sizeof(float));
input.read(reinterpret_cast<char*>(playbackSamples.data()), static_cast<std::streamsize>(playbackSamples.size() * sizeof(float)));
}
std::printf("asio playback-f32-mono: path=%s samples=%zu gain=%.5f\n", playFloat32MonoPath, playbackSamples.size(), playGain);
std::printf("asio playback-f32-mono: path=%s samples=%zu gain=%.5f outputChannel=%ld\n", playFloat32MonoPath, playbackSamples.size(), playGain, playOutputChannel);
}
if (captureSeconds > 0.0 || monitorSweep || !playbackSamples.empty() || emitJsonBlocks)
@ -820,6 +833,7 @@ int main(int argc, char** argv)
g_capture.sweepGain = sweepGain;
g_capture.playbackSamples = playbackSamples.empty() ? nullptr : &playbackSamples;
g_capture.playbackGain = playGain;
g_capture.playbackOutputChannel = playOutputChannel;
g_capture.emitJsonBlocks = emitJsonBlocks;
g_capture.sweepFrame = 0;
g_capture.sweepEnergy.assign(sweepFrequencies.size() * static_cast<size_t>(inputs), 0.0);

View file

@ -23,9 +23,14 @@ flowchart TD
C["Leap timing/IR driver"] --> R
D["network feed producers"] --> R
R --> N["native reservoir handles"]
N --> E["Fensalir GPU fusion + UI"]
R --> W["Mimir Well typed pages + timing state"]
W --> D["Fensalir reservoir daemon"]
N --> D
D --> E["Fensalir GPU fusion surfaces"]
R --> U["Mimir editor thin synced-video UI"]
N --> F["Faust/native DSP"]
E --> G["Spout2/program video"]
U --> V["EVE/native shared-texture program stream"]
F --> H["program stems + spatial bed"]
G --> I["OBS"]
H --> I
@ -40,12 +45,26 @@ Ownership:
phones, microcontrollers, Raven, and Starfire can participate without
becoming independent clock authorities.
- `Mimir.Runtime` owns app-level stream buffers and synchronization.
- `Mimir.Well` owns typed live pages, configured composites, timing pressure,
feature signals, and capture-body publication. It is the source the heavy
reservoir worker drinks from; it is not a renderer.
- Native capture workers own device reads.
- Fensalir owns dense visual fusion, material/brush/splat reconciliation,
D3D12 interop, runtime UI, and Spout2 publication.
- The future Fensalir reservoir daemon owns dense visual fusion,
material/brush/splat reconciliation, reservoir budgets, job scheduling, GPU
residency, and any CPU/GPU worker share thrown at field reconstruction. It
drinks from the Well and publishes typed reservoir/program surfaces.
- The Mimir editor owns only responsive 2D synced-video presentation controls
and program-layer placement for the current cut. It must not accidentally run
reservoir history work just because a frame is being drawn.
- Fensalir's in-app renderer owns D3D12 interop and lowering of explicit
surfaces. A flat video compositor backend is still missing: current
FieldEvidence camera resources are declarations/claims, not a production
textured-quad draw path.
- Faust/native DSP owns hot audio alignment, suppression, separation,
spatialization, and stem generation.
- OBS owns broadcast controls.
- EVE owns native decode/composite for its fullscreen display path; it does not
own layout, WebKit compatibility, or timing authority.
Invariant: the live window is bounded, in memory, and has one timing authority.
No private history outlives the rolling buffer.
@ -78,6 +97,14 @@ probes can emit per-frame JSON metadata so Fensalir sees real sensor cadence in
the rolling buffers while the direct ABI driver is being cut. It does not carry
pixels and does not own the final six-camera hot path. Multi-camera probes are
one process with declared accepted source ids, not one process per camera.
The `ffmpeg-rawvideo` adapter is the live network-pixel edge for Raven setup:
FFmpeg owns SRT receive/decode and writes exact raw frames to stdout;
`MimirFfmpegRawVideoStreamSource` owns frame-boundary reads, BGRA/NV12 payload
geometry, sample descriptors, and insertion into the normal synchronization
path. `config/mimir-runtime.raven-eve.example.json` declares `raven-display`
on SRT port `5200`. EveCanvas owns Eve camera/mic capture directly and sends
typed frame-events to `Mimir.EveSensorReceiver` on WebSocket ports `8793` and
`8794`.
## OBS Bridge Utility
@ -99,9 +126,13 @@ not become the synchronized program authority.
## Audio Field
The six-microphone path is separate from the bridge. Scarlett speaker loopback
is the current timing authority when active calibration is playing, but Scarlett
production capture belongs on ASIO rather than WASAPI shared mode.
The six-microphone path is separate from the bridge. Current local routing is:
Windows system default output is Scarlett for monitoring, while Starfire
Realtek is the active chirp-emission and loopback-measurement path. Any active
chirp runner that emits to the default output is therefore wrong unless it
explicitly selects Realtek. Scarlett production capture belongs on ASIO rather
than WASAPI shared mode, and Scarlett remains the professional mic/monitor
interface rather than the current chirp speaker authority.
`MimirBioacousticTimeline` owns the active runtime watermark described in
[[docs/bioacoustic-timeline-watermark|Bioacoustic Timeline Watermark]]: a
low-gain birdsong-like word language with 128 self-identifying word positions,
@ -152,14 +183,17 @@ diagnostic reference for constrained chirplet-transform work. The active
receiver is the bioacoustic motif timeline; the analyzer only emits active
timing reports from matched canonical anchors. The hub also owns cached reports
plus smoothed per-source sync state with delay-slope/SRO in ppm.
`MimirRuntime` runs analysis
online as a bounded rotating service and can print live telemetry with
`MIMIR_SYNC_TELEMETRY_SECONDS`. UI and telemetry are passive readers of cached
sync state; they must not invoke the analyzer. Current app testing proves
loopback wakeup, live mic buffers, and confident online sync states, but the
next proof is stable canonical anchors through real mic streams. Camera mics are
spatial/context witnesses; Focusrite devices are dialogue anchors. Fractional
delay and the hot resampler belong in Faust/native DSP.
`MimirRuntime` runs ordinary audio analysis online as a bounded rotating service
and can print live telemetry with `MIMIR_SYNC_TELEMETRY_SECONDS`. UI and
telemetry are passive readers of cached sync state; they must not invoke the
analyzer. Complex-contour matched-filter analysis is excluded from render-thread
and Well-loop cadence by default even when the contour capability is configured;
`MIMIR_COMPLEX_CONTOUR_RUNTIME_INLINE=1` is a diagnostic override, not the live
owner. Current app testing proves loopback wakeup, live mic buffers, and
confident online sync states, but the next proof is stable canonical anchors
through real mic streams. Camera mics are spatial/context witnesses; Focusrite
devices are dialogue anchors. Fractional delay and the hot resampler belong in
Faust/native DSP.
The WASAPI cadence probe is now a format/state diagnostic: it can request
shared or exclusive sample rates, bit depths, channel counts, and float/PCM
@ -279,9 +313,150 @@ inside BufferSmoke artifact replay. When `enableComplexContourRuntime` is true,
extracts Float32 windows from rolling audio buffers to publish
`evidence=complex-contour` reports. The synthetic runtime-shaped proof
`--complex-contour-runtime-self-test` at 192 kHz recovers a 693.5-sample delay
with about 0.219 us error from rolling buffers. The next real-world proof is to
run that lane through live Scarlett loopback and mics, not only stored ASIO
artifacts.
with about 0.219 us error from rolling buffers. In live app/Well operation this
is a configured and publishable evidence capability, not inline cadence
authority: expensive matched-filter execution belongs to a worker queue that can
commit cached reports back to the hub or CultMesh. The next real-world proof is
to run that worker lane through live Scarlett loopback and mics, not only stored
ASIO artifacts.
`MimirCalibratedAudioCompositeBuilder` is the first C# diagnostic owner for
turning that acoustic evidence into a corrected composite stream. It consumes
sample windows plus `MimirAudioSynchronizationState` delay/confidence and
per-band response evidence, applies fractional alignment, sparse measured-band
equalization, calibrated-band coherence scoring, incoherent-band suppression,
and confidence/noise-weighted summing, then emits composite samples plus source
flatness/weight/coherence reports and per-band coherence/suppression reports.
This is not the hot room-DSP
authority and it does not make room nulls disappear; Faust/native DSP still
owns the production alignment, suppression, separation, and stem lane.
`Mimir.BufferSmoke --calibrated-audio-composite-smoke` proves the contract on
synthetic delayed, colored, noisy mic paths: the uncalibrated mix measures
-3.171 dB SNR, the calibrated composite measures 15.135 dB SNR, and response
flatness improves from 0.548 to 1.000.
`Mimir.BufferSmoke --dual-mic-coherence-cancellation-smoke` proves the
two-witness cancellation cut: shared tones keep about 0.999 coherence, a
mic-local 3.52 kHz interferer scores about 0.001 coherence and 0.948
suppression, its measured energy falls from 0.039195 to 0.001306, and SNR rises
from 1.366 dB to 16.304 dB.
`scripts/record-kiyo-dual-mic-composite.ps1` is the first listenable Faust-chain
bridge receipt for that path. It starts Kiyo FFmpeg capture and Scarlett ASIO
raw capture together, converts ASIO channels 0/1 through
`--asio-dual-mic-faust-cleaner-wav`, and muxes the Faust dialogue WAV with Kiyo
video. The offline command compiles/runs
`faust/mimir_alignment_actuator.dsp` first, then feeds the aligned
shotgun/cardioid stems into `faust/mimir_dual_mic_dialogue_cleaner.dsp`; the
older C# composite stays a scoring/audition contract, not the current listening
receipt. The fresh 2026-06-01 receipt at
`artifacts/runtime/kiyo-dual-mic-composite/20260601-051044` and
`artifacts/runtime/kiyo-dual-mic-composite/kiyo-dual-mic-composite-latest.mp4`
has 10.000 s of 640x480 30 fps Kiyo video and 10.000 s of 48 kHz mono AAC from
the Faust dialogue stem. It estimated the cardioid 359.398 samples late
(7487.452 us), applied holdback 359.398/0.000 samples to shotgun/cardioid, and
wrote both dialogue and residual WAVs. The first muxed receipt was nearly
silent because `NormalizePeak` only attenuated hot samples; after the helper was
fixed to amplify quiet receipts too, the regenerated dialogue WAV peaks at
-0.724 dBFS with RMS -14.202 dBFS. The actual denoise authority is Faust:
`faust/mimir_dual_mic_dialogue_cleaner.dsp` compiles as a two-input/two-output
aligned shotgun/cardioid dialogue cleaner with smooth multiband voice shaping,
witness subtraction, and slow downward expansion. `MimirRuntime` queues it as
`dual-mic-dialogue-cleaner` with `host_voice` and residual `ambient` output
stems. It is fed from the alignment actuator's published `aligned_source_*`
stem frame, not from raw mic buffers, so Faust alignment is a real structural
precursor to cleanup. `Mimir.BufferSmoke --dialogue-cleaner-routing-smoke`
proves the boundary: alignment frames queue the cleaner DSP and input block, and
cleaner output frames are what enter `NativeProgram` stem publication. This is
the first hot-lane cleaner socket, not proof that the final studio denoiser is
solved.
The right rail `Output` panel is the app feature for local broadcast-proof
recording and live publish control, not the isolated Kiyo/Faust mic receipt.
`MimirProgramRecorder` owns `Record` / `Stop`, child-process status, timestamped
MP4 receipts under `C:\Users\Meta\Videos\Mimir` by default, and process-tree
shutdown. Its current actuator is `scripts/record-stream-proof-output.ps1`, so
app recording produces the same stream-proof composite intended for
Yggdrasil/public judgment: Raven program feed, Kiyo picture-in-picture, Starfire
Scarlett audio, and NVENC/AAC broadcast settings. `MIMIR_RECORD_START_RAVEN=1`
asks the app to start Raven's sender for the receipt. `MimirProgramStreamer`
owns `Live` / `Stop` for the `Mimir.Broadcast` RTMP path; it requires
`MIMIR_STREAM_INPUT` unless `MIMIR_STREAM_ALLOW_TEST_PATTERN=1` explicitly
authorizes a synthetic smoke stream. Later native recording/streaming should
replace these actuators behind this owner, not add another UI truth.
`MimirTargetedBioacousticProbePlanner` is the matching "ask the room a better
question" owner. It consumes per-source spectral residuals and optional source
pathology notes, then emits gain-bounded shaped probe bands for the
bioacoustic renderer to spend energy on. The first smoke models the two
professional 192 kHz Scarlett mics only: `asio-ch0-shotgun` is explicitly
noisy and low-heavy, `asio-ch1-cardioid` is cleaner, and the planner prioritizes
the shotgun's weak/unmeasured high-band evidence before lower residuals.
`Mimir.BufferSmoke --targeted-bioacoustic-probe-smoke` selects four probe
bands, led by about 18.6 kHz as an unmeasured band at the max bounded gain,
then about 9.9 kHz weak response, 3.6 kHz phase instability, and 910 Hz
magnitude residual. This is the intended first physical strategy: improve
shotgun/cardioid confidence at 192 kHz before asking every connected mic to
help solve the room field.
`MimirBioacousticProbeScheduler` now owns the first closed-loop cadence layer
above that planner. It reads smoothed sync states plus derived per-band
frequency-response residuals from `MimirSynchronizationHub`, backs probe cadence
off as aggregate sync/frequency confidence rises, emits targeted probe plans
only when the confidence/duty budget says to spend sound, and exposes
`mimir-bioacoustic-probe-schedule` plus per-source telemetry in `MimirRuntime`.
`Mimir.BufferSmoke --bioacoustic-probe-scheduler-smoke` proves the visible
climb: simulated probe feedback raises sync confidence from 0.360 to 0.920 and
frequency-response confidence from 0.558 to 0.955 over four scheduled probes.
The scheduler is not the output-device authority; in the current physical
routing the eventual live renderer must explicitly select Starfire Realtek.
That Realtek renderer now exists as a tooling edge. `Mimir.WasapiLoopback`
targets `net10.0-windows`, enumerates active render endpoints, selects by
friendly-name substring, and can play mono Float32 probe audio through the
matched WASAPI render endpoint. `Mimir.BufferSmoke
--targeted-bioacoustic-probe-realtk-live` renders a targeted probe, starts
Scarlett ASIO capture, plays it through the selected Realtek speaker endpoint,
then scores the captured shotgun/cardioid response. The first proof selected
`Speakers (Realtek(R) Audio)` and wrote
`artifacts/wasapi/targeted-bioacoustic-realtk-live/20260601-033851`; at gain
0.75 the shotgun peak was 0.407887 and cardioid peak was 0.074494, but 18.6 kHz
remained near-zero confidence while cardioid 3.6 kHz was the strongest printed
band at 0.098 confidence. Realtek emission is now real; high-band probe choice
is still a bad physical question for this room/speaker path. The default
targeted probe planner now caps selection at 12 kHz and biases measurable
low/mid bands instead of favoring ultrasonic-ish uncertainty that ordinary
speakers and mics cannot answer well.
Starfire's USB-connected PS Move is now a paired audio/visual timing actuator
through `Mimir.PsMoveProbe --paired-chirp-train`. The worker renders one raw
Float32 chirp train, asks `Mimir.WasapiLoopback` to emit it, and schedules Move
LED on/off phases from the same `PairedChirpEvent` documents. The schedule is
the owner: every visual pulse must share the same event id and planned offset
as its audio chirp. The 2026-06-02 receipt
`artifacts/runtime/starfire-usb-move-paired-20260602-144135/paired-events.jsonl`
proved three emitted chirps and three USB LED pulses, with LED onsets matching
planned 150/400/650 ms at millisecond receipt resolution after the HID train
moved to high-resolution stopwatch waits.
The calibration runner is now closed-loop instead of a blind full-spectrum
sweep: `--bioacoustic-realtk-scheduled-calibration-live` asks the scheduler for
each probe, emits through Realtek, captures Scarlett, updates residuals, and
then asks the scheduler again. It is a live physical I/O diagnostic harness
seeded from the current Scarlett residual model; production still needs measured
runtime confidence state wired into the same scheduler/emitter path. The
2026-06-01 receipt at
`artifacts/wasapi/realtk-scheduled-calibration-live/20260601-035354` ran four
scheduled steps at gain 1.15 and wrote `scheduled-response-bands.csv` plus
`balance-plan.md`. Local peak-vs-percentile-floor scoring found strong usable
response near 910 Hz, 3.6 kHz, and 9.84-9.87 kHz on both Scarlett mics.
The scheduler now filters confidence/status to the same 120 Hz-12 kHz
schedulable window as the targeted planner, so old 18.5 kHz residual scars can
remain recorded without owning live cadence or weakest-band status.
The first live run preserves two artifacts:
`artifacts/asio/targeted-bioacoustic-live/20260601-030836` at playback gain
1.0 and `artifacts/asio/targeted-bioacoustic-live/20260601-030857` at gain
2.0. Loopback energy scaled, but 18.6 kHz stayed nearly absent in the physical
mics; the second pass measured shotgun -51.475 dB and cardioid -56.585 dB
relative to loopback at that band. The 910 Hz probe was strongest physically,
especially shotgun at -21.994 dB relative to loopback. Current high-band
probing should be treated as failed path evidence, not a calibration win.
Louder follow-ups at gain 4 and gain 8 stayed below clipping; gain 8 reached
shotgun peak 0.140715 and cardioid peak 0.027774. Even then, 18.6 kHz remained
unusable at shotgun -54.910 dB and cardioid -63.258 dB relative to loopback.
Bias the next probe language toward usable low/mid anchors unless the speaker,
mic placement, or output path changes enough to make high-band evidence real.
That real-world proof now has a first receipt:
`calibration/bioacoustic/complex-contour-live-20260525-063229.md`. A freshly
rendered canary-packet witness was played through Focusrite ASIO and captured
@ -292,12 +467,158 @@ channel model loaded. This proves the contour/channel-model path survives a
fresh DAC/speaker/room/mic/ADC pass, but low physical confidence means the next
cut is stronger direct-path confidence and independently measured path truth,
not a victory lap.
A follow-up receipt,
`calibration/bioacoustic/complex-contour-live-20260528-002313.md`, keeps the
interface loopback proof healthy: `asio-ch2` to `asio-ch3` measured `-0.062 us`
with `0.956` confidence and 257 direct hits. Physical inputs are not currently
usable timing signals: `asio-ch0` and `asio-ch1` produced zero contour hits
against 1480 reference hits, with very low RMS. The next physical proof is
blocked on mic gain/routing/source placement before receiver logic changes.
After arming both mics, `complex-contour-live-20260528-080604.md` showed both
physical inputs were alive but was accidentally routed to headphones. The
monitor-routed proof `complex-contour-live-20260528-080727.md` is the first
credible fresh physical contour receipt: loopback is `-0.014 us` at 0.971
confidence, `asio-ch0` lands `-0.410 us` from its stored path seed at 0.524
confidence with 24 direct hits, and `asio-ch1` lands `-2.467 us` at 0.363
confidence with 14 direct hits. User-provided physical mapping: `asio-ch0` is
the shotgun, about 1.5 m from the right monitor and 3.0 m from the left;
`asio-ch1` is the cardioid, about 0.5 m from the left monitor and 1.5 m from
the right. Repeat captures are needed before updating the channel model or
letting physical mic reports drive actuator authority.
`complex-contour-monitor-split-20260528-081813.md` adds
`--play-output-channel` to the ASIO probe and captures output 0 and output 1
separately. Loopback proves the ASIO output buffers are distinct, but physical
mic delays are nearly identical across outputs: shotgun differs by about
20.885 us and cardioid by about 24.789 us. That does not match the declared
left/right monitor geometry, so the downstream monitor path is probably
summing, mirroring, or otherwise not isolating the physical speakers.
The first actuator proof now exists: `faust/mimir_alignment_actuator.dsp` owns
six channels of bounded fractional delay/gain controls for Faust/native DSP,
and `Mimir.BufferSmoke --bioacoustic-actuator-self-test --sample-rate 48000
--delay-samples 317.375` proves the control loop shape by estimating a
bioacoustic delay, applying fractional correction, and remeasuring residual
below printed microsecond precision.
below printed microsecond precision. `MimirAlignmentActuatorBank` is now the
runtime command owner above that Faust surface: it converts smoothed sync states
into nonnegative holdback commands, keeps source-to-`sourceN` slots stable,
reports overflow beyond the six-source profile, and exposes the reference
holdback separately. `MimirRuntime` updates that command frame on the sync
analysis cadence, queues it into `AquariumAudioDocument` as an engine audio
control frame, declares `faust/mimir_alignment_actuator.dsp` as an
`AquariumStreamingDspProgram`, and telemetry can print the current DSP targets.
Mimir also queues sample-bearing `AquariumStreamingAudioBlock` lanes from the
latest actuator source buffers, using the bank-assigned Faust `sourceN` control
path as channel authority. Fensalir can compile/host/process that persistent
Faust graph now. The engine now publishes processed output lanes as
`AquariumAudioStemFrame` records through `IAquariumAudioStemBus`, with Mimir's
actuator program declaring deterministic aligned-source stem names. OBS-facing
publication now has a first readiness surface:
`MimirObsStemPublicationState` consumes those frames without copying audio and
reports ready, missing, and unconfigured stems against the alignment actuator
stem-bus profile. `MimirObsStemSharedMemoryPublisher` writes the validated stem
frames into `Local\MimirObsStemBus`, and `native/obs_stem_source` is the first
OBS source plugin bundle: `Mimir Audio Stem` reads one named stem from that map
and submits it through libobs audio, while `Mimir Program Texture` opens
Fensalir's named shared D3D12 program-output texture and draws it through an
OBS-readable shared D3D11 bridge texture. The video path is GPU-to-GPU and does
not require Spout2 or CPU readback; the remaining copy is the explicit D3D12 to
libobs-D3D11 boundary. Fensalir publishes `Global\MimirFensalirProgramFence` as
a program-output publication fence, and the OBS source opens that fence before
copying. This prevents blind read-before-producer-completion without coupling
OBS to Fensalir's private frame fence. An optional texture ring is available
when Fensalir and OBS agree on `FENSALIR_PROGRAM_OUTPUT_RING_COUNT`; OBS selects
the latest completed slot from the program-output fence value. OBS can publish
`Global\MimirObsProgramConsumerFence`; when Fensalir is launched with
`FENSALIR_PROGRAM_OUTPUT_CONSUMER_FENCE_NAME` pointing at that fence, it skips
publication instead of reusing a ring slot OBS has not acknowledged.
`scripts/build-obs-stem-plugin.ps1` stages the upstream OBS plugin-template SDK
under `artifacts/obs-sdk/` and builds the plugin DLL locally.
The rendering migration's program-output receipt phase is structurally
complete: OBS consumes final Mimir/Fensalir surfaces and stems through the
native plugin, raw feeds remain debug inputs, and the ledger records the live
program texture, texture ring, dedicated producer fence, and consumer-fence
proofs.
The public self-hosted edge has its first deployable path. `src/Mimir.Broadcast`
builds the Starfire-side FFmpeg/NVENC push command for
`rtmp://127.0.0.1:11935/live/mimir`; the port is an SSH local forward to
Yggdrasil's localhost RTMP listener. Yggdrasil owns nginx RTMP ingest and HLS
segment serving only. It must not transcode or compose in v1. The public static
viewer route is `https://gamecult.org/livestream`, owned by the root
`gamecult-site` repo, and it reads
`https://streampixels.gamecult.org/mimir/live/hls/mimir.m3u8`. On 2026-05-30
the deployed Yggdrasil origin validated nginx, listened on `127.0.0.1:1935`,
returned `mimir-live-ok`, and generated four HLS segments plus `mimir.m3u8`
from an eight-second Starfire synthetic NVENC push. `live.mimir.gamecult.org`
is no longer required for v1 because StreamPixels already has DNS and TLS on
Yggdrasil, and Mimir's own static site does not own the viewer.
`MimirSynchronizedBufferPlanner` is now the low-level aligned-buffer primitive:
it picks one canonical presentation time inside the retained rolling window and
returns per-stream slices for cameras, network display feeds, and audio. Timing
corrections may target either one source or a shared `clockDomainId`. The Raven
shape is explicit: Raven controls both display pixels and an audio timing signal
routed into Scarlett, so Scarlett-decoded Raven audio evidence can earn a
`raven-sync` correction that is applied to the `raven-display` buffer. Network
arrival timestamps remain metadata, not timing authority.
The first operational ingest config is `config/mimir-runtime.raven-eve.example.json`:
Raven screen capture arrives over SRT and decodes to raw BGRA through FFmpeg;
Eve camera/mic arrive as EveCanvas-native WebSocket frame-events. All three
enter the same rolling-buffer model as local sources. Eve's sensor authority now
lives in EveCanvas, not an external camera-streaming app.
`MimirPresentationControlState` now owns the Fensalir program-control intent:
video feed visibility/solo/opacity/layer order, audio mute/solo/gain, and
global LUT preset selection. The `Mimir Program` panel exposes those controls
without giving OBS or bridge endpoints composition authority. Video controls
filter production surface intents before Fensalir composition; audio controls
modify Faust gain controls and sample gain before Fensalir streaming DSP. LUT
presets are typed postprocess state, with exposure/bloom currently mapped into
`GraphicsSettings` and LUT texture sampling left as the next renderer hook.
`MimirSceneEditorState` is now the separate Mimir-window editor owner. It owns
the editor camera, dynamic scene nodes, selected node, visibility/lock state,
2D-plane transforms, reset commands, and grab/rotate/resize gizmo mode. Rolling
video buffers derive sensor-feed panel nodes; SDF text panels and model import
requests create editor graph nodes. `MimirRuntime` presents that editor through
the Fensalir camera plus derived spline outlines, selection handles, and the
`Mimir Editor` hierarchy/transform panel. The center preview requests a
Fensalir-owned blit of the current output buffer, then draws Mimir's normalized
placement overlays on top. This editor is not the OBS program output authority;
it is a live program-surface witness. The placement commit primitive keeps
layers inside the program frame, snaps near edges/center, and preserves source
aspect while resizing through direct preview handles, sliders, or the scale
gizmo. Explicit transform/source controls are inspector-only and gated to a
selected program source; the center canvas handles are the primary compositor
edit path. Empty-canvas clicks clear source selection; hover/active handles and
snap guides come from the shared Eve dashboard document state that Fensalir's
Direct2D and D3D12 paths lower locally; arrows nudge the selected layer,
PageUp/PageDown reorder layers, and the inspector owns the selected source
contain/cover/stretch fit mode. Eve is the portable dashboard authority, while
CultUI is only the C# reference API shape for the document. The shared element
contract now includes typed CultCache/CultMesh field bindings: document
schema/id, field path, value kind, access, authority, and optional command id.
Callbacks are native lowering actuators; the binding is the portable
Verse-backed state contract. World SDF glyph rendering, ASSIMP-style
mesh decoding/upload, and pixel-accurate gizmo hit-testing are explicit
Fensalir renderer cuts, mapped in
[[docs/scene-editor-control-surface|Mimir Scene Editor Control Surface]].
`Mimir.EveDashboard` is now a small native Eve dashboard broker. It serves
`/eve/deck` plus the compatibility `/eve/dashboard` endpoint, exposes provider
manifests at `/eve/deck/providers`, and routes retained dashboard snapshots and
commands for provider-owned trees. Built-in providers are the switchboard,
`mimir.stream.layout`, `voidbot.swarm`, and `yggdrasil.streampixels.edge`;
launch-time external providers can be registered as `id|title|ws-url`
descriptors for LAN or SSH-forwarded TCP surfaces. EveCanvas renders the
dashboard locally in UIKit, including the scene graph and multitouch
pan/pinch/rotate source panels, and now treats provider-card taps as
`open-provider` commands. `voidbot.swarm` reads VoidBot's existing
`.voidbot/status/swarm-state.json` projection, publishes avatar URLs on
identity nodes, and Eve renders a native VoidBot-style cockpit: CTB rail,
selected Face/status pane, state tree, and detail panel.
Dashboard manifests and states have typed CultMesh-compatible document shapes
`mimir.eve_dashboard_manifest.v1` and `mimir.eve_dashboard_state.v1`, while the
live iPad lane remains WebSocket/TCP because it is the proven tunnelable
transport. `mimir.eve_dashboard_state.v1` now mirrors each retained element's
optional binding object so clients can move fields into CultCache without
turning `bindNodeId` into a second authority. The Mimir stream-layout provider
still uses fixture state; the next cut is binding it to live
`MimirPresentationControlState` and `MimirSceneEditorState`.
## Visual Fusion
@ -305,23 +626,299 @@ Visual fusion belongs in Fensalir over current reservoir claims. Native capture
workers provide frames; Fensalir owns feature extraction, matching, material
fitting, render budgeting, and publication.
The live debug spectrum view now has two separate authorities:
Fast stereo depth is now mapped as a future Fensalir D3D12 compute lane, not an
external library dependency. The provenance note
`research/d3d12-stereo-depth-provenance-2026-05-29.md` records `libSGM` as the
current permissive north-star reference: Mimir/Fensalir should rebuild the SGM
shape over synchronized rectified texture pairs, calibration state, typed GPU
resources, and explicit fences. CUDA, TensorRT, and monocular depth demos remain
research references only.
The first live contract exists: `MimirStereoDepthConfigurations` names the
libSGM-provenance D3D12 SGM profile, and
`MimirFensalirFieldLowering.BuildStereoDepthCandidateFrame` lowers caller-
declared rectified stereo input textures into a GPU-resident compute-writable
disparity SurfacePage, confidence texture, Height FieldEvidence claim, and
`AquariumFieldStereoDepthLowering` sidecar. The live
`BuildLeapPackedStereoDepthCandidateFrame` path now recognizes a live
`LeapStereoIr` rolling video window, reuses its declared packed R8G8 texture as
both left and right resource identity, names the left/right observations as
packed R/G lanes, and emits the same depth lowering automatically from the
runtime frame. The lowering sidecar is the dispatch contract: it binds the
libSGM-provenance profile, calibration id, camera pair, inputs, disparity
output, min disparity, disparity levels, aggregation paths, census radius,
P1/P2 smoothness penalties, and depth range. The current D3D12 SGM profile
publishes min disparity 0, 128 disparities, four paths, census radius 2, P1 8,
and P2 96. Fensalir now has a first packed-Leap D3D12 kernel that writes R16F
disparity through a crude SAD/block-match pass. This is intentionally a small
kernel-shaped proof, not full libSGM-style SGM or calibrated metric depth yet.
The first point-cloud root is also code-visible now:
`MimirPointCloudConfigurations.LeapDisparityPointCloudRoot` records the
stereo-disparity projection profile and provenance, while
`MimirFensalirFieldLowering.BuildLeapPackedStereoPointCloudCandidateFrame`
declares a GPU-resident `FieldMesh` point-list resource derived from the Leap
R16F disparity SurfacePage. `MimirRuntime` merges that Mesh claim into the same
evidence frame as the Leap stereo-depth lowering. `Mimir.BufferSmoke
--leap-point-cloud-root-smoke` proves the combined depth/point-cloud contract
plans as one `SurfacePage` packet and one `Mesh` packet with zero deferred
requests. Fensalir branch `codex/leap-packed-depth` now has the first native
surface-claim lane too. It preserves Mesh `SourceUri=derived-from:*`, allocates
generated point-list vertex/index buffers as UAV-capable GPU resources, fills
them from the disparity SurfacePage in `D3D12PointCloudFromDisparityCS`, and
writes approximated normals, curvature, SDF, PBR, and radiosity reservoirs into
the existing fractal splat buffers. The point-list draw path is diagnostic-only;
the verified live presentation path is Fensalir's splat/reservoir resolver.
The 2026-05-31 live capture
`artifacts/runtime/mimir-leap-surface-claims-clean.png` proves the clean path:
LeapUVC frames uploaded through a Fensalir-owned D3D12 texture, packed stereo
dispatched to R16F disparity, point-cloud compute generated 19,200 surface
claims, the point-list mesh renderer stayed at `rendered 0`, and the reservoir
resolver presented visible Leap-derived splats. The follow-up
`artifacts/runtime/mimir-leap-history-split-shader-visible.png` proves the
history pass now starts without `AQUARIUM_DISABLE_RESERVOIR_HISTORY`: Fensalir
compiles a dedicated lean `D3D12ReservoirHistoryUpdate.hlsl` and writes the
current resolver sample into the four-row history/resolved target. Temporal and
spatial reuse are deliberately not restored in that lean shader yet; the
remaining truth gaps are calibrated projection constants, full SGM, temporal
reservoir reuse split into a compilable shader, and a global
residual/calibration owner.
- `AquariumBufferFieldFrame` is the real Fensalir contract. Mimir expresses
each rolling audio spectrum window as a buffer-backed spline tube field:
frequency lives in the spline domain, amplitude shapes the tube path, and
appearance/probe policy can reference tangent, curvature, normal, derivative,
object domain, and parent-space transforms. This is the surface the future
compute lowering consumes.
- `AquariumSplineFrame` is only the immediate preview witness. It renders the
same spectrum windows as camera-facing Catmull-Rom tube ribbons so humans can
sanity-check ASIO loopback and mic buffers before the reservoir path owns the
draw. It must not become the final visual authority.
Fensalir must not be treated as a traditional rendering pipeline. Mimir does
not ask it to "draw a thing" and hope post-processing makes the result true.
Mimir submits evidence, buffers, constraints, and surface intent; Fensalir
lowers those into field claims with travel/depth, metadata, control, and
reservoir-guide lanes. If a path only paints pixels, it is a fallback/debug
draw, not the Perfect Machine surface.
The intended Fensalir lowering is buffer field expression -> compute-generated
stochastic SDF splat probes -> spatiotemporal splat reservoir -> reservoir
sampling for temporally antialiased surface contribution. Splat density is a
function of visual contribution, not uniform polyline tessellation.
The live payload boundary is now explicit. Mimir declares each live native/GPU
payload view as an `AquariumFieldResourceDeclaration` with resource key, kind,
residency, shader access, format, shape, valid time range, version, and native
handle metadata. Field claims and lowering requests reference those
`mimir:resource:*` keys. The handle string is only a name; Fensalir validation,
planning, and renderer-owned resource slots own typed resource resolution before
shader lowering. Mimir-declared rendering buffers are Fensalir GPU resources in
the same runtime; native/shared handle metadata is an import edge, not a
separate authority regime. Rendering-relevant buffers move to GPU residency as
early as possible and stay there; CPU readback/tessellation is diagnostic only.
Camera descriptors now enter this bridge too. The preferred hot path is
Fensalir-owned texture leasing: the runtime receives `AquariumRuntimeServices`,
asks Fensalir's field resource broker for a keyed D3D12 `Texture2D` lease, the
camera/decode producer writes that shared texture and signals the producer
fence, then Mimir carries the same resource key through FieldEvidence. Fensalir
waits on committed producer fence values before resolving shader reads. Shared
foreign texture handles remain an import edge, not the primary camera authority.
`MimirVideoCaptureDriverSource` propagates the lease client to drivers that
implement `IMimirFensalirTextureLeaseReceiver`, so direct camera drivers can
request the destination texture before emitting a frame descriptor.
Each camera backend must use the closest-to-device path available and expose
its unavoidable copy count. Managed/process wrappers are diagnostic bring-up
surfaces, not the production camera hot path.
For raw single-plane system-memory frames, `MimirVideoCaptureDriverSource`
performs the one declared CPU-to-GPU upload into the Fensalir lease, drops the
managed payload from the live sample, and increments `UnavoidableCopyCount`.
NV12 CPU upload is rejected until Fensalir owns a real planar upload path;
device/GPU NV12 producers should write the leased texture directly.
`MimirKsVideoCaptureDriver` plus `native/camera_capture/mimir_camera_capture.dll`
is the first production-shaped KS/UVC driver: it opens a Kernel Streaming pin
in process, queues uncompressed UVC frames, and feeds the same
`IMimirVideoCaptureDriver` source path. MJPG/H264 still need a device/GPU
decode producer; the KS driver rejects compressed capture formats.
`MimirPs3EyeVideoCaptureDriver` plus
`native/camera_capture/mimir_ps3eye_capture.dll` does the same for PS3 Eye raw
WinUSB/libusb capture, emitting Bayer8 frames through the common upload lane.
Driver smokes have pulled and uploaded real frames for LeapUVC 640x240 YUY2,
both PS3 Eyes at 320x240 Bayer8, Kiyo Pro 1920x1080 YUY2, and regular Kiyo
640x480 YUY2. Regular Kiyo 1280x720 YUY2 did not open.
`config/mimir-runtime.perfect-machine.local.json` is the current local
all-sensor runtime profile: Leap packed stereo IR, both PS3 Eyes at
320x240@187, Kiyo Pro RGB, and regular Kiyo RGB. `Mimir.BufferSmoke
--perfect-machine-live-smoke` opens all five video sources through the runtime,
attaches Fensalir texture leases, and requires five camera Texture2D resources,
one Leap stereo-depth lowering, one disparity SurfacePage, and one derived
point-cloud Mesh. Fensalir now plans those camera Feature/SensorObservation
claims through its `GpuSensorFusion` backend instead of deferring them, then
derives compute input metadata and SRVs from resolved FieldEvidence Texture2D
resources. The current dispatch samples LeapStereoIr, both Bayer8 PS3 Eyes,
Kiyo Pro YUY2, and regular Kiyo YUY2. Fensalir binds YUY2 through the
Microsoft-documented `R8G8B8A8_UNORM` SRV view over `DXGI_FORMAT_YUY2` and
decodes `Y0/U/Y1/V` in compute; packed camera video is not silently treated as
ordinary RGBA.
`MimirMediaFoundationGpuVideoCaptureDriver` plus
`native/camera_capture/mimir_mf_gpu_capture.dll` is the compressed camera path:
Media Foundation SourceReader runs with a D3D11 device manager, decodes MJPG or
H264 camera frames on the GPU, copies the decoded GPU surface into a shared BGRA
D3D11 texture, and publishes a `shared-d3d11-texture` handle with no live CPU
payload bytes. Kiyo Pro 1920x1080 MJPG->RGB32 and H264->RGB32 smokes both
produced valid GPU handles/resources. Direct NV12 shared texture creation
failed in D3D11, so NV12 needs plane-aware interop or a GPU bridge-copy cut.
Metadata-only cadence frames do not create camera surface intents or fake
payload requests. The old direct `AquariumGpuSensorFrame` bridge proof has been
removed from Mimir's active proof path. Camera image claims currently defer
until Fensalir owns a selected visual-fusion lowering for camera textures.
The current smokes prove planning, not visible packet rendering:
`--fensalir-field-evidence-smoke` now produces one planned resource-backed
`TubeField` packet from Mimir's spectrum intent, and
`--fensalir-field-dsl-resource-smoke` proves the Fensalir DSL can bind the same
kind of declared resource directly. `--fensalir-camera-observation-smoke`
proves the camera observation/resource split. Fensalir now has the first D3D12 resolver
cut for structured/curve buffers: it imports shared GPU buffers by handle and
allocates engine-owned GPU slots only for Fensalir-produced resources. The DSL
can describe a 2D rolling float buffer as Catmull-Rom XY tubes with modulo
column addressing, amplitude power/normalization, radius, ramp texture path, and
emission scale. Fensalir now resolves the first non-buffer resource family too:
Texture2D local assets bind as TubeField ramps, surface pages and volume
textures allocate shader-readable GPU textures, and mesh packages own
`Mesh.Vertices`/`Mesh.Indices` GPU buffer shape under one resource key. The DSL
can plan generic resource-backed claims over those declarations. The next
blocker is no longer resolver ownership; it is selected render lowerings that
consume camera/feature textures, mesh/page/volume resources as geometry,
height/SDF/material pages, or density/extinction/SDF3D domains. Mesh layout
authority is split by source:
imported/user meshes use the standard `PositionNormalUvColor` layout, and
generated meshes can be `PipelinePrivate` so each lowering owns the bytes it
emits and consumes. TubeField is the first concrete generated-mesh consumer:
its compute pass emits private vertex/index/indirect buffers and render binds
them as `D3D12PipelinePrivateGeneratedMesh` before applying source/ramp/material
state. The DrawIndexed indirect command signature has been lifted to the
generated-mesh lane instead of being TubeField-owned. TubeField expansion now
requires a planned TubeField backend packet for the lowering's claim, and
validation rejects TubeSpline metadata whose claim/resource authority is split.
Mimir's typed surface-intent lowering now emits the matching TubeSpline
lowering for audio spectrum/waveform Tube claims.
Mimir's active proof path also no longer uses the direct
`AquariumAcousticFieldFrame` builder: sync states lower into FieldEvidence
calibration constraints. Fensalir now selects audio-path `Phase` and
`Confidence` claims as `DebugOverlay` backend packets, so calibration
timing/path evidence can be inspected through the same FieldEvidence authority
while true acoustic source candidates and confidence volumes remain later
fusion work. The first synthetic acoustic source candidate proof now exists:
Mimir lowers an SRP/PHAT-style localized source with calibration/source
identity and a world-space confidence envelope, and Fensalir plans it as a
FieldEvidence `DebugOverlay` packet. Live room geometry must still come from an
explicit measured coordinate frame, not inferred from the recent two-distance
monitor notes. Phase 5's synthetic marker proof now exists too: Mimir lowers a
deterministic multi-camera marker candidate with calibration/marker identity and
world-space support, and Fensalir plans it as `DebugOverlay` evidence while
ambiguous raw camera features still defer.
LED strips now have a typed calibration-evidence socket instead of being folded
into fake depth. `MimirLedSplineFieldCandidate` carries ordered per-camera
bright-curve samples in that sensor's image plane and local clipspace/frustum,
whether the LEDs were temporally coded, and whether the sample indices are
stable. `BuildLedSplineCandidateFrame` lowers that shared curve as
deterministic camera-sensor Feature evidence; Fensalir plans it as a
`DebugOverlay` packet through the same FieldEvidence authority. Addressable or
blink-coded strips can provide exact cross-camera LED identity. Plain identical
white LEDs still provide a strong spline/epipolar constraint, but their point
correspondence is ambiguous under occlusion, saturation, repeated spacing, and
motion blur until temporal continuity, Leap/body anchors, or the residual
solver disambiguates it. `MimirLedSplineCurveSolver` is the first per-sensor
curve owner: it orders bright detections and computes image/clipspace curve
samples. `MimirCameraRigCalibrationSolver` is the first higher-level frustum
fit owner: given stable LED indices and a scene-space LED curve anchor, it
minimizes ray-to-curve distance and emits bounded camera translation updates.
Individual cameras do not write calibration state. Rotation, focal length, and
distortion solving are still future cuts, not hidden behavior. Exposure/gain
sweeps are scored by the recovered spline, not raw brightness:
`MimirLedSplineQualityScorer` combines detected LED count, curve coverage,
spacing coherence, smoothness, exposure fitness, and saturation penalty, and
`MimirLedSplineSweepSelector` picks the best calibration-usable setting. Once
`MimirLedSplineFrameAnalyzer` connects physical frames to that score by
extracting bright luma components, solving the sensor-local curve, and rejecting
clipped blobs as unusable calibration evidence even when they are easy to
detect. `Mimir.BufferSmoke --led-spline-live-score` is the current live sweep
surface: KS/UVC cameras get manual exposure/gain writes before each scoring
window, stale frames are drained, luma is extracted from the actual live frame
format, and the best spline setting is reported per source. On 2026-05-31 with
`--expected-leds 38 --minimum-luma 0.55`, both Kiyo cameras selected exposure
`-9`, gain `8` (`kiyo-pro-rgb` score `0.666`; `kiyo-basic-rgb` score `0.650`).
LeapUVC rejected generic KS exposure/gain writes and only found two LED
components; PS3 Eye feeds are still fixed-control until the WinUSB wrapper owns
gain/exposure. PS3 Eyes now also have a sparse tracking bootstrap:
`MimirSparseFeatureTracker` detects grid-suppressed Bayer/luma corner features,
links them frame-to-frame, and emits `MimirFeatureTrackCameraObservation`
records with sensor-local image/clipspace position, velocity, age, and
confidence. `MimirFeatureTrackFieldCandidate` lowers those observations as
camera Feature claims through Fensalir field evidence. A one-second live pass
at 320x240@187 with sensitive defaults returned 160 live tracks from each Eye:
160 stable tracks from `ps3-eye-0` and 156 stable tracks from `ps3-eye-1`.
This is a CPU diagnostic owner; the hot production path remains Fensalir D3D12
feature extraction. A full 2026-05-31 Fensalir headless run using the
all-sensor profile started all five video sources, declared eight field
resources, resolved six Texture2D resources, dispatched one packed-Leap
stereo-depth kernel, generated one point-cloud/surface stream, reported 19,200
visible reservoir splats, and captured
`artifacts/runtime/mimir-perfect-machine-all-sensors-headless.png`. The
follow-up GPU feature-extraction proof planned 12 packets with zero deferred
requests, dispatched sensor fusion over three sampleable camera textures,
generated 49,152 sparse gaussians at about 0.059 ms/frame sensor-fusion GPU
time, and captured
`artifacts/runtime/mimir-perfect-machine-gpu-feature-extraction-headless.png`.
The YUY2 decode proof then dispatched all five camera textures with zero
unsupported field textures, generated 81,920 sparse gaussians at about 0.08
ms/frame sensor-fusion GPU time, and captured
`artifacts/runtime/mimir-perfect-machine-yuy2-gpu-feature-extraction-headless.png`.
Once the
frustum fit is coherent, detected
surface candidates can be sampled across every synchronized view before
Fensalir resolves them into 4D Gaussian/surface splat claims.
The current teardown/migration map is
`docs/fensalir-rendering-rebuild-migration.md`, paired with Fensalir's
`docs/rendering-teardown-rebuild-protocol.md`. The intended cut is explicit:
Mimir publishes typed physical observations, calibration constraints, and
surface intent; Fensalir owns field domains, field claims, candidate selection,
track/reservoir state, selected lowerings, temporal guide lanes, and
presentation.
The reservoir can resolve pixel-level claims without requiring pixel-sized
contents. Claim support belongs to the represented domain. A smooth flat surface
can be a few huge surface claims; a heightfield terrain belongs in a quadtree of
brush-painted surface tiles, subdivided only where projected footprint,
curvature, material/brush detail, silhouette risk, or temporal uncertainty make
larger claims dishonest. Fensalir's SDF probe/lowering stage should emit
surface splats at that detail-adaptive density, not one splat per pixel by
default.
The live debug spectrum view is now field-evidence owned. Mimir no longer
submits direct `AquariumSplineFrame` trails for the spectrum dashboard and does
not submit `AquariumBufferFieldFrame` / fractal `ReservoirSplats` for it either.
Mimir declares the rolling resource, publishes a Tube claim plus
`AquariumFieldTubeSplineLowering`, and Fensalir owns planning, generated mesh
expansion, and TubeField material rendering. The spectrum resource is not just
a handle: Mimir uploads the current spectral amplitudes as row-major Float32
data into a Fensalir-owned GPU structured buffer before TubeField compute reads
it. TubeField writes stable field ids, real tube normals, coverage/confidence,
and domain-validity guide data into the same scene metadata/control/reservoir
guide targets consumed by reservoir resolve for spatiotemporal reconstruction.
Fensalir's shared reservoir history update now runs as a compute pass over
four structured rows per pixel and emits the resolved HDR field texture for
bloom/presentation, so the presentation shader is no longer a hidden history
owner.
TubeField now feeds the shared reservoir from GPU-resident rolling-buffer
column packets instead of generated segment packets. Fensalir emits one column
packet per logical spectrum history/source column, bins those columns into
screen tiles, and evaluates Catmull-Rom tube SDF/material samples directly from
the source buffer during the ReSTIR passes. Generated mesh draw args are zeroed
for the live path and remain only a diagnostic reference.
`Mimir.BufferSmoke --mimir-spectrum-upload-smoke` verifies this through the
actual `MimirRuntime` frame path and confirms the legacy direct spline and
buffer-field dashboard inputs are empty. The same path declares the local
blackbody ramp as a Texture2D field resource and binds the TubeField material by
resource key. Runtime spectrum columns now represent `(history age, source
lane)` pairs rather than only the latest spectra; Mimir emits one TubeField
claim/lowering per source, each striding through the shared resource to render
its own age trail at `z = age * 0.1`. The resource advertises fixed history and
source-lane capacities, so content updates and source topology changes do not
resize the GPU buffer while the rolling trail fills. `MIMIR_SPECTRUM_SOURCE_LANES`
sets the fixed lane budget; surplus sources are truncated and reported in the
runtime UI. Physical history slots are now addressed as a ring with
`RollingOffset`, so logical age no longer requires moving every column before
shader sampling. Mimir emits one Float32 resource upload for the newest
spectrum ring slot with an explicit element offset instead of uploading the
full fixed-capacity backing buffer every frame. Fensalir owns rolling-slot
validity and clamps TubeField dispatch so invalid older slots are not sampled
after renderer allocation, reset, or partial update.
`MIMIR_SPECTRUM_TUBE_SUBDIVISIONS` is the runtime cost lever for lowering
Catmull-Rom subdivision count when Fensalir reports TubeField truncation.
## Known Risks

View file

@ -17,8 +17,8 @@ Get-Content .\state\evidence.jsonl -Tail 8
## Current Shape
- Public brand/Face is Mimir.
- Mimir's Face doctrine lives in [[docs/mimir-face|Mimir Face]].
- Public brand/Persona is Mimir.
- Mimir's Persona doctrine lives in [[docs/mimir-face|Mimir Persona]].
- VoidBot persona/state live under `.voidbot/voice/` and `.voidbot/state/`.
Commit `.voidbot` dirt in a separate state commit whenever it appears.
- V1 still has FFmpeg/SRT/OBS bridge utilities for LAN ingest.
@ -43,13 +43,199 @@ Get-Content .\state\evidence.jsonl -Tail 8
- `native/reservoir` owns the lower shared-edge typed-handle invariant for
Fensalir/Faust binding work.
- Fensalir owns production GPU fusion, UI, and Spout2 publication.
- `MimirSceneEditorState` is currently a 2D OBS-shaped program mixer owner, not
a camera/3D scene editor. Synced video buffers become program layers with
normalized placement; the Eve/CultUI surface is Video Sources on the left, a
chrome-free program canvas in the center, Audio Mixer docked underneath that
canvas, and a Unity-style Inspector on the right for the selected layer's
properties. The center preview requests Fensalir's current output-buffer blit
as its background and overlays Mimir's normalized layer placements on top.
Placement commits snap near frame edges/center and preserve source aspect
while resizing through direct preview handles, sliders, or the scale gizmo.
Explicit source/transform controls are inspector-only and only visible when a
program source is selected.
Hover/active handle state, snap guides, empty-canvas deselect, keyboard
nudging, PageUp/PageDown layer ordering, and selected-source
contain/cover/stretch fit mode are wired through `MimirSceneEditorState`.
Camera/SDF/model editor affordances have no live UI authority, and editor
camera/SDF/spline output no longer contributes to the runtime frame. Smoke
with `--scene-editor-smoke`.
- The Audio Mixer dock is a horizontal CultUI channel-strip bank, not a
selected-stem inspector form. Six fixed slots read the live
`presentationControls.AudioFeeds` order so Scarlett ASIO mono inputs,
stereo loopback, and Raven loopback can appear as parallel strips with M/S,
gain, and compact sync/chirp-response readouts derived from
`MimirSynchronizationHub.AudioSynchronizationStates`. Do not move sample
alignment, chirp response mapping, or ambient/noise neutralization authority
into the UI; the surface only controls and observes runtime audio state.
- Program surface placement is seeded from CultCache config
`config/mimir-program-surface.cc` unless `--cache`,
`AQUARIUM_CULTCACHE_PATH`, or `MIMIR_PROGRAM_SURFACE_CONFIG` points
elsewhere. The default `mimir.program.default` document puts
`raven-display` full-frame at layer 0 and `kiyo-pro-rgb` as a top-left
picture-in-picture at layer 1. `MimirSceneEditorState` applies matching
source placements once when those feeds appear, then inspector/drag edits own
subsequent placement.
- The left Video Sources rail is a direct source selector, not a previous/next
navigator. It renders visible rows for live video feeds using CultUI
row-level visibility, with a Select action and placement summary per source.
Keep configured/offline source inventory as a future source-state owner;
the current rail lists live synced video feeds.
- Live camera image buffers should use Fensalir-owned texture leases when they
are rendering inputs: Mimir asks the engine broker for a keyed D3D12
Texture2D/fence lease, writes decoded frames into that texture, commits the
producer fence value, and lowers the same resource key through FieldEvidence.
`MimirVideoCaptureDriverSource` forwards that lease client to direct drivers
that implement `IMimirFensalirTextureLeaseReceiver`.
Camera backends must use the closest-to-device path available and report
unavoidable copies; process or managed convenience layers are diagnostics.
Raw single-plane CPU-origin frames can use the explicit broker upload lane,
which increments `UnavoidableCopyCount` and clears live managed payload bytes
after upload. NV12 CPU upload is rejected until Fensalir owns planar copy;
device/GPU NV12 producers should write the leased texture directly.
The first direct KS/UVC driver is `MimirKsVideoCaptureDriver` backed by
`native/camera_capture/mimir_camera_capture.dll`; the example config
`config/mimir-runtime.ks-camera.example.json` targets LeapUVC 640x240 YUY2.
The same native folder also has `mimir_ps3eye_capture.dll` and
`MimirPs3EyeVideoCaptureDriver` for PS3 Eye Bayer8 frames through WinUSB/libusb.
Fresh driver smokes pulled and uploaded real frames for LeapUVC 640x240 YUY2,
both PS3 Eyes at 320x240 Bayer8, Kiyo Pro 1920x1080 YUY2, and regular Kiyo
640x480 YUY2. Regular Kiyo 1280x720 YUY2 did not open.
Compressed Kiyo Pro MJPG/H264 now have a first GPU-resident path:
`MimirMediaFoundationGpuVideoCaptureDriver` uses
`native/camera_capture/mimir_mf_gpu_capture.dll` to decode through Media
Foundation with a D3D11 device manager, copy into a shared BGRA D3D11 texture,
and publish `shared-d3d11-texture` with zero live CPU payload bytes.
1920x1080 MJPG->RGB32 and H264->RGB32 smokes passed. Direct shared NV12
texture creation failed, so NV12 remains future plane-aware interop or
GPU bridge-copy work.
Foreign shared texture handles are import edges, not the primary hot path.
- The current Mimir/Fensalir rendering teardown map is
[[docs/fensalir-rendering-rebuild-migration|Fensalir Rendering Rebuild
Migration]]. Mimir's production visual output path must publish typed
physical observations, calibration constraints, and surface intent; Fensalir
turns those into field claims, selected lowerings, reusable evidence,
temporal guide lanes, and presentation.
- Fast depth provenance now lives at
[[research/d3d12-stereo-depth-provenance-2026-05-29|D3D12 Stereo Depth
Provenance]]. `libSGM` is the current permissive north-star reference for a
future Fensalir-owned D3D12 stereo SGM compute lane. Do not import CUDA or
TensorRT as live depth authority.
The first code contract is in `MimirStereoDepthConfigurations` and
`MimirFensalirFieldLowering`. Run
`dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --stereo-depth-contract-smoke`
to verify caller-declared rectified left/right input textures, or
`dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --leap-packed-stereo-depth-smoke`
to verify live `LeapStereoIr` packed R8G8 lowering. The Leap path reuses the
same declared texture as left/right resource identity and treats R/G as the
packed observation lanes. Fensalir now has a first packed-Leap D3D12
SAD/block-match disparity writer, not full libSGM-style SGM or calibrated
metric depth. Mimir also now declares the first Leap point-cloud root:
`MimirPointCloudConfigurations.LeapDisparityPointCloudRoot` and
`BuildLeapPackedStereoPointCloudCandidateFrame` emit a GPU-resident
`FieldMesh` point-list resource derived from that disparity SurfacePage. Run
`dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --leap-point-cloud-root-smoke`
to verify the combined `SurfacePage` plus `Mesh` packet plan. This is a
typed socket on the Mimir side. Fensalir branch `codex/leap-packed-depth` now
has the first matching render lane: generated point-list Mesh buffers are
filled from the disparity SurfacePage on D3D12 and rendered through the scene
candidate targets. The next cut is live Leap/editor verification plus
calibrated projection constants and global residual/calibration ownership.
- Fensalir also owns the EVE-facing dashboard pixels through
`Global\MimirFensalirProgramTexture`. `src/Mimir.EveRelay` opens that shared
texture, encodes H.264 Annex-B with NVENC by default, and serves EveCanvas
over `/stream`; EveCanvas decodes with `AVSampleBufferDisplayLayer`. The
launch script uses an SSH reverse tunnel to Eve when Windows firewall blocks
direct inbound TCP.
- `src/Mimir.Broadcast` is the first self-hosted public sender. Starfire pushes
one final H.264/AAC NVENC RTMP stream to
`rtmp://127.0.0.1:11935/live/mimir` through an SSH local forward to
Yggdrasil. Yggdrasil's nginx RTMP/HLS origin is deployed and has produced
HLS segments from a bounded synthetic push. Public HLS is served through the
existing StreamPixels TLS host, not a new Mimir media hostname. The public
static viewer belongs to the root GameCult site at
`https://gamecult.org/livestream` and reads
`https://streampixels.gamecult.org/mimir/live/hls/mimir.m3u8`.
- Raven/Eve ingest now has a network-pixel receiver config:
`config/mimir-runtime.raven-eve.example.json` starts an FFmpeg-backed
raw-video listener for `raven-display` on SRT port `5200` and Eve sensor
frame-event receivers for `eve-camera` on WebSocket port `8793` and `eve-mic`
on port `8794`. `scripts/start-raven-screen-capture-sender.ps1` launches
Raven desktop capture with `gdigrab` plus NVENC. EveCanvas owns Eve camera/mic
capture directly through AVFoundation/AVAudioEngine; no external iOS camera
app owns that path.
- `src/Mimir.RavenDaemon` is the Raven-local capture mux owner for the
stream-proof path. It supervises display capture plus Realtek render loopback
through the existing FFmpeg/WASAPI pipeline, publishes
`mimir.raven_capture_mux_state.v1` through CultMesh/CultCache, and exposes
Eve dashboard state on port `8801`. `scripts/start-raven-daemon.ps1` defaults
to Starfire over WireGuard at `10.77.0.2:5200`. The older Raven sender scripts
remain diagnostics, not canonical long-running status owners.
- Faust/native DSP owns hot audio alignment, separation, spatialization, and
synchronized stems.
- The current listenable dual-mic receipt is the Faust-chain bridge, not the
older C# composite. `scripts/record-kiyo-dual-mic-composite.ps1` records Kiyo
plus Scarlett ASIO, then `--asio-dual-mic-faust-cleaner-wav` compiles/runs
`faust/mimir_alignment_actuator.dsp` before
`faust/mimir_dual_mic_dialogue_cleaner.dsp` and muxes the dialogue stem into
`artifacts/runtime/kiyo-dual-mic-composite/kiyo-dual-mic-composite-latest.mp4`.
Fresh run `20260601-051044` measured cardioid delay 359.398 samples
(7487.452 us) at 48 kHz and applied holdback 359.398/0.000 to
shotgun/cardioid before cleanup.
- The edge-to-edge Mimir editor shell now uses headerless retained dashboard
panels. The inspector expands vertically; the right rail `Output` panel is a
fixed-height OBS-like control block below it. Eve is the portable dashboard
authority; the C# CultUI-shaped Aquarium contract is only the reference API
shape, and Fensalir Direct2D/D3D12 are native lowerings of the same document.
Elements can now carry typed CultCache/CultMesh bindings with document
schema/id, field path, value kind, access, authority, and optional command id;
native callbacks remain local lowering actuators, not portable state owners.
`MimirProgramRecorder` owns `Record` / `Stop`, status text, and timestamped
receipts under `C:\Users\Meta\Videos\Mimir` by default
(`MIMIR_RECORD_OUTPUT_DIR` overrides). It delegates to
`scripts/record-stream-proof-output.ps1`, so the app records the same Raven +
Kiyo PIP + Starfire Scarlett NVENC/AAC composite intended for
Yggdrasil/public judgment. `MIMIR_RECORD_START_RAVEN=1` asks it to launch the
Raven sender too. `MimirProgramStreamer` owns `Live` / `Stop` for
`Mimir.Broadcast`; it requires `MIMIR_STREAM_INPUT` unless
`MIMIR_STREAM_ALLOW_TEST_PATTERN=1` explicitly allows a synthetic smoke.
Native program-output recording/streaming should replace those actuators
behind the same owner.
- OBS receives final program surfaces; it does not own synchronization.
- The online Well stack exists. `Mimir.EveSensorReceiver` has a subscriber
WebSocket path beside its observation ingest path, and `Mimir.VerseRecorder`
writes subscribed observations to
`C:\Users\Meta\Videos\Mimir\VerseCaptures\<session>\session.json` plus
`observations.jsonl`. Recorder now also recognizes
`mimir.well_capture_page.v1`, strips inline base64 sample bodies into
`bodies/page-*.bin`, and records `mimir.recorder_body_index.v1` refs in
`bodies/index.jsonl` so offline reconstruction tools can resolve payloads
without treating JSONL as the body store. `Mimir.Well` loads
`config/mimir-runtime.well.local.json`, starts every configured source it
can, polls `MimirSynchronizationHub` with a bounded per-source budget, updates
audio sync/probe state, and publishes `mimir.well_snapshot.v1` records with
source buffers, composite controls, publication/stem contract, and sync
status. It also publishes bounded `mimir.well_capture_page.v1` records for
synchronized frame slices with timing metadata, configured composite state,
and inline source bodies when CPU bytes exist and fit the configured body
limit. GPU-handle-only samples remain metadata/body-handle evidence until
Fensalir/native program-output recording owns rendered composite bodies.
`scripts/start-online-verse-daemons.ps1` supervises the relay,
recorder, Nightwing typed Eyes/Move/builtin-camera/builtin-mic witness, the
music-synced de-Bruijn/microtonal Move gesture worker, and `Mimir.Well`.
Current live proof `online-verse-daemons-20260601-212502` wrote Well,
Nightwing builtin camera/mic, and Eye/Move records. Raven and Eve lanes are
configured but empty unless their producers are feeding the ports. Sync frame
status is degraded until a camera-clock/global-time correction owner maps
ASIO device time and camera driver time into one canonical timeline.
## Current Pressure
- Implement the first concrete Leap direct capture driver and measure cadence.
- Verify the live Leap point cloud in the Mimir editor, then add calibrated
projection constants and the global residual/calibration owner.
- Cut the Phase 1 Mimir-to-Fensalir bridge DTOs for rolling windows,
observations, calibration constraints, and surface intent before adding
another backend-specific visual path.
- Add remaining camera drivers through `IMimirVideoCaptureDriver`.
- Add native audio capture workers for mic, loopback, and network audio feeds.
- Turn the current loopback-referenced chirplet measurement into the real

View file

@ -11,11 +11,18 @@ Use it to shape native implementation decisions:
- direct camera and audio drivers;
- pinned/native buffers;
- GPU feature extraction, flow, matching, and fusion;
- GPU feature extraction, flow, matching, depth, and fusion;
- Fensalir D3D12 compute/render paths;
- Faust/native DSP;
- OBS-facing program outputs.
Visual/GPU research entry points:
- [[d3d12-stereo-depth-provenance-2026-05-29|D3D12 Stereo Depth Provenance]]:
libSGM is the current permissive north-star reference for a future
Fensalir-owned D3D12 compute stereo-disparity lane. External CUDA/TensorRT
projects are provenance only, not runtime authority.
Audio timing research entry points:
- [[chirplet-sync-decoder/summary|Chirplet Sync Decoder Summary]]: chirplet

View file

@ -0,0 +1,170 @@
# D3D12 Stereo Depth Provenance
Date: 2026-05-29
## Purpose
This note records the current fast GPU depth-estimation search as provenance for
a future Mimir/Fensalir D3D12 compute implementation.
The north star is `libSGM`: not as a dependency, not as a CUDA bridge, and not
as a permission slip to add an adapter. The useful shape is a fast,
deterministic stereo disparity pipeline whose ownership can be rebuilt inside
Fensalir's D3D12 resource model.
## Authority Map
Owner:
- Fensalir owns the live D3D12 compute kernels, GPU resources, fences,
scheduling, and depth/surface claim lowering.
Inputs:
- Rectified synchronized camera texture pairs from Mimir-declared
`Texture2D` resources.
- Camera calibration: intrinsics, extrinsics, baseline, rectification map, and
valid-time/version metadata.
- Rolling-window timing chosen by `MimirSynchronizedBufferPlanner`.
Outputs:
- GPU-resident disparity/depth resources, preferably fixed format and size per
calibrated camera pair.
- FieldEvidence claims for depth support, confidence, invalid/occluded pixels,
and later surface/material lowerings.
- Diagnostics: kernel timings, disparity range, confidence histogram, invalid
ratio, calibration version, and source texture fence values.
Derived State:
- Depth is derived from synchronized stereo textures plus calibration.
- Per-frame disparity confidence is derived evidence, not a clock or camera
authority.
- Monocular model depth remains optional reference/assistive evidence unless a
later invariant gives it a named role.
Forbidden Writers:
- CUDA runtime code must not become the live path.
- TensorRT/ONNX monocular demos must not own visual geometry.
- CPU image copies/readback must not be inserted for convenience.
- OBS, EVE, or bridge utilities must not decide depth or synchronization.
Shared Paths:
- Direct user capture, replayed rolling-window frames, future Raven/phone
camera feeds, and calibration-session frames must all enter through typed
resources and use the same depth compute/claim path once they are calibrated
stereo pairs.
Deletion Line:
- If a prototype requires persistent CPU depth images, a side transport daemon,
or a CUDA compatibility layer, it is research-only and does not graduate into
the live machine.
## Reference Shape: libSGM
Repository: https://github.com/fixstars/libSGM
License: Apache-2.0.
Why it matters:
- It is a CUDA implementation of Semi-Global Matching for stereo disparity.
- It was built by the Fixstars/adaskit ADAS effort, so it is pointed at the
same autonomous-driving pressure field rather than generic image demos.
- Reported benchmark in the upstream README: `1024 x 440`, disparity `128`,
four SGM paths, subpixel enabled:
- GTX 1080 Ti: `2.0 ms` / `495.1 FPS`
- RTX 3080: `1.5 ms` / `651.3 FPS`
- Tegra X2: `28.5 ms` / `35.1 FPS`
- Xavier MAXN: `9.0 ms` / `110.7 FPS`
Port lesson:
- Use the pipeline and data movement as a model: matching cost, path
aggregation, winner/subpixel selection, confidence/invalid handling.
- Rebuild the kernels in HLSL/D3D12 compute against Fensalir-owned resources.
- Preserve the ADAS-style assumption that calibrated stereo gives metric depth
when baseline and rectification are real.
## Secondary References
Fast-ACVNet: https://github.com/gangweiX/Fast-ACVNet
- License: MIT.
- Learned real-time stereo matching; useful for quality and failure-mode
comparison.
- Upstream reports `45 ms` for Fast-ACVNet+ in its KITTI/Scene Flow table.
- Not the first Mimir implementation target because it brings model training,
weights, and framework deployment complexity.
UniMatch / GMStereo: https://github.com/autonomousvision/unimatch
- License: MIT.
- Unified flow/stereo/depth research code.
- Useful as a modern learned-stereo comparison point and possible later
confidence/feature guide, not a kernel blueprint.
Depth Anything V2 Small: https://github.com/DepthAnything/Depth-Anything-V2
- License caveat: V2 Small is Apache-2.0; Base/Large/Giant are CC-BY-NC-4.0.
- Excellent monocular relative-depth reference, especially through TensorRT
ports, but monocular output is not metric scene truth by itself.
Depth Anything TensorRT CLI:
https://github.com/spacewalk01/depth-anything-tensorrt
- License: MIT.
- Reports fast RTX 4090 inference for Depth-Anything-S/B/L.
- Useful as performance context only. It must not displace calibrated stereo
geometry or Fensalir's D3D12 ownership.
Fast-FoundationStereo:
https://github.com/NVlabs/Fast-FoundationStereo
- Important research reference for real-time zero-shot stereo.
- License is research/non-commercial only, so it is not a permissive live
dependency candidate.
## D3D12 Compute Implementation Sketch
First implementation should stay boring and inspectable:
1. Rectify or require pre-rectified stereo pairs.
2. Convert sampled luma/feature texture into a compact matching domain.
3. Build per-pixel disparity matching costs over a fixed disparity range.
4. Aggregate along a small fixed path set, starting with four directions to
match the `libSGM` reference benchmark shape.
5. Select winner disparity and subpixel refinement.
6. Emit disparity, confidence, and invalid/occlusion masks as GPU resources.
7. Lower those resources into FieldEvidence depth/support claims.
Important D3D12 resource notes:
- Prefer persistent UAV/SRV resources sized by camera pair profile.
- Use explicit producer/consumer fences already established for camera texture
leases and program-output publication.
- Keep disparity range and path count as profile-level cost levers, not
per-frame mode flags scattered through the renderer.
- Do not read back depth except for gated diagnostics and receipts.
## Initial Verification Targets
- Synthetic rectified stereo pair with known constant disparity.
- Slanted plane with known disparity gradient.
- Occlusion edge where invalid/confidence behavior is visible.
- Real Leap stereo IR pair once calibration is pinned.
- Full rolling-window path: camera textures -> depth compute -> FieldEvidence
claim -> Fensalir debug overlay/resource receipt.
Negative checks:
- No CUDA runtime loaded.
- No CPU disparity image in the live path.
- No monocular model output accepted as metric depth without a named scale
authority.
- No untyped texture handle accepted without resource declaration and fence
provenance.

19
run-spectrum-preview.cmd Normal file
View file

@ -0,0 +1,19 @@
@echo off
setlocal
set "ROOT=%~dp0"
set "APP=%ROOT%src\Mimir.App\bin\Debug\net10.0\Mimir.App.exe"
if not exist "%APP%" (
echo Mimir.App.exe was not found.
echo Build it first with:
echo dotnet build "%ROOT%src\Mimir.App\Mimir.App.csproj"
pause
exit /b 1
)
set "MIMIR_SYNTHETIC_SPECTRUM_PREVIEW=1"
set "MIMIR_SPECTRUM_INTERVAL_SECONDS=0.016"
set "MIMIR_SPECTRUM_SOURCE_LANES=8"
start "Mimir Spectrum Preview" "%APP%"

View file

@ -0,0 +1,35 @@
param(
[string]$Configuration = "Release",
[string]$TemplateVersion = "master",
[switch]$SkipTests
)
$ErrorActionPreference = "Stop"
$root = Split-Path -Parent $PSScriptRoot
$sdkRoot = Join-Path $root "artifacts\obs-sdk"
$templateRoot = Join-Path $sdkRoot "obs-plugintemplate"
$depsRoot = Join-Path $templateRoot ".deps"
$buildRoot = Join-Path $templateRoot "build_x64_local"
$pluginBuildRoot = Join-Path $root "native\obs_stem_source\build"
New-Item -ItemType Directory -Force $sdkRoot | Out-Null
if (-not (Test-Path $templateRoot)) {
git clone --depth 1 --branch $TemplateVersion https://github.com/obsproject/obs-plugintemplate.git $templateRoot
}
if (-not (Test-Path (Join-Path $depsRoot "cmake\libobsConfig.cmake"))) {
cmake -S $templateRoot -B $buildRoot -G "Visual Studio 17 2022" -A x64 -DENABLE_FRONTEND_API=OFF -DENABLE_QT=OFF
}
cmake `
-S (Join-Path $root "native\obs_stem_source") `
-B $pluginBuildRoot `
-Dlibobs_DIR="$(Join-Path $depsRoot 'cmake')" `
-DCMAKE_PREFIX_PATH="$depsRoot"
cmake --build $pluginBuildRoot --config $Configuration --parallel
if (-not $SkipTests) {
ctest --test-dir $pluginBuildRoot -C $Configuration --output-on-failure
}

View file

@ -0,0 +1,71 @@
param(
[string]$ObsRoot = "",
[string]$PluginRoot = "",
[string]$Configuration = "Release",
[switch]$SkipBuild,
[switch]$WhatIf
)
$ErrorActionPreference = "Stop"
$repoRoot = Split-Path -Parent $PSScriptRoot
$pluginDll = Join-Path $repoRoot "native\obs_stem_source\build\$Configuration\mimir_obs_stem_source.dll"
$localeSourceDir = Join-Path $repoRoot "native\obs_stem_source\data\locale"
if (-not [string]::IsNullOrWhiteSpace($ObsRoot)) {
$targetDir = Join-Path $ObsRoot "obs-plugins\64bit"
$targetLocaleDir = Join-Path $ObsRoot "data\obs-plugins\mimir_obs_stem_source\locale"
} else {
if ([string]::IsNullOrWhiteSpace($PluginRoot)) {
if ($env:OBS_PLUGIN_ROOT) {
$PluginRoot = $env:OBS_PLUGIN_ROOT
} else {
$PluginRoot = Join-Path $env:ALLUSERSPROFILE "obs-studio\plugins"
}
}
$targetDir = Join-Path $PluginRoot "mimir_obs_stem_source\bin\64bit"
$targetLocaleDir = Join-Path $PluginRoot "mimir_obs_stem_source\data\locale"
}
$targetDll = Join-Path $targetDir "mimir_obs_stem_source.dll"
if (-not $SkipBuild) {
& (Join-Path $repoRoot "scripts\build-obs-stem-plugin.ps1") -Configuration $Configuration
}
if (-not (Test-Path $pluginDll)) {
throw "Built plugin DLL was not found at '$pluginDll'."
}
if (-not (Test-Path $localeSourceDir)) {
throw "Plugin locale directory was not found at '$localeSourceDir'."
}
if (-not (Test-Path $targetDir)) {
if ($WhatIf) {
Write-Host "Would create $targetDir"
} else {
New-Item -ItemType Directory -Force $targetDir | Out-Null
}
}
if (-not (Test-Path $targetLocaleDir)) {
if ($WhatIf) {
Write-Host "Would create $targetLocaleDir"
} else {
New-Item -ItemType Directory -Force $targetLocaleDir | Out-Null
}
}
if ($WhatIf) {
Write-Host "Would copy $pluginDll -> $targetDll"
Get-ChildItem -LiteralPath $localeSourceDir -File | ForEach-Object {
Write-Host "Would copy $($_.FullName) -> $(Join-Path $targetLocaleDir $_.Name)"
}
return
}
Copy-Item -LiteralPath $pluginDll -Destination $targetDll -Force
Copy-Item -Path (Join-Path $localeSourceDir "*") -Destination $targetLocaleDir -Force
Write-Host "Installed Mimir OBS plugin: $targetDll"
Write-Host "OBS source types: Mimir Program Texture, Mimir Audio Stem, Muninn Stream, Muninn Video, Muninn Audio"

41
scripts/raven-audio.ps1 Normal file
View file

@ -0,0 +1,41 @@
param(
[string]$TargetHost = "192.168.1.66",
[int]$Port = 5202,
[int]$SampleRate = 48000,
[int]$Channels = 2,
[string]$FfmpegPath = "ffmpeg",
[string]$LogRoot = "C:\Meta\Mimir\logs",
[switch]$DryRun
)
$ErrorActionPreference = "Stop"
New-Item -ItemType Directory -Force -Path $LogRoot | Out-Null
$repo = Split-Path -Parent $PSScriptRoot
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
$stdoutLog = Join-Path $LogRoot "raven-realtk-loopback-$timestamp.out.log"
$stderrLog = Join-Path $LogRoot "raven-realtk-loopback-$timestamp.err.log"
$endpoint = "srt://${TargetHost}:${Port}?mode=caller&latency=120000"
$captureScript = Join-Path $repo "scripts\wasapi-loopback-capture.ps1"
$command = "& `"$captureScript`" -Output - -SampleRate $SampleRate -Channels $Channels | & `"$FfmpegPath`" -hide_banner -nostdin -loglevel warning -f f32le -ar $SampleRate -ac $Channels -i pipe:0 -f f32le `"$endpoint`""
Write-Host "Raven Realtek loopback sender:"
Write-Host " target: $endpoint"
Write-Host " format: f32le ${Channels}ch ${SampleRate}Hz"
Write-Host " stdout: $stdoutLog"
Write-Host " stderr: $stderrLog"
if ($DryRun) {
Write-Host $command
exit 0
}
Start-Process `
-FilePath "powershell" `
-ArgumentList @("-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", $command) `
-WindowStyle Hidden `
-RedirectStandardOutput $stdoutLog `
-RedirectStandardError $stderrLog `
-PassThru |
ForEach-Object { Write-Host "Started Raven Realtek loopback sender pid=$($_.Id)" }

View file

@ -0,0 +1,158 @@
param(
[int]$DurationSeconds = 15,
[string]$OutputPath = "artifacts\runtime\kiyo-dual-mic-composite\kiyo-dual-mic-composite.mp4",
[string]$LogRoot = "artifacts\runtime\kiyo-dual-mic-composite",
[string]$FfmpegPath = "ffmpeg",
[string]$AsioProbePath = "native\probes\asio_audio_cadence\build\Release\asio_audio_cadence.exe",
[string]$KiyoVideoDevice = "USB Video Device",
[int]$KiyoWidth = 640,
[int]$KiyoHeight = 480,
[int]$KiyoFrameRate = 30,
[int]$SampleRate = 48000,
[int]$AsioChannels = 4,
[int]$ShotgunChannel = 0,
[int]$CardioidChannel = 1,
[switch]$DryRun
)
$ErrorActionPreference = "Stop"
function Quote-ProcessArgument([string]$Value) {
if ($Value.Length -eq 0) {
return '""'
}
if ($Value -notmatch '[\s"]') {
return $Value
}
return '"' + $Value.Replace('"', '\"') + '"'
}
function Start-LoggedProcess([string]$FileName, [string[]]$Arguments, [string]$StdoutPath, [string]$StderrPath) {
$startInfo = [System.Diagnostics.ProcessStartInfo]::new()
$startInfo.FileName = $FileName
$startInfo.UseShellExecute = $false
$startInfo.RedirectStandardOutput = $true
$startInfo.RedirectStandardError = $true
$startInfo.Arguments = ($Arguments | ForEach-Object { Quote-ProcessArgument $_ }) -join " "
$process = [System.Diagnostics.Process]::Start($startInfo)
if (-not $process) {
throw "Could not start $FileName"
}
$stdoutStream = [System.IO.File]::Open($StdoutPath, [System.IO.FileMode]::Create, [System.IO.FileAccess]::Write, [System.IO.FileShare]::Read)
$stderrStream = [System.IO.File]::Open($StderrPath, [System.IO.FileMode]::Create, [System.IO.FileAccess]::Write, [System.IO.FileShare]::Read)
return [pscustomobject]@{
Process = $process
StdoutTask = $process.StandardOutput.BaseStream.CopyToAsync($stdoutStream)
StderrTask = $process.StandardError.BaseStream.CopyToAsync($stderrStream)
StdoutStream = $stdoutStream
StderrStream = $stderrStream
}
}
function Wait-LoggedProcess($Handle, [int]$TimeoutSeconds, [string]$Name) {
if (-not $Handle.Process.WaitForExit($TimeoutSeconds * 1000)) {
try { $Handle.Process.Kill($true) } catch {}
throw "$Name timed out after ${TimeoutSeconds}s."
}
[void]$Handle.StdoutTask.GetAwaiter().GetResult()
[void]$Handle.StderrTask.GetAwaiter().GetResult()
$Handle.StdoutStream.Dispose()
$Handle.StderrStream.Dispose()
if ($Handle.Process.ExitCode -ne 0) {
throw "$Name failed with exit code $($Handle.Process.ExitCode)."
}
}
$repo = Split-Path -Parent $PSScriptRoot
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
$absoluteLogRoot = if ([System.IO.Path]::IsPathRooted($LogRoot)) { $LogRoot } else { Join-Path $repo $LogRoot }
$runDirectory = Join-Path $absoluteLogRoot $timestamp
New-Item -ItemType Directory -Force -Path $runDirectory | Out-Null
$absoluteOutputPath = if ([System.IO.Path]::IsPathRooted($OutputPath)) { $OutputPath } else { Join-Path $repo $OutputPath }
$outputDirectory = Split-Path -Parent $absoluteOutputPath
New-Item -ItemType Directory -Force -Path $outputDirectory | Out-Null
$asioRaw = Join-Path $runDirectory "scarlett-f32-interleaved.raw"
$kiyoMkv = Join-Path $runDirectory "kiyo-video.mkv"
$compositeWav = Join-Path $runDirectory "dual-mic-composite.wav"
$absoluteAsioProbe = if ([System.IO.Path]::IsPathRooted($AsioProbePath)) { $AsioProbePath } else { Join-Path $repo $AsioProbePath }
$asioArgs = @(
"--set-sample-rate", $SampleRate.ToString(),
"--record-f32-interleaved", $asioRaw,
"--capture-seconds", $DurationSeconds.ToString()
)
$kiyoArgs = @(
"-hide_banner",
"-y",
"-thread_queue_size", "1024",
"-rtbufsize", "256M",
"-f", "dshow",
"-video_size", "${KiyoWidth}x${KiyoHeight}",
"-framerate", $KiyoFrameRate.ToString(),
"-i", "video=$KiyoVideoDevice",
"-t", $DurationSeconds.ToString(),
"-r", $KiyoFrameRate.ToString(),
"-c:v", "h264_nvenc",
"-preset", "p4",
"-tune", "ll",
"-pix_fmt", "yuv420p",
$kiyoMkv
)
Write-Host "Kiyo + computed dual-mic composite recording:"
Write-Host " Kiyo: $KiyoVideoDevice ${KiyoWidth}x${KiyoHeight}@$KiyoFrameRate"
Write-Host " ASIO: $absoluteAsioProbe sampleRate=$SampleRate channels=$AsioChannels shotgun=$ShotgunChannel cardioid=$CardioidChannel"
Write-Host " Run dir: $runDirectory"
Write-Host " Output: $absoluteOutputPath"
if ($DryRun) {
Write-Host ($absoluteAsioProbe + " " + ($asioArgs -join " "))
Write-Host ($FfmpegPath + " " + ($kiyoArgs -join " "))
exit 0
}
$asio = Start-LoggedProcess $absoluteAsioProbe $asioArgs (Join-Path $runDirectory "asio.out.log") (Join-Path $runDirectory "asio.err.log")
$kiyo = Start-LoggedProcess $FfmpegPath $kiyoArgs (Join-Path $runDirectory "kiyo.out.log") (Join-Path $runDirectory "kiyo.err.log")
Wait-LoggedProcess $asio ($DurationSeconds + 15) "ASIO capture"
Wait-LoggedProcess $kiyo ($DurationSeconds + 15) "Kiyo capture"
dotnet run --project (Join-Path $repo "src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj") -- `
--asio-dual-mic-faust-cleaner-wav `
--input $asioRaw `
--output $compositeWav `
--sample-rate $SampleRate `
--channels $AsioChannels `
--shotgun-channel $ShotgunChannel `
--cardioid-channel $CardioidChannel
if ($LASTEXITCODE -ne 0) {
throw "Dual-mic composite WAV generation failed."
}
$muxArgs = @(
"-hide_banner",
"-y",
"-i", $kiyoMkv,
"-i", $compositeWav,
"-map", "0:v:0",
"-map", "1:a:0",
"-c:v", "copy",
"-c:a", "aac",
"-b:a", "192k",
"-ar", "48000",
"-movflags", "+faststart",
$absoluteOutputPath
)
$mux = Start-LoggedProcess $FfmpegPath $muxArgs (Join-Path $runDirectory "mux.out.log") (Join-Path $runDirectory "mux.err.log")
Wait-LoggedProcess $mux 60 "Mux"
Write-Host "Recording complete:"
Write-Host " $absoluteOutputPath"
Write-Host " composite wav: $compositeWav"
Write-Host " report: $([System.IO.Path]::ChangeExtension($compositeWav, '.json'))"

View file

@ -0,0 +1,201 @@
param(
[int]$RavenPort = 5204,
[int]$DurationSeconds = 20,
[int]$StartupTimeoutSeconds = 20,
[int]$Width = 1280,
[int]$Height = 720,
[int]$FrameRate = 30,
[string]$OutputPath = "artifacts\runtime\stream-proof\stream-proof-output.mp4",
[string]$LogRoot = "artifacts\runtime\stream-proof",
[string]$FfmpegPath = "ffmpeg",
[switch]$StartRavenSender,
[string]$RavenHost = "madman's lullaby@192.168.1.84",
[string]$RavenAddress = "192.168.1.84",
[string]$RavenRepo = "C:\Meta\Mimir",
[string]$StarfireHost = "192.168.1.66",
[switch]$RavenSenderListens,
[ValidateSet("srt", "tcp-listener")]
[string]$RavenSenderTransport = "srt",
[string]$RavenInputOverride = "",
[string]$KiyoVideoDevice = "USB Video Device",
[string]$StarfireScarlettAudioDevice = "Analogue 1 + 2 (2- Focusrite USB Audio)",
[int]$KiyoWidth = 640,
[int]$KiyoHeight = 480,
[int]$KiyoFrameRate = 30,
[int]$KiyoCornerWidth = 320,
[int]$KiyoMargin = 28,
[double]$RavenOffsetSeconds = 0.0,
[double]$KiyoOffsetSeconds = 0.0,
[double]$MicOffsetSeconds = 0.0,
[switch]$NoStarfireScarlett,
[switch]$DryRun
)
$ErrorActionPreference = "Stop"
function Quote-ProcessArgument([string]$Value) {
if ($Value.Length -eq 0) {
return '""'
}
if ($Value -notmatch '[\s"]') {
return $Value
}
return '"' + $Value.Replace('"', '\"') + '"'
}
$repo = Split-Path -Parent $PSScriptRoot
$absoluteOutputPath = if ([System.IO.Path]::IsPathRooted($OutputPath)) {
$OutputPath
} else {
Join-Path $repo $OutputPath
}
$outputDirectory = Split-Path -Parent $absoluteOutputPath
New-Item -ItemType Directory -Force -Path $outputDirectory | Out-Null
$absoluteLogRoot = if ([System.IO.Path]::IsPathRooted($LogRoot)) {
$LogRoot
} else {
Join-Path $repo $LogRoot
}
New-Item -ItemType Directory -Force -Path $absoluteLogRoot | Out-Null
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
$stdoutLog = Join-Path $absoluteLogRoot "stream-proof-record-$timestamp.out.log"
$stderrLog = Join-Path $absoluteLogRoot "stream-proof-record-$timestamp.err.log"
$ravenEndpoint = if ($RavenInputOverride.Length -gt 0) {
$RavenInputOverride
} elseif ($RavenSenderListens) {
"srt://${RavenAddress}:${RavenPort}?mode=caller&latency=120000&timeout=30000000"
} else {
"srt://0.0.0.0:${RavenPort}?mode=listener&latency=120000&timeout=30000000"
}
$ravenSender = "${RavenRepo}\scripts\start-raven-av-sender.ps1"
if ($StartRavenSender) {
$taskName = "MimirRavenAvSender$RavenPort"
$remoteCmd = "${RavenRepo}\scripts\run-raven-av-$RavenPort.cmd"
$remoteTargetHost = if ($RavenSenderListens) { "0.0.0.0" } else { $StarfireHost }
$remoteSrtMode = if ($RavenSenderListens) { "listener" } else { "caller" }
$remoteCommand = "powershell.exe -NoProfile -ExecutionPolicy Bypass -File $ravenSender -TargetHost $remoteTargetHost -Port $RavenPort -SrtMode $remoteSrtMode -Transport $RavenSenderTransport"
$taskTime = (Get-Date).AddMinutes(1).ToString("HH:mm")
Write-Host "Starting Raven muxed A/V sender on ${RavenHost} through interactive task ${taskName}: $remoteCommand"
if (-not $DryRun) {
ssh $RavenHost "cmd /c echo $remoteCommand ^> $remoteCmd" | Out-Null
ssh $RavenHost "schtasks /Delete /TN $taskName /F" | Out-Null
ssh $RavenHost "schtasks /Create /TN $taskName /SC ONCE /ST $taskTime /TR $remoteCmd /F /IT" | Out-Host
ssh $RavenHost "schtasks /Run /TN $taskName" | Out-Host
if ($LASTEXITCODE -ne 0) {
throw "Could not start Raven muxed A/V sender task on $RavenHost."
}
Start-Sleep -Milliseconds 1500
}
}
$arguments = @(
"-hide_banner",
"-y",
"-thread_queue_size", "1024",
"-itsoffset", $RavenOffsetSeconds.ToString([Globalization.CultureInfo]::InvariantCulture),
"-i", $ravenEndpoint,
"-thread_queue_size", "1024",
"-rtbufsize", "256M",
"-itsoffset", $KiyoOffsetSeconds.ToString([Globalization.CultureInfo]::InvariantCulture),
"-f", "dshow",
"-video_size", "${KiyoWidth}x${KiyoHeight}",
"-framerate", $KiyoFrameRate.ToString(),
"-i", "video=$KiyoVideoDevice"
)
if (-not $NoStarfireScarlett) {
$arguments += @(
"-thread_queue_size", "1024",
"-rtbufsize", "256M",
"-itsoffset", $MicOffsetSeconds.ToString([Globalization.CultureInfo]::InvariantCulture),
"-f", "dshow",
"-i", "audio=$StarfireScarlettAudioDevice"
)
}
$scaledKiyoHeight = [Math]::Max(1, [int][Math]::Round($KiyoCornerWidth * ($KiyoHeight / [double]$KiyoWidth)))
$overlayX = $KiyoMargin
$overlayY = $KiyoMargin
if ($NoStarfireScarlett) {
$filter = "[0:v]scale=${Width}:${Height},setsar=1,hqdn3d=1.2:1.0:3.0:2.0[base];" +
"[1:v]scale=${KiyoCornerWidth}:${scaledKiyoHeight},setsar=1,hqdn3d=1.4:1.1:3.4:2.2[pip];" +
"[base][pip]overlay=${overlayX}:${overlayY}:format=auto[v];" +
"[0:a]aresample=async=1:first_pts=0,afftdn=nr=8:nf=-35[a]"
} else {
$filter = "[0:v]scale=${Width}:${Height},setsar=1,hqdn3d=1.2:1.0:3.0:2.0[base];" +
"[1:v]scale=${KiyoCornerWidth}:${scaledKiyoHeight},setsar=1,hqdn3d=1.4:1.1:3.4:2.2[pip];" +
"[base][pip]overlay=${overlayX}:${overlayY}:format=auto[v];" +
"[0:a]aresample=async=1:first_pts=0,afftdn=nr=8:nf=-35[ravenAudio];" +
"[2:a]aresample=async=1:first_pts=0,afftdn=nr=10:nf=-38,volume=1.0[heroMic];" +
"[ravenAudio][heroMic]amix=inputs=2:duration=longest:normalize=0[a]"
}
$arguments += @(
"-filter_complex", $filter,
"-map", "[v]",
"-map", "[a]",
"-t", $DurationSeconds.ToString(),
"-r", $FrameRate.ToString(),
"-c:v", "h264_nvenc",
"-preset", "p4",
"-tune", "ll",
"-rc", "cbr",
"-b:v", "4500k",
"-maxrate", "4500k",
"-bufsize", "9000k",
"-g", ($FrameRate * 2).ToString(),
"-pix_fmt", "yuv420p",
"-c:a", "aac",
"-b:a", "160k",
"-ar", "48000",
"-movflags", "+faststart",
$absoluteOutputPath
)
Write-Host "Stream-proof recording:"
Write-Host " Raven input: $ravenEndpoint"
Write-Host " Raven SRT mode: $(if ($RavenSenderListens) { 'Raven listener, Starfire caller' } else { 'Starfire listener, Raven caller' })"
Write-Host " Kiyo PIP: $KiyoVideoDevice ${KiyoWidth}x${KiyoHeight}@$KiyoFrameRate -> ${KiyoCornerWidth}x${scaledKiyoHeight}"
Write-Host " Raven audio: Realtek/default render loopback in muxed NVENC/SRT feed"
Write-Host " Starfire Scarlett: $(if ($NoStarfireScarlett) { 'disabled' } else { $StarfireScarlettAudioDevice })"
Write-Host " sync role: Starfire Realtek emits chirps; Raven Realtek carries co-streamer game/program audio; Starfire Scarlett captures hero mics plus Raven shotgun"
Write-Host " Offsets: raven=${RavenOffsetSeconds}s kiyo=${KiyoOffsetSeconds}s mic=${MicOffsetSeconds}s"
Write-Host " Output: $absoluteOutputPath"
Write-Host " stdout: $stdoutLog"
Write-Host " stderr: $stderrLog"
if ($DryRun) {
Write-Host ($FfmpegPath + " " + ($arguments -join " "))
exit 0
}
$startInfo = [System.Diagnostics.ProcessStartInfo]::new()
$startInfo.FileName = $FfmpegPath
$startInfo.UseShellExecute = $false
$startInfo.RedirectStandardOutput = $true
$startInfo.RedirectStandardError = $true
$startInfo.Arguments = ($arguments | ForEach-Object { Quote-ProcessArgument $_ }) -join " "
$process = [System.Diagnostics.Process]::Start($startInfo)
if (-not $process) {
throw "Could not start FFmpeg recorder."
}
$stdoutTask = $process.StandardOutput.BaseStream.CopyToAsync([System.IO.File]::Open($stdoutLog, [System.IO.FileMode]::Create, [System.IO.FileAccess]::Write, [System.IO.FileShare]::Read))
$stderrTask = $process.StandardError.BaseStream.CopyToAsync([System.IO.File]::Open($stderrLog, [System.IO.FileMode]::Create, [System.IO.FileAccess]::Write, [System.IO.FileShare]::Read))
$timeoutMs = [Math]::Max(1, $DurationSeconds + $StartupTimeoutSeconds) * 1000
if (-not $process.WaitForExit($timeoutMs)) {
try { $process.Kill($true) } catch {}
throw "FFmpeg recorder timed out after $($DurationSeconds + $StartupTimeoutSeconds)s. Check $stderrLog."
}
[void]$stdoutTask.GetAwaiter().GetResult()
[void]$stderrTask.GetAwaiter().GetResult()
if ($process.ExitCode -ne 0) {
throw "FFmpeg recorder failed with exit code $($process.ExitCode). Check $stderrLog."
}

View file

@ -0,0 +1,188 @@
param(
[string]$ObsExe = "C:\Program Files\obs-studio\bin\64bit\obs64.exe",
[string]$ObsConfigRoot = "$env:APPDATA\obs-studio",
[int[]]$ExpectedUdpPorts = @(5204, 17874),
[int]$StartupWaitSeconds = 30,
[switch]$NoStart
)
$ErrorActionPreference = "Stop"
function Stop-Obs {
$processes = Get-Process obs64,obs-browser-page,obs-ffmpeg-mux -ErrorAction SilentlyContinue
if (-not $processes) {
return
}
& "$env:SystemRoot\System32\taskkill.exe" /IM obs64.exe /IM obs-browser-page.exe /IM obs-ffmpeg-mux.exe /T /F | Write-Host
$deadline = (Get-Date).AddSeconds(15)
do {
Start-Sleep -Milliseconds 500
$remaining = Get-Process obs64,obs-browser-page,obs-ffmpeg-mux -ErrorAction SilentlyContinue
if (-not $remaining) {
return
}
} while ((Get-Date) -lt $deadline)
$remaining | ForEach-Object {
try {
Stop-Process -Id $_.Id -Force -ErrorAction Stop
} catch {
Write-Host "Failed to stop lingering $($_.ProcessName) PID $($_.Id): $($_.Exception.Message)"
}
}
Start-Sleep -Seconds 1
}
function Remove-ObsCrashSentinel {
param([string]$ConfigRoot)
$sentinel = Join-Path $ConfigRoot ".sentinel"
if (-not (Test-Path -LiteralPath $sentinel)) {
return
}
$resolvedSentinel = Resolve-Path -LiteralPath $sentinel
$resolvedConfigRoot = Resolve-Path -LiteralPath $ConfigRoot
$expected = Join-Path $resolvedConfigRoot.Path ".sentinel"
if ($resolvedSentinel.Path -ne $expected) {
throw "Refusing to remove unexpected OBS sentinel path '$($resolvedSentinel.Path)'."
}
Remove-Item -LiteralPath $resolvedSentinel.Path -Recurse -Force
Write-Host "Removed OBS crash sentinel: $($resolvedSentinel.Path)"
}
function Get-UdpListenerLines {
param([int[]]$Ports)
$patterns = $Ports | ForEach-Object { ":$_" }
netstat -ano -p udp | Where-Object {
$line = $_
$patterns | Where-Object { $line -match [regex]::Escape($_) }
}
}
function Get-ObservedUdpPorts {
param([int[]]$Ports)
$listeners = @(Get-UdpListenerLines -Ports $Ports)
$observed = @{}
foreach ($port in $Ports) {
$pattern = ":\s*$port\s+"
if ($listeners | Where-Object { $_ -match $pattern }) {
$observed[$port] = $true
}
}
[pscustomobject]@{
Lines = $listeners
Ports = $observed
}
}
function Get-LatestObsLog {
param([string]$ConfigRoot)
$logRoot = Join-Path $ConfigRoot "logs"
if (-not (Test-Path -LiteralPath $logRoot)) {
return $null
}
Get-ChildItem -LiteralPath $logRoot -Filter "*.txt" |
Sort-Object LastWriteTime -Descending |
Select-Object -First 1
}
function Assert-LatestObsLogWitness {
param(
[string]$ConfigRoot,
[int]$WaitSeconds = 30
)
$requiredPatterns = @(
"Mimir OBS sources loaded: mimir_audio_stem_source, mimir_program_texture_source, muninn_stream_source, muninn_video_source, muninn_audio_source",
"mimir_obs_stem_source.dll"
)
$deadline = (Get-Date).AddSeconds($WaitSeconds)
do {
$latestLog = Get-LatestObsLog -ConfigRoot $ConfigRoot
if ($latestLog) {
$missing = @()
foreach ($pattern in $requiredPatterns) {
if (-not (Select-String -LiteralPath $latestLog.FullName -SimpleMatch -Pattern $pattern -Quiet)) {
$missing += $pattern
}
}
if ($missing.Count -eq 0) {
Write-Host "Verified OBS log witness: $($latestLog.FullName)"
return
}
}
Start-Sleep -Seconds 1
} while ((Get-Date) -lt $deadline)
if (-not $latestLog) {
throw "OBS started, but no OBS log file was found to verify Muninn source registration."
}
Write-Host "Newest OBS log: $($latestLog.FullName) ($($latestLog.Length) bytes)"
if ($latestLog.Length -gt 0) {
Get-Content -LiteralPath $latestLog.FullName -Tail 60 | Write-Host
}
throw "OBS log is missing the expected Mimir Muninn source registration witness after waiting ${WaitSeconds}s: $($requiredPatterns -join '; ')."
}
if (-not (Test-Path -LiteralPath $ObsExe)) {
throw "OBS executable was not found at '$ObsExe'."
}
if (-not (Test-Path -LiteralPath $ObsConfigRoot)) {
throw "OBS config root was not found at '$ObsConfigRoot'."
}
Stop-Obs
Remove-ObsCrashSentinel -ConfigRoot $ObsConfigRoot
if ($NoStart) {
Write-Host "OBS stopped and crash sentinel cleared."
return
}
$workingDirectory = Split-Path -Parent $ObsExe
Start-Process -FilePath $ObsExe -WorkingDirectory $workingDirectory
Write-Host "Started OBS: $ObsExe"
$deadline = (Get-Date).AddSeconds($StartupWaitSeconds)
$listeners = @()
do {
Start-Sleep -Seconds 1
$obs = Get-Process obs64 -ErrorAction SilentlyContinue
if (-not $obs) {
throw "OBS did not remain running after startup."
}
$observed = Get-ObservedUdpPorts -Ports $ExpectedUdpPorts
$listeners = @($observed.Lines)
$missingPorts = @($ExpectedUdpPorts | Where-Object { -not $observed.Ports.ContainsKey($_) })
if ($missingPorts.Count -eq 0) {
break
}
} while ((Get-Date) -lt $deadline)
if ($missingPorts.Count -ne 0) {
$latestLog = Get-LatestObsLog -ConfigRoot $ObsConfigRoot
if ($latestLog) {
Write-Host "Newest OBS log: $($latestLog.FullName) ($($latestLog.Length) bytes)"
if ($latestLog.Length -gt 0) {
Get-Content -LiteralPath $latestLog.FullName -Tail 40 | Write-Host
}
}
throw "OBS is running, but expected UDP listener ports are missing: $($missingPorts -join ', ')."
}
Write-Host "OBS process:"
$obs | Select-Object Id, ProcessName, StartTime, Responding | Format-Table -AutoSize
Write-Host "Observed Muninn UDP listeners:"
$listeners | ForEach-Object { Write-Host $_ }
Assert-LatestObsLogWitness -ConfigRoot $ObsConfigRoot

View file

@ -0,0 +1,141 @@
param(
[string] $Target = "eve",
[string] $SharedTextureName = "Global\MimirFensalirProgramTexture",
[string] $SharedFenceName = "Global\MimirFensalirProgramFence",
[int] $SharedTextureRingCount = 1,
[string] $Configuration = "Debug",
[int] $Width = 1620,
[int] $Height = 2160,
[int] $Fps = 30,
[int] $Port = 8792,
[string] $LanHost = "192.168.1.66",
[string] $Encoder = "h264_nvenc",
[int] $Quality = 26,
[string] $FfmpegPath = "",
[switch] $NoReverseTunnel,
[string] $MimirLogPath = "logs\eve-dashboard-mimir.log",
[string] $RelayLogPath = "logs\eve-dashboard-relay.log",
[string] $TunnelLogPath = "logs\eve-dashboard-ssh-tunnel.log",
[string[]] $AppArguments = @()
)
$ErrorActionPreference = "Stop"
$repoRoot = Split-Path -Parent $PSScriptRoot
$mimirLog = Join-Path $repoRoot $MimirLogPath
$mimirErr = [System.IO.Path]::ChangeExtension($mimirLog, ".err.log")
$relayLog = Join-Path $repoRoot $RelayLogPath
$relayErr = [System.IO.Path]::ChangeExtension($relayLog, ".err.log")
$tunnelLog = Join-Path $repoRoot $TunnelLogPath
$tunnelErr = [System.IO.Path]::ChangeExtension($tunnelLog, ".err.log")
New-Item -ItemType Directory -Path (Split-Path -Parent $mimirLog) -Force | Out-Null
New-Item -ItemType Directory -Path (Split-Path -Parent $relayLog) -Force | Out-Null
New-Item -ItemType Directory -Path (Split-Path -Parent $tunnelLog) -Force | Out-Null
Write-Host "Checking EVE SSH target: $Target"
ssh $Target "uname -a >/tmp/mimir-eve-dashboard-check.txt && cat /tmp/mimir-eve-dashboard-check.txt"
$env:FENSALIR_PROGRAM_OUTPUT_D3D12 = "1"
$env:FENSALIR_PROGRAM_OUTPUT_NAME = $SharedTextureName
$env:FENSALIR_PROGRAM_OUTPUT_FENCE_NAME = $SharedFenceName
$env:FENSALIR_PROGRAM_OUTPUT_RING_COUNT = [string] $SharedTextureRingCount
$mimirProject = Join-Path $repoRoot "src\Mimir.App\Mimir.App.csproj"
$mimirArgs = @(
"run",
"--project",
$mimirProject,
"-c",
$Configuration,
"--"
) + $AppArguments
Write-Host "Starting Mimir/Fensalir program output: $SharedTextureName"
Write-Host "Program output fence: $SharedFenceName"
Write-Host "Program output ring slots: $SharedTextureRingCount"
$mimirProcess = Start-Process -FilePath "dotnet" `
-ArgumentList $mimirArgs `
-WorkingDirectory $repoRoot `
-RedirectStandardOutput $mimirLog `
-RedirectStandardError $mimirErr `
-WindowStyle Hidden `
-PassThru
Write-Host "Mimir PID $($mimirProcess.Id); waiting for shared texture publication..."
$deadline = [DateTimeOffset]::Now.AddSeconds(30)
do {
Start-Sleep -Milliseconds 250
$mimirText = if (Test-Path $mimirLog) { Get-Content -Raw $mimirLog } else { "" }
if ($mimirText -match "D3D12 program output shared texture") {
break
}
} while ([DateTimeOffset]::Now -lt $deadline -and -not $mimirProcess.HasExited)
if ($mimirProcess.HasExited) {
throw "Mimir exited before publishing the shared program texture. Check $mimirLog and $mimirErr."
}
$relayProject = Join-Path $repoRoot "src\Mimir.EveRelay\Mimir.EveRelay.csproj"
$relayArgs = @(
"run",
"--project",
$relayProject,
"-c",
$Configuration,
"--no-restore",
"--",
"--shared-texture", $SharedTextureName,
"--width", $Width,
"--height", $Height,
"--fps", $Fps,
"--port", $Port,
"--lan-host", $LanHost,
"--encoder", $Encoder,
"--quality", $Quality
)
if (-not [string]::IsNullOrWhiteSpace($FfmpegPath)) {
$relayArgs += @("--ffmpeg", $FfmpegPath)
}
Write-Host "Starting H.264 Eve relay on port $Port"
$relayProcess = Start-Process -FilePath "dotnet" `
-ArgumentList $relayArgs `
-WorkingDirectory $repoRoot `
-RedirectStandardOutput $relayLog `
-RedirectStandardError $relayErr `
-WindowStyle Hidden `
-PassThru
if (-not $NoReverseTunnel) {
$forwardPattern = "-R ${Port}:127.0.0.1:${Port}"
$existingTunnel = Get-CimInstance Win32_Process -Filter "name = 'ssh.exe'" |
Where-Object { $_.CommandLine -like "*$forwardPattern*" -and $_.CommandLine -like "* $Target*" } |
Select-Object -First 1
if ($existingTunnel) {
Write-Host "Using existing EVE reverse tunnel PID $($existingTunnel.ProcessId)."
} else {
Write-Host "Starting EVE reverse tunnel: $Target 127.0.0.1:$Port -> Starfire 127.0.0.1:$Port"
$tunnelProcess = Start-Process -FilePath "ssh" `
-ArgumentList @("-N", "-R", "${Port}:127.0.0.1:${Port}", $Target) `
-RedirectStandardOutput $tunnelLog `
-RedirectStandardError $tunnelErr `
-WindowStyle Hidden `
-PassThru
Write-Host "Tunnel PID $($tunnelProcess.Id)."
}
Start-Sleep -Milliseconds 500
ssh $Target "curl -s --max-time 3 http://127.0.0.1:$Port/health || true"
}
Write-Host "Relay PID $($relayProcess.Id). Launching EveCanvas."
ssh $Target "uiopen --bundleid org.gamecult.evecanvas"
Write-Host "Poll with:"
Write-Host " Get-Content -Tail 40 '$mimirLog'"
Write-Host " Get-Content -Tail 40 '$mimirErr'"
Write-Host " Get-Content -Tail 40 '$relayLog'"
Write-Host " Get-Content -Tail 40 '$relayErr'"
Write-Host " Get-Content -Tail 40 '$tunnelLog'"
Write-Host " Get-Content -Tail 40 '$tunnelErr'"

View file

@ -0,0 +1,53 @@
param(
[string] $Target = "eve",
[string] $SharedTextureName = "Global\MimirFensalirProgramTexture",
[string] $SharedFenceName = "Global\MimirFensalirProgramFence",
[int] $SharedTextureRingCount = 1,
[string] $Configuration = "Debug",
[string] $LogPath = "logs\eve-program-output-fensalir.log",
[string[]] $AppArguments = @()
)
$ErrorActionPreference = "Stop"
$repoRoot = Split-Path -Parent $PSScriptRoot
$absoluteLogPath = Join-Path $repoRoot $LogPath
$absoluteErrorLogPath = [System.IO.Path]::ChangeExtension($absoluteLogPath, ".err.log")
$logDirectory = Split-Path -Parent $absoluteLogPath
if (-not (Test-Path $logDirectory)) {
New-Item -ItemType Directory -Path $logDirectory | Out-Null
}
Write-Host "Checking EVE SSH target: $Target"
ssh $Target "uname -a >/tmp/mimir-eve-program-output-check.txt && cat /tmp/mimir-eve-program-output-check.txt"
$env:FENSALIR_PROGRAM_OUTPUT_D3D12 = "1"
$env:FENSALIR_PROGRAM_OUTPUT_NAME = $SharedTextureName
$env:FENSALIR_PROGRAM_OUTPUT_FENCE_NAME = $SharedFenceName
$env:FENSALIR_PROGRAM_OUTPUT_RING_COUNT = [string] $SharedTextureRingCount
$projectPath = Join-Path $repoRoot "src\Mimir.App\Mimir.App.csproj"
$dotnetArgs = @(
"run",
"--project",
$projectPath,
"-c",
$Configuration,
"--"
) + $AppArguments
Write-Host "Starting Mimir/Fensalir with shared D3D12 output: $SharedTextureName"
Write-Host "Program output fence: $SharedFenceName"
Write-Host "Program output ring slots: $SharedTextureRingCount"
Write-Host "Log: $absoluteLogPath"
$process = Start-Process -FilePath "dotnet" `
-ArgumentList $dotnetArgs `
-WorkingDirectory $repoRoot `
-RedirectStandardOutput $absoluteLogPath `
-RedirectStandardError $absoluteErrorLogPath `
-WindowStyle Hidden `
-PassThru
Write-Host "Started PID $($process.Id). Poll with:"
Write-Host " Get-Content -Tail 40 '$absoluteLogPath'"
Write-Host " Get-Content -Tail 40 '$absoluteErrorLogPath'"

View file

@ -0,0 +1,103 @@
param(
[string]$WellLog = "",
[string]$CultCache = "",
[int]$Port = 8799,
[int]$PollMs = 100,
[int]$WorkerMs = 4,
[int]$MaxReservoirQueue = 120,
[double]$GpuBudget = 0.50,
[double]$CpuBudget = 0.25,
[switch]$NoBuild,
[switch]$FromStart
)
$ErrorActionPreference = "Stop"
$repo = Resolve-Path (Join-Path $PSScriptRoot "..")
$project = Join-Path $repo "src\Mimir.FensalirDaemon\Mimir.FensalirDaemon.csproj"
if (-not $NoBuild) {
dotnet build $project | Out-Host
}
if ([string]::IsNullOrWhiteSpace($CultCache)) {
$CultCache = Join-Path $repo "state\fensalir-daemon.ccmp"
}
if ([string]::IsNullOrWhiteSpace($WellLog)) {
$runtimeRoot = Join-Path $repo "artifacts\runtime"
if (Test-Path $runtimeRoot) {
$WellLog = Get-ChildItem -Path $runtimeRoot -Recurse -File -Include *.log,*.jsonl |
Where-Object { $_.Length -gt 0 } |
Sort-Object LastWriteTimeUtc -Descending |
Where-Object {
try {
Select-String -Path $_.FullName -Pattern 'mimir\.well_' -Quiet -ErrorAction Stop
} catch {
$false
}
} |
Select-Object -First 1 -ExpandProperty FullName
}
}
$runId = Get-Date -Format "yyyyMMdd-HHmmss"
$runDir = Join-Path $repo "artifacts\runtime\fensalir-daemon-$runId"
New-Item -ItemType Directory -Force -Path $runDir | Out-Null
$stdout = Join-Path $runDir "fensalir-daemon.out.log"
$stderr = Join-Path $runDir "fensalir-daemon.err.log"
$manifest = Join-Path $runDir "supervisor.json"
$dll = Join-Path $repo "src\Mimir.FensalirDaemon\bin\Debug\net10.0\Mimir.FensalirDaemon.dll"
$rootVerse = "asgard"
$locatedService = "$rootVerse.starfire.mimir-fensalir"
$tuiAddress = "$locatedService/eve/tui"
$guiAddress = "$locatedService/eve/gui"
$bridgeRoute = "ws://127.0.0.1:$Port/eve/deck"
$arguments = @(
$dll,
"--port", $Port,
"--cultcache", $CultCache,
"--poll-ms", $PollMs,
"--worker-ms", $WorkerMs,
"--max-reservoir-queue", $MaxReservoirQueue,
"--gpu-budget", $GpuBudget.ToString([Globalization.CultureInfo]::InvariantCulture),
"--cpu-budget", $CpuBudget.ToString([Globalization.CultureInfo]::InvariantCulture)
)
if (-not [string]::IsNullOrWhiteSpace($WellLog)) {
$arguments += @("--well-log", $WellLog)
}
if ($FromStart) {
$arguments += "--from-start"
}
$process = Start-Process -FilePath "dotnet" -ArgumentList $arguments -WorkingDirectory $repo -RedirectStandardOutput $stdout -RedirectStandardError $stderr -PassThru -WindowStyle Hidden
[pscustomobject]@{
daemon = "Mimir.FensalirDaemon"
pid = $process.Id
port = $Port
workerMs = $WorkerMs
maxReservoirQueue = $MaxReservoirQueue
wellLog = $WellLog
cultCache = $CultCache
rootVerse = $rootVerse
canonicalService = "$rootVerse.mimir-fensalir"
locatedService = $locatedService
cultMeshAddress = $tuiAddress
providerSpec = "mimir-fensalir-daemon|Mimir Fensalir Daemon|$tuiAddress"
endpoints = @($tuiAddress, $guiAddress)
routes = @(
[ordered]@{ kind = "websocket-bridge"; address = $bridgeRoute; carries = $tuiAddress; note = "Fensalir Eve deck bridge route; not service identity." },
[ordered]@{ kind = "websocket-bridge"; address = "$bridgeRoute/cultmesh"; carries = $tuiAddress; note = "Legacy CultMesh bridge route; not service identity." }
)
stdout = $stdout
stderr = $stderr
started = (Get-Date).ToString("o")
} | ConvertTo-Json | Set-Content -Path $manifest -Encoding UTF8
Write-Host "Started Mimir.FensalirDaemon PID $($process.Id)"
Write-Host "Health: http://127.0.0.1:$Port/health"
Write-Host "Eve provider: mimir-fensalir-daemon|Mimir Fensalir Daemon|$tuiAddress"
Write-Host "Bridge route: $bridgeRoute"
Write-Host "Manifest: $manifest"

View file

@ -0,0 +1,103 @@
param(
[switch]$Headless,
[int]$HeadlessWidth = 1280,
[int]$HeadlessHeight = 720,
[string]$CaptureFrame = "",
[string]$NightwingHost = "nightwing",
[string]$WitnessUrl = "ws://192.168.1.66:8796/eve/periwinkle",
[string]$ConfigPath = "E:\Projects\Mimir\config\mimir-runtime.mvp-leap-eyes-kiyo.local.json"
)
$ErrorActionPreference = "Stop"
$repo = Split-Path -Parent $PSScriptRoot
$rootVerse = "asgard"
$locatedService = "$rootVerse.starfire.mimir"
$cultMeshAddress = "$locatedService/periwinkle/eve/tui"
$logDir = Join-Path $repo "artifacts\runtime\mvp-leap-eyes-kiyo"
New-Item -ItemType Directory -Force -Path $logDir | Out-Null
function Test-CommandOk {
param([string[]]$Command, [string]$Name)
$process = Start-Process -FilePath $Command[0] -ArgumentList $Command[1..($Command.Length - 1)] -NoNewWindow -Wait -PassThru -RedirectStandardOutput (Join-Path $logDir "$Name.out.log") -RedirectStandardError (Join-Path $logDir "$Name.err.log")
return $process.ExitCode -eq 0
}
$kiyo = Get-PnpDevice -PresentOnly | Where-Object { $_.InstanceId -match "VID_1532&PID_0E05&MI_00" } | Select-Object -First 1
$leap = Get-PnpDevice -PresentOnly | Where-Object { $_.InstanceId -match "VID_F182&PID_0003&MI_00" } | Select-Object -First 1
if (-not $kiyo) {
throw "Kiyo Pro camera interface is not present. MVP needs Kiyo Pro RGB."
}
if (-not $leap) {
throw "LeapUVC camera interface is not present. Replug/wake the Leap before launching this MVP profile."
}
$nightwingCheck = ssh $NightwingHost "test -e /dev/video2 -a -e /dev/video3 && command -v python3 >/dev/null && echo ok" 2>$null
if ($LASTEXITCODE -ne 0 -or $nightwingCheck -notmatch "ok") {
throw "Nightwing does not expose /dev/video2 + /dev/video3 with python3 available."
}
scp (Join-Path $repo "tools\nw_eye_cap.py") "${NightwingHost}:/tmp/nw_eye_cap.py" | Out-Null
if ($LASTEXITCODE -ne 0) {
throw "Could not stage nw_eye_cap.py on Nightwing."
}
scp (Join-Path $repo "tools\nw_move_hint.py") "${NightwingHost}:/tmp/nw_move_hint.py" | Out-Null
if ($LASTEXITCODE -ne 0) {
throw "Could not stage nw_move_hint.py on Nightwing."
}
scp (Join-Path $repo "tools\nightwing_typed_witness_publisher.py") "${NightwingHost}:/tmp/nightwing_typed_witness_publisher.py" | Out-Null
if ($LASTEXITCODE -ne 0) {
throw "Could not stage nightwing_typed_witness_publisher.py on Nightwing."
}
$receiverRunning = Get-CimInstance Win32_Process |
Where-Object { $_.CommandLine -match "Mimir.EveSensorReceiver" -and $_.CommandLine -match "--port 8796" } |
Select-Object -First 1
if (-not $receiverRunning) {
$receiverOut = Join-Path $logDir "nightwing-witness-receiver.out.log"
$receiverErr = Join-Path $logDir "nightwing-witness-receiver.err.log"
Start-Process -WindowStyle Hidden -FilePath "dotnet" -ArgumentList @(
"run",
"--no-build",
"--project",
(Join-Path $repo "src\Mimir.EveSensorReceiver\Mimir.EveSensorReceiver.csproj"),
"--",
"--port",
"8796",
"--path",
"/eve/periwinkle",
"--source-id",
"nightwing-witness",
"--type",
"cultmesh-observation"
) -RedirectStandardOutput $receiverOut -RedirectStandardError $receiverErr
Start-Sleep -Milliseconds 800
}
$remotePublisher = 'ps -eo pid=,comm=,args= | awk ''$2=="python3" && $0 ~ /\/tmp\/nightwing_typed_witness_publisher.py/ && $0 ~ /--track-eyes/ {print $1}'' | xargs -r kill; nohup python3 /tmp/nightwing_typed_witness_publisher.py --url ''' + $WitnessUrl + ''' --track-eyes --interval 0.12 > /tmp/mimir-nightwing-mvp-witness.log 2>&1 &'
ssh $NightwingHost $remotePublisher | Out-Null
if ($LASTEXITCODE -ne 0) {
throw "Could not start Nightwing typed witness publisher."
}
$env:MIMIR_RUNTIME_CONFIG = $ConfigPath
$args = @("run", "--project", (Join-Path $repo "src\Mimir.App\Mimir.App.csproj"))
if ($Headless) {
$args += "--"
$args += "--headless"
$args += "--headless-width"
$args += "$HeadlessWidth"
$args += "--headless-height"
$args += "$HeadlessHeight"
if ($CaptureFrame) {
$args += "--capture-frame"
$args += $CaptureFrame
}
}
Write-Host "MIMIR_RUNTIME_CONFIG=$ConfigPath"
Write-Host "CultMesh address: $cultMeshAddress"
Write-Host "Bridge route: $WitnessUrl"
Write-Host "Starting MVP: Starfire Leap + Kiyo Pro, Nightwing Eye/Move witness claims"
dotnet @args

View file

@ -0,0 +1,376 @@
param(
[string]$NightwingHost = "nightwing",
[int]$VerseRelayPort = 8798,
[string]$WitnessUrl = "ws://192.168.1.66:8798/eve/periwinkle",
[string]$SubscribeUrl = "ws://127.0.0.1:8798/eve/periwinkle/subscribe",
[string]$ConfigPath = "E:\Projects\Mimir\config\mimir-runtime.well.local.json",
[string]$WellPublishUrl = "ws://127.0.0.1:8798/eve/periwinkle",
[string]$RecorderRoot = "C:\Users\Meta\Videos\Mimir\VerseCaptures",
[int]$FensalirPort = 8799,
[int]$FensalirWorkerMs = 2,
[int]$FensalirMaxReservoirQueue = 120,
[int]$DurationSeconds = 0,
[int]$MoveFps = 120,
[int]$MoveFftSize = 256,
[string[]]$Move = @(
"move-00-07-04-a3-97-72-usb=/dev/hidraw1:#35ff6c"
),
[switch]$SkipRuntime,
[switch]$SkipFensalir,
[switch]$SkipMoves,
[switch]$SkipNightwing,
[switch]$DryRun
)
$ErrorActionPreference = "Stop"
$repo = Split-Path -Parent $PSScriptRoot
$rootVerse = "asgard"
$locatedMimir = "$rootVerse.starfire.mimir"
$locatedFensalir = "$rootVerse.starfire.mimir-fensalir"
$periwinkleTui = "$locatedMimir/periwinkle/eve/tui"
$periwinkleGui = "$locatedMimir/periwinkle/eve/gui"
$runId = Get-Date -Format "yyyyMMdd-HHmmss"
$runDir = Join-Path $repo "artifacts\runtime\online-verse-daemons-$runId"
New-Item -ItemType Directory -Force -Path $runDir | Out-Null
New-Item -ItemType Directory -Force -Path $RecorderRoot | Out-Null
function Start-LoggedProcess {
param(
[string]$Name,
[string]$FilePath,
[string[]]$ArgumentList,
[hashtable]$Environment = @{}
)
$stdout = Join-Path $runDir "$Name.out.log"
$stderr = Join-Path $runDir "$Name.err.log"
if ($DryRun) {
Write-Host "DRY $Name`: $FilePath $($ArgumentList -join ' ')"
return $null
}
$previous = @{}
foreach ($key in $Environment.Keys) {
$previous[$key] = [Environment]::GetEnvironmentVariable($key, "Process")
[Environment]::SetEnvironmentVariable($key, [string]$Environment[$key], "Process")
}
try {
$process = Start-Process `
-FilePath $FilePath `
-ArgumentList $ArgumentList `
-WorkingDirectory $repo `
-WindowStyle Hidden `
-RedirectStandardOutput $stdout `
-RedirectStandardError $stderr `
-PassThru
} finally {
foreach ($key in $Environment.Keys) {
[Environment]::SetEnvironmentVariable($key, $previous[$key], "Process")
}
}
Set-Content -Path (Join-Path $runDir "$Name.pid") -Value $process.Id
Write-Host "$Name pid=$($process.Id) stdout=$stdout stderr=$stderr"
return $process
}
function Wait-HttpHealth {
param(
[string]$Url,
[int]$TimeoutSeconds = 45
)
if ($DryRun) {
Write-Host "DRY wait health: $Url"
return
}
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
do {
try {
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 3
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
Write-Host "health ok: $Url"
return
}
} catch {
Start-Sleep -Milliseconds 500
}
} while ((Get-Date) -lt $deadline)
throw "Timed out waiting for $Url"
}
$manifest = [ordered]@{
kind = "mimir.online_verse_daemon_supervisor.v1"
runId = $runId
runDir = $runDir
rootVerse = $rootVerse
canonicalService = "$rootVerse.mimir"
locatedService = $locatedMimir
cultMeshAddress = $periwinkleTui
endpoints = @($periwinkleTui, $periwinkleGui)
routes = @(
[ordered]@{ kind = "websocket-bridge"; address = $WitnessUrl; carries = $periwinkleTui; note = "Nightwing witness bridge route; not service identity." },
[ordered]@{ kind = "websocket-bridge"; address = $SubscribeUrl; carries = $periwinkleTui; note = "Recorder subscription bridge route; not service identity." },
[ordered]@{ kind = "websocket-bridge"; address = $WellPublishUrl; carries = $periwinkleTui; note = "Well publication bridge route; not service identity." }
)
witnessUrl = $WitnessUrl
subscribeUrl = $SubscribeUrl
wellPublishUrl = $WellPublishUrl
verseRelayPort = $VerseRelayPort
recorderRoot = $RecorderRoot
fensalirPort = $FensalirPort
configPath = $ConfigPath
startedAt = (Get-Date).ToString("o")
durationSeconds = $DurationSeconds
moves = $Move
processes = @{}
}
Write-Host "Online Verse daemon run: $runDir"
if (-not $DryRun) {
Get-CimInstance Win32_Process |
Where-Object {
($_.CommandLine -match "Mimir.EveSensorReceiver" -and $_.CommandLine -match "--port $VerseRelayPort") -or
($_.CommandLine -match "Mimir.VerseRecorder") -or
($_.CommandLine -match "Mimir.FensalirDaemon") -or
($_.CommandLine -match "Mimir.Well") -or
($_.CommandLine -match "Mimir.BufferSmoke" -and $_.CommandLine -match "--poll-ms")
} |
ForEach-Object {
try { Stop-Process -Id $_.ProcessId -Force -ErrorAction Stop } catch {}
}
}
$receiver = Start-LoggedProcess `
-Name "verse-relay" `
-FilePath "dotnet" `
-ArgumentList @(
"run",
"--project",
(Join-Path $repo "src\Mimir.EveSensorReceiver\Mimir.EveSensorReceiver.csproj"),
"--",
"--port", "$VerseRelayPort",
"--path", "/eve/periwinkle",
"--subscribe-path", "/eve/periwinkle/subscribe",
"--source-id", "nightwing-witness",
"--type", "cultmesh-observation"
)
if ($receiver) { $manifest.processes["verseRelayPid"] = $receiver.Id }
Wait-HttpHealth -Url "http://127.0.0.1:$VerseRelayPort/health"
$recorderArgs = @(
"run",
"--project",
(Join-Path $repo "src\Mimir.VerseRecorder\Mimir.VerseRecorder.csproj"),
"--",
"--url", $SubscribeUrl,
"--out-dir", $RecorderRoot,
"--run-id", $runId,
"--write-bodies", "true",
"--body-page-bytes", "134217728"
)
if ($DurationSeconds -gt 0) {
$recorderArgs += "--seconds"
$recorderArgs += "$DurationSeconds"
}
$recorder = Start-LoggedProcess -Name "verse-recorder" -FilePath "dotnet" -ArgumentList $recorderArgs
if ($recorder) { $manifest.processes["verseRecorderPid"] = $recorder.Id }
if (-not $SkipFensalir) {
$fensalirWellLog = Join-Path (Join-Path $RecorderRoot $runId) "observations.jsonl"
$fensalirCache = Join-Path $runDir "fensalir-daemon.ccmp"
$fensalir = Start-LoggedProcess `
-Name "fensalir-daemon" `
-FilePath "dotnet" `
-ArgumentList @(
"run",
"--project",
(Join-Path $repo "src\Mimir.FensalirDaemon\Mimir.FensalirDaemon.csproj"),
"--",
"--port", "$FensalirPort",
"--well-log", $fensalirWellLog,
"--cultcache", $fensalirCache,
"--poll-ms", "50",
"--worker-ms", "$FensalirWorkerMs",
"--max-reservoir-queue", "$FensalirMaxReservoirQueue"
)
if ($fensalir) {
$manifest.processes["fensalirDaemonPid"] = $fensalir.Id
$manifest.processes["fensalirWellLog"] = $fensalirWellLog
$manifest.processes["fensalirCultCache"] = $fensalirCache
$manifest.processes["fensalirProviderSpec"] = "mimir-fensalir-daemon|Mimir Fensalir Daemon|$locatedFensalir/eve/tui"
$manifest.processes["fensalirGui"] = "$locatedFensalir/eve/gui"
$manifest.processes["fensalirBridgeRoute"] = "ws://127.0.0.1:$FensalirPort/eve/deck"
Wait-HttpHealth -Url "http://127.0.0.1:$FensalirPort/health"
}
}
if (-not $SkipNightwing) {
if ($DryRun) {
Write-Host "DRY Nightwing stage and witness start on $NightwingHost"
} else {
scp (Join-Path $repo "tools\nw_eye_cap.py") "${NightwingHost}:/tmp/nw_eye_cap.py" | Out-Null
scp (Join-Path $repo "tools\nw_move_hint.py") "${NightwingHost}:/tmp/nw_move_hint.py" | Out-Null
scp (Join-Path $repo "tools\nightwing_typed_witness_publisher.py") "${NightwingHost}:/tmp/nightwing_typed_witness_publisher.py" | Out-Null
$remoteLog = "~/.local/state/gamecult/mimir-online-verse-witness-$runId.log"
$remotePid = "~/.local/state/gamecult/mimir-online-verse-witness-$runId.pid"
$remoteScript = @"
mkdir -p ~/.local/state/gamecult
pkill -f '/tmp/nightwing_typed_witness_publisher.py.*--track-eyes' || true
nohup python3 /tmp/nightwing_typed_witness_publisher.py --url '$WitnessUrl' --track-eyes --track-builtin-camera --track-builtin-mic --interval 0.10 --eye-devices /dev/video2,/dev/video3 --eye-window-seconds 0.08 --tracking-stride 6 > $remoteLog 2>&1 < /dev/null &
sleep 0.5
pgrep -n -f '/tmp/nightwing_typed_witness_publisher.py.*--track-eyes' > $remotePid
cat $remotePid
"@
$remoteB64 = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($remoteScript))
$nightwingPid = ssh $NightwingHost "printf '%s' '$remoteB64' | base64 -d | bash"
$manifest.processes["nightwingWitnessPid"] = ($nightwingPid | Select-Object -First 1)
$manifest.processes["nightwingWitnessLog"] = $remoteLog
}
}
if (-not $SkipMoves) {
if (-not $DryRun) {
Get-CimInstance Win32_Process |
Where-Object { $_.CommandLine -match "online_move_music_sync.py" } |
ForEach-Object {
try { Stop-Process -Id $_.ProcessId -Force -ErrorAction Stop } catch {}
}
}
$moveArgs = @(
".\tools\online_move_music_sync.py",
"--duration", $(if ($DurationSeconds -gt 0) { "$DurationSeconds" } else { "31536000" }),
"--out-dir", (Join-Path $runDir "move-sync"),
"--device", "Realtek",
"--asio-music-channels", "0,1,2,3",
"--asio-music-source-name", "0=shotgun-mic",
"--asio-music-source-name", "1=cardioid-mic",
"--asio-music-source-name", "2=starfire-loopback-l",
"--asio-music-source-name", "3=starfire-loopback-r-or-synth",
"--asio-drain-blocks", "1024",
"--fps", "$MoveFps",
"--fft-size", "$MoveFftSize",
"--onset-cooldown-seconds", "0.065",
"--warmup-seconds", "2.0",
"--loudness-history-seconds", "14.0",
"--loudness-threshold", "0.86",
"--hit-loudness-threshold", "0.84",
"--loudness-floor", "0.006",
"--loudness-exponent", "2.6",
"--quiet-brightness", "0.0",
"--max-brightness", "0.34",
"--brightness-exponent", "0.88",
"--score-subdivisions", "8",
"--score-instrument-spacing", "2",
"--score-release", "0.68",
"--score-loudness-threshold", "0.90",
"--score-min-loudness-gate", "0.08",
"--score-loudness-exponent", "1.8",
"--score-min-flux-percentile", "0.92",
"--score-min-onset-intensity", "0.42",
"--score-min-music-confidence", "0.30",
"--score-min-improv-density", "0.04",
"--score-max-improv-density", "0.22",
"--score-target-confidence", "0.78",
"--score-confidence-gesture-gain", "0.34",
"--score-confidence-accent-gain", "0.55",
"--score-min-voice-strength", "0.08",
"--score-min-voice-body", "0.004",
"--score-voice-release-seconds", "1.2",
"--score-voice-decay", "0.72",
"--score-downbeat-min-onset", "0.62",
"--score-call-response-min-onset", "0.48",
"--score-min-accent", "0.025",
"--score-max-envelope", "0.78",
"--score-ensemble-loudness-threshold", "0.96",
"--score-ensemble-min-loudness-gate", "0.25",
"--score-ensemble-min-onset-intensity", "0.86",
"--score-ensemble-min-music-confidence", "0.55",
"--score-ensemble-min-accent", "0.30",
"--score-min-tempo-confidence", "0.0",
"--debruijn-polyrhythm",
"--harmonic-base", "1.5",
"--microtonal-cents", "17",
"--voice-glide-rate", "3.4",
"--voice-vibrato-hz", "5.2",
"--voice-vibrato-cents", "18.0",
"--voice-note-hue-mix", "0.46",
"--voice-vibrato-hue-width", "0.012",
"--score-voice-roles", "violin-syrinx,harp-arpeggio,peck-syrinx",
"--violin-glide-rate", "1.35",
"--violin-release", "0.93",
"--violin-call-seconds", "0.72",
"--harp-slot-spacing", "5",
"--harp-min-onset-intensity", "0.58",
"--harp-call-seconds", "0.18",
"--peck-min-onset-intensity", "0.34",
"--peck-call-seconds", "0.11"
)
$moveArgs += "--emit-bioacoustic-realtk"
$moveArgs += "--bioacoustic-song"
$moveArgs += "aquasynth-formant-weaver"
$moveArgs += "--bioacoustic-device"
$moveArgs += "Realtek"
$moveArgs += "--bioacoustic-gain"
$moveArgs += "1.05"
$moveArgs += "--bioacoustic-loop-seconds"
$moveArgs += "0.42"
$moveArgs += "--bioacoustic-min-interval-seconds"
$moveArgs += "0.18"
$moveArgs += "--bioacoustic-max-active-calls"
$moveArgs += "3"
$moveArgs += "--bioacoustic-render-timeout-seconds"
$moveArgs += "15"
foreach ($spec in $Move) {
$moveArgs += "--move"
$moveArgs += $spec
}
$python = "C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe"
$moveProcess = Start-LoggedProcess -Name "move-sync" -FilePath $python -ArgumentList $moveArgs
if ($moveProcess) { $manifest.processes["moveSyncPid"] = $moveProcess.Id }
}
if (-not $SkipRuntime) {
$runtime = Start-LoggedProcess `
-Name "mimir-well" `
-FilePath "dotnet" `
-ArgumentList @(
"run",
"--project",
(Join-Path $repo "src\Mimir.Well\Mimir.Well.csproj"),
"--",
"--seconds", $(if ($DurationSeconds -gt 0) { "$DurationSeconds" } else { "0" }),
"--poll-ms", "5",
"--publish-url", $WellPublishUrl,
"--node-id", "starfire",
"--publish-ms", "250",
"--sync-ms", "250",
"--presentation-delay-ms", "2500",
"--max-samples-per-source", "32",
"--visual-calibration", "false",
"--visual-calibration-ms", "250",
"--visual-expected-leds", "38",
"--visual-minimum-luma", "0.55",
"--visual-setting-seconds", "0.75",
"--visual-resweep-seconds", "12.0",
"--capture-pages", "true",
"--capture-ms", "250",
"--capture-max-body-bytes", "4194304",
"--capture-inline-bodies", "false",
"--stream-frames", "true",
"--stream-frame-inline-bodies", "true"
) `
-Environment @{
"MIMIR_RUNTIME_CONFIG" = $ConfigPath
"MIMIR_SYNC_TELEMETRY_SECONDS" = "1"
}
if ($runtime) { $manifest.processes["mimirRuntimePid"] = $runtime.Id }
}
$manifestPath = Join-Path $runDir "supervisor.json"
$manifest | ConvertTo-Json -Depth 6 | Set-Content -Path $manifestPath
Write-Host "Supervisor manifest: $manifestPath"

View file

@ -0,0 +1,69 @@
param(
[int]$InputPort = 5200,
[int]$VideoPort = 5210,
[int]$AudioPort = 5212,
[int]$AudioSampleRate = 48000,
[int]$AudioChannels = 2,
[string]$FfmpegPath = "ffmpeg",
[string]$LogRoot = "E:\Projects\Mimir\artifacts\runtime\stream-proof",
[switch]$DryRun
)
$ErrorActionPreference = "Stop"
New-Item -ItemType Directory -Force -Path $LogRoot | Out-Null
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
$stdoutLog = Join-Path $LogRoot "raven-av-demux-$timestamp.out.log"
$stderrLog = Join-Path $LogRoot "raven-av-demux-$timestamp.err.log"
$inputEndpoint = "srt://0.0.0.0:${InputPort}?mode=listener&latency=120000&timeout=5000000"
$videoEndpoint = "srt://127.0.0.1:${VideoPort}?mode=caller&latency=20000"
$audioEndpoint = "srt://127.0.0.1:${AudioPort}?mode=caller&latency=20000"
$arguments = @(
"-hide_banner",
"-nostdin",
"-loglevel", "warning",
"-fflags", "nobuffer",
"-flags", "low_delay",
"-i", $inputEndpoint,
"-map", "0:v:0",
"-an",
"-c:v", "copy",
"-f", "mpegts",
$videoEndpoint,
"-map", "0:a:0",
"-vn",
"-acodec", "pcm_f32le",
"-ar", $AudioSampleRate.ToString(),
"-ac", $AudioChannels.ToString(),
"-f", "f32le",
$audioEndpoint
)
Write-Host "Raven muxed A/V demux:"
Write-Host " input: $inputEndpoint"
Write-Host " video: $videoEndpoint"
Write-Host " audio: $audioEndpoint"
Write-Host " stdout: $stdoutLog"
Write-Host " stderr: $stderrLog"
if ($DryRun) {
Write-Host ($FfmpegPath + " " + ($arguments -join " "))
exit 0
}
Get-CimInstance Win32_Process |
Where-Object { $_.ProcessName -like "ffmpeg*" -and $_.CommandLine -match [regex]::Escape($inputEndpoint) } |
ForEach-Object {
try { Stop-Process -Id $_.ProcessId -Force -ErrorAction Stop } catch {}
}
$process = Start-Process `
-FilePath $FfmpegPath `
-ArgumentList $arguments `
-WindowStyle Hidden `
-RedirectStandardOutput $stdoutLog `
-RedirectStandardError $stderrLog `
-PassThru
Write-Host "Started Raven muxed A/V demux pid=$($process.Id)"

View file

@ -0,0 +1,142 @@
param(
[string]$TargetHost = "192.168.1.66",
[int]$Port = 5200,
[int]$Width = 1920,
[int]$Height = 1080,
[int]$Framerate = 30,
[int]$AudioSampleRate = 48000,
[int]$AudioChannels = 2,
[ValidateSet("ddagrab", "gdigrab")]
[string]$VideoCapture = "ddagrab",
[int]$DdagrabOutputIndex = 0,
[ValidateSet("srt", "tcp-listener")]
[string]$Transport = "srt",
[ValidateSet("caller", "listener")]
[string]$SrtMode = "caller",
[string]$FfmpegPath = "ffmpeg",
[string]$Source = "desktop",
[string]$VideoBitrate = "12000k",
[string]$AudioBitrate = "192k",
[string]$LogRoot = "C:\Meta\Mimir\logs",
[string]$WasapiLoopbackPath = "",
[switch]$DryRun
)
$ErrorActionPreference = "Stop"
function Quote-CmdArgument([string]$Value) {
return '"' + $Value.Replace('"', '\"') + '"'
}
New-Item -ItemType Directory -Force -Path $LogRoot | Out-Null
$repo = Split-Path -Parent $PSScriptRoot
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
$stdoutLog = Join-Path $LogRoot "raven-av-srt-$timestamp.out.log"
$stderrLog = Join-Path $LogRoot "raven-av-srt-$timestamp.err.log"
$endpoint = if ($Transport -eq "tcp-listener") {
"tcp://${TargetHost}:${Port}?listen=1"
} else {
"srt://${TargetHost}:${Port}?mode=${SrtMode}&latency=120000&timeout=30000000"
}
$videoSize = "${Width}x${Height}"
$gop = [Math]::Max(1, $Framerate * 2)
$captureScript = Join-Path $repo "scripts\wasapi-loopback-capture.ps1"
$loopbackExe = if ([string]::IsNullOrWhiteSpace($WasapiLoopbackPath)) {
Join-Path $repo "tools\Mimir.WasapiLoopback\Mimir.WasapiLoopback.exe"
} else {
$WasapiLoopbackPath
}
$audioCaptureCommand = if (Test-Path $loopbackExe) {
(Quote-CmdArgument $loopbackExe) +
" --output stdout --sample-rate $AudioSampleRate --channels $AudioChannels"
} else {
"powershell.exe -NoProfile -ExecutionPolicy Bypass -File " +
(Quote-CmdArgument $captureScript) +
" -Output " +
(Quote-CmdArgument "stdout") +
" -SampleRate $AudioSampleRate -Channels $AudioChannels"
}
$videoInputArgs = if ($VideoCapture -eq "ddagrab") {
@(
"-thread_queue_size", "1024",
"-f", "lavfi",
"-i", "ddagrab=framerate=${Framerate}:output_idx=${DdagrabOutputIndex}:draw_mouse=1"
)
} else {
@(
"-thread_queue_size", "1024",
"-f", "gdigrab",
"-framerate", $Framerate.ToString(),
"-video_size", $videoSize,
"-i", $Source
)
}
$videoEncodeArgs = @(
"-c:v", "h264_nvenc",
"-preset", "p4",
"-tune", "ll",
"-b:v", $VideoBitrate,
"-maxrate", $VideoBitrate,
"-bufsize", "24000k",
"-g", $gop.ToString()
)
if ($VideoCapture -eq "gdigrab") {
$videoEncodeArgs += @("-pix_fmt", "yuv420p")
}
$ffmpegArgs = @(
"-hide_banner",
"-loglevel", "warning"
)
$ffmpegArgs += $videoInputArgs
$ffmpegArgs += @(
"-thread_queue_size", "1024",
"-f", "f32le",
"-ar", $AudioSampleRate.ToString(),
"-ac", $AudioChannels.ToString(),
"-i", "pipe:0",
"-map", "0:v:0",
"-map", "1:a:0"
)
$ffmpegArgs += $videoEncodeArgs
$ffmpegArgs += @(
"-c:a", "aac",
"-b:a", $AudioBitrate,
"-ar", $AudioSampleRate.ToString(),
"-ac", $AudioChannels.ToString(),
"-f", "mpegts",
$endpoint
)
$quotedFfmpegArgs = $ffmpegArgs | ForEach-Object { Quote-CmdArgument $_ }
$command = $audioCaptureCommand +
" | " +
(Quote-CmdArgument $FfmpegPath) +
" " +
($quotedFfmpegArgs -join " ")
Write-Host "Raven muxed A/V sender:"
Write-Host " ffmpeg: $FfmpegPath"
Write-Host " target: $endpoint"
Write-Host " transport: $Transport"
Write-Host " video: $(if ($VideoCapture -eq "ddagrab") { "ddagrab output $DdagrabOutputIndex" } else { "$Source $videoSize" })@$Framerate via h264_nvenc"
Write-Host " audio: Raven Realtek/default render loopback ${AudioChannels}ch ${AudioSampleRate}Hz f32 -> AAC"
Write-Host " loopback: $(if (Test-Path $loopbackExe) { $loopbackExe } else { "$captureScript (diagnostic fallback)" })"
Write-Host " sync role: Raven Realtek is co-streamer game/program loopback packaged with NVENC; Starfire Realtek owns chirp emission"
Write-Host " stdout: $stdoutLog"
Write-Host " stderr: $stderrLog"
if ($DryRun) {
Write-Host $command
exit 0
}
Start-Process `
-FilePath "cmd.exe" `
-ArgumentList ("/d /s /c `"$command`"") `
-WindowStyle Hidden `
-RedirectStandardOutput $stdoutLog `
-RedirectStandardError $stderrLog `
-PassThru |
ForEach-Object { Write-Host "Started Raven muxed A/V sender pid=$($_.Id)" }

View file

@ -0,0 +1,95 @@
param(
[string]$TargetHost = "10.77.0.2",
[int]$Port = 5200,
[string]$ObsTargetHost = "",
[int]$ObsPort = 5204,
[int]$EvePort = 8801,
[int]$Width = 1920,
[int]$Height = 1080,
[int]$Framerate = 30,
[int]$AudioSampleRate = 48000,
[int]$AudioChannels = 2,
[ValidateSet("ddagrab", "gdigrab")]
[string]$VideoCapture = "ddagrab",
[int]$DdagrabOutputIndex = 0,
[string]$AudioDevice = "Realtek",
[string]$FfmpegPath = "ffmpeg",
[string]$WasapiLoopbackPath = "",
[string]$CultMeshCachePath = "C:\Meta\Mimir\state\raven-capture-mux.ccmp",
[string]$LogRoot = "C:\Meta\Mimir\logs",
[switch]$NoObsTarget,
[switch]$NoCultNetServer,
[switch]$DryRun
)
$ErrorActionPreference = "Stop"
$repo = Split-Path -Parent $PSScriptRoot
if ([string]::IsNullOrWhiteSpace($ObsTargetHost)) {
$ObsTargetHost = $TargetHost
}
$project = Join-Path $repo "src\Mimir.RavenDaemon\Mimir.RavenDaemon.csproj"
$runtimeArgs = @(
"run",
"--project", $project,
"--",
"--target-host", $TargetHost,
"--port", $Port,
"--obs-target-host", $ObsTargetHost,
"--obs-port", $ObsPort,
"--eve-port", $EvePort,
"--width", $Width,
"--height", $Height,
"--framerate", $Framerate,
"--audio-sample-rate", $AudioSampleRate,
"--audio-channels", $AudioChannels,
"--video-capture", $VideoCapture,
"--ddagrab-output-index", $DdagrabOutputIndex,
"--audio-device", $AudioDevice,
"--ffmpeg", $FfmpegPath,
"--cultmesh-cache", $CultMeshCachePath,
"--log-root", $LogRoot
)
if ($NoObsTarget) {
$runtimeArgs += "--no-obs-target"
}
if (-not [string]::IsNullOrWhiteSpace($WasapiLoopbackPath)) {
$runtimeArgs += @("--wasapi-loopback", $WasapiLoopbackPath)
}
if ($NoCultNetServer) {
$runtimeArgs += "--no-cultnet-server"
}
if ($DryRun) {
$runtimeArgs += "--dry-run"
& dotnet @runtimeArgs
exit $LASTEXITCODE
}
New-Item -ItemType Directory -Force -Path $LogRoot | Out-Null
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $CultMeshCachePath) | Out-Null
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
$stdoutLog = Join-Path $LogRoot "mimir-raven-daemon-$timestamp.out.log"
$stderrLog = Join-Path $LogRoot "mimir-raven-daemon-$timestamp.err.log"
$process = Start-Process `
-FilePath "dotnet" `
-ArgumentList $runtimeArgs `
-WindowStyle Hidden `
-RedirectStandardOutput $stdoutLog `
-RedirectStandardError $stderrLog `
-PassThru
Write-Host "Started Mimir.RavenDaemon pid=$($process.Id)"
Write-Host " Eve/CultMesh: ws://0.0.0.0:${EvePort}/eve/deck"
Write-Host " Health: http://127.0.0.1:${EvePort}/health"
Write-Host " Mimir target: srt://${TargetHost}:${Port}"
if (-not $NoObsTarget) {
Write-Host " OBS target: srt://${ObsTargetHost}:${ObsPort}"
}
Write-Host " CultMesh cache: $CultMeshCachePath"
Write-Host " stdout: $stdoutLog"
Write-Host " stderr: $stderrLog"

View file

@ -0,0 +1,66 @@
param(
[string]$TargetHost = "192.168.1.66",
[int]$Port = 5200,
[int]$Width = 1920,
[int]$Height = 1080,
[int]$Framerate = 30,
[string]$FfmpegPath = "ffmpeg",
[string]$Source = "desktop",
[string]$VideoBitrate = "12000k",
[string]$LogRoot = "C:\Meta\Mimir\logs",
[switch]$DryRun
)
$ErrorActionPreference = "Stop"
New-Item -ItemType Directory -Force -Path $LogRoot | Out-Null
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
$stdoutLog = Join-Path $LogRoot "raven-screen-srt-$timestamp.out.log"
$stderrLog = Join-Path $LogRoot "raven-screen-srt-$timestamp.err.log"
$endpoint = "srt://${TargetHost}:${Port}?mode=caller&latency=120000"
$videoSize = "${Width}x${Height}"
$gop = [Math]::Max(1, $Framerate * 2)
$arguments = @(
"-hide_banner",
"-nostdin",
"-loglevel", "warning",
"-f", "gdigrab",
"-framerate", $Framerate.ToString(),
"-video_size", $videoSize,
"-i", $Source,
"-an",
"-c:v", "h264_nvenc",
"-preset", "p4",
"-tune", "ll",
"-b:v", $VideoBitrate,
"-maxrate", $VideoBitrate,
"-bufsize", "24000k",
"-pix_fmt", "yuv420p",
"-g", $gop.ToString(),
"-f", "mpegts",
$endpoint
)
Write-Host "Raven screen sender:"
Write-Host " ffmpeg: $FfmpegPath"
Write-Host " target: $endpoint"
Write-Host " source: $Source $videoSize@$Framerate"
Write-Host " stdout: $stdoutLog"
Write-Host " stderr: $stderrLog"
if ($DryRun)
{
Write-Host ($FfmpegPath + " " + ($arguments -join " "))
exit 0
}
$process = Start-Process `
-FilePath $FfmpegPath `
-ArgumentList $arguments `
-WindowStyle Hidden `
-RedirectStandardOutput $stdoutLog `
-RedirectStandardError $stderrLog `
-PassThru
Write-Host "Started Raven screen sender pid=$($process.Id)"

View file

@ -0,0 +1,146 @@
param(
[switch]$Headless,
[int]$HeadlessWidth = 1280,
[int]$HeadlessHeight = 720,
[string]$CaptureFrame = "",
[string]$NightwingHost = "nightwing",
[switch]$StartRavenSender,
[string]$RavenHost = "madman's lullaby@192.168.1.84",
[string]$RavenRepo = "C:\Meta\Mimir",
[string]$StarfireHost = "192.168.1.66",
[string]$WitnessUrl = "ws://192.168.1.66:8796/eve/periwinkle",
[string]$ConfigPath = "E:\Projects\Mimir\config\mimir-runtime.stream-proof.local.json",
[switch]$ProgramOutput,
[string]$SharedTextureName = "Global\MimirFensalirProgramTexture",
[string]$SharedFenceName = "Global\MimirFensalirProgramFence",
[int]$SharedTextureRingCount = 1,
[switch]$DryRun
)
$ErrorActionPreference = "Stop"
$repo = Split-Path -Parent $PSScriptRoot
$logDir = Join-Path $repo "artifacts\runtime\stream-proof"
New-Item -ItemType Directory -Force -Path $logDir | Out-Null
if ($DryRun) {
Write-Host "MIMIR_RUNTIME_CONFIG=$ConfigPath"
if ($ProgramOutput) {
Write-Host "FENSALIR_PROGRAM_OUTPUT_D3D12=1"
Write-Host "FENSALIR_PROGRAM_OUTPUT_NAME=$SharedTextureName"
Write-Host "FENSALIR_PROGRAM_OUTPUT_FENCE_NAME=$SharedFenceName"
Write-Host "FENSALIR_PROGRAM_OUTPUT_RING_COUNT=$SharedTextureRingCount"
}
Write-Host "Would stage Nightwing witness tools to $NightwingHost and start --track-eyes publisher at $WitnessUrl"
if ($StartRavenSender) {
Write-Host "Would start Raven muxed A/V sender on ${RavenHost}: ${RavenRepo}\scripts\start-raven-av-sender.ps1 -TargetHost $StarfireHost -Port 5200"
}
Write-Host "Would start local Raven A/V demux: scripts\start-raven-av-demux.ps1"
Write-Host "Would launch: dotnet run --project $repo\src\Mimir.App\Mimir.App.csproj"
Write-Host "Program layout: raven-display full-frame, kiyo-pro-rgb picture-in-picture"
exit 0
}
$kiyo = Get-PnpDevice -PresentOnly | Where-Object { $_.InstanceId -match "VID_1532&PID_0E05&MI_00" } | Select-Object -First 1
$leap = Get-PnpDevice -PresentOnly | Where-Object { $_.InstanceId -match "VID_F182&PID_0003&MI_00" } | Select-Object -First 1
if (-not $kiyo) {
throw "Kiyo Pro camera interface is not present. Stream proof needs the Kiyo AR program view."
}
if (-not $leap) {
throw "LeapUVC camera interface is not present. Replug/wake the Leap before launching this profile."
}
$nightwingCheck = ssh $NightwingHost "test -e /dev/video2 -a -e /dev/video3 && command -v python3 >/dev/null && echo ok" 2>$null
if ($LASTEXITCODE -ne 0 -or $nightwingCheck -notmatch "ok") {
throw "Nightwing does not expose /dev/video2 + /dev/video3 with python3 available."
}
scp (Join-Path $repo "tools\nw_eye_cap.py") "${NightwingHost}:/tmp/nw_eye_cap.py" | Out-Null
if ($LASTEXITCODE -ne 0) {
throw "Could not stage nw_eye_cap.py on Nightwing."
}
scp (Join-Path $repo "tools\nw_move_hint.py") "${NightwingHost}:/tmp/nw_move_hint.py" | Out-Null
if ($LASTEXITCODE -ne 0) {
throw "Could not stage nw_move_hint.py on Nightwing."
}
scp (Join-Path $repo "tools\nightwing_typed_witness_publisher.py") "${NightwingHost}:/tmp/nightwing_typed_witness_publisher.py" | Out-Null
if ($LASTEXITCODE -ne 0) {
throw "Could not stage nightwing_typed_witness_publisher.py on Nightwing."
}
$receiverRunning = Get-CimInstance Win32_Process |
Where-Object { $_.CommandLine -match "Mimir.EveSensorReceiver" -and $_.CommandLine -match "--port 8796" } |
Select-Object -First 1
if (-not $receiverRunning) {
$receiverOut = Join-Path $logDir "nightwing-witness-receiver.out.log"
$receiverErr = Join-Path $logDir "nightwing-witness-receiver.err.log"
Start-Process -WindowStyle Hidden -FilePath "dotnet" -ArgumentList @(
"run",
"--no-build",
"--project",
(Join-Path $repo "src\Mimir.EveSensorReceiver\Mimir.EveSensorReceiver.csproj"),
"--",
"--port",
"8796",
"--path",
"/eve/periwinkle",
"--source-id",
"nightwing-witness",
"--type",
"cultmesh-observation"
) -RedirectStandardOutput $receiverOut -RedirectStandardError $receiverErr
Start-Sleep -Milliseconds 800
}
$remotePublisher = 'ps -eo pid=,comm=,args= | awk ''$2=="python3" && $0 ~ /\/tmp\/nightwing_typed_witness_publisher.py/ && $0 ~ /--track-eyes/ {print $1}'' | xargs -r kill; nohup python3 /tmp/nightwing_typed_witness_publisher.py --url ''' + $WitnessUrl + ''' --track-eyes --interval 0.12 > /tmp/mimir-nightwing-stream-proof.log 2>&1 &'
ssh $NightwingHost $remotePublisher | Out-Null
if ($LASTEXITCODE -ne 0) {
throw "Could not start Nightwing typed witness publisher."
}
& (Join-Path $repo "scripts\start-raven-av-demux.ps1")
if ($StartRavenSender) {
$taskName = "MimirRavenAvSender5200"
$ravenSender = "${RavenRepo}\scripts\start-raven-av-sender.ps1"
$remoteCmd = "${RavenRepo}\scripts\run-raven-av-5200.cmd"
$taskTime = (Get-Date).AddMinutes(1).ToString("HH:mm")
ssh $RavenHost "cmd /c echo powershell.exe -NoProfile -ExecutionPolicy Bypass -File $ravenSender -TargetHost $StarfireHost -Port 5200 ^> $remoteCmd" | Out-Null
ssh $RavenHost "schtasks /Delete /TN $taskName /F" | Out-Null
ssh $RavenHost "schtasks /Create /TN $taskName /SC ONCE /ST $taskTime /TR $remoteCmd /F /IT" | Out-Host
ssh $RavenHost "schtasks /Run /TN $taskName" | Out-Host
if ($LASTEXITCODE -ne 0) {
throw "Could not start Raven muxed A/V sender task on $RavenHost."
}
Start-Sleep -Milliseconds 1500
}
$env:MIMIR_RUNTIME_CONFIG = $ConfigPath
if ($ProgramOutput) {
$env:FENSALIR_PROGRAM_OUTPUT_D3D12 = "1"
$env:FENSALIR_PROGRAM_OUTPUT_NAME = $SharedTextureName
$env:FENSALIR_PROGRAM_OUTPUT_FENCE_NAME = $SharedFenceName
$env:FENSALIR_PROGRAM_OUTPUT_RING_COUNT = [string]$SharedTextureRingCount
}
$args = @("run", "--project", (Join-Path $repo "src\Mimir.App\Mimir.App.csproj"))
if ($Headless) {
$args += "--"
$args += "--headless"
$args += "--headless-width"
$args += "$HeadlessWidth"
$args += "--headless-height"
$args += "$HeadlessHeight"
if ($CaptureFrame) {
$args += "--capture-frame"
$args += $CaptureFrame
}
}
Write-Host "MIMIR_RUNTIME_CONFIG=$ConfigPath"
Write-Host "Starting stream proof: Leap + Kiyo Pro + Raven screen/audio, Nightwing Eye/Move witness claims"
if ($ProgramOutput) {
Write-Host "Program output texture: $SharedTextureName"
}
Write-Host "Program layout: raven-display full-frame, kiyo-pro-rgb picture-in-picture"
dotnet @args

View file

@ -0,0 +1,74 @@
param(
[string]$WellLog = "",
[string]$MapName = "Local\MimirObsStemBus",
[string]$CompositeStemId = "well_composite",
[int]$PollMs = 25,
[switch]$FromStart
)
$ErrorActionPreference = "Stop"
$repo = Split-Path -Parent $PSScriptRoot
if ([string]::IsNullOrWhiteSpace($WellLog)) {
$runtime = Join-Path $repo "artifacts\runtime"
$latest = Get-ChildItem -Path $runtime -Filter "verse-relay.out.log" -Recurse -ErrorAction SilentlyContinue |
Sort-Object LastWriteTimeUtc -Descending |
Select-Object -First 1
if (-not $latest) {
throw "No verse-relay.out.log found under $runtime. Pass -WellLog explicitly."
}
$WellLog = $latest.FullName
}
$runDir = Join-Path $repo ("artifacts\runtime\well-obs-daemon-" + (Get-Date -Format "yyyyMMdd-HHmmss"))
New-Item -ItemType Directory -Force -Path $runDir | Out-Null
$stdout = Join-Path $runDir "well-obs-daemon.out.log"
$stderr = Join-Path $runDir "well-obs-daemon.err.log"
$daemonExe = Join-Path $repo "src\Mimir.WellObsDaemon\bin\Debug\net10.0-windows\Mimir.WellObsDaemon.exe"
if (-not (Test-Path $daemonExe)) {
dotnet build (Join-Path $repo "src\Mimir.WellObsDaemon\Mimir.WellObsDaemon.csproj")
}
$daemonArgs = @(
"--well-log",
$WellLog,
"--map",
$MapName,
"--composite-stem-id",
$CompositeStemId,
"--poll-ms",
"$PollMs"
)
if ($FromStart) {
$daemonArgs += "--from-start"
}
$process = Start-Process `
-FilePath $daemonExe `
-ArgumentList $daemonArgs `
-WorkingDirectory $repo `
-WindowStyle Hidden `
-RedirectStandardOutput $stdout `
-RedirectStandardError $stderr `
-PassThru
$manifest = [ordered]@{
document = "mimir.well_obs_daemon.supervisor.v1"
pid = $process.Id
wellLog = $WellLog
mapName = $MapName
compositeStemId = $CompositeStemId
stdout = $stdout
stderr = $stderr
startedAt = (Get-Date).ToUniversalTime().ToString("O")
}
$manifestPath = Join-Path $runDir "supervisor.json"
$manifest | ConvertTo-Json -Depth 6 | Set-Content -Path $manifestPath -Encoding UTF8
Write-Output "Mimir Well OBS daemon PID $($process.Id)"
Write-Output "Supervisor manifest: $manifestPath"
Write-Output "Requires Well capture pages with inline audio bodies enabled."
Write-Output "Poll log: Get-Content -Tail 20 '$stderr'"

View file

@ -112,7 +112,7 @@ namespace Mimir {
}
public static void Capture(string outputPath, double seconds, int targetRate, int targetChannels, string roleName) {
Stream output = outputPath == "-" ? Console.OpenStandardOutput() : new FileStream(outputPath, FileMode.Create, FileAccess.Write, FileShare.Read);
Stream output = IsStdoutPath(outputPath) ? Console.OpenStandardOutput() : new FileStream(outputPath, FileMode.Create, FileAccess.Write, FileShare.Read);
try {
IMMDeviceEnumerator enumerator = (IMMDeviceEnumerator)(new MMDeviceEnumerator());
IMMDevice device;
@ -160,6 +160,12 @@ namespace Mimir {
Check(audioClient.Start(), "Start");
DateTime end = seconds > 0 ? DateTime.UtcNow.AddSeconds(seconds) : DateTime.MaxValue;
byte[] pcm = new byte[8192];
int sourceRate = Math.Max(1, (int)fmt.nSamplesPerSec);
double sourceFramesPerTargetFrame = sourceRate / (double)Math.Max(1, targetRate);
long sourceFramesSeen = 0;
double nextOutputSourceFrame = 0.0;
float[] previousFrame = new float[Math.Max(1, targetChannels)];
bool hasPreviousFrame = false;
try {
while (DateTime.UtcNow < end) {
uint packetFrames;
@ -180,24 +186,40 @@ namespace Mimir {
int sourceBits = (int)fmt.wBitsPerSample;
int sourceBytesPerSample = Math.Max(1, sourceBits / 8);
int frameBytes = targetChannels * 4;
int needed = checked((int)frames * frameBytes);
long packetStart = sourceFramesSeen;
long packetEnd = packetStart + frames;
int outputFrames = 0;
double countCursor = nextOutputSourceFrame;
while (countCursor < packetEnd) {
outputFrames++;
countCursor += sourceFramesPerTargetFrame;
}
int needed = checked(outputFrames * frameBytes);
if (pcm.Length < needed) pcm = new byte[needed];
int dst = 0;
for (int frame = 0; frame < frames; frame++) {
float[] left = new float[targetChannels];
float[] right = new float[targetChannels];
while (nextOutputSourceFrame < packetEnd) {
long leftAbsolute = (long)Math.Floor(nextOutputSourceFrame);
long rightAbsolute = Math.Min(packetEnd - 1, leftAbsolute + 1);
double fraction = nextOutputSourceFrame - leftAbsolute;
ReadFrame(leftAbsolute, packetStart, data, flags, sourceChannels, sourceBytesPerFrame, sourceBytesPerSample, sourceBits, fmt.wFormatTag, activeFormatPtr, previousFrame, hasPreviousFrame, left);
ReadFrame(rightAbsolute, packetStart, data, flags, sourceChannels, sourceBytesPerFrame, sourceBytesPerSample, sourceBits, fmt.wFormatTag, activeFormatPtr, previousFrame, hasPreviousFrame, right);
for (int ch = 0; ch < targetChannels; ch++) {
float sample = 0f;
if ((flags & AUDCLNT_BUFFERFLAGS_SILENT) == 0) {
int srcCh = Math.Min(ch, sourceChannels - 1);
IntPtr src = IntPtr.Add(data, frame * sourceBytesPerFrame + srcCh * sourceBytesPerSample);
sample = ReadSample(src, sourceBits, fmt.wFormatTag, mixFormatPtr);
}
float sample = (float)(left[ch] + (right[ch] - left[ch]) * fraction);
byte[] bytes = BitConverter.GetBytes(sample);
Buffer.BlockCopy(bytes, 0, pcm, dst, 4);
dst += 4;
}
nextOutputSourceFrame += sourceFramesPerTargetFrame;
}
if (frames > 0) {
ReadFrame(packetEnd - 1, packetStart, data, flags, sourceChannels, sourceBytesPerFrame, sourceBytesPerSample, sourceBits, fmt.wFormatTag, activeFormatPtr, previousFrame, false, previousFrame);
hasPreviousFrame = true;
}
output.Write(pcm, 0, needed);
output.Flush();
sourceFramesSeen = packetEnd;
}
finally {
Check(captureClient.ReleaseBuffer(frames), "ReleaseBuffer");
@ -216,7 +238,30 @@ namespace Mimir {
}
}
finally {
if (outputPath != "-") output.Dispose();
if (!IsStdoutPath(outputPath)) output.Dispose();
}
}
static bool IsStdoutPath(string outputPath) {
return String.Equals(outputPath, "-", StringComparison.Ordinal) ||
String.Equals(outputPath, "stdout", StringComparison.OrdinalIgnoreCase);
}
static void ReadFrame(long absoluteFrame, long packetStart, IntPtr data, uint flags, int sourceChannels, int sourceBytesPerFrame, int sourceBytesPerSample, int sourceBits, ushort tag, IntPtr formatPtr, float[] previousFrame, bool hasPreviousFrame, float[] target) {
if (absoluteFrame < packetStart) {
for (int ch = 0; ch < target.Length; ch++) target[ch] = hasPreviousFrame ? previousFrame[ch] : 0f;
return;
}
int packetFrame = checked((int)(absoluteFrame - packetStart));
for (int ch = 0; ch < target.Length; ch++) {
float sample = 0f;
if ((flags & AUDCLNT_BUFFERFLAGS_SILENT) == 0) {
int srcCh = Math.Min(ch, sourceChannels - 1);
IntPtr src = IntPtr.Add(data, packetFrame * sourceBytesPerFrame + srcCh * sourceBytesPerSample);
sample = ReadSample(src, sourceBits, tag, formatPtr);
}
target[ch] = sample;
}
}

View file

@ -6,8 +6,8 @@ import MimirThemeLock from "./quartz/components/MimirThemeLock";
const MimirGraphShell = Component.GameCultGraphSpaShell({
stylesheetHref:
"/static/epiphany-graph/assets/viewer.css?v=graph-mimir-corpus-20260524",
moduleSrc: "/static/epiphany-graph/assets/viewer.js?v=graph-mimir-corpus-20260524",
"/static/norn-graph/assets/viewer.css?v=graph-mimir-corpus-20260524",
moduleSrc: "/static/norn-graph/assets/viewer.js?v=graph-mimir-corpus-20260524",
config: {
title: "Mimir Knowledge Graph",
architectureDescription:

View file

@ -0,0 +1,12 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net10.0</TargetFramework>
<AssemblyName>Mimir.Broadcast</AssemblyName>
<RootNamespace>Mimir.Broadcast</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>

View file

@ -0,0 +1,279 @@
using System.Diagnostics;
using System.Text;
namespace Mimir.Broadcast;
internal static class Program
{
public static async Task<int> Main(string[] args)
{
var options = MimirBroadcastOptions.Parse(args);
if (options.ShowHelp)
{
Console.WriteLine(MimirBroadcastOptions.HelpText);
return 0;
}
var plan = MimirBroadcastPlan.FromOptions(options);
if (options.PrintCommand || options.DryRun)
{
Console.WriteLine(plan.CommandLine);
}
if (options.Smoke)
{
return RunSmoke(plan);
}
if (options.DryRun || options.PrintCommand)
{
return 0;
}
return await RunFfmpegAsync(options, plan).ConfigureAwait(false);
}
private static int RunSmoke(MimirBroadcastPlan plan)
{
var command = plan.CommandLine;
var ok = command.Contains("h264_nvenc", StringComparison.Ordinal) &&
command.Contains("-f flv", StringComparison.Ordinal) &&
command.Contains("rtmp://127.0.0.1:11935/live/mimir", StringComparison.Ordinal) &&
command.Contains("-g 60", StringComparison.Ordinal) &&
command.Contains("-sc_threshold 0", StringComparison.Ordinal);
Console.WriteLine($"mimir-broadcast-smoke nvenc={command.Contains("h264_nvenc", StringComparison.Ordinal)} rtmp={command.Contains("rtmp://", StringComparison.Ordinal)} keyframeCadence={command.Contains("-g 60", StringComparison.Ordinal)}");
return ok ? 0 : 1;
}
private static async Task<int> RunFfmpegAsync(MimirBroadcastOptions options, MimirBroadcastPlan plan)
{
var start = new ProcessStartInfo
{
FileName = options.FfmpegPath,
UseShellExecute = false,
RedirectStandardError = true,
RedirectStandardOutput = true,
};
foreach (var argument in plan.Arguments)
{
start.ArgumentList.Add(argument);
}
using var process = Process.Start(start);
if (process == null)
{
Console.Error.WriteLine("Could not start ffmpeg.");
return 1;
}
var stdout = PumpAsync(process.StandardOutput, Console.Out);
var stderr = PumpAsync(process.StandardError, Console.Error);
await process.WaitForExitAsync().ConfigureAwait(false);
await Task.WhenAll(stdout, stderr).ConfigureAwait(false);
return process.ExitCode;
}
private static async Task PumpAsync(StreamReader reader, TextWriter writer)
{
while (await reader.ReadLineAsync().ConfigureAwait(false) is { } line)
{
writer.WriteLine(line);
}
}
}
internal sealed record MimirBroadcastPlan(IReadOnlyList<string> Arguments)
{
public string CommandLine => $"ffmpeg {string.Join(' ', Arguments.Select(Quote))}";
public static MimirBroadcastPlan FromOptions(MimirBroadcastOptions options)
{
var args = new List<string>();
if (options.Realtime)
{
args.Add("-re");
}
if (options.TestPattern)
{
args.AddRange(["-f", "lavfi", "-i", $"testsrc2=size={options.Width}x{options.Height}:rate={options.FrameRate}"]);
args.AddRange(["-f", "lavfi", "-i", $"sine=frequency=997:sample_rate={options.AudioSampleRate}"]);
}
else if (!string.IsNullOrWhiteSpace(options.InputPath))
{
args.AddRange(["-i", options.InputPath]);
}
else
{
throw new InvalidOperationException("Use --test-pattern or provide --input.");
}
var audioMap = options.TestPattern ? "1:a:0" : "0:a:0?";
args.AddRange([
"-map", "0:v:0",
"-map", audioMap,
"-c:v", "h264_nvenc",
"-preset", options.NvencPreset,
"-tune", "ll",
"-rc", "cbr",
"-b:v", $"{options.VideoBitrateKbps}k",
"-maxrate", $"{options.VideoBitrateKbps}k",
"-bufsize", $"{options.VideoBitrateKbps * 2}k",
"-g", options.GopFrames.ToStringInvariant(),
"-keyint_min", options.GopFrames.ToStringInvariant(),
"-sc_threshold", "0",
"-pix_fmt", "yuv420p",
"-c:a", "aac",
"-b:a", $"{options.AudioBitrateKbps}k",
"-ar", options.AudioSampleRate.ToStringInvariant(),
]);
if (options.DurationSeconds > 0)
{
args.AddRange(["-t", options.DurationSeconds.ToStringInvariant()]);
}
args.AddRange([
"-f", "flv",
options.TargetUrl,
]);
return new MimirBroadcastPlan(args);
}
private static string Quote(string value)
{
if (value.Length == 0)
{
return "\"\"";
}
return value.Any(static c => char.IsWhiteSpace(c) || c == '"')
? $"\"{value.Replace("\"", "\\\"", StringComparison.Ordinal)}\""
: value;
}
}
internal sealed record MimirBroadcastOptions
{
public string FfmpegPath { get; init; } = "ffmpeg";
public string TargetUrl { get; init; } = "rtmp://127.0.0.1:11935/live/mimir";
public string InputPath { get; init; } = "";
public bool TestPattern { get; init; } = true;
public bool Realtime { get; init; } = true;
public bool PrintCommand { get; init; }
public bool DryRun { get; init; }
public bool Smoke { get; init; }
public bool ShowHelp { get; init; }
public int Width { get; init; } = 1280;
public int Height { get; init; } = 720;
public int FrameRate { get; init; } = 30;
public int GopFrames { get; init; } = 60;
public int VideoBitrateKbps { get; init; } = 4500;
public int AudioSampleRate { get; init; } = 48_000;
public int AudioBitrateKbps { get; init; } = 160;
public string NvencPreset { get; init; } = "p4";
public int DurationSeconds { get; init; }
public static string HelpText => """
Mimir.Broadcast
Push a Starfire-encoded Mimir program stream to the Yggdrasil RTMP/HLS edge.
The default command is a synthetic NVENC smoke source:
dotnet run --project src/Mimir.Broadcast -- --print-command
Options:
--target <url> RTMP target. Default: rtmp://127.0.0.1:11935/live/mimir
--ffmpeg <path> FFmpeg executable. Default: ffmpeg
--input <path> Input media path instead of the synthetic test pattern.
--no-test-pattern Require --input.
--width <px> Synthetic width. Default: 1280
--height <px> Synthetic height. Default: 720
--fps <n> Synthetic frame rate. Default: 30
--video-kbps <n> CBR video bitrate. Default: 4500
--audio-kbps <n> AAC bitrate. Default: 160
--gop <frames> Keyframe cadence. Default: 60
--preset <name> NVENC preset. Default: p4
--duration-seconds <n> Stop after this many seconds. Default: run until stopped.
--print-command Print the generated FFmpeg command.
--dry-run Print command and exit.
--smoke Validate the generated command shape and exit.
""";
public static MimirBroadcastOptions Parse(IReadOnlyList<string> args)
{
var options = new MimirBroadcastOptions();
for (var index = 0; index < args.Count; index++)
{
var arg = args[index];
options = arg switch
{
"--help" or "-h" => options with { ShowHelp = true },
"--print-command" => options with { PrintCommand = true },
"--dry-run" => options with { DryRun = true, PrintCommand = true },
"--smoke" => options with { Smoke = true },
"--no-test-pattern" => options with { TestPattern = false },
"--target" => options with { TargetUrl = RequiredValue(args, ref index, arg) },
"--ffmpeg" => options with { FfmpegPath = RequiredValue(args, ref index, arg) },
"--input" => options with { InputPath = RequiredValue(args, ref index, arg), TestPattern = false },
"--width" => options with { Width = ParsePositiveInt(args, ref index, arg) },
"--height" => options with { Height = ParsePositiveInt(args, ref index, arg) },
"--fps" => options with { FrameRate = ParsePositiveInt(args, ref index, arg) },
"--video-kbps" => options with { VideoBitrateKbps = ParsePositiveInt(args, ref index, arg) },
"--audio-kbps" => options with { AudioBitrateKbps = ParsePositiveInt(args, ref index, arg) },
"--gop" => options with { GopFrames = ParsePositiveInt(args, ref index, arg) },
"--preset" => options with { NvencPreset = RequiredValue(args, ref index, arg) },
"--duration-seconds" => options with { DurationSeconds = ParsePositiveInt(args, ref index, arg) },
_ => throw new ArgumentException($"Unknown argument `{arg}`."),
};
}
return options;
}
private static string RequiredValue(IReadOnlyList<string> args, ref int index, string option)
{
if (index + 1 >= args.Count || args[index + 1].StartsWith("-", StringComparison.Ordinal))
{
throw new ArgumentException($"{option} requires a value.");
}
index++;
return args[index];
}
private static int ParsePositiveInt(IReadOnlyList<string> args, ref int index, string option)
{
var text = RequiredValue(args, ref index, option);
if (!int.TryParse(text, out var value) || value <= 0)
{
throw new ArgumentException($"{option} requires a positive integer.");
}
return value;
}
}
internal static class InvariantFormatting
{
public static string ToStringInvariant(this int value) =>
value.ToString(System.Globalization.CultureInfo.InvariantCulture);
}

Some files were not shown because too many files have changed in this diff Show more