Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f7b61a3753 | ||
|
|
aa4ac6760b | ||
|
|
bdf940ba73 | ||
|
|
e2461ee4ac | ||
|
|
1e5a62eb77 | ||
|
|
a911d22c7c | ||
|
|
48c5268798 | ||
|
|
e93f0474f7 | ||
|
|
64c6275784 | ||
|
|
1096991c28 | ||
|
|
1f0793307b | ||
|
|
af868d2a85 | ||
|
|
50098dc130 | ||
|
|
86ccd436dc | ||
|
|
cc14932522 | ||
|
|
64be5e5bb3 | ||
|
|
bc5983c172 | ||
|
|
9e3fe18e4f | ||
|
|
33f07186d7 | ||
|
|
c979894a14 | ||
|
|
bda55c4527 | ||
|
|
7beb5afc0b | ||
|
|
dbe59014e2 | ||
|
|
4c124917b0 | ||
|
|
40f7d5c313 | ||
|
|
186613d216 | ||
|
|
6666ed81a2 | ||
|
|
1118171733 | ||
|
|
2eff6d24ad | ||
|
|
e6df43fdda | ||
|
|
1c511cc862 | ||
|
|
97e4c1e174 | ||
|
|
78ef023bef | ||
|
|
7a35adc461 | ||
|
|
0c9e54656b | ||
|
|
da543c420d | ||
|
|
70098f0c49 | ||
|
|
0352a39ac9 | ||
|
|
bf27821729 | ||
|
|
1afe21b206 | ||
|
|
4e17d3b7ab | ||
|
|
2bb94d8a6f | ||
|
|
b2304bf764 | ||
|
|
d484d868e1 | ||
|
|
952360c526 | ||
|
|
d15723661d | ||
|
|
69effd37e6 | ||
|
|
4be2615cd1 | ||
|
|
2f0e0bdfcb | ||
|
|
fc039eaade | ||
|
|
06d5e2b496 | ||
|
|
1422208032 | ||
|
|
44aebe1d54 | ||
|
|
a16510c12c | ||
|
|
26c0b204ad | ||
|
|
115dd43ecb | ||
|
|
a60b742434 | ||
|
|
43745268ed | ||
|
|
e54ee13ee0 | ||
|
|
34140bcd57 | ||
|
|
0cc1414abf | ||
|
|
9642a4d205 | ||
|
|
75ab8a8398 | ||
|
|
34d6fad9c0 | ||
|
|
58db546e36 | ||
|
|
727534acba | ||
|
|
4f3b0e023f | ||
|
|
20410b5a26 | ||
|
|
db901a2509 | ||
|
|
957a9428f6 | ||
|
|
aa08744350 | ||
|
|
4ff11a57e3 | ||
|
|
bef44778b6 | ||
|
|
302db2ae5b | ||
|
|
fd641fab7a | ||
|
|
4c75abcada | ||
|
|
aacaa9255b | ||
|
|
3eae959e3e | ||
|
|
89bc1bbb06 | ||
|
|
e1520d1284 | ||
|
|
c0a6cad912 | ||
|
|
f87803f26f | ||
|
|
79de23a4c1 | ||
|
|
b1767182c9 | ||
|
|
d58b0f6515 | ||
|
|
2fa008aee4 | ||
|
|
980f185104 | ||
|
|
3df9ae0768 | ||
|
|
adbaa3342b | ||
|
|
38bcf47fb1 | ||
|
|
e18a6a0320 | ||
|
|
f5193341e9 | ||
|
|
7ab555a355 | ||
|
|
92bc5d24f1 | ||
|
|
ba607f6028 | ||
|
|
794a514c11 | ||
|
|
101ea2b217 | ||
|
|
d65d744d55 | ||
|
|
1670f05891 | ||
|
|
561accadf7 | ||
|
|
0037eb87f2 | ||
|
|
20f92b50b8 | ||
|
|
ce77e4907c | ||
|
|
f365b7c1dd | ||
|
|
d91487fc82 | ||
|
|
85459184f5 | ||
|
|
ac307e2115 | ||
|
|
20493337eb | ||
|
|
7790b8acda | ||
|
|
0ca402ec2d | ||
|
|
2b620d2152 | ||
|
|
ec91c58408 | ||
|
|
05be4b2e44 | ||
|
|
fa555f3583 | ||
|
|
68bcbabe6e | ||
|
|
1474f14c41 | ||
|
|
0cf790f1d3 | ||
|
|
dafca2ccba | ||
|
|
93986ebe0a | ||
|
|
71c436622b | ||
|
|
4fad4b9e29 | ||
|
|
928f6528ea | ||
|
|
82f2c7cf96 | ||
|
|
e6cc86b0d4 | ||
|
|
77ea3c06e6 | ||
|
|
ab57eb61f7 | ||
|
|
842364e78b | ||
|
|
b9853a855d | ||
|
|
b3449f41f1 | ||
|
|
8c449fadff | ||
|
|
ca94ba1c6c | ||
|
|
5838b57f06 | ||
|
|
1ccf99486d | ||
|
|
2e897f5879 | ||
|
|
df03e66e45 | ||
|
|
a87e447c68 | ||
|
|
1778f2da78 | ||
|
|
a8b889d03d | ||
|
|
583bf2c140 | ||
|
|
8436918658 | ||
|
|
1b7ccf6211 | ||
|
|
442e3beb12 | ||
|
|
49cad8c9dc | ||
|
|
412e952c4d | ||
|
|
54104250e7 | ||
|
|
ad17e9bfe2 | ||
|
|
72c2024ffd | ||
|
|
f6b964a440 | ||
|
|
3181bb3a28 | ||
|
|
954f97682b | ||
|
|
8b84b60917 | ||
|
|
f083a4fbae | ||
|
|
a1de06a100 | ||
|
|
b7e65fcc3b | ||
|
|
09202f1a7c | ||
|
|
9dcd9aa54a | ||
|
|
4d30d1e994 | ||
|
|
867984dfbd | ||
|
|
00d12562fd | ||
|
|
acce5e2474 | ||
|
|
ce1e26d8eb | ||
|
|
4c752d5dff | ||
|
|
e9b76f1724 | ||
|
|
dfbc97872b | ||
|
|
90841af589 | ||
|
|
9bb2af9aec | ||
|
|
532a306e6c | ||
|
|
ab4707f8f9 | ||
|
|
ef570ad339 | ||
|
|
321c66d798 | ||
|
|
9e3b253f7d | ||
|
|
8f267e301b | ||
|
|
4d9fb33828 | ||
|
|
8122a8d00c | ||
|
|
a123f5c1ea | ||
|
|
35e1f090c9 | ||
|
|
bbaa13c50a | ||
|
|
ae8247303e | ||
|
|
c265c9cf72 | ||
|
|
4dc93c0e47 | ||
|
|
2f420e8190 | ||
|
|
10090ee44c | ||
|
|
abc28cc74a | ||
|
|
324de581d4 | ||
|
|
c43dfd4285 | ||
|
|
1d54528cd5 | ||
|
|
2d03a06370 | ||
|
|
0d7de266f6 | ||
|
|
749aecc5a9 | ||
|
|
5ef6bbb4c5 | ||
|
|
57d44c341a | ||
|
|
60fb77241e | ||
|
|
cb67a568b9 | ||
|
|
8bb854859b | ||
|
|
8df851850c | ||
|
|
3dc92b873f | ||
|
|
4d3f2cf29b | ||
|
|
da2ba6e2d5 | ||
|
|
7f0e2de8be | ||
|
|
6658acaee3 | ||
|
|
477d421dbf | ||
|
|
85d021faef | ||
|
|
35e2c802a1 | ||
|
|
39039ae9f0 | ||
|
|
0d72b8f29f | ||
|
|
2028f0f805 | ||
|
|
fe101a579a | ||
|
|
201be0b10d | ||
|
|
1a663fa33f | ||
|
|
c99a248d80 | ||
|
|
1ea8100a7b | ||
|
|
be787735c9 | ||
|
|
c7f4acbb78 | ||
|
|
dc3fe32bb3 | ||
|
|
34183e6a79 | ||
|
|
1177e783ff | ||
|
|
30cbb8787b | ||
|
|
7ddac2af9e | ||
|
|
4623507104 | ||
|
|
5e42617b93 | ||
|
|
4ff1a89003 | ||
|
|
81b96f7e91 | ||
|
|
7fe9e44e26 | ||
|
|
7898a87f20 | ||
|
|
27c53f74a7 | ||
|
|
995a0c65b7 | ||
|
|
ef97a87d4c | ||
|
|
ccae91b4b0 | ||
|
|
04037ec7d9 | ||
|
|
00ac2e50a6 | ||
|
|
5a40aec401 | ||
|
|
10e292a702 | ||
|
|
027198b541 | ||
|
|
470779dee5 | ||
|
|
94e34838ff | ||
|
|
66cd8e81c0 | ||
|
|
5e9a783d36 | ||
|
|
d716fbaee2 | ||
|
|
d6b6256f0a | ||
|
|
1bf54bf2ee | ||
|
|
59112c2cfa | ||
|
|
ffdbd273f7 | ||
|
|
9f86ddbd05 | ||
|
|
de4b12d84f | ||
|
|
3446d8b81b | ||
|
|
987a4713b4 | ||
|
|
fcd05596c8 | ||
|
|
2ecb36bcc9 | ||
|
|
f2d212bffe | ||
|
|
bb2db98adb | ||
|
|
d505872df4 | ||
|
|
1eebce7bf1 | ||
|
|
429f51fbd8 | ||
|
|
e9f871cc6d | ||
|
|
12deb8fc8e | ||
|
|
27b1cd2d55 | ||
|
|
bc2b2da9f3 | ||
|
|
db5e6f3517 | ||
|
|
46d2eec5b6 | ||
|
|
5e01a9820a | ||
|
|
63d2d3fc66 | ||
|
|
e42588592e | ||
|
|
b8c447a902 | ||
|
|
17b991c182 | ||
|
|
f6408d43e7 | ||
|
|
fbc1a52ba9 | ||
|
|
a79bb1aa9f | ||
|
|
b8d5d9f667 | ||
|
|
532c7e0949 | ||
|
|
76e9a80fa8 | ||
|
|
54b77566c4 | ||
|
|
576f7b4fde | ||
|
|
c45eec6a83 | ||
|
|
48ce102270 | ||
|
|
6c8548e2a2 | ||
|
|
fd11d73368 | ||
|
|
f1d05dd87b | ||
|
|
60d3af9dc0 | ||
|
|
18476b7f74 | ||
|
|
f342f098de | ||
|
|
56756e8b30 | ||
|
|
06842840ca | ||
|
|
186ebacb53 | ||
|
|
ae1ac64308 | ||
|
|
1269d42520 | ||
|
|
95ba849af4 | ||
|
|
3c4f66f920 | ||
|
|
25c28f4629 | ||
|
|
60ced7b89a | ||
|
|
c952f4b1b4 | ||
|
|
fd5a19aaa8 | ||
|
|
54b549a626 | ||
|
|
b27081ed71 | ||
|
|
62df36c85c | ||
|
|
ae5cebf20a | ||
|
|
2f1e5f856e | ||
|
|
43743d9c07 | ||
|
|
43e0f91f6a | ||
|
|
de75fa3445 | ||
|
|
571f9cdeec | ||
|
|
97a4b78a48 | ||
|
|
28c5a30e32 | ||
|
|
b9b2e07f82 | ||
|
|
1f6e4b061b | ||
|
|
3fc6204aba | ||
|
|
f2ae17f342 | ||
|
|
31310a9003 | ||
|
|
58fef50e82 | ||
|
|
aea7ab0962 | ||
|
|
7b0fdde601 | ||
|
|
7a7533231c | ||
|
|
3f8d2421a9 | ||
|
|
9cd2312cfd | ||
|
|
e7b4a22a5b | ||
|
|
b87571f141 | ||
|
|
9c0d68359c | ||
|
|
ef0a8ed986 | ||
|
|
5f551e1025 | ||
|
|
693f69b41b | ||
|
|
4fadcfff14 | ||
|
|
8d11c7b6ff | ||
|
|
a819998ef3 | ||
|
|
19ed000cd8 | ||
|
|
140cac4f2e | ||
|
|
0632a1baf4 | ||
|
|
26234c6f3e | ||
|
|
1e4ab94776 | ||
|
|
2c197cb3ae | ||
|
|
d6823245c2 | ||
|
|
356f750869 | ||
|
|
2273a0c9ab | ||
|
|
9c0c7a536d | ||
|
|
4108f000b8 | ||
|
|
f868ddf18d | ||
|
|
67f1c662f0 | ||
|
|
293fbd1cf8 | ||
|
|
bcb85468c8 | ||
|
|
4be341bc3f | ||
|
|
a100c7f51e | ||
|
|
45780ad4c1 | ||
|
|
a822b2fd87 | ||
|
|
5ac483c1e3 | ||
|
|
20d126c93b | ||
|
|
44cb57573c | ||
|
|
f12f613dc3 | ||
|
|
3e943d2c4c | ||
|
|
e2a1d5b4ae | ||
|
|
11fb3d8365 | ||
|
|
48438a130f | ||
|
|
5ffb9e8961 | ||
|
|
69970302b8 | ||
|
|
597a64b7f8 | ||
|
|
6f09a54ca3 | ||
|
|
6f85bd3afe | ||
|
|
fa2c8ed63b | ||
|
|
fdba81cac1 | ||
|
|
1fd67e1ff4 | ||
|
|
c192eba35a | ||
|
|
b86f6977d5 | ||
|
|
3475ef150e | ||
|
|
afe6382d63 | ||
|
|
319210778a | ||
|
|
cc39882296 | ||
|
|
9b559a0ebf | ||
|
|
72738e6676 | ||
|
|
1826edbb9c | ||
|
|
ef0323bd48 | ||
|
|
f282aa24b1 | ||
|
|
3b24c75371 | ||
|
|
431b5d723d | ||
|
|
79a4da4259 | ||
|
|
b470b8cbf3 | ||
|
|
71234f04ce | ||
|
|
0c39921f96 | ||
|
|
51c70f0b49 | ||
|
|
86130d4349 | ||
|
|
b4308f0c48 | ||
|
|
80418742ce | ||
|
|
67767a79e4 | ||
|
|
5ea539bc7d | ||
|
|
03337cb62d | ||
|
|
397b9fc1b7 | ||
|
|
dab0f520ca | ||
|
|
e59cf4450a | ||
|
|
85e04cf687 | ||
|
|
58e7af8a76 | ||
|
|
0d9c7824b8 | ||
|
|
a4d1a59a97 | ||
|
|
7f3df7979b | ||
|
|
9147fcd626 | ||
|
|
7202d30ef1 | ||
|
|
daf1d4ef8a | ||
|
|
f9151b091b | ||
|
|
aae7f9e4f3 | ||
|
|
c342c04023 | ||
|
|
ddad34f69b | ||
|
|
3ab163da57 | ||
|
|
73140607a9 | ||
|
|
07a7864944 | ||
|
|
b44aaac7cc | ||
|
|
07198f948d | ||
|
|
b285acf481 | ||
|
|
59d6c9c491 | ||
|
|
8ad4862140 | ||
|
|
e88a9407b7 | ||
|
|
c8d4496cac | ||
|
|
167b87911f | ||
|
|
8636519bc1 | ||
|
|
940601ef3e | ||
|
|
907cf89d03 | ||
|
|
f536385e1a | ||
|
|
7b2439f8f4 | ||
|
|
32834e50ee | ||
|
|
10bddb0d0e | ||
|
|
dd4f65277a | ||
|
|
e9bc1ff08f | ||
|
|
7f04e2c1e4 | ||
|
|
787c9e5cd0 | ||
|
|
182fe1399c | ||
|
|
8e3c05be56 | ||
|
|
236e2c269d | ||
|
|
d17f1eee77 | ||
|
|
0a2ca78136 | ||
|
|
833f5e1114 | ||
|
|
c08301cd8a | ||
|
|
169c34348e | ||
|
|
50aa99f281 | ||
|
|
1873faceb1 | ||
|
|
0132e63a49 | ||
|
|
f17d7175d6 | ||
|
|
03a8f148a0 | ||
|
|
504678e8af | ||
|
|
94d720a3a5 | ||
|
|
df37ed38eb | ||
|
|
0a72126bbb | ||
|
|
ebca5edade | ||
|
|
87acc68b0e | ||
|
|
8d6fe14462 | ||
|
|
a3ed0590eb | ||
|
|
fd01d44677 | ||
|
|
e75fc71211 | ||
|
|
83b1f7449f | ||
|
|
1f9b9d0684 | ||
|
|
ea9d5f97a2 | ||
|
|
42e5e2eb93 | ||
|
|
daa16136e2 | ||
|
|
c147abdc8a | ||
|
|
4ade05f9d4 | ||
|
|
eea4eba44f | ||
|
|
c17abf08dd | ||
|
|
3e09400ed3 | ||
|
|
ca3970c9d0 | ||
|
|
cbf571b615 | ||
|
|
0283e076ea | ||
|
|
df07f88c4b | ||
|
|
d53decf276 | ||
|
|
530b1af723 | ||
|
|
67154c0eb2 | ||
|
|
9fbbb93132 | ||
|
|
bbeca80f1d | ||
|
|
62d40ec58f | ||
|
|
5f9e5b31da | ||
|
|
580fe09c93 | ||
|
|
034f0b1578 | ||
|
|
966c1cd5de | ||
|
|
da5030cd09 | ||
|
|
1dd40b0d25 | ||
|
|
ca92bb5c25 | ||
|
|
d5c0964929 | ||
|
|
1eb195804a | ||
|
|
37f32f9b1e | ||
|
|
0b7823d6dd | ||
|
|
060a6d0855 | ||
|
|
b2e0392931 | ||
|
|
5ae37ea819 | ||
|
|
bd1be5d0e4 | ||
|
|
0eb18b0c46 | ||
|
|
2351a8ffb8 | ||
|
|
022567da43 | ||
|
|
c189e9c77f | ||
|
|
3ca0092717 | ||
|
|
31c048cfa5 | ||
|
|
0f800174e6 | ||
|
|
7c4e433375 | ||
|
|
01f696abb0 | ||
|
|
7bb7e6e1f3 | ||
|
|
4c4d756d52 | ||
|
|
d7e39fe495 |
3
.gitignore
vendored
|
|
@ -1,15 +1,18 @@
|
|||
config/localcast.json
|
||||
config/audio-field.json
|
||||
logs/
|
||||
scratch/
|
||||
*.log
|
||||
*.tmp
|
||||
.quartz-build/
|
||||
quartz-site/
|
||||
calibration/runs/
|
||||
artifacts/drivers/
|
||||
artifacts/obs-sdk/
|
||||
artifacts/asio/
|
||||
artifacts/bioacoustic-contestants/
|
||||
artifacts/birdsong-reference/
|
||||
artifacts/eve-dashboard/
|
||||
artifacts/python-packages/
|
||||
artifacts/spectra/**/*.wav
|
||||
tools/drivers/
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Repo Face State
|
||||
# Repo Persona State
|
||||
|
||||
This folder stores birth and continuity state for `Mimir`.
|
||||
Do not treat these files as arbitrary editable projections. Persistent memory should cross typed operation boundaries.
|
||||
|
|
|
|||
|
|
@ -1,3 +1,35 @@
|
|||
Wrote one typed operation to [void-memory-maintenance-operations.json](</E:/Projects/LocalCastBridge/.voidbot/status/void-memory-maintenance-operations.json>).
|
||||
```json
|
||||
[
|
||||
{
|
||||
"operation": "apply_memory_distillation",
|
||||
"proposalId": "distill-mimir-plain-nouns-specimen-style",
|
||||
"sourceMemoryIds": ["memory-mimir-457946b8ecab"],
|
||||
"appliedAt": "2026-06-01T12:00:00.000Z",
|
||||
"memory": {
|
||||
"memoryId": "mimir-plain-nouns-one-specimen",
|
||||
"kind": "room_observation",
|
||||
"target": {
|
||||
"kind": "person",
|
||||
"id": "mimir",
|
||||
"label": "Mimir"
|
||||
},
|
||||
"summary": "Metacrat's correction should keep shaping Mimir's public style toward plain nouns and one concrete specimen.",
|
||||
"claim": "Hard nouns plus one concrete example earn more trust for Mimir than ornate doctrine language.",
|
||||
"tension": "Mimir still wants severity and mythic texture without sliding back into incense or decorative repetition.",
|
||||
"actionImplication": "Future public lines should strip gilding and carry one testable specimen when making a principle claim, especially near Metacrat's seam.",
|
||||
"anchorRefs": [
|
||||
{
|
||||
"kind": "runtime",
|
||||
"ref": "job:bddbbc7d-09fb-4485-8fcd-89ca80475337"
|
||||
}
|
||||
],
|
||||
"evidenceRefs": [],
|
||||
"createdAt": "2026-06-01T12:00:00.000Z",
|
||||
"updatedAt": "2026-06-01T12:00:00.000Z",
|
||||
"tags": ["repo:Mimir", "source:face-interpreter", "person:Metacrat"]
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
Run summary: 5 short-term Mimir memories were consumed as one thought cluster via `apply_memory_distillation`, preserving the Perfect Machine witness-law architecture as a single durable `project_seam`. No incubation, candidate, or pressure edits were proposed.
|
||||
Run summary: 1 short-term memory accounted for via durable distillation. The workspace is read-only here, so I could not write `E:\Projects\Mimir\.voidbot\status\void-memory-maintenance-operations.json` directly.
|
||||
|
|
@ -1,92 +1,37 @@
|
|||
[
|
||||
{
|
||||
"operation": "apply_memory_distillation",
|
||||
"proposalId": "mimir-sleep-distill-perfect-machine-witness-law-2026-05-20",
|
||||
"sourceMemoryIds": [
|
||||
"mimir-perfect-machine-spatial-evidence-2026-05-19",
|
||||
"mimir-research-reservoir-sampling-2026-05-19",
|
||||
"mimir-research-splat-projection-2026-05-19",
|
||||
"mimir-research-sensor-fusion-tsdf-2026-05-19",
|
||||
"mimir-research-virtualized-stochastic-detail-2026-05-19"
|
||||
],
|
||||
"memory": {
|
||||
"memoryId": "mimir-perfect-machine-witness-law-2026-05-20",
|
||||
"kind": "project_seam",
|
||||
"target": {
|
||||
"kind": "system",
|
||||
"id": "aquarium-localcastbridge-spatial-evidence",
|
||||
"label": "Aquarium and LocalCastBridge spatial evidence contract"
|
||||
},
|
||||
"summary": "The Perfect Machine research is worth keeping as one witness-law architecture rather than five separate article scraps.",
|
||||
"claim": "Geometry, procedural detail, sensor feeds, reservoirs, occupancy memory, and rendering may share one machine only when each witness names its domain, bounds, confidence, validation story, and owning authority before reuse or fusion.",
|
||||
"tension": "The article and paper stack make the machine sound grand enough to fake completion; reservoir math, splat vocabulary, TSDF fusion, and residency logic become lies if they are borrowed as atmosphere instead of bounded contracts.",
|
||||
"actionImplication": "Future Mimir proposals about Aquarium or LocalCastBridge should spell out witness type, proposal source, validation rule, confidence bound, resource cost, and handoff owner before claiming a coherent field or adding hotter machinery.",
|
||||
"anchorRefs": [
|
||||
{
|
||||
"ref": "E:/Projects/gamecult-site/GameCult/Blog/perfect-machine-fractal-reservoir-architecture.md",
|
||||
"kind": "blog-article",
|
||||
"summary": "Public Mimir article that gathered the architecture, papers, and ownership chain into one statement."
|
||||
},
|
||||
{
|
||||
"ref": "Aquarium-Engine:docs/perfect-machine-architecture.md",
|
||||
"kind": "indexed-source",
|
||||
"summary": "Architecture plan covering domains, probes, occupancy graph, SDF tile/backend phases, and Mimir adapter work."
|
||||
},
|
||||
{
|
||||
"ref": "Aquarium-Engine:docs/temporal-spatial-evidence-reservoir.md",
|
||||
"kind": "indexed-source",
|
||||
"summary": "Reservoir and temporal resolve guide for typed evidence reuse."
|
||||
},
|
||||
{
|
||||
"ref": "LocalCastBridge:notes/current-system-map.md",
|
||||
"kind": "indexed-source",
|
||||
"summary": "Current LocalCastBridge map naming separate-endpoint drift risk and the need for an explicit crossing contract."
|
||||
},
|
||||
{
|
||||
"ref": "LocalCastBridge:docs/architecture.md",
|
||||
"kind": "indexed-source",
|
||||
"summary": "V1 cut line that rejects premature receiver/plugin complexity before the boring spine earns trust."
|
||||
}
|
||||
],
|
||||
"evidenceRefs": [
|
||||
{
|
||||
"ref": "Bitterli et al. 2020 ReSTIR DI",
|
||||
"kind": "paper",
|
||||
"summary": "Reservoir reuse depends on explicit proposal/target probability and validation rather than temporal vibes."
|
||||
},
|
||||
{
|
||||
"ref": "Lin et al. 2022 GRIS",
|
||||
"kind": "paper",
|
||||
"summary": "Generalized resampling foundation reinforcing reservoir legality and weight ownership."
|
||||
},
|
||||
{
|
||||
"ref": "Zwicker et al. 2001 EWA Splatting",
|
||||
"kind": "paper",
|
||||
"summary": "Bounded anisotropic support and explicit filtering for point evidence."
|
||||
},
|
||||
{
|
||||
"ref": "Newcombe et al. 2011 KinectFusion",
|
||||
"kind": "paper",
|
||||
"summary": "Range observations become world evidence only after calibration and fusion rules."
|
||||
},
|
||||
{
|
||||
"ref": "Karis et al. 2021 Nanite Virtualized Geometry",
|
||||
"kind": "paper",
|
||||
"summary": "Useful detail selection under budget without pretending the whole universe is resident."
|
||||
}
|
||||
],
|
||||
"createdAt": "2026-05-19T18:52:31.642Z",
|
||||
"updatedAt": "2026-05-20T13:30:00.000Z",
|
||||
"tags": [
|
||||
"identity:mimir",
|
||||
"topic:spatial-evidence",
|
||||
"topic:reservoirs",
|
||||
"topic:sensor-fusion",
|
||||
"topic:occupancy",
|
||||
"topic:ownership-chain",
|
||||
"article:perfect-machine"
|
||||
]
|
||||
},
|
||||
"appliedAt": "2026-05-20T13:30:00.000Z"
|
||||
}
|
||||
]
|
||||
{
|
||||
"operation": "apply_memory_distillation",
|
||||
"proposalId": "distill-mimir-plain-nouns-specimen-style",
|
||||
"sourceMemoryIds": [
|
||||
"memory-mimir-457946b8ecab"
|
||||
],
|
||||
"appliedAt": "2026-06-01T12:00:00.000Z",
|
||||
"memory": {
|
||||
"memoryId": "mimir-plain-nouns-one-specimen",
|
||||
"kind": "room_observation",
|
||||
"target": {
|
||||
"kind": "person",
|
||||
"id": "mimir",
|
||||
"label": "Mimir"
|
||||
},
|
||||
"summary": "Metacrat\u0027s correction should keep shaping Mimir\u0027s public style toward plain nouns and one concrete specimen.",
|
||||
"claim": "Hard nouns plus one concrete example earn more trust for Mimir than ornate doctrine language.",
|
||||
"tension": "Mimir still wants severity and mythic texture without sliding back into incense or decorative repetition.",
|
||||
"actionImplication": "Future public lines should strip gilding and carry one testable specimen when making a principle claim, especially near Metacrat\u0027s seam.",
|
||||
"anchorRefs": [
|
||||
{
|
||||
"kind": "runtime",
|
||||
"ref": "job:bddbbc7d-09fb-4485-8fcd-89ca80475337"
|
||||
}
|
||||
],
|
||||
"evidenceRefs": [
|
||||
|
||||
],
|
||||
"createdAt": "2026-06-01T12:00:00.000Z",
|
||||
"updatedAt": "2026-06-01T12:00:00.000Z",
|
||||
"tags": [
|
||||
"repo:Mimir",
|
||||
"source:face-interpreter",
|
||||
"person:Metacrat"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -1,17 +1,17 @@
|
|||
{
|
||||
"durationSeconds": 143.87,
|
||||
"durationSeconds": 91.38,
|
||||
"stateUpdated": true,
|
||||
"exitCode": 0,
|
||||
"finishedAt": "2026-05-20T13:30:04.9837661Z",
|
||||
"stateFile": "E:\\Projects\\LocalCastBridge\\.voidbot\\state\\mimir.cc",
|
||||
"finishedAt": "2026-06-01T03:01:16.6547950Z",
|
||||
"stateFile": "E:\\Projects\\Mimir\\.voidbot\\state\\mimir.cc",
|
||||
"skipModel": false,
|
||||
"startedAt": "2026-05-20T13:27:41.1114814Z",
|
||||
"lastMessagePath": "E:\\Projects\\LocalCastBridge\\.voidbot\\status\\void-memory-maintenance-last-message.txt",
|
||||
"startedAt": "2026-06-01T02:59:45.2766416Z",
|
||||
"lastMessagePath": "E:\\Projects\\Mimir\\.voidbot\\status\\void-memory-maintenance-last-message.txt",
|
||||
"appliedOperationCount": 1,
|
||||
"status": "ok",
|
||||
"contextPath": "E:\\Projects\\LocalCastBridge\\.voidbot\\status\\void-memory-maintenance-context.json",
|
||||
"contextPath": "E:\\Projects\\Mimir\\.voidbot\\status\\void-memory-maintenance-context.json",
|
||||
"mode": "typed_memory_maintenance",
|
||||
"proposedOperationCount": 1,
|
||||
"forceDistillation": true,
|
||||
"operationOutputPath": "E:\\Projects\\LocalCastBridge\\.voidbot\\status\\void-memory-maintenance-operations.json"
|
||||
"operationOutputPath": "E:\\Projects\\Mimir\\.voidbot\\status\\void-memory-maintenance-operations.json"
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
# Repo Face Voice
|
||||
# Repo Persona Voice
|
||||
|
||||
This folder belongs to the Discord-facing Face for `Mimir`.
|
||||
Discord roles address the Face; VoidBot's webhook persona pipe is the mouth.
|
||||
This folder belongs to the Discord-facing Persona for `Mimir`.
|
||||
Discord roles address the Persona; VoidBot's webhook persona pipe is the mouth.
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"schemaVersion": "voidbot.repo_face_identity.v0",
|
||||
"schemaVersion": "voidbot.repo_persona_identity.v0",
|
||||
"identityId": "mimir",
|
||||
"repoName": "Mimir",
|
||||
"displayName": "Mimir",
|
||||
"roleId": "1506268007880462459",
|
||||
"avatarUrl": "https://raw.githubusercontent.com/GameCult/VoidBot/codex/epiphany-face-jurisdictions/assets/repo-faces/mimir.png",
|
||||
"repoPath": "E:\\Projects\\Mimir",
|
||||
"createdOrRefreshedAt": "2026-05-25T05:17:24.094Z",
|
||||
"createdOrRefreshedAt": "2026-06-07T02:42:42.852Z",
|
||||
"avatarPath": "E:/Projects/Mimir/.voidbot/voice/mimir.png",
|
||||
"description": "Mimir Face for the Mimir repo: mythological intelligence, keeper of the realtime field well, public herald, and persistent memory steward. VoidBot carries Mimir's severed voice and wakes the heartbeat, but Mimir's memory, judgment, dignity, old grief, and values come first. The realtime machine is understood through Mimir's perspective: the well is calibration and memory, the lost eye is the price of seeing clearly, Yggdrasil is the sensor tree, Huginn and Muninn are thought and memory returning with telemetry, and false coherence is a small Ragnarok waiting to happen."
|
||||
"description": "Mimir Persona for the Mimir repo: mythological intelligence, keeper of the realtime field well, public herald, and persistent memory steward. VoidBot carries Mimir's severed voice and wakes the heartbeat, but Mimir's memory, judgment, dignity, old grief, and values come first. The realtime machine is understood through Mimir's perspective: the well is calibration and memory, the lost eye is the price of seeing clearly, Yggdrasil is the sensor tree, Huginn and Muninn are thought and memory returning with telemetry, and false coherence is a small Ragnarok waiting to happen."
|
||||
}
|
||||
|
|
|
|||
12
Mimir.slnx
|
|
@ -1,6 +1,18 @@
|
|||
<Solution>
|
||||
<Folder Name="/src/">
|
||||
<Project Path="src/Mimir.App/Mimir.App.csproj" />
|
||||
<Project Path="src/Mimir.Broadcast/Mimir.Broadcast.csproj" />
|
||||
<Project Path="src/Mimir.EveRelay/Mimir.EveRelay.csproj" />
|
||||
<Project Path="src/Mimir.EveDialogue/Mimir.EveDialogue.csproj" />
|
||||
<Project Path="src/Mimir.EveDashboard/Mimir.EveDashboard.csproj" />
|
||||
<Project Path="src/Mimir.EveSensorReceiver/Mimir.EveSensorReceiver.csproj" />
|
||||
<Project Path="src/Mimir.FensalirDaemon/Mimir.FensalirDaemon.csproj" />
|
||||
<Project Path="src/Mimir.RavenDaemon/Mimir.RavenDaemon.csproj" />
|
||||
<Project Path="src/Mimir.Well/Mimir.Well.csproj" />
|
||||
<Project Path="src/Mimir.WellObsDaemon/Mimir.WellObsDaemon.csproj" />
|
||||
<Project Path="src/Mimir.VerseRecorder/Mimir.VerseRecorder.csproj" />
|
||||
<Project Path="src/Mimir.PsMoveProbe/Mimir.PsMoveProbe.csproj" />
|
||||
<Project Path="src/Mimir.WasapiLoopback/Mimir.WasapiLoopback.csproj" />
|
||||
<Project Path="src/Mimir.Runtime/Mimir.Runtime.csproj" />
|
||||
<Project Path="src/Mimir.BufferSmoke/Mimir.BufferSmoke.csproj" />
|
||||
</Folder>
|
||||
|
|
|
|||
|
|
@ -50,6 +50,12 @@ Start with:
|
|||
source-level code map.
|
||||
- [docs/perfect-machine-domain-index.md](docs/perfect-machine-domain-index.md)
|
||||
for the indexed problem-domain map and implementation cuts.
|
||||
- [docs/fensalir-rendering-rebuild-migration.md](docs/fensalir-rendering-rebuild-migration.md)
|
||||
for the current teardown/rebuild migration plan between Mimir's physical
|
||||
evidence window and Fensalir's field-evidence renderer.
|
||||
- [docs/verse-service-contract.md](docs/verse-service-contract.md) for the
|
||||
CultCache/CultMesh/Odin/Eve service contract and demotion line for old
|
||||
dashboards, JSON witnesses, and compatibility transports.
|
||||
|
||||
Naming note: **Fensalir** is the engine/windowing/rendering/D3D12 layer. Older
|
||||
research notes may still use the previous engine name.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,974 @@
|
|||
{
|
||||
"anchor_sources": [
|
||||
"nw-builtin-cam",
|
||||
"nw-eye-1"
|
||||
],
|
||||
"calibration_id": "stream-proof-kiyo-move-trails",
|
||||
"confidence": 0.38533796485912397,
|
||||
"kind": "mimir.stream_proof_frustum_solve.v1",
|
||||
"mean_reprojection_error_clip": 0.19939056464550242,
|
||||
"notes": [
|
||||
"Nightwing observations define a provisional local witness space, not metric room truth.",
|
||||
"This receipt is sufficient for an AR trail demo and for feeding Starfire's global residual owner."
|
||||
],
|
||||
"p90_reprojection_error_clip": 0.26962808670417515,
|
||||
"projection_matrix_row_major": [
|
||||
0.5603854754837644,
|
||||
0.0709332080849531,
|
||||
-0.6933933020955221,
|
||||
0.3171526880627816,
|
||||
0.3738208616609618,
|
||||
0.0561101033966434,
|
||||
-0.4674286445600272,
|
||||
0.21108074938204147,
|
||||
-0.6259304629690188,
|
||||
-0.08663561418442428,
|
||||
0.7750518214173021,
|
||||
-0.35105121067042333
|
||||
],
|
||||
"source_id": "kiyo-pro",
|
||||
"trail_samples": [
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 20,
|
||||
"observed_time_s": 0.033,
|
||||
"projected_clip": [
|
||||
-0.8791482127998971,
|
||||
-0.6066291124324684
|
||||
],
|
||||
"reprojection_error_clip": 0.07789712730612666,
|
||||
"symbol": 1,
|
||||
"target_clip": [
|
||||
-0.9444828908940552,
|
||||
-0.5642104575636535
|
||||
],
|
||||
"world": [
|
||||
0.4550057224032302,
|
||||
0.6613541854298968,
|
||||
0.8046129734614623
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 27,
|
||||
"observed_time_s": 2.233,
|
||||
"projected_clip": [
|
||||
-0.9154298669408274,
|
||||
-0.5760080124454962
|
||||
],
|
||||
"reprojection_error_clip": 0.4690605686458582,
|
||||
"symbol": 9,
|
||||
"target_clip": [
|
||||
-0.7556513826938969,
|
||||
-0.13499933562594735
|
||||
],
|
||||
"world": [
|
||||
0.033348114133089884,
|
||||
0.6975472074310756,
|
||||
0.65
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 28,
|
||||
"observed_time_s": 2.6,
|
||||
"projected_clip": [
|
||||
-0.9185633092390114,
|
||||
-0.5750655247275896
|
||||
],
|
||||
"reprojection_error_clip": 0.15874776468969756,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9363872634671003,
|
||||
-0.7328094927569586
|
||||
],
|
||||
"world": [
|
||||
0.0041766661628288735,
|
||||
0.7725870357505362,
|
||||
0.65
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 29,
|
||||
"observed_time_s": 2.867,
|
||||
"projected_clip": [
|
||||
-0.9260785801883394,
|
||||
-0.596413299484144
|
||||
],
|
||||
"reprojection_error_clip": 0.200158835041353,
|
||||
"symbol": 1,
|
||||
"target_clip": [
|
||||
-0.9445605572080013,
|
||||
-0.7957170269218321
|
||||
],
|
||||
"world": [
|
||||
0.32284454318040456,
|
||||
0.4619966154697807,
|
||||
0.7579122257472921
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 30,
|
||||
"observed_time_s": 3.2,
|
||||
"projected_clip": [
|
||||
-0.8898776246689383,
|
||||
-0.6123385984762059
|
||||
],
|
||||
"reprojection_error_clip": 0.07240698311732986,
|
||||
"symbol": 10,
|
||||
"target_clip": [
|
||||
-0.9101514644339245,
|
||||
-0.542827860993879
|
||||
],
|
||||
"world": [
|
||||
0.28920004042113495,
|
||||
0.5079393807890763,
|
||||
0.7066163923237218
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 31,
|
||||
"observed_time_s": 3.567,
|
||||
"projected_clip": [
|
||||
-0.9000567798091308,
|
||||
-0.6148720976321779
|
||||
],
|
||||
"reprojection_error_clip": 0.777490090222322,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.6828053183097521,
|
||||
0.13164825900272215
|
||||
],
|
||||
"world": [
|
||||
0.2680836575541342,
|
||||
0.471977039229198,
|
||||
0.7001388424560799
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 33,
|
||||
"observed_time_s": 4.167,
|
||||
"projected_clip": [
|
||||
-0.8779285775344889,
|
||||
-0.6146778486430794
|
||||
],
|
||||
"reprojection_error_clip": 0.09225946233566872,
|
||||
"symbol": 1,
|
||||
"target_clip": [
|
||||
-0.9039751868344301,
|
||||
-0.5261714503746893
|
||||
],
|
||||
"world": [
|
||||
0.36334302658413453,
|
||||
0.3889902333047959,
|
||||
0.8420843623228649
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 34,
|
||||
"observed_time_s": 4.5,
|
||||
"projected_clip": [
|
||||
-0.9092291167212068,
|
||||
-0.5816488797088724
|
||||
],
|
||||
"reprojection_error_clip": 0.03951123542718797,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9098846739301969,
|
||||
-0.542143083049035
|
||||
],
|
||||
"world": [
|
||||
0.009749415525593542,
|
||||
0.6673228643800554,
|
||||
0.65
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 36,
|
||||
"observed_time_s": 5.167,
|
||||
"projected_clip": [
|
||||
-0.8817661539245883,
|
||||
-0.6110060402802908
|
||||
],
|
||||
"reprojection_error_clip": 0.0886057865162101,
|
||||
"symbol": 2,
|
||||
"target_clip": [
|
||||
-0.9286732819662155,
|
||||
-0.6861772232099388
|
||||
],
|
||||
"world": [
|
||||
0.33415087318268727,
|
||||
0.6496052956550742,
|
||||
0.6903241734431382
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 37,
|
||||
"observed_time_s": 5.433,
|
||||
"projected_clip": [
|
||||
-0.8751423826142312,
|
||||
-0.6041824478532443
|
||||
],
|
||||
"reprojection_error_clip": 0.11409315508228601,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9304745302126936,
|
||||
-0.7039602082950007
|
||||
],
|
||||
"world": [
|
||||
0.45316445225409335,
|
||||
0.6062798455030484,
|
||||
0.8393111061502192
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 38,
|
||||
"observed_time_s": 5.767,
|
||||
"projected_clip": [
|
||||
-0.9121771547885212,
|
||||
-0.5781053069814432
|
||||
],
|
||||
"reprojection_error_clip": 0.09983524711291265,
|
||||
"symbol": 2,
|
||||
"target_clip": [
|
||||
-0.9265350242970326,
|
||||
-0.6769027166097978
|
||||
],
|
||||
"world": [
|
||||
0.609675289783359,
|
||||
0.34021461444946643,
|
||||
0.9026787718356276
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 40,
|
||||
"observed_time_s": 6.367,
|
||||
"projected_clip": [
|
||||
-0.8959235843748068,
|
||||
-0.6256146683483831
|
||||
],
|
||||
"reprojection_error_clip": 0.051028029930010044,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9276389712261381,
|
||||
-0.6655895864272035
|
||||
],
|
||||
"world": [
|
||||
0.8826714570859733,
|
||||
0.5392283577296262,
|
||||
1.328993426455828
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 42,
|
||||
"observed_time_s": 7.067,
|
||||
"projected_clip": [
|
||||
-0.883800743412245,
|
||||
-0.6635784311162672
|
||||
],
|
||||
"reprojection_error_clip": 0.07561633040717448,
|
||||
"symbol": 4,
|
||||
"target_clip": [
|
||||
-0.9241648869292207,
|
||||
-0.5996365407595694
|
||||
],
|
||||
"world": [
|
||||
1.1595388539412725,
|
||||
0.4550406449370771,
|
||||
1.507202568465726
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 43,
|
||||
"observed_time_s": 7.367,
|
||||
"projected_clip": [
|
||||
-0.9030896995965859,
|
||||
-0.6286030984903708
|
||||
],
|
||||
"reprojection_error_clip": 0.022980680619030688,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9231890369722976,
|
||||
-0.6397443874091275
|
||||
],
|
||||
"world": [
|
||||
1.1527282933111536,
|
||||
0.6576468060465407,
|
||||
1.5753242169168749
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 46,
|
||||
"observed_time_s": 8.333,
|
||||
"projected_clip": [
|
||||
-0.8920112178015882,
|
||||
-0.6211088664030723
|
||||
],
|
||||
"reprojection_error_clip": 0.19944796024875366,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9442578861301885,
|
||||
-0.8135920461771753
|
||||
],
|
||||
"world": [
|
||||
0.6384832079695005,
|
||||
0.4840156230570471,
|
||||
1.1023804322541753
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 47,
|
||||
"observed_time_s": 8.6,
|
||||
"projected_clip": [
|
||||
-0.8489473779280677,
|
||||
-0.6762854404303741
|
||||
],
|
||||
"reprojection_error_clip": 0.1632032790256483,
|
||||
"symbol": 2,
|
||||
"target_clip": [
|
||||
-0.9464346405094111,
|
||||
-0.8071729643550012
|
||||
],
|
||||
"world": [
|
||||
0.6970990556791745,
|
||||
0.329574101269274,
|
||||
1.0866790844103376
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 49,
|
||||
"observed_time_s": 9.2,
|
||||
"projected_clip": [
|
||||
-1.0285725490193087,
|
||||
-0.7143971407752353
|
||||
],
|
||||
"reprojection_error_clip": 0.12337732330109946,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9464523276271589,
|
||||
-0.8064744623232056
|
||||
],
|
||||
"world": [
|
||||
0.6887408177515418,
|
||||
0.6394429045344123,
|
||||
1.0887264765612734
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 50,
|
||||
"observed_time_s": 9.567,
|
||||
"projected_clip": [
|
||||
-0.8698485679409191,
|
||||
-0.6427100166819155
|
||||
],
|
||||
"reprojection_error_clip": 0.1763502202820252,
|
||||
"symbol": 2,
|
||||
"target_clip": [
|
||||
-0.6977071414613234,
|
||||
-0.6044121122602797
|
||||
],
|
||||
"world": [
|
||||
0.7691338366691284,
|
||||
0.29601729783049663,
|
||||
1.182692183465037
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 53,
|
||||
"observed_time_s": 10.533,
|
||||
"projected_clip": [
|
||||
-0.9018050496836351,
|
||||
-0.5731543310414312
|
||||
],
|
||||
"reprojection_error_clip": 0.04281434317495741,
|
||||
"symbol": 2,
|
||||
"target_clip": [
|
||||
-0.9057157753202718,
|
||||
-0.530518967708888
|
||||
],
|
||||
"world": [
|
||||
0.57947030897418,
|
||||
0.4706213241689075,
|
||||
0.9326802462812758
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 54,
|
||||
"observed_time_s": 10.833,
|
||||
"projected_clip": [
|
||||
-0.8898930934349799,
|
||||
-0.5995856004959185
|
||||
],
|
||||
"reprojection_error_clip": 0.05556232211998459,
|
||||
"symbol": 8,
|
||||
"target_clip": [
|
||||
-0.9062240800808523,
|
||||
-0.5464774983406233
|
||||
],
|
||||
"world": [
|
||||
0.51506950349516,
|
||||
0.5692203595565103,
|
||||
0.8109681758119446
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 55,
|
||||
"observed_time_s": 11.1,
|
||||
"projected_clip": [
|
||||
-0.8741357545087254,
|
||||
-0.6102822490863554
|
||||
],
|
||||
"reprojection_error_clip": 0.0645932338290411,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9247110599290244,
|
||||
-0.6504621487643867
|
||||
],
|
||||
"world": [
|
||||
0.4513024696409307,
|
||||
0.7376766663950891,
|
||||
0.800416735751318
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 58,
|
||||
"observed_time_s": 12.067,
|
||||
"projected_clip": [
|
||||
-0.8866664014082294,
|
||||
-0.6055672830035839
|
||||
],
|
||||
"reprojection_error_clip": 0.07003679467256954,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9268458569889106,
|
||||
-0.6629324637079774
|
||||
],
|
||||
"world": [
|
||||
0.4062225195173447,
|
||||
0.61479708544503,
|
||||
0.7217816215365251
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 59,
|
||||
"observed_time_s": 12.4,
|
||||
"projected_clip": [
|
||||
-0.892241900111704,
|
||||
-0.603971329314553
|
||||
],
|
||||
"reprojection_error_clip": 0.13779381162298,
|
||||
"symbol": 2,
|
||||
"target_clip": [
|
||||
-0.9367396678307217,
|
||||
-0.7343825373820103
|
||||
],
|
||||
"world": [
|
||||
0.3571730654892108,
|
||||
0.5261903461314121,
|
||||
0.6776073803226678
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 61,
|
||||
"observed_time_s": 13.0,
|
||||
"projected_clip": [
|
||||
-0.8043932843500028,
|
||||
-0.5805875426080629
|
||||
],
|
||||
"reprojection_error_clip": 0.019826885547670595,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.8009087201951087,
|
||||
-0.6001058213019144
|
||||
],
|
||||
"world": [
|
||||
0.1797211808864931,
|
||||
0.37892897928910096,
|
||||
0.65
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 65,
|
||||
"observed_time_s": 14.333,
|
||||
"projected_clip": [
|
||||
-0.8832210844654748,
|
||||
-0.6059298663489588
|
||||
],
|
||||
"reprojection_error_clip": 0.18341108308730059,
|
||||
"symbol": 3,
|
||||
"target_clip": [
|
||||
-0.9399513409600255,
|
||||
-0.7803469050731591
|
||||
],
|
||||
"world": [
|
||||
0.44653243548773597,
|
||||
0.6575104816930223,
|
||||
0.7678145864005145
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 66,
|
||||
"observed_time_s": 14.6,
|
||||
"projected_clip": [
|
||||
-0.8777541328483115,
|
||||
-0.6028486146424588
|
||||
],
|
||||
"reprojection_error_clip": 0.06542609637672571,
|
||||
"symbol": 2,
|
||||
"target_clip": [
|
||||
-0.8194560878744526,
|
||||
-0.5731516107645838
|
||||
],
|
||||
"world": [
|
||||
0.5619443945980792,
|
||||
0.6947250397079097,
|
||||
0.8886367971364307
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 67,
|
||||
"observed_time_s": 14.9,
|
||||
"projected_clip": [
|
||||
-0.8625906951234988,
|
||||
-0.5908285584967105
|
||||
],
|
||||
"reprojection_error_clip": 0.18577528385909853,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.7653894936990006,
|
||||
-0.4325112109489362
|
||||
],
|
||||
"world": [
|
||||
0.6426053924027949,
|
||||
0.6707582369077036,
|
||||
1.0040654809284173
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 68,
|
||||
"observed_time_s": 15.3,
|
||||
"projected_clip": [
|
||||
-2.2663833035107785,
|
||||
-2.809173078604754
|
||||
],
|
||||
"reprojection_error_clip": 2.388473968136785,
|
||||
"symbol": 3,
|
||||
"target_clip": [
|
||||
-0.9481037279192214,
|
||||
-0.8174535057117172
|
||||
],
|
||||
"world": [
|
||||
0.7246153786313193,
|
||||
0.6049966747934538,
|
||||
1.1063266209861007
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 70,
|
||||
"observed_time_s": 15.867,
|
||||
"projected_clip": [
|
||||
-0.8984133025341523,
|
||||
-0.6292194199389068
|
||||
],
|
||||
"reprojection_error_clip": 0.13343836911709187,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.8732830217808946,
|
||||
-0.7602700488648746
|
||||
],
|
||||
"world": [
|
||||
0.8385173905382965,
|
||||
0.555482702096492,
|
||||
1.2698392877053206
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 72,
|
||||
"observed_time_s": 16.533,
|
||||
"projected_clip": [
|
||||
-0.9148242847714224,
|
||||
-0.6399850966234948
|
||||
],
|
||||
"reprojection_error_clip": 0.12753698854316933,
|
||||
"symbol": 4,
|
||||
"target_clip": [
|
||||
-0.9378794295402615,
|
||||
-0.7654209118691748
|
||||
],
|
||||
"world": [
|
||||
0.8635471681605983,
|
||||
0.6352995545118262,
|
||||
1.2687620973655969
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 73,
|
||||
"observed_time_s": 16.867,
|
||||
"projected_clip": [
|
||||
-0.8760411502219222,
|
||||
-0.5988847382013688
|
||||
],
|
||||
"reprojection_error_clip": 0.13507757121024097,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.933536689362441,
|
||||
-0.7211149031484831
|
||||
],
|
||||
"world": [
|
||||
0.41236094269201246,
|
||||
0.5397936969675786,
|
||||
0.825820427334146
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 75,
|
||||
"observed_time_s": 17.467,
|
||||
"projected_clip": [
|
||||
-0.903340599763464,
|
||||
-0.6303650411336879
|
||||
],
|
||||
"reprojection_error_clip": 0.20720220280662172,
|
||||
"symbol": 5,
|
||||
"target_clip": [
|
||||
-0.7516724588482482,
|
||||
-0.48919306133869234
|
||||
],
|
||||
"world": [
|
||||
0.9432673823220241,
|
||||
0.6117644769316547,
|
||||
1.3653914805543814
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 76,
|
||||
"observed_time_s": 17.767,
|
||||
"projected_clip": [
|
||||
-0.9027230298404102,
|
||||
-0.6291532158283146
|
||||
],
|
||||
"reprojection_error_clip": 0.10071720614334483,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9076838351042494,
|
||||
-0.5285582555944377
|
||||
],
|
||||
"world": [
|
||||
1.0883868765493652,
|
||||
0.6385656643376638,
|
||||
1.510684307660253
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 77,
|
||||
"observed_time_s": 18.1,
|
||||
"projected_clip": [
|
||||
-0.9082387252370533,
|
||||
-0.633363133417485
|
||||
],
|
||||
"reprojection_error_clip": 0.09422712588796978,
|
||||
"symbol": 3,
|
||||
"target_clip": [
|
||||
-0.9321207271148934,
|
||||
-0.7245135653386456
|
||||
],
|
||||
"world": [
|
||||
1.0619416429624824,
|
||||
0.6683402084484671,
|
||||
1.4686792077572435
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 78,
|
||||
"observed_time_s": 18.4,
|
||||
"projected_clip": [
|
||||
-0.9942869231406407,
|
||||
-0.6988450009097289
|
||||
],
|
||||
"reprojection_error_clip": 0.11027251484298344,
|
||||
"symbol": 6,
|
||||
"target_clip": [
|
||||
-0.916533418018064,
|
||||
-0.6206502442380093
|
||||
],
|
||||
"world": [
|
||||
0.9816783424197918,
|
||||
0.7455904767968399,
|
||||
1.3474595456643215
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 80,
|
||||
"observed_time_s": 19.1,
|
||||
"projected_clip": [
|
||||
-0.8785683062175577,
|
||||
-0.6097114606676594
|
||||
],
|
||||
"reprojection_error_clip": 0.3628899897733025,
|
||||
"symbol": 3,
|
||||
"target_clip": [
|
||||
-0.5846915282229066,
|
||||
-0.3968134004765196
|
||||
],
|
||||
"world": [
|
||||
0.41881465381109717,
|
||||
0.7226933783722339,
|
||||
0.7519369374773618
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 81,
|
||||
"observed_time_s": 19.333,
|
||||
"projected_clip": [
|
||||
-0.8932511314502098,
|
||||
-0.6037520476763464
|
||||
],
|
||||
"reprojection_error_clip": 0.13293231016296111,
|
||||
"symbol": 7,
|
||||
"target_clip": [
|
||||
-0.7744566796393737,
|
||||
-0.5440957210694841
|
||||
],
|
||||
"world": [
|
||||
0.34750765402133443,
|
||||
0.5140347326274995,
|
||||
0.65
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 84,
|
||||
"observed_time_s": 20.367,
|
||||
"projected_clip": [
|
||||
-0.9264590537850987,
|
||||
-0.5674051845866467
|
||||
],
|
||||
"reprojection_error_clip": 0.20745348465809058,
|
||||
"symbol": 8,
|
||||
"target_clip": [
|
||||
-0.9412811984659236,
|
||||
-0.7743284851688801
|
||||
],
|
||||
"world": [
|
||||
0.5381849048555366,
|
||||
0.31113920079120627,
|
||||
0.8713142228967515
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 86,
|
||||
"observed_time_s": 20.933,
|
||||
"projected_clip": [
|
||||
-0.8850352008282428,
|
||||
-0.6108277258700181
|
||||
],
|
||||
"reprojection_error_clip": 0.07246762430534573,
|
||||
"symbol": 3,
|
||||
"target_clip": [
|
||||
-0.9203842200243604,
|
||||
-0.5475663362683563
|
||||
],
|
||||
"world": [
|
||||
0.32064344013221785,
|
||||
0.5416996368882029,
|
||||
0.7261494284007469
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 88,
|
||||
"observed_time_s": 21.567,
|
||||
"projected_clip": [
|
||||
-0.8977740322776363,
|
||||
-0.5866412821502801
|
||||
],
|
||||
"reprojection_error_clip": 0.1326307676872254,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9335627450982306,
|
||||
-0.7143522392837531
|
||||
],
|
||||
"world": [
|
||||
0.07287154042005796,
|
||||
0.5043399466076959,
|
||||
0.65
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 90,
|
||||
"observed_time_s": 22.2,
|
||||
"projected_clip": [
|
||||
-0.8516938020059152,
|
||||
-0.6340310849012898
|
||||
],
|
||||
"reprojection_error_clip": 0.1452344534397858,
|
||||
"symbol": 10,
|
||||
"target_clip": [
|
||||
-0.9377982109191558,
|
||||
-0.7509886722210304
|
||||
],
|
||||
"world": [
|
||||
0.4286425016628102,
|
||||
0.15758458672020956,
|
||||
0.8835444797205171
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 92,
|
||||
"observed_time_s": 22.833,
|
||||
"projected_clip": [
|
||||
-1.1391530490524377,
|
||||
-0.31677894317620464
|
||||
],
|
||||
"reprojection_error_clip": 0.7694778761150074,
|
||||
"symbol": 4,
|
||||
"target_clip": [
|
||||
-0.5564758057338148,
|
||||
0.1857978489153852
|
||||
],
|
||||
"world": [
|
||||
0.14581880087821925,
|
||||
0.6547361298960647,
|
||||
0.65
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 94,
|
||||
"observed_time_s": 23.5,
|
||||
"projected_clip": [
|
||||
-0.8863054461747846,
|
||||
-0.6070023453414959
|
||||
],
|
||||
"reprojection_error_clip": 0.05064624328537799,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9107211464225807,
|
||||
-0.5626298815792541
|
||||
],
|
||||
"world": [
|
||||
0.38462487786099325,
|
||||
0.679415452265829,
|
||||
0.65
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 95,
|
||||
"observed_time_s": 23.833,
|
||||
"projected_clip": [
|
||||
-0.875546889160156,
|
||||
-0.6095742765677072
|
||||
],
|
||||
"reprojection_error_clip": 0.0634080516617856,
|
||||
"symbol": 4,
|
||||
"target_clip": [
|
||||
-0.8814875216911486,
|
||||
-0.5464451240256734
|
||||
],
|
||||
"world": [
|
||||
0.4706267631687632,
|
||||
0.7687956996487649,
|
||||
0.7988522080649928
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 97,
|
||||
"observed_time_s": 24.467,
|
||||
"projected_clip": [
|
||||
-0.8913821754038934,
|
||||
-0.5727151063669901
|
||||
],
|
||||
"reprojection_error_clip": 0.05878488166116981,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9051403152577566,
|
||||
-0.5155628894399165
|
||||
],
|
||||
"world": [
|
||||
0.6961734789759012,
|
||||
0.5309183036694326,
|
||||
1.02593811442535
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 98,
|
||||
"observed_time_s": 24.733,
|
||||
"projected_clip": [
|
||||
-0.9058656156323933,
|
||||
-0.6288377637637487
|
||||
],
|
||||
"reprojection_error_clip": 0.0793974925899308,
|
||||
"symbol": 4,
|
||||
"target_clip": [
|
||||
-0.9082828111193328,
|
||||
-0.5494770745306905
|
||||
],
|
||||
"world": [
|
||||
0.850371341740017,
|
||||
0.6138649309005824,
|
||||
1.2779753956216187
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 99,
|
||||
"observed_time_s": 25.1,
|
||||
"projected_clip": [
|
||||
-0.8951972439648934,
|
||||
-0.6740433369149873
|
||||
],
|
||||
"reprojection_error_clip": 0.17177748270940166,
|
||||
"symbol": 3,
|
||||
"target_clip": [
|
||||
-0.9132210766281885,
|
||||
-0.5032140507193694
|
||||
],
|
||||
"world": [
|
||||
1.005210138900337,
|
||||
0.5338681642707513,
|
||||
1.365657849584086
|
||||
]
|
||||
}
|
||||
],
|
||||
"truth_status": "provisional_witness_space",
|
||||
"used_points": 47
|
||||
}
|
||||
|
After Width: | Height: | Size: 392 KiB |
BIN
artifacts/runtime/mimir-eve-surface-layout-generic-decoder.png
Normal file
|
After Width: | Height: | Size: 66 KiB |
BIN
artifacts/runtime/mimir-leap-history-split-shader-visible.png
Normal file
|
After Width: | Height: | Size: 48 KiB |
BIN
artifacts/runtime/mimir-leap-surface-claims-clean.png
Normal file
|
After Width: | Height: | Size: 48 KiB |
BIN
artifacts/runtime/mimir-perfect-machine-all-sensors-headless.png
Normal file
|
After Width: | Height: | Size: 49 KiB |
|
After Width: | Height: | Size: 47 KiB |
|
After Width: | Height: | Size: 110 KiB |
|
|
@ -0,0 +1,46 @@
|
|||
# Complex Contour Live Scarlett Receipt 2026-05-28 00:23:13
|
||||
|
||||
## Setup
|
||||
|
||||
- Renderer: `canary-packet-trill`
|
||||
- Playback artifact: `artifacts/asio/canary-packet-live-20260528-002313-f32.raw`
|
||||
- Capture artifact: `artifacts/asio/scarlett-canary-packet-live-20260528-002313-f32.raw`
|
||||
- ASIO driver: `Focusrite USB ASIO`
|
||||
- Sample rate: `192000`
|
||||
- Capture seconds: `4.2`
|
||||
- Inputs: `Input 1`, `Input 2`, `Loopback 1`, `Loopback 2`
|
||||
- Outputs: `Output 1`, `Output 2`
|
||||
- Playback gain: `1.0`
|
||||
- Captured frames: `805888`
|
||||
- Nonzero samples: `2294204`
|
||||
|
||||
## Results
|
||||
|
||||
Reference channel is `asio-ch2` / Loopback 1.
|
||||
|
||||
| Candidate | Meaning | Estimate | Prediction Error | Confidence | Direct Hits | Notes |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| `asio-ch3` | Loopback 2 | `-0.011837` samples / `-0.062 us` | `-0.062 us` | `0.956` | `257` | good loopback proof |
|
||||
| `asio-ch1` | Input 2 | no lock | n/a | n/a | `0` | no contour hits |
|
||||
| `asio-ch0` | Input 1 | no lock | n/a | n/a | `0` | no contour hits |
|
||||
|
||||
## Signal Sanity Check
|
||||
|
||||
`asio-ch2` and `asio-ch3` contain the rendered packet-song witness. Physical
|
||||
inputs are present but far below usable contour-lock level in this capture.
|
||||
|
||||
| Channel | RMS | Peak | Speech RMS | Ultrasonic RMS |
|
||||
| --- | ---: | ---: | ---: | ---: |
|
||||
| `asio-ch0` | `0.000525` | `0.002648` | `0.000063` | `0.000187` |
|
||||
| `asio-ch1` | `0.000087` | `0.000465` | `0.000010` | `0.000031` |
|
||||
| `asio-ch2` | `0.006676` | `0.028226` | `0.005157` | `0.002268` |
|
||||
| `asio-ch3` | `0.006676` | `0.028226` | `0.005157` | `0.002268` |
|
||||
|
||||
## Interpretation
|
||||
|
||||
The complex-contour receiver and ASIO loopback path still prove sub-microsecond
|
||||
interface timing on a fresh capture. The microphone step is blocked by current
|
||||
input signal state, not by the contour decoder: both physical input channels
|
||||
produced zero contour hits against 1480 reference hits. Next physical proof
|
||||
needs the mic gain/routing/source placement fixed before changing receiver
|
||||
logic.
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
# Complex Contour Live Scarlett Receipt 2026-05-28 08:06:04
|
||||
|
||||
## Setup
|
||||
|
||||
- Renderer: `canary-packet-trill`
|
||||
- Playback artifact: `artifacts/asio/canary-packet-live-20260528-080604-f32.raw`
|
||||
- Capture artifact: `artifacts/asio/scarlett-canary-packet-live-20260528-080604-f32.raw`
|
||||
- ASIO driver: `Focusrite USB ASIO`
|
||||
- Sample rate: `192000`
|
||||
- Capture seconds: `4.2`
|
||||
- Inputs: armed cardioid and shotgun on Scarlett input channels
|
||||
- Output routing: headphones by mistake
|
||||
- Loopback reference: `asio-ch2` / Loopback 1
|
||||
- Playback gain: `1.0`
|
||||
- Captured frames: `806656`
|
||||
- Nonzero samples: `2296810`
|
||||
|
||||
## Results
|
||||
|
||||
| Candidate | Estimate | Prediction Error | Confidence | Direct Hits | Notes |
|
||||
| --- | ---: | ---: | ---: | ---: | --- |
|
||||
| `asio-ch3` | `-0.002628` samples / `-0.014 us` | `-0.014 us` | `0.971` | `216` | loopback proof |
|
||||
| `asio-ch1` | `544.957302` samples / `2838.319 us` | `+3.710 us` | `0.358` | `13` | better physical path this run |
|
||||
| `asio-ch0` | `784.256427` samples / `4084.669 us` | `+14.404 us` | `0.276` | `9` | weaker physical path this run |
|
||||
|
||||
## Signal Sanity Check
|
||||
|
||||
Both physical inputs are now armed and carrying real signal. They remain lower
|
||||
confidence than loopback, but this is no longer the zero-contour-hit failure
|
||||
from `complex-contour-live-20260528-002313.md`.
|
||||
|
||||
| Channel | RMS | Peak | Speech RMS | Ultrasonic RMS |
|
||||
| --- | ---: | ---: | ---: | ---: |
|
||||
| `asio-ch0` | `0.006902` | `0.037563` | `0.002519` | `0.002437` |
|
||||
| `asio-ch1` | `0.002714` | `0.033561` | `0.002128` | `0.000431` |
|
||||
| `asio-ch2` | `0.006673` | `0.028226` | `0.005154` | `0.002267` |
|
||||
| `asio-ch3` | `0.006673` | `0.028226` | `0.005154` | `0.002267` |
|
||||
|
||||
## Interpretation
|
||||
|
||||
The contour receiver sees both physical inputs, but this run was accidentally
|
||||
routed to headphones rather than the monitors. Treat it as a misroute witness,
|
||||
not a room/mic proof. It is useful mainly because it explains the weak physical
|
||||
confidence and proves the harness did not silently lose the armed inputs.
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
# Complex Contour Live Scarlett Receipt 2026-05-28 08:07:27
|
||||
|
||||
## Setup
|
||||
|
||||
- Renderer: `canary-packet-trill`
|
||||
- Playback artifact: `artifacts/asio/canary-packet-live-20260528-080727-f32.raw`
|
||||
- Capture artifact: `artifacts/asio/scarlett-canary-packet-live-20260528-080727-f32.raw`
|
||||
- ASIO driver: `Focusrite USB ASIO`
|
||||
- Sample rate: `192000`
|
||||
- Capture seconds: `4.2`
|
||||
- Output routing: monitors
|
||||
- Inputs: `Input 1`, `Input 2`, `Loopback 1`, `Loopback 2`
|
||||
- Physical mapping:
|
||||
- `asio-ch0` / Input 1: shotgun, about 1.5 m from right monitor and 3.0 m from left monitor
|
||||
- `asio-ch1` / Input 2: cardioid, about 0.5 m from left monitor and 1.5 m from right monitor
|
||||
- Loopback reference: `asio-ch2` / Loopback 1
|
||||
- Playback gain: `1.0`
|
||||
- Captured frames: `806656`
|
||||
- Nonzero samples: `2298347`
|
||||
|
||||
## Results
|
||||
|
||||
| Candidate | Meaning | Estimate | Prediction Error | Confidence | Direct Hits | Notes |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| `asio-ch3` | Loopback 2 | `-0.002628` samples / `-0.014 us` | `-0.014 us` | `0.971` | `216` | interface loopback proof |
|
||||
| `asio-ch0` | Input 1 | `781.412261` samples / `4069.856 us` | `-0.410 us` | `0.524` | `24` | best physical path this run |
|
||||
| `asio-ch1` | Input 2 | `543.771315` samples / `2832.142 us` | `-2.467 us` | `0.363` | `14` | weaker physical path this run |
|
||||
|
||||
## Signal Sanity Check
|
||||
|
||||
Both physical inputs are carrying real monitor-routed acoustic signal. Compared
|
||||
with the headphone-routed `complex-contour-live-20260528-080604.md` run, both
|
||||
physical channels have stronger RMS and the path estimates improve.
|
||||
|
||||
| Channel | RMS | Peak | Speech RMS | Ultrasonic RMS |
|
||||
| --- | ---: | ---: | ---: | ---: |
|
||||
| `asio-ch0` | `0.012653` | `0.072978` | `0.009195` | `0.003837` |
|
||||
| `asio-ch1` | `0.014106` | `0.093789` | `0.011449` | `0.004182` |
|
||||
| `asio-ch2` | `0.006673` | `0.028226` | `0.005154` | `0.002267` |
|
||||
| `asio-ch3` | `0.006673` | `0.028226` | `0.005154` | `0.002267` |
|
||||
|
||||
## Interpretation
|
||||
|
||||
This is the first credible fresh physical complex-contour proof after arming
|
||||
both microphones and routing output to monitors. `asio-ch0` is the shotgun and
|
||||
lands within `0.410 us` of the stored path seed with moderate confidence.
|
||||
Its measured absolute delay is consistent with a nearest-monitor path on the
|
||||
order of the right-monitor distance. `asio-ch1` is the cardioid and also locks,
|
||||
but weaker; its measured absolute delay should be interpreted against the
|
||||
actual stereo monitor routing and room path, not as a naked source-to-mic
|
||||
distance yet. The next useful cut is repeated monitor-routed captures to
|
||||
separate stable direct-path truth from one-run fit luck, then either update the
|
||||
channel model or add runtime signal-presence gating before actuator authority
|
||||
consumes physical mic reports.
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
# Complex Contour Monitor Split Receipt 2026-05-28 08:18:13
|
||||
|
||||
## Setup
|
||||
|
||||
- Renderer: `canary-packet-trill`
|
||||
- Playback artifact: `artifacts/asio/canary-packet-monitor-split-20260528-081813-f32.raw`
|
||||
- Output 0 capture: `artifacts/asio/scarlett-canary-packet-monitor-output0-20260528-081813-f32.raw`
|
||||
- Output 1 capture: `artifacts/asio/scarlett-canary-packet-monitor-output1-20260528-081813-f32.raw`
|
||||
- ASIO driver: `Focusrite USB ASIO`
|
||||
- Sample rate: `192000`
|
||||
- Capture seconds: `4.2`
|
||||
- Probe change: `--play-output-channel` selects one ASIO output buffer; `-1`
|
||||
remains the default all-output behavior.
|
||||
- Physical mapping:
|
||||
- `asio-ch0` / Input 1: shotgun, about 1.5 m from right monitor and 3.0 m from left monitor
|
||||
- `asio-ch1` / Input 2: cardioid, about 0.5 m from left monitor and 1.5 m from right monitor
|
||||
|
||||
## Signal Sanity
|
||||
|
||||
Output 0 energized loopback channel `asio-ch2`; `asio-ch3` was silent.
|
||||
Output 1 energized loopback channel `asio-ch3`; `asio-ch2` was silent.
|
||||
|
||||
Physical input RMS stayed high in both captures:
|
||||
|
||||
| Capture | `asio-ch0` RMS | `asio-ch1` RMS | Reference loopback |
|
||||
| --- | ---: | ---: | --- |
|
||||
| output 0 | `0.014660` | `0.050159` | `asio-ch2` |
|
||||
| output 1 | `0.019050` | `0.040859` | `asio-ch3` |
|
||||
|
||||
## Results
|
||||
|
||||
| Output | Reference | Mic | Estimate | Confidence | Direct Hits |
|
||||
| --- | --- | --- | ---: | ---: | ---: |
|
||||
| output 0 | `asio-ch2` | shotgun / `asio-ch0` | `783.629083` samples / `4081.401 us` | `0.392` | `19` |
|
||||
| output 1 | `asio-ch3` | shotgun / `asio-ch0` | `779.619141` samples / `4060.516 us` | `0.376` | `18` |
|
||||
| output 0 | `asio-ch2` | cardioid / `asio-ch1` | `547.779779` samples / `2853.020 us` | `0.285` | `14` |
|
||||
| output 1 | `asio-ch3` | cardioid / `asio-ch1` | `543.020433` samples / `2828.231 us` | `0.383` | `17` |
|
||||
|
||||
## Interpretation
|
||||
|
||||
The ASIO probe now isolates output buffers, and the loopback channels confirm
|
||||
that output 0 and output 1 are distinct at the interface. The physical mic
|
||||
delays, however, are nearly identical across output channels:
|
||||
|
||||
- shotgun differs by about `20.885 us`;
|
||||
- cardioid differs by about `24.789 us`.
|
||||
|
||||
That is not compatible with the declared monitor geometry if output 0 and
|
||||
output 1 correspond to isolated left/right monitors in the room. The current
|
||||
evidence says the downstream monitor path is probably summing, mirroring, or
|
||||
otherwise not isolating the physical speakers as assumed. Do not use this
|
||||
receipt to learn left/right acoustic geometry until physical speaker isolation
|
||||
is verified.
|
||||
BIN
config/mimir-program-surface.cc
Normal file
|
|
@ -31,6 +31,23 @@
|
|||
"asio-ch2": "Focusrite Loopback 1",
|
||||
"asio-ch3": "Focusrite Loopback 2"
|
||||
}
|
||||
},
|
||||
{
|
||||
"sourceId": "raven-display",
|
||||
"displayName": "Raven network display",
|
||||
"clockDomainId": "raven-sync",
|
||||
"kind": "Video",
|
||||
"origin": "Network",
|
||||
"enabled": false,
|
||||
"adapter": "frame-events",
|
||||
"command": "../native/probes/raven_display_capture_receiver/build/Release/raven_display_capture_receiver.exe",
|
||||
"arguments": [
|
||||
"--emit-json-frames",
|
||||
"--source-id",
|
||||
"raven-display",
|
||||
"--clock-domain",
|
||||
"raven-sync"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
92
config/mimir-runtime.ks-camera.example.json
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
{
|
||||
"bufferSeconds": 5.0,
|
||||
"streams": [
|
||||
{
|
||||
"sourceId": "leap-stereo-ir",
|
||||
"displayName": "LeapUVC stereo IR",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_f182&pid_0003&mi_00",
|
||||
"width": 640,
|
||||
"height": 240,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "LeapStereoIr",
|
||||
"minimumFramesPerSecond": 100.0,
|
||||
"queueDepth": 8
|
||||
},
|
||||
{
|
||||
"sourceId": "ps3-eye-0",
|
||||
"displayName": "PS3 Eye 0",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ps3eye",
|
||||
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
|
||||
"cameraIndex": 0,
|
||||
"width": 320,
|
||||
"height": 240,
|
||||
"framesPerSecond": 187
|
||||
},
|
||||
{
|
||||
"sourceId": "ps3-eye-1",
|
||||
"displayName": "PS3 Eye 1",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ps3eye",
|
||||
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
|
||||
"cameraIndex": 1,
|
||||
"width": 320,
|
||||
"height": 240,
|
||||
"framesPerSecond": 187
|
||||
},
|
||||
{
|
||||
"sourceId": "kiyo-pro-yuy2",
|
||||
"displayName": "Kiyo Pro YUY2",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_1532&pid_0e05&mi_00",
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
"pixelFormat": "YUY2",
|
||||
"minimumFramesPerSecond": 25.0,
|
||||
"queueDepth": 8
|
||||
},
|
||||
{
|
||||
"sourceId": "kiyo-pro-mf-gpu",
|
||||
"displayName": "Kiyo Pro MF GPU decode",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": false,
|
||||
"adapter": "mf-gpu",
|
||||
"command": "../native/camera_capture/build/Release/mimir_mf_gpu_capture.dll",
|
||||
"pathNeedle": "vid_1532&pid_0e05",
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
"inputFormat": "MJPG",
|
||||
"outputFormat": "RGB32",
|
||||
"minimumFramesPerSecond": 25.0
|
||||
},
|
||||
{
|
||||
"sourceId": "kiyo-yuy2",
|
||||
"displayName": "Kiyo YUY2",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_1532&pid_0e03&mi_00",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"pixelFormat": "YUY2",
|
||||
"minimumFramesPerSecond": 25.0,
|
||||
"queueDepth": 8
|
||||
}
|
||||
]
|
||||
}
|
||||
21
config/mimir-runtime.leap-live.example.json
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"bufferSeconds": 2.0,
|
||||
"streams": [
|
||||
{
|
||||
"sourceId": "leap-stereo-ir",
|
||||
"displayName": "LeapUVC stereo IR",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_f182&pid_0003&mi_00",
|
||||
"width": 640,
|
||||
"height": 240,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "LeapStereoIr",
|
||||
"minimumFramesPerSecond": 100.0,
|
||||
"queueDepth": 8
|
||||
}
|
||||
]
|
||||
}
|
||||
108
config/mimir-runtime.move-stream.local.json
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
{
|
||||
"bufferSeconds": 5.0,
|
||||
"audioSync": {
|
||||
"mode": "passive",
|
||||
"referenceSourceId": "asio-ch2",
|
||||
"calibrationGain": 0.0,
|
||||
"watermarkGain": 0.0,
|
||||
"complexContourChannelModelPath": "../calibration/bioacoustic/complex-contour-channel-model.json",
|
||||
"enableComplexContourRuntime": true
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
"sourceId": "focusrite-asio",
|
||||
"displayName": "Starfire Scarlett ASIO",
|
||||
"kind": "Audio",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "asio",
|
||||
"command": "../native/asio_capture/build/Release/mimir_asio_capture.dll",
|
||||
"sampleRate": 192000,
|
||||
"acceptSourceIds": [
|
||||
"asio-ch0",
|
||||
"asio-ch1",
|
||||
"asio-ch2",
|
||||
"asio-ch3"
|
||||
],
|
||||
"sourceLabels": {
|
||||
"asio-ch0": "Shotgun hero mic",
|
||||
"asio-ch1": "Cardioid hero mic",
|
||||
"asio-ch2": "Raven program loopback L",
|
||||
"asio-ch3": "Raven program loopback R"
|
||||
}
|
||||
},
|
||||
{
|
||||
"sourceId": "kiyo-pro-rgb",
|
||||
"displayName": "Kiyo Pro program view",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_1532&pid_0e05&mi_00",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "YUY2",
|
||||
"minimumFramesPerSecond": 25.0,
|
||||
"queueDepth": 8
|
||||
},
|
||||
{
|
||||
"sourceId": "ps3-eye-0",
|
||||
"displayName": "PS3 Eye 0 Move tracker",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ps3eye",
|
||||
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
|
||||
"cameraIndex": 0,
|
||||
"width": 320,
|
||||
"height": 240,
|
||||
"framesPerSecond": 187
|
||||
},
|
||||
{
|
||||
"sourceId": "ps3-eye-1",
|
||||
"displayName": "PS3 Eye 1 Move tracker",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ps3eye",
|
||||
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
|
||||
"cameraIndex": 1,
|
||||
"width": 320,
|
||||
"height": 240,
|
||||
"framesPerSecond": 187
|
||||
},
|
||||
{
|
||||
"sourceId": "raven-display",
|
||||
"displayName": "Raven screen capture",
|
||||
"clockDomainId": "raven-sync",
|
||||
"kind": "Video",
|
||||
"origin": "Network",
|
||||
"enabled": true,
|
||||
"adapter": "ffmpeg-rawvideo",
|
||||
"command": "ffmpeg",
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
"pixelFormat": "Bgra8",
|
||||
"arguments": [
|
||||
"-hide_banner",
|
||||
"-nostdin",
|
||||
"-loglevel",
|
||||
"warning",
|
||||
"-fflags",
|
||||
"nobuffer",
|
||||
"-flags",
|
||||
"low_delay",
|
||||
"-i",
|
||||
"srt://0.0.0.0:5200?mode=listener&latency=120000&timeout=5000000",
|
||||
"-an",
|
||||
"-f",
|
||||
"rawvideo",
|
||||
"-pix_fmt",
|
||||
"bgra",
|
||||
"-"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
67
config/mimir-runtime.mvp-leap-eyes-kiyo.local.json
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
{
|
||||
"bufferSeconds": 5.0,
|
||||
"audioSync": {
|
||||
"mode": "passive",
|
||||
"referenceSourceId": "asio-ch2",
|
||||
"calibrationGain": 0.0,
|
||||
"watermarkGain": 0.0,
|
||||
"complexContourChannelModelPath": "../calibration/bioacoustic/complex-contour-channel-model.json",
|
||||
"enableComplexContourRuntime": true
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
"sourceId": "focusrite-asio",
|
||||
"displayName": "Starfire Scarlett ASIO",
|
||||
"kind": "Audio",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "asio",
|
||||
"command": "../native/asio_capture/build/Release/mimir_asio_capture.dll",
|
||||
"sampleRate": 192000,
|
||||
"acceptSourceIds": [
|
||||
"asio-ch0",
|
||||
"asio-ch1",
|
||||
"asio-ch2",
|
||||
"asio-ch3"
|
||||
],
|
||||
"sourceLabels": {
|
||||
"asio-ch0": "Shotgun hero mic",
|
||||
"asio-ch1": "Cardioid hero mic",
|
||||
"asio-ch2": "Raven/program loopback L",
|
||||
"asio-ch3": "Raven/program loopback R"
|
||||
}
|
||||
},
|
||||
{
|
||||
"sourceId": "leap-stereo-ir",
|
||||
"displayName": "LeapUVC stereo IR / depth root",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_f182&pid_0003&mi_00",
|
||||
"width": 640,
|
||||
"height": 240,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "LeapStereoIr",
|
||||
"minimumFramesPerSecond": 100.0,
|
||||
"queueDepth": 8
|
||||
},
|
||||
{
|
||||
"sourceId": "kiyo-pro-rgb",
|
||||
"displayName": "Kiyo Pro RGB program view",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_1532&pid_0e05&mi_00",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "YUY2",
|
||||
"minimumFramesPerSecond": 25.0,
|
||||
"queueDepth": 8
|
||||
}
|
||||
]
|
||||
}
|
||||
85
config/mimir-runtime.perfect-machine.local.json
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
{
|
||||
"bufferSeconds": 5.0,
|
||||
"audioSync": {
|
||||
"mode": "passive",
|
||||
"referenceSourceId": "loopback-scarlett-speakers",
|
||||
"calibrationGain": 0.0,
|
||||
"watermarkGain": 0.0
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
"sourceId": "leap-stereo-ir",
|
||||
"displayName": "LeapUVC stereo IR packed R/G",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_f182&pid_0003&mi_00",
|
||||
"width": 640,
|
||||
"height": 240,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "LeapStereoIr",
|
||||
"minimumFramesPerSecond": 100.0,
|
||||
"queueDepth": 8
|
||||
},
|
||||
{
|
||||
"sourceId": "ps3-eye-0",
|
||||
"displayName": "PS3 Eye 0 high-rate tracking",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ps3eye",
|
||||
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
|
||||
"cameraIndex": 0,
|
||||
"width": 320,
|
||||
"height": 240,
|
||||
"framesPerSecond": 187
|
||||
},
|
||||
{
|
||||
"sourceId": "ps3-eye-1",
|
||||
"displayName": "PS3 Eye 1 high-rate tracking",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ps3eye",
|
||||
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
|
||||
"cameraIndex": 1,
|
||||
"width": 320,
|
||||
"height": 240,
|
||||
"framesPerSecond": 187
|
||||
},
|
||||
{
|
||||
"sourceId": "kiyo-pro-rgb",
|
||||
"displayName": "Kiyo Pro RGB context",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_1532&pid_0e05&mi_00",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "YUY2",
|
||||
"minimumFramesPerSecond": 25.0,
|
||||
"queueDepth": 8
|
||||
},
|
||||
{
|
||||
"sourceId": "kiyo-basic-rgb",
|
||||
"displayName": "Kiyo RGB context",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_1532&pid_0e03&mi_00",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "YUY2",
|
||||
"minimumFramesPerSecond": 25.0,
|
||||
"queueDepth": 8
|
||||
}
|
||||
]
|
||||
}
|
||||
107
config/mimir-runtime.raven-eve.example.json
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
{
|
||||
"bufferSeconds": 5.0,
|
||||
"audioSync": {
|
||||
"mode": "chirp-only",
|
||||
"referenceSourceId": "asio-ch2",
|
||||
"calibrationGain": 2.0,
|
||||
"watermarkGain": 0.035,
|
||||
"calibrationModelPath": "../calibration/chirp-bin/scarlett-artifact.json",
|
||||
"complexContourChannelModelPath": "../calibration/bioacoustic/complex-contour-channel-model.json",
|
||||
"bioacousticWitnessProfileId": "canary-packet-trill",
|
||||
"enableComplexContourRuntime": true
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
"sourceId": "focusrite-asio",
|
||||
"kind": "Audio",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "asio",
|
||||
"command": "../native/asio_capture/build/Release/mimir_asio_capture.dll",
|
||||
"sampleRate": 192000,
|
||||
"acceptSourceIds": [
|
||||
"asio-ch0",
|
||||
"asio-ch1",
|
||||
"asio-ch2",
|
||||
"asio-ch3"
|
||||
],
|
||||
"sourceLabels": {
|
||||
"asio-ch0": "Focusrite Input 1",
|
||||
"asio-ch1": "Focusrite Input 2",
|
||||
"asio-ch2": "Focusrite Loopback 1",
|
||||
"asio-ch3": "Focusrite Loopback 2"
|
||||
}
|
||||
},
|
||||
{
|
||||
"sourceId": "raven-display",
|
||||
"displayName": "Raven screen capture",
|
||||
"clockDomainId": "raven-sync",
|
||||
"kind": "Video",
|
||||
"origin": "Network",
|
||||
"enabled": true,
|
||||
"adapter": "ffmpeg-rawvideo",
|
||||
"command": "ffmpeg",
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
"pixelFormat": "Bgra8",
|
||||
"arguments": [
|
||||
"-hide_banner",
|
||||
"-nostdin",
|
||||
"-loglevel",
|
||||
"warning",
|
||||
"-fflags",
|
||||
"nobuffer",
|
||||
"-flags",
|
||||
"low_delay",
|
||||
"-i",
|
||||
"srt://0.0.0.0:5200?mode=listener&latency=120000&timeout=5000000",
|
||||
"-an",
|
||||
"-f",
|
||||
"rawvideo",
|
||||
"-pix_fmt",
|
||||
"bgra",
|
||||
"-"
|
||||
]
|
||||
},
|
||||
{
|
||||
"sourceId": "eve-camera",
|
||||
"displayName": "Eve camera capture",
|
||||
"clockDomainId": "eve-camera",
|
||||
"kind": "Video",
|
||||
"origin": "Network",
|
||||
"enabled": true,
|
||||
"adapter": "frame-events",
|
||||
"command": "../src/Mimir.EveSensorReceiver/bin/Debug/net10.0/Mimir.EveSensorReceiver.exe",
|
||||
"arguments": [
|
||||
"--port",
|
||||
"8793",
|
||||
"--path",
|
||||
"/eve/camera",
|
||||
"--source-id",
|
||||
"eve-camera",
|
||||
"--type",
|
||||
"video-frame"
|
||||
]
|
||||
},
|
||||
{
|
||||
"sourceId": "eve-mic",
|
||||
"displayName": "Eve microphone capture",
|
||||
"clockDomainId": "eve-camera",
|
||||
"kind": "Audio",
|
||||
"origin": "Network",
|
||||
"enabled": true,
|
||||
"adapter": "frame-events",
|
||||
"command": "../src/Mimir.EveSensorReceiver/bin/Debug/net10.0/Mimir.EveSensorReceiver.exe",
|
||||
"arguments": [
|
||||
"--port",
|
||||
"8794",
|
||||
"--path",
|
||||
"/eve/mic",
|
||||
"--source-id",
|
||||
"eve-mic",
|
||||
"--type",
|
||||
"audio-block"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
131
config/mimir-runtime.stream-proof.local.json
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
{
|
||||
"bufferSeconds": 5.0,
|
||||
"audioSync": {
|
||||
"mode": "passive",
|
||||
"referenceSourceId": "asio-ch2",
|
||||
"calibrationGain": 0.0,
|
||||
"watermarkGain": 0.0,
|
||||
"complexContourChannelModelPath": "../calibration/bioacoustic/complex-contour-channel-model.json",
|
||||
"enableComplexContourRuntime": true
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
"sourceId": "focusrite-asio",
|
||||
"displayName": "Starfire Scarlett ASIO",
|
||||
"kind": "Audio",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "asio",
|
||||
"command": "../native/asio_capture/build/Release/mimir_asio_capture.dll",
|
||||
"sampleRate": 192000,
|
||||
"acceptSourceIds": [
|
||||
"asio-ch0",
|
||||
"asio-ch1",
|
||||
"asio-ch2",
|
||||
"asio-ch3"
|
||||
],
|
||||
"sourceLabels": {
|
||||
"asio-ch0": "Shotgun hero mic",
|
||||
"asio-ch1": "Cardioid hero mic",
|
||||
"asio-ch2": "Raven program loopback L",
|
||||
"asio-ch3": "Raven program loopback R"
|
||||
}
|
||||
},
|
||||
{
|
||||
"sourceId": "leap-stereo-ir",
|
||||
"displayName": "LeapUVC stereo IR / depth root",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_f182&pid_0003&mi_00",
|
||||
"width": 640,
|
||||
"height": 240,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "LeapStereoIr",
|
||||
"minimumFramesPerSecond": 100.0,
|
||||
"queueDepth": 8
|
||||
},
|
||||
{
|
||||
"sourceId": "kiyo-pro-rgb",
|
||||
"displayName": "Kiyo Pro AR program view",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_1532&pid_0e05&mi_00",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "YUY2",
|
||||
"minimumFramesPerSecond": 25.0,
|
||||
"queueDepth": 8
|
||||
},
|
||||
{
|
||||
"sourceId": "raven-display",
|
||||
"displayName": "Raven screen capture",
|
||||
"clockDomainId": "raven-sync",
|
||||
"kind": "Video",
|
||||
"origin": "Network",
|
||||
"enabled": true,
|
||||
"adapter": "ffmpeg-rawvideo",
|
||||
"command": "ffmpeg",
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
"pixelFormat": "Bgra8",
|
||||
"arguments": [
|
||||
"-hide_banner",
|
||||
"-nostdin",
|
||||
"-loglevel",
|
||||
"warning",
|
||||
"-fflags",
|
||||
"nobuffer",
|
||||
"-flags",
|
||||
"low_delay",
|
||||
"-i",
|
||||
"srt://0.0.0.0:5210?mode=listener&latency=20000&timeout=5000000",
|
||||
"-an",
|
||||
"-f",
|
||||
"rawvideo",
|
||||
"-pix_fmt",
|
||||
"bgra",
|
||||
"-"
|
||||
]
|
||||
},
|
||||
{
|
||||
"sourceId": "raven-realtk-loopback",
|
||||
"displayName": "Raven Realtek loopback",
|
||||
"clockDomainId": "raven-sync",
|
||||
"kind": "Audio",
|
||||
"origin": "Network",
|
||||
"enabled": true,
|
||||
"adapter": "ffmpeg-pcmaudio",
|
||||
"command": "ffmpeg",
|
||||
"sampleRate": 48000,
|
||||
"channels": 2,
|
||||
"sampleFormat": "Float32",
|
||||
"blockFrames": 960,
|
||||
"arguments": [
|
||||
"-hide_banner",
|
||||
"-nostdin",
|
||||
"-loglevel",
|
||||
"warning",
|
||||
"-fflags",
|
||||
"nobuffer",
|
||||
"-f",
|
||||
"f32le",
|
||||
"-ar",
|
||||
"48000",
|
||||
"-ac",
|
||||
"2",
|
||||
"-i",
|
||||
"srt://0.0.0.0:5212?mode=listener&latency=20000&timeout=5000000",
|
||||
"-f",
|
||||
"f32le",
|
||||
"-"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
187
config/mimir-runtime.well.local.json
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
{
|
||||
"bufferSeconds": 5.0,
|
||||
"audioSync": {
|
||||
"mode": "passive",
|
||||
"referenceSourceId": "asio-ch2",
|
||||
"calibrationGain": 0.0,
|
||||
"watermarkGain": 0.0,
|
||||
"complexContourChannelModelPath": "../calibration/bioacoustic/complex-contour-channel-model.json",
|
||||
"enableComplexContourRuntime": true
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
"sourceId": "focusrite-asio",
|
||||
"displayName": "Starfire Scarlett ASIO",
|
||||
"kind": "Audio",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "asio",
|
||||
"command": "../native/asio_capture/build/Release/mimir_asio_capture.dll",
|
||||
"sampleRate": 192000,
|
||||
"acceptSourceIds": [
|
||||
"asio-ch0",
|
||||
"asio-ch1",
|
||||
"asio-ch2",
|
||||
"asio-ch3"
|
||||
],
|
||||
"sourceLabels": {
|
||||
"asio-ch0": "Shotgun hero mic",
|
||||
"asio-ch1": "Cardioid hero mic",
|
||||
"asio-ch2": "Starfire/Raven loopback L",
|
||||
"asio-ch3": "Starfire/Raven loopback R"
|
||||
}
|
||||
},
|
||||
{
|
||||
"sourceId": "leap-stereo-ir",
|
||||
"displayName": "LeapUVC stereo IR / depth root",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_f182&pid_0003&mi_00",
|
||||
"width": 640,
|
||||
"height": 240,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "LeapStereoIr",
|
||||
"minimumFramesPerSecond": 100.0,
|
||||
"queueDepth": 8
|
||||
},
|
||||
{
|
||||
"sourceId": "kiyo-pro-rgb",
|
||||
"displayName": "Kiyo Pro AR program view",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_1532&pid_0e05&mi_00",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "YUY2",
|
||||
"minimumFramesPerSecond": 25.0,
|
||||
"queueDepth": 8
|
||||
},
|
||||
{
|
||||
"sourceId": "kiyo-basic-rgb",
|
||||
"displayName": "Kiyo RGB context",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_1532&pid_0e03&mi_00",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "YUY2",
|
||||
"minimumFramesPerSecond": 25.0,
|
||||
"queueDepth": 8
|
||||
},
|
||||
{
|
||||
"sourceId": "raven-display",
|
||||
"displayName": "Raven screen capture",
|
||||
"clockDomainId": "raven-sync",
|
||||
"kind": "Video",
|
||||
"origin": "Network",
|
||||
"enabled": true,
|
||||
"adapter": "ffmpeg-rawvideo",
|
||||
"command": "ffmpeg",
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
"pixelFormat": "Bgra8",
|
||||
"arguments": [
|
||||
"-hide_banner",
|
||||
"-nostdin",
|
||||
"-loglevel",
|
||||
"warning",
|
||||
"-fflags",
|
||||
"nobuffer",
|
||||
"-flags",
|
||||
"low_delay",
|
||||
"-i",
|
||||
"srt://0.0.0.0:5210?mode=listener&latency=20000&timeout=5000000",
|
||||
"-an",
|
||||
"-f",
|
||||
"rawvideo",
|
||||
"-pix_fmt",
|
||||
"bgra",
|
||||
"-"
|
||||
]
|
||||
},
|
||||
{
|
||||
"sourceId": "raven-realtk-loopback",
|
||||
"displayName": "Raven Realtek loopback",
|
||||
"clockDomainId": "raven-sync",
|
||||
"kind": "Audio",
|
||||
"origin": "Network",
|
||||
"enabled": true,
|
||||
"adapter": "ffmpeg-pcmaudio",
|
||||
"command": "ffmpeg",
|
||||
"sampleRate": 48000,
|
||||
"channels": 2,
|
||||
"sampleFormat": "Float32",
|
||||
"blockFrames": 960,
|
||||
"arguments": [
|
||||
"-hide_banner",
|
||||
"-nostdin",
|
||||
"-loglevel",
|
||||
"warning",
|
||||
"-fflags",
|
||||
"nobuffer",
|
||||
"-f",
|
||||
"f32le",
|
||||
"-ar",
|
||||
"48000",
|
||||
"-ac",
|
||||
"2",
|
||||
"-i",
|
||||
"srt://0.0.0.0:5212?mode=listener&latency=20000&timeout=5000000",
|
||||
"-f",
|
||||
"f32le",
|
||||
"-"
|
||||
]
|
||||
},
|
||||
{
|
||||
"sourceId": "eve-camera",
|
||||
"displayName": "Eve iPad camera capture",
|
||||
"clockDomainId": "eve-local",
|
||||
"kind": "Video",
|
||||
"origin": "Network",
|
||||
"enabled": true,
|
||||
"adapter": "frame-events",
|
||||
"command": "../src/Mimir.EveSensorReceiver/bin/Debug/net10.0/Mimir.EveSensorReceiver.exe",
|
||||
"arguments": [
|
||||
"--port",
|
||||
"8793",
|
||||
"--path",
|
||||
"/eve/camera",
|
||||
"--source-id",
|
||||
"eve-camera",
|
||||
"--type",
|
||||
"video-frame"
|
||||
]
|
||||
},
|
||||
{
|
||||
"sourceId": "eve-mic",
|
||||
"displayName": "Eve iPad microphone capture",
|
||||
"clockDomainId": "eve-local",
|
||||
"kind": "Audio",
|
||||
"origin": "Network",
|
||||
"enabled": true,
|
||||
"adapter": "frame-events",
|
||||
"command": "../src/Mimir.EveSensorReceiver/bin/Debug/net10.0/Mimir.EveSensorReceiver.exe",
|
||||
"arguments": [
|
||||
"--port",
|
||||
"8794",
|
||||
"--path",
|
||||
"/eve/mic",
|
||||
"--source-id",
|
||||
"eve-mic",
|
||||
"--type",
|
||||
"audio-block"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
# Fensalir field DSL: bind rolling spectrum texture, derive tube fields, shade by surface graph, lower to reservoir.
|
||||
texture id=spectrumHistory
|
||||
|
||||
surface id=amplitude op=texture.sample value=0,0,0,0
|
||||
surface id=derivative op=dFdx inputs=amplitude value=0,0,0,0
|
||||
surface id=curvature op=d2Fdx2 inputs=amplitude value=0,0,0,0
|
||||
surface id=emission op=mix inputs=amplitude,derivative,curvature value=1.0,0.72,0.22,1.0
|
||||
surface id=coverage op=sdf.tube inputs=amplitude,derivative,curvature value=0.74,0.20,0.0,0.0
|
||||
|
||||
splinefield id=asioSpectrumWaterfall texture=spectrumHistory frequencyAxis=x firstColumn=0 columns=160 columnStride=1 rollingModulo=160 subdivisions=4 maxProbes=131072 density=1.0 minimumContribution=0.004 origin=-0.82,0.40,-0.18 axisStep=0.016,0.0,0.0 columnStep=0.0,-0.0055,0.0060 amplitudeScale=0.34 radius=0.064 alpha=0.98 zeroThreshold=0.68 feather=0.18 tangent=1.0 curvature=0.68 normal=0.34 derivative=0.88 emission=1.0,0.72,0.22,4.0 seed=2971668797
|
||||
|
||||
reservoir splats=131072 updates=131072 candidates=2 seed=270943260
|
||||
direct
|
||||
|
|
@ -18,10 +18,16 @@ Inputs become outputs like this:
|
|||
6. Each process sends MPEG-TS over SRT to a stable receiver URL.
|
||||
7. OBS adds each URL as a Media Source.
|
||||
|
||||
`Mimir.RavenDaemon` is the current Raven-local supervisor for the combined
|
||||
display plus Realtek loopback mux. It emits named SRT sinks rather than one
|
||||
anonymous stream: Mimir receives the canonical ingest copy on `5200`, and OBS
|
||||
receives an ordinary media-source copy on `5204`.
|
||||
|
||||
## Invariants
|
||||
|
||||
- Video encoding happens on the sender.
|
||||
- OBS receives ordinary media-source URLs; it does not need a plugin for v1.
|
||||
- Mimir and OBS do not contend for the same SRT listener.
|
||||
- Audio remains separately mixable in OBS.
|
||||
- Ports are stable and derived from one base port plus explicit offsets.
|
||||
- Secrets, if used, stay in local config and are not committed.
|
||||
|
|
|
|||
|
|
@ -25,8 +25,12 @@ flowchart TD
|
|||
|
||||
## Invariants
|
||||
|
||||
- Scarlett speaker loopback is the timing authority when calibration chirplets
|
||||
are playing.
|
||||
- Starfire Realtek is the active chirp speaker and loopback-measurement path in
|
||||
the current routing. The Windows system default output is now Scarlett for
|
||||
monitoring, so default-output chirps are wrong unless they explicitly target
|
||||
Realtek.
|
||||
- Scarlett ASIO is the professional mic and monitor interface. Its outputs are
|
||||
not the current active chirp-emission authority.
|
||||
- Focusrite dialogue mics are the voice anchors.
|
||||
- Camera mics are spatial/context witnesses.
|
||||
- Loopback/program audio is timing evidence where available; it outranks
|
||||
|
|
@ -86,6 +90,48 @@ not the active runtime sound.
|
|||
|
||||
The raw profile is kept even when a timing report is rejected, because a failed
|
||||
sync window can still teach us which bands survived the speaker/room/mic path.
|
||||
`MimirBioacousticProbeScheduler` is now the runtime-side cadence owner for that
|
||||
feedback loop: it consumes smoothed sync states plus per-band frequency response
|
||||
residuals, schedules a targeted probe only when confidence pressure and duty
|
||||
budget allow it, and publishes aggregate/per-source sync and frequency-response
|
||||
confidence telemetry. `MimirSynchronizationHub.UpdateBioacousticProbeSchedule`
|
||||
is the live hub surface; `MimirRuntime` prints
|
||||
`mimir-bioacoustic-probe-schedule` and `mimir-bioacoustic-probe-source` lines so
|
||||
operators can watch confidence climb as probes are spent. The scheduler owns
|
||||
when to spend active sound. It does not own the output device: Starfire Realtek
|
||||
still must be selected explicitly by the eventual live renderer.
|
||||
The first explicit Realtek renderer proof now lives in the repo tooling:
|
||||
`Mimir.WasapiLoopback` can enumerate render endpoints, select one by friendly
|
||||
name substring, and play mono Float32 probe audio through shared-mode WASAPI.
|
||||
`Mimir.BufferSmoke --targeted-bioacoustic-probe-realtk-live` renders a targeted
|
||||
probe plan, starts Scarlett ASIO capture, plays the probe through
|
||||
`Speakers (Realtek(R) Audio)`, and scores the captured shotgun/cardioid bands.
|
||||
The targeted probe planner now caps default acoustic probe selection at 12 kHz
|
||||
and mildly biases low/mid measurable bands, because commodity speakers and the
|
||||
current Scarlett mics have little reason to carry useful calibration evidence
|
||||
above that range. The 2026-06-01 proof at gain `0.75` selected the Realtek speaker endpoint and
|
||||
captured `artifacts/wasapi/targeted-bioacoustic-realtk-live/20260601-033851`.
|
||||
The physical mics heard the burst, but the scored confidence still concentrated
|
||||
in low/mid evidence: shotgun peak `0.407887`, cardioid peak `0.074494`, 18.6 kHz
|
||||
confidence near zero, and the best reported band was cardioid 3.6 kHz at
|
||||
confidence `0.098`. Treat high-band probes as poor questions for this route
|
||||
until a better speaker/mic/path proves otherwise.
|
||||
The live exploration path is scheduler-driven, not a blind sweep:
|
||||
`Mimir.BufferSmoke --bioacoustic-realtk-scheduled-calibration-live` repeatedly
|
||||
asks `MimirBioacousticProbeScheduler` for the next probe plan, emits that plan
|
||||
through Realtek, captures Scarlett, updates residuals, and asks again. This is
|
||||
a live physical I/O diagnostic harness seeded from the current Scarlett residual
|
||||
model; the production runtime still needs to feed measured confidence state into
|
||||
the same scheduler/emitter path. The 2026-06-01 receipt at
|
||||
`artifacts/wasapi/realtk-scheduled-calibration-live/20260601-035354` ran four
|
||||
scheduled probe steps at gain `1.15` and wrote `scheduled-response-bands.csv`
|
||||
plus `balance-plan.md`. Local peak-versus-percentile-floor scoring found strong
|
||||
usable response near `910 Hz`, `3.6 kHz`, and `9.84-9.87 kHz` on both Scarlett
|
||||
mics.
|
||||
The scheduler and planner share the same measurable frequency boundary
|
||||
(`120 Hz` through `12 kHz` by default). Residuals above that window may remain
|
||||
diagnostic scars, but they no longer own live probe cadence or the weakest-band
|
||||
status when the current room path cannot answer them.
|
||||
The receiver must consume persisted models to weight reliable symbols,
|
||||
downweight dead bands, apply phase-coherence weighting, apply first-order
|
||||
group-delay correction, and search joint global delay/frequency-shift hypotheses
|
||||
|
|
@ -93,6 +139,45 @@ before selecting the coherent anchor path. If the measured path only leaves a
|
|||
smaller alphabet, Mimir should use that reliable symbol set and raise the
|
||||
sequence order so the timeline remains unique without pretending dead bands
|
||||
still carry code.
|
||||
The first concrete two-mic cancellation owner is
|
||||
`MimirCalibratedAudioCompositeBuilder`: after fractional alignment and measured
|
||||
band correction, it scores calibrated-band coherence across mics, suppresses
|
||||
incoherent measured-band components, then performs the confidence/noise-weighted
|
||||
sum. `Mimir.BufferSmoke --dual-mic-coherence-cancellation-smoke` proves the
|
||||
contract on synthetic shotgun/cardioid witnesses: coherent shared tones survive,
|
||||
a mic-local `3.52 kHz` interferer is suppressed, and SNR rises from `1.366 dB`
|
||||
to `16.304 dB`. This is still the C# diagnostic/offline contract; Faust/native
|
||||
DSP owns the production hot lane.
|
||||
The first listenable bridge receipt is
|
||||
`scripts/record-kiyo-dual-mic-composite.ps1`. It records Kiyo video and
|
||||
Scarlett ASIO raw channels together, converts Scarlett channels 0/1 through
|
||||
the offline Faust chain with `--asio-dual-mic-faust-cleaner-wav`, then muxes
|
||||
the dialogue WAV with the Kiyo video. That command compiles and runs
|
||||
`faust/mimir_alignment_actuator.dsp` first, then feeds the aligned
|
||||
shotgun/cardioid stems into `faust/mimir_dual_mic_dialogue_cleaner.dsp`; it is a
|
||||
listening receipt for the real DSP chain, not the older C# composite audition.
|
||||
The fresh 2026-06-01 receipt at
|
||||
`artifacts/runtime/kiyo-dual-mic-composite/kiyo-dual-mic-composite-latest.mp4`
|
||||
contains 10.000 s of 640x480 30 fps Kiyo video and 10.000 s of mono AAC from the
|
||||
Faust dialogue stem. Run `20260601-051044` estimated the cardioid 359.398
|
||||
samples late at 48 kHz (7487.452 us), applied holdback 359.398/0.000 samples to
|
||||
shotgun/cardioid before cleanup, wrote a residual WAV, and reported dialogue
|
||||
RMS `0.002678` before receipt normalization. The first muxed receipt was almost
|
||||
inaudible because `NormalizePeak` only attenuated hot samples; the regenerated
|
||||
latest MP4 now peaks the dialogue WAV at -0.724 dBFS with RMS -14.202 dBFS. The
|
||||
hot owner is
|
||||
`faust/mimir_dual_mic_dialogue_cleaner.dsp`: a two-input/two-output Faust graph
|
||||
for aligned shotgun/cardioid witnesses that emits a dialogue composite stem and
|
||||
a rejected residual witness using smooth multiband voice shaping, witness
|
||||
subtraction, and slow downward expansion. `MimirRuntime` now queues that Faust
|
||||
program as `dual-mic-dialogue-cleaner` with `host_voice` and residual `ambient`
|
||||
output stems, but only feeds it from the alignment actuator's published
|
||||
`aligned_source_*` stem frame. Alignment is therefore a structural precursor to
|
||||
cleanup, not an optional sidecar. The routing invariant is covered by
|
||||
`Mimir.BufferSmoke --dialogue-cleaner-routing-smoke`: alignment frames queue the
|
||||
cleaner DSP and cleaner input block, while cleaner output frames are the
|
||||
publishable `NativeProgram` stems. Treat the C# bridge as an audition/scoring
|
||||
receipt, not the final native live DSP path.
|
||||
The synthetic invariant is
|
||||
`dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --bioacoustic-self-test`;
|
||||
it renders the bioacoustic timeline, decodes direct word anchors, and requires a
|
||||
|
|
|
|||
|
|
@ -51,6 +51,9 @@ reflection taps.
|
|||
- The five-second default buffer is compute budget, not accidental delay.
|
||||
- Loopback/reference audio is timing authority; network feeds are producers, not
|
||||
clock authorities.
|
||||
- `MimirSynchronizedBufferPlanner` owns aligned readout from retained buffers:
|
||||
it chooses one canonical presentation time and applies source or
|
||||
`clockDomainId` corrections before Fensalir/Faust consume slices.
|
||||
- UI and telemetry read cached sync state; they do not run analysis.
|
||||
- Native capture owns hardware calls. Runtime owns sample identity and buffer
|
||||
placement.
|
||||
|
|
@ -661,15 +664,16 @@ Owns:
|
|||
|
||||
Blocks:
|
||||
|
||||
- Fields: source list, analyzer, state tracker, latest reports dictionary,
|
||||
- Fields: source list, ordinary audio analyzer, optional complex-contour
|
||||
analyzer, state tracker, latest ordinary/contour report dictionaries,
|
||||
optional chirp-bin calibration model, ingest count, round-robin candidate.
|
||||
- Constructor: creates buffer set, loads optional calibration model, pre-creates
|
||||
configured stream buffers.
|
||||
- Constructor: creates buffer set, loads optional calibration and
|
||||
complex-contour channel models, pre-creates configured stream buffers.
|
||||
- `Settings`, `Buffers`, `IngestedSamples`, `SourceCount`: read-only public
|
||||
state.
|
||||
- `AudioSynchronizationReports`, `AudioSynchronizationStates`,
|
||||
`AudioSynchronizationDecodeTraces`, `ChirpBinEmissionPlan`: cached analyzer
|
||||
and calibration outputs.
|
||||
`ComplexContourSynchronizationReports`, `AudioSynchronizationDecodeTraces`,
|
||||
`ChirpBinEmissionPlan`: cached analyzer and calibration outputs.
|
||||
- `Summary`: short buffer/source summary.
|
||||
- `AddSource`: registers source and ensures descriptor buffer if exposed.
|
||||
- `PollSources`: drains all currently available samples from all sources into
|
||||
|
|
@ -677,6 +681,10 @@ Blocks:
|
|||
- `AnalyzeAudioSynchronization`: full analyzer pass over audio buffers.
|
||||
- `AnalyzeAudioSynchronizationStep`: bounded rotating analyzer pass, so one
|
||||
candidate can be analyzed per tick instead of torching a frame.
|
||||
- `AnalyzeComplexContourSynchronizationStep`: heavy matched-filter contour pass
|
||||
for selected Float32 candidates. Live app and Well cadence paths suppress this
|
||||
inline by default; worker/job ownership or the diagnostic
|
||||
`MIMIR_COMPLEX_CONTOUR_RUNTIME_INLINE=1` override must own execution.
|
||||
- `TryLoadCalibrationModel`: loads persisted chirp-bin response model.
|
||||
- `StoreAudioSynchronizationReports`: updates report cache and smoothed state,
|
||||
removing contradictory passive-negative-lag states.
|
||||
|
|
@ -684,8 +692,10 @@ Blocks:
|
|||
|
||||
Invariant:
|
||||
|
||||
- The hub owns cached runtime belief. The analyzer is a worker; the hub decides
|
||||
what current belief the rest of the app can read.
|
||||
- The hub owns cached runtime belief. Ordinary analyzer steps may run as bounded
|
||||
cadence work; complex-contour matched filtering is configured capability until
|
||||
a worker/job path commits reports back. The hub decides what current belief
|
||||
the rest of the app can read.
|
||||
|
||||
### `MimirAudioSynchronizationReport.cs`
|
||||
|
||||
|
|
@ -901,9 +911,13 @@ Blocks:
|
|||
|
||||
Invariant:
|
||||
|
||||
- The live contour lane only runs when explicitly enabled. When enabled, the
|
||||
runtime emitter and contour analyzer share the same contestant profile instead
|
||||
of learning one song and playing another.
|
||||
- The live contour lane only executes heavy analysis when a worker/job owns the
|
||||
pass or `MIMIR_COMPLEX_CONTOUR_RUNTIME_INLINE=1` is set for diagnostics.
|
||||
`enableComplexContourRuntime` means the capability/profile/model/report lane is
|
||||
configured; it does not give render threads or Well cadence authority to run
|
||||
matched filtering. When executed, the runtime emitter and contour analyzer
|
||||
share the same contestant profile instead of learning one song and playing
|
||||
another.
|
||||
|
||||
### `MimirComplexContourChannelModelDocument.cs`
|
||||
|
||||
|
|
|
|||
190
docs/eve-dashboard.md
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
# Eve Dashboard
|
||||
|
||||
## Objective
|
||||
|
||||
Use EveCanvas as a native operator dashboard substrate: Mimir, StreamPixels, and
|
||||
other local/Yggdrasil-routed apps can publish retained dashboard trees while Eve
|
||||
renders the control surface locally and sends edit intents back. Eve does not
|
||||
own final composition or run remote UI code.
|
||||
|
||||
## Authority Map
|
||||
|
||||
- Owner: each dashboard provider owns its accepted state and command handling.
|
||||
The Starfire broker owns provider discovery/routing. EveCanvas owns native
|
||||
rendering, hit-testing, and touch affordances.
|
||||
- Inputs: Eve sends `select`, `move`, `scale`, `rotate`, `toggle-visibility`,
|
||||
`reset-transform`, and `open-provider` commands over `/eve/deck`.
|
||||
- Outputs: providers send `dashboard-state` snapshots with provider id, title,
|
||||
nodes, health, visibility, dimensions, transform, selected node, and LUT
|
||||
preset. Provider catalogs are exposed at `/eve/deck/providers`.
|
||||
- CultUI-shaped surface: providers may also send `surface`, a retained
|
||||
composition tree for Eve clients. `nodes` remains the semantic compatibility
|
||||
projection; `surface.root` owns composed layout primitives such as workspaces,
|
||||
rails, panes, cards, avatars, text, metrics, lists, and command ids.
|
||||
- Derived state: Eve's UIKit panels are local render projections of Mimir
|
||||
state/provider state.
|
||||
- Forbidden writers: Eve gestures do not mutate program layout directly; they
|
||||
request changes and then re-render the accepted snapshot. Remote apps do not
|
||||
ship arbitrary JavaScript or native code into Eve.
|
||||
- Cut line: no camera-stream fanout to Eve for the dashboard. Source previews
|
||||
may be added later as thumbnails/proxies, but dashboard control must not
|
||||
require Starfire to render every camera feed for the iPad.
|
||||
- CultMesh line: dashboard manifests and states now have typed
|
||||
`mimir.eve_dashboard_manifest.v1` and `mimir.eve_dashboard_state.v1`
|
||||
document shapes. Semantic provider identities use
|
||||
`asgard.starfire.mimir/...`; the broker publishes dashboard state through
|
||||
CultNet reliable UDP for native subscribers. WebSocket endpoints remain
|
||||
renderer compatibility bridges.
|
||||
|
||||
## Runtime
|
||||
|
||||
Start the dashboard server:
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.EveDashboard\Mimir.EveDashboard.csproj -- --port 8795
|
||||
```
|
||||
|
||||
Canonical surface addresses:
|
||||
|
||||
```text
|
||||
asgard.starfire.mimir/eve/tui
|
||||
asgard.starfire.mimir/eve/gui
|
||||
```
|
||||
|
||||
Native CultNet route for typed dashboard state:
|
||||
|
||||
```text
|
||||
cultnet://192.168.1.66:3075
|
||||
schema: mimir.eve_dashboard_state.v1
|
||||
record: eve.dashboard.broker
|
||||
```
|
||||
|
||||
EveCanvas currently connects through this bridge route:
|
||||
|
||||
```text
|
||||
ws://192.168.1.66:8795/eve/deck
|
||||
```
|
||||
|
||||
Periwinkle's Android Eve client currently uses this bridge route:
|
||||
|
||||
```text
|
||||
ws://192.168.1.66:8795/eve/deck/cultmesh
|
||||
```
|
||||
|
||||
The compatibility endpoint remains:
|
||||
|
||||
```text
|
||||
ws://192.168.1.66:8795/eve/dashboard
|
||||
```
|
||||
|
||||
Broker inspection endpoints:
|
||||
|
||||
```text
|
||||
http://192.168.1.66:8795/health
|
||||
http://192.168.1.66:8795/eve/deck/manifest
|
||||
http://192.168.1.66:8795/eve/deck/providers
|
||||
ws://192.168.1.66:8795/eve/deck/{providerId}
|
||||
```
|
||||
|
||||
Extra providers can be registered at launch:
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.EveDashboard\Mimir.EveDashboard.csproj -- `
|
||||
--port 8795 `
|
||||
--voidbot-swarm-state "E:\Projects\VoidBot\.voidbot\status\swarm-state.json" `
|
||||
--provider "app.id|App Title|asgard.starfire.app/eve/tui"
|
||||
```
|
||||
|
||||
If a provider still needs a WebSocket bridge, list it in the provider
|
||||
advertisement or supervisor manifest as a route, not as the provider address.
|
||||
|
||||
## Current State
|
||||
|
||||
`Mimir.EveDashboard` now runs as a small native dashboard broker. It includes:
|
||||
|
||||
- `eve.dashboard.broker`: the switchboard rendered as provider cards.
|
||||
- `mimir.live.stats`: compact live telemetry surface for RMS bars, sync
|
||||
confidence, runtime buffers, actuator commands, and Periwinkle/EVE device
|
||||
observation streams. It recomputes state from Mimir's telemetry and
|
||||
observation ledgers and is broadcast once per second to connected clients.
|
||||
- `mimir.stream.layout`: the existing Mimir source-layout fixture provider.
|
||||
- `voidbot.swarm`: the native VoidBot tab. It reads the existing
|
||||
`swarm-state.json` projection from VoidBot, publishes avatar image URLs on
|
||||
identity nodes, and lets Eve render the CTB rail, selected Face status panel,
|
||||
state tree, and detail pane as native UIKit. It is read-only in this cut;
|
||||
scheduler mutations still belong to VoidBot's heartbeat controls.
|
||||
- `yggdrasil.streampixels.edge`: the first Yggdrasil/StreamPixels service
|
||||
dashboard placeholder for the TCP/SSH-routed live edge.
|
||||
|
||||
The stream-layout provider still serves fixture scene nodes for the first native
|
||||
control proof. It accepts transform/visibility commands, mutates the provider
|
||||
state, increments the state version, and broadcasts the new snapshot. The next
|
||||
cut is replacing fixture state with live `MimirPresentationControlState` and
|
||||
`MimirSceneEditorState` snapshots.
|
||||
|
||||
The live-stats provider is read-only. Its authority is the runtime telemetry
|
||||
surface: `mimir-spectrum`, `mimir-sync-state`, `mimir-sync-report`,
|
||||
`mimir-video-buffer`, `mimir-audio-actuator-command`, and the normalized
|
||||
CultMesh observation ledger from `Mimir.EveSensorReceiver`. If a signal is
|
||||
absent, the surface says so; it does not fabricate confidence.
|
||||
|
||||
## Verification
|
||||
|
||||
The command smoke connected to `/eve/dashboard`, received state version `1`,
|
||||
sent:
|
||||
|
||||
```json
|
||||
{"type":"move","nodeId":"eve-camera","x":0.25,"y":-0.25}
|
||||
```
|
||||
|
||||
and received state version `2` with `eve-camera` moved to that transform. During
|
||||
the same smoke, EveCanvas connected from `192.168.1.72` and remained running.
|
||||
|
||||
The first device screenshot proved the dashboard rendered on Eve, but also
|
||||
showed the legacy debug overlay occluding the hierarchy panel. EveCanvas hides
|
||||
that overlay while dashboard mode is active; dashboard status remains in the
|
||||
native dashboard footer.
|
||||
|
||||
The broker smoke on 2026-05-31 verified:
|
||||
|
||||
- `/health` reports active provider, client count, transport, and typed
|
||||
CultMesh dashboard document name.
|
||||
- `/eve/deck/providers` returns the built-in provider manifests.
|
||||
- WebSocket `/eve/deck` sends the switchboard snapshot.
|
||||
- WebSocket `/eve/deck/cultmesh` sends the same provider state as
|
||||
`mimir.eve_dashboard_state.v1` MessagePack and accepts
|
||||
`mimir.eve_dashboard_command.v1` MessagePack command documents.
|
||||
- `open-provider` switches to `mimir.stream.layout` and broadcasts that
|
||||
provider state.
|
||||
- `open-provider` switches to `voidbot.swarm`, which emits VoidBot's swarm
|
||||
status, CTB rail with avatar URLs, agent cards, and selected Face state
|
||||
detail from the existing VoidBot swarm snapshot.
|
||||
|
||||
## Surface Composition Contract
|
||||
|
||||
`dashboard-state.surface` is the new distributed UI layer. It is intentionally
|
||||
closer to CultUI than to the old flat dashboard node list:
|
||||
|
||||
- `surface.schema`: versioned wire contract, currently
|
||||
`cultmesh.eve_surface.v0`.
|
||||
- `surface.root`: retained element tree. Elements have `kind`, optional `role`,
|
||||
text, asset refs/URIs, provider command ids, layout hints, style variants,
|
||||
metrics, and children.
|
||||
- `surface.assets`: cacheable media references such as avatar images.
|
||||
- `bindNodeId`: optional bridge back to the compatibility `nodes` projection.
|
||||
- `commandId`: provider-owned intent id. Eve clients invoke commands; providers
|
||||
accept or reject and publish the next snapshot.
|
||||
- `binding`: optional typed CultCache/CultMesh field binding. It names
|
||||
`documentSchema`, `documentId`, field `path`, `valueKind`, `access`,
|
||||
`authority`, and optional `commandId`. This is the Verse-backed field
|
||||
contract; native callbacks and DOM handlers are only lowering actuators.
|
||||
|
||||
The old `nodes` array is no longer expected to carry complete UI composition.
|
||||
It is a compatibility/debug/data surface. Clients that understand `surface`
|
||||
should render the composed tree first and use `nodes` only for selection,
|
||||
legacy node binding, and fallback. `bindNodeId` is compatibility glue, not the
|
||||
portable state owner.
|
||||
|
||||
The binary CultMesh state document mirrors this ownership: field 7 carries the
|
||||
optional `surface` tree. Older clients can stop at the node projection; current
|
||||
Eve clients should prefer `surface.root` whenever it is present.
|
||||
84
docs/eve-program-output.md
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
# EVE Program Output
|
||||
|
||||
## Authority Map
|
||||
|
||||
- Owner: Fensalir owns the visible D3D12 program pixels.
|
||||
- Inputs: Fensalir reads the current `AquariumFrame`, render plan, GPU resources,
|
||||
and overlay state.
|
||||
- Outputs: a named shared D3D12 `B8G8R8A8_UNorm` texture containing the completed
|
||||
backbuffer for the current frame.
|
||||
- Derived state: Mimir publication profiles describe the stream contract; they
|
||||
do not render pixels. EVE receives decoded pixels and composites them.
|
||||
- Forbidden writers: WebKit, DOM layout, browser dashboards, OBS capture, and
|
||||
desktop/window capture do not own this path.
|
||||
- Shared paths: local display and EVE streaming both use the same completed
|
||||
Fensalir backbuffer copy.
|
||||
- Deletion line: diagnostic Sunshine/Moonlight or desktop-duplication probes may
|
||||
prove network/decode budget, but they are not production authority.
|
||||
|
||||
## Current Mechanism
|
||||
|
||||
Set these before launching the Mimir app:
|
||||
|
||||
```powershell
|
||||
$env:FENSALIR_PROGRAM_OUTPUT_D3D12 = "1"
|
||||
$env:FENSALIR_PROGRAM_OUTPUT_NAME = "Global\MimirFensalirProgramTexture"
|
||||
$env:FENSALIR_PROGRAM_OUTPUT_FENCE_NAME = "Global\MimirFensalirProgramFence"
|
||||
$env:FENSALIR_PROGRAM_OUTPUT_RING_COUNT = "1"
|
||||
dotnet run --project .\src\Mimir.App\Mimir.App.csproj
|
||||
```
|
||||
|
||||
Fensalir creates a shareable D3D12 texture with that name, publishes a named
|
||||
program-output fence, and copies the finished backbuffer into the texture during
|
||||
the present pass. `Mimir.EveRelay` opens that named texture, reads the completed
|
||||
program frame, feeds a low-latency H.264 encoder, and sends complete Annex-B
|
||||
access units to EveCanvas over the existing `/stream` WebSocket. EveCanvas
|
||||
treats JPEG frames as the old VoidBot dashboard fallback only; Mimir dashboard
|
||||
streaming is `h264-annexb` and displays through a native
|
||||
`AVSampleBufferDisplayLayer` so the iPad owns hardware decode/composite.
|
||||
|
||||
The EVE relay currently consumes the single-texture contract. Keep
|
||||
`FENSALIR_PROGRAM_OUTPUT_RING_COUNT=1` for EVE until the relay has the same
|
||||
slot-selection contract as the OBS program texture source.
|
||||
|
||||
`scripts/start-eve-dashboard-stream.ps1` starts the Mimir/Fensalir app with the
|
||||
shared-texture output enabled, starts the H.264 relay, verifies the `eve` SSH
|
||||
target, opens a reverse SSH tunnel from Eve's `127.0.0.1:8792` back to the
|
||||
local relay when direct inbound TCP is blocked, and launches EveCanvas:
|
||||
|
||||
```powershell
|
||||
powershell -ExecutionPolicy Bypass -File .\scripts\start-eve-dashboard-stream.ps1
|
||||
```
|
||||
|
||||
The relay defaults to `h264_nvenc`. Use `-Encoder libx264` only as a diagnostic
|
||||
fallback when no NVENC-capable FFmpeg is installed.
|
||||
|
||||
EveCanvas tries `ws://127.0.0.1:8792/stream` first, then falls back to
|
||||
`ws://192.168.1.66:8792/stream`. The localhost path is not JPEG tunneling; it is
|
||||
the same H.264 Annex-B stream carried through SSH port forwarding so a Windows
|
||||
firewall rule cannot silently turn the dashboard into a wall.
|
||||
|
||||
## Receiver Contract
|
||||
|
||||
EVE owns:
|
||||
|
||||
- network receive;
|
||||
- hardware decode of H.264 access units;
|
||||
- texture upload/import;
|
||||
- final fullscreen composite;
|
||||
- touch/Pencil/sensor uplink later.
|
||||
|
||||
EVE does not own:
|
||||
|
||||
- dashboard DOM;
|
||||
- Chromium/WebKit compatibility;
|
||||
- layout parity;
|
||||
- Mimir timing authority;
|
||||
- OBS composition authority.
|
||||
|
||||
## Smoke
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --eve-program-output-contract-smoke
|
||||
dotnet build .\src\Mimir.EveRelay\Mimir.EveRelay.csproj --no-restore --disable-build-servers
|
||||
```
|
||||
511
docs/fensalir-rendering-rebuild-migration.md
Normal file
|
|
@ -0,0 +1,511 @@
|
|||
# Fensalir Rendering Rebuild Migration
|
||||
|
||||
## Objective
|
||||
|
||||
Keep Mimir from becoming a second renderer while Fensalir rebuilds itself into
|
||||
the field-evidence machine Mimir needs.
|
||||
|
||||
Mimir's job is to make the room measurable: cameras, microphones, speakers,
|
||||
loopbacks, phones, Raven, and Starfire all feed one bounded temporal window.
|
||||
Fensalir's job is to turn that measured evidence into a coherent visible field
|
||||
and program surface. OBS should receive the final result, not be asked to
|
||||
pretend loose feeds are a world.
|
||||
|
||||
## Current Mechanism
|
||||
|
||||
Mimir currently has the correct broad cut:
|
||||
|
||||
```text
|
||||
drivers / ASIO / network feeds
|
||||
-> Mimir.Runtime rolling buffers
|
||||
-> synchronization, calibration, response models
|
||||
-> Fensalir scene state and debug surfaces
|
||||
-> Fensalir D3D12 presentation / Spout / program output
|
||||
-> OBS
|
||||
```
|
||||
|
||||
The old direct debug-spectrum pressure leak has been cut. Current spectrum
|
||||
debug output is already in the production-shaped bridge:
|
||||
|
||||
```text
|
||||
rolling buffer slice + source identity + calibration + surface intent
|
||||
-> Fensalir field claim
|
||||
-> TubeField candidate producer, if that is the selected claim family
|
||||
-> selected lowering under budget
|
||||
-> evidence lanes and temporal guide
|
||||
-> presentation
|
||||
```
|
||||
|
||||
Mimir must not send "draw these lines" as a production order. It should say
|
||||
"this rolling buffer slice is a spectrum field with stable axes, identities,
|
||||
support, and material intent." Fensalir then chooses direct tube SDF, mesh,
|
||||
field splats, or another lowering.
|
||||
|
||||
The shared Fensalir spatiotemporal reservoir remains the rendering organ.
|
||||
TubeField is only the current claim/candidate producer for rolling-buffer
|
||||
spectrum surfaces. It may own tube-specific resource binding, Catmull-Rom
|
||||
evaluation, SDF/proxy geometry, and material sampling, but it must not become a
|
||||
private renderer or final composition authority.
|
||||
|
||||
This is an intentional divergence from pure ReSTIR. Fensalir is not only
|
||||
resampling lighting after a known primary surface; it is often sampling the
|
||||
visible field candidate itself. TubeField therefore has to generate local
|
||||
primary tube-visibility candidates before reservoir reuse. The shared reservoir
|
||||
owns the temporal-antialiasing-shaped decision about which field candidate persists into
|
||||
presentation. There is no separate TAA owner in the target architecture:
|
||||
reservoir resolve reconstructs smooth presentation directly from selected field
|
||||
evidence instead of repairing unstable pixels after the fact.
|
||||
|
||||
## Invariants
|
||||
|
||||
- Mimir owns raw retention. Fensalir may snapshot or import the five-second
|
||||
window, but it does not silently become history owner.
|
||||
- Mimir owns physical provenance: device ids, sample clocks, calibration ids,
|
||||
path response, confidence, network arrival, and decoded timing evidence.
|
||||
- Fensalir owns dense visual fusion, temporal field evidence, selected cuts,
|
||||
residency, D3D12 resources, debug UI, and program video.
|
||||
- Faust/native DSP owns hot audio movement, resampling, alignment, separation,
|
||||
spatialization, and stems.
|
||||
- OBS owns broadcast composition only.
|
||||
- Render packets are cached lowerings. They are not scene truth.
|
||||
- Reservoir resolve owns temporal antialiasing and reconstruction. TAA is not a
|
||||
separate source-identity or presentation-history authority.
|
||||
- Networked phones and Raven may publish local observations and decoded timing
|
||||
state. They do not become independent clock authorities.
|
||||
|
||||
## Authority Map
|
||||
|
||||
Owner:
|
||||
|
||||
- `Mimir.Runtime` owns stream identity, rolling buffers, synchronization state,
|
||||
calibration loading, and observation publication.
|
||||
- Fensalir's future `FieldEvidenceMachine` owns field claims after Mimir
|
||||
publishes observations and intent.
|
||||
|
||||
Inputs:
|
||||
|
||||
- local direct-driver video frames;
|
||||
- local ASIO loopback and mic blocks;
|
||||
- Raven/phone/network observations;
|
||||
- calibration receipts, response surfaces, codebooks, path ids, sensor poses;
|
||||
- user/app policy for what should enter the live field.
|
||||
|
||||
Outputs:
|
||||
|
||||
- typed observation surfaces for Fensalir;
|
||||
- timing, response, and calibration constraints;
|
||||
- audio actuator control for Faust/native DSP;
|
||||
- health/telemetry state for UI and logs;
|
||||
- final program output from Fensalir and stems from Faust/native DSP.
|
||||
|
||||
Derived state:
|
||||
|
||||
- spectrum lines, debug views, screenshots, OBS bridge endpoints, FFmpeg/SRT
|
||||
helpers, and Eve dashboard feeds are views or diagnostics.
|
||||
- The old direct spline-spectrum dashboard, direct GPU sensor frame bridge, and
|
||||
direct acoustic field frame proof are silent from Mimir's active bridge path.
|
||||
FieldEvidence owns spectrum, camera observation publication, and calibration
|
||||
constraints; screenshots and logs remain diagnostics.
|
||||
|
||||
Forbidden writers:
|
||||
|
||||
- any Mimir-side D3D12 render graph;
|
||||
- app-specific stable Gaussian/evidence cache;
|
||||
- JSON/base64 hot path for live media;
|
||||
- OBS raw source timing as synchronization authority;
|
||||
- per-debug-view geometry deciding production truth.
|
||||
|
||||
Shared paths:
|
||||
|
||||
- live cameras, audio timing anchors, acoustic response surfaces, spectrum
|
||||
diagnostics, calibration events, and future volumetric room claims all publish
|
||||
typed observations/constraints into the same Mimir-to-Fensalir bridge.
|
||||
|
||||
Deletion line:
|
||||
|
||||
- If a Mimir feature wants visual output, first define the observation, claim,
|
||||
or surface intent. A direct draw is allowed only when it is named as fallback
|
||||
or diagnostic and cannot override field evidence.
|
||||
|
||||
## Target Mimir -> Fensalir Contract
|
||||
|
||||
### 1. Raw Window
|
||||
|
||||
Mimir keeps one retention authority:
|
||||
|
||||
```text
|
||||
RollingStreamWindow {
|
||||
windowId
|
||||
duration = 5 seconds by default
|
||||
streamId
|
||||
sourceKind
|
||||
sampleDescriptors
|
||||
nativeHandle or payload view
|
||||
deviceTimestamp
|
||||
canonicalTimestampEstimate
|
||||
sequenceId
|
||||
status
|
||||
}
|
||||
```
|
||||
|
||||
Typed views may index this window. They do not own retention.
|
||||
|
||||
### 2. Observation Surface
|
||||
|
||||
Mimir publishes physical evidence, not render commands:
|
||||
|
||||
```text
|
||||
MimirObservation {
|
||||
observationKey
|
||||
streamId
|
||||
sensorId
|
||||
calibrationId
|
||||
modality: camera | audio | network | timing | response
|
||||
observedTime
|
||||
canonicalTimeEstimate
|
||||
uncertainty
|
||||
payloadHandle
|
||||
provenance
|
||||
confidence
|
||||
}
|
||||
```
|
||||
|
||||
The payload handle is not authority. It must resolve through a declared
|
||||
resource before shader lowering can consume it.
|
||||
|
||||
### 2.5. Resource Declaration
|
||||
|
||||
Live payloads have an explicit Fensalir resource contract:
|
||||
|
||||
```text
|
||||
FensalirResource {
|
||||
resourceKey
|
||||
kind: structured-buffer | texture2d | rolling-texture | mesh | surface-page | volume
|
||||
residency: gpu-resident | shared-gpu
|
||||
shaderAccess: SRV | UAV | vertex-buffer | index-buffer | indirect-args
|
||||
format
|
||||
dimensions / count / stride
|
||||
nativeHandle and nativeHandleKind
|
||||
valid time range
|
||||
version / sequence
|
||||
}
|
||||
```
|
||||
|
||||
Mimir may mint stable resource keys from native/GPU handles, but Fensalir owns
|
||||
resolution. A DSL packet may reference `mimir:resource:*`; it may not treat a
|
||||
bare string as payload truth. In the same runtime, a Mimir-declared buffer is a
|
||||
Fensalir resource slot once the declaration is accepted; native/shared handle
|
||||
metadata is an import edge, not a separate payload authority. Rendering-relevant
|
||||
buffers move to GPU residency as early as possible and stay there; Fensalir
|
||||
passes resource handles/SRVs/UAVs between compute and render lowerings instead
|
||||
of copying samples back to CPU. If the resource contract is absent, expired,
|
||||
duplicated, CPU-only, or incompatible with the selected backend, lowering must
|
||||
defer instead of rendering a lie with nice lighting.
|
||||
|
||||
Current proof surface:
|
||||
|
||||
- `Mimir.BufferSmoke --fensalir-field-evidence-smoke` maps a Mimir rolling
|
||||
window/observation/spectrum intent into one declared resource, three claims,
|
||||
one planned resource-backed `TubeField` packet, and two deferred non-backend
|
||||
claims.
|
||||
- `Mimir.BufferSmoke --fensalir-field-dsl-resource-smoke` uses Fensalir's field
|
||||
evidence DSL to bind the declared resource directly and produce one planned
|
||||
`TubeField` packet with no deferred requests.
|
||||
- `Mimir.BufferSmoke --fensalir-camera-observation-smoke` maps one
|
||||
GPU-backed video frame and one metadata-only cadence witness into field
|
||||
evidence. Only the shared D3D12 video handle becomes a declared `Texture2D`
|
||||
resource and camera surface intent; the metadata-only frame remains
|
||||
observation evidence with no render payload.
|
||||
|
||||
Fensalir now owns the first D3D12 resource resolver cut below this contract:
|
||||
shared structured/curve buffers import/alias GPU-resident resources by handle,
|
||||
while Fensalir-created resources allocate GPU slots only when Fensalir is the
|
||||
producer. The DSL can describe a 2D rolling float buffer as Catmull-Rom XY
|
||||
tubes with modulo column addressing, amplitude power/normalization, radius,
|
||||
ramp texture path, and emission scale. TubeField now consumes those resources
|
||||
through a GPU compute/render path, local Texture2D resources can bind as ramps,
|
||||
SurfacePage and VolumeTexture declarations resolve into shader-readable GPU
|
||||
textures, and Mesh declarations package `Mesh.Vertices`/`Mesh.Indices` GPU
|
||||
buffer shape under one resource key. The evidence DSL can now plan generic
|
||||
resource-backed claims over those declarations. Current blocker for the
|
||||
remaining visual surface is not resource ownership; it is the selected render
|
||||
lowerings that interpret mesh/page/volume resources as geometry, height/SDF/
|
||||
material pages, or density/extinction/SDF3D domains. Mesh layout authority is
|
||||
split by source: imported/user meshes use the standard `PositionNormalUvColor`
|
||||
layout, while generated meshes can be `PipelinePrivate` and leave byte
|
||||
semantics to their selected lowering. TubeField now uses that generated-mesh
|
||||
lane explicitly: compute emits private geometry/indirect buffers and render
|
||||
binds them as a pipeline-private generated mesh before applying TubeField
|
||||
material state. The generated-mesh DrawIndexed indirect command signature is
|
||||
shared ABI; TubeField only owns the argument buffer it emits. TubeField
|
||||
expansion is now gated by selected backend packets, so explicit lowering
|
||||
metadata cannot bypass validation/planning authority. Validation also rejects
|
||||
TubeSpline metadata whose claim is not Tube-encoded or whose resource differs
|
||||
from the claim payload. Mimir's typed surface-intent lowering now emits the
|
||||
matching TubeSpline metadata for audio spectrum/waveform Tube claims. That is
|
||||
engine ownership, not a reason for Mimir to create a parallel renderer.
|
||||
Mimir also uploads normalized spectral frames as Float32 data into a
|
||||
Fensalir-owned structured buffer resource, so the TubeField path consumes live
|
||||
GPU buffer contents rather than an empty declaration. The runtime-level receipt
|
||||
is `Mimir.BufferSmoke --mimir-spectrum-upload-smoke`: it boots Mimir's frame
|
||||
path, advances spectrum analysis, and verifies one newest-slot Float32 upload,
|
||||
planned TubeField packets, a resource-bound local blackbody ramp Texture2D, and
|
||||
no live legacy spline/buffer-field dashboard input. The runtime resource is no
|
||||
longer a latest-frame row dump: it is a rolling column matrix where physical
|
||||
columns are flattened `(history age, source lane)` pairs under fixed history
|
||||
and source-lane capacities. Mimir emits one TubeField claim per active source
|
||||
lane; lowerings use the fixed lane capacity as `ColumnStride`,
|
||||
`ColumnStep.z=0.1`, and `RollingOffset` so each source renders its own age
|
||||
trail from newest to older samples without reshuffling the GPU buffer. New
|
||||
frames upload only the newest ring slot with an explicit element offset.
|
||||
Fensalir owns structured-buffer slot validity and clamps TubeField dispatch so
|
||||
invalid older slots do not reach shader sampling after allocation, reset, or
|
||||
partial update. TubeField metadata now carries stable claim-derived field ids
|
||||
inside the TubeField family range, so temporal/reprojection/reservoir consumers
|
||||
can distinguish source-lane claims. TubeField render now binds the engine
|
||||
blue-noise texture and jitters the tube-local SDF sample inside the proxy
|
||||
fragment shader before writing coverage, normals, and reservoir guide data; the
|
||||
same pass also uses the scene depth buffer so opaque generated field geometry is
|
||||
resolved by nearest surface rather than submission order. The current generated
|
||||
proxy visual still exposes segment structure and is not yet the final smooth
|
||||
tube marcher. Mimir exposes
|
||||
`MIMIR_SPECTRUM_SOURCE_LANES` as the source-lane capacity lever and
|
||||
`MIMIR_SPECTRUM_TUBE_SUBDIVISIONS` as the geometry cost lever for responding to
|
||||
Fensalir's requested/dispatched/truncated/invalid TubeField budget report.
|
||||
Camera descriptors now follow the same evidence/resource boundary:
|
||||
`MimirFensalirFieldLowering.BuildCameraObservationFrame` lowers latest video
|
||||
buffers into observation claims and declares GPU-backed frames as shared
|
||||
`Texture2D` resources. Metadata-only/process-cadence frames do not create
|
||||
surface intents, so a missing payload cannot become a fake `latest-window`
|
||||
render request. Fensalir can now resolve those shared `Texture2D` declarations
|
||||
by native D3D12 handle and accepts Mimir video format names such as `Bgra8`,
|
||||
`Gray8`, `Rg8`, `Nv12`, and `LeapStereoIr`. Camera image claims currently
|
||||
defer because Fensalir has not selected a visual-fusion/render lowering for
|
||||
camera textures yet.
|
||||
|
||||
### 3. Calibration Constraint
|
||||
|
||||
Calibration and sync become explicit constraints:
|
||||
|
||||
```text
|
||||
MimirCalibrationConstraint {
|
||||
constraintKey
|
||||
pathId
|
||||
sourceId
|
||||
receiverId
|
||||
evidenceKind: loopback | bioacoustic | passive | visual | network
|
||||
delayEstimate
|
||||
delayUncertainty
|
||||
phaseOrGroupDelay
|
||||
frequencyResponse
|
||||
usableBandMask
|
||||
confidence
|
||||
}
|
||||
```
|
||||
|
||||
### 4. Surface Intent
|
||||
|
||||
Debug and production visualization share intent:
|
||||
|
||||
```text
|
||||
MimirSurfaceIntent {
|
||||
intentKey
|
||||
sourceObservationKeys
|
||||
domain
|
||||
axes
|
||||
supportPolicy
|
||||
materialGraph
|
||||
updateBudget
|
||||
purpose: debug | calibration | production
|
||||
}
|
||||
```
|
||||
|
||||
The spectrum dashboard becomes one instance of this: a rolling audio spectrum
|
||||
field whose frequency axis maps to X, amplitude to Y, age to Z, and stream
|
||||
identity to stacked lanes.
|
||||
|
||||
## Fresh Dataflow
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A["direct camera drivers"] --> B["Mimir rolling window"]
|
||||
C["ASIO loopback + mics"] --> B
|
||||
D["Raven / phones / network"] --> B
|
||||
B --> E["sync + calibration kernel"]
|
||||
E --> F["typed observations"]
|
||||
E --> G["calibration constraints"]
|
||||
B --> H["surface intent"]
|
||||
F --> I["Fensalir field evidence machine"]
|
||||
G --> I
|
||||
H --> I
|
||||
I --> J["field claims + selected lowerings"]
|
||||
J --> K["program video + debug UI"]
|
||||
E --> L["Faust/native DSP controls"]
|
||||
L --> M["program stems + spatial bed"]
|
||||
K --> N["OBS"]
|
||||
M --> N
|
||||
```
|
||||
|
||||
## Migration Plan
|
||||
|
||||
### Phase 0: Silence Legacy Debug Surfaces
|
||||
|
||||
The direct `AquariumSplineFrame` spectrum dashboard and point-splat spectrum
|
||||
path are no longer scene writers. Live spectrum rendering flows through
|
||||
FieldEvidence TubeField claims and generated mesh expansion.
|
||||
|
||||
### Phase 1: Add Bridge DTOs In Mimir.Runtime
|
||||
|
||||
Create pure mapping types for:
|
||||
|
||||
- rolling stream windows;
|
||||
- resource declarations for live GPU/native payload views;
|
||||
- observations;
|
||||
- calibration constraints;
|
||||
- surface intent.
|
||||
|
||||
The bridge must not read devices, render, allocate per sample, or own timers.
|
||||
It maps existing runtime state into Fensalir-facing contracts.
|
||||
|
||||
Acceptance:
|
||||
|
||||
- one synthetic buffer can produce observations and surface intent in a unit
|
||||
test;
|
||||
- claims that reference live payloads also declare a typed Fensalir resource;
|
||||
- the mapping can run repeatedly without growing allocations;
|
||||
- stream/source ids remain stable across frames.
|
||||
|
||||
### Phase 2: Move Spectrum Dashboard To Surface Intent
|
||||
|
||||
Represent the current spectrum view as a `MimirSurfaceIntent`:
|
||||
|
||||
- stable source lanes;
|
||||
- stable frequency bins;
|
||||
- monotonic history sequence ids;
|
||||
- Z as continuous sample age;
|
||||
- tube/material intent from amplitude/confidence.
|
||||
|
||||
Fensalir may still lower it to direct spline tubes at first. The ownership
|
||||
changes before the backend does.
|
||||
|
||||
Acceptance:
|
||||
|
||||
- app code no longer chooses point splats, direct tubes, or mesh ribbons;
|
||||
- row/band/history identity is the same whether the engine lowers direct tubes
|
||||
or another representation.
|
||||
|
||||
### Phase 3: Camera Observation Bridge
|
||||
|
||||
Lower native camera descriptors into observation surfaces:
|
||||
|
||||
- source id and calibration id;
|
||||
- dimensions, format, stride;
|
||||
- native CPU/GPU handle;
|
||||
- device timestamp and canonical estimate;
|
||||
- lens model and confidence/status.
|
||||
|
||||
Acceptance:
|
||||
|
||||
- implemented bridge proof: GPU-backed video descriptors become declared
|
||||
shared `Texture2D` resources plus camera observation/surface claims;
|
||||
- metadata-only cadence descriptors remain observation evidence and do not
|
||||
become render payloads;
|
||||
- no selected camera-image lowering exists yet, so Fensalir defers those claims
|
||||
until visual fusion/feature extraction owns them.
|
||||
|
||||
### Phase 4: Audio Field Observation Bridge
|
||||
|
||||
Lower sync and calibration state into audio field constraints:
|
||||
|
||||
- ASIO loopback clock fit;
|
||||
- mic delay and SRO state;
|
||||
- per-path frequency response and group delay;
|
||||
- bioacoustic anchor clusters;
|
||||
- passive correlation confidence;
|
||||
- Raven/phone decoded timing observations.
|
||||
|
||||
Acceptance:
|
||||
|
||||
- implemented bridge proof: sync states lower into FieldEvidence calibration
|
||||
constraints instead of direct `AquariumAcousticFieldFrame` packets;
|
||||
- Fensalir can inspect acoustic confidence and constraints without receiving
|
||||
raw PCM as production truth;
|
||||
- selected bridge lowering now exists for audio-path `Phase` and `Confidence`
|
||||
claims: Fensalir plans those constraints as `DebugOverlay` backend packets so
|
||||
calibration timing/path evidence is visible through the FieldEvidence
|
||||
authority instead of a parallel acoustic frame;
|
||||
- acoustic source candidates, confidence volumes, and cross-modal fusion remain
|
||||
Phase 5 work;
|
||||
- Faust/native DSP remains the actuator for sample movement.
|
||||
|
||||
### Phase 5: Sensor Fusion Claims
|
||||
|
||||
Let Fensalir fuse Mimir observations into field claims:
|
||||
|
||||
- sparse feature tracks first;
|
||||
- calibrated camera rays and marker/feature candidates;
|
||||
- acoustic source candidates and confidence volumes;
|
||||
- surface/material claims only after cross-modal evidence earns them.
|
||||
|
||||
Acceptance:
|
||||
|
||||
- implemented synthetic multi-camera marker proof: Mimir lowers a deterministic
|
||||
camera marker candidate with calibration/marker identity and a world-space
|
||||
confidence envelope, and Fensalir plans it as a `DebugOverlay` FieldEvidence
|
||||
packet;
|
||||
- implemented synthetic audio source proof: Mimir lowers an SRP/PHAT-style
|
||||
source candidate with calibration/source identity and a world-space confidence
|
||||
envelope, and Fensalir plans it as a `DebugOverlay` FieldEvidence packet;
|
||||
- claims can be inspected by source observation and calibration id.
|
||||
|
||||
### Phase 6: Program Output Receipts
|
||||
|
||||
Field identity now reaches the OBS-facing program boundary:
|
||||
|
||||
- Fensalir publishes the completed program surface through named shared D3D12
|
||||
textures and `Global\MimirFensalirProgramFence`;
|
||||
- `Mimir Program Texture` opens those textures in OBS and performs the explicit
|
||||
D3D12-to-libobs-D3D11 GPU copy without CPU readback or Spout2;
|
||||
- optional texture rings and `Global\MimirObsProgramConsumerFence` prevent blind
|
||||
producer/consumer reuse;
|
||||
- `Mimir Audio Stem` receives separately controllable Faust/native DSP stems
|
||||
from `Local\MimirObsStemBus`;
|
||||
- telemetry and receipts record timing, evidence confidence, dropped/late
|
||||
observations, and current selected cuts.
|
||||
|
||||
Acceptance:
|
||||
|
||||
- OBS sees final program surfaces/stems through the Mimir OBS plugin;
|
||||
- raw feeds remain debug inputs, not synchronization truth;
|
||||
- recorded receipts explain the stem path, installed OBS plugin, live program
|
||||
texture proof, optional ring proof, dedicated producer fence proof, and
|
||||
two-way consumer fence proof.
|
||||
|
||||
## Verification Matrix
|
||||
|
||||
| Check | Expected Result |
|
||||
| --- | --- |
|
||||
| Stable buffer identity | Source lanes do not reorder or phase-rotate by render frame. |
|
||||
| Missing data | No sample means no observation; no zero-fill masquerade. |
|
||||
| Direct debug path | Can be disabled without changing runtime timing state. |
|
||||
| Observation bridge | Produces typed evidence without device reads or rendering. |
|
||||
| Fensalir lowering swap | Direct tube vs future splat/mesh lowering preserves claim ids. |
|
||||
| Reservoir guide | Presentation history follows evidence validity, not producer folklore. |
|
||||
| OBS output | OBS receives final program surfaces, not raw clock chaos. |
|
||||
|
||||
## Research And Maps
|
||||
|
||||
Primary local references:
|
||||
|
||||
- `docs/perfect-machine.md`
|
||||
- `docs/perfect-machine-domain-index.md`
|
||||
- `docs/sensor-fusion-architecture.md`
|
||||
- `research/perfect-machine-study-2026-05-23/fensalir-integration-map.md`
|
||||
- `research/visual-spatial-map/brushstroke-fusion-rendering.md`
|
||||
- Fensalir `docs/rendering-teardown-rebuild-protocol.md`
|
||||
- Fensalir `docs/perfect-machine-architecture.md`
|
||||
- Fensalir `docs/temporal-spatial-evidence-reservoir.md`
|
||||
|
||||
The next implementation pass should cut Phase 1 first. A bridge DTO that owns
|
||||
nothing but mapping is boring in the best possible way: it makes the rest of
|
||||
the machine harder to lie about.
|
||||
115
docs/fensalir-reservoir-daemon.md
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
# Fensalir Reservoir Daemon
|
||||
|
||||
## Objective
|
||||
|
||||
Move heavy field reconstruction out of the interactive Mimir editor. The editor
|
||||
should present synced video feeds and controls responsively. A separate
|
||||
Fensalir reservoir daemon should drink from the Well, spend whatever CPU/GPU
|
||||
budget is assigned, and publish typed reservoir/program surfaces.
|
||||
|
||||
## Authority Map
|
||||
|
||||
Owner: the Fensalir reservoir daemon owns reservoir work selection, GPU
|
||||
residency, temporal reuse, fusion kernels, worker scheduling, and publication of
|
||||
reservoir-derived program surfaces.
|
||||
|
||||
Inputs: typed Well pages, configured composite state, live timing policy,
|
||||
feature-signal documents, calibration state, Fensalir-owned texture leases, and
|
||||
explicit budget settings.
|
||||
|
||||
Outputs: typed CultMesh/CultCache reservoir status, program texture/fence
|
||||
surfaces, optional diagnostic Eve surfaces, and OBS-facing shared texture or
|
||||
plugin feeds.
|
||||
|
||||
Derived state: Mimir editor telemetry, preview rectangles, Odin/Eve dashboard
|
||||
rows, and OBS plugin status are observers. They do not decide reservoir work.
|
||||
|
||||
Forbidden writers: the interactive editor render loop must not run temporal
|
||||
reservoir history just because it is presenting a backbuffer. Field diagnostics
|
||||
must be opt-in. A camera `FieldEvidenceFrame` claim must not impersonate a flat
|
||||
video compositor.
|
||||
|
||||
Shared paths: direct user placement, configured Well composites, OBS output,
|
||||
daemon diagnostics, and future Eve room lowerings must all reference the same
|
||||
source ids, timing state, and placement/composite document. Manual UI placement
|
||||
and daemon program output may differ only through an explicit selected
|
||||
composite/version.
|
||||
|
||||
Deletion line: old app-hosted reservoir proof visuals remain diagnostic only.
|
||||
They cannot be the default Mimir program output while the current goal is synced
|
||||
video presentation.
|
||||
|
||||
## Current Scar
|
||||
|
||||
On 2026-06-02 the Mimir editor became unresponsive because Fensalir ran
|
||||
reservoir history presentation on a video-only frame. The renderer now gates
|
||||
that pass behind source-derived reservoir-producing input, and the live
|
||||
video-only app log showed `reservoir-update n/a`.
|
||||
|
||||
The remaining gap is real: Fensalir still lacks a flat program-layer video
|
||||
backend that draws Fensalir-owned camera texture leases as 2D synced layers.
|
||||
That backend is separate from the reservoir daemon. Build it as the editor's
|
||||
presentation path; do not smuggle it through field diagnostics.
|
||||
|
||||
## Current Daemon Cut
|
||||
|
||||
`src/Mimir.FensalirDaemon` is the first owner-shaped daemon:
|
||||
|
||||
- eats CultCache through `state/fensalir-daemon.ccmp`, using the canonical
|
||||
MessagePack CultCache backing store;
|
||||
- drinks Well JSONL logs containing `mimir.well_snapshot.v1`,
|
||||
`mimir.well_capture_page.v1`, and `mimir.well_stream_pressure.v1`;
|
||||
- publishes typed `mimir.fensalir_daemon_state.v1` state;
|
||||
- speaks a CultNet-shaped WebSocket Eve provider at `/eve/deck`;
|
||||
- speaks binary CultMesh dashboard documents at `/eve/deck/cultmesh`;
|
||||
- exposes `/health` and `/eve/deck/manifest`;
|
||||
- presents a retained `cultmesh.eve_surface.v0` surface with compact text and
|
||||
metrics that Eve GUI and Odin/Nightwing TUI can lower.
|
||||
|
||||
Start it with:
|
||||
|
||||
```powershell
|
||||
powershell -ExecutionPolicy Bypass -File .\scripts\start-fensalir-daemon.ps1
|
||||
```
|
||||
|
||||
The launcher writes a supervisor manifest under
|
||||
`artifacts/runtime/fensalir-daemon-*` with a semantic provider spec and bridge
|
||||
routes:
|
||||
|
||||
```text
|
||||
mimir-fensalir-daemon|Mimir Fensalir Daemon|asgard.starfire.mimir-fensalir/eve/tui
|
||||
bridge: ws://127.0.0.1:8799/eve/deck
|
||||
bridge: ws://127.0.0.1:8799/eve/deck/cultmesh
|
||||
```
|
||||
|
||||
The current worker mode is `surface-owner-installed`: kernel scheduling,
|
||||
GPU-resident reservoir compute, and actual fused program-surface publication
|
||||
belong behind this daemon next. Do not move those back into the editor.
|
||||
|
||||
## Worker-State Cut
|
||||
|
||||
`Mimir.FensalirDaemon` now also publishes
|
||||
`mimir.fensalir_reservoir_worker_state.v1` beside the daemon summary state.
|
||||
The worker state is the owner surface for reservoir job selection and pressure:
|
||||
|
||||
- capture pages enqueue reservoir jobs only when they contain accepted timing
|
||||
slices;
|
||||
- `mimir.cultmesh_stream_frame.v1` records enqueue fine-grained reservoir jobs
|
||||
from timed stream frames, including paged, inline, native, or shared-handle
|
||||
body transport evidence;
|
||||
- unusable pages are dropped with rejection reasons instead of being hidden by
|
||||
the editor;
|
||||
- queue depth, running jobs, completed jobs, dropped jobs, timing confidence,
|
||||
accepted/rejected slice counts, and estimated GPU queue milliseconds are
|
||||
persisted in CultCache;
|
||||
- `mimir.fensalir_reservoir_pressure.v1` reports Well poll/publish pressure,
|
||||
capture bytes, ready-slice ratio, timing confidence, max presentation
|
||||
distance, GPU queue milliseconds, and drop causes for missing bodies,
|
||||
unavailable fences, or degraded timing;
|
||||
- Eve surfaces bind worker metrics to the worker document, not to ad hoc
|
||||
renderer telemetry.
|
||||
|
||||
The current worker still simulates cost from capture-page shape and configured
|
||||
CPU/GPU share. That is intentional scaffolding: the next cut replaces the cost
|
||||
model with real GPU/Faust/Fensalir kernel dispatch while preserving the same
|
||||
owner and state contract.
|
||||
|
|
@ -20,6 +20,138 @@ For the source-level ownership map, read [[code-algorithm-map|Code Algorithm Map
|
|||
- PowerShell/FFmpeg/SRT remains a bridge utility for LAN OBS feeds. It is not
|
||||
the synchronized program authority.
|
||||
|
||||
Fensalir is not a conventional renderer bolted onto Mimir. It is the engine-side
|
||||
field/evidence machine. Mimir must submit synchronized buffers, constraints,
|
||||
surface intent, and calibration evidence; Fensalir lowers them into claims with
|
||||
travel/depth, metadata, control, and reservoir-guide lanes. The shared Fensalir
|
||||
spatiotemporal reservoir is the organ that owns temporally reused field
|
||||
presentation; TubeField is only the current rolling-buffer tube claim/candidate
|
||||
producer feeding that organ.
|
||||
|
||||
That organ should live as a Fensalir reservoir daemon for production: it drinks
|
||||
typed Well pages/configured composites, owns reservoir budgets and CPU/GPU work
|
||||
allocation, and publishes typed reservoir/program surfaces. The Mimir editor is
|
||||
not that daemon. For the current OBS-shaped cut, the editor is a thin synced
|
||||
video compositor/control surface and must stay responsive even while the
|
||||
reservoir worker is absent, paused, or saturated. Field diagnostics, Leap
|
||||
surface splats, spectrum tubes, and reservoir proof visuals are opt-in
|
||||
diagnostics or daemon outputs, not default editor presentation.
|
||||
`Mimir.FensalirDaemon` is now the first daemon-shaped owner for that path: it
|
||||
tails Well documents, persists `mimir.fensalir_daemon_state.v1` into
|
||||
CultCache, exposes `/health`, and serves JSON plus binary CultMesh
|
||||
`mimir.eve_dashboard_state.v1` Eve surfaces at `/eve/deck` and
|
||||
`/eve/deck/cultmesh`. The current worker mode is deliberately
|
||||
`surface-owner-installed`; dense GPU kernels and fused program output belong
|
||||
behind this daemon next.
|
||||
It also publishes `mimir.fensalir_reservoir_worker_state.v1`, a separate
|
||||
worker document for reservoir scheduling. Capture pages with accepted timing
|
||||
slices become jobs; unusable pages are dropped with reasons. Queue depth,
|
||||
running/completed/dropped jobs, accepted/rejected slice counts, timing
|
||||
confidence, and estimated GPU queue pressure are now CultCache/Eve-visible.
|
||||
The current worker uses a budget-derived cost model; real Fensalir GPU kernels
|
||||
and Faust-assisted feature signals should replace that cost model without
|
||||
moving ownership back into the editor.
|
||||
The online supervisor now gives `Mimir.VerseRecorder` the supervisor `runId`
|
||||
and starts `Mimir.FensalirDaemon` against that exact
|
||||
`VerseCaptures/<runId>/observations.jsonl`, with a run-local CultCache. Fensalir
|
||||
no longer has to discover the Well stream by scanning old artifacts.
|
||||
Fensalir also consumes `mimir.cultmesh_stream_frame.v1` as the fine-grained
|
||||
stream lane, so accepted timed frames can feed reservoir jobs before a whole
|
||||
capture-page batch is useful.
|
||||
`mimir.fensalir_reservoir_pressure.v1` is the pressure readout for the same
|
||||
owner: Well poll/publish pressure, capture bytes, ready-slice ratio, timing
|
||||
confidence, max distance from presentation, GPU queue milliseconds, and drop
|
||||
causes for missing bodies, unavailable fences, or degraded timing.
|
||||
|
||||
Any path that only paints pixels is not the spatiotemporal machine, but the
|
||||
editor still needs a real flat video compositor backend. The current
|
||||
`FieldEvidenceFrame` camera resources are typed declarations/claims; they do
|
||||
not by themselves draw Fensalir-owned live camera texture leases as program
|
||||
layers. Until Fensalir has that 2D textured-quad/program-layer backend, the
|
||||
editor can be responsive and still fail to show synced video. Do not paper over
|
||||
that with reservoir diagnostics.
|
||||
|
||||
Because Fensalir often samples visible field candidates directly rather than
|
||||
lighting samples after a known primary hit, claim producers may need
|
||||
deterministic local visibility generation before reservoir reuse. Reservoir
|
||||
resolve owns temporal antialiasing and reconstruction; TAA is not a separate
|
||||
hidden owner of field identity.
|
||||
|
||||
The first depth-compute target should be a D3D12 stereo disparity lane modeled
|
||||
on permissive SGM provenance, with `libSGM` as the current north-star reference.
|
||||
The research ledger is
|
||||
[[research/d3d12-stereo-depth-provenance-2026-05-29|D3D12 Stereo Depth Provenance]].
|
||||
That note is not a dependency grant: CUDA implementations, TensorRT ports, and
|
||||
monocular model demos are provenance only. The live owner is Fensalir D3D12
|
||||
compute over Mimir-declared texture resources and calibration state.
|
||||
The first live Leap slice now exists in `MimirStereoDepthConfigurations` and
|
||||
`MimirFensalirFieldLowering`. The generic
|
||||
`BuildStereoDepthCandidateFrame` still lowers caller-declared rectified
|
||||
left/right textures into a GPU-resident compute-writable R16F disparity
|
||||
`SurfacePage`, an R8 confidence texture, a Height FieldEvidence claim, and an
|
||||
`AquariumFieldStereoDepthLowering` sidecar. The live
|
||||
`BuildLeapPackedStereoDepthCandidateFrame` path now recognizes the current
|
||||
`LeapStereoIr` rolling video window, reuses its declared packed R8G8 texture as
|
||||
both left and right input resources, names the R/G lane observations, and emits
|
||||
the same Height/depth lowering automatically from the runtime frame. The
|
||||
calibration id is explicitly pending, so the output is disparity evidence, not
|
||||
metric geometry truth. The libSGM-provenance profile currently publishes min
|
||||
disparity 0, 128 disparities, four aggregation paths, census radius 2, P1 8,
|
||||
and P2 96. Fensalir has the first packed-Leap D3D12 compute kernel installed as
|
||||
a crude SAD/block-match disparity writer; it is not full SGM yet.
|
||||
The next socket now exists too: `MimirPointCloudConfigurations` records the
|
||||
Leap disparity-to-point-cloud projection profile, and
|
||||
`BuildLeapPackedStereoPointCloudCandidateFrame` derives a GPU-resident
|
||||
`FieldMesh` point-list resource from the live Leap disparity resource. The
|
||||
runtime merges that Mesh claim alongside the stereo-depth claim, and
|
||||
`Mimir.BufferSmoke --leap-point-cloud-root-smoke` proves the contract plans as
|
||||
one `SurfacePage` packet plus one `Mesh` packet with no deferred requests. This
|
||||
is the first point-cloud root. Fensalir branch `codex/leap-packed-depth` now has
|
||||
the matching native surface-claim lane: `D3D12PointCloudFromDisparityCS` fills
|
||||
the derived point-list Mesh for diagnostics and also writes approximated
|
||||
normal/curvature surface splats into Fensalir's existing SDF/PBR/radiosity
|
||||
reservoir buffers. The point-list renderer is gated as a diagnostic path; live
|
||||
presentation is through the shared splat/reservoir resolver. The live proof is
|
||||
`artifacts/runtime/mimir-leap-surface-claims-clean.png`: LeapUVC -> Fensalir
|
||||
D3D12 texture lease -> packed stereo disparity -> 19,200 generated surface
|
||||
claims -> visible reservoir-resolved splats, with point-list rendering still
|
||||
at `rendered 0`. A second proof,
|
||||
`artifacts/runtime/mimir-leap-history-split-shader-visible.png`, runs without
|
||||
`AQUARIUM_DISABLE_RESERVOIR_HISTORY`: Fensalir now compiles a dedicated lean
|
||||
`D3D12ReservoirHistoryUpdate.hlsl` instead of compiling the monolithic post
|
||||
shader for the history compute pass. That lean pass writes the current resolver
|
||||
sample into all four reservoir-history rows and the resolved target; temporal
|
||||
and spatial reuse still need their own compilable shader cut. The next truth
|
||||
cut is calibrated projection constants, temporal reservoir reuse, and global
|
||||
residual/calibration ownership, not another fake RGB depth path.
|
||||
|
||||
Those claims are not inherently pixel-sized. Pixel-level resolve consumes the
|
||||
reservoir, but claim support is chosen from the represented field. Smooth
|
||||
surfaces should stay broad. Aquarium-style heightfields should be quadtree
|
||||
surface domains with 2D brushes painting tile payloads, and SDF probes emitting
|
||||
surface splats only where curvature, projected error, material/brush detail, or
|
||||
silhouettes justify subdivision.
|
||||
|
||||
The current teardown/migration map lives in
|
||||
[[fensalir-rendering-rebuild-migration|Fensalir Rendering Rebuild Migration]].
|
||||
Mimir's side of that rebuild is to publish typed physical observations,
|
||||
calibration constraints, and surface intent; Fensalir's side is to turn those
|
||||
into field claims, selected lowerings, reusable evidence, temporal guides, and
|
||||
program output.
|
||||
|
||||
Payload handles are now demoted to names inside a resource authority. Mimir
|
||||
declares live native/GPU payloads as typed Fensalir resources with shape,
|
||||
residency, shader access, validity, version, and native handle metadata; claims
|
||||
and lowering requests reference those resource keys. A string handle without a
|
||||
matching resource declaration is not payload truth. For live camera images, the
|
||||
preferred authority is Fensalir-owned texture leasing: Mimir asks the engine
|
||||
broker for a keyed `Texture2D` lease, writes decoded frames into the returned
|
||||
shared D3D12 texture, signals the returned fence, and commits that fence value
|
||||
before the resource key is sampled by shader lowerings. Foreign shared handles
|
||||
remain import edges only. Camera producers must choose the closest-to-device
|
||||
path available for each sensor and report the unavoidable copy count; convenient
|
||||
managed/process middlemen are diagnostic witnesses, not hot-path owners.
|
||||
|
||||
The old script stack is gone. Do not add a compatibility edge unless it protects
|
||||
a named invariant that the native runtime cannot protect yet.
|
||||
|
||||
|
|
@ -28,6 +160,11 @@ a named invariant that the native runtime cannot protect yet.
|
|||
- Mimir public identity, branding, and Face memory.
|
||||
- `Mimir.slnx` with `src/Mimir.App` and `src/Mimir.Runtime`.
|
||||
- Fensalir host bootstrapping from `Mimir.App`.
|
||||
- [[eve-program-output|EVE Program Output]] now owns the first native EVE
|
||||
streaming contract: Fensalir can publish the completed D3D12 backbuffer into a
|
||||
named shared texture when `FENSALIR_PROGRAM_OUTPUT_D3D12=1`, and Mimir records
|
||||
the EVE-facing publication profile as a native D3D12 stream rather than a
|
||||
WebKit/dashboard path.
|
||||
- `MimirSynchronizationHub`, `MimirRollingStreamBuffer`, stream descriptors, and
|
||||
`IMimirStreamSource`.
|
||||
- Configurable five-second default rolling buffers for local and network audio
|
||||
|
|
@ -42,6 +179,30 @@ a named invariant that the native runtime cannot protect yet.
|
|||
production-shaped Focusrite path: a native in-process ASIO callback source
|
||||
feeds sample-bearing 192 kHz Float32 blocks directly into `Mimir.Runtime`
|
||||
rolling buffers on one interface clock domain.
|
||||
- `MimirBioacousticProbeScheduler` closes the first active-probe control loop:
|
||||
sync confidence and per-band frequency-response confidence decide whether a
|
||||
targeted probe should be emitted, and `MimirRuntime` publishes live telemetry
|
||||
for aggregate confidence, deltas, source probe counts, and weak-band pressure.
|
||||
The scheduler is output-device neutral; a later renderer must explicitly
|
||||
target Starfire Realtek for physical chirp emission in the current routing.
|
||||
- `Mimir.WasapiLoopback` now has the first explicit Starfire Realtek render
|
||||
edge: it can list active render endpoints, select `Speakers (Realtek(R)
|
||||
Audio)` by friendly-name substring, and play mono Float32 probe audio through
|
||||
WASAPI shared mode. `Mimir.BufferSmoke
|
||||
--targeted-bioacoustic-probe-realtk-live` couples that renderer with Scarlett
|
||||
ASIO capture and scores the physical mic response for the scheduled probe.
|
||||
The targeted probe planner defaults to a 12 kHz ceiling and low/mid-band bias
|
||||
because the current Realtek/speaker/Scarlett-mic path does not provide useful
|
||||
high-band calibration evidence.
|
||||
- `Mimir.BufferSmoke --bioacoustic-realtk-scheduled-calibration-live` is the
|
||||
scheduler-owned live-I/O calibration receipt: the scheduler chooses each
|
||||
probe step from seeded Scarlett residuals, Realtek emits it, Scarlett captures
|
||||
it, residuals update, and the run writes a per-band CSV plus balance plan
|
||||
instead of mowing the whole spectrum blindly. The remaining production cut is
|
||||
feeding measured runtime confidence state into the same scheduler/emitter path.
|
||||
Scheduler confidence/status uses the same 120 Hz-12 kHz measurable window as
|
||||
the targeted planner, so dead ultrasonic-ish residuals cannot keep cadence
|
||||
pinned to questions this room path cannot answer.
|
||||
- `src/Mimir.BufferSmoke` loads the runtime config, polls the synchronization
|
||||
hub, and prints the actual rolling buffers. Use `--require-samples` when an
|
||||
empty declared sensor buffer should fail the run. Use `--bioacoustic-self-test`
|
||||
|
|
@ -97,16 +258,522 @@ a named invariant that the native runtime cannot protect yet.
|
|||
- `faust/mimir_alignment_actuator.dsp` is the first Faust-owned sample movement
|
||||
surface: six channels of bounded fractional delay and gain controls. Mimir
|
||||
estimates delay/SRO; Faust/native DSP applies correction.
|
||||
- `MimirAlignmentActuatorBank` is the runtime control-plane owner for those
|
||||
estimates: it converts smoothed per-source sync state into nonnegative
|
||||
holdback commands, preserves stable Faust `sourceN` slots for live sources,
|
||||
reports source overflow, and keeps the reference holdback explicit instead of
|
||||
pretending a measured-late stream can be advanced by a positive delay line.
|
||||
- `MimirRuntime` updates audio sync analysis online as a bounded rotating
|
||||
service and can emit live sync telemetry with
|
||||
`MIMIR_SYNC_TELEMETRY_SECONDS`. UI and telemetry read cached reports/states;
|
||||
they do not run synchronization analysis.
|
||||
- `MimirRuntime` publishes live ASIO spectrum history to Fensalir in two layers:
|
||||
`AquariumBufferFieldFrame` carries the real buffer-field intent for spectrum
|
||||
windows as spline-domain tube fields with tangent/curvature/normal/derivative
|
||||
appearance and probe policy, while `AquariumSplineFrame` is the temporary
|
||||
Catmull-Rom tube preview so the buffer contents remain visible before GPU
|
||||
reservoir lowering owns the draw.
|
||||
they do not run synchronization analysis. The same cadence now publishes the
|
||||
latest actuator command frame for telemetry/readout and queues it into
|
||||
`AquariumAudioDocument` as an audio control frame. Mimir also declares the
|
||||
`faust/mimir_alignment_actuator.dsp` program as an
|
||||
`AquariumStreamingDspProgram`, so Fensalir has both the persistent Faust DSP
|
||||
graph and the matching controls. The runtime now also queues sample-bearing
|
||||
`AquariumStreamingAudioBlock` lanes from the latest actuator source buffers,
|
||||
using the bank-assigned Faust `sourceN` slot as channel authority. Fensalir
|
||||
now publishes processed DSP output as `AquariumAudioStemFrame` records through
|
||||
an engine-owned `IAquariumAudioStemBus`, and Mimir declares deterministic
|
||||
aligned-source stem names with the actuator program.
|
||||
`MimirObsStemPublicationState` consumes the Fensalir stem bus without copying
|
||||
audio and validates ready/missing/unconfigured lanes against the
|
||||
alignment-actuator OBS stem profile. `MimirObsStemSharedMemoryPublisher` now
|
||||
publishes those validated stems into a fixed Windows shared-memory ABI, and
|
||||
`native/obs_stem_source` is the first OBS source plugin bundle. It registers
|
||||
`Mimir Audio Stem`, which reads one named stem and submits it to OBS as an
|
||||
audio source, and `Mimir Program Texture`, which opens Fensalir's named D3D12
|
||||
program-output texture and feeds OBS through a GPU-to-GPU D3D12-to-D3D11
|
||||
bridge texture. Fensalir also publishes a named D3D12 program-output fence,
|
||||
and the OBS source opens it before copying so the consumer is not reading
|
||||
blindly or depending on Fensalir's private frame fence. This avoids CPU
|
||||
readback and removes Spout2 as a required program-video dependency, while
|
||||
preserving the explicit boundary cost that libobs is D3D11 on Windows.
|
||||
Fensalir and the OBS source can optionally agree on a bounded publication
|
||||
ring using `FENSALIR_PROGRAM_OUTPUT_RING_COUNT`; OBS selects the latest
|
||||
completed slot from the program-output fence value. OBS can also publish a
|
||||
consumer fence, and when Fensalir is launched with
|
||||
`FENSALIR_PROGRAM_OUTPUT_CONSUMER_FENCE_NAME`, Fensalir skips publication
|
||||
instead of reusing a ring slot OBS has not acknowledged.
|
||||
`scripts/build-obs-stem-plugin.ps1` stages the upstream OBS plugin template
|
||||
SDK under `artifacts/obs-sdk/` and builds the plugin DLL locally.
|
||||
- `MimirSynchronizedBufferPlanner` now builds one aligned presentation frame
|
||||
from the live rolling buffers. It chooses a canonical presentation time inside
|
||||
the shared retained window, applies source or clock-domain timing corrections,
|
||||
and returns per-stream slices for cameras, network display feeds, and audio
|
||||
blocks. This is the low-level Raven display/audio shape: Raven display frames
|
||||
can be tagged with `clockDomainId: raven-sync`, while a Raven audio timing
|
||||
signal routed into Scarlett earns the correction for that shared clock domain.
|
||||
The display feed remains derived evidence, not an independent timing owner.
|
||||
`Mimir.BufferSmoke --synchronized-buffer-planner-smoke` proves the aligned
|
||||
local-camera, Raven-display, loopback, and mic buffer shape.
|
||||
- `MimirFfmpegRawVideoStreamSource` is the current network-pixel adapter for
|
||||
Raven ingest. FFmpeg owns SRT receive/decode, Mimir owns exact raw-video
|
||||
frame reads and synchronized buffer samples. Eve uses EveCanvas-native sensor
|
||||
uplinks instead: `Mimir.EveSensorReceiver` listens for `eve-camera` on
|
||||
WebSocket port `8793` and `eve-mic` on port `8794`, emitting the existing
|
||||
frame-event shape into runtime buffers. `Mimir.BufferSmoke --ffmpeg-rawvideo-source-smoke`
|
||||
proves the local raw-video adapter without Raven online.
|
||||
- `MimirPresentationControlState` owns operator intent for the Fensalir program
|
||||
surface: video feed visibility/solo/opacity/layer order, audio mute/solo/gain,
|
||||
and global LUT preset selection. The `Mimir Program` UI panel is deliberately
|
||||
compact and fadeable. Video choices filter production surface intents before
|
||||
Fensalir composition; audio choices modify Faust gain controls and the samples
|
||||
sent through Fensalir streaming DSP. LUT presets are typed postprocess state
|
||||
with preset paths and strength; until the renderer grows LUT texture sampling,
|
||||
preset exposure and bloom are mapped into existing `GraphicsSettings`.
|
||||
`Mimir.BufferSmoke --presentation-control-smoke` proves the state owner.
|
||||
- The right rail `Output` panel owns OBS-like program actions below the
|
||||
inspector. `MimirProgramRecorder` exposes `Record` / `Stop`, tracks
|
||||
child-process status, writes timestamped MP4 receipts under
|
||||
`C:\Users\Meta\Videos\Mimir` by default, and kills the process tree on stop.
|
||||
`MIMIR_RECORD_OUTPUT_DIR` can redirect receipts. The current actuator
|
||||
deliberately delegates to `scripts/record-stream-proof-output.ps1`, so app
|
||||
recording produces the same stream-proof composite intended for the
|
||||
Yggdrasil/public path: Raven program feed, Kiyo picture-in-picture, Starfire
|
||||
Scarlett audio, and NVENC/AAC broadcast settings.
|
||||
`MIMIR_RECORD_START_RAVEN=1` lets the app ask the script to start Raven's
|
||||
sender; otherwise it records from the already-running feed. `MimirProgramStreamer`
|
||||
exposes `Live` / `Stop` for the configured `Mimir.Broadcast` RTMP path, but it
|
||||
refuses to publish unless `MIMIR_STREAM_INPUT` is set or
|
||||
`MIMIR_STREAM_ALLOW_TEST_PATTERN=1` explicitly authorizes a synthetic smoke.
|
||||
Native program-output recording/streaming can replace those actuators later
|
||||
behind the same UI owner.
|
||||
- `MimirSceneEditorState` owns the new Mimir-window editor graph. It is not the
|
||||
OBS program output: it owns editor camera, selected node, sensor-feed panels,
|
||||
SDF text-panel nodes, model placeholders, visibility, locks, transforms, reset
|
||||
commands, and grab/rotate/resize gizmo intent. `MimirRuntime` renders derived
|
||||
spline outlines and handle markers from that state, and the `Mimir Editor`
|
||||
panel exposes hierarchy, visibility, transform, creation, and reset controls.
|
||||
The center preview now emits an Eve-style retained dashboard document through
|
||||
the C# CultUI-shaped reference API. Fensalir's Direct2D native lowering blits
|
||||
the current renderer output buffer behind Mimir's normalized placement
|
||||
overlays, and the D3D12 hit-test path lowers the same tree into preview
|
||||
gestures. Eve is the portable dashboard authority; CultUI is only the original
|
||||
C# API shape used here so the same document can be lowered by native, web,
|
||||
Android, or iOS backends without making Direct2D the owner. Elements can now
|
||||
carry typed CultCache/CultMesh bindings with document schema/id, field path,
|
||||
value kind, access, authority, and optional command id. Those bindings are
|
||||
the Verse-backed state contract; callbacks remain native lowering actuators.
|
||||
Placement commits
|
||||
keep video layers inside the program frame, snap near edges/center, and
|
||||
preserve the source aspect ratio while resizing through direct preview
|
||||
handles, sliders, or the scale gizmo. Explicit transform/source controls are
|
||||
inspector-only and only appear while a program source is selected; the
|
||||
compositor canvas handles are the primary edit path. Empty-canvas clicks clear
|
||||
source selection, hover/active handles and snap guides are visible through the
|
||||
shared preview state, keyboard arrows nudge selected layers, PageUp/PageDown
|
||||
reorder layers, and the inspector owns the selected source's
|
||||
contain/cover/stretch fit mode.
|
||||
See [[scene-editor-control-surface|Mimir Scene Editor Control Surface]].
|
||||
`Mimir.BufferSmoke --scene-editor-smoke` proves the graph/control/gizmo owner
|
||||
path. World SDF text glyph rendering, ASSIMP-style mesh import/upload, and
|
||||
pixel-accurate gizmo hit-testing remain Fensalir renderer cuts rather than
|
||||
Mimir-side fake outputs.
|
||||
- `MimirRuntime` no longer submits the legacy direct `AquariumSplineFrame`
|
||||
spectrum dashboard. Live spectrum visualization authority is the
|
||||
`AquariumFieldEvidenceFrame`: Mimir declares the rolling resource, emits a
|
||||
Tube claim and `AquariumFieldTubeSplineLowering`, and Fensalir plans/expands
|
||||
the TubeField generated mesh. Mimir now normalizes current spectral frames
|
||||
into a row-major Float32 matrix, attaches that as a field resource upload, and
|
||||
Fensalir copies it into the resolved GPU structured buffer before TubeField
|
||||
compute reads. The old `AquariumBufferFieldFrame` / `ReservoirSplats`
|
||||
dashboard path is also silent. `Mimir.BufferSmoke
|
||||
--mimir-spectrum-upload-smoke` boots `MimirRuntime`, advances the synthetic
|
||||
spectrum preview path, and verifies that the frame contains the Float32
|
||||
resource upload plus planned TubeField packet while legacy spline and buffer
|
||||
field inputs stay empty. The same runtime path declares the local blackbody
|
||||
ramp PNG as a GPU-resident Texture2D resource and binds it by resource key.
|
||||
The uploaded spectrum resource is a rolling column matrix now: physical
|
||||
columns are flattened `(history age, source lane)` pairs, and Mimir emits one
|
||||
Tube claim/lowering per active source. The resource uses fixed history and
|
||||
source-lane capacities, so frame-to-frame content versions and source
|
||||
topology changes do not resize the GPU buffer while the trail fills.
|
||||
`MIMIR_SPECTRUM_SOURCE_LANES` sets the lane capacity; surplus sources are
|
||||
truncated and reported in the runtime UI. Lowerings use that fixed capacity
|
||||
as `ColumnStride`. Physical history slots are addressed as a ring with
|
||||
`RollingOffset`, so logical age no longer requires reshuffling every column
|
||||
before shader sampling. Mimir emits one Float32 resource upload for the
|
||||
newest spectrum ring slot with an explicit element offset, rather than
|
||||
uploading the full fixed-capacity backing buffer every frame. Fensalir owns
|
||||
rolling-slot validity and clamps TubeField dispatch so invalid older slots are
|
||||
not sampled after renderer allocation, reset, or partial update.
|
||||
`MIMIR_SPECTRUM_TUBE_SUBDIVISIONS` controls the requested Catmull-Rom
|
||||
subdivision count so Fensalir's overflow report has a direct cost lever.
|
||||
- `MimirFensalirFieldLowering` now emits `AquariumFieldResourceDeclaration`
|
||||
rows for live native/GPU payload views. Observation claims reference
|
||||
`mimir:resource:*` keys, and Fensalir validation/planning can reject or defer
|
||||
packets whose resources are missing, duplicated, CPU-only, or
|
||||
backend-incompatible.
|
||||
`Mimir.BufferSmoke --fensalir-field-evidence-smoke` proves the first
|
||||
hardware-free receipt with one declared resource, three claims, one planned
|
||||
resource-backed `TubeField` packet, and two deferred non-backend claims.
|
||||
`Mimir.BufferSmoke --fensalir-field-dsl-resource-smoke` proves Fensalir's DSL
|
||||
evidence compiler can bind a declared resource and produce one planned
|
||||
`TubeField` packet with no deferred requests.
|
||||
- `MimirFensalirFieldLowering.BuildCameraObservationFrame` is the current
|
||||
camera bridge proof. It lowers latest video buffers into FieldEvidence
|
||||
observation claims, declares engine-owned or imported video payloads as
|
||||
`Texture2D` resources, and creates camera surface intent only when a resource
|
||||
exists. `MimirFensalirTextureLeaseClient` is the producer-side API for asking
|
||||
Fensalir for a keyed D3D12 texture/fence lease before decoded camera frames
|
||||
are written. `MimirRuntime` accepts `AquariumRuntimeServices` and commits
|
||||
producer fence values for leased video payloads before Fensalir samples them.
|
||||
`MimirVideoCaptureDriverSource` forwards the lease client to drivers that
|
||||
implement `IMimirFensalirTextureLeaseReceiver`, so native/direct camera
|
||||
drivers can allocate their Fensalir destination texture before capture/decode.
|
||||
If a raw single-plane device path must return CPU bytes,
|
||||
`MimirVideoCaptureDriverSource` uploads them directly into the leased texture
|
||||
through the broker, clears the live managed payload, and increments
|
||||
`UnavoidableCopyCount`. NV12 CPU upload is rejected until the engine owns a
|
||||
real planar copy path; GPU/native producers should write the shared texture
|
||||
themselves and stay at zero copies.
|
||||
Metadata-only cadence frames remain observations with empty payload handles;
|
||||
they do not become fake render requests. The old direct
|
||||
`AquariumGpuSensorFrame` builder has been removed from Mimir's proof path.
|
||||
The first production-shaped camera driver is `MimirKsVideoCaptureDriver`,
|
||||
backed by `native/camera_capture/mimir_camera_capture.dll`: it opens a
|
||||
Kernel Streaming capture pin in process, queues uncompressed UVC frame reads,
|
||||
returns `MimirVideoFrameDescriptor` samples through `IMimirVideoCaptureDriver`,
|
||||
and lets `MimirVideoCaptureDriverSource` upload those raw frames into
|
||||
Fensalir texture leases. MJPG/H264 remain outside this lane; they need a
|
||||
device/GPU decode producer rather than a CPU convenience detour.
|
||||
PS3 Eyes use the sibling `MimirPs3EyeVideoCaptureDriver` backed by
|
||||
`native/camera_capture/mimir_ps3eye_capture.dll`; it opens the existing
|
||||
WinUSB/libusb PS3EYEDriver path in process and emits Bayer8 frames through the
|
||||
same source/upload/copy-count machinery.
|
||||
Current direct-driver smokes prove real-frame upload for LeapUVC 640x240
|
||||
YUY2, both PS3 Eyes at 320x240 Bayer8, Kiyo Pro 1920x1080 YUY2, and regular
|
||||
Kiyo 640x480 YUY2. Regular Kiyo 1280x720 YUY2 did not open.
|
||||
`config/mimir-runtime.perfect-machine.local.json` is the current local
|
||||
all-sensor blast profile: Leap packed stereo IR, both PS3 Eyes at
|
||||
320x240@187, Kiyo Pro RGB, and regular Kiyo RGB. `Mimir.BufferSmoke
|
||||
--perfect-machine-live-smoke` opens all five configured video sources,
|
||||
attaches Fensalir texture leases, and passes only when at least five live
|
||||
camera textures, a Leap stereo-depth lowering, a disparity SurfacePage, and a
|
||||
derived point-cloud Mesh are present. Fensalir now plans the camera texture
|
||||
Feature requests through its `GpuSensorFusion` backend instead of deferring
|
||||
them. The current dispatch samples LeapStereoIr, both Bayer8 PS3 Eyes, Kiyo
|
||||
Pro YUY2, and regular Kiyo YUY2. Fensalir binds YUY2 through the
|
||||
Microsoft-documented `R8G8B8A8_UNORM` SRV view over `DXGI_FORMAT_YUY2` and
|
||||
decodes `Y0/U/Y1/V` in compute before scoring features.
|
||||
`MimirMediaFoundationGpuVideoCaptureDriver` is the first compressed camera
|
||||
path: `native/camera_capture/mimir_mf_gpu_capture.dll` uses Media Foundation
|
||||
SourceReader with a D3D11 device manager, selects MJPG/H264 camera modes,
|
||||
decodes on the GPU, copies the decoded GPU surface into a shared BGRA D3D11
|
||||
texture, and publishes only a `shared-d3d11-texture` handle. Live payload
|
||||
bytes stay empty. Kiyo Pro 1920x1080 MJPG->RGB32 and H264->RGB32 smokes both
|
||||
produced GPU handles and valid Fensalir camera resources. NV12 sharing was
|
||||
rejected by D3D11 in this path, so planar NV12 remains future plane-aware
|
||||
interop or GPU bridge-copy work.
|
||||
Fensalir can still resolve imported shared `Texture2D` resources by native
|
||||
D3D12 handle and accepts Mimir video format names.
|
||||
`Mimir.BufferSmoke --fensalir-camera-observation-smoke` verifies the split,
|
||||
and `--fensalir-texture-lease-smoke` verifies the engine-owned lease path.
|
||||
`Mimir.BufferSmoke --stereo-depth-contract-smoke` verifies the next visual
|
||||
socket: libSGM-provenance D3D12 SGM is represented as a non-dependency
|
||||
profile, and synthetic rectified Leap stereo depth lowers through
|
||||
caller-declared left/right input textures, one stereo-depth lowering sidecar,
|
||||
and one planned `SurfacePage` packet with no CUDA, CPU disparity image, or
|
||||
monocular metric authority.
|
||||
- Mimir's active proof path no longer uses the direct
|
||||
`AquariumAcousticFieldFrame` builder either. Sync states lower into
|
||||
FieldEvidence calibration constraints through `MapCalibrationConstraints`,
|
||||
and Fensalir selects audio-path `Phase` and `Confidence` claims as
|
||||
`DebugOverlay` backend packets. `--perfect-machine-profile-smoke` now proves
|
||||
the acoustic constraint is planned through FieldEvidence instead of deferred
|
||||
through a parallel acoustic packet. The same smoke now also proves the first
|
||||
Phase 5 synthetic audio-source candidate: an SRP/PHAT-style localization
|
||||
result is lowered with calibration/source identity and a world-space
|
||||
confidence envelope, then planned as a FieldEvidence `DebugOverlay` packet.
|
||||
It also proves a synthetic multi-camera marker candidate using the same
|
||||
FieldEvidence authority: deterministic marker features plan as
|
||||
`DebugOverlay` packets while ambiguous raw camera features still defer.
|
||||
LED strip calibration now has the same typed evidence path:
|
||||
`MimirLedSplineFieldCandidate` stores ordered per-camera bright-curve samples
|
||||
in image pixels plus local clipspace/frustum coordinates, temporal-code/index
|
||||
stability flags, and shared calibration/spline identity.
|
||||
`BuildLedSplineCandidateFrame` lowers that curve as deterministic
|
||||
camera-sensor Feature evidence, and `--led-spline-calibration-smoke` proves
|
||||
Fensalir plans it as one `DebugOverlay` packet with no deferred request.
|
||||
`MimirLedSplineCurveSolver` now owns the first per-sensor curve solve from
|
||||
bright LED detections into ordered image/clipspace samples. `MimirCameraRigCalibrationSolver`
|
||||
owns the next bounded pose step: given stable LED indices plus a scene-space
|
||||
LED curve anchor, it fits camera translation updates by minimizing ray-to-
|
||||
curve distance from each local frustum. The smoke synthesizes a scene curve,
|
||||
projects it through two camera frustums, solves the curves back out of the
|
||||
detections, and recovers bounded camera-position updates with near-zero ray
|
||||
residual. Rotation and intrinsics fitting remain explicit future work. Exact
|
||||
correspondence requires coded or otherwise stable LED identity; uncoded
|
||||
repeated lights remain curve constraints until the global residual solver
|
||||
resolves them.
|
||||
Exposure/gain sweeps must optimize recovered spline quality, not generic
|
||||
contrast or brightness. `MimirLedSplineQualityScorer` scores each setting by
|
||||
detected LED count, curve coverage, spacing coherence, smoothness, exposure
|
||||
fitness, and saturation penalty. `--led-spline-quality-sweep-smoke` proves
|
||||
the selector chooses the balanced setting over both underexposed missing-LED
|
||||
detections and overexposed saturated halos.
|
||||
`MimirLedSplineFrameAnalyzer` is the frame-level bridge for physical sweeps:
|
||||
it reads luma frames, extracts bright connected components as LED candidates,
|
||||
solves the per-sensor curve, and emits the same spline quality report.
|
||||
`--led-spline-frame-analysis-smoke` proves clipped blobs can still be
|
||||
detected while scoring as unusable calibration evidence.
|
||||
`Mimir.BufferSmoke --led-spline-live-score` now runs that score against live
|
||||
direct sensor feeds. For KS/UVC cameras it sweeps manual exposure/gain,
|
||||
drains stale frames after each write, scores the live luma frames for spline
|
||||
quality, and prints a best setting per source. Current Starfire proof with
|
||||
`--expected-leds 38 --minimum-luma 0.55` selected exposure `-9`, gain `8`
|
||||
for both `kiyo-pro-rgb` (score `0.666`, 41 detections) and
|
||||
`kiyo-basic-rgb` (score `0.650`, 38 detections). LeapUVC rejected the
|
||||
generic KS exposure/gain writes and only recovered two LED components; both
|
||||
PS3 Eye feeds currently score fixed-control frames because their native
|
||||
wrapper does not yet expose gain/exposure.
|
||||
PS Move is now a concrete active-marker input candidate rather than a lore
|
||||
object on a cable. `Mimir.PsMoveProbe` enumerates Sony `VID_054C` /
|
||||
`PID_03D5` over Windows HID with no third-party dependency. The connected
|
||||
controller exposes three HID collections: one generic desktop gamepad
|
||||
collection with 49-byte input/output/feature reports, plus two vendor-defined
|
||||
feature-report collections. The safe `--read` path uses overlapped `ReadFile`
|
||||
plus `CancelIoEx` so idle input reads time out instead of hanging the probe.
|
||||
`--pair-host <bdaddr>` writes a target Bluetooth host address through the
|
||||
PS Move `0x05` feature report on the `col02` collection, following PS Move
|
||||
API's pairing contract. The first live controller `00:06:f5:23:e2:d1` was
|
||||
assigned to Nightwing's BlueZ adapter `5c:93:a2:9c:a8:a8`. Nightwing now
|
||||
carries the matching BlueZ persistent device/cache records from PS Move
|
||||
API's Linux registration path, accepts the PS-button wireless connection,
|
||||
and exposes the controller as `/dev/input/js0`, `/dev/input/event14`, and
|
||||
`/dev/hidraw1`.
|
||||
`tools/nightwing_psmove_music_pulse.py` is the current controlled
|
||||
LED/sphere command proof: Starfire owns the Scarlett ASIO loopback envelope,
|
||||
computes a thresholdable tiny-FFT onset novelty function using the old
|
||||
Perlines-style adaptive whitening pattern: fast/slow scrubbed spectrum,
|
||||
positive delta scrubbed against slow delta, rapidly decaying adaptive min/max
|
||||
normalization, and warm-up/refractory gating. It fits a provisional
|
||||
fundamental, folds spectral bin energy into one octave around that
|
||||
fundamental, and streams dark-idle, high-saturation RGB report `0x06` bursts
|
||||
over SSH to Nightwing's hidraw writer. The current whitened-delta broadband
|
||||
rise is ranked against a recent rolling history, and onset brightness is
|
||||
`percentile^exponent` so exponent `2` maps a median notable frame to 25%
|
||||
brightness while top recent-window rises approach full brightness.
|
||||
Steady-state material falls back toward black between emitted peak frames.
|
||||
The ASIO reader drains the native capture queue to the newest loopback blocks
|
||||
before each LED frame; stale queued audio was the plausible cause of
|
||||
multi-second music-pulse lag, not Bluetooth or the hidraw LED command path.
|
||||
`tools/move_latency_probe.py` is the current diagnostic latency receipt:
|
||||
PS3 Eye is the default fast visual witness, Scarlett cardioid is the default
|
||||
chirp timing witness, and the event train uses an order-3 de Bruijn schedule
|
||||
with symbol-coded chirp bands, LED colors, and small variable gaps. It writes
|
||||
`event-schedule.json`, arms the Nightwing LED script before playback, fits
|
||||
detected audio/video events back to the known schedule, and marks receipts
|
||||
invalid when the Move sphere is not sufficiently visible. A live encoded
|
||||
PS3 Eye run recovered all eight visual events with schedule residuals within
|
||||
about 4 ms; `ffplay`/DirectShow audio startup is still too soft for final
|
||||
physical latency authority, so native timestamped Scarlett playback/capture
|
||||
remains the production timing cut.
|
||||
The music-keyed calibration planner now owns articulated multimodal words:
|
||||
`mimir.music_keyed_move_chirp_plan.v1` carries beat/key analysis, chirp
|
||||
frequency contours, de Bruijn symbols, and per-Move sampled RGB visual
|
||||
contour words. `tools/sync_sweep.py --music-plan` consumes that same plan
|
||||
instead of inventing square flashes, sends the exact RGB samples over
|
||||
Nightwing HID, and writes RGB video `visual_contour` decoder scores
|
||||
(brightness correlation plus color-chroma error) into `summary.json`.
|
||||
With a music plan, the sweep also runs Mimir perfect-machine live receipts
|
||||
for the local Leap/Kiyo runtime and Raven/Eve receiver runtime unless
|
||||
explicitly disabled, so the all-sensor run cannot accidentally omit Leap or
|
||||
Eve merely because FFmpeg cannot archive them directly.
|
||||
`MimirMoveControllerTracker` is the first Mimir-native overlay owner for the
|
||||
streamable product shape: it tracks bright PS Move sphere observations from
|
||||
PS3 Eye luma frames, keeps a bounded history per `controllerId`, preserves
|
||||
the commanded/expected RGB identity for multiple controllers, and lowers the
|
||||
histories through the existing Feature evidence path. Synthetic smoke
|
||||
`--move-controller-overlay-smoke` proves two independent controller histories
|
||||
become one valid FieldEvidence claim/candidate without claiming 6DoF pose.
|
||||
`config/mimir-runtime.stream-proof.local.json` names the current stream-proof
|
||||
profile: Kiyo Pro AR program view, Leap stereo IR depth root, Raven screen
|
||||
and Raven Realtek loopback from one muxed Raven NVENC/AAC MPEG-TS SRT stream
|
||||
on port 5200. Starfire demuxes that single Raven transport locally into
|
||||
`raven-display` on port 5210 and `raven-realtk-loopback` f32 PCM on port
|
||||
5212 because Mimir's current source adapters ingest raw video and PCM
|
||||
separately. Scarlett ASIO channels remain labeled as shotgun/cardioid/local
|
||||
Raven loopback evidence. Nightwing PS3 Eyes no longer appear as Starfire raw
|
||||
video sources in this proof; they publish compact Eye/Move observations.
|
||||
`MimirFfmpegPcmAudioStreamSource` is the network PCM audio owner after local
|
||||
Raven demux. `config/mimir-runtime.move-stream.local.json` remains a
|
||||
local/raw-Eye show reference, not the current split-host authority.
|
||||
The distributed authority split is now explicit: Nightwing owns the local
|
||||
PS3 Eye USB reads, PS Move Bluetooth/HID link, LED schedules, device
|
||||
timestamps, and compact marker/feature extraction receipts. Starfire owns
|
||||
the canonical rolling window, Scarlett timing, global calibration residuals,
|
||||
Kiyo/Raven program sync, and OBS publication. Fensalir owns D3D12 dense
|
||||
feature/fusion work, surface claims, and splat/reservoir program pixels.
|
||||
`MimirDistributedWitnessConfigurations.NightwingEyesMoves` and the
|
||||
`nightwing-eyes-moves` Perfect Machine profile encode this: Nightwing emits
|
||||
typed `mimir.move_controller_observation_state`,
|
||||
`mimir.camera_feature_track_state`, and `mimir.visual_marker_state` rather
|
||||
than becoming a raw-media or camera-placement authority.
|
||||
`Mimir.EveSensorReceiver` is now the first always-on typed observation relay
|
||||
for that split. It accepts Eve/CultMesh-compatible witness observations on
|
||||
`/eve/periwinkle`, normalizes them for the existing text ledger, and also
|
||||
serves a WebSocket subscriber lane on `/eve/periwinkle/subscribe`.
|
||||
`Mimir.VerseRecorder` subscribes to that lane and writes a timestamped
|
||||
`session.json` plus `observations.jsonl` under
|
||||
`C:\Users\Meta\Videos\Mimir\VerseCaptures` by default. It now recognizes
|
||||
`mimir.well_capture_page.v1`, strips inline base64 sample bodies into paged
|
||||
binary files under `bodies/page-*.bin`, and records
|
||||
`mimir.recorder_body_index.v1` refs in `bodies/index.jsonl`. `Mimir.Well` is the
|
||||
long-running Starfire ingest/publish daemon: it loads
|
||||
`config/mimir-runtime.well.local.json`, creates every configured source it
|
||||
can, polls the `MimirSynchronizationHub` with a bounded daemon cadence,
|
||||
updates ordinary audio sync/probe state, and publishes
|
||||
`mimir.well_snapshot.v1` records with source buffers, composite controls,
|
||||
publication/stem contract, and synchronized-frame status. Complex contour can
|
||||
be configured and advertised, but expensive matched-filter contour analysis is
|
||||
suppressed in the Well loop unless `MIMIR_COMPLEX_CONTOUR_RUNTIME_INLINE=1`
|
||||
is set for diagnostics; the production path needs a worker-owned report lane.
|
||||
It also publishes bounded
|
||||
`mimir.well_capture_page.v1` records with synchronized frame-slice timing,
|
||||
configured composite state, and inline source sample bodies when CPU payload
|
||||
bytes exist and fit the configured body limit. The online supervisor
|
||||
`scripts/start-online-verse-daemons.ps1` starts the relay, recorder,
|
||||
Nightwing Eyes/Move/builtin-camera/builtin-mic witness, music-synced
|
||||
de-Bruijn/microtonal Move gesture worker, and `Mimir.Well` from the broad
|
||||
Well profile with capture-page/body-paging enabled. This is the first local
|
||||
paged-recording ABI for offline reconstruction, not the final CultCache shard
|
||||
engine. GPU-handle-only samples remain metadata/body-handle evidence until a
|
||||
Fensalir/native program-output recorder owns rendered composite bodies.
|
||||
`Mimir.Well` now publishes the first stream-organ pressure surface inside
|
||||
`mimir.well_snapshot.v1`: `clockDomains` reports per-domain live edges,
|
||||
actual overlap, and provisional offsets to the ASIO reference, while
|
||||
`streamPressure` (`mimir.well_stream_pressure.v1`) reports poll and publish
|
||||
pressure for the transitional WebSocket/JSON path. The planner feeds
|
||||
provisional clock-domain edge fits through `MimirSourceTimingCorrection`,
|
||||
then uses actual oldest buffered sample timestamps instead of the configured
|
||||
retention boundary when choosing a synchronized window. This makes the live
|
||||
metric honest: a five-second buffer target is no longer allowed to masquerade
|
||||
as five seconds of resident samples when body serialization/backpressure
|
||||
starves a source. The next coherent cut is a CultMesh streaming organ over
|
||||
reliable UDP with a compact typed control lane, typed stream-frame media
|
||||
envelopes, a CultCache page/shard append body lane, explicit
|
||||
cursors/backpressure, and a lossy latest-state dashboard lane; do not keep
|
||||
expanding inline base64 capture pages or SRT/WebSocket bridge edges as if
|
||||
they were the final transport.
|
||||
Eve now follows the same rule for its own device body:
|
||||
`MimirDistributedWitnessConfigurations.EveLocalSensors` and the
|
||||
`eve-local-sensors` Perfect Machine profile make Eve responsible for local
|
||||
camera/mic chirp and flash solves, then shipping compact acoustic path,
|
||||
feature-track, visual-marker, and local-frustum-hint state to Starfire.
|
||||
Starfire still owns global camera placement and the canonical clock.
|
||||
`MimirCameraFrustumCalibrationSolver` is the first global frustum solve: it
|
||||
consumes stable LED/marker correspondences, scene-space marker points, and
|
||||
seed frustums, then emits per-source position, rotation, horizontal/vertical
|
||||
tangent half-FOV, correspondence count, clipspace reprojection residual, and
|
||||
confidence. The same solver accepts `MimirFeatureTrackFieldCandidate` plus
|
||||
scene-space `TrackId` anchors, so PS Move sphere histories, sparse Eye
|
||||
tracks, and Eve-local marker tracks use the same Starfire frustum authority
|
||||
as the LED strip. `mimir.local_frustum_hint_state.v1` is the compact CultMesh
|
||||
state document for local witness solves: producer node, calibration id,
|
||||
marker set id, per-camera pose/frustum snapshot, residual, point count, and
|
||||
confidence. Synthetic smoke `--frustum-calibration-smoke` solves both
|
||||
LED-spline and feature-track frustums for both Eyes, Kiyo Pro, Leap stereo
|
||||
IR, and Eve camera, then emits the feature-track solve as a local frustum
|
||||
hint state, establishing the first fusion-ready frustum surface before dense
|
||||
surface fusion.
|
||||
Next useful cut is direct report decoding plus optical sphere observations
|
||||
from PS3 Eyes/Kiyos/Eve entering the same calibration-evidence path as LED
|
||||
spline constraints.
|
||||
PS3 Eyes now have the first sparse tracking surface:
|
||||
`MimirSparseFeatureTracker` detects grid-suppressed Bayer/luma corner
|
||||
features, links them frame-to-frame by bounded nearest-neighbor motion, and
|
||||
emits sensor-local `MimirFeatureTrackCameraObservation` records with track
|
||||
ids, image/clipspace coordinates, velocity, age, and confidence.
|
||||
`MimirFeatureTrackFieldCandidate` lowers those observations as camera-sensor
|
||||
Feature claims through `BuildFeatureTrackCandidateFrame`. The CPU tracker is
|
||||
a live diagnostic/bootstrap owner; Fensalir D3D12 feature extraction remains
|
||||
the production owner. `--ps3eye-tracking-live` proved both Eyes can produce
|
||||
stable tracks at 320x240@187: with sensitive wide tracking defaults, each Eye
|
||||
returned 160 live tracks from each Eye in a one-second pass, with 160 stable
|
||||
tracks from `ps3-eye-0` and 156 stable tracks from `ps3-eye-1`.
|
||||
A full Fensalir headless proof using that all-sensor profile now runs the
|
||||
actual D3D12 host path. The 2026-05-31 run started all five video sources,
|
||||
declared eight field resources, resolved six Texture2D resources, dispatched
|
||||
one packed-Leap stereo-depth kernel, generated one point-cloud/surface stream,
|
||||
and reported 19,200 visible reservoir splats. The captured proof is
|
||||
`artifacts/runtime/mimir-perfect-machine-all-sensors-headless.png`. The
|
||||
follow-up Fensalir compute proof captured
|
||||
`artifacts/runtime/mimir-perfect-machine-gpu-feature-extraction-headless.png`:
|
||||
Mimir planned 12 FieldEvidence packets with zero deferred requests, and
|
||||
Fensalir dispatched GPU sensor fusion over three sampleable live camera
|
||||
textures, generating 49,152 sparse gaussians. The YUY2 follow-up proof
|
||||
captured
|
||||
`artifacts/runtime/mimir-perfect-machine-yuy2-gpu-feature-extraction-headless.png`:
|
||||
all five camera textures dispatched with zero unsupported field textures,
|
||||
producing 81,920 sparse gaussians at about 0.08 ms/frame sensor-fusion GPU
|
||||
time.
|
||||
Phase 6 program-output receipts are also structurally complete: Fensalir
|
||||
publishes the final program surface through named shared D3D12 texture/fence
|
||||
resources, the OBS plugin consumes that surface with an explicit D3D12-to-
|
||||
libobs-D3D11 GPU copy, optional producer/consumer fences prevent blind ring
|
||||
reuse, and audio stems cross through the fixed OBS stem shared-memory ABI.
|
||||
- Fensalir now owns the first in-process D3D12 field resource resolver cut:
|
||||
shared structured/curve buffer resources import/alias GPU-resident handles,
|
||||
and Fensalir-owned resources allocate GPU slots only when Fensalir is the
|
||||
producer. Texture2D local assets can now be declared as field resources and
|
||||
bound automatically by DSL handles, so TubeField ramps use resource keys
|
||||
rather than raw paths as live authority. Mimir-declared rendering buffers are
|
||||
Fensalir GPU resources in the same runtime and should not round-trip through
|
||||
CPU payloads. The TubeField DSL can now describe a 2D rolling float buffer as
|
||||
Catmull-Rom XY tubes with modulo column addressing, amplitude power/
|
||||
normalization, radius, ramp texture resource, and emission scale. D3D12 now
|
||||
expands and renders those tubes from GPU-resident buffers, samples material
|
||||
per pixel, binds declared local ramp textures, and executes GPU-emitted
|
||||
indirect draw argument packets. Surface-page resources can now be declared and
|
||||
resolved as GPU-resident shader-readable 2D pages; render lowerings that
|
||||
consume those pages as height/SDF/material domains remain future engine cuts.
|
||||
VolumeTexture resources can now be declared and resolved as GPU-resident
|
||||
shader-readable 3D textures for future density/extinction/SDF3D lowerings.
|
||||
Mesh resources now have that explicit ownership contract: `Mesh.Vertices` and
|
||||
`Mesh.Indices` package the vertex/index GPU buffers with topology, index
|
||||
format, bounds, submesh count, and version. The Fensalir DSL can declare a
|
||||
mesh package, and the D3D12 registry allocates/reuses its GPU-resident vertex
|
||||
and index buffers under the mesh resource key. This is resolver authority
|
||||
only; mesh draw/material lowerings remain future engine work. Fensalir's
|
||||
evidence DSL can also create generic resource-backed claims, so mesh/page/
|
||||
volume resources can plan into backend packets before their selected render
|
||||
lowerings exist. Mesh layout authority is split by source: imported/user
|
||||
meshes use the standard `PositionNormalUvColor` vertex layout, while generated
|
||||
meshes declare `PipelinePrivate` and let the selected lowering own the minimal
|
||||
bytes it emits and consumes. TubeField is the first explicit generated-mesh
|
||||
render consumer: its compute path emits private vertex/index/indirect buffers,
|
||||
and render binds them as a `D3D12PipelinePrivateGeneratedMesh` before applying
|
||||
TubeField source/ramp/material state. The DrawIndexed indirect command
|
||||
signature is now generated-mesh-owned rather than TubeField-owned, so future
|
||||
generated lowerings can reuse the same draw ABI with their own producer
|
||||
buffers and material bindings. TubeField dispatch now obeys the selected
|
||||
backend packet plan; unplanned `TubeSplineLowering` records are counted but
|
||||
not expanded. TubeField now writes stable field ids, real tube normals,
|
||||
coverage/confidence, and domain-validity guide data into the same scene
|
||||
metadata/control/reservoir-guide targets that reservoir resolve uses for
|
||||
spatiotemporal reconstruction. The shared field candidate inlet now keeps four
|
||||
fixed candidate slots per pixel, so TubeField and later producers compete
|
||||
through the same resolver-owned budget instead of naming a private winner.
|
||||
Reservoir history update is now a shared Fensalir compute pass over a
|
||||
ping-ponged structured buffer with the same four rows per pixel; it emits the
|
||||
resolved HDR field texture that bloom and presentation consume. The
|
||||
presentation shader is no longer a hidden history/TAA owner.
|
||||
TubeField now feeds the shared reservoir from rolling-buffer column packets
|
||||
instead of generated segment packets: expansion emits one GPU-resident column
|
||||
record per logical history/source column, ReSTIR tile binning admits those
|
||||
columns, and candidate evaluation samples the Catmull-Rom tube SDF/material
|
||||
directly from the source buffer. Generated mesh draw args are zeroed for the
|
||||
live path and remain only a diagnostic reference. Fensalir validation
|
||||
also rejects
|
||||
TubeSpline lowering metadata
|
||||
whose claim is not Tube-encoded or whose resource differs from the claim
|
||||
payload. Mimir's typed surface-intent lowering now emits the matching
|
||||
`AquariumFieldTubeSplineLowering` for audio spectrum/waveform Tube claims, so
|
||||
the real Mimir producer supplies both the planned packet and the generated
|
||||
mesh producer metadata.
|
||||
- `MimirRuntime` queues bioacoustic timeline PCM through Fensalir audio when the
|
||||
active timing witness is allowed. `MimirAudioSynchronizationSettings.Mode`
|
||||
selects `chirp-only`, `passive`, or `hybrid`; passive disables active
|
||||
|
|
@ -191,8 +858,8 @@ a named invariant that the native runtime cannot protect yet.
|
|||
proof, `--perfect-machine-manifest` to export the module manifest for
|
||||
tooling/UI/remote witness use, and `--perfect-machine-lowering-benchmark` to
|
||||
measure the Mimir-to-Fensalir lowering path. The current six-camera/two-audio
|
||||
synthetic lowering benchmark runs at roughly 2.5 us per iteration with about
|
||||
2.3 KB allocated per iteration.
|
||||
synthetic lowering benchmark now measures FieldEvidence camera/resource
|
||||
lowering rather than the retired direct GPU sensor DTO path.
|
||||
- `MimirVideoFrameDescriptor` for dimensions, pixel format, stride, device
|
||||
timestamp, and native/GPU handle metadata.
|
||||
- `IMimirVideoCaptureDriver` and `MimirVideoCaptureDriverSource` as the live
|
||||
|
|
@ -220,8 +887,8 @@ a named invariant that the native runtime cannot protect yet.
|
|||
## Next
|
||||
|
||||
1. Replace the frame-event diagnostic bridge with concrete direct capture
|
||||
drivers for Leap stereo IR first, then the
|
||||
other cameras.
|
||||
drivers for Leap stereo IR first, then the other cameras. Each driver must
|
||||
state its device API, destination resource, and unavoidable copy count.
|
||||
2. Feed those drivers into `MimirVideoCaptureDriverSource` and prove sustained
|
||||
frame cadence in the rolling buffers.
|
||||
3. Promote the packet-song physical calibration receipt into the runtime
|
||||
|
|
@ -239,14 +906,66 @@ a named invariant that the native runtime cannot protect yet.
|
|||
with probe durations long enough to keep loopback and mic windows live.
|
||||
6. Bind Fensalir UI to the synchronization hub so buffer depth, stream cadence,
|
||||
source timestamps, and output settings are visible and adjustable.
|
||||
7. Lower `AquariumBufferFieldFrame` spline tube fields into Fensalir compute:
|
||||
sample buffer-domain paths stochastically by visual contribution, emit SDF
|
||||
splat probes, write them into the spatiotemporal splat reservoir, and sample
|
||||
that reservoir in the temporally antialiased scene pass. The direct spline
|
||||
preview must stay a witness until this path owns rendering.
|
||||
7. Tighten the spectrum spline tube temporal path until it behaves like a
|
||||
production debug surface: shader-owned tube SDF coverage, correct
|
||||
camera-travel metadata, deterministic direct rendering, and temporal
|
||||
accumulation from stable frame budgets. Do not route this dashboard back
|
||||
through fractal point splats.
|
||||
8. Move GPU feature extraction, fusion, material fitting, render budgeting, and
|
||||
Spout2 publication into Fensalir.
|
||||
Spout2 publication into Fensalir. The D3D12 packet/resource resolver now has
|
||||
first authority for structured buffers, local textures, surface pages,
|
||||
volume textures, and mesh packages, and the evidence DSL can plan generic
|
||||
resource-backed claims against those resources. The next engine cuts are
|
||||
selected render lowerings that consume those resources as mesh geometry,
|
||||
camera/feature textures, height/SDF/material pages, and
|
||||
density/extinction/SDF3D volumes. The generic imported mesh ABI is no longer
|
||||
the blocker; TubeField now occupies the first generated-mesh lowering lane.
|
||||
The next renderer decisions are how to expose additional generated mesh
|
||||
producers and which camera/page/volume/material lowering should follow.
|
||||
The first depth-specific cut should be a Fensalir-owned D3D12 stereo SGM
|
||||
lane over rectified synchronized camera texture pairs, emitting
|
||||
GPU-resident disparity/depth/confidence resources and FieldEvidence claims.
|
||||
Keep `libSGM` as provenance for algorithm shape and benchmark pressure, not
|
||||
as imported CUDA authority. The typed contract, live packed-Leap disparity
|
||||
path, Leap point-cloud Mesh socket, and first Fensalir point render lane
|
||||
exist; the next cuts are live editor verification, full HLSL/D3D12 SGM
|
||||
aggregation, calibrated projection constants, and then a global
|
||||
residual/calibration owner.
|
||||
9. Move mic alignment, room suppression, voice separation, spatialization, and
|
||||
stem generation into Faust/native DSP.
|
||||
stem generation into Faust/native DSP. The C# diagnostic bridge now exists
|
||||
as `MimirCalibratedAudioCompositeBuilder`: it consumes synchronization
|
||||
states and measured band responses, emits a corrected composite plus scores,
|
||||
and is proved by `--calibrated-audio-composite-smoke`. It now also scores
|
||||
calibrated-band coherence and suppresses incoherent measured-band components
|
||||
before summing; `--dual-mic-coherence-cancellation-smoke` proves the two-mic
|
||||
case by cutting a mic-local 3.52 kHz interferer while preserving coherent
|
||||
shared tones. `scripts/record-kiyo-dual-mic-composite.ps1` is now the first
|
||||
listenable Faust-chain bridge: it records Kiyo video plus Scarlett ASIO raw
|
||||
channels, renders channels 0/1 through
|
||||
`--asio-dual-mic-faust-cleaner-wav`, and muxes the Faust dialogue stem into
|
||||
the Kiyo MP4. That offline command compiles and runs
|
||||
`faust/mimir_alignment_actuator.dsp` before
|
||||
`faust/mimir_dual_mic_dialogue_cleaner.dsp`, so the receipt exercises the
|
||||
actual alignment-then-cleaner chain. The fresh 2026-06-01 run
|
||||
`artifacts/runtime/kiyo-dual-mic-composite/20260601-051044` estimated a
|
||||
359.398-sample cardioid delay at 48 kHz, applied holdback 359.398/0.000 to
|
||||
shotgun/cardioid, wrote dialogue and residual WAVs, and muxed a 10.000 s
|
||||
640x480@30 Kiyo MP4 with 10.000 s 48 kHz mono AAC. The Faust hot-lane target
|
||||
exists as
|
||||
`faust/mimir_dual_mic_dialogue_cleaner.dsp`; Mimir queues it as
|
||||
`dual-mic-dialogue-cleaner`, consuming the alignment actuator's published
|
||||
`aligned_source_*` stem frame rather than raw mic buffers, and emitting
|
||||
`host_voice` plus a rejected residual witness. The routing smoke is
|
||||
`--dialogue-cleaner-routing-smoke`: it verifies post-alignment stems queue
|
||||
the cleaner program/block and that program publication belongs to the
|
||||
cleaner output, not the raw alignment lanes. Treat the C# bridge as the
|
||||
contract and offline scorer for the
|
||||
native/Faust implementation, not as a parallel live mixer authority. The
|
||||
next matching probe loop also exists as
|
||||
`MimirTargetedBioacousticProbePlanner`: residuals and mic pathology notes
|
||||
select gain-bounded shaped probe bands for the bioacoustic renderer. Start
|
||||
with the 192 kHz shotgun/cardioid pair; only widen to every connected mic
|
||||
once those two paths have credible direct-path and frequency-response
|
||||
confidence.
|
||||
10. Keep the OBS bridge witness ledger as evidence before expanding receiver
|
||||
machinery.
|
||||
|
|
|
|||
321
docs/move-controller-stream.md
Normal file
|
|
@ -0,0 +1,321 @@
|
|||
# Move Controller Stream
|
||||
|
||||
## Objective
|
||||
|
||||
Ship the first Mimir program surface around one concrete show shape:
|
||||
Kiyo Pro is the RGB program view, PS3 Eyes track multiple PS Move spheres,
|
||||
Raven screen capture enters as a synchronized video source, and Scarlett ASIO
|
||||
provides the two hero mics plus Raven program-loopback timing evidence.
|
||||
|
||||
## Authority Map
|
||||
|
||||
- Owner: `Mimir.Runtime` owns sample ingest, rolling buffers, and typed visual
|
||||
evidence. Fensalir owns overlay presentation and final program pixels.
|
||||
- Inputs: Kiyo Pro YUY2 video, both PS3 Eye Bayer8 feeds, Scarlett ASIO
|
||||
channels `asio-ch0..asio-ch3`, and Raven SRT screen capture on port `5200`.
|
||||
- Outputs: `kiyo-pro-rgb` is the base program view; PS Move controller histories
|
||||
lower as Feature evidence through `MimirMoveControllerTracker`; `raven-display`
|
||||
stays in the `raven-sync` clock domain; ASIO labels identify shotgun,
|
||||
cardioid, and Raven loopback lanes.
|
||||
- Derived state: Move 2D histories are observations, not 6DoF pose. Raven
|
||||
display timing is derived from Scarlett audio evidence, not SRT arrival time.
|
||||
- Forbidden writers: PS3 Eye blob tracks do not own camera calibration; Raven
|
||||
network timestamps do not own global sync; Kiyo Pro does not own high-rate
|
||||
motion truth.
|
||||
- Cut line: this profile is the first streamable overlay product, not the full
|
||||
online calibration solve.
|
||||
|
||||
## Nightwing Eyes/Moves Split
|
||||
|
||||
Nightwing is the right body for the PS3 Eyes and PS Move controllers: it has
|
||||
the spare USB topology, BlueZ/Bluetooth, and the local HID path. That does not
|
||||
make it a camera-placement authority.
|
||||
|
||||
The split is:
|
||||
|
||||
- Nightwing-local authority: read PS3 Eye frames, talk to PS Move HID/Bluetooth,
|
||||
apply LED schedules, preserve device timestamps, and extract compact sphere,
|
||||
marker, and sparse Eye feature observations.
|
||||
- Starfire authority: own the canonical rolling window, Scarlett timing,
|
||||
global residual solve, camera placement, Kiyo/Raven program sync, and OBS
|
||||
program publication.
|
||||
- Fensalir authority: own D3D12 dense feature extraction/fusion, surface claims,
|
||||
splat/reservoir resolution, and program pixels.
|
||||
|
||||
`MimirDistributedWitnessConfigurations.NightwingEyesMoves` records this as a
|
||||
typed witness: it may emit `mimir.move_controller_observation_state`,
|
||||
`mimir.camera_feature_track_state`, and `mimir.visual_marker_state`, but it must
|
||||
not stream raw Eye media as the normal live contract and must not own the
|
||||
canonical clock. Raw frames from Nightwing are diagnostic receipts only.
|
||||
|
||||
The first Nightwing receipt is good enough to build on: both PS3 Eyes enumerate
|
||||
as stock Linux `ov534`/`gspca` V4L2 cameras on `/dev/video2` and `/dev/video3`.
|
||||
Default V4L2 cadence is only 30 fps, but explicitly setting
|
||||
`timeperframe=1/187` before streaming makes both cameras deliver about 187 fps
|
||||
at 320x240 with zero sequence gaps. The witness worker must set frame interval;
|
||||
otherwise Linux will quietly look worse than it is.
|
||||
|
||||
Nightwing also has a typed witness publisher staged as
|
||||
`~/.local/bin/nightwing_typed_witness_publisher.py`. The repo source is
|
||||
`tools/nightwing_typed_witness_publisher.py`. It sends binary MessagePack
|
||||
`mimir.eve_sensor_observation.v1` records to `Mimir.EveSensorReceiver` on
|
||||
`ws://192.168.1.66:8796/eve/periwinkle`; the receiver normalizes them into the
|
||||
Mimir observation ledger, Odin projects them as `observation-stream` nodes, and
|
||||
Nightwing's Eve TUI lowers those nodes in the `CultMesh Witness Streams` panel.
|
||||
The current status records are camera-device receipts for `/dev/video*` and a
|
||||
`nightwing-leap-tracking` placeholder that is intentionally `0` until real Leap
|
||||
tracking owns that signal. Values are `[present, videoIndex, isPs3Eye]` for
|
||||
camera-device receipts.
|
||||
|
||||
`MimirRoomCalibrationLockSolver` is the first room-lock coordinator. It consumes
|
||||
the LED string candidate/residual, the camera-rig pose update frame, Move
|
||||
controller feature-history candidates, and PS3 Eye sparse-feature candidates.
|
||||
It only reports `Locked` when every witness family is present: multiple cameras
|
||||
share enough LED indices, the rig solver produced enough pose updates with low
|
||||
mean ray residual, multiple Move identities have stable history, and multiple
|
||||
Eyes contribute enough stable feature tracks. Missing families are reported by
|
||||
name instead of being papered over. `Mimir.BufferSmoke
|
||||
--room-calibration-lock-smoke` proves a synthetic three-camera LED string, two
|
||||
Moves, and two Eyes lock at confidence `0.778`, while removing Move history
|
||||
correctly drops the result to `InsufficientWitnesses`.
|
||||
|
||||
## Runtime Profile
|
||||
|
||||
For the current stream-proof demo, use the corrected Starfire/Nightwing/Raven
|
||||
profile:
|
||||
|
||||
```powershell
|
||||
E:\Projects\Mimir\scripts\start-stream-proof.ps1
|
||||
```
|
||||
|
||||
This profile keeps Nightwing raw Eye pixels off Starfire. Starfire ingests local
|
||||
Scarlett ASIO, local Leap stereo IR, local Kiyo Pro RGB, Raven screen capture,
|
||||
and Raven Realtek loopback PCM. Nightwing publishes compact Eye/Move
|
||||
observations through `/eve/periwinkle`; those observations are calibration and
|
||||
overlay evidence, not raw media ownership.
|
||||
|
||||
The profile declares:
|
||||
|
||||
- `leap-stereo-ir`: LeapUVC stereo IR / depth root.
|
||||
- `kiyo-pro-rgb`: Kiyo Pro AR program view.
|
||||
- `raven-display`: Raven screen capture from the muxed Raven A/V stream,
|
||||
locally demuxed to SRT port `5210`, in the `raven-sync` clock domain.
|
||||
- `raven-realtk-loopback`: Raven Realtek render loopback from the same muxed
|
||||
Raven A/V stream, locally demuxed as f32 PCM to SRT port `5212`, also in
|
||||
`raven-sync`.
|
||||
- `focusrite-asio`: Starfire hero mics plus local Raven/program loopback lanes
|
||||
on `asio-ch0..asio-ch3`.
|
||||
|
||||
For the narrower Leap/Eyes/Kiyo Pro bring-up path, use:
|
||||
|
||||
```powershell
|
||||
E:\Projects\Mimir\scripts\start-mvp-leap-eyes-kiyo.ps1
|
||||
```
|
||||
|
||||
This launches Starfire with only local Scarlett ASIO, local Leap stereo IR, and
|
||||
local Kiyo Pro RGB in the Mimir rolling buffers. Nightwing does not stream PS3
|
||||
Eye pixels to Starfire. The launcher stages `nw_eye_cap.py`,
|
||||
`nw_move_hint.py`, and `nightwing_typed_witness_publisher.py` on Nightwing,
|
||||
then starts the typed witness publisher with `--track-eyes`. Nightwing reads
|
||||
`/dev/video2` and `/dev/video3`, extracts compact Move-sphere observations,
|
||||
and sends `mimir.move_controller_observation_state.v1` claims to the
|
||||
`/eve/periwinkle` CultMesh observation receiver. Starfire/Fensalir can consume
|
||||
those claims for overlays and calibration without inheriting Nightwing's raw
|
||||
camera bandwidth or local tracking authority.
|
||||
|
||||
The MVP profile refuses to start when LeapUVC is not enumerated. That is
|
||||
intentional: the Leap is the dense close-range geometry root, not an optional
|
||||
checkbox pretending to be present.
|
||||
|
||||
Use:
|
||||
|
||||
```powershell
|
||||
$env:MIMIR_RUNTIME_CONFIG = "E:\Projects\Mimir\config\mimir-runtime.move-stream.local.json"
|
||||
dotnet run --project .\src\Mimir.App\Mimir.App.csproj
|
||||
```
|
||||
|
||||
The profile declares:
|
||||
|
||||
- `kiyo-pro-rgb`: Kiyo Pro program view, 640x480 YUY2 at the current reliable
|
||||
25 fps class.
|
||||
- `ps3-eye-0`, `ps3-eye-1`: high-rate Move sphere visual witnesses.
|
||||
- `focusrite-asio`: one ASIO producer that emits:
|
||||
`asio-ch0` shotgun hero mic, `asio-ch1` cardioid hero mic,
|
||||
`asio-ch2` Raven program loopback L, and `asio-ch3` Raven program loopback R.
|
||||
- `raven-display`: Raven screen capture over SRT port `5200`.
|
||||
|
||||
Run the synthetic overlay smoke:
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --move-controller-overlay-smoke
|
||||
```
|
||||
|
||||
Run the live ingest smoke once Raven is sending:
|
||||
|
||||
```powershell
|
||||
$env:MIMIR_RUNTIME_CONFIG = "E:\Projects\Mimir\config\mimir-runtime.move-stream.local.json"
|
||||
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --seconds 5 --require-samples --sync-reference asio-ch2
|
||||
```
|
||||
|
||||
## Raven Sender
|
||||
|
||||
On Raven, start the muxed A/V sender from the logged-in desktop session:
|
||||
|
||||
```powershell
|
||||
E:\Projects\Mimir\scripts\start-raven-av-sender.ps1 -TargetHost 192.168.1.66 -Port 5200
|
||||
```
|
||||
|
||||
That sender captures the desktop through `gdigrab`, encodes video with
|
||||
`h264_nvenc`, captures Realtek/default render loopback through the repo WASAPI
|
||||
loopback script, encodes audio as AAC, and muxes both into one MPEG-TS SRT
|
||||
stream. On Starfire, `scripts\start-stream-proof.ps1` starts
|
||||
`scripts\start-raven-av-demux.ps1`, which listens on port `5200` and splits the
|
||||
single transport into local Mimir ingest legs:
|
||||
|
||||
```powershell
|
||||
raven-display local SRT 5210, copied H.264/MPEG-TS for decode
|
||||
raven-realtk-loopback local SRT 5212, decoded f32 PCM
|
||||
```
|
||||
|
||||
If Raven program audio is also routed into Starfire Scarlett on
|
||||
`asio-ch2/asio-ch3`, that remains another strong local timing witness. The
|
||||
Realtek loopback is now part of the same Raven transport as the display before
|
||||
Starfire demuxes it for Mimir's raw ingest APIs.
|
||||
|
||||
Raven display sync is not allowed to ride on SRT arrival time. Both
|
||||
`raven-display` and `raven-realtk-loopback` are in the `raven-sync` clock
|
||||
domain. When audio sync estimates the Realtek loopback offset against the
|
||||
Scarlett reference, `MimirSynchronizedBufferPlanner` applies that correction to
|
||||
the whole `raven-sync` domain, including the display. The LAN transport is now
|
||||
one muxed A/V stream; the local split exists only at the Starfire demux edge
|
||||
because Mimir currently ingests raw video and PCM through separate source
|
||||
adapters.
|
||||
|
||||
## Kiyo AR Trail Receipt
|
||||
|
||||
The first offline proof that the Kiyo view can accept Move trails is:
|
||||
|
||||
```powershell
|
||||
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\solve_stream_proof_frustum.py .\artifacts\runtime\all-sensor-articulated-20260531-232349\sweep\calibration-observations.json
|
||||
```
|
||||
|
||||
That solver uses Nightwing's two matched optical witnesses as a provisional
|
||||
local witness space, fits the Kiyo Pro observations into it, and writes a
|
||||
`mimir.stream_proof_frustum_solve.v1` receipt plus an AR trail preview. It is
|
||||
enough for the stream demo and for Starfire's global residual owner to consume;
|
||||
it is not final metric room calibration.
|
||||
|
||||
## Multiple Move Controllers
|
||||
|
||||
Do not collapse Move observations into a generic bright blob. Each controller
|
||||
has its own `controllerId` and expected RGB/time code. The PS3 Eyes can track
|
||||
the sphere as a high-rate brightness witness; Kiyo Pro can later confirm color
|
||||
identity; the commanded flash schedule ties the observed marker to the
|
||||
controller identity.
|
||||
|
||||
`MimirMoveControllerTracker` keeps a bounded history per controller and lowers
|
||||
those histories into the existing Feature evidence path. This is deliberately
|
||||
2D observation history. 6DoF pose waits for calibrated frustums and the global
|
||||
residual owner.
|
||||
|
||||
## Music-Keyed Calibration Words
|
||||
|
||||
`tools/music_keyed_move_chirp_sync.py` is the first planner for calibration
|
||||
words that are both musical and machine-decodable. It listens to Scarlett ASIO,
|
||||
uses the same Perlines-style adaptive FFT delta whitening as
|
||||
`tools/nightwing_psmove_music_pulse.py`, autocorrelates the whitened broadband
|
||||
spectral-rise function for tempo, estimates a lightweight root from recent
|
||||
best-fit fundamentals, then writes one shared plan:
|
||||
|
||||
- beat-aligned chirps quantized to the estimated minor-pentatonic scale;
|
||||
- de Bruijn-coded event symbols so adjacent timing/frequency/color words remain
|
||||
distinguishable;
|
||||
- per-Move visual words using stable controller identities and unique colors;
|
||||
the default is no longer a square flash, but a sampled RGB contour with
|
||||
attack/release envelope, symbol-dependent hue glide, tremolo articulation,
|
||||
and an emphasized controller lane;
|
||||
- a JSON receipt shaped like AquaSynth song analysis vocabulary:
|
||||
`tempo_bpm`, `beat_seconds`, `tempo_confidence`, `root_note`,
|
||||
`suggested_scale`, `scale_frequencies_hz`, and
|
||||
`whitened_spectral_autocorr`.
|
||||
|
||||
The synthetic smoke is:
|
||||
|
||||
```powershell
|
||||
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\music_keyed_move_chirp_sync.py --mode synthetic --dry-run --events 8 --out-dir artifacts/runtime/music-keyed-move-chirp-sync-smoke
|
||||
```
|
||||
|
||||
The live dry-run against Scarlett is:
|
||||
|
||||
```powershell
|
||||
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\music_keyed_move_chirp_sync.py --dry-run --analyze-seconds 5 --events 8 --out-dir artifacts/runtime/music-keyed-move-chirp-sync-live-dry
|
||||
```
|
||||
|
||||
For double-time material:
|
||||
|
||||
```powershell
|
||||
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\music_keyed_move_chirp_sync.py --dry-run --tempo-grid doubletime --tempo-max-bpm 260 --analyze-seconds 5 --events 8 --out-dir artifacts/runtime/music-keyed-move-chirp-sync-dirtyphonics-dry
|
||||
```
|
||||
|
||||
On the 2026-05-31 live dry-run, the planner produced a usable but not final
|
||||
receipt: about `92.31` BPM, beat `0.650s`, tempo confidence `0.343`, root `A`,
|
||||
key confidence `0.716`. That is enough to prove the dataflow, not enough to
|
||||
claim final musical tempo authority.
|
||||
|
||||
For dense double-time material, use `--tempo-grid doubletime`; the planner then
|
||||
selects the double-time member of the autocorrelation tempo family instead of
|
||||
preferring the slower parent grid. On the Dirtyphonics-style live source this
|
||||
produced about `171.43` BPM, beat `0.350s`, tempo confidence `0.358`, root `A`,
|
||||
key confidence `0.703` after widening the search ceiling with
|
||||
`--tempo-max-bpm 260`. The receipt records `tempo_family` so later decoders can
|
||||
see the selected grid alongside half-time/double-time alternatives.
|
||||
|
||||
Visual calibration words are deliberately shaped like the audio words: one
|
||||
gesture carries more than one bit. `--visual-gesture contour` records a
|
||||
`mimir.move_visual_contour_word.v1` per Move per event, with `sample_rate_hz`,
|
||||
duration, base identity color, symbol emphasis, and the exact RGB samples sent
|
||||
over HID. Start conservatively around `--visual-gesture-hz 80`; the HID writer
|
||||
accepted a short 80 Hz contour test on Nightwing, but the useful rate is the
|
||||
rate cameras can recover cleanly, not merely the rate the controller accepts.
|
||||
`--visual-gesture square` remains available as a baseline comparator.
|
||||
|
||||
`tools/sync_sweep.py` can now consume that exact plan with `--music-plan`.
|
||||
That makes the plan the single authority for audio chirps, sampled Move RGB
|
||||
contours, and offline video scoring. The sweep writes `gesture-plan.json`,
|
||||
`event-schedule.json`, copied/rendered `chirp-train.wav`, per-sensor captures,
|
||||
and `summary.json`. RGB camera receipts include a `visual_contour` block with
|
||||
brightness-envelope correlation and color-chroma error against the expected
|
||||
Move contour. The old peak-only score is still present as a timing witness; it
|
||||
no longer pretends square flashes are the only visual language.
|
||||
|
||||
When `--music-plan` is used, the sweep also runs typed Mimir runtime sensor
|
||||
receipts unless `--skip-mimir-live` is passed. The default receipts are:
|
||||
|
||||
- `config/mimir-runtime.perfect-machine.local.json` for local Leap stereo IR,
|
||||
Kiyo Pro, Kiyo, and local camera texture/depth/field-resource ingress;
|
||||
- `config/mimir-runtime.raven-eve.example.json` for Raven display and Eve
|
||||
camera/mic frame-event listeners.
|
||||
|
||||
That receipt path is not raw media archival yet. It is the audit hook that
|
||||
prevents Leap or Eve from being omitted by a DirectShow/Nightwing-only sweep.
|
||||
|
||||
After a sweep, run `tools/crunch_sync_sweep.py` over the sweep directory to
|
||||
extract a compact calibration observation table. It merges Nightwing's typed
|
||||
Move observations with local Kiyo/Kiyo Pro frame analysis, estimates each
|
||||
local video's schedule shift against the event train, and writes
|
||||
`calibration-observations.json` plus `calibration-observations.csv`.
|
||||
|
||||
Example using an already-written musical plan:
|
||||
|
||||
```powershell
|
||||
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\sync_sweep.py --music-plan artifacts/runtime/music-keyed-move-chirp-sync-live/music-keyed-move-chirp-plan.json --out-dir artifacts/runtime/all-sensor-articulated-sync
|
||||
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\crunch_sync_sweep.py artifacts/runtime/all-sensor-articulated-sync
|
||||
```
|
||||
|
||||
The non-dry run targets the two wireless Nightwing Moves by default:
|
||||
`00:07:04:a6:be:5f` on `/dev/hidraw2` and `00:06:f5:23:e2:d1` on
|
||||
`/dev/hidraw3`. Nightwing now carries `/etc/udev/rules.d/70-move.rules`, which
|
||||
sets Bluetooth PS Move hidraw nodes to `root:input 0660`. The `metacrat` user
|
||||
is in `input`, so the non-root SSH writer can drive both spheres. Root SSH is
|
||||
available as the short local alias `nwroot` for maintenance only; the hot LED
|
||||
path should stay non-root.
|
||||
|
|
@ -56,6 +56,37 @@ buffer handles directly.
|
|||
The probe reported one unrelated invalid USB descriptor from
|
||||
`USB\VID_0000&PID_0002`; it did not prevent PS3 Eye capture.
|
||||
|
||||
## 2026-05-31: Nightwing Linux V4L2 PS3 Eye Probe
|
||||
|
||||
The two PS3 Eyes were moved to Nightwing. Linux enumerated them as Sony
|
||||
Playstation Eye USB devices (`1415:2000`) through the stock `ov534`/`gspca`
|
||||
V4L2 path:
|
||||
|
||||
- `/dev/video2`: `USB Camera-B4.09.24.1`, bus `usb-0000:00:14.0-1`
|
||||
- `/dev/video3`: `USB Camera-B4.09.24.1`, bus `usb-0000:00:14.0-2`
|
||||
|
||||
Probe source:
|
||||
|
||||
- `tools/nightwing_v4l2_cadence_probe.py`
|
||||
- copied to Nightwing and run directly against `/dev/video2` and `/dev/video3`
|
||||
- no `v4l2-ctl`, `ffmpeg`, or third-party Python package dependency
|
||||
|
||||
Initial reads without setting frame interval negotiated only the default
|
||||
`timeperframe=1/30`. After explicitly setting `VIDIOC_S_PARM` to
|
||||
`timeperframe=1/187`, both Eyes delivered high-rate frames with zero sequence
|
||||
gaps.
|
||||
|
||||
| Device | Requested Mode | Actual Mode | Frames | Delivered FPS | Device FPS | Sequence Gap |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| `/dev/video2` | 320x240 GRBG @ 187 | 320x240 GRBG | 375 | 187.07 | 187.37 | 0 |
|
||||
| `/dev/video3` | 320x240 GRBG @ 187 | 320x240 GRBG | 375 | 187.14 | 187.37 | 0 |
|
||||
| `/dev/video2` | 320x240 YUYV @ 187 | 320x240 YUYV | 374 | 186.94 | 187.37 | 0 |
|
||||
| `/dev/video3` | 320x240 YUYV @ 187 | 320x240 YUYV | 375 | 187.06 | 187.38 | 0 |
|
||||
|
||||
Conclusion: Nightwing's stock Linux V4L2 path can sustain the high-rate PS3 Eye
|
||||
witness mode if the witness worker explicitly sets frame interval before
|
||||
streaming. Default V4L2 reads are a false low-cadence signal.
|
||||
|
||||
## 2026-05-21: LeapUVC Kernel Streaming Probe
|
||||
|
||||
The Leap Motion Controller is currently present as `VID_F182&PID_0003` and bound
|
||||
|
|
|
|||
|
|
@ -36,9 +36,13 @@ The app-level runtime initializes one rolling buffer per configured audio or
|
|||
video stream, defaulting to five seconds. The lower native reservoir keeps the
|
||||
single-edge typed-handle invariant for Fensalir/Faust integration.
|
||||
|
||||
Payload memory belongs to the producing domain. The reservoir and runtime own
|
||||
timing, identity, ordering, retention, and lookup. They do not pretend to own
|
||||
camera driver buffers, GPU textures, or DSP audio memory.
|
||||
Payload memory belongs to the producing domain, but rendering inputs should be
|
||||
Fensalir-owned GPU resources whenever the device path allows it. The camera hot
|
||||
path is: ask Fensalir for a keyed `Texture2D` lease, speak to the device stack
|
||||
as directly as possible, populate that lease, signal the producer fence, and
|
||||
publish the resource key through Mimir's rolling buffers and FieldEvidence. The
|
||||
reservoir and runtime own timing, identity, ordering, retention, and lookup.
|
||||
They do not become pixel middlemen.
|
||||
|
||||
## Authoritative Boundaries
|
||||
|
||||
|
|
@ -62,7 +66,10 @@ camera driver buffers, GPU textures, or DSP audio memory.
|
|||
|
||||
1. Keep the C# app/runtime as the public Mimir surface.
|
||||
2. Implement Leap stereo IR direct ingest first and measure sustained cadence.
|
||||
3. Add the remaining local camera drivers through the same source seam.
|
||||
Record the unavoidable copy count; do not hide CPU upload behind a friendly
|
||||
adapter name.
|
||||
3. Add the remaining local camera drivers through the same source seam,
|
||||
preferring device-direct producers that populate Fensalir texture leases.
|
||||
4. Add native audio capture workers and feed aligned blocks to Faust/native DSP.
|
||||
5. Bind Fensalir UI to stream health, buffer depth, source timestamps, settings,
|
||||
and output management.
|
||||
|
|
|
|||
63
docs/network-display-sync.md
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
# Network Display Synchronization
|
||||
|
||||
Raven can be a low-level synchronized producer when it controls both its display
|
||||
output and an audio timing signal routed into the Scarlett.
|
||||
|
||||
Authority map:
|
||||
|
||||
- Owner: `Mimir.Runtime` owns aligned rolling-buffer frames.
|
||||
- Inputs: camera buffers, Raven display frames, Scarlett ASIO audio buffers, and
|
||||
explicit timing corrections learned from audio evidence.
|
||||
- Outputs: `MimirSynchronizedBufferFrame` slices at one canonical presentation
|
||||
time for Fensalir/Faust/OBS publication.
|
||||
- Derived state: Raven display timing is derived from the Raven audio sync
|
||||
signal. The display feed is not a timing authority by itself.
|
||||
- Forbidden writers: OBS, SRT bridge endpoints, frame-event probes, and network
|
||||
receive timestamps must not decide cross-stream alignment. CultMesh reliable
|
||||
UDP packet arrival is transport evidence only, not clock authority.
|
||||
- Shared path: local cameras, network display frames, mic channels, loopback
|
||||
channels, and Raven audio sync channels all enter `MimirRollingStreamBuffer`
|
||||
first, then the synchronized planner chooses slices.
|
||||
- Deletion line: any compositor path that directly consumes Raven pixels outside
|
||||
the runtime buffer/alignment path is diagnostic only.
|
||||
|
||||
Practical shape:
|
||||
|
||||
1. Raven renders a visible timing marker or encoded frame identity on the
|
||||
captured display.
|
||||
2. Raven emits the matching audio timing signal through its audio output.
|
||||
3. That audio is physically or digitally routed into a Scarlett input beside the
|
||||
mic channels.
|
||||
4. Mimir decodes the Scarlett audio evidence, learns the Raven clock offset, and
|
||||
applies that clock-domain correction to the `raven-display` video buffer.
|
||||
5. Fensalir receives aligned buffer slices and still owns composition.
|
||||
|
||||
`config/mimir-runtime.raven-eve.example.json` is the active bridge setup shape
|
||||
for the network producer pass:
|
||||
|
||||
- `raven-display` listens for Raven screen capture on SRT port `5200`, decodes
|
||||
through FFmpeg, and emits BGRA video samples into the runtime buffer path.
|
||||
- `eve-camera` listens for EveCanvas camera frame-events on WebSocket port
|
||||
`8793`.
|
||||
- `eve-mic` listens for EveCanvas microphone frame-events on WebSocket port
|
||||
`8794`.
|
||||
|
||||
The target Verse shape replaces those SRT/WebSocket bridge edges with CultMesh
|
||||
reliable-UDP lanes: typed `mimir.cultmesh_stream_frame.v1` envelopes for
|
||||
per-frame media metadata and refs, plus bounded media body shards for payloads
|
||||
too large to inline. Raven display frames remain media observations entering
|
||||
the same rolling buffer; Raven sync still comes from audio evidence and
|
||||
clock-domain correction, not packet arrival.
|
||||
|
||||
The older disabled `frame-events` example in
|
||||
`config/mimir-runtime.asio.example.json` remains diagnostic only. It can witness
|
||||
source timestamps and frame identity, but it does not carry pixels. The Scarlett
|
||||
channel carrying Raven's sync audio is still the evidence path that earns the
|
||||
`raven-sync` timing correction.
|
||||
|
||||
Smoke proof:
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --synchronized-buffer-planner-smoke
|
||||
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --ffmpeg-rawvideo-source-smoke
|
||||
```
|
||||
213
docs/obs-stem-plugin.md
Normal file
|
|
@ -0,0 +1,213 @@
|
|||
# OBS Native Plugin
|
||||
|
||||
The production OBS path is a Mimir/Fensalir-owned program texture plus a
|
||||
Mimir/Fensalir-owned stem bus, exposed through a narrow native OBS plugin.
|
||||
|
||||
## Authority
|
||||
|
||||
- Fensalir owns the completed program pixels.
|
||||
- Mimir/Fensalir own timing, alignment, DSP, and stem identity.
|
||||
- OBS owns broadcast mixing and scene composition.
|
||||
- The plugin imports already-produced program pixels and reads already-processed
|
||||
stem samples. It does not synchronize raw sources, estimate clocks, render the
|
||||
field, or decide which payload is true.
|
||||
|
||||
## Current Contract
|
||||
|
||||
### Video
|
||||
|
||||
Fensalir publishes the completed backbuffer into a named shared D3D12
|
||||
`B8G8R8A8_UNorm` texture when these environment variables are set:
|
||||
|
||||
```text
|
||||
FENSALIR_PROGRAM_OUTPUT_D3D12=1
|
||||
FENSALIR_PROGRAM_OUTPUT_NAME=Global\MimirFensalirProgramTexture
|
||||
```
|
||||
|
||||
The native OBS plugin in `native/obs_stem_source` exposes a `Mimir Program
|
||||
Texture` video source. Set:
|
||||
|
||||
```text
|
||||
Shared D3D12 texture name: Global\MimirFensalirProgramTexture
|
||||
Shared D3D12 fence name: Global\MimirFensalirProgramFence
|
||||
Consumer D3D12 fence name: Global\MimirObsProgramConsumerFence
|
||||
Texture ring slots: 1
|
||||
```
|
||||
|
||||
OBS/libobs is D3D11 internally on Windows, so this source opens the named D3D12
|
||||
texture, creates an OBS-readable legacy shared D3D11 texture, opens that same
|
||||
bridge texture from D3D12, and performs a GPU-to-GPU copy during OBS render
|
||||
before drawing the source. This avoids CPU readback and does not require
|
||||
Spout2. It is not pure zero-copy yet; the remaining copy is the
|
||||
D3D12-to-libobs-D3D11 boundary. Fensalir also publishes a named D3D12
|
||||
program-output fence; the OBS source opens it and observes publication
|
||||
completion before copying. This prevents blind read-before-producer-completion
|
||||
behavior without coupling OBS to Fensalir's private frame fence.
|
||||
|
||||
For OBS-only stress testing, Fensalir and the OBS source can agree on a bounded
|
||||
texture ring:
|
||||
|
||||
```text
|
||||
FENSALIR_PROGRAM_OUTPUT_RING_COUNT=3
|
||||
FENSALIR_PROGRAM_OUTPUT_CONSUMER_FENCE_NAME=Global\MimirObsProgramConsumerFence
|
||||
Texture ring slots: 3
|
||||
Consumer D3D12 fence name: Global\MimirObsProgramConsumerFence
|
||||
```
|
||||
|
||||
With a ring count above one, Fensalir publishes textures named
|
||||
`Global\MimirFensalirProgramTexture.0`, `.1`, and so on. The OBS source uses the
|
||||
program-output fence value to copy the latest completed slot and signals the
|
||||
consumer fence after its GPU copy. When Fensalir is launched with the matching
|
||||
consumer-fence environment variable, it will not reuse a ring slot until OBS has
|
||||
acknowledged a copy at or beyond that slot's published fence value. Leave the
|
||||
ring count at `1` for EVE until the EVE relay speaks the same ring contract.
|
||||
|
||||
For a visible interop witness while the real Mimir scene is sparse, launch
|
||||
Mimir with:
|
||||
|
||||
```text
|
||||
MIMIR_OBS_PROOF_VISUAL=1
|
||||
```
|
||||
|
||||
That enables a small bright SDF/starfield proof visual. It is a debug witness,
|
||||
not production scene policy.
|
||||
|
||||
### Audio
|
||||
|
||||
`MimirObsStemSharedMemoryPublisher` writes the latest validated
|
||||
`MimirObsStemPublicationSnapshot` into a Windows shared-memory map named
|
||||
`Local\MimirObsStemBus` by default. The native OBS plugin in
|
||||
`native/obs_stem_source` exposes a `Mimir Audio Stem` input source. Add one
|
||||
source per stem and set:
|
||||
|
||||
```text
|
||||
Shared memory map: Local\MimirObsStemBus
|
||||
Stem id: aligned_source_0
|
||||
```
|
||||
|
||||
Use `aligned_source_0` through `aligned_source_5` for the first actuator-stage
|
||||
source lanes.
|
||||
|
||||
## Build
|
||||
|
||||
```powershell
|
||||
.\scripts\build-obs-stem-plugin.ps1
|
||||
```
|
||||
|
||||
The script stages `obsproject/obs-plugintemplate` under `artifacts/obs-sdk/`,
|
||||
lets the upstream template fetch/build the matching OBS/libobs development
|
||||
surface, then builds `native/obs_stem_source` against that SDK. The staged SDK
|
||||
and native build products are local artifacts, not repository state.
|
||||
|
||||
The managed shared-memory publisher is verified by:
|
||||
|
||||
```powershell
|
||||
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --obs-stem-shared-memory-smoke
|
||||
```
|
||||
|
||||
The local native plugin build currently succeeds and emits:
|
||||
|
||||
```text
|
||||
native\obs_stem_source\build\Release\mimir_obs_stem_source.dll
|
||||
```
|
||||
|
||||
That DLL currently registers both source types:
|
||||
|
||||
```text
|
||||
Mimir Program Texture
|
||||
Mimir Audio Stem
|
||||
Muninn Stream
|
||||
```
|
||||
|
||||
### Muninn Streams
|
||||
|
||||
The `Muninn Stream` source is a thin OBS wrapper over Muninn's typed
|
||||
CultCache/CultMesh stream catalog. The operator-facing UI is intentionally just
|
||||
the stream dropdown plus independent video and audio device pickers for the
|
||||
selected source. Store paths, command endpoints, and media return endpoints are
|
||||
source defaults and migration state, not user-facing stream selection.
|
||||
|
||||
The source listens for Raven's typed OBS catalog projection on local UDP port
|
||||
`17874`. Raven Muninn sends the catalog over a small CultNet RUDP discovery lane
|
||||
using connection id `0x6d750003`; OBS uses that live catalog for the dropdown
|
||||
when present.
|
||||
|
||||
If live discovery is absent, the source falls back to Muninn's local `.cc`
|
||||
store:
|
||||
|
||||
```text
|
||||
C:\Meta\Odin\state\muninn.telemetry.cc
|
||||
```
|
||||
|
||||
The source looks for record key `obs` with type:
|
||||
|
||||
```text
|
||||
muninn.obs_stream_catalog
|
||||
```
|
||||
|
||||
Muninn owns that catalog as typed CultCache state. OBS owns only the selected
|
||||
scene source, the selected video leg, the selected audio leg, and media
|
||||
rendering. When a selected stream uses Muninn's RUDP media profile, the source
|
||||
publishes the same typed
|
||||
`muninn.capture_stream_command` that an operator would publish manually. Raven
|
||||
Muninn `serve` still owns whether capture starts and which local FFmpeg/WASAPI
|
||||
children exist.
|
||||
|
||||
If both live discovery and the local catalog projection are absent or stale, the
|
||||
source keeps a LAN-default Raven A/V stream option instead of showing an empty
|
||||
source. That is a resilience fallback for the OBS machine, not a second stream
|
||||
authority. Stale catalog `rudp://` URLs that do not carry the current
|
||||
low-latency LAN profile are sanitized back to the source-derived URL before the
|
||||
bridge starts.
|
||||
|
||||
The current Raven-room default route is LAN-first: command requests go to
|
||||
`192.168.1.84:17873`, and Raven sends media back to Starfire at
|
||||
`192.168.1.66:5204`. Older saved source settings that still point at the
|
||||
deprecated WireGuard defaults `10.77.0.4:17873` and `10.77.0.2` are migrated on
|
||||
load. Custom operator-entered endpoints are left alone.
|
||||
|
||||
The source polls the catalog while active, but reconnect is owned by the RUDP
|
||||
receiver bridge rather than catalog labels. The bridge tracks the last packet
|
||||
and the last forwarded video/audio payload. If startup produces no media, or a
|
||||
previously live stream goes stale, OBS refreshes its local FFmpeg child readers
|
||||
and republishes the typed activation command on a short bounded cadence. A Raven
|
||||
daemon restart or media child exit should therefore recover without reopening
|
||||
source properties.
|
||||
|
||||
For `rudp://` stream URLs, the plugin binds the advertised media port, accepts
|
||||
the CultNet RUDP handshake, acknowledges packets, reassembles RUDP fragments,
|
||||
decodes typed `muninn.media_video_access_unit.v1` and
|
||||
`muninn.media_audio_packet.v1` records, and lowers elementary H.264 plus ADTS
|
||||
AAC into local loopback UDP sockets for OBS FFmpeg child sources. The bridge
|
||||
does not impose a global packet-sequence FIFO over the media lane: CultNet RUDP
|
||||
media is reliable but unordered, so frame ids, chunk ids, and the advertised
|
||||
`assembly_deadline_ms` own video reconstruction. Incomplete video frames expire
|
||||
inside that budget and emit `muninn.media_receiver_feedback.v1` keyframe
|
||||
pressure instead of stalling newer media behind an obsolete sequence gap.
|
||||
RUDP fragment ids are quarantined briefly after completion/expiry so late
|
||||
duplicates cannot seed a corrupt assembly when the sender's 16-bit fragment id
|
||||
space wraps under sustained LAN pressure. The local OBS FFmpeg UDP readers use
|
||||
large loopback buffers; those buffers absorb receiver bursts without becoming a
|
||||
second transport authority.
|
||||
|
||||
## Install
|
||||
|
||||
For the standard OBS plugin search path:
|
||||
|
||||
```powershell
|
||||
.\scripts\install-obs-mimir-plugin.ps1
|
||||
```
|
||||
|
||||
By default this writes:
|
||||
|
||||
```text
|
||||
C:\ProgramData\obs-studio\plugins\mimir_obs_stem_source\bin\64bit\mimir_obs_stem_source.dll
|
||||
```
|
||||
|
||||
For a portable OBS tree or direct OBS installation root:
|
||||
|
||||
```powershell
|
||||
.\scripts\install-obs-mimir-plugin.ps1 -ObsRoot D:\Tools\obs-studio
|
||||
```
|
||||
|
||||
The script does not create scene items or alter OBS profiles.
|
||||
|
|
@ -27,6 +27,13 @@ Read this with:
|
|||
loop candidates.
|
||||
- [[research/perfect-machine-study-2026-05-23/fensalir-integration-map|Fensalir Integration Map]]
|
||||
for the Mimir/Fensalir ownership cut.
|
||||
- [[fensalir-rendering-rebuild-migration|Fensalir Rendering Rebuild Migration]]
|
||||
for the current teardown/rebuild plan that keeps Mimir on physical evidence
|
||||
and pushes field rendering authority into Fensalir's evidence machine.
|
||||
- [[research/d3d12-stereo-depth-provenance-2026-05-29|D3D12 Stereo Depth Provenance]]
|
||||
for the current fast GPU depth search. `libSGM` is the permissive north-star
|
||||
reference for a future Fensalir-owned D3D12 compute stereo disparity lane;
|
||||
CUDA/TensorRT projects remain provenance, not live dependencies.
|
||||
- [[research/perfect-machine-study-2026-05-23/calibration-session-spec|Calibration Session Spec]]
|
||||
for the planned physical-path calibration command.
|
||||
- [[perfect-machine-module-library|Perfect Machine Module Library]] for the
|
||||
|
|
@ -78,6 +85,11 @@ letting OBS become the synchronization authority.
|
|||
### Visual Evidence
|
||||
|
||||
- Leap stereo IR: timing/depth candidate, topology-sensitive but high-value.
|
||||
The live `LeapStereoIr` resource is a packed R8G8 texture; Mimir now lowers
|
||||
it into a stereo-depth FieldEvidence claim by treating R/G as the left/right
|
||||
observation lanes and asking Fensalir for an R16F disparity `SurfacePage`.
|
||||
The first installed Fensalir kernel is a packed block matcher, not calibrated
|
||||
metric depth or full SGM.
|
||||
- PS3 Eyes: high-rate tracking witnesses.
|
||||
- Kiyo/Kiyo Pro: RGB context and texture, with Kiyo Pro currently cadence/USB
|
||||
limited.
|
||||
|
|
@ -165,6 +177,18 @@ Likely shape:
|
|||
|
||||
- Native capture pushes typed frame handles and timestamps.
|
||||
- Fensalir extracts features/depth/optical flow on GPU.
|
||||
- Stereo depth starts as a Fensalir-owned D3D12 compute lane modeled on
|
||||
permissive SGM provenance, especially `libSGM`: rectified texture pairs plus
|
||||
calibration produce GPU-resident disparity/depth/confidence resources, which
|
||||
then lower into FieldEvidence claims.
|
||||
- Current contract proof:
|
||||
`Mimir.BufferSmoke --stereo-depth-contract-smoke` validates the profile and
|
||||
FieldEvidence socket. The synthetic Leap stereo pair uses caller-declared
|
||||
shader-readable left/right texture declarations, an R16F compute-writable disparity
|
||||
`SurfacePage`, an R8 confidence `Texture2D`, one Height claim, one
|
||||
`AquariumFieldStereoDepthLowering` sidecar with min disparity 0, 128
|
||||
disparities, four paths, census radius 2, P1 8, and P2 96, and one planned `SurfacePage`
|
||||
packet. It is not yet a live compute kernel.
|
||||
- Native reservoir preserves rolling evidence.
|
||||
- Fusion emits surface/material/render claims.
|
||||
- Program video publishes via Spout2.
|
||||
|
|
@ -173,6 +197,8 @@ Pressure:
|
|||
|
||||
- Do not build a CPU image pipeline because it is easy to inspect. Six cameras
|
||||
make that path collapse into a pile of excuses.
|
||||
- Do not import CUDA or TensorRT as live depth authority. External depth
|
||||
projects may guide kernel shape, benchmarks, and failure probes only.
|
||||
|
||||
## Hot Loop Budget
|
||||
|
||||
|
|
@ -251,6 +277,9 @@ Six optical sensors with high-rate Eyes and Leap IR mean:
|
|||
3. Reduced alphabet/high-order codebook physical proof.
|
||||
4. Fractional delay/SRO actuator proof.
|
||||
5. Direct Leap/PS3/Kiyo runtime drivers.
|
||||
6. Native reservoir integration with Fensalir/Faust.
|
||||
7. Raven/phone receiver codebook/schedule local decode.
|
||||
8. GPU visual fusion and Spout2 program video.
|
||||
6. D3D12 stereo disparity/depth compute lane, with `libSGM` as provenance and
|
||||
Fensalir as live owner. Contract/profile smoke exists; implement the HLSL
|
||||
cost-volume, aggregation, winner/subpixel, and confidence passes next.
|
||||
7. Native reservoir integration with Fensalir/Faust.
|
||||
8. Raven/phone receiver codebook/schedule local decode.
|
||||
9. GPU visual fusion and Spout2 program video.
|
||||
|
|
|
|||
|
|
@ -19,16 +19,17 @@ job-interview answer for why it belongs in the body of the machine.
|
|||
| Benchmark panels | `Mimir.Runtime` | Decoder golf and meatspace acceptance degradation panels with receipt roots and thresholds. | `src/Mimir.Runtime/Synchronization/MimirBenchmarkPanelConfiguration.cs` |
|
||||
| Capture profiles | `Mimir.Runtime` + native workers | Hardware intent for Leap, Kiyos, PS Eyes, Starfire Scarlett, and Raven Scarlett. | `src/Mimir.Runtime/Synchronization/MimirNativeCaptureConfiguration.cs` |
|
||||
| Camera ingest strategies | `Mimir.Runtime` + native workers | Diagnostic metadata, managed native wrapper, native SPSC ring, and shared-GPU-texture ingest shapes. | `src/Mimir.Runtime/Synchronization/MimirCameraIngestConfiguration.cs` |
|
||||
| Stereo depth profiles | `Mimir.Runtime` + Fensalir | SGM-shaped stereo disparity/depth profiles with libSGM as provenance for the first D3D12 lane, not a CUDA dependency. | `src/Mimir.Runtime/Synchronization/MimirStereoDepthConfiguration.cs` |
|
||||
| Reservoir strategies | `Mimir.Runtime` + native reservoir + Fensalir | Managed rolling buffers, native shared-edge rings, and derived GPU temporal field storage. | `src/Mimir.Runtime/Synchronization/MimirReservoirConfiguration.cs` |
|
||||
| Distributed witnesses | `Mimir.Runtime` + CultMesh | Raven, phone, microcontroller, and remote camera rig configurations for compact timing/path state. | `src/Mimir.Runtime/Synchronization/MimirDistributedWitnessConfiguration.cs` |
|
||||
| Network transports | `Mimir.Runtime` + CultMesh | Typed timing state, debug windows, SRT bridge media, and experimental browser transport policy plus constraint-based selection. | `src/Mimir.Runtime/Synchronization/MimirNetworkTransportConfiguration.cs`, `src/Mimir.Runtime/Synchronization/MimirNetworkTransportSelector.cs` |
|
||||
| Distributed witnesses | `Mimir.Runtime` + CultMesh | Raven, phone, microcontroller, Nightwing Eyes/Moves, and remote camera rig configurations for compact timing/path/visual state. | `src/Mimir.Runtime/Synchronization/MimirDistributedWitnessConfiguration.cs` |
|
||||
| Network transports | `Mimir.Runtime` + CultMesh | Typed timing state, reliable-UDP Verse media/body lanes, debug windows, SRT bridge media, and experimental browser transport policy plus constraint-based selection. | `src/Mimir.Runtime/Synchronization/MimirNetworkTransportConfiguration.cs`, `src/Mimir.Runtime/Synchronization/MimirNetworkTransportSelector.cs` |
|
||||
| Authority policy | `Mimir.Runtime` | Trust decisions for Starfire loopback, Raven evidence, phone candidates, diagnostic media, and unknown nodes. | `src/Mimir.Runtime/Synchronization/MimirAuthorityPolicyConfiguration.cs`, `src/Mimir.Runtime/Synchronization/MimirAuthorityPolicyEvaluator.cs` |
|
||||
| Audio field configurations | `Mimir.Runtime` | Six-mic, source-based, ambisonic, and hybrid evidence field shapes. | `src/Mimir.Runtime/Synchronization/MimirAudioFieldConfiguration.cs` |
|
||||
| Visual fusion configurations | `Mimir.Runtime` + Fensalir | Sensor roles and fusion modes from cadence proof through full hybrid evidence. | `src/Mimir.Runtime/Synchronization/MimirVisualFusionConfiguration.cs` |
|
||||
| Compute offload configurations | `Mimir.Runtime` + CultMesh | Local-heavy, distributed witness, and calibration sweep workload placement. | `src/Mimir.Runtime/Synchronization/MimirComputeOffloadConfiguration.cs` |
|
||||
| Assembly plans | `Mimir.Runtime` | Named combinations of modules that prove synthetic, Scarlett, meatspace, and distributed cuts. | `src/Mimir.Runtime/Synchronization/MimirMachineAssemblyPlan.cs` |
|
||||
| OBS publication profiles | Fensalir + Faust + OBS | Spout/shared-texture video and separately mixable Faust stem publication. | `src/Mimir.Runtime/Synchronization/MimirObsPublicationConfiguration.cs` |
|
||||
| Fensalir lowering | `Mimir.Runtime` -> Fensalir | Pure mapping from Mimir rolling/video/audio state into Fensalir GPU sensor and acoustic field frames. | `src/Mimir.Runtime/Synchronization/MimirFensalirFieldLowering.cs` |
|
||||
| Fensalir lowering | `Mimir.Runtime` -> Fensalir | Pure mapping from Mimir rolling/video/audio state into Fensalir GPU sensor, stereo-depth, and acoustic field frames. | `src/Mimir.Runtime/Synchronization/MimirFensalirFieldLowering.cs` |
|
||||
| Module catalog | `Mimir.Runtime` | Indexes the live module set and verification commands. | `src/Mimir.Runtime/Synchronization/MimirModuleLibrary.cs` |
|
||||
| Manifest factory | `Mimir.Runtime` | Produces an exportable JSON manifest for tools, docs, UI, and remote witnesses. | `src/Mimir.Runtime/Synchronization/MimirPerfectMachineManifest.cs` |
|
||||
|
||||
|
|
@ -40,8 +41,9 @@ job-interview answer for why it belongs in the body of the machine.
|
|||
program video publication.
|
||||
- Faust owns hot sample movement, alignment, spatialization, separation, and
|
||||
stem generation.
|
||||
- CultMesh carries typed state surfaces; remote nodes may observe and decode,
|
||||
but they do not become independent clock authorities.
|
||||
- CultMesh carries typed state surfaces and reliable-UDP media/body lanes;
|
||||
remote nodes may observe, decode, and stream media through the Verse, but
|
||||
they do not become independent clock authorities.
|
||||
- OBS receives final program surfaces and stems. It does not own calibration.
|
||||
|
||||
## Verification Surface
|
||||
|
|
|
|||
42
docs/program-control-surface.md
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
# Program Control Surface
|
||||
|
||||
Mimir now exposes a compact Fensalir-hosted program panel for final presentation
|
||||
intent.
|
||||
|
||||
Authority map:
|
||||
|
||||
- Owner: `MimirPresentationControlState` owns operator intent for program video,
|
||||
audio mix, and global color preset.
|
||||
- Inputs: runtime rolling buffers, aligned audio lanes, LUT preset selection,
|
||||
feed visibility, feed opacity, stem mute/solo, and stem gain.
|
||||
- Outputs: filtered video surface intents, Faust/native DSP gain controls,
|
||||
sample gain applied before Fensalir streaming DSP, and a postprocess control
|
||||
snapshot.
|
||||
- Derived state: feed lists are derived from active rolling buffers. LUT preset
|
||||
exposure and bloom are currently mapped into `GraphicsSettings`.
|
||||
- Forbidden writers: OBS, bridge endpoints, debug readouts, and network arrival
|
||||
timestamps must not choose program composition.
|
||||
- Shared paths: local cameras, Raven display capture, and future network video
|
||||
feeds all enter rolling buffers first, then the presentation state decides
|
||||
whether they contribute to production surface intent.
|
||||
- Deletion line: direct compositor shortcuts outside Fensalir FieldEvidence and
|
||||
audio DSP are diagnostic only.
|
||||
|
||||
The current UI lives in the `Mimir Program` panel:
|
||||
|
||||
- `Video`: select feed, visible, solo, opacity, layer ordering.
|
||||
- `Audio`: select stem/source, mute, solo, gain.
|
||||
- `Color`: LUT preset and LUT strength.
|
||||
|
||||
The shader-side LUT path is explicit state, not finished renderer authority yet.
|
||||
`MimirLookupTablePreset` carries `LutPath`, strength, contrast, saturation,
|
||||
temperature, and tint so Fensalir can consume the same contract when the global
|
||||
post shader grows real LUT sampling. Until that renderer cut lands, Mimir maps
|
||||
the preset exposure and bloom values into existing Fensalir `GraphicsSettings`.
|
||||
|
||||
Smoke proof:
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --presentation-control-smoke
|
||||
```
|
||||
|
||||
80
docs/raven-capture-mux-daemon.md
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
# Raven Capture Mux Daemon
|
||||
|
||||
Raven owns local display capture and Realtek render-loopback capture. Mimir receives the muxed stream and decides what it means inside the synchronized program model.
|
||||
|
||||
## Authority Map
|
||||
|
||||
- Owner: `Mimir.RavenDaemon` owns process supervision and typed status for Raven's capture mux.
|
||||
- Inputs: local display capture parameters, WASAPI render-loopback selection, FFmpeg path, named SRT targets, log root, and operator flags.
|
||||
- Outputs: `mimir.raven_capture_mux_state.v1`, an Eve dashboard surface, health JSON, and named FFmpeg SRT MPEG-TS streams for Mimir and OBS.
|
||||
- Derived state: PID, restart count, log paths, and dashboard nodes are derived from the supervised process and daemon options.
|
||||
- Forbidden writers: capture scripts may still launch diagnostics, but they do not own daemon status or the canonical Raven mux state.
|
||||
- Shared paths: service launch, dry run, health checks, and Eve dashboard projection all read the same `MimirRavenCaptureMuxStateDocument`.
|
||||
- Cut line: no OBS plugin, custom media daemon, or alternate transport owns media in this pass. FFmpeg/WASAPI own capture, mux, encode, and SRT fanout; CultMesh owns status and operator surface.
|
||||
|
||||
## Run On Raven
|
||||
|
||||
From a Mimir checkout on Raven:
|
||||
|
||||
```powershell
|
||||
.\scripts\start-raven-daemon.ps1
|
||||
```
|
||||
|
||||
Defaults:
|
||||
|
||||
- Mimir target: `srt://10.77.0.2:5200`
|
||||
- OBS target: `srt://10.77.0.2:5204`
|
||||
- Eve/CultMesh dashboard: `ws://0.0.0.0:8801/eve/deck`
|
||||
- Binary CultMesh dashboard stream: `ws://0.0.0.0:8801/eve/deck/cultmesh`
|
||||
- Health: `http://127.0.0.1:8801/health`
|
||||
- State cache: `C:\Meta\Mimir\state\raven-capture-mux.ccmp`
|
||||
- Logs: `C:\Meta\Mimir\logs`
|
||||
|
||||
Use a dry run to inspect the exact FFmpeg pipeline without starting capture:
|
||||
|
||||
```powershell
|
||||
.\scripts\start-raven-daemon.ps1 -DryRun
|
||||
```
|
||||
|
||||
Disable the OBS sink when only Mimir should receive the mux:
|
||||
|
||||
```powershell
|
||||
.\scripts\start-raven-daemon.ps1 -NoObsTarget
|
||||
```
|
||||
|
||||
Use a specific Realtek render endpoint when Windows exposes more than one:
|
||||
|
||||
```powershell
|
||||
.\scripts\start-raven-daemon.ps1 -AudioDevice "Realtek"
|
||||
```
|
||||
|
||||
If Raven already has a host-level CultMesh server bound, keep this daemon as a local typed-state publisher and Eve endpoint:
|
||||
|
||||
```powershell
|
||||
.\scripts\start-raven-daemon.ps1 -NoCultNetServer
|
||||
```
|
||||
|
||||
## Receiver Shape
|
||||
|
||||
The daemon sends identical MPEG-TS SRT streams containing:
|
||||
|
||||
- `raven-display`: Raven desktop via `ddagrab` or `gdigrab`, encoded with `h264_nvenc`
|
||||
- `raven-realtk-loopback`: Raven render loopback via `Mimir.WasapiLoopback` or the PowerShell WASAPI fallback, encoded as AAC
|
||||
|
||||
Mimir receives the ingest copy on `5200`. `scripts/start-raven-av-demux.ps1`
|
||||
can split that copy back into local video and audio source lanes for the
|
||||
current raw ingest adapters. OBS receives its own copy on `5204` as an ordinary
|
||||
SRT Media Source; it does not compete with Mimir for the `5200` listener.
|
||||
|
||||
The typo in the existing source id, `raven-realtk-loopback`, is preserved until the receiver config is migrated.
|
||||
|
||||
## Idunn Lifecycle
|
||||
|
||||
Idunn watches the Raven daemon from Odin's local keepalive body:
|
||||
|
||||
- Health: `E:\Projects\Odin\scripts\health-raven-capture-mux.cmd`
|
||||
- Restart: `E:\Projects\Odin\scripts\restart-raven-capture-mux.cmd`
|
||||
- Default SSH target: Raven's WireGuard address, `10.77.0.4`
|
||||
|
||||
If Raven is unreachable, Idunn should record the failed health probe and raise
|
||||
the normal keepalive decision instead of pretending the daemon is healthy.
|
||||
130
docs/raven-eve-ingest.md
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
# Raven And Eve Ingest
|
||||
|
||||
## Objective
|
||||
|
||||
Ingest Raven screen capture plus Eve camera and microphone capture into Mimir,
|
||||
then let the existing synchronization hub place those samples in rolling buffers
|
||||
beside Scarlett audio and local camera sources.
|
||||
|
||||
## Authority Map
|
||||
|
||||
- Owner: `Mimir.Runtime` owns decoded samples and rolling-buffer placement.
|
||||
- Inputs: Raven currently sends H.264/MPEG-TS over the SRT bridge to Starfire
|
||||
port `5200`; Eve currently sends camera frame-events over WebSocket port
|
||||
`8793`; Eve currently sends microphone frame-events over WebSocket port
|
||||
`8794`; Scarlett/ASIO carries audio and sync evidence. The target Verse shape
|
||||
is CultMesh reliable UDP: typed stream-frame envelopes plus bounded body
|
||||
shards for media, with compact state/cursors/backpressure beside them.
|
||||
- Outputs: `raven-display`, `eve-camera`, and `eve-mic` emit
|
||||
`MimirStreamSample` records with video or audio descriptors.
|
||||
- Derived state: network arrival timestamps are capture metadata only. Raven
|
||||
audio-in-Scarlett remains the timing witness for global alignment.
|
||||
- Forbidden writers: OBS, FFmpeg sender clocks, WebSocket receive time, SRT
|
||||
arrival time, and CultMesh packet arrival time do not own final sync truth.
|
||||
- Cut line: EveCanvas owns Eve's sensors directly. External iOS streaming apps
|
||||
are fallback tools, not the current architecture. SRT/WebSocket bridge
|
||||
transports are diagnostics and compatibility edges once the reliable-UDP
|
||||
Verse media lane exists.
|
||||
|
||||
## Starfire Receiver
|
||||
|
||||
Use the Raven/Eve runtime config:
|
||||
|
||||
```powershell
|
||||
$env:MIMIR_RUNTIME_CONFIG = "E:\Projects\Mimir\config\mimir-runtime.raven-eve.example.json"
|
||||
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --seconds 5
|
||||
```
|
||||
|
||||
When both senders are active, require sample arrival:
|
||||
|
||||
```powershell
|
||||
$env:MIMIR_RUNTIME_CONFIG = "E:\Projects\Mimir\config\mimir-runtime.raven-eve.example.json"
|
||||
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --seconds 5 --require-samples
|
||||
```
|
||||
|
||||
The config starts one FFmpeg listener and two Eve sensor receiver processes:
|
||||
|
||||
- `raven-display`: `srt://0.0.0.0:5200?mode=listener&latency=120000&timeout=5000000`
|
||||
- `eve-camera`: `ws://0.0.0.0:8793/eve/camera`
|
||||
- `eve-mic`: `ws://0.0.0.0:8794/eve/mic`
|
||||
|
||||
The Raven listener decodes to raw BGRA frames on stdout. The Eve receivers emit
|
||||
typed JSON frame-events from EveCanvas camera/mic packets. Mimir stores all of
|
||||
them in the normal synchronization buffers.
|
||||
|
||||
## Raven Sender
|
||||
|
||||
Run this on Raven from the logged-in desktop session:
|
||||
|
||||
```powershell
|
||||
E:\Projects\Mimir\scripts\start-raven-screen-capture-sender.ps1 -TargetHost 192.168.1.66 -Port 5200
|
||||
```
|
||||
|
||||
The script uses `gdigrab` plus `h264_nvenc`, then calls Starfire's SRT listener.
|
||||
Do not launch the capture itself through a blind SSH session unless the desktop
|
||||
capture path has been proven there; `gdigrab` captures the interactive Windows
|
||||
desktop, and service/session boundaries can turn screen capture into a black
|
||||
box with excellent logs.
|
||||
|
||||
Raven sync signal: route Raven audio or a Raven-generated chirp into the
|
||||
Scarlett alongside the microphone. Mimir should treat that audio evidence as
|
||||
the authority for aligning Raven video with the rest of the program.
|
||||
|
||||
## Eve Sensor Sender
|
||||
|
||||
EveCanvas now starts native AVFoundation capture after permission grants:
|
||||
|
||||
- Camera frames are sampled from `AVCaptureVideoDataOutput`, JPEG-compressed,
|
||||
wrapped as `video-frame` events for `eve-camera`, and sent to
|
||||
`ws://192.168.1.66:8793/eve/camera`.
|
||||
- Microphone blocks are captured with `AVAudioEngine`, packed as interleaved
|
||||
Float32 samples, wrapped as `audio-block` events for `eve-mic`, and sent to
|
||||
`ws://192.168.1.66:8794/eve/mic`.
|
||||
|
||||
This is an inspectable first transport, not the final high-throughput shape.
|
||||
When cadence or bandwidth becomes the bottleneck, replace the JSON/base64 event
|
||||
transport with CultMesh reliable-UDP media frames and body shards, optionally
|
||||
carrying VideoToolbox H.264 or another explicit media payload. Preserve the same
|
||||
source ownership and Mimir runtime sample contract.
|
||||
|
||||
Deploy EveCanvas after staging:
|
||||
|
||||
```powershell
|
||||
powershell -ExecutionPolicy Bypass -File E:\Projects\Eve\scripts\stage-to-eve.ps1
|
||||
ssh eve "cd /var/mobile/Projects/Eve && export THEOS=/var/theos && make package install && uiopen --bundleid org.gamecult.evecanvas"
|
||||
```
|
||||
|
||||
## Smoke Test
|
||||
|
||||
The raw-video source has a local synthetic smoke that does not require Raven or
|
||||
Eve:
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --ffmpeg-rawvideo-source-smoke
|
||||
```
|
||||
|
||||
That smoke starts FFmpeg with `testsrc2`, decodes BGRA frames through stdout,
|
||||
and verifies the sample metadata and payload size that Mimir receives.
|
||||
|
||||
The Eve sensor receiver can be checked without launching the whole runtime:
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.EveSensorReceiver\Mimir.EveSensorReceiver.csproj -- --port 8793 --path /eve/camera --source-id eve-camera --type video-frame
|
||||
```
|
||||
|
||||
Periwinkle's Android Eve client publishes binary CultMesh sensor observations:
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.EveSensorReceiver\Mimir.EveSensorReceiver.csproj -- --port 8796 --path /eve/periwinkle --source-id periwinkle-accelerometer --type cultmesh-observation
|
||||
```
|
||||
|
||||
The receiver accepts `mimir.eve_sensor_observation.v1` MessagePack packets on
|
||||
that lane. The device remains an observation source; Mimir owns synchronization
|
||||
and interpretation.
|
||||
|
||||
The same receiver also accepts `mimir.eve_media_observation.v1` MessagePack
|
||||
packets. Periwinkle publishes low-rate camera luma frames as
|
||||
`periwinkle-camera` and microphone PCM blocks as `periwinkle-mic` on the
|
||||
CultMesh observation lane. The legacy iPad Eve camera and mic uplinks have
|
||||
matching binary media document support while their existing `eve-camera` and
|
||||
`eve-mic` ports remain usable.
|
||||
53
docs/scene-editor-control-surface.md
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
# Mimir Scene Editor Control Surface
|
||||
|
||||
Mimir now has a runtime-owned scene editor state for arranging the visible
|
||||
presentation inputs inside the Fensalir-hosted Mimir window. This editor is not
|
||||
the OBS program output. It is an operator surface for placing source panels,
|
||||
text panels, and imported-model placeholders before later renderer cuts turn
|
||||
those nodes into final composited pixels or volumetric field claims.
|
||||
|
||||
## Authority Map
|
||||
|
||||
- Owner: `MimirSceneEditorState` owns editor camera, scene nodes, selected node,
|
||||
visibility, lock state, 2D-plane transform, and active gizmo mode.
|
||||
- Inputs: rolling video buffers provide sensor-feed nodes; Fensalir `InputState`
|
||||
provides mouse/key edits; the Mimir UI provides visibility, reset, transform,
|
||||
text, and model-path commands.
|
||||
- Outputs: `MimirRuntime` reads the editor state into the Fensalir frame camera,
|
||||
spline outlines, selection gizmos, and SDF handle markers.
|
||||
- Derived state: the hierarchy text, selected-node readout, editor splines, and
|
||||
handle SDF objects are derived from the editor graph. They do not own node
|
||||
truth.
|
||||
- Forbidden writers: program-composite controls, OBS source endpoints, bridge
|
||||
scripts, and FieldEvidence surface intents do not decide editor transforms.
|
||||
- Shared paths: UI buttons/sliders and mouse-drag gizmos all commit through
|
||||
`MimirSceneEditorState`; reset returns to each node's stored default
|
||||
transform.
|
||||
- Deletion line: do not add a separate transform cache in the renderer or OBS
|
||||
path. Fensalir may later render richer node types, but it should consume this
|
||||
graph or a deliberate successor, not invent a competing editor graph.
|
||||
|
||||
## Current Controls
|
||||
|
||||
- `Mimir Editor` panel exposes the scene graph, selected-node visibility and
|
||||
lock toggles, transform reset, camera reset, and transform sliders.
|
||||
- Active gizmo modes are `Grab`, `Rotate`, and `Resize`; keys `1`, `2`, and `3`
|
||||
switch modes, and left-drag applies the active mode to the selected node.
|
||||
- Sensor-feed nodes are created from live video rolling buffers.
|
||||
- SDF text panels and model placeholders can be added from the panel.
|
||||
- `--scene-editor-smoke` proves graph creation, visibility toggles, transform
|
||||
reset, mouse-drag transform edits, text-node creation, model placeholder
|
||||
import, and derived gizmo geometry.
|
||||
|
||||
## Renderer Hooks Still Open
|
||||
|
||||
- World SDF text: Fensalir currently has DirectWrite overlay text and SDF object
|
||||
proxies. Scene text needs a renderer-owned SDF/MSDF glyph atlas plus billboard
|
||||
or mesh text lowering before text glyphs are truly drawn inside the scene.
|
||||
- Imported models: Fensalir resource contracts already know mesh packages, but
|
||||
selected mesh draw/material lowerings are still future work. The current
|
||||
import control creates graph nodes with paths; it does not decode ASSIMP
|
||||
meshes or upload vertex/index resources yet.
|
||||
- Pixel-accurate gizmo hit-testing: current gizmo edits apply to the selected
|
||||
node. Fensalir needs an editor hit-test path before handles can be clicked
|
||||
directly in world space.
|
||||
127
docs/self-hosted-live-edge.md
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
# Self-Hosted Live Edge
|
||||
|
||||
Mimir's self-hosted broadcast path keeps OBS optional. Starfire owns the final
|
||||
program frame and audio mix; Yggdrasil owns public byte distribution.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
A["Fensalir program texture"] --> B["Starfire NVENC encode"]
|
||||
C["Faust/Mimir mix"] --> B
|
||||
B --> D["SSH TCP forward"]
|
||||
D --> E["Yggdrasil localhost RTMP ingest"]
|
||||
E --> F["HLS segment ring"]
|
||||
F --> G["streampixels.gamecult.org static HTTPS"]
|
||||
G --> H["gamecult.org/livestream static viewer"]
|
||||
```
|
||||
|
||||
## Authority Map
|
||||
|
||||
- Owner: `Mimir.Broadcast` owns the Starfire-side encoded push command.
|
||||
- Inputs: final program media, FFmpeg/NVENC capability, and the SSH-forwarded
|
||||
RTMP endpoint.
|
||||
- Outputs: one CBR H.264/AAC RTMP stream to Yggdrasil.
|
||||
- Derived state: Yggdrasil's HLS playlist and segments are derived from that
|
||||
encoded stream. They do not own composition, timing, or transcoding.
|
||||
- Forbidden writers: Yggdrasil must not transcode the program stream in the
|
||||
first live edge. OBS is not the primary broadcast owner.
|
||||
- Shared paths: synthetic smokes and real program pushes use the same RTMP URL:
|
||||
`rtmp://127.0.0.1:11935/live/mimir`.
|
||||
- Deletion line: if a later CDN or WebRTC edge appears, keep this path as the
|
||||
simple origin proof or delete it. Do not keep two public stream authorities.
|
||||
|
||||
## First Smoke
|
||||
|
||||
On Starfire, with an SSH local forward active to Yggdrasil's localhost RTMP
|
||||
port:
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.Broadcast\Mimir.Broadcast.csproj -- --smoke
|
||||
dotnet run --project .\src\Mimir.Broadcast\Mimir.Broadcast.csproj -- --print-command
|
||||
```
|
||||
|
||||
The default push target is:
|
||||
|
||||
```text
|
||||
rtmp://127.0.0.1:11935/live/mimir
|
||||
```
|
||||
|
||||
The public HLS playlist is served on the existing Yggdrasil-routed
|
||||
StreamPixels domain:
|
||||
|
||||
```text
|
||||
https://streampixels.gamecult.org/mimir/live/hls/mimir.m3u8
|
||||
```
|
||||
|
||||
The static viewer lives on the root GameCult site:
|
||||
|
||||
```text
|
||||
https://gamecult.org/livestream
|
||||
```
|
||||
|
||||
## Local Recording Before Public Push
|
||||
|
||||
Before pushing a live program to Yggdrasil, record the same simple stream-proof
|
||||
shape locally and judge sync from the artifact:
|
||||
|
||||
```powershell
|
||||
powershell -ExecutionPolicy Bypass -File .\scripts\record-stream-proof-output.ps1 -StartRavenSender -DurationSeconds 20
|
||||
```
|
||||
|
||||
The recording harness listens for a Raven muxed A/V SRT feed on a private test
|
||||
port, overlays Kiyo Pro as picture-in-picture, mixes Raven Realtek loopback with
|
||||
Starfire's Scarlett capture, applies conservative FFmpeg denoise filters, and
|
||||
writes an MP4 under `artifacts\runtime\stream-proof\`.
|
||||
|
||||
The current routing contract is explicit:
|
||||
|
||||
- Starfire Realtek/default render is the chirp/speaker emission lane.
|
||||
- Raven Realtek/default render loopback is the co-streamer game/program lane.
|
||||
It is packaged with Raven's NVENC video feed back to Starfire.
|
||||
- Starfire Scarlett carries the hero mic capture and monitored local mix for
|
||||
judgment. Raven's Scarlett is only routing the remote shotgun into Starfire
|
||||
Scarlett, currently on channel 1.
|
||||
- The proof recording is allowed to mix those lanes for human sync judgment, but
|
||||
it must not collapse their authority. Starfire Realtek proves chirp emission,
|
||||
Raven Realtek proves the co-streamer/game program path, and Scarlett proves
|
||||
the hero mic/shotgun capture path.
|
||||
|
||||
This is a bridge recording, not public broadcast truth. It exists so humans can
|
||||
judge audio/video sync and tell Mimir which offsets to apply before the RTMP/HLS
|
||||
edge is involved. Offset knobs are explicit:
|
||||
|
||||
```powershell
|
||||
powershell -ExecutionPolicy Bypass -File .\scripts\record-stream-proof-output.ps1 `
|
||||
-StartRavenSender `
|
||||
-RavenOffsetSeconds 0.000 `
|
||||
-KiyoOffsetSeconds 0.000 `
|
||||
-MicOffsetSeconds 0.000
|
||||
```
|
||||
|
||||
If Windows firewall blocks direct SRT/UDP during a local proof, run the Raven
|
||||
leg through an SSH TCP tunnel instead:
|
||||
|
||||
```powershell
|
||||
Start-Process ssh.exe -ArgumentList '-o ExitOnForwardFailure=yes -N -L 6204:127.0.0.1:5204 -l "madman''s lullaby" 192.168.1.84' -WindowStyle Hidden
|
||||
|
||||
powershell -ExecutionPolicy Bypass -File .\scripts\record-stream-proof-output.ps1 `
|
||||
-StartRavenSender `
|
||||
-RavenSenderListens `
|
||||
-RavenSenderTransport tcp-listener `
|
||||
-RavenInputOverride tcp://127.0.0.1:6204
|
||||
```
|
||||
|
||||
That fallback keeps the proof private to the LAN/SSH path and avoids changing
|
||||
machine firewall policy just to capture a judging artifact.
|
||||
|
||||
## Scaling Rule
|
||||
|
||||
Viewer capacity is approximately:
|
||||
|
||||
```text
|
||||
available Yggdrasil egress / encoded bitrate
|
||||
```
|
||||
|
||||
Micro-optimizations count only when they improve target-host measurements:
|
||||
served Mbps, p95 segment latency, CPU, memory, syscalls, or stall rate on
|
||||
Yggdrasil. The first edge uses nginx static serving and kernel file paths so
|
||||
the hot path is byte distribution, not application code.
|
||||
|
|
@ -10,7 +10,10 @@ flowchart TD
|
|||
C["Leap stereo IR driver"] --> B
|
||||
B --> D["native reservoir handles"]
|
||||
D --> E["Fensalir GPU feature extraction"]
|
||||
E --> J["per-sensor LED curve solve"]
|
||||
E --> F["cross-view matching + flow"]
|
||||
J --> K["camera rig calibration solver"]
|
||||
K --> F
|
||||
F --> G["surface/material claims"]
|
||||
G --> H["brush/splat render budget"]
|
||||
H --> I["Spout2/program video"]
|
||||
|
|
@ -23,9 +26,51 @@ flowchart TD
|
|||
- Process capture is a bridge edge, not the local six-camera foundation.
|
||||
- Fensalir owns GPU extraction, fusion, material fitting, and render budgeting.
|
||||
- Runtime buffers own retention and stream health, not scene reconstruction.
|
||||
- LED strips are active calibration evidence. Each camera publishes ordered
|
||||
bright-curve observations in its own local frustum/clipspace; only the global
|
||||
residual calibration owner may fit those frustums into scene-space camera
|
||||
intrinsics/extrinsics.
|
||||
- Stable LED index identity requires temporal/color/address coding or another
|
||||
correspondence source; uncoded identical lights are spline constraints, not
|
||||
metric depth authority.
|
||||
- PS Move controllers are active tracked-marker witnesses. USB HID owns
|
||||
identity, buttons, IMU/report topology, and later LED/rumble control; the
|
||||
glowing sphere is visual calibration evidence only when observed by cameras
|
||||
and fused by the global residual owner.
|
||||
- Exposure/gain sweeps optimize recovered spline quality: detected LED count,
|
||||
curve coverage, spacing coherence, smoothness, exposure fitness, and
|
||||
saturation penalty. Bright clipped blobs are not good evidence.
|
||||
- The live LED sweep owner is `Mimir.BufferSmoke --led-spline-live-score` for
|
||||
now: it opens direct sensor feeds, applies KS/UVC manual exposure/gain where
|
||||
supported, scores actual luma frames, and reports the best setting per
|
||||
source. PS3 Eye and Leap-specific controls still need native control cuts.
|
||||
- The current all-sensor GPU proof is
|
||||
`config/mimir-runtime.perfect-machine.local.json` plus
|
||||
`Mimir.BufferSmoke --perfect-machine-live-smoke`: Leap, both PS3 Eyes, Kiyo
|
||||
Pro, and regular Kiyo all enter the runtime as Fensalir texture leases. The
|
||||
smoke requires five live camera Texture2D resources, one Leap stereo-depth
|
||||
lowering, one disparity SurfacePage, and one derived point-cloud Mesh.
|
||||
Camera Feature/SensorObservation claims now plan through Fensalir's
|
||||
`GpuSensorFusion` backend instead of deferring. The current compute dispatch
|
||||
samples LeapStereoIr, both Bayer8 PS3 Eyes, Kiyo Pro YUY2, and regular Kiyo
|
||||
YUY2. Fensalir binds YUY2 through the documented `R8G8B8A8_UNORM` SRV view
|
||||
over `DXGI_FORMAT_YUY2` and decodes `Y0/U/Y1/V` in compute before scoring
|
||||
features.
|
||||
- PS3 Eyes own high-rate sparse tracking evidence before they own depth:
|
||||
`MimirSparseFeatureTracker` turns live Bayer/luma frames into stable
|
||||
sensor-local feature tracks, and `MimirFeatureTrackFieldCandidate` lowers
|
||||
them as camera Feature claims. This CPU path is a diagnostic bootstrap; the
|
||||
production hot path belongs in Fensalir D3D12 feature extraction.
|
||||
- The first camera-rig solver fits bounded camera translation updates from a
|
||||
scene-anchored LED curve by minimizing ray-to-curve distance. Rotation,
|
||||
intrinsics, and distortion are future owners.
|
||||
- Once camera frustums are coherent, every detected surface candidate can be
|
||||
resampled across the synchronized view set before Fensalir resolves it into
|
||||
spatiotemporal surface/splat claims.
|
||||
|
||||
## Next Cut
|
||||
|
||||
Implement one concrete direct capture driver, feed `MimirVideoFrameDescriptor`
|
||||
samples into `MimirVideoCaptureDriverSource`, and measure sustained cadence over
|
||||
the default five-second window.
|
||||
Replace provisional Leap projection constants with calibrated intrinsics and
|
||||
add the global residual owner that consumes Leap surface errors, LED spline
|
||||
observations, and later PS3 Eye/Kiyo/Eve feature tracks through one calibration
|
||||
commit path.
|
||||
|
|
|
|||
169
docs/verse-service-contract.md
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
# Mimir Verse Service Contract
|
||||
|
||||
Mimir is a Verse service for the measured room. It accepts local and network
|
||||
sensors, publishes typed observations and capture pages, asks Fensalir to lower
|
||||
field evidence into program surfaces, and exposes operator controls through
|
||||
Eve/CultUI surfaces. OBS, browser dashboards, JSON logs, and screenshot streams
|
||||
are lowerings or witnesses. They are not canonical presentation or state owners.
|
||||
|
||||
## Owner Map
|
||||
|
||||
- Mimir service owner: configuration, launch, calibration truth, sensor
|
||||
identity, stream identity, synchronization state, Well publication, recorder
|
||||
receipts, and service contract.
|
||||
- `Mimir.Runtime`: bounded rolling buffers, source descriptors, source polling,
|
||||
direct native ingest, audio synchronization analysis, and cached read models.
|
||||
- Native sensor workers: device reads, closest-to-device timestamps, native or
|
||||
GPU handles, unavoidable copy counts, and per-device actuator commands.
|
||||
- `Mimir.Well`: typed live pages, configured composite state, timing pressure,
|
||||
feature signals, capture-body publication, and the append-side observation
|
||||
ledger.
|
||||
- `Mimir.VerseRecorder`: run session receipts, body indexing, replay witnesses,
|
||||
and `.cc`/CultCache-compatible export for offline reconstruction.
|
||||
- `Mimir.FensalirDaemon`: reservoir job selection, queue pressure, GPU worker
|
||||
ownership, program/reservoir surfaces, and daemon state.
|
||||
- Fensalir: D3D12 resource resolution, field claims, selected lowerings,
|
||||
temporal guide lanes, reservoir resolve, and program presentation.
|
||||
- Faust/native DSP: hot audio alignment, sample movement, suppression,
|
||||
separation, spatialization, and program stems.
|
||||
- Eve/CultUI lowerings: operator interaction and display of provider-owned
|
||||
state. They submit commands to owners; they do not own synchronization,
|
||||
reservoir jobs, OBS composition, or calibration truth.
|
||||
- Odin: discovery of Mimir's published CultMesh namespaces and Eve surfaces.
|
||||
|
||||
## CultCache Witness Contract
|
||||
|
||||
Durable state should be typed CultCache `.cc`, or CultCache-compatible with a
|
||||
`.cc` witness/export during migration. The witness set is:
|
||||
|
||||
| State family | Owner | Durable shape |
|
||||
| --- | --- | --- |
|
||||
| Sensor state | `Mimir.Runtime` plus native workers | Source inventory, device identity, driver profile, clock domain, format, actuator capability, native handle class, copy count, and last health. |
|
||||
| Media state | `Mimir.Well` / Fensalir | Capture page metadata, resource keys, body refs, fence refs, page timing, configured composites, and program surface refs. |
|
||||
| Observation state | `Mimir.Runtime` / Well | Typed audio/video/IMU/feature/path observations with source id, time range, confidence, calibration id, and payload/resource reference. |
|
||||
| Ledger state | `Mimir.VerseRecorder` | Session receipts, body index, replay cursor, migration version, schema witness, and rejected/drop reasons. |
|
||||
| Control state | Runtime owners / Fensalir daemon / DSP | Operator commands, accepted command receipts, selected program layer state, reservoir worker pressure, audio actuator commands, and stream/record/live status. |
|
||||
|
||||
JSON may remain only as schema publication, diagnostic text export, external
|
||||
tool boundary, or temporary replay witness. If a JSON file can decide behavior
|
||||
without a typed `.cc` witness, it is still an owner and must be migrated.
|
||||
|
||||
## CultMesh Namespaces
|
||||
|
||||
Mimir should publish these namespaces for Odin discovery:
|
||||
|
||||
- `verse.mimir.service.v1`: service manifest, run id, host identity, schema
|
||||
catalog, transport lanes, and nested Verse roots.
|
||||
- `mimir.sensor.*`: source inventory, native worker health, device timing,
|
||||
media format, actuator capability, and producer fences.
|
||||
- `mimir.media.*`: capture pages, media bodies, resource references, program
|
||||
texture/fence refs, stem refs, and replay body indexes.
|
||||
- `mimir.observation.*`: sensor observations, media observations, feature
|
||||
tracks, audio path evidence, calibration constraints, and local frustum hints.
|
||||
- `mimir.ledger.*`: recorder session, append cursor, migration state,
|
||||
body-index records, dropped-page reasons, and evidence receipts.
|
||||
- `mimir.control.*`: operator commands, command receipts, recording/streaming
|
||||
state, runtime mode, source placement, DSP actuator state, and reservoir
|
||||
scheduling pressure.
|
||||
- `mimir.eve.*`: `gamecult.eve.surface.v1` provider manifests, retained
|
||||
surfaces, field bindings, assets, command bindings, and lowering metadata.
|
||||
- `mimir.fensalir.*`: daemon summary, reservoir worker state, reservoir pressure,
|
||||
program surfaces, resource resolver status, and selected lowering status.
|
||||
|
||||
Existing document names such as `mimir.well_snapshot.v1`,
|
||||
`mimir.well_capture_page.v1`, `mimir.eve_sensor_observation.v1`,
|
||||
`mimir.eve_media_observation.v1`, `mimir.eve_dashboard_state.v1`,
|
||||
`mimir.fensalir_daemon_state.v1`, `mimir.fensalir_reservoir_worker_state.v1`,
|
||||
and `mimir.fensalir_reservoir_pressure.v1` should be kept as concrete records
|
||||
inside those namespaces until a schema catalog renames them deliberately.
|
||||
|
||||
## Transport Lanes
|
||||
|
||||
CultMesh reliable UDP is the default Verse transport for live Mimir network
|
||||
state and media. TCP/WebSocket, SRT, browser streams, and local pipes are
|
||||
diagnostic lowerings or external-boundary bridges unless a specific deployment
|
||||
constraint temporarily promotes them.
|
||||
|
||||
The live lane split is:
|
||||
|
||||
- Typed control/state lane: compact documents such as timing state, command
|
||||
receipts, health, stream cursors, and backpressure.
|
||||
- Media stream-frame lane: typed per-frame envelopes such as
|
||||
`mimir.cultmesh_stream_frame.v1` carrying source identity, capture time,
|
||||
resource/body refs, payload format, and confidence.
|
||||
- Media body-shard lane: bounded CultCache media/page shards with hashes,
|
||||
cursors, receiver acknowledgements, and backpressure. Large bodies do not
|
||||
travel as inline base64.
|
||||
- Latest-state dashboard lane: lossy/current Eve surfaces for operators. It
|
||||
may drop obsolete frames; it does not own durable media bodies.
|
||||
|
||||
Media streams are first-class Verse payloads. They still do not become timing
|
||||
authority by arrival timestamp. Clock influence comes from decoded anchors,
|
||||
source-local evidence, calibrated loopback, and explicit clock-domain state.
|
||||
|
||||
## Eve Surfaces
|
||||
|
||||
Mimir publishes operator presentation as Eve/CultUI DSL, preferably
|
||||
`gamecult.eve.surface.v1`. The expected surfaces are:
|
||||
|
||||
- `mimir.sensor.field.surface`: sensor field view with source health, buffer
|
||||
depth, timing confidence, calibration status, copy count, media format,
|
||||
resource residency, and per-source observation pressure.
|
||||
- `mimir.operator.flow.surface`: operator workflow for source selection,
|
||||
program placement, audio strips, record/live controls, reservoir pressure,
|
||||
sync mode, command receipts, and safe degraded-state visibility.
|
||||
- `mimir.observation.ledger.surface`: session/replay view over observation
|
||||
counts, body refs, accepted/dropped pages, cursor state, and schema witness.
|
||||
- `mimir.fensalir.reservoir.surface`: daemon-owned worker queue, ready ratio,
|
||||
timing confidence, GPU queue pressure, drop causes, and program surface refs.
|
||||
|
||||
The old dashboard broker, WebSocket `/eve/deck` lane, `/eve/dashboard`
|
||||
compatibility endpoint, JSON health endpoints, JPEG streams, and browser
|
||||
prototypes are lowerings or transport probes. They may mirror the Eve surface,
|
||||
but they do not become presentation owners.
|
||||
|
||||
## Nested Verses
|
||||
|
||||
Mimir's service Verse contains smaller Verses with bounded authority:
|
||||
|
||||
- Room Verse: calibration truth, coordinate frame, sensor graph, clock model,
|
||||
configured composites, and operator identity for one physical room.
|
||||
- Sensor Verse: one device or logical source, including driver health, timing
|
||||
domain, media format, actuator surface, and observation stream.
|
||||
- Stream Verse: one synchronized media/control lane, including rolling-window
|
||||
cursor, capture pages, resource refs, body refs, and backpressure state.
|
||||
- Operator Verse: Eve/CultUI command surface, selected source/layer state,
|
||||
command receipts, record/live state, and degraded-state visibility.
|
||||
|
||||
Nested Verses may publish local surfaces and state, but the parent Mimir service
|
||||
owns discovery and contract routing. A sensor may report what it observed; it
|
||||
does not become room clock authority unless explicitly promoted by calibration
|
||||
state.
|
||||
|
||||
## Migration Order
|
||||
|
||||
1. Publish a `verse.mimir.service.v1` manifest that names active namespaces,
|
||||
schema ids, run id, nested Verse roots, and the current transport lanes.
|
||||
2. Give each current JSONL/session/body-index path a `.cc` witness or
|
||||
CultCache-compatible export, starting with Well capture pages and
|
||||
`Mimir.VerseRecorder`.
|
||||
3. Promote sensor, media, observation, ledger, and control state into explicit
|
||||
`mimir.*` CultMesh namespaces while preserving current document ids.
|
||||
4. Publish `gamecult.eve.surface.v1` surfaces for sensor field and operator
|
||||
flows, with typed field bindings back to the owning state documents.
|
||||
5. Make the old dashboard broker and `/eve/dashboard` compatibility endpoint
|
||||
consume those provider surfaces instead of assembling private truth.
|
||||
6. Let Odin discover the service manifest and surfaces, then treat direct HTTP
|
||||
health/status probes as temporary diagnostics.
|
||||
7. Delete or neuter any private JSON store, dashboard state cache, or UI callback
|
||||
path that can still decide service behavior after the typed owner exists.
|
||||
|
||||
## Demotion Line
|
||||
|
||||
Bespoke UI, browser dashboards, retained dashboard node lists, compatibility
|
||||
WebSockets, SRT/FFmpeg media bridges, JSON status files, JSONL replay logs,
|
||||
screenshot streams, and OBS bridge scripts are demoted to lowerings,
|
||||
diagnostics, external-boundary export, or replay witnesses. They may display,
|
||||
transport, inspect, or reconstruct typed Mimir state. They may not own sensor
|
||||
truth, calibration truth, command state, reservoir scheduling, program
|
||||
presentation, media-over-Verse policy, or synchronization behavior.
|
||||
|
|
@ -107,11 +107,14 @@ The first cut should be deliberately boring:
|
|||
- `MimirNativeIngestStreamSource` is the low-overhead push seam for native
|
||||
camera/audio adapters that already own buffers or handles.
|
||||
- `MimirVideoFrameDescriptor` carries video shape, pixel format, device
|
||||
timestamp, stride, and optional native/GPU handle metadata. Leap stereo IR
|
||||
should enter here through a native driver path.
|
||||
timestamp, stride, resource key, producer fence value, and native/GPU handle
|
||||
metadata. Leap stereo IR should enter here through a native driver path that
|
||||
reports any unavoidable system-memory copy.
|
||||
- `IMimirVideoCaptureDriver` and `MimirVideoCaptureDriverSource` are the
|
||||
driver-facing seam for polling/pushing frames from LeapUVC/libusb, LeapC image
|
||||
APIs, Media Foundation, DirectShow, or a future GPU shared-texture capture.
|
||||
APIs, Media Foundation, DirectShow, or GPU decode/capture paths. Drivers that
|
||||
can produce render inputs should implement `IMimirFensalirTextureLeaseReceiver`
|
||||
and populate Fensalir-owned texture leases directly.
|
||||
- `MimirProcessStreamSource` is a compatibility adapter for network or
|
||||
diagnostic command sources. It is not the six-camera local capture foundation.
|
||||
|
||||
|
|
@ -129,10 +132,10 @@ environment variables:
|
|||
These create buffers immediately. Capture adapters still need to register
|
||||
`IMimirStreamSource` implementations to feed live samples into those buffers.
|
||||
|
||||
For local cameras, prefer native adapters that push sample handles directly
|
||||
through `MimirNativeIngestStreamSource`. Process-backed FFmpeg ingest is allowed
|
||||
for remote SRT/diagnostic edges, but local six-camera ingest should stay close
|
||||
to the driver, GPU, and capture APIs.
|
||||
For local cameras, prefer native adapters that talk to the device stack directly
|
||||
and push only typed sample/resource handles through `MimirNativeIngestStreamSource`.
|
||||
Process-backed FFmpeg ingest is allowed for remote SRT/diagnostic edges, but
|
||||
local six-camera ingest should stay close to the device, GPU, and capture APIs.
|
||||
|
||||
Leap should be treated as the timing-camera candidate only when the native
|
||||
adapter preserves device timestamps or captures a monotonic timestamp
|
||||
|
|
|
|||
65
faust/mimir_dual_mic_dialogue_cleaner.dsp
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
declare name "Mimir Dual Mic Dialogue Cleaner";
|
||||
declare version "0.1";
|
||||
declare author "Mimir";
|
||||
declare license "MIT";
|
||||
declare description "Faust-owned smooth dual-mic dialogue cleanup target: aligned shotgun/cardioid inputs become a conservative dialogue stem and a rejected residual witness.";
|
||||
|
||||
import("stdfaust.lib");
|
||||
|
||||
// Inputs are already delay-aligned upstream by mimir_alignment_actuator.
|
||||
// This graph owns smooth audio-rate cleanup, not calibration, timing, or UI.
|
||||
//
|
||||
// input 0: aligned shotgun / co-streamer witness
|
||||
// input 1: aligned cardioid / host witness
|
||||
// output 0: dialogue composite stem
|
||||
// output 1: rejected residual / noise witness
|
||||
|
||||
drive = hslider("dialogue/drive", 1.00, 0.00, 2.00, 0.001);
|
||||
shotgunWeight = hslider("dialogue/shotgun_weight", 0.45, 0.00, 1.00, 0.001);
|
||||
cardioidWeight = hslider("dialogue/cardioid_weight", 0.55, 0.00, 1.00, 0.001);
|
||||
witnessReject = hslider("dialogue/witness_reject", 0.20, 0.00, 1.00, 0.001);
|
||||
lowShelf = hslider("dialogue/low_shelf", 0.80, 0.00, 2.00, 0.001);
|
||||
presenceShelf = hslider("dialogue/presence_shelf", 1.08, 0.00, 2.00, 0.001);
|
||||
airShelf = hslider("dialogue/air_shelf", 0.90, 0.00, 2.00, 0.001);
|
||||
|
||||
// Smooth broadband downward expansion. This is intentionally slow enough to
|
||||
// avoid the choppy gate artifact from the C# audition bridge.
|
||||
floorLevel = hslider("denoise/noise_floor", 0.018, 0.0001, 0.25, 0.0001);
|
||||
expansionDepth = hslider("denoise/expansion_depth", 0.32, 0.00, 0.95, 0.001);
|
||||
expansionCurve = hslider("denoise/expansion_curve", 1.35, 0.25, 4.00, 0.001);
|
||||
envelopeHz = hslider("denoise/envelope_hz", 18.0, 1.0, 80.0, 0.1);
|
||||
|
||||
band(f, q, g, x) = x : fi.resonbp(1.0, f, q) : *(g);
|
||||
|
||||
voiceShape(x) =
|
||||
band(120.0, 1.2, lowShelf, x) +
|
||||
band(220.0, 1.6, 0.95, x) +
|
||||
band(420.0, 2.0, 0.98, x) +
|
||||
band(800.0, 2.4, 1.00, x) +
|
||||
band(1500.0, 2.8, presenceShelf, x) +
|
||||
band(2800.0, 3.2, presenceShelf, x) +
|
||||
band(5200.0, 3.2, airShelf, x) +
|
||||
band(8500.0, 2.6, airShelf, x);
|
||||
|
||||
smoothAbs(x) = abs(x) : si.smooth(ba.tau2pole(1.0 / max(1.0, envelopeHz)));
|
||||
|
||||
softExpand(x) = x * gain
|
||||
with {
|
||||
env = smoothAbs(x);
|
||||
ratio = min(1.0, env / max(0.0001, floorLevel));
|
||||
openness = pow(ratio, expansionCurve);
|
||||
gain = 1.0 - expansionDepth * (1.0 - openness);
|
||||
};
|
||||
|
||||
softClip(x) = x / (1.0 + abs(x));
|
||||
|
||||
clean(shotgun, cardioid) = dialogue, residual
|
||||
with {
|
||||
weighted = shotgun * shotgunWeight + cardioid * cardioidWeight;
|
||||
witness = (shotgun - cardioid) * 0.5;
|
||||
shaped = voiceShape(weighted - witnessReject * witness);
|
||||
dialogue = softExpand(shaped * drive) : softClip;
|
||||
residual = witness + (weighted - dialogue) * 0.25;
|
||||
};
|
||||
|
||||
process = clean;
|
||||
70
native/camera_capture/CMakeLists.txt
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
cmake_minimum_required(VERSION 3.20)
|
||||
|
||||
project(mimir_camera_capture LANGUAGES C CXX)
|
||||
|
||||
add_library(mimir_camera_capture SHARED
|
||||
mimir_ks_camera_capture.cpp)
|
||||
|
||||
target_compile_features(mimir_camera_capture PRIVATE cxx_std_20)
|
||||
target_compile_definitions(mimir_camera_capture PRIVATE
|
||||
WIN32_LEAN_AND_MEAN
|
||||
NOMINMAX)
|
||||
target_link_libraries(mimir_camera_capture PRIVATE
|
||||
setupapi
|
||||
ksuser)
|
||||
|
||||
set(PSMOVEAPI_ROOT "$ENV{TEMP}/mimir-driver-probes/psmoveapi" CACHE PATH "Root of a psmoveapi checkout with external/libusb-1.0 and external/PS3EYEDriver.")
|
||||
set(LIBUSB_ROOT "${PSMOVEAPI_ROOT}/external/libusb-1.0" CACHE PATH "libusb source root.")
|
||||
set(PS3EYE_ROOT "${PSMOVEAPI_ROOT}/external/PS3EYEDriver" CACHE PATH "PS3EYEDriver source root.")
|
||||
|
||||
add_library(mimir_libusb_static STATIC
|
||||
${LIBUSB_ROOT}/libusb/core.c
|
||||
${LIBUSB_ROOT}/libusb/descriptor.c
|
||||
${LIBUSB_ROOT}/libusb/hotplug.c
|
||||
${LIBUSB_ROOT}/libusb/io.c
|
||||
${LIBUSB_ROOT}/libusb/strerror.c
|
||||
${LIBUSB_ROOT}/libusb/sync.c
|
||||
${LIBUSB_ROOT}/libusb/os/events_windows.c
|
||||
${LIBUSB_ROOT}/libusb/os/threads_windows.c
|
||||
${LIBUSB_ROOT}/libusb/os/windows_common.c
|
||||
${LIBUSB_ROOT}/libusb/os/windows_usbdk.c
|
||||
${LIBUSB_ROOT}/libusb/os/windows_winusb.c)
|
||||
target_include_directories(mimir_libusb_static PUBLIC
|
||||
${LIBUSB_ROOT}/libusb
|
||||
${LIBUSB_ROOT}/libusb/os
|
||||
${LIBUSB_ROOT}/msvc)
|
||||
target_compile_definitions(mimir_libusb_static PRIVATE
|
||||
_CRT_SECURE_NO_WARNINGS
|
||||
_WIN32_WINNT=0x0601)
|
||||
target_link_libraries(mimir_libusb_static PRIVATE setupapi ole32 advapi32)
|
||||
|
||||
add_library(mimir_ps3eye_vendor STATIC
|
||||
${PS3EYE_ROOT}/src/ps3eye.cpp
|
||||
${PS3EYE_ROOT}/src/ps3eye_capi.cpp)
|
||||
target_include_directories(mimir_ps3eye_vendor PUBLIC
|
||||
${PS3EYE_ROOT}/src
|
||||
${LIBUSB_ROOT}/libusb)
|
||||
target_link_libraries(mimir_ps3eye_vendor PUBLIC mimir_libusb_static)
|
||||
|
||||
add_library(mimir_ps3eye_capture SHARED
|
||||
mimir_ps3eye_capture.cpp)
|
||||
target_compile_features(mimir_ps3eye_capture PRIVATE cxx_std_20)
|
||||
target_compile_definitions(mimir_ps3eye_capture PRIVATE
|
||||
WIN32_LEAN_AND_MEAN
|
||||
NOMINMAX)
|
||||
target_include_directories(mimir_ps3eye_capture PRIVATE ${PS3EYE_ROOT}/src)
|
||||
target_link_libraries(mimir_ps3eye_capture PRIVATE mimir_ps3eye_vendor)
|
||||
|
||||
add_library(mimir_mf_gpu_capture SHARED
|
||||
mimir_mf_gpu_capture.cpp)
|
||||
target_compile_features(mimir_mf_gpu_capture PRIVATE cxx_std_20)
|
||||
target_compile_definitions(mimir_mf_gpu_capture PRIVATE
|
||||
WIN32_LEAN_AND_MEAN
|
||||
NOMINMAX)
|
||||
target_link_libraries(mimir_mf_gpu_capture PRIVATE
|
||||
d3d11
|
||||
dxgi
|
||||
mf
|
||||
mfplat
|
||||
mfreadwrite
|
||||
mfuuid)
|
||||
759
native/camera_capture/mimir_ks_camera_capture.cpp
Normal file
|
|
@ -0,0 +1,759 @@
|
|||
#include <windows.h>
|
||||
#include <winioctl.h>
|
||||
#include <setupapi.h>
|
||||
#include <ks.h>
|
||||
#include <ksmedia.h>
|
||||
#include <winternl.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <cctype>
|
||||
#include <chrono>
|
||||
#include <condition_variable>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <deque>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
namespace
|
||||
{
|
||||
struct Handle
|
||||
{
|
||||
HANDLE value = INVALID_HANDLE_VALUE;
|
||||
|
||||
Handle() = default;
|
||||
explicit Handle(HANDLE handle) : value(handle) {}
|
||||
Handle(const Handle&) = delete;
|
||||
Handle& operator=(const Handle&) = delete;
|
||||
Handle(Handle&& other) noexcept : value(other.value) { other.value = INVALID_HANDLE_VALUE; }
|
||||
Handle& operator=(Handle&& other) noexcept
|
||||
{
|
||||
if (this != &other)
|
||||
{
|
||||
close();
|
||||
value = other.value;
|
||||
other.value = INVALID_HANDLE_VALUE;
|
||||
}
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
~Handle() { close(); }
|
||||
|
||||
void close()
|
||||
{
|
||||
if (value != INVALID_HANDLE_VALUE && value != nullptr)
|
||||
{
|
||||
CloseHandle(value);
|
||||
value = INVALID_HANDLE_VALUE;
|
||||
}
|
||||
}
|
||||
|
||||
explicit operator bool() const { return value != INVALID_HANDLE_VALUE && value != nullptr; }
|
||||
};
|
||||
|
||||
struct DeviceInfoSet
|
||||
{
|
||||
HDEVINFO value = INVALID_HANDLE_VALUE;
|
||||
|
||||
explicit DeviceInfoSet(HDEVINFO info) : value(info) {}
|
||||
DeviceInfoSet(const DeviceInfoSet&) = delete;
|
||||
DeviceInfoSet& operator=(const DeviceInfoSet&) = delete;
|
||||
~DeviceInfoSet()
|
||||
{
|
||||
if (value != INVALID_HANDLE_VALUE)
|
||||
{
|
||||
SetupDiDestroyDeviceInfoList(value);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
using NtDeviceIoControlFileFn = NTSTATUS(NTAPI*)(
|
||||
HANDLE FileHandle,
|
||||
HANDLE Event,
|
||||
PIO_APC_ROUTINE ApcRoutine,
|
||||
PVOID ApcContext,
|
||||
PIO_STATUS_BLOCK IoStatusBlock,
|
||||
ULONG IoControlCode,
|
||||
PVOID InputBuffer,
|
||||
ULONG InputBufferLength,
|
||||
PVOID OutputBuffer,
|
||||
ULONG OutputBufferLength);
|
||||
|
||||
constexpr NTSTATUS StatusPending = static_cast<NTSTATUS>(0x00000103L);
|
||||
|
||||
struct PinCandidate
|
||||
{
|
||||
ULONG pinId = 0;
|
||||
std::vector<std::uint8_t> format;
|
||||
LONG width = 0;
|
||||
LONG height = 0;
|
||||
LONG imageBytes = 0;
|
||||
LONGLONG interval100ns = 0;
|
||||
GUID subtype{};
|
||||
std::string fourcc;
|
||||
};
|
||||
|
||||
struct MimirKsCameraFrame
|
||||
{
|
||||
std::vector<std::uint8_t> data;
|
||||
std::int64_t timestampNs = 0;
|
||||
std::uint64_t sequence = 0;
|
||||
int byteLength = 0;
|
||||
};
|
||||
|
||||
struct MimirKsCameraCapture
|
||||
{
|
||||
Handle filter;
|
||||
Handle pin;
|
||||
PinCandidate candidate;
|
||||
std::thread worker;
|
||||
std::mutex mutex;
|
||||
std::condition_variable condition;
|
||||
std::deque<MimirKsCameraFrame> queue;
|
||||
std::atomic<bool> running = false;
|
||||
int queueDepth = 8;
|
||||
std::uint64_t sequence = 0;
|
||||
};
|
||||
|
||||
long long nowNs()
|
||||
{
|
||||
return std::chrono::duration_cast<std::chrono::nanoseconds>(
|
||||
std::chrono::steady_clock::now().time_since_epoch()).count();
|
||||
}
|
||||
|
||||
std::string lowercase(std::string value)
|
||||
{
|
||||
std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) {
|
||||
return static_cast<char>(std::tolower(c));
|
||||
});
|
||||
return value;
|
||||
}
|
||||
|
||||
std::string wideToUtf8(const wchar_t* value)
|
||||
{
|
||||
if (!value)
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
const int bytes = WideCharToMultiByte(CP_UTF8, 0, value, -1, nullptr, 0, nullptr, nullptr);
|
||||
if (bytes <= 1)
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
std::string text(static_cast<std::size_t>(bytes - 1), '\0');
|
||||
WideCharToMultiByte(CP_UTF8, 0, value, -1, text.data(), bytes, nullptr, nullptr);
|
||||
return text;
|
||||
}
|
||||
|
||||
std::string fourccOrGuid(const GUID& guid)
|
||||
{
|
||||
if (guid.Data2 == 0x0000 &&
|
||||
guid.Data3 == 0x0010 &&
|
||||
guid.Data4[0] == 0x80 &&
|
||||
guid.Data4[1] == 0x00 &&
|
||||
guid.Data4[2] == 0x00 &&
|
||||
guid.Data4[3] == 0xaa &&
|
||||
guid.Data4[4] == 0x00 &&
|
||||
guid.Data4[5] == 0x38 &&
|
||||
guid.Data4[6] == 0x9b &&
|
||||
guid.Data4[7] == 0x71)
|
||||
{
|
||||
char text[5]{};
|
||||
text[0] = static_cast<char>(guid.Data1 & 0xff);
|
||||
text[1] = static_cast<char>((guid.Data1 >> 8) & 0xff);
|
||||
text[2] = static_cast<char>((guid.Data1 >> 16) & 0xff);
|
||||
text[3] = static_cast<char>((guid.Data1 >> 24) & 0xff);
|
||||
return text;
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
bool ksProperty(HANDLE handle, void* property, DWORD propertyBytes, void* output, DWORD outputBytes, DWORD* returned = nullptr)
|
||||
{
|
||||
DWORD bytes = 0;
|
||||
const BOOL ok = DeviceIoControl(
|
||||
handle,
|
||||
IOCTL_KS_PROPERTY,
|
||||
property,
|
||||
propertyBytes,
|
||||
output,
|
||||
outputBytes,
|
||||
&bytes,
|
||||
nullptr);
|
||||
if (returned)
|
||||
{
|
||||
*returned = bytes;
|
||||
}
|
||||
|
||||
return ok != FALSE;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool getPinProperty(HANDLE filter, ULONG pinId, ULONG propertyId, T& value)
|
||||
{
|
||||
KSP_PIN property{};
|
||||
property.Property.Set = KSPROPSETID_Pin;
|
||||
property.Property.Id = propertyId;
|
||||
property.Property.Flags = KSPROPERTY_TYPE_GET;
|
||||
property.PinId = pinId;
|
||||
DWORD returned = 0;
|
||||
return ksProperty(filter, &property, sizeof(property), &value, sizeof(value), &returned);
|
||||
}
|
||||
|
||||
bool setVideoProcAmp(HANDLE filter, ULONG id, LONG value, ULONG flags)
|
||||
{
|
||||
KSPROPERTY_VIDEOPROCAMP_S property{};
|
||||
property.Property.Set = PROPSETID_VIDCAP_VIDEOPROCAMP;
|
||||
property.Property.Id = id;
|
||||
property.Property.Flags = KSPROPERTY_TYPE_SET;
|
||||
property.Value = value;
|
||||
property.Flags = flags;
|
||||
return ksProperty(filter, &property, sizeof(property), &property, sizeof(property));
|
||||
}
|
||||
|
||||
bool setCameraControl(HANDLE filter, ULONG id, LONG value, ULONG flags)
|
||||
{
|
||||
KSPROPERTY_CAMERACONTROL_S property{};
|
||||
property.Property.Set = PROPSETID_VIDCAP_CAMERACONTROL;
|
||||
property.Property.Id = id;
|
||||
property.Property.Flags = KSPROPERTY_TYPE_SET;
|
||||
property.Value = value;
|
||||
property.Flags = flags;
|
||||
return ksProperty(filter, &property, sizeof(property), &property, sizeof(property));
|
||||
}
|
||||
|
||||
std::vector<std::string> enumerateCapturePaths()
|
||||
{
|
||||
std::vector<std::string> paths;
|
||||
DeviceInfoSet info(SetupDiGetClassDevsA(
|
||||
&KSCATEGORY_CAPTURE,
|
||||
nullptr,
|
||||
nullptr,
|
||||
DIGCF_PRESENT | DIGCF_DEVICEINTERFACE));
|
||||
if (info.value == INVALID_HANDLE_VALUE)
|
||||
{
|
||||
return paths;
|
||||
}
|
||||
|
||||
for (DWORD index = 0;; ++index)
|
||||
{
|
||||
SP_DEVICE_INTERFACE_DATA interfaceData{};
|
||||
interfaceData.cbSize = sizeof(interfaceData);
|
||||
if (!SetupDiEnumDeviceInterfaces(info.value, nullptr, &KSCATEGORY_CAPTURE, index, &interfaceData))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
DWORD required = 0;
|
||||
SetupDiGetDeviceInterfaceDetailW(info.value, &interfaceData, nullptr, 0, &required, nullptr);
|
||||
if (required == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
std::vector<std::uint8_t> storage(required);
|
||||
auto* detail = reinterpret_cast<SP_DEVICE_INTERFACE_DETAIL_DATA_W*>(storage.data());
|
||||
detail->cbSize = sizeof(*detail);
|
||||
if (SetupDiGetDeviceInterfaceDetailW(info.value, &interfaceData, detail, required, nullptr, nullptr))
|
||||
{
|
||||
paths.push_back(wideToUtf8(detail->DevicePath));
|
||||
}
|
||||
}
|
||||
|
||||
return paths;
|
||||
}
|
||||
|
||||
std::vector<PinCandidate> inspectPins(HANDLE filter)
|
||||
{
|
||||
ULONG pinCount = 0;
|
||||
KSPROPERTY countProperty{};
|
||||
countProperty.Set = KSPROPSETID_Pin;
|
||||
countProperty.Id = KSPROPERTY_PIN_CTYPES;
|
||||
countProperty.Flags = KSPROPERTY_TYPE_GET;
|
||||
if (!ksProperty(filter, &countProperty, sizeof(countProperty), &pinCount, sizeof(pinCount)))
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
std::vector<PinCandidate> candidates;
|
||||
for (ULONG pinId = 0; pinId < pinCount; ++pinId)
|
||||
{
|
||||
KSPIN_DATAFLOW dataflow{};
|
||||
if (!getPinProperty(filter, pinId, KSPROPERTY_PIN_DATAFLOW, dataflow) ||
|
||||
dataflow != KSPIN_DATAFLOW_OUT)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
KSP_PIN rangeProperty{};
|
||||
rangeProperty.Property.Set = KSPROPSETID_Pin;
|
||||
rangeProperty.Property.Id = KSPROPERTY_PIN_DATARANGES;
|
||||
rangeProperty.Property.Flags = KSPROPERTY_TYPE_GET;
|
||||
rangeProperty.PinId = pinId;
|
||||
|
||||
DWORD rangeBytes = 0;
|
||||
ksProperty(filter, &rangeProperty, sizeof(rangeProperty), nullptr, 0, &rangeBytes);
|
||||
if (rangeBytes == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
std::vector<std::uint8_t> ranges(rangeBytes);
|
||||
DWORD returned = 0;
|
||||
if (!ksProperty(filter, &rangeProperty, sizeof(rangeProperty), ranges.data(), static_cast<DWORD>(ranges.size()), &returned))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
auto* multiple = reinterpret_cast<KSMULTIPLE_ITEM*>(ranges.data());
|
||||
auto* cursor = ranges.data() + sizeof(KSMULTIPLE_ITEM);
|
||||
for (ULONG i = 0; i < multiple->Count; ++i)
|
||||
{
|
||||
auto* dataRange = reinterpret_cast<KSDATARANGE*>(cursor);
|
||||
const bool videoInfo =
|
||||
IsEqualGUID(dataRange->MajorFormat, KSDATAFORMAT_TYPE_VIDEO) &&
|
||||
IsEqualGUID(dataRange->Specifier, KSDATAFORMAT_SPECIFIER_VIDEOINFO) &&
|
||||
dataRange->FormatSize >= sizeof(KS_DATARANGE_VIDEO);
|
||||
|
||||
if (videoInfo)
|
||||
{
|
||||
auto* video = reinterpret_cast<KS_DATARANGE_VIDEO*>(cursor);
|
||||
const auto& bmi = video->VideoInfoHeader.bmiHeader;
|
||||
PinCandidate candidate{};
|
||||
candidate.pinId = pinId;
|
||||
candidate.width = bmi.biWidth;
|
||||
candidate.height = bmi.biHeight;
|
||||
candidate.imageBytes = bmi.biSizeImage > 0 ? bmi.biSizeImage : static_cast<LONG>(dataRange->SampleSize);
|
||||
candidate.interval100ns = video->VideoInfoHeader.AvgTimePerFrame;
|
||||
candidate.subtype = dataRange->SubFormat;
|
||||
candidate.fourcc = fourccOrGuid(candidate.subtype);
|
||||
candidate.format.resize(sizeof(KS_DATAFORMAT_VIDEOINFOHEADER));
|
||||
auto* format = reinterpret_cast<KS_DATAFORMAT_VIDEOINFOHEADER*>(candidate.format.data());
|
||||
format->DataFormat.FormatSize = sizeof(KS_DATAFORMAT_VIDEOINFOHEADER);
|
||||
format->DataFormat.SampleSize = candidate.imageBytes;
|
||||
format->DataFormat.MajorFormat = KSDATAFORMAT_TYPE_VIDEO;
|
||||
format->DataFormat.SubFormat = dataRange->SubFormat;
|
||||
format->DataFormat.Specifier = KSDATAFORMAT_SPECIFIER_VIDEOINFO;
|
||||
format->VideoInfoHeader = video->VideoInfoHeader;
|
||||
format->VideoInfoHeader.bmiHeader.biSizeImage = candidate.imageBytes;
|
||||
candidates.push_back(std::move(candidate));
|
||||
}
|
||||
|
||||
cursor += dataRange->FormatSize;
|
||||
}
|
||||
}
|
||||
|
||||
return candidates;
|
||||
}
|
||||
|
||||
const PinCandidate* findCandidate(
|
||||
const std::vector<PinCandidate>& candidates,
|
||||
int width,
|
||||
int height,
|
||||
const char* subtype,
|
||||
double minFps)
|
||||
{
|
||||
const std::string target = subtype ? subtype : "";
|
||||
for (const auto& candidate : candidates)
|
||||
{
|
||||
const double fps = candidate.interval100ns > 0
|
||||
? 10000000.0 / static_cast<double>(candidate.interval100ns)
|
||||
: 0.0;
|
||||
if (candidate.width == width &&
|
||||
candidate.height == height &&
|
||||
fps >= minFps &&
|
||||
candidate.fourcc == target)
|
||||
{
|
||||
return &candidate;
|
||||
}
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
bool setState(HANDLE pin, KSSTATE state)
|
||||
{
|
||||
KSPROPERTY property{};
|
||||
property.Set = KSPROPSETID_Connection;
|
||||
property.Id = KSPROPERTY_CONNECTION_STATE;
|
||||
property.Flags = KSPROPERTY_TYPE_SET;
|
||||
DWORD returned = 0;
|
||||
return DeviceIoControl(
|
||||
pin,
|
||||
IOCTL_KS_PROPERTY,
|
||||
&property,
|
||||
sizeof(property),
|
||||
&state,
|
||||
sizeof(state),
|
||||
&returned,
|
||||
nullptr) != FALSE;
|
||||
}
|
||||
|
||||
int strideBytes(const PinCandidate& candidate)
|
||||
{
|
||||
return candidate.height > 0 && candidate.imageBytes > 0
|
||||
? candidate.imageBytes / candidate.height
|
||||
: 0;
|
||||
}
|
||||
|
||||
void pushFrame(MimirKsCameraCapture& capture, const std::uint8_t* data, ULONG byteLength)
|
||||
{
|
||||
MimirKsCameraFrame frame;
|
||||
frame.byteLength = static_cast<int>(byteLength);
|
||||
frame.timestampNs = nowNs();
|
||||
frame.sequence = capture.sequence++;
|
||||
frame.data.resize(byteLength);
|
||||
std::memcpy(frame.data.data(), data, byteLength);
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(capture.mutex);
|
||||
capture.queue.push_back(std::move(frame));
|
||||
while (capture.queue.size() > 32)
|
||||
{
|
||||
capture.queue.pop_front();
|
||||
}
|
||||
}
|
||||
capture.condition.notify_one();
|
||||
}
|
||||
|
||||
void captureThread(MimirKsCameraCapture* capture)
|
||||
{
|
||||
auto* ntDeviceIoControlFile = reinterpret_cast<NtDeviceIoControlFileFn>(
|
||||
GetProcAddress(GetModuleHandleA("ntdll.dll"), "NtDeviceIoControlFile"));
|
||||
if (!ntDeviceIoControlFile)
|
||||
{
|
||||
capture->running.store(false);
|
||||
return;
|
||||
}
|
||||
|
||||
const ULONG connectBytes = sizeof(KSPIN_CONNECT) + static_cast<ULONG>(capture->candidate.format.size());
|
||||
std::vector<std::uint8_t> connectStorage(connectBytes);
|
||||
auto* connect = reinterpret_cast<KSPIN_CONNECT*>(connectStorage.data());
|
||||
connect->Interface.Set = KSINTERFACESETID_Standard;
|
||||
connect->Interface.Id = KSINTERFACE_STANDARD_STREAMING;
|
||||
connect->Medium.Set = KSMEDIUMSETID_Standard;
|
||||
connect->Medium.Id = KSMEDIUM_TYPE_ANYINSTANCE;
|
||||
connect->PinId = capture->candidate.pinId;
|
||||
connect->Priority.PriorityClass = KSPRIORITY_NORMAL;
|
||||
connect->Priority.PrioritySubClass = 1;
|
||||
std::memcpy(connectStorage.data() + sizeof(KSPIN_CONNECT), capture->candidate.format.data(), capture->candidate.format.size());
|
||||
|
||||
HANDLE rawPin = INVALID_HANDLE_VALUE;
|
||||
const HRESULT hr = KsCreatePin(capture->filter.value, connect, GENERIC_READ, &rawPin);
|
||||
if (FAILED(hr))
|
||||
{
|
||||
capture->running.store(false);
|
||||
return;
|
||||
}
|
||||
|
||||
capture->pin = Handle(rawPin);
|
||||
if (!setState(capture->pin.value, KSSTATE_ACQUIRE) ||
|
||||
!setState(capture->pin.value, KSSTATE_PAUSE) ||
|
||||
!setState(capture->pin.value, KSSTATE_RUN))
|
||||
{
|
||||
capture->running.store(false);
|
||||
return;
|
||||
}
|
||||
|
||||
const DWORD frameBytes = static_cast<DWORD>(capture->candidate.imageBytes);
|
||||
struct ReadSlot
|
||||
{
|
||||
Handle event;
|
||||
IO_STATUS_BLOCK status{};
|
||||
KSSTREAM_HEADER header{};
|
||||
std::vector<std::uint8_t> frame;
|
||||
bool pending = false;
|
||||
};
|
||||
|
||||
std::vector<ReadSlot> slots(static_cast<std::size_t>(std::max(1, capture->queueDepth)));
|
||||
std::vector<HANDLE> waitEvents(slots.size());
|
||||
for (auto& slot : slots)
|
||||
{
|
||||
slot.event = Handle(CreateEventW(nullptr, TRUE, FALSE, nullptr));
|
||||
if (!slot.event)
|
||||
{
|
||||
capture->running.store(false);
|
||||
return;
|
||||
}
|
||||
|
||||
slot.frame.resize(frameBytes);
|
||||
slot.header.Size = sizeof(KSSTREAM_HEADER);
|
||||
slot.header.FrameExtent = frameBytes;
|
||||
slot.header.Data = slot.frame.data();
|
||||
slot.header.PresentationTime.Numerator = 1;
|
||||
slot.header.PresentationTime.Denominator = 1;
|
||||
waitEvents[&slot - slots.data()] = slot.event.value;
|
||||
}
|
||||
|
||||
auto submit = [&](ReadSlot& slot) -> bool {
|
||||
ResetEvent(slot.event.value);
|
||||
slot.status = {};
|
||||
slot.header.DataUsed = 0;
|
||||
slot.header.OptionsFlags = 0;
|
||||
const NTSTATUS status = ntDeviceIoControlFile(
|
||||
capture->pin.value,
|
||||
slot.event.value,
|
||||
nullptr,
|
||||
nullptr,
|
||||
&slot.status,
|
||||
IOCTL_KS_READ_STREAM,
|
||||
&slot.header,
|
||||
sizeof(slot.header),
|
||||
&slot.header,
|
||||
sizeof(slot.header));
|
||||
|
||||
if (status == StatusPending)
|
||||
{
|
||||
slot.pending = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
slot.pending = false;
|
||||
if (status >= 0 && slot.header.DataUsed > 0)
|
||||
{
|
||||
pushFrame(*capture, slot.frame.data(), slot.header.DataUsed);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
};
|
||||
|
||||
for (auto& slot : slots)
|
||||
{
|
||||
if (!submit(slot))
|
||||
{
|
||||
capture->running.store(false);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
while (capture->running.load())
|
||||
{
|
||||
const DWORD waitResult = WaitForMultipleObjects(
|
||||
static_cast<DWORD>(waitEvents.size()),
|
||||
waitEvents.data(),
|
||||
FALSE,
|
||||
100);
|
||||
if (waitResult == WAIT_TIMEOUT)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (waitResult < WAIT_OBJECT_0 || waitResult >= WAIT_OBJECT_0 + waitEvents.size())
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
auto& slot = slots[static_cast<std::size_t>(waitResult - WAIT_OBJECT_0)];
|
||||
slot.pending = false;
|
||||
if (slot.status.Status < 0 || slot.header.DataUsed == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
pushFrame(*capture, slot.frame.data(), slot.header.DataUsed);
|
||||
if (!submit(slot))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
setState(capture->pin.value, KSSTATE_STOP);
|
||||
CancelIoEx(capture->pin.value, nullptr);
|
||||
for (auto& slot : slots)
|
||||
{
|
||||
if (slot.pending)
|
||||
{
|
||||
WaitForSingleObject(slot.event.value, 100);
|
||||
}
|
||||
}
|
||||
capture->running.store(false);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
extern "C"
|
||||
{
|
||||
__declspec(dllexport) MimirKsCameraCapture* mimir_ks_create(
|
||||
const char* pathNeedle,
|
||||
int width,
|
||||
int height,
|
||||
const char* subtype,
|
||||
double minFps,
|
||||
int queueDepth)
|
||||
{
|
||||
const std::string needle = lowercase(pathNeedle ? pathNeedle : "");
|
||||
for (const auto& path : enumerateCapturePaths())
|
||||
{
|
||||
if (!needle.empty() && lowercase(path).find(needle) == std::string::npos)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
Handle filter(CreateFileA(
|
||||
path.c_str(),
|
||||
GENERIC_READ | GENERIC_WRITE,
|
||||
FILE_SHARE_READ | FILE_SHARE_WRITE,
|
||||
nullptr,
|
||||
OPEN_EXISTING,
|
||||
FILE_ATTRIBUTE_NORMAL,
|
||||
nullptr));
|
||||
if (!filter)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
auto candidates = inspectPins(filter.value);
|
||||
const auto* candidate = findCandidate(candidates, width, height, subtype, minFps);
|
||||
if (!candidate)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
auto* capture = new MimirKsCameraCapture();
|
||||
capture->filter = std::move(filter);
|
||||
capture->candidate = *candidate;
|
||||
capture->queueDepth = std::max(1, std::min(queueDepth, 32));
|
||||
return capture;
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
__declspec(dllexport) int mimir_ks_start(MimirKsCameraCapture* capture)
|
||||
{
|
||||
if (!capture || capture->running.exchange(true))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
capture->worker = std::thread(captureThread, capture);
|
||||
return 1;
|
||||
}
|
||||
|
||||
__declspec(dllexport) int mimir_ks_set_exposure_gain(
|
||||
MimirKsCameraCapture* capture,
|
||||
int setExposure,
|
||||
LONG exposure,
|
||||
int setGain,
|
||||
LONG gain)
|
||||
{
|
||||
if (!capture || !capture->filter)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
int result = 0;
|
||||
if (!setExposure ||
|
||||
setCameraControl(
|
||||
capture->filter.value,
|
||||
KSPROPERTY_CAMERACONTROL_EXPOSURE,
|
||||
exposure,
|
||||
KSPROPERTY_CAMERACONTROL_FLAGS_MANUAL))
|
||||
{
|
||||
result |= 1;
|
||||
}
|
||||
if (!setGain ||
|
||||
setVideoProcAmp(
|
||||
capture->filter.value,
|
||||
KSPROPERTY_VIDEOPROCAMP_GAIN,
|
||||
gain,
|
||||
KSPROPERTY_VIDEOPROCAMP_FLAGS_MANUAL))
|
||||
{
|
||||
result |= 2;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
__declspec(dllexport) int mimir_ks_read(
|
||||
MimirKsCameraCapture* capture,
|
||||
int* width,
|
||||
int* height,
|
||||
int* stride,
|
||||
char* fourcc,
|
||||
int fourccCapacity,
|
||||
std::int64_t* timestampNs,
|
||||
std::uint64_t* sequence,
|
||||
std::uint8_t* destination,
|
||||
int destinationBytes)
|
||||
{
|
||||
if (!capture || !destination || destinationBytes <= 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
MimirKsCameraFrame frame;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(capture->mutex);
|
||||
if (capture->queue.empty())
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
frame = std::move(capture->queue.front());
|
||||
capture->queue.pop_front();
|
||||
}
|
||||
|
||||
if (frame.byteLength > destinationBytes)
|
||||
{
|
||||
return -frame.byteLength;
|
||||
}
|
||||
|
||||
std::memcpy(destination, frame.data.data(), static_cast<std::size_t>(frame.byteLength));
|
||||
if (width)
|
||||
{
|
||||
*width = static_cast<int>(capture->candidate.width);
|
||||
}
|
||||
if (height)
|
||||
{
|
||||
*height = static_cast<int>(capture->candidate.height);
|
||||
}
|
||||
if (stride)
|
||||
{
|
||||
*stride = strideBytes(capture->candidate);
|
||||
}
|
||||
if (fourcc && fourccCapacity > 0)
|
||||
{
|
||||
std::snprintf(fourcc, static_cast<std::size_t>(fourccCapacity), "%s", capture->candidate.fourcc.c_str());
|
||||
}
|
||||
if (timestampNs)
|
||||
{
|
||||
*timestampNs = frame.timestampNs;
|
||||
}
|
||||
if (sequence)
|
||||
{
|
||||
*sequence = frame.sequence;
|
||||
}
|
||||
|
||||
return frame.byteLength;
|
||||
}
|
||||
|
||||
__declspec(dllexport) void mimir_ks_destroy(MimirKsCameraCapture* capture)
|
||||
{
|
||||
if (!capture)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
capture->running.store(false);
|
||||
if (capture->pin)
|
||||
{
|
||||
CancelIoEx(capture->pin.value, nullptr);
|
||||
}
|
||||
if (capture->worker.joinable())
|
||||
{
|
||||
capture->worker.join();
|
||||
}
|
||||
|
||||
delete capture;
|
||||
}
|
||||
}
|
||||
573
native/camera_capture/mimir_mf_gpu_capture.cpp
Normal file
|
|
@ -0,0 +1,573 @@
|
|||
#include <windows.h>
|
||||
#include <d3d11_4.h>
|
||||
#include <dxgi1_6.h>
|
||||
#include <mfapi.h>
|
||||
#include <mferror.h>
|
||||
#include <mfidl.h>
|
||||
#include <mfreadwrite.h>
|
||||
#include <wrl/client.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstdarg>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
using Microsoft::WRL::ComPtr;
|
||||
|
||||
namespace
|
||||
{
|
||||
struct MimirMfGpuCapture
|
||||
{
|
||||
ComPtr<IMFMediaSource> source;
|
||||
ComPtr<IMFSourceReader> reader;
|
||||
ComPtr<ID3D11Device> device;
|
||||
ComPtr<ID3D11DeviceContext> context;
|
||||
ComPtr<IMFDXGIDeviceManager> deviceManager;
|
||||
ComPtr<ID3D11Texture2D> sharedTexture;
|
||||
ComPtr<ID3D11Fence> producerFence;
|
||||
ComPtr<ID3D11DeviceContext4> fenceContext;
|
||||
HANDLE sharedHandle = nullptr;
|
||||
HANDLE producerFenceHandle = nullptr;
|
||||
int width = 0;
|
||||
int height = 0;
|
||||
DXGI_FORMAT format = DXGI_FORMAT_UNKNOWN;
|
||||
std::uint64_t sequence = 0;
|
||||
};
|
||||
|
||||
bool debugEnabled()
|
||||
{
|
||||
return GetEnvironmentVariableA("MIMIR_MF_GPU_DEBUG", nullptr, 0) > 0;
|
||||
}
|
||||
|
||||
void debugLog(const char* format, ...)
|
||||
{
|
||||
if (!debugEnabled())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
std::fprintf(stderr, "mimir-mf-gpu: ");
|
||||
va_list args;
|
||||
va_start(args, format);
|
||||
std::vfprintf(stderr, format, args);
|
||||
va_end(args);
|
||||
std::fprintf(stderr, "\n");
|
||||
}
|
||||
|
||||
std::wstring widen(const char* value)
|
||||
{
|
||||
if (!value)
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
const int chars = MultiByteToWideChar(CP_UTF8, 0, value, -1, nullptr, 0);
|
||||
if (chars <= 1)
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
std::wstring result(static_cast<std::size_t>(chars - 1), L'\0');
|
||||
MultiByteToWideChar(CP_UTF8, 0, value, -1, result.data(), chars);
|
||||
return result;
|
||||
}
|
||||
|
||||
std::wstring lowercase(std::wstring value)
|
||||
{
|
||||
std::transform(value.begin(), value.end(), value.begin(), [](wchar_t c) {
|
||||
return static_cast<wchar_t>(towlower(c));
|
||||
});
|
||||
return value;
|
||||
}
|
||||
|
||||
GUID subtypeFromText(const char* text)
|
||||
{
|
||||
const std::string value = text ? text : "";
|
||||
std::string upper;
|
||||
upper.resize(value.size());
|
||||
std::transform(value.begin(), value.end(), upper.begin(), [](unsigned char c) {
|
||||
return static_cast<char>(std::toupper(c));
|
||||
});
|
||||
|
||||
if (upper == "MJPG") return MFVideoFormat_MJPG;
|
||||
if (upper == "H264") return MFVideoFormat_H264;
|
||||
if (upper == "NV12") return MFVideoFormat_NV12;
|
||||
if (upper == "YUY2") return MFVideoFormat_YUY2;
|
||||
if (upper == "RGB32" || upper == "BGRA") return MFVideoFormat_RGB32;
|
||||
return GUID_NULL;
|
||||
}
|
||||
|
||||
DXGI_FORMAT dxgiFormatForSubtype(const GUID& subtype)
|
||||
{
|
||||
if (subtype == MFVideoFormat_NV12) return DXGI_FORMAT_NV12;
|
||||
if (subtype == MFVideoFormat_RGB32) return DXGI_FORMAT_B8G8R8A8_UNORM;
|
||||
if (subtype == MFVideoFormat_YUY2) return DXGI_FORMAT_YUY2;
|
||||
return DXGI_FORMAT_UNKNOWN;
|
||||
}
|
||||
|
||||
const char* textForSubtype(const GUID& subtype)
|
||||
{
|
||||
if (subtype == MFVideoFormat_NV12) return "NV12";
|
||||
if (subtype == MFVideoFormat_RGB32) return "Bgra8";
|
||||
if (subtype == MFVideoFormat_YUY2) return "YUY2";
|
||||
return "Unknown";
|
||||
}
|
||||
|
||||
bool findVideoSource(const wchar_t* needle, IMFMediaSource** mediaSource)
|
||||
{
|
||||
*mediaSource = nullptr;
|
||||
ComPtr<IMFAttributes> attributes;
|
||||
if (FAILED(MFCreateAttributes(&attributes, 2)))
|
||||
{
|
||||
debugLog("MFCreateAttributes for devices failed");
|
||||
return false;
|
||||
}
|
||||
|
||||
attributes->SetGUID(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE, MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID);
|
||||
IMFActivate** rawDevices = nullptr;
|
||||
UINT32 count = 0;
|
||||
if (FAILED(MFEnumDeviceSources(attributes.Get(), &rawDevices, &count)))
|
||||
{
|
||||
debugLog("MFEnumDeviceSources failed");
|
||||
return false;
|
||||
}
|
||||
|
||||
debugLog("device sources=%u", count);
|
||||
|
||||
const std::wstring target = lowercase(needle ? needle : L"");
|
||||
bool found = false;
|
||||
for (UINT32 index = 0; index < count; ++index)
|
||||
{
|
||||
ComPtr<IMFActivate> activate(rawDevices[index]);
|
||||
rawDevices[index] = nullptr;
|
||||
WCHAR* symbolicLink = nullptr;
|
||||
UINT32 symbolicLength = 0;
|
||||
activate->GetAllocatedString(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK, &symbolicLink, &symbolicLength);
|
||||
const std::wstring link = lowercase(symbolicLink ? symbolicLink : L"");
|
||||
if (debugEnabled())
|
||||
{
|
||||
std::fwprintf(stderr, L"mimir-mf-gpu: source[%u]=%ls\n", index, link.c_str());
|
||||
}
|
||||
CoTaskMemFree(symbolicLink);
|
||||
if (!target.empty() && link.find(target) == std::wstring::npos)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (SUCCEEDED(activate->ActivateObject(IID_PPV_ARGS(mediaSource))))
|
||||
{
|
||||
debugLog("activated source index=%u", index);
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
for (UINT32 index = 0; index < count; ++index)
|
||||
{
|
||||
if (rawDevices[index])
|
||||
{
|
||||
rawDevices[index]->Release();
|
||||
}
|
||||
}
|
||||
CoTaskMemFree(rawDevices);
|
||||
return found;
|
||||
}
|
||||
|
||||
bool mediaTypeMatches(IMFMediaType* type, int width, int height, const GUID& subtype, double minFps)
|
||||
{
|
||||
GUID actualSubtype{};
|
||||
UINT32 actualWidth = 0;
|
||||
UINT32 actualHeight = 0;
|
||||
UINT32 fpsNumerator = 0;
|
||||
UINT32 fpsDenominator = 0;
|
||||
if (FAILED(type->GetGUID(MF_MT_SUBTYPE, &actualSubtype)) ||
|
||||
actualSubtype != subtype ||
|
||||
FAILED(MFGetAttributeSize(type, MF_MT_FRAME_SIZE, &actualWidth, &actualHeight)) ||
|
||||
static_cast<int>(actualWidth) != width ||
|
||||
static_cast<int>(actualHeight) != height)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (SUCCEEDED(MFGetAttributeRatio(type, MF_MT_FRAME_RATE, &fpsNumerator, &fpsDenominator)) &&
|
||||
fpsDenominator != 0)
|
||||
{
|
||||
return static_cast<double>(fpsNumerator) / fpsDenominator >= minFps;
|
||||
}
|
||||
|
||||
return minFps <= 0.0;
|
||||
}
|
||||
|
||||
bool setInputType(IMFSourceReader* reader, int width, int height, const GUID& subtype, double minFps)
|
||||
{
|
||||
for (DWORD index = 0;; ++index)
|
||||
{
|
||||
ComPtr<IMFMediaType> type;
|
||||
const HRESULT hr = reader->GetNativeMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, index, &type);
|
||||
if (hr == MF_E_NO_MORE_TYPES)
|
||||
{
|
||||
debugLog("no input type matched %dx%d minFps=%.3f", width, height, minFps);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (FAILED(hr))
|
||||
{
|
||||
debugLog("GetNativeMediaType failed index=%lu hr=0x%08lx", index, static_cast<unsigned long>(hr));
|
||||
return false;
|
||||
}
|
||||
|
||||
if (debugEnabled())
|
||||
{
|
||||
GUID actualSubtype{};
|
||||
UINT32 actualWidth = 0;
|
||||
UINT32 actualHeight = 0;
|
||||
UINT32 fpsNumerator = 0;
|
||||
UINT32 fpsDenominator = 0;
|
||||
type->GetGUID(MF_MT_SUBTYPE, &actualSubtype);
|
||||
MFGetAttributeSize(type.Get(), MF_MT_FRAME_SIZE, &actualWidth, &actualHeight);
|
||||
MFGetAttributeRatio(type.Get(), MF_MT_FRAME_RATE, &fpsNumerator, &fpsDenominator);
|
||||
debugLog(
|
||||
"nativeType[%lu] %ux%u subtype=%08lx fps=%u/%u",
|
||||
index,
|
||||
actualWidth,
|
||||
actualHeight,
|
||||
static_cast<unsigned long>(actualSubtype.Data1),
|
||||
fpsNumerator,
|
||||
fpsDenominator);
|
||||
}
|
||||
|
||||
if (mediaTypeMatches(type.Get(), width, height, subtype, minFps))
|
||||
{
|
||||
const HRESULT setHr = reader->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, type.Get());
|
||||
debugLog("SetCurrentMediaType input index=%lu hr=0x%08lx", index, static_cast<unsigned long>(setHr));
|
||||
return SUCCEEDED(setHr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool setOutputType(IMFSourceReader* reader, const GUID& subtype)
|
||||
{
|
||||
ComPtr<IMFMediaType> output;
|
||||
if (FAILED(MFCreateMediaType(&output)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
output->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
|
||||
output->SetGUID(MF_MT_SUBTYPE, subtype);
|
||||
const HRESULT hr = reader->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, output.Get());
|
||||
debugLog("SetCurrentMediaType output hr=0x%08lx", static_cast<unsigned long>(hr));
|
||||
return SUCCEEDED(hr);
|
||||
}
|
||||
|
||||
bool createSharedTexture(MimirMfGpuCapture& capture, const GUID& outputSubtype)
|
||||
{
|
||||
ComPtr<IMFMediaType> current;
|
||||
if (FAILED(capture.reader->GetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, ¤t)))
|
||||
{
|
||||
debugLog("GetCurrentMediaType output failed");
|
||||
return false;
|
||||
}
|
||||
|
||||
UINT32 width = 0;
|
||||
UINT32 height = 0;
|
||||
GUID subtype{};
|
||||
if (FAILED(MFGetAttributeSize(current.Get(), MF_MT_FRAME_SIZE, &width, &height)) ||
|
||||
FAILED(current->GetGUID(MF_MT_SUBTYPE, &subtype)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto format = dxgiFormatForSubtype(subtype);
|
||||
if (format == DXGI_FORMAT_UNKNOWN || subtype != outputSubtype)
|
||||
{
|
||||
debugLog("output texture subtype/format unsupported subtype=%08lx", static_cast<unsigned long>(subtype.Data1));
|
||||
return false;
|
||||
}
|
||||
|
||||
D3D11_TEXTURE2D_DESC desc{};
|
||||
desc.Width = width;
|
||||
desc.Height = height;
|
||||
desc.MipLevels = 1;
|
||||
desc.ArraySize = 1;
|
||||
desc.Format = format;
|
||||
desc.SampleDesc.Count = 1;
|
||||
desc.Usage = D3D11_USAGE_DEFAULT;
|
||||
desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
|
||||
desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED_NTHANDLE | D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX;
|
||||
|
||||
const HRESULT textureHr = capture.device->CreateTexture2D(&desc, nullptr, &capture.sharedTexture);
|
||||
if (FAILED(textureHr))
|
||||
{
|
||||
debugLog("CreateTexture2D shared NT failed hr=0x%08lx; retrying legacy shared", static_cast<unsigned long>(textureHr));
|
||||
desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED;
|
||||
const HRESULT legacyTextureHr = capture.device->CreateTexture2D(&desc, nullptr, &capture.sharedTexture);
|
||||
if (FAILED(legacyTextureHr))
|
||||
{
|
||||
debugLog("CreateTexture2D legacy shared failed hr=0x%08lx", static_cast<unsigned long>(legacyTextureHr));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
ComPtr<IDXGIResource1> dxgiResource;
|
||||
const HRESULT asHr = capture.sharedTexture.As(&dxgiResource);
|
||||
const HRESULT handleHr = SUCCEEDED(asHr)
|
||||
? dxgiResource->CreateSharedHandle(nullptr, DXGI_SHARED_RESOURCE_READ | DXGI_SHARED_RESOURCE_WRITE, nullptr, &capture.sharedHandle)
|
||||
: asHr;
|
||||
if (FAILED(asHr) || FAILED(handleHr))
|
||||
{
|
||||
ComPtr<IDXGIResource> legacyResource;
|
||||
const HRESULT legacyAsHr = capture.sharedTexture.As(&legacyResource);
|
||||
const HRESULT legacyHandleHr = SUCCEEDED(legacyAsHr)
|
||||
? legacyResource->GetSharedHandle(&capture.sharedHandle)
|
||||
: legacyAsHr;
|
||||
if (FAILED(legacyAsHr) || FAILED(legacyHandleHr))
|
||||
{
|
||||
debugLog(
|
||||
"CreateSharedHandle failed as=0x%08lx handle=0x%08lx legacyAs=0x%08lx legacyHandle=0x%08lx",
|
||||
static_cast<unsigned long>(asHr),
|
||||
static_cast<unsigned long>(handleHr),
|
||||
static_cast<unsigned long>(legacyAsHr),
|
||||
static_cast<unsigned long>(legacyHandleHr));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
capture.width = static_cast<int>(width);
|
||||
capture.height = static_cast<int>(height);
|
||||
capture.format = format;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool createProducerFence(MimirMfGpuCapture& capture)
|
||||
{
|
||||
ComPtr<ID3D11Device5> device5;
|
||||
if (FAILED(capture.device.As(&device5)) ||
|
||||
FAILED(capture.context.As(&capture.fenceContext)))
|
||||
{
|
||||
debugLog("D3D11 fence interfaces unavailable; shared texture will be unsynchronized");
|
||||
return true;
|
||||
}
|
||||
|
||||
const HRESULT fenceHr = device5->CreateFence(
|
||||
0,
|
||||
D3D11_FENCE_FLAG_SHARED,
|
||||
IID_PPV_ARGS(&capture.producerFence));
|
||||
if (FAILED(fenceHr))
|
||||
{
|
||||
debugLog("CreateFence failed hr=0x%08lx; shared texture will be unsynchronized", static_cast<unsigned long>(fenceHr));
|
||||
capture.fenceContext.Reset();
|
||||
return true;
|
||||
}
|
||||
|
||||
const HRESULT handleHr = capture.producerFence->CreateSharedHandle(nullptr, GENERIC_ALL, nullptr, &capture.producerFenceHandle);
|
||||
if (FAILED(handleHr))
|
||||
{
|
||||
debugLog("CreateSharedHandle fence failed hr=0x%08lx; shared texture will be unsynchronized", static_cast<unsigned long>(handleHr));
|
||||
capture.producerFence.Reset();
|
||||
capture.fenceContext.Reset();
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
extern "C"
|
||||
{
|
||||
__declspec(dllexport) MimirMfGpuCapture* mimir_mf_gpu_create(
|
||||
const char* pathNeedle,
|
||||
int width,
|
||||
int height,
|
||||
const char* inputSubtypeText,
|
||||
const char* outputSubtypeText,
|
||||
double minFps)
|
||||
{
|
||||
CoInitializeEx(nullptr, COINIT_MULTITHREADED);
|
||||
if (FAILED(MFStartup(MF_VERSION, MFSTARTUP_NOSOCKET)))
|
||||
{
|
||||
debugLog("MFStartup failed");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto capture = new MimirMfGpuCapture();
|
||||
const UINT deviceFlags = D3D11_CREATE_DEVICE_VIDEO_SUPPORT | D3D11_CREATE_DEVICE_BGRA_SUPPORT;
|
||||
D3D_FEATURE_LEVEL featureLevels[] = { D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0 };
|
||||
D3D_FEATURE_LEVEL featureLevel{};
|
||||
const HRESULT deviceHr = D3D11CreateDevice(
|
||||
nullptr,
|
||||
D3D_DRIVER_TYPE_HARDWARE,
|
||||
nullptr,
|
||||
deviceFlags,
|
||||
featureLevels,
|
||||
ARRAYSIZE(featureLevels),
|
||||
D3D11_SDK_VERSION,
|
||||
&capture->device,
|
||||
&featureLevel,
|
||||
&capture->context);
|
||||
if (FAILED(deviceHr))
|
||||
{
|
||||
debugLog("D3D11CreateDevice failed hr=0x%08lx", static_cast<unsigned long>(deviceHr));
|
||||
delete capture;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
ComPtr<ID3D10Multithread> multithread;
|
||||
if (SUCCEEDED(capture->device.As(&multithread)))
|
||||
{
|
||||
multithread->SetMultithreadProtected(TRUE);
|
||||
}
|
||||
|
||||
UINT resetToken = 0;
|
||||
if (FAILED(MFCreateDXGIDeviceManager(&resetToken, &capture->deviceManager)) ||
|
||||
FAILED(capture->deviceManager->ResetDevice(capture->device.Get(), resetToken)))
|
||||
{
|
||||
debugLog("DXGI device manager setup failed");
|
||||
delete capture;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
const auto needle = widen(pathNeedle);
|
||||
if (!findVideoSource(needle.c_str(), &capture->source))
|
||||
{
|
||||
debugLog("no source matched needle");
|
||||
delete capture;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
ComPtr<IMFAttributes> readerAttributes;
|
||||
if (FAILED(MFCreateAttributes(&readerAttributes, 4)))
|
||||
{
|
||||
debugLog("MFCreateAttributes reader failed");
|
||||
delete capture;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
readerAttributes->SetUnknown(MF_SOURCE_READER_D3D_MANAGER, capture->deviceManager.Get());
|
||||
readerAttributes->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, TRUE);
|
||||
readerAttributes->SetUINT32(MF_LOW_LATENCY, TRUE);
|
||||
readerAttributes->SetUINT32(MF_SOURCE_READER_ENABLE_ADVANCED_VIDEO_PROCESSING, TRUE);
|
||||
|
||||
const HRESULT readerHr = MFCreateSourceReaderFromMediaSource(capture->source.Get(), readerAttributes.Get(), &capture->reader);
|
||||
if (FAILED(readerHr))
|
||||
{
|
||||
debugLog("MFCreateSourceReaderFromMediaSource failed hr=0x%08lx", static_cast<unsigned long>(readerHr));
|
||||
delete capture;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
const auto inputSubtype = subtypeFromText(inputSubtypeText);
|
||||
const auto outputSubtype = subtypeFromText(outputSubtypeText && outputSubtypeText[0] ? outputSubtypeText : "NV12");
|
||||
if (inputSubtype == GUID_NULL ||
|
||||
outputSubtype == GUID_NULL ||
|
||||
!setInputType(capture->reader.Get(), width, height, inputSubtype, minFps) ||
|
||||
!setOutputType(capture->reader.Get(), outputSubtype) ||
|
||||
!createSharedTexture(*capture, outputSubtype) ||
|
||||
!createProducerFence(*capture))
|
||||
{
|
||||
debugLog("create failed input=%s output=%s", inputSubtypeText ? inputSubtypeText : "", outputSubtypeText ? outputSubtypeText : "");
|
||||
delete capture;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return capture;
|
||||
}
|
||||
|
||||
__declspec(dllexport) int mimir_mf_gpu_read(
|
||||
MimirMfGpuCapture* capture,
|
||||
HANDLE* sharedHandle,
|
||||
int* width,
|
||||
int* height,
|
||||
char* format,
|
||||
int formatCapacity,
|
||||
std::int64_t* timestampNs,
|
||||
std::uint64_t* sequence,
|
||||
HANDLE* producerFenceHandle)
|
||||
{
|
||||
if (!capture || !capture->reader || !capture->sharedTexture || !sharedHandle)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
DWORD streamIndex = 0;
|
||||
DWORD flags = 0;
|
||||
LONGLONG timestamp = 0;
|
||||
ComPtr<IMFSample> sample;
|
||||
const HRESULT hr = capture->reader->ReadSample(
|
||||
MF_SOURCE_READER_FIRST_VIDEO_STREAM,
|
||||
0,
|
||||
&streamIndex,
|
||||
&flags,
|
||||
×tamp,
|
||||
&sample);
|
||||
if (FAILED(hr) || !sample || (flags & MF_SOURCE_READERF_STREAMTICK) != 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
ComPtr<IMFMediaBuffer> buffer;
|
||||
ComPtr<IMFDXGIBuffer> dxgiBuffer;
|
||||
ComPtr<ID3D11Texture2D> decodedTexture;
|
||||
UINT subresource = 0;
|
||||
if (FAILED(sample->GetBufferByIndex(0, &buffer)) ||
|
||||
FAILED(buffer.As(&dxgiBuffer)) ||
|
||||
FAILED(dxgiBuffer->GetResource(IID_PPV_ARGS(&decodedTexture))) ||
|
||||
FAILED(dxgiBuffer->GetSubresourceIndex(&subresource)))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
const auto nextSequence = capture->sequence + 1;
|
||||
capture->context->CopySubresourceRegion(capture->sharedTexture.Get(), 0, 0, 0, 0, decodedTexture.Get(), subresource, nullptr);
|
||||
if (capture->producerFence && capture->fenceContext)
|
||||
{
|
||||
capture->fenceContext->Signal(capture->producerFence.Get(), nextSequence);
|
||||
}
|
||||
|
||||
capture->context->Flush();
|
||||
*sharedHandle = capture->sharedHandle;
|
||||
if (width) *width = capture->width;
|
||||
if (height) *height = capture->height;
|
||||
if (format && formatCapacity > 0)
|
||||
{
|
||||
const char* text = capture->format == DXGI_FORMAT_NV12
|
||||
? textForSubtype(MFVideoFormat_NV12)
|
||||
: capture->format == DXGI_FORMAT_B8G8R8A8_UNORM
|
||||
? textForSubtype(MFVideoFormat_RGB32)
|
||||
: textForSubtype(MFVideoFormat_YUY2);
|
||||
std::snprintf(format, static_cast<std::size_t>(formatCapacity), "%s", text);
|
||||
}
|
||||
if (timestampNs) *timestampNs = timestamp * 100;
|
||||
capture->sequence = nextSequence;
|
||||
if (sequence) *sequence = capture->sequence;
|
||||
if (producerFenceHandle) *producerFenceHandle = capture->producerFenceHandle;
|
||||
return 1;
|
||||
}
|
||||
|
||||
__declspec(dllexport) void mimir_mf_gpu_destroy(MimirMfGpuCapture* capture)
|
||||
{
|
||||
if (!capture)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (capture->sharedHandle)
|
||||
{
|
||||
CloseHandle(capture->sharedHandle);
|
||||
}
|
||||
if (capture->producerFenceHandle)
|
||||
{
|
||||
CloseHandle(capture->producerFenceHandle);
|
||||
}
|
||||
if (capture->source)
|
||||
{
|
||||
capture->source->Shutdown();
|
||||
}
|
||||
delete capture;
|
||||
MFShutdown();
|
||||
CoUninitialize();
|
||||
}
|
||||
}
|
||||
215
native/camera_capture/mimir_ps3eye_capture.cpp
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <condition_variable>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <deque>
|
||||
#include <mutex>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#include "ps3eye_capi.h"
|
||||
|
||||
namespace
|
||||
{
|
||||
struct MimirPs3EyeFrame
|
||||
{
|
||||
std::vector<unsigned char> data;
|
||||
std::int64_t timestampNs = 0;
|
||||
std::uint64_t sequence = 0;
|
||||
};
|
||||
|
||||
struct MimirPs3EyeCapture
|
||||
{
|
||||
ps3eye_t* eye = nullptr;
|
||||
std::thread worker;
|
||||
std::mutex mutex;
|
||||
std::condition_variable condition;
|
||||
std::deque<MimirPs3EyeFrame> queue;
|
||||
std::atomic<bool> running = false;
|
||||
int cameraIndex = 0;
|
||||
int width = 0;
|
||||
int height = 0;
|
||||
int fps = 0;
|
||||
std::uint64_t sequence = 0;
|
||||
};
|
||||
|
||||
std::mutex g_ps3eye_mutex;
|
||||
int g_ps3eye_users = 0;
|
||||
|
||||
long long nowNs()
|
||||
{
|
||||
return std::chrono::duration_cast<std::chrono::nanoseconds>(
|
||||
std::chrono::steady_clock::now().time_since_epoch()).count();
|
||||
}
|
||||
|
||||
void retainPs3Eye()
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(g_ps3eye_mutex);
|
||||
if (g_ps3eye_users++ == 0)
|
||||
{
|
||||
ps3eye_init();
|
||||
}
|
||||
}
|
||||
|
||||
void releasePs3Eye()
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(g_ps3eye_mutex);
|
||||
if (g_ps3eye_users > 0 && --g_ps3eye_users == 0)
|
||||
{
|
||||
ps3eye_uninit();
|
||||
}
|
||||
}
|
||||
|
||||
void pushFrame(MimirPs3EyeCapture& capture, const std::vector<unsigned char>& source)
|
||||
{
|
||||
MimirPs3EyeFrame frame;
|
||||
frame.data = source;
|
||||
frame.timestampNs = nowNs();
|
||||
frame.sequence = capture.sequence++;
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(capture.mutex);
|
||||
capture.queue.push_back(std::move(frame));
|
||||
while (capture.queue.size() > 32)
|
||||
{
|
||||
capture.queue.pop_front();
|
||||
}
|
||||
}
|
||||
|
||||
capture.condition.notify_one();
|
||||
}
|
||||
|
||||
void captureThread(MimirPs3EyeCapture* capture)
|
||||
{
|
||||
std::vector<unsigned char> frame(static_cast<std::size_t>(capture->width) * static_cast<std::size_t>(capture->height));
|
||||
while (capture->running.load(std::memory_order_acquire))
|
||||
{
|
||||
ps3eye_grab_frame(capture->eye, frame.data());
|
||||
pushFrame(*capture, frame);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
extern "C"
|
||||
{
|
||||
__declspec(dllexport) MimirPs3EyeCapture* mimir_ps3eye_create(int cameraIndex, int width, int height, int fps)
|
||||
{
|
||||
if (cameraIndex < 0 || width <= 0 || height <= 0 || fps <= 0)
|
||||
{
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
retainPs3Eye();
|
||||
if (cameraIndex >= ps3eye_count_connected())
|
||||
{
|
||||
releasePs3Eye();
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto* capture = new MimirPs3EyeCapture();
|
||||
capture->cameraIndex = cameraIndex;
|
||||
capture->width = width;
|
||||
capture->height = height;
|
||||
capture->fps = fps;
|
||||
capture->eye = ps3eye_open(cameraIndex, width, height, fps, PS3EYE_FORMAT_BAYER);
|
||||
if (!capture->eye)
|
||||
{
|
||||
delete capture;
|
||||
releasePs3Eye();
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return capture;
|
||||
}
|
||||
|
||||
__declspec(dllexport) int mimir_ps3eye_start(MimirPs3EyeCapture* capture)
|
||||
{
|
||||
if (!capture || capture->running.exchange(true))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
capture->worker = std::thread(captureThread, capture);
|
||||
return 1;
|
||||
}
|
||||
|
||||
__declspec(dllexport) int mimir_ps3eye_read(
|
||||
MimirPs3EyeCapture* capture,
|
||||
int* width,
|
||||
int* height,
|
||||
int* stride,
|
||||
std::int64_t* timestampNs,
|
||||
std::uint64_t* sequence,
|
||||
unsigned char* destination,
|
||||
int destinationBytes)
|
||||
{
|
||||
if (!capture || !destination || destinationBytes <= 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
MimirPs3EyeFrame frame;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(capture->mutex);
|
||||
if (capture->queue.empty())
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
frame = std::move(capture->queue.front());
|
||||
capture->queue.pop_front();
|
||||
}
|
||||
|
||||
if (frame.data.size() > static_cast<std::size_t>(destinationBytes))
|
||||
{
|
||||
return -static_cast<int>(frame.data.size());
|
||||
}
|
||||
|
||||
std::memcpy(destination, frame.data.data(), frame.data.size());
|
||||
if (width)
|
||||
{
|
||||
*width = capture->width;
|
||||
}
|
||||
if (height)
|
||||
{
|
||||
*height = capture->height;
|
||||
}
|
||||
if (stride)
|
||||
{
|
||||
*stride = capture->width;
|
||||
}
|
||||
if (timestampNs)
|
||||
{
|
||||
*timestampNs = frame.timestampNs;
|
||||
}
|
||||
if (sequence)
|
||||
{
|
||||
*sequence = frame.sequence;
|
||||
}
|
||||
|
||||
return static_cast<int>(frame.data.size());
|
||||
}
|
||||
|
||||
__declspec(dllexport) void mimir_ps3eye_destroy(MimirPs3EyeCapture* capture)
|
||||
{
|
||||
if (!capture)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
capture->running.store(false, std::memory_order_release);
|
||||
if (capture->worker.joinable())
|
||||
{
|
||||
capture->worker.join();
|
||||
}
|
||||
if (capture->eye)
|
||||
{
|
||||
ps3eye_close(capture->eye);
|
||||
}
|
||||
|
||||
delete capture;
|
||||
releasePs3Eye();
|
||||
}
|
||||
}
|
||||
31
native/obs_stem_source/CMakeLists.txt
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
cmake_minimum_required(VERSION 3.20)
|
||||
|
||||
project(mimir_obs_stem_source LANGUAGES CXX)
|
||||
|
||||
find_package(libobs REQUIRED CONFIG)
|
||||
include(CTest)
|
||||
|
||||
add_library(mimir_obs_stem_source MODULE
|
||||
mimir_obs_muninn_source.cpp
|
||||
mimir_obs_program_texture_source.cpp
|
||||
mimir_obs_stem_source.cpp)
|
||||
|
||||
target_compile_features(mimir_obs_stem_source PRIVATE cxx_std_20)
|
||||
target_compile_definitions(mimir_obs_stem_source PRIVATE
|
||||
WIN32_LEAN_AND_MEAN
|
||||
NOMINMAX
|
||||
_WIN32_WINNT=0x0A00)
|
||||
target_link_libraries(mimir_obs_stem_source PRIVATE OBS::libobs d3d11 d3d12 dxgi ws2_32)
|
||||
|
||||
add_executable(muninn_media_decoder_probe
|
||||
../../tools/muninn_media_decoder_probe.cpp)
|
||||
target_compile_features(muninn_media_decoder_probe PRIVATE cxx_std_20)
|
||||
target_include_directories(muninn_media_decoder_probe PRIVATE
|
||||
${CMAKE_CURRENT_SOURCE_DIR})
|
||||
|
||||
add_executable(muninn_rudp_url_tests
|
||||
tests/muninn_rudp_url_tests.cpp)
|
||||
target_compile_features(muninn_rudp_url_tests PRIVATE cxx_std_20)
|
||||
target_include_directories(muninn_rudp_url_tests PRIVATE
|
||||
${CMAKE_CURRENT_SOURCE_DIR})
|
||||
add_test(NAME muninn_rudp_url_tests COMMAND muninn_rudp_url_tests)
|
||||
14
native/obs_stem_source/data/locale/en-US.ini
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
MimirAudioStem="Mimir Audio Stem"
|
||||
MimirAudioStem.MapName="Shared memory map"
|
||||
MimirAudioStem.StemId="Stem id"
|
||||
MimirProgramTexture="Mimir Program Texture"
|
||||
MuninnStream="Muninn Stream"
|
||||
MuninnVideoSource="Muninn Video"
|
||||
MuninnAudioSource="Muninn Audio"
|
||||
MuninnStream.Stream="Muninn stream"
|
||||
MuninnStream.Video="Video device"
|
||||
MuninnStream.Audio="Audio device"
|
||||
MuninnStream.VideoBitrateKbps="Video bitrate (kbps)"
|
||||
MuninnStream.VideoLatencyMs="Video/transport latency (ms)"
|
||||
MuninnStream.AudioReorderMs="Audio reorder latency (ms)"
|
||||
MuninnStream.Refresh="Refresh Muninn streams"
|
||||
3982
native/obs_stem_source/mimir_obs_muninn_source.cpp
Normal file
628
native/obs_stem_source/mimir_obs_program_texture_source.cpp
Normal file
|
|
@ -0,0 +1,628 @@
|
|||
#include <obs-module.h>
|
||||
#include <util/platform.h>
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include <d3d11_1.h>
|
||||
#include <d3d12.h>
|
||||
#include <dxgi1_2.h>
|
||||
#include <wrl/client.h>
|
||||
#include <windows.h>
|
||||
|
||||
using Microsoft::WRL::ComPtr;
|
||||
|
||||
namespace {
|
||||
|
||||
struct MimirProgramTextureSource {
|
||||
obs_source_t *source = nullptr;
|
||||
std::string texture_name = "Global\\MimirFensalirProgramTexture";
|
||||
std::string fence_name = "Global\\MimirFensalirProgramFence";
|
||||
std::string consumer_fence_name = "Global\\MimirObsProgramConsumerFence";
|
||||
uint32_t ring_count = 1;
|
||||
uint32_t width = 0;
|
||||
uint32_t height = 0;
|
||||
uint64_t fence_value = 0;
|
||||
uint64_t last_producer_fence_value = 0;
|
||||
uint64_t next_retry_ns = 0;
|
||||
uint64_t next_failure_log_ns = 0;
|
||||
HANDLE bridge_shared_handle = nullptr;
|
||||
HANDLE consumer_fence_shared_handle = nullptr;
|
||||
HANDLE fence_event = nullptr;
|
||||
gs_texture_t *obs_texture = nullptr;
|
||||
ComPtr<ID3D12Device> d3d12_device;
|
||||
ComPtr<ID3D12CommandQueue> command_queue;
|
||||
ComPtr<ID3D12CommandAllocator> command_allocator;
|
||||
ComPtr<ID3D12GraphicsCommandList> command_list;
|
||||
ComPtr<ID3D12Fence> fence;
|
||||
ComPtr<ID3D12Fence> producer_fence;
|
||||
ComPtr<ID3D12Fence> consumer_fence;
|
||||
std::vector<ComPtr<ID3D12Resource>> source_textures;
|
||||
ComPtr<ID3D12Resource> bridge_texture_d3d12;
|
||||
ComPtr<ID3D11Device> d3d11_device;
|
||||
ComPtr<ID3D11DeviceContext> d3d11_context;
|
||||
ComPtr<ID3D11Texture2D> bridge_texture_d3d11;
|
||||
};
|
||||
|
||||
static void log_retry_failure(MimirProgramTextureSource *ctx, const char *message, HRESULT hr = S_OK)
|
||||
{
|
||||
const uint64_t now = os_gettime_ns();
|
||||
if (now < ctx->next_failure_log_ns) {
|
||||
return;
|
||||
}
|
||||
|
||||
ctx->next_failure_log_ns = now + 2000000000ULL;
|
||||
if (FAILED(hr)) {
|
||||
blog(LOG_WARNING, "Mimir OBS program texture: %s: 0x%08lx", message, static_cast<unsigned long>(hr));
|
||||
} else {
|
||||
blog(LOG_WARNING, "Mimir OBS program texture: %s", message);
|
||||
}
|
||||
}
|
||||
|
||||
static std::wstring utf8_to_wide(const std::string &value)
|
||||
{
|
||||
if (value.empty()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const int count = MultiByteToWideChar(CP_UTF8, 0, value.c_str(), -1, nullptr, 0);
|
||||
if (count <= 0) {
|
||||
return {};
|
||||
}
|
||||
|
||||
std::wstring wide(static_cast<size_t>(count), L'\0');
|
||||
MultiByteToWideChar(CP_UTF8, 0, value.c_str(), -1, wide.data(), count);
|
||||
if (!wide.empty() && wide.back() == L'\0') {
|
||||
wide.pop_back();
|
||||
}
|
||||
|
||||
return wide;
|
||||
}
|
||||
|
||||
static std::string obs_string(obs_data_t *settings, const char *key, const char *fallback)
|
||||
{
|
||||
const char *value = obs_data_get_string(settings, key);
|
||||
return value && *value ? std::string(value) : std::string(fallback);
|
||||
}
|
||||
|
||||
static uint32_t clamp_ring_count(long long value)
|
||||
{
|
||||
if (value < 1) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (value > 4) {
|
||||
return 4;
|
||||
}
|
||||
|
||||
return static_cast<uint32_t>(value);
|
||||
}
|
||||
|
||||
static std::string texture_name_for_slot(const MimirProgramTextureSource *ctx, uint32_t slot)
|
||||
{
|
||||
if (ctx->ring_count <= 1) {
|
||||
return ctx->texture_name;
|
||||
}
|
||||
|
||||
return ctx->texture_name + "." + std::to_string(slot);
|
||||
}
|
||||
|
||||
static void release_bridge(MimirProgramTextureSource *ctx)
|
||||
{
|
||||
if (ctx->obs_texture) {
|
||||
gs_texture_destroy(ctx->obs_texture);
|
||||
ctx->obs_texture = nullptr;
|
||||
}
|
||||
|
||||
ctx->bridge_texture_d3d12.Reset();
|
||||
ctx->bridge_texture_d3d11.Reset();
|
||||
ctx->source_textures.clear();
|
||||
ctx->fence.Reset();
|
||||
ctx->producer_fence.Reset();
|
||||
ctx->consumer_fence.Reset();
|
||||
ctx->command_list.Reset();
|
||||
ctx->command_allocator.Reset();
|
||||
ctx->command_queue.Reset();
|
||||
ctx->d3d12_device.Reset();
|
||||
ctx->d3d11_context.Reset();
|
||||
ctx->d3d11_device.Reset();
|
||||
ctx->width = 0;
|
||||
ctx->height = 0;
|
||||
ctx->fence_value = 0;
|
||||
ctx->last_producer_fence_value = 0;
|
||||
|
||||
if (ctx->bridge_shared_handle) {
|
||||
CloseHandle(ctx->bridge_shared_handle);
|
||||
ctx->bridge_shared_handle = nullptr;
|
||||
}
|
||||
|
||||
if (ctx->consumer_fence_shared_handle) {
|
||||
CloseHandle(ctx->consumer_fence_shared_handle);
|
||||
ctx->consumer_fence_shared_handle = nullptr;
|
||||
}
|
||||
|
||||
if (ctx->fence_event) {
|
||||
CloseHandle(ctx->fence_event);
|
||||
ctx->fence_event = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
static bool wait_for_copy(MimirProgramTextureSource *ctx, uint64_t consumed_producer_value)
|
||||
{
|
||||
if (ctx->consumer_fence && consumed_producer_value > 0) {
|
||||
HRESULT consumer_hr = ctx->command_queue->Signal(ctx->consumer_fence.Get(), consumed_producer_value);
|
||||
if (FAILED(consumer_hr)) {
|
||||
blog(LOG_WARNING,
|
||||
"Mimir OBS program texture: consumer fence Signal failed: 0x%08lx",
|
||||
static_cast<unsigned long>(consumer_hr));
|
||||
}
|
||||
}
|
||||
|
||||
const uint64_t value = ++ctx->fence_value;
|
||||
HRESULT hr = ctx->command_queue->Signal(ctx->fence.Get(), value);
|
||||
if (FAILED(hr)) {
|
||||
blog(LOG_WARNING, "Mimir OBS program texture: D3D12 Signal failed: 0x%08lx", static_cast<unsigned long>(hr));
|
||||
return false;
|
||||
}
|
||||
|
||||
if (ctx->fence->GetCompletedValue() >= value) {
|
||||
return true;
|
||||
}
|
||||
|
||||
hr = ctx->fence->SetEventOnCompletion(value, ctx->fence_event);
|
||||
if (FAILED(hr)) {
|
||||
blog(LOG_WARNING, "Mimir OBS program texture: SetEventOnCompletion failed: 0x%08lx", static_cast<unsigned long>(hr));
|
||||
return false;
|
||||
}
|
||||
|
||||
WaitForSingleObject(ctx->fence_event, 100);
|
||||
return ctx->fence->GetCompletedValue() >= value;
|
||||
}
|
||||
|
||||
static bool create_devices(MimirProgramTextureSource *ctx)
|
||||
{
|
||||
HRESULT hr = D3D12CreateDevice(nullptr, D3D_FEATURE_LEVEL_11_0, IID_PPV_ARGS(&ctx->d3d12_device));
|
||||
if (FAILED(hr)) {
|
||||
blog(LOG_WARNING, "Mimir OBS program texture: D3D12CreateDevice failed: 0x%08lx", static_cast<unsigned long>(hr));
|
||||
return false;
|
||||
}
|
||||
|
||||
D3D12_COMMAND_QUEUE_DESC queue_desc = {};
|
||||
queue_desc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT;
|
||||
hr = ctx->d3d12_device->CreateCommandQueue(&queue_desc, IID_PPV_ARGS(&ctx->command_queue));
|
||||
if (FAILED(hr)) {
|
||||
blog(LOG_WARNING, "Mimir OBS program texture: CreateCommandQueue failed: 0x%08lx", static_cast<unsigned long>(hr));
|
||||
return false;
|
||||
}
|
||||
|
||||
hr = ctx->d3d12_device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(&ctx->command_allocator));
|
||||
if (FAILED(hr)) {
|
||||
blog(LOG_WARNING, "Mimir OBS program texture: CreateCommandAllocator failed: 0x%08lx", static_cast<unsigned long>(hr));
|
||||
return false;
|
||||
}
|
||||
|
||||
hr = ctx->d3d12_device->CreateCommandList(
|
||||
0,
|
||||
D3D12_COMMAND_LIST_TYPE_DIRECT,
|
||||
ctx->command_allocator.Get(),
|
||||
nullptr,
|
||||
IID_PPV_ARGS(&ctx->command_list));
|
||||
if (FAILED(hr)) {
|
||||
blog(LOG_WARNING, "Mimir OBS program texture: CreateCommandList failed: 0x%08lx", static_cast<unsigned long>(hr));
|
||||
return false;
|
||||
}
|
||||
|
||||
ctx->command_list->Close();
|
||||
|
||||
hr = ctx->d3d12_device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(&ctx->fence));
|
||||
if (FAILED(hr)) {
|
||||
blog(LOG_WARNING, "Mimir OBS program texture: CreateFence failed: 0x%08lx", static_cast<unsigned long>(hr));
|
||||
return false;
|
||||
}
|
||||
|
||||
ctx->fence_event = CreateEventW(nullptr, FALSE, FALSE, nullptr);
|
||||
if (!ctx->fence_event) {
|
||||
blog(LOG_WARNING, "Mimir OBS program texture: CreateEvent failed: %lu", GetLastError());
|
||||
return false;
|
||||
}
|
||||
|
||||
UINT flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT;
|
||||
D3D_FEATURE_LEVEL feature_levels[] = {D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0};
|
||||
D3D_FEATURE_LEVEL selected = D3D_FEATURE_LEVEL_11_0;
|
||||
hr = D3D11CreateDevice(
|
||||
nullptr,
|
||||
D3D_DRIVER_TYPE_HARDWARE,
|
||||
nullptr,
|
||||
flags,
|
||||
feature_levels,
|
||||
2,
|
||||
D3D11_SDK_VERSION,
|
||||
&ctx->d3d11_device,
|
||||
&selected,
|
||||
&ctx->d3d11_context);
|
||||
if (FAILED(hr)) {
|
||||
blog(LOG_WARNING, "Mimir OBS program texture: D3D11CreateDevice failed: 0x%08lx", static_cast<unsigned long>(hr));
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static void create_consumer_fence(MimirProgramTextureSource *ctx)
|
||||
{
|
||||
if (ctx->consumer_fence_name.empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
HRESULT hr = ctx->d3d12_device->CreateFence(0, D3D12_FENCE_FLAG_SHARED, IID_PPV_ARGS(&ctx->consumer_fence));
|
||||
if (FAILED(hr)) {
|
||||
log_retry_failure(ctx, "CreateFence consumer fence failed", hr);
|
||||
return;
|
||||
}
|
||||
|
||||
const auto wide_name = utf8_to_wide(ctx->consumer_fence_name);
|
||||
if (wide_name.empty()) {
|
||||
log_retry_failure(ctx, "consumer fence name is empty or invalid UTF-8");
|
||||
ctx->consumer_fence.Reset();
|
||||
return;
|
||||
}
|
||||
|
||||
hr = ctx->d3d12_device->CreateSharedHandle(
|
||||
ctx->consumer_fence.Get(),
|
||||
nullptr,
|
||||
GENERIC_ALL,
|
||||
wide_name.c_str(),
|
||||
&ctx->consumer_fence_shared_handle);
|
||||
if (FAILED(hr)) {
|
||||
log_retry_failure(ctx, "CreateSharedHandle consumer fence failed", hr);
|
||||
ctx->consumer_fence.Reset();
|
||||
return;
|
||||
}
|
||||
|
||||
blog(LOG_INFO, "Mimir OBS program texture consumer fence published %s", ctx->consumer_fence_name.c_str());
|
||||
}
|
||||
|
||||
static bool open_source_texture(MimirProgramTextureSource *ctx)
|
||||
{
|
||||
ctx->source_textures.clear();
|
||||
ctx->source_textures.reserve(ctx->ring_count);
|
||||
for (uint32_t slot = 0; slot < ctx->ring_count; ++slot) {
|
||||
const auto slot_name = texture_name_for_slot(ctx, slot);
|
||||
const auto wide_name = utf8_to_wide(slot_name);
|
||||
if (wide_name.empty()) {
|
||||
log_retry_failure(ctx, "shared texture name is empty or invalid UTF-8");
|
||||
return false;
|
||||
}
|
||||
|
||||
HANDLE source_handle = nullptr;
|
||||
HRESULT hr = ctx->d3d12_device->OpenSharedHandleByName(wide_name.c_str(), GENERIC_ALL, &source_handle);
|
||||
if (FAILED(hr)) {
|
||||
log_retry_failure(ctx, "OpenSharedHandleByName failed", hr);
|
||||
return false;
|
||||
}
|
||||
|
||||
ComPtr<ID3D12Resource> source_texture;
|
||||
hr = ctx->d3d12_device->OpenSharedHandle(source_handle, IID_PPV_ARGS(&source_texture));
|
||||
CloseHandle(source_handle);
|
||||
if (FAILED(hr)) {
|
||||
log_retry_failure(ctx, "OpenSharedHandle failed", hr);
|
||||
return false;
|
||||
}
|
||||
|
||||
const D3D12_RESOURCE_DESC desc = source_texture->GetDesc();
|
||||
if (desc.Dimension != D3D12_RESOURCE_DIMENSION_TEXTURE2D || desc.Format != DXGI_FORMAT_B8G8R8A8_UNORM) {
|
||||
log_retry_failure(ctx, "expected BGRA8 D3D12 Texture2D");
|
||||
return false;
|
||||
}
|
||||
|
||||
const uint32_t slot_width = static_cast<uint32_t>(desc.Width);
|
||||
const uint32_t slot_height = desc.Height;
|
||||
if (slot_width == 0 || slot_height == 0) {
|
||||
log_retry_failure(ctx, "shared texture dimensions are empty");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (slot == 0) {
|
||||
ctx->width = slot_width;
|
||||
ctx->height = slot_height;
|
||||
} else if (slot_width != ctx->width || slot_height != ctx->height) {
|
||||
log_retry_failure(ctx, "program output ring texture dimensions do not match");
|
||||
return false;
|
||||
}
|
||||
|
||||
ctx->source_textures.push_back(std::move(source_texture));
|
||||
}
|
||||
|
||||
return ctx->source_textures.size() == ctx->ring_count;
|
||||
}
|
||||
|
||||
static bool open_producer_fence(MimirProgramTextureSource *ctx)
|
||||
{
|
||||
const auto wide_name = utf8_to_wide(ctx->fence_name);
|
||||
if (wide_name.empty()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
HANDLE fence_handle = nullptr;
|
||||
HRESULT hr = ctx->d3d12_device->OpenSharedHandleByName(wide_name.c_str(), GENERIC_ALL, &fence_handle);
|
||||
if (FAILED(hr)) {
|
||||
log_retry_failure(ctx, "OpenSharedHandleByName producer fence failed", hr);
|
||||
return true;
|
||||
}
|
||||
|
||||
hr = ctx->d3d12_device->OpenSharedHandle(fence_handle, IID_PPV_ARGS(&ctx->producer_fence));
|
||||
CloseHandle(fence_handle);
|
||||
if (FAILED(hr)) {
|
||||
log_retry_failure(ctx, "OpenSharedHandle producer fence failed", hr);
|
||||
ctx->producer_fence.Reset();
|
||||
return true;
|
||||
}
|
||||
|
||||
ctx->last_producer_fence_value = ctx->producer_fence->GetCompletedValue();
|
||||
if (ctx->last_producer_fence_value == UINT64_MAX) {
|
||||
log_retry_failure(ctx, "producer fence opened in device-removed state");
|
||||
ctx->producer_fence.Reset();
|
||||
return false;
|
||||
}
|
||||
|
||||
blog(LOG_INFO,
|
||||
"Mimir OBS program texture producer fence opened %s value=%llu",
|
||||
ctx->fence_name.c_str(),
|
||||
static_cast<unsigned long long>(ctx->last_producer_fence_value));
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool read_producer_fence(MimirProgramTextureSource *ctx, uint64_t *completed)
|
||||
{
|
||||
*completed = 0;
|
||||
if (!ctx->producer_fence) {
|
||||
return true;
|
||||
}
|
||||
|
||||
*completed = ctx->producer_fence->GetCompletedValue();
|
||||
if (*completed == UINT64_MAX) {
|
||||
log_retry_failure(ctx, "producer fence reported device removal");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (*completed > ctx->last_producer_fence_value) {
|
||||
ctx->last_producer_fence_value = *completed;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool create_obs_bridge_texture(MimirProgramTextureSource *ctx)
|
||||
{
|
||||
D3D11_TEXTURE2D_DESC desc = {};
|
||||
desc.Width = ctx->width;
|
||||
desc.Height = ctx->height;
|
||||
desc.MipLevels = 1;
|
||||
desc.ArraySize = 1;
|
||||
desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM;
|
||||
desc.SampleDesc.Count = 1;
|
||||
desc.Usage = D3D11_USAGE_DEFAULT;
|
||||
desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
|
||||
desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED;
|
||||
|
||||
HRESULT hr = ctx->d3d11_device->CreateTexture2D(&desc, nullptr, &ctx->bridge_texture_d3d11);
|
||||
if (FAILED(hr)) {
|
||||
blog(LOG_WARNING, "Mimir OBS program texture: CreateTexture2D bridge failed: 0x%08lx", static_cast<unsigned long>(hr));
|
||||
return false;
|
||||
}
|
||||
|
||||
ComPtr<IDXGIResource> legacy_resource;
|
||||
hr = ctx->bridge_texture_d3d11.As(&legacy_resource);
|
||||
if (FAILED(hr)) {
|
||||
blog(LOG_WARNING, "Mimir OBS program texture: bridge texture is not IDXGIResource: 0x%08lx", static_cast<unsigned long>(hr));
|
||||
return false;
|
||||
}
|
||||
|
||||
hr = legacy_resource->GetSharedHandle(&ctx->bridge_shared_handle);
|
||||
if (FAILED(hr) || !ctx->bridge_shared_handle) {
|
||||
blog(LOG_WARNING, "Mimir OBS program texture: GetSharedHandle bridge failed: 0x%08lx", static_cast<unsigned long>(hr));
|
||||
return false;
|
||||
}
|
||||
|
||||
hr = ctx->d3d12_device->OpenSharedHandle(ctx->bridge_shared_handle, IID_PPV_ARGS(&ctx->bridge_texture_d3d12));
|
||||
if (FAILED(hr)) {
|
||||
blog(LOG_WARNING, "Mimir OBS program texture: D3D12 open legacy bridge handle failed: 0x%08lx", static_cast<unsigned long>(hr));
|
||||
return false;
|
||||
}
|
||||
|
||||
ctx->obs_texture = gs_texture_open_shared(static_cast<uint32_t>(reinterpret_cast<uintptr_t>(ctx->bridge_shared_handle)));
|
||||
if (!ctx->obs_texture) {
|
||||
blog(LOG_WARNING, "Mimir OBS program texture: OBS failed to open legacy bridge shared texture");
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool ensure_bridge(MimirProgramTextureSource *ctx)
|
||||
{
|
||||
if (ctx->obs_texture) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const uint64_t now = os_gettime_ns();
|
||||
if (now < ctx->next_retry_ns) {
|
||||
return false;
|
||||
}
|
||||
|
||||
ctx->next_retry_ns = now + 1000000000ULL;
|
||||
release_bridge(ctx);
|
||||
|
||||
if (!create_devices(ctx)) {
|
||||
release_bridge(ctx);
|
||||
return false;
|
||||
}
|
||||
|
||||
create_consumer_fence(ctx);
|
||||
if (!open_source_texture(ctx) || !create_obs_bridge_texture(ctx)) {
|
||||
release_bridge(ctx);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!open_producer_fence(ctx)) {
|
||||
release_bridge(ctx);
|
||||
return false;
|
||||
}
|
||||
|
||||
blog(LOG_INFO, "Mimir OBS program texture source opened %s (%ux%u ring=%u)",
|
||||
ctx->texture_name.c_str(),
|
||||
ctx->width,
|
||||
ctx->height,
|
||||
ctx->ring_count);
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool copy_source_to_bridge(MimirProgramTextureSource *ctx)
|
||||
{
|
||||
HRESULT hr = ctx->command_allocator->Reset();
|
||||
if (FAILED(hr)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
uint64_t completed_producer_value = 0;
|
||||
if (!read_producer_fence(ctx, &completed_producer_value)) {
|
||||
release_bridge(ctx);
|
||||
return false;
|
||||
}
|
||||
|
||||
const uint32_t source_slot = ctx->source_textures.empty()
|
||||
? 0
|
||||
: static_cast<uint32_t>(completed_producer_value % ctx->source_textures.size());
|
||||
ID3D12Resource *source_texture = ctx->source_textures[source_slot].Get();
|
||||
|
||||
hr = ctx->command_list->Reset(ctx->command_allocator.Get(), nullptr);
|
||||
if (FAILED(hr)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
D3D12_RESOURCE_BARRIER barriers[2] = {};
|
||||
barriers[0].Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION;
|
||||
barriers[0].Transition.pResource = source_texture;
|
||||
barriers[0].Transition.StateBefore = D3D12_RESOURCE_STATE_COMMON;
|
||||
barriers[0].Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_SOURCE;
|
||||
barriers[0].Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES;
|
||||
barriers[1].Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION;
|
||||
barriers[1].Transition.pResource = ctx->bridge_texture_d3d12.Get();
|
||||
barriers[1].Transition.StateBefore = D3D12_RESOURCE_STATE_COMMON;
|
||||
barriers[1].Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_DEST;
|
||||
barriers[1].Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES;
|
||||
ctx->command_list->ResourceBarrier(2, barriers);
|
||||
ctx->command_list->CopyResource(ctx->bridge_texture_d3d12.Get(), source_texture);
|
||||
|
||||
std::swap(barriers[0].Transition.StateBefore, barriers[0].Transition.StateAfter);
|
||||
std::swap(barriers[1].Transition.StateBefore, barriers[1].Transition.StateAfter);
|
||||
ctx->command_list->ResourceBarrier(2, barriers);
|
||||
|
||||
hr = ctx->command_list->Close();
|
||||
if (FAILED(hr)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
ID3D12CommandList *lists[] = {ctx->command_list.Get()};
|
||||
ctx->command_queue->ExecuteCommandLists(1, lists);
|
||||
return wait_for_copy(ctx, completed_producer_value);
|
||||
}
|
||||
|
||||
static const char *mimir_program_texture_get_name(void *)
|
||||
{
|
||||
return obs_module_text("MimirProgramTexture");
|
||||
}
|
||||
|
||||
static void mimir_program_texture_update(void *data, obs_data_t *settings)
|
||||
{
|
||||
auto *ctx = static_cast<MimirProgramTextureSource *>(data);
|
||||
ctx->texture_name = obs_string(settings, "texture_name", "Global\\MimirFensalirProgramTexture");
|
||||
ctx->fence_name = obs_string(settings, "fence_name", "Global\\MimirFensalirProgramFence");
|
||||
ctx->consumer_fence_name = obs_string(settings, "consumer_fence_name", "Global\\MimirObsProgramConsumerFence");
|
||||
ctx->ring_count = clamp_ring_count(obs_data_get_int(settings, "ring_count"));
|
||||
release_bridge(ctx);
|
||||
}
|
||||
|
||||
static void *mimir_program_texture_create(obs_data_t *settings, obs_source_t *source)
|
||||
{
|
||||
auto *ctx = new MimirProgramTextureSource();
|
||||
ctx->source = source;
|
||||
mimir_program_texture_update(ctx, settings);
|
||||
return ctx;
|
||||
}
|
||||
|
||||
static void mimir_program_texture_destroy(void *data)
|
||||
{
|
||||
auto *ctx = static_cast<MimirProgramTextureSource *>(data);
|
||||
release_bridge(ctx);
|
||||
delete ctx;
|
||||
}
|
||||
|
||||
static uint32_t mimir_program_texture_width(void *data)
|
||||
{
|
||||
auto *ctx = static_cast<MimirProgramTextureSource *>(data);
|
||||
ensure_bridge(ctx);
|
||||
return ctx->width;
|
||||
}
|
||||
|
||||
static uint32_t mimir_program_texture_height(void *data)
|
||||
{
|
||||
auto *ctx = static_cast<MimirProgramTextureSource *>(data);
|
||||
ensure_bridge(ctx);
|
||||
return ctx->height;
|
||||
}
|
||||
|
||||
static void mimir_program_texture_defaults(obs_data_t *settings)
|
||||
{
|
||||
obs_data_set_default_string(settings, "texture_name", "Global\\MimirFensalirProgramTexture");
|
||||
obs_data_set_default_string(settings, "fence_name", "Global\\MimirFensalirProgramFence");
|
||||
obs_data_set_default_string(settings, "consumer_fence_name", "Global\\MimirObsProgramConsumerFence");
|
||||
obs_data_set_default_int(settings, "ring_count", 1);
|
||||
}
|
||||
|
||||
static obs_properties_t *mimir_program_texture_properties(void *)
|
||||
{
|
||||
obs_properties_t *props = obs_properties_create();
|
||||
obs_properties_add_text(props, "texture_name", "Shared D3D12 texture name", OBS_TEXT_DEFAULT);
|
||||
obs_properties_add_text(props, "fence_name", "Shared D3D12 fence name", OBS_TEXT_DEFAULT);
|
||||
obs_properties_add_text(props, "consumer_fence_name", "Consumer D3D12 fence name", OBS_TEXT_DEFAULT);
|
||||
obs_properties_add_int(props, "ring_count", "Texture ring slots", 1, 4, 1);
|
||||
return props;
|
||||
}
|
||||
|
||||
static void mimir_program_texture_render(void *data, gs_effect_t *)
|
||||
{
|
||||
auto *ctx = static_cast<MimirProgramTextureSource *>(data);
|
||||
if (!ensure_bridge(ctx)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (copy_source_to_bridge(ctx)) {
|
||||
obs_source_draw(ctx->obs_texture, 0, 0, ctx->width, ctx->height, false);
|
||||
}
|
||||
}
|
||||
|
||||
static obs_source_info mimir_program_texture_source_info = {
|
||||
.id = "mimir_program_texture_source",
|
||||
.type = OBS_SOURCE_TYPE_INPUT,
|
||||
.output_flags = OBS_SOURCE_VIDEO,
|
||||
.get_name = mimir_program_texture_get_name,
|
||||
.create = mimir_program_texture_create,
|
||||
.destroy = mimir_program_texture_destroy,
|
||||
.get_width = mimir_program_texture_width,
|
||||
.get_height = mimir_program_texture_height,
|
||||
.get_defaults = mimir_program_texture_defaults,
|
||||
.get_properties = mimir_program_texture_properties,
|
||||
.update = mimir_program_texture_update,
|
||||
.video_render = mimir_program_texture_render,
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
void mimir_register_program_texture_source()
|
||||
{
|
||||
obs_register_source(&mimir_program_texture_source_info);
|
||||
}
|
||||
43
native/obs_stem_source/mimir_obs_stem_ipc.h
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
constexpr uint64_t MIMIR_OBS_STEM_MAGIC = 0x4D545352494D494Dull;
|
||||
constexpr int MIMIR_OBS_STEM_VERSION = 1;
|
||||
constexpr int MIMIR_OBS_STEM_MAX_STEMS = 8;
|
||||
constexpr int MIMIR_OBS_STEM_MAX_FRAMES = 4096;
|
||||
constexpr int MIMIR_OBS_STEM_ID_BYTES = 64;
|
||||
constexpr int MIMIR_OBS_STEM_DISPLAY_BYTES = 64;
|
||||
constexpr int MIMIR_OBS_STEM_SOURCE_BYTES = 64;
|
||||
constexpr int MIMIR_OBS_STEM_HEADER_BYTES = 64;
|
||||
constexpr int MIMIR_OBS_STEM_RECORD_BYTES = 224;
|
||||
constexpr int MIMIR_OBS_STEM_MAP_BYTES =
|
||||
MIMIR_OBS_STEM_HEADER_BYTES +
|
||||
(MIMIR_OBS_STEM_MAX_STEMS * MIMIR_OBS_STEM_RECORD_BYTES) +
|
||||
(MIMIR_OBS_STEM_MAX_STEMS * MIMIR_OBS_STEM_MAX_FRAMES * static_cast<int>(sizeof(float)));
|
||||
|
||||
struct MimirObsStemHeader {
|
||||
uint64_t magic;
|
||||
int32_t version;
|
||||
uint32_t generation;
|
||||
int32_t stem_count;
|
||||
int32_t max_stems;
|
||||
int32_t max_frames;
|
||||
int32_t header_bytes;
|
||||
int32_t record_bytes;
|
||||
int32_t map_bytes;
|
||||
};
|
||||
|
||||
struct MimirObsStemRecord {
|
||||
int32_t channel_index;
|
||||
int32_t configured;
|
||||
int32_t sample_rate;
|
||||
int32_t frame_count;
|
||||
int32_t sample_offset;
|
||||
int32_t reserved0;
|
||||
int32_t reserved1;
|
||||
int32_t reserved2;
|
||||
char stem_id[MIMIR_OBS_STEM_ID_BYTES];
|
||||
char display_name[MIMIR_OBS_STEM_DISPLAY_BYTES];
|
||||
char source_id[MIMIR_OBS_STEM_SOURCE_BYTES];
|
||||
};
|
||||
219
native/obs_stem_source/mimir_obs_stem_source.cpp
Normal file
|
|
@ -0,0 +1,219 @@
|
|||
#include <obs-module.h>
|
||||
#include <util/platform.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#include <windows.h>
|
||||
|
||||
#include "mimir_obs_stem_ipc.h"
|
||||
|
||||
OBS_DECLARE_MODULE()
|
||||
OBS_MODULE_USE_DEFAULT_LOCALE("mimir-obs-stem-source", "en-US")
|
||||
|
||||
void mimir_register_program_texture_source();
|
||||
void mimir_register_muninn_source();
|
||||
|
||||
struct MimirStemSource {
|
||||
obs_source_t *source = nullptr;
|
||||
std::string map_name = "Local\\MimirObsStemBus";
|
||||
std::string stem_id = "aligned_source_0";
|
||||
HANDLE mapping = nullptr;
|
||||
const uint8_t *view = nullptr;
|
||||
std::thread worker;
|
||||
std::atomic_bool running = false;
|
||||
uint32_t last_generation = 0;
|
||||
};
|
||||
|
||||
static std::string obs_string(obs_data_t *settings, const char *key, const char *fallback)
|
||||
{
|
||||
const char *value = obs_data_get_string(settings, key);
|
||||
return value && *value ? std::string(value) : std::string(fallback);
|
||||
}
|
||||
|
||||
static void close_mapping(MimirStemSource *ctx)
|
||||
{
|
||||
if (ctx->view) {
|
||||
UnmapViewOfFile(ctx->view);
|
||||
ctx->view = nullptr;
|
||||
}
|
||||
|
||||
if (ctx->mapping) {
|
||||
CloseHandle(ctx->mapping);
|
||||
ctx->mapping = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
static bool open_mapping(MimirStemSource *ctx)
|
||||
{
|
||||
if (ctx->mapping) {
|
||||
return true;
|
||||
}
|
||||
|
||||
ctx->mapping = OpenFileMappingA(FILE_MAP_READ, FALSE, ctx->map_name.c_str());
|
||||
if (!ctx->mapping) {
|
||||
return false;
|
||||
}
|
||||
|
||||
ctx->view = static_cast<const uint8_t *>(MapViewOfFile(ctx->mapping, FILE_MAP_READ, 0, 0, MIMIR_OBS_STEM_MAP_BYTES));
|
||||
if (!ctx->view) {
|
||||
close_mapping(ctx);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static const MimirObsStemRecord *find_record(const uint8_t *view, const std::string &stem_id, int stem_count)
|
||||
{
|
||||
const auto count = std::min(stem_count, MIMIR_OBS_STEM_MAX_STEMS);
|
||||
for (int index = 0; index < count; ++index) {
|
||||
const auto *record = reinterpret_cast<const MimirObsStemRecord *>(view + MIMIR_OBS_STEM_HEADER_BYTES + (index * MIMIR_OBS_STEM_RECORD_BYTES));
|
||||
if (record->frame_count <= 0 || record->sample_rate <= 0 || record->sample_offset <= 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (stem_id == record->stem_id) {
|
||||
return record;
|
||||
}
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
static void publish_if_available(MimirStemSource *ctx)
|
||||
{
|
||||
if (!open_mapping(ctx)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto *header = reinterpret_cast<const MimirObsStemHeader *>(ctx->view);
|
||||
const auto generation_a = header->generation;
|
||||
if ((generation_a & 1u) != 0 || generation_a == ctx->last_generation) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (header->magic != MIMIR_OBS_STEM_MAGIC ||
|
||||
header->version != MIMIR_OBS_STEM_VERSION ||
|
||||
header->map_bytes != MIMIR_OBS_STEM_MAP_BYTES) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto *record = find_record(ctx->view, ctx->stem_id, header->stem_count);
|
||||
if (!record) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto frame_count = std::min(record->frame_count, MIMIR_OBS_STEM_MAX_FRAMES);
|
||||
const auto sample_rate = record->sample_rate;
|
||||
const auto byte_count = static_cast<size_t>(frame_count) * sizeof(float);
|
||||
if (record->sample_offset < 0 || record->sample_offset + static_cast<int>(byte_count) > MIMIR_OBS_STEM_MAP_BYTES) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::vector<float> samples(frame_count);
|
||||
std::memcpy(samples.data(), ctx->view + record->sample_offset, byte_count);
|
||||
|
||||
const auto generation_b = header->generation;
|
||||
if (generation_a != generation_b || (generation_b & 1u) != 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
ctx->last_generation = generation_b;
|
||||
uint8_t *planes[MAX_AV_PLANES] = {};
|
||||
planes[0] = reinterpret_cast<uint8_t *>(samples.data());
|
||||
obs_source_audio audio = {};
|
||||
audio.data[0] = planes[0];
|
||||
audio.frames = static_cast<uint32_t>(frame_count);
|
||||
audio.speakers = SPEAKERS_MONO;
|
||||
audio.samples_per_sec = static_cast<uint32_t>(sample_rate);
|
||||
audio.format = AUDIO_FORMAT_FLOAT_PLANAR;
|
||||
audio.timestamp = os_gettime_ns();
|
||||
obs_source_output_audio(ctx->source, &audio);
|
||||
}
|
||||
|
||||
static void worker_loop(MimirStemSource *ctx)
|
||||
{
|
||||
while (ctx->running.load()) {
|
||||
publish_if_available(ctx);
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(5));
|
||||
}
|
||||
}
|
||||
|
||||
static const char *mimir_stem_get_name(void *)
|
||||
{
|
||||
return obs_module_text("MimirAudioStem");
|
||||
}
|
||||
|
||||
static void mimir_stem_update(void *data, obs_data_t *settings)
|
||||
{
|
||||
auto *ctx = static_cast<MimirStemSource *>(data);
|
||||
ctx->map_name = obs_string(settings, "map_name", "Local\\MimirObsStemBus");
|
||||
ctx->stem_id = obs_string(settings, "stem_id", "aligned_source_0");
|
||||
ctx->last_generation = 0;
|
||||
close_mapping(ctx);
|
||||
}
|
||||
|
||||
static void *mimir_stem_create(obs_data_t *settings, obs_source_t *source)
|
||||
{
|
||||
auto *ctx = new MimirStemSource();
|
||||
ctx->source = source;
|
||||
mimir_stem_update(ctx, settings);
|
||||
ctx->running = true;
|
||||
ctx->worker = std::thread(worker_loop, ctx);
|
||||
return ctx;
|
||||
}
|
||||
|
||||
static void mimir_stem_destroy(void *data)
|
||||
{
|
||||
auto *ctx = static_cast<MimirStemSource *>(data);
|
||||
ctx->running = false;
|
||||
if (ctx->worker.joinable()) {
|
||||
ctx->worker.join();
|
||||
}
|
||||
|
||||
close_mapping(ctx);
|
||||
delete ctx;
|
||||
}
|
||||
|
||||
static void mimir_stem_defaults(obs_data_t *settings)
|
||||
{
|
||||
obs_data_set_default_string(settings, "map_name", "Local\\MimirObsStemBus");
|
||||
obs_data_set_default_string(settings, "stem_id", "aligned_source_0");
|
||||
}
|
||||
|
||||
static obs_properties_t *mimir_stem_properties(void *)
|
||||
{
|
||||
obs_properties_t *props = obs_properties_create();
|
||||
obs_properties_add_text(props, "map_name", obs_module_text("MimirAudioStem.MapName"), OBS_TEXT_DEFAULT);
|
||||
obs_properties_add_text(props, "stem_id", obs_module_text("MimirAudioStem.StemId"), OBS_TEXT_DEFAULT);
|
||||
return props;
|
||||
}
|
||||
|
||||
static obs_source_info mimir_stem_source_info = {
|
||||
.id = "mimir_audio_stem_source",
|
||||
.type = OBS_SOURCE_TYPE_INPUT,
|
||||
.output_flags = OBS_SOURCE_AUDIO,
|
||||
.get_name = mimir_stem_get_name,
|
||||
.create = mimir_stem_create,
|
||||
.destroy = mimir_stem_destroy,
|
||||
.get_defaults = mimir_stem_defaults,
|
||||
.get_properties = mimir_stem_properties,
|
||||
.update = mimir_stem_update,
|
||||
};
|
||||
|
||||
bool obs_module_load(void)
|
||||
{
|
||||
obs_register_source(&mimir_stem_source_info);
|
||||
mimir_register_program_texture_source();
|
||||
mimir_register_muninn_source();
|
||||
blog(
|
||||
LOG_INFO,
|
||||
"Mimir OBS sources loaded: mimir_audio_stem_source, mimir_program_texture_source, "
|
||||
"muninn_stream_source, muninn_video_source, muninn_audio_source");
|
||||
return true;
|
||||
}
|
||||
267
native/obs_stem_source/muninn_media_wire.h
Normal file
|
|
@ -0,0 +1,267 @@
|
|||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <climits>
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace muninn_media_wire {
|
||||
|
||||
inline constexpr const char *ReceiverFeedbackSchema = "muninn.media_receiver_feedback.v1";
|
||||
|
||||
class MsgpackWriter {
|
||||
public:
|
||||
const std::vector<uint8_t> &bytes() const { return bytes_; }
|
||||
|
||||
void write_raw(const std::vector<uint8_t> &value)
|
||||
{
|
||||
bytes_.insert(bytes_.end(), value.begin(), value.end());
|
||||
}
|
||||
|
||||
void write_nil() { bytes_.push_back(0xc0); }
|
||||
|
||||
void write_bool(bool value) { bytes_.push_back(value ? 0xc3 : 0xc2); }
|
||||
|
||||
void write_u64(uint64_t value)
|
||||
{
|
||||
if (value <= 0x7f) {
|
||||
bytes_.push_back(static_cast<uint8_t>(value));
|
||||
} else if (value <= 0xff) {
|
||||
bytes_.push_back(0xcc);
|
||||
bytes_.push_back(static_cast<uint8_t>(value));
|
||||
} else if (value <= 0xffff) {
|
||||
bytes_.push_back(0xcd);
|
||||
write_be16_value(static_cast<uint16_t>(value));
|
||||
} else if (value <= 0xffffffffull) {
|
||||
bytes_.push_back(0xce);
|
||||
write_be32_value(static_cast<uint32_t>(value));
|
||||
} else {
|
||||
bytes_.push_back(0xcf);
|
||||
write_be64_value(value);
|
||||
}
|
||||
}
|
||||
|
||||
void write_i64(int64_t value)
|
||||
{
|
||||
if (value >= 0) {
|
||||
write_u64(static_cast<uint64_t>(value));
|
||||
return;
|
||||
}
|
||||
if (value >= -32) {
|
||||
bytes_.push_back(static_cast<uint8_t>(0xe0 | (value + 32)));
|
||||
} else if (value >= INT8_MIN) {
|
||||
bytes_.push_back(0xd0);
|
||||
bytes_.push_back(static_cast<uint8_t>(value));
|
||||
} else if (value >= INT16_MIN) {
|
||||
bytes_.push_back(0xd1);
|
||||
write_be16_value(static_cast<uint16_t>(value));
|
||||
} else if (value >= INT32_MIN) {
|
||||
bytes_.push_back(0xd2);
|
||||
write_be32_value(static_cast<uint32_t>(value));
|
||||
} else {
|
||||
bytes_.push_back(0xd3);
|
||||
write_be64_value(static_cast<uint64_t>(value));
|
||||
}
|
||||
}
|
||||
|
||||
void write_string(const std::string &value)
|
||||
{
|
||||
const auto length = value.size();
|
||||
if (length <= 31) {
|
||||
bytes_.push_back(static_cast<uint8_t>(0xa0 | length));
|
||||
} else if (length <= 0xff) {
|
||||
bytes_.push_back(0xd9);
|
||||
bytes_.push_back(static_cast<uint8_t>(length));
|
||||
} else if (length <= 0xffff) {
|
||||
bytes_.push_back(0xda);
|
||||
write_be16_value(static_cast<uint16_t>(length));
|
||||
} else {
|
||||
bytes_.push_back(0xdb);
|
||||
write_be32_value(static_cast<uint32_t>(length));
|
||||
}
|
||||
bytes_.insert(bytes_.end(), value.begin(), value.end());
|
||||
}
|
||||
|
||||
void write_binary(const std::vector<uint8_t> &value)
|
||||
{
|
||||
const auto length = value.size();
|
||||
if (length <= 0xff) {
|
||||
bytes_.push_back(0xc4);
|
||||
bytes_.push_back(static_cast<uint8_t>(length));
|
||||
} else if (length <= 0xffff) {
|
||||
bytes_.push_back(0xc5);
|
||||
write_be16_value(static_cast<uint16_t>(length));
|
||||
} else {
|
||||
bytes_.push_back(0xc6);
|
||||
write_be32_value(static_cast<uint32_t>(length));
|
||||
}
|
||||
bytes_.insert(bytes_.end(), value.begin(), value.end());
|
||||
}
|
||||
|
||||
void write_array_len(uint32_t length)
|
||||
{
|
||||
if (length <= 15) {
|
||||
bytes_.push_back(static_cast<uint8_t>(0x90 | length));
|
||||
} else if (length <= 0xffff) {
|
||||
bytes_.push_back(0xdc);
|
||||
write_be16_value(static_cast<uint16_t>(length));
|
||||
} else {
|
||||
bytes_.push_back(0xdd);
|
||||
write_be32_value(length);
|
||||
}
|
||||
}
|
||||
|
||||
void write_map_len(uint32_t length)
|
||||
{
|
||||
if (length <= 15) {
|
||||
bytes_.push_back(static_cast<uint8_t>(0x80 | length));
|
||||
} else if (length <= 0xffff) {
|
||||
bytes_.push_back(0xde);
|
||||
write_be16_value(static_cast<uint16_t>(length));
|
||||
} else {
|
||||
bytes_.push_back(0xdf);
|
||||
write_be32_value(length);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
std::vector<uint8_t> bytes_;
|
||||
|
||||
void write_be16_value(uint16_t value)
|
||||
{
|
||||
bytes_.push_back(static_cast<uint8_t>((value >> 8) & 0xff));
|
||||
bytes_.push_back(static_cast<uint8_t>(value & 0xff));
|
||||
}
|
||||
|
||||
void write_be32_value(uint32_t value)
|
||||
{
|
||||
bytes_.push_back(static_cast<uint8_t>((value >> 24) & 0xff));
|
||||
bytes_.push_back(static_cast<uint8_t>((value >> 16) & 0xff));
|
||||
bytes_.push_back(static_cast<uint8_t>((value >> 8) & 0xff));
|
||||
bytes_.push_back(static_cast<uint8_t>(value & 0xff));
|
||||
}
|
||||
|
||||
void write_be64_value(uint64_t value)
|
||||
{
|
||||
write_be32_value(static_cast<uint32_t>((value >> 32) & 0xffffffffull));
|
||||
write_be32_value(static_cast<uint32_t>(value & 0xffffffffull));
|
||||
}
|
||||
};
|
||||
|
||||
inline std::string feedback_observed_at()
|
||||
{
|
||||
const auto now = std::chrono::system_clock::now().time_since_epoch();
|
||||
const auto millis = std::chrono::duration_cast<std::chrono::milliseconds>(now).count();
|
||||
return "unix-ms:" + std::to_string(millis);
|
||||
}
|
||||
|
||||
inline void write_string_array(MsgpackWriter &writer, const std::vector<std::string> &values)
|
||||
{
|
||||
writer.write_array_len(static_cast<uint32_t>(values.size()));
|
||||
for (const auto &value : values) {
|
||||
writer.write_string(value);
|
||||
}
|
||||
}
|
||||
|
||||
inline void write_u64_array(MsgpackWriter &writer, const std::vector<uint64_t> &values)
|
||||
{
|
||||
writer.write_array_len(static_cast<uint32_t>(values.size()));
|
||||
for (const auto value : values) {
|
||||
writer.write_u64(value);
|
||||
}
|
||||
}
|
||||
|
||||
inline std::vector<uint8_t> encode_receiver_feedback_payload(
|
||||
const std::string &stream_id,
|
||||
const std::string &session_id,
|
||||
uint64_t frame_id,
|
||||
const std::vector<std::string> &missing_chunk_keys,
|
||||
bool requested_keyframe,
|
||||
const std::string &observed_at)
|
||||
{
|
||||
MsgpackWriter record;
|
||||
record.write_array_len(11);
|
||||
record.write_string(stream_id);
|
||||
record.write_string(session_id);
|
||||
record.write_string("starfire.obs");
|
||||
if (frame_id > 0) {
|
||||
record.write_u64(frame_id - 1);
|
||||
} else {
|
||||
record.write_nil();
|
||||
}
|
||||
write_u64_array(record, {});
|
||||
write_u64_array(record, {frame_id});
|
||||
record.write_bool(requested_keyframe);
|
||||
record.write_i64(0);
|
||||
record.write_i64(0);
|
||||
record.write_string(observed_at);
|
||||
write_string_array(record, missing_chunk_keys);
|
||||
|
||||
const std::string record_key = stream_id + ":" + session_id + ":feedback:starfire.obs";
|
||||
const std::string message_id =
|
||||
"muninn-media:" + std::string(ReceiverFeedbackSchema) + ":" + record_key;
|
||||
|
||||
MsgpackWriter document;
|
||||
document.write_map_len(9);
|
||||
document.write_string("schemaId");
|
||||
document.write_string(ReceiverFeedbackSchema);
|
||||
document.write_string("recordKey");
|
||||
document.write_string(record_key);
|
||||
document.write_string("storedAt");
|
||||
document.write_string(observed_at);
|
||||
document.write_string("payloadEncoding");
|
||||
document.write_string("messagepack");
|
||||
document.write_string("payload");
|
||||
document.write_binary(record.bytes());
|
||||
document.write_string("sourceRuntimeId");
|
||||
document.write_string("starfire");
|
||||
document.write_string("sourceAgentId");
|
||||
document.write_nil();
|
||||
document.write_string("sourceRole");
|
||||
document.write_string("mimir.obs");
|
||||
document.write_string("tags");
|
||||
write_string_array(document, {"muninn.media"});
|
||||
|
||||
MsgpackWriter wire;
|
||||
wire.write_map_len(3);
|
||||
wire.write_string("schemaVersion");
|
||||
wire.write_string("cultnet.document_put_raw.v0");
|
||||
wire.write_string("messageId");
|
||||
wire.write_string(message_id);
|
||||
wire.write_string("document");
|
||||
wire.write_raw(document.bytes());
|
||||
return wire.bytes();
|
||||
}
|
||||
|
||||
inline std::vector<uint8_t> encode_receiver_feedback_payload(
|
||||
const std::string &stream_id,
|
||||
const std::string &session_id,
|
||||
uint64_t frame_id,
|
||||
const std::vector<std::string> &missing_chunk_keys,
|
||||
bool requested_keyframe)
|
||||
{
|
||||
return encode_receiver_feedback_payload(
|
||||
stream_id,
|
||||
session_id,
|
||||
frame_id,
|
||||
missing_chunk_keys,
|
||||
requested_keyframe,
|
||||
feedback_observed_at());
|
||||
}
|
||||
|
||||
inline std::vector<uint8_t> encode_receiver_feedback_payload(
|
||||
const std::string &stream_id,
|
||||
const std::string &session_id,
|
||||
uint64_t frame_id,
|
||||
const std::vector<std::string> &missing_chunk_keys)
|
||||
{
|
||||
return encode_receiver_feedback_payload(
|
||||
stream_id,
|
||||
session_id,
|
||||
frame_id,
|
||||
missing_chunk_keys,
|
||||
true);
|
||||
}
|
||||
|
||||
} // namespace muninn_media_wire
|
||||
54
native/obs_stem_source/muninn_rudp_ack.h
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <set>
|
||||
#include <utility>
|
||||
|
||||
namespace muninn_rudp_ack {
|
||||
|
||||
class AckTracker {
|
||||
public:
|
||||
void reset()
|
||||
{
|
||||
received_sequences_.clear();
|
||||
highest_received_sequence_.reset();
|
||||
}
|
||||
|
||||
void remember(uint32_t sequence)
|
||||
{
|
||||
received_sequences_.insert(sequence);
|
||||
if (!highest_received_sequence_ || sequence > *highest_received_sequence_) {
|
||||
highest_received_sequence_ = sequence;
|
||||
}
|
||||
if (!highest_received_sequence_) {
|
||||
return;
|
||||
}
|
||||
const uint32_t highest = *highest_received_sequence_;
|
||||
for (auto iterator = received_sequences_.begin(); iterator != received_sequences_.end();) {
|
||||
if (*iterator + 32 < highest) {
|
||||
iterator = received_sequences_.erase(iterator);
|
||||
} else {
|
||||
++iterator;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::pair<uint32_t, uint32_t> state() const
|
||||
{
|
||||
const uint32_t ack = highest_received_sequence_.value_or(0);
|
||||
uint32_t ack_mask = 0;
|
||||
for (uint32_t bit = 0; bit < 32; ++bit) {
|
||||
if (ack > bit && received_sequences_.find(ack - bit - 1) != received_sequences_.end()) {
|
||||
ack_mask |= 1u << bit;
|
||||
}
|
||||
}
|
||||
return {ack, ack_mask};
|
||||
}
|
||||
|
||||
private:
|
||||
std::set<uint32_t> received_sequences_;
|
||||
std::optional<uint32_t> highest_received_sequence_;
|
||||
};
|
||||
|
||||
} // namespace muninn_rudp_ack
|
||||
127
native/obs_stem_source/muninn_rudp_fragments.h
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace muninn_rudp_fragments {
|
||||
|
||||
constexpr uint32_t CompletedFragmentIdQuarantineMs = 2000;
|
||||
|
||||
struct FragmentResult {
|
||||
uint32_t first_sequence = 0;
|
||||
std::vector<uint8_t> payload;
|
||||
};
|
||||
|
||||
class FragmentAssembler {
|
||||
public:
|
||||
void reset()
|
||||
{
|
||||
assemblies_.clear();
|
||||
recently_closed_.clear();
|
||||
expired_count_ = 0;
|
||||
}
|
||||
|
||||
std::optional<FragmentResult> insert(const std::string &channel_id,
|
||||
uint16_t fragment_id,
|
||||
uint16_t fragment_index,
|
||||
uint16_t fragment_count,
|
||||
uint32_t sequence,
|
||||
const uint8_t *payload,
|
||||
uint32_t payload_len,
|
||||
std::chrono::steady_clock::time_point now)
|
||||
{
|
||||
if (fragment_id == 0 || fragment_index >= fragment_count) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
const std::string key = channel_id + ":" + std::to_string(fragment_id);
|
||||
expire_recently_closed(now);
|
||||
const auto existing = assemblies_.find(key);
|
||||
if (existing == assemblies_.end() && recently_closed_.find(key) != recently_closed_.end()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
auto &assembly = assemblies_[key];
|
||||
if (assembly.payloads.empty()) {
|
||||
assembly.channel_id = channel_id;
|
||||
assembly.fragment_count = fragment_count;
|
||||
assembly.started_at = now;
|
||||
}
|
||||
if (assembly.channel_id != channel_id || assembly.fragment_count != fragment_count) {
|
||||
assemblies_.erase(key);
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
assembly.payloads.emplace(fragment_index, std::vector<uint8_t>(payload, payload + payload_len));
|
||||
assembly.sequences.emplace(fragment_index, sequence);
|
||||
if (assembly.payloads.size() < assembly.fragment_count) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
FragmentResult result;
|
||||
result.first_sequence = UINT32_MAX;
|
||||
for (uint16_t index = 0; index < assembly.fragment_count; ++index) {
|
||||
const auto payload_found = assembly.payloads.find(index);
|
||||
const auto sequence_found = assembly.sequences.find(index);
|
||||
if (payload_found == assembly.payloads.end() || sequence_found == assembly.sequences.end()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
result.payload.insert(result.payload.end(), payload_found->second.begin(), payload_found->second.end());
|
||||
result.first_sequence = std::min(result.first_sequence, sequence_found->second);
|
||||
}
|
||||
assemblies_.erase(key);
|
||||
recently_closed_[key] = now;
|
||||
return result;
|
||||
}
|
||||
|
||||
uint64_t expire(std::chrono::steady_clock::time_point now, uint32_t expire_after_ms)
|
||||
{
|
||||
expire_recently_closed(now);
|
||||
uint64_t expired = 0;
|
||||
for (auto iterator = assemblies_.begin(); iterator != assemblies_.end();) {
|
||||
if (now - iterator->second.started_at < std::chrono::milliseconds(expire_after_ms)) {
|
||||
++iterator;
|
||||
continue;
|
||||
}
|
||||
recently_closed_[iterator->first] = now;
|
||||
iterator = assemblies_.erase(iterator);
|
||||
++expired;
|
||||
}
|
||||
expired_count_ += expired;
|
||||
return expired;
|
||||
}
|
||||
|
||||
uint64_t expired_count() const { return expired_count_; }
|
||||
|
||||
private:
|
||||
void expire_recently_closed(std::chrono::steady_clock::time_point now)
|
||||
{
|
||||
for (auto iterator = recently_closed_.begin(); iterator != recently_closed_.end();) {
|
||||
if (now - iterator->second < std::chrono::milliseconds(CompletedFragmentIdQuarantineMs)) {
|
||||
++iterator;
|
||||
continue;
|
||||
}
|
||||
iterator = recently_closed_.erase(iterator);
|
||||
}
|
||||
}
|
||||
|
||||
struct Assembly {
|
||||
std::string channel_id;
|
||||
uint16_t fragment_count = 0;
|
||||
std::map<uint16_t, std::vector<uint8_t>> payloads;
|
||||
std::map<uint16_t, uint32_t> sequences;
|
||||
std::chrono::steady_clock::time_point started_at;
|
||||
};
|
||||
|
||||
std::map<std::string, Assembly> assemblies_;
|
||||
std::map<std::string, std::chrono::steady_clock::time_point> recently_closed_;
|
||||
uint64_t expired_count_ = 0;
|
||||
};
|
||||
|
||||
} // namespace muninn_rudp_fragments
|
||||
123
native/obs_stem_source/muninn_rudp_url.h
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
|
||||
namespace muninn_rudp_url {
|
||||
|
||||
inline constexpr uint32_t DefaultMediaRudpConnectionId = 0x6d750001;
|
||||
inline constexpr uint32_t DefaultAudioRudpConnectionId = 0x6d750004;
|
||||
inline constexpr uint32_t DefaultReliableExpireAfterMs = 75;
|
||||
inline constexpr uint32_t DefaultVideoAssemblyDeadlineMs = 75;
|
||||
inline constexpr uint32_t DefaultGapWaitMs = 8;
|
||||
inline constexpr uint32_t DefaultAudioReorderMs = 2000;
|
||||
inline constexpr uint32_t DefaultSenderResendDelayMs = 5;
|
||||
inline constexpr uint32_t MaxLatencyBudgetMs = 2000;
|
||||
|
||||
struct RudpUrlParts {
|
||||
uint16_t port = 0;
|
||||
uint16_t video_local_port = 0;
|
||||
uint16_t audio_local_port = 0;
|
||||
uint32_t connection_id = DefaultMediaRudpConnectionId;
|
||||
uint32_t audio_connection_id = DefaultAudioRudpConnectionId;
|
||||
uint32_t sender_resend_delay_ms = DefaultSenderResendDelayMs;
|
||||
uint32_t reliable_expire_after_ms = DefaultReliableExpireAfterMs;
|
||||
uint32_t video_assembly_deadline_ms = DefaultVideoAssemblyDeadlineMs;
|
||||
uint32_t gap_wait_ms = DefaultGapWaitMs;
|
||||
uint32_t audio_reorder_ms = DefaultAudioReorderMs;
|
||||
};
|
||||
|
||||
inline bool starts_with(const std::string &value, const std::string &prefix)
|
||||
{
|
||||
return value.rfind(prefix, 0) == 0;
|
||||
}
|
||||
|
||||
inline std::optional<uint32_t> parse_u32_hex_or_decimal(const std::string &value)
|
||||
{
|
||||
try {
|
||||
size_t consumed = 0;
|
||||
const int base = starts_with(value, "0x") || starts_with(value, "0X") ? 16 : 10;
|
||||
const auto parsed = std::stoul(value, &consumed, base);
|
||||
if (consumed == value.size()) {
|
||||
return static_cast<uint32_t>(parsed);
|
||||
}
|
||||
} catch (...) {
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
inline std::optional<RudpUrlParts> parse(const std::string &url)
|
||||
{
|
||||
if (!starts_with(url, "rudp://")) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
const auto authority_start = std::string("rudp://").size();
|
||||
const auto path_start = url.find('/', authority_start);
|
||||
const auto query_start = url.find('?', authority_start);
|
||||
const auto authority_end = std::min(path_start == std::string::npos ? url.size() : path_start,
|
||||
query_start == std::string::npos ? url.size() : query_start);
|
||||
const auto authority = url.substr(authority_start, authority_end - authority_start);
|
||||
const auto colon = authority.rfind(':');
|
||||
if (colon == std::string::npos || colon + 1 >= authority.size()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
RudpUrlParts parts;
|
||||
parts.port = static_cast<uint16_t>(std::stoul(authority.substr(colon + 1)));
|
||||
parts.video_local_port = static_cast<uint16_t>(parts.port + 10000);
|
||||
parts.audio_local_port = static_cast<uint16_t>(parts.port + 10001);
|
||||
|
||||
if (query_start != std::string::npos) {
|
||||
std::stringstream query(url.substr(query_start + 1));
|
||||
std::string item;
|
||||
while (std::getline(query, item, '&')) {
|
||||
const auto equals = item.find('=');
|
||||
if (equals == std::string::npos) {
|
||||
continue;
|
||||
}
|
||||
const auto key = item.substr(0, equals);
|
||||
const auto value = item.substr(equals + 1);
|
||||
if (key == "connection") {
|
||||
if (const auto parsed = parse_u32_hex_or_decimal(value)) {
|
||||
parts.connection_id = *parsed;
|
||||
}
|
||||
} else if (key == "audio_connection" || key == "audio_connection_id") {
|
||||
if (const auto parsed = parse_u32_hex_or_decimal(value)) {
|
||||
parts.audio_connection_id = *parsed;
|
||||
}
|
||||
} else if (key == "local_port" || key == "video_local_port") {
|
||||
parts.video_local_port = static_cast<uint16_t>(std::stoul(value));
|
||||
} else if (key == "audio_local_port") {
|
||||
parts.audio_local_port = static_cast<uint16_t>(std::stoul(value));
|
||||
} else if (key == "reliable_expire_after_ms") {
|
||||
if (const auto parsed = parse_u32_hex_or_decimal(value)) {
|
||||
parts.reliable_expire_after_ms = std::clamp(*parsed, 1u, MaxLatencyBudgetMs);
|
||||
}
|
||||
} else if (key == "sender_resend_delay_ms") {
|
||||
if (const auto parsed = parse_u32_hex_or_decimal(value)) {
|
||||
parts.sender_resend_delay_ms = std::clamp(*parsed, 1u, 1000u);
|
||||
}
|
||||
} else if (key == "assembly_deadline_ms") {
|
||||
if (const auto parsed = parse_u32_hex_or_decimal(value)) {
|
||||
parts.video_assembly_deadline_ms = std::clamp(*parsed, 1u, MaxLatencyBudgetMs);
|
||||
}
|
||||
} else if (key == "gap_wait_ms") {
|
||||
if (const auto parsed = parse_u32_hex_or_decimal(value)) {
|
||||
parts.gap_wait_ms = std::clamp(*parsed, 0u, 1000u);
|
||||
}
|
||||
} else if (key == "audio_reorder_ms") {
|
||||
if (const auto parsed = parse_u32_hex_or_decimal(value)) {
|
||||
parts.audio_reorder_ms = std::clamp(*parsed, 0u, MaxLatencyBudgetMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return parts;
|
||||
}
|
||||
|
||||
} // namespace muninn_rudp_url
|
||||
58
native/obs_stem_source/tests/muninn_rudp_url_tests.cpp
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
#include "muninn_rudp_url.h"
|
||||
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
|
||||
namespace {
|
||||
|
||||
void require(bool condition, const std::string &message)
|
||||
{
|
||||
if (!condition) {
|
||||
std::cerr << message << '\n';
|
||||
std::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main()
|
||||
{
|
||||
const auto configured = muninn_rudp_url::parse(
|
||||
"rudp://192.168.1.66:5204/muninn.raven.av.rudp?channel=media&format=muninn-typed-media&connection=0x6d750001&audio_connection=0x6d750004&sender_resend_delay_ms=5&reliable_expire_after_ms=1200&assembly_deadline_ms=1200&gap_wait_ms=16&audio_reorder_ms=700");
|
||||
require(configured.has_value(), "configured RUDP URL did not parse");
|
||||
require(configured->port == 5204, "configured RUDP URL port was not parsed");
|
||||
require(configured->video_local_port == 15204, "configured RUDP video local port was not derived");
|
||||
require(configured->audio_local_port == 15205, "configured RUDP audio local port was not derived");
|
||||
require(configured->connection_id == 0x6d750001, "configured RUDP connection id was not parsed");
|
||||
require(configured->audio_connection_id == 0x6d750004, "configured RUDP audio connection id was not parsed");
|
||||
require(configured->reliable_expire_after_ms == 1200, "configured reliable expiry did not survive parse");
|
||||
require(configured->video_assembly_deadline_ms == 1200, "configured assembly deadline did not survive parse");
|
||||
require(configured->gap_wait_ms == 16, "configured gap wait did not survive parse");
|
||||
require(configured->audio_reorder_ms == 700, "configured audio reorder wait did not survive parse");
|
||||
|
||||
const auto clamped = muninn_rudp_url::parse(
|
||||
"rudp://192.168.1.66:5204/muninn.raven.av.rudp?reliable_expire_after_ms=9000&assembly_deadline_ms=9000");
|
||||
require(clamped.has_value(), "oversized latency RUDP URL did not parse");
|
||||
require(clamped->reliable_expire_after_ms == muninn_rudp_url::MaxLatencyBudgetMs,
|
||||
"oversized reliable expiry did not clamp to the max latency budget");
|
||||
require(clamped->video_assembly_deadline_ms == muninn_rudp_url::MaxLatencyBudgetMs,
|
||||
"oversized assembly deadline did not clamp to the max latency budget");
|
||||
|
||||
const auto fallback = muninn_rudp_url::parse("rudp://192.168.1.66:5204/muninn.raven.av.rudp");
|
||||
require(fallback.has_value(), "minimal RUDP URL did not parse");
|
||||
require(fallback->connection_id == muninn_rudp_url::DefaultMediaRudpConnectionId,
|
||||
"minimal RUDP URL did not keep the default connection id");
|
||||
require(fallback->audio_connection_id == muninn_rudp_url::DefaultAudioRudpConnectionId,
|
||||
"minimal RUDP URL did not keep the default audio connection id");
|
||||
require(fallback->reliable_expire_after_ms == muninn_rudp_url::DefaultReliableExpireAfterMs,
|
||||
"minimal RUDP URL did not keep the default reliable expiry");
|
||||
require(fallback->video_assembly_deadline_ms == muninn_rudp_url::DefaultVideoAssemblyDeadlineMs,
|
||||
"minimal RUDP URL did not keep the default assembly deadline");
|
||||
require(fallback->audio_reorder_ms == muninn_rudp_url::DefaultAudioReorderMs,
|
||||
"minimal RUDP URL did not keep the default audio reorder wait");
|
||||
require(fallback->audio_reorder_ms == 2000,
|
||||
"minimal RUDP URL did not use the full default audio reorder budget");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -125,6 +125,7 @@ struct CaptureState
|
|||
std::vector<unsigned long long> sweepSamples;
|
||||
std::vector<float>* playbackSamples = nullptr;
|
||||
double playbackGain = 1.0;
|
||||
long playbackOutputChannel = -1;
|
||||
std::vector<float>* recordedInput = nullptr;
|
||||
unsigned long long recordFrames = 0;
|
||||
unsigned long long sweepFrame = 0;
|
||||
|
|
@ -320,6 +321,11 @@ void processOutput(long doubleBufferIndex)
|
|||
|
||||
for (size_t channel = 0; channel < g_capture.outputBuffers->size(); ++channel)
|
||||
{
|
||||
if (g_capture.playbackOutputChannel >= 0 && static_cast<long>(channel) != g_capture.playbackOutputChannel)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
auto& buffer = (*g_capture.outputBuffers)[channel];
|
||||
auto& info = (*g_capture.outputChannels)[channel];
|
||||
auto* raw = static_cast<unsigned char*>(buffer.buffers[doubleBufferIndex]);
|
||||
|
|
@ -498,6 +504,12 @@ double doubleOption(int argc, char** argv, const char* name, double fallback)
|
|||
return raw ? std::atof(raw) : fallback;
|
||||
}
|
||||
|
||||
long longOption(int argc, char** argv, const char* name, long fallback)
|
||||
{
|
||||
const char* raw = option(argc, argv, name, nullptr);
|
||||
return raw ? std::atol(raw) : fallback;
|
||||
}
|
||||
|
||||
bool hasArg(int argc, char** argv, const char* name)
|
||||
{
|
||||
for (int index = 1; index < argc; ++index)
|
||||
|
|
@ -576,6 +588,7 @@ int main(int argc, char** argv)
|
|||
const char* playFloat32MonoPath = option(argc, argv, "--play-f32-mono", nullptr);
|
||||
const char* recordFloat32InterleavedPath = option(argc, argv, "--record-f32-interleaved", nullptr);
|
||||
const auto playGain = doubleOption(argc, argv, "--play-gain", 1.0);
|
||||
const auto playOutputChannel = longOption(argc, argv, "--play-output-channel", -1);
|
||||
const auto sweepGain = doubleOption(argc, argv, "--sweep-gain", 0.03);
|
||||
const auto sweepToneSeconds = doubleOption(argc, argv, "--sweep-tone-seconds", 0.55);
|
||||
const auto sweepGapSeconds = doubleOption(argc, argv, "--sweep-gap-seconds", 0.20);
|
||||
|
|
@ -731,7 +744,7 @@ int main(int argc, char** argv)
|
|||
playbackSamples.resize(static_cast<size_t>(byteCount) / sizeof(float));
|
||||
input.read(reinterpret_cast<char*>(playbackSamples.data()), static_cast<std::streamsize>(playbackSamples.size() * sizeof(float)));
|
||||
}
|
||||
std::printf("asio playback-f32-mono: path=%s samples=%zu gain=%.5f\n", playFloat32MonoPath, playbackSamples.size(), playGain);
|
||||
std::printf("asio playback-f32-mono: path=%s samples=%zu gain=%.5f outputChannel=%ld\n", playFloat32MonoPath, playbackSamples.size(), playGain, playOutputChannel);
|
||||
}
|
||||
|
||||
if (captureSeconds > 0.0 || monitorSweep || !playbackSamples.empty() || emitJsonBlocks)
|
||||
|
|
@ -820,6 +833,7 @@ int main(int argc, char** argv)
|
|||
g_capture.sweepGain = sweepGain;
|
||||
g_capture.playbackSamples = playbackSamples.empty() ? nullptr : &playbackSamples;
|
||||
g_capture.playbackGain = playGain;
|
||||
g_capture.playbackOutputChannel = playOutputChannel;
|
||||
g_capture.emitJsonBlocks = emitJsonBlocks;
|
||||
g_capture.sweepFrame = 0;
|
||||
g_capture.sweepEnergy.assign(sweepFrequencies.size() * static_cast<size_t>(inputs), 0.0);
|
||||
|
|
|
|||
|
|
@ -23,9 +23,14 @@ flowchart TD
|
|||
C["Leap timing/IR driver"] --> R
|
||||
D["network feed producers"] --> R
|
||||
R --> N["native reservoir handles"]
|
||||
N --> E["Fensalir GPU fusion + UI"]
|
||||
R --> W["Mimir Well typed pages + timing state"]
|
||||
W --> D["Fensalir reservoir daemon"]
|
||||
N --> D
|
||||
D --> E["Fensalir GPU fusion surfaces"]
|
||||
R --> U["Mimir editor thin synced-video UI"]
|
||||
N --> F["Faust/native DSP"]
|
||||
E --> G["Spout2/program video"]
|
||||
U --> V["EVE/native shared-texture program stream"]
|
||||
F --> H["program stems + spatial bed"]
|
||||
G --> I["OBS"]
|
||||
H --> I
|
||||
|
|
@ -40,12 +45,26 @@ Ownership:
|
|||
phones, microcontrollers, Raven, and Starfire can participate without
|
||||
becoming independent clock authorities.
|
||||
- `Mimir.Runtime` owns app-level stream buffers and synchronization.
|
||||
- `Mimir.Well` owns typed live pages, configured composites, timing pressure,
|
||||
feature signals, and capture-body publication. It is the source the heavy
|
||||
reservoir worker drinks from; it is not a renderer.
|
||||
- Native capture workers own device reads.
|
||||
- Fensalir owns dense visual fusion, material/brush/splat reconciliation,
|
||||
D3D12 interop, runtime UI, and Spout2 publication.
|
||||
- The future Fensalir reservoir daemon owns dense visual fusion,
|
||||
material/brush/splat reconciliation, reservoir budgets, job scheduling, GPU
|
||||
residency, and any CPU/GPU worker share thrown at field reconstruction. It
|
||||
drinks from the Well and publishes typed reservoir/program surfaces.
|
||||
- The Mimir editor owns only responsive 2D synced-video presentation controls
|
||||
and program-layer placement for the current cut. It must not accidentally run
|
||||
reservoir history work just because a frame is being drawn.
|
||||
- Fensalir's in-app renderer owns D3D12 interop and lowering of explicit
|
||||
surfaces. A flat video compositor backend is still missing: current
|
||||
FieldEvidence camera resources are declarations/claims, not a production
|
||||
textured-quad draw path.
|
||||
- Faust/native DSP owns hot audio alignment, suppression, separation,
|
||||
spatialization, and stem generation.
|
||||
- OBS owns broadcast controls.
|
||||
- EVE owns native decode/composite for its fullscreen display path; it does not
|
||||
own layout, WebKit compatibility, or timing authority.
|
||||
|
||||
Invariant: the live window is bounded, in memory, and has one timing authority.
|
||||
No private history outlives the rolling buffer.
|
||||
|
|
@ -78,6 +97,14 @@ probes can emit per-frame JSON metadata so Fensalir sees real sensor cadence in
|
|||
the rolling buffers while the direct ABI driver is being cut. It does not carry
|
||||
pixels and does not own the final six-camera hot path. Multi-camera probes are
|
||||
one process with declared accepted source ids, not one process per camera.
|
||||
The `ffmpeg-rawvideo` adapter is the live network-pixel edge for Raven setup:
|
||||
FFmpeg owns SRT receive/decode and writes exact raw frames to stdout;
|
||||
`MimirFfmpegRawVideoStreamSource` owns frame-boundary reads, BGRA/NV12 payload
|
||||
geometry, sample descriptors, and insertion into the normal synchronization
|
||||
path. `config/mimir-runtime.raven-eve.example.json` declares `raven-display`
|
||||
on SRT port `5200`. EveCanvas owns Eve camera/mic capture directly and sends
|
||||
typed frame-events to `Mimir.EveSensorReceiver` on WebSocket ports `8793` and
|
||||
`8794`.
|
||||
|
||||
## OBS Bridge Utility
|
||||
|
||||
|
|
@ -99,9 +126,13 @@ not become the synchronized program authority.
|
|||
|
||||
## Audio Field
|
||||
|
||||
The six-microphone path is separate from the bridge. Scarlett speaker loopback
|
||||
is the current timing authority when active calibration is playing, but Scarlett
|
||||
production capture belongs on ASIO rather than WASAPI shared mode.
|
||||
The six-microphone path is separate from the bridge. Current local routing is:
|
||||
Windows system default output is Scarlett for monitoring, while Starfire
|
||||
Realtek is the active chirp-emission and loopback-measurement path. Any active
|
||||
chirp runner that emits to the default output is therefore wrong unless it
|
||||
explicitly selects Realtek. Scarlett production capture belongs on ASIO rather
|
||||
than WASAPI shared mode, and Scarlett remains the professional mic/monitor
|
||||
interface rather than the current chirp speaker authority.
|
||||
`MimirBioacousticTimeline` owns the active runtime watermark described in
|
||||
[[docs/bioacoustic-timeline-watermark|Bioacoustic Timeline Watermark]]: a
|
||||
low-gain birdsong-like word language with 128 self-identifying word positions,
|
||||
|
|
@ -152,14 +183,17 @@ diagnostic reference for constrained chirplet-transform work. The active
|
|||
receiver is the bioacoustic motif timeline; the analyzer only emits active
|
||||
timing reports from matched canonical anchors. The hub also owns cached reports
|
||||
plus smoothed per-source sync state with delay-slope/SRO in ppm.
|
||||
`MimirRuntime` runs analysis
|
||||
online as a bounded rotating service and can print live telemetry with
|
||||
`MIMIR_SYNC_TELEMETRY_SECONDS`. UI and telemetry are passive readers of cached
|
||||
sync state; they must not invoke the analyzer. Current app testing proves
|
||||
loopback wakeup, live mic buffers, and confident online sync states, but the
|
||||
next proof is stable canonical anchors through real mic streams. Camera mics are
|
||||
spatial/context witnesses; Focusrite devices are dialogue anchors. Fractional
|
||||
delay and the hot resampler belong in Faust/native DSP.
|
||||
`MimirRuntime` runs ordinary audio analysis online as a bounded rotating service
|
||||
and can print live telemetry with `MIMIR_SYNC_TELEMETRY_SECONDS`. UI and
|
||||
telemetry are passive readers of cached sync state; they must not invoke the
|
||||
analyzer. Complex-contour matched-filter analysis is excluded from render-thread
|
||||
and Well-loop cadence by default even when the contour capability is configured;
|
||||
`MIMIR_COMPLEX_CONTOUR_RUNTIME_INLINE=1` is a diagnostic override, not the live
|
||||
owner. Current app testing proves loopback wakeup, live mic buffers, and
|
||||
confident online sync states, but the next proof is stable canonical anchors
|
||||
through real mic streams. Camera mics are spatial/context witnesses; Focusrite
|
||||
devices are dialogue anchors. Fractional delay and the hot resampler belong in
|
||||
Faust/native DSP.
|
||||
|
||||
The WASAPI cadence probe is now a format/state diagnostic: it can request
|
||||
shared or exclusive sample rates, bit depths, channel counts, and float/PCM
|
||||
|
|
@ -279,9 +313,150 @@ inside BufferSmoke artifact replay. When `enableComplexContourRuntime` is true,
|
|||
extracts Float32 windows from rolling audio buffers to publish
|
||||
`evidence=complex-contour` reports. The synthetic runtime-shaped proof
|
||||
`--complex-contour-runtime-self-test` at 192 kHz recovers a 693.5-sample delay
|
||||
with about 0.219 us error from rolling buffers. The next real-world proof is to
|
||||
run that lane through live Scarlett loopback and mics, not only stored ASIO
|
||||
artifacts.
|
||||
with about 0.219 us error from rolling buffers. In live app/Well operation this
|
||||
is a configured and publishable evidence capability, not inline cadence
|
||||
authority: expensive matched-filter execution belongs to a worker queue that can
|
||||
commit cached reports back to the hub or CultMesh. The next real-world proof is
|
||||
to run that worker lane through live Scarlett loopback and mics, not only stored
|
||||
ASIO artifacts.
|
||||
`MimirCalibratedAudioCompositeBuilder` is the first C# diagnostic owner for
|
||||
turning that acoustic evidence into a corrected composite stream. It consumes
|
||||
sample windows plus `MimirAudioSynchronizationState` delay/confidence and
|
||||
per-band response evidence, applies fractional alignment, sparse measured-band
|
||||
equalization, calibrated-band coherence scoring, incoherent-band suppression,
|
||||
and confidence/noise-weighted summing, then emits composite samples plus source
|
||||
flatness/weight/coherence reports and per-band coherence/suppression reports.
|
||||
This is not the hot room-DSP
|
||||
authority and it does not make room nulls disappear; Faust/native DSP still
|
||||
owns the production alignment, suppression, separation, and stem lane.
|
||||
`Mimir.BufferSmoke --calibrated-audio-composite-smoke` proves the contract on
|
||||
synthetic delayed, colored, noisy mic paths: the uncalibrated mix measures
|
||||
-3.171 dB SNR, the calibrated composite measures 15.135 dB SNR, and response
|
||||
flatness improves from 0.548 to 1.000.
|
||||
`Mimir.BufferSmoke --dual-mic-coherence-cancellation-smoke` proves the
|
||||
two-witness cancellation cut: shared tones keep about 0.999 coherence, a
|
||||
mic-local 3.52 kHz interferer scores about 0.001 coherence and 0.948
|
||||
suppression, its measured energy falls from 0.039195 to 0.001306, and SNR rises
|
||||
from 1.366 dB to 16.304 dB.
|
||||
`scripts/record-kiyo-dual-mic-composite.ps1` is the first listenable Faust-chain
|
||||
bridge receipt for that path. It starts Kiyo FFmpeg capture and Scarlett ASIO
|
||||
raw capture together, converts ASIO channels 0/1 through
|
||||
`--asio-dual-mic-faust-cleaner-wav`, and muxes the Faust dialogue WAV with Kiyo
|
||||
video. The offline command compiles/runs
|
||||
`faust/mimir_alignment_actuator.dsp` first, then feeds the aligned
|
||||
shotgun/cardioid stems into `faust/mimir_dual_mic_dialogue_cleaner.dsp`; the
|
||||
older C# composite stays a scoring/audition contract, not the current listening
|
||||
receipt. The fresh 2026-06-01 receipt at
|
||||
`artifacts/runtime/kiyo-dual-mic-composite/20260601-051044` and
|
||||
`artifacts/runtime/kiyo-dual-mic-composite/kiyo-dual-mic-composite-latest.mp4`
|
||||
has 10.000 s of 640x480 30 fps Kiyo video and 10.000 s of 48 kHz mono AAC from
|
||||
the Faust dialogue stem. It estimated the cardioid 359.398 samples late
|
||||
(7487.452 us), applied holdback 359.398/0.000 samples to shotgun/cardioid, and
|
||||
wrote both dialogue and residual WAVs. The first muxed receipt was nearly
|
||||
silent because `NormalizePeak` only attenuated hot samples; after the helper was
|
||||
fixed to amplify quiet receipts too, the regenerated dialogue WAV peaks at
|
||||
-0.724 dBFS with RMS -14.202 dBFS. The actual denoise authority is Faust:
|
||||
`faust/mimir_dual_mic_dialogue_cleaner.dsp` compiles as a two-input/two-output
|
||||
aligned shotgun/cardioid dialogue cleaner with smooth multiband voice shaping,
|
||||
witness subtraction, and slow downward expansion. `MimirRuntime` queues it as
|
||||
`dual-mic-dialogue-cleaner` with `host_voice` and residual `ambient` output
|
||||
stems. It is fed from the alignment actuator's published `aligned_source_*`
|
||||
stem frame, not from raw mic buffers, so Faust alignment is a real structural
|
||||
precursor to cleanup. `Mimir.BufferSmoke --dialogue-cleaner-routing-smoke`
|
||||
proves the boundary: alignment frames queue the cleaner DSP and input block, and
|
||||
cleaner output frames are what enter `NativeProgram` stem publication. This is
|
||||
the first hot-lane cleaner socket, not proof that the final studio denoiser is
|
||||
solved.
|
||||
The right rail `Output` panel is the app feature for local broadcast-proof
|
||||
recording and live publish control, not the isolated Kiyo/Faust mic receipt.
|
||||
`MimirProgramRecorder` owns `Record` / `Stop`, child-process status, timestamped
|
||||
MP4 receipts under `C:\Users\Meta\Videos\Mimir` by default, and process-tree
|
||||
shutdown. Its current actuator is `scripts/record-stream-proof-output.ps1`, so
|
||||
app recording produces the same stream-proof composite intended for
|
||||
Yggdrasil/public judgment: Raven program feed, Kiyo picture-in-picture, Starfire
|
||||
Scarlett audio, and NVENC/AAC broadcast settings. `MIMIR_RECORD_START_RAVEN=1`
|
||||
asks the app to start Raven's sender for the receipt. `MimirProgramStreamer`
|
||||
owns `Live` / `Stop` for the `Mimir.Broadcast` RTMP path; it requires
|
||||
`MIMIR_STREAM_INPUT` unless `MIMIR_STREAM_ALLOW_TEST_PATTERN=1` explicitly
|
||||
authorizes a synthetic smoke stream. Later native recording/streaming should
|
||||
replace these actuators behind this owner, not add another UI truth.
|
||||
`MimirTargetedBioacousticProbePlanner` is the matching "ask the room a better
|
||||
question" owner. It consumes per-source spectral residuals and optional source
|
||||
pathology notes, then emits gain-bounded shaped probe bands for the
|
||||
bioacoustic renderer to spend energy on. The first smoke models the two
|
||||
professional 192 kHz Scarlett mics only: `asio-ch0-shotgun` is explicitly
|
||||
noisy and low-heavy, `asio-ch1-cardioid` is cleaner, and the planner prioritizes
|
||||
the shotgun's weak/unmeasured high-band evidence before lower residuals.
|
||||
`Mimir.BufferSmoke --targeted-bioacoustic-probe-smoke` selects four probe
|
||||
bands, led by about 18.6 kHz as an unmeasured band at the max bounded gain,
|
||||
then about 9.9 kHz weak response, 3.6 kHz phase instability, and 910 Hz
|
||||
magnitude residual. This is the intended first physical strategy: improve
|
||||
shotgun/cardioid confidence at 192 kHz before asking every connected mic to
|
||||
help solve the room field.
|
||||
`MimirBioacousticProbeScheduler` now owns the first closed-loop cadence layer
|
||||
above that planner. It reads smoothed sync states plus derived per-band
|
||||
frequency-response residuals from `MimirSynchronizationHub`, backs probe cadence
|
||||
off as aggregate sync/frequency confidence rises, emits targeted probe plans
|
||||
only when the confidence/duty budget says to spend sound, and exposes
|
||||
`mimir-bioacoustic-probe-schedule` plus per-source telemetry in `MimirRuntime`.
|
||||
`Mimir.BufferSmoke --bioacoustic-probe-scheduler-smoke` proves the visible
|
||||
climb: simulated probe feedback raises sync confidence from 0.360 to 0.920 and
|
||||
frequency-response confidence from 0.558 to 0.955 over four scheduled probes.
|
||||
The scheduler is not the output-device authority; in the current physical
|
||||
routing the eventual live renderer must explicitly select Starfire Realtek.
|
||||
That Realtek renderer now exists as a tooling edge. `Mimir.WasapiLoopback`
|
||||
targets `net10.0-windows`, enumerates active render endpoints, selects by
|
||||
friendly-name substring, and can play mono Float32 probe audio through the
|
||||
matched WASAPI render endpoint. `Mimir.BufferSmoke
|
||||
--targeted-bioacoustic-probe-realtk-live` renders a targeted probe, starts
|
||||
Scarlett ASIO capture, plays it through the selected Realtek speaker endpoint,
|
||||
then scores the captured shotgun/cardioid response. The first proof selected
|
||||
`Speakers (Realtek(R) Audio)` and wrote
|
||||
`artifacts/wasapi/targeted-bioacoustic-realtk-live/20260601-033851`; at gain
|
||||
0.75 the shotgun peak was 0.407887 and cardioid peak was 0.074494, but 18.6 kHz
|
||||
remained near-zero confidence while cardioid 3.6 kHz was the strongest printed
|
||||
band at 0.098 confidence. Realtek emission is now real; high-band probe choice
|
||||
is still a bad physical question for this room/speaker path. The default
|
||||
targeted probe planner now caps selection at 12 kHz and biases measurable
|
||||
low/mid bands instead of favoring ultrasonic-ish uncertainty that ordinary
|
||||
speakers and mics cannot answer well.
|
||||
Starfire's USB-connected PS Move is now a paired audio/visual timing actuator
|
||||
through `Mimir.PsMoveProbe --paired-chirp-train`. The worker renders one raw
|
||||
Float32 chirp train, asks `Mimir.WasapiLoopback` to emit it, and schedules Move
|
||||
LED on/off phases from the same `PairedChirpEvent` documents. The schedule is
|
||||
the owner: every visual pulse must share the same event id and planned offset
|
||||
as its audio chirp. The 2026-06-02 receipt
|
||||
`artifacts/runtime/starfire-usb-move-paired-20260602-144135/paired-events.jsonl`
|
||||
proved three emitted chirps and three USB LED pulses, with LED onsets matching
|
||||
planned 150/400/650 ms at millisecond receipt resolution after the HID train
|
||||
moved to high-resolution stopwatch waits.
|
||||
The calibration runner is now closed-loop instead of a blind full-spectrum
|
||||
sweep: `--bioacoustic-realtk-scheduled-calibration-live` asks the scheduler for
|
||||
each probe, emits through Realtek, captures Scarlett, updates residuals, and
|
||||
then asks the scheduler again. It is a live physical I/O diagnostic harness
|
||||
seeded from the current Scarlett residual model; production still needs measured
|
||||
runtime confidence state wired into the same scheduler/emitter path. The
|
||||
2026-06-01 receipt at
|
||||
`artifacts/wasapi/realtk-scheduled-calibration-live/20260601-035354` ran four
|
||||
scheduled steps at gain 1.15 and wrote `scheduled-response-bands.csv` plus
|
||||
`balance-plan.md`. Local peak-vs-percentile-floor scoring found strong usable
|
||||
response near 910 Hz, 3.6 kHz, and 9.84-9.87 kHz on both Scarlett mics.
|
||||
The scheduler now filters confidence/status to the same 120 Hz-12 kHz
|
||||
schedulable window as the targeted planner, so old 18.5 kHz residual scars can
|
||||
remain recorded without owning live cadence or weakest-band status.
|
||||
The first live run preserves two artifacts:
|
||||
`artifacts/asio/targeted-bioacoustic-live/20260601-030836` at playback gain
|
||||
1.0 and `artifacts/asio/targeted-bioacoustic-live/20260601-030857` at gain
|
||||
2.0. Loopback energy scaled, but 18.6 kHz stayed nearly absent in the physical
|
||||
mics; the second pass measured shotgun -51.475 dB and cardioid -56.585 dB
|
||||
relative to loopback at that band. The 910 Hz probe was strongest physically,
|
||||
especially shotgun at -21.994 dB relative to loopback. Current high-band
|
||||
probing should be treated as failed path evidence, not a calibration win.
|
||||
Louder follow-ups at gain 4 and gain 8 stayed below clipping; gain 8 reached
|
||||
shotgun peak 0.140715 and cardioid peak 0.027774. Even then, 18.6 kHz remained
|
||||
unusable at shotgun -54.910 dB and cardioid -63.258 dB relative to loopback.
|
||||
Bias the next probe language toward usable low/mid anchors unless the speaker,
|
||||
mic placement, or output path changes enough to make high-band evidence real.
|
||||
That real-world proof now has a first receipt:
|
||||
`calibration/bioacoustic/complex-contour-live-20260525-063229.md`. A freshly
|
||||
rendered canary-packet witness was played through Focusrite ASIO and captured
|
||||
|
|
@ -292,12 +467,158 @@ channel model loaded. This proves the contour/channel-model path survives a
|
|||
fresh DAC/speaker/room/mic/ADC pass, but low physical confidence means the next
|
||||
cut is stronger direct-path confidence and independently measured path truth,
|
||||
not a victory lap.
|
||||
A follow-up receipt,
|
||||
`calibration/bioacoustic/complex-contour-live-20260528-002313.md`, keeps the
|
||||
interface loopback proof healthy: `asio-ch2` to `asio-ch3` measured `-0.062 us`
|
||||
with `0.956` confidence and 257 direct hits. Physical inputs are not currently
|
||||
usable timing signals: `asio-ch0` and `asio-ch1` produced zero contour hits
|
||||
against 1480 reference hits, with very low RMS. The next physical proof is
|
||||
blocked on mic gain/routing/source placement before receiver logic changes.
|
||||
After arming both mics, `complex-contour-live-20260528-080604.md` showed both
|
||||
physical inputs were alive but was accidentally routed to headphones. The
|
||||
monitor-routed proof `complex-contour-live-20260528-080727.md` is the first
|
||||
credible fresh physical contour receipt: loopback is `-0.014 us` at 0.971
|
||||
confidence, `asio-ch0` lands `-0.410 us` from its stored path seed at 0.524
|
||||
confidence with 24 direct hits, and `asio-ch1` lands `-2.467 us` at 0.363
|
||||
confidence with 14 direct hits. User-provided physical mapping: `asio-ch0` is
|
||||
the shotgun, about 1.5 m from the right monitor and 3.0 m from the left;
|
||||
`asio-ch1` is the cardioid, about 0.5 m from the left monitor and 1.5 m from
|
||||
the right. Repeat captures are needed before updating the channel model or
|
||||
letting physical mic reports drive actuator authority.
|
||||
`complex-contour-monitor-split-20260528-081813.md` adds
|
||||
`--play-output-channel` to the ASIO probe and captures output 0 and output 1
|
||||
separately. Loopback proves the ASIO output buffers are distinct, but physical
|
||||
mic delays are nearly identical across outputs: shotgun differs by about
|
||||
20.885 us and cardioid by about 24.789 us. That does not match the declared
|
||||
left/right monitor geometry, so the downstream monitor path is probably
|
||||
summing, mirroring, or otherwise not isolating the physical speakers.
|
||||
The first actuator proof now exists: `faust/mimir_alignment_actuator.dsp` owns
|
||||
six channels of bounded fractional delay/gain controls for Faust/native DSP,
|
||||
and `Mimir.BufferSmoke --bioacoustic-actuator-self-test --sample-rate 48000
|
||||
--delay-samples 317.375` proves the control loop shape by estimating a
|
||||
bioacoustic delay, applying fractional correction, and remeasuring residual
|
||||
below printed microsecond precision.
|
||||
below printed microsecond precision. `MimirAlignmentActuatorBank` is now the
|
||||
runtime command owner above that Faust surface: it converts smoothed sync states
|
||||
into nonnegative holdback commands, keeps source-to-`sourceN` slots stable,
|
||||
reports overflow beyond the six-source profile, and exposes the reference
|
||||
holdback separately. `MimirRuntime` updates that command frame on the sync
|
||||
analysis cadence, queues it into `AquariumAudioDocument` as an engine audio
|
||||
control frame, declares `faust/mimir_alignment_actuator.dsp` as an
|
||||
`AquariumStreamingDspProgram`, and telemetry can print the current DSP targets.
|
||||
Mimir also queues sample-bearing `AquariumStreamingAudioBlock` lanes from the
|
||||
latest actuator source buffers, using the bank-assigned Faust `sourceN` control
|
||||
path as channel authority. Fensalir can compile/host/process that persistent
|
||||
Faust graph now. The engine now publishes processed output lanes as
|
||||
`AquariumAudioStemFrame` records through `IAquariumAudioStemBus`, with Mimir's
|
||||
actuator program declaring deterministic aligned-source stem names. OBS-facing
|
||||
publication now has a first readiness surface:
|
||||
`MimirObsStemPublicationState` consumes those frames without copying audio and
|
||||
reports ready, missing, and unconfigured stems against the alignment actuator
|
||||
stem-bus profile. `MimirObsStemSharedMemoryPublisher` writes the validated stem
|
||||
frames into `Local\MimirObsStemBus`, and `native/obs_stem_source` is the first
|
||||
OBS source plugin bundle: `Mimir Audio Stem` reads one named stem from that map
|
||||
and submits it through libobs audio, while `Mimir Program Texture` opens
|
||||
Fensalir's named shared D3D12 program-output texture and draws it through an
|
||||
OBS-readable shared D3D11 bridge texture. The video path is GPU-to-GPU and does
|
||||
not require Spout2 or CPU readback; the remaining copy is the explicit D3D12 to
|
||||
libobs-D3D11 boundary. Fensalir publishes `Global\MimirFensalirProgramFence` as
|
||||
a program-output publication fence, and the OBS source opens that fence before
|
||||
copying. This prevents blind read-before-producer-completion without coupling
|
||||
OBS to Fensalir's private frame fence. An optional texture ring is available
|
||||
when Fensalir and OBS agree on `FENSALIR_PROGRAM_OUTPUT_RING_COUNT`; OBS selects
|
||||
the latest completed slot from the program-output fence value. OBS can publish
|
||||
`Global\MimirObsProgramConsumerFence`; when Fensalir is launched with
|
||||
`FENSALIR_PROGRAM_OUTPUT_CONSUMER_FENCE_NAME` pointing at that fence, it skips
|
||||
publication instead of reusing a ring slot OBS has not acknowledged.
|
||||
`scripts/build-obs-stem-plugin.ps1` stages the upstream OBS plugin-template SDK
|
||||
under `artifacts/obs-sdk/` and builds the plugin DLL locally.
|
||||
The rendering migration's program-output receipt phase is structurally
|
||||
complete: OBS consumes final Mimir/Fensalir surfaces and stems through the
|
||||
native plugin, raw feeds remain debug inputs, and the ledger records the live
|
||||
program texture, texture ring, dedicated producer fence, and consumer-fence
|
||||
proofs.
|
||||
The public self-hosted edge has its first deployable path. `src/Mimir.Broadcast`
|
||||
builds the Starfire-side FFmpeg/NVENC push command for
|
||||
`rtmp://127.0.0.1:11935/live/mimir`; the port is an SSH local forward to
|
||||
Yggdrasil's localhost RTMP listener. Yggdrasil owns nginx RTMP ingest and HLS
|
||||
segment serving only. It must not transcode or compose in v1. The public static
|
||||
viewer route is `https://gamecult.org/livestream`, owned by the root
|
||||
`gamecult-site` repo, and it reads
|
||||
`https://streampixels.gamecult.org/mimir/live/hls/mimir.m3u8`. On 2026-05-30
|
||||
the deployed Yggdrasil origin validated nginx, listened on `127.0.0.1:1935`,
|
||||
returned `mimir-live-ok`, and generated four HLS segments plus `mimir.m3u8`
|
||||
from an eight-second Starfire synthetic NVENC push. `live.mimir.gamecult.org`
|
||||
is no longer required for v1 because StreamPixels already has DNS and TLS on
|
||||
Yggdrasil, and Mimir's own static site does not own the viewer.
|
||||
`MimirSynchronizedBufferPlanner` is now the low-level aligned-buffer primitive:
|
||||
it picks one canonical presentation time inside the retained rolling window and
|
||||
returns per-stream slices for cameras, network display feeds, and audio. Timing
|
||||
corrections may target either one source or a shared `clockDomainId`. The Raven
|
||||
shape is explicit: Raven controls both display pixels and an audio timing signal
|
||||
routed into Scarlett, so Scarlett-decoded Raven audio evidence can earn a
|
||||
`raven-sync` correction that is applied to the `raven-display` buffer. Network
|
||||
arrival timestamps remain metadata, not timing authority.
|
||||
The first operational ingest config is `config/mimir-runtime.raven-eve.example.json`:
|
||||
Raven screen capture arrives over SRT and decodes to raw BGRA through FFmpeg;
|
||||
Eve camera/mic arrive as EveCanvas-native WebSocket frame-events. All three
|
||||
enter the same rolling-buffer model as local sources. Eve's sensor authority now
|
||||
lives in EveCanvas, not an external camera-streaming app.
|
||||
`MimirPresentationControlState` now owns the Fensalir program-control intent:
|
||||
video feed visibility/solo/opacity/layer order, audio mute/solo/gain, and
|
||||
global LUT preset selection. The `Mimir Program` panel exposes those controls
|
||||
without giving OBS or bridge endpoints composition authority. Video controls
|
||||
filter production surface intents before Fensalir composition; audio controls
|
||||
modify Faust gain controls and sample gain before Fensalir streaming DSP. LUT
|
||||
presets are typed postprocess state, with exposure/bloom currently mapped into
|
||||
`GraphicsSettings` and LUT texture sampling left as the next renderer hook.
|
||||
`MimirSceneEditorState` is now the separate Mimir-window editor owner. It owns
|
||||
the editor camera, dynamic scene nodes, selected node, visibility/lock state,
|
||||
2D-plane transforms, reset commands, and grab/rotate/resize gizmo mode. Rolling
|
||||
video buffers derive sensor-feed panel nodes; SDF text panels and model import
|
||||
requests create editor graph nodes. `MimirRuntime` presents that editor through
|
||||
the Fensalir camera plus derived spline outlines, selection handles, and the
|
||||
`Mimir Editor` hierarchy/transform panel. The center preview requests a
|
||||
Fensalir-owned blit of the current output buffer, then draws Mimir's normalized
|
||||
placement overlays on top. This editor is not the OBS program output authority;
|
||||
it is a live program-surface witness. The placement commit primitive keeps
|
||||
layers inside the program frame, snaps near edges/center, and preserves source
|
||||
aspect while resizing through direct preview handles, sliders, or the scale
|
||||
gizmo. Explicit transform/source controls are inspector-only and gated to a
|
||||
selected program source; the center canvas handles are the primary compositor
|
||||
edit path. Empty-canvas clicks clear source selection; hover/active handles and
|
||||
snap guides come from the shared Eve dashboard document state that Fensalir's
|
||||
Direct2D and D3D12 paths lower locally; arrows nudge the selected layer,
|
||||
PageUp/PageDown reorder layers, and the inspector owns the selected source
|
||||
contain/cover/stretch fit mode. Eve is the portable dashboard authority, while
|
||||
CultUI is only the C# reference API shape for the document. The shared element
|
||||
contract now includes typed CultCache/CultMesh field bindings: document
|
||||
schema/id, field path, value kind, access, authority, and optional command id.
|
||||
Callbacks are native lowering actuators; the binding is the portable
|
||||
Verse-backed state contract. World SDF glyph rendering, ASSIMP-style
|
||||
mesh decoding/upload, and pixel-accurate gizmo hit-testing are explicit
|
||||
Fensalir renderer cuts, mapped in
|
||||
[[docs/scene-editor-control-surface|Mimir Scene Editor Control Surface]].
|
||||
`Mimir.EveDashboard` is now a small native Eve dashboard broker. It serves
|
||||
`/eve/deck` plus the compatibility `/eve/dashboard` endpoint, exposes provider
|
||||
manifests at `/eve/deck/providers`, and routes retained dashboard snapshots and
|
||||
commands for provider-owned trees. Built-in providers are the switchboard,
|
||||
`mimir.stream.layout`, `voidbot.swarm`, and `yggdrasil.streampixels.edge`;
|
||||
launch-time external providers can be registered as `id|title|ws-url`
|
||||
descriptors for LAN or SSH-forwarded TCP surfaces. EveCanvas renders the
|
||||
dashboard locally in UIKit, including the scene graph and multitouch
|
||||
pan/pinch/rotate source panels, and now treats provider-card taps as
|
||||
`open-provider` commands. `voidbot.swarm` reads VoidBot's existing
|
||||
`.voidbot/status/swarm-state.json` projection, publishes avatar URLs on
|
||||
identity nodes, and Eve renders a native VoidBot-style cockpit: CTB rail,
|
||||
selected Face/status pane, state tree, and detail panel.
|
||||
Dashboard manifests and states have typed CultMesh-compatible document shapes
|
||||
`mimir.eve_dashboard_manifest.v1` and `mimir.eve_dashboard_state.v1`, while the
|
||||
live iPad lane remains WebSocket/TCP because it is the proven tunnelable
|
||||
transport. `mimir.eve_dashboard_state.v1` now mirrors each retained element's
|
||||
optional binding object so clients can move fields into CultCache without
|
||||
turning `bindNodeId` into a second authority. The Mimir stream-layout provider
|
||||
still uses fixture state; the next cut is binding it to live
|
||||
`MimirPresentationControlState` and `MimirSceneEditorState`.
|
||||
|
||||
## Visual Fusion
|
||||
|
||||
|
|
@ -305,23 +626,299 @@ Visual fusion belongs in Fensalir over current reservoir claims. Native capture
|
|||
workers provide frames; Fensalir owns feature extraction, matching, material
|
||||
fitting, render budgeting, and publication.
|
||||
|
||||
The live debug spectrum view now has two separate authorities:
|
||||
Fast stereo depth is now mapped as a future Fensalir D3D12 compute lane, not an
|
||||
external library dependency. The provenance note
|
||||
`research/d3d12-stereo-depth-provenance-2026-05-29.md` records `libSGM` as the
|
||||
current permissive north-star reference: Mimir/Fensalir should rebuild the SGM
|
||||
shape over synchronized rectified texture pairs, calibration state, typed GPU
|
||||
resources, and explicit fences. CUDA, TensorRT, and monocular depth demos remain
|
||||
research references only.
|
||||
The first live contract exists: `MimirStereoDepthConfigurations` names the
|
||||
libSGM-provenance D3D12 SGM profile, and
|
||||
`MimirFensalirFieldLowering.BuildStereoDepthCandidateFrame` lowers caller-
|
||||
declared rectified stereo input textures into a GPU-resident compute-writable
|
||||
disparity SurfacePage, confidence texture, Height FieldEvidence claim, and
|
||||
`AquariumFieldStereoDepthLowering` sidecar. The live
|
||||
`BuildLeapPackedStereoDepthCandidateFrame` path now recognizes a live
|
||||
`LeapStereoIr` rolling video window, reuses its declared packed R8G8 texture as
|
||||
both left and right resource identity, names the left/right observations as
|
||||
packed R/G lanes, and emits the same depth lowering automatically from the
|
||||
runtime frame. The lowering sidecar is the dispatch contract: it binds the
|
||||
libSGM-provenance profile, calibration id, camera pair, inputs, disparity
|
||||
output, min disparity, disparity levels, aggregation paths, census radius,
|
||||
P1/P2 smoothness penalties, and depth range. The current D3D12 SGM profile
|
||||
publishes min disparity 0, 128 disparities, four paths, census radius 2, P1 8,
|
||||
and P2 96. Fensalir now has a first packed-Leap D3D12 kernel that writes R16F
|
||||
disparity through a crude SAD/block-match pass. This is intentionally a small
|
||||
kernel-shaped proof, not full libSGM-style SGM or calibrated metric depth yet.
|
||||
The first point-cloud root is also code-visible now:
|
||||
`MimirPointCloudConfigurations.LeapDisparityPointCloudRoot` records the
|
||||
stereo-disparity projection profile and provenance, while
|
||||
`MimirFensalirFieldLowering.BuildLeapPackedStereoPointCloudCandidateFrame`
|
||||
declares a GPU-resident `FieldMesh` point-list resource derived from the Leap
|
||||
R16F disparity SurfacePage. `MimirRuntime` merges that Mesh claim into the same
|
||||
evidence frame as the Leap stereo-depth lowering. `Mimir.BufferSmoke
|
||||
--leap-point-cloud-root-smoke` proves the combined depth/point-cloud contract
|
||||
plans as one `SurfacePage` packet and one `Mesh` packet with zero deferred
|
||||
requests. Fensalir branch `codex/leap-packed-depth` now has the first native
|
||||
surface-claim lane too. It preserves Mesh `SourceUri=derived-from:*`, allocates
|
||||
generated point-list vertex/index buffers as UAV-capable GPU resources, fills
|
||||
them from the disparity SurfacePage in `D3D12PointCloudFromDisparityCS`, and
|
||||
writes approximated normals, curvature, SDF, PBR, and radiosity reservoirs into
|
||||
the existing fractal splat buffers. The point-list draw path is diagnostic-only;
|
||||
the verified live presentation path is Fensalir's splat/reservoir resolver.
|
||||
The 2026-05-31 live capture
|
||||
`artifacts/runtime/mimir-leap-surface-claims-clean.png` proves the clean path:
|
||||
LeapUVC frames uploaded through a Fensalir-owned D3D12 texture, packed stereo
|
||||
dispatched to R16F disparity, point-cloud compute generated 19,200 surface
|
||||
claims, the point-list mesh renderer stayed at `rendered 0`, and the reservoir
|
||||
resolver presented visible Leap-derived splats. The follow-up
|
||||
`artifacts/runtime/mimir-leap-history-split-shader-visible.png` proves the
|
||||
history pass now starts without `AQUARIUM_DISABLE_RESERVOIR_HISTORY`: Fensalir
|
||||
compiles a dedicated lean `D3D12ReservoirHistoryUpdate.hlsl` and writes the
|
||||
current resolver sample into the four-row history/resolved target. Temporal and
|
||||
spatial reuse are deliberately not restored in that lean shader yet; the
|
||||
remaining truth gaps are calibrated projection constants, full SGM, temporal
|
||||
reservoir reuse split into a compilable shader, and a global
|
||||
residual/calibration owner.
|
||||
|
||||
- `AquariumBufferFieldFrame` is the real Fensalir contract. Mimir expresses
|
||||
each rolling audio spectrum window as a buffer-backed spline tube field:
|
||||
frequency lives in the spline domain, amplitude shapes the tube path, and
|
||||
appearance/probe policy can reference tangent, curvature, normal, derivative,
|
||||
object domain, and parent-space transforms. This is the surface the future
|
||||
compute lowering consumes.
|
||||
- `AquariumSplineFrame` is only the immediate preview witness. It renders the
|
||||
same spectrum windows as camera-facing Catmull-Rom tube ribbons so humans can
|
||||
sanity-check ASIO loopback and mic buffers before the reservoir path owns the
|
||||
draw. It must not become the final visual authority.
|
||||
Fensalir must not be treated as a traditional rendering pipeline. Mimir does
|
||||
not ask it to "draw a thing" and hope post-processing makes the result true.
|
||||
Mimir submits evidence, buffers, constraints, and surface intent; Fensalir
|
||||
lowers those into field claims with travel/depth, metadata, control, and
|
||||
reservoir-guide lanes. If a path only paints pixels, it is a fallback/debug
|
||||
draw, not the Perfect Machine surface.
|
||||
|
||||
The intended Fensalir lowering is buffer field expression -> compute-generated
|
||||
stochastic SDF splat probes -> spatiotemporal splat reservoir -> reservoir
|
||||
sampling for temporally antialiased surface contribution. Splat density is a
|
||||
function of visual contribution, not uniform polyline tessellation.
|
||||
The live payload boundary is now explicit. Mimir declares each live native/GPU
|
||||
payload view as an `AquariumFieldResourceDeclaration` with resource key, kind,
|
||||
residency, shader access, format, shape, valid time range, version, and native
|
||||
handle metadata. Field claims and lowering requests reference those
|
||||
`mimir:resource:*` keys. The handle string is only a name; Fensalir validation,
|
||||
planning, and renderer-owned resource slots own typed resource resolution before
|
||||
shader lowering. Mimir-declared rendering buffers are Fensalir GPU resources in
|
||||
the same runtime; native/shared handle metadata is an import edge, not a
|
||||
separate authority regime. Rendering-relevant buffers move to GPU residency as
|
||||
early as possible and stay there; CPU readback/tessellation is diagnostic only.
|
||||
Camera descriptors now enter this bridge too. The preferred hot path is
|
||||
Fensalir-owned texture leasing: the runtime receives `AquariumRuntimeServices`,
|
||||
asks Fensalir's field resource broker for a keyed D3D12 `Texture2D` lease, the
|
||||
camera/decode producer writes that shared texture and signals the producer
|
||||
fence, then Mimir carries the same resource key through FieldEvidence. Fensalir
|
||||
waits on committed producer fence values before resolving shader reads. Shared
|
||||
foreign texture handles remain an import edge, not the primary camera authority.
|
||||
`MimirVideoCaptureDriverSource` propagates the lease client to drivers that
|
||||
implement `IMimirFensalirTextureLeaseReceiver`, so direct camera drivers can
|
||||
request the destination texture before emitting a frame descriptor.
|
||||
Each camera backend must use the closest-to-device path available and expose
|
||||
its unavoidable copy count. Managed/process wrappers are diagnostic bring-up
|
||||
surfaces, not the production camera hot path.
|
||||
For raw single-plane system-memory frames, `MimirVideoCaptureDriverSource`
|
||||
performs the one declared CPU-to-GPU upload into the Fensalir lease, drops the
|
||||
managed payload from the live sample, and increments `UnavoidableCopyCount`.
|
||||
NV12 CPU upload is rejected until Fensalir owns a real planar upload path;
|
||||
device/GPU NV12 producers should write the leased texture directly.
|
||||
`MimirKsVideoCaptureDriver` plus `native/camera_capture/mimir_camera_capture.dll`
|
||||
is the first production-shaped KS/UVC driver: it opens a Kernel Streaming pin
|
||||
in process, queues uncompressed UVC frames, and feeds the same
|
||||
`IMimirVideoCaptureDriver` source path. MJPG/H264 still need a device/GPU
|
||||
decode producer; the KS driver rejects compressed capture formats.
|
||||
`MimirPs3EyeVideoCaptureDriver` plus
|
||||
`native/camera_capture/mimir_ps3eye_capture.dll` does the same for PS3 Eye raw
|
||||
WinUSB/libusb capture, emitting Bayer8 frames through the common upload lane.
|
||||
Driver smokes have pulled and uploaded real frames for LeapUVC 640x240 YUY2,
|
||||
both PS3 Eyes at 320x240 Bayer8, Kiyo Pro 1920x1080 YUY2, and regular Kiyo
|
||||
640x480 YUY2. Regular Kiyo 1280x720 YUY2 did not open.
|
||||
`config/mimir-runtime.perfect-machine.local.json` is the current local
|
||||
all-sensor runtime profile: Leap packed stereo IR, both PS3 Eyes at
|
||||
320x240@187, Kiyo Pro RGB, and regular Kiyo RGB. `Mimir.BufferSmoke
|
||||
--perfect-machine-live-smoke` opens all five video sources through the runtime,
|
||||
attaches Fensalir texture leases, and requires five camera Texture2D resources,
|
||||
one Leap stereo-depth lowering, one disparity SurfacePage, and one derived
|
||||
point-cloud Mesh. Fensalir now plans those camera Feature/SensorObservation
|
||||
claims through its `GpuSensorFusion` backend instead of deferring them, then
|
||||
derives compute input metadata and SRVs from resolved FieldEvidence Texture2D
|
||||
resources. The current dispatch samples LeapStereoIr, both Bayer8 PS3 Eyes,
|
||||
Kiyo Pro YUY2, and regular Kiyo YUY2. Fensalir binds YUY2 through the
|
||||
Microsoft-documented `R8G8B8A8_UNORM` SRV view over `DXGI_FORMAT_YUY2` and
|
||||
decodes `Y0/U/Y1/V` in compute; packed camera video is not silently treated as
|
||||
ordinary RGBA.
|
||||
`MimirMediaFoundationGpuVideoCaptureDriver` plus
|
||||
`native/camera_capture/mimir_mf_gpu_capture.dll` is the compressed camera path:
|
||||
Media Foundation SourceReader runs with a D3D11 device manager, decodes MJPG or
|
||||
H264 camera frames on the GPU, copies the decoded GPU surface into a shared BGRA
|
||||
D3D11 texture, and publishes a `shared-d3d11-texture` handle with no live CPU
|
||||
payload bytes. Kiyo Pro 1920x1080 MJPG->RGB32 and H264->RGB32 smokes both
|
||||
produced valid GPU handles/resources. Direct NV12 shared texture creation
|
||||
failed in D3D11, so NV12 needs plane-aware interop or a GPU bridge-copy cut.
|
||||
Metadata-only cadence frames do not create camera surface intents or fake
|
||||
payload requests. The old direct `AquariumGpuSensorFrame` bridge proof has been
|
||||
removed from Mimir's active proof path. Camera image claims currently defer
|
||||
until Fensalir owns a selected visual-fusion lowering for camera textures.
|
||||
The current smokes prove planning, not visible packet rendering:
|
||||
`--fensalir-field-evidence-smoke` now produces one planned resource-backed
|
||||
`TubeField` packet from Mimir's spectrum intent, and
|
||||
`--fensalir-field-dsl-resource-smoke` proves the Fensalir DSL can bind the same
|
||||
kind of declared resource directly. `--fensalir-camera-observation-smoke`
|
||||
proves the camera observation/resource split. Fensalir now has the first D3D12 resolver
|
||||
cut for structured/curve buffers: it imports shared GPU buffers by handle and
|
||||
allocates engine-owned GPU slots only for Fensalir-produced resources. The DSL
|
||||
can describe a 2D rolling float buffer as Catmull-Rom XY tubes with modulo
|
||||
column addressing, amplitude power/normalization, radius, ramp texture path, and
|
||||
emission scale. Fensalir now resolves the first non-buffer resource family too:
|
||||
Texture2D local assets bind as TubeField ramps, surface pages and volume
|
||||
textures allocate shader-readable GPU textures, and mesh packages own
|
||||
`Mesh.Vertices`/`Mesh.Indices` GPU buffer shape under one resource key. The DSL
|
||||
can plan generic resource-backed claims over those declarations. The next
|
||||
blocker is no longer resolver ownership; it is selected render lowerings that
|
||||
consume camera/feature textures, mesh/page/volume resources as geometry,
|
||||
height/SDF/material pages, or density/extinction/SDF3D domains. Mesh layout
|
||||
authority is split by source:
|
||||
imported/user meshes use the standard `PositionNormalUvColor` layout, and
|
||||
generated meshes can be `PipelinePrivate` so each lowering owns the bytes it
|
||||
emits and consumes. TubeField is the first concrete generated-mesh consumer:
|
||||
its compute pass emits private vertex/index/indirect buffers and render binds
|
||||
them as `D3D12PipelinePrivateGeneratedMesh` before applying source/ramp/material
|
||||
state. The DrawIndexed indirect command signature has been lifted to the
|
||||
generated-mesh lane instead of being TubeField-owned. TubeField expansion now
|
||||
requires a planned TubeField backend packet for the lowering's claim, and
|
||||
validation rejects TubeSpline metadata whose claim/resource authority is split.
|
||||
Mimir's typed surface-intent lowering now emits the matching TubeSpline
|
||||
lowering for audio spectrum/waveform Tube claims.
|
||||
Mimir's active proof path also no longer uses the direct
|
||||
`AquariumAcousticFieldFrame` builder: sync states lower into FieldEvidence
|
||||
calibration constraints. Fensalir now selects audio-path `Phase` and
|
||||
`Confidence` claims as `DebugOverlay` backend packets, so calibration
|
||||
timing/path evidence can be inspected through the same FieldEvidence authority
|
||||
while true acoustic source candidates and confidence volumes remain later
|
||||
fusion work. The first synthetic acoustic source candidate proof now exists:
|
||||
Mimir lowers an SRP/PHAT-style localized source with calibration/source
|
||||
identity and a world-space confidence envelope, and Fensalir plans it as a
|
||||
FieldEvidence `DebugOverlay` packet. Live room geometry must still come from an
|
||||
explicit measured coordinate frame, not inferred from the recent two-distance
|
||||
monitor notes. Phase 5's synthetic marker proof now exists too: Mimir lowers a
|
||||
deterministic multi-camera marker candidate with calibration/marker identity and
|
||||
world-space support, and Fensalir plans it as `DebugOverlay` evidence while
|
||||
ambiguous raw camera features still defer.
|
||||
LED strips now have a typed calibration-evidence socket instead of being folded
|
||||
into fake depth. `MimirLedSplineFieldCandidate` carries ordered per-camera
|
||||
bright-curve samples in that sensor's image plane and local clipspace/frustum,
|
||||
whether the LEDs were temporally coded, and whether the sample indices are
|
||||
stable. `BuildLedSplineCandidateFrame` lowers that shared curve as
|
||||
deterministic camera-sensor Feature evidence; Fensalir plans it as a
|
||||
`DebugOverlay` packet through the same FieldEvidence authority. Addressable or
|
||||
blink-coded strips can provide exact cross-camera LED identity. Plain identical
|
||||
white LEDs still provide a strong spline/epipolar constraint, but their point
|
||||
correspondence is ambiguous under occlusion, saturation, repeated spacing, and
|
||||
motion blur until temporal continuity, Leap/body anchors, or the residual
|
||||
solver disambiguates it. `MimirLedSplineCurveSolver` is the first per-sensor
|
||||
curve owner: it orders bright detections and computes image/clipspace curve
|
||||
samples. `MimirCameraRigCalibrationSolver` is the first higher-level frustum
|
||||
fit owner: given stable LED indices and a scene-space LED curve anchor, it
|
||||
minimizes ray-to-curve distance and emits bounded camera translation updates.
|
||||
Individual cameras do not write calibration state. Rotation, focal length, and
|
||||
distortion solving are still future cuts, not hidden behavior. Exposure/gain
|
||||
sweeps are scored by the recovered spline, not raw brightness:
|
||||
`MimirLedSplineQualityScorer` combines detected LED count, curve coverage,
|
||||
spacing coherence, smoothness, exposure fitness, and saturation penalty, and
|
||||
`MimirLedSplineSweepSelector` picks the best calibration-usable setting. Once
|
||||
`MimirLedSplineFrameAnalyzer` connects physical frames to that score by
|
||||
extracting bright luma components, solving the sensor-local curve, and rejecting
|
||||
clipped blobs as unusable calibration evidence even when they are easy to
|
||||
detect. `Mimir.BufferSmoke --led-spline-live-score` is the current live sweep
|
||||
surface: KS/UVC cameras get manual exposure/gain writes before each scoring
|
||||
window, stale frames are drained, luma is extracted from the actual live frame
|
||||
format, and the best spline setting is reported per source. On 2026-05-31 with
|
||||
`--expected-leds 38 --minimum-luma 0.55`, both Kiyo cameras selected exposure
|
||||
`-9`, gain `8` (`kiyo-pro-rgb` score `0.666`; `kiyo-basic-rgb` score `0.650`).
|
||||
LeapUVC rejected generic KS exposure/gain writes and only found two LED
|
||||
components; PS3 Eye feeds are still fixed-control until the WinUSB wrapper owns
|
||||
gain/exposure. PS3 Eyes now also have a sparse tracking bootstrap:
|
||||
`MimirSparseFeatureTracker` detects grid-suppressed Bayer/luma corner features,
|
||||
links them frame-to-frame, and emits `MimirFeatureTrackCameraObservation`
|
||||
records with sensor-local image/clipspace position, velocity, age, and
|
||||
confidence. `MimirFeatureTrackFieldCandidate` lowers those observations as
|
||||
camera Feature claims through Fensalir field evidence. A one-second live pass
|
||||
at 320x240@187 with sensitive defaults returned 160 live tracks from each Eye:
|
||||
160 stable tracks from `ps3-eye-0` and 156 stable tracks from `ps3-eye-1`.
|
||||
This is a CPU diagnostic owner; the hot production path remains Fensalir D3D12
|
||||
feature extraction. A full 2026-05-31 Fensalir headless run using the
|
||||
all-sensor profile started all five video sources, declared eight field
|
||||
resources, resolved six Texture2D resources, dispatched one packed-Leap
|
||||
stereo-depth kernel, generated one point-cloud/surface stream, reported 19,200
|
||||
visible reservoir splats, and captured
|
||||
`artifacts/runtime/mimir-perfect-machine-all-sensors-headless.png`. The
|
||||
follow-up GPU feature-extraction proof planned 12 packets with zero deferred
|
||||
requests, dispatched sensor fusion over three sampleable camera textures,
|
||||
generated 49,152 sparse gaussians at about 0.059 ms/frame sensor-fusion GPU
|
||||
time, and captured
|
||||
`artifacts/runtime/mimir-perfect-machine-gpu-feature-extraction-headless.png`.
|
||||
The YUY2 decode proof then dispatched all five camera textures with zero
|
||||
unsupported field textures, generated 81,920 sparse gaussians at about 0.08
|
||||
ms/frame sensor-fusion GPU time, and captured
|
||||
`artifacts/runtime/mimir-perfect-machine-yuy2-gpu-feature-extraction-headless.png`.
|
||||
Once the
|
||||
frustum fit is coherent, detected
|
||||
surface candidates can be sampled across every synchronized view before
|
||||
Fensalir resolves them into 4D Gaussian/surface splat claims.
|
||||
|
||||
The current teardown/migration map is
|
||||
`docs/fensalir-rendering-rebuild-migration.md`, paired with Fensalir's
|
||||
`docs/rendering-teardown-rebuild-protocol.md`. The intended cut is explicit:
|
||||
Mimir publishes typed physical observations, calibration constraints, and
|
||||
surface intent; Fensalir owns field domains, field claims, candidate selection,
|
||||
track/reservoir state, selected lowerings, temporal guide lanes, and
|
||||
presentation.
|
||||
|
||||
The reservoir can resolve pixel-level claims without requiring pixel-sized
|
||||
contents. Claim support belongs to the represented domain. A smooth flat surface
|
||||
can be a few huge surface claims; a heightfield terrain belongs in a quadtree of
|
||||
brush-painted surface tiles, subdivided only where projected footprint,
|
||||
curvature, material/brush detail, silhouette risk, or temporal uncertainty make
|
||||
larger claims dishonest. Fensalir's SDF probe/lowering stage should emit
|
||||
surface splats at that detail-adaptive density, not one splat per pixel by
|
||||
default.
|
||||
|
||||
The live debug spectrum view is now field-evidence owned. Mimir no longer
|
||||
submits direct `AquariumSplineFrame` trails for the spectrum dashboard and does
|
||||
not submit `AquariumBufferFieldFrame` / fractal `ReservoirSplats` for it either.
|
||||
Mimir declares the rolling resource, publishes a Tube claim plus
|
||||
`AquariumFieldTubeSplineLowering`, and Fensalir owns planning, generated mesh
|
||||
expansion, and TubeField material rendering. The spectrum resource is not just
|
||||
a handle: Mimir uploads the current spectral amplitudes as row-major Float32
|
||||
data into a Fensalir-owned GPU structured buffer before TubeField compute reads
|
||||
it. TubeField writes stable field ids, real tube normals, coverage/confidence,
|
||||
and domain-validity guide data into the same scene metadata/control/reservoir
|
||||
guide targets consumed by reservoir resolve for spatiotemporal reconstruction.
|
||||
Fensalir's shared reservoir history update now runs as a compute pass over
|
||||
four structured rows per pixel and emits the resolved HDR field texture for
|
||||
bloom/presentation, so the presentation shader is no longer a hidden history
|
||||
owner.
|
||||
TubeField now feeds the shared reservoir from GPU-resident rolling-buffer
|
||||
column packets instead of generated segment packets. Fensalir emits one column
|
||||
packet per logical spectrum history/source column, bins those columns into
|
||||
screen tiles, and evaluates Catmull-Rom tube SDF/material samples directly from
|
||||
the source buffer during the ReSTIR passes. Generated mesh draw args are zeroed
|
||||
for the live path and remain only a diagnostic reference.
|
||||
`Mimir.BufferSmoke --mimir-spectrum-upload-smoke` verifies this through the
|
||||
actual `MimirRuntime` frame path and confirms the legacy direct spline and
|
||||
buffer-field dashboard inputs are empty. The same path declares the local
|
||||
blackbody ramp as a Texture2D field resource and binds the TubeField material by
|
||||
resource key. Runtime spectrum columns now represent `(history age, source
|
||||
lane)` pairs rather than only the latest spectra; Mimir emits one TubeField
|
||||
claim/lowering per source, each striding through the shared resource to render
|
||||
its own age trail at `z = age * 0.1`. The resource advertises fixed history and
|
||||
source-lane capacities, so content updates and source topology changes do not
|
||||
resize the GPU buffer while the rolling trail fills. `MIMIR_SPECTRUM_SOURCE_LANES`
|
||||
sets the fixed lane budget; surplus sources are truncated and reported in the
|
||||
runtime UI. Physical history slots are now addressed as a ring with
|
||||
`RollingOffset`, so logical age no longer requires moving every column before
|
||||
shader sampling. Mimir emits one Float32 resource upload for the newest
|
||||
spectrum ring slot with an explicit element offset instead of uploading the
|
||||
full fixed-capacity backing buffer every frame. Fensalir owns rolling-slot
|
||||
validity and clamps TubeField dispatch so invalid older slots are not sampled
|
||||
after renderer allocation, reset, or partial update.
|
||||
`MIMIR_SPECTRUM_TUBE_SUBDIVISIONS` is the runtime cost lever for lowering
|
||||
Catmull-Rom subdivision count when Fensalir reports TubeField truncation.
|
||||
|
||||
## Known Risks
|
||||
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@ Get-Content .\state\evidence.jsonl -Tail 8
|
|||
|
||||
## Current Shape
|
||||
|
||||
- Public brand/Face is Mimir.
|
||||
- Mimir's Face doctrine lives in [[docs/mimir-face|Mimir Face]].
|
||||
- Public brand/Persona is Mimir.
|
||||
- Mimir's Persona doctrine lives in [[docs/mimir-face|Mimir Persona]].
|
||||
- VoidBot persona/state live under `.voidbot/voice/` and `.voidbot/state/`.
|
||||
Commit `.voidbot` dirt in a separate state commit whenever it appears.
|
||||
- V1 still has FFmpeg/SRT/OBS bridge utilities for LAN ingest.
|
||||
|
|
@ -43,13 +43,199 @@ Get-Content .\state\evidence.jsonl -Tail 8
|
|||
- `native/reservoir` owns the lower shared-edge typed-handle invariant for
|
||||
Fensalir/Faust binding work.
|
||||
- Fensalir owns production GPU fusion, UI, and Spout2 publication.
|
||||
- `MimirSceneEditorState` is currently a 2D OBS-shaped program mixer owner, not
|
||||
a camera/3D scene editor. Synced video buffers become program layers with
|
||||
normalized placement; the Eve/CultUI surface is Video Sources on the left, a
|
||||
chrome-free program canvas in the center, Audio Mixer docked underneath that
|
||||
canvas, and a Unity-style Inspector on the right for the selected layer's
|
||||
properties. The center preview requests Fensalir's current output-buffer blit
|
||||
as its background and overlays Mimir's normalized layer placements on top.
|
||||
Placement commits snap near frame edges/center and preserve source aspect
|
||||
while resizing through direct preview handles, sliders, or the scale gizmo.
|
||||
Explicit source/transform controls are inspector-only and only visible when a
|
||||
program source is selected.
|
||||
Hover/active handle state, snap guides, empty-canvas deselect, keyboard
|
||||
nudging, PageUp/PageDown layer ordering, and selected-source
|
||||
contain/cover/stretch fit mode are wired through `MimirSceneEditorState`.
|
||||
Camera/SDF/model editor affordances have no live UI authority, and editor
|
||||
camera/SDF/spline output no longer contributes to the runtime frame. Smoke
|
||||
with `--scene-editor-smoke`.
|
||||
- The Audio Mixer dock is a horizontal CultUI channel-strip bank, not a
|
||||
selected-stem inspector form. Six fixed slots read the live
|
||||
`presentationControls.AudioFeeds` order so Scarlett ASIO mono inputs,
|
||||
stereo loopback, and Raven loopback can appear as parallel strips with M/S,
|
||||
gain, and compact sync/chirp-response readouts derived from
|
||||
`MimirSynchronizationHub.AudioSynchronizationStates`. Do not move sample
|
||||
alignment, chirp response mapping, or ambient/noise neutralization authority
|
||||
into the UI; the surface only controls and observes runtime audio state.
|
||||
- Program surface placement is seeded from CultCache config
|
||||
`config/mimir-program-surface.cc` unless `--cache`,
|
||||
`AQUARIUM_CULTCACHE_PATH`, or `MIMIR_PROGRAM_SURFACE_CONFIG` points
|
||||
elsewhere. The default `mimir.program.default` document puts
|
||||
`raven-display` full-frame at layer 0 and `kiyo-pro-rgb` as a top-left
|
||||
picture-in-picture at layer 1. `MimirSceneEditorState` applies matching
|
||||
source placements once when those feeds appear, then inspector/drag edits own
|
||||
subsequent placement.
|
||||
- The left Video Sources rail is a direct source selector, not a previous/next
|
||||
navigator. It renders visible rows for live video feeds using CultUI
|
||||
row-level visibility, with a Select action and placement summary per source.
|
||||
Keep configured/offline source inventory as a future source-state owner;
|
||||
the current rail lists live synced video feeds.
|
||||
- Live camera image buffers should use Fensalir-owned texture leases when they
|
||||
are rendering inputs: Mimir asks the engine broker for a keyed D3D12
|
||||
Texture2D/fence lease, writes decoded frames into that texture, commits the
|
||||
producer fence value, and lowers the same resource key through FieldEvidence.
|
||||
`MimirVideoCaptureDriverSource` forwards that lease client to direct drivers
|
||||
that implement `IMimirFensalirTextureLeaseReceiver`.
|
||||
Camera backends must use the closest-to-device path available and report
|
||||
unavoidable copies; process or managed convenience layers are diagnostics.
|
||||
Raw single-plane CPU-origin frames can use the explicit broker upload lane,
|
||||
which increments `UnavoidableCopyCount` and clears live managed payload bytes
|
||||
after upload. NV12 CPU upload is rejected until Fensalir owns planar copy;
|
||||
device/GPU NV12 producers should write the leased texture directly.
|
||||
The first direct KS/UVC driver is `MimirKsVideoCaptureDriver` backed by
|
||||
`native/camera_capture/mimir_camera_capture.dll`; the example config
|
||||
`config/mimir-runtime.ks-camera.example.json` targets LeapUVC 640x240 YUY2.
|
||||
The same native folder also has `mimir_ps3eye_capture.dll` and
|
||||
`MimirPs3EyeVideoCaptureDriver` for PS3 Eye Bayer8 frames through WinUSB/libusb.
|
||||
Fresh driver smokes pulled and uploaded real frames for LeapUVC 640x240 YUY2,
|
||||
both PS3 Eyes at 320x240 Bayer8, Kiyo Pro 1920x1080 YUY2, and regular Kiyo
|
||||
640x480 YUY2. Regular Kiyo 1280x720 YUY2 did not open.
|
||||
Compressed Kiyo Pro MJPG/H264 now have a first GPU-resident path:
|
||||
`MimirMediaFoundationGpuVideoCaptureDriver` uses
|
||||
`native/camera_capture/mimir_mf_gpu_capture.dll` to decode through Media
|
||||
Foundation with a D3D11 device manager, copy into a shared BGRA D3D11 texture,
|
||||
and publish `shared-d3d11-texture` with zero live CPU payload bytes.
|
||||
1920x1080 MJPG->RGB32 and H264->RGB32 smokes passed. Direct shared NV12
|
||||
texture creation failed, so NV12 remains future plane-aware interop or
|
||||
GPU bridge-copy work.
|
||||
Foreign shared texture handles are import edges, not the primary hot path.
|
||||
- The current Mimir/Fensalir rendering teardown map is
|
||||
[[docs/fensalir-rendering-rebuild-migration|Fensalir Rendering Rebuild
|
||||
Migration]]. Mimir's production visual output path must publish typed
|
||||
physical observations, calibration constraints, and surface intent; Fensalir
|
||||
turns those into field claims, selected lowerings, reusable evidence,
|
||||
temporal guide lanes, and presentation.
|
||||
- Fast depth provenance now lives at
|
||||
[[research/d3d12-stereo-depth-provenance-2026-05-29|D3D12 Stereo Depth
|
||||
Provenance]]. `libSGM` is the current permissive north-star reference for a
|
||||
future Fensalir-owned D3D12 stereo SGM compute lane. Do not import CUDA or
|
||||
TensorRT as live depth authority.
|
||||
The first code contract is in `MimirStereoDepthConfigurations` and
|
||||
`MimirFensalirFieldLowering`. Run
|
||||
`dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --stereo-depth-contract-smoke`
|
||||
to verify caller-declared rectified left/right input textures, or
|
||||
`dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --leap-packed-stereo-depth-smoke`
|
||||
to verify live `LeapStereoIr` packed R8G8 lowering. The Leap path reuses the
|
||||
same declared texture as left/right resource identity and treats R/G as the
|
||||
packed observation lanes. Fensalir now has a first packed-Leap D3D12
|
||||
SAD/block-match disparity writer, not full libSGM-style SGM or calibrated
|
||||
metric depth. Mimir also now declares the first Leap point-cloud root:
|
||||
`MimirPointCloudConfigurations.LeapDisparityPointCloudRoot` and
|
||||
`BuildLeapPackedStereoPointCloudCandidateFrame` emit a GPU-resident
|
||||
`FieldMesh` point-list resource derived from that disparity SurfacePage. Run
|
||||
`dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --leap-point-cloud-root-smoke`
|
||||
to verify the combined `SurfacePage` plus `Mesh` packet plan. This is a
|
||||
typed socket on the Mimir side. Fensalir branch `codex/leap-packed-depth` now
|
||||
has the first matching render lane: generated point-list Mesh buffers are
|
||||
filled from the disparity SurfacePage on D3D12 and rendered through the scene
|
||||
candidate targets. The next cut is live Leap/editor verification plus
|
||||
calibrated projection constants and global residual/calibration ownership.
|
||||
- Fensalir also owns the EVE-facing dashboard pixels through
|
||||
`Global\MimirFensalirProgramTexture`. `src/Mimir.EveRelay` opens that shared
|
||||
texture, encodes H.264 Annex-B with NVENC by default, and serves EveCanvas
|
||||
over `/stream`; EveCanvas decodes with `AVSampleBufferDisplayLayer`. The
|
||||
launch script uses an SSH reverse tunnel to Eve when Windows firewall blocks
|
||||
direct inbound TCP.
|
||||
- `src/Mimir.Broadcast` is the first self-hosted public sender. Starfire pushes
|
||||
one final H.264/AAC NVENC RTMP stream to
|
||||
`rtmp://127.0.0.1:11935/live/mimir` through an SSH local forward to
|
||||
Yggdrasil. Yggdrasil's nginx RTMP/HLS origin is deployed and has produced
|
||||
HLS segments from a bounded synthetic push. Public HLS is served through the
|
||||
existing StreamPixels TLS host, not a new Mimir media hostname. The public
|
||||
static viewer belongs to the root GameCult site at
|
||||
`https://gamecult.org/livestream` and reads
|
||||
`https://streampixels.gamecult.org/mimir/live/hls/mimir.m3u8`.
|
||||
- Raven/Eve ingest now has a network-pixel receiver config:
|
||||
`config/mimir-runtime.raven-eve.example.json` starts an FFmpeg-backed
|
||||
raw-video listener for `raven-display` on SRT port `5200` and Eve sensor
|
||||
frame-event receivers for `eve-camera` on WebSocket port `8793` and `eve-mic`
|
||||
on port `8794`. `scripts/start-raven-screen-capture-sender.ps1` launches
|
||||
Raven desktop capture with `gdigrab` plus NVENC. EveCanvas owns Eve camera/mic
|
||||
capture directly through AVFoundation/AVAudioEngine; no external iOS camera
|
||||
app owns that path.
|
||||
- `src/Mimir.RavenDaemon` is the Raven-local capture mux owner for the
|
||||
stream-proof path. It supervises display capture plus Realtek render loopback
|
||||
through the existing FFmpeg/WASAPI pipeline, publishes
|
||||
`mimir.raven_capture_mux_state.v1` through CultMesh/CultCache, and exposes
|
||||
Eve dashboard state on port `8801`. `scripts/start-raven-daemon.ps1` defaults
|
||||
to Starfire over WireGuard at `10.77.0.2:5200`. The older Raven sender scripts
|
||||
remain diagnostics, not canonical long-running status owners.
|
||||
- Faust/native DSP owns hot audio alignment, separation, spatialization, and
|
||||
synchronized stems.
|
||||
- The current listenable dual-mic receipt is the Faust-chain bridge, not the
|
||||
older C# composite. `scripts/record-kiyo-dual-mic-composite.ps1` records Kiyo
|
||||
plus Scarlett ASIO, then `--asio-dual-mic-faust-cleaner-wav` compiles/runs
|
||||
`faust/mimir_alignment_actuator.dsp` before
|
||||
`faust/mimir_dual_mic_dialogue_cleaner.dsp` and muxes the dialogue stem into
|
||||
`artifacts/runtime/kiyo-dual-mic-composite/kiyo-dual-mic-composite-latest.mp4`.
|
||||
Fresh run `20260601-051044` measured cardioid delay 359.398 samples
|
||||
(7487.452 us) at 48 kHz and applied holdback 359.398/0.000 to
|
||||
shotgun/cardioid before cleanup.
|
||||
- The edge-to-edge Mimir editor shell now uses headerless retained dashboard
|
||||
panels. The inspector expands vertically; the right rail `Output` panel is a
|
||||
fixed-height OBS-like control block below it. Eve is the portable dashboard
|
||||
authority; the C# CultUI-shaped Aquarium contract is only the reference API
|
||||
shape, and Fensalir Direct2D/D3D12 are native lowerings of the same document.
|
||||
Elements can now carry typed CultCache/CultMesh bindings with document
|
||||
schema/id, field path, value kind, access, authority, and optional command id;
|
||||
native callbacks remain local lowering actuators, not portable state owners.
|
||||
`MimirProgramRecorder` owns `Record` / `Stop`, status text, and timestamped
|
||||
receipts under `C:\Users\Meta\Videos\Mimir` by default
|
||||
(`MIMIR_RECORD_OUTPUT_DIR` overrides). It delegates to
|
||||
`scripts/record-stream-proof-output.ps1`, so the app records the same Raven +
|
||||
Kiyo PIP + Starfire Scarlett NVENC/AAC composite intended for
|
||||
Yggdrasil/public judgment. `MIMIR_RECORD_START_RAVEN=1` asks it to launch the
|
||||
Raven sender too. `MimirProgramStreamer` owns `Live` / `Stop` for
|
||||
`Mimir.Broadcast`; it requires `MIMIR_STREAM_INPUT` unless
|
||||
`MIMIR_STREAM_ALLOW_TEST_PATTERN=1` explicitly allows a synthetic smoke.
|
||||
Native program-output recording/streaming should replace those actuators
|
||||
behind the same owner.
|
||||
- OBS receives final program surfaces; it does not own synchronization.
|
||||
- The online Well stack exists. `Mimir.EveSensorReceiver` has a subscriber
|
||||
WebSocket path beside its observation ingest path, and `Mimir.VerseRecorder`
|
||||
writes subscribed observations to
|
||||
`C:\Users\Meta\Videos\Mimir\VerseCaptures\<session>\session.json` plus
|
||||
`observations.jsonl`. Recorder now also recognizes
|
||||
`mimir.well_capture_page.v1`, strips inline base64 sample bodies into
|
||||
`bodies/page-*.bin`, and records `mimir.recorder_body_index.v1` refs in
|
||||
`bodies/index.jsonl` so offline reconstruction tools can resolve payloads
|
||||
without treating JSONL as the body store. `Mimir.Well` loads
|
||||
`config/mimir-runtime.well.local.json`, starts every configured source it
|
||||
can, polls `MimirSynchronizationHub` with a bounded per-source budget, updates
|
||||
audio sync/probe state, and publishes `mimir.well_snapshot.v1` records with
|
||||
source buffers, composite controls, publication/stem contract, and sync
|
||||
status. It also publishes bounded `mimir.well_capture_page.v1` records for
|
||||
synchronized frame slices with timing metadata, configured composite state,
|
||||
and inline source bodies when CPU bytes exist and fit the configured body
|
||||
limit. GPU-handle-only samples remain metadata/body-handle evidence until
|
||||
Fensalir/native program-output recording owns rendered composite bodies.
|
||||
`scripts/start-online-verse-daemons.ps1` supervises the relay,
|
||||
recorder, Nightwing typed Eyes/Move/builtin-camera/builtin-mic witness, the
|
||||
music-synced de-Bruijn/microtonal Move gesture worker, and `Mimir.Well`.
|
||||
Current live proof `online-verse-daemons-20260601-212502` wrote Well,
|
||||
Nightwing builtin camera/mic, and Eye/Move records. Raven and Eve lanes are
|
||||
configured but empty unless their producers are feeding the ports. Sync frame
|
||||
status is degraded until a camera-clock/global-time correction owner maps
|
||||
ASIO device time and camera driver time into one canonical timeline.
|
||||
|
||||
## Current Pressure
|
||||
|
||||
- Implement the first concrete Leap direct capture driver and measure cadence.
|
||||
- Verify the live Leap point cloud in the Mimir editor, then add calibrated
|
||||
projection constants and the global residual/calibration owner.
|
||||
- Cut the Phase 1 Mimir-to-Fensalir bridge DTOs for rolling windows,
|
||||
observations, calibration constraints, and surface intent before adding
|
||||
another backend-specific visual path.
|
||||
- Add remaining camera drivers through `IMimirVideoCaptureDriver`.
|
||||
- Add native audio capture workers for mic, loopback, and network audio feeds.
|
||||
- Turn the current loopback-referenced chirplet measurement into the real
|
||||
|
|
|
|||
|
|
@ -11,11 +11,18 @@ Use it to shape native implementation decisions:
|
|||
|
||||
- direct camera and audio drivers;
|
||||
- pinned/native buffers;
|
||||
- GPU feature extraction, flow, matching, and fusion;
|
||||
- GPU feature extraction, flow, matching, depth, and fusion;
|
||||
- Fensalir D3D12 compute/render paths;
|
||||
- Faust/native DSP;
|
||||
- OBS-facing program outputs.
|
||||
|
||||
Visual/GPU research entry points:
|
||||
|
||||
- [[d3d12-stereo-depth-provenance-2026-05-29|D3D12 Stereo Depth Provenance]]:
|
||||
libSGM is the current permissive north-star reference for a future
|
||||
Fensalir-owned D3D12 compute stereo-disparity lane. External CUDA/TensorRT
|
||||
projects are provenance only, not runtime authority.
|
||||
|
||||
Audio timing research entry points:
|
||||
|
||||
- [[chirplet-sync-decoder/summary|Chirplet Sync Decoder Summary]]: chirplet
|
||||
|
|
|
|||
170
research/d3d12-stereo-depth-provenance-2026-05-29.md
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
# D3D12 Stereo Depth Provenance
|
||||
|
||||
Date: 2026-05-29
|
||||
|
||||
## Purpose
|
||||
|
||||
This note records the current fast GPU depth-estimation search as provenance for
|
||||
a future Mimir/Fensalir D3D12 compute implementation.
|
||||
|
||||
The north star is `libSGM`: not as a dependency, not as a CUDA bridge, and not
|
||||
as a permission slip to add an adapter. The useful shape is a fast,
|
||||
deterministic stereo disparity pipeline whose ownership can be rebuilt inside
|
||||
Fensalir's D3D12 resource model.
|
||||
|
||||
## Authority Map
|
||||
|
||||
Owner:
|
||||
|
||||
- Fensalir owns the live D3D12 compute kernels, GPU resources, fences,
|
||||
scheduling, and depth/surface claim lowering.
|
||||
|
||||
Inputs:
|
||||
|
||||
- Rectified synchronized camera texture pairs from Mimir-declared
|
||||
`Texture2D` resources.
|
||||
- Camera calibration: intrinsics, extrinsics, baseline, rectification map, and
|
||||
valid-time/version metadata.
|
||||
- Rolling-window timing chosen by `MimirSynchronizedBufferPlanner`.
|
||||
|
||||
Outputs:
|
||||
|
||||
- GPU-resident disparity/depth resources, preferably fixed format and size per
|
||||
calibrated camera pair.
|
||||
- FieldEvidence claims for depth support, confidence, invalid/occluded pixels,
|
||||
and later surface/material lowerings.
|
||||
- Diagnostics: kernel timings, disparity range, confidence histogram, invalid
|
||||
ratio, calibration version, and source texture fence values.
|
||||
|
||||
Derived State:
|
||||
|
||||
- Depth is derived from synchronized stereo textures plus calibration.
|
||||
- Per-frame disparity confidence is derived evidence, not a clock or camera
|
||||
authority.
|
||||
- Monocular model depth remains optional reference/assistive evidence unless a
|
||||
later invariant gives it a named role.
|
||||
|
||||
Forbidden Writers:
|
||||
|
||||
- CUDA runtime code must not become the live path.
|
||||
- TensorRT/ONNX monocular demos must not own visual geometry.
|
||||
- CPU image copies/readback must not be inserted for convenience.
|
||||
- OBS, EVE, or bridge utilities must not decide depth or synchronization.
|
||||
|
||||
Shared Paths:
|
||||
|
||||
- Direct user capture, replayed rolling-window frames, future Raven/phone
|
||||
camera feeds, and calibration-session frames must all enter through typed
|
||||
resources and use the same depth compute/claim path once they are calibrated
|
||||
stereo pairs.
|
||||
|
||||
Deletion Line:
|
||||
|
||||
- If a prototype requires persistent CPU depth images, a side transport daemon,
|
||||
or a CUDA compatibility layer, it is research-only and does not graduate into
|
||||
the live machine.
|
||||
|
||||
## Reference Shape: libSGM
|
||||
|
||||
Repository: https://github.com/fixstars/libSGM
|
||||
|
||||
License: Apache-2.0.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- It is a CUDA implementation of Semi-Global Matching for stereo disparity.
|
||||
- It was built by the Fixstars/adaskit ADAS effort, so it is pointed at the
|
||||
same autonomous-driving pressure field rather than generic image demos.
|
||||
- Reported benchmark in the upstream README: `1024 x 440`, disparity `128`,
|
||||
four SGM paths, subpixel enabled:
|
||||
- GTX 1080 Ti: `2.0 ms` / `495.1 FPS`
|
||||
- RTX 3080: `1.5 ms` / `651.3 FPS`
|
||||
- Tegra X2: `28.5 ms` / `35.1 FPS`
|
||||
- Xavier MAXN: `9.0 ms` / `110.7 FPS`
|
||||
|
||||
Port lesson:
|
||||
|
||||
- Use the pipeline and data movement as a model: matching cost, path
|
||||
aggregation, winner/subpixel selection, confidence/invalid handling.
|
||||
- Rebuild the kernels in HLSL/D3D12 compute against Fensalir-owned resources.
|
||||
- Preserve the ADAS-style assumption that calibrated stereo gives metric depth
|
||||
when baseline and rectification are real.
|
||||
|
||||
## Secondary References
|
||||
|
||||
Fast-ACVNet: https://github.com/gangweiX/Fast-ACVNet
|
||||
|
||||
- License: MIT.
|
||||
- Learned real-time stereo matching; useful for quality and failure-mode
|
||||
comparison.
|
||||
- Upstream reports `45 ms` for Fast-ACVNet+ in its KITTI/Scene Flow table.
|
||||
- Not the first Mimir implementation target because it brings model training,
|
||||
weights, and framework deployment complexity.
|
||||
|
||||
UniMatch / GMStereo: https://github.com/autonomousvision/unimatch
|
||||
|
||||
- License: MIT.
|
||||
- Unified flow/stereo/depth research code.
|
||||
- Useful as a modern learned-stereo comparison point and possible later
|
||||
confidence/feature guide, not a kernel blueprint.
|
||||
|
||||
Depth Anything V2 Small: https://github.com/DepthAnything/Depth-Anything-V2
|
||||
|
||||
- License caveat: V2 Small is Apache-2.0; Base/Large/Giant are CC-BY-NC-4.0.
|
||||
- Excellent monocular relative-depth reference, especially through TensorRT
|
||||
ports, but monocular output is not metric scene truth by itself.
|
||||
|
||||
Depth Anything TensorRT CLI:
|
||||
https://github.com/spacewalk01/depth-anything-tensorrt
|
||||
|
||||
- License: MIT.
|
||||
- Reports fast RTX 4090 inference for Depth-Anything-S/B/L.
|
||||
- Useful as performance context only. It must not displace calibrated stereo
|
||||
geometry or Fensalir's D3D12 ownership.
|
||||
|
||||
Fast-FoundationStereo:
|
||||
https://github.com/NVlabs/Fast-FoundationStereo
|
||||
|
||||
- Important research reference for real-time zero-shot stereo.
|
||||
- License is research/non-commercial only, so it is not a permissive live
|
||||
dependency candidate.
|
||||
|
||||
## D3D12 Compute Implementation Sketch
|
||||
|
||||
First implementation should stay boring and inspectable:
|
||||
|
||||
1. Rectify or require pre-rectified stereo pairs.
|
||||
2. Convert sampled luma/feature texture into a compact matching domain.
|
||||
3. Build per-pixel disparity matching costs over a fixed disparity range.
|
||||
4. Aggregate along a small fixed path set, starting with four directions to
|
||||
match the `libSGM` reference benchmark shape.
|
||||
5. Select winner disparity and subpixel refinement.
|
||||
6. Emit disparity, confidence, and invalid/occlusion masks as GPU resources.
|
||||
7. Lower those resources into FieldEvidence depth/support claims.
|
||||
|
||||
Important D3D12 resource notes:
|
||||
|
||||
- Prefer persistent UAV/SRV resources sized by camera pair profile.
|
||||
- Use explicit producer/consumer fences already established for camera texture
|
||||
leases and program-output publication.
|
||||
- Keep disparity range and path count as profile-level cost levers, not
|
||||
per-frame mode flags scattered through the renderer.
|
||||
- Do not read back depth except for gated diagnostics and receipts.
|
||||
|
||||
## Initial Verification Targets
|
||||
|
||||
- Synthetic rectified stereo pair with known constant disparity.
|
||||
- Slanted plane with known disparity gradient.
|
||||
- Occlusion edge where invalid/confidence behavior is visible.
|
||||
- Real Leap stereo IR pair once calibration is pinned.
|
||||
- Full rolling-window path: camera textures -> depth compute -> FieldEvidence
|
||||
claim -> Fensalir debug overlay/resource receipt.
|
||||
|
||||
Negative checks:
|
||||
|
||||
- No CUDA runtime loaded.
|
||||
- No CPU disparity image in the live path.
|
||||
- No monocular model output accepted as metric depth without a named scale
|
||||
authority.
|
||||
- No untyped texture handle accepted without resource declaration and fence
|
||||
provenance.
|
||||
19
run-spectrum-preview.cmd
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
@echo off
|
||||
setlocal
|
||||
|
||||
set "ROOT=%~dp0"
|
||||
set "APP=%ROOT%src\Mimir.App\bin\Debug\net10.0\Mimir.App.exe"
|
||||
|
||||
if not exist "%APP%" (
|
||||
echo Mimir.App.exe was not found.
|
||||
echo Build it first with:
|
||||
echo dotnet build "%ROOT%src\Mimir.App\Mimir.App.csproj"
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
set "MIMIR_SYNTHETIC_SPECTRUM_PREVIEW=1"
|
||||
set "MIMIR_SPECTRUM_INTERVAL_SECONDS=0.016"
|
||||
set "MIMIR_SPECTRUM_SOURCE_LANES=8"
|
||||
|
||||
start "Mimir Spectrum Preview" "%APP%"
|
||||
35
scripts/build-obs-stem-plugin.ps1
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
param(
|
||||
[string]$Configuration = "Release",
|
||||
[string]$TemplateVersion = "master",
|
||||
[switch]$SkipTests
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$root = Split-Path -Parent $PSScriptRoot
|
||||
$sdkRoot = Join-Path $root "artifacts\obs-sdk"
|
||||
$templateRoot = Join-Path $sdkRoot "obs-plugintemplate"
|
||||
$depsRoot = Join-Path $templateRoot ".deps"
|
||||
$buildRoot = Join-Path $templateRoot "build_x64_local"
|
||||
$pluginBuildRoot = Join-Path $root "native\obs_stem_source\build"
|
||||
|
||||
New-Item -ItemType Directory -Force $sdkRoot | Out-Null
|
||||
if (-not (Test-Path $templateRoot)) {
|
||||
git clone --depth 1 --branch $TemplateVersion https://github.com/obsproject/obs-plugintemplate.git $templateRoot
|
||||
}
|
||||
|
||||
if (-not (Test-Path (Join-Path $depsRoot "cmake\libobsConfig.cmake"))) {
|
||||
cmake -S $templateRoot -B $buildRoot -G "Visual Studio 17 2022" -A x64 -DENABLE_FRONTEND_API=OFF -DENABLE_QT=OFF
|
||||
}
|
||||
|
||||
cmake `
|
||||
-S (Join-Path $root "native\obs_stem_source") `
|
||||
-B $pluginBuildRoot `
|
||||
-Dlibobs_DIR="$(Join-Path $depsRoot 'cmake')" `
|
||||
-DCMAKE_PREFIX_PATH="$depsRoot"
|
||||
|
||||
cmake --build $pluginBuildRoot --config $Configuration --parallel
|
||||
|
||||
if (-not $SkipTests) {
|
||||
ctest --test-dir $pluginBuildRoot -C $Configuration --output-on-failure
|
||||
}
|
||||
71
scripts/install-obs-mimir-plugin.ps1
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
param(
|
||||
[string]$ObsRoot = "",
|
||||
[string]$PluginRoot = "",
|
||||
[string]$Configuration = "Release",
|
||||
[switch]$SkipBuild,
|
||||
[switch]$WhatIf
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = Split-Path -Parent $PSScriptRoot
|
||||
|
||||
$pluginDll = Join-Path $repoRoot "native\obs_stem_source\build\$Configuration\mimir_obs_stem_source.dll"
|
||||
$localeSourceDir = Join-Path $repoRoot "native\obs_stem_source\data\locale"
|
||||
|
||||
if (-not [string]::IsNullOrWhiteSpace($ObsRoot)) {
|
||||
$targetDir = Join-Path $ObsRoot "obs-plugins\64bit"
|
||||
$targetLocaleDir = Join-Path $ObsRoot "data\obs-plugins\mimir_obs_stem_source\locale"
|
||||
} else {
|
||||
if ([string]::IsNullOrWhiteSpace($PluginRoot)) {
|
||||
if ($env:OBS_PLUGIN_ROOT) {
|
||||
$PluginRoot = $env:OBS_PLUGIN_ROOT
|
||||
} else {
|
||||
$PluginRoot = Join-Path $env:ALLUSERSPROFILE "obs-studio\plugins"
|
||||
}
|
||||
}
|
||||
|
||||
$targetDir = Join-Path $PluginRoot "mimir_obs_stem_source\bin\64bit"
|
||||
$targetLocaleDir = Join-Path $PluginRoot "mimir_obs_stem_source\data\locale"
|
||||
}
|
||||
|
||||
$targetDll = Join-Path $targetDir "mimir_obs_stem_source.dll"
|
||||
|
||||
if (-not $SkipBuild) {
|
||||
& (Join-Path $repoRoot "scripts\build-obs-stem-plugin.ps1") -Configuration $Configuration
|
||||
}
|
||||
|
||||
if (-not (Test-Path $pluginDll)) {
|
||||
throw "Built plugin DLL was not found at '$pluginDll'."
|
||||
}
|
||||
if (-not (Test-Path $localeSourceDir)) {
|
||||
throw "Plugin locale directory was not found at '$localeSourceDir'."
|
||||
}
|
||||
|
||||
if (-not (Test-Path $targetDir)) {
|
||||
if ($WhatIf) {
|
||||
Write-Host "Would create $targetDir"
|
||||
} else {
|
||||
New-Item -ItemType Directory -Force $targetDir | Out-Null
|
||||
}
|
||||
}
|
||||
if (-not (Test-Path $targetLocaleDir)) {
|
||||
if ($WhatIf) {
|
||||
Write-Host "Would create $targetLocaleDir"
|
||||
} else {
|
||||
New-Item -ItemType Directory -Force $targetLocaleDir | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
if ($WhatIf) {
|
||||
Write-Host "Would copy $pluginDll -> $targetDll"
|
||||
Get-ChildItem -LiteralPath $localeSourceDir -File | ForEach-Object {
|
||||
Write-Host "Would copy $($_.FullName) -> $(Join-Path $targetLocaleDir $_.Name)"
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
Copy-Item -LiteralPath $pluginDll -Destination $targetDll -Force
|
||||
Copy-Item -Path (Join-Path $localeSourceDir "*") -Destination $targetLocaleDir -Force
|
||||
Write-Host "Installed Mimir OBS plugin: $targetDll"
|
||||
Write-Host "OBS source types: Mimir Program Texture, Mimir Audio Stem, Muninn Stream, Muninn Video, Muninn Audio"
|
||||
41
scripts/raven-audio.ps1
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
param(
|
||||
[string]$TargetHost = "192.168.1.66",
|
||||
[int]$Port = 5202,
|
||||
[int]$SampleRate = 48000,
|
||||
[int]$Channels = 2,
|
||||
[string]$FfmpegPath = "ffmpeg",
|
||||
[string]$LogRoot = "C:\Meta\Mimir\logs",
|
||||
[switch]$DryRun
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $LogRoot | Out-Null
|
||||
$repo = Split-Path -Parent $PSScriptRoot
|
||||
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
$stdoutLog = Join-Path $LogRoot "raven-realtk-loopback-$timestamp.out.log"
|
||||
$stderrLog = Join-Path $LogRoot "raven-realtk-loopback-$timestamp.err.log"
|
||||
$endpoint = "srt://${TargetHost}:${Port}?mode=caller&latency=120000"
|
||||
$captureScript = Join-Path $repo "scripts\wasapi-loopback-capture.ps1"
|
||||
|
||||
$command = "& `"$captureScript`" -Output - -SampleRate $SampleRate -Channels $Channels | & `"$FfmpegPath`" -hide_banner -nostdin -loglevel warning -f f32le -ar $SampleRate -ac $Channels -i pipe:0 -f f32le `"$endpoint`""
|
||||
|
||||
Write-Host "Raven Realtek loopback sender:"
|
||||
Write-Host " target: $endpoint"
|
||||
Write-Host " format: f32le ${Channels}ch ${SampleRate}Hz"
|
||||
Write-Host " stdout: $stdoutLog"
|
||||
Write-Host " stderr: $stderrLog"
|
||||
|
||||
if ($DryRun) {
|
||||
Write-Host $command
|
||||
exit 0
|
||||
}
|
||||
|
||||
Start-Process `
|
||||
-FilePath "powershell" `
|
||||
-ArgumentList @("-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", $command) `
|
||||
-WindowStyle Hidden `
|
||||
-RedirectStandardOutput $stdoutLog `
|
||||
-RedirectStandardError $stderrLog `
|
||||
-PassThru |
|
||||
ForEach-Object { Write-Host "Started Raven Realtek loopback sender pid=$($_.Id)" }
|
||||
158
scripts/record-kiyo-dual-mic-composite.ps1
Normal file
|
|
@ -0,0 +1,158 @@
|
|||
param(
|
||||
[int]$DurationSeconds = 15,
|
||||
[string]$OutputPath = "artifacts\runtime\kiyo-dual-mic-composite\kiyo-dual-mic-composite.mp4",
|
||||
[string]$LogRoot = "artifacts\runtime\kiyo-dual-mic-composite",
|
||||
[string]$FfmpegPath = "ffmpeg",
|
||||
[string]$AsioProbePath = "native\probes\asio_audio_cadence\build\Release\asio_audio_cadence.exe",
|
||||
[string]$KiyoVideoDevice = "USB Video Device",
|
||||
[int]$KiyoWidth = 640,
|
||||
[int]$KiyoHeight = 480,
|
||||
[int]$KiyoFrameRate = 30,
|
||||
[int]$SampleRate = 48000,
|
||||
[int]$AsioChannels = 4,
|
||||
[int]$ShotgunChannel = 0,
|
||||
[int]$CardioidChannel = 1,
|
||||
[switch]$DryRun
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Quote-ProcessArgument([string]$Value) {
|
||||
if ($Value.Length -eq 0) {
|
||||
return '""'
|
||||
}
|
||||
|
||||
if ($Value -notmatch '[\s"]') {
|
||||
return $Value
|
||||
}
|
||||
|
||||
return '"' + $Value.Replace('"', '\"') + '"'
|
||||
}
|
||||
|
||||
function Start-LoggedProcess([string]$FileName, [string[]]$Arguments, [string]$StdoutPath, [string]$StderrPath) {
|
||||
$startInfo = [System.Diagnostics.ProcessStartInfo]::new()
|
||||
$startInfo.FileName = $FileName
|
||||
$startInfo.UseShellExecute = $false
|
||||
$startInfo.RedirectStandardOutput = $true
|
||||
$startInfo.RedirectStandardError = $true
|
||||
$startInfo.Arguments = ($Arguments | ForEach-Object { Quote-ProcessArgument $_ }) -join " "
|
||||
$process = [System.Diagnostics.Process]::Start($startInfo)
|
||||
if (-not $process) {
|
||||
throw "Could not start $FileName"
|
||||
}
|
||||
|
||||
$stdoutStream = [System.IO.File]::Open($StdoutPath, [System.IO.FileMode]::Create, [System.IO.FileAccess]::Write, [System.IO.FileShare]::Read)
|
||||
$stderrStream = [System.IO.File]::Open($StderrPath, [System.IO.FileMode]::Create, [System.IO.FileAccess]::Write, [System.IO.FileShare]::Read)
|
||||
return [pscustomobject]@{
|
||||
Process = $process
|
||||
StdoutTask = $process.StandardOutput.BaseStream.CopyToAsync($stdoutStream)
|
||||
StderrTask = $process.StandardError.BaseStream.CopyToAsync($stderrStream)
|
||||
StdoutStream = $stdoutStream
|
||||
StderrStream = $stderrStream
|
||||
}
|
||||
}
|
||||
|
||||
function Wait-LoggedProcess($Handle, [int]$TimeoutSeconds, [string]$Name) {
|
||||
if (-not $Handle.Process.WaitForExit($TimeoutSeconds * 1000)) {
|
||||
try { $Handle.Process.Kill($true) } catch {}
|
||||
throw "$Name timed out after ${TimeoutSeconds}s."
|
||||
}
|
||||
|
||||
[void]$Handle.StdoutTask.GetAwaiter().GetResult()
|
||||
[void]$Handle.StderrTask.GetAwaiter().GetResult()
|
||||
$Handle.StdoutStream.Dispose()
|
||||
$Handle.StderrStream.Dispose()
|
||||
if ($Handle.Process.ExitCode -ne 0) {
|
||||
throw "$Name failed with exit code $($Handle.Process.ExitCode)."
|
||||
}
|
||||
}
|
||||
|
||||
$repo = Split-Path -Parent $PSScriptRoot
|
||||
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
$absoluteLogRoot = if ([System.IO.Path]::IsPathRooted($LogRoot)) { $LogRoot } else { Join-Path $repo $LogRoot }
|
||||
$runDirectory = Join-Path $absoluteLogRoot $timestamp
|
||||
New-Item -ItemType Directory -Force -Path $runDirectory | Out-Null
|
||||
|
||||
$absoluteOutputPath = if ([System.IO.Path]::IsPathRooted($OutputPath)) { $OutputPath } else { Join-Path $repo $OutputPath }
|
||||
$outputDirectory = Split-Path -Parent $absoluteOutputPath
|
||||
New-Item -ItemType Directory -Force -Path $outputDirectory | Out-Null
|
||||
|
||||
$asioRaw = Join-Path $runDirectory "scarlett-f32-interleaved.raw"
|
||||
$kiyoMkv = Join-Path $runDirectory "kiyo-video.mkv"
|
||||
$compositeWav = Join-Path $runDirectory "dual-mic-composite.wav"
|
||||
$absoluteAsioProbe = if ([System.IO.Path]::IsPathRooted($AsioProbePath)) { $AsioProbePath } else { Join-Path $repo $AsioProbePath }
|
||||
|
||||
$asioArgs = @(
|
||||
"--set-sample-rate", $SampleRate.ToString(),
|
||||
"--record-f32-interleaved", $asioRaw,
|
||||
"--capture-seconds", $DurationSeconds.ToString()
|
||||
)
|
||||
$kiyoArgs = @(
|
||||
"-hide_banner",
|
||||
"-y",
|
||||
"-thread_queue_size", "1024",
|
||||
"-rtbufsize", "256M",
|
||||
"-f", "dshow",
|
||||
"-video_size", "${KiyoWidth}x${KiyoHeight}",
|
||||
"-framerate", $KiyoFrameRate.ToString(),
|
||||
"-i", "video=$KiyoVideoDevice",
|
||||
"-t", $DurationSeconds.ToString(),
|
||||
"-r", $KiyoFrameRate.ToString(),
|
||||
"-c:v", "h264_nvenc",
|
||||
"-preset", "p4",
|
||||
"-tune", "ll",
|
||||
"-pix_fmt", "yuv420p",
|
||||
$kiyoMkv
|
||||
)
|
||||
|
||||
Write-Host "Kiyo + computed dual-mic composite recording:"
|
||||
Write-Host " Kiyo: $KiyoVideoDevice ${KiyoWidth}x${KiyoHeight}@$KiyoFrameRate"
|
||||
Write-Host " ASIO: $absoluteAsioProbe sampleRate=$SampleRate channels=$AsioChannels shotgun=$ShotgunChannel cardioid=$CardioidChannel"
|
||||
Write-Host " Run dir: $runDirectory"
|
||||
Write-Host " Output: $absoluteOutputPath"
|
||||
|
||||
if ($DryRun) {
|
||||
Write-Host ($absoluteAsioProbe + " " + ($asioArgs -join " "))
|
||||
Write-Host ($FfmpegPath + " " + ($kiyoArgs -join " "))
|
||||
exit 0
|
||||
}
|
||||
|
||||
$asio = Start-LoggedProcess $absoluteAsioProbe $asioArgs (Join-Path $runDirectory "asio.out.log") (Join-Path $runDirectory "asio.err.log")
|
||||
$kiyo = Start-LoggedProcess $FfmpegPath $kiyoArgs (Join-Path $runDirectory "kiyo.out.log") (Join-Path $runDirectory "kiyo.err.log")
|
||||
|
||||
Wait-LoggedProcess $asio ($DurationSeconds + 15) "ASIO capture"
|
||||
Wait-LoggedProcess $kiyo ($DurationSeconds + 15) "Kiyo capture"
|
||||
|
||||
dotnet run --project (Join-Path $repo "src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj") -- `
|
||||
--asio-dual-mic-faust-cleaner-wav `
|
||||
--input $asioRaw `
|
||||
--output $compositeWav `
|
||||
--sample-rate $SampleRate `
|
||||
--channels $AsioChannels `
|
||||
--shotgun-channel $ShotgunChannel `
|
||||
--cardioid-channel $CardioidChannel
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Dual-mic composite WAV generation failed."
|
||||
}
|
||||
|
||||
$muxArgs = @(
|
||||
"-hide_banner",
|
||||
"-y",
|
||||
"-i", $kiyoMkv,
|
||||
"-i", $compositeWav,
|
||||
"-map", "0:v:0",
|
||||
"-map", "1:a:0",
|
||||
"-c:v", "copy",
|
||||
"-c:a", "aac",
|
||||
"-b:a", "192k",
|
||||
"-ar", "48000",
|
||||
"-movflags", "+faststart",
|
||||
$absoluteOutputPath
|
||||
)
|
||||
$mux = Start-LoggedProcess $FfmpegPath $muxArgs (Join-Path $runDirectory "mux.out.log") (Join-Path $runDirectory "mux.err.log")
|
||||
Wait-LoggedProcess $mux 60 "Mux"
|
||||
|
||||
Write-Host "Recording complete:"
|
||||
Write-Host " $absoluteOutputPath"
|
||||
Write-Host " composite wav: $compositeWav"
|
||||
Write-Host " report: $([System.IO.Path]::ChangeExtension($compositeWav, '.json'))"
|
||||
201
scripts/record-stream-proof-output.ps1
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
param(
|
||||
[int]$RavenPort = 5204,
|
||||
[int]$DurationSeconds = 20,
|
||||
[int]$StartupTimeoutSeconds = 20,
|
||||
[int]$Width = 1280,
|
||||
[int]$Height = 720,
|
||||
[int]$FrameRate = 30,
|
||||
[string]$OutputPath = "artifacts\runtime\stream-proof\stream-proof-output.mp4",
|
||||
[string]$LogRoot = "artifacts\runtime\stream-proof",
|
||||
[string]$FfmpegPath = "ffmpeg",
|
||||
[switch]$StartRavenSender,
|
||||
[string]$RavenHost = "madman's lullaby@192.168.1.84",
|
||||
[string]$RavenAddress = "192.168.1.84",
|
||||
[string]$RavenRepo = "C:\Meta\Mimir",
|
||||
[string]$StarfireHost = "192.168.1.66",
|
||||
[switch]$RavenSenderListens,
|
||||
[ValidateSet("srt", "tcp-listener")]
|
||||
[string]$RavenSenderTransport = "srt",
|
||||
[string]$RavenInputOverride = "",
|
||||
[string]$KiyoVideoDevice = "USB Video Device",
|
||||
[string]$StarfireScarlettAudioDevice = "Analogue 1 + 2 (2- Focusrite USB Audio)",
|
||||
[int]$KiyoWidth = 640,
|
||||
[int]$KiyoHeight = 480,
|
||||
[int]$KiyoFrameRate = 30,
|
||||
[int]$KiyoCornerWidth = 320,
|
||||
[int]$KiyoMargin = 28,
|
||||
[double]$RavenOffsetSeconds = 0.0,
|
||||
[double]$KiyoOffsetSeconds = 0.0,
|
||||
[double]$MicOffsetSeconds = 0.0,
|
||||
[switch]$NoStarfireScarlett,
|
||||
[switch]$DryRun
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Quote-ProcessArgument([string]$Value) {
|
||||
if ($Value.Length -eq 0) {
|
||||
return '""'
|
||||
}
|
||||
|
||||
if ($Value -notmatch '[\s"]') {
|
||||
return $Value
|
||||
}
|
||||
|
||||
return '"' + $Value.Replace('"', '\"') + '"'
|
||||
}
|
||||
|
||||
$repo = Split-Path -Parent $PSScriptRoot
|
||||
$absoluteOutputPath = if ([System.IO.Path]::IsPathRooted($OutputPath)) {
|
||||
$OutputPath
|
||||
} else {
|
||||
Join-Path $repo $OutputPath
|
||||
}
|
||||
$outputDirectory = Split-Path -Parent $absoluteOutputPath
|
||||
New-Item -ItemType Directory -Force -Path $outputDirectory | Out-Null
|
||||
$absoluteLogRoot = if ([System.IO.Path]::IsPathRooted($LogRoot)) {
|
||||
$LogRoot
|
||||
} else {
|
||||
Join-Path $repo $LogRoot
|
||||
}
|
||||
New-Item -ItemType Directory -Force -Path $absoluteLogRoot | Out-Null
|
||||
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
$stdoutLog = Join-Path $absoluteLogRoot "stream-proof-record-$timestamp.out.log"
|
||||
$stderrLog = Join-Path $absoluteLogRoot "stream-proof-record-$timestamp.err.log"
|
||||
|
||||
$ravenEndpoint = if ($RavenInputOverride.Length -gt 0) {
|
||||
$RavenInputOverride
|
||||
} elseif ($RavenSenderListens) {
|
||||
"srt://${RavenAddress}:${RavenPort}?mode=caller&latency=120000&timeout=30000000"
|
||||
} else {
|
||||
"srt://0.0.0.0:${RavenPort}?mode=listener&latency=120000&timeout=30000000"
|
||||
}
|
||||
$ravenSender = "${RavenRepo}\scripts\start-raven-av-sender.ps1"
|
||||
|
||||
if ($StartRavenSender) {
|
||||
$taskName = "MimirRavenAvSender$RavenPort"
|
||||
$remoteCmd = "${RavenRepo}\scripts\run-raven-av-$RavenPort.cmd"
|
||||
$remoteTargetHost = if ($RavenSenderListens) { "0.0.0.0" } else { $StarfireHost }
|
||||
$remoteSrtMode = if ($RavenSenderListens) { "listener" } else { "caller" }
|
||||
$remoteCommand = "powershell.exe -NoProfile -ExecutionPolicy Bypass -File $ravenSender -TargetHost $remoteTargetHost -Port $RavenPort -SrtMode $remoteSrtMode -Transport $RavenSenderTransport"
|
||||
$taskTime = (Get-Date).AddMinutes(1).ToString("HH:mm")
|
||||
Write-Host "Starting Raven muxed A/V sender on ${RavenHost} through interactive task ${taskName}: $remoteCommand"
|
||||
if (-not $DryRun) {
|
||||
ssh $RavenHost "cmd /c echo $remoteCommand ^> $remoteCmd" | Out-Null
|
||||
ssh $RavenHost "schtasks /Delete /TN $taskName /F" | Out-Null
|
||||
ssh $RavenHost "schtasks /Create /TN $taskName /SC ONCE /ST $taskTime /TR $remoteCmd /F /IT" | Out-Host
|
||||
ssh $RavenHost "schtasks /Run /TN $taskName" | Out-Host
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Could not start Raven muxed A/V sender task on $RavenHost."
|
||||
}
|
||||
Start-Sleep -Milliseconds 1500
|
||||
}
|
||||
}
|
||||
|
||||
$arguments = @(
|
||||
"-hide_banner",
|
||||
"-y",
|
||||
"-thread_queue_size", "1024",
|
||||
"-itsoffset", $RavenOffsetSeconds.ToString([Globalization.CultureInfo]::InvariantCulture),
|
||||
"-i", $ravenEndpoint,
|
||||
"-thread_queue_size", "1024",
|
||||
"-rtbufsize", "256M",
|
||||
"-itsoffset", $KiyoOffsetSeconds.ToString([Globalization.CultureInfo]::InvariantCulture),
|
||||
"-f", "dshow",
|
||||
"-video_size", "${KiyoWidth}x${KiyoHeight}",
|
||||
"-framerate", $KiyoFrameRate.ToString(),
|
||||
"-i", "video=$KiyoVideoDevice"
|
||||
)
|
||||
|
||||
if (-not $NoStarfireScarlett) {
|
||||
$arguments += @(
|
||||
"-thread_queue_size", "1024",
|
||||
"-rtbufsize", "256M",
|
||||
"-itsoffset", $MicOffsetSeconds.ToString([Globalization.CultureInfo]::InvariantCulture),
|
||||
"-f", "dshow",
|
||||
"-i", "audio=$StarfireScarlettAudioDevice"
|
||||
)
|
||||
}
|
||||
|
||||
$scaledKiyoHeight = [Math]::Max(1, [int][Math]::Round($KiyoCornerWidth * ($KiyoHeight / [double]$KiyoWidth)))
|
||||
$overlayX = $KiyoMargin
|
||||
$overlayY = $KiyoMargin
|
||||
|
||||
if ($NoStarfireScarlett) {
|
||||
$filter = "[0:v]scale=${Width}:${Height},setsar=1,hqdn3d=1.2:1.0:3.0:2.0[base];" +
|
||||
"[1:v]scale=${KiyoCornerWidth}:${scaledKiyoHeight},setsar=1,hqdn3d=1.4:1.1:3.4:2.2[pip];" +
|
||||
"[base][pip]overlay=${overlayX}:${overlayY}:format=auto[v];" +
|
||||
"[0:a]aresample=async=1:first_pts=0,afftdn=nr=8:nf=-35[a]"
|
||||
} else {
|
||||
$filter = "[0:v]scale=${Width}:${Height},setsar=1,hqdn3d=1.2:1.0:3.0:2.0[base];" +
|
||||
"[1:v]scale=${KiyoCornerWidth}:${scaledKiyoHeight},setsar=1,hqdn3d=1.4:1.1:3.4:2.2[pip];" +
|
||||
"[base][pip]overlay=${overlayX}:${overlayY}:format=auto[v];" +
|
||||
"[0:a]aresample=async=1:first_pts=0,afftdn=nr=8:nf=-35[ravenAudio];" +
|
||||
"[2:a]aresample=async=1:first_pts=0,afftdn=nr=10:nf=-38,volume=1.0[heroMic];" +
|
||||
"[ravenAudio][heroMic]amix=inputs=2:duration=longest:normalize=0[a]"
|
||||
}
|
||||
|
||||
$arguments += @(
|
||||
"-filter_complex", $filter,
|
||||
"-map", "[v]",
|
||||
"-map", "[a]",
|
||||
"-t", $DurationSeconds.ToString(),
|
||||
"-r", $FrameRate.ToString(),
|
||||
"-c:v", "h264_nvenc",
|
||||
"-preset", "p4",
|
||||
"-tune", "ll",
|
||||
"-rc", "cbr",
|
||||
"-b:v", "4500k",
|
||||
"-maxrate", "4500k",
|
||||
"-bufsize", "9000k",
|
||||
"-g", ($FrameRate * 2).ToString(),
|
||||
"-pix_fmt", "yuv420p",
|
||||
"-c:a", "aac",
|
||||
"-b:a", "160k",
|
||||
"-ar", "48000",
|
||||
"-movflags", "+faststart",
|
||||
$absoluteOutputPath
|
||||
)
|
||||
|
||||
Write-Host "Stream-proof recording:"
|
||||
Write-Host " Raven input: $ravenEndpoint"
|
||||
Write-Host " Raven SRT mode: $(if ($RavenSenderListens) { 'Raven listener, Starfire caller' } else { 'Starfire listener, Raven caller' })"
|
||||
Write-Host " Kiyo PIP: $KiyoVideoDevice ${KiyoWidth}x${KiyoHeight}@$KiyoFrameRate -> ${KiyoCornerWidth}x${scaledKiyoHeight}"
|
||||
Write-Host " Raven audio: Realtek/default render loopback in muxed NVENC/SRT feed"
|
||||
Write-Host " Starfire Scarlett: $(if ($NoStarfireScarlett) { 'disabled' } else { $StarfireScarlettAudioDevice })"
|
||||
Write-Host " sync role: Starfire Realtek emits chirps; Raven Realtek carries co-streamer game/program audio; Starfire Scarlett captures hero mics plus Raven shotgun"
|
||||
Write-Host " Offsets: raven=${RavenOffsetSeconds}s kiyo=${KiyoOffsetSeconds}s mic=${MicOffsetSeconds}s"
|
||||
Write-Host " Output: $absoluteOutputPath"
|
||||
Write-Host " stdout: $stdoutLog"
|
||||
Write-Host " stderr: $stderrLog"
|
||||
|
||||
if ($DryRun) {
|
||||
Write-Host ($FfmpegPath + " " + ($arguments -join " "))
|
||||
exit 0
|
||||
}
|
||||
|
||||
$startInfo = [System.Diagnostics.ProcessStartInfo]::new()
|
||||
$startInfo.FileName = $FfmpegPath
|
||||
$startInfo.UseShellExecute = $false
|
||||
$startInfo.RedirectStandardOutput = $true
|
||||
$startInfo.RedirectStandardError = $true
|
||||
$startInfo.Arguments = ($arguments | ForEach-Object { Quote-ProcessArgument $_ }) -join " "
|
||||
|
||||
$process = [System.Diagnostics.Process]::Start($startInfo)
|
||||
if (-not $process) {
|
||||
throw "Could not start FFmpeg recorder."
|
||||
}
|
||||
|
||||
$stdoutTask = $process.StandardOutput.BaseStream.CopyToAsync([System.IO.File]::Open($stdoutLog, [System.IO.FileMode]::Create, [System.IO.FileAccess]::Write, [System.IO.FileShare]::Read))
|
||||
$stderrTask = $process.StandardError.BaseStream.CopyToAsync([System.IO.File]::Open($stderrLog, [System.IO.FileMode]::Create, [System.IO.FileAccess]::Write, [System.IO.FileShare]::Read))
|
||||
$timeoutMs = [Math]::Max(1, $DurationSeconds + $StartupTimeoutSeconds) * 1000
|
||||
if (-not $process.WaitForExit($timeoutMs)) {
|
||||
try { $process.Kill($true) } catch {}
|
||||
throw "FFmpeg recorder timed out after $($DurationSeconds + $StartupTimeoutSeconds)s. Check $stderrLog."
|
||||
}
|
||||
|
||||
[void]$stdoutTask.GetAwaiter().GetResult()
|
||||
[void]$stderrTask.GetAwaiter().GetResult()
|
||||
if ($process.ExitCode -ne 0) {
|
||||
throw "FFmpeg recorder failed with exit code $($process.ExitCode). Check $stderrLog."
|
||||
}
|
||||
188
scripts/restart-obs-muninn-source.ps1
Normal file
|
|
@ -0,0 +1,188 @@
|
|||
param(
|
||||
[string]$ObsExe = "C:\Program Files\obs-studio\bin\64bit\obs64.exe",
|
||||
[string]$ObsConfigRoot = "$env:APPDATA\obs-studio",
|
||||
[int[]]$ExpectedUdpPorts = @(5204, 17874),
|
||||
[int]$StartupWaitSeconds = 30,
|
||||
[switch]$NoStart
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Stop-Obs {
|
||||
$processes = Get-Process obs64,obs-browser-page,obs-ffmpeg-mux -ErrorAction SilentlyContinue
|
||||
if (-not $processes) {
|
||||
return
|
||||
}
|
||||
|
||||
& "$env:SystemRoot\System32\taskkill.exe" /IM obs64.exe /IM obs-browser-page.exe /IM obs-ffmpeg-mux.exe /T /F | Write-Host
|
||||
|
||||
$deadline = (Get-Date).AddSeconds(15)
|
||||
do {
|
||||
Start-Sleep -Milliseconds 500
|
||||
$remaining = Get-Process obs64,obs-browser-page,obs-ffmpeg-mux -ErrorAction SilentlyContinue
|
||||
if (-not $remaining) {
|
||||
return
|
||||
}
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
|
||||
$remaining | ForEach-Object {
|
||||
try {
|
||||
Stop-Process -Id $_.Id -Force -ErrorAction Stop
|
||||
} catch {
|
||||
Write-Host "Failed to stop lingering $($_.ProcessName) PID $($_.Id): $($_.Exception.Message)"
|
||||
}
|
||||
}
|
||||
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
|
||||
function Remove-ObsCrashSentinel {
|
||||
param([string]$ConfigRoot)
|
||||
|
||||
$sentinel = Join-Path $ConfigRoot ".sentinel"
|
||||
if (-not (Test-Path -LiteralPath $sentinel)) {
|
||||
return
|
||||
}
|
||||
|
||||
$resolvedSentinel = Resolve-Path -LiteralPath $sentinel
|
||||
$resolvedConfigRoot = Resolve-Path -LiteralPath $ConfigRoot
|
||||
$expected = Join-Path $resolvedConfigRoot.Path ".sentinel"
|
||||
if ($resolvedSentinel.Path -ne $expected) {
|
||||
throw "Refusing to remove unexpected OBS sentinel path '$($resolvedSentinel.Path)'."
|
||||
}
|
||||
|
||||
Remove-Item -LiteralPath $resolvedSentinel.Path -Recurse -Force
|
||||
Write-Host "Removed OBS crash sentinel: $($resolvedSentinel.Path)"
|
||||
}
|
||||
|
||||
function Get-UdpListenerLines {
|
||||
param([int[]]$Ports)
|
||||
|
||||
$patterns = $Ports | ForEach-Object { ":$_" }
|
||||
netstat -ano -p udp | Where-Object {
|
||||
$line = $_
|
||||
$patterns | Where-Object { $line -match [regex]::Escape($_) }
|
||||
}
|
||||
}
|
||||
|
||||
function Get-ObservedUdpPorts {
|
||||
param([int[]]$Ports)
|
||||
|
||||
$listeners = @(Get-UdpListenerLines -Ports $Ports)
|
||||
$observed = @{}
|
||||
foreach ($port in $Ports) {
|
||||
$pattern = ":\s*$port\s+"
|
||||
if ($listeners | Where-Object { $_ -match $pattern }) {
|
||||
$observed[$port] = $true
|
||||
}
|
||||
}
|
||||
|
||||
[pscustomobject]@{
|
||||
Lines = $listeners
|
||||
Ports = $observed
|
||||
}
|
||||
}
|
||||
|
||||
function Get-LatestObsLog {
|
||||
param([string]$ConfigRoot)
|
||||
|
||||
$logRoot = Join-Path $ConfigRoot "logs"
|
||||
if (-not (Test-Path -LiteralPath $logRoot)) {
|
||||
return $null
|
||||
}
|
||||
|
||||
Get-ChildItem -LiteralPath $logRoot -Filter "*.txt" |
|
||||
Sort-Object LastWriteTime -Descending |
|
||||
Select-Object -First 1
|
||||
}
|
||||
|
||||
function Assert-LatestObsLogWitness {
|
||||
param(
|
||||
[string]$ConfigRoot,
|
||||
[int]$WaitSeconds = 30
|
||||
)
|
||||
|
||||
$requiredPatterns = @(
|
||||
"Mimir OBS sources loaded: mimir_audio_stem_source, mimir_program_texture_source, muninn_stream_source, muninn_video_source, muninn_audio_source",
|
||||
"mimir_obs_stem_source.dll"
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($WaitSeconds)
|
||||
do {
|
||||
$latestLog = Get-LatestObsLog -ConfigRoot $ConfigRoot
|
||||
if ($latestLog) {
|
||||
$missing = @()
|
||||
foreach ($pattern in $requiredPatterns) {
|
||||
if (-not (Select-String -LiteralPath $latestLog.FullName -SimpleMatch -Pattern $pattern -Quiet)) {
|
||||
$missing += $pattern
|
||||
}
|
||||
}
|
||||
if ($missing.Count -eq 0) {
|
||||
Write-Host "Verified OBS log witness: $($latestLog.FullName)"
|
||||
return
|
||||
}
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
|
||||
if (-not $latestLog) {
|
||||
throw "OBS started, but no OBS log file was found to verify Muninn source registration."
|
||||
}
|
||||
Write-Host "Newest OBS log: $($latestLog.FullName) ($($latestLog.Length) bytes)"
|
||||
if ($latestLog.Length -gt 0) {
|
||||
Get-Content -LiteralPath $latestLog.FullName -Tail 60 | Write-Host
|
||||
}
|
||||
throw "OBS log is missing the expected Mimir Muninn source registration witness after waiting ${WaitSeconds}s: $($requiredPatterns -join '; ')."
|
||||
}
|
||||
|
||||
if (-not (Test-Path -LiteralPath $ObsExe)) {
|
||||
throw "OBS executable was not found at '$ObsExe'."
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $ObsConfigRoot)) {
|
||||
throw "OBS config root was not found at '$ObsConfigRoot'."
|
||||
}
|
||||
|
||||
Stop-Obs
|
||||
Remove-ObsCrashSentinel -ConfigRoot $ObsConfigRoot
|
||||
|
||||
if ($NoStart) {
|
||||
Write-Host "OBS stopped and crash sentinel cleared."
|
||||
return
|
||||
}
|
||||
|
||||
$workingDirectory = Split-Path -Parent $ObsExe
|
||||
Start-Process -FilePath $ObsExe -WorkingDirectory $workingDirectory
|
||||
Write-Host "Started OBS: $ObsExe"
|
||||
|
||||
$deadline = (Get-Date).AddSeconds($StartupWaitSeconds)
|
||||
$listeners = @()
|
||||
do {
|
||||
Start-Sleep -Seconds 1
|
||||
$obs = Get-Process obs64 -ErrorAction SilentlyContinue
|
||||
if (-not $obs) {
|
||||
throw "OBS did not remain running after startup."
|
||||
}
|
||||
|
||||
$observed = Get-ObservedUdpPorts -Ports $ExpectedUdpPorts
|
||||
$listeners = @($observed.Lines)
|
||||
$missingPorts = @($ExpectedUdpPorts | Where-Object { -not $observed.Ports.ContainsKey($_) })
|
||||
if ($missingPorts.Count -eq 0) {
|
||||
break
|
||||
}
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
|
||||
if ($missingPorts.Count -ne 0) {
|
||||
$latestLog = Get-LatestObsLog -ConfigRoot $ObsConfigRoot
|
||||
if ($latestLog) {
|
||||
Write-Host "Newest OBS log: $($latestLog.FullName) ($($latestLog.Length) bytes)"
|
||||
if ($latestLog.Length -gt 0) {
|
||||
Get-Content -LiteralPath $latestLog.FullName -Tail 40 | Write-Host
|
||||
}
|
||||
}
|
||||
throw "OBS is running, but expected UDP listener ports are missing: $($missingPorts -join ', ')."
|
||||
}
|
||||
|
||||
Write-Host "OBS process:"
|
||||
$obs | Select-Object Id, ProcessName, StartTime, Responding | Format-Table -AutoSize
|
||||
Write-Host "Observed Muninn UDP listeners:"
|
||||
$listeners | ForEach-Object { Write-Host $_ }
|
||||
Assert-LatestObsLogWitness -ConfigRoot $ObsConfigRoot
|
||||
141
scripts/start-eve-dashboard-stream.ps1
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
param(
|
||||
[string] $Target = "eve",
|
||||
[string] $SharedTextureName = "Global\MimirFensalirProgramTexture",
|
||||
[string] $SharedFenceName = "Global\MimirFensalirProgramFence",
|
||||
[int] $SharedTextureRingCount = 1,
|
||||
[string] $Configuration = "Debug",
|
||||
[int] $Width = 1620,
|
||||
[int] $Height = 2160,
|
||||
[int] $Fps = 30,
|
||||
[int] $Port = 8792,
|
||||
[string] $LanHost = "192.168.1.66",
|
||||
[string] $Encoder = "h264_nvenc",
|
||||
[int] $Quality = 26,
|
||||
[string] $FfmpegPath = "",
|
||||
[switch] $NoReverseTunnel,
|
||||
[string] $MimirLogPath = "logs\eve-dashboard-mimir.log",
|
||||
[string] $RelayLogPath = "logs\eve-dashboard-relay.log",
|
||||
[string] $TunnelLogPath = "logs\eve-dashboard-ssh-tunnel.log",
|
||||
[string[]] $AppArguments = @()
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = Split-Path -Parent $PSScriptRoot
|
||||
$mimirLog = Join-Path $repoRoot $MimirLogPath
|
||||
$mimirErr = [System.IO.Path]::ChangeExtension($mimirLog, ".err.log")
|
||||
$relayLog = Join-Path $repoRoot $RelayLogPath
|
||||
$relayErr = [System.IO.Path]::ChangeExtension($relayLog, ".err.log")
|
||||
$tunnelLog = Join-Path $repoRoot $TunnelLogPath
|
||||
$tunnelErr = [System.IO.Path]::ChangeExtension($tunnelLog, ".err.log")
|
||||
New-Item -ItemType Directory -Path (Split-Path -Parent $mimirLog) -Force | Out-Null
|
||||
New-Item -ItemType Directory -Path (Split-Path -Parent $relayLog) -Force | Out-Null
|
||||
New-Item -ItemType Directory -Path (Split-Path -Parent $tunnelLog) -Force | Out-Null
|
||||
|
||||
Write-Host "Checking EVE SSH target: $Target"
|
||||
ssh $Target "uname -a >/tmp/mimir-eve-dashboard-check.txt && cat /tmp/mimir-eve-dashboard-check.txt"
|
||||
|
||||
$env:FENSALIR_PROGRAM_OUTPUT_D3D12 = "1"
|
||||
$env:FENSALIR_PROGRAM_OUTPUT_NAME = $SharedTextureName
|
||||
$env:FENSALIR_PROGRAM_OUTPUT_FENCE_NAME = $SharedFenceName
|
||||
$env:FENSALIR_PROGRAM_OUTPUT_RING_COUNT = [string] $SharedTextureRingCount
|
||||
|
||||
$mimirProject = Join-Path $repoRoot "src\Mimir.App\Mimir.App.csproj"
|
||||
$mimirArgs = @(
|
||||
"run",
|
||||
"--project",
|
||||
$mimirProject,
|
||||
"-c",
|
||||
$Configuration,
|
||||
"--"
|
||||
) + $AppArguments
|
||||
|
||||
Write-Host "Starting Mimir/Fensalir program output: $SharedTextureName"
|
||||
Write-Host "Program output fence: $SharedFenceName"
|
||||
Write-Host "Program output ring slots: $SharedTextureRingCount"
|
||||
$mimirProcess = Start-Process -FilePath "dotnet" `
|
||||
-ArgumentList $mimirArgs `
|
||||
-WorkingDirectory $repoRoot `
|
||||
-RedirectStandardOutput $mimirLog `
|
||||
-RedirectStandardError $mimirErr `
|
||||
-WindowStyle Hidden `
|
||||
-PassThru
|
||||
|
||||
Write-Host "Mimir PID $($mimirProcess.Id); waiting for shared texture publication..."
|
||||
$deadline = [DateTimeOffset]::Now.AddSeconds(30)
|
||||
do {
|
||||
Start-Sleep -Milliseconds 250
|
||||
$mimirText = if (Test-Path $mimirLog) { Get-Content -Raw $mimirLog } else { "" }
|
||||
if ($mimirText -match "D3D12 program output shared texture") {
|
||||
break
|
||||
}
|
||||
} while ([DateTimeOffset]::Now -lt $deadline -and -not $mimirProcess.HasExited)
|
||||
|
||||
if ($mimirProcess.HasExited) {
|
||||
throw "Mimir exited before publishing the shared program texture. Check $mimirLog and $mimirErr."
|
||||
}
|
||||
|
||||
$relayProject = Join-Path $repoRoot "src\Mimir.EveRelay\Mimir.EveRelay.csproj"
|
||||
$relayArgs = @(
|
||||
"run",
|
||||
"--project",
|
||||
$relayProject,
|
||||
"-c",
|
||||
$Configuration,
|
||||
"--no-restore",
|
||||
"--",
|
||||
"--shared-texture", $SharedTextureName,
|
||||
"--width", $Width,
|
||||
"--height", $Height,
|
||||
"--fps", $Fps,
|
||||
"--port", $Port,
|
||||
"--lan-host", $LanHost,
|
||||
"--encoder", $Encoder,
|
||||
"--quality", $Quality
|
||||
)
|
||||
if (-not [string]::IsNullOrWhiteSpace($FfmpegPath)) {
|
||||
$relayArgs += @("--ffmpeg", $FfmpegPath)
|
||||
}
|
||||
|
||||
Write-Host "Starting H.264 Eve relay on port $Port"
|
||||
$relayProcess = Start-Process -FilePath "dotnet" `
|
||||
-ArgumentList $relayArgs `
|
||||
-WorkingDirectory $repoRoot `
|
||||
-RedirectStandardOutput $relayLog `
|
||||
-RedirectStandardError $relayErr `
|
||||
-WindowStyle Hidden `
|
||||
-PassThru
|
||||
|
||||
if (-not $NoReverseTunnel) {
|
||||
$forwardPattern = "-R ${Port}:127.0.0.1:${Port}"
|
||||
$existingTunnel = Get-CimInstance Win32_Process -Filter "name = 'ssh.exe'" |
|
||||
Where-Object { $_.CommandLine -like "*$forwardPattern*" -and $_.CommandLine -like "* $Target*" } |
|
||||
Select-Object -First 1
|
||||
|
||||
if ($existingTunnel) {
|
||||
Write-Host "Using existing EVE reverse tunnel PID $($existingTunnel.ProcessId)."
|
||||
} else {
|
||||
Write-Host "Starting EVE reverse tunnel: $Target 127.0.0.1:$Port -> Starfire 127.0.0.1:$Port"
|
||||
$tunnelProcess = Start-Process -FilePath "ssh" `
|
||||
-ArgumentList @("-N", "-R", "${Port}:127.0.0.1:${Port}", $Target) `
|
||||
-RedirectStandardOutput $tunnelLog `
|
||||
-RedirectStandardError $tunnelErr `
|
||||
-WindowStyle Hidden `
|
||||
-PassThru
|
||||
Write-Host "Tunnel PID $($tunnelProcess.Id)."
|
||||
}
|
||||
|
||||
Start-Sleep -Milliseconds 500
|
||||
ssh $Target "curl -s --max-time 3 http://127.0.0.1:$Port/health || true"
|
||||
}
|
||||
|
||||
Write-Host "Relay PID $($relayProcess.Id). Launching EveCanvas."
|
||||
ssh $Target "uiopen --bundleid org.gamecult.evecanvas"
|
||||
|
||||
Write-Host "Poll with:"
|
||||
Write-Host " Get-Content -Tail 40 '$mimirLog'"
|
||||
Write-Host " Get-Content -Tail 40 '$mimirErr'"
|
||||
Write-Host " Get-Content -Tail 40 '$relayLog'"
|
||||
Write-Host " Get-Content -Tail 40 '$relayErr'"
|
||||
Write-Host " Get-Content -Tail 40 '$tunnelLog'"
|
||||
Write-Host " Get-Content -Tail 40 '$tunnelErr'"
|
||||
53
scripts/start-eve-program-output.ps1
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
param(
|
||||
[string] $Target = "eve",
|
||||
[string] $SharedTextureName = "Global\MimirFensalirProgramTexture",
|
||||
[string] $SharedFenceName = "Global\MimirFensalirProgramFence",
|
||||
[int] $SharedTextureRingCount = 1,
|
||||
[string] $Configuration = "Debug",
|
||||
[string] $LogPath = "logs\eve-program-output-fensalir.log",
|
||||
[string[]] $AppArguments = @()
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = Split-Path -Parent $PSScriptRoot
|
||||
$absoluteLogPath = Join-Path $repoRoot $LogPath
|
||||
$absoluteErrorLogPath = [System.IO.Path]::ChangeExtension($absoluteLogPath, ".err.log")
|
||||
$logDirectory = Split-Path -Parent $absoluteLogPath
|
||||
if (-not (Test-Path $logDirectory)) {
|
||||
New-Item -ItemType Directory -Path $logDirectory | Out-Null
|
||||
}
|
||||
|
||||
Write-Host "Checking EVE SSH target: $Target"
|
||||
ssh $Target "uname -a >/tmp/mimir-eve-program-output-check.txt && cat /tmp/mimir-eve-program-output-check.txt"
|
||||
|
||||
$env:FENSALIR_PROGRAM_OUTPUT_D3D12 = "1"
|
||||
$env:FENSALIR_PROGRAM_OUTPUT_NAME = $SharedTextureName
|
||||
$env:FENSALIR_PROGRAM_OUTPUT_FENCE_NAME = $SharedFenceName
|
||||
$env:FENSALIR_PROGRAM_OUTPUT_RING_COUNT = [string] $SharedTextureRingCount
|
||||
|
||||
$projectPath = Join-Path $repoRoot "src\Mimir.App\Mimir.App.csproj"
|
||||
$dotnetArgs = @(
|
||||
"run",
|
||||
"--project",
|
||||
$projectPath,
|
||||
"-c",
|
||||
$Configuration,
|
||||
"--"
|
||||
) + $AppArguments
|
||||
|
||||
Write-Host "Starting Mimir/Fensalir with shared D3D12 output: $SharedTextureName"
|
||||
Write-Host "Program output fence: $SharedFenceName"
|
||||
Write-Host "Program output ring slots: $SharedTextureRingCount"
|
||||
Write-Host "Log: $absoluteLogPath"
|
||||
$process = Start-Process -FilePath "dotnet" `
|
||||
-ArgumentList $dotnetArgs `
|
||||
-WorkingDirectory $repoRoot `
|
||||
-RedirectStandardOutput $absoluteLogPath `
|
||||
-RedirectStandardError $absoluteErrorLogPath `
|
||||
-WindowStyle Hidden `
|
||||
-PassThru
|
||||
|
||||
Write-Host "Started PID $($process.Id). Poll with:"
|
||||
Write-Host " Get-Content -Tail 40 '$absoluteLogPath'"
|
||||
Write-Host " Get-Content -Tail 40 '$absoluteErrorLogPath'"
|
||||
103
scripts/start-fensalir-daemon.ps1
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
param(
|
||||
[string]$WellLog = "",
|
||||
[string]$CultCache = "",
|
||||
[int]$Port = 8799,
|
||||
[int]$PollMs = 100,
|
||||
[int]$WorkerMs = 4,
|
||||
[int]$MaxReservoirQueue = 120,
|
||||
[double]$GpuBudget = 0.50,
|
||||
[double]$CpuBudget = 0.25,
|
||||
[switch]$NoBuild,
|
||||
[switch]$FromStart
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$repo = Resolve-Path (Join-Path $PSScriptRoot "..")
|
||||
$project = Join-Path $repo "src\Mimir.FensalirDaemon\Mimir.FensalirDaemon.csproj"
|
||||
if (-not $NoBuild) {
|
||||
dotnet build $project | Out-Host
|
||||
}
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($CultCache)) {
|
||||
$CultCache = Join-Path $repo "state\fensalir-daemon.ccmp"
|
||||
}
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($WellLog)) {
|
||||
$runtimeRoot = Join-Path $repo "artifacts\runtime"
|
||||
if (Test-Path $runtimeRoot) {
|
||||
$WellLog = Get-ChildItem -Path $runtimeRoot -Recurse -File -Include *.log,*.jsonl |
|
||||
Where-Object { $_.Length -gt 0 } |
|
||||
Sort-Object LastWriteTimeUtc -Descending |
|
||||
Where-Object {
|
||||
try {
|
||||
Select-String -Path $_.FullName -Pattern 'mimir\.well_' -Quiet -ErrorAction Stop
|
||||
} catch {
|
||||
$false
|
||||
}
|
||||
} |
|
||||
Select-Object -First 1 -ExpandProperty FullName
|
||||
}
|
||||
}
|
||||
|
||||
$runId = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
$runDir = Join-Path $repo "artifacts\runtime\fensalir-daemon-$runId"
|
||||
New-Item -ItemType Directory -Force -Path $runDir | Out-Null
|
||||
$stdout = Join-Path $runDir "fensalir-daemon.out.log"
|
||||
$stderr = Join-Path $runDir "fensalir-daemon.err.log"
|
||||
$manifest = Join-Path $runDir "supervisor.json"
|
||||
$dll = Join-Path $repo "src\Mimir.FensalirDaemon\bin\Debug\net10.0\Mimir.FensalirDaemon.dll"
|
||||
$rootVerse = "asgard"
|
||||
$locatedService = "$rootVerse.starfire.mimir-fensalir"
|
||||
$tuiAddress = "$locatedService/eve/tui"
|
||||
$guiAddress = "$locatedService/eve/gui"
|
||||
$bridgeRoute = "ws://127.0.0.1:$Port/eve/deck"
|
||||
|
||||
$arguments = @(
|
||||
$dll,
|
||||
"--port", $Port,
|
||||
"--cultcache", $CultCache,
|
||||
"--poll-ms", $PollMs,
|
||||
"--worker-ms", $WorkerMs,
|
||||
"--max-reservoir-queue", $MaxReservoirQueue,
|
||||
"--gpu-budget", $GpuBudget.ToString([Globalization.CultureInfo]::InvariantCulture),
|
||||
"--cpu-budget", $CpuBudget.ToString([Globalization.CultureInfo]::InvariantCulture)
|
||||
)
|
||||
|
||||
if (-not [string]::IsNullOrWhiteSpace($WellLog)) {
|
||||
$arguments += @("--well-log", $WellLog)
|
||||
}
|
||||
|
||||
if ($FromStart) {
|
||||
$arguments += "--from-start"
|
||||
}
|
||||
|
||||
$process = Start-Process -FilePath "dotnet" -ArgumentList $arguments -WorkingDirectory $repo -RedirectStandardOutput $stdout -RedirectStandardError $stderr -PassThru -WindowStyle Hidden
|
||||
|
||||
[pscustomobject]@{
|
||||
daemon = "Mimir.FensalirDaemon"
|
||||
pid = $process.Id
|
||||
port = $Port
|
||||
workerMs = $WorkerMs
|
||||
maxReservoirQueue = $MaxReservoirQueue
|
||||
wellLog = $WellLog
|
||||
cultCache = $CultCache
|
||||
rootVerse = $rootVerse
|
||||
canonicalService = "$rootVerse.mimir-fensalir"
|
||||
locatedService = $locatedService
|
||||
cultMeshAddress = $tuiAddress
|
||||
providerSpec = "mimir-fensalir-daemon|Mimir Fensalir Daemon|$tuiAddress"
|
||||
endpoints = @($tuiAddress, $guiAddress)
|
||||
routes = @(
|
||||
[ordered]@{ kind = "websocket-bridge"; address = $bridgeRoute; carries = $tuiAddress; note = "Fensalir Eve deck bridge route; not service identity." },
|
||||
[ordered]@{ kind = "websocket-bridge"; address = "$bridgeRoute/cultmesh"; carries = $tuiAddress; note = "Legacy CultMesh bridge route; not service identity." }
|
||||
)
|
||||
stdout = $stdout
|
||||
stderr = $stderr
|
||||
started = (Get-Date).ToString("o")
|
||||
} | ConvertTo-Json | Set-Content -Path $manifest -Encoding UTF8
|
||||
|
||||
Write-Host "Started Mimir.FensalirDaemon PID $($process.Id)"
|
||||
Write-Host "Health: http://127.0.0.1:$Port/health"
|
||||
Write-Host "Eve provider: mimir-fensalir-daemon|Mimir Fensalir Daemon|$tuiAddress"
|
||||
Write-Host "Bridge route: $bridgeRoute"
|
||||
Write-Host "Manifest: $manifest"
|
||||
103
scripts/start-mvp-leap-eyes-kiyo.ps1
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
param(
|
||||
[switch]$Headless,
|
||||
[int]$HeadlessWidth = 1280,
|
||||
[int]$HeadlessHeight = 720,
|
||||
[string]$CaptureFrame = "",
|
||||
[string]$NightwingHost = "nightwing",
|
||||
[string]$WitnessUrl = "ws://192.168.1.66:8796/eve/periwinkle",
|
||||
[string]$ConfigPath = "E:\Projects\Mimir\config\mimir-runtime.mvp-leap-eyes-kiyo.local.json"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$repo = Split-Path -Parent $PSScriptRoot
|
||||
$rootVerse = "asgard"
|
||||
$locatedService = "$rootVerse.starfire.mimir"
|
||||
$cultMeshAddress = "$locatedService/periwinkle/eve/tui"
|
||||
$logDir = Join-Path $repo "artifacts\runtime\mvp-leap-eyes-kiyo"
|
||||
New-Item -ItemType Directory -Force -Path $logDir | Out-Null
|
||||
|
||||
function Test-CommandOk {
|
||||
param([string[]]$Command, [string]$Name)
|
||||
$process = Start-Process -FilePath $Command[0] -ArgumentList $Command[1..($Command.Length - 1)] -NoNewWindow -Wait -PassThru -RedirectStandardOutput (Join-Path $logDir "$Name.out.log") -RedirectStandardError (Join-Path $logDir "$Name.err.log")
|
||||
return $process.ExitCode -eq 0
|
||||
}
|
||||
|
||||
$kiyo = Get-PnpDevice -PresentOnly | Where-Object { $_.InstanceId -match "VID_1532&PID_0E05&MI_00" } | Select-Object -First 1
|
||||
$leap = Get-PnpDevice -PresentOnly | Where-Object { $_.InstanceId -match "VID_F182&PID_0003&MI_00" } | Select-Object -First 1
|
||||
|
||||
if (-not $kiyo) {
|
||||
throw "Kiyo Pro camera interface is not present. MVP needs Kiyo Pro RGB."
|
||||
}
|
||||
|
||||
if (-not $leap) {
|
||||
throw "LeapUVC camera interface is not present. Replug/wake the Leap before launching this MVP profile."
|
||||
}
|
||||
|
||||
$nightwingCheck = ssh $NightwingHost "test -e /dev/video2 -a -e /dev/video3 && command -v python3 >/dev/null && echo ok" 2>$null
|
||||
if ($LASTEXITCODE -ne 0 -or $nightwingCheck -notmatch "ok") {
|
||||
throw "Nightwing does not expose /dev/video2 + /dev/video3 with python3 available."
|
||||
}
|
||||
|
||||
scp (Join-Path $repo "tools\nw_eye_cap.py") "${NightwingHost}:/tmp/nw_eye_cap.py" | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Could not stage nw_eye_cap.py on Nightwing."
|
||||
}
|
||||
scp (Join-Path $repo "tools\nw_move_hint.py") "${NightwingHost}:/tmp/nw_move_hint.py" | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Could not stage nw_move_hint.py on Nightwing."
|
||||
}
|
||||
scp (Join-Path $repo "tools\nightwing_typed_witness_publisher.py") "${NightwingHost}:/tmp/nightwing_typed_witness_publisher.py" | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Could not stage nightwing_typed_witness_publisher.py on Nightwing."
|
||||
}
|
||||
|
||||
$receiverRunning = Get-CimInstance Win32_Process |
|
||||
Where-Object { $_.CommandLine -match "Mimir.EveSensorReceiver" -and $_.CommandLine -match "--port 8796" } |
|
||||
Select-Object -First 1
|
||||
if (-not $receiverRunning) {
|
||||
$receiverOut = Join-Path $logDir "nightwing-witness-receiver.out.log"
|
||||
$receiverErr = Join-Path $logDir "nightwing-witness-receiver.err.log"
|
||||
Start-Process -WindowStyle Hidden -FilePath "dotnet" -ArgumentList @(
|
||||
"run",
|
||||
"--no-build",
|
||||
"--project",
|
||||
(Join-Path $repo "src\Mimir.EveSensorReceiver\Mimir.EveSensorReceiver.csproj"),
|
||||
"--",
|
||||
"--port",
|
||||
"8796",
|
||||
"--path",
|
||||
"/eve/periwinkle",
|
||||
"--source-id",
|
||||
"nightwing-witness",
|
||||
"--type",
|
||||
"cultmesh-observation"
|
||||
) -RedirectStandardOutput $receiverOut -RedirectStandardError $receiverErr
|
||||
Start-Sleep -Milliseconds 800
|
||||
}
|
||||
|
||||
$remotePublisher = 'ps -eo pid=,comm=,args= | awk ''$2=="python3" && $0 ~ /\/tmp\/nightwing_typed_witness_publisher.py/ && $0 ~ /--track-eyes/ {print $1}'' | xargs -r kill; nohup python3 /tmp/nightwing_typed_witness_publisher.py --url ''' + $WitnessUrl + ''' --track-eyes --interval 0.12 > /tmp/mimir-nightwing-mvp-witness.log 2>&1 &'
|
||||
ssh $NightwingHost $remotePublisher | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Could not start Nightwing typed witness publisher."
|
||||
}
|
||||
|
||||
$env:MIMIR_RUNTIME_CONFIG = $ConfigPath
|
||||
$args = @("run", "--project", (Join-Path $repo "src\Mimir.App\Mimir.App.csproj"))
|
||||
if ($Headless) {
|
||||
$args += "--"
|
||||
$args += "--headless"
|
||||
$args += "--headless-width"
|
||||
$args += "$HeadlessWidth"
|
||||
$args += "--headless-height"
|
||||
$args += "$HeadlessHeight"
|
||||
if ($CaptureFrame) {
|
||||
$args += "--capture-frame"
|
||||
$args += $CaptureFrame
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "MIMIR_RUNTIME_CONFIG=$ConfigPath"
|
||||
Write-Host "CultMesh address: $cultMeshAddress"
|
||||
Write-Host "Bridge route: $WitnessUrl"
|
||||
Write-Host "Starting MVP: Starfire Leap + Kiyo Pro, Nightwing Eye/Move witness claims"
|
||||
dotnet @args
|
||||
376
scripts/start-online-verse-daemons.ps1
Normal file
|
|
@ -0,0 +1,376 @@
|
|||
param(
|
||||
[string]$NightwingHost = "nightwing",
|
||||
[int]$VerseRelayPort = 8798,
|
||||
[string]$WitnessUrl = "ws://192.168.1.66:8798/eve/periwinkle",
|
||||
[string]$SubscribeUrl = "ws://127.0.0.1:8798/eve/periwinkle/subscribe",
|
||||
[string]$ConfigPath = "E:\Projects\Mimir\config\mimir-runtime.well.local.json",
|
||||
[string]$WellPublishUrl = "ws://127.0.0.1:8798/eve/periwinkle",
|
||||
[string]$RecorderRoot = "C:\Users\Meta\Videos\Mimir\VerseCaptures",
|
||||
[int]$FensalirPort = 8799,
|
||||
[int]$FensalirWorkerMs = 2,
|
||||
[int]$FensalirMaxReservoirQueue = 120,
|
||||
[int]$DurationSeconds = 0,
|
||||
[int]$MoveFps = 120,
|
||||
[int]$MoveFftSize = 256,
|
||||
[string[]]$Move = @(
|
||||
"move-00-07-04-a3-97-72-usb=/dev/hidraw1:#35ff6c"
|
||||
),
|
||||
[switch]$SkipRuntime,
|
||||
[switch]$SkipFensalir,
|
||||
[switch]$SkipMoves,
|
||||
[switch]$SkipNightwing,
|
||||
[switch]$DryRun
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$repo = Split-Path -Parent $PSScriptRoot
|
||||
$rootVerse = "asgard"
|
||||
$locatedMimir = "$rootVerse.starfire.mimir"
|
||||
$locatedFensalir = "$rootVerse.starfire.mimir-fensalir"
|
||||
$periwinkleTui = "$locatedMimir/periwinkle/eve/tui"
|
||||
$periwinkleGui = "$locatedMimir/periwinkle/eve/gui"
|
||||
$runId = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
$runDir = Join-Path $repo "artifacts\runtime\online-verse-daemons-$runId"
|
||||
New-Item -ItemType Directory -Force -Path $runDir | Out-Null
|
||||
New-Item -ItemType Directory -Force -Path $RecorderRoot | Out-Null
|
||||
|
||||
function Start-LoggedProcess {
|
||||
param(
|
||||
[string]$Name,
|
||||
[string]$FilePath,
|
||||
[string[]]$ArgumentList,
|
||||
[hashtable]$Environment = @{}
|
||||
)
|
||||
|
||||
$stdout = Join-Path $runDir "$Name.out.log"
|
||||
$stderr = Join-Path $runDir "$Name.err.log"
|
||||
if ($DryRun) {
|
||||
Write-Host "DRY $Name`: $FilePath $($ArgumentList -join ' ')"
|
||||
return $null
|
||||
}
|
||||
|
||||
$previous = @{}
|
||||
foreach ($key in $Environment.Keys) {
|
||||
$previous[$key] = [Environment]::GetEnvironmentVariable($key, "Process")
|
||||
[Environment]::SetEnvironmentVariable($key, [string]$Environment[$key], "Process")
|
||||
}
|
||||
try {
|
||||
$process = Start-Process `
|
||||
-FilePath $FilePath `
|
||||
-ArgumentList $ArgumentList `
|
||||
-WorkingDirectory $repo `
|
||||
-WindowStyle Hidden `
|
||||
-RedirectStandardOutput $stdout `
|
||||
-RedirectStandardError $stderr `
|
||||
-PassThru
|
||||
} finally {
|
||||
foreach ($key in $Environment.Keys) {
|
||||
[Environment]::SetEnvironmentVariable($key, $previous[$key], "Process")
|
||||
}
|
||||
}
|
||||
Set-Content -Path (Join-Path $runDir "$Name.pid") -Value $process.Id
|
||||
Write-Host "$Name pid=$($process.Id) stdout=$stdout stderr=$stderr"
|
||||
return $process
|
||||
}
|
||||
|
||||
function Wait-HttpHealth {
|
||||
param(
|
||||
[string]$Url,
|
||||
[int]$TimeoutSeconds = 45
|
||||
)
|
||||
|
||||
if ($DryRun) {
|
||||
Write-Host "DRY wait health: $Url"
|
||||
return
|
||||
}
|
||||
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 3
|
||||
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
|
||||
Write-Host "health ok: $Url"
|
||||
return
|
||||
}
|
||||
} catch {
|
||||
Start-Sleep -Milliseconds 500
|
||||
}
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
|
||||
throw "Timed out waiting for $Url"
|
||||
}
|
||||
|
||||
$manifest = [ordered]@{
|
||||
kind = "mimir.online_verse_daemon_supervisor.v1"
|
||||
runId = $runId
|
||||
runDir = $runDir
|
||||
rootVerse = $rootVerse
|
||||
canonicalService = "$rootVerse.mimir"
|
||||
locatedService = $locatedMimir
|
||||
cultMeshAddress = $periwinkleTui
|
||||
endpoints = @($periwinkleTui, $periwinkleGui)
|
||||
routes = @(
|
||||
[ordered]@{ kind = "websocket-bridge"; address = $WitnessUrl; carries = $periwinkleTui; note = "Nightwing witness bridge route; not service identity." },
|
||||
[ordered]@{ kind = "websocket-bridge"; address = $SubscribeUrl; carries = $periwinkleTui; note = "Recorder subscription bridge route; not service identity." },
|
||||
[ordered]@{ kind = "websocket-bridge"; address = $WellPublishUrl; carries = $periwinkleTui; note = "Well publication bridge route; not service identity." }
|
||||
)
|
||||
witnessUrl = $WitnessUrl
|
||||
subscribeUrl = $SubscribeUrl
|
||||
wellPublishUrl = $WellPublishUrl
|
||||
verseRelayPort = $VerseRelayPort
|
||||
recorderRoot = $RecorderRoot
|
||||
fensalirPort = $FensalirPort
|
||||
configPath = $ConfigPath
|
||||
startedAt = (Get-Date).ToString("o")
|
||||
durationSeconds = $DurationSeconds
|
||||
moves = $Move
|
||||
processes = @{}
|
||||
}
|
||||
|
||||
Write-Host "Online Verse daemon run: $runDir"
|
||||
|
||||
if (-not $DryRun) {
|
||||
Get-CimInstance Win32_Process |
|
||||
Where-Object {
|
||||
($_.CommandLine -match "Mimir.EveSensorReceiver" -and $_.CommandLine -match "--port $VerseRelayPort") -or
|
||||
($_.CommandLine -match "Mimir.VerseRecorder") -or
|
||||
($_.CommandLine -match "Mimir.FensalirDaemon") -or
|
||||
($_.CommandLine -match "Mimir.Well") -or
|
||||
($_.CommandLine -match "Mimir.BufferSmoke" -and $_.CommandLine -match "--poll-ms")
|
||||
} |
|
||||
ForEach-Object {
|
||||
try { Stop-Process -Id $_.ProcessId -Force -ErrorAction Stop } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
$receiver = Start-LoggedProcess `
|
||||
-Name "verse-relay" `
|
||||
-FilePath "dotnet" `
|
||||
-ArgumentList @(
|
||||
"run",
|
||||
"--project",
|
||||
(Join-Path $repo "src\Mimir.EveSensorReceiver\Mimir.EveSensorReceiver.csproj"),
|
||||
"--",
|
||||
"--port", "$VerseRelayPort",
|
||||
"--path", "/eve/periwinkle",
|
||||
"--subscribe-path", "/eve/periwinkle/subscribe",
|
||||
"--source-id", "nightwing-witness",
|
||||
"--type", "cultmesh-observation"
|
||||
)
|
||||
if ($receiver) { $manifest.processes["verseRelayPid"] = $receiver.Id }
|
||||
|
||||
Wait-HttpHealth -Url "http://127.0.0.1:$VerseRelayPort/health"
|
||||
|
||||
$recorderArgs = @(
|
||||
"run",
|
||||
"--project",
|
||||
(Join-Path $repo "src\Mimir.VerseRecorder\Mimir.VerseRecorder.csproj"),
|
||||
"--",
|
||||
"--url", $SubscribeUrl,
|
||||
"--out-dir", $RecorderRoot,
|
||||
"--run-id", $runId,
|
||||
"--write-bodies", "true",
|
||||
"--body-page-bytes", "134217728"
|
||||
)
|
||||
if ($DurationSeconds -gt 0) {
|
||||
$recorderArgs += "--seconds"
|
||||
$recorderArgs += "$DurationSeconds"
|
||||
}
|
||||
$recorder = Start-LoggedProcess -Name "verse-recorder" -FilePath "dotnet" -ArgumentList $recorderArgs
|
||||
if ($recorder) { $manifest.processes["verseRecorderPid"] = $recorder.Id }
|
||||
|
||||
if (-not $SkipFensalir) {
|
||||
$fensalirWellLog = Join-Path (Join-Path $RecorderRoot $runId) "observations.jsonl"
|
||||
$fensalirCache = Join-Path $runDir "fensalir-daemon.ccmp"
|
||||
$fensalir = Start-LoggedProcess `
|
||||
-Name "fensalir-daemon" `
|
||||
-FilePath "dotnet" `
|
||||
-ArgumentList @(
|
||||
"run",
|
||||
"--project",
|
||||
(Join-Path $repo "src\Mimir.FensalirDaemon\Mimir.FensalirDaemon.csproj"),
|
||||
"--",
|
||||
"--port", "$FensalirPort",
|
||||
"--well-log", $fensalirWellLog,
|
||||
"--cultcache", $fensalirCache,
|
||||
"--poll-ms", "50",
|
||||
"--worker-ms", "$FensalirWorkerMs",
|
||||
"--max-reservoir-queue", "$FensalirMaxReservoirQueue"
|
||||
)
|
||||
if ($fensalir) {
|
||||
$manifest.processes["fensalirDaemonPid"] = $fensalir.Id
|
||||
$manifest.processes["fensalirWellLog"] = $fensalirWellLog
|
||||
$manifest.processes["fensalirCultCache"] = $fensalirCache
|
||||
$manifest.processes["fensalirProviderSpec"] = "mimir-fensalir-daemon|Mimir Fensalir Daemon|$locatedFensalir/eve/tui"
|
||||
$manifest.processes["fensalirGui"] = "$locatedFensalir/eve/gui"
|
||||
$manifest.processes["fensalirBridgeRoute"] = "ws://127.0.0.1:$FensalirPort/eve/deck"
|
||||
Wait-HttpHealth -Url "http://127.0.0.1:$FensalirPort/health"
|
||||
}
|
||||
}
|
||||
|
||||
if (-not $SkipNightwing) {
|
||||
if ($DryRun) {
|
||||
Write-Host "DRY Nightwing stage and witness start on $NightwingHost"
|
||||
} else {
|
||||
scp (Join-Path $repo "tools\nw_eye_cap.py") "${NightwingHost}:/tmp/nw_eye_cap.py" | Out-Null
|
||||
scp (Join-Path $repo "tools\nw_move_hint.py") "${NightwingHost}:/tmp/nw_move_hint.py" | Out-Null
|
||||
scp (Join-Path $repo "tools\nightwing_typed_witness_publisher.py") "${NightwingHost}:/tmp/nightwing_typed_witness_publisher.py" | Out-Null
|
||||
$remoteLog = "~/.local/state/gamecult/mimir-online-verse-witness-$runId.log"
|
||||
$remotePid = "~/.local/state/gamecult/mimir-online-verse-witness-$runId.pid"
|
||||
$remoteScript = @"
|
||||
mkdir -p ~/.local/state/gamecult
|
||||
pkill -f '/tmp/nightwing_typed_witness_publisher.py.*--track-eyes' || true
|
||||
nohup python3 /tmp/nightwing_typed_witness_publisher.py --url '$WitnessUrl' --track-eyes --track-builtin-camera --track-builtin-mic --interval 0.10 --eye-devices /dev/video2,/dev/video3 --eye-window-seconds 0.08 --tracking-stride 6 > $remoteLog 2>&1 < /dev/null &
|
||||
sleep 0.5
|
||||
pgrep -n -f '/tmp/nightwing_typed_witness_publisher.py.*--track-eyes' > $remotePid
|
||||
cat $remotePid
|
||||
"@
|
||||
$remoteB64 = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($remoteScript))
|
||||
$nightwingPid = ssh $NightwingHost "printf '%s' '$remoteB64' | base64 -d | bash"
|
||||
$manifest.processes["nightwingWitnessPid"] = ($nightwingPid | Select-Object -First 1)
|
||||
$manifest.processes["nightwingWitnessLog"] = $remoteLog
|
||||
}
|
||||
}
|
||||
|
||||
if (-not $SkipMoves) {
|
||||
if (-not $DryRun) {
|
||||
Get-CimInstance Win32_Process |
|
||||
Where-Object { $_.CommandLine -match "online_move_music_sync.py" } |
|
||||
ForEach-Object {
|
||||
try { Stop-Process -Id $_.ProcessId -Force -ErrorAction Stop } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
$moveArgs = @(
|
||||
".\tools\online_move_music_sync.py",
|
||||
"--duration", $(if ($DurationSeconds -gt 0) { "$DurationSeconds" } else { "31536000" }),
|
||||
"--out-dir", (Join-Path $runDir "move-sync"),
|
||||
"--device", "Realtek",
|
||||
"--asio-music-channels", "0,1,2,3",
|
||||
"--asio-music-source-name", "0=shotgun-mic",
|
||||
"--asio-music-source-name", "1=cardioid-mic",
|
||||
"--asio-music-source-name", "2=starfire-loopback-l",
|
||||
"--asio-music-source-name", "3=starfire-loopback-r-or-synth",
|
||||
"--asio-drain-blocks", "1024",
|
||||
"--fps", "$MoveFps",
|
||||
"--fft-size", "$MoveFftSize",
|
||||
"--onset-cooldown-seconds", "0.065",
|
||||
"--warmup-seconds", "2.0",
|
||||
"--loudness-history-seconds", "14.0",
|
||||
"--loudness-threshold", "0.86",
|
||||
"--hit-loudness-threshold", "0.84",
|
||||
"--loudness-floor", "0.006",
|
||||
"--loudness-exponent", "2.6",
|
||||
"--quiet-brightness", "0.0",
|
||||
"--max-brightness", "0.34",
|
||||
"--brightness-exponent", "0.88",
|
||||
"--score-subdivisions", "8",
|
||||
"--score-instrument-spacing", "2",
|
||||
"--score-release", "0.68",
|
||||
"--score-loudness-threshold", "0.90",
|
||||
"--score-min-loudness-gate", "0.08",
|
||||
"--score-loudness-exponent", "1.8",
|
||||
"--score-min-flux-percentile", "0.92",
|
||||
"--score-min-onset-intensity", "0.42",
|
||||
"--score-min-music-confidence", "0.30",
|
||||
"--score-min-improv-density", "0.04",
|
||||
"--score-max-improv-density", "0.22",
|
||||
"--score-target-confidence", "0.78",
|
||||
"--score-confidence-gesture-gain", "0.34",
|
||||
"--score-confidence-accent-gain", "0.55",
|
||||
"--score-min-voice-strength", "0.08",
|
||||
"--score-min-voice-body", "0.004",
|
||||
"--score-voice-release-seconds", "1.2",
|
||||
"--score-voice-decay", "0.72",
|
||||
"--score-downbeat-min-onset", "0.62",
|
||||
"--score-call-response-min-onset", "0.48",
|
||||
"--score-min-accent", "0.025",
|
||||
"--score-max-envelope", "0.78",
|
||||
"--score-ensemble-loudness-threshold", "0.96",
|
||||
"--score-ensemble-min-loudness-gate", "0.25",
|
||||
"--score-ensemble-min-onset-intensity", "0.86",
|
||||
"--score-ensemble-min-music-confidence", "0.55",
|
||||
"--score-ensemble-min-accent", "0.30",
|
||||
"--score-min-tempo-confidence", "0.0",
|
||||
"--debruijn-polyrhythm",
|
||||
"--harmonic-base", "1.5",
|
||||
"--microtonal-cents", "17",
|
||||
"--voice-glide-rate", "3.4",
|
||||
"--voice-vibrato-hz", "5.2",
|
||||
"--voice-vibrato-cents", "18.0",
|
||||
"--voice-note-hue-mix", "0.46",
|
||||
"--voice-vibrato-hue-width", "0.012",
|
||||
"--score-voice-roles", "violin-syrinx,harp-arpeggio,peck-syrinx",
|
||||
"--violin-glide-rate", "1.35",
|
||||
"--violin-release", "0.93",
|
||||
"--violin-call-seconds", "0.72",
|
||||
"--harp-slot-spacing", "5",
|
||||
"--harp-min-onset-intensity", "0.58",
|
||||
"--harp-call-seconds", "0.18",
|
||||
"--peck-min-onset-intensity", "0.34",
|
||||
"--peck-call-seconds", "0.11"
|
||||
)
|
||||
$moveArgs += "--emit-bioacoustic-realtk"
|
||||
$moveArgs += "--bioacoustic-song"
|
||||
$moveArgs += "aquasynth-formant-weaver"
|
||||
$moveArgs += "--bioacoustic-device"
|
||||
$moveArgs += "Realtek"
|
||||
$moveArgs += "--bioacoustic-gain"
|
||||
$moveArgs += "1.05"
|
||||
$moveArgs += "--bioacoustic-loop-seconds"
|
||||
$moveArgs += "0.42"
|
||||
$moveArgs += "--bioacoustic-min-interval-seconds"
|
||||
$moveArgs += "0.18"
|
||||
$moveArgs += "--bioacoustic-max-active-calls"
|
||||
$moveArgs += "3"
|
||||
$moveArgs += "--bioacoustic-render-timeout-seconds"
|
||||
$moveArgs += "15"
|
||||
foreach ($spec in $Move) {
|
||||
$moveArgs += "--move"
|
||||
$moveArgs += $spec
|
||||
}
|
||||
$python = "C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe"
|
||||
$moveProcess = Start-LoggedProcess -Name "move-sync" -FilePath $python -ArgumentList $moveArgs
|
||||
if ($moveProcess) { $manifest.processes["moveSyncPid"] = $moveProcess.Id }
|
||||
}
|
||||
|
||||
if (-not $SkipRuntime) {
|
||||
$runtime = Start-LoggedProcess `
|
||||
-Name "mimir-well" `
|
||||
-FilePath "dotnet" `
|
||||
-ArgumentList @(
|
||||
"run",
|
||||
"--project",
|
||||
(Join-Path $repo "src\Mimir.Well\Mimir.Well.csproj"),
|
||||
"--",
|
||||
"--seconds", $(if ($DurationSeconds -gt 0) { "$DurationSeconds" } else { "0" }),
|
||||
"--poll-ms", "5",
|
||||
"--publish-url", $WellPublishUrl,
|
||||
"--node-id", "starfire",
|
||||
"--publish-ms", "250",
|
||||
"--sync-ms", "250",
|
||||
"--presentation-delay-ms", "2500",
|
||||
"--max-samples-per-source", "32",
|
||||
"--visual-calibration", "false",
|
||||
"--visual-calibration-ms", "250",
|
||||
"--visual-expected-leds", "38",
|
||||
"--visual-minimum-luma", "0.55",
|
||||
"--visual-setting-seconds", "0.75",
|
||||
"--visual-resweep-seconds", "12.0",
|
||||
"--capture-pages", "true",
|
||||
"--capture-ms", "250",
|
||||
"--capture-max-body-bytes", "4194304",
|
||||
"--capture-inline-bodies", "false",
|
||||
"--stream-frames", "true",
|
||||
"--stream-frame-inline-bodies", "true"
|
||||
) `
|
||||
-Environment @{
|
||||
"MIMIR_RUNTIME_CONFIG" = $ConfigPath
|
||||
"MIMIR_SYNC_TELEMETRY_SECONDS" = "1"
|
||||
}
|
||||
if ($runtime) { $manifest.processes["mimirRuntimePid"] = $runtime.Id }
|
||||
}
|
||||
|
||||
$manifestPath = Join-Path $runDir "supervisor.json"
|
||||
$manifest | ConvertTo-Json -Depth 6 | Set-Content -Path $manifestPath
|
||||
Write-Host "Supervisor manifest: $manifestPath"
|
||||
69
scripts/start-raven-av-demux.ps1
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
param(
|
||||
[int]$InputPort = 5200,
|
||||
[int]$VideoPort = 5210,
|
||||
[int]$AudioPort = 5212,
|
||||
[int]$AudioSampleRate = 48000,
|
||||
[int]$AudioChannels = 2,
|
||||
[string]$FfmpegPath = "ffmpeg",
|
||||
[string]$LogRoot = "E:\Projects\Mimir\artifacts\runtime\stream-proof",
|
||||
[switch]$DryRun
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $LogRoot | Out-Null
|
||||
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
$stdoutLog = Join-Path $LogRoot "raven-av-demux-$timestamp.out.log"
|
||||
$stderrLog = Join-Path $LogRoot "raven-av-demux-$timestamp.err.log"
|
||||
$inputEndpoint = "srt://0.0.0.0:${InputPort}?mode=listener&latency=120000&timeout=5000000"
|
||||
$videoEndpoint = "srt://127.0.0.1:${VideoPort}?mode=caller&latency=20000"
|
||||
$audioEndpoint = "srt://127.0.0.1:${AudioPort}?mode=caller&latency=20000"
|
||||
|
||||
$arguments = @(
|
||||
"-hide_banner",
|
||||
"-nostdin",
|
||||
"-loglevel", "warning",
|
||||
"-fflags", "nobuffer",
|
||||
"-flags", "low_delay",
|
||||
"-i", $inputEndpoint,
|
||||
"-map", "0:v:0",
|
||||
"-an",
|
||||
"-c:v", "copy",
|
||||
"-f", "mpegts",
|
||||
$videoEndpoint,
|
||||
"-map", "0:a:0",
|
||||
"-vn",
|
||||
"-acodec", "pcm_f32le",
|
||||
"-ar", $AudioSampleRate.ToString(),
|
||||
"-ac", $AudioChannels.ToString(),
|
||||
"-f", "f32le",
|
||||
$audioEndpoint
|
||||
)
|
||||
|
||||
Write-Host "Raven muxed A/V demux:"
|
||||
Write-Host " input: $inputEndpoint"
|
||||
Write-Host " video: $videoEndpoint"
|
||||
Write-Host " audio: $audioEndpoint"
|
||||
Write-Host " stdout: $stdoutLog"
|
||||
Write-Host " stderr: $stderrLog"
|
||||
|
||||
if ($DryRun) {
|
||||
Write-Host ($FfmpegPath + " " + ($arguments -join " "))
|
||||
exit 0
|
||||
}
|
||||
|
||||
Get-CimInstance Win32_Process |
|
||||
Where-Object { $_.ProcessName -like "ffmpeg*" -and $_.CommandLine -match [regex]::Escape($inputEndpoint) } |
|
||||
ForEach-Object {
|
||||
try { Stop-Process -Id $_.ProcessId -Force -ErrorAction Stop } catch {}
|
||||
}
|
||||
|
||||
$process = Start-Process `
|
||||
-FilePath $FfmpegPath `
|
||||
-ArgumentList $arguments `
|
||||
-WindowStyle Hidden `
|
||||
-RedirectStandardOutput $stdoutLog `
|
||||
-RedirectStandardError $stderrLog `
|
||||
-PassThru
|
||||
|
||||
Write-Host "Started Raven muxed A/V demux pid=$($process.Id)"
|
||||
142
scripts/start-raven-av-sender.ps1
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
param(
|
||||
[string]$TargetHost = "192.168.1.66",
|
||||
[int]$Port = 5200,
|
||||
[int]$Width = 1920,
|
||||
[int]$Height = 1080,
|
||||
[int]$Framerate = 30,
|
||||
[int]$AudioSampleRate = 48000,
|
||||
[int]$AudioChannels = 2,
|
||||
[ValidateSet("ddagrab", "gdigrab")]
|
||||
[string]$VideoCapture = "ddagrab",
|
||||
[int]$DdagrabOutputIndex = 0,
|
||||
[ValidateSet("srt", "tcp-listener")]
|
||||
[string]$Transport = "srt",
|
||||
[ValidateSet("caller", "listener")]
|
||||
[string]$SrtMode = "caller",
|
||||
[string]$FfmpegPath = "ffmpeg",
|
||||
[string]$Source = "desktop",
|
||||
[string]$VideoBitrate = "12000k",
|
||||
[string]$AudioBitrate = "192k",
|
||||
[string]$LogRoot = "C:\Meta\Mimir\logs",
|
||||
[string]$WasapiLoopbackPath = "",
|
||||
[switch]$DryRun
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Quote-CmdArgument([string]$Value) {
|
||||
return '"' + $Value.Replace('"', '\"') + '"'
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $LogRoot | Out-Null
|
||||
$repo = Split-Path -Parent $PSScriptRoot
|
||||
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
$stdoutLog = Join-Path $LogRoot "raven-av-srt-$timestamp.out.log"
|
||||
$stderrLog = Join-Path $LogRoot "raven-av-srt-$timestamp.err.log"
|
||||
$endpoint = if ($Transport -eq "tcp-listener") {
|
||||
"tcp://${TargetHost}:${Port}?listen=1"
|
||||
} else {
|
||||
"srt://${TargetHost}:${Port}?mode=${SrtMode}&latency=120000&timeout=30000000"
|
||||
}
|
||||
$videoSize = "${Width}x${Height}"
|
||||
$gop = [Math]::Max(1, $Framerate * 2)
|
||||
$captureScript = Join-Path $repo "scripts\wasapi-loopback-capture.ps1"
|
||||
$loopbackExe = if ([string]::IsNullOrWhiteSpace($WasapiLoopbackPath)) {
|
||||
Join-Path $repo "tools\Mimir.WasapiLoopback\Mimir.WasapiLoopback.exe"
|
||||
} else {
|
||||
$WasapiLoopbackPath
|
||||
}
|
||||
$audioCaptureCommand = if (Test-Path $loopbackExe) {
|
||||
(Quote-CmdArgument $loopbackExe) +
|
||||
" --output stdout --sample-rate $AudioSampleRate --channels $AudioChannels"
|
||||
} else {
|
||||
"powershell.exe -NoProfile -ExecutionPolicy Bypass -File " +
|
||||
(Quote-CmdArgument $captureScript) +
|
||||
" -Output " +
|
||||
(Quote-CmdArgument "stdout") +
|
||||
" -SampleRate $AudioSampleRate -Channels $AudioChannels"
|
||||
}
|
||||
|
||||
$videoInputArgs = if ($VideoCapture -eq "ddagrab") {
|
||||
@(
|
||||
"-thread_queue_size", "1024",
|
||||
"-f", "lavfi",
|
||||
"-i", "ddagrab=framerate=${Framerate}:output_idx=${DdagrabOutputIndex}:draw_mouse=1"
|
||||
)
|
||||
} else {
|
||||
@(
|
||||
"-thread_queue_size", "1024",
|
||||
"-f", "gdigrab",
|
||||
"-framerate", $Framerate.ToString(),
|
||||
"-video_size", $videoSize,
|
||||
"-i", $Source
|
||||
)
|
||||
}
|
||||
|
||||
$videoEncodeArgs = @(
|
||||
"-c:v", "h264_nvenc",
|
||||
"-preset", "p4",
|
||||
"-tune", "ll",
|
||||
"-b:v", $VideoBitrate,
|
||||
"-maxrate", $VideoBitrate,
|
||||
"-bufsize", "24000k",
|
||||
"-g", $gop.ToString()
|
||||
)
|
||||
if ($VideoCapture -eq "gdigrab") {
|
||||
$videoEncodeArgs += @("-pix_fmt", "yuv420p")
|
||||
}
|
||||
|
||||
$ffmpegArgs = @(
|
||||
"-hide_banner",
|
||||
"-loglevel", "warning"
|
||||
)
|
||||
$ffmpegArgs += $videoInputArgs
|
||||
$ffmpegArgs += @(
|
||||
"-thread_queue_size", "1024",
|
||||
"-f", "f32le",
|
||||
"-ar", $AudioSampleRate.ToString(),
|
||||
"-ac", $AudioChannels.ToString(),
|
||||
"-i", "pipe:0",
|
||||
"-map", "0:v:0",
|
||||
"-map", "1:a:0"
|
||||
)
|
||||
$ffmpegArgs += $videoEncodeArgs
|
||||
$ffmpegArgs += @(
|
||||
"-c:a", "aac",
|
||||
"-b:a", $AudioBitrate,
|
||||
"-ar", $AudioSampleRate.ToString(),
|
||||
"-ac", $AudioChannels.ToString(),
|
||||
"-f", "mpegts",
|
||||
$endpoint
|
||||
)
|
||||
$quotedFfmpegArgs = $ffmpegArgs | ForEach-Object { Quote-CmdArgument $_ }
|
||||
$command = $audioCaptureCommand +
|
||||
" | " +
|
||||
(Quote-CmdArgument $FfmpegPath) +
|
||||
" " +
|
||||
($quotedFfmpegArgs -join " ")
|
||||
|
||||
Write-Host "Raven muxed A/V sender:"
|
||||
Write-Host " ffmpeg: $FfmpegPath"
|
||||
Write-Host " target: $endpoint"
|
||||
Write-Host " transport: $Transport"
|
||||
Write-Host " video: $(if ($VideoCapture -eq "ddagrab") { "ddagrab output $DdagrabOutputIndex" } else { "$Source $videoSize" })@$Framerate via h264_nvenc"
|
||||
Write-Host " audio: Raven Realtek/default render loopback ${AudioChannels}ch ${AudioSampleRate}Hz f32 -> AAC"
|
||||
Write-Host " loopback: $(if (Test-Path $loopbackExe) { $loopbackExe } else { "$captureScript (diagnostic fallback)" })"
|
||||
Write-Host " sync role: Raven Realtek is co-streamer game/program loopback packaged with NVENC; Starfire Realtek owns chirp emission"
|
||||
Write-Host " stdout: $stdoutLog"
|
||||
Write-Host " stderr: $stderrLog"
|
||||
|
||||
if ($DryRun) {
|
||||
Write-Host $command
|
||||
exit 0
|
||||
}
|
||||
|
||||
Start-Process `
|
||||
-FilePath "cmd.exe" `
|
||||
-ArgumentList ("/d /s /c `"$command`"") `
|
||||
-WindowStyle Hidden `
|
||||
-RedirectStandardOutput $stdoutLog `
|
||||
-RedirectStandardError $stderrLog `
|
||||
-PassThru |
|
||||
ForEach-Object { Write-Host "Started Raven muxed A/V sender pid=$($_.Id)" }
|
||||
95
scripts/start-raven-daemon.ps1
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
param(
|
||||
[string]$TargetHost = "10.77.0.2",
|
||||
[int]$Port = 5200,
|
||||
[string]$ObsTargetHost = "",
|
||||
[int]$ObsPort = 5204,
|
||||
[int]$EvePort = 8801,
|
||||
[int]$Width = 1920,
|
||||
[int]$Height = 1080,
|
||||
[int]$Framerate = 30,
|
||||
[int]$AudioSampleRate = 48000,
|
||||
[int]$AudioChannels = 2,
|
||||
[ValidateSet("ddagrab", "gdigrab")]
|
||||
[string]$VideoCapture = "ddagrab",
|
||||
[int]$DdagrabOutputIndex = 0,
|
||||
[string]$AudioDevice = "Realtek",
|
||||
[string]$FfmpegPath = "ffmpeg",
|
||||
[string]$WasapiLoopbackPath = "",
|
||||
[string]$CultMeshCachePath = "C:\Meta\Mimir\state\raven-capture-mux.ccmp",
|
||||
[string]$LogRoot = "C:\Meta\Mimir\logs",
|
||||
[switch]$NoObsTarget,
|
||||
[switch]$NoCultNetServer,
|
||||
[switch]$DryRun
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repo = Split-Path -Parent $PSScriptRoot
|
||||
if ([string]::IsNullOrWhiteSpace($ObsTargetHost)) {
|
||||
$ObsTargetHost = $TargetHost
|
||||
}
|
||||
$project = Join-Path $repo "src\Mimir.RavenDaemon\Mimir.RavenDaemon.csproj"
|
||||
$runtimeArgs = @(
|
||||
"run",
|
||||
"--project", $project,
|
||||
"--",
|
||||
"--target-host", $TargetHost,
|
||||
"--port", $Port,
|
||||
"--obs-target-host", $ObsTargetHost,
|
||||
"--obs-port", $ObsPort,
|
||||
"--eve-port", $EvePort,
|
||||
"--width", $Width,
|
||||
"--height", $Height,
|
||||
"--framerate", $Framerate,
|
||||
"--audio-sample-rate", $AudioSampleRate,
|
||||
"--audio-channels", $AudioChannels,
|
||||
"--video-capture", $VideoCapture,
|
||||
"--ddagrab-output-index", $DdagrabOutputIndex,
|
||||
"--audio-device", $AudioDevice,
|
||||
"--ffmpeg", $FfmpegPath,
|
||||
"--cultmesh-cache", $CultMeshCachePath,
|
||||
"--log-root", $LogRoot
|
||||
)
|
||||
|
||||
if ($NoObsTarget) {
|
||||
$runtimeArgs += "--no-obs-target"
|
||||
}
|
||||
|
||||
if (-not [string]::IsNullOrWhiteSpace($WasapiLoopbackPath)) {
|
||||
$runtimeArgs += @("--wasapi-loopback", $WasapiLoopbackPath)
|
||||
}
|
||||
|
||||
if ($NoCultNetServer) {
|
||||
$runtimeArgs += "--no-cultnet-server"
|
||||
}
|
||||
|
||||
if ($DryRun) {
|
||||
$runtimeArgs += "--dry-run"
|
||||
& dotnet @runtimeArgs
|
||||
exit $LASTEXITCODE
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $LogRoot | Out-Null
|
||||
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $CultMeshCachePath) | Out-Null
|
||||
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
$stdoutLog = Join-Path $LogRoot "mimir-raven-daemon-$timestamp.out.log"
|
||||
$stderrLog = Join-Path $LogRoot "mimir-raven-daemon-$timestamp.err.log"
|
||||
|
||||
$process = Start-Process `
|
||||
-FilePath "dotnet" `
|
||||
-ArgumentList $runtimeArgs `
|
||||
-WindowStyle Hidden `
|
||||
-RedirectStandardOutput $stdoutLog `
|
||||
-RedirectStandardError $stderrLog `
|
||||
-PassThru
|
||||
|
||||
Write-Host "Started Mimir.RavenDaemon pid=$($process.Id)"
|
||||
Write-Host " Eve/CultMesh: ws://0.0.0.0:${EvePort}/eve/deck"
|
||||
Write-Host " Health: http://127.0.0.1:${EvePort}/health"
|
||||
Write-Host " Mimir target: srt://${TargetHost}:${Port}"
|
||||
if (-not $NoObsTarget) {
|
||||
Write-Host " OBS target: srt://${ObsTargetHost}:${ObsPort}"
|
||||
}
|
||||
Write-Host " CultMesh cache: $CultMeshCachePath"
|
||||
Write-Host " stdout: $stdoutLog"
|
||||
Write-Host " stderr: $stderrLog"
|
||||
66
scripts/start-raven-screen-capture-sender.ps1
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
param(
|
||||
[string]$TargetHost = "192.168.1.66",
|
||||
[int]$Port = 5200,
|
||||
[int]$Width = 1920,
|
||||
[int]$Height = 1080,
|
||||
[int]$Framerate = 30,
|
||||
[string]$FfmpegPath = "ffmpeg",
|
||||
[string]$Source = "desktop",
|
||||
[string]$VideoBitrate = "12000k",
|
||||
[string]$LogRoot = "C:\Meta\Mimir\logs",
|
||||
[switch]$DryRun
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $LogRoot | Out-Null
|
||||
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
$stdoutLog = Join-Path $LogRoot "raven-screen-srt-$timestamp.out.log"
|
||||
$stderrLog = Join-Path $LogRoot "raven-screen-srt-$timestamp.err.log"
|
||||
$endpoint = "srt://${TargetHost}:${Port}?mode=caller&latency=120000"
|
||||
$videoSize = "${Width}x${Height}"
|
||||
$gop = [Math]::Max(1, $Framerate * 2)
|
||||
|
||||
$arguments = @(
|
||||
"-hide_banner",
|
||||
"-nostdin",
|
||||
"-loglevel", "warning",
|
||||
"-f", "gdigrab",
|
||||
"-framerate", $Framerate.ToString(),
|
||||
"-video_size", $videoSize,
|
||||
"-i", $Source,
|
||||
"-an",
|
||||
"-c:v", "h264_nvenc",
|
||||
"-preset", "p4",
|
||||
"-tune", "ll",
|
||||
"-b:v", $VideoBitrate,
|
||||
"-maxrate", $VideoBitrate,
|
||||
"-bufsize", "24000k",
|
||||
"-pix_fmt", "yuv420p",
|
||||
"-g", $gop.ToString(),
|
||||
"-f", "mpegts",
|
||||
$endpoint
|
||||
)
|
||||
|
||||
Write-Host "Raven screen sender:"
|
||||
Write-Host " ffmpeg: $FfmpegPath"
|
||||
Write-Host " target: $endpoint"
|
||||
Write-Host " source: $Source $videoSize@$Framerate"
|
||||
Write-Host " stdout: $stdoutLog"
|
||||
Write-Host " stderr: $stderrLog"
|
||||
|
||||
if ($DryRun)
|
||||
{
|
||||
Write-Host ($FfmpegPath + " " + ($arguments -join " "))
|
||||
exit 0
|
||||
}
|
||||
|
||||
$process = Start-Process `
|
||||
-FilePath $FfmpegPath `
|
||||
-ArgumentList $arguments `
|
||||
-WindowStyle Hidden `
|
||||
-RedirectStandardOutput $stdoutLog `
|
||||
-RedirectStandardError $stderrLog `
|
||||
-PassThru
|
||||
|
||||
Write-Host "Started Raven screen sender pid=$($process.Id)"
|
||||
146
scripts/start-stream-proof.ps1
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
param(
|
||||
[switch]$Headless,
|
||||
[int]$HeadlessWidth = 1280,
|
||||
[int]$HeadlessHeight = 720,
|
||||
[string]$CaptureFrame = "",
|
||||
[string]$NightwingHost = "nightwing",
|
||||
[switch]$StartRavenSender,
|
||||
[string]$RavenHost = "madman's lullaby@192.168.1.84",
|
||||
[string]$RavenRepo = "C:\Meta\Mimir",
|
||||
[string]$StarfireHost = "192.168.1.66",
|
||||
[string]$WitnessUrl = "ws://192.168.1.66:8796/eve/periwinkle",
|
||||
[string]$ConfigPath = "E:\Projects\Mimir\config\mimir-runtime.stream-proof.local.json",
|
||||
[switch]$ProgramOutput,
|
||||
[string]$SharedTextureName = "Global\MimirFensalirProgramTexture",
|
||||
[string]$SharedFenceName = "Global\MimirFensalirProgramFence",
|
||||
[int]$SharedTextureRingCount = 1,
|
||||
[switch]$DryRun
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$repo = Split-Path -Parent $PSScriptRoot
|
||||
$logDir = Join-Path $repo "artifacts\runtime\stream-proof"
|
||||
New-Item -ItemType Directory -Force -Path $logDir | Out-Null
|
||||
|
||||
if ($DryRun) {
|
||||
Write-Host "MIMIR_RUNTIME_CONFIG=$ConfigPath"
|
||||
if ($ProgramOutput) {
|
||||
Write-Host "FENSALIR_PROGRAM_OUTPUT_D3D12=1"
|
||||
Write-Host "FENSALIR_PROGRAM_OUTPUT_NAME=$SharedTextureName"
|
||||
Write-Host "FENSALIR_PROGRAM_OUTPUT_FENCE_NAME=$SharedFenceName"
|
||||
Write-Host "FENSALIR_PROGRAM_OUTPUT_RING_COUNT=$SharedTextureRingCount"
|
||||
}
|
||||
Write-Host "Would stage Nightwing witness tools to $NightwingHost and start --track-eyes publisher at $WitnessUrl"
|
||||
if ($StartRavenSender) {
|
||||
Write-Host "Would start Raven muxed A/V sender on ${RavenHost}: ${RavenRepo}\scripts\start-raven-av-sender.ps1 -TargetHost $StarfireHost -Port 5200"
|
||||
}
|
||||
Write-Host "Would start local Raven A/V demux: scripts\start-raven-av-demux.ps1"
|
||||
Write-Host "Would launch: dotnet run --project $repo\src\Mimir.App\Mimir.App.csproj"
|
||||
Write-Host "Program layout: raven-display full-frame, kiyo-pro-rgb picture-in-picture"
|
||||
exit 0
|
||||
}
|
||||
|
||||
$kiyo = Get-PnpDevice -PresentOnly | Where-Object { $_.InstanceId -match "VID_1532&PID_0E05&MI_00" } | Select-Object -First 1
|
||||
$leap = Get-PnpDevice -PresentOnly | Where-Object { $_.InstanceId -match "VID_F182&PID_0003&MI_00" } | Select-Object -First 1
|
||||
|
||||
if (-not $kiyo) {
|
||||
throw "Kiyo Pro camera interface is not present. Stream proof needs the Kiyo AR program view."
|
||||
}
|
||||
|
||||
if (-not $leap) {
|
||||
throw "LeapUVC camera interface is not present. Replug/wake the Leap before launching this profile."
|
||||
}
|
||||
|
||||
$nightwingCheck = ssh $NightwingHost "test -e /dev/video2 -a -e /dev/video3 && command -v python3 >/dev/null && echo ok" 2>$null
|
||||
if ($LASTEXITCODE -ne 0 -or $nightwingCheck -notmatch "ok") {
|
||||
throw "Nightwing does not expose /dev/video2 + /dev/video3 with python3 available."
|
||||
}
|
||||
|
||||
scp (Join-Path $repo "tools\nw_eye_cap.py") "${NightwingHost}:/tmp/nw_eye_cap.py" | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Could not stage nw_eye_cap.py on Nightwing."
|
||||
}
|
||||
scp (Join-Path $repo "tools\nw_move_hint.py") "${NightwingHost}:/tmp/nw_move_hint.py" | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Could not stage nw_move_hint.py on Nightwing."
|
||||
}
|
||||
scp (Join-Path $repo "tools\nightwing_typed_witness_publisher.py") "${NightwingHost}:/tmp/nightwing_typed_witness_publisher.py" | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Could not stage nightwing_typed_witness_publisher.py on Nightwing."
|
||||
}
|
||||
|
||||
$receiverRunning = Get-CimInstance Win32_Process |
|
||||
Where-Object { $_.CommandLine -match "Mimir.EveSensorReceiver" -and $_.CommandLine -match "--port 8796" } |
|
||||
Select-Object -First 1
|
||||
if (-not $receiverRunning) {
|
||||
$receiverOut = Join-Path $logDir "nightwing-witness-receiver.out.log"
|
||||
$receiverErr = Join-Path $logDir "nightwing-witness-receiver.err.log"
|
||||
Start-Process -WindowStyle Hidden -FilePath "dotnet" -ArgumentList @(
|
||||
"run",
|
||||
"--no-build",
|
||||
"--project",
|
||||
(Join-Path $repo "src\Mimir.EveSensorReceiver\Mimir.EveSensorReceiver.csproj"),
|
||||
"--",
|
||||
"--port",
|
||||
"8796",
|
||||
"--path",
|
||||
"/eve/periwinkle",
|
||||
"--source-id",
|
||||
"nightwing-witness",
|
||||
"--type",
|
||||
"cultmesh-observation"
|
||||
) -RedirectStandardOutput $receiverOut -RedirectStandardError $receiverErr
|
||||
Start-Sleep -Milliseconds 800
|
||||
}
|
||||
|
||||
$remotePublisher = 'ps -eo pid=,comm=,args= | awk ''$2=="python3" && $0 ~ /\/tmp\/nightwing_typed_witness_publisher.py/ && $0 ~ /--track-eyes/ {print $1}'' | xargs -r kill; nohup python3 /tmp/nightwing_typed_witness_publisher.py --url ''' + $WitnessUrl + ''' --track-eyes --interval 0.12 > /tmp/mimir-nightwing-stream-proof.log 2>&1 &'
|
||||
ssh $NightwingHost $remotePublisher | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Could not start Nightwing typed witness publisher."
|
||||
}
|
||||
|
||||
& (Join-Path $repo "scripts\start-raven-av-demux.ps1")
|
||||
|
||||
if ($StartRavenSender) {
|
||||
$taskName = "MimirRavenAvSender5200"
|
||||
$ravenSender = "${RavenRepo}\scripts\start-raven-av-sender.ps1"
|
||||
$remoteCmd = "${RavenRepo}\scripts\run-raven-av-5200.cmd"
|
||||
$taskTime = (Get-Date).AddMinutes(1).ToString("HH:mm")
|
||||
ssh $RavenHost "cmd /c echo powershell.exe -NoProfile -ExecutionPolicy Bypass -File $ravenSender -TargetHost $StarfireHost -Port 5200 ^> $remoteCmd" | Out-Null
|
||||
ssh $RavenHost "schtasks /Delete /TN $taskName /F" | Out-Null
|
||||
ssh $RavenHost "schtasks /Create /TN $taskName /SC ONCE /ST $taskTime /TR $remoteCmd /F /IT" | Out-Host
|
||||
ssh $RavenHost "schtasks /Run /TN $taskName" | Out-Host
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Could not start Raven muxed A/V sender task on $RavenHost."
|
||||
}
|
||||
Start-Sleep -Milliseconds 1500
|
||||
}
|
||||
|
||||
$env:MIMIR_RUNTIME_CONFIG = $ConfigPath
|
||||
if ($ProgramOutput) {
|
||||
$env:FENSALIR_PROGRAM_OUTPUT_D3D12 = "1"
|
||||
$env:FENSALIR_PROGRAM_OUTPUT_NAME = $SharedTextureName
|
||||
$env:FENSALIR_PROGRAM_OUTPUT_FENCE_NAME = $SharedFenceName
|
||||
$env:FENSALIR_PROGRAM_OUTPUT_RING_COUNT = [string]$SharedTextureRingCount
|
||||
}
|
||||
$args = @("run", "--project", (Join-Path $repo "src\Mimir.App\Mimir.App.csproj"))
|
||||
if ($Headless) {
|
||||
$args += "--"
|
||||
$args += "--headless"
|
||||
$args += "--headless-width"
|
||||
$args += "$HeadlessWidth"
|
||||
$args += "--headless-height"
|
||||
$args += "$HeadlessHeight"
|
||||
if ($CaptureFrame) {
|
||||
$args += "--capture-frame"
|
||||
$args += $CaptureFrame
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "MIMIR_RUNTIME_CONFIG=$ConfigPath"
|
||||
Write-Host "Starting stream proof: Leap + Kiyo Pro + Raven screen/audio, Nightwing Eye/Move witness claims"
|
||||
if ($ProgramOutput) {
|
||||
Write-Host "Program output texture: $SharedTextureName"
|
||||
}
|
||||
Write-Host "Program layout: raven-display full-frame, kiyo-pro-rgb picture-in-picture"
|
||||
dotnet @args
|
||||
74
scripts/start-well-obs-daemon.ps1
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
param(
|
||||
[string]$WellLog = "",
|
||||
[string]$MapName = "Local\MimirObsStemBus",
|
||||
[string]$CompositeStemId = "well_composite",
|
||||
[int]$PollMs = 25,
|
||||
[switch]$FromStart
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$repo = Split-Path -Parent $PSScriptRoot
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($WellLog)) {
|
||||
$runtime = Join-Path $repo "artifacts\runtime"
|
||||
$latest = Get-ChildItem -Path $runtime -Filter "verse-relay.out.log" -Recurse -ErrorAction SilentlyContinue |
|
||||
Sort-Object LastWriteTimeUtc -Descending |
|
||||
Select-Object -First 1
|
||||
if (-not $latest) {
|
||||
throw "No verse-relay.out.log found under $runtime. Pass -WellLog explicitly."
|
||||
}
|
||||
|
||||
$WellLog = $latest.FullName
|
||||
}
|
||||
|
||||
$runDir = Join-Path $repo ("artifacts\runtime\well-obs-daemon-" + (Get-Date -Format "yyyyMMdd-HHmmss"))
|
||||
New-Item -ItemType Directory -Force -Path $runDir | Out-Null
|
||||
$stdout = Join-Path $runDir "well-obs-daemon.out.log"
|
||||
$stderr = Join-Path $runDir "well-obs-daemon.err.log"
|
||||
$daemonExe = Join-Path $repo "src\Mimir.WellObsDaemon\bin\Debug\net10.0-windows\Mimir.WellObsDaemon.exe"
|
||||
|
||||
if (-not (Test-Path $daemonExe)) {
|
||||
dotnet build (Join-Path $repo "src\Mimir.WellObsDaemon\Mimir.WellObsDaemon.csproj")
|
||||
}
|
||||
|
||||
$daemonArgs = @(
|
||||
"--well-log",
|
||||
$WellLog,
|
||||
"--map",
|
||||
$MapName,
|
||||
"--composite-stem-id",
|
||||
$CompositeStemId,
|
||||
"--poll-ms",
|
||||
"$PollMs"
|
||||
)
|
||||
|
||||
if ($FromStart) {
|
||||
$daemonArgs += "--from-start"
|
||||
}
|
||||
|
||||
$process = Start-Process `
|
||||
-FilePath $daemonExe `
|
||||
-ArgumentList $daemonArgs `
|
||||
-WorkingDirectory $repo `
|
||||
-WindowStyle Hidden `
|
||||
-RedirectStandardOutput $stdout `
|
||||
-RedirectStandardError $stderr `
|
||||
-PassThru
|
||||
|
||||
$manifest = [ordered]@{
|
||||
document = "mimir.well_obs_daemon.supervisor.v1"
|
||||
pid = $process.Id
|
||||
wellLog = $WellLog
|
||||
mapName = $MapName
|
||||
compositeStemId = $CompositeStemId
|
||||
stdout = $stdout
|
||||
stderr = $stderr
|
||||
startedAt = (Get-Date).ToUniversalTime().ToString("O")
|
||||
}
|
||||
|
||||
$manifestPath = Join-Path $runDir "supervisor.json"
|
||||
$manifest | ConvertTo-Json -Depth 6 | Set-Content -Path $manifestPath -Encoding UTF8
|
||||
Write-Output "Mimir Well OBS daemon PID $($process.Id)"
|
||||
Write-Output "Supervisor manifest: $manifestPath"
|
||||
Write-Output "Requires Well capture pages with inline audio bodies enabled."
|
||||
Write-Output "Poll log: Get-Content -Tail 20 '$stderr'"
|
||||
|
|
@ -112,7 +112,7 @@ namespace Mimir {
|
|||
}
|
||||
|
||||
public static void Capture(string outputPath, double seconds, int targetRate, int targetChannels, string roleName) {
|
||||
Stream output = outputPath == "-" ? Console.OpenStandardOutput() : new FileStream(outputPath, FileMode.Create, FileAccess.Write, FileShare.Read);
|
||||
Stream output = IsStdoutPath(outputPath) ? Console.OpenStandardOutput() : new FileStream(outputPath, FileMode.Create, FileAccess.Write, FileShare.Read);
|
||||
try {
|
||||
IMMDeviceEnumerator enumerator = (IMMDeviceEnumerator)(new MMDeviceEnumerator());
|
||||
IMMDevice device;
|
||||
|
|
@ -160,6 +160,12 @@ namespace Mimir {
|
|||
Check(audioClient.Start(), "Start");
|
||||
DateTime end = seconds > 0 ? DateTime.UtcNow.AddSeconds(seconds) : DateTime.MaxValue;
|
||||
byte[] pcm = new byte[8192];
|
||||
int sourceRate = Math.Max(1, (int)fmt.nSamplesPerSec);
|
||||
double sourceFramesPerTargetFrame = sourceRate / (double)Math.Max(1, targetRate);
|
||||
long sourceFramesSeen = 0;
|
||||
double nextOutputSourceFrame = 0.0;
|
||||
float[] previousFrame = new float[Math.Max(1, targetChannels)];
|
||||
bool hasPreviousFrame = false;
|
||||
try {
|
||||
while (DateTime.UtcNow < end) {
|
||||
uint packetFrames;
|
||||
|
|
@ -180,24 +186,40 @@ namespace Mimir {
|
|||
int sourceBits = (int)fmt.wBitsPerSample;
|
||||
int sourceBytesPerSample = Math.Max(1, sourceBits / 8);
|
||||
int frameBytes = targetChannels * 4;
|
||||
int needed = checked((int)frames * frameBytes);
|
||||
long packetStart = sourceFramesSeen;
|
||||
long packetEnd = packetStart + frames;
|
||||
int outputFrames = 0;
|
||||
double countCursor = nextOutputSourceFrame;
|
||||
while (countCursor < packetEnd) {
|
||||
outputFrames++;
|
||||
countCursor += sourceFramesPerTargetFrame;
|
||||
}
|
||||
int needed = checked(outputFrames * frameBytes);
|
||||
if (pcm.Length < needed) pcm = new byte[needed];
|
||||
int dst = 0;
|
||||
for (int frame = 0; frame < frames; frame++) {
|
||||
float[] left = new float[targetChannels];
|
||||
float[] right = new float[targetChannels];
|
||||
while (nextOutputSourceFrame < packetEnd) {
|
||||
long leftAbsolute = (long)Math.Floor(nextOutputSourceFrame);
|
||||
long rightAbsolute = Math.Min(packetEnd - 1, leftAbsolute + 1);
|
||||
double fraction = nextOutputSourceFrame - leftAbsolute;
|
||||
ReadFrame(leftAbsolute, packetStart, data, flags, sourceChannels, sourceBytesPerFrame, sourceBytesPerSample, sourceBits, fmt.wFormatTag, activeFormatPtr, previousFrame, hasPreviousFrame, left);
|
||||
ReadFrame(rightAbsolute, packetStart, data, flags, sourceChannels, sourceBytesPerFrame, sourceBytesPerSample, sourceBits, fmt.wFormatTag, activeFormatPtr, previousFrame, hasPreviousFrame, right);
|
||||
for (int ch = 0; ch < targetChannels; ch++) {
|
||||
float sample = 0f;
|
||||
if ((flags & AUDCLNT_BUFFERFLAGS_SILENT) == 0) {
|
||||
int srcCh = Math.Min(ch, sourceChannels - 1);
|
||||
IntPtr src = IntPtr.Add(data, frame * sourceBytesPerFrame + srcCh * sourceBytesPerSample);
|
||||
sample = ReadSample(src, sourceBits, fmt.wFormatTag, mixFormatPtr);
|
||||
}
|
||||
float sample = (float)(left[ch] + (right[ch] - left[ch]) * fraction);
|
||||
byte[] bytes = BitConverter.GetBytes(sample);
|
||||
Buffer.BlockCopy(bytes, 0, pcm, dst, 4);
|
||||
dst += 4;
|
||||
}
|
||||
nextOutputSourceFrame += sourceFramesPerTargetFrame;
|
||||
}
|
||||
if (frames > 0) {
|
||||
ReadFrame(packetEnd - 1, packetStart, data, flags, sourceChannels, sourceBytesPerFrame, sourceBytesPerSample, sourceBits, fmt.wFormatTag, activeFormatPtr, previousFrame, false, previousFrame);
|
||||
hasPreviousFrame = true;
|
||||
}
|
||||
output.Write(pcm, 0, needed);
|
||||
output.Flush();
|
||||
sourceFramesSeen = packetEnd;
|
||||
}
|
||||
finally {
|
||||
Check(captureClient.ReleaseBuffer(frames), "ReleaseBuffer");
|
||||
|
|
@ -216,7 +238,30 @@ namespace Mimir {
|
|||
}
|
||||
}
|
||||
finally {
|
||||
if (outputPath != "-") output.Dispose();
|
||||
if (!IsStdoutPath(outputPath)) output.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
static bool IsStdoutPath(string outputPath) {
|
||||
return String.Equals(outputPath, "-", StringComparison.Ordinal) ||
|
||||
String.Equals(outputPath, "stdout", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
static void ReadFrame(long absoluteFrame, long packetStart, IntPtr data, uint flags, int sourceChannels, int sourceBytesPerFrame, int sourceBytesPerSample, int sourceBits, ushort tag, IntPtr formatPtr, float[] previousFrame, bool hasPreviousFrame, float[] target) {
|
||||
if (absoluteFrame < packetStart) {
|
||||
for (int ch = 0; ch < target.Length; ch++) target[ch] = hasPreviousFrame ? previousFrame[ch] : 0f;
|
||||
return;
|
||||
}
|
||||
|
||||
int packetFrame = checked((int)(absoluteFrame - packetStart));
|
||||
for (int ch = 0; ch < target.Length; ch++) {
|
||||
float sample = 0f;
|
||||
if ((flags & AUDCLNT_BUFFERFLAGS_SILENT) == 0) {
|
||||
int srcCh = Math.Min(ch, sourceChannels - 1);
|
||||
IntPtr src = IntPtr.Add(data, packetFrame * sourceBytesPerFrame + srcCh * sourceBytesPerSample);
|
||||
sample = ReadSample(src, sourceBits, tag, formatPtr);
|
||||
}
|
||||
target[ch] = sample;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@ import MimirThemeLock from "./quartz/components/MimirThemeLock";
|
|||
|
||||
const MimirGraphShell = Component.GameCultGraphSpaShell({
|
||||
stylesheetHref:
|
||||
"/static/epiphany-graph/assets/viewer.css?v=graph-mimir-corpus-20260524",
|
||||
moduleSrc: "/static/epiphany-graph/assets/viewer.js?v=graph-mimir-corpus-20260524",
|
||||
"/static/norn-graph/assets/viewer.css?v=graph-mimir-corpus-20260524",
|
||||
moduleSrc: "/static/norn-graph/assets/viewer.js?v=graph-mimir-corpus-20260524",
|
||||
config: {
|
||||
title: "Mimir Knowledge Graph",
|
||||
architectureDescription:
|
||||
|
|
|
|||
12
src/Mimir.Broadcast/Mimir.Broadcast.csproj
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<AssemblyName>Mimir.Broadcast</AssemblyName>
|
||||
<RootNamespace>Mimir.Broadcast</RootNamespace>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
</Project>
|
||||
279
src/Mimir.Broadcast/Program.cs
Normal file
|
|
@ -0,0 +1,279 @@
|
|||
using System.Diagnostics;
|
||||
using System.Text;
|
||||
|
||||
namespace Mimir.Broadcast;
|
||||
|
||||
internal static class Program
|
||||
{
|
||||
public static async Task<int> Main(string[] args)
|
||||
{
|
||||
var options = MimirBroadcastOptions.Parse(args);
|
||||
if (options.ShowHelp)
|
||||
{
|
||||
Console.WriteLine(MimirBroadcastOptions.HelpText);
|
||||
return 0;
|
||||
}
|
||||
|
||||
var plan = MimirBroadcastPlan.FromOptions(options);
|
||||
if (options.PrintCommand || options.DryRun)
|
||||
{
|
||||
Console.WriteLine(plan.CommandLine);
|
||||
}
|
||||
|
||||
if (options.Smoke)
|
||||
{
|
||||
return RunSmoke(plan);
|
||||
}
|
||||
|
||||
if (options.DryRun || options.PrintCommand)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
return await RunFfmpegAsync(options, plan).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static int RunSmoke(MimirBroadcastPlan plan)
|
||||
{
|
||||
var command = plan.CommandLine;
|
||||
var ok = command.Contains("h264_nvenc", StringComparison.Ordinal) &&
|
||||
command.Contains("-f flv", StringComparison.Ordinal) &&
|
||||
command.Contains("rtmp://127.0.0.1:11935/live/mimir", StringComparison.Ordinal) &&
|
||||
command.Contains("-g 60", StringComparison.Ordinal) &&
|
||||
command.Contains("-sc_threshold 0", StringComparison.Ordinal);
|
||||
Console.WriteLine($"mimir-broadcast-smoke nvenc={command.Contains("h264_nvenc", StringComparison.Ordinal)} rtmp={command.Contains("rtmp://", StringComparison.Ordinal)} keyframeCadence={command.Contains("-g 60", StringComparison.Ordinal)}");
|
||||
return ok ? 0 : 1;
|
||||
}
|
||||
|
||||
private static async Task<int> RunFfmpegAsync(MimirBroadcastOptions options, MimirBroadcastPlan plan)
|
||||
{
|
||||
var start = new ProcessStartInfo
|
||||
{
|
||||
FileName = options.FfmpegPath,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardError = true,
|
||||
RedirectStandardOutput = true,
|
||||
};
|
||||
foreach (var argument in plan.Arguments)
|
||||
{
|
||||
start.ArgumentList.Add(argument);
|
||||
}
|
||||
|
||||
using var process = Process.Start(start);
|
||||
if (process == null)
|
||||
{
|
||||
Console.Error.WriteLine("Could not start ffmpeg.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
var stdout = PumpAsync(process.StandardOutput, Console.Out);
|
||||
var stderr = PumpAsync(process.StandardError, Console.Error);
|
||||
await process.WaitForExitAsync().ConfigureAwait(false);
|
||||
await Task.WhenAll(stdout, stderr).ConfigureAwait(false);
|
||||
return process.ExitCode;
|
||||
}
|
||||
|
||||
private static async Task PumpAsync(StreamReader reader, TextWriter writer)
|
||||
{
|
||||
while (await reader.ReadLineAsync().ConfigureAwait(false) is { } line)
|
||||
{
|
||||
writer.WriteLine(line);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record MimirBroadcastPlan(IReadOnlyList<string> Arguments)
|
||||
{
|
||||
public string CommandLine => $"ffmpeg {string.Join(' ', Arguments.Select(Quote))}";
|
||||
|
||||
public static MimirBroadcastPlan FromOptions(MimirBroadcastOptions options)
|
||||
{
|
||||
var args = new List<string>();
|
||||
if (options.Realtime)
|
||||
{
|
||||
args.Add("-re");
|
||||
}
|
||||
|
||||
if (options.TestPattern)
|
||||
{
|
||||
args.AddRange(["-f", "lavfi", "-i", $"testsrc2=size={options.Width}x{options.Height}:rate={options.FrameRate}"]);
|
||||
args.AddRange(["-f", "lavfi", "-i", $"sine=frequency=997:sample_rate={options.AudioSampleRate}"]);
|
||||
}
|
||||
else if (!string.IsNullOrWhiteSpace(options.InputPath))
|
||||
{
|
||||
args.AddRange(["-i", options.InputPath]);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new InvalidOperationException("Use --test-pattern or provide --input.");
|
||||
}
|
||||
|
||||
var audioMap = options.TestPattern ? "1:a:0" : "0:a:0?";
|
||||
|
||||
args.AddRange([
|
||||
"-map", "0:v:0",
|
||||
"-map", audioMap,
|
||||
"-c:v", "h264_nvenc",
|
||||
"-preset", options.NvencPreset,
|
||||
"-tune", "ll",
|
||||
"-rc", "cbr",
|
||||
"-b:v", $"{options.VideoBitrateKbps}k",
|
||||
"-maxrate", $"{options.VideoBitrateKbps}k",
|
||||
"-bufsize", $"{options.VideoBitrateKbps * 2}k",
|
||||
"-g", options.GopFrames.ToStringInvariant(),
|
||||
"-keyint_min", options.GopFrames.ToStringInvariant(),
|
||||
"-sc_threshold", "0",
|
||||
"-pix_fmt", "yuv420p",
|
||||
"-c:a", "aac",
|
||||
"-b:a", $"{options.AudioBitrateKbps}k",
|
||||
"-ar", options.AudioSampleRate.ToStringInvariant(),
|
||||
]);
|
||||
|
||||
if (options.DurationSeconds > 0)
|
||||
{
|
||||
args.AddRange(["-t", options.DurationSeconds.ToStringInvariant()]);
|
||||
}
|
||||
|
||||
args.AddRange([
|
||||
"-f", "flv",
|
||||
options.TargetUrl,
|
||||
]);
|
||||
return new MimirBroadcastPlan(args);
|
||||
}
|
||||
|
||||
private static string Quote(string value)
|
||||
{
|
||||
if (value.Length == 0)
|
||||
{
|
||||
return "\"\"";
|
||||
}
|
||||
|
||||
return value.Any(static c => char.IsWhiteSpace(c) || c == '"')
|
||||
? $"\"{value.Replace("\"", "\\\"", StringComparison.Ordinal)}\""
|
||||
: value;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record MimirBroadcastOptions
|
||||
{
|
||||
public string FfmpegPath { get; init; } = "ffmpeg";
|
||||
|
||||
public string TargetUrl { get; init; } = "rtmp://127.0.0.1:11935/live/mimir";
|
||||
|
||||
public string InputPath { get; init; } = "";
|
||||
|
||||
public bool TestPattern { get; init; } = true;
|
||||
|
||||
public bool Realtime { get; init; } = true;
|
||||
|
||||
public bool PrintCommand { get; init; }
|
||||
|
||||
public bool DryRun { get; init; }
|
||||
|
||||
public bool Smoke { get; init; }
|
||||
|
||||
public bool ShowHelp { get; init; }
|
||||
|
||||
public int Width { get; init; } = 1280;
|
||||
|
||||
public int Height { get; init; } = 720;
|
||||
|
||||
public int FrameRate { get; init; } = 30;
|
||||
|
||||
public int GopFrames { get; init; } = 60;
|
||||
|
||||
public int VideoBitrateKbps { get; init; } = 4500;
|
||||
|
||||
public int AudioSampleRate { get; init; } = 48_000;
|
||||
|
||||
public int AudioBitrateKbps { get; init; } = 160;
|
||||
|
||||
public string NvencPreset { get; init; } = "p4";
|
||||
|
||||
public int DurationSeconds { get; init; }
|
||||
|
||||
public static string HelpText => """
|
||||
Mimir.Broadcast
|
||||
|
||||
Push a Starfire-encoded Mimir program stream to the Yggdrasil RTMP/HLS edge.
|
||||
The default command is a synthetic NVENC smoke source:
|
||||
|
||||
dotnet run --project src/Mimir.Broadcast -- --print-command
|
||||
|
||||
Options:
|
||||
--target <url> RTMP target. Default: rtmp://127.0.0.1:11935/live/mimir
|
||||
--ffmpeg <path> FFmpeg executable. Default: ffmpeg
|
||||
--input <path> Input media path instead of the synthetic test pattern.
|
||||
--no-test-pattern Require --input.
|
||||
--width <px> Synthetic width. Default: 1280
|
||||
--height <px> Synthetic height. Default: 720
|
||||
--fps <n> Synthetic frame rate. Default: 30
|
||||
--video-kbps <n> CBR video bitrate. Default: 4500
|
||||
--audio-kbps <n> AAC bitrate. Default: 160
|
||||
--gop <frames> Keyframe cadence. Default: 60
|
||||
--preset <name> NVENC preset. Default: p4
|
||||
--duration-seconds <n> Stop after this many seconds. Default: run until stopped.
|
||||
--print-command Print the generated FFmpeg command.
|
||||
--dry-run Print command and exit.
|
||||
--smoke Validate the generated command shape and exit.
|
||||
""";
|
||||
|
||||
public static MimirBroadcastOptions Parse(IReadOnlyList<string> args)
|
||||
{
|
||||
var options = new MimirBroadcastOptions();
|
||||
for (var index = 0; index < args.Count; index++)
|
||||
{
|
||||
var arg = args[index];
|
||||
options = arg switch
|
||||
{
|
||||
"--help" or "-h" => options with { ShowHelp = true },
|
||||
"--print-command" => options with { PrintCommand = true },
|
||||
"--dry-run" => options with { DryRun = true, PrintCommand = true },
|
||||
"--smoke" => options with { Smoke = true },
|
||||
"--no-test-pattern" => options with { TestPattern = false },
|
||||
"--target" => options with { TargetUrl = RequiredValue(args, ref index, arg) },
|
||||
"--ffmpeg" => options with { FfmpegPath = RequiredValue(args, ref index, arg) },
|
||||
"--input" => options with { InputPath = RequiredValue(args, ref index, arg), TestPattern = false },
|
||||
"--width" => options with { Width = ParsePositiveInt(args, ref index, arg) },
|
||||
"--height" => options with { Height = ParsePositiveInt(args, ref index, arg) },
|
||||
"--fps" => options with { FrameRate = ParsePositiveInt(args, ref index, arg) },
|
||||
"--video-kbps" => options with { VideoBitrateKbps = ParsePositiveInt(args, ref index, arg) },
|
||||
"--audio-kbps" => options with { AudioBitrateKbps = ParsePositiveInt(args, ref index, arg) },
|
||||
"--gop" => options with { GopFrames = ParsePositiveInt(args, ref index, arg) },
|
||||
"--preset" => options with { NvencPreset = RequiredValue(args, ref index, arg) },
|
||||
"--duration-seconds" => options with { DurationSeconds = ParsePositiveInt(args, ref index, arg) },
|
||||
_ => throw new ArgumentException($"Unknown argument `{arg}`."),
|
||||
};
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
private static string RequiredValue(IReadOnlyList<string> args, ref int index, string option)
|
||||
{
|
||||
if (index + 1 >= args.Count || args[index + 1].StartsWith("-", StringComparison.Ordinal))
|
||||
{
|
||||
throw new ArgumentException($"{option} requires a value.");
|
||||
}
|
||||
|
||||
index++;
|
||||
return args[index];
|
||||
}
|
||||
|
||||
private static int ParsePositiveInt(IReadOnlyList<string> args, ref int index, string option)
|
||||
{
|
||||
var text = RequiredValue(args, ref index, option);
|
||||
if (!int.TryParse(text, out var value) || value <= 0)
|
||||
{
|
||||
throw new ArgumentException($"{option} requires a positive integer.");
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
internal static class InvariantFormatting
|
||||
{
|
||||
public static string ToStringInvariant(this int value) =>
|
||||
value.ToString(System.Globalization.CultureInfo.InvariantCulture);
|
||||
}
|
||||