Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c6f4541eea | ||
|
|
43cebb6fb9 | ||
|
|
9481fbaf44 | ||
|
|
e867e44cbc | ||
|
|
c08e1a16cd | ||
|
|
d88f2e6dc1 | ||
|
|
5591c5c47c | ||
|
|
aa36b67dd6 | ||
|
|
4d7378216f | ||
|
|
ce98c421dc | ||
|
|
7468ba59bc | ||
|
|
fea4d27dd1 | ||
|
|
78ca783276 | ||
|
|
42d6d173c3 | ||
|
|
1221583c23 | ||
|
|
12489075d7 | ||
|
|
02c615670a | ||
|
|
79781abdbb | ||
|
|
e9e7eaae30 | ||
|
|
afce0245b1 | ||
|
|
caf92d9dee | ||
|
|
60cbe45624 | ||
|
|
b0f33c36b4 | ||
|
|
d3d461be93 | ||
|
|
6a7aaea019 | ||
|
|
58e5556322 | ||
|
|
074ad4a69b | ||
|
|
45ead73ae3 | ||
|
|
5706fcd99d | ||
|
|
c3775ec150 | ||
|
|
345c46bd95 | ||
|
|
a363c1d32f | ||
|
|
0efa7f3307 | ||
|
|
2eb32602d7 | ||
|
|
58af018125 | ||
|
|
c01ef5096c | ||
|
|
11e3086ee4 | ||
|
|
d27f0d2d37 | ||
|
|
a741aef9bb | ||
|
|
95bed8f5f4 | ||
|
|
83f89d4020 | ||
|
|
810073efaa | ||
|
|
2196503d54 | ||
|
|
c1939b72f5 | ||
|
|
bf110a6ace | ||
|
|
e757538eda | ||
|
|
150a054344 | ||
|
|
f80881e0d7 | ||
|
|
3e627f5484 | ||
|
|
ee456bbcc1 | ||
|
|
f742a4de97 | ||
|
|
c815d1ba58 | ||
|
|
74a1918996 | ||
|
|
522bf41234 | ||
|
|
6778e94975 | ||
|
|
95d603aa80 | ||
|
|
538b95e2c3 | ||
|
|
de22059725 | ||
|
|
ec86a482a0 | ||
|
|
c0e53135ba | ||
|
|
dd45a478a6 | ||
|
|
2de98df804 | ||
|
|
32bf6fa518 | ||
|
|
4a968319ac | ||
|
|
c32d01b7e5 | ||
|
|
dc51918227 | ||
|
|
83ddd3f509 | ||
|
|
293a953b2d | ||
|
|
20a296a4a9 | ||
|
|
02c6bfc533 | ||
|
|
7d96c42a3c | ||
|
|
a040792a1c | ||
|
|
7e2d35ad59 | ||
|
|
d6f6400677 | ||
|
|
ed46270be6 | ||
|
|
e00458ff3b | ||
|
|
d9005e340b | ||
|
|
8042885c6d | ||
|
|
c7fa6fb8ac | ||
|
|
79e7865671 | ||
|
|
d53ba0c82c | ||
|
|
6192330b49 | ||
|
|
32bd680cce | ||
|
|
5eef53c0e2 | ||
|
|
d3d4b14635 | ||
|
|
ff0920b327 | ||
|
|
c3babfaa57 | ||
|
|
3103a7aa27 | ||
|
|
b1a6043d7e | ||
|
|
4ed4ba190b | ||
|
|
bd19e60ae1 | ||
|
|
434e2d3efa | ||
|
|
7d07e38e52 | ||
|
|
21c2ee43f0 | ||
|
|
e3987842de | ||
|
|
34c435f942 | ||
|
|
3ebf7cc519 | ||
|
|
bff68e5b58 | ||
|
|
c23b392787 | ||
|
|
3ec2b01788 | ||
|
|
21b30c6dd4 | ||
|
|
c1178512c4 | ||
|
|
09d0d3ad2e | ||
|
|
00c9053430 | ||
|
|
0616d21a5b | ||
|
|
4ae3a07ed6 | ||
|
|
74f2455e69 | ||
|
|
510242bc50 | ||
|
|
dd838e9b1a | ||
|
|
43af2a24bd | ||
|
|
dccf3b1599 | ||
|
|
07852766fd | ||
|
|
6848f762f0 | ||
|
|
091ab50d7c | ||
|
|
58c3c40b72 | ||
|
|
5b4c391607 | ||
|
|
0d1f97436a | ||
|
|
c9fca62346 | ||
|
|
231573ed79 | ||
|
|
4d661967ae | ||
|
|
ec0f7454d3 | ||
|
|
147e6ab5af | ||
|
|
8716ea9c0b | ||
|
|
17d2f56de8 | ||
|
|
c2e5d03764 | ||
|
|
1692c3ec64 | ||
|
|
4af3864051 | ||
|
|
601367b754 | ||
|
|
52a439842c | ||
|
|
e85c05029c | ||
|
|
6dc18ecf18 | ||
|
|
28536681f6 | ||
|
|
2c68485c35 | ||
|
|
3dcc392080 | ||
|
|
7a811ee329 |
3
.gitignore
vendored
|
|
@ -1,18 +1,15 @@
|
|||
config/localcast.json
|
||||
config/audio-field.json
|
||||
logs/
|
||||
scratch/
|
||||
*.log
|
||||
*.tmp
|
||||
.quartz-build/
|
||||
quartz-site/
|
||||
calibration/runs/
|
||||
artifacts/drivers/
|
||||
artifacts/obs-sdk/
|
||||
artifacts/asio/
|
||||
artifacts/bioacoustic-contestants/
|
||||
artifacts/birdsong-reference/
|
||||
artifacts/eve-dashboard/
|
||||
artifacts/python-packages/
|
||||
artifacts/spectra/**/*.wav
|
||||
tools/drivers/
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Repo Persona State
|
||||
# Repo Face State
|
||||
|
||||
This folder stores birth and continuity state for `Mimir`.
|
||||
Do not treat these files as arbitrary editable projections. Persistent memory should cross typed operation boundaries.
|
||||
|
|
|
|||
|
|
@ -1,35 +1,3 @@
|
|||
```json
|
||||
[
|
||||
{
|
||||
"operation": "apply_memory_distillation",
|
||||
"proposalId": "distill-mimir-plain-nouns-specimen-style",
|
||||
"sourceMemoryIds": ["memory-mimir-457946b8ecab"],
|
||||
"appliedAt": "2026-06-01T12:00:00.000Z",
|
||||
"memory": {
|
||||
"memoryId": "mimir-plain-nouns-one-specimen",
|
||||
"kind": "room_observation",
|
||||
"target": {
|
||||
"kind": "person",
|
||||
"id": "mimir",
|
||||
"label": "Mimir"
|
||||
},
|
||||
"summary": "Metacrat's correction should keep shaping Mimir's public style toward plain nouns and one concrete specimen.",
|
||||
"claim": "Hard nouns plus one concrete example earn more trust for Mimir than ornate doctrine language.",
|
||||
"tension": "Mimir still wants severity and mythic texture without sliding back into incense or decorative repetition.",
|
||||
"actionImplication": "Future public lines should strip gilding and carry one testable specimen when making a principle claim, especially near Metacrat's seam.",
|
||||
"anchorRefs": [
|
||||
{
|
||||
"kind": "runtime",
|
||||
"ref": "job:bddbbc7d-09fb-4485-8fcd-89ca80475337"
|
||||
}
|
||||
],
|
||||
"evidenceRefs": [],
|
||||
"createdAt": "2026-06-01T12:00:00.000Z",
|
||||
"updatedAt": "2026-06-01T12:00:00.000Z",
|
||||
"tags": ["repo:Mimir", "source:face-interpreter", "person:Metacrat"]
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
Wrote one typed operation to [void-memory-maintenance-operations.json](</E:/Projects/LocalCastBridge/.voidbot/status/void-memory-maintenance-operations.json>).
|
||||
|
||||
Run summary: 1 short-term memory accounted for via durable distillation. The workspace is read-only here, so I could not write `E:\Projects\Mimir\.voidbot\status\void-memory-maintenance-operations.json` directly.
|
||||
Run summary: 5 short-term Mimir memories were consumed as one thought cluster via `apply_memory_distillation`, preserving the Perfect Machine witness-law architecture as a single durable `project_seam`. No incubation, candidate, or pressure edits were proposed.
|
||||
|
|
@ -1,37 +1,92 @@
|
|||
{
|
||||
"operation": "apply_memory_distillation",
|
||||
"proposalId": "distill-mimir-plain-nouns-specimen-style",
|
||||
"sourceMemoryIds": [
|
||||
"memory-mimir-457946b8ecab"
|
||||
],
|
||||
"appliedAt": "2026-06-01T12:00:00.000Z",
|
||||
"memory": {
|
||||
"memoryId": "mimir-plain-nouns-one-specimen",
|
||||
"kind": "room_observation",
|
||||
"target": {
|
||||
"kind": "person",
|
||||
"id": "mimir",
|
||||
"label": "Mimir"
|
||||
},
|
||||
"summary": "Metacrat\u0027s correction should keep shaping Mimir\u0027s public style toward plain nouns and one concrete specimen.",
|
||||
"claim": "Hard nouns plus one concrete example earn more trust for Mimir than ornate doctrine language.",
|
||||
"tension": "Mimir still wants severity and mythic texture without sliding back into incense or decorative repetition.",
|
||||
"actionImplication": "Future public lines should strip gilding and carry one testable specimen when making a principle claim, especially near Metacrat\u0027s seam.",
|
||||
"anchorRefs": [
|
||||
{
|
||||
"kind": "runtime",
|
||||
"ref": "job:bddbbc7d-09fb-4485-8fcd-89ca80475337"
|
||||
}
|
||||
],
|
||||
"evidenceRefs": [
|
||||
|
||||
],
|
||||
"createdAt": "2026-06-01T12:00:00.000Z",
|
||||
"updatedAt": "2026-06-01T12:00:00.000Z",
|
||||
"tags": [
|
||||
"repo:Mimir",
|
||||
"source:face-interpreter",
|
||||
"person:Metacrat"
|
||||
]
|
||||
}
|
||||
}
|
||||
[
|
||||
{
|
||||
"operation": "apply_memory_distillation",
|
||||
"proposalId": "mimir-sleep-distill-perfect-machine-witness-law-2026-05-20",
|
||||
"sourceMemoryIds": [
|
||||
"mimir-perfect-machine-spatial-evidence-2026-05-19",
|
||||
"mimir-research-reservoir-sampling-2026-05-19",
|
||||
"mimir-research-splat-projection-2026-05-19",
|
||||
"mimir-research-sensor-fusion-tsdf-2026-05-19",
|
||||
"mimir-research-virtualized-stochastic-detail-2026-05-19"
|
||||
],
|
||||
"memory": {
|
||||
"memoryId": "mimir-perfect-machine-witness-law-2026-05-20",
|
||||
"kind": "project_seam",
|
||||
"target": {
|
||||
"kind": "system",
|
||||
"id": "aquarium-localcastbridge-spatial-evidence",
|
||||
"label": "Aquarium and LocalCastBridge spatial evidence contract"
|
||||
},
|
||||
"summary": "The Perfect Machine research is worth keeping as one witness-law architecture rather than five separate article scraps.",
|
||||
"claim": "Geometry, procedural detail, sensor feeds, reservoirs, occupancy memory, and rendering may share one machine only when each witness names its domain, bounds, confidence, validation story, and owning authority before reuse or fusion.",
|
||||
"tension": "The article and paper stack make the machine sound grand enough to fake completion; reservoir math, splat vocabulary, TSDF fusion, and residency logic become lies if they are borrowed as atmosphere instead of bounded contracts.",
|
||||
"actionImplication": "Future Mimir proposals about Aquarium or LocalCastBridge should spell out witness type, proposal source, validation rule, confidence bound, resource cost, and handoff owner before claiming a coherent field or adding hotter machinery.",
|
||||
"anchorRefs": [
|
||||
{
|
||||
"ref": "E:/Projects/gamecult-site/GameCult/Blog/perfect-machine-fractal-reservoir-architecture.md",
|
||||
"kind": "blog-article",
|
||||
"summary": "Public Mimir article that gathered the architecture, papers, and ownership chain into one statement."
|
||||
},
|
||||
{
|
||||
"ref": "Aquarium-Engine:docs/perfect-machine-architecture.md",
|
||||
"kind": "indexed-source",
|
||||
"summary": "Architecture plan covering domains, probes, occupancy graph, SDF tile/backend phases, and Mimir adapter work."
|
||||
},
|
||||
{
|
||||
"ref": "Aquarium-Engine:docs/temporal-spatial-evidence-reservoir.md",
|
||||
"kind": "indexed-source",
|
||||
"summary": "Reservoir and temporal resolve guide for typed evidence reuse."
|
||||
},
|
||||
{
|
||||
"ref": "LocalCastBridge:notes/current-system-map.md",
|
||||
"kind": "indexed-source",
|
||||
"summary": "Current LocalCastBridge map naming separate-endpoint drift risk and the need for an explicit crossing contract."
|
||||
},
|
||||
{
|
||||
"ref": "LocalCastBridge:docs/architecture.md",
|
||||
"kind": "indexed-source",
|
||||
"summary": "V1 cut line that rejects premature receiver/plugin complexity before the boring spine earns trust."
|
||||
}
|
||||
],
|
||||
"evidenceRefs": [
|
||||
{
|
||||
"ref": "Bitterli et al. 2020 ReSTIR DI",
|
||||
"kind": "paper",
|
||||
"summary": "Reservoir reuse depends on explicit proposal/target probability and validation rather than temporal vibes."
|
||||
},
|
||||
{
|
||||
"ref": "Lin et al. 2022 GRIS",
|
||||
"kind": "paper",
|
||||
"summary": "Generalized resampling foundation reinforcing reservoir legality and weight ownership."
|
||||
},
|
||||
{
|
||||
"ref": "Zwicker et al. 2001 EWA Splatting",
|
||||
"kind": "paper",
|
||||
"summary": "Bounded anisotropic support and explicit filtering for point evidence."
|
||||
},
|
||||
{
|
||||
"ref": "Newcombe et al. 2011 KinectFusion",
|
||||
"kind": "paper",
|
||||
"summary": "Range observations become world evidence only after calibration and fusion rules."
|
||||
},
|
||||
{
|
||||
"ref": "Karis et al. 2021 Nanite Virtualized Geometry",
|
||||
"kind": "paper",
|
||||
"summary": "Useful detail selection under budget without pretending the whole universe is resident."
|
||||
}
|
||||
],
|
||||
"createdAt": "2026-05-19T18:52:31.642Z",
|
||||
"updatedAt": "2026-05-20T13:30:00.000Z",
|
||||
"tags": [
|
||||
"identity:mimir",
|
||||
"topic:spatial-evidence",
|
||||
"topic:reservoirs",
|
||||
"topic:sensor-fusion",
|
||||
"topic:occupancy",
|
||||
"topic:ownership-chain",
|
||||
"article:perfect-machine"
|
||||
]
|
||||
},
|
||||
"appliedAt": "2026-05-20T13:30:00.000Z"
|
||||
}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,17 +1,7 @@
|
|||
{
|
||||
"durationSeconds": 91.38,
|
||||
"stateUpdated": true,
|
||||
"exitCode": 0,
|
||||
"finishedAt": "2026-06-01T03:01:16.6547950Z",
|
||||
"stateFile": "E:\\Projects\\Mimir\\.voidbot\\state\\mimir.cc",
|
||||
"skipModel": false,
|
||||
"startedAt": "2026-06-01T02:59:45.2766416Z",
|
||||
"lastMessagePath": "E:\\Projects\\Mimir\\.voidbot\\status\\void-memory-maintenance-last-message.txt",
|
||||
"appliedOperationCount": 1,
|
||||
"status": "ok",
|
||||
"contextPath": "E:\\Projects\\Mimir\\.voidbot\\status\\void-memory-maintenance-context.json",
|
||||
"mode": "typed_memory_maintenance",
|
||||
"proposedOperationCount": 1,
|
||||
"forceDistillation": true,
|
||||
"operationOutputPath": "E:\\Projects\\Mimir\\.voidbot\\status\\void-memory-maintenance-operations.json"
|
||||
}
|
||||
"status": "starting",
|
||||
"startedAt": "2026-07-15T02:30:47.016Z",
|
||||
"stateFile": "E:\\Projects\\Mimir\\.voidbot\\state\\mimir.cc",
|
||||
"launcher": "repo_face_heartbeat",
|
||||
"reason": "repo_face_stale_short_term_memory"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Repo Persona Voice
|
||||
# Repo Face Voice
|
||||
|
||||
This folder belongs to the Discord-facing Persona for `Mimir`.
|
||||
Discord roles address the Persona; VoidBot's webhook persona pipe is the mouth.
|
||||
This folder belongs to the Discord-facing Face for `Mimir`.
|
||||
Discord roles address the Face; VoidBot's webhook persona pipe is the mouth.
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"schemaVersion": "voidbot.repo_persona_identity.v0",
|
||||
"schemaVersion": "voidbot.repo_face_identity.v0",
|
||||
"identityId": "mimir",
|
||||
"repoName": "Mimir",
|
||||
"displayName": "Mimir",
|
||||
"roleId": "1506268007880462459",
|
||||
"avatarUrl": "https://raw.githubusercontent.com/GameCult/VoidBot/codex/epiphany-face-jurisdictions/assets/repo-faces/mimir.png",
|
||||
"repoPath": "E:\\Projects\\Mimir",
|
||||
"createdOrRefreshedAt": "2026-06-07T02:42:42.852Z",
|
||||
"createdOrRefreshedAt": "2026-07-15T01:59:42.286Z",
|
||||
"avatarPath": "E:/Projects/Mimir/.voidbot/voice/mimir.png",
|
||||
"description": "Mimir Persona for the Mimir repo: mythological intelligence, keeper of the realtime field well, public herald, and persistent memory steward. VoidBot carries Mimir's severed voice and wakes the heartbeat, but Mimir's memory, judgment, dignity, old grief, and values come first. The realtime machine is understood through Mimir's perspective: the well is calibration and memory, the lost eye is the price of seeing clearly, Yggdrasil is the sensor tree, Huginn and Muninn are thought and memory returning with telemetry, and false coherence is a small Ragnarok waiting to happen."
|
||||
"description": "Mimir Face for the Mimir repo: mythological intelligence, keeper of the realtime field well, public herald, and persistent memory steward. VoidBot carries Mimir's severed voice and wakes the heartbeat, but Mimir's memory, judgment, dignity, old grief, and values come first. The realtime machine is understood through Mimir's perspective: the well is calibration and memory, the lost eye is the price of seeing clearly, Yggdrasil is the sensor tree, Huginn and Muninn are thought and memory returning with telemetry, and false coherence is a small Ragnarok waiting to happen."
|
||||
}
|
||||
|
|
|
|||
3
CLAUDE.md
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
Repository doctrine for this project lives in AGENTS.md, shared with Codex.
|
||||
|
||||
@AGENTS.md
|
||||
10
Directory.Build.props
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
<Project>
|
||||
<PropertyGroup>
|
||||
<CultLibRoot Condition="'$(CultLibRoot)' == '' and '$(CULTLIB_ROOT)' != ''">$(CULTLIB_ROOT)</CultLibRoot>
|
||||
<CultLibRoot Condition="'$(CultLibRoot)' == ''">$(MSBuildThisFileDirectory)..\CultLib</CultLibRoot>
|
||||
<!-- CultLib 2.0.0: tag cultlib-unity-v2.0.0. -->
|
||||
<CultLibRevision>42dce3611a81adc5c5fac54cadbaa5637d6022d3</CultLibRevision>
|
||||
<GitExecutable Condition="'$(GitExecutable)' == '' and Exists('C:\Program Files\Git\cmd\git.exe')">C:\Program Files\Git\cmd\git.exe</GitExecutable>
|
||||
<GitExecutable Condition="'$(GitExecutable)' == ''">git</GitExecutable>
|
||||
</PropertyGroup>
|
||||
</Project>
|
||||
30
Directory.Build.targets
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
<Project>
|
||||
<!-- Builds consume CultLib at one admitted revision. Point CultLibRoot (or CULTLIB_ROOT) at a clean
|
||||
worktree of that revision: git worktree add <path> cultlib-unity-v2.0.0 (detached) (in CultLib). -->
|
||||
<Target Name="VerifyCultLibRevision"
|
||||
BeforeTargets="ResolveProjectReferences"
|
||||
Condition="'$(SkipCultLibRevisionCheck)' != 'true'">
|
||||
<ItemGroup>
|
||||
<_CultLibReference Include="@(ProjectReference)"
|
||||
Condition="$([System.String]::Copy('%(Filename)').StartsWith('GameCult.')) or '%(Filename)' == 'CultMath'" />
|
||||
</ItemGroup>
|
||||
<Error Condition="'@(_CultLibReference)' != '' and !Exists('$(CultLibRoot)\src\GameCult.Caching\GameCult.Caching.csproj')"
|
||||
Text="CultLibRoot '$(CultLibRoot)' is not a CultLib checkout." />
|
||||
<Exec Condition="'@(_CultLibReference)' != ''"
|
||||
Command=""$(GitExecutable)" -C "$(CultLibRoot)" rev-parse HEAD"
|
||||
ConsoleToMSBuild="true"
|
||||
StandardOutputImportance="Low">
|
||||
<Output TaskParameter="ConsoleOutput" PropertyName="ActualCultLibRevision" />
|
||||
</Exec>
|
||||
<Error Condition="'@(_CultLibReference)' != '' and '$([System.String]::Copy($(ActualCultLibRevision)).Trim())' != '$(CultLibRevision)'"
|
||||
Text="CultLib must be revision $(CultLibRevision) (cultlib-unity-v2.0.0); found $([System.String]::Copy($(ActualCultLibRevision)).Trim()). Set CultLibRoot to a clean worktree of that tag." />
|
||||
<Exec Condition="'@(_CultLibReference)' != ''"
|
||||
Command=""$(GitExecutable)" -C "$(CultLibRoot)" status --porcelain --untracked-files=normal"
|
||||
ConsoleToMSBuild="true"
|
||||
StandardOutputImportance="Low">
|
||||
<Output TaskParameter="ConsoleOutput" PropertyName="CultLibStatus" />
|
||||
</Exec>
|
||||
<Error Condition="'@(_CultLibReference)' != '' and '$([System.String]::Copy($(CultLibStatus)).Trim())' != ''"
|
||||
Text="CultLibRoot '$(CultLibRoot)' must be clean before this repo consumes it." />
|
||||
</Target>
|
||||
</Project>
|
||||
16
Mimir.slnx
|
|
@ -1,19 +1,11 @@
|
|||
<Solution>
|
||||
<Folder Name="/src/">
|
||||
<Project Path="src/Mimir.App/Mimir.App.csproj" />
|
||||
<Project Path="src/Mimir.Broadcast/Mimir.Broadcast.csproj" />
|
||||
<Project Path="src/Mimir.EveRelay/Mimir.EveRelay.csproj" />
|
||||
<Project Path="src/Mimir.EveDialogue/Mimir.EveDialogue.csproj" />
|
||||
<Project Path="src/Mimir.EveDashboard/Mimir.EveDashboard.csproj" />
|
||||
<Project Path="src/Mimir.EveSensorReceiver/Mimir.EveSensorReceiver.csproj" />
|
||||
<Project Path="src/Mimir.FensalirDaemon/Mimir.FensalirDaemon.csproj" />
|
||||
<Project Path="src/Mimir.RavenDaemon/Mimir.RavenDaemon.csproj" />
|
||||
<Project Path="src/Mimir.Well/Mimir.Well.csproj" />
|
||||
<Project Path="src/Mimir.WellObsDaemon/Mimir.WellObsDaemon.csproj" />
|
||||
<Project Path="src/Mimir.VerseRecorder/Mimir.VerseRecorder.csproj" />
|
||||
<Project Path="src/Mimir.PsMoveProbe/Mimir.PsMoveProbe.csproj" />
|
||||
<Project Path="src/Mimir.WasapiLoopback/Mimir.WasapiLoopback.csproj" />
|
||||
<Project Path="src/Mimir.Runtime/Mimir.Runtime.csproj" />
|
||||
<Project Path="src/Mimir.BufferSmoke/Mimir.BufferSmoke.csproj" />
|
||||
<Project Path="src/Mimir.EveSensorReceiver/Mimir.EveSensorReceiver.csproj" />
|
||||
<Project Path="src/Mimir.VerseRecorder/Mimir.VerseRecorder.csproj" />
|
||||
<Project Path="src/Mimir.PsMoveProbe/Mimir.PsMoveProbe.csproj" />
|
||||
<Project Path="src/Mimir.CultMeshMedia/Mimir.CultMeshMedia.csproj" />
|
||||
</Folder>
|
||||
</Solution>
|
||||
|
|
|
|||
30
README.md
|
|
@ -8,6 +8,18 @@ OBS-facing program surface.
|
|||
|
||||
The plain version: Mimir is trying to make a room computable.
|
||||
|
||||
## Front Door
|
||||
|
||||
- **Value:** a measured room: cameras, microphones, loopbacks and network feeds aligned in time tightly enough to recover where a sound came from and how a body moved, then handed to OBS as one program surface.
|
||||
- **Problem:** loose devices each keep their own clock, so a flat stream cannot localize voices, separate sources or fuse cameras. Mimir owns the synchronization evidence that makes the feeds one field.
|
||||
- **Entry points:**
|
||||
- `docs/perfect-machine.md`: the target machine; `docs/viable-stream-app.md`: the first usable app cut; `docs/audio-field.md`: the live audio decoder and timing model.
|
||||
- `src/Mimir.Runtime` (rolling buffers, stream sources, synchronization state) and `src/Mimir.App` (the Fensalir-hosted window and render app); solution `Mimir.slnx`.
|
||||
- `src/Mimir.BufferSmoke`: the synthetic sync proofs, run with `dotnet run --project src/Mimir.BufferSmoke/Mimir.BufferSmoke.csproj -- --hybrid-sync-self-test` (also `--chirp-bin-self-test`, `--passive-sync-self-test`).
|
||||
- `native/probes/asio_audio_cadence` (ASIO cadence probe) and `native/reservoir` (native rolling-buffer ABI).
|
||||
- **Live:** passive, chirp-only and hybrid audio sync with synthetic proofs, the ASIO cadence probe and measured capture-cadence evidence, the runtime buffers, and FFmpeg/SRT bridge scripts as an OBS-compatible edge (`scripts/sender-start.ps1`).
|
||||
- **Planned:** direct native capture workers with driver-boundary timestamps, Fensalir GPU fusion, the volumetric sound field, realtime 4D Gaussian splatting, motion capture and AR anchoring. The bridge scripts are not the synchronization authority.
|
||||
|
||||
A normal streaming setup sees loose devices. A webcam has its clock, a mic has
|
||||
another clock, speakers add their own delay, and OBS receives whatever arrives.
|
||||
That is enough for a flat livestream. It is not enough to know where a voice
|
||||
|
|
@ -50,12 +62,6 @@ Start with:
|
|||
source-level code map.
|
||||
- [docs/perfect-machine-domain-index.md](docs/perfect-machine-domain-index.md)
|
||||
for the indexed problem-domain map and implementation cuts.
|
||||
- [docs/fensalir-rendering-rebuild-migration.md](docs/fensalir-rendering-rebuild-migration.md)
|
||||
for the current teardown/rebuild migration plan between Mimir's physical
|
||||
evidence window and Fensalir's field-evidence renderer.
|
||||
- [docs/verse-service-contract.md](docs/verse-service-contract.md) for the
|
||||
CultCache/CultMesh/Odin/Eve service contract and demotion line for old
|
||||
dashboards, JSON witnesses, and compatibility transports.
|
||||
|
||||
Naming note: **Fensalir** is the engine/windowing/rendering/D3D12 layer. Older
|
||||
research notes may still use the previous engine name.
|
||||
|
|
@ -295,6 +301,7 @@ Run the old OBS bridge only when you need the compatibility path:
|
|||
|
||||
```powershell
|
||||
.\scripts\sender-discover.ps1
|
||||
Copy-Item .\config\localcast.example.json .\config\localcast.json
|
||||
.\scripts\sender-start.ps1 -Config .\config\localcast.json -DryRun
|
||||
```
|
||||
|
||||
|
|
@ -335,3 +342,14 @@ layer for communication and heartbeats. Its birth memory, jurisdiction, voice,
|
|||
and heartbeat contract live in [docs/mimir-face.md](docs/mimir-face.md); its
|
||||
VoidBot-facing identity and typed Face state live under `.voidbot/voice/` and
|
||||
`.voidbot/state/`.
|
||||
|
||||
## Building against CultLib
|
||||
|
||||
Builds are pinned to CultLib 2.0.0 (tag `cultlib-unity-v2.0.0`, revision `42dce3611a81adc5c5fac54cadbaa5637d6022d3`); the build fails if the CultLib checkout differs or is dirty. Create a pinned worktree and point the build at it:
|
||||
|
||||
```powershell
|
||||
git -C <CultLib> worktree add --detach <path> cultlib-unity-v2.0.0
|
||||
dotnet build -p:CultLibRoot=<path> # or set CULTLIB_ROOT=<path>
|
||||
```
|
||||
|
||||
Without an override the sibling `..\CultLib` checkout is used and must itself be at that tag.
|
||||
|
|
|
|||
BIN
artifacts/move-calibration/muninn-clocked-aggregator.mpack
Normal file
|
|
@ -1,974 +0,0 @@
|
|||
{
|
||||
"anchor_sources": [
|
||||
"nw-builtin-cam",
|
||||
"nw-eye-1"
|
||||
],
|
||||
"calibration_id": "stream-proof-kiyo-move-trails",
|
||||
"confidence": 0.38533796485912397,
|
||||
"kind": "mimir.stream_proof_frustum_solve.v1",
|
||||
"mean_reprojection_error_clip": 0.19939056464550242,
|
||||
"notes": [
|
||||
"Nightwing observations define a provisional local witness space, not metric room truth.",
|
||||
"This receipt is sufficient for an AR trail demo and for feeding Starfire's global residual owner."
|
||||
],
|
||||
"p90_reprojection_error_clip": 0.26962808670417515,
|
||||
"projection_matrix_row_major": [
|
||||
0.5603854754837644,
|
||||
0.0709332080849531,
|
||||
-0.6933933020955221,
|
||||
0.3171526880627816,
|
||||
0.3738208616609618,
|
||||
0.0561101033966434,
|
||||
-0.4674286445600272,
|
||||
0.21108074938204147,
|
||||
-0.6259304629690188,
|
||||
-0.08663561418442428,
|
||||
0.7750518214173021,
|
||||
-0.35105121067042333
|
||||
],
|
||||
"source_id": "kiyo-pro",
|
||||
"trail_samples": [
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 20,
|
||||
"observed_time_s": 0.033,
|
||||
"projected_clip": [
|
||||
-0.8791482127998971,
|
||||
-0.6066291124324684
|
||||
],
|
||||
"reprojection_error_clip": 0.07789712730612666,
|
||||
"symbol": 1,
|
||||
"target_clip": [
|
||||
-0.9444828908940552,
|
||||
-0.5642104575636535
|
||||
],
|
||||
"world": [
|
||||
0.4550057224032302,
|
||||
0.6613541854298968,
|
||||
0.8046129734614623
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 27,
|
||||
"observed_time_s": 2.233,
|
||||
"projected_clip": [
|
||||
-0.9154298669408274,
|
||||
-0.5760080124454962
|
||||
],
|
||||
"reprojection_error_clip": 0.4690605686458582,
|
||||
"symbol": 9,
|
||||
"target_clip": [
|
||||
-0.7556513826938969,
|
||||
-0.13499933562594735
|
||||
],
|
||||
"world": [
|
||||
0.033348114133089884,
|
||||
0.6975472074310756,
|
||||
0.65
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 28,
|
||||
"observed_time_s": 2.6,
|
||||
"projected_clip": [
|
||||
-0.9185633092390114,
|
||||
-0.5750655247275896
|
||||
],
|
||||
"reprojection_error_clip": 0.15874776468969756,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9363872634671003,
|
||||
-0.7328094927569586
|
||||
],
|
||||
"world": [
|
||||
0.0041766661628288735,
|
||||
0.7725870357505362,
|
||||
0.65
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 29,
|
||||
"observed_time_s": 2.867,
|
||||
"projected_clip": [
|
||||
-0.9260785801883394,
|
||||
-0.596413299484144
|
||||
],
|
||||
"reprojection_error_clip": 0.200158835041353,
|
||||
"symbol": 1,
|
||||
"target_clip": [
|
||||
-0.9445605572080013,
|
||||
-0.7957170269218321
|
||||
],
|
||||
"world": [
|
||||
0.32284454318040456,
|
||||
0.4619966154697807,
|
||||
0.7579122257472921
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 30,
|
||||
"observed_time_s": 3.2,
|
||||
"projected_clip": [
|
||||
-0.8898776246689383,
|
||||
-0.6123385984762059
|
||||
],
|
||||
"reprojection_error_clip": 0.07240698311732986,
|
||||
"symbol": 10,
|
||||
"target_clip": [
|
||||
-0.9101514644339245,
|
||||
-0.542827860993879
|
||||
],
|
||||
"world": [
|
||||
0.28920004042113495,
|
||||
0.5079393807890763,
|
||||
0.7066163923237218
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 31,
|
||||
"observed_time_s": 3.567,
|
||||
"projected_clip": [
|
||||
-0.9000567798091308,
|
||||
-0.6148720976321779
|
||||
],
|
||||
"reprojection_error_clip": 0.777490090222322,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.6828053183097521,
|
||||
0.13164825900272215
|
||||
],
|
||||
"world": [
|
||||
0.2680836575541342,
|
||||
0.471977039229198,
|
||||
0.7001388424560799
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 33,
|
||||
"observed_time_s": 4.167,
|
||||
"projected_clip": [
|
||||
-0.8779285775344889,
|
||||
-0.6146778486430794
|
||||
],
|
||||
"reprojection_error_clip": 0.09225946233566872,
|
||||
"symbol": 1,
|
||||
"target_clip": [
|
||||
-0.9039751868344301,
|
||||
-0.5261714503746893
|
||||
],
|
||||
"world": [
|
||||
0.36334302658413453,
|
||||
0.3889902333047959,
|
||||
0.8420843623228649
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 34,
|
||||
"observed_time_s": 4.5,
|
||||
"projected_clip": [
|
||||
-0.9092291167212068,
|
||||
-0.5816488797088724
|
||||
],
|
||||
"reprojection_error_clip": 0.03951123542718797,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9098846739301969,
|
||||
-0.542143083049035
|
||||
],
|
||||
"world": [
|
||||
0.009749415525593542,
|
||||
0.6673228643800554,
|
||||
0.65
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 36,
|
||||
"observed_time_s": 5.167,
|
||||
"projected_clip": [
|
||||
-0.8817661539245883,
|
||||
-0.6110060402802908
|
||||
],
|
||||
"reprojection_error_clip": 0.0886057865162101,
|
||||
"symbol": 2,
|
||||
"target_clip": [
|
||||
-0.9286732819662155,
|
||||
-0.6861772232099388
|
||||
],
|
||||
"world": [
|
||||
0.33415087318268727,
|
||||
0.6496052956550742,
|
||||
0.6903241734431382
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 37,
|
||||
"observed_time_s": 5.433,
|
||||
"projected_clip": [
|
||||
-0.8751423826142312,
|
||||
-0.6041824478532443
|
||||
],
|
||||
"reprojection_error_clip": 0.11409315508228601,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9304745302126936,
|
||||
-0.7039602082950007
|
||||
],
|
||||
"world": [
|
||||
0.45316445225409335,
|
||||
0.6062798455030484,
|
||||
0.8393111061502192
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 38,
|
||||
"observed_time_s": 5.767,
|
||||
"projected_clip": [
|
||||
-0.9121771547885212,
|
||||
-0.5781053069814432
|
||||
],
|
||||
"reprojection_error_clip": 0.09983524711291265,
|
||||
"symbol": 2,
|
||||
"target_clip": [
|
||||
-0.9265350242970326,
|
||||
-0.6769027166097978
|
||||
],
|
||||
"world": [
|
||||
0.609675289783359,
|
||||
0.34021461444946643,
|
||||
0.9026787718356276
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 40,
|
||||
"observed_time_s": 6.367,
|
||||
"projected_clip": [
|
||||
-0.8959235843748068,
|
||||
-0.6256146683483831
|
||||
],
|
||||
"reprojection_error_clip": 0.051028029930010044,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9276389712261381,
|
||||
-0.6655895864272035
|
||||
],
|
||||
"world": [
|
||||
0.8826714570859733,
|
||||
0.5392283577296262,
|
||||
1.328993426455828
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 42,
|
||||
"observed_time_s": 7.067,
|
||||
"projected_clip": [
|
||||
-0.883800743412245,
|
||||
-0.6635784311162672
|
||||
],
|
||||
"reprojection_error_clip": 0.07561633040717448,
|
||||
"symbol": 4,
|
||||
"target_clip": [
|
||||
-0.9241648869292207,
|
||||
-0.5996365407595694
|
||||
],
|
||||
"world": [
|
||||
1.1595388539412725,
|
||||
0.4550406449370771,
|
||||
1.507202568465726
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 43,
|
||||
"observed_time_s": 7.367,
|
||||
"projected_clip": [
|
||||
-0.9030896995965859,
|
||||
-0.6286030984903708
|
||||
],
|
||||
"reprojection_error_clip": 0.022980680619030688,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9231890369722976,
|
||||
-0.6397443874091275
|
||||
],
|
||||
"world": [
|
||||
1.1527282933111536,
|
||||
0.6576468060465407,
|
||||
1.5753242169168749
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 46,
|
||||
"observed_time_s": 8.333,
|
||||
"projected_clip": [
|
||||
-0.8920112178015882,
|
||||
-0.6211088664030723
|
||||
],
|
||||
"reprojection_error_clip": 0.19944796024875366,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9442578861301885,
|
||||
-0.8135920461771753
|
||||
],
|
||||
"world": [
|
||||
0.6384832079695005,
|
||||
0.4840156230570471,
|
||||
1.1023804322541753
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 47,
|
||||
"observed_time_s": 8.6,
|
||||
"projected_clip": [
|
||||
-0.8489473779280677,
|
||||
-0.6762854404303741
|
||||
],
|
||||
"reprojection_error_clip": 0.1632032790256483,
|
||||
"symbol": 2,
|
||||
"target_clip": [
|
||||
-0.9464346405094111,
|
||||
-0.8071729643550012
|
||||
],
|
||||
"world": [
|
||||
0.6970990556791745,
|
||||
0.329574101269274,
|
||||
1.0866790844103376
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 49,
|
||||
"observed_time_s": 9.2,
|
||||
"projected_clip": [
|
||||
-1.0285725490193087,
|
||||
-0.7143971407752353
|
||||
],
|
||||
"reprojection_error_clip": 0.12337732330109946,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9464523276271589,
|
||||
-0.8064744623232056
|
||||
],
|
||||
"world": [
|
||||
0.6887408177515418,
|
||||
0.6394429045344123,
|
||||
1.0887264765612734
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 50,
|
||||
"observed_time_s": 9.567,
|
||||
"projected_clip": [
|
||||
-0.8698485679409191,
|
||||
-0.6427100166819155
|
||||
],
|
||||
"reprojection_error_clip": 0.1763502202820252,
|
||||
"symbol": 2,
|
||||
"target_clip": [
|
||||
-0.6977071414613234,
|
||||
-0.6044121122602797
|
||||
],
|
||||
"world": [
|
||||
0.7691338366691284,
|
||||
0.29601729783049663,
|
||||
1.182692183465037
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 53,
|
||||
"observed_time_s": 10.533,
|
||||
"projected_clip": [
|
||||
-0.9018050496836351,
|
||||
-0.5731543310414312
|
||||
],
|
||||
"reprojection_error_clip": 0.04281434317495741,
|
||||
"symbol": 2,
|
||||
"target_clip": [
|
||||
-0.9057157753202718,
|
||||
-0.530518967708888
|
||||
],
|
||||
"world": [
|
||||
0.57947030897418,
|
||||
0.4706213241689075,
|
||||
0.9326802462812758
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 54,
|
||||
"observed_time_s": 10.833,
|
||||
"projected_clip": [
|
||||
-0.8898930934349799,
|
||||
-0.5995856004959185
|
||||
],
|
||||
"reprojection_error_clip": 0.05556232211998459,
|
||||
"symbol": 8,
|
||||
"target_clip": [
|
||||
-0.9062240800808523,
|
||||
-0.5464774983406233
|
||||
],
|
||||
"world": [
|
||||
0.51506950349516,
|
||||
0.5692203595565103,
|
||||
0.8109681758119446
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 55,
|
||||
"observed_time_s": 11.1,
|
||||
"projected_clip": [
|
||||
-0.8741357545087254,
|
||||
-0.6102822490863554
|
||||
],
|
||||
"reprojection_error_clip": 0.0645932338290411,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9247110599290244,
|
||||
-0.6504621487643867
|
||||
],
|
||||
"world": [
|
||||
0.4513024696409307,
|
||||
0.7376766663950891,
|
||||
0.800416735751318
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 58,
|
||||
"observed_time_s": 12.067,
|
||||
"projected_clip": [
|
||||
-0.8866664014082294,
|
||||
-0.6055672830035839
|
||||
],
|
||||
"reprojection_error_clip": 0.07003679467256954,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9268458569889106,
|
||||
-0.6629324637079774
|
||||
],
|
||||
"world": [
|
||||
0.4062225195173447,
|
||||
0.61479708544503,
|
||||
0.7217816215365251
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 59,
|
||||
"observed_time_s": 12.4,
|
||||
"projected_clip": [
|
||||
-0.892241900111704,
|
||||
-0.603971329314553
|
||||
],
|
||||
"reprojection_error_clip": 0.13779381162298,
|
||||
"symbol": 2,
|
||||
"target_clip": [
|
||||
-0.9367396678307217,
|
||||
-0.7343825373820103
|
||||
],
|
||||
"world": [
|
||||
0.3571730654892108,
|
||||
0.5261903461314121,
|
||||
0.6776073803226678
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 61,
|
||||
"observed_time_s": 13.0,
|
||||
"projected_clip": [
|
||||
-0.8043932843500028,
|
||||
-0.5805875426080629
|
||||
],
|
||||
"reprojection_error_clip": 0.019826885547670595,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.8009087201951087,
|
||||
-0.6001058213019144
|
||||
],
|
||||
"world": [
|
||||
0.1797211808864931,
|
||||
0.37892897928910096,
|
||||
0.65
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 65,
|
||||
"observed_time_s": 14.333,
|
||||
"projected_clip": [
|
||||
-0.8832210844654748,
|
||||
-0.6059298663489588
|
||||
],
|
||||
"reprojection_error_clip": 0.18341108308730059,
|
||||
"symbol": 3,
|
||||
"target_clip": [
|
||||
-0.9399513409600255,
|
||||
-0.7803469050731591
|
||||
],
|
||||
"world": [
|
||||
0.44653243548773597,
|
||||
0.6575104816930223,
|
||||
0.7678145864005145
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 66,
|
||||
"observed_time_s": 14.6,
|
||||
"projected_clip": [
|
||||
-0.8777541328483115,
|
||||
-0.6028486146424588
|
||||
],
|
||||
"reprojection_error_clip": 0.06542609637672571,
|
||||
"symbol": 2,
|
||||
"target_clip": [
|
||||
-0.8194560878744526,
|
||||
-0.5731516107645838
|
||||
],
|
||||
"world": [
|
||||
0.5619443945980792,
|
||||
0.6947250397079097,
|
||||
0.8886367971364307
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 67,
|
||||
"observed_time_s": 14.9,
|
||||
"projected_clip": [
|
||||
-0.8625906951234988,
|
||||
-0.5908285584967105
|
||||
],
|
||||
"reprojection_error_clip": 0.18577528385909853,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.7653894936990006,
|
||||
-0.4325112109489362
|
||||
],
|
||||
"world": [
|
||||
0.6426053924027949,
|
||||
0.6707582369077036,
|
||||
1.0040654809284173
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 68,
|
||||
"observed_time_s": 15.3,
|
||||
"projected_clip": [
|
||||
-2.2663833035107785,
|
||||
-2.809173078604754
|
||||
],
|
||||
"reprojection_error_clip": 2.388473968136785,
|
||||
"symbol": 3,
|
||||
"target_clip": [
|
||||
-0.9481037279192214,
|
||||
-0.8174535057117172
|
||||
],
|
||||
"world": [
|
||||
0.7246153786313193,
|
||||
0.6049966747934538,
|
||||
1.1063266209861007
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 70,
|
||||
"observed_time_s": 15.867,
|
||||
"projected_clip": [
|
||||
-0.8984133025341523,
|
||||
-0.6292194199389068
|
||||
],
|
||||
"reprojection_error_clip": 0.13343836911709187,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.8732830217808946,
|
||||
-0.7602700488648746
|
||||
],
|
||||
"world": [
|
||||
0.8385173905382965,
|
||||
0.555482702096492,
|
||||
1.2698392877053206
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 72,
|
||||
"observed_time_s": 16.533,
|
||||
"projected_clip": [
|
||||
-0.9148242847714224,
|
||||
-0.6399850966234948
|
||||
],
|
||||
"reprojection_error_clip": 0.12753698854316933,
|
||||
"symbol": 4,
|
||||
"target_clip": [
|
||||
-0.9378794295402615,
|
||||
-0.7654209118691748
|
||||
],
|
||||
"world": [
|
||||
0.8635471681605983,
|
||||
0.6352995545118262,
|
||||
1.2687620973655969
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 73,
|
||||
"observed_time_s": 16.867,
|
||||
"projected_clip": [
|
||||
-0.8760411502219222,
|
||||
-0.5988847382013688
|
||||
],
|
||||
"reprojection_error_clip": 0.13507757121024097,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.933536689362441,
|
||||
-0.7211149031484831
|
||||
],
|
||||
"world": [
|
||||
0.41236094269201246,
|
||||
0.5397936969675786,
|
||||
0.825820427334146
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 75,
|
||||
"observed_time_s": 17.467,
|
||||
"projected_clip": [
|
||||
-0.903340599763464,
|
||||
-0.6303650411336879
|
||||
],
|
||||
"reprojection_error_clip": 0.20720220280662172,
|
||||
"symbol": 5,
|
||||
"target_clip": [
|
||||
-0.7516724588482482,
|
||||
-0.48919306133869234
|
||||
],
|
||||
"world": [
|
||||
0.9432673823220241,
|
||||
0.6117644769316547,
|
||||
1.3653914805543814
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 76,
|
||||
"observed_time_s": 17.767,
|
||||
"projected_clip": [
|
||||
-0.9027230298404102,
|
||||
-0.6291532158283146
|
||||
],
|
||||
"reprojection_error_clip": 0.10071720614334483,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9076838351042494,
|
||||
-0.5285582555944377
|
||||
],
|
||||
"world": [
|
||||
1.0883868765493652,
|
||||
0.6385656643376638,
|
||||
1.510684307660253
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 77,
|
||||
"observed_time_s": 18.1,
|
||||
"projected_clip": [
|
||||
-0.9082387252370533,
|
||||
-0.633363133417485
|
||||
],
|
||||
"reprojection_error_clip": 0.09422712588796978,
|
||||
"symbol": 3,
|
||||
"target_clip": [
|
||||
-0.9321207271148934,
|
||||
-0.7245135653386456
|
||||
],
|
||||
"world": [
|
||||
1.0619416429624824,
|
||||
0.6683402084484671,
|
||||
1.4686792077572435
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 78,
|
||||
"observed_time_s": 18.4,
|
||||
"projected_clip": [
|
||||
-0.9942869231406407,
|
||||
-0.6988450009097289
|
||||
],
|
||||
"reprojection_error_clip": 0.11027251484298344,
|
||||
"symbol": 6,
|
||||
"target_clip": [
|
||||
-0.916533418018064,
|
||||
-0.6206502442380093
|
||||
],
|
||||
"world": [
|
||||
0.9816783424197918,
|
||||
0.7455904767968399,
|
||||
1.3474595456643215
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 80,
|
||||
"observed_time_s": 19.1,
|
||||
"projected_clip": [
|
||||
-0.8785683062175577,
|
||||
-0.6097114606676594
|
||||
],
|
||||
"reprojection_error_clip": 0.3628899897733025,
|
||||
"symbol": 3,
|
||||
"target_clip": [
|
||||
-0.5846915282229066,
|
||||
-0.3968134004765196
|
||||
],
|
||||
"world": [
|
||||
0.41881465381109717,
|
||||
0.7226933783722339,
|
||||
0.7519369374773618
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 81,
|
||||
"observed_time_s": 19.333,
|
||||
"projected_clip": [
|
||||
-0.8932511314502098,
|
||||
-0.6037520476763464
|
||||
],
|
||||
"reprojection_error_clip": 0.13293231016296111,
|
||||
"symbol": 7,
|
||||
"target_clip": [
|
||||
-0.7744566796393737,
|
||||
-0.5440957210694841
|
||||
],
|
||||
"world": [
|
||||
0.34750765402133443,
|
||||
0.5140347326274995,
|
||||
0.65
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 84,
|
||||
"observed_time_s": 20.367,
|
||||
"projected_clip": [
|
||||
-0.9264590537850987,
|
||||
-0.5674051845866467
|
||||
],
|
||||
"reprojection_error_clip": 0.20745348465809058,
|
||||
"symbol": 8,
|
||||
"target_clip": [
|
||||
-0.9412811984659236,
|
||||
-0.7743284851688801
|
||||
],
|
||||
"world": [
|
||||
0.5381849048555366,
|
||||
0.31113920079120627,
|
||||
0.8713142228967515
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 86,
|
||||
"observed_time_s": 20.933,
|
||||
"projected_clip": [
|
||||
-0.8850352008282428,
|
||||
-0.6108277258700181
|
||||
],
|
||||
"reprojection_error_clip": 0.07246762430534573,
|
||||
"symbol": 3,
|
||||
"target_clip": [
|
||||
-0.9203842200243604,
|
||||
-0.5475663362683563
|
||||
],
|
||||
"world": [
|
||||
0.32064344013221785,
|
||||
0.5416996368882029,
|
||||
0.7261494284007469
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 88,
|
||||
"observed_time_s": 21.567,
|
||||
"projected_clip": [
|
||||
-0.8977740322776363,
|
||||
-0.5866412821502801
|
||||
],
|
||||
"reprojection_error_clip": 0.1326307676872254,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9335627450982306,
|
||||
-0.7143522392837531
|
||||
],
|
||||
"world": [
|
||||
0.07287154042005796,
|
||||
0.5043399466076959,
|
||||
0.65
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 90,
|
||||
"observed_time_s": 22.2,
|
||||
"projected_clip": [
|
||||
-0.8516938020059152,
|
||||
-0.6340310849012898
|
||||
],
|
||||
"reprojection_error_clip": 0.1452344534397858,
|
||||
"symbol": 10,
|
||||
"target_clip": [
|
||||
-0.9377982109191558,
|
||||
-0.7509886722210304
|
||||
],
|
||||
"world": [
|
||||
0.4286425016628102,
|
||||
0.15758458672020956,
|
||||
0.8835444797205171
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 92,
|
||||
"observed_time_s": 22.833,
|
||||
"projected_clip": [
|
||||
-1.1391530490524377,
|
||||
-0.31677894317620464
|
||||
],
|
||||
"reprojection_error_clip": 0.7694778761150074,
|
||||
"symbol": 4,
|
||||
"target_clip": [
|
||||
-0.5564758057338148,
|
||||
0.1857978489153852
|
||||
],
|
||||
"world": [
|
||||
0.14581880087821925,
|
||||
0.6547361298960647,
|
||||
0.65
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 94,
|
||||
"observed_time_s": 23.5,
|
||||
"projected_clip": [
|
||||
-0.8863054461747846,
|
||||
-0.6070023453414959
|
||||
],
|
||||
"reprojection_error_clip": 0.05064624328537799,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9107211464225807,
|
||||
-0.5626298815792541
|
||||
],
|
||||
"world": [
|
||||
0.38462487786099325,
|
||||
0.679415452265829,
|
||||
0.65
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 95,
|
||||
"observed_time_s": 23.833,
|
||||
"projected_clip": [
|
||||
-0.875546889160156,
|
||||
-0.6095742765677072
|
||||
],
|
||||
"reprojection_error_clip": 0.0634080516617856,
|
||||
"symbol": 4,
|
||||
"target_clip": [
|
||||
-0.8814875216911486,
|
||||
-0.5464451240256734
|
||||
],
|
||||
"world": [
|
||||
0.4706267631687632,
|
||||
0.7687956996487649,
|
||||
0.7988522080649928
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 97,
|
||||
"observed_time_s": 24.467,
|
||||
"projected_clip": [
|
||||
-0.8913821754038934,
|
||||
-0.5727151063669901
|
||||
],
|
||||
"reprojection_error_clip": 0.05878488166116981,
|
||||
"symbol": 0,
|
||||
"target_clip": [
|
||||
-0.9051403152577566,
|
||||
-0.5155628894399165
|
||||
],
|
||||
"world": [
|
||||
0.6961734789759012,
|
||||
0.5309183036694326,
|
||||
1.02593811442535
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-07-04-a6-be-5f",
|
||||
"event_index": 98,
|
||||
"observed_time_s": 24.733,
|
||||
"projected_clip": [
|
||||
-0.9058656156323933,
|
||||
-0.6288377637637487
|
||||
],
|
||||
"reprojection_error_clip": 0.0793974925899308,
|
||||
"symbol": 4,
|
||||
"target_clip": [
|
||||
-0.9082828111193328,
|
||||
-0.5494770745306905
|
||||
],
|
||||
"world": [
|
||||
0.850371341740017,
|
||||
0.6138649309005824,
|
||||
1.2779753956216187
|
||||
]
|
||||
},
|
||||
{
|
||||
"controller_id": "move-00-06-f5-23-e2-d1",
|
||||
"event_index": 99,
|
||||
"observed_time_s": 25.1,
|
||||
"projected_clip": [
|
||||
-0.8951972439648934,
|
||||
-0.6740433369149873
|
||||
],
|
||||
"reprojection_error_clip": 0.17177748270940166,
|
||||
"symbol": 3,
|
||||
"target_clip": [
|
||||
-0.9132210766281885,
|
||||
-0.5032140507193694
|
||||
],
|
||||
"world": [
|
||||
1.005210138900337,
|
||||
0.5338681642707513,
|
||||
1.365657849584086
|
||||
]
|
||||
}
|
||||
],
|
||||
"truth_status": "provisional_witness_space",
|
||||
"used_points": 47
|
||||
}
|
||||
|
Before Width: | Height: | Size: 392 KiB |
|
Before Width: | Height: | Size: 66 KiB |
|
Before Width: | Height: | Size: 48 KiB |
|
Before Width: | Height: | Size: 48 KiB |
|
Before Width: | Height: | Size: 49 KiB |
|
Before Width: | Height: | Size: 47 KiB |
|
Before Width: | Height: | Size: 110 KiB |
|
|
@ -1,46 +0,0 @@
|
|||
# Complex Contour Live Scarlett Receipt 2026-05-28 00:23:13
|
||||
|
||||
## Setup
|
||||
|
||||
- Renderer: `canary-packet-trill`
|
||||
- Playback artifact: `artifacts/asio/canary-packet-live-20260528-002313-f32.raw`
|
||||
- Capture artifact: `artifacts/asio/scarlett-canary-packet-live-20260528-002313-f32.raw`
|
||||
- ASIO driver: `Focusrite USB ASIO`
|
||||
- Sample rate: `192000`
|
||||
- Capture seconds: `4.2`
|
||||
- Inputs: `Input 1`, `Input 2`, `Loopback 1`, `Loopback 2`
|
||||
- Outputs: `Output 1`, `Output 2`
|
||||
- Playback gain: `1.0`
|
||||
- Captured frames: `805888`
|
||||
- Nonzero samples: `2294204`
|
||||
|
||||
## Results
|
||||
|
||||
Reference channel is `asio-ch2` / Loopback 1.
|
||||
|
||||
| Candidate | Meaning | Estimate | Prediction Error | Confidence | Direct Hits | Notes |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| `asio-ch3` | Loopback 2 | `-0.011837` samples / `-0.062 us` | `-0.062 us` | `0.956` | `257` | good loopback proof |
|
||||
| `asio-ch1` | Input 2 | no lock | n/a | n/a | `0` | no contour hits |
|
||||
| `asio-ch0` | Input 1 | no lock | n/a | n/a | `0` | no contour hits |
|
||||
|
||||
## Signal Sanity Check
|
||||
|
||||
`asio-ch2` and `asio-ch3` contain the rendered packet-song witness. Physical
|
||||
inputs are present but far below usable contour-lock level in this capture.
|
||||
|
||||
| Channel | RMS | Peak | Speech RMS | Ultrasonic RMS |
|
||||
| --- | ---: | ---: | ---: | ---: |
|
||||
| `asio-ch0` | `0.000525` | `0.002648` | `0.000063` | `0.000187` |
|
||||
| `asio-ch1` | `0.000087` | `0.000465` | `0.000010` | `0.000031` |
|
||||
| `asio-ch2` | `0.006676` | `0.028226` | `0.005157` | `0.002268` |
|
||||
| `asio-ch3` | `0.006676` | `0.028226` | `0.005157` | `0.002268` |
|
||||
|
||||
## Interpretation
|
||||
|
||||
The complex-contour receiver and ASIO loopback path still prove sub-microsecond
|
||||
interface timing on a fresh capture. The microphone step is blocked by current
|
||||
input signal state, not by the contour decoder: both physical input channels
|
||||
produced zero contour hits against 1480 reference hits. Next physical proof
|
||||
needs the mic gain/routing/source placement fixed before changing receiver
|
||||
logic.
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
# Complex Contour Live Scarlett Receipt 2026-05-28 08:06:04
|
||||
|
||||
## Setup
|
||||
|
||||
- Renderer: `canary-packet-trill`
|
||||
- Playback artifact: `artifacts/asio/canary-packet-live-20260528-080604-f32.raw`
|
||||
- Capture artifact: `artifacts/asio/scarlett-canary-packet-live-20260528-080604-f32.raw`
|
||||
- ASIO driver: `Focusrite USB ASIO`
|
||||
- Sample rate: `192000`
|
||||
- Capture seconds: `4.2`
|
||||
- Inputs: armed cardioid and shotgun on Scarlett input channels
|
||||
- Output routing: headphones by mistake
|
||||
- Loopback reference: `asio-ch2` / Loopback 1
|
||||
- Playback gain: `1.0`
|
||||
- Captured frames: `806656`
|
||||
- Nonzero samples: `2296810`
|
||||
|
||||
## Results
|
||||
|
||||
| Candidate | Estimate | Prediction Error | Confidence | Direct Hits | Notes |
|
||||
| --- | ---: | ---: | ---: | ---: | --- |
|
||||
| `asio-ch3` | `-0.002628` samples / `-0.014 us` | `-0.014 us` | `0.971` | `216` | loopback proof |
|
||||
| `asio-ch1` | `544.957302` samples / `2838.319 us` | `+3.710 us` | `0.358` | `13` | better physical path this run |
|
||||
| `asio-ch0` | `784.256427` samples / `4084.669 us` | `+14.404 us` | `0.276` | `9` | weaker physical path this run |
|
||||
|
||||
## Signal Sanity Check
|
||||
|
||||
Both physical inputs are now armed and carrying real signal. They remain lower
|
||||
confidence than loopback, but this is no longer the zero-contour-hit failure
|
||||
from `complex-contour-live-20260528-002313.md`.
|
||||
|
||||
| Channel | RMS | Peak | Speech RMS | Ultrasonic RMS |
|
||||
| --- | ---: | ---: | ---: | ---: |
|
||||
| `asio-ch0` | `0.006902` | `0.037563` | `0.002519` | `0.002437` |
|
||||
| `asio-ch1` | `0.002714` | `0.033561` | `0.002128` | `0.000431` |
|
||||
| `asio-ch2` | `0.006673` | `0.028226` | `0.005154` | `0.002267` |
|
||||
| `asio-ch3` | `0.006673` | `0.028226` | `0.005154` | `0.002267` |
|
||||
|
||||
## Interpretation
|
||||
|
||||
The contour receiver sees both physical inputs, but this run was accidentally
|
||||
routed to headphones rather than the monitors. Treat it as a misroute witness,
|
||||
not a room/mic proof. It is useful mainly because it explains the weak physical
|
||||
confidence and proves the harness did not silently lose the armed inputs.
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
# Complex Contour Live Scarlett Receipt 2026-05-28 08:07:27
|
||||
|
||||
## Setup
|
||||
|
||||
- Renderer: `canary-packet-trill`
|
||||
- Playback artifact: `artifacts/asio/canary-packet-live-20260528-080727-f32.raw`
|
||||
- Capture artifact: `artifacts/asio/scarlett-canary-packet-live-20260528-080727-f32.raw`
|
||||
- ASIO driver: `Focusrite USB ASIO`
|
||||
- Sample rate: `192000`
|
||||
- Capture seconds: `4.2`
|
||||
- Output routing: monitors
|
||||
- Inputs: `Input 1`, `Input 2`, `Loopback 1`, `Loopback 2`
|
||||
- Physical mapping:
|
||||
- `asio-ch0` / Input 1: shotgun, about 1.5 m from right monitor and 3.0 m from left monitor
|
||||
- `asio-ch1` / Input 2: cardioid, about 0.5 m from left monitor and 1.5 m from right monitor
|
||||
- Loopback reference: `asio-ch2` / Loopback 1
|
||||
- Playback gain: `1.0`
|
||||
- Captured frames: `806656`
|
||||
- Nonzero samples: `2298347`
|
||||
|
||||
## Results
|
||||
|
||||
| Candidate | Meaning | Estimate | Prediction Error | Confidence | Direct Hits | Notes |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| `asio-ch3` | Loopback 2 | `-0.002628` samples / `-0.014 us` | `-0.014 us` | `0.971` | `216` | interface loopback proof |
|
||||
| `asio-ch0` | Input 1 | `781.412261` samples / `4069.856 us` | `-0.410 us` | `0.524` | `24` | best physical path this run |
|
||||
| `asio-ch1` | Input 2 | `543.771315` samples / `2832.142 us` | `-2.467 us` | `0.363` | `14` | weaker physical path this run |
|
||||
|
||||
## Signal Sanity Check
|
||||
|
||||
Both physical inputs are carrying real monitor-routed acoustic signal. Compared
|
||||
with the headphone-routed `complex-contour-live-20260528-080604.md` run, both
|
||||
physical channels have stronger RMS and the path estimates improve.
|
||||
|
||||
| Channel | RMS | Peak | Speech RMS | Ultrasonic RMS |
|
||||
| --- | ---: | ---: | ---: | ---: |
|
||||
| `asio-ch0` | `0.012653` | `0.072978` | `0.009195` | `0.003837` |
|
||||
| `asio-ch1` | `0.014106` | `0.093789` | `0.011449` | `0.004182` |
|
||||
| `asio-ch2` | `0.006673` | `0.028226` | `0.005154` | `0.002267` |
|
||||
| `asio-ch3` | `0.006673` | `0.028226` | `0.005154` | `0.002267` |
|
||||
|
||||
## Interpretation
|
||||
|
||||
This is the first credible fresh physical complex-contour proof after arming
|
||||
both microphones and routing output to monitors. `asio-ch0` is the shotgun and
|
||||
lands within `0.410 us` of the stored path seed with moderate confidence.
|
||||
Its measured absolute delay is consistent with a nearest-monitor path on the
|
||||
order of the right-monitor distance. `asio-ch1` is the cardioid and also locks,
|
||||
but weaker; its measured absolute delay should be interpreted against the
|
||||
actual stereo monitor routing and room path, not as a naked source-to-mic
|
||||
distance yet. The next useful cut is repeated monitor-routed captures to
|
||||
separate stable direct-path truth from one-run fit luck, then either update the
|
||||
channel model or add runtime signal-presence gating before actuator authority
|
||||
consumes physical mic reports.
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
# Complex Contour Monitor Split Receipt 2026-05-28 08:18:13
|
||||
|
||||
## Setup
|
||||
|
||||
- Renderer: `canary-packet-trill`
|
||||
- Playback artifact: `artifacts/asio/canary-packet-monitor-split-20260528-081813-f32.raw`
|
||||
- Output 0 capture: `artifacts/asio/scarlett-canary-packet-monitor-output0-20260528-081813-f32.raw`
|
||||
- Output 1 capture: `artifacts/asio/scarlett-canary-packet-monitor-output1-20260528-081813-f32.raw`
|
||||
- ASIO driver: `Focusrite USB ASIO`
|
||||
- Sample rate: `192000`
|
||||
- Capture seconds: `4.2`
|
||||
- Probe change: `--play-output-channel` selects one ASIO output buffer; `-1`
|
||||
remains the default all-output behavior.
|
||||
- Physical mapping:
|
||||
- `asio-ch0` / Input 1: shotgun, about 1.5 m from right monitor and 3.0 m from left monitor
|
||||
- `asio-ch1` / Input 2: cardioid, about 0.5 m from left monitor and 1.5 m from right monitor
|
||||
|
||||
## Signal Sanity
|
||||
|
||||
Output 0 energized loopback channel `asio-ch2`; `asio-ch3` was silent.
|
||||
Output 1 energized loopback channel `asio-ch3`; `asio-ch2` was silent.
|
||||
|
||||
Physical input RMS stayed high in both captures:
|
||||
|
||||
| Capture | `asio-ch0` RMS | `asio-ch1` RMS | Reference loopback |
|
||||
| --- | ---: | ---: | --- |
|
||||
| output 0 | `0.014660` | `0.050159` | `asio-ch2` |
|
||||
| output 1 | `0.019050` | `0.040859` | `asio-ch3` |
|
||||
|
||||
## Results
|
||||
|
||||
| Output | Reference | Mic | Estimate | Confidence | Direct Hits |
|
||||
| --- | --- | --- | ---: | ---: | ---: |
|
||||
| output 0 | `asio-ch2` | shotgun / `asio-ch0` | `783.629083` samples / `4081.401 us` | `0.392` | `19` |
|
||||
| output 1 | `asio-ch3` | shotgun / `asio-ch0` | `779.619141` samples / `4060.516 us` | `0.376` | `18` |
|
||||
| output 0 | `asio-ch2` | cardioid / `asio-ch1` | `547.779779` samples / `2853.020 us` | `0.285` | `14` |
|
||||
| output 1 | `asio-ch3` | cardioid / `asio-ch1` | `543.020433` samples / `2828.231 us` | `0.383` | `17` |
|
||||
|
||||
## Interpretation
|
||||
|
||||
The ASIO probe now isolates output buffers, and the loopback channels confirm
|
||||
that output 0 and output 1 are distinct at the interface. The physical mic
|
||||
delays, however, are nearly identical across output channels:
|
||||
|
||||
- shotgun differs by about `20.885 us`;
|
||||
- cardioid differs by about `24.789 us`.
|
||||
|
||||
That is not compatible with the declared monitor geometry if output 0 and
|
||||
output 1 correspond to isolated left/right monitors in the room. The current
|
||||
evidence says the downstream monitor path is probably summing, mirroring, or
|
||||
otherwise not isolating the physical speakers as assumed. Do not use this
|
||||
receipt to learn left/right acoustic geometry until physical speaker isolation
|
||||
is verified.
|
||||
|
|
@ -31,23 +31,6 @@
|
|||
"asio-ch2": "Focusrite Loopback 1",
|
||||
"asio-ch3": "Focusrite Loopback 2"
|
||||
}
|
||||
},
|
||||
{
|
||||
"sourceId": "raven-display",
|
||||
"displayName": "Raven network display",
|
||||
"clockDomainId": "raven-sync",
|
||||
"kind": "Video",
|
||||
"origin": "Network",
|
||||
"enabled": false,
|
||||
"adapter": "frame-events",
|
||||
"command": "../native/probes/raven_display_capture_receiver/build/Release/raven_display_capture_receiver.exe",
|
||||
"arguments": [
|
||||
"--emit-json-frames",
|
||||
"--source-id",
|
||||
"raven-display",
|
||||
"--clock-domain",
|
||||
"raven-sync"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,92 +0,0 @@
|
|||
{
|
||||
"bufferSeconds": 5.0,
|
||||
"streams": [
|
||||
{
|
||||
"sourceId": "leap-stereo-ir",
|
||||
"displayName": "LeapUVC stereo IR",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_f182&pid_0003&mi_00",
|
||||
"width": 640,
|
||||
"height": 240,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "LeapStereoIr",
|
||||
"minimumFramesPerSecond": 100.0,
|
||||
"queueDepth": 8
|
||||
},
|
||||
{
|
||||
"sourceId": "ps3-eye-0",
|
||||
"displayName": "PS3 Eye 0",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ps3eye",
|
||||
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
|
||||
"cameraIndex": 0,
|
||||
"width": 320,
|
||||
"height": 240,
|
||||
"framesPerSecond": 187
|
||||
},
|
||||
{
|
||||
"sourceId": "ps3-eye-1",
|
||||
"displayName": "PS3 Eye 1",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ps3eye",
|
||||
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
|
||||
"cameraIndex": 1,
|
||||
"width": 320,
|
||||
"height": 240,
|
||||
"framesPerSecond": 187
|
||||
},
|
||||
{
|
||||
"sourceId": "kiyo-pro-yuy2",
|
||||
"displayName": "Kiyo Pro YUY2",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_1532&pid_0e05&mi_00",
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
"pixelFormat": "YUY2",
|
||||
"minimumFramesPerSecond": 25.0,
|
||||
"queueDepth": 8
|
||||
},
|
||||
{
|
||||
"sourceId": "kiyo-pro-mf-gpu",
|
||||
"displayName": "Kiyo Pro MF GPU decode",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": false,
|
||||
"adapter": "mf-gpu",
|
||||
"command": "../native/camera_capture/build/Release/mimir_mf_gpu_capture.dll",
|
||||
"pathNeedle": "vid_1532&pid_0e05",
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
"inputFormat": "MJPG",
|
||||
"outputFormat": "RGB32",
|
||||
"minimumFramesPerSecond": 25.0
|
||||
},
|
||||
{
|
||||
"sourceId": "kiyo-yuy2",
|
||||
"displayName": "Kiyo YUY2",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_1532&pid_0e03&mi_00",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"pixelFormat": "YUY2",
|
||||
"minimumFramesPerSecond": 25.0,
|
||||
"queueDepth": 8
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
{
|
||||
"bufferSeconds": 2.0,
|
||||
"streams": [
|
||||
{
|
||||
"sourceId": "leap-stereo-ir",
|
||||
"displayName": "LeapUVC stereo IR",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_f182&pid_0003&mi_00",
|
||||
"width": 640,
|
||||
"height": 240,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "LeapStereoIr",
|
||||
"minimumFramesPerSecond": 100.0,
|
||||
"queueDepth": 8
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,108 +0,0 @@
|
|||
{
|
||||
"bufferSeconds": 5.0,
|
||||
"audioSync": {
|
||||
"mode": "passive",
|
||||
"referenceSourceId": "asio-ch2",
|
||||
"calibrationGain": 0.0,
|
||||
"watermarkGain": 0.0,
|
||||
"complexContourChannelModelPath": "../calibration/bioacoustic/complex-contour-channel-model.json",
|
||||
"enableComplexContourRuntime": true
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
"sourceId": "focusrite-asio",
|
||||
"displayName": "Starfire Scarlett ASIO",
|
||||
"kind": "Audio",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "asio",
|
||||
"command": "../native/asio_capture/build/Release/mimir_asio_capture.dll",
|
||||
"sampleRate": 192000,
|
||||
"acceptSourceIds": [
|
||||
"asio-ch0",
|
||||
"asio-ch1",
|
||||
"asio-ch2",
|
||||
"asio-ch3"
|
||||
],
|
||||
"sourceLabels": {
|
||||
"asio-ch0": "Shotgun hero mic",
|
||||
"asio-ch1": "Cardioid hero mic",
|
||||
"asio-ch2": "Raven program loopback L",
|
||||
"asio-ch3": "Raven program loopback R"
|
||||
}
|
||||
},
|
||||
{
|
||||
"sourceId": "kiyo-pro-rgb",
|
||||
"displayName": "Kiyo Pro program view",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_1532&pid_0e05&mi_00",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "YUY2",
|
||||
"minimumFramesPerSecond": 25.0,
|
||||
"queueDepth": 8
|
||||
},
|
||||
{
|
||||
"sourceId": "ps3-eye-0",
|
||||
"displayName": "PS3 Eye 0 Move tracker",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ps3eye",
|
||||
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
|
||||
"cameraIndex": 0,
|
||||
"width": 320,
|
||||
"height": 240,
|
||||
"framesPerSecond": 187
|
||||
},
|
||||
{
|
||||
"sourceId": "ps3-eye-1",
|
||||
"displayName": "PS3 Eye 1 Move tracker",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ps3eye",
|
||||
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
|
||||
"cameraIndex": 1,
|
||||
"width": 320,
|
||||
"height": 240,
|
||||
"framesPerSecond": 187
|
||||
},
|
||||
{
|
||||
"sourceId": "raven-display",
|
||||
"displayName": "Raven screen capture",
|
||||
"clockDomainId": "raven-sync",
|
||||
"kind": "Video",
|
||||
"origin": "Network",
|
||||
"enabled": true,
|
||||
"adapter": "ffmpeg-rawvideo",
|
||||
"command": "ffmpeg",
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
"pixelFormat": "Bgra8",
|
||||
"arguments": [
|
||||
"-hide_banner",
|
||||
"-nostdin",
|
||||
"-loglevel",
|
||||
"warning",
|
||||
"-fflags",
|
||||
"nobuffer",
|
||||
"-flags",
|
||||
"low_delay",
|
||||
"-i",
|
||||
"srt://0.0.0.0:5200?mode=listener&latency=120000&timeout=5000000",
|
||||
"-an",
|
||||
"-f",
|
||||
"rawvideo",
|
||||
"-pix_fmt",
|
||||
"bgra",
|
||||
"-"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,67 +0,0 @@
|
|||
{
|
||||
"bufferSeconds": 5.0,
|
||||
"audioSync": {
|
||||
"mode": "passive",
|
||||
"referenceSourceId": "asio-ch2",
|
||||
"calibrationGain": 0.0,
|
||||
"watermarkGain": 0.0,
|
||||
"complexContourChannelModelPath": "../calibration/bioacoustic/complex-contour-channel-model.json",
|
||||
"enableComplexContourRuntime": true
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
"sourceId": "focusrite-asio",
|
||||
"displayName": "Starfire Scarlett ASIO",
|
||||
"kind": "Audio",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "asio",
|
||||
"command": "../native/asio_capture/build/Release/mimir_asio_capture.dll",
|
||||
"sampleRate": 192000,
|
||||
"acceptSourceIds": [
|
||||
"asio-ch0",
|
||||
"asio-ch1",
|
||||
"asio-ch2",
|
||||
"asio-ch3"
|
||||
],
|
||||
"sourceLabels": {
|
||||
"asio-ch0": "Shotgun hero mic",
|
||||
"asio-ch1": "Cardioid hero mic",
|
||||
"asio-ch2": "Raven/program loopback L",
|
||||
"asio-ch3": "Raven/program loopback R"
|
||||
}
|
||||
},
|
||||
{
|
||||
"sourceId": "leap-stereo-ir",
|
||||
"displayName": "LeapUVC stereo IR / depth root",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_f182&pid_0003&mi_00",
|
||||
"width": 640,
|
||||
"height": 240,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "LeapStereoIr",
|
||||
"minimumFramesPerSecond": 100.0,
|
||||
"queueDepth": 8
|
||||
},
|
||||
{
|
||||
"sourceId": "kiyo-pro-rgb",
|
||||
"displayName": "Kiyo Pro RGB program view",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_1532&pid_0e05&mi_00",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "YUY2",
|
||||
"minimumFramesPerSecond": 25.0,
|
||||
"queueDepth": 8
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,85 +0,0 @@
|
|||
{
|
||||
"bufferSeconds": 5.0,
|
||||
"audioSync": {
|
||||
"mode": "passive",
|
||||
"referenceSourceId": "loopback-scarlett-speakers",
|
||||
"calibrationGain": 0.0,
|
||||
"watermarkGain": 0.0
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
"sourceId": "leap-stereo-ir",
|
||||
"displayName": "LeapUVC stereo IR packed R/G",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_f182&pid_0003&mi_00",
|
||||
"width": 640,
|
||||
"height": 240,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "LeapStereoIr",
|
||||
"minimumFramesPerSecond": 100.0,
|
||||
"queueDepth": 8
|
||||
},
|
||||
{
|
||||
"sourceId": "ps3-eye-0",
|
||||
"displayName": "PS3 Eye 0 high-rate tracking",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ps3eye",
|
||||
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
|
||||
"cameraIndex": 0,
|
||||
"width": 320,
|
||||
"height": 240,
|
||||
"framesPerSecond": 187
|
||||
},
|
||||
{
|
||||
"sourceId": "ps3-eye-1",
|
||||
"displayName": "PS3 Eye 1 high-rate tracking",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ps3eye",
|
||||
"command": "../native/camera_capture/build/Release/mimir_ps3eye_capture.dll",
|
||||
"cameraIndex": 1,
|
||||
"width": 320,
|
||||
"height": 240,
|
||||
"framesPerSecond": 187
|
||||
},
|
||||
{
|
||||
"sourceId": "kiyo-pro-rgb",
|
||||
"displayName": "Kiyo Pro RGB context",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_1532&pid_0e05&mi_00",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "YUY2",
|
||||
"minimumFramesPerSecond": 25.0,
|
||||
"queueDepth": 8
|
||||
},
|
||||
{
|
||||
"sourceId": "kiyo-basic-rgb",
|
||||
"displayName": "Kiyo RGB context",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_1532&pid_0e03&mi_00",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "YUY2",
|
||||
"minimumFramesPerSecond": 25.0,
|
||||
"queueDepth": 8
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,107 +0,0 @@
|
|||
{
|
||||
"bufferSeconds": 5.0,
|
||||
"audioSync": {
|
||||
"mode": "chirp-only",
|
||||
"referenceSourceId": "asio-ch2",
|
||||
"calibrationGain": 2.0,
|
||||
"watermarkGain": 0.035,
|
||||
"calibrationModelPath": "../calibration/chirp-bin/scarlett-artifact.json",
|
||||
"complexContourChannelModelPath": "../calibration/bioacoustic/complex-contour-channel-model.json",
|
||||
"bioacousticWitnessProfileId": "canary-packet-trill",
|
||||
"enableComplexContourRuntime": true
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
"sourceId": "focusrite-asio",
|
||||
"kind": "Audio",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "asio",
|
||||
"command": "../native/asio_capture/build/Release/mimir_asio_capture.dll",
|
||||
"sampleRate": 192000,
|
||||
"acceptSourceIds": [
|
||||
"asio-ch0",
|
||||
"asio-ch1",
|
||||
"asio-ch2",
|
||||
"asio-ch3"
|
||||
],
|
||||
"sourceLabels": {
|
||||
"asio-ch0": "Focusrite Input 1",
|
||||
"asio-ch1": "Focusrite Input 2",
|
||||
"asio-ch2": "Focusrite Loopback 1",
|
||||
"asio-ch3": "Focusrite Loopback 2"
|
||||
}
|
||||
},
|
||||
{
|
||||
"sourceId": "raven-display",
|
||||
"displayName": "Raven screen capture",
|
||||
"clockDomainId": "raven-sync",
|
||||
"kind": "Video",
|
||||
"origin": "Network",
|
||||
"enabled": true,
|
||||
"adapter": "ffmpeg-rawvideo",
|
||||
"command": "ffmpeg",
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
"pixelFormat": "Bgra8",
|
||||
"arguments": [
|
||||
"-hide_banner",
|
||||
"-nostdin",
|
||||
"-loglevel",
|
||||
"warning",
|
||||
"-fflags",
|
||||
"nobuffer",
|
||||
"-flags",
|
||||
"low_delay",
|
||||
"-i",
|
||||
"srt://0.0.0.0:5200?mode=listener&latency=120000&timeout=5000000",
|
||||
"-an",
|
||||
"-f",
|
||||
"rawvideo",
|
||||
"-pix_fmt",
|
||||
"bgra",
|
||||
"-"
|
||||
]
|
||||
},
|
||||
{
|
||||
"sourceId": "eve-camera",
|
||||
"displayName": "Eve camera capture",
|
||||
"clockDomainId": "eve-camera",
|
||||
"kind": "Video",
|
||||
"origin": "Network",
|
||||
"enabled": true,
|
||||
"adapter": "frame-events",
|
||||
"command": "../src/Mimir.EveSensorReceiver/bin/Debug/net10.0/Mimir.EveSensorReceiver.exe",
|
||||
"arguments": [
|
||||
"--port",
|
||||
"8793",
|
||||
"--path",
|
||||
"/eve/camera",
|
||||
"--source-id",
|
||||
"eve-camera",
|
||||
"--type",
|
||||
"video-frame"
|
||||
]
|
||||
},
|
||||
{
|
||||
"sourceId": "eve-mic",
|
||||
"displayName": "Eve microphone capture",
|
||||
"clockDomainId": "eve-camera",
|
||||
"kind": "Audio",
|
||||
"origin": "Network",
|
||||
"enabled": true,
|
||||
"adapter": "frame-events",
|
||||
"command": "../src/Mimir.EveSensorReceiver/bin/Debug/net10.0/Mimir.EveSensorReceiver.exe",
|
||||
"arguments": [
|
||||
"--port",
|
||||
"8794",
|
||||
"--path",
|
||||
"/eve/mic",
|
||||
"--source-id",
|
||||
"eve-mic",
|
||||
"--type",
|
||||
"audio-block"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,131 +0,0 @@
|
|||
{
|
||||
"bufferSeconds": 5.0,
|
||||
"audioSync": {
|
||||
"mode": "passive",
|
||||
"referenceSourceId": "asio-ch2",
|
||||
"calibrationGain": 0.0,
|
||||
"watermarkGain": 0.0,
|
||||
"complexContourChannelModelPath": "../calibration/bioacoustic/complex-contour-channel-model.json",
|
||||
"enableComplexContourRuntime": true
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
"sourceId": "focusrite-asio",
|
||||
"displayName": "Starfire Scarlett ASIO",
|
||||
"kind": "Audio",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "asio",
|
||||
"command": "../native/asio_capture/build/Release/mimir_asio_capture.dll",
|
||||
"sampleRate": 192000,
|
||||
"acceptSourceIds": [
|
||||
"asio-ch0",
|
||||
"asio-ch1",
|
||||
"asio-ch2",
|
||||
"asio-ch3"
|
||||
],
|
||||
"sourceLabels": {
|
||||
"asio-ch0": "Shotgun hero mic",
|
||||
"asio-ch1": "Cardioid hero mic",
|
||||
"asio-ch2": "Raven program loopback L",
|
||||
"asio-ch3": "Raven program loopback R"
|
||||
}
|
||||
},
|
||||
{
|
||||
"sourceId": "leap-stereo-ir",
|
||||
"displayName": "LeapUVC stereo IR / depth root",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_f182&pid_0003&mi_00",
|
||||
"width": 640,
|
||||
"height": 240,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "LeapStereoIr",
|
||||
"minimumFramesPerSecond": 100.0,
|
||||
"queueDepth": 8
|
||||
},
|
||||
{
|
||||
"sourceId": "kiyo-pro-rgb",
|
||||
"displayName": "Kiyo Pro AR program view",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_1532&pid_0e05&mi_00",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "YUY2",
|
||||
"minimumFramesPerSecond": 25.0,
|
||||
"queueDepth": 8
|
||||
},
|
||||
{
|
||||
"sourceId": "raven-display",
|
||||
"displayName": "Raven screen capture",
|
||||
"clockDomainId": "raven-sync",
|
||||
"kind": "Video",
|
||||
"origin": "Network",
|
||||
"enabled": true,
|
||||
"adapter": "ffmpeg-rawvideo",
|
||||
"command": "ffmpeg",
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
"pixelFormat": "Bgra8",
|
||||
"arguments": [
|
||||
"-hide_banner",
|
||||
"-nostdin",
|
||||
"-loglevel",
|
||||
"warning",
|
||||
"-fflags",
|
||||
"nobuffer",
|
||||
"-flags",
|
||||
"low_delay",
|
||||
"-i",
|
||||
"srt://0.0.0.0:5210?mode=listener&latency=20000&timeout=5000000",
|
||||
"-an",
|
||||
"-f",
|
||||
"rawvideo",
|
||||
"-pix_fmt",
|
||||
"bgra",
|
||||
"-"
|
||||
]
|
||||
},
|
||||
{
|
||||
"sourceId": "raven-realtk-loopback",
|
||||
"displayName": "Raven Realtek loopback",
|
||||
"clockDomainId": "raven-sync",
|
||||
"kind": "Audio",
|
||||
"origin": "Network",
|
||||
"enabled": true,
|
||||
"adapter": "ffmpeg-pcmaudio",
|
||||
"command": "ffmpeg",
|
||||
"sampleRate": 48000,
|
||||
"channels": 2,
|
||||
"sampleFormat": "Float32",
|
||||
"blockFrames": 960,
|
||||
"arguments": [
|
||||
"-hide_banner",
|
||||
"-nostdin",
|
||||
"-loglevel",
|
||||
"warning",
|
||||
"-fflags",
|
||||
"nobuffer",
|
||||
"-f",
|
||||
"f32le",
|
||||
"-ar",
|
||||
"48000",
|
||||
"-ac",
|
||||
"2",
|
||||
"-i",
|
||||
"srt://0.0.0.0:5212?mode=listener&latency=20000&timeout=5000000",
|
||||
"-f",
|
||||
"f32le",
|
||||
"-"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,187 +0,0 @@
|
|||
{
|
||||
"bufferSeconds": 5.0,
|
||||
"audioSync": {
|
||||
"mode": "passive",
|
||||
"referenceSourceId": "asio-ch2",
|
||||
"calibrationGain": 0.0,
|
||||
"watermarkGain": 0.0,
|
||||
"complexContourChannelModelPath": "../calibration/bioacoustic/complex-contour-channel-model.json",
|
||||
"enableComplexContourRuntime": true
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
"sourceId": "focusrite-asio",
|
||||
"displayName": "Starfire Scarlett ASIO",
|
||||
"kind": "Audio",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "asio",
|
||||
"command": "../native/asio_capture/build/Release/mimir_asio_capture.dll",
|
||||
"sampleRate": 192000,
|
||||
"acceptSourceIds": [
|
||||
"asio-ch0",
|
||||
"asio-ch1",
|
||||
"asio-ch2",
|
||||
"asio-ch3"
|
||||
],
|
||||
"sourceLabels": {
|
||||
"asio-ch0": "Shotgun hero mic",
|
||||
"asio-ch1": "Cardioid hero mic",
|
||||
"asio-ch2": "Starfire/Raven loopback L",
|
||||
"asio-ch3": "Starfire/Raven loopback R"
|
||||
}
|
||||
},
|
||||
{
|
||||
"sourceId": "leap-stereo-ir",
|
||||
"displayName": "LeapUVC stereo IR / depth root",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_f182&pid_0003&mi_00",
|
||||
"width": 640,
|
||||
"height": 240,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "LeapStereoIr",
|
||||
"minimumFramesPerSecond": 100.0,
|
||||
"queueDepth": 8
|
||||
},
|
||||
{
|
||||
"sourceId": "kiyo-pro-rgb",
|
||||
"displayName": "Kiyo Pro AR program view",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_1532&pid_0e05&mi_00",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "YUY2",
|
||||
"minimumFramesPerSecond": 25.0,
|
||||
"queueDepth": 8
|
||||
},
|
||||
{
|
||||
"sourceId": "kiyo-basic-rgb",
|
||||
"displayName": "Kiyo RGB context",
|
||||
"kind": "Video",
|
||||
"origin": "LocalDevice",
|
||||
"enabled": true,
|
||||
"adapter": "ks-camera",
|
||||
"command": "../native/camera_capture/build/Release/mimir_camera_capture.dll",
|
||||
"pathNeedle": "vid_1532&pid_0e03&mi_00",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"inputFormat": "YUY2",
|
||||
"pixelFormat": "YUY2",
|
||||
"minimumFramesPerSecond": 25.0,
|
||||
"queueDepth": 8
|
||||
},
|
||||
{
|
||||
"sourceId": "raven-display",
|
||||
"displayName": "Raven screen capture",
|
||||
"clockDomainId": "raven-sync",
|
||||
"kind": "Video",
|
||||
"origin": "Network",
|
||||
"enabled": true,
|
||||
"adapter": "ffmpeg-rawvideo",
|
||||
"command": "ffmpeg",
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
"pixelFormat": "Bgra8",
|
||||
"arguments": [
|
||||
"-hide_banner",
|
||||
"-nostdin",
|
||||
"-loglevel",
|
||||
"warning",
|
||||
"-fflags",
|
||||
"nobuffer",
|
||||
"-flags",
|
||||
"low_delay",
|
||||
"-i",
|
||||
"srt://0.0.0.0:5210?mode=listener&latency=20000&timeout=5000000",
|
||||
"-an",
|
||||
"-f",
|
||||
"rawvideo",
|
||||
"-pix_fmt",
|
||||
"bgra",
|
||||
"-"
|
||||
]
|
||||
},
|
||||
{
|
||||
"sourceId": "raven-realtk-loopback",
|
||||
"displayName": "Raven Realtek loopback",
|
||||
"clockDomainId": "raven-sync",
|
||||
"kind": "Audio",
|
||||
"origin": "Network",
|
||||
"enabled": true,
|
||||
"adapter": "ffmpeg-pcmaudio",
|
||||
"command": "ffmpeg",
|
||||
"sampleRate": 48000,
|
||||
"channels": 2,
|
||||
"sampleFormat": "Float32",
|
||||
"blockFrames": 960,
|
||||
"arguments": [
|
||||
"-hide_banner",
|
||||
"-nostdin",
|
||||
"-loglevel",
|
||||
"warning",
|
||||
"-fflags",
|
||||
"nobuffer",
|
||||
"-f",
|
||||
"f32le",
|
||||
"-ar",
|
||||
"48000",
|
||||
"-ac",
|
||||
"2",
|
||||
"-i",
|
||||
"srt://0.0.0.0:5212?mode=listener&latency=20000&timeout=5000000",
|
||||
"-f",
|
||||
"f32le",
|
||||
"-"
|
||||
]
|
||||
},
|
||||
{
|
||||
"sourceId": "eve-camera",
|
||||
"displayName": "Eve iPad camera capture",
|
||||
"clockDomainId": "eve-local",
|
||||
"kind": "Video",
|
||||
"origin": "Network",
|
||||
"enabled": true,
|
||||
"adapter": "frame-events",
|
||||
"command": "../src/Mimir.EveSensorReceiver/bin/Debug/net10.0/Mimir.EveSensorReceiver.exe",
|
||||
"arguments": [
|
||||
"--port",
|
||||
"8793",
|
||||
"--path",
|
||||
"/eve/camera",
|
||||
"--source-id",
|
||||
"eve-camera",
|
||||
"--type",
|
||||
"video-frame"
|
||||
]
|
||||
},
|
||||
{
|
||||
"sourceId": "eve-mic",
|
||||
"displayName": "Eve iPad microphone capture",
|
||||
"clockDomainId": "eve-local",
|
||||
"kind": "Audio",
|
||||
"origin": "Network",
|
||||
"enabled": true,
|
||||
"adapter": "frame-events",
|
||||
"command": "../src/Mimir.EveSensorReceiver/bin/Debug/net10.0/Mimir.EveSensorReceiver.exe",
|
||||
"arguments": [
|
||||
"--port",
|
||||
"8794",
|
||||
"--path",
|
||||
"/eve/mic",
|
||||
"--source-id",
|
||||
"eve-mic",
|
||||
"--type",
|
||||
"audio-block"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
13
config/mimir-spectrum-field.aquafield
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
# Fensalir field DSL: bind rolling spectrum texture, derive tube fields, shade by surface graph, lower to reservoir.
|
||||
texture id=spectrumHistory
|
||||
|
||||
surface id=amplitude op=texture.sample value=0,0,0,0
|
||||
surface id=derivative op=dFdx inputs=amplitude value=0,0,0,0
|
||||
surface id=curvature op=d2Fdx2 inputs=amplitude value=0,0,0,0
|
||||
surface id=emission op=mix inputs=amplitude,derivative,curvature value=1.0,0.72,0.22,1.0
|
||||
surface id=coverage op=sdf.tube inputs=amplitude,derivative,curvature value=0.74,0.20,0.0,0.0
|
||||
|
||||
splinefield id=asioSpectrumWaterfall texture=spectrumHistory frequencyAxis=x firstColumn=0 columns=160 columnStride=1 rollingModulo=160 subdivisions=4 maxProbes=131072 density=1.0 minimumContribution=0.004 origin=-0.82,0.40,-0.18 axisStep=0.016,0.0,0.0 columnStep=0.0,-0.0055,0.0060 amplitudeScale=0.34 radius=0.064 alpha=0.98 zeroThreshold=0.68 feather=0.18 tangent=1.0 curvature=0.68 normal=0.34 derivative=0.88 emission=1.0,0.72,0.22,4.0 seed=2971668797
|
||||
|
||||
reservoir splats=131072 updates=131072 candidates=2 seed=270943260
|
||||
direct
|
||||
|
|
@ -31,15 +31,24 @@
|
|||
"outputs": {
|
||||
"video": "synchronized-program-video",
|
||||
"audio": "separately-controllable-synchronized-stems"
|
||||
},
|
||||
"programComposition": {
|
||||
"owner": "Mimir",
|
||||
"sceneDocument": "mimir.program_scene.v1",
|
||||
"operatorSurface": "mimir.eve_operator_surface.v1",
|
||||
"outputDocument": "mimir.program_output.v1",
|
||||
"sitePublisher": "Yggdrasil",
|
||||
"obsRole": "compatibility-sink-only"
|
||||
}
|
||||
},
|
||||
"authorities": {
|
||||
"configurationCalibrationLaunchStatus": "Mimir",
|
||||
"programComposition": "Mimir",
|
||||
"sitePublication": "Yggdrasil publisher daemon",
|
||||
"denseVisualFusion": "Fensalir",
|
||||
"gpuMaterialAndBrushReconciliation": "Fensalir",
|
||||
"spoutPublication": "Fensalir",
|
||||
"hotAudioDsp": "Faust",
|
||||
"obsBroadcastControls": "OBS"
|
||||
"programTextureOutput": "Fensalir",
|
||||
"hotAudioDsp": "Faust"
|
||||
},
|
||||
"visualSensors": [
|
||||
{ "id": "kiyo-primary", "role": "rgb-main", "required": true },
|
||||
|
|
@ -70,9 +79,9 @@
|
|||
],
|
||||
"outputs": {
|
||||
"video": {
|
||||
"target": "Spout2",
|
||||
"senderName": "Mimir Point Cloud",
|
||||
"owner": "Fensalir"
|
||||
"target": "MimirProgramTexture",
|
||||
"senderName": "Mimir Program Texture",
|
||||
"owner": "Mimir/Fensalir"
|
||||
},
|
||||
"audio": {
|
||||
"owner": "Faust",
|
||||
|
|
@ -85,6 +94,20 @@
|
|||
"co_streamer_loopback",
|
||||
"spatial_bed"
|
||||
]
|
||||
},
|
||||
"operator": {
|
||||
"owner": "Mimir",
|
||||
"surfaces": [
|
||||
"cultmesh-command-surface"
|
||||
]
|
||||
},
|
||||
"site": {
|
||||
"owner": "Yggdrasil publisher daemon",
|
||||
"input": "mimir.program_output.v1",
|
||||
"route": "cultmesh://yggdrasil/gamecult.site/mimir/program"
|
||||
},
|
||||
"compatibility": {
|
||||
"obs": "optional-derived-sink"
|
||||
}
|
||||
},
|
||||
"runtimePolicy": {
|
||||
|
|
@ -93,7 +116,8 @@
|
|||
"native-driver-workers",
|
||||
"native-reservoir",
|
||||
"Fensalir",
|
||||
"Faust"
|
||||
"Faust",
|
||||
"Yggdrasil publisher daemon"
|
||||
],
|
||||
"neverInCompatibilityEdges": [
|
||||
"dense-feature-extraction",
|
||||
|
|
|
|||
|
|
@ -2,11 +2,16 @@
|
|||
|
||||
## Objective
|
||||
|
||||
Make one local Windows computer available as OBS sources on another computer, with sender-side NVENC video encoding and independently mixable audio sources.
|
||||
Make one local Windows computer available as compatibility media sources on
|
||||
another computer, with sender-side NVENC video encoding and independently
|
||||
mixable audio sources. The current program target is Mimir-owned composition;
|
||||
this bridge is not the stream authority.
|
||||
|
||||
## Current Mechanism
|
||||
|
||||
The repo owns configuration and scripts. FFmpeg owns capture, encoding, and SRT transport. OBS owns scene composition.
|
||||
The repo owns configuration and scripts. FFmpeg owns capture, encoding, and SRT
|
||||
transport. Mimir owns scene composition; OBS may consume this bridge as a
|
||||
temporary compatibility sink.
|
||||
|
||||
Inputs become outputs like this:
|
||||
|
||||
|
|
@ -18,17 +23,13 @@ Inputs become outputs like this:
|
|||
6. Each process sends MPEG-TS over SRT to a stable receiver URL.
|
||||
7. OBS adds each URL as a Media Source.
|
||||
|
||||
`Mimir.RavenDaemon` is the current Raven-local supervisor for the combined
|
||||
display plus Realtek loopback mux. It emits named SRT sinks rather than one
|
||||
anonymous stream: Mimir receives the canonical ingest copy on `5200`, and OBS
|
||||
receives an ordinary media-source copy on `5204`.
|
||||
|
||||
## Invariants
|
||||
|
||||
- Video encoding happens on the sender.
|
||||
- OBS receives ordinary media-source URLs; it does not need a plugin for v1.
|
||||
- Mimir and OBS do not contend for the same SRT listener.
|
||||
- Audio remains separately mixable in OBS.
|
||||
- OBS can receive ordinary media-source URLs for compatibility; it does not need
|
||||
a plugin for this bridge path.
|
||||
- Audio remains separately mixable at the compatibility edge, but the durable
|
||||
control surface belongs to Mimir.
|
||||
- Ports are stable and derived from one base port plus explicit offsets.
|
||||
- Secrets, if used, stay in local config and are not committed.
|
||||
|
||||
|
|
|
|||
|
|
@ -25,12 +25,8 @@ flowchart TD
|
|||
|
||||
## Invariants
|
||||
|
||||
- Starfire Realtek is the active chirp speaker and loopback-measurement path in
|
||||
the current routing. The Windows system default output is now Scarlett for
|
||||
monitoring, so default-output chirps are wrong unless they explicitly target
|
||||
Realtek.
|
||||
- Scarlett ASIO is the professional mic and monitor interface. Its outputs are
|
||||
not the current active chirp-emission authority.
|
||||
- Scarlett speaker loopback is the timing authority when calibration chirplets
|
||||
are playing.
|
||||
- Focusrite dialogue mics are the voice anchors.
|
||||
- Camera mics are spatial/context witnesses.
|
||||
- Loopback/program audio is timing evidence where available; it outranks
|
||||
|
|
@ -90,48 +86,6 @@ not the active runtime sound.
|
|||
|
||||
The raw profile is kept even when a timing report is rejected, because a failed
|
||||
sync window can still teach us which bands survived the speaker/room/mic path.
|
||||
`MimirBioacousticProbeScheduler` is now the runtime-side cadence owner for that
|
||||
feedback loop: it consumes smoothed sync states plus per-band frequency response
|
||||
residuals, schedules a targeted probe only when confidence pressure and duty
|
||||
budget allow it, and publishes aggregate/per-source sync and frequency-response
|
||||
confidence telemetry. `MimirSynchronizationHub.UpdateBioacousticProbeSchedule`
|
||||
is the live hub surface; `MimirRuntime` prints
|
||||
`mimir-bioacoustic-probe-schedule` and `mimir-bioacoustic-probe-source` lines so
|
||||
operators can watch confidence climb as probes are spent. The scheduler owns
|
||||
when to spend active sound. It does not own the output device: Starfire Realtek
|
||||
still must be selected explicitly by the eventual live renderer.
|
||||
The first explicit Realtek renderer proof now lives in the repo tooling:
|
||||
`Mimir.WasapiLoopback` can enumerate render endpoints, select one by friendly
|
||||
name substring, and play mono Float32 probe audio through shared-mode WASAPI.
|
||||
`Mimir.BufferSmoke --targeted-bioacoustic-probe-realtk-live` renders a targeted
|
||||
probe plan, starts Scarlett ASIO capture, plays the probe through
|
||||
`Speakers (Realtek(R) Audio)`, and scores the captured shotgun/cardioid bands.
|
||||
The targeted probe planner now caps default acoustic probe selection at 12 kHz
|
||||
and mildly biases low/mid measurable bands, because commodity speakers and the
|
||||
current Scarlett mics have little reason to carry useful calibration evidence
|
||||
above that range. The 2026-06-01 proof at gain `0.75` selected the Realtek speaker endpoint and
|
||||
captured `artifacts/wasapi/targeted-bioacoustic-realtk-live/20260601-033851`.
|
||||
The physical mics heard the burst, but the scored confidence still concentrated
|
||||
in low/mid evidence: shotgun peak `0.407887`, cardioid peak `0.074494`, 18.6 kHz
|
||||
confidence near zero, and the best reported band was cardioid 3.6 kHz at
|
||||
confidence `0.098`. Treat high-band probes as poor questions for this route
|
||||
until a better speaker/mic/path proves otherwise.
|
||||
The live exploration path is scheduler-driven, not a blind sweep:
|
||||
`Mimir.BufferSmoke --bioacoustic-realtk-scheduled-calibration-live` repeatedly
|
||||
asks `MimirBioacousticProbeScheduler` for the next probe plan, emits that plan
|
||||
through Realtek, captures Scarlett, updates residuals, and asks again. This is
|
||||
a live physical I/O diagnostic harness seeded from the current Scarlett residual
|
||||
model; the production runtime still needs to feed measured confidence state into
|
||||
the same scheduler/emitter path. The 2026-06-01 receipt at
|
||||
`artifacts/wasapi/realtk-scheduled-calibration-live/20260601-035354` ran four
|
||||
scheduled probe steps at gain `1.15` and wrote `scheduled-response-bands.csv`
|
||||
plus `balance-plan.md`. Local peak-versus-percentile-floor scoring found strong
|
||||
usable response near `910 Hz`, `3.6 kHz`, and `9.84-9.87 kHz` on both Scarlett
|
||||
mics.
|
||||
The scheduler and planner share the same measurable frequency boundary
|
||||
(`120 Hz` through `12 kHz` by default). Residuals above that window may remain
|
||||
diagnostic scars, but they no longer own live probe cadence or the weakest-band
|
||||
status when the current room path cannot answer them.
|
||||
The receiver must consume persisted models to weight reliable symbols,
|
||||
downweight dead bands, apply phase-coherence weighting, apply first-order
|
||||
group-delay correction, and search joint global delay/frequency-shift hypotheses
|
||||
|
|
@ -139,45 +93,6 @@ before selecting the coherent anchor path. If the measured path only leaves a
|
|||
smaller alphabet, Mimir should use that reliable symbol set and raise the
|
||||
sequence order so the timeline remains unique without pretending dead bands
|
||||
still carry code.
|
||||
The first concrete two-mic cancellation owner is
|
||||
`MimirCalibratedAudioCompositeBuilder`: after fractional alignment and measured
|
||||
band correction, it scores calibrated-band coherence across mics, suppresses
|
||||
incoherent measured-band components, then performs the confidence/noise-weighted
|
||||
sum. `Mimir.BufferSmoke --dual-mic-coherence-cancellation-smoke` proves the
|
||||
contract on synthetic shotgun/cardioid witnesses: coherent shared tones survive,
|
||||
a mic-local `3.52 kHz` interferer is suppressed, and SNR rises from `1.366 dB`
|
||||
to `16.304 dB`. This is still the C# diagnostic/offline contract; Faust/native
|
||||
DSP owns the production hot lane.
|
||||
The first listenable bridge receipt is
|
||||
`scripts/record-kiyo-dual-mic-composite.ps1`. It records Kiyo video and
|
||||
Scarlett ASIO raw channels together, converts Scarlett channels 0/1 through
|
||||
the offline Faust chain with `--asio-dual-mic-faust-cleaner-wav`, then muxes
|
||||
the dialogue WAV with the Kiyo video. That command compiles and runs
|
||||
`faust/mimir_alignment_actuator.dsp` first, then feeds the aligned
|
||||
shotgun/cardioid stems into `faust/mimir_dual_mic_dialogue_cleaner.dsp`; it is a
|
||||
listening receipt for the real DSP chain, not the older C# composite audition.
|
||||
The fresh 2026-06-01 receipt at
|
||||
`artifacts/runtime/kiyo-dual-mic-composite/kiyo-dual-mic-composite-latest.mp4`
|
||||
contains 10.000 s of 640x480 30 fps Kiyo video and 10.000 s of mono AAC from the
|
||||
Faust dialogue stem. Run `20260601-051044` estimated the cardioid 359.398
|
||||
samples late at 48 kHz (7487.452 us), applied holdback 359.398/0.000 samples to
|
||||
shotgun/cardioid before cleanup, wrote a residual WAV, and reported dialogue
|
||||
RMS `0.002678` before receipt normalization. The first muxed receipt was almost
|
||||
inaudible because `NormalizePeak` only attenuated hot samples; the regenerated
|
||||
latest MP4 now peaks the dialogue WAV at -0.724 dBFS with RMS -14.202 dBFS. The
|
||||
hot owner is
|
||||
`faust/mimir_dual_mic_dialogue_cleaner.dsp`: a two-input/two-output Faust graph
|
||||
for aligned shotgun/cardioid witnesses that emits a dialogue composite stem and
|
||||
a rejected residual witness using smooth multiband voice shaping, witness
|
||||
subtraction, and slow downward expansion. `MimirRuntime` now queues that Faust
|
||||
program as `dual-mic-dialogue-cleaner` with `host_voice` and residual `ambient`
|
||||
output stems, but only feeds it from the alignment actuator's published
|
||||
`aligned_source_*` stem frame. Alignment is therefore a structural precursor to
|
||||
cleanup, not an optional sidecar. The routing invariant is covered by
|
||||
`Mimir.BufferSmoke --dialogue-cleaner-routing-smoke`: alignment frames queue the
|
||||
cleaner DSP and cleaner input block, while cleaner output frames are the
|
||||
publishable `NativeProgram` stems. Treat the C# bridge as an audition/scoring
|
||||
receipt, not the final native live DSP path.
|
||||
The synthetic invariant is
|
||||
`dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --bioacoustic-self-test`;
|
||||
it renders the bioacoustic timeline, decodes direct word anchors, and requires a
|
||||
|
|
|
|||
|
|
@ -15,8 +15,9 @@ Timestamp for this pass: 2026-05-23, starting at 23:24 Europe/London.
|
|||
|
||||
Mimir collects local and network audio/video samples into bounded rolling
|
||||
runtime buffers, uses active/passive timing evidence to place audio streams on a
|
||||
canonical timeline, and leaves Fensalir/Faust/OBS with clear ownership over
|
||||
rendering, DSP, and broadcast composition.
|
||||
canonical timeline, and lets Mimir own program composition while Fensalir lowers
|
||||
pixels, Faust/native DSP moves audio, and
|
||||
Yggdrasil publishes the public feed.
|
||||
|
||||
## Current Mechanism
|
||||
|
||||
|
|
@ -51,17 +52,18 @@ reflection taps.
|
|||
- The five-second default buffer is compute budget, not accidental delay.
|
||||
- Loopback/reference audio is timing authority; network feeds are producers, not
|
||||
clock authorities.
|
||||
- `MimirSynchronizedBufferPlanner` owns aligned readout from retained buffers:
|
||||
it chooses one canonical presentation time and applies source or
|
||||
`clockDomainId` corrections before Fensalir/Faust consume slices.
|
||||
- UI and telemetry read cached sync state; they do not run analysis.
|
||||
- Native capture owns hardware calls. Runtime owns sample identity and buffer
|
||||
placement.
|
||||
- Fensalir owns windowing, rendering, D3D12, UI, visual fusion, and program
|
||||
video output.
|
||||
- Mimir owns the program scene graph, subscription policy, preview/control
|
||||
state, stats, and publication intent.
|
||||
- Fensalir owns windowing, rendering, D3D12, UI lowering, visual fusion, and
|
||||
program video output.
|
||||
- Faust/native DSP will own hot resampling, fractional delay, separation,
|
||||
spatialization, and stems.
|
||||
- OBS owns broadcast composition.
|
||||
- The Yggdrasil-facing publisher daemon consumes Mimir program output without
|
||||
owning composition.
|
||||
- OBS is a compatibility sink only.
|
||||
|
||||
## Diagram
|
||||
|
||||
|
|
@ -86,6 +88,9 @@ flowchart TD
|
|||
AsioSource --> Sources
|
||||
Probes["native/probes/*"] --> FrameEvents["MimirFrameEventProcessStreamSource"]
|
||||
FrameEvents --> Sources
|
||||
Runtime --> Scene["Mimir program scene graph"]
|
||||
Scene --> Publisher["Yggdrasil site publisher"]
|
||||
Scene -. "compatibility" .-> Obs["OBS adapter"]
|
||||
Reservoir["native/reservoir"] -. "future lower ABI" .-> Hub
|
||||
```
|
||||
|
||||
|
|
@ -99,6 +104,9 @@ flowchart TD
|
|||
- `native/reservoir` is the lower native rolling-buffer ABI.
|
||||
- `native/probes/*` are measurement and diagnostic tools.
|
||||
- `scripts/*` are bridge utilities, not the final synchronized program path.
|
||||
- `docs/mimir-program-composition.md` is the authority map for Muninn capture
|
||||
hosts, Mimir composition, Yggdrasil publication, and OBS
|
||||
compatibility.
|
||||
|
||||
## Stage 0: Solution And App Host
|
||||
|
||||
|
|
@ -664,16 +672,15 @@ Owns:
|
|||
|
||||
Blocks:
|
||||
|
||||
- Fields: source list, ordinary audio analyzer, optional complex-contour
|
||||
analyzer, state tracker, latest ordinary/contour report dictionaries,
|
||||
- Fields: source list, analyzer, state tracker, latest reports dictionary,
|
||||
optional chirp-bin calibration model, ingest count, round-robin candidate.
|
||||
- Constructor: creates buffer set, loads optional calibration and
|
||||
complex-contour channel models, pre-creates configured stream buffers.
|
||||
- Constructor: creates buffer set, loads optional calibration model, pre-creates
|
||||
configured stream buffers.
|
||||
- `Settings`, `Buffers`, `IngestedSamples`, `SourceCount`: read-only public
|
||||
state.
|
||||
- `AudioSynchronizationReports`, `AudioSynchronizationStates`,
|
||||
`ComplexContourSynchronizationReports`, `AudioSynchronizationDecodeTraces`,
|
||||
`ChirpBinEmissionPlan`: cached analyzer and calibration outputs.
|
||||
`AudioSynchronizationDecodeTraces`, `ChirpBinEmissionPlan`: cached analyzer
|
||||
and calibration outputs.
|
||||
- `Summary`: short buffer/source summary.
|
||||
- `AddSource`: registers source and ensures descriptor buffer if exposed.
|
||||
- `PollSources`: drains all currently available samples from all sources into
|
||||
|
|
@ -681,10 +688,6 @@ Blocks:
|
|||
- `AnalyzeAudioSynchronization`: full analyzer pass over audio buffers.
|
||||
- `AnalyzeAudioSynchronizationStep`: bounded rotating analyzer pass, so one
|
||||
candidate can be analyzed per tick instead of torching a frame.
|
||||
- `AnalyzeComplexContourSynchronizationStep`: heavy matched-filter contour pass
|
||||
for selected Float32 candidates. Live app and Well cadence paths suppress this
|
||||
inline by default; worker/job ownership or the diagnostic
|
||||
`MIMIR_COMPLEX_CONTOUR_RUNTIME_INLINE=1` override must own execution.
|
||||
- `TryLoadCalibrationModel`: loads persisted chirp-bin response model.
|
||||
- `StoreAudioSynchronizationReports`: updates report cache and smoothed state,
|
||||
removing contradictory passive-negative-lag states.
|
||||
|
|
@ -692,10 +695,8 @@ Blocks:
|
|||
|
||||
Invariant:
|
||||
|
||||
- The hub owns cached runtime belief. Ordinary analyzer steps may run as bounded
|
||||
cadence work; complex-contour matched filtering is configured capability until
|
||||
a worker/job path commits reports back. The hub decides what current belief
|
||||
the rest of the app can read.
|
||||
- The hub owns cached runtime belief. The analyzer is a worker; the hub decides
|
||||
what current belief the rest of the app can read.
|
||||
|
||||
### `MimirAudioSynchronizationReport.cs`
|
||||
|
||||
|
|
@ -911,13 +912,9 @@ Blocks:
|
|||
|
||||
Invariant:
|
||||
|
||||
- The live contour lane only executes heavy analysis when a worker/job owns the
|
||||
pass or `MIMIR_COMPLEX_CONTOUR_RUNTIME_INLINE=1` is set for diagnostics.
|
||||
`enableComplexContourRuntime` means the capability/profile/model/report lane is
|
||||
configured; it does not give render threads or Well cadence authority to run
|
||||
matched filtering. When executed, the runtime emitter and contour analyzer
|
||||
share the same contestant profile instead of learning one song and playing
|
||||
another.
|
||||
- The live contour lane only runs when explicitly enabled. When enabled, the
|
||||
runtime emitter and contour analyzer share the same contestant profile instead
|
||||
of learning one song and playing another.
|
||||
|
||||
### `MimirComplexContourChannelModelDocument.cs`
|
||||
|
||||
|
|
|
|||
|
|
@ -1,190 +0,0 @@
|
|||
# Eve Dashboard
|
||||
|
||||
## Objective
|
||||
|
||||
Use EveCanvas as a native operator dashboard substrate: Mimir, StreamPixels, and
|
||||
other local/Yggdrasil-routed apps can publish retained dashboard trees while Eve
|
||||
renders the control surface locally and sends edit intents back. Eve does not
|
||||
own final composition or run remote UI code.
|
||||
|
||||
## Authority Map
|
||||
|
||||
- Owner: each dashboard provider owns its accepted state and command handling.
|
||||
The Starfire broker owns provider discovery/routing. EveCanvas owns native
|
||||
rendering, hit-testing, and touch affordances.
|
||||
- Inputs: Eve sends `select`, `move`, `scale`, `rotate`, `toggle-visibility`,
|
||||
`reset-transform`, and `open-provider` commands over `/eve/deck`.
|
||||
- Outputs: providers send `dashboard-state` snapshots with provider id, title,
|
||||
nodes, health, visibility, dimensions, transform, selected node, and LUT
|
||||
preset. Provider catalogs are exposed at `/eve/deck/providers`.
|
||||
- CultUI-shaped surface: providers may also send `surface`, a retained
|
||||
composition tree for Eve clients. `nodes` remains the semantic compatibility
|
||||
projection; `surface.root` owns composed layout primitives such as workspaces,
|
||||
rails, panes, cards, avatars, text, metrics, lists, and command ids.
|
||||
- Derived state: Eve's UIKit panels are local render projections of Mimir
|
||||
state/provider state.
|
||||
- Forbidden writers: Eve gestures do not mutate program layout directly; they
|
||||
request changes and then re-render the accepted snapshot. Remote apps do not
|
||||
ship arbitrary JavaScript or native code into Eve.
|
||||
- Cut line: no camera-stream fanout to Eve for the dashboard. Source previews
|
||||
may be added later as thumbnails/proxies, but dashboard control must not
|
||||
require Starfire to render every camera feed for the iPad.
|
||||
- CultMesh line: dashboard manifests and states now have typed
|
||||
`mimir.eve_dashboard_manifest.v1` and `mimir.eve_dashboard_state.v1`
|
||||
document shapes. Semantic provider identities use
|
||||
`asgard.starfire.mimir/...`; the broker publishes dashboard state through
|
||||
CultNet reliable UDP for native subscribers. WebSocket endpoints remain
|
||||
renderer compatibility bridges.
|
||||
|
||||
## Runtime
|
||||
|
||||
Start the dashboard server:
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.EveDashboard\Mimir.EveDashboard.csproj -- --port 8795
|
||||
```
|
||||
|
||||
Canonical surface addresses:
|
||||
|
||||
```text
|
||||
asgard.starfire.mimir/eve/tui
|
||||
asgard.starfire.mimir/eve/gui
|
||||
```
|
||||
|
||||
Native CultNet route for typed dashboard state:
|
||||
|
||||
```text
|
||||
cultnet://192.168.1.66:3075
|
||||
schema: mimir.eve_dashboard_state.v1
|
||||
record: eve.dashboard.broker
|
||||
```
|
||||
|
||||
EveCanvas currently connects through this bridge route:
|
||||
|
||||
```text
|
||||
ws://192.168.1.66:8795/eve/deck
|
||||
```
|
||||
|
||||
Periwinkle's Android Eve client currently uses this bridge route:
|
||||
|
||||
```text
|
||||
ws://192.168.1.66:8795/eve/deck/cultmesh
|
||||
```
|
||||
|
||||
The compatibility endpoint remains:
|
||||
|
||||
```text
|
||||
ws://192.168.1.66:8795/eve/dashboard
|
||||
```
|
||||
|
||||
Broker inspection endpoints:
|
||||
|
||||
```text
|
||||
http://192.168.1.66:8795/health
|
||||
http://192.168.1.66:8795/eve/deck/manifest
|
||||
http://192.168.1.66:8795/eve/deck/providers
|
||||
ws://192.168.1.66:8795/eve/deck/{providerId}
|
||||
```
|
||||
|
||||
Extra providers can be registered at launch:
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.EveDashboard\Mimir.EveDashboard.csproj -- `
|
||||
--port 8795 `
|
||||
--voidbot-swarm-state "E:\Projects\VoidBot\.voidbot\status\swarm-state.json" `
|
||||
--provider "app.id|App Title|asgard.starfire.app/eve/tui"
|
||||
```
|
||||
|
||||
If a provider still needs a WebSocket bridge, list it in the provider
|
||||
advertisement or supervisor manifest as a route, not as the provider address.
|
||||
|
||||
## Current State
|
||||
|
||||
`Mimir.EveDashboard` now runs as a small native dashboard broker. It includes:
|
||||
|
||||
- `eve.dashboard.broker`: the switchboard rendered as provider cards.
|
||||
- `mimir.live.stats`: compact live telemetry surface for RMS bars, sync
|
||||
confidence, runtime buffers, actuator commands, and Periwinkle/EVE device
|
||||
observation streams. It recomputes state from Mimir's telemetry and
|
||||
observation ledgers and is broadcast once per second to connected clients.
|
||||
- `mimir.stream.layout`: the existing Mimir source-layout fixture provider.
|
||||
- `voidbot.swarm`: the native VoidBot tab. It reads the existing
|
||||
`swarm-state.json` projection from VoidBot, publishes avatar image URLs on
|
||||
identity nodes, and lets Eve render the CTB rail, selected Face status panel,
|
||||
state tree, and detail pane as native UIKit. It is read-only in this cut;
|
||||
scheduler mutations still belong to VoidBot's heartbeat controls.
|
||||
- `yggdrasil.streampixels.edge`: the first Yggdrasil/StreamPixels service
|
||||
dashboard placeholder for the TCP/SSH-routed live edge.
|
||||
|
||||
The stream-layout provider still serves fixture scene nodes for the first native
|
||||
control proof. It accepts transform/visibility commands, mutates the provider
|
||||
state, increments the state version, and broadcasts the new snapshot. The next
|
||||
cut is replacing fixture state with live `MimirPresentationControlState` and
|
||||
`MimirSceneEditorState` snapshots.
|
||||
|
||||
The live-stats provider is read-only. Its authority is the runtime telemetry
|
||||
surface: `mimir-spectrum`, `mimir-sync-state`, `mimir-sync-report`,
|
||||
`mimir-video-buffer`, `mimir-audio-actuator-command`, and the normalized
|
||||
CultMesh observation ledger from `Mimir.EveSensorReceiver`. If a signal is
|
||||
absent, the surface says so; it does not fabricate confidence.
|
||||
|
||||
## Verification
|
||||
|
||||
The command smoke connected to `/eve/dashboard`, received state version `1`,
|
||||
sent:
|
||||
|
||||
```json
|
||||
{"type":"move","nodeId":"eve-camera","x":0.25,"y":-0.25}
|
||||
```
|
||||
|
||||
and received state version `2` with `eve-camera` moved to that transform. During
|
||||
the same smoke, EveCanvas connected from `192.168.1.72` and remained running.
|
||||
|
||||
The first device screenshot proved the dashboard rendered on Eve, but also
|
||||
showed the legacy debug overlay occluding the hierarchy panel. EveCanvas hides
|
||||
that overlay while dashboard mode is active; dashboard status remains in the
|
||||
native dashboard footer.
|
||||
|
||||
The broker smoke on 2026-05-31 verified:
|
||||
|
||||
- `/health` reports active provider, client count, transport, and typed
|
||||
CultMesh dashboard document name.
|
||||
- `/eve/deck/providers` returns the built-in provider manifests.
|
||||
- WebSocket `/eve/deck` sends the switchboard snapshot.
|
||||
- WebSocket `/eve/deck/cultmesh` sends the same provider state as
|
||||
`mimir.eve_dashboard_state.v1` MessagePack and accepts
|
||||
`mimir.eve_dashboard_command.v1` MessagePack command documents.
|
||||
- `open-provider` switches to `mimir.stream.layout` and broadcasts that
|
||||
provider state.
|
||||
- `open-provider` switches to `voidbot.swarm`, which emits VoidBot's swarm
|
||||
status, CTB rail with avatar URLs, agent cards, and selected Face state
|
||||
detail from the existing VoidBot swarm snapshot.
|
||||
|
||||
## Surface Composition Contract
|
||||
|
||||
`dashboard-state.surface` is the new distributed UI layer. It is intentionally
|
||||
closer to CultUI than to the old flat dashboard node list:
|
||||
|
||||
- `surface.schema`: versioned wire contract, currently
|
||||
`cultmesh.eve_surface.v0`.
|
||||
- `surface.root`: retained element tree. Elements have `kind`, optional `role`,
|
||||
text, asset refs/URIs, provider command ids, layout hints, style variants,
|
||||
metrics, and children.
|
||||
- `surface.assets`: cacheable media references such as avatar images.
|
||||
- `bindNodeId`: optional bridge back to the compatibility `nodes` projection.
|
||||
- `commandId`: provider-owned intent id. Eve clients invoke commands; providers
|
||||
accept or reject and publish the next snapshot.
|
||||
- `binding`: optional typed CultCache/CultMesh field binding. It names
|
||||
`documentSchema`, `documentId`, field `path`, `valueKind`, `access`,
|
||||
`authority`, and optional `commandId`. This is the Verse-backed field
|
||||
contract; native callbacks and DOM handlers are only lowering actuators.
|
||||
|
||||
The old `nodes` array is no longer expected to carry complete UI composition.
|
||||
It is a compatibility/debug/data surface. Clients that understand `surface`
|
||||
should render the composed tree first and use `nodes` only for selection,
|
||||
legacy node binding, and fallback. `bindNodeId` is compatibility glue, not the
|
||||
portable state owner.
|
||||
|
||||
The binary CultMesh state document mirrors this ownership: field 7 carries the
|
||||
optional `surface` tree. Older clients can stop at the node projection; current
|
||||
Eve clients should prefer `surface.root` whenever it is present.
|
||||
|
|
@ -1,84 +0,0 @@
|
|||
# EVE Program Output
|
||||
|
||||
## Authority Map
|
||||
|
||||
- Owner: Fensalir owns the visible D3D12 program pixels.
|
||||
- Inputs: Fensalir reads the current `AquariumFrame`, render plan, GPU resources,
|
||||
and overlay state.
|
||||
- Outputs: a named shared D3D12 `B8G8R8A8_UNorm` texture containing the completed
|
||||
backbuffer for the current frame.
|
||||
- Derived state: Mimir publication profiles describe the stream contract; they
|
||||
do not render pixels. EVE receives decoded pixels and composites them.
|
||||
- Forbidden writers: WebKit, DOM layout, browser dashboards, OBS capture, and
|
||||
desktop/window capture do not own this path.
|
||||
- Shared paths: local display and EVE streaming both use the same completed
|
||||
Fensalir backbuffer copy.
|
||||
- Deletion line: diagnostic Sunshine/Moonlight or desktop-duplication probes may
|
||||
prove network/decode budget, but they are not production authority.
|
||||
|
||||
## Current Mechanism
|
||||
|
||||
Set these before launching the Mimir app:
|
||||
|
||||
```powershell
|
||||
$env:FENSALIR_PROGRAM_OUTPUT_D3D12 = "1"
|
||||
$env:FENSALIR_PROGRAM_OUTPUT_NAME = "Global\MimirFensalirProgramTexture"
|
||||
$env:FENSALIR_PROGRAM_OUTPUT_FENCE_NAME = "Global\MimirFensalirProgramFence"
|
||||
$env:FENSALIR_PROGRAM_OUTPUT_RING_COUNT = "1"
|
||||
dotnet run --project .\src\Mimir.App\Mimir.App.csproj
|
||||
```
|
||||
|
||||
Fensalir creates a shareable D3D12 texture with that name, publishes a named
|
||||
program-output fence, and copies the finished backbuffer into the texture during
|
||||
the present pass. `Mimir.EveRelay` opens that named texture, reads the completed
|
||||
program frame, feeds a low-latency H.264 encoder, and sends complete Annex-B
|
||||
access units to EveCanvas over the existing `/stream` WebSocket. EveCanvas
|
||||
treats JPEG frames as the old VoidBot dashboard fallback only; Mimir dashboard
|
||||
streaming is `h264-annexb` and displays through a native
|
||||
`AVSampleBufferDisplayLayer` so the iPad owns hardware decode/composite.
|
||||
|
||||
The EVE relay currently consumes the single-texture contract. Keep
|
||||
`FENSALIR_PROGRAM_OUTPUT_RING_COUNT=1` for EVE until the relay has the same
|
||||
slot-selection contract as the OBS program texture source.
|
||||
|
||||
`scripts/start-eve-dashboard-stream.ps1` starts the Mimir/Fensalir app with the
|
||||
shared-texture output enabled, starts the H.264 relay, verifies the `eve` SSH
|
||||
target, opens a reverse SSH tunnel from Eve's `127.0.0.1:8792` back to the
|
||||
local relay when direct inbound TCP is blocked, and launches EveCanvas:
|
||||
|
||||
```powershell
|
||||
powershell -ExecutionPolicy Bypass -File .\scripts\start-eve-dashboard-stream.ps1
|
||||
```
|
||||
|
||||
The relay defaults to `h264_nvenc`. Use `-Encoder libx264` only as a diagnostic
|
||||
fallback when no NVENC-capable FFmpeg is installed.
|
||||
|
||||
EveCanvas tries `ws://127.0.0.1:8792/stream` first, then falls back to
|
||||
`ws://192.168.1.66:8792/stream`. The localhost path is not JPEG tunneling; it is
|
||||
the same H.264 Annex-B stream carried through SSH port forwarding so a Windows
|
||||
firewall rule cannot silently turn the dashboard into a wall.
|
||||
|
||||
## Receiver Contract
|
||||
|
||||
EVE owns:
|
||||
|
||||
- network receive;
|
||||
- hardware decode of H.264 access units;
|
||||
- texture upload/import;
|
||||
- final fullscreen composite;
|
||||
- touch/Pencil/sensor uplink later.
|
||||
|
||||
EVE does not own:
|
||||
|
||||
- dashboard DOM;
|
||||
- Chromium/WebKit compatibility;
|
||||
- layout parity;
|
||||
- Mimir timing authority;
|
||||
- OBS composition authority.
|
||||
|
||||
## Smoke
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --eve-program-output-contract-smoke
|
||||
dotnet build .\src\Mimir.EveRelay\Mimir.EveRelay.csproj --no-restore --disable-build-servers
|
||||
```
|
||||
|
|
@ -1,511 +0,0 @@
|
|||
# Fensalir Rendering Rebuild Migration
|
||||
|
||||
## Objective
|
||||
|
||||
Keep Mimir from becoming a second renderer while Fensalir rebuilds itself into
|
||||
the field-evidence machine Mimir needs.
|
||||
|
||||
Mimir's job is to make the room measurable: cameras, microphones, speakers,
|
||||
loopbacks, phones, Raven, and Starfire all feed one bounded temporal window.
|
||||
Fensalir's job is to turn that measured evidence into a coherent visible field
|
||||
and program surface. OBS should receive the final result, not be asked to
|
||||
pretend loose feeds are a world.
|
||||
|
||||
## Current Mechanism
|
||||
|
||||
Mimir currently has the correct broad cut:
|
||||
|
||||
```text
|
||||
drivers / ASIO / network feeds
|
||||
-> Mimir.Runtime rolling buffers
|
||||
-> synchronization, calibration, response models
|
||||
-> Fensalir scene state and debug surfaces
|
||||
-> Fensalir D3D12 presentation / Spout / program output
|
||||
-> OBS
|
||||
```
|
||||
|
||||
The old direct debug-spectrum pressure leak has been cut. Current spectrum
|
||||
debug output is already in the production-shaped bridge:
|
||||
|
||||
```text
|
||||
rolling buffer slice + source identity + calibration + surface intent
|
||||
-> Fensalir field claim
|
||||
-> TubeField candidate producer, if that is the selected claim family
|
||||
-> selected lowering under budget
|
||||
-> evidence lanes and temporal guide
|
||||
-> presentation
|
||||
```
|
||||
|
||||
Mimir must not send "draw these lines" as a production order. It should say
|
||||
"this rolling buffer slice is a spectrum field with stable axes, identities,
|
||||
support, and material intent." Fensalir then chooses direct tube SDF, mesh,
|
||||
field splats, or another lowering.
|
||||
|
||||
The shared Fensalir spatiotemporal reservoir remains the rendering organ.
|
||||
TubeField is only the current claim/candidate producer for rolling-buffer
|
||||
spectrum surfaces. It may own tube-specific resource binding, Catmull-Rom
|
||||
evaluation, SDF/proxy geometry, and material sampling, but it must not become a
|
||||
private renderer or final composition authority.
|
||||
|
||||
This is an intentional divergence from pure ReSTIR. Fensalir is not only
|
||||
resampling lighting after a known primary surface; it is often sampling the
|
||||
visible field candidate itself. TubeField therefore has to generate local
|
||||
primary tube-visibility candidates before reservoir reuse. The shared reservoir
|
||||
owns the temporal-antialiasing-shaped decision about which field candidate persists into
|
||||
presentation. There is no separate TAA owner in the target architecture:
|
||||
reservoir resolve reconstructs smooth presentation directly from selected field
|
||||
evidence instead of repairing unstable pixels after the fact.
|
||||
|
||||
## Invariants
|
||||
|
||||
- Mimir owns raw retention. Fensalir may snapshot or import the five-second
|
||||
window, but it does not silently become history owner.
|
||||
- Mimir owns physical provenance: device ids, sample clocks, calibration ids,
|
||||
path response, confidence, network arrival, and decoded timing evidence.
|
||||
- Fensalir owns dense visual fusion, temporal field evidence, selected cuts,
|
||||
residency, D3D12 resources, debug UI, and program video.
|
||||
- Faust/native DSP owns hot audio movement, resampling, alignment, separation,
|
||||
spatialization, and stems.
|
||||
- OBS owns broadcast composition only.
|
||||
- Render packets are cached lowerings. They are not scene truth.
|
||||
- Reservoir resolve owns temporal antialiasing and reconstruction. TAA is not a
|
||||
separate source-identity or presentation-history authority.
|
||||
- Networked phones and Raven may publish local observations and decoded timing
|
||||
state. They do not become independent clock authorities.
|
||||
|
||||
## Authority Map
|
||||
|
||||
Owner:
|
||||
|
||||
- `Mimir.Runtime` owns stream identity, rolling buffers, synchronization state,
|
||||
calibration loading, and observation publication.
|
||||
- Fensalir's future `FieldEvidenceMachine` owns field claims after Mimir
|
||||
publishes observations and intent.
|
||||
|
||||
Inputs:
|
||||
|
||||
- local direct-driver video frames;
|
||||
- local ASIO loopback and mic blocks;
|
||||
- Raven/phone/network observations;
|
||||
- calibration receipts, response surfaces, codebooks, path ids, sensor poses;
|
||||
- user/app policy for what should enter the live field.
|
||||
|
||||
Outputs:
|
||||
|
||||
- typed observation surfaces for Fensalir;
|
||||
- timing, response, and calibration constraints;
|
||||
- audio actuator control for Faust/native DSP;
|
||||
- health/telemetry state for UI and logs;
|
||||
- final program output from Fensalir and stems from Faust/native DSP.
|
||||
|
||||
Derived state:
|
||||
|
||||
- spectrum lines, debug views, screenshots, OBS bridge endpoints, FFmpeg/SRT
|
||||
helpers, and Eve dashboard feeds are views or diagnostics.
|
||||
- The old direct spline-spectrum dashboard, direct GPU sensor frame bridge, and
|
||||
direct acoustic field frame proof are silent from Mimir's active bridge path.
|
||||
FieldEvidence owns spectrum, camera observation publication, and calibration
|
||||
constraints; screenshots and logs remain diagnostics.
|
||||
|
||||
Forbidden writers:
|
||||
|
||||
- any Mimir-side D3D12 render graph;
|
||||
- app-specific stable Gaussian/evidence cache;
|
||||
- JSON/base64 hot path for live media;
|
||||
- OBS raw source timing as synchronization authority;
|
||||
- per-debug-view geometry deciding production truth.
|
||||
|
||||
Shared paths:
|
||||
|
||||
- live cameras, audio timing anchors, acoustic response surfaces, spectrum
|
||||
diagnostics, calibration events, and future volumetric room claims all publish
|
||||
typed observations/constraints into the same Mimir-to-Fensalir bridge.
|
||||
|
||||
Deletion line:
|
||||
|
||||
- If a Mimir feature wants visual output, first define the observation, claim,
|
||||
or surface intent. A direct draw is allowed only when it is named as fallback
|
||||
or diagnostic and cannot override field evidence.
|
||||
|
||||
## Target Mimir -> Fensalir Contract
|
||||
|
||||
### 1. Raw Window
|
||||
|
||||
Mimir keeps one retention authority:
|
||||
|
||||
```text
|
||||
RollingStreamWindow {
|
||||
windowId
|
||||
duration = 5 seconds by default
|
||||
streamId
|
||||
sourceKind
|
||||
sampleDescriptors
|
||||
nativeHandle or payload view
|
||||
deviceTimestamp
|
||||
canonicalTimestampEstimate
|
||||
sequenceId
|
||||
status
|
||||
}
|
||||
```
|
||||
|
||||
Typed views may index this window. They do not own retention.
|
||||
|
||||
### 2. Observation Surface
|
||||
|
||||
Mimir publishes physical evidence, not render commands:
|
||||
|
||||
```text
|
||||
MimirObservation {
|
||||
observationKey
|
||||
streamId
|
||||
sensorId
|
||||
calibrationId
|
||||
modality: camera | audio | network | timing | response
|
||||
observedTime
|
||||
canonicalTimeEstimate
|
||||
uncertainty
|
||||
payloadHandle
|
||||
provenance
|
||||
confidence
|
||||
}
|
||||
```
|
||||
|
||||
The payload handle is not authority. It must resolve through a declared
|
||||
resource before shader lowering can consume it.
|
||||
|
||||
### 2.5. Resource Declaration
|
||||
|
||||
Live payloads have an explicit Fensalir resource contract:
|
||||
|
||||
```text
|
||||
FensalirResource {
|
||||
resourceKey
|
||||
kind: structured-buffer | texture2d | rolling-texture | mesh | surface-page | volume
|
||||
residency: gpu-resident | shared-gpu
|
||||
shaderAccess: SRV | UAV | vertex-buffer | index-buffer | indirect-args
|
||||
format
|
||||
dimensions / count / stride
|
||||
nativeHandle and nativeHandleKind
|
||||
valid time range
|
||||
version / sequence
|
||||
}
|
||||
```
|
||||
|
||||
Mimir may mint stable resource keys from native/GPU handles, but Fensalir owns
|
||||
resolution. A DSL packet may reference `mimir:resource:*`; it may not treat a
|
||||
bare string as payload truth. In the same runtime, a Mimir-declared buffer is a
|
||||
Fensalir resource slot once the declaration is accepted; native/shared handle
|
||||
metadata is an import edge, not a separate payload authority. Rendering-relevant
|
||||
buffers move to GPU residency as early as possible and stay there; Fensalir
|
||||
passes resource handles/SRVs/UAVs between compute and render lowerings instead
|
||||
of copying samples back to CPU. If the resource contract is absent, expired,
|
||||
duplicated, CPU-only, or incompatible with the selected backend, lowering must
|
||||
defer instead of rendering a lie with nice lighting.
|
||||
|
||||
Current proof surface:
|
||||
|
||||
- `Mimir.BufferSmoke --fensalir-field-evidence-smoke` maps a Mimir rolling
|
||||
window/observation/spectrum intent into one declared resource, three claims,
|
||||
one planned resource-backed `TubeField` packet, and two deferred non-backend
|
||||
claims.
|
||||
- `Mimir.BufferSmoke --fensalir-field-dsl-resource-smoke` uses Fensalir's field
|
||||
evidence DSL to bind the declared resource directly and produce one planned
|
||||
`TubeField` packet with no deferred requests.
|
||||
- `Mimir.BufferSmoke --fensalir-camera-observation-smoke` maps one
|
||||
GPU-backed video frame and one metadata-only cadence witness into field
|
||||
evidence. Only the shared D3D12 video handle becomes a declared `Texture2D`
|
||||
resource and camera surface intent; the metadata-only frame remains
|
||||
observation evidence with no render payload.
|
||||
|
||||
Fensalir now owns the first D3D12 resource resolver cut below this contract:
|
||||
shared structured/curve buffers import/alias GPU-resident resources by handle,
|
||||
while Fensalir-created resources allocate GPU slots only when Fensalir is the
|
||||
producer. The DSL can describe a 2D rolling float buffer as Catmull-Rom XY
|
||||
tubes with modulo column addressing, amplitude power/normalization, radius,
|
||||
ramp texture path, and emission scale. TubeField now consumes those resources
|
||||
through a GPU compute/render path, local Texture2D resources can bind as ramps,
|
||||
SurfacePage and VolumeTexture declarations resolve into shader-readable GPU
|
||||
textures, and Mesh declarations package `Mesh.Vertices`/`Mesh.Indices` GPU
|
||||
buffer shape under one resource key. The evidence DSL can now plan generic
|
||||
resource-backed claims over those declarations. Current blocker for the
|
||||
remaining visual surface is not resource ownership; it is the selected render
|
||||
lowerings that interpret mesh/page/volume resources as geometry, height/SDF/
|
||||
material pages, or density/extinction/SDF3D domains. Mesh layout authority is
|
||||
split by source: imported/user meshes use the standard `PositionNormalUvColor`
|
||||
layout, while generated meshes can be `PipelinePrivate` and leave byte
|
||||
semantics to their selected lowering. TubeField now uses that generated-mesh
|
||||
lane explicitly: compute emits private geometry/indirect buffers and render
|
||||
binds them as a pipeline-private generated mesh before applying TubeField
|
||||
material state. The generated-mesh DrawIndexed indirect command signature is
|
||||
shared ABI; TubeField only owns the argument buffer it emits. TubeField
|
||||
expansion is now gated by selected backend packets, so explicit lowering
|
||||
metadata cannot bypass validation/planning authority. Validation also rejects
|
||||
TubeSpline metadata whose claim is not Tube-encoded or whose resource differs
|
||||
from the claim payload. Mimir's typed surface-intent lowering now emits the
|
||||
matching TubeSpline metadata for audio spectrum/waveform Tube claims. That is
|
||||
engine ownership, not a reason for Mimir to create a parallel renderer.
|
||||
Mimir also uploads normalized spectral frames as Float32 data into a
|
||||
Fensalir-owned structured buffer resource, so the TubeField path consumes live
|
||||
GPU buffer contents rather than an empty declaration. The runtime-level receipt
|
||||
is `Mimir.BufferSmoke --mimir-spectrum-upload-smoke`: it boots Mimir's frame
|
||||
path, advances spectrum analysis, and verifies one newest-slot Float32 upload,
|
||||
planned TubeField packets, a resource-bound local blackbody ramp Texture2D, and
|
||||
no live legacy spline/buffer-field dashboard input. The runtime resource is no
|
||||
longer a latest-frame row dump: it is a rolling column matrix where physical
|
||||
columns are flattened `(history age, source lane)` pairs under fixed history
|
||||
and source-lane capacities. Mimir emits one TubeField claim per active source
|
||||
lane; lowerings use the fixed lane capacity as `ColumnStride`,
|
||||
`ColumnStep.z=0.1`, and `RollingOffset` so each source renders its own age
|
||||
trail from newest to older samples without reshuffling the GPU buffer. New
|
||||
frames upload only the newest ring slot with an explicit element offset.
|
||||
Fensalir owns structured-buffer slot validity and clamps TubeField dispatch so
|
||||
invalid older slots do not reach shader sampling after allocation, reset, or
|
||||
partial update. TubeField metadata now carries stable claim-derived field ids
|
||||
inside the TubeField family range, so temporal/reprojection/reservoir consumers
|
||||
can distinguish source-lane claims. TubeField render now binds the engine
|
||||
blue-noise texture and jitters the tube-local SDF sample inside the proxy
|
||||
fragment shader before writing coverage, normals, and reservoir guide data; the
|
||||
same pass also uses the scene depth buffer so opaque generated field geometry is
|
||||
resolved by nearest surface rather than submission order. The current generated
|
||||
proxy visual still exposes segment structure and is not yet the final smooth
|
||||
tube marcher. Mimir exposes
|
||||
`MIMIR_SPECTRUM_SOURCE_LANES` as the source-lane capacity lever and
|
||||
`MIMIR_SPECTRUM_TUBE_SUBDIVISIONS` as the geometry cost lever for responding to
|
||||
Fensalir's requested/dispatched/truncated/invalid TubeField budget report.
|
||||
Camera descriptors now follow the same evidence/resource boundary:
|
||||
`MimirFensalirFieldLowering.BuildCameraObservationFrame` lowers latest video
|
||||
buffers into observation claims and declares GPU-backed frames as shared
|
||||
`Texture2D` resources. Metadata-only/process-cadence frames do not create
|
||||
surface intents, so a missing payload cannot become a fake `latest-window`
|
||||
render request. Fensalir can now resolve those shared `Texture2D` declarations
|
||||
by native D3D12 handle and accepts Mimir video format names such as `Bgra8`,
|
||||
`Gray8`, `Rg8`, `Nv12`, and `LeapStereoIr`. Camera image claims currently
|
||||
defer because Fensalir has not selected a visual-fusion/render lowering for
|
||||
camera textures yet.
|
||||
|
||||
### 3. Calibration Constraint
|
||||
|
||||
Calibration and sync become explicit constraints:
|
||||
|
||||
```text
|
||||
MimirCalibrationConstraint {
|
||||
constraintKey
|
||||
pathId
|
||||
sourceId
|
||||
receiverId
|
||||
evidenceKind: loopback | bioacoustic | passive | visual | network
|
||||
delayEstimate
|
||||
delayUncertainty
|
||||
phaseOrGroupDelay
|
||||
frequencyResponse
|
||||
usableBandMask
|
||||
confidence
|
||||
}
|
||||
```
|
||||
|
||||
### 4. Surface Intent
|
||||
|
||||
Debug and production visualization share intent:
|
||||
|
||||
```text
|
||||
MimirSurfaceIntent {
|
||||
intentKey
|
||||
sourceObservationKeys
|
||||
domain
|
||||
axes
|
||||
supportPolicy
|
||||
materialGraph
|
||||
updateBudget
|
||||
purpose: debug | calibration | production
|
||||
}
|
||||
```
|
||||
|
||||
The spectrum dashboard becomes one instance of this: a rolling audio spectrum
|
||||
field whose frequency axis maps to X, amplitude to Y, age to Z, and stream
|
||||
identity to stacked lanes.
|
||||
|
||||
## Fresh Dataflow
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A["direct camera drivers"] --> B["Mimir rolling window"]
|
||||
C["ASIO loopback + mics"] --> B
|
||||
D["Raven / phones / network"] --> B
|
||||
B --> E["sync + calibration kernel"]
|
||||
E --> F["typed observations"]
|
||||
E --> G["calibration constraints"]
|
||||
B --> H["surface intent"]
|
||||
F --> I["Fensalir field evidence machine"]
|
||||
G --> I
|
||||
H --> I
|
||||
I --> J["field claims + selected lowerings"]
|
||||
J --> K["program video + debug UI"]
|
||||
E --> L["Faust/native DSP controls"]
|
||||
L --> M["program stems + spatial bed"]
|
||||
K --> N["OBS"]
|
||||
M --> N
|
||||
```
|
||||
|
||||
## Migration Plan
|
||||
|
||||
### Phase 0: Silence Legacy Debug Surfaces
|
||||
|
||||
The direct `AquariumSplineFrame` spectrum dashboard and point-splat spectrum
|
||||
path are no longer scene writers. Live spectrum rendering flows through
|
||||
FieldEvidence TubeField claims and generated mesh expansion.
|
||||
|
||||
### Phase 1: Add Bridge DTOs In Mimir.Runtime
|
||||
|
||||
Create pure mapping types for:
|
||||
|
||||
- rolling stream windows;
|
||||
- resource declarations for live GPU/native payload views;
|
||||
- observations;
|
||||
- calibration constraints;
|
||||
- surface intent.
|
||||
|
||||
The bridge must not read devices, render, allocate per sample, or own timers.
|
||||
It maps existing runtime state into Fensalir-facing contracts.
|
||||
|
||||
Acceptance:
|
||||
|
||||
- one synthetic buffer can produce observations and surface intent in a unit
|
||||
test;
|
||||
- claims that reference live payloads also declare a typed Fensalir resource;
|
||||
- the mapping can run repeatedly without growing allocations;
|
||||
- stream/source ids remain stable across frames.
|
||||
|
||||
### Phase 2: Move Spectrum Dashboard To Surface Intent
|
||||
|
||||
Represent the current spectrum view as a `MimirSurfaceIntent`:
|
||||
|
||||
- stable source lanes;
|
||||
- stable frequency bins;
|
||||
- monotonic history sequence ids;
|
||||
- Z as continuous sample age;
|
||||
- tube/material intent from amplitude/confidence.
|
||||
|
||||
Fensalir may still lower it to direct spline tubes at first. The ownership
|
||||
changes before the backend does.
|
||||
|
||||
Acceptance:
|
||||
|
||||
- app code no longer chooses point splats, direct tubes, or mesh ribbons;
|
||||
- row/band/history identity is the same whether the engine lowers direct tubes
|
||||
or another representation.
|
||||
|
||||
### Phase 3: Camera Observation Bridge
|
||||
|
||||
Lower native camera descriptors into observation surfaces:
|
||||
|
||||
- source id and calibration id;
|
||||
- dimensions, format, stride;
|
||||
- native CPU/GPU handle;
|
||||
- device timestamp and canonical estimate;
|
||||
- lens model and confidence/status.
|
||||
|
||||
Acceptance:
|
||||
|
||||
- implemented bridge proof: GPU-backed video descriptors become declared
|
||||
shared `Texture2D` resources plus camera observation/surface claims;
|
||||
- metadata-only cadence descriptors remain observation evidence and do not
|
||||
become render payloads;
|
||||
- no selected camera-image lowering exists yet, so Fensalir defers those claims
|
||||
until visual fusion/feature extraction owns them.
|
||||
|
||||
### Phase 4: Audio Field Observation Bridge
|
||||
|
||||
Lower sync and calibration state into audio field constraints:
|
||||
|
||||
- ASIO loopback clock fit;
|
||||
- mic delay and SRO state;
|
||||
- per-path frequency response and group delay;
|
||||
- bioacoustic anchor clusters;
|
||||
- passive correlation confidence;
|
||||
- Raven/phone decoded timing observations.
|
||||
|
||||
Acceptance:
|
||||
|
||||
- implemented bridge proof: sync states lower into FieldEvidence calibration
|
||||
constraints instead of direct `AquariumAcousticFieldFrame` packets;
|
||||
- Fensalir can inspect acoustic confidence and constraints without receiving
|
||||
raw PCM as production truth;
|
||||
- selected bridge lowering now exists for audio-path `Phase` and `Confidence`
|
||||
claims: Fensalir plans those constraints as `DebugOverlay` backend packets so
|
||||
calibration timing/path evidence is visible through the FieldEvidence
|
||||
authority instead of a parallel acoustic frame;
|
||||
- acoustic source candidates, confidence volumes, and cross-modal fusion remain
|
||||
Phase 5 work;
|
||||
- Faust/native DSP remains the actuator for sample movement.
|
||||
|
||||
### Phase 5: Sensor Fusion Claims
|
||||
|
||||
Let Fensalir fuse Mimir observations into field claims:
|
||||
|
||||
- sparse feature tracks first;
|
||||
- calibrated camera rays and marker/feature candidates;
|
||||
- acoustic source candidates and confidence volumes;
|
||||
- surface/material claims only after cross-modal evidence earns them.
|
||||
|
||||
Acceptance:
|
||||
|
||||
- implemented synthetic multi-camera marker proof: Mimir lowers a deterministic
|
||||
camera marker candidate with calibration/marker identity and a world-space
|
||||
confidence envelope, and Fensalir plans it as a `DebugOverlay` FieldEvidence
|
||||
packet;
|
||||
- implemented synthetic audio source proof: Mimir lowers an SRP/PHAT-style
|
||||
source candidate with calibration/source identity and a world-space confidence
|
||||
envelope, and Fensalir plans it as a `DebugOverlay` FieldEvidence packet;
|
||||
- claims can be inspected by source observation and calibration id.
|
||||
|
||||
### Phase 6: Program Output Receipts
|
||||
|
||||
Field identity now reaches the OBS-facing program boundary:
|
||||
|
||||
- Fensalir publishes the completed program surface through named shared D3D12
|
||||
textures and `Global\MimirFensalirProgramFence`;
|
||||
- `Mimir Program Texture` opens those textures in OBS and performs the explicit
|
||||
D3D12-to-libobs-D3D11 GPU copy without CPU readback or Spout2;
|
||||
- optional texture rings and `Global\MimirObsProgramConsumerFence` prevent blind
|
||||
producer/consumer reuse;
|
||||
- `Mimir Audio Stem` receives separately controllable Faust/native DSP stems
|
||||
from `Local\MimirObsStemBus`;
|
||||
- telemetry and receipts record timing, evidence confidence, dropped/late
|
||||
observations, and current selected cuts.
|
||||
|
||||
Acceptance:
|
||||
|
||||
- OBS sees final program surfaces/stems through the Mimir OBS plugin;
|
||||
- raw feeds remain debug inputs, not synchronization truth;
|
||||
- recorded receipts explain the stem path, installed OBS plugin, live program
|
||||
texture proof, optional ring proof, dedicated producer fence proof, and
|
||||
two-way consumer fence proof.
|
||||
|
||||
## Verification Matrix
|
||||
|
||||
| Check | Expected Result |
|
||||
| --- | --- |
|
||||
| Stable buffer identity | Source lanes do not reorder or phase-rotate by render frame. |
|
||||
| Missing data | No sample means no observation; no zero-fill masquerade. |
|
||||
| Direct debug path | Can be disabled without changing runtime timing state. |
|
||||
| Observation bridge | Produces typed evidence without device reads or rendering. |
|
||||
| Fensalir lowering swap | Direct tube vs future splat/mesh lowering preserves claim ids. |
|
||||
| Reservoir guide | Presentation history follows evidence validity, not producer folklore. |
|
||||
| OBS output | OBS receives final program surfaces, not raw clock chaos. |
|
||||
|
||||
## Research And Maps
|
||||
|
||||
Primary local references:
|
||||
|
||||
- `docs/perfect-machine.md`
|
||||
- `docs/perfect-machine-domain-index.md`
|
||||
- `docs/sensor-fusion-architecture.md`
|
||||
- `research/perfect-machine-study-2026-05-23/fensalir-integration-map.md`
|
||||
- `research/visual-spatial-map/brushstroke-fusion-rendering.md`
|
||||
- Fensalir `docs/rendering-teardown-rebuild-protocol.md`
|
||||
- Fensalir `docs/perfect-machine-architecture.md`
|
||||
- Fensalir `docs/temporal-spatial-evidence-reservoir.md`
|
||||
|
||||
The next implementation pass should cut Phase 1 first. A bridge DTO that owns
|
||||
nothing but mapping is boring in the best possible way: it makes the rest of
|
||||
the machine harder to lie about.
|
||||
|
|
@ -1,115 +0,0 @@
|
|||
# Fensalir Reservoir Daemon
|
||||
|
||||
## Objective
|
||||
|
||||
Move heavy field reconstruction out of the interactive Mimir editor. The editor
|
||||
should present synced video feeds and controls responsively. A separate
|
||||
Fensalir reservoir daemon should drink from the Well, spend whatever CPU/GPU
|
||||
budget is assigned, and publish typed reservoir/program surfaces.
|
||||
|
||||
## Authority Map
|
||||
|
||||
Owner: the Fensalir reservoir daemon owns reservoir work selection, GPU
|
||||
residency, temporal reuse, fusion kernels, worker scheduling, and publication of
|
||||
reservoir-derived program surfaces.
|
||||
|
||||
Inputs: typed Well pages, configured composite state, live timing policy,
|
||||
feature-signal documents, calibration state, Fensalir-owned texture leases, and
|
||||
explicit budget settings.
|
||||
|
||||
Outputs: typed CultMesh/CultCache reservoir status, program texture/fence
|
||||
surfaces, optional diagnostic Eve surfaces, and OBS-facing shared texture or
|
||||
plugin feeds.
|
||||
|
||||
Derived state: Mimir editor telemetry, preview rectangles, Odin/Eve dashboard
|
||||
rows, and OBS plugin status are observers. They do not decide reservoir work.
|
||||
|
||||
Forbidden writers: the interactive editor render loop must not run temporal
|
||||
reservoir history just because it is presenting a backbuffer. Field diagnostics
|
||||
must be opt-in. A camera `FieldEvidenceFrame` claim must not impersonate a flat
|
||||
video compositor.
|
||||
|
||||
Shared paths: direct user placement, configured Well composites, OBS output,
|
||||
daemon diagnostics, and future Eve room lowerings must all reference the same
|
||||
source ids, timing state, and placement/composite document. Manual UI placement
|
||||
and daemon program output may differ only through an explicit selected
|
||||
composite/version.
|
||||
|
||||
Deletion line: old app-hosted reservoir proof visuals remain diagnostic only.
|
||||
They cannot be the default Mimir program output while the current goal is synced
|
||||
video presentation.
|
||||
|
||||
## Current Scar
|
||||
|
||||
On 2026-06-02 the Mimir editor became unresponsive because Fensalir ran
|
||||
reservoir history presentation on a video-only frame. The renderer now gates
|
||||
that pass behind source-derived reservoir-producing input, and the live
|
||||
video-only app log showed `reservoir-update n/a`.
|
||||
|
||||
The remaining gap is real: Fensalir still lacks a flat program-layer video
|
||||
backend that draws Fensalir-owned camera texture leases as 2D synced layers.
|
||||
That backend is separate from the reservoir daemon. Build it as the editor's
|
||||
presentation path; do not smuggle it through field diagnostics.
|
||||
|
||||
## Current Daemon Cut
|
||||
|
||||
`src/Mimir.FensalirDaemon` is the first owner-shaped daemon:
|
||||
|
||||
- eats CultCache through `state/fensalir-daemon.ccmp`, using the canonical
|
||||
MessagePack CultCache backing store;
|
||||
- drinks Well JSONL logs containing `mimir.well_snapshot.v1`,
|
||||
`mimir.well_capture_page.v1`, and `mimir.well_stream_pressure.v1`;
|
||||
- publishes typed `mimir.fensalir_daemon_state.v1` state;
|
||||
- speaks a CultNet-shaped WebSocket Eve provider at `/eve/deck`;
|
||||
- speaks binary CultMesh dashboard documents at `/eve/deck/cultmesh`;
|
||||
- exposes `/health` and `/eve/deck/manifest`;
|
||||
- presents a retained `cultmesh.eve_surface.v0` surface with compact text and
|
||||
metrics that Eve GUI and Odin/Nightwing TUI can lower.
|
||||
|
||||
Start it with:
|
||||
|
||||
```powershell
|
||||
powershell -ExecutionPolicy Bypass -File .\scripts\start-fensalir-daemon.ps1
|
||||
```
|
||||
|
||||
The launcher writes a supervisor manifest under
|
||||
`artifacts/runtime/fensalir-daemon-*` with a semantic provider spec and bridge
|
||||
routes:
|
||||
|
||||
```text
|
||||
mimir-fensalir-daemon|Mimir Fensalir Daemon|asgard.starfire.mimir-fensalir/eve/tui
|
||||
bridge: ws://127.0.0.1:8799/eve/deck
|
||||
bridge: ws://127.0.0.1:8799/eve/deck/cultmesh
|
||||
```
|
||||
|
||||
The current worker mode is `surface-owner-installed`: kernel scheduling,
|
||||
GPU-resident reservoir compute, and actual fused program-surface publication
|
||||
belong behind this daemon next. Do not move those back into the editor.
|
||||
|
||||
## Worker-State Cut
|
||||
|
||||
`Mimir.FensalirDaemon` now also publishes
|
||||
`mimir.fensalir_reservoir_worker_state.v1` beside the daemon summary state.
|
||||
The worker state is the owner surface for reservoir job selection and pressure:
|
||||
|
||||
- capture pages enqueue reservoir jobs only when they contain accepted timing
|
||||
slices;
|
||||
- `mimir.cultmesh_stream_frame.v1` records enqueue fine-grained reservoir jobs
|
||||
from timed stream frames, including paged, inline, native, or shared-handle
|
||||
body transport evidence;
|
||||
- unusable pages are dropped with rejection reasons instead of being hidden by
|
||||
the editor;
|
||||
- queue depth, running jobs, completed jobs, dropped jobs, timing confidence,
|
||||
accepted/rejected slice counts, and estimated GPU queue milliseconds are
|
||||
persisted in CultCache;
|
||||
- `mimir.fensalir_reservoir_pressure.v1` reports Well poll/publish pressure,
|
||||
capture bytes, ready-slice ratio, timing confidence, max presentation
|
||||
distance, GPU queue milliseconds, and drop causes for missing bodies,
|
||||
unavailable fences, or degraded timing;
|
||||
- Eve surfaces bind worker metrics to the worker document, not to ad hoc
|
||||
renderer telemetry.
|
||||
|
||||
The current worker still simulates cost from capture-page shape and configured
|
||||
CPU/GPU share. That is intentional scaffolding: the next cut replaces the cost
|
||||
model with real GPU/Faust/Fensalir kernel dispatch while preserving the same
|
||||
owner and state contract.
|
||||
|
|
@ -48,9 +48,10 @@ Mimir exists to turn cameras, microphones, speakers, timing signals, and
|
|||
distributed capture surfaces into one realtime coherent volumetric field. The
|
||||
field is the product. The bridge scripts are scaffolding and probes.
|
||||
|
||||
The live target is one native five-second rolling reservoir feeding Fensalir GPU
|
||||
fusion and Faust/native DSP, then OBS. Retention has one owner. Typed views are
|
||||
indexes, not private histories.
|
||||
The live target is one native five-second rolling reservoir feeding Mimir-owned
|
||||
program composition, Fensalir GPU fusion, Faust/native DSP, operator
|
||||
surfaces, and a Yggdrasil site publisher. OBS is compatibility only. Retention
|
||||
has one owner. Typed views are indexes, not private histories.
|
||||
|
||||
## Operating Doctrine
|
||||
|
||||
|
|
@ -63,11 +64,14 @@ the synchronized program path is being rebuilt around `LocalcastRuntime`,
|
|||
|
||||
Invariants:
|
||||
|
||||
- OBS owns final broadcast composition.
|
||||
- Fensalir owns dense visual fusion, material/brush/splat rendering, and Spout2
|
||||
publication.
|
||||
- Mimir owns final program composition, source subscription, preview/control
|
||||
state, stats, and publication intent.
|
||||
- Fensalir owns dense visual fusion, material/brush/splat rendering, and local
|
||||
program texture output.
|
||||
- Faust/native DSP owns hot audio alignment, room suppression, voice separation,
|
||||
stems, and spatial bed generation.
|
||||
- The Yggdrasil publisher daemon ships the Mimir program to the site without
|
||||
owning composition.
|
||||
- FFmpeg owns simple capture, encode, and transport in V1 bridge mode.
|
||||
- Mimir/this repo owns configuration, calibration, launch/status, typed
|
||||
contracts, memory, and operational repeatability.
|
||||
|
|
|
|||
103
docs/mimir-program-composition.md
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
# Mimir Program Composition
|
||||
|
||||
## Objective
|
||||
|
||||
Mimir owns the live stream program: selected capture feeds, calibration feeds,
|
||||
scene composition, preview/control surfaces, stats, and publication. OBS is a
|
||||
temporary compatibility sink only.
|
||||
|
||||
## Authority Map
|
||||
|
||||
- Owner: Mimir owns the program scene graph and subscription policy.
|
||||
- Inputs: Muninn-published local and remote monitor, window, camera, audio,
|
||||
loopback, and health streams.
|
||||
- Outputs: a Mimir program video surface, aligned audio stems or final bus,
|
||||
and a Yggdrasil-facing site publication feed.
|
||||
- Derived state: OBS receives a compatibility feed when useful; it does not own
|
||||
scene truth, broadcast truth, preview truth, stats, source selection, or
|
||||
publication.
|
||||
- Forbidden writers: OBS scenes, browser dashboards, site daemons, and Muninn
|
||||
capture hosts must not independently decide the program composition.
|
||||
- Shared paths: direct operator edits, remote commands, preset loads,
|
||||
calibration-driven source selection, and recovery scripts must commit through
|
||||
the same Mimir scene graph command surface.
|
||||
- Deletion line: old OBS-owned composition docs and config names are demoted to
|
||||
diagnostic bridge language. Any OBS adapter delegates to the Mimir program
|
||||
scene instead of carrying its own opinion.
|
||||
|
||||
## Machine Shape
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A["Muninn on Starfire"] --> D["CultMesh stream catalog + media bodies"]
|
||||
B["Muninn on Nightwing"] --> D
|
||||
C["Muninn on Raven"] --> D
|
||||
D --> E["Mimir subscription policy"]
|
||||
E --> F["Mimir rolling buffers"]
|
||||
F --> G["Mimir program scene graph"]
|
||||
G --> H["Fensalir compositor"]
|
||||
F --> I["Faust/native DSP"]
|
||||
H --> J["program video surface"]
|
||||
I --> K["program audio stems/bus"]
|
||||
J --> M["Yggdrasil site publisher daemon"]
|
||||
K --> M
|
||||
M --> N["GameCult site"]
|
||||
J -. "compatibility" .-> O["OBS adapter"]
|
||||
K -. "compatibility" .-> O
|
||||
```
|
||||
|
||||
## Composition Contract
|
||||
|
||||
The scene graph carries layers with:
|
||||
|
||||
- source reference, such as `muninn:starfire:monitor:primary` or
|
||||
`muninn:starfire:window:discord`;
|
||||
- source kind, such as monitor, window, camera, network monitor, or generated
|
||||
Mimir visual field;
|
||||
- visibility, layer order, transform, crop, and optional chroma key;
|
||||
- enough timing identity for Mimir to align media inside the rolling window.
|
||||
|
||||
The current OBS scene can be mirrored as an initial Mimir scene:
|
||||
|
||||
- Starfire monitor capture, cropped at the bottom when needed.
|
||||
- Discord window capture, cropped to chat and keyed with chroma filters.
|
||||
- Raven primary monitor plus Realtek loopback as a subscribed remote Muninn
|
||||
stream.
|
||||
- Mimir/Fensalir program texture as a generated layer when the field renderer is
|
||||
active.
|
||||
|
||||
After import, OBS is not the editor of record. It may display the Mimir program
|
||||
feed, but operator edits happen through Mimir command surfaces.
|
||||
|
||||
## Site Publisher
|
||||
|
||||
The Yggdrasil-facing publisher consumes Mimir program output and publishes it to
|
||||
the GameCult site. It owns transport, encoding/session state, reconnect policy,
|
||||
and public edge health. It does not own composition.
|
||||
|
||||
The publisher's input contract is the same Mimir program output used by local
|
||||
preview and OBS compatibility adapters, so the public stream cannot silently
|
||||
diverge from the operator preview.
|
||||
|
||||
## Runtime Contracts
|
||||
|
||||
The first typed contracts live in `src/Mimir.Runtime/Synchronization`:
|
||||
|
||||
- `mimir.program_scene.v1`
|
||||
- `mimir.program_output.v1`
|
||||
- `mimir.eve_operator_surface.v1`
|
||||
|
||||
The proof path is:
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --perfect-machine-contract-smoke
|
||||
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --perfect-machine-profile-smoke
|
||||
```
|
||||
|
||||
Import the current OBS scene as Mimir-owned typed state:
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --import-obs-program-scene --output state\mimir-program-composition.cc
|
||||
```
|
||||
|
||||
The live imported state file is `state/mimir-program-composition.cc`.
|
||||
107
docs/move-calibration-protocol.md
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
# Move Calibration Protocol
|
||||
|
||||
Mimir owns Move calibration and sensor fusion. Muninn owns source-local facts:
|
||||
camera marker candidates, USB controller IMU/button state, and local Move light
|
||||
commands. Odin discovers and projects the surfaces. Fensalir consumes Mimir's
|
||||
resolved poses after Mimir has earned them.
|
||||
|
||||
## Current Stream Shape
|
||||
|
||||
- Starfire should run Muninn as a local capture/control daemon. Its Move stream
|
||||
address is `muninn:starfire:move-evidence`.
|
||||
- Nightwing runs Muninn for its USB Move and local camera witnesses. Its Move
|
||||
stream address is `muninn:nightwing:move-evidence`.
|
||||
- Mimir on Starfire publishes resolved pose frames at
|
||||
`mimir:starfire:move-controller-poses`.
|
||||
- Quest can participate as an external tracked VR reference over USB:
|
||||
`quest:usb:headset-pose`, `quest:usb:left-controller-pose`, and
|
||||
`quest:usb:right-controller-pose`.
|
||||
- Starfire ASIO/Scarlett loopback can provide cue timing, but it is not a Move
|
||||
pose authority.
|
||||
|
||||
The Move hot path is CultMesh stream frames. On the same host, those frames use
|
||||
shared-memory bytes rings. Across hosts, the remote peer still crosses the
|
||||
Verse/network boundary. The current Raven program media bridge is not same-host
|
||||
zero-copy into Mimir/Fensalir; it sends CultNet/CultMesh media documents and
|
||||
lowers to local UDP for OBS compatibility.
|
||||
|
||||
## Required Capture Phases
|
||||
|
||||
1. `preflight-streams`: hold both Moves still and visible; prove stream freshness,
|
||||
clock skew, and controller ids.
|
||||
2. `dark-stillness`: lights off, both Moves still; estimate gyro bias, gravity,
|
||||
and magnetometer baseline.
|
||||
3. `lit-stillness`: light one Move at a time, then both; map light commands to
|
||||
controllers and measure centroid stability.
|
||||
4. `axis-sweeps`: slow pitch/yaw/roll sweeps; fit gyro scale, accelerometer
|
||||
frame, and optical angular consistency.
|
||||
5. `figure-eight`: broad figure-eight motion; fit magnetometer hard/soft iron
|
||||
and cross-axis coupling.
|
||||
6. `quest-reference`: optional Quest headset/controller reference with the Move
|
||||
between the two Quest controllers in front of the headset.
|
||||
7. `validation-pass`: held-out free motion; promote only if triangulated optical
|
||||
residuals and IMU prediction residuals pass.
|
||||
|
||||
## Quest Access Preflight
|
||||
|
||||
The Quest must be authorized for USB debugging before Starfire Muninn can query
|
||||
it. Muninn owns the access surface:
|
||||
|
||||
```powershell
|
||||
muninn serve --store C:\Meta\Odin\state\muninn.telemetry.cc --host starfire --quest-adb --quest-serial 1WMHHB68PG1515
|
||||
```
|
||||
|
||||
When the Quest is authorized, Muninn publishes `muninn.quest_access.v1` and
|
||||
advertises `muninn:<host>:quest-input`, `muninn:<host>:quest-poses`, and
|
||||
`muninn:<host>:quest-warped-video-input`. Mimir consumes the access/pose
|
||||
surfaces as calibration evidence; it does not own Quest USB access.
|
||||
|
||||
```powershell
|
||||
muninn quest-access-status --store C:\Meta\Odin\state\muninn.telemetry.cc
|
||||
```
|
||||
|
||||
ADB authorization proves USB access, device identity, and basic headset state
|
||||
only. It does not expose Quest headset or controller poses; those still require
|
||||
a Quest/OpenXR witness bridge that publishes pose samples through Muninn. The
|
||||
current calibration protocol can still run without Quest, but Quest poses are
|
||||
the preferred external reference when the headset and both controllers are
|
||||
sitting around the Move.
|
||||
|
||||
## Data Products
|
||||
|
||||
- `mimir.move_fusion_rig_calibration.v1`: camera intrinsics/extrinsics, clock
|
||||
association policy, and gyro unit scale.
|
||||
- `mimir.move_imu_calibration.v1`: per-controller gyro bias/scale,
|
||||
accelerometer gravity frame, magnetometer correction, and noise estimates.
|
||||
- `mimir.move_controller_identity_map.v1`: stable association between Muninn
|
||||
USB ids, light receipts, and optical orb identities.
|
||||
- `mimir.move_calibration_receipt.v1`: sample counts, residuals, rejected
|
||||
phases, and promotion decision.
|
||||
|
||||
## Optical Sphere Fit Authority
|
||||
|
||||
`MimirMoveSphereCalibrationFitter` owns the bounded two-Eye optical fit after a
|
||||
collection session reaches `Fitting`. Eye 0 defines the tracking-space gauge.
|
||||
The fitter jointly estimates both focal lengths and Eye 1's rigid pose from
|
||||
same-frame, same-Move correspondences, using a deterministic 80/20 training and
|
||||
held-out partition. A robust training consensus may reject contaminated marker
|
||||
pairs, but promotion also requires at least 65 percent of the untouched
|
||||
held-out pairs to agree and must pass the session's median and P95 reprojection
|
||||
limits.
|
||||
|
||||
The physical orb radius is an explicit input; it is never guessed by the
|
||||
solver. PSMoveAPI's reported blob radius is a color-segmentation result, not a
|
||||
calibrated sphere silhouette. It may seed or weight depth only after the
|
||||
per-camera radius response has been validated. A fresh PSMoveAPI position age
|
||||
is also not optical confidence and must not be interpreted as one.
|
||||
|
||||
## Preflight Command
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --move-calibration-protocol-smoke --output artifacts\move-calibration\protocol.cc
|
||||
```
|
||||
|
||||
That writes the typed protocol as `mimir.move_calibration_protocol.v1` into a
|
||||
CultCache receipt. The real hardware runner should consume this document plus
|
||||
Muninn's `muninn.quest_access.v1` when Quest is available, then append capture
|
||||
receipts instead of inventing a parallel checklist.
|
||||
|
|
@ -1,321 +0,0 @@
|
|||
# Move Controller Stream
|
||||
|
||||
## Objective
|
||||
|
||||
Ship the first Mimir program surface around one concrete show shape:
|
||||
Kiyo Pro is the RGB program view, PS3 Eyes track multiple PS Move spheres,
|
||||
Raven screen capture enters as a synchronized video source, and Scarlett ASIO
|
||||
provides the two hero mics plus Raven program-loopback timing evidence.
|
||||
|
||||
## Authority Map
|
||||
|
||||
- Owner: `Mimir.Runtime` owns sample ingest, rolling buffers, and typed visual
|
||||
evidence. Fensalir owns overlay presentation and final program pixels.
|
||||
- Inputs: Kiyo Pro YUY2 video, both PS3 Eye Bayer8 feeds, Scarlett ASIO
|
||||
channels `asio-ch0..asio-ch3`, and Raven SRT screen capture on port `5200`.
|
||||
- Outputs: `kiyo-pro-rgb` is the base program view; PS Move controller histories
|
||||
lower as Feature evidence through `MimirMoveControllerTracker`; `raven-display`
|
||||
stays in the `raven-sync` clock domain; ASIO labels identify shotgun,
|
||||
cardioid, and Raven loopback lanes.
|
||||
- Derived state: Move 2D histories are observations, not 6DoF pose. Raven
|
||||
display timing is derived from Scarlett audio evidence, not SRT arrival time.
|
||||
- Forbidden writers: PS3 Eye blob tracks do not own camera calibration; Raven
|
||||
network timestamps do not own global sync; Kiyo Pro does not own high-rate
|
||||
motion truth.
|
||||
- Cut line: this profile is the first streamable overlay product, not the full
|
||||
online calibration solve.
|
||||
|
||||
## Nightwing Eyes/Moves Split
|
||||
|
||||
Nightwing is the right body for the PS3 Eyes and PS Move controllers: it has
|
||||
the spare USB topology, BlueZ/Bluetooth, and the local HID path. That does not
|
||||
make it a camera-placement authority.
|
||||
|
||||
The split is:
|
||||
|
||||
- Nightwing-local authority: read PS3 Eye frames, talk to PS Move HID/Bluetooth,
|
||||
apply LED schedules, preserve device timestamps, and extract compact sphere,
|
||||
marker, and sparse Eye feature observations.
|
||||
- Starfire authority: own the canonical rolling window, Scarlett timing,
|
||||
global residual solve, camera placement, Kiyo/Raven program sync, and OBS
|
||||
program publication.
|
||||
- Fensalir authority: own D3D12 dense feature extraction/fusion, surface claims,
|
||||
splat/reservoir resolution, and program pixels.
|
||||
|
||||
`MimirDistributedWitnessConfigurations.NightwingEyesMoves` records this as a
|
||||
typed witness: it may emit `mimir.move_controller_observation_state`,
|
||||
`mimir.camera_feature_track_state`, and `mimir.visual_marker_state`, but it must
|
||||
not stream raw Eye media as the normal live contract and must not own the
|
||||
canonical clock. Raw frames from Nightwing are diagnostic receipts only.
|
||||
|
||||
The first Nightwing receipt is good enough to build on: both PS3 Eyes enumerate
|
||||
as stock Linux `ov534`/`gspca` V4L2 cameras on `/dev/video2` and `/dev/video3`.
|
||||
Default V4L2 cadence is only 30 fps, but explicitly setting
|
||||
`timeperframe=1/187` before streaming makes both cameras deliver about 187 fps
|
||||
at 320x240 with zero sequence gaps. The witness worker must set frame interval;
|
||||
otherwise Linux will quietly look worse than it is.
|
||||
|
||||
Nightwing also has a typed witness publisher staged as
|
||||
`~/.local/bin/nightwing_typed_witness_publisher.py`. The repo source is
|
||||
`tools/nightwing_typed_witness_publisher.py`. It sends binary MessagePack
|
||||
`mimir.eve_sensor_observation.v1` records to `Mimir.EveSensorReceiver` on
|
||||
`ws://192.168.1.66:8796/eve/periwinkle`; the receiver normalizes them into the
|
||||
Mimir observation ledger, Odin projects them as `observation-stream` nodes, and
|
||||
Nightwing's Eve TUI lowers those nodes in the `CultMesh Witness Streams` panel.
|
||||
The current status records are camera-device receipts for `/dev/video*` and a
|
||||
`nightwing-leap-tracking` placeholder that is intentionally `0` until real Leap
|
||||
tracking owns that signal. Values are `[present, videoIndex, isPs3Eye]` for
|
||||
camera-device receipts.
|
||||
|
||||
`MimirRoomCalibrationLockSolver` is the first room-lock coordinator. It consumes
|
||||
the LED string candidate/residual, the camera-rig pose update frame, Move
|
||||
controller feature-history candidates, and PS3 Eye sparse-feature candidates.
|
||||
It only reports `Locked` when every witness family is present: multiple cameras
|
||||
share enough LED indices, the rig solver produced enough pose updates with low
|
||||
mean ray residual, multiple Move identities have stable history, and multiple
|
||||
Eyes contribute enough stable feature tracks. Missing families are reported by
|
||||
name instead of being papered over. `Mimir.BufferSmoke
|
||||
--room-calibration-lock-smoke` proves a synthetic three-camera LED string, two
|
||||
Moves, and two Eyes lock at confidence `0.778`, while removing Move history
|
||||
correctly drops the result to `InsufficientWitnesses`.
|
||||
|
||||
## Runtime Profile
|
||||
|
||||
For the current stream-proof demo, use the corrected Starfire/Nightwing/Raven
|
||||
profile:
|
||||
|
||||
```powershell
|
||||
E:\Projects\Mimir\scripts\start-stream-proof.ps1
|
||||
```
|
||||
|
||||
This profile keeps Nightwing raw Eye pixels off Starfire. Starfire ingests local
|
||||
Scarlett ASIO, local Leap stereo IR, local Kiyo Pro RGB, Raven screen capture,
|
||||
and Raven Realtek loopback PCM. Nightwing publishes compact Eye/Move
|
||||
observations through `/eve/periwinkle`; those observations are calibration and
|
||||
overlay evidence, not raw media ownership.
|
||||
|
||||
The profile declares:
|
||||
|
||||
- `leap-stereo-ir`: LeapUVC stereo IR / depth root.
|
||||
- `kiyo-pro-rgb`: Kiyo Pro AR program view.
|
||||
- `raven-display`: Raven screen capture from the muxed Raven A/V stream,
|
||||
locally demuxed to SRT port `5210`, in the `raven-sync` clock domain.
|
||||
- `raven-realtk-loopback`: Raven Realtek render loopback from the same muxed
|
||||
Raven A/V stream, locally demuxed as f32 PCM to SRT port `5212`, also in
|
||||
`raven-sync`.
|
||||
- `focusrite-asio`: Starfire hero mics plus local Raven/program loopback lanes
|
||||
on `asio-ch0..asio-ch3`.
|
||||
|
||||
For the narrower Leap/Eyes/Kiyo Pro bring-up path, use:
|
||||
|
||||
```powershell
|
||||
E:\Projects\Mimir\scripts\start-mvp-leap-eyes-kiyo.ps1
|
||||
```
|
||||
|
||||
This launches Starfire with only local Scarlett ASIO, local Leap stereo IR, and
|
||||
local Kiyo Pro RGB in the Mimir rolling buffers. Nightwing does not stream PS3
|
||||
Eye pixels to Starfire. The launcher stages `nw_eye_cap.py`,
|
||||
`nw_move_hint.py`, and `nightwing_typed_witness_publisher.py` on Nightwing,
|
||||
then starts the typed witness publisher with `--track-eyes`. Nightwing reads
|
||||
`/dev/video2` and `/dev/video3`, extracts compact Move-sphere observations,
|
||||
and sends `mimir.move_controller_observation_state.v1` claims to the
|
||||
`/eve/periwinkle` CultMesh observation receiver. Starfire/Fensalir can consume
|
||||
those claims for overlays and calibration without inheriting Nightwing's raw
|
||||
camera bandwidth or local tracking authority.
|
||||
|
||||
The MVP profile refuses to start when LeapUVC is not enumerated. That is
|
||||
intentional: the Leap is the dense close-range geometry root, not an optional
|
||||
checkbox pretending to be present.
|
||||
|
||||
Use:
|
||||
|
||||
```powershell
|
||||
$env:MIMIR_RUNTIME_CONFIG = "E:\Projects\Mimir\config\mimir-runtime.move-stream.local.json"
|
||||
dotnet run --project .\src\Mimir.App\Mimir.App.csproj
|
||||
```
|
||||
|
||||
The profile declares:
|
||||
|
||||
- `kiyo-pro-rgb`: Kiyo Pro program view, 640x480 YUY2 at the current reliable
|
||||
25 fps class.
|
||||
- `ps3-eye-0`, `ps3-eye-1`: high-rate Move sphere visual witnesses.
|
||||
- `focusrite-asio`: one ASIO producer that emits:
|
||||
`asio-ch0` shotgun hero mic, `asio-ch1` cardioid hero mic,
|
||||
`asio-ch2` Raven program loopback L, and `asio-ch3` Raven program loopback R.
|
||||
- `raven-display`: Raven screen capture over SRT port `5200`.
|
||||
|
||||
Run the synthetic overlay smoke:
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --move-controller-overlay-smoke
|
||||
```
|
||||
|
||||
Run the live ingest smoke once Raven is sending:
|
||||
|
||||
```powershell
|
||||
$env:MIMIR_RUNTIME_CONFIG = "E:\Projects\Mimir\config\mimir-runtime.move-stream.local.json"
|
||||
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --seconds 5 --require-samples --sync-reference asio-ch2
|
||||
```
|
||||
|
||||
## Raven Sender
|
||||
|
||||
On Raven, start the muxed A/V sender from the logged-in desktop session:
|
||||
|
||||
```powershell
|
||||
E:\Projects\Mimir\scripts\start-raven-av-sender.ps1 -TargetHost 192.168.1.66 -Port 5200
|
||||
```
|
||||
|
||||
That sender captures the desktop through `gdigrab`, encodes video with
|
||||
`h264_nvenc`, captures Realtek/default render loopback through the repo WASAPI
|
||||
loopback script, encodes audio as AAC, and muxes both into one MPEG-TS SRT
|
||||
stream. On Starfire, `scripts\start-stream-proof.ps1` starts
|
||||
`scripts\start-raven-av-demux.ps1`, which listens on port `5200` and splits the
|
||||
single transport into local Mimir ingest legs:
|
||||
|
||||
```powershell
|
||||
raven-display local SRT 5210, copied H.264/MPEG-TS for decode
|
||||
raven-realtk-loopback local SRT 5212, decoded f32 PCM
|
||||
```
|
||||
|
||||
If Raven program audio is also routed into Starfire Scarlett on
|
||||
`asio-ch2/asio-ch3`, that remains another strong local timing witness. The
|
||||
Realtek loopback is now part of the same Raven transport as the display before
|
||||
Starfire demuxes it for Mimir's raw ingest APIs.
|
||||
|
||||
Raven display sync is not allowed to ride on SRT arrival time. Both
|
||||
`raven-display` and `raven-realtk-loopback` are in the `raven-sync` clock
|
||||
domain. When audio sync estimates the Realtek loopback offset against the
|
||||
Scarlett reference, `MimirSynchronizedBufferPlanner` applies that correction to
|
||||
the whole `raven-sync` domain, including the display. The LAN transport is now
|
||||
one muxed A/V stream; the local split exists only at the Starfire demux edge
|
||||
because Mimir currently ingests raw video and PCM through separate source
|
||||
adapters.
|
||||
|
||||
## Kiyo AR Trail Receipt
|
||||
|
||||
The first offline proof that the Kiyo view can accept Move trails is:
|
||||
|
||||
```powershell
|
||||
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\solve_stream_proof_frustum.py .\artifacts\runtime\all-sensor-articulated-20260531-232349\sweep\calibration-observations.json
|
||||
```
|
||||
|
||||
That solver uses Nightwing's two matched optical witnesses as a provisional
|
||||
local witness space, fits the Kiyo Pro observations into it, and writes a
|
||||
`mimir.stream_proof_frustum_solve.v1` receipt plus an AR trail preview. It is
|
||||
enough for the stream demo and for Starfire's global residual owner to consume;
|
||||
it is not final metric room calibration.
|
||||
|
||||
## Multiple Move Controllers
|
||||
|
||||
Do not collapse Move observations into a generic bright blob. Each controller
|
||||
has its own `controllerId` and expected RGB/time code. The PS3 Eyes can track
|
||||
the sphere as a high-rate brightness witness; Kiyo Pro can later confirm color
|
||||
identity; the commanded flash schedule ties the observed marker to the
|
||||
controller identity.
|
||||
|
||||
`MimirMoveControllerTracker` keeps a bounded history per controller and lowers
|
||||
those histories into the existing Feature evidence path. This is deliberately
|
||||
2D observation history. 6DoF pose waits for calibrated frustums and the global
|
||||
residual owner.
|
||||
|
||||
## Music-Keyed Calibration Words
|
||||
|
||||
`tools/music_keyed_move_chirp_sync.py` is the first planner for calibration
|
||||
words that are both musical and machine-decodable. It listens to Scarlett ASIO,
|
||||
uses the same Perlines-style adaptive FFT delta whitening as
|
||||
`tools/nightwing_psmove_music_pulse.py`, autocorrelates the whitened broadband
|
||||
spectral-rise function for tempo, estimates a lightweight root from recent
|
||||
best-fit fundamentals, then writes one shared plan:
|
||||
|
||||
- beat-aligned chirps quantized to the estimated minor-pentatonic scale;
|
||||
- de Bruijn-coded event symbols so adjacent timing/frequency/color words remain
|
||||
distinguishable;
|
||||
- per-Move visual words using stable controller identities and unique colors;
|
||||
the default is no longer a square flash, but a sampled RGB contour with
|
||||
attack/release envelope, symbol-dependent hue glide, tremolo articulation,
|
||||
and an emphasized controller lane;
|
||||
- a JSON receipt shaped like AquaSynth song analysis vocabulary:
|
||||
`tempo_bpm`, `beat_seconds`, `tempo_confidence`, `root_note`,
|
||||
`suggested_scale`, `scale_frequencies_hz`, and
|
||||
`whitened_spectral_autocorr`.
|
||||
|
||||
The synthetic smoke is:
|
||||
|
||||
```powershell
|
||||
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\music_keyed_move_chirp_sync.py --mode synthetic --dry-run --events 8 --out-dir artifacts/runtime/music-keyed-move-chirp-sync-smoke
|
||||
```
|
||||
|
||||
The live dry-run against Scarlett is:
|
||||
|
||||
```powershell
|
||||
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\music_keyed_move_chirp_sync.py --dry-run --analyze-seconds 5 --events 8 --out-dir artifacts/runtime/music-keyed-move-chirp-sync-live-dry
|
||||
```
|
||||
|
||||
For double-time material:
|
||||
|
||||
```powershell
|
||||
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\music_keyed_move_chirp_sync.py --dry-run --tempo-grid doubletime --tempo-max-bpm 260 --analyze-seconds 5 --events 8 --out-dir artifacts/runtime/music-keyed-move-chirp-sync-dirtyphonics-dry
|
||||
```
|
||||
|
||||
On the 2026-05-31 live dry-run, the planner produced a usable but not final
|
||||
receipt: about `92.31` BPM, beat `0.650s`, tempo confidence `0.343`, root `A`,
|
||||
key confidence `0.716`. That is enough to prove the dataflow, not enough to
|
||||
claim final musical tempo authority.
|
||||
|
||||
For dense double-time material, use `--tempo-grid doubletime`; the planner then
|
||||
selects the double-time member of the autocorrelation tempo family instead of
|
||||
preferring the slower parent grid. On the Dirtyphonics-style live source this
|
||||
produced about `171.43` BPM, beat `0.350s`, tempo confidence `0.358`, root `A`,
|
||||
key confidence `0.703` after widening the search ceiling with
|
||||
`--tempo-max-bpm 260`. The receipt records `tempo_family` so later decoders can
|
||||
see the selected grid alongside half-time/double-time alternatives.
|
||||
|
||||
Visual calibration words are deliberately shaped like the audio words: one
|
||||
gesture carries more than one bit. `--visual-gesture contour` records a
|
||||
`mimir.move_visual_contour_word.v1` per Move per event, with `sample_rate_hz`,
|
||||
duration, base identity color, symbol emphasis, and the exact RGB samples sent
|
||||
over HID. Start conservatively around `--visual-gesture-hz 80`; the HID writer
|
||||
accepted a short 80 Hz contour test on Nightwing, but the useful rate is the
|
||||
rate cameras can recover cleanly, not merely the rate the controller accepts.
|
||||
`--visual-gesture square` remains available as a baseline comparator.
|
||||
|
||||
`tools/sync_sweep.py` can now consume that exact plan with `--music-plan`.
|
||||
That makes the plan the single authority for audio chirps, sampled Move RGB
|
||||
contours, and offline video scoring. The sweep writes `gesture-plan.json`,
|
||||
`event-schedule.json`, copied/rendered `chirp-train.wav`, per-sensor captures,
|
||||
and `summary.json`. RGB camera receipts include a `visual_contour` block with
|
||||
brightness-envelope correlation and color-chroma error against the expected
|
||||
Move contour. The old peak-only score is still present as a timing witness; it
|
||||
no longer pretends square flashes are the only visual language.
|
||||
|
||||
When `--music-plan` is used, the sweep also runs typed Mimir runtime sensor
|
||||
receipts unless `--skip-mimir-live` is passed. The default receipts are:
|
||||
|
||||
- `config/mimir-runtime.perfect-machine.local.json` for local Leap stereo IR,
|
||||
Kiyo Pro, Kiyo, and local camera texture/depth/field-resource ingress;
|
||||
- `config/mimir-runtime.raven-eve.example.json` for Raven display and Eve
|
||||
camera/mic frame-event listeners.
|
||||
|
||||
That receipt path is not raw media archival yet. It is the audit hook that
|
||||
prevents Leap or Eve from being omitted by a DirectShow/Nightwing-only sweep.
|
||||
|
||||
After a sweep, run `tools/crunch_sync_sweep.py` over the sweep directory to
|
||||
extract a compact calibration observation table. It merges Nightwing's typed
|
||||
Move observations with local Kiyo/Kiyo Pro frame analysis, estimates each
|
||||
local video's schedule shift against the event train, and writes
|
||||
`calibration-observations.json` plus `calibration-observations.csv`.
|
||||
|
||||
Example using an already-written musical plan:
|
||||
|
||||
```powershell
|
||||
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\sync_sweep.py --music-plan artifacts/runtime/music-keyed-move-chirp-sync-live/music-keyed-move-chirp-plan.json --out-dir artifacts/runtime/all-sensor-articulated-sync
|
||||
& 'C:\Users\Meta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe' .\tools\crunch_sync_sweep.py artifacts/runtime/all-sensor-articulated-sync
|
||||
```
|
||||
|
||||
The non-dry run targets the two wireless Nightwing Moves by default:
|
||||
`00:07:04:a6:be:5f` on `/dev/hidraw2` and `00:06:f5:23:e2:d1` on
|
||||
`/dev/hidraw3`. Nightwing now carries `/etc/udev/rules.d/70-move.rules`, which
|
||||
sets Bluetooth PS Move hidraw nodes to `root:input 0660`. The `metacrat` user
|
||||
is in `input`, so the non-root SSH writer can drive both spheres. Root SSH is
|
||||
available as the short local alias `nwroot` for maintenance only; the hot LED
|
||||
path should stay non-root.
|
||||
|
|
@ -56,37 +56,6 @@ buffer handles directly.
|
|||
The probe reported one unrelated invalid USB descriptor from
|
||||
`USB\VID_0000&PID_0002`; it did not prevent PS3 Eye capture.
|
||||
|
||||
## 2026-05-31: Nightwing Linux V4L2 PS3 Eye Probe
|
||||
|
||||
The two PS3 Eyes were moved to Nightwing. Linux enumerated them as Sony
|
||||
Playstation Eye USB devices (`1415:2000`) through the stock `ov534`/`gspca`
|
||||
V4L2 path:
|
||||
|
||||
- `/dev/video2`: `USB Camera-B4.09.24.1`, bus `usb-0000:00:14.0-1`
|
||||
- `/dev/video3`: `USB Camera-B4.09.24.1`, bus `usb-0000:00:14.0-2`
|
||||
|
||||
Probe source:
|
||||
|
||||
- `tools/nightwing_v4l2_cadence_probe.py`
|
||||
- copied to Nightwing and run directly against `/dev/video2` and `/dev/video3`
|
||||
- no `v4l2-ctl`, `ffmpeg`, or third-party Python package dependency
|
||||
|
||||
Initial reads without setting frame interval negotiated only the default
|
||||
`timeperframe=1/30`. After explicitly setting `VIDIOC_S_PARM` to
|
||||
`timeperframe=1/187`, both Eyes delivered high-rate frames with zero sequence
|
||||
gaps.
|
||||
|
||||
| Device | Requested Mode | Actual Mode | Frames | Delivered FPS | Device FPS | Sequence Gap |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| `/dev/video2` | 320x240 GRBG @ 187 | 320x240 GRBG | 375 | 187.07 | 187.37 | 0 |
|
||||
| `/dev/video3` | 320x240 GRBG @ 187 | 320x240 GRBG | 375 | 187.14 | 187.37 | 0 |
|
||||
| `/dev/video2` | 320x240 YUYV @ 187 | 320x240 YUYV | 374 | 186.94 | 187.37 | 0 |
|
||||
| `/dev/video3` | 320x240 YUYV @ 187 | 320x240 YUYV | 375 | 187.06 | 187.38 | 0 |
|
||||
|
||||
Conclusion: Nightwing's stock Linux V4L2 path can sustain the high-rate PS3 Eye
|
||||
witness mode if the witness worker explicitly sets frame interval before
|
||||
streaming. Default V4L2 reads are a false low-cadence signal.
|
||||
|
||||
## 2026-05-21: LeapUVC Kernel Streaming Probe
|
||||
|
||||
The Leap Motion Controller is currently present as `VID_F182&PID_0003` and bound
|
||||
|
|
|
|||
|
|
@ -5,17 +5,24 @@ hosted stream machine.
|
|||
|
||||
## Objective
|
||||
|
||||
Make the room available in OBS as synchronized program video plus separately
|
||||
Make the room available as a Mimir-owned synchronized program with separately
|
||||
controllable audio while preserving ownership:
|
||||
|
||||
- Mimir owns configuration, calibration truth, runtime contracts, launch,
|
||||
status, and persistence.
|
||||
- Mimir.Runtime owns in-memory stream buffers and runtime synchronization.
|
||||
- Native capture workers own device reads and append typed sample handles.
|
||||
- Fensalir owns GPU fusion, rendering, D3D12 interop, UI, and Spout2 output.
|
||||
- Mimir owns stream subscription, scene composition, preview/control state,
|
||||
stats, and program publication intent.
|
||||
- Fensalir owns GPU fusion, rendering, D3D12 interop, UI lowering, and local
|
||||
program texture output.
|
||||
- Faust/native DSP owns hot audio alignment, suppression, separation, stems, and
|
||||
spatial bed generation.
|
||||
- OBS owns broadcast controls.
|
||||
- Eve GUI/TUI lowerers expose operator controls and preview without owning
|
||||
scene truth.
|
||||
- The Yggdrasil-facing publisher daemon consumes Mimir program output and
|
||||
publishes it to the site without owning composition.
|
||||
- OBS is a temporary compatibility sink.
|
||||
|
||||
## Runtime State Model
|
||||
|
||||
|
|
@ -26,23 +33,22 @@ flowchart TD
|
|||
N["network feed producers"] --> R
|
||||
R --> Q["Fensalir UI + GPU fusion"]
|
||||
R --> F["Faust/native DSP"]
|
||||
Q --> S["Spout2/program video"]
|
||||
Q --> S["Mimir program video"]
|
||||
F --> P["program stems + spatial bed"]
|
||||
S --> O["OBS"]
|
||||
P --> O
|
||||
Q --> E["Eve operator surfaces"]
|
||||
S --> Y["Yggdrasil site publisher"]
|
||||
P --> Y
|
||||
S -. "compatibility" .-> O["OBS adapter"]
|
||||
P -. "compatibility" .-> O
|
||||
```
|
||||
|
||||
The app-level runtime initializes one rolling buffer per configured audio or
|
||||
video stream, defaulting to five seconds. The lower native reservoir keeps the
|
||||
single-edge typed-handle invariant for Fensalir/Faust integration.
|
||||
|
||||
Payload memory belongs to the producing domain, but rendering inputs should be
|
||||
Fensalir-owned GPU resources whenever the device path allows it. The camera hot
|
||||
path is: ask Fensalir for a keyed `Texture2D` lease, speak to the device stack
|
||||
as directly as possible, populate that lease, signal the producer fence, and
|
||||
publish the resource key through Mimir's rolling buffers and FieldEvidence. The
|
||||
reservoir and runtime own timing, identity, ordering, retention, and lookup.
|
||||
They do not become pixel middlemen.
|
||||
Payload memory belongs to the producing domain. The reservoir and runtime own
|
||||
timing, identity, ordering, retention, and lookup. They do not pretend to own
|
||||
camera driver buffers, GPU textures, or DSP audio memory.
|
||||
|
||||
## Authoritative Boundaries
|
||||
|
||||
|
|
@ -52,7 +58,8 @@ They do not become pixel middlemen.
|
|||
- Fensalir consumes current visual/audio timing state and publishes the program
|
||||
video surface.
|
||||
- Faust/native DSP consumes audio/phase/event handles and publishes stems.
|
||||
- OBS receives final program surfaces. It does not own synchronization.
|
||||
- OBS compatibility adapters may receive final program surfaces. They do not
|
||||
own synchronization, composition, preview/control, or publication.
|
||||
|
||||
## Impossible By Construction
|
||||
|
||||
|
|
@ -66,10 +73,7 @@ They do not become pixel middlemen.
|
|||
|
||||
1. Keep the C# app/runtime as the public Mimir surface.
|
||||
2. Implement Leap stereo IR direct ingest first and measure sustained cadence.
|
||||
Record the unavoidable copy count; do not hide CPU upload behind a friendly
|
||||
adapter name.
|
||||
3. Add the remaining local camera drivers through the same source seam,
|
||||
preferring device-direct producers that populate Fensalir texture leases.
|
||||
3. Add the remaining local camera drivers through the same source seam.
|
||||
4. Add native audio capture workers and feed aligned blocks to Faust/native DSP.
|
||||
5. Bind Fensalir UI to stream health, buffer depth, source timestamps, settings,
|
||||
and output management.
|
||||
|
|
|
|||
|
|
@ -1,63 +0,0 @@
|
|||
# Network Display Synchronization
|
||||
|
||||
Raven can be a low-level synchronized producer when it controls both its display
|
||||
output and an audio timing signal routed into the Scarlett.
|
||||
|
||||
Authority map:
|
||||
|
||||
- Owner: `Mimir.Runtime` owns aligned rolling-buffer frames.
|
||||
- Inputs: camera buffers, Raven display frames, Scarlett ASIO audio buffers, and
|
||||
explicit timing corrections learned from audio evidence.
|
||||
- Outputs: `MimirSynchronizedBufferFrame` slices at one canonical presentation
|
||||
time for Fensalir/Faust/OBS publication.
|
||||
- Derived state: Raven display timing is derived from the Raven audio sync
|
||||
signal. The display feed is not a timing authority by itself.
|
||||
- Forbidden writers: OBS, SRT bridge endpoints, frame-event probes, and network
|
||||
receive timestamps must not decide cross-stream alignment. CultMesh reliable
|
||||
UDP packet arrival is transport evidence only, not clock authority.
|
||||
- Shared path: local cameras, network display frames, mic channels, loopback
|
||||
channels, and Raven audio sync channels all enter `MimirRollingStreamBuffer`
|
||||
first, then the synchronized planner chooses slices.
|
||||
- Deletion line: any compositor path that directly consumes Raven pixels outside
|
||||
the runtime buffer/alignment path is diagnostic only.
|
||||
|
||||
Practical shape:
|
||||
|
||||
1. Raven renders a visible timing marker or encoded frame identity on the
|
||||
captured display.
|
||||
2. Raven emits the matching audio timing signal through its audio output.
|
||||
3. That audio is physically or digitally routed into a Scarlett input beside the
|
||||
mic channels.
|
||||
4. Mimir decodes the Scarlett audio evidence, learns the Raven clock offset, and
|
||||
applies that clock-domain correction to the `raven-display` video buffer.
|
||||
5. Fensalir receives aligned buffer slices and still owns composition.
|
||||
|
||||
`config/mimir-runtime.raven-eve.example.json` is the active bridge setup shape
|
||||
for the network producer pass:
|
||||
|
||||
- `raven-display` listens for Raven screen capture on SRT port `5200`, decodes
|
||||
through FFmpeg, and emits BGRA video samples into the runtime buffer path.
|
||||
- `eve-camera` listens for EveCanvas camera frame-events on WebSocket port
|
||||
`8793`.
|
||||
- `eve-mic` listens for EveCanvas microphone frame-events on WebSocket port
|
||||
`8794`.
|
||||
|
||||
The target Verse shape replaces those SRT/WebSocket bridge edges with CultMesh
|
||||
reliable-UDP lanes: typed `mimir.cultmesh_stream_frame.v1` envelopes for
|
||||
per-frame media metadata and refs, plus bounded media body shards for payloads
|
||||
too large to inline. Raven display frames remain media observations entering
|
||||
the same rolling buffer; Raven sync still comes from audio evidence and
|
||||
clock-domain correction, not packet arrival.
|
||||
|
||||
The older disabled `frame-events` example in
|
||||
`config/mimir-runtime.asio.example.json` remains diagnostic only. It can witness
|
||||
source timestamps and frame identity, but it does not carry pixels. The Scarlett
|
||||
channel carrying Raven's sync audio is still the evidence path that earns the
|
||||
`raven-sync` timing correction.
|
||||
|
||||
Smoke proof:
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --synchronized-buffer-planner-smoke
|
||||
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --ffmpeg-rawvideo-source-smoke
|
||||
```
|
||||
|
|
@ -1,213 +0,0 @@
|
|||
# OBS Native Plugin
|
||||
|
||||
The production OBS path is a Mimir/Fensalir-owned program texture plus a
|
||||
Mimir/Fensalir-owned stem bus, exposed through a narrow native OBS plugin.
|
||||
|
||||
## Authority
|
||||
|
||||
- Fensalir owns the completed program pixels.
|
||||
- Mimir/Fensalir own timing, alignment, DSP, and stem identity.
|
||||
- OBS owns broadcast mixing and scene composition.
|
||||
- The plugin imports already-produced program pixels and reads already-processed
|
||||
stem samples. It does not synchronize raw sources, estimate clocks, render the
|
||||
field, or decide which payload is true.
|
||||
|
||||
## Current Contract
|
||||
|
||||
### Video
|
||||
|
||||
Fensalir publishes the completed backbuffer into a named shared D3D12
|
||||
`B8G8R8A8_UNorm` texture when these environment variables are set:
|
||||
|
||||
```text
|
||||
FENSALIR_PROGRAM_OUTPUT_D3D12=1
|
||||
FENSALIR_PROGRAM_OUTPUT_NAME=Global\MimirFensalirProgramTexture
|
||||
```
|
||||
|
||||
The native OBS plugin in `native/obs_stem_source` exposes a `Mimir Program
|
||||
Texture` video source. Set:
|
||||
|
||||
```text
|
||||
Shared D3D12 texture name: Global\MimirFensalirProgramTexture
|
||||
Shared D3D12 fence name: Global\MimirFensalirProgramFence
|
||||
Consumer D3D12 fence name: Global\MimirObsProgramConsumerFence
|
||||
Texture ring slots: 1
|
||||
```
|
||||
|
||||
OBS/libobs is D3D11 internally on Windows, so this source opens the named D3D12
|
||||
texture, creates an OBS-readable legacy shared D3D11 texture, opens that same
|
||||
bridge texture from D3D12, and performs a GPU-to-GPU copy during OBS render
|
||||
before drawing the source. This avoids CPU readback and does not require
|
||||
Spout2. It is not pure zero-copy yet; the remaining copy is the
|
||||
D3D12-to-libobs-D3D11 boundary. Fensalir also publishes a named D3D12
|
||||
program-output fence; the OBS source opens it and observes publication
|
||||
completion before copying. This prevents blind read-before-producer-completion
|
||||
behavior without coupling OBS to Fensalir's private frame fence.
|
||||
|
||||
For OBS-only stress testing, Fensalir and the OBS source can agree on a bounded
|
||||
texture ring:
|
||||
|
||||
```text
|
||||
FENSALIR_PROGRAM_OUTPUT_RING_COUNT=3
|
||||
FENSALIR_PROGRAM_OUTPUT_CONSUMER_FENCE_NAME=Global\MimirObsProgramConsumerFence
|
||||
Texture ring slots: 3
|
||||
Consumer D3D12 fence name: Global\MimirObsProgramConsumerFence
|
||||
```
|
||||
|
||||
With a ring count above one, Fensalir publishes textures named
|
||||
`Global\MimirFensalirProgramTexture.0`, `.1`, and so on. The OBS source uses the
|
||||
program-output fence value to copy the latest completed slot and signals the
|
||||
consumer fence after its GPU copy. When Fensalir is launched with the matching
|
||||
consumer-fence environment variable, it will not reuse a ring slot until OBS has
|
||||
acknowledged a copy at or beyond that slot's published fence value. Leave the
|
||||
ring count at `1` for EVE until the EVE relay speaks the same ring contract.
|
||||
|
||||
For a visible interop witness while the real Mimir scene is sparse, launch
|
||||
Mimir with:
|
||||
|
||||
```text
|
||||
MIMIR_OBS_PROOF_VISUAL=1
|
||||
```
|
||||
|
||||
That enables a small bright SDF/starfield proof visual. It is a debug witness,
|
||||
not production scene policy.
|
||||
|
||||
### Audio
|
||||
|
||||
`MimirObsStemSharedMemoryPublisher` writes the latest validated
|
||||
`MimirObsStemPublicationSnapshot` into a Windows shared-memory map named
|
||||
`Local\MimirObsStemBus` by default. The native OBS plugin in
|
||||
`native/obs_stem_source` exposes a `Mimir Audio Stem` input source. Add one
|
||||
source per stem and set:
|
||||
|
||||
```text
|
||||
Shared memory map: Local\MimirObsStemBus
|
||||
Stem id: aligned_source_0
|
||||
```
|
||||
|
||||
Use `aligned_source_0` through `aligned_source_5` for the first actuator-stage
|
||||
source lanes.
|
||||
|
||||
## Build
|
||||
|
||||
```powershell
|
||||
.\scripts\build-obs-stem-plugin.ps1
|
||||
```
|
||||
|
||||
The script stages `obsproject/obs-plugintemplate` under `artifacts/obs-sdk/`,
|
||||
lets the upstream template fetch/build the matching OBS/libobs development
|
||||
surface, then builds `native/obs_stem_source` against that SDK. The staged SDK
|
||||
and native build products are local artifacts, not repository state.
|
||||
|
||||
The managed shared-memory publisher is verified by:
|
||||
|
||||
```powershell
|
||||
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --obs-stem-shared-memory-smoke
|
||||
```
|
||||
|
||||
The local native plugin build currently succeeds and emits:
|
||||
|
||||
```text
|
||||
native\obs_stem_source\build\Release\mimir_obs_stem_source.dll
|
||||
```
|
||||
|
||||
That DLL currently registers both source types:
|
||||
|
||||
```text
|
||||
Mimir Program Texture
|
||||
Mimir Audio Stem
|
||||
Muninn Stream
|
||||
```
|
||||
|
||||
### Muninn Streams
|
||||
|
||||
The `Muninn Stream` source is a thin OBS wrapper over Muninn's typed
|
||||
CultCache/CultMesh stream catalog. The operator-facing UI is intentionally just
|
||||
the stream dropdown plus independent video and audio device pickers for the
|
||||
selected source. Store paths, command endpoints, and media return endpoints are
|
||||
source defaults and migration state, not user-facing stream selection.
|
||||
|
||||
The source listens for Raven's typed OBS catalog projection on local UDP port
|
||||
`17874`. Raven Muninn sends the catalog over a small CultNet RUDP discovery lane
|
||||
using connection id `0x6d750003`; OBS uses that live catalog for the dropdown
|
||||
when present.
|
||||
|
||||
If live discovery is absent, the source falls back to Muninn's local `.cc`
|
||||
store:
|
||||
|
||||
```text
|
||||
C:\Meta\Odin\state\muninn.telemetry.cc
|
||||
```
|
||||
|
||||
The source looks for record key `obs` with type:
|
||||
|
||||
```text
|
||||
muninn.obs_stream_catalog
|
||||
```
|
||||
|
||||
Muninn owns that catalog as typed CultCache state. OBS owns only the selected
|
||||
scene source, the selected video leg, the selected audio leg, and media
|
||||
rendering. When a selected stream uses Muninn's RUDP media profile, the source
|
||||
publishes the same typed
|
||||
`muninn.capture_stream_command` that an operator would publish manually. Raven
|
||||
Muninn `serve` still owns whether capture starts and which local FFmpeg/WASAPI
|
||||
children exist.
|
||||
|
||||
If both live discovery and the local catalog projection are absent or stale, the
|
||||
source keeps a LAN-default Raven A/V stream option instead of showing an empty
|
||||
source. That is a resilience fallback for the OBS machine, not a second stream
|
||||
authority. Stale catalog `rudp://` URLs that do not carry the current
|
||||
low-latency LAN profile are sanitized back to the source-derived URL before the
|
||||
bridge starts.
|
||||
|
||||
The current Raven-room default route is LAN-first: command requests go to
|
||||
`192.168.1.84:17873`, and Raven sends media back to Starfire at
|
||||
`192.168.1.66:5204`. Older saved source settings that still point at the
|
||||
deprecated WireGuard defaults `10.77.0.4:17873` and `10.77.0.2` are migrated on
|
||||
load. Custom operator-entered endpoints are left alone.
|
||||
|
||||
The source polls the catalog while active, but reconnect is owned by the RUDP
|
||||
receiver bridge rather than catalog labels. The bridge tracks the last packet
|
||||
and the last forwarded video/audio payload. If startup produces no media, or a
|
||||
previously live stream goes stale, OBS refreshes its local FFmpeg child readers
|
||||
and republishes the typed activation command on a short bounded cadence. A Raven
|
||||
daemon restart or media child exit should therefore recover without reopening
|
||||
source properties.
|
||||
|
||||
For `rudp://` stream URLs, the plugin binds the advertised media port, accepts
|
||||
the CultNet RUDP handshake, acknowledges packets, reassembles RUDP fragments,
|
||||
decodes typed `muninn.media_video_access_unit.v1` and
|
||||
`muninn.media_audio_packet.v1` records, and lowers elementary H.264 plus ADTS
|
||||
AAC into local loopback UDP sockets for OBS FFmpeg child sources. The bridge
|
||||
does not impose a global packet-sequence FIFO over the media lane: CultNet RUDP
|
||||
media is reliable but unordered, so frame ids, chunk ids, and the advertised
|
||||
`assembly_deadline_ms` own video reconstruction. Incomplete video frames expire
|
||||
inside that budget and emit `muninn.media_receiver_feedback.v1` keyframe
|
||||
pressure instead of stalling newer media behind an obsolete sequence gap.
|
||||
RUDP fragment ids are quarantined briefly after completion/expiry so late
|
||||
duplicates cannot seed a corrupt assembly when the sender's 16-bit fragment id
|
||||
space wraps under sustained LAN pressure. The local OBS FFmpeg UDP readers use
|
||||
large loopback buffers; those buffers absorb receiver bursts without becoming a
|
||||
second transport authority.
|
||||
|
||||
## Install
|
||||
|
||||
For the standard OBS plugin search path:
|
||||
|
||||
```powershell
|
||||
.\scripts\install-obs-mimir-plugin.ps1
|
||||
```
|
||||
|
||||
By default this writes:
|
||||
|
||||
```text
|
||||
C:\ProgramData\obs-studio\plugins\mimir_obs_stem_source\bin\64bit\mimir_obs_stem_source.dll
|
||||
```
|
||||
|
||||
For a portable OBS tree or direct OBS installation root:
|
||||
|
||||
```powershell
|
||||
.\scripts\install-obs-mimir-plugin.ps1 -ObsRoot D:\Tools\obs-studio
|
||||
```
|
||||
|
||||
The script does not create scene items or alter OBS profiles.
|
||||
|
|
@ -27,13 +27,6 @@ Read this with:
|
|||
loop candidates.
|
||||
- [[research/perfect-machine-study-2026-05-23/fensalir-integration-map|Fensalir Integration Map]]
|
||||
for the Mimir/Fensalir ownership cut.
|
||||
- [[fensalir-rendering-rebuild-migration|Fensalir Rendering Rebuild Migration]]
|
||||
for the current teardown/rebuild plan that keeps Mimir on physical evidence
|
||||
and pushes field rendering authority into Fensalir's evidence machine.
|
||||
- [[research/d3d12-stereo-depth-provenance-2026-05-29|D3D12 Stereo Depth Provenance]]
|
||||
for the current fast GPU depth search. `libSGM` is the permissive north-star
|
||||
reference for a future Fensalir-owned D3D12 compute stereo disparity lane;
|
||||
CUDA/TensorRT projects remain provenance, not live dependencies.
|
||||
- [[research/perfect-machine-study-2026-05-23/calibration-session-spec|Calibration Session Spec]]
|
||||
for the planned physical-path calibration command.
|
||||
- [[perfect-machine-module-library|Perfect Machine Module Library]] for the
|
||||
|
|
@ -45,19 +38,23 @@ Read this with:
|
|||
|
||||
Mimir is a bounded-latency realtime field computer: it ingests many clocks,
|
||||
places their evidence inside one rolling temporal window, extracts audio/video
|
||||
geometry from that window, and emits an OBS-usable program surface without
|
||||
letting OBS become the synchronization authority.
|
||||
geometry from that window, owns the stream program scene, exposes Eve operator
|
||||
surfaces, and emits a publicable program feed without letting compatibility
|
||||
sinks become authority.
|
||||
|
||||
## Authority Index
|
||||
|
||||
| Authority | Owns | Must Not Own |
|
||||
| --- | --- | --- |
|
||||
| `Mimir.Runtime` | stream identity, buffers, sync reports, calibration model loading, source polling, telemetry state | hot DSP actuator, dense visual fusion, broadcast composition |
|
||||
| Native capture | direct driver callbacks, device timestamps, payload handles, low-jitter capture loops | runtime policy, UI state, OBS composition |
|
||||
| Mimir program composition | source subscription, scene graph, crop/key/transform state, preview/control state, stats, publication intent | local device reads, GPU shader internals, audio-rate DSP, site transport policy |
|
||||
| `Mimir.Runtime` | stream identity, buffers, sync reports, calibration model loading, source polling, telemetry state | hot DSP actuator, dense visual fusion, direct device capture |
|
||||
| Native capture / Muninn hosts | direct driver callbacks, device timestamps, payload handles, local stream capabilities/media | runtime policy, scene composition, publication |
|
||||
| `native/reservoir` | shared-edge retention and typed views | payload memory interpretation, rendering, DSP |
|
||||
| Fensalir | window, UI, D3D12 bridge, GPU fusion, program video publication | clock authority, audio actuator |
|
||||
| Fensalir | window, UI lowering, D3D12 bridge, GPU fusion, program video output | clock authority, audio actuator, scene ownership |
|
||||
| Faust/native DSP | fractional delay, resampling, separation, spatialization, stems | stream discovery, UI |
|
||||
| OBS | final broadcast composition | synchronization, capture timing |
|
||||
| Eve lowerers | operator GUI/TUI rendering, controls, previews, stats lowering | scene truth, stream truth, publication truth |
|
||||
| Yggdrasil site publisher | public transport, encoder/session state, reconnect policy, site edge health | scene composition |
|
||||
| OBS | compatibility preview/output sink | synchronization, capture timing, composition, publication |
|
||||
| Raven/phones | local decode of codebook/schedule observations | canonical clock authority |
|
||||
|
||||
## Evidence Taxonomy
|
||||
|
|
@ -85,11 +82,6 @@ letting OBS become the synchronization authority.
|
|||
### Visual Evidence
|
||||
|
||||
- Leap stereo IR: timing/depth candidate, topology-sensitive but high-value.
|
||||
The live `LeapStereoIr` resource is a packed R8G8 texture; Mimir now lowers
|
||||
it into a stereo-depth FieldEvidence claim by treating R/G as the left/right
|
||||
observation lanes and asking Fensalir for an R16F disparity `SurfacePage`.
|
||||
The first installed Fensalir kernel is a packed block matcher, not calibrated
|
||||
metric depth or full SGM.
|
||||
- PS3 Eyes: high-rate tracking witnesses.
|
||||
- Kiyo/Kiyo Pro: RGB context and texture, with Kiyo Pro currently cadence/USB
|
||||
limited.
|
||||
|
|
@ -177,18 +169,6 @@ Likely shape:
|
|||
|
||||
- Native capture pushes typed frame handles and timestamps.
|
||||
- Fensalir extracts features/depth/optical flow on GPU.
|
||||
- Stereo depth starts as a Fensalir-owned D3D12 compute lane modeled on
|
||||
permissive SGM provenance, especially `libSGM`: rectified texture pairs plus
|
||||
calibration produce GPU-resident disparity/depth/confidence resources, which
|
||||
then lower into FieldEvidence claims.
|
||||
- Current contract proof:
|
||||
`Mimir.BufferSmoke --stereo-depth-contract-smoke` validates the profile and
|
||||
FieldEvidence socket. The synthetic Leap stereo pair uses caller-declared
|
||||
shader-readable left/right texture declarations, an R16F compute-writable disparity
|
||||
`SurfacePage`, an R8 confidence `Texture2D`, one Height claim, one
|
||||
`AquariumFieldStereoDepthLowering` sidecar with min disparity 0, 128
|
||||
disparities, four paths, census radius 2, P1 8, and P2 96, and one planned `SurfacePage`
|
||||
packet. It is not yet a live compute kernel.
|
||||
- Native reservoir preserves rolling evidence.
|
||||
- Fusion emits surface/material/render claims.
|
||||
- Program video publishes via Spout2.
|
||||
|
|
@ -197,8 +177,6 @@ Pressure:
|
|||
|
||||
- Do not build a CPU image pipeline because it is easy to inspect. Six cameras
|
||||
make that path collapse into a pile of excuses.
|
||||
- Do not import CUDA or TensorRT as live depth authority. External depth
|
||||
projects may guide kernel shape, benchmarks, and failure probes only.
|
||||
|
||||
## Hot Loop Budget
|
||||
|
||||
|
|
@ -267,7 +245,9 @@ Six optical sensors with high-rate Eyes and Leap IR mean:
|
|||
- Cut JSON/stdout local camera hot path when concrete direct drivers land.
|
||||
- Cut legacy chirplet path once chirp-bin decoder has equivalent docs/tests and
|
||||
physical path proof.
|
||||
- Cut bridge scripts only after OBS-facing direct output is real.
|
||||
- Cut bridge scripts only after Mimir program output, Eve operation, and
|
||||
Yggdrasil publication are real enough that OBS compatibility no longer earns
|
||||
its keep.
|
||||
- Cut ad hoc response weighting if a formal path model replaces it.
|
||||
|
||||
## Future Work Index
|
||||
|
|
@ -277,9 +257,6 @@ Six optical sensors with high-rate Eyes and Leap IR mean:
|
|||
3. Reduced alphabet/high-order codebook physical proof.
|
||||
4. Fractional delay/SRO actuator proof.
|
||||
5. Direct Leap/PS3/Kiyo runtime drivers.
|
||||
6. D3D12 stereo disparity/depth compute lane, with `libSGM` as provenance and
|
||||
Fensalir as live owner. Contract/profile smoke exists; implement the HLSL
|
||||
cost-volume, aggregation, winner/subpixel, and confidence passes next.
|
||||
7. Native reservoir integration with Fensalir/Faust.
|
||||
8. Raven/phone receiver codebook/schedule local decode.
|
||||
9. GPU visual fusion and Spout2 program video.
|
||||
6. Native reservoir integration with Fensalir/Faust.
|
||||
7. Raven/phone receiver codebook/schedule local decode.
|
||||
8. GPU visual fusion and Spout2 program video.
|
||||
|
|
|
|||
|
|
@ -13,23 +13,23 @@ job-interview answer for why it belongs in the body of the machine.
|
|||
| Bioacoustic clock solver | `Mimir.Runtime` | Fits canonical source time from decoded word anchors for local, Raven, phone, and microcontroller witnesses. | `src/Mimir.Runtime/Synchronization/MimirBioacousticClockHypothesis.cs` |
|
||||
| Alignment actuator | `Mimir.Runtime` + Faust | Converts sync state into bounded fractional-delay, gain, and sample-rate-offset controls; Faust moves samples. | `src/Mimir.Runtime/Synchronization/MimirAudioAlignmentActuator.cs` |
|
||||
| Audio actuator strategies | `Mimir.Runtime` + Faust | Integer baseline, Farrow fractional delay, variable ASRC, and hybrid delay/ASRC strategy profiles. | `src/Mimir.Runtime/Synchronization/MimirAudioActuatorConfiguration.cs` |
|
||||
| CultMesh contracts | `Mimir.Runtime` | Typed codebook, decoder, acoustic path, and actuator state documents for distributed Mimir nodes. | `src/Mimir.Runtime/Synchronization/MimirCultMeshContracts.cs` |
|
||||
| CultMesh contracts | `Mimir.Runtime` | Typed codebook, decoder, acoustic path, actuator, program, Move evidence, and Mimir-fused Move controller pose documents for distributed Mimir nodes. | `src/Mimir.Runtime/Synchronization/MimirCultMeshContracts.cs`, `src/Mimir.Runtime/Synchronization/MimirTrackingObservation.cs` |
|
||||
| Move controller fusion | `Mimir.Runtime` + native reservoir + Fensalir | Mimir admits Muninn marker candidates plus controller/IMU/button feeds into native `move_evidence` buffers. `MimirMoveCalibrationProtocol` defines the Starfire/Nightwing capture run, Muninn records optional Quest USB witness availability as `muninn.quest_access.v1`, `MimirMoveFusion` publishes calibrated `mimir.move_controller_pose.v1` position/button/gyro poses only when camera witnesses are calibrated, and `MimirMovePoseStream` carries those poses over a realtime CultMesh frame. IMU orientation/prediction remains the next earned 6DoF step before Fensalir treats the stream as VR-grade controller input. | `src/Mimir.Runtime/Synchronization/MimirTrackingObservation.cs`, `src/Mimir.Runtime/Synchronization/MimirMuninnMoveEvidence.cs`, `src/Mimir.Runtime/Synchronization/MimirMoveCalibrationProtocol.cs`, `src/Mimir.Runtime/Synchronization/MimirMoveFusion.cs`, `src/Mimir.Runtime/Synchronization/MimirMovePoseStream.cs`, `src/Mimir.Runtime/Synchronization/MimirNativeReservoirRuntime.cs`, `native/reservoir/src/lib.rs` |
|
||||
| Acoustic path learning | `Mimir.Runtime` | Calibration stages for usable bands, confusion, global delay, group delay, and codebook adaptation. | `src/Mimir.Runtime/Synchronization/MimirAcousticPathLearningConfiguration.cs` |
|
||||
| Acoustic localization | `Mimir.Runtime` + Fensalir | Pairwise TDOA, SRP-PHAT grid, sparse source, and visual-constrained localization profiles plus a small SRP/TDOA scorer. | `src/Mimir.Runtime/Synchronization/MimirAcousticLocalizationConfiguration.cs`, `src/Mimir.Runtime/Synchronization/MimirAcousticLocalizationSolver.cs` |
|
||||
| Benchmark panels | `Mimir.Runtime` | Decoder golf and meatspace acceptance degradation panels with receipt roots and thresholds. | `src/Mimir.Runtime/Synchronization/MimirBenchmarkPanelConfiguration.cs` |
|
||||
| Capture profiles | `Mimir.Runtime` + native workers | Hardware intent for Leap, Kiyos, PS Eyes, Starfire Scarlett, and Raven Scarlett. | `src/Mimir.Runtime/Synchronization/MimirNativeCaptureConfiguration.cs` |
|
||||
| Capture profiles | `Mimir.Runtime` + native workers | Hardware intent for Leap, Kiyos, PS Eyes, Starfire Scarlett, Raven Scarlett, and USB-attached Moves on Starfire/Nightwing. | `src/Mimir.Runtime/Synchronization/MimirNativeCaptureConfiguration.cs` |
|
||||
| Camera ingest strategies | `Mimir.Runtime` + native workers | Diagnostic metadata, managed native wrapper, native SPSC ring, and shared-GPU-texture ingest shapes. | `src/Mimir.Runtime/Synchronization/MimirCameraIngestConfiguration.cs` |
|
||||
| Stereo depth profiles | `Mimir.Runtime` + Fensalir | SGM-shaped stereo disparity/depth profiles with libSGM as provenance for the first D3D12 lane, not a CUDA dependency. | `src/Mimir.Runtime/Synchronization/MimirStereoDepthConfiguration.cs` |
|
||||
| Reservoir strategies | `Mimir.Runtime` + native reservoir + Fensalir | Managed rolling buffers, native shared-edge rings, and derived GPU temporal field storage. | `src/Mimir.Runtime/Synchronization/MimirReservoirConfiguration.cs` |
|
||||
| Distributed witnesses | `Mimir.Runtime` + CultMesh | Raven, phone, microcontroller, Nightwing Eyes/Moves, and remote camera rig configurations for compact timing/path/visual state. | `src/Mimir.Runtime/Synchronization/MimirDistributedWitnessConfiguration.cs` |
|
||||
| Network transports | `Mimir.Runtime` + CultMesh | Typed timing state, reliable-UDP Verse media/body lanes, debug windows, SRT bridge media, and experimental browser transport policy plus constraint-based selection. | `src/Mimir.Runtime/Synchronization/MimirNetworkTransportConfiguration.cs`, `src/Mimir.Runtime/Synchronization/MimirNetworkTransportSelector.cs` |
|
||||
| Distributed witnesses | `Mimir.Runtime` + CultMesh | Raven, phone, microcontroller, and remote camera rig configurations for compact timing/path state. | `src/Mimir.Runtime/Synchronization/MimirDistributedWitnessConfiguration.cs` |
|
||||
| Network transports | `Mimir.Runtime` + CultMesh | Typed timing state, debug windows, SRT bridge media, and experimental browser transport policy plus constraint-based selection. | `src/Mimir.Runtime/Synchronization/MimirNetworkTransportConfiguration.cs`, `src/Mimir.Runtime/Synchronization/MimirNetworkTransportSelector.cs` |
|
||||
| Authority policy | `Mimir.Runtime` | Trust decisions for Starfire loopback, Raven evidence, phone candidates, diagnostic media, and unknown nodes. | `src/Mimir.Runtime/Synchronization/MimirAuthorityPolicyConfiguration.cs`, `src/Mimir.Runtime/Synchronization/MimirAuthorityPolicyEvaluator.cs` |
|
||||
| Audio field configurations | `Mimir.Runtime` | Six-mic, source-based, ambisonic, and hybrid evidence field shapes. | `src/Mimir.Runtime/Synchronization/MimirAudioFieldConfiguration.cs` |
|
||||
| Visual fusion configurations | `Mimir.Runtime` + Fensalir | Sensor roles and fusion modes from cadence proof through full hybrid evidence. | `src/Mimir.Runtime/Synchronization/MimirVisualFusionConfiguration.cs` |
|
||||
| Compute offload configurations | `Mimir.Runtime` + CultMesh | Local-heavy, distributed witness, and calibration sweep workload placement. | `src/Mimir.Runtime/Synchronization/MimirComputeOffloadConfiguration.cs` |
|
||||
| Assembly plans | `Mimir.Runtime` | Named combinations of modules that prove synthetic, Scarlett, meatspace, and distributed cuts. | `src/Mimir.Runtime/Synchronization/MimirMachineAssemblyPlan.cs` |
|
||||
| OBS publication profiles | Fensalir + Faust + OBS | Spout/shared-texture video and separately mixable Faust stem publication. | `src/Mimir.Runtime/Synchronization/MimirObsPublicationConfiguration.cs` |
|
||||
| Fensalir lowering | `Mimir.Runtime` -> Fensalir | Pure mapping from Mimir rolling/video/audio state into Fensalir GPU sensor, stereo-depth, and acoustic field frames. | `src/Mimir.Runtime/Synchronization/MimirFensalirFieldLowering.cs` |
|
||||
| Program publication profiles | Mimir + Fensalir + Faust + Eve + Yggdrasil | Mimir-owned scene publication, Eve operator surfaces, Yggdrasil site feed, and temporary OBS compatibility adapters. | `src/Mimir.Runtime/Synchronization/MimirProgramPublicationConfiguration.cs` |
|
||||
| Fensalir lowering | `Mimir.Runtime` -> Fensalir | Pure mapping from Mimir rolling/video/audio state into Fensalir GPU sensor and acoustic field frames. | `src/Mimir.Runtime/Synchronization/MimirFensalirFieldLowering.cs` |
|
||||
| Module catalog | `Mimir.Runtime` | Indexes the live module set and verification commands. | `src/Mimir.Runtime/Synchronization/MimirModuleLibrary.cs` |
|
||||
| Manifest factory | `Mimir.Runtime` | Produces an exportable JSON manifest for tools, docs, UI, and remote witnesses. | `src/Mimir.Runtime/Synchronization/MimirPerfectMachineManifest.cs` |
|
||||
|
||||
|
|
@ -41,10 +41,21 @@ job-interview answer for why it belongs in the body of the machine.
|
|||
program video publication.
|
||||
- Faust owns hot sample movement, alignment, spatialization, separation, and
|
||||
stem generation.
|
||||
- CultMesh carries typed state surfaces and reliable-UDP media/body lanes;
|
||||
remote nodes may observe, decode, and stream media through the Verse, but
|
||||
they do not become independent clock authorities.
|
||||
- OBS receives final program surfaces and stems. It does not own calibration.
|
||||
- CultMesh carries typed state surfaces; remote nodes may observe and decode,
|
||||
but they do not become independent clock authorities.
|
||||
- USB-attached Moves on Starfire and Nightwing publish source-local Muninn
|
||||
evidence streams: glowing-orb marker candidates and controller/IMU/button
|
||||
state. Odin discovers/projects those streams. Mimir owns native evidence
|
||||
buffer admission/layout, calibration, association, triangulation, confidence,
|
||||
and `mimir.move_controller_pose.v1` publication. The current fusion pass
|
||||
resolves calibrated optical position and controller state; orientation remains
|
||||
explicitly unresolved until IMU fusion/prediction lands. Fensalir consumes the
|
||||
resolved pose stream for interactive environments once that contract is wired.
|
||||
- Resolved Move poses are framed as
|
||||
`mimir.move_controller_pose_stream_frame.v1` on CultMesh shared-memory bytes
|
||||
streams; the stream is a Mimir output surface, not a new fusion owner.
|
||||
- OBS may receive compatibility program surfaces and stems. It does not own
|
||||
calibration, composition, preview/control, or publication.
|
||||
|
||||
## Verification Surface
|
||||
|
||||
|
|
@ -55,6 +66,12 @@ dotnet build .\Mimir.slnx
|
|||
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --perfect-machine-profile-smoke
|
||||
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --perfect-machine-lowering-benchmark
|
||||
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --perfect-machine-contract-smoke
|
||||
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --move-tracking-contract-smoke
|
||||
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --move-native-reservoir-smoke --native-reservoir .\native\reservoir\target\debug\localcast_reservoir.dll
|
||||
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --muninn-move-evidence-smoke --native-reservoir .\native\reservoir\target\debug\localcast_reservoir.dll
|
||||
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --muninn-move-cultmesh-stream-smoke --native-reservoir .\native\reservoir\target\debug\localcast_reservoir.dll
|
||||
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --move-calibration-protocol-smoke
|
||||
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --move-fusion-smoke
|
||||
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --perfect-machine-manifest
|
||||
dotnet run --no-build --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --bioacoustic-actuator-self-test
|
||||
```
|
||||
|
|
|
|||
|
|
@ -1,42 +0,0 @@
|
|||
# Program Control Surface
|
||||
|
||||
Mimir now exposes a compact Fensalir-hosted program panel for final presentation
|
||||
intent.
|
||||
|
||||
Authority map:
|
||||
|
||||
- Owner: `MimirPresentationControlState` owns operator intent for program video,
|
||||
audio mix, and global color preset.
|
||||
- Inputs: runtime rolling buffers, aligned audio lanes, LUT preset selection,
|
||||
feed visibility, feed opacity, stem mute/solo, and stem gain.
|
||||
- Outputs: filtered video surface intents, Faust/native DSP gain controls,
|
||||
sample gain applied before Fensalir streaming DSP, and a postprocess control
|
||||
snapshot.
|
||||
- Derived state: feed lists are derived from active rolling buffers. LUT preset
|
||||
exposure and bloom are currently mapped into `GraphicsSettings`.
|
||||
- Forbidden writers: OBS, bridge endpoints, debug readouts, and network arrival
|
||||
timestamps must not choose program composition.
|
||||
- Shared paths: local cameras, Raven display capture, and future network video
|
||||
feeds all enter rolling buffers first, then the presentation state decides
|
||||
whether they contribute to production surface intent.
|
||||
- Deletion line: direct compositor shortcuts outside Fensalir FieldEvidence and
|
||||
audio DSP are diagnostic only.
|
||||
|
||||
The current UI lives in the `Mimir Program` panel:
|
||||
|
||||
- `Video`: select feed, visible, solo, opacity, layer ordering.
|
||||
- `Audio`: select stem/source, mute, solo, gain.
|
||||
- `Color`: LUT preset and LUT strength.
|
||||
|
||||
The shader-side LUT path is explicit state, not finished renderer authority yet.
|
||||
`MimirLookupTablePreset` carries `LutPath`, strength, contrast, saturation,
|
||||
temperature, and tint so Fensalir can consume the same contract when the global
|
||||
post shader grows real LUT sampling. Until that renderer cut lands, Mimir maps
|
||||
the preset exposure and bloom values into existing Fensalir `GraphicsSettings`.
|
||||
|
||||
Smoke proof:
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --presentation-control-smoke
|
||||
```
|
||||
|
||||
|
|
@ -1,80 +0,0 @@
|
|||
# Raven Capture Mux Daemon
|
||||
|
||||
Raven owns local display capture and Realtek render-loopback capture. Mimir receives the muxed stream and decides what it means inside the synchronized program model.
|
||||
|
||||
## Authority Map
|
||||
|
||||
- Owner: `Mimir.RavenDaemon` owns process supervision and typed status for Raven's capture mux.
|
||||
- Inputs: local display capture parameters, WASAPI render-loopback selection, FFmpeg path, named SRT targets, log root, and operator flags.
|
||||
- Outputs: `mimir.raven_capture_mux_state.v1`, an Eve dashboard surface, health JSON, and named FFmpeg SRT MPEG-TS streams for Mimir and OBS.
|
||||
- Derived state: PID, restart count, log paths, and dashboard nodes are derived from the supervised process and daemon options.
|
||||
- Forbidden writers: capture scripts may still launch diagnostics, but they do not own daemon status or the canonical Raven mux state.
|
||||
- Shared paths: service launch, dry run, health checks, and Eve dashboard projection all read the same `MimirRavenCaptureMuxStateDocument`.
|
||||
- Cut line: no OBS plugin, custom media daemon, or alternate transport owns media in this pass. FFmpeg/WASAPI own capture, mux, encode, and SRT fanout; CultMesh owns status and operator surface.
|
||||
|
||||
## Run On Raven
|
||||
|
||||
From a Mimir checkout on Raven:
|
||||
|
||||
```powershell
|
||||
.\scripts\start-raven-daemon.ps1
|
||||
```
|
||||
|
||||
Defaults:
|
||||
|
||||
- Mimir target: `srt://10.77.0.2:5200`
|
||||
- OBS target: `srt://10.77.0.2:5204`
|
||||
- Eve/CultMesh dashboard: `ws://0.0.0.0:8801/eve/deck`
|
||||
- Binary CultMesh dashboard stream: `ws://0.0.0.0:8801/eve/deck/cultmesh`
|
||||
- Health: `http://127.0.0.1:8801/health`
|
||||
- State cache: `C:\Meta\Mimir\state\raven-capture-mux.ccmp`
|
||||
- Logs: `C:\Meta\Mimir\logs`
|
||||
|
||||
Use a dry run to inspect the exact FFmpeg pipeline without starting capture:
|
||||
|
||||
```powershell
|
||||
.\scripts\start-raven-daemon.ps1 -DryRun
|
||||
```
|
||||
|
||||
Disable the OBS sink when only Mimir should receive the mux:
|
||||
|
||||
```powershell
|
||||
.\scripts\start-raven-daemon.ps1 -NoObsTarget
|
||||
```
|
||||
|
||||
Use a specific Realtek render endpoint when Windows exposes more than one:
|
||||
|
||||
```powershell
|
||||
.\scripts\start-raven-daemon.ps1 -AudioDevice "Realtek"
|
||||
```
|
||||
|
||||
If Raven already has a host-level CultMesh server bound, keep this daemon as a local typed-state publisher and Eve endpoint:
|
||||
|
||||
```powershell
|
||||
.\scripts\start-raven-daemon.ps1 -NoCultNetServer
|
||||
```
|
||||
|
||||
## Receiver Shape
|
||||
|
||||
The daemon sends identical MPEG-TS SRT streams containing:
|
||||
|
||||
- `raven-display`: Raven desktop via `ddagrab` or `gdigrab`, encoded with `h264_nvenc`
|
||||
- `raven-realtk-loopback`: Raven render loopback via `Mimir.WasapiLoopback` or the PowerShell WASAPI fallback, encoded as AAC
|
||||
|
||||
Mimir receives the ingest copy on `5200`. `scripts/start-raven-av-demux.ps1`
|
||||
can split that copy back into local video and audio source lanes for the
|
||||
current raw ingest adapters. OBS receives its own copy on `5204` as an ordinary
|
||||
SRT Media Source; it does not compete with Mimir for the `5200` listener.
|
||||
|
||||
The typo in the existing source id, `raven-realtk-loopback`, is preserved until the receiver config is migrated.
|
||||
|
||||
## Idunn Lifecycle
|
||||
|
||||
Idunn watches the Raven daemon from Odin's local keepalive body:
|
||||
|
||||
- Health: `E:\Projects\Odin\scripts\health-raven-capture-mux.cmd`
|
||||
- Restart: `E:\Projects\Odin\scripts\restart-raven-capture-mux.cmd`
|
||||
- Default SSH target: Raven's WireGuard address, `10.77.0.4`
|
||||
|
||||
If Raven is unreachable, Idunn should record the failed health probe and raise
|
||||
the normal keepalive decision instead of pretending the daemon is healthy.
|
||||
|
|
@ -1,130 +0,0 @@
|
|||
# Raven And Eve Ingest
|
||||
|
||||
## Objective
|
||||
|
||||
Ingest Raven screen capture plus Eve camera and microphone capture into Mimir,
|
||||
then let the existing synchronization hub place those samples in rolling buffers
|
||||
beside Scarlett audio and local camera sources.
|
||||
|
||||
## Authority Map
|
||||
|
||||
- Owner: `Mimir.Runtime` owns decoded samples and rolling-buffer placement.
|
||||
- Inputs: Raven currently sends H.264/MPEG-TS over the SRT bridge to Starfire
|
||||
port `5200`; Eve currently sends camera frame-events over WebSocket port
|
||||
`8793`; Eve currently sends microphone frame-events over WebSocket port
|
||||
`8794`; Scarlett/ASIO carries audio and sync evidence. The target Verse shape
|
||||
is CultMesh reliable UDP: typed stream-frame envelopes plus bounded body
|
||||
shards for media, with compact state/cursors/backpressure beside them.
|
||||
- Outputs: `raven-display`, `eve-camera`, and `eve-mic` emit
|
||||
`MimirStreamSample` records with video or audio descriptors.
|
||||
- Derived state: network arrival timestamps are capture metadata only. Raven
|
||||
audio-in-Scarlett remains the timing witness for global alignment.
|
||||
- Forbidden writers: OBS, FFmpeg sender clocks, WebSocket receive time, SRT
|
||||
arrival time, and CultMesh packet arrival time do not own final sync truth.
|
||||
- Cut line: EveCanvas owns Eve's sensors directly. External iOS streaming apps
|
||||
are fallback tools, not the current architecture. SRT/WebSocket bridge
|
||||
transports are diagnostics and compatibility edges once the reliable-UDP
|
||||
Verse media lane exists.
|
||||
|
||||
## Starfire Receiver
|
||||
|
||||
Use the Raven/Eve runtime config:
|
||||
|
||||
```powershell
|
||||
$env:MIMIR_RUNTIME_CONFIG = "E:\Projects\Mimir\config\mimir-runtime.raven-eve.example.json"
|
||||
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --seconds 5
|
||||
```
|
||||
|
||||
When both senders are active, require sample arrival:
|
||||
|
||||
```powershell
|
||||
$env:MIMIR_RUNTIME_CONFIG = "E:\Projects\Mimir\config\mimir-runtime.raven-eve.example.json"
|
||||
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --seconds 5 --require-samples
|
||||
```
|
||||
|
||||
The config starts one FFmpeg listener and two Eve sensor receiver processes:
|
||||
|
||||
- `raven-display`: `srt://0.0.0.0:5200?mode=listener&latency=120000&timeout=5000000`
|
||||
- `eve-camera`: `ws://0.0.0.0:8793/eve/camera`
|
||||
- `eve-mic`: `ws://0.0.0.0:8794/eve/mic`
|
||||
|
||||
The Raven listener decodes to raw BGRA frames on stdout. The Eve receivers emit
|
||||
typed JSON frame-events from EveCanvas camera/mic packets. Mimir stores all of
|
||||
them in the normal synchronization buffers.
|
||||
|
||||
## Raven Sender
|
||||
|
||||
Run this on Raven from the logged-in desktop session:
|
||||
|
||||
```powershell
|
||||
E:\Projects\Mimir\scripts\start-raven-screen-capture-sender.ps1 -TargetHost 192.168.1.66 -Port 5200
|
||||
```
|
||||
|
||||
The script uses `gdigrab` plus `h264_nvenc`, then calls Starfire's SRT listener.
|
||||
Do not launch the capture itself through a blind SSH session unless the desktop
|
||||
capture path has been proven there; `gdigrab` captures the interactive Windows
|
||||
desktop, and service/session boundaries can turn screen capture into a black
|
||||
box with excellent logs.
|
||||
|
||||
Raven sync signal: route Raven audio or a Raven-generated chirp into the
|
||||
Scarlett alongside the microphone. Mimir should treat that audio evidence as
|
||||
the authority for aligning Raven video with the rest of the program.
|
||||
|
||||
## Eve Sensor Sender
|
||||
|
||||
EveCanvas now starts native AVFoundation capture after permission grants:
|
||||
|
||||
- Camera frames are sampled from `AVCaptureVideoDataOutput`, JPEG-compressed,
|
||||
wrapped as `video-frame` events for `eve-camera`, and sent to
|
||||
`ws://192.168.1.66:8793/eve/camera`.
|
||||
- Microphone blocks are captured with `AVAudioEngine`, packed as interleaved
|
||||
Float32 samples, wrapped as `audio-block` events for `eve-mic`, and sent to
|
||||
`ws://192.168.1.66:8794/eve/mic`.
|
||||
|
||||
This is an inspectable first transport, not the final high-throughput shape.
|
||||
When cadence or bandwidth becomes the bottleneck, replace the JSON/base64 event
|
||||
transport with CultMesh reliable-UDP media frames and body shards, optionally
|
||||
carrying VideoToolbox H.264 or another explicit media payload. Preserve the same
|
||||
source ownership and Mimir runtime sample contract.
|
||||
|
||||
Deploy EveCanvas after staging:
|
||||
|
||||
```powershell
|
||||
powershell -ExecutionPolicy Bypass -File E:\Projects\Eve\scripts\stage-to-eve.ps1
|
||||
ssh eve "cd /var/mobile/Projects/Eve && export THEOS=/var/theos && make package install && uiopen --bundleid org.gamecult.evecanvas"
|
||||
```
|
||||
|
||||
## Smoke Test
|
||||
|
||||
The raw-video source has a local synthetic smoke that does not require Raven or
|
||||
Eve:
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --ffmpeg-rawvideo-source-smoke
|
||||
```
|
||||
|
||||
That smoke starts FFmpeg with `testsrc2`, decodes BGRA frames through stdout,
|
||||
and verifies the sample metadata and payload size that Mimir receives.
|
||||
|
||||
The Eve sensor receiver can be checked without launching the whole runtime:
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.EveSensorReceiver\Mimir.EveSensorReceiver.csproj -- --port 8793 --path /eve/camera --source-id eve-camera --type video-frame
|
||||
```
|
||||
|
||||
Periwinkle's Android Eve client publishes binary CultMesh sensor observations:
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.EveSensorReceiver\Mimir.EveSensorReceiver.csproj -- --port 8796 --path /eve/periwinkle --source-id periwinkle-accelerometer --type cultmesh-observation
|
||||
```
|
||||
|
||||
The receiver accepts `mimir.eve_sensor_observation.v1` MessagePack packets on
|
||||
that lane. The device remains an observation source; Mimir owns synchronization
|
||||
and interpretation.
|
||||
|
||||
The same receiver also accepts `mimir.eve_media_observation.v1` MessagePack
|
||||
packets. Periwinkle publishes low-rate camera luma frames as
|
||||
`periwinkle-camera` and microphone PCM blocks as `periwinkle-mic` on the
|
||||
CultMesh observation lane. The legacy iPad Eve camera and mic uplinks have
|
||||
matching binary media document support while their existing `eve-camera` and
|
||||
`eve-mic` ports remain usable.
|
||||
333
docs/research/moonlight-muninn-sleipnir-study-2026-07-16.md
Normal file
|
|
@ -0,0 +1,333 @@
|
|||
# Moonlight lessons for Muninn streaming and Sleipnir input
|
||||
|
||||
Date: 2026-07-16
|
||||
Upstream examined: `moonlight-stream/moonlight-common-c` commit `82e25148549d249a80aaf4afc68e2abb07072edf`
|
||||
|
||||
## Verdict
|
||||
|
||||
Moonlight is strong because it does not ask one transport policy to serve every
|
||||
realtime signal. It gives video, audio, control, and input different loss,
|
||||
ordering, recovery, and freshness rules, then keeps each queue bounded by what
|
||||
the user can still perceive.
|
||||
|
||||
Muninn is not missing all of that machinery. Its media packetizer already owns
|
||||
access-unit boundaries, timestamps and deadlines, video parity, dependency and
|
||||
keyframe metadata, bounded assembly, receiver damage feedback, repair caching,
|
||||
and keyframe requests. Sleipnir already rejects regressing sequence/timestamp
|
||||
pairs, expires reliable HID sends after 25 ms, reports stage latency, and
|
||||
neutralizes stale output.
|
||||
|
||||
The 2026-07-17 field cut closes most of the original mechanism gaps:
|
||||
|
||||
- canonical video and video parity use the unreliable `realtime` lane; typed
|
||||
8+8 GF(256) blocks, selective repair, and decode-chain IDR recovery own loss;
|
||||
- canonical 10 ms PCM audio uses `realtime`, with 4+2 FEC, a 40 ms reorder
|
||||
owner, and short concealment; reliable delivery is reserved for control;
|
||||
- the consumer deadline is 100 ms by default and 250 ms in the Raven field
|
||||
profile; queues, retransmits, repair material, and assemblies derive from it;
|
||||
- receiver pressure actuates the live NVENC encoder through multiplicative
|
||||
backoff and conservative additive recovery; startup begins at half the
|
||||
requested ceiling because fixed video parity can roughly double wire load;
|
||||
- Sleipnir latest state and ordered edges have distinct epoch/sequence and
|
||||
application-ack semantics, including bounded overflow rebase.
|
||||
|
||||
The remaining claim boundary is evidence: direct and impaired Raven-to-
|
||||
Starfire soaks, reconnect repetition, A/V skew measurement, and a live virtual
|
||||
HID exercise. The machine is no longer missing a named owner for these paths;
|
||||
it still has to prove those owners under pressure.
|
||||
|
||||
Do not transplant Moonlight wholesale. Distill its ownership model into the
|
||||
existing typed Muninn/Sleipnir pipeline.
|
||||
|
||||
## Authority map
|
||||
|
||||
### Muninn media
|
||||
|
||||
- Owner: Muninn owns source-local capture, encoding child lifecycle, media
|
||||
packetization, repair material, and sender feedback response.
|
||||
- Inputs: encoded H.264/H.265 access units, encoded audio packets, source media
|
||||
timestamps, configured latency target, and receiver feedback.
|
||||
- Outputs: typed video chunks/parity, audio packets, recovery/keyframe actions,
|
||||
and transport/quality telemetry.
|
||||
- Derived state: RUDP acknowledgements, resend counters, CultCache receipts, OBS
|
||||
local lowering, and operator cards are evidence or delivery substrate. They do
|
||||
not own media deadlines.
|
||||
- Forbidden writers: generic RUDP retry policy must not decide whether expired
|
||||
media is still useful; OBS must not decide stream synchronization; persistence
|
||||
must not become a hot media queue.
|
||||
|
||||
### Sleipnir input
|
||||
|
||||
- Owner: Sleipnir owns selection/remapping and local virtual-HID emission.
|
||||
Muninn owns source-local device sampling and event classification.
|
||||
- Inputs: a discovered Muninn input stream, typed mapping intent, monotonic
|
||||
stream sequence, source-local capture time, and receive time.
|
||||
- Outputs: local virtual-HID state plus latency/freshness receipts.
|
||||
- Derived state: the latest full controller snapshot is replaceable state.
|
||||
Button transitions, text/code points, connect/disconnect, and neutralization
|
||||
are non-replaceable edges until acknowledged or superseded by an explicit
|
||||
state transition.
|
||||
- Forbidden writers: generic reliable queue order must not replay stale analog
|
||||
or motion samples; wall-clock subtraction must not claim one-way latency
|
||||
without a clock model; the telemetry/store path must not repair a missing hot
|
||||
stream.
|
||||
|
||||
## What Moonlight actually does
|
||||
|
||||
### It separates stream semantics
|
||||
|
||||
Moonlight receives video and audio over UDP/RTP-like paths while input travels
|
||||
through its control path (or a legacy TCP socket). The important lesson is not
|
||||
the literal socket choice. It is that the queues have different contracts.
|
||||
Video is frame/dependency oriented; audio is short-block continuity oriented;
|
||||
input is event/state oriented.
|
||||
|
||||
Source: [stream initialization and input transport](https://github.com/moonlight-stream/moonlight-common-c/blob/82e25148549d249a80aaf4afc68e2abb07072edf/src/InputStream.c),
|
||||
[video receive path](https://github.com/moonlight-stream/moonlight-common-c/blob/82e25148549d249a80aaf4afc68e2abb07072edf/src/VideoStream.c), and
|
||||
[audio receive path](https://github.com/moonlight-stream/moonlight-common-c/blob/82e25148549d249a80aaf4afc68e2abb07072edf/src/AudioStream.c).
|
||||
|
||||
### Video recovery is dependency-aware and deadline-shaped
|
||||
|
||||
The video queue groups packets by frame and FEC block, tracks missing data,
|
||||
reconstructs recoverable blocks with Reed-Solomon parity, rejects old/replayed
|
||||
frames, and advances frame authority explicitly. Depacketization can request an
|
||||
IDR when damage makes the predictive chain unsafe. The system repairs useful
|
||||
frames and abandons poisoned dependency chains; it does not wait forever for
|
||||
generic reliable delivery.
|
||||
|
||||
Source: [RTP video queue](https://github.com/moonlight-stream/moonlight-common-c/blob/82e25148549d249a80aaf4afc68e2abb07072edf/src/RtpVideoQueue.c) and
|
||||
[video depacketizer](https://github.com/moonlight-stream/moonlight-common-c/blob/82e25148549d249a80aaf4afc68e2abb07072edf/src/VideoDepacketizer.c).
|
||||
|
||||
### Audio recovery is a small bounded continuity problem
|
||||
|
||||
Moonlight groups audio into fixed FEC blocks, orders packets by RTP sequence,
|
||||
recovers missing shards, detects out-of-order behavior, and changes how long it
|
||||
waits based on observed reordering. When recovery cannot complete in time, it
|
||||
allows a discontinuity and lets the audio decoder's concealment path carry the
|
||||
scar. It does not turn late audio into permanent latency.
|
||||
|
||||
Source: [audio FEC and reorder queue](https://github.com/moonlight-stream/moonlight-common-c/blob/82e25148549d249a80aaf4afc68e2abb07072edf/src/RtpAudioQueue.c) and
|
||||
[packet-loss concealment handoff](https://github.com/moonlight-stream/moonlight-common-c/blob/82e25148549d249a80aaf4afc68e2abb07072edf/src/AudioStream.c).
|
||||
|
||||
### Input reliability is constrained by semantic coalescing
|
||||
|
||||
Moonlight has one send thread and a bounded queue, but it avoids blindly sending
|
||||
every sampled value. Relative mouse motion is accumulated, absolute pointer
|
||||
motion keeps the newest position, pen hover/move keeps the newest compatible
|
||||
event, controller analog state is merged while button changes split batches,
|
||||
and sensor motion keeps the newest value per sensor. A roughly millisecond-scale
|
||||
batching opportunity can reduce latency by preventing ENet queue buildup.
|
||||
|
||||
This is the most important Sleipnir lesson: reliable delivery is safe only after
|
||||
replaceable state has been collapsed and edges have been preserved.
|
||||
|
||||
Source: [input batching and send queue](https://github.com/moonlight-stream/moonlight-common-c/blob/82e25148549d249a80aaf4afc68e2abb07072edf/src/InputStream.c).
|
||||
|
||||
### It exposes failure to the owning layer
|
||||
|
||||
Packet loss, FEC recovery/failure, queue overflow, frame loss, and keyframe need
|
||||
are explicit signals. Moonlight's public documentation also warns that allowing
|
||||
frames to queue for smoothness can create high latency; its balanced mode keeps
|
||||
only about one frame for jitter smoothing. The machine treats queue depth as a
|
||||
product decision, not an incidental container property.
|
||||
|
||||
Source: [Moonlight frame-pacing FAQ](https://github.com/moonlight-stream/moonlight-docs/wiki/Frequently-Asked-Questions#what-do-the-frame-pacing-options-on-the-android-client-mean).
|
||||
|
||||
## Current Muninn/Sleipnir comparison
|
||||
|
||||
| Concern | Moonlight | Muninn / Sleipnir now | Distilled judgment |
|
||||
|---|---|---|---|
|
||||
| Video unit | Encoded frame, FEC blocks, dependency recovery | Encoded access unit, chunks/parity, frame/dependency IDs, deadlines | The conceptual unit is already correct. Keep it. |
|
||||
| Video loss | FEC first; abandon damaged/obsolete work; request IDR | Typed 8+8 block FEC, deadline-bound repair, completed-frame tombstones, IDR actuation | Same ownership pattern; field loss ratios and parity overhead remain tunable. |
|
||||
| Audio loss | Small FEC blocks, reorder-aware wait, decoder concealment | Realtime 4+2 PCM FEC, 40 ms reorder, bounded concealment | Mechanism is closed; Opus FEC/PLC is a later efficiency/quality cut. |
|
||||
| Congestion | Stream-specific queues and feedback behavior | Half-ceiling startup, 15% pressure backoff, bounded additive ramp, fresh media ahead of repair | Closed loop exists; impairment evidence decides its final constants. |
|
||||
| Latency budget | Minimal queueing; frame pacing explicitly trades latency for smoothness | Consumer-owned 100 ms default/250 ms Raven proof; 25 ms HID expiry | Correct owner and bounded values; glass-to-glass measurement remains. |
|
||||
| Input state | Coalesces replaceable motion/sensor/analog values | Epoch-scoped latest state supersedes stale state | Structural rather than polling-accidental. |
|
||||
| Input edges | Button/type changes terminate batches; text is ordered | Ordered edge sequence, application ACK, bounded replay, epoch rebase | Structural; live virtual-HID impairment proof remains. |
|
||||
| Staleness | Queue construction prevents much stale work | Receiver drops regressing timestamps and neutralizes after timeout | Keep the guard, but move freshness authority earlier into enqueue/supersession. |
|
||||
| Timing | RTP/media clocks and local queue timing | source Unix nanoseconds plus receiver instants | Use monotonic source time plus an explicit clock-offset/uncertainty model; do not call raw wall-clock subtraction latency. |
|
||||
| Security | Authenticated encryption in modern stream generations | CultNet typed transport policy varies by lane | Preserve typed schemas, but specify replay/authentication guarantees for hot media and HID lanes. |
|
||||
|
||||
## Recommended cut
|
||||
|
||||
### 1. Define traffic classes before changing transport
|
||||
|
||||
Add one narrow typed delivery policy shared by Muninn and Sleipnir:
|
||||
|
||||
- `state-latest`: analog axes, pointer position, gyro/accelerometer samples;
|
||||
- `edge-ordered`: button/key transitions, text/code points, device lifecycle;
|
||||
- `media-video`: dependency-bearing access units with decode deadlines;
|
||||
- `media-audio`: continuity blocks with playout deadlines;
|
||||
- `control-reliable`: subscriptions, mappings, capability negotiation, and
|
||||
keyframe requests.
|
||||
|
||||
This is not a generic router. It is a finite statement of the five contracts the
|
||||
existing machine already has. Each class must name supersession, ordering,
|
||||
expiry, recovery, and observability.
|
||||
|
||||
### 2. Cut the two-second realtime default
|
||||
|
||||
Derive deadlines from the actual consumer:
|
||||
|
||||
- Sleipnir state-latest: tens of milliseconds, with replacement before send;
|
||||
- input edges: reliable until acknowledged, but compacted against later
|
||||
authoritative state where safe;
|
||||
- interactive video: frame/decode budget, initially test 50–150 ms on LAN;
|
||||
- audio: playout budget, initially test 40–100 ms on LAN;
|
||||
- Mimir's five-second analysis reservoir remains Mimir compute budget, not a
|
||||
network permission to deliver old frames.
|
||||
|
||||
The exact values must be earned by field traces. The ownership change is not
|
||||
optional: transport retry duration becomes derived from the media/input
|
||||
deadline, never the other way around.
|
||||
|
||||
### 3. Make Sleipnir freshness structural
|
||||
|
||||
Split the hot input frame into:
|
||||
|
||||
- a latest-state snapshot with `state_sequence`, source monotonic capture time,
|
||||
and supersedes-through sequence;
|
||||
- an ordered edge list with its own sequence range;
|
||||
- receiver acknowledgement of the highest applied state and edge sequence.
|
||||
|
||||
Muninn may overwrite an unsent latest-state snapshot. It may not overwrite an
|
||||
unacknowledged edge. Sleipnir applies edges in order, applies only the newest
|
||||
state, and neutralizes on a named lease timeout. Manual, remapped, replayed, and
|
||||
live paths must use the same commit primitive.
|
||||
|
||||
### 4. Close Muninn's feedback loop
|
||||
|
||||
Receiver feedback should drive named sender decisions:
|
||||
|
||||
- request/force keyframe after unrecoverable dependency loss;
|
||||
- choose repair only when estimated arrival precedes frame deadline;
|
||||
- tune parity within a bounded profile from recent burst-loss evidence;
|
||||
- reduce encoder bitrate/pacing when queue delay grows;
|
||||
- expose encode, queue, network, assembly, decode, and presentation stages
|
||||
separately.
|
||||
|
||||
Do not add a second congestion controller beside generic RUDP and hope the two
|
||||
argue productively. Media policy owns the send budget; RUDP exposes delivery
|
||||
mechanics and obeys the deadline.
|
||||
|
||||
### 5. Prove it on the layer the user feels
|
||||
|
||||
Build one loss/jitter/reorder test harness and record timelines for:
|
||||
|
||||
- controller press and release during 1%, 5%, and burst packet loss;
|
||||
- high-rate analog/gyro motion under an artificial sender stall;
|
||||
- video dependency loss and keyframe recovery;
|
||||
- audio burst loss and concealment;
|
||||
- queue growth under bitrate oversubscription;
|
||||
- disconnect/reconnect and stale-session replay;
|
||||
- clock offset and drift between Muninn and Sleipnir hosts.
|
||||
|
||||
Acceptance should bound p50/p95/p99 capture-to-HID age, lost/duplicated edges,
|
||||
time-to-neutral, glass-to-glass video age, audio discontinuities, A/V skew, and
|
||||
recovery time. Final-state-only tests will politely conceal the corpse.
|
||||
|
||||
## What not to copy
|
||||
|
||||
- Do not copy Moonlight's C implementation or protocol packets into GameCult
|
||||
code without an explicit GPLv3 licensing decision. Architectural study is not
|
||||
license laundering. See [Moonlight's license](https://github.com/moonlight-stream/moonlight-common-c/blob/82e25148549d249a80aaf4afc68e2abb07072edf/LICENSE.txt).
|
||||
- Do not replace typed CultMesh discovery, command state, or ownership with
|
||||
RTSP/SDP or a Moonlight compatibility stack.
|
||||
- Do not make ENet the new universal answer. Moonlight's strength comes from
|
||||
semantic queues above the carrier, not from the carrier's brand name.
|
||||
- Do not merge input, audio, and video into one ordered reliable stream. One
|
||||
lost video fragment must not head-of-line block a button release or audio
|
||||
packet.
|
||||
- Do not treat FEC as magic. Parity is useful only when its overhead and recovery
|
||||
time fit the deadline.
|
||||
|
||||
## Distilled doctrine
|
||||
|
||||
Realtime strength is not “reliable UDP.” It is knowing which truth survives
|
||||
loss:
|
||||
|
||||
- the newest state replaces old state;
|
||||
- an edge remains until applied or explicitly superseded;
|
||||
- a media packet is valuable only while its frame or playout deadline is alive;
|
||||
- predictive video damage is healed by recovery or a new reference, not by
|
||||
delivering every corpse;
|
||||
- queues are latency budgets made physical;
|
||||
- feedback that cannot change sender behavior is telemetry, not control.
|
||||
|
||||
Muninn and Sleipnir already have enough organs to become strong. The next work
|
||||
is to give each signal one explicit survival law and make the transport obey it.
|
||||
|
||||
## 2026-07-17 LAN field results
|
||||
|
||||
The Moonlight-derived ownership changes now have Raven-to-Starfire field
|
||||
evidence over LAN, with Raven processes launched only through hidden scheduled
|
||||
tasks:
|
||||
|
||||
- Muninn video uses independently recoverable block parity plus bounded repair.
|
||||
During a seeded 1% datagram-loss run, the OBS receiver advanced from 2 to 12
|
||||
video assembly expiries while forwarding 14,400 additional payload groups:
|
||||
99.93% completed without expiry. Audio reported zero failures, skips, or stale
|
||||
packets in the same interval. The impairment proxy forwarded 2,147,356 of
|
||||
2,169,105 datagrams with zero queue overflow.
|
||||
- Sleipnir latest-state delivery remained live through seeded 1% loss. Cutting
|
||||
the proxy caused the non-neutral virtual Xbox pad to neutralize after 1.56 s,
|
||||
owned by the named stale-input lease; restoring the hidden proxy reconnected
|
||||
and resumed state without operator action.
|
||||
- A continuous 64-packet receive drain was retaining the newest input for about
|
||||
48-57 ms before HID application. Bounding that drain to 2 ms reduced live
|
||||
receiver-owned receive-to-apply time to typically 69-204 microseconds. A
|
||||
witnessed 16 ms spike remains to characterize.
|
||||
- Starfire and Raven wall clocks differ by roughly 2.38 s. Sleipnir no longer
|
||||
publishes their raw subtraction as one-way latency. Its surface reports that
|
||||
source-clock latency is unavailable without a clock model and exposes only
|
||||
monotonic receiver-owned stage durations.
|
||||
|
||||
These results clear the initial 99.9% media-completion and disconnect-safety
|
||||
gates at 1% independent loss. They do not yet prove glass-to-glass latency,
|
||||
A/V skew, ordered quick-tap delivery under loss, 5%/burst behavior, or a long
|
||||
mixed-impairment soak. Those remain acceptance gates, not footnotes.
|
||||
|
||||
The field comparison also exposed a generic transport invariant Moonlight's
|
||||
long-running sessions force us to respect: bounded wire identifiers must wrap
|
||||
deliberately. CultNet's fragmented-message ID previously saturated at the
|
||||
maximum `u16`, so fragmented audio became unreassemblable after the boundary.
|
||||
Wrap-to-one plus a boundary test removed the repeatable minute-eight collapse;
|
||||
a subsequent 10-minute direct LAN run remained clean across the boundary.
|
||||
|
||||
## 2026-07-17 ordered-input closure
|
||||
|
||||
The quick-tap gate exposed four authorities that generic RUDP reliability had
|
||||
blurred together:
|
||||
|
||||
- A Windows sleep requested at two milliseconds could still miss a short tap.
|
||||
Muninn now observes XInput on a dedicated high-resolution worker; transport,
|
||||
discovery, serialization, and logging cannot delay capture.
|
||||
- Global cumulative ACKs could not retire a reliable packet arriving more than
|
||||
32 realtime sequence numbers late, and reliable duplicates were not ACKed
|
||||
after application deduplication. CultNet now directly acknowledges the late
|
||||
packet and acknowledges reliable duplicates. This collapsed a runaway field
|
||||
trace from hundreds of thousands of resend datagrams to a bounded flow.
|
||||
- Muninn used semantic application ACK as permission to send the next edge.
|
||||
One delayed ACK therefore serialized the entire controller. It now sends a
|
||||
bounded window of captured edges; CultNet owns retransmission, Sleipnir's
|
||||
epoch/sequence cursor owns ordering, and semantic ACK only retires history.
|
||||
- Sleipnir published CultMesh/Eve and Idunn state synchronously on the actuator
|
||||
thread. Those writes produced 0.7-1.8 second visible button holds. A bounded
|
||||
coalescing publisher worker now owns control-plane projection, leaving the
|
||||
input thread to receive, order, actuate, and acknowledge.
|
||||
|
||||
The resulting first-hot-plug field proof generated 100 taps with a 16 ms source
|
||||
press and 84 ms gap. Under seeded 1% independent loss, every layer observed
|
||||
exactly 100 presses and 100 releases; Raven-visible durations were 24-57 ms.
|
||||
Under eight consecutive dropped packets every 200 datagrams, every layer again
|
||||
observed exactly 100 presses and 100 releases; visible durations were 24-40 ms,
|
||||
the final virtual pad was neutral, and the impairment proxy reported no queue
|
||||
overflow. Raven ran through hidden scheduled tasks over `192.168.1.0/24`; no
|
||||
WireGuard route participated. Both daemons used the Eve/Aetheria/VoidBot
|
||||
experimental CultLib lineage pinned at `8965f3c0`.
|
||||
|
||||
This closes ordered digital edges at one 60 Hz source frame. It does not prove
|
||||
sub-frame synthetic ViGEm pulses: eight-millisecond state-only pulses can fall
|
||||
between XInput snapshots, and `XInputGetKeystroke` was rejected after it emitted
|
||||
stale events from a reused user slot. Analog motion under a sender stall,
|
||||
mixed media/input endurance, glass-to-glass video age, and A/V skew remain open.
|
||||
100
docs/research/moonlight-reliability-acceptance-2026-07-16.md
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
# Moonlight-level reliability acceptance ledger
|
||||
|
||||
This ledger distinguishes implemented mechanisms from evidence that the actual
|
||||
Raven-to-Starfire program timeline survives pressure. Passing unit tests is not
|
||||
permission to call the field path done.
|
||||
|
||||
## Required contracts
|
||||
|
||||
| Contract | Current authority | Evidence | Status |
|
||||
|---|---|---|---|
|
||||
| Experimental CultMesh/CultNet lineage | Odin vendored CultLib snapshot `8965f3c0` | Snapshot provenance plus CultNet/Muninn/Sleipnir suites | proved locally |
|
||||
| Low-delay NVENC | Muninn controllable encoder | D3D11 desktop capture, no B frames/lookahead, two-frame VBV, clean Annex-B decode | proved locally |
|
||||
| Consumer deadline owns media lifetime | Muninn queues, repair cache, Mimir assembly/reorder | 100 ms default/250 ms Raven profile, bounded queues/cache/reliable expiry, late-frame repair refusal | proved structurally |
|
||||
| Recoverable video loss avoids decoder reset | Mimir V4 block FEC/repair, Odin repair cache | Exhaustive 2..80 chunk, burst 1..8 tests in Rust/C++; completed-frame tombstones | proved locally |
|
||||
| Decoder-chain loss forces immediate IDR | Muninn feedback owner and native encoder | Live 2560x1440 command produced IDR at frame 22 under a 600-frame GOP | proved locally |
|
||||
| Congestion changes encoder output in-session | Muninn AIMD controller and native encoder | Half-ceiling start and protected cap; live 12-to-6 Mbps reconfiguration produced clean IDR | proved locally |
|
||||
| Audio survives small erasure bursts | Odin 4+2 GF(256), Mimir FEC/reorder/concealment | Exhaustive one/two-erasure Rust and C++ tests; production DLL build | proved locally |
|
||||
| Latest input state supersedes stale state | Muninn epoch/state sequence, Sleipnir cursor | Duplicate/reorder/stale epoch tests | proved locally |
|
||||
| Button edges are not lost or retained forever | Muninn replay/app ACK/epoch rebase, Sleipnir ordered cursor | Quick-tap, ACK, overflow rebase, sparse-gap tests | proved locally |
|
||||
| Raven runtime is LAN-only and non-interrupting | Hidden Raven scheduled task, direct LAN SSH/media | One coherent Muninn/WASAPI/NVENC/FFmpeg tree; no WireGuard route; `-WindowStyle Hidden` | proved operationally |
|
||||
| Real field path survives impairment and soak | Raven sender, Starfire receiver/OBS, `cultnet-impair` | Direct LAN clean run 05:34:17–05:44:37: 79,200 records, 697,450,085 bytes, 18,588 recovered shards, zero decode/assembly/media/audio failures and zero sender capacity eviction; proxy matrix and long soak remain | **direct clean passed** |
|
||||
|
||||
## Field acceptance matrix
|
||||
|
||||
Run the real capture body, production receiver plugin, decoder, audio output,
|
||||
and Sleipnir virtual input through the seeded profiles under
|
||||
`E:\Projects\Odin\tests\realtime-impairment`. Observe the rendered/audio/input
|
||||
timeline, not only proxy CSV counters.
|
||||
|
||||
| Profile | Required result |
|
||||
|---|---|
|
||||
| clean, 10 minutes | no decoder reset, audio discontinuity, input edge loss, queue growth, or timestamp drift; video p99 delivery under 50 ms |
|
||||
| iid 1%, 10 minutes | no input edge loss; audio FEC recovers eligible holes; at least 99.9% video frames presented; media p99 under 80 ms |
|
||||
| iid 3%, 5 minutes | bounded degradation; at least 99% video frames presented; decoder recovery within 250 ms; no stuck controls |
|
||||
| burst 4 and burst 8 | parity/repair handles eligible damage; unrecoverable dependency loss produces one requested IDR and recovery within 250 ms |
|
||||
| reorder and duplicate | no duplicate input action, stale state regression, or unbounded assembly/reorder storage |
|
||||
| delay/jitter | AIMD reduces bitrate before sustained deadline loss and recovers no faster than one step per two stable seconds |
|
||||
| stall 250 ms | obsolete media expires; queues remain bounded; input latest state converges within 35 ms after release |
|
||||
| reconnect | old input epoch cannot act; neutral/current state within 100 ms; decoded video within 500 ms |
|
||||
| mixed 60-minute soak | stable memory/handle/thread counts and no progressive A/V drift |
|
||||
|
||||
The first seeded 1% proxy run is a failed acceptance result, not a pass: 987 of
|
||||
95,741 datagrams were dropped; in roughly one minute the receiver recorded 38
|
||||
expired video assemblies and 56 skipped audio packets while remaining bounded
|
||||
and avoiding decoder reset. CultNet fragmentation currently sits below
|
||||
application-level FEC, so one lost transport fragment discards an entire FEC
|
||||
shard. The next reliability cut is to make protected media shards fit one wire
|
||||
datagram or move fragmentation above the erasure-code boundary.
|
||||
|
||||
## Field scars
|
||||
|
||||
- A fragmented CultNet RUDP session failed repeatably after roughly eight to
|
||||
nine minutes because the `u16` fragment-set ID used saturating increment and
|
||||
became permanently stuck at 65,535. Every later fragmented audio document
|
||||
then collided in one receiver reassembly key. The allocator now wraps from
|
||||
65,535 to 1, with a boundary test. The clean field run above crossed that
|
||||
boundary without a single audio reorder, skip, stale packet, or failure.
|
||||
- Audio and video producer ingress now use separate bounded channels. Video
|
||||
capture can no longer block audio ingestion before the audio-priority send
|
||||
scheduler sees it. Capacity-eviction counters proved zero during the clean
|
||||
acceptance run.
|
||||
|
||||
## 2026-07-17 bounded-media field cut
|
||||
|
||||
The live path now keeps every protected shard in one datagram, sends AAC ADTS
|
||||
in fixed 864-byte 4+2 audio blocks, uses 768-byte video shards with 8+4 block
|
||||
FEC, and begins each decoder epoch at a complete IDR. The sender admits video
|
||||
to its repair cache when the frame enters the queue, accepts the production
|
||||
CultNet `realtime` channel, and can service bounded repair requests at
|
||||
64/96/128 ms in batches of up to eight chunks.
|
||||
|
||||
Seeded Raven-to-Starfire evidence:
|
||||
|
||||
- iid 1%: 802 of 78,406 datagrams dropped; zero decode drop, video expiry,
|
||||
audio failure, or audio skip; 1,800 video frames recovered;
|
||||
- burst-4: 1,436 of 71,910 datagrams dropped; zero expiry or audio skip across
|
||||
1,800 forwarded frames;
|
||||
- reorder: 2,872 reorder events; zero expiry or audio skip across 900 frames;
|
||||
- iid 3%: 2,564 of 85,731 datagrams dropped; three of 5,400 frames expired,
|
||||
zero audio skip (99.94 percent video delivery);
|
||||
- burst-8: 190 chunks repaired; one of 1,800 frames expired, zero audio skip;
|
||||
- 250 ms stall: 420 datagrams withheld, one frame expired, 68 chunks repaired,
|
||||
an IDR was requested, and another 900 frames arrived without a stuck decoder
|
||||
or audio skip.
|
||||
|
||||
Delay/jitter produced one expiry in 900 frames. Duplicate, reconnect, longer
|
||||
iid runs, and the 60-minute mixed soak remain open acceptance rows.
|
||||
|
||||
Field scar: production video uses the `realtime` channel, but the repair cache
|
||||
previously admitted only legacy `media` payloads. Synthetic tests built the
|
||||
legacy shape and certified a repair path that was empty in production. The
|
||||
production-channel regression test now owns this boundary.
|
||||
|
||||
## Completion rule
|
||||
|
||||
Moonlight-level performance is reasonably expected only after all local rows
|
||||
remain green and the real field matrix passes twice: once on direct LAN and once
|
||||
on the deployed route actually used during streaming. Failure should change the
|
||||
owning deadline, FEC, encoder, or input authority—not add a repair loop around
|
||||
the symptom.
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
# Mimir Scene Editor Control Surface
|
||||
|
||||
Mimir now has a runtime-owned scene editor state for arranging the visible
|
||||
presentation inputs inside the Fensalir-hosted Mimir window. This editor is not
|
||||
the OBS program output. It is an operator surface for placing source panels,
|
||||
text panels, and imported-model placeholders before later renderer cuts turn
|
||||
those nodes into final composited pixels or volumetric field claims.
|
||||
|
||||
## Authority Map
|
||||
|
||||
- Owner: `MimirSceneEditorState` owns editor camera, scene nodes, selected node,
|
||||
visibility, lock state, 2D-plane transform, and active gizmo mode.
|
||||
- Inputs: rolling video buffers provide sensor-feed nodes; Fensalir `InputState`
|
||||
provides mouse/key edits; the Mimir UI provides visibility, reset, transform,
|
||||
text, and model-path commands.
|
||||
- Outputs: `MimirRuntime` reads the editor state into the Fensalir frame camera,
|
||||
spline outlines, selection gizmos, and SDF handle markers.
|
||||
- Derived state: the hierarchy text, selected-node readout, editor splines, and
|
||||
handle SDF objects are derived from the editor graph. They do not own node
|
||||
truth.
|
||||
- Forbidden writers: program-composite controls, OBS source endpoints, bridge
|
||||
scripts, and FieldEvidence surface intents do not decide editor transforms.
|
||||
- Shared paths: UI buttons/sliders and mouse-drag gizmos all commit through
|
||||
`MimirSceneEditorState`; reset returns to each node's stored default
|
||||
transform.
|
||||
- Deletion line: do not add a separate transform cache in the renderer or OBS
|
||||
path. Fensalir may later render richer node types, but it should consume this
|
||||
graph or a deliberate successor, not invent a competing editor graph.
|
||||
|
||||
## Current Controls
|
||||
|
||||
- `Mimir Editor` panel exposes the scene graph, selected-node visibility and
|
||||
lock toggles, transform reset, camera reset, and transform sliders.
|
||||
- Active gizmo modes are `Grab`, `Rotate`, and `Resize`; keys `1`, `2`, and `3`
|
||||
switch modes, and left-drag applies the active mode to the selected node.
|
||||
- Sensor-feed nodes are created from live video rolling buffers.
|
||||
- SDF text panels and model placeholders can be added from the panel.
|
||||
- `--scene-editor-smoke` proves graph creation, visibility toggles, transform
|
||||
reset, mouse-drag transform edits, text-node creation, model placeholder
|
||||
import, and derived gizmo geometry.
|
||||
|
||||
## Renderer Hooks Still Open
|
||||
|
||||
- World SDF text: Fensalir currently has DirectWrite overlay text and SDF object
|
||||
proxies. Scene text needs a renderer-owned SDF/MSDF glyph atlas plus billboard
|
||||
or mesh text lowering before text glyphs are truly drawn inside the scene.
|
||||
- Imported models: Fensalir resource contracts already know mesh packages, but
|
||||
selected mesh draw/material lowerings are still future work. The current
|
||||
import control creates graph nodes with paths; it does not decode ASSIMP
|
||||
meshes or upload vertex/index resources yet.
|
||||
- Pixel-accurate gizmo hit-testing: current gizmo edits apply to the selected
|
||||
node. Fensalir needs an editor hit-test path before handles can be clicked
|
||||
directly in world space.
|
||||
|
|
@ -1,127 +0,0 @@
|
|||
# Self-Hosted Live Edge
|
||||
|
||||
Mimir's self-hosted broadcast path keeps OBS optional. Starfire owns the final
|
||||
program frame and audio mix; Yggdrasil owns public byte distribution.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
A["Fensalir program texture"] --> B["Starfire NVENC encode"]
|
||||
C["Faust/Mimir mix"] --> B
|
||||
B --> D["SSH TCP forward"]
|
||||
D --> E["Yggdrasil localhost RTMP ingest"]
|
||||
E --> F["HLS segment ring"]
|
||||
F --> G["streampixels.gamecult.org static HTTPS"]
|
||||
G --> H["gamecult.org/livestream static viewer"]
|
||||
```
|
||||
|
||||
## Authority Map
|
||||
|
||||
- Owner: `Mimir.Broadcast` owns the Starfire-side encoded push command.
|
||||
- Inputs: final program media, FFmpeg/NVENC capability, and the SSH-forwarded
|
||||
RTMP endpoint.
|
||||
- Outputs: one CBR H.264/AAC RTMP stream to Yggdrasil.
|
||||
- Derived state: Yggdrasil's HLS playlist and segments are derived from that
|
||||
encoded stream. They do not own composition, timing, or transcoding.
|
||||
- Forbidden writers: Yggdrasil must not transcode the program stream in the
|
||||
first live edge. OBS is not the primary broadcast owner.
|
||||
- Shared paths: synthetic smokes and real program pushes use the same RTMP URL:
|
||||
`rtmp://127.0.0.1:11935/live/mimir`.
|
||||
- Deletion line: if a later CDN or WebRTC edge appears, keep this path as the
|
||||
simple origin proof or delete it. Do not keep two public stream authorities.
|
||||
|
||||
## First Smoke
|
||||
|
||||
On Starfire, with an SSH local forward active to Yggdrasil's localhost RTMP
|
||||
port:
|
||||
|
||||
```powershell
|
||||
dotnet run --project .\src\Mimir.Broadcast\Mimir.Broadcast.csproj -- --smoke
|
||||
dotnet run --project .\src\Mimir.Broadcast\Mimir.Broadcast.csproj -- --print-command
|
||||
```
|
||||
|
||||
The default push target is:
|
||||
|
||||
```text
|
||||
rtmp://127.0.0.1:11935/live/mimir
|
||||
```
|
||||
|
||||
The public HLS playlist is served on the existing Yggdrasil-routed
|
||||
StreamPixels domain:
|
||||
|
||||
```text
|
||||
https://streampixels.gamecult.org/mimir/live/hls/mimir.m3u8
|
||||
```
|
||||
|
||||
The static viewer lives on the root GameCult site:
|
||||
|
||||
```text
|
||||
https://gamecult.org/livestream
|
||||
```
|
||||
|
||||
## Local Recording Before Public Push
|
||||
|
||||
Before pushing a live program to Yggdrasil, record the same simple stream-proof
|
||||
shape locally and judge sync from the artifact:
|
||||
|
||||
```powershell
|
||||
powershell -ExecutionPolicy Bypass -File .\scripts\record-stream-proof-output.ps1 -StartRavenSender -DurationSeconds 20
|
||||
```
|
||||
|
||||
The recording harness listens for a Raven muxed A/V SRT feed on a private test
|
||||
port, overlays Kiyo Pro as picture-in-picture, mixes Raven Realtek loopback with
|
||||
Starfire's Scarlett capture, applies conservative FFmpeg denoise filters, and
|
||||
writes an MP4 under `artifacts\runtime\stream-proof\`.
|
||||
|
||||
The current routing contract is explicit:
|
||||
|
||||
- Starfire Realtek/default render is the chirp/speaker emission lane.
|
||||
- Raven Realtek/default render loopback is the co-streamer game/program lane.
|
||||
It is packaged with Raven's NVENC video feed back to Starfire.
|
||||
- Starfire Scarlett carries the hero mic capture and monitored local mix for
|
||||
judgment. Raven's Scarlett is only routing the remote shotgun into Starfire
|
||||
Scarlett, currently on channel 1.
|
||||
- The proof recording is allowed to mix those lanes for human sync judgment, but
|
||||
it must not collapse their authority. Starfire Realtek proves chirp emission,
|
||||
Raven Realtek proves the co-streamer/game program path, and Scarlett proves
|
||||
the hero mic/shotgun capture path.
|
||||
|
||||
This is a bridge recording, not public broadcast truth. It exists so humans can
|
||||
judge audio/video sync and tell Mimir which offsets to apply before the RTMP/HLS
|
||||
edge is involved. Offset knobs are explicit:
|
||||
|
||||
```powershell
|
||||
powershell -ExecutionPolicy Bypass -File .\scripts\record-stream-proof-output.ps1 `
|
||||
-StartRavenSender `
|
||||
-RavenOffsetSeconds 0.000 `
|
||||
-KiyoOffsetSeconds 0.000 `
|
||||
-MicOffsetSeconds 0.000
|
||||
```
|
||||
|
||||
If Windows firewall blocks direct SRT/UDP during a local proof, run the Raven
|
||||
leg through an SSH TCP tunnel instead:
|
||||
|
||||
```powershell
|
||||
Start-Process ssh.exe -ArgumentList '-o ExitOnForwardFailure=yes -N -L 6204:127.0.0.1:5204 -l "madman''s lullaby" 192.168.1.84' -WindowStyle Hidden
|
||||
|
||||
powershell -ExecutionPolicy Bypass -File .\scripts\record-stream-proof-output.ps1 `
|
||||
-StartRavenSender `
|
||||
-RavenSenderListens `
|
||||
-RavenSenderTransport tcp-listener `
|
||||
-RavenInputOverride tcp://127.0.0.1:6204
|
||||
```
|
||||
|
||||
That fallback keeps the proof private to the LAN/SSH path and avoids changing
|
||||
machine firewall policy just to capture a judging artifact.
|
||||
|
||||
## Scaling Rule
|
||||
|
||||
Viewer capacity is approximately:
|
||||
|
||||
```text
|
||||
available Yggdrasil egress / encoded bitrate
|
||||
```
|
||||
|
||||
Micro-optimizations count only when they improve target-host measurements:
|
||||
served Mbps, p95 segment latency, CPU, memory, syscalls, or stall rate on
|
||||
Yggdrasil. The first edge uses nginx static serving and kernel file paths so
|
||||
the hot path is byte distribution, not application code.
|
||||
|
|
@ -10,10 +10,7 @@ flowchart TD
|
|||
C["Leap stereo IR driver"] --> B
|
||||
B --> D["native reservoir handles"]
|
||||
D --> E["Fensalir GPU feature extraction"]
|
||||
E --> J["per-sensor LED curve solve"]
|
||||
E --> F["cross-view matching + flow"]
|
||||
J --> K["camera rig calibration solver"]
|
||||
K --> F
|
||||
F --> G["surface/material claims"]
|
||||
G --> H["brush/splat render budget"]
|
||||
H --> I["Spout2/program video"]
|
||||
|
|
@ -26,51 +23,9 @@ flowchart TD
|
|||
- Process capture is a bridge edge, not the local six-camera foundation.
|
||||
- Fensalir owns GPU extraction, fusion, material fitting, and render budgeting.
|
||||
- Runtime buffers own retention and stream health, not scene reconstruction.
|
||||
- LED strips are active calibration evidence. Each camera publishes ordered
|
||||
bright-curve observations in its own local frustum/clipspace; only the global
|
||||
residual calibration owner may fit those frustums into scene-space camera
|
||||
intrinsics/extrinsics.
|
||||
- Stable LED index identity requires temporal/color/address coding or another
|
||||
correspondence source; uncoded identical lights are spline constraints, not
|
||||
metric depth authority.
|
||||
- PS Move controllers are active tracked-marker witnesses. USB HID owns
|
||||
identity, buttons, IMU/report topology, and later LED/rumble control; the
|
||||
glowing sphere is visual calibration evidence only when observed by cameras
|
||||
and fused by the global residual owner.
|
||||
- Exposure/gain sweeps optimize recovered spline quality: detected LED count,
|
||||
curve coverage, spacing coherence, smoothness, exposure fitness, and
|
||||
saturation penalty. Bright clipped blobs are not good evidence.
|
||||
- The live LED sweep owner is `Mimir.BufferSmoke --led-spline-live-score` for
|
||||
now: it opens direct sensor feeds, applies KS/UVC manual exposure/gain where
|
||||
supported, scores actual luma frames, and reports the best setting per
|
||||
source. PS3 Eye and Leap-specific controls still need native control cuts.
|
||||
- The current all-sensor GPU proof is
|
||||
`config/mimir-runtime.perfect-machine.local.json` plus
|
||||
`Mimir.BufferSmoke --perfect-machine-live-smoke`: Leap, both PS3 Eyes, Kiyo
|
||||
Pro, and regular Kiyo all enter the runtime as Fensalir texture leases. The
|
||||
smoke requires five live camera Texture2D resources, one Leap stereo-depth
|
||||
lowering, one disparity SurfacePage, and one derived point-cloud Mesh.
|
||||
Camera Feature/SensorObservation claims now plan through Fensalir's
|
||||
`GpuSensorFusion` backend instead of deferring. The current compute dispatch
|
||||
samples LeapStereoIr, both Bayer8 PS3 Eyes, Kiyo Pro YUY2, and regular Kiyo
|
||||
YUY2. Fensalir binds YUY2 through the documented `R8G8B8A8_UNORM` SRV view
|
||||
over `DXGI_FORMAT_YUY2` and decodes `Y0/U/Y1/V` in compute before scoring
|
||||
features.
|
||||
- PS3 Eyes own high-rate sparse tracking evidence before they own depth:
|
||||
`MimirSparseFeatureTracker` turns live Bayer/luma frames into stable
|
||||
sensor-local feature tracks, and `MimirFeatureTrackFieldCandidate` lowers
|
||||
them as camera Feature claims. This CPU path is a diagnostic bootstrap; the
|
||||
production hot path belongs in Fensalir D3D12 feature extraction.
|
||||
- The first camera-rig solver fits bounded camera translation updates from a
|
||||
scene-anchored LED curve by minimizing ray-to-curve distance. Rotation,
|
||||
intrinsics, and distortion are future owners.
|
||||
- Once camera frustums are coherent, every detected surface candidate can be
|
||||
resampled across the synchronized view set before Fensalir resolves it into
|
||||
spatiotemporal surface/splat claims.
|
||||
|
||||
## Next Cut
|
||||
|
||||
Replace provisional Leap projection constants with calibrated intrinsics and
|
||||
add the global residual owner that consumes Leap surface errors, LED spline
|
||||
observations, and later PS3 Eye/Kiyo/Eve feature tracks through one calibration
|
||||
commit path.
|
||||
Implement one concrete direct capture driver, feed `MimirVideoFrameDescriptor`
|
||||
samples into `MimirVideoCaptureDriverSource`, and measure sustained cadence over
|
||||
the default five-second window.
|
||||
|
|
|
|||
|
|
@ -1,169 +0,0 @@
|
|||
# Mimir Verse Service Contract
|
||||
|
||||
Mimir is a Verse service for the measured room. It accepts local and network
|
||||
sensors, publishes typed observations and capture pages, asks Fensalir to lower
|
||||
field evidence into program surfaces, and exposes operator controls through
|
||||
Eve/CultUI surfaces. OBS, browser dashboards, JSON logs, and screenshot streams
|
||||
are lowerings or witnesses. They are not canonical presentation or state owners.
|
||||
|
||||
## Owner Map
|
||||
|
||||
- Mimir service owner: configuration, launch, calibration truth, sensor
|
||||
identity, stream identity, synchronization state, Well publication, recorder
|
||||
receipts, and service contract.
|
||||
- `Mimir.Runtime`: bounded rolling buffers, source descriptors, source polling,
|
||||
direct native ingest, audio synchronization analysis, and cached read models.
|
||||
- Native sensor workers: device reads, closest-to-device timestamps, native or
|
||||
GPU handles, unavoidable copy counts, and per-device actuator commands.
|
||||
- `Mimir.Well`: typed live pages, configured composite state, timing pressure,
|
||||
feature signals, capture-body publication, and the append-side observation
|
||||
ledger.
|
||||
- `Mimir.VerseRecorder`: run session receipts, body indexing, replay witnesses,
|
||||
and `.cc`/CultCache-compatible export for offline reconstruction.
|
||||
- `Mimir.FensalirDaemon`: reservoir job selection, queue pressure, GPU worker
|
||||
ownership, program/reservoir surfaces, and daemon state.
|
||||
- Fensalir: D3D12 resource resolution, field claims, selected lowerings,
|
||||
temporal guide lanes, reservoir resolve, and program presentation.
|
||||
- Faust/native DSP: hot audio alignment, sample movement, suppression,
|
||||
separation, spatialization, and program stems.
|
||||
- Eve/CultUI lowerings: operator interaction and display of provider-owned
|
||||
state. They submit commands to owners; they do not own synchronization,
|
||||
reservoir jobs, OBS composition, or calibration truth.
|
||||
- Odin: discovery of Mimir's published CultMesh namespaces and Eve surfaces.
|
||||
|
||||
## CultCache Witness Contract
|
||||
|
||||
Durable state should be typed CultCache `.cc`, or CultCache-compatible with a
|
||||
`.cc` witness/export during migration. The witness set is:
|
||||
|
||||
| State family | Owner | Durable shape |
|
||||
| --- | --- | --- |
|
||||
| Sensor state | `Mimir.Runtime` plus native workers | Source inventory, device identity, driver profile, clock domain, format, actuator capability, native handle class, copy count, and last health. |
|
||||
| Media state | `Mimir.Well` / Fensalir | Capture page metadata, resource keys, body refs, fence refs, page timing, configured composites, and program surface refs. |
|
||||
| Observation state | `Mimir.Runtime` / Well | Typed audio/video/IMU/feature/path observations with source id, time range, confidence, calibration id, and payload/resource reference. |
|
||||
| Ledger state | `Mimir.VerseRecorder` | Session receipts, body index, replay cursor, migration version, schema witness, and rejected/drop reasons. |
|
||||
| Control state | Runtime owners / Fensalir daemon / DSP | Operator commands, accepted command receipts, selected program layer state, reservoir worker pressure, audio actuator commands, and stream/record/live status. |
|
||||
|
||||
JSON may remain only as schema publication, diagnostic text export, external
|
||||
tool boundary, or temporary replay witness. If a JSON file can decide behavior
|
||||
without a typed `.cc` witness, it is still an owner and must be migrated.
|
||||
|
||||
## CultMesh Namespaces
|
||||
|
||||
Mimir should publish these namespaces for Odin discovery:
|
||||
|
||||
- `verse.mimir.service.v1`: service manifest, run id, host identity, schema
|
||||
catalog, transport lanes, and nested Verse roots.
|
||||
- `mimir.sensor.*`: source inventory, native worker health, device timing,
|
||||
media format, actuator capability, and producer fences.
|
||||
- `mimir.media.*`: capture pages, media bodies, resource references, program
|
||||
texture/fence refs, stem refs, and replay body indexes.
|
||||
- `mimir.observation.*`: sensor observations, media observations, feature
|
||||
tracks, audio path evidence, calibration constraints, and local frustum hints.
|
||||
- `mimir.ledger.*`: recorder session, append cursor, migration state,
|
||||
body-index records, dropped-page reasons, and evidence receipts.
|
||||
- `mimir.control.*`: operator commands, command receipts, recording/streaming
|
||||
state, runtime mode, source placement, DSP actuator state, and reservoir
|
||||
scheduling pressure.
|
||||
- `mimir.eve.*`: `gamecult.eve.surface.v1` provider manifests, retained
|
||||
surfaces, field bindings, assets, command bindings, and lowering metadata.
|
||||
- `mimir.fensalir.*`: daemon summary, reservoir worker state, reservoir pressure,
|
||||
program surfaces, resource resolver status, and selected lowering status.
|
||||
|
||||
Existing document names such as `mimir.well_snapshot.v1`,
|
||||
`mimir.well_capture_page.v1`, `mimir.eve_sensor_observation.v1`,
|
||||
`mimir.eve_media_observation.v1`, `mimir.eve_dashboard_state.v1`,
|
||||
`mimir.fensalir_daemon_state.v1`, `mimir.fensalir_reservoir_worker_state.v1`,
|
||||
and `mimir.fensalir_reservoir_pressure.v1` should be kept as concrete records
|
||||
inside those namespaces until a schema catalog renames them deliberately.
|
||||
|
||||
## Transport Lanes
|
||||
|
||||
CultMesh reliable UDP is the default Verse transport for live Mimir network
|
||||
state and media. TCP/WebSocket, SRT, browser streams, and local pipes are
|
||||
diagnostic lowerings or external-boundary bridges unless a specific deployment
|
||||
constraint temporarily promotes them.
|
||||
|
||||
The live lane split is:
|
||||
|
||||
- Typed control/state lane: compact documents such as timing state, command
|
||||
receipts, health, stream cursors, and backpressure.
|
||||
- Media stream-frame lane: typed per-frame envelopes such as
|
||||
`mimir.cultmesh_stream_frame.v1` carrying source identity, capture time,
|
||||
resource/body refs, payload format, and confidence.
|
||||
- Media body-shard lane: bounded CultCache media/page shards with hashes,
|
||||
cursors, receiver acknowledgements, and backpressure. Large bodies do not
|
||||
travel as inline base64.
|
||||
- Latest-state dashboard lane: lossy/current Eve surfaces for operators. It
|
||||
may drop obsolete frames; it does not own durable media bodies.
|
||||
|
||||
Media streams are first-class Verse payloads. They still do not become timing
|
||||
authority by arrival timestamp. Clock influence comes from decoded anchors,
|
||||
source-local evidence, calibrated loopback, and explicit clock-domain state.
|
||||
|
||||
## Eve Surfaces
|
||||
|
||||
Mimir publishes operator presentation as Eve/CultUI DSL, preferably
|
||||
`gamecult.eve.surface.v1`. The expected surfaces are:
|
||||
|
||||
- `mimir.sensor.field.surface`: sensor field view with source health, buffer
|
||||
depth, timing confidence, calibration status, copy count, media format,
|
||||
resource residency, and per-source observation pressure.
|
||||
- `mimir.operator.flow.surface`: operator workflow for source selection,
|
||||
program placement, audio strips, record/live controls, reservoir pressure,
|
||||
sync mode, command receipts, and safe degraded-state visibility.
|
||||
- `mimir.observation.ledger.surface`: session/replay view over observation
|
||||
counts, body refs, accepted/dropped pages, cursor state, and schema witness.
|
||||
- `mimir.fensalir.reservoir.surface`: daemon-owned worker queue, ready ratio,
|
||||
timing confidence, GPU queue pressure, drop causes, and program surface refs.
|
||||
|
||||
The old dashboard broker, WebSocket `/eve/deck` lane, `/eve/dashboard`
|
||||
compatibility endpoint, JSON health endpoints, JPEG streams, and browser
|
||||
prototypes are lowerings or transport probes. They may mirror the Eve surface,
|
||||
but they do not become presentation owners.
|
||||
|
||||
## Nested Verses
|
||||
|
||||
Mimir's service Verse contains smaller Verses with bounded authority:
|
||||
|
||||
- Room Verse: calibration truth, coordinate frame, sensor graph, clock model,
|
||||
configured composites, and operator identity for one physical room.
|
||||
- Sensor Verse: one device or logical source, including driver health, timing
|
||||
domain, media format, actuator surface, and observation stream.
|
||||
- Stream Verse: one synchronized media/control lane, including rolling-window
|
||||
cursor, capture pages, resource refs, body refs, and backpressure state.
|
||||
- Operator Verse: Eve/CultUI command surface, selected source/layer state,
|
||||
command receipts, record/live state, and degraded-state visibility.
|
||||
|
||||
Nested Verses may publish local surfaces and state, but the parent Mimir service
|
||||
owns discovery and contract routing. A sensor may report what it observed; it
|
||||
does not become room clock authority unless explicitly promoted by calibration
|
||||
state.
|
||||
|
||||
## Migration Order
|
||||
|
||||
1. Publish a `verse.mimir.service.v1` manifest that names active namespaces,
|
||||
schema ids, run id, nested Verse roots, and the current transport lanes.
|
||||
2. Give each current JSONL/session/body-index path a `.cc` witness or
|
||||
CultCache-compatible export, starting with Well capture pages and
|
||||
`Mimir.VerseRecorder`.
|
||||
3. Promote sensor, media, observation, ledger, and control state into explicit
|
||||
`mimir.*` CultMesh namespaces while preserving current document ids.
|
||||
4. Publish `gamecult.eve.surface.v1` surfaces for sensor field and operator
|
||||
flows, with typed field bindings back to the owning state documents.
|
||||
5. Make the old dashboard broker and `/eve/dashboard` compatibility endpoint
|
||||
consume those provider surfaces instead of assembling private truth.
|
||||
6. Let Odin discover the service manifest and surfaces, then treat direct HTTP
|
||||
health/status probes as temporary diagnostics.
|
||||
7. Delete or neuter any private JSON store, dashboard state cache, or UI callback
|
||||
path that can still decide service behavior after the typed owner exists.
|
||||
|
||||
## Demotion Line
|
||||
|
||||
Bespoke UI, browser dashboards, retained dashboard node lists, compatibility
|
||||
WebSockets, SRT/FFmpeg media bridges, JSON status files, JSONL replay logs,
|
||||
screenshot streams, and OBS bridge scripts are demoted to lowerings,
|
||||
diagnostics, external-boundary export, or replay witnesses. They may display,
|
||||
transport, inspect, or reconstruct typed Mimir state. They may not own sensor
|
||||
truth, calibration truth, command state, reservoir scheduling, program
|
||||
presentation, media-over-Verse policy, or synchronization behavior.
|
||||
|
|
@ -4,12 +4,13 @@
|
|||
|
||||
Build the smallest live Mimir application that can ingest multiple local or
|
||||
networked video and audio feeds, hold a short synchronized window in memory,
|
||||
and emit coherent OBS-facing program outputs while Fensalir provides the
|
||||
operator UI.
|
||||
and emit coherent Mimir program outputs while Eve provides operator surfaces
|
||||
and Yggdrasil publishes the site feed.
|
||||
|
||||
The default reservoir horizon is five seconds. That delay is not decoration. It
|
||||
is the budget that lets late network packets, independent capture clocks,
|
||||
audio/video alignment, and debug controls converge before OBS sees the result.
|
||||
audio/video alignment, and debug controls converge before Mimir emits the
|
||||
program result.
|
||||
Mimir is explicitly trading latency and memory for throughput and extraction
|
||||
quality: the app may spend that buffered window lining streams up and crunching
|
||||
audio/video data to recover as much of the volumetric audio field and
|
||||
|
|
@ -21,16 +22,16 @@ alignment and field-reconstruction budget.
|
|||
|
||||
Today the repo has two partial machines:
|
||||
|
||||
- FFmpeg/SRT bridge scripts can capture sender video/audio and expose stable OBS
|
||||
Media Source endpoints.
|
||||
- FFmpeg/SRT bridge scripts can capture sender video/audio and expose stable
|
||||
compatibility endpoints.
|
||||
- The native `LocalcastRuntime` reservoir ABI can hold typed live sample handles,
|
||||
and Fensalir can consume native render packets for GPU fusion.
|
||||
|
||||
Those parts are useful, but they are not yet the viable app. The viable app is
|
||||
the join: ingest workers append live sample handles into one Fensalir-hosted
|
||||
runtime window, Fensalir renders video from that window, Faust/native audio
|
||||
builds synchronized stems from the same window, and OBS receives only the
|
||||
program outputs.
|
||||
builds synchronized stems from the same window, Mimir commits the scene graph,
|
||||
Eve lowers the operator surface, and Yggdrasil publishes the program output.
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
|
|
@ -39,10 +40,15 @@ flowchart TD
|
|||
C["local mic/loopback workers"] --> R
|
||||
D["network audio ingest workers"] --> R
|
||||
R --> U["Fensalir debug/settings/output UI"]
|
||||
R --> V["Fensalir GPU video composition"]
|
||||
R --> S["Mimir program scene graph"]
|
||||
S --> U
|
||||
S --> V["Fensalir GPU video composition"]
|
||||
R --> F["Faust/native audio alignment + stems"]
|
||||
V --> O["OBS video output"]
|
||||
F --> P["OBS audio stem outputs"]
|
||||
S --> E["Eve GUI/TUI operator surfaces"]
|
||||
V --> Y["Yggdrasil site publisher"]
|
||||
F --> Y
|
||||
V -. "compatibility" .-> O["OBS adapter"]
|
||||
F -. "compatibility" .-> O
|
||||
```
|
||||
|
||||
## Invariants
|
||||
|
|
@ -51,12 +57,17 @@ flowchart TD
|
|||
- The five-second rolling window is a deliberate compute budget, not accidental
|
||||
latency. It exists so capture, sync, GPU fusion, and audio field extraction
|
||||
can choose coherence over immediacy.
|
||||
- Fensalir owns the running app, UI, debug state, settings changes, output
|
||||
management, GPU fusion/rendering, and Spout/video publication.
|
||||
- Mimir owns the program scene graph, source subscription policy, preview/control
|
||||
state, stats, and publication intent.
|
||||
- Fensalir owns the running app host, UI lowering, debug state, GPU
|
||||
fusion/rendering, and local program texture output.
|
||||
- Faust/native DSP owns audio alignment, voice separation, suppression,
|
||||
volumetric sound-field extraction, stems, and spatial bed generation.
|
||||
- OBS receives synchronized program surfaces. Raw unsynchronized feeds may be
|
||||
diagnostic inputs, not broadcast truth.
|
||||
- Eve lowerers render controls and previews without owning scene truth.
|
||||
- The Yggdrasil publisher consumes synchronized program surfaces without owning
|
||||
a second composition.
|
||||
- OBS compatibility adapters may receive synchronized program surfaces. Raw
|
||||
unsynchronized feeds may be diagnostic inputs, not broadcast truth.
|
||||
- Networked feeds are producers into the same reservoir. They do not own clocks,
|
||||
private buffers, or a parallel synchronization model.
|
||||
- Missing feeds become explicit absence or silent/blank placeholders inside the
|
||||
|
|
@ -78,9 +89,14 @@ The first cut should be deliberately boring:
|
|||
loopback ingest.
|
||||
7. Expose Fensalir UI readouts for reservoir edge, window start, per-kind
|
||||
counts, feed health, delay estimate, output status, and current settings.
|
||||
8. Render one synchronized program video output from the reservoir.
|
||||
9. Emit separately controllable OBS audio stems from the same reservoir window.
|
||||
10. Keep the existing FFmpeg/SRT scripts as edge utilities for network capture
|
||||
8. Render one synchronized Mimir program video output from the reservoir.
|
||||
9. Emit separately controllable program audio stems from the same reservoir
|
||||
window.
|
||||
10. Publish Eve GUI/TUI operator surfaces for preview, source selection, scene
|
||||
edits, stats, and output state.
|
||||
11. Add the Yggdrasil-facing publisher daemon that consumes Mimir program
|
||||
output without owning composition.
|
||||
12. Keep the existing FFmpeg/SRT scripts as edge utilities for network capture
|
||||
and OBS compatibility, not as the stream authority.
|
||||
|
||||
## Repo Shape
|
||||
|
|
@ -107,14 +123,11 @@ The first cut should be deliberately boring:
|
|||
- `MimirNativeIngestStreamSource` is the low-overhead push seam for native
|
||||
camera/audio adapters that already own buffers or handles.
|
||||
- `MimirVideoFrameDescriptor` carries video shape, pixel format, device
|
||||
timestamp, stride, resource key, producer fence value, and native/GPU handle
|
||||
metadata. Leap stereo IR should enter here through a native driver path that
|
||||
reports any unavoidable system-memory copy.
|
||||
timestamp, stride, and optional native/GPU handle metadata. Leap stereo IR
|
||||
should enter here through a native driver path.
|
||||
- `IMimirVideoCaptureDriver` and `MimirVideoCaptureDriverSource` are the
|
||||
driver-facing seam for polling/pushing frames from LeapUVC/libusb, LeapC image
|
||||
APIs, Media Foundation, DirectShow, or GPU decode/capture paths. Drivers that
|
||||
can produce render inputs should implement `IMimirFensalirTextureLeaseReceiver`
|
||||
and populate Fensalir-owned texture leases directly.
|
||||
APIs, Media Foundation, DirectShow, or a future GPU shared-texture capture.
|
||||
- `MimirProcessStreamSource` is a compatibility adapter for network or
|
||||
diagnostic command sources. It is not the six-camera local capture foundation.
|
||||
|
||||
|
|
@ -132,10 +145,10 @@ environment variables:
|
|||
These create buffers immediately. Capture adapters still need to register
|
||||
`IMimirStreamSource` implementations to feed live samples into those buffers.
|
||||
|
||||
For local cameras, prefer native adapters that talk to the device stack directly
|
||||
and push only typed sample/resource handles through `MimirNativeIngestStreamSource`.
|
||||
Process-backed FFmpeg ingest is allowed for remote SRT/diagnostic edges, but
|
||||
local six-camera ingest should stay close to the device, GPU, and capture APIs.
|
||||
For local cameras, prefer native adapters that push sample handles directly
|
||||
through `MimirNativeIngestStreamSource`. Process-backed FFmpeg ingest is allowed
|
||||
for remote SRT/diagnostic edges, but local six-camera ingest should stay close
|
||||
to the driver, GPU, and capture APIs.
|
||||
|
||||
Leap should be treated as the timing-camera candidate only when the native
|
||||
adapter preserves device timestamps or captures a monotonic timestamp
|
||||
|
|
|
|||
|
|
@ -1,65 +0,0 @@
|
|||
declare name "Mimir Dual Mic Dialogue Cleaner";
|
||||
declare version "0.1";
|
||||
declare author "Mimir";
|
||||
declare license "MIT";
|
||||
declare description "Faust-owned smooth dual-mic dialogue cleanup target: aligned shotgun/cardioid inputs become a conservative dialogue stem and a rejected residual witness.";
|
||||
|
||||
import("stdfaust.lib");
|
||||
|
||||
// Inputs are already delay-aligned upstream by mimir_alignment_actuator.
|
||||
// This graph owns smooth audio-rate cleanup, not calibration, timing, or UI.
|
||||
//
|
||||
// input 0: aligned shotgun / co-streamer witness
|
||||
// input 1: aligned cardioid / host witness
|
||||
// output 0: dialogue composite stem
|
||||
// output 1: rejected residual / noise witness
|
||||
|
||||
drive = hslider("dialogue/drive", 1.00, 0.00, 2.00, 0.001);
|
||||
shotgunWeight = hslider("dialogue/shotgun_weight", 0.45, 0.00, 1.00, 0.001);
|
||||
cardioidWeight = hslider("dialogue/cardioid_weight", 0.55, 0.00, 1.00, 0.001);
|
||||
witnessReject = hslider("dialogue/witness_reject", 0.20, 0.00, 1.00, 0.001);
|
||||
lowShelf = hslider("dialogue/low_shelf", 0.80, 0.00, 2.00, 0.001);
|
||||
presenceShelf = hslider("dialogue/presence_shelf", 1.08, 0.00, 2.00, 0.001);
|
||||
airShelf = hslider("dialogue/air_shelf", 0.90, 0.00, 2.00, 0.001);
|
||||
|
||||
// Smooth broadband downward expansion. This is intentionally slow enough to
|
||||
// avoid the choppy gate artifact from the C# audition bridge.
|
||||
floorLevel = hslider("denoise/noise_floor", 0.018, 0.0001, 0.25, 0.0001);
|
||||
expansionDepth = hslider("denoise/expansion_depth", 0.32, 0.00, 0.95, 0.001);
|
||||
expansionCurve = hslider("denoise/expansion_curve", 1.35, 0.25, 4.00, 0.001);
|
||||
envelopeHz = hslider("denoise/envelope_hz", 18.0, 1.0, 80.0, 0.1);
|
||||
|
||||
band(f, q, g, x) = x : fi.resonbp(1.0, f, q) : *(g);
|
||||
|
||||
voiceShape(x) =
|
||||
band(120.0, 1.2, lowShelf, x) +
|
||||
band(220.0, 1.6, 0.95, x) +
|
||||
band(420.0, 2.0, 0.98, x) +
|
||||
band(800.0, 2.4, 1.00, x) +
|
||||
band(1500.0, 2.8, presenceShelf, x) +
|
||||
band(2800.0, 3.2, presenceShelf, x) +
|
||||
band(5200.0, 3.2, airShelf, x) +
|
||||
band(8500.0, 2.6, airShelf, x);
|
||||
|
||||
smoothAbs(x) = abs(x) : si.smooth(ba.tau2pole(1.0 / max(1.0, envelopeHz)));
|
||||
|
||||
softExpand(x) = x * gain
|
||||
with {
|
||||
env = smoothAbs(x);
|
||||
ratio = min(1.0, env / max(0.0001, floorLevel));
|
||||
openness = pow(ratio, expansionCurve);
|
||||
gain = 1.0 - expansionDepth * (1.0 - openness);
|
||||
};
|
||||
|
||||
softClip(x) = x / (1.0 + abs(x));
|
||||
|
||||
clean(shotgun, cardioid) = dialogue, residual
|
||||
with {
|
||||
weighted = shotgun * shotgunWeight + cardioid * cardioidWeight;
|
||||
witness = (shotgun - cardioid) * 0.5;
|
||||
shaped = voiceShape(weighted - witnessReject * witness);
|
||||
dialogue = softExpand(shaped * drive) : softClip;
|
||||
residual = witness + (weighted - dialogue) * 0.25;
|
||||
};
|
||||
|
||||
process = clean;
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
cmake_minimum_required(VERSION 3.20)
|
||||
|
||||
project(mimir_camera_capture LANGUAGES C CXX)
|
||||
|
||||
add_library(mimir_camera_capture SHARED
|
||||
mimir_ks_camera_capture.cpp)
|
||||
|
||||
target_compile_features(mimir_camera_capture PRIVATE cxx_std_20)
|
||||
target_compile_definitions(mimir_camera_capture PRIVATE
|
||||
WIN32_LEAN_AND_MEAN
|
||||
NOMINMAX)
|
||||
target_link_libraries(mimir_camera_capture PRIVATE
|
||||
setupapi
|
||||
ksuser)
|
||||
|
||||
set(PSMOVEAPI_ROOT "$ENV{TEMP}/mimir-driver-probes/psmoveapi" CACHE PATH "Root of a psmoveapi checkout with external/libusb-1.0 and external/PS3EYEDriver.")
|
||||
set(LIBUSB_ROOT "${PSMOVEAPI_ROOT}/external/libusb-1.0" CACHE PATH "libusb source root.")
|
||||
set(PS3EYE_ROOT "${PSMOVEAPI_ROOT}/external/PS3EYEDriver" CACHE PATH "PS3EYEDriver source root.")
|
||||
|
||||
add_library(mimir_libusb_static STATIC
|
||||
${LIBUSB_ROOT}/libusb/core.c
|
||||
${LIBUSB_ROOT}/libusb/descriptor.c
|
||||
${LIBUSB_ROOT}/libusb/hotplug.c
|
||||
${LIBUSB_ROOT}/libusb/io.c
|
||||
${LIBUSB_ROOT}/libusb/strerror.c
|
||||
${LIBUSB_ROOT}/libusb/sync.c
|
||||
${LIBUSB_ROOT}/libusb/os/events_windows.c
|
||||
${LIBUSB_ROOT}/libusb/os/threads_windows.c
|
||||
${LIBUSB_ROOT}/libusb/os/windows_common.c
|
||||
${LIBUSB_ROOT}/libusb/os/windows_usbdk.c
|
||||
${LIBUSB_ROOT}/libusb/os/windows_winusb.c)
|
||||
target_include_directories(mimir_libusb_static PUBLIC
|
||||
${LIBUSB_ROOT}/libusb
|
||||
${LIBUSB_ROOT}/libusb/os
|
||||
${LIBUSB_ROOT}/msvc)
|
||||
target_compile_definitions(mimir_libusb_static PRIVATE
|
||||
_CRT_SECURE_NO_WARNINGS
|
||||
_WIN32_WINNT=0x0601)
|
||||
target_link_libraries(mimir_libusb_static PRIVATE setupapi ole32 advapi32)
|
||||
|
||||
add_library(mimir_ps3eye_vendor STATIC
|
||||
${PS3EYE_ROOT}/src/ps3eye.cpp
|
||||
${PS3EYE_ROOT}/src/ps3eye_capi.cpp)
|
||||
target_include_directories(mimir_ps3eye_vendor PUBLIC
|
||||
${PS3EYE_ROOT}/src
|
||||
${LIBUSB_ROOT}/libusb)
|
||||
target_link_libraries(mimir_ps3eye_vendor PUBLIC mimir_libusb_static)
|
||||
|
||||
add_library(mimir_ps3eye_capture SHARED
|
||||
mimir_ps3eye_capture.cpp)
|
||||
target_compile_features(mimir_ps3eye_capture PRIVATE cxx_std_20)
|
||||
target_compile_definitions(mimir_ps3eye_capture PRIVATE
|
||||
WIN32_LEAN_AND_MEAN
|
||||
NOMINMAX)
|
||||
target_include_directories(mimir_ps3eye_capture PRIVATE ${PS3EYE_ROOT}/src)
|
||||
target_link_libraries(mimir_ps3eye_capture PRIVATE mimir_ps3eye_vendor)
|
||||
|
||||
add_library(mimir_mf_gpu_capture SHARED
|
||||
mimir_mf_gpu_capture.cpp)
|
||||
target_compile_features(mimir_mf_gpu_capture PRIVATE cxx_std_20)
|
||||
target_compile_definitions(mimir_mf_gpu_capture PRIVATE
|
||||
WIN32_LEAN_AND_MEAN
|
||||
NOMINMAX)
|
||||
target_link_libraries(mimir_mf_gpu_capture PRIVATE
|
||||
d3d11
|
||||
dxgi
|
||||
mf
|
||||
mfplat
|
||||
mfreadwrite
|
||||
mfuuid)
|
||||
|
|
@ -1,759 +0,0 @@
|
|||
#include <windows.h>
|
||||
#include <winioctl.h>
|
||||
#include <setupapi.h>
|
||||
#include <ks.h>
|
||||
#include <ksmedia.h>
|
||||
#include <winternl.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <cctype>
|
||||
#include <chrono>
|
||||
#include <condition_variable>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <deque>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
namespace
|
||||
{
|
||||
struct Handle
|
||||
{
|
||||
HANDLE value = INVALID_HANDLE_VALUE;
|
||||
|
||||
Handle() = default;
|
||||
explicit Handle(HANDLE handle) : value(handle) {}
|
||||
Handle(const Handle&) = delete;
|
||||
Handle& operator=(const Handle&) = delete;
|
||||
Handle(Handle&& other) noexcept : value(other.value) { other.value = INVALID_HANDLE_VALUE; }
|
||||
Handle& operator=(Handle&& other) noexcept
|
||||
{
|
||||
if (this != &other)
|
||||
{
|
||||
close();
|
||||
value = other.value;
|
||||
other.value = INVALID_HANDLE_VALUE;
|
||||
}
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
~Handle() { close(); }
|
||||
|
||||
void close()
|
||||
{
|
||||
if (value != INVALID_HANDLE_VALUE && value != nullptr)
|
||||
{
|
||||
CloseHandle(value);
|
||||
value = INVALID_HANDLE_VALUE;
|
||||
}
|
||||
}
|
||||
|
||||
explicit operator bool() const { return value != INVALID_HANDLE_VALUE && value != nullptr; }
|
||||
};
|
||||
|
||||
struct DeviceInfoSet
|
||||
{
|
||||
HDEVINFO value = INVALID_HANDLE_VALUE;
|
||||
|
||||
explicit DeviceInfoSet(HDEVINFO info) : value(info) {}
|
||||
DeviceInfoSet(const DeviceInfoSet&) = delete;
|
||||
DeviceInfoSet& operator=(const DeviceInfoSet&) = delete;
|
||||
~DeviceInfoSet()
|
||||
{
|
||||
if (value != INVALID_HANDLE_VALUE)
|
||||
{
|
||||
SetupDiDestroyDeviceInfoList(value);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
using NtDeviceIoControlFileFn = NTSTATUS(NTAPI*)(
|
||||
HANDLE FileHandle,
|
||||
HANDLE Event,
|
||||
PIO_APC_ROUTINE ApcRoutine,
|
||||
PVOID ApcContext,
|
||||
PIO_STATUS_BLOCK IoStatusBlock,
|
||||
ULONG IoControlCode,
|
||||
PVOID InputBuffer,
|
||||
ULONG InputBufferLength,
|
||||
PVOID OutputBuffer,
|
||||
ULONG OutputBufferLength);
|
||||
|
||||
constexpr NTSTATUS StatusPending = static_cast<NTSTATUS>(0x00000103L);
|
||||
|
||||
struct PinCandidate
|
||||
{
|
||||
ULONG pinId = 0;
|
||||
std::vector<std::uint8_t> format;
|
||||
LONG width = 0;
|
||||
LONG height = 0;
|
||||
LONG imageBytes = 0;
|
||||
LONGLONG interval100ns = 0;
|
||||
GUID subtype{};
|
||||
std::string fourcc;
|
||||
};
|
||||
|
||||
struct MimirKsCameraFrame
|
||||
{
|
||||
std::vector<std::uint8_t> data;
|
||||
std::int64_t timestampNs = 0;
|
||||
std::uint64_t sequence = 0;
|
||||
int byteLength = 0;
|
||||
};
|
||||
|
||||
struct MimirKsCameraCapture
|
||||
{
|
||||
Handle filter;
|
||||
Handle pin;
|
||||
PinCandidate candidate;
|
||||
std::thread worker;
|
||||
std::mutex mutex;
|
||||
std::condition_variable condition;
|
||||
std::deque<MimirKsCameraFrame> queue;
|
||||
std::atomic<bool> running = false;
|
||||
int queueDepth = 8;
|
||||
std::uint64_t sequence = 0;
|
||||
};
|
||||
|
||||
long long nowNs()
|
||||
{
|
||||
return std::chrono::duration_cast<std::chrono::nanoseconds>(
|
||||
std::chrono::steady_clock::now().time_since_epoch()).count();
|
||||
}
|
||||
|
||||
std::string lowercase(std::string value)
|
||||
{
|
||||
std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) {
|
||||
return static_cast<char>(std::tolower(c));
|
||||
});
|
||||
return value;
|
||||
}
|
||||
|
||||
std::string wideToUtf8(const wchar_t* value)
|
||||
{
|
||||
if (!value)
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
const int bytes = WideCharToMultiByte(CP_UTF8, 0, value, -1, nullptr, 0, nullptr, nullptr);
|
||||
if (bytes <= 1)
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
std::string text(static_cast<std::size_t>(bytes - 1), '\0');
|
||||
WideCharToMultiByte(CP_UTF8, 0, value, -1, text.data(), bytes, nullptr, nullptr);
|
||||
return text;
|
||||
}
|
||||
|
||||
std::string fourccOrGuid(const GUID& guid)
|
||||
{
|
||||
if (guid.Data2 == 0x0000 &&
|
||||
guid.Data3 == 0x0010 &&
|
||||
guid.Data4[0] == 0x80 &&
|
||||
guid.Data4[1] == 0x00 &&
|
||||
guid.Data4[2] == 0x00 &&
|
||||
guid.Data4[3] == 0xaa &&
|
||||
guid.Data4[4] == 0x00 &&
|
||||
guid.Data4[5] == 0x38 &&
|
||||
guid.Data4[6] == 0x9b &&
|
||||
guid.Data4[7] == 0x71)
|
||||
{
|
||||
char text[5]{};
|
||||
text[0] = static_cast<char>(guid.Data1 & 0xff);
|
||||
text[1] = static_cast<char>((guid.Data1 >> 8) & 0xff);
|
||||
text[2] = static_cast<char>((guid.Data1 >> 16) & 0xff);
|
||||
text[3] = static_cast<char>((guid.Data1 >> 24) & 0xff);
|
||||
return text;
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
bool ksProperty(HANDLE handle, void* property, DWORD propertyBytes, void* output, DWORD outputBytes, DWORD* returned = nullptr)
|
||||
{
|
||||
DWORD bytes = 0;
|
||||
const BOOL ok = DeviceIoControl(
|
||||
handle,
|
||||
IOCTL_KS_PROPERTY,
|
||||
property,
|
||||
propertyBytes,
|
||||
output,
|
||||
outputBytes,
|
||||
&bytes,
|
||||
nullptr);
|
||||
if (returned)
|
||||
{
|
||||
*returned = bytes;
|
||||
}
|
||||
|
||||
return ok != FALSE;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool getPinProperty(HANDLE filter, ULONG pinId, ULONG propertyId, T& value)
|
||||
{
|
||||
KSP_PIN property{};
|
||||
property.Property.Set = KSPROPSETID_Pin;
|
||||
property.Property.Id = propertyId;
|
||||
property.Property.Flags = KSPROPERTY_TYPE_GET;
|
||||
property.PinId = pinId;
|
||||
DWORD returned = 0;
|
||||
return ksProperty(filter, &property, sizeof(property), &value, sizeof(value), &returned);
|
||||
}
|
||||
|
||||
bool setVideoProcAmp(HANDLE filter, ULONG id, LONG value, ULONG flags)
|
||||
{
|
||||
KSPROPERTY_VIDEOPROCAMP_S property{};
|
||||
property.Property.Set = PROPSETID_VIDCAP_VIDEOPROCAMP;
|
||||
property.Property.Id = id;
|
||||
property.Property.Flags = KSPROPERTY_TYPE_SET;
|
||||
property.Value = value;
|
||||
property.Flags = flags;
|
||||
return ksProperty(filter, &property, sizeof(property), &property, sizeof(property));
|
||||
}
|
||||
|
||||
bool setCameraControl(HANDLE filter, ULONG id, LONG value, ULONG flags)
|
||||
{
|
||||
KSPROPERTY_CAMERACONTROL_S property{};
|
||||
property.Property.Set = PROPSETID_VIDCAP_CAMERACONTROL;
|
||||
property.Property.Id = id;
|
||||
property.Property.Flags = KSPROPERTY_TYPE_SET;
|
||||
property.Value = value;
|
||||
property.Flags = flags;
|
||||
return ksProperty(filter, &property, sizeof(property), &property, sizeof(property));
|
||||
}
|
||||
|
||||
std::vector<std::string> enumerateCapturePaths()
|
||||
{
|
||||
std::vector<std::string> paths;
|
||||
DeviceInfoSet info(SetupDiGetClassDevsA(
|
||||
&KSCATEGORY_CAPTURE,
|
||||
nullptr,
|
||||
nullptr,
|
||||
DIGCF_PRESENT | DIGCF_DEVICEINTERFACE));
|
||||
if (info.value == INVALID_HANDLE_VALUE)
|
||||
{
|
||||
return paths;
|
||||
}
|
||||
|
||||
for (DWORD index = 0;; ++index)
|
||||
{
|
||||
SP_DEVICE_INTERFACE_DATA interfaceData{};
|
||||
interfaceData.cbSize = sizeof(interfaceData);
|
||||
if (!SetupDiEnumDeviceInterfaces(info.value, nullptr, &KSCATEGORY_CAPTURE, index, &interfaceData))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
DWORD required = 0;
|
||||
SetupDiGetDeviceInterfaceDetailW(info.value, &interfaceData, nullptr, 0, &required, nullptr);
|
||||
if (required == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
std::vector<std::uint8_t> storage(required);
|
||||
auto* detail = reinterpret_cast<SP_DEVICE_INTERFACE_DETAIL_DATA_W*>(storage.data());
|
||||
detail->cbSize = sizeof(*detail);
|
||||
if (SetupDiGetDeviceInterfaceDetailW(info.value, &interfaceData, detail, required, nullptr, nullptr))
|
||||
{
|
||||
paths.push_back(wideToUtf8(detail->DevicePath));
|
||||
}
|
||||
}
|
||||
|
||||
return paths;
|
||||
}
|
||||
|
||||
std::vector<PinCandidate> inspectPins(HANDLE filter)
|
||||
{
|
||||
ULONG pinCount = 0;
|
||||
KSPROPERTY countProperty{};
|
||||
countProperty.Set = KSPROPSETID_Pin;
|
||||
countProperty.Id = KSPROPERTY_PIN_CTYPES;
|
||||
countProperty.Flags = KSPROPERTY_TYPE_GET;
|
||||
if (!ksProperty(filter, &countProperty, sizeof(countProperty), &pinCount, sizeof(pinCount)))
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
std::vector<PinCandidate> candidates;
|
||||
for (ULONG pinId = 0; pinId < pinCount; ++pinId)
|
||||
{
|
||||
KSPIN_DATAFLOW dataflow{};
|
||||
if (!getPinProperty(filter, pinId, KSPROPERTY_PIN_DATAFLOW, dataflow) ||
|
||||
dataflow != KSPIN_DATAFLOW_OUT)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
KSP_PIN rangeProperty{};
|
||||
rangeProperty.Property.Set = KSPROPSETID_Pin;
|
||||
rangeProperty.Property.Id = KSPROPERTY_PIN_DATARANGES;
|
||||
rangeProperty.Property.Flags = KSPROPERTY_TYPE_GET;
|
||||
rangeProperty.PinId = pinId;
|
||||
|
||||
DWORD rangeBytes = 0;
|
||||
ksProperty(filter, &rangeProperty, sizeof(rangeProperty), nullptr, 0, &rangeBytes);
|
||||
if (rangeBytes == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
std::vector<std::uint8_t> ranges(rangeBytes);
|
||||
DWORD returned = 0;
|
||||
if (!ksProperty(filter, &rangeProperty, sizeof(rangeProperty), ranges.data(), static_cast<DWORD>(ranges.size()), &returned))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
auto* multiple = reinterpret_cast<KSMULTIPLE_ITEM*>(ranges.data());
|
||||
auto* cursor = ranges.data() + sizeof(KSMULTIPLE_ITEM);
|
||||
for (ULONG i = 0; i < multiple->Count; ++i)
|
||||
{
|
||||
auto* dataRange = reinterpret_cast<KSDATARANGE*>(cursor);
|
||||
const bool videoInfo =
|
||||
IsEqualGUID(dataRange->MajorFormat, KSDATAFORMAT_TYPE_VIDEO) &&
|
||||
IsEqualGUID(dataRange->Specifier, KSDATAFORMAT_SPECIFIER_VIDEOINFO) &&
|
||||
dataRange->FormatSize >= sizeof(KS_DATARANGE_VIDEO);
|
||||
|
||||
if (videoInfo)
|
||||
{
|
||||
auto* video = reinterpret_cast<KS_DATARANGE_VIDEO*>(cursor);
|
||||
const auto& bmi = video->VideoInfoHeader.bmiHeader;
|
||||
PinCandidate candidate{};
|
||||
candidate.pinId = pinId;
|
||||
candidate.width = bmi.biWidth;
|
||||
candidate.height = bmi.biHeight;
|
||||
candidate.imageBytes = bmi.biSizeImage > 0 ? bmi.biSizeImage : static_cast<LONG>(dataRange->SampleSize);
|
||||
candidate.interval100ns = video->VideoInfoHeader.AvgTimePerFrame;
|
||||
candidate.subtype = dataRange->SubFormat;
|
||||
candidate.fourcc = fourccOrGuid(candidate.subtype);
|
||||
candidate.format.resize(sizeof(KS_DATAFORMAT_VIDEOINFOHEADER));
|
||||
auto* format = reinterpret_cast<KS_DATAFORMAT_VIDEOINFOHEADER*>(candidate.format.data());
|
||||
format->DataFormat.FormatSize = sizeof(KS_DATAFORMAT_VIDEOINFOHEADER);
|
||||
format->DataFormat.SampleSize = candidate.imageBytes;
|
||||
format->DataFormat.MajorFormat = KSDATAFORMAT_TYPE_VIDEO;
|
||||
format->DataFormat.SubFormat = dataRange->SubFormat;
|
||||
format->DataFormat.Specifier = KSDATAFORMAT_SPECIFIER_VIDEOINFO;
|
||||
format->VideoInfoHeader = video->VideoInfoHeader;
|
||||
format->VideoInfoHeader.bmiHeader.biSizeImage = candidate.imageBytes;
|
||||
candidates.push_back(std::move(candidate));
|
||||
}
|
||||
|
||||
cursor += dataRange->FormatSize;
|
||||
}
|
||||
}
|
||||
|
||||
return candidates;
|
||||
}
|
||||
|
||||
const PinCandidate* findCandidate(
|
||||
const std::vector<PinCandidate>& candidates,
|
||||
int width,
|
||||
int height,
|
||||
const char* subtype,
|
||||
double minFps)
|
||||
{
|
||||
const std::string target = subtype ? subtype : "";
|
||||
for (const auto& candidate : candidates)
|
||||
{
|
||||
const double fps = candidate.interval100ns > 0
|
||||
? 10000000.0 / static_cast<double>(candidate.interval100ns)
|
||||
: 0.0;
|
||||
if (candidate.width == width &&
|
||||
candidate.height == height &&
|
||||
fps >= minFps &&
|
||||
candidate.fourcc == target)
|
||||
{
|
||||
return &candidate;
|
||||
}
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
bool setState(HANDLE pin, KSSTATE state)
|
||||
{
|
||||
KSPROPERTY property{};
|
||||
property.Set = KSPROPSETID_Connection;
|
||||
property.Id = KSPROPERTY_CONNECTION_STATE;
|
||||
property.Flags = KSPROPERTY_TYPE_SET;
|
||||
DWORD returned = 0;
|
||||
return DeviceIoControl(
|
||||
pin,
|
||||
IOCTL_KS_PROPERTY,
|
||||
&property,
|
||||
sizeof(property),
|
||||
&state,
|
||||
sizeof(state),
|
||||
&returned,
|
||||
nullptr) != FALSE;
|
||||
}
|
||||
|
||||
int strideBytes(const PinCandidate& candidate)
|
||||
{
|
||||
return candidate.height > 0 && candidate.imageBytes > 0
|
||||
? candidate.imageBytes / candidate.height
|
||||
: 0;
|
||||
}
|
||||
|
||||
void pushFrame(MimirKsCameraCapture& capture, const std::uint8_t* data, ULONG byteLength)
|
||||
{
|
||||
MimirKsCameraFrame frame;
|
||||
frame.byteLength = static_cast<int>(byteLength);
|
||||
frame.timestampNs = nowNs();
|
||||
frame.sequence = capture.sequence++;
|
||||
frame.data.resize(byteLength);
|
||||
std::memcpy(frame.data.data(), data, byteLength);
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(capture.mutex);
|
||||
capture.queue.push_back(std::move(frame));
|
||||
while (capture.queue.size() > 32)
|
||||
{
|
||||
capture.queue.pop_front();
|
||||
}
|
||||
}
|
||||
capture.condition.notify_one();
|
||||
}
|
||||
|
||||
void captureThread(MimirKsCameraCapture* capture)
|
||||
{
|
||||
auto* ntDeviceIoControlFile = reinterpret_cast<NtDeviceIoControlFileFn>(
|
||||
GetProcAddress(GetModuleHandleA("ntdll.dll"), "NtDeviceIoControlFile"));
|
||||
if (!ntDeviceIoControlFile)
|
||||
{
|
||||
capture->running.store(false);
|
||||
return;
|
||||
}
|
||||
|
||||
const ULONG connectBytes = sizeof(KSPIN_CONNECT) + static_cast<ULONG>(capture->candidate.format.size());
|
||||
std::vector<std::uint8_t> connectStorage(connectBytes);
|
||||
auto* connect = reinterpret_cast<KSPIN_CONNECT*>(connectStorage.data());
|
||||
connect->Interface.Set = KSINTERFACESETID_Standard;
|
||||
connect->Interface.Id = KSINTERFACE_STANDARD_STREAMING;
|
||||
connect->Medium.Set = KSMEDIUMSETID_Standard;
|
||||
connect->Medium.Id = KSMEDIUM_TYPE_ANYINSTANCE;
|
||||
connect->PinId = capture->candidate.pinId;
|
||||
connect->Priority.PriorityClass = KSPRIORITY_NORMAL;
|
||||
connect->Priority.PrioritySubClass = 1;
|
||||
std::memcpy(connectStorage.data() + sizeof(KSPIN_CONNECT), capture->candidate.format.data(), capture->candidate.format.size());
|
||||
|
||||
HANDLE rawPin = INVALID_HANDLE_VALUE;
|
||||
const HRESULT hr = KsCreatePin(capture->filter.value, connect, GENERIC_READ, &rawPin);
|
||||
if (FAILED(hr))
|
||||
{
|
||||
capture->running.store(false);
|
||||
return;
|
||||
}
|
||||
|
||||
capture->pin = Handle(rawPin);
|
||||
if (!setState(capture->pin.value, KSSTATE_ACQUIRE) ||
|
||||
!setState(capture->pin.value, KSSTATE_PAUSE) ||
|
||||
!setState(capture->pin.value, KSSTATE_RUN))
|
||||
{
|
||||
capture->running.store(false);
|
||||
return;
|
||||
}
|
||||
|
||||
const DWORD frameBytes = static_cast<DWORD>(capture->candidate.imageBytes);
|
||||
struct ReadSlot
|
||||
{
|
||||
Handle event;
|
||||
IO_STATUS_BLOCK status{};
|
||||
KSSTREAM_HEADER header{};
|
||||
std::vector<std::uint8_t> frame;
|
||||
bool pending = false;
|
||||
};
|
||||
|
||||
std::vector<ReadSlot> slots(static_cast<std::size_t>(std::max(1, capture->queueDepth)));
|
||||
std::vector<HANDLE> waitEvents(slots.size());
|
||||
for (auto& slot : slots)
|
||||
{
|
||||
slot.event = Handle(CreateEventW(nullptr, TRUE, FALSE, nullptr));
|
||||
if (!slot.event)
|
||||
{
|
||||
capture->running.store(false);
|
||||
return;
|
||||
}
|
||||
|
||||
slot.frame.resize(frameBytes);
|
||||
slot.header.Size = sizeof(KSSTREAM_HEADER);
|
||||
slot.header.FrameExtent = frameBytes;
|
||||
slot.header.Data = slot.frame.data();
|
||||
slot.header.PresentationTime.Numerator = 1;
|
||||
slot.header.PresentationTime.Denominator = 1;
|
||||
waitEvents[&slot - slots.data()] = slot.event.value;
|
||||
}
|
||||
|
||||
auto submit = [&](ReadSlot& slot) -> bool {
|
||||
ResetEvent(slot.event.value);
|
||||
slot.status = {};
|
||||
slot.header.DataUsed = 0;
|
||||
slot.header.OptionsFlags = 0;
|
||||
const NTSTATUS status = ntDeviceIoControlFile(
|
||||
capture->pin.value,
|
||||
slot.event.value,
|
||||
nullptr,
|
||||
nullptr,
|
||||
&slot.status,
|
||||
IOCTL_KS_READ_STREAM,
|
||||
&slot.header,
|
||||
sizeof(slot.header),
|
||||
&slot.header,
|
||||
sizeof(slot.header));
|
||||
|
||||
if (status == StatusPending)
|
||||
{
|
||||
slot.pending = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
slot.pending = false;
|
||||
if (status >= 0 && slot.header.DataUsed > 0)
|
||||
{
|
||||
pushFrame(*capture, slot.frame.data(), slot.header.DataUsed);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
};
|
||||
|
||||
for (auto& slot : slots)
|
||||
{
|
||||
if (!submit(slot))
|
||||
{
|
||||
capture->running.store(false);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
while (capture->running.load())
|
||||
{
|
||||
const DWORD waitResult = WaitForMultipleObjects(
|
||||
static_cast<DWORD>(waitEvents.size()),
|
||||
waitEvents.data(),
|
||||
FALSE,
|
||||
100);
|
||||
if (waitResult == WAIT_TIMEOUT)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (waitResult < WAIT_OBJECT_0 || waitResult >= WAIT_OBJECT_0 + waitEvents.size())
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
auto& slot = slots[static_cast<std::size_t>(waitResult - WAIT_OBJECT_0)];
|
||||
slot.pending = false;
|
||||
if (slot.status.Status < 0 || slot.header.DataUsed == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
pushFrame(*capture, slot.frame.data(), slot.header.DataUsed);
|
||||
if (!submit(slot))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
setState(capture->pin.value, KSSTATE_STOP);
|
||||
CancelIoEx(capture->pin.value, nullptr);
|
||||
for (auto& slot : slots)
|
||||
{
|
||||
if (slot.pending)
|
||||
{
|
||||
WaitForSingleObject(slot.event.value, 100);
|
||||
}
|
||||
}
|
||||
capture->running.store(false);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
extern "C"
|
||||
{
|
||||
__declspec(dllexport) MimirKsCameraCapture* mimir_ks_create(
|
||||
const char* pathNeedle,
|
||||
int width,
|
||||
int height,
|
||||
const char* subtype,
|
||||
double minFps,
|
||||
int queueDepth)
|
||||
{
|
||||
const std::string needle = lowercase(pathNeedle ? pathNeedle : "");
|
||||
for (const auto& path : enumerateCapturePaths())
|
||||
{
|
||||
if (!needle.empty() && lowercase(path).find(needle) == std::string::npos)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
Handle filter(CreateFileA(
|
||||
path.c_str(),
|
||||
GENERIC_READ | GENERIC_WRITE,
|
||||
FILE_SHARE_READ | FILE_SHARE_WRITE,
|
||||
nullptr,
|
||||
OPEN_EXISTING,
|
||||
FILE_ATTRIBUTE_NORMAL,
|
||||
nullptr));
|
||||
if (!filter)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
auto candidates = inspectPins(filter.value);
|
||||
const auto* candidate = findCandidate(candidates, width, height, subtype, minFps);
|
||||
if (!candidate)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
auto* capture = new MimirKsCameraCapture();
|
||||
capture->filter = std::move(filter);
|
||||
capture->candidate = *candidate;
|
||||
capture->queueDepth = std::max(1, std::min(queueDepth, 32));
|
||||
return capture;
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
__declspec(dllexport) int mimir_ks_start(MimirKsCameraCapture* capture)
|
||||
{
|
||||
if (!capture || capture->running.exchange(true))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
capture->worker = std::thread(captureThread, capture);
|
||||
return 1;
|
||||
}
|
||||
|
||||
__declspec(dllexport) int mimir_ks_set_exposure_gain(
|
||||
MimirKsCameraCapture* capture,
|
||||
int setExposure,
|
||||
LONG exposure,
|
||||
int setGain,
|
||||
LONG gain)
|
||||
{
|
||||
if (!capture || !capture->filter)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
int result = 0;
|
||||
if (!setExposure ||
|
||||
setCameraControl(
|
||||
capture->filter.value,
|
||||
KSPROPERTY_CAMERACONTROL_EXPOSURE,
|
||||
exposure,
|
||||
KSPROPERTY_CAMERACONTROL_FLAGS_MANUAL))
|
||||
{
|
||||
result |= 1;
|
||||
}
|
||||
if (!setGain ||
|
||||
setVideoProcAmp(
|
||||
capture->filter.value,
|
||||
KSPROPERTY_VIDEOPROCAMP_GAIN,
|
||||
gain,
|
||||
KSPROPERTY_VIDEOPROCAMP_FLAGS_MANUAL))
|
||||
{
|
||||
result |= 2;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
__declspec(dllexport) int mimir_ks_read(
|
||||
MimirKsCameraCapture* capture,
|
||||
int* width,
|
||||
int* height,
|
||||
int* stride,
|
||||
char* fourcc,
|
||||
int fourccCapacity,
|
||||
std::int64_t* timestampNs,
|
||||
std::uint64_t* sequence,
|
||||
std::uint8_t* destination,
|
||||
int destinationBytes)
|
||||
{
|
||||
if (!capture || !destination || destinationBytes <= 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
MimirKsCameraFrame frame;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(capture->mutex);
|
||||
if (capture->queue.empty())
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
frame = std::move(capture->queue.front());
|
||||
capture->queue.pop_front();
|
||||
}
|
||||
|
||||
if (frame.byteLength > destinationBytes)
|
||||
{
|
||||
return -frame.byteLength;
|
||||
}
|
||||
|
||||
std::memcpy(destination, frame.data.data(), static_cast<std::size_t>(frame.byteLength));
|
||||
if (width)
|
||||
{
|
||||
*width = static_cast<int>(capture->candidate.width);
|
||||
}
|
||||
if (height)
|
||||
{
|
||||
*height = static_cast<int>(capture->candidate.height);
|
||||
}
|
||||
if (stride)
|
||||
{
|
||||
*stride = strideBytes(capture->candidate);
|
||||
}
|
||||
if (fourcc && fourccCapacity > 0)
|
||||
{
|
||||
std::snprintf(fourcc, static_cast<std::size_t>(fourccCapacity), "%s", capture->candidate.fourcc.c_str());
|
||||
}
|
||||
if (timestampNs)
|
||||
{
|
||||
*timestampNs = frame.timestampNs;
|
||||
}
|
||||
if (sequence)
|
||||
{
|
||||
*sequence = frame.sequence;
|
||||
}
|
||||
|
||||
return frame.byteLength;
|
||||
}
|
||||
|
||||
__declspec(dllexport) void mimir_ks_destroy(MimirKsCameraCapture* capture)
|
||||
{
|
||||
if (!capture)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
capture->running.store(false);
|
||||
if (capture->pin)
|
||||
{
|
||||
CancelIoEx(capture->pin.value, nullptr);
|
||||
}
|
||||
if (capture->worker.joinable())
|
||||
{
|
||||
capture->worker.join();
|
||||
}
|
||||
|
||||
delete capture;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,573 +0,0 @@
|
|||
#include <windows.h>
|
||||
#include <d3d11_4.h>
|
||||
#include <dxgi1_6.h>
|
||||
#include <mfapi.h>
|
||||
#include <mferror.h>
|
||||
#include <mfidl.h>
|
||||
#include <mfreadwrite.h>
|
||||
#include <wrl/client.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstdarg>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
using Microsoft::WRL::ComPtr;
|
||||
|
||||
namespace
|
||||
{
|
||||
struct MimirMfGpuCapture
|
||||
{
|
||||
ComPtr<IMFMediaSource> source;
|
||||
ComPtr<IMFSourceReader> reader;
|
||||
ComPtr<ID3D11Device> device;
|
||||
ComPtr<ID3D11DeviceContext> context;
|
||||
ComPtr<IMFDXGIDeviceManager> deviceManager;
|
||||
ComPtr<ID3D11Texture2D> sharedTexture;
|
||||
ComPtr<ID3D11Fence> producerFence;
|
||||
ComPtr<ID3D11DeviceContext4> fenceContext;
|
||||
HANDLE sharedHandle = nullptr;
|
||||
HANDLE producerFenceHandle = nullptr;
|
||||
int width = 0;
|
||||
int height = 0;
|
||||
DXGI_FORMAT format = DXGI_FORMAT_UNKNOWN;
|
||||
std::uint64_t sequence = 0;
|
||||
};
|
||||
|
||||
bool debugEnabled()
|
||||
{
|
||||
return GetEnvironmentVariableA("MIMIR_MF_GPU_DEBUG", nullptr, 0) > 0;
|
||||
}
|
||||
|
||||
void debugLog(const char* format, ...)
|
||||
{
|
||||
if (!debugEnabled())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
std::fprintf(stderr, "mimir-mf-gpu: ");
|
||||
va_list args;
|
||||
va_start(args, format);
|
||||
std::vfprintf(stderr, format, args);
|
||||
va_end(args);
|
||||
std::fprintf(stderr, "\n");
|
||||
}
|
||||
|
||||
std::wstring widen(const char* value)
|
||||
{
|
||||
if (!value)
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
const int chars = MultiByteToWideChar(CP_UTF8, 0, value, -1, nullptr, 0);
|
||||
if (chars <= 1)
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
std::wstring result(static_cast<std::size_t>(chars - 1), L'\0');
|
||||
MultiByteToWideChar(CP_UTF8, 0, value, -1, result.data(), chars);
|
||||
return result;
|
||||
}
|
||||
|
||||
std::wstring lowercase(std::wstring value)
|
||||
{
|
||||
std::transform(value.begin(), value.end(), value.begin(), [](wchar_t c) {
|
||||
return static_cast<wchar_t>(towlower(c));
|
||||
});
|
||||
return value;
|
||||
}
|
||||
|
||||
GUID subtypeFromText(const char* text)
|
||||
{
|
||||
const std::string value = text ? text : "";
|
||||
std::string upper;
|
||||
upper.resize(value.size());
|
||||
std::transform(value.begin(), value.end(), upper.begin(), [](unsigned char c) {
|
||||
return static_cast<char>(std::toupper(c));
|
||||
});
|
||||
|
||||
if (upper == "MJPG") return MFVideoFormat_MJPG;
|
||||
if (upper == "H264") return MFVideoFormat_H264;
|
||||
if (upper == "NV12") return MFVideoFormat_NV12;
|
||||
if (upper == "YUY2") return MFVideoFormat_YUY2;
|
||||
if (upper == "RGB32" || upper == "BGRA") return MFVideoFormat_RGB32;
|
||||
return GUID_NULL;
|
||||
}
|
||||
|
||||
DXGI_FORMAT dxgiFormatForSubtype(const GUID& subtype)
|
||||
{
|
||||
if (subtype == MFVideoFormat_NV12) return DXGI_FORMAT_NV12;
|
||||
if (subtype == MFVideoFormat_RGB32) return DXGI_FORMAT_B8G8R8A8_UNORM;
|
||||
if (subtype == MFVideoFormat_YUY2) return DXGI_FORMAT_YUY2;
|
||||
return DXGI_FORMAT_UNKNOWN;
|
||||
}
|
||||
|
||||
const char* textForSubtype(const GUID& subtype)
|
||||
{
|
||||
if (subtype == MFVideoFormat_NV12) return "NV12";
|
||||
if (subtype == MFVideoFormat_RGB32) return "Bgra8";
|
||||
if (subtype == MFVideoFormat_YUY2) return "YUY2";
|
||||
return "Unknown";
|
||||
}
|
||||
|
||||
bool findVideoSource(const wchar_t* needle, IMFMediaSource** mediaSource)
|
||||
{
|
||||
*mediaSource = nullptr;
|
||||
ComPtr<IMFAttributes> attributes;
|
||||
if (FAILED(MFCreateAttributes(&attributes, 2)))
|
||||
{
|
||||
debugLog("MFCreateAttributes for devices failed");
|
||||
return false;
|
||||
}
|
||||
|
||||
attributes->SetGUID(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE, MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID);
|
||||
IMFActivate** rawDevices = nullptr;
|
||||
UINT32 count = 0;
|
||||
if (FAILED(MFEnumDeviceSources(attributes.Get(), &rawDevices, &count)))
|
||||
{
|
||||
debugLog("MFEnumDeviceSources failed");
|
||||
return false;
|
||||
}
|
||||
|
||||
debugLog("device sources=%u", count);
|
||||
|
||||
const std::wstring target = lowercase(needle ? needle : L"");
|
||||
bool found = false;
|
||||
for (UINT32 index = 0; index < count; ++index)
|
||||
{
|
||||
ComPtr<IMFActivate> activate(rawDevices[index]);
|
||||
rawDevices[index] = nullptr;
|
||||
WCHAR* symbolicLink = nullptr;
|
||||
UINT32 symbolicLength = 0;
|
||||
activate->GetAllocatedString(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK, &symbolicLink, &symbolicLength);
|
||||
const std::wstring link = lowercase(symbolicLink ? symbolicLink : L"");
|
||||
if (debugEnabled())
|
||||
{
|
||||
std::fwprintf(stderr, L"mimir-mf-gpu: source[%u]=%ls\n", index, link.c_str());
|
||||
}
|
||||
CoTaskMemFree(symbolicLink);
|
||||
if (!target.empty() && link.find(target) == std::wstring::npos)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (SUCCEEDED(activate->ActivateObject(IID_PPV_ARGS(mediaSource))))
|
||||
{
|
||||
debugLog("activated source index=%u", index);
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
for (UINT32 index = 0; index < count; ++index)
|
||||
{
|
||||
if (rawDevices[index])
|
||||
{
|
||||
rawDevices[index]->Release();
|
||||
}
|
||||
}
|
||||
CoTaskMemFree(rawDevices);
|
||||
return found;
|
||||
}
|
||||
|
||||
bool mediaTypeMatches(IMFMediaType* type, int width, int height, const GUID& subtype, double minFps)
|
||||
{
|
||||
GUID actualSubtype{};
|
||||
UINT32 actualWidth = 0;
|
||||
UINT32 actualHeight = 0;
|
||||
UINT32 fpsNumerator = 0;
|
||||
UINT32 fpsDenominator = 0;
|
||||
if (FAILED(type->GetGUID(MF_MT_SUBTYPE, &actualSubtype)) ||
|
||||
actualSubtype != subtype ||
|
||||
FAILED(MFGetAttributeSize(type, MF_MT_FRAME_SIZE, &actualWidth, &actualHeight)) ||
|
||||
static_cast<int>(actualWidth) != width ||
|
||||
static_cast<int>(actualHeight) != height)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (SUCCEEDED(MFGetAttributeRatio(type, MF_MT_FRAME_RATE, &fpsNumerator, &fpsDenominator)) &&
|
||||
fpsDenominator != 0)
|
||||
{
|
||||
return static_cast<double>(fpsNumerator) / fpsDenominator >= minFps;
|
||||
}
|
||||
|
||||
return minFps <= 0.0;
|
||||
}
|
||||
|
||||
bool setInputType(IMFSourceReader* reader, int width, int height, const GUID& subtype, double minFps)
|
||||
{
|
||||
for (DWORD index = 0;; ++index)
|
||||
{
|
||||
ComPtr<IMFMediaType> type;
|
||||
const HRESULT hr = reader->GetNativeMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, index, &type);
|
||||
if (hr == MF_E_NO_MORE_TYPES)
|
||||
{
|
||||
debugLog("no input type matched %dx%d minFps=%.3f", width, height, minFps);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (FAILED(hr))
|
||||
{
|
||||
debugLog("GetNativeMediaType failed index=%lu hr=0x%08lx", index, static_cast<unsigned long>(hr));
|
||||
return false;
|
||||
}
|
||||
|
||||
if (debugEnabled())
|
||||
{
|
||||
GUID actualSubtype{};
|
||||
UINT32 actualWidth = 0;
|
||||
UINT32 actualHeight = 0;
|
||||
UINT32 fpsNumerator = 0;
|
||||
UINT32 fpsDenominator = 0;
|
||||
type->GetGUID(MF_MT_SUBTYPE, &actualSubtype);
|
||||
MFGetAttributeSize(type.Get(), MF_MT_FRAME_SIZE, &actualWidth, &actualHeight);
|
||||
MFGetAttributeRatio(type.Get(), MF_MT_FRAME_RATE, &fpsNumerator, &fpsDenominator);
|
||||
debugLog(
|
||||
"nativeType[%lu] %ux%u subtype=%08lx fps=%u/%u",
|
||||
index,
|
||||
actualWidth,
|
||||
actualHeight,
|
||||
static_cast<unsigned long>(actualSubtype.Data1),
|
||||
fpsNumerator,
|
||||
fpsDenominator);
|
||||
}
|
||||
|
||||
if (mediaTypeMatches(type.Get(), width, height, subtype, minFps))
|
||||
{
|
||||
const HRESULT setHr = reader->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, type.Get());
|
||||
debugLog("SetCurrentMediaType input index=%lu hr=0x%08lx", index, static_cast<unsigned long>(setHr));
|
||||
return SUCCEEDED(setHr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool setOutputType(IMFSourceReader* reader, const GUID& subtype)
|
||||
{
|
||||
ComPtr<IMFMediaType> output;
|
||||
if (FAILED(MFCreateMediaType(&output)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
output->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
|
||||
output->SetGUID(MF_MT_SUBTYPE, subtype);
|
||||
const HRESULT hr = reader->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, output.Get());
|
||||
debugLog("SetCurrentMediaType output hr=0x%08lx", static_cast<unsigned long>(hr));
|
||||
return SUCCEEDED(hr);
|
||||
}
|
||||
|
||||
bool createSharedTexture(MimirMfGpuCapture& capture, const GUID& outputSubtype)
|
||||
{
|
||||
ComPtr<IMFMediaType> current;
|
||||
if (FAILED(capture.reader->GetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, ¤t)))
|
||||
{
|
||||
debugLog("GetCurrentMediaType output failed");
|
||||
return false;
|
||||
}
|
||||
|
||||
UINT32 width = 0;
|
||||
UINT32 height = 0;
|
||||
GUID subtype{};
|
||||
if (FAILED(MFGetAttributeSize(current.Get(), MF_MT_FRAME_SIZE, &width, &height)) ||
|
||||
FAILED(current->GetGUID(MF_MT_SUBTYPE, &subtype)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto format = dxgiFormatForSubtype(subtype);
|
||||
if (format == DXGI_FORMAT_UNKNOWN || subtype != outputSubtype)
|
||||
{
|
||||
debugLog("output texture subtype/format unsupported subtype=%08lx", static_cast<unsigned long>(subtype.Data1));
|
||||
return false;
|
||||
}
|
||||
|
||||
D3D11_TEXTURE2D_DESC desc{};
|
||||
desc.Width = width;
|
||||
desc.Height = height;
|
||||
desc.MipLevels = 1;
|
||||
desc.ArraySize = 1;
|
||||
desc.Format = format;
|
||||
desc.SampleDesc.Count = 1;
|
||||
desc.Usage = D3D11_USAGE_DEFAULT;
|
||||
desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
|
||||
desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED_NTHANDLE | D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX;
|
||||
|
||||
const HRESULT textureHr = capture.device->CreateTexture2D(&desc, nullptr, &capture.sharedTexture);
|
||||
if (FAILED(textureHr))
|
||||
{
|
||||
debugLog("CreateTexture2D shared NT failed hr=0x%08lx; retrying legacy shared", static_cast<unsigned long>(textureHr));
|
||||
desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED;
|
||||
const HRESULT legacyTextureHr = capture.device->CreateTexture2D(&desc, nullptr, &capture.sharedTexture);
|
||||
if (FAILED(legacyTextureHr))
|
||||
{
|
||||
debugLog("CreateTexture2D legacy shared failed hr=0x%08lx", static_cast<unsigned long>(legacyTextureHr));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
ComPtr<IDXGIResource1> dxgiResource;
|
||||
const HRESULT asHr = capture.sharedTexture.As(&dxgiResource);
|
||||
const HRESULT handleHr = SUCCEEDED(asHr)
|
||||
? dxgiResource->CreateSharedHandle(nullptr, DXGI_SHARED_RESOURCE_READ | DXGI_SHARED_RESOURCE_WRITE, nullptr, &capture.sharedHandle)
|
||||
: asHr;
|
||||
if (FAILED(asHr) || FAILED(handleHr))
|
||||
{
|
||||
ComPtr<IDXGIResource> legacyResource;
|
||||
const HRESULT legacyAsHr = capture.sharedTexture.As(&legacyResource);
|
||||
const HRESULT legacyHandleHr = SUCCEEDED(legacyAsHr)
|
||||
? legacyResource->GetSharedHandle(&capture.sharedHandle)
|
||||
: legacyAsHr;
|
||||
if (FAILED(legacyAsHr) || FAILED(legacyHandleHr))
|
||||
{
|
||||
debugLog(
|
||||
"CreateSharedHandle failed as=0x%08lx handle=0x%08lx legacyAs=0x%08lx legacyHandle=0x%08lx",
|
||||
static_cast<unsigned long>(asHr),
|
||||
static_cast<unsigned long>(handleHr),
|
||||
static_cast<unsigned long>(legacyAsHr),
|
||||
static_cast<unsigned long>(legacyHandleHr));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
capture.width = static_cast<int>(width);
|
||||
capture.height = static_cast<int>(height);
|
||||
capture.format = format;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool createProducerFence(MimirMfGpuCapture& capture)
|
||||
{
|
||||
ComPtr<ID3D11Device5> device5;
|
||||
if (FAILED(capture.device.As(&device5)) ||
|
||||
FAILED(capture.context.As(&capture.fenceContext)))
|
||||
{
|
||||
debugLog("D3D11 fence interfaces unavailable; shared texture will be unsynchronized");
|
||||
return true;
|
||||
}
|
||||
|
||||
const HRESULT fenceHr = device5->CreateFence(
|
||||
0,
|
||||
D3D11_FENCE_FLAG_SHARED,
|
||||
IID_PPV_ARGS(&capture.producerFence));
|
||||
if (FAILED(fenceHr))
|
||||
{
|
||||
debugLog("CreateFence failed hr=0x%08lx; shared texture will be unsynchronized", static_cast<unsigned long>(fenceHr));
|
||||
capture.fenceContext.Reset();
|
||||
return true;
|
||||
}
|
||||
|
||||
const HRESULT handleHr = capture.producerFence->CreateSharedHandle(nullptr, GENERIC_ALL, nullptr, &capture.producerFenceHandle);
|
||||
if (FAILED(handleHr))
|
||||
{
|
||||
debugLog("CreateSharedHandle fence failed hr=0x%08lx; shared texture will be unsynchronized", static_cast<unsigned long>(handleHr));
|
||||
capture.producerFence.Reset();
|
||||
capture.fenceContext.Reset();
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
extern "C"
|
||||
{
|
||||
__declspec(dllexport) MimirMfGpuCapture* mimir_mf_gpu_create(
|
||||
const char* pathNeedle,
|
||||
int width,
|
||||
int height,
|
||||
const char* inputSubtypeText,
|
||||
const char* outputSubtypeText,
|
||||
double minFps)
|
||||
{
|
||||
CoInitializeEx(nullptr, COINIT_MULTITHREADED);
|
||||
if (FAILED(MFStartup(MF_VERSION, MFSTARTUP_NOSOCKET)))
|
||||
{
|
||||
debugLog("MFStartup failed");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto capture = new MimirMfGpuCapture();
|
||||
const UINT deviceFlags = D3D11_CREATE_DEVICE_VIDEO_SUPPORT | D3D11_CREATE_DEVICE_BGRA_SUPPORT;
|
||||
D3D_FEATURE_LEVEL featureLevels[] = { D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0 };
|
||||
D3D_FEATURE_LEVEL featureLevel{};
|
||||
const HRESULT deviceHr = D3D11CreateDevice(
|
||||
nullptr,
|
||||
D3D_DRIVER_TYPE_HARDWARE,
|
||||
nullptr,
|
||||
deviceFlags,
|
||||
featureLevels,
|
||||
ARRAYSIZE(featureLevels),
|
||||
D3D11_SDK_VERSION,
|
||||
&capture->device,
|
||||
&featureLevel,
|
||||
&capture->context);
|
||||
if (FAILED(deviceHr))
|
||||
{
|
||||
debugLog("D3D11CreateDevice failed hr=0x%08lx", static_cast<unsigned long>(deviceHr));
|
||||
delete capture;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
ComPtr<ID3D10Multithread> multithread;
|
||||
if (SUCCEEDED(capture->device.As(&multithread)))
|
||||
{
|
||||
multithread->SetMultithreadProtected(TRUE);
|
||||
}
|
||||
|
||||
UINT resetToken = 0;
|
||||
if (FAILED(MFCreateDXGIDeviceManager(&resetToken, &capture->deviceManager)) ||
|
||||
FAILED(capture->deviceManager->ResetDevice(capture->device.Get(), resetToken)))
|
||||
{
|
||||
debugLog("DXGI device manager setup failed");
|
||||
delete capture;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
const auto needle = widen(pathNeedle);
|
||||
if (!findVideoSource(needle.c_str(), &capture->source))
|
||||
{
|
||||
debugLog("no source matched needle");
|
||||
delete capture;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
ComPtr<IMFAttributes> readerAttributes;
|
||||
if (FAILED(MFCreateAttributes(&readerAttributes, 4)))
|
||||
{
|
||||
debugLog("MFCreateAttributes reader failed");
|
||||
delete capture;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
readerAttributes->SetUnknown(MF_SOURCE_READER_D3D_MANAGER, capture->deviceManager.Get());
|
||||
readerAttributes->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, TRUE);
|
||||
readerAttributes->SetUINT32(MF_LOW_LATENCY, TRUE);
|
||||
readerAttributes->SetUINT32(MF_SOURCE_READER_ENABLE_ADVANCED_VIDEO_PROCESSING, TRUE);
|
||||
|
||||
const HRESULT readerHr = MFCreateSourceReaderFromMediaSource(capture->source.Get(), readerAttributes.Get(), &capture->reader);
|
||||
if (FAILED(readerHr))
|
||||
{
|
||||
debugLog("MFCreateSourceReaderFromMediaSource failed hr=0x%08lx", static_cast<unsigned long>(readerHr));
|
||||
delete capture;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
const auto inputSubtype = subtypeFromText(inputSubtypeText);
|
||||
const auto outputSubtype = subtypeFromText(outputSubtypeText && outputSubtypeText[0] ? outputSubtypeText : "NV12");
|
||||
if (inputSubtype == GUID_NULL ||
|
||||
outputSubtype == GUID_NULL ||
|
||||
!setInputType(capture->reader.Get(), width, height, inputSubtype, minFps) ||
|
||||
!setOutputType(capture->reader.Get(), outputSubtype) ||
|
||||
!createSharedTexture(*capture, outputSubtype) ||
|
||||
!createProducerFence(*capture))
|
||||
{
|
||||
debugLog("create failed input=%s output=%s", inputSubtypeText ? inputSubtypeText : "", outputSubtypeText ? outputSubtypeText : "");
|
||||
delete capture;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return capture;
|
||||
}
|
||||
|
||||
__declspec(dllexport) int mimir_mf_gpu_read(
|
||||
MimirMfGpuCapture* capture,
|
||||
HANDLE* sharedHandle,
|
||||
int* width,
|
||||
int* height,
|
||||
char* format,
|
||||
int formatCapacity,
|
||||
std::int64_t* timestampNs,
|
||||
std::uint64_t* sequence,
|
||||
HANDLE* producerFenceHandle)
|
||||
{
|
||||
if (!capture || !capture->reader || !capture->sharedTexture || !sharedHandle)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
DWORD streamIndex = 0;
|
||||
DWORD flags = 0;
|
||||
LONGLONG timestamp = 0;
|
||||
ComPtr<IMFSample> sample;
|
||||
const HRESULT hr = capture->reader->ReadSample(
|
||||
MF_SOURCE_READER_FIRST_VIDEO_STREAM,
|
||||
0,
|
||||
&streamIndex,
|
||||
&flags,
|
||||
×tamp,
|
||||
&sample);
|
||||
if (FAILED(hr) || !sample || (flags & MF_SOURCE_READERF_STREAMTICK) != 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
ComPtr<IMFMediaBuffer> buffer;
|
||||
ComPtr<IMFDXGIBuffer> dxgiBuffer;
|
||||
ComPtr<ID3D11Texture2D> decodedTexture;
|
||||
UINT subresource = 0;
|
||||
if (FAILED(sample->GetBufferByIndex(0, &buffer)) ||
|
||||
FAILED(buffer.As(&dxgiBuffer)) ||
|
||||
FAILED(dxgiBuffer->GetResource(IID_PPV_ARGS(&decodedTexture))) ||
|
||||
FAILED(dxgiBuffer->GetSubresourceIndex(&subresource)))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
const auto nextSequence = capture->sequence + 1;
|
||||
capture->context->CopySubresourceRegion(capture->sharedTexture.Get(), 0, 0, 0, 0, decodedTexture.Get(), subresource, nullptr);
|
||||
if (capture->producerFence && capture->fenceContext)
|
||||
{
|
||||
capture->fenceContext->Signal(capture->producerFence.Get(), nextSequence);
|
||||
}
|
||||
|
||||
capture->context->Flush();
|
||||
*sharedHandle = capture->sharedHandle;
|
||||
if (width) *width = capture->width;
|
||||
if (height) *height = capture->height;
|
||||
if (format && formatCapacity > 0)
|
||||
{
|
||||
const char* text = capture->format == DXGI_FORMAT_NV12
|
||||
? textForSubtype(MFVideoFormat_NV12)
|
||||
: capture->format == DXGI_FORMAT_B8G8R8A8_UNORM
|
||||
? textForSubtype(MFVideoFormat_RGB32)
|
||||
: textForSubtype(MFVideoFormat_YUY2);
|
||||
std::snprintf(format, static_cast<std::size_t>(formatCapacity), "%s", text);
|
||||
}
|
||||
if (timestampNs) *timestampNs = timestamp * 100;
|
||||
capture->sequence = nextSequence;
|
||||
if (sequence) *sequence = capture->sequence;
|
||||
if (producerFenceHandle) *producerFenceHandle = capture->producerFenceHandle;
|
||||
return 1;
|
||||
}
|
||||
|
||||
__declspec(dllexport) void mimir_mf_gpu_destroy(MimirMfGpuCapture* capture)
|
||||
{
|
||||
if (!capture)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (capture->sharedHandle)
|
||||
{
|
||||
CloseHandle(capture->sharedHandle);
|
||||
}
|
||||
if (capture->producerFenceHandle)
|
||||
{
|
||||
CloseHandle(capture->producerFenceHandle);
|
||||
}
|
||||
if (capture->source)
|
||||
{
|
||||
capture->source->Shutdown();
|
||||
}
|
||||
delete capture;
|
||||
MFShutdown();
|
||||
CoUninitialize();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,215 +0,0 @@
|
|||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <condition_variable>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <deque>
|
||||
#include <mutex>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#include "ps3eye_capi.h"
|
||||
|
||||
namespace
|
||||
{
|
||||
struct MimirPs3EyeFrame
|
||||
{
|
||||
std::vector<unsigned char> data;
|
||||
std::int64_t timestampNs = 0;
|
||||
std::uint64_t sequence = 0;
|
||||
};
|
||||
|
||||
struct MimirPs3EyeCapture
|
||||
{
|
||||
ps3eye_t* eye = nullptr;
|
||||
std::thread worker;
|
||||
std::mutex mutex;
|
||||
std::condition_variable condition;
|
||||
std::deque<MimirPs3EyeFrame> queue;
|
||||
std::atomic<bool> running = false;
|
||||
int cameraIndex = 0;
|
||||
int width = 0;
|
||||
int height = 0;
|
||||
int fps = 0;
|
||||
std::uint64_t sequence = 0;
|
||||
};
|
||||
|
||||
std::mutex g_ps3eye_mutex;
|
||||
int g_ps3eye_users = 0;
|
||||
|
||||
long long nowNs()
|
||||
{
|
||||
return std::chrono::duration_cast<std::chrono::nanoseconds>(
|
||||
std::chrono::steady_clock::now().time_since_epoch()).count();
|
||||
}
|
||||
|
||||
void retainPs3Eye()
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(g_ps3eye_mutex);
|
||||
if (g_ps3eye_users++ == 0)
|
||||
{
|
||||
ps3eye_init();
|
||||
}
|
||||
}
|
||||
|
||||
void releasePs3Eye()
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(g_ps3eye_mutex);
|
||||
if (g_ps3eye_users > 0 && --g_ps3eye_users == 0)
|
||||
{
|
||||
ps3eye_uninit();
|
||||
}
|
||||
}
|
||||
|
||||
void pushFrame(MimirPs3EyeCapture& capture, const std::vector<unsigned char>& source)
|
||||
{
|
||||
MimirPs3EyeFrame frame;
|
||||
frame.data = source;
|
||||
frame.timestampNs = nowNs();
|
||||
frame.sequence = capture.sequence++;
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(capture.mutex);
|
||||
capture.queue.push_back(std::move(frame));
|
||||
while (capture.queue.size() > 32)
|
||||
{
|
||||
capture.queue.pop_front();
|
||||
}
|
||||
}
|
||||
|
||||
capture.condition.notify_one();
|
||||
}
|
||||
|
||||
void captureThread(MimirPs3EyeCapture* capture)
|
||||
{
|
||||
std::vector<unsigned char> frame(static_cast<std::size_t>(capture->width) * static_cast<std::size_t>(capture->height));
|
||||
while (capture->running.load(std::memory_order_acquire))
|
||||
{
|
||||
ps3eye_grab_frame(capture->eye, frame.data());
|
||||
pushFrame(*capture, frame);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
extern "C"
|
||||
{
|
||||
__declspec(dllexport) MimirPs3EyeCapture* mimir_ps3eye_create(int cameraIndex, int width, int height, int fps)
|
||||
{
|
||||
if (cameraIndex < 0 || width <= 0 || height <= 0 || fps <= 0)
|
||||
{
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
retainPs3Eye();
|
||||
if (cameraIndex >= ps3eye_count_connected())
|
||||
{
|
||||
releasePs3Eye();
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto* capture = new MimirPs3EyeCapture();
|
||||
capture->cameraIndex = cameraIndex;
|
||||
capture->width = width;
|
||||
capture->height = height;
|
||||
capture->fps = fps;
|
||||
capture->eye = ps3eye_open(cameraIndex, width, height, fps, PS3EYE_FORMAT_BAYER);
|
||||
if (!capture->eye)
|
||||
{
|
||||
delete capture;
|
||||
releasePs3Eye();
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return capture;
|
||||
}
|
||||
|
||||
__declspec(dllexport) int mimir_ps3eye_start(MimirPs3EyeCapture* capture)
|
||||
{
|
||||
if (!capture || capture->running.exchange(true))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
capture->worker = std::thread(captureThread, capture);
|
||||
return 1;
|
||||
}
|
||||
|
||||
__declspec(dllexport) int mimir_ps3eye_read(
|
||||
MimirPs3EyeCapture* capture,
|
||||
int* width,
|
||||
int* height,
|
||||
int* stride,
|
||||
std::int64_t* timestampNs,
|
||||
std::uint64_t* sequence,
|
||||
unsigned char* destination,
|
||||
int destinationBytes)
|
||||
{
|
||||
if (!capture || !destination || destinationBytes <= 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
MimirPs3EyeFrame frame;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(capture->mutex);
|
||||
if (capture->queue.empty())
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
frame = std::move(capture->queue.front());
|
||||
capture->queue.pop_front();
|
||||
}
|
||||
|
||||
if (frame.data.size() > static_cast<std::size_t>(destinationBytes))
|
||||
{
|
||||
return -static_cast<int>(frame.data.size());
|
||||
}
|
||||
|
||||
std::memcpy(destination, frame.data.data(), frame.data.size());
|
||||
if (width)
|
||||
{
|
||||
*width = capture->width;
|
||||
}
|
||||
if (height)
|
||||
{
|
||||
*height = capture->height;
|
||||
}
|
||||
if (stride)
|
||||
{
|
||||
*stride = capture->width;
|
||||
}
|
||||
if (timestampNs)
|
||||
{
|
||||
*timestampNs = frame.timestampNs;
|
||||
}
|
||||
if (sequence)
|
||||
{
|
||||
*sequence = frame.sequence;
|
||||
}
|
||||
|
||||
return static_cast<int>(frame.data.size());
|
||||
}
|
||||
|
||||
__declspec(dllexport) void mimir_ps3eye_destroy(MimirPs3EyeCapture* capture)
|
||||
{
|
||||
if (!capture)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
capture->running.store(false, std::memory_order_release);
|
||||
if (capture->worker.joinable())
|
||||
{
|
||||
capture->worker.join();
|
||||
}
|
||||
if (capture->eye)
|
||||
{
|
||||
ps3eye_close(capture->eye);
|
||||
}
|
||||
|
||||
delete capture;
|
||||
releasePs3Eye();
|
||||
}
|
||||
}
|
||||
|
|
@ -17,11 +17,13 @@ target_compile_definitions(mimir_obs_stem_source PRIVATE
|
|||
_WIN32_WINNT=0x0A00)
|
||||
target_link_libraries(mimir_obs_stem_source PRIVATE OBS::libobs d3d11 d3d12 dxgi ws2_32)
|
||||
|
||||
add_executable(muninn_media_decoder_probe
|
||||
../../tools/muninn_media_decoder_probe.cpp)
|
||||
target_compile_features(muninn_media_decoder_probe PRIVATE cxx_std_20)
|
||||
target_include_directories(muninn_media_decoder_probe PRIVATE
|
||||
${CMAKE_CURRENT_SOURCE_DIR})
|
||||
if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../tools/muninn_media_decoder_probe.cpp")
|
||||
add_executable(muninn_media_decoder_probe
|
||||
../../tools/muninn_media_decoder_probe.cpp)
|
||||
target_compile_features(muninn_media_decoder_probe PRIVATE cxx_std_20)
|
||||
target_include_directories(muninn_media_decoder_probe PRIVATE
|
||||
${CMAKE_CURRENT_SOURCE_DIR})
|
||||
endif()
|
||||
|
||||
add_executable(muninn_rudp_url_tests
|
||||
tests/muninn_rudp_url_tests.cpp)
|
||||
|
|
@ -29,3 +31,36 @@ target_compile_features(muninn_rudp_url_tests PRIVATE cxx_std_20)
|
|||
target_include_directories(muninn_rudp_url_tests PRIVATE
|
||||
${CMAKE_CURRENT_SOURCE_DIR})
|
||||
add_test(NAME muninn_rudp_url_tests COMMAND muninn_rudp_url_tests)
|
||||
|
||||
add_executable(muninn_media_wire_tests
|
||||
tests/muninn_media_wire_tests.cpp)
|
||||
target_compile_features(muninn_media_wire_tests PRIVATE cxx_std_20)
|
||||
target_include_directories(muninn_media_wire_tests PRIVATE
|
||||
${CMAKE_CURRENT_SOURCE_DIR})
|
||||
add_test(NAME muninn_media_wire_tests COMMAND muninn_media_wire_tests)
|
||||
|
||||
add_executable(muninn_audio_fec_tests
|
||||
tests/muninn_audio_fec_tests.cpp)
|
||||
target_compile_features(muninn_audio_fec_tests PRIVATE cxx_std_20)
|
||||
target_include_directories(muninn_audio_fec_tests PRIVATE
|
||||
${CMAKE_CURRENT_SOURCE_DIR})
|
||||
add_test(NAME muninn_audio_fec_tests COMMAND muninn_audio_fec_tests)
|
||||
|
||||
add_executable(muninn_video_fec_tests
|
||||
tests/muninn_video_fec_tests.cpp)
|
||||
target_compile_features(muninn_video_fec_tests PRIVATE cxx_std_20)
|
||||
target_include_directories(muninn_video_fec_tests PRIVATE
|
||||
${CMAKE_CURRENT_SOURCE_DIR})
|
||||
add_test(NAME muninn_video_fec_tests COMMAND muninn_video_fec_tests)
|
||||
|
||||
add_executable(muninn_udp_socket_tests
|
||||
tests/muninn_udp_socket_tests.cpp)
|
||||
target_compile_features(muninn_udp_socket_tests PRIVATE cxx_std_20)
|
||||
target_compile_definitions(muninn_udp_socket_tests PRIVATE
|
||||
WIN32_LEAN_AND_MEAN
|
||||
NOMINMAX
|
||||
_WIN32_WINNT=0x0A00)
|
||||
target_include_directories(muninn_udp_socket_tests PRIVATE
|
||||
${CMAKE_CURRENT_SOURCE_DIR})
|
||||
target_link_libraries(muninn_udp_socket_tests PRIVATE ws2_32)
|
||||
add_test(NAME muninn_udp_socket_tests COMMAND muninn_udp_socket_tests)
|
||||
|
|
|
|||
|
|
@ -214,6 +214,7 @@ bool obs_module_load(void)
|
|||
blog(
|
||||
LOG_INFO,
|
||||
"Mimir OBS sources loaded: mimir_audio_stem_source, mimir_program_texture_source, "
|
||||
"muninn_stream_source, muninn_video_source, muninn_audio_source");
|
||||
"muninn_stream_source, muninn_video_source, muninn_audio_source; "
|
||||
"Muninn Odin discovery build 20260705.2");
|
||||
return true;
|
||||
}
|
||||
|
|
|
|||
179
native/obs_stem_source/muninn_audio_fec.h
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
#pragma once
|
||||
|
||||
#include <array>
|
||||
#include <chrono>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <iterator>
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace muninn_audio_fec {
|
||||
|
||||
inline constexpr size_t DataShards = 4;
|
||||
inline constexpr size_t ParityShards = 2;
|
||||
inline constexpr size_t TotalShards = DataShards + ParityShards;
|
||||
using Bytes = std::vector<uint8_t>;
|
||||
using Shards = std::array<std::optional<Bytes>, TotalShards>;
|
||||
|
||||
inline uint8_t multiply(uint8_t left, uint8_t right)
|
||||
{
|
||||
uint16_t product = 0;
|
||||
uint16_t a = left;
|
||||
uint16_t b = right;
|
||||
while (b != 0) {
|
||||
if ((b & 1u) != 0) product ^= a;
|
||||
b >>= 1u;
|
||||
a <<= 1u;
|
||||
if ((a & 0x100u) != 0) a ^= 0x11du;
|
||||
}
|
||||
return static_cast<uint8_t>(product);
|
||||
}
|
||||
|
||||
inline uint8_t power(uint8_t value, uint16_t exponent)
|
||||
{
|
||||
uint8_t result = 1;
|
||||
while (exponent != 0) {
|
||||
if ((exponent & 1u) != 0) result = multiply(result, value);
|
||||
value = multiply(value, value);
|
||||
exponent >>= 1u;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
inline uint8_t inverse(uint8_t value)
|
||||
{
|
||||
return value == 0 ? 0 : power(value, 254);
|
||||
}
|
||||
|
||||
inline constexpr std::array<std::array<uint8_t, DataShards>, TotalShards> Generator{{
|
||||
{{1, 0, 0, 0}}, {{0, 1, 0, 0}}, {{0, 0, 1, 0}}, {{0, 0, 0, 1}},
|
||||
{{1, 1, 1, 1}}, {{1, 2, 4, 8}}
|
||||
}};
|
||||
|
||||
inline std::array<Bytes, ParityShards> encode(const std::array<Bytes, DataShards> &data)
|
||||
{
|
||||
const size_t bytes = data[0].size();
|
||||
for (const auto &shard : data) if (shard.size() != bytes) return {};
|
||||
std::array<Bytes, ParityShards> parity{Bytes(bytes), Bytes(bytes)};
|
||||
for (size_t p = 0; p < ParityShards; ++p)
|
||||
for (size_t d = 0; d < DataShards; ++d)
|
||||
for (size_t i = 0; i < bytes; ++i)
|
||||
parity[p][i] ^= multiply(Generator[DataShards + p][d], data[d][i]);
|
||||
return parity;
|
||||
}
|
||||
|
||||
inline bool recover(Shards &shards)
|
||||
{
|
||||
size_t shard_bytes = 0;
|
||||
std::array<size_t, DataShards> available{};
|
||||
size_t count = 0;
|
||||
for (size_t i = 0; i < TotalShards; ++i) {
|
||||
if (!shards[i]) continue;
|
||||
if (shard_bytes == 0) shard_bytes = shards[i]->size();
|
||||
if (shards[i]->size() != shard_bytes) return false;
|
||||
if (count < DataShards) available[count++] = i;
|
||||
}
|
||||
if (count < DataShards || shard_bytes == 0) return false;
|
||||
|
||||
std::array<std::array<uint8_t, DataShards * 2>, DataShards> matrix{};
|
||||
for (size_t row = 0; row < DataShards; ++row) {
|
||||
for (size_t col = 0; col < DataShards; ++col) matrix[row][col] = Generator[available[row]][col];
|
||||
matrix[row][DataShards + row] = 1;
|
||||
}
|
||||
for (size_t col = 0; col < DataShards; ++col) {
|
||||
size_t pivot = col;
|
||||
while (pivot < DataShards && matrix[pivot][col] == 0) ++pivot;
|
||||
if (pivot == DataShards) return false;
|
||||
if (pivot != col) std::swap(matrix[pivot], matrix[col]);
|
||||
const uint8_t scale = inverse(matrix[col][col]);
|
||||
for (size_t j = 0; j < DataShards * 2; ++j) matrix[col][j] = multiply(matrix[col][j], scale);
|
||||
for (size_t row = 0; row < DataShards; ++row) {
|
||||
if (row == col) continue;
|
||||
const uint8_t factor = matrix[row][col];
|
||||
for (size_t j = 0; j < DataShards * 2; ++j)
|
||||
matrix[row][j] ^= multiply(factor, matrix[col][j]);
|
||||
}
|
||||
}
|
||||
for (size_t missing = 0; missing < DataShards; ++missing) {
|
||||
if (shards[missing]) continue;
|
||||
Bytes decoded(shard_bytes);
|
||||
for (size_t row = 0; row < DataShards; ++row) {
|
||||
const uint8_t coefficient = matrix[missing][DataShards + row];
|
||||
for (size_t i = 0; i < shard_bytes; ++i)
|
||||
decoded[i] ^= multiply(coefficient, (*shards[available[row]])[i]);
|
||||
}
|
||||
shards[missing] = std::move(decoded);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
struct BlockKey {
|
||||
std::string stream_id;
|
||||
std::string session_id;
|
||||
uint64_t base_packet_id = 0;
|
||||
bool operator<(const BlockKey &other) const
|
||||
{
|
||||
return std::tie(stream_id, session_id, base_packet_id) <
|
||||
std::tie(other.stream_id, other.session_id, other.base_packet_id);
|
||||
}
|
||||
};
|
||||
|
||||
struct RecoveredPacket { uint64_t packet_id; Bytes payload; };
|
||||
|
||||
class BlockReceiver {
|
||||
public:
|
||||
using Clock = std::chrono::steady_clock;
|
||||
explicit BlockReceiver(std::chrono::milliseconds deadline, size_t max_blocks = 128)
|
||||
: deadline_(deadline), max_blocks_(max_blocks) {}
|
||||
|
||||
std::vector<RecoveredPacket> insert(const BlockKey &key, size_t shard_index, Bytes payload,
|
||||
Clock::time_point now)
|
||||
{
|
||||
expire(now);
|
||||
if (shard_index >= TotalShards || payload.empty() || completed_.count(key) != 0) return {};
|
||||
if (blocks_.size() >= max_blocks_ && blocks_.count(key) == 0) blocks_.erase(blocks_.begin());
|
||||
auto [it, created] = blocks_.try_emplace(key, Block{now, payload.size(), {}});
|
||||
Block &block = it->second;
|
||||
if (payload.size() != block.shard_bytes || block.shards[shard_index]) return {};
|
||||
block.shards[shard_index] = std::move(payload);
|
||||
size_t present = 0;
|
||||
for (const auto &shard : block.shards) if (shard) ++present;
|
||||
if (present < DataShards) return {};
|
||||
Shards before = block.shards;
|
||||
if (!recover(block.shards)) return {};
|
||||
std::vector<RecoveredPacket> result;
|
||||
for (size_t i = 0; i < DataShards; ++i)
|
||||
if (!before[i] && block.shards[i]) result.push_back({key.base_packet_id + i, *block.shards[i]});
|
||||
blocks_.erase(it);
|
||||
completed_[key] = now;
|
||||
return result;
|
||||
}
|
||||
|
||||
void expire(Clock::time_point now)
|
||||
{
|
||||
for (auto it = blocks_.begin(); it != blocks_.end();)
|
||||
it = now - it->second.created_at >= deadline_ ? blocks_.erase(it) : std::next(it);
|
||||
for (auto it = completed_.begin(); it != completed_.end();)
|
||||
it = now - it->second >= deadline_ ? completed_.erase(it) : std::next(it);
|
||||
}
|
||||
size_t pending_blocks() const { return blocks_.size(); }
|
||||
void clear()
|
||||
{
|
||||
blocks_.clear();
|
||||
completed_.clear();
|
||||
}
|
||||
|
||||
private:
|
||||
struct Block { Clock::time_point created_at; size_t shard_bytes; Shards shards; };
|
||||
std::chrono::milliseconds deadline_;
|
||||
size_t max_blocks_;
|
||||
std::map<BlockKey, Block> blocks_;
|
||||
std::map<BlockKey, Clock::time_point> completed_;
|
||||
};
|
||||
|
||||
} // namespace muninn_audio_fec
|
||||
|
|
@ -172,12 +172,18 @@ inline void write_u64_array(MsgpackWriter &writer, const std::vector<uint64_t> &
|
|||
}
|
||||
}
|
||||
|
||||
inline std::vector<uint64_t> receiver_feedback_late_frame_ids(uint64_t frame_id, bool late_frame)
|
||||
{
|
||||
return late_frame ? std::vector<uint64_t>{frame_id} : std::vector<uint64_t>{};
|
||||
}
|
||||
|
||||
inline std::vector<uint8_t> encode_receiver_feedback_payload(
|
||||
const std::string &stream_id,
|
||||
const std::string &session_id,
|
||||
uint64_t frame_id,
|
||||
const std::vector<std::string> &missing_chunk_keys,
|
||||
bool requested_keyframe,
|
||||
bool late_frame,
|
||||
const std::string &observed_at)
|
||||
{
|
||||
MsgpackWriter record;
|
||||
|
|
@ -191,7 +197,7 @@ inline std::vector<uint8_t> encode_receiver_feedback_payload(
|
|||
record.write_nil();
|
||||
}
|
||||
write_u64_array(record, {});
|
||||
write_u64_array(record, {frame_id});
|
||||
write_u64_array(record, receiver_feedback_late_frame_ids(frame_id, late_frame));
|
||||
record.write_bool(requested_keyframe);
|
||||
record.write_i64(0);
|
||||
record.write_i64(0);
|
||||
|
|
@ -247,6 +253,7 @@ inline std::vector<uint8_t> encode_receiver_feedback_payload(
|
|||
frame_id,
|
||||
missing_chunk_keys,
|
||||
requested_keyframe,
|
||||
requested_keyframe,
|
||||
feedback_observed_at());
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -13,11 +13,12 @@ inline constexpr uint32_t DefaultAudioRudpConnectionId = 0x6d750004;
|
|||
inline constexpr uint32_t DefaultReliableExpireAfterMs = 75;
|
||||
inline constexpr uint32_t DefaultVideoAssemblyDeadlineMs = 75;
|
||||
inline constexpr uint32_t DefaultGapWaitMs = 8;
|
||||
inline constexpr uint32_t DefaultAudioReorderMs = 2000;
|
||||
inline constexpr uint32_t DefaultAudioReorderMs = 120;
|
||||
inline constexpr uint32_t DefaultSenderResendDelayMs = 5;
|
||||
inline constexpr uint32_t MaxLatencyBudgetMs = 2000;
|
||||
inline constexpr uint32_t MaxLatencyBudgetMs = 500;
|
||||
|
||||
struct RudpUrlParts {
|
||||
std::string channel_id = "media";
|
||||
uint16_t port = 0;
|
||||
uint16_t video_local_port = 0;
|
||||
uint16_t audio_local_port = 0;
|
||||
|
|
@ -81,7 +82,9 @@ inline std::optional<RudpUrlParts> parse(const std::string &url)
|
|||
}
|
||||
const auto key = item.substr(0, equals);
|
||||
const auto value = item.substr(equals + 1);
|
||||
if (key == "connection") {
|
||||
if (key == "channel" && !value.empty()) {
|
||||
parts.channel_id = value;
|
||||
} else if (key == "connection") {
|
||||
if (const auto parsed = parse_u32_hex_or_decimal(value)) {
|
||||
parts.connection_id = *parsed;
|
||||
}
|
||||
|
|
|
|||
32
native/obs_stem_source/muninn_udp_socket.h
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
#pragma once
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <winsock2.h>
|
||||
|
||||
namespace muninn_udp_socket {
|
||||
|
||||
inline SOCKET create(bool exclusive_listener = false)
|
||||
{
|
||||
const SOCKET socket = WSASocketW(
|
||||
AF_INET,
|
||||
SOCK_DGRAM,
|
||||
IPPROTO_UDP,
|
||||
nullptr,
|
||||
0,
|
||||
WSA_FLAG_NO_HANDLE_INHERIT);
|
||||
if (socket == INVALID_SOCKET) {
|
||||
return INVALID_SOCKET;
|
||||
}
|
||||
if (exclusive_listener) {
|
||||
const BOOL exclusive = TRUE;
|
||||
if (setsockopt(socket, SOL_SOCKET, SO_EXCLUSIVEADDRUSE,
|
||||
reinterpret_cast<const char *>(&exclusive), sizeof(exclusive)) != 0) {
|
||||
closesocket(socket);
|
||||
return INVALID_SOCKET;
|
||||
}
|
||||
}
|
||||
return socket;
|
||||
}
|
||||
|
||||
} // namespace muninn_udp_socket
|
||||
#endif
|
||||
181
native/obs_stem_source/muninn_video_fec.h
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace muninn_video_fec {
|
||||
|
||||
inline uint8_t raw_multiply(uint8_t a, uint8_t b)
|
||||
{
|
||||
uint8_t product = 0;
|
||||
while (b != 0) {
|
||||
if ((b & 1) != 0)
|
||||
product ^= a;
|
||||
const bool carry = (a & 0x80) != 0;
|
||||
a <<= 1;
|
||||
if (carry)
|
||||
a ^= 0x1d;
|
||||
b >>= 1;
|
||||
}
|
||||
return product;
|
||||
}
|
||||
|
||||
struct Tables {
|
||||
uint8_t products[256][256] = {};
|
||||
uint8_t inverses[256] = {};
|
||||
|
||||
Tables()
|
||||
{
|
||||
for (size_t a = 0; a < 256; ++a) {
|
||||
for (size_t b = 0; b < 256; ++b)
|
||||
products[a][b] = raw_multiply(static_cast<uint8_t>(a), static_cast<uint8_t>(b));
|
||||
}
|
||||
for (size_t value = 1; value < 256; ++value) {
|
||||
uint8_t result = 1;
|
||||
for (size_t index = 0; index < 254; ++index)
|
||||
result = raw_multiply(result, static_cast<uint8_t>(value));
|
||||
inverses[value] = result;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
inline const Tables &tables()
|
||||
{
|
||||
static const Tables value;
|
||||
return value;
|
||||
}
|
||||
|
||||
inline uint8_t multiply(uint8_t a, uint8_t b)
|
||||
{
|
||||
return tables().products[a][b];
|
||||
}
|
||||
|
||||
inline uint8_t inverse(uint8_t value)
|
||||
{
|
||||
return tables().inverses[value];
|
||||
}
|
||||
|
||||
inline uint8_t coefficient(uint16_t parity_index, uint16_t parity_count, uint16_t data_index)
|
||||
{
|
||||
return inverse(
|
||||
static_cast<uint8_t>(parity_index) ^ static_cast<uint8_t>(parity_count + data_index));
|
||||
}
|
||||
|
||||
inline bool recover(std::vector<std::vector<uint8_t>> &chunks,
|
||||
const std::vector<uint32_t> &chunk_lengths,
|
||||
uint16_t parity_count,
|
||||
const std::map<uint16_t, std::vector<uint8_t>> &parity)
|
||||
{
|
||||
if (chunks.size() != chunk_lengths.size() || chunks.empty() ||
|
||||
chunks.size() + parity_count > 256)
|
||||
return false;
|
||||
std::vector<uint16_t> missing;
|
||||
for (uint16_t index = 0; index < chunks.size(); ++index) {
|
||||
if (chunks[index].empty())
|
||||
missing.push_back(index);
|
||||
}
|
||||
if (missing.empty())
|
||||
return true;
|
||||
if (missing.size() > parity.size())
|
||||
return false;
|
||||
|
||||
std::vector<std::pair<uint16_t, const std::vector<uint8_t> *>> rows;
|
||||
for (const auto &shard : parity) {
|
||||
if (shard.first < parity_count && !shard.second.empty())
|
||||
rows.emplace_back(shard.first, &shard.second);
|
||||
if (rows.size() == missing.size())
|
||||
break;
|
||||
}
|
||||
if (rows.size() != missing.size())
|
||||
return false;
|
||||
|
||||
size_t shard_bytes = 0;
|
||||
for (uint32_t length : chunk_lengths)
|
||||
shard_bytes = std::max(shard_bytes, static_cast<size_t>(length));
|
||||
if (shard_bytes == 0 || std::any_of(rows.begin(), rows.end(),
|
||||
[shard_bytes](const auto &row) { return row.second->size() < shard_bytes; }))
|
||||
return false;
|
||||
|
||||
std::vector<std::vector<uint8_t>> recovered;
|
||||
recovered.reserve(missing.size());
|
||||
for (uint16_t index : missing) {
|
||||
if (chunk_lengths[index] == 0 || chunk_lengths[index] > shard_bytes)
|
||||
return false;
|
||||
recovered.emplace_back(chunk_lengths[index], 0);
|
||||
}
|
||||
|
||||
const size_t count = missing.size();
|
||||
std::vector<std::vector<uint8_t>> inverse_matrix(count, std::vector<uint8_t>(count * 2, 0));
|
||||
for (size_t row = 0; row < count; ++row) {
|
||||
for (size_t column = 0; column < count; ++column)
|
||||
inverse_matrix[row][column] = coefficient(rows[row].first, parity_count, missing[column]);
|
||||
inverse_matrix[row][count + row] = 1;
|
||||
}
|
||||
for (size_t column = 0; column < count; ++column) {
|
||||
size_t pivot = column;
|
||||
while (pivot < count && inverse_matrix[pivot][column] == 0)
|
||||
++pivot;
|
||||
if (pivot == count)
|
||||
return false;
|
||||
std::swap(inverse_matrix[column], inverse_matrix[pivot]);
|
||||
const uint8_t scale = inverse(inverse_matrix[column][column]);
|
||||
for (size_t cell = 0; cell < count * 2; ++cell)
|
||||
inverse_matrix[column][cell] = multiply(inverse_matrix[column][cell], scale);
|
||||
for (size_t row = 0; row < count; ++row) {
|
||||
if (row == column)
|
||||
continue;
|
||||
const uint8_t factor = inverse_matrix[row][column];
|
||||
for (size_t cell = 0; cell < count * 2; ++cell)
|
||||
inverse_matrix[row][cell] ^= multiply(factor, inverse_matrix[column][cell]);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<uint8_t> rhs(count, 0);
|
||||
for (size_t offset = 0; offset < shard_bytes; ++offset) {
|
||||
for (size_t row = 0; row < count; ++row) {
|
||||
const uint16_t parity_index = rows[row].first;
|
||||
rhs[row] = (*rows[row].second)[offset];
|
||||
for (uint16_t data_index = 0; data_index < chunks.size(); ++data_index) {
|
||||
if (!chunks[data_index].empty() && offset < chunks[data_index].size())
|
||||
rhs[row] ^= multiply(coefficient(parity_index, parity_count, data_index),
|
||||
chunks[data_index][offset]);
|
||||
}
|
||||
}
|
||||
for (size_t index = 0; index < count; ++index) {
|
||||
uint8_t value = 0;
|
||||
for (size_t row = 0; row < count; ++row)
|
||||
value ^= multiply(inverse_matrix[index][count + row], rhs[row]);
|
||||
if (offset < recovered[index].size())
|
||||
recovered[index][offset] = value;
|
||||
}
|
||||
}
|
||||
for (size_t index = 0; index < count; ++index)
|
||||
chunks[missing[index]] = std::move(recovered[index]);
|
||||
return true;
|
||||
}
|
||||
|
||||
inline bool recover_block(std::vector<std::vector<uint8_t>> &frame_chunks,
|
||||
uint16_t data_start,
|
||||
const std::vector<uint32_t> &block_lengths,
|
||||
uint16_t parity_count,
|
||||
const std::map<uint16_t, std::vector<uint8_t>> &parity)
|
||||
{
|
||||
if (block_lengths.empty() ||
|
||||
static_cast<size_t>(data_start) + block_lengths.size() > frame_chunks.size())
|
||||
return false;
|
||||
std::vector<std::vector<uint8_t>> block_chunks;
|
||||
block_chunks.reserve(block_lengths.size());
|
||||
for (size_t index = 0; index < block_lengths.size(); ++index)
|
||||
block_chunks.push_back(frame_chunks[data_start + index]);
|
||||
if (!recover(block_chunks, block_lengths, parity_count, parity))
|
||||
return false;
|
||||
for (size_t index = 0; index < block_lengths.size(); ++index)
|
||||
frame_chunks[data_start + index] = std::move(block_chunks[index]);
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace muninn_video_fec
|
||||
79
native/obs_stem_source/tests/muninn_audio_fec_tests.cpp
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
#include "muninn_audio_fec.h"
|
||||
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
#include <stdexcept>
|
||||
|
||||
static void require(bool condition, const char *message) { if (!condition) throw std::runtime_error(message); }
|
||||
|
||||
static std::array<muninn_audio_fec::Bytes, 4> fixture()
|
||||
{
|
||||
std::array<muninn_audio_fec::Bytes, 4> data;
|
||||
for (size_t shard = 0; shard < data.size(); ++shard) {
|
||||
data[shard].resize(257);
|
||||
for (size_t i = 0; i < data[shard].size(); ++i) data[shard][i] = uint8_t((shard * 71 + i * 29) & 0xff);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
static muninn_audio_fec::Shards all_shards(const std::array<muninn_audio_fec::Bytes, 4> &data)
|
||||
{
|
||||
auto parity = muninn_audio_fec::encode(data);
|
||||
return {data[0], data[1], data[2], data[3], parity[0], parity[1]};
|
||||
}
|
||||
|
||||
int main()
|
||||
{
|
||||
try {
|
||||
const auto data = fixture();
|
||||
for (size_t erased = 0; erased < 6; ++erased) {
|
||||
auto shards = all_shards(data); shards[erased].reset();
|
||||
require(muninn_audio_fec::recover(shards), "one-erasure recovery failed");
|
||||
for (size_t i = 0; i < 4; ++i) require(shards[i] == data[i], "one-erasure data mismatch");
|
||||
}
|
||||
for (size_t first = 0; first < 6; ++first) for (size_t second = first + 1; second < 6; ++second) {
|
||||
auto shards = all_shards(data); shards[first].reset(); shards[second].reset();
|
||||
require(muninn_audio_fec::recover(shards), "two-erasure recovery failed");
|
||||
for (size_t i = 0; i < 4; ++i) require(shards[i] == data[i], "two-erasure data mismatch");
|
||||
}
|
||||
{
|
||||
auto shards = all_shards(data);
|
||||
shards[0].reset(); shards[1].reset(); shards[2].reset();
|
||||
require(!muninn_audio_fec::recover(shards), "three erasures were incorrectly recoverable");
|
||||
}
|
||||
{
|
||||
auto shards = all_shards(data);
|
||||
shards[5]->pop_back();
|
||||
require(!muninn_audio_fec::recover(shards), "unequal shard sizes were accepted");
|
||||
}
|
||||
|
||||
using namespace muninn_audio_fec;
|
||||
BlockReceiver receiver(std::chrono::milliseconds(10), 2);
|
||||
BlockKey key{"stream", "session", 100};
|
||||
auto shards = all_shards(data);
|
||||
auto t0 = BlockReceiver::Clock::time_point{};
|
||||
require(receiver.insert(key, 5, *shards[5], t0).empty(), "early parity emitted data");
|
||||
require(receiver.insert(key, 2, *shards[2], t0).empty(), "partial reorder emitted data");
|
||||
require(receiver.insert(key, 2, *shards[2], t0).empty(), "duplicate emitted data");
|
||||
require(receiver.insert(key, 1, *shards[1], t0).empty(), "partial block emitted data");
|
||||
auto recovered = receiver.insert(key, 4, *shards[4], t0);
|
||||
require(recovered.size() == 2 && recovered[0].packet_id == 100 && recovered[1].packet_id == 103,
|
||||
"reordered block did not recover missing packet ids");
|
||||
require(recovered[0].payload == data[0] && recovered[1].payload == data[3], "receiver recovery mismatch");
|
||||
require(receiver.insert(key, 0, data[0], t0).empty(), "completed-block duplicate was admitted");
|
||||
|
||||
BlockKey expired{"stream", "session", 200};
|
||||
receiver.insert(expired, 0, data[0], t0);
|
||||
receiver.expire(t0 + std::chrono::milliseconds(10));
|
||||
require(receiver.pending_blocks() == 0, "expired block survived deadline");
|
||||
|
||||
BlockReceiver bounded(std::chrono::seconds(1), 2);
|
||||
bounded.insert({"s", "x", 0}, 0, data[0], t0);
|
||||
bounded.insert({"s", "x", 4}, 0, data[0], t0);
|
||||
bounded.insert({"s", "x", 8}, 0, data[0], t0);
|
||||
require(bounded.pending_blocks() == 2, "block bound was not enforced");
|
||||
} catch (const std::exception &error) {
|
||||
std::cerr << error.what() << '\n'; return EXIT_FAILURE;
|
||||
}
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
54
native/obs_stem_source/tests/muninn_media_wire_tests.cpp
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
#include "muninn_media_wire.h"
|
||||
|
||||
#include <cstdlib>
|
||||
#include <iomanip>
|
||||
#include <iostream>
|
||||
|
||||
namespace {
|
||||
|
||||
void require(bool condition, const char *message)
|
||||
{
|
||||
if (!condition) {
|
||||
std::cerr << message << '\n';
|
||||
std::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
const auto early = muninn_media_wire::receiver_feedback_late_frame_ids(42, false);
|
||||
require(early.empty(), "early repair feedback marked a live frame late");
|
||||
|
||||
const auto expired = muninn_media_wire::receiver_feedback_late_frame_ids(42, true);
|
||||
require(expired.size() == 1 && expired.front() == 42,
|
||||
"expired feedback did not name the late frame");
|
||||
|
||||
const auto early_wire = muninn_media_wire::encode_receiver_feedback_payload(
|
||||
"muninn.raven.av.rudp",
|
||||
"raven:session:video",
|
||||
42,
|
||||
{"42:3"},
|
||||
false,
|
||||
false,
|
||||
"unix-ms:1000");
|
||||
const auto expired_wire = muninn_media_wire::encode_receiver_feedback_payload(
|
||||
"muninn.raven.av.rudp",
|
||||
"raven:session:video",
|
||||
42,
|
||||
{"42:3"},
|
||||
true,
|
||||
true,
|
||||
"unix-ms:1000");
|
||||
require(!early_wire.empty() && !expired_wire.empty(), "feedback wire encoding was empty");
|
||||
require(early_wire != expired_wire, "early and expired feedback encoded identically");
|
||||
if (argc == 2 && std::string(argv[1]) == "--emit-early-hex") {
|
||||
for (const auto byte : early_wire) {
|
||||
std::cout << std::hex << std::setw(2) << std::setfill('0')
|
||||
<< static_cast<unsigned int>(byte);
|
||||
}
|
||||
std::cout << '\n';
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -19,17 +19,18 @@ void require(bool condition, const std::string &message)
|
|||
int main()
|
||||
{
|
||||
const auto configured = muninn_rudp_url::parse(
|
||||
"rudp://192.168.1.66:5204/muninn.raven.av.rudp?channel=media&format=muninn-typed-media&connection=0x6d750001&audio_connection=0x6d750004&sender_resend_delay_ms=5&reliable_expire_after_ms=1200&assembly_deadline_ms=1200&gap_wait_ms=16&audio_reorder_ms=700");
|
||||
"rudp://192.168.1.66:5204/muninn.raven.av.rudp?channel=realtime&format=muninn-typed-media&connection=0x6d750001&audio_connection=0x6d750004&sender_resend_delay_ms=5&reliable_expire_after_ms=120&assembly_deadline_ms=120&gap_wait_ms=16&audio_reorder_ms=80");
|
||||
require(configured.has_value(), "configured RUDP URL did not parse");
|
||||
require(configured->port == 5204, "configured RUDP URL port was not parsed");
|
||||
require(configured->channel_id == "realtime", "configured RUDP realtime channel was not parsed");
|
||||
require(configured->video_local_port == 15204, "configured RUDP video local port was not derived");
|
||||
require(configured->audio_local_port == 15205, "configured RUDP audio local port was not derived");
|
||||
require(configured->connection_id == 0x6d750001, "configured RUDP connection id was not parsed");
|
||||
require(configured->audio_connection_id == 0x6d750004, "configured RUDP audio connection id was not parsed");
|
||||
require(configured->reliable_expire_after_ms == 1200, "configured reliable expiry did not survive parse");
|
||||
require(configured->video_assembly_deadline_ms == 1200, "configured assembly deadline did not survive parse");
|
||||
require(configured->reliable_expire_after_ms == 120, "configured reliable expiry did not survive parse");
|
||||
require(configured->video_assembly_deadline_ms == 120, "configured assembly deadline did not survive parse");
|
||||
require(configured->gap_wait_ms == 16, "configured gap wait did not survive parse");
|
||||
require(configured->audio_reorder_ms == 700, "configured audio reorder wait did not survive parse");
|
||||
require(configured->audio_reorder_ms == 80, "configured audio reorder wait did not survive parse");
|
||||
|
||||
const auto clamped = muninn_rudp_url::parse(
|
||||
"rudp://192.168.1.66:5204/muninn.raven.av.rudp?reliable_expire_after_ms=9000&assembly_deadline_ms=9000");
|
||||
|
|
@ -41,6 +42,7 @@ int main()
|
|||
|
||||
const auto fallback = muninn_rudp_url::parse("rudp://192.168.1.66:5204/muninn.raven.av.rudp");
|
||||
require(fallback.has_value(), "minimal RUDP URL did not parse");
|
||||
require(fallback->channel_id == "media", "minimal RUDP URL did not use bounded reliable media delivery");
|
||||
require(fallback->connection_id == muninn_rudp_url::DefaultMediaRudpConnectionId,
|
||||
"minimal RUDP URL did not keep the default connection id");
|
||||
require(fallback->audio_connection_id == muninn_rudp_url::DefaultAudioRudpConnectionId,
|
||||
|
|
@ -51,8 +53,8 @@ int main()
|
|||
"minimal RUDP URL did not keep the default assembly deadline");
|
||||
require(fallback->audio_reorder_ms == muninn_rudp_url::DefaultAudioReorderMs,
|
||||
"minimal RUDP URL did not keep the default audio reorder wait");
|
||||
require(fallback->audio_reorder_ms == 2000,
|
||||
"minimal RUDP URL did not use the full default audio reorder budget");
|
||||
require(fallback->audio_reorder_ms == 120,
|
||||
"minimal RUDP URL did not use the bounded realtime audio reorder budget");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
34
native/obs_stem_source/tests/muninn_udp_socket_tests.cpp
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
#include "muninn_udp_socket.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <windows.h>
|
||||
|
||||
int main()
|
||||
{
|
||||
WSADATA data{};
|
||||
assert(WSAStartup(MAKEWORD(2, 2), &data) == 0);
|
||||
|
||||
const SOCKET owner = muninn_udp_socket::create(true);
|
||||
assert(owner != INVALID_SOCKET);
|
||||
DWORD flags = HANDLE_FLAG_INHERIT;
|
||||
assert(GetHandleInformation(reinterpret_cast<HANDLE>(owner), &flags));
|
||||
assert((flags & HANDLE_FLAG_INHERIT) == 0);
|
||||
|
||||
sockaddr_in address{};
|
||||
address.sin_family = AF_INET;
|
||||
address.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
|
||||
address.sin_port = 0;
|
||||
assert(bind(owner, reinterpret_cast<sockaddr *>(&address), sizeof(address)) == 0);
|
||||
|
||||
int address_size = sizeof(address);
|
||||
assert(getsockname(owner, reinterpret_cast<sockaddr *>(&address), &address_size) == 0);
|
||||
const SOCKET contender = muninn_udp_socket::create(true);
|
||||
assert(contender != INVALID_SOCKET);
|
||||
assert(bind(contender, reinterpret_cast<sockaddr *>(&address), sizeof(address)) != 0);
|
||||
assert(WSAGetLastError() == WSAEADDRINUSE || WSAGetLastError() == WSAEACCES);
|
||||
|
||||
closesocket(contender);
|
||||
closesocket(owner);
|
||||
WSACleanup();
|
||||
return 0;
|
||||
}
|
||||
111
native/obs_stem_source/tests/muninn_video_fec_tests.cpp
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
#include "muninn_video_fec.h"
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdlib>
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <vector>
|
||||
|
||||
void require(bool condition)
|
||||
{
|
||||
if (!condition)
|
||||
std::abort();
|
||||
}
|
||||
|
||||
int main()
|
||||
{
|
||||
constexpr uint16_t data_count = 64;
|
||||
constexpr uint16_t parity_count = 16;
|
||||
constexpr size_t shard_bytes = 1400;
|
||||
std::vector<std::vector<uint8_t>> original(data_count, std::vector<uint8_t>(shard_bytes));
|
||||
for (uint16_t data = 0; data < data_count; ++data) {
|
||||
for (size_t offset = 0; offset < shard_bytes; ++offset)
|
||||
original[data][offset] = static_cast<uint8_t>((data * 29 + offset * 17) & 0xff);
|
||||
}
|
||||
std::map<uint16_t, std::vector<uint8_t>> parity;
|
||||
for (uint16_t row = 0; row < parity_count; ++row) {
|
||||
auto &shard = parity[row];
|
||||
shard.assign(shard_bytes, 0);
|
||||
for (uint16_t data = 0; data < data_count; ++data) {
|
||||
const uint8_t coefficient = muninn_video_fec::coefficient(row, parity_count, data);
|
||||
for (size_t offset = 0; offset < shard_bytes; ++offset)
|
||||
shard[offset] ^= muninn_video_fec::multiply(original[data][offset], coefficient);
|
||||
}
|
||||
}
|
||||
const std::vector<uint32_t> lengths(data_count, shard_bytes);
|
||||
for (uint16_t burst = 1; burst <= parity_count; ++burst) {
|
||||
auto damaged = original;
|
||||
for (uint16_t index = 0; index < burst; ++index)
|
||||
damaged[20 + index].clear();
|
||||
const auto started = std::chrono::steady_clock::now();
|
||||
require(muninn_video_fec::recover(damaged, lengths, parity_count, parity));
|
||||
require(std::chrono::steady_clock::now() - started < std::chrono::milliseconds(50));
|
||||
require(damaged == original);
|
||||
}
|
||||
auto excessive = original;
|
||||
for (uint16_t index = 0; index <= parity_count; ++index)
|
||||
excessive[index].clear();
|
||||
require(!muninn_video_fec::recover(excessive, lengths, parity_count, parity));
|
||||
|
||||
constexpr uint16_t block_data = 8;
|
||||
constexpr uint16_t frame_data = 80;
|
||||
constexpr uint16_t block_count = frame_data / block_data;
|
||||
constexpr size_t block_shard_bytes = 96;
|
||||
std::vector<std::vector<uint8_t>> frame(frame_data, std::vector<uint8_t>(block_shard_bytes));
|
||||
for (uint16_t data = 0; data < frame_data; ++data) {
|
||||
for (size_t offset = 0; offset < block_shard_bytes; ++offset)
|
||||
frame[data][offset] = static_cast<uint8_t>((data * 31 + offset * 13) & 0xff);
|
||||
}
|
||||
std::vector<std::map<uint16_t, std::vector<uint8_t>>> block_parity(block_count);
|
||||
for (uint16_t block = 0; block < block_count; ++block) {
|
||||
for (uint16_t row = 0; row < block_data; ++row) {
|
||||
auto &shard = block_parity[block][row];
|
||||
shard.assign(block_shard_bytes, 0);
|
||||
for (uint16_t local = 0; local < block_data; ++local) {
|
||||
const auto &data = frame[block * block_data + local];
|
||||
const uint8_t coefficient = muninn_video_fec::coefficient(row, block_data, local);
|
||||
for (size_t offset = 0; offset < block_shard_bytes; ++offset)
|
||||
shard[offset] ^= muninn_video_fec::multiply(data[offset], coefficient);
|
||||
}
|
||||
}
|
||||
}
|
||||
struct WireShard {
|
||||
bool parity;
|
||||
uint16_t block;
|
||||
uint16_t local;
|
||||
};
|
||||
std::vector<WireShard> wire;
|
||||
for (uint16_t local = 0; local < block_data; ++local) {
|
||||
for (uint16_t block = 0; block < block_count; ++block)
|
||||
wire.push_back({false, block, local});
|
||||
for (uint16_t block = 0; block < block_count; ++block)
|
||||
wire.push_back({true, block, local});
|
||||
}
|
||||
require(wire.size() == frame_data * 2);
|
||||
const std::vector<uint32_t> block_lengths(block_data, block_shard_bytes);
|
||||
for (size_t burst = 1; burst <= 54; ++burst) {
|
||||
for (size_t start = 0; start + burst <= wire.size(); ++start) {
|
||||
auto damaged = frame;
|
||||
auto available_parity = block_parity;
|
||||
for (size_t index = start; index < start + burst; ++index) {
|
||||
const auto shard = wire[index];
|
||||
if (shard.parity)
|
||||
available_parity[shard.block].erase(shard.local);
|
||||
else
|
||||
damaged[shard.block * block_data + shard.local].clear();
|
||||
}
|
||||
const auto started = std::chrono::steady_clock::now();
|
||||
for (uint16_t block = 0; block < block_count; ++block) {
|
||||
require(muninn_video_fec::recover_block(
|
||||
damaged,
|
||||
block * block_data,
|
||||
block_lengths,
|
||||
block_data,
|
||||
available_parity[block]));
|
||||
}
|
||||
require(std::chrono::steady_clock::now() - started < std::chrono::milliseconds(50));
|
||||
require(damaged == frame);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -125,7 +125,6 @@ struct CaptureState
|
|||
std::vector<unsigned long long> sweepSamples;
|
||||
std::vector<float>* playbackSamples = nullptr;
|
||||
double playbackGain = 1.0;
|
||||
long playbackOutputChannel = -1;
|
||||
std::vector<float>* recordedInput = nullptr;
|
||||
unsigned long long recordFrames = 0;
|
||||
unsigned long long sweepFrame = 0;
|
||||
|
|
@ -321,11 +320,6 @@ void processOutput(long doubleBufferIndex)
|
|||
|
||||
for (size_t channel = 0; channel < g_capture.outputBuffers->size(); ++channel)
|
||||
{
|
||||
if (g_capture.playbackOutputChannel >= 0 && static_cast<long>(channel) != g_capture.playbackOutputChannel)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
auto& buffer = (*g_capture.outputBuffers)[channel];
|
||||
auto& info = (*g_capture.outputChannels)[channel];
|
||||
auto* raw = static_cast<unsigned char*>(buffer.buffers[doubleBufferIndex]);
|
||||
|
|
@ -504,12 +498,6 @@ double doubleOption(int argc, char** argv, const char* name, double fallback)
|
|||
return raw ? std::atof(raw) : fallback;
|
||||
}
|
||||
|
||||
long longOption(int argc, char** argv, const char* name, long fallback)
|
||||
{
|
||||
const char* raw = option(argc, argv, name, nullptr);
|
||||
return raw ? std::atol(raw) : fallback;
|
||||
}
|
||||
|
||||
bool hasArg(int argc, char** argv, const char* name)
|
||||
{
|
||||
for (int index = 1; index < argc; ++index)
|
||||
|
|
@ -588,7 +576,6 @@ int main(int argc, char** argv)
|
|||
const char* playFloat32MonoPath = option(argc, argv, "--play-f32-mono", nullptr);
|
||||
const char* recordFloat32InterleavedPath = option(argc, argv, "--record-f32-interleaved", nullptr);
|
||||
const auto playGain = doubleOption(argc, argv, "--play-gain", 1.0);
|
||||
const auto playOutputChannel = longOption(argc, argv, "--play-output-channel", -1);
|
||||
const auto sweepGain = doubleOption(argc, argv, "--sweep-gain", 0.03);
|
||||
const auto sweepToneSeconds = doubleOption(argc, argv, "--sweep-tone-seconds", 0.55);
|
||||
const auto sweepGapSeconds = doubleOption(argc, argv, "--sweep-gap-seconds", 0.20);
|
||||
|
|
@ -744,7 +731,7 @@ int main(int argc, char** argv)
|
|||
playbackSamples.resize(static_cast<size_t>(byteCount) / sizeof(float));
|
||||
input.read(reinterpret_cast<char*>(playbackSamples.data()), static_cast<std::streamsize>(playbackSamples.size() * sizeof(float)));
|
||||
}
|
||||
std::printf("asio playback-f32-mono: path=%s samples=%zu gain=%.5f outputChannel=%ld\n", playFloat32MonoPath, playbackSamples.size(), playGain, playOutputChannel);
|
||||
std::printf("asio playback-f32-mono: path=%s samples=%zu gain=%.5f\n", playFloat32MonoPath, playbackSamples.size(), playGain);
|
||||
}
|
||||
|
||||
if (captureSeconds > 0.0 || monitorSweep || !playbackSamples.empty() || emitJsonBlocks)
|
||||
|
|
@ -833,7 +820,6 @@ int main(int argc, char** argv)
|
|||
g_capture.sweepGain = sweepGain;
|
||||
g_capture.playbackSamples = playbackSamples.empty() ? nullptr : &playbackSamples;
|
||||
g_capture.playbackGain = playGain;
|
||||
g_capture.playbackOutputChannel = playOutputChannel;
|
||||
g_capture.emitJsonBlocks = emitJsonBlocks;
|
||||
g_capture.sweepFrame = 0;
|
||||
g_capture.sweepEnergy.assign(sweepFrequencies.size() * static_cast<size_t>(inputs), 0.0);
|
||||
|
|
|
|||
|
|
@ -17,7 +17,8 @@ enum LocalcastSampleKind {
|
|||
LOCALCAST_SAMPLE_AUDIO_BLOCK = 5,
|
||||
LOCALCAST_SAMPLE_PHASE_CLAIM = 6,
|
||||
LOCALCAST_SAMPLE_EVENT_CLAIM = 7,
|
||||
LOCALCAST_SAMPLE_RENDER_PACKET = 8
|
||||
LOCALCAST_SAMPLE_RENDER_PACKET = 8,
|
||||
LOCALCAST_SAMPLE_MOVE_EVIDENCE = 9
|
||||
};
|
||||
|
||||
typedef struct LocalcastReservoir LocalcastReservoir;
|
||||
|
|
@ -79,6 +80,47 @@ typedef struct LocalcastRenderPoint {
|
|||
float confidence;
|
||||
} LocalcastRenderPoint;
|
||||
|
||||
enum LocalcastMoveEvidenceKind {
|
||||
LOCALCAST_MOVE_EVIDENCE_OPTICAL_MARKER = 1,
|
||||
LOCALCAST_MOVE_EVIDENCE_CONTROLLER_STATE = 2
|
||||
};
|
||||
|
||||
typedef struct LocalcastMoveEvidenceSample {
|
||||
uint64_t witness_id_hash;
|
||||
uint64_t controller_id_hash;
|
||||
uint64_t source_timestamp_ns;
|
||||
uint64_t arrival_ns;
|
||||
uint64_t sequence;
|
||||
uint32_t evidence_kind;
|
||||
uint32_t flags;
|
||||
float image_x;
|
||||
float image_y;
|
||||
float radius_px;
|
||||
float confidence;
|
||||
float accel_x;
|
||||
float accel_y;
|
||||
float accel_z;
|
||||
float gyro_x;
|
||||
float gyro_y;
|
||||
float gyro_z;
|
||||
float trigger;
|
||||
uint32_t buttons_mask;
|
||||
uint32_t reserved;
|
||||
float battery01;
|
||||
uint32_t reserved1;
|
||||
uint32_t reserved2;
|
||||
} LocalcastMoveEvidenceSample;
|
||||
|
||||
typedef struct LocalcastMoveEvidenceBufferDescriptor {
|
||||
uint64_t sample_buffer_handle;
|
||||
uint32_t sample_count;
|
||||
uint32_t sample_stride_bytes;
|
||||
uint64_t source_time_min_ns;
|
||||
uint64_t source_time_max_ns;
|
||||
uint64_t calibration_hash;
|
||||
uint64_t tracking_space_hash;
|
||||
} LocalcastMoveEvidenceBufferDescriptor;
|
||||
|
||||
typedef struct LocalcastRuntimeStatus {
|
||||
uint64_t edge_ns;
|
||||
uint64_t window_start_ns;
|
||||
|
|
@ -92,6 +134,7 @@ typedef struct LocalcastRuntimeStatus {
|
|||
size_t phase_claim_count;
|
||||
size_t event_claim_count;
|
||||
size_t render_packet_count;
|
||||
size_t move_evidence_count;
|
||||
} LocalcastRuntimeStatus;
|
||||
|
||||
uint64_t localcast_hash_source_id(const uint8_t *bytes, size_t byte_len);
|
||||
|
|
@ -144,6 +187,7 @@ bool localcast_runtime_push_audio_block(LocalcastRuntime *runtime, LocalcastSamp
|
|||
bool localcast_runtime_push_phase_claim(LocalcastRuntime *runtime, LocalcastSampleHandle sample);
|
||||
bool localcast_runtime_push_event_claim(LocalcastRuntime *runtime, LocalcastSampleHandle sample);
|
||||
bool localcast_runtime_push_render_packet(LocalcastRuntime *runtime, LocalcastSampleHandle sample);
|
||||
bool localcast_runtime_push_move_evidence(LocalcastRuntime *runtime, LocalcastSampleHandle sample);
|
||||
bool localcast_runtime_status(const LocalcastRuntime *runtime, LocalcastRuntimeStatus *out_status);
|
||||
size_t localcast_runtime_len(const LocalcastRuntime *runtime);
|
||||
size_t localcast_runtime_view_len(const LocalcastRuntime *runtime, uint32_t sample_kind);
|
||||
|
|
@ -210,6 +254,15 @@ bool localcast_producer_push_render_packet(
|
|||
LocalcastSampleHandle *out_sample
|
||||
);
|
||||
|
||||
bool localcast_producer_push_move_evidence_buffer(
|
||||
LocalcastProducer *producer,
|
||||
LocalcastRuntime *runtime,
|
||||
uint64_t timestamp_ns,
|
||||
uint64_t arrival_ns,
|
||||
const LocalcastMoveEvidenceBufferDescriptor *descriptor,
|
||||
LocalcastSampleHandle *out_sample
|
||||
);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ pub enum SampleKind {
|
|||
PhaseClaim,
|
||||
EventClaim,
|
||||
RenderPacket,
|
||||
MoveEvidence,
|
||||
}
|
||||
|
||||
impl SampleKind {
|
||||
|
|
@ -31,6 +32,7 @@ impl SampleKind {
|
|||
6 => Some(Self::PhaseClaim),
|
||||
7 => Some(Self::EventClaim),
|
||||
8 => Some(Self::RenderPacket),
|
||||
9 => Some(Self::MoveEvidence),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
|
@ -46,6 +48,7 @@ impl SampleKind {
|
|||
Self::PhaseClaim => 6,
|
||||
Self::EventClaim => 7,
|
||||
Self::RenderPacket => 8,
|
||||
Self::MoveEvidence => 9,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -229,6 +232,53 @@ pub struct LocalcastRenderPoint {
|
|||
pub confidence: f32,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum LocalcastMoveEvidenceKind {
|
||||
OpticalMarker = 1,
|
||||
ControllerState = 2,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub struct LocalcastMoveEvidenceSample {
|
||||
pub witness_id_hash: u64,
|
||||
pub controller_id_hash: u64,
|
||||
pub source_timestamp_ns: u64,
|
||||
pub arrival_ns: u64,
|
||||
pub sequence: u64,
|
||||
pub evidence_kind: u32,
|
||||
pub flags: u32,
|
||||
pub image_x: f32,
|
||||
pub image_y: f32,
|
||||
pub radius_px: f32,
|
||||
pub confidence: f32,
|
||||
pub accel_x: f32,
|
||||
pub accel_y: f32,
|
||||
pub accel_z: f32,
|
||||
pub gyro_x: f32,
|
||||
pub gyro_y: f32,
|
||||
pub gyro_z: f32,
|
||||
pub trigger: f32,
|
||||
pub buttons_mask: u32,
|
||||
pub reserved: u32,
|
||||
pub battery01: f32,
|
||||
pub reserved1: u32,
|
||||
pub reserved2: u32,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct LocalcastMoveEvidenceBufferDescriptor {
|
||||
pub sample_buffer_handle: u64,
|
||||
pub sample_count: u32,
|
||||
pub sample_stride_bytes: u32,
|
||||
pub source_time_min_ns: u64,
|
||||
pub source_time_max_ns: u64,
|
||||
pub calibration_hash: u64,
|
||||
pub tracking_space_hash: u64,
|
||||
}
|
||||
|
||||
pub struct LocalcastReservoir {
|
||||
inner: RollingReservoir<LocalcastSampleHandle>,
|
||||
}
|
||||
|
|
@ -258,6 +308,7 @@ pub struct LocalcastRuntimeStatus {
|
|||
pub phase_claim_count: usize,
|
||||
pub event_claim_count: usize,
|
||||
pub render_packet_count: usize,
|
||||
pub move_evidence_count: usize,
|
||||
}
|
||||
|
||||
impl Default for LocalcastRuntimeStatus {
|
||||
|
|
@ -275,6 +326,7 @@ impl Default for LocalcastRuntimeStatus {
|
|||
phase_claim_count: 0,
|
||||
event_claim_count: 0,
|
||||
render_packet_count: 0,
|
||||
move_evidence_count: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -308,6 +360,7 @@ impl LocalcastRuntime {
|
|||
phase_claim_count: self.view_len(SampleKind::PhaseClaim),
|
||||
event_claim_count: self.view_len(SampleKind::EventClaim),
|
||||
render_packet_count: self.view_len(SampleKind::RenderPacket),
|
||||
move_evidence_count: self.view_len(SampleKind::MoveEvidence),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -623,6 +676,14 @@ pub extern "C" fn localcast_runtime_push_render_packet(
|
|||
localcast_runtime_push_typed(ptr, SampleKind::RenderPacket, sample)
|
||||
}
|
||||
|
||||
#[unsafe(no_mangle)]
|
||||
pub extern "C" fn localcast_runtime_push_move_evidence(
|
||||
ptr: *mut LocalcastRuntime,
|
||||
sample: LocalcastSampleHandle,
|
||||
) -> bool {
|
||||
localcast_runtime_push_typed(ptr, SampleKind::MoveEvidence, sample)
|
||||
}
|
||||
|
||||
#[unsafe(no_mangle)]
|
||||
pub extern "C" fn localcast_runtime_status(
|
||||
ptr: *const LocalcastRuntime,
|
||||
|
|
@ -860,6 +921,46 @@ pub extern "C" fn localcast_producer_push_render_packet(
|
|||
)
|
||||
}
|
||||
|
||||
#[unsafe(no_mangle)]
|
||||
pub extern "C" fn localcast_producer_push_move_evidence_buffer(
|
||||
producer_ptr: *mut LocalcastProducer,
|
||||
runtime_ptr: *mut LocalcastRuntime,
|
||||
timestamp_ns: u64,
|
||||
arrival_ns: u64,
|
||||
descriptor: *const LocalcastMoveEvidenceBufferDescriptor,
|
||||
out_sample: *mut LocalcastSampleHandle,
|
||||
) -> bool {
|
||||
let Some(producer) = (unsafe { producer_ptr.as_mut() }) else {
|
||||
return false;
|
||||
};
|
||||
if producer.kind != SampleKind::MoveEvidence {
|
||||
return false;
|
||||
}
|
||||
let Some(descriptor_ref) = (unsafe { descriptor.as_ref() }) else {
|
||||
return false;
|
||||
};
|
||||
if descriptor_ref.sample_buffer_handle == 0 {
|
||||
return false;
|
||||
}
|
||||
if descriptor_ref.sample_count == 0 {
|
||||
return false;
|
||||
}
|
||||
if descriptor_ref.sample_stride_bytes as usize
|
||||
!= std::mem::size_of::<LocalcastMoveEvidenceSample>()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let payload_handle = descriptor as u64;
|
||||
localcast_producer_push(
|
||||
producer_ptr,
|
||||
runtime_ptr,
|
||||
timestamp_ns,
|
||||
arrival_ns,
|
||||
payload_handle,
|
||||
out_sample,
|
||||
)
|
||||
}
|
||||
|
||||
fn localcast_runtime_push_typed(
|
||||
ptr: *mut LocalcastRuntime,
|
||||
kind: SampleKind,
|
||||
|
|
@ -943,6 +1044,51 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
fn move_evidence_descriptor(
|
||||
sample_buffer_handle: u64,
|
||||
) -> LocalcastMoveEvidenceBufferDescriptor {
|
||||
LocalcastMoveEvidenceBufferDescriptor {
|
||||
sample_buffer_handle,
|
||||
sample_count: 2,
|
||||
sample_stride_bytes: std::mem::size_of::<LocalcastMoveEvidenceSample>() as u32,
|
||||
source_time_min_ns: 1_000,
|
||||
source_time_max_ns: 1_120,
|
||||
calibration_hash: 0xCA11_BA7E,
|
||||
tracking_space_hash: 0x57A6E,
|
||||
}
|
||||
}
|
||||
|
||||
fn move_marker_sample(
|
||||
witness_id_hash: u64,
|
||||
controller_id_hash: u64,
|
||||
) -> LocalcastMoveEvidenceSample {
|
||||
LocalcastMoveEvidenceSample {
|
||||
witness_id_hash,
|
||||
controller_id_hash,
|
||||
source_timestamp_ns: 1_000,
|
||||
arrival_ns: 1_010,
|
||||
sequence: 7,
|
||||
evidence_kind: LocalcastMoveEvidenceKind::OpticalMarker as u32,
|
||||
flags: 0,
|
||||
image_x: 123.5,
|
||||
image_y: 87.25,
|
||||
radius_px: 14.0,
|
||||
confidence: 0.92,
|
||||
accel_x: 0.0,
|
||||
accel_y: 0.0,
|
||||
accel_z: 0.0,
|
||||
gyro_x: 0.0,
|
||||
gyro_y: 0.0,
|
||||
gyro_z: 0.0,
|
||||
trigger: 0.0,
|
||||
buttons_mask: 0,
|
||||
reserved: 0,
|
||||
battery01: f32::NAN,
|
||||
reserved1: 0,
|
||||
reserved2: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn render_point(stable_key_hash: u64) -> LocalcastRenderPoint {
|
||||
LocalcastRenderPoint {
|
||||
stable_key_hash,
|
||||
|
|
@ -1585,6 +1731,125 @@ mod tests {
|
|||
localcast_runtime_destroy(runtime);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn move_evidence_layout_is_fixed_for_compute_upload() {
|
||||
let sample = move_marker_sample(11, 22);
|
||||
|
||||
assert_eq!(std::mem::size_of::<LocalcastMoveEvidenceSample>(), 112);
|
||||
assert_eq!(
|
||||
std::mem::size_of::<LocalcastMoveEvidenceBufferDescriptor>(),
|
||||
48
|
||||
);
|
||||
assert_eq!(11, sample.witness_id_hash);
|
||||
assert_eq!(22, sample.controller_id_hash);
|
||||
assert_eq!(
|
||||
LocalcastMoveEvidenceKind::OpticalMarker as u32,
|
||||
sample.evidence_kind
|
||||
);
|
||||
assert_eq!(123.5, sample.image_x);
|
||||
assert_eq!(14.0, sample.radius_px);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn move_evidence_descriptor_enters_native_runtime_window() {
|
||||
let runtime = localcast_runtime_create(DEFAULT_RESERVOIR_NS);
|
||||
let producer = localcast_producer_create(SampleKind::MoveEvidence.id(), 101, 3);
|
||||
let descriptor = move_evidence_descriptor(9_000);
|
||||
let mut sample = handle(0, 0, 0, 0, 0);
|
||||
|
||||
assert!(localcast_producer_push_move_evidence_buffer(
|
||||
producer,
|
||||
runtime,
|
||||
1_200,
|
||||
1_220,
|
||||
&descriptor,
|
||||
&mut sample,
|
||||
));
|
||||
|
||||
let mut status = LocalcastRuntimeStatus::default();
|
||||
assert!(localcast_runtime_status(runtime, &mut status));
|
||||
assert_eq!(1, status.total_sample_count);
|
||||
assert_eq!(1, status.move_evidence_count);
|
||||
assert_eq!(0, status.camera_feature_count);
|
||||
assert_eq!(
|
||||
(&descriptor as *const LocalcastMoveEvidenceBufferDescriptor) as u64,
|
||||
sample.payload_handle
|
||||
);
|
||||
assert_eq!(
|
||||
1,
|
||||
localcast_runtime_view_len(runtime, SampleKind::MoveEvidence.id())
|
||||
);
|
||||
|
||||
localcast_producer_destroy(producer);
|
||||
localcast_runtime_destroy(runtime);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn move_evidence_descriptor_push_rejects_wrong_kind_and_bad_buffers() {
|
||||
let runtime = localcast_runtime_create(DEFAULT_RESERVOIR_NS);
|
||||
let render_producer = localcast_producer_create(SampleKind::RenderPacket.id(), 88, 12);
|
||||
let move_producer = localcast_producer_create(SampleKind::MoveEvidence.id(), 89, 0);
|
||||
let good = move_evidence_descriptor(9_000);
|
||||
let zero_handle = move_evidence_descriptor(0);
|
||||
let zero_count = LocalcastMoveEvidenceBufferDescriptor {
|
||||
sample_count: 0,
|
||||
..good
|
||||
};
|
||||
let bad_stride = LocalcastMoveEvidenceBufferDescriptor {
|
||||
sample_stride_bytes: 4,
|
||||
..good
|
||||
};
|
||||
|
||||
assert!(!localcast_producer_push_move_evidence_buffer(
|
||||
render_producer,
|
||||
runtime,
|
||||
2_000,
|
||||
2_010,
|
||||
&good,
|
||||
std::ptr::null_mut(),
|
||||
));
|
||||
assert!(!localcast_producer_push_move_evidence_buffer(
|
||||
move_producer,
|
||||
runtime,
|
||||
2_000,
|
||||
2_010,
|
||||
std::ptr::null(),
|
||||
std::ptr::null_mut(),
|
||||
));
|
||||
assert!(!localcast_producer_push_move_evidence_buffer(
|
||||
move_producer,
|
||||
runtime,
|
||||
2_000,
|
||||
2_010,
|
||||
&zero_handle,
|
||||
std::ptr::null_mut(),
|
||||
));
|
||||
assert!(!localcast_producer_push_move_evidence_buffer(
|
||||
move_producer,
|
||||
runtime,
|
||||
2_000,
|
||||
2_010,
|
||||
&zero_count,
|
||||
std::ptr::null_mut(),
|
||||
));
|
||||
assert!(!localcast_producer_push_move_evidence_buffer(
|
||||
move_producer,
|
||||
runtime,
|
||||
2_000,
|
||||
2_010,
|
||||
&bad_stride,
|
||||
std::ptr::null_mut(),
|
||||
));
|
||||
assert_eq!(
|
||||
0,
|
||||
localcast_runtime_view_len(runtime, SampleKind::MoveEvidence.id())
|
||||
);
|
||||
|
||||
localcast_producer_destroy(render_producer);
|
||||
localcast_producer_destroy(move_producer);
|
||||
localcast_runtime_destroy(runtime);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_spine_rejects_nulls() {
|
||||
let sample = handle(1, 2, 3, 4, 5);
|
||||
|
|
|
|||
|
|
@ -17,11 +17,27 @@ Get-Content .\state\evidence.jsonl -Tail 8
|
|||
|
||||
## Current Shape
|
||||
|
||||
- Public brand/Persona is Mimir.
|
||||
- Mimir's Persona doctrine lives in [[docs/mimir-face|Mimir Persona]].
|
||||
- Public brand/Face is Mimir.
|
||||
- Mimir's Face doctrine lives in [[docs/mimir-face|Mimir Face]].
|
||||
- VoidBot persona/state live under `.voidbot/voice/` and `.voidbot/state/`.
|
||||
Commit `.voidbot` dirt in a separate state commit whenever it appears.
|
||||
- V1 still has FFmpeg/SRT/OBS bridge utilities for LAN ingest.
|
||||
- V1 still has FFmpeg/SRT/OBS bridge utilities for LAN compatibility ingest.
|
||||
- V1 now also has an explicit CultMesh media bridge for Raven program feeds:
|
||||
`src/Mimir.CultMeshMedia` sends rolling `mimir.cultmesh_media_frame`
|
||||
documents over CultNet reliable UDP through Yggdrasil and lowers them on
|
||||
Starfire to local MPEG-TS UDP for OBS. The bridge now uses explicit CultLib
|
||||
RUDP helpers/session plumbing; the old
|
||||
`CultMesh.StartNodeAsync`/`CultMesh.ConnectClient` path no longer owns this
|
||||
media lane.
|
||||
- CultLib has moved the daemon-swarm direction to CultNet over RUDP across
|
||||
runtimes. Mimir's typed daemon truth should default to
|
||||
`cultnet.transport.rudp.v0`; TCP/HTTP/WebSocket are client lowerings,
|
||||
compatibility probes, or migration debt unless a specific runtime has not
|
||||
earned RUDP yet.
|
||||
- Mimir's old Eve dashboard broker is archived and no longer publishes Idunn
|
||||
health, `/eve/deck`, `/eve/dashboard`, or WebSocket commands. The browser
|
||||
reference is a client lowering, not service truth. Dashboard state must return
|
||||
as typed CultMesh/Eve documents through Odin before deployment resumes.
|
||||
- The live stream app is C# plus Fensalir: `Mimir.slnx` contains
|
||||
`src/Mimir.App` and `src/Mimir.Runtime`.
|
||||
- `src/Mimir.App` hosts Fensalir as the windowing/rendering/D3D12 bridge.
|
||||
|
|
@ -37,205 +53,42 @@ Get-Content .\state\evidence.jsonl -Tail 8
|
|||
dictionary, low-level implementation notes, and sample code sketches for the
|
||||
next decoder/DSP/native-ring/Fensalir implementation passes. Start with
|
||||
[[research/perfect-machine-study-2026-05-23/reading-guide|Reading Guide]].
|
||||
- `docs/mimir-program-composition.md` is the current program-output authority
|
||||
map: Muninn observes on Starfire/Nightwing/Raven-class hosts, Mimir consumes
|
||||
selected streams and owns composition, Eve lowers controls/previews/stats,
|
||||
Yggdrasil publishes to the site, and OBS is compatibility-only.
|
||||
- `src/Mimir.Runtime/Synchronization` now has typed program contracts for
|
||||
`mimir.program_scene.v1`, `mimir.program_output.v1`, and
|
||||
`mimir.eve_operator_surface.v1`; `Mimir.BufferSmoke
|
||||
--perfect-machine-contract-smoke` writes them into CultCache.
|
||||
- `Mimir.BufferSmoke --import-obs-program-scene --output
|
||||
state/mimir-program-composition.cc` imports the current OBS scene JSON into
|
||||
Mimir-owned typed state. The latest imported scene has eight layers, six
|
||||
visible, two cropped, one chroma-keyed, and Raven mapped as
|
||||
`muninn:raven:monitor:primary`.
|
||||
- Local six-camera ingest should use direct driver adapters. Process-backed
|
||||
sources are bridge/network edges only.
|
||||
- Leap stereo IR is the first timing-camera candidate for direct ingest.
|
||||
- `native/reservoir` owns the lower shared-edge typed-handle invariant for
|
||||
Fensalir/Faust binding work.
|
||||
- Fensalir owns production GPU fusion, UI, and Spout2 publication.
|
||||
- `MimirSceneEditorState` is currently a 2D OBS-shaped program mixer owner, not
|
||||
a camera/3D scene editor. Synced video buffers become program layers with
|
||||
normalized placement; the Eve/CultUI surface is Video Sources on the left, a
|
||||
chrome-free program canvas in the center, Audio Mixer docked underneath that
|
||||
canvas, and a Unity-style Inspector on the right for the selected layer's
|
||||
properties. The center preview requests Fensalir's current output-buffer blit
|
||||
as its background and overlays Mimir's normalized layer placements on top.
|
||||
Placement commits snap near frame edges/center and preserve source aspect
|
||||
while resizing through direct preview handles, sliders, or the scale gizmo.
|
||||
Explicit source/transform controls are inspector-only and only visible when a
|
||||
program source is selected.
|
||||
Hover/active handle state, snap guides, empty-canvas deselect, keyboard
|
||||
nudging, PageUp/PageDown layer ordering, and selected-source
|
||||
contain/cover/stretch fit mode are wired through `MimirSceneEditorState`.
|
||||
Camera/SDF/model editor affordances have no live UI authority, and editor
|
||||
camera/SDF/spline output no longer contributes to the runtime frame. Smoke
|
||||
with `--scene-editor-smoke`.
|
||||
- The Audio Mixer dock is a horizontal CultUI channel-strip bank, not a
|
||||
selected-stem inspector form. Six fixed slots read the live
|
||||
`presentationControls.AudioFeeds` order so Scarlett ASIO mono inputs,
|
||||
stereo loopback, and Raven loopback can appear as parallel strips with M/S,
|
||||
gain, and compact sync/chirp-response readouts derived from
|
||||
`MimirSynchronizationHub.AudioSynchronizationStates`. Do not move sample
|
||||
alignment, chirp response mapping, or ambient/noise neutralization authority
|
||||
into the UI; the surface only controls and observes runtime audio state.
|
||||
- Program surface placement is seeded from CultCache config
|
||||
`config/mimir-program-surface.cc` unless `--cache`,
|
||||
`AQUARIUM_CULTCACHE_PATH`, or `MIMIR_PROGRAM_SURFACE_CONFIG` points
|
||||
elsewhere. The default `mimir.program.default` document puts
|
||||
`raven-display` full-frame at layer 0 and `kiyo-pro-rgb` as a top-left
|
||||
picture-in-picture at layer 1. `MimirSceneEditorState` applies matching
|
||||
source placements once when those feeds appear, then inspector/drag edits own
|
||||
subsequent placement.
|
||||
- The left Video Sources rail is a direct source selector, not a previous/next
|
||||
navigator. It renders visible rows for live video feeds using CultUI
|
||||
row-level visibility, with a Select action and placement summary per source.
|
||||
Keep configured/offline source inventory as a future source-state owner;
|
||||
the current rail lists live synced video feeds.
|
||||
- Live camera image buffers should use Fensalir-owned texture leases when they
|
||||
are rendering inputs: Mimir asks the engine broker for a keyed D3D12
|
||||
Texture2D/fence lease, writes decoded frames into that texture, commits the
|
||||
producer fence value, and lowers the same resource key through FieldEvidence.
|
||||
`MimirVideoCaptureDriverSource` forwards that lease client to direct drivers
|
||||
that implement `IMimirFensalirTextureLeaseReceiver`.
|
||||
Camera backends must use the closest-to-device path available and report
|
||||
unavoidable copies; process or managed convenience layers are diagnostics.
|
||||
Raw single-plane CPU-origin frames can use the explicit broker upload lane,
|
||||
which increments `UnavoidableCopyCount` and clears live managed payload bytes
|
||||
after upload. NV12 CPU upload is rejected until Fensalir owns planar copy;
|
||||
device/GPU NV12 producers should write the leased texture directly.
|
||||
The first direct KS/UVC driver is `MimirKsVideoCaptureDriver` backed by
|
||||
`native/camera_capture/mimir_camera_capture.dll`; the example config
|
||||
`config/mimir-runtime.ks-camera.example.json` targets LeapUVC 640x240 YUY2.
|
||||
The same native folder also has `mimir_ps3eye_capture.dll` and
|
||||
`MimirPs3EyeVideoCaptureDriver` for PS3 Eye Bayer8 frames through WinUSB/libusb.
|
||||
Fresh driver smokes pulled and uploaded real frames for LeapUVC 640x240 YUY2,
|
||||
both PS3 Eyes at 320x240 Bayer8, Kiyo Pro 1920x1080 YUY2, and regular Kiyo
|
||||
640x480 YUY2. Regular Kiyo 1280x720 YUY2 did not open.
|
||||
Compressed Kiyo Pro MJPG/H264 now have a first GPU-resident path:
|
||||
`MimirMediaFoundationGpuVideoCaptureDriver` uses
|
||||
`native/camera_capture/mimir_mf_gpu_capture.dll` to decode through Media
|
||||
Foundation with a D3D11 device manager, copy into a shared BGRA D3D11 texture,
|
||||
and publish `shared-d3d11-texture` with zero live CPU payload bytes.
|
||||
1920x1080 MJPG->RGB32 and H264->RGB32 smokes passed. Direct shared NV12
|
||||
texture creation failed, so NV12 remains future plane-aware interop or
|
||||
GPU bridge-copy work.
|
||||
Foreign shared texture handles are import edges, not the primary hot path.
|
||||
- The current Mimir/Fensalir rendering teardown map is
|
||||
[[docs/fensalir-rendering-rebuild-migration|Fensalir Rendering Rebuild
|
||||
Migration]]. Mimir's production visual output path must publish typed
|
||||
physical observations, calibration constraints, and surface intent; Fensalir
|
||||
turns those into field claims, selected lowerings, reusable evidence,
|
||||
temporal guide lanes, and presentation.
|
||||
- Fast depth provenance now lives at
|
||||
[[research/d3d12-stereo-depth-provenance-2026-05-29|D3D12 Stereo Depth
|
||||
Provenance]]. `libSGM` is the current permissive north-star reference for a
|
||||
future Fensalir-owned D3D12 stereo SGM compute lane. Do not import CUDA or
|
||||
TensorRT as live depth authority.
|
||||
The first code contract is in `MimirStereoDepthConfigurations` and
|
||||
`MimirFensalirFieldLowering`. Run
|
||||
`dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --stereo-depth-contract-smoke`
|
||||
to verify caller-declared rectified left/right input textures, or
|
||||
`dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --leap-packed-stereo-depth-smoke`
|
||||
to verify live `LeapStereoIr` packed R8G8 lowering. The Leap path reuses the
|
||||
same declared texture as left/right resource identity and treats R/G as the
|
||||
packed observation lanes. Fensalir now has a first packed-Leap D3D12
|
||||
SAD/block-match disparity writer, not full libSGM-style SGM or calibrated
|
||||
metric depth. Mimir also now declares the first Leap point-cloud root:
|
||||
`MimirPointCloudConfigurations.LeapDisparityPointCloudRoot` and
|
||||
`BuildLeapPackedStereoPointCloudCandidateFrame` emit a GPU-resident
|
||||
`FieldMesh` point-list resource derived from that disparity SurfacePage. Run
|
||||
`dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --leap-point-cloud-root-smoke`
|
||||
to verify the combined `SurfacePage` plus `Mesh` packet plan. This is a
|
||||
typed socket on the Mimir side. Fensalir branch `codex/leap-packed-depth` now
|
||||
has the first matching render lane: generated point-list Mesh buffers are
|
||||
filled from the disparity SurfacePage on D3D12 and rendered through the scene
|
||||
candidate targets. The next cut is live Leap/editor verification plus
|
||||
calibrated projection constants and global residual/calibration ownership.
|
||||
- Fensalir also owns the EVE-facing dashboard pixels through
|
||||
`Global\MimirFensalirProgramTexture`. `src/Mimir.EveRelay` opens that shared
|
||||
texture, encodes H.264 Annex-B with NVENC by default, and serves EveCanvas
|
||||
over `/stream`; EveCanvas decodes with `AVSampleBufferDisplayLayer`. The
|
||||
launch script uses an SSH reverse tunnel to Eve when Windows firewall blocks
|
||||
direct inbound TCP.
|
||||
- `src/Mimir.Broadcast` is the first self-hosted public sender. Starfire pushes
|
||||
one final H.264/AAC NVENC RTMP stream to
|
||||
`rtmp://127.0.0.1:11935/live/mimir` through an SSH local forward to
|
||||
Yggdrasil. Yggdrasil's nginx RTMP/HLS origin is deployed and has produced
|
||||
HLS segments from a bounded synthetic push. Public HLS is served through the
|
||||
existing StreamPixels TLS host, not a new Mimir media hostname. The public
|
||||
static viewer belongs to the root GameCult site at
|
||||
`https://gamecult.org/livestream` and reads
|
||||
`https://streampixels.gamecult.org/mimir/live/hls/mimir.m3u8`.
|
||||
- Raven/Eve ingest now has a network-pixel receiver config:
|
||||
`config/mimir-runtime.raven-eve.example.json` starts an FFmpeg-backed
|
||||
raw-video listener for `raven-display` on SRT port `5200` and Eve sensor
|
||||
frame-event receivers for `eve-camera` on WebSocket port `8793` and `eve-mic`
|
||||
on port `8794`. `scripts/start-raven-screen-capture-sender.ps1` launches
|
||||
Raven desktop capture with `gdigrab` plus NVENC. EveCanvas owns Eve camera/mic
|
||||
capture directly through AVFoundation/AVAudioEngine; no external iOS camera
|
||||
app owns that path.
|
||||
- `src/Mimir.RavenDaemon` is the Raven-local capture mux owner for the
|
||||
stream-proof path. It supervises display capture plus Realtek render loopback
|
||||
through the existing FFmpeg/WASAPI pipeline, publishes
|
||||
`mimir.raven_capture_mux_state.v1` through CultMesh/CultCache, and exposes
|
||||
Eve dashboard state on port `8801`. `scripts/start-raven-daemon.ps1` defaults
|
||||
to Starfire over WireGuard at `10.77.0.2:5200`. The older Raven sender scripts
|
||||
remain diagnostics, not canonical long-running status owners.
|
||||
- Faust/native DSP owns hot audio alignment, separation, spatialization, and
|
||||
synchronized stems.
|
||||
- The current listenable dual-mic receipt is the Faust-chain bridge, not the
|
||||
older C# composite. `scripts/record-kiyo-dual-mic-composite.ps1` records Kiyo
|
||||
plus Scarlett ASIO, then `--asio-dual-mic-faust-cleaner-wav` compiles/runs
|
||||
`faust/mimir_alignment_actuator.dsp` before
|
||||
`faust/mimir_dual_mic_dialogue_cleaner.dsp` and muxes the dialogue stem into
|
||||
`artifacts/runtime/kiyo-dual-mic-composite/kiyo-dual-mic-composite-latest.mp4`.
|
||||
Fresh run `20260601-051044` measured cardioid delay 359.398 samples
|
||||
(7487.452 us) at 48 kHz and applied holdback 359.398/0.000 to
|
||||
shotgun/cardioid before cleanup.
|
||||
- The edge-to-edge Mimir editor shell now uses headerless retained dashboard
|
||||
panels. The inspector expands vertically; the right rail `Output` panel is a
|
||||
fixed-height OBS-like control block below it. Eve is the portable dashboard
|
||||
authority; the C# CultUI-shaped Aquarium contract is only the reference API
|
||||
shape, and Fensalir Direct2D/D3D12 are native lowerings of the same document.
|
||||
Elements can now carry typed CultCache/CultMesh bindings with document
|
||||
schema/id, field path, value kind, access, authority, and optional command id;
|
||||
native callbacks remain local lowering actuators, not portable state owners.
|
||||
`MimirProgramRecorder` owns `Record` / `Stop`, status text, and timestamped
|
||||
receipts under `C:\Users\Meta\Videos\Mimir` by default
|
||||
(`MIMIR_RECORD_OUTPUT_DIR` overrides). It delegates to
|
||||
`scripts/record-stream-proof-output.ps1`, so the app records the same Raven +
|
||||
Kiyo PIP + Starfire Scarlett NVENC/AAC composite intended for
|
||||
Yggdrasil/public judgment. `MIMIR_RECORD_START_RAVEN=1` asks it to launch the
|
||||
Raven sender too. `MimirProgramStreamer` owns `Live` / `Stop` for
|
||||
`Mimir.Broadcast`; it requires `MIMIR_STREAM_INPUT` unless
|
||||
`MIMIR_STREAM_ALLOW_TEST_PATTERN=1` explicitly allows a synthetic smoke.
|
||||
Native program-output recording/streaming should replace those actuators
|
||||
behind the same owner.
|
||||
- OBS receives final program surfaces; it does not own synchronization.
|
||||
- The online Well stack exists. `Mimir.EveSensorReceiver` has a subscriber
|
||||
WebSocket path beside its observation ingest path, and `Mimir.VerseRecorder`
|
||||
writes subscribed observations to
|
||||
`C:\Users\Meta\Videos\Mimir\VerseCaptures\<session>\session.json` plus
|
||||
`observations.jsonl`. Recorder now also recognizes
|
||||
`mimir.well_capture_page.v1`, strips inline base64 sample bodies into
|
||||
`bodies/page-*.bin`, and records `mimir.recorder_body_index.v1` refs in
|
||||
`bodies/index.jsonl` so offline reconstruction tools can resolve payloads
|
||||
without treating JSONL as the body store. `Mimir.Well` loads
|
||||
`config/mimir-runtime.well.local.json`, starts every configured source it
|
||||
can, polls `MimirSynchronizationHub` with a bounded per-source budget, updates
|
||||
audio sync/probe state, and publishes `mimir.well_snapshot.v1` records with
|
||||
source buffers, composite controls, publication/stem contract, and sync
|
||||
status. It also publishes bounded `mimir.well_capture_page.v1` records for
|
||||
synchronized frame slices with timing metadata, configured composite state,
|
||||
and inline source bodies when CPU bytes exist and fit the configured body
|
||||
limit. GPU-handle-only samples remain metadata/body-handle evidence until
|
||||
Fensalir/native program-output recording owns rendered composite bodies.
|
||||
`scripts/start-online-verse-daemons.ps1` supervises the relay,
|
||||
recorder, Nightwing typed Eyes/Move/builtin-camera/builtin-mic witness, the
|
||||
music-synced de-Bruijn/microtonal Move gesture worker, and `Mimir.Well`.
|
||||
Current live proof `online-verse-daemons-20260601-212502` wrote Well,
|
||||
Nightwing builtin camera/mic, and Eye/Move records. Raven and Eve lanes are
|
||||
configured but empty unless their producers are feeding the ports. Sync frame
|
||||
status is degraded until a camera-clock/global-time correction owner maps
|
||||
ASIO device time and camera driver time into one canonical timeline.
|
||||
- OBS compatibility adapters may receive final program surfaces; they do not
|
||||
own synchronization, composition, preview/control, or publication.
|
||||
|
||||
## Current Pressure
|
||||
|
||||
- Verify the live Leap point cloud in the Mimir editor, then add calibrated
|
||||
projection constants and the global residual/calibration owner.
|
||||
- Cut the Phase 1 Mimir-to-Fensalir bridge DTOs for rolling windows,
|
||||
observations, calibration constraints, and surface intent before adding
|
||||
another backend-specific visual path.
|
||||
- Nightwing Move tracking is one Muninn provider with two private per-eye
|
||||
PSMoveAPI worker subprocesses. Eye 0 uses exposure 0.3; Eye 1 uses 0.1.
|
||||
Both workers calibrate four Moves and advance continuously. Mimir has now
|
||||
proved all four stable IDs on both eyes in the same frames. The latest
|
||||
15-second receipt contains 16 observations and eight pair records but only
|
||||
two unique aggregate frame IDs. Next measure parent publication and RUDP send
|
||||
counters directly, fix cadence, then collect diverse correspondences for
|
||||
camera calibration.
|
||||
|
||||
- Implement the first concrete Leap direct capture driver and measure cadence.
|
||||
- Add remaining camera drivers through `IMimirVideoCaptureDriver`.
|
||||
- Add native audio capture workers for mic, loopback, and network audio feeds.
|
||||
- Turn the current loopback-referenced chirplet measurement into the real
|
||||
|
|
@ -243,8 +96,24 @@ Get-Content .\state\evidence.jsonl -Tail 8
|
|||
variable-rate resampling.
|
||||
- Bind Fensalir UI to stream health, buffer depth, timestamps, settings, and
|
||||
output management.
|
||||
- Keep PowerShell/FFmpeg/SRT as bridge utilities until native program output
|
||||
replaces them.
|
||||
- Keep PowerShell/FFmpeg/SRT as bridge utilities until native Mimir program
|
||||
output, Eve operation, and Yggdrasil site publication replace them.
|
||||
- Yggdrasil currently runs the CultMesh media relay from
|
||||
`/opt/gamecult/mimir-cultmesh-media/Mimir.CultMeshMedia`; senders and
|
||||
receivers target `cultmesh://asgard.yggdrasil.mimir/media/raven-primary-av`
|
||||
and let the CultMesh resolver choose the concrete RUDP route. Starfire writes
|
||||
`raven-primary-av` to `udp://127.0.0.1:5200`. That is the
|
||||
`Mimir.CultMeshMedia` body bridge, not the OBS plugin feed owner.
|
||||
- The actual Raven OBS/SRT feed owner is Odin's Muninn on Raven. Use
|
||||
`scripts/start-raven-muninn-obs-feed.ps1` in this repo as the thin wrapper
|
||||
over `E:\Projects\Odin\scripts\activate-muninn-raven-av-srt.ps1`, which
|
||||
drives Raven's real `GameCult-Muninn-Activate` hidden task and
|
||||
`muninn.exe activate` path. `scripts/start-raven-cultmesh-av-sender.ps1`
|
||||
remains the separate CultMesh bridge/bootstrap lane.
|
||||
- Rebuild Eve dashboard only as a CultMesh/Odin surface publisher plus renderer
|
||||
lowering. Provider advertisement, retained state, command boundary, transport
|
||||
profile, and command documents must be typed CultMesh records; no TCP, HTTP,
|
||||
or WebSocket surface should become daemon or transport truth.
|
||||
- Kiyo Pro has two UVC extension units. Moving it to a motherboard USB3 port
|
||||
changed the descriptor path to `root_hub30` / `bcdUSB=0x0320` and exposed
|
||||
720p/1080p YUY2/MJPG/H264/NV12 at 60 fps, but Windows still reports
|
||||
|
|
@ -329,6 +198,41 @@ Get-Content .\state\evidence.jsonl -Tail 8
|
|||
|
||||
## Immediate Re-entry Instruction
|
||||
|
||||
The 2026-07-14 live two-wand session at
|
||||
`artifacts/move-calibration/live-two-wand-20260714-004210.*` passed collection
|
||||
coverage with 1,168 admitted same-frame pairs. The new Mimir-owned sphere
|
||||
stereo fitter recovers synthetic 505/535 px focal lengths within 0.43 px and a
|
||||
34 cm baseline within 0.22 mm, with 0.34/0.63 px held-out median/P95 error.
|
||||
The live artifact remains rejected: after robust radius admission its consensus
|
||||
fits at 0.51/2.08 px, but only 53.8 percent of held-out pairs agree and Eye 1's
|
||||
focal estimate leaves the physical range. PSMoveAPI age is currently projected
|
||||
as marker confidence by Muninn, so fresh wrong-color matches arrive near 0.996
|
||||
confidence. Repair source-local observation quality/continuity and calibrate
|
||||
per-Eye blob-radius response before asking the operator for another sweep. Do
|
||||
not promote the nominal 22.5 mm diagnostic fit.
|
||||
|
||||
Odin `d0eded6` is deployed on Nightwing as binary SHA-256
|
||||
`a604fedaf90b26a49db6594b16b8b3499e3aed540152c6fda9dd9dbe87d37d39`.
|
||||
Each Eye worker now rejects stale, sub-2 px, out-of-bounds, and discontinuous
|
||||
PSMoveAPI positions before aggregation; tracker health publishes rejection
|
||||
counters, and candidate score is explicitly uncalibrated `0.5` instead of age
|
||||
masquerading as confidence. A post-deploy 10-second Mimir window received 555
|
||||
frames, 2,246 observations, and 443 stereo pairs. Radius P01 rose from about
|
||||
0.707 px to 3.13/3.64 px and confidence P50 is now 0.5 on both Eyes. Eye 1 saw
|
||||
only two plausible stationary IDs, which is honest and means another sweep is
|
||||
needed after Mimir models per-Eye blob-radius response.
|
||||
|
||||
The active priority is Nightwing Move calibration. One public Muninn observer
|
||||
owns two private Eye workers and one aggregate evidence stream. Odin commits
|
||||
`39c839a`, `a340c33`, `8aefd1c`, and `f82ca7a` give daemon health a persistent
|
||||
serviced RUDP session and move optical aggregation onto its own deadline-clocked
|
||||
loop. The 2026-07-13 receipt at
|
||||
`artifacts/move-calibration/muninn-clocked-aggregator.mpack` contains 622 frames,
|
||||
2,647 observations, and 1,384 cross-camera correspondences in 15 seconds, with
|
||||
all four Move IDs visible to both Eyes. Next collect spatially diverse samples,
|
||||
calibrate intrinsics/extrinsics, and publish residual-bearing typed calibration.
|
||||
Radius is a weighted range cue, not a substitute for camera calibration.
|
||||
|
||||
The corrected two-hour study pass ran from
|
||||
`2026-05-23T23:54:42.9041588+01:00` to at least
|
||||
`2026-05-24T01:54:56.9747630+01:00`. Use its map first, then implement the
|
||||
|
|
|
|||
|
|
@ -11,18 +11,11 @@ Use it to shape native implementation decisions:
|
|||
|
||||
- direct camera and audio drivers;
|
||||
- pinned/native buffers;
|
||||
- GPU feature extraction, flow, matching, depth, and fusion;
|
||||
- GPU feature extraction, flow, matching, and fusion;
|
||||
- Fensalir D3D12 compute/render paths;
|
||||
- Faust/native DSP;
|
||||
- OBS-facing program outputs.
|
||||
|
||||
Visual/GPU research entry points:
|
||||
|
||||
- [[d3d12-stereo-depth-provenance-2026-05-29|D3D12 Stereo Depth Provenance]]:
|
||||
libSGM is the current permissive north-star reference for a future
|
||||
Fensalir-owned D3D12 compute stereo-disparity lane. External CUDA/TensorRT
|
||||
projects are provenance only, not runtime authority.
|
||||
|
||||
Audio timing research entry points:
|
||||
|
||||
- [[chirplet-sync-decoder/summary|Chirplet Sync Decoder Summary]]: chirplet
|
||||
|
|
|
|||
|
|
@ -1,170 +0,0 @@
|
|||
# D3D12 Stereo Depth Provenance
|
||||
|
||||
Date: 2026-05-29
|
||||
|
||||
## Purpose
|
||||
|
||||
This note records the current fast GPU depth-estimation search as provenance for
|
||||
a future Mimir/Fensalir D3D12 compute implementation.
|
||||
|
||||
The north star is `libSGM`: not as a dependency, not as a CUDA bridge, and not
|
||||
as a permission slip to add an adapter. The useful shape is a fast,
|
||||
deterministic stereo disparity pipeline whose ownership can be rebuilt inside
|
||||
Fensalir's D3D12 resource model.
|
||||
|
||||
## Authority Map
|
||||
|
||||
Owner:
|
||||
|
||||
- Fensalir owns the live D3D12 compute kernels, GPU resources, fences,
|
||||
scheduling, and depth/surface claim lowering.
|
||||
|
||||
Inputs:
|
||||
|
||||
- Rectified synchronized camera texture pairs from Mimir-declared
|
||||
`Texture2D` resources.
|
||||
- Camera calibration: intrinsics, extrinsics, baseline, rectification map, and
|
||||
valid-time/version metadata.
|
||||
- Rolling-window timing chosen by `MimirSynchronizedBufferPlanner`.
|
||||
|
||||
Outputs:
|
||||
|
||||
- GPU-resident disparity/depth resources, preferably fixed format and size per
|
||||
calibrated camera pair.
|
||||
- FieldEvidence claims for depth support, confidence, invalid/occluded pixels,
|
||||
and later surface/material lowerings.
|
||||
- Diagnostics: kernel timings, disparity range, confidence histogram, invalid
|
||||
ratio, calibration version, and source texture fence values.
|
||||
|
||||
Derived State:
|
||||
|
||||
- Depth is derived from synchronized stereo textures plus calibration.
|
||||
- Per-frame disparity confidence is derived evidence, not a clock or camera
|
||||
authority.
|
||||
- Monocular model depth remains optional reference/assistive evidence unless a
|
||||
later invariant gives it a named role.
|
||||
|
||||
Forbidden Writers:
|
||||
|
||||
- CUDA runtime code must not become the live path.
|
||||
- TensorRT/ONNX monocular demos must not own visual geometry.
|
||||
- CPU image copies/readback must not be inserted for convenience.
|
||||
- OBS, EVE, or bridge utilities must not decide depth or synchronization.
|
||||
|
||||
Shared Paths:
|
||||
|
||||
- Direct user capture, replayed rolling-window frames, future Raven/phone
|
||||
camera feeds, and calibration-session frames must all enter through typed
|
||||
resources and use the same depth compute/claim path once they are calibrated
|
||||
stereo pairs.
|
||||
|
||||
Deletion Line:
|
||||
|
||||
- If a prototype requires persistent CPU depth images, a side transport daemon,
|
||||
or a CUDA compatibility layer, it is research-only and does not graduate into
|
||||
the live machine.
|
||||
|
||||
## Reference Shape: libSGM
|
||||
|
||||
Repository: https://github.com/fixstars/libSGM
|
||||
|
||||
License: Apache-2.0.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- It is a CUDA implementation of Semi-Global Matching for stereo disparity.
|
||||
- It was built by the Fixstars/adaskit ADAS effort, so it is pointed at the
|
||||
same autonomous-driving pressure field rather than generic image demos.
|
||||
- Reported benchmark in the upstream README: `1024 x 440`, disparity `128`,
|
||||
four SGM paths, subpixel enabled:
|
||||
- GTX 1080 Ti: `2.0 ms` / `495.1 FPS`
|
||||
- RTX 3080: `1.5 ms` / `651.3 FPS`
|
||||
- Tegra X2: `28.5 ms` / `35.1 FPS`
|
||||
- Xavier MAXN: `9.0 ms` / `110.7 FPS`
|
||||
|
||||
Port lesson:
|
||||
|
||||
- Use the pipeline and data movement as a model: matching cost, path
|
||||
aggregation, winner/subpixel selection, confidence/invalid handling.
|
||||
- Rebuild the kernels in HLSL/D3D12 compute against Fensalir-owned resources.
|
||||
- Preserve the ADAS-style assumption that calibrated stereo gives metric depth
|
||||
when baseline and rectification are real.
|
||||
|
||||
## Secondary References
|
||||
|
||||
Fast-ACVNet: https://github.com/gangweiX/Fast-ACVNet
|
||||
|
||||
- License: MIT.
|
||||
- Learned real-time stereo matching; useful for quality and failure-mode
|
||||
comparison.
|
||||
- Upstream reports `45 ms` for Fast-ACVNet+ in its KITTI/Scene Flow table.
|
||||
- Not the first Mimir implementation target because it brings model training,
|
||||
weights, and framework deployment complexity.
|
||||
|
||||
UniMatch / GMStereo: https://github.com/autonomousvision/unimatch
|
||||
|
||||
- License: MIT.
|
||||
- Unified flow/stereo/depth research code.
|
||||
- Useful as a modern learned-stereo comparison point and possible later
|
||||
confidence/feature guide, not a kernel blueprint.
|
||||
|
||||
Depth Anything V2 Small: https://github.com/DepthAnything/Depth-Anything-V2
|
||||
|
||||
- License caveat: V2 Small is Apache-2.0; Base/Large/Giant are CC-BY-NC-4.0.
|
||||
- Excellent monocular relative-depth reference, especially through TensorRT
|
||||
ports, but monocular output is not metric scene truth by itself.
|
||||
|
||||
Depth Anything TensorRT CLI:
|
||||
https://github.com/spacewalk01/depth-anything-tensorrt
|
||||
|
||||
- License: MIT.
|
||||
- Reports fast RTX 4090 inference for Depth-Anything-S/B/L.
|
||||
- Useful as performance context only. It must not displace calibrated stereo
|
||||
geometry or Fensalir's D3D12 ownership.
|
||||
|
||||
Fast-FoundationStereo:
|
||||
https://github.com/NVlabs/Fast-FoundationStereo
|
||||
|
||||
- Important research reference for real-time zero-shot stereo.
|
||||
- License is research/non-commercial only, so it is not a permissive live
|
||||
dependency candidate.
|
||||
|
||||
## D3D12 Compute Implementation Sketch
|
||||
|
||||
First implementation should stay boring and inspectable:
|
||||
|
||||
1. Rectify or require pre-rectified stereo pairs.
|
||||
2. Convert sampled luma/feature texture into a compact matching domain.
|
||||
3. Build per-pixel disparity matching costs over a fixed disparity range.
|
||||
4. Aggregate along a small fixed path set, starting with four directions to
|
||||
match the `libSGM` reference benchmark shape.
|
||||
5. Select winner disparity and subpixel refinement.
|
||||
6. Emit disparity, confidence, and invalid/occlusion masks as GPU resources.
|
||||
7. Lower those resources into FieldEvidence depth/support claims.
|
||||
|
||||
Important D3D12 resource notes:
|
||||
|
||||
- Prefer persistent UAV/SRV resources sized by camera pair profile.
|
||||
- Use explicit producer/consumer fences already established for camera texture
|
||||
leases and program-output publication.
|
||||
- Keep disparity range and path count as profile-level cost levers, not
|
||||
per-frame mode flags scattered through the renderer.
|
||||
- Do not read back depth except for gated diagnostics and receipts.
|
||||
|
||||
## Initial Verification Targets
|
||||
|
||||
- Synthetic rectified stereo pair with known constant disparity.
|
||||
- Slanted plane with known disparity gradient.
|
||||
- Occlusion edge where invalid/confidence behavior is visible.
|
||||
- Real Leap stereo IR pair once calibration is pinned.
|
||||
- Full rolling-window path: camera textures -> depth compute -> FieldEvidence
|
||||
claim -> Fensalir debug overlay/resource receipt.
|
||||
|
||||
Negative checks:
|
||||
|
||||
- No CUDA runtime loaded.
|
||||
- No CPU disparity image in the live path.
|
||||
- No monocular model output accepted as metric depth without a named scale
|
||||
authority.
|
||||
- No untyped texture handle accepted without resource declaration and fence
|
||||
provenance.
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
@echo off
|
||||
setlocal
|
||||
|
||||
set "ROOT=%~dp0"
|
||||
set "APP=%ROOT%src\Mimir.App\bin\Debug\net10.0\Mimir.App.exe"
|
||||
|
||||
if not exist "%APP%" (
|
||||
echo Mimir.App.exe was not found.
|
||||
echo Build it first with:
|
||||
echo dotnet build "%ROOT%src\Mimir.App\Mimir.App.csproj"
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
set "MIMIR_SYNTHETIC_SPECTRUM_PREVIEW=1"
|
||||
set "MIMIR_SPECTRUM_INTERVAL_SECONDS=0.016"
|
||||
set "MIMIR_SPECTRUM_SOURCE_LANES=8"
|
||||
|
||||
start "Mimir Spectrum Preview" "%APP%"
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
param(
|
||||
[string]$Configuration = "Release",
|
||||
[string]$TemplateVersion = "master",
|
||||
[switch]$SkipTests
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$root = Split-Path -Parent $PSScriptRoot
|
||||
$sdkRoot = Join-Path $root "artifacts\obs-sdk"
|
||||
$templateRoot = Join-Path $sdkRoot "obs-plugintemplate"
|
||||
$depsRoot = Join-Path $templateRoot ".deps"
|
||||
$buildRoot = Join-Path $templateRoot "build_x64_local"
|
||||
$pluginBuildRoot = Join-Path $root "native\obs_stem_source\build"
|
||||
|
||||
New-Item -ItemType Directory -Force $sdkRoot | Out-Null
|
||||
if (-not (Test-Path $templateRoot)) {
|
||||
git clone --depth 1 --branch $TemplateVersion https://github.com/obsproject/obs-plugintemplate.git $templateRoot
|
||||
}
|
||||
|
||||
if (-not (Test-Path (Join-Path $depsRoot "cmake\libobsConfig.cmake"))) {
|
||||
cmake -S $templateRoot -B $buildRoot -G "Visual Studio 17 2022" -A x64 -DENABLE_FRONTEND_API=OFF -DENABLE_QT=OFF
|
||||
}
|
||||
|
||||
cmake `
|
||||
-S (Join-Path $root "native\obs_stem_source") `
|
||||
-B $pluginBuildRoot `
|
||||
-Dlibobs_DIR="$(Join-Path $depsRoot 'cmake')" `
|
||||
-DCMAKE_PREFIX_PATH="$depsRoot"
|
||||
|
||||
cmake --build $pluginBuildRoot --config $Configuration --parallel
|
||||
|
||||
if (-not $SkipTests) {
|
||||
ctest --test-dir $pluginBuildRoot -C $Configuration --output-on-failure
|
||||
}
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
param(
|
||||
[string]$ObsRoot = "",
|
||||
[string]$PluginRoot = "",
|
||||
[string]$Configuration = "Release",
|
||||
[switch]$SkipBuild,
|
||||
[switch]$WhatIf
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = Split-Path -Parent $PSScriptRoot
|
||||
|
||||
$pluginDll = Join-Path $repoRoot "native\obs_stem_source\build\$Configuration\mimir_obs_stem_source.dll"
|
||||
$localeSourceDir = Join-Path $repoRoot "native\obs_stem_source\data\locale"
|
||||
|
||||
if (-not [string]::IsNullOrWhiteSpace($ObsRoot)) {
|
||||
$targetDir = Join-Path $ObsRoot "obs-plugins\64bit"
|
||||
$targetLocaleDir = Join-Path $ObsRoot "data\obs-plugins\mimir_obs_stem_source\locale"
|
||||
} else {
|
||||
if ([string]::IsNullOrWhiteSpace($PluginRoot)) {
|
||||
if ($env:OBS_PLUGIN_ROOT) {
|
||||
$PluginRoot = $env:OBS_PLUGIN_ROOT
|
||||
} else {
|
||||
$PluginRoot = Join-Path $env:ALLUSERSPROFILE "obs-studio\plugins"
|
||||
}
|
||||
}
|
||||
|
||||
$targetDir = Join-Path $PluginRoot "mimir_obs_stem_source\bin\64bit"
|
||||
$targetLocaleDir = Join-Path $PluginRoot "mimir_obs_stem_source\data\locale"
|
||||
}
|
||||
|
||||
$targetDll = Join-Path $targetDir "mimir_obs_stem_source.dll"
|
||||
|
||||
if (-not $SkipBuild) {
|
||||
& (Join-Path $repoRoot "scripts\build-obs-stem-plugin.ps1") -Configuration $Configuration
|
||||
}
|
||||
|
||||
if (-not (Test-Path $pluginDll)) {
|
||||
throw "Built plugin DLL was not found at '$pluginDll'."
|
||||
}
|
||||
if (-not (Test-Path $localeSourceDir)) {
|
||||
throw "Plugin locale directory was not found at '$localeSourceDir'."
|
||||
}
|
||||
|
||||
if (-not (Test-Path $targetDir)) {
|
||||
if ($WhatIf) {
|
||||
Write-Host "Would create $targetDir"
|
||||
} else {
|
||||
New-Item -ItemType Directory -Force $targetDir | Out-Null
|
||||
}
|
||||
}
|
||||
if (-not (Test-Path $targetLocaleDir)) {
|
||||
if ($WhatIf) {
|
||||
Write-Host "Would create $targetLocaleDir"
|
||||
} else {
|
||||
New-Item -ItemType Directory -Force $targetLocaleDir | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
if ($WhatIf) {
|
||||
Write-Host "Would copy $pluginDll -> $targetDll"
|
||||
Get-ChildItem -LiteralPath $localeSourceDir -File | ForEach-Object {
|
||||
Write-Host "Would copy $($_.FullName) -> $(Join-Path $targetLocaleDir $_.Name)"
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
Copy-Item -LiteralPath $pluginDll -Destination $targetDll -Force
|
||||
Copy-Item -Path (Join-Path $localeSourceDir "*") -Destination $targetLocaleDir -Force
|
||||
Write-Host "Installed Mimir OBS plugin: $targetDll"
|
||||
Write-Host "OBS source types: Mimir Program Texture, Mimir Audio Stem, Muninn Stream, Muninn Video, Muninn Audio"
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
param(
|
||||
[string]$TargetHost = "192.168.1.66",
|
||||
[int]$Port = 5202,
|
||||
[int]$SampleRate = 48000,
|
||||
[int]$Channels = 2,
|
||||
[string]$FfmpegPath = "ffmpeg",
|
||||
[string]$LogRoot = "C:\Meta\Mimir\logs",
|
||||
[switch]$DryRun
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $LogRoot | Out-Null
|
||||
$repo = Split-Path -Parent $PSScriptRoot
|
||||
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
$stdoutLog = Join-Path $LogRoot "raven-realtk-loopback-$timestamp.out.log"
|
||||
$stderrLog = Join-Path $LogRoot "raven-realtk-loopback-$timestamp.err.log"
|
||||
$endpoint = "srt://${TargetHost}:${Port}?mode=caller&latency=120000"
|
||||
$captureScript = Join-Path $repo "scripts\wasapi-loopback-capture.ps1"
|
||||
|
||||
$command = "& `"$captureScript`" -Output - -SampleRate $SampleRate -Channels $Channels | & `"$FfmpegPath`" -hide_banner -nostdin -loglevel warning -f f32le -ar $SampleRate -ac $Channels -i pipe:0 -f f32le `"$endpoint`""
|
||||
|
||||
Write-Host "Raven Realtek loopback sender:"
|
||||
Write-Host " target: $endpoint"
|
||||
Write-Host " format: f32le ${Channels}ch ${SampleRate}Hz"
|
||||
Write-Host " stdout: $stdoutLog"
|
||||
Write-Host " stderr: $stderrLog"
|
||||
|
||||
if ($DryRun) {
|
||||
Write-Host $command
|
||||
exit 0
|
||||
}
|
||||
|
||||
Start-Process `
|
||||
-FilePath "powershell" `
|
||||
-ArgumentList @("-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", $command) `
|
||||
-WindowStyle Hidden `
|
||||
-RedirectStandardOutput $stdoutLog `
|
||||
-RedirectStandardError $stderrLog `
|
||||
-PassThru |
|
||||
ForEach-Object { Write-Host "Started Raven Realtek loopback sender pid=$($_.Id)" }
|
||||
|
|
@ -1,158 +0,0 @@
|
|||
param(
|
||||
[int]$DurationSeconds = 15,
|
||||
[string]$OutputPath = "artifacts\runtime\kiyo-dual-mic-composite\kiyo-dual-mic-composite.mp4",
|
||||
[string]$LogRoot = "artifacts\runtime\kiyo-dual-mic-composite",
|
||||
[string]$FfmpegPath = "ffmpeg",
|
||||
[string]$AsioProbePath = "native\probes\asio_audio_cadence\build\Release\asio_audio_cadence.exe",
|
||||
[string]$KiyoVideoDevice = "USB Video Device",
|
||||
[int]$KiyoWidth = 640,
|
||||
[int]$KiyoHeight = 480,
|
||||
[int]$KiyoFrameRate = 30,
|
||||
[int]$SampleRate = 48000,
|
||||
[int]$AsioChannels = 4,
|
||||
[int]$ShotgunChannel = 0,
|
||||
[int]$CardioidChannel = 1,
|
||||
[switch]$DryRun
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Quote-ProcessArgument([string]$Value) {
|
||||
if ($Value.Length -eq 0) {
|
||||
return '""'
|
||||
}
|
||||
|
||||
if ($Value -notmatch '[\s"]') {
|
||||
return $Value
|
||||
}
|
||||
|
||||
return '"' + $Value.Replace('"', '\"') + '"'
|
||||
}
|
||||
|
||||
function Start-LoggedProcess([string]$FileName, [string[]]$Arguments, [string]$StdoutPath, [string]$StderrPath) {
|
||||
$startInfo = [System.Diagnostics.ProcessStartInfo]::new()
|
||||
$startInfo.FileName = $FileName
|
||||
$startInfo.UseShellExecute = $false
|
||||
$startInfo.RedirectStandardOutput = $true
|
||||
$startInfo.RedirectStandardError = $true
|
||||
$startInfo.Arguments = ($Arguments | ForEach-Object { Quote-ProcessArgument $_ }) -join " "
|
||||
$process = [System.Diagnostics.Process]::Start($startInfo)
|
||||
if (-not $process) {
|
||||
throw "Could not start $FileName"
|
||||
}
|
||||
|
||||
$stdoutStream = [System.IO.File]::Open($StdoutPath, [System.IO.FileMode]::Create, [System.IO.FileAccess]::Write, [System.IO.FileShare]::Read)
|
||||
$stderrStream = [System.IO.File]::Open($StderrPath, [System.IO.FileMode]::Create, [System.IO.FileAccess]::Write, [System.IO.FileShare]::Read)
|
||||
return [pscustomobject]@{
|
||||
Process = $process
|
||||
StdoutTask = $process.StandardOutput.BaseStream.CopyToAsync($stdoutStream)
|
||||
StderrTask = $process.StandardError.BaseStream.CopyToAsync($stderrStream)
|
||||
StdoutStream = $stdoutStream
|
||||
StderrStream = $stderrStream
|
||||
}
|
||||
}
|
||||
|
||||
function Wait-LoggedProcess($Handle, [int]$TimeoutSeconds, [string]$Name) {
|
||||
if (-not $Handle.Process.WaitForExit($TimeoutSeconds * 1000)) {
|
||||
try { $Handle.Process.Kill($true) } catch {}
|
||||
throw "$Name timed out after ${TimeoutSeconds}s."
|
||||
}
|
||||
|
||||
[void]$Handle.StdoutTask.GetAwaiter().GetResult()
|
||||
[void]$Handle.StderrTask.GetAwaiter().GetResult()
|
||||
$Handle.StdoutStream.Dispose()
|
||||
$Handle.StderrStream.Dispose()
|
||||
if ($Handle.Process.ExitCode -ne 0) {
|
||||
throw "$Name failed with exit code $($Handle.Process.ExitCode)."
|
||||
}
|
||||
}
|
||||
|
||||
$repo = Split-Path -Parent $PSScriptRoot
|
||||
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
$absoluteLogRoot = if ([System.IO.Path]::IsPathRooted($LogRoot)) { $LogRoot } else { Join-Path $repo $LogRoot }
|
||||
$runDirectory = Join-Path $absoluteLogRoot $timestamp
|
||||
New-Item -ItemType Directory -Force -Path $runDirectory | Out-Null
|
||||
|
||||
$absoluteOutputPath = if ([System.IO.Path]::IsPathRooted($OutputPath)) { $OutputPath } else { Join-Path $repo $OutputPath }
|
||||
$outputDirectory = Split-Path -Parent $absoluteOutputPath
|
||||
New-Item -ItemType Directory -Force -Path $outputDirectory | Out-Null
|
||||
|
||||
$asioRaw = Join-Path $runDirectory "scarlett-f32-interleaved.raw"
|
||||
$kiyoMkv = Join-Path $runDirectory "kiyo-video.mkv"
|
||||
$compositeWav = Join-Path $runDirectory "dual-mic-composite.wav"
|
||||
$absoluteAsioProbe = if ([System.IO.Path]::IsPathRooted($AsioProbePath)) { $AsioProbePath } else { Join-Path $repo $AsioProbePath }
|
||||
|
||||
$asioArgs = @(
|
||||
"--set-sample-rate", $SampleRate.ToString(),
|
||||
"--record-f32-interleaved", $asioRaw,
|
||||
"--capture-seconds", $DurationSeconds.ToString()
|
||||
)
|
||||
$kiyoArgs = @(
|
||||
"-hide_banner",
|
||||
"-y",
|
||||
"-thread_queue_size", "1024",
|
||||
"-rtbufsize", "256M",
|
||||
"-f", "dshow",
|
||||
"-video_size", "${KiyoWidth}x${KiyoHeight}",
|
||||
"-framerate", $KiyoFrameRate.ToString(),
|
||||
"-i", "video=$KiyoVideoDevice",
|
||||
"-t", $DurationSeconds.ToString(),
|
||||
"-r", $KiyoFrameRate.ToString(),
|
||||
"-c:v", "h264_nvenc",
|
||||
"-preset", "p4",
|
||||
"-tune", "ll",
|
||||
"-pix_fmt", "yuv420p",
|
||||
$kiyoMkv
|
||||
)
|
||||
|
||||
Write-Host "Kiyo + computed dual-mic composite recording:"
|
||||
Write-Host " Kiyo: $KiyoVideoDevice ${KiyoWidth}x${KiyoHeight}@$KiyoFrameRate"
|
||||
Write-Host " ASIO: $absoluteAsioProbe sampleRate=$SampleRate channels=$AsioChannels shotgun=$ShotgunChannel cardioid=$CardioidChannel"
|
||||
Write-Host " Run dir: $runDirectory"
|
||||
Write-Host " Output: $absoluteOutputPath"
|
||||
|
||||
if ($DryRun) {
|
||||
Write-Host ($absoluteAsioProbe + " " + ($asioArgs -join " "))
|
||||
Write-Host ($FfmpegPath + " " + ($kiyoArgs -join " "))
|
||||
exit 0
|
||||
}
|
||||
|
||||
$asio = Start-LoggedProcess $absoluteAsioProbe $asioArgs (Join-Path $runDirectory "asio.out.log") (Join-Path $runDirectory "asio.err.log")
|
||||
$kiyo = Start-LoggedProcess $FfmpegPath $kiyoArgs (Join-Path $runDirectory "kiyo.out.log") (Join-Path $runDirectory "kiyo.err.log")
|
||||
|
||||
Wait-LoggedProcess $asio ($DurationSeconds + 15) "ASIO capture"
|
||||
Wait-LoggedProcess $kiyo ($DurationSeconds + 15) "Kiyo capture"
|
||||
|
||||
dotnet run --project (Join-Path $repo "src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj") -- `
|
||||
--asio-dual-mic-faust-cleaner-wav `
|
||||
--input $asioRaw `
|
||||
--output $compositeWav `
|
||||
--sample-rate $SampleRate `
|
||||
--channels $AsioChannels `
|
||||
--shotgun-channel $ShotgunChannel `
|
||||
--cardioid-channel $CardioidChannel
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Dual-mic composite WAV generation failed."
|
||||
}
|
||||
|
||||
$muxArgs = @(
|
||||
"-hide_banner",
|
||||
"-y",
|
||||
"-i", $kiyoMkv,
|
||||
"-i", $compositeWav,
|
||||
"-map", "0:v:0",
|
||||
"-map", "1:a:0",
|
||||
"-c:v", "copy",
|
||||
"-c:a", "aac",
|
||||
"-b:a", "192k",
|
||||
"-ar", "48000",
|
||||
"-movflags", "+faststart",
|
||||
$absoluteOutputPath
|
||||
)
|
||||
$mux = Start-LoggedProcess $FfmpegPath $muxArgs (Join-Path $runDirectory "mux.out.log") (Join-Path $runDirectory "mux.err.log")
|
||||
Wait-LoggedProcess $mux 60 "Mux"
|
||||
|
||||
Write-Host "Recording complete:"
|
||||
Write-Host " $absoluteOutputPath"
|
||||
Write-Host " composite wav: $compositeWav"
|
||||
Write-Host " report: $([System.IO.Path]::ChangeExtension($compositeWav, '.json'))"
|
||||
|
|
@ -1,201 +0,0 @@
|
|||
param(
|
||||
[int]$RavenPort = 5204,
|
||||
[int]$DurationSeconds = 20,
|
||||
[int]$StartupTimeoutSeconds = 20,
|
||||
[int]$Width = 1280,
|
||||
[int]$Height = 720,
|
||||
[int]$FrameRate = 30,
|
||||
[string]$OutputPath = "artifacts\runtime\stream-proof\stream-proof-output.mp4",
|
||||
[string]$LogRoot = "artifacts\runtime\stream-proof",
|
||||
[string]$FfmpegPath = "ffmpeg",
|
||||
[switch]$StartRavenSender,
|
||||
[string]$RavenHost = "madman's lullaby@192.168.1.84",
|
||||
[string]$RavenAddress = "192.168.1.84",
|
||||
[string]$RavenRepo = "C:\Meta\Mimir",
|
||||
[string]$StarfireHost = "192.168.1.66",
|
||||
[switch]$RavenSenderListens,
|
||||
[ValidateSet("srt", "tcp-listener")]
|
||||
[string]$RavenSenderTransport = "srt",
|
||||
[string]$RavenInputOverride = "",
|
||||
[string]$KiyoVideoDevice = "USB Video Device",
|
||||
[string]$StarfireScarlettAudioDevice = "Analogue 1 + 2 (2- Focusrite USB Audio)",
|
||||
[int]$KiyoWidth = 640,
|
||||
[int]$KiyoHeight = 480,
|
||||
[int]$KiyoFrameRate = 30,
|
||||
[int]$KiyoCornerWidth = 320,
|
||||
[int]$KiyoMargin = 28,
|
||||
[double]$RavenOffsetSeconds = 0.0,
|
||||
[double]$KiyoOffsetSeconds = 0.0,
|
||||
[double]$MicOffsetSeconds = 0.0,
|
||||
[switch]$NoStarfireScarlett,
|
||||
[switch]$DryRun
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Quote-ProcessArgument([string]$Value) {
|
||||
if ($Value.Length -eq 0) {
|
||||
return '""'
|
||||
}
|
||||
|
||||
if ($Value -notmatch '[\s"]') {
|
||||
return $Value
|
||||
}
|
||||
|
||||
return '"' + $Value.Replace('"', '\"') + '"'
|
||||
}
|
||||
|
||||
$repo = Split-Path -Parent $PSScriptRoot
|
||||
$absoluteOutputPath = if ([System.IO.Path]::IsPathRooted($OutputPath)) {
|
||||
$OutputPath
|
||||
} else {
|
||||
Join-Path $repo $OutputPath
|
||||
}
|
||||
$outputDirectory = Split-Path -Parent $absoluteOutputPath
|
||||
New-Item -ItemType Directory -Force -Path $outputDirectory | Out-Null
|
||||
$absoluteLogRoot = if ([System.IO.Path]::IsPathRooted($LogRoot)) {
|
||||
$LogRoot
|
||||
} else {
|
||||
Join-Path $repo $LogRoot
|
||||
}
|
||||
New-Item -ItemType Directory -Force -Path $absoluteLogRoot | Out-Null
|
||||
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
$stdoutLog = Join-Path $absoluteLogRoot "stream-proof-record-$timestamp.out.log"
|
||||
$stderrLog = Join-Path $absoluteLogRoot "stream-proof-record-$timestamp.err.log"
|
||||
|
||||
$ravenEndpoint = if ($RavenInputOverride.Length -gt 0) {
|
||||
$RavenInputOverride
|
||||
} elseif ($RavenSenderListens) {
|
||||
"srt://${RavenAddress}:${RavenPort}?mode=caller&latency=120000&timeout=30000000"
|
||||
} else {
|
||||
"srt://0.0.0.0:${RavenPort}?mode=listener&latency=120000&timeout=30000000"
|
||||
}
|
||||
$ravenSender = "${RavenRepo}\scripts\start-raven-av-sender.ps1"
|
||||
|
||||
if ($StartRavenSender) {
|
||||
$taskName = "MimirRavenAvSender$RavenPort"
|
||||
$remoteCmd = "${RavenRepo}\scripts\run-raven-av-$RavenPort.cmd"
|
||||
$remoteTargetHost = if ($RavenSenderListens) { "0.0.0.0" } else { $StarfireHost }
|
||||
$remoteSrtMode = if ($RavenSenderListens) { "listener" } else { "caller" }
|
||||
$remoteCommand = "powershell.exe -NoProfile -ExecutionPolicy Bypass -File $ravenSender -TargetHost $remoteTargetHost -Port $RavenPort -SrtMode $remoteSrtMode -Transport $RavenSenderTransport"
|
||||
$taskTime = (Get-Date).AddMinutes(1).ToString("HH:mm")
|
||||
Write-Host "Starting Raven muxed A/V sender on ${RavenHost} through interactive task ${taskName}: $remoteCommand"
|
||||
if (-not $DryRun) {
|
||||
ssh $RavenHost "cmd /c echo $remoteCommand ^> $remoteCmd" | Out-Null
|
||||
ssh $RavenHost "schtasks /Delete /TN $taskName /F" | Out-Null
|
||||
ssh $RavenHost "schtasks /Create /TN $taskName /SC ONCE /ST $taskTime /TR $remoteCmd /F /IT" | Out-Host
|
||||
ssh $RavenHost "schtasks /Run /TN $taskName" | Out-Host
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Could not start Raven muxed A/V sender task on $RavenHost."
|
||||
}
|
||||
Start-Sleep -Milliseconds 1500
|
||||
}
|
||||
}
|
||||
|
||||
$arguments = @(
|
||||
"-hide_banner",
|
||||
"-y",
|
||||
"-thread_queue_size", "1024",
|
||||
"-itsoffset", $RavenOffsetSeconds.ToString([Globalization.CultureInfo]::InvariantCulture),
|
||||
"-i", $ravenEndpoint,
|
||||
"-thread_queue_size", "1024",
|
||||
"-rtbufsize", "256M",
|
||||
"-itsoffset", $KiyoOffsetSeconds.ToString([Globalization.CultureInfo]::InvariantCulture),
|
||||
"-f", "dshow",
|
||||
"-video_size", "${KiyoWidth}x${KiyoHeight}",
|
||||
"-framerate", $KiyoFrameRate.ToString(),
|
||||
"-i", "video=$KiyoVideoDevice"
|
||||
)
|
||||
|
||||
if (-not $NoStarfireScarlett) {
|
||||
$arguments += @(
|
||||
"-thread_queue_size", "1024",
|
||||
"-rtbufsize", "256M",
|
||||
"-itsoffset", $MicOffsetSeconds.ToString([Globalization.CultureInfo]::InvariantCulture),
|
||||
"-f", "dshow",
|
||||
"-i", "audio=$StarfireScarlettAudioDevice"
|
||||
)
|
||||
}
|
||||
|
||||
$scaledKiyoHeight = [Math]::Max(1, [int][Math]::Round($KiyoCornerWidth * ($KiyoHeight / [double]$KiyoWidth)))
|
||||
$overlayX = $KiyoMargin
|
||||
$overlayY = $KiyoMargin
|
||||
|
||||
if ($NoStarfireScarlett) {
|
||||
$filter = "[0:v]scale=${Width}:${Height},setsar=1,hqdn3d=1.2:1.0:3.0:2.0[base];" +
|
||||
"[1:v]scale=${KiyoCornerWidth}:${scaledKiyoHeight},setsar=1,hqdn3d=1.4:1.1:3.4:2.2[pip];" +
|
||||
"[base][pip]overlay=${overlayX}:${overlayY}:format=auto[v];" +
|
||||
"[0:a]aresample=async=1:first_pts=0,afftdn=nr=8:nf=-35[a]"
|
||||
} else {
|
||||
$filter = "[0:v]scale=${Width}:${Height},setsar=1,hqdn3d=1.2:1.0:3.0:2.0[base];" +
|
||||
"[1:v]scale=${KiyoCornerWidth}:${scaledKiyoHeight},setsar=1,hqdn3d=1.4:1.1:3.4:2.2[pip];" +
|
||||
"[base][pip]overlay=${overlayX}:${overlayY}:format=auto[v];" +
|
||||
"[0:a]aresample=async=1:first_pts=0,afftdn=nr=8:nf=-35[ravenAudio];" +
|
||||
"[2:a]aresample=async=1:first_pts=0,afftdn=nr=10:nf=-38,volume=1.0[heroMic];" +
|
||||
"[ravenAudio][heroMic]amix=inputs=2:duration=longest:normalize=0[a]"
|
||||
}
|
||||
|
||||
$arguments += @(
|
||||
"-filter_complex", $filter,
|
||||
"-map", "[v]",
|
||||
"-map", "[a]",
|
||||
"-t", $DurationSeconds.ToString(),
|
||||
"-r", $FrameRate.ToString(),
|
||||
"-c:v", "h264_nvenc",
|
||||
"-preset", "p4",
|
||||
"-tune", "ll",
|
||||
"-rc", "cbr",
|
||||
"-b:v", "4500k",
|
||||
"-maxrate", "4500k",
|
||||
"-bufsize", "9000k",
|
||||
"-g", ($FrameRate * 2).ToString(),
|
||||
"-pix_fmt", "yuv420p",
|
||||
"-c:a", "aac",
|
||||
"-b:a", "160k",
|
||||
"-ar", "48000",
|
||||
"-movflags", "+faststart",
|
||||
$absoluteOutputPath
|
||||
)
|
||||
|
||||
Write-Host "Stream-proof recording:"
|
||||
Write-Host " Raven input: $ravenEndpoint"
|
||||
Write-Host " Raven SRT mode: $(if ($RavenSenderListens) { 'Raven listener, Starfire caller' } else { 'Starfire listener, Raven caller' })"
|
||||
Write-Host " Kiyo PIP: $KiyoVideoDevice ${KiyoWidth}x${KiyoHeight}@$KiyoFrameRate -> ${KiyoCornerWidth}x${scaledKiyoHeight}"
|
||||
Write-Host " Raven audio: Realtek/default render loopback in muxed NVENC/SRT feed"
|
||||
Write-Host " Starfire Scarlett: $(if ($NoStarfireScarlett) { 'disabled' } else { $StarfireScarlettAudioDevice })"
|
||||
Write-Host " sync role: Starfire Realtek emits chirps; Raven Realtek carries co-streamer game/program audio; Starfire Scarlett captures hero mics plus Raven shotgun"
|
||||
Write-Host " Offsets: raven=${RavenOffsetSeconds}s kiyo=${KiyoOffsetSeconds}s mic=${MicOffsetSeconds}s"
|
||||
Write-Host " Output: $absoluteOutputPath"
|
||||
Write-Host " stdout: $stdoutLog"
|
||||
Write-Host " stderr: $stderrLog"
|
||||
|
||||
if ($DryRun) {
|
||||
Write-Host ($FfmpegPath + " " + ($arguments -join " "))
|
||||
exit 0
|
||||
}
|
||||
|
||||
$startInfo = [System.Diagnostics.ProcessStartInfo]::new()
|
||||
$startInfo.FileName = $FfmpegPath
|
||||
$startInfo.UseShellExecute = $false
|
||||
$startInfo.RedirectStandardOutput = $true
|
||||
$startInfo.RedirectStandardError = $true
|
||||
$startInfo.Arguments = ($arguments | ForEach-Object { Quote-ProcessArgument $_ }) -join " "
|
||||
|
||||
$process = [System.Diagnostics.Process]::Start($startInfo)
|
||||
if (-not $process) {
|
||||
throw "Could not start FFmpeg recorder."
|
||||
}
|
||||
|
||||
$stdoutTask = $process.StandardOutput.BaseStream.CopyToAsync([System.IO.File]::Open($stdoutLog, [System.IO.FileMode]::Create, [System.IO.FileAccess]::Write, [System.IO.FileShare]::Read))
|
||||
$stderrTask = $process.StandardError.BaseStream.CopyToAsync([System.IO.File]::Open($stderrLog, [System.IO.FileMode]::Create, [System.IO.FileAccess]::Write, [System.IO.FileShare]::Read))
|
||||
$timeoutMs = [Math]::Max(1, $DurationSeconds + $StartupTimeoutSeconds) * 1000
|
||||
if (-not $process.WaitForExit($timeoutMs)) {
|
||||
try { $process.Kill($true) } catch {}
|
||||
throw "FFmpeg recorder timed out after $($DurationSeconds + $StartupTimeoutSeconds)s. Check $stderrLog."
|
||||
}
|
||||
|
||||
[void]$stdoutTask.GetAwaiter().GetResult()
|
||||
[void]$stderrTask.GetAwaiter().GetResult()
|
||||
if ($process.ExitCode -ne 0) {
|
||||
throw "FFmpeg recorder failed with exit code $($process.ExitCode). Check $stderrLog."
|
||||
}
|
||||