From 3dcc39208051517a5fdbf1f2935ff19c70a959ef Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 9 Jun 2026 02:21:32 +0100 Subject: [PATCH] Move program composition authority into Mimir --- config/perfect-machine.example.json | 42 ++++- docs/architecture.md | 15 +- docs/code-algorithm-map.md | 24 ++- docs/implementation-plan.md | 54 ++++-- docs/mimir-face.md | 17 +- docs/mimir-program-composition.md | 111 ++++++++++++ docs/native-rebuild-plan.md | 25 ++- docs/perfect-machine-domain-index.md | 20 ++- docs/perfect-machine-module-library.md | 5 +- docs/viable-stream-app.md | 50 ++++-- notes/current-system-map.md | 77 ++++++++- notes/fresh-workspace-handoff.md | 29 +++- src/Mimir.BufferSmoke/Program.cs | 40 ++++- .../MimirComputeOffloadConfiguration.cs | 6 +- .../Synchronization/MimirCultMeshContracts.cs | 153 +++++++++++++++++ .../Synchronization/MimirModuleLibrary.cs | 8 +- .../MimirObsPublicationConfiguration.cs | 83 --------- .../MimirPerfectMachineManifest.cs | 4 +- .../MimirPerfectMachineProfiles.cs | 54 ++++-- .../MimirProgramPublicationConfiguration.cs | 159 ++++++++++++++++++ .../MimirReservoirConfiguration.cs | 2 +- state/evidence.jsonl | 3 + state/map.yaml | 39 ++++- 23 files changed, 827 insertions(+), 193 deletions(-) create mode 100644 docs/mimir-program-composition.md delete mode 100644 src/Mimir.Runtime/Synchronization/MimirObsPublicationConfiguration.cs create mode 100644 src/Mimir.Runtime/Synchronization/MimirProgramPublicationConfiguration.cs diff --git a/config/perfect-machine.example.json b/config/perfect-machine.example.json index e3d8dab..36868d7 100644 --- a/config/perfect-machine.example.json +++ b/config/perfect-machine.example.json @@ -31,15 +31,25 @@ "outputs": { "video": "synchronized-program-video", "audio": "separately-controllable-synchronized-stems" + }, + "programComposition": { + "owner": "Mimir", + "sceneDocument": "mimir.program_scene.v1", + "operatorSurface": "mimir.eve_operator_surface.v1", + "outputDocument": "mimir.program_output.v1", + "sitePublisher": "Yggdrasil", + "obsRole": "compatibility-sink-only" } }, "authorities": { "configurationCalibrationLaunchStatus": "Mimir", + "programComposition": "Mimir", + "operatorSurfaces": "Eve", + "sitePublication": "Yggdrasil publisher daemon", "denseVisualFusion": "Fensalir", "gpuMaterialAndBrushReconciliation": "Fensalir", - "spoutPublication": "Fensalir", - "hotAudioDsp": "Faust", - "obsBroadcastControls": "OBS" + "programTextureOutput": "Fensalir", + "hotAudioDsp": "Faust" }, "visualSensors": [ { "id": "kiyo-primary", "role": "rgb-main", "required": true }, @@ -70,9 +80,9 @@ ], "outputs": { "video": { - "target": "Spout2", - "senderName": "Mimir Point Cloud", - "owner": "Fensalir" + "target": "MimirProgramTexture", + "senderName": "Mimir Program Texture", + "owner": "Mimir/Fensalir" }, "audio": { "owner": "Faust", @@ -85,6 +95,22 @@ "co_streamer_loopback", "spatial_bed" ] + }, + "operator": { + "owner": "Mimir/Eve", + "surfaces": [ + "eve-gui-compositor", + "eve-tui-operator", + "cultmesh-command-surface" + ] + }, + "site": { + "owner": "Yggdrasil publisher daemon", + "input": "mimir.program_output.v1", + "route": "cultmesh://yggdrasil/gamecult.site/mimir/program" + }, + "compatibility": { + "obs": "optional-derived-sink" } }, "runtimePolicy": { @@ -93,7 +119,9 @@ "native-driver-workers", "native-reservoir", "Fensalir", - "Faust" + "Faust", + "Eve", + "Yggdrasil publisher daemon" ], "neverInCompatibilityEdges": [ "dense-feature-extraction", diff --git a/docs/architecture.md b/docs/architecture.md index 607fcfa..c2195d9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2,11 +2,16 @@ ## Objective -Make one local Windows computer available as OBS sources on another computer, with sender-side NVENC video encoding and independently mixable audio sources. +Make one local Windows computer available as compatibility media sources on +another computer, with sender-side NVENC video encoding and independently +mixable audio sources. The current program target is Mimir-owned composition; +this bridge is not the stream authority. ## Current Mechanism -The repo owns configuration and scripts. FFmpeg owns capture, encoding, and SRT transport. OBS owns scene composition. +The repo owns configuration and scripts. FFmpeg owns capture, encoding, and SRT +transport. Mimir owns scene composition; OBS may consume this bridge as a +temporary compatibility sink. Inputs become outputs like this: @@ -21,8 +26,10 @@ Inputs become outputs like this: ## Invariants - Video encoding happens on the sender. -- OBS receives ordinary media-source URLs; it does not need a plugin for v1. -- Audio remains separately mixable in OBS. +- OBS can receive ordinary media-source URLs for compatibility; it does not need + a plugin for this bridge path. +- Audio remains separately mixable at the compatibility edge, but the durable + control surface belongs to Mimir/Eve. - Ports are stable and derived from one base port plus explicit offsets. - Secrets, if used, stay in local config and are not committed. diff --git a/docs/code-algorithm-map.md b/docs/code-algorithm-map.md index 8ebd9be..7be4449 100644 --- a/docs/code-algorithm-map.md +++ b/docs/code-algorithm-map.md @@ -15,8 +15,9 @@ Timestamp for this pass: 2026-05-23, starting at 23:24 Europe/London. Mimir collects local and network audio/video samples into bounded rolling runtime buffers, uses active/passive timing evidence to place audio streams on a -canonical timeline, and leaves Fensalir/Faust/OBS with clear ownership over -rendering, DSP, and broadcast composition. +canonical timeline, and lets Mimir own program composition while Fensalir lowers +pixels, Faust/native DSP moves audio, Eve exposes operator surfaces, and +Yggdrasil publishes the public feed. ## Current Mechanism @@ -54,11 +55,17 @@ reflection taps. - UI and telemetry read cached sync state; they do not run analysis. - Native capture owns hardware calls. Runtime owns sample identity and buffer placement. -- Fensalir owns windowing, rendering, D3D12, UI, visual fusion, and program - video output. +- Mimir owns the program scene graph, subscription policy, preview/control + state, stats, and publication intent. +- Fensalir owns windowing, rendering, D3D12, UI lowering, visual fusion, and + program video output. - Faust/native DSP will own hot resampling, fractional delay, separation, spatialization, and stems. -- OBS owns broadcast composition. +- Eve GUI/TUI lowerers render Mimir operator surfaces without owning scene + truth. +- The Yggdrasil-facing publisher daemon consumes Mimir program output without + owning composition. +- OBS is a compatibility sink only. ## Diagram @@ -83,6 +90,10 @@ flowchart TD AsioSource --> Sources Probes["native/probes/*"] --> FrameEvents["MimirFrameEventProcessStreamSource"] FrameEvents --> Sources + Runtime --> Scene["Mimir program scene graph"] + Scene --> Eve["Eve operator surfaces"] + Scene --> Publisher["Yggdrasil site publisher"] + Scene -. "compatibility" .-> Obs["OBS adapter"] Reservoir["native/reservoir"] -. "future lower ABI" .-> Hub ``` @@ -96,6 +107,9 @@ flowchart TD - `native/reservoir` is the lower native rolling-buffer ABI. - `native/probes/*` are measurement and diagnostic tools. - `scripts/*` are bridge utilities, not the final synchronized program path. +- `docs/mimir-program-composition.md` is the authority map for Muninn capture + hosts, Mimir composition, Eve operation, Yggdrasil publication, and OBS + compatibility. ## Stage 0: Solution And App Host diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 608b6c4..a44944a 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -13,12 +13,12 @@ For the source-level ownership map, read [[code-algorithm-map|Code Algorithm Map ingest, one rolling buffer per configured audio/video stream, and the synchronization hub Fensalir can inspect. - The default five-second window is an intentional latency/memory trade: use it - to line up streams and extract the volumetric audio/video field before OBS - sees program output. + to line up streams and extract the volumetric audio/video field before Mimir + emits program output. - `native/reservoir` owns the lower native rolling-buffer invariant for Fensalir/Faust integration. -- PowerShell/FFmpeg/SRT remains a bridge utility for LAN OBS feeds. It is not - the synchronized program authority. +- PowerShell/FFmpeg/SRT remains a bridge utility for LAN compatibility feeds. + It is not the synchronized program or publication authority. The old script stack is gone. Do not add a compatibility edge unless it protects a named invariant that the native runtime cannot protect yet. @@ -184,7 +184,7 @@ a named invariant that the native runtime cannot protect yet. profiles, language/emission profiles, path-learning sessions, benchmark panels, audio actuator strategies, native capture profiles, camera ingest strategies, reservoir strategies, audio/visual field profiles, compute - offload profiles, OBS publication profiles, CultMesh contracts, Fensalir + offload profiles, program publication profiles, CultMesh contracts, Fensalir lowering, and assembly plans. Use `Mimir.BufferSmoke --perfect-machine-profile-smoke` to prove the catalog assembles, `--perfect-machine-contract-smoke` to write a CultCache contract @@ -202,14 +202,40 @@ a named invariant that the native runtime cannot protect yet. source-id hashing, producer helpers, and typed audio/render payload descriptors. - Windows bridge scripts for sender discovery/start/stop and simple OBS Media - Source ingest. + Source compatibility ingest. +- `src/Mimir.Runtime/Synchronization` now contains the first typed program + composition contracts: `mimir.program_scene.v1`, `mimir.program_output.v1`, + and `mimir.eve_operator_surface.v1`. These make Mimir's scene graph, + Yggdrasil/site publication route, and Eve operator control/preview surface + manifestable through CultMesh. +- `docs/mimir-program-composition.md` is the current authority map for the new + stream-program architecture: Muninn observes on Starfire/Nightwing/Raven-class + hosts, Mimir consumes selected streams and composes the program, Eve lowers + controls/previews/stats, Yggdrasil publishes to the site, and OBS is + compatibility-only. +- `src/Mimir.CultMeshMedia` is the first explicit CultMesh media/body bridge: + `relay` hosts the document relay on CultNet UDP `3075`, `send` reads an + MPEG-TS byte stream from stdin and publishes rolling + `mimir.cultmesh_media_frame` documents, and `recv` subscribes to those + documents and writes ordered MPEG-TS bytes to a Starfire-local UDP endpoint + for compatibility sinks. +- `scripts/start-raven-cultmesh-av-sender.ps1`, + `scripts/start-yggdrasil-cultmesh-media-relay.ps1`, and + `scripts/start-starfire-cultmesh-av-receiver.ps1` are the CultMesh bridge + operators. Raven capture defaults to FFmpeg desktop frames plus Mimir's + WASAPI loopback capture muxed as H.264/AAC MPEG-TS; DirectShow audio remains + an explicit fallback. - Documentation for OBS receiver setup, native rebuild boundaries, the viable stream app, and the Mimir Face. ## Temporary - Audio and video may still traverse separate OBS/SRT endpoints during bridge - testing so OBS can preserve independent controls. + testing, but Mimir/Eve owns independent controls. +- The CultMesh media bridge still lowers to local UDP for compatibility sinks + because OBS is not a CultMesh consumer. Network transit between Raven, + Yggdrasil, and Starfire is the CultMesh/CultNet path; OBS-local UDP is an + egress adapter only. - Process-backed stream sources are only acceptable for network bridge feeds or diagnostics. Six-camera local ingest belongs behind direct capture drivers. - Frame-event process sources are diagnostic only. They prove source cadence and @@ -239,14 +265,20 @@ a named invariant that the native runtime cannot protect yet. with probe durations long enough to keep loopback and mic windows live. 6. Bind Fensalir UI to the synchronization hub so buffer depth, stream cadence, source timestamps, and output settings are visible and adjustable. -7. Lower `AquariumBufferFieldFrame` spline tube fields into Fensalir compute: +7. Implement the Mimir program scene graph as the shared commit primitive for + source subscription, transforms, crop, chroma key, visibility, layer order, + preview, and output publication. Import the current OBS scene only as an + initial mirror, then make Eve GUI/TUI the operator surface. +8. Add the Yggdrasil-facing site publisher daemon that consumes the Mimir + program output and publishes it without owning a second composition. +9. Lower `AquariumBufferFieldFrame` spline tube fields into Fensalir compute: sample buffer-domain paths stochastically by visual contribution, emit SDF splat probes, write them into the spatiotemporal splat reservoir, and sample that reservoir in the temporally antialiased scene pass. The direct spline preview must stay a witness until this path owns rendering. -8. Move GPU feature extraction, fusion, material fitting, render budgeting, and +10. Move GPU feature extraction, fusion, material fitting, render budgeting, and Spout2 publication into Fensalir. -9. Move mic alignment, room suppression, voice separation, spatialization, and +11. Move mic alignment, room suppression, voice separation, spatialization, and stem generation into Faust/native DSP. -10. Keep the OBS bridge witness ledger as evidence before expanding receiver +12. Keep the OBS bridge witness ledger as evidence before expanding receiver machinery. diff --git a/docs/mimir-face.md b/docs/mimir-face.md index 3c9ea4d..40efc7f 100644 --- a/docs/mimir-face.md +++ b/docs/mimir-face.md @@ -48,9 +48,10 @@ Mimir exists to turn cameras, microphones, speakers, timing signals, and distributed capture surfaces into one realtime coherent volumetric field. The field is the product. The bridge scripts are scaffolding and probes. -The live target is one native five-second rolling reservoir feeding Fensalir GPU -fusion and Faust/native DSP, then OBS. Retention has one owner. Typed views are -indexes, not private histories. +The live target is one native five-second rolling reservoir feeding Mimir-owned +program composition, Fensalir GPU fusion, Faust/native DSP, Eve operator +surfaces, and a Yggdrasil site publisher. OBS is compatibility only. Retention +has one owner. Typed views are indexes, not private histories. ## Operating Doctrine @@ -63,11 +64,15 @@ the synchronized program path is being rebuilt around `LocalcastRuntime`, Invariants: -- OBS owns final broadcast composition. -- Fensalir owns dense visual fusion, material/brush/splat rendering, and Spout2 - publication. +- Mimir owns final program composition, source subscription, preview/control + state, stats, and publication intent. +- Fensalir owns dense visual fusion, material/brush/splat rendering, and local + program texture output. - Faust/native DSP owns hot audio alignment, room suppression, voice separation, stems, and spatial bed generation. +- Eve lowerers render operator controls and preview without owning scene truth. +- The Yggdrasil publisher daemon ships the Mimir program to the site without + owning composition. - FFmpeg owns simple capture, encode, and transport in V1 bridge mode. - Mimir/this repo owns configuration, calibration, launch/status, typed contracts, memory, and operational repeatability. diff --git a/docs/mimir-program-composition.md b/docs/mimir-program-composition.md new file mode 100644 index 0000000..2febc26 --- /dev/null +++ b/docs/mimir-program-composition.md @@ -0,0 +1,111 @@ +# Mimir Program Composition + +## Objective + +Mimir owns the live stream program: selected capture feeds, calibration feeds, +scene composition, preview/control surfaces, stats, and publication. OBS is a +temporary compatibility sink only. + +## Authority Map + +- Owner: Mimir owns the program scene graph and subscription policy. +- Inputs: Muninn-published local and remote monitor, window, camera, audio, + loopback, and health streams. +- Outputs: a Mimir program video surface, aligned audio stems or final bus, Eve + GUI/TUI operator surfaces, and a Yggdrasil-facing site publication feed. +- Derived state: OBS receives a compatibility feed when useful; it does not own + scene truth, broadcast truth, preview truth, stats, source selection, or + publication. +- Forbidden writers: OBS scenes, browser dashboards, site daemons, and Muninn + capture hosts must not independently decide the program composition. +- Shared paths: direct operator edits, remote Eve commands, preset loads, + calibration-driven source selection, and recovery scripts must commit through + the same Mimir scene graph command surface. +- Deletion line: old OBS-owned composition docs and config names are demoted to + diagnostic bridge language. Any OBS adapter delegates to the Mimir program + scene instead of carrying its own opinion. + +## Machine Shape + +```mermaid +flowchart TD + A["Muninn on Starfire"] --> D["CultMesh stream catalog + media bodies"] + B["Muninn on Nightwing"] --> D + C["Muninn on Raven"] --> D + D --> E["Mimir subscription policy"] + E --> F["Mimir rolling buffers"] + F --> G["Mimir program scene graph"] + G --> H["Fensalir compositor"] + F --> I["Faust/native DSP"] + H --> J["program video surface"] + I --> K["program audio stems/bus"] + G --> L["Eve GUI/TUI operator surfaces"] + J --> M["Yggdrasil site publisher daemon"] + K --> M + M --> N["GameCult site"] + J -. "compatibility" .-> O["OBS adapter"] + K -. "compatibility" .-> O +``` + +## Composition Contract + +The scene graph carries layers with: + +- source reference, such as `muninn:starfire:monitor:primary` or + `muninn:starfire:window:discord`; +- source kind, such as monitor, window, camera, network monitor, or generated + Mimir visual field; +- visibility, layer order, transform, crop, and optional chroma key; +- enough timing identity for Mimir to align media inside the rolling window. + +The current OBS scene can be mirrored as an initial Mimir scene: + +- Starfire monitor capture, cropped at the bottom when needed. +- Discord window capture, cropped to chat and keyed with chroma filters. +- Raven primary monitor plus Realtek loopback as a subscribed remote Muninn + stream. +- Mimir/Fensalir program texture as a generated layer when the field renderer is + active. + +After import, OBS is not the editor of record. It may display the Mimir program +feed, but operator edits happen through Mimir/Eve command surfaces. + +## Eve Operator Surface + +Mimir publishes Eve GUI/TUI documents for: + +- live program preview; +- source catalog and subscription state; +- layer transform, crop, chroma key, visibility, and ordering; +- audio stem gain/mute/route controls; +- stream health, latency, buffer depth, timing confidence, and publication + status; +- emergency scene selection and publisher recovery commands. + +Eve lowerers render those controls on any device without owning the underlying +scene graph. + +## Site Publisher + +The Yggdrasil-facing publisher consumes Mimir program output and publishes it to +the GameCult site. It owns transport, encoding/session state, reconnect policy, +and public edge health. It does not own composition. + +The publisher's input contract is the same Mimir program output used by local +preview and OBS compatibility adapters, so the public stream cannot silently +diverge from the operator preview. + +## Runtime Contracts + +The first typed contracts live in `src/Mimir.Runtime/Synchronization`: + +- `mimir.program_scene.v1` +- `mimir.program_output.v1` +- `mimir.eve_operator_surface.v1` + +The proof path is: + +```powershell +dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --perfect-machine-contract-smoke +dotnet run --project .\src\Mimir.BufferSmoke\Mimir.BufferSmoke.csproj -- --perfect-machine-profile-smoke +``` diff --git a/docs/native-rebuild-plan.md b/docs/native-rebuild-plan.md index 39c648b..a76edff 100644 --- a/docs/native-rebuild-plan.md +++ b/docs/native-rebuild-plan.md @@ -5,17 +5,24 @@ hosted stream machine. ## Objective -Make the room available in OBS as synchronized program video plus separately +Make the room available as a Mimir-owned synchronized program with separately controllable audio while preserving ownership: - Mimir owns configuration, calibration truth, runtime contracts, launch, status, and persistence. - Mimir.Runtime owns in-memory stream buffers and runtime synchronization. - Native capture workers own device reads and append typed sample handles. -- Fensalir owns GPU fusion, rendering, D3D12 interop, UI, and Spout2 output. +- Mimir owns stream subscription, scene composition, preview/control state, + stats, and program publication intent. +- Fensalir owns GPU fusion, rendering, D3D12 interop, UI lowering, and local + program texture output. - Faust/native DSP owns hot audio alignment, suppression, separation, stems, and spatial bed generation. -- OBS owns broadcast controls. +- Eve GUI/TUI lowerers expose operator controls and preview without owning + scene truth. +- The Yggdrasil-facing publisher daemon consumes Mimir program output and + publishes it to the site without owning composition. +- OBS is a temporary compatibility sink. ## Runtime State Model @@ -26,10 +33,13 @@ flowchart TD N["network feed producers"] --> R R --> Q["Fensalir UI + GPU fusion"] R --> F["Faust/native DSP"] - Q --> S["Spout2/program video"] + Q --> S["Mimir program video"] F --> P["program stems + spatial bed"] - S --> O["OBS"] - P --> O + Q --> E["Eve operator surfaces"] + S --> Y["Yggdrasil site publisher"] + P --> Y + S -. "compatibility" .-> O["OBS adapter"] + P -. "compatibility" .-> O ``` The app-level runtime initializes one rolling buffer per configured audio or @@ -48,7 +58,8 @@ camera driver buffers, GPU textures, or DSP audio memory. - Fensalir consumes current visual/audio timing state and publishes the program video surface. - Faust/native DSP consumes audio/phase/event handles and publishes stems. -- OBS receives final program surfaces. It does not own synchronization. +- OBS compatibility adapters may receive final program surfaces. They do not + own synchronization, composition, preview/control, or publication. ## Impossible By Construction diff --git a/docs/perfect-machine-domain-index.md b/docs/perfect-machine-domain-index.md index 11c86ce..cd627b3 100644 --- a/docs/perfect-machine-domain-index.md +++ b/docs/perfect-machine-domain-index.md @@ -38,19 +38,23 @@ Read this with: Mimir is a bounded-latency realtime field computer: it ingests many clocks, places their evidence inside one rolling temporal window, extracts audio/video -geometry from that window, and emits an OBS-usable program surface without -letting OBS become the synchronization authority. +geometry from that window, owns the stream program scene, exposes Eve operator +surfaces, and emits a publicable program feed without letting compatibility +sinks become authority. ## Authority Index | Authority | Owns | Must Not Own | | --- | --- | --- | -| `Mimir.Runtime` | stream identity, buffers, sync reports, calibration model loading, source polling, telemetry state | hot DSP actuator, dense visual fusion, broadcast composition | -| Native capture | direct driver callbacks, device timestamps, payload handles, low-jitter capture loops | runtime policy, UI state, OBS composition | +| Mimir program composition | source subscription, scene graph, crop/key/transform state, preview/control state, stats, publication intent | local device reads, GPU shader internals, audio-rate DSP, site transport policy | +| `Mimir.Runtime` | stream identity, buffers, sync reports, calibration model loading, source polling, telemetry state | hot DSP actuator, dense visual fusion, direct device capture | +| Native capture / Muninn hosts | direct driver callbacks, device timestamps, payload handles, local stream capabilities/media | runtime policy, scene composition, publication | | `native/reservoir` | shared-edge retention and typed views | payload memory interpretation, rendering, DSP | -| Fensalir | window, UI, D3D12 bridge, GPU fusion, program video publication | clock authority, audio actuator | +| Fensalir | window, UI lowering, D3D12 bridge, GPU fusion, program video output | clock authority, audio actuator, scene ownership | | Faust/native DSP | fractional delay, resampling, separation, spatialization, stems | stream discovery, UI | -| OBS | final broadcast composition | synchronization, capture timing | +| Eve lowerers | operator GUI/TUI rendering, controls, previews, stats lowering | scene truth, stream truth, publication truth | +| Yggdrasil site publisher | public transport, encoder/session state, reconnect policy, site edge health | scene composition | +| OBS | compatibility preview/output sink | synchronization, capture timing, composition, publication | | Raven/phones | local decode of codebook/schedule observations | canonical clock authority | ## Evidence Taxonomy @@ -241,7 +245,9 @@ Six optical sensors with high-rate Eyes and Leap IR mean: - Cut JSON/stdout local camera hot path when concrete direct drivers land. - Cut legacy chirplet path once chirp-bin decoder has equivalent docs/tests and physical path proof. -- Cut bridge scripts only after OBS-facing direct output is real. +- Cut bridge scripts only after Mimir program output, Eve operation, and + Yggdrasil publication are real enough that OBS compatibility no longer earns + its keep. - Cut ad hoc response weighting if a formal path model replaces it. ## Future Work Index diff --git a/docs/perfect-machine-module-library.md b/docs/perfect-machine-module-library.md index fd0e44d..c5fa824 100644 --- a/docs/perfect-machine-module-library.md +++ b/docs/perfect-machine-module-library.md @@ -27,7 +27,7 @@ job-interview answer for why it belongs in the body of the machine. | Visual fusion configurations | `Mimir.Runtime` + Fensalir | Sensor roles and fusion modes from cadence proof through full hybrid evidence. | `src/Mimir.Runtime/Synchronization/MimirVisualFusionConfiguration.cs` | | Compute offload configurations | `Mimir.Runtime` + CultMesh | Local-heavy, distributed witness, and calibration sweep workload placement. | `src/Mimir.Runtime/Synchronization/MimirComputeOffloadConfiguration.cs` | | Assembly plans | `Mimir.Runtime` | Named combinations of modules that prove synthetic, Scarlett, meatspace, and distributed cuts. | `src/Mimir.Runtime/Synchronization/MimirMachineAssemblyPlan.cs` | -| OBS publication profiles | Fensalir + Faust + OBS | Spout/shared-texture video and separately mixable Faust stem publication. | `src/Mimir.Runtime/Synchronization/MimirObsPublicationConfiguration.cs` | +| Program publication profiles | Mimir + Fensalir + Faust + Eve + Yggdrasil | Mimir-owned scene publication, Eve operator surfaces, Yggdrasil site feed, and temporary OBS compatibility adapters. | `src/Mimir.Runtime/Synchronization/MimirProgramPublicationConfiguration.cs` | | Fensalir lowering | `Mimir.Runtime` -> Fensalir | Pure mapping from Mimir rolling/video/audio state into Fensalir GPU sensor and acoustic field frames. | `src/Mimir.Runtime/Synchronization/MimirFensalirFieldLowering.cs` | | Module catalog | `Mimir.Runtime` | Indexes the live module set and verification commands. | `src/Mimir.Runtime/Synchronization/MimirModuleLibrary.cs` | | Manifest factory | `Mimir.Runtime` | Produces an exportable JSON manifest for tools, docs, UI, and remote witnesses. | `src/Mimir.Runtime/Synchronization/MimirPerfectMachineManifest.cs` | @@ -42,7 +42,8 @@ job-interview answer for why it belongs in the body of the machine. stem generation. - CultMesh carries typed state surfaces; remote nodes may observe and decode, but they do not become independent clock authorities. -- OBS receives final program surfaces and stems. It does not own calibration. +- OBS may receive compatibility program surfaces and stems. It does not own + calibration, composition, preview/control, or publication. ## Verification Surface diff --git a/docs/viable-stream-app.md b/docs/viable-stream-app.md index a304701..eb58c07 100644 --- a/docs/viable-stream-app.md +++ b/docs/viable-stream-app.md @@ -4,12 +4,13 @@ Build the smallest live Mimir application that can ingest multiple local or networked video and audio feeds, hold a short synchronized window in memory, -and emit coherent OBS-facing program outputs while Fensalir provides the -operator UI. +and emit coherent Mimir program outputs while Eve provides operator surfaces +and Yggdrasil publishes the site feed. The default reservoir horizon is five seconds. That delay is not decoration. It is the budget that lets late network packets, independent capture clocks, -audio/video alignment, and debug controls converge before OBS sees the result. +audio/video alignment, and debug controls converge before Mimir emits the +program result. Mimir is explicitly trading latency and memory for throughput and extraction quality: the app may spend that buffered window lining streams up and crunching audio/video data to recover as much of the volumetric audio field and @@ -21,16 +22,16 @@ alignment and field-reconstruction budget. Today the repo has two partial machines: -- FFmpeg/SRT bridge scripts can capture sender video/audio and expose stable OBS - Media Source endpoints. +- FFmpeg/SRT bridge scripts can capture sender video/audio and expose stable + compatibility endpoints. - The native `LocalcastRuntime` reservoir ABI can hold typed live sample handles, and Fensalir can consume native render packets for GPU fusion. Those parts are useful, but they are not yet the viable app. The viable app is the join: ingest workers append live sample handles into one Fensalir-hosted runtime window, Fensalir renders video from that window, Faust/native audio -builds synchronized stems from the same window, and OBS receives only the -program outputs. +builds synchronized stems from the same window, Mimir commits the scene graph, +Eve lowers the operator surface, and Yggdrasil publishes the program output. ```mermaid flowchart TD @@ -39,10 +40,15 @@ flowchart TD C["local mic/loopback workers"] --> R D["network audio ingest workers"] --> R R --> U["Fensalir debug/settings/output UI"] - R --> V["Fensalir GPU video composition"] + R --> S["Mimir program scene graph"] + S --> U + S --> V["Fensalir GPU video composition"] R --> F["Faust/native audio alignment + stems"] - V --> O["OBS video output"] - F --> P["OBS audio stem outputs"] + S --> E["Eve GUI/TUI operator surfaces"] + V --> Y["Yggdrasil site publisher"] + F --> Y + V -. "compatibility" .-> O["OBS adapter"] + F -. "compatibility" .-> O ``` ## Invariants @@ -51,12 +57,17 @@ flowchart TD - The five-second rolling window is a deliberate compute budget, not accidental latency. It exists so capture, sync, GPU fusion, and audio field extraction can choose coherence over immediacy. -- Fensalir owns the running app, UI, debug state, settings changes, output - management, GPU fusion/rendering, and Spout/video publication. +- Mimir owns the program scene graph, source subscription policy, preview/control + state, stats, and publication intent. +- Fensalir owns the running app host, UI lowering, debug state, GPU + fusion/rendering, and local program texture output. - Faust/native DSP owns audio alignment, voice separation, suppression, volumetric sound-field extraction, stems, and spatial bed generation. -- OBS receives synchronized program surfaces. Raw unsynchronized feeds may be - diagnostic inputs, not broadcast truth. +- Eve lowerers render controls and previews without owning scene truth. +- The Yggdrasil publisher consumes synchronized program surfaces without owning + a second composition. +- OBS compatibility adapters may receive synchronized program surfaces. Raw + unsynchronized feeds may be diagnostic inputs, not broadcast truth. - Networked feeds are producers into the same reservoir. They do not own clocks, private buffers, or a parallel synchronization model. - Missing feeds become explicit absence or silent/blank placeholders inside the @@ -78,9 +89,14 @@ The first cut should be deliberately boring: loopback ingest. 7. Expose Fensalir UI readouts for reservoir edge, window start, per-kind counts, feed health, delay estimate, output status, and current settings. -8. Render one synchronized program video output from the reservoir. -9. Emit separately controllable OBS audio stems from the same reservoir window. -10. Keep the existing FFmpeg/SRT scripts as edge utilities for network capture +8. Render one synchronized Mimir program video output from the reservoir. +9. Emit separately controllable program audio stems from the same reservoir + window. +10. Publish Eve GUI/TUI operator surfaces for preview, source selection, scene + edits, stats, and output state. +11. Add the Yggdrasil-facing publisher daemon that consumes Mimir program + output without owning composition. +12. Keep the existing FFmpeg/SRT scripts as edge utilities for network capture and OBS compatibility, not as the stream authority. ## Repo Shape diff --git a/notes/current-system-map.md b/notes/current-system-map.md index 90077fb..0317e12 100644 --- a/notes/current-system-map.md +++ b/notes/current-system-map.md @@ -25,10 +25,13 @@ flowchart TD R --> N["native reservoir handles"] N --> E["Fensalir GPU fusion + UI"] N --> F["Faust/native DSP"] - E --> G["Spout2/program video"] + E --> G["Mimir program video"] F --> H["program stems + spatial bed"] - G --> I["OBS"] - H --> I + E --> I["Eve GUI/TUI operator surfaces"] + G --> J["Yggdrasil site publisher"] + H --> J + G -. "compatibility" .-> K["OBS adapter"] + H -. "compatibility" .-> K ``` Ownership: @@ -41,11 +44,18 @@ Ownership: becoming independent clock authorities. - `Mimir.Runtime` owns app-level stream buffers and synchronization. - Native capture workers own device reads. +- Mimir owns program composition, source subscription policy, preview/control + state, stats, and publication intent. - Fensalir owns dense visual fusion, material/brush/splat reconciliation, - D3D12 interop, runtime UI, and Spout2 publication. + D3D12 interop, runtime UI lowering, and local program texture output. - Faust/native DSP owns hot audio alignment, suppression, separation, spatialization, and stem generation. -- OBS owns broadcast controls. +- Eve GUI/TUI lowerers render Mimir's operator surfaces on any device without + owning scene truth. +- The Yggdrasil-facing publisher daemon consumes Mimir program output and + publishes it to the site without owning composition. +- OBS is a temporary compatibility sink, not a composition or broadcast + authority. Invariant: the live window is bounded, in memory, and has one timing authority. No private history outlives the rolling buffer. @@ -54,7 +64,7 @@ No private history outlives the rolling buffer. [[docs/viable-stream-app|Viable Stream App]] defines the near-term app target. Fensalir hosts the running Mimir app, keeps the default five-second runtime in -memory, exposes debug/settings/output controls, and emits synchronized OBS +memory, exposes debug/settings/output controls, and emits synchronized Mimir program video plus separately controllable audio stems. `Mimir.Runtime` currently provides: @@ -79,6 +89,25 @@ the rolling buffers while the direct ABI driver is being cut. It does not carry pixels and does not own the final six-camera hot path. Multi-camera probes are one process with declared accepted source ids, not one process per camera. +## Program Composition + +[[docs/mimir-program-composition|Mimir Program Composition]] is the live +authority map for stream-program output. Muninn runs on Starfire, Nightwing, and +Raven-class capture hosts to publish local stream capabilities and selected +media bodies. Mimir consumes only the streams it needs for calibration and +composition, commits one typed scene graph, emits Eve GUI/TUI operator surfaces, +and publishes program output locally plus through a Yggdrasil-facing site +publisher. + +The first typed program contracts live in `src/Mimir.Runtime/Synchronization`: + +- `mimir.program_scene.v1` +- `mimir.program_output.v1` +- `mimir.eve_operator_surface.v1` + +OBS scene JSON is import/mirror evidence for the initial crop/key/transform +layout. It is not the editor of record after Mimir owns the scene graph. + ## OBS Bridge Utility ```mermaid @@ -97,6 +126,42 @@ flowchart TD The bridge is useful because it is inspectable and already speaks OBS. It does not become the synchronized program authority. +## CultMesh Media Bridge + +```mermaid +flowchart TD + A["Raven FFmpeg desktop + WASAPI loopback mux"] --> B["Mimir.CultMeshMedia send"] + B --> C["CultNet reliable UDP media-frame documents"] + C --> D["Yggdrasil CultMesh relay on 10.77.0.1:3075"] + D --> E["Starfire Mimir.CultMeshMedia recv"] + E --> F["local MPEG-TS UDP udp://127.0.0.1:5200"] + F --> G["OBS Raven Monitor + Realtek"] +``` + +Owner: `src/Mimir.CultMeshMedia` owns the live Raven-to-Starfire media body +bridge over CultMesh/CultNet. FFmpeg still owns Raven capture and encoding. +Mimir owns composition. OBS may consume the Starfire-local MPEG-TS endpoint as a +compatibility sink while Mimir/Fensalir program output and site publication +mature. + +Invariant: Raven media body state is live and bounded. Sender writes rolling +`mimir.cultmesh_media_frame` slot documents rather than an unbounded durable +video archive. Yggdrasil relays CultNet reliable UDP over the WireGuard mesh. +Starfire lowers the subscribed stream to local UDP for compatibility sinks +because OBS is not a CultMesh runtime. + +Current deployment: Yggdrasil runs the relay from +`/opt/gamecult/mimir-cultmesh-media/Mimir.CultMeshMedia` with cache +`/var/lib/gamecult/mimir/cultmesh-media.cc` and log +`/var/log/gamecult/mimir-cultmesh-media.log`. Starfire runs the receiver from +`artifacts/mimir-cultmesh-media-win-x64-selfcontained/Mimir.CultMeshMedia.exe` +and writes `raven-primary-av` to `udp://127.0.0.1:5200`. OBS source +`Raven Monitor + Realtek` points at that local UDP endpoint. + +Raven SSH over WireGuard timed out during the initial deployment pass, so the +Raven sender still needs to be launched on Raven directly with +`scripts/start-raven-cultmesh-av-sender.ps1`. + ## Audio Field The six-microphone path is separate from the bridge. Scarlett speaker loopback diff --git a/notes/fresh-workspace-handoff.md b/notes/fresh-workspace-handoff.md index f164d87..aa1eda9 100644 --- a/notes/fresh-workspace-handoff.md +++ b/notes/fresh-workspace-handoff.md @@ -21,7 +21,11 @@ Get-Content .\state\evidence.jsonl -Tail 8 - Mimir's Face doctrine lives in [[docs/mimir-face|Mimir Face]]. - VoidBot persona/state live under `.voidbot/voice/` and `.voidbot/state/`. Commit `.voidbot` dirt in a separate state commit whenever it appears. -- V1 still has FFmpeg/SRT/OBS bridge utilities for LAN ingest. +- V1 still has FFmpeg/SRT/OBS bridge utilities for LAN compatibility ingest. +- V1 now also has an explicit CultMesh media bridge for Raven program feeds: + `src/Mimir.CultMeshMedia` sends rolling `mimir.cultmesh_media_frame` + documents over CultNet reliable UDP through Yggdrasil and lowers them on + Starfire to local MPEG-TS UDP for OBS. - The live stream app is C# plus Fensalir: `Mimir.slnx` contains `src/Mimir.App` and `src/Mimir.Runtime`. - `src/Mimir.App` hosts Fensalir as the windowing/rendering/D3D12 bridge. @@ -37,6 +41,14 @@ Get-Content .\state\evidence.jsonl -Tail 8 dictionary, low-level implementation notes, and sample code sketches for the next decoder/DSP/native-ring/Fensalir implementation passes. Start with [[research/perfect-machine-study-2026-05-23/reading-guide|Reading Guide]]. +- `docs/mimir-program-composition.md` is the current program-output authority + map: Muninn observes on Starfire/Nightwing/Raven-class hosts, Mimir consumes + selected streams and owns composition, Eve lowers controls/previews/stats, + Yggdrasil publishes to the site, and OBS is compatibility-only. +- `src/Mimir.Runtime/Synchronization` now has typed program contracts for + `mimir.program_scene.v1`, `mimir.program_output.v1`, and + `mimir.eve_operator_surface.v1`; `Mimir.BufferSmoke + --perfect-machine-contract-smoke` writes them into CultCache. - Local six-camera ingest should use direct driver adapters. Process-backed sources are bridge/network edges only. - Leap stereo IR is the first timing-camera candidate for direct ingest. @@ -45,7 +57,8 @@ Get-Content .\state\evidence.jsonl -Tail 8 - Fensalir owns production GPU fusion, UI, and Spout2 publication. - Faust/native DSP owns hot audio alignment, separation, spatialization, and synchronized stems. -- OBS receives final program surfaces; it does not own synchronization. +- OBS compatibility adapters may receive final program surfaces; they do not + own synchronization, composition, preview/control, or publication. ## Current Pressure @@ -57,8 +70,16 @@ Get-Content .\state\evidence.jsonl -Tail 8 variable-rate resampling. - Bind Fensalir UI to stream health, buffer depth, timestamps, settings, and output management. -- Keep PowerShell/FFmpeg/SRT as bridge utilities until native program output - replaces them. +- Keep PowerShell/FFmpeg/SRT as bridge utilities until native Mimir program + output, Eve operation, and Yggdrasil site publication replace them. +- Yggdrasil currently runs the CultMesh media relay on UDP `3075` from + `/opt/gamecult/mimir-cultmesh-media/Mimir.CultMeshMedia`; Starfire currently + runs the receiver PID from + `artifacts/mimir-cultmesh-media-win-x64-selfcontained/Mimir.CultMeshMedia.exe` + and writes `raven-primary-av` to `udp://127.0.0.1:5200`. OBS source + `Raven Monitor + Realtek` has been repointed from old SRT to that local UDP + endpoint. Raven SSH timed out, so launch + `scripts/start-raven-cultmesh-av-sender.ps1` on Raven directly. - Kiyo Pro has two UVC extension units. Moving it to a motherboard USB3 port changed the descriptor path to `root_hub30` / `bcdUSB=0x0320` and exposed 720p/1080p YUY2/MJPG/H264/NV12 at 60 fps, but Windows still reports diff --git a/src/Mimir.BufferSmoke/Program.cs b/src/Mimir.BufferSmoke/Program.cs index fe3617b..e9e9ae4 100644 --- a/src/Mimir.BufferSmoke/Program.cs +++ b/src/Mimir.BufferSmoke/Program.cs @@ -1230,7 +1230,7 @@ static int RunPerfectMachineProfileSmoke() var computePlans = MimirComputeOffloadConfigurations.BuiltIn; var assemblyPlans = MimirMachineAssemblyPlans.BuiltIn; var captureProfiles = MimirNativeCaptureConfigurations.BuiltIn; - var publications = MimirObsPublicationConfigurations.BuiltIn; + var publications = MimirProgramPublicationConfigurations.BuiltIn; var moduleCatalog = MimirModuleLibrary.Entries; var languageProfiles = MimirBioacousticLanguageConfigurations.BuiltIn; var pathLearningProfiles = MimirAcousticPathLearningConfigurations.BuiltIn; @@ -1261,6 +1261,15 @@ static int RunPerfectMachineProfileSmoke() "scarlett-host-mic-actuator", MimirAlignmentActuatorProfile.SixSourceFaust.Id, command); + var scene = MimirCultMeshContractFactory.CreateObsSceneMirror("mimir-current-program"); + var output = MimirCultMeshContractFactory.CreateProgramOutput( + "mimir-site-program", + scene.SceneId, + MimirProgramPublicationConfigurations.YggdrasilSiteProgram); + var eveSurface = MimirCultMeshContractFactory.CreateOperatorSurface( + "mimir-eve-gui-compositor", + scene.SceneId, + MimirProgramPublicationConfigurations.OperatorSurfaces[0]); var observations = new[] { new MimirBioacousticWordObservation(0, MimirBioacousticTimeline.Default.EventForIndex(0).StartSeconds * MimirBioacousticTimeline.SampleRate + 317.375, 0.95), @@ -1342,6 +1351,8 @@ static int RunPerfectMachineProfileSmoke() $"perfect-machine-codebook id={codebook.CodebookId} motifs={codebook.Motifs.Length} decoder={decoder.Configuration.Id} augmentations={decoder.Configuration.TemplateAugmentations.Length}"); Console.WriteLine( $"perfect-machine-actuator source={actuator.SourceId} delay={actuator.TargetDelaySamples:0.000} ratio={actuator.ResampleRatio:0.000000000} controls={actuator.FaustControls.Length}"); + Console.WriteLine( + $"perfect-machine-program scene={scene.SceneId} layers={scene.Layers.Length} output={output.OutputId} route={output.PublisherRoute} eveCommands={eveSurface.CommandTopics.Length}"); Console.WriteLine( $"perfect-machine-clock anchors={clock?.AnchorCount ?? 0} coverage={clock?.AnchorCoverage ?? 0:0.000} offset={clock?.SourceOffsetSamples ?? double.NaN:0.000} confidence={clock?.Confidence ?? 0:0.000}"); Console.WriteLine( @@ -1530,6 +1541,19 @@ static async Task RunPerfectMachineContractSmokeAsync(string outputPath) MimirAlignmentActuatorProfile.SixSourceFaust.Id, command, createdAt); + var scene = MimirCultMeshContractFactory.CreateObsSceneMirror( + "mimir-current-program", + createdAt); + var output = MimirCultMeshContractFactory.CreateProgramOutput( + "mimir-yggdrasil-site-program", + scene.SceneId, + MimirProgramPublicationConfigurations.YggdrasilSiteProgram, + createdAt); + var eveSurface = MimirCultMeshContractFactory.CreateOperatorSurface( + "mimir-eve-gui-compositor", + scene.SceneId, + MimirProgramPublicationConfigurations.OperatorSurfaces[0], + createdAt); Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(outputPath)) ?? "."); using var cache = await CultCacheMessagePack.OpenAsync(outputPath, new CultCacheOpenOptions @@ -1552,10 +1576,22 @@ static async Task RunPerfectMachineContractSmokeAsync(string outputPath) actuator, new CultRecordHandle(new CultRecordKey($"mimir-actuator:{actuator.ActuatorId}"))) .ConfigureAwait(false); + await cache.UpsertAsync( + scene, + new CultRecordHandle(new CultRecordKey($"mimir-program-scene:{scene.SceneId}"))) + .ConfigureAwait(false); + await cache.UpsertAsync( + output, + new CultRecordHandle(new CultRecordKey($"mimir-program-output:{output.OutputId}"))) + .ConfigureAwait(false); + await cache.UpsertAsync( + eveSurface, + new CultRecordHandle(new CultRecordKey($"mimir-eve-surface:{eveSurface.SurfaceId}"))) + .ConfigureAwait(false); await cache.FlushAsync().ConfigureAwait(false); Console.WriteLine( - $"perfect-machine-contract-smoke path={outputPath} codebookMotifs={codebook.Motifs.Length} decoder={decoder.Configuration.Id} pathBands={pathState.BandResponses.Length} actuatorControls={actuator.FaustControls.Length}"); + $"perfect-machine-contract-smoke path={outputPath} codebookMotifs={codebook.Motifs.Length} decoder={decoder.Configuration.Id} pathBands={pathState.BandResponses.Length} actuatorControls={actuator.FaustControls.Length} sceneLayers={scene.Layers.Length} programOutput={output.OutputId} eveCommands={eveSurface.CommandTopics.Length}"); return 0; } diff --git a/src/Mimir.Runtime/Synchronization/MimirComputeOffloadConfiguration.cs b/src/Mimir.Runtime/Synchronization/MimirComputeOffloadConfiguration.cs index 5b63587..35f3f92 100644 --- a/src/Mimir.Runtime/Synchronization/MimirComputeOffloadConfiguration.cs +++ b/src/Mimir.Runtime/Synchronization/MimirComputeOffloadConfiguration.cs @@ -11,7 +11,8 @@ public enum MimirWorkloadKind CameraCapture, VisualFeatureExtraction, VisualFusion, - ObsPublication + ProgramComposition, + ProgramPublication } public sealed record MimirWorkloadPlacement( @@ -42,7 +43,8 @@ public static class MimirComputeOffloadConfigurations new(MimirWorkloadKind.CameraCapture, MimirComputeBackend.NativeSimd, "native capture workers", CanRunRemote: true, RequiresRealtime: true, "Drivers own device reads and timestamps."), new(MimirWorkloadKind.VisualFeatureExtraction, MimirComputeBackend.D3D12Compute, "Fensalir", CanRunRemote: false, RequiresRealtime: true, "GPU owns dense feature work."), new(MimirWorkloadKind.VisualFusion, MimirComputeBackend.D3D12Compute, "Fensalir", CanRunRemote: false, RequiresRealtime: true, "Fensalir owns temporal evidence field."), - new(MimirWorkloadKind.ObsPublication, MimirComputeBackend.D3D12Compute, "OBS/Fensalir output", CanRunRemote: false, RequiresRealtime: true, "OBS receives program surfaces, not sync authority.") + new(MimirWorkloadKind.ProgramComposition, MimirComputeBackend.D3D12Compute, "Mimir + Fensalir", CanRunRemote: false, RequiresRealtime: true, "Mimir owns the scene graph; Fensalir lowers it into program pixels."), + new(MimirWorkloadKind.ProgramPublication, MimirComputeBackend.D3D12Compute, "Mimir + Yggdrasil publisher", CanRunRemote: false, RequiresRealtime: true, "The site publisher consumes the Mimir program output; OBS is only an optional compatibility sink.") ]); public static MimirComputeOffloadConfiguration DistributedWitnesses { get; } = new( diff --git a/src/Mimir.Runtime/Synchronization/MimirCultMeshContracts.cs b/src/Mimir.Runtime/Synchronization/MimirCultMeshContracts.cs index 2298b7b..3276d43 100644 --- a/src/Mimir.Runtime/Synchronization/MimirCultMeshContracts.cs +++ b/src/Mimir.Runtime/Synchronization/MimirCultMeshContracts.cs @@ -104,6 +104,73 @@ public sealed record MimirFaustControlSnapshot( [property: Key(0)] string Path, [property: Key(1)] float Value); +[CultDocument("mimir.program_scene", "mimir.program_scene.v1")] +[MessagePackObject] +public sealed record MimirProgramSceneDocument( + [property: Key(0)] + [property: CultName] + string SceneId, + [property: Key(1)] string UpdatedAtUtc, + [property: Key(2)] int CanvasWidth, + [property: Key(3)] int CanvasHeight, + [property: Key(4)] string Owner, + [property: Key(5)] MimirProgramSceneLayer[] Layers); + +[MessagePackObject] +public sealed record MimirProgramSceneLayer( + [property: Key(0)] string LayerId, + [property: Key(1)] string SourceRef, + [property: Key(2)] string SourceKind, + [property: Key(3)] bool Visible, + [property: Key(4)] double X, + [property: Key(5)] double Y, + [property: Key(6)] double Width, + [property: Key(7)] double Height, + [property: Key(8)] int ZIndex, + [property: Key(9)] MimirProgramCrop Crop, + [property: Key(10)] MimirProgramChromaKey? ChromaKey); + +[MessagePackObject] +public sealed record MimirProgramCrop( + [property: Key(0)] double Left, + [property: Key(1)] double Top, + [property: Key(2)] double Right, + [property: Key(3)] double Bottom); + +[MessagePackObject] +public sealed record MimirProgramChromaKey( + [property: Key(0)] uint KeyColorRgba, + [property: Key(1)] double Similarity, + [property: Key(2)] double Smoothness, + [property: Key(3)] double Spill); + +[CultDocument("mimir.program_output", "mimir.program_output.v1")] +[MessagePackObject] +public sealed record MimirProgramOutputDocument( + [property: Key(0)] + [property: CultName] + string OutputId, + [property: Key(1)] string UpdatedAtUtc, + [property: Key(2)] string SceneId, + [property: Key(3)] string VideoSurface, + [property: Key(4)] string AudioBus, + [property: Key(5)] string PublisherRoute, + [property: Key(6)] bool DiagnosticOnly); + +[CultDocument("mimir.eve_operator_surface", "mimir.eve_operator_surface.v1")] +[MessagePackObject] +public sealed record MimirEveOperatorSurfaceDocument( + [property: Key(0)] + [property: CultName] + string SurfaceId, + [property: Key(1)] string UpdatedAtUtc, + [property: Key(2)] string SceneId, + [property: Key(3)] bool CanPreviewProgram, + [property: Key(4)] bool CanEditComposition, + [property: Key(5)] bool CanSelectSources, + [property: Key(6)] bool CanShowStats, + [property: Key(7)] string[] CommandTopics); + public static class MimirCultMeshContractFactory { public static MimirBioacousticCodebookState CreateCodebookState( @@ -175,6 +242,92 @@ public static class MimirCultMeshContractFactory .Select(pair => new MimirFaustControlSnapshot(pair.Key, pair.Value)) .ToArray()); + public static MimirProgramSceneDocument CreateObsSceneMirror( + string sceneId, + DateTimeOffset? updatedAt = null) => + new( + sceneId, + (updatedAt ?? DateTimeOffset.UtcNow).ToString("O"), + CanvasWidth: 1920, + CanvasHeight: 1080, + Owner: "Mimir", + Layers: + [ + new( + "starfire-monitor", + "muninn:starfire:monitor:primary", + "monitor", + Visible: true, + X: 0, + Y: 0, + Width: 1920, + Height: 1080, + ZIndex: 0, + new MimirProgramCrop(Left: 0, Top: 0, Right: 0, Bottom: 48), + ChromaKey: null), + new( + "discord-chat", + "muninn:starfire:window:discord", + "window", + Visible: true, + X: 1225.5, + Y: 0.5, + Width: 1012.2, + Height: 569.4, + ZIndex: 10, + new MimirProgramCrop(Left: 350, Top: 38, Right: 271, Bottom: 87), + new MimirProgramChromaKey(0xff2b7c5a, Similarity: 1, Smoothness: 2, Spill: 1)), + new( + "raven-monitor", + "muninn:raven:monitor:primary", + "network-monitor", + Visible: true, + X: 0, + Y: 0, + Width: 2560, + Height: 1440, + ZIndex: 20, + new MimirProgramCrop(Left: 0, Top: 0, Right: 0, Bottom: 0), + ChromaKey: null) + ]); + + public static MimirProgramOutputDocument CreateProgramOutput( + string outputId, + string sceneId, + MimirProgramPublicationConfiguration publication, + DateTimeOffset? updatedAt = null) => + new( + outputId, + (updatedAt ?? DateTimeOffset.UtcNow).ToString("O"), + sceneId, + publication.VideoSurfaceName, + publication.AudioKind.ToString(), + publication.SitePublisherRoute, + publication.DiagnosticOnly); + + public static MimirEveOperatorSurfaceDocument CreateOperatorSurface( + string surfaceId, + string sceneId, + MimirProgramControlSurface surface, + DateTimeOffset? updatedAt = null) => + new( + surfaceId, + (updatedAt ?? DateTimeOffset.UtcNow).ToString("O"), + sceneId, + surface.CanPreviewProgram, + surface.CanEditComposition, + surface.CanSelectSources, + surface.CanShowStats, + CommandTopics: + [ + "mimir.program.scene.select", + "mimir.program.layer.transform", + "mimir.program.layer.crop", + "mimir.program.layer.chroma_key", + "mimir.program.source.subscribe", + "mimir.program.output.publish" + ]); + private static MimirBioacousticDecoderConfigurationSnapshot Snapshot(MimirBioacousticDecoderConfiguration configuration) => new( configuration.Id, diff --git a/src/Mimir.Runtime/Synchronization/MimirModuleLibrary.cs b/src/Mimir.Runtime/Synchronization/MimirModuleLibrary.cs index d016807..b09d387 100644 --- a/src/Mimir.Runtime/Synchronization/MimirModuleLibrary.cs +++ b/src/Mimir.Runtime/Synchronization/MimirModuleLibrary.cs @@ -111,10 +111,10 @@ public static class MimirModuleLibrary MimirMachineAssemblyPlans.BuiltIn.Select(plan => plan.Id).ToArray(), ["dotnet run --project .\\src\\Mimir.BufferSmoke\\Mimir.BufferSmoke.csproj -- --perfect-machine-profile-smoke"]), new( - "obs-publication", - "Fensalir + Faust + OBS", - "OBS-facing program surface and separately mixable audio stem configurations.", - MimirObsPublicationConfigurations.BuiltIn.Select(profile => profile.Id).ToArray(), + "program-publication", + "Mimir + Fensalir + Faust + Eve + Yggdrasil", + "Mimir-owned program composition, Eve operator surfaces, Yggdrasil site publication, and temporary OBS compatibility adapters.", + MimirProgramPublicationConfigurations.BuiltIn.Select(profile => profile.Id).ToArray(), ["dotnet run --project .\\src\\Mimir.BufferSmoke\\Mimir.BufferSmoke.csproj -- --perfect-machine-profile-smoke"]) ]; } diff --git a/src/Mimir.Runtime/Synchronization/MimirObsPublicationConfiguration.cs b/src/Mimir.Runtime/Synchronization/MimirObsPublicationConfiguration.cs deleted file mode 100644 index 35a8edc..0000000 --- a/src/Mimir.Runtime/Synchronization/MimirObsPublicationConfiguration.cs +++ /dev/null @@ -1,83 +0,0 @@ -namespace Mimir.Runtime.Synchronization; - -public enum MimirObsVideoPublicationKind -{ - Spout2, - SharedD3D12Texture, - SrtDiagnosticBridge, - WindowCaptureDiagnostic -} - -public enum MimirObsAudioPublicationKind -{ - FaustStemBus, - AsioOutputPair, - WasapiLoopbackDiagnostic, - SrtDiagnosticBridge -} - -public sealed record MimirObsAudioStem( - string Id, - string DisplayName, - int ChannelCount, - bool UserMixable, - bool CarriesTimingWitness, - string Notes); - -public sealed record MimirObsPublicationConfiguration( - string Id, - string Description, - MimirObsVideoPublicationKind VideoKind, - string VideoSourceName, - MimirObsAudioPublicationKind AudioKind, - IReadOnlyList AudioStems, - double TargetPresentationDelaySeconds, - bool DiagnosticOnly); - -public static class MimirObsPublicationConfigurations -{ - public static IReadOnlyList ProgramStems { get; } = - [ - new("host_voice", "Host voice", 1, UserMixable: true, CarriesTimingWitness: false, "Primary Starfire dialogue stem."), - new("co_streamer_voice", "Co-streamer voice", 1, UserMixable: true, CarriesTimingWitness: false, "Raven dialogue after network clock fit."), - new("ambient", "Ambient room", 2, UserMixable: true, CarriesTimingWitness: false, "Room bed after suppression/separation."), - new("transients", "Transients", 2, UserMixable: true, CarriesTimingWitness: false, "Keyboard, desk, impacts, and short events."), - new("local_loopback", "Local loopback", 2, UserMixable: true, CarriesTimingWitness: true, "Starfire program loopback and watermark witness."), - new("remote_loopback", "Remote loopback", 2, UserMixable: true, CarriesTimingWitness: true, "Raven program loopback and network-delay witness."), - new("spatial_bed", "Spatial bed", 4, UserMixable: true, CarriesTimingWitness: false, "Encoded volumetric ambience or ambisonic fold.") - ]; - - public static MimirObsPublicationConfiguration NativeProgram { get; } = new( - "native-program-spout-faust", - "Production target: Fensalir publishes synchronized video, Faust publishes aligned stems.", - MimirObsVideoPublicationKind.Spout2, - "Mimir Point Cloud", - MimirObsAudioPublicationKind.FaustStemBus, - ProgramStems, - TargetPresentationDelaySeconds: 5.0, - DiagnosticOnly: false); - - public static MimirObsPublicationConfiguration TextureInteropProgram { get; } = NativeProgram with - { - Id = "native-program-d3d12-shared-texture", - VideoKind = MimirObsVideoPublicationKind.SharedD3D12Texture, - VideoSourceName = "Mimir D3D12 Program Texture" - }; - - public static MimirObsPublicationConfiguration DiagnosticSrtBridge { get; } = new( - "diagnostic-srt-bridge", - "OBS-facing fallback for bridge tests only. It does not own synchronized program truth.", - MimirObsVideoPublicationKind.SrtDiagnosticBridge, - "Mimir Diagnostic SRT", - MimirObsAudioPublicationKind.SrtDiagnosticBridge, - ProgramStems.Where(stem => stem.Id is "host_voice" or "local_loopback").ToArray(), - TargetPresentationDelaySeconds: 1.5, - DiagnosticOnly: true); - - public static IReadOnlyList BuiltIn { get; } = - [ - NativeProgram, - TextureInteropProgram, - DiagnosticSrtBridge - ]; -} diff --git a/src/Mimir.Runtime/Synchronization/MimirPerfectMachineManifest.cs b/src/Mimir.Runtime/Synchronization/MimirPerfectMachineManifest.cs index 4e20d8f..fe4de73 100644 --- a/src/Mimir.Runtime/Synchronization/MimirPerfectMachineManifest.cs +++ b/src/Mimir.Runtime/Synchronization/MimirPerfectMachineManifest.cs @@ -20,7 +20,7 @@ public sealed record MimirPerfectMachineManifest( IReadOnlyList AudioFields, IReadOnlyList VisualFields, IReadOnlyList ComputePlans, - IReadOnlyList Publications, + IReadOnlyList Publications, IReadOnlyList AssemblyPlans); public static class MimirPerfectMachineManifestFactory @@ -48,6 +48,6 @@ public static class MimirPerfectMachineManifestFactory MimirAudioFieldConfigurations.BuiltIn, MimirVisualFusionConfigurations.BuiltIn, MimirComputeOffloadConfigurations.BuiltIn, - MimirObsPublicationConfigurations.BuiltIn, + MimirProgramPublicationConfigurations.BuiltIn, MimirMachineAssemblyPlans.BuiltIn); } diff --git a/src/Mimir.Runtime/Synchronization/MimirPerfectMachineProfiles.cs b/src/Mimir.Runtime/Synchronization/MimirPerfectMachineProfiles.cs index 9a09926..d872b9b 100644 --- a/src/Mimir.Runtime/Synchronization/MimirPerfectMachineProfiles.cs +++ b/src/Mimir.Runtime/Synchronization/MimirPerfectMachineProfiles.cs @@ -4,10 +4,11 @@ public enum MimirPerfectMachineNodeRole { StarfireAuthority, RavenRemote, + MuninnHost, PhoneWitness, MicrocontrollerWitness, CalibrationBench, - ObsProgramHost + MimirProgramPublisher } public enum MimirAudioFieldModel @@ -38,7 +39,7 @@ public sealed record MimirPerfectMachineProfile( IReadOnlyList ComputeBackends, bool EmitsWitness, bool OwnsCanonicalClock, - bool PublishesObsProgram, + bool PublishesProgram, TimeSpan RollingWindow, string[] RequiredCultMeshDocuments); @@ -46,7 +47,7 @@ public static class MimirPerfectMachineProfiles { public static MimirPerfectMachineProfile StarfireAuthority { get; } = new( "starfire-authority", - "Heavy local workstation: ASIO loopback authority, runtime buffers, calibration truth, Fensalir/Faust lowering, OBS-facing program output.", + "Heavy local workstation: ASIO loopback authority, runtime buffers, calibration truth, Mimir composition, Fensalir/Faust lowering, Eve operator surfaces, and program publication.", MimirPerfectMachineNodeRole.StarfireAuthority, MimirBioacousticDecoderConfiguration.BaselineMfccIndex, MimirAlignmentActuatorProfile.SixSourceFaust, @@ -54,7 +55,7 @@ public static class MimirPerfectMachineProfiles [MimirComputeBackend.ManagedScalar, MimirComputeBackend.NativeSimd, MimirComputeBackend.D3D12Compute, MimirComputeBackend.FaustNativeDsp], EmitsWitness: true, OwnsCanonicalClock: true, - PublishesObsProgram: true, + PublishesProgram: true, RollingWindow: TimeSpan.FromSeconds(5), RequiredCultMeshDocuments: [ @@ -74,7 +75,7 @@ public static class MimirPerfectMachineProfiles [MimirComputeBackend.ManagedScalar, MimirComputeBackend.RemoteCultMeshWorker], EmitsWitness: false, OwnsCanonicalClock: false, - PublishesObsProgram: false, + PublishesProgram: false, RollingWindow: TimeSpan.FromSeconds(5), RequiredCultMeshDocuments: [ @@ -93,7 +94,7 @@ public static class MimirPerfectMachineProfiles [MimirComputeBackend.ManagedScalar, MimirComputeBackend.RemoteCultMeshWorker], EmitsWitness: false, OwnsCanonicalClock: false, - PublishesObsProgram: false, + PublishesProgram: false, RollingWindow: TimeSpan.FromSeconds(3), RequiredCultMeshDocuments: [ @@ -111,7 +112,7 @@ public static class MimirPerfectMachineProfiles [MimirComputeBackend.RemoteCultMeshWorker], EmitsWitness: false, OwnsCanonicalClock: false, - PublishesObsProgram: false, + PublishesProgram: false, RollingWindow: TimeSpan.FromSeconds(2), RequiredCultMeshDocuments: [ @@ -129,7 +130,7 @@ public static class MimirPerfectMachineProfiles [MimirComputeBackend.ManagedScalar, MimirComputeBackend.NativeSimd, MimirComputeBackend.D3D12Compute], EmitsWitness: true, OwnsCanonicalClock: false, - PublishesObsProgram: false, + PublishesProgram: false, RollingWindow: TimeSpan.FromSeconds(8), RequiredCultMeshDocuments: [ @@ -138,22 +139,44 @@ public static class MimirPerfectMachineProfiles "mimir.acoustic_path_state.v1" ]); - public static MimirPerfectMachineProfile ObsProgramHost { get; } = new( - "obs-program-host", - "Program-output profile: consumes aligned stems/spatial bed and publishes OBS-facing surfaces without becoming sync authority.", - MimirPerfectMachineNodeRole.ObsProgramHost, + public static MimirPerfectMachineProfile MuninnHost { get; } = new( + "muninn-host", + "Capture host profile: Muninn observes local screens, windows, loopbacks, cameras, and device health, publishes typed stream capabilities/media, and does not compose the final program.", + MimirPerfectMachineNodeRole.MuninnHost, + MimirBioacousticDecoderConfiguration.CompactFastIndex, + MimirAlignmentActuatorProfile.SixSourceFaust, + MimirAudioFieldModel.AlignedStems, + [MimirComputeBackend.ManagedScalar, MimirComputeBackend.RemoteCultMeshWorker], + EmitsWitness: false, + OwnsCanonicalClock: false, + PublishesProgram: false, + RollingWindow: TimeSpan.FromSeconds(5), + RequiredCultMeshDocuments: + [ + "muninn.telemetry_surface.v1", + "muninn.capture_stream.v1", + "mimir.cultmesh_media_frame.v1" + ]); + + public static MimirPerfectMachineProfile MimirProgramPublisher { get; } = new( + "mimir-program-publisher", + "Program-output profile: consumes selected Muninn/Mimir streams, owns scene composition and Eve controls, and publishes the program locally and through Yggdrasil without OBS authority.", + MimirPerfectMachineNodeRole.MimirProgramPublisher, MimirBioacousticDecoderConfiguration.BaselineMfccIndex, MimirAlignmentActuatorProfile.SixSourceFaust, MimirAudioFieldModel.SourceBasedSpatialBus, [MimirComputeBackend.FaustNativeDsp, MimirComputeBackend.D3D12Compute], EmitsWitness: false, OwnsCanonicalClock: false, - PublishesObsProgram: true, + PublishesProgram: true, RollingWindow: TimeSpan.FromSeconds(5), RequiredCultMeshDocuments: [ "mimir.acoustic_path_state.v1", - "mimir.actuator_state.v1" + "mimir.actuator_state.v1", + "mimir.program_scene.v1", + "mimir.program_output.v1", + "mimir.eve_operator_surface.v1" ]); public static IReadOnlyList All { get; } = @@ -163,6 +186,7 @@ public static class MimirPerfectMachineProfiles PhoneWitness, MicrocontrollerWitness, CalibrationBench, - ObsProgramHost + MuninnHost, + MimirProgramPublisher ]; } diff --git a/src/Mimir.Runtime/Synchronization/MimirProgramPublicationConfiguration.cs b/src/Mimir.Runtime/Synchronization/MimirProgramPublicationConfiguration.cs new file mode 100644 index 0000000..5dafc6d --- /dev/null +++ b/src/Mimir.Runtime/Synchronization/MimirProgramPublicationConfiguration.cs @@ -0,0 +1,159 @@ +namespace Mimir.Runtime.Synchronization; + +public enum MimirProgramVideoPublicationKind +{ + FensalirProgramTexture, + SharedD3D12Texture, + CultMeshMediaBody, + SrtDiagnosticBridge, + ObsCompatibilityAdapter +} + +public enum MimirProgramAudioPublicationKind +{ + FaustStemBus, + AsioOutputPair, + CultMeshMediaBody, + WasapiLoopbackDiagnostic, + ObsCompatibilityAdapter +} + +public enum MimirProgramControlPublicationKind +{ + EveGui, + EveTui, + CultMeshCommandSurface +} + +public sealed record MimirProgramAudioStem( + string Id, + string DisplayName, + int ChannelCount, + bool UserMixable, + bool CarriesTimingWitness, + string Notes); + +public sealed record MimirProgramControlSurface( + string Id, + string Description, + MimirProgramControlPublicationKind Kind, + bool CanPreviewProgram, + bool CanEditComposition, + bool CanSelectSources, + bool CanShowStats); + +public sealed record MimirProgramPublicationConfiguration( + string Id, + string Description, + MimirProgramVideoPublicationKind VideoKind, + string VideoSurfaceName, + MimirProgramAudioPublicationKind AudioKind, + IReadOnlyList AudioStems, + IReadOnlyList ControlSurfaces, + bool PublishesSiteFeed, + string SitePublisherRoute, + double TargetPresentationDelaySeconds, + bool DiagnosticOnly); + +public static class MimirProgramPublicationConfigurations +{ + public static IReadOnlyList ProgramStems { get; } = + [ + new("host_voice", "Host voice", 1, UserMixable: true, CarriesTimingWitness: false, "Primary Starfire dialogue stem."), + new("co_streamer_voice", "Co-streamer voice", 1, UserMixable: true, CarriesTimingWitness: false, "Raven dialogue after network clock fit."), + new("ambient", "Ambient room", 2, UserMixable: true, CarriesTimingWitness: false, "Room bed after suppression/separation."), + new("transients", "Transients", 2, UserMixable: true, CarriesTimingWitness: false, "Keyboard, desk, impacts, and short events."), + new("local_loopback", "Local loopback", 2, UserMixable: true, CarriesTimingWitness: true, "Starfire program loopback and watermark witness."), + new("remote_loopback", "Remote loopback", 2, UserMixable: true, CarriesTimingWitness: true, "Raven program loopback and network-delay witness."), + new("spatial_bed", "Spatial bed", 4, UserMixable: true, CarriesTimingWitness: false, "Encoded volumetric ambience or ambisonic fold.") + ]; + + public static IReadOnlyList OperatorSurfaces { get; } = + [ + new( + "eve-gui-compositor", + "Primary Eve GUI composition surface: preview, source selection, transforms, crop/key controls, audio stem controls, and stream health.", + MimirProgramControlPublicationKind.EveGui, + CanPreviewProgram: true, + CanEditComposition: true, + CanSelectSources: true, + CanShowStats: true), + new( + "eve-tui-operator", + "Compact Eve TUI operator surface for source health, live stats, scene selection, and emergency command paths.", + MimirProgramControlPublicationKind.EveTui, + CanPreviewProgram: false, + CanEditComposition: false, + CanSelectSources: true, + CanShowStats: true), + new( + "cultmesh-command-surface", + "Typed CultMesh command surface consumed by Eve lowerers and trusted devices; Mimir remains the scene authority.", + MimirProgramControlPublicationKind.CultMeshCommandSurface, + CanPreviewProgram: false, + CanEditComposition: true, + CanSelectSources: true, + CanShowStats: true) + ]; + + public static MimirProgramPublicationConfiguration NativeProgram { get; } = new( + "mimir-native-program", + "Production target: Mimir owns composition, Fensalir publishes synchronized program video, Faust publishes aligned stems, and Eve exposes control/preview/stat surfaces.", + MimirProgramVideoPublicationKind.FensalirProgramTexture, + "Mimir Program Texture", + MimirProgramAudioPublicationKind.FaustStemBus, + ProgramStems, + OperatorSurfaces, + PublishesSiteFeed: false, + SitePublisherRoute: "", + TargetPresentationDelaySeconds: 5.0, + DiagnosticOnly: false); + + public static MimirProgramPublicationConfiguration YggdrasilSiteProgram { get; } = NativeProgram with + { + Id = "mimir-yggdrasil-site-program", + Description = "Site publishing target: a Yggdrasil-facing daemon consumes the same Mimir program output and publishes it to the public site without owning a second composition.", + VideoKind = MimirProgramVideoPublicationKind.CultMeshMediaBody, + AudioKind = MimirProgramAudioPublicationKind.CultMeshMediaBody, + PublishesSiteFeed = true, + SitePublisherRoute = "cultmesh://yggdrasil/gamecult.site/mimir/program" + }; + + public static MimirProgramPublicationConfiguration TextureInteropProgram { get; } = NativeProgram with + { + Id = "native-program-d3d12-shared-texture", + VideoKind = MimirProgramVideoPublicationKind.SharedD3D12Texture, + VideoSurfaceName = "Mimir D3D12 Program Texture" + }; + + public static MimirProgramPublicationConfiguration ObsCompatibilityAdapter { get; } = new( + "obs-compatibility-adapter", + "Temporary OBS compatibility sink for preview/testing while Mimir owns composition and Yggdrasil/site publication matures.", + MimirProgramVideoPublicationKind.ObsCompatibilityAdapter, + "Mimir OBS Compatibility Feed", + MimirProgramAudioPublicationKind.ObsCompatibilityAdapter, + ProgramStems.Where(stem => stem.Id is "host_voice" or "local_loopback").ToArray(), + OperatorSurfaces, + PublishesSiteFeed: false, + SitePublisherRoute: "", + TargetPresentationDelaySeconds: 1.5, + DiagnosticOnly: true); + + public static MimirProgramPublicationConfiguration DiagnosticSrtBridge { get; } = ObsCompatibilityAdapter with + { + Id = "diagnostic-srt-bridge", + Description = "Legacy SRT bridge for bridge tests only. It is not scene, composition, or broadcast authority.", + VideoKind = MimirProgramVideoPublicationKind.SrtDiagnosticBridge, + AudioKind = MimirProgramAudioPublicationKind.WasapiLoopbackDiagnostic, + VideoSurfaceName = "Mimir Diagnostic SRT" + }; + + public static IReadOnlyList BuiltIn { get; } = + [ + NativeProgram, + YggdrasilSiteProgram, + TextureInteropProgram, + ObsCompatibilityAdapter, + DiagnosticSrtBridge + ]; +} diff --git a/src/Mimir.Runtime/Synchronization/MimirReservoirConfiguration.cs b/src/Mimir.Runtime/Synchronization/MimirReservoirConfiguration.cs index 86760d3..7a6a8ee 100644 --- a/src/Mimir.Runtime/Synchronization/MimirReservoirConfiguration.cs +++ b/src/Mimir.Runtime/Synchronization/MimirReservoirConfiguration.cs @@ -33,7 +33,7 @@ public static class MimirReservoirConfigurations new("phase_claim", "audio phase/group-delay evidence", "Mimir.Runtime", RetainsHistory: false, "Calibration/normalization input."), new("event_claim", "timeline anchor", "Mimir.Runtime", RetainsHistory: false, "Bioacoustic/passive/calibration events."), new("surface_claim", "spatial surface observation", "Fensalir", RetainsHistory: false, "Fusion candidate."), - new("render_packet", "OBS-facing packet", "Fensalir/Faust", RetainsHistory: false, "Output-ready view.") + new("render_packet", "Mimir program packet", "Mimir/Fensalir/Faust", RetainsHistory: false, "Output-ready program view.") ]; public static MimirReservoirConfiguration ManagedRuntime { get; } = new( diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 0150da4..113d935 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -119,3 +119,6 @@ {"ts": "2026-05-25T13:56:03+01:00", "type": "decision", "status": "ok", "note": "Fensalir spectrum visualization authority split: Mimir now emits AquariumBufferFieldFrame spline tube fields as the real buffer-domain contract, with tangent/curvature/normal/derivative appearance and probe policy for future compute SDF splat reservoir lowering. AquariumSplineFrame remains only the immediate Catmull-Rom preview witness until compute lowering owns rendering."} {"ts": "2026-05-25T15:55:00+01:00", "type": "decision", "status": "ok", "note": "Fensalir/Mimir spectrum visualization moved from C# spline authority to DSL-authored texture field authority. config/mimir-spectrum-field.aquafield binds a rolling spectrum texture, declares surface graph nodes, and asks for texture-dimension splinefield lowering. Fensalir now compiles Aquarium field DSL text, uploads texture samples/program packets, and ProgramMode=3 in D3D12FractalReservoirCompute emits stochastic reservoir splats from the texture field. Current canary is mechanically through the reservoir at 131072 splats with low CPU record time, but visual framing/material is still not the final beautiful spectrum surface."} {"ts": "2026-05-25T17:43:25+01:00", "type": "decision", "status": "ok", "note": "Fensalir direct splinefield correction: texture-derived tube trails are direct SDF envelope surfaces, not point-splat fractal dust. AquariumFieldScriptCompiler supports direct; D3D12Renderer now derives AquariumSplineFrame tube-envelope trails from BufferFieldFrame texture spline programs, the spline pipeline declares triangle topology, and D3D12SplinePS writes color, metadata/control, and reservoirGuide outputs. Mimir capture artifacts/runtime/mimir-direct-sdf-tube-spectrum.png proves the DSL-authored ASIO spectrum appears through the direct SDF tube path with ~0.64 ms record CPU in the canary."} +{"ts": "2026-06-08T09:29:43+01:00", "type": "decision", "status": "ok", "note": "Added CultMesh media bridge for Raven program feed: Raven sender publishes rolling mimir.cultmesh_media_frame MPEG-TS chunks over CultNet reliable UDP to Yggdrasil 10.77.0.1:3075; Starfire receiver lowers raven-primary-av to local UDP 127.0.0.1:5200 for OBS. Yggdrasil relay and Starfire receiver were started; Raven SSH timed out, so sender must be launched on Raven directly."} +{"ts": "2026-06-08T09:59:22+01:00", "type": "verification", "status": "blocked", "note": "CultMesh media bridge verification: Yggdrasil UFW was opened for 3075/udp on wg-gamecult and Starfire receiver now connects from 10.77.0.2. Synthetic sender on Yggdrasil connects and sends chunks without User Not Verified after CultLib pre-verification schema-v0 policy patch, but relay cache does not receive mimir.cultmesh_media_frame updates and Starfire receiver logs no frames. Remaining blocker is CultNet document put/apply path, not WireGuard routing or OBS endpoint."} +{"ts": "2026-06-09T02:17:25+01:00", "type": "decision", "status": "ok", "note": "Mimir program composition authority moved out of OBS: Muninn hosts publish local stream capabilities/media; Mimir owns selected-stream subscription, scene graph, Eve operator surfaces, and program publication intent; Yggdrasil-facing publisher daemon consumes Mimir output; OBS is compatibility-only. Added typed mimir.program_scene.v1, mimir.program_output.v1, and mimir.eve_operator_surface.v1 contracts and BufferSmoke proof coverage."} diff --git a/state/map.yaml b/state/map.yaml index 0ae93ef..e42d7ec 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -5,7 +5,7 @@ workspace: implementation_name: Mimir public surface / native runtime contracts premise: > Mimir is the local realtime field machine: cameras, microphones, speakers, - network feeds, and timing signals become one coherent OBS-facing volumetric + network feeds, and timing signals become one coherent Mimir-owned volumetric program surface. The live path is a C# Fensalir-hosted app with in-memory rolling buffers and native capture/runtime boundaries. @@ -13,7 +13,7 @@ objective: primary: > Build one synchronized spatial stream machine from six cameras, six microphones, two speakers, Leap timing, Fensalir GPU rendering, Faust DSP, - and OBS output. + Eve operator surfaces, and Yggdrasil site publication. current: > Promote the active bioacoustic packet-song receiver into runtime state: self-identifying song contours should pin multiple time/frequency anchors @@ -22,9 +22,9 @@ objective: constraints: - Sender is Windows. - - Receiver uses OBS. + - OBS is a temporary compatibility sink, not a composition or broadcast authority. - Video encode must use NVENC when the bridge path is active. - - Audio sources must remain separately controllable in OBS. + - Audio sources must remain separately controllable through Mimir/Eve and publishable as stems. - V1 bridge is LAN-only. - Persistent state must clarify rehydrate, not become transcript tax. - Live ingest belongs to Mimir.Runtime, native drivers, Fensalir, and @@ -33,22 +33,31 @@ constraints: memory. - Networked feeds are producers into the live runtime, not independent clock authorities. + - CultMesh/CultNet is the default Raven-to-Starfire media transit. OBS may + consume a Starfire-local UDP MPEG-TS egress as a compatibility sink, but + Mimir owns program selection, composition, preview/control, and publication. - Mimir explicitly trades latency and memory for throughput and field quality: the default five-second buffer is compute budget for stream alignment, GPU/audio analysis, and volumetric sound-field extraction. invariants: - - OBS composition authority stays in OBS. + - Mimir owns composition, preview/control state, source subscription policy, + and program publication intent. - FFmpeg owns capture, encoding, and transport only for bridge utilities. - One stable bridge port equals one OBS source endpoint. - One rolling live window owns the timing edge and expiry. - The rolling window is not accidental latency. It must preserve enough time - to line up feeds and crunch data before OBS-facing output is emitted. + to line up feeds and crunch data before Mimir program output is emitted. - Typed views may index the rolling buffer, but no per-kind ring owns retention. - - Fensalir owns production UI, GPU fusion, D3D12 interop, and Spout2 output. + - Fensalir owns GPU lowering for the Mimir program scene graph, D3D12 interop, + and local program texture output. - Faust/native DSP owns hot audio alignment, separation, spatialization, and stems. + - Eve GUI/TUI surfaces lower Mimir's operator controls, previews, and stats on + any device without owning scene truth. + - A Yggdrasil-facing publisher daemon consumes Mimir program output and + publishes it to the site without owning composition. - Local secrets remain untracked. - Mimir is a CultMesh app. Codebooks, schedules, calibration receipts, path response, and compute-offload work need typed mesh-syncable state surfaces @@ -271,7 +280,7 @@ accepted_design: library distilled from the research notes: node profiles, decoder/language profiles, path learning, benchmark panels, actuator strategies, capture and ingest strategies, reservoir strategies, field profiles, compute offload, - OBS publication, CultMesh contracts, Fensalir lowering, and exportable + program publication, CultMesh contracts, Fensalir lowering, and exportable manifest generation. - `docs/perfect-machine-module-library.md` indexes those reusable runtime modules and their smoke commands. @@ -279,6 +288,18 @@ accepted_design: for the current repo code surface. - `docs/perfect-machine-domain-index.md` indexes Mimir's full problem domain, current evidence taxonomy, hot paths, and implementation cuts. + - `docs/mimir-program-composition.md` owns the current program-composition + authority map: Muninn observes, Mimir composes/controls, Eve lowers operator + surfaces, Yggdrasil publishes, and OBS is compatibility-only. + - `src/Mimir.Runtime/Synchronization` now includes typed program contracts: + `mimir.program_scene.v1`, `mimir.program_output.v1`, and + `mimir.eve_operator_surface.v1`. + - `src/Mimir.CultMeshMedia` owns the first explicit CultMesh media/body + bridge. Raven publishes muxed H.264/AAC MPEG-TS chunks as rolling + `mimir.cultmesh_media_frame` documents over CultNet reliable UDP, + Yggdrasil relays them on `10.77.0.1:3075`, and Starfire lowers the + subscribed stream to `udp://127.0.0.1:5200` for the OBS source + `Raven Monitor + Realtek`. - `research/perfect-machine-study-2026-05-23/` preserves the architecture rumination, optimization ledger, research references, source/hotspot audits, Fensalir/native boundary maps, benchmark plan, calibration-session spec, @@ -305,6 +326,8 @@ active_subgoals: settings, and output management. - Keep PowerShell/FFmpeg/SRT as bridge utilities until direct program output replaces them. + - Launch Raven's CultMesh sender directly on Raven because SSH to Raven over + WireGuard timed out during the initial relay deployment pass. current_status: summary: >