From cb36734aa85c7eba3b5ae1ad769e2077b9449552 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sat, 23 May 2026 23:46:00 +0100 Subject: [PATCH] Index Mimir perfect machine domain --- README.md | 2 + docs/code-algorithm-map.md | 4 + docs/perfect-machine-domain-index.md | 239 ++++++++++++++++++ .../architecture-rumination.md | 213 ++++++++++++++++ .../perfect-machine-study-2026-05-23/index.md | 67 +++++ .../optimization-ledger.md | 174 +++++++++++++ .../references.md | 70 +++++ .../samples/Avx2DechirpGoertzelSketch.cpp | 82 ++++++ .../samples/ChirpBinScore.compute.hlsl | 60 +++++ .../samples/FarrowFractionalDelaySketch.cpp | 45 ++++ .../samples/README.md | 22 ++ .../samples/SpscAudioBlockRingSketch.cpp | 56 ++++ .../samples/StreamingChirpBinDecoderSketch.cs | 188 ++++++++++++++ 13 files changed, 1222 insertions(+) create mode 100644 docs/perfect-machine-domain-index.md create mode 100644 research/perfect-machine-study-2026-05-23/architecture-rumination.md create mode 100644 research/perfect-machine-study-2026-05-23/index.md create mode 100644 research/perfect-machine-study-2026-05-23/optimization-ledger.md create mode 100644 research/perfect-machine-study-2026-05-23/references.md create mode 100644 research/perfect-machine-study-2026-05-23/samples/Avx2DechirpGoertzelSketch.cpp create mode 100644 research/perfect-machine-study-2026-05-23/samples/ChirpBinScore.compute.hlsl create mode 100644 research/perfect-machine-study-2026-05-23/samples/FarrowFractionalDelaySketch.cpp create mode 100644 research/perfect-machine-study-2026-05-23/samples/README.md create mode 100644 research/perfect-machine-study-2026-05-23/samples/SpscAudioBlockRingSketch.cpp create mode 100644 research/perfect-machine-study-2026-05-23/samples/StreamingChirpBinDecoderSketch.cs diff --git a/README.md b/README.md index 6516241..63eb101 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,8 @@ Start with: camera/sensor evidence. - [docs/code-algorithm-map.md](docs/code-algorithm-map.md) for the current source-level code map. +- [docs/perfect-machine-domain-index.md](docs/perfect-machine-domain-index.md) + for the indexed problem-domain map and implementation cuts. Naming note: **Fensalir** is the engine/windowing/rendering/D3D12 layer. Older research notes may still use the previous engine name. diff --git a/docs/code-algorithm-map.md b/docs/code-algorithm-map.md index 377bb2d..2a954c0 100644 --- a/docs/code-algorithm-map.md +++ b/docs/code-algorithm-map.md @@ -1418,6 +1418,10 @@ Linked notes: - `research/feedback-calibration/summary.md`: room/speaker/mic calibration. - `research/perfect-machine-hypotheses/summary.md`: hypotheses added by this documentation pass. +- `docs/perfect-machine-domain-index.md`: indexed domain map for future + implementation passes. +- `research/perfect-machine-study-2026-05-23/`: architecture rumination, + optimization ledger, references, and sample implementation sketches. Hypotheses: diff --git a/docs/perfect-machine-domain-index.md b/docs/perfect-machine-domain-index.md new file mode 100644 index 0000000..15e5133 --- /dev/null +++ b/docs/perfect-machine-domain-index.md @@ -0,0 +1,239 @@ +# Mimir Perfect Machine Domain Index + +## What This File Is + +This is the domain index for future implementation passes. It organizes the +whole Mimir problem space by authority, evidence type, algorithm, hot path, and +open architectural cut. + +Read this with: + +- `docs/code-algorithm-map.md` for source-level ownership. +- `docs/audio-field.md` for current audio sync narrative. +- `docs/native-capture-cadence.md` for measured camera/USB evidence. +- `research/perfect-machine-study-2026-05-23/architecture-rumination.md` for + the long-form design argument. +- `research/perfect-machine-study-2026-05-23/index.md` for the study directory + and sample-code index. +- `research/perfect-machine-study-2026-05-23/optimization-ledger.md` for hot + loop candidates. +- `research/perfect-machine-study-2026-05-23/samples/README.md` for sample + implementation sketches. + +## One-Sentence Machine + +Mimir is a bounded-latency realtime field computer: it ingests many clocks, +places their evidence inside one rolling temporal window, extracts audio/video +geometry from that window, and emits an OBS-usable program surface without +letting OBS become the synchronization authority. + +## Authority Index + +| Authority | Owns | Must Not Own | +| --- | --- | --- | +| `Mimir.Runtime` | stream identity, buffers, sync reports, calibration model loading, source polling, telemetry state | hot DSP actuator, dense visual fusion, broadcast composition | +| Native capture | direct driver callbacks, device timestamps, payload handles, low-jitter capture loops | runtime policy, UI state, OBS composition | +| `native/reservoir` | shared-edge retention and typed views | payload memory interpretation, rendering, DSP | +| Fensalir | window, UI, D3D12 bridge, GPU fusion, program video publication | clock authority, audio actuator | +| Faust/native DSP | fractional delay, resampling, separation, spatialization, stems | stream discovery, UI | +| OBS | final broadcast composition | synchronization, capture timing | +| Raven/phones | local decode of codebook/schedule observations | canonical clock authority | + +## Evidence Taxonomy + +### Audio Timing Evidence + +- ASIO loopback: strongest local reference; same interface clock as Scarlett mic + channels on Starfire. +- Physical mic chirp-bin decode: acoustic timing plus response surface; current + failure mode is sparse symbols and room/path coloration. +- Passive program audio: relative delay/drift hint from real content; useful + when confidence is earned, not a canonical code timeline. +- Network audio: delayed observations from Raven/phones; must carry decoded + local timing quality, not become a remote clock. + +### Audio Calibration Evidence + +- Magnitude response: which emitted bins survive per output/mic path. +- Confusion matrix: expected symbol versus observed bin energy/symbol. +- Phase coherence: whether a path has stable phase enough for group-delay + correction. +- Delay hypotheses: global delay/bin-shift candidates for whole-window decode. +- SRO: delay slope over time, feeding resampler control. + +### Visual Evidence + +- Leap stereo IR: timing/depth candidate, topology-sensitive but high-value. +- PS3 Eyes: high-rate tracking witnesses. +- Kiyo/Kiyo Pro: RGB context and texture, with Kiyo Pro currently cadence/USB + limited. +- Future Fensalir GPU state: features, rays, surface/material claims, splats, + confidence, and render packets. + +### Network Evidence + +- SRT/FFmpeg bridge: useful v1 transport witness, not synchronized program + authority. +- Future Raven/phone agents: source-local decode/calibration reporters. + +## Algorithm Families + +### Controlled Chirp-Bin Active Decode + +Purpose: + +- Place each mic stream on canonical time. +- Learn response/confusion/phase/group-delay. + +Current live shape: + +1. render deterministic fixed-slope chirp-bin codebook; +2. detect candidate event windows via cheap energy trace; +3. dechirp each candidate; +4. score a small bin bank; +5. preserve ambiguous symbol/time candidates; +6. select code-valid de Bruijn triplets; +7. fit a coherent source clock; +8. refine source offset against scheduled waveform; +9. emit calibration model evidence. + +Pressure: + +- The hot loop must move from scalar per-window scoring toward SIMD/GPU batched + dechirp/bin evaluation once stream counts rise. + +### Passive GCC-PHAT Program-Audio Sync + +Purpose: + +- Use existing music/game audio when it is informative. + +Current live shape: + +1. window reference/candidate audio; +2. mean remove, preemphasize, Hann window; +3. FFT both; +4. phase-normalized cross-spectrum; +5. inverse FFT to lag correlation; +6. peak/refinement/confidence. + +Pressure: + +- Passive confidence should be multi-window and stateful before it can suppress + active watermark emission for long periods. + +### Fractional Delay And SRO Actuation + +Purpose: + +- Turn measurement into aligned samples. + +Likely shape: + +- Per-source delay state drives a fractional delay line. +- SRO state drives an asynchronous sample-rate converter. +- Fast path belongs in Faust/native DSP, not C#. +- Control loop consumes `MimirAudioSynchronizationState`. + +Candidate filters: + +- Farrow/Lagrange fractional delay for tiny offset corrections. +- Polyphase windowed-sinc resampler for higher quality. +- PLL-like state on delay slope to keep drift stable. + +### Visual Fusion + +Purpose: + +- Turn synchronized frames into a usable volumetric scene. + +Likely shape: + +- Native capture pushes typed frame handles and timestamps. +- Fensalir extracts features/depth/optical flow on GPU. +- Native reservoir preserves rolling evidence. +- Fusion emits surface/material/render claims. +- Program video publishes via Spout2. + +Pressure: + +- Do not build a CPU image pipeline because it is easy to inspect. Six cameras + make that path collapse into a pile of excuses. + +## Hot Loop Budget + +### Audio + +Given 192 kHz, four local Scarlett channels, and future Raven/phone sources: + +- 2 seconds = 384,000 samples per channel. +- Four channels = 1.5 million samples per analysis window. +- Analysis every 100 ms can easily become a full CPU core if windows are + rescanned naively. + +Rules: + +- Incremental streaming state beats full-window rescans. +- Precompute chirp kernels. +- Dechirp/bin score in batches. +- Reuse aligned buffers. +- Separate proposal, classification, and global code solving. + +### Video + +Six optical sensors with high-rate Eyes and Leap IR mean: + +- CPU copying/compression/debug JSON is poison in the hot path. +- Capture should produce native handles or stable byte spans with timestamps. +- GPU work should happen once inside Fensalir, not through throwaway staging + formats. + +## Implementation Index + +### Current Code To Read First + +- `src/Mimir.Runtime/MimirRuntime.cs`: runtime loop and witness emission. +- `src/Mimir.Runtime/Synchronization/MimirSynchronizationHub.cs`: source/buffer + and report authority. +- `src/Mimir.Runtime/Synchronization/MimirAudioSynchronizationAnalyzer.cs`: + buffer-to-report analyzer. +- `src/Mimir.Runtime/Synchronization/MimirChirpBinTimeline.cs`: active decoder. +- `src/Mimir.Runtime/Synchronization/MimirChirpBinCalibrationModel.cs`: + response/confusion/delay model. +- `native/asio_capture/mimir_asio_capture.cpp`: runtime ASIO bridge. +- `native/reservoir/src/lib.rs`: native shared-edge reservoir. + +### Current Code To Treat As Temporary + +- `MimirFrameEventProcessStreamSource`: diagnostic JSON-line adapter. +- `MimirProcessStreamSource`: bridge/network raw stdout adapter. +- WASAPI probe sample transport. +- FFmpeg/SRT sender scripts. +- Legacy dense chirplet path. + +### Current Code To Preserve Conceptually + +- Descriptor/sample/buffer identity model. +- Hub report/state cache authority. +- ASIO in-process source shape. +- Chirp-bin codebook/calibration separation. +- Native reservoir shared-edge invariant. + +## Cut Lines + +- Cut JSON/stdout local camera hot path when concrete direct drivers land. +- Cut legacy chirplet path once chirp-bin decoder has equivalent docs/tests and + physical path proof. +- Cut bridge scripts only after OBS-facing direct output is real. +- Cut ad hoc response weighting if a formal path model replaces it. + +## Future Work Index + +1. Streaming chirp-bin decoder state. +2. Calibration-weighted physical decode proof. +3. Reduced alphabet/high-order codebook physical proof. +4. Fractional delay/SRO actuator proof. +5. Direct Leap/PS3/Kiyo runtime drivers. +6. Native reservoir integration with Fensalir/Faust. +7. Raven/phone receiver codebook/schedule local decode. +8. GPU visual fusion and Spout2 program video. diff --git a/research/perfect-machine-study-2026-05-23/architecture-rumination.md b/research/perfect-machine-study-2026-05-23/architecture-rumination.md new file mode 100644 index 0000000..2a6f736 --- /dev/null +++ b/research/perfect-machine-study-2026-05-23/architecture-rumination.md @@ -0,0 +1,213 @@ +# Architecture Rumination: Building Mimir Without Jenga + +## Clock + +- User correction received after the first documentation pass: spend the full + dedicated pass mapping, researching, distilling, ruminating, and writing + sample options. +- Active pass start recorded locally: `2026-05-23T23:41:19+01:00`. + +## Prime Cut + +Mimir is not an app that "captures devices." That framing is too small and it +drags the architecture toward adapters, UI panels, and pile-of-feeds thinking. +Mimir is a realtime field reconstruction machine. Devices are evidence +producers. The rolling buffer is the measurement window. Synchronization is the +act that makes independent evidence comparable. Output is a negotiated product, +not a raw feed. + +The architecture should be judged by whether each subsystem makes field +evidence more coherent: + +- Can this block preserve timing? +- Can this block preserve identity? +- Can this block preserve enough raw information for a later better algorithm? +- Can this block be explained as "X owns Y so Z remains true"? + +If not, it is probably scaffolding or residue. + +## Current Machine + +### Runtime + +`MimirRuntime` is a Fensalir runtime with an intentionally empty scene. It owns +application cadence, audio witness emission, telemetry, and UI readouts. It +does not own driver details. It does not own DSP. It should eventually own less +of the calibration renderer too: witness generation can stay controlled by the +runtime, but sample production and hot-path synthesis should be native/DSP once +the shape is settled. + +### Hub + +`MimirSynchronizationHub` is the correct authority for current runtime belief. +It owns source registration, buffer append, analyzer scheduling, report cache, +and state smoothing. The strongest invariant here is that UI/telemetry must not +invoke analysis. That separation prevents accidental workload explosions. + +### Buffers + +`MimirRollingStreamBuffer` is simple and right: one stream identity, one bounded +queue, one edge. It is not the final native memory layout, but the ownership +model is sound. The native reservoir should be the lower version of the same +idea: one edge, typed views, no per-kind retention. + +### Active Audio Decoder + +`MimirChirpBinTimeline` is the live active sync machine. It is conceptually +right because the emitter and receiver share the same codebook and schedule. +The current code still has scalar and full-window pressure, but the shape is no +longer the wrong dense chirplet wall. It is a controlled receiver: + +- proposal; +- dechirp; +- bin score; +- ambiguity preservation; +- code-constrained path selection; +- clock fit; +- calibration evidence. + +The next coherent cut is not "add more rejection." It is streaming state and +calibration-weighted decode that changes the likelihood surface before path +selection. + +### Passive Audio Decoder + +`MimirPassiveAudioSynchronizationEstimator` is useful but should be demoted in +the mental model. It is a passive delay/drift witness, not canonical time. It +can suppress active watermark when confidence is stable. It should not invent +timeline anchors. + +### Native ASIO + +`native/asio_capture` is a correct direction: COM and ASIO callbacks stay in +native code, C# reads bounded channel blocks. This pattern should repeat for +camera drivers. + +### Native Probes + +The probes are research instruments. They should keep telling us what hardware +does, but their stdout/JSON shape must not leak into production. + +## The Perfect Machine Shape + +### Audio Spine + +1. Native ASIO capture writes input and loopback channels into native audio + blocks with device sample counters. +2. Mimir runtime indexes those blocks as stream samples. +3. Active/passive analyzers update per-source delay/SRO/response state. +4. Faust/native DSP consumes the same blocks plus state and emits aligned stems. +5. OBS receives stems/spatial bed; it never sees unsynchronized raw mics as the + truth. + +### Visual Spine + +1. Native camera drivers capture frames with device or immediate receipt time. +2. Runtime indexes handles into rolling buffers/reservoir. +3. Fensalir GPU extracts features/flow/depth/surface/material claims. +4. Reservoir preserves rolling visual evidence by time. +5. Fensalir emits program video via Spout2. + +### Distributed Spine + +Raven and phones should decode local time from codebook/schedule state. They +should report: + +- local anchor observations; +- confidence; +- response surface; +- fitted clock; +- transport timestamp only as metadata. + +They should not ask Starfire to bless every sample over the network. Network +latency is exactly what the codebook is meant to escape. + +## Things The Current Code Gets Right + +- The app/runtime split is clean. +- The UI is not the analyzer. +- Stream identity is explicit. +- Direct ASIO is in process and sample-bearing. +- Calibration model includes magnitude, confusion, phase, delay, and adaptive + codebook planning. +- Runtime emission can consume the calibration plan. +- Native reservoir already encodes the single-edge invariant. + +## Things Still Too Soft + +- Active decoder still rescans windows rather than owning a streaming state. +- Physical acoustic decode is not yet robust enough. +- The actuator does not exist. +- Direct camera runtime drivers do not exist. +- Native reservoir is not wired into the C#/Fensalir/Faust flow. +- Calibration model is loaded, but not yet proven as the decisive fix for + meatspace. +- Raven/phone receiver shape is only conceptual. + +## Candidate Cuts + +### Cut 1: Streaming Chirp-Bin Decoder + +Owns: + +- rolling PCM window; +- energy proposal ring; +- candidate frame cache; +- anchor trellis; +- clock fit state. + +Reason: + +- Avoid full-window rescans every analysis tick. + +### Cut 2: Calibration-Weighted Likelihood + +Owns: + +- path-specific symbol likelihood from response model; +- phase/group-delay correction before code selection; +- global delay/bin-shift hypothesis scoring. + +Reason: + +- Meatspace failure is not merely noisy timing. It is path-shaped symbol + distortion. + +### Cut 3: Native DSP Actuator + +Owns: + +- fractional delay; +- resampler; +- SRO control; +- phase/group-delay correction; +- stem alignment. + +Reason: + +- The runtime estimates state. DSP moves samples. + +### Cut 4: Direct Camera Driver Worker + +Owns: + +- device open/configure; +- async read queue; +- timestamp preservation; +- native payload handle. + +Reason: + +- JSON/stdout cannot be allowed near six-camera hot loops. + +## Rejected Temptations + +- More mode flags in analyzer: the problem is evidence ownership, not a switch + shortage. +- More outlier rejection before calibration weighting: that hides the response + surface instead of learning it. +- Routing all sources through process adapters because it is easy to debug: + that is how bandwidth dies in a formal suit. +- Treating 192 kHz as magic: it improves timing granularity and ultrasonic + options only where transducers and path response actually preserve signal. + diff --git a/research/perfect-machine-study-2026-05-23/index.md b/research/perfect-machine-study-2026-05-23/index.md new file mode 100644 index 0000000..f19e020 --- /dev/null +++ b/research/perfect-machine-study-2026-05-23/index.md @@ -0,0 +1,67 @@ +# 2026-05-23 Perfect Machine Study Index + +## Purpose + +This directory is the deeper study pass that sits beside the source map. Use it +when the next implementation pass needs more than "where is the file?" and +needs the reason a cut exists. + +## Documents + +| File | Use It For | +| --- | --- | +| `architecture-rumination.md` | System ownership, current machine critique, coherent next cuts. | +| `optimization-ledger.md` | Hot loop risks, low-level implementation options, micro-optimization candidates. | +| `references.md` | Research links for chirplets, CSS/LoRa, GCC-PHAT, D3D12/FFT/GPU compute. | +| `samples/README.md` | Index of sample implementation sketches. | + +## Sample Code + +| Sample | Future Production Target | Question It Answers | +| --- | --- | --- | +| `samples/StreamingChirpBinDecoderSketch.cs` | `MimirChirpBinStreamingDecoder` or equivalent runtime class | What state removes full-window rescans? | +| `samples/Avx2DechirpGoertzelSketch.cpp` | Native CPU scorer for chirp-bin candidates | What does SIMD dechirp/bin scoring look like? | +| `samples/ChirpBinScore.compute.hlsl` | Fensalir/D3D12 compute scorer | How would batched candidate/bin scoring map to GPU? | +| `samples/FarrowFractionalDelaySketch.cpp` | Faust/native DSP actuator | What is the smallest fractional delay proof? | +| `samples/SpscAudioBlockRingSketch.cpp` | Native capture worker handoff | What shape should low-jitter audio block transfer take? | + +## Best Next Implementation Sequence + +1. Build a streaming chirp-bin decoder state object beside the current analyzer, + but keep the old analyzer as proof oracle until outputs match. +2. Load a real physical mic calibration model and compare weighted versus + unweighted symbol likelihood on stored Scarlett captures. +3. Emit reduced reliable-bin codebook from the physical model and measure + meatspace anchor rate. +4. Add native/Faust actuator proof using logged `MimirAudioSynchronizationState` + before touching live program output. +5. Port one direct camera path into `IMimirVideoCaptureDriverSource`; Leap first + if timing/depth value wins, PS3 Eye first if raw driver throughput is the + faster proof. + +## Index Terms + +- active sync +- ASIO +- AVX2 +- chirp-bin +- chirplet +- codebook adaptation +- dechirp +- D3D12 compute +- Farrow fractional delay +- Faust actuator +- GCC-PHAT +- group delay +- LoRa CSS +- native reservoir +- passive sync +- Raven +- response/confusion matrix +- sample-rate offset +- SPSC ring +- Starfire +- streaming decoder +- volumetric audio +- volumetric visual fusion + diff --git a/research/perfect-machine-study-2026-05-23/optimization-ledger.md b/research/perfect-machine-study-2026-05-23/optimization-ledger.md new file mode 100644 index 0000000..15d831a --- /dev/null +++ b/research/perfect-machine-study-2026-05-23/optimization-ledger.md @@ -0,0 +1,174 @@ +# Optimization Ledger + +## What This File Is + +This is the low-level implementation ledger for Mimir's most likely hot paths. +It is not a benchmark result. It is a map of what to test, why it might matter, +and where the current code is likely to pay unnecessary cost. + +## Hot Path 1: Chirp-Bin Decode + +### Current Shape + +- Build an energy trace across a window. +- Classify local candidate windows. +- Dechirp and score each bin. +- Run code-valid anchor selection. +- Fit clock and refine offset. + +### Bottleneck Risk + +The current scalar/classic C# version is correct enough to reason about, but it +still encourages repeated full-window work. Six or more audio streams at 192 kHz +make that expensive quickly. + +### Preferred Evolution + +1. Streaming energy proposal ring. +2. Precomputed chirp kernels by sample rate and symbol plan. +3. SIMD dechirp/bin score on CPU for small channel counts. +4. Batched GPU compute for many candidate windows or many remote receivers. +5. Keep the codebook trellis on CPU unless profiling proves otherwise; it is + branchy and small. + +### Micro-Optimization Ideas + +- Structure-of-arrays for sine/cosine kernels. +- Process bins in groups of 8 floats on AVX2/FMA. +- Keep hot candidate windows contiguous and aligned. +- Avoid allocating candidate arrays inside every analysis tick. +- Cache symbol reliability weights in dense arrays. +- Precompute expected event sample offsets for the active window. +- Use a ring of candidate frames keyed by absolute sample index. + +### Sample References + +- `samples/StreamingChirpBinDecoderSketch.cs` +- `samples/Avx2DechirpGoertzelSketch.cpp` +- `samples/ChirpBinScore.compute.hlsl` + +## Hot Path 2: Passive GCC-PHAT + +### Current Shape + +- Allocate complex arrays per estimate. +- FFT reference/candidate. +- Normalize cross-spectrum. +- Inverse FFT. +- Search lags. + +### Bottleneck Risk + +Passive sync is cheaper than dense chirplet matching, but repeated allocations +and full FFTs per source pair can still dominate if called too often. + +### Preferred Evolution + +- Reuse FFT buffers/plans. +- Keep reference spectrum cached for the current analysis edge. +- Batch candidate transforms. +- Move to native FFTW/KissFFT/MKL/cuFFT only after measured C# FFT cost matters. +- Use passive only as confidence/drift support, not as mandatory every-tick work. + +### Micro-Optimization Ideas + +- Window into preallocated `Complex` spans. +- Precompute Hann window. +- Preemphasis in one pass with mean removal. +- Limit lag search by physical/network horizon. +- Use parabolic interpolation only around plausible peaks. + +## Hot Path 3: Fractional Delay And SRO Actuator + +### Current Shape + +- Not built. + +### Preferred Evolution + +- Prototype native/Faust fractional delay using Farrow/Lagrange for small + sub-sample corrections. +- Add a higher quality polyphase sinc path for program output. +- Drive both from smoothed delay/SRO state. +- Keep per-source state in DSP, not in UI/runtime strings. + +### Micro-Optimization Ideas + +- Fixed filter order for predictable SIMD. +- Interleave channels only where the DSP kernel wants it. +- Separate control-rate state update from audio-rate sample processing. +- Use denormal guards. + +### Sample Reference + +- `samples/FarrowFractionalDelaySketch.cpp` + +## Hot Path 4: Native Rolling Buffers + +### Current Shape + +- C# rolling buffers store sample envelopes. +- Rust reservoir stores one shared-edge native rolling buffer with typed views. + +### Bottleneck Risk + +The C# buffer shape is fine for current proof state, but payload-heavy audio and +video should move to native memory handles. Copies and per-sample allocations +will become visible as source count rises. + +### Preferred Evolution + +- Native SPSC rings per capture worker feeding a shared reservoir index. +- Payload handles point to native/audio/GPU memory owned by capture/DSP/Fensalir. +- Runtime stores metadata and current belief. + +### Micro-Optimization Ideas + +- Power-of-two ring capacities. +- Single writer per capture device. +- Cache-line padded head/tail counters. +- Batch publish blocks. +- Avoid sharing mutable payload ownership across subsystems. + +### Sample Reference + +- `samples/SpscAudioBlockRingSketch.cpp` + +## Hot Path 5: Camera Capture And GPU Fusion + +### Current Shape + +- Native probes prove direct driver access and cadence. +- Runtime direct driver seam exists. +- Fensalir fusion not wired yet. + +### Bottleneck Risk + +Six cameras make CPU copies and process bridges fail. Leap/PS3/Kiyo sources +need direct capture, stable timestamps, and GPU-friendly payloads. + +### Preferred Evolution + +- Direct KS/libusb/vendor driver workers. +- Native payload handles into runtime/reservoir. +- Fensalir consumes current window and uploads/processes on GPU. +- Use D3D12 shared resources where possible. + +### Micro-Optimization Ideas + +- Queue multiple async reads per camera. +- Keep camera buffers pinned/native. +- Avoid decode unless the algorithm needs decoded pixels. +- Do per-camera feature extraction in compute, then fuse compact features. + +## External References + +- LoRa/CSS receivers repeatedly validate dechirp plus FFT/bin scoring as the + natural controlled-chirp demodulator shape. +- HLSL Shader Model 6 wave intrinsics are relevant for reductions and FFT-like + kernels inside D3D12 compute. +- cuFFT callbacks show a general trick: combine preprocessing with transform + load/store to avoid extra memory bandwidth. +- FFTW wisdom/alignment notes matter if we move passive or chirp-bin batches to + native FFT plans. + diff --git a/research/perfect-machine-study-2026-05-23/references.md b/research/perfect-machine-study-2026-05-23/references.md new file mode 100644 index 0000000..47b0faf --- /dev/null +++ b/research/perfect-machine-study-2026-05-23/references.md @@ -0,0 +1,70 @@ +# Research References + +## Chirplet And Chirp Analysis + +- Steve Mann, *The Chirplet Transform: A new signal analysis technique based on + affine relationships in the time-frequency plane*: + https://www.media.mit.edu/publications/the-chirplet-transform-a-new-signal-analysis-technique-based-on-affine-relationships-in-the-time-frequency-plane/ +- General linear chirplet transform overview: + https://www.sciencedirect.com/science/article/pii/S0888327015003994 +- High-resolution chirplet transform parameter analysis: + https://arxiv.org/abs/2108.00572 + +## Chirp Spread Spectrum / LoRa Receiver Shape + +- SDR-LoRa design notes, including synchronization via dechirp and FFT: + https://www.sciencedirect.com/science/article/pii/S1389128624000264 +- OpenLoRa demodulator evaluation: + https://openlora.wisc.edu/demodulators-evaluated/ +- OpenLoRa NSDI paper: + https://www.usenix.org/system/files/nsdi23-mishra.pdf +- CSS receiver design: + https://arxiv.org/abs/2105.02833 +- LoRa/CSS tutorial: + https://arxiv.org/abs/2310.10503 +- I/Q CSS with coherent detector and channel estimation: + https://arxiv.org/abs/2009.10421 +- Data-over-audio reference implementation trail: + https://github.com/cawfree/OpenChirp + +## Passive Audio Timing / GCC-PHAT + +- Improved GCC-PHAT weighting for time-delay estimation: + https://pmc.ncbi.nlm.nih.gov/articles/PMC9571281/ +- Acoustic DoA paper noting GCC-PHAT normalization and interpolation: + https://pmc.ncbi.nlm.nih.gov/articles/PMC11014389/ +- Parametrized GCC-PHAT features for time-delay estimation: + https://www.isca-archive.org/interspeech_2021/salvati21_interspeech.html +- Complexity/accuracy of GCC-PHAT methods: + https://arxiv.org/abs/1811.11787 + +## Low-Level Transform / Compute References + +- HLSL Shader Model 6 wave intrinsics: + https://learn.microsoft.com/windows/win32/direct3dhlsl/hlsl-shader-model-6-0-features-for-direct3d-12 +- DirectX Shader Compiler wave intrinsics notes: + https://github.com/microsoft/directxshadercompiler/wiki/wave-intrinsics +- cuFFT callback documentation: + https://docs.nvidia.com/cuda/archive/12.5.0/cufft/index.html +- NVIDIA cuFFT callback performance article: + https://developer.nvidia.com/blog/cuda-pro-tip-use-cufft-callbacks-custom-data-processing/ +- FFTW reference: + https://www.fftw.org/fftw2_doc/fftw_3.html +- FFTW SIMD alignment: + https://www.fftw.org/~fftw/doc/SIMD-alignment-and-fftw_005fmalloc.html +- Chirp Z-transform theory: + https://pyffs.readthedocs.io/en/stable/theory/CZT.html + +## How These References Steer Mimir + +- Dechirp-plus-bin scoring is the right hot active receiver shape for controlled + chirps. +- Dense generic chirplet transforms are research/reference tools, not the + runtime hot loop unless the codebook stops being controlled. +- GCC-PHAT is still valuable for passive relative delay, but it must remain an + evidence source with confidence, not a canonical timestamp source. +- GPU or native FFT paths should be considered only where batching is real; for + a small fixed bin bank, SIMD Goertzel/dechirp may beat FFT overhead. +- If a transform path moves to GPU, fuse dechirp/window/preprocess with transform + load/store where possible to avoid memory bandwidth waste. + diff --git a/research/perfect-machine-study-2026-05-23/samples/Avx2DechirpGoertzelSketch.cpp b/research/perfect-machine-study-2026-05-23/samples/Avx2DechirpGoertzelSketch.cpp new file mode 100644 index 0000000..9fa50c7 --- /dev/null +++ b/research/perfect-machine-study-2026-05-23/samples/Avx2DechirpGoertzelSketch.cpp @@ -0,0 +1,82 @@ +// Sketch only. Compile-time target would be /arch:AVX2 plus FMA. +// This illustrates a CPU hot loop for dechirp + small-bin scoring. + +#include +#include +#include + +struct KernelSoA +{ + const float* cosBase; + const float* sinBase; + const float* binCos; + const float* binSin; + int sampleCount; + int binCount; +}; + +struct BinScore +{ + float real; + float imag; + float energy; +}; + +static inline float hsum256(__m256 value) +{ + __m128 low = _mm256_castps256_ps128(value); + __m128 high = _mm256_extractf128_ps(value, 1); + __m128 sum = _mm_add_ps(low, high); + sum = _mm_hadd_ps(sum, sum); + sum = _mm_hadd_ps(sum, sum); + return _mm_cvtss_f32(sum); +} + +void score_dechirped_bins_avx2( + const float* samples, + const KernelSoA& kernel, + BinScore* outScores) +{ + for (int bin = 0; bin < kernel.binCount; ++bin) + { + __m256 accRe = _mm256_setzero_ps(); + __m256 accIm = _mm256_setzero_ps(); + + const float* binCos = kernel.binCos + bin * kernel.sampleCount; + const float* binSin = kernel.binSin + bin * kernel.sampleCount; + + int i = 0; + for (; i + 7 < kernel.sampleCount; i += 8) + { + __m256 x = _mm256_loadu_ps(samples + i); + __m256 chirpC = _mm256_loadu_ps(kernel.cosBase + i); + __m256 chirpS = _mm256_loadu_ps(kernel.sinBase + i); + __m256 toneC = _mm256_loadu_ps(binCos + i); + __m256 toneS = _mm256_loadu_ps(binSin + i); + + // Real input multiplied by conjugate chirp, then projected onto + // the candidate bin tone. Algebra can be fused further once the + // final kernel convention is frozen. + __m256 deRe = _mm256_mul_ps(x, chirpC); + __m256 deIm = _mm256_mul_ps(x, _mm256_sub_ps(_mm256_setzero_ps(), chirpS)); + + accRe = _mm256_fmadd_ps(deRe, toneC, accRe); + accRe = _mm256_fnmadd_ps(deIm, toneS, accRe); + accIm = _mm256_fmadd_ps(deRe, toneS, accIm); + accIm = _mm256_fmadd_ps(deIm, toneC, accIm); + } + + float re = hsum256(accRe); + float im = hsum256(accIm); + for (; i < kernel.sampleCount; ++i) + { + float deRe = samples[i] * kernel.cosBase[i]; + float deIm = -samples[i] * kernel.sinBase[i]; + re += deRe * binCos[i] - deIm * binSin[i]; + im += deRe * binSin[i] + deIm * binCos[i]; + } + + outScores[bin] = { re, im, re * re + im * im }; + } +} + diff --git a/research/perfect-machine-study-2026-05-23/samples/ChirpBinScore.compute.hlsl b/research/perfect-machine-study-2026-05-23/samples/ChirpBinScore.compute.hlsl new file mode 100644 index 0000000..aace671 --- /dev/null +++ b/research/perfect-machine-study-2026-05-23/samples/ChirpBinScore.compute.hlsl @@ -0,0 +1,60 @@ +// Sketch only. D3D12 compute shape for batched chirp-bin scoring. +// One thread group scores one candidate window against one symbol/bin block. + +cbuffer ChirpScoreConstants : register(b0) +{ + uint SampleCount; + uint BinCount; + uint CandidateCount; + uint KernelStride; +}; + +StructuredBuffer Samples : register(t0); +StructuredBuffer CandidateSampleOffsets : register(t1); +StructuredBuffer DechirpKernel : register(t2); +StructuredBuffer BinKernel : register(t3); +RWStructuredBuffer Scores : register(u0); + +groupshared float2 Partial[256]; + +[numthreads(256, 1, 1)] +void main(uint3 groupId : SV_GroupID, uint groupIndex : SV_GroupIndex) +{ + uint candidate = groupId.x; + uint bin = groupId.y; + uint offset = CandidateSampleOffsets[candidate]; + + float2 acc = float2(0.0, 0.0); + for (uint i = groupIndex; i < SampleCount; i += 256) + { + float x = Samples[offset + i]; + float2 chirp = DechirpKernel[i]; + float2 tone = BinKernel[bin * KernelStride + i]; + + float2 dechirped = float2(x * chirp.x, x * chirp.y); + acc += float2( + dechirped.x * tone.x - dechirped.y * tone.y, + dechirped.x * tone.y + dechirped.y * tone.x); + } + + Partial[groupIndex] = acc; + GroupMemoryBarrierWithGroupSync(); + + for (uint stride = 128; stride > 0; stride >>= 1) + { + if (groupIndex < stride) + { + Partial[groupIndex] += Partial[groupIndex + stride]; + } + + GroupMemoryBarrierWithGroupSync(); + } + + if (groupIndex == 0) + { + float2 z = Partial[0]; + float energy = dot(z, z); + Scores[candidate * BinCount + bin] = float4(z.x, z.y, energy, 0.0); + } +} + diff --git a/research/perfect-machine-study-2026-05-23/samples/FarrowFractionalDelaySketch.cpp b/research/perfect-machine-study-2026-05-23/samples/FarrowFractionalDelaySketch.cpp new file mode 100644 index 0000000..a5a34e9 --- /dev/null +++ b/research/perfect-machine-study-2026-05-23/samples/FarrowFractionalDelaySketch.cpp @@ -0,0 +1,45 @@ +// Sketch only. Cubic Farrow fractional-delay actuator for small delay changes. + +#include +#include +#include + +class FarrowFractionalDelay +{ +public: + float process(float x, double delaySamples) + { + push(x); + + int whole = static_cast(std::floor(delaySamples)); + double mu = delaySamples - whole; + + float xm1 = read(whole + 3); + float x0 = read(whole + 2); + float x1 = read(whole + 1); + float x2 = read(whole + 0); + + // Cubic Lagrange form. Good enough for an actuator proof; production + // should compare against polyphase sinc for final program quality. + double c0 = x0; + double c1 = 0.5 * (x1 - xm1); + double c2 = xm1 - 2.5 * x0 + 2.0 * x1 - 0.5 * x2; + double c3 = 0.5 * (x2 - xm1) + 1.5 * (x0 - x1); + return static_cast(((c3 * mu + c2) * mu + c1) * mu + c0); + } + +private: + std::array ring{}; + unsigned cursor = 0; + + void push(float x) + { + ring[cursor++ & (ring.size() - 1)] = x; + } + + float read(int delay) const + { + return ring[(cursor - 1 - static_cast(delay)) & (ring.size() - 1)]; + } +}; + diff --git a/research/perfect-machine-study-2026-05-23/samples/README.md b/research/perfect-machine-study-2026-05-23/samples/README.md new file mode 100644 index 0000000..b1f916b --- /dev/null +++ b/research/perfect-machine-study-2026-05-23/samples/README.md @@ -0,0 +1,22 @@ +# Sample Implementation Sketches + +These are not production files. They are indexed sketches for future cuts, +written to make implementation options concrete without sneaking half-built +machinery into the app. + +## Files + +- `StreamingChirpBinDecoderSketch.cs`: C# state model for an incremental + chirp-bin decoder that avoids full-window rescans. +- `Avx2DechirpGoertzelSketch.cpp`: AVX2/FMA-shaped CPU scoring kernel sketch. +- `ChirpBinScore.compute.hlsl`: D3D12 compute shader shape for batched + candidate/bin scoring. +- `FarrowFractionalDelaySketch.cpp`: native fractional delay actuator sketch. +- `SpscAudioBlockRingSketch.cpp`: native single-producer/single-consumer audio + block ring sketch for capture worker handoff. + +## Rule + +Promote one of these only after a benchmark or integration test proves it owns a +real invariant. Sample code is a microscope, not a new wall in the house. + diff --git a/research/perfect-machine-study-2026-05-23/samples/SpscAudioBlockRingSketch.cpp b/research/perfect-machine-study-2026-05-23/samples/SpscAudioBlockRingSketch.cpp new file mode 100644 index 0000000..24e0e70 --- /dev/null +++ b/research/perfect-machine-study-2026-05-23/samples/SpscAudioBlockRingSketch.cpp @@ -0,0 +1,56 @@ +// Sketch only. Native capture handoff ring for one producer and one consumer. + +#include +#include +#include + +struct AudioBlockHandle +{ + std::uint64_t dataHandle; + std::uint64_t timestampNs; + std::uint64_t sequence; + std::uint32_t frameCount; + std::uint32_t sampleRate; + std::uint32_t channelCount; + std::uint32_t format; +}; + +template +class SpscAudioBlockRing +{ +public: + static_assert((CapacityPowerOfTwo & (CapacityPowerOfTwo - 1)) == 0); + + bool push(const AudioBlockHandle& block) + { + auto head = head_.load(std::memory_order_relaxed); + auto next = head + 1; + if (next - tail_.load(std::memory_order_acquire) > CapacityPowerOfTwo) + { + return false; + } + + blocks_[head & (CapacityPowerOfTwo - 1)] = block; + head_.store(next, std::memory_order_release); + return true; + } + + bool pop(AudioBlockHandle& block) + { + auto tail = tail_.load(std::memory_order_relaxed); + if (tail == head_.load(std::memory_order_acquire)) + { + return false; + } + + block = blocks_[tail & (CapacityPowerOfTwo - 1)]; + tail_.store(tail + 1, std::memory_order_release); + return true; + } + +private: + alignas(64) std::atomic head_{0}; + alignas(64) std::atomic tail_{0}; + std::array blocks_{}; +}; + diff --git a/research/perfect-machine-study-2026-05-23/samples/StreamingChirpBinDecoderSketch.cs b/research/perfect-machine-study-2026-05-23/samples/StreamingChirpBinDecoderSketch.cs new file mode 100644 index 0000000..846997e --- /dev/null +++ b/research/perfect-machine-study-2026-05-23/samples/StreamingChirpBinDecoderSketch.cs @@ -0,0 +1,188 @@ +// Sketch only. This shows the state shape for an incremental chirp-bin decoder. +// It is not compiled by Mimir. + +namespace Mimir.Research.Samples; + +public sealed class StreamingChirpBinDecoderSketch +{ + private readonly ChirpBinPlan plan; + private readonly Ring pcm; + private readonly Ring energy; + private readonly List candidates = []; + private readonly List anchors = []; + private long absoluteSampleCursor; + private long lastScannedSample; + + public StreamingChirpBinDecoderSketch(ChirpBinPlan plan, int maxWindowSamples) + { + this.plan = plan; + pcm = new Ring(NextPowerOfTwo(maxWindowSamples)); + energy = new Ring(NextPowerOfTwo(maxWindowSamples / plan.HopSamples + 8)); + } + + public void Append(ReadOnlySpan samples) + { + foreach (var sample in samples) + { + pcm.Push(sample); + absoluteSampleCursor++; + } + } + + public DecodeSnapshot DecodeAvailable() + { + ScanEnergyIncrementally(); + ClassifyNewProposals(); + PruneOldCandidates(); + SolveAnchorsIncrementally(); + var fit = FitClock(anchors); + return new DecodeSnapshot([.. candidates], [.. anchors], fit); + } + + private void ScanEnergyIncrementally() + { + var firstPossible = lastScannedSample; + var lastPossible = absoluteSampleCursor - plan.WindowSamples; + for (var start = firstPossible; start <= lastPossible; start += plan.HopSamples) + { + var window = pcm.ReadAbsolute(start, plan.WindowSamples); + energy.Push(Rms(window)); + if (energy.Latest > plan.EnergyThreshold) + { + candidates.Add(new CandidateFrame(start, Energy: energy.Latest)); + } + } + + lastScannedSample = Math.Max(lastScannedSample, lastPossible); + } + + private void ClassifyNewProposals() + { + foreach (var candidate in candidates.Where(candidate => candidate.Symbols.Length == 0)) + { + var window = pcm.ReadAbsolute(candidate.AbsoluteSample, plan.WindowSamples); + candidate.Symbols = plan.Score(window); + } + } + + private void SolveAnchorsIncrementally() + { + // Production version should use a bounded trellis over the new tail, + // not rebuild the entire anchor path. + var tail = candidates + .Where(candidate => candidate.Symbols.Length > 0) + .OrderBy(candidate => candidate.AbsoluteSample) + .TakeLast(plan.TrellisTailFrames) + .ToArray(); + + foreach (var triple in ConsecutiveTriples(tail)) + { + if (!plan.TryMapTriplet(triple, out var eventIndex, out var confidence)) + { + continue; + } + + anchors.Add(new TimelineAnchor( + eventIndex, + triple[1].AbsoluteSample, + confidence)); + } + } + + private void PruneOldCandidates() + { + var keepAfter = absoluteSampleCursor - pcm.Capacity; + candidates.RemoveAll(candidate => candidate.AbsoluteSample < keepAfter); + anchors.RemoveAll(anchor => anchor.ObservedSample < keepAfter); + } + + private static double Rms(ReadOnlySpan samples) + { + var sum = 0.0; + foreach (var sample in samples) + { + sum += sample * sample; + } + + return Math.Sqrt(sum / Math.Max(1, samples.Length)); + } + + private static IEnumerable ConsecutiveTriples(CandidateFrame[] frames) + { + for (var index = 0; index + 2 < frames.Length; index++) + { + yield return [frames[index], frames[index + 1], frames[index + 2]]; + } + } + + private static ClockFit? FitClock(IReadOnlyList anchors) => null; + + private static int NextPowerOfTwo(int value) + { + var power = 1; + while (power < value) + { + power <<= 1; + } + + return power; + } +} + +public sealed record ChirpBinPlan( + int WindowSamples, + int HopSamples, + int TrellisTailFrames, + double EnergyThreshold, + Func, SymbolScore[]> Score, + TryMapTripletDelegate TryMapTriplet); + +public delegate bool TryMapTripletDelegate( + IReadOnlyList frames, + out ulong eventIndex, + out double confidence); + +public sealed record CandidateFrame(long AbsoluteSample, double Energy) +{ + public SymbolScore[] Symbols { get; set; } = []; +} + +public sealed record SymbolScore(int SymbolId, double Score, double FractionalOffset); + +public sealed record TimelineAnchor(ulong EventIndex, long ObservedSample, double Confidence); + +public sealed record ClockFit(double SourceOffsetSamples, double SampleRate, double Confidence); + +public sealed record DecodeSnapshot( + IReadOnlyList Candidates, + IReadOnlyList Anchors, + ClockFit? ClockFit); + +public sealed class Ring +{ + private readonly T[] data; + private long cursor; + + public Ring(int capacity) + { + data = new T[capacity]; + } + + public int Capacity => data.Length; + + public T Latest => data[(cursor - 1) & (data.Length - 1)]; + + public void Push(T value) + { + data[cursor & (data.Length - 1)] = value; + cursor++; + } + + public ReadOnlySpan ReadAbsolute(long absoluteStart, int count) + { + // Sketch simplification: production should copy wraparound into a + // scratch span or expose two spans. + throw new NotImplementedException(); + } +} +