Feed Perlines from Fensalir audio stems

docs/system-map.md

@@ -11,15 +11,17 @@ reservoir, denoising, and presentation machinery used by the engine.
 
 ## Authority Map
 
-- Owner: `PerlinesSignalProcessor` owns audio synthesis, FFT whitening, key
+- Owner: `PerlinesSignalProcessor` owns FFT whitening, key detection, and
-  detection, and projection from frequency bins into musical scale-note lanes.
+  projection from frequency bins into musical scale-note lanes.
-- Inputs: frame time and delta time from the Fensalir runtime loop.
+- Inputs: a Fensalir-owned Windows system-loopback capture stem when available;
+  otherwise frame time and delta time drive the procedural fallback signal.
 - Outputs: one normalized Float32 column of scale-note energy per frame,
   transposed into note-owned TubeField splines for rendering.
 - Derived state: detected key, flux, tonal center, and energy are diagnostic
   readouts derived from the same spectral pass.
-- Forbidden writers: render passes, UI controls, and tube lowerings do not
+- Forbidden writers: render passes, UI controls, tube lowerings, and audio stem
-  decide spectral content. They only consume the field evidence.
+  metadata do not decide spectral content. They only feed or consume the field
+  evidence path.
 - Shared path: visible splines, TubeField evidence, headless captures, and UI
   telemetry all read the same rolling note reservoir and version counter.
 - Deletion line: Unity trail ownership, arbitrary band-to-spline assignment,
@@ -27,7 +29,10 @@ reservoir, denoising, and presentation machinery used by the engine.
 
 ## Pipeline
 
-1. Procedural audio is synthesized as a test signal.
+1. Perlines requests a system-loopback capture stem from the Fensalir audio
+   host, drains published audio stem frames when available, and mixes their
+   channels to a mono analysis window. If no stem frame arrives, a procedural
+   signal is synthesized as a test fallback.
 2. A Hann-windowed FFT measures the current frame.
 3. Each FFT bin is divided by a smoothed envelope, producing adaptive whitening
    so persistent loud bands stop monopolizing the field.
@@ -49,6 +54,8 @@ reservoir, denoising, and presentation machinery used by the engine.
 
 - Musical lanes are keyed scale notes, not arbitrary FFT bins.
 - Each TubeField spline belongs to one detected-key note lane.
+- Live audio stems are inputs to the same spectral owner as the fallback signal;
+  they do not bypass whitening, key detection, or lane projection.
 - Perlines uploads a compact 70 by 96 Float32 resource, not a native-resolution
   buffer.
 - The showy dance splines are derived from the same note reservoir. They do not

src/Perlines/PerlinesRuntime.cs

@@ -7,7 +7,7 @@ using Aquarium.Engine.Ui;
 
 namespace Perlines;
 
-public sealed class PerlinesRuntime : IAquariumRuntime
+public sealed class PerlinesRuntime : IAquariumRuntime, IAquariumRuntimeServicesReceiver
 {
     private const int FftSize = 2048;
     private const int NoteLaneCount = 70;
@@ -17,6 +17,7 @@ public sealed class PerlinesRuntime : IAquariumRuntime
     private const string ClaimKey = "claim:perlines:whitened-fft:tube";
     private const string CandidateKey = "candidate:perlines:whitened-fft:tube";
     private const string ProducerKey = "perlines:adaptive-whitener";
+    private const string LoopbackProfileId = "perlines:system-loopback";
 
     private readonly PerlinesSignalProcessor signal = new(FftSize, NoteLaneCount, sampleRate: 48000.0f);
     private readonly SmoothNoise danceNoise = new(0xDA7A);
@@ -24,6 +25,8 @@ public sealed class PerlinesRuntime : IAquariumRuntime
     private readonly float[] rollingNotes = new float[NoteLaneCount * HistoryColumns];
     private readonly float[] tubeSamples = new float[NoteLaneCount * HistoryColumns];
     private readonly float[] uploadColumn = new float[NoteLaneCount];
+    private AquariumRuntimeServices services = AquariumRuntimeServices.Empty;
+    private float[] stemMixBuffer = [];
     private float timeSeconds;
     private float previousTimeSeconds;
     private float cameraYaw;
@@ -31,8 +34,11 @@ public sealed class PerlinesRuntime : IAquariumRuntime
     private float cameraDistance = 7.8f;
     private int rollingColumn;
     private ulong resourceVersion = 1;
+    private long consumedAudioSequence = -1;
     private bool autoOrbit = true;
     private bool freezeSignal;
+    private bool usedAudioStemThisFrame;
+    private string audioInputLabel = "synthetic";
 
     public AquariumRuntimeOptions Options { get; private set; }
 
@@ -79,6 +85,7 @@ public sealed class PerlinesRuntime : IAquariumRuntime
                 panel.Slider("Camera Distance", () => cameraDistance, value => cameraDistance = Math.Clamp(value, 4.5f, 13.0f), 4.5f, 13.0f, "0.00");
                 panel.Readout("FFT", () => $"{FftSize} samples / {NoteLaneCount} scale-note lanes / v{resourceVersion}");
                 panel.Readout("Key", () => signal.DetectedKeyName);
+                panel.Readout("Input", () => audioInputLabel);
                 panel.Readout("Whitening", () => $"flux {signal.LastFlux:0.000} / tonal center {signal.LastCentroid:0.000} / energy {signal.LastEnergy:0.000}");
                 panel.Readout("Renderer", () => "Fensalir TubeField + shared reservoir + final reconstruction");
             })
@@ -92,6 +99,12 @@ public sealed class PerlinesRuntime : IAquariumRuntime
 
     public void Start()
     {
+        Audio.EnqueueSystemLoopbackCapture(
+            LoopbackProfileId,
+            "windows-default-render",
+            "System mix",
+            enabled: true,
+            sequence: 1);
         Console.WriteLine("Perlines adaptive FFT tube field booted.");
     }
 
@@ -103,7 +116,13 @@ public sealed class PerlinesRuntime : IAquariumRuntime
 
         if (!freezeSignal)
         {
-            signal.Advance(timeSeconds, safeDelta, uploadColumn);
+            usedAudioStemThisFrame = TryAdvanceFromAudioStem(safeDelta, uploadColumn);
+            if (!usedAudioStemThisFrame)
+            {
+                signal.Advance(timeSeconds, safeDelta, uploadColumn);
+                audioInputLabel = "synthetic fallback";
+            }
+
             rollingColumn = (rollingColumn + 1) % HistoryColumns;
             Array.Copy(uploadColumn, 0, rollingNotes, rollingColumn * NoteLaneCount, NoteLaneCount);
             resourceVersion++;
@@ -159,6 +178,11 @@ public sealed class PerlinesRuntime : IAquariumRuntime
     {
     }
 
+    public void AttachServices(AquariumRuntimeServices runtimeServices)
+    {
+        services = runtimeServices;
+    }
+
     internal void SetOptions(AquariumRuntimeOptions options)
     {
         Options = options;
@@ -305,6 +329,61 @@ public sealed class PerlinesRuntime : IAquariumRuntime
         };
     }
 
+    private bool TryAdvanceFromAudioStem(float deltaSeconds, Span<float> output)
+    {
+        var frame = services.AudioStems
+            .DrainPublishedFrames(maxFrames: 16)
+            .Where(candidate => candidate.Channels.Count > 0 && candidate.FrameCount > 0 && candidate.SampleRate > 0)
+            .OrderBy(candidate => candidate.Sequence)
+            .LastOrDefault();
+        if (frame is null || frame.Sequence == consumedAudioSequence)
+        {
+            return false;
+        }
+
+        consumedAudioSequence = frame.Sequence;
+        var frameCount = frame.FrameCount;
+        if (stemMixBuffer.Length < frameCount)
+        {
+            stemMixBuffer = new float[frameCount];
+        }
+
+        Array.Clear(stemMixBuffer, 0, frameCount);
+        var mixedChannels = 0;
+        foreach (var channel in frame.Channels)
+        {
+            if (channel.Samples.Length == 0)
+            {
+                continue;
+            }
+
+            var count = Math.Min(frameCount, channel.Samples.Length);
+            for (var index = 0; index < count; index++)
+            {
+                stemMixBuffer[index] += channel.Samples[index];
+            }
+
+            mixedChannels++;
+        }
+
+        if (mixedChannels == 0)
+        {
+            return false;
+        }
+
+        var scale = 1.0f / mixedChannels;
+        for (var index = 0; index < frameCount; index++)
+        {
+            stemMixBuffer[index] = Math.Clamp(stemMixBuffer[index] * scale, -1.0f, 1.0f);
+        }
+
+        signal.AdvanceFromSamples(stemMixBuffer.AsSpan(0, frameCount), frame.SampleRate, deltaSeconds, output);
+        var stem = frame.Channels[0];
+        var source = string.IsNullOrWhiteSpace(stem.DisplayName) ? stem.StemId : stem.DisplayName;
+        audioInputLabel = $"{frame.ProfileId} / {source} / seq {frame.Sequence}";
+        return true;
+    }
+
     private AquariumSplineFrame BuildDanceSplineFrame()
     {
         var splines = new List<AquariumSpline3D>(NoteLaneCount);

src/Perlines/PerlinesSignalProcessor.cs

@@ -12,7 +12,7 @@ internal sealed class PerlinesSignalProcessor
 
     private readonly int fftSize;
     private readonly int laneCount;
-    private readonly float sampleRate;
+    private float sampleRate;
     private readonly int binCount;
     private readonly float[] window;
     private readonly float[] envelope;
@@ -20,6 +20,7 @@ internal sealed class PerlinesSignalProcessor
     private readonly float[] chroma;
     private readonly float[] smoothedChroma;
     private readonly float[] previousLanes;
+    private readonly float[] externalSamples;
     private readonly Complex[] spectrum;
     private readonly SmoothNoise noise = new(0x5EED);
     private float phaseA;
@@ -27,6 +28,8 @@ internal sealed class PerlinesSignalProcessor
     private float phaseC;
     private int detectedRoot = 9;
     private bool detectedMinor;
+    private int externalWriteIndex;
+    private int externalSampleCount;
 
     public float LastEnergy { get; private set; }
 
@@ -53,6 +56,7 @@ internal sealed class PerlinesSignalProcessor
         chroma = new float[12];
         smoothedChroma = new float[12];
         previousLanes = new float[laneCount];
+        externalSamples = new float[fftSize];
         spectrum = new Complex[fftSize];
         Array.Fill(envelope, 0.08f);
 
@@ -70,6 +74,30 @@ internal sealed class PerlinesSignalProcessor
         }
 
         FillSignal(timeSeconds);
+        Analyze(deltaSeconds, output);
+    }
+
+    public void AdvanceFromSamples(ReadOnlySpan<float> samples, int inputSampleRate, float deltaSeconds, Span<float> output)
+    {
+        if (output.Length < laneCount)
+        {
+            throw new ArgumentException("Output span is smaller than the configured lane count.", nameof(output));
+        }
+
+        if (samples.IsEmpty || inputSampleRate <= 0)
+        {
+            Advance(0.0f, deltaSeconds, output);
+            return;
+        }
+
+        sampleRate = inputSampleRate;
+        PushExternalSamples(samples);
+        FillExternalWindow();
+        Analyze(deltaSeconds, output);
+    }
+
+    private void Analyze(float deltaSeconds, Span<float> output)
+    {
         Fft(spectrum);
         WhitenBins(Math.Max(deltaSeconds, 1.0f / 240.0f));
         DetectKey();
@@ -100,6 +128,34 @@ internal sealed class PerlinesSignalProcessor
         }
     }
 
+    private void PushExternalSamples(ReadOnlySpan<float> samples)
+    {
+        foreach (var sample in samples)
+        {
+            externalSamples[externalWriteIndex] = Math.Clamp(sample, -1.0f, 1.0f);
+            externalWriteIndex = (externalWriteIndex + 1) % fftSize;
+            externalSampleCount = Math.Min(externalSampleCount + 1, fftSize);
+        }
+    }
+
+    private void FillExternalWindow()
+    {
+        var available = Math.Max(0, Math.Min(externalSampleCount, fftSize));
+        var first = (externalWriteIndex - available + fftSize) % fftSize;
+        var pad = fftSize - available;
+        for (var index = 0; index < pad; index++)
+        {
+            spectrum[index] = Complex.Zero;
+        }
+
+        for (var index = 0; index < available; index++)
+        {
+            var sourceIndex = (first + index) % fftSize;
+            var targetIndex = pad + index;
+            spectrum[targetIndex] = new Complex(externalSamples[sourceIndex] * window[targetIndex], 0.0f);
+        }
+    }
+
     private void WhitenBins(float safeDelta)
     {
         Array.Clear(chroma);