From 27d0ad669a345422b2f583db90accbad2bee7910 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 31 May 2026 02:35:46 +0100 Subject: [PATCH] Feed Perlines from Fensalir audio stems --- docs/system-map.md | 19 ++++-- src/Perlines/PerlinesRuntime.cs | 83 ++++++++++++++++++++++++- src/Perlines/PerlinesSignalProcessor.cs | 58 ++++++++++++++++- 3 files changed, 151 insertions(+), 9 deletions(-) diff --git a/docs/system-map.md b/docs/system-map.md index 85c545f..51b46c9 100644 --- a/docs/system-map.md +++ b/docs/system-map.md @@ -11,15 +11,17 @@ reservoir, denoising, and presentation machinery used by the engine. ## Authority Map -- Owner: `PerlinesSignalProcessor` owns audio synthesis, FFT whitening, key - detection, and projection from frequency bins into musical scale-note lanes. -- Inputs: frame time and delta time from the Fensalir runtime loop. +- Owner: `PerlinesSignalProcessor` owns FFT whitening, key detection, and + projection from frequency bins into musical scale-note lanes. +- Inputs: a Fensalir-owned Windows system-loopback capture stem when available; + otherwise frame time and delta time drive the procedural fallback signal. - Outputs: one normalized Float32 column of scale-note energy per frame, transposed into note-owned TubeField splines for rendering. - Derived state: detected key, flux, tonal center, and energy are diagnostic readouts derived from the same spectral pass. -- Forbidden writers: render passes, UI controls, and tube lowerings do not - decide spectral content. They only consume the field evidence. +- Forbidden writers: render passes, UI controls, tube lowerings, and audio stem + metadata do not decide spectral content. They only feed or consume the field + evidence path. - Shared path: visible splines, TubeField evidence, headless captures, and UI telemetry all read the same rolling note reservoir and version counter. - Deletion line: Unity trail ownership, arbitrary band-to-spline assignment, @@ -27,7 +29,10 @@ reservoir, denoising, and presentation machinery used by the engine. ## Pipeline -1. Procedural audio is synthesized as a test signal. +1. Perlines requests a system-loopback capture stem from the Fensalir audio + host, drains published audio stem frames when available, and mixes their + channels to a mono analysis window. If no stem frame arrives, a procedural + signal is synthesized as a test fallback. 2. A Hann-windowed FFT measures the current frame. 3. Each FFT bin is divided by a smoothed envelope, producing adaptive whitening so persistent loud bands stop monopolizing the field. @@ -49,6 +54,8 @@ reservoir, denoising, and presentation machinery used by the engine. - Musical lanes are keyed scale notes, not arbitrary FFT bins. - Each TubeField spline belongs to one detected-key note lane. +- Live audio stems are inputs to the same spectral owner as the fallback signal; + they do not bypass whitening, key detection, or lane projection. - Perlines uploads a compact 70 by 96 Float32 resource, not a native-resolution buffer. - The showy dance splines are derived from the same note reservoir. They do not diff --git a/src/Perlines/PerlinesRuntime.cs b/src/Perlines/PerlinesRuntime.cs index 8ae4e40..24ce775 100644 --- a/src/Perlines/PerlinesRuntime.cs +++ b/src/Perlines/PerlinesRuntime.cs @@ -7,7 +7,7 @@ using Aquarium.Engine.Ui; namespace Perlines; -public sealed class PerlinesRuntime : IAquariumRuntime +public sealed class PerlinesRuntime : IAquariumRuntime, IAquariumRuntimeServicesReceiver { private const int FftSize = 2048; private const int NoteLaneCount = 70; @@ -17,6 +17,7 @@ public sealed class PerlinesRuntime : IAquariumRuntime private const string ClaimKey = "claim:perlines:whitened-fft:tube"; private const string CandidateKey = "candidate:perlines:whitened-fft:tube"; private const string ProducerKey = "perlines:adaptive-whitener"; + private const string LoopbackProfileId = "perlines:system-loopback"; private readonly PerlinesSignalProcessor signal = new(FftSize, NoteLaneCount, sampleRate: 48000.0f); private readonly SmoothNoise danceNoise = new(0xDA7A); @@ -24,6 +25,8 @@ public sealed class PerlinesRuntime : IAquariumRuntime private readonly float[] rollingNotes = new float[NoteLaneCount * HistoryColumns]; private readonly float[] tubeSamples = new float[NoteLaneCount * HistoryColumns]; private readonly float[] uploadColumn = new float[NoteLaneCount]; + private AquariumRuntimeServices services = AquariumRuntimeServices.Empty; + private float[] stemMixBuffer = []; private float timeSeconds; private float previousTimeSeconds; private float cameraYaw; @@ -31,8 +34,11 @@ public sealed class PerlinesRuntime : IAquariumRuntime private float cameraDistance = 7.8f; private int rollingColumn; private ulong resourceVersion = 1; + private long consumedAudioSequence = -1; private bool autoOrbit = true; private bool freezeSignal; + private bool usedAudioStemThisFrame; + private string audioInputLabel = "synthetic"; public AquariumRuntimeOptions Options { get; private set; } @@ -79,6 +85,7 @@ public sealed class PerlinesRuntime : IAquariumRuntime panel.Slider("Camera Distance", () => cameraDistance, value => cameraDistance = Math.Clamp(value, 4.5f, 13.0f), 4.5f, 13.0f, "0.00"); panel.Readout("FFT", () => $"{FftSize} samples / {NoteLaneCount} scale-note lanes / v{resourceVersion}"); panel.Readout("Key", () => signal.DetectedKeyName); + panel.Readout("Input", () => audioInputLabel); panel.Readout("Whitening", () => $"flux {signal.LastFlux:0.000} / tonal center {signal.LastCentroid:0.000} / energy {signal.LastEnergy:0.000}"); panel.Readout("Renderer", () => "Fensalir TubeField + shared reservoir + final reconstruction"); }) @@ -92,6 +99,12 @@ public sealed class PerlinesRuntime : IAquariumRuntime public void Start() { + Audio.EnqueueSystemLoopbackCapture( + LoopbackProfileId, + "windows-default-render", + "System mix", + enabled: true, + sequence: 1); Console.WriteLine("Perlines adaptive FFT tube field booted."); } @@ -103,7 +116,13 @@ public sealed class PerlinesRuntime : IAquariumRuntime if (!freezeSignal) { - signal.Advance(timeSeconds, safeDelta, uploadColumn); + usedAudioStemThisFrame = TryAdvanceFromAudioStem(safeDelta, uploadColumn); + if (!usedAudioStemThisFrame) + { + signal.Advance(timeSeconds, safeDelta, uploadColumn); + audioInputLabel = "synthetic fallback"; + } + rollingColumn = (rollingColumn + 1) % HistoryColumns; Array.Copy(uploadColumn, 0, rollingNotes, rollingColumn * NoteLaneCount, NoteLaneCount); resourceVersion++; @@ -159,6 +178,11 @@ public sealed class PerlinesRuntime : IAquariumRuntime { } + public void AttachServices(AquariumRuntimeServices runtimeServices) + { + services = runtimeServices; + } + internal void SetOptions(AquariumRuntimeOptions options) { Options = options; @@ -305,6 +329,61 @@ public sealed class PerlinesRuntime : IAquariumRuntime }; } + private bool TryAdvanceFromAudioStem(float deltaSeconds, Span output) + { + var frame = services.AudioStems + .DrainPublishedFrames(maxFrames: 16) + .Where(candidate => candidate.Channels.Count > 0 && candidate.FrameCount > 0 && candidate.SampleRate > 0) + .OrderBy(candidate => candidate.Sequence) + .LastOrDefault(); + if (frame is null || frame.Sequence == consumedAudioSequence) + { + return false; + } + + consumedAudioSequence = frame.Sequence; + var frameCount = frame.FrameCount; + if (stemMixBuffer.Length < frameCount) + { + stemMixBuffer = new float[frameCount]; + } + + Array.Clear(stemMixBuffer, 0, frameCount); + var mixedChannels = 0; + foreach (var channel in frame.Channels) + { + if (channel.Samples.Length == 0) + { + continue; + } + + var count = Math.Min(frameCount, channel.Samples.Length); + for (var index = 0; index < count; index++) + { + stemMixBuffer[index] += channel.Samples[index]; + } + + mixedChannels++; + } + + if (mixedChannels == 0) + { + return false; + } + + var scale = 1.0f / mixedChannels; + for (var index = 0; index < frameCount; index++) + { + stemMixBuffer[index] = Math.Clamp(stemMixBuffer[index] * scale, -1.0f, 1.0f); + } + + signal.AdvanceFromSamples(stemMixBuffer.AsSpan(0, frameCount), frame.SampleRate, deltaSeconds, output); + var stem = frame.Channels[0]; + var source = string.IsNullOrWhiteSpace(stem.DisplayName) ? stem.StemId : stem.DisplayName; + audioInputLabel = $"{frame.ProfileId} / {source} / seq {frame.Sequence}"; + return true; + } + private AquariumSplineFrame BuildDanceSplineFrame() { var splines = new List(NoteLaneCount); diff --git a/src/Perlines/PerlinesSignalProcessor.cs b/src/Perlines/PerlinesSignalProcessor.cs index 0168127..b4e2bd0 100644 --- a/src/Perlines/PerlinesSignalProcessor.cs +++ b/src/Perlines/PerlinesSignalProcessor.cs @@ -12,7 +12,7 @@ internal sealed class PerlinesSignalProcessor private readonly int fftSize; private readonly int laneCount; - private readonly float sampleRate; + private float sampleRate; private readonly int binCount; private readonly float[] window; private readonly float[] envelope; @@ -20,6 +20,7 @@ internal sealed class PerlinesSignalProcessor private readonly float[] chroma; private readonly float[] smoothedChroma; private readonly float[] previousLanes; + private readonly float[] externalSamples; private readonly Complex[] spectrum; private readonly SmoothNoise noise = new(0x5EED); private float phaseA; @@ -27,6 +28,8 @@ internal sealed class PerlinesSignalProcessor private float phaseC; private int detectedRoot = 9; private bool detectedMinor; + private int externalWriteIndex; + private int externalSampleCount; public float LastEnergy { get; private set; } @@ -53,6 +56,7 @@ internal sealed class PerlinesSignalProcessor chroma = new float[12]; smoothedChroma = new float[12]; previousLanes = new float[laneCount]; + externalSamples = new float[fftSize]; spectrum = new Complex[fftSize]; Array.Fill(envelope, 0.08f); @@ -70,6 +74,30 @@ internal sealed class PerlinesSignalProcessor } FillSignal(timeSeconds); + Analyze(deltaSeconds, output); + } + + public void AdvanceFromSamples(ReadOnlySpan samples, int inputSampleRate, float deltaSeconds, Span output) + { + if (output.Length < laneCount) + { + throw new ArgumentException("Output span is smaller than the configured lane count.", nameof(output)); + } + + if (samples.IsEmpty || inputSampleRate <= 0) + { + Advance(0.0f, deltaSeconds, output); + return; + } + + sampleRate = inputSampleRate; + PushExternalSamples(samples); + FillExternalWindow(); + Analyze(deltaSeconds, output); + } + + private void Analyze(float deltaSeconds, Span output) + { Fft(spectrum); WhitenBins(Math.Max(deltaSeconds, 1.0f / 240.0f)); DetectKey(); @@ -100,6 +128,34 @@ internal sealed class PerlinesSignalProcessor } } + private void PushExternalSamples(ReadOnlySpan samples) + { + foreach (var sample in samples) + { + externalSamples[externalWriteIndex] = Math.Clamp(sample, -1.0f, 1.0f); + externalWriteIndex = (externalWriteIndex + 1) % fftSize; + externalSampleCount = Math.Min(externalSampleCount + 1, fftSize); + } + } + + private void FillExternalWindow() + { + var available = Math.Max(0, Math.Min(externalSampleCount, fftSize)); + var first = (externalWriteIndex - available + fftSize) % fftSize; + var pad = fftSize - available; + for (var index = 0; index < pad; index++) + { + spectrum[index] = Complex.Zero; + } + + for (var index = 0; index < available; index++) + { + var sourceIndex = (first + index) % fftSize; + var targetIndex = pad + index; + spectrum[targetIndex] = new Complex(externalSamples[sourceIndex] * window[targetIndex], 0.0f); + } + } + private void WhitenBins(float safeDelta) { Array.Clear(chroma);