GameCult/Perlines
Watch
9
0
Fork
You've already forked Perlines
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Feed Perlines from Fensalir audio stems

Browse source
This commit is contained in:
Meta Vens 2026-05-31 02:35:46 +01:00
commit 27d0ad669a
3 changed files with 151 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

19
docs/system-map.md
View file

@ -11,15 +11,17 @@ reservoir, denoising, and presentation machinery used by the engine.
## Authority Map
- Owner: `PerlinesSignalProcessor` owns audio synthesis, FFT whitening, key
detection, and projection from frequency bins into musical scale-note lanes.
- Inputs: frame time and delta time from the Fensalir runtime loop.
- Owner: `PerlinesSignalProcessor` owns FFT whitening, key detection, and
projection from frequency bins into musical scale-note lanes.
- Inputs: a Fensalir-owned Windows system-loopback capture stem when available;
otherwise frame time and delta time drive the procedural fallback signal.
- Outputs: one normalized Float32 column of scale-note energy per frame,
transposed into note-owned TubeField splines for rendering.
- Derived state: detected key, flux, tonal center, and energy are diagnostic
readouts derived from the same spectral pass.
- Forbidden writers: render passes, UI controls, and tube lowerings do not
decide spectral content. They only consume the field evidence.
- Forbidden writers: render passes, UI controls, tube lowerings, and audio stem
metadata do not decide spectral content. They only feed or consume the field
evidence path.
- Shared path: visible splines, TubeField evidence, headless captures, and UI
telemetry all read the same rolling note reservoir and version counter.
- Deletion line: Unity trail ownership, arbitrary band-to-spline assignment,
@ -27,7 +29,10 @@ reservoir, denoising, and presentation machinery used by the engine.
## Pipeline
1. Procedural audio is synthesized as a test signal.
1. Perlines requests a system-loopback capture stem from the Fensalir audio
host, drains published audio stem frames when available, and mixes their
channels to a mono analysis window. If no stem frame arrives, a procedural
signal is synthesized as a test fallback.
2. A Hann-windowed FFT measures the current frame.
3. Each FFT bin is divided by a smoothed envelope, producing adaptive whitening
so persistent loud bands stop monopolizing the field.
@ -49,6 +54,8 @@ reservoir, denoising, and presentation machinery used by the engine.
- Musical lanes are keyed scale notes, not arbitrary FFT bins.
- Each TubeField spline belongs to one detected-key note lane.
- Live audio stems are inputs to the same spectral owner as the fallback signal;
they do not bypass whitening, key detection, or lane projection.
- Perlines uploads a compact 70 by 96 Float32 resource, not a native-resolution
buffer.
- The showy dance splines are derived from the same note reservoir. They do not

83
src/Perlines/PerlinesRuntime.cs
View file

@ -7,7 +7,7 @@ using Aquarium.Engine.Ui;
namespace Perlines;
public sealed class PerlinesRuntime : IAquariumRuntime
public sealed class PerlinesRuntime : IAquariumRuntime, IAquariumRuntimeServicesReceiver
{
private const int FftSize = 2048;
private const int NoteLaneCount = 70;
@ -17,6 +17,7 @@ public sealed class PerlinesRuntime : IAquariumRuntime
private const string ClaimKey = "claim:perlines:whitened-fft:tube";
private const string CandidateKey = "candidate:perlines:whitened-fft:tube";
private const string ProducerKey = "perlines:adaptive-whitener";
private const string LoopbackProfileId = "perlines:system-loopback";
private readonly PerlinesSignalProcessor signal = new(FftSize, NoteLaneCount, sampleRate: 48000.0f);
private readonly SmoothNoise danceNoise = new(0xDA7A);
@ -24,6 +25,8 @@ public sealed class PerlinesRuntime : IAquariumRuntime
private readonly float[] rollingNotes = new float[NoteLaneCount * HistoryColumns];
private readonly float[] tubeSamples = new float[NoteLaneCount * HistoryColumns];
private readonly float[] uploadColumn = new float[NoteLaneCount];
private AquariumRuntimeServices services = AquariumRuntimeServices.Empty;
private float[] stemMixBuffer = [];
private float timeSeconds;
private float previousTimeSeconds;
private float cameraYaw;
@ -31,8 +34,11 @@ public sealed class PerlinesRuntime : IAquariumRuntime
private float cameraDistance = 7.8f;
private int rollingColumn;
private ulong resourceVersion = 1;
private long consumedAudioSequence = -1;
private bool autoOrbit = true;
private bool freezeSignal;
private bool usedAudioStemThisFrame;
private string audioInputLabel = "synthetic";
public AquariumRuntimeOptions Options { get; private set; }
@ -79,6 +85,7 @@ public sealed class PerlinesRuntime : IAquariumRuntime
panel.Slider("Camera Distance", () => cameraDistance, value => cameraDistance = Math.Clamp(value, 4.5f, 13.0f), 4.5f, 13.0f, "0.00");
panel.Readout("FFT", () => $"{FftSize} samples / {NoteLaneCount} scale-note lanes / v{resourceVersion}");
panel.Readout("Key", () => signal.DetectedKeyName);
panel.Readout("Input", () => audioInputLabel);
panel.Readout("Whitening", () => $"flux {signal.LastFlux:0.000} / tonal center {signal.LastCentroid:0.000} / energy {signal.LastEnergy:0.000}");
panel.Readout("Renderer", () => "Fensalir TubeField + shared reservoir + final reconstruction");
})
@ -92,6 +99,12 @@ public sealed class PerlinesRuntime : IAquariumRuntime
public void Start()
{
Audio.EnqueueSystemLoopbackCapture(
LoopbackProfileId,
"windows-default-render",
"System mix",
enabled: true,
sequence: 1);
Console.WriteLine("Perlines adaptive FFT tube field booted.");
}
@ -103,7 +116,13 @@ public sealed class PerlinesRuntime : IAquariumRuntime
if (!freezeSignal)
{
signal.Advance(timeSeconds, safeDelta, uploadColumn);
usedAudioStemThisFrame = TryAdvanceFromAudioStem(safeDelta, uploadColumn);
if (!usedAudioStemThisFrame)
{
signal.Advance(timeSeconds, safeDelta, uploadColumn);
audioInputLabel = "synthetic fallback";
}
rollingColumn = (rollingColumn + 1) % HistoryColumns;
Array.Copy(uploadColumn, 0, rollingNotes, rollingColumn * NoteLaneCount, NoteLaneCount);
resourceVersion++;
@ -159,6 +178,11 @@ public sealed class PerlinesRuntime : IAquariumRuntime
{
}
public void AttachServices(AquariumRuntimeServices runtimeServices)
{
services = runtimeServices;
}
internal void SetOptions(AquariumRuntimeOptions options)
{
Options = options;
@ -305,6 +329,61 @@ public sealed class PerlinesRuntime : IAquariumRuntime
};
}
private bool TryAdvanceFromAudioStem(float deltaSeconds, Span<float> output)
{
var frame = services.AudioStems
.DrainPublishedFrames(maxFrames: 16)
.Where(candidate => candidate.Channels.Count > 0 && candidate.FrameCount > 0 && candidate.SampleRate > 0)
.OrderBy(candidate => candidate.Sequence)
.LastOrDefault();
if (frame is null || frame.Sequence == consumedAudioSequence)
{
return false;
}
consumedAudioSequence = frame.Sequence;
var frameCount = frame.FrameCount;
if (stemMixBuffer.Length < frameCount)
{
stemMixBuffer = new float[frameCount];
}
Array.Clear(stemMixBuffer, 0, frameCount);
var mixedChannels = 0;
foreach (var channel in frame.Channels)
{
if (channel.Samples.Length == 0)
{
continue;
}
var count = Math.Min(frameCount, channel.Samples.Length);
for (var index = 0; index < count; index++)
{
stemMixBuffer[index] += channel.Samples[index];
}
mixedChannels++;
}
if (mixedChannels == 0)
{
return false;
}
var scale = 1.0f / mixedChannels;
for (var index = 0; index < frameCount; index++)
{
stemMixBuffer[index] = Math.Clamp(stemMixBuffer[index] * scale, -1.0f, 1.0f);
}
signal.AdvanceFromSamples(stemMixBuffer.AsSpan(0, frameCount), frame.SampleRate, deltaSeconds, output);
var stem = frame.Channels[0];
var source = string.IsNullOrWhiteSpace(stem.DisplayName) ? stem.StemId : stem.DisplayName;
audioInputLabel = $"{frame.ProfileId} / {source} / seq {frame.Sequence}";
return true;
}
private AquariumSplineFrame BuildDanceSplineFrame()
{
var splines = new List<AquariumSpline3D>(NoteLaneCount);

58
src/Perlines/PerlinesSignalProcessor.cs
View file

@ -12,7 +12,7 @@ internal sealed class PerlinesSignalProcessor
private readonly int fftSize;
private readonly int laneCount;
private readonly float sampleRate;
private float sampleRate;
private readonly int binCount;
private readonly float[] window;
private readonly float[] envelope;
@ -20,6 +20,7 @@ internal sealed class PerlinesSignalProcessor
private readonly float[] chroma;
private readonly float[] smoothedChroma;
private readonly float[] previousLanes;
private readonly float[] externalSamples;
private readonly Complex[] spectrum;
private readonly SmoothNoise noise = new(0x5EED);
private float phaseA;
@ -27,6 +28,8 @@ internal sealed class PerlinesSignalProcessor
private float phaseC;
private int detectedRoot = 9;
private bool detectedMinor;
private int externalWriteIndex;
private int externalSampleCount;
public float LastEnergy { get; private set; }
@ -53,6 +56,7 @@ internal sealed class PerlinesSignalProcessor
chroma = new float[12];
smoothedChroma = new float[12];
previousLanes = new float[laneCount];
externalSamples = new float[fftSize];
spectrum = new Complex[fftSize];
Array.Fill(envelope, 0.08f);
@ -70,6 +74,30 @@ internal sealed class PerlinesSignalProcessor
}
FillSignal(timeSeconds);
Analyze(deltaSeconds, output);
}
public void AdvanceFromSamples(ReadOnlySpan<float> samples, int inputSampleRate, float deltaSeconds, Span<float> output)
{
if (output.Length < laneCount)
{
throw new ArgumentException("Output span is smaller than the configured lane count.", nameof(output));
}
if (samples.IsEmpty || inputSampleRate <= 0)
{
Advance(0.0f, deltaSeconds, output);
return;
}
sampleRate = inputSampleRate;
PushExternalSamples(samples);
FillExternalWindow();
Analyze(deltaSeconds, output);
}
private void Analyze(float deltaSeconds, Span<float> output)
{
Fft(spectrum);
WhitenBins(Math.Max(deltaSeconds, 1.0f / 240.0f));
DetectKey();
@ -100,6 +128,34 @@ internal sealed class PerlinesSignalProcessor
}
}
private void PushExternalSamples(ReadOnlySpan<float> samples)
{
foreach (var sample in samples)
{
externalSamples[externalWriteIndex] = Math.Clamp(sample, -1.0f, 1.0f);
externalWriteIndex = (externalWriteIndex + 1) % fftSize;
externalSampleCount = Math.Min(externalSampleCount + 1, fftSize);
}
}
private void FillExternalWindow()
{
var available = Math.Max(0, Math.Min(externalSampleCount, fftSize));
var first = (externalWriteIndex - available + fftSize) % fftSize;
var pad = fftSize - available;
for (var index = 0; index < pad; index++)
{
spectrum[index] = Complex.Zero;
}
for (var index = 0; index < available; index++)
{
var sourceIndex = (first + index) % fftSize;
var targetIndex = pad + index;
spectrum[targetIndex] = new Complex(externalSamples[sourceIndex] * window[targetIndex], 0.0f);
}
}
private void WhitenBins(float safeDelta)
{
Array.Clear(chroma);