491 lines
20 KiB
C#
491 lines
20 KiB
C#
using System.Numerics;
|
|
|
|
namespace AquaSynth.Dsl;
|
|
|
|
public sealed record AudioAnalysisConfig(
|
|
float SampleRate = 44100,
|
|
float GateFloor = 0.0005f,
|
|
float GateRatio = 0.03f,
|
|
int FftSize = 512,
|
|
int HopSize = 128,
|
|
int MelBandCount = 32,
|
|
float MinFrequencyHz = 40,
|
|
float MaxFrequencyHz = 16000);
|
|
|
|
public sealed record AudioFeatures(
|
|
float AttackSeconds,
|
|
float DurationSeconds,
|
|
float Peak,
|
|
float Rms,
|
|
float ZeroCrossingRate,
|
|
float SpectralCentroidHz,
|
|
float SpectralRolloffHz);
|
|
|
|
public sealed record Spectrogram(int Frames, int Bands, float[] Values)
|
|
{
|
|
public float At(int frame, int band) => Values[frame * Bands + band];
|
|
}
|
|
|
|
public sealed record AudioAnalysis(AudioFeatures Features, Spectrogram LogMelSpectrogram, float[] RmsEnvelope);
|
|
|
|
public sealed record AudioComparison(
|
|
AudioAnalysis Reference,
|
|
AudioAnalysis Candidate,
|
|
float DurationRatio,
|
|
float RmsRatio,
|
|
float ZeroCrossingRatio,
|
|
float CentroidRatio,
|
|
float EnvelopeDistance,
|
|
float LogMelDistance,
|
|
float LogMelCosineSimilarity,
|
|
AudioArticulationComparison Articulation,
|
|
float Score);
|
|
|
|
public sealed record AudioArticulationComparison(
|
|
float EnvelopeCosineSimilarity,
|
|
float ActiveFrameRatio,
|
|
float SilenceMismatch,
|
|
float EnvelopeFluxRatio,
|
|
float SpectralFluxRatio,
|
|
float MotorBandRatio,
|
|
float SpeechBandRatio,
|
|
float ArticulationScore);
|
|
|
|
public sealed class AudioAnalyzer
|
|
{
|
|
private readonly AudioAnalysisConfig _config;
|
|
private readonly int _fftSize;
|
|
private readonly Complex[] _fftBuffer;
|
|
private readonly float[] _spectrumBuffer;
|
|
private readonly int[] _melEdges;
|
|
|
|
public AudioAnalyzer(AudioAnalysisConfig? config = null)
|
|
{
|
|
_config = config ?? new AudioAnalysisConfig();
|
|
_fftSize = NextPowerOfTwo(Math.Max(32, _config.FftSize));
|
|
_fftBuffer = new Complex[_fftSize];
|
|
_spectrumBuffer = new float[_fftSize / 2 + 1];
|
|
_melEdges = MelBandEdges(Math.Max(1, _config.MelBandCount), _fftSize, _config.SampleRate, _config.MinFrequencyHz, _config.MaxFrequencyHz);
|
|
}
|
|
|
|
public AudioAnalysisConfig Config => _config;
|
|
|
|
public AudioAnalysis Analyze(ReadOnlySpan<float> samples)
|
|
{
|
|
var features = ExtractFeatures(samples);
|
|
var envelope = RmsEnvelope(samples, _fftSize, _config.HopSize);
|
|
var spectrogram = LogMelSpectrogram(samples);
|
|
return new AudioAnalysis(features, spectrogram, envelope);
|
|
}
|
|
|
|
public AudioComparison Compare(ReadOnlySpan<float> referenceSamples, ReadOnlySpan<float> candidateSamples) =>
|
|
FromAnalysis(Analyze(referenceSamples), Analyze(candidateSamples), CompareArticulation(referenceSamples, candidateSamples));
|
|
|
|
private AudioFeatures ExtractFeatures(ReadOnlySpan<float> samples)
|
|
{
|
|
var peak = 0f;
|
|
foreach (var sample in samples) peak = Math.Max(peak, Math.Abs(sample));
|
|
var gate = Math.Max(peak * _config.GateRatio, _config.GateFloor);
|
|
|
|
var first = 0;
|
|
while (first < samples.Length && Math.Abs(samples[first]) < gate) first++;
|
|
if (first >= samples.Length) first = 0;
|
|
|
|
var last = samples.Length - 1;
|
|
while (last > first && Math.Abs(samples[last]) < gate) last--;
|
|
|
|
var active = samples.Length == 0 ? ReadOnlySpan<float>.Empty : samples[first..(last + 1)];
|
|
var durationSeconds = (last - first + (samples.Length == 0 ? 0 : 1)) / Math.Max(1, _config.SampleRate);
|
|
var zeroCrossings = 0;
|
|
for (var i = 1; i < active.Length; i++)
|
|
{
|
|
if (MathF.Sign(active[i - 1]) != MathF.Sign(active[i])) zeroCrossings++;
|
|
}
|
|
|
|
var spectrum = AveragePowerSpectrum(active);
|
|
var shape = SpectralShape(spectrum, _config.SampleRate, 0.85f);
|
|
return new AudioFeatures(
|
|
first / Math.Max(1, _config.SampleRate),
|
|
durationSeconds,
|
|
peak,
|
|
MathF.Sqrt(MeanSquare(active)),
|
|
zeroCrossings / Math.Max(durationSeconds, 1 / _config.SampleRate),
|
|
shape.CentroidHz,
|
|
shape.RolloffHz);
|
|
}
|
|
|
|
private Spectrogram LogMelSpectrogram(ReadOnlySpan<float> samples)
|
|
{
|
|
var bands = Math.Max(1, _config.MelBandCount);
|
|
if (samples.IsEmpty) return new Spectrogram(1, bands, new float[bands]);
|
|
|
|
var values = new List<float>();
|
|
var frames = 0;
|
|
for (var start = 0; start < samples.Length; start += Math.Max(1, _config.HopSize))
|
|
{
|
|
WriteFramePowerSpectrum(samples, start);
|
|
for (var band = 0; band < bands; band++)
|
|
{
|
|
var left = _melEdges[band];
|
|
var center = Math.Max(_melEdges[band + 1], left + 1);
|
|
var right = Math.Max(_melEdges[band + 2], center + 1);
|
|
var energy = 0f;
|
|
var weightSum = 0f;
|
|
for (var bin = left; bin < Math.Min(right, _spectrumBuffer.Length); bin++)
|
|
{
|
|
var weight = Math.Max(0, bin <= center
|
|
? (bin - left) / (float)Math.Max(1, center - left)
|
|
: (right - bin) / (float)Math.Max(1, right - center));
|
|
energy += _spectrumBuffer[bin] * weight;
|
|
weightSum += weight;
|
|
}
|
|
values.Add(MathF.Log(energy / Math.Max(1, weightSum) + 1e-9f));
|
|
}
|
|
frames++;
|
|
}
|
|
|
|
var array = values.ToArray();
|
|
NormalizeInPlace(array);
|
|
return new Spectrogram(frames, bands, array);
|
|
}
|
|
|
|
private float[] AveragePowerSpectrum(ReadOnlySpan<float> samples)
|
|
{
|
|
var sum = new float[_fftSize / 2 + 1];
|
|
if (samples.IsEmpty) return sum;
|
|
|
|
var frames = 0f;
|
|
for (var start = 0; start < samples.Length; start += Math.Max(1, _fftSize / 2))
|
|
{
|
|
WriteFramePowerSpectrum(samples, start);
|
|
for (var i = 0; i < sum.Length; i++) sum[i] += _spectrumBuffer[i];
|
|
frames++;
|
|
}
|
|
|
|
for (var i = 0; i < sum.Length; i++) sum[i] /= frames;
|
|
return sum;
|
|
}
|
|
|
|
private void WriteFramePowerSpectrum(ReadOnlySpan<float> samples, int start)
|
|
{
|
|
for (var i = 0; i < _fftSize; i++)
|
|
{
|
|
var sample = start + i < samples.Length ? samples[start + i] : 0;
|
|
_fftBuffer[i] = new Complex(sample * Hann(i, _fftSize), 0);
|
|
}
|
|
|
|
Fft(_fftBuffer);
|
|
for (var bin = 0; bin < _spectrumBuffer.Length; bin++)
|
|
{
|
|
_spectrumBuffer[bin] = (float)(_fftBuffer[bin].Real * _fftBuffer[bin].Real + _fftBuffer[bin].Imaginary * _fftBuffer[bin].Imaginary);
|
|
}
|
|
}
|
|
|
|
public static AudioAnalysis AnalyzeAudio(ReadOnlySpan<float> samples, AudioAnalysisConfig? config = null) =>
|
|
new AudioAnalyzer(config).Analyze(samples);
|
|
|
|
public static AudioComparison CompareAudio(ReadOnlySpan<float> referenceSamples, ReadOnlySpan<float> candidateSamples, AudioAnalysisConfig? config = null) =>
|
|
new AudioAnalyzer(config).Compare(referenceSamples, candidateSamples);
|
|
|
|
private static AudioComparison FromAnalysis(AudioAnalysis reference, AudioAnalysis candidate, AudioArticulationComparison articulation)
|
|
{
|
|
var durationRatio = SafeRatio(candidate.Features.DurationSeconds, reference.Features.DurationSeconds);
|
|
var rmsRatio = SafeRatio(candidate.Features.Rms, reference.Features.Rms);
|
|
var zeroCrossingRatio = SafeRatio(candidate.Features.ZeroCrossingRate, reference.Features.ZeroCrossingRate);
|
|
var centroidRatio = SafeRatio(candidate.Features.SpectralCentroidHz, reference.Features.SpectralCentroidHz);
|
|
var envelopeDistance = NormalizedDistance(reference.RmsEnvelope, candidate.RmsEnvelope);
|
|
var logMelDistance = NormalizedDistance(reference.LogMelSpectrogram.Values, candidate.LogMelSpectrogram.Values);
|
|
var logMelCosine = CosineSimilarity(reference.LogMelSpectrogram.Values, candidate.LogMelSpectrogram.Values);
|
|
var ratioPenalty = RatioDistance(durationRatio) + RatioDistance(rmsRatio) + RatioDistance(zeroCrossingRatio) * 0.5f + RatioDistance(centroidRatio) * 0.5f;
|
|
var score = 1 / (1 + envelopeDistance * 0.7f + logMelDistance + ratioPenalty);
|
|
return new AudioComparison(reference, candidate, durationRatio, rmsRatio, zeroCrossingRatio, centroidRatio, envelopeDistance, logMelDistance, logMelCosine, articulation, score);
|
|
}
|
|
|
|
private AudioArticulationComparison CompareArticulation(ReadOnlySpan<float> referenceSamples, ReadOnlySpan<float> candidateSamples)
|
|
{
|
|
var reference = ArticulationFrames(referenceSamples);
|
|
var candidate = ArticulationFrames(candidateSamples);
|
|
var envelopeCosine = CosineSimilarity(reference.Envelope, candidate.Envelope);
|
|
var activeFrameRatio = SafeRatio(candidate.ActiveRatio, reference.ActiveRatio);
|
|
var silenceMismatch = BinaryMismatch(reference.Active, candidate.Active);
|
|
var envelopeFluxRatio = SafeRatio(candidate.EnvelopeFlux, reference.EnvelopeFlux);
|
|
var spectralFluxRatio = SafeRatio(candidate.SpectralFlux, reference.SpectralFlux);
|
|
var motorBandRatio = SafeRatio(candidate.MotorBandShare, reference.MotorBandShare);
|
|
var speechBandRatio = SafeRatio(candidate.SpeechBandShare, reference.SpeechBandShare);
|
|
var penalty =
|
|
(1 - Math.Clamp(envelopeCosine, 0, 1)) * 1.4f +
|
|
silenceMismatch * 1.7f +
|
|
RatioDistance(activeFrameRatio) * 0.9f +
|
|
RatioDistance(envelopeFluxRatio) * 0.7f +
|
|
RatioDistance(spectralFluxRatio) * 0.7f +
|
|
RatioDistance(motorBandRatio) * 0.6f +
|
|
RatioDistance(speechBandRatio) * 0.8f;
|
|
var score = 1 / (1 + penalty);
|
|
return new AudioArticulationComparison(
|
|
envelopeCosine,
|
|
activeFrameRatio,
|
|
silenceMismatch,
|
|
envelopeFluxRatio,
|
|
spectralFluxRatio,
|
|
motorBandRatio,
|
|
speechBandRatio,
|
|
score);
|
|
}
|
|
|
|
private ArticulationFrameSet ArticulationFrames(ReadOnlySpan<float> samples)
|
|
{
|
|
const int frameSize = 1024;
|
|
const int hopSize = 256;
|
|
if (samples.IsEmpty) return new ArticulationFrameSet([0], [false], 0, 0, 0, 0, 0);
|
|
|
|
var envelope = new List<float>();
|
|
var active = new List<bool>();
|
|
var speechBand = 0f;
|
|
var motorBand = 0f;
|
|
var totalBand = 0f;
|
|
var spectralFlux = 0f;
|
|
var previousSpectrum = Array.Empty<float>();
|
|
var peak = 0f;
|
|
foreach (var sample in samples) peak = Math.Max(peak, Math.Abs(sample));
|
|
var gate = Math.Max(peak * 0.035f, _config.GateFloor);
|
|
|
|
for (var start = 0; start < samples.Length; start += hopSize)
|
|
{
|
|
WriteFramePowerSpectrum(samples, start);
|
|
var end = Math.Min(start + frameSize, samples.Length);
|
|
var rms = MathF.Sqrt(MeanSquare(samples[start..end]));
|
|
envelope.Add(MathF.Log(1e-7f + rms));
|
|
active.Add(rms >= gate);
|
|
|
|
var current = _spectrumBuffer.ToArray();
|
|
if (previousSpectrum.Length == current.Length)
|
|
{
|
|
spectralFlux += PositiveSpectralFlux(previousSpectrum, current);
|
|
}
|
|
previousSpectrum = current;
|
|
|
|
for (var bin = 0; bin < current.Length; bin++)
|
|
{
|
|
var frequency = bin * _config.SampleRate / Math.Max(1, _fftSize);
|
|
var energy = current[bin];
|
|
totalBand += energy;
|
|
if (frequency is >= 80 and < 200) motorBand += energy;
|
|
if (frequency is >= 500 and < 2500) speechBand += energy;
|
|
}
|
|
}
|
|
|
|
var fluxFrames = Math.Max(1, envelope.Count - 1);
|
|
return new ArticulationFrameSet(
|
|
envelope.ToArray(),
|
|
active.ToArray(),
|
|
active.Count(item => item) / (float)Math.Max(1, active.Count),
|
|
EnvelopeFlux(envelope),
|
|
spectralFlux / fluxFrames,
|
|
motorBand / Math.Max(float.Epsilon, totalBand),
|
|
speechBand / Math.Max(float.Epsilon, totalBand));
|
|
}
|
|
|
|
private static void Fft(Complex[] buffer)
|
|
{
|
|
var n = buffer.Length;
|
|
for (var i = 1; i < n; i++)
|
|
{
|
|
var bit = n >> 1;
|
|
var j = i;
|
|
while ((j & bit) != 0) j ^= bit;
|
|
j ^= bit;
|
|
if (i < j) (buffer[i], buffer[j]) = (buffer[j], buffer[i]);
|
|
}
|
|
|
|
for (var len = 2; len <= n; len <<= 1)
|
|
{
|
|
var angle = -2 * Math.PI / len;
|
|
var wLen = new Complex(Math.Cos(angle), Math.Sin(angle));
|
|
for (var i = 0; i < n; i += len)
|
|
{
|
|
var w = Complex.One;
|
|
for (var j = 0; j < len / 2; j++)
|
|
{
|
|
var u = buffer[i + j];
|
|
var v = buffer[i + j + len / 2] * w;
|
|
buffer[i + j] = u + v;
|
|
buffer[i + j + len / 2] = u - v;
|
|
w *= wLen;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private static float[] RmsEnvelope(ReadOnlySpan<float> samples, int windowSize, int hopSize)
|
|
{
|
|
windowSize = Math.Max(16, windowSize);
|
|
hopSize = Math.Max(1, hopSize);
|
|
if (samples.IsEmpty) return [0];
|
|
|
|
var envelope = new List<float>();
|
|
for (var start = 0; start < samples.Length; start += hopSize)
|
|
{
|
|
var end = Math.Min(start + windowSize, samples.Length);
|
|
envelope.Add(MathF.Sqrt(MeanSquare(samples[start..end])));
|
|
}
|
|
return envelope.ToArray();
|
|
}
|
|
|
|
private static int[] MelBandEdges(int bandCount, int fftSize, float sampleRate, float minFrequencyHz, float maxFrequencyHz)
|
|
{
|
|
var minMel = HzToMel(Math.Max(0, minFrequencyHz));
|
|
var maxMel = HzToMel(Math.Max(minFrequencyHz + 1, Math.Min(maxFrequencyHz, sampleRate * 0.5f)));
|
|
return Enumerable.Range(0, bandCount + 2)
|
|
.Select(index =>
|
|
{
|
|
var t = index / (float)(bandCount + 1);
|
|
var hz = MelToHz(minMel + (maxMel - minMel) * t);
|
|
return (int)MathF.Round(hz / Math.Max(1, sampleRate) * fftSize);
|
|
})
|
|
.ToArray();
|
|
}
|
|
|
|
private static (float CentroidHz, float RolloffHz) SpectralShape(IReadOnlyList<float> spectrum, float sampleRate, float rolloffPortion)
|
|
{
|
|
var total = spectrum.Sum();
|
|
if (total <= float.Epsilon) return (0, 0);
|
|
|
|
var weighted = 0f;
|
|
var cumulative = 0f;
|
|
var rolloff = 0f;
|
|
for (var bin = 0; bin < spectrum.Count; bin++)
|
|
{
|
|
var frequency = bin * sampleRate * 0.5f / Math.Max(1, spectrum.Count - 1);
|
|
weighted += frequency * spectrum[bin];
|
|
cumulative += spectrum[bin];
|
|
if (rolloff == 0 && cumulative >= total * rolloffPortion) rolloff = frequency;
|
|
}
|
|
return (weighted / total, rolloff);
|
|
}
|
|
|
|
private static float NormalizedDistance(IReadOnlyList<float> reference, IReadOnlyList<float> candidate)
|
|
{
|
|
var length = Math.Max(1, Math.Max(reference.Count, candidate.Count));
|
|
var error = 0f;
|
|
var scale = 0f;
|
|
for (var i = 0; i < length; i++)
|
|
{
|
|
var a = ResampledAt(reference, i, length);
|
|
var b = ResampledAt(candidate, i, length);
|
|
var delta = a - b;
|
|
error += delta * delta;
|
|
scale += a * a + b * b;
|
|
}
|
|
return MathF.Sqrt(error / Math.Max(float.Epsilon, scale));
|
|
}
|
|
|
|
public static float CosineSimilarity(IReadOnlyList<float> reference, IReadOnlyList<float> candidate)
|
|
{
|
|
var length = Math.Max(1, Math.Max(reference.Count, candidate.Count));
|
|
var dot = 0f;
|
|
var left = 0f;
|
|
var right = 0f;
|
|
for (var i = 0; i < length; i++)
|
|
{
|
|
var a = ResampledAt(reference, i, length);
|
|
var b = ResampledAt(candidate, i, length);
|
|
dot += a * b;
|
|
left += a * a;
|
|
right += b * b;
|
|
}
|
|
|
|
return dot / MathF.Max(0.000001f, MathF.Sqrt(left) * MathF.Sqrt(right));
|
|
}
|
|
|
|
private static float BinaryMismatch(IReadOnlyList<bool> reference, IReadOnlyList<bool> candidate)
|
|
{
|
|
var length = Math.Max(1, Math.Max(reference.Count, candidate.Count));
|
|
var mismatches = 0;
|
|
for (var i = 0; i < length; i++)
|
|
{
|
|
if (BoolResampledAt(reference, i, length) != BoolResampledAt(candidate, i, length)) mismatches++;
|
|
}
|
|
|
|
return mismatches / (float)length;
|
|
}
|
|
|
|
private static bool BoolResampledAt(IReadOnlyList<bool> values, int index, int targetLength)
|
|
{
|
|
if (values.Count == 0) return false;
|
|
if (values.Count == 1 || targetLength <= 1) return values[0];
|
|
var position = index * (values.Count - 1f) / (targetLength - 1);
|
|
return values[(int)MathF.Round(position)];
|
|
}
|
|
|
|
private static float EnvelopeFlux(IReadOnlyList<float> envelope)
|
|
{
|
|
if (envelope.Count < 2) return 0;
|
|
var sum = 0f;
|
|
for (var i = 1; i < envelope.Count; i++) sum += MathF.Abs(envelope[i] - envelope[i - 1]);
|
|
return sum / (envelope.Count - 1);
|
|
}
|
|
|
|
private static float PositiveSpectralFlux(IReadOnlyList<float> previous, IReadOnlyList<float> current)
|
|
{
|
|
var sum = 0f;
|
|
var scale = 0f;
|
|
var length = Math.Min(previous.Count, current.Count);
|
|
for (var i = 0; i < length; i++)
|
|
{
|
|
var previousLog = MathF.Log(1e-12f + previous[i]);
|
|
var currentLog = MathF.Log(1e-12f + current[i]);
|
|
sum += MathF.Max(0, currentLog - previousLog);
|
|
scale += MathF.Abs(previousLog) + MathF.Abs(currentLog);
|
|
}
|
|
|
|
return sum / Math.Max(1e-6f, scale);
|
|
}
|
|
|
|
private static float ResampledAt(IReadOnlyList<float> values, int index, int targetLength)
|
|
{
|
|
if (values.Count == 0) return 0;
|
|
if (values.Count == 1 || targetLength <= 1) return values[0];
|
|
var position = index * (values.Count - 1f) / (targetLength - 1);
|
|
var left = (int)MathF.Floor(position);
|
|
var right = Math.Min(left + 1, values.Count - 1);
|
|
var t = position - left;
|
|
return values[left] * (1 - t) + values[right] * t;
|
|
}
|
|
|
|
private static void NormalizeInPlace(float[] values)
|
|
{
|
|
if (values.Length == 0) return;
|
|
var mean = values.Sum() / values.Length;
|
|
var variance = values.Select(value => (value - mean) * (value - mean)).Sum() / values.Length;
|
|
var scale = MathF.Sqrt(variance).ClampMin(1e-6f);
|
|
for (var i = 0; i < values.Length; i++) values[i] = (values[i] - mean) / scale;
|
|
}
|
|
|
|
private static float MeanSquare(ReadOnlySpan<float> samples)
|
|
{
|
|
var sum = 0f;
|
|
foreach (var sample in samples) sum += sample * sample;
|
|
return sum / Math.Max(1, samples.Length);
|
|
}
|
|
|
|
private static float Hann(int index, int size) => size <= 1 ? 1 : 0.5f - 0.5f * MathF.Cos(MathF.Tau * index / (size - 1));
|
|
private static float HzToMel(float hz) => 2595 * MathF.Log10(1 + hz / 700);
|
|
private static float MelToHz(float mel) => 700 * (MathF.Pow(10, mel / 2595) - 1);
|
|
private static float SafeRatio(float candidate, float reference) => candidate / Math.Max(float.Epsilon, reference);
|
|
private static float RatioDistance(float ratio) => MathF.Abs(MathF.Log(Math.Max(float.Epsilon, ratio)));
|
|
private static int NextPowerOfTwo(int value) => 1 << (int)MathF.Ceiling(MathF.Log2(value));
|
|
|
|
private sealed record ArticulationFrameSet(
|
|
float[] Envelope,
|
|
bool[] Active,
|
|
float ActiveRatio,
|
|
float EnvelopeFlux,
|
|
float SpectralFlux,
|
|
float MotorBandShare,
|
|
float SpeechBandShare);
|
|
}
|
|
|
|
file static class FloatExtensions
|
|
{
|
|
public static float ClampMin(this float value, float min) => Math.Max(value, min);
|
|
}
|