Add native ASIO runtime source
This commit is contained in:
parent
f0d3cf5815
commit
fad7eb63b7
20 changed files with 925 additions and 134 deletions
225
src/Mimir.Runtime/Synchronization/MimirAsioStreamSource.cs
Normal file
225
src/Mimir.Runtime/Synchronization/MimirAsioStreamSource.cs
Normal file
|
|
@ -0,0 +1,225 @@
|
|||
using System.Runtime.InteropServices;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Runtime.Versioning;
|
||||
|
||||
namespace Mimir.Runtime.Synchronization;
|
||||
|
||||
public sealed record MimirAsioStreamSourceOptions(
|
||||
string NativeLibraryPath,
|
||||
string DriverClsid,
|
||||
int SampleRate,
|
||||
IReadOnlyList<string> SourceIds);
|
||||
|
||||
[SupportedOSPlatform("windows")]
|
||||
public sealed class MimirAsioStreamSource : IMimirStreamSource
|
||||
{
|
||||
private const string DefaultClsid = "{AC4D0455-50D7-4498-B3CD-9A41D130B759}";
|
||||
private readonly ConcurrentQueue<MimirStreamSample> samples = new();
|
||||
private readonly ManualResetEventSlim started = new();
|
||||
private readonly Thread captureThread;
|
||||
private readonly CancellationTokenSource cancellation = new();
|
||||
private readonly string[] sourceIds;
|
||||
private int sampleRate;
|
||||
private string? startError;
|
||||
private bool disposed;
|
||||
|
||||
public MimirAsioStreamSource(
|
||||
MimirStreamDescriptor descriptor,
|
||||
MimirAsioStreamSourceOptions options)
|
||||
{
|
||||
Descriptor = descriptor;
|
||||
Options = options;
|
||||
sourceIds = options.SourceIds.Count > 0
|
||||
? options.SourceIds.ToArray()
|
||||
: Enumerable.Range(0, 8).Select(index => $"asio-ch{index}").ToArray();
|
||||
captureThread = new Thread(CaptureLoop)
|
||||
{
|
||||
IsBackground = true,
|
||||
Name = "Mimir Focusrite ASIO capture",
|
||||
};
|
||||
captureThread.SetApartmentState(ApartmentState.STA);
|
||||
captureThread.Start();
|
||||
if (!started.Wait(TimeSpan.FromSeconds(5)))
|
||||
{
|
||||
throw new InvalidOperationException("Timed out starting Focusrite ASIO capture source");
|
||||
}
|
||||
|
||||
if (startError != null)
|
||||
{
|
||||
throw new InvalidOperationException(startError);
|
||||
}
|
||||
}
|
||||
|
||||
public MimirStreamDescriptor Descriptor { get; }
|
||||
|
||||
public MimirAsioStreamSourceOptions Options { get; }
|
||||
|
||||
public bool ExposesDescriptorBuffer => false;
|
||||
|
||||
public bool TryRead(out MimirStreamSample sample)
|
||||
{
|
||||
sample = default;
|
||||
ObjectDisposedException.ThrowIf(disposed, this);
|
||||
return samples.TryDequeue(out sample);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
disposed = true;
|
||||
cancellation.Cancel();
|
||||
captureThread.Join(TimeSpan.FromSeconds(2));
|
||||
cancellation.Dispose();
|
||||
started.Dispose();
|
||||
}
|
||||
|
||||
private void CaptureLoop()
|
||||
{
|
||||
nint handle = 0;
|
||||
try
|
||||
{
|
||||
Native.Load(Options.NativeLibraryPath);
|
||||
handle = Native.Create(
|
||||
string.IsNullOrWhiteSpace(Options.DriverClsid) ? DefaultClsid : Options.DriverClsid,
|
||||
Options.SampleRate,
|
||||
out sampleRate,
|
||||
out _,
|
||||
out var maxFrames);
|
||||
if (handle == 0)
|
||||
{
|
||||
startError = "Could not create Focusrite ASIO capture source";
|
||||
started.Set();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!Native.Start(handle))
|
||||
{
|
||||
startError = "Could not start Focusrite ASIO capture source";
|
||||
started.Set();
|
||||
return;
|
||||
}
|
||||
|
||||
started.Set();
|
||||
var sampleBuffer = new float[Math.Max(1, maxFrames)];
|
||||
while (!cancellation.IsCancellationRequested)
|
||||
{
|
||||
if (!Native.Read(
|
||||
handle,
|
||||
out var channel,
|
||||
out var timestampNs,
|
||||
out var sequence,
|
||||
out var frameCount,
|
||||
sampleBuffer,
|
||||
sampleBuffer.Length))
|
||||
{
|
||||
Thread.Sleep(1);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (channel < 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var sourceId = channel < sourceIds.Length ? sourceIds[channel] : $"asio-ch{channel}";
|
||||
var bytes = new byte[frameCount * sizeof(float)];
|
||||
MemoryMarshal.AsBytes(sampleBuffer.AsSpan(0, frameCount)).CopyTo(bytes);
|
||||
samples.Enqueue(new MimirStreamSample(
|
||||
sourceId,
|
||||
MimirStreamKind.Audio,
|
||||
Descriptor.Origin,
|
||||
timestampNs,
|
||||
DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() * 1_000_000L,
|
||||
sequence,
|
||||
0,
|
||||
bytes.Length,
|
||||
bytes,
|
||||
AudioBlock: new MimirAudioBlockDescriptor(
|
||||
sampleRate,
|
||||
1,
|
||||
MimirAudioSampleFormat.Float32,
|
||||
frameCount,
|
||||
timestampNs)));
|
||||
|
||||
while (samples.Count > 8192 && samples.TryDequeue(out _))
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
startError = ex.Message;
|
||||
started.Set();
|
||||
}
|
||||
finally
|
||||
{
|
||||
Native.Destroy(handle);
|
||||
}
|
||||
}
|
||||
|
||||
private static unsafe class Native
|
||||
{
|
||||
private static nint library;
|
||||
private static delegate* unmanaged[Stdcall]<byte*, double, out int, out int, out int, nint> create;
|
||||
private static delegate* unmanaged[Stdcall]<nint, int> start;
|
||||
private static delegate* unmanaged[Stdcall]<nint, out int, out long, out ulong, out int, float*, int, int> read;
|
||||
private static delegate* unmanaged[Stdcall]<nint, void> destroy;
|
||||
|
||||
public static void Load(string path)
|
||||
{
|
||||
if (library != 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(path))
|
||||
{
|
||||
path = Path.Combine(AppContext.BaseDirectory, "mimir_asio_capture.dll");
|
||||
}
|
||||
|
||||
library = NativeLibrary.Load(path);
|
||||
create = (delegate* unmanaged[Stdcall]<byte*, double, out int, out int, out int, nint>)NativeLibrary.GetExport(library, "mimir_asio_create");
|
||||
start = (delegate* unmanaged[Stdcall]<nint, int>)NativeLibrary.GetExport(library, "mimir_asio_start");
|
||||
read = (delegate* unmanaged[Stdcall]<nint, out int, out long, out ulong, out int, float*, int, int>)NativeLibrary.GetExport(library, "mimir_asio_read");
|
||||
destroy = (delegate* unmanaged[Stdcall]<nint, void>)NativeLibrary.GetExport(library, "mimir_asio_destroy");
|
||||
}
|
||||
|
||||
public static nint Create(string clsid, int requestedSampleRate, out int sampleRate, out int inputCount, out int preferredBufferSize)
|
||||
{
|
||||
var bytes = System.Text.Encoding.UTF8.GetBytes(clsid + "\0");
|
||||
fixed (byte* clsidPtr = bytes)
|
||||
{
|
||||
return create(clsidPtr, requestedSampleRate, out sampleRate, out inputCount, out preferredBufferSize);
|
||||
}
|
||||
}
|
||||
|
||||
public static bool Start(nint handle) => start(handle) != 0;
|
||||
|
||||
public static bool Read(
|
||||
nint handle,
|
||||
out int channel,
|
||||
out long timestampNs,
|
||||
out ulong sequence,
|
||||
out int frameCount,
|
||||
float[] samples,
|
||||
int maxFrames)
|
||||
{
|
||||
fixed (float* samplePtr = samples)
|
||||
{
|
||||
return read(handle, out channel, out timestampNs, out sequence, out frameCount, samplePtr, maxFrames) != 0;
|
||||
}
|
||||
}
|
||||
|
||||
public static void Destroy(nint handle)
|
||||
{
|
||||
if (handle != 0)
|
||||
{
|
||||
destroy(handle);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -39,11 +39,12 @@ public sealed record MimirChirpBinCalibrationModel(
|
|||
public static MimirChirpBinCalibrationModel FromDecodes(
|
||||
string referenceSourceId,
|
||||
int sampleRate,
|
||||
IReadOnlyDictionary<string, MimirChirpletStreamDecode> decodes)
|
||||
IReadOnlyDictionary<string, MimirChirpletStreamDecode> decodes,
|
||||
string outputSourceId = "main-speakers")
|
||||
{
|
||||
var paths = decodes
|
||||
.OrderBy(pair => pair.Key, StringComparer.Ordinal)
|
||||
.Select(pair => MimirChirpBinPathCalibration.FromDecode(pair.Key, sampleRate, pair.Value))
|
||||
.Select(pair => MimirChirpBinPathCalibration.FromDecode(outputSourceId, pair.Key, sampleRate, pair.Value))
|
||||
.ToArray();
|
||||
var emissionPlan = MimirChirpBinCodebookPlan.ForSharedEmission(paths);
|
||||
paths = paths.Select(path => path with { EmissionPlan = emissionPlan }).ToArray();
|
||||
|
|
@ -58,6 +59,7 @@ public sealed record MimirChirpBinCalibrationModel(
|
|||
}
|
||||
|
||||
public sealed record MimirChirpBinPathCalibration(
|
||||
string OutputSourceId,
|
||||
string SourceId,
|
||||
int SampleRate,
|
||||
MimirChirpBinCalibrationProfile Profile,
|
||||
|
|
@ -79,7 +81,20 @@ public sealed record MimirChirpBinPathCalibration(
|
|||
return symbol?.MeanTimingResidualSamples ?? 0.0;
|
||||
}
|
||||
|
||||
public double PhaseWeight(int symbolId, double observedPhaseRadians)
|
||||
{
|
||||
var symbol = Symbols.FirstOrDefault(item => item.SymbolId == symbolId);
|
||||
if (symbol == null || symbol.ObservationCount <= 0)
|
||||
{
|
||||
return 1.0;
|
||||
}
|
||||
|
||||
var error = Math.Abs(AngularDistance(observedPhaseRadians, symbol.MeanPhaseRadians));
|
||||
return Math.Clamp(1.0 - error / Math.PI, 0.15, 1.0);
|
||||
}
|
||||
|
||||
public static MimirChirpBinPathCalibration FromDecode(
|
||||
string outputSourceId,
|
||||
string sourceId,
|
||||
int sampleRate,
|
||||
MimirChirpletStreamDecode decode)
|
||||
|
|
@ -99,9 +114,11 @@ public sealed record MimirChirpBinPathCalibration(
|
|||
(correct / (double)items.Length) *
|
||||
(1.0 / (1.0 + residualSpread / Math.Max(1.0, sampleRate * 0.00025)));
|
||||
var phase = CircularMean(items.Select(item => item.PhaseRadians));
|
||||
var phaseCoherence = PhaseCoherence(items.Select(item => item.PhaseRadians));
|
||||
return new MimirChirpBinSymbolCalibration(
|
||||
group.Key,
|
||||
reliability,
|
||||
phaseCoherence,
|
||||
items.Average(item => item.ExpectedCenterHz),
|
||||
items.Average(item => item.ObservedCenterHz),
|
||||
items.Average(item => item.ObservedEnergy),
|
||||
|
|
@ -115,6 +132,7 @@ public sealed record MimirChirpBinPathCalibration(
|
|||
var hypotheses = BuildDelayHypotheses(decode);
|
||||
var plan = MimirChirpBinCodebookPlan.FromSymbols(symbols);
|
||||
return new MimirChirpBinPathCalibration(
|
||||
outputSourceId,
|
||||
sourceId,
|
||||
sampleRate,
|
||||
profile,
|
||||
|
|
@ -201,6 +219,27 @@ public sealed record MimirChirpBinPathCalibration(
|
|||
|
||||
return count == 0 ? 0.0 : Math.Atan2(sin / count, cos / count);
|
||||
}
|
||||
|
||||
private static double PhaseCoherence(IEnumerable<double> radians)
|
||||
{
|
||||
var sin = 0.0;
|
||||
var cos = 0.0;
|
||||
var count = 0;
|
||||
foreach (var value in radians)
|
||||
{
|
||||
sin += Math.Sin(value);
|
||||
cos += Math.Cos(value);
|
||||
count++;
|
||||
}
|
||||
|
||||
return count == 0 ? 0.0 : Math.Clamp(Math.Sqrt(sin * sin + cos * cos) / count, 0.0, 1.0);
|
||||
}
|
||||
|
||||
private static double AngularDistance(double a, double b)
|
||||
{
|
||||
var delta = Math.Abs(a - b) % (Math.PI * 2.0);
|
||||
return delta > Math.PI ? Math.PI * 2.0 - delta : delta;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record MimirChirpBinCodebookPlan(
|
||||
|
|
@ -253,6 +292,7 @@ public sealed record MimirChirpBinCodebookPlan(
|
|||
public sealed record MimirChirpBinSymbolCalibration(
|
||||
int SymbolId,
|
||||
double Reliability,
|
||||
double PhaseCoherence,
|
||||
double ExpectedCenterHz,
|
||||
double ObservedCenterHz,
|
||||
double MeanEnergy,
|
||||
|
|
|
|||
|
|
@ -296,7 +296,8 @@ public sealed class MimirChirpBinTimeline
|
|||
var correctedEnergy = ScoreSymbolEnergy(
|
||||
samples.Slice((int)Math.Round(correctedOffset), chirpSamples),
|
||||
sampleRate,
|
||||
canonicalSymbol) * (calibration?.SymbolWeight(canonicalSymbol) ?? 1.0);
|
||||
canonicalSymbol) *
|
||||
(calibration?.SymbolWeight(canonicalSymbol) ?? 1.0);
|
||||
correctedOffset -= calibration?.GroupDelayCorrectionSamples(canonicalSymbol) ?? 0.0;
|
||||
correctedCandidates.Add(new MimirChirpletSymbolCandidate(
|
||||
canonicalSymbol,
|
||||
|
|
@ -350,7 +351,9 @@ public sealed class MimirChirpBinTimeline
|
|||
.Select((score, symbol) => new MimirChirpletSymbolCandidate(
|
||||
symbol,
|
||||
sampleOffset - (calibration?.GroupDelayCorrectionSamples(symbol) ?? 0.0),
|
||||
score.Energy * (calibration?.SymbolWeight(symbol) ?? 1.0),
|
||||
score.Energy *
|
||||
(calibration?.SymbolWeight(symbol) ?? 1.0) *
|
||||
(calibration?.PhaseWeight(symbol, score.PhaseRadians) ?? 1.0),
|
||||
bandResponses))
|
||||
.OrderByDescending(candidate => candidate.Energy)
|
||||
.Take(MaxSymbolCandidatesPerFrame)
|
||||
|
|
@ -459,35 +462,45 @@ public sealed class MimirChirpBinTimeline
|
|||
|
||||
foreach (var candidatePath in CandidateSymbolPaths(window))
|
||||
{
|
||||
var code = CodeKey(candidatePath.Select(candidate => candidate.SymbolId));
|
||||
if (!plan.CodeToIndex.TryGetValue(code, out var eventIndex))
|
||||
foreach (var globalBinShift in GlobalBinShiftHypotheses(calibration))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
var canonicalSymbols = candidatePath
|
||||
.Select(candidate => ShiftSymbol(candidate.SymbolId, -globalBinShift))
|
||||
.ToArray();
|
||||
var code = CodeKey(canonicalSymbols);
|
||||
if (!plan.CodeToIndex.TryGetValue(code, out var eventIndex))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var firstEvent = Default.EventForIndex((ulong)eventIndex, calibration?.EmissionPlan);
|
||||
var gapError = 0.0;
|
||||
for (var gap = 0; gap < candidatePath.Count - 1; gap++)
|
||||
{
|
||||
var measuredGap = (candidatePath[gap + 1].SampleOffset - candidatePath[gap].SampleOffset) / sampleRate;
|
||||
gapError += Math.Abs(measuredGap - EventSpacingSeconds);
|
||||
}
|
||||
var firstEvent = Default.EventForIndex((ulong)eventIndex, calibration?.EmissionPlan);
|
||||
var gapError = 0.0;
|
||||
for (var gap = 0; gap < candidatePath.Count - 1; gap++)
|
||||
{
|
||||
var measuredGap = (candidatePath[gap + 1].SampleOffset - candidatePath[gap].SampleOffset) / sampleRate;
|
||||
gapError += Math.Abs(measuredGap - EventSpacingSeconds);
|
||||
}
|
||||
|
||||
var gapConfidence = 1.0 / (1.0 + gapError / 0.003);
|
||||
if (gapConfidence < 0.55)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
var gapConfidence = 1.0 / (1.0 + gapError / 0.003);
|
||||
if (gapConfidence < 0.55)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var energyConfidence = Math.Clamp(candidatePath.Average(candidate => candidate.Energy), 0.0, 1.0);
|
||||
candidates.Add(new MimirChirpletTimelineAnchor(
|
||||
(ulong)eventIndex,
|
||||
firstEvent.StartSeconds,
|
||||
candidatePath[0].SampleOffset,
|
||||
gapConfidence * 0.60 + energyConfidence * 0.40,
|
||||
candidatePath
|
||||
.Select(candidate => new MimirChirpletSymbolObservation(candidate.SymbolId, candidate.SampleOffset, candidate.Energy))
|
||||
.ToArray()));
|
||||
var energyConfidence = Math.Clamp(candidatePath.Average(candidate => candidate.Energy), 0.0, 1.0);
|
||||
var binShiftPenalty = 1.0 / (1.0 + Math.Abs(globalBinShift) * 0.10);
|
||||
candidates.Add(new MimirChirpletTimelineAnchor(
|
||||
(ulong)eventIndex,
|
||||
firstEvent.StartSeconds,
|
||||
candidatePath[0].SampleOffset,
|
||||
(gapConfidence * 0.60 + energyConfidence * 0.40) * binShiftPenalty,
|
||||
candidatePath
|
||||
.Select((candidate, candidateIndex) => new MimirChirpletSymbolObservation(
|
||||
canonicalSymbols[candidateIndex],
|
||||
candidate.SampleOffset,
|
||||
candidate.Energy))
|
||||
.ToArray()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -519,6 +532,33 @@ public sealed class MimirChirpBinTimeline
|
|||
return Recurse(0);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<int> GlobalBinShiftHypotheses(MimirChirpBinPathCalibration? calibration)
|
||||
{
|
||||
var shifts = new HashSet<int> { 0 };
|
||||
if (calibration != null)
|
||||
{
|
||||
foreach (var hypothesis in calibration.DelayHypotheses)
|
||||
{
|
||||
shifts.Add(Math.Clamp(hypothesis.BinShift, -MaxTimingBinShift, MaxTimingBinShift));
|
||||
}
|
||||
|
||||
foreach (var group in calibration.Confusion
|
||||
.GroupBy(observation => observation.BinShift)
|
||||
.OrderByDescending(group => group.Count())
|
||||
.Take(4))
|
||||
{
|
||||
shifts.Add(Math.Clamp(group.Key, -MaxTimingBinShift, MaxTimingBinShift));
|
||||
}
|
||||
}
|
||||
|
||||
for (var shift = -MaxTimingBinShift; shift <= MaxTimingBinShift; shift++)
|
||||
{
|
||||
shifts.Add(shift);
|
||||
}
|
||||
|
||||
return shifts.OrderBy(shift => Math.Abs(shift)).ThenBy(shift => shift).ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<MimirChirpletTimelineAnchor> SelectCoherentAnchorPath(
|
||||
IReadOnlyList<MimirChirpletTimelineAnchor> candidates,
|
||||
int sampleRate,
|
||||
|
|
|
|||
|
|
@ -119,6 +119,24 @@ public sealed class MimirRuntimeConfiguration
|
|||
return new MimirNativeIngestStreamSource(ToDescriptor(stream));
|
||||
}
|
||||
|
||||
if (string.Equals(stream.Adapter, "asio", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (!OperatingSystem.IsWindows())
|
||||
{
|
||||
throw new PlatformNotSupportedException("The Mimir ASIO stream source requires Windows.");
|
||||
}
|
||||
|
||||
#pragma warning disable CA1416
|
||||
return new MimirAsioStreamSource(
|
||||
ToDescriptor(stream),
|
||||
new MimirAsioStreamSourceOptions(
|
||||
ResolveCommand(stream.Command, configDirectory),
|
||||
stream.DriverClsid,
|
||||
stream.SampleRate,
|
||||
stream.AcceptSourceIds));
|
||||
#pragma warning restore CA1416
|
||||
}
|
||||
|
||||
if (string.Equals(stream.Adapter, "frame-events", StringComparison.OrdinalIgnoreCase)
|
||||
|| string.Equals(stream.Adapter, "json-lines", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
|
|
@ -243,4 +261,8 @@ public sealed class MimirStreamConfig
|
|||
public string[] AcceptSourceIds { get; set; } = [];
|
||||
|
||||
public int ChunkBytes { get; set; } = 65_536;
|
||||
|
||||
public int SampleRate { get; set; } = 192_000;
|
||||
|
||||
public string DriverClsid { get; set; } = "";
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue