Add native and process stream sources

This commit is contained in:
Meta Vens 2026-05-21 18:27:39 +01:00
commit a22fa0745c
14 changed files with 413 additions and 3 deletions

View file

@ -0,0 +1,44 @@
using System.Collections.Concurrent;
namespace Mimir.Runtime.Synchronization;
public sealed class MimirNativeIngestStreamSource : IMimirStreamSource
{
private readonly ConcurrentQueue<MimirStreamSample> samples = new();
private ulong sequence;
public MimirNativeIngestStreamSource(MimirStreamDescriptor descriptor)
{
Descriptor = descriptor;
}
public MimirStreamDescriptor Descriptor { get; }
public void Push(
long timestampNs,
long arrivalNs,
ulong payloadHandle,
int byteLength = 0,
ReadOnlyMemory<byte> data = default)
{
samples.Enqueue(new MimirStreamSample(
Descriptor.SourceId,
Descriptor.Kind,
Descriptor.Origin,
timestampNs,
arrivalNs,
sequence++,
payloadHandle,
byteLength,
data));
}
public bool TryRead(out MimirStreamSample sample)
{
return samples.TryDequeue(out sample);
}
public void Dispose()
{
}
}

View file

@ -0,0 +1,134 @@
using System.Collections.Concurrent;
using System.Diagnostics;
namespace Mimir.Runtime.Synchronization;
public sealed record MimirProcessStreamSourceOptions(
string Command,
IReadOnlyList<string> Arguments,
int ChunkBytes = 65_536);
public sealed class MimirProcessStreamSource : IMimirStreamSource
{
private readonly ConcurrentQueue<MimirStreamSample> samples = new();
private readonly CancellationTokenSource cancellation = new();
private readonly Process process;
private readonly Task stdoutTask;
private readonly Task stderrTask;
private ulong sequence;
private bool disposed;
public MimirProcessStreamSource(
MimirStreamDescriptor descriptor,
MimirProcessStreamSourceOptions options)
{
Descriptor = descriptor;
Options = options;
process = StartProcess(options);
stdoutTask = Task.Run(() => ReadStdout(options.ChunkBytes, cancellation.Token));
stderrTask = Task.Run(() => DrainStderr(cancellation.Token));
}
public MimirStreamDescriptor Descriptor { get; }
public MimirProcessStreamSourceOptions Options { get; }
public bool TryRead(out MimirStreamSample sample)
{
return samples.TryDequeue(out sample);
}
private Process StartProcess(MimirProcessStreamSourceOptions options)
{
var startInfo = new ProcessStartInfo(options.Command)
{
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true,
};
foreach (var argument in options.Arguments)
{
startInfo.ArgumentList.Add(argument);
}
return Process.Start(startInfo)
?? throw new InvalidOperationException($"Could not start stream source process: {options.Command}");
}
private async Task ReadStdout(int chunkBytes, CancellationToken token)
{
var stream = process.StandardOutput.BaseStream;
while (!token.IsCancellationRequested)
{
var buffer = new byte[chunkBytes];
var read = await stream.ReadAsync(buffer.AsMemory(0, buffer.Length), token).ConfigureAwait(false);
if (read <= 0)
{
break;
}
if (read != buffer.Length)
{
Array.Resize(ref buffer, read);
}
var now = Stopwatch.GetTimestamp();
var timestampNs = StopwatchTicksToNs(now);
samples.Enqueue(new MimirStreamSample(
Descriptor.SourceId,
Descriptor.Kind,
Descriptor.Origin,
timestampNs,
timestampNs,
sequence++,
0,
buffer.Length,
buffer));
}
}
private async Task DrainStderr(CancellationToken token)
{
while (!token.IsCancellationRequested)
{
var line = await process.StandardError.ReadLineAsync(token).ConfigureAwait(false);
if (line == null)
{
break;
}
}
}
private static long StopwatchTicksToNs(long ticks)
{
return checked((long)(ticks * (1_000_000_000.0 / Stopwatch.Frequency)));
}
public void Dispose()
{
if (disposed)
{
return;
}
disposed = true;
cancellation.Cancel();
try
{
if (!process.HasExited)
{
process.Kill(entireProcessTree: true);
}
}
catch
{
}
process.Dispose();
cancellation.Dispose();
GC.SuppressFinalize(stdoutTask);
GC.SuppressFinalize(stderrTask);
}
}

View file

@ -0,0 +1,153 @@
using System.Text.Json;
namespace Mimir.Runtime.Synchronization;
public sealed class MimirRuntimeConfiguration
{
public MimirSynchronizationSettings Settings { get; init; } = new();
public IReadOnlyList<IMimirStreamSource> Sources { get; init; } = [];
public static MimirRuntimeConfiguration Load()
{
var path = ResolveConfigPath();
if (path == null)
{
return new MimirRuntimeConfiguration
{
Settings = MimirSynchronizationSettings.FromEnvironment(),
};
}
var model = JsonSerializer.Deserialize<MimirRuntimeConfigFile>(
File.ReadAllText(path),
new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true,
}) ?? new MimirRuntimeConfigFile();
var bufferDuration = model.BufferSeconds > 0.0
? TimeSpan.FromSeconds(Math.Clamp(model.BufferSeconds, 0.25, 60.0))
: MimirSynchronizationSettings.FromEnvironment().BufferDuration;
var sourceModels = model.Streams
.Where(stream => stream.Enabled)
.ToArray();
var streams = sourceModels
.Select(ToDescriptor)
.ToArray();
var sources = sourceModels
.Select(TryCreateSource)
.Where(source => source != null)
.Cast<IMimirStreamSource>()
.ToArray();
return new MimirRuntimeConfiguration
{
Settings = new MimirSynchronizationSettings
{
BufferDuration = bufferDuration,
Streams = streams,
},
Sources = sources,
};
}
private static string? ResolveConfigPath()
{
var configured = Environment.GetEnvironmentVariable("MIMIR_RUNTIME_CONFIG");
if (!string.IsNullOrWhiteSpace(configured) && File.Exists(configured))
{
return configured;
}
var current = AppContext.BaseDirectory;
for (var index = 0; index < 8; index++)
{
var candidate = Path.Combine(current, "config", "mimir-runtime.json");
if (File.Exists(candidate))
{
return candidate;
}
var parent = Directory.GetParent(current);
if (parent == null)
{
break;
}
current = parent.FullName;
}
return null;
}
private static MimirStreamDescriptor ToDescriptor(MimirStreamConfig stream)
{
return new MimirStreamDescriptor(
stream.SourceId,
ParseKind(stream.Kind),
ParseOrigin(stream.Origin),
stream.Enabled);
}
private static IMimirStreamSource? TryCreateSource(MimirStreamConfig stream)
{
if (!string.Equals(stream.Adapter, "process", StringComparison.OrdinalIgnoreCase))
{
return null;
}
if (string.IsNullOrWhiteSpace(stream.Command))
{
return null;
}
return new MimirProcessStreamSource(
ToDescriptor(stream),
new MimirProcessStreamSourceOptions(
stream.Command,
stream.Arguments,
Math.Max(1024, stream.ChunkBytes)));
}
private static MimirStreamKind ParseKind(string value)
{
return Enum.TryParse<MimirStreamKind>(value, ignoreCase: true, out var kind)
? kind
: throw new InvalidOperationException($"Unknown Mimir stream kind: {value}");
}
private static MimirStreamOrigin ParseOrigin(string value)
{
return Enum.TryParse<MimirStreamOrigin>(value, ignoreCase: true, out var origin)
? origin
: throw new InvalidOperationException($"Unknown Mimir stream origin: {value}");
}
}
public sealed class MimirRuntimeConfigFile
{
public double BufferSeconds { get; set; } = 5.0;
public List<MimirStreamConfig> Streams { get; set; } = [];
}
public sealed class MimirStreamConfig
{
public string SourceId { get; set; } = "";
public string Kind { get; set; } = "Video";
public string Origin { get; set; } = "LocalDevice";
public bool Enabled { get; set; } = true;
public string Adapter { get; set; } = "process";
public string Command { get; set; } = "";
public string[] Arguments { get; set; } = [];
public int ChunkBytes { get; set; } = 65_536;
}

View file

@ -8,4 +8,5 @@ public readonly record struct MimirStreamSample(
long ArrivalNs,
ulong Sequence,
ulong PayloadHandle,
int ByteLength = 0);
int ByteLength = 0,
ReadOnlyMemory<byte> Data = default);