Add native and process stream sources
This commit is contained in:
parent
2d179d9e2f
commit
a22fa0745c
14 changed files with 413 additions and 3 deletions
|
|
@ -0,0 +1,44 @@
|
|||
using System.Collections.Concurrent;
|
||||
|
||||
namespace Mimir.Runtime.Synchronization;
|
||||
|
||||
public sealed class MimirNativeIngestStreamSource : IMimirStreamSource
|
||||
{
|
||||
private readonly ConcurrentQueue<MimirStreamSample> samples = new();
|
||||
private ulong sequence;
|
||||
|
||||
public MimirNativeIngestStreamSource(MimirStreamDescriptor descriptor)
|
||||
{
|
||||
Descriptor = descriptor;
|
||||
}
|
||||
|
||||
public MimirStreamDescriptor Descriptor { get; }
|
||||
|
||||
public void Push(
|
||||
long timestampNs,
|
||||
long arrivalNs,
|
||||
ulong payloadHandle,
|
||||
int byteLength = 0,
|
||||
ReadOnlyMemory<byte> data = default)
|
||||
{
|
||||
samples.Enqueue(new MimirStreamSample(
|
||||
Descriptor.SourceId,
|
||||
Descriptor.Kind,
|
||||
Descriptor.Origin,
|
||||
timestampNs,
|
||||
arrivalNs,
|
||||
sequence++,
|
||||
payloadHandle,
|
||||
byteLength,
|
||||
data));
|
||||
}
|
||||
|
||||
public bool TryRead(out MimirStreamSample sample)
|
||||
{
|
||||
return samples.TryDequeue(out sample);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
134
src/Mimir.Runtime/Synchronization/MimirProcessStreamSource.cs
Normal file
134
src/Mimir.Runtime/Synchronization/MimirProcessStreamSource.cs
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace Mimir.Runtime.Synchronization;
|
||||
|
||||
public sealed record MimirProcessStreamSourceOptions(
|
||||
string Command,
|
||||
IReadOnlyList<string> Arguments,
|
||||
int ChunkBytes = 65_536);
|
||||
|
||||
public sealed class MimirProcessStreamSource : IMimirStreamSource
|
||||
{
|
||||
private readonly ConcurrentQueue<MimirStreamSample> samples = new();
|
||||
private readonly CancellationTokenSource cancellation = new();
|
||||
private readonly Process process;
|
||||
private readonly Task stdoutTask;
|
||||
private readonly Task stderrTask;
|
||||
private ulong sequence;
|
||||
private bool disposed;
|
||||
|
||||
public MimirProcessStreamSource(
|
||||
MimirStreamDescriptor descriptor,
|
||||
MimirProcessStreamSourceOptions options)
|
||||
{
|
||||
Descriptor = descriptor;
|
||||
Options = options;
|
||||
process = StartProcess(options);
|
||||
stdoutTask = Task.Run(() => ReadStdout(options.ChunkBytes, cancellation.Token));
|
||||
stderrTask = Task.Run(() => DrainStderr(cancellation.Token));
|
||||
}
|
||||
|
||||
public MimirStreamDescriptor Descriptor { get; }
|
||||
|
||||
public MimirProcessStreamSourceOptions Options { get; }
|
||||
|
||||
public bool TryRead(out MimirStreamSample sample)
|
||||
{
|
||||
return samples.TryDequeue(out sample);
|
||||
}
|
||||
|
||||
private Process StartProcess(MimirProcessStreamSourceOptions options)
|
||||
{
|
||||
var startInfo = new ProcessStartInfo(options.Command)
|
||||
{
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true,
|
||||
};
|
||||
|
||||
foreach (var argument in options.Arguments)
|
||||
{
|
||||
startInfo.ArgumentList.Add(argument);
|
||||
}
|
||||
|
||||
return Process.Start(startInfo)
|
||||
?? throw new InvalidOperationException($"Could not start stream source process: {options.Command}");
|
||||
}
|
||||
|
||||
private async Task ReadStdout(int chunkBytes, CancellationToken token)
|
||||
{
|
||||
var stream = process.StandardOutput.BaseStream;
|
||||
while (!token.IsCancellationRequested)
|
||||
{
|
||||
var buffer = new byte[chunkBytes];
|
||||
var read = await stream.ReadAsync(buffer.AsMemory(0, buffer.Length), token).ConfigureAwait(false);
|
||||
if (read <= 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (read != buffer.Length)
|
||||
{
|
||||
Array.Resize(ref buffer, read);
|
||||
}
|
||||
|
||||
var now = Stopwatch.GetTimestamp();
|
||||
var timestampNs = StopwatchTicksToNs(now);
|
||||
samples.Enqueue(new MimirStreamSample(
|
||||
Descriptor.SourceId,
|
||||
Descriptor.Kind,
|
||||
Descriptor.Origin,
|
||||
timestampNs,
|
||||
timestampNs,
|
||||
sequence++,
|
||||
0,
|
||||
buffer.Length,
|
||||
buffer));
|
||||
}
|
||||
}
|
||||
|
||||
private async Task DrainStderr(CancellationToken token)
|
||||
{
|
||||
while (!token.IsCancellationRequested)
|
||||
{
|
||||
var line = await process.StandardError.ReadLineAsync(token).ConfigureAwait(false);
|
||||
if (line == null)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static long StopwatchTicksToNs(long ticks)
|
||||
{
|
||||
return checked((long)(ticks * (1_000_000_000.0 / Stopwatch.Frequency)));
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
disposed = true;
|
||||
cancellation.Cancel();
|
||||
try
|
||||
{
|
||||
if (!process.HasExited)
|
||||
{
|
||||
process.Kill(entireProcessTree: true);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
}
|
||||
|
||||
process.Dispose();
|
||||
cancellation.Dispose();
|
||||
GC.SuppressFinalize(stdoutTask);
|
||||
GC.SuppressFinalize(stderrTask);
|
||||
}
|
||||
}
|
||||
153
src/Mimir.Runtime/Synchronization/MimirRuntimeConfiguration.cs
Normal file
153
src/Mimir.Runtime/Synchronization/MimirRuntimeConfiguration.cs
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
using System.Text.Json;
|
||||
|
||||
namespace Mimir.Runtime.Synchronization;
|
||||
|
||||
public sealed class MimirRuntimeConfiguration
|
||||
{
|
||||
public MimirSynchronizationSettings Settings { get; init; } = new();
|
||||
|
||||
public IReadOnlyList<IMimirStreamSource> Sources { get; init; } = [];
|
||||
|
||||
public static MimirRuntimeConfiguration Load()
|
||||
{
|
||||
var path = ResolveConfigPath();
|
||||
if (path == null)
|
||||
{
|
||||
return new MimirRuntimeConfiguration
|
||||
{
|
||||
Settings = MimirSynchronizationSettings.FromEnvironment(),
|
||||
};
|
||||
}
|
||||
|
||||
var model = JsonSerializer.Deserialize<MimirRuntimeConfigFile>(
|
||||
File.ReadAllText(path),
|
||||
new JsonSerializerOptions
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
}) ?? new MimirRuntimeConfigFile();
|
||||
|
||||
var bufferDuration = model.BufferSeconds > 0.0
|
||||
? TimeSpan.FromSeconds(Math.Clamp(model.BufferSeconds, 0.25, 60.0))
|
||||
: MimirSynchronizationSettings.FromEnvironment().BufferDuration;
|
||||
|
||||
var sourceModels = model.Streams
|
||||
.Where(stream => stream.Enabled)
|
||||
.ToArray();
|
||||
var streams = sourceModels
|
||||
.Select(ToDescriptor)
|
||||
.ToArray();
|
||||
var sources = sourceModels
|
||||
.Select(TryCreateSource)
|
||||
.Where(source => source != null)
|
||||
.Cast<IMimirStreamSource>()
|
||||
.ToArray();
|
||||
|
||||
return new MimirRuntimeConfiguration
|
||||
{
|
||||
Settings = new MimirSynchronizationSettings
|
||||
{
|
||||
BufferDuration = bufferDuration,
|
||||
Streams = streams,
|
||||
},
|
||||
Sources = sources,
|
||||
};
|
||||
}
|
||||
|
||||
private static string? ResolveConfigPath()
|
||||
{
|
||||
var configured = Environment.GetEnvironmentVariable("MIMIR_RUNTIME_CONFIG");
|
||||
if (!string.IsNullOrWhiteSpace(configured) && File.Exists(configured))
|
||||
{
|
||||
return configured;
|
||||
}
|
||||
|
||||
var current = AppContext.BaseDirectory;
|
||||
for (var index = 0; index < 8; index++)
|
||||
{
|
||||
var candidate = Path.Combine(current, "config", "mimir-runtime.json");
|
||||
if (File.Exists(candidate))
|
||||
{
|
||||
return candidate;
|
||||
}
|
||||
|
||||
var parent = Directory.GetParent(current);
|
||||
if (parent == null)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
current = parent.FullName;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static MimirStreamDescriptor ToDescriptor(MimirStreamConfig stream)
|
||||
{
|
||||
return new MimirStreamDescriptor(
|
||||
stream.SourceId,
|
||||
ParseKind(stream.Kind),
|
||||
ParseOrigin(stream.Origin),
|
||||
stream.Enabled);
|
||||
}
|
||||
|
||||
private static IMimirStreamSource? TryCreateSource(MimirStreamConfig stream)
|
||||
{
|
||||
if (!string.Equals(stream.Adapter, "process", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(stream.Command))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new MimirProcessStreamSource(
|
||||
ToDescriptor(stream),
|
||||
new MimirProcessStreamSourceOptions(
|
||||
stream.Command,
|
||||
stream.Arguments,
|
||||
Math.Max(1024, stream.ChunkBytes)));
|
||||
}
|
||||
|
||||
private static MimirStreamKind ParseKind(string value)
|
||||
{
|
||||
return Enum.TryParse<MimirStreamKind>(value, ignoreCase: true, out var kind)
|
||||
? kind
|
||||
: throw new InvalidOperationException($"Unknown Mimir stream kind: {value}");
|
||||
}
|
||||
|
||||
private static MimirStreamOrigin ParseOrigin(string value)
|
||||
{
|
||||
return Enum.TryParse<MimirStreamOrigin>(value, ignoreCase: true, out var origin)
|
||||
? origin
|
||||
: throw new InvalidOperationException($"Unknown Mimir stream origin: {value}");
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class MimirRuntimeConfigFile
|
||||
{
|
||||
public double BufferSeconds { get; set; } = 5.0;
|
||||
|
||||
public List<MimirStreamConfig> Streams { get; set; } = [];
|
||||
}
|
||||
|
||||
public sealed class MimirStreamConfig
|
||||
{
|
||||
public string SourceId { get; set; } = "";
|
||||
|
||||
public string Kind { get; set; } = "Video";
|
||||
|
||||
public string Origin { get; set; } = "LocalDevice";
|
||||
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
public string Adapter { get; set; } = "process";
|
||||
|
||||
public string Command { get; set; } = "";
|
||||
|
||||
public string[] Arguments { get; set; } = [];
|
||||
|
||||
public int ChunkBytes { get; set; } = 65_536;
|
||||
}
|
||||
|
|
@ -8,4 +8,5 @@ public readonly record struct MimirStreamSample(
|
|||
long ArrivalNs,
|
||||
ulong Sequence,
|
||||
ulong PayloadHandle,
|
||||
int ByteLength = 0);
|
||||
int ByteLength = 0,
|
||||
ReadOnlyMemory<byte> Data = default);
|
||||
|
|
|
|||
Loading…
Reference in a new issue