Actuate receiver-requested Muninn IDR

This commit is contained in:
Meta Vens 2026-07-16 21:17:56 +01:00
commit a6567233bb
5 changed files with 435 additions and 25 deletions

View file

@ -171,6 +171,7 @@ struct Options {
audio_sample_rate: u32,
audio_channels: u32,
ffmpeg_path: String,
video_encoder_path: Option<PathBuf>,
loopback_script: PathBuf,
log_root: PathBuf,
interval_seconds: Option<u64>,
@ -1217,6 +1218,10 @@ fn spawn_capture_stream_activation(
"--log-root".to_string(),
options.log_root.display().to_string(),
]);
if let Some(path) = &options.video_encoder_path {
args.push("--video-encoder".to_string());
args.push(path.display().to_string());
}
Command::new(exe)
.args(args)
.stdin(Stdio::null())
@ -1857,18 +1862,36 @@ fn run_rudp_mux_once(
};
let mut video_ffmpeg = if options.capture_video {
let (program, args, controllable) = if let Some(path) = &options.video_encoder_path {
(
path.as_os_str(),
muninn_controllable_video_encoder_args(options),
true,
)
} else {
(
std::ffi::OsStr::new(&options.ffmpeg_path),
rudp_video_ffmpeg_args(options),
false,
)
};
Some(
Command::new(&options.ffmpeg_path)
.args(rudp_video_ffmpeg_args(options))
.stdin(Stdio::null())
Command::new(program)
.args(args)
.stdin(if controllable { Stdio::piped() } else { Stdio::null() })
.stdout(Stdio::piped())
.stderr(fs::File::create(&video_ffmpeg_stderr)?)
.spawn()
.with_context(|| format!("starting {} video encoder", options.ffmpeg_path))?,
.with_context(|| format!("starting {} video encoder", program.to_string_lossy()))?,
)
} else {
None
};
let mut video_encoder_control = if options.video_encoder_path.is_some() {
video_ffmpeg.as_mut().and_then(|process| process.stdin.take())
} else {
None
};
let video_ffmpeg_stdout = if let Some(process) = video_ffmpeg.as_mut() {
Some(
process
@ -2052,10 +2075,12 @@ fn run_rudp_mux_once(
&mut video_send_pacer,
payloads_dropped,
)?;
record_receiver_keyframe_pressure(
if record_receiver_keyframe_pressure(
&receiver_feedback,
&mut handled_keyframe_requests,
);
) {
request_video_encoder_idr(video_encoder_control.as_mut())?;
}
poll_rudp_resends_with_backpressure(&mut video_transport)?;
poll_rudp_resends_with_backpressure(&mut audio_transport)?;
republish_running_stream_if_due(
@ -2126,10 +2151,12 @@ fn run_rudp_mux_once(
&mut video_send_pacer,
payloads_dropped,
)?;
record_receiver_keyframe_pressure(
if record_receiver_keyframe_pressure(
&receiver_feedback,
&mut handled_keyframe_requests,
);
) {
request_video_encoder_idr(video_encoder_control.as_mut())?;
}
poll_rudp_resends_with_backpressure(&mut video_transport)?;
poll_rudp_resends_with_backpressure(&mut audio_transport)?;
republish_running_stream_if_due(
@ -2196,10 +2223,12 @@ fn run_rudp_mux_once(
&mut video_send_pacer,
payloads_dropped,
)?;
record_receiver_keyframe_pressure(
if record_receiver_keyframe_pressure(
&receiver_feedback,
&mut handled_keyframe_requests,
);
) {
request_video_encoder_idr(video_encoder_control.as_mut())?;
}
poll_rudp_resends_with_backpressure(&mut video_transport)?;
poll_rudp_resends_with_backpressure(&mut audio_transport)?;
republish_running_stream_if_due(
@ -2619,14 +2648,25 @@ fn default_rudp_mux_restart_delay(restart_count: u32) -> Duration {
fn record_receiver_keyframe_pressure(
receiver_feedback: &MuninnRudpReceiverFeedbackStats,
handled_keyframe_requests: &mut u64,
) {
) -> bool {
if receiver_feedback.requested_keyframes <= *handled_keyframe_requests {
return;
return false;
}
*handled_keyframe_requests = receiver_feedback.requested_keyframes;
eprintln!(
"Muninn RUDP receiver requested a fresh keyframe; continuing current low-latency encoder session until explicit encoder control exists."
);
true
}
fn request_video_encoder_idr<W: std::io::Write + ?Sized>(control: Option<&mut W>) -> Result<()> {
if let Some(control) = control {
control.write_all(b"IDR\n")?;
control.flush()?;
eprintln!("Muninn RUDP receiver invalidated the decode chain; requested next-frame IDR.");
} else {
eprintln!(
"Muninn RUDP receiver invalidated the decode chain; scheduled quarter-second IDR remains the recovery ceiling because the configured FFmpeg CLI has no live encoder command surface."
);
}
Ok(())
}
fn rudp_media_progress_detail(
@ -9840,6 +9880,22 @@ fn rudp_video_ffmpeg_args(options: &Options) -> Vec<String> {
]
}
fn muninn_controllable_video_encoder_args(options: &Options) -> Vec<String> {
vec![
"--input".to_string(),
format!(
"ddagrab=framerate={}:output_idx={}:draw_mouse=1",
options.framerate, options.ddagrab_output_index
),
"--framerate".to_string(),
options.framerate.max(1).to_string(),
"--bitrate-kbps".to_string(),
options.rudp_video_bitrate_kbps.to_string(),
"--gop-frames".to_string(),
muninn_rudp_video_gop_frames(options).to_string(),
]
}
fn rudp_audio_ffmpeg_args(options: &Options) -> Vec<String> {
vec![
"-hide_banner".to_string(),
@ -9953,6 +10009,7 @@ impl Options {
audio_sample_rate: 48000,
audio_channels: 2,
ffmpeg_path: "ffmpeg".to_string(),
video_encoder_path: None,
loopback_script: PathBuf::from("scripts/wasapi-loopback-capture.ps1"),
log_root: PathBuf::from("C:/Meta/Odin/logs/muninn"),
interval_seconds: None,
@ -10106,6 +10163,12 @@ impl Options {
options.audio_channels = take_value(&mut args, "--audio-channels")?.parse()?
}
"--ffmpeg" => options.ffmpeg_path = take_value(&mut args, "--ffmpeg")?,
"--video-encoder" => {
options.video_encoder_path = Some(PathBuf::from(take_value(
&mut args,
"--video-encoder",
)?))
}
"--loopback-script" => {
options.loopback_script =
PathBuf::from(take_value(&mut args, "--loopback-script")?)
@ -11535,6 +11598,46 @@ mod tests {
assert_eq!(muninn_rudp_video_gop_frames(&sixty_fps), 15);
}
#[test]
fn controllable_video_encoder_owns_capture_and_recovery_shape() {
let options = Options::parse(
[
"activate",
"--media-transport",
"rudp",
"--framerate",
"60",
"--rudp-video-bitrate-kbps",
"16000",
"--ddagrab-output-index",
"1",
"--video-encoder",
"C:/GameCult/muninn-video-encoder.exe",
]
.into_iter()
.map(String::from),
)
.unwrap();
assert_eq!(
options.video_encoder_path,
Some(PathBuf::from("C:/GameCult/muninn-video-encoder.exe"))
);
assert_eq!(
muninn_controllable_video_encoder_args(&options),
vec![
"--input",
"ddagrab=framerate=60:output_idx=1:draw_mouse=1",
"--framerate",
"60",
"--bitrate-kbps",
"16000",
"--gop-frames",
"15",
]
);
}
#[test]
fn default_rudp_media_packet_size_keeps_late_typed_video_wire_under_udp_mtu() {
let mut access_unit = Vec::new();
@ -11897,24 +12000,31 @@ mod tests {
}
#[test]
fn receiver_feedback_keyframe_requests_are_recorded_without_encoder_restart() {
fn receiver_feedback_keyframe_requests_are_edge_triggered() {
let mut handled = 0;
let mut receiver_feedback = MuninnRudpReceiverFeedbackStats::default();
record_receiver_keyframe_pressure(&receiver_feedback, &mut handled);
assert!(!record_receiver_keyframe_pressure(&receiver_feedback, &mut handled));
assert_eq!(handled, 0);
receiver_feedback.requested_keyframes = 1;
record_receiver_keyframe_pressure(&receiver_feedback, &mut handled);
assert!(record_receiver_keyframe_pressure(&receiver_feedback, &mut handled));
assert_eq!(handled, 1);
record_receiver_keyframe_pressure(&receiver_feedback, &mut handled);
assert!(!record_receiver_keyframe_pressure(&receiver_feedback, &mut handled));
assert_eq!(handled, 1);
receiver_feedback.requested_keyframes = 2;
record_receiver_keyframe_pressure(&receiver_feedback, &mut handled);
assert!(record_receiver_keyframe_pressure(&receiver_feedback, &mut handled));
assert_eq!(handled, 2);
}
#[test]
fn receiver_keyframe_pressure_writes_live_encoder_idr_command() {
let mut commands = Vec::new();
request_video_encoder_idr(Some(&mut commands)).unwrap();
assert_eq!(commands, b"IDR\n");
}
#[test]
fn rudp_media_receiver_feedback_updates_sender_pressure_stats() {
let feedback = crate::media_packetizer::build_receiver_feedback(

View file

@ -99,11 +99,10 @@ The machine is reliable when each class loses only what its contract permits.
- Recoverable missing video chunks remain repair/FEC damage. Only explicit or
dependency-derived decode-chain invalidation requests a keyframe.
Still open: real encoder keyframe actuation, bounded long-disconnect admission
for unacknowledged HID edges, adaptive bitrate/parity from receiver pressure,
and field acceptance runs through the impairment harness. The current encoder
still guarantees a scheduled IDR every quarter second, so missing dynamic IDR
actuation has a hard recovery ceiling rather than an unbounded decoder stall.
Still open: bounded long-disconnect admission for unacknowledged HID edges,
adaptive bitrate/parity from receiver pressure, packaging the controllable
encoder on each capture body, and field acceptance runs through the impairment
harness. The scheduled IDR every quarter second remains a fallback ceiling.
Receiver audit correction on 2026-07-16: the native Mimir/OBS receiver already
contains production XOR parity reconstruction. Its early repair feedback was
@ -129,3 +128,13 @@ loss, delay/jitter, reorder, duplication, and stalls with a bounded scheduled
queue and CSV evidence. Profiles live under `tests/realtime-impairment`; the
tool's six deterministic policy tests and release build pass. It observes the
transport without becoming a second transport authority.
The encoder command loop is now closed. `native/muninn-video-encoder` owns the
long-lived libavdevice desktop capture and `h264_nvenc` context. Muninn retains
the receiver-feedback decision and writes one `IDR` command only when a new
decode-chain invalidation edge arrives. The encoder marks the next submitted
`AVFrame` intra with NVENC `forced-idr=1`; it does not restart video, audio, or
CultNet. A live 2560x1440 D3D11 desktop run with a 600-frame scheduled GOP
received `IDR` over stdin and emitted a second cleanly decoded IDR at frame 22.
The synthetic 640x360 verification emitted IDRs at frames 0 and 10 and decoded
without errors. The quarter-second scheduled GOP stays as independent fallback.

View file

@ -0,0 +1,22 @@
cmake_minimum_required(VERSION 3.24)
project(muninn_video_encoder LANGUAGES CXX)
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(FFMPEG_ROOT "" CACHE PATH "FFmpeg development root containing include/ and lib/")
if(NOT FFMPEG_ROOT)
message(FATAL_ERROR "Set FFMPEG_ROOT to the OBS/FFmpeg dependency root")
endif()
add_executable(muninn-video-encoder main.cpp)
target_include_directories(muninn-video-encoder PRIVATE "${FFMPEG_ROOT}/include")
target_link_directories(muninn-video-encoder PRIVATE "${FFMPEG_ROOT}/lib")
target_link_libraries(muninn-video-encoder PRIVATE
avcodec avdevice avfilter avformat avutil)
if(MSVC)
target_compile_options(muninn-video-encoder PRIVATE /W4 /permissive-)
else()
target_compile_options(muninn-video-encoder PRIVATE -Wall -Wextra -Wpedantic -Werror)
endif()

View file

@ -0,0 +1,25 @@
# Muninn controllable video encoder
This process owns desktop capture frames and the long-lived NVENC context for
Muninn's realtime video leg. Annex-B H.264 is written to stdout. Stdin accepts
one command per line:
- `IDR` marks the next submitted frame `AV_PICTURE_TYPE_I`. With NVENC
`forced-idr=1`, FFmpeg lowers that request to an IDR rather than an ordinary
intra frame.
- `QUIT` ends capture cleanly.
The process does not own transport, audio, packetization, deadlines, or repair.
Muninn owns those and writes `IDR` only when receiver feedback proves the decode
chain is invalid.
Build against the same FFmpeg development snapshot shipped with OBS:
```powershell
cmake -S . -B build -DFFMPEG_ROOT=E:/path/to/obs-deps
cmake --build build --config Release
```
`--input` defaults to `ddagrab=framerate=60:output_idx=0:draw_mouse=1`.
`--force-idr-frame N` exists only for deterministic encoder verification.
`--frames N` bounds deterministic verification runs.

View file

@ -0,0 +1,244 @@
#include <atomic>
#include <cerrno>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <optional>
#include <string>
#include <string_view>
#include <thread>
#ifdef _WIN32
#include <fcntl.h>
#include <io.h>
#endif
extern "C" {
#include <libavcodec/avcodec.h>
#include <libavdevice/avdevice.h>
#include <libavformat/avformat.h>
#include <libavutil/avutil.h>
#include <libavutil/error.h>
#include <libavutil/opt.h>
}
namespace {
struct Options {
std::string input = "ddagrab=framerate=60:output_idx=0:draw_mouse=1";
int framerate = 60;
int bitrate_kbps = 12000;
int gop_frames = 15;
std::optional<std::uint64_t> max_frames;
std::optional<std::uint64_t> force_idr_frame;
};
std::string ff_error(int code)
{
char text[AV_ERROR_MAX_STRING_SIZE]{};
av_strerror(code, text, sizeof(text));
return text;
}
[[noreturn]] void fail(const std::string &message)
{
std::cerr << "muninn-video-encoder: " << message << '\n';
std::exit(1);
}
int parse_positive(const char *value, const char *name)
{
char *end = nullptr;
errno = 0;
const long parsed = std::strtol(value, &end, 10);
if (errno != 0 || end == value || *end != '\0' || parsed <= 0 || parsed > INT32_MAX)
fail(std::string("invalid ") + name);
return static_cast<int>(parsed);
}
Options parse_options(int argc, char **argv)
{
Options result;
for (int index = 1; index < argc; ++index) {
const std::string_view arg(argv[index]);
const auto take = [&]() -> const char * {
if (++index >= argc) fail(std::string("missing value for ") + std::string(arg));
return argv[index];
};
if (arg == "--input") result.input = take();
else if (arg == "--framerate") result.framerate = parse_positive(take(), "framerate");
else if (arg == "--bitrate-kbps") result.bitrate_kbps = parse_positive(take(), "bitrate");
else if (arg == "--gop-frames") result.gop_frames = parse_positive(take(), "gop");
else if (arg == "--force-idr-frame")
result.force_idr_frame = static_cast<std::uint64_t>(parse_positive(take(), "force frame"));
else if (arg == "--frames")
result.max_frames = static_cast<std::uint64_t>(parse_positive(take(), "frames"));
else fail("unknown argument: " + std::string(arg));
}
return result;
}
struct FormatCloser { void operator()(AVFormatContext *value) const { avformat_close_input(&value); } };
struct CodecCloser { void operator()(AVCodecContext *value) const { avcodec_free_context(&value); } };
struct FrameCloser { void operator()(AVFrame *value) const { av_frame_free(&value); } };
struct PacketCloser { void operator()(AVPacket *value) const { av_packet_free(&value); } };
using FormatPtr = std::unique_ptr<AVFormatContext, FormatCloser>;
using CodecPtr = std::unique_ptr<AVCodecContext, CodecCloser>;
using FramePtr = std::unique_ptr<AVFrame, FrameCloser>;
using PacketPtr = std::unique_ptr<AVPacket, PacketCloser>;
void write_packet(const AVPacket *packet)
{
if (packet->size == 0) return;
if (std::fwrite(packet->data, 1, static_cast<size_t>(packet->size), stdout) !=
static_cast<size_t>(packet->size))
fail("writing Annex-B stdout failed");
std::fflush(stdout);
}
void drain_encoder(AVCodecContext *encoder, AVPacket *packet)
{
while (true) {
const int result = avcodec_receive_packet(encoder, packet);
if (result == AVERROR(EAGAIN) || result == AVERROR_EOF) return;
if (result < 0) fail("receiving encoded packet: " + ff_error(result));
write_packet(packet);
av_packet_unref(packet);
}
}
} // namespace
int main(int argc, char **argv)
{
#ifdef _WIN32
if (_setmode(_fileno(stdout), _O_BINARY) == -1)
fail("setting stdout binary mode failed");
#endif
const Options options = parse_options(argc, argv);
avdevice_register_all();
const AVInputFormat *lavfi = av_find_input_format("lavfi");
if (!lavfi) fail("this FFmpeg build has no lavfi input device");
AVFormatContext *raw_input = nullptr;
int result = avformat_open_input(&raw_input, options.input.c_str(), lavfi, nullptr);
if (result < 0) fail("opening capture input: " + ff_error(result));
FormatPtr input(raw_input);
result = avformat_find_stream_info(input.get(), nullptr);
if (result < 0) fail("reading capture stream information: " + ff_error(result));
const int video_stream = av_find_best_stream(input.get(), AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0);
if (video_stream < 0) fail("capture input has no video stream");
const AVCodec *decoder_codec = avcodec_find_decoder(
input->streams[video_stream]->codecpar->codec_id);
if (!decoder_codec) fail("capture frame decoder is unavailable");
CodecPtr decoder(avcodec_alloc_context3(decoder_codec));
if (!decoder) fail("allocating capture decoder");
result = avcodec_parameters_to_context(decoder.get(), input->streams[video_stream]->codecpar);
if (result < 0) fail("copying capture parameters: " + ff_error(result));
result = avcodec_open2(decoder.get(), decoder_codec, nullptr);
if (result < 0) fail("opening capture decoder: " + ff_error(result));
const AVCodec *encoder_codec = avcodec_find_encoder_by_name("h264_nvenc");
if (!encoder_codec) fail("h264_nvenc is unavailable");
CodecPtr encoder(avcodec_alloc_context3(encoder_codec));
if (!encoder) fail("allocating h264_nvenc context");
PacketPtr input_packet(av_packet_alloc());
PacketPtr output_packet(av_packet_alloc());
FramePtr frame(av_frame_alloc());
if (!input_packet || !output_packet || !frame) fail("allocating frame pipeline");
std::atomic_bool force_next_idr{false};
std::atomic_bool quit{false};
std::thread command_reader([&] {
std::string command;
while (std::getline(std::cin, command)) {
if (command == "IDR") force_next_idr.store(true, std::memory_order_release);
else if (command == "QUIT") { quit.store(true, std::memory_order_release); break; }
}
});
command_reader.detach();
bool encoder_open = false;
std::uint64_t frame_number = 0;
while (!quit.load(std::memory_order_acquire)) {
result = av_read_frame(input.get(), input_packet.get());
if (result == AVERROR_EOF) break;
if (result < 0) fail("reading capture frame: " + ff_error(result));
if (input_packet->stream_index != video_stream) {
av_packet_unref(input_packet.get());
continue;
}
result = avcodec_send_packet(decoder.get(), input_packet.get());
av_packet_unref(input_packet.get());
if (result < 0) fail("submitting capture packet: " + ff_error(result));
while ((result = avcodec_receive_frame(decoder.get(), frame.get())) >= 0) {
if (!encoder_open) {
encoder->width = frame->width;
encoder->height = frame->height;
encoder->pix_fmt = static_cast<AVPixelFormat>(frame->format);
encoder->time_base = AVRational{1, options.framerate};
encoder->framerate = AVRational{options.framerate, 1};
encoder->bit_rate = static_cast<int64_t>(options.bitrate_kbps) * 1000;
encoder->rc_max_rate = encoder->bit_rate;
encoder->rc_buffer_size = static_cast<int>(encoder->bit_rate / options.framerate * 2);
encoder->gop_size = options.gop_frames;
encoder->max_b_frames = 0;
if (frame->hw_frames_ctx) encoder->hw_frames_ctx = av_buffer_ref(frame->hw_frames_ctx);
AVDictionary *settings = nullptr;
av_dict_set(&settings, "preset", "p1", 0);
av_dict_set(&settings, "tune", "ull", 0);
av_dict_set(&settings, "zerolatency", "1", 0);
av_dict_set(&settings, "delay", "0", 0);
av_dict_set(&settings, "rc", "cbr", 0);
av_dict_set(&settings, "rc-lookahead", "0", 0);
av_dict_set(&settings, "multipass", "disabled", 0);
av_dict_set(&settings, "strict_gop", "1", 0);
av_dict_set(&settings, "forced-idr", "1", 0);
av_dict_set(&settings, "aud", "1", 0);
result = avcodec_open2(encoder.get(), encoder_codec, &settings);
av_dict_free(&settings);
if (result < 0) fail("opening h264_nvenc: " + ff_error(result));
if (encoder->extradata && encoder->extradata_size > 0) {
if (std::fwrite(encoder->extradata, 1,
static_cast<size_t>(encoder->extradata_size), stdout) !=
static_cast<size_t>(encoder->extradata_size))
fail("writing encoder headers failed");
}
std::cerr << "muninn-video-encoder: " << encoder->width << 'x'
<< encoder->height << " format=" << encoder->pix_fmt
<< " headers=" << encoder->extradata_size << '\n';
encoder_open = true;
}
const bool scheduled = options.force_idr_frame == frame_number;
if (scheduled || force_next_idr.exchange(false, std::memory_order_acq_rel))
frame->pict_type = AV_PICTURE_TYPE_I;
else
frame->pict_type = AV_PICTURE_TYPE_NONE;
frame->pts = static_cast<int64_t>(frame_number++);
result = avcodec_send_frame(encoder.get(), frame.get());
av_frame_unref(frame.get());
if (result < 0) fail("submitting frame to h264_nvenc: " + ff_error(result));
drain_encoder(encoder.get(), output_packet.get());
if (options.max_frames && frame_number >= *options.max_frames) {
quit.store(true, std::memory_order_release);
break;
}
}
if (quit.load(std::memory_order_acquire)) break;
if (result != AVERROR(EAGAIN) && result != AVERROR_EOF)
fail("decoding capture frame: " + ff_error(result));
}
if (encoder_open) {
result = avcodec_send_frame(encoder.get(), nullptr);
if (result >= 0) drain_encoder(encoder.get(), output_packet.get());
}
return 0;
}