diff --git a/crates/muninn-daemon/src/main.rs b/crates/muninn-daemon/src/main.rs index e687544..d4ed5e8 100644 --- a/crates/muninn-daemon/src/main.rs +++ b/crates/muninn-daemon/src/main.rs @@ -171,6 +171,7 @@ struct Options { audio_sample_rate: u32, audio_channels: u32, ffmpeg_path: String, + video_encoder_path: Option, loopback_script: PathBuf, log_root: PathBuf, interval_seconds: Option, @@ -1217,6 +1218,10 @@ fn spawn_capture_stream_activation( "--log-root".to_string(), options.log_root.display().to_string(), ]); + if let Some(path) = &options.video_encoder_path { + args.push("--video-encoder".to_string()); + args.push(path.display().to_string()); + } Command::new(exe) .args(args) .stdin(Stdio::null()) @@ -1857,18 +1862,36 @@ fn run_rudp_mux_once( }; let mut video_ffmpeg = if options.capture_video { + let (program, args, controllable) = if let Some(path) = &options.video_encoder_path { + ( + path.as_os_str(), + muninn_controllable_video_encoder_args(options), + true, + ) + } else { + ( + std::ffi::OsStr::new(&options.ffmpeg_path), + rudp_video_ffmpeg_args(options), + false, + ) + }; Some( - Command::new(&options.ffmpeg_path) - .args(rudp_video_ffmpeg_args(options)) - .stdin(Stdio::null()) + Command::new(program) + .args(args) + .stdin(if controllable { Stdio::piped() } else { Stdio::null() }) .stdout(Stdio::piped()) .stderr(fs::File::create(&video_ffmpeg_stderr)?) .spawn() - .with_context(|| format!("starting {} video encoder", options.ffmpeg_path))?, + .with_context(|| format!("starting {} video encoder", program.to_string_lossy()))?, ) } else { None }; + let mut video_encoder_control = if options.video_encoder_path.is_some() { + video_ffmpeg.as_mut().and_then(|process| process.stdin.take()) + } else { + None + }; let video_ffmpeg_stdout = if let Some(process) = video_ffmpeg.as_mut() { Some( process @@ -2052,10 +2075,12 @@ fn run_rudp_mux_once( &mut video_send_pacer, payloads_dropped, )?; - record_receiver_keyframe_pressure( + if record_receiver_keyframe_pressure( &receiver_feedback, &mut handled_keyframe_requests, - ); + ) { + request_video_encoder_idr(video_encoder_control.as_mut())?; + } poll_rudp_resends_with_backpressure(&mut video_transport)?; poll_rudp_resends_with_backpressure(&mut audio_transport)?; republish_running_stream_if_due( @@ -2126,10 +2151,12 @@ fn run_rudp_mux_once( &mut video_send_pacer, payloads_dropped, )?; - record_receiver_keyframe_pressure( + if record_receiver_keyframe_pressure( &receiver_feedback, &mut handled_keyframe_requests, - ); + ) { + request_video_encoder_idr(video_encoder_control.as_mut())?; + } poll_rudp_resends_with_backpressure(&mut video_transport)?; poll_rudp_resends_with_backpressure(&mut audio_transport)?; republish_running_stream_if_due( @@ -2196,10 +2223,12 @@ fn run_rudp_mux_once( &mut video_send_pacer, payloads_dropped, )?; - record_receiver_keyframe_pressure( + if record_receiver_keyframe_pressure( &receiver_feedback, &mut handled_keyframe_requests, - ); + ) { + request_video_encoder_idr(video_encoder_control.as_mut())?; + } poll_rudp_resends_with_backpressure(&mut video_transport)?; poll_rudp_resends_with_backpressure(&mut audio_transport)?; republish_running_stream_if_due( @@ -2619,14 +2648,25 @@ fn default_rudp_mux_restart_delay(restart_count: u32) -> Duration { fn record_receiver_keyframe_pressure( receiver_feedback: &MuninnRudpReceiverFeedbackStats, handled_keyframe_requests: &mut u64, -) { +) -> bool { if receiver_feedback.requested_keyframes <= *handled_keyframe_requests { - return; + return false; } *handled_keyframe_requests = receiver_feedback.requested_keyframes; - eprintln!( - "Muninn RUDP receiver requested a fresh keyframe; continuing current low-latency encoder session until explicit encoder control exists." - ); + true +} + +fn request_video_encoder_idr(control: Option<&mut W>) -> Result<()> { + if let Some(control) = control { + control.write_all(b"IDR\n")?; + control.flush()?; + eprintln!("Muninn RUDP receiver invalidated the decode chain; requested next-frame IDR."); + } else { + eprintln!( + "Muninn RUDP receiver invalidated the decode chain; scheduled quarter-second IDR remains the recovery ceiling because the configured FFmpeg CLI has no live encoder command surface." + ); + } + Ok(()) } fn rudp_media_progress_detail( @@ -9840,6 +9880,22 @@ fn rudp_video_ffmpeg_args(options: &Options) -> Vec { ] } +fn muninn_controllable_video_encoder_args(options: &Options) -> Vec { + vec![ + "--input".to_string(), + format!( + "ddagrab=framerate={}:output_idx={}:draw_mouse=1", + options.framerate, options.ddagrab_output_index + ), + "--framerate".to_string(), + options.framerate.max(1).to_string(), + "--bitrate-kbps".to_string(), + options.rudp_video_bitrate_kbps.to_string(), + "--gop-frames".to_string(), + muninn_rudp_video_gop_frames(options).to_string(), + ] +} + fn rudp_audio_ffmpeg_args(options: &Options) -> Vec { vec![ "-hide_banner".to_string(), @@ -9953,6 +10009,7 @@ impl Options { audio_sample_rate: 48000, audio_channels: 2, ffmpeg_path: "ffmpeg".to_string(), + video_encoder_path: None, loopback_script: PathBuf::from("scripts/wasapi-loopback-capture.ps1"), log_root: PathBuf::from("C:/Meta/Odin/logs/muninn"), interval_seconds: None, @@ -10106,6 +10163,12 @@ impl Options { options.audio_channels = take_value(&mut args, "--audio-channels")?.parse()? } "--ffmpeg" => options.ffmpeg_path = take_value(&mut args, "--ffmpeg")?, + "--video-encoder" => { + options.video_encoder_path = Some(PathBuf::from(take_value( + &mut args, + "--video-encoder", + )?)) + } "--loopback-script" => { options.loopback_script = PathBuf::from(take_value(&mut args, "--loopback-script")?) @@ -11535,6 +11598,46 @@ mod tests { assert_eq!(muninn_rudp_video_gop_frames(&sixty_fps), 15); } + #[test] + fn controllable_video_encoder_owns_capture_and_recovery_shape() { + let options = Options::parse( + [ + "activate", + "--media-transport", + "rudp", + "--framerate", + "60", + "--rudp-video-bitrate-kbps", + "16000", + "--ddagrab-output-index", + "1", + "--video-encoder", + "C:/GameCult/muninn-video-encoder.exe", + ] + .into_iter() + .map(String::from), + ) + .unwrap(); + + assert_eq!( + options.video_encoder_path, + Some(PathBuf::from("C:/GameCult/muninn-video-encoder.exe")) + ); + assert_eq!( + muninn_controllable_video_encoder_args(&options), + vec![ + "--input", + "ddagrab=framerate=60:output_idx=1:draw_mouse=1", + "--framerate", + "60", + "--bitrate-kbps", + "16000", + "--gop-frames", + "15", + ] + ); + } + #[test] fn default_rudp_media_packet_size_keeps_late_typed_video_wire_under_udp_mtu() { let mut access_unit = Vec::new(); @@ -11897,24 +12000,31 @@ mod tests { } #[test] - fn receiver_feedback_keyframe_requests_are_recorded_without_encoder_restart() { + fn receiver_feedback_keyframe_requests_are_edge_triggered() { let mut handled = 0; let mut receiver_feedback = MuninnRudpReceiverFeedbackStats::default(); - record_receiver_keyframe_pressure(&receiver_feedback, &mut handled); + assert!(!record_receiver_keyframe_pressure(&receiver_feedback, &mut handled)); assert_eq!(handled, 0); receiver_feedback.requested_keyframes = 1; - record_receiver_keyframe_pressure(&receiver_feedback, &mut handled); + assert!(record_receiver_keyframe_pressure(&receiver_feedback, &mut handled)); assert_eq!(handled, 1); - record_receiver_keyframe_pressure(&receiver_feedback, &mut handled); + assert!(!record_receiver_keyframe_pressure(&receiver_feedback, &mut handled)); assert_eq!(handled, 1); receiver_feedback.requested_keyframes = 2; - record_receiver_keyframe_pressure(&receiver_feedback, &mut handled); + assert!(record_receiver_keyframe_pressure(&receiver_feedback, &mut handled)); assert_eq!(handled, 2); } + #[test] + fn receiver_keyframe_pressure_writes_live_encoder_idr_command() { + let mut commands = Vec::new(); + request_video_encoder_idr(Some(&mut commands)).unwrap(); + assert_eq!(commands, b"IDR\n"); + } + #[test] fn rudp_media_receiver_feedback_updates_sender_pressure_stats() { let feedback = crate::media_packetizer::build_receiver_feedback( diff --git a/docs/realtime-delivery-authority.md b/docs/realtime-delivery-authority.md index 0333cd6..fef4457 100644 --- a/docs/realtime-delivery-authority.md +++ b/docs/realtime-delivery-authority.md @@ -99,11 +99,10 @@ The machine is reliable when each class loses only what its contract permits. - Recoverable missing video chunks remain repair/FEC damage. Only explicit or dependency-derived decode-chain invalidation requests a keyframe. -Still open: real encoder keyframe actuation, bounded long-disconnect admission -for unacknowledged HID edges, adaptive bitrate/parity from receiver pressure, -and field acceptance runs through the impairment harness. The current encoder -still guarantees a scheduled IDR every quarter second, so missing dynamic IDR -actuation has a hard recovery ceiling rather than an unbounded decoder stall. +Still open: bounded long-disconnect admission for unacknowledged HID edges, +adaptive bitrate/parity from receiver pressure, packaging the controllable +encoder on each capture body, and field acceptance runs through the impairment +harness. The scheduled IDR every quarter second remains a fallback ceiling. Receiver audit correction on 2026-07-16: the native Mimir/OBS receiver already contains production XOR parity reconstruction. Its early repair feedback was @@ -129,3 +128,13 @@ loss, delay/jitter, reorder, duplication, and stalls with a bounded scheduled queue and CSV evidence. Profiles live under `tests/realtime-impairment`; the tool's six deterministic policy tests and release build pass. It observes the transport without becoming a second transport authority. + +The encoder command loop is now closed. `native/muninn-video-encoder` owns the +long-lived libavdevice desktop capture and `h264_nvenc` context. Muninn retains +the receiver-feedback decision and writes one `IDR` command only when a new +decode-chain invalidation edge arrives. The encoder marks the next submitted +`AVFrame` intra with NVENC `forced-idr=1`; it does not restart video, audio, or +CultNet. A live 2560x1440 D3D11 desktop run with a 600-frame scheduled GOP +received `IDR` over stdin and emitted a second cleanly decoded IDR at frame 22. +The synthetic 640x360 verification emitted IDRs at frames 0 and 10 and decoded +without errors. The quarter-second scheduled GOP stays as independent fallback. diff --git a/native/muninn-video-encoder/CMakeLists.txt b/native/muninn-video-encoder/CMakeLists.txt new file mode 100644 index 0000000..ab2c36c --- /dev/null +++ b/native/muninn-video-encoder/CMakeLists.txt @@ -0,0 +1,22 @@ +cmake_minimum_required(VERSION 3.24) +project(muninn_video_encoder LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +set(FFMPEG_ROOT "" CACHE PATH "FFmpeg development root containing include/ and lib/") +if(NOT FFMPEG_ROOT) + message(FATAL_ERROR "Set FFMPEG_ROOT to the OBS/FFmpeg dependency root") +endif() + +add_executable(muninn-video-encoder main.cpp) +target_include_directories(muninn-video-encoder PRIVATE "${FFMPEG_ROOT}/include") +target_link_directories(muninn-video-encoder PRIVATE "${FFMPEG_ROOT}/lib") +target_link_libraries(muninn-video-encoder PRIVATE + avcodec avdevice avfilter avformat avutil) + +if(MSVC) + target_compile_options(muninn-video-encoder PRIVATE /W4 /permissive-) +else() + target_compile_options(muninn-video-encoder PRIVATE -Wall -Wextra -Wpedantic -Werror) +endif() diff --git a/native/muninn-video-encoder/README.md b/native/muninn-video-encoder/README.md new file mode 100644 index 0000000..eba3f35 --- /dev/null +++ b/native/muninn-video-encoder/README.md @@ -0,0 +1,25 @@ +# Muninn controllable video encoder + +This process owns desktop capture frames and the long-lived NVENC context for +Muninn's realtime video leg. Annex-B H.264 is written to stdout. Stdin accepts +one command per line: + +- `IDR` marks the next submitted frame `AV_PICTURE_TYPE_I`. With NVENC + `forced-idr=1`, FFmpeg lowers that request to an IDR rather than an ordinary + intra frame. +- `QUIT` ends capture cleanly. + +The process does not own transport, audio, packetization, deadlines, or repair. +Muninn owns those and writes `IDR` only when receiver feedback proves the decode +chain is invalid. + +Build against the same FFmpeg development snapshot shipped with OBS: + +```powershell +cmake -S . -B build -DFFMPEG_ROOT=E:/path/to/obs-deps +cmake --build build --config Release +``` + +`--input` defaults to `ddagrab=framerate=60:output_idx=0:draw_mouse=1`. +`--force-idr-frame N` exists only for deterministic encoder verification. +`--frames N` bounds deterministic verification runs. diff --git a/native/muninn-video-encoder/main.cpp b/native/muninn-video-encoder/main.cpp new file mode 100644 index 0000000..66e4d51 --- /dev/null +++ b/native/muninn-video-encoder/main.cpp @@ -0,0 +1,244 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#include +#endif + +extern "C" { +#include +#include +#include +#include +#include +#include +} + +namespace { + +struct Options { + std::string input = "ddagrab=framerate=60:output_idx=0:draw_mouse=1"; + int framerate = 60; + int bitrate_kbps = 12000; + int gop_frames = 15; + std::optional max_frames; + std::optional force_idr_frame; +}; + +std::string ff_error(int code) +{ + char text[AV_ERROR_MAX_STRING_SIZE]{}; + av_strerror(code, text, sizeof(text)); + return text; +} + +[[noreturn]] void fail(const std::string &message) +{ + std::cerr << "muninn-video-encoder: " << message << '\n'; + std::exit(1); +} + +int parse_positive(const char *value, const char *name) +{ + char *end = nullptr; + errno = 0; + const long parsed = std::strtol(value, &end, 10); + if (errno != 0 || end == value || *end != '\0' || parsed <= 0 || parsed > INT32_MAX) + fail(std::string("invalid ") + name); + return static_cast(parsed); +} + +Options parse_options(int argc, char **argv) +{ + Options result; + for (int index = 1; index < argc; ++index) { + const std::string_view arg(argv[index]); + const auto take = [&]() -> const char * { + if (++index >= argc) fail(std::string("missing value for ") + std::string(arg)); + return argv[index]; + }; + if (arg == "--input") result.input = take(); + else if (arg == "--framerate") result.framerate = parse_positive(take(), "framerate"); + else if (arg == "--bitrate-kbps") result.bitrate_kbps = parse_positive(take(), "bitrate"); + else if (arg == "--gop-frames") result.gop_frames = parse_positive(take(), "gop"); + else if (arg == "--force-idr-frame") + result.force_idr_frame = static_cast(parse_positive(take(), "force frame")); + else if (arg == "--frames") + result.max_frames = static_cast(parse_positive(take(), "frames")); + else fail("unknown argument: " + std::string(arg)); + } + return result; +} + +struct FormatCloser { void operator()(AVFormatContext *value) const { avformat_close_input(&value); } }; +struct CodecCloser { void operator()(AVCodecContext *value) const { avcodec_free_context(&value); } }; +struct FrameCloser { void operator()(AVFrame *value) const { av_frame_free(&value); } }; +struct PacketCloser { void operator()(AVPacket *value) const { av_packet_free(&value); } }; + +using FormatPtr = std::unique_ptr; +using CodecPtr = std::unique_ptr; +using FramePtr = std::unique_ptr; +using PacketPtr = std::unique_ptr; + +void write_packet(const AVPacket *packet) +{ + if (packet->size == 0) return; + if (std::fwrite(packet->data, 1, static_cast(packet->size), stdout) != + static_cast(packet->size)) + fail("writing Annex-B stdout failed"); + std::fflush(stdout); +} + +void drain_encoder(AVCodecContext *encoder, AVPacket *packet) +{ + while (true) { + const int result = avcodec_receive_packet(encoder, packet); + if (result == AVERROR(EAGAIN) || result == AVERROR_EOF) return; + if (result < 0) fail("receiving encoded packet: " + ff_error(result)); + write_packet(packet); + av_packet_unref(packet); + } +} + +} // namespace + +int main(int argc, char **argv) +{ +#ifdef _WIN32 + if (_setmode(_fileno(stdout), _O_BINARY) == -1) + fail("setting stdout binary mode failed"); +#endif + const Options options = parse_options(argc, argv); + avdevice_register_all(); + + const AVInputFormat *lavfi = av_find_input_format("lavfi"); + if (!lavfi) fail("this FFmpeg build has no lavfi input device"); + AVFormatContext *raw_input = nullptr; + int result = avformat_open_input(&raw_input, options.input.c_str(), lavfi, nullptr); + if (result < 0) fail("opening capture input: " + ff_error(result)); + FormatPtr input(raw_input); + result = avformat_find_stream_info(input.get(), nullptr); + if (result < 0) fail("reading capture stream information: " + ff_error(result)); + const int video_stream = av_find_best_stream(input.get(), AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0); + if (video_stream < 0) fail("capture input has no video stream"); + + const AVCodec *decoder_codec = avcodec_find_decoder( + input->streams[video_stream]->codecpar->codec_id); + if (!decoder_codec) fail("capture frame decoder is unavailable"); + CodecPtr decoder(avcodec_alloc_context3(decoder_codec)); + if (!decoder) fail("allocating capture decoder"); + result = avcodec_parameters_to_context(decoder.get(), input->streams[video_stream]->codecpar); + if (result < 0) fail("copying capture parameters: " + ff_error(result)); + result = avcodec_open2(decoder.get(), decoder_codec, nullptr); + if (result < 0) fail("opening capture decoder: " + ff_error(result)); + + const AVCodec *encoder_codec = avcodec_find_encoder_by_name("h264_nvenc"); + if (!encoder_codec) fail("h264_nvenc is unavailable"); + CodecPtr encoder(avcodec_alloc_context3(encoder_codec)); + if (!encoder) fail("allocating h264_nvenc context"); + + PacketPtr input_packet(av_packet_alloc()); + PacketPtr output_packet(av_packet_alloc()); + FramePtr frame(av_frame_alloc()); + if (!input_packet || !output_packet || !frame) fail("allocating frame pipeline"); + + std::atomic_bool force_next_idr{false}; + std::atomic_bool quit{false}; + std::thread command_reader([&] { + std::string command; + while (std::getline(std::cin, command)) { + if (command == "IDR") force_next_idr.store(true, std::memory_order_release); + else if (command == "QUIT") { quit.store(true, std::memory_order_release); break; } + } + }); + command_reader.detach(); + + bool encoder_open = false; + std::uint64_t frame_number = 0; + while (!quit.load(std::memory_order_acquire)) { + result = av_read_frame(input.get(), input_packet.get()); + if (result == AVERROR_EOF) break; + if (result < 0) fail("reading capture frame: " + ff_error(result)); + if (input_packet->stream_index != video_stream) { + av_packet_unref(input_packet.get()); + continue; + } + result = avcodec_send_packet(decoder.get(), input_packet.get()); + av_packet_unref(input_packet.get()); + if (result < 0) fail("submitting capture packet: " + ff_error(result)); + + while ((result = avcodec_receive_frame(decoder.get(), frame.get())) >= 0) { + if (!encoder_open) { + encoder->width = frame->width; + encoder->height = frame->height; + encoder->pix_fmt = static_cast(frame->format); + encoder->time_base = AVRational{1, options.framerate}; + encoder->framerate = AVRational{options.framerate, 1}; + encoder->bit_rate = static_cast(options.bitrate_kbps) * 1000; + encoder->rc_max_rate = encoder->bit_rate; + encoder->rc_buffer_size = static_cast(encoder->bit_rate / options.framerate * 2); + encoder->gop_size = options.gop_frames; + encoder->max_b_frames = 0; + if (frame->hw_frames_ctx) encoder->hw_frames_ctx = av_buffer_ref(frame->hw_frames_ctx); + AVDictionary *settings = nullptr; + av_dict_set(&settings, "preset", "p1", 0); + av_dict_set(&settings, "tune", "ull", 0); + av_dict_set(&settings, "zerolatency", "1", 0); + av_dict_set(&settings, "delay", "0", 0); + av_dict_set(&settings, "rc", "cbr", 0); + av_dict_set(&settings, "rc-lookahead", "0", 0); + av_dict_set(&settings, "multipass", "disabled", 0); + av_dict_set(&settings, "strict_gop", "1", 0); + av_dict_set(&settings, "forced-idr", "1", 0); + av_dict_set(&settings, "aud", "1", 0); + result = avcodec_open2(encoder.get(), encoder_codec, &settings); + av_dict_free(&settings); + if (result < 0) fail("opening h264_nvenc: " + ff_error(result)); + if (encoder->extradata && encoder->extradata_size > 0) { + if (std::fwrite(encoder->extradata, 1, + static_cast(encoder->extradata_size), stdout) != + static_cast(encoder->extradata_size)) + fail("writing encoder headers failed"); + } + std::cerr << "muninn-video-encoder: " << encoder->width << 'x' + << encoder->height << " format=" << encoder->pix_fmt + << " headers=" << encoder->extradata_size << '\n'; + encoder_open = true; + } + + const bool scheduled = options.force_idr_frame == frame_number; + if (scheduled || force_next_idr.exchange(false, std::memory_order_acq_rel)) + frame->pict_type = AV_PICTURE_TYPE_I; + else + frame->pict_type = AV_PICTURE_TYPE_NONE; + frame->pts = static_cast(frame_number++); + result = avcodec_send_frame(encoder.get(), frame.get()); + av_frame_unref(frame.get()); + if (result < 0) fail("submitting frame to h264_nvenc: " + ff_error(result)); + drain_encoder(encoder.get(), output_packet.get()); + if (options.max_frames && frame_number >= *options.max_frames) { + quit.store(true, std::memory_order_release); + break; + } + } + if (quit.load(std::memory_order_acquire)) break; + if (result != AVERROR(EAGAIN) && result != AVERROR_EOF) + fail("decoding capture frame: " + ff_error(result)); + } + + if (encoder_open) { + result = avcodec_send_frame(encoder.get(), nullptr); + if (result >= 0) drain_encoder(encoder.get(), output_packet.get()); + } + return 0; +}