Stabilize Muninn RUDP media under LAN pressure
This commit is contained in:
parent
470e648b16
commit
7e344b8554
1 changed files with 167 additions and 121 deletions
|
|
@ -27,7 +27,7 @@ use serde_json::json;
|
|||
use std::collections::HashMap;
|
||||
use std::env;
|
||||
use std::fs;
|
||||
use std::io::Read;
|
||||
use std::io::{ErrorKind, Read};
|
||||
#[cfg(not(windows))]
|
||||
use std::io::Write;
|
||||
use std::net::{SocketAddr, UdpSocket};
|
||||
|
|
@ -42,8 +42,6 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
|||
#[cfg(windows)]
|
||||
use std::ffi::OsStr;
|
||||
#[cfg(unix)]
|
||||
use std::io::ErrorKind;
|
||||
#[cfg(unix)]
|
||||
use std::os::fd::AsRawFd;
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
|
|
@ -56,7 +54,7 @@ const MUNINN_COMMAND_RUDP_CONNECTION_ID: u32 = 0x6d75_0002;
|
|||
const MUNINN_MEDIA_RUDP_CONNECTION_ID: u32 = 0x6d75_0001;
|
||||
const MUNINN_MEDIA_SEND_QUEUE_DEADLINE_MS: u64 = 75;
|
||||
const MUNINN_RUDP_MEDIA_PROFILE_ID: &str = "muninn.rudp.low_latency_h264_lan.v1";
|
||||
const MUNINN_RUDP_MEDIA_VIDEO_BITRATE_KBPS: u32 = 12_000;
|
||||
const MUNINN_RUDP_MEDIA_VIDEO_BITRATE_KBPS: u32 = 8_000;
|
||||
const MUNINN_RUDP_MEDIA_PACKET_BYTES: usize = 480;
|
||||
const MUNINN_RUDP_IPV4_UDP_PAYLOAD_BYTES: usize = 1_472;
|
||||
const MUNINN_RUDP_FIXED_HEADER_BYTES: usize = 36;
|
||||
|
|
@ -64,10 +62,9 @@ const MUNINN_RUDP_MEDIA_MAX_FRAGMENT_BYTES: usize = MUNINN_RUDP_IPV4_UDP_PAYLOAD
|
|||
- MUNINN_RUDP_FIXED_HEADER_BYTES
|
||||
- crate::media_packetizer::MUNINN_MEDIA_RUDP_CHANNEL.len();
|
||||
const MUNINN_RUDP_MEDIA_RESEND_DELAY_MS: u64 = 5;
|
||||
const MUNINN_RUDP_MEDIA_RELIABLE_EXPIRE_AFTER_MS: u64 = 75;
|
||||
const MUNINN_RUDP_MEDIA_RECEIVER_ASSEMBLY_DEADLINE_MS: u64 = 75;
|
||||
const MUNINN_RUDP_MEDIA_RELIABLE_EXPIRE_AFTER_MS: u64 = 250;
|
||||
const MUNINN_RUDP_MEDIA_RECEIVER_ASSEMBLY_DEADLINE_MS: u64 = 150;
|
||||
const MUNINN_RUDP_MEDIA_RECEIVER_GAP_WAIT_MS: u64 = 8;
|
||||
const MUNINN_MEDIA_RECEIVER_KEYFRAME_RESTART_DELAY_MS: u64 = 100;
|
||||
const PS_MOVE_LED_REPORT_LEN: usize = 49;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
|
|
@ -638,6 +635,10 @@ fn spawn_capture_stream_activation(
|
|||
command: &MuninnCaptureStreamCommandRecord,
|
||||
) -> Result<Child> {
|
||||
let exe = env::current_exe().context("resolving current Muninn executable")?;
|
||||
let activation_stderr = options.log_root.join(format!(
|
||||
"muninn-activation-{}.err.log",
|
||||
safe_log_component(&command.command_id)
|
||||
));
|
||||
let mut args = vec![
|
||||
"activate".to_string(),
|
||||
"--store".to_string(),
|
||||
|
|
@ -683,11 +684,29 @@ fn spawn_capture_stream_activation(
|
|||
.args(args)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.stderr(fs::File::create(&activation_stderr).with_context(|| {
|
||||
format!(
|
||||
"creating activation stderr log {}",
|
||||
activation_stderr.display()
|
||||
)
|
||||
})?)
|
||||
.spawn()
|
||||
.context("spawning daemon-owned Muninn capture activation child")
|
||||
}
|
||||
|
||||
fn safe_log_component(value: &str) -> String {
|
||||
value
|
||||
.chars()
|
||||
.map(|ch| {
|
||||
if ch.is_ascii_alphanumeric() || matches!(ch, '.' | '-' | '_') {
|
||||
ch
|
||||
} else {
|
||||
'_'
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn active_move_state_sources(sources: Vec<MoveStateSource>) -> Vec<ActiveMoveStateSource> {
|
||||
sources.into_iter().map(active_move_state_source).collect()
|
||||
}
|
||||
|
|
@ -1256,7 +1275,7 @@ fn run_rudp_mux_once(
|
|||
timebase_den: 90_000,
|
||||
deadline_delay_ticks: i64::from(video_frame_duration_ticks(options)?),
|
||||
max_payload_bytes: options.media_packet_bytes.max(256),
|
||||
max_pending_bytes: options.media_packet_bytes.max(256) * 256,
|
||||
max_pending_bytes: options.media_packet_bytes.max(256) * 4096,
|
||||
source_runtime_id: options.host_id.clone(),
|
||||
source_role: "muninn.rudp.video".to_string(),
|
||||
},
|
||||
|
|
@ -1295,19 +1314,11 @@ fn run_rudp_mux_once(
|
|||
) {
|
||||
payloads_dropped += 1;
|
||||
poll_rudp_media_receiver_feedback(&mut transport, &mut receiver_feedback)?;
|
||||
if receiver_feedback_requests_encoder_restart(
|
||||
record_receiver_keyframe_pressure(
|
||||
&receiver_feedback,
|
||||
&mut handled_keyframe_requests,
|
||||
) {
|
||||
let expired = transport.stats().reliable_packets_expired;
|
||||
break Ok(rudp_media_keyframe_restart(
|
||||
payloads_sent,
|
||||
payloads_dropped,
|
||||
expired,
|
||||
&receiver_feedback,
|
||||
));
|
||||
}
|
||||
transport.poll_resends()?;
|
||||
);
|
||||
poll_rudp_resends_with_backpressure(&mut transport)?;
|
||||
if payloads_dropped == 1 || payloads_dropped % 300 == 0 {
|
||||
let expired = transport.stats().reliable_packets_expired;
|
||||
eprintln!(
|
||||
|
|
@ -1324,23 +1335,21 @@ fn run_rudp_mux_once(
|
|||
}
|
||||
|
||||
let payload_len = queued.payload.payload.len();
|
||||
transport
|
||||
.send(queued.payload.channel_id, queued.payload.payload)
|
||||
.context("sending typed Muninn media payload over RUDP media")?;
|
||||
if !send_rudp_media_payload_with_backpressure(
|
||||
&mut transport,
|
||||
queued.payload,
|
||||
queued.queued_at,
|
||||
Duration::from_millis(media_profile.sender_queue_deadline_ms),
|
||||
)? {
|
||||
payloads_dropped += 1;
|
||||
continue;
|
||||
}
|
||||
poll_rudp_media_receiver_feedback(&mut transport, &mut receiver_feedback)?;
|
||||
if receiver_feedback_requests_encoder_restart(
|
||||
record_receiver_keyframe_pressure(
|
||||
&receiver_feedback,
|
||||
&mut handled_keyframe_requests,
|
||||
) {
|
||||
let expired = transport.stats().reliable_packets_expired;
|
||||
break Ok(rudp_media_keyframe_restart(
|
||||
payloads_sent,
|
||||
payloads_dropped,
|
||||
expired,
|
||||
&receiver_feedback,
|
||||
));
|
||||
}
|
||||
transport.poll_resends()?;
|
||||
);
|
||||
poll_rudp_resends_with_backpressure(&mut transport)?;
|
||||
payloads_sent += 1;
|
||||
if payloads_sent == 1 || payloads_sent % 900 == 0 {
|
||||
let expired = transport.stats().reliable_packets_expired;
|
||||
|
|
@ -1358,19 +1367,11 @@ fn run_rudp_mux_once(
|
|||
Ok(Err(error)) => break Err(error),
|
||||
Err(mpsc::RecvTimeoutError::Timeout) => {
|
||||
poll_rudp_media_receiver_feedback(&mut transport, &mut receiver_feedback)?;
|
||||
if receiver_feedback_requests_encoder_restart(
|
||||
record_receiver_keyframe_pressure(
|
||||
&receiver_feedback,
|
||||
&mut handled_keyframe_requests,
|
||||
) {
|
||||
let expired = transport.stats().reliable_packets_expired;
|
||||
break Ok(rudp_media_keyframe_restart(
|
||||
payloads_sent,
|
||||
payloads_dropped,
|
||||
expired,
|
||||
&receiver_feedback,
|
||||
));
|
||||
}
|
||||
transport.poll_resends()?;
|
||||
);
|
||||
poll_rudp_resends_with_backpressure(&mut transport)?;
|
||||
}
|
||||
Err(mpsc::RecvTimeoutError::Disconnected) => {
|
||||
let expired = transport.stats().reliable_packets_expired;
|
||||
|
|
@ -1431,38 +1432,63 @@ fn media_payload_queue_age_exceeded(queued_at: Instant, now: Instant, max_age: D
|
|||
now.saturating_duration_since(queued_at) > max_age
|
||||
}
|
||||
|
||||
fn poll_rudp_resends_with_backpressure(
|
||||
transport: &mut CultNetRudpSocketTransportConnection,
|
||||
) -> Result<()> {
|
||||
match transport.poll_resends() {
|
||||
Ok(()) => Ok(()),
|
||||
Err(error) if is_would_block_error(&error) => Ok(()),
|
||||
Err(error) => Err(error).context("polling Muninn RUDP media resends"),
|
||||
}
|
||||
}
|
||||
|
||||
fn send_rudp_media_payload_with_backpressure(
|
||||
transport: &mut CultNetRudpSocketTransportConnection,
|
||||
payload: MuninnMediaSendPayload,
|
||||
queued_at: Instant,
|
||||
max_age: Duration,
|
||||
) -> Result<bool> {
|
||||
loop {
|
||||
match transport.send(payload.channel_id, payload.payload.clone()) {
|
||||
Ok(()) => return Ok(true),
|
||||
Err(error) if is_would_block_error(&error) => {
|
||||
if media_payload_queue_age_exceeded(queued_at, Instant::now(), max_age) {
|
||||
return Ok(false);
|
||||
}
|
||||
poll_rudp_resends_with_backpressure(transport)?;
|
||||
thread::sleep(Duration::from_millis(1));
|
||||
}
|
||||
Err(error) => {
|
||||
return Err(error).context("sending typed Muninn media payload over RUDP media");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_would_block_error(error: &anyhow::Error) -> bool {
|
||||
error.chain().any(|cause| {
|
||||
cause
|
||||
.downcast_ref::<std::io::Error>()
|
||||
.is_some_and(|io| io.kind() == ErrorKind::WouldBlock)
|
||||
|| cause.to_string().contains("os error 10035")
|
||||
})
|
||||
}
|
||||
|
||||
fn default_rudp_mux_restart_delay(restart_count: u32) -> Duration {
|
||||
Duration::from_secs((2 + restart_count).min(30) as u64)
|
||||
}
|
||||
|
||||
fn receiver_keyframe_restart_delay() -> Duration {
|
||||
Duration::from_millis(MUNINN_MEDIA_RECEIVER_KEYFRAME_RESTART_DELAY_MS)
|
||||
}
|
||||
|
||||
fn receiver_feedback_requests_encoder_restart(
|
||||
fn record_receiver_keyframe_pressure(
|
||||
receiver_feedback: &MuninnRudpReceiverFeedbackStats,
|
||||
handled_keyframe_requests: &mut u64,
|
||||
) -> bool {
|
||||
) {
|
||||
if receiver_feedback.requested_keyframes <= *handled_keyframe_requests {
|
||||
return false;
|
||||
return;
|
||||
}
|
||||
*handled_keyframe_requests = receiver_feedback.requested_keyframes;
|
||||
true
|
||||
}
|
||||
|
||||
fn rudp_media_keyframe_restart(
|
||||
sent: u64,
|
||||
queue_dropped: u64,
|
||||
reliable_expired: u64,
|
||||
receiver_feedback: &MuninnRudpReceiverFeedbackStats,
|
||||
) -> RudpMuxRestart {
|
||||
RudpMuxRestart {
|
||||
detail: format!(
|
||||
"receiver requested a fresh keyframe; {}",
|
||||
rudp_media_progress_detail(sent, queue_dropped, reliable_expired, receiver_feedback)
|
||||
),
|
||||
delay: receiver_keyframe_restart_delay(),
|
||||
}
|
||||
eprintln!(
|
||||
"Muninn RUDP receiver requested a fresh keyframe; continuing current low-latency encoder session until explicit encoder control exists."
|
||||
);
|
||||
}
|
||||
|
||||
fn rudp_media_progress_detail(
|
||||
|
|
@ -1488,10 +1514,14 @@ fn poll_rudp_media_receiver_feedback(
|
|||
transport: &mut CultNetRudpSocketTransportConnection,
|
||||
stats: &mut MuninnRudpReceiverFeedbackStats,
|
||||
) -> Result<()> {
|
||||
while let Some(frame) = transport.receive_once()? {
|
||||
record_rudp_media_receiver_feedback(&frame, stats)?;
|
||||
loop {
|
||||
match transport.receive_once() {
|
||||
Ok(Some(frame)) => record_rudp_media_receiver_feedback(&frame, stats)?,
|
||||
Ok(None) => return Ok(()),
|
||||
Err(error) if is_would_block_error(&error) => return Ok(()),
|
||||
Err(error) => return Err(error).context("polling Muninn RUDP media feedback"),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn record_rudp_media_receiver_feedback(
|
||||
|
|
@ -5095,6 +5125,11 @@ fn muninn_rudp_video_vbv_buffer_arg(options: &Options, profile: &MuninnRudpMedia
|
|||
format!("{frame_budget_kbits}k")
|
||||
}
|
||||
|
||||
fn muninn_rudp_video_gop_frames(options: &Options) -> u32 {
|
||||
let _ = options;
|
||||
1
|
||||
}
|
||||
|
||||
fn loopback_args(options: &Options) -> Vec<String> {
|
||||
vec![
|
||||
"-NoProfile".to_string(),
|
||||
|
|
@ -5151,6 +5186,14 @@ fn rudp_video_ffmpeg_args(options: &Options) -> Vec<String> {
|
|||
"cbr".to_string(),
|
||||
"-rc-lookahead".to_string(),
|
||||
profile.video_rc_lookahead.to_string(),
|
||||
"-multipass".to_string(),
|
||||
"disabled".to_string(),
|
||||
"-strict_gop".to_string(),
|
||||
"1".to_string(),
|
||||
"-nonref_p".to_string(),
|
||||
"1".to_string(),
|
||||
"-b_ref_mode".to_string(),
|
||||
"disabled".to_string(),
|
||||
"-b:v".to_string(),
|
||||
muninn_rudp_video_bitrate_arg(&profile),
|
||||
"-maxrate".to_string(),
|
||||
|
|
@ -5158,9 +5201,15 @@ fn rudp_video_ffmpeg_args(options: &Options) -> Vec<String> {
|
|||
"-bufsize".to_string(),
|
||||
muninn_rudp_video_vbv_buffer_arg(options, &profile),
|
||||
"-g".to_string(),
|
||||
options.framerate.max(1).to_string(),
|
||||
muninn_rudp_video_gop_frames(options).to_string(),
|
||||
"-keyint_min".to_string(),
|
||||
muninn_rudp_video_gop_frames(options).to_string(),
|
||||
"-forced-idr".to_string(),
|
||||
"1".to_string(),
|
||||
"-fps_mode".to_string(),
|
||||
"cfr".to_string(),
|
||||
"-r".to_string(),
|
||||
options.framerate.max(1).to_string(),
|
||||
"-f".to_string(),
|
||||
profile.video_codec.to_string(),
|
||||
"pipe:1".to_string(),
|
||||
|
|
@ -5259,6 +5308,10 @@ fn ffmpeg_args(options: &Options) -> Vec<String> {
|
|||
"30".to_string(),
|
||||
"-forced-idr".to_string(),
|
||||
"1".to_string(),
|
||||
"-fps_mode".to_string(),
|
||||
"cfr".to_string(),
|
||||
"-r".to_string(),
|
||||
options.framerate.max(1).to_string(),
|
||||
"-c:a".to_string(),
|
||||
"aac".to_string(),
|
||||
"-b:a".to_string(),
|
||||
|
|
@ -6020,7 +6073,7 @@ mod tests {
|
|||
assert_eq!(
|
||||
plan.targets,
|
||||
vec![
|
||||
"rudp://10.77.0.2:5204/muninn.raven.av.rudp?channel=media&format=muninn-typed-media&connection=0x6d750001&profile=muninn.rudp.low_latency_h264_lan.v1&sender_resend_delay_ms=5&reliable_expire_after_ms=75&assembly_deadline_ms=75&gap_wait_ms=8"
|
||||
"rudp://10.77.0.2:5204/muninn.raven.av.rudp?channel=media&format=muninn-typed-media&connection=0x6d750001&profile=muninn.rudp.low_latency_h264_lan.v1&sender_resend_delay_ms=5&reliable_expire_after_ms=250&assembly_deadline_ms=150&gap_wait_ms=8"
|
||||
]
|
||||
);
|
||||
assert!(!plan.command_line.contains("tee"));
|
||||
|
|
@ -6061,19 +6114,47 @@ mod tests {
|
|||
);
|
||||
assert!(
|
||||
args.windows(2)
|
||||
.any(|pair| pair[0] == "-b:v" && pair[1] == "12000k")
|
||||
.any(|pair| pair[0] == "-multipass" && pair[1] == "disabled")
|
||||
);
|
||||
assert!(
|
||||
args.windows(2)
|
||||
.any(|pair| pair[0] == "-maxrate" && pair[1] == "12000k")
|
||||
.any(|pair| pair[0] == "-strict_gop" && pair[1] == "1")
|
||||
);
|
||||
assert!(
|
||||
args.windows(2)
|
||||
.any(|pair| pair[0] == "-bufsize" && pair[1] == "200k")
|
||||
.any(|pair| pair[0] == "-nonref_p" && pair[1] == "1")
|
||||
);
|
||||
assert!(
|
||||
args.windows(2)
|
||||
.any(|pair| pair[0] == "-g" && pair[1] == "60")
|
||||
.any(|pair| pair[0] == "-b_ref_mode" && pair[1] == "disabled")
|
||||
);
|
||||
assert!(
|
||||
args.windows(2)
|
||||
.any(|pair| pair[0] == "-b:v" && pair[1] == "8000k")
|
||||
);
|
||||
assert!(
|
||||
args.windows(2)
|
||||
.any(|pair| pair[0] == "-maxrate" && pair[1] == "8000k")
|
||||
);
|
||||
assert!(
|
||||
args.windows(2)
|
||||
.any(|pair| pair[0] == "-bufsize" && pair[1] == "134k")
|
||||
);
|
||||
assert!(
|
||||
args.windows(2)
|
||||
.any(|pair| pair[0] == "-g" && pair[1] == "1")
|
||||
);
|
||||
assert!(
|
||||
args.windows(2)
|
||||
.any(|pair| pair[0] == "-keyint_min" && pair[1] == "1")
|
||||
);
|
||||
assert!(
|
||||
args.windows(2)
|
||||
.any(|pair| pair[0] == "-fps_mode" && pair[1] == "cfr")
|
||||
);
|
||||
assert!(
|
||||
args.windows(2)
|
||||
.any(|pair| pair[0] == "-r" && pair[1] == "60")
|
||||
);
|
||||
assert!(
|
||||
args.windows(2)
|
||||
|
|
@ -6100,11 +6181,11 @@ mod tests {
|
|||
|
||||
assert_eq!(
|
||||
muninn_rudp_video_vbv_buffer_arg(&thirty_fps, &profile),
|
||||
"400k"
|
||||
"267k"
|
||||
);
|
||||
assert_eq!(
|
||||
muninn_rudp_video_vbv_buffer_arg(&sixty_fps, &profile),
|
||||
"200k"
|
||||
"134k"
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -6240,59 +6321,24 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn receiver_feedback_keyframe_requests_are_single_restart_edges() {
|
||||
fn receiver_feedback_keyframe_requests_are_recorded_without_encoder_restart() {
|
||||
let mut handled = 0;
|
||||
let mut receiver_feedback = MuninnRudpReceiverFeedbackStats::default();
|
||||
|
||||
assert!(!receiver_feedback_requests_encoder_restart(
|
||||
&receiver_feedback,
|
||||
&mut handled
|
||||
));
|
||||
record_receiver_keyframe_pressure(&receiver_feedback, &mut handled);
|
||||
assert_eq!(handled, 0);
|
||||
|
||||
receiver_feedback.requested_keyframes = 1;
|
||||
assert!(receiver_feedback_requests_encoder_restart(
|
||||
&receiver_feedback,
|
||||
&mut handled
|
||||
));
|
||||
record_receiver_keyframe_pressure(&receiver_feedback, &mut handled);
|
||||
assert_eq!(handled, 1);
|
||||
record_receiver_keyframe_pressure(&receiver_feedback, &mut handled);
|
||||
assert_eq!(handled, 1);
|
||||
assert!(!receiver_feedback_requests_encoder_restart(
|
||||
&receiver_feedback,
|
||||
&mut handled
|
||||
));
|
||||
|
||||
receiver_feedback.requested_keyframes = 2;
|
||||
assert!(receiver_feedback_requests_encoder_restart(
|
||||
&receiver_feedback,
|
||||
&mut handled
|
||||
));
|
||||
record_receiver_keyframe_pressure(&receiver_feedback, &mut handled);
|
||||
assert_eq!(handled, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn receiver_keyframe_restart_uses_short_delay_and_live_pressure_detail() {
|
||||
let receiver_feedback = MuninnRudpReceiverFeedbackStats {
|
||||
feedback_records: 1,
|
||||
requested_keyframes: 1,
|
||||
late_frames: 2,
|
||||
missing_video_chunks: 3,
|
||||
highest_decodable_frame_id: Some(40),
|
||||
};
|
||||
|
||||
let restart = rudp_media_keyframe_restart(120, 4, 7, &receiver_feedback);
|
||||
|
||||
assert_eq!(restart.delay, receiver_keyframe_restart_delay());
|
||||
assert!(
|
||||
restart
|
||||
.detail
|
||||
.contains("receiver requested a fresh keyframe")
|
||||
);
|
||||
assert!(restart.detail.contains("sent=120"));
|
||||
assert!(restart.detail.contains("queue_dropped=4"));
|
||||
assert!(restart.detail.contains("reliable_expired=7"));
|
||||
assert!(restart.detail.contains("receiver_keyframes=1"));
|
||||
assert_eq!(default_rudp_mux_restart_delay(0), Duration::from_secs(2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rudp_media_receiver_feedback_updates_sender_pressure_stats() {
|
||||
let feedback = crate::media_packetizer::build_receiver_feedback(
|
||||
|
|
|
|||
Loading…
Reference in a new issue