From fb01f6dcc740d91ddbea34340dfe4ef85017cf47 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 17 Jul 2026 05:48:17 +0100 Subject: [PATCH] Make Muninn media deadline-owned and wrap-safe --- crates/muninn-daemon/src/main.rs | 178 ++++++--- crates/muninn-daemon/src/media_packetizer.rs | 384 ++++++++++++++----- crates/odin-core/src/documents.rs | 20 +- vendor/cultnet-rs/src/rudp.rs | 250 +++++++++++- 4 files changed, 674 insertions(+), 158 deletions(-) diff --git a/crates/muninn-daemon/src/main.rs b/crates/muninn-daemon/src/main.rs index 099ab3b..e1810fe 100644 --- a/crates/muninn-daemon/src/main.rs +++ b/crates/muninn-daemon/src/main.rs @@ -73,7 +73,7 @@ const MUNINN_RUDP_MEDIA_VIDEO_BITRATE_KBPS: u32 = 12_000; const MUNINN_RUDP_MEDIA_VBV_FRAME_BUDGETS: u32 = 1; const MUNINN_RUDP_MEDIA_LOW_DELAY_KEY_FRAME_SCALE: u32 = 4; const MUNINN_RUDP_MEDIA_VIDEO_DPB_SIZE: u32 = 1; -const MUNINN_RUDP_MEDIA_PACKET_BYTES: usize = 848; +const MUNINN_RUDP_MEDIA_PACKET_BYTES: usize = 832; const MUNINN_RUDP_IPV4_UDP_PAYLOAD_BYTES: usize = 1_472; const MUNINN_RUDP_FIXED_HEADER_BYTES: usize = 36; const MUNINN_RUDP_MEDIA_MAX_FRAGMENT_BYTES: usize = MUNINN_RUDP_IPV4_UDP_PAYLOAD_BYTES @@ -1861,7 +1861,7 @@ fn run_rudp_mux_once( options, node, MUNINN_MEDIA_RUDP_CONNECTION_ID, - None, + Some(media_profile.receiver_assembly_deadline_ms), "video", )?; let mut audio_transport = open_media_rudp_transport( @@ -2016,13 +2016,17 @@ fn run_rudp_mux_once( restart_count, ); - let (payload_tx, payload_rx) = + let (video_payload_tx, video_payload_rx) = + mpsc::sync_channel::>>( + MUNINN_RUDP_MEDIA_PAYLOAD_CHANNEL_CAPACITY, + ); + let (audio_payload_tx, audio_payload_rx) = mpsc::sync_channel::>>( MUNINN_RUDP_MEDIA_PAYLOAD_CHANNEL_CAPACITY, ); video_sender = if let Some(stdout) = video_ffmpeg_stdout { Some(video_rudp_payload_reader( - payload_tx.clone(), + video_payload_tx.clone(), stdout, VideoAnnexBStreamSendConfig { stream_id: options.stream_id.clone(), @@ -2045,7 +2049,7 @@ fn run_rudp_mux_once( }; audio_sender = if let Some(stdout) = audio_ffmpeg_stdout { Some(audio_rudp_payload_reader( - payload_tx.clone(), + audio_payload_tx.clone(), stdout, AudioPcmStreamSendConfig { stream_id: options.stream_id.clone(), @@ -2070,7 +2074,8 @@ fn run_rudp_mux_once( } else { None }; - drop(payload_tx); + drop(video_payload_tx); + drop(audio_payload_tx); let mut payloads_sent = 0_u64; let mut payloads_queue_expired = 0_u64; @@ -2097,17 +2102,26 @@ fn run_rudp_mux_once( ); let mut audio_send_pacer = MuninnRudpMediaSendPacer::new(0, Duration::ZERO); let mut pending_payloads = PendingMuninnMediaSendQueues::default(); - let mut payload_channel_disconnected = false; + let mut video_payload_channel_disconnected = false; + let mut audio_payload_channel_disconnected = false; loop { - if pending_payloads.is_empty() && !payload_channel_disconnected { - payload_channel_disconnected = receive_pending_media_payloads( - &payload_rx, + if !audio_payload_channel_disconnected { + audio_payload_channel_disconnected = drain_available_media_payloads( + &audio_payload_rx, &mut pending_payloads, - Duration::from_millis(5), )?; - } else if !payload_channel_disconnected { - payload_channel_disconnected = - drain_available_media_payloads(&payload_rx, &mut pending_payloads)?; + } + if !video_payload_channel_disconnected { + video_payload_channel_disconnected = drain_available_media_payloads( + &video_payload_rx, + &mut pending_payloads, + )?; + } + if pending_payloads.is_empty() + && !(video_payload_channel_disconnected && audio_payload_channel_disconnected) + { + thread::sleep(Duration::from_millis(1)); + continue; } if let Some(queued) = pending_payloads.pop_next() { @@ -2136,6 +2150,7 @@ fn run_rudp_mux_once( media_profile.receiver_assembly_deadline_ms, )?; poll_rudp_resends_with_backpressure(&mut video_transport)?; + drain_rudp_transport_receipts(&mut audio_transport)?; poll_rudp_resends_with_backpressure(&mut audio_transport)?; if payloads_dropped == 1 || payloads_dropped % 300 == 0 { let expired = reliable_packets_expired(&video_transport, &audio_transport); @@ -2201,12 +2216,13 @@ fn run_rudp_mux_once( media_profile.receiver_assembly_deadline_ms, )?; poll_rudp_resends_with_backpressure(&mut video_transport)?; + drain_rudp_transport_receipts(&mut audio_transport)?; poll_rudp_resends_with_backpressure(&mut audio_transport)?; payloads_sent += 1; if payloads_sent == 1 || payloads_sent % 900 == 0 { let expired = reliable_packets_expired(&video_transport, &audio_transport); eprintln!( - "{}; pending_audio={} pending_video={}; latest {:?} payload was {payload_len} bytes.", + "{}; video_rx_bytes={} video_reliable_pending={} audio_rx_bytes={} audio_reliable_pending={} pending_audio={} pending_video={} audio_capacity_dropped={} video_capacity_dropped={}; latest {:?} payload was {payload_len} bytes.", rudp_media_progress_detail( payloads_sent, payloads_dropped, @@ -2215,15 +2231,21 @@ fn run_rudp_mux_once( expired, &receiver_feedback ), + video_transport.stats().bytes_received, + video_transport.pending_reliable_packets(), + audio_transport.stats().bytes_received, + audio_transport.pending_reliable_packets(), pending_payloads.audio_len(), pending_payloads.video_len(), + pending_payloads.audio_capacity_dropped(), + pending_payloads.video_capacity_dropped(), queued.kind ); } continue; } - if payload_channel_disconnected { + if video_payload_channel_disconnected && audio_payload_channel_disconnected { let expired = reliable_packets_expired(&video_transport, &audio_transport); let payloads_dropped = payloads_queue_expired + payloads_send_expired; break Ok(RudpMuxRestart { @@ -2262,6 +2284,7 @@ fn run_rudp_mux_once( media_profile.receiver_assembly_deadline_ms, )?; poll_rudp_resends_with_backpressure(&mut video_transport)?; + drain_rudp_transport_receipts(&mut audio_transport)?; poll_rudp_resends_with_backpressure(&mut audio_transport)?; } } @@ -2309,6 +2332,8 @@ struct PendingMuninnMediaSendQueues { audio: VecDeque>, active_video: VecDeque, video: VecDeque>, + audio_capacity_dropped: u64, + video_capacity_dropped: u64, } impl PendingMuninnMediaSendQueues { @@ -2328,7 +2353,11 @@ impl PendingMuninnMediaSendQueues { > MUNINN_RUDP_MEDIA_PENDING_AUDIO_CAPACITY && !self.audio.is_empty() { - self.audio.pop_front(); + if let Some(dropped) = self.audio.pop_front() { + self.audio_capacity_dropped = self + .audio_capacity_dropped + .saturating_add(dropped.len() as u64); + } } self.audio.push_back(group); } @@ -2337,7 +2366,11 @@ impl PendingMuninnMediaSendQueues { > MUNINN_RUDP_MEDIA_PENDING_VIDEO_CAPACITY && !self.video.is_empty() { - self.video.pop_front(); + if let Some(dropped) = self.video.pop_front() { + self.video_capacity_dropped = self + .video_capacity_dropped + .saturating_add(dropped.len() as u64); + } } self.video.push_back(group); } @@ -2375,6 +2408,14 @@ impl PendingMuninnMediaSendQueues { fn video_len(&self) -> usize { self.active_video.len() + self.video.iter().map(VecDeque::len).sum::() } + + fn audio_capacity_dropped(&self) -> u64 { + self.audio_capacity_dropped + } + + fn video_capacity_dropped(&self) -> u64 { + self.video_capacity_dropped + } } #[derive(Clone, Debug, Default, PartialEq, Eq)] @@ -2405,12 +2446,16 @@ struct MuninnVideoBitrateController { } impl MuninnVideoBitrateController { - fn new(max_kbps: u32, now: Instant) -> Self { - let max_kbps = max_kbps.max(1_000); - let current_kbps = (max_kbps.saturating_mul(2) / 3).max(1_000); + fn new(configured_kbps: u32, now: Instant) -> Self { + let configured_kbps = configured_kbps.max(1_000); + // Block FEC can roughly double the encoder payload on the wire. Keep + // additive probing below the configured payload ceiling so clean video + // does not have to lose a frame to discover the protected wire limit. + let max_kbps = (configured_kbps / 2).max(1_000); + let current_kbps = (configured_kbps / 2).max(1_000).min(max_kbps); Self { current_kbps, - min_kbps: (max_kbps / 4).max(1_000), + min_kbps: (configured_kbps / 4).max(1_000).min(max_kbps), max_kbps, last_late_frames: 0, last_keyframes: 0, @@ -2689,22 +2734,6 @@ fn queue_muninn_video_payloads_by_access_unit( queue_muninn_media_payloads(tx, current, QueuedMuninnMediaKind::Video) } -fn receive_pending_media_payloads( - rx: &mpsc::Receiver>>, - pending: &mut PendingMuninnMediaSendQueues, - timeout: Duration, -) -> Result { - match rx.recv_timeout(timeout) { - Ok(Ok(payloads)) => { - pending.push_group(payloads); - drain_available_media_payloads(rx, pending) - } - Ok(Err(error)) => Err(error), - Err(mpsc::RecvTimeoutError::Timeout) => Ok(false), - Err(mpsc::RecvTimeoutError::Disconnected) => Ok(true), - } -} - fn drain_available_media_payloads( rx: &mpsc::Receiver>>, pending: &mut PendingMuninnMediaSendQueues, @@ -3017,6 +3046,7 @@ fn open_media_rudp_transport( connection_id, reliable_expire_after_ms, ))?; + transport.set_reliable_expire_after_ms(reliable_expire_after_ms); transport.connect(options.stream_id.as_bytes().to_vec())?; let deadline = Instant::now() + Duration::from_secs(5); while !transport.connected() { @@ -3132,12 +3162,26 @@ fn muninn_media_rudp_options( options } +fn drain_rudp_transport_receipts( + transport: &mut CultNetRudpSocketTransportConnection, +) -> Result<()> { + loop { + match transport.receive_once() { + Ok(Some(_)) => continue, + Ok(None) => return Ok(()), + Err(error) if is_would_block_error(&error) => return Ok(()), + Err(error) => return Err(error).context("draining Muninn RUDP transport receipts"), + } + } +} + fn reliable_packets_expired( video_transport: &CultNetRudpSocketTransportConnection, audio_transport: &CultNetRudpSocketTransportConnection, ) -> u64 { - let _ = (video_transport, audio_transport); - 0 + video_transport + .reliable_packets_expired() + .saturating_add(audio_transport.reliable_packets_expired()) } fn video_rudp_payload_reader( @@ -11998,12 +12042,12 @@ mod tests { .unwrap(); assert_eq!( media_channel.delivery, - cultnet_rs::CultNetTransportDelivery::Unreliable + cultnet_rs::CultNetTransportDelivery::Reliable ); } #[test] - fn rudp_audio_transport_uses_separate_realtime_connection() { + fn rudp_audio_transport_uses_separate_media_connection() { let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); let endpoint: SocketAddr = "127.0.0.1:5204".parse().unwrap(); let profile = muninn_rudp_media_profile(); @@ -12100,6 +12144,48 @@ mod tests { assert!(pending.is_empty()); } + #[test] + fn reliable_media_packets_expire_at_the_consumer_deadline() { + use cultnet_rs::{ + CultNetRudpSendOptions, CultNetRudpSession, CultNetRudpSessionOptions, + }; + + let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 77, + initial_sequence: 1, + resend_delay_ms: 5, + max_pending_reliable_packets: Some(16), + }); + let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 77, + initial_sequence: 1, + resend_delay_ms: 5, + max_pending_reliable_packets: Some(16), + }); + sender.set_reliable_expire_after_ms(Some(250)); + let connect = sender.create_connect(0, Vec::new()).unwrap(); + let accept = receiver.accept_connect(&connect, 0, Vec::new()).unwrap(); + sender.receive(&accept, 0).unwrap(); + let packet = sender + .send( + crate::media_packetizer::MUNINN_MEDIA_RUDP_CHANNEL, + vec![1, 2, 3], + CultNetRudpSendOptions { + reliable: true, + ordered: false, + sequenced: false, + now_ms: 10, + }, + ) + .unwrap(); + assert!(packet.reliable); + assert!(sender.due_resends(14).is_empty()); + assert_eq!(sender.due_resends(15).len(), 1); + assert!(sender.due_resends(261).is_empty()); + assert!(sender.pending_reliable_sequences().is_empty()); + assert_eq!(sender.reliable_packets_expired(), 1); + } + #[test] fn rudp_media_pending_queues_are_structurally_bounded() { let queued_at = Instant::now(); @@ -12393,7 +12479,7 @@ mod tests { feedback.late_frames = 1; assert_eq!( controller.observe(&feedback, 0, 100_000, start + Duration::from_millis(500)), - Some(6_800) + Some(5_100) ); assert_eq!( controller.observe(&feedback, 0, 100_000, start + Duration::from_secs(1)), @@ -12405,7 +12491,7 @@ mod tests { ); assert_eq!( controller.observe(&feedback, 0, 100_000, start + Duration::from_millis(10_500)), - Some(7_040) + Some(5_220) ); } @@ -12421,10 +12507,10 @@ mod tests { let adjusted = controller .observe(&feedback, 0, 100_000, start + Duration::from_millis(500)) .unwrap(); - assert_eq!(adjusted, 9_066); + assert_eq!(adjusted, 6_800); let mut commands = Vec::new(); request_video_encoder_bitrate(Some(&mut commands), adjusted).unwrap(); - assert_eq!(commands, b"BITRATE 9066\n"); + assert_eq!(commands, b"BITRATE 6800\n"); } #[test] diff --git a/crates/muninn-daemon/src/media_packetizer.rs b/crates/muninn-daemon/src/media_packetizer.rs index 2ffbbe4..1d58bee 100644 --- a/crates/muninn-daemon/src/media_packetizer.rs +++ b/crates/muninn-daemon/src/media_packetizer.rs @@ -16,7 +16,8 @@ use std::collections::BTreeMap; // Deadline-bound audio/video must use the experimental CultNet realtime lane. // The `media` lane is reliable without expiry and can accumulate corpses behind // loss; application-level parity/repair and playout deadlines own recovery. -pub const MUNINN_MEDIA_RUDP_CHANNEL: &str = "realtime"; +pub const MUNINN_MEDIA_RUDP_CHANNEL: &str = "media"; +pub const MUNINN_MEDIA_REALTIME_RUDP_CHANNEL: &str = "realtime"; #[derive(Clone, Debug, PartialEq, Eq)] pub struct VideoAccessUnit { @@ -309,6 +310,10 @@ struct VideoParityShardWirePayload<'a>( u16, u16, u16, + u16, + u16, + u16, + u16, u32, u32, #[serde(with = "serde_bytes")] &'a [u8], @@ -1132,7 +1137,7 @@ pub fn packetize_video_access_unit( Ok(records) } -const MUNINN_VIDEO_PARITY_SHARDS: u16 = 16; +const MUNINN_VIDEO_FEC_BLOCK_DATA_SHARDS: u16 = 8; fn video_fec_coefficient(parity_index: u16, parity_count: u16, data_index: u16) -> u8 { debug_assert!(u32::from(parity_count) + u32::from(data_index) <= 255); @@ -1183,53 +1188,71 @@ pub fn build_video_parity_shards( } } - let mut chunk_payload_len = 0_usize; - let mut last_chunk_payload_bytes = 0_u32; for index in 0..first.chunk_count { - let chunk = by_index - .get(&index) - .ok_or_else(|| anyhow!("video parity missing chunk_index {index}"))?; - chunk_payload_len = chunk_payload_len.max(chunk.payload.len()); - if index == first.chunk_count - 1 { - last_chunk_payload_bytes = u32::try_from(chunk.payload.len()) - .context("video parity last chunk payload length exceeds u32")?; + if !by_index.contains_key(&index) { + return Err(anyhow!("video parity missing chunk_index {index}")); } } - let chunk_payload_bytes = u32::try_from(chunk_payload_len) - .context("video parity chunk payload length exceeds u32")?; - let parity_count = first - .chunk_count - .min(MUNINN_VIDEO_PARITY_SHARDS) - .min(256_u16.saturating_sub(first.chunk_count)); - let mut records = Vec::with_capacity(parity_count as usize); - for parity_index in 0..parity_count { - let mut parity = vec![0_u8; chunk_payload_len]; - for index in 0..first.chunk_count { - let chunk = by_index.get(&index).expect("chunk index was checked above"); - let coefficient = video_fec_coefficient(parity_index, parity_count, index); - for (offset, byte) in chunk.payload.iter().enumerate() { - parity[offset] ^= gf256_mul(*byte, coefficient); + + let block_count = first.chunk_count.div_ceil(MUNINN_VIDEO_FEC_BLOCK_DATA_SHARDS); + let mut records = Vec::with_capacity(first.chunk_count as usize); + for block_index in 0..block_count { + let block_data_start = block_index * MUNINN_VIDEO_FEC_BLOCK_DATA_SHARDS; + let block_data_count = (first.chunk_count - block_data_start) + .min(MUNINN_VIDEO_FEC_BLOCK_DATA_SHARDS); + let parity_count = MUNINN_VIDEO_FEC_BLOCK_DATA_SHARDS; + let block_end = block_data_start + block_data_count; + let chunk_payload_len = (block_data_start..block_end) + .map(|index| by_index.get(&index).expect("chunk index checked").payload.len()) + .max() + .expect("block contains data"); + let last_chunk_payload_bytes = u32::try_from( + by_index + .get(&(block_end - 1)) + .expect("block last chunk checked") + .payload + .len(), + ) + .context("video parity block last chunk payload length exceeds u32")?; + let chunk_payload_bytes = u32::try_from(chunk_payload_len) + .context("video parity chunk payload length exceeds u32")?; + + for parity_index in 0..parity_count { + let mut parity = vec![0_u8; chunk_payload_len]; + for local_index in 0..block_data_count { + let chunk = by_index + .get(&(block_data_start + local_index)) + .expect("block chunk checked"); + let coefficient = + video_fec_coefficient(parity_index, parity_count, local_index); + for (offset, byte) in chunk.payload.iter().enumerate() { + parity[offset] ^= gf256_mul(*byte, coefficient); + } } + records.push(MuninnMediaVideoParityShardRecord { + stream_id: first.stream_id.clone(), + session_id: first.session_id.clone(), + frame_id: first.frame_id, + codec: first.codec.clone(), + pts_ticks: first.pts_ticks, + duration_ticks: first.duration_ticks, + timebase_num: first.timebase_num, + timebase_den: first.timebase_den, + keyframe: first.keyframe, + dependency_frame_id: first.dependency_frame_id, + deadline_ticks: first.deadline_ticks, + chunk_count: first.chunk_count, + block_index, + block_count, + block_data_start, + block_data_count, + parity_index, + parity_count, + chunk_payload_bytes, + last_chunk_payload_bytes, + payload: parity, + }); } - records.push(MuninnMediaVideoParityShardRecord { - stream_id: first.stream_id.clone(), - session_id: first.session_id.clone(), - frame_id: first.frame_id, - codec: first.codec.clone(), - pts_ticks: first.pts_ticks, - duration_ticks: first.duration_ticks, - timebase_num: first.timebase_num, - timebase_den: first.timebase_den, - keyframe: first.keyframe, - dependency_frame_id: first.dependency_frame_id, - deadline_ticks: first.deadline_ticks, - chunk_count: first.chunk_count, - parity_index, - parity_count, - chunk_payload_bytes, - last_chunk_payload_bytes, - payload: parity, - }); } Ok(records) } @@ -1251,25 +1274,48 @@ fn video_wire_records_with_parity( } let frame_records = &records[offset..end]; let parity = build_video_parity_shards(frame_records)?; - let parity_split = parity.len().div_ceil(2); - wire_records.extend( - parity[..parity_split] + if parity.is_empty() { + wire_records.extend( + frame_records + .iter() + .cloned() + .map(MuninnMediaWireRecord::Video), + ); + offset = end; + continue; + } + let block_count = parity + .first() + .map_or(0_usize, |record| record.block_count as usize); + let mut block_lanes = vec![Vec::new(); block_count]; + for block_index in 0..block_count { + let block_parity = parity .iter() - .cloned() - .map(MuninnMediaWireRecord::VideoParity), - ); - wire_records.extend( - frame_records - .iter() - .cloned() - .map(MuninnMediaWireRecord::Video), - ); - wire_records.extend( - parity[parity_split..] - .iter() - .cloned() - .map(MuninnMediaWireRecord::VideoParity), - ); + .filter(|record| record.block_index as usize == block_index) + .collect::>(); + let Some(first_parity) = block_parity.first() else { + return Err(anyhow!("video FEC block {block_index} has no parity")); + }; + for local_index in 0..first_parity.parity_count as usize { + if local_index < first_parity.block_data_count as usize { + let global_index = first_parity.block_data_start as usize + local_index; + block_lanes[block_index].push(MuninnMediaWireRecord::Video( + frame_records[global_index].clone(), + )); + } + block_lanes[block_index].push(MuninnMediaWireRecord::VideoParity( + block_parity[local_index].clone(), + )); + } + } + let lane_len = block_lanes.iter().map(Vec::len).max().unwrap_or(0); + for lane_position in 0..lane_len { + for lane in &block_lanes { + if let Some(record) = lane.get(lane_position) { + wire_records.push(record.clone()); + } + } + } offset = end; } Ok(wire_records) @@ -1977,8 +2023,19 @@ fn wire_payloads_to_media_send(payloads: Vec>) -> Vec) -> MuninnMediaSendPayload { + let channel_id = match decode_media_wire_record(&payload) { + Ok( + MuninnMediaWireRecord::Video(_) + | MuninnMediaWireRecord::VideoParity(_) + | MuninnMediaWireRecord::Audio(_) + | MuninnMediaWireRecord::AudioParity(_), + ) => { + MUNINN_MEDIA_REALTIME_RUDP_CHANNEL + } + _ => MUNINN_MEDIA_RUDP_CHANNEL, + }; MuninnMediaSendPayload { - channel_id: MUNINN_MEDIA_RUDP_CHANNEL, + channel_id, payload, } } @@ -2105,6 +2162,10 @@ fn encode_video_parity_record_payload( record.dependency_frame_id, record.deadline_ticks, record.chunk_count, + record.block_index, + record.block_count, + record.block_data_start, + record.block_data_count, record.parity_index, record.parity_count, record.chunk_payload_bytes, @@ -2248,14 +2309,28 @@ fn validate_video_parity_record(record: &MuninnMediaVideoParityShardRecord) -> R "video parity media record chunk_count must be non-zero" )); } - if record.parity_count == 0 || record.parity_index >= record.parity_count { - return Err(anyhow!( - "video parity media record has invalid parity stripe metadata" - )); + let expected_block_count = record + .chunk_count + .div_ceil(MUNINN_VIDEO_FEC_BLOCK_DATA_SHARDS); + if record.block_count != expected_block_count + || record.block_index >= record.block_count + || record.block_data_count == 0 + || record.block_data_count > MUNINN_VIDEO_FEC_BLOCK_DATA_SHARDS + || record.block_data_start + != record.block_index * MUNINN_VIDEO_FEC_BLOCK_DATA_SHARDS + || record + .block_data_start + .checked_add(record.block_data_count) + .is_none_or(|end| end > record.chunk_count) + { + return Err(anyhow!("video parity media record has invalid FEC block metadata")); } - if record.parity_count > record.chunk_count { + if record.parity_count < record.block_data_count + || record.parity_count > MUNINN_VIDEO_FEC_BLOCK_DATA_SHARDS + || record.parity_index >= record.parity_count + { return Err(anyhow!( - "video parity media record parity_count exceeds chunk_count" + "video parity media record has invalid block parity metadata" )); } if record.chunk_payload_bytes == 0 || record.last_chunk_payload_bytes == 0 { @@ -2378,8 +2453,12 @@ fn video_record_key(record: &MuninnMediaVideoAccessUnitRecord) -> String { fn video_parity_record_key(record: &MuninnMediaVideoParityShardRecord) -> String { format!( - "{}:{}:video-parity:{}:{}", - record.stream_id, record.session_id, record.frame_id, record.parity_index + "{}:{}:video-parity:{}:{}:{}", + record.stream_id, + record.session_id, + record.frame_id, + record.block_index, + record.parity_index ) } @@ -2860,22 +2939,24 @@ mod tests { let parity = build_video_parity_shards(&records)?; assert_eq!(records.len(), 20); - assert_eq!(parity.len(), 16); - assert!( - parity.iter().enumerate().all( - |(index, shard)| shard.parity_index == index as u16 && shard.parity_count == 16 - ) - ); + assert_eq!(parity.len(), 24); + assert_eq!(parity.iter().map(|shard| shard.block_index).collect::>(), + [vec![0; 8], vec![1; 8], vec![2; 8]].concat()); + assert!(parity.iter().all(|shard| shard.parity_count == 8)); assert!(parity.iter().all(|shard| shard.chunk_payload_bytes == 2)); let missing_index = 18_u16; - let shard = &parity[7]; + let shard = &parity[18]; let mut recovered = shard.payload.clone(); - for chunk in records.iter().filter(|chunk| chunk.chunk_index != missing_index) { + for chunk in records.iter().filter(|chunk| { + chunk.chunk_index >= shard.block_data_start + && chunk.chunk_index < shard.block_data_start + shard.block_data_count + && chunk.chunk_index != missing_index + }) { let coefficient = video_fec_coefficient( shard.parity_index, shard.parity_count, - chunk.chunk_index, + chunk.chunk_index - shard.block_data_start, ); for (offset, byte) in chunk.payload.iter().enumerate() { recovered[offset] ^= gf256_mul(*byte, coefficient); @@ -2884,7 +2965,7 @@ mod tests { let inverse = gf256_inv(video_fec_coefficient( shard.parity_index, shard.parity_count, - missing_index, + missing_index - shard.block_data_start, )); for byte in &mut recovered { *byte = gf256_mul(*byte, inverse); @@ -2895,7 +2976,7 @@ mod tests { } #[test] - fn video_wire_bookends_data_with_independent_parity() -> Result<()> { + fn video_wire_round_robins_independent_fec_blocks() -> Result<()> { let access_unit = VideoAccessUnit { bytes: (1_u8..=40).collect(), keyframe: true, @@ -2917,10 +2998,124 @@ mod tests { )?; let wire = video_wire_records_with_parity(&records)?; - assert_eq!(wire.len(), 36); - assert!(wire[..8].iter().all(|record| matches!(record, MuninnMediaWireRecord::VideoParity(_)))); - assert!(wire[8..28].iter().all(|record| matches!(record, MuninnMediaWireRecord::Video(_)))); - assert!(wire[28..].iter().all(|record| matches!(record, MuninnMediaWireRecord::VideoParity(_)))); + assert_eq!(wire.len(), 44); + assert_eq!( + wire[..3] + .iter() + .map(|record| match record { + MuninnMediaWireRecord::Video(record) => record.chunk_index, + _ => u16::MAX, + }) + .collect::>(), + vec![0, 8, 16] + ); + assert_eq!( + wire[3..6] + .iter() + .map(|record| match record { + MuninnMediaWireRecord::VideoParity(record) => record.block_index, + _ => u16::MAX, + }) + .collect::>(), + vec![0, 1, 2] + ); + Ok(()) + } + + #[test] + fn video_block_layout_survives_every_observed_contiguous_burst_alignment() -> Result<()> { + let access_unit = VideoAccessUnit { + bytes: (0..80).map(|index| (index * 17) as u8).collect(), + keyframe: true, + }; + let records = packetize_video_access_unit( + VideoFramePacketizeOptions { + stream_id: "muninn.raven.av.rudp", + session_id: "session-1", + codec: "h264", + frame_id: 10, + pts_ticks: 30_000, + duration_ticks: 3_000, + timebase_num: 1, + timebase_den: 90_000, + deadline_ticks: 52_500, + max_payload_bytes: 1, + }, + &access_unit, + )?; + let wire = video_wire_records_with_parity(&records)?; + assert_eq!(wire.len(), 160); + + for burst_len in 1..=54_usize { + for burst_start in 0..=wire.len() - burst_len { + let mut missing_data = [0_u16; 10]; + let mut surviving_parity = [8_u16; 10]; + for record in &wire[burst_start..burst_start + burst_len] { + match record { + MuninnMediaWireRecord::Video(record) => { + missing_data[record.chunk_index as usize / 8] += 1; + } + MuninnMediaWireRecord::VideoParity(record) => { + surviving_parity[record.block_index as usize] -= 1; + } + _ => unreachable!("video wire contains only video records"), + } + } + assert!(missing_data + .iter() + .zip(surviving_parity) + .all(|(missing, parity)| *missing <= parity)); + } + } + Ok(()) + } + + #[test] + fn video_tail_block_survives_eight_packet_burst_at_every_alignment() -> Result<()> { + for chunk_count in 2_usize..=80 { + let access_unit = VideoAccessUnit { + bytes: (0..chunk_count).map(|index| (index * 19) as u8).collect(), + keyframe: false, + }; + let records = packetize_video_access_unit( + VideoFramePacketizeOptions { + stream_id: "muninn.raven.av.rudp", + session_id: "session-1", + codec: "h264", + frame_id: 11, + pts_ticks: 33_000, + duration_ticks: 3_000, + timebase_num: 1, + timebase_den: 90_000, + deadline_ticks: 55_500, + max_payload_bytes: 1, + }, + &access_unit, + )?; + let wire = video_wire_records_with_parity(&records)?; + for burst_len in 1..=8_usize.min(wire.len()) { + for burst_start in 0..=wire.len() - burst_len { + let block_count = chunk_count.div_ceil(8); + let mut missing_data = vec![0_u16; block_count]; + let mut surviving_parity = vec![8_u16; block_count]; + for record in &wire[burst_start..burst_start + burst_len] { + match record { + MuninnMediaWireRecord::Video(record) => { + missing_data[record.chunk_index as usize / 8] += 1; + } + MuninnMediaWireRecord::VideoParity(record) => { + surviving_parity[record.block_index as usize] -= 1; + } + _ => unreachable!("video wire contains only video records"), + } + } + assert!(missing_data + .iter() + .zip(surviving_parity) + .all(|(missing, parity)| *missing <= parity)); + } + } + } Ok(()) } @@ -3950,7 +4145,7 @@ mod tests { } #[test] - fn media_send_payloads_pin_video_to_media_channel() -> Result<()> { + fn media_send_payloads_pin_video_to_realtime_channel() -> Result<()> { let mut stream = Vec::new(); stream.extend_from_slice(&start_code()); stream.extend_from_slice(&[0x65, 0x80]); @@ -3977,7 +4172,7 @@ mod tests { )?; assert_eq!(payloads.len(), 1); - assert_eq!(payloads[0].channel_id, MUNINN_MEDIA_RUDP_CHANNEL); + assert_eq!(payloads[0].channel_id, MUNINN_MEDIA_REALTIME_RUDP_CHANNEL); let MuninnMediaWireRecord::Video(record) = decode_media_wire_record(&payloads[0].payload)? else { panic!("expected video media record"); @@ -4055,7 +4250,10 @@ mod tests { let second_push = sender.push("2026-06-18T00:00:00Z", &second_frame)?; assert_eq!(second_push.len(), 1); - assert_eq!(second_push[0].channel_id, MUNINN_MEDIA_RUDP_CHANNEL); + assert_eq!( + second_push[0].channel_id, + MUNINN_MEDIA_REALTIME_RUDP_CHANNEL + ); let MuninnMediaWireRecord::Video(first) = decode_media_wire_record(&second_push[0].payload)? else { @@ -4134,8 +4332,8 @@ mod tests { let second_push = sender.push("2026-06-18T00:00:00Z", &tail)?; assert_eq!(second_push.len(), 2); - assert_eq!(second_push[0].channel_id, MUNINN_MEDIA_RUDP_CHANNEL); - assert_eq!(second_push[1].channel_id, MUNINN_MEDIA_RUDP_CHANNEL); + assert_eq!(second_push[0].channel_id, MUNINN_MEDIA_REALTIME_RUDP_CHANNEL); + assert_eq!(second_push[1].channel_id, MUNINN_MEDIA_REALTIME_RUDP_CHANNEL); let MuninnMediaWireRecord::Audio(first) = decode_media_wire_record(&second_push[0].payload)? @@ -4241,7 +4439,7 @@ mod tests { } #[test] - fn media_send_payload_pins_audio_to_media_channel() -> Result<()> { + fn media_send_payload_pins_audio_to_realtime_channel() -> Result<()> { let payload = audio_packet_send_payload( AudioPacketWireOptions { packetize: AudioPacketizeOptions { @@ -4262,7 +4460,7 @@ mod tests { &[0xf8, 0xff, 0xfe], )?; - assert_eq!(payload.channel_id, MUNINN_MEDIA_RUDP_CHANNEL); + assert_eq!(payload.channel_id, MUNINN_MEDIA_REALTIME_RUDP_CHANNEL); let MuninnMediaWireRecord::Audio(record) = decode_media_wire_record(&payload.payload)? else { panic!("expected audio media record"); diff --git a/crates/odin-core/src/documents.rs b/crates/odin-core/src/documents.rs index 7b0e2a7..bb39000 100644 --- a/crates/odin-core/src/documents.rs +++ b/crates/odin-core/src/documents.rs @@ -36,7 +36,7 @@ pub const MUNINN_TELEMETRY_SURFACE_SCHEMA: &str = "muninn.telemetry_surface.v1"; pub const MUNINN_CAPTURE_STREAM_SCHEMA: &str = "muninn.capture_stream.v1"; pub const MUNINN_CAPTURE_STREAM_COMMAND_SCHEMA: &str = "muninn.capture_stream_command.v1"; pub const MUNINN_MEDIA_VIDEO_ACCESS_UNIT_SCHEMA: &str = "muninn.media_video_access_unit.v1"; -pub const MUNINN_MEDIA_VIDEO_PARITY_SHARD_SCHEMA: &str = "muninn.media_video_parity_shard.v3"; +pub const MUNINN_MEDIA_VIDEO_PARITY_SHARD_SCHEMA: &str = "muninn.media_video_parity_shard.v4"; pub const MUNINN_MEDIA_AUDIO_PACKET_SCHEMA: &str = "muninn.media_audio_packet.v1"; pub const MUNINN_MEDIA_AUDIO_PARITY_SHARD_SCHEMA: &str = "muninn.media_audio_parity_shard.v1"; pub const MUNINN_MEDIA_RECEIVER_FEEDBACK_SCHEMA: &str = "muninn.media_receiver_feedback.v1"; @@ -829,7 +829,7 @@ pub struct MuninnMediaVideoAccessUnitRecord { #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache( type = "muninn.media_video_parity_shard", - schema = "muninn.media_video_parity_shard.v3" + schema = "muninn.media_video_parity_shard.v4" )] pub struct MuninnMediaVideoParityShardRecord { #[cultcache(key = 0)] @@ -857,14 +857,22 @@ pub struct MuninnMediaVideoParityShardRecord { #[cultcache(key = 11)] pub chunk_count: u16, #[cultcache(key = 12)] - pub parity_index: u16, + pub block_index: u16, #[cultcache(key = 13)] - pub parity_count: u16, + pub block_count: u16, #[cultcache(key = 14)] - pub chunk_payload_bytes: u32, + pub block_data_start: u16, #[cultcache(key = 15)] - pub last_chunk_payload_bytes: u32, + pub block_data_count: u16, #[cultcache(key = 16)] + pub parity_index: u16, + #[cultcache(key = 17)] + pub parity_count: u16, + #[cultcache(key = 18)] + pub chunk_payload_bytes: u32, + #[cultcache(key = 19)] + pub last_chunk_payload_bytes: u32, + #[cultcache(key = 20)] pub payload: Vec, } diff --git a/vendor/cultnet-rs/src/rudp.rs b/vendor/cultnet-rs/src/rudp.rs index 2692bb8..a0fbf9a 100644 --- a/vendor/cultnet-rs/src/rudp.rs +++ b/vendor/cultnet-rs/src/rudp.rs @@ -105,6 +105,7 @@ pub struct CultNetRudpSendOptions { #[derive(Clone, Debug, PartialEq, Eq)] struct PendingReliablePacket { packet: CultNetRudpPacket, + first_sent_at_ms: u64, last_sent_at_ms: u64, } @@ -127,6 +128,8 @@ pub struct CultNetRudpSession { connection_id: u32, resend_delay_ms: u64, max_pending_reliable_packets: Option, + reliable_expire_after_ms: Option, + reliable_packets_expired: u64, initial_sequence: u32, next_sequence: u32, next_fragment_id: u16, @@ -146,6 +149,8 @@ impl CultNetRudpSession { connection_id: options.connection_id, resend_delay_ms: options.resend_delay_ms, max_pending_reliable_packets: options.max_pending_reliable_packets, + reliable_expire_after_ms: None, + reliable_packets_expired: 0, initial_sequence: options.initial_sequence, next_sequence: options.initial_sequence, next_fragment_id: 1, @@ -176,6 +181,14 @@ impl CultNetRudpSession { self.pending_reliable.keys().copied().collect() } + pub fn reliable_packets_expired(&self) -> u64 { + self.reliable_packets_expired + } + + pub fn set_reliable_expire_after_ms(&mut self, expire_after_ms: Option) { + self.reliable_expire_after_ms = expire_after_ms; + } + pub fn last_received_at_ms(&self) -> Option { self.last_received_at_ms } @@ -495,6 +508,15 @@ impl CultNetRudpSession { } pub fn due_resends(&mut self, now_ms: u64) -> Vec { + if let Some(expire_after_ms) = self.reliable_expire_after_ms { + let before = self.pending_reliable.len(); + self.pending_reliable.retain(|_, pending| { + now_ms.saturating_sub(pending.first_sent_at_ms) <= expire_after_ms + }); + self.reliable_packets_expired = self + .reliable_packets_expired + .saturating_add((before - self.pending_reliable.len()) as u64); + } let mut due = Vec::new(); for pending in self.pending_reliable.values_mut() { if now_ms.saturating_sub(pending.last_sent_at_ms) >= self.resend_delay_ms { @@ -569,6 +591,7 @@ impl CultNetRudpSession { packet.sequence, PendingReliablePacket { packet, + first_sent_at_ms: now_ms, last_sent_at_ms: now_ms, }, ); @@ -580,7 +603,11 @@ impl CultNetRudpSession { } if let Some(limit) = self.max_pending_reliable_packets { if self.pending_reliable.len() + packet_count > limit { - return Err(anyhow!("RUDP reliable send queue is full")); + return Err(std::io::Error::new( + ErrorKind::WouldBlock, + "RUDP reliable send queue is full", + ) + .into()); } } Ok(()) @@ -795,7 +822,7 @@ impl CultNetRudpSession { fn allocate_fragment_id(&mut self) -> u16 { let fragment_id = self.next_fragment_id; - self.next_fragment_id = self.next_fragment_id.saturating_add(1); + self.next_fragment_id = self.next_fragment_id.wrapping_add(1); if self.next_fragment_id == 0 { self.next_fragment_id = 1; } @@ -887,6 +914,8 @@ pub struct CultNetRudpSocketTransportConnection { max_fragment_bytes: Option, disconnect_reason: Option>, pong_payloads: VecDeque>, + pending_outbound_packets: VecDeque, + max_pending_outbound_packets: usize, } impl CultNetRudpSocketTransportConnection { @@ -907,14 +936,16 @@ impl CultNetRudpSocketTransportConnection { let max_pending_reliable_packets = options .max_pending_reliable_packets .map(|value| value as usize); + let max_pending_outbound_packets = max_pending_reliable_packets.unwrap_or(1_024).max(1); + let session = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: options.connection_id, + initial_sequence: options.initial_sequence, + resend_delay_ms: options.resend_delay_ms, + max_pending_reliable_packets, + }); Ok(Self { socket: options.socket, - session: CultNetRudpSession::new(CultNetRudpSessionOptions { - connection_id: options.connection_id, - initial_sequence: options.initial_sequence, - resend_delay_ms: options.resend_delay_ms, - max_pending_reliable_packets, - }), + session, mode: options.mode, remote_addr: options.remote_addr, profile, @@ -923,6 +954,8 @@ impl CultNetRudpSocketTransportConnection { max_fragment_bytes: options.max_fragment_bytes.map(|value| value as usize), disconnect_reason: None, pong_payloads: VecDeque::new(), + pending_outbound_packets: VecDeque::new(), + max_pending_outbound_packets, }) } @@ -934,6 +967,18 @@ impl CultNetRudpSocketTransportConnection { self.stats.clone() } + pub fn reliable_packets_expired(&self) -> u64 { + self.session.reliable_packets_expired() + } + + pub fn pending_reliable_packets(&self) -> usize { + self.session.pending_reliable_sequences().len() + } + + pub fn set_reliable_expire_after_ms(&mut self, expire_after_ms: Option) { + self.session.set_reliable_expire_after_ms(expire_after_ms); + } + pub fn disconnect_reason(&self) -> Option<&[u8]> { self.disconnect_reason.as_deref() } @@ -953,13 +998,23 @@ impl CultNetRudpSocketTransportConnection { } pub fn send(&mut self, channel_id: &str, payload: Vec) -> Result<()> { + self.flush_pending_outbound_packets()?; + let packet_count = self.outbound_packet_count(payload.len())?; + if self.pending_outbound_packets.len().saturating_add(packet_count) + > self.max_pending_outbound_packets + { + return Err(std::io::Error::new( + ErrorKind::WouldBlock, + "CultNet RUDP outbound packet queue is full", + ) + .into()); + } let options = channel_send_options(channel_id, now_ms()); let packets = self.session .send_many(channel_id, payload, options, self.max_fragment_bytes)?; - for packet in packets { - self.send_packet(&packet)?; - } + self.pending_outbound_packets.extend(packets); + self.flush_pending_outbound_packets()?; self.stats.frames_sent += 1; Ok(()) } @@ -1079,9 +1134,38 @@ impl CultNetRudpSocketTransportConnection { } pub fn poll_resends(&mut self) -> Result<()> { - for packet in self.session.due_resends(now_ms()) { - self.send_packet(&packet)?; + self.flush_pending_outbound_packets()?; + if !self.pending_outbound_packets.is_empty() { + return Ok(()); } + for packet in self.session.due_resends(now_ms()) { + if self.pending_outbound_packets.len() >= self.max_pending_outbound_packets { + break; + } + self.pending_outbound_packets.push_back(packet); + } + self.flush_pending_outbound_packets()?; + Ok(()) + } + + fn outbound_packet_count(&self, payload_bytes: usize) -> Result { + match self.max_fragment_bytes { + Some(0) => Err(anyhow!("RUDP max_fragment_bytes must be greater than zero")), + Some(max_fragment_bytes) if payload_bytes > max_fragment_bytes => { + Ok(payload_bytes.div_ceil(max_fragment_bytes)) + } + _ => Ok(1), + } + } + + fn flush_pending_outbound_packets(&mut self) -> Result<()> { + let Some(remote_addr) = self.remote_addr else { + return Err(anyhow!("RUDP socket transport does not have a remote endpoint")); + }; + let sent_bytes = flush_rudp_outbound_queue(&mut self.pending_outbound_packets, |wire| { + self.socket.send_to(wire, remote_addr) + })?; + self.stats.bytes_sent += sent_bytes as u64; Ok(()) } @@ -1098,6 +1182,33 @@ impl CultNetRudpSocketTransportConnection { } } +fn flush_rudp_outbound_queue( + pending: &mut VecDeque, + mut send: F, +) -> Result +where + F: FnMut(&[u8]) -> std::io::Result, +{ + let mut sent_bytes = 0_usize; + while let Some(packet) = pending.front() { + let wire = encode_rudp_packet(packet)?; + match send(&wire) { + Ok(sent) => { + sent_bytes = sent_bytes.saturating_add(sent); + pending.pop_front(); + } + Err(error) + if error.kind() == ErrorKind::WouldBlock + || error.kind() == ErrorKind::TimedOut => + { + break; + } + Err(error) => return Err(error.into()), + } + } + Ok(sent_bytes) +} + pub struct CultNetRudpReconnectLoop where F: FnMut() -> Result, @@ -1409,3 +1520,116 @@ fn packet_type_from_code(code: u8) -> Result { _ => Err(anyhow!("Unsupported CultNet RUDP packet type {code}")), } } + +#[cfg(test)] +mod socket_transport_tests { + use super::*; + + fn data_packet(sequence: u32) -> CultNetRudpPacket { + CultNetRudpPacket { + packet_type: CultNetRudpPacketType::Data, + connection_id: 7, + sequence, + ack: 0, + ack_mask: 0, + channel_id: "realtime".to_string(), + reliable: false, + ordered: false, + sequenced: false, + fragment_id: 0, + fragment_index: 0, + fragment_count: 0, + payload: vec![sequence as u8], + } + } + + #[test] + fn fragment_ids_wrap_to_one_instead_of_sticking_at_u16_max() { + let mut session = CultNetRudpSession::new(CultNetRudpSessionOptions::default()); + session.next_fragment_id = u16::MAX; + + assert_eq!(session.allocate_fragment_id(), u16::MAX); + assert_eq!(session.allocate_fragment_id(), 1); + assert_eq!(session.allocate_fragment_id(), 2); + } + + #[test] + fn outbound_queue_preserves_allocated_packets_across_would_block() -> Result<()> { + let mut pending = VecDeque::from([data_packet(41), data_packet(42)]); + let mut attempts = 0_usize; + let sent = flush_rudp_outbound_queue(&mut pending, |_| { + attempts += 1; + Err(std::io::Error::from(ErrorKind::WouldBlock)) + })?; + assert_eq!(sent, 0); + assert_eq!(attempts, 1); + assert_eq!(pending.iter().map(|packet| packet.sequence).collect::>(), vec![41, 42]); + + let mut sent_sequences = Vec::new(); + let sent = flush_rudp_outbound_queue(&mut pending, |wire| { + sent_sequences.push(decode_rudp_packet(wire).unwrap().sequence); + Ok(wire.len()) + })?; + assert!(sent > 0); + assert_eq!(sent_sequences, vec![41, 42]); + assert!(pending.is_empty()); + Ok(()) + } + + #[test] + fn exact_ack_releases_one_reliable_packet() -> Result<()> { + let mut session = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 7, + ..CultNetRudpSessionOptions::default() + }); + let connect = session.create_connect(1, Vec::new())?; + session.receive( + &CultNetRudpPacket { + packet_type: CultNetRudpPacketType::Accept, + connection_id: 7, + sequence: 1, + ack: connect.sequence, + ack_mask: 0, + channel_id: "control".to_string(), + reliable: true, + ordered: true, + sequenced: false, + fragment_id: 0, + fragment_index: 0, + fragment_count: 0, + payload: Vec::new(), + }, + 2, + )?; + let packet = session.send( + "media", + vec![1, 2, 3], + CultNetRudpSendOptions { + reliable: true, + now_ms: 10, + ..CultNetRudpSendOptions::default() + }, + )?; + assert_eq!(session.pending_reliable_sequences(), vec![packet.sequence]); + session.receive( + &CultNetRudpPacket { + packet_type: CultNetRudpPacketType::Ack, + connection_id: 7, + sequence: 99, + ack: packet.sequence, + ack_mask: 0, + channel_id: "control".to_string(), + reliable: false, + ordered: false, + sequenced: false, + fragment_id: 0, + fragment_index: 0, + fragment_count: 0, + payload: Vec::new(), + }, + 11, + )?; + assert!(session.pending_reliable_sequences().is_empty()); + Ok(()) + } +}