Make Muninn media deadline-owned and wrap-safe

This commit is contained in:
Meta Vens 2026-07-17 05:48:17 +01:00
commit fb01f6dcc7
4 changed files with 673 additions and 157 deletions

View file

@ -73,7 +73,7 @@ const MUNINN_RUDP_MEDIA_VIDEO_BITRATE_KBPS: u32 = 12_000;
const MUNINN_RUDP_MEDIA_VBV_FRAME_BUDGETS: u32 = 1;
const MUNINN_RUDP_MEDIA_LOW_DELAY_KEY_FRAME_SCALE: u32 = 4;
const MUNINN_RUDP_MEDIA_VIDEO_DPB_SIZE: u32 = 1;
const MUNINN_RUDP_MEDIA_PACKET_BYTES: usize = 848;
const MUNINN_RUDP_MEDIA_PACKET_BYTES: usize = 832;
const MUNINN_RUDP_IPV4_UDP_PAYLOAD_BYTES: usize = 1_472;
const MUNINN_RUDP_FIXED_HEADER_BYTES: usize = 36;
const MUNINN_RUDP_MEDIA_MAX_FRAGMENT_BYTES: usize = MUNINN_RUDP_IPV4_UDP_PAYLOAD_BYTES
@ -1861,7 +1861,7 @@ fn run_rudp_mux_once(
options,
node,
MUNINN_MEDIA_RUDP_CONNECTION_ID,
None,
Some(media_profile.receiver_assembly_deadline_ms),
"video",
)?;
let mut audio_transport = open_media_rudp_transport(
@ -2016,13 +2016,17 @@ fn run_rudp_mux_once(
restart_count,
);
let (payload_tx, payload_rx) =
let (video_payload_tx, video_payload_rx) =
mpsc::sync_channel::<Result<Vec<QueuedMuninnMediaSendPayload>>>(
MUNINN_RUDP_MEDIA_PAYLOAD_CHANNEL_CAPACITY,
);
let (audio_payload_tx, audio_payload_rx) =
mpsc::sync_channel::<Result<Vec<QueuedMuninnMediaSendPayload>>>(
MUNINN_RUDP_MEDIA_PAYLOAD_CHANNEL_CAPACITY,
);
video_sender = if let Some(stdout) = video_ffmpeg_stdout {
Some(video_rudp_payload_reader(
payload_tx.clone(),
video_payload_tx.clone(),
stdout,
VideoAnnexBStreamSendConfig {
stream_id: options.stream_id.clone(),
@ -2045,7 +2049,7 @@ fn run_rudp_mux_once(
};
audio_sender = if let Some(stdout) = audio_ffmpeg_stdout {
Some(audio_rudp_payload_reader(
payload_tx.clone(),
audio_payload_tx.clone(),
stdout,
AudioPcmStreamSendConfig {
stream_id: options.stream_id.clone(),
@ -2070,7 +2074,8 @@ fn run_rudp_mux_once(
} else {
None
};
drop(payload_tx);
drop(video_payload_tx);
drop(audio_payload_tx);
let mut payloads_sent = 0_u64;
let mut payloads_queue_expired = 0_u64;
@ -2097,17 +2102,26 @@ fn run_rudp_mux_once(
);
let mut audio_send_pacer = MuninnRudpMediaSendPacer::new(0, Duration::ZERO);
let mut pending_payloads = PendingMuninnMediaSendQueues::default();
let mut payload_channel_disconnected = false;
let mut video_payload_channel_disconnected = false;
let mut audio_payload_channel_disconnected = false;
loop {
if pending_payloads.is_empty() && !payload_channel_disconnected {
payload_channel_disconnected = receive_pending_media_payloads(
&payload_rx,
if !audio_payload_channel_disconnected {
audio_payload_channel_disconnected = drain_available_media_payloads(
&audio_payload_rx,
&mut pending_payloads,
Duration::from_millis(5),
)?;
} else if !payload_channel_disconnected {
payload_channel_disconnected =
drain_available_media_payloads(&payload_rx, &mut pending_payloads)?;
}
if !video_payload_channel_disconnected {
video_payload_channel_disconnected = drain_available_media_payloads(
&video_payload_rx,
&mut pending_payloads,
)?;
}
if pending_payloads.is_empty()
&& !(video_payload_channel_disconnected && audio_payload_channel_disconnected)
{
thread::sleep(Duration::from_millis(1));
continue;
}
if let Some(queued) = pending_payloads.pop_next() {
@ -2136,6 +2150,7 @@ fn run_rudp_mux_once(
media_profile.receiver_assembly_deadline_ms,
)?;
poll_rudp_resends_with_backpressure(&mut video_transport)?;
drain_rudp_transport_receipts(&mut audio_transport)?;
poll_rudp_resends_with_backpressure(&mut audio_transport)?;
if payloads_dropped == 1 || payloads_dropped % 300 == 0 {
let expired = reliable_packets_expired(&video_transport, &audio_transport);
@ -2201,12 +2216,13 @@ fn run_rudp_mux_once(
media_profile.receiver_assembly_deadline_ms,
)?;
poll_rudp_resends_with_backpressure(&mut video_transport)?;
drain_rudp_transport_receipts(&mut audio_transport)?;
poll_rudp_resends_with_backpressure(&mut audio_transport)?;
payloads_sent += 1;
if payloads_sent == 1 || payloads_sent % 900 == 0 {
let expired = reliable_packets_expired(&video_transport, &audio_transport);
eprintln!(
"{}; pending_audio={} pending_video={}; latest {:?} payload was {payload_len} bytes.",
"{}; video_rx_bytes={} video_reliable_pending={} audio_rx_bytes={} audio_reliable_pending={} pending_audio={} pending_video={} audio_capacity_dropped={} video_capacity_dropped={}; latest {:?} payload was {payload_len} bytes.",
rudp_media_progress_detail(
payloads_sent,
payloads_dropped,
@ -2215,15 +2231,21 @@ fn run_rudp_mux_once(
expired,
&receiver_feedback
),
video_transport.stats().bytes_received,
video_transport.pending_reliable_packets(),
audio_transport.stats().bytes_received,
audio_transport.pending_reliable_packets(),
pending_payloads.audio_len(),
pending_payloads.video_len(),
pending_payloads.audio_capacity_dropped(),
pending_payloads.video_capacity_dropped(),
queued.kind
);
}
continue;
}
if payload_channel_disconnected {
if video_payload_channel_disconnected && audio_payload_channel_disconnected {
let expired = reliable_packets_expired(&video_transport, &audio_transport);
let payloads_dropped = payloads_queue_expired + payloads_send_expired;
break Ok(RudpMuxRestart {
@ -2262,6 +2284,7 @@ fn run_rudp_mux_once(
media_profile.receiver_assembly_deadline_ms,
)?;
poll_rudp_resends_with_backpressure(&mut video_transport)?;
drain_rudp_transport_receipts(&mut audio_transport)?;
poll_rudp_resends_with_backpressure(&mut audio_transport)?;
}
}
@ -2309,6 +2332,8 @@ struct PendingMuninnMediaSendQueues {
audio: VecDeque<VecDeque<QueuedMuninnMediaSendPayload>>,
active_video: VecDeque<QueuedMuninnMediaSendPayload>,
video: VecDeque<VecDeque<QueuedMuninnMediaSendPayload>>,
audio_capacity_dropped: u64,
video_capacity_dropped: u64,
}
impl PendingMuninnMediaSendQueues {
@ -2328,7 +2353,11 @@ impl PendingMuninnMediaSendQueues {
> MUNINN_RUDP_MEDIA_PENDING_AUDIO_CAPACITY
&& !self.audio.is_empty()
{
self.audio.pop_front();
if let Some(dropped) = self.audio.pop_front() {
self.audio_capacity_dropped = self
.audio_capacity_dropped
.saturating_add(dropped.len() as u64);
}
}
self.audio.push_back(group);
}
@ -2337,7 +2366,11 @@ impl PendingMuninnMediaSendQueues {
> MUNINN_RUDP_MEDIA_PENDING_VIDEO_CAPACITY
&& !self.video.is_empty()
{
self.video.pop_front();
if let Some(dropped) = self.video.pop_front() {
self.video_capacity_dropped = self
.video_capacity_dropped
.saturating_add(dropped.len() as u64);
}
}
self.video.push_back(group);
}
@ -2375,6 +2408,14 @@ impl PendingMuninnMediaSendQueues {
fn video_len(&self) -> usize {
self.active_video.len() + self.video.iter().map(VecDeque::len).sum::<usize>()
}
fn audio_capacity_dropped(&self) -> u64 {
self.audio_capacity_dropped
}
fn video_capacity_dropped(&self) -> u64 {
self.video_capacity_dropped
}
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
@ -2405,12 +2446,16 @@ struct MuninnVideoBitrateController {
}
impl MuninnVideoBitrateController {
fn new(max_kbps: u32, now: Instant) -> Self {
let max_kbps = max_kbps.max(1_000);
let current_kbps = (max_kbps.saturating_mul(2) / 3).max(1_000);
fn new(configured_kbps: u32, now: Instant) -> Self {
let configured_kbps = configured_kbps.max(1_000);
// Block FEC can roughly double the encoder payload on the wire. Keep
// additive probing below the configured payload ceiling so clean video
// does not have to lose a frame to discover the protected wire limit.
let max_kbps = (configured_kbps / 2).max(1_000);
let current_kbps = (configured_kbps / 2).max(1_000).min(max_kbps);
Self {
current_kbps,
min_kbps: (max_kbps / 4).max(1_000),
min_kbps: (configured_kbps / 4).max(1_000).min(max_kbps),
max_kbps,
last_late_frames: 0,
last_keyframes: 0,
@ -2689,22 +2734,6 @@ fn queue_muninn_video_payloads_by_access_unit(
queue_muninn_media_payloads(tx, current, QueuedMuninnMediaKind::Video)
}
fn receive_pending_media_payloads(
rx: &mpsc::Receiver<Result<Vec<QueuedMuninnMediaSendPayload>>>,
pending: &mut PendingMuninnMediaSendQueues,
timeout: Duration,
) -> Result<bool> {
match rx.recv_timeout(timeout) {
Ok(Ok(payloads)) => {
pending.push_group(payloads);
drain_available_media_payloads(rx, pending)
}
Ok(Err(error)) => Err(error),
Err(mpsc::RecvTimeoutError::Timeout) => Ok(false),
Err(mpsc::RecvTimeoutError::Disconnected) => Ok(true),
}
}
fn drain_available_media_payloads(
rx: &mpsc::Receiver<Result<Vec<QueuedMuninnMediaSendPayload>>>,
pending: &mut PendingMuninnMediaSendQueues,
@ -3017,6 +3046,7 @@ fn open_media_rudp_transport(
connection_id,
reliable_expire_after_ms,
))?;
transport.set_reliable_expire_after_ms(reliable_expire_after_ms);
transport.connect(options.stream_id.as_bytes().to_vec())?;
let deadline = Instant::now() + Duration::from_secs(5);
while !transport.connected() {
@ -3132,12 +3162,26 @@ fn muninn_media_rudp_options(
options
}
fn drain_rudp_transport_receipts(
transport: &mut CultNetRudpSocketTransportConnection,
) -> Result<()> {
loop {
match transport.receive_once() {
Ok(Some(_)) => continue,
Ok(None) => return Ok(()),
Err(error) if is_would_block_error(&error) => return Ok(()),
Err(error) => return Err(error).context("draining Muninn RUDP transport receipts"),
}
}
}
fn reliable_packets_expired(
video_transport: &CultNetRudpSocketTransportConnection,
audio_transport: &CultNetRudpSocketTransportConnection,
) -> u64 {
let _ = (video_transport, audio_transport);
0
video_transport
.reliable_packets_expired()
.saturating_add(audio_transport.reliable_packets_expired())
}
fn video_rudp_payload_reader<R>(
@ -11998,12 +12042,12 @@ mod tests {
.unwrap();
assert_eq!(
media_channel.delivery,
cultnet_rs::CultNetTransportDelivery::Unreliable
cultnet_rs::CultNetTransportDelivery::Reliable
);
}
#[test]
fn rudp_audio_transport_uses_separate_realtime_connection() {
fn rudp_audio_transport_uses_separate_media_connection() {
let socket = UdpSocket::bind("127.0.0.1:0").unwrap();
let endpoint: SocketAddr = "127.0.0.1:5204".parse().unwrap();
let profile = muninn_rudp_media_profile();
@ -12100,6 +12144,48 @@ mod tests {
assert!(pending.is_empty());
}
#[test]
fn reliable_media_packets_expire_at_the_consumer_deadline() {
use cultnet_rs::{
CultNetRudpSendOptions, CultNetRudpSession, CultNetRudpSessionOptions,
};
let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions {
connection_id: 77,
initial_sequence: 1,
resend_delay_ms: 5,
max_pending_reliable_packets: Some(16),
});
let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions {
connection_id: 77,
initial_sequence: 1,
resend_delay_ms: 5,
max_pending_reliable_packets: Some(16),
});
sender.set_reliable_expire_after_ms(Some(250));
let connect = sender.create_connect(0, Vec::new()).unwrap();
let accept = receiver.accept_connect(&connect, 0, Vec::new()).unwrap();
sender.receive(&accept, 0).unwrap();
let packet = sender
.send(
crate::media_packetizer::MUNINN_MEDIA_RUDP_CHANNEL,
vec![1, 2, 3],
CultNetRudpSendOptions {
reliable: true,
ordered: false,
sequenced: false,
now_ms: 10,
},
)
.unwrap();
assert!(packet.reliable);
assert!(sender.due_resends(14).is_empty());
assert_eq!(sender.due_resends(15).len(), 1);
assert!(sender.due_resends(261).is_empty());
assert!(sender.pending_reliable_sequences().is_empty());
assert_eq!(sender.reliable_packets_expired(), 1);
}
#[test]
fn rudp_media_pending_queues_are_structurally_bounded() {
let queued_at = Instant::now();
@ -12393,7 +12479,7 @@ mod tests {
feedback.late_frames = 1;
assert_eq!(
controller.observe(&feedback, 0, 100_000, start + Duration::from_millis(500)),
Some(6_800)
Some(5_100)
);
assert_eq!(
controller.observe(&feedback, 0, 100_000, start + Duration::from_secs(1)),
@ -12405,7 +12491,7 @@ mod tests {
);
assert_eq!(
controller.observe(&feedback, 0, 100_000, start + Duration::from_millis(10_500)),
Some(7_040)
Some(5_220)
);
}
@ -12421,10 +12507,10 @@ mod tests {
let adjusted = controller
.observe(&feedback, 0, 100_000, start + Duration::from_millis(500))
.unwrap();
assert_eq!(adjusted, 9_066);
assert_eq!(adjusted, 6_800);
let mut commands = Vec::new();
request_video_encoder_bitrate(Some(&mut commands), adjusted).unwrap();
assert_eq!(commands, b"BITRATE 9066\n");
assert_eq!(commands, b"BITRATE 6800\n");
}
#[test]

View file

@ -16,7 +16,8 @@ use std::collections::BTreeMap;
// Deadline-bound audio/video must use the experimental CultNet realtime lane.
// The `media` lane is reliable without expiry and can accumulate corpses behind
// loss; application-level parity/repair and playout deadlines own recovery.
pub const MUNINN_MEDIA_RUDP_CHANNEL: &str = "realtime";
pub const MUNINN_MEDIA_RUDP_CHANNEL: &str = "media";
pub const MUNINN_MEDIA_REALTIME_RUDP_CHANNEL: &str = "realtime";
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct VideoAccessUnit {
@ -309,6 +310,10 @@ struct VideoParityShardWirePayload<'a>(
u16,
u16,
u16,
u16,
u16,
u16,
u16,
u32,
u32,
#[serde(with = "serde_bytes")] &'a [u8],
@ -1132,7 +1137,7 @@ pub fn packetize_video_access_unit(
Ok(records)
}
const MUNINN_VIDEO_PARITY_SHARDS: u16 = 16;
const MUNINN_VIDEO_FEC_BLOCK_DATA_SHARDS: u16 = 8;
fn video_fec_coefficient(parity_index: u16, parity_count: u16, data_index: u16) -> u8 {
debug_assert!(u32::from(parity_count) + u32::from(data_index) <= 255);
@ -1183,53 +1188,71 @@ pub fn build_video_parity_shards(
}
}
let mut chunk_payload_len = 0_usize;
let mut last_chunk_payload_bytes = 0_u32;
for index in 0..first.chunk_count {
let chunk = by_index
.get(&index)
.ok_or_else(|| anyhow!("video parity missing chunk_index {index}"))?;
chunk_payload_len = chunk_payload_len.max(chunk.payload.len());
if index == first.chunk_count - 1 {
last_chunk_payload_bytes = u32::try_from(chunk.payload.len())
.context("video parity last chunk payload length exceeds u32")?;
if !by_index.contains_key(&index) {
return Err(anyhow!("video parity missing chunk_index {index}"));
}
}
let chunk_payload_bytes = u32::try_from(chunk_payload_len)
.context("video parity chunk payload length exceeds u32")?;
let parity_count = first
.chunk_count
.min(MUNINN_VIDEO_PARITY_SHARDS)
.min(256_u16.saturating_sub(first.chunk_count));
let mut records = Vec::with_capacity(parity_count as usize);
for parity_index in 0..parity_count {
let mut parity = vec![0_u8; chunk_payload_len];
for index in 0..first.chunk_count {
let chunk = by_index.get(&index).expect("chunk index was checked above");
let coefficient = video_fec_coefficient(parity_index, parity_count, index);
for (offset, byte) in chunk.payload.iter().enumerate() {
parity[offset] ^= gf256_mul(*byte, coefficient);
let block_count = first.chunk_count.div_ceil(MUNINN_VIDEO_FEC_BLOCK_DATA_SHARDS);
let mut records = Vec::with_capacity(first.chunk_count as usize);
for block_index in 0..block_count {
let block_data_start = block_index * MUNINN_VIDEO_FEC_BLOCK_DATA_SHARDS;
let block_data_count = (first.chunk_count - block_data_start)
.min(MUNINN_VIDEO_FEC_BLOCK_DATA_SHARDS);
let parity_count = MUNINN_VIDEO_FEC_BLOCK_DATA_SHARDS;
let block_end = block_data_start + block_data_count;
let chunk_payload_len = (block_data_start..block_end)
.map(|index| by_index.get(&index).expect("chunk index checked").payload.len())
.max()
.expect("block contains data");
let last_chunk_payload_bytes = u32::try_from(
by_index
.get(&(block_end - 1))
.expect("block last chunk checked")
.payload
.len(),
)
.context("video parity block last chunk payload length exceeds u32")?;
let chunk_payload_bytes = u32::try_from(chunk_payload_len)
.context("video parity chunk payload length exceeds u32")?;
for parity_index in 0..parity_count {
let mut parity = vec![0_u8; chunk_payload_len];
for local_index in 0..block_data_count {
let chunk = by_index
.get(&(block_data_start + local_index))
.expect("block chunk checked");
let coefficient =
video_fec_coefficient(parity_index, parity_count, local_index);
for (offset, byte) in chunk.payload.iter().enumerate() {
parity[offset] ^= gf256_mul(*byte, coefficient);
}
}
records.push(MuninnMediaVideoParityShardRecord {
stream_id: first.stream_id.clone(),
session_id: first.session_id.clone(),
frame_id: first.frame_id,
codec: first.codec.clone(),
pts_ticks: first.pts_ticks,
duration_ticks: first.duration_ticks,
timebase_num: first.timebase_num,
timebase_den: first.timebase_den,
keyframe: first.keyframe,
dependency_frame_id: first.dependency_frame_id,
deadline_ticks: first.deadline_ticks,
chunk_count: first.chunk_count,
block_index,
block_count,
block_data_start,
block_data_count,
parity_index,
parity_count,
chunk_payload_bytes,
last_chunk_payload_bytes,
payload: parity,
});
}
records.push(MuninnMediaVideoParityShardRecord {
stream_id: first.stream_id.clone(),
session_id: first.session_id.clone(),
frame_id: first.frame_id,
codec: first.codec.clone(),
pts_ticks: first.pts_ticks,
duration_ticks: first.duration_ticks,
timebase_num: first.timebase_num,
timebase_den: first.timebase_den,
keyframe: first.keyframe,
dependency_frame_id: first.dependency_frame_id,
deadline_ticks: first.deadline_ticks,
chunk_count: first.chunk_count,
parity_index,
parity_count,
chunk_payload_bytes,
last_chunk_payload_bytes,
payload: parity,
});
}
Ok(records)
}
@ -1251,25 +1274,48 @@ fn video_wire_records_with_parity(
}
let frame_records = &records[offset..end];
let parity = build_video_parity_shards(frame_records)?;
let parity_split = parity.len().div_ceil(2);
wire_records.extend(
parity[..parity_split]
if parity.is_empty() {
wire_records.extend(
frame_records
.iter()
.cloned()
.map(MuninnMediaWireRecord::Video),
);
offset = end;
continue;
}
let block_count = parity
.first()
.map_or(0_usize, |record| record.block_count as usize);
let mut block_lanes = vec![Vec::new(); block_count];
for block_index in 0..block_count {
let block_parity = parity
.iter()
.cloned()
.map(MuninnMediaWireRecord::VideoParity),
);
wire_records.extend(
frame_records
.iter()
.cloned()
.map(MuninnMediaWireRecord::Video),
);
wire_records.extend(
parity[parity_split..]
.iter()
.cloned()
.map(MuninnMediaWireRecord::VideoParity),
);
.filter(|record| record.block_index as usize == block_index)
.collect::<Vec<_>>();
let Some(first_parity) = block_parity.first() else {
return Err(anyhow!("video FEC block {block_index} has no parity"));
};
for local_index in 0..first_parity.parity_count as usize {
if local_index < first_parity.block_data_count as usize {
let global_index = first_parity.block_data_start as usize + local_index;
block_lanes[block_index].push(MuninnMediaWireRecord::Video(
frame_records[global_index].clone(),
));
}
block_lanes[block_index].push(MuninnMediaWireRecord::VideoParity(
block_parity[local_index].clone(),
));
}
}
let lane_len = block_lanes.iter().map(Vec::len).max().unwrap_or(0);
for lane_position in 0..lane_len {
for lane in &block_lanes {
if let Some(record) = lane.get(lane_position) {
wire_records.push(record.clone());
}
}
}
offset = end;
}
Ok(wire_records)
@ -1977,8 +2023,19 @@ fn wire_payloads_to_media_send(payloads: Vec<Vec<u8>>) -> Vec<MuninnMediaSendPay
}
fn wire_payload_to_media_send(payload: Vec<u8>) -> MuninnMediaSendPayload {
let channel_id = match decode_media_wire_record(&payload) {
Ok(
MuninnMediaWireRecord::Video(_)
| MuninnMediaWireRecord::VideoParity(_)
| MuninnMediaWireRecord::Audio(_)
| MuninnMediaWireRecord::AudioParity(_),
) => {
MUNINN_MEDIA_REALTIME_RUDP_CHANNEL
}
_ => MUNINN_MEDIA_RUDP_CHANNEL,
};
MuninnMediaSendPayload {
channel_id: MUNINN_MEDIA_RUDP_CHANNEL,
channel_id,
payload,
}
}
@ -2105,6 +2162,10 @@ fn encode_video_parity_record_payload(
record.dependency_frame_id,
record.deadline_ticks,
record.chunk_count,
record.block_index,
record.block_count,
record.block_data_start,
record.block_data_count,
record.parity_index,
record.parity_count,
record.chunk_payload_bytes,
@ -2248,14 +2309,28 @@ fn validate_video_parity_record(record: &MuninnMediaVideoParityShardRecord) -> R
"video parity media record chunk_count must be non-zero"
));
}
if record.parity_count == 0 || record.parity_index >= record.parity_count {
return Err(anyhow!(
"video parity media record has invalid parity stripe metadata"
));
let expected_block_count = record
.chunk_count
.div_ceil(MUNINN_VIDEO_FEC_BLOCK_DATA_SHARDS);
if record.block_count != expected_block_count
|| record.block_index >= record.block_count
|| record.block_data_count == 0
|| record.block_data_count > MUNINN_VIDEO_FEC_BLOCK_DATA_SHARDS
|| record.block_data_start
!= record.block_index * MUNINN_VIDEO_FEC_BLOCK_DATA_SHARDS
|| record
.block_data_start
.checked_add(record.block_data_count)
.is_none_or(|end| end > record.chunk_count)
{
return Err(anyhow!("video parity media record has invalid FEC block metadata"));
}
if record.parity_count > record.chunk_count {
if record.parity_count < record.block_data_count
|| record.parity_count > MUNINN_VIDEO_FEC_BLOCK_DATA_SHARDS
|| record.parity_index >= record.parity_count
{
return Err(anyhow!(
"video parity media record parity_count exceeds chunk_count"
"video parity media record has invalid block parity metadata"
));
}
if record.chunk_payload_bytes == 0 || record.last_chunk_payload_bytes == 0 {
@ -2378,8 +2453,12 @@ fn video_record_key(record: &MuninnMediaVideoAccessUnitRecord) -> String {
fn video_parity_record_key(record: &MuninnMediaVideoParityShardRecord) -> String {
format!(
"{}:{}:video-parity:{}:{}",
record.stream_id, record.session_id, record.frame_id, record.parity_index
"{}:{}:video-parity:{}:{}:{}",
record.stream_id,
record.session_id,
record.frame_id,
record.block_index,
record.parity_index
)
}
@ -2860,22 +2939,24 @@ mod tests {
let parity = build_video_parity_shards(&records)?;
assert_eq!(records.len(), 20);
assert_eq!(parity.len(), 16);
assert!(
parity.iter().enumerate().all(
|(index, shard)| shard.parity_index == index as u16 && shard.parity_count == 16
)
);
assert_eq!(parity.len(), 24);
assert_eq!(parity.iter().map(|shard| shard.block_index).collect::<Vec<_>>(),
[vec![0; 8], vec![1; 8], vec![2; 8]].concat());
assert!(parity.iter().all(|shard| shard.parity_count == 8));
assert!(parity.iter().all(|shard| shard.chunk_payload_bytes == 2));
let missing_index = 18_u16;
let shard = &parity[7];
let shard = &parity[18];
let mut recovered = shard.payload.clone();
for chunk in records.iter().filter(|chunk| chunk.chunk_index != missing_index) {
for chunk in records.iter().filter(|chunk| {
chunk.chunk_index >= shard.block_data_start
&& chunk.chunk_index < shard.block_data_start + shard.block_data_count
&& chunk.chunk_index != missing_index
}) {
let coefficient = video_fec_coefficient(
shard.parity_index,
shard.parity_count,
chunk.chunk_index,
chunk.chunk_index - shard.block_data_start,
);
for (offset, byte) in chunk.payload.iter().enumerate() {
recovered[offset] ^= gf256_mul(*byte, coefficient);
@ -2884,7 +2965,7 @@ mod tests {
let inverse = gf256_inv(video_fec_coefficient(
shard.parity_index,
shard.parity_count,
missing_index,
missing_index - shard.block_data_start,
));
for byte in &mut recovered {
*byte = gf256_mul(*byte, inverse);
@ -2895,7 +2976,7 @@ mod tests {
}
#[test]
fn video_wire_bookends_data_with_independent_parity() -> Result<()> {
fn video_wire_round_robins_independent_fec_blocks() -> Result<()> {
let access_unit = VideoAccessUnit {
bytes: (1_u8..=40).collect(),
keyframe: true,
@ -2917,10 +2998,124 @@ mod tests {
)?;
let wire = video_wire_records_with_parity(&records)?;
assert_eq!(wire.len(), 36);
assert!(wire[..8].iter().all(|record| matches!(record, MuninnMediaWireRecord::VideoParity(_))));
assert!(wire[8..28].iter().all(|record| matches!(record, MuninnMediaWireRecord::Video(_))));
assert!(wire[28..].iter().all(|record| matches!(record, MuninnMediaWireRecord::VideoParity(_))));
assert_eq!(wire.len(), 44);
assert_eq!(
wire[..3]
.iter()
.map(|record| match record {
MuninnMediaWireRecord::Video(record) => record.chunk_index,
_ => u16::MAX,
})
.collect::<Vec<_>>(),
vec![0, 8, 16]
);
assert_eq!(
wire[3..6]
.iter()
.map(|record| match record {
MuninnMediaWireRecord::VideoParity(record) => record.block_index,
_ => u16::MAX,
})
.collect::<Vec<_>>(),
vec![0, 1, 2]
);
Ok(())
}
#[test]
fn video_block_layout_survives_every_observed_contiguous_burst_alignment() -> Result<()> {
let access_unit = VideoAccessUnit {
bytes: (0..80).map(|index| (index * 17) as u8).collect(),
keyframe: true,
};
let records = packetize_video_access_unit(
VideoFramePacketizeOptions {
stream_id: "muninn.raven.av.rudp",
session_id: "session-1",
codec: "h264",
frame_id: 10,
pts_ticks: 30_000,
duration_ticks: 3_000,
timebase_num: 1,
timebase_den: 90_000,
deadline_ticks: 52_500,
max_payload_bytes: 1,
},
&access_unit,
)?;
let wire = video_wire_records_with_parity(&records)?;
assert_eq!(wire.len(), 160);
for burst_len in 1..=54_usize {
for burst_start in 0..=wire.len() - burst_len {
let mut missing_data = [0_u16; 10];
let mut surviving_parity = [8_u16; 10];
for record in &wire[burst_start..burst_start + burst_len] {
match record {
MuninnMediaWireRecord::Video(record) => {
missing_data[record.chunk_index as usize / 8] += 1;
}
MuninnMediaWireRecord::VideoParity(record) => {
surviving_parity[record.block_index as usize] -= 1;
}
_ => unreachable!("video wire contains only video records"),
}
}
assert!(missing_data
.iter()
.zip(surviving_parity)
.all(|(missing, parity)| *missing <= parity));
}
}
Ok(())
}
#[test]
fn video_tail_block_survives_eight_packet_burst_at_every_alignment() -> Result<()> {
for chunk_count in 2_usize..=80 {
let access_unit = VideoAccessUnit {
bytes: (0..chunk_count).map(|index| (index * 19) as u8).collect(),
keyframe: false,
};
let records = packetize_video_access_unit(
VideoFramePacketizeOptions {
stream_id: "muninn.raven.av.rudp",
session_id: "session-1",
codec: "h264",
frame_id: 11,
pts_ticks: 33_000,
duration_ticks: 3_000,
timebase_num: 1,
timebase_den: 90_000,
deadline_ticks: 55_500,
max_payload_bytes: 1,
},
&access_unit,
)?;
let wire = video_wire_records_with_parity(&records)?;
for burst_len in 1..=8_usize.min(wire.len()) {
for burst_start in 0..=wire.len() - burst_len {
let block_count = chunk_count.div_ceil(8);
let mut missing_data = vec![0_u16; block_count];
let mut surviving_parity = vec![8_u16; block_count];
for record in &wire[burst_start..burst_start + burst_len] {
match record {
MuninnMediaWireRecord::Video(record) => {
missing_data[record.chunk_index as usize / 8] += 1;
}
MuninnMediaWireRecord::VideoParity(record) => {
surviving_parity[record.block_index as usize] -= 1;
}
_ => unreachable!("video wire contains only video records"),
}
}
assert!(missing_data
.iter()
.zip(surviving_parity)
.all(|(missing, parity)| *missing <= parity));
}
}
}
Ok(())
}
@ -3950,7 +4145,7 @@ mod tests {
}
#[test]
fn media_send_payloads_pin_video_to_media_channel() -> Result<()> {
fn media_send_payloads_pin_video_to_realtime_channel() -> Result<()> {
let mut stream = Vec::new();
stream.extend_from_slice(&start_code());
stream.extend_from_slice(&[0x65, 0x80]);
@ -3977,7 +4172,7 @@ mod tests {
)?;
assert_eq!(payloads.len(), 1);
assert_eq!(payloads[0].channel_id, MUNINN_MEDIA_RUDP_CHANNEL);
assert_eq!(payloads[0].channel_id, MUNINN_MEDIA_REALTIME_RUDP_CHANNEL);
let MuninnMediaWireRecord::Video(record) = decode_media_wire_record(&payloads[0].payload)?
else {
panic!("expected video media record");
@ -4055,7 +4250,10 @@ mod tests {
let second_push = sender.push("2026-06-18T00:00:00Z", &second_frame)?;
assert_eq!(second_push.len(), 1);
assert_eq!(second_push[0].channel_id, MUNINN_MEDIA_RUDP_CHANNEL);
assert_eq!(
second_push[0].channel_id,
MUNINN_MEDIA_REALTIME_RUDP_CHANNEL
);
let MuninnMediaWireRecord::Video(first) =
decode_media_wire_record(&second_push[0].payload)?
else {
@ -4134,8 +4332,8 @@ mod tests {
let second_push = sender.push("2026-06-18T00:00:00Z", &tail)?;
assert_eq!(second_push.len(), 2);
assert_eq!(second_push[0].channel_id, MUNINN_MEDIA_RUDP_CHANNEL);
assert_eq!(second_push[1].channel_id, MUNINN_MEDIA_RUDP_CHANNEL);
assert_eq!(second_push[0].channel_id, MUNINN_MEDIA_REALTIME_RUDP_CHANNEL);
assert_eq!(second_push[1].channel_id, MUNINN_MEDIA_REALTIME_RUDP_CHANNEL);
let MuninnMediaWireRecord::Audio(first) =
decode_media_wire_record(&second_push[0].payload)?
@ -4241,7 +4439,7 @@ mod tests {
}
#[test]
fn media_send_payload_pins_audio_to_media_channel() -> Result<()> {
fn media_send_payload_pins_audio_to_realtime_channel() -> Result<()> {
let payload = audio_packet_send_payload(
AudioPacketWireOptions {
packetize: AudioPacketizeOptions {
@ -4262,7 +4460,7 @@ mod tests {
&[0xf8, 0xff, 0xfe],
)?;
assert_eq!(payload.channel_id, MUNINN_MEDIA_RUDP_CHANNEL);
assert_eq!(payload.channel_id, MUNINN_MEDIA_REALTIME_RUDP_CHANNEL);
let MuninnMediaWireRecord::Audio(record) = decode_media_wire_record(&payload.payload)?
else {
panic!("expected audio media record");

View file

@ -36,7 +36,7 @@ pub const MUNINN_TELEMETRY_SURFACE_SCHEMA: &str = "muninn.telemetry_surface.v1";
pub const MUNINN_CAPTURE_STREAM_SCHEMA: &str = "muninn.capture_stream.v1";
pub const MUNINN_CAPTURE_STREAM_COMMAND_SCHEMA: &str = "muninn.capture_stream_command.v1";
pub const MUNINN_MEDIA_VIDEO_ACCESS_UNIT_SCHEMA: &str = "muninn.media_video_access_unit.v1";
pub const MUNINN_MEDIA_VIDEO_PARITY_SHARD_SCHEMA: &str = "muninn.media_video_parity_shard.v3";
pub const MUNINN_MEDIA_VIDEO_PARITY_SHARD_SCHEMA: &str = "muninn.media_video_parity_shard.v4";
pub const MUNINN_MEDIA_AUDIO_PACKET_SCHEMA: &str = "muninn.media_audio_packet.v1";
pub const MUNINN_MEDIA_AUDIO_PARITY_SHARD_SCHEMA: &str = "muninn.media_audio_parity_shard.v1";
pub const MUNINN_MEDIA_RECEIVER_FEEDBACK_SCHEMA: &str = "muninn.media_receiver_feedback.v1";
@ -829,7 +829,7 @@ pub struct MuninnMediaVideoAccessUnitRecord {
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
#[cultcache(
type = "muninn.media_video_parity_shard",
schema = "muninn.media_video_parity_shard.v3"
schema = "muninn.media_video_parity_shard.v4"
)]
pub struct MuninnMediaVideoParityShardRecord {
#[cultcache(key = 0)]
@ -857,14 +857,22 @@ pub struct MuninnMediaVideoParityShardRecord {
#[cultcache(key = 11)]
pub chunk_count: u16,
#[cultcache(key = 12)]
pub parity_index: u16,
pub block_index: u16,
#[cultcache(key = 13)]
pub parity_count: u16,
pub block_count: u16,
#[cultcache(key = 14)]
pub chunk_payload_bytes: u32,
pub block_data_start: u16,
#[cultcache(key = 15)]
pub last_chunk_payload_bytes: u32,
pub block_data_count: u16,
#[cultcache(key = 16)]
pub parity_index: u16,
#[cultcache(key = 17)]
pub parity_count: u16,
#[cultcache(key = 18)]
pub chunk_payload_bytes: u32,
#[cultcache(key = 19)]
pub last_chunk_payload_bytes: u32,
#[cultcache(key = 20)]
pub payload: Vec<u8>,
}

View file

@ -105,6 +105,7 @@ pub struct CultNetRudpSendOptions {
#[derive(Clone, Debug, PartialEq, Eq)]
struct PendingReliablePacket {
packet: CultNetRudpPacket,
first_sent_at_ms: u64,
last_sent_at_ms: u64,
}
@ -127,6 +128,8 @@ pub struct CultNetRudpSession {
connection_id: u32,
resend_delay_ms: u64,
max_pending_reliable_packets: Option<usize>,
reliable_expire_after_ms: Option<u64>,
reliable_packets_expired: u64,
initial_sequence: u32,
next_sequence: u32,
next_fragment_id: u16,
@ -146,6 +149,8 @@ impl CultNetRudpSession {
connection_id: options.connection_id,
resend_delay_ms: options.resend_delay_ms,
max_pending_reliable_packets: options.max_pending_reliable_packets,
reliable_expire_after_ms: None,
reliable_packets_expired: 0,
initial_sequence: options.initial_sequence,
next_sequence: options.initial_sequence,
next_fragment_id: 1,
@ -176,6 +181,14 @@ impl CultNetRudpSession {
self.pending_reliable.keys().copied().collect()
}
pub fn reliable_packets_expired(&self) -> u64 {
self.reliable_packets_expired
}
pub fn set_reliable_expire_after_ms(&mut self, expire_after_ms: Option<u64>) {
self.reliable_expire_after_ms = expire_after_ms;
}
pub fn last_received_at_ms(&self) -> Option<u64> {
self.last_received_at_ms
}
@ -495,6 +508,15 @@ impl CultNetRudpSession {
}
pub fn due_resends(&mut self, now_ms: u64) -> Vec<CultNetRudpPacket> {
if let Some(expire_after_ms) = self.reliable_expire_after_ms {
let before = self.pending_reliable.len();
self.pending_reliable.retain(|_, pending| {
now_ms.saturating_sub(pending.first_sent_at_ms) <= expire_after_ms
});
self.reliable_packets_expired = self
.reliable_packets_expired
.saturating_add((before - self.pending_reliable.len()) as u64);
}
let mut due = Vec::new();
for pending in self.pending_reliable.values_mut() {
if now_ms.saturating_sub(pending.last_sent_at_ms) >= self.resend_delay_ms {
@ -569,6 +591,7 @@ impl CultNetRudpSession {
packet.sequence,
PendingReliablePacket {
packet,
first_sent_at_ms: now_ms,
last_sent_at_ms: now_ms,
},
);
@ -580,7 +603,11 @@ impl CultNetRudpSession {
}
if let Some(limit) = self.max_pending_reliable_packets {
if self.pending_reliable.len() + packet_count > limit {
return Err(anyhow!("RUDP reliable send queue is full"));
return Err(std::io::Error::new(
ErrorKind::WouldBlock,
"RUDP reliable send queue is full",
)
.into());
}
}
Ok(())
@ -795,7 +822,7 @@ impl CultNetRudpSession {
fn allocate_fragment_id(&mut self) -> u16 {
let fragment_id = self.next_fragment_id;
self.next_fragment_id = self.next_fragment_id.saturating_add(1);
self.next_fragment_id = self.next_fragment_id.wrapping_add(1);
if self.next_fragment_id == 0 {
self.next_fragment_id = 1;
}
@ -887,6 +914,8 @@ pub struct CultNetRudpSocketTransportConnection {
max_fragment_bytes: Option<usize>,
disconnect_reason: Option<Vec<u8>>,
pong_payloads: VecDeque<Vec<u8>>,
pending_outbound_packets: VecDeque<CultNetRudpPacket>,
max_pending_outbound_packets: usize,
}
impl CultNetRudpSocketTransportConnection {
@ -907,14 +936,16 @@ impl CultNetRudpSocketTransportConnection {
let max_pending_reliable_packets = options
.max_pending_reliable_packets
.map(|value| value as usize);
let max_pending_outbound_packets = max_pending_reliable_packets.unwrap_or(1_024).max(1);
let session = CultNetRudpSession::new(CultNetRudpSessionOptions {
connection_id: options.connection_id,
initial_sequence: options.initial_sequence,
resend_delay_ms: options.resend_delay_ms,
max_pending_reliable_packets,
});
Ok(Self {
socket: options.socket,
session: CultNetRudpSession::new(CultNetRudpSessionOptions {
connection_id: options.connection_id,
initial_sequence: options.initial_sequence,
resend_delay_ms: options.resend_delay_ms,
max_pending_reliable_packets,
}),
session,
mode: options.mode,
remote_addr: options.remote_addr,
profile,
@ -923,6 +954,8 @@ impl CultNetRudpSocketTransportConnection {
max_fragment_bytes: options.max_fragment_bytes.map(|value| value as usize),
disconnect_reason: None,
pong_payloads: VecDeque::new(),
pending_outbound_packets: VecDeque::new(),
max_pending_outbound_packets,
})
}
@ -934,6 +967,18 @@ impl CultNetRudpSocketTransportConnection {
self.stats.clone()
}
pub fn reliable_packets_expired(&self) -> u64 {
self.session.reliable_packets_expired()
}
pub fn pending_reliable_packets(&self) -> usize {
self.session.pending_reliable_sequences().len()
}
pub fn set_reliable_expire_after_ms(&mut self, expire_after_ms: Option<u64>) {
self.session.set_reliable_expire_after_ms(expire_after_ms);
}
pub fn disconnect_reason(&self) -> Option<&[u8]> {
self.disconnect_reason.as_deref()
}
@ -953,13 +998,23 @@ impl CultNetRudpSocketTransportConnection {
}
pub fn send(&mut self, channel_id: &str, payload: Vec<u8>) -> Result<()> {
self.flush_pending_outbound_packets()?;
let packet_count = self.outbound_packet_count(payload.len())?;
if self.pending_outbound_packets.len().saturating_add(packet_count)
> self.max_pending_outbound_packets
{
return Err(std::io::Error::new(
ErrorKind::WouldBlock,
"CultNet RUDP outbound packet queue is full",
)
.into());
}
let options = channel_send_options(channel_id, now_ms());
let packets =
self.session
.send_many(channel_id, payload, options, self.max_fragment_bytes)?;
for packet in packets {
self.send_packet(&packet)?;
}
self.pending_outbound_packets.extend(packets);
self.flush_pending_outbound_packets()?;
self.stats.frames_sent += 1;
Ok(())
}
@ -1079,9 +1134,38 @@ impl CultNetRudpSocketTransportConnection {
}
pub fn poll_resends(&mut self) -> Result<()> {
for packet in self.session.due_resends(now_ms()) {
self.send_packet(&packet)?;
self.flush_pending_outbound_packets()?;
if !self.pending_outbound_packets.is_empty() {
return Ok(());
}
for packet in self.session.due_resends(now_ms()) {
if self.pending_outbound_packets.len() >= self.max_pending_outbound_packets {
break;
}
self.pending_outbound_packets.push_back(packet);
}
self.flush_pending_outbound_packets()?;
Ok(())
}
fn outbound_packet_count(&self, payload_bytes: usize) -> Result<usize> {
match self.max_fragment_bytes {
Some(0) => Err(anyhow!("RUDP max_fragment_bytes must be greater than zero")),
Some(max_fragment_bytes) if payload_bytes > max_fragment_bytes => {
Ok(payload_bytes.div_ceil(max_fragment_bytes))
}
_ => Ok(1),
}
}
fn flush_pending_outbound_packets(&mut self) -> Result<()> {
let Some(remote_addr) = self.remote_addr else {
return Err(anyhow!("RUDP socket transport does not have a remote endpoint"));
};
let sent_bytes = flush_rudp_outbound_queue(&mut self.pending_outbound_packets, |wire| {
self.socket.send_to(wire, remote_addr)
})?;
self.stats.bytes_sent += sent_bytes as u64;
Ok(())
}
@ -1098,6 +1182,33 @@ impl CultNetRudpSocketTransportConnection {
}
}
fn flush_rudp_outbound_queue<F>(
pending: &mut VecDeque<CultNetRudpPacket>,
mut send: F,
) -> Result<usize>
where
F: FnMut(&[u8]) -> std::io::Result<usize>,
{
let mut sent_bytes = 0_usize;
while let Some(packet) = pending.front() {
let wire = encode_rudp_packet(packet)?;
match send(&wire) {
Ok(sent) => {
sent_bytes = sent_bytes.saturating_add(sent);
pending.pop_front();
}
Err(error)
if error.kind() == ErrorKind::WouldBlock
|| error.kind() == ErrorKind::TimedOut =>
{
break;
}
Err(error) => return Err(error.into()),
}
}
Ok(sent_bytes)
}
pub struct CultNetRudpReconnectLoop<F>
where
F: FnMut() -> Result<CultNetRudpSocketTransportConnection>,
@ -1409,3 +1520,116 @@ fn packet_type_from_code(code: u8) -> Result<CultNetRudpPacketType> {
_ => Err(anyhow!("Unsupported CultNet RUDP packet type {code}")),
}
}
#[cfg(test)]
mod socket_transport_tests {
use super::*;
fn data_packet(sequence: u32) -> CultNetRudpPacket {
CultNetRudpPacket {
packet_type: CultNetRudpPacketType::Data,
connection_id: 7,
sequence,
ack: 0,
ack_mask: 0,
channel_id: "realtime".to_string(),
reliable: false,
ordered: false,
sequenced: false,
fragment_id: 0,
fragment_index: 0,
fragment_count: 0,
payload: vec![sequence as u8],
}
}
#[test]
fn fragment_ids_wrap_to_one_instead_of_sticking_at_u16_max() {
let mut session = CultNetRudpSession::new(CultNetRudpSessionOptions::default());
session.next_fragment_id = u16::MAX;
assert_eq!(session.allocate_fragment_id(), u16::MAX);
assert_eq!(session.allocate_fragment_id(), 1);
assert_eq!(session.allocate_fragment_id(), 2);
}
#[test]
fn outbound_queue_preserves_allocated_packets_across_would_block() -> Result<()> {
let mut pending = VecDeque::from([data_packet(41), data_packet(42)]);
let mut attempts = 0_usize;
let sent = flush_rudp_outbound_queue(&mut pending, |_| {
attempts += 1;
Err(std::io::Error::from(ErrorKind::WouldBlock))
})?;
assert_eq!(sent, 0);
assert_eq!(attempts, 1);
assert_eq!(pending.iter().map(|packet| packet.sequence).collect::<Vec<_>>(), vec![41, 42]);
let mut sent_sequences = Vec::new();
let sent = flush_rudp_outbound_queue(&mut pending, |wire| {
sent_sequences.push(decode_rudp_packet(wire).unwrap().sequence);
Ok(wire.len())
})?;
assert!(sent > 0);
assert_eq!(sent_sequences, vec![41, 42]);
assert!(pending.is_empty());
Ok(())
}
#[test]
fn exact_ack_releases_one_reliable_packet() -> Result<()> {
let mut session = CultNetRudpSession::new(CultNetRudpSessionOptions {
connection_id: 7,
..CultNetRudpSessionOptions::default()
});
let connect = session.create_connect(1, Vec::new())?;
session.receive(
&CultNetRudpPacket {
packet_type: CultNetRudpPacketType::Accept,
connection_id: 7,
sequence: 1,
ack: connect.sequence,
ack_mask: 0,
channel_id: "control".to_string(),
reliable: true,
ordered: true,
sequenced: false,
fragment_id: 0,
fragment_index: 0,
fragment_count: 0,
payload: Vec::new(),
},
2,
)?;
let packet = session.send(
"media",
vec![1, 2, 3],
CultNetRudpSendOptions {
reliable: true,
now_ms: 10,
..CultNetRudpSendOptions::default()
},
)?;
assert_eq!(session.pending_reliable_sequences(), vec![packet.sequence]);
session.receive(
&CultNetRudpPacket {
packet_type: CultNetRudpPacketType::Ack,
connection_id: 7,
sequence: 99,
ack: packet.sequence,
ack_mask: 0,
channel_id: "control".to_string(),
reliable: false,
ordered: false,
sequenced: false,
fragment_id: 0,
fragment_index: 0,
fragment_count: 0,
payload: Vec::new(),
},
11,
)?;
assert!(session.pending_reliable_sequences().is_empty());
Ok(())
}
}