Recover bounded Muninn audio loss

This commit is contained in:
Meta Vens 2026-07-16 21:02:01 +01:00
commit 58e5556322
6 changed files with 378 additions and 9 deletions

View file

@ -38,3 +38,10 @@ target_compile_features(muninn_media_wire_tests PRIVATE cxx_std_20)
target_include_directories(muninn_media_wire_tests PRIVATE
${CMAKE_CURRENT_SOURCE_DIR})
add_test(NAME muninn_media_wire_tests COMMAND muninn_media_wire_tests)
add_executable(muninn_audio_fec_tests
tests/muninn_audio_fec_tests.cpp)
target_compile_features(muninn_audio_fec_tests PRIVATE cxx_std_20)
target_include_directories(muninn_audio_fec_tests PRIVATE
${CMAKE_CURRENT_SOURCE_DIR})
add_test(NAME muninn_audio_fec_tests COMMAND muninn_audio_fec_tests)

View file

@ -8,6 +8,7 @@
#include <util/platform.h>
#include "muninn_media_wire.h"
#include "muninn_audio_fec.h"
#include "muninn_rudp_ack.h"
#include "muninn_rudp_fragments.h"
#include "muninn_rudp_url.h"
@ -70,11 +71,11 @@ constexpr uint32_t MuninnCatalogDiscoveryConnectionId = 0x6d750003;
constexpr uint32_t MuninnCommandRudpConnectionId = 0x6d750002;
constexpr uint32_t MuninnDefaultMediaPacketBytes = 848;
constexpr uint32_t MuninnDefaultRudpVideoBitrateKbps = 12000;
constexpr uint32_t MuninnDefaultRudpLatencyBudgetMs = 2000;
constexpr uint32_t MuninnDefaultRudpLatencyBudgetMs = 100;
constexpr uint32_t MuninnDefaultRudpAudioReorderMs = muninn_rudp_url::DefaultAudioReorderMs;
constexpr uint32_t MuninnMaxRudpVideoBitrateKbps = 100000;
constexpr uint32_t MuninnMaxRudpLatencyBudgetMs = 2000;
constexpr uint32_t MuninnMaxRudpAudioReorderMs = 2000;
constexpr uint32_t MuninnMaxRudpLatencyBudgetMs = 500;
constexpr uint32_t MuninnMaxRudpAudioReorderMs = 250;
constexpr size_t MuninnAudioReorderMaxPackets = 128;
constexpr uint64_t MuninnAudioConcealmentMaxPackets = 4;
constexpr size_t MuninnAudioQueueMaxPackets = 32;
@ -200,7 +201,7 @@ static std::optional<uint16_t> allocate_udp_loopback_port()
}
struct MuninnMediaPayload {
enum class Kind { Unknown, Video, VideoParity, Audio };
enum class Kind { Unknown, Video, VideoParity, Audio, AudioParity };
Kind kind = Kind::Unknown;
std::string stream_id;
std::string session_id;
@ -211,6 +212,11 @@ struct MuninnMediaPayload {
uint32_t audio_timebase_num = 0;
uint32_t audio_timebase_den = 0;
uint64_t audio_deadline_ticks = 0;
uint64_t audio_base_packet_id = 0;
uint16_t audio_data_shard_count = 0;
uint16_t audio_parity_index = 0;
uint16_t audio_parity_shard_count = 0;
uint32_t audio_shard_payload_bytes = 0;
bool keyframe = false;
std::optional<uint64_t> dependency_frame_id;
uint16_t chunk_index = 0;
@ -699,6 +705,35 @@ static std::optional<MuninnMediaPayload> decode_muninn_media_payload(const uint8
media.payload = record.read_bytes();
return media;
}
if (schema_id == "muninn.media_audio_parity_shard.v1") {
if (record_len < 14) {
return std::nullopt;
}
MuninnMediaPayload media;
media.kind = MuninnMediaPayload::Kind::AudioParity;
media.stream_id = record.read_string();
media.session_id = record.read_string();
media.audio_base_packet_id = record.read_unsigned();
record.skip(); // codec
record.skip(); // base_pts_ticks
record.skip(); // packet_duration_ticks
record.skip(); // timebase_num
record.skip(); // timebase_den
media.audio_deadline_ticks = record.read_unsigned();
media.audio_data_shard_count = static_cast<uint16_t>(record.read_unsigned());
media.audio_parity_index = static_cast<uint16_t>(record.read_unsigned());
media.audio_parity_shard_count = static_cast<uint16_t>(record.read_unsigned());
media.audio_shard_payload_bytes = static_cast<uint32_t>(record.read_unsigned());
media.payload = record.read_bytes();
if (media.audio_data_shard_count != muninn_audio_fec::DataShards ||
media.audio_parity_shard_count != muninn_audio_fec::ParityShards ||
media.audio_parity_index >= muninn_audio_fec::ParityShards ||
media.audio_shard_payload_bytes == 0 ||
media.payload.size() != media.audio_shard_payload_bytes) {
return std::nullopt;
}
return media;
}
} catch (const std::exception &error) {
blog(LOG_WARNING, "Muninn RUDP bridge could not decode typed media payload: %s", error.what());
}
@ -1149,6 +1184,12 @@ private:
return;
}
forwarded_bytes = *audio_forwarded_bytes;
} else if (media->kind == MuninnMediaPayload::Kind::AudioParity) {
const auto audio_forwarded_bytes = forward_audio_parity(*media, audio_socket, audio_addr);
if (!audio_forwarded_bytes) {
return;
}
forwarded_bytes = *audio_forwarded_bytes;
} else {
++payloads_decoded_dropped_;
log_bridge_pressure("dropped unknown media payload");
@ -1194,6 +1235,11 @@ private:
audio_session_id_ = media.session_id;
}
audio_fec_data_cache_[media.audio_packet_id] = MuninnPendingAudioPacket{media.payload, now};
while (audio_fec_data_cache_.size() > MuninnAudioFecDataCacheMaxPackets) {
audio_fec_data_cache_.erase(audio_fec_data_cache_.begin());
}
if (!next_audio_packet_ready_) {
next_audio_packet_ready_ = true;
next_audio_packet_id_ = media.audio_packet_id;
@ -1217,6 +1263,58 @@ private:
return flush_audio_packets(now, audio_socket, audio_addr);
}
std::optional<size_t> forward_audio_parity(const MuninnMediaPayload &media,
SOCKET audio_socket,
const sockaddr_in &audio_addr)
{
if (media.stream_id.empty() || media.session_id.empty() || media.payload.empty()) {
++payloads_decoded_dropped_;
return std::nullopt;
}
const auto now = std::chrono::steady_clock::now();
if (audio_stream_id_ != media.stream_id || audio_session_id_ != media.session_id) {
reset_audio_reorder();
audio_stream_id_ = media.stream_id;
audio_session_id_ = media.session_id;
}
const muninn_audio_fec::BlockKey key{
media.stream_id, media.session_id, media.audio_base_packet_id};
std::vector<muninn_audio_fec::RecoveredPacket> recovered;
for (size_t index = 0; index < muninn_audio_fec::DataShards; ++index) {
const auto packet = audio_fec_data_cache_.find(media.audio_base_packet_id + index);
if (packet != audio_fec_data_cache_.end()) {
auto newly_recovered = audio_fec_receiver_.insert(
key, index, packet->second.payload, now);
recovered.insert(recovered.end(),
std::make_move_iterator(newly_recovered.begin()),
std::make_move_iterator(newly_recovered.end()));
}
}
auto parity_recovered = audio_fec_receiver_.insert(
key,
muninn_audio_fec::DataShards + media.audio_parity_index,
media.payload,
now);
recovered.insert(recovered.end(),
std::make_move_iterator(parity_recovered.begin()),
std::make_move_iterator(parity_recovered.end()));
for (auto &packet : recovered) {
if (!next_audio_packet_ready_) {
next_audio_packet_ready_ = true;
next_audio_packet_id_ = packet.packet_id;
}
if (packet.packet_id >= next_audio_packet_id_ &&
pending_audio_packets_.emplace(
packet.packet_id,
MuninnPendingAudioPacket{std::move(packet.payload), now}).second) {
++audio_packets_fec_recovered_;
}
}
return flush_audio_packets(now, audio_socket, audio_addr);
}
std::optional<size_t> flush_audio_packets(std::chrono::steady_clock::time_point now,
SOCKET audio_socket,
const sockaddr_in &audio_addr)
@ -1302,6 +1400,8 @@ private:
audio_stream_id_.clear();
audio_session_id_.clear();
pending_audio_packets_.clear();
audio_fec_data_cache_.clear();
audio_fec_receiver_.clear();
next_audio_packet_ready_ = false;
next_audio_packet_id_ = 0;
audio_gap_started_at_ = std::chrono::steady_clock::time_point::min();
@ -1942,12 +2042,16 @@ private:
uint64_t audio_packets_skipped_ = 0;
uint64_t audio_packets_stale_ = 0;
uint64_t audio_packets_concealed_ = 0;
uint64_t audio_packets_fec_recovered_ = 0;
std::string audio_stream_id_;
std::string audio_session_id_;
bool next_audio_packet_ready_ = false;
uint64_t next_audio_packet_id_ = 0;
std::chrono::steady_clock::time_point audio_gap_started_at_ = std::chrono::steady_clock::time_point::min();
std::map<uint64_t, MuninnPendingAudioPacket> pending_audio_packets_;
static constexpr size_t MuninnAudioFecDataCacheMaxPackets = 512;
std::map<uint64_t, MuninnPendingAudioPacket> audio_fec_data_cache_;
muninn_audio_fec::BlockReceiver audio_fec_receiver_{std::chrono::milliseconds(40), 128};
bool media_sequence_initialized_ = false;
uint32_t last_media_sequence_ = 0;
uint64_t media_sequence_gap_events_ = 0;

View file

@ -0,0 +1,179 @@
#pragma once
#include <array>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <iterator>
#include <map>
#include <optional>
#include <string>
#include <tuple>
#include <utility>
#include <vector>
namespace muninn_audio_fec {
inline constexpr size_t DataShards = 4;
inline constexpr size_t ParityShards = 2;
inline constexpr size_t TotalShards = DataShards + ParityShards;
using Bytes = std::vector<uint8_t>;
using Shards = std::array<std::optional<Bytes>, TotalShards>;
inline uint8_t multiply(uint8_t left, uint8_t right)
{
uint16_t product = 0;
uint16_t a = left;
uint16_t b = right;
while (b != 0) {
if ((b & 1u) != 0) product ^= a;
b >>= 1u;
a <<= 1u;
if ((a & 0x100u) != 0) a ^= 0x11du;
}
return static_cast<uint8_t>(product);
}
inline uint8_t power(uint8_t value, uint16_t exponent)
{
uint8_t result = 1;
while (exponent != 0) {
if ((exponent & 1u) != 0) result = multiply(result, value);
value = multiply(value, value);
exponent >>= 1u;
}
return result;
}
inline uint8_t inverse(uint8_t value)
{
return value == 0 ? 0 : power(value, 254);
}
inline constexpr std::array<std::array<uint8_t, DataShards>, TotalShards> Generator{{
{{1, 0, 0, 0}}, {{0, 1, 0, 0}}, {{0, 0, 1, 0}}, {{0, 0, 0, 1}},
{{1, 1, 1, 1}}, {{1, 2, 4, 8}}
}};
inline std::array<Bytes, ParityShards> encode(const std::array<Bytes, DataShards> &data)
{
const size_t bytes = data[0].size();
for (const auto &shard : data) if (shard.size() != bytes) return {};
std::array<Bytes, ParityShards> parity{Bytes(bytes), Bytes(bytes)};
for (size_t p = 0; p < ParityShards; ++p)
for (size_t d = 0; d < DataShards; ++d)
for (size_t i = 0; i < bytes; ++i)
parity[p][i] ^= multiply(Generator[DataShards + p][d], data[d][i]);
return parity;
}
inline bool recover(Shards &shards)
{
size_t shard_bytes = 0;
std::array<size_t, DataShards> available{};
size_t count = 0;
for (size_t i = 0; i < TotalShards; ++i) {
if (!shards[i]) continue;
if (shard_bytes == 0) shard_bytes = shards[i]->size();
if (shards[i]->size() != shard_bytes) return false;
if (count < DataShards) available[count++] = i;
}
if (count < DataShards || shard_bytes == 0) return false;
std::array<std::array<uint8_t, DataShards * 2>, DataShards> matrix{};
for (size_t row = 0; row < DataShards; ++row) {
for (size_t col = 0; col < DataShards; ++col) matrix[row][col] = Generator[available[row]][col];
matrix[row][DataShards + row] = 1;
}
for (size_t col = 0; col < DataShards; ++col) {
size_t pivot = col;
while (pivot < DataShards && matrix[pivot][col] == 0) ++pivot;
if (pivot == DataShards) return false;
if (pivot != col) std::swap(matrix[pivot], matrix[col]);
const uint8_t scale = inverse(matrix[col][col]);
for (size_t j = 0; j < DataShards * 2; ++j) matrix[col][j] = multiply(matrix[col][j], scale);
for (size_t row = 0; row < DataShards; ++row) {
if (row == col) continue;
const uint8_t factor = matrix[row][col];
for (size_t j = 0; j < DataShards * 2; ++j)
matrix[row][j] ^= multiply(factor, matrix[col][j]);
}
}
for (size_t missing = 0; missing < DataShards; ++missing) {
if (shards[missing]) continue;
Bytes decoded(shard_bytes);
for (size_t row = 0; row < DataShards; ++row) {
const uint8_t coefficient = matrix[missing][DataShards + row];
for (size_t i = 0; i < shard_bytes; ++i)
decoded[i] ^= multiply(coefficient, (*shards[available[row]])[i]);
}
shards[missing] = std::move(decoded);
}
return true;
}
struct BlockKey {
std::string stream_id;
std::string session_id;
uint64_t base_packet_id = 0;
bool operator<(const BlockKey &other) const
{
return std::tie(stream_id, session_id, base_packet_id) <
std::tie(other.stream_id, other.session_id, other.base_packet_id);
}
};
struct RecoveredPacket { uint64_t packet_id; Bytes payload; };
class BlockReceiver {
public:
using Clock = std::chrono::steady_clock;
explicit BlockReceiver(std::chrono::milliseconds deadline, size_t max_blocks = 128)
: deadline_(deadline), max_blocks_(max_blocks) {}
std::vector<RecoveredPacket> insert(const BlockKey &key, size_t shard_index, Bytes payload,
Clock::time_point now)
{
expire(now);
if (shard_index >= TotalShards || payload.empty() || completed_.count(key) != 0) return {};
if (blocks_.size() >= max_blocks_ && blocks_.count(key) == 0) blocks_.erase(blocks_.begin());
auto [it, created] = blocks_.try_emplace(key, Block{now, payload.size(), {}});
Block &block = it->second;
if (payload.size() != block.shard_bytes || block.shards[shard_index]) return {};
block.shards[shard_index] = std::move(payload);
size_t present = 0;
for (const auto &shard : block.shards) if (shard) ++present;
if (present < DataShards) return {};
Shards before = block.shards;
if (!recover(block.shards)) return {};
std::vector<RecoveredPacket> result;
for (size_t i = 0; i < DataShards; ++i)
if (!before[i] && block.shards[i]) result.push_back({key.base_packet_id + i, *block.shards[i]});
blocks_.erase(it);
completed_[key] = now;
return result;
}
void expire(Clock::time_point now)
{
for (auto it = blocks_.begin(); it != blocks_.end();)
it = now - it->second.created_at >= deadline_ ? blocks_.erase(it) : std::next(it);
for (auto it = completed_.begin(); it != completed_.end();)
it = now - it->second >= deadline_ ? completed_.erase(it) : std::next(it);
}
size_t pending_blocks() const { return blocks_.size(); }
void clear()
{
blocks_.clear();
completed_.clear();
}
private:
struct Block { Clock::time_point created_at; size_t shard_bytes; Shards shards; };
std::chrono::milliseconds deadline_;
size_t max_blocks_;
std::map<BlockKey, Block> blocks_;
std::map<BlockKey, Clock::time_point> completed_;
};
} // namespace muninn_audio_fec

View file

@ -15,7 +15,7 @@ inline constexpr uint32_t DefaultVideoAssemblyDeadlineMs = 75;
inline constexpr uint32_t DefaultGapWaitMs = 8;
inline constexpr uint32_t DefaultAudioReorderMs = 40;
inline constexpr uint32_t DefaultSenderResendDelayMs = 5;
inline constexpr uint32_t MaxLatencyBudgetMs = 2000;
inline constexpr uint32_t MaxLatencyBudgetMs = 500;
struct RudpUrlParts {
uint16_t port = 0;

View file

@ -0,0 +1,79 @@
#include "muninn_audio_fec.h"
#include <cstdlib>
#include <iostream>
#include <stdexcept>
static void require(bool condition, const char *message) { if (!condition) throw std::runtime_error(message); }
static std::array<muninn_audio_fec::Bytes, 4> fixture()
{
std::array<muninn_audio_fec::Bytes, 4> data;
for (size_t shard = 0; shard < data.size(); ++shard) {
data[shard].resize(257);
for (size_t i = 0; i < data[shard].size(); ++i) data[shard][i] = uint8_t((shard * 71 + i * 29) & 0xff);
}
return data;
}
static muninn_audio_fec::Shards all_shards(const std::array<muninn_audio_fec::Bytes, 4> &data)
{
auto parity = muninn_audio_fec::encode(data);
return {data[0], data[1], data[2], data[3], parity[0], parity[1]};
}
int main()
{
try {
const auto data = fixture();
for (size_t erased = 0; erased < 6; ++erased) {
auto shards = all_shards(data); shards[erased].reset();
require(muninn_audio_fec::recover(shards), "one-erasure recovery failed");
for (size_t i = 0; i < 4; ++i) require(shards[i] == data[i], "one-erasure data mismatch");
}
for (size_t first = 0; first < 6; ++first) for (size_t second = first + 1; second < 6; ++second) {
auto shards = all_shards(data); shards[first].reset(); shards[second].reset();
require(muninn_audio_fec::recover(shards), "two-erasure recovery failed");
for (size_t i = 0; i < 4; ++i) require(shards[i] == data[i], "two-erasure data mismatch");
}
{
auto shards = all_shards(data);
shards[0].reset(); shards[1].reset(); shards[2].reset();
require(!muninn_audio_fec::recover(shards), "three erasures were incorrectly recoverable");
}
{
auto shards = all_shards(data);
shards[5]->pop_back();
require(!muninn_audio_fec::recover(shards), "unequal shard sizes were accepted");
}
using namespace muninn_audio_fec;
BlockReceiver receiver(std::chrono::milliseconds(10), 2);
BlockKey key{"stream", "session", 100};
auto shards = all_shards(data);
auto t0 = BlockReceiver::Clock::time_point{};
require(receiver.insert(key, 5, *shards[5], t0).empty(), "early parity emitted data");
require(receiver.insert(key, 2, *shards[2], t0).empty(), "partial reorder emitted data");
require(receiver.insert(key, 2, *shards[2], t0).empty(), "duplicate emitted data");
require(receiver.insert(key, 1, *shards[1], t0).empty(), "partial block emitted data");
auto recovered = receiver.insert(key, 4, *shards[4], t0);
require(recovered.size() == 2 && recovered[0].packet_id == 100 && recovered[1].packet_id == 103,
"reordered block did not recover missing packet ids");
require(recovered[0].payload == data[0] && recovered[1].payload == data[3], "receiver recovery mismatch");
require(receiver.insert(key, 0, data[0], t0).empty(), "completed-block duplicate was admitted");
BlockKey expired{"stream", "session", 200};
receiver.insert(expired, 0, data[0], t0);
receiver.expire(t0 + std::chrono::milliseconds(10));
require(receiver.pending_blocks() == 0, "expired block survived deadline");
BlockReceiver bounded(std::chrono::seconds(1), 2);
bounded.insert({"s", "x", 0}, 0, data[0], t0);
bounded.insert({"s", "x", 4}, 0, data[0], t0);
bounded.insert({"s", "x", 8}, 0, data[0], t0);
require(bounded.pending_blocks() == 2, "block bound was not enforced");
} catch (const std::exception &error) {
std::cerr << error.what() << '\n'; return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}

View file

@ -19,17 +19,17 @@ void require(bool condition, const std::string &message)
int main()
{
const auto configured = muninn_rudp_url::parse(
"rudp://192.168.1.66:5204/muninn.raven.av.rudp?channel=media&format=muninn-typed-media&connection=0x6d750001&audio_connection=0x6d750004&sender_resend_delay_ms=5&reliable_expire_after_ms=1200&assembly_deadline_ms=1200&gap_wait_ms=16&audio_reorder_ms=700");
"rudp://192.168.1.66:5204/muninn.raven.av.rudp?channel=media&format=muninn-typed-media&connection=0x6d750001&audio_connection=0x6d750004&sender_resend_delay_ms=5&reliable_expire_after_ms=120&assembly_deadline_ms=120&gap_wait_ms=16&audio_reorder_ms=80");
require(configured.has_value(), "configured RUDP URL did not parse");
require(configured->port == 5204, "configured RUDP URL port was not parsed");
require(configured->video_local_port == 15204, "configured RUDP video local port was not derived");
require(configured->audio_local_port == 15205, "configured RUDP audio local port was not derived");
require(configured->connection_id == 0x6d750001, "configured RUDP connection id was not parsed");
require(configured->audio_connection_id == 0x6d750004, "configured RUDP audio connection id was not parsed");
require(configured->reliable_expire_after_ms == 1200, "configured reliable expiry did not survive parse");
require(configured->video_assembly_deadline_ms == 1200, "configured assembly deadline did not survive parse");
require(configured->reliable_expire_after_ms == 120, "configured reliable expiry did not survive parse");
require(configured->video_assembly_deadline_ms == 120, "configured assembly deadline did not survive parse");
require(configured->gap_wait_ms == 16, "configured gap wait did not survive parse");
require(configured->audio_reorder_ms == 700, "configured audio reorder wait did not survive parse");
require(configured->audio_reorder_ms == 80, "configured audio reorder wait did not survive parse");
const auto clamped = muninn_rudp_url::parse(
"rudp://192.168.1.66:5204/muninn.raven.av.rudp?reliable_expire_after_ms=9000&assembly_deadline_ms=9000");