diff --git a/native/obs_stem_source/CMakeLists.txt b/native/obs_stem_source/CMakeLists.txt index c4f0444..090a2ff 100644 --- a/native/obs_stem_source/CMakeLists.txt +++ b/native/obs_stem_source/CMakeLists.txt @@ -38,3 +38,10 @@ target_compile_features(muninn_media_wire_tests PRIVATE cxx_std_20) target_include_directories(muninn_media_wire_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) add_test(NAME muninn_media_wire_tests COMMAND muninn_media_wire_tests) + +add_executable(muninn_audio_fec_tests + tests/muninn_audio_fec_tests.cpp) +target_compile_features(muninn_audio_fec_tests PRIVATE cxx_std_20) +target_include_directories(muninn_audio_fec_tests PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}) +add_test(NAME muninn_audio_fec_tests COMMAND muninn_audio_fec_tests) diff --git a/native/obs_stem_source/mimir_obs_muninn_source.cpp b/native/obs_stem_source/mimir_obs_muninn_source.cpp index 864d4c0..3600db3 100644 --- a/native/obs_stem_source/mimir_obs_muninn_source.cpp +++ b/native/obs_stem_source/mimir_obs_muninn_source.cpp @@ -8,6 +8,7 @@ #include #include "muninn_media_wire.h" +#include "muninn_audio_fec.h" #include "muninn_rudp_ack.h" #include "muninn_rudp_fragments.h" #include "muninn_rudp_url.h" @@ -70,11 +71,11 @@ constexpr uint32_t MuninnCatalogDiscoveryConnectionId = 0x6d750003; constexpr uint32_t MuninnCommandRudpConnectionId = 0x6d750002; constexpr uint32_t MuninnDefaultMediaPacketBytes = 848; constexpr uint32_t MuninnDefaultRudpVideoBitrateKbps = 12000; -constexpr uint32_t MuninnDefaultRudpLatencyBudgetMs = 2000; +constexpr uint32_t MuninnDefaultRudpLatencyBudgetMs = 100; constexpr uint32_t MuninnDefaultRudpAudioReorderMs = muninn_rudp_url::DefaultAudioReorderMs; constexpr uint32_t MuninnMaxRudpVideoBitrateKbps = 100000; -constexpr uint32_t MuninnMaxRudpLatencyBudgetMs = 2000; -constexpr uint32_t MuninnMaxRudpAudioReorderMs = 2000; +constexpr uint32_t MuninnMaxRudpLatencyBudgetMs = 500; +constexpr uint32_t MuninnMaxRudpAudioReorderMs = 250; constexpr size_t MuninnAudioReorderMaxPackets = 128; constexpr uint64_t MuninnAudioConcealmentMaxPackets = 4; constexpr size_t MuninnAudioQueueMaxPackets = 32; @@ -200,7 +201,7 @@ static std::optional allocate_udp_loopback_port() } struct MuninnMediaPayload { - enum class Kind { Unknown, Video, VideoParity, Audio }; + enum class Kind { Unknown, Video, VideoParity, Audio, AudioParity }; Kind kind = Kind::Unknown; std::string stream_id; std::string session_id; @@ -211,6 +212,11 @@ struct MuninnMediaPayload { uint32_t audio_timebase_num = 0; uint32_t audio_timebase_den = 0; uint64_t audio_deadline_ticks = 0; + uint64_t audio_base_packet_id = 0; + uint16_t audio_data_shard_count = 0; + uint16_t audio_parity_index = 0; + uint16_t audio_parity_shard_count = 0; + uint32_t audio_shard_payload_bytes = 0; bool keyframe = false; std::optional dependency_frame_id; uint16_t chunk_index = 0; @@ -699,6 +705,35 @@ static std::optional decode_muninn_media_payload(const uint8 media.payload = record.read_bytes(); return media; } + if (schema_id == "muninn.media_audio_parity_shard.v1") { + if (record_len < 14) { + return std::nullopt; + } + MuninnMediaPayload media; + media.kind = MuninnMediaPayload::Kind::AudioParity; + media.stream_id = record.read_string(); + media.session_id = record.read_string(); + media.audio_base_packet_id = record.read_unsigned(); + record.skip(); // codec + record.skip(); // base_pts_ticks + record.skip(); // packet_duration_ticks + record.skip(); // timebase_num + record.skip(); // timebase_den + media.audio_deadline_ticks = record.read_unsigned(); + media.audio_data_shard_count = static_cast(record.read_unsigned()); + media.audio_parity_index = static_cast(record.read_unsigned()); + media.audio_parity_shard_count = static_cast(record.read_unsigned()); + media.audio_shard_payload_bytes = static_cast(record.read_unsigned()); + media.payload = record.read_bytes(); + if (media.audio_data_shard_count != muninn_audio_fec::DataShards || + media.audio_parity_shard_count != muninn_audio_fec::ParityShards || + media.audio_parity_index >= muninn_audio_fec::ParityShards || + media.audio_shard_payload_bytes == 0 || + media.payload.size() != media.audio_shard_payload_bytes) { + return std::nullopt; + } + return media; + } } catch (const std::exception &error) { blog(LOG_WARNING, "Muninn RUDP bridge could not decode typed media payload: %s", error.what()); } @@ -1149,6 +1184,12 @@ private: return; } forwarded_bytes = *audio_forwarded_bytes; + } else if (media->kind == MuninnMediaPayload::Kind::AudioParity) { + const auto audio_forwarded_bytes = forward_audio_parity(*media, audio_socket, audio_addr); + if (!audio_forwarded_bytes) { + return; + } + forwarded_bytes = *audio_forwarded_bytes; } else { ++payloads_decoded_dropped_; log_bridge_pressure("dropped unknown media payload"); @@ -1194,6 +1235,11 @@ private: audio_session_id_ = media.session_id; } + audio_fec_data_cache_[media.audio_packet_id] = MuninnPendingAudioPacket{media.payload, now}; + while (audio_fec_data_cache_.size() > MuninnAudioFecDataCacheMaxPackets) { + audio_fec_data_cache_.erase(audio_fec_data_cache_.begin()); + } + if (!next_audio_packet_ready_) { next_audio_packet_ready_ = true; next_audio_packet_id_ = media.audio_packet_id; @@ -1217,6 +1263,58 @@ private: return flush_audio_packets(now, audio_socket, audio_addr); } + std::optional forward_audio_parity(const MuninnMediaPayload &media, + SOCKET audio_socket, + const sockaddr_in &audio_addr) + { + if (media.stream_id.empty() || media.session_id.empty() || media.payload.empty()) { + ++payloads_decoded_dropped_; + return std::nullopt; + } + const auto now = std::chrono::steady_clock::now(); + if (audio_stream_id_ != media.stream_id || audio_session_id_ != media.session_id) { + reset_audio_reorder(); + audio_stream_id_ = media.stream_id; + audio_session_id_ = media.session_id; + } + + const muninn_audio_fec::BlockKey key{ + media.stream_id, media.session_id, media.audio_base_packet_id}; + std::vector recovered; + for (size_t index = 0; index < muninn_audio_fec::DataShards; ++index) { + const auto packet = audio_fec_data_cache_.find(media.audio_base_packet_id + index); + if (packet != audio_fec_data_cache_.end()) { + auto newly_recovered = audio_fec_receiver_.insert( + key, index, packet->second.payload, now); + recovered.insert(recovered.end(), + std::make_move_iterator(newly_recovered.begin()), + std::make_move_iterator(newly_recovered.end())); + } + } + auto parity_recovered = audio_fec_receiver_.insert( + key, + muninn_audio_fec::DataShards + media.audio_parity_index, + media.payload, + now); + recovered.insert(recovered.end(), + std::make_move_iterator(parity_recovered.begin()), + std::make_move_iterator(parity_recovered.end())); + + for (auto &packet : recovered) { + if (!next_audio_packet_ready_) { + next_audio_packet_ready_ = true; + next_audio_packet_id_ = packet.packet_id; + } + if (packet.packet_id >= next_audio_packet_id_ && + pending_audio_packets_.emplace( + packet.packet_id, + MuninnPendingAudioPacket{std::move(packet.payload), now}).second) { + ++audio_packets_fec_recovered_; + } + } + return flush_audio_packets(now, audio_socket, audio_addr); + } + std::optional flush_audio_packets(std::chrono::steady_clock::time_point now, SOCKET audio_socket, const sockaddr_in &audio_addr) @@ -1302,6 +1400,8 @@ private: audio_stream_id_.clear(); audio_session_id_.clear(); pending_audio_packets_.clear(); + audio_fec_data_cache_.clear(); + audio_fec_receiver_.clear(); next_audio_packet_ready_ = false; next_audio_packet_id_ = 0; audio_gap_started_at_ = std::chrono::steady_clock::time_point::min(); @@ -1942,12 +2042,16 @@ private: uint64_t audio_packets_skipped_ = 0; uint64_t audio_packets_stale_ = 0; uint64_t audio_packets_concealed_ = 0; + uint64_t audio_packets_fec_recovered_ = 0; std::string audio_stream_id_; std::string audio_session_id_; bool next_audio_packet_ready_ = false; uint64_t next_audio_packet_id_ = 0; std::chrono::steady_clock::time_point audio_gap_started_at_ = std::chrono::steady_clock::time_point::min(); std::map pending_audio_packets_; + static constexpr size_t MuninnAudioFecDataCacheMaxPackets = 512; + std::map audio_fec_data_cache_; + muninn_audio_fec::BlockReceiver audio_fec_receiver_{std::chrono::milliseconds(40), 128}; bool media_sequence_initialized_ = false; uint32_t last_media_sequence_ = 0; uint64_t media_sequence_gap_events_ = 0; diff --git a/native/obs_stem_source/muninn_audio_fec.h b/native/obs_stem_source/muninn_audio_fec.h new file mode 100644 index 0000000..e5ca7d9 --- /dev/null +++ b/native/obs_stem_source/muninn_audio_fec.h @@ -0,0 +1,179 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace muninn_audio_fec { + +inline constexpr size_t DataShards = 4; +inline constexpr size_t ParityShards = 2; +inline constexpr size_t TotalShards = DataShards + ParityShards; +using Bytes = std::vector; +using Shards = std::array, TotalShards>; + +inline uint8_t multiply(uint8_t left, uint8_t right) +{ + uint16_t product = 0; + uint16_t a = left; + uint16_t b = right; + while (b != 0) { + if ((b & 1u) != 0) product ^= a; + b >>= 1u; + a <<= 1u; + if ((a & 0x100u) != 0) a ^= 0x11du; + } + return static_cast(product); +} + +inline uint8_t power(uint8_t value, uint16_t exponent) +{ + uint8_t result = 1; + while (exponent != 0) { + if ((exponent & 1u) != 0) result = multiply(result, value); + value = multiply(value, value); + exponent >>= 1u; + } + return result; +} + +inline uint8_t inverse(uint8_t value) +{ + return value == 0 ? 0 : power(value, 254); +} + +inline constexpr std::array, TotalShards> Generator{{ + {{1, 0, 0, 0}}, {{0, 1, 0, 0}}, {{0, 0, 1, 0}}, {{0, 0, 0, 1}}, + {{1, 1, 1, 1}}, {{1, 2, 4, 8}} +}}; + +inline std::array encode(const std::array &data) +{ + const size_t bytes = data[0].size(); + for (const auto &shard : data) if (shard.size() != bytes) return {}; + std::array parity{Bytes(bytes), Bytes(bytes)}; + for (size_t p = 0; p < ParityShards; ++p) + for (size_t d = 0; d < DataShards; ++d) + for (size_t i = 0; i < bytes; ++i) + parity[p][i] ^= multiply(Generator[DataShards + p][d], data[d][i]); + return parity; +} + +inline bool recover(Shards &shards) +{ + size_t shard_bytes = 0; + std::array available{}; + size_t count = 0; + for (size_t i = 0; i < TotalShards; ++i) { + if (!shards[i]) continue; + if (shard_bytes == 0) shard_bytes = shards[i]->size(); + if (shards[i]->size() != shard_bytes) return false; + if (count < DataShards) available[count++] = i; + } + if (count < DataShards || shard_bytes == 0) return false; + + std::array, DataShards> matrix{}; + for (size_t row = 0; row < DataShards; ++row) { + for (size_t col = 0; col < DataShards; ++col) matrix[row][col] = Generator[available[row]][col]; + matrix[row][DataShards + row] = 1; + } + for (size_t col = 0; col < DataShards; ++col) { + size_t pivot = col; + while (pivot < DataShards && matrix[pivot][col] == 0) ++pivot; + if (pivot == DataShards) return false; + if (pivot != col) std::swap(matrix[pivot], matrix[col]); + const uint8_t scale = inverse(matrix[col][col]); + for (size_t j = 0; j < DataShards * 2; ++j) matrix[col][j] = multiply(matrix[col][j], scale); + for (size_t row = 0; row < DataShards; ++row) { + if (row == col) continue; + const uint8_t factor = matrix[row][col]; + for (size_t j = 0; j < DataShards * 2; ++j) + matrix[row][j] ^= multiply(factor, matrix[col][j]); + } + } + for (size_t missing = 0; missing < DataShards; ++missing) { + if (shards[missing]) continue; + Bytes decoded(shard_bytes); + for (size_t row = 0; row < DataShards; ++row) { + const uint8_t coefficient = matrix[missing][DataShards + row]; + for (size_t i = 0; i < shard_bytes; ++i) + decoded[i] ^= multiply(coefficient, (*shards[available[row]])[i]); + } + shards[missing] = std::move(decoded); + } + return true; +} + +struct BlockKey { + std::string stream_id; + std::string session_id; + uint64_t base_packet_id = 0; + bool operator<(const BlockKey &other) const + { + return std::tie(stream_id, session_id, base_packet_id) < + std::tie(other.stream_id, other.session_id, other.base_packet_id); + } +}; + +struct RecoveredPacket { uint64_t packet_id; Bytes payload; }; + +class BlockReceiver { +public: + using Clock = std::chrono::steady_clock; + explicit BlockReceiver(std::chrono::milliseconds deadline, size_t max_blocks = 128) + : deadline_(deadline), max_blocks_(max_blocks) {} + + std::vector insert(const BlockKey &key, size_t shard_index, Bytes payload, + Clock::time_point now) + { + expire(now); + if (shard_index >= TotalShards || payload.empty() || completed_.count(key) != 0) return {}; + if (blocks_.size() >= max_blocks_ && blocks_.count(key) == 0) blocks_.erase(blocks_.begin()); + auto [it, created] = blocks_.try_emplace(key, Block{now, payload.size(), {}}); + Block &block = it->second; + if (payload.size() != block.shard_bytes || block.shards[shard_index]) return {}; + block.shards[shard_index] = std::move(payload); + size_t present = 0; + for (const auto &shard : block.shards) if (shard) ++present; + if (present < DataShards) return {}; + Shards before = block.shards; + if (!recover(block.shards)) return {}; + std::vector result; + for (size_t i = 0; i < DataShards; ++i) + if (!before[i] && block.shards[i]) result.push_back({key.base_packet_id + i, *block.shards[i]}); + blocks_.erase(it); + completed_[key] = now; + return result; + } + + void expire(Clock::time_point now) + { + for (auto it = blocks_.begin(); it != blocks_.end();) + it = now - it->second.created_at >= deadline_ ? blocks_.erase(it) : std::next(it); + for (auto it = completed_.begin(); it != completed_.end();) + it = now - it->second >= deadline_ ? completed_.erase(it) : std::next(it); + } + size_t pending_blocks() const { return blocks_.size(); } + void clear() + { + blocks_.clear(); + completed_.clear(); + } + +private: + struct Block { Clock::time_point created_at; size_t shard_bytes; Shards shards; }; + std::chrono::milliseconds deadline_; + size_t max_blocks_; + std::map blocks_; + std::map completed_; +}; + +} // namespace muninn_audio_fec diff --git a/native/obs_stem_source/muninn_rudp_url.h b/native/obs_stem_source/muninn_rudp_url.h index a7f45c5..ffbfbd2 100644 --- a/native/obs_stem_source/muninn_rudp_url.h +++ b/native/obs_stem_source/muninn_rudp_url.h @@ -15,7 +15,7 @@ inline constexpr uint32_t DefaultVideoAssemblyDeadlineMs = 75; inline constexpr uint32_t DefaultGapWaitMs = 8; inline constexpr uint32_t DefaultAudioReorderMs = 40; inline constexpr uint32_t DefaultSenderResendDelayMs = 5; -inline constexpr uint32_t MaxLatencyBudgetMs = 2000; +inline constexpr uint32_t MaxLatencyBudgetMs = 500; struct RudpUrlParts { uint16_t port = 0; diff --git a/native/obs_stem_source/tests/muninn_audio_fec_tests.cpp b/native/obs_stem_source/tests/muninn_audio_fec_tests.cpp new file mode 100644 index 0000000..bbefab7 --- /dev/null +++ b/native/obs_stem_source/tests/muninn_audio_fec_tests.cpp @@ -0,0 +1,79 @@ +#include "muninn_audio_fec.h" + +#include +#include +#include + +static void require(bool condition, const char *message) { if (!condition) throw std::runtime_error(message); } + +static std::array fixture() +{ + std::array data; + for (size_t shard = 0; shard < data.size(); ++shard) { + data[shard].resize(257); + for (size_t i = 0; i < data[shard].size(); ++i) data[shard][i] = uint8_t((shard * 71 + i * 29) & 0xff); + } + return data; +} + +static muninn_audio_fec::Shards all_shards(const std::array &data) +{ + auto parity = muninn_audio_fec::encode(data); + return {data[0], data[1], data[2], data[3], parity[0], parity[1]}; +} + +int main() +{ + try { + const auto data = fixture(); + for (size_t erased = 0; erased < 6; ++erased) { + auto shards = all_shards(data); shards[erased].reset(); + require(muninn_audio_fec::recover(shards), "one-erasure recovery failed"); + for (size_t i = 0; i < 4; ++i) require(shards[i] == data[i], "one-erasure data mismatch"); + } + for (size_t first = 0; first < 6; ++first) for (size_t second = first + 1; second < 6; ++second) { + auto shards = all_shards(data); shards[first].reset(); shards[second].reset(); + require(muninn_audio_fec::recover(shards), "two-erasure recovery failed"); + for (size_t i = 0; i < 4; ++i) require(shards[i] == data[i], "two-erasure data mismatch"); + } + { + auto shards = all_shards(data); + shards[0].reset(); shards[1].reset(); shards[2].reset(); + require(!muninn_audio_fec::recover(shards), "three erasures were incorrectly recoverable"); + } + { + auto shards = all_shards(data); + shards[5]->pop_back(); + require(!muninn_audio_fec::recover(shards), "unequal shard sizes were accepted"); + } + + using namespace muninn_audio_fec; + BlockReceiver receiver(std::chrono::milliseconds(10), 2); + BlockKey key{"stream", "session", 100}; + auto shards = all_shards(data); + auto t0 = BlockReceiver::Clock::time_point{}; + require(receiver.insert(key, 5, *shards[5], t0).empty(), "early parity emitted data"); + require(receiver.insert(key, 2, *shards[2], t0).empty(), "partial reorder emitted data"); + require(receiver.insert(key, 2, *shards[2], t0).empty(), "duplicate emitted data"); + require(receiver.insert(key, 1, *shards[1], t0).empty(), "partial block emitted data"); + auto recovered = receiver.insert(key, 4, *shards[4], t0); + require(recovered.size() == 2 && recovered[0].packet_id == 100 && recovered[1].packet_id == 103, + "reordered block did not recover missing packet ids"); + require(recovered[0].payload == data[0] && recovered[1].payload == data[3], "receiver recovery mismatch"); + require(receiver.insert(key, 0, data[0], t0).empty(), "completed-block duplicate was admitted"); + + BlockKey expired{"stream", "session", 200}; + receiver.insert(expired, 0, data[0], t0); + receiver.expire(t0 + std::chrono::milliseconds(10)); + require(receiver.pending_blocks() == 0, "expired block survived deadline"); + + BlockReceiver bounded(std::chrono::seconds(1), 2); + bounded.insert({"s", "x", 0}, 0, data[0], t0); + bounded.insert({"s", "x", 4}, 0, data[0], t0); + bounded.insert({"s", "x", 8}, 0, data[0], t0); + require(bounded.pending_blocks() == 2, "block bound was not enforced"); + } catch (const std::exception &error) { + std::cerr << error.what() << '\n'; return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} diff --git a/native/obs_stem_source/tests/muninn_rudp_url_tests.cpp b/native/obs_stem_source/tests/muninn_rudp_url_tests.cpp index fd65938..2cdebd0 100644 --- a/native/obs_stem_source/tests/muninn_rudp_url_tests.cpp +++ b/native/obs_stem_source/tests/muninn_rudp_url_tests.cpp @@ -19,17 +19,17 @@ void require(bool condition, const std::string &message) int main() { const auto configured = muninn_rudp_url::parse( - "rudp://192.168.1.66:5204/muninn.raven.av.rudp?channel=media&format=muninn-typed-media&connection=0x6d750001&audio_connection=0x6d750004&sender_resend_delay_ms=5&reliable_expire_after_ms=1200&assembly_deadline_ms=1200&gap_wait_ms=16&audio_reorder_ms=700"); + "rudp://192.168.1.66:5204/muninn.raven.av.rudp?channel=media&format=muninn-typed-media&connection=0x6d750001&audio_connection=0x6d750004&sender_resend_delay_ms=5&reliable_expire_after_ms=120&assembly_deadline_ms=120&gap_wait_ms=16&audio_reorder_ms=80"); require(configured.has_value(), "configured RUDP URL did not parse"); require(configured->port == 5204, "configured RUDP URL port was not parsed"); require(configured->video_local_port == 15204, "configured RUDP video local port was not derived"); require(configured->audio_local_port == 15205, "configured RUDP audio local port was not derived"); require(configured->connection_id == 0x6d750001, "configured RUDP connection id was not parsed"); require(configured->audio_connection_id == 0x6d750004, "configured RUDP audio connection id was not parsed"); - require(configured->reliable_expire_after_ms == 1200, "configured reliable expiry did not survive parse"); - require(configured->video_assembly_deadline_ms == 1200, "configured assembly deadline did not survive parse"); + require(configured->reliable_expire_after_ms == 120, "configured reliable expiry did not survive parse"); + require(configured->video_assembly_deadline_ms == 120, "configured assembly deadline did not survive parse"); require(configured->gap_wait_ms == 16, "configured gap wait did not survive parse"); - require(configured->audio_reorder_ms == 700, "configured audio reorder wait did not survive parse"); + require(configured->audio_reorder_ms == 80, "configured audio reorder wait did not survive parse"); const auto clamped = muninn_rudp_url::parse( "rudp://192.168.1.66:5204/muninn.raven.av.rudp?reliable_expire_after_ms=9000&assembly_deadline_ms=9000");