feat(app): add PLI feedback, jitter reordering, and hardware decode
Loss recovery for the streaming path: - PLI over signaling: the depacketizer now reports damaged frames (DepacketizeResult) and the receiver asks the sender for a keyframe (SessionPli, rate-limited to one per 500 ms). The sender keeps the signaling channel open during the session and honors PLIs through the new thread-safe SenderPipeline::request_keyframe(). Recovery takes one frame time instead of waiting out the GOP. - RtpJitterBuffer: reorders RTP packets by sequence number (16 packets / 60 ms) before the in-order depacketizer, so Wi-Fi reordering is not misread as loss; in-order streams release immediately, and a straggler older than the delivered sequence is discarded. - Hardware H.264 decode probe: DecoderFactory tries h264_v4l2m2m (the VideoCore path on the Pi) with an automatic software fallback and a clear journal line for the chosen path; --swdecode opts out. Validated: PLI end-to-end with a probe that drops a mid-keyframe packet over real UDP (receiver logged the damaged frame and the PLI arrived with the session id); hardware probe fails cleanly and falls back on this desktop; jitter reordering covered by unit tests. meson test 5/5 in both build configurations, valgrind clean.
This commit is contained in:
@@ -74,13 +74,13 @@ struct ReceiverSink {
|
||||
int last_height = 0;
|
||||
|
||||
void on_packet(sc::RtpPacket packet) {
|
||||
std::optional<std::vector<std::byte>> access_unit = depacketizer.depacketize(packet);
|
||||
if (!access_unit.has_value()) {
|
||||
const sc::DepacketizeResult result = depacketizer.depacketize(packet);
|
||||
if (!result.access_unit.has_value()) {
|
||||
return;
|
||||
}
|
||||
|
||||
sc::EncodedFrame encoded;
|
||||
encoded.data = std::move(*access_unit);
|
||||
encoded.data = std::move(*result.access_unit);
|
||||
encoded.rtp_timestamp = packet.header.timestamp;
|
||||
|
||||
auto decoded_result = decoder->decode(encoded);
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
@@ -288,13 +289,17 @@ void test_depacketize_roundtrip() {
|
||||
|
||||
sc::H264Depacketizer depacketizer;
|
||||
std::optional<std::vector<std::byte>> completed;
|
||||
bool any_dropped = false;
|
||||
for (const sc::RtpPacket& packet : packets) {
|
||||
if (auto result = depacketizer.depacketize(packet)) {
|
||||
const sc::DepacketizeResult result = depacketizer.depacketize(packet);
|
||||
if (result.access_unit.has_value()) {
|
||||
check(!completed.has_value(), "only one completion");
|
||||
completed = std::move(result);
|
||||
completed = std::move(result.access_unit);
|
||||
}
|
||||
any_dropped = any_dropped || result.frame_dropped;
|
||||
}
|
||||
check(completed.has_value(), "frame completed");
|
||||
check(!any_dropped, "no dropped frames in a clean stream");
|
||||
check(equal_bytes(*completed, access_unit), "access unit round-trip");
|
||||
}
|
||||
|
||||
@@ -305,9 +310,11 @@ void test_depacketizer_drops_gapped_frames() {
|
||||
check(packets.size() == 3, "gap test packet count");
|
||||
|
||||
sc::H264Depacketizer depacketizer;
|
||||
check(!depacketizer.depacketize(packets[0]).has_value(), "first fu chunk accepted");
|
||||
check(!depacketizer.depacketize(packets[0]).access_unit.has_value(), "first fu chunk accepted");
|
||||
// packets[1] is lost in transit; the tail cannot complete the frame.
|
||||
check(!depacketizer.depacketize(packets[2]).has_value(), "tail after gap dropped");
|
||||
const sc::DepacketizeResult tail = depacketizer.depacketize(packets[2]);
|
||||
check(!tail.access_unit.has_value(), "tail after gap dropped");
|
||||
check(tail.frame_dropped, "drop reported after gap");
|
||||
}
|
||||
|
||||
void test_depacketizer_separate_frames() {
|
||||
@@ -319,12 +326,79 @@ void test_depacketizer_separate_frames() {
|
||||
const std::vector<sc::RtpPacket> second = packetizer.packetize(make_frame(f2, 90000));
|
||||
|
||||
sc::H264Depacketizer depacketizer;
|
||||
const std::optional<std::vector<std::byte>> au1 = depacketizer.depacketize(first[0]);
|
||||
check(au1.has_value() && equal_bytes(*au1, f1), "first frame");
|
||||
const sc::DepacketizeResult au1 = depacketizer.depacketize(first[0]);
|
||||
check(au1.access_unit.has_value() && equal_bytes(*au1.access_unit, f1), "first frame");
|
||||
check(!au1.frame_dropped, "first frame not dropped");
|
||||
|
||||
// Same RTP timestamp on purpose: the marker alone separates frames.
|
||||
const std::optional<std::vector<std::byte>> au2 = depacketizer.depacketize(second[0]);
|
||||
check(au2.has_value() && equal_bytes(*au2, f2), "second frame with same timestamp");
|
||||
const sc::DepacketizeResult au2 = depacketizer.depacketize(second[0]);
|
||||
check(au2.access_unit.has_value() && equal_bytes(*au2.access_unit, f2), "second frame with same timestamp");
|
||||
check(!au2.frame_dropped, "second frame not dropped");
|
||||
}
|
||||
|
||||
void test_jitter_buffer_in_order() {
|
||||
sc::RtpJitterBuffer jitter;
|
||||
sc::RtpPacket packet;
|
||||
packet.header.sequence_number = 100;
|
||||
|
||||
std::vector<sc::RtpPacket> released = jitter.push(packet);
|
||||
check(released.size() == 1 && released[0].header.sequence_number == 100, "in-order releases immediately");
|
||||
|
||||
packet.header.sequence_number = 101;
|
||||
released = jitter.push(std::move(packet));
|
||||
check(released.size() == 1 && released[0].header.sequence_number == 101, "next packet releases too");
|
||||
}
|
||||
|
||||
void test_jitter_buffer_reorders() {
|
||||
sc::RtpJitterBuffer jitter;
|
||||
|
||||
sc::RtpPacket first;
|
||||
first.header.sequence_number = 1;
|
||||
std::vector<sc::RtpPacket> released = jitter.push(std::move(first));
|
||||
check(released.size() == 1 && released[0].header.sequence_number == 1, "first packet releases");
|
||||
|
||||
// Arrives ahead of its predecessor: held, not delivered.
|
||||
sc::RtpPacket third;
|
||||
third.header.sequence_number = 3;
|
||||
released = jitter.push(std::move(third));
|
||||
check(released.empty(), "gap holds packets");
|
||||
|
||||
sc::RtpPacket second;
|
||||
second.header.sequence_number = 2;
|
||||
released = jitter.push(std::move(second));
|
||||
check(released.size() == 2, "held packets release in order");
|
||||
check(released[0].header.sequence_number == 2 && released[1].header.sequence_number == 3,
|
||||
"released in sequence order");
|
||||
}
|
||||
|
||||
void test_jitter_buffer_overflow_and_stragglers() {
|
||||
// Small depth: a persistent gap overflows the buffer and flushes what
|
||||
// is there, so genuine loss reaches the depacketizer instead of
|
||||
// stalling delivery.
|
||||
sc::RtpJitterBuffer jitter(4, std::chrono::milliseconds{500});
|
||||
|
||||
sc::RtpPacket packet;
|
||||
packet.header.sequence_number = 10;
|
||||
check(jitter.push(std::move(packet)).size() == 1, "first releases");
|
||||
|
||||
std::vector<sc::RtpPacket> released;
|
||||
for (std::uint16_t sequence = 12; sequence < 17; ++sequence) {
|
||||
sc::RtpPacket missing;
|
||||
missing.header.sequence_number = sequence;
|
||||
for (sc::RtpPacket out : jitter.push(std::move(missing))) {
|
||||
released.push_back(std::move(out));
|
||||
}
|
||||
}
|
||||
check(released.size() == 5, "overflow flushes the backlog");
|
||||
for (std::size_t i = 0; i < released.size(); ++i) {
|
||||
check(released[i].header.sequence_number == 12 + i, "flushed in order");
|
||||
}
|
||||
|
||||
// A straggler older than the delivered sequence is discarded, not
|
||||
// re-inserted out of order.
|
||||
sc::RtpPacket straggler;
|
||||
straggler.header.sequence_number = 11;
|
||||
check(jitter.push(std::move(straggler)).empty(), "straggler discarded");
|
||||
}
|
||||
|
||||
void test_sequence_wrap() {
|
||||
@@ -370,6 +444,9 @@ int main() {
|
||||
test_depacketize_roundtrip();
|
||||
test_depacketizer_drops_gapped_frames();
|
||||
test_depacketizer_separate_frames();
|
||||
test_jitter_buffer_in_order();
|
||||
test_jitter_buffer_reorders();
|
||||
test_jitter_buffer_overflow_and_stragglers();
|
||||
test_sequence_wrap();
|
||||
test_default_config_randomizes();
|
||||
test_empty_inputs();
|
||||
|
||||
@@ -93,6 +93,27 @@ int main() {
|
||||
check(answer.session_id == "test-session-0001", "session id echoes");
|
||||
check(answer.rtp_endpoint.port == advertised_rtp_port, "rtp port in answer");
|
||||
|
||||
// PLI: the receiver side asks for a keyframe mid-session; the client
|
||||
// must receive it with the session id intact.
|
||||
std::promise<sc::SessionPli> pli_promise;
|
||||
auto pli_future = pli_promise.get_future();
|
||||
std::atomic<bool> pli_seen{false};
|
||||
client->on_message([&](const sc::SignalingMessage& message) {
|
||||
if (const sc::SessionPli* pli = std::get_if<sc::SessionPli>(&message)) {
|
||||
if (!pli_seen.exchange(true)) {
|
||||
pli_promise.set_value(*pli);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
sc::SessionPli pli;
|
||||
pli.session_id = "test-session-0001";
|
||||
server->send(pli);
|
||||
|
||||
check(pli_future.wait_for(std::chrono::seconds(5)) == std::future_status::ready, "pli received");
|
||||
const sc::SessionPli received_pli = pli_future.get();
|
||||
check(received_pli.session_id == "test-session-0001", "pli session id echoes");
|
||||
|
||||
client->disconnect();
|
||||
server->disconnect();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user