feat(app): add PLI feedback, jitter reordering, and hardware decode
Loss recovery for the streaming path: - PLI over signaling: the depacketizer now reports damaged frames (DepacketizeResult) and the receiver asks the sender for a keyframe (SessionPli, rate-limited to one per 500 ms). The sender keeps the signaling channel open during the session and honors PLIs through the new thread-safe SenderPipeline::request_keyframe(). Recovery takes one frame time instead of waiting out the GOP. - RtpJitterBuffer: reorders RTP packets by sequence number (16 packets / 60 ms) before the in-order depacketizer, so Wi-Fi reordering is not misread as loss; in-order streams release immediately, and a straggler older than the delivered sequence is discarded. - Hardware H.264 decode probe: DecoderFactory tries h264_v4l2m2m (the VideoCore path on the Pi) with an automatic software fallback and a clear journal line for the chosen path; --swdecode opts out. Validated: PLI end-to-end with a probe that drops a mid-keyframe packet over real UDP (receiver logged the damaged frame and the PLI arrived with the session id); hardware probe fails cleanly and falls back on this desktop; jitter reordering covered by unit tests. meson test 5/5 in both build configurations, valgrind clean.
This commit is contained in:
@@ -18,6 +18,7 @@ struct ReceiveCommand {
|
||||
int local_rtp_port = 5004;
|
||||
int signaling_port = 5005;
|
||||
bool fullscreen = false;
|
||||
bool software_decode = false; // --swdecode disables the hardware probe
|
||||
};
|
||||
|
||||
struct DiscoverCommand {
|
||||
|
||||
@@ -39,6 +39,10 @@ class SenderPipeline {
|
||||
bool start();
|
||||
void stop();
|
||||
|
||||
// Ask the sender to encode its next frame as a keyframe. Thread-safe;
|
||||
// used by the PLI feedback path.
|
||||
void request_keyframe();
|
||||
|
||||
private:
|
||||
class Impl;
|
||||
std::unique_ptr<Impl> impl_;
|
||||
|
||||
@@ -22,6 +22,9 @@ struct DecoderConfig {
|
||||
int width = 0;
|
||||
int height = 0;
|
||||
std::vector<std::byte> extradata; // SPS/PPS for H.264
|
||||
// Probe a hardware decoder (v4l2 mem2mem) first and fall back to the
|
||||
// software decoder automatically. Disable with --swdecode.
|
||||
bool hardware_accel = true;
|
||||
};
|
||||
|
||||
class Decoder {
|
||||
|
||||
@@ -42,14 +42,23 @@ class H264Packetizer {
|
||||
std::uint16_t next_sequence_number_ = 0;
|
||||
};
|
||||
|
||||
// Result of feeding one packet to the depacketizer.
|
||||
struct DepacketizeResult {
|
||||
// The completed access unit (Annex-B with 3-byte start codes) when the
|
||||
// packet closed an undamaged frame.
|
||||
std::optional<std::vector<std::byte>> access_unit;
|
||||
// True when this call discarded a frame as damaged (packet loss or an
|
||||
// unsupported packetization). Pipelines use it to request a keyframe.
|
||||
bool frame_dropped = false;
|
||||
};
|
||||
|
||||
// Reassembles RFC 6184 packet streams (single NAL unit packets and FU-A)
|
||||
// into Annex-B access units. Packets must arrive in order; frames damaged by
|
||||
// sequence gaps or missing fragments are dropped silently.
|
||||
// sequence gaps or missing fragments are reported via DepacketizeResult.
|
||||
class H264Depacketizer {
|
||||
public:
|
||||
// Feed one packet. Returns the completed access unit (Annex-B with 3-byte
|
||||
// start codes) when the packet closes a frame, nullopt otherwise.
|
||||
std::optional<std::vector<std::byte>> depacketize(const RtpPacket& packet);
|
||||
// Feed one packet.
|
||||
DepacketizeResult depacketize(const RtpPacket& packet);
|
||||
|
||||
private:
|
||||
void drop_frame();
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <span>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace sc {
|
||||
@@ -31,4 +35,29 @@ struct RtpPacket {
|
||||
static std::optional<RtpPacket> parse(std::span<const std::byte> in) noexcept;
|
||||
};
|
||||
|
||||
// Reorders RTP packets by sequence number before depacketization so that a
|
||||
// reordering link (Wi-Fi) does not read as loss. Delivery stays in order;
|
||||
// only aged-out or overflowing buffers release out of order, which the
|
||||
// downstream gap detection still handles for genuine loss.
|
||||
class RtpJitterBuffer {
|
||||
public:
|
||||
explicit RtpJitterBuffer(std::size_t max_depth = 16,
|
||||
std::chrono::milliseconds max_delay = std::chrono::milliseconds{60});
|
||||
|
||||
// Insert one packet and return the packets now ready for in-order
|
||||
// delivery. In-order streams release immediately (zero added latency);
|
||||
// a straggler older than the next expected sequence is discarded.
|
||||
std::vector<RtpPacket> push(RtpPacket packet);
|
||||
|
||||
// Discard everything still buffered.
|
||||
void clear();
|
||||
|
||||
private:
|
||||
std::size_t max_depth_;
|
||||
std::chrono::milliseconds max_delay_;
|
||||
std::mutex mutex_;
|
||||
std::map<std::uint16_t, std::pair<std::chrono::steady_clock::time_point, RtpPacket>> buffer_;
|
||||
std::optional<std::uint16_t> next_expected_;
|
||||
};
|
||||
|
||||
} // namespace sc
|
||||
|
||||
@@ -31,7 +31,13 @@ struct SessionAnswer {
|
||||
Endpoint rtp_endpoint;
|
||||
};
|
||||
|
||||
using SignalingMessage = std::variant<SessionOffer, SessionAnswer>;
|
||||
// Picture Loss Indication: the receiver asks the sender for a keyframe
|
||||
// after discarding a damaged frame.
|
||||
struct SessionPli {
|
||||
std::string session_id;
|
||||
};
|
||||
|
||||
using SignalingMessage = std::variant<SessionOffer, SessionAnswer, SessionPli>;
|
||||
|
||||
class SignalingChannel {
|
||||
public:
|
||||
|
||||
Reference in New Issue
Block a user