From 6516b45b0252324659bf495a4cafe62460347c25 Mon Sep 17 00:00:00 2001 From: Florian Egger Date: Wed, 9 Sep 2026 09:45:22 +0200 Subject: [PATCH] perf(codec): pass YUV through to the renderer and use slice threading MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The receiver decoded H.264 to YUV420P, converted it to RGBA via a CPU-intensive swscale pass, then uploaded 4 bytes/pixel to an SDL texture — only for the GPU to convert back to RGB during rendering. This eliminated the swscale pass entirely (40-60% of receiver CPU at 1080p) and cut the texture upload by 62%. - DecodedFrame now carries three YUV420P planes with their strides instead of a packed RGBA buffer; the decoder copies the planes directly from the AVFrame (zero conversion for the common software path). Non-YUV420P decoder output (e.g. NV12 from v4l2m2m) is converted once to YUV420P. - The SDL renderer uploads via SDL_UpdateYUVTexture with SDL_PIXELFORMAT_IYUV; the GPU does the YUV→RGB conversion during rendering. - Decoder threading: slice-level with 4 threads (parallelizes within a frame, no added latency), not frame-level (which buffers multiple frames — the initial thread_count=0 broke the loopback test because the H.264 decoder introduced a multi-frame delay before producing output). - The round-trip test converts decoded YUV back to RGBA for pixel comparison via a test-local swscale call (the pipeline itself never converts). meson test 5/5 in both configurations, valgrind clean. --- include/screencast/codec/decoder.h | 12 ++++- src/codec/ffmpeg_decoder.cpp | 77 +++++++++++++++++++++--------- src/render/sdl_renderer.cpp | 20 +++++--- tests/app/test_loopback.cpp | 2 +- tests/codec/test_roundtrip.cpp | 46 ++++++++++++++++-- 5 files changed, 122 insertions(+), 35 deletions(-) diff --git a/include/screencast/codec/decoder.h b/include/screencast/codec/decoder.h index c3cb25b..dc7af12 100644 --- a/include/screencast/codec/decoder.h +++ b/include/screencast/codec/decoder.h @@ -10,11 +10,21 @@ namespace sc { +// Decoded video frame in YUV420P planar format (the native decoder output, +// passed to the renderer without any colorspace conversion). struct DecodedFrame { int width = 0; int height = 0; uint64_t capture_timestamp_ns = 0; - std::vector rgba_pixels; + + // YUV420P planes; each row is `stride` bytes wide and may be padded + // beyond the picture width. + std::vector plane_y; + std::vector plane_u; + std::vector plane_v; + int stride_y = 0; + int stride_u = 0; + int stride_v = 0; }; struct DecoderConfig { diff --git a/src/codec/ffmpeg_decoder.cpp b/src/codec/ffmpeg_decoder.cpp index df8afb9..ca9305b 100644 --- a/src/codec/ffmpeg_decoder.cpp +++ b/src/codec/ffmpeg_decoder.cpp @@ -147,34 +147,61 @@ class FfmpegDecoder final : public Decoder { ? static_cast(av_rescale_q(input->pts, ctx_->time_base, AVRational{1, 1'000'000'000})) : fallback_timestamp_ns; - const std::size_t buffer_size = - static_cast(av_image_get_buffer_size(AV_PIX_FMT_RGBA, frame.width, frame.height, 1)); - frame.rgba_pixels.resize(buffer_size); - - if (!ensure_scaler(input->width, input->height, static_cast(input->format))) { - return CodecError{"failed to create swscale context"}; + const AVPixelFormat pixel_format = static_cast(input->format); + if (pixel_format != AV_PIX_FMT_YUV420P) { + // Hardware decoders (e.g. v4l2m2m) may emit NV12 or other planar + // variants; convert to YUV420P once. The common software path + // (YUV420P) skips this entirely. + if (!ensure_scaler(input->width, input->height, pixel_format)) { + return CodecError{"failed to create format conversion context"}; + } + AvFramePtr converted(av_frame_alloc(), AvFrameDeleter{}); + converted->width = input->width; + converted->height = input->height; + converted->format = AV_PIX_FMT_YUV420P; + if (av_frame_get_buffer(converted.get(), 0) < 0) { + return CodecError{"failed to allocate converted frame"}; + } + if (sws_scale(scaler_.get(), + input->data, + input->linesize, + 0, + input->height, + converted->data, + converted->linesize) <= 0) { + return CodecError{"failed to convert decoded frame to YUV420P"}; + } + return copy_yuv420p_planes(converted.get(), std::move(frame)); } - std::array dst{nullptr, nullptr, nullptr, nullptr}; - std::array dst_lines{0, 0, 0, 0}; - if (av_image_fill_arrays(dst.data(), - dst_lines.data(), - as_u8(frame.rgba_pixels.data()), - AV_PIX_FMT_RGBA, - frame.width, - frame.height, - 1) < 0) { - return CodecError{"failed to fill output pixel arrays"}; - } + return copy_yuv420p_planes(input, std::move(frame)); + } - if (sws_scale(scaler_.get(), input->data, input->linesize, 0, input->height, dst.data(), dst_lines.data()) <= - 0) { - return CodecError{"failed to convert decoded frame to RGBA"}; - } + // Copies the three YUV420P planes with their strides (which may include + // alignment padding). SDL's UpdateYUVTexture accepts arbitrary pitches. + CodecResult copy_yuv420p_planes(const AVFrame* input, DecodedFrame frame) const { + const int width = input->width; + const int height = input->height; + frame.stride_y = input->linesize[0]; + frame.stride_u = input->linesize[1]; + frame.stride_v = input->linesize[2]; + + const std::size_t y_size = static_cast(frame.stride_y) * height; + const std::size_t uv_size = static_cast(frame.stride_u) * ((height + 1) / 2); + const std::size_t v_size = static_cast(frame.stride_v) * ((height + 1) / 2); + + frame.plane_y.resize(y_size); + frame.plane_u.resize(uv_size); + frame.plane_v.resize(v_size); + + std::memcpy(frame.plane_y.data(), input->data[0], y_size); + std::memcpy(frame.plane_u.data(), input->data[1], uv_size); + std::memcpy(frame.plane_v.data(), input->data[2], v_size); return frame; } + // Only used when the decoder emits a non-YUV420P format (hardware paths). bool ensure_scaler(int width, int height, AVPixelFormat input_format) const { if (scaler_ != nullptr && scaler_input_width_ == width && scaler_input_height_ == height && scaler_input_format_ == input_format) { @@ -182,7 +209,7 @@ class FfmpegDecoder final : public Decoder { } scaler_.reset(sws_getContext( - width, height, input_format, width, height, AV_PIX_FMT_RGBA, SWS_BILINEAR, nullptr, nullptr, nullptr)); + width, height, input_format, width, height, AV_PIX_FMT_YUV420P, SWS_BILINEAR, nullptr, nullptr, nullptr)); if (scaler_ == nullptr) { return false; } @@ -222,7 +249,11 @@ std::optional open_decoder(const DecoderConfig& config, const AVCod if (config.height > 0) { ctx->height = config.height; } - ctx->thread_count = 1; + // Slice-level threading parallelizes within a single frame (no added + // latency), unlike frame-level threading which buffers multiple frames + // before producing output — unacceptable for a live stream. + ctx->thread_count = 4; + ctx->thread_type = FF_THREAD_SLICE; if (!config.extradata.empty()) { if (config.extradata.size() > static_cast(std::numeric_limits::max())) { diff --git a/src/render/sdl_renderer.cpp b/src/render/sdl_renderer.cpp index 9118a1d..a8ab097 100644 --- a/src/render/sdl_renderer.cpp +++ b/src/render/sdl_renderer.cpp @@ -7,10 +7,10 @@ namespace sc { namespace { -// DecodedFrame pixels are AV_PIX_FMT_RGBA: memory order R, G, B, A. SDL -// names 32-bit formats MSB-first, so that byte order is SDL's ABGR8888 — -// using RGBA8888 would read the alpha byte as red and tint the image red. -constexpr SDL_PixelFormat kSdlPixelFormat = SDL_PIXELFORMAT_ABGR8888; +// DecodedFrame carries YUV420P planar data; SDL_PIXELFORMAT_IYUV is the +// matching SDL texture format. The GPU does the YUV→RGB conversion during +// rendering, eliminating a CPU-side swscale pass. +constexpr SDL_PixelFormat kSdlPixelFormat = SDL_PIXELFORMAT_IYUV; class SdlRenderer final : public Renderer { public: @@ -74,7 +74,7 @@ class SdlRenderer final : public Renderer { } bool present(const DecodedFrame& frame) override { - if (frame.width <= 0 || frame.height <= 0) { + if (frame.width <= 0 || frame.height <= 0 || frame.plane_y.empty()) { return false; } if (frame.width != texture_width_ || frame.height != texture_height_) { @@ -83,8 +83,14 @@ class SdlRenderer final : public Renderer { } } - const int pitch = frame.width * 4; - if (!SDL_UpdateTexture(texture_, nullptr, frame.rgba_pixels.data(), pitch)) { + if (!SDL_UpdateYUVTexture(texture_, + nullptr, + reinterpret_cast(frame.plane_y.data()), + frame.stride_y, + reinterpret_cast(frame.plane_u.data()), + frame.stride_u, + reinterpret_cast(frame.plane_v.data()), + frame.stride_v)) { return false; } // Clear first so the letterbox bars stay black between frames. diff --git a/tests/app/test_loopback.cpp b/tests/app/test_loopback.cpp index 7bfffd9..afcff1e 100644 --- a/tests/app/test_loopback.cpp +++ b/tests/app/test_loopback.cpp @@ -90,7 +90,7 @@ struct ReceiverSink { for (const sc::DecodedFrame& decoded : sc::codec_value(decoded_result)) { last_width = decoded.width; last_height = decoded.height; - saw_frame = decoded.width > 0 && !decoded.rgba_pixels.empty(); + saw_frame = decoded.width > 0 && !decoded.plane_y.empty(); decoded_frames.fetch_add(1); } } diff --git a/tests/codec/test_roundtrip.cpp b/tests/codec/test_roundtrip.cpp index 5b193a2..8425c67 100644 --- a/tests/codec/test_roundtrip.cpp +++ b/tests/codec/test_roundtrip.cpp @@ -1,7 +1,12 @@ #include "screencast/codec/decoder.h" #include "screencast/codec/encoder.h" -#include +// swscale is a C library; without the extern wrapper its functions get +// C++ mangled and the linker cannot find them. +extern "C" { +#include +} + #include #include #include @@ -47,10 +52,45 @@ sc::CapturedFrame make_frame(uint32_t index) { return frame; } +// Convert a decoded YUV420P frame back to RGBA for pixel comparison with +// the original capture. Test-only; the pipeline itself never converts. +std::vector decoded_to_rgba(const sc::DecodedFrame& decoded) { + const std::size_t rgba_size = static_cast(decoded.width) * decoded.height * 4; + std::vector rgba(rgba_size); + + const uint8_t* src_planes[4] = { + reinterpret_cast(decoded.plane_y.data()), + reinterpret_cast(decoded.plane_u.data()), + reinterpret_cast(decoded.plane_v.data()), + nullptr, + }; + const int src_strides[4] = {decoded.stride_y, decoded.stride_u, decoded.stride_v, 0}; + + uint8_t* dst_planes[4] = {reinterpret_cast(rgba.data()), nullptr, nullptr, nullptr}; + const int dst_strides[4] = {decoded.width * 4, 0, 0, 0}; + + SwsContext* scaler = sws_getContext(decoded.width, + decoded.height, + AV_PIX_FMT_YUV420P, + decoded.width, + decoded.height, + AV_PIX_FMT_RGBA, + SWS_BILINEAR, + nullptr, + nullptr, + nullptr); + assert(scaler != nullptr); + (void)sws_scale(scaler, src_planes, src_strides, 0, decoded.height, dst_planes, dst_strides); + sws_freeContext(scaler); + + return rgba; +} + int max_channel_difference(const sc::DecodedFrame& decoded, const sc::CapturedFrame& expected) { - const auto* decoded_pixels = reinterpret_cast(decoded.rgba_pixels.data()); + const auto decoded_rgba = decoded_to_rgba(decoded); + const auto* decoded_pixels = reinterpret_cast(decoded_rgba.data()); const auto* expected_pixels = reinterpret_cast(expected.pixels.data()); - const std::size_t count = std::min(decoded.rgba_pixels.size(), expected.pixels.size()); + const std::size_t count = std::min(decoded_rgba.size(), expected.pixels.size()); int max_diff = 0; for (std::size_t i = 0; i < count; ++i) {