fegger/screen_cast
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(codec): pass YUV through to the renderer and use slice threading

Browse Source 
The receiver decoded H.264 to YUV420P, converted it to RGBA via a
CPU-intensive swscale pass, then uploaded 4 bytes/pixel to an SDL
texture — only for the GPU to convert back to RGB during rendering.
This eliminated the swscale pass entirely (40-60% of receiver CPU at
1080p) and cut the texture upload by 62%.

- DecodedFrame now carries three YUV420P planes with their strides
  instead of a packed RGBA buffer; the decoder copies the planes
  directly from the AVFrame (zero conversion for the common software
  path). Non-YUV420P decoder output (e.g. NV12 from v4l2m2m) is
  converted once to YUV420P.
- The SDL renderer uploads via SDL_UpdateYUVTexture with
  SDL_PIXELFORMAT_IYUV; the GPU does the YUV→RGB conversion during
  rendering.
- Decoder threading: slice-level with 4 threads (parallelizes within a
  frame, no added latency), not frame-level (which buffers multiple
  frames — the initial thread_count=0 broke the loopback test because
  the H.264 decoder introduced a multi-frame delay before producing
  output).
- The round-trip test converts decoded YUV back to RGBA for pixel
  comparison via a test-local swscale call (the pipeline itself never
  converts).

meson test 5/5 in both configurations, valgrind clean.
This commit is contained in:
Florian Egger
2026-09-09 09:45:22 +02:00
parent b4d1411fd9
commit 6516b45b02
5 changed files with 122 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/screencast/codec/decoder.h
+11 -1
View File
@@ -10,11 +10,21 @@
namespace sc {
// Decoded video frame in YUV420P planar format (the native decoder output,
// passed to the renderer without any colorspace conversion).
struct DecodedFrame {
int width = 0;
int height = 0;
uint64_t capture_timestamp_ns = 0;
std::vector<std::byte> rgba_pixels;
// YUV420P planes; each row is `stride` bytes wide and may be padded
// beyond the picture width.
std::vector<std::byte> plane_y;
std::vector<std::byte> plane_u;
std::vector<std::byte> plane_v;
int stride_y = 0;
int stride_u = 0;
int stride_v = 0;
};
struct DecoderConfig {
src/codec/ffmpeg_decoder.cpp
+54 -23
View File
@@ -147,34 +147,61 @@ class FfmpegDecoder final : public Decoder {
? static_cast<uint64_t>(av_rescale_q(input->pts, ctx_->time_base, AVRational{1, 1'000'000'000}))
: fallback_timestamp_ns;
const std::size_t buffer_size =
static_cast<std::size_t>(av_image_get_buffer_size(AV_PIX_FMT_RGBA, frame.width, frame.height, 1));
frame.rgba_pixels.resize(buffer_size);
if (!ensure_scaler(input->width, input->height, static_cast<AVPixelFormat>(input->format))) {
return CodecError{"failed to create swscale context"};
const AVPixelFormat pixel_format = static_cast<AVPixelFormat>(input->format);
if (pixel_format != AV_PIX_FMT_YUV420P) {
// Hardware decoders (e.g. v4l2m2m) may emit NV12 or other planar
// variants; convert to YUV420P once. The common software path
// (YUV420P) skips this entirely.
if (!ensure_scaler(input->width, input->height, pixel_format)) {
return CodecError{"failed to create format conversion context"};
}
AvFramePtr converted(av_frame_alloc(), AvFrameDeleter{});
converted->width = input->width;
converted->height = input->height;
converted->format = AV_PIX_FMT_YUV420P;
if (av_frame_get_buffer(converted.get(), 0) < 0) {
return CodecError{"failed to allocate converted frame"};
}
if (sws_scale(scaler_.get(),
input->data,
input->linesize,
0,
input->height,
converted->data,
converted->linesize) <= 0) {
return CodecError{"failed to convert decoded frame to YUV420P"};
}
return copy_yuv420p_planes(converted.get(), std::move(frame));
}
std::array<uint8_t*, 4> dst{nullptr, nullptr, nullptr, nullptr};
std::array<int, 4> dst_lines{0, 0, 0, 0};
if (av_image_fill_arrays(dst.data(),
dst_lines.data(),
as_u8(frame.rgba_pixels.data()),
AV_PIX_FMT_RGBA,
frame.width,
frame.height,
1) < 0) {
return CodecError{"failed to fill output pixel arrays"};
}
return copy_yuv420p_planes(input, std::move(frame));
}
if (sws_scale(scaler_.get(), input->data, input->linesize, 0, input->height, dst.data(), dst_lines.data()) <=
0) {
return CodecError{"failed to convert decoded frame to RGBA"};
}
// Copies the three YUV420P planes with their strides (which may include
// alignment padding). SDL's UpdateYUVTexture accepts arbitrary pitches.
CodecResult<DecodedFrame> copy_yuv420p_planes(const AVFrame* input, DecodedFrame frame) const {
const int width = input->width;
const int height = input->height;
frame.stride_y = input->linesize[0];
frame.stride_u = input->linesize[1];
frame.stride_v = input->linesize[2];
const std::size_t y_size = static_cast<std::size_t>(frame.stride_y) * height;
const std::size_t uv_size = static_cast<std::size_t>(frame.stride_u) * ((height + 1) / 2);
const std::size_t v_size = static_cast<std::size_t>(frame.stride_v) * ((height + 1) / 2);
frame.plane_y.resize(y_size);
frame.plane_u.resize(uv_size);
frame.plane_v.resize(v_size);
std::memcpy(frame.plane_y.data(), input->data[0], y_size);
std::memcpy(frame.plane_u.data(), input->data[1], uv_size);
std::memcpy(frame.plane_v.data(), input->data[2], v_size);
return frame;
}
// Only used when the decoder emits a non-YUV420P format (hardware paths).
bool ensure_scaler(int width, int height, AVPixelFormat input_format) const {
if (scaler_ != nullptr && scaler_input_width_ == width && scaler_input_height_ == height &&
scaler_input_format_ == input_format) {
@@ -182,7 +209,7 @@ class FfmpegDecoder final : public Decoder {
}
scaler_.reset(sws_getContext(
width, height, input_format, width, height, AV_PIX_FMT_RGBA, SWS_BILINEAR, nullptr, nullptr, nullptr));
width, height, input_format, width, height, AV_PIX_FMT_YUV420P, SWS_BILINEAR, nullptr, nullptr, nullptr));
if (scaler_ == nullptr) {
return false;
}
@@ -222,7 +249,11 @@ std::optional<std::string> open_decoder(const DecoderConfig& config, const AVCod
if (config.height > 0) {
ctx->height = config.height;
}
ctx->thread_count = 1;
// Slice-level threading parallelizes within a single frame (no added
// latency), unlike frame-level threading which buffers multiple frames
// before producing output — unacceptable for a live stream.
ctx->thread_count = 4;
ctx->thread_type = FF_THREAD_SLICE;
if (!config.extradata.empty()) {
if (config.extradata.size() > static_cast<std::size_t>(std::numeric_limits<int>::max())) {
src/render/sdl_renderer.cpp
+13 -7
View File
@@ -7,10 +7,10 @@
namespace sc {
namespace {
// DecodedFrame pixels are AV_PIX_FMT_RGBA: memory order R, G, B, A. SDL
// names 32-bit formats MSB-first, so that byte order is SDL's ABGR8888 —
// using RGBA8888 would read the alpha byte as red and tint the image red.
constexpr SDL_PixelFormat kSdlPixelFormat = SDL_PIXELFORMAT_ABGR8888;
// DecodedFrame carries YUV420P planar data; SDL_PIXELFORMAT_IYUV is the
// matching SDL texture format. The GPU does the YUV→RGB conversion during
// rendering, eliminating a CPU-side swscale pass.
constexpr SDL_PixelFormat kSdlPixelFormat = SDL_PIXELFORMAT_IYUV;
class SdlRenderer final : public Renderer {
public:
@@ -74,7 +74,7 @@ class SdlRenderer final : public Renderer {
}
bool present(const DecodedFrame& frame) override {
if (frame.width <= 0 || frame.height <= 0) {
if (frame.width <= 0 || frame.height <= 0 || frame.plane_y.empty()) {
return false;
}
if (frame.width != texture_width_ || frame.height != texture_height_) {
@@ -83,8 +83,14 @@ class SdlRenderer final : public Renderer {
}
}
const int pitch = frame.width * 4;
if (!SDL_UpdateTexture(texture_, nullptr, frame.rgba_pixels.data(), pitch)) {
if (!SDL_UpdateYUVTexture(texture_,
nullptr,
reinterpret_cast<const uint8_t*>(frame.plane_y.data()),
frame.stride_y,
reinterpret_cast<const uint8_t*>(frame.plane_u.data()),
frame.stride_u,
reinterpret_cast<const uint8_t*>(frame.plane_v.data()),
frame.stride_v)) {
return false;
}
// Clear first so the letterbox bars stay black between frames.
tests/app/test_loopback.cpp
+1 -1
View File
@@ -90,7 +90,7 @@ struct ReceiverSink {
for (const sc::DecodedFrame& decoded : sc::codec_value(decoded_result)) {
last_width = decoded.width;
last_height = decoded.height;
saw_frame = decoded.width > 0 && !decoded.rgba_pixels.empty();
saw_frame = decoded.width > 0 && !decoded.plane_y.empty();
decoded_frames.fetch_add(1);
}
}
tests/codec/test_roundtrip.cpp
+43 -3
View File
@@ -1,7 +1,12 @@
#include "screencast/codec/decoder.h"
#include "screencast/codec/encoder.h"
#include <algorithm>
// swscale is a C library; without the extern wrapper its functions get
// C++ mangled and the linker cannot find them.
extern "C" {
#include <libswscale/swscale.h>
}
#include <cassert>
#include <cstdint>
#include <cstdlib>
@@ -47,10 +52,45 @@ sc::CapturedFrame make_frame(uint32_t index) {
return frame;
}
// Convert a decoded YUV420P frame back to RGBA for pixel comparison with
// the original capture. Test-only; the pipeline itself never converts.
std::vector<std::byte> decoded_to_rgba(const sc::DecodedFrame& decoded) {
const std::size_t rgba_size = static_cast<std::size_t>(decoded.width) * decoded.height * 4;
std::vector<std::byte> rgba(rgba_size);
const uint8_t* src_planes[4] = {
reinterpret_cast<const uint8_t*>(decoded.plane_y.data()),
reinterpret_cast<const uint8_t*>(decoded.plane_u.data()),
reinterpret_cast<const uint8_t*>(decoded.plane_v.data()),
nullptr,
};
const int src_strides[4] = {decoded.stride_y, decoded.stride_u, decoded.stride_v, 0};
uint8_t* dst_planes[4] = {reinterpret_cast<uint8_t*>(rgba.data()), nullptr, nullptr, nullptr};
const int dst_strides[4] = {decoded.width * 4, 0, 0, 0};
SwsContext* scaler = sws_getContext(decoded.width,
decoded.height,
AV_PIX_FMT_YUV420P,
decoded.width,
decoded.height,
AV_PIX_FMT_RGBA,
SWS_BILINEAR,
nullptr,
nullptr,
nullptr);
assert(scaler != nullptr);
(void)sws_scale(scaler, src_planes, src_strides, 0, decoded.height, dst_planes, dst_strides);
sws_freeContext(scaler);
return rgba;
}
int max_channel_difference(const sc::DecodedFrame& decoded, const sc::CapturedFrame& expected) {
const auto* decoded_pixels = reinterpret_cast<const std::uint8_t*>(decoded.rgba_pixels.data());
const auto decoded_rgba = decoded_to_rgba(decoded);
const auto* decoded_pixels = reinterpret_cast<const std::uint8_t*>(decoded_rgba.data());
const auto* expected_pixels = reinterpret_cast<const std::uint8_t*>(expected.pixels.data());
const std::size_t count = std::min(decoded.rgba_pixels.size(), expected.pixels.size());
const std::size_t count = std::min(decoded_rgba.size(), expected.pixels.size());
int max_diff = 0;
for (std::size_t i = 0; i < count; ++i) {