fix(android): harden the receiver after the review pass

- SignalingServer.send() never throws: a broken signaling connection
  drops the peer instead of killing the RTP reader thread via requestPli()
- H264Decoder: assign the codec before configure/start (no orphaned
  instance on failure); feed() reports input-queue timeouts so the
  pipeline requests a keyframe instead of dropping the frame silently
- Park offers that arrive without a surface as pendingOffer and configure
  on attachSurface(): a codec configured without a surface can never take
  one (setOutputSurface refuses it); the late configure sends a PLI
- Forget destroyed surfaces (onSurfaceTextureDestroyed -> detachSurface)
- Show the status overlay again on later messages
- Accumulate NAL bytes in ByteArrayOutputStreams, not boxed ArrayList<Int>
- Delete the dead NSD reflection fallback (ResolutionListener never
  existed; the classic API is present from API 16 through 36)
- Hold decoderLock around all MediaCodec calls (not thread-safe) and make
  requestPli thread-safe
- Update RUNBOOK quirks (NSD, setOutputSurface) and MEMORY notes
This commit is contained in:
2026-09-10 12:19:21 +02:00
parent bbe4f21a3b
commit c6722d164b
7 changed files with 297 additions and 134 deletions
@@ -52,7 +52,13 @@ class ReceiverActivity : Activity() {
fitVideo()
}
override fun onSurfaceTextureDestroyed(surface: SurfaceTexture): Boolean = true
override fun onSurfaceTextureDestroyed(surface: SurfaceTexture): Boolean {
// The TextureView is about to release this SurfaceTexture; the
// pipeline must forget it or a later offer would configure the
// decoder against a dead surface.
pipeline?.detachSurface()
return true
}
override fun onSurfaceTextureUpdated(surface: SurfaceTexture) = Unit
}
@@ -85,7 +91,12 @@ class ReceiverActivity : Activity() {
val metrics = resources.displayMetrics
metrics.widthPixels to metrics.heightPixels
},
onStatus = { text -> ui.post { statusView.text = text } },
onStatus = { text ->
ui.post {
statusView.text = text
statusView.visibility = View.VISIBLE
}
},
onFirstFrame = { ui.post { statusView.visibility = View.GONE } },
onVideoSize = { _, _ -> ui.post { fitVideo() } },
)
@@ -35,11 +35,26 @@ class H264Decoder {
MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, 320, 240)
}
format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, MAX_INPUT_SIZE)
codec = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_VIDEO_AVC).also { c ->
// The surface must be passed to configure(); setOutputSurface()
// is only legal before configuration.
c.configure(format, renderSurface, null, 0)
c.start()
// Assign before configuring so a configure()/start() failure cannot
// orphan the created instance: MediaCodec has no finalizer and each
// unreleased instance holds a scarce native codec slot. The caller
// follows an exception with release(), which is a no-op on null.
val created = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_VIDEO_AVC)
codec = created
try {
// The surface must go into configure(): the codec must start in
// surface mode. setOutputSurface() afterwards only switches an
// already-surface-mode codec; a codec configured without a
// surface can never take one.
created.configure(format, renderSurface, null, 0)
created.start()
} catch (e: Exception) {
codec = null
try {
created.release()
} catch (ignored: Exception) {
}
throw e
}
configured = true
}
@@ -48,19 +63,39 @@ class H264Decoder {
@Synchronized
fun attachSurface(surface: Surface) {
renderSurface = surface
// Legal only in surface mode: setOutputSurface() dynamically switches
// an output surface on a codec configured WITH one (per the platform
// docs); on a ByteBuffer-mode codec it throws IllegalStateException.
codec?.setOutputSurface(surface)
}
/** Queues one access unit. [presentationUs] must be monotonic (the RTP timestamp). */
fun feed(data: ByteArray, presentationUs: Long, isKeyFrame: Boolean) {
/**
* Forgets a destroyed render surface: frames still decode (returning
* their buffers and keeping the queue moving) but are not rendered
* until the next [attachSurface]. API 35's detachOutputSurface() is the
* codec-side equivalent.
*/
@Synchronized
fun detachSurface() {
renderSurface = null
}
/**
* Queues one access unit. [presentationUs] must be monotonic (the RTP timestamp).
* @return false when the codec's input queue was full and the frame was
* dropped; the caller should request a keyframe, not flush (the codec
* is healthy, merely busy).
*/
fun feed(data: ByteArray, presentationUs: Long, isKeyFrame: Boolean): Boolean {
val c = codec ?: throw IllegalStateException("decoder not configured")
val index = c.dequeueInputBuffer(10_000)
if (index < 0) return
val buffer = c.getInputBuffer(index) ?: return
if (index < 0) return false
val buffer = c.getInputBuffer(index) ?: return false
buffer.clear()
buffer.put(data)
val flags = if (isKeyFrame) MediaCodec.BUFFER_FLAG_SYNC_FRAME else 0
val flags = if (isKeyFrame) MediaCodec.BUFFER_FLAG_KEY_FRAME else 0
c.queueInputBuffer(index, 0, data.size, presentationUs, flags)
return true
}
/**
@@ -12,7 +12,6 @@ import screen_cast.signaling.SessionAnswer
import screen_cast.signaling.SessionOffer
import screen_cast.signaling.SessionPli
import screen_cast.signaling.SignalingServer
import java.lang.reflect.Proxy
import java.net.DatagramPacket
import java.net.DatagramSocket
import java.net.InetSocketAddress
@@ -54,16 +53,17 @@ class ReceiverPipeline(
private var signalingPort = 0
private var signaling: SignalingServer? = null
private var readerThread: Thread? = null
private var nsdInfo: NsdServiceInfo? = null
@Volatile private var registrationListener: NsdManager.RegistrationListener? = null
@Volatile private var legacyNsdListener: Any? = null
// Guarded by decoderLock: currentSurface, pendingOffer, decoder.
@Volatile private var decoder: H264Decoder? = null
@Volatile private var depacketizer = H264Depacketizer()
private val jitter = JitterBuffer()
@Volatile private var currentSurface: Surface? = null
private var currentSurface: Surface? = null
private var pendingOffer: SessionOffer? = null
@Volatile private var activeSession = ""
private val firstFrameSeen = AtomicBoolean(false)
private val pliLock = Any()
private var lastPliAtMs = 0L
@Volatile private var videoWidth = 0
@Volatile private var videoHeight = 0
@@ -117,10 +117,60 @@ class ReceiverPipeline(
)
}
/** Points the decoder (current or future) at a render surface. */
/**
* Points the decoder (current or future) at a render surface. A pending
* offer (accepted while no surface existed) configures its decoder now.
*/
fun attachSurface(surface: Surface) {
currentSurface = surface
synchronized(decoderLock) { decoder?.attachSurface(surface) }
var configuredFromPending = false
synchronized(decoderLock) {
currentSurface = surface
decoder?.attachSurface(surface)
val offer = pendingOffer
if (offer != null) {
val created = configureDecoderLocked(offer.width, offer.height)
if (created != null) {
pendingOffer = null
decoder = created
configuredFromPending = true
} else {
// Keep the offer: the next attachSurface retries the
// configure instead of abandoning the session.
onStatus("Could not start the decoder")
}
}
}
if (configuredFromPending) {
// The sender is already streaming P-frames; the late-configured
// decoder needs a keyframe (SPS/PPS + IDR) to start producing
// output — the sender only emits one when asked.
requestPli()
}
}
/** Forgets a destroyed render surface so a later offer cannot configure against it. */
fun detachSurface() {
synchronized(decoderLock) {
currentSurface = null
decoder?.detachSurface()
}
}
/**
* Creates and configures a decoder for the stream size; caller holds
* [decoderLock]. Returns null on failure with the codec released.
*/
private fun configureDecoderLocked(width: Int, height: Int): H264Decoder? {
val fresh = H264Decoder()
try {
// Surface first: configure() needs it before the codec starts.
currentSurface?.let { fresh.attachSurface(it) }
fresh.configure(width, height)
} catch (e: Exception) {
fresh.release()
return null
}
return fresh
}
/** Stops listening; the pipeline can be started again. */
@@ -142,6 +192,7 @@ class ReceiverPipeline(
synchronized(decoderLock) {
decoder?.release()
decoder = null
pendingOffer = null
}
firstFrameSeen.set(false)
onStatus("Stopped")
@@ -154,23 +205,23 @@ class ReceiverPipeline(
}
// The offer's width/height are informational (the C++ sender leaves
// them 0); the bitstream carries SPS/PPS at every keyframe.
val fresh = H264Decoder()
val surface = currentSurface
var ok = true
var decoderReady = true
synchronized(decoderLock) {
pendingOffer = null
decoder?.release()
try {
// Surface first: configure() needs it before the codec starts.
surface?.let { fresh.attachSurface(it) }
fresh.configure(offer.width, offer.height)
decoder = fresh
} catch (e: Exception) {
ok = false
decoder = null
decoder = null
if (currentSurface != null) {
decoder = configureDecoderLocked(offer.width, offer.height)
decoderReady = decoder != null
} else {
// No surface yet (the window is between surfaces): remember
// the offer; attachSurface configures the decoder. Configuring
// without a surface is not an option — that codec could never
// take one afterwards (setOutputSurface refuses it).
pendingOffer = offer
}
}
if (!ok) {
fresh.release()
if (!decoderReady) {
onStatus("Could not start the decoder")
return
}
@@ -193,7 +244,10 @@ class ReceiverPipeline(
displayHeight = displayHeight,
),
)
onStatus("Session ${offer.sessionId} negotiated — waiting for the first frame…")
onStatus(
if (decoder != null) "Session ${offer.sessionId} negotiated — waiting for the first frame…"
else "Session ${offer.sessionId} negotiated — waiting for the display surface…"
)
}
private fun readLoop() {
@@ -224,19 +278,28 @@ class ReceiverPipeline(
val accessUnit = result.accessUnit
if (accessUnit != null) {
val activeDecoder = synchronized(decoderLock) { decoder }
if (activeDecoder == null) return
val presentationUs = packet.header.timestamp.toLong() and 0xFFFFFFFFL
try {
val presentationUs = packet.header.timestamp.toLong() and 0xFFFFFFFFL
activeDecoder.feed(accessUnit, presentationUs, result.isKeyFrame)
activeDecoder.drain()
if (firstFrameSeen.compareAndSet(false, true)) {
onFirstFrame()
// MediaCodec is not thread-safe: attachSurface() (main
// thread) and onOffer (signaling thread) synchronize on the
// same lock around their codec calls.
synchronized(decoderLock) {
val activeDecoder = decoder ?: return
if (!activeDecoder.feed(accessUnit, presentationUs, result.isKeyFrame)) {
// Input queue full: the dropped frame corrupts the
// GOP until the next keyframe — ask for one. No
// flush; the codec is healthy, merely busy.
requestPli()
}
activeDecoder.drain()
if (firstFrameSeen.compareAndSet(false, true)) {
onFirstFrame()
}
updateVideoSize()
}
updateVideoSize()
} catch (e: Exception) {
onStatus("Decoder error (${e.message}) — requesting a keyframe…")
activeDecoder.flush()
synchronized(decoderLock) { decoder?.flush() }
requestPli()
}
}
@@ -247,7 +310,7 @@ class ReceiverPipeline(
private fun updateVideoSize() {
val size = synchronized(decoderLock) { decoder?.outputSize() } ?: return
if (size != null && (size.first != videoWidth || size.second != videoHeight)) {
if (size.first != videoWidth || size.second != videoHeight) {
videoWidth = size.first
videoHeight = size.second
android.util.Log.i(TAG, "video size: ${size.first}x${size.second}")
@@ -255,15 +318,18 @@ class ReceiverPipeline(
}
}
// Rate-limited keyframe request (single-threaded access from the reader).
// Rate-limited keyframe request, callable from any thread (the reader
// thread and the main thread's attachSurface both reach it).
private fun requestPli() {
val session = activeSession
if (session.isEmpty()) return
val now = System.currentTimeMillis()
if (now - lastPliAtMs < PLI_MIN_INTERVAL_MS) return
lastPliAtMs = now
synchronized(pliLock) {
if (now - lastPliAtMs < PLI_MIN_INTERVAL_MS) return
lastPliAtMs = now
}
android.util.Log.i(TAG, "PLI requested for session $session")
signaling?.send(screen_cast.signaling.SessionPli(session))
signaling?.send(SessionPli(session))
}
private fun advertiseNsd(port: Int) {
@@ -272,70 +338,38 @@ class ReceiverPipeline(
setServiceType(SERVICE_TYPE)
setPort(port)
}
nsdInfo = info
// The NSD registration API was replaced in API 36 (ResolutionListener
// removed). Try the new API first, then the legacy one via reflection
// so the same build works on both.
try {
val listener = object : NsdManager.RegistrationListener {
override fun onServiceRegistered(serviceInfo: NsdServiceInfo) {
// Registered: the sender's mDNS browser should see it now.
}
override fun onServiceUnregistered(serviceInfo: NsdServiceInfo) = Unit
override fun onRegistrationFailed(serviceInfo: NsdServiceInfo, errorCode: Int) {
onStatus("mDNS registration failed — reach this receiver with --peer $localIp:$port")
}
override fun onUnregistrationFailed(serviceInfo: NsdServiceInfo, errorCode: Int) = Unit
// The classic RegistrationListener API exists from API 16 through 36
// (verified against the android-36 SDK with javap), so no fallback is
// needed — the previously reflected "ResolutionListener" never
// existed at any API level and could only ever fail.
val listener = object : NsdManager.RegistrationListener {
override fun onServiceRegistered(serviceInfo: NsdServiceInfo) {
// Registered: the sender's mDNS browser should see it now.
}
registrationListener = listener
nsdManager.registerService(info, 0, listener)
} catch (e: Throwable) {
try {
registerNsdLegacy(info, port)
} catch (e2: Throwable) {
onStatus("mDNS unavailable (${e2.message}) — reach this receiver with --peer $localIp:$port")
}
}
}
/** Pre-API-36 registration API (removed from the API 36 SDK; reflection). */
private fun registerNsdLegacy(info: NsdServiceInfo, port: Int) {
val listenerClass = Class.forName("android.net.nsd.NsdManager\$ResolutionListener")
val proxy = Proxy.newProxyInstance(
listenerClass.classLoader,
arrayOf(listenerClass),
) { _, method, _ ->
if (method.name == "onResolutionFailed") {
override fun onServiceUnregistered(serviceInfo: NsdServiceInfo) = Unit
override fun onRegistrationFailed(serviceInfo: NsdServiceInfo, errorCode: Int) {
onStatus("mDNS registration failed — reach this receiver with --peer $localIp:$port")
}
null
override fun onUnregistrationFailed(serviceInfo: NsdServiceInfo, errorCode: Int) = Unit
}
try {
nsdManager.registerService(info, 0, listener)
registrationListener = listener
} catch (e: Exception) {
onStatus("mDNS unavailable (${e.message}) — reach this receiver with --peer $localIp:$port")
}
legacyNsdListener = proxy
NsdManager::class.java
.getMethod("registerService", NsdServiceInfo::class.java, Int::class.javaPrimitiveType, listenerClass)
.invoke(nsdManager, info, 0, proxy)
}
private fun unregisterNsd() {
val newListener = registrationListener
val legacyListener = legacyNsdListener
val info = nsdInfo
val listener = registrationListener ?: return
try {
when {
newListener != null -> nsdManager.unregisterService(newListener)
legacyListener != null && info != null -> NsdManager::class.java
.getMethod("unregisterService", NsdServiceInfo::class.java)
.invoke(nsdManager, info)
else -> return
}
} catch (ignored: Throwable) {
nsdManager.unregisterService(listener)
} catch (ignored: Exception) {
// already unregistered
}
registrationListener = null
legacyNsdListener = null
nsdInfo = null
}
}
@@ -1,5 +1,7 @@
package screen_cast.rtp
import java.io.ByteArrayOutputStream
/** Result of feeding one packet to the depacketizer. */
data class DepacketizeResult(
/** Completed access unit (Annex-B with 3-byte start codes) when the frame closed undamaged. */
@@ -26,9 +28,11 @@ class H264Depacketizer {
private var frameStarted = false
private var frameDamaged = false
private var frameTimestamp = 0
private val accessUnit = ArrayList<Int>()
// Growable byte accumulators: keyframes reach hundreds of KB, and boxing
// each byte (the ArrayList<Int> this replaced) churned the GC hard.
private val accessUnit = ByteArrayOutputStream()
private var fuActive = false
private val fuNal = ArrayList<Int>()
private val fuNal = ByteArrayOutputStream()
/** Feed one packet (in sequence order, from the jitter buffer). */
fun depacketize(packet: RtpPacket): DepacketizeResult {
@@ -39,7 +43,7 @@ class H264Depacketizer {
val expected = (last + 1) and 0xFFFF
if (packet.header.sequenceNumber != expected) {
fuActive = false
fuNal.clear()
fuNal.reset()
if (frameStarted) {
frameDamaged = true
}
@@ -57,7 +61,7 @@ class H264Depacketizer {
frameStarted = true
frameDamaged = false
frameTimestamp = packet.header.timestamp
accessUnit.clear()
accessUnit.reset()
}
val payload = packet.payload
@@ -70,10 +74,10 @@ class H264Depacketizer {
// The previous fragmented NAL never received its end packet.
frameDamaged = true
fuActive = false
fuNal.clear()
fuNal.reset()
}
appendStartCode()
payload.forEach { accessUnit.add(it.toInt() and 0xFF) }
accessUnit.write(payload)
}
type == FU_A -> {
@@ -91,11 +95,11 @@ class H264Depacketizer {
frameDamaged = true
}
fuActive = true
fuNal.clear()
fuNal.reset()
// The FU indicator keeps the original NAL's F bit (0) and NRI,
// and declares type 28; the FU header carries S/E plus the real type.
fuNal.add((payload[0].toInt() and 0xE0) or (fuHeader and 0x1F))
fragment.forEach { fuNal.add(it.toInt() and 0xFF) }
fuNal.write((payload[0].toInt() and 0xE0) or (fuHeader and 0x1F))
fuNal.write(fragment)
}
!fuActive -> {
@@ -104,12 +108,12 @@ class H264Depacketizer {
}
else -> {
fragment.forEach { fuNal.add(it.toInt() and 0xFF) }
fuNal.write(fragment)
if (end) {
appendStartCode()
accessUnit.addAll(fuNal)
fuNal.writeTo(accessUnit)
fuActive = false
fuNal.clear()
fuNal.reset()
}
}
}
@@ -132,11 +136,11 @@ class H264Depacketizer {
// The marker arrived while a NAL was still fragmented.
frameDamaged = true
fuActive = false
fuNal.clear()
fuNal.reset()
}
if (!frameDamaged && accessUnit.isNotEmpty()) {
val unit = ByteArray(accessUnit.size) { accessUnit[it].toByte() }
if (!frameDamaged && accessUnit.size() > 0) {
val unit = accessUnit.toByteArray()
result = result.copy(accessUnit = unit, isKeyFrame = containsParameterSets(unit))
} else {
// The frame that just ended is unusable.
@@ -147,9 +151,9 @@ class H264Depacketizer {
}
private fun appendStartCode() {
accessUnit.add(0)
accessUnit.add(0)
accessUnit.add(1)
accessUnit.write(0)
accessUnit.write(0)
accessUnit.write(1)
}
/** The sender repeats SPS/PPS in-band at every keyframe; sniff for NAL types 7/8. */
@@ -168,8 +172,8 @@ class H264Depacketizer {
private fun dropFrame() {
frameStarted = false
frameDamaged = false
accessUnit.clear()
accessUnit.reset()
fuActive = false
fuNal.clear()
fuNal.reset()
}
}
@@ -34,12 +34,29 @@ class SignalingServer(
return server.localPort
}
/**
* Sends to the current peer; never throws. A lost control message (a
* PLI, an answer) is recoverable — the session re-negotiates or the next
* keyframe arrives — but an exception here would kill the RTP reader
* thread, which reaches send() from requestPli(). Mirrors the C++ server,
* which ignores write failures.
*/
fun send(message: SignalingMessage) {
val bytes = SignalingMessage.serialize(message).toByteArray(Charsets.UTF_8)
val bytes = try {
SignalingMessage.serialize(message).toByteArray(Charsets.UTF_8)
} catch (e: Exception) {
return // unreachable for our message types; serialize is total
}
synchronized(peerLock) {
peerOut?.let { out ->
val out = peerOut ?: return
try {
out.write(bytes)
out.flush()
} catch (e: Exception) {
// The peer is gone (e.g. the sender died while media still
// flows). Dropping peerOut also closes the socket; the next
// accepted offer installs a fresh one.
peerOut = null
}
}
}