From 3b59dd31ae3eb3a1090dd8ad70a1de08ce279cce Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 4 Jun 2026 16:54:39 -0400 Subject: [PATCH 1/5] feat: Enhance Android H.264 streaming support and refactor related components - Updated StreamQualityLimits struct to be cloneable and public. - Introduced stream_quality_limits_for_payload function to resolve quality limits from payload. - Refactored Android H.264 socket handling to utilize new quality resolution logic. - Renamed native encoder function to encode BGRA frames instead of RGBA. - Removed deprecated RGBA WebRTC transport handling in favor of H.264. - Implemented shared video frame encoding for Android using native H.264 encoder. - Improved error handling and logging for stream quality updates and encoding failures. - Updated SimDeck documentation to reflect changes in Android live viewing capabilities. --- README.md | 2 +- docs/api/rest.md | 2 +- docs/guide/architecture.md | 2 +- docs/guide/troubleshooting.md | 2 + docs/guide/video.md | 15 +- packages/client/src/api/types.ts | 2 +- .../stream/streamWorkerClient.test.ts | 66 +- .../src/features/stream/streamWorkerClient.ts | 337 +----- packages/server/Cargo.lock | 295 +---- packages/server/Cargo.toml | 3 - .../server/native/bridge/XCWNativeBridge.h | 1 + .../server/native/bridge/XCWNativeBridge.m | 106 ++ packages/server/src/android.rs | 1063 ++++++++--------- packages/server/src/api/routes.rs | 55 +- packages/server/src/native/ffi.rs | 4 +- packages/server/src/transport/webrtc.rs | 843 ++++--------- skills/simdeck/SKILL.md | 2 +- 17 files changed, 920 insertions(+), 1880 deletions(-) diff --git a/README.md b/README.md index 8b1a7942..449ba0fc 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ view inside the editor. ## Features -- Supports streaming both iOS simulators and Android emulators +- Supports native H.264 streaming for both iOS simulators and Android emulators - Full simulator control & inspection using private iOS accessibility APIs and Android UIAutomator - available using `simdeck` CLI - Real-time screen `describe` command using accessibility view tree - available in token-efficient format for agents - Profiling built-in: CPU, memory, disk writes, network throughput, hang signals, and stack sampling diff --git a/docs/api/rest.md b/docs/api/rest.md index 5e5828fa..5ccd3d0f 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -173,7 +173,7 @@ Performance query parameters: | `GET` | `/api/simulators/{udid}/control` | Alias for input control WebSocket | | `POST` | `/api/simulators/{udid}/refresh` | Request a fresh frame or keyframe | -For normal clients, copy the browser behavior instead of hand-coding a raw decoder. The UI supports WebRTC first and H.264 WebSocket fallback. +For normal clients, copy the browser behavior instead of hand-coding a raw decoder. The UI supports WebRTC first and H.264 WebSocket fallback. Android emulator IDs use the same endpoints; their H.264 frames are produced from the emulator `-share-vid` display surface, not screenshot polling. Minimal WebRTC request: diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index a0a5a77f..5e3339b1 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -26,7 +26,7 @@ This is why a long-lived service feels faster than repeatedly calling lower-leve ## Video flow -The browser opens a live stream for the selected device. SimDeck sends fresh frames, drops stale ones when a client falls behind, and lets the browser request refreshes. The UI can use WebRTC or H.264-over-WebSocket fallback depending on browser support and network behavior. +The browser opens a live stream for the selected device. SimDeck sends fresh frames, drops stale ones when a client falls behind, and lets the browser request refreshes. iOS frames come from the native display bridge and are encoded on the Mac; Android frames come from the emulator `-share-vid` shared display surface and are encoded on the Mac. The UI can use WebRTC or H.264-over-WebSocket fallback depending on browser support and network behavior. Tune this from the user-facing controls or with: diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index 9f3830c1..99f02ea2 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -137,6 +137,8 @@ curl http://127.0.0.1:4310/api/metrics If `frames_dropped_server` keeps climbing, the client or network cannot keep up. Move closer to the host, reduce quality, or switch to software encoding. +For Android emulator streams, SimDeck uses the emulator `-share-vid` shared display surface. If Android video never starts, confirm `adb devices` shows the emulator as `device`, that it has fully booted, and that externally launched emulators were started with `-share-vid`. SimDeck-owned Android boots add the flag automatically. + ### Browser cannot establish WebRTC Force the H.264 WebSocket fallback while testing: diff --git a/docs/guide/video.md b/docs/guide/video.md index 372ca1b4..cdb9d108 100644 --- a/docs/guide/video.md +++ b/docs/guide/video.md @@ -3,12 +3,15 @@ SimDeck streams live device video to the browser. Local sessions default to high quality. Remote or constrained sessions can trade detail for lower CPU and latency. iOS simulator H.264 uses VideoToolbox for hardware encoding and x264 for software encoding. +Android emulator H.264 uses the emulator `-share-vid` display surface. SimDeck reads BGRA frames from the `videmulator` shared memory region and encodes them on the Mac, so normal Android live video stays on the native shared display path. ## When encoding runs -SimDeck starts encoding when a browser stream needs H.264 frames. The server -requests an initial keyframe to answer the WebRTC or H.264 WebSocket viewer, -then keeps a shared refresh pump active while frame subscribers exist. +SimDeck starts encoding when a browser stream needs H.264 frames. For iOS, the +server requests an initial keyframe to answer the WebRTC or H.264 WebSocket +viewer, then keeps a shared refresh pump active while frame subscribers exist. +For Android, SimDeck starts emulators with `-share-vid`, maps the shared display +region, and feeds changed BGRA frames into the native host H.264 encoder. The browser reports whether the page and stream canvas are foreground. When all known viewers are hidden or the last frame subscriber disconnects, the native @@ -58,6 +61,12 @@ simdeck service restart --video-codec software | `hardware` | Dedicated local machines where VideoToolbox hardware H.264 is reliable. | | `software` | x264 software H.264 for CI, screen recording conflicts, or hardware encoder stalls. | +The codec setting controls iOS simulator host encoding. Android emulator streams +use a dedicated host encoder for shared display frames; set +`SIMDECK_ANDROID_VIDEO_CODEC=hardware` or `software` before starting the service +when you need to override Android's encoder choice. Stream quality controls the +encoded Android frame size. + When multiple simulator streams run at the same time, `auto` keeps one active stream on the hardware encoder path and routes additional active auto streams to software encoding. This avoids saturating the shared VideoToolbox hardware diff --git a/packages/client/src/api/types.ts b/packages/client/src/api/types.ts index 2c9b8f05..e6da623a 100644 --- a/packages/client/src/api/types.ts +++ b/packages/client/src/api/types.ts @@ -50,7 +50,7 @@ export interface SimulatorMetadata { isBooted: boolean; android?: { avdName?: string; - grpcPort?: number; + consolePort?: number; serial?: string; }; privateDisplay?: PrivateDisplayInfo; diff --git a/packages/client/src/features/stream/streamWorkerClient.test.ts b/packages/client/src/features/stream/streamWorkerClient.test.ts index eddc2054..03e6ab3a 100644 --- a/packages/client/src/features/stream/streamWorkerClient.test.ts +++ b/packages/client/src/features/stream/streamWorkerClient.test.ts @@ -4,7 +4,6 @@ import { buildStreamTarget, initialStreamBackend, preferredStreamBackend, - shouldUseLocalAndroidRgbaWebRtc, } from "./streamWorkerClient"; describe("streamWorkerClient", () => { @@ -25,16 +24,16 @@ describe("streamWorkerClient", () => { expect(preferredStreamBackend(target)).toBe("webrtc"); }); - it("treats explicit RGBA transport as a WebRTC backend", () => { + it("ignores unknown stream query parameters", () => { const previousWindow = globalThis.window; Object.defineProperty(globalThis, "window", { configurable: true, - value: { location: { search: "?stream=rgba" } }, + value: { location: { search: "?stream=unknown" } }, }); try { expect(preferredStreamBackend(buildStreamTarget("android:Pixel_8"))).toBe( - "webrtc", + "auto", ); } finally { Object.defineProperty(globalThis, "window", { @@ -62,63 +61,4 @@ describe("streamWorkerClient", () => { ).RTCPeerConnection = previousPeerConnection; } }); - - it("uses RGBA WebRTC transport for local loopback Android streams", () => { - const previousWindow = globalThis.window; - Object.defineProperty(globalThis, "window", { - configurable: true, - value: { location: { hostname: "127.0.0.1", search: "" } }, - }); - - try { - expect( - shouldUseLocalAndroidRgbaWebRtc( - buildStreamTarget("android:Pixel_8", { - platform: "android-emulator", - transport: "auto", - }), - ), - ).toBe(true); - } finally { - Object.defineProperty(globalThis, "window", { - configurable: true, - value: previousWindow, - }); - } - }); - - it("keeps Android RGBA disabled for h264 or remote streams", () => { - const previousWindow = globalThis.window; - const location = { hostname: "127.0.0.1", search: "?stream=h264" }; - Object.defineProperty(globalThis, "window", { - configurable: true, - value: { location }, - }); - - try { - expect( - shouldUseLocalAndroidRgbaWebRtc( - buildStreamTarget("android:Pixel_8", { - platform: "android-emulator", - transport: "auto", - }), - ), - ).toBe(false); - location.search = ""; - expect( - shouldUseLocalAndroidRgbaWebRtc( - buildStreamTarget("android:Pixel_8", { - platform: "android-emulator", - remote: true, - transport: "auto", - }), - ), - ).toBe(false); - } finally { - Object.defineProperty(globalThis, "window", { - configurable: true, - value: previousWindow, - }); - } - }); }); diff --git a/packages/client/src/features/stream/streamWorkerClient.ts b/packages/client/src/features/stream/streamWorkerClient.ts index b49842a4..fdcee912 100644 --- a/packages/client/src/features/stream/streamWorkerClient.ts +++ b/packages/client/src/features/stream/streamWorkerClient.ts @@ -18,7 +18,6 @@ import type { const HAVE_CURRENT_DATA = 2; const WEBRTC_CONTROL_CHANNEL_LABEL = "simdeck-control"; const WEBRTC_TELEMETRY_CHANNEL_LABEL = "simdeck-telemetry"; -const WEBRTC_RGBA_CHANNEL_LABEL = "simdeck-rgba"; const WEBRTC_FIRST_FRAME_TIMEOUT_MS = 10000; const WEBRTC_STALLED_FRAME_TIMEOUT_MS = 3000; const WEBRTC_LOCAL_RECEIVER_BUFFER_SECONDS = 0.001; @@ -34,10 +33,6 @@ const H264_WS_HEADER_BYTES = 40; const H264_WS_MAGIC = 0x53444831; const H264_WS_FLAG_KEYFRAME = 1 << 0; const H264_WS_FLAG_CONFIG = 1 << 1; -const WEBRTC_RGBA_CHUNK_HEADER_BYTES = 48; -const WEBRTC_RGBA_CHUNK_MAGIC = 0x53445243; -const WEBRTC_RGBA_VERSION = 1; -const WEBRTC_RGBA_FORMAT_RGBA8888 = 1; const H264_WS_LOCAL_AUTO_PROFILES: StreamQualityPreset[] = [ "low", "economy", @@ -309,34 +304,6 @@ interface H264WebSocketFrame { width: number; } -interface WebRtcRgbaFrame { - height: number; - payload: Uint8Array; - sequence: number; - timestampUs: number; - width: number; -} - -interface WebRtcRgbaChunk { - chunkOffset: number; - height: number; - payload: Uint8Array; - payloadBytes: number; - sequence: number; - timestampUs: number; - width: number; -} - -interface WebRtcRgbaAssembly { - buffer: Uint8Array; - height: number; - receivedBytes: number; - receivedRanges: Array<[number, number]>; - sequence: number; - timestampUs: number; - width: number; -} - interface WebCodecsVideoFrame { close(): void; codedHeight?: number; @@ -1185,53 +1152,6 @@ function parseH264WebSocketFrame(data: unknown): H264WebSocketFrame | null { }; } -function parseWebRtcRgbaChunk(data: unknown): WebRtcRgbaChunk | null { - const bytes = bytesFromBinaryMessage(data); - if (!bytes || bytes.byteLength < WEBRTC_RGBA_CHUNK_HEADER_BYTES) { - return null; - } - const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); - if ( - view.getUint32(0, false) !== WEBRTC_RGBA_CHUNK_MAGIC || - view.getUint8(4) !== WEBRTC_RGBA_VERSION || - view.getUint8(5) !== WEBRTC_RGBA_FORMAT_RGBA8888 - ) { - return null; - } - const headerBytes = view.getUint16(6, false); - if ( - headerBytes < WEBRTC_RGBA_CHUNK_HEADER_BYTES || - headerBytes > bytes.byteLength - ) { - return null; - } - const width = view.getUint32(24, false); - const height = view.getUint32(28, false); - const payloadBytes = view.getUint32(32, false); - const chunkOffset = view.getUint32(36, false); - const chunkBytes = view.getUint32(40, false); - const payloadEnd = headerBytes + chunkBytes; - if ( - width <= 0 || - height <= 0 || - payloadBytes !== width * height * 4 || - chunkBytes <= 0 || - chunkOffset + chunkBytes > payloadBytes || - payloadEnd > bytes.byteLength - ) { - return null; - } - return { - chunkOffset, - height, - payload: bytes.subarray(headerBytes, payloadEnd), - payloadBytes, - sequence: Number(view.getBigUint64(8, false)), - timestampUs: Number(view.getBigUint64(16, false)), - width, - }; -} - function bytesFromBinaryMessage(data: unknown): Uint8Array | null { if (ArrayBuffer.isView(data)) { return new Uint8Array(data.buffer, data.byteOffset, data.byteLength); @@ -1259,16 +1179,6 @@ function bytesFromBinaryMessage(data: unknown): Uint8Array | null { return null; } -function rangeAlreadyReceived( - ranges: Array<[number, number]>, - start: number, - end: number, -): boolean { - return ranges.some( - ([rangeStart, rangeEnd]) => start >= rangeStart && end <= rangeEnd, - ); -} - function h264CodecStringFromAvcC(config: Uint8Array): string | null { if (config.byteLength < 4 || config[0] !== 1) { return null; @@ -1318,10 +1228,6 @@ class WebRtcStreamClient implements StreamClientBackend { private receiverStatsSeen = false; private streamingReported = false; private shouldReconnect = false; - private latestRgbaSequence = -1; - private rgbaAssemblies = new Map(); - private rgbaChannel: RTCDataChannel | null = null; - private rgbaMode = false; private streamConfigGeneration = 0; private streamTarget: StreamConnectTarget | null = null; private telemetryChannel: RTCDataChannel | null = null; @@ -1432,20 +1338,16 @@ class WebRtcStreamClient implements StreamClientBackend { iceTransportPolicy: iceTransportPolicy(health), }); this.peerConnection = peerConnection; - const useRgbaTransport = shouldUseLocalAndroidRgbaWebRtc(target); - this.rgbaMode = useRgbaTransport; this.attachDiagnostics(peerConnection, target, generation); - if (!useRgbaTransport) { - this.startReceiverStatsPolling(peerConnection, target, generation); - const transceiver = peerConnection.addTransceiver("video", { - direction: "recvonly", - }); - configureReceiverCodecPreferences(transceiver); - configureLowLatencyReceiver( - transceiver.receiver, - receiverBufferSeconds(target), - ); - } + this.startReceiverStatsPolling(peerConnection, target, generation); + const transceiver = peerConnection.addTransceiver("video", { + direction: "recvonly", + }); + configureReceiverCodecPreferences(transceiver); + configureLowLatencyReceiver( + transceiver.receiver, + receiverBufferSeconds(target), + ); const controlChannel = peerConnection.createDataChannel( WEBRTC_CONTROL_CHANNEL_LABEL, { @@ -1474,20 +1376,7 @@ class WebRtcStreamClient implements StreamClientBackend { } }); - peerConnection.ondatachannel = (event) => { - if ( - generation !== this.connectGeneration || - event.channel.label !== WEBRTC_RGBA_CHANNEL_LABEL - ) { - return; - } - this.attachRgbaDataChannel(event.channel, generation); - }; - peerConnection.ontrack = (event) => { - if (useRgbaTransport) { - return; - } if (generation !== this.connectGeneration) { return; } @@ -1581,7 +1470,6 @@ class WebRtcStreamClient implements StreamClientBackend { await this.negotiatePeerConnection(peerConnection, target, generation, { detailPrefix: "local", - transport: useRgbaTransport ? "rgba" : undefined, }); this.scheduleFrameWatchdog(target, generation); } catch (error) { @@ -1627,7 +1515,6 @@ class WebRtcStreamClient implements StreamClientBackend { options: { detailPrefix: string; iceRestart?: boolean; - transport?: "rgba"; }, ) { const offer = safariBaselineH264Offer( @@ -1661,7 +1548,6 @@ class WebRtcStreamClient implements StreamClientBackend { const response = await postWebRtcOfferWithAuthRetry( target, localDescription, - options.transport, ); const answer = (await response.json()) as WebRtcAnswerPayload; if (generation !== this.connectGeneration) { @@ -1715,11 +1601,6 @@ class WebRtcStreamClient implements StreamClientBackend { activeWebRtcTelemetryChannel = null; } this.telemetryChannel = null; - this.rgbaChannel?.close(); - this.rgbaChannel = null; - this.rgbaAssemblies.clear(); - this.latestRgbaSequence = -1; - this.rgbaMode = false; this.peerConnection?.close(); this.peerConnection = null; } @@ -2122,172 +2003,6 @@ class WebRtcStreamClient implements StreamClientBackend { } } - private attachRgbaDataChannel(channel: RTCDataChannel, generation: number) { - this.rgbaChannel?.close(); - this.rgbaChannel = channel; - channel.binaryType = "arraybuffer"; - channel.addEventListener("message", (event) => { - if ( - generation !== this.connectGeneration || - channel !== this.rgbaChannel - ) { - return; - } - if (hasArrayBufferMethod(event.data)) { - void event.data.arrayBuffer().then((buffer) => { - if ( - generation === this.connectGeneration && - channel === this.rgbaChannel - ) { - this.handleRgbaMessage(buffer); - } - }); - return; - } - this.handleRgbaMessage(event.data); - }); - channel.addEventListener("close", () => { - if (this.rgbaChannel === channel) { - this.rgbaChannel = null; - } - }); - } - - private handleRgbaMessage(data: unknown) { - const chunk = parseWebRtcRgbaChunk(data); - if (!chunk) { - this.stats.h264ParseFailures += 1; - this.onMessage({ type: "stats", stats: { ...this.stats } }); - return; - } - this.stats.receivedPackets += 1; - if (chunk.sequence < this.latestRgbaSequence) { - this.stats.droppedFrames += 1; - this.onMessage({ type: "stats", stats: { ...this.stats } }); - return; - } - const frame = this.appendRgbaChunk(chunk); - if (!frame) { - return; - } - this.stats.decodedFrames += 1; - this.stats.width = frame.width; - this.stats.height = frame.height; - this.stats.frameSequence = frame.sequence; - this.stats.codec = "webrtc-rgba"; - this.latestRgbaSequence = frame.sequence; - this.paintRgbaFrame(frame); - } - - private appendRgbaChunk(chunk: WebRtcRgbaChunk): WebRtcRgbaFrame | null { - let assembly = this.rgbaAssemblies.get(chunk.sequence); - if ( - assembly && - (assembly.width !== chunk.width || - assembly.height !== chunk.height || - assembly.buffer.byteLength !== chunk.payloadBytes) - ) { - this.rgbaAssemblies.delete(chunk.sequence); - assembly = undefined; - } - if (!assembly) { - assembly = { - buffer: new Uint8Array(chunk.payloadBytes), - height: chunk.height, - receivedBytes: 0, - receivedRanges: [], - sequence: chunk.sequence, - timestampUs: chunk.timestampUs, - width: chunk.width, - }; - this.rgbaAssemblies.set(chunk.sequence, assembly); - this.trimRgbaAssemblies(chunk.sequence); - } - assembly.buffer.set(chunk.payload, chunk.chunkOffset); - const chunkEnd = chunk.chunkOffset + chunk.payload.byteLength; - if ( - !rangeAlreadyReceived( - assembly.receivedRanges, - chunk.chunkOffset, - chunkEnd, - ) - ) { - assembly.receivedRanges.push([chunk.chunkOffset, chunkEnd]); - assembly.receivedBytes += chunk.payload.byteLength; - } - if (assembly.receivedBytes < assembly.buffer.byteLength) { - return null; - } - this.rgbaAssemblies.delete(chunk.sequence); - return { - height: assembly.height, - payload: assembly.buffer, - sequence: assembly.sequence, - timestampUs: assembly.timestampUs, - width: assembly.width, - }; - } - - private trimRgbaAssemblies(latestSequence: number) { - while (this.rgbaAssemblies.size > 3) { - const firstKey = this.rgbaAssemblies.keys().next().value; - if (typeof firstKey !== "number") { - break; - } - this.rgbaAssemblies.delete(firstKey); - } - for (const sequence of this.rgbaAssemblies.keys()) { - if (latestSequence - sequence > 2) { - this.rgbaAssemblies.delete(sequence); - } - } - } - - private paintRgbaFrame(frame: WebRtcRgbaFrame) { - const canvas = this.canvas; - const context = this.ensureCanvasContext(); - if (!canvas || !context) { - return; - } - this.syncCanvasSize(frame.width, frame.height); - const startedAt = performance.now(); - const rgba = new Uint8ClampedArray( - frame.payload.buffer as ArrayBuffer, - frame.payload.byteOffset, - frame.payload.byteLength, - ); - try { - context.putImageData( - new ImageData(rgba, frame.width, frame.height), - 0, - 0, - ); - } catch { - this.stats.droppedFrames += 1; - this.onMessage({ type: "stats", stats: { ...this.stats } }); - return; - } - const finishedAt = performance.now(); - const previousFrameAt = this.lastVideoFrameAt; - this.lastVideoFrameAt = finishedAt; - this.hasRenderedFrame = true; - this.stats.renderedFrames += 1; - this.stats.latestRenderMs = finishedAt - startedAt; - this.stats.maxRenderMs = Math.max( - this.stats.maxRenderMs, - this.stats.latestRenderMs, - ); - this.stats.averageRenderMs = - this.stats.averageRenderMs <= 0 - ? this.stats.latestRenderMs - : this.stats.averageRenderMs * 0.9 + this.stats.latestRenderMs * 0.1; - this.stats.latestFrameGapMs = - previousFrameAt > 0 ? finishedAt - previousFrameAt : 0; - this.reportVideoConfig(frame.width, frame.height); - this.onMessage({ type: "stats", stats: { ...this.stats } }); - this.reportWebRtcStreaming(); - } - private drawVideoFrame = () => { this.videoFrameCallback = 0; if (!this.canvas || !this.video) { @@ -2362,9 +2077,7 @@ class WebRtcStreamClient implements StreamClientBackend { this.onMessage({ type: "status", status: { - detail: this.rgbaMode - ? "WebRTC RGBA first frame rendered" - : "WebRTC first video frame rendered", + detail: "WebRTC first video frame rendered", state: "streaming", }, }); @@ -2546,9 +2259,8 @@ function streamErrorIsServerUnreachable(message: string): boolean { async function postWebRtcOfferWithAuthRetry( target: StreamConnectTarget, localDescription: RTCSessionDescription, - transport?: "rgba", ): Promise { - const response = await postWebRtcOffer(target, localDescription, transport); + const response = await postWebRtcOffer(target, localDescription); if (response.status !== 401) { if (!response.ok) { throw new Error(await response.text()); @@ -2559,7 +2271,7 @@ async function postWebRtcOfferWithAuthRetry( throw new Error(await response.text()); } await fetchHealth(); - const retry = await postWebRtcOffer(target, localDescription, transport); + const retry = await postWebRtcOffer(target, localDescription); if (!retry.ok) { throw new Error(await retry.text()); } @@ -2625,7 +2337,6 @@ function streamQualityQuery(config: StreamConfig | undefined): string { function postWebRtcOffer( target: StreamConnectTarget, localDescription: RTCSessionDescription, - transport?: "rgba", ): Promise { return fetch( apiUrl(`/api/simulators/${encodeURIComponent(target.udid)}/webrtc/offer`), @@ -2636,7 +2347,6 @@ function postWebRtcOffer( streamConfig: target.streamConfig ? streamQualityPayload(target.streamConfig) : undefined, - transport, type: localDescription.type, }), headers: apiHeaders(), @@ -2688,25 +2398,6 @@ function shouldUseRemoteH264AutoProfile( return Boolean(target?.remote) || !isLoopbackHost(window.location.hostname); } -export function shouldUseLocalAndroidRgbaWebRtc( - target: StreamConnectTarget, -): boolean { - const stream = new URLSearchParams(window.location.search).get("stream"); - if ( - target.transport === "h264" || - stream === "h264" || - stream === "h264-ws" - ) { - return false; - } - return ( - !target.remote && - isLoopbackHost(window.location.hostname) && - (target.platform === "android-emulator" || - target.udid.startsWith("android:")) - ); -} - function configureLowLatencyReceiver( receiver: RTCRtpReceiver, bufferSeconds: number | null, @@ -3113,9 +2804,7 @@ export function preferredStreamBackend( if (value === "h264" || value === "h264-ws") { return "h264-ws"; } - return value === "webrtc" || value === "rgba" || value === "webrtc-rgba" - ? "webrtc" - : "auto"; + return value === "webrtc" ? "webrtc" : "auto"; } export function initialStreamBackend( diff --git a/packages/server/Cargo.lock b/packages/server/Cargo.lock index 49029348..3a6f0bf0 100644 --- a/packages/server/Cargo.lock +++ b/packages/server/Cargo.lock @@ -150,28 +150,6 @@ dependencies = [ "syn", ] -[[package]] -name = "async-stream" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "async-trait" version = "0.1.89" @@ -195,40 +173,13 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "axum" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" -dependencies = [ - "async-trait", - "axum-core 0.4.5", - "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "itoa", - "matchit 0.7.3", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper", - "tower 0.5.3", - "tower-layer", - "tower-service", -] - [[package]] name = "axum" version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ - "axum-core 0.5.6", + "axum-core", "base64", "bytes", "form_urlencoded", @@ -239,7 +190,7 @@ dependencies = [ "hyper", "hyper-util", "itoa", - "matchit 0.8.4", + "matchit", "memchr", "mime", "percent-encoding", @@ -252,32 +203,12 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-tungstenite", - "tower 0.5.3", + "tower", "tower-layer", "tower-service", "tracing", ] -[[package]] -name = "axum-core" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "rustversion", - "sync_wrapper", - "tower-layer", - "tower-service", -] - [[package]] name = "axum-core" version = "0.5.6" @@ -642,12 +573,6 @@ dependencies = [ "spki", ] -[[package]] -name = "either" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" - [[package]] name = "elliptic-curve" version = "0.13.8" @@ -707,12 +632,6 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - [[package]] name = "foldhash" version = "0.1.5" @@ -884,31 +803,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "h2" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap 2.14.0", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.15.5" @@ -1015,7 +909,6 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2", "http", "http-body", "httparse", @@ -1024,20 +917,6 @@ dependencies = [ "pin-project-lite", "smallvec", "tokio", - "want", -] - -[[package]] -name = "hyper-timeout" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" -dependencies = [ - "hyper", - "hyper-util", - "pin-project-lite", - "tokio", - "tower-service", ] [[package]] @@ -1047,17 +926,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ "bytes", - "futures-channel", - "futures-util", "http", "http-body", "hyper", - "libc", "pin-project-lite", - "socket2 0.6.3", "tokio", "tower-service", - "tracing", ] [[package]] @@ -1181,16 +1055,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - [[package]] name = "indexmap" version = "2.14.0" @@ -1245,15 +1109,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.18" @@ -1320,12 +1175,6 @@ dependencies = [ "regex-automata", ] -[[package]] -name = "matchit" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" - [[package]] name = "matchit" version = "0.8.4" @@ -1565,26 +1414,6 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" -[[package]] -name = "pin-project" -version = "1.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf0d9e68100b3a7989b4901972f265cd542e560a3a8a724e1e20322f4d06ce9" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a990e22f43e84855daf260dded30524ef4a9021cc7541c26540500a50b624389" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1614,7 +1443,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092791278e026273c1b65bbdcfbba3a300f2994c896bd01ab01da613c29c46f1" dependencies = [ "base64", - "indexmap 2.14.0", + "indexmap", "quick-xml", "serde", "time", @@ -1690,29 +1519,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "prost" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-derive" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "pxfm" version = "0.1.29" @@ -2099,7 +1905,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.14.0", + "indexmap", "itoa", "ryu", "serde", @@ -2168,7 +1974,7 @@ name = "simdeck-server" version = "0.1.29" dependencies = [ "anyhow", - "axum 0.8.9", + "axum", "base64", "bytes", "cc", @@ -2178,7 +1984,6 @@ dependencies = [ "http", "libc", "plist", - "prost", "qrcode", "regex", "roxmltree", @@ -2188,9 +1993,7 @@ dependencies = [ "sha2", "thiserror 2.0.18", "tokio", - "tokio-stream", "tokio-tungstenite", - "tonic", "tower-http", "tracing", "tracing-subscriber", @@ -2440,17 +2243,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tokio-stream" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-tungstenite" version = "0.29.0" @@ -2476,56 +2268,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tonic" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" -dependencies = [ - "async-stream", - "async-trait", - "axum 0.7.9", - "base64", - "bytes", - "h2", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-timeout", - "hyper-util", - "percent-encoding", - "pin-project", - "prost", - "socket2 0.5.10", - "tokio", - "tokio-stream", - "tower 0.4.13", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "indexmap 1.9.3", - "pin-project", - "pin-project-lite", - "rand 0.8.6", - "slab", - "tokio", - "tokio-util", - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "tower" version = "0.5.3" @@ -2642,12 +2384,6 @@ dependencies = [ "tracing-log", ] -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - [[package]] name = "tungstenite" version = "0.29.0" @@ -2787,15 +2523,6 @@ dependencies = [ "atomic-waker", ] -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -2882,7 +2609,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap 2.14.0", + "indexmap", "wasm-encoder", "wasmparser", ] @@ -2895,7 +2622,7 @@ checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ "bitflags 2.11.1", "hashbrown 0.15.5", - "indexmap 2.14.0", + "indexmap", "semver", ] @@ -3252,7 +2979,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck", - "indexmap 2.14.0", + "indexmap", "prettyplease", "syn", "wasm-metadata", @@ -3283,7 +3010,7 @@ checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", "bitflags 2.11.1", - "indexmap 2.14.0", + "indexmap", "log", "serde", "serde_derive", @@ -3302,7 +3029,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap 2.14.0", + "indexmap", "log", "semver", "serde", diff --git a/packages/server/Cargo.toml b/packages/server/Cargo.toml index 230acc2f..ae5586d8 100644 --- a/packages/server/Cargo.toml +++ b/packages/server/Cargo.toml @@ -15,7 +15,6 @@ hex = "0.4" http = "1.1" libc = "0.2" plist = "1.7" -prost = "0.13" qrcode = "0.14" roxmltree = "0.20" regex = "1.11" @@ -25,9 +24,7 @@ serde_yaml = "0.9" sha2 = "0.10" thiserror = "2.0" tokio = { version = "1.42", features = ["fs", "io-util", "macros", "process", "rt-multi-thread", "signal", "sync", "time"] } -tokio-stream = "0.1" tokio-tungstenite = "0.29" -tonic = { version = "0.12", features = ["transport"] } tower-http = { version = "0.6", features = ["cors", "fs", "trace"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } diff --git a/packages/server/native/bridge/XCWNativeBridge.h b/packages/server/native/bridge/XCWNativeBridge.h index 35ffc647..c54d287e 100644 --- a/packages/server/native/bridge/XCWNativeBridge.h +++ b/packages/server/native/bridge/XCWNativeBridge.h @@ -107,6 +107,7 @@ void xcw_native_session_set_frame_callback(void * _Nonnull handle, xcw_native_fr void * _Nullable xcw_native_h264_encoder_create(xcw_native_frame_callback _Nullable callback, void * _Nullable user_data, char * _Nullable * _Nullable error_message); void xcw_native_h264_encoder_destroy(void * _Nullable handle); bool xcw_native_h264_encoder_encode_rgba(void * _Nonnull handle, const uint8_t * _Nonnull rgba, size_t length, uint32_t width, uint32_t height, uint64_t timestamp_us, char * _Nullable * _Nullable error_message); +bool xcw_native_h264_encoder_encode_bgra(void * _Nonnull handle, const uint8_t * _Nonnull bgra, size_t length, uint32_t width, uint32_t height, uint64_t timestamp_us, char * _Nullable * _Nullable error_message); void xcw_native_h264_encoder_request_keyframe(void * _Nonnull handle); void xcw_native_free_string(char * _Nullable value); diff --git a/packages/server/native/bridge/XCWNativeBridge.m b/packages/server/native/bridge/XCWNativeBridge.m index 1c2746d7..a64fe336 100644 --- a/packages/server/native/bridge/XCWNativeBridge.m +++ b/packages/server/native/bridge/XCWNativeBridge.m @@ -161,6 +161,11 @@ - (BOOL)encodeRGBA:(const uint8_t *)rgba width:(uint32_t)width height:(uint32_t)height error:(NSError * _Nullable __autoreleasing *)error; +- (BOOL)encodeBGRA:(const uint8_t *)bgra + length:(size_t)length + width:(uint32_t)width + height:(uint32_t)height + error:(NSError * _Nullable __autoreleasing *)error; - (void)requestKeyFrame; - (void)invalidate; @@ -186,11 +191,14 @@ - (instancetype)initWithFrameCallback:(xcw_native_frame_callback)callback @synchronized (XCWNativeH264Encoder.class) { const char *previousCodec = getenv("SIMDECK_VIDEO_CODEC"); char *previousCodecCopy = previousCodec != NULL ? strdup(previousCodec) : NULL; + const char *previousRealtimeStream = getenv("SIMDECK_REALTIME_STREAM"); + char *previousRealtimeStreamCopy = previousRealtimeStream != NULL ? strdup(previousRealtimeStream) : NULL; const char *androidCodec = getenv("SIMDECK_ANDROID_VIDEO_CODEC"); if (androidCodec == NULL || strlen(androidCodec) == 0) { androidCodec = "software"; } setenv("SIMDECK_VIDEO_CODEC", androidCodec, 1); + setenv("SIMDECK_REALTIME_STREAM", "1", 1); _encoder = [[XCWH264Encoder alloc] initWithOutputHandler:^(NSData *sampleData, uint64_t timestampUs, BOOL isKeyFrame, @@ -220,6 +228,12 @@ - (instancetype)initWithFrameCallback:(xcw_native_frame_callback)callback } else { unsetenv("SIMDECK_VIDEO_CODEC"); } + if (previousRealtimeStreamCopy != NULL) { + setenv("SIMDECK_REALTIME_STREAM", previousRealtimeStreamCopy, 1); + free(previousRealtimeStreamCopy); + } else { + unsetenv("SIMDECK_REALTIME_STREAM"); + } } return self; } @@ -305,6 +319,76 @@ - (BOOL)encodeRGBA:(const uint8_t *)rgba return YES; } +- (BOOL)encodeBGRA:(const uint8_t *)bgra + length:(size_t)length + width:(uint32_t)width + height:(uint32_t)height + error:(NSError * _Nullable __autoreleasing *)error { + if (bgra == NULL || width == 0 || height == 0) { + if (error != NULL) { + *error = [NSError errorWithDomain:@"SimDeck.NativeH264Encoder" + code:1 + userInfo:@{ NSLocalizedDescriptionKey: @"BGRA frame input was empty." }]; + } + return NO; + } + size_t expectedLength = (size_t)width * (size_t)height * 4; + if (length < expectedLength) { + if (error != NULL) { + *error = [NSError errorWithDomain:@"SimDeck.NativeH264Encoder" + code:2 + userInfo:@{ NSLocalizedDescriptionKey: @"BGRA frame input was truncated." }]; + } + return NO; + } + + NSDictionary *attributes = @{ + (__bridge NSString *)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA), + (__bridge NSString *)kCVPixelBufferWidthKey: @(width), + (__bridge NSString *)kCVPixelBufferHeightKey: @(height), + (__bridge NSString *)kCVPixelBufferIOSurfacePropertiesKey: @{}, + }; + CVPixelBufferRef pixelBuffer = NULL; + CVReturn createStatus = CVPixelBufferCreate(kCFAllocatorDefault, + (size_t)width, + (size_t)height, + kCVPixelFormatType_32BGRA, + (__bridge CFDictionaryRef)attributes, + &pixelBuffer); + if (createStatus != kCVReturnSuccess || pixelBuffer == NULL) { + if (error != NULL) { + *error = [NSError errorWithDomain:@"SimDeck.NativeH264Encoder" + code:createStatus + userInfo:@{ NSLocalizedDescriptionKey: @"Unable to allocate a VideoToolbox pixel buffer." }]; + } + return NO; + } + + CVReturn lockStatus = CVPixelBufferLockBaseAddress(pixelBuffer, 0); + if (lockStatus != kCVReturnSuccess) { + CVPixelBufferRelease(pixelBuffer); + if (error != NULL) { + *error = [NSError errorWithDomain:@"SimDeck.NativeH264Encoder" + code:lockStatus + userInfo:@{ NSLocalizedDescriptionKey: @"Unable to lock a VideoToolbox pixel buffer." }]; + } + return NO; + } + + uint8_t *dst = CVPixelBufferGetBaseAddress(pixelBuffer); + size_t dstRowBytes = CVPixelBufferGetBytesPerRow(pixelBuffer); + size_t srcRowBytes = (size_t)width * 4; + for (uint32_t y = 0; y < height; y += 1) { + memcpy(dst + ((size_t)y * dstRowBytes), + bgra + ((size_t)y * srcRowBytes), + srcRowBytes); + } + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + [_encoder encodePixelBuffer:pixelBuffer]; + CVPixelBufferRelease(pixelBuffer); + return YES; +} + - (void)requestKeyFrame { [_encoder requestKeyFrame]; } @@ -1361,6 +1445,28 @@ bool xcw_native_h264_encoder_encode_rgba(void *handle, } } +bool xcw_native_h264_encoder_encode_bgra(void *handle, + const uint8_t *bgra, + size_t length, + uint32_t width, + uint32_t height, + uint64_t timestamp_us, + char **error_message) { + (void)timestamp_us; + @autoreleasepool { + NSError *error = nil; + BOOL ok = [XCWNativeH264EncoderFromHandle(handle) encodeBGRA:bgra + length:length + width:width + height:height + error:&error]; + if (!ok) { + XCWSetErrorMessage(error_message, error); + } + return ok; + } +} + void xcw_native_h264_encoder_request_keyframe(void *handle) { @autoreleasepool { [XCWNativeH264EncoderFromHandle(handle) requestKeyFrame]; diff --git a/packages/server/src/android.rs b/packages/server/src/android.rs index adc2c794..9a0768c8 100644 --- a/packages/server/src/android.rs +++ b/packages/server/src/android.rs @@ -1,22 +1,26 @@ use crate::error::AppError; -use bytes::BytesMut; -use http::uri::PathAndQuery; use serde_json::{json, Value}; use std::collections::HashMap; use std::env; +use std::ffi::CString; use std::ffi::OsString; use std::io::{Read, Write}; +use std::mem::MaybeUninit; +use std::os::fd::RawFd; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; +use std::ptr; use std::sync::{Mutex, OnceLock}; use std::thread; use std::time::{Duration, Instant}; -use tonic::metadata::MetadataValue; -use tonic::transport::Endpoint; const ANDROID_ID_PREFIX: &str = "android:"; -const DEFAULT_GRPC_PORT_BASE: u16 = 8554; -const ANDROID_GRPC_FRAME_MESSAGE_LIMIT: usize = 64 * 1024 * 1024; +const DEFAULT_EMULATOR_CONSOLE_PORT_BASE: u16 = 5554; +const ANDROID_H264_DEFAULT_MAX_EDGE: u32 = 1440; +const ANDROID_SHARED_VIDEO_HEADER_BYTES: usize = 3072; +const ANDROID_SHARED_VIDEO_PIXEL_BYTES: usize = 4; +const ANDROID_SHARED_VIDEO_MAX_DIMENSION: u32 = 8192; +const ANDROID_SHARED_VIDEO_MAX_FRAME_BYTES: usize = 256 * 1024 * 1024; const ANDROID_TOUCH_SWIPE_THRESHOLD: f64 = 0.025; const ANDROID_TOUCH_MIN_DURATION_MS: u128 = 80; const ANDROID_TOUCH_MAX_DURATION_MS: u128 = 1500; @@ -24,7 +28,7 @@ const ANDROID_COMMAND_TIMEOUT: Duration = Duration::from_secs(30); const ANDROID_UIAUTOMATOR_DUMP_ATTEMPTS: usize = 10; const ANDROID_UIAUTOMATOR_DUMP_RETRY_DELAY: Duration = Duration::from_millis(250); const RUNNING_EMULATOR_CACHE_TTL: Duration = Duration::from_secs(2); -const AVD_GRPC_PORT_CACHE_TTL: Duration = Duration::from_secs(60); +const AVD_CONSOLE_PORT_CACHE_TTL: Duration = Duration::from_secs(60); const SCREEN_SIZE_CACHE_TTL: Duration = Duration::from_secs(1); const MODIFIER_SHIFT: u32 = 1 << 0; const MODIFIER_CONTROL: u32 = 1 << 1; @@ -75,7 +79,7 @@ pub struct AndroidDevice { pub avd_name: String, pub serial: Option, pub is_booted: bool, - pub grpc_port: u16, + pub console_port: u16, } #[derive(Clone, Debug)] @@ -85,24 +89,38 @@ pub struct AndroidEmulatorSpec { pub system_image_identifier: String, } +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct AndroidH264StreamQuality { + pub max_edge: Option, + pub min_bitrate: Option, + pub bits_per_pixel: Option, +} + #[derive(Debug)] -pub struct AndroidFrame { +pub struct AndroidSharedVideoFrame { + pub timestamp_us: u64, pub width: u32, pub height: u32, - pub timestamp_us: u64, - pub rgba: Vec, + pub bgra: Vec, } -pub struct AndroidGrpcFrameStream { - inner: tonic::Streaming, - target: Option, +pub struct AndroidSharedVideoFrameStream { + handle: String, + fd: RawFd, + ptr: *mut u8, + length: usize, + last_sequence: Option, } +unsafe impl Send for AndroidSharedVideoFrameStream {} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct AndroidFrameTarget { +struct AndroidSharedVideoHeader { width: u32, height: u32, - rotation_quarter_turns: u16, + fps: u32, + sequence: u32, + timestamp_us: u64, } #[derive(Debug)] @@ -232,16 +250,18 @@ impl AndroidBridge { } let running = self.running_emulators().unwrap_or_default(); - Ok(avds - .into_iter() + avds.into_iter() .enumerate() - .map(|(index, avd_name)| AndroidDevice { - serial: running.get(&avd_name).cloned(), - is_booted: running.contains_key(&avd_name), - grpc_port: DEFAULT_GRPC_PORT_BASE + index as u16, - avd_name, + .map(|(index, avd_name)| { + let console_port = console_port_for_avd_index(index)?; + Ok(AndroidDevice { + serial: running.get(&avd_name).cloned(), + is_booted: running.contains_key(&avd_name), + console_port, + avd_name, + }) }) - .collect()) + .collect() } pub fn enrich_devices(&self, devices: Vec) -> Vec { @@ -318,8 +338,11 @@ impl AndroidBridge { if self.resolve_serial(&avd_name).is_ok() { return Ok(false); } - let grpc_port = self.grpc_port_for_avd(&avd_name)?; - let grpc_port = grpc_port.to_string(); + let console_port = self.console_port_for_avd(&avd_name)?; + let adb_port = console_port.checked_add(1).ok_or_else(|| { + AppError::native("Android emulator console port overflowed while booting.") + })?; + let emulator_ports = format!("{console_port},{adb_port}"); let is_windows = cfg!(target_os = "windows"); let window_mode = if is_windows { "-qt-hide-window" @@ -337,7 +360,7 @@ impl AndroidBridge { if is_windows { args.extend(["-feature", "-Vulkan"]); } - args.extend(["-grpc", &grpc_port]); + args.extend(["-ports", &emulator_ports, "-share-vid"]); Command::new(self.emulator_path()) .args(args) .stdin(Stdio::null()) @@ -781,77 +804,12 @@ impl AndroidBridge { })) } - pub async fn grpc_frame_stream( + pub fn shared_video_frame_stream( &self, id: &str, - max_edge: Option, - ) -> Result { - let avd_name = avd_from_id(id)?; - let port = self.grpc_port_for_avd(&avd_name)?; - let serial = self.resolve_serial(&avd_name)?; - let mut format = grpc::ImageFormat { - format: grpc::image_format::ImgFormat::Rgba8888 as i32, - width: 0, - height: 0, - display: 0, - transport: None, - }; - let target = self - .display_metrics_for_serial(&serial) - .ok() - .map(|metrics| AndroidFrameTarget { - width: metrics.width.round().max(1.0) as u32, - height: metrics.height.round().max(1.0) as u32, - rotation_quarter_turns: metrics.rotation_quarter_turns, - }); - if let (Some(max_edge), Some(target)) = (max_edge, target) { - let max_edge = max_edge.clamp(240, 2400); - let largest = target.width.max(target.height); - if largest > max_edge { - if target.width >= target.height { - format.width = max_edge; - } else { - format.height = max_edge; - } - } - } - - let endpoint = Endpoint::from_shared(format!("http://127.0.0.1:{port}")) - .map_err(|error| AppError::native(format!("Invalid Android gRPC endpoint: {error}")))? - .connect() - .await - .map_err(|error| { - AppError::native(format!( - "Unable to connect to Android emulator gRPC: {error}" - )) - })?; - let mut grpc = tonic::client::Grpc::new(endpoint) - .max_decoding_message_size(ANDROID_GRPC_FRAME_MESSAGE_LIMIT); - grpc.ready().await.map_err(|error| { - AppError::native(format!("Android emulator gRPC is not ready: {error}")) - })?; - let path = PathAndQuery::from_static( - "/android.emulation.control.EmulatorController/streamScreenshot", - ); - let mut request = tonic::Request::new(format); - if let Some(token) = self.emulator_grpc_token(&serial, port) { - let value = MetadataValue::try_from(format!("Bearer {token}")).map_err(|error| { - AppError::native(format!("Invalid Android emulator gRPC token: {error}")) - })?; - request.metadata_mut().insert("authorization", value); - } - let response = grpc - .server_streaming(request, path, tonic::codec::ProstCodec::default()) - .await - .map_err(|error| { - AppError::native(format!( - "Android emulator screenshot stream failed: {error}" - )) - })?; - Ok(AndroidGrpcFrameStream { - inner: response.into_inner(), - target, - }) + ) -> Result { + let handle = self.shared_video_memory_handle(id)?; + AndroidSharedVideoFrameStream::open(handle) } pub fn accessibility_tree( @@ -958,7 +916,7 @@ impl AndroidBridge { "android": { "avdName": device.avd_name, "serial": device.serial, - "grpcPort": device.grpc_port, + "consolePort": device.console_port, }, "privateDisplay": private_display, }) @@ -968,6 +926,19 @@ impl AndroidBridge { self.resolve_serial(&avd_from_id(id)?) } + fn shared_video_memory_handle(&self, id: &str) -> Result { + let avd_name = avd_from_id(id)?; + if let Ok(serial) = self.resolve_serial(&avd_name) { + if let Some(port) = console_port_from_serial(&serial) { + return Ok(format!("videmulator{port}")); + } + } + Ok(format!( + "videmulator{}", + self.console_port_for_avd(&avd_name)? + )) + } + fn resolve_serial(&self, avd_name: &str) -> Result { if let Some(serial) = self.running_emulators()?.remove(avd_name) { return Ok(serial); @@ -1034,11 +1005,11 @@ impl AndroidBridge { }) } - fn grpc_port_for_avd(&self, avd_name: &str) -> Result { + fn console_port_for_avd(&self, avd_name: &str) -> Result { static CACHE: OnceLock>> = OnceLock::new(); let cache = CACHE.get_or_init(|| Mutex::new(None)); if let Some((updated_at, ports)) = cache.lock().unwrap().as_ref() { - if updated_at.elapsed() < AVD_GRPC_PORT_CACHE_TTL { + if updated_at.elapsed() < AVD_CONSOLE_PORT_CACHE_TTL { if let Some(port) = ports.get(avd_name) { return Ok(*port); } @@ -1051,8 +1022,8 @@ impl AndroidBridge { .map(str::trim) .filter(|line| !line.is_empty()) .enumerate() - .map(|(index, name)| (name.to_owned(), DEFAULT_GRPC_PORT_BASE + index as u16)) - .collect::>(); + .map(|(index, name)| Ok((name.to_owned(), console_port_for_avd_index(index)?))) + .collect::, AppError>>()?; let port = ports .get(avd_name) .copied() @@ -1168,202 +1139,347 @@ impl AndroidBridge { fn avd_dir(&self, avd_name: &str) -> PathBuf { home_dir().join(format!(".android/avd/{avd_name}.avd")) } +} - fn emulator_grpc_token(&self, serial: &str, port: u16) -> Option { - self.discovery_path_grpc_token(serial, port) - .or_else(|| per_instance_grpc_token(port)) - .or_else(global_grpc_token) +impl AndroidSharedVideoFrameStream { + fn open(handle: String) -> Result { + let (fd, ptr, length) = open_android_shared_video_memory(&handle)?; + Ok(Self { + handle, + fd, + ptr, + length, + last_sequence: None, + }) } - fn discovery_path_grpc_token(&self, serial: &str, port: u16) -> Option { - let output = self - .run_adb(["-s", serial, "emu", "avd", "discoverypath"]) - .ok()?; - let path = output - .lines() - .map(str::trim) - .find(|line| { - !line.is_empty() - && *line != "OK" - && (line.ends_with(".ini") || line.contains("avd")) - }) - .map(PathBuf::from)?; - let contents = std::fs::read_to_string(path).ok()?; - grpc_token_from_discovery_ini(&contents, port) + pub fn next_frame( + &mut self, + quality: AndroidH264StreamQuality, + ) -> Result, AppError> { + for _ in 0..4 { + let header = self.read_header()?; + if Some(header.sequence) == self.last_sequence { + return Ok(None); + } + let pixel_bytes = android_shared_video_pixel_bytes(header.width, header.height)?; + let end = ANDROID_SHARED_VIDEO_HEADER_BYTES + .checked_add(pixel_bytes) + .ok_or_else(|| AppError::native("Android shared video frame size overflowed."))?; + if end > self.length { + return Err(AppError::native(format!( + "Android emulator shared video `{}` is too small for {}x{} BGRA frames.", + self.handle, header.width, header.height + ))); + } + + let mut bgra = vec![0u8; pixel_bytes]; + unsafe { + ptr::copy_nonoverlapping( + self.ptr.add(ANDROID_SHARED_VIDEO_HEADER_BYTES), + bgra.as_mut_ptr(), + pixel_bytes, + ); + } + let confirm = self.read_header()?; + if header.width != confirm.width + || header.height != confirm.height + || header.sequence != confirm.sequence + { + continue; + } + self.last_sequence = Some(header.sequence); + return Ok(Some(android_shared_video_frame_from_pixels( + header, bgra, quality, + )?)); + } + Ok(None) + } + + fn read_header(&self) -> Result { + read_android_shared_video_header_volatile(self.ptr, self.length) } } -impl AndroidGrpcFrameStream { - pub async fn next_frame(&mut self) -> Result, AppError> { - let Some(image) = self.inner.message().await.map_err(|error| { - AppError::native(format!( - "Android emulator screenshot stream failed: {error}" - )) - })? - else { - return Ok(None); - }; - let format = image.format.ok_or_else(|| { - AppError::native("Android emulator screenshot did not include an image format.") - })?; - let width = if format.width > 0 { - format.width - } else { - image.width - }; - let height = if format.height > 0 { - format.height - } else { - image.height - }; - if width == 0 || height == 0 { - return Err(AppError::native( - "Android emulator screenshot did not include dimensions.", - )); +impl Drop for AndroidSharedVideoFrameStream { + fn drop(&mut self) { + unsafe { + if !self.ptr.is_null() && self.length > 0 { + libc::munmap(self.ptr.cast(), self.length); + } + if self.fd >= 0 { + libc::close(self.fd); + } } - let rgba = rgba_display_order( - &image.image, - width, - height, - grpc::image_format::ImgFormat::try_from(format.format) - .unwrap_or(grpc::image_format::ImgFormat::Rgba8888), - )?; - let (width, height, mut rgba) = - normalize_android_frame_orientation(width, height, rgba, self.target); - flatten_android_frame_alpha(&mut rgba, width, height); - Ok(Some(AndroidFrame { - width, - height, - timestamp_us: image.timestamp_us, - rgba, - })) } } -fn normalize_android_frame_orientation( - width: u32, - height: u32, - mut rgba: Vec, - target: Option, -) -> (u32, u32, Vec) { - let Some(target) = target else { - return (width, height, rgba); - }; - if width == 0 || height == 0 || target.width == 0 || target.height == 0 { - return (width, height, rgba); +fn android_h264_stream_dimensions( + source_width: u32, + source_height: u32, + max_edge: Option, +) -> (u32, u32) { + let source_width = source_width.max(2); + let source_height = source_height.max(2); + let max_edge = max_edge + .unwrap_or(ANDROID_H264_DEFAULT_MAX_EDGE) + .clamp(240, 4096); + let largest = source_width.max(source_height); + if largest <= max_edge { + return ( + round_android_h264_dimension(source_width), + round_android_h264_dimension(source_height), + ); } + let scale = max_edge as f64 / largest as f64; + ( + round_android_h264_dimension((source_width as f64 * scale).round() as u32), + round_android_h264_dimension((source_height as f64 * scale).round() as u32), + ) +} - let (width, height) = if (width > height) == (target.width > target.height) { - (width, height) +fn round_android_h264_dimension(value: u32) -> u32 { + let rounded = value.max(2); + if rounded % 2 == 0 { + rounded } else { - rgba = if target.rotation_quarter_turns == 3 { - rotate_rgba_counterclockwise(&rgba, width, height) - } else { - rotate_rgba_clockwise(&rgba, width, height) - }; - (height, width) - }; - - if target.width > target.height { - rotate_rgba_180_in_place(&mut rgba, width, height); + rounded.saturating_sub(1).max(2) } - - (width, height, rgba) } -fn rotate_rgba_clockwise(rgba: &[u8], width: u32, height: u32) -> Vec { - let width = width as usize; - let height = height as usize; - let mut out = vec![0; rgba.len()]; - for y in 0..height { - for x in 0..width { - let src = (y * width + x) * 4; - let dst_x = height - 1 - y; - let dst_y = x; - let dst = (dst_y * height + dst_x) * 4; - out[dst..dst + 4].copy_from_slice(&rgba[src..src + 4]); - } +fn open_android_shared_video_memory(handle: &str) -> Result<(RawFd, *mut u8, usize), AppError> { + let mut names = vec![handle.to_owned()]; + if !handle.starts_with('/') { + names.push(format!("/{handle}")); } - out -} + let mut last_error = None; + for name in names { + let c_name = CString::new(name.as_str()) + .map_err(|_| AppError::native("Android shared video handle contains a NUL byte."))?; + let fd = unsafe { libc::shm_open(c_name.as_ptr(), libc::O_RDONLY, 0) }; + if fd < 0 { + last_error = Some(std::io::Error::last_os_error()); + continue; + } + + let mut stat = MaybeUninit::::uninit(); + if unsafe { libc::fstat(fd, stat.as_mut_ptr()) } != 0 { + let error = std::io::Error::last_os_error(); + unsafe { + libc::close(fd); + } + last_error = Some(error); + continue; + } + let stat = unsafe { stat.assume_init() }; + let length = usize::try_from(stat.st_size).map_err(|_| { + unsafe { + libc::close(fd); + } + AppError::native(format!( + "Android emulator shared video `{handle}` reported an invalid size." + )) + })?; + if length <= ANDROID_SHARED_VIDEO_HEADER_BYTES { + unsafe { + libc::close(fd); + } + return Err(AppError::native(format!( + "Android emulator shared video `{handle}` is smaller than the display header." + ))); + } -fn rotate_rgba_180_in_place(rgba: &mut [u8], width: u32, height: u32) { - let pixel_count = width as usize * height as usize; - for pixel in 0..(pixel_count / 2) { - let opposite = pixel_count - 1 - pixel; - for channel in 0..4 { - rgba.swap(pixel * 4 + channel, opposite * 4 + channel); + let ptr = unsafe { + libc::mmap( + ptr::null_mut(), + length, + libc::PROT_READ, + libc::MAP_SHARED, + fd, + 0, + ) + }; + if ptr == libc::MAP_FAILED { + let error = std::io::Error::last_os_error(); + unsafe { + libc::close(fd); + } + last_error = Some(error); + continue; } + return Ok((fd, ptr.cast::(), length)); } + + Err(AppError::native(format!( + "Android emulator shared video `{handle}` is unavailable. Start the emulator through SimDeck or launch it with `-share-vid`{}.", + last_error + .as_ref() + .map(|error| format!(" ({error})")) + .unwrap_or_default() + ))) } -fn rotate_rgba_counterclockwise(rgba: &[u8], width: u32, height: u32) -> Vec { - let width = width as usize; - let height = height as usize; - let mut out = vec![0; rgba.len()]; - for y in 0..height { - for x in 0..width { - let src = (y * width + x) * 4; - let dst_x = y; - let dst_y = width - 1 - x; - let dst = (dst_y * height + dst_x) * 4; - out[dst..dst + 4].copy_from_slice(&rgba[src..src + 4]); - } +fn read_android_shared_video_header_volatile( + ptr: *const u8, + length: usize, +) -> Result { + if ptr.is_null() || length < ANDROID_SHARED_VIDEO_HEADER_BYTES { + return Err(AppError::native( + "Android emulator shared video header is unavailable.", + )); } - out + let width = unsafe { read_volatile_le_u32(ptr, 0) }; + let height = unsafe { read_volatile_le_u32(ptr, 4) }; + let fps = unsafe { read_volatile_le_u32(ptr, 8) }; + let sequence = unsafe { read_volatile_le_u32(ptr, 12) }; + let timestamp_low = unsafe { read_volatile_le_u32(ptr, 16) }; + let timestamp_high = unsafe { read_volatile_le_u32(ptr, 20) }; + validate_android_shared_video_header(AndroidSharedVideoHeader { + width, + height, + fps, + sequence, + timestamp_us: (u64::from(timestamp_high) << 32) | u64::from(timestamp_low), + }) } -fn flatten_android_frame_alpha(rgba: &mut [u8], width: u32, height: u32) { - if !rgba.chunks_exact(4).any(|pixel| pixel[3] != 255) { - return; +unsafe fn read_volatile_le_u32(ptr: *const u8, offset: usize) -> u32 { + let mut bytes = [0u8; 4]; + for (index, byte) in bytes.iter_mut().enumerate() { + *byte = ptr.add(offset + index).read_volatile(); } + u32::from_le_bytes(bytes) +} - let width = width as usize; - let height = height as usize; - let Some(default_fill) = first_opaque_rgb(rgba) else { - for pixel in rgba.chunks_exact_mut(4) { - pixel[3] = 255; - } - return; - }; +#[cfg(test)] +fn android_shared_video_frame_from_slice( + bytes: &[u8], + last_sequence: Option, + quality: AndroidH264StreamQuality, +) -> Result, AppError> { + let header = read_android_shared_video_header(bytes)?; + if Some(header.sequence) == last_sequence { + return Ok(None); + } + let pixel_bytes = android_shared_video_pixel_bytes(header.width, header.height)?; + let end = ANDROID_SHARED_VIDEO_HEADER_BYTES + .checked_add(pixel_bytes) + .ok_or_else(|| AppError::native("Android shared video frame size overflowed."))?; + let pixels = bytes + .get(ANDROID_SHARED_VIDEO_HEADER_BYTES..end) + .ok_or_else(|| AppError::native("Android shared video frame was truncated."))? + .to_vec(); + Ok(Some(android_shared_video_frame_from_pixels( + header, pixels, quality, + )?)) +} - for y in 0..height { - let row_start = y * width * 4; - let row = &mut rgba[row_start..row_start + width * 4]; - let mut fill = first_opaque_rgb(row).unwrap_or(default_fill); - for pixel in row.chunks_exact_mut(4) { - if pixel[3] == 255 { - fill = [pixel[0], pixel[1], pixel[2]]; - continue; - } - composite_pixel_over_rgb(pixel, fill); - } +#[cfg(test)] +fn read_android_shared_video_header(bytes: &[u8]) -> Result { + if bytes.len() < 24 { + return Err(AppError::native( + "Android emulator shared video header was truncated.", + )); } + let width = u32::from_le_bytes(bytes[0..4].try_into().unwrap()); + let height = u32::from_le_bytes(bytes[4..8].try_into().unwrap()); + let fps = u32::from_le_bytes(bytes[8..12].try_into().unwrap()); + let sequence = u32::from_le_bytes(bytes[12..16].try_into().unwrap()); + let timestamp_low = u32::from_le_bytes(bytes[16..20].try_into().unwrap()); + let timestamp_high = u32::from_le_bytes(bytes[20..24].try_into().unwrap()); + validate_android_shared_video_header(AndroidSharedVideoHeader { + width, + height, + fps, + sequence, + timestamp_us: (u64::from(timestamp_high) << 32) | u64::from(timestamp_low), + }) } -fn first_opaque_rgb(rgba: &[u8]) -> Option<[u8; 3]> { - rgba.chunks_exact(4) - .find(|pixel| pixel[3] == 255) - .map(|pixel| [pixel[0], pixel[1], pixel[2]]) +fn validate_android_shared_video_header( + header: AndroidSharedVideoHeader, +) -> Result { + if header.width == 0 + || header.height == 0 + || header.fps == 0 + || header.width > ANDROID_SHARED_VIDEO_MAX_DIMENSION + || header.height > ANDROID_SHARED_VIDEO_MAX_DIMENSION + { + return Err(AppError::native(format!( + "Android emulator shared video reported invalid dimensions {}x{}.", + header.width, header.height + ))); + } + Ok(header) } -fn composite_pixel_over_rgb(pixel: &mut [u8], background: [u8; 3]) { - let alpha = u32::from(pixel[3]); - if alpha == 0 { - pixel[0] = background[0]; - pixel[1] = background[1]; - pixel[2] = background[2]; - pixel[3] = 255; - return; +fn android_shared_video_pixel_bytes(width: u32, height: u32) -> Result { + let bytes = usize::try_from(width) + .ok() + .and_then(|width| { + usize::try_from(height) + .ok() + .and_then(|height| width.checked_mul(height)) + }) + .and_then(|pixels| pixels.checked_mul(ANDROID_SHARED_VIDEO_PIXEL_BYTES)) + .ok_or_else(|| AppError::native("Android shared video frame size overflowed."))?; + if bytes == 0 || bytes > ANDROID_SHARED_VIDEO_MAX_FRAME_BYTES { + return Err(AppError::native( + "Android shared video frame size was outside supported bounds.", + )); } + Ok(bytes) +} + +fn android_shared_video_frame_from_pixels( + header: AndroidSharedVideoHeader, + bgra: Vec, + quality: AndroidH264StreamQuality, +) -> Result { + let (width, height, bgra) = + scale_android_bgra_frame(header.width, header.height, bgra, quality.max_edge)?; + Ok(AndroidSharedVideoFrame { + timestamp_us: header.timestamp_us, + width, + height, + bgra, + }) +} - for channel in 0..3 { - pixel[channel] = ((u32::from(pixel[channel]) * alpha - + u32::from(background[channel]) * (255 - alpha) - + 127) - / 255) as u8; +fn scale_android_bgra_frame( + width: u32, + height: u32, + bgra: Vec, + max_edge: Option, +) -> Result<(u32, u32, Vec), AppError> { + let expected = android_shared_video_pixel_bytes(width, height)?; + if bgra.len() < expected { + return Err(AppError::native( + "Android shared video frame contained truncated BGRA pixels.", + )); + } + let (target_width, target_height) = android_h264_stream_dimensions(width, height, max_edge); + if target_width == width && target_height == height { + return Ok((width, height, bgra)); + } + + let output_len = android_shared_video_pixel_bytes(target_width, target_height)?; + let mut output = vec![0u8; output_len]; + let source_width = width as usize; + let source_height = height as usize; + let target_width_usize = target_width as usize; + let target_height_usize = target_height as usize; + for y in 0..target_height_usize { + let source_y = y * source_height / target_height_usize; + for x in 0..target_width_usize { + let source_x = x * source_width / target_width_usize; + let src = ((source_y * source_width) + source_x) * ANDROID_SHARED_VIDEO_PIXEL_BYTES; + let dst = ((y * target_width_usize) + x) * ANDROID_SHARED_VIDEO_PIXEL_BYTES; + output[dst..dst + ANDROID_SHARED_VIDEO_PIXEL_BYTES] + .copy_from_slice(&bgra[src..src + ANDROID_SHARED_VIDEO_PIXEL_BYTES]); + } } - pixel[3] = 255; + Ok((target_width, target_height, output)) } fn run_command_text(program: PathBuf, args: [&str; N]) -> Result { @@ -1712,6 +1828,24 @@ fn parse_online_emulator_serials(output: &str) -> Vec { .collect() } +fn console_port_for_avd_index(index: usize) -> Result { + let index = u16::try_from(index).map_err(|_| { + AppError::native("Android emulator console port overflowed while deriving ports.") + })?; + let console_offset = index.checked_mul(2).ok_or_else(|| { + AppError::native("Android emulator console port overflowed while deriving ports.") + })?; + DEFAULT_EMULATOR_CONSOLE_PORT_BASE + .checked_add(console_offset) + .ok_or_else(|| { + AppError::native("Android emulator console port overflowed while deriving ports.") + }) +} + +fn console_port_from_serial(serial: &str) -> Option { + serial.strip_prefix("emulator-")?.parse::().ok() +} + fn is_android_component_name(value: &str) -> bool { value .split_once('/') @@ -1777,120 +1911,6 @@ fn home_dir() -> PathBuf { .unwrap_or_else(|| Path::new("/").to_path_buf()) } -fn per_instance_grpc_token(port: u16) -> Option { - for running_dir in emulator_discovery_dirs() { - let entries = match std::fs::read_dir(running_dir) { - Ok(entries) => entries, - Err(_) => continue, - }; - let port_value = port.to_string(); - for entry in entries.flatten() { - let path = entry.path(); - if path.extension().and_then(|ext| ext.to_str()) != Some("ini") { - continue; - } - let contents = std::fs::read_to_string(path).ok()?; - let fields = parse_ini(&contents); - if fields.get("grpc.port") == Some(&port_value) { - if let Some(token) = fields.get("grpc.token").filter(|token| !token.is_empty()) { - return Some(token.to_owned()); - } - } - } - } - None -} - -fn emulator_discovery_dirs() -> Vec { - let mut dirs = Vec::new(); - dirs.push(home_dir().join("Library/Caches/TemporaryItems/avd/running")); - dirs.push(std::env::temp_dir().join("avd/running")); - if cfg!(target_os = "linux") { - if let Some(user) = env::var_os("USER") { - dirs.push( - Path::new("/tmp").join(format!("android-{}/avd/running", user.to_string_lossy())), - ); - } - dirs.push(Path::new("/tmp").join("avd/running")); - } - dirs -} - -fn grpc_token_from_discovery_ini(contents: &str, port: u16) -> Option { - let port_value = port.to_string(); - let fields = parse_ini(contents); - (fields.get("grpc.port") == Some(&port_value)) - .then(|| fields.get("grpc.token")) - .flatten() - .filter(|token| !token.is_empty()) - .map(ToOwned::to_owned) -} - -fn global_grpc_token() -> Option { - std::fs::read_to_string(home_dir().join(".emulator_console_auth_token")) - .ok() - .map(|token| token.trim().to_owned()) - .filter(|token| !token.is_empty()) -} - -fn parse_ini(contents: &str) -> HashMap { - contents - .lines() - .filter_map(|line| { - let line = line.trim(); - let (key, value) = line.split_once('=')?; - Some((key.trim().to_owned(), value.trim().to_owned())) - }) - .collect() -} - -fn rgba_display_order( - image: &[u8], - width: u32, - height: u32, - format: grpc::image_format::ImgFormat, -) -> Result, AppError> { - let width = width as usize; - let height = height as usize; - match format { - grpc::image_format::ImgFormat::Rgba8888 => { - let row_len = width * 4; - if image.len() < row_len * height { - return Err(AppError::native( - "Android emulator returned a truncated RGBA frame.", - )); - } - Ok(image[..row_len * height].to_vec()) - } - grpc::image_format::ImgFormat::Rgb888 => { - let src_row_len = width * 3; - if image.len() < src_row_len * height { - return Err(AppError::native( - "Android emulator returned a truncated RGB frame.", - )); - } - let mut out = BytesMut::with_capacity(width * height * 4); - out.resize(width * height * 4, 255); - for y in 0..height { - let src_row = y * src_row_len; - let dst_row = y * width * 4; - for x in 0..width { - let src = src_row + x * 3; - let dst = dst_row + x * 4; - out[dst] = image[src]; - out[dst + 1] = image[src + 1]; - out[dst + 2] = image[src + 2]; - out[dst + 3] = 255; - } - } - Ok(out.to_vec()) - } - grpc::image_format::ImgFormat::Png => Err(AppError::native( - "Android emulator gRPC returned PNG instead of raw pixels.", - )), - } -} - fn extract_xml(output: &str) -> &str { output .find(", - } - - pub mod image_format { - #[derive( - Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration, - )] - #[repr(i32)] - pub enum ImgFormat { - Png = 0, - Rgba8888 = 1, - Rgb888 = 2, - } + #[test] + fn android_shared_video_frame_skips_existing_sequence() { + let mut bytes = vec![0u8; ANDROID_SHARED_VIDEO_HEADER_BYTES + 2 * 2 * 4]; + bytes[0..4].copy_from_slice(&2u32.to_le_bytes()); + bytes[4..8].copy_from_slice(&2u32.to_le_bytes()); + bytes[8..12].copy_from_slice(&60u32.to_le_bytes()); + bytes[12..16].copy_from_slice(&7u32.to_le_bytes()); + + let frame = android_shared_video_frame_from_slice( + &bytes, + Some(7), + AndroidH264StreamQuality::default(), + ) + .unwrap(); + + assert!(frame.is_none()); } - #[derive(Clone, PartialEq, ::prost::Message)] - pub struct Image { - #[prost(message, optional, tag = "1")] - pub format: Option, - #[prost(uint32, tag = "2")] - pub width: u32, - #[prost(uint32, tag = "3")] - pub height: u32, - #[prost(bytes = "vec", tag = "4")] - pub image: Vec, - #[prost(uint32, tag = "5")] - pub seq: u32, - #[prost(uint64, tag = "6")] - pub timestamp_us: u64, + #[test] + fn android_shared_video_frame_scales_bgra_to_even_max_edge() { + let mut bytes = vec![0u8; ANDROID_SHARED_VIDEO_HEADER_BYTES + 480 * 480 * 4]; + bytes[0..4].copy_from_slice(&480u32.to_le_bytes()); + bytes[4..8].copy_from_slice(&480u32.to_le_bytes()); + bytes[8..12].copy_from_slice(&60u32.to_le_bytes()); + bytes[12..16].copy_from_slice(&1u32.to_le_bytes()); + for y in 0..480usize { + for x in 0..480usize { + let offset = ANDROID_SHARED_VIDEO_HEADER_BYTES + ((y * 480 + x) * 4); + bytes[offset..offset + 4].copy_from_slice(&[x as u8, y as u8, 0xaa, 0xff]); + } + } + + let frame = android_shared_video_frame_from_slice( + &bytes, + None, + AndroidH264StreamQuality { + max_edge: Some(240), + min_bitrate: None, + bits_per_pixel: None, + }, + ) + .unwrap() + .unwrap(); + + assert_eq!((frame.width, frame.height), (240, 240)); + assert_eq!(frame.bgra.len(), 240 * 240 * 4); + assert_eq!(&frame.bgra[..8], &[0, 0, 0xaa, 0xff, 2, 0, 0xaa, 0xff]); + let second_row = 240 * 4; + assert_eq!( + &frame.bgra[second_row..second_row + 8], + &[0, 2, 0xaa, 0xff, 2, 2, 0xaa, 0xff] + ); } } diff --git a/packages/server/src/api/routes.rs b/packages/server/src/api/routes.rs index 033628de..5465fb48 100644 --- a/packages/server/src/api/routes.rs +++ b/packages/server/src/api/routes.rs @@ -386,12 +386,12 @@ struct ActiveStreamQualityState { video_codec: String, } -#[derive(Debug, Eq, PartialEq)] -struct StreamQualityLimits { - max_edge: u32, - fps: u32, - min_bitrate: u32, - bits_per_pixel: u32, +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct StreamQualityLimits { + pub(crate) max_edge: u32, + pub(crate) fps: u32, + pub(crate) min_bitrate: u32, + pub(crate) bits_per_pixel: u32, } const STREAM_QUALITY_PROFILES: &[StreamQualityProfile] = &[ @@ -1531,6 +1531,19 @@ fn stream_quality_profile_value(profile: &StreamQualityProfile) -> Value { }) } +pub(crate) fn stream_quality_limits_for_payload( + payload: &StreamQualityPayload, +) -> Result { + let profile = payload + .profile + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(stream_quality_profile) + .transpose()?; + Ok(resolved_stream_quality_limits(payload, profile)) +} + fn resolved_stream_quality_limits( payload: &StreamQualityPayload, profile: Option, @@ -2940,8 +2953,13 @@ async fn handle_android_h264_socket( state.android.clone(), state.metrics.clone(), udid.clone(), - initial_quality.max_edge, - true, + match android_h264_quality_from_stream_payload(&initial_quality) { + Ok(quality) => quality, + Err(error) => { + tracing::debug!("Invalid Android H264 WebSocket quality for {udid}: {error}"); + return; + } + }, ) .await { @@ -3149,13 +3167,30 @@ fn handle_android_h264_socket_message( source.request_refresh(); } } - H264SocketMessage::StreamQuality { config: _ } => { - source.request_keyframe(); + H264SocketMessage::StreamQuality { config } => { + match android_h264_quality_from_stream_payload(&config) { + Ok(quality) => source.reconfigure_h264(quality), + Err(error) => { + tracing::debug!("Android H264 WebSocket stream quality update failed: {error}"); + source.request_keyframe(); + } + } } } true } +fn android_h264_quality_from_stream_payload( + payload: &StreamQualityPayload, +) -> Result { + let limits = stream_quality_limits_for_payload(payload)?; + Ok(android::AndroidH264StreamQuality { + max_edge: Some(limits.max_edge), + min_bitrate: Some(limits.min_bitrate), + bits_per_pixel: Some(limits.bits_per_pixel), + }) +} + #[derive(Debug, Deserialize)] #[serde(tag = "type", rename_all = "camelCase")] enum H264SocketMessage { diff --git a/packages/server/src/native/ffi.rs b/packages/server/src/native/ffi.rs index 8a8a47b5..4fe9e724 100644 --- a/packages/server/src/native/ffi.rs +++ b/packages/server/src/native/ffi.rs @@ -340,9 +340,9 @@ unsafe extern "C" { error_message: *mut *mut c_char, ) -> *mut c_void; pub fn xcw_native_h264_encoder_destroy(handle: *mut c_void); - pub fn xcw_native_h264_encoder_encode_rgba( + pub fn xcw_native_h264_encoder_encode_bgra( handle: *mut c_void, - rgba: *const u8, + bgra: *const u8, length: usize, width: u32, height: u32, diff --git a/packages/server/src/transport/webrtc.rs b/packages/server/src/transport/webrtc.rs index 2391ceda..d2ca9da4 100644 --- a/packages/server/src/transport/webrtc.rs +++ b/packages/server/src/transport/webrtc.rs @@ -2,19 +2,22 @@ use crate::android; use crate::api::routes::{ apply_stream_client_foreground_from_stats, apply_stream_quality_payload, bridge_input_session_for_control, run_bridge_multitouch_control_message, run_control_message, - run_toggle_appearance_control, run_tvos_control_message, AppState, ControlMessage, - StreamQualityPayload, TvosControlTouchGesture, + run_toggle_appearance_control, run_tvos_control_message, stream_quality_limits_for_payload, + AppState, ControlMessage, StreamQualityLimits, StreamQualityPayload, TvosControlTouchGesture, }; use crate::error::AppError; use crate::metrics::counters::ClientStreamStats; use crate::native::ffi; use crate::transport::packet::{FramePacket, SharedFrame}; -use bytes::{BufMut, Bytes, BytesMut}; +use bytes::Bytes; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, VecDeque}; use std::ffi::{c_void, CStr}; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::ptr; +use std::slice; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, OnceLock, RwLock, Weak}; +use std::thread; use std::time::Duration; use tokio::sync::{broadcast, mpsc}; use tokio::task; @@ -23,9 +26,7 @@ use tracing::{info, warn}; use webrtc::api::interceptor_registry::register_default_interceptors; use webrtc::api::media_engine::{MediaEngine, MIME_TYPE_H264}; use webrtc::api::APIBuilder; -use webrtc::data_channel::data_channel_init::RTCDataChannelInit; use webrtc::data_channel::data_channel_message::DataChannelMessage; -use webrtc::data_channel::data_channel_state::RTCDataChannelState; use webrtc::data_channel::RTCDataChannel; use webrtc::ice_transport::ice_server::RTCIceServer; use webrtc::interceptor::registry::Registry; @@ -51,7 +52,6 @@ const ANNEX_B_START_CODE: &[u8] = &[0, 0, 0, 1]; const DEFAULT_STUN_URL: &str = "stun:stun.l.google.com:19302"; const WEBRTC_CONTROL_CHANNEL_LABEL: &str = "simdeck-control"; const WEBRTC_TELEMETRY_CHANNEL_LABEL: &str = "simdeck-telemetry"; -const WEBRTC_RGBA_CHANNEL_LABEL: &str = "simdeck-rgba"; const WEBRTC_DEFAULT_LOCAL_STREAM_FPS: u32 = 60; const WEBRTC_MAX_LOCAL_STREAM_FPS: u32 = 240; const WEBRTC_WRITE_TIMEOUT: Duration = Duration::from_millis(120); @@ -64,14 +64,8 @@ const WEBRTC_MULTITOUCH_INPUT_IDLE_TIMEOUT: Duration = Duration::from_secs(5); const WEBRTC_RTP_OUTBOUND_MTU: usize = 1200; const WEBRTC_PEER_DISCONNECTED_TIMEOUT: Duration = Duration::from_secs(12); const ANDROID_WEBRTC_FRAME_BROADCAST_CAPACITY: usize = 128; -const ANDROID_WEBRTC_RAW_FRAME_BROADCAST_CAPACITY: usize = 8; -const ANDROID_WEBRTC_RGBA_CHUNK_HEADER_BYTES: usize = 48; -const ANDROID_WEBRTC_RGBA_CHUNK_BYTES: usize = 256 * 1024; -const ANDROID_WEBRTC_RGBA_CHUNK_MAGIC: u32 = 0x5344_5243; // "SDRC" -const ANDROID_WEBRTC_RGBA_VERSION: u8 = 1; -const ANDROID_WEBRTC_RGBA_FORMAT_RGBA8888: u8 = 1; -const ANDROID_WEBRTC_RGBA_BUFFERED_FRAME_LIMIT: usize = 2; -const ANDROID_WEBRTC_FPS: u64 = 30; +const ANDROID_WEBRTC_FPS: u64 = 60; +const ANDROID_SHARED_VIDEO_RETRY_DELAY: Duration = Duration::from_millis(200); static WEBRTC_MEDIA_STREAMS: OnceLock>>> = OnceLock::new(); const MAX_WEBRTC_MEDIA_STREAMS_PER_UDID: usize = 16; @@ -110,6 +104,26 @@ pub struct WebRtcVideoMetadata { pub height: u32, } +fn android_h264_quality_from_payload( + payload: Option<&StreamQualityPayload>, +) -> Result { + let Some(payload) = payload else { + return Ok(android::AndroidH264StreamQuality::default()); + }; + let limits = stream_quality_limits_for_payload(payload)?; + Ok(android_h264_quality_from_limits(limits)) +} + +fn android_h264_quality_from_limits( + limits: StreamQualityLimits, +) -> android::AndroidH264StreamQuality { + android::AndroidH264StreamQuality { + max_edge: Some(limits.max_edge), + min_bitrate: Some(limits.min_bitrate), + bits_per_pixel: Some(limits.bits_per_pixel), + } +} + #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct ClientIceServer { @@ -124,7 +138,7 @@ pub async fn create_answer( state: AppState, udid: String, payload: WebRtcOfferPayload, - peer_is_loopback: bool, + _peer_is_loopback: bool, ) -> Result { if payload.kind != "offer" { return Err(AppError::bad_request( @@ -132,27 +146,9 @@ pub async fn create_answer( )); } let is_android = android::is_android_id(&udid); - let transport = payload - .transport - .as_deref() - .map(str::trim) - .filter(|value| !value.is_empty()); - if transport.is_some_and(|value| value.eq_ignore_ascii_case("rgba")) { - if !is_android { - return Err(AppError::bad_request( - "RGBA WebRTC transport is only supported for Android emulators.", - )); - } - if !peer_is_loopback { - return Err(AppError::bad_request( - "RGBA WebRTC transport is only available to loopback clients.", - )); - } - return create_android_rgba_answer(state, udid, payload).await; - } if payload.transport.is_some() { return Err(AppError::bad_request( - "Unsupported WebRTC transport. Supported transports are media tracks and Android loopback RGBA.", + "Unsupported WebRTC transport. SimDeck streams WebRTC video over media tracks.", )); } if !is_android { @@ -167,8 +163,7 @@ pub async fn create_answer( state.android.clone(), state.metrics.clone(), udid.clone(), - None, - true, + android_h264_quality_from_payload(payload.stream_config.as_ref())?, ) .await?, ) @@ -363,127 +358,6 @@ pub async fn create_answer( }) } -async fn create_android_rgba_answer( - state: AppState, - udid: String, - payload: WebRtcOfferPayload, -) -> Result { - let source = AndroidWebRtcSource::start( - state.android.clone(), - state.metrics.clone(), - udid.clone(), - None, - false, - ) - .await?; - info!( - "Android RGBA WebRTC offer for {udid}: remote_candidates={} remote_candidate_types={} ice_servers={} ice_transport_policy={}", - count_sdp_candidates(&payload.sdp), - summarize_sdp_candidate_types(&payload.sdp), - std::env::var("SIMDECK_WEBRTC_ICE_SERVERS") - .ok() - .filter(|value| !value.trim().is_empty()) - .unwrap_or_else(|| DEFAULT_STUN_URL.to_owned()), - ice_transport_policy_label() - ); - - let api = APIBuilder::new().build(); - let peer_connection = Arc::new( - api.new_peer_connection(RTCConfiguration { - ice_servers: ice_servers(), - ice_transport_policy: ice_transport_policy(), - ..Default::default() - }) - .await - .map_err(|error| AppError::internal(format!("create WebRTC peer connection: {error}")))?, - ); - register_diagnostics(&peer_connection, &udid); - let (stream_control_tx, stream_control_rx) = mpsc::unbounded_channel(); - register_android_data_channel( - &peer_connection, - source.clone(), - state.clone(), - udid.clone(), - stream_control_tx, - ); - let rgba_channel = peer_connection - .create_data_channel( - WEBRTC_RGBA_CHANNEL_LABEL, - Some(RTCDataChannelInit { - ordered: Some(false), - max_retransmits: Some(0), - ..Default::default() - }), - ) - .await - .map_err(|error| AppError::internal(format!("create RGBA WebRTC data channel: {error}")))?; - - let fast_gather = - has_sdp_candidate_type(&payload.sdp, "host") && ice_transport_policy_label() == "all"; - let offer = RTCSessionDescription::offer(payload.sdp) - .map_err(|error| AppError::bad_request(format!("invalid WebRTC offer: {error}")))?; - peer_connection - .set_remote_description(offer) - .await - .map_err(|error| AppError::bad_request(format!("set remote WebRTC offer: {error}")))?; - let answer = peer_connection - .create_answer(None) - .await - .map_err(|error| AppError::internal(format!("create WebRTC answer: {error}")))?; - let mut gather_complete = peer_connection.gathering_complete_promise().await; - peer_connection - .set_local_description(answer) - .await - .map_err(|error| AppError::internal(format!("set WebRTC answer: {error}")))?; - let gather_timeout = if fast_gather { - WEBRTC_FAST_ICE_GATHER_TIMEOUT - } else { - WEBRTC_FULL_ICE_GATHER_TIMEOUT - }; - let gather_result = time::timeout(gather_timeout, gather_complete.recv()).await; - let mut local_description = peer_connection - .local_description() - .await - .ok_or_else(|| AppError::internal("WebRTC local description was not set."))?; - if gather_result.is_err() && count_sdp_candidates(&local_description.sdp) == 0 { - let _ = time::timeout(WEBRTC_FULL_ICE_GATHER_TIMEOUT, gather_complete.recv()).await; - local_description = peer_connection - .local_description() - .await - .ok_or_else(|| AppError::internal("WebRTC local description was not set."))?; - } - info!( - "Android RGBA WebRTC answer for {udid}: local_candidates={} local_candidate_types={}", - count_sdp_candidates(&local_description.sdp), - summarize_sdp_candidate_types(&local_description.sdp) - ); - - let (cancellation_token, cancellation) = - register_webrtc_media_stream(&udid, payload.client_id.as_deref(), true); - tokio::spawn( - WebRtcRgbaStream { - state, - udid, - source, - peer_connection, - rgba_channel, - cancellation_token, - cancellation, - stream_control_rx, - } - .run(), - ); - - Ok(WebRtcAnswerPayload { - sdp: local_description.sdp, - kind: "answer".to_owned(), - video: WebRtcVideoMetadata { - width: 0, - height: 0, - }, - }) -} - fn register_diagnostics( peer_connection: &Arc, udid: &str, @@ -801,8 +675,15 @@ fn attach_android_data_channel( let _ = stream_control_tx.send(command); } WebRtcDataChannelMessage::StreamQuality { config } => { - let _ = config; - source.request_keyframe(); + match android_h264_quality_from_payload(Some(&config)) { + Ok(quality) => source.reconfigure_h264(quality), + Err(error) => { + warn!( + "Android WebRTC stream quality update failed for {udid}: {error}" + ); + source.request_keyframe(); + } + } } } return; @@ -1432,12 +1313,13 @@ pub(crate) struct AndroidWebRtcSource { struct AndroidWebRtcSourceInner { udid: String, - encoder_handle: AtomicUsize, - callback_user_data: AtomicUsize, shutdown_tx: broadcast::Sender<()>, sender: broadcast::Sender, - raw_sender: broadcast::Sender>, latest_keyframe: RwLock>, + frame_sequence: AtomicU64, + quality: Mutex, + encoder_handle: AtomicUsize, + callback_user_data: AtomicUsize, metrics: Arc, } @@ -1449,142 +1331,25 @@ impl AndroidWebRtcSource { bridge: android::AndroidBridge, metrics: Arc, udid: String, - max_edge: Option, - encode_h264: bool, + quality: android::AndroidH264StreamQuality, ) -> Result { - let mut frame_stream = bridge.grpc_frame_stream(&udid, max_edge).await?; let (sender, _) = broadcast::channel(ANDROID_WEBRTC_FRAME_BROADCAST_CAPACITY); - let (raw_sender, _) = broadcast::channel(ANDROID_WEBRTC_RAW_FRAME_BROADCAST_CAPACITY); let (shutdown_tx, _) = broadcast::channel(1); let inner = Arc::new(AndroidWebRtcSourceInner { udid: udid.clone(), - encoder_handle: AtomicUsize::new(0), - callback_user_data: AtomicUsize::new(0), shutdown_tx, sender, - raw_sender, latest_keyframe: RwLock::new(None), + frame_sequence: AtomicU64::new(0), + quality: Mutex::new(quality), + encoder_handle: AtomicUsize::new(0), + callback_user_data: AtomicUsize::new(0), metrics, }); - if encode_h264 { - let user_data = Weak::into_raw(Arc::downgrade(&inner)) as *mut c_void; - let mut error = std::ptr::null_mut(); - let handle = unsafe { - ffi::xcw_native_h264_encoder_create( - Some(android_h264_encoder_frame_callback), - user_data, - &mut error, - ) - }; - if handle.is_null() { - unsafe { - let _ = Weak::from_raw(user_data as *const AndroidWebRtcSourceInner); - } - return Err(unsafe { take_native_error(error) }.unwrap_or_else(|| { - AppError::native("Unable to create Android H.264 encoder.") - })); - } - inner - .encoder_handle - .store(handle as usize, Ordering::Release); - inner - .callback_user_data - .store(user_data as usize, Ordering::Release); - } let source = Self { inner }; - let latest_frame = Arc::new(Mutex::new(None::>)); - let reader_inner = Arc::downgrade(&source.inner); - let reader_latest_frame = latest_frame.clone(); - let mut reader_shutdown_rx = source.inner.shutdown_tx.subscribe(); - tokio::spawn(async move { - loop { - tokio::select! { - _ = reader_shutdown_rx.recv() => break, - frame = frame_stream.next_frame() => { - match frame { - Ok(Some(frame)) => { - let frame = Arc::new(frame); - *reader_latest_frame.lock().unwrap() = Some(frame); - } - Ok(None) => break, - Err(error) => { - let udid = reader_inner - .upgrade() - .map(|inner| inner.udid.clone()) - .unwrap_or_else(|| "android".to_owned()); - warn!("Android WebRTC raw frame stream failed for {udid}: {error}"); - break; - } - } - } - } - } - }); - - if encode_h264 { - let encoder_inner = Arc::downgrade(&source.inner); - let encoder_latest_frame = latest_frame; - let mut encoder_shutdown_rx = source.inner.shutdown_tx.subscribe(); - tokio::spawn(async move { - let min_frame_gap = android_webrtc_frame_interval(); - let mut ticker = time::interval(min_frame_gap); - ticker.set_missed_tick_behavior(time::MissedTickBehavior::Skip); - loop { - tokio::select! { - _ = encoder_shutdown_rx.recv() => break, - _ = ticker.tick() => { - let Some(inner) = encoder_inner.upgrade() else { - break; - }; - let frame = encoder_latest_frame.lock().unwrap().clone(); - let Some(frame) = frame else { - continue; - }; - if inner.latest_keyframe.read().unwrap().is_none() { - inner.request_keyframe(); - } - let handle = inner.encoder_handle.load(Ordering::Acquire); - let udid = inner.udid.clone(); - let encode_result = task::spawn_blocking(move || { - encode_android_rgba_frame(handle, &frame) - }) - .await - .map_err(|error| AppError::internal(format!("Failed to join Android encoder task: {error}"))) - .and_then(|result| result); - if let Err(error) = encode_result { - warn!("Android VideoToolbox encode failed for {udid}: {error}"); - } - } - } - } - }); - source.request_keyframe(); - } else { - let raw_inner = Arc::downgrade(&source.inner); - let raw_latest_frame = latest_frame; - let mut raw_shutdown_rx = source.inner.shutdown_tx.subscribe(); - tokio::spawn(async move { - let min_frame_gap = android_webrtc_frame_interval(); - let mut ticker = time::interval(min_frame_gap); - ticker.set_missed_tick_behavior(time::MissedTickBehavior::Skip); - loop { - tokio::select! { - _ = raw_shutdown_rx.recv() => break, - _ = ticker.tick() => { - let Some(inner) = raw_inner.upgrade() else { - break; - }; - let frame = raw_latest_frame.lock().unwrap().clone(); - let Some(frame) = frame else { - continue; - }; - let _ = inner.raw_sender.send(frame); - } - } - } - }); - } + source.inner.create_native_encoder()?; + spawn_android_shared_video_encoder(bridge, &source.inner); Ok(source) } @@ -1592,10 +1357,6 @@ impl AndroidWebRtcSource { self.inner.sender.subscribe() } - fn subscribe_raw(&self) -> broadcast::Receiver> { - self.inner.raw_sender.subscribe() - } - pub(crate) async fn wait_for_keyframe( &self, timeout_duration: Duration, @@ -1635,67 +1396,178 @@ impl AndroidWebRtcSource { pub(crate) fn request_keyframe(&self) { self.inner.request_keyframe(); } + + pub(crate) fn reconfigure_h264(&self, quality: android::AndroidH264StreamQuality) { + self.inner.reconfigure_h264(quality); + } +} + +fn spawn_android_shared_video_encoder( + bridge: android::AndroidBridge, + inner: &Arc, +) { + let weak_inner = Arc::downgrade(inner); + let udid = inner.udid.clone(); + let mut shutdown_rx = inner.shutdown_tx.subscribe(); + thread::spawn(move || loop { + if android_shutdown_requested(&mut shutdown_rx) || weak_inner.upgrade().is_none() { + break; + } + let mut stream = match bridge.shared_video_frame_stream(&udid) { + Ok(stream) => stream, + Err(error) => { + warn!("Android shared-video stream is not ready for {udid}: {error}"); + thread::sleep(ANDROID_SHARED_VIDEO_RETRY_DELAY); + continue; + } + }; + info!("Android shared-video stream attached for {udid}"); + loop { + if android_shutdown_requested(&mut shutdown_rx) { + return; + } + let Some(inner) = weak_inner.upgrade() else { + return; + }; + let quality = inner.h264_quality(); + match stream.next_frame(quality) { + Ok(Some(frame)) => { + if let Err(error) = inner.encode_android_shared_video_frame(&frame) { + warn!("Android shared-video encode failed for {udid}: {error}"); + thread::sleep(ANDROID_SHARED_VIDEO_RETRY_DELAY); + } + } + Ok(None) => {} + Err(error) => { + warn!("Android shared-video frame read failed for {udid}: {error}"); + break; + } + } + thread::sleep(android_webrtc_frame_interval()); + } + }); +} + +fn android_shutdown_requested(receiver: &mut broadcast::Receiver<()>) -> bool { + !matches!( + receiver.try_recv(), + Err(broadcast::error::TryRecvError::Empty) + ) } impl Drop for AndroidWebRtcSourceInner { fn drop(&mut self) { let _ = self.shutdown_tx.send(()); - let encoder_handle = self.encoder_handle.load(Ordering::Acquire); - let callback_user_data = self.callback_user_data.load(Ordering::Acquire); - unsafe { - if encoder_handle != 0 { - ffi::xcw_native_h264_encoder_destroy(encoder_handle as *mut c_void); + let handle = self.encoder_handle.swap(0, Ordering::AcqRel); + if handle != 0 { + unsafe { + ffi::xcw_native_h264_encoder_destroy(handle as *mut c_void); } - if callback_user_data != 0 { - let _ = Weak::from_raw(callback_user_data as *const AndroidWebRtcSourceInner); + } + let user_data = self.callback_user_data.swap(0, Ordering::AcqRel); + if user_data != 0 { + unsafe { + let _ = Weak::from_raw(user_data as *const AndroidWebRtcSourceInner); } } } } -unsafe extern "C" fn android_h264_encoder_frame_callback( - frame: *const ffi::xcw_native_frame, - user_data: *mut c_void, -) { - if frame.is_null() || user_data.is_null() { - return; +impl AndroidWebRtcSourceInner { + fn create_native_encoder(self: &Arc) -> Result<(), AppError> { + let weak = Arc::downgrade(self); + let user_data = Weak::into_raw(weak) as *mut c_void; + let mut error = ptr::null_mut(); + let handle = unsafe { + ffi::xcw_native_h264_encoder_create( + Some(android_h264_encoder_frame_callback), + user_data, + &mut error, + ) + }; + if handle.is_null() { + unsafe { + let _ = Weak::from_raw(user_data as *const AndroidWebRtcSourceInner); + } + return Err(unsafe { + take_native_error(error, "Unable to create Android native H.264 encoder.") + }); + } + self.callback_user_data + .store(user_data as usize, Ordering::Release); + self.encoder_handle + .store(handle as usize, Ordering::Release); + Ok(()) } - let weak = unsafe { Weak::from_raw(user_data as *const AndroidWebRtcSourceInner) }; - if let Some(inner) = weak.upgrade() { - unsafe { - inner.handle_encoded_frame(&*frame); + fn h264_quality(&self) -> android::AndroidH264StreamQuality { + *self.quality.lock().unwrap() + } + + fn reconfigure_h264(&self, quality: android::AndroidH264StreamQuality) { + let mut current = self.quality.lock().unwrap(); + if *current != quality { + *current = quality; + *self.latest_keyframe.write().unwrap() = None; } + drop(current); + self.request_keyframe(); } - let _ = Weak::into_raw(weak); -} -impl AndroidWebRtcSourceInner { fn request_keyframe(&self) { self.metrics .keyframe_requests .fetch_add(1, Ordering::Relaxed); - let encoder_handle = self.encoder_handle.load(Ordering::Acquire); - if encoder_handle == 0 { - return; + let handle = self.encoder_handle.load(Ordering::Acquire); + if handle != 0 { + unsafe { + ffi::xcw_native_h264_encoder_request_keyframe(handle as *mut c_void); + } } - unsafe { - ffi::xcw_native_h264_encoder_request_keyframe(encoder_handle as *mut c_void); + } + + fn encode_android_shared_video_frame( + &self, + frame: &android::AndroidSharedVideoFrame, + ) -> Result<(), AppError> { + let handle = self.encoder_handle.load(Ordering::Acquire); + if handle == 0 { + return Err(AppError::native( + "Android native H.264 encoder is not available.", + )); + } + let mut error = ptr::null_mut(); + let ok = unsafe { + ffi::xcw_native_h264_encoder_encode_bgra( + handle as *mut c_void, + frame.bgra.as_ptr(), + frame.bgra.len(), + frame.width, + frame.height, + frame.timestamp_us, + &mut error, + ) + }; + if !ok { + return Err(unsafe { take_native_error(error, "Android native H.264 encode failed.") }); } + Ok(()) } fn handle_encoded_frame(&self, frame: &ffi::xcw_native_frame) { - let description = unsafe { copy_native_shared_bytes(frame.description) }; let Some(data) = (unsafe { copy_native_shared_bytes(frame.data) }) else { return; }; + let description = unsafe { copy_native_shared_bytes(frame.description) }; + let codec = unsafe { native_c_string(frame.codec) }; + let frame_sequence = self.frame_sequence.fetch_add(1, Ordering::Relaxed) + 1; let packet = Arc::new(FramePacket { - frame_sequence: frame.frame_sequence, + frame_sequence, timestamp_us: frame.timestamp_us, is_keyframe: frame.is_keyframe, width: frame.width, height: frame.height, - codec: native_c_string(frame.codec), + codec, description, data, }); @@ -1710,115 +1582,51 @@ impl AndroidWebRtcSourceInner { } } -fn encode_android_rgba_frame( - encoder_handle: usize, - frame: &android::AndroidFrame, -) -> Result<(), AppError> { - unsafe { - let mut error = std::ptr::null_mut(); - let ok = ffi::xcw_native_h264_encoder_encode_rgba( - encoder_handle as *mut c_void, - frame.rgba.as_ptr(), - frame.rgba.len(), - frame.width, - frame.height, - frame.timestamp_us, - &mut error, - ); - if ok { - Ok(()) - } else { - Err(take_native_error(error) - .unwrap_or_else(|| AppError::native("Android VideoToolbox encode failed."))) - } +unsafe extern "C" fn android_h264_encoder_frame_callback( + frame: *const ffi::xcw_native_frame, + user_data: *mut c_void, +) { + if frame.is_null() || user_data.is_null() { + return; } -} -fn android_rgba_webrtc_frame_chunks( - sequence: u64, - frame: &android::AndroidFrame, -) -> Option> { - let expected_bytes = frame.width as usize * frame.height as usize * 4; - if expected_bytes == 0 || frame.rgba.len() != expected_bytes { - return None; - } - let mut chunks = Vec::with_capacity(frame.rgba.len().div_ceil(ANDROID_WEBRTC_RGBA_CHUNK_BYTES)); - for (chunk_index, chunk) in frame - .rgba - .chunks(ANDROID_WEBRTC_RGBA_CHUNK_BYTES) - .enumerate() - { - let chunk_offset = chunk_index * ANDROID_WEBRTC_RGBA_CHUNK_BYTES; - let mut bytes = - BytesMut::with_capacity(ANDROID_WEBRTC_RGBA_CHUNK_HEADER_BYTES + chunk.len()); - bytes.put_u32(ANDROID_WEBRTC_RGBA_CHUNK_MAGIC); - bytes.put_u8(ANDROID_WEBRTC_RGBA_VERSION); - bytes.put_u8(ANDROID_WEBRTC_RGBA_FORMAT_RGBA8888); - bytes.put_u16(ANDROID_WEBRTC_RGBA_CHUNK_HEADER_BYTES as u16); - bytes.put_u64(sequence); - bytes.put_u64(frame.timestamp_us); - bytes.put_u32(frame.width); - bytes.put_u32(frame.height); - bytes.put_u32(frame.rgba.len() as u32); - bytes.put_u32(chunk_offset as u32); - bytes.put_u32(chunk.len() as u32); - bytes.put_u8(if chunk_offset + chunk.len() >= frame.rgba.len() { - 1 - } else { - 0 - }); - bytes.put_u8(0); - bytes.put_u16(0); - bytes.extend_from_slice(chunk); - chunks.push(bytes.freeze()); + let weak = Weak::from_raw(user_data as *const AndroidWebRtcSourceInner); + if let Some(inner) = weak.upgrade() { + inner.handle_encoded_frame(&*frame); } - Some(chunks) + let _ = Weak::into_raw(weak); } unsafe fn copy_native_shared_bytes(bytes: ffi::xcw_native_shared_bytes) -> Option { - if bytes.data.is_null() || bytes.length == 0 { - if !bytes.owner.is_null() { - unsafe { - ffi::xcw_native_release_shared_bytes(bytes); - } - } - return None; - } - - let copied = - unsafe { Bytes::copy_from_slice(std::slice::from_raw_parts(bytes.data, bytes.length)) }; - unsafe { - ffi::xcw_native_release_shared_bytes(bytes); - } - Some(copied) -} - -fn native_c_string(ptr: *const i8) -> Option { - if ptr.is_null() { - return None; - } - let value = unsafe { CStr::from_ptr(ptr) } - .to_string_lossy() - .trim() - .to_owned(); - if value.is_empty() { + let copied = if bytes.data.is_null() || bytes.length == 0 { None } else { - Some(value) - } + Some(Bytes::copy_from_slice(slice::from_raw_parts( + bytes.data, + bytes.length, + ))) + }; + ffi::xcw_native_release_shared_bytes(bytes); + copied } -unsafe fn take_native_error(raw: *mut i8) -> Option { - if raw.is_null() { +unsafe fn native_c_string(value: *const i8) -> Option { + if value.is_null() { return None; } - let message = unsafe { CStr::from_ptr(raw) } - .to_string_lossy() - .into_owned(); - unsafe { - ffi::xcw_native_free_string(raw); + CStr::from_ptr(value).to_str().ok().map(ToOwned::to_owned) +} + +unsafe fn take_native_error(error: *mut i8, fallback: &str) -> AppError { + if error.is_null() { + return AppError::native(fallback); } - Some(AppError::native(message)) + let message = CStr::from_ptr(error) + .to_str() + .map(ToOwned::to_owned) + .unwrap_or_else(|_| fallback.to_owned()); + ffi::xcw_native_free_string(error); + AppError::native(message) } fn android_webrtc_frame_interval() -> Duration { @@ -1909,135 +1717,6 @@ struct WebRtcMediaStream { stream_control_rx: mpsc::UnboundedReceiver, } -struct WebRtcRgbaStream { - state: AppState, - source: AndroidWebRtcSource, - udid: String, - peer_connection: Arc, - rgba_channel: Arc, - cancellation_token: broadcast::Sender<()>, - cancellation: broadcast::Receiver<()>, - stream_control_rx: mpsc::UnboundedReceiver, -} - -impl WebRtcRgbaStream { - async fn run(self) { - let Self { - state, - source, - udid, - peer_connection, - rgba_channel, - cancellation_token, - mut cancellation, - mut stream_control_rx, - } = self; - let mut rx = source.subscribe_raw(); - let mut peer_state_interval = time::interval(Duration::from_millis(250)); - let mut peer_disconnected_since: Option = None; - let mut sequence = 0u64; - let _guard = WebRtcMetricsGuard::new(state.metrics.clone()); - rgba_channel.on_open(Box::new({ - let udid = udid.clone(); - move || { - let udid = udid.clone(); - Box::pin(async move { - info!("Android RGBA WebRTC data channel open for {udid}"); - }) - } - })); - - loop { - tokio::select! { - _ = cancellation.recv() => { - warn!("Android RGBA WebRTC stream replaced for {udid}"); - break; - } - _ = peer_state_interval.tick() => { - let peer_state = peer_connection.connection_state(); - if matches!(peer_state, RTCPeerConnectionState::Closed | RTCPeerConnectionState::Failed) { - warn!("Android RGBA WebRTC stream closing for {udid}: peer state {peer_state}"); - break; - } - if peer_state == RTCPeerConnectionState::Disconnected { - let disconnected_since = - peer_disconnected_since.get_or_insert_with(time::Instant::now); - if disconnected_since.elapsed() >= WEBRTC_PEER_DISCONNECTED_TIMEOUT { - warn!("Android RGBA WebRTC stream closing for {udid}: peer state {peer_state}"); - break; - } - } else { - peer_disconnected_since = None; - } - } - command = stream_control_rx.recv() => { - let Some(command) = command else { - continue; - }; - if command.force_keyframe || command.snapshot { - source.request_refresh(); - } - } - frame = rx.recv() => { - let frame = match frame { - Ok(frame) => frame, - Err(broadcast::error::RecvError::Lagged(skipped)) => { - state - .metrics - .frames_dropped_server - .fetch_add(skipped, Ordering::Relaxed); - continue; - } - Err(broadcast::error::RecvError::Closed) => { - warn!("Android RGBA WebRTC stream closing for {udid}: raw frame channel closed"); - break; - } - }; - if rgba_channel.ready_state() != RTCDataChannelState::Open { - state.metrics.frames_dropped_server.fetch_add(1, Ordering::Relaxed); - continue; - } - let Some(chunks) = android_rgba_webrtc_frame_chunks(sequence, &frame) else { - state.metrics.frames_dropped_server.fetch_add(1, Ordering::Relaxed); - continue; - }; - let buffered_amount = rgba_channel.buffered_amount().await; - if buffered_amount > frame.rgba.len() * ANDROID_WEBRTC_RGBA_BUFFERED_FRAME_LIMIT { - state.metrics.frames_dropped_server.fetch_add(1, Ordering::Relaxed); - continue; - } - sequence = sequence.wrapping_add(1); - let mut sent_frame = true; - for chunk in chunks { - let send_result = time::timeout(WEBRTC_REALTIME_WRITE_TIMEOUT, rgba_channel.send(&chunk)).await; - match send_result { - Ok(Ok(_)) => {} - Ok(Err(error)) => { - warn!("Android RGBA WebRTC data channel send failed for {udid}: {error}"); - sent_frame = false; - break; - } - Err(_) => { - sent_frame = false; - break; - } - } - } - if sent_frame { - state.metrics.frames_sent.fetch_add(1, Ordering::Relaxed); - } else { - state.metrics.frames_dropped_server.fetch_add(1, Ordering::Relaxed); - } - } - } - } - - warn!("Android RGBA WebRTC stream ended for {udid}"); - clear_webrtc_media_stream(&udid, &cancellation_token); - let _ = peer_connection.close().await; - } -} - impl WebRtcMediaStream { async fn run(self) { let Self { @@ -2643,14 +2322,11 @@ impl Drop for WebRtcMetricsGuard { #[cfg(test)] mod tests { use super::{ - android_rgba_webrtc_frame_chunks, append_avcc_parameter_sets, append_length_prefixed_nalus, - h264_annex_b_sample, h264_frame_has_idr, h264_frame_is_decoder_sync, h264_sdp_fmtp_line, - is_annex_b, is_h264_codec, rtcp_packet_requests_keyframe, rtp_packet_pacing, - WebRtcMetricsGuard, WebRtcSendTiming, ANDROID_WEBRTC_RGBA_CHUNK_BYTES, - ANDROID_WEBRTC_RGBA_CHUNK_HEADER_BYTES, ANDROID_WEBRTC_RGBA_CHUNK_MAGIC, - ANDROID_WEBRTC_RGBA_FORMAT_RGBA8888, ANDROID_WEBRTC_RGBA_VERSION, ANNEX_B_START_CODE, + append_avcc_parameter_sets, append_length_prefixed_nalus, h264_annex_b_sample, + h264_frame_has_idr, h264_frame_is_decoder_sync, h264_sdp_fmtp_line, is_annex_b, + is_h264_codec, rtcp_packet_requests_keyframe, rtp_packet_pacing, WebRtcMetricsGuard, + WebRtcSendTiming, ANNEX_B_START_CODE, }; - use crate::android; use crate::metrics::counters::Metrics; use crate::transport::packet::FramePacket; use bytes::Bytes; @@ -3135,71 +2811,6 @@ mod tests { ); } - #[test] - fn android_rgba_webrtc_frame_chunks_use_fixed_binary_header() { - let rgba = (0..(320 * 240 * 4)) - .map(|value| (value % 251) as u8) - .collect::>(); - let frame = android::AndroidFrame { - width: 320, - height: 240, - timestamp_us: 123_456, - rgba: rgba.clone(), - }; - - let chunks = android_rgba_webrtc_frame_chunks(7, &frame).unwrap(); - - assert_eq!(chunks.len(), 2); - let first = &chunks[0]; - assert_eq!( - u32::from_be_bytes(first[0..4].try_into().unwrap()), - ANDROID_WEBRTC_RGBA_CHUNK_MAGIC - ); - assert_eq!(first[4], ANDROID_WEBRTC_RGBA_VERSION); - assert_eq!(first[5], ANDROID_WEBRTC_RGBA_FORMAT_RGBA8888); - assert_eq!( - u16::from_be_bytes(first[6..8].try_into().unwrap()) as usize, - ANDROID_WEBRTC_RGBA_CHUNK_HEADER_BYTES - ); - assert_eq!(u64::from_be_bytes(first[8..16].try_into().unwrap()), 7); - assert_eq!( - u64::from_be_bytes(first[16..24].try_into().unwrap()), - 123_456 - ); - assert_eq!(u32::from_be_bytes(first[24..28].try_into().unwrap()), 320); - assert_eq!(u32::from_be_bytes(first[28..32].try_into().unwrap()), 240); - assert_eq!( - u32::from_be_bytes(first[32..36].try_into().unwrap()) as usize, - rgba.len() - ); - assert_eq!(u32::from_be_bytes(first[36..40].try_into().unwrap()), 0); - assert_eq!( - u32::from_be_bytes(first[40..44].try_into().unwrap()) as usize, - ANDROID_WEBRTC_RGBA_CHUNK_BYTES - ); - assert_eq!(first[44], 0); - - let second = &chunks[1]; - assert_eq!( - u32::from_be_bytes(second[36..40].try_into().unwrap()) as usize, - ANDROID_WEBRTC_RGBA_CHUNK_BYTES - ); - assert_eq!( - u32::from_be_bytes(second[40..44].try_into().unwrap()) as usize, - rgba.len() - ANDROID_WEBRTC_RGBA_CHUNK_BYTES - ); - assert_eq!(second[44], 1); - let reassembled = chunks - .iter() - .flat_map(|chunk| { - chunk[ANDROID_WEBRTC_RGBA_CHUNK_HEADER_BYTES..] - .iter() - .copied() - }) - .collect::>(); - assert_eq!(reassembled, rgba); - } - trait CloneFrameForTest { fn clone_for_test(&self, frame_sequence: u64) -> Self; } diff --git a/skills/simdeck/SKILL.md b/skills/simdeck/SKILL.md index fda1b65d..9341b57d 100644 --- a/skills/simdeck/SKILL.md +++ b/skills/simdeck/SKILL.md @@ -5,7 +5,7 @@ description: Use for simulator lifecycle, app install/launch, live viewing, UI i # SimDeck Agent Guide -SimDeck automates iOS Simulators and Android emulators. Use the CLI for automation and the browser UI for live human visibility. iOS works with NativeScript, UIKit, SwiftUI, React Native, Expo, and Flutter apps; Android works through ADB, emulator lifecycle, screenshots, logs, and UIAutomator hierarchy dumps. +SimDeck automates iOS Simulators and Android emulators. Use the CLI for automation and the browser UI for live human visibility. iOS works with NativeScript, UIKit, SwiftUI, React Native, Expo, and Flutter apps; Android live viewing uses the emulator `-share-vid` shared display surface plus SimDeck's native host H.264 encoder, with lifecycle, screenshots, logs, and UIAutomator hierarchy dumps also handled through ADB. SimDeck uses one long-running local service. Run `simdeck` first; it starts or reuses the background service and prints the browser URL plus pairing code. From 66eeeed5017339dd7761be8202ab292ef293435e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 4 Jun 2026 22:17:22 -0400 Subject: [PATCH 2/5] fix: improve android display streaming --- README.md | 4 +- docs/api/health.md | 4 +- docs/api/rest.md | 17 +- docs/cli/commands.md | 6 - docs/cli/flags.md | 25 +- docs/extensions/browser-client.md | 1 - docs/guide/architecture.md | 2 +- docs/guide/service.md | 19 +- docs/guide/troubleshooting.md | 6 +- docs/guide/video.md | 38 +- packages/client/src/app/AppShell.tsx | 196 ++- .../src/features/simulators/SimulatorMenu.tsx | 19 +- packages/client/src/features/stream/stats.ts | 4 - .../client/src/features/stream/streamTypes.ts | 6 +- .../stream/streamWorkerClient.test.ts | 6 +- .../src/features/stream/streamWorkerClient.ts | 1062 +---------------- .../src/features/stream/useLiveStream.ts | 3 +- .../server/native/bridge/XCWNativeBridge.m | 2 +- packages/server/native_stubs.c | 16 + packages/server/src/android.rs | 489 +++++--- packages/server/src/api/routes.rs | 545 +-------- packages/server/src/main.rs | 154 ++- packages/server/src/metrics/counters.rs | 8 - packages/server/src/service.rs | 7 + packages/server/src/transport/webrtc.rs | 190 ++- skills/simdeck/SKILL.md | 3 + 26 files changed, 807 insertions(+), 2025 deletions(-) diff --git a/README.md b/README.md index 449ba0fc..4e8ab93e 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,9 @@ Normal service restarts preserve that token so paired clients stay connected. Use `simdeck service reset` only when you want to rotate the service token and restart the LaunchAgent. The service uses port 4310 unless you pass `-p` or `--port`. +SimDeck-owned Android emulator boots use host GPU rendering by default; use +`simdeck service restart --android-gpu auto` or +`--android-gpu swiftshader_indirect` only as a machine-specific fallback. Use `simdeck service kill` when you want to stop every SimDeck service process, including services started from another checkout or installed binary. @@ -130,7 +133,6 @@ simdeck pasteboard get simdeck screenshot --output screen.png simdeck screenshot --with-bezel --output screen-bezel.png simdeck record --seconds 5 --output screen-recording.mp4 -simdeck stream --frames 120 > stream.h264 simdeck describe simdeck describe --format agent --max-depth 4 simdeck describe --format agent --max-depth 4 --interactive diff --git a/docs/api/health.md b/docs/api/health.md index 1c734eaa..6100e50f 100644 --- a/docs/api/health.md +++ b/docs/api/health.md @@ -21,6 +21,7 @@ Example: "serverKind": "launchAgent", "timestamp": 1714094761.234, "videoCodec": "auto", + "androidGpu": "host", "lowLatency": false, "realtimeStream": true, "localStreamFps": 60, @@ -46,6 +47,7 @@ Important fields: | `httpPort` | Port serving UI and API | | `serverKind` | `launchAgent` or `standalone` | | `videoCodec` | Requested codec mode: `auto`, `hardware`, or `software` | +| `androidGpu` | Android emulator renderer mode for SimDeck-owned boots | | `streamQuality` | Active stream profile and limits | | `webRtc` | ICE settings the browser should use | @@ -90,7 +92,7 @@ Content-Type: application/json "clientId": "browser-ABC", "kind": "viewport", "udid": "9D7E5BB7-...", - "codec": "h264", + "codec": "video/H264/103", "decodedFps": 59.7, "droppedFps": 0.0, "latestRenderMs": 6.2 diff --git a/docs/api/rest.md b/docs/api/rest.md index 5ccd3d0f..5acba0e8 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -165,15 +165,14 @@ Performance query parameters: ## Live video -| Method | Path | Purpose | -| ------ | ------------------------------------- | -------------------------------------- | -| `POST` | `/api/simulators/{udid}/webrtc/offer` | WebRTC offer/answer stream setup | -| `GET` | `/api/simulators/{udid}/h264` | H.264 WebSocket fallback | -| `GET` | `/api/simulators/{udid}/input` | Input WebSocket for fallback transport | -| `GET` | `/api/simulators/{udid}/control` | Alias for input control WebSocket | -| `POST` | `/api/simulators/{udid}/refresh` | Request a fresh frame or keyframe | - -For normal clients, copy the browser behavior instead of hand-coding a raw decoder. The UI supports WebRTC first and H.264 WebSocket fallback. Android emulator IDs use the same endpoints; their H.264 frames are produced from the emulator `-share-vid` display surface, not screenshot polling. +| Method | Path | Purpose | +| ------ | ------------------------------------- | ------------------------------------- | +| `POST` | `/api/simulators/{udid}/webrtc/offer` | WebRTC offer/answer stream setup | +| `GET` | `/api/simulators/{udid}/input` | Input WebSocket fallback for controls | +| `GET` | `/api/simulators/{udid}/control` | Alias for input control WebSocket | +| `POST` | `/api/simulators/{udid}/refresh` | Request a fresh frame or keyframe | + +For normal clients, copy the browser behavior instead of hand-coding a raw decoder. The UI uses the WebRTC offer endpoint for live video. Android emulator IDs use the same WebRTC endpoint; their H.264 frames are produced from the emulator `-share-vid` display surface, not screenshot polling. Minimal WebRTC request: diff --git a/docs/cli/commands.md b/docs/cli/commands.md index 0b543610..69a604d1 100644 --- a/docs/cli/commands.md +++ b/docs/cli/commands.md @@ -207,12 +207,6 @@ simdeck logs --seconds 30 --limit 200 simdeck chrome-profile ``` -Diagnostic iOS H.264 stream: - -```sh -simdeck stream --frames 120 > stream.h264 -``` - ## Studio and providers For hosted Studio workflows: diff --git a/docs/cli/flags.md b/docs/cli/flags.md index 79741f39..33e8aa09 100644 --- a/docs/cli/flags.md +++ b/docs/cli/flags.md @@ -26,18 +26,19 @@ When `service restart` is run without `--port`, it preserves the installed LaunchAgent port or the current singleton service port before falling back to `4310`. -| Flag | Default | Notes | -| ---------------------------- | -------------- | --------------------------------------------------------------------------------- | -| `--port ` / `-p` | `4310` | HTTP port; `service restart` preserves the existing service port when omitted | -| `--bind ` | `127.0.0.1` | Use `0.0.0.0` or `::` for LAN access | -| `--advertise-host ` | detected | Host printed for remote browsers | -| `--client-root ` | bundled client | Static client directory | -| `--video-codec ` | `auto` | `auto`, `hardware`, or `software` | -| `--stream-quality ` | `full` | `full`, `balanced`, `economy`, `low`, `tiny`, `ci-software`, and related profiles | -| `--local-stream-fps ` | `60` | Local stream frame target | -| `--low-latency` | off | Conservative software H.264 profile | -| `--open` | off | Open the browser after starting the service | -| `--autostart` / `-a` | off | Register the service as a macOS LaunchAgent | +| Flag | Default | Notes | +| ---------------------------- | -------------- | ---------------------------------------------------------------------------------------------------------------------- | +| `--port ` / `-p` | `4310` | HTTP port; `service restart` preserves the existing service port when omitted | +| `--bind ` | `127.0.0.1` | Use `0.0.0.0` or `::` for LAN access | +| `--advertise-host ` | detected | Host printed for remote browsers | +| `--client-root ` | bundled client | Static client directory | +| `--video-codec ` | `auto` | `auto`, `hardware`, or `software` | +| `--android-gpu ` | `host` | Android emulator renderer: `host`, `auto`, `software`, `lavapipe`, `swiftshader`, `swangle`, or `swiftshader_indirect` | +| `--stream-quality ` | `full` | `full`, `balanced`, `smooth`, `economy`, `low`, `tiny`, `ci-software`, and related profiles | +| `--local-stream-fps ` | `60` | Local stream frame target | +| `--low-latency` | off | Conservative software H.264 profile | +| `--open` | off | Open the browser after starting the service | +| `--autostart` / `-a` | off | Register the service as a macOS LaunchAgent | ## `describe` diff --git a/docs/extensions/browser-client.md b/docs/extensions/browser-client.md index 6933c95d..2417161a 100644 --- a/docs/extensions/browser-client.md +++ b/docs/extensions/browser-client.md @@ -38,7 +38,6 @@ Force a stream transport while debugging: ```text http://127.0.0.1:4310?stream=webrtc -http://127.0.0.1:4310?stream=h264 ``` Use the default URL for normal operation. diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index 5e3339b1..63226317 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -26,7 +26,7 @@ This is why a long-lived service feels faster than repeatedly calling lower-leve ## Video flow -The browser opens a live stream for the selected device. SimDeck sends fresh frames, drops stale ones when a client falls behind, and lets the browser request refreshes. iOS frames come from the native display bridge and are encoded on the Mac; Android frames come from the emulator `-share-vid` shared display surface and are encoded on the Mac. The UI can use WebRTC or H.264-over-WebSocket fallback depending on browser support and network behavior. +The browser opens a live WebRTC stream for the selected device. SimDeck sends fresh frames, drops stale ones when a client falls behind, and lets the browser request refreshes. iOS frames come from the native display bridge and are encoded on the Mac; Android frames come from the emulator `-share-vid` shared display surface and are encoded on the Mac. Tune this from the user-facing controls or with: diff --git a/docs/guide/service.md b/docs/guide/service.md index 81f886ad..4291b90c 100644 --- a/docs/guide/service.md +++ b/docs/guide/service.md @@ -63,15 +63,16 @@ LaunchAgent port or the current singleton service port before falling back to These options are accepted by `simdeck`, `service start`, `service restart`, `service on`, and `service reset`: -| Flag | Default | Use it when | -| ---------------------------- | ----------- | ------------------------------------------ | -| `--port ` / `-p` | `4310` | You want a specific service port | -| `--bind ` | `127.0.0.1` | You need LAN access with `0.0.0.0` or `::` | -| `--advertise-host ` | detected | Remote browsers need a specific host or IP | -| `--video-codec ` | `auto` | You need to force encoder behavior | -| `--stream-quality ` | `full` | You want lower CPU or bandwidth use | -| `--local-stream-fps ` | `60` | You want a different local stream target | -| `--client-root ` | bundled UI | You are serving a custom static client | +| Flag | Default | Use it when | +| ---------------------------- | ----------- | ------------------------------------------------- | +| `--port ` / `-p` | `4310` | You want a specific service port | +| `--bind ` | `127.0.0.1` | You need LAN access with `0.0.0.0` or `::` | +| `--advertise-host ` | detected | Remote browsers need a specific host or IP | +| `--video-codec ` | `auto` | You need to force encoder behavior | +| `--android-gpu ` | `host` | You need to change Android emulator GPU rendering | +| `--stream-quality ` | `full` | You want lower CPU or bandwidth use | +| `--local-stream-fps ` | `60` | You want a different local stream target | +| `--client-root ` | bundled UI | You are serving a custom static client | ## Restart CoreSimulator diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index 99f02ea2..7ad04818 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -137,14 +137,14 @@ curl http://127.0.0.1:4310/api/metrics If `frames_dropped_server` keeps climbing, the client or network cannot keep up. Move closer to the host, reduce quality, or switch to software encoding. -For Android emulator streams, SimDeck uses the emulator `-share-vid` shared display surface. If Android video never starts, confirm `adb devices` shows the emulator as `device`, that it has fully booted, and that externally launched emulators were started with `-share-vid`. SimDeck-owned Android boots add the flag automatically. +For Android emulator streams, SimDeck uses the emulator `-share-vid` shared display surface. If Android video never starts, confirm `adb devices` shows the emulator as `device`, that it has fully booted, and that externally launched emulators were started with `-share-vid`. SimDeck-owned Android boots add the flag automatically and default to `--android-gpu host`; try `simdeck service restart --android-gpu auto` or `--android-gpu swiftshader_indirect` only when host rendering is unstable. ### Browser cannot establish WebRTC -Force the H.264 WebSocket fallback while testing: +Use `?stream=webrtc` while testing to make transport selection explicit: ```text -http://127.0.0.1:4310?stream=h264 +http://127.0.0.1:4310?stream=webrtc ``` For routed remote sessions, configure TURN as described in [Video and streaming](/guide/video#remote-browsers). diff --git a/docs/guide/video.md b/docs/guide/video.md index cdb9d108..9937032c 100644 --- a/docs/guide/video.md +++ b/docs/guide/video.md @@ -1,6 +1,6 @@ # Video and streaming -SimDeck streams live device video to the browser. Local sessions default to high quality. Remote or constrained sessions can trade detail for lower CPU and latency. +SimDeck streams live device video to the browser. Local sessions default to full-resolution 60 fps. Remote or constrained sessions can trade detail for lower CPU and latency. iOS simulator H.264 uses VideoToolbox for hardware encoding and x264 for software encoding. Android emulator H.264 uses the emulator `-share-vid` display surface. SimDeck reads BGRA frames from the `videmulator` shared memory region and encodes them on the Mac, so normal Android live video stays on the native shared display path. @@ -8,10 +8,12 @@ Android emulator H.264 uses the emulator `-share-vid` display surface. SimDeck r ## When encoding runs SimDeck starts encoding when a browser stream needs H.264 frames. For iOS, the -server requests an initial keyframe to answer the WebRTC or H.264 WebSocket -viewer, then keeps a shared refresh pump active while frame subscribers exist. +server requests an initial keyframe to answer the WebRTC viewer, then keeps a +shared refresh pump active while frame subscribers exist. For Android, SimDeck starts emulators with `-share-vid`, maps the shared display region, and feeds changed BGRA frames into the native host H.264 encoder. +SimDeck-owned Android boots also default to `-gpu host`, matching the native +emulator app's accelerated renderer while staying in headless shared-video mode. The browser reports whether the page and stream canvas are foreground. When all known viewers are hidden or the last frame subscriber disconnects, the native @@ -38,7 +40,8 @@ Common profiles: | Profile | Use it for | | ------------- | --------------------------------------- | -| `full` | Local browser on a fast Mac | +| `full` | Default local full-resolution 60 fps | +| `smooth` | Full-size 60 fps with lower bitrate | | `balanced` | Good local quality with less bandwidth | | `economy` | Remote browser or busy machine | | `low` | Slower Wi-Fi or shared hosts | @@ -47,6 +50,18 @@ Common profiles: The browser also has stream controls for transport, resolution, FPS, and refresh. +## Pick an Android GPU mode + +SimDeck-owned Android emulator boots use host GPU acceleration by default: + +```sh +simdeck service restart --android-gpu host +``` + +Use `auto` to let the Android emulator choose the renderer. Use +`swiftshader_indirect`, `swiftshader`, `software`, `lavapipe`, or `swangle` only +when host rendering is unstable on a specific machine. + ## Pick a codec ```sh @@ -61,11 +76,11 @@ simdeck service restart --video-codec software | `hardware` | Dedicated local machines where VideoToolbox hardware H.264 is reliable. | | `software` | x264 software H.264 for CI, screen recording conflicts, or hardware encoder stalls. | -The codec setting controls iOS simulator host encoding. Android emulator streams -use a dedicated host encoder for shared display frames; set -`SIMDECK_ANDROID_VIDEO_CODEC=hardware` or `software` before starting the service -when you need to override Android's encoder choice. Stream quality controls the -encoded Android frame size. +The codec setting controls simulator host encoding. Android emulator streams use +the same service codec by default for shared display frames; set +`SIMDECK_ANDROID_VIDEO_CODEC=auto`, `hardware`, or `software` before starting the +service only when you need an Android-specific encoder override. Stream quality +controls the encoded Android frame size. When multiple simulator streams run at the same time, `auto` keeps one active stream on the hardware encoder path and routes additional active auto streams to @@ -78,15 +93,14 @@ For very constrained software sessions: simdeck service restart --video-codec software --low-latency ``` -## WebRTC and fallback +## WebRTC -The browser tries WebRTC first. If WebRTC cannot render a frame, the UI can fall back to H.264 over WebSocket when the browser supports WebCodecs. +The browser uses WebRTC for live video. SimDeck no longer exposes a separate H.264 WebSocket video transport. Force a mode while debugging: ```text http://127.0.0.1:4310?stream=webrtc -http://127.0.0.1:4310?stream=h264 ``` ## Remote browsers diff --git a/packages/client/src/app/AppShell.tsx b/packages/client/src/app/AppShell.tsx index a670cea6..499f93a3 100644 --- a/packages/client/src/app/AppShell.tsx +++ b/packages/client/src/app/AppShell.tsx @@ -133,9 +133,6 @@ const REMOTE_STREAM_DEFAULTS: StreamConfig = { fps: 30, quality: "balanced", }; -const H264_WS_DEFAULT_FPS = 60; -const H264_WS_LOCAL_DEFAULT_QUALITY: StreamQualityPreset = "full"; -const H264_WS_REMOTE_DEFAULT_QUALITY: StreamQualityPreset = "auto"; const CONTROL_BACKLOG_DROP_BYTES = 4096; const STREAM_CONFIG_USER_CHANGE_GRACE_MS = 1000; const STREAM_ENCODER_VALUES = new Set([ @@ -143,11 +140,7 @@ const STREAM_ENCODER_VALUES = new Set([ "hardware", "software", ]); -const STREAM_TRANSPORT_VALUES = new Set([ - "auto", - "h264", - "webrtc", -]); +const STREAM_TRANSPORT_VALUES = new Set(["auto", "webrtc"]); const MOBILE_VIEWPORT_MEDIA_QUERY = "(max-width: 600px)"; const CHROME_RENDERER_ASSET_VERSION = "chrome-renderer-button-overlay-23"; clearLegacyVolatileUiState(); @@ -280,9 +273,6 @@ function shouldUseRemoteStreamDefault(apiRoot: string): boolean { function readStreamTransportQueryParam(): StreamTransport { const value = new URLSearchParams(window.location.search).get("stream"); - if (value === "h264-ws") { - return "h264"; - } return value && STREAM_TRANSPORT_VALUES.has(value as StreamTransport) ? (value as StreamTransport) : "auto"; @@ -290,19 +280,9 @@ function readStreamTransportQueryParam(): StreamTransport { function defaultStreamConfigForTransport( remote: boolean, - transport: StreamTransport, + _transport: StreamTransport, ): StreamConfig { const base = remote ? REMOTE_STREAM_DEFAULTS : LOCAL_STREAM_DEFAULTS; - if (transport === "h264") { - return { - ...base, - fps: H264_WS_DEFAULT_FPS, - maxEdge: undefined, - quality: remote - ? H264_WS_REMOTE_DEFAULT_QUALITY - : H264_WS_LOCAL_DEFAULT_QUALITY, - }; - } return base; } @@ -752,38 +732,39 @@ export function AppShell({ [], ); - const syncStreamConfig = useCallback(async () => { - const requestId = ++streamConfigRequestIdRef.current; - try { - const response = await apiRequest( - "/api/stream-quality", - ); - if (requestId !== streamConfigRequestIdRef.current) { - return; - } - if ( - Date.now() - streamConfigUserChangeAtRef.current < - STREAM_CONFIG_USER_CHANGE_GRACE_MS - ) { - return; - } - if (streamTransport === "h264" && !streamConfigUserTouchedRef.current) { - return; - } - setStreamConfig((current) => - mergeStreamQualityResponse(current, response, { - preserveAutoQuality: streamTransport === "h264", - }), - ); - } catch { - // Keep the existing local/default selection; the stream path will surface - // provider reachability errors separately. - } finally { - if (requestId === streamConfigRequestIdRef.current) { - setStreamConfigReady(true); + const syncStreamConfig = useCallback( + async (options?: { ignoreUserGrace?: boolean }) => { + const requestId = ++streamConfigRequestIdRef.current; + try { + const response = await apiRequest( + "/api/stream-quality", + ); + if (requestId !== streamConfigRequestIdRef.current) { + return; + } + if ( + !options?.ignoreUserGrace && + Date.now() - streamConfigUserChangeAtRef.current < + STREAM_CONFIG_USER_CHANGE_GRACE_MS + ) { + return; + } + setStreamConfig((current) => + mergeStreamQualityResponse(current, response, { + preserveAutoQuality: false, + }), + ); + } catch { + // Keep the existing local/default selection; the stream path will surface + // provider reachability errors separately. + } finally { + if (requestId === streamConfigRequestIdRef.current) { + setStreamConfigReady(true); + } } - } - }, [streamTransport]); + }, + [streamTransport], + ); useEffect(() => { let cancelled = false; @@ -828,8 +809,10 @@ export function AppShell({ ) { return; } + setStreamConfigReady(false); + void syncStreamConfig({ ignoreUserGrace: true }); void refreshRef.current(); - }, [streamStatus.error, streamStatus.state]); + }, [streamStatus.error, streamStatus.state, syncStreamConfig]); const updateStreamEncoder = useCallback((encoder: StreamEncoder) => { streamConfigUserTouchedRef.current = true; @@ -855,27 +838,10 @@ export function AppShell({ setStreamConfig((current) => ({ ...current, maxEdge: undefined, quality })); }, []); - const updateStreamTransport = useCallback( - (transport: StreamTransport) => { - setStreamTransport(transport); - writeStreamTransportQueryParam(transport); - if (transport !== "h264" || streamConfigUserTouchedRef.current) { - return; - } - streamConfigUserChangeAtRef.current = Date.now(); - setStreamConfigReady(true); - setStreamConfigApplyKey((current) => current + 1); - setStreamConfig((current) => ({ - ...current, - fps: H264_WS_DEFAULT_FPS, - maxEdge: undefined, - quality: remoteStream - ? H264_WS_REMOTE_DEFAULT_QUALITY - : H264_WS_LOCAL_DEFAULT_QUALITY, - })); - }, - [remoteStream], - ); + const updateStreamTransport = useCallback((transport: StreamTransport) => { + setStreamTransport(transport); + writeStreamTransportQueryParam(transport); + }, []); useEffect(() => { if ( @@ -1930,7 +1896,7 @@ export function AppShell({ const screenOnlyStyle = !viewportChromeProfile && chromeProfile && chromeProfile.screenWidth > 0 ? isAndroidViewport - ? androidScreenRadiusStyle(chromeProfile, effectiveDeviceNaturalSize) + ? androidScreenRadiusStyle(chromeProfile) : ({ borderRadius: `${Math.min( chromeProfile.cornerRadius * @@ -3142,60 +3108,78 @@ export function AppShell({ function androidScreenRadiusStyle( chromeProfile: ChromeProfile, - displaySize: Size | null, ): CSSProperties | null { - const screenWidth = - displaySize && displaySize.width > 0 - ? displaySize.width - : chromeProfile.screenWidth; - if (screenWidth <= 0) { + const screenWidth = chromeProfile.screenWidth; + const screenHeight = + chromeProfile.screenHeight > 0 ? chromeProfile.screenHeight : screenWidth; + if (screenWidth <= 0 || screenHeight <= 0) { return null; } - const scale = DEVICE_SCREEN_WIDTH / screenWidth; - const maxRadius = DEVICE_SCREEN_WIDTH / 2; const radii = chromeProfile.cornerRadii; - const topLeft = scaledScreenRadius( + const topLeft = screenRadiusPercentPair( radii?.topLeft ?? chromeProfile.cornerRadius, - scale, - maxRadius, + screenWidth, + screenHeight, ); - const topRight = scaledScreenRadius( + const topRight = screenRadiusPercentPair( radii?.topRight ?? chromeProfile.cornerRadius, - scale, - maxRadius, + screenWidth, + screenHeight, ); - const bottomRight = scaledScreenRadius( + const bottomRight = screenRadiusPercentPair( radii?.bottomRight ?? chromeProfile.cornerRadius, - scale, - maxRadius, + screenWidth, + screenHeight, ); - const bottomLeft = scaledScreenRadius( + const bottomLeft = screenRadiusPercentPair( radii?.bottomLeft ?? chromeProfile.cornerRadius, - scale, - maxRadius, + screenWidth, + screenHeight, ); - if (topLeft <= 0 && topRight <= 0 && bottomRight <= 0 && bottomLeft <= 0) { + if ( + topLeft.radius <= 0 && + topRight.radius <= 0 && + bottomRight.radius <= 0 && + bottomLeft.radius <= 0 + ) { return null; } - const borderRadius = `${topLeft}px ${topRight}px ${bottomRight}px ${bottomLeft}px`; + const borderRadius = [ + `${topLeft.x} ${topRight.x} ${bottomRight.x} ${bottomLeft.x}`, + `${topLeft.y} ${topRight.y} ${bottomRight.y} ${bottomLeft.y}`, + ].join(" / "); + return { borderRadius, - borderTopLeftRadius: `${topLeft}px`, - borderTopRightRadius: `${topRight}px`, - borderBottomRightRadius: `${bottomRight}px`, - borderBottomLeftRadius: `${bottomLeft}px`, - clipPath: `inset(0 round ${borderRadius})`, + overflow: "hidden", }; } -function scaledScreenRadius(radius: number, scale: number, maxRadius: number) { +function screenRadiusPercentPair( + radius: number, + screenWidth: number, + screenHeight: number, +) { if (!Number.isFinite(radius) || radius <= 0) { - return 0; + return { + radius: 0, + x: "0%", + y: "0%", + }; } - return Math.min(radius * scale, maxRadius); + const clampedRadius = Math.min(radius, screenWidth / 2, screenHeight / 2); + return { + radius: clampedRadius, + x: formatCssPercent((clampedRadius / screenWidth) * 100), + y: formatCssPercent((clampedRadius / screenHeight) * 100), + }; +} + +function formatCssPercent(value: number) { + return `${Number(value.toFixed(6))}%`; } function androidDisplayKeyForSimulator(simulator: SimulatorMetadata): string { diff --git a/packages/client/src/features/simulators/SimulatorMenu.tsx b/packages/client/src/features/simulators/SimulatorMenu.tsx index 0347f7bf..1b963f81 100644 --- a/packages/client/src/features/simulators/SimulatorMenu.tsx +++ b/packages/client/src/features/simulators/SimulatorMenu.tsx @@ -93,7 +93,7 @@ export function SimulatorMenu({ const fpsOptions = remoteStream ? REMOTE_STREAM_FPS_OPTIONS : LOCAL_STREAM_FPS_OPTIONS; - const qualityOptions = H264_STREAM_QUALITY_OPTIONS; + const qualityOptions = STREAM_QUALITY_OPTIONS; const activeQualityOption = qualityOptions.some( (option) => option.value === streamConfig.quality, ) @@ -368,7 +368,6 @@ const STREAM_ENCODERS: Array<{ label: string; value: StreamEncoder }> = [ const STREAM_TRANSPORTS: Array<{ label: string; value: StreamTransport }> = [ { label: "Auto", value: "auto" }, { label: "WebRTC", value: "webrtc" }, - { label: "H264 WS", value: "h264" }, ]; const LOCAL_STREAM_FPS_OPTIONS: Array<{ label: string; value: StreamFps }> = [ @@ -383,41 +382,41 @@ const REMOTE_STREAM_FPS_OPTIONS: Array<{ label: string; value: StreamFps }> = [ { label: "60", value: 60 }, ]; -const H264_STREAM_QUALITY_OPTIONS: Array<{ +const STREAM_QUALITY_OPTIONS: Array<{ label: string; value: StreamQualityPreset; }> = [ { label: "Auto", value: "auto" }, - { label: "Full", value: "full" }, + { label: "Full res", value: "full" }, { label: "1280", value: "balanced" }, + { label: "Smooth", value: "smooth" }, { label: "1080", value: "economy" }, { label: "720", value: "low" }, { label: "540", value: "tiny" }, ]; -const H264_QUALITY_LABELS: Partial> = { +const STREAM_QUALITY_LABELS: Partial> = { auto: "Auto", balanced: "1280px", economy: "1080px", full: "Full res", low: "720px", quality: "Full+", - smooth: "1170px", + smooth: "Smooth", tiny: "540px", }; function streamQualityOptionLabel(quality: StreamQualityPreset): string { - return H264_QUALITY_LABELS[quality] ?? quality; + return STREAM_QUALITY_LABELS[quality] ?? quality; } function formatStreamConfigSummary( streamConfig: StreamConfig, transport: StreamTransport, ): string { - const transportLabel = - transport === "h264" ? "H264 WS" : transport.toUpperCase(); + const transportLabel = transport.toUpperCase(); const resolution = - H264_QUALITY_LABELS[streamConfig.quality] ?? + STREAM_QUALITY_LABELS[streamConfig.quality] ?? (typeof streamConfig.maxEdge === "number" && streamConfig.maxEdge > 0 ? `${streamConfig.maxEdge}px` : "Full res"); diff --git a/packages/client/src/features/stream/stats.ts b/packages/client/src/features/stream/stats.ts index 4162e69f..92fd86ad 100644 --- a/packages/client/src/features/stream/stats.ts +++ b/packages/client/src/features/stream/stats.ts @@ -9,10 +9,6 @@ export function createEmptyStreamStats(): StreamStats { decoderDroppedFrames: 0, droppedFrames: 0, frameSequence: 0, - h264ParseFailures: 0, - h264SocketBytes: 0, - h264SocketMessages: 0, - h264SocketMessageType: "", height: 0, iceRestartReason: "", iceRestarts: 0, diff --git a/packages/client/src/features/stream/streamTypes.ts b/packages/client/src/features/stream/streamTypes.ts index 4e9b6099..364feb5f 100644 --- a/packages/client/src/features/stream/streamTypes.ts +++ b/packages/client/src/features/stream/streamTypes.ts @@ -11,7 +11,7 @@ export interface StreamConnectTarget { export type StreamEncoder = "auto" | "hardware" | "software"; export type StreamFps = number; -export type StreamTransport = "auto" | "h264" | "webrtc"; +export type StreamTransport = "auto" | "webrtc"; export type StreamQualityPreset = | "auto" | "balanced" @@ -52,10 +52,6 @@ export interface StreamStats extends Size { decoderDroppedFrames: number; droppedFrames: number; frameSequence: number; - h264ParseFailures: number; - h264SocketBytes: number; - h264SocketMessages: number; - h264SocketMessageType: string; iceRestartReason: string; iceRestarts: number; latestFrameGapMs: number; diff --git a/packages/client/src/features/stream/streamWorkerClient.test.ts b/packages/client/src/features/stream/streamWorkerClient.test.ts index 03e6ab3a..ef3293c4 100644 --- a/packages/client/src/features/stream/streamWorkerClient.test.ts +++ b/packages/client/src/features/stream/streamWorkerClient.test.ts @@ -7,13 +7,13 @@ import { } from "./streamWorkerClient"; describe("streamWorkerClient", () => { - it("uses the common H264 WebSocket preference for Android emulator streams", () => { + it("ignores removed legacy stream transport preferences", () => { const target = buildStreamTarget("android:emulator-5554", { platform: "android-emulator", - transport: "h264", + transport: "h264" as never, }); - expect(preferredStreamBackend(target)).toBe("h264-ws"); + expect(preferredStreamBackend(target)).toBe("auto"); }); it("uses the common WebRTC preference for Android emulator streams", () => { diff --git a/packages/client/src/features/stream/streamWorkerClient.ts b/packages/client/src/features/stream/streamWorkerClient.ts index fdcee912..60f7b01b 100644 --- a/packages/client/src/features/stream/streamWorkerClient.ts +++ b/packages/client/src/features/stream/streamWorkerClient.ts @@ -27,40 +27,14 @@ const WEBRTC_REMOTE_DISCONNECTED_GRACE_MS = 10000; const WEBRTC_REMOTE_ICE_RESTART_GRACE_MS = 1500; const WEBRTC_RECONNECT_BASE_DELAY_MS = 250; const WEBRTC_RECONNECT_MAX_DELAY_MS = 1000; -const H264_WS_FIRST_FRAME_TIMEOUT_MS = 10000; -const H264_WS_STALLED_FRAME_TIMEOUT_MS = 5000; -const H264_WS_HEADER_BYTES = 40; -const H264_WS_MAGIC = 0x53444831; -const H264_WS_FLAG_KEYFRAME = 1 << 0; -const H264_WS_FLAG_CONFIG = 1 << 1; -const H264_WS_LOCAL_AUTO_PROFILES: StreamQualityPreset[] = [ - "low", - "economy", - "smooth", - "balanced", - "full", -]; -const H264_WS_REMOTE_AUTO_PROFILES: StreamQualityPreset[] = [ - "tiny", - "low", - "economy", - "smooth", - "balanced", - "full", -]; -const H264_WS_LOCAL_INITIAL_AUTO_PROFILE: StreamQualityPreset = "full"; -const H264_WS_REMOTE_INITIAL_AUTO_PROFILE: StreamQualityPreset = "low"; -const H264_WS_ADAPTIVE_SAMPLE_MS = 1000; -const H264_WS_AUTO_STABLE_SAMPLES_TO_UPGRADE = 3; const CONTROL_BACKLOG_DROP_BYTES = 4096; let activeWebRtcControlChannel: RTCDataChannel | null = null; let activeWebRtcTelemetryChannel: RTCDataChannel | null = null; let activeInputSocket: WebSocket | null = null; -let activeH264StreamSocket: WebSocket | null = null; let activeStreamClient: StreamWorkerClient | null = null; -export type StreamBackend = "h264-ws" | "webrtc"; +export type StreamBackend = "webrtc"; export function sendWebRtcControlMessage( encoded: string, @@ -74,10 +48,7 @@ export function sendWebRtcControlMessage( export function sendStreamClientStats(stats: unknown): boolean { const encoded = JSON.stringify({ stats, type: "clientStats" }); - return ( - sendDataChannelMessage(activeWebRtcTelemetryChannel, encoded) || - sendWebSocketMessage(activeH264StreamSocket, encoded) - ); + return sendDataChannelMessage(activeWebRtcTelemetryChannel, encoded); } export function sendWebRtcStreamControl(options: { @@ -97,10 +68,7 @@ function sendStreamQualityConfig(config: StreamConfig): boolean { config: streamQualityPayload(config), type: "streamQuality", }); - return ( - sendDataChannelMessage(activeWebRtcControlChannel, encoded) || - sendWebSocketMessage(activeH264StreamSocket, encoded) - ); + return sendDataChannelMessage(activeWebRtcControlChannel, encoded); } function sendDataChannelMessage( @@ -252,28 +220,10 @@ function webSocketApiUrl(path: string): string { return url.toString(); } -function isLoopbackHost(hostname: string): boolean { - return ( - hostname === "localhost" || - hostname === "127.0.0.1" || - hostname === "::1" || - hostname.endsWith(".localhost") - ); -} - export function canUseWebRtc(): boolean { return typeof RTCPeerConnection === "function"; } -export function canUseH264WebSocket(): boolean { - const { EncodedVideoChunk, VideoDecoder } = webCodecsConstructors(); - return ( - typeof WebSocket === "function" && - typeof VideoDecoder === "function" && - typeof EncodedVideoChunk === "function" - ); -} - interface StreamClientBackend { attachCanvas(canvasElement: HTMLCanvasElement): void; clear(): void; @@ -294,67 +244,6 @@ export interface VisualArtifactSample { meanDiff: number; } -interface H264WebSocketFrame { - config: Uint8Array; - height: number; - isKeyFrame: boolean; - payload: Uint8Array; - sequence: number; - timestampUs: number; - width: number; -} - -interface WebCodecsVideoFrame { - close(): void; - codedHeight?: number; - codedWidth?: number; - displayHeight?: number; - displayWidth?: number; - timestamp?: number; -} - -interface WebCodecsVideoDecoderConfig { - codec: string; - codedHeight?: number; - codedWidth?: number; - description?: BufferSource; - hardwareAcceleration?: - | "no-preference" - | "prefer-hardware" - | "prefer-software"; - optimizeForLatency?: boolean; -} - -interface WebCodecsVideoDecoder { - readonly decodeQueueSize: number; - readonly state?: string; - close(): void; - configure(config: WebCodecsVideoDecoderConfig): void; - decode(chunk: WebCodecsEncodedVideoChunk): void; -} - -interface WebCodecsEncodedVideoChunk { - readonly byteLength?: number; -} - -interface WebCodecsEncodedVideoChunkConstructor { - new (init: { - data: BufferSource; - timestamp: number; - type: "delta" | "key"; - }): WebCodecsEncodedVideoChunk; -} - -interface WebCodecsVideoDecoderConstructor { - new (init: { - error: (error: Error) => void; - output: (frame: WebCodecsVideoFrame) => void; - }): WebCodecsVideoDecoder; - isConfigSupported?: (config: WebCodecsVideoDecoderConfig) => Promise<{ - supported: boolean; - }>; -} - interface WebRtcAnswerPayload extends RTCSessionDescriptionInit { video?: { height?: number; @@ -362,848 +251,6 @@ interface WebRtcAnswerPayload extends RTCSessionDescriptionInit { }; } -interface PendingVideoFrame { - frame: WebCodecsVideoFrame; - sequence: number | null; -} - -function webCodecsConstructors(): { - EncodedVideoChunk?: WebCodecsEncodedVideoChunkConstructor; - VideoDecoder?: WebCodecsVideoDecoderConstructor; -} { - return globalThis as typeof globalThis & { - EncodedVideoChunk?: WebCodecsEncodedVideoChunkConstructor; - VideoDecoder?: WebCodecsVideoDecoderConstructor; - }; -} - -function hasArrayBufferMethod( - value: unknown, -): value is { arrayBuffer: () => Promise } { - return ( - typeof value === "object" && - value !== null && - "arrayBuffer" in value && - typeof (value as { arrayBuffer?: unknown }).arrayBuffer === "function" - ); -} - -class H264WebSocketStreamClient implements StreamClientBackend { - private adaptiveInterval = 0; - private adaptiveLastAt = 0; - private adaptiveLastDecodedFrames = 0; - private adaptiveLastRenderedFrames = 0; - private autoProfile: StreamQualityPreset = - H264_WS_REMOTE_INITIAL_AUTO_PROFILE; - private autoProfileStableSamples = 0; - private animationFrame = 0; - private canvas: HTMLCanvasElement | null = null; - private canvasContext: CanvasRenderingContext2D | null = null; - private connectGeneration = 0; - private decoder: WebCodecsVideoDecoder | null = null; - private decoderConfigKey = ""; - private frameWatchdogTimeout = 0; - private inputSocket: WebSocket | null = null; - private h264DecodeTimestampUs = 0; - private lastFrameAt = 0; - private pendingFrame: PendingVideoFrame | null = null; - private pendingFrameSequences = new Map(); - private reportedVideoHeight = 0; - private reportedVideoWidth = 0; - private shouldReconnect = false; - private stats: StreamStats = createEmptyStreamStats(); - private stalledFrameWatchdogCount = 0; - private streamingReported = false; - private streamConfig?: StreamConfig; - private streamSocket: WebSocket | null = null; - private streamTarget: StreamConnectTarget | null = null; - private waitingForKeyFrame = true; - - constructor( - private readonly onMessage: (message: WorkerToMainMessage) => void, - ) {} - - attachCanvas(canvasElement: HTMLCanvasElement) { - this.canvas = canvasElement; - this.canvasContext = canvasElement.getContext("2d", { - alpha: false, - desynchronized: true, - }); - } - - async connect(target: StreamConnectTarget) { - this.disconnect(); - if (!this.canvas) { - return; - } - if (!canUseH264WebSocket()) { - this.handleError("H264 WebSocket requires browser WebCodecs support."); - return; - } - - const generation = ++this.connectGeneration; - this.autoProfile = initialH264AutoProfile(target); - this.autoProfileStableSamples = 0; - const effectiveConfig = h264WebSocketStreamConfig( - target.streamConfig, - this.autoProfile, - ); - this.shouldReconnect = true; - this.streamTarget = target; - this.streamConfig = target.streamConfig; - this.streamingReported = false; - this.waitingForKeyFrame = true; - this.h264DecodeTimestampUs = 0; - this.stats = createEmptyStreamStats(); - this.stats.codec = "h264-ws"; - this.stats.waitingForKeyFrame = true; - this.lastFrameAt = 0; - this.stalledFrameWatchdogCount = 0; - this.onMessage({ type: "stats", stats: { ...this.stats } }); - this.onMessage({ - type: "status", - status: { detail: "Opening H264 WebSocket stream", state: "connecting" }, - }); - - const socket = new WebSocket( - webSocketApiUrl( - `/api/simulators/${encodeURIComponent(target.udid)}/h264${streamQualityQuery(effectiveConfig)}`, - ), - ); - socket.binaryType = "arraybuffer"; - this.streamSocket = socket; - socket.addEventListener("open", () => { - if (socket === this.streamSocket) { - socket.binaryType = "arraybuffer"; - activeH264StreamSocket = socket; - this.startAdaptiveQuality(generation); - } - }); - socket.addEventListener("message", (event) => { - if (socket !== this.streamSocket) { - return; - } - this.recordH264SocketMessage(event.data); - if (hasArrayBufferMethod(event.data)) { - void event.data.arrayBuffer().then((buffer) => { - if (socket === this.streamSocket) { - this.handleSocketMessage(buffer); - } - }); - return; - } - this.handleSocketMessage(event.data); - }); - socket.addEventListener("close", () => { - if (activeH264StreamSocket === socket) { - activeH264StreamSocket = null; - } - if (socket === this.streamSocket && this.shouldReconnect) { - this.handleError("H264 WebSocket stream closed."); - } - }); - socket.addEventListener("error", () => { - if (socket === this.streamSocket) { - this.handleError("H264 WebSocket stream failed."); - } - }); - - this.connectInputSocket(target, generation); - this.scheduleFrameWatchdog(generation); - } - - disconnect() { - this.shouldReconnect = false; - this.connectGeneration += 1; - this.clearAdaptiveQuality(); - this.clearFrameWatchdog(); - window.cancelAnimationFrame(this.animationFrame); - this.animationFrame = 0; - this.closePendingFrame(); - this.closeDecoder(); - this.pendingFrameSequences.clear(); - this.h264DecodeTimestampUs = 0; - this.streamSocket?.close(); - if (activeH264StreamSocket === this.streamSocket) { - activeH264StreamSocket = null; - } - this.streamSocket = null; - this.inputSocket?.close(); - if (activeInputSocket === this.inputSocket) { - activeInputSocket = null; - } - this.inputSocket = null; - this.streamingReported = false; - this.streamTarget = null; - this.lastFrameAt = 0; - this.stalledFrameWatchdogCount = 0; - this.waitingForKeyFrame = true; - this.reportedVideoHeight = 0; - this.reportedVideoWidth = 0; - } - - destroy() { - this.disconnect(); - } - - clear() { - if (!this.canvas) { - return; - } - this.ensureCanvasContext()?.clearRect( - 0, - 0, - this.canvas.width, - this.canvas.height, - ); - } - - sendControl(payload: unknown): boolean { - if ( - payload && - typeof payload === "object" && - "type" in payload && - payload.type === "streamControl" - ) { - return sendWebSocketMessage(this.streamSocket, JSON.stringify(payload)); - } - return sendWebSocketMessage(this.inputSocket, JSON.stringify(payload)); - } - - async applyStreamConfig(config?: StreamConfig) { - this.autoProfile = initialH264AutoProfile(this.streamTarget); - this.autoProfileStableSamples = 0; - const effectiveConfig = h264WebSocketStreamConfig(config, this.autoProfile); - this.streamConfig = config; - this.clearAdaptiveQuality(); - if (config?.quality === "auto") { - this.startAdaptiveQuality(this.connectGeneration); - } - if (effectiveConfig) { - if (!sendStreamQualityConfig(effectiveConfig)) { - await postStreamConfigWithAuthRetry(effectiveConfig, { - remote: this.streamTarget?.remote, - }); - } - this.sendControl({ - forceKeyframe: true, - snapshot: true, - type: "streamControl", - }); - } - } - - private connectInputSocket(target: StreamConnectTarget, generation: number) { - const socket = new WebSocket( - webSocketApiUrl( - `/api/simulators/${encodeURIComponent(target.udid)}/input`, - ), - ); - this.inputSocket = socket; - activeInputSocket = socket; - socket.addEventListener("open", () => { - if (generation === this.connectGeneration) { - activeInputSocket = socket; - } - }); - socket.addEventListener("close", () => { - if (activeInputSocket === socket) { - activeInputSocket = null; - } - }); - socket.addEventListener("error", () => { - if (generation === this.connectGeneration) { - console.warn("H264 input WebSocket failed."); - } - }); - } - - private handleSocketMessage(data: unknown) { - const frame = parseH264WebSocketFrame(data); - if (!frame) { - this.stats.h264ParseFailures += 1; - this.onMessage({ type: "stats", stats: { ...this.stats } }); - return; - } - this.stats.receivedPackets += 1; - this.stats.frameSequence = frame.sequence; - this.stats.width = frame.width; - this.stats.height = frame.height; - this.stats.codec = "h264-ws"; - - if (this.waitingForKeyFrame && !frame.isKeyFrame) { - this.stats.droppedFrames += 1; - this.stats.waitingForKeyFrame = true; - this.onMessage({ type: "stats", stats: { ...this.stats } }); - this.sendControl({ forceKeyframe: true, type: "streamControl" }); - return; - } - - if (!this.ensureDecoderConfigured(frame)) { - return; - } - const decoder = this.decoder; - if (!decoder) { - return; - } - if (decoder.state === "closed") { - this.closeDecoder(); - this.waitingForKeyFrame = true; - this.stats.waitingForKeyFrame = true; - this.sendControl({ forceKeyframe: true, type: "streamControl" }); - return; - } - this.stats.decodeQueueSize = decoder.decodeQueueSize; - - const { EncodedVideoChunk } = webCodecsConstructors(); - if (!EncodedVideoChunk) { - this.handleError("H264 WebSocket requires EncodedVideoChunk support."); - return; - } - const decodeTimestampUs = this.nextDecodeTimestampUs(frame.timestampUs); - try { - this.pendingFrameSequences.set(decodeTimestampUs, frame.sequence); - this.trimPendingFrameSequenceMap(); - decoder.decode( - new EncodedVideoChunk({ - data: frame.payload as BufferSource, - timestamp: decodeTimestampUs, - type: frame.isKeyFrame ? "key" : "delta", - }), - ); - this.waitingForKeyFrame = false; - this.stats.waitingForKeyFrame = false; - this.stats.decodeQueueSize = decoder.decodeQueueSize; - } catch (error) { - this.pendingFrameSequences.delete(decodeTimestampUs); - this.closeDecoder(); - this.waitingForKeyFrame = true; - this.stats.waitingForKeyFrame = true; - this.stats.decoderDroppedFrames += 1; - this.sendControl({ forceKeyframe: true, type: "streamControl" }); - this.onMessage({ type: "stats", stats: { ...this.stats } }); - if (!this.streamingReported) { - this.onMessage({ - type: "status", - status: { detail: "Recovering H264 decoder", state: "connecting" }, - }); - } - } - } - - private ensureDecoderConfigured(frame: H264WebSocketFrame): boolean { - const { VideoDecoder } = webCodecsConstructors(); - if (!VideoDecoder) { - this.handleError("H264 WebSocket requires VideoDecoder support."); - return false; - } - if (this.decoder?.state === "closed") { - this.closeDecoder(); - } - if (this.decoder && frame.config.byteLength === 0) { - return true; - } - const configKey = h264DecoderConfigKey(frame); - if (this.decoder && this.decoderConfigKey === configKey) { - return true; - } - - this.closeDecoder(); - const decoder = new VideoDecoder({ - error: (error) => { - this.stats.decoderDroppedFrames += 1; - this.waitingForKeyFrame = true; - this.stats.waitingForKeyFrame = true; - if (this.decoder === decoder) { - this.closeDecoder(); - } - this.sendControl({ forceKeyframe: true, type: "streamControl" }); - this.onMessage({ type: "stats", stats: { ...this.stats } }); - if (!this.streamingReported) { - this.onMessage({ - type: "status", - status: { detail: "Recovering H264 decoder", state: "connecting" }, - }); - } - }, - output: (videoFrame) => this.queueDecodedFrame(videoFrame), - }); - const config: WebCodecsVideoDecoderConfig = { - codec: h264CodecStringFromAvcC(frame.config) ?? "avc1.42E01F", - codedHeight: frame.height || undefined, - codedWidth: frame.width || undefined, - hardwareAcceleration: "prefer-hardware", - optimizeForLatency: true, - }; - if (frame.config.byteLength > 0) { - config.description = arrayBufferCopy(frame.config); - } - try { - decoder.configure(config); - } catch (error) { - decoder.close(); - this.handleError(error instanceof Error ? error.message : String(error)); - return false; - } - this.decoder = decoder; - this.decoderConfigKey = configKey; - return true; - } - - private nextDecodeTimestampUs(sourceTimestampUs: number): number { - const sourceTimestamp = Number.isFinite(sourceTimestampUs) - ? Math.max(0, Math.floor(sourceTimestampUs)) - : 0; - this.h264DecodeTimestampUs = Math.max( - sourceTimestamp, - this.h264DecodeTimestampUs + 1, - ); - return this.h264DecodeTimestampUs; - } - - private queueDecodedFrame(videoFrame: WebCodecsVideoFrame) { - const sequence = this.takeRenderedFrameSequence(videoFrame); - this.stats.decodedFrames += 1; - if (this.pendingFrame) { - this.stats.droppedFrames += 1; - this.closePendingFrame(); - } - this.pendingFrame = { frame: videoFrame, sequence }; - this.schedulePaint(); - } - - private schedulePaint() { - if (this.animationFrame) { - return; - } - this.animationFrame = window.requestAnimationFrame(this.paintPendingFrame); - } - - private readonly paintPendingFrame = () => { - this.animationFrame = 0; - const pending = this.pendingFrame; - this.pendingFrame = null; - if (!pending) { - return; - } - this.paintDecodedFrame(pending); - }; - - private paintDecodedFrame(pending: PendingVideoFrame) { - const canvas = this.canvas; - if (!canvas) { - pending.frame.close(); - return; - } - const videoFrame = pending.frame; - const width = - videoFrame.displayWidth ?? videoFrame.codedWidth ?? this.stats.width; - const height = - videoFrame.displayHeight ?? videoFrame.codedHeight ?? this.stats.height; - this.syncCanvasSize(width, height); - const startedAt = performance.now(); - try { - this.ensureCanvasContext()?.drawImage( - videoFrame as unknown as CanvasImageSource, - 0, - 0, - canvas.width, - canvas.height, - ); - } finally { - videoFrame.close(); - } - const finishedAt = performance.now(); - const previousFrameAt = this.lastFrameAt; - this.lastFrameAt = finishedAt; - this.stalledFrameWatchdogCount = 0; - this.reportVideoConfig(canvas.width, canvas.height); - this.stats.codec = "h264-ws"; - if (pending.sequence !== null) { - this.stats.frameSequence = pending.sequence; - } - this.stats.renderedFrames += 1; - this.stats.latestRenderMs = finishedAt - startedAt; - this.stats.maxRenderMs = Math.max( - this.stats.maxRenderMs, - this.stats.latestRenderMs, - ); - this.stats.averageRenderMs = - this.stats.averageRenderMs <= 0 - ? this.stats.latestRenderMs - : this.stats.averageRenderMs * 0.9 + this.stats.latestRenderMs * 0.1; - this.stats.latestFrameGapMs = - previousFrameAt > 0 ? finishedAt - previousFrameAt : 0; - this.stats.decodeQueueSize = this.decoder?.decodeQueueSize ?? 0; - this.stats.waitingForKeyFrame = false; - this.onMessage({ type: "stats", stats: { ...this.stats } }); - if (!this.streamingReported) { - this.streamingReported = true; - this.onMessage({ - type: "status", - status: { - detail: "H264 WebSocket stream connected", - state: "streaming", - }, - }); - } - } - - private ensureCanvasContext(): CanvasRenderingContext2D | null { - const canvas = this.canvas; - if (!canvas) { - this.canvasContext = null; - return null; - } - if (this.canvasContext?.canvas === canvas) { - return this.canvasContext; - } - this.canvasContext = canvas.getContext("2d", { - alpha: false, - desynchronized: true, - }); - return this.canvasContext; - } - - private syncCanvasSize(width: number, height: number) { - if (!this.canvas) { - return; - } - const nextWidth = Math.max(1, Math.round(width)); - const nextHeight = Math.max(1, Math.round(height)); - if (this.canvas.width !== nextWidth) { - this.canvas.width = nextWidth; - } - if (this.canvas.height !== nextHeight) { - this.canvas.height = nextHeight; - } - } - - private reportVideoConfig(width: number, height: number) { - if ( - this.reportedVideoWidth === width && - this.reportedVideoHeight === height - ) { - return; - } - this.reportedVideoWidth = width; - this.reportedVideoHeight = height; - this.onMessage({ type: "video-config", size: { height, width } }); - } - - private scheduleFrameWatchdog(generation: number) { - this.clearFrameWatchdog(); - this.frameWatchdogTimeout = window.setTimeout( - () => { - this.frameWatchdogTimeout = 0; - if (generation !== this.connectGeneration || !this.shouldReconnect) { - return; - } - const now = performance.now(); - if (this.lastFrameAt <= 0) { - this.handleError("H264 WebSocket stream did not render a frame."); - return; - } - if (now - this.lastFrameAt > H264_WS_STALLED_FRAME_TIMEOUT_MS) { - this.stalledFrameWatchdogCount += 1; - this.sendControl({ snapshot: true, type: "streamControl" }); - this.sendControl({ forceKeyframe: true, type: "streamControl" }); - if (this.stalledFrameWatchdogCount >= 2 && this.streamTarget) { - const target = this.streamTarget; - this.onMessage({ - type: "status", - status: { - detail: "Reconnecting stalled H264 WebSocket stream", - state: "connecting", - }, - }); - void this.connect({ - ...target, - streamConfig: this.streamConfig, - }); - return; - } - } else { - this.stalledFrameWatchdogCount = 0; - } - this.scheduleFrameWatchdog(generation); - }, - this.lastFrameAt > 0 - ? H264_WS_STALLED_FRAME_TIMEOUT_MS - : H264_WS_FIRST_FRAME_TIMEOUT_MS, - ); - } - - private clearFrameWatchdog() { - if (!this.frameWatchdogTimeout) { - return; - } - window.clearTimeout(this.frameWatchdogTimeout); - this.frameWatchdogTimeout = 0; - } - - private closeDecoder() { - try { - this.decoder?.close(); - } catch { - // Closing a failed decoder is best effort. - } - this.decoder = null; - this.decoderConfigKey = ""; - this.pendingFrameSequences.clear(); - } - - private closePendingFrame() { - if (!this.pendingFrame) { - return; - } - try { - this.pendingFrame.frame.close(); - } catch { - // VideoFrame cleanup is best effort during disconnect/replacement. - } - this.pendingFrame = null; - } - - private takeRenderedFrameSequence( - videoFrame: WebCodecsVideoFrame, - ): number | null { - if (typeof videoFrame.timestamp !== "number") { - return null; - } - const sequence = this.pendingFrameSequences.get(videoFrame.timestamp); - this.pendingFrameSequences.delete(videoFrame.timestamp); - return typeof sequence === "number" ? sequence : null; - } - - private trimPendingFrameSequenceMap() { - while (this.pendingFrameSequences.size > 256) { - const firstKey = this.pendingFrameSequences.keys().next().value; - if (typeof firstKey !== "number") { - break; - } - this.pendingFrameSequences.delete(firstKey); - } - } - - private recordH264SocketMessage(data: unknown) { - this.stats.h264SocketMessages += 1; - const byteLength = - typeof data === "string" - ? data.length - : typeof (data as { byteLength?: unknown })?.byteLength === "number" - ? ((data as { byteLength: number }).byteLength ?? 0) - : typeof (data as { size?: unknown })?.size === "number" - ? ((data as { size: number }).size ?? 0) - : 0; - this.stats.h264SocketBytes += Math.max(0, byteLength); - this.stats.h264SocketMessageType = Object.prototype.toString.call(data); - } - - private startAdaptiveQuality(generation: number) { - this.clearAdaptiveQuality(); - if (this.streamConfig?.quality !== "auto") { - return; - } - this.adaptiveLastAt = performance.now(); - this.adaptiveLastDecodedFrames = this.stats.decodedFrames; - this.adaptiveLastRenderedFrames = this.stats.renderedFrames; - this.adaptiveInterval = window.setInterval(() => { - if (generation !== this.connectGeneration || !this.shouldReconnect) { - this.clearAdaptiveQuality(); - return; - } - void this.sampleAdaptiveQuality(); - }, H264_WS_ADAPTIVE_SAMPLE_MS); - } - - private clearAdaptiveQuality() { - if (!this.adaptiveInterval) { - return; - } - window.clearInterval(this.adaptiveInterval); - this.adaptiveInterval = 0; - } - - private async sampleAdaptiveQuality() { - if (this.streamConfig?.quality !== "auto") { - this.clearAdaptiveQuality(); - return; - } - const now = performance.now(); - const elapsedSeconds = Math.max((now - this.adaptiveLastAt) / 1000, 0.001); - const renderedDelta = - this.stats.renderedFrames - this.adaptiveLastRenderedFrames; - const decodedDelta = - this.stats.decodedFrames - this.adaptiveLastDecodedFrames; - this.adaptiveLastAt = now; - this.adaptiveLastRenderedFrames = this.stats.renderedFrames; - this.adaptiveLastDecodedFrames = this.stats.decodedFrames; - - const renderedFps = renderedDelta / elapsedSeconds; - const decodedFps = decodedDelta / elapsedSeconds; - const underPressure = - this.stats.decodeQueueSize > 1 || - this.stats.latestFrameGapMs > 80 || - this.stats.averageRenderMs > 4; - if (underPressure) { - this.autoProfileStableSamples = 0; - await this.setAutoProfile(this.nextLowerAutoProfile()); - return; - } - - if ( - renderedFps > 0 && - decodedFps > 0 && - this.stats.decodeQueueSize === 0 && - this.stats.latestFrameGapMs < 40 && - this.stats.averageRenderMs < 2 - ) { - this.autoProfileStableSamples += 1; - } else { - this.autoProfileStableSamples = 0; - } - if ( - this.autoProfileStableSamples >= H264_WS_AUTO_STABLE_SAMPLES_TO_UPGRADE - ) { - this.autoProfileStableSamples = 0; - await this.setAutoProfile(this.nextHigherAutoProfile()); - } - } - - private nextLowerAutoProfile(): StreamQualityPreset { - const profiles = h264AutoProfiles(this.streamTarget); - const index = profiles.indexOf(this.autoProfile); - return profiles[Math.max(0, index - 1)] ?? this.autoProfile; - } - - private nextHigherAutoProfile(): StreamQualityPreset { - const profiles = h264AutoProfiles(this.streamTarget); - const index = profiles.indexOf(this.autoProfile); - return ( - profiles[Math.min(profiles.length - 1, Math.max(0, index) + 1)] ?? - this.autoProfile - ); - } - - private async setAutoProfile(profile: StreamQualityPreset) { - if (profile === this.autoProfile || this.streamConfig?.quality !== "auto") { - return; - } - this.autoProfile = profile; - const effectiveConfig = h264WebSocketStreamConfig( - this.streamConfig, - this.autoProfile, - ); - if (!effectiveConfig) { - return; - } - if (!sendStreamQualityConfig(effectiveConfig)) { - await postStreamConfigWithAuthRetry(effectiveConfig, { - remote: this.streamTarget?.remote, - }).catch(() => { - // Stream-quality adaptation is opportunistic; the stream socket handles reachability. - }); - } - this.sendControl({ forceKeyframe: true, type: "streamControl" }); - } - - private handleError(message: string) { - this.onMessage({ - type: "status", - status: { error: message.replace(/\.$/, ""), state: "error" }, - }); - } -} - -function parseH264WebSocketFrame(data: unknown): H264WebSocketFrame | null { - const bytes = bytesFromBinaryMessage(data); - if (!bytes || bytes.byteLength < H264_WS_HEADER_BYTES) { - return null; - } - const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); - if (view.getUint32(0, false) !== H264_WS_MAGIC || view.getUint8(4) !== 1) { - return null; - } - const flags = view.getUint8(5); - const headerBytes = view.getUint16(6, false); - if (headerBytes < H264_WS_HEADER_BYTES || headerBytes > bytes.byteLength) { - return null; - } - const configBytes = view.getUint32(32, false); - const payloadBytes = view.getUint32(36, false); - const payloadOffset = headerBytes + configBytes; - const payloadEnd = payloadOffset + payloadBytes; - if (payloadEnd > bytes.byteLength) { - return null; - } - const config = - flags & H264_WS_FLAG_CONFIG - ? bytes.subarray(headerBytes, payloadOffset) - : new Uint8Array(); - return { - config, - height: view.getUint32(28, false), - isKeyFrame: Boolean(flags & H264_WS_FLAG_KEYFRAME), - payload: bytes.subarray(payloadOffset, payloadEnd), - sequence: Number(view.getBigUint64(8, false)), - timestampUs: Number(view.getBigUint64(16, false)), - width: view.getUint32(24, false), - }; -} - -function bytesFromBinaryMessage(data: unknown): Uint8Array | null { - if (ArrayBuffer.isView(data)) { - return new Uint8Array(data.buffer, data.byteOffset, data.byteLength); - } - if (data instanceof ArrayBuffer) { - return new Uint8Array(data); - } - if ( - Object.prototype.toString.call(data) === "[object ArrayBuffer]" && - typeof (data as { byteLength?: unknown }).byteLength === "number" - ) { - return new Uint8Array(data as ArrayBuffer); - } - if ( - typeof data === "object" && - data !== null && - typeof (data as { byteLength?: unknown }).byteLength === "number" - ) { - try { - return new Uint8Array(data as ArrayBufferLike); - } catch { - return null; - } - } - return null; -} - -function h264CodecStringFromAvcC(config: Uint8Array): string | null { - if (config.byteLength < 4 || config[0] !== 1) { - return null; - } - return `avc1.${hexByte(config[1])}${hexByte(config[2])}${hexByte(config[3])}`; -} - -function h264DecoderConfigKey(frame: H264WebSocketFrame): string { - const codec = h264CodecStringFromAvcC(frame.config) ?? "avc1.42E01F"; - const prefix = frame.config.byteLength - ? Array.from(frame.config.slice(0, 16), hexByte).join("") - : ""; - return `${codec}:${frame.width}x${frame.height}:${frame.config.byteLength}:${prefix}`; -} - -function arrayBufferCopy(bytes: Uint8Array): ArrayBuffer { - const copy = new Uint8Array(bytes.byteLength); - copy.set(bytes); - return copy.buffer; -} - -function hexByte(byte: number): string { - return byte.toString(16).padStart(2, "0").toUpperCase(); -} - class WebRtcStreamClient implements StreamClientBackend { private animationFrame = 0; private canvas: HTMLCanvasElement | null = null; @@ -2322,18 +1369,6 @@ function streamQualityPayload(config: StreamConfig): { }; } -function streamQualityQuery(config: StreamConfig | undefined): string { - if (!config) { - return ""; - } - const params = new URLSearchParams(); - const payload = streamQualityPayload(config); - params.set("fps", String(payload.fps)); - params.set("profile", payload.profile); - params.set("videoCodec", payload.videoCodec); - return `?${params.toString()}`; -} - function postWebRtcOffer( target: StreamConnectTarget, localDescription: RTCSessionDescription, @@ -2355,49 +1390,6 @@ function postWebRtcOffer( ); } -function h264WebSocketStreamConfig( - config: StreamConfig | undefined, - autoProfile: StreamQualityPreset, -): StreamConfig | undefined { - if (!config) { - return config; - } - if (config.quality === "auto") { - return { - ...config, - fps: Math.min(config.fps || 60, 60), - maxEdge: undefined, - quality: autoProfile, - }; - } - if (config.quality !== "quality") { - return config; - } - return config; -} - -function h264AutoProfiles( - target: StreamConnectTarget | null, -): StreamQualityPreset[] { - return shouldUseRemoteH264AutoProfile(target) - ? H264_WS_REMOTE_AUTO_PROFILES - : H264_WS_LOCAL_AUTO_PROFILES; -} - -function initialH264AutoProfile( - target: StreamConnectTarget | null, -): StreamQualityPreset { - return shouldUseRemoteH264AutoProfile(target) - ? H264_WS_REMOTE_INITIAL_AUTO_PROFILE - : H264_WS_LOCAL_INITIAL_AUTO_PROFILE; -} - -function shouldUseRemoteH264AutoProfile( - target: StreamConnectTarget | null, -): boolean { - return Boolean(target?.remote) || !isLoopbackHost(window.location.hostname); -} - function configureLowLatencyReceiver( receiver: RTCRtpReceiver, bufferSeconds: number | null, @@ -2756,11 +1748,7 @@ export class StreamWorkerClient { return; } this.backend?.destroy(); - if (kind === "h264-ws") { - this.backend = new H264WebSocketStreamClient(this.handleBackendMessage); - } else { - this.backend = new WebRtcStreamClient(this.handleBackendMessage); - } + this.backend = new WebRtcStreamClient(this.handleBackendMessage); this.backendKind = kind; if (this.canvasElement) { this.backend.attachCanvas(this.canvasElement); @@ -2768,29 +1756,6 @@ export class StreamWorkerClient { } private readonly handleBackendMessage = (message: WorkerToMainMessage) => { - if ( - message.type === "status" && - message.status.state === "error" && - preferredStreamBackend(this.target) === "auto" && - this.target - ) { - const nextBackend = nextAutoFallbackBackend(this.backendKind); - if (!nextBackend) { - this.onMessage(message); - return; - } - const target = this.target; - this.setBackend(nextBackend); - this.onMessage({ - type: "status", - status: { - detail: "Falling back to H264 WebSocket", - state: "connecting", - }, - }); - this.backend?.connect(target); - return; - } this.onMessage(message); }; } @@ -2801,9 +1766,6 @@ export function preferredStreamBackend( const value = target?.transport ?? new URLSearchParams(window.location.search).get("stream"); - if (value === "h264" || value === "h264-ws") { - return "h264-ws"; - } return value === "webrtc" ? "webrtc" : "auto"; } @@ -2811,20 +1773,8 @@ export function initialStreamBackend( target: StreamConnectTarget, ): StreamBackend { const preferredBackend = preferredStreamBackend(target); - if (preferredBackend === "h264-ws") { - return canUseH264WebSocket() ? "h264-ws" : "webrtc"; - } if (preferredBackend === "webrtc") { - return canUseWebRtc() ? "webrtc" : "h264-ws"; - } - return canUseWebRtc() ? "webrtc" : "h264-ws"; -} - -function nextAutoFallbackBackend( - current: StreamBackend | null, -): StreamBackend | null { - if (current === "webrtc") { - return canUseH264WebSocket() ? "h264-ws" : null; + return "webrtc"; } - return null; + return "webrtc"; } diff --git a/packages/client/src/features/stream/useLiveStream.ts b/packages/client/src/features/stream/useLiveStream.ts index 77990731..95e7321c 100644 --- a/packages/client/src/features/stream/useLiveStream.ts +++ b/packages/client/src/features/stream/useLiveStream.ts @@ -495,7 +495,6 @@ export function useLiveStream({ if ( sendStreamClientStats(payload) || remote || - streamTransport === "h264" || streamTransport === "webrtc" || streamTransport === "auto" ) { @@ -532,7 +531,7 @@ export function useLiveStream({ runtimeInfo, stats, status, - streamBackend: stats.codec === "h264-ws" ? "h264-ws" : "webrtc", + streamBackend: "webrtc", streamCanvasKey: `stream-${streamCanvasRevision}`, }; } diff --git a/packages/server/native/bridge/XCWNativeBridge.m b/packages/server/native/bridge/XCWNativeBridge.m index a64fe336..71d09457 100644 --- a/packages/server/native/bridge/XCWNativeBridge.m +++ b/packages/server/native/bridge/XCWNativeBridge.m @@ -195,7 +195,7 @@ - (instancetype)initWithFrameCallback:(xcw_native_frame_callback)callback char *previousRealtimeStreamCopy = previousRealtimeStream != NULL ? strdup(previousRealtimeStream) : NULL; const char *androidCodec = getenv("SIMDECK_ANDROID_VIDEO_CODEC"); if (androidCodec == NULL || strlen(androidCodec) == 0) { - androidCodec = "software"; + androidCodec = (previousCodec != NULL && strlen(previousCodec) > 0) ? previousCodec : "auto"; } setenv("SIMDECK_VIDEO_CODEC", androidCodec, 1); setenv("SIMDECK_REALTIME_STREAM", "1", 1); diff --git a/packages/server/native_stubs.c b/packages/server/native_stubs.c index 7b3ae92a..698c0272 100644 --- a/packages/server/native_stubs.c +++ b/packages/server/native_stubs.c @@ -583,6 +583,22 @@ bool xcw_native_h264_encoder_encode_rgba(void *handle, const uint8_t *rgba, return false; } +bool xcw_native_h264_encoder_encode_bgra(void *handle, const uint8_t *bgra, + uintptr_t length, uint32_t width, + uint32_t height, + uint64_t timestamp_us, + char **error_message) { + (void)handle; + (void)bgra; + (void)length; + (void)width; + (void)height; + (void)timestamp_us; + xcw_set_error(error_message, + "H.264 encoding is only available in the macOS native bridge."); + return false; +} + void xcw_native_h264_encoder_request_keyframe(void *handle) { (void)handle; } void xcw_native_free_string(char *value) { free(value); } diff --git a/packages/server/src/android.rs b/packages/server/src/android.rs index 9a0768c8..096ea91a 100644 --- a/packages/server/src/android.rs +++ b/packages/server/src/android.rs @@ -6,6 +6,7 @@ use std::ffi::CString; use std::ffi::OsString; use std::io::{Read, Write}; use std::mem::MaybeUninit; +use std::net::{SocketAddr, TcpStream}; use std::os::fd::RawFd; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; @@ -16,15 +17,14 @@ use std::time::{Duration, Instant}; const ANDROID_ID_PREFIX: &str = "android:"; const DEFAULT_EMULATOR_CONSOLE_PORT_BASE: u16 = 5554; -const ANDROID_H264_DEFAULT_MAX_EDGE: u32 = 1440; -const ANDROID_SHARED_VIDEO_HEADER_BYTES: usize = 3072; +const ANDROID_EMULATOR_DEFAULT_GPU_MODE: &str = "host"; +const ANDROID_EMULATOR_GPU_ENV: &str = "SIMDECK_ANDROID_GPU"; +const ANDROID_SHARED_VIDEO_HEADER_BYTES: usize = std::mem::size_of::(); const ANDROID_SHARED_VIDEO_PIXEL_BYTES: usize = 4; const ANDROID_SHARED_VIDEO_MAX_DIMENSION: u32 = 8192; const ANDROID_SHARED_VIDEO_MAX_FRAME_BYTES: usize = 256 * 1024 * 1024; -const ANDROID_TOUCH_SWIPE_THRESHOLD: f64 = 0.025; -const ANDROID_TOUCH_MIN_DURATION_MS: u128 = 80; -const ANDROID_TOUCH_MAX_DURATION_MS: u128 = 1500; const ANDROID_COMMAND_TIMEOUT: Duration = Duration::from_secs(30); +const ANDROID_CONSOLE_TIMEOUT: Duration = Duration::from_secs(2); const ANDROID_UIAUTOMATOR_DUMP_ATTEMPTS: usize = 10; const ANDROID_UIAUTOMATOR_DUMP_RETRY_DELAY: Duration = Duration::from_millis(250); const RUNNING_EMULATOR_CACHE_TTL: Duration = Duration::from_secs(2); @@ -38,6 +38,7 @@ const MODIFIER_CAPS_LOCK: u32 = 1 << 4; type TimedMap = Option<(Instant, HashMap)>; type DisplayMetricsCache = HashMap; +type ConsoleConnectionCache = HashMap; #[derive(Clone, Copy, Debug, PartialEq)] struct AndroidDisplayMetrics { @@ -114,6 +115,7 @@ pub struct AndroidSharedVideoFrameStream { unsafe impl Send for AndroidSharedVideoFrameStream {} +#[repr(C)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] struct AndroidSharedVideoHeader { width: u32, @@ -123,45 +125,6 @@ struct AndroidSharedVideoHeader { timestamp_us: u64, } -#[derive(Debug)] -pub struct AndroidTouchGesture { - started_at: Instant, - start_x: f64, - start_y: f64, -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub enum AndroidTouchAction { - None, - Tap { - x: f64, - y: f64, - }, - Swipe { - start_x: f64, - start_y: f64, - end_x: f64, - end_y: f64, - duration_ms: u64, - }, -} - -impl AndroidTouchAction { - pub fn perform(self, bridge: &AndroidBridge, id: &str) -> Result<(), AppError> { - match self { - AndroidTouchAction::None => Ok(()), - AndroidTouchAction::Tap { x, y } => bridge.send_tap_adb(id, x, y), - AndroidTouchAction::Swipe { - start_x, - start_y, - end_x, - end_y, - duration_ms, - } => bridge.send_swipe_adb(id, start_x, start_y, end_x, end_y, duration_ms), - } - } -} - pub fn is_android_id(id: &str) -> bool { id.starts_with(ANDROID_ID_PREFIX) } @@ -177,61 +140,6 @@ pub fn id_for_avd(avd_name: &str) -> String { format!("{ANDROID_ID_PREFIX}{avd_name}") } -pub fn update_touch_gesture( - active_touch: &mut Option, - x: f64, - y: f64, - phase: &str, -) -> Result { - if !x.is_finite() || !y.is_finite() { - return Err(AppError::bad_request( - "`x` and `y` must be finite normalized numbers.", - )); - } - let x = x.clamp(0.0, 1.0); - let y = y.clamp(0.0, 1.0); - - match phase { - "began" => { - *active_touch = Some(AndroidTouchGesture { - started_at: Instant::now(), - start_x: x, - start_y: y, - }); - Ok(AndroidTouchAction::None) - } - "moved" => Ok(AndroidTouchAction::None), - "ended" => { - let touch = active_touch.take().unwrap_or(AndroidTouchGesture { - started_at: Instant::now(), - start_x: x, - start_y: y, - }); - let distance = ((x - touch.start_x).powi(2) + (y - touch.start_y).powi(2)).sqrt(); - if distance < ANDROID_TOUCH_SWIPE_THRESHOLD { - return Ok(AndroidTouchAction::Tap { x, y }); - } - Ok(AndroidTouchAction::Swipe { - start_x: touch.start_x, - start_y: touch.start_y, - end_x: x, - end_y: y, - duration_ms: touch - .started_at - .elapsed() - .as_millis() - .clamp(ANDROID_TOUCH_MIN_DURATION_MS, ANDROID_TOUCH_MAX_DURATION_MS) - as u64, - }) - } - "cancelled" => { - *active_touch = None; - Ok(AndroidTouchAction::None) - } - _ => Ok(AndroidTouchAction::None), - } -} - impl AndroidBridge { pub fn list_devices(&self) -> Result, AppError> { if !self.emulator_path().exists() { @@ -343,6 +251,7 @@ impl AndroidBridge { AppError::native("Android emulator console port overflowed while booting.") })?; let emulator_ports = format!("{console_port},{adb_port}"); + let gpu_mode = android_emulator_gpu_mode()?; let is_windows = cfg!(target_os = "windows"); let window_mode = if is_windows { "-qt-hide-window" @@ -355,7 +264,7 @@ impl AndroidBridge { window_mode, "-no-audio", "-gpu", - "swiftshader_indirect", + gpu_mode.as_str(), ]; if is_windows { args.extend(["-feature", "-Vulkan"]); @@ -506,27 +415,15 @@ impl AndroidBridge { } pub fn send_touch(&self, id: &str, x: f64, y: f64, phase: &str) -> Result<(), AppError> { - match phase { - "ended" => self.send_tap_adb(id, x, y), - _ => Ok(()), - } - } - - fn send_tap_adb(&self, id: &str, x: f64, y: f64) -> Result<(), AppError> { let serial = self.serial_for_id(id)?; let (width, height) = self.screen_size_for_serial(&serial)?; - let px = (x.clamp(0.0, 1.0) * (width - 1.0)).round().max(0.0); - let py = (y.clamp(0.0, 1.0) * (height - 1.0)).round().max(0.0); - self.run_adb([ - "-s", + let px = (x.clamp(0.0, 1.0) * (width - 1.0)).round().max(0.0) as u32; + let py = (y.clamp(0.0, 1.0) * (height - 1.0)).round().max(0.0) as u32; + let button_state = android_mouse_button_state_for_touch_phase(phase)?; + self.run_console_command_for_serial( &serial, - "shell", - "input", - "tap", - &px.to_string(), - &py.to_string(), - ])?; - Ok(()) + &format!("event mouse {px} {py} 0 {button_state}"), + ) } pub fn send_swipe( @@ -1088,6 +985,15 @@ impl AndroidBridge { .remove(serial); } + fn run_console_command_for_serial(&self, serial: &str, command: &str) -> Result<(), AppError> { + let port = console_port_from_serial(serial).ok_or_else(|| { + AppError::native(format!( + "Android emulator serial `{serial}` does not expose a console port." + )) + })?; + run_android_console_command(port, command) + } + fn run_adb_shell(&self, serial: &str, script: &str) -> Result { self.run_adb(["-s", serial, "shell", script]) } @@ -1173,14 +1079,9 @@ impl AndroidSharedVideoFrameStream { ))); } - let mut bgra = vec![0u8; pixel_bytes]; - unsafe { - ptr::copy_nonoverlapping( - self.ptr.add(ANDROID_SHARED_VIDEO_HEADER_BYTES), - bgra.as_mut_ptr(), - pixel_bytes, - ); - } + let pixel_ptr = unsafe { self.ptr.add(ANDROID_SHARED_VIDEO_HEADER_BYTES) }; + let frame = + unsafe { android_shared_video_frame_from_memory(header, pixel_ptr, quality)? }; let confirm = self.read_header()?; if header.width != confirm.width || header.height != confirm.height @@ -1189,9 +1090,7 @@ impl AndroidSharedVideoFrameStream { continue; } self.last_sequence = Some(header.sequence); - return Ok(Some(android_shared_video_frame_from_pixels( - header, bgra, quality, - )?)); + return Ok(Some(frame)); } Ok(None) } @@ -1221,9 +1120,12 @@ fn android_h264_stream_dimensions( ) -> (u32, u32) { let source_width = source_width.max(2); let source_height = source_height.max(2); - let max_edge = max_edge - .unwrap_or(ANDROID_H264_DEFAULT_MAX_EDGE) - .clamp(240, 4096); + let Some(max_edge) = max_edge.map(|value| value.clamp(240, 4096)) else { + return ( + round_android_h264_dimension(source_width), + round_android_h264_dimension(source_height), + ); + }; let largest = source_width.max(source_height); if largest <= max_edge { return ( @@ -1240,7 +1142,7 @@ fn android_h264_stream_dimensions( fn round_android_h264_dimension(value: u32) -> u32 { let rounded = value.max(2); - if rounded % 2 == 0 { + if rounded.is_multiple_of(2) { rounded } else { rounded.saturating_sub(1).max(2) @@ -1367,16 +1269,15 @@ fn android_shared_video_frame_from_slice( .ok_or_else(|| AppError::native("Android shared video frame size overflowed."))?; let pixels = bytes .get(ANDROID_SHARED_VIDEO_HEADER_BYTES..end) - .ok_or_else(|| AppError::native("Android shared video frame was truncated."))? - .to_vec(); - Ok(Some(android_shared_video_frame_from_pixels( - header, pixels, quality, - )?)) + .ok_or_else(|| AppError::native("Android shared video frame was truncated."))?; + Ok(Some(unsafe { + android_shared_video_frame_from_memory(header, pixels.as_ptr(), quality)? + })) } #[cfg(test)] fn read_android_shared_video_header(bytes: &[u8]) -> Result { - if bytes.len() < 24 { + if bytes.len() < ANDROID_SHARED_VIDEO_HEADER_BYTES { return Err(AppError::native( "Android emulator shared video header was truncated.", )); @@ -1413,6 +1314,37 @@ fn validate_android_shared_video_header( Ok(header) } +fn android_emulator_gpu_mode() -> Result { + android_emulator_gpu_mode_from_value(env::var(ANDROID_EMULATOR_GPU_ENV).ok().as_deref()) +} + +fn android_emulator_gpu_mode_from_value(value: Option<&str>) -> Result { + let raw = value + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or(ANDROID_EMULATOR_DEFAULT_GPU_MODE); + let mode = raw.to_ascii_lowercase().replace('-', "_"); + if !android_emulator_gpu_mode_supported(&mode) { + return Err(AppError::native(format!( + "Unsupported Android emulator GPU mode `{raw}`. Use auto, host, software, lavapipe, swiftshader, swangle, or swiftshader_indirect." + ))); + } + Ok(mode) +} + +fn android_emulator_gpu_mode_supported(value: &str) -> bool { + matches!( + value, + "auto" + | "host" + | "software" + | "lavapipe" + | "swiftshader" + | "swangle" + | "swiftshader_indirect" + ) +} + fn android_shared_video_pixel_bytes(width: u32, height: u32) -> Result { let bytes = usize::try_from(width) .ok() @@ -1431,13 +1363,17 @@ fn android_shared_video_pixel_bytes(width: u32, height: u32) -> Result, + source_bgra: *const u8, quality: AndroidH264StreamQuality, ) -> Result { - let (width, height, bgra) = - scale_android_bgra_frame(header.width, header.height, bgra, quality.max_edge)?; + let (width, height, bgra) = copy_scaled_android_bgra_from_memory( + header.width, + header.height, + source_bgra, + quality.max_edge, + )?; Ok(AndroidSharedVideoFrame { timestamp_us: header.timestamp_us, width, @@ -1446,25 +1382,26 @@ fn android_shared_video_frame_from_pixels( }) } -fn scale_android_bgra_frame( +unsafe fn copy_scaled_android_bgra_from_memory( width: u32, height: u32, - bgra: Vec, + source_bgra: *const u8, max_edge: Option, ) -> Result<(u32, u32, Vec), AppError> { - let expected = android_shared_video_pixel_bytes(width, height)?; - if bgra.len() < expected { + if source_bgra.is_null() { return Err(AppError::native( - "Android shared video frame contained truncated BGRA pixels.", + "Android shared video frame pixels are unavailable.", )); } + let source_len = android_shared_video_pixel_bytes(width, height)?; let (target_width, target_height) = android_h264_stream_dimensions(width, height, max_edge); + let target_len = android_shared_video_pixel_bytes(target_width, target_height)?; + let mut output = vec![0u8; target_len]; if target_width == width && target_height == height { - return Ok((width, height, bgra)); + ptr::copy_nonoverlapping(source_bgra, output.as_mut_ptr(), source_len); + return Ok((width, height, output)); } - let output_len = android_shared_video_pixel_bytes(target_width, target_height)?; - let mut output = vec![0u8; output_len]; let source_width = width as usize; let source_height = height as usize; let target_width_usize = target_width as usize; @@ -1475,8 +1412,11 @@ fn scale_android_bgra_frame( let source_x = x * source_width / target_width_usize; let src = ((source_y * source_width) + source_x) * ANDROID_SHARED_VIDEO_PIXEL_BYTES; let dst = ((y * target_width_usize) + x) * ANDROID_SHARED_VIDEO_PIXEL_BYTES; - output[dst..dst + ANDROID_SHARED_VIDEO_PIXEL_BYTES] - .copy_from_slice(&bgra[src..src + ANDROID_SHARED_VIDEO_PIXEL_BYTES]); + ptr::copy_nonoverlapping( + source_bgra.add(src), + output.as_mut_ptr().add(dst), + ANDROID_SHARED_VIDEO_PIXEL_BYTES, + ); } } Ok((target_width, target_height, output)) @@ -1846,6 +1786,16 @@ fn console_port_from_serial(serial: &str) -> Option { serial.strip_prefix("emulator-")?.parse::().ok() } +fn android_mouse_button_state_for_touch_phase(phase: &str) -> Result { + match phase.trim().to_ascii_lowercase().as_str() { + "began" | "down" | "moved" => Ok(1), + "ended" | "up" | "cancelled" => Ok(0), + _ => Err(AppError::bad_request( + "`phase` must be `began`, `moved`, `ended`, `cancelled`, `down`, or `up`.", + )), + } +} + fn is_android_component_name(value: &str) -> bool { value .split_once('/') @@ -2008,6 +1958,119 @@ fn android_display_metrics_cache() -> &'static Mutex { CACHE.get_or_init(|| Mutex::new(HashMap::new())) } +fn android_console_connection_cache() -> &'static Mutex { + static CACHE: OnceLock> = OnceLock::new(); + CACHE.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn run_android_console_command(port: u16, command: &str) -> Result<(), AppError> { + let mut cache = android_console_connection_cache().lock().unwrap(); + for attempt in 0..2 { + if let std::collections::hash_map::Entry::Vacant(entry) = cache.entry(port) { + entry.insert(connect_android_console(port)?); + } + let stream = cache + .get_mut(&port) + .ok_or_else(|| AppError::native("Android emulator console connection was missing."))?; + match write_android_console_command(stream, command) { + Ok(()) => return Ok(()), + Err(error) if attempt == 0 => { + cache.remove(&port); + tracing::debug!("Android emulator console command retry on {port}: {error}"); + } + Err(error) => { + cache.remove(&port); + return Err(error); + } + } + } + Err(AppError::native( + "Android emulator console command failed unexpectedly.", + )) +} + +fn connect_android_console(port: u16) -> Result { + let address = SocketAddr::from(([127, 0, 0, 1], port)); + let mut stream = + TcpStream::connect_timeout(&address, ANDROID_CONSOLE_TIMEOUT).map_err(|error| { + AppError::native(format!( + "Unable to connect to Android emulator console on port {port}: {error}" + )) + })?; + stream + .set_read_timeout(Some(ANDROID_CONSOLE_TIMEOUT)) + .map_err(|error| { + AppError::native(format!("Unable to configure console timeout: {error}")) + })?; + stream + .set_write_timeout(Some(ANDROID_CONSOLE_TIMEOUT)) + .map_err(|error| { + AppError::native(format!("Unable to configure console timeout: {error}")) + })?; + let greeting = read_android_console_response(&mut stream)?; + if greeting.contains("Authentication required") { + let token_path = home_dir().join(".emulator_console_auth_token"); + let token = std::fs::read_to_string(&token_path).map_err(|error| { + AppError::native(format!( + "Unable to read Android emulator console auth token at {}: {error}", + token_path.display() + )) + })?; + write_android_console_command(&mut stream, &format!("auth {}", token.trim()))?; + } + Ok(stream) +} + +fn write_android_console_command(stream: &mut TcpStream, command: &str) -> Result<(), AppError> { + stream + .write_all(command.as_bytes()) + .and_then(|_| stream.write_all(b"\n")) + .map_err(|error| { + AppError::native(format!("Android emulator console write failed: {error}")) + })?; + let response = read_android_console_response(stream)?; + if android_console_response_ok(&response) { + Ok(()) + } else { + Err(AppError::native(format!( + "Android emulator console rejected `{command}`: {}", + response.trim() + ))) + } +} + +fn read_android_console_response(stream: &mut TcpStream) -> Result { + let mut response = String::new(); + let mut buffer = [0u8; 1024]; + loop { + let bytes_read = stream.read(&mut buffer).map_err(|error| { + AppError::native(format!("Android emulator console read failed: {error}")) + })?; + if bytes_read == 0 { + return Err(AppError::native( + "Android emulator console closed the connection.", + )); + } + response.push_str(&String::from_utf8_lossy(&buffer[..bytes_read])); + if android_console_response_complete(&response) { + return Ok(response); + } + } +} + +fn android_console_response_complete(response: &str) -> bool { + response + .lines() + .any(|line| line.trim() == "OK" || line.trim_start().starts_with("KO")) +} + +fn android_console_response_ok(response: &str) -> bool { + response.lines().any(|line| line.trim() == "OK") + && !response + .lines() + .any(|line| line.trim_start().starts_with("KO")) +} + fn parse_android_display_metrics(output: &str) -> Option { let rotation = parse_android_display_rotation(output).unwrap_or(0); let corner_radii = parse_android_rounded_corners(output).unwrap_or(AndroidCornerRadii::ZERO); @@ -2353,39 +2416,62 @@ mod tests { } #[test] - fn android_touch_gesture_resolves_tap_on_end() { - let mut active = None; - - assert_eq!( - update_touch_gesture(&mut active, 0.4, 0.6, "began").unwrap(), - AndroidTouchAction::None - ); - assert_eq!( - update_touch_gesture(&mut active, 0.41, 0.6, "ended").unwrap(), - AndroidTouchAction::Tap { x: 0.41, y: 0.6 } - ); + fn android_touch_phases_map_to_emulator_mouse_button_state() { + for phase in ["began", "down", "moved"] { + assert_eq!( + android_mouse_button_state_for_touch_phase(phase).unwrap(), + 1 + ); + } + for phase in ["ended", "up", "cancelled"] { + assert_eq!( + android_mouse_button_state_for_touch_phase(phase).unwrap(), + 0 + ); + } + assert!(android_mouse_button_state_for_touch_phase("hover").is_err()); } #[test] - fn android_touch_gesture_resolves_swipe_on_end() { - let mut active = None; + fn android_console_response_status_detection() { + assert!(android_console_response_complete( + "Android Console\r\nOK\r\n" + )); + assert!(android_console_response_ok("Android Console\r\nOK\r\n")); + assert!(android_console_response_complete("KO: bad command\r\n")); + assert!(!android_console_response_ok("KO: bad command\r\n")); + assert!(!android_console_response_complete("Android Console\r\n")); + } + #[test] + fn android_emulator_gpu_mode_defaults_to_host() { + assert_eq!(android_emulator_gpu_mode_from_value(None).unwrap(), "host"); assert_eq!( - update_touch_gesture(&mut active, 0.1, 0.2, "began").unwrap(), - AndroidTouchAction::None - ); - assert_eq!( - update_touch_gesture(&mut active, 0.8, 0.2, "ended").unwrap(), - AndroidTouchAction::Swipe { - start_x: 0.1, - start_y: 0.2, - end_x: 0.8, - end_y: 0.2, - duration_ms: 80, - } + android_emulator_gpu_mode_from_value(Some(" ")).unwrap(), + "host" ); } + #[test] + fn android_emulator_gpu_mode_accepts_supported_modes() { + for (input, expected) in [ + ("auto", "auto"), + ("host", "host"), + ("software", "software"), + ("lavapipe", "lavapipe"), + ("swiftshader", "swiftshader"), + ("swangle", "swangle"), + ("swiftshader_indirect", "swiftshader_indirect"), + ("swiftshader-indirect", "swiftshader_indirect"), + ] { + assert_eq!( + android_emulator_gpu_mode_from_value(Some(input)).unwrap(), + expected + ); + } + assert!(android_emulator_gpu_mode_from_value(Some("bad-gpu")).is_err()); + } + #[test] fn android_key_code_maps_usb_hid_keyboard_usages() { assert_eq!(android_key_code(4), 29); @@ -2544,6 +2630,10 @@ abcd1234\tdevice #[test] fn android_h264_stream_dimensions_scale_evenly() { + assert_eq!( + android_h264_stream_dimensions(1440, 3120, None), + (1440, 3120) + ); assert_eq!( android_h264_stream_dimensions(1080, 2400, Some(720)), (324, 720) @@ -2554,6 +2644,11 @@ abcd1234\tdevice ); } + #[test] + fn android_shared_video_header_matches_emulator_video_info_layout() { + assert_eq!(ANDROID_SHARED_VIDEO_HEADER_BYTES, 24); + } + #[test] fn android_shared_video_frame_reads_header_and_pixels() { let mut bytes = vec![0u8; ANDROID_SHARED_VIDEO_HEADER_BYTES + 4 * 4 * 4]; @@ -2642,4 +2737,42 @@ abcd1234\tdevice &[0, 2, 0xaa, 0xff, 2, 2, 0xaa, 0xff] ); } + + #[test] + fn android_shared_video_frame_scales_directly_from_memory() { + let mut bytes = vec![0u8; ANDROID_SHARED_VIDEO_HEADER_BYTES + 480 * 480 * 4]; + bytes[0..4].copy_from_slice(&480u32.to_le_bytes()); + bytes[4..8].copy_from_slice(&480u32.to_le_bytes()); + bytes[8..12].copy_from_slice(&60u32.to_le_bytes()); + bytes[12..16].copy_from_slice(&1u32.to_le_bytes()); + for y in 0..480usize { + for x in 0..480usize { + let offset = ANDROID_SHARED_VIDEO_HEADER_BYTES + ((y * 480 + x) * 4); + bytes[offset..offset + 4].copy_from_slice(&[x as u8, y as u8, 0xaa, 0xff]); + } + } + let header = read_android_shared_video_header(&bytes).unwrap(); + let source = unsafe { bytes.as_ptr().add(ANDROID_SHARED_VIDEO_HEADER_BYTES) }; + let frame = unsafe { + android_shared_video_frame_from_memory( + header, + source, + AndroidH264StreamQuality { + max_edge: Some(240), + min_bitrate: None, + bits_per_pixel: None, + }, + ) + .unwrap() + }; + + assert_eq!((frame.width, frame.height), (240, 240)); + assert_eq!(frame.bgra.len(), 240 * 240 * 4); + assert_eq!(&frame.bgra[..8], &[0, 0, 0xaa, 0xff, 2, 0, 0xaa, 0xff]); + let second_row = 240 * 4; + assert_eq!( + &frame.bgra[second_row..second_row + 8], + &[0, 2, 0xaa, 0xff, 2, 2, 0xaa, 0xff] + ); + } } diff --git a/packages/server/src/api/routes.rs b/packages/server/src/api/routes.rs index 5465fb48..dd811743 100644 --- a/packages/server/src/api/routes.rs +++ b/packages/server/src/api/routes.rs @@ -19,8 +19,6 @@ use crate::performance::{ use crate::simulators::registry::SessionRegistry; use crate::simulators::session::SimulatorSession; use crate::static_files; -use crate::transport::packet::FramePacket; -use crate::transport::webrtc::AndroidWebRtcSource; use crate::webkit; use axum::body::Body; use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade}; @@ -30,7 +28,7 @@ use axum::middleware::{from_fn_with_state, Next}; use axum::response::{IntoResponse, Redirect, Response}; use axum::routing::{any, get, post}; use axum::{Json, Router}; -use bytes::{Bytes, BytesMut}; +use bytes::Bytes; use futures::{SinkExt, StreamExt}; use regex::Regex; use serde::Deserialize; @@ -44,7 +42,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; use tokio::net::TcpStream; use tokio::process::Command; -use tokio::sync::{broadcast, mpsc, Mutex}; +use tokio::sync::{mpsc, Mutex}; use tokio::task; use tokio::time::timeout; use tower_http::trace::{DefaultMakeSpan, DefaultOnFailure, TraceLayer}; @@ -53,12 +51,6 @@ use tracing::Level; const SIMULATOR_INVENTORY_CACHE_TTL: Duration = Duration::from_secs(5); const SIMULATOR_INVENTORY_TIMEOUT: Duration = Duration::from_secs(8); const SIMULATOR_INVENTORY_FORCE_REFRESH_TIMEOUT: Duration = Duration::from_secs(90); -const H264_WS_MAGIC: &[u8; 4] = b"SDH1"; -const H264_WS_HEADER_LEN: usize = 40; -const H264_WS_FLAG_KEYFRAME: u8 = 1 << 0; -const H264_WS_FLAG_CONFIG: u8 = 1 << 1; -const H264_WS_SEND_TIMEOUT: Duration = Duration::from_secs(2); -const H264_WS_KEYFRAME_WAIT_TIMEOUT: Duration = Duration::from_secs(3); const STREAM_CLIENT_FOREGROUND_TTL: Duration = Duration::from_secs(30); const CHROME_DEVTOOLS_DISCOVERY_TIMEOUT: Duration = Duration::from_millis(900); const MULTITOUCH_INPUT_IDLE_TIMEOUT: Duration = Duration::from_secs(5); @@ -438,7 +430,7 @@ const STREAM_QUALITY_PROFILES: &[StreamQualityProfile] = &[ StreamQualityProfile { id: "smooth", label: "Smooth", - max_edge: 1170, + max_edge: 4096, fps: 60, min_bitrate: 4_000_000, bits_per_pixel: 5, @@ -469,7 +461,8 @@ const STREAM_QUALITY_PROFILES: &[StreamQualityProfile] = &[ }, ]; -const VISIBLE_STREAM_QUALITY_PROFILE_IDS: &[&str] = &["full", "balanced", "economy", "low", "tiny"]; +const VISIBLE_STREAM_QUALITY_PROFILE_IDS: &[&str] = + &["full", "balanced", "smooth", "economy", "low", "tiny"]; static STREAM_CONFIG_LOCK: OnceLock> = OnceLock::new(); @@ -847,7 +840,7 @@ pub fn router(state: AppState) -> Router { .route("/api/simulators/{udid}/action", post(simulator_action)) .route("/api/simulators/{udid}/control", get(control_socket)) .route("/api/simulators/{udid}/input", get(control_socket)) - .route("/api/simulators/{udid}/h264", get(h264_socket)) + .route("/api/simulators/{udid}/h264", get(removed_h264_stream)) .route("/api/simulators/{udid}/webrtc/offer", post(webrtc_offer)) .route("/api/simulators/{udid}/chrome-profile", get(chrome_profile)) .route("/api/simulators/{udid}/chrome.png", get(chrome_png)) @@ -979,6 +972,7 @@ async fn health(State(state): State) -> Json { "serverKind": state.config.server_kind.as_str(), "timestamp": SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or(Duration::ZERO).as_secs_f64(), "videoCodec": video_codec, + "androidGpu": active_android_gpu(), "lowLatency": state.config.low_latency, "realtimeStream": crate::transport::webrtc::realtime_stream_enabled(), "localStreamFps": env_u32("SIMDECK_LOCAL_STREAM_FPS", 60, 15, 240), @@ -997,6 +991,13 @@ fn active_video_codec(config: &Config) -> String { .unwrap_or_else(|| config.video_codec.clone()) } +fn active_android_gpu() -> String { + std::env::var("SIMDECK_ANDROID_GPU") + .ok() + .and_then(|value| normalize_android_gpu(&value).map(ToOwned::to_owned)) + .unwrap_or_else(|| "host".to_owned()) +} + fn now_ms() -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) @@ -1013,6 +1014,19 @@ fn normalize_video_codec(codec: &str) -> Option<&'static str> { } } +fn normalize_android_gpu(mode: &str) -> Option<&'static str> { + match mode.trim().to_ascii_lowercase().replace('-', "_").as_str() { + "auto" => Some("auto"), + "host" => Some("host"), + "software" => Some("software"), + "lavapipe" => Some("lavapipe"), + "swiftshader" => Some("swiftshader"), + "swangle" => Some("swangle"), + "swiftshader_indirect" => Some("swiftshader_indirect"), + _ => None, + } +} + async fn metrics(State(state): State) -> Json { let mut snapshot = json_value!(state.metrics.snapshot()); if let Some(object) = snapshot.as_object_mut() { @@ -2699,18 +2713,18 @@ async fn control_socket( websocket.on_upgrade(move |socket| handle_control_socket(state, udid, socket)) } -async fn h264_socket( - State(state): State, - Path(udid): Path, - Query(query): Query, - websocket: WebSocketUpgrade, -) -> impl IntoResponse { - websocket.on_upgrade(move |socket| handle_h264_socket(state, udid, query, socket)) +async fn removed_h264_stream(Path(_udid): Path) -> impl IntoResponse { + ( + StatusCode::GONE, + Json(json_value!({ + "ok": false, + "error": "H.264 WebSocket video streaming has been removed. Use /api/simulators/{udid}/webrtc/offer.", + })), + ) } async fn handle_android_control_socket(state: AppState, udid: String, socket: WebSocket) { let (mut sender, mut receiver) = socket.split(); - let mut active_touch: Option = None; let _ = sender .send(Message::Text( json_value!({ "type": "ready", "udid": udid, "platform": "android-emulator" }) @@ -2734,7 +2748,7 @@ async fn handle_android_control_socket(state: AppState, udid: String, socket: We }; let state = state.clone(); let udid = udid.clone(); - let _ = run_android_control_message(state, udid, control_message, &mut active_touch).await; + let _ = run_android_control_message(state, udid, control_message).await; } } @@ -2742,17 +2756,16 @@ async fn run_android_control_message( state: AppState, udid: String, message: ControlMessage, - active_touch: &mut Option, ) -> Result<(), AppError> { match message { ControlMessage::Touch { x, y, phase } => { - handle_android_control_touch(state, udid, x, y, phase, active_touch).await + handle_android_control_touch(state, udid, x, y, phase).await } ControlMessage::EdgeTouch { x, y, phase, .. } => { - handle_android_control_touch(state, udid, x, y, phase, active_touch).await + handle_android_control_touch(state, udid, x, y, phase).await } ControlMessage::MultiTouch { x1, y1, phase, .. } => { - handle_android_control_touch(state, udid, x1, y1, phase, active_touch).await + handle_android_control_touch(state, udid, x1, y1, phase).await } other => { run_android_action(state, move |android| match other { @@ -2801,13 +2814,11 @@ async fn handle_android_control_touch( x: f64, y: f64, phase: String, - active_touch: &mut Option, ) -> Result<(), AppError> { - let action = android::update_touch_gesture(active_touch, x, y, &phase)?; - if matches!(action, android::AndroidTouchAction::None) { - return Ok(()); - } - run_android_action(state, move |android| action.perform(&android, &udid)).await + run_android_action(state, move |android| { + android.send_touch(&udid, x, y, &phase) + }) + .await } async fn webrtc_offer( @@ -2821,442 +2832,6 @@ async fn webrtc_offer( .map(Json) } -async fn handle_h264_socket( - state: AppState, - udid: String, - initial_quality: StreamQualityPayload, - socket: WebSocket, -) { - if android::is_android_id(&udid) { - handle_android_h264_socket(state, udid, initial_quality, socket).await; - return; - } - - if initial_quality.has_any_value() { - if let Err(error) = apply_stream_quality_payload(&state, &initial_quality) { - tracing::debug!("Failed to apply H264 WebSocket stream quality for {udid}: {error}"); - } - } - let session = match state.registry.get_or_create_async(&udid).await { - Ok(session) => session, - Err(error) => { - tracing::debug!("Failed to create H264 WebSocket session for {udid}: {error}"); - return; - } - }; - if let Err(error) = session.ensure_started_async().await { - tracing::debug!("Failed to start H264 WebSocket session for {udid}: {error}"); - return; - } - - let mut subscription = session.subscribe(); - let (mut sender, mut receiver) = socket.split(); - let mut decoder_synced = false; - let mut last_sent_sequence: Option = None; - - let initial_keyframe = if let Some(keyframe) = session - .latest_keyframe() - .filter(|frame| h264_ws_frame_is_decoder_sync(frame)) - { - Some(keyframe) - } else { - session - .wait_for_keyframe(H264_WS_KEYFRAME_WAIT_TIMEOUT) - .await - .filter(|frame| h264_ws_frame_is_decoder_sync(frame)) - }; - - if let Some(keyframe) = initial_keyframe { - if h264_ws_frame_is_supported(&keyframe) { - let message_bytes = h264_ws_frame_message(&keyframe); - let message = Message::Binary(message_bytes); - if timeout(H264_WS_SEND_TIMEOUT, sender.send(message)) - .await - .ok() - .and_then(Result::ok) - .is_none() - { - return; - } - last_sent_sequence = Some(keyframe.frame_sequence); - decoder_synced = true; - } - } else { - session.request_keyframe(); - } - - loop { - tokio::select! { - received = receiver.next() => { - let Some(received) = received else { break }; - let message = match received { - Ok(message) => message, - Err(error) => { - tracing::debug!("H264 WebSocket closed for {udid}: {error}"); - break; - } - }; - if !handle_h264_socket_message(&state, &session, &message) { - break; - } - } - frame = subscription.recv() => { - let frame = match frame { - Ok(frame) => frame, - Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => { - decoder_synced = false; - session.request_keyframe(); - continue; - } - Err(tokio::sync::broadcast::error::RecvError::Closed) => break, - }; - if !h264_ws_frame_is_supported(&frame) { - continue; - } - if last_sent_sequence - .map(|sequence| frame.frame_sequence <= sequence) - .unwrap_or(false) - { - continue; - } - if !decoder_synced && !frame.is_keyframe { - session.request_keyframe(); - continue; - } - let is_keyframe = frame.is_keyframe; - let message_bytes = h264_ws_frame_message(&frame); - let message = Message::Binary(message_bytes); - if timeout(H264_WS_SEND_TIMEOUT, sender.send(message)) - .await - .ok() - .and_then(Result::ok) - .is_none() - { - break; - } - last_sent_sequence = Some(frame.frame_sequence); - if is_keyframe { - decoder_synced = true; - } - } - } - } -} - -async fn handle_android_h264_socket( - state: AppState, - udid: String, - initial_quality: StreamQualityPayload, - socket: WebSocket, -) { - let source = match AndroidWebRtcSource::start( - state.android.clone(), - state.metrics.clone(), - udid.clone(), - match android_h264_quality_from_stream_payload(&initial_quality) { - Ok(quality) => quality, - Err(error) => { - tracing::debug!("Invalid Android H264 WebSocket quality for {udid}: {error}"); - return; - } - }, - ) - .await - { - Ok(source) => source, - Err(error) => { - tracing::debug!("Failed to create Android H264 WebSocket source for {udid}: {error}"); - return; - } - }; - - let mut subscription = source.subscribe(); - let (mut sender, mut receiver) = socket.split(); - let mut decoder_synced = false; - let mut last_sent_sequence: Option = None; - - let initial_keyframe = source - .wait_for_keyframe(H264_WS_KEYFRAME_WAIT_TIMEOUT) - .await - .filter(|frame| h264_ws_frame_is_decoder_sync(frame)); - - if let Some(keyframe) = initial_keyframe { - if h264_ws_frame_is_supported(&keyframe) { - let message_bytes = h264_ws_frame_message(&keyframe); - let message = Message::Binary(message_bytes); - if timeout(H264_WS_SEND_TIMEOUT, sender.send(message)) - .await - .ok() - .and_then(Result::ok) - .is_none() - { - return; - } - last_sent_sequence = Some(keyframe.frame_sequence); - decoder_synced = true; - } - } else { - source.request_keyframe(); - } - - loop { - tokio::select! { - received = receiver.next() => { - let Some(received) = received else { break }; - let message = match received { - Ok(message) => message, - Err(error) => { - tracing::debug!("Android H264 WebSocket closed for {udid}: {error}"); - break; - } - }; - if !handle_android_h264_socket_message(&state, &source, &message) { - break; - } - } - frame = subscription.recv() => { - let frame = match frame { - Ok(frame) => frame, - Err(broadcast::error::RecvError::Lagged(_)) => { - decoder_synced = false; - source.request_keyframe(); - continue; - } - Err(broadcast::error::RecvError::Closed) => break, - }; - if !h264_ws_frame_is_supported(&frame) { - continue; - } - if last_sent_sequence - .map(|sequence| frame.frame_sequence <= sequence) - .unwrap_or(false) - { - continue; - } - if !decoder_synced && !frame.is_keyframe { - source.request_keyframe(); - continue; - } - let is_keyframe = frame.is_keyframe; - let message_bytes = h264_ws_frame_message(&frame); - let message = Message::Binary(message_bytes); - if timeout(H264_WS_SEND_TIMEOUT, sender.send(message)) - .await - .ok() - .and_then(Result::ok) - .is_none() - { - break; - } - last_sent_sequence = Some(frame.frame_sequence); - if is_keyframe { - decoder_synced = true; - } - } - } - } -} - -fn handle_h264_socket_message( - state: &AppState, - session: &SimulatorSession, - message: &Message, -) -> bool { - let text = match message { - Message::Text(text) => text.as_str(), - Message::Binary(bytes) => match std::str::from_utf8(bytes) { - Ok(text) => text, - Err(_) => return true, - }, - Message::Close(_) => return false, - Message::Ping(_) | Message::Pong(_) => return true, - }; - let Ok(message) = serde_json::from_str::(text) else { - return true; - }; - match message { - H264SocketMessage::ClientStats { stats } => { - if !stats.client_id.trim().is_empty() && !stats.kind.trim().is_empty() { - apply_stream_client_foreground_from_stats(state, &stats); - state.metrics.record_client_stream_stats(*stats); - } - } - H264SocketMessage::StreamControl { - client_id, - force_keyframe, - foreground, - snapshot, - } => { - apply_stream_client_foreground(state, session, &client_id, foreground); - if force_keyframe.unwrap_or(false) { - session.request_keyframe(); - } - if snapshot.unwrap_or(false) { - session.request_refresh(); - } - } - H264SocketMessage::StreamQuality { config } => { - if let Err(error) = apply_stream_quality_payload(state, &config) { - tracing::debug!("Failed to apply H264 WebSocket stream quality: {error}"); - } else { - session.request_keyframe(); - } - } - } - true -} - -fn apply_stream_client_foreground( - state: &AppState, - session: &SimulatorSession, - client_id: &Option, - foreground: Option, -) { - let Some(foreground) = foreground else { - return; - }; - let Some(client_id) = client_id - .as_deref() - .map(str::trim) - .filter(|value| !value.is_empty()) - else { - return; - }; - let (any_foreground, changed) = - state - .stream_clients - .record(session.udid(), client_id, foreground); - if changed { - session.set_client_foreground(any_foreground); - } -} - -fn handle_android_h264_socket_message( - state: &AppState, - source: &AndroidWebRtcSource, - message: &Message, -) -> bool { - let text = match message { - Message::Text(text) => text.as_str(), - Message::Binary(bytes) => match std::str::from_utf8(bytes) { - Ok(text) => text, - Err(_) => return true, - }, - Message::Close(_) => return false, - Message::Ping(_) | Message::Pong(_) => return true, - }; - let Ok(message) = serde_json::from_str::(text) else { - return true; - }; - match message { - H264SocketMessage::ClientStats { stats } => { - if !stats.client_id.trim().is_empty() && !stats.kind.trim().is_empty() { - state.metrics.record_client_stream_stats(*stats); - } - } - H264SocketMessage::StreamControl { - client_id: _, - force_keyframe, - foreground: _, - snapshot, - } => { - if force_keyframe.unwrap_or(false) { - source.request_keyframe(); - } - if snapshot.unwrap_or(false) { - source.request_refresh(); - } - } - H264SocketMessage::StreamQuality { config } => { - match android_h264_quality_from_stream_payload(&config) { - Ok(quality) => source.reconfigure_h264(quality), - Err(error) => { - tracing::debug!("Android H264 WebSocket stream quality update failed: {error}"); - source.request_keyframe(); - } - } - } - } - true -} - -fn android_h264_quality_from_stream_payload( - payload: &StreamQualityPayload, -) -> Result { - let limits = stream_quality_limits_for_payload(payload)?; - Ok(android::AndroidH264StreamQuality { - max_edge: Some(limits.max_edge), - min_bitrate: Some(limits.min_bitrate), - bits_per_pixel: Some(limits.bits_per_pixel), - }) -} - -#[derive(Debug, Deserialize)] -#[serde(tag = "type", rename_all = "camelCase")] -enum H264SocketMessage { - ClientStats { - stats: Box, - }, - StreamControl { - #[serde(rename = "clientId")] - client_id: Option, - #[serde(rename = "forceKeyframe")] - force_keyframe: Option, - foreground: Option, - snapshot: Option, - }, - StreamQuality { - config: StreamQualityPayload, - }, -} - -fn h264_ws_frame_is_supported(frame: &FramePacket) -> bool { - frame - .codec - .as_deref() - .map(|codec| { - let codec = codec.to_ascii_lowercase(); - codec == "h264" || codec.starts_with("avc1") - }) - .unwrap_or(true) -} - -fn h264_ws_frame_is_decoder_sync(frame: &FramePacket) -> bool { - h264_ws_frame_is_supported(frame) - && frame.is_keyframe - && frame - .description - .as_ref() - .map(|description| !description.is_empty()) - .unwrap_or(false) -} - -fn h264_ws_frame_message(frame: &FramePacket) -> Bytes { - let config = frame.description.as_deref().unwrap_or(&[]); - let flags = if frame.is_keyframe { - H264_WS_FLAG_KEYFRAME - } else { - 0 - } | if config.is_empty() { - 0 - } else { - H264_WS_FLAG_CONFIG - }; - let mut message = BytesMut::with_capacity(H264_WS_HEADER_LEN + config.len() + frame.data.len()); - message.extend_from_slice(H264_WS_MAGIC); - message.extend_from_slice(&[1, flags]); - message.extend_from_slice(&(H264_WS_HEADER_LEN as u16).to_be_bytes()); - message.extend_from_slice(&frame.frame_sequence.to_be_bytes()); - message.extend_from_slice(&frame.timestamp_us.to_be_bytes()); - message.extend_from_slice(&frame.width.to_be_bytes()); - message.extend_from_slice(&frame.height.to_be_bytes()); - message.extend_from_slice(&(config.len() as u32).to_be_bytes()); - message.extend_from_slice(&(frame.data.len() as u32).to_be_bytes()); - message.extend_from_slice(config); - message.extend_from_slice(&frame.data); - message.freeze() -} - async fn handle_control_socket(state: AppState, udid: String, socket: WebSocket) { let session = match state.registry.get_or_create_async(&udid).await { Ok(session) => session, @@ -6092,8 +5667,6 @@ mod tests { }; use crate::inspector::PublishedInspector; use crate::metrics::counters::ClientStreamStats; - use crate::transport::packet::FramePacket; - use bytes::Bytes; use serde_json::{json, Value}; fn selector() -> ElementSelectorPayload { @@ -6796,36 +6369,4 @@ mod tests { vec!["error", "springboard", "debug"] ); } - - #[test] - fn h264_ws_frame_message_uses_fixed_binary_header() { - let frame = FramePacket { - frame_sequence: 9, - timestamp_us: 456, - is_keyframe: true, - width: 390, - height: 844, - codec: Some("h264".to_owned()), - description: Some(Bytes::from_static(b"avcc")), - data: Bytes::from_static(b"h264-sample"), - }; - - let message = super::h264_ws_frame_message(&frame); - - assert_eq!(&message[0..4], b"SDH1"); - assert_eq!(message[4], 1); - assert_eq!( - message[5], - super::H264_WS_FLAG_KEYFRAME | super::H264_WS_FLAG_CONFIG - ); - assert_eq!(u16::from_be_bytes([message[6], message[7]]), 40); - assert_eq!(u64::from_be_bytes(message[8..16].try_into().unwrap()), 9); - assert_eq!(u64::from_be_bytes(message[16..24].try_into().unwrap()), 456); - assert_eq!(u32::from_be_bytes(message[24..28].try_into().unwrap()), 390); - assert_eq!(u32::from_be_bytes(message[28..32].try_into().unwrap()), 844); - assert_eq!(u32::from_be_bytes(message[32..36].try_into().unwrap()), 4); - assert_eq!(u32::from_be_bytes(message[36..40].try_into().unwrap()), 11); - assert_eq!(&message[40..44], b"avcc"); - assert_eq!(&message[44..], b"h264-sample"); - } } diff --git a/packages/server/src/main.rs b/packages/server/src/main.rs index 0a8425bf..de931088 100644 --- a/packages/server/src/main.rs +++ b/packages/server/src/main.rs @@ -190,6 +190,8 @@ enum Command { client_root: Option, #[arg(long, value_enum, default_value_t = VideoCodecMode::Auto)] video_codec: VideoCodecMode, + #[arg(long, value_enum, default_value_t = AndroidGpuMode::Host)] + android_gpu: AndroidGpuMode, #[arg(long)] low_latency: bool, #[arg(long, value_enum)] @@ -308,11 +310,6 @@ enum Command { #[arg(long, default_value_t = 5.0, value_parser = parse_positive_seconds_arg)] seconds: f64, }, - Stream { - udid: Option, - #[arg(long, default_value_t = 0)] - frames: u64, - }, #[command(name = "describe", visible_alias = "snapshot")] DescribeUi { udid: Option, @@ -661,6 +658,8 @@ enum ServiceCommand { client_root: Option, #[arg(long, value_enum, default_value_t = VideoCodecMode::Auto)] video_codec: VideoCodecMode, + #[arg(long, value_enum, default_value_t = AndroidGpuMode::Host)] + android_gpu: AndroidGpuMode, #[arg(long)] low_latency: bool, #[arg(long, value_enum)] @@ -679,6 +678,8 @@ enum ServiceCommand { client_root: Option, #[arg(long, value_enum, default_value_t = VideoCodecMode::Auto)] video_codec: VideoCodecMode, + #[arg(long, value_enum, default_value_t = AndroidGpuMode::Host)] + android_gpu: AndroidGpuMode, #[arg(long)] low_latency: bool, #[arg(long, value_enum)] @@ -702,6 +703,8 @@ enum ServiceCommand { client_root: Option, #[arg(long, value_enum, default_value_t = VideoCodecMode::Auto)] video_codec: VideoCodecMode, + #[arg(long, value_enum, default_value_t = AndroidGpuMode::Host)] + android_gpu: AndroidGpuMode, #[arg(long)] low_latency: bool, #[arg(long, value_enum)] @@ -720,6 +723,8 @@ enum ServiceCommand { client_root: Option, #[arg(long, value_enum, default_value_t = VideoCodecMode::Auto)] video_codec: VideoCodecMode, + #[arg(long, value_enum, default_value_t = AndroidGpuMode::Host)] + android_gpu: AndroidGpuMode, #[arg(long)] low_latency: bool, #[arg(long, value_enum)] @@ -750,6 +755,8 @@ enum ServiceCommand { client_root: Option, #[arg(long, value_enum, default_value_t = VideoCodecMode::Auto)] video_codec: VideoCodecMode, + #[arg(long, value_enum, default_value_t = AndroidGpuMode::Host)] + android_gpu: AndroidGpuMode, #[arg(long)] low_latency: bool, #[arg(long, value_enum)] @@ -861,6 +868,18 @@ enum VideoCodecMode { Software, } +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +enum AndroidGpuMode { + Auto, + Host, + Software, + Lavapipe, + Swiftshader, + Swangle, + #[value(name = "swiftshader_indirect", alias = "swiftshader-indirect")] + SwiftshaderIndirect, +} + #[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] enum ServerKindArg { LaunchAgent, @@ -945,6 +964,8 @@ struct ServiceMetadata { log_path: Option, #[serde(default, skip_serializing_if = "Option::is_none")] video_codec: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + android_gpu: Option, #[serde(default)] low_latency: bool, #[serde(default)] @@ -982,6 +1003,7 @@ struct ServiceLaunchOptions { advertise_host: Option, client_root: Option, video_codec: VideoCodecMode, + android_gpu: AndroidGpuMode, low_latency: bool, realtime_stream: bool, allow_port_probe: bool, @@ -1010,6 +1032,20 @@ impl VideoCodecMode { } } +impl AndroidGpuMode { + fn as_emulator_value(self) -> &'static str { + match self { + Self::Auto => "auto", + Self::Host => "host", + Self::Software => "software", + Self::Lavapipe => "lavapipe", + Self::Swiftshader => "swiftshader", + Self::Swangle => "swangle", + Self::SwiftshaderIndirect => "swiftshader_indirect", + } + } +} + fn parse_video_codec_mode(value: &str) -> Option { match value { "auto" => Some(VideoCodecMode::Auto), @@ -1019,6 +1055,19 @@ fn parse_video_codec_mode(value: &str) -> Option { } } +fn parse_android_gpu_mode(value: &str) -> Option { + match value.trim().to_ascii_lowercase().replace('-', "_").as_str() { + "auto" => Some(AndroidGpuMode::Auto), + "host" => Some(AndroidGpuMode::Host), + "software" => Some(AndroidGpuMode::Software), + "lavapipe" => Some(AndroidGpuMode::Lavapipe), + "swiftshader" => Some(AndroidGpuMode::Swiftshader), + "swangle" => Some(AndroidGpuMode::Swangle), + "swiftshader_indirect" => Some(AndroidGpuMode::SwiftshaderIndirect), + _ => None, + } +} + fn parse_stream_quality_profile(value: &str) -> Option { match value { "quality" => Some(StreamQualityProfileArg::Quality), @@ -1085,7 +1134,7 @@ fn stream_quality_env_for_profile(profile: &str) -> anyhow::Result Ok(StreamQualityEnvironment { profile: "smooth", - max_edge: 1170, + max_edge: 4096, fps: 60, min_bitrate: 4_000_000, bits_per_pixel: 5, @@ -1192,6 +1241,7 @@ impl Default for ServiceLaunchOptions { advertise_host: None, client_root: None, video_codec: VideoCodecMode::Auto, + android_gpu: AndroidGpuMode::Host, low_latency: false, realtime_stream: false, allow_port_probe: false, @@ -1262,6 +1312,7 @@ fn ensure_launch_agent_service(options: ServiceLaunchOptions) -> anyhow::Result< advertise_host: options.advertise_host.clone(), client_root: options.client_root.clone(), video_codec: options.video_codec, + android_gpu: options.android_gpu, low_latency: options.low_latency, stream_quality_profile: options.stream_quality_profile.clone(), local_stream_fps: options.local_stream_fps, @@ -1304,6 +1355,8 @@ fn start_project_service(options: ServiceLaunchOptions) -> anyhow::Result, client_root: Option, video_codec: VideoCodecMode, + android_gpu: AndroidGpuMode, low_latency: bool, stream_quality: Option, local_stream_fps: Option, @@ -2429,6 +2489,7 @@ impl Default for DefaultServiceLaunchOptions { advertise_host: None, client_root: None, video_codec: VideoCodecMode::Auto, + android_gpu: AndroidGpuMode::Host, low_latency: false, stream_quality: None, local_stream_fps: None, @@ -2494,6 +2555,14 @@ fn no_command_action_from_args_slice(args: &[String]) -> Option options.video_codec = parse_video_codec_mode(value.strip_prefix("--video-codec=")?)?; } + "--android-gpu" => { + i += 1; + options.android_gpu = parse_android_gpu_mode(args.get(i)?)?; + } + value if value.starts_with("--android-gpu=") => { + options.android_gpu = + parse_android_gpu_mode(value.strip_prefix("--android-gpu=")?)?; + } "--stream-quality" => { i += 1; options.stream_quality = Some(parse_stream_quality_profile(args.get(i)?)?); @@ -2601,6 +2670,7 @@ fn run_default_service(options: DefaultServiceLaunchOptions) -> anyhow::Result<( advertise_host: options.advertise_host, client_root: options.client_root, video_codec: options.video_codec, + android_gpu: options.android_gpu, low_latency: options.low_latency, realtime_stream: false, allow_port_probe: !options.autostart, @@ -2649,6 +2719,7 @@ fn restart_detached_service(options: ServiceLaunchOptions) -> anyhow::Result<()> advertise_host: options.advertise_host, client_root: options.client_root, video_codec: options.video_codec, + android_gpu: options.android_gpu, low_latency: options.low_latency, stream_quality_profile: options.stream_quality_profile, local_stream_fps: options.local_stream_fps, @@ -2681,6 +2752,7 @@ struct PairGlobalServiceOptions { advertise_host: Option, client_root: Option, video_codec: VideoCodecMode, + android_gpu: AndroidGpuMode, low_latency: bool, stream_quality: Option, local_stream_fps: Option, @@ -2694,6 +2766,7 @@ fn pair_global_service(options: PairGlobalServiceOptions) -> anyhow::Result<()> advertise_host, client_root, video_codec, + android_gpu, low_latency, stream_quality, local_stream_fps, @@ -2729,6 +2802,7 @@ fn pair_global_service(options: PairGlobalServiceOptions) -> anyhow::Result<()> advertise_host, client_root, video_codec, + android_gpu, low_latency, stream_quality_profile: local_stream_quality_profile(low_latency, stream_quality), local_stream_fps, @@ -2863,6 +2937,7 @@ fn expose_to_studio(options: StudioExposeOptions) -> anyhow::Result<()> { advertise_host: None, client_root: None, video_codec: options.video_codec, + android_gpu: AndroidGpuMode::Host, low_latency: options.low_latency, realtime_stream: true, allow_port_probe: false, @@ -3545,6 +3620,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + android_gpu, low_latency, stream_quality, local_stream_fps, @@ -3555,6 +3631,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + android_gpu, low_latency, stream_quality, local_stream_fps, @@ -3613,6 +3690,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + android_gpu, low_latency, stream_quality, local_stream_fps, @@ -3624,6 +3702,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + android_gpu, low_latency, realtime_stream: false, allow_port_probe: false, @@ -3641,6 +3720,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + android_gpu, low_latency, stream_quality, local_stream_fps, @@ -3653,6 +3733,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + android_gpu, low_latency, stream_quality_profile: local_stream_quality_profile( low_latency, @@ -3669,6 +3750,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + android_gpu, low_latency, stream_quality, local_stream_fps, @@ -3681,6 +3763,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + android_gpu, low_latency, realtime_stream: false, allow_port_probe: false, @@ -3697,6 +3780,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + android_gpu, low_latency, stream_quality, local_stream_fps, @@ -3709,6 +3793,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + android_gpu, low_latency, stream_quality_profile: local_stream_quality_profile( low_latency, @@ -3730,6 +3815,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + android_gpu, low_latency, stream_quality, local_stream_fps, @@ -3760,6 +3846,7 @@ fn main() -> anyhow::Result<()> { started_at: now_secs(), log_path: service_log_path().ok(), video_codec: Some(video_codec.as_env_value().to_owned()), + android_gpu: Some(android_gpu.as_emulator_value().to_owned()), low_latency, realtime_stream: crate::transport::webrtc::realtime_stream_enabled() || low_latency @@ -3777,6 +3864,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + android_gpu, low_latency, stream_quality_profile, local_stream_fps, @@ -4174,10 +4262,6 @@ fn main() -> anyhow::Result<()> { } Ok(()) } - Command::Stream { udid, frames } => { - let udid = resolve_device_udid(udid.as_deref())?; - run_stream_stdout(&bridge, udid, frames) - } Command::DescribeUi { udid, point, @@ -5011,6 +5095,7 @@ struct ServiceOptions { advertise_host: Option, client_root: Option, video_codec: VideoCodecMode, + android_gpu: AndroidGpuMode, low_latency: bool, stream_quality_profile: Option, local_stream_fps: Option, @@ -5025,6 +5110,7 @@ fn serve_with_appkit( advertise_host: Option, client_root: Option, video_codec: VideoCodecMode, + android_gpu: AndroidGpuMode, low_latency: bool, stream_quality_profile: Option, local_stream_fps: Option, @@ -5033,6 +5119,7 @@ fn serve_with_appkit( pairing_code: Option, ) -> anyhow::Result<()> { std::env::set_var("SIMDECK_VIDEO_CODEC", video_codec.as_env_value()); + std::env::set_var("SIMDECK_ANDROID_GPU", android_gpu.as_emulator_value()); std::env::set_var("SIMDECK_LOW_LATENCY", if low_latency { "1" } else { "0" }); if let Some(local_stream_fps) = local_stream_fps { std::env::set_var("SIMDECK_LOCAL_STREAM_FPS", local_stream_fps.to_string()); @@ -5490,39 +5577,6 @@ fn default_recording_path(udid: &str) -> PathBuf { PathBuf::from(format!("Simulator Recording - {udid} - {timestamp}.mp4")) } -fn run_stream_stdout(bridge: &NativeBridge, udid: String, frames: u64) -> anyhow::Result<()> { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_time() - .build() - .context("create stream runtime")?; - let _runtime_guard = runtime.enter(); - let metrics = Arc::new(Metrics::default()); - let session = simulators::session::SimulatorSession::new(bridge, udid, metrics) - .map_err(|error| anyhow::anyhow!("{error}"))?; - session - .ensure_started() - .map_err(|error| anyhow::anyhow!("{error}"))?; - session.request_keyframe(); - - let mut receiver = session.subscribe(); - let mut stdout = io::stdout().lock(); - let mut written = 0u64; - runtime.block_on(async { - loop { - if frames > 0 && written >= frames { - break; - } - let frame = receiver.recv().await?; - let sample = crate::transport::webrtc::h264_annex_b_sample(&frame) - .map_err(|error| anyhow::anyhow!("encode Annex B frame: {error}"))?; - stdout.write_all(&sample)?; - stdout.flush()?; - written += 1; - } - anyhow::Ok(()) - }) -} - #[allow(clippy::too_many_arguments)] fn describe_ui_snapshot( bridge: &NativeBridge, @@ -6128,11 +6182,12 @@ mod tests { render_agent_accessibility_tree, render_qr_code, run_maestro_command, server_health_watchdog_should_restart, service_addresses, service_matches_launch_options, service_post_error_is_retryable, simdeck_open_link, simdeck_pair_url, - studio_service_restart_args, workspace_service_process_is_current, Cli, Command, - ElementSelector, NoCommandAction, PairingAddress, ServiceCommand, ServiceLaunchOptions, - ServiceMetadata, StreamQualityProfileArg, StudioExposeOptions, TapCommandTarget, - VideoCodecMode, WorkspaceServiceProcess, YamlValue, DEFAULT_LOCAL_STREAM_QUALITY_PROFILE, - SERVER_HEALTH_WATCHDOG_FAILURE_THRESHOLD, SERVER_HEALTH_WATCHDOG_HTTP_FAILURE_THRESHOLD, + studio_service_restart_args, workspace_service_process_is_current, AndroidGpuMode, Cli, + Command, ElementSelector, NoCommandAction, PairingAddress, ServiceCommand, + ServiceLaunchOptions, ServiceMetadata, StreamQualityProfileArg, StudioExposeOptions, + TapCommandTarget, VideoCodecMode, WorkspaceServiceProcess, YamlValue, + DEFAULT_LOCAL_STREAM_QUALITY_PROFILE, SERVER_HEALTH_WATCHDOG_FAILURE_THRESHOLD, + SERVER_HEALTH_WATCHDOG_HTTP_FAILURE_THRESHOLD, }; use clap::Parser; use std::collections::HashMap; @@ -6159,6 +6214,7 @@ mod tests { started_at: 1, log_path: None, video_codec: Some(VideoCodecMode::Auto.as_env_value().to_owned()), + android_gpu: Some(AndroidGpuMode::Host.as_emulator_value().to_owned()), low_latency: false, realtime_stream: true, stream_quality_profile: Some(DEFAULT_LOCAL_STREAM_QUALITY_PROFILE.to_owned()), @@ -6178,6 +6234,7 @@ mod tests { advertise_host: advertise_host.map(str::to_owned), client_root: client_root.map(PathBuf::from), video_codec: VideoCodecMode::Auto, + android_gpu: AndroidGpuMode::Host, low_latency: false, realtime_stream: false, allow_port_probe: false, @@ -6418,6 +6475,7 @@ mod tests { started_at: 1, log_path: None, video_codec: Some(VideoCodecMode::Auto.as_env_value().to_owned()), + android_gpu: Some(AndroidGpuMode::Host.as_emulator_value().to_owned()), low_latency: false, realtime_stream: true, stream_quality_profile: Some(DEFAULT_LOCAL_STREAM_QUALITY_PROFILE.to_owned()), diff --git a/packages/server/src/metrics/counters.rs b/packages/server/src/metrics/counters.rs index 2ef135b4..7667658a 100644 --- a/packages/server/src/metrics/counters.rs +++ b/packages/server/src/metrics/counters.rs @@ -82,10 +82,6 @@ pub struct ClientStreamStats { pub rendered_frames: Option, pub decoder_dropped_frames: Option, pub dropped_frames: Option, - pub h264_parse_failures: Option, - pub h264_socket_bytes: Option, - pub h264_socket_messages: Option, - pub h264_socket_message_type: Option, pub packets_lost: Option, pub presentation_dropped_frames: Option, pub ice_restarts: Option, @@ -261,10 +257,6 @@ mod tests { rendered_frames: None, decoder_dropped_frames: None, dropped_frames: None, - h264_parse_failures: None, - h264_socket_bytes: None, - h264_socket_messages: None, - h264_socket_message_type: None, packets_lost: None, presentation_dropped_frames: None, ice_restarts: None, diff --git a/packages/server/src/service.rs b/packages/server/src/service.rs index 96996f00..c85ce310 100644 --- a/packages/server/src/service.rs +++ b/packages/server/src/service.rs @@ -333,6 +333,8 @@ fn service_options_match_arguments( && argument_value(arguments, "--client-root").as_deref() == Some(client_root.as_ref()) && argument_value(arguments, "--video-codec").as_deref() == Some(options.video_codec.as_env_value()) + && argument_value(arguments, "--android-gpu").as_deref() + == Some(options.android_gpu.as_emulator_value()) && argument_value(arguments, "--server-kind").as_deref() == Some("launch-agent") && arguments .windows(2) @@ -615,6 +617,8 @@ fn plist_contents( client_root.to_string_lossy().into_owned(), "--video-codec".to_string(), options.video_codec.as_env_value().to_string(), + "--android-gpu".to_string(), + options.android_gpu.as_emulator_value().to_string(), "--server-kind".to_string(), "launch-agent".to_string(), ]; @@ -728,6 +732,7 @@ mod tests { advertise_host: None, client_root: None, video_codec: crate::VideoCodecMode::Auto, + android_gpu: crate::AndroidGpuMode::Host, low_latency: false, stream_quality_profile: None, local_stream_fps: None, @@ -749,6 +754,8 @@ mod tests { "/tmp/client".to_owned(), "--video-codec".to_owned(), options.video_codec.as_env_value().to_owned(), + "--android-gpu".to_owned(), + options.android_gpu.as_emulator_value().to_owned(), "--server-kind".to_owned(), "launch-agent".to_owned(), ]; diff --git a/packages/server/src/transport/webrtc.rs b/packages/server/src/transport/webrtc.rs index d2ca9da4..26c7a328 100644 --- a/packages/server/src/transport/webrtc.rs +++ b/packages/server/src/transport/webrtc.rs @@ -3,7 +3,7 @@ use crate::api::routes::{ apply_stream_client_foreground_from_stats, apply_stream_quality_payload, bridge_input_session_for_control, run_bridge_multitouch_control_message, run_control_message, run_toggle_appearance_control, run_tvos_control_message, stream_quality_limits_for_payload, - AppState, ControlMessage, StreamQualityLimits, StreamQualityPayload, TvosControlTouchGesture, + AppState, ControlMessage, StreamQualityPayload, TvosControlTouchGesture, }; use crate::error::AppError; use crate::metrics::counters::ClientStreamStats; @@ -63,8 +63,8 @@ const WEBRTC_FULL_ICE_GATHER_TIMEOUT: Duration = Duration::from_secs(3); const WEBRTC_MULTITOUCH_INPUT_IDLE_TIMEOUT: Duration = Duration::from_secs(5); const WEBRTC_RTP_OUTBOUND_MTU: usize = 1200; const WEBRTC_PEER_DISCONNECTED_TIMEOUT: Duration = Duration::from_secs(12); -const ANDROID_WEBRTC_FRAME_BROADCAST_CAPACITY: usize = 128; -const ANDROID_WEBRTC_FPS: u64 = 60; +const ANDROID_WEBRTC_FRAME_BROADCAST_CAPACITY: usize = 1; +const ANDROID_WEBRTC_DEFAULT_POLL_FPS: u64 = 120; const ANDROID_SHARED_VIDEO_RETRY_DELAY: Duration = Duration::from_millis(200); static WEBRTC_MEDIA_STREAMS: OnceLock>>> = OnceLock::new(); @@ -111,17 +111,20 @@ fn android_h264_quality_from_payload( return Ok(android::AndroidH264StreamQuality::default()); }; let limits = stream_quality_limits_for_payload(payload)?; - Ok(android_h264_quality_from_limits(limits)) -} - -fn android_h264_quality_from_limits( - limits: StreamQualityLimits, -) -> android::AndroidH264StreamQuality { - android::AndroidH264StreamQuality { - max_edge: Some(limits.max_edge), + let unscaled_profile = payload + .profile + .as_deref() + .map(str::trim) + .is_some_and(|profile| matches!(profile, "full" | "quality" | "smooth")); + Ok(android::AndroidH264StreamQuality { + max_edge: if unscaled_profile { + None + } else { + Some(limits.max_edge) + }, min_bitrate: Some(limits.min_bitrate), bits_per_pixel: Some(limits.bits_per_pixel), - } + }) } #[derive(Debug, Clone, Serialize)] @@ -709,7 +712,6 @@ async fn run_android_webrtc_control_queue( mut receiver: mpsc::UnboundedReceiver, ) { let mut pending = VecDeque::new(); - let mut active_touch: Option = None; loop { let mut message = match pending.pop_front() { Some(message) => message, @@ -729,13 +731,8 @@ async fn run_android_webrtc_control_queue( } } - if let Err(error) = run_android_webrtc_control_message( - state.clone(), - udid.clone(), - message, - &mut active_touch, - ) - .await + if let Err(error) = + run_android_webrtc_control_message(state.clone(), udid.clone(), message).await { warn!("Android WebRTC control message failed for {udid}: {error}"); } @@ -746,7 +743,6 @@ async fn run_android_webrtc_control_message( state: AppState, udid: String, message: ControlMessage, - active_touch: &mut Option, ) -> Result<(), AppError> { match message { ControlMessage::Touch { x, y, phase } => { @@ -761,7 +757,6 @@ async fn run_android_webrtc_control_message( x.clamp(0.0, 1.0), y.clamp(0.0, 1.0), phase, - active_touch, ) .await; } @@ -777,7 +772,6 @@ async fn run_android_webrtc_control_message( x.clamp(0.0, 1.0), y.clamp(0.0, 1.0), phase, - active_touch, ) .await; } @@ -793,7 +787,6 @@ async fn run_android_webrtc_control_message( x1.clamp(0.0, 1.0), y1.clamp(0.0, 1.0), phase, - active_touch, ) .await; } @@ -849,13 +842,8 @@ async fn handle_android_webrtc_touch( x: f64, y: f64, phase: String, - active_touch: &mut Option, ) -> Result<(), AppError> { - let action = android::update_touch_gesture(active_touch, x, y, &phase)?; - if matches!(action, android::AndroidTouchAction::None) { - return Ok(()); - } - task::spawn_blocking(move || action.perform(&state.android, &udid)) + task::spawn_blocking(move || state.android.send_touch(&udid, x, y, &phase)) .await .map_err(|error| { AppError::internal(format!("Failed to join Android touch task: {error}")) @@ -1422,6 +1410,8 @@ fn spawn_android_shared_video_encoder( } }; info!("Android shared-video stream attached for {udid}"); + let poll_interval = android_webrtc_poll_interval(); + let mut next_poll_at = Instant::now(); loop { if android_shutdown_requested(&mut shutdown_rx) { return; @@ -1443,7 +1433,7 @@ fn spawn_android_shared_video_encoder( break; } } - thread::sleep(android_webrtc_frame_interval()); + sleep_until_next_android_poll(&mut next_poll_at, poll_interval); } }); } @@ -1629,8 +1619,26 @@ unsafe fn take_native_error(error: *mut i8, fallback: &str) -> AppError { AppError::native(message) } -fn android_webrtc_frame_interval() -> Duration { - Duration::from_micros(1_000_000 / ANDROID_WEBRTC_FPS) +fn android_webrtc_poll_interval() -> Duration { + let fps = std::env::var("SIMDECK_ANDROID_SHARED_VIDEO_POLL_FPS") + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(ANDROID_WEBRTC_DEFAULT_POLL_FPS) + .clamp(60, u64::from(WEBRTC_MAX_LOCAL_STREAM_FPS)); + Duration::from_micros(1_000_000 / fps) +} + +fn sleep_until_next_android_poll(next_poll_at: &mut Instant, interval: Duration) { + let target = next_poll_at + .checked_add(interval) + .unwrap_or_else(Instant::now); + *next_poll_at = target; + let now = Instant::now(); + if target > now { + thread::sleep(target - now); + } else { + *next_poll_at = now; + } } #[derive(Clone)] @@ -1640,6 +1648,10 @@ enum WebRtcVideoSource { } impl WebRtcVideoSource { + fn uses_frame_timestamps_for_realtime(&self) -> bool { + matches!(self, Self::Android(_)) + } + fn subscribe(&self) -> WebRtcFrameReceiver { match self { Self::Simulator(session) => WebRtcFrameReceiver::Simulator(session.subscribe()), @@ -1746,7 +1758,12 @@ impl WebRtcMediaStream { let mut waiting_for_keyframe = false; let mut peer_disconnected_since: Option = None; let _guard = WebRtcMetricsGuard::new(state.metrics.clone()); - let first_frame_duration = send_timing.duration_for(&first_frame, realtime_stream); + let uses_frame_timestamps_for_realtime = source.uses_frame_timestamps_for_realtime(); + let first_frame_duration = send_timing.duration_for( + &first_frame, + realtime_stream, + uses_frame_timestamps_for_realtime, + ); match write_frame_sample_with_timeout( &video_track, @@ -1844,7 +1861,11 @@ impl WebRtcMediaStream { state.metrics.frames_dropped_server.fetch_add(1, Ordering::Relaxed); continue; } - let duration = send_timing.duration_for(&frame, realtime_stream); + let duration = send_timing.duration_for( + &frame, + realtime_stream, + uses_frame_timestamps_for_realtime, + ); let write_result = write_frame_sample_with_timeout( &video_track, &mut packetizer, @@ -2253,6 +2274,7 @@ impl WebRtcSendTiming { &mut self, frame: &crate::transport::packet::FramePacket, realtime_stream: bool, + use_frame_timestamps_for_realtime: bool, ) -> Duration { const MIN_FRAME_DURATION_US: u64 = 1_000; const DEFAULT_FRAME_DURATION_US: u64 = 16_667; @@ -2267,7 +2289,7 @@ impl WebRtcSendTiming { .try_into() .unwrap_or(DEFAULT_FRAME_DURATION_US); - if realtime_stream { + if realtime_stream && !use_frame_timestamps_for_realtime { self.last_timestamp_us = Some(frame.timestamp_us); return default_duration; } @@ -2322,11 +2344,13 @@ impl Drop for WebRtcMetricsGuard { #[cfg(test)] mod tests { use super::{ - append_avcc_parameter_sets, append_length_prefixed_nalus, h264_annex_b_sample, - h264_frame_has_idr, h264_frame_is_decoder_sync, h264_sdp_fmtp_line, is_annex_b, - is_h264_codec, rtcp_packet_requests_keyframe, rtp_packet_pacing, WebRtcMetricsGuard, - WebRtcSendTiming, ANNEX_B_START_CODE, + android_h264_quality_from_payload, append_avcc_parameter_sets, + append_length_prefixed_nalus, h264_annex_b_sample, h264_frame_has_idr, + h264_frame_is_decoder_sync, h264_sdp_fmtp_line, is_annex_b, is_h264_codec, + rtcp_packet_requests_keyframe, rtp_packet_pacing, WebRtcMetricsGuard, WebRtcSendTiming, + ANNEX_B_START_CODE, }; + use crate::api::routes::StreamQualityPayload; use crate::metrics::counters::Metrics; use crate::transport::packet::FramePacket; use bytes::Bytes; @@ -2381,6 +2405,44 @@ mod tests { assert!(!rtcp_packet_requests_keyframe(&SenderReport::default())); } + #[test] + fn android_full_size_quality_keeps_native_dimensions() { + for (profile, min_bitrate, bits_per_pixel) in + [("full", 12_000_000, 4), ("smooth", 4_000_000, 5)] + { + let payload = StreamQualityPayload { + profile: Some(profile.to_owned()), + video_codec: None, + max_edge: None, + fps: Some(60), + min_bitrate: None, + bits_per_pixel: None, + }; + + let quality = android_h264_quality_from_payload(Some(&payload)).unwrap(); + + assert_eq!(quality.max_edge, None); + assert_eq!(quality.min_bitrate, Some(min_bitrate)); + assert_eq!(quality.bits_per_pixel, Some(bits_per_pixel)); + } + } + + #[test] + fn android_scaled_quality_applies_profile_edge() { + let payload = StreamQualityPayload { + profile: Some("balanced".to_owned()), + video_codec: None, + max_edge: None, + fps: Some(60), + min_bitrate: None, + bits_per_pixel: None, + }; + + let quality = android_h264_quality_from_payload(Some(&payload)).unwrap(); + + assert_eq!(quality.max_edge, Some(1280)); + } + #[test] fn realtime_h264_advertises_retransmission_feedback() { let feedback = super::h264_rtcp_feedback(); @@ -2549,11 +2611,11 @@ mod tests { }; assert_eq!( - timing.duration_for(&first, false), + timing.duration_for(&first, false, false), Duration::from_micros(16_667) ); assert_eq!( - timing.duration_for(&second, false), + timing.duration_for(&second, false, false), Duration::from_micros(33_333) ); } @@ -2581,15 +2643,15 @@ mod tests { }; assert_eq!( - timing.duration_for(&first, false), + timing.duration_for(&first, false, false), Duration::from_micros(16_667) ); assert_eq!( - timing.duration_for(&backwards, false), + timing.duration_for(&backwards, false, false), Duration::from_micros(16_667) ); assert_eq!( - timing.duration_for(&huge_gap, false), + timing.duration_for(&huge_gap, false, false), Duration::from_micros(100_000) ); } @@ -2631,13 +2693,47 @@ mod tests { }; assert_eq!( - timing.duration_for(&first, true), + timing.duration_for(&first, true, false), + super::realtime_sample_duration() + ); + assert_eq!( + timing.duration_for(&second, true, false), super::realtime_sample_duration() ); + } + + #[test] + fn realtime_android_send_timing_can_use_frame_timestamps() { + let mut timing = WebRtcSendTiming::new(); + let first = FramePacket { + frame_sequence: 1, + timestamp_us: 10_000, + is_keyframe: true, + width: 100, + height: 100, + codec: Some("h264".to_owned()), + description: None, + data: Bytes::from_static(&[0, 0, 1, 0x65]), + }; + let second = FramePacket { + frame_sequence: 2, + timestamp_us: 18_333, + is_keyframe: false, + width: 100, + height: 100, + codec: Some("h264".to_owned()), + description: None, + data: Bytes::from_static(&[0, 0, 1, 0x41]), + }; + assert_eq!( - timing.duration_for(&second, true), + timing.duration_for(&first, true, true), super::realtime_sample_duration() ); + assert_eq!( + timing.duration_for(&second, true, true), + Duration::from_micros(8_333) + ); } #[test] diff --git a/skills/simdeck/SKILL.md b/skills/simdeck/SKILL.md index 9341b57d..61573d70 100644 --- a/skills/simdeck/SKILL.md +++ b/skills/simdeck/SKILL.md @@ -98,6 +98,9 @@ Build apps with project tooling. Android devices use IDs like `android:Pixel_8_API_36`. `simdeck list` discovers AVDs from the Android SDK. +SimDeck-owned Android boots use the emulator shared-video surface plus +`--android-gpu host` by default. Use `simdeck service restart --android-gpu auto` +or `--android-gpu swiftshader_indirect` only when host GPU rendering is unstable. ## Fast agent inspection From 331653c878775beaa6178bab80de19907cb04641 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 4 Jun 2026 22:35:06 -0400 Subject: [PATCH 3/5] fix: harden android and webrtc ci --- packages/server/src/android.rs | 128 +++++++++++++++++++-------------- scripts/integration/webrtc.mjs | 6 +- 2 files changed, 78 insertions(+), 56 deletions(-) diff --git a/packages/server/src/android.rs b/packages/server/src/android.rs index 096ea91a..f9af1cd7 100644 --- a/packages/server/src/android.rs +++ b/packages/server/src/android.rs @@ -2,11 +2,14 @@ use crate::error::AppError; use serde_json::{json, Value}; use std::collections::HashMap; use std::env; +#[cfg(unix)] use std::ffi::CString; use std::ffi::OsString; use std::io::{Read, Write}; +#[cfg(unix)] use std::mem::MaybeUninit; use std::net::{SocketAddr, TcpStream}; +#[cfg(unix)] use std::os::fd::RawFd; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; @@ -36,6 +39,9 @@ const MODIFIER_OPTION: u32 = 1 << 2; const MODIFIER_COMMAND: u32 = 1 << 3; const MODIFIER_CAPS_LOCK: u32 = 1 << 4; +#[cfg(not(unix))] +type RawFd = i32; + type TimedMap = Option<(Instant, HashMap)>; type DisplayMetricsCache = HashMap; type ConsoleConnectionCache = HashMap; @@ -1102,6 +1108,7 @@ impl AndroidSharedVideoFrameStream { impl Drop for AndroidSharedVideoFrameStream { fn drop(&mut self) { + #[cfg(unix)] unsafe { if !self.ptr.is_null() && self.length > 0 { libc::munmap(self.ptr.cast(), self.length); @@ -1150,75 +1157,86 @@ fn round_android_h264_dimension(value: u32) -> u32 { } fn open_android_shared_video_memory(handle: &str) -> Result<(RawFd, *mut u8, usize), AppError> { - let mut names = vec![handle.to_owned()]; - if !handle.starts_with('/') { - names.push(format!("/{handle}")); - } - let mut last_error = None; - for name in names { - let c_name = CString::new(name.as_str()) - .map_err(|_| AppError::native("Android shared video handle contains a NUL byte."))?; - let fd = unsafe { libc::shm_open(c_name.as_ptr(), libc::O_RDONLY, 0) }; - if fd < 0 { - last_error = Some(std::io::Error::last_os_error()); - continue; - } + #[cfg(not(unix))] + { + return Err(AppError::native(format!( + "Android emulator shared video `{handle}` requires POSIX shared memory and is not available on this host." + ))); + } - let mut stat = MaybeUninit::::uninit(); - if unsafe { libc::fstat(fd, stat.as_mut_ptr()) } != 0 { - let error = std::io::Error::last_os_error(); - unsafe { - libc::close(fd); - } - last_error = Some(error); - continue; + #[cfg(unix)] + { + let mut names = vec![handle.to_owned()]; + if !handle.starts_with('/') { + names.push(format!("/{handle}")); } - let stat = unsafe { stat.assume_init() }; - let length = usize::try_from(stat.st_size).map_err(|_| { - unsafe { - libc::close(fd); + let mut last_error = None; + for name in names { + let c_name = CString::new(name.as_str()).map_err(|_| { + AppError::native("Android shared video handle contains a NUL byte.") + })?; + let fd = unsafe { libc::shm_open(c_name.as_ptr(), libc::O_RDONLY, 0) }; + if fd < 0 { + last_error = Some(std::io::Error::last_os_error()); + continue; } - AppError::native(format!( - "Android emulator shared video `{handle}` reported an invalid size." - )) - })?; - if length <= ANDROID_SHARED_VIDEO_HEADER_BYTES { - unsafe { - libc::close(fd); + + let mut stat = MaybeUninit::::uninit(); + if unsafe { libc::fstat(fd, stat.as_mut_ptr()) } != 0 { + let error = std::io::Error::last_os_error(); + unsafe { + libc::close(fd); + } + last_error = Some(error); + continue; + } + let stat = unsafe { stat.assume_init() }; + let length = usize::try_from(stat.st_size).map_err(|_| { + unsafe { + libc::close(fd); + } + AppError::native(format!( + "Android emulator shared video `{handle}` reported an invalid size." + )) + })?; + if length <= ANDROID_SHARED_VIDEO_HEADER_BYTES { + unsafe { + libc::close(fd); + } + return Err(AppError::native(format!( + "Android emulator shared video `{handle}` is smaller than the display header." + ))); } - return Err(AppError::native(format!( - "Android emulator shared video `{handle}` is smaller than the display header." - ))); - } - let ptr = unsafe { - libc::mmap( - ptr::null_mut(), - length, - libc::PROT_READ, - libc::MAP_SHARED, - fd, - 0, - ) - }; - if ptr == libc::MAP_FAILED { - let error = std::io::Error::last_os_error(); - unsafe { - libc::close(fd); + let ptr = unsafe { + libc::mmap( + ptr::null_mut(), + length, + libc::PROT_READ, + libc::MAP_SHARED, + fd, + 0, + ) + }; + if ptr == libc::MAP_FAILED { + let error = std::io::Error::last_os_error(); + unsafe { + libc::close(fd); + } + last_error = Some(error); + continue; } - last_error = Some(error); - continue; + return Ok((fd, ptr.cast::(), length)); } - return Ok((fd, ptr.cast::(), length)); - } - Err(AppError::native(format!( + Err(AppError::native(format!( "Android emulator shared video `{handle}` is unavailable. Start the emulator through SimDeck or launch it with `-share-vid`{}.", last_error .as_ref() .map(|error| format!(" ({error})")) .unwrap_or_default() ))) + } } fn read_android_shared_video_header_volatile( diff --git a/scripts/integration/webrtc.mjs b/scripts/integration/webrtc.mjs index f6f4dfb3..9e060c9a 100644 --- a/scripts/integration/webrtc.mjs +++ b/scripts/integration/webrtc.mjs @@ -28,6 +28,10 @@ const coreSimulatorCommandTimeoutMs = Number( const simdeckBootTimeoutMs = Number( process.env.SIMDECK_INTEGRATION_BOOT_TIMEOUT_MS ?? "300000", ); +const fixtureLaunchMaxRecoveries = Number( + process.env.SIMDECK_INTEGRATION_FIXTURE_LAUNCH_MAX_RECOVERIES ?? + (process.env.CI === "true" ? "2" : "1"), +); let simulatorUDID = ""; let serverProcess = null; @@ -188,7 +192,7 @@ async function waitForHealth() { async function launchFixtureWithRecovery(appPath, options = {}) { const recoveryCount = options.recoveryCount ?? 0; - const maxRecoveries = options.maxRecoveries ?? 1; + const maxRecoveries = options.maxRecoveries ?? fixtureLaunchMaxRecoveries; simdeckJson(["install", simulatorUDID, appPath], { timeoutMs: 60_000, From 7ea2045024935c64403ce330f279689ef204872c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 4 Jun 2026 23:00:53 -0400 Subject: [PATCH 4/5] fix: stabilize simulator integration retry --- scripts/integration/cli.mjs | 40 +++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/scripts/integration/cli.mjs b/scripts/integration/cli.mjs index 42cdd603..e244099f 100644 --- a/scripts/integration/cli.mjs +++ b/scripts/integration/cli.mjs @@ -352,8 +352,8 @@ async function main() { "CLI boot after erase", () => retrySimdeckJson(["boot", simulatorUDID], "CLI boot after erase", { - attempts: 3, - delayMs: 3_000, + attempts: process.env.CI === "true" ? 4 : 3, + delayMs: 5_000, timeoutMs: 180_000, }), { phase: phaseSimulatorLifecycle }, @@ -744,8 +744,8 @@ function startServer() { ); } -async function waitForHealth() { - const deadline = Date.now() + 30_000; +async function waitForHealth(options = {}) { + const deadline = Date.now() + (options.timeoutMs ?? 30_000); while (Date.now() < deadline) { try { const health = await httpJson("GET", "/api/health"); @@ -766,6 +766,7 @@ function simdeckJson(args, options = {}) { async function retrySimdeckJson(args, label, options = {}) { const attempts = options.attempts ?? 6; const delayMs = options.delayMs ?? 2_000; + const healthTimeoutMs = options.healthTimeoutMs ?? 60_000; let lastError; for (let attempt = 1; attempt <= attempts; attempt += 1) { try { @@ -775,6 +776,16 @@ async function retrySimdeckJson(args, label, options = {}) { if (attempt === attempts) { break; } + if (isServiceConnectionError(error)) { + logStep( + `${label} lost service connection on attempt ${attempt}; waiting for health before retry`, + ); + try { + await waitForHealth({ timeoutMs: healthTimeoutMs }); + } catch (healthError) { + lastError = healthError; + } + } await sleep(delayMs); } } @@ -786,6 +797,7 @@ async function retrySimdeckJson(args, label, options = {}) { async function retrySimdeckText(args, label, options = {}) { const attempts = options.attempts ?? 6; const delayMs = options.delayMs ?? 2_000; + const healthTimeoutMs = options.healthTimeoutMs ?? 60_000; let lastError; for (let attempt = 1; attempt <= attempts; attempt += 1) { try { @@ -795,6 +807,16 @@ async function retrySimdeckText(args, label, options = {}) { if (attempt === attempts) { break; } + if (isServiceConnectionError(error)) { + logStep( + `${label} lost service connection on attempt ${attempt}; waiting for health before retry`, + ); + try { + await waitForHealth({ timeoutMs: healthTimeoutMs }); + } catch (healthError) { + lastError = healthError; + } + } await sleep(delayMs); } } @@ -803,6 +825,16 @@ async function retrySimdeckText(args, label, options = {}) { ); } +function isServiceConnectionError(error) { + const message = String(error?.message ?? error).toLowerCase(); + return ( + message.includes("connect to simdeck service") || + message.includes("connection refused") || + message.includes("connection reset") || + message.includes("failed to connect") + ); +} + async function retrySimdeckTextUntil( args, label, From 1fcabf1e7f2da750365d78916ebacc9f76d9e62e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 4 Jun 2026 23:16:22 -0400 Subject: [PATCH 5/5] fix: retry webrtc reference screenshot in ci --- scripts/integration/webrtc.mjs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/integration/webrtc.mjs b/scripts/integration/webrtc.mjs index 9e060c9a..99a19a3e 100644 --- a/scripts/integration/webrtc.mjs +++ b/scripts/integration/webrtc.mjs @@ -102,9 +102,15 @@ async function main() { await launchFixtureWithRecovery(fixture.appPath); const screenshotPath = path.join(tempRoot, "reference.png"); - simdeckJson(["screenshot", simulatorUDID, "--output", screenshotPath], { - timeoutMs: 30_000, - }); + await retrySimdeckJson( + ["screenshot", simulatorUDID, "--output", screenshotPath], + "WebRTC reference screenshot", + { + attempts: process.env.CI === "true" ? 3 : 2, + delayMs: 5_000, + timeoutMs: process.env.CI === "true" ? 120_000 : 60_000, + }, + ); const { width, height } = pngSize(screenshotPath); console.log(`reference screenshot ${width}x${height}`);