diff --git a/Cargo.lock b/Cargo.lock index 32b8e09d4e..7074d9fcde 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1143,6 +1143,18 @@ dependencies = [ "workspace-hack", ] +[[package]] +name = "cap-camera-effects" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytemuck", + "ndarray 0.16.1", + "ort", + "tracing", + "wgpu", +] + [[package]] name = "cap-camera-ffmpeg" version = "0.1.0" @@ -1235,6 +1247,7 @@ dependencies = [ "bytes", "cap-audio", "cap-camera", + "cap-camera-effects", "cap-editor", "cap-enc-ffmpeg", "cap-export", @@ -1270,12 +1283,15 @@ dependencies = [ "md5", "nix 0.29.0", "objc", + "objc2 0.6.2", "objc2-app-kit", + "objc2-foundation 0.3.1", "opentelemetry", "opentelemetry-otlp", "opentelemetry_sdk", "parakeet-rs", "png 0.17.16", + "pollster", "posthog-rs", "rand 0.8.5", "regex", @@ -1669,6 +1685,7 @@ dependencies = [ "bezier_easing", "build-time", "bytemuck", + "cap-camera-effects", "cap-cursor-info", "cap-flags", "cap-project", @@ -5726,6 +5743,21 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308d96db8debc727c3fd9744aac51751243420e46edf401010908da7f8d5e57c" +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "ndarray" version = "0.17.2" @@ -6644,7 +6676,7 @@ version = "2.0.0-rc.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" dependencies = [ - "ndarray", + "ndarray 0.17.2", "ort-sys", "smallvec", "tracing", @@ -6731,7 +6763,7 @@ checksum = "ac2c29bb70e3b63ddfa9af7cbb66f87a200550a5f6a5ac82fabf527b270c6615" dependencies = [ "eyre", "hound", - "ndarray", + "ndarray 0.17.2", "ort", "realfft", "serde", @@ -7064,6 +7096,12 @@ dependencies = [ "windows-sys 0.61.0", ] +[[package]] +name = "pollster" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3a9f18d041e6d0e102a0a46750538147e5e8992d3b4873aaafee2520b00ce3" + [[package]] name = "portable-atomic" version = "1.11.1" diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index b3b36c52b3..d41d28fecb 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -78,6 +78,7 @@ log = "0.4.20" cap-audio = { path = "../../../crates/audio" } cap-camera = { path = "../../../crates/camera", features = ["serde", "specta"] } +cap-camera-effects = { path = "../../../crates/camera-effects" } cap-utils = { path = "../../../crates/utils" } cap-project = { path = "../../../crates/project" } cap-rendering = { path = "../../../crates/rendering" } @@ -102,6 +103,7 @@ tokio-stream = { version = "0.1.17", features = ["sync"] } md5 = "0.7.0" tokio-util = "0.7.15" wgpu.workspace = true +pollster = "0.4" bytemuck = "1.23.1" kameo = "0.17.2" tauri-plugin-sentry = "0.5.0" @@ -128,6 +130,14 @@ objc2-app-kit = { version = "0.3.0", features = [ "NSWindow", "NSResponder", "NSHapticFeedback", + "NSWorkspace", + "NSRunningApplication", +] } +objc2 = "0.6.0" +objc2-foundation = { version = "0.3.0", features = [ + "NSNotification", + "NSString", + "NSThread", ] } cocoa = "0.26.0" objc = "0.2.7" @@ -142,6 +152,7 @@ parakeet-rs = "0.3.4" windows = { workspace = true, features = [ "Win32_Foundation", "Win32_System", + "Win32_System_Power", "Win32_UI_WindowsAndMessaging", "Win32_Graphics_Gdi", ] } diff --git a/apps/desktop/src-tauri/assets/backgrounds/blue/1.jpg b/apps/desktop/src-tauri/assets/backgrounds/blue/1.jpg index 73a2468ba0..35e69ce4ec 100644 Binary files a/apps/desktop/src-tauri/assets/backgrounds/blue/1.jpg and b/apps/desktop/src-tauri/assets/backgrounds/blue/1.jpg differ diff --git a/apps/desktop/src-tauri/assets/backgrounds/blue/2.jpg b/apps/desktop/src-tauri/assets/backgrounds/blue/2.jpg index e9634ad2c4..ea288229a8 100644 Binary files a/apps/desktop/src-tauri/assets/backgrounds/blue/2.jpg and b/apps/desktop/src-tauri/assets/backgrounds/blue/2.jpg differ diff --git a/apps/desktop/src-tauri/assets/backgrounds/blue/3.jpg b/apps/desktop/src-tauri/assets/backgrounds/blue/3.jpg index 2b981089f7..e051280d5f 100644 Binary files a/apps/desktop/src-tauri/assets/backgrounds/blue/3.jpg and b/apps/desktop/src-tauri/assets/backgrounds/blue/3.jpg differ diff --git a/apps/desktop/src-tauri/assets/backgrounds/blue/4.jpg b/apps/desktop/src-tauri/assets/backgrounds/blue/4.jpg index e0f2dfdd08..ffbb9c624c 100644 Binary files a/apps/desktop/src-tauri/assets/backgrounds/blue/4.jpg and b/apps/desktop/src-tauri/assets/backgrounds/blue/4.jpg differ diff --git a/apps/desktop/src-tauri/assets/backgrounds/blue/5.jpg b/apps/desktop/src-tauri/assets/backgrounds/blue/5.jpg index a7d4cab335..69cbad5781 100644 Binary files a/apps/desktop/src-tauri/assets/backgrounds/blue/5.jpg and b/apps/desktop/src-tauri/assets/backgrounds/blue/5.jpg differ diff --git a/apps/desktop/src-tauri/assets/backgrounds/cities/liverpool.jpg b/apps/desktop/src-tauri/assets/backgrounds/cities/liverpool.jpg new file mode 100644 index 0000000000..8bd41bcb6b Binary files /dev/null and b/apps/desktop/src-tauri/assets/backgrounds/cities/liverpool.jpg differ diff --git a/apps/desktop/src-tauri/assets/backgrounds/cities/london.jpg b/apps/desktop/src-tauri/assets/backgrounds/cities/london.jpg new file mode 100644 index 0000000000..f991ddd61a Binary files /dev/null and b/apps/desktop/src-tauri/assets/backgrounds/cities/london.jpg differ diff --git a/apps/desktop/src-tauri/assets/backgrounds/cities/miami.jpg b/apps/desktop/src-tauri/assets/backgrounds/cities/miami.jpg new file mode 100644 index 0000000000..13e6d3c8d8 Binary files /dev/null and b/apps/desktop/src-tauri/assets/backgrounds/cities/miami.jpg differ diff --git a/apps/desktop/src-tauri/assets/backgrounds/cities/monaco.jpg b/apps/desktop/src-tauri/assets/backgrounds/cities/monaco.jpg new file mode 100644 index 0000000000..3d061d9d28 Binary files /dev/null and b/apps/desktop/src-tauri/assets/backgrounds/cities/monaco.jpg differ diff --git a/apps/desktop/src-tauri/assets/backgrounds/cities/nyc.jpg b/apps/desktop/src-tauri/assets/backgrounds/cities/nyc.jpg new file mode 100644 index 0000000000..dd57cd9848 Binary files /dev/null and b/apps/desktop/src-tauri/assets/backgrounds/cities/nyc.jpg differ diff --git a/apps/desktop/src-tauri/assets/backgrounds/cities/rome.jpg b/apps/desktop/src-tauri/assets/backgrounds/cities/rome.jpg new file mode 100644 index 0000000000..f8a080544d Binary files /dev/null and b/apps/desktop/src-tauri/assets/backgrounds/cities/rome.jpg differ diff --git a/apps/desktop/src-tauri/assets/backgrounds/cities/santorini.jpg b/apps/desktop/src-tauri/assets/backgrounds/cities/santorini.jpg new file mode 100644 index 0000000000..a2b9819864 Binary files /dev/null and b/apps/desktop/src-tauri/assets/backgrounds/cities/santorini.jpg differ diff --git a/apps/desktop/src-tauri/assets/backgrounds/cities/sf.jpg b/apps/desktop/src-tauri/assets/backgrounds/cities/sf.jpg new file mode 100644 index 0000000000..9547a740cf Binary files /dev/null and b/apps/desktop/src-tauri/assets/backgrounds/cities/sf.jpg differ diff --git a/apps/desktop/src-tauri/assets/backgrounds/purple/1.jpg b/apps/desktop/src-tauri/assets/backgrounds/purple/1.jpg index da0b2c847b..fae67a19f3 100644 Binary files a/apps/desktop/src-tauri/assets/backgrounds/purple/1.jpg and b/apps/desktop/src-tauri/assets/backgrounds/purple/1.jpg differ diff --git a/apps/desktop/src-tauri/assets/backgrounds/purple/2.jpg b/apps/desktop/src-tauri/assets/backgrounds/purple/2.jpg index 0a15123c0a..c8bccd31c1 100644 Binary files a/apps/desktop/src-tauri/assets/backgrounds/purple/2.jpg and b/apps/desktop/src-tauri/assets/backgrounds/purple/2.jpg differ diff --git a/apps/desktop/src-tauri/assets/backgrounds/purple/3.jpg b/apps/desktop/src-tauri/assets/backgrounds/purple/3.jpg index 188f27644f..623b50c2d2 100644 Binary files a/apps/desktop/src-tauri/assets/backgrounds/purple/3.jpg and b/apps/desktop/src-tauri/assets/backgrounds/purple/3.jpg differ diff --git a/apps/desktop/src-tauri/assets/backgrounds/purple/4.jpg b/apps/desktop/src-tauri/assets/backgrounds/purple/4.jpg index 63f3a5fb3e..01b07aeb33 100644 Binary files a/apps/desktop/src-tauri/assets/backgrounds/purple/4.jpg and b/apps/desktop/src-tauri/assets/backgrounds/purple/4.jpg differ diff --git a/apps/desktop/src-tauri/assets/backgrounds/purple/5.jpg b/apps/desktop/src-tauri/assets/backgrounds/purple/5.jpg index 2e64ecc45e..dd8367f6a1 100644 Binary files a/apps/desktop/src-tauri/assets/backgrounds/purple/5.jpg and b/apps/desktop/src-tauri/assets/backgrounds/purple/5.jpg differ diff --git a/apps/desktop/src-tauri/assets/backgrounds/purple/6.jpg b/apps/desktop/src-tauri/assets/backgrounds/purple/6.jpg index 26c75e5e5e..12de9cb52c 100644 Binary files a/apps/desktop/src-tauri/assets/backgrounds/purple/6.jpg and b/apps/desktop/src-tauri/assets/backgrounds/purple/6.jpg differ diff --git a/apps/desktop/src-tauri/src/camera.rs b/apps/desktop/src-tauri/src/camera.rs index 88ef3e84d7..7fedbc9211 100644 --- a/apps/desktop/src-tauri/src/camera.rs +++ b/apps/desktop/src-tauri/src/camera.rs @@ -51,6 +51,8 @@ pub struct CameraPreviewState { pub size: f32, pub shape: CameraPreviewShape, pub mirrored: bool, + #[serde(default)] + pub background_blur: cap_project::BackgroundBlurMode, } impl Default for CameraPreviewState { @@ -59,6 +61,7 @@ impl Default for CameraPreviewState { size: DEFAULT_CAMERA_SIZE, shape: CameraPreviewShape::default(), mirrored: false, + background_blur: cap_project::BackgroundBlurMode::Off, } } } @@ -541,6 +544,20 @@ impl InitializedCameraPreview { ..Default::default() }); + let blur_processor = match cap_camera_effects::BlurProcessor::new( + &device, + wgpu::TextureFormat::Rgba8Unorm, + ) { + Ok(bp) => { + info!("Camera background blur processor initialized"); + Some(bp) + } + Err(e) => { + warn!("Failed to initialize camera background blur: {e}"); + None + } + }; + let mut renderer = Renderer { surface: Some(surface), surface_config, @@ -555,6 +572,8 @@ impl InitializedCameraPreview { uniform_bind_group, texture: Cached::default(), aspect_ratio: Cached::default(), + blur_processor, + blur_source_texture: None, }; renderer.update_state_uniforms(default_state); @@ -563,11 +582,7 @@ impl InitializedCameraPreview { .await; renderer.reconfigure_gpu_surface(size.0, size.1); - let initial_surface = renderer.surface.as_ref().and_then(|s| { - s.get_current_texture() - .map_err(|err| error!("Error getting camera renderer surface texture: {err:?}")) - .ok() - }); + let initial_surface = renderer.acquire_surface_texture(); if let Some(surface) = initial_surface { let output_width = 5; let output_height = 5; @@ -607,6 +622,8 @@ struct Renderer { uniform_bind_group: wgpu::BindGroup, texture: Cached<(u32, u32), PreparedTexture>, aspect_ratio: Cached, + blur_processor: Option, + blur_source_texture: Option, } impl Renderer { @@ -657,9 +674,7 @@ impl Renderer { Ok(ReconfigureEvent::Resume) => { is_paused = false; while camera_rx.try_recv().is_ok() {} - if let Some(surface) = &self.surface - && let Ok(texture) = surface.get_current_texture() - { + if let Some(texture) = self.acquire_surface_texture() { let (buffer, stride) = render_solid_frame([0x11, 0x11, 0x11, 0xFF], 5, 5); PreparedTexture::init( @@ -775,13 +790,7 @@ impl Renderer { self.sync_ratio_uniform_and_resize_window_to_it(&window, &state, aspect_ratio) .await; - let surface_result = self.surface.as_ref().and_then(|s| { - s.get_current_texture() - .map_err(|err| { - error!("Error getting camera renderer surface texture: {err:?}") - }) - .ok() - }); + let surface_result = self.acquire_surface_texture(); if let Some(surface) = surface_result { let window_px = (clamp_size(state.size) as u32) * GPU_SURFACE_SCALE; let output_width = window_px.max(320).min(frame.inner.width()); @@ -805,8 +814,24 @@ impl Renderer { continue 'main_loop; } - self.texture - .get_or_init((output_width, output_height), || { + let frame_data = resampler_frame.data(0); + let frame_stride = resampler_frame.stride(0) as u32; + + let blur_mode = blur_mode_from_project(state.background_blur); + let blurred = if let Some(mode) = blur_mode { + self.run_background_blur( + frame_data, + frame_stride, + output_width, + output_height, + mode, + ) + } else { + false + }; + + let prepared = + self.texture.get_or_init((output_width, output_height), || { PreparedTexture::init( self.device.clone(), self.queue.clone(), @@ -817,12 +842,43 @@ impl Renderer { output_width, output_height, ) - }) - .render( - &surface, - resampler_frame.data(0), - resampler_frame.stride(0) as u32, + }); + + if blurred { + let blur_output = self + .blur_processor + .as_mut() + .and_then(|p| p.process_returning_output()) + .expect("blurred flag guarantees output"); + let mut encoder = self.device.create_command_encoder( + &wgpu::CommandEncoderDescriptor { + label: Some("Blur Copy"), + }, + ); + encoder.copy_texture_to_texture( + wgpu::TexelCopyTextureInfo { + texture: blur_output, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyTextureInfo { + texture: &prepared.texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::Extent3d { + width: output_width, + height: output_height, + depth_or_array_layers: 1, + }, ); + self.queue.submit(std::iter::once(encoder.finish())); + prepared.render_no_upload(&surface); + } else { + prepared.render(&surface, frame_data, frame_stride); + } surface.present(); } } @@ -873,9 +929,7 @@ impl Renderer { } Err(ReconfigureEvent::Resume) => { while camera_rx.try_recv().is_ok() {} - if let Some(surface) = &self.surface - && let Ok(texture) = surface.get_current_texture() - { + if let Some(texture) = self.acquire_surface_texture() { let (buffer, stride) = render_solid_frame([0x11, 0x11, 0x11, 0xFF], 5, 5); PreparedTexture::init( self.device.clone(), @@ -899,6 +953,73 @@ impl Renderer { } } + fn run_background_blur( + &mut self, + frame_data: &[u8], + frame_stride: u32, + width: u32, + height: u32, + mode: cap_camera_effects::BlurMode, + ) -> bool { + self.ensure_blur_source_texture(width, height); + + let (Some(src_tex), Some(processor)) = ( + self.blur_source_texture.as_ref(), + self.blur_processor.as_mut(), + ) else { + return false; + }; + + self.queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: src_tex, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + frame_data, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(frame_stride), + rows_per_image: Some(height), + }, + wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + ); + + processor.process(&self.device, &self.queue, src_tex, mode); + true + } + + fn ensure_blur_source_texture(&mut self, width: u32, height: u32) -> &wgpu::Texture { + if self + .blur_source_texture + .as_ref() + .is_none_or(|t| t.width() != width || t.height() != height) + { + self.blur_source_texture = Some(self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("Blur Source"), + size: wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::COPY_DST + | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + })); + } + self.blur_source_texture.as_ref().unwrap() + } + async fn cleanup_for_shutdown(&mut self, window: &WebviewWindow) { info!("Camera preview shutdown requested. Cleaning up..."); @@ -922,6 +1043,34 @@ impl Renderer { tokio::time::sleep(Duration::from_millis(50)).await; } + fn acquire_surface_texture(&mut self) -> Option { + let surface = self.surface.as_ref()?; + match surface.get_current_texture() { + Ok(texture) => Some(texture), + Err(wgpu::SurfaceError::Lost | wgpu::SurfaceError::Outdated) => { + tracing::warn!("Camera preview surface lost/outdated; reconfiguring"); + surface.configure(&self.device, &self.surface_config); + match surface.get_current_texture() { + Ok(texture) => Some(texture), + Err(err) => { + tracing::error!( + "Camera preview surface still failing after reconfigure: {err:?}" + ); + None + } + } + } + Err(wgpu::SurfaceError::Timeout) => { + tracing::debug!("Camera preview surface acquire timed out; skipping frame"); + None + } + Err(err) => { + tracing::error!("Error getting camera renderer surface texture: {err:?}"); + None + } + } + } + fn reconfigure_gpu_surface(&mut self, window_width: u32, window_height: u32) { self.surface_config.width = if window_width > 0 { window_width * GPU_SURFACE_SCALE @@ -1164,6 +1313,16 @@ mod tests { } } +fn blur_mode_from_project( + mode: cap_project::BackgroundBlurMode, +) -> Option { + match mode { + cap_project::BackgroundBlurMode::Off => None, + cap_project::BackgroundBlurMode::Light => Some(cap_camera_effects::BlurMode::Light), + cap_project::BackgroundBlurMode::Heavy => Some(cap_camera_effects::BlurMode::Heavy), + } +} + fn render_solid_frame(color: [u8; 4], width: u32, height: u32) -> (Vec, u32) { let pixel_count = (height * width) as usize; let buffer: Vec = color @@ -1243,6 +1402,45 @@ impl PreparedTexture { } } + pub fn render_no_upload(&self, surface: &SurfaceTexture) { + let surface_view = surface + .texture + .create_view(&wgpu::TextureViewDescriptor::default()); + + let mut encoder = self + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor { label: None }); + + { + let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: None, + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &surface_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color { + r: 0.0, + g: 0.0, + b: 0.0, + a: 0.0, + }), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + + render_pass.set_pipeline(&self.render_pipeline); + render_pass.set_bind_group(0, &self.bind_group, &[]); + render_pass.set_bind_group(1, &self.uniform_bind_group, &[]); + render_pass.draw(0..6, 0..1); + } + + self.queue.submit(Some(encoder.finish())); + } + pub fn render(&self, surface: &SurfaceTexture, buffer: &[u8], stride: u32) { let surface_view = surface .texture diff --git a/apps/desktop/src-tauri/src/camera.wgsl b/apps/desktop/src-tauri/src/camera.wgsl index 5caa8f2f18..9b985b3bd0 100644 --- a/apps/desktop/src-tauri/src/camera.wgsl +++ b/apps/desktop/src-tauri/src/camera.wgsl @@ -168,15 +168,12 @@ fn fs_main(in: VertexOutput) -> @location(0) vec4 { mask = 1.0; } - // Apply the mask with cleaner alpha handling to reduce ghosting if (mask < 0.05) { return vec4(0.0, 0.0, 0.0, 0.0); } - // Sample the camera texture let camera_color = textureSample(t_camera, s_camera, final_uv); - // Use sharper alpha cutoff to reduce ghosting let final_alpha = select(1.0, mask, mask < 0.95); return vec4(camera_color.rgb, final_alpha); } diff --git a/apps/desktop/src-tauri/src/camera_legacy.rs b/apps/desktop/src-tauri/src/camera_legacy.rs index fd01e654cf..ceb6a73a26 100644 --- a/apps/desktop/src-tauri/src/camera_legacy.rs +++ b/apps/desktop/src-tauri/src/camera_legacy.rs @@ -1,15 +1,34 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicU8, Ordering}; use std::time::Instant; use cap_recording::FFmpegVideoFrame; use flume::Sender; +use tokio::sync::watch; use tokio_util::sync::CancellationToken; use crate::frame_ws::{WSFrame, create_frame_ws}; +const WS_READBACK_PENDING: u8 = 0; +const WS_READBACK_READY_OK: u8 = 1; +const WS_READBACK_READY_ERR: u8 = 2; + +enum WsReadbackState { + Idle, + InFlight(Arc), +} + +struct WsReadback { + buffer: wgpu::Buffer, + state: WsReadbackState, +} + const WS_PREVIEW_MAX_WIDTH: u32 = 640; const WS_PREVIEW_MAX_HEIGHT: u32 = 360; -pub async fn create_camera_preview_ws() -> (Sender, u16, CancellationToken) { +pub async fn create_camera_preview_ws( + blur_rx: watch::Receiver, +) -> (Sender, u16, CancellationToken) { let (camera_tx, camera_rx) = flume::bounded::(4); let (frame_tx, _) = tokio::sync::broadcast::channel::(4); let frame_tx_clone = frame_tx.clone(); @@ -18,6 +37,9 @@ pub async fn create_camera_preview_ws() -> (Sender, u16, Cance let mut converter: Option<(Pixel, ffmpeg::software::scaling::Context)> = None; let mut reusable_frame: Option = None; + let mut blur_rx = blur_rx; + + let mut blur_state = WsBlurState::new(); while let Ok(raw_frame) = camera_rx.recv() { let mut frame = raw_frame.inner; @@ -26,6 +48,15 @@ pub async fn create_camera_preview_ws() -> (Sender, u16, Cance frame = newer.inner; } + let blur_mode = *blur_rx.borrow_and_update(); + let blur_enabled = blur_mode != cap_project::BackgroundBlurMode::Off; + let effects_mode = match blur_mode { + cap_project::BackgroundBlurMode::Off | cap_project::BackgroundBlurMode::Light => { + cap_camera_effects::BlurMode::Light + } + cap_project::BackgroundBlurMode::Heavy => cap_camera_effects::BlurMode::Heavy, + }; + let needs_convert = frame.format() != Pixel::RGBA || frame.width() > WS_PREVIEW_MAX_WIDTH || frame.height() > WS_PREVIEW_MAX_HEIGHT; @@ -73,12 +104,37 @@ pub async fn create_camera_preview_ws() -> (Sender, u16, Cance continue; } + let (data, width, height, stride) = if blur_enabled { + match blur_state.process( + out_frame.data(0), + out_frame.width(), + out_frame.height(), + out_frame.stride(0) as u32, + effects_mode, + ) { + Some(blurred) => blurred, + None => ( + std::sync::Arc::new(out_frame.data(0).to_vec()), + out_frame.width(), + out_frame.height(), + out_frame.stride(0) as u32, + ), + } + } else { + ( + std::sync::Arc::new(out_frame.data(0).to_vec()), + out_frame.width(), + out_frame.height(), + out_frame.stride(0) as u32, + ) + }; + frame_tx_clone .send(WSFrame { - data: std::sync::Arc::new(out_frame.data(0).to_vec()), - width: out_frame.width(), - height: out_frame.height(), - stride: out_frame.stride(0) as u32, + data, + width, + height, + stride, frame_number: 0, target_time_ns: 0, format: crate::frame_ws::WSFrameFormat::Rgba, @@ -86,12 +142,37 @@ pub async fn create_camera_preview_ws() -> (Sender, u16, Cance }) .ok(); } else { + let (data, width, height, stride) = if blur_enabled { + match blur_state.process( + frame.data(0), + frame.width(), + frame.height(), + frame.stride(0) as u32, + effects_mode, + ) { + Some(blurred) => blurred, + None => ( + std::sync::Arc::new(frame.data(0).to_vec()), + frame.width(), + frame.height(), + frame.stride(0) as u32, + ), + } + } else { + ( + std::sync::Arc::new(frame.data(0).to_vec()), + frame.width(), + frame.height(), + frame.stride(0) as u32, + ) + }; + frame_tx_clone .send(WSFrame { - data: std::sync::Arc::new(frame.data(0).to_vec()), - width: frame.width(), - height: frame.height(), - stride: frame.stride(0) as u32, + data, + width, + height, + stride, frame_number: 0, target_time_ns: 0, format: crate::frame_ws::WSFrameFormat::Rgba, @@ -105,3 +186,268 @@ pub async fn create_camera_preview_ws() -> (Sender, u16, Cance (camera_tx, camera_ws_port, _shutdown) } + +struct WsBlurState { + processor: Option, + init_attempted: bool, +} + +struct WsBlurResources { + device: wgpu::Device, + queue: wgpu::Queue, + processor: cap_camera_effects::BlurProcessor, + source_texture: Option<(u32, u32, wgpu::Texture)>, + readbacks: Option<(u32, u32, [WsReadback; 2])>, + current_idx: usize, +} + +impl WsBlurState { + fn new() -> Self { + Self { + processor: None, + init_attempted: false, + } + } + + fn process( + &mut self, + rgba_data: &[u8], + width: u32, + height: u32, + stride: u32, + mode: cap_camera_effects::BlurMode, + ) -> Option<(Arc>, u32, u32, u32)> { + if !self.init_attempted { + self.init_attempted = true; + self.processor = init_headless_blur(); + } + + let res = self.processor.as_mut()?; + + let src = match &res.source_texture { + Some((w, h, t)) if *w == width && *h == height => t, + _ => { + let tex = res.device.create_texture(&wgpu::TextureDescriptor { + label: Some("WS Blur Source"), + size: wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::COPY_DST + | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + res.source_texture = Some((width, height, tex)); + &res.source_texture.as_ref().unwrap().2 + } + }; + + res.queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: src, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + rgba_data, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(stride), + rows_per_image: Some(height), + }, + wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + ); + + let bytes_per_row_aligned = (width * 4 + 255) & !255; + let buf_size = (bytes_per_row_aligned * height) as u64; + + let readbacks_match = matches!( + &res.readbacks, + Some((w, h, _)) if *w == width && *h == height + ); + if !readbacks_match { + let make_buf = |label: &str| { + res.device.create_buffer(&wgpu::BufferDescriptor { + label: Some(label), + size: buf_size, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }) + }; + let readbacks = [ + WsReadback { + buffer: make_buf("WS Blur Readback 0"), + state: WsReadbackState::Idle, + }, + WsReadback { + buffer: make_buf("WS Blur Readback 1"), + state: WsReadbackState::Idle, + }, + ]; + res.readbacks = Some((width, height, readbacks)); + res.current_idx = 0; + } + + let _ = res.device.poll(wgpu::PollType::Poll); + + let current_idx = res.current_idx; + let prev_idx = 1 - current_idx; + + let prev_data = try_drain_readback( + &mut res.readbacks.as_mut().unwrap().2[prev_idx], + width, + height, + bytes_per_row_aligned, + ); + let curr_data = try_drain_readback( + &mut res.readbacks.as_mut().unwrap().2[current_idx], + width, + height, + bytes_per_row_aligned, + ); + let blurred_out = prev_data.or(curr_data); + + let issue_idx = if matches!( + res.readbacks.as_ref().unwrap().2[current_idx].state, + WsReadbackState::Idle + ) { + Some(current_idx) + } else if matches!( + res.readbacks.as_ref().unwrap().2[prev_idx].state, + WsReadbackState::Idle + ) { + Some(prev_idx) + } else { + None + }; + + if let Some(idx) = issue_idx { + let output = res.processor.process(&res.device, &res.queue, src, mode); + + let mut encoder = res + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("WS Blur Copy"), + }); + + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: output, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: &res.readbacks.as_ref().unwrap().2[idx].buffer, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(bytes_per_row_aligned), + rows_per_image: Some(height), + }, + }, + wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + ); + + res.queue.submit(std::iter::once(encoder.finish())); + + let status = Arc::new(AtomicU8::new(WS_READBACK_PENDING)); + let status_cb = status.clone(); + res.readbacks.as_ref().unwrap().2[idx] + .buffer + .slice(..) + .map_async(wgpu::MapMode::Read, move |result| { + let code = if result.is_ok() { + WS_READBACK_READY_OK + } else { + WS_READBACK_READY_ERR + }; + status_cb.store(code, Ordering::Release); + }); + + res.readbacks.as_mut().unwrap().2[idx].state = WsReadbackState::InFlight(status); + res.current_idx = 1 - idx; + } + + blurred_out.map(|out| (Arc::new(out), width, height, width * 4)) + } +} + +fn try_drain_readback( + readback: &mut WsReadback, + width: u32, + height: u32, + bytes_per_row_aligned: u32, +) -> Option> { + let WsReadbackState::InFlight(status) = &readback.state else { + return None; + }; + match status.load(Ordering::Acquire) { + WS_READBACK_READY_OK => { + let slice = readback.buffer.slice(..); + let data = slice.get_mapped_range(); + let row_bytes = (width * 4) as usize; + let mut out = Vec::with_capacity(row_bytes * height as usize); + for row in 0..height as usize { + let start = row * bytes_per_row_aligned as usize; + out.extend_from_slice(&data[start..start + row_bytes]); + } + drop(data); + readback.buffer.unmap(); + readback.state = WsReadbackState::Idle; + Some(out) + } + WS_READBACK_READY_ERR => { + readback.state = WsReadbackState::Idle; + None + } + _ => None, + } +} + +fn init_headless_blur() -> Option { + let instance = wgpu::Instance::default(); + let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions { + power_preference: wgpu::PowerPreference::LowPower, + force_fallback_adapter: false, + compatible_surface: None, + })) + .ok()?; + + let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor { + label: Some("WS Blur Device"), + required_features: wgpu::Features::empty(), + required_limits: + wgpu::Limits::downlevel_webgl2_defaults().using_resolution(adapter.limits()), + memory_hints: Default::default(), + trace: wgpu::Trace::Off, + })) + .ok()?; + + let processor = + cap_camera_effects::BlurProcessor::new(&device, wgpu::TextureFormat::Rgba8Unorm).ok()?; + + tracing::info!("WebSocket camera blur processor initialized (headless)"); + + Some(WsBlurResources { + device, + queue, + processor, + source_texture: None, + readbacks: None, + current_idx: 0, + }) +} diff --git a/apps/desktop/src-tauri/src/fake_window.rs b/apps/desktop/src-tauri/src/fake_window.rs index 3ed756fc8d..200fac8617 100644 --- a/apps/desktop/src-tauri/src/fake_window.rs +++ b/apps/desktop/src-tauri/src/fake_window.rs @@ -1,9 +1,17 @@ use cap_recording::sources::screen_capture::ScreenCaptureTarget; use scap_targets::{Display, DisplayId, Window as ScapWindow, bounds::LogicalBounds}; -use std::{collections::HashMap, sync::Arc, time::Duration}; +use std::{ + collections::HashMap, + sync::{ + Arc, Mutex, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, +}; use tauri::{AppHandle, Manager, WebviewWindow}; use tokio::{sync::RwLock, time::sleep}; -use tracing::instrument; +use tokio_util::sync::CancellationToken; +use tracing::{debug, instrument}; use crate::{App, ArcLock, RecordingState}; @@ -11,9 +19,63 @@ const RECORDING_CONTROLS_LABEL: &str = "in-progress-recording"; const RECORDING_CONTROLS_WIDTH: f64 = 320.0; const RECORDING_CONTROLS_HEIGHT: f64 = 150.0; const RECORDING_CONTROLS_OFFSET_Y: f64 = 120.0; +const TICK_INTERVAL: Duration = Duration::from_millis(50); +const DEAD_WINDOW_ERROR_THRESHOLD: u8 = 5; pub struct FakeWindowBounds(pub Arc>>>); +struct TokenEntry { + id: u64, + token: CancellationToken, +} + +#[derive(Default)] +pub struct FakeWindowListeners { + tokens: Mutex>, + next_id: AtomicU64, +} + +impl FakeWindowListeners { + fn register(&self, label: String) -> (u64, CancellationToken) { + let id = self.next_id.fetch_add(1, Ordering::Relaxed); + let token = CancellationToken::new(); + let mut guard = self.tokens.lock().unwrap_or_else(|e| e.into_inner()); + if let Some(previous) = guard.insert( + label, + TokenEntry { + id, + token: token.clone(), + }, + ) { + previous.token.cancel(); + } + (id, token) + } + + fn finish(&self, label: &str, id: u64) { + let mut guard = self.tokens.lock().unwrap_or_else(|e| e.into_inner()); + if let Some(current) = guard.get(label) + && current.id == id + { + guard.remove(label); + } + } + + pub fn cancel(&self, label: &str) { + let mut guard = self.tokens.lock().unwrap_or_else(|e| e.into_inner()); + if let Some(entry) = guard.remove(label) { + entry.token.cancel(); + } + } + + pub fn cancel_all(&self) { + let mut guard = self.tokens.lock().unwrap_or_else(|e| e.into_inner()); + for (_, entry) in guard.drain() { + entry.token.cancel(); + } + } +} + #[tauri::command] #[specta::specta] #[instrument(skip(state))] @@ -107,15 +169,40 @@ pub fn calculate_recording_controls_position_for_target( pub fn spawn_fake_window_listener(app: AppHandle, window: WebviewWindow) { window.set_ignore_cursor_events(true).ok(); - let is_recording_controls = window.label() == RECORDING_CONTROLS_LABEL; + let label = window.label().to_string(); + let is_recording_controls = label == RECORDING_CONTROLS_LABEL; + let listeners = app.state::(); + let (listener_id, token) = listeners.register(label.clone()); tokio::spawn(async move { + let listeners = app.state::(); let state = app.state::(); let mut current_display_id: Option = get_display_id_for_cursor(); let mut last_target_pos: Option<(f64, f64)> = None; + let mut consecutive_errors: u8 = 0; loop { - sleep(Duration::from_millis(1000 / 20)).await; + tokio::select! { + biased; + _ = token.cancelled() => { + debug!(window = %label, "Fake window listener cancelled"); + break; + } + _ = sleep(TICK_INTERVAL) => {} + } + + if crate::app_is_exiting(&app) { + break; + } + + if crate::power_observer::is_system_asleep() { + continue; + } + + if !app.webview_windows().contains_key(&label) { + debug!(window = %label, "Fake window listener stopping: window no longer exists"); + break; + } if is_recording_controls { let capture_target = app.state::>().try_read().ok().and_then(|s| { @@ -170,8 +257,19 @@ pub fn spawn_fake_window_listener(app: AppHandle, window: WebviewWindow) { let map = state.0.read().await; - let Some(windows) = map.get(window.label()) else { - window.set_ignore_cursor_events(true).ok(); + let Some(windows) = map.get(&label) else { + if window.set_ignore_cursor_events(true).is_err() { + consecutive_errors = consecutive_errors.saturating_add(1); + if consecutive_errors >= DEAD_WINDOW_ERROR_THRESHOLD { + debug!( + window = %label, + "Fake window listener stopping: window handle is no longer responsive" + ); + break; + } + } else { + consecutive_errors = 0; + } continue; }; @@ -180,10 +278,20 @@ pub fn spawn_fake_window_listener(app: AppHandle, window: WebviewWindow) { window.cursor_position(), window.scale_factor(), ) else { + consecutive_errors = consecutive_errors.saturating_add(1); + if consecutive_errors >= DEAD_WINDOW_ERROR_THRESHOLD { + debug!( + window = %label, + "Fake window listener stopping: repeated failures querying window state" + ); + break; + } let _ = window.set_ignore_cursor_events(true); continue; }; + consecutive_errors = 0; + let mut ignore = true; for bounds in windows.values() { @@ -215,9 +323,29 @@ pub fn spawn_fake_window_listener(app: AppHandle, window: WebviewWindow) { window.set_ignore_cursor_events(ignore).ok(); } } + + listeners.finish(&label, listener_id); + + { + let mut map = state.0.write().await; + map.remove(&label); + } }); } +pub fn cancel_fake_window_listener(app: &AppHandle, label: &str) { + if let Some(listeners) = app.try_state::() { + listeners.cancel(label); + } +} + +pub fn cancel_all_fake_window_listeners(app: &AppHandle) { + if let Some(listeners) = app.try_state::() { + listeners.cancel_all(); + } +} + pub fn init(app: &AppHandle) { app.manage(FakeWindowBounds(Default::default())); + app.manage(FakeWindowListeners::default()); } diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index ba96738df3..87d6d6edf4 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -24,6 +24,7 @@ mod panel_manager; mod permissions; mod platform; mod posthog; +mod power_observer; mod presets; mod recording; mod recording_settings; @@ -37,6 +38,7 @@ mod update_project_names; mod upload; pub mod web_api; mod window_exclusion; +mod window_position_persistence; mod windows; use audio::AppSounds; @@ -329,6 +331,7 @@ pub struct App { #[deprecated = "can be removed when native camera preview is ready"] camera_ws_sender: flume::Sender, camera_preview: CameraPreviewManager, + camera_blur_tx: tokio::sync::watch::Sender, handle: AppHandle, recording_state: RecordingState, recording_logging_handle: LoggingHandle, @@ -868,7 +871,11 @@ fn monitor_name_for_position(pos_x: f64, pos_y: f64) -> Option { .filter(|name| !name.trim().is_empty()) } -fn update_camera_window_position_settings(settings: &mut GeneralSettingsStore, x: f64, y: f64) { +pub(crate) fn update_camera_window_position_settings( + settings: &mut GeneralSettingsStore, + x: f64, + y: f64, +) { let display_id = display_id_for_position(x, y); let monitor_name = monitor_name_for_position(x, y); let position = general_settings::WindowPosition { x, y, display_id }; @@ -977,6 +984,11 @@ fn spawn_devices_snapshot_emitter(app_handle: AppHandle) { break; } + if power_observer::is_system_asleep() { + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + continue; + } + let permissions = permissions::do_permissions_check(false); let Some((cameras, microphones)) = collect_device_inventory( || app_is_exiting(&app_handle), @@ -1045,40 +1057,121 @@ fn spawn_devices_snapshot_emitter(app_handle: AppHandle) { } }); } + +fn spawn_system_resume_detector(app_handle: AppHandle) { + const TICK: Duration = Duration::from_secs(5); + const WAKE_THRESHOLD: Duration = Duration::from_secs(20); + + tokio::spawn(async move { + let mut last_instant = std::time::Instant::now(); + let mut last_system_time = std::time::SystemTime::now(); + + loop { + if app_is_exiting(&app_handle) { + break; + } + + tokio::time::sleep(TICK).await; + + if app_is_exiting(&app_handle) { + break; + } + + let now_instant = std::time::Instant::now(); + let now_system = std::time::SystemTime::now(); + + let monotonic_delta = now_instant.saturating_duration_since(last_instant); + let wall_delta = now_system + .duration_since(last_system_time) + .unwrap_or_default(); + + last_instant = now_instant; + last_system_time = now_system; + + let drift = wall_delta.saturating_sub(monotonic_delta); + let slept = drift >= WAKE_THRESHOLD || monotonic_delta >= TICK + WAKE_THRESHOLD; + + if slept { + tracing::warn!( + monotonic_ms = monotonic_delta.as_millis(), + wall_ms = wall_delta.as_millis(), + "System resume drift detected; scheduling resume recovery" + ); + + schedule_resume_recovery(app_handle.clone()); + } + } + }); +} + +pub(crate) fn schedule_resume_recovery(app_handle: AppHandle) { + tauri::async_runtime::spawn(async move { + if app_is_exiting(&app_handle) { + return; + } + + tokio::time::sleep(Duration::from_millis(500)).await; + + if app_is_exiting(&app_handle) { + return; + } + + #[cfg(target_os = "macos")] + { + let prewarmer = app_handle.state::(); + prewarmer.request(true).await; + } + + if !app_is_exiting(&app_handle) { + let _ = RequestScreenCapturePrewarm { force: true }.emit(&app_handle); + } + + if !app_is_exiting(&app_handle) { + let snapshot = get_devices_snapshot().await; + let _ = snapshot.emit(&app_handle); + } + }); +} + async fn cleanup_camera_window(app: AppHandle, session_id: u64) { if app_is_exiting(&app) { return; } let state = app.state::>(); - let mut app_state = state.write().await; - if app_is_exiting(&app) { - return; - } + let camera_feed = { + let mut app_state = state.write().await; - let current_session_id = app_state - .camera_preview - .session_id_handle() - .load(Ordering::Acquire); - - if current_session_id != session_id { - tracing::info!( - "Camera cleanup aborted: session mismatch (cleanup session {} vs current {})", - session_id, - current_session_id - ); - return; - } + if app_is_exiting(&app) { + return; + } - if app_state.camera_cleanup_done { - return; - } + let current_session_id = app_state + .camera_preview + .session_id_handle() + .load(Ordering::Acquire); + + if current_session_id != session_id { + tracing::info!( + "Camera cleanup aborted: session mismatch (cleanup session {} vs current {})", + session_id, + current_session_id + ); + return; + } + + if app_state.camera_cleanup_done { + return; + } + + app_state.camera_cleanup_done = true; + app_state.camera_preview.pause(); - app_state.camera_cleanup_done = true; - app_state.camera_preview.pause(); + if app_state.is_recording_active_or_pending() { + return; + } - if !app_state.is_recording_active_or_pending() { let has_visible_target_overlay = app.webview_windows().iter().any(|(label, window)| { label.starts_with("target-select-overlay-") && window.is_visible().unwrap_or(false) }); @@ -1099,11 +1192,21 @@ async fn cleanup_camera_window(app: AppHandle, session_id: u64) { return; } - if !has_visible_target_overlay { - let _ = app_state.camera_feed.ask(feeds::camera::RemoveInput).await; - app_state.camera_in_use = false; + if has_visible_target_overlay { + return; } - } + + app_state.camera_feed.clone() + }; + + let _ = tokio::time::timeout( + APP_EXIT_STEP_TIMEOUT, + camera_feed.ask(feeds::camera::RemoveInput), + ) + .await; + + let mut app_state = state.write().await; + app_state.camera_in_use = false; } async fn cleanup_camera_after_overlay_close(app: AppHandle, captured_session_id: u64) { @@ -1169,6 +1272,8 @@ async fn cleanup_camera_after_overlay_close(app: AppHandle, captured_session_id: } async fn cleanup_app_resources_for_exit(app: &AppHandle) { + power_observer::uninstall(app); + fake_window::cancel_all_fake_window_listeners(app); close_target_select_overlays(app); let (mic_feed, camera_feed, camera_shutdown) = { @@ -1276,6 +1381,11 @@ fn spawn_microphone_watcher(app_handle: AppHandle) { break; } + if power_observer::is_system_asleep() { + tokio::time::sleep(Duration::from_secs(1)).await; + continue; + } + let (should_check, label, is_marked) = { let guard = state.read().await; ( @@ -1342,6 +1452,11 @@ fn spawn_camera_watcher(app_handle: AppHandle) { break; } + if power_observer::is_system_asleep() { + tokio::time::sleep(Duration::from_secs(1)).await; + continue; + } + let (should_check, camera_id, is_marked) = { let guard = state.read().await; ( @@ -3246,12 +3361,16 @@ async fn set_camera_preview_state( app: MutableState<'_, App>, state: CameraPreviewState, ) -> Result<(), String> { - app.read() - .await + let app_guard = app.read().await; + let blur_mode = state.background_blur; + app_guard .camera_preview .set_state(state) .map_err(|err| format!("Error saving camera window state: {err}"))?; + app_guard.camera_blur_tx.send(blur_mode).ok(); + drop(app_guard); + Ok(()) } @@ -3312,6 +3431,24 @@ async fn refresh_camera_feed(state: MutableState<'_, App>) -> Result<(), String> Ok(()) } +#[tauri::command] +#[specta::specta] +#[instrument(skip(app, state))] +async fn destroy_camera_window(app: AppHandle, state: MutableState<'_, App>) -> Result<(), String> { + let shutdown_rx = { + let mut app_state = state.write().await; + app_state.camera_preview.begin_shutdown() + }; + + if let Some(rx) = shutdown_rx { + let _ = tokio::time::timeout(std::time::Duration::from_millis(500), rx).await; + } + + windows::cleanup_camera_window(&app, None, true, true).await; + + Ok(()) +} + #[tauri::command] #[specta::specta] #[instrument(skip(app))] @@ -3480,6 +3617,7 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { set_camera_window_position, ignore_camera_window_position, await_camera_preview_ready, + destroy_camera_window, refresh_camera_feed, captions::create_dir, captions::save_model_file, @@ -3549,7 +3687,10 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { ) .expect("Failed to export typescript bindings"); - let (camera_tx, camera_ws_port, _shutdown) = camera_legacy::create_camera_preview_ws().await; + let (camera_blur_tx, camera_blur_rx) = + tokio::sync::watch::channel(cap_project::BackgroundBlurMode::Off); + let (camera_tx, camera_ws_port, _shutdown) = + camera_legacy::create_camera_preview_ws(camera_blur_rx).await; let camera_ws_sender = camera_tx.clone(); let (mic_samples_tx, mic_samples_rx) = flume::bounded(8); @@ -3775,6 +3916,7 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { camera_ws_sender, handle: app.clone(), camera_preview, + camera_blur_tx, recording_state: RecordingState::None, recording_logging_handle, mic_feed, @@ -3822,6 +3964,9 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { spawn_mic_error_handler(app.clone(), mic_error_rx); spawn_device_watchers(app.clone()); spawn_devices_snapshot_emitter(app.clone()); + spawn_system_resume_detector(app.clone()); + power_observer::install(&app); + window_position_persistence::install(&app); tokio::spawn(check_notification_permissions(app.clone())); @@ -3963,17 +4108,28 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { let app = app.clone(); tokio::spawn(async move { let state = app.state::>(); - let app_state = &mut *state.write().await; - - app_state.camera_preview.pause(); - - let _ = - app_state.mic_feed.ask(microphone::RemoveInput).await; - let _ = app_state - .camera_feed - .ask(feeds::camera::RemoveInput) - .await; + let (mic_feed, camera_feed) = { + let mut app_state = state.write().await; + app_state.camera_preview.pause(); + ( + app_state.mic_feed.clone(), + app_state.camera_feed.clone(), + ) + }; + + let _ = tokio::time::timeout( + APP_EXIT_STEP_TIMEOUT, + mic_feed.ask(microphone::RemoveInput), + ) + .await; + let _ = tokio::time::timeout( + APP_EXIT_STEP_TIMEOUT, + camera_feed.ask(feeds::camera::RemoveInput), + ) + .await; + + let mut app_state = state.write().await; app_state.selected_mic_label = None; app_state.camera_in_use = false; }); @@ -3984,6 +4140,7 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { } } WindowEvent::Destroyed => { + fake_window::cancel_fake_window_listener(app, label); if app_is_exiting(app) { return; } @@ -4003,19 +4160,37 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { tokio::spawn(async move { let state = app.state::>(); - let app_state = &mut *state.write().await; - - if !app_state.is_recording_active_or_pending() { - let _ = - app_state.mic_feed.ask(microphone::RemoveInput).await; - let _ = app_state - .camera_feed - .ask(feeds::camera::RemoveInput) - .await; - app_state.selected_mic_label = None; - app_state.selected_camera_id = None; - app_state.camera_in_use = false; + let feeds = { + let app_state = state.read().await; + if app_state.is_recording_active_or_pending() { + None + } else { + Some(( + app_state.mic_feed.clone(), + app_state.camera_feed.clone(), + )) + } + }; + + if let Some((mic_feed, camera_feed)) = feeds { + let _ = tokio::time::timeout( + APP_EXIT_STEP_TIMEOUT, + mic_feed.ask(microphone::RemoveInput), + ) + .await; + let _ = tokio::time::timeout( + APP_EXIT_STEP_TIMEOUT, + camera_feed.ask(feeds::camera::RemoveInput), + ) + .await; + + let mut app_state = state.write().await; + if !app_state.is_recording_active_or_pending() { + app_state.selected_mic_label = None; + app_state.selected_camera_id = None; + app_state.camera_in_use = false; + } } }); } @@ -4161,27 +4336,25 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { CapWindowId::Main => { let display_id = display_id_for_position(logical_pos.x, logical_pos.y); - let _ = GeneralSettingsStore::update(app, |settings| { - settings.main_window_position = - Some(general_settings::WindowPosition { - x: logical_pos.x, - y: logical_pos.y, - display_id, - }); - }); + window_position_persistence::queue_main_position( + app, + general_settings::WindowPosition { + x: logical_pos.x, + y: logical_pos.y, + display_id, + }, + ); } CapWindowId::Camera => { let guard = app.state::(); if guard.should_ignore() { return; } - let _ = GeneralSettingsStore::update(app, |settings| { - update_camera_window_position_settings( - settings, - logical_pos.x, - logical_pos.y, - ); - }); + window_position_persistence::queue_camera_position( + app, + logical_pos.x, + logical_pos.y, + ); } _ => {} } diff --git a/apps/desktop/src-tauri/src/power_observer.rs b/apps/desktop/src-tauri/src/power_observer.rs new file mode 100644 index 0000000000..71e354470b --- /dev/null +++ b/apps/desktop/src-tauri/src/power_observer.rs @@ -0,0 +1,273 @@ +use std::sync::atomic::{AtomicBool, Ordering}; +use tauri::AppHandle; +use tracing::info; + +static SYSTEM_ASLEEP: AtomicBool = AtomicBool::new(false); + +pub fn is_system_asleep() -> bool { + SYSTEM_ASLEEP.load(Ordering::Acquire) +} + +fn mark_sleeping() { + SYSTEM_ASLEEP.store(true, Ordering::Release); +} + +fn mark_awake() { + SYSTEM_ASLEEP.store(false, Ordering::Release); +} + +pub fn on_system_will_sleep(_app: &AppHandle) { + mark_sleeping(); + info!("System going to sleep"); +} + +pub fn on_system_did_wake(app: &AppHandle) { + mark_awake(); + info!("System woke from sleep; scheduling recovery refresh"); + crate::schedule_resume_recovery(app.clone()); +} + +pub fn install(app: &AppHandle) { + #[cfg(target_os = "macos")] + macos::install(app); + + #[cfg(target_os = "windows")] + windows::install(app); + + #[cfg(not(any(target_os = "macos", target_os = "windows")))] + { + let _ = app; + } +} + +pub fn uninstall(app: &AppHandle) { + #[cfg(target_os = "macos")] + macos::uninstall(app); + + #[cfg(target_os = "windows")] + windows::uninstall(app); + + #[cfg(not(any(target_os = "macos", target_os = "windows")))] + { + let _ = app; + } +} + +#[cfg(target_os = "macos")] +mod macos { + use super::{on_system_did_wake, on_system_will_sleep}; + use objc2::{ + AnyThread, DeclaredClass, define_class, msg_send, + rc::{Retained, autoreleasepool}, + runtime::{AnyObject, NSObject}, + }; + use objc2_app_kit::NSWorkspace; + use objc2_foundation::{NSNotification, ns_string}; + use std::sync::{Mutex, OnceLock}; + use tauri::AppHandle; + use tracing::warn; + + struct ObserverIvars { + app: AppHandle, + } + + define_class!( + #[unsafe(super(NSObject))] + #[name = "CapPowerObserver"] + #[ivars = ObserverIvars] + struct CapPowerObserver; + + impl CapPowerObserver { + #[unsafe(method(handleWillSleep:))] + fn handle_will_sleep(&self, _notification: &NSNotification) { + on_system_will_sleep(&self.ivars().app); + } + + #[unsafe(method(handleDidWake:))] + fn handle_did_wake(&self, _notification: &NSNotification) { + on_system_did_wake(&self.ivars().app); + } + + #[unsafe(method(handleScreensDidSleep:))] + fn handle_screens_did_sleep(&self, _notification: &NSNotification) { + on_system_will_sleep(&self.ivars().app); + } + + #[unsafe(method(handleScreensDidWake:))] + fn handle_screens_did_wake(&self, _notification: &NSNotification) { + on_system_did_wake(&self.ivars().app); + } + } + ); + + impl CapPowerObserver { + fn new(app: AppHandle) -> Retained { + let this = Self::alloc().set_ivars(ObserverIvars { app }); + unsafe { msg_send![super(this), init] } + } + } + + static OBSERVER: OnceLock>>> = OnceLock::new(); + + fn slot() -> &'static Mutex>> { + OBSERVER.get_or_init(|| Mutex::new(None)) + } + + pub fn install(app: &AppHandle) { + let app = app.clone(); + let Err(err) = app.clone().run_on_main_thread(move || { + autoreleasepool(|_| unsafe { + let mut guard = slot().lock().unwrap_or_else(|e| e.into_inner()); + if guard.is_some() { + return; + } + + let workspace = NSWorkspace::sharedWorkspace(); + let center = workspace.notificationCenter(); + let observer = CapPowerObserver::new(app.clone()); + + let observer_obj: &AnyObject = &observer; + + center.addObserver_selector_name_object( + observer_obj, + objc2::sel!(handleWillSleep:), + Some(ns_string!("NSWorkspaceWillSleepNotification")), + None, + ); + center.addObserver_selector_name_object( + observer_obj, + objc2::sel!(handleDidWake:), + Some(ns_string!("NSWorkspaceDidWakeNotification")), + None, + ); + center.addObserver_selector_name_object( + observer_obj, + objc2::sel!(handleScreensDidSleep:), + Some(ns_string!("NSWorkspaceScreensDidSleepNotification")), + None, + ); + center.addObserver_selector_name_object( + observer_obj, + objc2::sel!(handleScreensDidWake:), + Some(ns_string!("NSWorkspaceScreensDidWakeNotification")), + None, + ); + + *guard = Some(observer); + }); + }) else { + return; + }; + warn!("Failed to install power observer on main thread: {err}"); + } + + pub fn uninstall(app: &AppHandle) { + let Err(err) = app.clone().run_on_main_thread(move || { + autoreleasepool(|_| unsafe { + let mut guard = slot().lock().unwrap_or_else(|e| e.into_inner()); + if let Some(observer) = guard.take() { + let workspace = NSWorkspace::sharedWorkspace(); + let center = workspace.notificationCenter(); + let observer_obj: &AnyObject = &observer; + center.removeObserver(observer_obj); + } + }); + }) else { + return; + }; + warn!("Failed to uninstall power observer on main thread: {err}"); + } +} + +#[cfg(target_os = "windows")] +mod windows { + use super::{on_system_did_wake, on_system_will_sleep}; + use ::windows::Win32::Foundation::{HANDLE, WIN32_ERROR}; + use ::windows::Win32::System::Power::{ + DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS, HPOWERNOTIFY, PowerRegisterSuspendResumeNotification, + PowerUnregisterSuspendResumeNotification, + }; + use ::windows::Win32::UI::WindowsAndMessaging::{ + DEVICE_NOTIFY_CALLBACK, PBT_APMRESUMEAUTOMATIC, PBT_APMSUSPEND, + }; + use std::ffi::c_void; + use std::sync::{Mutex, OnceLock}; + use tauri::AppHandle; + use tracing::warn; + + struct RegistrationHandle { + handle: HPOWERNOTIFY, + app: Box, + } + + unsafe impl Send for RegistrationHandle {} + unsafe impl Sync for RegistrationHandle {} + + static REGISTRATION: OnceLock>> = OnceLock::new(); + + fn slot() -> &'static Mutex> { + REGISTRATION.get_or_init(|| Mutex::new(None)) + } + + unsafe extern "system" fn power_callback( + context: *const c_void, + event_type: u32, + _setting: *const c_void, + ) -> u32 { + if context.is_null() { + return 0; + } + let app = unsafe { &*(context as *const AppHandle) }; + match event_type { + PBT_APMSUSPEND => on_system_will_sleep(app), + PBT_APMRESUMEAUTOMATIC => on_system_did_wake(app), + _ => {} + } + 0 + } + + pub fn install(app: &AppHandle) { + let mut guard = slot().lock().unwrap_or_else(|e| e.into_inner()); + if guard.is_some() { + return; + } + + let boxed_app = Box::new(app.clone()); + let context = Box::as_ref(&boxed_app) as *const AppHandle as *mut c_void; + + let params = DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS { + Callback: Some(power_callback), + Context: context, + }; + + let mut raw_handle: *mut c_void = std::ptr::null_mut(); + let result = unsafe { + PowerRegisterSuspendResumeNotification( + DEVICE_NOTIFY_CALLBACK, + HANDLE(¶ms as *const _ as *mut c_void), + &mut raw_handle, + ) + }; + + if result != WIN32_ERROR(0) { + warn!( + code = result.0, + "PowerRegisterSuspendResumeNotification failed" + ); + return; + } + + *guard = Some(RegistrationHandle { + handle: HPOWERNOTIFY(raw_handle as isize), + app: boxed_app, + }); + } + + pub fn uninstall(_app: &AppHandle) { + let mut guard = slot().lock().unwrap_or_else(|e| e.into_inner()); + if let Some(reg) = guard.take() { + let _ = unsafe { PowerUnregisterSuspendResumeNotification(reg.handle) }; + drop(reg.app); + } + } +} diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 4f740b9e20..251e332858 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -597,16 +597,17 @@ pub async fn start_recording( let mut app_state = state_mtx.write().await; app_state.was_camera_only_recording = true; - let current_mirrored = app_state + let (current_mirrored, current_background_blur) = app_state .camera_preview .get_state() - .map(|s| s.mirrored) - .unwrap_or(false); + .map(|s| (s.mirrored, s.background_blur)) + .unwrap_or_default(); let camera_state = CameraPreviewState { size: crate::camera::CAMERA_PRESET_LARGE, shape: CameraPreviewShape::Full, mirrored: current_mirrored, + background_blur: current_background_blur, }; if let Err(err) = app_state.camera_preview.set_state(camera_state) { @@ -2734,6 +2735,10 @@ fn project_config_from_recording( config.camera.rounding = 25.0; } } + + config.camera.background_blur = cap_project::BackgroundBlurConfig { + mode: camera_preview_state.background_blur, + }; } let timeline_segments = recordings diff --git a/apps/desktop/src-tauri/src/window_position_persistence.rs b/apps/desktop/src-tauri/src/window_position_persistence.rs new file mode 100644 index 0000000000..55b4b802fb --- /dev/null +++ b/apps/desktop/src-tauri/src/window_position_persistence.rs @@ -0,0 +1,125 @@ +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use tauri::{AppHandle, Manager}; +use tokio::sync::Notify; + +use crate::general_settings::{GeneralSettingsStore, WindowPosition}; + +#[derive(Default)] +struct PendingState { + main: Option, + camera_position: Option<(f64, f64)>, +} + +pub struct WindowPositionPersistence { + pending: Mutex, + notify: Arc, +} + +impl WindowPositionPersistence { + fn new() -> Arc { + Arc::new(Self { + pending: Mutex::new(PendingState::default()), + notify: Arc::new(Notify::new()), + }) + } + + pub fn queue_main(&self, position: WindowPosition) { + { + let mut guard = self.pending.lock().unwrap_or_else(|e| e.into_inner()); + guard.main = Some(position); + } + self.notify.notify_one(); + } + + pub fn queue_camera(&self, x: f64, y: f64) { + { + let mut guard = self.pending.lock().unwrap_or_else(|e| e.into_inner()); + guard.camera_position = Some((x, y)); + } + self.notify.notify_one(); + } + + fn take_pending(&self) -> PendingState { + let mut guard = self.pending.lock().unwrap_or_else(|e| e.into_inner()); + std::mem::take(&mut *guard) + } +} + +pub fn install(app: &AppHandle) { + let persistence = WindowPositionPersistence::new(); + app.manage(persistence.clone()); + + let app_handle = app.clone(); + let notify = persistence.notify.clone(); + tokio::spawn(async move { + const DEBOUNCE: Duration = Duration::from_millis(350); + const MIN_INTERVAL: Duration = Duration::from_millis(150); + + let mut last_flush = std::time::Instant::now() + .checked_sub(MIN_INTERVAL) + .unwrap_or_else(std::time::Instant::now); + + loop { + notify.notified().await; + + if crate::app_is_exiting(&app_handle) { + break; + } + + tokio::time::sleep(DEBOUNCE).await; + + if crate::app_is_exiting(&app_handle) { + break; + } + + let elapsed = last_flush.elapsed(); + let remaining = MIN_INTERVAL.saturating_sub(elapsed); + if !remaining.is_zero() { + tokio::time::sleep(remaining).await; + } + + let Some(persistence) = app_handle.try_state::>() else { + break; + }; + let pending = persistence.take_pending(); + + if pending.main.is_none() && pending.camera_position.is_none() { + continue; + } + + let write_app = app_handle.clone(); + let write_result = tokio::task::spawn_blocking(move || { + GeneralSettingsStore::update(&write_app, |settings| { + if let Some(main) = pending.main { + settings.main_window_position = Some(main); + } + if let Some((x, y)) = pending.camera_position { + crate::update_camera_window_position_settings(settings, x, y); + } + }) + }) + .await; + + match write_result { + Ok(Ok(())) => {} + Ok(Err(err)) => tracing::warn!("Failed to persist window position: {err}"), + Err(err) => tracing::warn!("Window position persistence task panicked: {err}"), + } + + last_flush = std::time::Instant::now(); + } + }); +} + +pub fn queue_main_position(app: &AppHandle, position: WindowPosition) { + if let Some(persistence) = app.try_state::>() { + persistence.queue_main(position); + } +} + +pub fn queue_camera_position(app: &AppHandle, x: f64, y: f64) { + if let Some(persistence) = app.try_state::>() { + persistence.queue_camera(x, y); + } +} diff --git a/apps/desktop/src-tauri/src/windows.rs b/apps/desktop/src-tauri/src/windows.rs index a717898a50..3d33e4e19b 100644 --- a/apps/desktop/src-tauri/src/windows.rs +++ b/apps/desktop/src-tauri/src/windows.rs @@ -250,7 +250,7 @@ async fn restore_main_window_inputs(app: &AppHandle) { } } -async fn cleanup_camera_window( +pub(crate) async fn cleanup_camera_window( app: &AppHandle, window: Option<&WebviewWindow>, #[allow(unused_variables)] reset_panel: bool, diff --git a/apps/desktop/src-tauri/tauri.conf.json b/apps/desktop/src-tauri/tauri.conf.json index 29432db8f4..3e67d2280a 100644 --- a/apps/desktop/src-tauri/tauri.conf.json +++ b/apps/desktop/src-tauri/tauri.conf.json @@ -49,6 +49,7 @@ "resources": { "assets/backgrounds/macOS/*": "assets/backgrounds/macOS/", "assets/backgrounds/blue/*": "assets/backgrounds/blue/", + "assets/backgrounds/cities/*": "assets/backgrounds/cities/", "assets/backgrounds/dark/*": "assets/backgrounds/dark/", "assets/backgrounds/orange/*": "assets/backgrounds/orange/", "assets/backgrounds/purple/*": "assets/backgrounds/purple/", diff --git a/apps/desktop/src/routes/(window-chrome)/settings/experimental.tsx b/apps/desktop/src/routes/(window-chrome)/settings/experimental.tsx index bfbec2f6f4..4f83f70ed4 100644 --- a/apps/desktop/src/routes/(window-chrome)/settings/experimental.tsx +++ b/apps/desktop/src/routes/(window-chrome)/settings/experimental.tsx @@ -1,4 +1,3 @@ -import { WebviewWindow } from "@tauri-apps/api/webviewWindow"; import { type } from "@tauri-apps/plugin-os"; import { createResource, Show } from "solid-js"; import { createStore } from "solid-js/store"; @@ -41,10 +40,9 @@ function Inner(props: { if (key === "enableNativeCameraPreview") { await commands.setCameraInput(null, true); try { - const cameraWindow = await WebviewWindow.getByLabel("camera"); - await cameraWindow?.close(); + await commands.destroyCameraWindow(); } catch (error) { - console.error("Failed to close camera window", error); + console.error("Failed to destroy camera window", error); } } }; diff --git a/apps/desktop/src/routes/camera.tsx b/apps/desktop/src/routes/camera.tsx index 3bb0c6be99..18c063c46c 100644 --- a/apps/desktop/src/routes/camera.tsx +++ b/apps/desktop/src/routes/camera.tsx @@ -21,7 +21,7 @@ import { Show, Suspense, } from "solid-js"; -import { createStore } from "solid-js/store"; +import { createStore, type SetStoreFunction } from "solid-js/store"; import { generalSettingsStore } from "~/store"; import { createTauriEventListener } from "~/utils/createEventListener"; import { createCameraMutation } from "~/utils/queries"; @@ -30,10 +30,12 @@ import { commands, events } from "~/utils/tauri"; import { RecordingOptionsProvider } from "./(window-chrome)/OptionsContext"; type CameraWindowShape = "round" | "square" | "full"; +type BackgroundBlurMode = "off" | "light" | "heavy"; type CameraWindowState = { size: number; shape: CameraWindowShape; mirrored: boolean; + backgroundBlur: BackgroundBlurMode | boolean; }; const CAMERA_MIN_SIZE = 150; @@ -50,8 +52,33 @@ const getCameraOnlyInitialState = (): CameraWindowState => ({ size: CAMERA_PRESET_LARGE, shape: "full", mirrored: false, + backgroundBlur: "off", }); +const BLUR_MODES: BackgroundBlurMode[] = ["off", "light", "heavy"]; + +const cycleBlurMode = ( + current: BackgroundBlurMode | boolean, +): BackgroundBlurMode => { + if (typeof current === "boolean") { + return current ? "heavy" : "light"; + } + const idx = BLUR_MODES.indexOf(current); + return BLUR_MODES[(idx + 1) % BLUR_MODES.length]; +}; + +const blurModeLabel = (mode: BackgroundBlurMode | boolean): string => { + if (typeof mode === "boolean") return mode ? "Blur" : ""; + switch (mode) { + case "light": + return "Light"; + case "heavy": + return "Heavy"; + default: + return ""; + } +}; + let ignoreMoveUntil = 0; const ignoreMoveFor = (durationMs: number) => { @@ -189,19 +216,12 @@ function NativeCameraPreviewPage(props: { disconnected: Accessor }) { size: CAMERA_DEFAULT_SIZE, shape: "round", mirrored: false, + backgroundBlur: "off" as BackgroundBlurMode, }, ), { name: "cameraWindowState" }, ); - const [isResizing, setIsResizing] = createSignal(false); - const [resizeStart, setResizeStart] = createSignal({ - size: 0, - x: 0, - y: 0, - corner: "", - }); - const applyCameraOnlyDefaults = () => { const cameraOnlyState = getCameraOnlyInitialState(); setState("size", cameraOnlyState.size); @@ -263,7 +283,17 @@ function NativeCameraPreviewPage(props: { disconnected: Accessor }) { if (clampedSize !== currentSize) { setState("size", clampedSize); } - commands.setCameraPreviewState(state); + commands.setCameraPreviewState({ + size: state.size, + shape: state.shape, + mirrored: state.mirrored, + background_blur: + (typeof state.backgroundBlur === "boolean" + ? state.backgroundBlur + ? "heavy" + : "off" + : state.backgroundBlur) ?? "off", + }); }); const [cameraPreviewReady] = createResource(() => @@ -278,60 +308,6 @@ function NativeCameraPreviewPage(props: { disconnected: Accessor }) { return 0.7 + normalized * 0.3; }; - const handleResizeStart = (corner: string) => (e: MouseEvent) => { - e.preventDefault(); - e.stopPropagation(); - setIsResizing(true); - setResizeStart({ size: state.size, x: e.clientX, y: e.clientY, corner }); - }; - - const handleResizeMove = (e: MouseEvent) => { - if (!isResizing()) return; - const start = resizeStart(); - const deltaX = e.clientX - start.x; - const deltaY = e.clientY - start.y; - - let delta = 0; - if (start.corner.includes("e") && start.corner.includes("s")) { - delta = Math.max(deltaX, deltaY); - } else if (start.corner.includes("e") && start.corner.includes("n")) { - delta = Math.max(deltaX, -deltaY); - } else if (start.corner.includes("w") && start.corner.includes("s")) { - delta = Math.max(-deltaX, deltaY); - } else if (start.corner.includes("w") && start.corner.includes("n")) { - delta = Math.max(-deltaX, -deltaY); - } else if (start.corner.includes("e")) { - delta = deltaX; - } else if (start.corner.includes("w")) { - delta = -deltaX; - } else if (start.corner.includes("s")) { - delta = deltaY; - } else if (start.corner.includes("n")) { - delta = -deltaY; - } - - const newSize = Math.max( - CAMERA_MIN_SIZE, - Math.min(CAMERA_MAX_SIZE, start.size + delta), - ); - setState("size", newSize); - }; - - const handleResizeEnd = () => { - setIsResizing(false); - }; - - createEffect(() => { - if (isResizing()) { - window.addEventListener("mousemove", handleResizeMove); - window.addEventListener("mouseup", handleResizeEnd); - onCleanup(() => { - window.removeEventListener("mousemove", handleResizeMove); - window.removeEventListener("mouseup", handleResizeEnd); - }); - } - }); - return (
}) { > + + setState("backgroundBlur", (b) => cycleBlurMode(b)) + } + > +
+ + + + {blurModeLabel(state.backgroundBlur)} + + +
+
-
-
-
-
- {/* The camera preview is rendered in Rust by wgpu */}
Loading camera...
@@ -431,6 +413,153 @@ function ControlButton( ); } +type ResizeCorner = "nw" | "ne" | "sw" | "se"; + +const RESIZE_CORNERS: readonly ResizeCorner[] = ["nw", "ne", "sw", "se"]; + +function CameraResizeHandles(props: { + state: CameraWindowState; + setState: SetStoreFunction; + toolbarHeight: number; +}) { + const [isResizing, setIsResizing] = createSignal(false); + const [activeCorner, setActiveCorner] = createSignal( + null, + ); + const [resizeStart, setResizeStart] = createSignal({ + size: 0, + x: 0, + y: 0, + corner: "nw" as ResizeCorner, + }); + + const handleResizeStart = (corner: ResizeCorner) => (e: MouseEvent) => { + if (e.button !== 0) return; + e.preventDefault(); + e.stopPropagation(); + setIsResizing(true); + setActiveCorner(corner); + setResizeStart({ + size: props.state.size, + x: e.clientX, + y: e.clientY, + corner, + }); + }; + + const handleResizeMove = (e: MouseEvent) => { + if (!isResizing()) return; + const start = resizeStart(); + const deltaX = e.clientX - start.x; + const deltaY = e.clientY - start.y; + + const hasE = start.corner.includes("e"); + const hasW = start.corner.includes("w"); + const hasS = start.corner.includes("s"); + const hasN = start.corner.includes("n"); + + const dx = hasE ? deltaX : hasW ? -deltaX : 0; + const dy = hasS ? deltaY : hasN ? -deltaY : 0; + + const delta = (hasE || hasW) && (hasN || hasS) ? Math.max(dx, dy) : dx + dy; + + const newSize = Math.max( + CAMERA_MIN_SIZE, + Math.min(CAMERA_MAX_SIZE, start.size + delta), + ); + props.setState("size", newSize); + }; + + const handleResizeEnd = () => { + setIsResizing(false); + setActiveCorner(null); + }; + + createEffect(() => { + if (!isResizing()) return; + window.addEventListener("mousemove", handleResizeMove); + window.addEventListener("mouseup", handleResizeEnd); + onCleanup(() => { + window.removeEventListener("mousemove", handleResizeMove); + window.removeEventListener("mouseup", handleResizeEnd); + }); + }); + + return ( +
+ {RESIZE_CORNERS.map((corner) => ( + + ))} +
+ ); +} + +function ResizeCornerHandle(props: { + corner: ResizeCorner; + onMouseDown: (e: MouseEvent) => void; + active: boolean; +}) { + const hitAreaClass = () => { + switch (props.corner) { + case "nw": + return "top-0 left-0 cursor-nw-resize"; + case "ne": + return "top-0 right-0 cursor-ne-resize"; + case "sw": + return "bottom-0 left-0 cursor-sw-resize"; + case "se": + return "bottom-0 right-0 cursor-se-resize"; + } + }; + + const bracketPositionClass = () => { + switch (props.corner) { + case "nw": + return "top-1.5 left-1.5 border-t-2 border-l-2 rounded-tl-[6px]"; + case "ne": + return "top-1.5 right-1.5 border-t-2 border-r-2 rounded-tr-[6px]"; + case "sw": + return "bottom-1.5 left-1.5 border-b-2 border-l-2 rounded-bl-[6px]"; + case "se": + return "bottom-1.5 right-1.5 border-b-2 border-r-2 rounded-br-[6px]"; + } + }; + + return ( +
+
+
+ ); +} + // Legacy stuff below function LegacyCameraPreviewPage(props: { disconnected: Accessor }) { @@ -444,6 +573,7 @@ function LegacyCameraPreviewPage(props: { disconnected: Accessor }) { size: CAMERA_DEFAULT_SIZE, shape: "round", mirrored: false, + backgroundBlur: "off" as BackgroundBlurMode, }, ), { name: "cameraWindowState" }, @@ -492,12 +622,18 @@ function LegacyCameraPreviewPage(props: { disconnected: Accessor }) { } }); - const [isResizing, setIsResizing] = createSignal(false); - const [resizeStart, setResizeStart] = createSignal({ - size: 0, - x: 0, - y: 0, - corner: "", + createEffect(() => { + commands.setCameraPreviewState({ + size: state.size, + shape: state.shape, + mirrored: state.mirrored, + background_blur: + (typeof state.backgroundBlur === "boolean" + ? state.backgroundBlur + ? "heavy" + : "off" + : state.backgroundBlur) ?? "off", + }); }); const [hasPositioned, setHasPositioned] = createSignal(isCameraOnlyMode()); @@ -792,60 +928,6 @@ function LegacyCameraPreviewPage(props: { disconnected: Accessor }) { return 0.7 + normalized * 0.3; }; - const handleResizeStart = (corner: string) => (e: MouseEvent) => { - e.preventDefault(); - e.stopPropagation(); - setIsResizing(true); - setResizeStart({ size: state.size, x: e.clientX, y: e.clientY, corner }); - }; - - const handleResizeMove = (e: MouseEvent) => { - if (!isResizing()) return; - const start = resizeStart(); - const deltaX = e.clientX - start.x; - const deltaY = e.clientY - start.y; - - let delta = 0; - if (start.corner.includes("e") && start.corner.includes("s")) { - delta = Math.max(deltaX, deltaY); - } else if (start.corner.includes("e") && start.corner.includes("n")) { - delta = Math.max(deltaX, -deltaY); - } else if (start.corner.includes("w") && start.corner.includes("s")) { - delta = Math.max(-deltaX, deltaY); - } else if (start.corner.includes("w") && start.corner.includes("n")) { - delta = Math.max(-deltaX, -deltaY); - } else if (start.corner.includes("e")) { - delta = deltaX; - } else if (start.corner.includes("w")) { - delta = -deltaX; - } else if (start.corner.includes("s")) { - delta = deltaY; - } else if (start.corner.includes("n")) { - delta = -deltaY; - } - - const newSize = Math.max( - CAMERA_MIN_SIZE, - Math.min(CAMERA_MAX_SIZE, start.size + delta), - ); - setState("size", newSize); - }; - - const handleResizeEnd = () => { - setIsResizing(false); - }; - - createEffect(() => { - if (isResizing()) { - window.addEventListener("mousemove", handleResizeMove); - window.addEventListener("mouseup", handleResizeEnd); - onCleanup(() => { - window.removeEventListener("mousemove", handleResizeMove); - window.removeEventListener("mouseup", handleResizeEnd); - }); - } - }); - const [_windowSize] = createResource( () => [ @@ -1016,28 +1098,35 @@ function LegacyCameraPreviewPage(props: { disconnected: Accessor }) { > + + setState("backgroundBlur", (b) => cycleBlurMode(b)) + } + > +
+ + + + {blurModeLabel(state.backgroundBlur)} + + +
+
-
-
-
-
setProject("camera", "mirror", mirror)} /> + + + options={[ + { name: "Off", value: "off" }, + { name: "Light Blur", value: "light" }, + { name: "Heavy Blur", value: "heavy" }, + ]} + optionValue="value" + optionTextValue="name" + value={ + ( + [ + { name: "Off", value: "off" }, + { name: "Light Blur", value: "light" }, + { name: "Heavy Blur", value: "heavy" }, + ] as const + ).find( + (v) => + v.value === (project.camera.backgroundBlur?.mode ?? "off"), + ) ?? { name: "Off", value: "off" } + } + onChange={(v) => { + if (v) + setProject("camera", "backgroundBlur", { + mode: v.value, + }); + }} + disallowEmptySelection + itemComponent={(props) => ( + + as={KSelect.Item} + item={props.item} + > + + {props.item.rawValue.name} + + + )} + > + + class="flex-1 text-sm text-left truncate text-[--gray-500] font-normal"> + {(state) => {state.selectedOption().name}} + + + as={(iconProps) => ( + + )} + /> + + + + as={KSelect.Content} + class={cx(topSlideAnimateClasses, "z-50")} + > + + class="overflow-y-auto max-h-32" + as={KSelect.Listbox} + /> + + + + options={CAMERA_SHAPES} diff --git a/apps/desktop/src/routes/screenshot-editor/popovers/BackgroundSettingsPopover.tsx b/apps/desktop/src/routes/screenshot-editor/popovers/BackgroundSettingsPopover.tsx index e6dab5748a..cfaf064932 100644 --- a/apps/desktop/src/routes/screenshot-editor/popovers/BackgroundSettingsPopover.tsx +++ b/apps/desktop/src/routes/screenshot-editor/popovers/BackgroundSettingsPopover.tsx @@ -93,6 +93,14 @@ const WALLPAPER_NAMES = [ "purple/4", "purple/5", "purple/6", + "cities/liverpool", + "cities/santorini", + "cities/miami", + "cities/monaco", + "cities/london", + "cities/rome", + "cities/sf", + "cities/nyc", "dark/1", "dark/2", "dark/3", @@ -117,6 +125,7 @@ const BACKGROUND_THEMES = { macOS: "macOS", dark: "Dark", blue: "Blue", + cities: "Cities", purple: "Purple", orange: "Orange", }; diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts index ee42935f04..c1f1153e32 100644 --- a/apps/desktop/src/utils/tauri.ts +++ b/apps/desktop/src/utils/tauri.ts @@ -289,6 +289,9 @@ async ignoreCameraWindowPosition(durationMs: number) : Promise { async awaitCameraPreviewReady() : Promise { return await TAURI_INVOKE("await_camera_preview_ready"); }, +async destroyCameraWindow() : Promise { + return await TAURI_INVOKE("destroy_camera_window"); +}, async refreshCameraFeed() : Promise { return await TAURI_INVOKE("refresh_camera_feed"); }, @@ -434,15 +437,17 @@ export type AudioInputLevelChange = number export type AudioMeta = { path: string; start_time?: number | null; device_id?: string | null } export type AuthSecret = { api_key: string } | { token: string; expires: number } export type AuthStore = { secret: AuthSecret; user_id: string | null; plan: Plan | null; organizations?: Organization[] } +export type BackgroundBlurConfig = { mode: BackgroundBlurMode } +export type BackgroundBlurMode = "off" | "light" | "heavy" export type BackgroundConfiguration = { source: BackgroundSource; blur: number; padding: number; rounding: number; roundingType: CornerStyle; inset: number; crop: Crop | null; shadow: number; advancedShadow: ShadowConfiguration | null; border: BorderConfiguration | null } export type BackgroundSource = { type: "wallpaper"; path: string | null } | { type: "image"; path: string | null } | { type: "color"; value: [number, number, number]; alpha?: number } | { type: "gradient"; from: [number, number, number]; to: [number, number, number]; angle?: number; noise_intensity?: number | null; noise_scale?: number | null; animated?: boolean | null; animation_speed?: number | null } export type BorderConfiguration = { enabled: boolean; width: number; color: [number, number, number]; opacity: number } -export type Camera = { hide: boolean; mirror: boolean; position: CameraPosition; size: number; zoomSize: number | null; rounding: number; shadow: number; advancedShadow: ShadowConfiguration | null; shape: CameraShape; roundingType: CornerStyle; scaleDuringZoom?: number } +export type Camera = { hide: boolean; mirror: boolean; position: CameraPosition; size: number; zoomSize: number | null; rounding: number; shadow: number; advancedShadow: ShadowConfiguration | null; shape: CameraShape; roundingType: CornerStyle; scaleDuringZoom?: number; backgroundBlur?: BackgroundBlurConfig } export type CameraFormatInfo = { width: number; height: number; frameRate: number } export type CameraInfo = { device_id: string; model_id: ModelIDType | null; display_name: string } export type CameraPosition = { x: CameraXPosition; y: CameraYPosition } export type CameraPreviewShape = "round" | "square" | "full" -export type CameraPreviewState = { size: number; shape: CameraPreviewShape; mirrored: boolean } +export type CameraPreviewState = { size: number; shape: CameraPreviewShape; mirrored: boolean; background_blur?: BackgroundBlurMode } export type CameraShape = "square" | "source" export type CameraWithFormats = { deviceId: string; displayName: string; modelId: string | null; formats: CameraFormatInfo[]; bestFormat: CameraFormatInfo | null } export type CameraXPosition = "left" | "center" | "right" diff --git a/crates/camera-effects/Cargo.toml b/crates/camera-effects/Cargo.toml new file mode 100644 index 0000000000..ba3babfb02 --- /dev/null +++ b/crates/camera-effects/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "cap-camera-effects" +version = "0.1.0" +edition = "2024" + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true +wgpu.workspace = true +tracing.workspace = true +bytemuck = { version = "1.7", features = ["derive"] } +ndarray = "0.16" +ort = { version = "2.0.0-rc.12", default-features = false, features = ["std", "ndarray", "download-binaries", "copy-dylibs", "tls-native"] } + +[target.'cfg(target_os = "macos")'.dependencies] +ort = { version = "2.0.0-rc.12", default-features = false, features = ["coreml"] } + +[target.'cfg(target_os = "windows")'.dependencies] +ort = { version = "2.0.0-rc.12", default-features = false, features = ["directml"] } diff --git a/crates/camera-effects/assets/modnet.onnx b/crates/camera-effects/assets/modnet.onnx new file mode 100644 index 0000000000..929e054b0d Binary files /dev/null and b/crates/camera-effects/assets/modnet.onnx differ diff --git a/crates/camera-effects/assets/selfie_segmentation.onnx b/crates/camera-effects/assets/selfie_segmentation.onnx new file mode 100644 index 0000000000..f926ab523d Binary files /dev/null and b/crates/camera-effects/assets/selfie_segmentation.onnx differ diff --git a/crates/camera-effects/src/blur_pipeline.rs b/crates/camera-effects/src/blur_pipeline.rs new file mode 100644 index 0000000000..20d2b9c50c --- /dev/null +++ b/crates/camera-effects/src/blur_pipeline.rs @@ -0,0 +1,390 @@ +use wgpu::util::DeviceExt; + +#[repr(C)] +#[derive(Debug, Copy, Clone, bytemuck::Pod, bytemuck::Zeroable)] +struct BlurUniforms { + direction: [f32; 2], + texel_size: [f32; 2], + intensity: f32, + _padding: f32, + _padding2: [f32; 2], +} + +pub struct BlurPipeline { + blur_pipeline: wgpu::RenderPipeline, + blur_bind_group_layout: wgpu::BindGroupLayout, + sampler: wgpu::Sampler, +} + +pub struct BlurPassInputs<'a> { + pub source: &'a wgpu::TextureView, + pub intermediate: &'a wgpu::TextureView, + pub output: &'a wgpu::TextureView, + pub width: u32, + pub height: u32, + pub intensity: f32, +} + +impl BlurPipeline { + pub fn new(device: &wgpu::Device) -> Self { + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("Gaussian Blur Shader"), + source: wgpu::ShaderSource::Wgsl(include_str!("shaders/gaussian_blur.wgsl").into()), + }); + + let blur_bind_group_layout = + device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("Blur Bind Group Layout"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + ], + }); + + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("Blur Pipeline Layout"), + bind_group_layouts: &[&blur_bind_group_layout], + push_constant_ranges: &[], + }); + + let blur_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("Blur Pipeline"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: Default::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8Unorm, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: Default::default(), + }), + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: Default::default(), + multiview: None, + cache: None, + }); + + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + address_mode_u: wgpu::AddressMode::ClampToEdge, + address_mode_v: wgpu::AddressMode::ClampToEdge, + address_mode_w: wgpu::AddressMode::ClampToEdge, + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + ..Default::default() + }); + + Self { + blur_pipeline, + blur_bind_group_layout, + sampler, + } + } + + pub fn blur_two_pass( + &self, + device: &wgpu::Device, + encoder: &mut wgpu::CommandEncoder, + inputs: BlurPassInputs<'_>, + ) { + let BlurPassInputs { + source, + intermediate, + output, + width, + height, + intensity, + } = inputs; + + let texel_size = [1.0 / width as f32, 1.0 / height as f32]; + + let h_uniforms = BlurUniforms { + direction: [1.0, 0.0], + texel_size, + intensity, + _padding: 0.0, + _padding2: [0.0, 0.0], + }; + let h_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("Blur H Uniform"), + contents: bytemuck::cast_slice(&[h_uniforms]), + usage: wgpu::BufferUsages::UNIFORM, + }); + let h_bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("Blur H Bind Group"), + layout: &self.blur_bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(source), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(&self.sampler), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: h_buffer.as_entire_binding(), + }, + ], + }); + + { + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("Blur Horizontal Pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: intermediate, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + pass.set_pipeline(&self.blur_pipeline); + pass.set_bind_group(0, &h_bind_group, &[]); + pass.draw(0..3, 0..1); + } + + let v_uniforms = BlurUniforms { + direction: [0.0, 1.0], + texel_size, + intensity, + _padding: 0.0, + _padding2: [0.0, 0.0], + }; + let v_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("Blur V Uniform"), + contents: bytemuck::cast_slice(&[v_uniforms]), + usage: wgpu::BufferUsages::UNIFORM, + }); + let v_bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("Blur V Bind Group"), + layout: &self.blur_bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(intermediate), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(&self.sampler), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: v_buffer.as_entire_binding(), + }, + ], + }); + + { + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("Blur Vertical Pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: output, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + pass.set_pipeline(&self.blur_pipeline); + pass.set_bind_group(0, &v_bind_group, &[]); + pass.draw(0..3, 0..1); + } + } +} + +pub struct CompositePipeline { + pipeline: wgpu::RenderPipeline, + bind_group_layout: wgpu::BindGroupLayout, + sampler: wgpu::Sampler, +} + +impl CompositePipeline { + pub fn new(device: &wgpu::Device, output_format: wgpu::TextureFormat) -> Self { + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("Mask Composite Shader"), + source: wgpu::ShaderSource::Wgsl(include_str!("shaders/mask_composite.wgsl").into()), + }); + + let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("Composite Bind Group Layout"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 3, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + ], + }); + + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("Composite Pipeline Layout"), + bind_group_layouts: &[&bind_group_layout], + push_constant_ranges: &[], + }); + + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("Composite Pipeline"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: Default::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + targets: &[Some(wgpu::ColorTargetState { + format: output_format, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: Default::default(), + }), + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: Default::default(), + multiview: None, + cache: None, + }); + + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + address_mode_u: wgpu::AddressMode::ClampToEdge, + address_mode_v: wgpu::AddressMode::ClampToEdge, + address_mode_w: wgpu::AddressMode::ClampToEdge, + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + ..Default::default() + }); + + Self { + pipeline, + bind_group_layout, + sampler, + } + } + + pub fn composite( + &self, + device: &wgpu::Device, + encoder: &mut wgpu::CommandEncoder, + sharp_view: &wgpu::TextureView, + blurred_view: &wgpu::TextureView, + mask_view: &wgpu::TextureView, + output_view: &wgpu::TextureView, + ) { + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("Composite Bind Group"), + layout: &self.bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(sharp_view), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(blurred_view), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::TextureView(mask_view), + }, + wgpu::BindGroupEntry { + binding: 3, + resource: wgpu::BindingResource::Sampler(&self.sampler), + }, + ], + }); + + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("Composite Pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: output_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + pass.set_pipeline(&self.pipeline); + pass.set_bind_group(0, &bind_group, &[]); + pass.draw(0..3, 0..1); + } +} diff --git a/crates/camera-effects/src/lib.rs b/crates/camera-effects/src/lib.rs new file mode 100644 index 0000000000..c876aaea1f --- /dev/null +++ b/crates/camera-effects/src/lib.rs @@ -0,0 +1,605 @@ +mod blur_pipeline; +mod segmentation; + +use std::sync::Arc; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::time::Instant; + +use blur_pipeline::{BlurPassInputs, BlurPipeline, CompositePipeline}; +use segmentation::SegmentationModel; + +const READBACK_PENDING: u8 = 0; +const READBACK_READY_OK: u8 = 1; +const READBACK_READY_ERR: u8 = 2; + +enum ReadbackState { + Idle, + InFlight(Arc), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BlurMode { + Light, + Heavy, +} + +const SEGMENTATION_SIZE: u32 = 256; +const INFERENCE_INTERVAL_MS: u64 = 66; +const EMA_ALPHA: f32 = 0.7; + +pub struct BlurProcessor { + model: SegmentationModel, + blur_pipeline: BlurPipeline, + composite_pipeline: CompositePipeline, + downsample_pipeline: DownsamplePipeline, + textures: Option, + mask_data: Vec, + smoothed_mask: Vec, + mask_scratch: Vec, + last_inference: Instant, + downsample_texture: wgpu::Texture, + downsample_view: wgpu::TextureView, + readback_buffer: wgpu::Buffer, + readback_bytes_per_row: u32, + readback_state: ReadbackState, + mask_dirty: bool, + output_generation: u64, +} + +struct DownsamplePipeline { + pipeline: wgpu::RenderPipeline, + bind_group_layout: wgpu::BindGroupLayout, + sampler: wgpu::Sampler, +} + +impl DownsamplePipeline { + fn new(device: &wgpu::Device) -> Self { + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("Downsample Shader"), + source: wgpu::ShaderSource::Wgsl(BLIT_SHADER.into()), + }); + + let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("Downsample BGL"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + ], + }); + + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("Downsample Pipeline Layout"), + bind_group_layouts: &[&bind_group_layout], + push_constant_ranges: &[], + }); + + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("Downsample Pipeline"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: Default::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8Unorm, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: Default::default(), + }), + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: Default::default(), + multiview: None, + cache: None, + }); + + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + ..Default::default() + }); + + Self { + pipeline, + bind_group_layout, + sampler, + } + } +} + +struct ProcessorTextures { + width: u32, + height: u32, + _blurred_texture: wgpu::Texture, + blurred_view: wgpu::TextureView, + _blur_intermediate: wgpu::Texture, + blur_intermediate_view: wgpu::TextureView, + mask_texture: wgpu::Texture, + mask_view: wgpu::TextureView, + output_texture: wgpu::Texture, + output_view: wgpu::TextureView, +} + +impl BlurProcessor { + pub fn new(device: &wgpu::Device, output_format: wgpu::TextureFormat) -> anyhow::Result { + let model = SegmentationModel::new()?; + let blur_pipeline = BlurPipeline::new(device); + let composite_pipeline = CompositePipeline::new(device, output_format); + let downsample_pipeline = DownsamplePipeline::new(device); + let pixel_count = (SEGMENTATION_SIZE * SEGMENTATION_SIZE) as usize; + + let downsample_texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some("Downsample 256"), + size: wgpu::Extent3d { + width: SEGMENTATION_SIZE, + height: SEGMENTATION_SIZE, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::COPY_SRC + | wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }); + let downsample_view = downsample_texture.create_view(&Default::default()); + + let readback_bytes_per_row = (SEGMENTATION_SIZE * 4).div_ceil(256) * 256; + let readback_buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("Segmentation Readback"), + size: (readback_bytes_per_row * SEGMENTATION_SIZE) as u64, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + + Ok(Self { + model, + blur_pipeline, + composite_pipeline, + downsample_pipeline, + textures: None, + mask_data: vec![0.0; pixel_count], + smoothed_mask: vec![0.0; pixel_count], + mask_scratch: vec![0.0; pixel_count], + last_inference: Instant::now() + .checked_sub(std::time::Duration::from_secs(1)) + .unwrap_or_else(Instant::now), + downsample_texture, + downsample_view, + readback_buffer, + readback_bytes_per_row, + readback_state: ReadbackState::Idle, + mask_dirty: true, + output_generation: 0, + }) + } + + pub fn output_generation(&self) -> u64 { + self.output_generation + } + + pub fn output_view(&self) -> Option<&wgpu::TextureView> { + self.textures.as_ref().map(|t| &t.output_view) + } + + pub fn process( + &mut self, + device: &wgpu::Device, + queue: &wgpu::Queue, + input_texture: &wgpu::Texture, + mode: BlurMode, + ) -> &wgpu::Texture { + let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("Background Blur Encoder"), + }); + + self.process_into_encoder(device, queue, input_texture, &mut encoder, mode); + + queue.submit(std::iter::once(encoder.finish())); + + &self + .textures + .as_ref() + .expect("textures initialized above") + .output_texture + } + + pub fn process_into_encoder( + &mut self, + device: &wgpu::Device, + queue: &wgpu::Queue, + input_texture: &wgpu::Texture, + encoder: &mut wgpu::CommandEncoder, + mode: BlurMode, + ) { + let width = input_texture.width(); + let height = input_texture.height(); + + self.ensure_textures(device, width, height); + let input_view = input_texture.create_view(&Default::default()); + + if self.last_inference.elapsed().as_millis() >= INFERENCE_INTERVAL_MS as u128 { + self.run_segmentation(device, queue, input_texture); + self.last_inference = Instant::now(); + self.mask_dirty = true; + } + + if self.mask_dirty { + self.upload_mask(queue); + self.mask_dirty = false; + } + + let textures = self.textures.as_ref().expect("textures initialized above"); + + let blur_intensity = match mode { + BlurMode::Light => 0.75, + BlurMode::Heavy => 2.0, + }; + + self.blur_pipeline.blur_two_pass( + device, + encoder, + BlurPassInputs { + source: &input_view, + intermediate: &textures.blur_intermediate_view, + output: &textures.blurred_view, + width, + height, + intensity: blur_intensity, + }, + ); + + self.composite_pipeline.composite( + device, + encoder, + &input_view, + &textures.blurred_view, + &textures.mask_view, + &textures.output_view, + ); + } + + pub fn process_returning_output(&mut self) -> Option<&wgpu::Texture> { + self.textures.as_ref().map(|t| &t.output_texture) + } + + fn ensure_textures(&mut self, device: &wgpu::Device, width: u32, height: u32) { + if let Some(t) = &self.textures + && t.width == width + && t.height == height + { + return; + } + + let create_rgba_texture = |label: &str, w: u32, h: u32, usage: wgpu::TextureUsages| { + device.create_texture(&wgpu::TextureDescriptor { + label: Some(label), + size: wgpu::Extent3d { + width: w, + height: h, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage, + view_formats: &[], + }) + }; + + let tex_usage = wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::COPY_SRC; + + let blurred = create_rgba_texture("Blurred Camera", width, height, tex_usage); + let blur_inter = create_rgba_texture("Blur Intermediate", width, height, tex_usage); + let output_texture = create_rgba_texture("Blur Output", width, height, tex_usage); + + let mask_texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some("Segmentation Mask"), + size: wgpu::Extent3d { + width: SEGMENTATION_SIZE, + height: SEGMENTATION_SIZE, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::R8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + + self.textures = Some(ProcessorTextures { + width, + height, + blurred_view: blurred.create_view(&Default::default()), + _blurred_texture: blurred, + blur_intermediate_view: blur_inter.create_view(&Default::default()), + _blur_intermediate: blur_inter, + mask_view: mask_texture.create_view(&Default::default()), + mask_texture, + output_view: output_texture.create_view(&Default::default()), + output_texture, + }); + self.output_generation = self.output_generation.wrapping_add(1); + self.mask_dirty = true; + } + + fn run_segmentation( + &mut self, + device: &wgpu::Device, + queue: &wgpu::Queue, + input_texture: &wgpu::Texture, + ) { + let rgba_256 = match self.readback_downsampled(device, queue, input_texture) { + Some(data) => data, + None => return, + }; + + match self.model.run_inference(&rgba_256) { + Ok(new_mask) => { + let pixel_count = (SEGMENTATION_SIZE * SEGMENTATION_SIZE) as usize; + if new_mask.len() >= pixel_count { + for (i, &raw) in new_mask.iter().take(pixel_count).enumerate() { + let v = refine_mask_value(raw); + self.smoothed_mask[i] = + EMA_ALPHA * v + (1.0 - EMA_ALPHA) * self.smoothed_mask[i]; + } + self.mask_data + .copy_from_slice(&self.smoothed_mask[..pixel_count]); + } + } + Err(e) => { + tracing::warn!("Segmentation inference failed: {e:#}"); + } + } + } + + fn readback_downsampled( + &mut self, + device: &wgpu::Device, + queue: &wgpu::Queue, + input_texture: &wgpu::Texture, + ) -> Option> { + let mut completed: Option> = None; + + if let ReadbackState::InFlight(status) = &self.readback_state { + let _ = device.poll(wgpu::PollType::Poll); + match status.load(Ordering::Acquire) { + READBACK_READY_OK => { + let slice = self.readback_buffer.slice(..); + let data = slice.get_mapped_range(); + let expected_row = (SEGMENTATION_SIZE * 4) as usize; + let bytes_per_row = self.readback_bytes_per_row as usize; + let mut out = Vec::with_capacity(expected_row * SEGMENTATION_SIZE as usize); + for row in 0..SEGMENTATION_SIZE as usize { + let start = row * bytes_per_row; + out.extend_from_slice(&data[start..start + expected_row]); + } + drop(data); + self.readback_buffer.unmap(); + self.readback_state = ReadbackState::Idle; + completed = Some(out); + } + READBACK_READY_ERR => { + self.readback_state = ReadbackState::Idle; + } + _ => {} + } + } + + if matches!(self.readback_state, ReadbackState::Idle) { + let input_view = input_texture.create_view(&Default::default()); + + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("Downsample BG"), + layout: &self.downsample_pipeline.bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&input_view), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(&self.downsample_pipeline.sampler), + }, + ], + }); + + let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("Downsample Encoder"), + }); + + { + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("Downsample Pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.downsample_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + pass.set_pipeline(&self.downsample_pipeline.pipeline); + pass.set_bind_group(0, &bind_group, &[]); + pass.draw(0..3, 0..1); + } + + let bytes_per_row = self.readback_bytes_per_row; + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: &self.downsample_texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: &self.readback_buffer, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(bytes_per_row), + rows_per_image: Some(SEGMENTATION_SIZE), + }, + }, + wgpu::Extent3d { + width: SEGMENTATION_SIZE, + height: SEGMENTATION_SIZE, + depth_or_array_layers: 1, + }, + ); + + queue.submit(std::iter::once(encoder.finish())); + + let status = Arc::new(AtomicU8::new(READBACK_PENDING)); + let status_cb = status.clone(); + self.readback_buffer + .slice(..) + .map_async(wgpu::MapMode::Read, move |result| { + let code = if result.is_ok() { + READBACK_READY_OK + } else { + READBACK_READY_ERR + }; + status_cb.store(code, Ordering::Release); + }); + + self.readback_state = ReadbackState::InFlight(status); + } + + completed + } + + fn upload_mask(&mut self, queue: &wgpu::Queue) { + let Some(textures) = &self.textures else { + return; + }; + + let w = SEGMENTATION_SIZE as usize; + + blur_mask_1d(&self.mask_data, &mut self.mask_scratch, w, true); + blur_mask_1d(&self.mask_scratch, &mut self.mask_data, w, false); + blur_mask_1d(&self.mask_data, &mut self.mask_scratch, w, true); + blur_mask_1d(&self.mask_scratch, &mut self.mask_data, w, false); + + let mask_u8: Vec = self + .mask_data + .iter() + .map(|&v| (v.clamp(0.0, 1.0) * 255.0) as u8) + .collect(); + + queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &textures.mask_texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + &mask_u8, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(SEGMENTATION_SIZE), + rows_per_image: Some(SEGMENTATION_SIZE), + }, + wgpu::Extent3d { + width: SEGMENTATION_SIZE, + height: SEGMENTATION_SIZE, + depth_or_array_layers: 1, + }, + ); + } +} + +fn blur_mask_1d(src: &[f32], dst: &mut [f32], width: usize, horizontal: bool) { + let kernel = [0.06136, 0.24477, 0.38774, 0.24477, 0.06136]; + let height = src.len() / width; + + for y in 0..height { + for x in 0..width { + let mut sum = 0.0; + for (ki, &weight) in kernel.iter().enumerate() { + let offset = ki as isize - 2; + let (sx, sy) = if horizontal { + ( + (x as isize + offset).clamp(0, width as isize - 1) as usize, + y, + ) + } else { + ( + x, + (y as isize + offset).clamp(0, height as isize - 1) as usize, + ) + }; + sum += src[sy * width + sx] * weight; + } + dst[y * width + x] = sum; + } + } +} + +fn refine_mask_value(raw: f32) -> f32 { + let clamped = raw.clamp(0.0, 1.0); + let shifted = (clamped - 0.5) * 6.0; + 1.0 / (1.0 + (-shifted).exp()) +} + +const BLIT_SHADER: &str = r" +@group(0) @binding(0) var src_tex: texture_2d; +@group(0) @binding(1) var src_sampler: sampler; + +struct VertexOutput { + @builtin(position) position: vec4, + @location(0) uv: vec2, +}; + +@vertex +fn vs_main(@builtin(vertex_index) vi: u32) -> VertexOutput { + var positions = array, 3>( + vec2(-1.0, -1.0), + vec2(3.0, -1.0), + vec2(-1.0, 3.0), + ); + var uvs = array, 3>( + vec2(0.0, 1.0), + vec2(2.0, 1.0), + vec2(0.0, -1.0), + ); + var out: VertexOutput; + out.position = vec4(positions[vi], 0.0, 1.0); + out.uv = uvs[vi]; + return out; +} + +@fragment +fn fs_main(in: VertexOutput) -> @location(0) vec4 { + return textureSample(src_tex, src_sampler, in.uv); +} +"; diff --git a/crates/camera-effects/src/segmentation.rs b/crates/camera-effects/src/segmentation.rs new file mode 100644 index 0000000000..471fcc6c82 --- /dev/null +++ b/crates/camera-effects/src/segmentation.rs @@ -0,0 +1,120 @@ +use anyhow::Context; +use ort::session::Session; +use ort::value::Value; + +const MODEL_BYTES: &[u8] = include_bytes!("../assets/selfie_segmentation.onnx"); +const MODEL_INPUT_SIZE: usize = 256; + +pub struct SegmentationModel { + session: Session, +} + +impl SegmentationModel { + pub fn new() -> anyhow::Result { + let session = create_session()?; + Ok(Self { session }) + } + + pub fn run_inference(&mut self, rgba_256x256: &[u8]) -> anyhow::Result> { + let channel_size = MODEL_INPUT_SIZE * MODEL_INPUT_SIZE; + let mut flat = vec![0.0f32; 3 * channel_size]; + + let (r_plane, rest) = flat.split_at_mut(channel_size); + let (g_plane, b_plane) = rest.split_at_mut(channel_size); + + for i in 0..channel_size { + let px = i * 4; + r_plane[i] = rgba_256x256[px] as f32 / 255.0; + g_plane[i] = rgba_256x256[px + 1] as f32 / 255.0; + b_plane[i] = rgba_256x256[px + 2] as f32 / 255.0; + } + + let shape: Vec = vec![1, 3, MODEL_INPUT_SIZE, MODEL_INPUT_SIZE]; + let input_value = Value::from_array((shape, flat.into_boxed_slice())) + .context("Failed to create input tensor")?; + + let outputs = self + .session + .run(ort::inputs!["pixel_values" => input_value]) + .context("ONNX inference failed")?; + + let output_value = &outputs["alphas"]; + let (_shape, raw_data) = output_value + .try_extract_tensor::() + .context("Failed to extract output tensor")?; + + Ok(raw_data.to_vec()) + } +} + +fn create_session() -> anyhow::Result { + let mut builder = Session::builder().context("Failed to create ONNX session builder")?; + + #[cfg(target_os = "macos")] + { + builder = try_register_coreml(builder); + } + + #[cfg(target_os = "windows")] + { + builder = try_register_directml(builder); + } + + let session = builder + .commit_from_memory(MODEL_BYTES) + .context("Failed to load selfie segmentation model")?; + + tracing::info!( + "Selfie segmentation model loaded, inputs: {:?}, outputs: {:?}", + session + .inputs() + .iter() + .map(|i| i.name()) + .collect::>(), + session + .outputs() + .iter() + .map(|o| o.name()) + .collect::>() + ); + + Ok(session) +} + +#[cfg(target_os = "macos")] +fn try_register_coreml( + builder: ort::session::builder::SessionBuilder, +) -> ort::session::builder::SessionBuilder { + match builder.with_execution_providers([ + ort::execution_providers::CoreMLExecutionProvider::default().build(), + ]) { + Ok(b) => { + tracing::info!("Camera background blur: CoreML execution provider registered"); + b + } + Err(e) => { + tracing::warn!("Camera background blur: CoreML EP registration failed, using CPU: {e}"); + e.recover() + } + } +} + +#[cfg(target_os = "windows")] +fn try_register_directml( + builder: ort::session::builder::SessionBuilder, +) -> ort::session::builder::SessionBuilder { + match builder.with_execution_providers([ + ort::execution_providers::DirectMLExecutionProvider::default().build(), + ]) { + Ok(b) => { + tracing::info!("Camera background blur: DirectML execution provider registered"); + b + } + Err(e) => { + tracing::warn!( + "Camera background blur: DirectML EP registration failed, using CPU: {e}" + ); + e.recover() + } + } +} diff --git a/crates/camera-effects/src/shaders/gaussian_blur.wgsl b/crates/camera-effects/src/shaders/gaussian_blur.wgsl new file mode 100644 index 0000000000..4386a8e316 --- /dev/null +++ b/crates/camera-effects/src/shaders/gaussian_blur.wgsl @@ -0,0 +1,54 @@ +struct BlurUniforms { + direction: vec2, + texel_size: vec2, + intensity: f32, + _padding: f32, + _padding2: vec2, +}; + +@group(0) @binding(0) var input_tex: texture_2d; +@group(0) @binding(1) var input_sampler: sampler; +@group(0) @binding(2) var uniforms: BlurUniforms; + +struct VertexOutput { + @builtin(position) position: vec4, + @location(0) uv: vec2, +}; + +@vertex +fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput { + var positions = array, 3>( + vec2(-1.0, -1.0), + vec2(3.0, -1.0), + vec2(-1.0, 3.0), + ); + var uvs = array, 3>( + vec2(0.0, 1.0), + vec2(2.0, 1.0), + vec2(0.0, -1.0), + ); + + var out: VertexOutput; + out.position = vec4(positions[vertex_index], 0.0, 1.0); + out.uv = uvs[vertex_index]; + return out; +} + +@fragment +fn fs_main(in: VertexOutput) -> @location(0) vec4 { + let weights = array(0.227027, 0.1945946, 0.1216216, 0.054054, 0.016216); + let offsets = array(0.0, 1.0, 2.0, 3.0, 4.0); + + let blur_scale = uniforms.intensity * 2.0; + let step = uniforms.direction * uniforms.texel_size * blur_scale; + + var color = textureSample(input_tex, input_sampler, in.uv) * weights[0]; + + for (var i = 1u; i < 5u; i = i + 1u) { + let offset = step * offsets[i]; + color += textureSample(input_tex, input_sampler, in.uv + offset) * weights[i]; + color += textureSample(input_tex, input_sampler, in.uv - offset) * weights[i]; + } + + return color; +} diff --git a/crates/camera-effects/src/shaders/mask_composite.wgsl b/crates/camera-effects/src/shaders/mask_composite.wgsl new file mode 100644 index 0000000000..d329ecd048 --- /dev/null +++ b/crates/camera-effects/src/shaders/mask_composite.wgsl @@ -0,0 +1,37 @@ +@group(0) @binding(0) var sharp_tex: texture_2d; +@group(0) @binding(1) var blurred_tex: texture_2d; +@group(0) @binding(2) var mask_tex: texture_2d; +@group(0) @binding(3) var tex_sampler: sampler; + +struct VertexOutput { + @builtin(position) position: vec4, + @location(0) uv: vec2, +}; + +@vertex +fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput { + var positions = array, 3>( + vec2(-1.0, -1.0), + vec2(3.0, -1.0), + vec2(-1.0, 3.0), + ); + var uvs = array, 3>( + vec2(0.0, 1.0), + vec2(2.0, 1.0), + vec2(0.0, -1.0), + ); + + var out: VertexOutput; + out.position = vec4(positions[vertex_index], 0.0, 1.0); + out.uv = uvs[vertex_index]; + return out; +} + +@fragment +fn fs_main(in: VertexOutput) -> @location(0) vec4 { + let sharp = textureSample(sharp_tex, tex_sampler, in.uv); + let blurred = textureSample(blurred_tex, tex_sampler, in.uv); + let mask_value = textureSample(mask_tex, tex_sampler, in.uv).r; + + return mix(blurred, sharp, mask_value); +} diff --git a/crates/project/src/configuration.rs b/crates/project/src/configuration.rs index 59b26b6b3a..e6607e5c18 100644 --- a/crates/project/src/configuration.rs +++ b/crates/project/src/configuration.rs @@ -296,6 +296,35 @@ pub struct CameraPosition { pub y: CameraYPosition, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Type, Default)] +#[serde(rename_all = "camelCase")] +pub enum BackgroundBlurMode { + #[default] + Off, + Light, + Heavy, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +#[serde(rename_all = "camelCase", default)] +pub struct BackgroundBlurConfig { + pub mode: BackgroundBlurMode, +} + +impl BackgroundBlurConfig { + pub fn is_active(&self) -> bool { + self.mode != BackgroundBlurMode::Off + } +} + +impl Default for BackgroundBlurConfig { + fn default() -> Self { + Self { + mode: BackgroundBlurMode::Off, + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize, Type)] #[serde(rename_all = "camelCase", default)] pub struct Camera { @@ -314,6 +343,8 @@ pub struct Camera { pub rounding_type: CornerStyle, #[serde(default = "Camera::default_scale_during_zoom")] pub scale_during_zoom: f32, + #[serde(default)] + pub background_blur: BackgroundBlurConfig, } #[derive(Debug, Clone, Copy, Serialize, Deserialize, Type, Default)] @@ -356,6 +387,7 @@ impl Default for Camera { shape: CameraShape::Square, rounding_type: CornerStyle::default(), scale_during_zoom: Self::default_scale_during_zoom(), + background_blur: BackgroundBlurConfig::default(), } } } diff --git a/crates/rendering/Cargo.toml b/crates/rendering/Cargo.toml index 8727008bcb..34ac9e4a2f 100644 --- a/crates/rendering/Cargo.toml +++ b/crates/rendering/Cargo.toml @@ -9,6 +9,7 @@ workspace = true [dependencies] anyhow.workspace = true bytemuck = { version = "1.7", features = ["derive"] } +cap-camera-effects = { path = "../camera-effects" } cap-flags = { path = "../flags" } cap-project = { path = "../project" } cap-video-decode = { path = "../video-decode" } diff --git a/crates/rendering/src/layers/camera.rs b/crates/rendering/src/layers/camera.rs index d7c9944b1f..e2be7f5dd5 100644 --- a/crates/rendering/src/layers/camera.rs +++ b/crates/rendering/src/layers/camera.rs @@ -18,6 +18,17 @@ pub struct CameraLayer { hidden: bool, last_recording_time: Option, yuv_converter: YuvToRgbaConverter, + blur_bind_group: Option, + blur_active: bool, + blur_cache: Option, +} + +#[derive(Clone, Copy)] +struct BlurCacheEntry { + recording_time: f32, + mode: cap_camera_effects::BlurMode, + texture_idx: usize, + output_generation: u64, } impl CameraLayer { @@ -65,6 +76,9 @@ impl CameraLayer { hidden: false, last_recording_time: None, yuv_converter, + blur_bind_group: None, + blur_active: false, + blur_cache: None, } } @@ -75,6 +89,8 @@ impl CameraLayer { uniforms: Option, frame_data: Option<(XY, &DecodedFrame, f32)>, ) { + self.blur_active = false; + let Some(uniforms) = uniforms else { self.hidden = true; return; @@ -327,6 +343,8 @@ impl CameraLayer { frame_data: Option<(XY, &DecodedFrame, f32)>, encoder: &mut wgpu::CommandEncoder, ) { + self.blur_active = false; + let Some(uniforms) = uniforms else { self.hidden = true; return; @@ -464,12 +482,75 @@ impl CameraLayer { } } + pub fn source_texture_for_blur(&self) -> Option<&wgpu::Texture> { + if self.hidden || self.last_recording_time.is_none() { + return None; + } + Some(&self.frame_textures[self.current_texture]) + } + + pub fn attach_shared_blur( + &mut self, + device: &wgpu::Device, + processor: &cap_camera_effects::BlurProcessor, + mode: cap_camera_effects::BlurMode, + ) { + if self.hidden || self.last_recording_time.is_none() { + return; + } + + let recording_time = self.last_recording_time.expect("checked above"); + let current = self.current_texture; + let processor_generation = processor.output_generation(); + + let cache_hit = matches!( + self.blur_cache, + Some(entry) + if entry.texture_idx == current + && entry.mode == mode + && (entry.recording_time - recording_time).abs() < 0.001 + && entry.output_generation == processor_generation + ) && self.blur_bind_group.is_some(); + + if cache_hit { + self.blur_active = true; + return; + } + + let Some(output_view) = processor.output_view() else { + return; + }; + + self.blur_bind_group = Some(self.pipeline.bind_group( + device, + &self.uniforms_buffer, + output_view, + )); + self.blur_cache = Some(BlurCacheEntry { + recording_time, + mode, + texture_idx: current, + output_generation: processor_generation, + }); + self.blur_active = true; + } + pub fn copy_to_texture(&mut self, _encoder: &mut wgpu::CommandEncoder) {} pub fn render(&self, pass: &mut wgpu::RenderPass<'_>) { - if !self.hidden - && let Some(bind_group) = &self.bind_groups[self.current_texture] - { + if self.hidden { + return; + } + + let bind_group = if self.blur_active { + self.blur_bind_group + .as_ref() + .or(self.bind_groups[self.current_texture].as_ref()) + } else { + self.bind_groups[self.current_texture].as_ref() + }; + + if let Some(bind_group) = bind_group { pass.set_pipeline(&self.pipeline.render_pipeline); pass.set_bind_group(0, bind_group, &[]); pass.draw(0..3, 0..1); diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index 1335c6a293..72b3f44d40 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -3407,6 +3407,8 @@ pub struct RendererLayers { text: TextLayer, captions: CaptionsLayer, keyboard: KeyboardLayer, + camera_blur_processor: Option, + camera_blur_init_failed: bool, } impl RendererLayers { @@ -3447,9 +3449,90 @@ impl RendererLayers { text: TextLayer::new(device, queue), captions: CaptionsLayer::new(device, queue), keyboard: KeyboardLayer::new(device, queue), + camera_blur_processor: None, + camera_blur_init_failed: false, } } + fn ensure_camera_blur_processor(&mut self, device: &wgpu::Device) { + if self.camera_blur_processor.is_none() && !self.camera_blur_init_failed { + match cap_camera_effects::BlurProcessor::new(device, wgpu::TextureFormat::Rgba8Unorm) { + Ok(processor) => { + self.camera_blur_processor = Some(processor); + } + Err(e) => { + tracing::warn!("Failed to init camera background blur in renderer: {e}"); + self.camera_blur_init_failed = true; + } + } + } + } + + fn run_shared_camera_blur( + &mut self, + device: &wgpu::Device, + queue: &wgpu::Queue, + mode: cap_camera_effects::BlurMode, + ) { + if self.camera.source_texture_for_blur().is_none() + && self.camera_only.source_texture_for_blur().is_none() + { + return; + } + + self.ensure_camera_blur_processor(device); + let Some(processor) = self.camera_blur_processor.as_mut() else { + return; + }; + + let source_texture = self + .camera + .source_texture_for_blur() + .or_else(|| self.camera_only.source_texture_for_blur()); + let Some(source_texture) = source_texture else { + return; + }; + + let _ = processor.process(device, queue, source_texture, mode); + + let processor: &cap_camera_effects::BlurProcessor = processor; + self.camera.attach_shared_blur(device, processor, mode); + self.camera_only.attach_shared_blur(device, processor, mode); + } + + fn run_shared_camera_blur_with_encoder( + &mut self, + device: &wgpu::Device, + queue: &wgpu::Queue, + encoder: &mut wgpu::CommandEncoder, + mode: cap_camera_effects::BlurMode, + ) { + if self.camera.source_texture_for_blur().is_none() + && self.camera_only.source_texture_for_blur().is_none() + { + return; + } + + self.ensure_camera_blur_processor(device); + let Some(processor) = self.camera_blur_processor.as_mut() else { + return; + }; + + let source_texture = self + .camera + .source_texture_for_blur() + .or_else(|| self.camera_only.source_texture_for_blur()); + let Some(source_texture) = source_texture else { + return; + }; + + processor.process_into_encoder(device, queue, source_texture, encoder, mode); + + let processor: &cap_camera_effects::BlurProcessor = processor; + self.camera.attach_shared_blur(device, processor, mode); + self.camera_only.attach_shared_blur(device, processor, mode); + } + pub fn prepare_for_video_dimensions( &mut self, device: &wgpu::Device, @@ -3537,6 +3620,10 @@ impl RendererLayers { }), ); + if let Some(mode) = blur_mode_from_config(&uniforms.project.camera.background_blur) { + self.run_shared_camera_blur(&constants.device, &constants.queue, mode); + } + self.text.prepare( &constants.device, &constants.queue, @@ -3629,6 +3716,15 @@ impl RendererLayers { encoder, ); + if let Some(mode) = blur_mode_from_config(&uniforms.project.camera.background_blur) { + self.run_shared_camera_blur_with_encoder( + &constants.device, + &constants.queue, + encoder, + mode, + ); + } + self.text.prepare( &constants.device, &constants.queue, @@ -3803,6 +3899,16 @@ async fn produce_frame( .await } +fn blur_mode_from_config( + config: &cap_project::BackgroundBlurConfig, +) -> Option { + match config.mode { + cap_project::BackgroundBlurMode::Off => None, + cap_project::BackgroundBlurMode::Light => Some(cap_camera_effects::BlurMode::Light), + cap_project::BackgroundBlurMode::Heavy => Some(cap_camera_effects::BlurMode::Heavy), + } +} + fn parse_color_component(hex_color: &str, index: usize) -> f32 { let color = hex_color.trim_start_matches('#'); diff --git a/packages/ui-solid/src/auto-imports.d.ts b/packages/ui-solid/src/auto-imports.d.ts index e5c040138b..2513f98d22 100644 --- a/packages/ui-solid/src/auto-imports.d.ts +++ b/packages/ui-solid/src/auto-imports.d.ts @@ -94,6 +94,7 @@ declare global { const IconLucideMicOff: typeof import('~icons/lucide/mic-off.jsx')['default'] const IconLucideMinus: typeof import('~icons/lucide/minus.jsx')['default'] const IconLucideMonitor: typeof import('~icons/lucide/monitor.jsx')['default'] + const IconLucidePersonStanding: typeof import('~icons/lucide/person-standing.jsx')['default'] const IconLucidePlus: typeof import('~icons/lucide/plus.jsx')['default'] const IconLucideRatio: typeof import('~icons/lucide/ratio.jsx')['default'] const IconLucideRectangleHorizontal: typeof import('~icons/lucide/rectangle-horizontal.jsx')['default']