From 5ee8692ab32ebb92055f997f0b9eacfc14f4d230 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 18 Jul 2024 22:28:47 +0100 Subject: [PATCH 1/5] fix: Add minimum_frame_interval to stream_config --- src/capturer/engine/mac/mod.rs | 69 +++++++++++++++++----------------- src/main.rs | 37 ++++++++---------- 2 files changed, 49 insertions(+), 57 deletions(-) diff --git a/src/capturer/engine/mac/mod.rs b/src/capturer/engine/mac/mod.rs index bea876f..2724bdc 100644 --- a/src/capturer/engine/mac/mod.rs +++ b/src/capturer/engine/mac/mod.rs @@ -3,20 +3,21 @@ use std::sync::mpsc; use screencapturekit::{ cm_sample_buffer::CMSampleBuffer, - sc_content_filter::{InitParams, SCContentFilter}, + sc_content_filter::{ InitParams, SCContentFilter }, sc_error_handler::StreamErrorHandler, - sc_output_handler::{SCStreamOutputType, StreamOutput}, + sc_output_handler::{ SCStreamOutputType, StreamOutput }, sc_shareable_content::SCShareableContent, sc_stream::SCStream, - sc_stream_configuration::{PixelFormat, SCStreamConfiguration}, + sc_stream_configuration::{ PixelFormat, SCStreamConfiguration }, sc_types::SCFrameStatus, }; -use screencapturekit_sys::os_types::geometry::{CGPoint, CGRect, CGSize}; +use screencapturekit_sys::os_types::geometry::{ CGPoint, CGRect, CGSize }; +use screencapturekit_sys::os_types::base::{ CMTime, CMTimeScale }; -use crate::capturer::{Area, Options, Point, Size}; -use crate::frame::{Frame, FrameType}; +use crate::capturer::{ Area, Options, Point, Size }; +use crate::frame::{ Frame, FrameType }; use crate::targets::Target; -use crate::{capturer::Resolution, targets}; +use crate::{ capturer::Resolution, targets }; mod pixelformat; @@ -66,7 +67,7 @@ impl StreamOutput for Capturer { } } self.tx.send(frame).unwrap_or(()); - }, + } _ => {} } } @@ -77,8 +78,7 @@ impl StreamOutput for Capturer { pub fn create_capturer(options: &Options, tx: mpsc::Sender) -> SCStream { // If no target is specified, capture the main display - let target = options - .target + let target = options.target .clone() .unwrap_or_else(|| Target::Display(targets::get_main_display())); @@ -87,8 +87,7 @@ pub fn create_capturer(options: &Options, tx: mpsc::Sender) -> SCStream { let params = match target { Target::Window(window) => { // Get SCWindow from window id - let sc_window = sc_shareable_content - .windows + let sc_window = sc_shareable_content.windows .into_iter() .find(|sc_win| sc_win.window_id == window.id) .unwrap(); @@ -99,8 +98,7 @@ pub fn create_capturer(options: &Options, tx: mpsc::Sender) -> SCStream { } Target::Display(display) => { // Get SCDisplay from display id - let sc_display = sc_shareable_content - .displays + let sc_display = sc_shareable_content.displays .into_iter() .find(|sc_dis| sc_dis.display_id == display.id) .unwrap(); @@ -108,17 +106,18 @@ pub fn create_capturer(options: &Options, tx: mpsc::Sender) -> SCStream { match &options.excluded_targets { None => InitParams::Display(sc_display), Some(excluded_targets) => { - let excluded_windows = sc_shareable_content - .windows + let excluded_windows = sc_shareable_content.windows .into_iter() .filter(|window| { excluded_targets .into_iter() - .find(|excluded_target| match excluded_target { - Target::Window(excluded_window) => { - excluded_window.id == window.window_id + .find(|excluded_target| { + match excluded_target { + Target::Window(excluded_window) => { + excluded_window.id == window.window_id + } + _ => false, } - _ => false, }) .is_some() }) @@ -160,21 +159,23 @@ pub fn create_capturer(options: &Options, tx: mpsc::Sender) -> SCStream { source_rect, pixel_format, shows_cursor: options.show_cursor, + minimum_frame_interval: CMTime { + value: 1, + timescale: options.fps as CMTimeScale, + epoch: 0, + flags: 0, + }, ..Default::default() }; let mut stream = SCStream::new(filter, stream_config, ErrorHandler); - stream.add_output( - Capturer::new(tx, options.output_type), - SCStreamOutputType::Screen, - ); + stream.add_output(Capturer::new(tx, options.output_type), SCStreamOutputType::Screen); stream } pub fn get_output_frame_size(options: &Options) -> [u32; 2] { - let target = options - .target + let target = options.target .clone() .unwrap_or_else(|| Target::Display(targets::get_main_display())); @@ -183,15 +184,15 @@ pub fn get_output_frame_size(options: &Options) -> [u32; 2] { // Calculate the output height & width based on the required resolution // Output width and height need to be multiplied by scale (or dpi) - let mut output_width = (source_rect.size.width as u32) * scale_factor as u32; - let mut output_height = (source_rect.size.height as u32) * scale_factor as u32; + let mut output_width = (source_rect.size.width as u32) * (scale_factor as u32); + let mut output_height = (source_rect.size.height as u32) * (scale_factor as u32); // 1200x800 match options.output_resolution { Resolution::Captured => {} _ => { - let [resolved_width, resolved_height] = options - .output_resolution - .value((source_rect.size.width as f32) / (source_rect.size.height as f32)); + let [resolved_width, resolved_height] = options.output_resolution.value( + (source_rect.size.width as f32) / (source_rect.size.height as f32) + ); // 1280 x 853 output_width = cmp::min(output_width, resolved_width); output_height = cmp::min(output_height, resolved_height); @@ -205,15 +206,13 @@ pub fn get_output_frame_size(options: &Options) -> [u32; 2] { } pub fn get_crop_area(options: &Options) -> Area { - let target = options - .target + let target = options.target .clone() .unwrap_or_else(|| Target::Display(targets::get_main_display())); let (width, height) = targets::get_target_dimensions(&target); - options - .crop_area + options.crop_area .as_ref() .map(|val| { let input_width = val.size.width + (val.size.width % 2.0); diff --git a/src/main.rs b/src/main.rs index 4446323..563b38e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,11 +1,7 @@ // This program is just a testing application // Refer to `lib.rs` for the library source code -use scap::{ - capturer::{Area, Capturer, Options, Point, Size}, - frame::Frame, - Target, -}; +use scap::{ capturer::{ Area, Capturer, Options, Point, Size }, frame::Frame, Target }; fn main() { // Check if the platform is supported @@ -24,22 +20,12 @@ fn main() { } } - // Get recording targets - let targets = scap::get_all_targets(); - println!("🎯 Targets: {:?}", targets); - - let vscode_win = targets - .into_iter() - .find(|target| match target { - Target::Display(_) => false, - Target::Window(w) => w.title.contains("Visual Studio Code"), - }) - .expect("Visual Studio Code window not found"); + // // Get recording targets + // let targets = scap::get_all_targets(); // Create Options let options = Options { fps: 60, - target: Some(vscode_win), show_cursor: true, show_highlight: true, excluded_targets: None, @@ -70,13 +56,17 @@ fn main() { Frame::YUVFrame(frame) => { println!( "Recieved YUV frame {} of width {} and height {} and pts {}", - i, frame.width, frame.height, frame.display_time + i, + frame.width, + frame.height, + frame.display_time ); } Frame::BGR0(frame) => { println!( "Received BGR0 frame of width {} and height {}", - frame.width, frame.height + frame.width, + frame.height ); } Frame::RGB(frame) => { @@ -94,19 +84,22 @@ fn main() { Frame::RGBx(frame) => { println!( "Recieved RGBx frame of width {} and height {}", - frame.width, frame.height + frame.width, + frame.height ); } Frame::XBGR(frame) => { println!( "Recieved xRGB frame of width {} and height {}", - frame.width, frame.height + frame.width, + frame.height ); } Frame::BGRx(frame) => { println!( "Recieved BGRx frame of width {} and height {}", - frame.width, frame.height + frame.width, + frame.height ); } Frame::BGRA(frame) => { From 43012cea0178dea1378727eb019517d1bacfc689 Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Tue, 23 Jul 2024 14:06:59 +0800 Subject: [PATCH 2/5] fix minimum frame interval --- src/capturer/engine/mac/mod.rs | 62 +++++++++++++++++++--------------- src/main.rs | 23 ++++++------- 2 files changed, 45 insertions(+), 40 deletions(-) diff --git a/src/capturer/engine/mac/mod.rs b/src/capturer/engine/mac/mod.rs index 2724bdc..fff3599 100644 --- a/src/capturer/engine/mac/mod.rs +++ b/src/capturer/engine/mac/mod.rs @@ -3,21 +3,21 @@ use std::sync::mpsc; use screencapturekit::{ cm_sample_buffer::CMSampleBuffer, - sc_content_filter::{ InitParams, SCContentFilter }, + sc_content_filter::{InitParams, SCContentFilter}, sc_error_handler::StreamErrorHandler, - sc_output_handler::{ SCStreamOutputType, StreamOutput }, + sc_output_handler::{SCStreamOutputType, StreamOutput}, sc_shareable_content::SCShareableContent, sc_stream::SCStream, - sc_stream_configuration::{ PixelFormat, SCStreamConfiguration }, + sc_stream_configuration::{PixelFormat, SCStreamConfiguration}, sc_types::SCFrameStatus, }; -use screencapturekit_sys::os_types::geometry::{ CGPoint, CGRect, CGSize }; -use screencapturekit_sys::os_types::base::{ CMTime, CMTimeScale }; +use screencapturekit_sys::os_types::base::{CMTime, CMTimeScale}; +use screencapturekit_sys::os_types::geometry::{CGPoint, CGRect, CGSize}; -use crate::capturer::{ Area, Options, Point, Size }; -use crate::frame::{ Frame, FrameType }; +use crate::capturer::{Area, Options, Point, Size}; +use crate::frame::{Frame, FrameType}; use crate::targets::Target; -use crate::{ capturer::Resolution, targets }; +use crate::{capturer::Resolution, targets}; mod pixelformat; @@ -67,7 +67,7 @@ impl StreamOutput for Capturer { } } self.tx.send(frame).unwrap_or(()); - } + }, _ => {} } } @@ -78,7 +78,8 @@ impl StreamOutput for Capturer { pub fn create_capturer(options: &Options, tx: mpsc::Sender) -> SCStream { // If no target is specified, capture the main display - let target = options.target + let target = options + .target .clone() .unwrap_or_else(|| Target::Display(targets::get_main_display())); @@ -87,7 +88,8 @@ pub fn create_capturer(options: &Options, tx: mpsc::Sender) -> SCStream { let params = match target { Target::Window(window) => { // Get SCWindow from window id - let sc_window = sc_shareable_content.windows + let sc_window = sc_shareable_content + .windows .into_iter() .find(|sc_win| sc_win.window_id == window.id) .unwrap(); @@ -98,7 +100,8 @@ pub fn create_capturer(options: &Options, tx: mpsc::Sender) -> SCStream { } Target::Display(display) => { // Get SCDisplay from display id - let sc_display = sc_shareable_content.displays + let sc_display = sc_shareable_content + .displays .into_iter() .find(|sc_dis| sc_dis.display_id == display.id) .unwrap(); @@ -106,18 +109,17 @@ pub fn create_capturer(options: &Options, tx: mpsc::Sender) -> SCStream { match &options.excluded_targets { None => InitParams::Display(sc_display), Some(excluded_targets) => { - let excluded_windows = sc_shareable_content.windows + let excluded_windows = sc_shareable_content + .windows .into_iter() .filter(|window| { excluded_targets .into_iter() - .find(|excluded_target| { - match excluded_target { - Target::Window(excluded_window) => { - excluded_window.id == window.window_id - } - _ => false, + .find(|excluded_target| match excluded_target { + Target::Window(excluded_window) => { + excluded_window.id == window.window_id } + _ => false, }) .is_some() }) @@ -163,19 +165,23 @@ pub fn create_capturer(options: &Options, tx: mpsc::Sender) -> SCStream { value: 1, timescale: options.fps as CMTimeScale, epoch: 0, - flags: 0, + flags: 1, }, ..Default::default() }; let mut stream = SCStream::new(filter, stream_config, ErrorHandler); - stream.add_output(Capturer::new(tx, options.output_type), SCStreamOutputType::Screen); + stream.add_output( + Capturer::new(tx, options.output_type), + SCStreamOutputType::Screen, + ); stream } pub fn get_output_frame_size(options: &Options) -> [u32; 2] { - let target = options.target + let target = options + .target .clone() .unwrap_or_else(|| Target::Display(targets::get_main_display())); @@ -190,9 +196,9 @@ pub fn get_output_frame_size(options: &Options) -> [u32; 2] { match options.output_resolution { Resolution::Captured => {} _ => { - let [resolved_width, resolved_height] = options.output_resolution.value( - (source_rect.size.width as f32) / (source_rect.size.height as f32) - ); + let [resolved_width, resolved_height] = options + .output_resolution + .value((source_rect.size.width as f32) / (source_rect.size.height as f32)); // 1280 x 853 output_width = cmp::min(output_width, resolved_width); output_height = cmp::min(output_height, resolved_height); @@ -206,13 +212,15 @@ pub fn get_output_frame_size(options: &Options) -> [u32; 2] { } pub fn get_crop_area(options: &Options) -> Area { - let target = options.target + let target = options + .target .clone() .unwrap_or_else(|| Target::Display(targets::get_main_display())); let (width, height) = targets::get_target_dimensions(&target); - options.crop_area + options + .crop_area .as_ref() .map(|val| { let input_width = val.size.width + (val.size.width % 2.0); diff --git a/src/main.rs b/src/main.rs index 563b38e..46d09f4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,11 @@ // This program is just a testing application // Refer to `lib.rs` for the library source code -use scap::{ capturer::{ Area, Capturer, Options, Point, Size }, frame::Frame, Target }; +use scap::{ + capturer::{Area, Capturer, Options, Point, Size}, + frame::Frame, + Target, +}; fn main() { // Check if the platform is supported @@ -56,17 +60,13 @@ fn main() { Frame::YUVFrame(frame) => { println!( "Recieved YUV frame {} of width {} and height {} and pts {}", - i, - frame.width, - frame.height, - frame.display_time + i, frame.width, frame.height, frame.display_time ); } Frame::BGR0(frame) => { println!( "Received BGR0 frame of width {} and height {}", - frame.width, - frame.height + frame.width, frame.height ); } Frame::RGB(frame) => { @@ -84,22 +84,19 @@ fn main() { Frame::RGBx(frame) => { println!( "Recieved RGBx frame of width {} and height {}", - frame.width, - frame.height + frame.width, frame.height ); } Frame::XBGR(frame) => { println!( "Recieved xRGB frame of width {} and height {}", - frame.width, - frame.height + frame.width, frame.height ); } Frame::BGRx(frame) => { println!( "Recieved BGRx frame of width {} and height {}", - frame.width, - frame.height + frame.width, frame.height ); } Frame::BGRA(frame) => { From baa31392b1f036380a933001eb98d70ddb3d51f5 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Tue, 23 Jul 2024 11:32:38 +0100 Subject: [PATCH 3/5] feat: Take change from brendonovich branch --- src/capturer/engine/mac/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/capturer/engine/mac/mod.rs b/src/capturer/engine/mac/mod.rs index 2724bdc..d969459 100644 --- a/src/capturer/engine/mac/mod.rs +++ b/src/capturer/engine/mac/mod.rs @@ -11,8 +11,8 @@ use screencapturekit::{ sc_stream_configuration::{ PixelFormat, SCStreamConfiguration }, sc_types::SCFrameStatus, }; -use screencapturekit_sys::os_types::geometry::{ CGPoint, CGRect, CGSize }; use screencapturekit_sys::os_types::base::{ CMTime, CMTimeScale }; +use screencapturekit_sys::os_types::geometry::{ CGPoint, CGRect, CGSize }; use crate::capturer::{ Area, Options, Point, Size }; use crate::frame::{ Frame, FrameType }; @@ -163,7 +163,7 @@ pub fn create_capturer(options: &Options, tx: mpsc::Sender) -> SCStream { value: 1, timescale: options.fps as CMTimeScale, epoch: 0, - flags: 0, + flags: 1, }, ..Default::default() }; From 5600cade51af4333e6e6afe77f6189108219b03b Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Tue, 23 Jul 2024 11:47:24 +0100 Subject: [PATCH 4/5] Revert "feat: Take change from brendonovich branch" This reverts commit baa31392b1f036380a933001eb98d70ddb3d51f5. --- src/capturer/engine/mac/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/capturer/engine/mac/mod.rs b/src/capturer/engine/mac/mod.rs index d969459..2724bdc 100644 --- a/src/capturer/engine/mac/mod.rs +++ b/src/capturer/engine/mac/mod.rs @@ -11,8 +11,8 @@ use screencapturekit::{ sc_stream_configuration::{ PixelFormat, SCStreamConfiguration }, sc_types::SCFrameStatus, }; -use screencapturekit_sys::os_types::base::{ CMTime, CMTimeScale }; use screencapturekit_sys::os_types::geometry::{ CGPoint, CGRect, CGSize }; +use screencapturekit_sys::os_types::base::{ CMTime, CMTimeScale }; use crate::capturer::{ Area, Options, Point, Size }; use crate::frame::{ Frame, FrameType }; @@ -163,7 +163,7 @@ pub fn create_capturer(options: &Options, tx: mpsc::Sender) -> SCStream { value: 1, timescale: options.fps as CMTimeScale, epoch: 0, - flags: 1, + flags: 0, }, ..Default::default() }; From fc71e6c6c0927342028a6d0fccbc38f60be54bbc Mon Sep 17 00:00:00 2001 From: PJ Essien Date: Tue, 23 Jul 2024 22:44:38 +0100 Subject: [PATCH 5/5] yield empty frames if the display hasn't changed --- src/capturer/engine/mac/mod.rs | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/src/capturer/engine/mac/mod.rs b/src/capturer/engine/mac/mod.rs index fff3599..9876ef4 100644 --- a/src/capturer/engine/mac/mod.rs +++ b/src/capturer/engine/mac/mod.rs @@ -14,7 +14,7 @@ use screencapturekit::{ use screencapturekit_sys::os_types::base::{CMTime, CMTimeScale}; use screencapturekit_sys::os_types::geometry::{CGPoint, CGRect, CGSize}; -use crate::capturer::{Area, Options, Point, Size}; +use crate::{capturer::{Area, Options, Point, Size}, frame::BGRAFrame}; use crate::frame::{Frame, FrameType}; use crate::targets::Target; use crate::{capturer::Resolution, targets}; @@ -46,28 +46,43 @@ impl StreamOutput for Capturer { let frame_status = &sample.frame_status; match frame_status { - SCFrameStatus::Complete => unsafe { - let frame; - match self.output_type { + SCFrameStatus::Complete | SCFrameStatus::Started => unsafe { + let frame = match self.output_type { FrameType::YUVFrame => { let yuvframe = pixelformat::create_yuv_frame(sample).unwrap(); - frame = Frame::YUVFrame(yuvframe); + Frame::YUVFrame(yuvframe) } FrameType::RGB => { let rgbframe = pixelformat::create_rgb_frame(sample).unwrap(); - frame = Frame::RGB(rgbframe); + Frame::RGB(rgbframe) } FrameType::BGR0 => { let bgrframe = pixelformat::create_bgr_frame(sample).unwrap(); - frame = Frame::BGR0(bgrframe); + Frame::BGR0(bgrframe) } FrameType::BGRAFrame => { let bgraframe = pixelformat::create_bgra_frame(sample).unwrap(); - frame = Frame::BGRA(bgraframe); + Frame::BGRA(bgraframe) } - } + }; self.tx.send(frame).unwrap_or(()); }, + SCFrameStatus::Idle => { + // Quick hack - just send an empty frame, and the caller can figure out how to handle it + match self.output_type { + FrameType::BGRAFrame => { + let display_time = sample.sys_ref.get_presentation_timestamp().value as u64; + let frame = BGRAFrame { + display_time, + width: 0, + height: 0, + data: vec![], + }; + self.tx.send(Frame::BGRA(frame)).unwrap_or(()); + }, + _ => {} + } + }, _ => {} } }