From 9406684d04fcc14e4464a06fdf960b0e1712c5ce Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Sat, 1 Feb 2025 18:47:42 +0800 Subject: [PATCH] cleanup rendering structs --- apps/desktop/src-tauri/src/recording.rs | 2 +- apps/desktop/src/utils/tauri.ts | 2 +- crates/editor/src/editor.rs | 29 ++++------- crates/editor/src/editor_instance.rs | 24 ++++------ crates/editor/src/playback.rs | 33 ++++++------- crates/export/src/lib.rs | 9 ++-- crates/media/src/feeds/audio.rs | 57 ++++++++++------------ crates/project/src/configuration.rs | 16 ++++--- crates/rendering/src/lib.rs | 64 +++++++++++-------------- 9 files changed, 99 insertions(+), 137 deletions(-) diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 9418255c..a4c00865 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -395,7 +395,7 @@ fn project_config_from_recording( .iter() .enumerate() .map(|(i, segment)| TimelineSegment { - recording_segment: Some(i as u32), + recording_segment: i as u32, start: 0.0, end: segment.duration(), timescale: 1.0, diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts index 96de5b35..84d27b61 100644 --- a/apps/desktop/src/utils/tauri.ts +++ b/apps/desktop/src/utils/tauri.ts @@ -293,7 +293,7 @@ export type SharingMeta = { id: string; link: string } export type ShowCapWindow = "Setup" | "Main" | { Settings: { page: string | null } } | { Editor: { project_id: string } } | "PrevRecordings" | "WindowCaptureOccluder" | { CaptureArea: { screen: CaptureScreen } } | { Camera: { ws_port: number } } | { InProgressRecording: { position: [number, number] | null } } | "Upgrade" | "SignIn" export type SingleSegment = { display: Display; camera?: CameraMeta | null; audio?: AudioMeta | null; cursor?: string | null } export type TimelineConfiguration = { segments: TimelineSegment[]; zoomSegments: ZoomSegment[] } -export type TimelineSegment = { recordingSegment: number | null; timescale: number; start: number; end: number } +export type TimelineSegment = { recordingSegment?: number; timescale: number; start: number; end: number } export type UploadMode = { Initial: { pre_created_video: PreCreatedVideo | null } } | "Reupload" export type UploadProgress = { progress: number; message: string } export type UploadResult = { Success: string } | "NotAuthenticated" | "PlanCheckFailed" | "UpgradeRequired" diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index 8fd2aa6b..6bb29e48 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -3,7 +3,8 @@ use std::{sync::Arc, time::Instant}; use cap_media::{feeds::RawCameraFrame, frame_ws::WSFrame}; use cap_project::{BackgroundSource, RecordingMeta, XY}; use cap_rendering::{ - decoder::DecodedFrame, produce_frame, ProjectRecordings, ProjectUniforms, RenderVideoConstants, + decoder::DecodedFrame, produce_frame, DecodedSegmentFrames, ProjectRecordings, ProjectUniforms, + RenderVideoConstants, }; use tokio::{ sync::{mpsc, oneshot}, @@ -12,11 +13,9 @@ use tokio::{ pub enum RendererMessage { RenderFrame { - screen_frame: DecodedFrame, - camera_frame: Option, + segment_frames: DecodedSegmentFrames, background: BackgroundSource, uniforms: ProjectUniforms, - time: f32, // Add this field finished: oneshot::Sender<()>, resolution_base: XY, }, @@ -47,9 +46,7 @@ impl Renderer { // Check camera duration if it exists if let Some(camera_path) = meta.content.camera_path() { - if let Ok(camera_duration) = - recordings.get_source_duration(&meta.path(&camera_path)) - { + if let Ok(camera_duration) = recordings.get_source_duration(&meta.path(&camera_path)) { max_duration = max_duration.max(camera_duration); } } @@ -77,11 +74,9 @@ impl Renderer { while let Some(msg) = self.rx.recv().await { match msg { RendererMessage::RenderFrame { - screen_frame, - camera_frame, + segment_frames, background, uniforms, - time, finished, resolution_base, } => { @@ -95,17 +90,13 @@ impl Renderer { let render_constants = self.render_constants.clone(); let frame_tx = self.frame_tx.clone(); - let total_frames = self.total_frames; frame_task = Some(tokio::spawn(async move { let frame = produce_frame( &render_constants, - &screen_frame, - &camera_frame, + segment_frames, cap_rendering::Background::from(background), &uniforms, - time, - total_frames, resolution_base, ) .await @@ -145,21 +136,17 @@ impl RendererHandle { pub async fn render_frame( &self, - screen_frame: DecodedFrame, - camera_frame: Option, + segment_frames: DecodedSegmentFrames, background: BackgroundSource, uniforms: ProjectUniforms, - time: f32, resolution_base: XY, ) { let (finished_tx, finished_rx) = oneshot::channel(); self.send(RendererMessage::RenderFrame { - screen_frame, - camera_frame, + segment_frames, background, uniforms, - time, // Pass the time finished: finished_tx, resolution_base, }) diff --git a/crates/editor/src/editor_instance.rs b/crates/editor/src/editor_instance.rs index aedbe93e..50e3b6d7 100644 --- a/crates/editor/src/editor_instance.rs +++ b/crates/editor/src/editor_instance.rs @@ -7,8 +7,8 @@ use cap_media::frame_ws::create_frame_ws; use cap_project::RecordingConfig; use cap_project::{CursorEvents, ProjectConfiguration, RecordingMeta, XY}; use cap_rendering::{ - get_duration, ProjectRecordings, ProjectUniforms, RecordingSegmentDecoders, RenderOptions, - RenderVideoConstants, SegmentVideoPaths, + get_duration, DecodedSegmentFrames, ProjectRecordings, ProjectUniforms, + RecordingSegmentDecoders, RenderOptions, RenderVideoConstants, SegmentVideoPaths, }; use std::ops::Deref; use std::sync::Mutex as StdMutex; @@ -244,37 +244,31 @@ impl EditorInstance { let project = self.project_config.1.borrow().clone(); - let frame_time = frame_number as f32 / fps as f32; + let frame_time = frame_number as f64 / fps as f64; - let Some((segment_time, segment)) = project - .timeline - .as_ref() - .map(|timeline| timeline.get_recording_time(frame_number as f64 / fps as f64)) - .unwrap_or(Some((frame_number as f64 / fps as f64, None))) - else { + let Some((segment_time, segment_i)) = project.get_segment_time(frame_time) else { continue; }; - let segment = &self.segments[segment.unwrap_or(0) as usize]; + let segment = &self.segments[segment_i as usize]; - if let Some((screen_frame, camera_frame)) = segment + if let Some(segment_frames) = segment .decoders .get_frames(segment_time as f32, !project.camera.hide) .await { self.renderer .render_frame( - screen_frame, - camera_frame, + segment_frames, project.background.source.clone(), ProjectUniforms::new( &self.render_constants, &project, - frame_time, + frame_number, + fps, resolution_base, get_is_upgraded(), ), - segment_time as f32, resolution_base, ) .await; diff --git a/crates/editor/src/playback.rs b/crates/editor/src/playback.rs index d0e2d7d2..e2326973 100644 --- a/crates/editor/src/playback.rs +++ b/crates/editor/src/playback.rs @@ -57,12 +57,11 @@ impl Playback { let mut frame_number = self.start_frame_number + 1; - let duration = self - .project - .borrow() - .timeline() - .map(|t| t.duration()) - .unwrap_or(f64::MAX); + let duration = if let Some(timeline) = &self.project.borrow().timeline { + timeline.duration() + } else { + f64::MAX + }; // TODO: make this work with >1 segment if self.segments[0].audio.is_some() { @@ -89,23 +88,21 @@ impl Playback { let frame_time = frame_number as f32 / fps as f32; - if let Some((segment_time, segment)) = project - .timeline() - .map(|t| t.get_recording_time(frame_time as f64)) - .unwrap_or(Some((frame_time as f64, None))) + if let Some((segment_time, segment_i)) = project.get_segment_time(frame_time as f64) { - let segment = &self.segments[segment.unwrap_or(0) as usize]; + let segment = &self.segments[segment_i as usize]; tokio::select! { _ = stop_rx.changed() => { break; }, data = segment.decoders.get_frames(segment_time as f32, !project.camera.hide) => { - if let Some((screen_frame, camera_frame)) = data { + if let Some(segment_frames) = data { let uniforms = ProjectUniforms::new( &self.render_constants, &project, - segment_time as f32, + frame_number, + fps, resolution_base, is_upgraded, ); @@ -113,11 +110,9 @@ impl Playback { self .renderer .render_frame( - screen_frame, - camera_frame, + segment_frames, project.background.source.clone(), - uniforms.clone(), - frame_time, + uniforms, resolution_base ) .await; @@ -226,7 +221,7 @@ impl AudioPlayback { // pre-recorded videos are obviously a fixed size let mut audio_renderer = AudioPlaybackBuffer::new(segments, output_info); let playhead = f64::from(start_frame_number) / f64::from(fps); - audio_renderer.set_playhead(playhead, project.borrow().timeline()); + audio_renderer.set_playhead(playhead, &project.borrow()); // Prerender enough for smooth playback // disabled bc it causes weirdness during playback atm @@ -242,7 +237,7 @@ impl AudioPlayback { .build_output_stream( &config, move |buffer: &mut [T], _info| { - audio_renderer.render(project.borrow().timeline()); + audio_renderer.render(&project.borrow()); audio_renderer.fill(buffer); }, |_| {}, diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 3482943a..fed660fb 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -201,7 +201,7 @@ where let audio_frame = if let Some(audio) = &mut audio { if frame_count == 0 { - audio.buffer.set_playhead(0., project.timeline()); + audio.buffer.set_playhead(0., &project); } let audio_info = audio.buffer.info(); @@ -209,16 +209,14 @@ where f64::from(audio_info.sample_rate) / f64::from(self.fps); let samples = estimated_samples_per_frame.ceil() as usize; - if let Some((_, frame_data)) = audio - .buffer - .next_frame_data(samples, project.timeline.as_ref().map(|t| t)) + if let Some((_, frame_data)) = + audio.buffer.next_frame_data(samples, &project) { let mut frame = audio_info .wrap_frame(unsafe { cast_f32_slice_to_bytes(&frame_data) }, 0); let pts = (frame_number as f64 * f64::from(audio_info.sample_rate) / f64::from(fps)) as i64; frame.set_pts(Some(pts)); - let audio_frame = &frame; Some(frame) } else { None @@ -239,7 +237,6 @@ where frame.padded_bytes_per_row as usize, ); video_frame.set_pts(Some(frame_number as i64)); - dbg!(video_frame.pts()); frame_tx .send(MP4Input { diff --git a/crates/media/src/feeds/audio.rs b/crates/media/src/feeds/audio.rs index 5c92b4bd..a2948910 100644 --- a/crates/media/src/feeds/audio.rs +++ b/crates/media/src/feeds/audio.rs @@ -1,4 +1,4 @@ -use cap_project::TimelineConfiguration; +use cap_project::{ProjectConfiguration, TimelineConfiguration}; use ffmpeg::{ codec::{context, decoder}, format::{ @@ -163,7 +163,7 @@ pub struct AudioFrameBuffer { #[derive(Clone, Copy, Debug)] pub struct AudioFrameBufferCursor { - segment_index: usize, + segment_index: u32, // excludes channels samples: usize, } @@ -188,22 +188,13 @@ impl AudioFrameBuffer { self.data[0].info } - pub fn set_playhead(&mut self, playhead: f64, maybe_timeline: Option<&TimelineConfiguration>) { + pub fn set_playhead(&mut self, playhead: f64, project: &ProjectConfiguration) { self.elapsed_samples = self.playhead_to_samples(playhead); - self.cursor = match maybe_timeline { - Some(timeline) => match timeline.get_recording_time(playhead) { - Some((time, segment)) => { - let index = segment.unwrap_or(0) as usize; - AudioFrameBufferCursor { - segment_index: index, - samples: self.playhead_to_samples(time), - } - } - None => AudioFrameBufferCursor { - segment_index: 0, - samples: self.data[0].buffer.len(), - }, + self.cursor = match project.get_segment_time(playhead) { + Some((segment_time, segment_i)) => AudioFrameBufferCursor { + segment_index: segment_i, + samples: self.playhead_to_samples(segment_time), }, None => AudioFrameBufferCursor { segment_index: 0, @@ -219,10 +210,10 @@ impl AudioFrameBuffer { // this will only seek if there is a significant change in actual vs expected next sample // (corresponding to a trim or split point). Currently this change is at least 0.2 seconds // - not sure we offer that much precision in the editor even! - let new_cursor = match timeline.get_recording_time(playhead) { - Some((time, segment)) => AudioFrameBufferCursor { - segment_index: segment.unwrap_or(0) as usize, - samples: self.playhead_to_samples(time), + let new_cursor = match timeline.get_segment_time(playhead) { + Some((segment_time, segment_i)) => AudioFrameBufferCursor { + segment_index: segment_i, + samples: self.playhead_to_samples(segment_time), }, None => AudioFrameBufferCursor { segment_index: 0, @@ -249,14 +240,14 @@ impl AudioFrameBuffer { pub fn next_frame( &mut self, requested_samples: usize, - timeline: Option<&TimelineConfiguration>, + project: &ProjectConfiguration, ) -> Option { let format = self.info().sample_format; let channels = self.info().channel_layout(); let sample_rate = self.info().sample_rate; let res = self - .next_frame_data(requested_samples, timeline) + .next_frame_data(requested_samples, project) .map(move |(samples, data)| { let mut raw_frame = FFAudio::new(format, samples, channels); raw_frame.set_rate(sample_rate); @@ -273,28 +264,28 @@ impl AudioFrameBuffer { pub fn next_frame_data<'a>( &'a mut self, samples: usize, - maybe_timeline: Option<&TimelineConfiguration>, + project: &ProjectConfiguration, ) -> Option<(usize, &'a [f32])> { - if let Some(timeline) = maybe_timeline { + if let Some(timeline) = &project.timeline { self.adjust_cursor(timeline); } + let channels = self.info().channels; - let data = &self.data[self.cursor.segment_index]; + let data = &self.data[self.cursor.segment_index as usize]; let buffer = &data.buffer; - if self.cursor.samples >= buffer.len() / self.info().channels { + if self.cursor.samples >= buffer.len() / channels { self.elapsed_samples += samples; return None; } - let samples = (samples).min((buffer.len() / self.info().channels) - self.cursor.samples); + let samples = (samples).min((buffer.len() / channels) - self.cursor.samples); let start = self.cursor; self.elapsed_samples += samples; self.cursor.samples += samples; Some(( samples, - &buffer - [start.samples * self.info().channels..self.cursor.samples * self.info().channels], + &buffer[start.samples * channels..self.cursor.samples * channels], )) } } @@ -330,10 +321,10 @@ impl AudioPlaybackBuffer { } } - pub fn set_playhead(&mut self, playhead: f64, maybe_timeline: Option<&TimelineConfiguration>) { + pub fn set_playhead(&mut self, playhead: f64, project: &ProjectConfiguration) { self.resampler.reset(); self.resampled_buffer.clear(); - self.frame_buffer.set_playhead(playhead, maybe_timeline); + self.frame_buffer.set_playhead(playhead, project); println!("Successful seek to sample {:?}", self.frame_buffer.cursor); } @@ -343,7 +334,7 @@ impl AudioPlaybackBuffer { <= 2 * (Self::PROCESSING_SAMPLES_COUNT as usize) * self.resampler.output.channels } - pub fn render(&mut self, timeline: Option<&TimelineConfiguration>) { + pub fn render(&mut self, project: &ProjectConfiguration) { if self.buffer_reaching_limit() { return; } @@ -352,7 +343,7 @@ impl AudioPlaybackBuffer { let next_frame = self .frame_buffer - .next_frame(Self::PROCESSING_SAMPLES_COUNT as usize, timeline); + .next_frame(Self::PROCESSING_SAMPLES_COUNT as usize, project); let maybe_rendered = match next_frame { Some(frame) => Some(self.resampler.queue_and_process_frame(&frame)), diff --git a/crates/project/src/configuration.rs b/crates/project/src/configuration.rs index 2f0325be..7e46d778 100644 --- a/crates/project/src/configuration.rs +++ b/crates/project/src/configuration.rs @@ -263,7 +263,8 @@ pub struct HotkeysConfiguration { #[derive(Type, Serialize, Deserialize, Clone, Debug, Default)] #[serde(rename_all = "camelCase")] pub struct TimelineSegment { - pub recording_segment: Option, + #[serde(default)] + pub recording_segment: u32, pub timescale: f64, pub start: f64, pub end: f64, @@ -307,13 +308,13 @@ pub struct TimelineConfiguration { } impl TimelineConfiguration { - pub fn get_recording_time(&self, tick_time: f64) -> Option<(f64, Option)> { + pub fn get_segment_time(&self, frame_time: f64) -> Option<(f64, u32)> { let mut accum_duration = 0.0; for segment in self.segments.iter() { - if tick_time < accum_duration + segment.duration() { + if frame_time < accum_duration + segment.duration() { return segment - .interpolate_time(tick_time - accum_duration) + .interpolate_time(frame_time - accum_duration) .map(|t| (t, segment.recording_segment)); } @@ -355,8 +356,11 @@ impl ProjectConfiguration { ) } - pub fn timeline(&self) -> Option<&TimelineConfiguration> { - self.timeline.as_ref() + pub fn get_segment_time(&self, frame_time: f64) -> Option<(f64, u32)> { + self.timeline + .as_ref() + .map(|t| t.get_segment_time(frame_time as f64)) + .unwrap_or(Some((frame_time as f64, 0))) } } diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index c36ec607..94a29bfe 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -132,19 +132,23 @@ impl RecordingSegmentDecoders { pub async fn get_frames( &self, - frame_time: f32, + segment_time: f32, needs_camera: bool, - ) -> Option<(DecodedFrame, Option)> { + ) -> Option { let (screen, camera) = tokio::join!( - self.screen.get_frame(frame_time), + self.screen.get_frame(segment_time), OptionFuture::from( needs_camera - .then(|| self.camera.as_ref().map(|d| d.get_frame(frame_time))) + .then(|| self.camera.as_ref().map(|d| d.get_frame(segment_time))) .flatten() ) ); - Some((screen?, camera.flatten())) + Some(DecodedSegmentFrames { + screen_frame: screen?, + camera_frame: camera.flatten(), + segment_time, + }) } } @@ -201,23 +205,10 @@ pub async fn render_video_to_channel( break; } - let frame_time = frame_number as f32 / fps as f32; + let frame_time = frame_number as f64 / fps as f64; - let segment_time = project - .timeline - .as_ref() - .and_then(|t| t.get_recording_time(frame_time as f64)) - .map(|(v, _)| v) - .unwrap_or(frame_time as f64); - - let segment_i = if let Some(timeline) = &project.timeline { - timeline - .get_recording_time(frame_number as f64 / fps as f64) - .map(|value| value.1) - .flatten() - .unwrap_or(0u32) - } else { - 0u32 + let Some((segment_time, segment_i)) = project.get_segment_time(frame_time) else { + break; }; let segment = &segments[segment_i as usize]; @@ -228,7 +219,7 @@ pub async fn render_video_to_channel( std::mem::replace(&mut frame_number, prev + 1) }; - if let Some((screen_frame, camera_frame)) = segment + if let Some(segment_frames) = segment .decoders .get_frames(segment_time as f32, !project.camera.hide) .await @@ -236,19 +227,17 @@ pub async fn render_video_to_channel( let uniforms = ProjectUniforms::new( &constants, &project, - frame_time as f32, + frame_number, + fps, resolution_base, is_upgraded, ); let frame = produce_frame( &constants, - &screen_frame, - &camera_frame, + segment_frames, background, &uniforms, - frame_time, - total_frames, resolution_base, ) .await?; @@ -711,12 +700,14 @@ impl ProjectUniforms { pub fn new( constants: &RenderVideoConstants, project: &ProjectConfiguration, - frame_time: f32, + frame_number: u32, + fps: u32, resolution_base: XY, is_upgraded: bool, ) -> Self { let options = &constants.options; let output_size = Self::get_output_size(options, project, resolution_base); + let frame_time = frame_number as f32 / fps as f32; let cursor_position = interpolate_cursor_position( &Default::default(), /*constants.cursor*/ @@ -907,14 +898,17 @@ pub struct RenderedFrame { pub padded_bytes_per_row: u32, } +pub struct DecodedSegmentFrames { + pub screen_frame: DecodedFrame, + pub camera_frame: Option, + pub segment_time: f32, +} + pub async fn produce_frame( constants: &RenderVideoConstants, - screen_frame: &Vec, - camera_frame: &Option, + segment_frames: DecodedSegmentFrames, background: Background, uniforms: &ProjectUniforms, - segment_time: f32, - total_frames: u32, resolution_base: XY, ) -> Result { let mut encoder = constants.device.create_command_encoder( @@ -1010,7 +1004,7 @@ pub async fn produce_frame( origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All, }, - screen_frame, + &segment_frames.screen_frame, wgpu::ImageDataLayout { offset: 0, bytes_per_row: Some(constants.options.screen_size.x * 4), @@ -1043,7 +1037,7 @@ pub async fn produce_frame( draw_cursor( constants, uniforms, - segment_time, + segment_frames.segment_time, &mut encoder, get_either(texture_views, !output_is_left), resolution_base, @@ -1052,7 +1046,7 @@ pub async fn produce_frame( // camera if let (Some(camera_size), Some(camera_frame), Some(uniforms)) = ( constants.options.camera_size, - camera_frame, + &segment_frames.camera_frame, &uniforms.camera, ) { let texture = constants.device.create_texture(