Skip to content

Commit

Permalink
cleanup rendering structs
Browse files Browse the repository at this point in the history
  • Loading branch information
@Brendonovich
Brendonovich committed Feb 1, 2025
1 parent c87951c commit 9406684
Show file tree
Hide file tree
Showing 9 changed files with 99 additions and 137 deletions.
2 changes: 1 addition & 1 deletion apps/desktop/src-tauri/src/recording.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -395,7 +395,7 @@ fn project_config_from_recording(
.iter()
.enumerate()
.map(|(i, segment)| TimelineSegment {
recording_segment: Some(i as u32),
recording_segment: i as u32,
start: 0.0,
end: segment.duration(),
timescale: 1.0,
Expand Down
2 changes: 1 addition & 1 deletion apps/desktop/src/utils/tauri.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,7 @@ export type SharingMeta = { id: string; link: string }
export type ShowCapWindow = "Setup" | "Main" | { Settings: { page: string | null } } | { Editor: { project_id: string } } | "PrevRecordings" | "WindowCaptureOccluder" | { CaptureArea: { screen: CaptureScreen } } | { Camera: { ws_port: number } } | { InProgressRecording: { position: [number, number] | null } } | "Upgrade" | "SignIn"
export type SingleSegment = { display: Display; camera?: CameraMeta | null; audio?: AudioMeta | null; cursor?: string | null }
export type TimelineConfiguration = { segments: TimelineSegment[]; zoomSegments: ZoomSegment[] }
export type TimelineSegment = { recordingSegment: number | null; timescale: number; start: number; end: number }
export type TimelineSegment = { recordingSegment?: number; timescale: number; start: number; end: number }
export type UploadMode = { Initial: { pre_created_video: PreCreatedVideo | null } } | "Reupload"
export type UploadProgress = { progress: number; message: string }
export type UploadResult = { Success: string } | "NotAuthenticated" | "PlanCheckFailed" | "UpgradeRequired"
Expand Down
29 changes: 8 additions & 21 deletions crates/editor/src/editor.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ use std::{sync::Arc, time::Instant};
use cap_media::{feeds::RawCameraFrame, frame_ws::WSFrame};
use cap_project::{BackgroundSource, RecordingMeta, XY};
use cap_rendering::{
decoder::DecodedFrame, produce_frame, ProjectRecordings, ProjectUniforms, RenderVideoConstants,
decoder::DecodedFrame, produce_frame, DecodedSegmentFrames, ProjectRecordings, ProjectUniforms,
RenderVideoConstants,
};
use tokio::{
sync::{mpsc, oneshot},
Expand All @@ -12,11 +13,9 @@ use tokio::{

pub enum RendererMessage {
RenderFrame {
screen_frame: DecodedFrame,
camera_frame: Option<DecodedFrame>,
segment_frames: DecodedSegmentFrames,
background: BackgroundSource,
uniforms: ProjectUniforms,
time: f32, // Add this field
finished: oneshot::Sender<()>,
resolution_base: XY<u32>,
},
Expand Down Expand Up @@ -47,9 +46,7 @@ impl Renderer {

// Check camera duration if it exists
if let Some(camera_path) = meta.content.camera_path() {
if let Ok(camera_duration) =
recordings.get_source_duration(&meta.path(&camera_path))
{
if let Ok(camera_duration) = recordings.get_source_duration(&meta.path(&camera_path)) {
max_duration = max_duration.max(camera_duration);
}
}
Expand Down Expand Up @@ -77,11 +74,9 @@ impl Renderer {
while let Some(msg) = self.rx.recv().await {
match msg {
RendererMessage::RenderFrame {
screen_frame,
camera_frame,
segment_frames,
background,
uniforms,
time,
finished,
resolution_base,
} => {
Expand All @@ -95,17 +90,13 @@ impl Renderer {

let render_constants = self.render_constants.clone();
let frame_tx = self.frame_tx.clone();
let total_frames = self.total_frames;

frame_task = Some(tokio::spawn(async move {
let frame = produce_frame(
&render_constants,
&screen_frame,
&camera_frame,
segment_frames,
cap_rendering::Background::from(background),
&uniforms,
time,
total_frames,
resolution_base,
)
.await
Expand Down Expand Up @@ -145,21 +136,17 @@ impl RendererHandle {

pub async fn render_frame(
&self,
screen_frame: DecodedFrame,
camera_frame: Option<DecodedFrame>,
segment_frames: DecodedSegmentFrames,
background: BackgroundSource,
uniforms: ProjectUniforms,
time: f32,
resolution_base: XY<u32>,
) {
let (finished_tx, finished_rx) = oneshot::channel();

self.send(RendererMessage::RenderFrame {
screen_frame,
camera_frame,
segment_frames,
background,
uniforms,
time, // Pass the time
finished: finished_tx,
resolution_base,
})
Expand Down
24 changes: 9 additions & 15 deletions crates/editor/src/editor_instance.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ use cap_media::frame_ws::create_frame_ws;
use cap_project::RecordingConfig;
use cap_project::{CursorEvents, ProjectConfiguration, RecordingMeta, XY};
use cap_rendering::{
get_duration, ProjectRecordings, ProjectUniforms, RecordingSegmentDecoders, RenderOptions,
RenderVideoConstants, SegmentVideoPaths,
get_duration, DecodedSegmentFrames, ProjectRecordings, ProjectUniforms,
RecordingSegmentDecoders, RenderOptions, RenderVideoConstants, SegmentVideoPaths,
};
use std::ops::Deref;
use std::sync::Mutex as StdMutex;
Expand Down Expand Up @@ -244,37 +244,31 @@ impl EditorInstance {

let project = self.project_config.1.borrow().clone();

let frame_time = frame_number as f32 / fps as f32;
let frame_time = frame_number as f64 / fps as f64;

let Some((segment_time, segment)) = project
.timeline
.as_ref()
.map(|timeline| timeline.get_recording_time(frame_number as f64 / fps as f64))
.unwrap_or(Some((frame_number as f64 / fps as f64, None)))
else {
let Some((segment_time, segment_i)) = project.get_segment_time(frame_time) else {
continue;
};

let segment = &self.segments[segment.unwrap_or(0) as usize];
let segment = &self.segments[segment_i as usize];

if let Some((screen_frame, camera_frame)) = segment
if let Some(segment_frames) = segment
.decoders
.get_frames(segment_time as f32, !project.camera.hide)
.await
{
self.renderer
.render_frame(
screen_frame,
camera_frame,
segment_frames,
project.background.source.clone(),
ProjectUniforms::new(
&self.render_constants,
&project,
frame_time,
frame_number,
fps,
resolution_base,
get_is_upgraded(),
),
segment_time as f32,
resolution_base,
)
.await;
Expand Down
33 changes: 14 additions & 19 deletions crates/editor/src/playback.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,12 +57,11 @@ impl Playback {

let mut frame_number = self.start_frame_number + 1;

let duration = self
.project
.borrow()
.timeline()
.map(|t| t.duration())
.unwrap_or(f64::MAX);
let duration = if let Some(timeline) = &self.project.borrow().timeline {
timeline.duration()
} else {
f64::MAX
};

// TODO: make this work with >1 segment
if self.segments[0].audio.is_some() {
Expand All @@ -89,35 +88,31 @@ impl Playback {

let frame_time = frame_number as f32 / fps as f32;

if let Some((segment_time, segment)) = project
.timeline()
.map(|t| t.get_recording_time(frame_time as f64))
.unwrap_or(Some((frame_time as f64, None)))
if let Some((segment_time, segment_i)) = project.get_segment_time(frame_time as f64)
{
let segment = &self.segments[segment.unwrap_or(0) as usize];
let segment = &self.segments[segment_i as usize];

tokio::select! {
_ = stop_rx.changed() => {
break;
},
data = segment.decoders.get_frames(segment_time as f32, !project.camera.hide) => {
if let Some((screen_frame, camera_frame)) = data {
if let Some(segment_frames) = data {
let uniforms = ProjectUniforms::new(
&self.render_constants,
&project,
segment_time as f32,
frame_number,
fps,
resolution_base,
is_upgraded,
);

self
.renderer
.render_frame(
screen_frame,
camera_frame,
segment_frames,
project.background.source.clone(),
uniforms.clone(),
frame_time,
uniforms,
resolution_base
)
.await;
Expand Down Expand Up @@ -226,7 +221,7 @@ impl AudioPlayback {
// pre-recorded videos are obviously a fixed size
let mut audio_renderer = AudioPlaybackBuffer::new(segments, output_info);
let playhead = f64::from(start_frame_number) / f64::from(fps);
audio_renderer.set_playhead(playhead, project.borrow().timeline());
audio_renderer.set_playhead(playhead, &project.borrow());

// Prerender enough for smooth playback
// disabled bc it causes weirdness during playback atm
Expand All @@ -242,7 +237,7 @@ impl AudioPlayback {
.build_output_stream(
&config,
move |buffer: &mut [T], _info| {
audio_renderer.render(project.borrow().timeline());
audio_renderer.render(&project.borrow());
audio_renderer.fill(buffer);
},
|_| {},
Expand Down
9 changes: 3 additions & 6 deletions crates/export/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,24 +201,22 @@ where

let audio_frame = if let Some(audio) = &mut audio {
if frame_count == 0 {
audio.buffer.set_playhead(0., project.timeline());
audio.buffer.set_playhead(0., &project);
}

let audio_info = audio.buffer.info();
let estimated_samples_per_frame =
f64::from(audio_info.sample_rate) / f64::from(self.fps);
let samples = estimated_samples_per_frame.ceil() as usize;

if let Some((_, frame_data)) = audio
.buffer
.next_frame_data(samples, project.timeline.as_ref().map(|t| t))
if let Some((_, frame_data)) =
audio.buffer.next_frame_data(samples, &project)
{
let mut frame = audio_info
.wrap_frame(unsafe { cast_f32_slice_to_bytes(&frame_data) }, 0);
let pts = (frame_number as f64 * f64::from(audio_info.sample_rate)
/ f64::from(fps)) as i64;
frame.set_pts(Some(pts));
let audio_frame = &frame;
Some(frame)
} else {
None
Expand All @@ -239,7 +237,6 @@ where
frame.padded_bytes_per_row as usize,
);
video_frame.set_pts(Some(frame_number as i64));
dbg!(video_frame.pts());

frame_tx
.send(MP4Input {
Expand Down
Loading

0 comments on commit 9406684

Please sign in to comment.