Handle audio samples as f32 internally (#252)

* some stuff

* fix crashes

* simplify AudioData

* cleanup

crates/export/src/lib.rs

@@ -1,6 +1,6 @@
 use cap_editor::Segment;
 use cap_media::{
-    data::{AudioInfo, RawVideoFormat, VideoInfo},
+    data::{cast_f32_slice_to_bytes, AudioInfo, RawVideoFormat, VideoInfo},
     encoders::{MP4Encoder, MP4Input},
     feeds::{AudioData, AudioFrameBuffer},
     MediaError,
@@ -207,17 +207,11 @@ where
                             .buffer
                             .next_frame_data(samples, project.timeline.as_ref().map(|t| t))
                         {
-                            let mut frame = audio_info.wrap_frame(&frame_data, 0);
+                            let mut frame = audio_info
+                                .wrap_frame(unsafe { cast_f32_slice_to_bytes(&frame_data) }, 0);
                             let pts = (frame_count as f64 * f64::from(audio_info.sample_rate)
                                 / f64::from(fps)) as i64;
                             frame.set_pts(Some(pts));
-                            // println!(
-                            //     "Export: Sending audio frame {} with PTS: {:?}, samples: {}, data size: {}",
-                            //     frame_count,
-                            //     frame.pts(),
-                            //     samples,
-                            //     frame_data.len()
-                            // );
                             Some(frame)
                         } else {
                             None
@@ -226,11 +220,6 @@ where
                         None
                     };
 
-                    // println!(
-                    //     "Export: Processing frame {} (size: {}x{}, padded_bytes_per_row: {})",
-                    //     frame_count, frame.width, frame.height, frame.padded_bytes_per_row
-                    // );
-
                     let mut video_frame = VideoInfo::from_raw(
                         RawVideoFormat::Rgba,
                         self.output_size.0,
@@ -244,12 +233,6 @@ where
                     );
                     video_frame.set_pts(Some(frame_count as i64));
 
-                    // println!(
-                    //     "Export: Sending frame {} to encoder (PTS: {:?})",
-                    //     frame_count,
-                    //     video_frame.pts()
-                    // );
-
                     frame_tx
                         .send(MP4Input {
                             audio: audio_frame,

crates/media/src/data.rs

@@ -204,6 +204,10 @@ impl AudioInfo {
     }
 }
 
+pub unsafe fn cast_f32_slice_to_bytes(slice: &[f32]) -> &[u8] {
+    std::slice::from_raw_parts(slice.as_ptr() as *const u8, slice.len() * f32::BYTE_SIZE)
+}
+
 #[derive(Debug, Copy, Clone)]
 pub struct VideoInfo {
     pub pixel_format: Pixel,

crates/media/src/encoders/mp4.rs

@@ -2,6 +2,7 @@ use crate::{
     data::{
         AudioInfo, FFAudio, FFPacket, FFRational, FFVideo, PlanarData, RawVideoFormat, VideoInfo,
     },
+    feeds::AudioData,
     pipeline::{audio_buffer::AudioBuffer, task::PipelineSinkTask},
     MediaError,
 };
@@ -110,7 +111,7 @@ impl MP4Encoder {
 
             let resampler = software::resampler(
                 (
-                    Sample::F64(format::sample::Type::Packed),
+                    AudioData::FORMAT,
                     audio_config.channel_layout(),
                     audio_config.sample_rate,
                 ),
@@ -211,31 +212,20 @@ impl MP4Encoder {
             };
 
             let mut output = ffmpeg::util::frame::Audio::empty();
+
             audio.resampler.run(&buffered_frame, &mut output).unwrap();
 
             // Preserve PTS from input frame
             if let Some(pts) = buffered_frame.pts() {
                 output.set_pts(Some(pts));
             }
 
-            // println!(
-            //     "MP4Encoder: Sending audio frame with PTS: {:?}, samples: {}",
-            //     output.pts(),
-            //     output.samples()
-            // );
-
             // Send frame to encoder
             audio.encoder.send_frame(&output).unwrap();
 
             // Process any encoded packets
             let mut encoded_packet = FFPacket::empty();
             while audio.encoder.receive_packet(&mut encoded_packet).is_ok() {
-                // println!(
-                //     "MP4Encoder: Writing audio packet with PTS: {:?}, size: {}",
-                //     encoded_packet.pts(),
-                //     encoded_packet.size()
-                // );
-
                 encoded_packet.set_stream(1);
                 encoded_packet.rescale_ts(
                     audio.encoder.time_base(),
@@ -257,21 +247,11 @@ impl MP4Encoder {
             .receive_packet(&mut encoded_packet)
             .is_ok()
         {
-            // println!(
-            //     "MP4Encoder: Got encoded packet with PTS: {:?}, DTS: {:?}",
-            //     encoded_packet.pts(),
-            //     encoded_packet.dts()
-            // );
             encoded_packet.set_stream(0); // Video is stream 0
             encoded_packet.rescale_ts(
                 self.video.encoder.time_base(),
                 self.output_ctx.stream(0).unwrap().time_base(),
             );
-            // println!(
-            //     "MP4Encoder: Writing packet with rescaled PTS: {:?}, DTS: {:?}",
-            //     encoded_packet.pts(),
-            //     encoded_packet.dts()
-            // );
             encoded_packet
                 .write_interleaved(&mut self.output_ctx)
                 .unwrap();