From 4c02475ba995cdc488c784e903024d128153c7d0 Mon Sep 17 00:00:00 2001 From: Luke Harries Date: Sun, 16 Jun 2024 21:38:04 -0400 Subject: [PATCH 1/2] ability to download --- .../sound-effects/video-to-sfx/app/page.tsx | 68 +++++++++++------- .../video-to-sfx/app/state/orchestrator.ts | 5 ++ .../video-to-sfx/app/state/player.ts | 2 + .../video-to-sfx/lib/mergeAndDownload.ts | 69 +++++++++++++++++++ .../video-to-sfx/package-lock.json | 29 ++++++++ .../sound-effects/video-to-sfx/package.json | 2 + 6 files changed, 152 insertions(+), 23 deletions(-) create mode 100644 examples/sound-effects/video-to-sfx/lib/mergeAndDownload.ts diff --git a/examples/sound-effects/video-to-sfx/app/page.tsx b/examples/sound-effects/video-to-sfx/app/page.tsx index ed32cb0..18eba86 100644 --- a/examples/sound-effects/video-to-sfx/app/page.tsx +++ b/examples/sound-effects/video-to-sfx/app/page.tsx @@ -13,7 +13,7 @@ import { observer } from "mobx-react"; import { cn } from "@/lib/utils"; import { autorun, reaction } from "mobx"; -export const HoverOverlay = ({ className }: { className?: string }) => { +const HoverOverlay = ({ className }: { className?: string }) => { return (
{ }; import { convertVideoToSFX } from "@/lib/videoToSFX"; import { useMutation } from "@tanstack/react-query"; +import { DownloadIcon } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { mergeAndDownload } from "@/lib/mergeAndDownload"; const LoadingIndicator = () => { const { ref, replay } = useScramble({ @@ -272,14 +275,31 @@ const Home = observer(() => { {orchestrator.sfxPlayers.map((player, index) => ( - orchestrator.play(index)} - onPause={() => orchestrator.stop()} - player={player} - active={orchestrator.activeIndex === index} - /> +
+ orchestrator.play(index)} + onPause={() => orchestrator.stop()} + player={player} + active={orchestrator.activeIndex === index} + /> + +
))} @@ -343,7 +363,7 @@ const SoundEffect = observer( scale: 1, transition: { ...springs.xxxslow(), delay: index * 0.1 }, }} - className="group relative h-16 rounded-xl w-full focus-visible:ring-gray-800 focus-visible:ring-2 focus-visible:outline-none" + className="flex justify-between group relative h-16 rounded-xl w-full focus-visible:ring-gray-800 focus-visible:ring-2 focus-visible:outline-none" onClick={() => { if (player.playing) { onPause?.(); @@ -362,19 +382,21 @@ const SoundEffect = observer( } }} > - -
- - - - +
+ +
+ + + + +
); diff --git a/examples/sound-effects/video-to-sfx/app/state/orchestrator.ts b/examples/sound-effects/video-to-sfx/app/state/orchestrator.ts index b46314a..0f4d04e 100644 --- a/examples/sound-effects/video-to-sfx/app/state/orchestrator.ts +++ b/examples/sound-effects/video-to-sfx/app/state/orchestrator.ts @@ -55,4 +55,9 @@ export class Orchestrator { stop() { this.playing = false; } + + getAudioUrl(index: number) { + const player = this.sfxPlayers[index]; + return player.data; + } } diff --git a/examples/sound-effects/video-to-sfx/app/state/player.ts b/examples/sound-effects/video-to-sfx/app/state/player.ts index fbae0a8..1ea453a 100644 --- a/examples/sound-effects/video-to-sfx/app/state/player.ts +++ b/examples/sound-effects/video-to-sfx/app/state/player.ts @@ -82,11 +82,13 @@ export class AudioPlayer { audio: HTMLAudioElement; progress: number = 0; playing: boolean; + data: string; constructor(data: string) { this.loaded = false; this.waveformLoaded = false; this.playing = false; + this.data = data; this._player = new Tone.Player( data, action(() => { diff --git a/examples/sound-effects/video-to-sfx/lib/mergeAndDownload.ts b/examples/sound-effects/video-to-sfx/lib/mergeAndDownload.ts new file mode 100644 index 0000000..2168315 --- /dev/null +++ b/examples/sound-effects/video-to-sfx/lib/mergeAndDownload.ts @@ -0,0 +1,69 @@ +export async function mergeAndDownload( + videoFile: File | null, + audioData: string +) { + const { FFmpeg } = await import("@ffmpeg/ffmpeg"); + const { fetchFile, toBlobURL } = await import("@ffmpeg/util"); + const ffmpeg = new FFmpeg(); + + const load = async () => { + const baseURL = "https://unpkg.com/@ffmpeg/core@0.12.6/dist/umd"; + ffmpeg.on("log", ({ message }) => { + console.log(message); + }); + await ffmpeg.load({ + coreURL: await toBlobURL(`${baseURL}/ffmpeg-core.js`, "text/javascript"), + wasmURL: await toBlobURL( + `${baseURL}/ffmpeg-core.wasm`, + "application/wasm" + ), + }); + }; + const process = async () => { + console.log("transcoding"); + if (!videoFile) { + throw new Error("No video file"); + } + + if (videoFile) { + await ffmpeg.writeFile( + "input.mp4", + await fetchFile(URL.createObjectURL(videoFile)) + ); + } + + await ffmpeg.writeFile("audio.mpeg", await fetchFile(audioData)); + + await ffmpeg.exec(["-v", "error", "-i", "input.mp4", "-f", "null", "-"]); + + await ffmpeg.exec(["-v", "error", "-i", "audio.mpeg", "-f", "null", "-"]); + + await ffmpeg.exec([ + "-v", + "verbose", + "-i", + "input.mp4", + "-i", + "audio.mpeg", + "-c:v", + "copy", // Copy the video codec + "-c:a", + "aac", // Transcode audio to AAC + "-strict", + "experimental", + "output.mp4", + ]); + console.log("transcoding completed"); + const data = await ffmpeg.readFile("output.mp4"); + const final_url = URL.createObjectURL( + new Blob([data.buffer], { type: "video/mp4" }) + ); + const downloadLinkFinalVideo = document.createElement("a"); + downloadLinkFinalVideo.href = final_url; + downloadLinkFinalVideo.download = "final_output.mp4"; + downloadLinkFinalVideo.click(); + }; + + await load(); + await process(); +} diff --git a/examples/sound-effects/video-to-sfx/package-lock.json b/examples/sound-effects/video-to-sfx/package-lock.json index d4a90d9..df0dad7 100644 --- a/examples/sound-effects/video-to-sfx/package-lock.json +++ b/examples/sound-effects/video-to-sfx/package-lock.json @@ -8,6 +8,8 @@ "name": "video-to-sfx", "version": "0.1.0", "dependencies": { + "@ffmpeg/ffmpeg": "^0.12.10", + "@ffmpeg/util": "^0.12.1", "@hookform/resolvers": "^3.6.0", "@radix-ui/react-accordion": "^1.1.2", "@radix-ui/react-alert-dialog": "^1.0.5", @@ -101,6 +103,33 @@ "node": ">=6.9.0" } }, + "node_modules/@ffmpeg/ffmpeg": { + "version": "0.12.10", + "resolved": "https://registry.npmjs.org/@ffmpeg/ffmpeg/-/ffmpeg-0.12.10.tgz", + "integrity": "sha512-lVtk8PW8e+NUzGZhPTWj2P1J4/NyuCrbDD3O9IGpSeLYtUZKBqZO8CNj1WYGghep/MXoM8e1qVY1GztTkf8YYQ==", + "dependencies": { + "@ffmpeg/types": "^0.12.2" + }, + "engines": { + "node": ">=18.x" + } + }, + "node_modules/@ffmpeg/types": { + "version": "0.12.2", + "resolved": "https://registry.npmjs.org/@ffmpeg/types/-/types-0.12.2.tgz", + "integrity": "sha512-NJtxwPoLb60/z1Klv0ueshguWQ/7mNm106qdHkB4HL49LXszjhjCCiL+ldHJGQ9ai2Igx0s4F24ghigy//ERdA==", + "engines": { + "node": ">=16.x" + } + }, + "node_modules/@ffmpeg/util": { + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@ffmpeg/util/-/util-0.12.1.tgz", + "integrity": "sha512-10jjfAKWaDyb8+nAkijcsi9wgz/y26LOc1NKJradNMyCIl6usQcBbhkjX5qhALrSBcOy6TOeksunTYa+a03qNQ==", + "engines": { + "node": ">=18.x" + } + }, "node_modules/@floating-ui/core": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.6.2.tgz", diff --git a/examples/sound-effects/video-to-sfx/package.json b/examples/sound-effects/video-to-sfx/package.json index 6739320..6dbe279 100644 --- a/examples/sound-effects/video-to-sfx/package.json +++ b/examples/sound-effects/video-to-sfx/package.json @@ -9,6 +9,8 @@ "lint": "next lint" }, "dependencies": { + "@ffmpeg/ffmpeg": "^0.12.10", + "@ffmpeg/util": "^0.12.1", "@hookform/resolvers": "^3.6.0", "@radix-ui/react-accordion": "^1.1.2", "@radix-ui/react-alert-dialog": "^1.0.5", From 26865b0cdc9d6d33be6d050487a075fe9b6b20a6 Mon Sep 17 00:00:00 2001 From: Luke Harries Date: Sun, 16 Jun 2024 21:40:22 -0400 Subject: [PATCH 2/2] add package --- examples/sound-effects/video-to-sfx/package-lock.json | 9 +++++++++ examples/sound-effects/video-to-sfx/package.json | 1 + 2 files changed, 10 insertions(+) diff --git a/examples/sound-effects/video-to-sfx/package-lock.json b/examples/sound-effects/video-to-sfx/package-lock.json index df0dad7..05bdeda 100644 --- a/examples/sound-effects/video-to-sfx/package-lock.json +++ b/examples/sound-effects/video-to-sfx/package-lock.json @@ -47,6 +47,7 @@ "date-fns": "^3.6.0", "embla-carousel-react": "^8.1.5", "framer-motion": "^11.2.10", + "geist": "^1.3.0", "input-otp": "^1.2.4", "lodash": "^4.17.21", "lucide-react": "^0.395.0", @@ -2430,6 +2431,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/geist": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/geist/-/geist-1.3.0.tgz", + "integrity": "sha512-IoGBfcqVEYB4bEwsfHd35jF4+X9LHRPYZymHL4YOltHSs9LJa24DYs1Z7rEMQ/lsEvaAIc61Y9aUxgcJaQ8lrg==", + "peerDependencies": { + "next": ">=13.2.0 <15.0.0-0" + } + }, "node_modules/get-nonce": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-nonce/-/get-nonce-1.0.1.tgz", diff --git a/examples/sound-effects/video-to-sfx/package.json b/examples/sound-effects/video-to-sfx/package.json index 6dbe279..f20d11b 100644 --- a/examples/sound-effects/video-to-sfx/package.json +++ b/examples/sound-effects/video-to-sfx/package.json @@ -48,6 +48,7 @@ "date-fns": "^3.6.0", "embla-carousel-react": "^8.1.5", "framer-motion": "^11.2.10", + "geist": "^1.3.0", "input-otp": "^1.2.4", "lodash": "^4.17.21", "lucide-react": "^0.395.0",