Merge pull request #94 from hughfenghen/feat/split-clip-by-time

Feat/split clip by time

packages/av-cliper/package.json

@@ -39,7 +39,7 @@
     "typescript": "^4.9.3",
     "vite": "^5.1.6",
     "vitest": "^1.4.0",
-    "webdriverio": "^8.34.0"
+    "webdriverio": "^8.36.1"
   },
   "dependencies": {
     "@types/dom-webcodecs": "^0.1.7",

packages/av-cliper/src/av-utils.ts

@@ -92,7 +92,7 @@ export async function decodeImg(
   };
   const imageDecoder = new ImageDecoder(init);
 
-  await Promise.all([imageDecoder.completed, imageDecoder.tracks.ready])
+  await Promise.all([imageDecoder.completed, imageDecoder.tracks.ready]);
 
   let frameCnt = imageDecoder.tracks.selectedTrack?.frameCount ?? 1;
 
@@ -209,8 +209,8 @@ export function autoReadStream<ST extends ReadableStream>(
   stream: ST,
   cbs: {
     onChunk: ST extends ReadableStream<infer DT>
-    ? (chunk: DT) => Promise<void>
-    : never;
+      ? (chunk: DT) => Promise<void>
+      : never;
     onDone: () => void;
   },
 ) {

packages/av-cliper/src/clips/__tests__/audio-clip.test.ts

@@ -93,3 +93,14 @@ test('seek', async () => {
   const { audio: audio3 } = await clip.tick(11e6);
   expect(audio3[0].length).toBe(DEFAULT_AUDIO_CONF.sampleRate * 2);
 });
+
+test('split by time', async () => {
+  const clip = new AudioClip((await fetch(m4a_44kHz_2chan)).body!);
+  const [preClip1, postClip2] = await clip.split(60e6);
+  expect(Math.round(preClip1.meta.duration / 1e6)).toBe(60);
+  expect(Math.round(postClip2.meta.duration / 1e6)).toBe(62);
+
+  const [preClip11, postClip12] = await preClip1.split(30e6);
+  expect(Math.round(preClip11.meta.duration / 1e6)).toBe(30);
+  expect(Math.round(postClip12.meta.duration / 1e6)).toBe(30);
+});

packages/av-cliper/src/clips/__tests__/embed-subtitles.test.ts

@@ -1,5 +1,4 @@
 import { expect, test } from 'vitest';
-// import '../../__tests__/mock';
 import { EmbedSubtitlesClip } from '../embed-subtitles-clip';
 
 const txt1 = `
@@ -80,3 +79,15 @@ test('EmbedSubtitles digital', async () => {
   expect(vf1?.timestamp).toBe(342000);
   expect(vf1?.duration).toBe(3218000 - 342000);
 });
+
+test('split by time', async () => {
+  const clip = new EmbedSubtitlesClip(txt1, {
+    videoWidth: 1280,
+    videoHeight: 720,
+  });
+  await clip.ready;
+  const [preClip, postClip] = await clip.split(30e6);
+  expect(clip.meta.duration).toBe(
+    preClip.meta.duration + postClip.meta.duration,
+  );
+});

packages/av-cliper/src/clips/__tests__/img-clip.test.ts

@@ -0,0 +1,16 @@
+import { expect, test } from 'vitest';
+import { ImgClip } from '..';
+
+const animated_gif = `//${location.host}/img/animated.gif`;
+
+test('split by time', async () => {
+  const clip = new ImgClip({
+    type: 'image/gif',
+    stream: (await fetch(animated_gif)).body!,
+  });
+  await clip.ready;
+  const [preClip, postClip] = await clip.split(1e6);
+  expect(clip.meta.duration).toBe(
+    preClip.meta.duration + postClip.meta.duration,
+  );
+});

packages/av-cliper/src/clips/__tests__/mp4-clip.test.ts

@@ -115,7 +115,7 @@ test('split track', async () => {
   expect(Math.round(trackClips[1].meta.duration / 1e6)).toBe(21);
 });
 
-test('split when only has video track', async () => {
+test('splitTrack when only has video track', async () => {
   const clip = new MP4Clip((await fetch(mp4_bunny_1)).body!, { audio: false });
   await clip.ready;
   const trackClips = await clip.splitTrack();
@@ -125,3 +125,16 @@ test('split when only has video track', async () => {
   expect(trackClips[0].meta.audioChanCount).toBe(0);
   expect(Math.round(trackClips[0].meta.duration / 1e6)).toBe(21);
 });
+
+test('split MP4Clip by time', async () => {
+  const clip = new MP4Clip((await fetch(mp4_bunny_1)).body!);
+  const [preClip1, postClip2] = await clip.split(10e6);
+  expect(Math.round(preClip1.meta.duration / 1e6)).toEqual(10);
+  expect(Math.round(postClip2.meta.duration / 1e6)).toEqual(11);
+  expect(preClip1.meta.audioChanCount).toBe(2);
+  expect(postClip2.meta.audioChanCount).toBe(2);
+  // second split
+  const [preClip11, postClip12] = await preClip1.split(5e6);
+  expect(Math.round(preClip11.meta.duration / 1e6)).toEqual(5);
+  expect(Math.round(postClip12.meta.duration / 1e6)).toEqual(5);
+});

packages/av-cliper/src/clips/audio-clip.ts

@@ -23,6 +23,14 @@ export class AudioClip implements IClip {
     height: 0,
   };
 
+  get meta() {
+    return {
+      duration: this.#meta.duration,
+      sampleRate: DEFAULT_AUDIO_CONF.sampleRate,
+      chanCount: 2,
+    };
+  }
+
   #chan0Buf = new Float32Array();
   #chan1Buf = new Float32Array();
   getPCMData(): Float32Array[] {
@@ -133,6 +141,20 @@ export class AudioClip implements IClip {
     return { audio, state: 'success' };
   }
 
+  async split(time: number) {
+    await this.ready;
+    const frameCnt = Math.ceil((time / 1e6) * DEFAULT_AUDIO_CONF.sampleRate);
+    const preSlice = new AudioClip(
+      this.getPCMData().map((chan) => chan.slice(0, frameCnt)),
+      this.#opts,
+    );
+    const postSlice = new AudioClip(
+      this.getPCMData().map((chan) => chan.slice(frameCnt)),
+      this.#opts,
+    );
+    return [preSlice, postSlice];
+  }
+
   async clone() {
     return new AudioClip(this.getPCMData(), this.#opts) as this;
   }

packages/av-cliper/src/clips/embed-subtitles-clip.ts

@@ -29,14 +29,26 @@ declare global {
   }
 }
 
+interface SubtitleStruct {
+  start: number;
+  end: number;
+  text: string;
+}
+
 export class EmbedSubtitlesClip implements IClip {
   ready: IClip['ready'];
 
-  #subtitles: Array<{
-    start: number;
-    end: number;
-    text: string;
-  }> = [];
+  #subtitles: SubtitleStruct[] = [];
+
+  #meta = {
+    width: 0,
+    height: 0,
+    duration: 0,
+  };
+
+  get meta() {
+    return { ...this.#meta };
+  }
 
   #opts: Required<IEmbedSubtitlesOpts> = {
     color: '#FFF',
@@ -68,15 +80,14 @@ export class EmbedSubtitlesClip implements IClip {
   #lineHeight = 0;
   #linePadding = 0;
 
-  #content;
-
-  constructor(content: string, opts: IEmbedSubtitlesOpts) {
-    this.#content = content;
-    this.#subtitles = parseSrt(content).map(({ start, end, text }) => ({
-      start: start * 1e6,
-      end: end * 1e6,
-      text,
-    }));
+  constructor(content: string | SubtitleStruct[], opts: IEmbedSubtitlesOpts) {
+    this.#subtitles = Array.isArray(content)
+      ? content
+      : parseSrt(content).map(({ start, end, text }) => ({
+          start: start * 1e6,
+          end: end * 1e6,
+          text,
+        }));
     if (this.#subtitles.length === 0) throw Error('No subtitles content');
 
     this.#opts = Object.assign(this.#opts, opts);
@@ -94,12 +105,13 @@ export class EmbedSubtitlesClip implements IClip {
     this.#ctx.textBaseline = 'top';
     this.#ctx.letterSpacing = letterSpacing ?? '0px';
 
-    // 字幕的宽高 由视频画面内容决定
-    this.ready = Promise.resolve({
+    this.#meta = {
       width: videoWidth,
       height: videoHeight,
       duration: this.#subtitles.at(-1)?.end ?? 0,
-    });
+    };
+    // 字幕的宽高 由视频画面内容决定
+    this.ready = Promise.resolve(this.meta);
   }
 
   #renderTxt(txt: string) {
@@ -225,8 +237,41 @@ export class EmbedSubtitlesClip implements IClip {
     return { video: vf.clone(), state: 'success' };
   }
 
+  async split(time: number) {
+    await this.ready;
+    let hitIdx = -1;
+    for (let i = 0; i < this.#subtitles.length; i++) {
+      const sub = this.#subtitles[i];
+      if (time > sub.start) continue;
+      hitIdx = i;
+      break;
+    }
+    if (hitIdx === -1) throw Error('Not found subtitle by time');
+    const preSlice = this.#subtitles.slice(0, hitIdx).map((s) => ({ ...s }));
+    let preLastIt = preSlice.at(-1);
+    let postFirstIt = null;
+    // 切割时间命中字幕区间，需要将当前字幕元素拆成前后两份
+    if (preLastIt != null && preLastIt.end > time) {
+      postFirstIt = {
+        start: 0,
+        end: preLastIt.end - time,
+        text: preLastIt.text,
+      };
+
+      preLastIt.end = time;
+    }
+    const postSlice = this.#subtitles
+      .slice(hitIdx)
+      .map((s) => ({ ...s, start: s.start - time, end: s.end - time }));
+    if (postFirstIt != null) postSlice.unshift(postFirstIt);
+    return [
+      new EmbedSubtitlesClip(preSlice, this.#opts),
+      new EmbedSubtitlesClip(postSlice, this.#opts),
+    ] as this[];
+  }
+
   async clone() {
-    return new EmbedSubtitlesClip(this.#content, this.#opts) as this;
+    return new EmbedSubtitlesClip(this.#subtitles.slice(0), this.#opts) as this;
   }
 
   destroy() {

packages/av-cliper/src/clips/img-clip.ts

@@ -2,6 +2,8 @@ import { decodeImg } from '../av-utils';
 import { Log } from '../log';
 import { IClip } from './iclip';
 
+type AnimateImgType = 'avif' | 'webp' | 'png' | 'gif';
+
 export class ImgClip implements IClip {
   ready: IClip['ready'];
 
@@ -12,6 +14,10 @@ export class ImgClip implements IClip {
     height: 0,
   };
 
+  get meta() {
+    return { ...this.#meta };
+  }
+
   #img: ImageBitmap | null = null;
 
   #frames: VideoFrame[] = [];
@@ -20,7 +26,7 @@ export class ImgClip implements IClip {
     dataSource:
       | ImageBitmap
       | VideoFrame[]
-      | { type: 'image/gif'; stream: ReadableStream },
+      | { type: `image/${AnimateImgType}`; stream: ReadableStream },
   ) {
     if (dataSource instanceof ImageBitmap) {
       this.#img = dataSource;
@@ -45,17 +51,21 @@ export class ImgClip implements IClip {
       };
       this.ready = Promise.resolve({ ...this.#meta, duration: Infinity });
     } else {
-      this.ready = this.#gifInit(dataSource.stream, dataSource.type).then(
-        () => ({
-          width: this.#meta.width,
-          height: this.#meta.height,
-          duration: Infinity,
-        }),
-      );
+      this.ready = this.#initAnimateImg(
+        dataSource.stream,
+        dataSource.type,
+      ).then(() => ({
+        width: this.#meta.width,
+        height: this.#meta.height,
+        duration: Infinity,
+      }));
     }
   }
 
-  async #gifInit(stream: ReadableStream, type: string) {
+  async #initAnimateImg(
+    stream: ReadableStream,
+    type: `image/${AnimateImgType}`,
+  ) {
     this.#frames = await decodeImg(stream, type);
     const firstVf = this.#frames[0];
     if (firstVf == null) throw Error('No frame available in gif');
@@ -90,6 +100,31 @@ export class ImgClip implements IClip {
     };
   }
 
+  async split(time: number) {
+    await this.ready;
+    if (this.#img != null) {
+      return [new ImgClip(this.#img), new ImgClip(this.#img)];
+    }
+    let hitIdx = -1;
+    for (let i = 0; i < this.#frames.length; i++) {
+      const vf = this.#frames[i];
+      if (time > vf.timestamp) continue;
+      hitIdx = i;
+      break;
+    }
+    if (hitIdx === -1) throw Error('Not found frame by time');
+    const preSlice = this.#frames
+      .slice(0, hitIdx)
+      .map((vf) => new VideoFrame(vf));
+    const postSlice = this.#frames.slice(hitIdx).map(
+      (vf) =>
+        new VideoFrame(vf, {
+          timestamp: vf.timestamp - time,
+        }),
+    );
+    return [new ImgClip(preSlice), new ImgClip(postSlice)] as this[];
+  }
+
   async clone() {
     await this.ready;
     return new ImgClip(this.#img ?? this.#frames) as this;