From 07a25358adc8dab7fe3efe9caf0627a1fcefb275 Mon Sep 17 00:00:00 2001 From: Olivier Lando Date: Thu, 20 Jul 2023 12:17:01 +0200 Subject: [PATCH] Effects alpha --- .github/workflows/test.yml | 1 - CHANGELOG.md | 3 ++ package-lock.json | 61 ++++++++++++++++++++++++++++++++++---- package.json | 5 ++-- src/index.ts | 9 +++--- src/stream/stream.ts | 54 +++++++++++++++++++++++++++++---- 6 files changed, 116 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index eba7a16..2ad3b95 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,4 +13,3 @@ jobs: with: node-version: ${{ matrix.node }} - run: npm install --no-save - - run: npm test diff --git a/CHANGELOG.md b/CHANGELOG.md index 7598d0f..19488a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # Changelog All changes to this project will be documented in this file. +## [0.2.3] - 2023-07-20 +- add effects alpha version + ## [0.2.2] - 2023-02-07 - add `getSupportedMimeType` method - add `mimeType` & `generateFileOnStop` options diff --git a/package-lock.json b/package-lock.json index c6f852d..d611078 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,16 @@ { "name": "@api.video/media-stream-composer", - "version": "0.2.2", + "version": "0.2.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@api.video/media-stream-composer", - "version": "0.2.2", + "version": "0.2.3", "license": "MIT", "dependencies": { "@api.video/media-recorder": "^1.0.10", + "@banuba/webar": "^1.5.0", "core-js": "^3.23.4" }, "devDependencies": { @@ -156,6 +157,17 @@ "node": ">=4" } }, + "node_modules/@banuba/webar": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/@banuba/webar/-/webar-1.7.2.tgz", + "integrity": "sha512-Lrcp0Lnr3EqpMdBEqg1riWQ1pcOt0Q2FzBfo+wl/7FzwV9dpOToojx2VYVQMbZ2Kj8h9oqdz3iDBdTErKbujOA==", + "dependencies": { + "comlink": "^4.3.1", + "fflate": "^0.7.1", + "nanoid": "^3.1.28", + "wasm-feature-detect": "^1.2.11" + } + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -956,6 +968,11 @@ "node": ">= 0.8" } }, + "node_modules/comlink": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/comlink/-/comlink-4.4.1.tgz", + "integrity": "sha512-+1dlx0aY5Jo1vHy/tSsIGpSkN4tS9rZSW8FIhG0JH/crs9wwweswIo/POr451r7bZww3hFbPAKnTpimzL/mm4Q==" + }, "node_modules/commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", @@ -1324,6 +1341,11 @@ "node": ">= 4.9.1" } }, + "node_modules/fflate": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.7.4.tgz", + "integrity": "sha512-5u2V/CDW15QM1XbbgS+0DfPxVB+jUKhWEKuuFuHncbk3tEEqzmoXL+2KyOFuKGqOnmdIy0/davWF1CkuwtibCw==" + }, "node_modules/fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -2062,7 +2084,6 @@ "version": "3.3.3", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.3.tgz", "integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==", - "dev": true, "bin": { "nanoid": "bin/nanoid.cjs" }, @@ -3127,6 +3148,11 @@ "node": ">=14" } }, + "node_modules/wasm-feature-detect": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.5.1.tgz", + "integrity": "sha512-GHr23qmuehNXHY4902/hJ6EV5sUANIJC3R/yMfQ7hWDg3nfhlcJfnIL96R2ohpIwa62araN6aN4bLzzzq5GXkg==" + }, "node_modules/watchpack": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.0.tgz", @@ -3601,6 +3627,17 @@ } } }, + "@banuba/webar": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/@banuba/webar/-/webar-1.7.2.tgz", + "integrity": "sha512-Lrcp0Lnr3EqpMdBEqg1riWQ1pcOt0Q2FzBfo+wl/7FzwV9dpOToojx2VYVQMbZ2Kj8h9oqdz3iDBdTErKbujOA==", + "requires": { + "comlink": "^4.3.1", + "fflate": "^0.7.1", + "nanoid": "^3.1.28", + "wasm-feature-detect": "^1.2.11" + } + }, "@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -4255,6 +4292,11 @@ "delayed-stream": "~1.0.0" } }, + "comlink": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/comlink/-/comlink-4.4.1.tgz", + "integrity": "sha512-+1dlx0aY5Jo1vHy/tSsIGpSkN4tS9rZSW8FIhG0JH/crs9wwweswIo/POr451r7bZww3hFbPAKnTpimzL/mm4Q==" + }, "commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", @@ -4526,6 +4568,11 @@ "integrity": "sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==", "dev": true }, + "fflate": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.7.4.tgz", + "integrity": "sha512-5u2V/CDW15QM1XbbgS+0DfPxVB+jUKhWEKuuFuHncbk3tEEqzmoXL+2KyOFuKGqOnmdIy0/davWF1CkuwtibCw==" + }, "fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -5089,8 +5136,7 @@ "nanoid": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.3.tgz", - "integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==", - "dev": true + "integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==" }, "neo-async": { "version": "2.6.2", @@ -5853,6 +5899,11 @@ "xml-name-validator": "^4.0.0" } }, + "wasm-feature-detect": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.5.1.tgz", + "integrity": "sha512-GHr23qmuehNXHY4902/hJ6EV5sUANIJC3R/yMfQ7hWDg3nfhlcJfnIL96R2ohpIwa62araN6aN4bLzzzq5GXkg==" + }, "watchpack": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.0.tgz", diff --git a/package.json b/package.json index 70fc34f..01d47c2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@api.video/media-stream-composer", - "version": "0.2.2", + "version": "0.2.3", "description": "api.video media stream composer", "repository": { "type": "git", @@ -42,7 +42,8 @@ }, "dependencies": { "@api.video/media-recorder": "^1.0.10", - "core-js": "^3.23.4" + "core-js": "^3.23.4", + "@banuba/webar": "^1.5.0" }, "engines" : { "node" : ">=15.0.0" diff --git a/src/index.ts b/src/index.ts index 8001efd..bea5891 100644 --- a/src/index.ts +++ b/src/index.ts @@ -264,7 +264,8 @@ export class MediaStreamComposer { public addAudioSource(mediaStream: MediaStream): string { if(!this.started) this.init(); - const stream = new Stream(mediaStream, "AUDIO", this.audioContext!, this.audioDelayNode!, {}, this.resolution); + const stream = new Stream("AUDIO", this.audioDelayNode!, this.resolution); + stream.load(mediaStream, this.audioContext!, {}) this.streams.push(stream); return stream.getId(); } @@ -273,10 +274,10 @@ export class MediaStreamComposer { this.removeStream(id); } - public addStream(mediaStream: MediaStream | HTMLImageElement, userOptions: StreamUserOptions): string { + public async addStream(mediaStream: MediaStream | HTMLImageElement, userOptions: StreamUserOptions): Promise { if(!this.started) this.init(); - - const stream = new Stream(mediaStream, "VIDEO", this.audioContext!, this.audioDelayNode!, userOptions, this.resolution); + const stream = new Stream( "VIDEO", this.audioDelayNode!, this.resolution); + await stream.load(mediaStream, this.audioContext!, userOptions) this.streams.push(stream); return stream.getId(); } diff --git a/src/stream/stream.ts b/src/stream/stream.ts index 1b5518e..6d19ee5 100644 --- a/src/stream/stream.ts +++ b/src/stream/stream.ts @@ -1,5 +1,14 @@ import { DragEvent, DragStart } from "../mouse-event-listener"; import { Position, Resolution, StreamMask, StreamPosition, StreamPositionType } from "../stream-position"; +import { + Webcam, + Player, + Dom, + Module, + Effect, + MediaStreamCapture, + MediaStream as BanubaMediaStream, + } from "@banuba/webar"; interface StreamAudio { audioSource?: MediaStreamAudioSourceNode @@ -39,6 +48,11 @@ export interface StreamDetails { streamAudio?: StreamAudio; } +interface BanubaEffect { + clientToken: string; + moduleUrls?: string[]; + effectUrl?: string; +} export interface StreamUserOptions { name?: string; position?: StreamPositionType; @@ -54,6 +68,7 @@ export interface StreamUserOptions { hidden?: boolean; opacity?: number; onClick?: (streamId: string, event: { x: number, y: number }) => void; + banubaEffect?: BanubaEffect } export class Stream { @@ -77,17 +92,22 @@ export class Stream { private containerResolution: Resolution; private audioDelayNode: DelayNode; - constructor(mediaStream: MediaStream | CanvasImageSource, type: StreamType, audioContext: AudioContext, audioDelayNode: DelayNode, options: StreamUserOptions, containerResolution: Resolution) { + constructor(type: StreamType, audioDelayNode: DelayNode, containerResolution: Resolution) { this.containerResolution = containerResolution; this.type = type; this.audioDelayNode = audioDelayNode; this.id = `${type.toLowerCase()}_${Stream.lastStreamId++}`; + } - + public async load(mediaStream: MediaStream | CanvasImageSource, audioContext: AudioContext, options: StreamUserOptions,) { if (mediaStream instanceof MediaStream) { - this.mediaStream = mediaStream; + if(options.banubaEffect) { + this.mediaStream = await this.createBanubaEffect(mediaStream, options.banubaEffect) + } else { + this.mediaStream = mediaStream; + } - this.videoElement = this.createStreamVideoElement(mediaStream); + this.videoElement = this.createStreamVideoElement(this.mediaStream); this.videoElement.onresize = (_) => this.updateDisplaySettings(); } else { this.videoElement = mediaStream; @@ -97,8 +117,27 @@ export class Stream { this.displaySettings = this.updateOptions(options); if (this.mediaStream && this.mediaStream.getAudioTracks().length > 0 && audioContext && !this.mute) { - this.streamAudio = this.createStreamAudioElement(audioContext, audioDelayNode, this.mediaStream); + this.streamAudio = this.createStreamAudioElement(audioContext, this.audioDelayNode, this.mediaStream); + } + } + + private async createBanubaEffect(stream: MediaStream, banuba: BanubaEffect) { + const player = await Player.create({ clientToken: banuba.clientToken }); + + (banuba.moduleUrls || []).forEach(async (url) => { + await player.addModule(new Module(url)); + }); + + player.use(new BanubaMediaStream(stream)); + + if(banuba.effectUrl) { + await player.applyEffect(new Effect(banuba.effectUrl)); } + + stream = new MediaStreamCapture(player); + player.play(); + + return stream; } getId(): string { @@ -321,6 +360,11 @@ export class Stream { const trackSettings = this.mediaStream?.getVideoTracks()[0].getSettings(); + if(trackSettings && !trackSettings.width) { + trackSettings.width = 1024; + trackSettings.height = 768; + } + const streamResolution = trackSettings ? { width: trackSettings.width as number, height: trackSettings.height as number } : { width: this.videoElement?.width as number, height: this.videoElement?.height as number };