From 7479f2f697b6bd21b544d5e9ebfa7f1723d27c06 Mon Sep 17 00:00:00 2001 From: Su-Yong Date: Wed, 4 Oct 2023 20:28:14 +0900 Subject: [PATCH 1/3] feat(ambient-mode): add ambient-mode plugin --- config/defaults.ts | 1 + index.ts | 2 + plugins/ambient-mode/back.ts | 10 ++++ plugins/ambient-mode/front.ts | 83 ++++++++++++++++++++++++++++++++++ plugins/ambient-mode/style.css | 7 +++ preload.ts | 2 + 6 files changed, 105 insertions(+) create mode 100644 plugins/ambient-mode/back.ts create mode 100644 plugins/ambient-mode/front.ts create mode 100644 plugins/ambient-mode/style.css diff --git a/config/defaults.ts b/config/defaults.ts index ee2e0c3c4..08369660f 100644 --- a/config/defaults.ts +++ b/config/defaults.ts @@ -77,6 +77,7 @@ const defaultConfig = { disableDefaultLists: [], }, 'album-color-theme': {}, + 'ambient-mode': {}, 'audio-compressor': {}, 'blur-nav-bar': {}, 'bypass-age-restrictions': {}, diff --git a/index.ts b/index.ts index 3af1f805d..c81229ee4 100644 --- a/index.ts +++ b/index.ts @@ -20,6 +20,7 @@ import { APP_PROTOCOL, handleProtocol, setupProtocolHandler } from './providers/ import adblocker from './plugins/adblocker/back'; import albumColorTheme from './plugins/album-color-theme/back'; +import ambientMode from './plugins/ambient-mode/back'; import blurNavigationBar from './plugins/blur-nav-bar/back'; import captionsSelector from './plugins/captions-selector/back'; import crossfade from './plugins/crossfade/back'; @@ -103,6 +104,7 @@ function onClosed() { const mainPlugins = { 'adblocker': adblocker, 'album-color-theme': albumColorTheme, + 'ambient-mode': ambientMode, 'blur-nav-bar': blurNavigationBar, 'captions-selector': captionsSelector, 'crossfade': crossfade, diff --git a/plugins/ambient-mode/back.ts b/plugins/ambient-mode/back.ts new file mode 100644 index 000000000..0f5015152 --- /dev/null +++ b/plugins/ambient-mode/back.ts @@ -0,0 +1,10 @@ +import { BrowserWindow } from 'electron'; + +import style from './style.css'; + +import { injectCSS } from '../utils'; + + +export default (win: BrowserWindow) => { + injectCSS(win.webContents, style); +}; diff --git a/plugins/ambient-mode/front.ts b/plugins/ambient-mode/front.ts new file mode 100644 index 000000000..b7b84e55e --- /dev/null +++ b/plugins/ambient-mode/front.ts @@ -0,0 +1,83 @@ +import { ConfigType } from '../../config/dynamic'; + +export default (_: ConfigType<'ambient-mode'>) => { + let unregister: (() => void) | null = null; + + const injectBlurVideo = (): (() => void) | null => { + const songVideo = document.querySelector('#song-video'); + const video = document.querySelector('#song-video .html5-video-container > video'); + const wrapper = document.querySelector('#song-video > .player-wrapper'); + + if (!songVideo) return null; + if (!video) return null; + if (!wrapper) return null; + + const blurCanvas = document.createElement('canvas'); + blurCanvas.classList.add('html5-blur-canvas'); + + const context = blurCanvas.getContext('2d'); + + const applyVideoAttributes = () => { + const rect = video.getBoundingClientRect(); + + blurCanvas.width = video.width || rect.width; + blurCanvas.height = video.height || rect.height; + }; + + const onSync = () => { + requestAnimationFrame(() => { + context?.drawImage(video, 0, 0, blurCanvas.width, blurCanvas.height); + }); + }; + + const observer = new MutationObserver((mutations) => { + mutations.forEach((mutation) => { + if (mutation.type === 'attributes') { + applyVideoAttributes(); + } + }); + }); + const resizeObserver = new ResizeObserver(() => { + applyVideoAttributes(); + }); + + /* hooking */ + video.addEventListener('timeupdate', onSync); + + applyVideoAttributes(); + observer.observe(songVideo, { attributes: true }); + resizeObserver.observe(songVideo); + + /* injecting */ + wrapper.prepend(blurCanvas); + + /* cleanup */ + return () => { + video.removeEventListener('timeupdate', onSync); + observer.disconnect(); + resizeObserver.disconnect(); + + wrapper.removeChild(blurCanvas); + }; + }; + + + const playerPage = document.querySelector('#player-page'); + const ytmusicAppLayout = document.querySelector('#layout'); + + const observer = new MutationObserver((mutationsList) => { + for (const mutation of mutationsList) { + if (mutation.type === 'attributes') { + const isPageOpen = ytmusicAppLayout?.hasAttribute('player-page-open'); + if (isPageOpen) { + unregister?.(); + unregister = injectBlurVideo() ?? null; + } + } + } + }); + + if (playerPage) { + observer.observe(playerPage, { attributes: true }); + } +}; \ No newline at end of file diff --git a/plugins/ambient-mode/style.css b/plugins/ambient-mode/style.css new file mode 100644 index 000000000..44f901648 --- /dev/null +++ b/plugins/ambient-mode/style.css @@ -0,0 +1,7 @@ +#song-video canvas.html5-blur-canvas{ + position: absolute; + left: 0; + top: 0; + + filter: blur(100px); +} diff --git a/preload.ts b/preload.ts index c2e0f74cd..1886958bc 100644 --- a/preload.ts +++ b/preload.ts @@ -7,6 +7,7 @@ import { setupSongControls } from './providers/song-controls-front'; import { startingPages } from './providers/extracted-data'; import albumColorThemeRenderer from './plugins/album-color-theme/front'; +import ambientModeRenderer from './plugins/ambient-mode/front'; import audioCompressorRenderer from './plugins/audio-compressor/front'; import bypassAgeRestrictionsRenderer from './plugins/bypass-age-restrictions/front'; import captionsSelectorRenderer from './plugins/captions-selector/front'; @@ -43,6 +44,7 @@ type PluginMapper = { const rendererPlugins: PluginMapper<'renderer'> = { 'album-color-theme': albumColorThemeRenderer, + 'ambient-mode': ambientModeRenderer, 'audio-compressor': audioCompressorRenderer, 'bypass-age-restrictions': bypassAgeRestrictionsRenderer, 'captions-selector': captionsSelectorRenderer, From 81b2303a6fa1dabddf6551986f9f8ee45ff9f1b0 Mon Sep 17 00:00:00 2001 From: Su-Yong Date: Wed, 4 Oct 2023 22:22:41 +0900 Subject: [PATCH 2/3] feat(ambient-mode): add ambient effect interpolation --- plugins/ambient-mode/front.ts | 80 ++++++++++++++++++++++++++++++----- 1 file changed, 69 insertions(+), 11 deletions(-) diff --git a/plugins/ambient-mode/front.ts b/plugins/ambient-mode/front.ts index b7b84e55e..567460b10 100644 --- a/plugins/ambient-mode/front.ts +++ b/plugins/ambient-mode/front.ts @@ -1,6 +1,11 @@ import { ConfigType } from '../../config/dynamic'; export default (_: ConfigType<'ambient-mode'>) => { + const interpolationLength = 3000; + const framerate = 30; + const interpolationFrame = (interpolationLength / 1000) * framerate; + const qualityRatio = 50; // width size + let unregister: (() => void) | null = null; const injectBlurVideo = (): (() => void) | null => { @@ -15,19 +20,55 @@ export default (_: ConfigType<'ambient-mode'>) => { const blurCanvas = document.createElement('canvas'); blurCanvas.classList.add('html5-blur-canvas'); - const context = blurCanvas.getContext('2d'); + const context = blurCanvas.getContext('2d', { willReadFrequently: true }); + + /* effect */ + let lastEffectWorkId: number | null = null; + const imageData: (ImageData | undefined)[] = []; + + const onSync = () => { + if (typeof lastEffectWorkId === 'number') cancelAnimationFrame(lastEffectWorkId); + + lastEffectWorkId = requestAnimationFrame(() => { + if (!context) return; + + context.globalAlpha = 1 / interpolationFrame; + const width = qualityRatio; + let height = Math.max(Math.floor(blurCanvas.height / blurCanvas.width * width), 1); + if (!Number.isFinite(height)) height = width; + + context.drawImage(video, 0, 0, width, height); + + const nowImageData = context.getImageData(0, 0, width, height); + if (nowImageData) { + imageData.unshift( + new ImageData( + new Uint8ClampedArray(nowImageData.data), + nowImageData.width, + nowImageData.height + ), + ); + } + imageData.length = framerate; + + for (let i = 1; i < interpolationFrame; i += 1) { + context.putImageData(imageData[i] ?? imageData[0]!, 0, 0); + } + + lastEffectWorkId = null; + }); + }; const applyVideoAttributes = () => { const rect = video.getBoundingClientRect(); - blurCanvas.width = video.width || rect.width; - blurCanvas.height = video.height || rect.height; - }; + const newWidth = Math.floor(video.width || rect.width); + const newHeight = Math.floor(video.height || rect.height); - const onSync = () => { - requestAnimationFrame(() => { - context?.drawImage(video, 0, 0, blurCanvas.width, blurCanvas.height); - }); + blurCanvas.width = qualityRatio; + blurCanvas.height = Math.floor(newHeight / newWidth * qualityRatio); + blurCanvas.style.width = `${newWidth}px`; + blurCanvas.style.height = `${newHeight}px`; }; const observer = new MutationObserver((mutations) => { @@ -42,20 +83,37 @@ export default (_: ConfigType<'ambient-mode'>) => { }); /* hooking */ - video.addEventListener('timeupdate', onSync); - + let canvasInterval: NodeJS.Timeout | null = null; + canvasInterval = setInterval(onSync, Math.max(1, Math.ceil(interpolationLength / interpolationFrame))); applyVideoAttributes(); observer.observe(songVideo, { attributes: true }); resizeObserver.observe(songVideo); + window.addEventListener('resize', applyVideoAttributes); + + const onPause = () => { + if (canvasInterval) clearInterval(canvasInterval); + canvasInterval = null; + }; + const onPlay = () => { + if (canvasInterval) clearInterval(canvasInterval); + canvasInterval = setInterval(onSync, Math.max(1, Math.ceil(interpolationLength / interpolationFrame))); + }; + songVideo.addEventListener('pause', onPause); + songVideo.addEventListener('play', onPlay); /* injecting */ wrapper.prepend(blurCanvas); /* cleanup */ return () => { - video.removeEventListener('timeupdate', onSync); + if (canvasInterval) clearInterval(canvasInterval); + + songVideo.removeEventListener('pause', onPause); + songVideo.removeEventListener('play', onPlay); + observer.disconnect(); resizeObserver.disconnect(); + window.removeEventListener('resize', applyVideoAttributes); wrapper.removeChild(blurCanvas); }; From 0c948d5ea10f6142c7532a25f0c89b3f55ba1cbf Mon Sep 17 00:00:00 2001 From: Su-Yong Date: Wed, 4 Oct 2023 23:43:45 +0900 Subject: [PATCH 3/3] feat(ambient-mode): improve performance --- plugins/ambient-mode/front.ts | 41 ++++++++++++++++------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/plugins/ambient-mode/front.ts b/plugins/ambient-mode/front.ts index 567460b10..253d32b03 100644 --- a/plugins/ambient-mode/front.ts +++ b/plugins/ambient-mode/front.ts @@ -1,10 +1,9 @@ import { ConfigType } from '../../config/dynamic'; export default (_: ConfigType<'ambient-mode'>) => { - const interpolationLength = 3000; - const framerate = 30; - const interpolationFrame = (interpolationLength / 1000) * framerate; - const qualityRatio = 50; // width size + const interpolationTime = 3000; // interpolation time (ms) + const framerate = 30; // frame + const qualityRatio = 50; // width size (pixel) let unregister: (() => void) | null = null; @@ -24,7 +23,7 @@ export default (_: ConfigType<'ambient-mode'>) => { /* effect */ let lastEffectWorkId: number | null = null; - const imageData: (ImageData | undefined)[] = []; + let lastImageData: ImageData | null = null; const onSync = () => { if (typeof lastEffectWorkId === 'number') cancelAnimationFrame(lastEffectWorkId); @@ -32,28 +31,21 @@ export default (_: ConfigType<'ambient-mode'>) => { lastEffectWorkId = requestAnimationFrame(() => { if (!context) return; - context.globalAlpha = 1 / interpolationFrame; const width = qualityRatio; let height = Math.max(Math.floor(blurCanvas.height / blurCanvas.width * width), 1); if (!Number.isFinite(height)) height = width; + context.globalAlpha = 1; + if (lastImageData) { + const frameOffset = (1 / framerate) * (1000 / interpolationTime); + context.globalAlpha = 1 - (frameOffset * 2); // because of alpha value must be < 1 + context.putImageData(lastImageData, 0, 0); + context.globalAlpha = frameOffset; + } context.drawImage(video, 0, 0, width, height); const nowImageData = context.getImageData(0, 0, width, height); - if (nowImageData) { - imageData.unshift( - new ImageData( - new Uint8ClampedArray(nowImageData.data), - nowImageData.width, - nowImageData.height - ), - ); - } - imageData.length = framerate; - - for (let i = 1; i < interpolationFrame; i += 1) { - context.putImageData(imageData[i] ?? imageData[0]!, 0, 0); - } + lastImageData = nowImageData; lastEffectWorkId = null; }); @@ -65,6 +57,8 @@ export default (_: ConfigType<'ambient-mode'>) => { const newWidth = Math.floor(video.width || rect.width); const newHeight = Math.floor(video.height || rect.height); + if (newWidth === 0 || newHeight === 0) return; + blurCanvas.width = qualityRatio; blurCanvas.height = Math.floor(newHeight / newWidth * qualityRatio); blurCanvas.style.width = `${newWidth}px`; @@ -84,7 +78,7 @@ export default (_: ConfigType<'ambient-mode'>) => { /* hooking */ let canvasInterval: NodeJS.Timeout | null = null; - canvasInterval = setInterval(onSync, Math.max(1, Math.ceil(interpolationLength / interpolationFrame))); + canvasInterval = setInterval(onSync, Math.max(1, Math.ceil(1000 / framerate))); applyVideoAttributes(); observer.observe(songVideo, { attributes: true }); resizeObserver.observe(songVideo); @@ -96,7 +90,7 @@ export default (_: ConfigType<'ambient-mode'>) => { }; const onPlay = () => { if (canvasInterval) clearInterval(canvasInterval); - canvasInterval = setInterval(onSync, Math.max(1, Math.ceil(interpolationLength / interpolationFrame))); + canvasInterval = setInterval(onSync, Math.max(1, Math.ceil(1000 / framerate))); }; songVideo.addEventListener('pause', onPause); songVideo.addEventListener('play', onPlay); @@ -130,6 +124,9 @@ export default (_: ConfigType<'ambient-mode'>) => { if (isPageOpen) { unregister?.(); unregister = injectBlurVideo() ?? null; + } else { + unregister?.(); + unregister = null; } } }