From 98b6bc5e80fdb669867081e7f8177b454a0e1abf Mon Sep 17 00:00:00 2001 From: Dong Nguyen Date: Mon, 24 Jun 2024 23:58:11 +0700 Subject: [PATCH] Add more utils & test cases --- CONTRIBUTING.md | 23 +- README.md | 219 +- SECURITY.md | 6 +- deno.json | 43 + mod.ts | 9 + scripts/build_npm.ts | 34 + tests/auto_discovery_test.ts | 64 + tests/linter_test.ts | 51 + tests/provider_test.ts | 94 + tests/test_data/bitchute.html | 74 + tests/test_data/bitchute.json | 1 + tests/test_data/facebook.json | 13 + tests/test_data/flickr-default.json | 22 + tests/test_data/flickr-sizelimit.json | 22 + tests/test_data/flickr_2iYctUr.json | 22 + tests/test_data/flickr_2iYctUr_640x480.json | 22 + tests/test_data/instagram_ic7kRDqOlt.json | 11 + tests/test_data/twitter-dark.json | 13 + tests/test_data/twitter.json | 13 + tests/test_data/youtube.json | 15 + tests/test_data/youtube_ciS8aCrX-9s.json | 15 + utils/auto_discovery.ts | 43 + utils/linker.ts | 15 + utils/provider.ts | 152 ++ utils/providers.latest.ts | 2179 +++++++++++++++++++ utils/retrieve.ts | 92 + 26 files changed, 3176 insertions(+), 91 deletions(-) create mode 100644 deno.json create mode 100644 mod.ts create mode 100644 scripts/build_npm.ts create mode 100644 tests/auto_discovery_test.ts create mode 100644 tests/linter_test.ts create mode 100644 tests/provider_test.ts create mode 100644 tests/test_data/bitchute.html create mode 100644 tests/test_data/bitchute.json create mode 100644 tests/test_data/facebook.json create mode 100644 tests/test_data/flickr-default.json create mode 100644 tests/test_data/flickr-sizelimit.json create mode 100644 tests/test_data/flickr_2iYctUr.json create mode 100644 tests/test_data/flickr_2iYctUr_640x480.json create mode 100644 tests/test_data/instagram_ic7kRDqOlt.json create mode 100644 tests/test_data/twitter-dark.json create mode 100644 tests/test_data/twitter.json create mode 100644 tests/test_data/youtube.json create mode 100644 tests/test_data/youtube_ciS8aCrX-9s.json create mode 100644 utils/auto_discovery.ts create mode 100644 utils/linker.ts create mode 100644 utils/provider.ts create mode 100644 utils/providers.latest.ts create mode 100644 utils/retrieve.ts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f48652c..cca90ce 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,27 +2,28 @@ Glad to see you here. -Collaborations and pull requests are always welcomed, though larger proposals should be discussed first. - -As an OSS, it's better to follow the Unix philosophy: "do one thing and do it well". +Collaborations and pull requests are always welcomed, though larger proposals +should be discussed first. +As an OSS, it's better to follow the Unix philosophy: "do one thing and do it +well". ## Third-party libraries -Please avoid using libaries other than those available in the standard library, unless necessary. - -This library needs to be simple and flexible to run on multiple platforms such as Deno, Bun, or even browser. +Please avoid using libaries other than those available in the standard library, +unless necessary. +This library needs to be simple and flexible to run on multiple platforms such +as Deno, Bun, or even browser. ## Documentation If you've changed APIs, please update README. - ## Clean commit histories -When you open a pull request, please ensure the commit history is clean. -Squash the commits into logical blocks, perhaps a single commit if that makes sense. +When you open a pull request, please ensure the commit history is clean. Squash +the commits into logical blocks, perhaps a single commit if that makes sense. What you want to avoid is commits such as "WIP" and "fix test" in the history. This is so we keep history on master clean and straightforward. @@ -32,9 +33,9 @@ For people new to git, please refer the following guides: - [Writing good commit messages](https://github.com/erlang/otp/wiki/writing-good-commit-messages) - [Commit Message Guidelines](https://gist.github.com/robertpainsi/b632364184e70900af4ab688decf6f53) - ## License -By contributing to `@extractus/oembed-extractor`, you agree that your contributions will be licensed under its [MIT license](LICENSE). +By contributing to `@extractus/oembed-extractor`, you agree that your +contributions will be licensed under its [MIT license](LICENSE). --- diff --git a/README.md b/README.md index 22165fb..892cf2f 100644 --- a/README.md +++ b/README.md @@ -1,54 +1,99 @@ - # oembed-extractor Extract oEmbed content from given URL. -[![NPM](https://badge.fury.io/js/@extractus%2Foembed-extractor.svg)](https://badge.fury.io/js/@extractus%2Foembed-extractor) ![CodeQL](https://github.com/extractus/oembed-extractor/workflows/CodeQL/badge.svg) [![CI test](https://github.com/extractus/oembed-extractor/workflows/ci-test/badge.svg)](https://github.com/extractus/oembed-extractor/actions) [![Coverage Status](https://coveralls.io/repos/github/extractus/oembed-extractor/badge.svg)](https://coveralls.io/github/extractus/oembed-extractor) +[![NPM](https://img.shields.io/npm/v/%40extractus%2Foembed-extractor?color=32bb24)](https://www.npmjs.com/package/@extractus/oembed-extractor) +[![JSR](https://jsr.io/badges/@extractus/oembed-extractor?color=32bb24)](https://jsr.io/@extractus/oembed-extractor) ## Demo - [Give it a try!](https://extractor-demos.pages.dev/oembed-extractor) - [Example FaaS](https://extractus.deno.dev/extract?apikey=rn0wbHos2e73W6ghQf705bdF&type=oembed&url=https://www.instagram.com/tv/CVlR5GFqF68/) +## Setup & Usage -## Install & Usage - -### Node.js +### Deno -```bash -npm i @extractus/oembed-extractor +https://jsr.io/@extractus/oembed-extractor -# pnpm -pnpm i @extractus/oembed-extractor +```ts +import { extract } from "@extractus/oembed-extractor"; -# yarn -yarn add @extractus/oembed-extractor +const result = await extract("https://www.youtube.com/watch?v=x2bqscVkGxk"); +console.log(result); ``` +You can use JSR packages without an install step using `jsr:` specifiers: + ```ts -import { extract } from '@extractus/oembed-extractor' +import { extract } from "jsr:@extractus/oembed-extractor"; -const result = await extract('https://www.youtube.com/watch?v=x2bqscVkGxk') -console.log(result) +const result = await extract("https://www.youtube.com/watch?v=x2bqscVkGxk"); +console.log(result); ``` -### Deno +You can also use `npm:` specifiers as before: ```ts -import { extract } from 'npm:@extractus/oembed-extractor' +import { extract } from "npm:@extractus/oembed-extractor"; + +const result = await extract("https://www.youtube.com/watch?v=x2bqscVkGxk"); +console.log(result); ``` -### Browser +Or import from esm.sh ```ts -import { extract } from "https://esm.sh/@extractus/oembed-extractor@latest" +import { extract } from "https://esm.sh/@extractus/oembed-extractor"; + +const result = await extract("https://www.youtube.com/watch?v=x2bqscVkGxk"); +console.log(result); +``` + +### Node.js & Bun + +https://www.npmjs.com/package/@extractus/oembed-extractor + +```bash +npm i @extractus/oembed-extractor +# pnpm +pnpm i @extractus/oembed-extractor +# yarn +yarn add @extractus/oembed-extractor +# bun +bun add @extractus/oembed-extractor +``` + +```js +import { extract } from "@extractus/oembed-extractor"; + +const result = await extract("https://www.youtube.com/watch?v=x2bqscVkGxk"); +console.log(result); +``` + +You can also use CJS style: + +```js +const { extract } = require("@extractus/oembed-extractor"); + +const result = await extract("https://www.youtube.com/watch?v=x2bqscVkGxk"); +console.log(result); ``` -Please check [the examples](examples) for reference. +### Browsers: +```html + +``` ## APIs @@ -66,13 +111,15 @@ extract(String url, Object params, Object fetchOptions) #### Parameters -##### `url` *required* +##### `url` _required_ -URL of a valid oEmbed resource, e.g. `https://www.youtube.com/watch?v=x2bqscVkGxk` +URL of a valid oEmbed resource, e.g. +`https://www.youtube.com/watch?v=x2bqscVkGxk` -##### `params` *optional* +##### `params` _optional_ -Optional argument `params` can be useful when you want to specify some additional customizations. +Optional argument `params` can be useful when you want to specify some +additional customizations. Here are several popular params: @@ -84,88 +131,96 @@ Here are several popular params: Note that some params are supported by these providers but not by the others. Please see the provider's oEmbed API docs carefully for exact information. -##### `fetchOptions` *optional* +##### `fetchOptions` _optional_ `fetchOptions` is an object that can have the following properties: - `headers`: to set request headers - `proxy`: another endpoint to forward the request to - `agent`: a HTTP proxy agent -- `signal`: AbortController signal or AbortSignal timeout to terminate the request +- `signal`: AbortController signal or AbortSignal timeout to terminate the + request You can use this param to set request headers to fetch. For example: ```js -import { extract } from '@extractus/oembed-extractor' +import { extract } from "@extractus/oembed-extractor"; -const url = 'https://codepen.io/ndaidong/pen/LYmLKBw' +const url = "https://codepen.io/ndaidong/pen/LYmLKBw"; extract(url, null, { headers: { - 'user-agent': 'Opera/9.60 (Windows NT 6.0; U; en) Presto/2.1.1' - } -}) + "user-agent": "Opera/9.60 (Windows NT 6.0; U; en) Presto/2.1.1", + }, +}); ``` -You can also specify a proxy endpoint to load remote content, instead of fetching directly. +You can also specify a proxy endpoint to load remote content, instead of +fetching directly. For example: ```js -import { extract } from '@extractus/oembed-extractor' +import { extract } from "@extractus/oembed-extractor"; -const url = 'https://codepen.io/ndaidong/pen/LYmLKBw' +const url = "https://codepen.io/ndaidong/pen/LYmLKBw"; extract(url, null, { headers: { - 'user-agent': 'Opera/9.60 (Windows NT 6.0; U; en) Presto/2.1.1' + "user-agent": "Opera/9.60 (Windows NT 6.0; U; en) Presto/2.1.1", }, proxy: { - target: 'https://your-secret-proxy.io/loadJson?url=', + target: "https://your-secret-proxy.io/loadJson?url=", headers: { - 'Proxy-Authorization': 'Bearer YWxhZGRpbjpvcGVuc2VzYW1l...' - } - } -}) + "Proxy-Authorization": "Bearer YWxhZGRpbjpvcGVuc2VzYW1l...", + }, + }, +}); ``` -With the above setting, request will be forwarded to `https://your-secret-proxy.io/loadJson?url={OEMBED_ENDPOINT}`. +With the above setting, request will be forwarded to +`https://your-secret-proxy.io/loadJson?url={OEMBED_ENDPOINT}`. -Another way to work with proxy is use `agent` option instead of `proxy` as below: +Another way to work with proxy is use `agent` option instead of `proxy` as +below: ```js -import { extract } from '@extractus/oembed-extractor' +import { extract } from "@extractus/oembed-extractor"; -import { HttpsProxyAgent } from 'https-proxy-agent' +import { HttpsProxyAgent } from "https-proxy-agent"; -const proxy = 'http://abc:RaNdoMpasswORd_country-France@proxy.packetstream.io:31113' +const proxy = + "http://abc:RaNdoMpasswORd_country-France@proxy.packetstream.io:31113"; -const url = 'https://codepen.io/ndaidong/pen/LYmLKBw' +const url = "https://codepen.io/ndaidong/pen/LYmLKBw"; const oembed = await extract(url, null, { agent: new HttpsProxyAgent(proxy), -}) -console.log('Run oembed-extractor with proxy:', proxy) -console.log(oembed) +}); +console.log("Run oembed-extractor with proxy:", proxy); +console.log(oembed); ``` -For more info about [https-proxy-agent](https://www.npmjs.com/package/https-proxy-agent), check [its repo](https://github.com/TooTallNate/proxy-agents). +For more info about +[https-proxy-agent](https://www.npmjs.com/package/https-proxy-agent), check +[its repo](https://github.com/TooTallNate/proxy-agents). -By default, there is no request timeout. You can use the option `signal` to cancel request at the right time. +By default, there is no request timeout. You can use the option `signal` to +cancel request at the right time. The common way is to use AbortControler: ```js -const controller = new AbortController() +const controller = new AbortController(); // stop after 5 seconds setTimeout(() => { - controller.abort() -}, 5000) + controller.abort(); +}, 5000); const oembed = await extract(url, null, { signal: controller.signal, -}) +}); ``` A newer solution is AbortSignal's `timeout()` static method: @@ -174,7 +229,7 @@ A newer solution is AbortSignal's `timeout()` static method: // stop after 5 seconds const oembed = await extract(url, null, { signal: AbortSignal.timeout(5000), -}) +}); ``` For more info: @@ -182,10 +237,10 @@ For more info: - [AbortController constructor](https://developer.mozilla.org/en-US/docs/Web/API/AbortController) - [AbortSignal: timeout() static method](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal/timeout_static) - ### `.setProviderList()` -Apply a list of providers to use, overriding the [default](src/utils/providers.orginal.json). +Apply a list of providers to use, overriding the +[default](src/utils/providers.orginal.json). #### Syntax @@ -195,45 +250,52 @@ setProviderList(Array providers) #### Parameters -##### `providers` *required* +##### `providers` _required_ List of providers to apply. For example: ```js -import { setProviderList } from '@extractus/oembed-extractor' +import { setProviderList } from "@extractus/oembed-extractor"; const providers = [ { - provider_name: 'Alpha', - provider_url: 'https://alpha.com', + provider_name: "Alpha", + provider_url: "https://alpha.com", endpoints: [ // endpoint definition here - ] + ], }, { - provider_name: 'Beta', - provider_url: 'https://beta.com', + provider_name: "Beta", + provider_url: "https://beta.com", endpoints: [ // endpoint definition here - ] - } -] + ], + }, +]; -setProviderList(providers) +setProviderList(providers); ``` -Default list of resource providers is synchronized from [oembed.com](http://oembed.com/providers.json). - -If you want to modify providers list, please make pull request on [iamcal/oembed](https://github.com/iamcal/oembed) then create issue/pr here to ask for sync. +Default list of resource providers is synchronized from +[oembed.com](http://oembed.com/providers.json). +If you want to modify providers list, please make pull request on +[iamcal/oembed](https://github.com/iamcal/oembed) then create issue/pr here to +ask for sync. ## Facebook and Instagram -In order to work with the links from Facebook and Instagram, you need a [reviewed Facebook's app](https://developers.facebook.com/docs/app-review) with [oEmbed Read](https://developers.facebook.com/docs/features-reference/oembed-read) permission. +In order to work with the links from Facebook and Instagram, you need a +[reviewed Facebook's app](https://developers.facebook.com/docs/app-review) with +[oEmbed Read](https://developers.facebook.com/docs/features-reference/oembed-read) +permission. -When seeing a link from Facebook or Instagram, `oembed-parser` will look for environment variables `FACEBOOK_APP_ID` and `FACEBOOK_CLIENT_TOKEN` to retrieve oembed data using your app credentials. +When seeing a link from Facebook or Instagram, `oembed-parser` will look for +environment variables `FACEBOOK_APP_ID` and `FACEBOOK_CLIENT_TOKEN` to retrieve +oembed data using your app credentials. For example: @@ -244,7 +306,6 @@ export FACEBOOK_CLIENT_TOKEN=your_client_token npm run eval https://www.instagram.com/tv/CVlR5GFqF68/ ``` - ## Test ```bash @@ -256,7 +317,6 @@ npm test ![oembed-extractor unit test](https://i.imgur.com/Nr5BgUx.png) - ## Quick evaluation ```bash @@ -267,16 +327,19 @@ npm run eval {URL_TO_PARSE_OEMBED} ``` ## License -The MIT License (MIT) +The MIT License (MIT) ## Support the project -If you find value from this open source project, you can support in the following ways: +If you find value from this open source project, you can support in the +following ways: - Give it a star ⭐ - Buy me a coffee: https://paypal.me/ndaidong 🍵 -- Subscribe [oEmbed Parser service](https://rapidapi.com/pwshub-pwshub-default/api/oembed-parser/) on RapidAPI 😉 +- Subscribe + [oEmbed Parser service](https://rapidapi.com/pwshub-pwshub-default/api/oembed-parser/) + on RapidAPI 😉 Thank you. diff --git a/SECURITY.md b/SECURITY.md index 77bf31c..aa4b468 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,9 +2,11 @@ ## Supported Versions -Due to resource limitations, only the latest stable minor release is getting bugfixes (including security ones). +Due to resource limitations, only the latest stable minor release is getting +bugfixes (including security ones). -So e.g. if the latest stable version is 3.1.4, then 3.1.x line will still get security fixes but older versions (like 3.0.x) won't get any fixes. +So e.g. if the latest stable version is 3.1.4, then 3.1.x line will still get +security fixes but older versions (like 3.0.x) won't get any fixes. Description above is a general rule and may be altered on case by case basis. diff --git a/deno.json b/deno.json new file mode 100644 index 0000000..fcf2d7c --- /dev/null +++ b/deno.json @@ -0,0 +1,43 @@ +{ + "name": "@extractus/oembed-extractor", + "version": "5.0.0-rc1", + "description": "Get oEmbed data from given URL.", + "homepage": "https://github.com/extractus/oembed-extractor", + "repository": { + "type": "git", + "url": "git+https://github.com/extractus/oembed-extractor.git" + }, + "author": "@extractus", + "license": "MIT", + "tasks": { + "build": "deno run -A ./scripts/build_npm.ts" + }, + "imports": { + "@deno/dnt": "jsr:@deno/dnt@^0.41.2" + }, + "exports": "./mod.ts", + "lint": { + "include": ["mod.ts", "utils/*.ts", "scripts/*.ts", "tests/*.ts"], + "exclude": ["npm"], + "rules": { + "tags": ["recommended"], + "include": [], + "exclude": ["no-explicit-any"] + } + }, + "test": { + "include": ["tests"], + "exclude": [] + }, + "publish": { + "include": [ + "LICENSE", + "README.md", + "SECURITY.md", + "CONTRIBUTING.md", + "mod.ts", + "utils/*.ts", + "tests/*.ts" + ] + } +} diff --git a/mod.ts b/mod.ts new file mode 100644 index 0000000..a871be9 --- /dev/null +++ b/mod.ts @@ -0,0 +1,9 @@ +// mod.ts + +export { + find as findProvider, + has as hasProvider, + set as setProviderList, +} from "./utils/provider.ts"; + +export { getJson } from "./utils/retrieve.ts"; diff --git a/scripts/build_npm.ts b/scripts/build_npm.ts new file mode 100644 index 0000000..f8df0ec --- /dev/null +++ b/scripts/build_npm.ts @@ -0,0 +1,34 @@ +// scripts/build_npm.ts +import { build, emptyDir } from "@deno/dnt"; + +import pkg from "../deno.json" with { type: "json" }; + +const outputDir = "./npm"; + +await emptyDir(outputDir); + +await build({ + importMap: "deno.json", + entryPoints: ["./mod.ts"], + outDir: outputDir, + shims: { + deno: true, + }, + package: { + name: pkg.name, + version: pkg.version, + description: pkg.description, + author: pkg.author, + repository: pkg.repository, + bugs: { + url: `${pkg.homepage}/issues`, + }, + license: pkg.license, + }, + postBuild() { + Deno.copyFileSync("LICENSE", "npm/LICENSE"); + Deno.copyFileSync("README.md", "npm/README.md"); + Deno.copyFileSync("SECURITY.md", "npm/SECURITY.md"); + Deno.copyFileSync("CONTRIBUTING.md", "npm/CONTRIBUTING.md"); + }, +}); diff --git a/tests/auto_discovery_test.ts b/tests/auto_discovery_test.ts new file mode 100644 index 0000000..a75ad25 --- /dev/null +++ b/tests/auto_discovery_test.ts @@ -0,0 +1,64 @@ +// auto_discovery_test.ts + +import { assertEquals, assertRejects } from "https://deno.land/std/testing/asserts.ts"; +import { mockFetch, resetFetch } from "jsr:@c4spar/mock-fetch"; +import { isObject, hasProperty } from "jsr:@ndaidong/bellajs"; + +import autoDiscovery from "../utils/auto_discovery.ts"; + +const parseUrl = (url: string): any => { + const re = new URL(url) + return { + baseUrl: `${re.protocol}//${re.host}`, + path: re.pathname, + } +} + +Deno.test("test if autoDiscovery() works correctly", async () => { + const url = 'https://www.bitchute.com/video/8hXWnkvA8Ao/' + const htmlFile = './tests/test_data/bitchute.html' + const jsonFile = './tests/test_data/bitchute.json' + + const { baseUrl, path } = parseUrl(url) + + const fakeHtml = await Deno.readTextFile(htmlFile) + const fakeJson = await Deno.readTextFile(jsonFile) + + mockFetch(`${baseUrl}${path}`, { + body: fakeHtml, + headers: { + 'Content-Type': 'text/html', + } + }); + + const endpoint = 'https://www.bitchute.com/oembed/' + const { baseUrl: endpointBaseUrl, path: endpointPath } = parseUrl(endpoint) + + const params = { + maxwidth: '600', + maxheight: '400', + } + + const queries = new URLSearchParams({ + url: 'https://www.bitchute.com/video/8hXWnkvA8Ao/', + format: 'json', + ...params, + }) + const target = `${endpointBaseUrl}${endpointPath}?${queries.toString()}` + mockFetch(target, { + body: fakeJson, + headers: { + 'Content-Type': 'application/json', + } + }) + + const result = await autoDiscovery(url, params) + + assertEquals(isObject(result), true) + assertEquals(hasProperty(result, 'method'), true) + assertEquals(hasProperty(result, 'title'), true) + assertEquals(result.method, "auto-discovery") + assertEquals(result.title, "2023 Praemium Imperiale White House Program") + + resetFetch(); +}); diff --git a/tests/linter_test.ts b/tests/linter_test.ts new file mode 100644 index 0000000..b8a542b --- /dev/null +++ b/tests/linter_test.ts @@ -0,0 +1,51 @@ +// linter_test.ts + +import { assertEquals } from "https://deno.land/std/testing/asserts.ts"; + +import { getDomain, isValid } from "../utils/linker.ts"; + +Deno.test("test if isValid() works correctly", async (t) => { + await t.step("isValid: returns true for valid HTTP URL", () => { + const url = "http://example.com"; + const result = isValid(url); + assertEquals(result, true); + }); + + await t.step("isValid: returns true for valid HTTPS URL", () => { + const url = "https://example.com"; + const result = isValid(url); + assertEquals(result, true); + }); + + await t.step("isValid: returns false for invalid URL", () => { + const url = "invalid-url"; + const result = isValid(url); + assertEquals(result, false); + }); + + await t.step("isValid: returns false for non-HTTP/HTTPS URL", () => { + const url = "ftp://example.com"; + const result = isValid(url); + assertEquals(result, false); + }); +}); + +Deno.test("test if isValid() works correctly", async (t) => { + await t.step("getDomain: returns domain without www", () => { + const url = "http://www.example.com"; + const result = getDomain(url); + assertEquals(result, "example.com"); + }); + + await t.step("getDomain: returns domain with subdomain", () => { + const url = "http://sub.example.com"; + const result = getDomain(url); + assertEquals(result, "sub.example.com"); + }); + + await t.step("getDomain: returns domain without www for HTTPS", () => { + const url = "https://www.example.com"; + const result = getDomain(url); + assertEquals(result, "example.com"); + }); +}); diff --git a/tests/provider_test.ts b/tests/provider_test.ts new file mode 100644 index 0000000..905e47f --- /dev/null +++ b/tests/provider_test.ts @@ -0,0 +1,94 @@ +// provider_test.ts + +import { assertEquals } from "https://deno.land/std/testing/asserts.ts"; + +import provider from "../utils/provider.ts"; + +Deno.test("test if provider.find() works correctly", async (t) => { + const cases = [ + { + url: "https://www.facebook.com/video.php?v=999999999", + fetchEndpoint: "https://graph.facebook.com/v16.0/oembed_video", + }, + { + url: "https://www.facebook.com/someone/photos/somephoto", + fetchEndpoint: "https://graph.facebook.com/v16.0/oembed_post", + }, + { + url: "https://www.facebook.com/someone/page", + fetchEndpoint: "https://graph.facebook.com/v16.0/oembed_page", + }, + { + url: "http://instagram.com/someone/p/somepage", + fetchEndpoint: "https://graph.facebook.com/v16.0/instagram_oembed", + }, + { + url: "https://www.edumedia-sciences.com/en/media/558-heredity", + fetchEndpoint: "https://www.edumedia-sciences.com/oembed.json", + }, + { + url: "https://vimeo.com/999999", + fetchEndpoint: "https://vimeo.com/api/oembed.json", + }, + { + url: "https://www.youtube.com/watch?v=9999999", + fetchEndpoint: "https://www.youtube.com/oembed", + }, + ]; + + for (const ucase of cases) { + const { url, fetchEndpoint } = ucase; + await t.step( + `provider.find("${url}") must return "${fetchEndpoint}"`, + () => { + const foundedProvider = provider.find(url); + assertEquals(foundedProvider?.endpoint, fetchEndpoint); + }, + ); + } + + await t.step('provider.find("abcdef") must return null', () => { + assertEquals(provider.find("abcdef"), null); + }); + + await t.step( + 'provider.find("https://somethingdoesnotexist.com") must return null', + () => { + assertEquals(provider.find("https://somethingdoesnotexist.com"), null); + }, + ); + + await t.step("test if provider set/get works correctly", () => { + const providerList = [ + { + provider_name: "Alpha", + provider_url: "https://alpha.com", + endpoints: [ + { + schemes: [ + "https://store.alpha.com/*", + ], + url: "https://api.alpha.com/oembed", + }, + ], + }, + { + provider_name: "Beta", + provider_url: "https://beta.com", + endpoints: [ + { + schemes: [ + "https://store.beta.com/*", + ], + url: "https://api.beta.com/oembed", + }, + ], + }, + ]; + assertEquals(provider.set(providerList), providerList.length); + + const newList = provider.get(); + assertEquals(newList.length, providerList.length); + assertEquals(newList[0].schemes[0], /store.alpha.com\/(.*)/i); + }); +}); diff --git a/tests/test_data/bitchute.html b/tests/test_data/bitchute.html new file mode 100644 index 0000000..0f7b80a --- /dev/null +++ b/tests/test_data/bitchute.html @@ -0,0 +1,74 @@ + + + + + 2023 Praemium Imperiale White House Program + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test_data/bitchute.json b/tests/test_data/bitchute.json new file mode 100644 index 0000000..f5588ac --- /dev/null +++ b/tests/test_data/bitchute.json @@ -0,0 +1 @@ +{"title": "2023 Praemium Imperiale White House Program", "provider_url": "https://www.bitchute.com/", "author_name": "TheWhiteHouse", "thumbnail_url": "https://static-3.bitchute.com/live/cover_images/zWsYVmCOu4JA/8hXWnkvA8Ao_320x180.jpg", "html": "", "author_url": "https://www.bitchute.com/channel/zWsYVmCOu4JA/", "height": "344", "thumbnail_height": "180", "version": "1.0", "type": "video", "thumbnail_width": "320", "provider_name": "BitChute", "width": "459"} \ No newline at end of file diff --git a/tests/test_data/facebook.json b/tests/test_data/facebook.json new file mode 100644 index 0000000..78bd08b --- /dev/null +++ b/tests/test_data/facebook.json @@ -0,0 +1,13 @@ +{ + "author_name": "Facebook", + "author_url": "https://www.facebook.com/facebook/", + "provider_url": "https://www.facebook.com", + "provider_name": "Facebook", + "success": true, + "height": null, + "html": "
\n
How to Share With Just Friends

How to share with just friends.

Posted by Facebook on Friday, December 5, 2014
", + "type": "video", + "version": "1.0", + "url": "https://www.facebook.com/facebook/videos/10153231379946729/", + "width": "100%" +} diff --git a/tests/test_data/flickr-default.json b/tests/test_data/flickr-default.json new file mode 100644 index 0000000..1c472ed --- /dev/null +++ b/tests/test_data/flickr-default.json @@ -0,0 +1,22 @@ +{ + "type": "photo", + "flickr_type": "photo", + "title": "ZB8T0193", + "author_name": "‮‭‬bees‬", + "author_url": "https://www.flickr.com/photos/bees/", + "width": 1024, + "height": 683, + "url": "https://live.staticflickr.com/3123/2341623661_7c99f48bbf_b.jpg", + "web_page": "https://www.flickr.com/photos/bees/2341623661/", + "thumbnail_url": "https://live.staticflickr.com/3123/2341623661_7c99f48bbf_q.jpg", + "thumbnail_width": 150, + "thumbnail_height": 150, + "web_page_short_url": "https://flic.kr/p/4yVr8K", + "license": "All Rights Reserved", + "license_id": 0, + "html": "\"ZB8T0193\"", + "version": "1.0", + "cache_age": 3600, + "provider_name": "Flickr", + "provider_url": "https://www.flickr.com/" +} \ No newline at end of file diff --git a/tests/test_data/flickr-sizelimit.json b/tests/test_data/flickr-sizelimit.json new file mode 100644 index 0000000..26d425b --- /dev/null +++ b/tests/test_data/flickr-sizelimit.json @@ -0,0 +1,22 @@ +{ + "type": "photo", + "flickr_type": "photo", + "title": "ZB8T0193", + "author_name": "‮‭‬bees‬", + "author_url": "https://www.flickr.com/photos/bees/", + "width": 500, + "height": 333, + "url": "https://live.staticflickr.com/3123/2341623661_7c99f48bbf.jpg", + "web_page": "https://www.flickr.com/photos/bees/2341623661/", + "thumbnail_url": "https://live.staticflickr.com/3123/2341623661_7c99f48bbf_q.jpg", + "thumbnail_width": 150, + "thumbnail_height": 150, + "web_page_short_url": "https://flic.kr/p/4yVr8K", + "license": "All Rights Reserved", + "license_id": 0, + "html": "\"ZB8T0193\"", + "version": "1.0", + "cache_age": 3600, + "provider_name": "Flickr", + "provider_url": "https://www.flickr.com/" +} \ No newline at end of file diff --git a/tests/test_data/flickr_2iYctUr.json b/tests/test_data/flickr_2iYctUr.json new file mode 100644 index 0000000..e35194e --- /dev/null +++ b/tests/test_data/flickr_2iYctUr.json @@ -0,0 +1,22 @@ +{ + "type": "photo", + "flickr_type": "photo", + "title": "20200401_165350", + "author_name": "NDD2012", + "author_url": "https://www.flickr.com/photos/66881897@N05/", + "width": 1024, + "height": 768, + "url": "https://live.staticflickr.com/65535/49862721477_645db2bc58_b.jpg", + "web_page": "https://www.flickr.com/photos/66881897@N05/49862721477/", + "thumbnail_url": "https://live.staticflickr.com/65535/49862721477_645db2bc58_q.jpg", + "thumbnail_width": 150, + "thumbnail_height": 150, + "web_page_short_url": "https://flic.kr/p/2iYctUr", + "license": "All Rights Reserved", + "license_id": 0, + "html": "\"20200401_165350\"", + "version": "1.0", + "cache_age": 3600, + "provider_name": "Flickr", + "provider_url": "https://www.flickr.com/" +} \ No newline at end of file diff --git a/tests/test_data/flickr_2iYctUr_640x480.json b/tests/test_data/flickr_2iYctUr_640x480.json new file mode 100644 index 0000000..c09eead --- /dev/null +++ b/tests/test_data/flickr_2iYctUr_640x480.json @@ -0,0 +1,22 @@ +{ + "type": "photo", + "flickr_type": "photo", + "title": "20200401_165350", + "author_name": "NDD2012", + "author_url": "https://www.flickr.com/photos/66881897@N05/", + "width": 640, + "height": 480, + "url": "https://live.staticflickr.com/65535/49862721477_645db2bc58_z.jpg", + "web_page": "https://www.flickr.com/photos/66881897@N05/49862721477/", + "thumbnail_url": "https://live.staticflickr.com/65535/49862721477_645db2bc58_q.jpg", + "thumbnail_width": 150, + "thumbnail_height": 150, + "web_page_short_url": "https://flic.kr/p/2iYctUr", + "license": "All Rights Reserved", + "license_id": 0, + "html": "\"20200401_165350\"", + "version": "1.0", + "cache_age": 3600, + "provider_name": "Flickr", + "provider_url": "https://www.flickr.com/" +} \ No newline at end of file diff --git a/tests/test_data/instagram_ic7kRDqOlt.json b/tests/test_data/instagram_ic7kRDqOlt.json new file mode 100644 index 0000000..136e9dd --- /dev/null +++ b/tests/test_data/instagram_ic7kRDqOlt.json @@ -0,0 +1,11 @@ +{ + "version": "1.0", + "author_name": "diegoquinteiro", + "provider_name": "Instagram", + "provider_url": "https://www.instagram.com/", + "type": "rich", + "width": 658, + "html": "

Live event: Release Lotus - New social network in Vietnamhttps://t.co/BWGejBZRWh#lotus #SocialNetwork

— Dong Nguyen (@ndaidong) September 16, 2019
\n\n", + "width": 550, + "height": null, + "type": "rich", + "cache_age": "3153600000", + "provider_name": "Twitter", + "provider_url": "https://twitter.com", + "version": "1.0" +} diff --git a/tests/test_data/twitter.json b/tests/test_data/twitter.json new file mode 100644 index 0000000..b137d24 --- /dev/null +++ b/tests/test_data/twitter.json @@ -0,0 +1,13 @@ +{ + "url": "https://twitter.com/ndaidong/status/1173592062878314497", + "author_name": "Dong Nguyen", + "author_url": "https://twitter.com/ndaidong", + "html": "

Live event: Release Lotus - New social network in Vietnamhttps://t.co/BWGejBZRWh#lotus #SocialNetwork

— Dong Nguyen (@ndaidong) September 16, 2019
\n\n", + "width": 550, + "height": null, + "type": "rich", + "cache_age": "3153600000", + "provider_name": "Twitter", + "provider_url": "https://twitter.com", + "version": "1.0" +} \ No newline at end of file diff --git a/tests/test_data/youtube.json b/tests/test_data/youtube.json new file mode 100644 index 0000000..0d67cc5 --- /dev/null +++ b/tests/test_data/youtube.json @@ -0,0 +1,15 @@ +{ + "title": "My dog trio", + "author_name": "Dong Nguyen", + "author_url": "https://www.youtube.com/c/DongNguyenbellajs", + "type": "video", + "height": 113, + "width": 200, + "version": "1.0", + "provider_name": "YouTube", + "provider_url": "https://www.youtube.com/", + "thumbnail_height": 360, + "thumbnail_width": 480, + "thumbnail_url": "https://i.ytimg.com/vi/qQpb1oCernE/hqdefault.jpg", + "html": "" +} \ No newline at end of file diff --git a/tests/test_data/youtube_ciS8aCrX-9s.json b/tests/test_data/youtube_ciS8aCrX-9s.json new file mode 100644 index 0000000..f3aa6da --- /dev/null +++ b/tests/test_data/youtube_ciS8aCrX-9s.json @@ -0,0 +1,15 @@ +{ + "title": "prOfile", + "author_name": "Ora Walters", + "author_url": "https://www.youtube.com/c/OwenWalters", + "type": "video", + "height": 150, + "width": 200, + "version": "1.0", + "provider_name": "YouTube", + "provider_url": "https://www.youtube.com/", + "thumbnail_height": 360, + "thumbnail_width": 480, + "thumbnail_url": "https://i.ytimg.com/vi/ciS8aCrX-9s/hqdefault.jpg", + "html": "" +} \ No newline at end of file diff --git a/utils/auto_discovery.ts b/utils/auto_discovery.ts new file mode 100644 index 0000000..e85df55 --- /dev/null +++ b/utils/auto_discovery.ts @@ -0,0 +1,43 @@ +// autoDiscovery.ts + +import { DOMParser } from "jsr:@b-fuze/deno-dom"; + +import { + getHtml, + getJson, + FetchOptions +} from "./retrieve.ts"; + +interface Params { + [key: string]: any; +} + +export default async ( + url: string, + params: Params = {}, + options: FetchOptions = {} +): Promise => { + const html = await getHtml(url, options); + const doc = new DOMParser().parseFromString(html, 'text/html'); + const elm = doc.querySelector('link[type="application/json+oembed"]'); + if (!elm) { + throw new Error("No oEmbed link found in the HTML"); + } + const href = elm.getAttribute('href'); + if (!href) { + throw new Error("oEmbed link does not contain an href attribute"); + } + const q = new URL(href); + const { origin, pathname, searchParams } = q; + + Object.keys(params).forEach((key) => { + if (!searchParams.has(key)) { + searchParams.append(key, params[key]); + } + }); + + const link = `${origin}${pathname}?${searchParams.toString()}`; + const body = await getJson(link, options); + body.method = 'auto-discovery'; + return body; +}; diff --git a/utils/linker.ts b/utils/linker.ts new file mode 100644 index 0000000..337fed4 --- /dev/null +++ b/utils/linker.ts @@ -0,0 +1,15 @@ +// linker.ts + +export const isValid = (url: string = ""): boolean => { + try { + const ourl = new URL(url); + return ourl !== null && ourl.protocol.startsWith("http"); + } catch { + return false; + } +}; + +export const getDomain = (url: string): string => { + const host = (new URL(url)).host; + return host.replace("www.", ""); +}; diff --git a/utils/provider.ts b/utils/provider.ts new file mode 100644 index 0000000..f7baef5 --- /dev/null +++ b/utils/provider.ts @@ -0,0 +1,152 @@ +// provider.ts + +import { getDomain, isValid as isValidURL } from "./linker.ts"; +import { providers as defaultProviderList } from "./providers.latest.ts"; + +export interface ProviderEndpoint { + schemes?: string[]; + url: string; + formats?: string[]; // "json" "xml" + discovery?: boolean; +} + +export interface Provider { + "provider_name": string; + "provider_url": string; + endpoints: ProviderEndpoint[]; +} + +export interface FindProviderResult { + "fetchEndpoint": string; + "provider_name": string; + "provider_url": string; +} + +interface SimplifiedProvider { + s: string[]; + e: string; +} + +interface ProcessedProvider { + endpoint: string; + schemes: RegExp[]; +} + +const toRegExp = (scheme: string = ""): RegExp => { + return new RegExp( + scheme.replace(/\\./g, ".").replace(/\*/g, "(.*)").replace(/\?/g, "\\?") + .replace(/,$/g, ""), + "i", + ); +}; + +const uniquify = (arr: string[] = []): string[] => { + return [...new Set(arr)]; +}; + +const undotted = (scheme: string = ""): string => { + return scheme.replace(/\./g, "\\."); +}; + +const removeProtocol = (url: string): string => { + return url.replace("https://", "").replace("http://", ""); +}; + +export const simplify = (providers: Provider[] = []): SimplifiedProvider[] => { + return providers.map((item) => { + const { endpoints } = item; + return endpoints.map((endpoint) => { + const { schemes = [], url } = endpoint; + const patterns = schemes.length > 0 + ? uniquify(schemes.map(removeProtocol).map(undotted)) + : []; + + return { + s: patterns, + e: removeProtocol(url).replace(/\{format\}/g, "json"), + }; + }); + }).reduce((prev, curr) => { + return prev.concat(curr); + }, []); +}; + +const providersFromList = ( + providers: SimplifiedProvider[] = [], +): ProcessedProvider[] => { + return providers.map((provider) => { + const { e: endpoint, s: schemes } = provider; + return { + endpoint: `https://${endpoint}`, + schemes: schemes.map(toRegExp), + }; + }); +}; + +const store = { + providers: providersFromList(defaultProviderList), +}; + +export const get = (): ProcessedProvider[] => { + return [...store.providers]; +}; + +export const set = (providers: Provider[] = []): number => { + store.providers = providersFromList(simplify(providers)); + return store.providers.length; +}; + +const compare = ( + url: string = "", + endpoint: string = "", + schemes: RegExp[] = [], +): boolean => { + if (!schemes.length) { + const domain = getDomain(url); + const endpointDomain = getDomain(endpoint); + return domain === endpointDomain; + } + return schemes.some((scheme) => { + return url.match(scheme); + }); +}; + +export const find = ( + url: string = "", +): { schemes: RegExp[]; endpoint: string; url: string } | null => { + if (!isValidURL(url)) { + return null; + } + + const providers = get(); + + for (let i = 0; i < providers.length; i++) { + const { endpoint, schemes } = providers[i]; + const isMatched = compare(url, endpoint, schemes); + if (isMatched) { + return { + schemes, + endpoint, + url, + }; + } + } + + return null; +}; + +export const has = (url: string = ""): boolean => { + return find(url) !== null; +}; + +export const getEndpoint = (url: string): string | null => { + const p = find(url); + return p ? p.endpoint : null; +}; + +export default { + find, + has, + get, + set, +}; diff --git a/utils/providers.latest.ts b/utils/providers.latest.ts new file mode 100644 index 0000000..a51445f --- /dev/null +++ b/utils/providers.latest.ts @@ -0,0 +1,2179 @@ +// provider data, synchronized at 2024-05-07T10:41:10.221Z + +export const providers = [ + { + "s": [ + "www\\.23hq\\.com/*/photo/*", + ], + "e": "www.23hq.com/23/oembed", + }, + { + "s": [ + "playout\\.3qsdn\\.com/embed/*", + ], + "e": "playout.3qsdn.com/oembed", + }, + { + "s": [ + "store\\.abraia\\.me/*", + ], + "e": "api.abraia.me/oembed", + }, + { + "s": [ + "play\\.acast\\.com/s/*", + ], + "e": "oembed.acast.com/v1/embed-player", + }, + { + "s": [ + "secure\\.actblue\\.com/donate/*", + ], + "e": "secure.actblue.com/cf/oembed", + }, + { + "s": [ + "adilo\\.bigcommand\\.com/watch/*", + ], + "e": "adilo.bigcommand.com/web/oembed", + }, + { + "s": [ + "vod\\.afreecatv\\.com/player/", + "v\\.afree\\.ca/ST/", + "vod\\.afreecatv\\.com/ST/", + "vod\\.afreecatv\\.com/PLAYER/STATION/", + "play\\.afreecatv\\.com/", + ], + "e": "openapi.afreecatv.com/oembed/embedinfo", + }, + { + "s": [ + "altium\\.com/viewer/*", + ], + "e": "viewer.altium.com/shell/oembed", + }, + { + "s": [ + "app\\.altrulabs\\.com/*/*?answer_id=*", + "app\\.altrulabs\\.com/player/*", + ], + "e": "api.altrulabs.com/api/v1/social/oembed", + }, + { + "s": [ + "live\\.amcharts\\.com/*", + ], + "e": "live.amcharts.com/oembed", + }, + { + "s": [ + "amtraker\\.com/trains/*", + "beta\\.amtraker\\.com/trains/*", + ], + "e": "api.amtraker.com/v2/oembed", + }, + { + "s": [ + "www\\.animatron\\.com/project/*", + "animatron\\.com/project/*", + ], + "e": "animatron.com/oembed/json", + }, + { + "s": [ + "animoto\\.com/play/*", + ], + "e": "animoto.com/oembeds/create", + }, + { + "s": [ + "anniemusic\\.app/t/*", + "anniemusic\\.app/p/*", + ], + "e": "api.anniemusic.app/api/v1/oembed", + }, + { + "s": [ + "storymaps\\.arcgis\\.com/stories/*", + ], + "e": "storymaps.arcgis.com/oembed", + }, + { + "s": [ + "app\\.archivos\\.digital/app/view/*", + ], + "e": "app.archivos.digital/oembed/", + }, + { + "s": [ + "*\\.studio\\.assemblrworld\\.com/creation/*", + "studio\\.assemblrworld\\.com/creation/*", + "*\\.app-edu\\.assemblrworld\\.com/Creation/*", + "app-edu\\.assemblrworld\\.com/Creation/*", + "assemblr\\.world/*", + "editor\\.assemblrworld\\.com/*", + "*\\.assemblrworld\\.com/creation/*", + "*\\.assemblrworld\\.com/Creation/*", + ], + "e": "studio.assemblrworld.com/api/oembed", + }, + { + "s": [ + "audio\\.com/*", + "www\\.audio\\.com/*", + ], + "e": "api.audio.com/oembed", + }, + { + "s": [ + "audioboom\\.com/channels/*", + "audioboom\\.com/channel/*", + "audioboom\\.com/playlists/*", + "audioboom\\.com/podcasts/*", + "audioboom\\.com/podcast/*", + "audioboom\\.com/posts/*", + "audioboom\\.com/episodes/*", + ], + "e": "audioboom.com/publishing/oembed.json", + }, + { + "s": [ + "audioclip\\.naver\\.com/channels/*/clips/*", + "audioclip\\.naver\\.com/audiobooks/*", + ], + "e": "audioclip.naver.com/oembed", + }, + { + "s": [ + "audiomack\\.com/*/song/*", + "audiomack\\.com/*/album/*", + "audiomack\\.com/*/playlist/*", + ], + "e": "audiomack.com/oembed", + }, + { + "s": [ + "podcasts\\.audiomeans\\.fr/*", + ], + "e": "podcasts.audiomeans.fr/services/oembed", + }, + { + "s": [ + "app\\.avocode\\.com/view/*", + ], + "e": "stage-embed.avocode.com/api/oembed", + }, + { + "s": [ + "backtracks\\.fm/*/*/e/*", + "backtracks\\.fm/*/s/*/*", + "backtracks\\.fm/*/*/*/*/e/*/*", + "backtracks\\.fm/*", + ], + "e": "backtracks.fm/oembed", + }, + { + "s": [ + "balsamiq\\.cloud/*", + ], + "e": "balsamiq.cloud/oembed", + }, + { + "s": [ + "beams\\.fm/*", + ], + "e": "api.beams.fm/oEmbed", + }, + { + "s": [], + "e": "www.beautiful.ai/api/oembed", + }, + { + "s": [ + "www\\.behance\\.net/gallery/*/*", + "www\\.behance\\.net/*/services/*/*", + ], + "e": "www.behance.net/services/oembed", + }, + { + "s": [ + "cloud\\.biqapp\\.com/*", + ], + "e": "biqapp.com/api/v1/video/oembed", + }, + { + "s": [ + "blackfire\\.io/profiles/*/graph", + "blackfire\\.io/profiles/compare/*/graph", + ], + "e": "blackfire.io/oembed", + }, + { + "s": [ + "blogcast\\.host/embed/*", + "blogcast\\.host/embedly/*", + ], + "e": "blogcast.host/oembed", + }, + { + "s": [ + "bsky\\.app/profile/*/post/*", + ], + "e": "embed.bsky.app/oembed", + }, + { + "s": [ + "www\\.bookingmood\\.com/embed/*/*", + ], + "e": "bookingmood.com/api/oembed", + }, + { + "s": [], + "e": "boxofficebuz.com/oembed", + }, + { + "s": [ + "view\\.briovr\\.com/api/v1/worlds/oembed/*", + ], + "e": "view.briovr.com/api/v1/worlds/oembed/", + }, + { + "s": [ + "www\\.bumper\\.com/oembed/bumper", + "www\\.bumper\\.com/oembed-s/bumper", + ], + "e": "www.bumper.com/oembed/bumper", + }, + { + "s": [ + "buttondown\\.email/*", + ], + "e": "buttondown.email/embed", + }, + { + "s": [ + "cmc\\.byzart\\.eu/files/*", + ], + "e": "cmc.byzart.eu/oembed/", + }, + { + "s": [ + "cacoo\\.com/diagrams/*", + ], + "e": "cacoo.com/oembed.json", + }, + { + "s": [ + "www\\.canva\\.com/design/*/view", + ], + "e": "www.canva.com/_oembed", + }, + { + "s": [ + "minesweeper\\.today/*", + ], + "e": "minesweeper.today/api/oembed", + }, + { + "s": [ + "img\\.catbo\\.at/*", + ], + "e": "img.catbo.at/oembed.json", + }, + { + "s": [ + "embeds\\.celero\\.io/*", + ], + "e": "api.celero.io/api/oembed", + }, + { + "s": [ + "view\\.ceros\\.com/*", + ], + "e": "view.ceros.com/oembed", + }, + { + "s": [ + "chainflix\\.net/video/*", + "chainflix\\.net/video/embed/*", + "*\\.chainflix\\.net/video/*", + "*\\.chainflix\\.net/video/embed/*", + ], + "e": "www.chainflix.net/video/oembed", + }, + { + "s": [ + "public\\.chartblocks\\.com/c/*", + ], + "e": "embed.chartblocks.com/1.0/oembed", + }, + { + "s": [ + "chirb\\.it/*", + ], + "e": "chirb.it/oembed.json", + }, + { + "s": [ + "chroco\\.ooo/mypage/*", + "chroco\\.ooo/story/*", + ], + "e": "chroco.ooo/embed", + }, + { + "s": [ + "www\\.circuitlab\\.com/circuit/*", + ], + "e": "www.circuitlab.com/circuit/oembed/", + }, + { + "s": [ + "www\\.clipland\\.com/v/*", + ], + "e": "www.clipland.com/api/oembed", + }, + { + "s": [ + "clyp\\.it/*", + "clyp\\.it/playlist/*", + ], + "e": "api.clyp.it/oembed/", + }, + { + "s": [ + "app\\.ilovecoco\\.video/*/embed", + ], + "e": "app.ilovecoco.video/api/oembed.json", + }, + { + "s": [ + "codehs\\.com/editor/share_abacus/*", + ], + "e": "codehs.com/api/sharedprogram/1/oembed/", + }, + { + "s": [ + "codepen\\.io/*", + ], + "e": "codepen.io/api/oembed", + }, + { + "s": [ + "codepoints\\.net/*", + "www\\.codepoints\\.net/*", + ], + "e": "codepoints.net/api/v1/oembed", + }, + { + "s": [ + "codesandbox\\.io/s/*", + "codesandbox\\.io/embed/*", + ], + "e": "codesandbox.io/oembed", + }, + { + "s": [ + "www\\.collegehumor\\.com/video/*", + ], + "e": "www.collegehumor.com/oembed.json", + }, + { + "s": [ + "commaful\\.com/play/*", + ], + "e": "commaful.com/api/oembed/", + }, + { + "s": [ + "coub\\.com/view/*", + "coub\\.com/embed/*", + ], + "e": "coub.com/api/oembed.json", + }, + { + "s": [ + "crowdranking\\.com/*/*", + ], + "e": "crowdranking.com/api/oembed.json", + }, + { + "s": [ + "crumb\\.sh/*", + ], + "e": "crumb.sh/oembed/", + }, + { + "s": [ + "cueup\\.io/user/*/sounds/*", + ], + "e": "gql.cueup.io/oembed", + }, + { + "s": [ + "*\\.curated\\.co/*", + ], + "e": "api.curated.co/oembed", + }, + { + "s": [ + "app\\.customerdb\\.com/share/*", + ], + "e": "app.customerdb.com/embed", + }, + { + "s": [ + "app\\.dadan\\.io/*", + "stage\\.dadan\\.io/*", + ], + "e": "app.dadan.io/api/video/oembed", + }, + { + "s": [ + "www\\.dailymotion\\.com/video/*", + ], + "e": "www.dailymotion.com/services/oembed", + }, + { + "s": [ + "dalexni\\.com/i/*", + ], + "e": "dalexni.com/oembed/", + }, + { + "s": [ + "datawrapper\\.dwcdn\\.net/*", + ], + "e": "api.datawrapper.de/v3/oembed/", + }, + { + "s": [ + "*\\.deseret\\.com/*", + ], + "e": "embed.deseret.com/", + }, + { + "s": [ + "*\\.deviantart\\.com/art/*", + "*\\.deviantart\\.com/*#/d*", + "fav\\.me/*", + "sta\\.sh/*", + "*\\.deviantart\\.com/*/art/*", + ], + "e": "backend.deviantart.com/oembed", + }, + { + "s": [ + "www\\.ultimedia\\.com/central/video/edit/id/*/topic_id/*/", + "www\\.ultimedia\\.com/default/index/videogeneric/id/*/showtitle/1/viewnc/1", + "www\\.ultimedia\\.com/default/index/videogeneric/id/*", + ], + "e": "www.ultimedia.com/api/search/oembed", + }, + { + "s": [ + "*\\.docdroid\\.net/*", + "docdro\\.id/*", + "*\\.docdroid\\.com/*", + ], + "e": "www.docdroid.net/api/oembed", + }, + { + "s": [ + "docswell\\.com/s/*/*", + "www\\.docswell\\.com/s/*/*", + ], + "e": "www.docswell.com/service/oembed", + }, + { + "s": [ + "dotsub\\.com/view/*", + ], + "e": "dotsub.com/services/oembed", + }, + { + "s": [ + "www\\.dreambroker\\.com/channel/*/*", + ], + "e": "dreambroker.com/channel/oembed", + }, + { + "s": [ + "d\\.tube/v/*", + ], + "e": "api.d.tube/oembed", + }, + { + "s": [ + "app\\.echoeshq\\.com/embed/*", + ], + "e": "api.echoeshq.com/oembed", + }, + { + "s": [], + "e": "www.edumedia-sciences.com/oembed.json", + }, + { + "s": [], + "e": "www.edumedia-sciences.com/oembed.xml", + }, + { + "s": [ + "egliseinfo\\.catholique\\.fr/*", + ], + "e": "egliseinfo.catholique.fr/api/oembed", + }, + { + "s": [ + "embedery\\.com/widget/*", + ], + "e": "embedery.com/api/oembed", + }, + { + "s": [ + "ethfiddle\\.com/*", + ], + "e": "ethfiddle.com/services/oembed/", + }, + { + "s": [ + "evt\\.live/*", + "evt\\.live/*/*", + "live\\.eventlive\\.pro/*", + "live\\.eventlive\\.pro/*/*", + ], + "e": "evt.live/api/oembed", + }, + { + "s": [ + "app\\.everviz\\.com/embed/*", + ], + "e": "api.everviz.com/oembed", + }, + { + "s": [ + "app\\.ex\\.co/stories/*", + "www\\.playbuzz\\.com/*", + ], + "e": "oembed.ex.co/item", + }, + { + "s": [ + "eyrie\\.io/board/*", + "eyrie\\.io/sparkfun/*", + ], + "e": "eyrie.io/v1/oembed", + }, + { + "s": [ + "www\\.facebook\\.com/*/posts/*", + "www\\.facebook\\.com/*/activity/*", + "www\\.facebook\\.com/*/photos/*", + "www\\.facebook\\.com/photo\\.php?fbid=*", + "www\\.facebook\\.com/photos/*", + "www\\.facebook\\.com/permalink\\.php?story_fbid=*", + "www\\.facebook\\.com/media/set?set=*", + "www\\.facebook\\.com/questions/*", + "www\\.facebook\\.com/notes/*/*/*", + ], + "e": "graph.facebook.com/v16.0/oembed_post", + }, + { + "s": [ + "www\\.facebook\\.com/*/videos/*", + "www\\.facebook\\.com/video\\.php?id=*", + "www\\.facebook\\.com/video\\.php?v=*", + ], + "e": "graph.facebook.com/v16.0/oembed_video", + }, + { + "s": [ + "www\\.facebook\\.com/*", + ], + "e": "graph.facebook.com/v16.0/oembed_page", + }, + { + "s": [ + "app\\.getfader\\.com/projects/*/publish", + ], + "e": "app.getfader.com/api/oembed", + }, + { + "s": [ + "faithlifetv\\.com/items/*", + "faithlifetv\\.com/items/resource/*/*", + "faithlifetv\\.com/media/*", + "faithlifetv\\.com/media/assets/*", + "faithlifetv\\.com/media/resource/*/*", + ], + "e": "faithlifetv.com/api/oembed", + }, + { + "s": [ + "www\\.figma\\.com/file/*", + ], + "e": "www.figma.com/api/oembed", + }, + { + "s": [ + "*\\.fireworktv\\.com/*", + "*\\.fireworktv\\.com/embed/*/v/*", + ], + "e": "www.fireworktv.com/oembed", + }, + { + "s": [ + "www\\.fite\\.tv/watch/*", + ], + "e": "www.fite.tv/oembed", + }, + { + "s": [ + "flat\\.io/score/*", + "*\\.flat\\.io/score/*", + ], + "e": "flat.io/services/oembed", + }, + { + "s": [ + "*\\.flickr\\.com/photos/*", + "flic\\.kr/p/*", + "*\\.*\\.flickr\\.com/*/*", + ], + "e": "www.flickr.com/services/oembed/", + }, + { + "s": [ + "public\\.flourish\\.studio/visualisation/*", + "public\\.flourish\\.studio/story/*", + ], + "e": "app.flourish.studio/api/v1/oembed", + }, + { + "s": [ + "flowhub\\.org/f/*", + "flowhub\\.org/s/*", + ], + "e": "flowhub.org/o/embed", + }, + { + "s": [ + "fooday\\.app/*/reviews/*", + "fooday\\.app/*/spots/*", + ], + "e": "fooday.app/oembed", + }, + { + "s": [ + "fiso\\.foxsports\\.com\\.au/isomorphic-widget/*", + ], + "e": "fiso.foxsports.com.au/oembed", + }, + { + "s": [ + "framebuzz\\.com/v/*", + ], + "e": "framebuzz.com/oembed/", + }, + { + "s": [ + "framer\\.com/share/*", + "framer\\.com/embed/*", + ], + "e": "api.framer.com/web/oembed", + }, + { + "s": [ + "*\\.geograph\\.org\\.uk/*", + "*\\.geograph\\.co\\.uk/*", + "*\\.geograph\\.ie/*", + "*\\.wikimedia\\.org/*_geograph\\.org\\.uk_*", + ], + "e": "api.geograph.org.uk/api/oembed", + }, + { + "s": [ + "*\\.geograph\\.org\\.gg/*", + "*\\.geograph\\.org\\.je/*", + "channel-islands\\.geograph\\.org/*", + "channel-islands\\.geographs\\.org/*", + "*\\.channel\\.geographs\\.org/*", + ], + "e": "www.geograph.org.gg/api/oembed", + }, + { + "s": [ + "geo-en\\.hlipp\\.de/*", + "geo\\.hlipp\\.de/*", + "germany\\.geograph\\.org/*", + ], + "e": "geo.hlipp.de/restapi.php/api/oembed", + }, + { + "s": [ + "gty\\.im/*", + ], + "e": "embed.gettyimages.com/oembed", + }, + { + "s": [ + "www\\.gifnote\\.com/play/*", + ], + "e": "www.gifnote.com/services/oembed", + }, + { + "s": [ + "giphy\\.com/gifs/*", + "giphy\\.com/clips/*", + "gph\\.is/*", + "media\\.giphy\\.com/media/*/giphy\\.gif", + ], + "e": "giphy.com/services/oembed", + }, + { + "s": [], + "e": "gloria.tv/oembed/", + }, + { + "s": [ + "view\\.gmetri\\.com/*", + "*\\.gmetri\\.com/*", + ], + "e": "embed.gmetri.com/oembed/", + }, + { + "s": [ + "app\\.gong\\.io/call?id=*", + ], + "e": "app.gong.io/oembed", + }, + { + "s": [ + "grain\\.co/highlight/*", + "grain\\.co/share/*", + "grain\\.com/share/*", + ], + "e": "api.grain.com/_/api/oembed", + }, + { + "s": [ + "gtchannel\\.com/watch/*", + ], + "e": "api.luminery.com/oembed", + }, + { + "s": [ + "www\\.gumlet\\.com/watch/*", + "play\\.gumlet\\.io/embed/*", + ], + "e": "api.gumlet.com/v1/oembed", + }, + { + "s": [ + "gyazo\\.com/*", + ], + "e": "api.gyazo.com/api/oembed", + }, + { + "s": [ + "core\\.hash\\.ai/@*", + ], + "e": "api.hash.ai/oembed", + }, + { + "s": [ + "hearthis\\.at/*/*/", + "hearthis\\.at/*/set/*/", + ], + "e": "hearthis.at/oembed/?format=json", + }, + { + "s": [ + "heyzine\\.com/flip-book/*", + "*\\.hflip\\.co/*", + "*\\.aflip\\.in/*", + ], + "e": "heyzine.com/api1/oembed", + }, + { + "s": [ + "player\\.hihaho\\.com/*", + ], + "e": "player.hihaho.com/services/oembed", + }, + { + "s": [ + "*\\.hippovideo\\.io/*", + ], + "e": "www.hippovideo.io/services/oembed", + }, + { + "s": [ + "homey\\.app/f/*", + "homey\\.app/*/flow/*", + ], + "e": "homey.app/api/oembed/flow", + }, + { + "s": [ + "huffduffer\\.com/*/*", + ], + "e": "huffduffer.com/oembed", + }, + { + "s": [ + "www\\.hulu\\.com/watch/*", + ], + "e": "www.hulu.com/api/oembed.json", + }, + { + "s": [ + "oembed\\.ideamapper\\.com/*", + ], + "e": "oembed.ideamapper.com", + }, + { + "s": [ + "*\\.idomoo\\.com/*", + ], + "e": "oembed.idomoo.com/oembed", + }, + { + "s": [ + "www\\.ifixit\\.com/Guide/View/*", + ], + "e": "www.ifixit.com/Embed", + }, + { + "s": [ + "ifttt\\.com/recipes/*", + ], + "e": "www.ifttt.com/oembed/", + }, + { + "s": [ + "www\\.iheart\\.com/podcast/*/*", + ], + "e": "www.iheart.com/oembed", + }, + { + "s": [ + "qr\\.imenupro\\.com/*", + ], + "e": "qr.imenupro.com/api/oembed", + }, + { + "s": [ + "incredible\\.dev/watch/*", + ], + "e": "oembed.incredible.dev/oembed", + }, + { + "s": [ + "player\\.indacolive\\.com/player/jwp/clients/*", + ], + "e": "player.indacolive.com/services/oembed", + }, + { + "s": [ + "infogram\\.com/*", + ], + "e": "infogram.com/oembed", + }, + { + "s": [ + "*\\.infoveave\\.net/E/*", + "*\\.infoveave\\.net/P/*", + ], + "e": "infoveave.net/services/oembed/", + }, + { + "s": [ + "www\\.injurymap\\.com/exercises/*", + ], + "e": "www.injurymap.com/services/oembed", + }, + { + "s": [ + "www\\.inoreader\\.com/oembed/", + ], + "e": "www.inoreader.com/oembed/api/", + }, + { + "s": [ + "*\\.inphood\\.com/*", + ], + "e": "api.inphood.com/oembed", + }, + { + "s": [ + "instagram\\.com/*/p/*,", + "www\\.instagram\\.com/*/p/*,", + "instagram\\.com/p/*", + "instagr\\.am/p/*", + "www\\.instagram\\.com/p/*", + "www\\.instagr\\.am/p/*", + "instagram\\.com/tv/*", + "instagr\\.am/tv/*", + "www\\.instagram\\.com/tv/*", + "www\\.instagr\\.am/tv/*", + "www\\.instagram\\.com/reel/*", + "instagram\\.com/reel/*", + "instagr\\.am/reel/*", + ], + "e": "graph.facebook.com/v16.0/instagram_oembed", + }, + { + "s": [ + "ppa\\.insticator\\.com/embed-unit/*", + ], + "e": "www.insticator.com/oembed", + }, + { + "s": [ + "issuu\\.com/*/docs/*", + ], + "e": "issuu.com/oembed", + }, + { + "s": [ + "play\\.itemis\\.io/*", + ], + "e": "create.storage.api.itemis.io/api/embed", + }, + { + "s": [ + "jovian\\.ml/*", + "jovian\\.ml/viewer*", + "*\\.jovian\\.ml/*", + "jovian\\.ai/*", + "jovian\\.ai/viewer*", + "*\\.jovian\\.ai/*", + "jovian\\.com/*", + "jovian\\.com/viewer*", + "*\\.jovian\\.com/*", + ], + "e": "api.jovian.com/oembed.json", + }, + { + "s": [ + "tv\\.kakao\\.com/channel/*/cliplink/*", + "tv\\.kakao\\.com/m/channel/*/cliplink/*", + "tv\\.kakao\\.com/channel/v/*", + "tv\\.kakao\\.com/channel/*/livelink/*", + "tv\\.kakao\\.com/m/channel/*/livelink/*", + "tv\\.kakao\\.com/channel/l/*", + ], + "e": "tv.kakao.com/oembed", + }, + { + "s": [ + "www\\.kickstarter\\.com/projects/*", + ], + "e": "www.kickstarter.com/services/oembed", + }, + { + "s": [ + "www\\.kidoju\\.com/en/x/*/*", + "www\\.kidoju\\.com/fr/x/*/*", + ], + "e": "www.kidoju.com/api/oembed", + }, + { + "s": [ + "halaman\\.email/form/*", + "aplikasi\\.kirim\\.email/form/*", + ], + "e": "halaman.email/service/oembed", + }, + { + "s": [ + "kit\\.co/*/*", + ], + "e": "embed.kit.co/oembed", + }, + { + "s": [ + "www\\.kitchenbowl\\.com/recipe/*", + ], + "e": "www.kitchenbowl.com/oembed", + }, + { + "s": [ + "app\\.kmdr\\.sh/h/*", + "app\\.kmdr\\.sh/history/*", + ], + "e": "api.kmdr.sh/services/oembed", + }, + { + "s": [ + "jdr\\.knacki\\.info/meuh/*", + ], + "e": "jdr.knacki.info/oembed", + }, + { + "s": [ + "knowledgepad\\.co/#/knowledge/*", + ], + "e": "api.spoonacular.com/knowledge/oembed", + }, + { + "s": [ + "*\\.kooapp\\.com/koo/*", + ], + "e": "embed.kooapp.com/services/oembed", + }, + { + "s": [ + "kurozora\\.app/episodes*", + "kurozora\\.app/songs*", + ], + "e": "kurozora.app/oembed", + }, + { + "s": [ + "learningapps\\.org/*", + ], + "e": "learningapps.org/oembed.php", + }, + { + "s": [ + "umotion-test\\.univ-lemans\\.fr/video/*", + ], + "e": "umotion-test.univ-lemans.fr/oembed", + }, + { + "s": [ + "pod\\.univ-lille\\.fr/video/*", + ], + "e": "pod.univ-lille.fr/video/oembed", + }, + { + "s": [ + "place\\.line\\.me/businesses/*", + ], + "e": "place.line.me/oembed", + }, + { + "s": [ + "livestream\\.com/accounts/*/events/*", + "livestream\\.com/accounts/*/events/*/videos/*", + "livestream\\.com/*/events/*", + "livestream\\.com/*/events/*/videos/*", + "livestream\\.com/*/*", + "livestream\\.com/*/*/videos/*", + ], + "e": "livestream.com/oembed", + }, + { + "s": [ + "lottiefiles\\.com/*", + "*\\.lottiefiles\\.com/*", + "*\\.lottie\\.host/*", + "lottie\\.host/*", + ], + "e": "embed.lottiefiles.com/oembed", + }, + { + "s": [ + "app\\.ludus\\.one/*", + ], + "e": "app.ludus.one/oembed", + }, + { + "s": [ + "*\\.lumiere\\.is/v/*", + ], + "e": "admin.lumiere.is/api/services/oembed", + }, + { + "s": [ + "mathembed\\.com/latex?inputText=*", + ], + "e": "mathembed.com/oembed", + }, + { + "s": [], + "e": "my.matterport.com/api/v1/models/oembed/", + }, + { + "s": [ + "me\\.me/i/*", + ], + "e": "me.me/oembed", + }, + { + "s": [ + "mdstrm\\.com/embed/*", + "mdstrm\\.com/live-stream/*", + "mdstrm\\.com/image/*", + ], + "e": "mdstrm.com/oembed", + }, + { + "s": [ + "medienarchiv\\.zhdk\\.ch/entries/*", + ], + "e": "medienarchiv.zhdk.ch/oembed.json", + }, + { + "s": [ + "mermaid\\.ink/img/*", + "mermaid\\.ink/svg/*", + ], + "e": "mermaid.ink/services/oembed", + }, + { + "s": [ + "*\\.microsoftstream\\.com/video/*", + "*\\.microsoftstream\\.com/channel/*", + ], + "e": "web.microsoftstream.com/oembed", + }, + { + "s": [ + "www\\.minervaknows\\.com/featured-recipes/*", + "www\\.minervaknows\\.com/themes/*", + "www\\.minervaknows\\.com/themes/*/recipes/*", + "app\\.minervaknows\\.com/recipes/*", + "app\\.minervaknows\\.com/recipes/*/follow", + ], + "e": "oembed.minervaknows.com", + }, + { + "s": [ + "miro\\.com/app/board/*", + ], + "e": "miro.com/api/v1/oembed", + }, + { + "s": [ + "www\\.mixcloud\\.com/*/*/", + ], + "e": "www.mixcloud.com/oembed/", + }, + { + "s": [ + "mixpanel\\.com/*", + ], + "e": "mixpanel.com/api/app/embed/oembed/", + }, + { + "s": [ + "www\\.mobypicture\\.com/user/*/view/*", + "moby\\.to/*", + ], + "e": "api.mobypicture.com/oEmbed", + }, + { + "s": [ + "musicboxmaniacs\\.com/explore/melody/*", + ], + "e": "musicboxmaniacs.com/embed/", + }, + { + "s": [ + "mybeweeg\\.com/w/*", + ], + "e": "mybeweeg.com/services/oembed", + }, + { + "s": [ + "namchey\\.com/embeds/*", + ], + "e": "namchey.com/api/oembed", + }, + { + "s": [ + "*\\.nanoo\\.tv/link/*", + "nanoo\\.tv/link/*", + "*\\.nanoo\\.pro/link/*", + "nanoo\\.pro/link/*", + "media\\.zhdk\\.ch/signatur/*", + "new\\.media\\.zhdk\\.ch/signatur/*", + ], + "e": "www.nanoo.tv/services/oembed", + }, + { + "s": [ + "www\\.nb\\.no/items/*", + ], + "e": "api.nb.no/catalog/v1/oembed", + }, + { + "s": [ + "naturalatlas\\.com/*", + "naturalatlas\\.com/*/*", + "naturalatlas\\.com/*/*/*", + "naturalatlas\\.com/*/*/*/*", + ], + "e": "naturalatlas.com/oembed.json", + }, + { + "s": [ + "ndla\\.no/*", + "ndla\\.no/article/*", + "ndla\\.no/audio/*", + "ndla\\.no/concept/*", + "ndla\\.no/image/*", + "ndla\\.no/video/*", + ], + "e": "ndla.no/oembed", + }, + { + "s": [ + "*\\.neetorecord\\.com/watch/*", + ], + "e": "api.neetorecord.com/api/v1/oembed", + }, + { + "s": [ + "*\\.nfb\\.ca/film/*", + ], + "e": "www.nfb.ca/remote/services/oembed/", + }, + { + "s": [ + "nopaste\\.ml/*", + ], + "e": "oembed.nopaste.ml", + }, + { + "s": [ + "observablehq\\.com/@*/*", + "observablehq\\.com/d/*", + "observablehq\\.com/embed/*", + ], + "e": "api.observablehq.com/oembed", + }, + { + "s": [ + "www\\.odds\\.com\\.au/*", + "odds\\.com\\.au/*", + ], + "e": "www.odds.com.au/api/oembed/", + }, + { + "s": [ + "song\\.link/*", + "album\\.link/*", + "artist\\.link/*", + "playlist\\.link/*", + "pods\\.link/*", + "mylink\\.page/*", + "odesli\\.co/*", + ], + "e": "song.link/oembed", + }, + { + "s": [ + "odysee\\.com/*/*", + "odysee\\.com/*", + ], + "e": "odysee.com/$/oembed", + }, + { + "s": [ + "official\\.fm/tracks/*", + "official\\.fm/playlists/*", + ], + "e": "official.fm/services/oembed.json", + }, + { + "s": [ + "omniscope\\.me/*", + ], + "e": "omniscope.me/_global_/oembed/json", + }, + { + "s": [ + "omny\\.fm/shows/*", + ], + "e": "omny.fm/oembed", + }, + { + "s": [ + "orbitvu\\.co/001/*/ov3601/view", + "orbitvu\\.co/001/*/ov3601/*/view", + "orbitvu\\.co/001/*/ov3602/*/view", + "orbitvu\\.co/001/*/2/orbittour/*/view", + "orbitvu\\.co/001/*/1/2/orbittour/*/view", + ], + "e": "orbitvu.co/service/oembed", + }, + { + "s": [ + "origits\\.com/v/*", + ], + "e": "origits.net/oembed", + }, + { + "s": [ + "origits\\.com/v/*", + ], + "e": "origits.com/oembed", + }, + { + "s": [ + "outplayed\\.tv/media/*", + ], + "e": "outplayed.tv/oembed", + }, + { + "s": [ + "overflow\\.io/s/*", + "overflow\\.io/embed/*", + ], + "e": "overflow.io/services/oembed", + }, + { + "s": [ + "www\\.oz\\.com/*/video/*", + ], + "e": "core.oz.com/oembed", + }, + { + "s": [ + "padlet\\.com/*", + ], + "e": "padlet.com/oembed/", + }, + { + "s": [ + "*\\.tv\\.pandavideo\\.com\\.br/embed/?v=*", + "*\\.tv\\.pandavideo\\.com\\.br/*/playlist\\.m3u8", + "dashboard\\.pandavideo\\.com\\.br/#/videos/*", + ], + "e": "api-v2.pandavideo.com.br/oembed", + }, + { + "s": [ + "pastery\\.net/*", + "www\\.pastery\\.net/*", + ], + "e": "www.pastery.net/oembed", + }, + { + "s": [ + "www\\.picturelfy\\.com/p/*", + ], + "e": "api.picturelfy.com/service/oembed/", + }, + { + "s": [ + "piggy\\.to/@*/*", + "piggy\\.to/view/*", + ], + "e": "piggy.to/oembed", + }, + { + "s": [ + "*\\.builder\\.pikasso\\.xyz/embed/*", + ], + "e": "builder.pikasso.xyz/api/oembed", + }, + { + "s": [], + "e": "beta.pingvp.com.kpnis.nl/p/oembed.php", + }, + { + "s": [ + "tools\\.pinpoll\\.com/embed/*", + ], + "e": "tools.pinpoll.com/oembed", + }, + { + "s": [ + "www\\.pinterest\\.com/*", + ], + "e": "www.pinterest.com/oembed.json", + }, + { + "s": [ + "player\\.pitchhub\\.com/en/public/player/*", + ], + "e": "player.pitchhub.com/en/public/oembed", + }, + { + "s": [ + "store\\.pixdor\\.com/place-marker-widget/*/show", + "store\\.pixdor\\.com/map/*/show", + ], + "e": "store.pixdor.com/oembed", + }, + { + "s": [ + "app\\.plusdocs\\.com/*/snapshots/*", + "app\\.plusdocs\\.com/*/pages/edit/*", + "app\\.plusdocs\\.com/*/pages/share/*", + ], + "e": "app.plusdocs.com/oembed", + }, + { + "s": [ + "*\\.podbean\\.com/e/*", + ], + "e": "api.podbean.com/v1/oembed", + }, + { + "s": [ + "*\\.polldaddy\\.com/s/*", + "*\\.polldaddy\\.com/poll/*", + "*\\.polldaddy\\.com/ratings/*", + ], + "e": "polldaddy.com/oembed/", + }, + { + "s": [ + "portfolium\\.com/entry/*", + ], + "e": "api.portfolium.com/oembed", + }, + { + "s": [ + "present\\.do/decks/*", + ], + "e": "gateway.cobalt.run/present/decks/oembed", + }, + { + "s": [ + "prezi\\.com/v/*", + "*\\.prezi\\.com/v/*", + ], + "e": "prezi.com/v/oembed", + }, + { + "s": [ + "qtpi\\.gg/fashion/*", + ], + "e": "qtpi.gg/fashion/oembed", + }, + { + "s": [ + "www\\.quiz\\.biz/quizz-*\\.html", + ], + "e": "www.quiz.biz/api/oembed", + }, + { + "s": [ + "www\\.quizz\\.biz/quizz-*\\.html", + ], + "e": "www.quizz.biz/api/oembed", + }, + { + "s": [ + "play\\.radiopublic\\.com/*", + "radiopublic\\.com/*", + "www\\.radiopublic\\.com/*", + "*\\.radiopublic\\.com/*", + ], + "e": "oembed.radiopublic.com/oembed", + }, + { + "s": [ + "raindrop\\.io/*", + "raindrop\\.io/*/*", + "raindrop\\.io/*/*/*", + "raindrop\\.io/*/*/*/*", + ], + "e": "pub.raindrop.io/api/oembed", + }, + { + "s": [ + "www\\.rcvis\\.com/v/*", + "www\\.rcvis\\.com/visualize=*", + "www\\.rcvis\\.com/ve/*", + "www\\.rcvis\\.com/visualizeEmbedded=*", + ], + "e": "animatron.com/oembed", + }, + { + "s": [ + "reddit\\.com/r/*/comments/*/*", + "www\\.reddit\\.com/r/*/comments/*/*", + ], + "e": "www.reddit.com/oembed", + }, + { + "s": [ + "rwire\\.com/*", + ], + "e": "publisher.releasewire.com/oembed/", + }, + { + "s": [ + "repl\\.it/@*/*", + "replit\\.com/@*/*", + ], + "e": "replit.com/data/oembed", + }, + { + "s": [ + "www\\.reverbnation\\.com/*", + "www\\.reverbnation\\.com/*/songs/*", + ], + "e": "www.reverbnation.com/oembed", + }, + { + "s": [ + "roomshare\\.jp/post/*", + "roomshare\\.jp/en/post/*", + ], + "e": "roomshare.jp/en/oembed.json", + }, + { + "s": [ + "roosterteeth\\.com/*", + ], + "e": "roosterteeth.com/oembed", + }, + { + "s": [], + "e": "rumble.com/api/Media/oembed.json", + }, + { + "s": [ + "embed\\.runkit\\.com/*,", + ], + "e": "embed.runkit.com/oembed", + }, + { + "s": [ + "octopus\\.saooti\\.com/main/pub/podcast/*", + ], + "e": "octopus.saooti.com/oembed", + }, + { + "s": [ + "videos\\.sapo\\.pt/*", + ], + "e": "videos.sapo.pt/oembed", + }, + { + "s": [ + "console\\.screen9\\.com/*", + "*\\.screen9\\.tv/*", + ], + "e": "api.screen9.com/oembed", + }, + { + "s": [ + "www\\.screencast\\.com/*", + ], + "e": "api.screencast.com/external/oembed", + }, + { + "s": [ + "www\\.screenr\\.com/*/", + ], + "e": "www.screenr.com/api/oembed.json", + }, + { + "s": [ + "www\\.scribblemaps\\.com/maps/view/*", + "scribblemaps\\.com/maps/view/*", + ], + "e": "scribblemaps.com/api/services/oembed.json", + }, + { + "s": [ + "www\\.scribd\\.com/doc/*", + ], + "e": "www.scribd.com/services/oembed/", + }, + { + "s": [ + "embed\\.sendtonews\\.com/oembed/*", + ], + "e": "embed.sendtonews.com/services/oembed", + }, + { + "s": [ + "shopshare\\.tv/shopboard/*", + "shopshare\\.tv/shopcast/*", + ], + "e": "shopshare.tv/api/shopcast/oembed", + }, + { + "s": [ + "www\\.shortnote\\.jp/view/notes/*", + ], + "e": "www.shortnote.jp/oembed/", + }, + { + "s": [ + "shoudio\\.com/*", + "shoud\\.io/*", + ], + "e": "shoudio.com/api/oembed", + }, + { + "s": [ + "app\\.getshow\\.io/iframe/*", + "*\\.getshow\\.io/share/*", + ], + "e": "api.getshow.io/oembed.json", + }, + { + "s": [ + "showtheway\\.io/to/*", + ], + "e": "showtheway.io/oembed", + }, + { + "s": [ + "simplecast\\.com/s/*", + ], + "e": "simplecast.com/oembed", + }, + { + "s": [ + "onsizzle\\.com/i/*", + ], + "e": "onsizzle.com/oembed", + }, + { + "s": [ + "sketchfab\\.com/*models/*", + "sketchfab\\.com/*/folders/*", + ], + "e": "sketchfab.com/oembed", + }, + { + "s": [ + "www\\.slideshare\\.net/*/*", + "fr\\.slideshare\\.net/*/*", + "de\\.slideshare\\.net/*/*", + "es\\.slideshare\\.net/*/*", + "pt\\.slideshare\\.net/*/*", + ], + "e": "www.slideshare.net/api/oembed/2", + }, + { + "s": [ + "smashnotes\\.com/p/*", + "smashnotes\\.com/p/*/e/* - https://smashnotes\\.com/p/*/e/*/s/*", + ], + "e": "smashnotes.com/services/oembed", + }, + { + "s": [ + "open\\.smeme\\.com/*", + ], + "e": "open.smeme.com/api/oembed", + }, + { + "s": [ + "www\\.smrthi\\.com/book/*", + ], + "e": "www.smrthi.com/api/oembed", + }, + { + "s": [ + "*\\.smugmug\\.com/*", + ], + "e": "api.smugmug.com/services/oembed/", + }, + { + "s": [ + "www\\.socialexplorer\\.com/*/explore", + "www\\.socialexplorer\\.com/*/view", + "www\\.socialexplorer\\.com/*/edit", + "www\\.socialexplorer\\.com/*/embed", + ], + "e": "www.socialexplorer.com/services/oembed/", + }, + { + "s": [ + "soundcloud\\.com/*", + "on\\.soundcloud\\.com/*", + "soundcloud\\.app\\.goog\\.gl/*", + ], + "e": "soundcloud.com/oembed", + }, + { + "s": [ + "speakerdeck\\.com/*/*", + ], + "e": "speakerdeck.com/oembed.json", + }, + { + "s": [ + "open\\.spotify\\.com/*", + "spotify:*", + ], + "e": "open.spotify.com/oembed", + }, + { + "s": [ + "*\\.spotlightr\\.com/watch/*", + "*\\.spotlightr\\.com/publish/*", + "*\\.cdn\\.spotlightr\\.com/watch/*", + "*\\.cdn\\.spotlightr\\.com/publish/*", + ], + "e": "api.spotlightr.com/getOEmbed", + }, + { + "s": [ + "*\\.spreaker\\.com/*", + ], + "e": "api.spreaker.com/oembed", + }, + { + "s": [ + "sproutvideo\\.com/videos/*", + "*\\.vids\\.io/videos/*", + ], + "e": "sproutvideo.com/oembed.json", + }, + { + "s": [ + "spyke\\.social/p/*", + "spyke\\.social/u/*", + "spyke\\.social/g/*", + "spyke\\.social/c/*", + "www\\.spyke\\.social/p/*", + "www\\.spyke\\.social/u/*", + "www\\.spyke\\.social/g/*", + "www\\.spyke\\.social/c/*", + ], + "e": "api.spyke.social/embed/oembed", + }, + { + "s": [ + "purl\\.stanford\\.edu/*", + ], + "e": "purl.stanford.edu/embed.json", + }, + { + "s": [ + "streamable\\.com/*", + ], + "e": "api.streamable.com/oembed.json", + }, + { + "s": [ + "s3m\\.io/*", + "23m\\.io/*", + ], + "e": "streamio.com/api/v1/oembed", + }, + { + "s": [ + "subscribi\\.io/api/oembed*", + ], + "e": "subscribi.io/api/oembed", + }, + { + "s": [ + "www\\.sudomemo\\.net/watch/*", + "flipnot\\.es/*", + ], + "e": "www.sudomemo.net/oembed", + }, + { + "s": [ + "www\\.sutori\\.com/story/*", + ], + "e": "www.sutori.com/api/oembed", + }, + { + "s": [ + "sway\\.com/*", + "www\\.sway\\.com/*", + ], + "e": "sway.com/api/v1.0/oembed", + }, + { + "s": [ + "sway\\.office\\.com/*", + ], + "e": "sway.office.com/api/v1.0/oembed", + }, + { + "s": [ + "share\\.synthesia\\.io/*", + ], + "e": "69jr5v75rc.execute-api.eu-west-1.amazonaws.com/prod/v2/oembed", + }, + { + "s": [ + "ted\\.com/talks/*", + "www\\.ted\\.com/talks/*", + ], + "e": "www.ted.com/services/v1/oembed.json", + }, + { + "s": [ + "www\\.nytimes\\.com/svc/oembed", + "nytimes\\.com/*", + "*\\.nytimes\\.com/*", + ], + "e": "www.nytimes.com/svc/oembed/json/", + }, + { + "s": [ + "theysaidso\\.com/image/*", + ], + "e": "theysaidso.com/extensions/oembed/", + }, + { + "s": [ + "www\\.tickcounter\\.com/widget/*", + "www\\.tickcounter\\.com/countdown/*", + "www\\.tickcounter\\.com/countup/*", + "www\\.tickcounter\\.com/ticker/*", + "www\\.tickcounter\\.com/clock/*", + "www\\.tickcounter\\.com/worldclock/*", + ], + "e": "www.tickcounter.com/oembed", + }, + { + "s": [ + "www\\.tiktok\\.com/*", + "www\\.tiktok\\.com/*/video/*", + ], + "e": "www.tiktok.com/oembed", + }, + { + "s": [ + "tonicaudio\\.com/take/*", + "tonicaudio\\.com/song/*", + "tnic\\.io/song/*", + "tnic\\.io/take/*", + ], + "e": "tonicaudio.com/oembed", + }, + { + "s": [ + "www\\.toornament\\.com/tournaments/*/information", + "www\\.toornament\\.com/tournaments/*/registration/", + "www\\.toornament\\.com/tournaments/*/matches/schedule", + "www\\.toornament\\.com/tournaments/*/stages/*/", + ], + "e": "widget.toornament.com/oembed", + }, + { + "s": [ + "www\\.topy\\.se/image/*", + ], + "e": "www.topy.se/oembed/", + }, + { + "s": [ + "app-test\\.totango\\.com/*", + ], + "e": "app-test.totango.com/oembed", + }, + { + "s": [ + "trackspace\\.upitup\\.com/*", + ], + "e": "trackspace.upitup.com/oembed", + }, + { + "s": [ + "trinitymedia\\.ai/player/*", + "trinitymedia\\.ai/player/*/*", + "trinitymedia\\.ai/player/*/*/*", + ], + "e": "trinitymedia.ai/services/oembed", + }, + { + "s": [ + "*\\.tumblr\\.com/post/*", + ], + "e": "www.tumblr.com/oembed/1.0", + }, + { + "s": [ + "www\\.tuxx\\.be/*", + ], + "e": "www.tuxx.be/services/oembed", + }, + { + "s": [ + "play\\.tvcf\\.co\\.kr/*", + "*\\.tvcf\\.co\\.kr/*", + ], + "e": "play.tvcf.co.kr/rest/oembed", + }, + { + "s": [ + "twinmotion\\.unrealengine\\.com/presentation/*", + "twinmotion\\.unrealengine\\.com/panorama/*", + ], + "e": "twinmotion.unrealengine.com/oembed", + }, + { + "s": [ + "twitter\\.com/*", + "twitter\\.com/*/status/*", + "*\\.twitter\\.com/*/status/*", + ], + "e": "publish.twitter.com/oembed", + }, + { + "s": [ + "play\\.typecast\\.ai/s/*", + "play\\.typecast\\.ai/e/*", + "play\\.typecast\\.ai/*", + ], + "e": "play.typecast.ai/oembed", + }, + { + "s": [], + "e": "typlog.com/oembed", + }, + { + "s": [ + "uapod\\.univ-antilles\\.fr/video/*", + ], + "e": "uapod.univ-antilles.fr/oembed", + }, + { + "s": [ + "map\\.cam\\.ac\\.uk/*", + ], + "e": "map.cam.ac.uk/oembed/", + }, + { + "s": [ + "mediatheque\\.univ-paris1\\.fr/video/*", + ], + "e": "mediatheque.univ-paris1.fr/oembed", + }, + { + "s": [ + "pod\\.u-pec\\.fr/video/*", + ], + "e": "pod.u-pec.fr/oembed", + }, + { + "s": [ + "*\\.ustream\\.tv/*", + "*\\.ustream\\.com/*", + ], + "e": "www.ustream.tv/oembed", + }, + { + "s": [ + "*\\.ustudio\\.com/embed/*", + "*\\.ustudio\\.com/embed/*/*", + ], + "e": "app.ustudio.com/api/v2/oembed", + }, + { + "s": [ + "veer\\.tv/videos/*", + ], + "e": "api.veer.tv/oembed", + }, + { + "s": [ + "veervr\\.tv/videos/*", + ], + "e": "api.veervr.tv/oembed", + }, + { + "s": [ + "www\\.vevo\\.com/*", + ], + "e": "www.vevo.com/oembed", + }, + { + "s": [ + "videfit\\.com/videos/*", + ], + "e": "videfit.com/oembed", + }, + { + "s": [ + "vidmount\\.com/*", + ], + "e": "vidmount.com/oembed", + }, + { + "s": [ + "*\\.vidyard\\.com/*", + "*\\.hubs\\.vidyard\\.com/*", + ], + "e": "api.vidyard.com/dashboard/v1.1/oembed", + }, + { + "s": [ + "vimeo\\.com/*", + "vimeo\\.com/album/*/video/*", + "vimeo\\.com/channels/*/*", + "vimeo\\.com/groups/*/videos/*", + "vimeo\\.com/ondemand/*/*", + "player\\.vimeo\\.com/video/*", + "vimeo\\.com/event/*/*", + ], + "e": "vimeo.com/api/oembed.json", + }, + { + "s": [ + "share\\.viostream\\.com/*", + ], + "e": "play.viostream.com/oembed", + }, + { + "s": [ + "www\\.viously\\.com/*/*", + ], + "e": "www.viously.com/oembed", + }, + { + "s": [ + "vizydrop\\.com/shared/*", + ], + "e": "vizydrop.com/oembed", + }, + { + "s": [ + "vlipsy\\.com/*", + ], + "e": "vlipsy.com/oembed", + }, + { + "s": [ + "www\\.vlive\\.tv/video/*", + ], + "e": "www.vlive.tv/oembed", + }, + { + "s": [ + "*\\.vouchfor\\.com/*", + ], + "e": "embed.vouchfor.com/v1/oembed", + }, + { + "s": [ + "article\\.voxsnap\\.com/*/*", + ], + "e": "data.voxsnap.com/oembed", + }, + { + "s": [ + "waltrack\\.net/product/*", + ], + "e": "waltrack.net/oembed", + }, + { + "s": [ + "watch\\.wave\\.video/*", + "embed\\.wave\\.video/*", + ], + "e": "embed.wave.video/oembed", + }, + { + "s": [ + "www\\.web3isgoinggreat\\.com/?id=*", + "www\\.web3isgoinggreat\\.com/single/*", + "www\\.web3isgoinggreat\\.com/embed/*", + ], + "e": "www.web3isgoinggreat.com/api/oembed", + }, + { + "s": [ + "play\\.wecandeo\\.com/video/v/*", + ], + "e": "play.wecandeo.com/oembed/", + }, + { + "s": [ + "whimsical\\.com/*", + ], + "e": "whimsical.com/api/oembed", + }, + { + "s": [ + "fast\\.wistia\\.com/embed/iframe/*", + "fast\\.wistia\\.com/embed/playlists/*", + "*\\.wistia\\.com/medias/*", + ], + "e": "fast.wistia.com/oembed.json", + }, + { + "s": [ + "*\\.wizer\\.me/learn/*", + "*\\.wizer\\.me/preview/*", + ], + "e": "app.wizer.me/api/oembed.json", + }, + { + "s": [ + "wokwi\\.com/share/*", + ], + "e": "wokwi.com/api/oembed", + }, + { + "s": [ + "*\\.wolframcloud\\.com/*", + ], + "e": "www.wolframcloud.com/oembed", + }, + { + "s": [ + "wordpress\\.com/*", + "*\\.wordpress\\.com/*", + "*\\.*\\.wordpress\\.com/*", + "wp\\.me/*", + ], + "e": "public-api.wordpress.com/oembed/", + }, + { + "s": [ + "x\\.com/*", + "x\\.com/*/status/*", + "*\\.x\\.com/*/status/*", + ], + "e": "publish.x.com/oembed", + }, + { + "s": [ + "*\\.youtube\\.com/watch*", + "*\\.youtube\\.com/v/*", + "youtu\\.be/*", + "*\\.youtube\\.com/playlist?list=*", + "youtube\\.com/playlist?list=*", + "*\\.youtube\\.com/shorts*", + ], + "e": "www.youtube.com/oembed", + }, + { + "s": [ + "www\\.yumpu\\.com/*/document/view/*/*", + ], + "e": "www.yumpu.com/services/oembed", + }, + { + "s": [ + "app\\.zeplin\\.io/project/*/screen/*", + "app\\.zeplin\\.io/project/*/screen/*/version/*", + "app\\.zeplin\\.io/project/*/styleguide/components?coid=*", + "app\\.zeplin\\.io/styleguide/*/components?coid=*", + ], + "e": "app.zeplin.io/embed", + }, + { + "s": [ + "app\\.zingsoft\\.com/embed/*", + "app\\.zingsoft\\.com/view/*", + ], + "e": "app.zingsoft.com/oembed", + }, + { + "s": [ + "*\\.znipe\\.tv/*", + ], + "e": "api.znipe.tv/v3/oembed/", + }, + { + "s": [ + "srv2\\.zoomable\\.ca/viewer\\.php*", + ], + "e": "srv2.zoomable.ca/oembed", + }, +]; diff --git a/utils/retrieve.ts b/utils/retrieve.ts new file mode 100644 index 0000000..23ea6cc --- /dev/null +++ b/utils/retrieve.ts @@ -0,0 +1,92 @@ +// retrieve.ts + +export const USER_AGENT = + "Mozilla/5.0 (X11; Linux x86_64; rv:126.0) Gecko/20100101 Firefox/126.0"; + +export interface ProxyConfig { + target?: string; + headers?: Record; +} + +export interface FetchOptions { + headers?: Record; + /** + * the values to configure proxy + * default: null + */ + proxy?: ProxyConfig; + /** + * http proxy agent + * default: null + */ + agent?: object; + /** + * signal to terminate request + * default: null + */ + signal?: object; +} + +const profetch = async ( + url: string, + options: { proxy?: ProxyConfig } = {}, +): Promise => { + const { proxy = {} } = options; + const { target, headers = {} } = proxy; + + const res = await fetch(target + encodeURIComponent(url), { + headers, + }); + + return res; +}; + +export const getHtml = async ( + url: string, + options: FetchOptions = {}, +): Promise => { + const { + headers = { + "user-agent": USER_AGENT, + }, + proxy = null, + } = options; + + const res = proxy + ? await profetch(url, { proxy }) + : await fetch(url, { headers }); + + if (!res.ok) { + throw new Error(`Request failed with error code ${res.status}`); + } + + const text = await res.text(); + return text; +}; + +export const getJson = async ( + url: string, + options: FetchOptions = {}, +): Promise => { + const { + headers = { + "user-agent": USER_AGENT, + }, + proxy = null, + } = options; + + const res = proxy + ? await profetch(url, { proxy }) + : await fetch(url, { headers }); + + if (!res.ok) { + throw new Error(`Request failed with error code ${res.status}`); + } + + try { + const text = await res.text(); + return JSON.parse(text.trim()); + } catch { + throw new Error("Failed to convert data to JSON object"); + } +};