diff --git a/.dockerignore b/.dockerignore index 75f7f37..34c47f9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -25,4 +25,5 @@ build *.secrets .devcontainer .vscode -**/act/* \ No newline at end of file +**/act/* +tmp-* diff --git a/.gitignore b/.gitignore index 3c305f5..beb432f 100644 --- a/.gitignore +++ b/.gitignore @@ -133,4 +133,5 @@ flatpak/generated-sources.json build !setupProxy.js **/generated-sources.* -.github/act/out \ No newline at end of file +.github/act/out +tmp-* diff --git a/.mocharc.json b/.mocharc.json index ad3584a..15d99f3 100644 --- a/.mocharc.json +++ b/.mocharc.json @@ -2,5 +2,6 @@ "reporter": "dot", "extension": "ts", "import": "tsx/esm", - "spec": "./src/backend/tests/**/*.test.ts" + "spec": "./src/backend/tests/**/*.test.ts", + "exit": true } diff --git a/config/lastfm.json.example b/config/lastfm.json.example index 7bfdcf5..3cb4fe7 100644 --- a/config/lastfm.json.example +++ b/config/lastfm.json.example @@ -1,6 +1,6 @@ [ { - "name": "myLastFm", + "name": "myLastFmClient", "enable": true, "configureAs": "client", "data": { @@ -8,5 +8,15 @@ "secret": "ec42e09d5ae0ee0f0816ca151008412a", "redirectUri": "http://localhost:9078/lastfm/callback" } + }, + { + "name": "myLastFmSource", + "enable": true, + "configureAs": "source", + "data": { + "apiKey": "a89cba1569901a0671d5a9875fed4be1", + "secret": "ec42e09d5ae0ee0f0816ca151008412a", + "redirectUri": "http://localhost:9078/lastfm/callback" + } } ] diff --git a/config/listenbrainz.json.example b/config/listenbrainz.json.example index 5bdf8dd..fe8b8bb 100644 --- a/config/listenbrainz.json.example +++ b/config/listenbrainz.json.example @@ -1,11 +1,20 @@ [ { - "name": "brainz", + "name": "brainzClient", "enable": true, "configureAs": "client", "data": { "token": "029b081ba-9156-4pe7-88e5-3be671f5ea2b", "username": "FoxxMD" } + }, + { + "name": "brainzSource", + "enable": true, + "configureAs": "source", + "data": { + "token": "029b081ba-9156-4pe7-88e5-3be671f5ea2b", + "username": "FoxxMD" + } } ] diff --git a/config/webscrobbler.json.example b/config/webscrobbler.json.example index 2eee548..3d69853 100644 --- a/config/webscrobbler.json.example +++ b/config/webscrobbler.json.example @@ -2,7 +2,7 @@ { "name": "MyWebScrobbler", "data": { - "slug": null, + "slug": "MyOptionalSlug", "whitelist": [], "blacklist": [] } diff --git a/package-lock.json b/package-lock.json index 61fe737..48acff7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47,6 +47,7 @@ "fast-deep-equal": "^3.1.3", "fixed-size-list": "^0.3.0", "formidable": "^3.5", + "glob": "^11.0.0", "gotify": "^1.1.0", "iso-websocket": "^0.3.0", "iti": "^0.6.0", @@ -122,7 +123,8 @@ "typescript": "^5.5.4", "typescript-eslint": "^7.0.1", "typescript-json-schema": "^0.61.0", - "vite": "^5.2.12" + "vite": "^5.2.12", + "with-local-tmp-dir": "^5.1.1" }, "engines": { "node": ">=18.19.1", @@ -633,6 +635,18 @@ "resolved": "https://registry.npmjs.org/@donedeal0/superdiff/-/superdiff-1.1.3.tgz", "integrity": "sha512-lp316jRb+OCCm8KbzidppWCKteISwgpWU6sxzzDI+PJtflfX87h/AHVQdcqY1f3/za2S869KliJbtJZuan1H2Q==" }, + "node_modules/@dword-design/chdir": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@dword-design/chdir/-/chdir-2.1.4.tgz", + "integrity": "sha512-cHpMTx4XepHQi3hmDnKma5YXU4pm01k9xKI1k8Wwq25cN3j0FCWbiji11hdrRYr6+uDC9mrb/M5Mo0okYiNRzA==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/dword-design" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.23.1", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.23.1.tgz", @@ -5446,20 +5460,22 @@ "integrity": "sha512-FDSgeMqa7GnJDxt/q0AbrxbfeTyxp4ImxEw1e4nw6NUHA5FMhFUq33dTXI4Xdgcj1VQ1q5QLWF6WxFrJ8KCBOg==" }, "node_modules/glob": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", - "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", - "deprecated": "Glob versions prior to v9 are no longer supported", - "dev": true, + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-11.0.0.tgz", + "integrity": "sha512-9UiX/Bl6J2yaBbxKoEBRm4Cipxgok8kQYcOPEhScPwebu2I0HoQOuYdIO6S3hLuWoZgpDpwQZMzTFxgpkyT76g==", "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^5.0.1", - "once": "^1.3.0" + "foreground-child": "^3.1.0", + "jackspeak": "^4.0.1", + "minimatch": "^10.0.0", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^2.0.0" + }, + "bin": { + "glob": "dist/esm/bin.mjs" }, "engines": { - "node": ">=12" + "node": "20 || >=22" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -5480,21 +5496,59 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, "dependencies": { "balanced-match": "^1.0.0" } }, + "node_modules/glob/node_modules/jackspeak": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.0.2.tgz", + "integrity": "sha512-bZsjR/iRjl1Nk1UkjGpAzLNfQtzuijhn2g+pbZb98HQ1Gk8vM9hfbxeMBP+M2/UUdwj0RqGG3mlvk2MsAqwvEw==", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob/node_modules/lru-cache": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.0.1.tgz", + "integrity": "sha512-CgeuL5uom6j/ZVrg7G/+1IXqRY8JXX4Hghfy5YE0EhoYQWvndP1kufu58cmZLNIDKnRhZrXfdS9urVWx98AipQ==", + "engines": { + "node": "20 || >=22" + } + }, "node_modules/glob/node_modules/minimatch": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", - "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", - "dev": true, + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.0.1.tgz", + "integrity": "sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ==", "dependencies": { "brace-expansion": "^2.0.1" }, "engines": { - "node": ">=10" + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob/node_modules/path-scurry": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.0.tgz", + "integrity": "sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg==", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/globals": { @@ -7063,6 +7117,26 @@ "balanced-match": "^1.0.0" } }, + "node_modules/mocha/node_modules/glob": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^5.0.1", + "once": "^1.3.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/mocha/node_modules/minimatch": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", @@ -10043,6 +10117,24 @@ "node": ">=0.6.0" } }, + "node_modules/tmp-promise": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-3.0.3.tgz", + "integrity": "sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==", + "dev": true, + "dependencies": { + "tmp": "^0.2.0" + } + }, + "node_modules/tmp-promise/node_modules/tmp": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz", + "integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==", + "dev": true, + "engines": { + "node": ">=14.14" + } + }, "node_modules/to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", @@ -11182,6 +11274,22 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/with-local-tmp-dir": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/with-local-tmp-dir/-/with-local-tmp-dir-5.1.1.tgz", + "integrity": "sha512-7HZx3lC2BNMqrkob/29+GxBZtt0J2j5qyYn77/FivEAKS2Q90Tlc6KaqjBAPnVo0B3OfHDtCBZG3i8S6kD8N8A==", + "dev": true, + "dependencies": { + "@dword-design/chdir": "^2.0.0", + "tmp-promise": "^3.0.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/dword-design" + } + }, "node_modules/word-wrap": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", diff --git a/package.json b/package.json index a5fe83a..f5c8cb9 100644 --- a/package.json +++ b/package.json @@ -82,6 +82,7 @@ "fast-deep-equal": "^3.1.3", "fixed-size-list": "^0.3.0", "formidable": "^3.5", + "glob": "^11.0.0", "gotify": "^1.1.0", "iso-websocket": "^0.3.0", "iti": "^0.6.0", @@ -157,7 +158,8 @@ "typescript": "^5.5.4", "typescript-eslint": "^7.0.1", "typescript-json-schema": "^0.61.0", - "vite": "^5.2.12" + "vite": "^5.2.12", + "with-local-tmp-dir": "^5.1.1" }, "browserslist": { "production": [ @@ -174,6 +176,11 @@ "overrides": { "spotify-web-api-node": { "superagent": "$superagent" + }, + "with-local-tmp-dir": { + "tmp-promise": { + "tmp": "0.2.1" + } } } } diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index 109b8fb..158f4ca 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -48,6 +48,10 @@ export const sourceTypes: SourceType[] = [ 'vlc' ]; +export const isSourceType = (data: string): data is SourceType => { + return sourceTypes.includes(data as SourceType); +} + export const lowGranularitySources: SourceType[] = ['subsonic', 'ytmusic']; export type ClientType = @@ -59,6 +63,9 @@ export const clientTypes: ClientType[] = [ 'lastfm', 'listenbrainz' ]; +export const isClientType = (data: string): data is ClientType => { + return clientTypes.includes(data as ClientType); +} export type InitState = 0 | 1 | 2; export const NOT_INITIALIZED: InitState = 0; diff --git a/src/backend/common/infrastructure/config/aioConfig.ts b/src/backend/common/infrastructure/config/aioConfig.ts index 05cd5e5..cfe6134 100644 --- a/src/backend/common/infrastructure/config/aioConfig.ts +++ b/src/backend/common/infrastructure/config/aioConfig.ts @@ -5,6 +5,7 @@ import { RequestRetryOptions } from "./common.js"; import { WebhookConfig } from "./health/webhooks.js"; import { CommonSourceOptions, SourceRetryOptions } from "./source/index.js"; import { SourceAIOConfig } from "./source/sources.js"; +import { ClientType, SourceType } from "../Atomic.js"; export interface SourceDefaults extends CommonSourceOptions { @@ -70,7 +71,22 @@ export interface AIOClientConfig { clients?: ClientAIOConfig[] } +export interface AIOClientRelaxedConfig { + clientDefaults?: RequestRetryOptions + clients?: object[] +} + export interface AIOSourceConfig { sourceDefaults?: SourceRetryOptions sources?: SourceAIOConfig[] } + +export interface AIOSourceRelaxedConfig { + sourceDefaults?: SourceRetryOptions + sources?: object[] +} + +export interface TypedConfig { + type: T + // [key: string]: any +} \ No newline at end of file diff --git a/src/backend/common/infrastructure/config/source/deezer.ts b/src/backend/common/infrastructure/config/source/deezer.ts index 93832b5..23839b1 100644 --- a/src/backend/common/infrastructure/config/source/deezer.ts +++ b/src/backend/common/infrastructure/config/source/deezer.ts @@ -19,7 +19,7 @@ export interface DeezerData extends CommonSourceData { * @default "http://localhost:9078/deezer/callback" * @examples ["http://localhost:9078/deezer/callback"] * */ - redirectUri: string + redirectUri?: string /** * optional, how long to wait before calling spotify for new tracks (in seconds) * diff --git a/src/backend/common/infrastructure/config/source/jellyfin.ts b/src/backend/common/infrastructure/config/source/jellyfin.ts index 2ecfc02..b3bd29b 100644 --- a/src/backend/common/infrastructure/config/source/jellyfin.ts +++ b/src/backend/common/infrastructure/config/source/jellyfin.ts @@ -89,3 +89,6 @@ export interface JellyApiSourceConfig extends CommonSourceConfig { export interface JellyApiSourceAIOConfig extends JellyApiSourceConfig { type: 'jellyfin' } + + +export type JellyfinCompatConfig = JellyApiSourceConfig | JellySourceConfig; \ No newline at end of file diff --git a/src/backend/common/infrastructure/config/source/spotify.ts b/src/backend/common/infrastructure/config/source/spotify.ts index 742e0da..6feae73 100644 --- a/src/backend/common/infrastructure/config/source/spotify.ts +++ b/src/backend/common/infrastructure/config/source/spotify.ts @@ -20,7 +20,7 @@ export interface SpotifySourceData extends CommonSourceData, PollingOptions { * @default "http://localhost:9078/callback" * @examples ["http://localhost:9078/callback"] * */ - redirectUri: string + redirectUri?: string /** * How long to wait before polling the source API for new tracks (in seconds) * diff --git a/src/backend/common/infrastructure/config/source/vlc.ts b/src/backend/common/infrastructure/config/source/vlc.ts index 43261cf..45f8caf 100644 --- a/src/backend/common/infrastructure/config/source/vlc.ts +++ b/src/backend/common/infrastructure/config/source/vlc.ts @@ -25,7 +25,7 @@ export interface VLCSourceOptions extends CommonSourceOptions { * * Used when VLC reports only the filename for the current audio track * */ - filenamePatterns?: string[] + filenamePatterns?: string | string[] /** * Log to DEBUG when a filename-only track is matched or not matched by filenamePatterns * diff --git a/src/backend/common/schema/aio-source.json b/src/backend/common/schema/aio-source.json index 997e3bd..a3417b0 100644 --- a/src/backend/common/schema/aio-source.json +++ b/src/backend/common/schema/aio-source.json @@ -374,8 +374,7 @@ }, "required": [ "clientId", - "clientSecret", - "redirectUri" + "clientSecret" ], "title": "DeezerData", "type": "object" @@ -2230,8 +2229,7 @@ }, "required": [ "clientId", - "clientSecret", - "redirectUri" + "clientSecret" ], "title": "SpotifySourceData", "type": "object" @@ -2507,12 +2505,19 @@ "type": "boolean" }, "filenamePatterns": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "string" + } + ], "description": "A list of regular expressions to use to extract metadata (title, album, artist) from a filename\n\nUsed when VLC reports only the filename for the current audio track", - "items": { - "type": "string" - }, - "title": "filenamePatterns", - "type": "array" + "title": "filenamePatterns" }, "logFilenamePatterns": { "default": false, diff --git a/src/backend/common/schema/aio.json b/src/backend/common/schema/aio.json index 2d15a1a..d4ad4af 100644 --- a/src/backend/common/schema/aio.json +++ b/src/backend/common/schema/aio.json @@ -651,8 +651,7 @@ }, "required": [ "clientId", - "clientSecret", - "redirectUri" + "clientSecret" ], "title": "DeezerData", "type": "object" @@ -3174,8 +3173,7 @@ }, "required": [ "clientId", - "clientSecret", - "redirectUri" + "clientSecret" ], "title": "SpotifySourceData", "type": "object" @@ -3451,12 +3449,19 @@ "type": "boolean" }, "filenamePatterns": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "string" + } + ], "description": "A list of regular expressions to use to extract metadata (title, album, artist) from a filename\n\nUsed when VLC reports only the filename for the current audio track", - "items": { - "type": "string" - }, - "title": "filenamePatterns", - "type": "array" + "title": "filenamePatterns" }, "logFilenamePatterns": { "default": false, diff --git a/src/backend/common/schema/source.json b/src/backend/common/schema/source.json index 7252576..79e5d83 100644 --- a/src/backend/common/schema/source.json +++ b/src/backend/common/schema/source.json @@ -425,8 +425,7 @@ }, "required": [ "clientId", - "clientSecret", - "redirectUri" + "clientSecret" ], "title": "DeezerData", "type": "object" @@ -2082,8 +2081,7 @@ }, "required": [ "clientId", - "clientSecret", - "redirectUri" + "clientSecret" ], "title": "SpotifySourceData", "type": "object" @@ -2335,12 +2333,19 @@ "type": "boolean" }, "filenamePatterns": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "string" + } + ], "description": "A list of regular expressions to use to extract metadata (title, album, artist) from a filename\n\nUsed when VLC reports only the filename for the current audio track", - "items": { - "type": "string" - }, - "title": "filenamePatterns", - "type": "array" + "title": "filenamePatterns" }, "logFilenamePatterns": { "default": false, diff --git a/src/backend/index.ts b/src/backend/index.ts index b74a34c..fe72ce9 100644 --- a/src/backend/index.ts +++ b/src/backend/index.ts @@ -16,6 +16,7 @@ import { initServer } from "./server/index.js"; import { createHeartbeatClientsTask } from "./tasks/heartbeatClients.js"; import { createHeartbeatSourcesTask } from "./tasks/heartbeatSources.js"; import { parseBool, readJson, sleep } from "./utils.js"; +import { getTsConfigGenerator } from './utils/SchemaUtils.js'; dayjs.extend(utc) dayjs.extend(isBetween); @@ -23,6 +24,7 @@ dayjs.extend(relativeTime); dayjs.extend(duration); dayjs.extend(timezone); +// eslint-disable-next-line prefer-arrow-functions/prefer-arrow-functions (async function () { const scheduler = new ToadScheduler() @@ -83,6 +85,10 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`) const root = getRoot({...config, logger}); initLogger.info(`Version: ${root.get('version')}`); + initLogger.info('Generating schema definitions...'); + getTsConfigGenerator(); + initLogger.info('Schema definitions generated'); + initServer(logger, appLoggerStream, output); if(process.env.IS_LOCAL === 'true') { @@ -113,6 +119,12 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`) const scrobbleSources = root.get('sources'); await scrobbleSources.buildSourcesFromConfig([]); + for(const source of scrobbleSources.sources) { + if(!source.isReady()) { + await source.initialize(); + } + } + scrobbleSources.logger.info('Finished initializing sources'); // check ambiguous client/source types like this for now const lastfmSources = scrobbleSources.getByType('lastfm'); diff --git a/src/backend/ioc.ts b/src/backend/ioc.ts index 4e43ea2..de08740 100644 --- a/src/backend/ioc.ts +++ b/src/backend/ioc.ts @@ -37,14 +37,22 @@ const createRoot = (options?: RootOptions) => { disableWeb = process.env.DISABLE_WEB === 'true'; } + const cEmitter = new WildcardEmitter(); + // do nothing, just catch + cEmitter.on('error', (e) => null); + const sEmitter = new WildcardEmitter(); + sEmitter.on('error', (e) => { + const f = e; + }); + return createContainer().add({ version, configDir: configDir, isProd: process.env.NODE_ENV !== undefined && (process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'prod'), port: process.env.PORT ?? port, disableWeb, - clientEmitter: () => new WildcardEmitter(), - sourceEmitter: () => new WildcardEmitter(), + clientEmitter: () => cEmitter, + sourceEmitter: () => sEmitter, notifierEmitter: () => new EventEmitter(), }).add((items) => { const localUrl = generateBaseURL(baseUrl, items.port) diff --git a/src/backend/scrobblers/ScrobbleClients.ts b/src/backend/scrobblers/ScrobbleClients.ts index 886147f..e649992 100644 --- a/src/backend/scrobblers/ScrobbleClients.ts +++ b/src/backend/scrobblers/ScrobbleClients.ts @@ -2,21 +2,22 @@ import { childLogger, Logger } from '@foxxmd/logging'; import dayjs, { Dayjs } from "dayjs"; import { PlayObject } from "../../core/Atomic.js"; -import { clientTypes, ConfigMeta } from "../common/infrastructure/Atomic.js"; +import { ClientType, clientTypes, ConfigMeta, isClientType } from "../common/infrastructure/Atomic.js"; import { AIOConfig } from "../common/infrastructure/config/aioConfig.js"; import { ClientAIOConfig, ClientConfig } from "../common/infrastructure/config/client/clients.js"; import { LastfmClientConfig } from "../common/infrastructure/config/client/lastfm.js"; import { ListenBrainzClientConfig } from "../common/infrastructure/config/client/listenbrainz.js"; import { MalojaClientConfig } from "../common/infrastructure/config/client/maloja.js"; -import * as aioSchema from '../common/schema/aio-client.json'; -import * as clientSchema from '../common/schema/client.json'; import { WildcardEmitter } from "../common/WildcardEmitter.js"; import { Notifiers } from "../notifier/Notifiers.js"; -import { joinedUrl, readJson, validateJson, } from "../utils.js"; +import { joinedUrl, readJson, thresholdResultSummary } from "../utils.js"; +import { getTypeSchemaFromConfigGenerator } from "../utils/SchemaUtils.js"; +import { validateJson } from "../utils/ValidationUtils.js"; import AbstractScrobbleClient from "./AbstractScrobbleClient.js"; import LastfmScrobbler from "./LastfmScrobbler.js"; import ListenbrainzScrobbler from "./ListenbrainzScrobbler.js"; import MalojaScrobbler from "./MalojaScrobbler.js"; +import { Definition } from 'typescript-json-schema'; type groupedNamedConfigs = {[key: string]: ParsedConfig[]}; @@ -30,6 +31,8 @@ export default class ScrobbleClients { configDir: string; localUrl: URL; + private schemaDefinitions: Record = {}; + emitter: WildcardEmitter; sourceEmitter: WildcardEmitter; @@ -74,6 +77,23 @@ export default class ScrobbleClients { return [clientsReady, messages]; } + private getSchemaByType = (type: ClientType): Definition => { + if(this.schemaDefinitions[type] === undefined) { + switch(type) { + case 'maloja': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("MalojaClientConfig"); + break; + case 'lastfm': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("LastfmClientConfig"); + break; + case 'listenbrainz': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("ListenBrainzClientConfig"); + break; + } + } + return this.schemaDefinitions[type]; + } + buildClientsFromConfig = async (notifier: Notifiers) => { const configs: ParsedConfig[] = []; @@ -84,27 +104,37 @@ export default class ScrobbleClients { // think this should stay as show-stopper since config could include important defaults (delay, retries) we don't want to ignore throw new Error('config.json could not be parsed'); } + + const relaxedSchema = getTypeSchemaFromConfigGenerator("AIOClientRelaxedConfig"); + let clientDefaults = {}; if (configFile !== undefined) { - const aioConfig = validateJson(configFile, aioSchema, this.logger); + const aioConfig = validateJson(configFile, relaxedSchema, this.logger); const { clients: mainConfigClientConfigs = [], clientDefaults: cd = {}, } = aioConfig; clientDefaults = cd; - // const validMainConfigs = mainConfigClientConfigs.reduce((acc: any, curr: any, i: any) => { - // if(curr === null) { - // this.logger.error(`The client config entry at index ${i} in config.json is null but should be an object, will not parse`); - // return acc; - // } - // if(typeof curr !== 'object') { - // this.logger.error(`The client config entry at index ${i} in config.json should be an object, will not parse`); - // return acc; - // } - // return acc.concat(curr); - // }, []); - for (const c of mainConfigClientConfigs) { + for (const [index, c] of mainConfigClientConfigs.entries()) { const {name = 'unnamed'} = c; + if(!isClientType(c.type.toLocaleLowerCase())) { + const invalidTypeMsg = `Client config ${index + 1} (${name}) in config.json has an invalid client type of '${c.type}'. Must be one of ${clientTypes.join(' | ')}`; + //this.emitter.emit('error', new Error(invalidTypeMsg)); + this.logger.error(invalidTypeMsg); + continue; + } + if(['lastfm','listenbrainz'].includes(c.type.toLocaleLowerCase()) && ((c as LastfmClientConfig | ListenBrainzClientConfig).configureAs === 'source')) { + this.logger.debug(`Skipping config ${index + 1} (${name}) in config.json because it is configured as a source.`); + continue; + } + try { + validateJson(c, this.getSchemaByType(c.type.toLocaleLowerCase() as ClientType), this.logger); + } catch (e) { + const err = new Error(`Client config ${index + 1} (${c.type} - ${name}) in config.json is invalid and will not be used.`, {cause: e}); + this.emitter.emit('error', err); + this.logger.error(err); + continue; + } configs.push({...c, name, source: 'config.json', @@ -176,7 +206,9 @@ export default class ScrobbleClients { try { rawClientConfigs = await readJson(`${this.configDir}/${clientType}.json`, {throwOnNotFound: false}); } catch (e) { - this.logger.error(`${clientType}.json config file could not be parsed`); + const errMsg = `${clientType}.json config file could not be parsed`; + this.emitter.emit('error', errMsg); + this.logger.error(errMsg); continue; } if (rawClientConfigs !== undefined) { @@ -193,8 +225,14 @@ export default class ScrobbleClients { continue; } for(const [i,rawConf] of rawClientConfigs.entries()) { + if(['lastfm','listenbrainz'].includes(clientType) && + ((rawConf as LastfmClientConfig | ListenBrainzClientConfig).configureAs === 'source')) + { + this.logger.debug(`Skipping config ${i + 1} from ${clientType}.json because it is configured as a source.`); + continue; + } try { - const validConfig = validateJson(rawConf, clientSchema, this.logger); + const validConfig = validateJson(rawConf, this.getSchemaByType(clientType), this.logger); // @ts-expect-error configureAs should exist const {configureAs = defaultConfigureAs} = validConfig; if (configureAs === 'client') { @@ -206,38 +244,14 @@ export default class ScrobbleClients { configs.push(parsedConfig); } } catch (e: any) { - this.logger.error(`The config entry at index ${i} from ${clientType}.json was not valid`); + const configErr = new Error(`The config entry at index ${i} from ${clientType}.json was not valid`, {cause: e}); + this.emitter.emit('error', configErr); + this.logger.error(configErr); } } -/* for (const [i,m] of clientConfigs.entries()) { - if(m === null) { - this.logger.error(`The config entry at index ${i} from ${clientType}.json is null`); - continue; - } - if (typeof m !== 'object') { - this.logger.error(`The config entry at index ${i} from ${clientType}.json was not an object, skipping`, m); - continue; - } - const {configureAs = defaultConfigureAs} = m; - if(configureAs === 'client') { - m.source = `${clientType}.json`; - m.type = clientType; - configs.push(m); - } - }*/ } } - // we have all possible client configurations so we'll check they are minimally valid - /*const validConfigs = configs.reduce((acc, c) => { - const isValid = isValidConfigStructure(c, {type: true, data: true}); - if (isValid !== true) { - this.logger.error(`Client config from ${c.source} with name [${c.name || 'unnamed'}] of type [${c.type || 'unknown'}] will not be used because it has structural errors: ${isValid.join(' | ')}`); - return acc; - } - return acc.concat(c); - }, []);*/ - // all client configs are minimally valid // now check that names are unique const nameGroupedConfigs = configs.reduce((acc: groupedNamedConfigs, curr: ParsedConfig) => { @@ -269,8 +283,9 @@ ${sources.join('\n')}`); try { await this.addClient(c, clientDefaults, notifier); } catch(e) { - this.logger.error(`Client ${c.name} was not added because it had unrecoverable errors`); - this.logger.error(e); + const addError = new Error(`Client ${c.name} was not added because it had unrecoverable errors`, {cause: e}); + this.emitter.emit('error', addError); + this.logger.error(addError); } } } diff --git a/src/backend/sources/ScrobbleSources.ts b/src/backend/sources/ScrobbleSources.ts index da33da3..2088dae 100644 --- a/src/backend/sources/ScrobbleSources.ts +++ b/src/backend/sources/ScrobbleSources.ts @@ -1,7 +1,7 @@ /* eslint-disable no-case-declarations */ import { childLogger, Logger } from '@foxxmd/logging'; import EventEmitter from "events"; -import { ConfigMeta, InternalConfig, SourceType, sourceTypes } from "../common/infrastructure/Atomic.js"; +import { ConfigMeta, InternalConfig, isSourceType, SourceType, sourceTypes } from "../common/infrastructure/Atomic.js"; import { AIOConfig, SourceDefaults } from "../common/infrastructure/config/aioConfig.js"; import { ChromecastSourceConfig } from "../common/infrastructure/config/source/chromecast.js"; import { DeezerData, DeezerSourceConfig } from "../common/infrastructure/config/source/deezer.js"; @@ -27,10 +27,9 @@ import { TautulliSourceConfig } from "../common/infrastructure/config/source/tau import { VLCData, VLCSourceConfig } from "../common/infrastructure/config/source/vlc.js"; import { WebScrobblerSourceConfig } from "../common/infrastructure/config/source/webscrobbler.js"; import { YTMusicSourceConfig } from "../common/infrastructure/config/source/ytmusic.js"; -import * as aioSchema from "../common/schema/aio-source.json"; -import * as sourceSchema from "../common/schema/source.json"; import { WildcardEmitter } from "../common/WildcardEmitter.js"; -import { parseBool, readJson, validateJson } from "../utils.js"; +import { parseBool, readJson } from "../utils.js"; +import { validateJson } from "../utils/ValidationUtils.js"; import AbstractSource from "./AbstractSource.js"; import { ChromecastSource } from "./ChromecastSource.js"; import DeezerSource from "./DeezerSource.js"; @@ -51,6 +50,8 @@ import TautulliSource from "./TautulliSource.js"; import { VLCSource } from "./VLCSource.js"; import { WebScrobblerSource } from "./WebScrobblerSource.js"; import YTMusicSource from "./YTMusicSource.js"; +import { Definition } from 'typescript-json-schema'; +import { getTypeSchemaFromConfigGenerator } from '../utils/SchemaUtils.js'; type groupedNamedConfigs = {[key: string]: ParsedConfig[]}; @@ -64,11 +65,13 @@ export default class ScrobbleSources { logger: Logger; internalConfig: InternalConfig; + private schemaDefinitions: Record = {}; + emitter: WildcardEmitter; constructor(emitter: EventEmitter, internal: InternalConfigOptional, parentLogger: Logger) { this.emitter = emitter; - this.logger = childLogger(parentLogger, 'Sources'); // winston.loggers.get('app').child({labels: ['Sources']}, mergeArr); + this.logger = childLogger(parentLogger, 'Sources'); this.internalConfig = { ...internal, logger: this.logger @@ -108,6 +111,68 @@ export default class ScrobbleSources { return [sourcesReady, messages]; } + private getSchemaByType = (type: SourceType): Definition => { + if(this.schemaDefinitions[type] === undefined) { + switch(type) { + case 'spotify': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("SpotifySourceConfig"); + break; + case 'plex': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("PlexSourceConfig"); + break; + case 'tautulli': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("TautulliSourceConfig"); + break; + case 'deezer': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("DeezerSourceConfig"); + break; + case 'subsonic': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("SubSonicSourceConfig"); + break; + case 'jellyfin': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("JellyfinCompatConfig"); + break; + case 'lastfm': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("LastfmSourceConfig"); + break; + case 'ytmusic': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("YTMusicSourceConfig"); + break; + case 'mpris': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("MPRISSourceConfig"); + break; + case 'mopidy': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("MopidySourceConfig"); + break; + case 'listenbrainz': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("ListenBrainzSourceConfig"); + break; + case 'jriver': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("JRiverSourceConfig"); + break; + case 'kodi': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("KodiSourceConfig"); + break; + case 'chromecast': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("ChromecastSourceConfig"); + break; + case 'webscrobbler': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("WebScrobblerSourceConfig"); + break; + case 'musikcube': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("MusikcubeSourceConfig"); + break; + case 'mpd': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("MPDSourceConfig"); + break; + case 'vlc': + this.schemaDefinitions[type] = getTypeSchemaFromConfigGenerator("VLCSourceConfig"); + break; + } + } + return this.schemaDefinitions[type]; + } + buildSourcesFromConfig = async (additionalConfigs: ParsedConfig[] = []) => { const configs: ParsedConfig[] = additionalConfigs; @@ -117,27 +182,38 @@ export default class ScrobbleSources { } catch (e) { throw new Error('config.json could not be parsed'); } + + const relaxedSchema = getTypeSchemaFromConfigGenerator("AIOSourceRelaxedConfig"); + let sourceDefaults = {}; if (configFile !== undefined) { - const aioConfig = validateJson(configFile, aioSchema, this.logger); + const aioConfig = validateJson(configFile, relaxedSchema, this.logger); const { sources: mainConfigSourcesConfigs = [], sourceDefaults: sd = {}, } = aioConfig; sourceDefaults = sd; -/* const validMainConfigs = mainConfigSourcesConfigs.reduce((acc: any, curr: any, i: any) => { - if(curr === null) { - this.logger.error(`The source config entry at index ${i} in config.json is null but should be an object, will not parse`); - return acc; + for (const [index, c] of mainConfigSourcesConfigs.entries()) { + const {name = 'unnamed'} = c; + if(!isSourceType(c.type.toLocaleLowerCase())) { + const invalidMsgType = `Source config ${index + 1} (${name}) in config.json has an invalid source type of '${c.type}'. Must be one of ${sourceTypes.join(' | ')}`; + this.emitter.emit('error', new Error(invalidMsgType)); + this.logger.error(invalidMsgType); + continue; } - if(typeof curr !== 'object') { - this.logger.error(`The source config entry at index ${i} in config.json should be an object, will not parse`); - return acc; + if(['lastfm','listenbrainz'].includes(c.type.toLocaleLowerCase()) && ((c as LastfmSourceConfig | ListenBrainzSourceConfig).configureAs !== 'source')) + { + this.logger.debug(`Skipping config ${index + 1} (${name}) in config.json because it is configured as a client.`); + continue; + } + try { + validateJson(c, this.getSchemaByType(c.type.toLocaleLowerCase() as SourceType), this.logger); + } catch (e) { + const err = new Error(`Source config ${index + 1} (${c.type} - ${name}) in config.json is invalid and will not be used.`, {cause: e}); + this.emitter.emit('error', err); + this.logger.error(err); + continue; } - return acc.concat(curr); - }, []);*/ - for (const c of mainConfigSourcesConfigs) { - const {name = 'unnamed'} = c; configs.push({...c, name, source: 'config.json', @@ -402,7 +478,9 @@ export default class ScrobbleSources { try { rawSourceConfigs = await readJson(`${this.internalConfig.configDir}/${sourceType}.json`, {throwOnNotFound: false}); } catch (e) { - this.logger.error(`${sourceType}.json config file could not be parsed`); + const errMsg = `${sourceType}.json config file could not be parsed`; + this.emitter.emit('error', errMsg); + this.logger.error(errMsg); continue; } if (rawSourceConfigs !== undefined) { @@ -419,8 +497,14 @@ export default class ScrobbleSources { continue; } for (const [i,rawConf] of sourceConfigs.entries()) { + if(['lastfm','listenbrainz'].includes(sourceType) && + ((rawConf as LastfmSourceConfig | ListenBrainzSourceConfig).configureAs !== 'source')) + { + this.logger.debug(`Skipping config ${i + 1} from ${sourceType}.json because it is configured as a client.`); + continue; + } try { - const validConfig = validateJson(rawConf, sourceSchema, this.logger); + const validConfig = validateJson(rawConf, this.getSchemaByType(sourceType), this.logger); // @ts-expect-error will eventually have all info (lazy) const parsedConfig: ParsedConfig = { @@ -428,38 +512,16 @@ export default class ScrobbleSources { source: `${sourceType}.json`, type: sourceType } - - if(!['lastfm','listenbrainz'].includes(sourceType) || ((validConfig as LastfmSourceConfig | ListenBrainzSourceConfig).configureAs === 'source')) { - configs.push(parsedConfig); - } else { - if('configureAs' in validConfig) { - if(validConfig.configureAs === 'source') { - configs.push(parsedConfig); - } else { - this.logger.verbose(`${sourceType} has 'configureAs: client' so will skip adding as a source`); - } - } else { - this.logger.verbose(`${sourceType} did not have 'configureAs' specified! Assuming 'client' so will skip adding as a source`); - } - } + configs.push(parsedConfig); } catch (e: any) { - this.logger.error(`The config entry at index ${i} from ${sourceType}.json was not valid`); + const configErr = new Error(`The config entry at index ${i} from ${sourceType}.json was not valid`, {cause: e}); + this.emitter.emit('error', configErr); + this.logger.error(configErr); } } } } - // we have all possible configurations so we'll check they are minimally valid -/* const validConfigs = configs.reduce((acc, c) => { - const isValid = isValidConfigStructure(c, {type: true, data: true}); - if (isValid !== true) { - // @ts-expect-error TS(2339): Property 'source' does not exist on type 'never'. - this.logger.error(`Source config from ${c.source} with name [${c.name || 'unnamed'}] of type [${c.type || 'unknown'}] will not be used because it has structural errors: ${isValid.join(' | ')}`); - return acc; - } - return acc.concat(c); - }, []);*/ - // finally! all configs are valid, structurally, and can now be passed to addClient // do a last check that names (within each type) are unique and warn if not, but add anyways const typeGroupedConfigs = configs.reduce((acc: groupedNamedConfigs, curr: ParsedConfig) => { @@ -496,8 +558,9 @@ export default class ScrobbleSources { try { await this.addSource(c, sourceDefaults); } catch(e) { - this.logger.error(`Source ${c.name} of type ${c.type} was not added because of unrecoverable errors`); - this.logger.error(e); + const addError = new Error(`Source ${c.name} of type ${c.type} was not added because of unrecoverable errors`, {cause: e}); + this.emitter.emit('error', addError); + this.logger.error(addError); } } } @@ -592,12 +655,5 @@ export default class ScrobbleSources { return; } this.sources.push(newSource); - if(!newSource.isReady()) { - if ((await newSource.initialize()) === false) { - this.logger.error(`${type} (${name}) source failed to initialize. Source needs to be successfully initialized before activity capture can begin.`); - } - } else { - newSource.logger.info('Fully Initialized!'); - } } } diff --git a/src/backend/tests/component/component.test.ts b/src/backend/tests/component/component.test.ts index 67fb858..6e4437a 100644 --- a/src/backend/tests/component/component.test.ts +++ b/src/backend/tests/component/component.test.ts @@ -23,7 +23,6 @@ describe('Play Transforms', function () { beforeEach(function() { component.config = {}; - // @ts-expect-error should be built on every test component.transformRules = undefined; }); diff --git a/src/backend/tests/config/config.test.ts b/src/backend/tests/config/config.test.ts new file mode 100644 index 0000000..a483674 --- /dev/null +++ b/src/backend/tests/config/config.test.ts @@ -0,0 +1,94 @@ +import { after, before, describe, it } from 'mocha'; +import chai, { assert, expect } from 'chai'; +import asPromised from 'chai-as-promised'; +import withLocalTmpDir from 'with-local-tmp-dir'; +import {constants, copyFile, access} from 'node:fs/promises'; +import path from "path"; +import {projectDir} from '../../common/index.js'; +import ScrobbleClients from '../../scrobblers/ScrobbleClients.js'; +import ScrobbleSources from '../../sources/ScrobbleSources.js'; +import EventEmitter from "events"; +import {loggerTest, loggerDebug} from '@foxxmd/logging'; +import { clientTypes, SourceType, sourceTypes } from '../../common/infrastructure/Atomic.js'; +import { Notifiers } from '../../notifier/Notifiers.js'; + +chai.use(asPromised); + +const samplePath = (name: string) => path.resolve(projectDir, 'config', `${name}.json.example`); + +describe('Sample Configs', function () { + + describe('Exist', function() { + describe('Source Configs', function () { + for(const componentType of sourceTypes) { + it(`Sample ${componentType}.json exists`, async function () { + await access(samplePath(componentType), constants.F_OK); + }); + } + + }); + describe('Client Configs', function () { + for(const componentType of clientTypes) { + it(`Sample ${componentType}.json exists`, async function () { + await access(samplePath(componentType), constants.F_OK); + }); + } + }); + }); + + describe('Parse and Validate Correctly', function () { + + describe('Source Configs', function () { + let reset: any; + + beforeEach(async function() { + reset = await withLocalTmpDir({unsafeCleanup: true}); + }); + + afterEach(async function() { + await reset(); + }); + + for(const componentType of sourceTypes) { + + //trueName = componentType; + it(`Sample ${componentType}.json parses and validates`, async function () { + + let emitter = new EventEmitter(); + await copyFile(samplePath(componentType), `${componentType}.json`); + const sources = new ScrobbleSources(emitter, { + localUrl: new URL('http://example.com'), + configDir: process.cwd(), + version: 'test' + }, loggerTest); + + await sources.buildSourcesFromConfig(); + expect(sources.sources).length(1); + }); + } + }); + + describe('Client Configs', function () { + let reset: any; + + beforeEach(async function() { + reset = await withLocalTmpDir({unsafeCleanup: true}); + }); + + afterEach(async function() { + await reset(); + }); + + for(const componentType of clientTypes) { + it(`Sample ${componentType}.json parses and validates`, async function () { + + let emitter = new EventEmitter(); + await copyFile(samplePath(componentType), `${componentType}.json`); + const clients = new ScrobbleClients(emitter, new EventEmitter, new URL('http://example.com'), process.cwd(), loggerTest); + await clients.buildClientsFromConfig(new Notifiers(new EventEmitter, new EventEmitter, new EventEmitter, loggerTest)); + expect(clients.clients).length(1); + }); + } + }); + }); +}); diff --git a/src/backend/tsconfig.json b/src/backend/tsconfig.json index 4e0a3cf..6bde68c 100644 --- a/src/backend/tsconfig.json +++ b/src/backend/tsconfig.json @@ -24,7 +24,7 @@ "exclude": [ "../../node_modules", "../../coverage", - "./tests/**/*", + "../../_site", "../../docsite", "../../build", diff --git a/src/backend/utils.ts b/src/backend/utils.ts index d7d5325..641cdc4 100644 --- a/src/backend/utils.ts +++ b/src/backend/utils.ts @@ -1,9 +1,7 @@ import { Logger } from '@foxxmd/logging'; -import { parseRegexSingle, SearchAndReplaceRegExp } from "@foxxmd/regex-buddy-core"; +import { parseRegexSingle } from "@foxxmd/regex-buddy-core"; import backoffStrategies from '@kenyip/backoff-strategies'; import address from "address"; -import * as AjvNS from 'ajv'; -import Ajv, { Schema } from 'ajv'; import { replaceResultTransformer, stripIndentTransformer, TemplateTag, trimResultTransformer } from 'common-tags'; import dayjs, { Dayjs } from 'dayjs'; import { Duration } from "dayjs/plugin/duration.js"; @@ -351,62 +349,6 @@ export const parseDurationFromTimestamp = (timestamp: any) => { }); } -export const createAjvFactory = (logger: Logger): AjvNS.default => { - const validator = new Ajv.default({logger: logger, verbose: true, strict: "log", allowUnionTypes: true}); - // https://ajv.js.org/strict-mode.html#unknown-keywords - validator.addKeyword('deprecationMessage'); - return validator; -} - -export const validateJson = (config: object, schema: Schema, logger: Logger): T => { - const ajv = createAjvFactory(logger); - const valid = ajv.validate(schema, config); - if (valid) { - return config as unknown as T; - } else { - logger.error('Json config was not valid. Please use schema to check validity.', {leaf: 'Config'}); - if (Array.isArray(ajv.errors)) { - for (const err of ajv.errors) { - const parts = [ - `At: ${err.instancePath}`, - ]; - let data; - if (typeof err.data === 'string') { - data = err.data; - } else if (err.data !== null && typeof err.data === 'object' && (err.data as any).name !== undefined) { - data = `Object named '${(err.data as any).name}'`; - } - if (data !== undefined) { - parts.push(`Data: ${data}`); - } - let suffix = ''; - if (err.params.allowedValues !== undefined) { - suffix = err.params.allowedValues.join(', '); - suffix = ` [${suffix}]`; - } - parts.push(`${err.keyword}: ${err.schemaPath} => ${err.message}${suffix}`); - - // if we have a reference in the description parse it out so we can log it here for context - if (err.parentSchema !== undefined && err.parentSchema.description !== undefined) { - const desc = err.parentSchema.description as string; - const seeIndex = desc.indexOf('[See]'); - if (seeIndex !== -1) { - let newLineIndex: number | undefined = desc.indexOf('\n', seeIndex); - if (newLineIndex === -1) { - newLineIndex = undefined; - } - const seeFragment = desc.slice(seeIndex + 5, newLineIndex); - parts.push(`See:${seeFragment}`); - } - } - - logger.error(`Schema Error:\r\n${parts.join('\r\n')}`, {leaf: 'Config'}); - } - } - throw new Error('Config schema validity failure'); - } -} - export const remoteHostIdentifiers = (req: Request): RemoteIdentityParts => { const remote = req.connection.remoteAddress; const proxyRemote = Array.isArray(req.headers["x-forwarded-for"]) ? req.headers["x-forwarded-for"][0] : req.headers["x-forwarded-for"]; diff --git a/src/backend/utils/SchemaUtils.ts b/src/backend/utils/SchemaUtils.ts new file mode 100644 index 0000000..9739de8 --- /dev/null +++ b/src/backend/utils/SchemaUtils.ts @@ -0,0 +1,46 @@ +import * as TJS from "typescript-json-schema"; +import {sync} from "glob"; +import { resolve } from "path"; +import { projectDir } from "../common/index.js"; + +// const includeOnly = [ +// sync(resolve(projectDir, "src/backend/**/*.ts"), {ignore: resolve(projectDir, "src/backend/tests/**/*")}), +// sync(resolve(projectDir, "src/core/**/*.ts")) +// ].flat(1); + +export const buildSchemaGenerator = (program?: TJS.Program, settings: TJS.PartialArgs = {}) => { + return TJS.buildGenerator(program, {...defaultGeneratorArgs, ...settings}); +} + +let configProgram: TJS.Program, +generatorFromConfig: TJS.JsonSchemaGenerator; + +export const getTsConfigProgram = (): TJS.Program => { + if(configProgram === undefined) { + const tsConfig = resolve(projectDir, "src/backend/tsconfig.json"); + configProgram = TJS.programFromConfig(tsConfig, + sync(resolve(projectDir, "src/backend/common/infrastructure/config/**/*.ts")) + ); + } + return configProgram; +} + +export const getTsConfigGenerator = (): TJS.JsonSchemaGenerator => { + if(generatorFromConfig === undefined) { + generatorFromConfig = buildSchemaGenerator(getTsConfigProgram()); + } + return generatorFromConfig; +} + +export const getTypeSchemaFromConfigGenerator = (type: string): TJS.Definition | null => { + return TJS.generateSchema(getTsConfigProgram(), type, undefined, [], getTsConfigGenerator()); +} + +export const defaultGeneratorArgs: TJS.PartialArgs = { + required: true, + titles: true, + validationKeywords: ['deprecationMessage'], + constAsEnum: true, + ref: true, + esModuleInterop: true +}; diff --git a/src/backend/utils/ValidationUtils.ts b/src/backend/utils/ValidationUtils.ts new file mode 100644 index 0000000..5ecbffa --- /dev/null +++ b/src/backend/utils/ValidationUtils.ts @@ -0,0 +1,58 @@ +import { Logger } from "@foxxmd/logging"; +import * as AjvNS from "ajv"; +import Ajv, { Schema } from "ajv"; + +export const createAjvFactory = (logger: Logger): AjvNS.default => { + const validator = new Ajv.default({logger: logger, verbose: true, strict: "log", allowUnionTypes: true}); + // https://ajv.js.org/strict-mode.html#unknown-keywords + validator.addKeyword('deprecationMessage'); + return validator; +} +export const validateJson = (config: object, schema: Schema, logger: Logger): T => { + const ajv = createAjvFactory(logger); + const valid = ajv.validate(schema, config); + if (valid) { + return config as unknown as T; + } else { + const schemaErrors = ['Json config was not valid. Please use schema to check validity.']; + if (Array.isArray(ajv.errors)) { + for (const err of ajv.errors) { + const parts = [ + `At: ${err.instancePath}`, + ]; + let data; + if (typeof err.data === 'string') { + data = err.data; + } else if (err.data !== null && typeof err.data === 'object' && (err.data as any).name !== undefined) { + data = `Object named '${(err.data as any).name}'`; + } + if (data !== undefined) { + parts.push(`Data: ${data}`); + } + let suffix = ''; + if (err.params.allowedValues !== undefined) { + suffix = err.params.allowedValues.join(', '); + suffix = ` [${suffix}]`; + } + parts.push(`${err.keyword}: ${err.schemaPath} => ${err.message}${suffix}`); + + // if we have a reference in the description parse it out so we can log it here for context + if (err.parentSchema !== undefined && err.parentSchema.description !== undefined) { + const desc = err.parentSchema.description as string; + const seeIndex = desc.indexOf('[See]'); + if (seeIndex !== -1) { + let newLineIndex: number | undefined = desc.indexOf('\n', seeIndex); + if (newLineIndex === -1) { + newLineIndex = undefined; + } + const seeFragment = desc.slice(seeIndex + 5, newLineIndex); + parts.push(`See:${seeFragment}`); + } + } + + schemaErrors.push(`Schema Error:\r\n${parts.join('\r\n')}`); + } + } + throw new Error(schemaErrors.join('\n\n')); + } +} diff --git a/tsconfig.json b/tsconfig.json index 1b606f4..109f70a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -16,7 +16,7 @@ "isolatedModules": true, "noEmit": true, "jsx": "react-jsx", - "sourceMap": true, + "sourceMap": false, }, "include": [ "src"