From 342e4d52ccd98c67262614d20b6cd60787350e0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Burkard?= <22095555+JeromeBu@users.noreply.github.com> Date: Wed, 7 Aug 2024 23:30:39 +0200 Subject: [PATCH 01/19] using db API v2 with postgres --- api/scripts/compile-data.ts | 17 +- api/src/core/adapters/dbApi/InMemoryDbApi.ts | 60 - api/src/core/adapters/dbApi/createGitDbApi.ts | 2 +- .../dbApi/kysely/createPgAgentRepository.ts | 3 +- .../kysely/createPgInstanceRepository.ts | 12 +- .../kysely/createPgSoftwareRepository.ts | 347 ++-- .../createPgUserAndReferentRepository.ts | 18 + .../dbApi/kysely/pgDbApi.integration.test.ts | 6 +- api/src/core/bootstrap.ts | 50 +- api/src/core/ports/DbApi.ts | 3 +- api/src/core/ports/DbApiV2.ts | 16 +- api/src/core/usecases/index.ts | 3 +- .../usecases/readWriteSillData/selectors.ts | 640 +++--- .../core/usecases/readWriteSillData/thunks.ts | 1806 +++++++++-------- .../suggestionAndAutoFill/selectors.ts | 16 +- .../usecases/suggestionAndAutoFill/thunks.ts | 21 - api/src/env.ts | 7 +- api/src/rpc/createTestCaller.ts | 19 +- api/src/rpc/router.ts | 250 ++- api/src/rpc/routes.e2e.test.ts | 104 +- api/src/rpc/start.ts | 112 +- api/src/tools/test.helpers.ts | 2 + .../tools/validateGithubWebhookSignature.ts | 50 - 23 files changed, 1829 insertions(+), 1735 deletions(-) delete mode 100644 api/src/core/adapters/dbApi/InMemoryDbApi.ts delete mode 100644 api/src/tools/validateGithubWebhookSignature.ts diff --git a/api/scripts/compile-data.ts b/api/scripts/compile-data.ts index 4fc09872..dad98948 100644 --- a/api/scripts/compile-data.ts +++ b/api/scripts/compile-data.ts @@ -1,14 +1,19 @@ +import { Kysely } from "kysely"; import { bootstrapCore } from "../src/core"; +import type { Database } from "../src/core/adapters/dbApi/kysely/kysely.database"; +import { createPgDialect } from "../src/core/adapters/dbApi/kysely/kysely.dialect"; import { env } from "../src/env"; (async () => { + const kyselyDb = new Kysely({ dialect: createPgDialect(env.databaseUrl) }); const { core } = await bootstrapCore({ "keycloakUserApiParams": undefined, "dbConfig": { - "dbKind": "git", - "dataRepoSshUrl": "git@github.com:codegouvfr/sill-data.git", - "sshPrivateKey": env.sshPrivateKeyForGit, - "sshPrivateKeyName": env.sshPrivateKeyForGitName + "dbKind": "kysely", + "kyselyDb": kyselyDb + // "dataRepoSshUrl": "git@github.com:codegouvfr/sill-data.git", + // "sshPrivateKey": env.sshPrivateKeyForGit, + // "sshPrivateKeyName": env.sshPrivateKeyForGitName }, "githubPersonalAccessTokenForApiRateLimit": env.githubPersonalAccessTokenForApiRateLimit, "doPerPerformPeriodicalCompilation": false, @@ -16,7 +21,9 @@ import { env } from "../src/env"; "externalSoftwareDataOrigin": env.externalSoftwareDataOrigin }); - await core.functions.readWriteSillData.manuallyTriggerNonIncrementalCompilation(); + console.log("core initialized, TODO TRIGGER INCREMENTAL compilation", core.functions); + + // await core.functions.readWriteSillData.manuallyTriggerNonIncrementalCompilation(); process.exit(0); })(); diff --git a/api/src/core/adapters/dbApi/InMemoryDbApi.ts b/api/src/core/adapters/dbApi/InMemoryDbApi.ts deleted file mode 100644 index e0861b53..00000000 --- a/api/src/core/adapters/dbApi/InMemoryDbApi.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { CompiledData } from "../../ports/CompileData"; -import type { Db, DbApi } from "../../ports/DbApi"; - -export class InMemoryDbApi implements DbApi { - #softwareRows: Db.SoftwareRow[] = []; - #agentRows: Db.AgentRow[] = []; - #softwareReferentRows: Db.SoftwareReferentRow[] = []; - #softwareUserRows: Db.SoftwareUserRow[] = []; - #instanceRows: Db.InstanceRow[] = []; - - #compiledData: CompiledData<"private"> = []; - - async fetchDb() { - return { - softwareRows: this.#softwareRows, - agentRows: this.#agentRows, - softwareReferentRows: this.#softwareReferentRows, - softwareUserRows: this.#softwareUserRows, - instanceRows: this.#instanceRows - }; - } - - async updateDb({ newDb }: { newDb: Db }) { - this.#softwareRows = newDb.softwareRows; - this.#agentRows = newDb.agentRows; - this.#softwareReferentRows = newDb.softwareReferentRows; - this.#softwareUserRows = newDb.softwareUserRows; - this.#instanceRows = newDb.instanceRows; - } - - async fetchCompiledData() { - return this.#compiledData; - } - - async updateCompiledData({ newCompiledData }: { newCompiledData: CompiledData<"private"> }) { - this.#compiledData = newCompiledData; - } - - // test helpers - - get softwareRows() { - return this.#softwareRows; - } - - get agentRows() { - return this.#agentRows; - } - - get softwareReferentRows() { - return this.#softwareReferentRows; - } - - get softwareUserRows() { - return this.#softwareUserRows; - } - - get instanceRows() { - return this.#instanceRows; - } -} diff --git a/api/src/core/adapters/dbApi/createGitDbApi.ts b/api/src/core/adapters/dbApi/createGitDbApi.ts index a2566a65..b8232646 100644 --- a/api/src/core/adapters/dbApi/createGitDbApi.ts +++ b/api/src/core/adapters/dbApi/createGitDbApi.ts @@ -24,7 +24,7 @@ export type GitDbApiParams = { sshPrivateKey: string; }; -export function createGitDbApi(params: GitDbApiParams): Db.DbApiAndInitializeCache { +export function createGitDbApi(params: GitDbApiParams): { dbApi: DbApi; initializeDbApiCache: () => Promise } { const { dataRepoSshUrl, sshPrivateKeyName, sshPrivateKey } = params; const dbApi: DbApi = { diff --git a/api/src/core/adapters/dbApi/kysely/createPgAgentRepository.ts b/api/src/core/adapters/dbApi/kysely/createPgAgentRepository.ts index e2e0db47..663b5ab1 100644 --- a/api/src/core/adapters/dbApi/kysely/createPgAgentRepository.ts +++ b/api/src/core/adapters/dbApi/kysely/createPgAgentRepository.ts @@ -4,7 +4,8 @@ import type { Database } from "./kysely.database"; export const createPgAgentRepository = (db: Kysely): AgentRepository => ({ add: async agent => { - await db.insertInto("agents").values(agent).execute(); + const { id } = await db.insertInto("agents").values(agent).returning("id").executeTakeFirstOrThrow(); + return id; }, update: async agent => { await db.updateTable("agents").set(agent).where("id", "=", agent.id).execute(); diff --git a/api/src/core/adapters/dbApi/kysely/createPgInstanceRepository.ts b/api/src/core/adapters/dbApi/kysely/createPgInstanceRepository.ts index 62429782..4d08fd0e 100644 --- a/api/src/core/adapters/dbApi/kysely/createPgInstanceRepository.ts +++ b/api/src/core/adapters/dbApi/kysely/createPgInstanceRepository.ts @@ -6,12 +6,12 @@ import { Instance } from "../../../usecases/readWriteSillData"; import { Database } from "./kysely.database"; export const createPgInstanceRepository = (db: Kysely): InstanceRepository => ({ - create: async ({ fromData, agentEmail }) => { - const { mainSoftwareSillId, organization, targetAudience, publicUrl, ...rest } = fromData; + create: async ({ formData, agentEmail }) => { + const { mainSoftwareSillId, organization, targetAudience, publicUrl, ...rest } = formData; assert>(); const now = Date.now(); - await db + const { instanceId } = await db .insertInto("instances") .values({ addedByAgentEmail: agentEmail, @@ -22,10 +22,12 @@ export const createPgInstanceRepository = (db: Kysely): InstanceReposi targetAudience, publicUrl }) + .returning("id as instanceId") .executeTakeFirstOrThrow(); + return instanceId; }, - update: async ({ fromData, instanceId }) => { - const { mainSoftwareSillId, organization, targetAudience, publicUrl, ...rest } = fromData; + update: async ({ formData, instanceId }) => { + const { mainSoftwareSillId, organization, targetAudience, publicUrl, ...rest } = formData; assert>(); const now = Date.now(); diff --git a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts index 7fe35a9b..f50d4ba2 100644 --- a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts +++ b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts @@ -30,7 +30,7 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi const now = Date.now(); - await db.transaction().execute(async trx => { + return db.transaction().execute(async trx => { const { softwareId } = await trx .insertInto("softwares") .values({ @@ -66,6 +66,8 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi .insertInto("softwares__similar_software_external_datas") .values(similarSoftwareExternalDataIds.map(similarExternalId => ({ softwareId, similarExternalId }))) .execute(); + + return softwareId; }); }, update: async ({ formData, softwareSillId, agentEmail }) => { @@ -117,107 +119,51 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi .where("id", "=", softwareSillId) .execute(); }, + getById: async (softwareId: number): Promise => { + console.log("getById : ", softwareId); + return makeGetSoftwareBuilder(db) + .where("id", "=", softwareId) + .executeTakeFirst() + .then((result): Software | undefined => { + if (!result) return; + const { + testUrls, + serviceProviders, + parentExternalData, + updateTime, + addedTime, + softwareExternalData, + similarExternalSoftwares, + ...software + } = result; + return { + ...convertNullValuesToUndefined(software), + updateTime: new Date(+updateTime).getTime(), + addedTime: new Date(+addedTime).getTime(), + serviceProviders: serviceProviders ?? [], + similarSoftwares: similarExternalSoftwares, + userAndReferentCountByOrganization: {}, + authors: (softwareExternalData?.developers ?? []).map(dev => ({ + authorName: dev.name, + authorUrl: `https://www.wikidata.org/wiki/${dev.id}` + })), + officialWebsiteUrl: + softwareExternalData?.websiteUrl ?? + software.comptoirDuLibreSoftware?.external_resources.website ?? + undefined, + codeRepositoryUrl: + softwareExternalData?.sourceUrl ?? + software.comptoirDuLibreSoftware?.external_resources.repository ?? + undefined, + documentationUrl: softwareExternalData?.documentationUrl ?? undefined, + comptoirDuLibreServiceProviderCount: software.comptoirDuLibreSoftware?.providers.length ?? 0, + testUrl: testUrls[0]?.url, + parentWikidataSoftware: parentExternalData ?? undefined + }; + }); + }, getAll: (): Promise => - db - .selectFrom("softwares as s") - .leftJoin("software_external_datas as ext", "ext.externalId", "s.externalId") - .leftJoin("compiled_softwares as cs", "cs.softwareId", "s.id") - .leftJoin("software_external_datas as parentExt", "s.parentSoftwareWikidataId", "parentExt.externalId") - .leftJoin( - "softwares__similar_software_external_datas", - "softwares__similar_software_external_datas.softwareId", - "s.id" - ) - .leftJoin( - "software_external_datas as similarExt", - "softwares__similar_software_external_datas.similarExternalId", - "similarExt.externalId" - ) - .groupBy([ - "s.id", - "cs.softwareId", - "cs.annuaireCnllServiceProviders", - "cs.comptoirDuLibreSoftware", - "cs.latestVersion", - "cs.serviceProviders", - "ext.externalId", - "parentExt.externalId" - ]) - .select([ - "s.id as softwareId", - "s.logoUrl", - "s.name as softwareName", - "s.description as softwareDescription", - "cs.serviceProviders", - "cs.latestVersion", - "s.testUrls", - "s.referencedSinceTime as addedTime", - "s.updateTime", - "s.dereferencing", - "s.categories", - ({ ref }) => - jsonBuildObject({ - isPresentInSupportContract: ref("isPresentInSupportContract"), - isFromFrenchPublicServices: ref("isFromFrenchPublicService"), - doRespectRgaa: ref("doRespectRgaa") - }).as("prerogatives"), - "s.comptoirDuLibreId", - "cs.comptoirDuLibreSoftware", - "s.versionMin", - "s.license", - "annuaireCnllServiceProviders", - "s.externalId", - "s.externalDataOrigin", - "s.softwareType", - ({ ref, ...qb }) => - qb - .case() - .when("parentExt.externalId", "is not", null) - .then( - jsonBuildObject({ - externalId: ref("ext.externalId"), - label: ref("ext.label"), - description: ref("ext.description") - }).$castTo() - ) - .else(null) - .end() - .as("parentExternalData"), - "s.keywords", - - ({ ref }) => - jsonBuildObject({ - externalId: ref("ext.externalId"), - externalDataOrigin: ref("ext.externalDataOrigin"), - developers: ref("ext.developers"), - label: ref("ext.label"), - description: ref("ext.description"), - isLibreSoftware: ref("ext.isLibreSoftware"), - logoUrl: ref("ext.logoUrl"), - framaLibreId: ref("ext.framaLibreId"), - websiteUrl: ref("ext.websiteUrl"), - sourceUrl: ref("ext.sourceUrl"), - documentationUrl: ref("ext.documentationUrl") - }).as("softwareExternalData"), - ({ ref, fn }) => - fn - .coalesce( - fn - .jsonAgg( - jsonBuildObject({ - isInSill: sql`false`, - externalId: ref("similarExt.externalId"), - label: ref("similarExt.label"), - description: ref("similarExt.description"), - isLibreSoftware: ref("similarExt.isLibreSoftware"), - externalDataOrigin: ref("similarExt.externalDataOrigin") - }).$castTo() - ) - .filterWhere("similarExt.externalId", "is not", null), - sql<[]>`'[]'` - ) - .as("similarExternalSoftwares") - ]) + makeGetSoftwareBuilder(db) .execute() .then(softwares => softwares.map( @@ -230,42 +176,39 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi softwareExternalData, similarExternalSoftwares, ...software - }): Software => { - return { - ...convertNullValuesToUndefined(software), - updateTime: new Date(+updateTime).getTime(), - addedTime: new Date(+addedTime).getTime(), - serviceProviders: serviceProviders ?? [], - similarSoftwares: similarExternalSoftwares, - // (similarSoftwares ?? []).map( - // (s): SimilarSoftware => ({ - // softwareName: - // typeof s.label === "string" ? s.label : Object.values(s.label)[0]!, - // softwareDescription: - // typeof s.label === "string" ? s.label : Object.values(s.label)[0]!, - // isInSill: true // TODO: check if this is true - // }) - // ) ?? [], - userAndReferentCountByOrganization: {}, - authors: (softwareExternalData?.developers ?? []).map(dev => ({ - authorName: dev.name, - authorUrl: `https://www.wikidata.org/wiki/${dev.id}` - })), - officialWebsiteUrl: - softwareExternalData?.websiteUrl ?? - software.comptoirDuLibreSoftware?.external_resources.website ?? - undefined, - codeRepositoryUrl: - softwareExternalData?.sourceUrl ?? - software.comptoirDuLibreSoftware?.external_resources.repository ?? - undefined, - documentationUrl: softwareExternalData?.documentationUrl ?? undefined, - comptoirDuLibreServiceProviderCount: - software.comptoirDuLibreSoftware?.providers.length ?? 0, - testUrl: testUrls[0]?.url, - parentWikidataSoftware: parentExternalData ?? undefined - }; - } + }): Software => ({ + ...convertNullValuesToUndefined(software), + updateTime: new Date(+updateTime).getTime(), + addedTime: new Date(+addedTime).getTime(), + serviceProviders: serviceProviders ?? [], + similarSoftwares: similarExternalSoftwares, + // (similarSoftwares ?? []).map( + // (s): SimilarSoftware => ({ + // softwareName: + // typeof s.label === "string" ? s.label : Object.values(s.label)[0]!, + // softwareDescription: + // typeof s.label === "string" ? s.label : Object.values(s.label)[0]!, + // isInSill: true // TODO: check if this is true + // }) + // ) ?? [], + userAndReferentCountByOrganization: {}, + authors: (softwareExternalData?.developers ?? []).map(dev => ({ + authorName: dev.name, + authorUrl: `https://www.wikidata.org/wiki/${dev.id}` + })), + officialWebsiteUrl: + softwareExternalData?.websiteUrl ?? + software.comptoirDuLibreSoftware?.external_resources.website ?? + undefined, + codeRepositoryUrl: + softwareExternalData?.sourceUrl ?? + software.comptoirDuLibreSoftware?.external_resources.repository ?? + undefined, + documentationUrl: softwareExternalData?.documentationUrl ?? undefined, + comptoirDuLibreServiceProviderCount: software.comptoirDuLibreSoftware?.providers.length ?? 0, + testUrl: testUrls[0]?.url, + parentWikidataSoftware: parentExternalData ?? undefined + }) ) ), getAllSillSoftwareExternalIds: async externalDataOrigin => @@ -275,5 +218,133 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi .where("externalDataOrigin", "=", externalDataOrigin) .execute() .then(rows => rows.map(row => row.externalId!)), - unreference: async () => {} + countAddedByAgent: async ({ agentEmail }) => { + const { count } = await db + .selectFrom("softwares") + .select(qb => qb.fn.countAll().as("count")) + .where("addedByAgentEmail", "=", agentEmail) + .executeTakeFirstOrThrow(); + return +count; + }, + unreference: async ({ softwareId, reason, time }) => { + const { versionMin } = await db + .selectFrom("softwares") + .select("versionMin") + .where("id", "=", softwareId) + .executeTakeFirstOrThrow(); + + await db + .updateTable("softwares") + .set({ + dereferencing: JSON.stringify({ + reason, + time, + lastRecommendedVersion: versionMin + }) + }) + .where("id", "=", softwareId) + .executeTakeFirstOrThrow(); + } }); + +const makeGetSoftwareBuilder = (db: Kysely) => + db + .selectFrom("softwares as s") + .leftJoin("software_external_datas as ext", "ext.externalId", "s.externalId") + .leftJoin("compiled_softwares as cs", "cs.softwareId", "s.id") + .leftJoin("software_external_datas as parentExt", "s.parentSoftwareWikidataId", "parentExt.externalId") + .leftJoin( + "softwares__similar_software_external_datas", + "softwares__similar_software_external_datas.softwareId", + "s.id" + ) + .leftJoin( + "software_external_datas as similarExt", + "softwares__similar_software_external_datas.similarExternalId", + "similarExt.externalId" + ) + .groupBy([ + "s.id", + "cs.softwareId", + "cs.annuaireCnllServiceProviders", + "cs.comptoirDuLibreSoftware", + "cs.latestVersion", + "cs.serviceProviders", + "ext.externalId", + "parentExt.externalId" + ]) + .select([ + "s.id as softwareId", + "s.logoUrl", + "s.name as softwareName", + "s.description as softwareDescription", + "cs.serviceProviders", + "cs.latestVersion", + "s.testUrls", + "s.referencedSinceTime as addedTime", + "s.updateTime", + "s.dereferencing", + "s.categories", + ({ ref }) => + jsonBuildObject({ + isPresentInSupportContract: ref("isPresentInSupportContract"), + isFromFrenchPublicServices: ref("isFromFrenchPublicService"), + doRespectRgaa: ref("doRespectRgaa") + }).as("prerogatives"), + "s.comptoirDuLibreId", + "cs.comptoirDuLibreSoftware", + "s.versionMin", + "s.license", + "annuaireCnllServiceProviders", + "s.externalId", + "s.externalDataOrigin", + "s.softwareType", + ({ ref, ...qb }) => + qb + .case() + .when("parentExt.externalId", "is not", null) + .then( + jsonBuildObject({ + externalId: ref("ext.externalId"), + label: ref("ext.label"), + description: ref("ext.description") + }).$castTo() + ) + .else(null) + .end() + .as("parentExternalData"), + "s.keywords", + + ({ ref }) => + jsonBuildObject({ + externalId: ref("ext.externalId"), + externalDataOrigin: ref("ext.externalDataOrigin"), + developers: ref("ext.developers"), + label: ref("ext.label"), + description: ref("ext.description"), + isLibreSoftware: ref("ext.isLibreSoftware"), + logoUrl: ref("ext.logoUrl"), + framaLibreId: ref("ext.framaLibreId"), + websiteUrl: ref("ext.websiteUrl"), + sourceUrl: ref("ext.sourceUrl"), + documentationUrl: ref("ext.documentationUrl") + }).as("softwareExternalData"), + ({ ref, fn }) => + fn + .coalesce( + fn + .jsonAgg( + jsonBuildObject({ + isInSill: sql`false`, + externalId: ref("similarExt.externalId"), + label: ref("similarExt.label"), + description: ref("similarExt.description"), + isLibreSoftware: ref("similarExt.isLibreSoftware"), + externalDataOrigin: ref("similarExt.externalDataOrigin") + }).$castTo() + ) + .filterWhere("similarExt.externalId", "is not", null), + sql<[]>`'[]'` + ) + .as("similarExternalSoftwares") + ]); diff --git a/api/src/core/adapters/dbApi/kysely/createPgUserAndReferentRepository.ts b/api/src/core/adapters/dbApi/kysely/createPgUserAndReferentRepository.ts index e9d8eb96..1adef111 100644 --- a/api/src/core/adapters/dbApi/kysely/createPgUserAndReferentRepository.ts +++ b/api/src/core/adapters/dbApi/kysely/createPgUserAndReferentRepository.ts @@ -12,6 +12,15 @@ export const createPgUserRepository = (db: Kysely): SoftwareUserReposi .where("softwareId", "=", softwareId) .where("agentId", "=", agentId) .execute(); + }, + countSoftwaresForAgent: async (params: { agentId: number }) => { + const { count } = await db + .selectFrom("software_users") + .select(qb => qb.fn.countAll().as("count")) + .where("agentId", "=", params.agentId) + .executeTakeFirstOrThrow(); + + return +count; } }); @@ -26,6 +35,15 @@ export const createPgReferentRepository = (db: Kysely): SoftwareRefere .where("agentId", "=", agentId) .execute(); }, + countSoftwaresForAgent: async (params: { agentId: number }) => { + const { count } = await db + .selectFrom("software_referents") + .select(qb => qb.fn.countAll().as("count")) + .where("agentId", "=", params.agentId) + .executeTakeFirstOrThrow(); + + return +count; + }, getTotalCount: async () => { const { total_referents } = await db .selectFrom("software_referents") diff --git a/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts b/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts index 3f73e313..fe5923a5 100644 --- a/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts +++ b/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts @@ -1,6 +1,6 @@ import { Kysely } from "kysely"; import { beforeEach, describe, expect, it, afterEach } from "vitest"; -import { expectPromiseToFailWith, expectToEqual } from "../../../../tools/test.helpers"; +import { expectPromiseToFailWith, expectToEqual, testPgUrl } from "../../../../tools/test.helpers"; import { Agent, DbApiV2 } from "../../../ports/DbApiV2"; import { SoftwareExternalData } from "../../../ports/GetSoftwareExternalData"; import { SoftwareFormData } from "../../../usecases/readWriteSillData"; @@ -70,7 +70,7 @@ const similarSoftwareExternalData: SoftwareExternalData = { license: "MIT" }; -const db = new Kysely({ dialect: createPgDialect("postgresql://sill:pg_password@localhost:5432/sill") }); +const db = new Kysely({ dialect: createPgDialect(testPgUrl) }); describe("pgDbApi", () => { let dbApi: DbApiV2; @@ -189,7 +189,7 @@ describe("pgDbApi", () => { console.log("saving instance"); await dbApi.instance.create({ agentEmail: agent.email, - fromData: { + formData: { mainSoftwareSillId: softwareId, organization: "test-orga", targetAudience: "test-audience", diff --git a/api/src/core/bootstrap.ts b/api/src/core/bootstrap.ts index b21133cd..1098de89 100644 --- a/api/src/core/bootstrap.ts +++ b/api/src/core/bootstrap.ts @@ -1,8 +1,9 @@ +import { Kysely } from "kysely"; import { createCore, createObjectThatThrowsIfAccessed, type GenericCore } from "redux-clean-architecture"; import { createCompileData } from "./adapters/compileData"; import { comptoirDuLibreApi } from "./adapters/comptoirDuLibreApi"; -import { createGitDbApi, type GitDbApiParams } from "./adapters/dbApi/createGitDbApi"; -import { InMemoryDbApi } from "./adapters/dbApi/InMemoryDbApi"; +import { createKyselyPgDbApi } from "./adapters/dbApi/kysely/createPgDbApi"; +import { Database } from "./adapters/dbApi/kysely/kysely.database"; import { getCnllPrestatairesSill } from "./adapters/getCnllPrestatairesSill"; import { getServiceProviders } from "./adapters/getServiceProviders"; import { createGetSoftwareLatestVersion } from "./adapters/getSoftwareLatestVersion"; @@ -13,17 +14,17 @@ import { getHalSoftwareOptions } from "./adapters/hal/getHalSoftwareOptions"; import { createKeycloakUserApi, type KeycloakUserApiParams } from "./adapters/userApi"; import type { CompileData } from "./ports/CompileData"; import type { ComptoirDuLibreApi } from "./ports/ComptoirDuLibreApi"; -import { DbApi, Db } from "./ports/DbApi"; +import { Db } from "./ports/DbApi"; +import { DbApiV2 } from "./ports/DbApiV2"; import type { ExternalDataOrigin, GetSoftwareExternalData } from "./ports/GetSoftwareExternalData"; import type { GetSoftwareExternalDataOptions } from "./ports/GetSoftwareExternalDataOptions"; import type { GetSoftwareLatestVersion } from "./ports/GetSoftwareLatestVersion"; import type { UserApi } from "./ports/UserApi"; import { usecases } from "./usecases"; -type GitDbConfig = { dbKind: "git" } & GitDbApiParams; -type InMemoryDbConfig = { dbKind: "inMemory" }; +type PgDbConfig = { dbKind: "kysely"; kyselyDb: Kysely }; -type DbConfig = GitDbConfig | InMemoryDbConfig; +type DbConfig = PgDbConfig; type ParamsOfBootstrapCore = { dbConfig: DbConfig; @@ -36,11 +37,10 @@ type ParamsOfBootstrapCore = { export type Context = { paramsOfBootstrapCore: ParamsOfBootstrapCore; - dbApi: DbApi; + dbApi: DbApiV2; userApi: UserApi; compileData: CompileData; comptoirDuLibreApi: ComptoirDuLibreApi; - getSoftwareExternalDataOptions: GetSoftwareExternalDataOptions; getSoftwareExternalData: GetSoftwareExternalData; getSoftwareLatestVersion: GetSoftwareLatestVersion; }; @@ -52,17 +52,20 @@ export type Thunks = Core["types"]["Thunks"]; export type CreateEvt = Core["types"]["CreateEvt"]; const getDbApiAndInitializeCache = (dbConfig: DbConfig): Db.DbApiAndInitializeCache => { - if (dbConfig.dbKind === "git") return createGitDbApi(dbConfig); - if (dbConfig.dbKind === "inMemory") + if (dbConfig.dbKind === "kysely") { return { - dbApi: new InMemoryDbApi(), + dbApi: createKyselyPgDbApi(dbConfig.kyselyDb), initializeDbApiCache: async () => {} }; - const shouldNotBeReached: never = dbConfig; + } + + const shouldNotBeReached: never = dbConfig.dbKind; throw new Error(`Unsupported case: ${shouldNotBeReached}`); }; -export async function bootstrapCore(params: ParamsOfBootstrapCore): Promise<{ core: Core; context: Context }> { +export async function bootstrapCore( + params: ParamsOfBootstrapCore +): Promise<{ dbApi: DbApiV2; context: Context; core: Core }> { const { dbConfig, keycloakUserApiParams, @@ -76,8 +79,7 @@ export async function bootstrapCore(params: ParamsOfBootstrapCore): Promise<{ co githubPersonalAccessTokenForApiRateLimit }); - const { getSoftwareExternalDataOptions, getSoftwareExternalData } = - getSoftwareExternalDataFunctions(externalSoftwareDataOrigin); + const { getSoftwareExternalData } = getSoftwareExternalDataFunctions(externalSoftwareDataOrigin); const { compileData } = createCompileData({ getSoftwareExternalData, @@ -105,21 +107,23 @@ export async function bootstrapCore(params: ParamsOfBootstrapCore): Promise<{ co userApi, compileData, comptoirDuLibreApi, - getSoftwareExternalDataOptions, getSoftwareExternalData, getSoftwareLatestVersion }; - const { core, dispatch } = createCore({ + const { core } = createCore({ context, usecases }); - await dispatch( - usecases.readWriteSillData.protectedThunks.initialize({ - doPerPerformPeriodicalCompilation - }) - ); + if (doPerPerformPeriodicalCompilation) { + console.log("TODO: doPerPerformPeriodicalCompilation"); + } + // await dispatch( + // usecases.readWriteSillData.protectedThunks.initialize({ + // doPerPerformPeriodicalCompilation + // }) + // ); if (doPerformCacheInitialization) { console.log("Performing cache initialization..."); @@ -127,7 +131,7 @@ export async function bootstrapCore(params: ParamsOfBootstrapCore): Promise<{ co await Promise.all([initializeDbApiCache(), initializeUserApiCache()]); } - return { core, context }; + return { dbApi, context, core }; } function getSoftwareExternalDataFunctions(externalSoftwareDataOrigin: ExternalDataOrigin): { diff --git a/api/src/core/ports/DbApi.ts b/api/src/core/ports/DbApi.ts index 0218dc76..86b9be85 100644 --- a/api/src/core/ports/DbApi.ts +++ b/api/src/core/ports/DbApi.ts @@ -1,4 +1,5 @@ import type { CompiledData } from "./CompileData"; +import { DbApiV2 } from "./DbApiV2"; export type DbApi = { fetchCompiledData: () => Promise>; @@ -95,7 +96,7 @@ export namespace Db { updateTime: number; }; - export type DbApiAndInitializeCache = { dbApi: DbApi; initializeDbApiCache: () => Promise }; + export type DbApiAndInitializeCache = { dbApi: DbApiV2; initializeDbApiCache: () => Promise }; } export type Os = "windows" | "linux" | "mac" | "android" | "ios"; diff --git a/api/src/core/ports/DbApiV2.ts b/api/src/core/ports/DbApiV2.ts index 1510b14a..a3ff5903 100644 --- a/api/src/core/ports/DbApiV2.ts +++ b/api/src/core/ports/DbApiV2.ts @@ -5,7 +5,7 @@ import type { CompiledData } from "./CompileData"; import type { ExternalDataOrigin } from "./GetSoftwareExternalData"; -type WithAgentEmail = { agentEmail: string }; +export type WithAgentEmail = { agentEmail: string }; export interface SoftwareRepository { create: ( @@ -13,7 +13,7 @@ export interface SoftwareRepository { formData: SoftwareFormData; externalDataOrigin: ExternalDataOrigin; } & WithAgentEmail - ) => Promise; + ) => Promise; update: ( params: { softwareSillId: number; @@ -21,13 +21,15 @@ export interface SoftwareRepository { } & WithAgentEmail ) => Promise; getAll: () => Promise; + getById: (id: number) => Promise; + countAddedByAgent: (params: { agentEmail: string }) => Promise; getAllSillSoftwareExternalIds: (externalDataOrigin: ExternalDataOrigin) => Promise; - unreference: () => {}; + unreference: (params: { softwareId: number; reason: string; time: number }) => Promise; } export interface InstanceRepository { - create: (params: { fromData: InstanceFormData } & WithAgentEmail) => Promise; - update: (params: { fromData: InstanceFormData; instanceId: number }) => Promise; + create: (params: { formData: InstanceFormData } & WithAgentEmail) => Promise; + update: (params: { formData: InstanceFormData; instanceId: number }) => Promise; getAll: () => Promise; } @@ -40,7 +42,7 @@ export type Agent = { }; export interface AgentRepository { - add: (agent: OmitFromExisting) => Promise; + add: (agent: OmitFromExisting) => Promise; update: (agent: Agent) => Promise; remove: (agentId: number) => Promise; getByEmail: (email: string) => Promise; @@ -50,12 +52,14 @@ export interface AgentRepository { export interface SoftwareReferentRepository { add: (params: Database["software_referents"]) => Promise; remove: (params: { softwareId: number; agentId: number }) => Promise; + countSoftwaresForAgent: (params: { agentId: number }) => Promise; getTotalCount: () => Promise; } export interface SoftwareUserRepository { add: (params: Database["software_users"]) => Promise; remove: (params: { softwareId: number; agentId: number }) => Promise; + countSoftwaresForAgent: (params: { agentId: number }) => Promise; } export type DbApiV2 = { diff --git a/api/src/core/usecases/index.ts b/api/src/core/usecases/index.ts index 65ef0b10..a960f1d7 100644 --- a/api/src/core/usecases/index.ts +++ b/api/src/core/usecases/index.ts @@ -1,7 +1,6 @@ -import * as readWriteSillData from "./readWriteSillData"; import * as suggestionAndAutoFill from "./suggestionAndAutoFill"; export const usecases = { - readWriteSillData, + // readWriteSillData, suggestionAndAutoFill }; diff --git a/api/src/core/usecases/readWriteSillData/selectors.ts b/api/src/core/usecases/readWriteSillData/selectors.ts index d5a705ba..821ee13b 100644 --- a/api/src/core/usecases/readWriteSillData/selectors.ts +++ b/api/src/core/usecases/readWriteSillData/selectors.ts @@ -1,319 +1,321 @@ -import type { State as RootState } from "../../bootstrap"; -import { createSelector } from "redux-clean-architecture"; -import { assert } from "tsafe/assert"; -import { compiledDataPrivateToPublic } from "../../ports/CompileData"; -import { removeDuplicates } from "evt/tools/reducers/removeDuplicates"; -import { exclude } from "tsafe/exclude"; -import { id } from "tsafe/id"; -import { SoftwareExternalData } from "../../ports/GetSoftwareExternalData"; -import { name } from "./state"; -import type { Software, Agent, Instance, DeclarationFormData } from "./types"; -import { CompiledData } from "../../ports/CompileData"; - -const sliceState = (state: RootState) => state[name]; - -const compiledData = createSelector(sliceState, state => state.compiledData); - -const similarSoftwarePartition = createSelector(compiledData, (compiledData): Software.SimilarSoftware[][] => { - const compiledSoftwareByWikidataId: { [wikidataId: string]: CompiledData.Software<"private"> } = {}; - - compiledData.forEach(software => { - if (software.softwareExternalData === undefined) { - return; - } - compiledSoftwareByWikidataId[software.softwareExternalData.externalId] = software; - }); - - const compiledSoftwareByName: { [name: string]: CompiledData.Software<"private"> } = {}; - - compiledData.forEach(software => { - compiledSoftwareByName[software.name] = software; - }); - - function wikidataSoftwareToSimilarSoftware( - externalSoftwareData: Pick< - SoftwareExternalData, - "externalId" | "label" | "description" | "isLibreSoftware" | "externalDataOrigin" - > - ): Software.SimilarSoftware { - const software = compiledSoftwareByWikidataId[externalSoftwareData.externalId]; - - if (software === undefined) { - return { - "isInSill": false, - "externalId": externalSoftwareData.externalId, - "externalDataOrigin": externalSoftwareData.externalDataOrigin, - "label": externalSoftwareData.label, - "description": externalSoftwareData.description, - "isLibreSoftware": externalSoftwareData.isLibreSoftware - }; - } - - return { - "isInSill": true, - "softwareName": software.name, - "softwareDescription": software.description - }; - } - - const similarSoftwarePartition: Software.SimilarSoftware[][] = []; - - compiledData.forEach(o => { - const softwareAlreadySeen = new Set(); - - for (const similarSoftwares of similarSoftwarePartition) { - for (const similarSoftware of similarSoftwares) { - if (!similarSoftware.isInSill) { - continue; - } - if (similarSoftware.softwareName === o.name) { - return; - } - softwareAlreadySeen.add(similarSoftware.softwareName); - } - } - - function getPartition(similarSoftware: Software.SimilarSoftware): Software.SimilarSoftware[] { - { - const id = similarSoftware.isInSill ? similarSoftware.softwareName : similarSoftware.externalId; - - if (softwareAlreadySeen.has(id)) { - return []; - } - - softwareAlreadySeen.add(id); - } - - return id([ - similarSoftware, - ...(() => { - if (!similarSoftware.isInSill) { - return []; - } - - const software = compiledSoftwareByName[similarSoftware.softwareName]; - - assert(software !== undefined); - - return software.similarExternalSoftwares - .map(wikidataSoftware => getPartition(wikidataSoftwareToSimilarSoftware(wikidataSoftware))) - .flat(); - })(), - ...compiledData - .map(software => { - const hasCurrentSimilarSoftwareInItsListOfSimilarSoftware = - software.similarExternalSoftwares.find(externalSoftware => { - const similarSoftware_i = wikidataSoftwareToSimilarSoftware(externalSoftware); - - if (similarSoftware.isInSill) { - return ( - similarSoftware_i.isInSill && - similarSoftware_i.softwareName === similarSoftware.softwareName - ); - } else { - return externalSoftware.externalId === similarSoftware.externalId; - } - }) !== undefined; - - if (!hasCurrentSimilarSoftwareInItsListOfSimilarSoftware) { - return undefined; - } - - return getPartition({ - "isInSill": true, - "softwareName": software.name, - "softwareDescription": software.description - }); - }) - .filter(exclude(undefined)) - .flat() - ]); - } - - similarSoftwarePartition.push( - getPartition( - id({ - "isInSill": true, - "softwareName": o.name, - "softwareDescription": o.description - }) - ) - ); - }); - - return similarSoftwarePartition; -}); - -const softwares = createSelector(compiledData, similarSoftwarePartition, (compiledData, similarSoftwarePartition) => { - return compiledData.map( - (o): Software => ({ - "serviceProviders": o.serviceProviders, - "logoUrl": o.logoUrl ?? o.softwareExternalData?.logoUrl ?? o.comptoirDuLibreSoftware?.logoUrl, - "softwareId": o.id, - "softwareName": o.name, - "softwareDescription": o.description, - "latestVersion": o.latestVersion, - "testUrl": o.testUrls[0]?.url, - "addedTime": o.referencedSinceTime, - "updateTime": o.updateTime, - "dereferencing": o.dereferencing, - "categories": o.categories, - "prerogatives": { - "doRespectRgaa": o.doRespectRgaa, - "isFromFrenchPublicServices": o.isFromFrenchPublicService, - "isPresentInSupportContract": o.isPresentInSupportContract - }, - "userAndReferentCountByOrganization": (() => { - const out: Software["userAndReferentCountByOrganization"] = {}; - - o.referents.forEach(referent => { - const entry = (out[referent.organization] ??= { "referentCount": 0, "userCount": 0 }); - entry.referentCount++; - }); - o.users.forEach(user => { - const entry = (out[user.organization] ??= { "referentCount": 0, "userCount": 0 }); - entry.userCount++; - }); - - return out; - })(), - "authors": - o.softwareExternalData?.developers.map(developer => ({ - "authorName": developer.name, - "authorUrl": `https://www.wikidata.org/wiki/${developer.id}` - })) ?? [], - "officialWebsiteUrl": - o.softwareExternalData?.websiteUrl ?? - o.comptoirDuLibreSoftware?.external_resources.website ?? - undefined, - "codeRepositoryUrl": - o.softwareExternalData?.sourceUrl ?? - o.comptoirDuLibreSoftware?.external_resources.repository ?? - undefined, - "documentationUrl": o.softwareExternalData?.documentationUrl, - "versionMin": o.versionMin, - "license": o.license, - "comptoirDuLibreServiceProviderCount": o.comptoirDuLibreSoftware?.providers.length ?? 0, - "annuaireCnllServiceProviders": o.annuaireCnllServiceProviders ?? [], - "comptoirDuLibreId": o.comptoirDuLibreSoftware?.id, - "externalId": o.softwareExternalData?.externalId, - "externalDataOrigin": o.softwareExternalData?.externalDataOrigin, - "softwareType": o.softwareType, - "parentWikidataSoftware": o.parentWikidataSoftware, - "similarSoftwares": (() => { - for (const similarSoftwares of similarSoftwarePartition) { - for (const similarSoftware of similarSoftwares) { - if (!similarSoftware.isInSill) { - continue; - } - if (similarSoftware.softwareName === o.name) { - return similarSoftwares.filter(item => item !== similarSoftware); - } - } - } - - return []; - })(), - "keywords": [...o.keywords, ...(o.comptoirDuLibreSoftware?.keywords ?? [])].reduce( - ...removeDuplicates((k1, k2) => k1.toLowerCase() === k2.toLowerCase()) - ) - }) - ); -}); - -const instances = createSelector(compiledData, (compiledData): Instance[] => - compiledData - .map(software => software.instances.map(instance => ({ ...instance, "mainSoftwareSillId": software.id }))) - .flat() - .map(({ id, organization, targetAudience, publicUrl, addedByAgentEmail, mainSoftwareSillId }) => ({ - id, - mainSoftwareSillId, - organization, - targetAudience, - publicUrl, - addedByAgentEmail - })) -); - -const agents = createSelector(sliceState, state => - state.db.agentRows.map((agentRow): Agent => { - const getSoftwareName = (softwareId: number) => { - const row = state.db.softwareRows.find(row => row.id === softwareId); - - assert(row !== undefined); - - return row.name; - }; - - return { - "email": agentRow.email, - "declarations": [ - ...state.db.softwareUserRows - .filter(row => row.agentEmail === agentRow.email) - .map((row): DeclarationFormData.User & { softwareName: string } => ({ - "declarationType": "user", - "usecaseDescription": row.useCaseDescription, - "os": row.os, - "version": row.version, - "serviceUrl": row.serviceUrl, - "softwareName": getSoftwareName(row.softwareId) - })), - ...state.db.softwareReferentRows - .filter(row => row.agentEmail === agentRow.email) - .map((row): DeclarationFormData.Referent & { softwareName: string } => ({ - "declarationType": "referent", - "isTechnicalExpert": row.isExpert, - "usecaseDescription": row.useCaseDescription, - "serviceUrl": row.serviceUrl, - "softwareName": getSoftwareName(row.softwareId) - })) - ], - "organization": agentRow.organization - }; - }) -); - -const aboutAndIsPublicByAgentEmail = createSelector( - sliceState, - (state): Record => - Object.fromEntries( - state.db.agentRows.map(agentRow => [ - agentRow.email, - { - "isPublic": agentRow.isPublic, - "about": agentRow.about - } - ]) - ) -); - -const referentCount = createSelector( - agents, - agents => - agents - .filter( - agent => - agent.declarations.find(declaration => declaration.declarationType === "referent") !== undefined - ) - .map(agent => agent.email) - .reduce(...removeDuplicates()).length -); - -const compiledDataPublicJson = createSelector(sliceState, state => { - const { compiledData } = state; - - return JSON.stringify(compiledDataPrivateToPublic(compiledData), null, 2); -}); - -export const protectedSelectors = { - similarSoftwarePartition -}; - -export const selectors = { - softwares, - instances, - agents, - referentCount, - compiledDataPublicJson, - aboutAndIsPublicByAgentEmail -}; +// import type { State as RootState } from "../../bootstrap"; +// import { createSelector } from "redux-clean-architecture"; +// import { assert } from "tsafe/assert"; +// import { compiledDataPrivateToPublic } from "../../ports/CompileData"; +// import { removeDuplicates } from "evt/tools/reducers/removeDuplicates"; +// import { exclude } from "tsafe/exclude"; +// import { id } from "tsafe/id"; +// import { SoftwareExternalData } from "../../ports/GetSoftwareExternalData"; +// import { name } from "./state"; +// import type { Software, Agent, Instance, DeclarationFormData } from "./types"; +// import { CompiledData } from "../../ports/CompileData"; +// +// const sliceState = (state: RootState) => state[name]; +// +// const compiledData = createSelector(sliceState, state => state.compiledData); +// +// const similarSoftwarePartition = createSelector(compiledData, (compiledData): Software.SimilarSoftware[][] => { +// const compiledSoftwareByWikidataId: { [wikidataId: string]: CompiledData.Software<"private"> } = {}; +// +// compiledData.forEach(software => { +// if (software.softwareExternalData === undefined) { +// return; +// } +// compiledSoftwareByWikidataId[software.softwareExternalData.externalId] = software; +// }); +// +// const compiledSoftwareByName: { [name: string]: CompiledData.Software<"private"> } = {}; +// +// compiledData.forEach(software => { +// compiledSoftwareByName[software.name] = software; +// }); +// +// function wikidataSoftwareToSimilarSoftware( +// externalSoftwareData: Pick< +// SoftwareExternalData, +// "externalId" | "label" | "description" | "isLibreSoftware" | "externalDataOrigin" +// > +// ): Software.SimilarSoftware { +// const software = compiledSoftwareByWikidataId[externalSoftwareData.externalId]; +// +// if (software === undefined) { +// return { +// "isInSill": false, +// "externalId": externalSoftwareData.externalId, +// "externalDataOrigin": externalSoftwareData.externalDataOrigin, +// "label": externalSoftwareData.label, +// "description": externalSoftwareData.description, +// "isLibreSoftware": externalSoftwareData.isLibreSoftware +// }; +// } +// +// return { +// "isInSill": true, +// "softwareName": software.name, +// "softwareDescription": software.description +// }; +// } +// +// const similarSoftwarePartition: Software.SimilarSoftware[][] = []; +// +// compiledData.forEach(o => { +// const softwareAlreadySeen = new Set(); +// +// for (const similarSoftwares of similarSoftwarePartition) { +// for (const similarSoftware of similarSoftwares) { +// if (!similarSoftware.isInSill) { +// continue; +// } +// if (similarSoftware.softwareName === o.name) { +// return; +// } +// softwareAlreadySeen.add(similarSoftware.softwareName); +// } +// } +// +// function getPartition(similarSoftware: Software.SimilarSoftware): Software.SimilarSoftware[] { +// { +// const id = similarSoftware.isInSill ? similarSoftware.softwareName : similarSoftware.externalId; +// +// if (softwareAlreadySeen.has(id)) { +// return []; +// } +// +// softwareAlreadySeen.add(id); +// } +// +// return id([ +// similarSoftware, +// ...(() => { +// if (!similarSoftware.isInSill) { +// return []; +// } +// +// const software = compiledSoftwareByName[similarSoftware.softwareName]; +// +// assert(software !== undefined); +// +// return software.similarExternalSoftwares +// .map(wikidataSoftware => getPartition(wikidataSoftwareToSimilarSoftware(wikidataSoftware))) +// .flat(); +// })(), +// ...compiledData +// .map(software => { +// const hasCurrentSimilarSoftwareInItsListOfSimilarSoftware = +// software.similarExternalSoftwares.find(externalSoftware => { +// const similarSoftware_i = wikidataSoftwareToSimilarSoftware(externalSoftware); +// +// if (similarSoftware.isInSill) { +// return ( +// similarSoftware_i.isInSill && +// similarSoftware_i.softwareName === similarSoftware.softwareName +// ); +// } else { +// return externalSoftware.externalId === similarSoftware.externalId; +// } +// }) !== undefined; +// +// if (!hasCurrentSimilarSoftwareInItsListOfSimilarSoftware) { +// return undefined; +// } +// +// return getPartition({ +// "isInSill": true, +// "softwareName": software.name, +// "softwareDescription": software.description +// }); +// }) +// .filter(exclude(undefined)) +// .flat() +// ]); +// } +// +// similarSoftwarePartition.push( +// getPartition( +// id({ +// "isInSill": true, +// "softwareName": o.name, +// "softwareDescription": o.description +// }) +// ) +// ); +// }); +// +// return similarSoftwarePartition; +// }); +// +// const softwares = createSelector(compiledData, similarSoftwarePartition, (compiledData, similarSoftwarePartition) => { +// return compiledData.map( +// (o): Software => ({ +// "serviceProviders": o.serviceProviders, +// "logoUrl": o.logoUrl ?? o.softwareExternalData?.logoUrl ?? o.comptoirDuLibreSoftware?.logoUrl, +// "softwareId": o.id, +// "softwareName": o.name, +// "softwareDescription": o.description, +// "latestVersion": o.latestVersion, +// "testUrl": o.testUrls[0]?.url, +// "addedTime": o.referencedSinceTime, +// "updateTime": o.updateTime, +// "dereferencing": o.dereferencing, +// "categories": o.categories, +// "prerogatives": { +// "doRespectRgaa": o.doRespectRgaa, +// "isFromFrenchPublicServices": o.isFromFrenchPublicService, +// "isPresentInSupportContract": o.isPresentInSupportContract +// }, +// "userAndReferentCountByOrganization": (() => { +// const out: Software["userAndReferentCountByOrganization"] = {}; +// +// o.referents.forEach(referent => { +// const entry = (out[referent.organization] ??= { "referentCount": 0, "userCount": 0 }); +// entry.referentCount++; +// }); +// o.users.forEach(user => { +// const entry = (out[user.organization] ??= { "referentCount": 0, "userCount": 0 }); +// entry.userCount++; +// }); +// +// return out; +// })(), +// "authors": +// o.softwareExternalData?.developers.map(developer => ({ +// "authorName": developer.name, +// "authorUrl": `https://www.wikidata.org/wiki/${developer.id}` +// })) ?? [], +// "officialWebsiteUrl": +// o.softwareExternalData?.websiteUrl ?? +// o.comptoirDuLibreSoftware?.external_resources.website ?? +// undefined, +// "codeRepositoryUrl": +// o.softwareExternalData?.sourceUrl ?? +// o.comptoirDuLibreSoftware?.external_resources.repository ?? +// undefined, +// "documentationUrl": o.softwareExternalData?.documentationUrl, +// "versionMin": o.versionMin, +// "license": o.license, +// "comptoirDuLibreServiceProviderCount": o.comptoirDuLibreSoftware?.providers.length ?? 0, +// "annuaireCnllServiceProviders": o.annuaireCnllServiceProviders ?? [], +// "comptoirDuLibreId": o.comptoirDuLibreSoftware?.id, +// "externalId": o.softwareExternalData?.externalId, +// "externalDataOrigin": o.softwareExternalData?.externalDataOrigin, +// "softwareType": o.softwareType, +// "parentWikidataSoftware": o.parentWikidataSoftware, +// "similarSoftwares": (() => { +// for (const similarSoftwares of similarSoftwarePartition) { +// for (const similarSoftware of similarSoftwares) { +// if (!similarSoftware.isInSill) { +// continue; +// } +// if (similarSoftware.softwareName === o.name) { +// return similarSoftwares.filter(item => item !== similarSoftware); +// } +// } +// } +// +// return []; +// })(), +// "keywords": [...o.keywords, ...(o.comptoirDuLibreSoftware?.keywords ?? [])].reduce( +// ...removeDuplicates((k1, k2) => k1.toLowerCase() === k2.toLowerCase()) +// ) +// }) +// ); +// }); +// +// const instances = createSelector(compiledData, (compiledData): Instance[] => +// compiledData +// .map(software => software.instances.map(instance => ({ ...instance, "mainSoftwareSillId": software.id }))) +// .flat() +// .map(({ id, organization, targetAudience, publicUrl, addedByAgentEmail, mainSoftwareSillId }) => ({ +// id, +// mainSoftwareSillId, +// organization, +// targetAudience, +// publicUrl, +// addedByAgentEmail +// })) +// ); +// +// const agents = createSelector(sliceState, state => +// state.db.agentRows.map((agentRow): Agent => { +// const getSoftwareName = (softwareId: number) => { +// const row = state.db.softwareRows.find(row => row.id === softwareId); +// +// assert(row !== undefined); +// +// return row.name; +// }; +// +// return { +// "email": agentRow.email, +// "declarations": [ +// ...state.db.softwareUserRows +// .filter(row => row.agentEmail === agentRow.email) +// .map((row): DeclarationFormData.User & { softwareName: string } => ({ +// "declarationType": "user", +// "usecaseDescription": row.useCaseDescription, +// "os": row.os, +// "version": row.version, +// "serviceUrl": row.serviceUrl, +// "softwareName": getSoftwareName(row.softwareId) +// })), +// ...state.db.softwareReferentRows +// .filter(row => row.agentEmail === agentRow.email) +// .map((row): DeclarationFormData.Referent & { softwareName: string } => ({ +// "declarationType": "referent", +// "isTechnicalExpert": row.isExpert, +// "usecaseDescription": row.useCaseDescription, +// "serviceUrl": row.serviceUrl, +// "softwareName": getSoftwareName(row.softwareId) +// })) +// ], +// "organization": agentRow.organization +// }; +// }) +// ); +// +// const aboutAndIsPublicByAgentEmail = createSelector( +// sliceState, +// (state): Record => +// Object.fromEntries( +// state.db.agentRows.map(agentRow => [ +// agentRow.email, +// { +// "isPublic": agentRow.isPublic, +// "about": agentRow.about +// } +// ]) +// ) +// ); +// +// const referentCount = createSelector( +// agents, +// agents => +// agents +// .filter( +// agent => +// agent.declarations.find(declaration => declaration.declarationType === "referent") !== undefined +// ) +// .map(agent => agent.email) +// .reduce(...removeDuplicates()).length +// ); +// +// const compiledDataPublicJson = createSelector(sliceState, state => { +// const { compiledData } = state; +// +// return JSON.stringify(compiledDataPrivateToPublic(compiledData), null, 2); +// }); +// +// export const protectedSelectors = { +// similarSoftwarePartition +// }; +// +// export const selectors = { +// softwares, +// instances, +// agents, +// referentCount, +// compiledDataPublicJson, +// aboutAndIsPublicByAgentEmail +// }; + +export const selectors = {}; diff --git a/api/src/core/usecases/readWriteSillData/thunks.ts b/api/src/core/usecases/readWriteSillData/thunks.ts index 2f3b117f..8f35a713 100644 --- a/api/src/core/usecases/readWriteSillData/thunks.ts +++ b/api/src/core/usecases/readWriteSillData/thunks.ts @@ -1,902 +1,904 @@ -import structuredClone from "@ungap/structured-clone"; -import type { Thunks } from "../../bootstrap"; -import { createUsecaseContextApi } from "redux-clean-architecture"; -import { Mutex } from "async-mutex"; -import { assert, type Equals } from "tsafe/assert"; -import type { Db } from "../../ports/DbApi"; -import { same } from "evt/tools/inDepth/same"; -import { Deferred } from "evt/tools/Deferred"; -import * as suggestionAndAutoFill from "../suggestionAndAutoFill"; -import { objectKeys } from "tsafe/objectKeys"; -import { name, actions } from "./state"; -import type { SoftwareFormData, DeclarationFormData, InstanceFormData, Agent } from "./types"; -import { selectors } from "./selectors"; - -const { getContext } = createUsecaseContextApi(() => ({ - "mutex": new Mutex() -})); - -export const protectedThunks = { - "initialize": - (params: { doPerPerformPeriodicalCompilation: boolean }) => - async (...args) => { - const { doPerPerformPeriodicalCompilation } = params; - - const [dispatch, getState, rootContext] = args; - - const { dbApi, evtAction } = rootContext; - - const [db, compiledData] = await Promise.all([dbApi.fetchDb(), dbApi.fetchCompiledData()]); - - dispatch( - actions.updated({ - db, - compiledData - }) - ); - - periodical_compilation: { - if (!doPerPerformPeriodicalCompilation) { - console.log("Periodical compilation disabled"); - break periodical_compilation; - } - - console.log("Periodical update enabled"); - - dispatch(privateThunks.triggerNonIncrementalCompilation({ "triggerType": "initial" })); - - setInterval( - async () => { - try { - dispatch(privateThunks.triggerNonIncrementalCompilation({ "triggerType": "periodical" })); - } catch (error) { - console.error(`Non incremental periodical compilation failed: ${String(error)}`); - } - }, - 4 * 3600 * 1000 //4 hour - ); - } - - evtAction - .pipe(action => action.usecaseName === name && action.actionName === "updated") - .toStateful() - .attach(() => { - setTimeout(() => { - const start = Date.now(); - - console.log("Starting cache refresh of readWriteSillData selectors"); - - objectKeys(selectors).forEach(selectorName => selectors[selectorName](getState())); - - console.log(`Cache refresh of readWriteSillData selectors done in ${Date.now() - start}ms`); - }, 500); - }); - } -} satisfies Thunks; - -export const thunks = { - "manuallyTriggerNonIncrementalCompilation": - () => - async (...args) => { - const [dispatch] = args; - - await dispatch(privateThunks.triggerNonIncrementalCompilation({ "triggerType": "manual" })); - }, - "notifyPushOnMainBranch": - (params: { commitMessage: string }) => - async (...args) => { - const { commitMessage } = params; - - const [dispatch, , { dbApi }] = args; - - await dispatch( - privateThunks.transaction(async () => ({ - "newDb": await dbApi.fetchDb(), - commitMessage - })) - ); - }, - "createSoftware": - (params: { formData: SoftwareFormData; agent: { email: string; organization: string } }) => - async (...args) => { - const [dispatch] = args; - - const { formData } = params; - - const agentRow = { ...params.agent }; - - await dispatch( - privateThunks.transaction(async newDb => { - const { softwareRows, agentRows } = newDb; - - assert( - softwareRows.find(s => { - const t = (name: string) => name.toLowerCase().replace(/ /g, "-"); - return t(s.name) === t(formData.softwareName); - }) === undefined, - "There is already a software with this name" - ); - - const softwareId = - newDb.softwareRows.map(({ id }) => id).reduce((prev, curr) => Math.max(prev, curr), 0) + 1; - - const now = Date.now(); - - softwareRows.push({ - "id": softwareId, - "name": formData.softwareName, - "description": formData.softwareDescription, - "referencedSinceTime": now, - "updateTime": now, - "dereferencing": undefined, - "isStillInObservation": false, - "parentSoftwareWikidataId": undefined, - "doRespectRgaa": formData.doRespectRgaa, - "isFromFrenchPublicService": formData.isFromFrenchPublicService, - "isPresentInSupportContract": formData.isPresentInSupportContract, - "similarSoftwareExternalDataIds": formData.similarSoftwareExternalDataIds, - "externalId": formData.externalId, - "comptoirDuLibreId": formData.comptoirDuLibreId, - "license": formData.softwareLicense, - "softwareType": formData.softwareType, - "versionMin": formData.softwareMinimalVersion, - "catalogNumeriqueGouvFrId": undefined, - "workshopUrls": [], - "testUrls": [], - "categories": [], - "generalInfoMd": undefined, - "addedByAgentEmail": agentRow.email, - "logoUrl": await dispatch( - privateThunks.getStorableLogo({ - "externalId": formData.externalId, - "logoUrlFromFormData": formData.softwareLogoUrl - }) - ), - "keywords": formData.softwareKeywords - }); - - if (agentRows.find(({ email }) => email === agentRow.email) === undefined) { - agentRows.push({ - "email": agentRow.email, - "organization": agentRow.organization, - "about": undefined, - "isPublic": false - }); - } - - return { - newDb, - "commitMessage": `Add software: ${formData.softwareName}` - }; - }) - ); - }, - "updateSoftware": - (params: { - softwareSillId: number; - formData: SoftwareFormData; - agent: { email: string; organization: string }; - }) => - async (...args): Promise => { - const [dispatch, getState] = args; - - const { softwareSillId, formData, agent } = params; - - await dispatch( - privateThunks.transaction(async newDb => { - const { softwareRows, softwareReferentRows } = newDb; - - assert( - softwareReferentRows.find(({ agentEmail }) => agentEmail === agentEmail) !== undefined, - "The user is not a referent of this software" - ); - - const index = softwareRows.findIndex(softwareRow => softwareRow.id === softwareSillId); - - assert(index !== -1, "The software does not exist"); - - let commitMessage = `${softwareRows[index].name} updated by ${agent.email}`; - - { - const { - id, - referencedSinceTime, - dereferencing, - isStillInObservation, - parentSoftwareWikidataId, - addedByAgentEmail, - catalogNumeriqueGouvFrId, - categories, - generalInfoMd, - testUrls, - workshopUrls, - logoUrl: logoUrlFromDb - } = softwareRows[index]; - - const { - comptoirDuLibreId, - isFromFrenchPublicService, - isPresentInSupportContract, - similarSoftwareExternalDataIds, - softwareDescription, - softwareLicense, - softwareMinimalVersion, - softwareName, - softwareType, - externalId, - softwareLogoUrl: logoUrlFromFormData, - softwareKeywords, - doRespectRgaa, - ...rest - } = formData; - - assert>(); - - softwareRows[index] = { - id, - referencedSinceTime, - "updateTime": Date.now(), - dereferencing, - isStillInObservation, - parentSoftwareWikidataId, - doRespectRgaa, - addedByAgentEmail, - catalogNumeriqueGouvFrId, - categories, - generalInfoMd, - testUrls, - workshopUrls, - comptoirDuLibreId, - isFromFrenchPublicService, - isPresentInSupportContract, - similarSoftwareExternalDataIds, - "description": softwareDescription, - "license": softwareLicense, - "versionMin": softwareMinimalVersion, - "name": softwareName, - "softwareType": softwareType, - externalId, - "logoUrl": (() => { - const state = getState()[name]; - - const software = state.compiledData.find(({ id }) => id === softwareSillId); - - assert(software !== undefined); - - if (software.logoUrl === logoUrlFromDb) { - return logoUrlFromDb; - } - - return logoUrlFromFormData; - })(), - "keywords": softwareKeywords - }; - } - - return { - commitMessage, - newDb - }; - }) - ); - }, - "createUserOrReferent": - (params: { - softwareName: string; - formData: DeclarationFormData; - agent: { email: string; organization: string }; - }) => - async (...args): Promise => { - const [dispatch] = args; - - const { formData, softwareName, agent } = params; - - await dispatch( - privateThunks.transaction(async newDb => { - const { agentRows, softwareReferentRows, softwareUserRows, softwareRows } = newDb; - - const softwareRow = softwareRows.find(row => row.name === softwareName); - - assert(softwareRow !== undefined, "Software not in SILL"); - - if (agentRows.find(row => row.email === agent.email) === undefined) { - agentRows.push({ - "email": agent.email, - "organization": agent.organization, - "about": undefined, - "isPublic": false - }); - } - - switch (formData.declarationType) { - case "referent": - { - assert( - softwareReferentRows.find( - row => row.softwareId === softwareRow.id && row.agentEmail === agent.email - ) === undefined, - "Agent already referent of this software" - ); - - softwareReferentRows.push({ - "softwareId": softwareRow.id, - "agentEmail": agent.email, - "isExpert": formData.isTechnicalExpert, - "serviceUrl": formData.serviceUrl, - "useCaseDescription": formData.usecaseDescription - }); - } - break; - case "user": - { - assert( - softwareUserRows.find( - row => row.softwareId === softwareRow.id && row.agentEmail === agent.email - ) === undefined, - "Agent already declared as user of this software" - ); - - softwareUserRows.push({ - "softwareId": softwareRow.id, - "agentEmail": agent.email, - "os": formData.os, - "serviceUrl": formData.serviceUrl, - "useCaseDescription": formData.usecaseDescription, - "version": formData.version - }); - } - break; - } - - return { - newDb, - "commitMessage": `Add ${agent.email} as ${formData.declarationType} of ${softwareName}` - }; - }) - ); - }, - "removeUserOrReferent": - (params: { softwareName: string; declarationType: "user" | "referent"; agentEmail: string }) => - async (...args): Promise => { - const [dispatch] = args; - - const { softwareName, declarationType, agentEmail } = params; - - await dispatch( - privateThunks.transaction(async newDb => { - const { agentRows, softwareReferentRows, softwareUserRows, softwareRows, instanceRows } = newDb; - - const softwareRow = softwareRows.find(row => row.name === softwareName); - - assert(softwareRow !== undefined, `There is no ${softwareName} in SILL`); - - const softwareDeclarationRows = ((): { agentEmail: string; softwareId: number }[] => { - switch (declarationType) { - case "referent": - return softwareReferentRows; - case "user": - return softwareUserRows; - } - })(); - - const softwareDeclarationRow = softwareDeclarationRows.find( - row => row.agentEmail === agentEmail && row.softwareId === softwareRow.id - ); - - assert( - softwareDeclarationRow !== undefined, - `There is no ${agentEmail} as ${declarationType} of ${softwareName}` - ); - - softwareDeclarationRows.splice(softwareDeclarationRows.indexOf(softwareDeclarationRow), 1); - - remove_agent_if_no_longer_referenced_anywhere: { - if (softwareReferentRows.find(row => row.agentEmail === agentEmail) !== undefined) { - break remove_agent_if_no_longer_referenced_anywhere; - } - - if (softwareUserRows.find(row => row.agentEmail === agentEmail) !== undefined) { - break remove_agent_if_no_longer_referenced_anywhere; - } - - if (softwareRows.find(row => row.addedByAgentEmail === agentEmail) !== undefined) { - break remove_agent_if_no_longer_referenced_anywhere; - } - - if (instanceRows.find(row => row.addedByAgentEmail === agentEmail) !== undefined) { - break remove_agent_if_no_longer_referenced_anywhere; - } - - const agentRow = agentRows.find(row => row.email === agentEmail); - - assert(agentRow !== undefined, `There is no ${agentEmail} in the database`); - - agentRows.splice(agentRows.indexOf(agentRow), 1); - } - - return { - newDb, - "commitMessage": `Remove ${agentEmail} as ${declarationType} of ${softwareName}` - }; - }) - ); - }, - "createInstance": - (params: { formData: InstanceFormData; agent: { email: string; organization: string } }) => - async (...args): Promise<{ instanceId: number }> => { - const { agent, formData } = params; - - const [dispatch] = args; - - const dInstanceId = new Deferred<{ instanceId: number }>(); - - await dispatch( - privateThunks.transaction(async newDb => { - const { instanceRows, softwareRows } = newDb; - - { - const fmtUrl = (url: string | undefined) => - url === undefined ? {} : url.toLowerCase().replace(/\/$/, ""); - - assert( - instanceRows.find( - row => - row.mainSoftwareSillId === formData.mainSoftwareSillId && - fmtUrl(row.publicUrl) === fmtUrl(formData.publicUrl) - ) === undefined, - "This instance is already referenced" - ); - } - - const softwareRow = softwareRows.find(row => row.id === formData.mainSoftwareSillId); - - assert(softwareRow !== undefined, "Can't create instance, software not in SILL"); - - const instanceId = - instanceRows.map(({ id }) => id).reduce((prev, curr) => Math.max(prev, curr), 0) + 1; - - dInstanceId.resolve({ instanceId }); - - const now = Date.now(); - - instanceRows.push({ - "id": instanceId, - "addedByAgentEmail": agent.email, - "organization": formData.organization, - "mainSoftwareSillId": formData.mainSoftwareSillId, - "publicUrl": formData.publicUrl, - "targetAudience": formData.targetAudience, - "referencedSinceTime": now, - "updateTime": now - }); - - return { - newDb, - "commitMessage": `Adding ${softwareRow.name} instance: ${formData.publicUrl}` - }; - }) - ); - - return dInstanceId.pr; - }, - "updateInstance": - (params: { instanceId: number; formData: InstanceFormData; agentEmail: string }) => - async (...args): Promise => { - const { instanceId, formData, agentEmail } = params; - - const [dispatch] = args; - - await dispatch( - privateThunks.transaction(async newDb => { - const { instanceRows } = newDb; - - const index = instanceRows.findIndex(row => row.id === instanceId); - - assert(index !== -1, "Can't update instance, it doesn't exist"); - - const { mainSoftwareSillId, organization, publicUrl, targetAudience, ...rest } = formData; - - assert>(); - - const { id, addedByAgentEmail, referencedSinceTime } = instanceRows[index]; - - instanceRows[index] = { - id, - addedByAgentEmail, - mainSoftwareSillId, - organization, - publicUrl, - targetAudience, - referencedSinceTime, - "updateTime": Date.now() - }; - - return { - newDb, - "commitMessage": `Instance ${formData.publicUrl} updated by ${agentEmail}` - }; - }) - ); - }, - "changeAgentOrganization": - (params: { userId: string; email: string; newOrganization: string }) => - async (...args) => { - const [dispatch, , rootContext] = args; - - const { userApi } = rootContext; - - const { userId, email, newOrganization } = params; - - await dispatch( - privateThunks.transaction(async newDb => { - const { agentRows } = newDb; - - const agentRow = agentRows.find(row => row.email === email); - - if (agentRow === undefined) { - return; - } - - const { organization: oldOrganization } = agentRow; - - agentRow.organization = newOrganization; - - return { - "result": undefined, - newDb, - "commitMessage": `Update ${email} organization from ${oldOrganization} to ${newOrganization}` - }; - }) - ); - - await userApi.updateUserOrganization({ - "organization": newOrganization, - userId - }); - }, - "updateUserEmail": - (params: { userId: string; email: string; newEmail: string }) => - async (...args) => { - const [dispatch, , rootContext] = args; - - const { userApi } = rootContext; - - const { userId, email, newEmail } = params; - - await dispatch( - privateThunks.transaction(async newDb => { - const { agentRows, softwareReferentRows, softwareUserRows, softwareRows } = newDb; - - const agent = agentRows.find(row => row.email === email); - - if (agent === undefined) { - return; - } - - agent.email = newEmail; - - softwareReferentRows - .filter(({ agentEmail }) => agentEmail === email) - .forEach(softwareReferentRow => (softwareReferentRow.agentEmail = newEmail)); - - softwareUserRows - .filter(({ agentEmail }) => agentEmail === email) - .forEach(softwareUserRow => (softwareUserRow.agentEmail = newEmail)); - - softwareRows.filter(({ addedByAgentEmail }) => addedByAgentEmail === newEmail); - - return { - "commitMessage": `Updating referent email from ${email} to ${newEmail}`, - newDb - }; - }) - ); - - await userApi.updateUserEmail({ - "email": newEmail, - userId - }); - }, - "getAgentIsPublic": - (params: { email: string }) => - (...args): boolean => { - const [, getState] = args; - - const { email } = params; - - const { isPublic = false } = selectors.aboutAndIsPublicByAgentEmail(getState())[email] ?? {}; - - return isPublic; - }, - "getAgent": - (params: { email: string }) => - async (...args): Promise => { - const [, getState] = args; - - const { email } = params; - - const state = getState(); - - const { agent } = (() => { - const agents = selectors.agents(state); - - const agent = agents.find(agent => agent.email === email); - - return { agent }; - })(); - - const { isPublic = false, about } = selectors.aboutAndIsPublicByAgentEmail(state)[email] ?? {}; - - return { - "email": email, - "organization": agent?.organization ?? "", - "declarations": agent?.declarations ?? [], - isPublic, - about - }; - }, - "updateIsAgentProfilePublic": - (params: { agent: { email: string; organization: string }; isPublic: boolean }) => - async (...args) => { - const [dispatch] = args; - - const { - agent: { email, organization }, - isPublic - } = params; - - await dispatch( - privateThunks.transaction(async newDb => { - const { agentRows } = newDb; - - let agentRow = agentRows.find(agentRow => agentRow.email === email); - - if (agentRow === undefined) { - agentRow = { - email, - organization, - "isPublic": false, - "about": undefined - }; - - agentRows.push(agentRow); - } - - agentRow.isPublic = isPublic; - - return { - newDb, - "commitMessage": `Making ${email} profile ${isPublic ? "public" : "private"}` - }; - }) - ); - }, - "updateAgentAbout": - (params: { agent: { email: string; organization: string }; about: string | undefined }) => - async (...args) => { - const [dispatch] = args; - - const { - agent: { email, organization }, - about - } = params; - - await dispatch( - privateThunks.transaction(async newDb => { - const { agentRows } = newDb; - - let agentRow = agentRows.find(agentRow => agentRow.email === email); - - if (agentRow === undefined) { - agentRow = { - email, - organization, - "isPublic": false, - "about": undefined - }; - - agentRows.push(agentRow); - } - - agentRow.about = about; - - return { - newDb, - "commitMessage": `Updating ${email} about markdown text` - }; - }) - ); - }, - "unreferenceSoftware": - (params: { softwareName: string; reason: string }) => - async (...args): Promise => { - const [dispatch] = args; - - const { softwareName, reason } = params; - - await dispatch( - privateThunks.transaction(async newDb => { - const { softwareRows } = newDb; - - const softwareRow = softwareRows.find(softwareRow => softwareRow.name === softwareName); - - assert(softwareRow !== undefined, `There is no ${softwareName} in the database`); - - softwareRow.dereferencing = { - "time": Date.now(), - reason, - "lastRecommendedVersion": softwareRow.versionMin - }; - - return { - newDb, - "commitMessage": `Dereferencing ${softwareName} because ${reason}` - }; - }) - ); - } -} satisfies Thunks; - -const privateThunks = { - "transaction": - (asyncReducer: (dbClone: Db) => Promise<{ newDb: Db; commitMessage: string } | undefined>) => - async (...args): Promise => { - const [dispatch, getState, rootContext] = args; - - const { compileData, dbApi } = rootContext; - - const { mutex } = getContext(rootContext); - - const dLocalStateUpdated = new Deferred(); - - mutex - .runExclusive(async () => { - let newDb = structuredClone(getState()[name].db); - - const reducerReturnValue = await asyncReducer(newDb); - - if (reducerReturnValue === undefined) { - return; - } - - const { commitMessage, newDb: newDbReturnedByReducer } = reducerReturnValue; - - if (newDbReturnedByReducer !== undefined) { - newDb = newDbReturnedByReducer; - } - - const state = getState()[name]; - - if (same(newDb, state.db)) { - return; - } - - //NOTE: It's important to call compileData first as it may crash - //and if it does it mean that if we have committed we'll end up with - //inconsistent state. - const newCompiledData = await compileData({ - "db": newDb, - "getCachedSoftware": ({ sillSoftwareId }) => - state.compiledData.find(({ id }) => id === sillSoftwareId) - }); - - dispatch( - actions.updated({ - "db": newDb, - "compiledData": newCompiledData - }) - ); - - dLocalStateUpdated.resolve(); - - try { - await Promise.all([ - dbApi.updateDb({ newDb, commitMessage }), - dbApi.updateCompiledData({ newCompiledData, commitMessage }) - ]); - } catch (error) { - console.error( - `Error while updating the DB, this is fatal, terminating the process now`, - String(error) - ); - process.exit(1); - } - }) - .catch(error => dLocalStateUpdated.reject(error)); - - await dLocalStateUpdated.pr; - }, - "triggerNonIncrementalCompilation": - (params: { triggerType: "periodical" | "manual" | "initial" }) => - async (...args) => { - console.log("Starting non incremental compilation"); - - const { triggerType } = params; - - const [dispatch, getState, rootContext] = args; - - const { dbApi, compileData } = rootContext; - - const { mutex } = getContext(rootContext); - - const dbBefore = structuredClone(getState()[name].db); - - console.log("Non incremental compilation started"); - - const newCompiledData = await compileData({ - "db": dbBefore, - "getCachedSoftware": undefined - }); - - const wasCanceled = await mutex.runExclusive(async (): Promise => { - const { db } = getState()[name]; - - if (!same(dbBefore, db)) { - //While we where re compiling there was some other transaction, - //Re-scheduling. - console.log( - "Re-scheduling non incremental compilation, db has changed (probably due to a concurrent transaction)" - ); - return true; - } - - await dbApi.updateCompiledData({ - newCompiledData, - "commitMessage": (() => { - switch (triggerType) { - case "initial": - return "Some data have changed while the backend was down"; - case "periodical": - return "Periodical update: Some Wikidata or other 3rd party source data have changed"; - case "manual": - return "Manual trigger: Some data have changed since last compilation"; - } - })() - }); - - dispatch( - actions.updated({ - db, - "compiledData": newCompiledData - }) - ); - - return false; - }); - - if (wasCanceled) { - console.log("Data have changed, re-scheduling non incremental compilation"); - await dispatch(privateThunks.triggerNonIncrementalCompilation(params)); - } - - console.log("Done with non incremental compilation"); - }, - /** Functions that returns logoUrlFromFormData if it's not the same as the one from the automatic suggestions */ - "getStorableLogo": - (params: { logoUrlFromFormData: string | undefined; externalId: string | undefined }) => - async (...args): Promise => { - const { logoUrlFromFormData, externalId } = params; - - const [dispatch] = args; - - if (logoUrlFromFormData === undefined) { - return undefined; - } - - if (externalId === undefined) { - return logoUrlFromFormData; - } - - const softwareFormAutoFillData = await dispatch( - suggestionAndAutoFill.thunks.getSoftwareFormAutoFillDataFromExternalAndOtherSources({ - externalId - }) - ); - - if (softwareFormAutoFillData.softwareLogoUrl === logoUrlFromFormData) { - return undefined; - } - - return logoUrlFromFormData; - } -} satisfies Thunks; +// import structuredClone from "@ungap/structured-clone"; +// import type { Thunks } from "../../bootstrap"; +// import { createUsecaseContextApi } from "redux-clean-architecture"; +// import { Mutex } from "async-mutex"; +// import { assert, type Equals } from "tsafe/assert"; +// import type { Db } from "../../ports/DbApi"; +// import { same } from "evt/tools/inDepth/same"; +// import { Deferred } from "evt/tools/Deferred"; +// import * as suggestionAndAutoFill from "../suggestionAndAutoFill"; +// import { objectKeys } from "tsafe/objectKeys"; +// import { name, actions } from "./state"; +// import type { SoftwareFormData, DeclarationFormData, InstanceFormData, Agent } from "./types"; +// import { selectors } from "./selectors"; +// +// const { getContext } = createUsecaseContextApi(() => ({ +// "mutex": new Mutex() +// })); +// +// export const protectedThunks = { +// "initialize": +// (params: { doPerPerformPeriodicalCompilation: boolean }) => +// async (...args) => { +// const { doPerPerformPeriodicalCompilation } = params; +// +// const [dispatch, getState, rootContext] = args; +// +// const { dbApi, evtAction } = rootContext; +// +// const [db, compiledData] = await Promise.all([dbApi.fetchDb(), dbApi.fetchCompiledData()]); +// +// dispatch( +// actions.updated({ +// db, +// compiledData +// }) +// ); +// +// periodical_compilation: { +// if (!doPerPerformPeriodicalCompilation) { +// console.log("Periodical compilation disabled"); +// break periodical_compilation; +// } +// +// console.log("Periodical update enabled"); +// +// dispatch(privateThunks.triggerNonIncrementalCompilation({ "triggerType": "initial" })); +// +// setInterval( +// async () => { +// try { +// dispatch(privateThunks.triggerNonIncrementalCompilation({ "triggerType": "periodical" })); +// } catch (error) { +// console.error(`Non incremental periodical compilation failed: ${String(error)}`); +// } +// }, +// 4 * 3600 * 1000 //4 hour +// ); +// } +// +// evtAction +// .pipe(action => action.usecaseName === name && action.actionName === "updated") +// .toStateful() +// .attach(() => { +// setTimeout(() => { +// const start = Date.now(); +// +// console.log("Starting cache refresh of readWriteSillData selectors"); +// +// objectKeys(selectors).forEach(selectorName => selectors[selectorName](getState())); +// +// console.log(`Cache refresh of readWriteSillData selectors done in ${Date.now() - start}ms`); +// }, 500); +// }); +// } +// } satisfies Thunks; +// +// export const thunks = { +// "manuallyTriggerNonIncrementalCompilation": +// () => +// async (...args) => { +// const [dispatch] = args; +// +// await dispatch(privateThunks.triggerNonIncrementalCompilation({ "triggerType": "manual" })); +// }, +// "notifyPushOnMainBranch": +// (params: { commitMessage: string }) => +// async (...args) => { +// const { commitMessage } = params; +// +// const [dispatch, , { dbApi }] = args; +// +// await dispatch( +// privateThunks.transaction(async () => ({ +// "newDb": await dbApi.fetchDb(), +// commitMessage +// })) +// ); +// }, +// "createSoftware": +// (params: { formData: SoftwareFormData; agent: { email: string; organization: string } }) => +// async (...args) => { +// const [dispatch] = args; +// +// const { formData } = params; +// +// const agentRow = { ...params.agent }; +// +// await dispatch( +// privateThunks.transaction(async newDb => { +// const { softwareRows, agentRows } = newDb; +// +// assert( +// softwareRows.find(s => { +// const t = (name: string) => name.toLowerCase().replace(/ /g, "-"); +// return t(s.name) === t(formData.softwareName); +// }) === undefined, +// "There is already a software with this name" +// ); +// +// const softwareId = +// newDb.softwareRows.map(({ id }) => id).reduce((prev, curr) => Math.max(prev, curr), 0) + 1; +// +// const now = Date.now(); +// +// softwareRows.push({ +// "id": softwareId, +// "name": formData.softwareName, +// "description": formData.softwareDescription, +// "referencedSinceTime": now, +// "updateTime": now, +// "dereferencing": undefined, +// "isStillInObservation": false, +// "parentSoftwareWikidataId": undefined, +// "doRespectRgaa": formData.doRespectRgaa, +// "isFromFrenchPublicService": formData.isFromFrenchPublicService, +// "isPresentInSupportContract": formData.isPresentInSupportContract, +// "similarSoftwareExternalDataIds": formData.similarSoftwareExternalDataIds, +// "externalId": formData.externalId, +// "comptoirDuLibreId": formData.comptoirDuLibreId, +// "license": formData.softwareLicense, +// "softwareType": formData.softwareType, +// "versionMin": formData.softwareMinimalVersion, +// "catalogNumeriqueGouvFrId": undefined, +// "workshopUrls": [], +// "testUrls": [], +// "categories": [], +// "generalInfoMd": undefined, +// "addedByAgentEmail": agentRow.email, +// "logoUrl": await dispatch( +// privateThunks.getStorableLogo({ +// "externalId": formData.externalId, +// "logoUrlFromFormData": formData.softwareLogoUrl +// }) +// ), +// "keywords": formData.softwareKeywords +// }); +// +// if (agentRows.find(({ email }) => email === agentRow.email) === undefined) { +// agentRows.push({ +// "email": agentRow.email, +// "organization": agentRow.organization, +// "about": undefined, +// "isPublic": false +// }); +// } +// +// return { +// newDb, +// "commitMessage": `Add software: ${formData.softwareName}` +// }; +// }) +// ); +// }, +// "updateSoftware": +// (params: { +// softwareSillId: number; +// formData: SoftwareFormData; +// agent: { email: string; organization: string }; +// }) => +// async (...args): Promise => { +// const [dispatch, getState] = args; +// +// const { softwareSillId, formData, agent } = params; +// +// await dispatch( +// privateThunks.transaction(async newDb => { +// const { softwareRows, softwareReferentRows } = newDb; +// +// assert( +// softwareReferentRows.find(({ agentEmail }) => agentEmail === agentEmail) !== undefined, +// "The user is not a referent of this software" +// ); +// +// const index = softwareRows.findIndex(softwareRow => softwareRow.id === softwareSillId); +// +// assert(index !== -1, "The software does not exist"); +// +// let commitMessage = `${softwareRows[index].name} updated by ${agent.email}`; +// +// { +// const { +// id, +// referencedSinceTime, +// dereferencing, +// isStillInObservation, +// parentSoftwareWikidataId, +// addedByAgentEmail, +// catalogNumeriqueGouvFrId, +// categories, +// generalInfoMd, +// testUrls, +// workshopUrls, +// logoUrl: logoUrlFromDb +// } = softwareRows[index]; +// +// const { +// comptoirDuLibreId, +// isFromFrenchPublicService, +// isPresentInSupportContract, +// similarSoftwareExternalDataIds, +// softwareDescription, +// softwareLicense, +// softwareMinimalVersion, +// softwareName, +// softwareType, +// externalId, +// softwareLogoUrl: logoUrlFromFormData, +// softwareKeywords, +// doRespectRgaa, +// ...rest +// } = formData; +// +// assert>(); +// +// softwareRows[index] = { +// id, +// referencedSinceTime, +// "updateTime": Date.now(), +// dereferencing, +// isStillInObservation, +// parentSoftwareWikidataId, +// doRespectRgaa, +// addedByAgentEmail, +// catalogNumeriqueGouvFrId, +// categories, +// generalInfoMd, +// testUrls, +// workshopUrls, +// comptoirDuLibreId, +// isFromFrenchPublicService, +// isPresentInSupportContract, +// similarSoftwareExternalDataIds, +// "description": softwareDescription, +// "license": softwareLicense, +// "versionMin": softwareMinimalVersion, +// "name": softwareName, +// "softwareType": softwareType, +// externalId, +// "logoUrl": (() => { +// const state = getState()[name]; +// +// const software = state.compiledData.find(({ id }) => id === softwareSillId); +// +// assert(software !== undefined); +// +// if (software.logoUrl === logoUrlFromDb) { +// return logoUrlFromDb; +// } +// +// return logoUrlFromFormData; +// })(), +// "keywords": softwareKeywords +// }; +// } +// +// return { +// commitMessage, +// newDb +// }; +// }) +// ); +// }, +// "createUserOrReferent": +// (params: { +// softwareName: string; +// formData: DeclarationFormData; +// agent: { email: string; organization: string }; +// }) => +// async (...args): Promise => { +// const [dispatch] = args; +// +// const { formData, softwareName, agent } = params; +// +// await dispatch( +// privateThunks.transaction(async newDb => { +// const { agentRows, softwareReferentRows, softwareUserRows, softwareRows } = newDb; +// +// const softwareRow = softwareRows.find(row => row.name === softwareName); +// +// assert(softwareRow !== undefined, "Software not in SILL"); +// +// if (agentRows.find(row => row.email === agent.email) === undefined) { +// agentRows.push({ +// "email": agent.email, +// "organization": agent.organization, +// "about": undefined, +// "isPublic": false +// }); +// } +// +// switch (formData.declarationType) { +// case "referent": +// { +// assert( +// softwareReferentRows.find( +// row => row.softwareId === softwareRow.id && row.agentEmail === agent.email +// ) === undefined, +// "Agent already referent of this software" +// ); +// +// softwareReferentRows.push({ +// "softwareId": softwareRow.id, +// "agentEmail": agent.email, +// "isExpert": formData.isTechnicalExpert, +// "serviceUrl": formData.serviceUrl, +// "useCaseDescription": formData.usecaseDescription +// }); +// } +// break; +// case "user": +// { +// assert( +// softwareUserRows.find( +// row => row.softwareId === softwareRow.id && row.agentEmail === agent.email +// ) === undefined, +// "Agent already declared as user of this software" +// ); +// +// softwareUserRows.push({ +// "softwareId": softwareRow.id, +// "agentEmail": agent.email, +// "os": formData.os, +// "serviceUrl": formData.serviceUrl, +// "useCaseDescription": formData.usecaseDescription, +// "version": formData.version +// }); +// } +// break; +// } +// +// return { +// newDb, +// "commitMessage": `Add ${agent.email} as ${formData.declarationType} of ${softwareName}` +// }; +// }) +// ); +// }, +// "removeUserOrReferent": +// (params: { softwareName: string; declarationType: "user" | "referent"; agentEmail: string }) => +// async (...args): Promise => { +// const [dispatch] = args; +// +// const { softwareName, declarationType, agentEmail } = params; +// +// await dispatch( +// privateThunks.transaction(async newDb => { +// const { agentRows, softwareReferentRows, softwareUserRows, softwareRows, instanceRows } = newDb; +// +// const softwareRow = softwareRows.find(row => row.name === softwareName); +// +// assert(softwareRow !== undefined, `There is no ${softwareName} in SILL`); +// +// const softwareDeclarationRows = ((): { agentEmail: string; softwareId: number }[] => { +// switch (declarationType) { +// case "referent": +// return softwareReferentRows; +// case "user": +// return softwareUserRows; +// } +// })(); +// +// const softwareDeclarationRow = softwareDeclarationRows.find( +// row => row.agentEmail === agentEmail && row.softwareId === softwareRow.id +// ); +// +// assert( +// softwareDeclarationRow !== undefined, +// `There is no ${agentEmail} as ${declarationType} of ${softwareName}` +// ); +// +// softwareDeclarationRows.splice(softwareDeclarationRows.indexOf(softwareDeclarationRow), 1); +// +// remove_agent_if_no_longer_referenced_anywhere: { +// if (softwareReferentRows.find(row => row.agentEmail === agentEmail) !== undefined) { +// break remove_agent_if_no_longer_referenced_anywhere; +// } +// +// if (softwareUserRows.find(row => row.agentEmail === agentEmail) !== undefined) { +// break remove_agent_if_no_longer_referenced_anywhere; +// } +// +// if (softwareRows.find(row => row.addedByAgentEmail === agentEmail) !== undefined) { +// break remove_agent_if_no_longer_referenced_anywhere; +// } +// +// if (instanceRows.find(row => row.addedByAgentEmail === agentEmail) !== undefined) { +// break remove_agent_if_no_longer_referenced_anywhere; +// } +// +// const agentRow = agentRows.find(row => row.email === agentEmail); +// +// assert(agentRow !== undefined, `There is no ${agentEmail} in the database`); +// +// agentRows.splice(agentRows.indexOf(agentRow), 1); +// } +// +// return { +// newDb, +// "commitMessage": `Remove ${agentEmail} as ${declarationType} of ${softwareName}` +// }; +// }) +// ); +// }, +// "createInstance": +// (params: { formData: InstanceFormData; agent: { email: string; organization: string } }) => +// async (...args): Promise<{ instanceId: number }> => { +// const { agent, formData } = params; +// +// const [dispatch] = args; +// +// const dInstanceId = new Deferred<{ instanceId: number }>(); +// +// await dispatch( +// privateThunks.transaction(async newDb => { +// const { instanceRows, softwareRows } = newDb; +// +// { +// const fmtUrl = (url: string | undefined) => +// url === undefined ? {} : url.toLowerCase().replace(/\/$/, ""); +// +// assert( +// instanceRows.find( +// row => +// row.mainSoftwareSillId === formData.mainSoftwareSillId && +// fmtUrl(row.publicUrl) === fmtUrl(formData.publicUrl) +// ) === undefined, +// "This instance is already referenced" +// ); +// } +// +// const softwareRow = softwareRows.find(row => row.id === formData.mainSoftwareSillId); +// +// assert(softwareRow !== undefined, "Can't create instance, software not in SILL"); +// +// const instanceId = +// instanceRows.map(({ id }) => id).reduce((prev, curr) => Math.max(prev, curr), 0) + 1; +// +// dInstanceId.resolve({ instanceId }); +// +// const now = Date.now(); +// +// instanceRows.push({ +// "id": instanceId, +// "addedByAgentEmail": agent.email, +// "organization": formData.organization, +// "mainSoftwareSillId": formData.mainSoftwareSillId, +// "publicUrl": formData.publicUrl, +// "targetAudience": formData.targetAudience, +// "referencedSinceTime": now, +// "updateTime": now +// }); +// +// return { +// newDb, +// "commitMessage": `Adding ${softwareRow.name} instance: ${formData.publicUrl}` +// }; +// }) +// ); +// +// return dInstanceId.pr; +// }, +// "updateInstance": +// (params: { instanceId: number; formData: InstanceFormData; agentEmail: string }) => +// async (...args): Promise => { +// const { instanceId, formData, agentEmail } = params; +// +// const [dispatch] = args; +// +// await dispatch( +// privateThunks.transaction(async newDb => { +// const { instanceRows } = newDb; +// +// const index = instanceRows.findIndex(row => row.id === instanceId); +// +// assert(index !== -1, "Can't update instance, it doesn't exist"); +// +// const { mainSoftwareSillId, organization, publicUrl, targetAudience, ...rest } = formData; +// +// assert>(); +// +// const { id, addedByAgentEmail, referencedSinceTime } = instanceRows[index]; +// +// instanceRows[index] = { +// id, +// addedByAgentEmail, +// mainSoftwareSillId, +// organization, +// publicUrl, +// targetAudience, +// referencedSinceTime, +// "updateTime": Date.now() +// }; +// +// return { +// newDb, +// "commitMessage": `Instance ${formData.publicUrl} updated by ${agentEmail}` +// }; +// }) +// ); +// }, +// "changeAgentOrganization": +// (params: { userId: string; email: string; newOrganization: string }) => +// async (...args) => { +// const [dispatch, , rootContext] = args; +// +// const { userApi } = rootContext; +// +// const { userId, email, newOrganization } = params; +// +// await dispatch( +// privateThunks.transaction(async newDb => { +// const { agentRows } = newDb; +// +// const agentRow = agentRows.find(row => row.email === email); +// +// if (agentRow === undefined) { +// return; +// } +// +// const { organization: oldOrganization } = agentRow; +// +// agentRow.organization = newOrganization; +// +// return { +// "result": undefined, +// newDb, +// "commitMessage": `Update ${email} organization from ${oldOrganization} to ${newOrganization}` +// }; +// }) +// ); +// +// await userApi.updateUserOrganization({ +// "organization": newOrganization, +// userId +// }); +// }, +// "updateUserEmail": +// (params: { userId: string; email: string; newEmail: string }) => +// async (...args) => { +// const [dispatch, , rootContext] = args; +// +// const { userApi } = rootContext; +// +// const { userId, email, newEmail } = params; +// +// await dispatch( +// privateThunks.transaction(async newDb => { +// const { agentRows, softwareReferentRows, softwareUserRows, softwareRows } = newDb; +// +// const agent = agentRows.find(row => row.email === email); +// +// if (agent === undefined) { +// return; +// } +// +// agent.email = newEmail; +// +// softwareReferentRows +// .filter(({ agentEmail }) => agentEmail === email) +// .forEach(softwareReferentRow => (softwareReferentRow.agentEmail = newEmail)); +// +// softwareUserRows +// .filter(({ agentEmail }) => agentEmail === email) +// .forEach(softwareUserRow => (softwareUserRow.agentEmail = newEmail)); +// +// softwareRows.filter(({ addedByAgentEmail }) => addedByAgentEmail === newEmail); +// +// return { +// "commitMessage": `Updating referent email from ${email} to ${newEmail}`, +// newDb +// }; +// }) +// ); +// +// await userApi.updateUserEmail({ +// "email": newEmail, +// userId +// }); +// }, +// "getAgentIsPublic": +// (params: { email: string }) => +// (...args): boolean => { +// const [, getState] = args; +// +// const { email } = params; +// +// const { isPublic = false } = selectors.aboutAndIsPublicByAgentEmail(getState())[email] ?? {}; +// +// return isPublic; +// }, +// "getAgent": +// (params: { email: string }) => +// async (...args): Promise => { +// const [, getState] = args; +// +// const { email } = params; +// +// const state = getState(); +// +// const { agent } = (() => { +// const agents = selectors.agents(state); +// +// const agent = agents.find(agent => agent.email === email); +// +// return { agent }; +// })(); +// +// const { isPublic = false, about } = selectors.aboutAndIsPublicByAgentEmail(state)[email] ?? {}; +// +// return { +// "email": email, +// "organization": agent?.organization ?? "", +// "declarations": agent?.declarations ?? [], +// isPublic, +// about +// }; +// }, +// "updateIsAgentProfilePublic": +// (params: { agent: { email: string; organization: string }; isPublic: boolean }) => +// async (...args) => { +// const [dispatch] = args; +// +// const { +// agent: { email, organization }, +// isPublic +// } = params; +// +// await dispatch( +// privateThunks.transaction(async newDb => { +// const { agentRows } = newDb; +// +// let agentRow = agentRows.find(agentRow => agentRow.email === email); +// +// if (agentRow === undefined) { +// agentRow = { +// email, +// organization, +// "isPublic": false, +// "about": undefined +// }; +// +// agentRows.push(agentRow); +// } +// +// agentRow.isPublic = isPublic; +// +// return { +// newDb, +// "commitMessage": `Making ${email} profile ${isPublic ? "public" : "private"}` +// }; +// }) +// ); +// }, +// "updateAgentAbout": +// (params: { agent: { email: string; organization: string }; about: string | undefined }) => +// async (...args) => { +// const [dispatch] = args; +// +// const { +// agent: { email, organization }, +// about +// } = params; +// +// await dispatch( +// privateThunks.transaction(async newDb => { +// const { agentRows } = newDb; +// +// let agentRow = agentRows.find(agentRow => agentRow.email === email); +// +// if (agentRow === undefined) { +// agentRow = { +// email, +// organization, +// "isPublic": false, +// "about": undefined +// }; +// +// agentRows.push(agentRow); +// } +// +// agentRow.about = about; +// +// return { +// newDb, +// "commitMessage": `Updating ${email} about markdown text` +// }; +// }) +// ); +// }, +// "unreferenceSoftware": +// (params: { softwareName: string; reason: string }) => +// async (...args): Promise => { +// const [dispatch] = args; +// +// const { softwareName, reason } = params; +// +// await dispatch( +// privateThunks.transaction(async newDb => { +// const { softwareRows } = newDb; +// +// const softwareRow = softwareRows.find(softwareRow => softwareRow.name === softwareName); +// +// assert(softwareRow !== undefined, `There is no ${softwareName} in the database`); +// +// softwareRow.dereferencing = { +// "time": Date.now(), +// reason, +// "lastRecommendedVersion": softwareRow.versionMin +// }; +// +// return { +// newDb, +// "commitMessage": `Dereferencing ${softwareName} because ${reason}` +// }; +// }) +// ); +// } +// } satisfies Thunks; +// +// const privateThunks = { +// "transaction": +// (asyncReducer: (dbClone: Db) => Promise<{ newDb: Db; commitMessage: string } | undefined>) => +// async (...args): Promise => { +// const [dispatch, getState, rootContext] = args; +// +// const { compileData, dbApi } = rootContext; +// +// const { mutex } = getContext(rootContext); +// +// const dLocalStateUpdated = new Deferred(); +// +// mutex +// .runExclusive(async () => { +// let newDb = structuredClone(getState()[name].db); +// +// const reducerReturnValue = await asyncReducer(newDb); +// +// if (reducerReturnValue === undefined) { +// return; +// } +// +// const { commitMessage, newDb: newDbReturnedByReducer } = reducerReturnValue; +// +// if (newDbReturnedByReducer !== undefined) { +// newDb = newDbReturnedByReducer; +// } +// +// const state = getState()[name]; +// +// if (same(newDb, state.db)) { +// return; +// } +// +// //NOTE: It's important to call compileData first as it may crash +// //and if it does it mean that if we have committed we'll end up with +// //inconsistent state. +// const newCompiledData = await compileData({ +// "db": newDb, +// "getCachedSoftware": ({ sillSoftwareId }) => +// state.compiledData.find(({ id }) => id === sillSoftwareId) +// }); +// +// dispatch( +// actions.updated({ +// "db": newDb, +// "compiledData": newCompiledData +// }) +// ); +// +// dLocalStateUpdated.resolve(); +// +// try { +// await Promise.all([ +// dbApi.updateDb({ newDb, commitMessage }), +// dbApi.updateCompiledData({ newCompiledData, commitMessage }) +// ]); +// } catch (error) { +// console.error( +// `Error while updating the DB, this is fatal, terminating the process now`, +// String(error) +// ); +// process.exit(1); +// } +// }) +// .catch(error => dLocalStateUpdated.reject(error)); +// +// await dLocalStateUpdated.pr; +// }, +// "triggerNonIncrementalCompilation": +// (params: { triggerType: "periodical" | "manual" | "initial" }) => +// async (...args) => { +// console.log("Starting non incremental compilation"); +// +// const { triggerType } = params; +// +// const [dispatch, getState, rootContext] = args; +// +// const { dbApi, compileData } = rootContext; +// +// const { mutex } = getContext(rootContext); +// +// const dbBefore = structuredClone(getState()[name].db); +// +// console.log("Non incremental compilation started"); +// +// const newCompiledData = await compileData({ +// "db": dbBefore, +// "getCachedSoftware": undefined +// }); +// +// const wasCanceled = await mutex.runExclusive(async (): Promise => { +// const { db } = getState()[name]; +// +// if (!same(dbBefore, db)) { +// //While we where re compiling there was some other transaction, +// //Re-scheduling. +// console.log( +// "Re-scheduling non incremental compilation, db has changed (probably due to a concurrent transaction)" +// ); +// return true; +// } +// +// await dbApi.updateCompiledData({ +// newCompiledData, +// "commitMessage": (() => { +// switch (triggerType) { +// case "initial": +// return "Some data have changed while the backend was down"; +// case "periodical": +// return "Periodical update: Some Wikidata or other 3rd party source data have changed"; +// case "manual": +// return "Manual trigger: Some data have changed since last compilation"; +// } +// })() +// }); +// +// dispatch( +// actions.updated({ +// db, +// "compiledData": newCompiledData +// }) +// ); +// +// return false; +// }); +// +// if (wasCanceled) { +// console.log("Data have changed, re-scheduling non incremental compilation"); +// await dispatch(privateThunks.triggerNonIncrementalCompilation(params)); +// } +// +// console.log("Done with non incremental compilation"); +// }, +// /** Functions that returns logoUrlFromFormData if it's not the same as the one from the automatic suggestions */ +// "getStorableLogo": +// (params: { logoUrlFromFormData: string | undefined; externalId: string | undefined }) => +// async (...args): Promise => { +// const { logoUrlFromFormData, externalId } = params; +// +// const [dispatch] = args; +// +// if (logoUrlFromFormData === undefined) { +// return undefined; +// } +// +// if (externalId === undefined) { +// return logoUrlFromFormData; +// } +// +// const softwareFormAutoFillData = await dispatch( +// suggestionAndAutoFill.thunks.getSoftwareFormAutoFillDataFromExternalAndOtherSources({ +// externalId +// }) +// ); +// +// if (softwareFormAutoFillData.softwareLogoUrl === logoUrlFromFormData) { +// return undefined; +// } +// +// return logoUrlFromFormData; +// } +// } satisfies Thunks; + +export const thunks = {}; diff --git a/api/src/core/usecases/suggestionAndAutoFill/selectors.ts b/api/src/core/usecases/suggestionAndAutoFill/selectors.ts index dee7264a..00d544a0 100644 --- a/api/src/core/usecases/suggestionAndAutoFill/selectors.ts +++ b/api/src/core/usecases/suggestionAndAutoFill/selectors.ts @@ -1,15 +1,5 @@ -import type { State as RootState } from "../../bootstrap"; -import { createSelector } from "redux-clean-architecture"; -import { exclude } from "tsafe/exclude"; +// import type { State as RootState } from "../../bootstrap"; +// import { createSelector } from "redux-clean-architecture"; +// import { exclude } from "tsafe/exclude"; export const selectors = undefined; - -export const privateSelectors = (() => { - const compiledData = (state: RootState) => state.readWriteSillData.compiledData; - - const sillWikidataIds = createSelector(compiledData, compiledData => - compiledData.map(software => software.softwareExternalData?.externalId).filter(exclude(undefined)) - ); - - return { sillWikidataIds }; -})(); diff --git a/api/src/core/usecases/suggestionAndAutoFill/thunks.ts b/api/src/core/usecases/suggestionAndAutoFill/thunks.ts index c09b7fed..d5c82a1f 100644 --- a/api/src/core/usecases/suggestionAndAutoFill/thunks.ts +++ b/api/src/core/usecases/suggestionAndAutoFill/thunks.ts @@ -4,29 +4,8 @@ import { assert } from "tsafe/assert"; import type { Language } from "../../ports/GetSoftwareExternalData"; import { createResolveLocalizedString } from "i18nifty/LocalizedString/reactless"; import { id } from "tsafe/id"; -import { privateSelectors } from "./selectors"; export const thunks = { - "getSoftwareExternalDataOptionsWithPresenceInSill": - (params: { queryString: string; language: Language }) => - async (...args) => { - const { queryString, language } = params; - - const [, getState, { getSoftwareExternalDataOptions }] = args; - - const queryResults = await getSoftwareExternalDataOptions({ queryString, language }); - - const sillWikidataIds = privateSelectors.sillWikidataIds(getState()); - - return queryResults.map(({ externalId, description, label, isLibreSoftware, externalDataOrigin }) => ({ - "externalId": externalId, - "description": description, - "label": label, - "isInSill": sillWikidataIds.includes(externalId), - isLibreSoftware, - "externalDataOrigin": externalDataOrigin - })); - }, "getSoftwareFormAutoFillDataFromExternalAndOtherSources": (params: { externalId: string }) => async (...args): Promise => { diff --git a/api/src/env.ts b/api/src/env.ts index 060c0633..cb20da4e 100644 --- a/api/src/env.ts +++ b/api/src/env.ts @@ -34,7 +34,8 @@ const zConfiguration = z.object({ "isDevEnvironnement": z.boolean().optional(), // Completely disable this instance and redirect to another url "redirectUrl": z.string().optional(), - "externalSoftwareDataOrigin": z.enum(["wikidata", "HAL"]).optional() + "externalSoftwareDataOrigin": z.enum(["wikidata", "HAL"]).optional(), + "databaseUrl": z.string() }); const getJsonConfiguration = () => { @@ -73,7 +74,9 @@ const getJsonConfiguration = () => { "githubWebhookSecret": process.env.SILL_WEBHOOK_SECRET, "port": parseInt(process.env.SILL_API_PORT ?? ""), "isDevEnvironnement": process.env.SILL_IS_DEV_ENVIRONNEMENT?.toLowerCase() === "true", - "externalSoftwareDataOrigin": process.env.SILL_EXTERNAL_SOFTWARE_DATA_ORIGIN + "externalSoftwareDataOrigin": process.env.SILL_EXTERNAL_SOFTWARE_DATA_ORIGIN, + "redirectUrl": process.env.SILL_REDIRECT_URL, + "databaseUrl": process.env.DATABASE_URL }; }; diff --git a/api/src/rpc/createTestCaller.ts b/api/src/rpc/createTestCaller.ts index 73e32c64..36675fef 100644 --- a/api/src/rpc/createTestCaller.ts +++ b/api/src/rpc/createTestCaller.ts @@ -1,6 +1,11 @@ +import { Kysely } from "kysely"; import { bootstrapCore } from "../core"; -import { InMemoryDbApi } from "../core/adapters/dbApi/InMemoryDbApi"; +import { Database } from "../core/adapters/dbApi/kysely/kysely.database"; +import { createPgDialect } from "../core/adapters/dbApi/kysely/kysely.dialect"; +import { getWikidataSoftware } from "../core/adapters/wikidata/getWikidataSoftware"; +import { getWikidataSoftwareOptions } from "../core/adapters/wikidata/getWikidataSoftwareOptions"; import { ExternalDataOrigin } from "../core/ports/GetSoftwareExternalData"; +import { testPgUrl } from "../tools/test.helpers"; import { createRouter } from "./router"; import { User } from "./user"; @@ -18,9 +23,10 @@ export type ApiCaller = Awaited>["apiCaller" export const createTestCaller = async ({ user }: TestCallerConfig = { user: defaultUser }) => { const externalSoftwareDataOrigin: ExternalDataOrigin = "wikidata"; + const kyselyDb = new Kysely({ dialect: createPgDialect(testPgUrl) }); - const { core, context } = await bootstrapCore({ - "dbConfig": { dbKind: "inMemory" }, + const { core, context, dbApi } = await bootstrapCore({ + "dbConfig": { dbKind: "kysely", kyselyDb }, "keycloakUserApiParams": undefined, "githubPersonalAccessTokenForApiRateLimit": "fake-token", "doPerPerformPeriodicalCompilation": false, @@ -36,14 +42,17 @@ export const createTestCaller = async ({ user }: TestCallerConfig = { user: defa const { router } = createRouter({ core, + dbApi, coreContext: context, keycloakParams: undefined, redirectUrl: undefined, externalSoftwareDataOrigin, readmeUrl: "http://readme.url", termsOfServiceUrl: "http://terms.url", - jwtClaimByUserKey + jwtClaimByUserKey, + getSoftwareExternalDataOptions: getWikidataSoftwareOptions, + getSoftwareExternalData: getWikidataSoftware }); - return { apiCaller: router.createCaller({ user }), inMemoryDb: context.dbApi as InMemoryDbApi }; + return { apiCaller: router.createCaller({ user }), kyselyDb }; }; diff --git a/api/src/rpc/router.ts b/api/src/rpc/router.ts index 9e39b7fa..e91c883f 100644 --- a/api/src/rpc/router.ts +++ b/api/src/rpc/router.ts @@ -9,7 +9,15 @@ import type { Equals, ReturnType } from "tsafe"; import { assert } from "tsafe/assert"; import { z } from "zod"; import type { Context as CoreContext, Core } from "../core"; -import { ExternalDataOrigin, Language, languages, type LocalizedString } from "../core/ports/GetSoftwareExternalData"; +import { DbApiV2 } from "../core/ports/DbApiV2"; +import { + ExternalDataOrigin, + GetSoftwareExternalData, + Language, + languages, + type LocalizedString +} from "../core/ports/GetSoftwareExternalData"; +import type { GetSoftwareExternalDataOptions } from "../core/ports/GetSoftwareExternalDataOptions"; import { DeclarationFormData, InstanceFormData, @@ -24,6 +32,7 @@ import type { Context } from "./context"; import type { User } from "./user"; export function createRouter(params: { + dbApi: DbApiV2; core: Core; coreContext: CoreContext; keycloakParams: @@ -36,16 +45,20 @@ export function createRouter(params: { readmeUrl: LocalizedString; redirectUrl: string | undefined; externalSoftwareDataOrigin: ExternalDataOrigin; + getSoftwareExternalDataOptions: GetSoftwareExternalDataOptions; + getSoftwareExternalData: GetSoftwareExternalData; }) { const { core, + dbApi, coreContext, keycloakParams, jwtClaimByUserKey, termsOfServiceUrl, readmeUrl, redirectUrl, - externalSoftwareDataOrigin + externalSoftwareDataOrigin: externalDataOrigin, + getSoftwareExternalDataOptions } = params; const t = initTRPC.context().create({ @@ -69,7 +82,7 @@ export function createRouter(params: { const router = t.router({ "getRedirectUrl": loggedProcedure.query(() => redirectUrl), - "getExternalSoftwareDataOrigin": loggedProcedure.query(() => externalSoftwareDataOrigin), + "getExternalSoftwareDataOrigin": loggedProcedure.query(() => externalDataOrigin), "getApiVersion": loggedProcedure.query( (() => { const out: string = JSON.parse( @@ -108,8 +121,8 @@ export function createRouter(params: { return () => organizationUserProfileAttributeName; })() ), - "getSoftwares": loggedProcedure.query(() => core.states.readWriteSillData.getSoftwares()), - "getInstances": loggedProcedure.query(() => core.states.readWriteSillData.getInstances()), + "getSoftwares": loggedProcedure.query(() => dbApi.software.getAll()), + "getInstances": loggedProcedure.query(() => dbApi.instance.getAll()), "getExternalSoftwareOptions": loggedProcedure .input( z.object({ @@ -117,7 +130,7 @@ export function createRouter(params: { "language": zLanguage }) ) - .query(({ ctx: { user }, input }) => { + .query(async ({ ctx: { user }, input }) => { if (user === undefined) { //To prevent abuse. throw new TRPCError({ "code": "UNAUTHORIZED" }); @@ -125,10 +138,19 @@ export function createRouter(params: { const { queryString, language } = input; - return core.functions.suggestionAndAutoFill.getSoftwareExternalDataOptionsWithPresenceInSill({ - queryString, - language - }); + const [queryResults, softwareExternalDataIds] = await Promise.all([ + getSoftwareExternalDataOptions({ queryString, language }), + dbApi.software.getAllSillSoftwareExternalIds(externalDataOrigin) + ]); + + return queryResults.map(({ externalId, description, label, isLibreSoftware, externalDataOrigin }) => ({ + "externalId": externalId, + "description": description, + "label": label, + "isInSill": softwareExternalDataIds.includes(externalId), + isLibreSoftware, + "externalDataOrigin": externalDataOrigin + })); }), "getSoftwareFormAutoFillDataFromExternalSoftwareAndOtherSources": loggedProcedure .input( @@ -165,13 +187,20 @@ export function createRouter(params: { // from readWriteSillData/thunks/getStorableLogo try { - await core.functions.readWriteSillData.createSoftware({ + await dbApi.software.create({ formData, - "agent": { - "email": user.email, - "organization": user.organization - } + agentEmail: user.email, + externalDataOrigin }); + const agent = await dbApi.agent.getByEmail(user.email); + if (!agent) { + await dbApi.agent.add({ + email: user.email, + organization: user.organization, + about: undefined, + isPublic: false + }); + } } catch (e) { throw new TRPCError({ "code": "INTERNAL_SERVER_ERROR", "message": String(e) }); } @@ -190,43 +219,72 @@ export function createRouter(params: { const { softwareSillId, formData } = input; - await core.functions.readWriteSillData.updateSoftware({ + await dbApi.software.update({ softwareSillId, formData, - "agent": { - "email": user.email, - "organization": user.organization - } + agentEmail: user.email }); }), "createUserOrReferent": loggedProcedure .input( z.object({ "formData": zDeclarationFormData, - "softwareName": z.string() + "softwareId": z.number() }) ) .mutation(async ({ ctx: { user }, input }) => { + console.log("createUserOrReferent, user :", user); if (user === undefined) { throw new TRPCError({ "code": "UNAUTHORIZED" }); } - const { formData, softwareName } = input; + const { formData, softwareId } = input; + + console.log("createUserOrReferent, softwareId :", softwareId); + console.log("createUserOrReferent, formData :", formData); + const software = await dbApi.software.getById(softwareId); + console.log("software", software); + if (!software) throw new TRPCError({ "code": "NOT_FOUND", message: "Software not found in SILL" }); + console.log("reached ?"); + + const agent = await dbApi.agent.getByEmail(user.email); + let agentId = agent?.id as number; + if (!agent) { + agentId = await dbApi.agent.add({ + email: user.email, + organization: user.organization, + about: undefined, + isPublic: false + }); + } - await core.functions.readWriteSillData.createUserOrReferent({ - formData, - softwareName, - "agent": { - "email": user.email, - "organization": user.organization - } - }); + switch (formData.declarationType) { + case "user": + await dbApi.softwareUser.add({ + softwareId, + agentId, + os: formData.os ?? null, + serviceUrl: formData.serviceUrl ?? null, + useCaseDescription: formData.usecaseDescription, + version: formData.version + }); + break; + case "referent": + await dbApi.softwareReferent.add({ + softwareId, + agentId, + isExpert: formData.isTechnicalExpert, + useCaseDescription: formData.usecaseDescription, + serviceUrl: formData.serviceUrl ?? null + }); + break; + } }), "removeUserOrReferent": loggedProcedure .input( z.object({ - "softwareName": z.string(), + "softwareId": z.number(), "declarationType": z.enum(["user", "referent"]) }) ) @@ -235,13 +293,49 @@ export function createRouter(params: { throw new TRPCError({ "code": "UNAUTHORIZED" }); } - const { softwareName, declarationType } = input; + const { softwareId, declarationType } = input; - await core.functions.readWriteSillData.removeUserOrReferent({ - softwareName, - "agentEmail": user.email, - declarationType - }); + const agent = await dbApi.agent.getByEmail(user.email); + if (!agent) throw new TRPCError({ "code": "NOT_FOUND", message: "Agent not found" }); + + const software = await dbApi.software.getById(softwareId); + if (!software) throw new TRPCError({ "code": "NOT_FOUND", message: "Software not found" }); + + switch (declarationType) { + case "user": { + await dbApi.softwareUser.remove({ + softwareId, + agentId: agent.id + }); + break; + } + + case "referent": { + await dbApi.softwareReferent.remove({ + softwareId, + agentId: agent.id + }); + break; + } + } + + const [ + numberOfSoftwareWhereThisAgentIsUser, + numberOfSoftwareWhereThisAgentIsReferent, + numberOfSoftwareAddedByThisAgent + ] = await Promise.all([ + dbApi.softwareUser.countSoftwaresForAgent({ agentId: agent.id }), + dbApi.softwareReferent.countSoftwaresForAgent({ agentId: agent.id }), + dbApi.software.countAddedByAgent({ agentEmail: agent.email }) + ]); + + if ( + numberOfSoftwareWhereThisAgentIsReferent === 0 && + numberOfSoftwareWhereThisAgentIsUser === 0 && + numberOfSoftwareAddedByThisAgent === 0 + ) { + await dbApi.agent.remove(agent.id); + } }), "createInstance": loggedProcedure @@ -257,12 +351,9 @@ export function createRouter(params: { const { formData } = input; - const { instanceId } = await core.functions.readWriteSillData.createInstance({ + const instanceId = await dbApi.instance.create({ formData, - "agent": { - "email": user.email, - "organization": user.organization - } + agentEmail: user.email }); return { instanceId }; @@ -281,19 +372,16 @@ export function createRouter(params: { const { instanceId, formData } = input; - await core.functions.readWriteSillData.updateInstance({ - instanceId, + await dbApi.instance.update({ formData, - "agentEmail": user.email + instanceId }); }), "getAgents": loggedProcedure.query(async ({ ctx: { user } }) => { if (user === undefined) { throw new TRPCError({ "code": "UNAUTHORIZED" }); } - - const agents = core.states.readWriteSillData.getAgents(); - + const agents = await dbApi.agent.getAll(); return { agents }; }), "updateIsAgentProfilePublic": loggedProcedure @@ -309,13 +397,9 @@ export function createRouter(params: { const { isPublic } = input; - await core.functions.readWriteSillData.updateIsAgentProfilePublic({ - "agent": { - "email": user.email, - "organization": user.organization - }, - isPublic - }); + const agent = await dbApi.agent.getByEmail(user.email); + if (!agent) throw new TRPCError({ "code": "NOT_FOUND", message: "Agent not found" }); + await dbApi.agent.update({ ...agent, isPublic }); }), "updateAgentAbout": loggedProcedure .input( @@ -330,13 +414,9 @@ export function createRouter(params: { const { about } = input; - await core.functions.readWriteSillData.updateAgentAbout({ - "agent": { - "email": user.email, - "organization": user.organization - }, - about - }); + const agent = await dbApi.agent.getByEmail(user.email); + if (!agent) throw new TRPCError({ "code": "NOT_FOUND", message: "Agent not found" }); + await dbApi.agent.update({ ...agent, about }); }), "getIsAgentProfilePublic": loggedProcedure .input( @@ -347,11 +427,9 @@ export function createRouter(params: { .query(async ({ input }) => { const { email } = input; - const isPublic = core.functions.readWriteSillData.getAgentIsPublic({ - email - }); + const agent = await dbApi.agent.getByEmail(email); - return { isPublic }; + return { isPublic: agent?.isPublic ?? false }; }), "getAgent": loggedProcedure .input( @@ -362,18 +440,12 @@ export function createRouter(params: { .query(async ({ ctx: { user }, input }) => { const { email } = input; - const isPublic = core.functions.readWriteSillData.getAgentIsPublic({ - email - }); + const agent = await dbApi.agent.getByEmail(email); - if (!isPublic && user === undefined) { + if (!agent?.isPublic && user === undefined) { throw new TRPCError({ "code": "UNAUTHORIZED" }); } - const agent = await core.functions.readWriteSillData.getAgent({ - email - }); - return { agent }; }), "getAllowedEmailRegexp": loggedProcedure.query(() => coreContext.userApi.getAllowedEmailRegexp()), @@ -393,11 +465,10 @@ export function createRouter(params: { const { newOrganization } = input; - await core.functions.readWriteSillData.changeAgentOrganization({ - "email": user.email, - newOrganization, - "userId": user.id - }); + const agent = await dbApi.agent.getByEmail(user.email); + if (!agent) throw new TRPCError({ "code": "NOT_FOUND", message: "Agent not found" }); + + await dbApi.agent.update({ ...agent, organization: newOrganization }); }), "updateEmail": loggedProcedure .input( @@ -414,15 +485,13 @@ export function createRouter(params: { assert(keycloakParams !== undefined); - await core.functions.readWriteSillData.updateUserEmail({ - "userId": user.id, - "email": user.email, - newEmail - }); + const agent = await dbApi.agent.getByEmail(user.email); + if (!agent) throw new TRPCError({ "code": "NOT_FOUND", message: "Agent not found" }); + await dbApi.agent.update({ ...agent, email: newEmail }); }), "getRegisteredUserCount": loggedProcedure.query(async () => coreContext.userApi.getUserCount()), "getTotalReferentCount": loggedProcedure.query(() => { - const referentCount = core.states.readWriteSillData.getReferentCount(); + const referentCount = dbApi.softwareReferent.getTotalCount(); return { referentCount }; }), "getTermsOfServiceUrl": loggedProcedure.query(() => termsOfServiceUrl), @@ -484,7 +553,7 @@ export function createRouter(params: { "unreferenceSoftware": loggedProcedure .input( z.object({ - "softwareName": z.string(), + "softwareId": z.number(), "reason": z.string() }) ) @@ -493,11 +562,12 @@ export function createRouter(params: { throw new TRPCError({ "code": "UNAUTHORIZED" }); } - const { softwareName, reason } = input; + const { softwareId, reason } = input; - await core.functions.readWriteSillData.unreferenceSoftware({ - softwareName, - reason + await dbApi.software.unreference({ + softwareId, + reason, + time: Date.now() }); }) }); diff --git a/api/src/rpc/routes.e2e.test.ts b/api/src/rpc/routes.e2e.test.ts index d6a8a2e9..75a7dbeb 100644 --- a/api/src/rpc/routes.e2e.test.ts +++ b/api/src/rpc/routes.e2e.test.ts @@ -1,6 +1,8 @@ +import { Kysely } from "kysely"; import { beforeAll, describe, expect, it } from "vitest"; -import { InMemoryDbApi } from "../core/adapters/dbApi/InMemoryDbApi"; +import { Database } from "../core/adapters/dbApi/kysely/kysely.database"; import { CompiledData } from "../core/ports/CompileData"; +import { InstanceFormData } from "../core/usecases/readWriteSillData"; import { createDeclarationFormData, createInstanceFormData, @@ -10,48 +12,45 @@ import { } from "../tools/test.helpers"; import { ApiCaller, createTestCaller, defaultUser } from "./createTestCaller"; -const expectedSoftwareId = 1; - const softwareFormData = createSoftwareFormData(); const declarationFormData = createDeclarationFormData(); -const instanceFormData = createInstanceFormData({ mainSoftwareSillId: expectedSoftwareId }); describe("RPC e2e tests", () => { let apiCaller: ApiCaller; - let inMemoryDb: InMemoryDbApi; + let kyselyDb: Kysely; describe("getAgents - wrong paths", () => { it("fails with UNAUTHORIZED if user is not logged in", async () => { - ({ apiCaller, inMemoryDb } = await createTestCaller({ user: undefined })); + ({ apiCaller, kyselyDb } = await createTestCaller({ user: undefined })); expect(apiCaller.getAgents()).rejects.toThrow("UNAUTHORIZED"); }); }); describe("createUserOrReferent - Wrong paths", () => { it("fails with UNAUTHORIZED if user is not logged in", async () => { - ({ apiCaller, inMemoryDb } = await createTestCaller({ user: undefined })); + ({ apiCaller, kyselyDb } = await createTestCaller({ user: undefined })); expect( apiCaller.createUserOrReferent({ formData: declarationFormData, - softwareName: "Some software" + softwareId: 123 }) ).rejects.toThrow("UNAUTHORIZED"); }); it("fails when software is not found in SILL", async () => { - ({ apiCaller, inMemoryDb } = await createTestCaller()); + ({ apiCaller, kyselyDb } = await createTestCaller()); expect( apiCaller.createUserOrReferent({ formData: declarationFormData, - softwareName: "Some software" + softwareId: 404 }) - ).rejects.toThrow("Software not in SILL"); + ).rejects.toThrow("Software not found in SILL"); }); }); describe("createSoftware - Wrong paths", () => { it("fails with UNAUTHORIZED if user is not logged in", async () => { - ({ apiCaller, inMemoryDb } = await createTestCaller({ user: undefined })); + ({ apiCaller, kyselyDb } = await createTestCaller({ user: undefined })); expect( apiCaller.createSoftware({ formData: softwareFormData @@ -64,8 +63,16 @@ describe("RPC e2e tests", () => { // because those tests are not isolated // (the order is important)⚠️ describe("Scenario - Add a new software then mark an agent as user of this software", () => { + let actualSoftwareId: number; + let instanceFormData: InstanceFormData; + beforeAll(async () => { - ({ apiCaller, inMemoryDb } = await createTestCaller()); + ({ apiCaller, kyselyDb } = await createTestCaller()); + await kyselyDb.deleteFrom("software_referents").execute(); + await kyselyDb.deleteFrom("software_users").execute(); + await kyselyDb.deleteFrom("agents").execute(); + await kyselyDb.deleteFrom("softwares").execute(); + await kyselyDb.deleteFrom("instances").execute(); }); it("gets the list of agents, which is initially empty", async () => { @@ -74,14 +81,16 @@ describe("RPC e2e tests", () => { }); it("adds a new software", async () => { - expect(inMemoryDb.softwareRows).toHaveLength(0); + expect(await getSoftwareRows()).toHaveLength(0); const initialSoftwares = await apiCaller.getSoftwares(); expectToEqual(initialSoftwares, []); await apiCaller.createSoftware({ formData: softwareFormData }); - expect(inMemoryDb.softwareRows).toHaveLength(1); + + const softwareRows = await getSoftwareRows(); + expect(softwareRows).toHaveLength(1); const expectedSoftware: Partial> = { "description": softwareFormData.softwareDescription, "externalId": softwareFormData.externalId, @@ -98,13 +107,25 @@ describe("RPC e2e tests", () => { "workshopUrls": [], "categories": [], "isStillInObservation": false, - "id": expectedSoftwareId + "id": expect.any(Number) }; - expectToMatchObject(inMemoryDb.softwareRows[0], { + actualSoftwareId = softwareRows[0].id; + + expectToMatchObject(softwareRows[0], { ...expectedSoftware, - "addedByAgentEmail": defaultUser.email, - "similarSoftwareExternalDataIds": softwareFormData.similarSoftwareExternalDataIds + "addedByAgentEmail": defaultUser.email + }); + const similars = await kyselyDb + .selectFrom("softwares__similar_software_external_datas") + .selectAll() + .execute(); + expect(similars).toHaveLength(softwareFormData.similarSoftwareExternalDataIds.length); + softwareFormData.similarSoftwareExternalDataIds.forEach(similarExternalId => { + expectToMatchObject(similars[0], { + softwareId: actualSoftwareId, + similarExternalId + }); }); }); @@ -124,40 +145,44 @@ describe("RPC e2e tests", () => { }); it("adds an agent as user of the software", async () => { - expect(inMemoryDb.agentRows).toHaveLength(1); - expect(inMemoryDb.softwareRows).toHaveLength(1); - expect(inMemoryDb.softwareUserRows).toHaveLength(0); + expect(await getAgentRows()).toHaveLength(1); + expect(await getSoftwareRows()).toHaveLength(1); + expect(await getSoftwareUserRows()).toHaveLength(0); + await apiCaller.createUserOrReferent({ formData: declarationFormData, - softwareName: "Some software" + softwareId: actualSoftwareId }); if (declarationFormData.declarationType !== "user") throw new Error("This test is only for user declaration"); - expect(inMemoryDb.softwareUserRows).toHaveLength(1); + const softwareUserRows = await getSoftwareUserRows(); + expect(softwareUserRows).toHaveLength(1); - expectToEqual(inMemoryDb.softwareUserRows[0], { - "agentEmail": defaultUser.email, - "softwareId": inMemoryDb.softwareRows[0].id, - "os": declarationFormData.os, - "serviceUrl": declarationFormData.serviceUrl, + expectToEqual(softwareUserRows[0], { + "agentId": expect.any(Number), + "softwareId": expect.any(Number), + "os": declarationFormData.os ?? null, + "serviceUrl": declarationFormData.serviceUrl ?? null, "useCaseDescription": declarationFormData.usecaseDescription, "version": declarationFormData.version }); }); it("adds an instance of the software", async () => { - expect(inMemoryDb.softwareRows).toHaveLength(1); - expect(inMemoryDb.instanceRows).toHaveLength(0); + instanceFormData = createInstanceFormData({ mainSoftwareSillId: actualSoftwareId }); + expect(await getSoftwareRows()).toHaveLength(1); + expect(await getInstanceRows()).toHaveLength(0); await apiCaller.createInstance({ formData: instanceFormData }); - expect(inMemoryDb.instanceRows).toHaveLength(1); - expectToMatchObject(inMemoryDb.instanceRows[0], { - "id": 1, + const instanceRows = await getInstanceRows(); + expect(instanceRows).toHaveLength(1); + expectToMatchObject(instanceRows[0], { + "id": expect.any(Number), "addedByAgentEmail": defaultUser.email, - "mainSoftwareSillId": expectedSoftwareId, + "mainSoftwareSillId": actualSoftwareId, "organization": instanceFormData.organization, "publicUrl": instanceFormData.publicUrl, "targetAudience": instanceFormData.targetAudience @@ -168,12 +193,17 @@ describe("RPC e2e tests", () => { const instances = await apiCaller.getInstances(); expect(instances).toHaveLength(1); expectToMatchObject(instances[0], { - "id": 1, - "mainSoftwareSillId": expectedSoftwareId, + "id": expect.any(Number), + "mainSoftwareSillId": actualSoftwareId, "organization": instanceFormData.organization, "publicUrl": instanceFormData.publicUrl, "targetAudience": instanceFormData.targetAudience }); }); }); + + const getSoftwareRows = async () => kyselyDb.selectFrom("softwares").selectAll().execute(); + const getAgentRows = () => kyselyDb.selectFrom("agents").selectAll().execute(); + const getSoftwareUserRows = () => kyselyDb.selectFrom("software_users").selectAll().execute(); + const getInstanceRows = () => kyselyDb.selectFrom("instances").selectAll().execute(); }); diff --git a/api/src/rpc/start.ts b/api/src/rpc/start.ts index b7c15752..204bd150 100644 --- a/api/src/rpc/start.ts +++ b/api/src/rpc/start.ts @@ -1,16 +1,28 @@ -import express from "express"; import * as trpcExpress from "@trpc/server/adapters/express"; -import cors from "cors"; -import { createValidateGitHubWebhookSignature } from "../tools/validateGithubWebhookSignature"; import compression from "compression"; -import { bootstrapCore } from "../core"; -import type { ExternalDataOrigin, LocalizedString } from "../core/ports/GetSoftwareExternalData"; -import { assert } from "tsafe/assert"; +import cors from "cors"; +import express from "express"; +import { Kysely } from "kysely"; +import { basename as pathBasename } from "path"; import type { Equals } from "tsafe"; +import { assert } from "tsafe/assert"; +import { bootstrapCore } from "../core"; +import { Database } from "../core/adapters/dbApi/kysely/kysely.database"; +import { createPgDialect } from "../core/adapters/dbApi/kysely/kysely.dialect"; +import { getHalSoftware } from "../core/adapters/hal/getHalSoftware"; +import { getHalSoftwareOptions } from "../core/adapters/hal/getHalSoftwareOptions"; +import { getWikidataSoftware } from "../core/adapters/wikidata/getWikidataSoftware"; +import { getWikidataSoftwareOptions } from "../core/adapters/wikidata/getWikidataSoftwareOptions"; +import { compiledDataPrivateToPublic } from "../core/ports/CompileData"; +import type { + ExternalDataOrigin, + GetSoftwareExternalData, + LocalizedString +} from "../core/ports/GetSoftwareExternalData"; +import type { GetSoftwareExternalDataOptions } from "../core/ports/GetSoftwareExternalDataOptions"; import { createContextFactory } from "./context"; -import type { User } from "./user"; import { createRouter } from "./router"; -import { basename as pathBasename } from "path"; +import type { User } from "./user"; export async function startRpcService(params: { keycloakParams?: { @@ -32,6 +44,7 @@ export async function startRpcService(params: { isDevEnvironnement: boolean; externalSoftwareDataOrigin: ExternalDataOrigin; redirectUrl?: string; + databaseUrl: string; }) { const { redirectUrl, @@ -47,6 +60,7 @@ export async function startRpcService(params: { githubPersonalAccessTokenForApiRateLimit, isDevEnvironnement, externalSoftwareDataOrigin, + databaseUrl, ...rest } = params; @@ -54,12 +68,16 @@ export async function startRpcService(params: { console.log({ isDevEnvironnement }); - const { core, context: coreContext } = await bootstrapCore({ + const kyselyDb = new Kysely({ dialect: createPgDialect(databaseUrl) }); + + const { + dbApi, + context: coreContext, + core + } = await bootstrapCore({ "dbConfig": { - "dbKind": "git", - dataRepoSshUrl, - "sshPrivateKeyName": sshPrivateKeyForGitName, - "sshPrivateKey": sshPrivateKeyForGit + "dbKind": "kysely", + "kyselyDb": kyselyDb }, "keycloakUserApiParams": keycloakParams === undefined @@ -90,7 +108,13 @@ export async function startRpcService(params: { } }); + const { getSoftwareExternalDataOptions, getSoftwareExternalData } = + getSoftwareExternalDataFunctions(externalSoftwareDataOrigin); + const { router } = createRouter({ + dbApi, + getSoftwareExternalDataOptions, + getSoftwareExternalData, core, coreContext, jwtClaimByUserKey, @@ -114,48 +138,13 @@ export async function startRpcService(params: { .use(compression()) .use((req, _res, next) => (console.log("⬅", req.method, req.path, req.body ?? req.query), next())) .use("/public/healthcheck", (...[, res]) => res.sendStatus(200)) - .post( - `*/ondataupdated`, - (() => { - if (githubWebhookSecret === undefined) { - return async (...[, res]) => res.sendStatus(410); - } - - const { validateGitHubWebhookSignature } = createValidateGitHubWebhookSignature({ - githubWebhookSecret - }); - - return async (req, res) => { - const reqBody = await validateGitHubWebhookSignature(req, res); - - console.log("Webhook signature OK"); - - if (redirectUrl !== undefined) { - console.log("Doing nothing with the webhook, this instance is effectively disabled"); - } - - if (reqBody.ref !== `refs/heads/main`) { - console.log(`Not a push on the main branch, doing nothing`); - res.sendStatus(200); - return; - } - - console.log("Push on main branch of data repo"); - - core.functions.readWriteSillData.notifyPushOnMainBranch({ - "commitMessage": reqBody.head_commit.message - }); - - res.sendStatus(200); - }; - })() - ) - .get(`*/sill.json`, (req, res) => { + .get(`*/sill.json`, async (req, res) => { if (redirectUrl !== undefined) { return res.redirect(redirectUrl + req.originalUrl); } - const compiledDataPublicJson = core.states.readWriteSillData.getCompiledDataPublicJson(); + const privateCompiledData = await dbApi.getCompiledDataPrivate(); + const compiledDataPublicJson = JSON.stringify(compiledDataPrivateToPublic(privateCompiledData)); res.setHeader("Content-Type", "application/json").send(Buffer.from(compiledDataPublicJson, "utf8")); }) @@ -179,3 +168,24 @@ export async function startRpcService(params: { ) .listen(port, () => console.log(`Listening on port ${port}`)); } + +function getSoftwareExternalDataFunctions(externalSoftwareDataOrigin: ExternalDataOrigin): { + "getSoftwareExternalDataOptions": GetSoftwareExternalDataOptions; + "getSoftwareExternalData": GetSoftwareExternalData; +} { + switch (externalSoftwareDataOrigin) { + case "wikidata": + return { + "getSoftwareExternalDataOptions": getWikidataSoftwareOptions, + "getSoftwareExternalData": getWikidataSoftware + }; + case "HAL": + return { + "getSoftwareExternalDataOptions": getHalSoftwareOptions, + "getSoftwareExternalData": getHalSoftware + }; + default: + const unreachableCase: never = externalSoftwareDataOrigin; + throw new Error(`Unreachable case: ${unreachableCase}`); + } +} diff --git a/api/src/tools/test.helpers.ts b/api/src/tools/test.helpers.ts index cc079f4b..8b98b748 100644 --- a/api/src/tools/test.helpers.ts +++ b/api/src/tools/test.helpers.ts @@ -2,6 +2,8 @@ import { expect } from "vitest"; import { Db } from "../core/ports/DbApi"; import { DeclarationFormData, InstanceFormData, SoftwareFormData } from "../core/usecases/readWriteSillData"; +export const testPgUrl = "postgresql://sill:pg_password@localhost:5432/sill"; + export const expectPromiseToFailWith = (promise: Promise, errorMessage: string) => { return expect(promise).rejects.toThrow(errorMessage); }; diff --git a/api/src/tools/validateGithubWebhookSignature.ts b/api/src/tools/validateGithubWebhookSignature.ts deleted file mode 100644 index a0f3d3a2..00000000 --- a/api/src/tools/validateGithubWebhookSignature.ts +++ /dev/null @@ -1,50 +0,0 @@ -import type { Request, Response } from "express"; -import { getRequestBody } from "../tools/getRequestBody"; -import * as crypto from "crypto"; -import { assert } from "tsafe/assert"; - -export type GitHubWebhookReqBody = { - ref: string; - repository: { - url: string; - }; - head_commit: { - message: string; - }; -}; - -export function createValidateGitHubWebhookSignature(params: { githubWebhookSecret: string }) { - const { githubWebhookSecret } = params; - - async function validateGitHubWebhookSignature(req: Request, res: Response): Promise { - const receivedHash = githubWebhookSecret === "NO VERIFY" ? null : req.header("X-Hub-Signature-256"); - - if (receivedHash === undefined) { - console.log("No authentication header"); - res.sendStatus(401); - await new Promise(() => { - /*never*/ - }); - assert(false); // Only to make the type checker happy - } - - const body = await getRequestBody(req); - - if (receivedHash !== null) { - const hash = "sha256=" + crypto.createHmac("sha256", githubWebhookSecret).update(body).digest("hex"); - - if (!crypto.timingSafeEqual(Buffer.from(receivedHash, "utf8"), Buffer.from(hash, "utf8"))) { - res.sendStatus(403); - await new Promise(() => { - /*never*/ - }); - } - - console.log("Webhook signature OK"); - } - - return JSON.parse(body.toString("utf8")); - } - - return { validateGitHubWebhookSignature }; -} From 848204c4493bb58c914d19dc627b70fe17b94594 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Burkard?= <22095555+JeromeBu@users.noreply.github.com> Date: Fri, 9 Aug 2024 14:28:00 +0200 Subject: [PATCH 02/19] fix agent query to get declarations at the same time --- .../dbApi/kysely/createPgAgentRepository.ts | 89 ++++++++++++++++--- .../dbApi/kysely/pgDbApi.integration.test.ts | 58 ++++++++++-- api/src/core/ports/DbApiV2.ts | 14 +-- .../core/usecases/readWriteSillData/types.ts | 2 + api/src/rpc/router.ts | 9 +- 5 files changed, 142 insertions(+), 30 deletions(-) diff --git a/api/src/core/adapters/dbApi/kysely/createPgAgentRepository.ts b/api/src/core/adapters/dbApi/kysely/createPgAgentRepository.ts index 663b5ab1..a5c32dae 100644 --- a/api/src/core/adapters/dbApi/kysely/createPgAgentRepository.ts +++ b/api/src/core/adapters/dbApi/kysely/createPgAgentRepository.ts @@ -1,6 +1,8 @@ -import type { Kysely, Selectable } from "kysely"; -import { Agent, AgentRepository } from "../../../ports/DbApiV2"; +import { Kysely, sql } from "kysely"; +import { AgentRepository } from "../../../ports/DbApiV2"; +import { Os } from "../../../usecases/readWriteSillData"; import type { Database } from "./kysely.database"; +import { jsonBuildObject, jsonStripNulls } from "./kysely.utils"; export const createPgAgentRepository = (db: Kysely): AgentRepository => ({ add: async agent => { @@ -14,19 +16,80 @@ export const createPgAgentRepository = (db: Kysely): AgentRepository = await db.deleteFrom("agents").where("id", "=", agentId).execute(); }, getByEmail: async email => { - const dbAgent = await db.selectFrom("agents").selectAll().where("email", "=", email).executeTakeFirst(); + const dbAgent = await makeGetAgentBuilder(db).where("email", "=", email).executeTakeFirst(); if (!dbAgent) return; - return toAgent(dbAgent); + + const { usersDeclarations, referentsDeclarations, ...rest } = dbAgent; + + return { + ...rest, + about: rest.about ?? undefined, + declarations: [...usersDeclarations, ...referentsDeclarations] + }; }, - getAll: async () => - db - .selectFrom("agents") - .selectAll() + getAll: () => + makeGetAgentBuilder(db) .execute() - .then(dbAgent => dbAgent.map(toAgent)) + .then(results => + results.map(({ usersDeclarations, referentsDeclarations, about, ...rest }) => ({ + ...rest, + about: about ?? undefined, + declarations: [...usersDeclarations, ...referentsDeclarations] + })) + ) }); -const toAgent = (row: Selectable): Agent => ({ - ...row, - about: row.about ?? undefined -}); +const makeGetAgentBuilder = (db: Kysely) => + db + .selectFrom("agents as a") + .leftJoin("software_users as u", "a.id", "u.agentId") + .leftJoin("softwares as us", "u.softwareId", "us.id") + .leftJoin("software_referents as r", "a.id", "r.agentId") + .leftJoin("softwares as rs", "r.softwareId", "rs.id") + .select([ + "a.id", + "a.email", + "a.isPublic", + "a.about", + "a.organization", + ({ ref, fn }) => + fn + .coalesce( + fn + .jsonAgg( + jsonStripNulls( + jsonBuildObject({ + declarationType: sql<"user">`'user'`, + serviceUrl: ref("u.serviceUrl"), + usecaseDescription: ref("u.useCaseDescription").$castTo(), + version: ref("u.version").$castTo(), + os: ref("u.os").$castTo(), + softwareName: ref("us.name").$castTo() + }) + ) + ) + .filterWhere("u.agentId", "is not", null), + sql<[]>`'[]'` + ) + .as("usersDeclarations"), + ({ ref, fn }) => + fn + .coalesce( + fn + .jsonAgg( + jsonStripNulls( + jsonBuildObject({ + declarationType: sql<"referent">`'referent'`, + isTechnicalExpert: ref("r.isExpert").$castTo(), + usecaseDescription: ref("r.useCaseDescription").$castTo(), + serviceUrl: ref("r.serviceUrl"), + softwareName: ref("rs.name").$castTo() + }) + ) + ) + .filterWhere("r.agentId", "is not", null), + sql<[]>`'[]'` + ) + .as("referentsDeclarations") + ]) + .groupBy("a.id"); diff --git a/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts b/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts index fe5923a5..0aa631b3 100644 --- a/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts +++ b/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts @@ -1,9 +1,9 @@ import { Kysely } from "kysely"; import { beforeEach, describe, expect, it, afterEach } from "vitest"; import { expectPromiseToFailWith, expectToEqual, testPgUrl } from "../../../../tools/test.helpers"; -import { Agent, DbApiV2 } from "../../../ports/DbApiV2"; +import { DbAgent, DbApiV2 } from "../../../ports/DbApiV2"; import { SoftwareExternalData } from "../../../ports/GetSoftwareExternalData"; -import { SoftwareFormData } from "../../../usecases/readWriteSillData"; +import { DeclarationFormData, SoftwareFormData } from "../../../usecases/readWriteSillData"; import { createKyselyPgDbApi } from "./createPgDbApi"; import { Database } from "./kysely.database"; import { createPgDialect } from "./kysely.dialect"; @@ -220,13 +220,59 @@ describe("pgDbApi", () => { about: "test about" }; console.log("inserting agent"); - await dbApi.agent.add(insertedAgent); + const agentId = await dbApi.agent.add(insertedAgent); + const softwareId = await dbApi.software.create({ + formData: softwareFormData, + agentEmail: insertedAgent.email, + externalDataOrigin: "wikidata" + }); + await db + .insertInto("software_users") + .values({ + agentId, + softwareId, + os: "mac", + useCaseDescription: "des trucs de user", + version: "1", + serviceUrl: "https://example.com" + }) + .execute(); + + await db + .insertInto("software_referents") + .values({ agentId, softwareId, useCaseDescription: "des trucs de référent", isExpert: true }) + .execute(); console.log("getting agent by email"); const agent = await dbApi.agent.getByEmail(insertedAgent.email); - expectToEqual(agent, { id: expect.any(Number), ...insertedAgent }); + const expectedDeclarations: (DeclarationFormData & { softwareName: string })[] = [ + { + declarationType: "user", + softwareName: softwareFormData.softwareName, + os: "mac", + version: "1", + serviceUrl: "https://example.com", + usecaseDescription: "des trucs de user" + }, + { + declarationType: "referent", + softwareName: softwareFormData.softwareName, + isTechnicalExpert: true, + usecaseDescription: "des trucs de référent", + serviceUrl: undefined + } + ]; + + expectToEqual(agent, { + id: expect.any(Number), + email: insertedAgent.email, + organization: insertedAgent.organization, + about: insertedAgent.about, + isPublic: insertedAgent.isPublic, + declarations: expectedDeclarations + }); - const updatedAgent: Agent = { + const updatedAgent: DbAgent = { id: agent!.id, organization: "updated-test-organization", about: "updated about", @@ -239,7 +285,7 @@ describe("pgDbApi", () => { console.log("getting all agents"); const allAgents = await dbApi.agent.getAll(); - expectToEqual(allAgents, [updatedAgent]); + expectToEqual(allAgents, [{ ...updatedAgent, declarations: expectedDeclarations }]); console.log("removing agent"); await dbApi.agent.remove(updatedAgent.id); diff --git a/api/src/core/ports/DbApiV2.ts b/api/src/core/ports/DbApiV2.ts index a3ff5903..551e34ab 100644 --- a/api/src/core/ports/DbApiV2.ts +++ b/api/src/core/ports/DbApiV2.ts @@ -1,5 +1,5 @@ import type { Database } from "../adapters/dbApi/kysely/kysely.database"; -import type { Instance, InstanceFormData, Software, SoftwareFormData } from "../usecases/readWriteSillData"; +import type { Agent, Instance, InstanceFormData, Software, SoftwareFormData } from "../usecases/readWriteSillData"; import type { OmitFromExisting } from "../utils"; import type { CompiledData } from "./CompileData"; @@ -33,7 +33,7 @@ export interface InstanceRepository { getAll: () => Promise; } -export type Agent = { +export type DbAgent = { id: number; email: string; organization: string; @@ -41,12 +41,14 @@ export type Agent = { isPublic: boolean; }; +type AgentWithAllDbFields = Agent & Pick; + export interface AgentRepository { - add: (agent: OmitFromExisting) => Promise; - update: (agent: Agent) => Promise; + add: (agent: OmitFromExisting) => Promise; + update: (agent: DbAgent) => Promise; remove: (agentId: number) => Promise; - getByEmail: (email: string) => Promise; - getAll: () => Promise; + getByEmail: (email: string) => Promise; + getAll: () => Promise; } export interface SoftwareReferentRepository { diff --git a/api/src/core/usecases/readWriteSillData/types.ts b/api/src/core/usecases/readWriteSillData/types.ts index d0b91bc5..f9409406 100644 --- a/api/src/core/usecases/readWriteSillData/types.ts +++ b/api/src/core/usecases/readWriteSillData/types.ts @@ -79,6 +79,8 @@ export type Agent = { email: string | undefined; organization: string; declarations: (DeclarationFormData & { softwareName: string })[]; + isPublic: boolean; + about: string | undefined; }; export type Instance = { diff --git a/api/src/rpc/router.ts b/api/src/rpc/router.ts index e91c883f..76cec493 100644 --- a/api/src/rpc/router.ts +++ b/api/src/rpc/router.ts @@ -19,6 +19,7 @@ import { } from "../core/ports/GetSoftwareExternalData"; import type { GetSoftwareExternalDataOptions } from "../core/ports/GetSoftwareExternalDataOptions"; import { + Agent, DeclarationFormData, InstanceFormData, Os, @@ -437,14 +438,12 @@ export function createRouter(params: { "email": z.string() }) ) - .query(async ({ ctx: { user }, input }) => { + .query(async ({ ctx: { user }, input }): Promise<{ agent: Agent }> => { const { email } = input; const agent = await dbApi.agent.getByEmail(email); - - if (!agent?.isPublic && user === undefined) { - throw new TRPCError({ "code": "UNAUTHORIZED" }); - } + if (agent === undefined) throw new TRPCError({ "code": "NOT_FOUND", message: "Agent not found" }); + if (!agent?.isPublic && user === undefined) throw new TRPCError({ "code": "UNAUTHORIZED" }); return { agent }; }), From 51d3e68f7de81374989b928b53a0a6aa3d102a9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Burkard?= <22095555+JeromeBu@users.noreply.github.com> Date: Fri, 9 Aug 2024 14:59:13 +0200 Subject: [PATCH 03/19] fix frontend types --- api/src/core/ports/DbApiV2.ts | 2 +- .../core/usecases/readWriteSillData/types.ts | 2 +- api/src/rpc/router.ts | 4 ++-- web/src/core/adapter/sillApiMock.ts | 21 ++++++++++++++----- .../core/usecases/declarationForm/state.ts | 1 + .../core/usecases/declarationForm/thunks.ts | 3 ++- .../usecases/declarationRemoval/thunks.ts | 6 +++--- .../core/usecases/softwareDetails/thunks.ts | 2 +- .../pages/softwareDetails/SoftwareDetails.tsx | 3 ++- web/src/ui/shared/DeclarationRemovalModal.tsx | 9 ++++++-- 10 files changed, 36 insertions(+), 17 deletions(-) diff --git a/api/src/core/ports/DbApiV2.ts b/api/src/core/ports/DbApiV2.ts index 551e34ab..4a5a6af7 100644 --- a/api/src/core/ports/DbApiV2.ts +++ b/api/src/core/ports/DbApiV2.ts @@ -41,7 +41,7 @@ export type DbAgent = { isPublic: boolean; }; -type AgentWithAllDbFields = Agent & Pick; +type AgentWithAllDbFields = Agent & Pick; export interface AgentRepository { add: (agent: OmitFromExisting) => Promise; diff --git a/api/src/core/usecases/readWriteSillData/types.ts b/api/src/core/usecases/readWriteSillData/types.ts index f9409406..0fa8b8df 100644 --- a/api/src/core/usecases/readWriteSillData/types.ts +++ b/api/src/core/usecases/readWriteSillData/types.ts @@ -76,7 +76,7 @@ export namespace Software { export type Agent = { //NOTE: Undefined if the agent isn't referent of at least one software // If it's the user the email is never undefined. - email: string | undefined; + email: string; organization: string; declarations: (DeclarationFormData & { softwareName: string })[]; isPublic: boolean; diff --git a/api/src/rpc/router.ts b/api/src/rpc/router.ts index 76cec493..80b1598f 100644 --- a/api/src/rpc/router.ts +++ b/api/src/rpc/router.ts @@ -489,8 +489,8 @@ export function createRouter(params: { await dbApi.agent.update({ ...agent, email: newEmail }); }), "getRegisteredUserCount": loggedProcedure.query(async () => coreContext.userApi.getUserCount()), - "getTotalReferentCount": loggedProcedure.query(() => { - const referentCount = dbApi.softwareReferent.getTotalCount(); + "getTotalReferentCount": loggedProcedure.query(async () => { + const referentCount = await dbApi.softwareReferent.getTotalCount(); return { referentCount }; }), "getTermsOfServiceUrl": loggedProcedure.query(() => termsOfServiceUrl), diff --git a/web/src/core/adapter/sillApiMock.ts b/web/src/core/adapter/sillApiMock.ts index 19709f71..5d9b2f3a 100644 --- a/web/src/core/adapter/sillApiMock.ts +++ b/web/src/core/adapter/sillApiMock.ts @@ -175,10 +175,10 @@ export const sillApi: SillApi = { "createUserOrReferent": async ({ formData }) => { console.log(`User or referent updated ${JSON.stringify(formData, null, 2)}`); }, - "removeUserOrReferent": async ({ declarationType, softwareName }) => { + "removeUserOrReferent": async ({ declarationType, softwareId }) => { console.log( `removed user or referent ${JSON.stringify( - { declarationType, softwareName }, + { declarationType, softwareId }, null, 2 )}` @@ -193,9 +193,14 @@ export const sillApi: SillApi = { "updateInstance": async params => { console.log(`Updating instance ${JSON.stringify(params)}`); }, - "getAgents": memoize(async () => ({ "agents": id([...agents]) }), { - "promise": true - }), + "getAgents": memoize( + async () => ({ + "agents": agents.map((agent, index) => ({ id: index, ...agent })) + }), + { + "promise": true + } + ), "changeAgentOrganization": async ({ newOrganization }) => { console.log(`Update organization -> ${newOrganization}`); }, @@ -1244,6 +1249,8 @@ const agents: ApiTypes.Agent[] = [ { "organization": "Développement durable", "email": "agent1@codegouv.fr", + "isPublic": true, + "about": undefined, "declarations": [ { "serviceUrl": "", @@ -1258,6 +1265,8 @@ const agents: ApiTypes.Agent[] = [ { "organization": "Babel", "email": "agent2@codegouv.fr", + "isPublic": true, + "about": undefined, "declarations": [ { "serviceUrl": "", @@ -1271,6 +1280,8 @@ const agents: ApiTypes.Agent[] = [ { "organization": "Éducation nationale", "email": "agent3@codegouv.fr", + "isPublic": true, + "about": undefined, "declarations": [ { "serviceUrl": "", diff --git a/web/src/core/usecases/declarationForm/state.ts b/web/src/core/usecases/declarationForm/state.ts index 7ae58a91..84618176 100644 --- a/web/src/core/usecases/declarationForm/state.ts +++ b/web/src/core/usecases/declarationForm/state.ts @@ -18,6 +18,7 @@ export namespace State { isSubmitting: boolean; software: { logoUrl: string | undefined; + softwareId: number; softwareName: string; referentCount: number; userCount: number; diff --git a/web/src/core/usecases/declarationForm/thunks.ts b/web/src/core/usecases/declarationForm/thunks.ts index 7e83a473..28d47761 100644 --- a/web/src/core/usecases/declarationForm/thunks.ts +++ b/web/src/core/usecases/declarationForm/thunks.ts @@ -36,6 +36,7 @@ export const thunks = { "software": { "logoUrl": software.logoUrl, softwareName, + softwareId: software.softwareId, "referentCount": Object.values( software.userAndReferentCountByOrganization ) @@ -143,7 +144,7 @@ export const thunks = { await sillApi.createUserOrReferent({ formData, - "softwareName": state.software.softwareName + "softwareId": state.software.softwareId }); dispatch(actions.triggerRedirect({ "isFormSubmitted": true })); diff --git a/web/src/core/usecases/declarationRemoval/thunks.ts b/web/src/core/usecases/declarationRemoval/thunks.ts index 66ad32fb..18daef7e 100644 --- a/web/src/core/usecases/declarationRemoval/thunks.ts +++ b/web/src/core/usecases/declarationRemoval/thunks.ts @@ -3,9 +3,9 @@ import { actions } from "./state"; export const thunks = { "removeAgentAsReferentOrUserFromSoftware": - (params: { softwareName: string; declarationType: "user" | "referent" }) => + (params: { softwareId: number; declarationType: "user" | "referent" }) => async (...args) => { - const { declarationType, softwareName } = params; + const { declarationType, softwareId } = params; const [dispatch, , { sillApi }] = args; @@ -13,7 +13,7 @@ export const thunks = { await sillApi.removeUserOrReferent({ declarationType, - softwareName + softwareId }); dispatch(actions.userOrReferentRemoved()); diff --git a/web/src/core/usecases/softwareDetails/thunks.ts b/web/src/core/usecases/softwareDetails/thunks.ts index b800b59b..2fd7de87 100644 --- a/web/src/core/usecases/softwareDetails/thunks.ts +++ b/web/src/core/usecases/softwareDetails/thunks.ts @@ -144,7 +144,7 @@ export const thunks = { const time = Date.now(); await sillApi.unreferenceSoftware({ - "softwareName": state.software.softwareName, + "softwareId": state.software.softwareId, reason }); diff --git a/web/src/ui/pages/softwareDetails/SoftwareDetails.tsx b/web/src/ui/pages/softwareDetails/SoftwareDetails.tsx index 0be653c4..3950bd52 100644 --- a/web/src/ui/pages/softwareDetails/SoftwareDetails.tsx +++ b/web/src/ui/pages/softwareDetails/SoftwareDetails.tsx @@ -315,7 +315,8 @@ export default function SoftwareDetails(props: Props) { onClick={() => openDeclarationRemovalModal({ declarationType, - "softwareName": software.softwareName + "softwareName": software.softwareName, + "softwareId": software.softwareId }) } > diff --git a/web/src/ui/shared/DeclarationRemovalModal.tsx b/web/src/ui/shared/DeclarationRemovalModal.tsx index e9171599..d561d87a 100644 --- a/web/src/ui/shared/DeclarationRemovalModal.tsx +++ b/web/src/ui/shared/DeclarationRemovalModal.tsx @@ -13,6 +13,7 @@ const modal = createModal({ }); type Params = { + softwareId: number; softwareName: string; declarationType: "user" | "referent"; }; @@ -47,7 +48,11 @@ export function DeclarationRemovalModal() { const params = evtParams.state; - const { softwareName = "", declarationType = "referent" } = params ?? {}; + const { + softwareName = "", + softwareId = 0, + declarationType = "referent" + } = params ?? {}; return ( declarationRemoval.removeAgentAsReferentOrUserFromSoftware({ - softwareName, + softwareId, declarationType }), "nativeButtonProps": { From 4e177f72a62673fae6c00b9e7e1738c67bf96166 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Burkard?= <22095555+JeromeBu@users.noreply.github.com> Date: Fri, 9 Aug 2024 17:38:10 +0200 Subject: [PATCH 04/19] fix parentExternalId query --- .../dbApi/kysely/createGetCompiledData.ts | 4 +- .../kysely/createPgSoftwareRepository.ts | 93 ++++++++++--------- .../adapters/dbApi/kysely/kysely.utils.ts | 4 +- .../dbApi/kysely/pgDbApi.integration.test.ts | 31 +++++-- api/src/rpc/routes.e2e.test.ts | 17 ++++ .../core/usecases/softwareCatalog/thunks.ts | 8 ++ .../core/usecases/softwareDetails/thunks.ts | 6 ++ 7 files changed, 107 insertions(+), 56 deletions(-) diff --git a/api/src/core/adapters/dbApi/kysely/createGetCompiledData.ts b/api/src/core/adapters/dbApi/kysely/createGetCompiledData.ts index 19d1f1fd..41c2549a 100644 --- a/api/src/core/adapters/dbApi/kysely/createGetCompiledData.ts +++ b/api/src/core/adapters/dbApi/kysely/createGetCompiledData.ts @@ -3,7 +3,7 @@ import { CompiledData } from "../../../ports/CompileData"; import { Db } from "../../../ports/DbApi"; import { ParentSoftwareExternalData, SoftwareExternalData } from "../../../ports/GetSoftwareExternalData"; import { Database } from "./kysely.database"; -import { convertNullValuesToUndefined, isNotNull, jsonBuildObject, jsonStripNulls } from "./kysely.utils"; +import { stripNullOrUndefinedValues, isNotNull, jsonBuildObject, jsonStripNulls } from "./kysely.utils"; export const createGetCompiledData = (db: Kysely) => async (): Promise> => { console.time("agentById query"); @@ -138,7 +138,7 @@ export const createGetCompiledData = (db: Kysely) => async (): Promise ...software }): CompiledData.Software<"private"> => { return { - ...convertNullValuesToUndefined(software), + ...stripNullOrUndefinedValues(software), updateTime: new Date(+updateTime).getTime(), referencedSinceTime: new Date(+referencedSinceTime).getTime(), doRespectRgaa, diff --git a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts index f50d4ba2..bc4f54a5 100644 --- a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts +++ b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts @@ -5,7 +5,7 @@ import { SoftwareRepository } from "../../../ports/DbApiV2"; import { ParentSoftwareExternalData } from "../../../ports/GetSoftwareExternalData"; import { Software } from "../../../usecases/readWriteSillData"; import { Database } from "./kysely.database"; -import { convertNullValuesToUndefined, jsonBuildObject } from "./kysely.utils"; +import { stripNullOrUndefinedValues, jsonBuildObject } from "./kysely.utils"; export const createPgSoftwareRepository = (db: Kysely): SoftwareRepository => ({ create: async ({ formData, externalDataOrigin, agentEmail }) => { @@ -136,8 +136,8 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi similarExternalSoftwares, ...software } = result; - return { - ...convertNullValuesToUndefined(software), + return stripNullOrUndefinedValues({ + ...software, updateTime: new Date(+updateTime).getTime(), addedTime: new Date(+addedTime).getTime(), serviceProviders: serviceProviders ?? [], @@ -149,17 +149,15 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi })), officialWebsiteUrl: softwareExternalData?.websiteUrl ?? - software.comptoirDuLibreSoftware?.external_resources.website ?? - undefined, + software.comptoirDuLibreSoftware?.external_resources.website, codeRepositoryUrl: softwareExternalData?.sourceUrl ?? - software.comptoirDuLibreSoftware?.external_resources.repository ?? - undefined, - documentationUrl: softwareExternalData?.documentationUrl ?? undefined, + software.comptoirDuLibreSoftware?.external_resources.repository, + documentationUrl: softwareExternalData?.documentationUrl, comptoirDuLibreServiceProviderCount: software.comptoirDuLibreSoftware?.providers.length ?? 0, testUrl: testUrls[0]?.url, - parentWikidataSoftware: parentExternalData ?? undefined - }; + parentWikidataSoftware: parentExternalData + }); }); }, getAll: (): Promise => @@ -176,39 +174,41 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi softwareExternalData, similarExternalSoftwares, ...software - }): Software => ({ - ...convertNullValuesToUndefined(software), - updateTime: new Date(+updateTime).getTime(), - addedTime: new Date(+addedTime).getTime(), - serviceProviders: serviceProviders ?? [], - similarSoftwares: similarExternalSoftwares, - // (similarSoftwares ?? []).map( - // (s): SimilarSoftware => ({ - // softwareName: - // typeof s.label === "string" ? s.label : Object.values(s.label)[0]!, - // softwareDescription: - // typeof s.label === "string" ? s.label : Object.values(s.label)[0]!, - // isInSill: true // TODO: check if this is true - // }) - // ) ?? [], - userAndReferentCountByOrganization: {}, - authors: (softwareExternalData?.developers ?? []).map(dev => ({ - authorName: dev.name, - authorUrl: `https://www.wikidata.org/wiki/${dev.id}` - })), - officialWebsiteUrl: - softwareExternalData?.websiteUrl ?? - software.comptoirDuLibreSoftware?.external_resources.website ?? - undefined, - codeRepositoryUrl: - softwareExternalData?.sourceUrl ?? - software.comptoirDuLibreSoftware?.external_resources.repository ?? - undefined, - documentationUrl: softwareExternalData?.documentationUrl ?? undefined, - comptoirDuLibreServiceProviderCount: software.comptoirDuLibreSoftware?.providers.length ?? 0, - testUrl: testUrls[0]?.url, - parentWikidataSoftware: parentExternalData ?? undefined - }) + }): Software => + stripNullOrUndefinedValues({ + ...software, + updateTime: new Date(+updateTime).getTime(), + addedTime: new Date(+addedTime).getTime(), + serviceProviders: serviceProviders ?? [], + similarSoftwares: similarExternalSoftwares, + // (similarSoftwares ?? []).map( + // (s): SimilarSoftware => ({ + // softwareName: + // typeof s.label === "string" ? s.label : Object.values(s.label)[0]!, + // softwareDescription: + // typeof s.label === "string" ? s.label : Object.values(s.label)[0]!, + // isInSill: true // TODO: check if this is true + // }) + // ) ?? [], + userAndReferentCountByOrganization: {}, + authors: (softwareExternalData?.developers ?? []).map(dev => ({ + authorName: dev.name, + authorUrl: `https://www.wikidata.org/wiki/${dev.id}` + })), + officialWebsiteUrl: + softwareExternalData?.websiteUrl ?? + software.comptoirDuLibreSoftware?.external_resources.website ?? + undefined, + codeRepositoryUrl: + softwareExternalData?.sourceUrl ?? + software.comptoirDuLibreSoftware?.external_resources.repository ?? + undefined, + documentationUrl: softwareExternalData?.documentationUrl ?? undefined, + comptoirDuLibreServiceProviderCount: + software.comptoirDuLibreSoftware?.providers.length ?? 0, + testUrl: testUrls[0]?.url, + parentWikidataSoftware: parentExternalData ?? undefined + }) ) ), getAllSillSoftwareExternalIds: async externalDataOrigin => @@ -273,6 +273,7 @@ const makeGetSoftwareBuilder = (db: Kysely) => "ext.externalId", "parentExt.externalId" ]) + .orderBy("s.id", "asc") .select([ "s.id as softwareId", "s.logoUrl", @@ -305,9 +306,9 @@ const makeGetSoftwareBuilder = (db: Kysely) => .when("parentExt.externalId", "is not", null) .then( jsonBuildObject({ - externalId: ref("ext.externalId"), - label: ref("ext.label"), - description: ref("ext.description") + externalId: ref("parentExt.externalId"), + label: ref("parentExt.label"), + description: ref("parentExt.description") }).$castTo() ) .else(null) diff --git a/api/src/core/adapters/dbApi/kysely/kysely.utils.ts b/api/src/core/adapters/dbApi/kysely/kysely.utils.ts index a7f5f4c1..daa9eb79 100644 --- a/api/src/core/adapters/dbApi/kysely/kysely.utils.ts +++ b/api/src/core/adapters/dbApi/kysely/kysely.utils.ts @@ -28,7 +28,7 @@ export const castSql = ( export const isNotNull = (value: T | null): value is T => value !== null; -export const convertNullValuesToUndefined = >( +export const stripNullOrUndefinedValues = >( obj: T ): { [K in keyof T]: null extends T[K] ? Exclude | undefined : T[K] } => - Object.fromEntries(Object.entries(obj).map(([key, value]) => [key, value === null ? undefined : value])) as any; + Object.fromEntries(Object.entries(obj).filter(([_, value]) => value !== null && value !== undefined)) as any; diff --git a/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts b/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts index 0aa631b3..0fbc9692 100644 --- a/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts +++ b/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts @@ -70,6 +70,13 @@ const similarSoftwareExternalData: SoftwareExternalData = { license: "MIT" }; +const parentSoftwareExternalData: SoftwareExternalData = { + ...similarSoftwareExternalData, + externalId: "Q-parent-external-id", + label: "Some parent software label", + description: { en: "Some parent software description" } +}; + const db = new Kysely({ dialect: createPgDialect(testPgUrl) }); describe("pgDbApi", () => { @@ -112,13 +119,21 @@ describe("pgDbApi", () => { }); describe("software", () => { - it("creates a software, than gets it with getAll", async () => { + it("creates a software, than gets it with getAll, than updates adding a parent", async () => { console.log("------ software scenario ------"); - await insertSoftwareExternalDataAndSoftware(); + const softwareId = await insertSoftwareExternalDataAndSoftware(); + + await db + .updateTable("softwares") + .set("parentSoftwareWikidataId", parentSoftwareExternalData.externalId) + .where("id", "=", softwareId) + .execute(); const softwares = await dbApi.software.getAll(); - expectToEqual(softwares[0], { + const actualSoftware = softwares[0]; + + expectToEqual(actualSoftware, { addedTime: expect.any(Number), updateTime: expect.any(Number), annuaireCnllServiceProviders: undefined, @@ -139,7 +154,11 @@ describe("pgDbApi", () => { license: "MIT", logoUrl: "https://example.com/logo.png", officialWebsiteUrl: softwareExternalData.websiteUrl, - parentWikidataSoftware: undefined, + parentWikidataSoftware: { + label: parentSoftwareExternalData.label, + description: parentSoftwareExternalData.description, + externalId: parentSoftwareExternalData.externalId + }, prerogatives: { doRespectRgaa: true, isFromFrenchPublicServices: false, @@ -384,7 +403,7 @@ describe("pgDbApi", () => { await db .insertInto("software_external_datas") .values( - [softwareExternalData, similarSoftwareExternalData].map(softExtData => ({ + [softwareExternalData, similarSoftwareExternalData, parentSoftwareExternalData].map(softExtData => ({ ...softExtData, developers: JSON.stringify(softExtData.developers), label: JSON.stringify(softExtData.label), @@ -393,7 +412,7 @@ describe("pgDbApi", () => { ) .execute(); - await dbApi.software.create({ + return dbApi.software.create({ formData: softwareFormData, agentEmail: agent.email, externalDataOrigin: "wikidata" diff --git a/api/src/rpc/routes.e2e.test.ts b/api/src/rpc/routes.e2e.test.ts index 75a7dbeb..9c78c2b4 100644 --- a/api/src/rpc/routes.e2e.test.ts +++ b/api/src/rpc/routes.e2e.test.ts @@ -1,6 +1,7 @@ import { Kysely } from "kysely"; import { beforeAll, describe, expect, it } from "vitest"; import { Database } from "../core/adapters/dbApi/kysely/kysely.database"; +import { stripNullOrUndefinedValues } from "../core/adapters/dbApi/kysely/kysely.utils"; import { CompiledData } from "../core/ports/CompileData"; import { InstanceFormData } from "../core/usecases/readWriteSillData"; import { @@ -15,6 +16,22 @@ import { ApiCaller, createTestCaller, defaultUser } from "./createTestCaller"; const softwareFormData = createSoftwareFormData(); const declarationFormData = createDeclarationFormData(); +describe("stripNullOrUndefined", () => { + it("removes null and undefined values", () => { + const stripped = stripNullOrUndefinedValues({ + "a": null, + "b": undefined, + "c": 0, + "d": 1, + "e": "", + "f": "yolo" + }); + expect(stripped.hasOwnProperty("a")).toBe(false); + expect(stripped.hasOwnProperty("b")).toBe(false); + expect(stripped).toStrictEqual({ "c": 0, "d": 1, "e": "", "f": "yolo" }); + }); +}); + describe("RPC e2e tests", () => { let apiCaller: ApiCaller; let kyselyDb: Kysely; diff --git a/web/src/core/usecases/softwareCatalog/thunks.ts b/web/src/core/usecases/softwareCatalog/thunks.ts index a7bdf756..75e4a3b7 100644 --- a/web/src/core/usecases/softwareCatalog/thunks.ts +++ b/web/src/core/usecases/softwareCatalog/thunks.ts @@ -258,6 +258,14 @@ function apiSoftwareToInternalSoftware(params: { }; } + console.log( + "resolving localized string in SOFTWARE CATALOG : ", + parentWikidataSoftware.label, + ` ( for software ${softwareName})` + ); + + console.log(parentWikidataSoftware); + return { "isInSill": false, "softwareName": resolveLocalizedString(parentWikidataSoftware.label), diff --git a/web/src/core/usecases/softwareDetails/thunks.ts b/web/src/core/usecases/softwareDetails/thunks.ts index 2fd7de87..e23cb3a2 100644 --- a/web/src/core/usecases/softwareDetails/thunks.ts +++ b/web/src/core/usecases/softwareDetails/thunks.ts @@ -220,6 +220,12 @@ function apiSoftwareToSoftware(params: { }; } + console.log( + "resolving localized string in SOFTWARE DETAILS : ", + parentWikidataSoftware_api.label, + ` ( for software ${softwareId} - ${softwareName})` + ); + return { "isInSill": false, "softwareName": resolveLocalizedString(parentWikidataSoftware_api.label), From d5c1d234d81df50512b1ede4cb4d7cd806ec7685 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Burkard?= <22095555+JeromeBu@users.noreply.github.com> Date: Fri, 9 Aug 2024 22:42:30 +0200 Subject: [PATCH 05/19] fix userAndReferentCountByOrganization in get softwares --- .../kysely/createPgSoftwareRepository.ts | 79 ++++++++++++++++--- .../dbApi/kysely/pgDbApi.integration.test.ts | 53 ++++++++----- .../core/usecases/softwareCatalog/thunks.ts | 10 +++ 3 files changed, 114 insertions(+), 28 deletions(-) diff --git a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts index bc4f54a5..ae322bbd 100644 --- a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts +++ b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts @@ -163,8 +163,9 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi getAll: (): Promise => makeGetSoftwareBuilder(db) .execute() - .then(softwares => - softwares.map( + .then(async softwares => { + const userAndReferentCountByOrganization = await getUserAndReferentCountByOrganizationBySoftwareId(db); + return softwares.map( ({ testUrls, serviceProviders, @@ -174,8 +175,8 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi softwareExternalData, similarExternalSoftwares, ...software - }): Software => - stripNullOrUndefinedValues({ + }): Software => { + return stripNullOrUndefinedValues({ ...software, updateTime: new Date(+updateTime).getTime(), addedTime: new Date(+addedTime).getTime(), @@ -190,7 +191,8 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi // isInSill: true // TODO: check if this is true // }) // ) ?? [], - userAndReferentCountByOrganization: {}, + userAndReferentCountByOrganization: + userAndReferentCountByOrganization[software.softwareId] ?? {}, authors: (softwareExternalData?.developers ?? []).map(dev => ({ authorName: dev.name, authorUrl: `https://www.wikidata.org/wiki/${dev.id}` @@ -208,9 +210,10 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi software.comptoirDuLibreSoftware?.providers.length ?? 0, testUrl: testUrls[0]?.url, parentWikidataSoftware: parentExternalData ?? undefined - }) - ) - ), + }); + } + ); + }), getAllSillSoftwareExternalIds: async externalDataOrigin => db .selectFrom("softwares") @@ -315,7 +318,6 @@ const makeGetSoftwareBuilder = (db: Kysely) => .end() .as("parentExternalData"), "s.keywords", - ({ ref }) => jsonBuildObject({ externalId: ref("ext.externalId"), @@ -349,3 +351,62 @@ const makeGetSoftwareBuilder = (db: Kysely) => ) .as("similarExternalSoftwares") ]); + +type CountForOrganisationAndSoftwareId = { + organization: string; + softwareId: number; + type: "user" | "referent"; + count: string; +}; + +type UserAndReferentCountByOrganizationBySoftwareId = Record< + string, + Record +>; + +const defaultCount = { + userCount: 0, + referentCount: 0 +}; + +const getUserAndReferentCountByOrganizationBySoftwareId = async ( + db: Kysely +): Promise => { + const softwareUserCountBySoftwareId: CountForOrganisationAndSoftwareId[] = await db + .selectFrom("software_users as u") + .innerJoin("agents as a", "a.id", "u.agentId") + .select([ + "u.softwareId", + "a.organization", + ({ fn }) => fn.countAll().as("count"), + sql<"user">`'userCount'`.as("type") + ]) + .groupBy(["a.organization", "u.softwareId"]) + .execute(); + + const softwareReferentCountBySoftwareId: CountForOrganisationAndSoftwareId[] = await db + .selectFrom("software_referents as r") + .innerJoin("agents as a", "a.id", "r.agentId") + .select([ + "r.softwareId", + "a.organization", + ({ fn }) => fn.countAll().as("count"), + sql<"referent">`'referentCount'`.as("type") + ]) + .groupBy(["a.organization", "r.softwareId"]) + .execute(); + + return [...softwareReferentCountBySoftwareId, ...softwareUserCountBySoftwareId].reduce( + (acc, { organization, softwareId, type, count }): UserAndReferentCountByOrganizationBySoftwareId => ({ + ...acc, + [softwareId]: { + ...(acc[softwareId] ?? {}), + [organization]: { + ...(acc[softwareId]?.[organization] ?? defaultCount), + [type]: +count + } + } + }), + {} as UserAndReferentCountByOrganizationBySoftwareId + ); +}; diff --git a/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts b/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts index 0fbc9692..26e7e3ea 100644 --- a/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts +++ b/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts @@ -77,6 +77,13 @@ const parentSoftwareExternalData: SoftwareExternalData = { description: { en: "Some parent software description" } }; +const insertedAgent = { + email: "test@test.com", + organization: "test-organization", + isPublic: true, + about: "test about" +}; + const db = new Kysely({ dialect: createPgDialect(testPgUrl) }); describe("pgDbApi", () => { @@ -121,7 +128,7 @@ describe("pgDbApi", () => { describe("software", () => { it("creates a software, than gets it with getAll, than updates adding a parent", async () => { console.log("------ software scenario ------"); - const softwareId = await insertSoftwareExternalDataAndSoftware(); + const { softwareId, agentId } = await insertSoftwareExternalDataAndSoftwareAndAgent(); await db .updateTable("softwares") @@ -129,6 +136,15 @@ describe("pgDbApi", () => { .where("id", "=", softwareId) .execute(); + await dbApi.softwareUser.add({ + agentId, + softwareId, + useCaseDescription: "des trucs de user", + os: "windows", + version: "1.0.0", + serviceUrl: "https://example.com" + }); + const softwares = await dbApi.software.getAll(); const actualSoftware = softwares[0]; @@ -189,7 +205,12 @@ describe("pgDbApi", () => { type: "desktop/mobile" }, testUrl: undefined, - userAndReferentCountByOrganization: {}, + userAndReferentCountByOrganization: { + [insertedAgent.organization]: { + userCount: 1, + referentCount: 0 + } + }, versionMin: "" }); @@ -202,7 +223,7 @@ describe("pgDbApi", () => { describe("instance", () => { it("creates an instance, than gets it with getAll", async () => { console.log("------ instance scenario ------"); - await insertSoftwareExternalDataAndSoftware(); + await insertSoftwareExternalDataAndSoftwareAndAgent(); const softwares = await dbApi.software.getAll(); const softwareId = softwares[0].softwareId; console.log("saving instance"); @@ -232,12 +253,6 @@ describe("pgDbApi", () => { describe("agents", () => { it("adds an agent, get it by email, updates it, getAll", async () => { console.log("------ agent scenario------"); - const insertedAgent = { - email: "test@test.com", - organization: "test-organization", - isPublic: true, - about: "test about" - }; console.log("inserting agent"); const agentId = await dbApi.agent.add(insertedAgent); const softwareId = await dbApi.software.create({ @@ -320,14 +335,7 @@ describe("pgDbApi", () => { let agentId: number; beforeEach(async () => { console.log("before -- setting up test with software and agent"); - await insertSoftwareExternalDataAndSoftware(); - - await dbApi.agent.add({ - email: "test@test.com", - organization: "test-organization", - isPublic: true, - about: "test about" - }); + await insertSoftwareExternalDataAndSoftwareAndAgent(); softwareId = (await dbApi.software.getAll())[0].softwareId; agentId = (await dbApi.agent.getAll())[0].id; @@ -399,7 +407,7 @@ describe("pgDbApi", () => { }); }); - const insertSoftwareExternalDataAndSoftware = async () => { + const insertSoftwareExternalDataAndSoftwareAndAgent = async () => { await db .insertInto("software_external_datas") .values( @@ -412,10 +420,17 @@ describe("pgDbApi", () => { ) .execute(); - return dbApi.software.create({ + const agentId = await dbApi.agent.add(insertedAgent); + + const softwareId = await dbApi.software.create({ formData: softwareFormData, agentEmail: agent.email, externalDataOrigin: "wikidata" }); + + return { + softwareId, + agentId + }; }; }); diff --git a/web/src/core/usecases/softwareCatalog/thunks.ts b/web/src/core/usecases/softwareCatalog/thunks.ts index 75e4a3b7..f98045fc 100644 --- a/web/src/core/usecases/softwareCatalog/thunks.ts +++ b/web/src/core/usecases/softwareCatalog/thunks.ts @@ -273,6 +273,16 @@ function apiSoftwareToInternalSoftware(params: { }; })(); + console.log({ + userAndReferentCountByOrganization, + "referentCount": Object.values(userAndReferentCountByOrganization) + .map(({ referentCount }) => referentCount) + .reduce((prev, curr) => prev + curr, 0), + "userCount": Object.values(userAndReferentCountByOrganization) + .map(({ userCount }) => userCount) + .reduce((prev, curr) => prev + curr, 0) + }); + return { logoUrl, softwareName, From bb3ce1efcd7bc74a1292a4847124880d2fc0fab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Burkard?= <22095555+JeromeBu@users.noreply.github.com> Date: Sat, 10 Aug 2024 15:45:54 +0200 Subject: [PATCH 06/19] fix sequences of ids when loading data in pg db from git repository --- api/scripts/load-git-repo-in-pg.ts | 5 ++++- .../kysely/createPgSoftwareRepository.ts | 20 +++++++++++++++---- .../dbApi/kysely/pgDbApi.integration.test.ts | 9 ++------- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/api/scripts/load-git-repo-in-pg.ts b/api/scripts/load-git-repo-in-pg.ts index 4f19cc0b..69b1ecdb 100644 --- a/api/scripts/load-git-repo-in-pg.ts +++ b/api/scripts/load-git-repo-in-pg.ts @@ -1,4 +1,4 @@ -import { InsertObject, Kysely } from "kysely"; +import { InsertObject, Kysely, sql } from "kysely"; import { z } from "zod"; import { createGitDbApi, GitDbApiParams } from "../src/core/adapters/dbApi/createGitDbApi"; import { Database } from "../src/core/adapters/dbApi/kysely/kysely.database"; @@ -63,6 +63,7 @@ const insertSoftwares = async (softwareRows: SoftwareRow[], db: Kysely })) ) .executeTakeFirst(); + await sql`SELECT setval('softwares_id_seq', (SELECT MAX(id) FROM softwares))`.execute(trx); await trx .insertInto("softwares__similar_software_external_datas") @@ -84,6 +85,7 @@ const insertAgents = async (agentRows: Db.AgentRow[], db: Kysely) => { await db.transaction().execute(async trx => { await trx.deleteFrom("agents").execute(); await trx.insertInto("agents").values(agentRows).executeTakeFirst(); + await sql`SELECT setval('agents_id_seq', (SELECT MAX(id) FROM agents))`.execute(trx); }); }; @@ -149,6 +151,7 @@ const insertInstances = async ({ instanceRows, db }: { instanceRows: Db.Instance await db.transaction().execute(async trx => { await trx.deleteFrom("instances").execute(); await trx.insertInto("instances").values(instanceRows).executeTakeFirst(); + await sql`SELECT setval('instances_id_seq', (SELECT MAX(id) FROM instances))`.execute(trx); }); }; diff --git a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts index ae322bbd..88806614 100644 --- a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts +++ b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts @@ -62,10 +62,21 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi .returning("id as softwareId") .executeTakeFirstOrThrow(); - await trx - .insertInto("softwares__similar_software_external_datas") - .values(similarSoftwareExternalDataIds.map(similarExternalId => ({ softwareId, similarExternalId }))) - .execute(); + console.log( + `inserted software correctly, softwareId is : ${softwareId}, about to insert similars : `, + similarSoftwareExternalDataIds + ); + + if (similarSoftwareExternalDataIds.length > 0) { + await trx + .insertInto("softwares__similar_software_external_datas") + .values( + similarSoftwareExternalDataIds.map(similarExternalId => ({ softwareId, similarExternalId })) + ) + .execute(); + } + + console.log("all good"); return softwareId; }); @@ -221,6 +232,7 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi .where("externalDataOrigin", "=", externalDataOrigin) .execute() .then(rows => rows.map(row => row.externalId!)), + countAddedByAgent: async ({ agentEmail }) => { const { count } = await db .selectFrom("softwares") diff --git a/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts b/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts index 26e7e3ea..f5cbf0e7 100644 --- a/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts +++ b/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts @@ -8,11 +8,6 @@ import { createKyselyPgDbApi } from "./createPgDbApi"; import { Database } from "./kysely.database"; import { createPgDialect } from "./kysely.dialect"; -const agent = { - id: 1, - email: "test@test.com", - organization: "test-orga" -}; const externalId = "external-id-111"; const similarExternalId = "external-id-222"; const softwareFormData: SoftwareFormData = { @@ -228,7 +223,7 @@ describe("pgDbApi", () => { const softwareId = softwares[0].softwareId; console.log("saving instance"); await dbApi.instance.create({ - agentEmail: agent.email, + agentEmail: insertedAgent.email, formData: { mainSoftwareSillId: softwareId, organization: "test-orga", @@ -424,7 +419,7 @@ describe("pgDbApi", () => { const softwareId = await dbApi.software.create({ formData: softwareFormData, - agentEmail: agent.email, + agentEmail: insertedAgent.email, externalDataOrigin: "wikidata" }); From d11456209aee5d00c479f23cc871d4b82db69dca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Burkard?= <22095555+JeromeBu@users.noreply.github.com> Date: Sat, 10 Aug 2024 16:09:31 +0200 Subject: [PATCH 07/19] handle case when trying to add a software with a name that already exists --- .../kysely/createPgSoftwareRepository.ts | 47 +++++++++++++++++-- .../1717162141365_create-initial-tables.ts | 2 +- api/src/core/ports/DbApiV2.ts | 1 + api/src/rpc/router.ts | 9 ++++ 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts index 88806614..257098d2 100644 --- a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts +++ b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts @@ -130,9 +130,47 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi .where("id", "=", softwareSillId) .execute(); }, - getById: async (softwareId: number): Promise => { - console.log("getById : ", softwareId); - return makeGetSoftwareBuilder(db) + getByName: async (softwareName: string): Promise => + makeGetSoftwareBuilder(db) + .where("name", "=", softwareName) + .executeTakeFirst() + .then((result): Software | undefined => { + if (!result) return; + const { + testUrls, + serviceProviders, + parentExternalData, + updateTime, + addedTime, + softwareExternalData, + similarExternalSoftwares, + ...software + } = result; + return stripNullOrUndefinedValues({ + ...software, + updateTime: new Date(+updateTime).getTime(), + addedTime: new Date(+addedTime).getTime(), + serviceProviders: serviceProviders ?? [], + similarSoftwares: similarExternalSoftwares, + userAndReferentCountByOrganization: {}, + authors: (softwareExternalData?.developers ?? []).map(dev => ({ + authorName: dev.name, + authorUrl: `https://www.wikidata.org/wiki/${dev.id}` + })), + officialWebsiteUrl: + softwareExternalData?.websiteUrl ?? + software.comptoirDuLibreSoftware?.external_resources.website, + codeRepositoryUrl: + softwareExternalData?.sourceUrl ?? + software.comptoirDuLibreSoftware?.external_resources.repository, + documentationUrl: softwareExternalData?.documentationUrl, + comptoirDuLibreServiceProviderCount: software.comptoirDuLibreSoftware?.providers.length ?? 0, + testUrl: testUrls[0]?.url, + parentWikidataSoftware: parentExternalData + }); + }), + getById: async (softwareId: number): Promise => + makeGetSoftwareBuilder(db) .where("id", "=", softwareId) .executeTakeFirst() .then((result): Software | undefined => { @@ -169,8 +207,7 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi testUrl: testUrls[0]?.url, parentWikidataSoftware: parentExternalData }); - }); - }, + }), getAll: (): Promise => makeGetSoftwareBuilder(db) .execute() diff --git a/api/src/core/adapters/dbApi/kysely/migrations/1717162141365_create-initial-tables.ts b/api/src/core/adapters/dbApi/kysely/migrations/1717162141365_create-initial-tables.ts index f2508b46..9f886f1f 100644 --- a/api/src/core/adapters/dbApi/kysely/migrations/1717162141365_create-initial-tables.ts +++ b/api/src/core/adapters/dbApi/kysely/migrations/1717162141365_create-initial-tables.ts @@ -19,7 +19,7 @@ export async function up(db: Kysely): Promise { .addColumn("externalId", "text") .addColumn("externalDataOrigin", sql`external_data_origin_type`) .addColumn("comptoirDuLibreId", "integer") - .addColumn("name", "text", col => col.notNull()) + .addColumn("name", "text", col => col.unique().notNull()) .addColumn("description", "text", col => col.notNull()) .addColumn("license", "text", col => col.notNull()) .addColumn("versionMin", "text", col => col.notNull()) diff --git a/api/src/core/ports/DbApiV2.ts b/api/src/core/ports/DbApiV2.ts index 4a5a6af7..0f938a74 100644 --- a/api/src/core/ports/DbApiV2.ts +++ b/api/src/core/ports/DbApiV2.ts @@ -22,6 +22,7 @@ export interface SoftwareRepository { ) => Promise; getAll: () => Promise; getById: (id: number) => Promise; + getByName: (name: string) => Promise; countAddedByAgent: (params: { agentEmail: string }) => Promise; getAllSillSoftwareExternalIds: (externalDataOrigin: ExternalDataOrigin) => Promise; unreference: (params: { softwareId: number; reason: string; time: number }) => Promise; diff --git a/api/src/rpc/router.ts b/api/src/rpc/router.ts index 80b1598f..3ac1035c 100644 --- a/api/src/rpc/router.ts +++ b/api/src/rpc/router.ts @@ -187,6 +187,15 @@ export function createRouter(params: { // TODO : there is some logic with logoUrl that should be moved here // from readWriteSillData/thunks/getStorableLogo + const existingSoftware = await dbApi.software.getByName(formData.softwareName.trim()); + + if (existingSoftware) { + throw new TRPCError({ + "code": "CONFLICT", + "message": `Software already exists with name : ${formData.softwareName.trim()}` + }); + } + try { await dbApi.software.create({ formData, From 75f8d80d563ddc900a9e9966eab8a8086a609042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Burkard?= <22095555+JeromeBu@users.noreply.github.com> Date: Sun, 11 Aug 2024 14:06:45 +0200 Subject: [PATCH 08/19] reference agent by Id in software and instances instead of by email --- api/scripts/load-git-repo-in-pg.ts | 46 +++++++++++++------ .../dbApi/kysely/createGetCompiledData.ts | 6 ++- .../dbApi/kysely/createPgAgentRepository.ts | 5 ++ .../kysely/createPgInstanceRepository.ts | 12 ++++- .../kysely/createPgSoftwareRepository.ts | 12 ++--- .../adapters/dbApi/kysely/kysely.database.ts | 4 +- .../1717162141365_create-initial-tables.ts | 4 +- .../dbApi/kysely/pgDbApi.integration.test.ts | 11 +++-- api/src/core/ports/DbApiV2.ts | 11 +++-- api/src/rpc/router.ts | 32 ++++++++----- api/src/rpc/routes.e2e.test.ts | 21 +++++++-- 11 files changed, 112 insertions(+), 52 deletions(-) diff --git a/api/scripts/load-git-repo-in-pg.ts b/api/scripts/load-git-repo-in-pg.ts index 69b1ecdb..28b2a69e 100644 --- a/api/scripts/load-git-repo-in-pg.ts +++ b/api/scripts/load-git-repo-in-pg.ts @@ -1,6 +1,7 @@ import { InsertObject, Kysely, sql } from "kysely"; import { z } from "zod"; import { createGitDbApi, GitDbApiParams } from "../src/core/adapters/dbApi/createGitDbApi"; +import { makeGetAgentIdByEmail } from "../src/core/adapters/dbApi/kysely/createPgAgentRepository"; import { Database } from "../src/core/adapters/dbApi/kysely/kysely.database"; import { createPgDialect } from "../src/core/adapters/dbApi/kysely/kysely.dialect"; import { CompiledData } from "../src/core/ports/CompileData"; @@ -20,22 +21,24 @@ const saveGitDbInPostgres = async ({ pgConfig, gitDbConfig }: Params) => { const { softwareRows, agentRows, softwareReferentRows, softwareUserRows, instanceRows } = await gitDbApi.fetchDb(); - await insertSoftwares(softwareRows, pgDb); await insertAgents(agentRows, pgDb); const agentIdByEmail = await makeGetAgentIdByEmail(pgDb); + + await insertSoftwares(softwareRows, agentIdByEmail, pgDb); await insertSoftwareReferents({ softwareReferentRows: softwareReferentRows, - agentIdByEmail: agentIdByEmail, + agentIdByEmail, db: pgDb }); await insertSoftwareUsers({ softwareUserRows: softwareUserRows, - agentIdByEmail: agentIdByEmail, + agentIdByEmail, db: pgDb }); await insertInstances({ instanceRows: instanceRows, + agentIdByEmail, db: pgDb }); @@ -43,7 +46,11 @@ const saveGitDbInPostgres = async ({ pgConfig, gitDbConfig }: Params) => { await insertCompiledSoftwaresAndSoftwareExternalData(compiledSoftwares, pgDb); }; -const insertSoftwares = async (softwareRows: SoftwareRow[], db: Kysely) => { +const insertSoftwares = async ( + softwareRows: SoftwareRow[], + agentIdByEmail: Record, + db: Kysely +) => { console.info("Deleting than Inserting softwares"); console.info("Number of softwares to insert : ", softwareRows.length); await db.transaction().execute(async trx => { @@ -52,8 +59,9 @@ const insertSoftwares = async (softwareRows: SoftwareRow[], db: Kysely await trx .insertInto("softwares") .values( - softwareRows.map(({ similarSoftwareExternalDataIds: _, ...row }) => ({ + softwareRows.map(({ similarSoftwareExternalDataIds: _, addedByAgentEmail, ...row }) => ({ ...row, + addedByAgentId: agentIdByEmail[addedByAgentEmail], dereferencing: row.dereferencing ? JSON.stringify(row.dereferencing) : null, softwareType: JSON.stringify(row.softwareType), workshopUrls: JSON.stringify(row.workshopUrls), @@ -83,18 +91,14 @@ const insertAgents = async (agentRows: Db.AgentRow[], db: Kysely) => { console.log("Deleting than Inserting agents"); console.info("Number of agents to insert : ", agentRows.length); await db.transaction().execute(async trx => { + await trx.deleteFrom("instances").execute(); + await trx.deleteFrom("softwares").execute(); await trx.deleteFrom("agents").execute(); await trx.insertInto("agents").values(agentRows).executeTakeFirst(); await sql`SELECT setval('agents_id_seq', (SELECT MAX(id) FROM agents))`.execute(trx); }); }; -const makeGetAgentIdByEmail = async (db: Kysely): Promise> => { - console.info("Fetching agents, to map email to id"); - const agents = await db.selectFrom("agents").select(["email", "id"]).execute(); - return agents.reduce((acc, agent) => ({ ...acc, [agent.email]: agent.id }), {}); -}; - const insertSoftwareReferents = async ({ softwareReferentRows, agentIdByEmail, @@ -145,12 +149,28 @@ const insertSoftwareUsers = async ({ }); }; -const insertInstances = async ({ instanceRows, db }: { instanceRows: Db.InstanceRow[]; db: Kysely }) => { +const insertInstances = async ({ + instanceRows, + agentIdByEmail, + db +}: { + instanceRows: Db.InstanceRow[]; + agentIdByEmail: Record; + db: Kysely; +}) => { console.info("Deleting than Inserting instances"); console.info("Number of instances to insert : ", instanceRows.length); await db.transaction().execute(async trx => { await trx.deleteFrom("instances").execute(); - await trx.insertInto("instances").values(instanceRows).executeTakeFirst(); + await trx + .insertInto("instances") + .values( + instanceRows.map(({ addedByAgentEmail, ...instanceRow }) => ({ + ...instanceRow, + addedByAgentId: agentIdByEmail[addedByAgentEmail] + })) + ) + .executeTakeFirst(); await sql`SELECT setval('instances_id_seq', (SELECT MAX(id) FROM instances))`.execute(trx); }); }; diff --git a/api/src/core/adapters/dbApi/kysely/createGetCompiledData.ts b/api/src/core/adapters/dbApi/kysely/createGetCompiledData.ts index 41c2549a..1eafeb64 100644 --- a/api/src/core/adapters/dbApi/kysely/createGetCompiledData.ts +++ b/api/src/core/adapters/dbApi/kysely/createGetCompiledData.ts @@ -45,7 +45,7 @@ export const createGetCompiledData = (db: Kysely) => async (): Promise ]) .select([ "s.id", - "s.addedByAgentEmail", + "s.addedByAgentId", "s.catalogNumeriqueGouvFrId", "s.categories", "s.dereferencing", @@ -120,6 +120,7 @@ export const createGetCompiledData = (db: Kysely) => async (): Promise console.time("software processing"); const processedSoftwares = results.map( ({ + addedByAgentId, externalDataSoftwareId, annuaireCnllServiceProviders, comptoirDuLibreSoftware, @@ -139,6 +140,7 @@ export const createGetCompiledData = (db: Kysely) => async (): Promise }): CompiledData.Software<"private"> => { return { ...stripNullOrUndefinedValues(software), + addedByAgentEmail: agentById[addedByAgentId].email, updateTime: new Date(+updateTime).getTime(), referencedSinceTime: new Date(+referencedSinceTime).getTime(), doRespectRgaa, @@ -172,7 +174,7 @@ export const createGetCompiledData = (db: Kysely) => async (): Promise organization: instance.organization!, targetAudience: instance.targetAudience!, publicUrl: instance.publicUrl ?? undefined, - addedByAgentEmail: instance.addedByAgentEmail!, + addedByAgentEmail: agentById[instance.addedByAgentId!].email, otherWikidataSoftwares: [] })) }; diff --git a/api/src/core/adapters/dbApi/kysely/createPgAgentRepository.ts b/api/src/core/adapters/dbApi/kysely/createPgAgentRepository.ts index a5c32dae..d15b30a1 100644 --- a/api/src/core/adapters/dbApi/kysely/createPgAgentRepository.ts +++ b/api/src/core/adapters/dbApi/kysely/createPgAgentRepository.ts @@ -93,3 +93,8 @@ const makeGetAgentBuilder = (db: Kysely) => .as("referentsDeclarations") ]) .groupBy("a.id"); + +export const makeGetAgentIdByEmail = async (db: Kysely): Promise> => { + const agents = await db.selectFrom("agents").select(["email", "id"]).execute(); + return agents.reduce((acc, agent) => ({ ...acc, [agent.email]: agent.id }), {}); +}; diff --git a/api/src/core/adapters/dbApi/kysely/createPgInstanceRepository.ts b/api/src/core/adapters/dbApi/kysely/createPgInstanceRepository.ts index 4d08fd0e..1a7713c4 100644 --- a/api/src/core/adapters/dbApi/kysely/createPgInstanceRepository.ts +++ b/api/src/core/adapters/dbApi/kysely/createPgInstanceRepository.ts @@ -6,7 +6,7 @@ import { Instance } from "../../../usecases/readWriteSillData"; import { Database } from "./kysely.database"; export const createPgInstanceRepository = (db: Kysely): InstanceRepository => ({ - create: async ({ formData, agentEmail }) => { + create: async ({ formData, agentId }) => { const { mainSoftwareSillId, organization, targetAudience, publicUrl, ...rest } = formData; assert>(); @@ -14,7 +14,7 @@ export const createPgInstanceRepository = (db: Kysely): InstanceReposi const { instanceId } = await db .insertInto("instances") .values({ - addedByAgentEmail: agentEmail, + addedByAgentId: agentId, updateTime: now, referencedSinceTime: now, mainSoftwareSillId, @@ -43,6 +43,14 @@ export const createPgInstanceRepository = (db: Kysely): InstanceReposi .where("id", "=", instanceId) .execute(); }, + countAddedByAgent: async ({ agentId }) => { + const { count } = await db + .selectFrom("instances") + .select(qb => qb.fn.countAll().as("count")) + .where("addedByAgentId", "=", agentId) + .executeTakeFirstOrThrow(); + return parseInt(count); + }, getAll: async () => db .selectFrom("instances as i") diff --git a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts index 257098d2..7117b3c3 100644 --- a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts +++ b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts @@ -8,7 +8,7 @@ import { Database } from "./kysely.database"; import { stripNullOrUndefinedValues, jsonBuildObject } from "./kysely.utils"; export const createPgSoftwareRepository = (db: Kysely): SoftwareRepository => ({ - create: async ({ formData, externalDataOrigin, agentEmail }) => { + create: async ({ formData, externalDataOrigin, agentId }) => { const { softwareName, softwareDescription, @@ -56,7 +56,7 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi testUrls: JSON.stringify([]), categories: JSON.stringify([]), generalInfoMd: undefined, - addedByAgentEmail: agentEmail, + addedByAgentId: agentId, keywords: JSON.stringify(softwareKeywords) }) .returning("id as softwareId") @@ -81,7 +81,7 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi return softwareId; }); }, - update: async ({ formData, softwareSillId, agentEmail }) => { + update: async ({ formData, softwareSillId, agentId }) => { const { softwareName, softwareDescription, @@ -124,7 +124,7 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi testUrls: JSON.stringify([]), categories: JSON.stringify([]), generalInfoMd: undefined, - addedByAgentEmail: agentEmail, + addedByAgentId: agentId, keywords: JSON.stringify(softwareKeywords) }) .where("id", "=", softwareSillId) @@ -270,11 +270,11 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi .execute() .then(rows => rows.map(row => row.externalId!)), - countAddedByAgent: async ({ agentEmail }) => { + countAddedByAgent: async ({ agentId }) => { const { count } = await db .selectFrom("softwares") .select(qb => qb.fn.countAll().as("count")) - .where("addedByAgentEmail", "=", agentEmail) + .where("addedByAgentId", "=", agentId) .executeTakeFirstOrThrow(); return +count; }, diff --git a/api/src/core/adapters/dbApi/kysely/kysely.database.ts b/api/src/core/adapters/dbApi/kysely/kysely.database.ts index fd490947..08b2d927 100644 --- a/api/src/core/adapters/dbApi/kysely/kysely.database.ts +++ b/api/src/core/adapters/dbApi/kysely/kysely.database.ts @@ -44,7 +44,7 @@ type InstancesTable = { organization: string; targetAudience: string; publicUrl: string | null; - addedByAgentEmail: string; + addedByAgentId: number; referencedSinceTime: number; updateTime: number; }; @@ -118,7 +118,7 @@ type SoftwaresTable = { >; categories: JSONColumnType; generalInfoMd: string | null; - addedByAgentEmail: string; + addedByAgentId: number; logoUrl: string | null; keywords: JSONColumnType; }; diff --git a/api/src/core/adapters/dbApi/kysely/migrations/1717162141365_create-initial-tables.ts b/api/src/core/adapters/dbApi/kysely/migrations/1717162141365_create-initial-tables.ts index 9f886f1f..c0656525 100644 --- a/api/src/core/adapters/dbApi/kysely/migrations/1717162141365_create-initial-tables.ts +++ b/api/src/core/adapters/dbApi/kysely/migrations/1717162141365_create-initial-tables.ts @@ -36,7 +36,7 @@ export async function up(db: Kysely): Promise { .addColumn("testUrls", "jsonb", col => col.notNull()) .addColumn("categories", "jsonb", col => col.notNull()) .addColumn("generalInfoMd", "text") - .addColumn("addedByAgentEmail", "text", col => col.notNull()) + .addColumn("addedByAgentId", "integer", col => col.notNull().references("agents.id")) .addColumn("dereferencing", "jsonb") .addColumn("referencedSinceTime", "bigint", col => col.notNull()) .addColumn("updateTime", "bigint", col => col.notNull()) @@ -88,7 +88,7 @@ export async function up(db: Kysely): Promise { .createTable("instances") .addColumn("id", "serial", col => col.primaryKey()) .addColumn("mainSoftwareSillId", "integer", col => col.notNull().references("softwares.id").onDelete("cascade")) - .addColumn("addedByAgentEmail", "text", col => col.notNull()) + .addColumn("addedByAgentId", "integer", col => col.notNull().references("agents.id")) .addColumn("organization", "text", col => col.notNull()) .addColumn("targetAudience", "text", col => col.notNull()) .addColumn("publicUrl", "text") diff --git a/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts b/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts index f5cbf0e7..135f3bb3 100644 --- a/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts +++ b/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts @@ -218,12 +218,12 @@ describe("pgDbApi", () => { describe("instance", () => { it("creates an instance, than gets it with getAll", async () => { console.log("------ instance scenario ------"); - await insertSoftwareExternalDataAndSoftwareAndAgent(); + const { agentId } = await insertSoftwareExternalDataAndSoftwareAndAgent(); const softwares = await dbApi.software.getAll(); const softwareId = softwares[0].softwareId; console.log("saving instance"); await dbApi.instance.create({ - agentEmail: insertedAgent.email, + agentId, formData: { mainSoftwareSillId: softwareId, organization: "test-orga", @@ -252,9 +252,10 @@ describe("pgDbApi", () => { const agentId = await dbApi.agent.add(insertedAgent); const softwareId = await dbApi.software.create({ formData: softwareFormData, - agentEmail: insertedAgent.email, + agentId, externalDataOrigin: "wikidata" }); + await db .insertInto("software_users") .values({ @@ -316,6 +317,8 @@ describe("pgDbApi", () => { const allAgents = await dbApi.agent.getAll(); expectToEqual(allAgents, [{ ...updatedAgent, declarations: expectedDeclarations }]); + await db.deleteFrom("softwares").where("addedByAgentId", "=", updatedAgent.id).execute(); + console.log("removing agent"); await dbApi.agent.remove(updatedAgent.id); @@ -419,7 +422,7 @@ describe("pgDbApi", () => { const softwareId = await dbApi.software.create({ formData: softwareFormData, - agentEmail: insertedAgent.email, + agentId, externalDataOrigin: "wikidata" }); diff --git a/api/src/core/ports/DbApiV2.ts b/api/src/core/ports/DbApiV2.ts index 0f938a74..00c1c934 100644 --- a/api/src/core/ports/DbApiV2.ts +++ b/api/src/core/ports/DbApiV2.ts @@ -5,32 +5,33 @@ import type { CompiledData } from "./CompileData"; import type { ExternalDataOrigin } from "./GetSoftwareExternalData"; -export type WithAgentEmail = { agentEmail: string }; +export type WithAgentId = { agentId: number }; export interface SoftwareRepository { create: ( params: { formData: SoftwareFormData; externalDataOrigin: ExternalDataOrigin; - } & WithAgentEmail + } & WithAgentId ) => Promise; update: ( params: { softwareSillId: number; formData: SoftwareFormData; - } & WithAgentEmail + } & WithAgentId ) => Promise; getAll: () => Promise; getById: (id: number) => Promise; getByName: (name: string) => Promise; - countAddedByAgent: (params: { agentEmail: string }) => Promise; + countAddedByAgent: (params: { agentId: number }) => Promise; getAllSillSoftwareExternalIds: (externalDataOrigin: ExternalDataOrigin) => Promise; unreference: (params: { softwareId: number; reason: string; time: number }) => Promise; } export interface InstanceRepository { - create: (params: { formData: InstanceFormData } & WithAgentEmail) => Promise; + create: (params: { formData: InstanceFormData } & WithAgentId) => Promise; update: (params: { formData: InstanceFormData; instanceId: number }) => Promise; + countAddedByAgent: (params: { agentId: number }) => Promise; getAll: () => Promise; } diff --git a/api/src/rpc/router.ts b/api/src/rpc/router.ts index 3ac1035c..380165a2 100644 --- a/api/src/rpc/router.ts +++ b/api/src/rpc/router.ts @@ -197,20 +197,22 @@ export function createRouter(params: { } try { - await dbApi.software.create({ - formData, - agentEmail: user.email, - externalDataOrigin - }); const agent = await dbApi.agent.getByEmail(user.email); + let agentId = agent?.id as number; if (!agent) { - await dbApi.agent.add({ + agentId = await dbApi.agent.add({ email: user.email, organization: user.organization, about: undefined, isPublic: false }); } + + await dbApi.software.create({ + formData, + agentId, + externalDataOrigin + }); } catch (e) { throw new TRPCError({ "code": "INTERNAL_SERVER_ERROR", "message": String(e) }); } @@ -229,10 +231,13 @@ export function createRouter(params: { const { softwareSillId, formData } = input; + const agent = await dbApi.agent.getByEmail(user.email); + if (!agent) throw new TRPCError({ "code": "NOT_FOUND", message: "Agent not found" }); + await dbApi.software.update({ softwareSillId, formData, - agentEmail: user.email + agentId: agent.id }); }), "createUserOrReferent": loggedProcedure @@ -332,17 +337,20 @@ export function createRouter(params: { const [ numberOfSoftwareWhereThisAgentIsUser, numberOfSoftwareWhereThisAgentIsReferent, - numberOfSoftwareAddedByThisAgent + numberOfSoftwareAddedByThisAgent, + numberOfInstanceAddedByThisAgent ] = await Promise.all([ dbApi.softwareUser.countSoftwaresForAgent({ agentId: agent.id }), dbApi.softwareReferent.countSoftwaresForAgent({ agentId: agent.id }), - dbApi.software.countAddedByAgent({ agentEmail: agent.email }) + dbApi.software.countAddedByAgent({ agentId: agent.id }), + dbApi.instance.countAddedByAgent({ agentId: agent.id }) ]); if ( numberOfSoftwareWhereThisAgentIsReferent === 0 && numberOfSoftwareWhereThisAgentIsUser === 0 && - numberOfSoftwareAddedByThisAgent === 0 + numberOfSoftwareAddedByThisAgent === 0 && + numberOfInstanceAddedByThisAgent === 0 ) { await dbApi.agent.remove(agent.id); } @@ -359,11 +367,13 @@ export function createRouter(params: { throw new TRPCError({ "code": "UNAUTHORIZED" }); } + const agent = await dbApi.agent.getByEmail(user.email); + if (!agent) throw new TRPCError({ "code": "NOT_FOUND", message: "Agent not found" }); const { formData } = input; const instanceId = await dbApi.instance.create({ formData, - agentEmail: user.email + agentId: agent.id }); return { instanceId }; diff --git a/api/src/rpc/routes.e2e.test.ts b/api/src/rpc/routes.e2e.test.ts index 9c78c2b4..24f05d3b 100644 --- a/api/src/rpc/routes.e2e.test.ts +++ b/api/src/rpc/routes.e2e.test.ts @@ -3,7 +3,8 @@ import { beforeAll, describe, expect, it } from "vitest"; import { Database } from "../core/adapters/dbApi/kysely/kysely.database"; import { stripNullOrUndefinedValues } from "../core/adapters/dbApi/kysely/kysely.utils"; import { CompiledData } from "../core/ports/CompileData"; -import { InstanceFormData } from "../core/usecases/readWriteSillData"; +import type { DbAgent } from "../core/ports/DbApiV2"; +import type { InstanceFormData } from "../core/usecases/readWriteSillData"; import { createDeclarationFormData, createInstanceFormData, @@ -82,14 +83,15 @@ describe("RPC e2e tests", () => { describe("Scenario - Add a new software then mark an agent as user of this software", () => { let actualSoftwareId: number; let instanceFormData: InstanceFormData; + let agent: DbAgent; beforeAll(async () => { ({ apiCaller, kyselyDb } = await createTestCaller()); await kyselyDb.deleteFrom("software_referents").execute(); await kyselyDb.deleteFrom("software_users").execute(); - await kyselyDb.deleteFrom("agents").execute(); - await kyselyDb.deleteFrom("softwares").execute(); await kyselyDb.deleteFrom("instances").execute(); + await kyselyDb.deleteFrom("softwares").execute(); + await kyselyDb.deleteFrom("agents").execute(); }); it("gets the list of agents, which is initially empty", async () => { @@ -106,6 +108,15 @@ describe("RPC e2e tests", () => { formData: softwareFormData }); + const { agents } = await apiCaller.getAgents(); + expect(agents).toHaveLength(1); + agent = agents[0]; + expectToMatchObject(agent, { + id: expect.any(Number), + email: defaultUser.email, + organization: defaultUser.organization + }); + const softwareRows = await getSoftwareRows(); expect(softwareRows).toHaveLength(1); const expectedSoftware: Partial> = { @@ -131,7 +142,7 @@ describe("RPC e2e tests", () => { expectToMatchObject(softwareRows[0], { ...expectedSoftware, - "addedByAgentEmail": defaultUser.email + "addedByAgentId": agent.id }); const similars = await kyselyDb .selectFrom("softwares__similar_software_external_datas") @@ -198,7 +209,7 @@ describe("RPC e2e tests", () => { expect(instanceRows).toHaveLength(1); expectToMatchObject(instanceRows[0], { "id": expect.any(Number), - "addedByAgentEmail": defaultUser.email, + "addedByAgentId": agent.id, "mainSoftwareSillId": actualSoftwareId, "organization": instanceFormData.organization, "publicUrl": instanceFormData.publicUrl, From dad210c460b1edeb84a055135ac13c321f0600e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Burkard?= <22095555+JeromeBu@users.noreply.github.com> Date: Wed, 9 Oct 2024 11:07:51 +0200 Subject: [PATCH 09/19] fix fetchExternalData --- api/scripts/load-git-repo-in-pg.ts | 3 + .../adapters/dbApi/kysely/createPgDbApi.ts | 4 + ...eatePgOtherSoftwareExtraDataRepositiory.ts | 36 ++ .../createPgSoftwareExternalDataRepository.ts | 20 + .../kysely/createPgSoftwareRepository.ts | 548 ++++++++++-------- .../1719576701920_add-compiled-tables.ts | 4 +- .../core/adapters/fetchExternalData.test.ts | 483 +++++++++++++++ api/src/core/adapters/fetchExternalData.ts | 148 +++++ api/src/core/bootstrap.ts | 3 + api/src/core/ports/DbApiV2.ts | 51 +- .../core/usecases/readWriteSillData/types.ts | 7 +- 11 files changed, 1047 insertions(+), 260 deletions(-) create mode 100644 api/src/core/adapters/dbApi/kysely/createPgOtherSoftwareExtraDataRepositiory.ts create mode 100644 api/src/core/adapters/dbApi/kysely/createPgSoftwareExternalDataRepository.ts create mode 100644 api/src/core/adapters/fetchExternalData.test.ts create mode 100644 api/src/core/adapters/fetchExternalData.ts diff --git a/api/scripts/load-git-repo-in-pg.ts b/api/scripts/load-git-repo-in-pg.ts index 28b2a69e..9bd139a2 100644 --- a/api/scripts/load-git-repo-in-pg.ts +++ b/api/scripts/load-git-repo-in-pg.ts @@ -21,6 +21,9 @@ const saveGitDbInPostgres = async ({ pgConfig, gitDbConfig }: Params) => { const { softwareRows, agentRows, softwareReferentRows, softwareUserRows, instanceRows } = await gitDbApi.fetchDb(); + const reactSofts = softwareRows.filter(s => s.name.toLowerCase().includes("react")); + console.log("REACT SOFT from gitDB: ", reactSofts); + await insertAgents(agentRows, pgDb); const agentIdByEmail = await makeGetAgentIdByEmail(pgDb); diff --git a/api/src/core/adapters/dbApi/kysely/createPgDbApi.ts b/api/src/core/adapters/dbApi/kysely/createPgDbApi.ts index 610d8014..cd73b7e7 100644 --- a/api/src/core/adapters/dbApi/kysely/createPgDbApi.ts +++ b/api/src/core/adapters/dbApi/kysely/createPgDbApi.ts @@ -3,6 +3,8 @@ import { DbApiV2 } from "../../../ports/DbApiV2"; import { createGetCompiledData } from "./createGetCompiledData"; import { createPgAgentRepository } from "./createPgAgentRepository"; import { createPgInstanceRepository } from "./createPgInstanceRepository"; +import { createPgOtherSoftwareExtraDataRepository } from "./createPgOtherSoftwareExtraDataRepositiory"; +import { createPgSoftwareExternalDataRepository } from "./createPgSoftwareExternalDataRepository"; import { createPgSoftwareRepository } from "./createPgSoftwareRepository"; import { createPgReferentRepository, createPgUserRepository } from "./createPgUserAndReferentRepository"; import { Database } from "./kysely.database"; @@ -10,6 +12,8 @@ import { Database } from "./kysely.database"; export const createKyselyPgDbApi = (db: Kysely): DbApiV2 => { return { software: createPgSoftwareRepository(db), + softwareExternalData: createPgSoftwareExternalDataRepository(db), + otherSoftwareExtraData: createPgOtherSoftwareExtraDataRepository(db), instance: createPgInstanceRepository(db), agent: createPgAgentRepository(db), softwareReferent: createPgReferentRepository(db), diff --git a/api/src/core/adapters/dbApi/kysely/createPgOtherSoftwareExtraDataRepositiory.ts b/api/src/core/adapters/dbApi/kysely/createPgOtherSoftwareExtraDataRepositiory.ts new file mode 100644 index 00000000..388e1ae2 --- /dev/null +++ b/api/src/core/adapters/dbApi/kysely/createPgOtherSoftwareExtraDataRepositiory.ts @@ -0,0 +1,36 @@ +import { Kysely } from "kysely"; +import { OtherSoftwareExtraDataRepository } from "../../../ports/DbApiV2"; +import { Database } from "./kysely.database"; + +export const createPgOtherSoftwareExtraDataRepository = (db: Kysely): OtherSoftwareExtraDataRepository => ({ + save: async ({ + softwareId, + annuaireCnllServiceProviders, + serviceProviders, + comptoirDuLibreSoftware, + latestVersion + }) => { + const pgValues = { + softwareId, + annuaireCnllServiceProviders: JSON.stringify(annuaireCnllServiceProviders), + serviceProviders: JSON.stringify(serviceProviders), + comptoirDuLibreSoftware: JSON.stringify(comptoirDuLibreSoftware), + latestVersion: JSON.stringify(latestVersion) + }; + + await db + .insertInto("compiled_softwares") + .values(pgValues) + .onConflict(oc => oc.column("softwareId").doUpdateSet(pgValues)) + .executeTakeFirst(); + }, + getBySoftwareId: async softwareId => + db.selectFrom("compiled_softwares").selectAll().where("softwareId", "=", softwareId).executeTakeFirst() +}); + +// const a = qb => ({ +// annuaireCnllServiceProviders: qb.ref("excluded.annuaireCnllServiceProviders"), +// serviceProviders: qb.ref("excluded.serviceProviders"), +// comptoirDuLibreSoftware: qb.ref("excluded.comptoirDuLibreSoftware"), +// latestVersion: qb.ref("excluded.latestVersion") +// }); diff --git a/api/src/core/adapters/dbApi/kysely/createPgSoftwareExternalDataRepository.ts b/api/src/core/adapters/dbApi/kysely/createPgSoftwareExternalDataRepository.ts new file mode 100644 index 00000000..3450ea77 --- /dev/null +++ b/api/src/core/adapters/dbApi/kysely/createPgSoftwareExternalDataRepository.ts @@ -0,0 +1,20 @@ +import { Kysely } from "kysely"; +import { SoftwareExternalDataRepository } from "../../../ports/DbApiV2"; +import { Database } from "./kysely.database"; + +export const createPgSoftwareExternalDataRepository = (db: Kysely): SoftwareExternalDataRepository => ({ + save: async softwareExternalData => { + const pgValues = { + ...softwareExternalData, + developers: JSON.stringify(softwareExternalData.developers), + label: JSON.stringify(softwareExternalData.label), + description: JSON.stringify(softwareExternalData.description) + }; + + await db + .insertInto("software_external_datas") + .values(pgValues) + .onConflict(oc => oc.column("externalId").doUpdateSet(pgValues)) + .executeTakeFirst(); + } +}); diff --git a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts index 7117b3c3..74b2d107 100644 --- a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts +++ b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts @@ -7,48 +7,121 @@ import { Software } from "../../../usecases/readWriteSillData"; import { Database } from "./kysely.database"; import { stripNullOrUndefinedValues, jsonBuildObject } from "./kysely.utils"; -export const createPgSoftwareRepository = (db: Kysely): SoftwareRepository => ({ - create: async ({ formData, externalDataOrigin, agentId }) => { - const { - softwareName, - softwareDescription, - softwareLicense, - softwareLogoUrl, - softwareMinimalVersion, - isPresentInSupportContract, - isFromFrenchPublicService, - doRespectRgaa, - similarSoftwareExternalDataIds, - softwareType, - externalId, - comptoirDuLibreId, - softwareKeywords, - ...rest - } = formData; +export const createPgSoftwareRepository = (db: Kysely): SoftwareRepository => { + const getBySoftwareId = makeGetSoftwareById(db); + return { + create: async ({ formData, externalDataOrigin, agentId }) => { + const { + softwareName, + softwareDescription, + softwareLicense, + softwareLogoUrl, + softwareMinimalVersion, + isPresentInSupportContract, + isFromFrenchPublicService, + doRespectRgaa, + similarSoftwareExternalDataIds, + softwareType, + externalId, + comptoirDuLibreId, + softwareKeywords, + ...rest + } = formData; - assert>(); + assert>(); - const now = Date.now(); + const now = Date.now(); - return db.transaction().execute(async trx => { - const { softwareId } = await trx - .insertInto("softwares") - .values({ + return db.transaction().execute(async trx => { + const { softwareId } = await trx + .insertInto("softwares") + .values({ + name: softwareName, + description: softwareDescription, + license: softwareLicense, + logoUrl: softwareLogoUrl, + versionMin: softwareMinimalVersion, + referencedSinceTime: now, + updateTime: now, + dereferencing: undefined, + isStillInObservation: false, + parentSoftwareWikidataId: undefined, + doRespectRgaa: doRespectRgaa, + isFromFrenchPublicService: isFromFrenchPublicService, + isPresentInSupportContract: isPresentInSupportContract, + externalId: externalId, + externalDataOrigin: externalDataOrigin, + comptoirDuLibreId: comptoirDuLibreId, + softwareType: JSON.stringify(softwareType), + catalogNumeriqueGouvFrId: undefined, + workshopUrls: JSON.stringify([]), + testUrls: JSON.stringify([]), + categories: JSON.stringify([]), + generalInfoMd: undefined, + addedByAgentId: agentId, + keywords: JSON.stringify(softwareKeywords) + }) + .returning("id as softwareId") + .executeTakeFirstOrThrow(); + + console.log( + `inserted software correctly, softwareId is : ${softwareId}, about to insert similars : `, + similarSoftwareExternalDataIds + ); + + if (similarSoftwareExternalDataIds.length > 0) { + await trx + .insertInto("softwares__similar_software_external_datas") + .values( + similarSoftwareExternalDataIds.map(similarExternalId => ({ + softwareId, + similarExternalId + })) + ) + .execute(); + } + + console.log("all good"); + + return softwareId; + }); + }, + update: async ({ formData, softwareSillId, agentId }) => { + const { + softwareName, + softwareDescription, + softwareLicense, + softwareLogoUrl, + softwareMinimalVersion, + isPresentInSupportContract, + isFromFrenchPublicService, + doRespectRgaa, + similarSoftwareExternalDataIds, + softwareType, + externalId, + comptoirDuLibreId, + softwareKeywords, + ...rest + } = formData; + + assert>(); + + const now = Date.now(); + await db + .updateTable("softwares") + .set({ name: softwareName, description: softwareDescription, license: softwareLicense, logoUrl: softwareLogoUrl, versionMin: softwareMinimalVersion, - referencedSinceTime: now, updateTime: now, - dereferencing: undefined, isStillInObservation: false, parentSoftwareWikidataId: undefined, doRespectRgaa: doRespectRgaa, isFromFrenchPublicService: isFromFrenchPublicService, isPresentInSupportContract: isPresentInSupportContract, externalId: externalId, - externalDataOrigin: externalDataOrigin, comptoirDuLibreId: comptoirDuLibreId, softwareType: JSON.stringify(softwareType), catalogNumeriqueGouvFrId: undefined, @@ -59,162 +132,16 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi addedByAgentId: agentId, keywords: JSON.stringify(softwareKeywords) }) - .returning("id as softwareId") - .executeTakeFirstOrThrow(); - - console.log( - `inserted software correctly, softwareId is : ${softwareId}, about to insert similars : `, - similarSoftwareExternalDataIds - ); - - if (similarSoftwareExternalDataIds.length > 0) { - await trx - .insertInto("softwares__similar_software_external_datas") - .values( - similarSoftwareExternalDataIds.map(similarExternalId => ({ softwareId, similarExternalId })) - ) - .execute(); - } - - console.log("all good"); - - return softwareId; - }); - }, - update: async ({ formData, softwareSillId, agentId }) => { - const { - softwareName, - softwareDescription, - softwareLicense, - softwareLogoUrl, - softwareMinimalVersion, - isPresentInSupportContract, - isFromFrenchPublicService, - doRespectRgaa, - similarSoftwareExternalDataIds, - softwareType, - externalId, - comptoirDuLibreId, - softwareKeywords, - ...rest - } = formData; - - assert>(); - - const now = Date.now(); - await db - .updateTable("softwares") - .set({ - name: softwareName, - description: softwareDescription, - license: softwareLicense, - logoUrl: softwareLogoUrl, - versionMin: softwareMinimalVersion, - updateTime: now, - isStillInObservation: false, - parentSoftwareWikidataId: undefined, - doRespectRgaa: doRespectRgaa, - isFromFrenchPublicService: isFromFrenchPublicService, - isPresentInSupportContract: isPresentInSupportContract, - externalId: externalId, - comptoirDuLibreId: comptoirDuLibreId, - softwareType: JSON.stringify(softwareType), - catalogNumeriqueGouvFrId: undefined, - workshopUrls: JSON.stringify([]), - testUrls: JSON.stringify([]), - categories: JSON.stringify([]), - generalInfoMd: undefined, - addedByAgentId: agentId, - keywords: JSON.stringify(softwareKeywords) - }) - .where("id", "=", softwareSillId) - .execute(); - }, - getByName: async (softwareName: string): Promise => - makeGetSoftwareBuilder(db) - .where("name", "=", softwareName) - .executeTakeFirst() - .then((result): Software | undefined => { - if (!result) return; - const { - testUrls, - serviceProviders, - parentExternalData, - updateTime, - addedTime, - softwareExternalData, - similarExternalSoftwares, - ...software - } = result; - return stripNullOrUndefinedValues({ - ...software, - updateTime: new Date(+updateTime).getTime(), - addedTime: new Date(+addedTime).getTime(), - serviceProviders: serviceProviders ?? [], - similarSoftwares: similarExternalSoftwares, - userAndReferentCountByOrganization: {}, - authors: (softwareExternalData?.developers ?? []).map(dev => ({ - authorName: dev.name, - authorUrl: `https://www.wikidata.org/wiki/${dev.id}` - })), - officialWebsiteUrl: - softwareExternalData?.websiteUrl ?? - software.comptoirDuLibreSoftware?.external_resources.website, - codeRepositoryUrl: - softwareExternalData?.sourceUrl ?? - software.comptoirDuLibreSoftware?.external_resources.repository, - documentationUrl: softwareExternalData?.documentationUrl, - comptoirDuLibreServiceProviderCount: software.comptoirDuLibreSoftware?.providers.length ?? 0, - testUrl: testUrls[0]?.url, - parentWikidataSoftware: parentExternalData - }); - }), - getById: async (softwareId: number): Promise => - makeGetSoftwareBuilder(db) - .where("id", "=", softwareId) - .executeTakeFirst() - .then((result): Software | undefined => { - if (!result) return; - const { - testUrls, - serviceProviders, - parentExternalData, - updateTime, - addedTime, - softwareExternalData, - similarExternalSoftwares, - ...software - } = result; - return stripNullOrUndefinedValues({ - ...software, - updateTime: new Date(+updateTime).getTime(), - addedTime: new Date(+addedTime).getTime(), - serviceProviders: serviceProviders ?? [], - similarSoftwares: similarExternalSoftwares, - userAndReferentCountByOrganization: {}, - authors: (softwareExternalData?.developers ?? []).map(dev => ({ - authorName: dev.name, - authorUrl: `https://www.wikidata.org/wiki/${dev.id}` - })), - officialWebsiteUrl: - softwareExternalData?.websiteUrl ?? - software.comptoirDuLibreSoftware?.external_resources.website, - codeRepositoryUrl: - softwareExternalData?.sourceUrl ?? - software.comptoirDuLibreSoftware?.external_resources.repository, - documentationUrl: softwareExternalData?.documentationUrl, - comptoirDuLibreServiceProviderCount: software.comptoirDuLibreSoftware?.providers.length ?? 0, - testUrl: testUrls[0]?.url, - parentWikidataSoftware: parentExternalData - }); - }), - getAll: (): Promise => - makeGetSoftwareBuilder(db) - .execute() - .then(async softwares => { - const userAndReferentCountByOrganization = await getUserAndReferentCountByOrganizationBySoftwareId(db); - return softwares.map( - ({ + .where("id", "=", softwareSillId) + .execute(); + }, + getByName: async (softwareName: string): Promise => + makeGetSoftwareBuilder(db) + .where("name", "=", softwareName) + .executeTakeFirst() + .then((result): Software | undefined => { + if (!result) return; + const { testUrls, serviceProviders, parentExternalData, @@ -223,81 +150,150 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi softwareExternalData, similarExternalSoftwares, ...software - }): Software => { - return stripNullOrUndefinedValues({ - ...software, - updateTime: new Date(+updateTime).getTime(), - addedTime: new Date(+addedTime).getTime(), - serviceProviders: serviceProviders ?? [], - similarSoftwares: similarExternalSoftwares, - // (similarSoftwares ?? []).map( - // (s): SimilarSoftware => ({ - // softwareName: - // typeof s.label === "string" ? s.label : Object.values(s.label)[0]!, - // softwareDescription: - // typeof s.label === "string" ? s.label : Object.values(s.label)[0]!, - // isInSill: true // TODO: check if this is true - // }) - // ) ?? [], - userAndReferentCountByOrganization: - userAndReferentCountByOrganization[software.softwareId] ?? {}, - authors: (softwareExternalData?.developers ?? []).map(dev => ({ - authorName: dev.name, - authorUrl: `https://www.wikidata.org/wiki/${dev.id}` - })), - officialWebsiteUrl: - softwareExternalData?.websiteUrl ?? - software.comptoirDuLibreSoftware?.external_resources.website ?? - undefined, - codeRepositoryUrl: - softwareExternalData?.sourceUrl ?? - software.comptoirDuLibreSoftware?.external_resources.repository ?? - undefined, - documentationUrl: softwareExternalData?.documentationUrl ?? undefined, - comptoirDuLibreServiceProviderCount: - software.comptoirDuLibreSoftware?.providers.length ?? 0, - testUrl: testUrls[0]?.url, - parentWikidataSoftware: parentExternalData ?? undefined - }); - } - ); - }), - getAllSillSoftwareExternalIds: async externalDataOrigin => - db - .selectFrom("softwares") - .select("externalId") - .where("externalDataOrigin", "=", externalDataOrigin) - .execute() - .then(rows => rows.map(row => row.externalId!)), + } = result; + return stripNullOrUndefinedValues({ + ...software, + updateTime: new Date(+updateTime).getTime(), + addedTime: new Date(+addedTime).getTime(), + serviceProviders: serviceProviders ?? [], + similarSoftwares: similarExternalSoftwares, + userAndReferentCountByOrganization: {}, + authors: (softwareExternalData?.developers ?? []).map(dev => ({ + authorName: dev.name, + authorUrl: `https://www.wikidata.org/wiki/${dev.id}` + })), + officialWebsiteUrl: + softwareExternalData?.websiteUrl ?? + software.comptoirDuLibreSoftware?.external_resources.website, + codeRepositoryUrl: + softwareExternalData?.sourceUrl ?? + software.comptoirDuLibreSoftware?.external_resources.repository, + documentationUrl: softwareExternalData?.documentationUrl, + comptoirDuLibreServiceProviderCount: software.comptoirDuLibreSoftware?.providers.length ?? 0, + testUrl: testUrls[0]?.url, + parentWikidataSoftware: parentExternalData + }); + }), + getById: getBySoftwareId, + getByIdWithLinkedSoftwaresExternalIds: async softwareId => { + const software = await getBySoftwareId(softwareId); + if (!software) return; - countAddedByAgent: async ({ agentId }) => { - const { count } = await db - .selectFrom("softwares") - .select(qb => qb.fn.countAll().as("count")) - .where("addedByAgentId", "=", agentId) - .executeTakeFirstOrThrow(); - return +count; - }, - unreference: async ({ softwareId, reason, time }) => { - const { versionMin } = await db - .selectFrom("softwares") - .select("versionMin") - .where("id", "=", softwareId) - .executeTakeFirstOrThrow(); + const { parentSoftwareExternalId, similarSoftwaresExternalIds } = await db + .selectFrom("softwares as s") + .leftJoin("softwares__similar_software_external_datas as sim", "sim.softwareId", "s.id") + .select([ + "s.parentSoftwareWikidataId as parentSoftwareExternalId", + qb => + qb.fn + .jsonAgg(qb.ref("sim.similarExternalId")) + .filterWhere("sim.similarExternalId", "is not", null) + .$castTo() + .as("similarSoftwaresExternalIds") + ]) + .groupBy("s.id") + .where("id", "=", softwareId) + .executeTakeFirstOrThrow(); + + return { + software, + similarSoftwaresExternalIds: similarSoftwaresExternalIds ?? [], + parentSoftwareExternalId: parentSoftwareExternalId ?? undefined + }; + }, + getAll: (): Promise => + makeGetSoftwareBuilder(db) + .execute() + .then(async softwares => { + const userAndReferentCountByOrganization = await getUserAndReferentCountByOrganizationBySoftwareId( + db + ); + return softwares.map( + ({ + testUrls, + serviceProviders, + parentExternalData, + updateTime, + addedTime, + softwareExternalData, + similarExternalSoftwares, + ...software + }): Software => { + return stripNullOrUndefinedValues({ + ...software, + updateTime: new Date(+updateTime).getTime(), + addedTime: new Date(+addedTime).getTime(), + serviceProviders: serviceProviders ?? [], + similarSoftwares: similarExternalSoftwares, + // (similarSoftwares ?? []).map( + // (s): SimilarSoftware => ({ + // softwareName: + // typeof s.label === "string" ? s.label : Object.values(s.label)[0]!, + // softwareDescription: + // typeof s.label === "string" ? s.label : Object.values(s.label)[0]!, + // isInSill: true // TODO: check if this is true + // }) + // ) ?? [], + userAndReferentCountByOrganization: + userAndReferentCountByOrganization[software.softwareId] ?? {}, + authors: (softwareExternalData?.developers ?? []).map(dev => ({ + authorName: dev.name, + authorUrl: `https://www.wikidata.org/wiki/${dev.id}` + })), + officialWebsiteUrl: + softwareExternalData?.websiteUrl ?? + software.comptoirDuLibreSoftware?.external_resources.website ?? + undefined, + codeRepositoryUrl: + softwareExternalData?.sourceUrl ?? + software.comptoirDuLibreSoftware?.external_resources.repository ?? + undefined, + documentationUrl: softwareExternalData?.documentationUrl ?? undefined, + comptoirDuLibreServiceProviderCount: + software.comptoirDuLibreSoftware?.providers.length ?? 0, + testUrl: testUrls[0]?.url, + parentWikidataSoftware: parentExternalData ?? undefined + }); + } + ); + }), + getAllSillSoftwareExternalIds: async externalDataOrigin => + db + .selectFrom("softwares") + .select("externalId") + .where("externalDataOrigin", "=", externalDataOrigin) + .execute() + .then(rows => rows.map(row => row.externalId!)), - await db - .updateTable("softwares") - .set({ - dereferencing: JSON.stringify({ - reason, - time, - lastRecommendedVersion: versionMin + countAddedByAgent: async ({ agentId }) => { + const { count } = await db + .selectFrom("softwares") + .select(qb => qb.fn.countAll().as("count")) + .where("addedByAgentId", "=", agentId) + .executeTakeFirstOrThrow(); + return +count; + }, + unreference: async ({ softwareId, reason, time }) => { + const { versionMin } = await db + .selectFrom("softwares") + .select("versionMin") + .where("id", "=", softwareId) + .executeTakeFirstOrThrow(); + + await db + .updateTable("softwares") + .set({ + dereferencing: JSON.stringify({ + reason, + time, + lastRecommendedVersion: versionMin + }) }) - }) - .where("id", "=", softwareId) - .executeTakeFirstOrThrow(); - } -}); + .where("id", "=", softwareId) + .executeTakeFirstOrThrow(); + } + }; +}; const makeGetSoftwareBuilder = (db: Kysely) => db @@ -459,3 +455,45 @@ const getUserAndReferentCountByOrganizationBySoftwareId = async ( {} as UserAndReferentCountByOrganizationBySoftwareId ); }; + +const makeGetSoftwareById = + (db: Kysely) => + async (softwareId: number): Promise => + makeGetSoftwareBuilder(db) + .where("id", "=", softwareId) + .executeTakeFirst() + .then((result): Software | undefined => { + if (!result) return; + const { + testUrls, + serviceProviders, + parentExternalData, + updateTime, + addedTime, + softwareExternalData, + similarExternalSoftwares, + ...software + } = result; + return stripNullOrUndefinedValues({ + ...software, + updateTime: new Date(+updateTime).getTime(), + addedTime: new Date(+addedTime).getTime(), + serviceProviders: serviceProviders ?? [], + similarSoftwares: similarExternalSoftwares, + userAndReferentCountByOrganization: {}, + authors: (softwareExternalData?.developers ?? []).map(dev => ({ + authorName: dev.name, + authorUrl: `https://www.wikidata.org/wiki/${dev.id}` + })), + officialWebsiteUrl: + softwareExternalData?.websiteUrl ?? + software.comptoirDuLibreSoftware?.external_resources.website, + codeRepositoryUrl: + softwareExternalData?.sourceUrl ?? + software.comptoirDuLibreSoftware?.external_resources.repository, + documentationUrl: softwareExternalData?.documentationUrl, + comptoirDuLibreServiceProviderCount: software.comptoirDuLibreSoftware?.providers.length ?? 0, + testUrl: testUrls[0]?.url, + parentWikidataSoftware: parentExternalData + }); + }); diff --git a/api/src/core/adapters/dbApi/kysely/migrations/1719576701920_add-compiled-tables.ts b/api/src/core/adapters/dbApi/kysely/migrations/1719576701920_add-compiled-tables.ts index 14eb2765..60b21b0d 100644 --- a/api/src/core/adapters/dbApi/kysely/migrations/1719576701920_add-compiled-tables.ts +++ b/api/src/core/adapters/dbApi/kysely/migrations/1719576701920_add-compiled-tables.ts @@ -3,7 +3,9 @@ import type { Kysely } from "kysely"; export async function up(db: Kysely): Promise { await db.schema .createTable("compiled_softwares") - .addColumn("softwareId", "integer", col => col.notNull().references("softwares.id").onDelete("cascade")) + .addColumn("softwareId", "integer", col => + col.notNull().unique().references("softwares.id").onDelete("cascade") + ) .addColumn("serviceProviders", "jsonb", col => col.notNull()) .addColumn("comptoirDuLibreSoftware", "jsonb") .addColumn("annuaireCnllServiceProviders", "jsonb") diff --git a/api/src/core/adapters/fetchExternalData.test.ts b/api/src/core/adapters/fetchExternalData.test.ts new file mode 100644 index 00000000..1515e827 --- /dev/null +++ b/api/src/core/adapters/fetchExternalData.test.ts @@ -0,0 +1,483 @@ +import { Kysely, sql } from "kysely"; +import { describe, it, beforeEach, expect } from "vitest"; +import { expectToEqual, testPgUrl } from "../../tools/test.helpers"; +import { DbApiV2 } from "../ports/DbApiV2"; +import { SoftwareFormData } from "../usecases/readWriteSillData"; +import { comptoirDuLibreApi } from "./comptoirDuLibreApi"; +import { createKyselyPgDbApi } from "./dbApi/kysely/createPgDbApi"; +import type { Database } from "./dbApi/kysely/kysely.database"; +import { createPgDialect } from "./dbApi/kysely/kysely.dialect"; +import { makeFetchAndSaveSoftwareExtraData } from "./fetchExternalData"; +import { getCnllPrestatairesSill } from "./getCnllPrestatairesSill"; +import { getServiceProviders } from "./getServiceProviders"; +import { createGetSoftwareLatestVersion } from "./getSoftwareLatestVersion"; +import { getWikidataSoftware } from "./wikidata/getWikidataSoftware"; + +const craSoftwareFormData = { + softwareType: { + type: "stack" + }, + externalId: "Q118629387", + comptoirDuLibreId: 1, + softwareName: "Create react app", + softwareDescription: "To create React apps.", + softwareLicense: "MIT", + softwareMinimalVersion: "1.0.0", + isPresentInSupportContract: true, + isFromFrenchPublicService: true, + similarSoftwareExternalDataIds: ["Q111590996" /* viteJS */], + softwareLogoUrl: "https://example.com/logo.png", + softwareKeywords: ["Productivity", "Task", "Management"], + doRespectRgaa: true +} satisfies SoftwareFormData; + +const apacheSoftwareId = 6; + +const insertApacheWithCorrectId = async (db: Kysely, agentId: number) => { + await sql` + INSERT INTO public.softwares + (id, "softwareType", "externalId", "externalDataOrigin", + "comptoirDuLibreId", name, description, license, "versionMin", + "isPresentInSupportContract", "isFromFrenchPublicService", "logoUrl", + keywords, "doRespectRgaa", "isStillInObservation", + "parentSoftwareWikidataId", "catalogNumeriqueGouvFrId", "workshopUrls", + "testUrls", categories, "generalInfoMd", "addedByAgentId", + dereferencing, "referencedSinceTime", "updateTime") + VALUES (${apacheSoftwareId}, + '{"os": {"ios": false, "mac": false, "linux": true, "android": false, "windows": false}, "type": "desktop/mobile"}', + 'Q11354', 'wikidata', 3737, 'Apache HTTP Server', + 'Serveur Web & Reverse Proxy', 'Apache-2.0', '212', true, false, + 'https://sill.code.gouv.fr/logo/apache-http.png', + '["serveur", "http", "web", "server", "apache"]', false, false, + null, null, '[]', '[]', '[]', null, ${agentId}, null, 1728462232094, + 1728462232094); + `.execute(db); +}; + +describe("fetches software extra data (from different providers)", () => { + let fetchAndSaveSoftwareExtraData: ReturnType; + let dbApi: DbApiV2; + let db: Kysely; + let craSoftwareId: number; + + beforeEach(async () => { + db = new Kysely({ dialect: createPgDialect(testPgUrl) }); + await db.deleteFrom("compiled_softwares").execute(); + await db.deleteFrom("software_external_datas").execute(); + await db.deleteFrom("software_users").execute(); + await db.deleteFrom("software_referents").execute(); + await db.deleteFrom("softwares").execute(); + await db.deleteFrom("agents").execute(); + + dbApi = createKyselyPgDbApi(db); + + const agentId = await dbApi.agent.add({ + email: "myuser@example.com", + organization: "myorg", + about: "my about", + isPublic: false + }); + + craSoftwareId = await dbApi.software.create({ + formData: craSoftwareFormData, + externalDataOrigin: "wikidata", + agentId + }); + + await insertApacheWithCorrectId(db, agentId); + + const { getSoftwareLatestVersion } = createGetSoftwareLatestVersion({ + githubPersonalAccessTokenForApiRateLimit: "" + }); + + fetchAndSaveSoftwareExtraData = makeFetchAndSaveSoftwareExtraData({ + dbApi, + getSoftwareExternalData: getWikidataSoftware, + comptoirDuLibreApi, + getCnllPrestatairesSill: getCnllPrestatairesSill, + getServiceProviders: getServiceProviders, + getSoftwareLatestVersion + }); + }); + + it("does nothing if the software is not found", async () => { + const softwareExternalDatas = await db.selectFrom("software_external_datas").selectAll().execute(); + expectToEqual(softwareExternalDatas, []); + + await fetchAndSaveSoftwareExtraData(404, {}); + + const updatedSoftwareExternalDatas = await db.selectFrom("software_external_datas").selectAll().execute(); + expectToEqual(updatedSoftwareExternalDatas, []); + }); + + it( + "gets software external data and saves it, and does not save other extra data if there is nothing relevant", + async () => { + const softwareExternalDatas = await db.selectFrom("software_external_datas").selectAll().execute(); + expect(softwareExternalDatas).toHaveLength(0); + + await fetchAndSaveSoftwareExtraData(craSoftwareId, {}); + + const updatedSoftwareExternalDatas = await db.selectFrom("software_external_datas").selectAll().execute(); + expectToEqual(updatedSoftwareExternalDatas, [ + { + description: "A framwork for creating react SPA that uses webpack as bundler", + developers: [], + documentationUrl: null, + externalDataOrigin: "wikidata", + externalId: craSoftwareFormData.externalId, + framaLibreId: null, + isLibreSoftware: true, + label: "create-react-app", + license: "MIT licence", + logoUrl: null, + sourceUrl: "https://github.com/facebook/create-react-app", + websiteUrl: "https://create-react-app.dev/" + }, + { + description: "open-source JavaScript module bundler", + developers: [ + { + id: "Q58482636", + name: "Evan You" + } + ], + documentationUrl: "https://vitejs.dev/guide/", + externalDataOrigin: "wikidata", + externalId: "Q111590996", + framaLibreId: null, + isLibreSoftware: true, + label: "Vite", + license: "MIT licence", + logoUrl: + "//upload.wikimedia.org/wikipedia/commons/thumb/f/f1/Vitejs-logo.svg/220px-Vitejs-logo.svg.png", + sourceUrl: "https://github.com/vitejs/vite", + websiteUrl: "https://vitejs.dev/" + } + ]); + + const otherExtraData = await db.selectFrom("compiled_softwares").selectAll().execute(); + expectToEqual(otherExtraData, []); + }, + { timeout: 10_000 } + ); + + it( + "gets software external data and saves it, and save other extra data", + async () => { + const softwareExternalDatas = await db.selectFrom("software_external_datas").selectAll().execute(); + expect(softwareExternalDatas).toHaveLength(0); + + await fetchAndSaveSoftwareExtraData(apacheSoftwareId, {}); + + const updatedSoftwareExternalDatas = await db.selectFrom("software_external_datas").selectAll().execute(); + expectToEqual(updatedSoftwareExternalDatas, [ + { + description: { + en: "open-source web server software", + fr: "serveur web sous licence libre" + }, + developers: [ + { + id: "Q489709", + name: "Apache Software Foundation" + } + ], + documentationUrl: null, + externalDataOrigin: "wikidata", + externalId: "Q11354", + framaLibreId: null, + isLibreSoftware: false, + label: "Apache HTTP Server", + license: "Apache License v2.0", + logoUrl: + "//upload.wikimedia.org/wikipedia/commons/thumb/1/10/Apache_HTTP_server_logo_%282019-present%29.svg/220px-Apache_HTTP_server_logo_%282019-present%29.svg.png", + sourceUrl: "https://github.com/apache/httpd", + websiteUrl: "https://httpd.apache.org/" + } + ]); + + const otherExtraData = await db.selectFrom("compiled_softwares").selectAll().execute(); + + expectToEqual(otherExtraData, [ + { + softwareId: apacheSoftwareId, + comptoirDuLibreSoftware: null, + annuaireCnllServiceProviders: [ + { + url: "https://annuaire.cnll.fr/societes/434940763", + name: "YPOK", + siren: "434940763" + }, + { + url: "https://annuaire.cnll.fr/societes/538420753", + name: "INNO3", + siren: "538420753" + }, + { + url: "https://annuaire.cnll.fr/societes/522588979", + name: "COMBODO", + siren: "522588979" + }, + { + url: "https://annuaire.cnll.fr/societes/452887441", + name: "APITUX", + siren: "452887441" + }, + { + url: "https://annuaire.cnll.fr/societes/820266211", + name: "POLLEN ROBOTICS", + siren: "820266211" + }, + { + url: "https://annuaire.cnll.fr/societes/437827959", + name: "ézéo", + siren: "437827959" + }, + { + url: "https://annuaire.cnll.fr/societes/483494589", + name: "CENTREON", + siren: "483494589" + }, + { + url: "https://annuaire.cnll.fr/societes/524457520", + name: "Lan2Net", + siren: "524457520" + }, + { + url: "https://annuaire.cnll.fr/societes/824429708", + name: "WORTEKS", + siren: "824429708" + }, + { + url: "https://annuaire.cnll.fr/societes/499277713", + name: "PLICIWEB SOLUTIONS", + siren: "499277713" + }, + { + url: "https://annuaire.cnll.fr/societes/450656731", + name: "Ryxéo", + siren: "450656731" + }, + { + url: "https://annuaire.cnll.fr/societes/813965662", + name: "DEBAMAX", + siren: "813965662" + }, + { + url: "https://annuaire.cnll.fr/societes/495075079", + name: "Telnowedge", + siren: "495075079" + }, + { + url: "https://annuaire.cnll.fr/societes/449989573", + name: "WebGeoDataVore", + siren: "449989573" + }, + { + url: "https://annuaire.cnll.fr/societes/821345345", + name: "Wiki Valley", + siren: "821345345" + }, + { + url: "https://annuaire.cnll.fr/societes/490932308", + name: "ALTER WAY", + siren: "490932308" + }, + { + url: "https://annuaire.cnll.fr/societes/443170139", + name: "Entr'ouvert", + siren: "443170139" + }, + { + url: "https://annuaire.cnll.fr/societes/451952295", + name: "EVOLIX", + siren: "451952295" + } + ], + serviceProviders: [ + { + name: "Oziolab", + cdlUrl: "https://comptoir-du-libre.org/fr/users/4075", + website: "https://www.oziolab.fr" + }, + { + name: "Lan2Net", + siren: "524457520", + cnllUrl: "https://annuaire.cnll.fr/societes/524457520" + }, + { + name: "DEBAMAX", + siren: "813965662", + cnllUrl: "https://annuaire.cnll.fr/societes/813965662" + }, + { + name: "Wiki Valley", + siren: "821345345", + cnllUrl: "https://annuaire.cnll.fr/societes/821345345" + }, + { + name: "INNO3", + siren: "538420753", + cnllUrl: "https://annuaire.cnll.fr/societes/538420753" + }, + { + name: "WebGeoDataVore", + siren: "449989573", + cnllUrl: "https://annuaire.cnll.fr/societes/449989573" + }, + { + name: "PLICIWEB SOLUTIONS", + siren: "499277713", + cnllUrl: "https://annuaire.cnll.fr/societes/499277713" + }, + { + name: "Probesys", + cdlUrl: "https://comptoir-du-libre.org/fr/users/398", + website: "https://www.probesys.com" + }, + { + name: "Ionzee", + cdlUrl: "https://comptoir-du-libre.org/fr/users/3681", + website: "" + }, + { + name: "Empreinte Digitale SCOP SA", + cdlUrl: "https://comptoir-du-libre.org/fr/users/2225", + website: "https://www.empreintedigitale.fr" + }, + { + name: "APITUX", + siren: "452887441", + cnllUrl: "https://annuaire.cnll.fr/societes/452887441" + }, + { + name: "CENTREON", + siren: "483494589", + cnllUrl: "https://annuaire.cnll.fr/societes/483494589" + }, + { + name: "Bearstech", + cdlUrl: "https://comptoir-du-libre.org/fr/users/3960", + website: "https://bearstech.com" + }, + { + name: "CIGALL", + cdlUrl: "https://comptoir-du-libre.org/fr/users/270", + website: "Https://www.cigall.fr" + }, + { + name: "ézéo", + siren: "437827959", + cnllUrl: "https://annuaire.cnll.fr/societes/437827959" + }, + { + name: "ALTER WAY", + siren: "490932308", + cnllUrl: "https://annuaire.cnll.fr/societes/490932308" + }, + { + name: "Azure Informatique", + cdlUrl: "https://comptoir-du-libre.org/fr/users/3962", + website: "https://www.azure-informatique.fr" + }, + { + name: "Your Own Net", + cdlUrl: "https://comptoir-du-libre.org/fr/users/162", + website: "https://yourownnet.net" + }, + { + name: "EDISSYUM Consulting", + cdlUrl: "https://comptoir-du-libre.org/fr/users/345", + website: "http://www.edissyum.com" + }, + { + name: "EVOLIX", + siren: "451952295", + cnllUrl: "https://annuaire.cnll.fr/societes/451952295" + }, + { + name: "Talan", + cdlUrl: "https://comptoir-du-libre.org/fr/users/4021", + website: "https://talan.com" + }, + { + name: "Microlinux", + cdlUrl: "https://comptoir-du-libre.org/fr/users/295", + website: "https://www.microlinux.fr" + }, + { + name: "decaris", + cdlUrl: "https://comptoir-du-libre.org/fr/users/643", + website: "https://mywebdatahome.com" + }, + { + name: "SIGMAZ Consilium", + cdlUrl: "https://comptoir-du-libre.org/fr/users/3893", + website: "https://sigmaz-consilium.fr/" + }, + { + name: "Ryxéo", + siren: "450656731", + cnllUrl: "https://annuaire.cnll.fr/societes/450656731" + }, + { + name: "Telnowedge", + siren: "495075079", + cnllUrl: "https://annuaire.cnll.fr/societes/495075079" + }, + { + name: "APLOSE", + cdlUrl: "https://comptoir-du-libre.org/fr/users/165", + website: "https://www.aplose.fr" + }, + { + name: "TEICEE", + cdlUrl: "https://comptoir-du-libre.org/fr/users/3838", + website: "https://www.teicee.com" + }, + { + name: "CAP-REL", + cdlUrl: "https://comptoir-du-libre.org/fr/users/3216", + website: "https://cap-rel.fr" + }, + { + name: "YPOK", + siren: "434940763", + cnllUrl: "https://annuaire.cnll.fr/societes/434940763" + }, + { + name: "Entr'ouvert", + siren: "443170139", + cnllUrl: "https://annuaire.cnll.fr/societes/443170139" + }, + { + name: "COMBODO", + siren: "522588979", + cnllUrl: "https://annuaire.cnll.fr/societes/522588979" + }, + { + name: "AUKFOOD", + cdlUrl: "https://comptoir-du-libre.org/fr/users/3288", + website: "https://www.aukfood.fr" + }, + { + name: "POLLEN ROBOTICS", + siren: "820266211", + cnllUrl: "https://annuaire.cnll.fr/societes/820266211" + }, + { + name: "WORTEKS", + siren: "824429708", + cdlUrl: "https://comptoir-du-libre.org/fr/users/255", + cnllUrl: "https://annuaire.cnll.fr/societes/824429708", + website: "https://www.worteks.com/fr/" + } + ], + latestVersion: null + } + ]); + }, + { timeout: 10_000 } + ); +}); diff --git a/api/src/core/adapters/fetchExternalData.ts b/api/src/core/adapters/fetchExternalData.ts new file mode 100644 index 00000000..600e5665 --- /dev/null +++ b/api/src/core/adapters/fetchExternalData.ts @@ -0,0 +1,148 @@ +import type { ComptoirDuLibreApi } from "../ports/ComptoirDuLibreApi"; +import { DbApiV2, OtherSoftwareExtraData } from "../ports/DbApiV2"; +import type { GetCnllPrestatairesSill } from "../ports/GetCnllPrestatairesSill"; +import { GetServiceProviders } from "../ports/GetServiceProviders"; +import type { GetSoftwareExternalData, SoftwareExternalData } from "../ports/GetSoftwareExternalData"; +import type { GetSoftwareLatestVersion } from "../ports/GetSoftwareLatestVersion"; +import { Software } from "../usecases/readWriteSillData"; +import { PgComptoirDuLibre } from "./dbApi/kysely/kysely.database"; + +type ExternalId = string; +type SoftwareExternalDataCacheBySoftwareId = Partial>; + +type FetchOtherExternalDataDependencies = { + getCnllPrestatairesSill: GetCnllPrestatairesSill; + comptoirDuLibreApi: ComptoirDuLibreApi; + getSoftwareLatestVersion: GetSoftwareLatestVersion; + getServiceProviders: GetServiceProviders; +}; + +export const makeFetchAndSaveSoftwareExtraData = ({ + getSoftwareExternalData, + dbApi, + ...otherExternalDataDeps +}: FetchOtherExternalDataDependencies & { + getSoftwareExternalData: GetSoftwareExternalData; + dbApi: DbApiV2; +}) => { + const getOtherExternalData = makeGetOtherExternalData(otherExternalDataDeps); + const getSoftwareExternalDataAndSaveIt = makeGetSoftwareExternalData({ dbApi, getSoftwareExternalData }); + + return async (softwareId: number, softwareExternalDataCache: SoftwareExternalDataCacheBySoftwareId) => { + const data = await dbApi.software.getByIdWithLinkedSoftwaresExternalIds(softwareId); + if (!data) return; + + const { software, similarSoftwaresExternalIds, parentSoftwareExternalId } = data; + + if (software.externalId) await getSoftwareExternalDataAndSaveIt(software.externalId, softwareExternalDataCache); + + if (parentSoftwareExternalId) + await getSoftwareExternalDataAndSaveIt(parentSoftwareExternalId, softwareExternalDataCache); + + if (similarSoftwaresExternalIds.length > 0) + await Promise.all( + similarSoftwaresExternalIds.map(similarExternalId => + getSoftwareExternalDataAndSaveIt(similarExternalId, softwareExternalDataCache) + ) + ); + + const existingOtherSoftwareExtraData = await dbApi.otherSoftwareExtraData.getBySoftwareId(software.softwareId); + const newOtherSoftwareExtraData = await getOtherExternalData(software, existingOtherSoftwareExtraData); + console.log("newOtherSoftwareExtraData : ", newOtherSoftwareExtraData); + + if (newOtherSoftwareExtraData) await dbApi.otherSoftwareExtraData.save(newOtherSoftwareExtraData); + }; +}; + +const makeGetSoftwareExternalData = + (deps: { getSoftwareExternalData: GetSoftwareExternalData; dbApi: DbApiV2 }) => + async (externalId: ExternalId, cache: SoftwareExternalDataCacheBySoftwareId) => { + if (cache[externalId]) return cache[externalId]; + + const softwareExternalData = await deps.getSoftwareExternalData(externalId); + if (softwareExternalData) { + await deps.dbApi.softwareExternalData.save(softwareExternalData); + cache[externalId] = softwareExternalData; + } + }; + +const makeGetOtherExternalData = + (deps: FetchOtherExternalDataDependencies) => + async ( + software: Software, + existingOtherSoftwareExtraData: OtherSoftwareExtraData | undefined + ): Promise => { + const [serviceProvidersBySoftwareId, cnllPrestatairesSill, latestVersion] = await Promise.all([ + deps.getServiceProviders(), + deps.getCnllPrestatairesSill(), + software.codeRepositoryUrl ? deps.getSoftwareLatestVersion(software.codeRepositoryUrl, "quick") : undefined + ]); + + const comptoirDuLibreSoftware = await getNewComptoirDuLibre({ + comptoirDuLibreApi: deps.comptoirDuLibreApi, + software, + otherSoftwareExtraDataInCache: existingOtherSoftwareExtraData + }); + + console.log("DATA GATHERED : "); + console.log({ + softwareId: software.softwareId, + softwareName: software.softwareName + }); + + const otherSoftwareExtraData: OtherSoftwareExtraData = { + softwareId: software.softwareId, + serviceProviders: serviceProvidersBySoftwareId[software.softwareId.toString()] ?? [], + comptoirDuLibreSoftware, + annuaireCnllServiceProviders: + cnllPrestatairesSill + .find(({ sill_id }) => sill_id === software.softwareId) + ?.prestataires.map(({ nom, siren, url }) => ({ + name: nom, + siren, + url + })) ?? null, + latestVersion: latestVersion ?? null + }; + + if ( + otherSoftwareExtraData.serviceProviders.length === 0 && + otherSoftwareExtraData.comptoirDuLibreSoftware === null && + otherSoftwareExtraData.annuaireCnllServiceProviders === null && + otherSoftwareExtraData.latestVersion === null + ) + return; + + return otherSoftwareExtraData; + }; + +const getNewComptoirDuLibre = async ({ + software, + comptoirDuLibreApi, + otherSoftwareExtraDataInCache +}: { + comptoirDuLibreApi: ComptoirDuLibreApi; + software: Software; + otherSoftwareExtraDataInCache: OtherSoftwareExtraData | undefined; +}): Promise => { + if (software.comptoirDuLibreId === undefined) return null; + const comptoirDuLibre = await comptoirDuLibreApi.getComptoirDuLibre(); + const comptoirDuLibreSoftware = comptoirDuLibre.softwares.find( + comptoirDuLibreSoftware => comptoirDuLibreSoftware.id === software.comptoirDuLibreId + ); + console.log("number of softwares in comptoir du libre : ", comptoirDuLibre.softwares.length); + console.log("comptoirDuLibreId : ", software.comptoirDuLibreId); + console.log("first : ", { name: comptoirDuLibre.softwares[0].name, id: comptoirDuLibre.softwares[0].id }); + console.log("comptoirDuLibreSoftware : ", comptoirDuLibreSoftware); + if (!comptoirDuLibreSoftware) return null; + + const [logoUrl, keywords] = + otherSoftwareExtraDataInCache?.comptoirDuLibreSoftware?.id === comptoirDuLibreSoftware.id + ? [] + : await Promise.all([ + comptoirDuLibreApi.getIconUrl({ comptoirDuLibreId: comptoirDuLibreSoftware.id }), + comptoirDuLibreApi.getKeywords({ comptoirDuLibreId: comptoirDuLibreSoftware.id }) + ]); + + return { ...comptoirDuLibreSoftware, logoUrl, keywords }; +}; diff --git a/api/src/core/bootstrap.ts b/api/src/core/bootstrap.ts index 1098de89..fc4838a5 100644 --- a/api/src/core/bootstrap.ts +++ b/api/src/core/bootstrap.ts @@ -118,6 +118,9 @@ export async function bootstrapCore( if (doPerPerformPeriodicalCompilation) { console.log("TODO: doPerPerformPeriodicalCompilation"); + // setTimeout(() => { + // compileData(); + // }); } // await dispatch( // usecases.readWriteSillData.protectedThunks.initialize({ diff --git a/api/src/core/ports/DbApiV2.ts b/api/src/core/ports/DbApiV2.ts index 00c1c934..5ee7729b 100644 --- a/api/src/core/ports/DbApiV2.ts +++ b/api/src/core/ports/DbApiV2.ts @@ -1,9 +1,17 @@ import type { Database } from "../adapters/dbApi/kysely/kysely.database"; -import type { Agent, Instance, InstanceFormData, Software, SoftwareFormData } from "../usecases/readWriteSillData"; +import type { + Agent, + Instance, + InstanceFormData, + ServiceProvider, + Software, + SoftwareFormData +} from "../usecases/readWriteSillData"; import type { OmitFromExisting } from "../utils"; import type { CompiledData } from "./CompileData"; +import { ComptoirDuLibre } from "./ComptoirDuLibreApi"; -import type { ExternalDataOrigin } from "./GetSoftwareExternalData"; +import type { ExternalDataOrigin, SoftwareExternalData } from "./GetSoftwareExternalData"; export type WithAgentId = { agentId: number }; @@ -22,14 +30,49 @@ export interface SoftwareRepository { ) => Promise; getAll: () => Promise; getById: (id: number) => Promise; + getByIdWithLinkedSoftwaresExternalIds: (id: number) => Promise< + | { + software: Software; + similarSoftwaresExternalIds: string[]; + parentSoftwareExternalId: string | undefined; + } + | undefined + >; getByName: (name: string) => Promise; countAddedByAgent: (params: { agentId: number }) => Promise; getAllSillSoftwareExternalIds: (externalDataOrigin: ExternalDataOrigin) => Promise; unreference: (params: { softwareId: number; reason: string; time: number }) => Promise; } +export interface SoftwareExternalDataRepository { + save: (softwareExternalData: SoftwareExternalData) => Promise; +} + +type CnllPrestataire = { + name: string; + siren: string; + url: string; +}; + +export type OtherSoftwareExtraData = { + softwareId: number; + serviceProviders: ServiceProvider[]; + comptoirDuLibreSoftware: ComptoirDuLibre.Software | null; + annuaireCnllServiceProviders: CnllPrestataire[] | null; + latestVersion: { semVer: string; publicationTime: number } | null; +}; + +export interface OtherSoftwareExtraDataRepository { + save: (otherSoftwareExtraData: OtherSoftwareExtraData) => Promise; + getBySoftwareId: (softwareId: number) => Promise; +} + export interface InstanceRepository { - create: (params: { formData: InstanceFormData } & WithAgentId) => Promise; + create: ( + params: { + formData: InstanceFormData; + } & WithAgentId + ) => Promise; update: (params: { formData: InstanceFormData; instanceId: number }) => Promise; countAddedByAgent: (params: { agentId: number }) => Promise; getAll: () => Promise; @@ -68,6 +111,8 @@ export interface SoftwareUserRepository { export type DbApiV2 = { software: SoftwareRepository; + softwareExternalData: SoftwareExternalDataRepository; + otherSoftwareExtraData: OtherSoftwareExtraDataRepository; instance: InstanceRepository; agent: AgentRepository; softwareReferent: SoftwareReferentRepository; diff --git a/api/src/core/usecases/readWriteSillData/types.ts b/api/src/core/usecases/readWriteSillData/types.ts index 0fa8b8df..f72f4ae3 100644 --- a/api/src/core/usecases/readWriteSillData/types.ts +++ b/api/src/core/usecases/readWriteSillData/types.ts @@ -69,7 +69,12 @@ export namespace Software { export namespace SimilarSoftware { export type ExternalSoftwareData = { isInSill: false } & SimilarSoftwareExternalData; - export type Sill = { isInSill: true; softwareName: string; softwareDescription: string }; + export type Sill = { + isInSill: true; + softwareId: number; + softwareName: string; + softwareDescription: string; + } & SimilarSoftwareExternalData; } } From 93917a53ee42a208ed701c6a4117a55022aa1c3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Burkard?= <22095555+JeromeBu@users.noreply.github.com> Date: Wed, 9 Oct 2024 12:11:34 +0200 Subject: [PATCH 10/19] add some logs when fetching external data --- api/src/core/adapters/comptoirDuLibreApi.ts | 5 +++++ api/src/core/adapters/fetchExternalData.ts | 12 +----------- api/src/core/adapters/getCnllPrestatairesSill.ts | 3 +++ api/src/core/adapters/getServiceProviders.ts | 3 +++ .../core/adapters/wikidata/getWikidataSoftware.ts | 3 +++ 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/api/src/core/adapters/comptoirDuLibreApi.ts b/api/src/core/adapters/comptoirDuLibreApi.ts index ccc72c6b..1e406222 100644 --- a/api/src/core/adapters/comptoirDuLibreApi.ts +++ b/api/src/core/adapters/comptoirDuLibreApi.ts @@ -9,6 +9,7 @@ export const comptoirDuLibreApi: ComptoirDuLibreApi = { "getComptoirDuLibre": memoize( async () => { try { + console.info("Fetching comptoir du libre..."); const res = await fetch(url); if (res.status !== 200) { @@ -26,6 +27,8 @@ export const comptoirDuLibreApi: ComptoirDuLibreApi = { "number_of_software": 0, "softwares": [] }; + } finally { + console.info("Fetching comptoir du libre finished"); } }, { "promise": true } @@ -34,6 +37,7 @@ export const comptoirDuLibreApi: ComptoirDuLibreApi = { let imgSrc: string | undefined; try { + console.info(`Fetching comptoir du libre icon, for comptoirDuLibreId : ${comptoirDuLibreId}`); const body = await fetch(`https://comptoir-du-libre.org/fr/softwares/${comptoirDuLibreId}`).then(r => r.text() ); @@ -55,6 +59,7 @@ export const comptoirDuLibreApi: ComptoirDuLibreApi = { let $: CheerioAPI; try { + console.info(`Fetching comptoir du libre keywords, for comptoirDuLibreId : ${comptoirDuLibreId}`); const body = await fetch(`https://comptoir-du-libre.org/fr/softwares/${comptoirDuLibreId}`).then(r => r.text() ); diff --git a/api/src/core/adapters/fetchExternalData.ts b/api/src/core/adapters/fetchExternalData.ts index 600e5665..c5629cf0 100644 --- a/api/src/core/adapters/fetchExternalData.ts +++ b/api/src/core/adapters/fetchExternalData.ts @@ -48,7 +48,6 @@ export const makeFetchAndSaveSoftwareExtraData = ({ const existingOtherSoftwareExtraData = await dbApi.otherSoftwareExtraData.getBySoftwareId(software.softwareId); const newOtherSoftwareExtraData = await getOtherExternalData(software, existingOtherSoftwareExtraData); - console.log("newOtherSoftwareExtraData : ", newOtherSoftwareExtraData); if (newOtherSoftwareExtraData) await dbApi.otherSoftwareExtraData.save(newOtherSoftwareExtraData); }; @@ -84,12 +83,6 @@ const makeGetOtherExternalData = otherSoftwareExtraDataInCache: existingOtherSoftwareExtraData }); - console.log("DATA GATHERED : "); - console.log({ - softwareId: software.softwareId, - softwareName: software.softwareName - }); - const otherSoftwareExtraData: OtherSoftwareExtraData = { softwareId: software.softwareId, serviceProviders: serviceProvidersBySoftwareId[software.softwareId.toString()] ?? [], @@ -130,10 +123,7 @@ const getNewComptoirDuLibre = async ({ const comptoirDuLibreSoftware = comptoirDuLibre.softwares.find( comptoirDuLibreSoftware => comptoirDuLibreSoftware.id === software.comptoirDuLibreId ); - console.log("number of softwares in comptoir du libre : ", comptoirDuLibre.softwares.length); - console.log("comptoirDuLibreId : ", software.comptoirDuLibreId); - console.log("first : ", { name: comptoirDuLibre.softwares[0].name, id: comptoirDuLibre.softwares[0].id }); - console.log("comptoirDuLibreSoftware : ", comptoirDuLibreSoftware); + if (!comptoirDuLibreSoftware) return null; const [logoUrl, keywords] = diff --git a/api/src/core/adapters/getCnllPrestatairesSill.ts b/api/src/core/adapters/getCnllPrestatairesSill.ts index 0aac888c..c75a7380 100644 --- a/api/src/core/adapters/getCnllPrestatairesSill.ts +++ b/api/src/core/adapters/getCnllPrestatairesSill.ts @@ -9,6 +9,7 @@ const url = "https://annuaire.cnll.fr/api/prestataires-sill.json"; export const getCnllPrestatairesSill: GetCnllPrestatairesSill = memoize( async () => { try { + console.info("Fetching cnll prestataires sill"); const res = await fetch(url, { "agent": new https.Agent({ "rejectUnauthorized": false }) }); if (res.status !== 200) { @@ -22,6 +23,8 @@ export const getCnllPrestatairesSill: GetCnllPrestatairesSill = memoize( } catch (error) { console.error(`Failed to fetch or parse ${url}: ${String(error)}`); return []; + } finally { + console.info("Fetching cnll prestataires sill finished"); } }, { "promise": true } diff --git a/api/src/core/adapters/getServiceProviders.ts b/api/src/core/adapters/getServiceProviders.ts index f4010f6b..25f9a72f 100644 --- a/api/src/core/adapters/getServiceProviders.ts +++ b/api/src/core/adapters/getServiceProviders.ts @@ -36,6 +36,7 @@ const url = "https://code.gouv.fr/data/sill-prestataires.json"; export const getServiceProviders: GetServiceProviders = memoize( async () => { try { + console.info("Fetching service providers"); const res = await fetch(url); if (res.status !== 200) { @@ -61,6 +62,8 @@ export const getServiceProviders: GetServiceProviders = memoize( } catch (error) { console.error(`Failed to fetch or parse ${url}: ${String(error)}`); return {}; + } finally { + console.info("Fetching service providers finished"); } }, { "promise": true } diff --git a/api/src/core/adapters/wikidata/getWikidataSoftware.ts b/api/src/core/adapters/wikidata/getWikidataSoftware.ts index e76dbc62..d3d72c89 100644 --- a/api/src/core/adapters/wikidata/getWikidataSoftware.ts +++ b/api/src/core/adapters/wikidata/getWikidataSoftware.ts @@ -242,6 +242,7 @@ export class WikidataFetchError extends Error { } async function fetchEntity(wikidataId: string): Promise<{ entity: Entity }> { + console.info(`Fetching wikidata, for wikidataId : ${wikidataId}`); const res = await fetch(`https://www.wikidata.org/wiki/Special:EntityData/${wikidataId}.json`).catch( () => undefined ); @@ -263,6 +264,8 @@ async function fetchEntity(wikidataId: string): Promise<{ entity: Entity }> { const entity = Object.values(json["entities"])[0] as Entity; + console.info(`Fetching wikidata finished, for wikidataId : ${wikidataId}`); + return { entity }; } From 05742fb66b73dd4ac3404ee893602f0848e01be9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Burkard?= <22095555+JeromeBu@users.noreply.github.com> Date: Wed, 9 Oct 2024 15:10:16 +0200 Subject: [PATCH 11/19] use fetchExternalData in bootstrap --- api/src/core/adapters/fetchExternalData.ts | 45 ++++++++++++++----- .../adapters/wikidata/getWikidataSoftware.ts | 8 ++-- api/src/core/bootstrap.ts | 43 +++++++++++------- api/src/core/ports/DbApi.ts | 3 -- api/src/rpc/start.ts | 3 +- 5 files changed, 66 insertions(+), 36 deletions(-) diff --git a/api/src/core/adapters/fetchExternalData.ts b/api/src/core/adapters/fetchExternalData.ts index c5629cf0..b44cbae0 100644 --- a/api/src/core/adapters/fetchExternalData.ts +++ b/api/src/core/adapters/fetchExternalData.ts @@ -17,14 +17,16 @@ type FetchOtherExternalDataDependencies = { getServiceProviders: GetServiceProviders; }; +type FetchAndSaveSoftwareExtraDataDependencies = FetchOtherExternalDataDependencies & { + getSoftwareExternalData: GetSoftwareExternalData; + dbApi: DbApiV2; +}; + export const makeFetchAndSaveSoftwareExtraData = ({ getSoftwareExternalData, dbApi, ...otherExternalDataDeps -}: FetchOtherExternalDataDependencies & { - getSoftwareExternalData: GetSoftwareExternalData; - dbApi: DbApiV2; -}) => { +}: FetchAndSaveSoftwareExtraDataDependencies) => { const getOtherExternalData = makeGetOtherExternalData(otherExternalDataDeps); const getSoftwareExternalDataAndSaveIt = makeGetSoftwareExternalData({ dbApi, getSoftwareExternalData }); @@ -33,18 +35,24 @@ export const makeFetchAndSaveSoftwareExtraData = ({ if (!data) return; const { software, similarSoftwaresExternalIds, parentSoftwareExternalId } = data; + console.log(`🚀${software.softwareName}`); - if (software.externalId) await getSoftwareExternalDataAndSaveIt(software.externalId, softwareExternalDataCache); + if (software.externalId) { + console.log(" • Soft: ", software.softwareName, " - Own wiki : ", software.externalId); + await getSoftwareExternalDataAndSaveIt(software.externalId, softwareExternalDataCache); + } - if (parentSoftwareExternalId) + if (parentSoftwareExternalId) { + console.log(" • Parent wiki : ", parentSoftwareExternalId); await getSoftwareExternalDataAndSaveIt(parentSoftwareExternalId, softwareExternalDataCache); + } - if (similarSoftwaresExternalIds.length > 0) - await Promise.all( - similarSoftwaresExternalIds.map(similarExternalId => - getSoftwareExternalDataAndSaveIt(similarExternalId, softwareExternalDataCache) - ) - ); + if (similarSoftwaresExternalIds.length > 0) { + for (const similarExternalId of similarSoftwaresExternalIds) { + console.log(" • Similar wiki : ", similarExternalId); + await getSoftwareExternalDataAndSaveIt(similarExternalId, softwareExternalDataCache); + } + } const existingOtherSoftwareExtraData = await dbApi.otherSoftwareExtraData.getBySoftwareId(software.softwareId); const newOtherSoftwareExtraData = await getOtherExternalData(software, existingOtherSoftwareExtraData); @@ -136,3 +144,16 @@ const getNewComptoirDuLibre = async ({ return { ...comptoirDuLibreSoftware, logoUrl, keywords }; }; + +export const makeFetchAndSaveExternalDataForAllSoftwares = (deps: FetchAndSaveSoftwareExtraDataDependencies) => { + const fetchOtherExternalData = makeFetchAndSaveSoftwareExtraData(deps); + return async () => { + const softwares = await deps.dbApi.software.getAll(); + + const softwareExternalDataCache: SoftwareExternalDataCacheBySoftwareId = {}; + + for (const software of softwares) { + await fetchOtherExternalData(software.softwareId, softwareExternalDataCache); + } + }; +}; diff --git a/api/src/core/adapters/wikidata/getWikidataSoftware.ts b/api/src/core/adapters/wikidata/getWikidataSoftware.ts index d3d72c89..2442bdeb 100644 --- a/api/src/core/adapters/wikidata/getWikidataSoftware.ts +++ b/api/src/core/adapters/wikidata/getWikidataSoftware.ts @@ -24,6 +24,7 @@ const { resolveLocalizedString } = createResolveLocalizedString({ export const getWikidataSoftware: GetSoftwareExternalData = memoize( async (wikidataId): Promise => { + console.info(` -> fetching wiki soft : ${wikidataId}`); const { entity } = (await fetchEntity(wikidataId).catch(error => { if (error instanceof WikidataFetchError) { @@ -47,6 +48,7 @@ export const getWikidataSoftware: GetSoftwareExternalData = memoize( return undefined; } + console.info(` -> fetching wiki license : ${licenseId}`); const { entity } = await fetchEntity(licenseId).catch(() => ({ "entity": undefined })); if (entity === undefined) { @@ -144,6 +146,7 @@ export const getWikidataSoftware: GetSoftwareExternalData = memoize( ...getClaimDataValue<"wikibase-entityid">("P172"), ...getClaimDataValue<"wikibase-entityid">("P178") ].map(async ({ id }) => { + console.info(` -> fetching wiki dev : ${id}`); const { entity } = await fetchEntity(id).catch(() => ({ "entity": undefined })); if (entity === undefined) { return undefined; @@ -242,7 +245,6 @@ export class WikidataFetchError extends Error { } async function fetchEntity(wikidataId: string): Promise<{ entity: Entity }> { - console.info(`Fetching wikidata, for wikidataId : ${wikidataId}`); const res = await fetch(`https://www.wikidata.org/wiki/Special:EntityData/${wikidataId}.json`).catch( () => undefined ); @@ -252,7 +254,7 @@ async function fetchEntity(wikidataId: string): Promise<{ entity: Entity }> { } if (res.status === 429) { - await new Promise(resolve => setTimeout(resolve, 100)); + await new Promise(resolve => setTimeout(resolve, 300)); return fetchEntity(wikidataId); } @@ -264,8 +266,6 @@ async function fetchEntity(wikidataId: string): Promise<{ entity: Entity }> { const entity = Object.values(json["entities"])[0] as Entity; - console.info(`Fetching wikidata finished, for wikidataId : ${wikidataId}`); - return { entity }; } diff --git a/api/src/core/bootstrap.ts b/api/src/core/bootstrap.ts index fc4838a5..acc879b3 100644 --- a/api/src/core/bootstrap.ts +++ b/api/src/core/bootstrap.ts @@ -1,9 +1,9 @@ import { Kysely } from "kysely"; import { createCore, createObjectThatThrowsIfAccessed, type GenericCore } from "redux-clean-architecture"; -import { createCompileData } from "./adapters/compileData"; import { comptoirDuLibreApi } from "./adapters/comptoirDuLibreApi"; import { createKyselyPgDbApi } from "./adapters/dbApi/kysely/createPgDbApi"; import { Database } from "./adapters/dbApi/kysely/kysely.database"; +import { makeFetchAndSaveExternalDataForAllSoftwares } from "./adapters/fetchExternalData"; import { getCnllPrestatairesSill } from "./adapters/getCnllPrestatairesSill"; import { getServiceProviders } from "./adapters/getServiceProviders"; import { createGetSoftwareLatestVersion } from "./adapters/getSoftwareLatestVersion"; @@ -12,9 +12,7 @@ import { getWikidataSoftwareOptions } from "./adapters/wikidata/getWikidataSoftw import { getHalSoftware } from "./adapters/hal/getHalSoftware"; import { getHalSoftwareOptions } from "./adapters/hal/getHalSoftwareOptions"; import { createKeycloakUserApi, type KeycloakUserApiParams } from "./adapters/userApi"; -import type { CompileData } from "./ports/CompileData"; import type { ComptoirDuLibreApi } from "./ports/ComptoirDuLibreApi"; -import { Db } from "./ports/DbApi"; import { DbApiV2 } from "./ports/DbApiV2"; import type { ExternalDataOrigin, GetSoftwareExternalData } from "./ports/GetSoftwareExternalData"; import type { GetSoftwareExternalDataOptions } from "./ports/GetSoftwareExternalDataOptions"; @@ -39,7 +37,7 @@ export type Context = { paramsOfBootstrapCore: ParamsOfBootstrapCore; dbApi: DbApiV2; userApi: UserApi; - compileData: CompileData; + // compileData: CompileData; comptoirDuLibreApi: ComptoirDuLibreApi; getSoftwareExternalData: GetSoftwareExternalData; getSoftwareLatestVersion: GetSoftwareLatestVersion; @@ -51,11 +49,10 @@ export type State = Core["types"]["State"]; export type Thunks = Core["types"]["Thunks"]; export type CreateEvt = Core["types"]["CreateEvt"]; -const getDbApiAndInitializeCache = (dbConfig: DbConfig): Db.DbApiAndInitializeCache => { +const getDbApiAndInitializeCache = (dbConfig: DbConfig): { dbApi: DbApiV2 } => { if (dbConfig.dbKind === "kysely") { return { - dbApi: createKyselyPgDbApi(dbConfig.kyselyDb), - initializeDbApiCache: async () => {} + dbApi: createKyselyPgDbApi(dbConfig.kyselyDb) }; } @@ -81,15 +78,17 @@ export async function bootstrapCore( const { getSoftwareExternalData } = getSoftwareExternalDataFunctions(externalSoftwareDataOrigin); - const { compileData } = createCompileData({ + const { dbApi } = getDbApiAndInitializeCache(dbConfig); + + const fetchAndSaveExternalDataForAllSoftwares = makeFetchAndSaveExternalDataForAllSoftwares({ getSoftwareExternalData, getCnllPrestatairesSill, comptoirDuLibreApi, getSoftwareLatestVersion, - getServiceProviders + getServiceProviders, + dbApi }); - - const { dbApi, initializeDbApiCache } = getDbApiAndInitializeCache(dbConfig); + // const { compileData } = createCompileData({}); const { userApi, initializeUserApiCache } = keycloakUserApiParams === undefined @@ -105,7 +104,6 @@ export async function bootstrapCore( "paramsOfBootstrapCore": params, dbApi, userApi, - compileData, comptoirDuLibreApi, getSoftwareExternalData, getSoftwareLatestVersion @@ -116,11 +114,24 @@ export async function bootstrapCore( usecases }); + console.log("doPerPerformPeriodicalCompilation : ", doPerPerformPeriodicalCompilation); if (doPerPerformPeriodicalCompilation) { - console.log("TODO: doPerPerformPeriodicalCompilation"); // setTimeout(() => { // compileData(); // }); + + const frequencyOfUpdate = 1000 * 5; // 5 seconds + + const updateSoftwareExternalData = async () => { + console.log("------ Updating software external data started ------"); + await fetchAndSaveExternalDataForAllSoftwares(); + console.log("------ Updating software external data finished ------"); + setTimeout(async () => { + await updateSoftwareExternalData(); + }, frequencyOfUpdate); + }; + + updateSoftwareExternalData(); } // await dispatch( // usecases.readWriteSillData.protectedThunks.initialize({ @@ -129,9 +140,9 @@ export async function bootstrapCore( // ); if (doPerformCacheInitialization) { - console.log("Performing cache initialization..."); - - await Promise.all([initializeDbApiCache(), initializeUserApiCache()]); + console.log("Performing user cache initialization..."); + await initializeUserApiCache(); + // await Promise.all([initializeDbApiCache(), initializeUserApiCache()]); } return { dbApi, context, core }; diff --git a/api/src/core/ports/DbApi.ts b/api/src/core/ports/DbApi.ts index 86b9be85..3d876003 100644 --- a/api/src/core/ports/DbApi.ts +++ b/api/src/core/ports/DbApi.ts @@ -1,5 +1,4 @@ import type { CompiledData } from "./CompileData"; -import { DbApiV2 } from "./DbApiV2"; export type DbApi = { fetchCompiledData: () => Promise>; @@ -95,8 +94,6 @@ export namespace Db { referencedSinceTime: number; updateTime: number; }; - - export type DbApiAndInitializeCache = { dbApi: DbApiV2; initializeDbApiCache: () => Promise }; } export type Os = "windows" | "linux" | "mac" | "android" | "ios"; diff --git a/api/src/rpc/start.ts b/api/src/rpc/start.ts index 204bd150..e8365bf1 100644 --- a/api/src/rpc/start.ts +++ b/api/src/rpc/start.ts @@ -89,7 +89,8 @@ export async function startRpcService(params: { "organizationUserProfileAttributeName": keycloakParams.organizationUserProfileAttributeName }, githubPersonalAccessTokenForApiRateLimit, - "doPerPerformPeriodicalCompilation": !isDevEnvironnement && redirectUrl === undefined, + "doPerPerformPeriodicalCompilation": true, + // "doPerPerformPeriodicalCompilation": !isDevEnvironnement && redirectUrl === undefined, "doPerformCacheInitialization": redirectUrl === undefined, "externalSoftwareDataOrigin": externalSoftwareDataOrigin }); From f4f0112ff605eb611c4da8761e464afa24a92cb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Burkard?= <22095555+JeromeBu@users.noreply.github.com> Date: Wed, 9 Oct 2024 16:12:26 +0200 Subject: [PATCH 12/19] only update extra data for software that have not been updated for more than 3 hours --- api/scripts/load-git-repo-in-pg.ts | 3 - api/src/core/adapters/comptoirDuLibreApi.ts | 4 +- .../kysely/createPgSoftwareRepository.ts | 128 ++++++++++-------- .../adapters/dbApi/kysely/kysely.database.ts | 1 + .../1717162141365_create-initial-tables.ts | 1 + .../dbApi/kysely/pgDbApi.integration.test.ts | 2 +- .../core/adapters/fetchExternalData.test.ts | 16 +++ api/src/core/adapters/fetchExternalData.ts | 11 +- .../core/adapters/getCnllPrestatairesSill.ts | 2 - api/src/core/adapters/getServiceProviders.ts | 2 - api/src/core/bootstrap.ts | 24 +--- api/src/core/ports/DbApiV2.ts | 7 +- 12 files changed, 112 insertions(+), 89 deletions(-) diff --git a/api/scripts/load-git-repo-in-pg.ts b/api/scripts/load-git-repo-in-pg.ts index 9bd139a2..28b2a69e 100644 --- a/api/scripts/load-git-repo-in-pg.ts +++ b/api/scripts/load-git-repo-in-pg.ts @@ -21,9 +21,6 @@ const saveGitDbInPostgres = async ({ pgConfig, gitDbConfig }: Params) => { const { softwareRows, agentRows, softwareReferentRows, softwareUserRows, instanceRows } = await gitDbApi.fetchDb(); - const reactSofts = softwareRows.filter(s => s.name.toLowerCase().includes("react")); - console.log("REACT SOFT from gitDB: ", reactSofts); - await insertAgents(agentRows, pgDb); const agentIdByEmail = await makeGetAgentIdByEmail(pgDb); diff --git a/api/src/core/adapters/comptoirDuLibreApi.ts b/api/src/core/adapters/comptoirDuLibreApi.ts index 1e406222..4b032e25 100644 --- a/api/src/core/adapters/comptoirDuLibreApi.ts +++ b/api/src/core/adapters/comptoirDuLibreApi.ts @@ -9,7 +9,7 @@ export const comptoirDuLibreApi: ComptoirDuLibreApi = { "getComptoirDuLibre": memoize( async () => { try { - console.info("Fetching comptoir du libre..."); + console.info("Fetching comptoir du libre"); const res = await fetch(url); if (res.status !== 200) { @@ -27,8 +27,6 @@ export const comptoirDuLibreApi: ComptoirDuLibreApi = { "number_of_software": 0, "softwares": [] }; - } finally { - console.info("Fetching comptoir du libre finished"); } }, { "promise": true } diff --git a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts index 74b2d107..ad11aaea 100644 --- a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts +++ b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts @@ -86,6 +86,13 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi return softwareId; }); }, + updateLastExtraDataFetchAt: async ({ softwareId }) => { + await db + .updateTable("softwares") + .set("lastExtraDataFetchAt", sql`now()`) + .where("id", "=", softwareId) + .executeTakeFirstOrThrow(); + }, update: async ({ formData, softwareSillId, agentId }) => { const { softwareName, @@ -201,62 +208,70 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi parentSoftwareExternalId: parentSoftwareExternalId ?? undefined }; }, - getAll: (): Promise => - makeGetSoftwareBuilder(db) - .execute() - .then(async softwares => { - const userAndReferentCountByOrganization = await getUserAndReferentCountByOrganizationBySoftwareId( - db - ); - return softwares.map( - ({ - testUrls, - serviceProviders, - parentExternalData, - updateTime, - addedTime, - softwareExternalData, - similarExternalSoftwares, - ...software - }): Software => { - return stripNullOrUndefinedValues({ - ...software, - updateTime: new Date(+updateTime).getTime(), - addedTime: new Date(+addedTime).getTime(), - serviceProviders: serviceProviders ?? [], - similarSoftwares: similarExternalSoftwares, - // (similarSoftwares ?? []).map( - // (s): SimilarSoftware => ({ - // softwareName: - // typeof s.label === "string" ? s.label : Object.values(s.label)[0]!, - // softwareDescription: - // typeof s.label === "string" ? s.label : Object.values(s.label)[0]!, - // isInSill: true // TODO: check if this is true - // }) - // ) ?? [], - userAndReferentCountByOrganization: - userAndReferentCountByOrganization[software.softwareId] ?? {}, - authors: (softwareExternalData?.developers ?? []).map(dev => ({ - authorName: dev.name, - authorUrl: `https://www.wikidata.org/wiki/${dev.id}` - })), - officialWebsiteUrl: - softwareExternalData?.websiteUrl ?? - software.comptoirDuLibreSoftware?.external_resources.website ?? - undefined, - codeRepositoryUrl: - softwareExternalData?.sourceUrl ?? - software.comptoirDuLibreSoftware?.external_resources.repository ?? - undefined, - documentationUrl: softwareExternalData?.documentationUrl ?? undefined, - comptoirDuLibreServiceProviderCount: - software.comptoirDuLibreSoftware?.providers.length ?? 0, - testUrl: testUrls[0]?.url, - parentWikidataSoftware: parentExternalData ?? undefined - }); - } - ); - }), + getAll: ({ onlyIfUpdatedMoreThan3HoursAgo } = {}): Promise => { + let builder = makeGetSoftwareBuilder(db); + + builder = onlyIfUpdatedMoreThan3HoursAgo + ? builder.where(eb => + eb.or([ + eb("lastExtraDataFetchAt", "is", null), + eb("lastExtraDataFetchAt", "<", sql`now() - interval '3 hours'`) + ]) + ) + : builder; + + return builder.execute().then(async softwares => { + const userAndReferentCountByOrganization = await getUserAndReferentCountByOrganizationBySoftwareId(db); + return softwares.map( + ({ + testUrls, + serviceProviders, + parentExternalData, + updateTime, + addedTime, + softwareExternalData, + similarExternalSoftwares, + ...software + }): Software => { + return stripNullOrUndefinedValues({ + ...software, + updateTime: new Date(+updateTime).getTime(), + addedTime: new Date(+addedTime).getTime(), + serviceProviders: serviceProviders ?? [], + similarSoftwares: similarExternalSoftwares, + // (similarSoftwares ?? []).map( + // (s): SimilarSoftware => ({ + // softwareName: + // typeof s.label === "string" ? s.label : Object.values(s.label)[0]!, + // softwareDescription: + // typeof s.label === "string" ? s.label : Object.values(s.label)[0]!, + // isInSill: true // TODO: check if this is true + // }) + // ) ?? [], + userAndReferentCountByOrganization: + userAndReferentCountByOrganization[software.softwareId] ?? {}, + authors: (softwareExternalData?.developers ?? []).map(dev => ({ + authorName: dev.name, + authorUrl: `https://www.wikidata.org/wiki/${dev.id}` + })), + officialWebsiteUrl: + softwareExternalData?.websiteUrl ?? + software.comptoirDuLibreSoftware?.external_resources.website ?? + undefined, + codeRepositoryUrl: + softwareExternalData?.sourceUrl ?? + software.comptoirDuLibreSoftware?.external_resources.repository ?? + undefined, + documentationUrl: softwareExternalData?.documentationUrl ?? undefined, + comptoirDuLibreServiceProviderCount: + software.comptoirDuLibreSoftware?.providers.length ?? 0, + testUrl: testUrls[0]?.url, + parentWikidataSoftware: parentExternalData ?? undefined + }); + } + ); + }); + }, getAllSillSoftwareExternalIds: async externalDataOrigin => db .selectFrom("softwares") @@ -332,6 +347,7 @@ const makeGetSoftwareBuilder = (db: Kysely) => "s.testUrls", "s.referencedSinceTime as addedTime", "s.updateTime", + "s.lastExtraDataFetchAt", "s.dereferencing", "s.categories", ({ ref }) => diff --git a/api/src/core/adapters/dbApi/kysely/kysely.database.ts b/api/src/core/adapters/dbApi/kysely/kysely.database.ts index 08b2d927..8bfb0996 100644 --- a/api/src/core/adapters/dbApi/kysely/kysely.database.ts +++ b/api/src/core/adapters/dbApi/kysely/kysely.database.ts @@ -92,6 +92,7 @@ type SoftwaresTable = { description: string; referencedSinceTime: number; updateTime: number; + lastExtraDataFetchAt: Date | null; dereferencing: JSONColumnType<{ reason?: string; time: number; diff --git a/api/src/core/adapters/dbApi/kysely/migrations/1717162141365_create-initial-tables.ts b/api/src/core/adapters/dbApi/kysely/migrations/1717162141365_create-initial-tables.ts index c0656525..545b3e32 100644 --- a/api/src/core/adapters/dbApi/kysely/migrations/1717162141365_create-initial-tables.ts +++ b/api/src/core/adapters/dbApi/kysely/migrations/1717162141365_create-initial-tables.ts @@ -40,6 +40,7 @@ export async function up(db: Kysely): Promise { .addColumn("dereferencing", "jsonb") .addColumn("referencedSinceTime", "bigint", col => col.notNull()) .addColumn("updateTime", "bigint", col => col.notNull()) + .addColumn("lastExtraDataFetchAt", "timestamptz") .execute(); await db.schema diff --git a/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts b/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts index 135f3bb3..69019277 100644 --- a/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts +++ b/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts @@ -140,7 +140,7 @@ describe("pgDbApi", () => { serviceUrl: "https://example.com" }); - const softwares = await dbApi.software.getAll(); + const softwares = await dbApi.software.getAll({ onlyIfUpdatedMoreThan3HoursAgo: true }); const actualSoftware = softwares[0]; diff --git a/api/src/core/adapters/fetchExternalData.test.ts b/api/src/core/adapters/fetchExternalData.test.ts index 1515e827..a8f5b30f 100644 --- a/api/src/core/adapters/fetchExternalData.test.ts +++ b/api/src/core/adapters/fetchExternalData.test.ts @@ -69,6 +69,8 @@ describe("fetches software extra data (from different providers)", () => { await db.deleteFrom("softwares").execute(); await db.deleteFrom("agents").execute(); + await sql`SELECT setval('softwares_id_seq', 11, false)`.execute(db); + dbApi = createKyselyPgDbApi(db); const agentId = await dbApi.agent.add({ @@ -116,6 +118,13 @@ describe("fetches software extra data (from different providers)", () => { const softwareExternalDatas = await db.selectFrom("software_external_datas").selectAll().execute(); expect(softwareExternalDatas).toHaveLength(0); + const { lastExtraDataFetchAt: initialLastExtraDataFetchAt } = await db + .selectFrom("softwares") + .select("lastExtraDataFetchAt") + .where("id", "=", craSoftwareId) + .executeTakeFirstOrThrow(); + expect(initialLastExtraDataFetchAt).toBe(null); + await fetchAndSaveSoftwareExtraData(craSoftwareId, {}); const updatedSoftwareExternalDatas = await db.selectFrom("software_external_datas").selectAll().execute(); @@ -158,6 +167,13 @@ describe("fetches software extra data (from different providers)", () => { const otherExtraData = await db.selectFrom("compiled_softwares").selectAll().execute(); expectToEqual(otherExtraData, []); + + const { lastExtraDataFetchAt } = await db + .selectFrom("softwares") + .select("lastExtraDataFetchAt") + .where("id", "=", craSoftwareId) + .executeTakeFirstOrThrow(); + expect(lastExtraDataFetchAt).toBeTruthy(); }, { timeout: 10_000 } ); diff --git a/api/src/core/adapters/fetchExternalData.ts b/api/src/core/adapters/fetchExternalData.ts index b44cbae0..fdb84ceb 100644 --- a/api/src/core/adapters/fetchExternalData.ts +++ b/api/src/core/adapters/fetchExternalData.ts @@ -38,18 +38,18 @@ export const makeFetchAndSaveSoftwareExtraData = ({ console.log(`🚀${software.softwareName}`); if (software.externalId) { - console.log(" • Soft: ", software.softwareName, " - Own wiki : ", software.externalId); + console.log(" • Soft: ", software.softwareName, " - Own wiki : ", software.externalId); await getSoftwareExternalDataAndSaveIt(software.externalId, softwareExternalDataCache); } if (parentSoftwareExternalId) { - console.log(" • Parent wiki : ", parentSoftwareExternalId); + console.log(" • Parent wiki : ", parentSoftwareExternalId); await getSoftwareExternalDataAndSaveIt(parentSoftwareExternalId, softwareExternalDataCache); } if (similarSoftwaresExternalIds.length > 0) { for (const similarExternalId of similarSoftwaresExternalIds) { - console.log(" • Similar wiki : ", similarExternalId); + console.log(" • Similar wiki : ", similarExternalId); await getSoftwareExternalDataAndSaveIt(similarExternalId, softwareExternalDataCache); } } @@ -58,6 +58,7 @@ export const makeFetchAndSaveSoftwareExtraData = ({ const newOtherSoftwareExtraData = await getOtherExternalData(software, existingOtherSoftwareExtraData); if (newOtherSoftwareExtraData) await dbApi.otherSoftwareExtraData.save(newOtherSoftwareExtraData); + await dbApi.software.updateLastExtraDataFetchAt({ softwareId: software.softwareId }); }; }; @@ -148,7 +149,9 @@ const getNewComptoirDuLibre = async ({ export const makeFetchAndSaveExternalDataForAllSoftwares = (deps: FetchAndSaveSoftwareExtraDataDependencies) => { const fetchOtherExternalData = makeFetchAndSaveSoftwareExtraData(deps); return async () => { - const softwares = await deps.dbApi.software.getAll(); + const softwares = await deps.dbApi.software.getAll({ onlyIfUpdatedMoreThan3HoursAgo: true }); + + console.info("About to update ${softwares.length} softwares"); const softwareExternalDataCache: SoftwareExternalDataCacheBySoftwareId = {}; diff --git a/api/src/core/adapters/getCnllPrestatairesSill.ts b/api/src/core/adapters/getCnllPrestatairesSill.ts index c75a7380..c72275b9 100644 --- a/api/src/core/adapters/getCnllPrestatairesSill.ts +++ b/api/src/core/adapters/getCnllPrestatairesSill.ts @@ -23,8 +23,6 @@ export const getCnllPrestatairesSill: GetCnllPrestatairesSill = memoize( } catch (error) { console.error(`Failed to fetch or parse ${url}: ${String(error)}`); return []; - } finally { - console.info("Fetching cnll prestataires sill finished"); } }, { "promise": true } diff --git a/api/src/core/adapters/getServiceProviders.ts b/api/src/core/adapters/getServiceProviders.ts index 25f9a72f..9365a537 100644 --- a/api/src/core/adapters/getServiceProviders.ts +++ b/api/src/core/adapters/getServiceProviders.ts @@ -62,8 +62,6 @@ export const getServiceProviders: GetServiceProviders = memoize( } catch (error) { console.error(`Failed to fetch or parse ${url}: ${String(error)}`); return {}; - } finally { - console.info("Fetching service providers finished"); } }, { "promise": true } diff --git a/api/src/core/bootstrap.ts b/api/src/core/bootstrap.ts index acc879b3..a0925091 100644 --- a/api/src/core/bootstrap.ts +++ b/api/src/core/bootstrap.ts @@ -114,13 +114,14 @@ export async function bootstrapCore( usecases }); + if (doPerformCacheInitialization) { + console.log("Performing user cache initialization..."); + await initializeUserApiCache(); + } + console.log("doPerPerformPeriodicalCompilation : ", doPerPerformPeriodicalCompilation); if (doPerPerformPeriodicalCompilation) { - // setTimeout(() => { - // compileData(); - // }); - - const frequencyOfUpdate = 1000 * 5; // 5 seconds + const frequencyOfUpdate = 1000 * 60 * 60 * 4; // 4 hours const updateSoftwareExternalData = async () => { console.log("------ Updating software external data started ------"); @@ -131,18 +132,7 @@ export async function bootstrapCore( }, frequencyOfUpdate); }; - updateSoftwareExternalData(); - } - // await dispatch( - // usecases.readWriteSillData.protectedThunks.initialize({ - // doPerPerformPeriodicalCompilation - // }) - // ); - - if (doPerformCacheInitialization) { - console.log("Performing user cache initialization..."); - await initializeUserApiCache(); - // await Promise.all([initializeDbApiCache(), initializeUserApiCache()]); + void updateSoftwareExternalData(); } return { dbApi, context, core }; diff --git a/api/src/core/ports/DbApiV2.ts b/api/src/core/ports/DbApiV2.ts index 5ee7729b..ad888b96 100644 --- a/api/src/core/ports/DbApiV2.ts +++ b/api/src/core/ports/DbApiV2.ts @@ -15,6 +15,10 @@ import type { ExternalDataOrigin, SoftwareExternalData } from "./GetSoftwareExte export type WithAgentId = { agentId: number }; +type GetSoftwareFilters = { + onlyIfUpdatedMoreThan3HoursAgo?: true; +}; + export interface SoftwareRepository { create: ( params: { @@ -28,7 +32,8 @@ export interface SoftwareRepository { formData: SoftwareFormData; } & WithAgentId ) => Promise; - getAll: () => Promise; + updateLastExtraDataFetchAt: (params: { softwareId: number }) => Promise; + getAll: (filters?: GetSoftwareFilters) => Promise; getById: (id: number) => Promise; getByIdWithLinkedSoftwaresExternalIds: (id: number) => Promise< | { From 3f083f054f669e6871bf119b08920ee656b82250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Burkard?= <22095555+JeromeBu@users.noreply.github.com> Date: Wed, 9 Oct 2024 16:31:08 +0200 Subject: [PATCH 13/19] configure app to run migration on app start --- api/package.json | 5 +++-- api/src/core/adapters/fetchExternalData.ts | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/api/package.json b/api/package.json index cc566149..1d754e08 100644 --- a/api/package.json +++ b/api/package.json @@ -10,13 +10,14 @@ "types": "dist/src/lib/index.d.ts", "scripts": { "migrate": "dotenv -e ../.env -- kysely migrate", + "db:up": "yarn migrate latest", "prepare": "[ ! -f .env.local.sh ] && cp .env.sh .env.local.sh || true", "test": "vitest --watch=false", "dev": "yarn build && yarn start", "build": "tsc", - "start": "dotenv -e ../.env -- forever dist/src/main.js", + "start": "yarn db:up && dotenv -e ../.env -- forever dist/src/main.js", "build-prod": "esbuild src/main.ts --bundle --platform=node --target=node20 --outfile=dist/index.js", - "start-prod": "dotenv -e ../.env -- forever dist/index.js", + "start-prod": "yarn db:up && dotenv -e ../.env -- forever dist/index.js", "_format": "prettier \"**/*.{ts,tsx,json,md}\"", "format": "yarn run _format --write", "format:check": "yarn run _format --list-different", diff --git a/api/src/core/adapters/fetchExternalData.ts b/api/src/core/adapters/fetchExternalData.ts index fdb84ceb..2e7b237d 100644 --- a/api/src/core/adapters/fetchExternalData.ts +++ b/api/src/core/adapters/fetchExternalData.ts @@ -151,7 +151,7 @@ export const makeFetchAndSaveExternalDataForAllSoftwares = (deps: FetchAndSaveSo return async () => { const softwares = await deps.dbApi.software.getAll({ onlyIfUpdatedMoreThan3HoursAgo: true }); - console.info("About to update ${softwares.length} softwares"); + console.info(`About to update ${softwares.length} softwares`); const softwareExternalDataCache: SoftwareExternalDataCacheBySoftwareId = {}; From 2353f4037c860a33eaa7c7b9e9642a09fe239d54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Burkard?= <22095555+JeromeBu@users.noreply.github.com> Date: Wed, 9 Oct 2024 16:35:36 +0200 Subject: [PATCH 14/19] add pre prod docker-compose --- docker-compose.preprod.yml | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 docker-compose.preprod.yml diff --git a/docker-compose.preprod.yml b/docker-compose.preprod.yml new file mode 100644 index 00000000..6df17ff4 --- /dev/null +++ b/docker-compose.preprod.yml @@ -0,0 +1,40 @@ +services: + api: + build: + context: "." + dockerfile: Dockerfile.api + env_file: .env + restart: unless-stopped + + web: + build: + context: "." + dockerfile: Dockerfile.web + restart: unless-stopped + + nginx: + image: nginx:alpine + ports: + - "8090:80" + volumes: + - ./nginx/:/etc/nginx/conf.d/ + depends_on: + - api + - web + restart: unless-stopped + + postgres: + image: postgres:16 + shm_size: 256m + environment: + POSTGRES_LOG_STATEMENTS: all + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + volumes: + - ./docker-data:/var/lib/postgresql/data + + adminer: + image: adminer + ports: + - "8091:8080" From ae4369f3ee1de7f874781e0d114379eca9057ff7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Burkard?= <22095555+JeromeBu@users.noreply.github.com> Date: Fri, 18 Oct 2024 10:36:58 +0200 Subject: [PATCH 15/19] update docker-compose to restart PG if it crashes --- docker-compose.preprod.yml | 2 ++ docker-compose.prod.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docker-compose.preprod.yml b/docker-compose.preprod.yml index 6df17ff4..9a81eed4 100644 --- a/docker-compose.preprod.yml +++ b/docker-compose.preprod.yml @@ -33,8 +33,10 @@ services: POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} volumes: - ./docker-data:/var/lib/postgresql/data + restart: unless-stopped adminer: image: adminer ports: - "8091:8080" + restart: unless-stopped diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 9abc6229..8ab7477b 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -33,8 +33,10 @@ services: POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} volumes: - ./docker-data:/var/lib/postgresql/data + restart: unless-stopped adminer: image: adminer ports: - "8081:8080" + restart: unless-stopped From df4ffe040f8552ed9bf76ff06162b4bb693d06df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Burkard?= <22095555+JeromeBu@users.noreply.github.com> Date: Fri, 18 Oct 2024 11:00:27 +0200 Subject: [PATCH 16/19] start incremental compilation for the first time after a delay --- api/src/core/bootstrap.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/src/core/bootstrap.ts b/api/src/core/bootstrap.ts index a0925091..c77d833d 100644 --- a/api/src/core/bootstrap.ts +++ b/api/src/core/bootstrap.ts @@ -132,7 +132,8 @@ export async function bootstrapCore( }, frequencyOfUpdate); }; - void updateSoftwareExternalData(); + // start the periodical compilation 2 min after api starts + void setTimeout(() => updateSoftwareExternalData(), 1000 * 60 * 2); } return { dbApi, context, core }; From 203173aff235e44a0024e51877d6d1a1fc44caff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Burkard?= <22095555+JeromeBu@users.noreply.github.com> Date: Fri, 18 Oct 2024 12:28:55 +0200 Subject: [PATCH 17/19] fix loading logos from wikidata or comptoir du libre --- .../kysely/createPgSoftwareRepository.ts | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts index ad11aaea..7acc9992 100644 --- a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts +++ b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts @@ -89,7 +89,11 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi updateLastExtraDataFetchAt: async ({ softwareId }) => { await db .updateTable("softwares") - .set("lastExtraDataFetchAt", sql`now()`) + .set( + "lastExtraDataFetchAt", + sql`now + ()` + ) .where("id", "=", softwareId) .executeTakeFirstOrThrow(); }, @@ -215,7 +219,13 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi ? builder.where(eb => eb.or([ eb("lastExtraDataFetchAt", "is", null), - eb("lastExtraDataFetchAt", "<", sql`now() - interval '3 hours'`) + eb( + "lastExtraDataFetchAt", + "<", + sql`now + () + - interval '3 hours'` + ) ]) ) : builder; @@ -339,7 +349,14 @@ const makeGetSoftwareBuilder = (db: Kysely) => .orderBy("s.id", "asc") .select([ "s.id as softwareId", - "s.logoUrl", + ({ fn, ref }) => + fn + .coalesce( + ref("s.logoUrl"), + ref("ext.logoUrl"), + sql`${ref("cs.comptoirDuLibreSoftware")} ->> 'logoUrl'` + ) + .as("logoUrl"), "s.name as softwareName", "s.description as softwareDescription", "cs.serviceProviders", From 615f5968dc4a11d4d4eba81c0808fa5b0219a83b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Burkard?= <22095555+JeromeBu@users.noreply.github.com> Date: Fri, 18 Oct 2024 13:08:12 +0200 Subject: [PATCH 18/19] fix comptoire du libre schema to keep logo --- api/src/core/ports/ComptoirDuLibreApi.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/src/core/ports/ComptoirDuLibreApi.ts b/api/src/core/ports/ComptoirDuLibreApi.ts index 8ee9b29e..774d4d62 100644 --- a/api/src/core/ports/ComptoirDuLibreApi.ts +++ b/api/src/core/ports/ComptoirDuLibreApi.ts @@ -43,6 +43,7 @@ export declare namespace ComptoirDuLibre { url: string; name: string; licence: string; + logoUrl?: string; external_resources: { website: string | null; repository: string | null; @@ -77,13 +78,14 @@ export const { zComptoirDuLibre } = (() => { assert>>(); - const zSoftware = z.object({ + const zSoftware: z.Schema = z.object({ "id": z.number(), "created": z.string(), "modified": z.string(), "url": z.string(), "name": z.string(), "licence": z.string(), + "logoUrl": z.string().optional(), "external_resources": z.object({ "website": z.union([z.string(), z.null()]), "repository": z.union([z.string(), z.null()]) From 55079d79506d32ba6494c0a450bc9f86927be5a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Burkard?= <22095555+JeromeBu@users.noreply.github.com> Date: Fri, 18 Oct 2024 14:00:48 +0200 Subject: [PATCH 19/19] fix fetch comptoir du libre logo and keywords, and cache condition --- .../core/adapters/fetchExternalData.test.ts | 81 ++++++++++++++----- api/src/core/adapters/fetchExternalData.ts | 11 ++- api/src/core/ports/ComptoirDuLibreApi.ts | 1 + 3 files changed, 72 insertions(+), 21 deletions(-) diff --git a/api/src/core/adapters/fetchExternalData.test.ts b/api/src/core/adapters/fetchExternalData.test.ts index a8f5b30f..4eccdf81 100644 --- a/api/src/core/adapters/fetchExternalData.test.ts +++ b/api/src/core/adapters/fetchExternalData.test.ts @@ -1,6 +1,6 @@ import { Kysely, sql } from "kysely"; import { describe, it, beforeEach, expect } from "vitest"; -import { expectToEqual, testPgUrl } from "../../tools/test.helpers"; +import { expectToEqual, expectToMatchObject, testPgUrl } from "../../tools/test.helpers"; import { DbApiV2 } from "../ports/DbApiV2"; import { SoftwareFormData } from "../usecases/readWriteSillData"; import { comptoirDuLibreApi } from "./comptoirDuLibreApi"; @@ -35,23 +35,53 @@ const apacheSoftwareId = 6; const insertApacheWithCorrectId = async (db: Kysely, agentId: number) => { await sql` - INSERT INTO public.softwares - (id, "softwareType", "externalId", "externalDataOrigin", - "comptoirDuLibreId", name, description, license, "versionMin", - "isPresentInSupportContract", "isFromFrenchPublicService", "logoUrl", - keywords, "doRespectRgaa", "isStillInObservation", - "parentSoftwareWikidataId", "catalogNumeriqueGouvFrId", "workshopUrls", - "testUrls", categories, "generalInfoMd", "addedByAgentId", - dereferencing, "referencedSinceTime", "updateTime") - VALUES (${apacheSoftwareId}, - '{"os": {"ios": false, "mac": false, "linux": true, "android": false, "windows": false}, "type": "desktop/mobile"}', - 'Q11354', 'wikidata', 3737, 'Apache HTTP Server', - 'Serveur Web & Reverse Proxy', 'Apache-2.0', '212', true, false, - 'https://sill.code.gouv.fr/logo/apache-http.png', - '["serveur", "http", "web", "server", "apache"]', false, false, - null, null, '[]', '[]', '[]', null, ${agentId}, null, 1728462232094, - 1728462232094); - `.execute(db); + INSERT INTO public.softwares + (id, "softwareType", "externalId", "externalDataOrigin", + "comptoirDuLibreId", name, description, license, "versionMin", + "isPresentInSupportContract", "isFromFrenchPublicService", "logoUrl", + keywords, "doRespectRgaa", "isStillInObservation", + "parentSoftwareWikidataId", "catalogNumeriqueGouvFrId", "workshopUrls", + "testUrls", categories, "generalInfoMd", "addedByAgentId", + dereferencing, "referencedSinceTime", "updateTime") + VALUES (${apacheSoftwareId}, + '{"os": {"ios": false, "mac": false, "linux": true, "android": false, "windows": false}, "type": "desktop/mobile"}', + 'Q11354', 'wikidata', 3737, 'Apache HTTP Server', + 'Serveur Web & Reverse Proxy', 'Apache-2.0', '212', true, false, + 'https://sill.code.gouv.fr/logo/apache-http.png', + '["serveur", "http", "web", "server", "apache"]', false, false, + null, null, '[]', '[]', '[]', null, ${agentId}, null, + 1728462232094, + 1728462232094); + `.execute(db); +}; + +const acceleroId = 2; +const insertAcceleroWithCorrectId = async (db: Kysely, agentId: number) => { + await sql` + INSERT INTO public.softwares (id, "softwareType", "externalId", + "externalDataOrigin", "comptoirDuLibreId", + name, description, license, "versionMin", + "isPresentInSupportContract", + "isFromFrenchPublicService", "logoUrl", + keywords, "doRespectRgaa", + "isStillInObservation", + "parentSoftwareWikidataId", + "catalogNumeriqueGouvFrId", + "workshopUrls", "testUrls", categories, + "generalInfoMd", "addedByAgentId", + dereferencing, "referencedSinceTime", + "updateTime") + VALUES (${acceleroId}, '{"type": "stack"}', 'Q2822666', 'wikidata', 304, + 'Acceleo', + 'Outil et/ou plugin de génération de tout ou partie du code', + 'EPL-2.0', '3.7.8', false, false, null, + '["modélisation", "génération", "code", "modeling", "code generation"]', + false, false, null, null, '[]', '[]', + '["Other Development Tools"]', null, ${agentId}, null, + 1514764800000, + 1514764800000); + `.execute(db); + return acceleroId; }; describe("fetches software extra data (from different providers)", () => { @@ -87,6 +117,7 @@ describe("fetches software extra data (from different providers)", () => { }); await insertApacheWithCorrectId(db, agentId); + await insertAcceleroWithCorrectId(db, agentId); const { getSoftwareLatestVersion } = createGetSoftwareLatestVersion({ githubPersonalAccessTokenForApiRateLimit: "" @@ -112,6 +143,20 @@ describe("fetches software extra data (from different providers)", () => { expectToEqual(updatedSoftwareExternalDatas, []); }); + it("fetches correctly the logoUrl from comptoir du libre", async () => { + const softwareExternalDatas = await db.selectFrom("software_external_datas").selectAll().execute(); + expectToEqual(softwareExternalDatas, []); + + await fetchAndSaveSoftwareExtraData(acceleroId, {}); + + const results = await db.selectFrom("compiled_softwares").select("comptoirDuLibreSoftware").execute(); + expect(results).toHaveLength(1); + expectToMatchObject(results[0]!.comptoirDuLibreSoftware, { + name: "Acceleo", + logoUrl: "https://comptoir-du-libre.org//img/files/Softwares/Acceleo/avatar/Acceleo.png" + }); + }); + it( "gets software external data and saves it, and does not save other extra data if there is nothing relevant", async () => { diff --git a/api/src/core/adapters/fetchExternalData.ts b/api/src/core/adapters/fetchExternalData.ts index 2e7b237d..5ccd1b97 100644 --- a/api/src/core/adapters/fetchExternalData.ts +++ b/api/src/core/adapters/fetchExternalData.ts @@ -28,7 +28,10 @@ export const makeFetchAndSaveSoftwareExtraData = ({ ...otherExternalDataDeps }: FetchAndSaveSoftwareExtraDataDependencies) => { const getOtherExternalData = makeGetOtherExternalData(otherExternalDataDeps); - const getSoftwareExternalDataAndSaveIt = makeGetSoftwareExternalData({ dbApi, getSoftwareExternalData }); + const getSoftwareExternalDataAndSaveIt = makeGetSoftwareExternalData({ + dbApi, + getSoftwareExternalData + }); return async (softwareId: number, softwareExternalDataCache: SoftwareExternalDataCacheBySoftwareId) => { const data = await dbApi.software.getByIdWithLinkedSoftwaresExternalIds(softwareId); @@ -135,9 +138,11 @@ const getNewComptoirDuLibre = async ({ if (!comptoirDuLibreSoftware) return null; + const alreadySavedCdlSoftware = otherSoftwareExtraDataInCache?.comptoirDuLibreSoftware; + const [logoUrl, keywords] = - otherSoftwareExtraDataInCache?.comptoirDuLibreSoftware?.id === comptoirDuLibreSoftware.id - ? [] + alreadySavedCdlSoftware && alreadySavedCdlSoftware.id === comptoirDuLibreSoftware.id + ? [alreadySavedCdlSoftware.logoUrl, alreadySavedCdlSoftware.keywords] : await Promise.all([ comptoirDuLibreApi.getIconUrl({ comptoirDuLibreId: comptoirDuLibreSoftware.id }), comptoirDuLibreApi.getKeywords({ comptoirDuLibreId: comptoirDuLibreSoftware.id }) diff --git a/api/src/core/ports/ComptoirDuLibreApi.ts b/api/src/core/ports/ComptoirDuLibreApi.ts index 774d4d62..b8511b78 100644 --- a/api/src/core/ports/ComptoirDuLibreApi.ts +++ b/api/src/core/ports/ComptoirDuLibreApi.ts @@ -44,6 +44,7 @@ export declare namespace ComptoirDuLibre { name: string; licence: string; logoUrl?: string; + keywords?: string[]; external_resources: { website: string | null; repository: string | null;