diff --git a/api/scripts/load-git-repo-in-pg.ts b/api/scripts/load-git-repo-in-pg.ts index 2a3fd19b..9fe3e587 100644 --- a/api/scripts/load-git-repo-in-pg.ts +++ b/api/scripts/load-git-repo-in-pg.ts @@ -48,13 +48,13 @@ const insertSoftwares = async (softwareRows: SoftwareRow[], db: Kysely console.info("Number of softwares to insert : ", softwareRows.length); await db.transaction().execute(async trx => { await trx.deleteFrom("softwares").execute(); + await trx.deleteFrom("softwares__similar_software_external_datas").execute(); await trx .insertInto("softwares") .values( - softwareRows.map(row => ({ + softwareRows.map(({ similarSoftwareExternalDataIds: _, ...row }) => ({ ...row, dereferencing: row.dereferencing ? JSON.stringify(row.dereferencing) : null, - similarSoftwareExternalDataIds: JSON.stringify(row.similarSoftwareExternalDataIds), softwareType: JSON.stringify(row.softwareType), workshopUrls: JSON.stringify(row.workshopUrls), testUrls: JSON.stringify(row.testUrls), @@ -63,6 +63,18 @@ const insertSoftwares = async (softwareRows: SoftwareRow[], db: Kysely })) ) .executeTakeFirst(); + + await trx + .insertInto("softwares__similar_software_external_datas") + .values( + softwareRows.flatMap(row => + Array.from(new Set(row.similarSoftwareExternalDataIds)).map(externalId => ({ + softwareId: row.id, + similarExternalId: externalId + })) + ) + ) + .execute(); }); }; @@ -168,6 +180,7 @@ const insertCompiledSoftwaresAndSoftwareExternalData = async ( .executeTakeFirst(); await trx.deleteFrom("software_external_datas").execute(); + await trx .insertInto("software_external_datas") .values( @@ -203,6 +216,25 @@ const insertCompiledSoftwaresAndSoftwareExternalData = async ( ) .onConflict(conflict => conflict.column("externalId").doNothing()) .executeTakeFirst(); + + await trx + .insertInto("software_external_datas") + .values( + compiledSoftwares + .filter(s => s.similarExternalSoftwares.length > 0) + .flatMap(s => + (s.similarExternalSoftwares ?? []).map(similarExternalSoftware => ({ + externalId: similarExternalSoftware.externalId, + externalDataOrigin: similarExternalSoftware.externalDataOrigin, + developers: JSON.stringify([]), + label: JSON.stringify(similarExternalSoftware?.label ?? {}), + description: JSON.stringify(similarExternalSoftware?.description ?? {}), + isLibreSoftware: similarExternalSoftware?.isLibreSoftware ?? false + })) + ) + ) + .onConflict(conflict => conflict.column("externalId").doNothing()) + .executeTakeFirst(); }); }; diff --git a/api/src/core/adapters/dbApi/kysely/createGetCompiledData.ts b/api/src/core/adapters/dbApi/kysely/createGetCompiledData.ts index 7fd440ab..19d1f1fd 100644 --- a/api/src/core/adapters/dbApi/kysely/createGetCompiledData.ts +++ b/api/src/core/adapters/dbApi/kysely/createGetCompiledData.ts @@ -3,7 +3,7 @@ import { CompiledData } from "../../../ports/CompileData"; import { Db } from "../../../ports/DbApi"; import { ParentSoftwareExternalData, SoftwareExternalData } from "../../../ports/GetSoftwareExternalData"; import { Database } from "./kysely.database"; -import { convertNullValuesToUndefined, isNotNull, jsonBuildObject } from "./kysely.utils"; +import { convertNullValuesToUndefined, isNotNull, jsonBuildObject, jsonStripNulls } from "./kysely.utils"; export const createGetCompiledData = (db: Kysely) => async (): Promise> => { console.time("agentById query"); @@ -90,35 +90,26 @@ export const createGetCompiledData = (db: Kysely) => async (): Promise .case() .when("ext.externalId", "is not", null) .then( - jsonBuildObject({ - externalId: ref("ext.externalId"), - externalDataOrigin: ref("ext.externalDataOrigin"), - developers: ref("ext.developers"), - label: ref("ext.label"), - description: ref("ext.description"), - isLibreSoftware: ref("ext.isLibreSoftware"), - logoUrl: ref("ext.logoUrl"), - framaLibreId: ref("ext.framaLibreId"), - websiteUrl: ref("ext.websiteUrl"), - sourceUrl: ref("ext.sourceUrl"), - documentationUrl: ref("ext.documentationUrl") - }).$castTo() + jsonStripNulls( + jsonBuildObject({ + externalId: ref("ext.externalId"), + externalDataOrigin: ref("ext.externalDataOrigin"), + developers: ref("ext.developers"), + label: ref("ext.label"), + description: ref("ext.description"), + isLibreSoftware: ref("ext.isLibreSoftware"), + logoUrl: ref("ext.logoUrl"), + framaLibreId: ref("ext.framaLibreId"), + websiteUrl: ref("ext.websiteUrl"), + sourceUrl: ref("ext.sourceUrl"), + documentationUrl: ref("ext.documentationUrl"), + license: ref("ext.license") + }) + ).$castTo() ) .end() .as("softwareExternalData"), - ({ ref, fn }) => - fn - .jsonAgg( - jsonBuildObject({ - externalId: ref("similarExt.externalId"), - label: ref("similarExt.label"), - description: ref("similarExt.description"), - isLibreSoftware: ref("similarExt.isLibreSoftware"), - externalDataOrigin: ref("similarExt.externalDataOrigin") - }).$castTo() - ) - .filterWhere("similarExt.externalId", "is not", null) - .as("similarExternalSoftwares"), + ({ fn }) => fn.jsonAgg("similarExt").distinct().as("similarExternalSoftwares"), ({ fn }) => fn.jsonAgg("users").distinct().as("users"), ({ fn }) => fn.jsonAgg("referents").distinct().as("referents"), ({ fn }) => fn.jsonAgg("instances").distinct().as("instances") @@ -158,7 +149,16 @@ export const createGetCompiledData = (db: Kysely) => async (): Promise parentWikidataSoftware: parentWikidataSoftware ?? undefined, dereferencing: dereferencing ?? undefined, serviceProviders: serviceProviders ?? [], - similarExternalSoftwares: similarExternalSoftwares ?? [], + similarExternalSoftwares: (similarExternalSoftwares ?? []) + .filter(isNotNull) + .map(similar => ({ + "externalId": similar.externalId!, + "externalDataOrigin": similar.externalDataOrigin!, + "label": similar.label!, + "description": similar.description!, + "isLibreSoftware": similar.isLibreSoftware! + })) + .sort((a, b) => a.externalId.localeCompare(b.externalId)), users: users.filter(isNotNull).map(user => ({ ...(user as any), organization: agentById[user.agentId!]?.organization @@ -167,8 +167,13 @@ export const createGetCompiledData = (db: Kysely) => async (): Promise ...(referent as any), organization: agentById[referent.agentId!]?.organization })), - instances: instances.filter(isNotNull).map(instance => ({ - ...(instance as any) + instances: (instances ?? []).filter(isNotNull).map(instance => ({ + id: instance.id!, + organization: instance.organization!, + targetAudience: instance.targetAudience!, + publicUrl: instance.publicUrl ?? undefined, + addedByAgentEmail: instance.addedByAgentEmail!, + otherWikidataSoftwares: [] })) }; } @@ -177,5 +182,6 @@ export const createGetCompiledData = (db: Kysely) => async (): Promise return processedSoftwares; }); + console.log("numberOfCompiledSoftwares : ", compliedSoftwares.length); return compliedSoftwares; }; diff --git a/api/src/core/adapters/dbApi/kysely/kysely.database.ts b/api/src/core/adapters/dbApi/kysely/kysely.database.ts index 8e7e20d7..00600321 100644 --- a/api/src/core/adapters/dbApi/kysely/kysely.database.ts +++ b/api/src/core/adapters/dbApi/kysely/kysely.database.ts @@ -107,7 +107,6 @@ type SoftwaresTable = { doRespectRgaa: boolean | null; isFromFrenchPublicService: boolean; isPresentInSupportContract: boolean; - similarSoftwareExternalDataIds: JSONColumnType; parentSoftwareWikidataId: string | null; externalId: string | null; externalDataOrigin: "wikidata" | "HAL" | null; diff --git a/api/src/core/adapters/dbApi/kysely/migrations/1717162141365_create-initial-tables.ts b/api/src/core/adapters/dbApi/kysely/migrations/1717162141365_create-initial-tables.ts index 93351b57..1f29be4e 100644 --- a/api/src/core/adapters/dbApi/kysely/migrations/1717162141365_create-initial-tables.ts +++ b/api/src/core/adapters/dbApi/kysely/migrations/1717162141365_create-initial-tables.ts @@ -30,7 +30,6 @@ export async function up(db: Kysely): Promise { .addColumn("doRespectRgaa", "boolean") // from ??? .addColumn("isStillInObservation", "boolean", col => col.notNull()) - .addColumn("similarSoftwareExternalDataIds", "jsonb") .addColumn("parentSoftwareWikidataId", "text") .addColumn("catalogNumeriqueGouvFrId", "text") .addColumn("workshopUrls", "jsonb", col => col.notNull()) diff --git a/api/src/core/adapters/dbApi/kysely/migrations/1720187269028_add-indexes.ts b/api/src/core/adapters/dbApi/kysely/migrations/1720187269028_add-indexes.ts index a5a48ea8..8c6e51ff 100644 --- a/api/src/core/adapters/dbApi/kysely/migrations/1720187269028_add-indexes.ts +++ b/api/src/core/adapters/dbApi/kysely/migrations/1720187269028_add-indexes.ts @@ -1,46 +1,54 @@ import type { Kysely } from "kysely"; const softwares_externalIdIdx = "softwares__externalId_idx"; +const softwares_parentExternalIdIdx = "softwares__parentExternalId_idx"; const compiledSoftwares_SoftwareIdIdx = "compiled_softwares__softwareId_idx"; -const compiledSoftwares_GroupByIdx = "compiled_softwares_group_by_idx"; +const software_similarExternalIdIdx = "softwares_similarExternalId_idx"; +const software_softwareIdIdx = "softwares_similarSoftwareId_idx"; const softwareReferents_softwareIdIdx = "softwareReferents_software_idx"; const softwareUsers_softwareIdIdx = "softwareUsers_software_idx"; const instances_mainSoftwareSillIdIdx = "instances_mainSoftwareSillId_idx"; export async function up(db: Kysely): Promise { await db.schema.createIndex(softwares_externalIdIdx).on("softwares").column("externalId").execute(); + await db.schema + .createIndex(softwares_parentExternalIdIdx) + .on("softwares") + .column("parentSoftwareWikidataId") + .execute(); + + await db.schema + .createIndex(software_similarExternalIdIdx) + .on("softwares__similar_software_external_datas") + .column("similarExternalId") + .execute(); + await db.schema + .createIndex(software_softwareIdIdx) + .on("softwares__similar_software_external_datas") + .column("softwareId") + .execute(); + await db.schema .createIndex(compiledSoftwares_SoftwareIdIdx) .on("compiled_softwares") .column("softwareId") .execute(); - await db.schema .createIndex(softwareReferents_softwareIdIdx) .on("software_referents") .column("softwareId") .execute(); - await db.schema.createIndex(softwareUsers_softwareIdIdx).on("software_users").column("softwareId").execute(); await db.schema.createIndex(instances_mainSoftwareSillIdIdx).on("instances").column("mainSoftwareSillId").execute(); - - // CREATE INDEX idx_compiled_softwares_group_by ON compiled_softwares (softwareId, annuaireCnllServiceProviders, comptoirDuLibreSoftware, latestVersion, parentWikidataSoftware, serviceProviders, similarExternalSoftwares, softwareExternalData); - await db.schema - .createIndex(compiledSoftwares_GroupByIdx) - .on("compiled_softwares") - .column("softwareId") - .column("annuaireCnllServiceProviders") - .column("comptoirDuLibreSoftware") - .column("latestVersion") - .column("serviceProviders") - .execute(); } export async function down(db: Kysely): Promise { + await db.schema.dropIndex(softwares_parentExternalIdIdx).execute(); await db.schema.dropIndex(softwares_externalIdIdx).execute(); + await db.schema.dropIndex(software_similarExternalIdIdx).execute(); + await db.schema.dropIndex(software_softwareIdIdx).execute(); await db.schema.dropIndex(compiledSoftwares_SoftwareIdIdx).execute(); await db.schema.dropIndex(softwareReferents_softwareIdIdx).execute(); await db.schema.dropIndex(softwareUsers_softwareIdIdx).execute(); await db.schema.dropIndex(instances_mainSoftwareSillIdIdx).execute(); - await db.schema.dropIndex(compiledSoftwares_GroupByIdx).execute(); } diff --git a/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts b/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts index 557de10a..15a59245 100644 --- a/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts +++ b/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts @@ -1,6 +1,8 @@ import { Kysely } from "kysely"; +import * as fs from "node:fs"; import { beforeEach, describe, expect, it, afterEach } from "vitest"; import { expectPromiseToFailWith, expectToEqual } from "../../../../tools/test.helpers"; +import { compiledDataPrivateToPublic } from "../../../ports/CompileData"; import { Agent, DbApiV2 } from "../../../ports/DbApiV2"; import { SoftwareExternalData } from "../../../ports/GetSoftwareExternalData"; import { SoftwareFormData } from "../../../usecases/readWriteSillData"; @@ -76,13 +78,22 @@ describe("pgDbApi", () => { describe("getCompiledDataPrivate", () => { it("gets private compiled data", async () => { const compiledDataPrivate = await dbApi.getCompiledDataPrivate(); - const { users, referents, instances, ...firstSoftware } = compiledDataPrivate.find(s => s.id === 42)!; - console.log(firstSoftware); + console.log("compiledDataPrivate.length : ", compiledDataPrivate.length); + // write softwares to file + const publicCompiledData = compiledDataPrivateToPublic(compiledDataPrivate); + publicCompiledData.sort((a, b) => (a.id >= b.id ? 1 : -1)); + const data = JSON.stringify(publicCompiledData, null, 2); + fs.writeFileSync("./my-ordered-from-db.json", data); + + console.log("publicCompiledData", JSON.stringify(publicCompiledData, null, 2)); // - console.log(`Users n = ${users?.length} : `, users); - console.log(`Referents n = ${referents?.length} : `, referents); - console.log(`Instances n = ${instances?.length} : `, instances); - expect(compiledDataPrivate).toHaveLength(100); + // const { users, referents, instances, ...firstSoftware } = compiledDataPrivate.find(s => s.id === 42)!; + // console.log(firstSoftware); + // // + // console.log(`Users n = ${users?.length} : `, users); + // console.log(`Referents n = ${referents?.length} : `, referents); + // console.log(`Instances n = ${instances?.length} : `, instances); + // expect(compiledDataPrivate).toHaveLength(100); }); });