improve query to get all private compile software

api/scripts/load-git-repo-in-pg.ts

@@ -48,13 +48,13 @@ const insertSoftwares = async (softwareRows: SoftwareRow[], db: Kysely<Database>
     console.info("Number of softwares to insert : ", softwareRows.length);
     await db.transaction().execute(async trx => {
         await trx.deleteFrom("softwares").execute();
+        await trx.deleteFrom("softwares__similar_software_external_datas").execute();
         await trx
             .insertInto("softwares")
             .values(
-                softwareRows.map(row => ({
+                softwareRows.map(({ similarSoftwareExternalDataIds: _, ...row }) => ({
                     ...row,
                     dereferencing: row.dereferencing ? JSON.stringify(row.dereferencing) : null,
-                    similarSoftwareExternalDataIds: JSON.stringify(row.similarSoftwareExternalDataIds),
                     softwareType: JSON.stringify(row.softwareType),
                     workshopUrls: JSON.stringify(row.workshopUrls),
                     testUrls: JSON.stringify(row.testUrls),
@@ -63,6 +63,18 @@ const insertSoftwares = async (softwareRows: SoftwareRow[], db: Kysely<Database>
                 }))
             )
             .executeTakeFirst();
+
+        await trx
+            .insertInto("softwares__similar_software_external_datas")
+            .values(
+                softwareRows.flatMap(row =>
+                    Array.from(new Set(row.similarSoftwareExternalDataIds)).map(externalId => ({
+                        softwareId: row.id,
+                        similarExternalId: externalId
+                    }))
+                )
+            )
+            .execute();
     });
 };
 
@@ -168,6 +180,7 @@ const insertCompiledSoftwaresAndSoftwareExternalData = async (
             .executeTakeFirst();
 
         await trx.deleteFrom("software_external_datas").execute();
+
         await trx
             .insertInto("software_external_datas")
             .values(
@@ -203,6 +216,25 @@ const insertCompiledSoftwaresAndSoftwareExternalData = async (
             )
             .onConflict(conflict => conflict.column("externalId").doNothing())
             .executeTakeFirst();
+
+        await trx
+            .insertInto("software_external_datas")
+            .values(
+                compiledSoftwares
+                    .filter(s => s.similarExternalSoftwares.length > 0)
+                    .flatMap(s =>
+                        (s.similarExternalSoftwares ?? []).map(similarExternalSoftware => ({
+                            externalId: similarExternalSoftware.externalId,
+                            externalDataOrigin: similarExternalSoftware.externalDataOrigin,
+                            developers: JSON.stringify([]),
+                            label: JSON.stringify(similarExternalSoftware?.label ?? {}),
+                            description: JSON.stringify(similarExternalSoftware?.description ?? {}),
+                            isLibreSoftware: similarExternalSoftware?.isLibreSoftware ?? false
+                        }))
+                    )
+            )
+            .onConflict(conflict => conflict.column("externalId").doNothing())
+            .executeTakeFirst();
     });
 };
 

api/src/core/adapters/dbApi/kysely/createGetCompiledData.ts

@@ -3,7 +3,7 @@ import { CompiledData } from "../../../ports/CompileData";
 import { Db } from "../../../ports/DbApi";
 import { ParentSoftwareExternalData, SoftwareExternalData } from "../../../ports/GetSoftwareExternalData";
 import { Database } from "./kysely.database";
-import { convertNullValuesToUndefined, isNotNull, jsonBuildObject } from "./kysely.utils";
+import { convertNullValuesToUndefined, isNotNull, jsonBuildObject, jsonStripNulls } from "./kysely.utils";
 
 export const createGetCompiledData = (db: Kysely<Database>) => async (): Promise<CompiledData<"private">> => {
     console.time("agentById query");
@@ -90,35 +90,26 @@ export const createGetCompiledData = (db: Kysely<Database>) => async (): Promise
                     .case()
                     .when("ext.externalId", "is not", null)
                     .then(
-                        jsonBuildObject({
-                            externalId: ref("ext.externalId"),
-                            externalDataOrigin: ref("ext.externalDataOrigin"),
-                            developers: ref("ext.developers"),
-                            label: ref("ext.label"),
-                            description: ref("ext.description"),
-                            isLibreSoftware: ref("ext.isLibreSoftware"),
-                            logoUrl: ref("ext.logoUrl"),
-                            framaLibreId: ref("ext.framaLibreId"),
-                            websiteUrl: ref("ext.websiteUrl"),
-                            sourceUrl: ref("ext.sourceUrl"),
-                            documentationUrl: ref("ext.documentationUrl")
-                        }).$castTo<SoftwareExternalData>()
+                        jsonStripNulls(
+                            jsonBuildObject({
+                                externalId: ref("ext.externalId"),
+                                externalDataOrigin: ref("ext.externalDataOrigin"),
+                                developers: ref("ext.developers"),
+                                label: ref("ext.label"),
+                                description: ref("ext.description"),
+                                isLibreSoftware: ref("ext.isLibreSoftware"),
+                                logoUrl: ref("ext.logoUrl"),
+                                framaLibreId: ref("ext.framaLibreId"),
+                                websiteUrl: ref("ext.websiteUrl"),
+                                sourceUrl: ref("ext.sourceUrl"),
+                                documentationUrl: ref("ext.documentationUrl"),
+                                license: ref("ext.license")
+                            })
+                        ).$castTo<SoftwareExternalData>()
                     )
                     .end()
                     .as("softwareExternalData"),
-            ({ ref, fn }) =>
-                fn
-                    .jsonAgg(
-                        jsonBuildObject({
-                            externalId: ref("similarExt.externalId"),
-                            label: ref("similarExt.label"),
-                            description: ref("similarExt.description"),
-                            isLibreSoftware: ref("similarExt.isLibreSoftware"),
-                            externalDataOrigin: ref("similarExt.externalDataOrigin")
-                        }).$castTo<SoftwareExternalData>()
-                    )
-                    .filterWhere("similarExt.externalId", "is not", null)
-                    .as("similarExternalSoftwares"),
+            ({ fn }) => fn.jsonAgg("similarExt").distinct().as("similarExternalSoftwares"),
             ({ fn }) => fn.jsonAgg("users").distinct().as("users"),
             ({ fn }) => fn.jsonAgg("referents").distinct().as("referents"),
             ({ fn }) => fn.jsonAgg("instances").distinct().as("instances")
@@ -158,7 +149,16 @@ export const createGetCompiledData = (db: Kysely<Database>) => async (): Promise
                         parentWikidataSoftware: parentWikidataSoftware ?? undefined,
                         dereferencing: dereferencing ?? undefined,
                         serviceProviders: serviceProviders ?? [],
-                        similarExternalSoftwares: similarExternalSoftwares ?? [],
+                        similarExternalSoftwares: (similarExternalSoftwares ?? [])
+                            .filter(isNotNull)
+                            .map(similar => ({
+                                "externalId": similar.externalId!,
+                                "externalDataOrigin": similar.externalDataOrigin!,
+                                "label": similar.label!,
+                                "description": similar.description!,
+                                "isLibreSoftware": similar.isLibreSoftware!
+                            }))
+                            .sort((a, b) => a.externalId.localeCompare(b.externalId)),
                         users: users.filter(isNotNull).map(user => ({
                             ...(user as any),
                             organization: agentById[user.agentId!]?.organization
@@ -167,8 +167,13 @@ export const createGetCompiledData = (db: Kysely<Database>) => async (): Promise
                             ...(referent as any),
                             organization: agentById[referent.agentId!]?.organization
                         })),
-                        instances: instances.filter(isNotNull).map(instance => ({
-                            ...(instance as any)
+                        instances: (instances ?? []).filter(isNotNull).map(instance => ({
+                            id: instance.id!,
+                            organization: instance.organization!,
+                            targetAudience: instance.targetAudience!,
+                            publicUrl: instance.publicUrl ?? undefined,
+                            addedByAgentEmail: instance.addedByAgentEmail!,
+                            otherWikidataSoftwares: []
                         }))
                     };
                 }
@@ -177,5 +182,6 @@ export const createGetCompiledData = (db: Kysely<Database>) => async (): Promise
             return processedSoftwares;
         });
 
+    console.log("numberOfCompiledSoftwares : ", compliedSoftwares.length);
     return compliedSoftwares;
 };

api/src/core/adapters/dbApi/kysely/kysely.database.ts

@@ -107,7 +107,6 @@ type SoftwaresTable = {
     doRespectRgaa: boolean | null;
     isFromFrenchPublicService: boolean;
     isPresentInSupportContract: boolean;
-    similarSoftwareExternalDataIds: JSONColumnType<string[]>;
     parentSoftwareWikidataId: string | null;
     externalId: string | null;
     externalDataOrigin: "wikidata" | "HAL" | null;

api/src/core/adapters/dbApi/kysely/migrations/1717162141365_create-initial-tables.ts

@@ -30,7 +30,6 @@ export async function up(db: Kysely<any>): Promise<void> {
         .addColumn("doRespectRgaa", "boolean")
         // from ???
         .addColumn("isStillInObservation", "boolean", col => col.notNull())
-        .addColumn("similarSoftwareExternalDataIds", "jsonb")
         .addColumn("parentSoftwareWikidataId", "text")
         .addColumn("catalogNumeriqueGouvFrId", "text")
         .addColumn("workshopUrls", "jsonb", col => col.notNull())

api/src/core/adapters/dbApi/kysely/migrations/1720187269028_add-indexes.ts

@@ -1,46 +1,54 @@
 import type { Kysely } from "kysely";
 
 const softwares_externalIdIdx = "softwares__externalId_idx";
+const softwares_parentExternalIdIdx = "softwares__parentExternalId_idx";
 const compiledSoftwares_SoftwareIdIdx = "compiled_softwares__softwareId_idx";
-const compiledSoftwares_GroupByIdx = "compiled_softwares_group_by_idx";
+const software_similarExternalIdIdx = "softwares_similarExternalId_idx";
+const software_softwareIdIdx = "softwares_similarSoftwareId_idx";
 const softwareReferents_softwareIdIdx = "softwareReferents_software_idx";
 const softwareUsers_softwareIdIdx = "softwareUsers_software_idx";
 const instances_mainSoftwareSillIdIdx = "instances_mainSoftwareSillId_idx";
 
 export async function up(db: Kysely<any>): Promise<void> {
     await db.schema.createIndex(softwares_externalIdIdx).on("softwares").column("externalId").execute();
+    await db.schema
+        .createIndex(softwares_parentExternalIdIdx)
+        .on("softwares")
+        .column("parentSoftwareWikidataId")
+        .execute();
+
+    await db.schema
+        .createIndex(software_similarExternalIdIdx)
+        .on("softwares__similar_software_external_datas")
+        .column("similarExternalId")
+        .execute();
+    await db.schema
+        .createIndex(software_softwareIdIdx)
+        .on("softwares__similar_software_external_datas")
+        .column("softwareId")
+        .execute();
+
     await db.schema
         .createIndex(compiledSoftwares_SoftwareIdIdx)
         .on("compiled_softwares")
         .column("softwareId")
         .execute();
-
     await db.schema
         .createIndex(softwareReferents_softwareIdIdx)
         .on("software_referents")
         .column("softwareId")
         .execute();
-
     await db.schema.createIndex(softwareUsers_softwareIdIdx).on("software_users").column("softwareId").execute();
     await db.schema.createIndex(instances_mainSoftwareSillIdIdx).on("instances").column("mainSoftwareSillId").execute();
-
-    // CREATE INDEX idx_compiled_softwares_group_by ON compiled_softwares (softwareId, annuaireCnllServiceProviders, comptoirDuLibreSoftware, latestVersion, parentWikidataSoftware, serviceProviders, similarExternalSoftwares, softwareExternalData);
-    await db.schema
-        .createIndex(compiledSoftwares_GroupByIdx)
-        .on("compiled_softwares")
-        .column("softwareId")
-        .column("annuaireCnllServiceProviders")
-        .column("comptoirDuLibreSoftware")
-        .column("latestVersion")
-        .column("serviceProviders")
-        .execute();
 }
 
 export async function down(db: Kysely<any>): Promise<void> {
+    await db.schema.dropIndex(softwares_parentExternalIdIdx).execute();
     await db.schema.dropIndex(softwares_externalIdIdx).execute();
+    await db.schema.dropIndex(software_similarExternalIdIdx).execute();
+    await db.schema.dropIndex(software_softwareIdIdx).execute();
     await db.schema.dropIndex(compiledSoftwares_SoftwareIdIdx).execute();
     await db.schema.dropIndex(softwareReferents_softwareIdIdx).execute();
     await db.schema.dropIndex(softwareUsers_softwareIdIdx).execute();
     await db.schema.dropIndex(instances_mainSoftwareSillIdIdx).execute();
-    await db.schema.dropIndex(compiledSoftwares_GroupByIdx).execute();
 }

api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts

@@ -1,6 +1,8 @@
 import { Kysely } from "kysely";
+import * as fs from "node:fs";
 import { beforeEach, describe, expect, it, afterEach } from "vitest";
 import { expectPromiseToFailWith, expectToEqual } from "../../../../tools/test.helpers";
+import { compiledDataPrivateToPublic } from "../../../ports/CompileData";
 import { Agent, DbApiV2 } from "../../../ports/DbApiV2";
 import { SoftwareExternalData } from "../../../ports/GetSoftwareExternalData";
 import { SoftwareFormData } from "../../../usecases/readWriteSillData";
@@ -76,13 +78,22 @@ describe("pgDbApi", () => {
     describe("getCompiledDataPrivate", () => {
         it("gets private compiled data", async () => {
             const compiledDataPrivate = await dbApi.getCompiledDataPrivate();
-            const { users, referents, instances, ...firstSoftware } = compiledDataPrivate.find(s => s.id === 42)!;
-            console.log(firstSoftware);
+            console.log("compiledDataPrivate.length : ", compiledDataPrivate.length);
+            // write softwares to file
+            const publicCompiledData = compiledDataPrivateToPublic(compiledDataPrivate);
+            publicCompiledData.sort((a, b) => (a.id >= b.id ? 1 : -1));
+            const data = JSON.stringify(publicCompiledData, null, 2);
+            fs.writeFileSync("./my-ordered-from-db.json", data);
+
+            console.log("publicCompiledData", JSON.stringify(publicCompiledData, null, 2));
             //
-            console.log(`Users n = ${users?.length} : `, users);
-            console.log(`Referents n = ${referents?.length} : `, referents);
-            console.log(`Instances n = ${instances?.length} : `, instances);
-            expect(compiledDataPrivate).toHaveLength(100);
+            // const { users, referents, instances, ...firstSoftware } = compiledDataPrivate.find(s => s.id === 42)!;
+            // console.log(firstSoftware);
+            // //
+            // console.log(`Users n = ${users?.length} : `, users);
+            // console.log(`Referents n = ${referents?.length} : `, referents);
+            // console.log(`Instances n = ${instances?.length} : `, instances);
+            // expect(compiledDataPrivate).toHaveLength(100);
         });
     });