use pg repository

api/package.json

@@ -10,13 +10,14 @@
     "types": "dist/src/lib/index.d.ts",
     "scripts": {
         "migrate": "dotenv -e ../.env -- kysely migrate",
+        "db:up": "yarn migrate latest",
         "prepare": "[ ! -f .env.local.sh ] && cp .env.sh .env.local.sh || true",
         "test": "vitest --watch=false",
         "dev": "yarn build && yarn start",
         "build": "tsc",
-        "start": "dotenv -e ../.env -- forever dist/src/main.js",
+        "start": "yarn db:up && dotenv -e ../.env -- forever dist/src/main.js",
         "build-prod": "esbuild src/main.ts --bundle --platform=node --target=node20 --outfile=dist/index.js",
-        "start-prod": "dotenv -e ../.env -- forever dist/index.js",
+        "start-prod": "yarn db:up && dotenv -e ../.env -- forever dist/index.js",
         "_format": "prettier \"**/*.{ts,tsx,json,md}\"",
         "format": "yarn run _format --write",
         "format:check": "yarn run _format --list-different",

api/scripts/compile-data.ts

@@ -1,22 +1,29 @@
+import { Kysely } from "kysely";
 import { bootstrapCore } from "../src/core";
+import type { Database } from "../src/core/adapters/dbApi/kysely/kysely.database";
+import { createPgDialect } from "../src/core/adapters/dbApi/kysely/kysely.dialect";
 import { env } from "../src/env";
 
 (async () => {
+    const kyselyDb = new Kysely<Database>({ dialect: createPgDialect(env.databaseUrl) });
     const { core } = await bootstrapCore({
         "keycloakUserApiParams": undefined,
         "dbConfig": {
-            "dbKind": "git",
-            "dataRepoSshUrl": "[email protected]:codegouvfr/sill-data.git",
-            "sshPrivateKey": env.sshPrivateKeyForGit,
-            "sshPrivateKeyName": env.sshPrivateKeyForGitName
+            "dbKind": "kysely",
+            "kyselyDb": kyselyDb
+            // "dataRepoSshUrl": "[email protected]:codegouvfr/sill-data.git",
+            // "sshPrivateKey": env.sshPrivateKeyForGit,
+            // "sshPrivateKeyName": env.sshPrivateKeyForGitName
         },
         "githubPersonalAccessTokenForApiRateLimit": env.githubPersonalAccessTokenForApiRateLimit,
         "doPerPerformPeriodicalCompilation": false,
         "doPerformCacheInitialization": false,
         "externalSoftwareDataOrigin": env.externalSoftwareDataOrigin
     });
 
-    await core.functions.readWriteSillData.manuallyTriggerNonIncrementalCompilation();
+    console.log("core initialized, TODO TRIGGER INCREMENTAL compilation", core.functions);
+
+    // await core.functions.readWriteSillData.manuallyTriggerNonIncrementalCompilation();
 
     process.exit(0);
 })();

api/scripts/load-git-repo-in-pg.ts

@@ -1,6 +1,7 @@
-import { InsertObject, Kysely } from "kysely";
+import { InsertObject, Kysely, sql } from "kysely";
 import { z } from "zod";
 import { createGitDbApi, GitDbApiParams } from "../src/core/adapters/dbApi/createGitDbApi";
+import { makeGetAgentIdByEmail } from "../src/core/adapters/dbApi/kysely/createPgAgentRepository";
 import { Database } from "../src/core/adapters/dbApi/kysely/kysely.database";
 import { createPgDialect } from "../src/core/adapters/dbApi/kysely/kysely.dialect";
 import { CompiledData } from "../src/core/ports/CompileData";
@@ -20,30 +21,36 @@ const saveGitDbInPostgres = async ({ pgConfig, gitDbConfig }: Params) => {
 
     const { softwareRows, agentRows, softwareReferentRows, softwareUserRows, instanceRows } = await gitDbApi.fetchDb();
 
-    await insertSoftwares(softwareRows, pgDb);
     await insertAgents(agentRows, pgDb);
 
     const agentIdByEmail = await makeGetAgentIdByEmail(pgDb);
+
+    await insertSoftwares(softwareRows, agentIdByEmail, pgDb);
     await insertSoftwareReferents({
         softwareReferentRows: softwareReferentRows,
-        agentIdByEmail: agentIdByEmail,
+        agentIdByEmail,
         db: pgDb
     });
     await insertSoftwareUsers({
         softwareUserRows: softwareUserRows,
-        agentIdByEmail: agentIdByEmail,
+        agentIdByEmail,
         db: pgDb
     });
     await insertInstances({
         instanceRows: instanceRows,
+        agentIdByEmail,
         db: pgDb
     });
 
     const compiledSoftwares = await gitDbApi.fetchCompiledData();
     await insertCompiledSoftwaresAndSoftwareExternalData(compiledSoftwares, pgDb);
 };
 
-const insertSoftwares = async (softwareRows: SoftwareRow[], db: Kysely<Database>) => {
+const insertSoftwares = async (
+    softwareRows: SoftwareRow[],
+    agentIdByEmail: Record<string, number>,
+    db: Kysely<Database>
+) => {
     console.info("Deleting than Inserting softwares");
     console.info("Number of softwares to insert : ", softwareRows.length);
     await db.transaction().execute(async trx => {
@@ -52,8 +59,9 @@ const insertSoftwares = async (softwareRows: SoftwareRow[], db: Kysely<Database>
         await trx
             .insertInto("softwares")
             .values(
-                softwareRows.map(({ similarSoftwareExternalDataIds: _, ...row }) => ({
+                softwareRows.map(({ similarSoftwareExternalDataIds: _, addedByAgentEmail, ...row }) => ({
                     ...row,
+                    addedByAgentId: agentIdByEmail[addedByAgentEmail],
                     dereferencing: row.dereferencing ? JSON.stringify(row.dereferencing) : null,
                     softwareType: JSON.stringify(row.softwareType),
                     workshopUrls: JSON.stringify(row.workshopUrls),
@@ -63,6 +71,7 @@ const insertSoftwares = async (softwareRows: SoftwareRow[], db: Kysely<Database>
                 }))
             )
             .executeTakeFirst();
+        await sql`SELECT setval('softwares_id_seq', (SELECT MAX(id) FROM softwares))`.execute(trx);
 
         await trx
             .insertInto("softwares__similar_software_external_datas")
@@ -82,17 +91,14 @@ const insertAgents = async (agentRows: Db.AgentRow[], db: Kysely<Database>) => {
     console.log("Deleting than Inserting agents");
     console.info("Number of agents to insert : ", agentRows.length);
     await db.transaction().execute(async trx => {
+        await trx.deleteFrom("instances").execute();
+        await trx.deleteFrom("softwares").execute();
         await trx.deleteFrom("agents").execute();
         await trx.insertInto("agents").values(agentRows).executeTakeFirst();
+        await sql`SELECT setval('agents_id_seq', (SELECT MAX(id) FROM agents))`.execute(trx);
     });
 };
 
-const makeGetAgentIdByEmail = async (db: Kysely<Database>): Promise<Record<string, number>> => {
-    console.info("Fetching agents, to map email to id");
-    const agents = await db.selectFrom("agents").select(["email", "id"]).execute();
-    return agents.reduce((acc, agent) => ({ ...acc, [agent.email]: agent.id }), {});
-};
-
 const insertSoftwareReferents = async ({
     softwareReferentRows,
     agentIdByEmail,
@@ -143,12 +149,29 @@ const insertSoftwareUsers = async ({
     });
 };
 
-const insertInstances = async ({ instanceRows, db }: { instanceRows: Db.InstanceRow[]; db: Kysely<Database> }) => {
+const insertInstances = async ({
+    instanceRows,
+    agentIdByEmail,
+    db
+}: {
+    instanceRows: Db.InstanceRow[];
+    agentIdByEmail: Record<string, number>;
+    db: Kysely<Database>;
+}) => {
     console.info("Deleting than Inserting instances");
     console.info("Number of instances to insert : ", instanceRows.length);
     await db.transaction().execute(async trx => {
         await trx.deleteFrom("instances").execute();
-        await trx.insertInto("instances").values(instanceRows).executeTakeFirst();
+        await trx
+            .insertInto("instances")
+            .values(
+                instanceRows.map(({ addedByAgentEmail, ...instanceRow }) => ({
+                    ...instanceRow,
+                    addedByAgentId: agentIdByEmail[addedByAgentEmail]
+                }))
+            )
+            .executeTakeFirst();
+        await sql`SELECT setval('instances_id_seq', (SELECT MAX(id) FROM instances))`.execute(trx);
     });
 };
 

api/src/core/adapters/comptoirDuLibreApi.ts

@@ -9,6 +9,7 @@ export const comptoirDuLibreApi: ComptoirDuLibreApi = {
     "getComptoirDuLibre": memoize(
         async () => {
             try {
+                console.info("Fetching comptoir du libre");
                 const res = await fetch(url);
 
                 if (res.status !== 200) {
@@ -34,6 +35,7 @@ export const comptoirDuLibreApi: ComptoirDuLibreApi = {
         let imgSrc: string | undefined;
 
         try {
+            console.info(`Fetching comptoir du libre icon, for comptoirDuLibreId : ${comptoirDuLibreId}`);
             const body = await fetch(`https://comptoir-du-libre.org/fr/softwares/${comptoirDuLibreId}`).then(r =>
                 r.text()
             );
@@ -55,6 +57,7 @@ export const comptoirDuLibreApi: ComptoirDuLibreApi = {
         let $: CheerioAPI;
 
         try {
+            console.info(`Fetching comptoir du libre keywords, for comptoirDuLibreId : ${comptoirDuLibreId}`);
             const body = await fetch(`https://comptoir-du-libre.org/fr/softwares/${comptoirDuLibreId}`).then(r =>
                 r.text()
             );

api/src/core/adapters/dbApi/createGitDbApi.ts

@@ -24,7 +24,7 @@ export type GitDbApiParams = {
     sshPrivateKey: string;
 };
 
-export function createGitDbApi(params: GitDbApiParams): Db.DbApiAndInitializeCache {
+export function createGitDbApi(params: GitDbApiParams): { dbApi: DbApi; initializeDbApiCache: () => Promise<void> } {
     const { dataRepoSshUrl, sshPrivateKeyName, sshPrivateKey } = params;
 
     const dbApi: DbApi = {

api/src/core/adapters/dbApi/kysely/createGetCompiledData.ts

@@ -3,7 +3,7 @@ import { CompiledData } from "../../../ports/CompileData";
 import { Db } from "../../../ports/DbApi";
 import { ParentSoftwareExternalData, SoftwareExternalData } from "../../../ports/GetSoftwareExternalData";
 import { Database } from "./kysely.database";
-import { convertNullValuesToUndefined, isNotNull, jsonBuildObject, jsonStripNulls } from "./kysely.utils";
+import { stripNullOrUndefinedValues, isNotNull, jsonBuildObject, jsonStripNulls } from "./kysely.utils";
 
 export const createGetCompiledData = (db: Kysely<Database>) => async (): Promise<CompiledData<"private">> => {
     console.time("agentById query");
@@ -45,7 +45,7 @@ export const createGetCompiledData = (db: Kysely<Database>) => async (): Promise
         ])
         .select([
             "s.id",
-            "s.addedByAgentEmail",
+            "s.addedByAgentId",
             "s.catalogNumeriqueGouvFrId",
             "s.categories",
             "s.dereferencing",
@@ -120,6 +120,7 @@ export const createGetCompiledData = (db: Kysely<Database>) => async (): Promise
             console.time("software processing");
             const processedSoftwares = results.map(
                 ({
+                    addedByAgentId,
                     externalDataSoftwareId,
                     annuaireCnllServiceProviders,
                     comptoirDuLibreSoftware,
@@ -138,7 +139,8 @@ export const createGetCompiledData = (db: Kysely<Database>) => async (): Promise
                     ...software
                 }): CompiledData.Software<"private"> => {
                     return {
-                        ...convertNullValuesToUndefined(software),
+                        ...stripNullOrUndefinedValues(software),
+                        addedByAgentEmail: agentById[addedByAgentId].email,
                         updateTime: new Date(+updateTime).getTime(),
                         referencedSinceTime: new Date(+referencedSinceTime).getTime(),
                         doRespectRgaa,
@@ -172,7 +174,7 @@ export const createGetCompiledData = (db: Kysely<Database>) => async (): Promise
                             organization: instance.organization!,
                             targetAudience: instance.targetAudience!,
                             publicUrl: instance.publicUrl ?? undefined,
-                            addedByAgentEmail: instance.addedByAgentEmail!,
+                            addedByAgentEmail: agentById[instance.addedByAgentId!].email,
                             otherWikidataSoftwares: []
                         }))
                     };