feat(ui): modules_to_packages endpoint (#101)

Signed-off-by: Tomas Pilar <[email protected]>

.dockerignore

@@ -1 +1,4 @@
 node_modules
+
+# Package DB is created during docker build
+static/package-db.sqlite

Dockerfile

@@ -1,3 +1,16 @@
+ARG ALPINE_VERSION="3.20"
+ARG GO_VERSION="1.23"
+ARG UPM_VERSION="2.5.2"
+
+FROM docker.io/golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS build-upm
+ARG UPM_VERSION
+WORKDIR /workdir
+RUN apk add --no-cache git make gcc g++ musl-dev sqlite && \
+  git clone --depth 1 --branch "v${UPM_VERSION}" https://github.com/replit/upm.git  && \
+  cd ./upm && \
+  export CGO_CFLAGS="-D_LARGEFILE64_SOURCE" && \
+  make internal/backends/python/pypi_map.sqlite
+
 FROM node:22.2-alpine AS base
 
 # Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
@@ -38,6 +51,9 @@ RUN deluser --remove-home node \
   && addgroup -S node -g 1001 \
   && adduser -S -G node -u 1001 node
 
+# UPM
+COPY --from=build-upm /workdir/upm/internal/backends/python/pypi_map.sqlite ./static/package-db.sqlite
+
 COPY --chown=node:node --from=deps ${APP_DIR}/package.json ./
 COPY --chown=node:node --from=deps ${APP_DIR}/node_modules ./node_modules
 COPY --chown=node:node --from=builder ${APP_DIR}/dist ./dist

package.json

@@ -74,6 +74,8 @@
     "pino": "^9.2.0",
     "prom-client": "^15.1.3",
     "remeda": "^2.2.2",
+    "sqlite": "^5.1.1",
+    "sqlite3": "^5.1.7",
     "yaml": "^2.5.0",
     "yauzl": "^3.1.3",
     "zod": "^3.23.8",

src/config.ts

@@ -185,3 +185,5 @@ export const VECTOR_STORE_FILE_QUOTA_DAILY = VECTOR_STORE_FILE_QUOTA_DAILY_RAW
   : Infinity;
 
 export const ARTIFACT_SECRET_RATE_LIMIT = parseInt(getEnv('ARTIFACT_SECRET_RATE_LIMIT', '25'));
+
+export const PACKAGE_DB = getEnv('PACKAGE_DB', './static/package-db.sqlite');

src/ui/dtos/modules-to-packages.ts

@@ -0,0 +1,42 @@
+/**
+ * Copyright 2024 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { FromSchema, JSONSchema } from 'json-schema-to-ts';
+
+export const modulesToPackagesQuerySchema = {
+  type: 'object',
+  required: ['modules'],
+  properties: {
+    modules: {
+      type: 'array',
+      minItems: 1,
+      items: { type: 'string' }
+    }
+  }
+} as const satisfies JSONSchema;
+export type ModulesToPackagesQuery = FromSchema<typeof modulesToPackagesQuerySchema>;
+
+export const modulesToPackagesResponseSchema = {
+  type: 'object',
+  required: ['packages'],
+  properties: {
+    packages: {
+      type: 'array',
+      items: { type: 'string' }
+    }
+  }
+} as const satisfies JSONSchema;
+export type ModulesToPackagesResponse = FromSchema<typeof modulesToPackagesResponseSchema>;

src/ui/ui.module.ts

@@ -17,10 +17,16 @@
 import { FastifyPluginAsyncJsonSchemaToTs } from '@fastify/type-provider-json-schema-to-ts';
 import { StatusCodes } from 'http-status-codes';
 
-import { lastAssistants } from './ui.service.js';
+import { lastAssistants, modulesToPackages } from './ui.service.js';
 import { lastAssistantsSchema } from './dtos/last_assistant.dto.js';
+import {
+  ModulesToPackagesQuery,
+  modulesToPackagesQuerySchema,
+  modulesToPackagesResponseSchema
+} from './dtos/modules-to-packages.js';
 
 import { Tag } from '@/swagger.js';
+import { AuthSecret } from '@/auth/utils.js';
 
 export const uiModule: FastifyPluginAsyncJsonSchemaToTs = async (app) => {
   app.get(
@@ -36,4 +42,23 @@ export const uiModule: FastifyPluginAsyncJsonSchemaToTs = async (app) => {
       return lastAssistants();
     }
   );
+
+  app.get<{ Querystring: ModulesToPackagesQuery }>(
+    '/ui/modules_to_packages',
+    {
+      schema: {
+        querystring: modulesToPackagesQuerySchema,
+        response: { [StatusCodes.OK]: modulesToPackagesResponseSchema },
+        tags: [Tag.BEE_API]
+      },
+      preHandler: app.auth([
+        AuthSecret.ACCESS_TOKEN,
+        AuthSecret.API_KEY,
+        AuthSecret.ARTIFACT_SECRET
+      ])
+    },
+    async (req) => {
+      return modulesToPackages(req.query);
+    }
+  );
 };

src/ui/ui.service.ts

@@ -14,12 +14,23 @@
  * limitations under the License.
  */
 
+import sqlite3 from 'sqlite3';
+import { open } from 'sqlite';
+
 import { LastAssistants } from './dtos/last_assistant.dto.js';
+import { ModulesToPackagesQuery, ModulesToPackagesResponse } from './dtos/modules-to-packages.js';
 
 import { ORM } from '@/database.js';
 import { Assistant } from '@/assistants/assistant.entity.js';
 import { toDto } from '@/assistants/assistants.service.js';
 import { getProjectPrincipal } from '@/administration/helpers.js';
+import { PACKAGE_DB } from '@/config.js';
+
+const db = await open({
+  filename: PACKAGE_DB,
+  driver: sqlite3.cached.Database,
+  mode: sqlite3.OPEN_READONLY
+});
 
 export async function lastAssistants(): Promise<LastAssistants> {
   const projectPrincipal = getProjectPrincipal();
@@ -49,3 +60,15 @@ export async function lastAssistants(): Promise<LastAssistants> {
 
   return assistants.map(toDto);
 }
+
+export async function modulesToPackages({
+  modules
+}: ModulesToPackagesQuery): Promise<ModulesToPackagesResponse> {
+  const results = await db.all<{ guess: string }[]>(
+    `SELECT DISTINCT guess FROM module_to_pypi_package WHERE module_name IN (${Array.from('?'.repeat(modules.length)).join(',')})`,
+    modules
+  );
+  return {
+    packages: results.map((result) => result.guess)
+  };
+}