📦 refactor: Move DB Models to @librechat/data-schemas (#6210)

* 🚀 feat: Introduce data schemas and refactor models to use @librechat/data-schemas * 🚀 feat: Add installation step for Data Schemas Package in backend review workflow * chore: Add `data-schemas` package to update/rebuild packages scripts * chore: Update Dockerfile to include data-schemas package build process * fix: add missing @rollup/plugin-typescript package * chore: Add GitHub Actions workflow for publishing data-schemas package --------- Co-authored-by: Danny Avila <[email protected]>
danny-avila · Mar 7, 2025 · b51cd21 · b51cd21
1 parent 4d04904
commit b51cd21
Show file tree

Hide file tree

Showing 77 changed files with 1,781 additions and 984 deletions.
diff --git a/.github/workflows/backend-review.yml b/.github/workflows/backend-review.yml
@@ -39,6 +39,9 @@ jobs:
       - name: Install MCP Package
         run: npm run build:mcp
 
+      - name: Install Data Schemas Package
+        run: npm run build:data-schemas
+
       - name: Create empty auth.json file
         run: |
           mkdir -p api/data

diff --git a/.github/workflows/data-schemas.yml b/.github/workflows/data-schemas.yml
@@ -0,0 +1,34 @@
+name: Node.js Package
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'packages/data-schemas/package.json'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 16
+      - run: cd packages/data-schemas && npm ci
+      - run: cd packages/data-schemas && npm run build
+
+  publish-npm:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 16
+          registry-url: 'https://registry.npmjs.org'
+      - run: cd packages/data-schemas && npm ci
+      - run: cd packages/data-schemas && npm run build
+      - run: cd packages/data-schemas && npm publish
+        env:
+          NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
diff --git a/.github/workflows/generate-release-changelog-pr.yml b/.github/workflows/generate-release-changelog-pr.yml
@@ -84,11 +84,11 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           sign-commits: true
-          commit-message: "chore: update CHANGELOG for release ${GITHUB_REF##*/}"
+          commit-message: "chore: update CHANGELOG for release ${{ github.ref_name }}"
           base: main
-          branch: "changelog/${GITHUB_REF##*/}"
+          branch: "changelog/${{ github.ref_name }}"
           reviewers: danny-avila
-          title: "chore: update CHANGELOG for release ${GITHUB_REF##*/}"
+          title: "chore: update CHANGELOG for release ${{ github.ref_name }}"
           body: |
             **Description**:
-            - This PR updates the CHANGELOG.md by removing the "Unreleased" section and adding new release notes for release ${GITHUB_REF##*/} above previous releases.
+            - This PR updates the CHANGELOG.md by removing the "Unreleased" section and adding new release notes for release ${{ github.ref_name }} above previous releases.
diff --git a/Dockerfile.multi b/Dockerfile.multi
@@ -11,6 +11,7 @@ RUN npm config set fetch-retry-maxtimeout 600000 && \
 COPY package*.json ./
 COPY packages/data-provider/package*.json ./packages/data-provider/
 COPY packages/mcp/package*.json ./packages/mcp/
+COPY packages/data-schemas/package*.json ./packages/data-schemas/
 COPY client/package*.json ./client/
 COPY api/package*.json ./api/
 
@@ -32,6 +33,13 @@ COPY packages/mcp ./
 COPY --from=data-provider-build /app/packages/data-provider/dist /app/packages/data-provider/dist
 RUN npm run build
 
+# Build data-schemas
+FROM base AS data-schemas-build
+WORKDIR /app/packages/data-schemas
+COPY packages/data-schemas ./
+COPY --from=data-provider-build /app/packages/data-provider/dist /app/packages/data-provider/dist
+RUN npm run build
+
 # Client build
 FROM base AS client-build
 WORKDIR /app/client
@@ -49,8 +57,9 @@ COPY api ./api
 COPY config ./config
 COPY --from=data-provider-build /app/packages/data-provider/dist ./packages/data-provider/dist
 COPY --from=mcp-build /app/packages/mcp/dist ./packages/mcp/dist
+COPY --from=data-schemas-build /app/packages/data-schemas/dist ./packages/data-schemas/dist
 COPY --from=client-build /app/client/dist ./client/dist
 WORKDIR /app/api
 EXPOSE 3080
 ENV HOST=0.0.0.0
-CMD ["node", "server/index.js"]
+CMD ["node", "server/index.js"]
diff --git a/api/lib/db/indexSync.js b/api/lib/db/indexSync.js
@@ -1,5 +1,5 @@
 const { MeiliSearch } = require('meilisearch');
-const Conversation = require('~/models/schema/convoSchema');
+const Conversation = require('~/models/Conversation');
 const Message = require('~/models/schema/messageSchema');
 const { isEnabled } = require('~/server/utils');
 const { logger } = require('~/config');

diff --git a/api/models/Action.js b/api/models/Action.js
@@ -1,5 +1,5 @@
 const mongoose = require('mongoose');
-const actionSchema = require('./schema/action');
+const { actionSchema } = require('@librechat/data-schemas');
 
 const Action = mongoose.model('action', actionSchema);
 

diff --git a/api/models/Agent.js b/api/models/Agent.js
@@ -9,7 +9,7 @@ const {
   removeAgentFromAllProjects,
 } = require('./Project');
 const getLogStores = require('~/cache/getLogStores');
-const agentSchema = require('./schema/agent');
+const { agentSchema } = require('@librechat/data-schemas');
 
 const Agent = mongoose.model('agent', agentSchema);
 

diff --git a/api/models/Assistant.js b/api/models/Assistant.js
@@ -1,5 +1,5 @@
 const mongoose = require('mongoose');
-const assistantSchema = require('./schema/assistant');
+const { assistantSchema } = require('@librechat/data-schemas');
 
 const Assistant = mongoose.model('assistant', assistantSchema);
 

diff --git a/api/models/Balance.js b/api/models/Balance.js
@@ -1,5 +1,5 @@
 const mongoose = require('mongoose');
-const balanceSchema = require('./schema/balance');
+const { balanceSchema } = require('@librechat/data-schemas');
 const { getMultiplier } = require('./tx');
 const { logger } = require('~/config');
 

diff --git a/api/models/Banner.js b/api/models/Banner.js
@@ -1,5 +1,9 @@
-const Banner = require('./schema/banner');
+const mongoose = require('mongoose');
 const logger = require('~/config/winston');
+const { bannerSchema } = require('@librechat/data-schemas');
+
+const Banner = mongoose.model('Banner', bannerSchema);
+
 /**
  * Retrieves the current active banner.
  * @returns {Promise<Object|null>} The active banner object or null if no active banner is found.

diff --git a/api/models/Categories.js b/api/models/Categories.js
@@ -1,5 +1,4 @@
 const { logger } = require('~/config');
-// const { Categories } = require('./schema/categories');
 
 const options = [
   {

diff --git a/api/models/ConversationTag.js b/api/models/ConversationTag.js
@@ -1,7 +1,11 @@
-const ConversationTag = require('./schema/conversationTagSchema');
+const mongoose = require('mongoose');
 const Conversation = require('./schema/convoSchema');
 const logger = require('~/config/winston');
 
+const { conversationTagSchema } = require('@librechat/data-schemas');
+
+const ConversationTag = mongoose.model('ConversationTag', conversationTagSchema);
+
 /**
  * Retrieves all conversation tags for a user.
  * @param {string} user - The user ID.

diff --git a/api/models/File.js b/api/models/File.js
@@ -1,13 +1,13 @@
 const mongoose = require('mongoose');
-const fileSchema = require('./schema/fileSchema');
+const { fileSchema } = require('@librechat/data-schemas');
 
 const File = mongoose.model('File', fileSchema);
 
 /**
  * Finds a file by its file_id with additional query options.
  * @param {string} file_id - The unique identifier of the file.
  * @param {object} options - Query options for filtering, projection, etc.
- * @returns {Promise<MongoFile>} A promise that resolves to the file document or null.
+ * @returns {Promise<IMongoFile>} A promise that resolves to the file document or null.
  */
 const findFileById = async (file_id, options = {}) => {
   return await File.findOne({ file_id, ...options }).lean();
@@ -17,7 +17,7 @@ const findFileById = async (file_id, options = {}) => {
  * Retrieves files matching a given filter, sorted by the most recently updated.
  * @param {Object} filter - The filter criteria to apply.
  * @param {Object} [_sortOptions] - Optional sort parameters.
- * @returns {Promise<Array<MongoFile>>} A promise that resolves to an array of file documents.
+ * @returns {Promise<Array<IMongoFile>>} A promise that resolves to an array of file documents.
  */
 const getFiles = async (filter, _sortOptions) => {
   const sortOptions = { updatedAt: -1, ..._sortOptions };
@@ -26,9 +26,9 @@ const getFiles = async (filter, _sortOptions) => {
 
 /**
  * Creates a new file with a TTL of 1 hour.
- * @param {MongoFile} data - The file data to be created, must contain file_id.
+ * @param {IMongoFile} data - The file data to be created, must contain file_id.
  * @param {boolean} disableTTL - Whether to disable the TTL.
- * @returns {Promise<MongoFile>} A promise that resolves to the created file document.
+ * @returns {Promise<IMongoFile>} A promise that resolves to the created file document.
  */
 const createFile = async (data, disableTTL) => {
   const fileData = {
@@ -48,8 +48,8 @@ const createFile = async (data, disableTTL) => {
 
 /**
  * Updates a file identified by file_id with new data and removes the TTL.
- * @param {MongoFile} data - The data to update, must contain file_id.
- * @returns {Promise<MongoFile>} A promise that resolves to the updated file document.
+ * @param {IMongoFile} data - The data to update, must contain file_id.
+ * @returns {Promise<IMongoFile>} A promise that resolves to the updated file document.
  */
 const updateFile = async (data) => {
   const { file_id, ...update } = data;
@@ -62,8 +62,8 @@ const updateFile = async (data) => {
 
 /**
  * Increments the usage of a file identified by file_id.
- * @param {MongoFile} data - The data to update, must contain file_id and the increment value for usage.
- * @returns {Promise<MongoFile>} A promise that resolves to the updated file document.
+ * @param {IMongoFile} data - The data to update, must contain file_id and the increment value for usage.
+ * @returns {Promise<IMongoFile>} A promise that resolves to the updated file document.
  */
 const updateFileUsage = async (data) => {
   const { file_id, inc = 1 } = data;
@@ -77,7 +77,7 @@ const updateFileUsage = async (data) => {
 /**
  * Deletes a file identified by file_id.
  * @param {string} file_id - The unique identifier of the file to delete.
- * @returns {Promise<MongoFile>} A promise that resolves to the deleted file document or null.
+ * @returns {Promise<IMongoFile>} A promise that resolves to the deleted file document or null.
  */
 const deleteFile = async (file_id) => {
   return await File.findOneAndDelete({ file_id }).lean();
@@ -86,7 +86,7 @@ const deleteFile = async (file_id) => {
 /**
  * Deletes a file identified by a filter.
  * @param {object} filter - The filter criteria to apply.
- * @returns {Promise<MongoFile>} A promise that resolves to the deleted file document or null.
+ * @returns {Promise<IMongoFile>} A promise that resolves to the deleted file document or null.
  */
 const deleteFileByFilter = async (filter) => {
   return await File.findOneAndDelete(filter).lean();

diff --git a/api/models/Key.js b/api/models/Key.js
@@ -1,4 +1,4 @@
 const mongoose = require('mongoose');
-const keySchema = require('./schema/key');
+const { keySchema } = require('@librechat/data-schemas');
 
 module.exports = mongoose.model('Key', keySchema);
diff --git a/api/models/Project.js b/api/models/Project.js
@@ -1,6 +1,6 @@
 const { model } = require('mongoose');
 const { GLOBAL_PROJECT_NAME } = require('librechat-data-provider').Constants;
-const projectSchema = require('~/models/schema/projectSchema');
+const { projectSchema } = require('@librechat/data-schemas');
 
 const Project = model('Project', projectSchema);
 
@@ -9,7 +9,7 @@ const Project = model('Project', projectSchema);
  *
  * @param {string} projectId - The ID of the project to find and return as a plain object.
  * @param {string|string[]} [fieldsToSelect] - The fields to include or exclude in the returned document.
- * @returns {Promise<MongoProject>} A plain object representing the project document, or `null` if no project is found.
+ * @returns {Promise<IMongoProject>} A plain object representing the project document, or `null` if no project is found.
  */
 const getProjectById = async function (projectId, fieldsToSelect = null) {
   const query = Project.findById(projectId);
@@ -27,7 +27,7 @@ const getProjectById = async function (projectId, fieldsToSelect = null) {
  *
  * @param {string} projectName - The name of the project to find or create.
  * @param {string|string[]} [fieldsToSelect] - The fields to include or exclude in the returned document.
- * @returns {Promise<MongoProject>} A plain object representing the project document.
+ * @returns {Promise<IMongoProject>} A plain object representing the project document.
  */
 const getProjectByName = async function (projectName, fieldsToSelect = null) {
   const query = { name: projectName };
@@ -47,7 +47,7 @@ const getProjectByName = async function (projectName, fieldsToSelect = null) {
  *
  * @param {string} projectId - The ID of the project to update.
  * @param {string[]} promptGroupIds - The array of prompt group IDs to add to the project.
- * @returns {Promise<MongoProject>} The updated project document.
+ * @returns {Promise<IMongoProject>} The updated project document.
  */
 const addGroupIdsToProject = async function (projectId, promptGroupIds) {
   return await Project.findByIdAndUpdate(
@@ -62,7 +62,7 @@ const addGroupIdsToProject = async function (projectId, promptGroupIds) {
  *
  * @param {string} projectId - The ID of the project to update.
  * @param {string[]} promptGroupIds - The array of prompt group IDs to remove from the project.
- * @returns {Promise<MongoProject>} The updated project document.
+ * @returns {Promise<IMongoProject>} The updated project document.
  */
 const removeGroupIdsFromProject = async function (projectId, promptGroupIds) {
   return await Project.findByIdAndUpdate(
@@ -87,7 +87,7 @@ const removeGroupFromAllProjects = async (promptGroupId) => {
  *
  * @param {string} projectId - The ID of the project to update.
  * @param {string[]} agentIds - The array of agent IDs to add to the project.
- * @returns {Promise<MongoProject>} The updated project document.
+ * @returns {Promise<IMongoProject>} The updated project document.
  */
 const addAgentIdsToProject = async function (projectId, agentIds) {
   return await Project.findByIdAndUpdate(
@@ -102,7 +102,7 @@ const addAgentIdsToProject = async function (projectId, agentIds) {
  *
  * @param {string} projectId - The ID of the project to update.
  * @param {string[]} agentIds - The array of agent IDs to remove from the project.
- * @returns {Promise<MongoProject>} The updated project document.
+ * @returns {Promise<IMongoProject>} The updated project document.
  */
 const removeAgentIdsFromProject = async function (projectId, agentIds) {
   return await Project.findByIdAndUpdate(

diff --git a/api/models/Prompt.js b/api/models/Prompt.js
@@ -1,3 +1,4 @@
+const mongoose = require('mongoose');
 const { ObjectId } = require('mongodb');
 const { SystemRoles, SystemCategories, Constants } = require('librechat-data-provider');
 const {
@@ -6,10 +7,13 @@ const {
   removeGroupIdsFromProject,
   removeGroupFromAllProjects,
 } = require('./Project');
-const { Prompt, PromptGroup } = require('./schema/promptSchema');
+const { promptGroupSchema, promptSchema } = require('@librechat/data-schemas');
 const { escapeRegExp } = require('~/server/utils');
 const { logger } = require('~/config');
 
+const PromptGroup = mongoose.model('PromptGroup', promptGroupSchema);
+const Prompt = mongoose.model('Prompt', promptSchema);
+
 /**
  * Create a pipeline for the aggregation to get prompt groups
  * @param {Object} query

diff --git a/api/models/Role.js b/api/models/Role.js
@@ -1,3 +1,4 @@
+const mongoose = require('mongoose');
 const {
   CacheKeys,
   SystemRoles,
@@ -12,9 +13,11 @@ const {
   temporaryChatPermissionsSchema,
 } = require('librechat-data-provider');
 const getLogStores = require('~/cache/getLogStores');
-const Role = require('~/models/schema/roleSchema');
+const { roleSchema } = require('@librechat/data-schemas');
 const { logger } = require('~/config');
 
+const Role = mongoose.model('Role', roleSchema);
+
 /**
  * Retrieve a role by name and convert the found role document to a plain object.
  * If the role with the given name doesn't exist and the name is a system defined role, create it and return the lean version.
@@ -168,6 +171,7 @@ const initializeRoles = async function () {
   }
 };
 module.exports = {
+  Role,
   getRoleByName,
   initializeRoles,
   updateRoleByName,

diff --git a/api/models/Role.spec.js b/api/models/Role.spec.js
@@ -8,7 +8,7 @@ const {
 } = require('librechat-data-provider');
 const { updateAccessPermissions, initializeRoles } = require('~/models/Role');
 const getLogStores = require('~/cache/getLogStores');
-const Role = require('~/models/schema/roleSchema');
+const { Role } = require('~/models/Role');
 
 // Mock the cache
 jest.mock('~/cache/getLogStores', () => {

diff --git a/api/models/Session.js b/api/models/Session.js
@@ -1,7 +1,7 @@
 const mongoose = require('mongoose');
 const signPayload = require('~/server/services/signPayload');
 const { hashToken } = require('~/server/utils/crypto');
-const sessionSchema = require('./schema/session');
+const { sessionSchema } = require('@librechat/data-schemas');
 const { logger } = require('~/config');
 
 const Session = mongoose.model('Session', sessionSchema);

diff --git a/api/models/Share.js b/api/models/Share.js
@@ -1,7 +1,9 @@
+const mongoose = require('mongoose');
 const { nanoid } = require('nanoid');
 const { Constants } = require('librechat-data-provider');
 const { Conversation } = require('~/models/Conversation');
-const SharedLink = require('./schema/shareSchema');
+const { shareSchema } = require('@librechat/data-schemas');
+const SharedLink = mongoose.model('SharedLink', shareSchema);
 const { getMessages } = require('./Message');
 const logger = require('~/config/winston');
 

diff --git a/api/models/Token.js b/api/models/Token.js
@@ -1,6 +1,6 @@
 const mongoose = require('mongoose');
 const { encryptV2 } = require('~/server/utils/crypto');
-const tokenSchema = require('./schema/tokenSchema');
+const { tokenSchema } = require('@librechat/data-schemas');
 const { logger } = require('~/config');
 
 /**