feat: 文件列表支持增量更新

src/bootstrap.ts

@@ -1,13 +1,15 @@
 import nodeCluster from 'cluster'
 import colors from 'colors/safe.js'
 import {HTTPError} from 'got'
+import {max} from 'lodash-es'
 import ms from 'ms'
 import {join} from 'path'
 import {fileURLToPath} from 'url'
 import {Cluster} from './cluster.js'
 import {config} from './config.js'
 import {logger} from './logger.js'
 import {TokenManager} from './token.js'
+import {IFileList} from './types.js'
 
 // eslint-disable-next-line @typescript-eslint/naming-convention
 const __dirname = fileURLToPath(new URL('.', import.meta.url))
@@ -56,7 +58,9 @@ export async function bootstrap(version: string): Promise<void> {
       process.send('ready')
     }
 
-    checkFileInterval = setTimeout(() => checkFile, ms('10m'))
+    checkFileInterval = setTimeout(() => {
+      void checkFile(files)
+    }, ms('10s'))
   } catch (e) {
     logger.fatal(e)
     if (process.env.NODE_ENV === 'development') {
@@ -65,19 +69,25 @@ export async function bootstrap(version: string): Promise<void> {
       cluster.exit(1)
     }
   }
-  async function checkFile(): Promise<void> {
+  async function checkFile(lastFileList: IFileList): Promise<void> {
     logger.debug('refresh files')
     try {
-      const files = await cluster.getFileList()
+      const lastModified = max(lastFileList.files.map((file) => file.mtime))
+      const fileList = await cluster.getFileList(lastModified)
+      if (fileList.files.length === 0) {
+        logger.debug('没有新文件')
+        return
+      }
+      lastFileList = fileList
       const configuration = await cluster.getConfiguration()
       await cluster.syncFiles(files, configuration.sync)
     } finally {
       checkFileInterval = setTimeout(() => {
-        checkFile().catch((e) => {
+        checkFile(lastFileList).catch((e) => {
           console.error('check file error')
           console.error(e)
         })
-      }, ms('10m'))
+      }, ms('10s'))
     }
   }
 

src/cluster.ts

@@ -1,5 +1,4 @@
 import {decompress} from '@mongodb-js/zstd'
-import avsc, {type schema} from 'avsc'
 import Bluebird from 'bluebird'
 import {ChildProcess, spawn} from 'child_process'
 import {MultiBar} from 'cli-progress'
@@ -26,6 +25,7 @@ import {connect, Socket} from 'socket.io-client'
 import {Tail} from 'tail'
 import {fileURLToPath} from 'url'
 import {config, type OpenbmclapiAgentConfiguration, OpenbmclapiAgentConfigurationSchema} from './config.js'
+import {FileListSchema} from './constants.js'
 import {validateFile} from './file.js'
 import {logger} from './logger.js'
 import MeasureRouteFactory from './measure.route.js'
@@ -123,25 +123,22 @@ export class Cluster {
     }
   }
 
-  public async getFileList(): Promise<IFileList> {
-    const fileListSchema = avsc.Type.forSchema({
-      type: 'array',
-      items: {
-        type: 'record',
-        fields: [
-          {name: 'path', type: 'string'},
-          {name: 'hash', type: 'string'},
-          {name: 'size', type: 'long'},
-        ],
-      } as schema.RecordType,
-    })
+  public async getFileList(lastModified?: number): Promise<IFileList> {
     const res = await this.got.get('openbmclapi/files', {
       responseType: 'buffer',
       cache: this.requestCache,
+      searchParams: {
+        lastModified,
+      },
     })
+    if (res.statusCode === 304) {
+      return {
+        files: [],
+      }
+    }
     const decompressed = await decompress(res.body)
     return {
-      files: fileListSchema.fromBuffer(Buffer.from(decompressed)) as IFileList['files'],
+      files: FileListSchema.fromBuffer(Buffer.from(decompressed)) as IFileList['files'],
     }
   }
 
@@ -433,6 +430,7 @@ export class Cluster {
       path: `/download/${hash}`,
       hash,
       size: res.body.length,
+      mtime: Date.now(),
     })
   }
 

src/constants.ts

@@ -0,0 +1,15 @@
+import avsc from 'avsc'
+
+export const FileListSchema = avsc.Type.forSchema({
+  type: 'array',
+  items: {
+    name: 'FileListEntry',
+    type: 'record',
+    fields: [
+      {name: 'path', type: 'string'},
+      {name: 'hash', type: 'string'},
+      {name: 'size', type: 'long'},
+      {name: 'mtime', type: 'long'},
+    ],
+  },
+})

src/types.ts

@@ -6,4 +6,5 @@ export interface IFileInfo {
   path: string
   hash: string
   size: number
+  mtime: number
 }