feat(k8s-client): chant to dynamic module

packages/k8s-client/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "description": "该模块是对 [@kubernetes/client-node](https://github.com/kubernetes-client/javascript) 的二次封装，从资源的维度重新组织了 api。",
   "keywords": [],
-  "author": "zhangpc",
+  "author": "Carrotzpc <[email protected]>",
   "main": "dist/index.js",
   "module": "dist/index.js",
   "types": "dist/index.d.ts",

packages/k8s-client/src/index.ts

@@ -1,2 +1,3 @@
 export * from './kubernetes.module';
 export * from './kubernetes.service';
+export * from './lib';

packages/k8s-client/src/kubernetes.module.ts

@@ -1,10 +1,22 @@
-import { Global, Module } from '@nestjs/common';
+import { DynamicModule, Global, Module } from '@nestjs/common';
 
-import { KubernetesService } from './kubernetes.service';
+import { K8S_CLIENT_CONFIG } from './kubernetes.constants';
+import { K8sConfig, KubernetesService } from './kubernetes.service';
 
 @Global()
-@Module({
-  providers: [KubernetesService],
-  exports: [KubernetesService],
-})
-export class KubernetesModule {}
+@Module({})
+export class KubernetesModule {
+  static forRoot(config: K8sConfig): DynamicModule {
+    return {
+      module: KubernetesModule,
+      providers: [
+        {
+          provide: K8S_CLIENT_CONFIG,
+          useValue: config,
+        },
+        KubernetesService,
+      ],
+      exports: [KubernetesService],
+    };
+  }
+}

packages/k8s-client/src/kubernetes.service.ts

@@ -2,14 +2,13 @@
 import * as K8s from '@kubernetes/client-node';
 import { dumpYaml, loadAllYaml } from '@kubernetes/client-node';
 import { WebSocketHandler } from '@kubernetes/client-node/dist/web-socket-handler';
-import { Injectable, Logger } from '@nestjs/common';
+import { Inject, Injectable, Logger } from '@nestjs/common';
 import { WebSocket } from 'ws';
 
+import { K8S_CLIENT_CONFIG } from './kubernetes.constants';
 import * as lib from './lib';
 import { ClientOptions } from './lib/interfaces';
-import { IS_PROD } from './utils';
 
-const { env } = process;
 const K8S_WS_PROTOCOLS = [
   'base64.channel.k8s.io', // 增加了 base64 channel
   'v4.channel.k8s.io',
@@ -22,37 +21,36 @@ interface K8sUser extends K8s.User {
   ip?: string;
 }
 
-const k8sApiTimeout = process.env.K8S_API_TIMEOUT || '12';
-const defaultInterceptor: K8s.Interceptor = requestOptions => {
-  requestOptions.timeout = Number.parseFloat(k8sApiTimeout) * 1000;
-};
-
 type KubernetesClientBase = Awaited<ReturnType<KubernetesService['getClientBase']>>;
 export interface KubernetesClient extends KubernetesClientBase {
   /** _sa 下的资源会使用 server 自己的 service account 来作为调用 k8s api 的凭证，且只能操作管理集群的资源*/
   _sa?: Awaited<KubernetesClientBase>;
 }
 
-// @Todo: 需要改为注入 module 时传入
 export interface K8sConfig {
-  cluster: K8s.Cluster;
-  saToken: string;
+  cluster?: K8s.Cluster;
+  /** ServiceAccount 类型的 token */
+  saToken?: string;
+  options?: {
+    /** 超时时间，单位 s，默认为 12 */
+    timeout?: number;
+  };
 }
 
 @Injectable()
 export class KubernetesService {
-  k8sConfig: K8sConfig = {
-    cluster: {
-      name: env.K8S_SERVER_NAME || 'k8s-cluster',
-      server: env.K8S_OIDC_PROXY_URL || 'https://172.22.96.133:6443',
-      skipTLSVerify: true,
-    },
-    saToken:
-      'eyJhbGciOiJSUzI1NiIsImtpZCI6InhUWE1zQUJBd3lvUG1nYUlqc19DaHUtTUM0alVBOERVdWthX19rR2Z4NjQifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJ5dW50aS1zeXN0ZW0iLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlY3JldC5uYW1lIjoieXVudGktc2VydmVyIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6Inl1bnRpLXNlcnZlciIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjRiYTU1OGJhLTJlNDYtNDJjMi05MDRkLTdjY2EwZWRmMzdiYyIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDp5dW50aS1zeXN0ZW06eXVudGktc2VydmVyIn0.E6kwkN-g7vbRe2GKU3NsjkLMXjA8Ff7LMzNfcIP1Evb4wJBMWfqPgtFJpJCKTjlroZiRED-Xkjmw9rGoRTkCMjpgLNTnoWda6X5ECK_Od781mUMLnGVCJtzINTlVumxa9GekK7c9-JKdUD0HfFwm_pfyaKMVpBqT8rYSbSc4TX1QtoQ935FVcOts7o3U2vMXIEDDf1joeLHLHF-9Kszo04O6x1OlULOyUpcXYR7fGrFvmkbBZzhOdT6m7Y2jbEpMukwaRGpGhtQSmZuriFndtpJW-8HLGnkeNimaR8OfbW3Gifkr0J3xZspooRNjmBnwMN9KLvXDL9TUPfXux46YCQ',
-  };
+  private readonly k8sConfig: K8sConfig;
+  private readonly defaultInterceptor: K8s.Interceptor;
 
   private logger = new Logger('KubernetesService');
 
+  constructor(@Inject(K8S_CLIENT_CONFIG) private config: K8sConfig) {
+    this.k8sConfig = config;
+    this.defaultInterceptor = requestOptions => {
+      requestOptions.timeout = (this.k8sConfig.options?.timeout || 12) * 1000;
+    };
+  }
+
   createKubeConfig(cluster: K8s.Cluster, user: K8s.User) {
     const kubeConfig = new K8s.KubeConfig();
     kubeConfig.loadFromClusterAndUser(cluster, user);
@@ -85,14 +83,6 @@ export class KubernetesService {
     if (!options._sa) {
       return client;
     }
-    if (IS_PROD) {
-      try {
-        // @Todo: 暂时去掉
-        // this.k8sConfig.saToken = readFileSync(K8S_SA_TOKEN_PATH).toString();
-      } catch (error) {
-        this.logger.error('read service account token failed', error);
-      }
-    }
     const saClient = await this.getClientBase(
       { name: 'sa-client', token: this.k8sConfig.saToken, ip: user.ip },
       // 只支持管理集群，不支持指定其他集群
@@ -109,6 +99,10 @@ export class KubernetesService {
    *
    */
   async getSaClient() {
+    if (!this.k8sConfig.saToken) {
+      this.logger.warn('getSaClient', 'saToken is required');
+      return;
+    }
     const saClient = await this.getClientBase({
       name: 'sa-client',
       token: this.k8sConfig.saToken,
@@ -167,7 +161,7 @@ export class KubernetesService {
   }
 
   private async getClientBase(user: K8sUser, options: ClientOptions = {}) {
-    const { cluster: specifiedCluster, interceptor = defaultInterceptor } = options;
+    const { cluster: specifiedCluster, interceptor = this.defaultInterceptor } = options;
     const defaultHeadersInterceptor: K8s.Interceptor = requestOptions => {
       this.logger.debug(
         `[${requestOptions.method}] ${(requestOptions as any).uri}`,

packages/server/.gitignore

@@ -39,3 +39,4 @@ lerna-debug.log*
 # dev
 /dolt/databases
 /deploy/k8s
+dolt/db/query.sql

packages/server/configs/config.default.yaml

@@ -24,7 +24,7 @@ db:
   host: 0.0.0.0
   port: 13306
   username: root
-  password: tenxcloud
+  password: yunti
   database: yunti
   logging: false
   timezone: Z
@@ -98,9 +98,16 @@ npm:
     mirror: https://jsd.onmicrosoft.cn
   unpkg: http://dev-unpkg.tenxcloud.net
 kubernetes:
-  name: kube-oidc-proxy
-  server: https://172.22.96.133:6443
-  skipTLSVerify: true
+  # k8s 集群配置
+  cluster:
+    name: kube-oidc-proxy
+    server: https://172.22.96.133:6443
+    skipTLSVerify: true
+  # k8s 集群 ServiceAccount token
+  saToken: ''
+  options:
+    # 超时时间，单位秒
+    timeout: 10
 minio:
   client:
     endPoint: ''

packages/server/dolt/servercfg.d/config.yaml

@@ -6,7 +6,7 @@ behavior:
 
 user:
   name: root
-  password: 'tenxcloud'
+  password: 'yunti'
 
 listener:
   host: 0.0.0.0

packages/server/src/app.module.ts

@@ -6,6 +6,7 @@ import { GraphQLModule } from '@nestjs/graphql';
 import { ScheduleModule } from '@nestjs/schedule';
 import { ServeStaticModule } from '@nestjs/serve-static';
 import { TypeOrmModule } from '@nestjs/typeorm';
+import { KubernetesModule } from '@yuntijs/k8s-client';
 import { Response } from 'express';
 import { DirectiveLocation, GraphQLDirective } from 'graphql';
 import { join } from 'node:path';
@@ -32,7 +33,7 @@ import { GRAPHQL_PATH } from './common/utils';
 import { ComponentsMembersModule } from './components-members/components-members.module';
 import { ComponentsVersionsModule } from './components-versions/components-versions.module';
 import { ComponentsModule } from './components/components.module';
-import serverConfig from './config/server.config';
+import serverConfig, { SERVER_CONFIG } from './config/server.config';
 import { GitModule } from './git/git.module';
 import { MergeRequestModule } from './merge-requests/merge-requests.module';
 import { MinioModule } from './minio/minio.module';
@@ -117,6 +118,7 @@ import { UsersModule } from './users/users.module';
         },
       },
     }),
+    KubernetesModule.forRoot(SERVER_CONFIG.kubernetes),
     UsersModule,
     PagesModule,
     AppsModule,

packages/server/src/common/entities/index.ts

@@ -1,7 +1,7 @@
 import { registerAs } from '@nestjs/config';
 import { TypeOrmModuleOptions } from '@nestjs/typeorm';
 
-import { serverConfig } from '../../config/server.config';
+import { SERVER_CONFIG } from '../../config/server.config';
 import { AppMember } from './apps-members.entity';
 import { App } from './apps.entity';
 import { Block } from './blocks.entity';
@@ -22,7 +22,7 @@ import { PublishChannel } from './publish-channels.entity';
 import { PublishRecord } from './publish-records.entity';
 import { User } from './users.entity';
 
-const { maxDataSources: maxDs, ...config } = serverConfig.db;
+const { maxDataSources: maxDs, ...config } = SERVER_CONFIG.db;
 
 /** yunti-server 与数据库建立的最大连接数 */
 export const maxDataSources = maxDs;

packages/server/src/common/utils/tools.ts

@@ -1,4 +1,4 @@
-import { K8s } from '@yuntijs/k8s-client/lib';
+import { K8s } from '@yuntijs/k8s-client';
 import { isObject } from 'lodash';
 import { customAlphabet } from 'nanoid';
 import { lowercase, numbers } from 'nanoid-dictionary';

packages/server/src/config/server.config.ts

@@ -12,6 +12,6 @@ function mergeCustomizer(objValue: any, srcValue: any) {
 const serverDefaultConfig = getConfigByPath(SERVER_DEFAULT_CONFIG_PATH);
 const serverRuntimeConfig = getConfigByPath(SERVER_CONFIG_PATH);
 
-export const serverConfig = mergeWith(serverDefaultConfig, serverRuntimeConfig, mergeCustomizer);
+export const SERVER_CONFIG = mergeWith(serverDefaultConfig, serverRuntimeConfig, mergeCustomizer);
 
-export default registerAs('server', () => serverConfig);
+export default registerAs('server', () => SERVER_CONFIG);

packages/server/src/main.ts

@@ -12,13 +12,13 @@ import { join } from 'node:path';
 import { AppModule } from './app.module';
 import { LoggerMiddleware } from './common/middleware/logger.middleware';
 import { IS_PROD } from './common/utils';
-import { serverConfig } from './config/server.config';
+import { SERVER_CONFIG } from './config/server.config';
 
 import ejs = require('ejs');
 
 async function bootstrap() {
   const app = await NestFactory.create<NestExpressApplication>(AppModule, {
-    logger: IS_PROD ? serverConfig.log.levels.split(',') : undefined,
+    logger: IS_PROD ? SERVER_CONFIG.log.levels.split(',') : undefined,
   });
 
   app.enableCors({
@@ -32,8 +32,8 @@ async function bootstrap() {
   app.set('trust proxy', true);
 
   // 启用 gzip 压缩
-  if (serverConfig.compression?.enabled) {
-    app.use(compression(serverConfig.compression));
+  if (SERVER_CONFIG.compression?.enabled) {
+    app.use(compression(SERVER_CONFIG.compression));
   }
 
   // ~ set ejs
@@ -44,7 +44,7 @@ async function bootstrap() {
   app.setBaseViewsDir(join(__dirname, '../../../', 'public'));
   app.setViewEngine('html');
 
-  app.use(json(serverConfig.bodyParser.json));
+  app.use(json(SERVER_CONFIG.bodyParser.json));
 
   app.use(
     helmet({
@@ -57,15 +57,15 @@ async function bootstrap() {
   app.use(
     session({
       store: new RedisStore({
-        client: new Redis(serverConfig.redis),
+        client: new Redis(SERVER_CONFIG.redis),
       }),
-      ...serverConfig.session,
+      ...SERVER_CONFIG.session,
     })
   );
 
   app.use(LoggerMiddleware);
 
-  await app.listen(serverConfig.web.port);
+  await app.listen(SERVER_CONFIG.web.port);
 
   // eslint-disable-next-line no-console
   console.log(`bff-server is running on: ${await app.getUrl()}`);

packages/server/src/pipelines/pipelines.module.ts

@@ -1,11 +1,9 @@
 import { Module } from '@nestjs/common';
-import { KubernetesModule } from '@yuntijs/k8s-client/kubernetes.module';
 
 import { PipelinesService } from './pipelines.service';
 
 @Module({
   providers: [PipelinesService],
   exports: [PipelinesService],
-  imports: [KubernetesModule],
 })
 export class PipelinesModule {}

packages/server/src/pipelines/pipelines.service.ts

@@ -1,7 +1,6 @@
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import { ConfigType } from '@nestjs/config';
-import { KubernetesService } from '@yuntijs/k8s-client/kubernetes.service';
-import { CRD, K8s } from '@yuntijs/k8s-client/lib';
+import { type CRD, K8s, KubernetesService } from '@yuntijs/k8s-client';
 
 import { extractPipelineRunStatus } from '@/common/utils';
 import serverConfig from '@/config/server.config';
@@ -150,6 +149,9 @@ export class PipelinesService {
   async makePipeLineRunInformer(namespace: string, pipeline: string) {
     const method = 'makePipeLineRunInformer';
     const k8s = await this.k8sService.getSaClient();
+    if (!k8s) {
+      return;
+    }
     const labelSelector = `tekton.dev/pipeline=${pipeline}`;
     const listFn = () => k8s.pipelineRun.list(namespace, { labelSelector });
     const { group, version, name } = k8s.pipelineRun;

packages/server/src/publish-records/publish-records.service.ts

@@ -1,7 +1,7 @@
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import { ConfigType } from '@nestjs/config';
 import { Timeout } from '@nestjs/schedule';
-import { CRD } from '@yuntijs/k8s-client/lib';
+import { CRD } from '@yuntijs/k8s-client';
 import { Like } from 'typeorm';
 
 import { AppsMembersService } from '@/apps-members/apps-members.service';
@@ -368,8 +368,12 @@ export class PublishRecordsService {
       pipeline?.namespace,
       pipeline?.publish?.name
     );
+    if (!informer) {
+      this.logger.warn(`[${method}] ⚠ informer start failed ⚠`);
+      return;
+    }
     const startInformer = () =>
-      informer.start().then(() => this.logger.log(`[${method}] informer stated ...`));
+      informer.start().then(() => this.logger.log(`[${method}] informer started ...`));
 
     // informer.on('add', (pr: CRD.PipelineRun) => {
     //   console.log(`Added: ${pr.metadata!.name}`);

packages/server/tsconfig.json

@@ -21,7 +21,7 @@
     "resolveJsonModule": true,
     "paths": {
       "@/*": ["src/*"],
-      "@yuntijs/k8s-client/*": ["../k8s-client/src/*"]
+      "@yuntijs/k8s-client": ["../k8s-client/src"]
     }
   },
   "exclude": ["dist"]