From b2cb9f5dfcd30a99e06dca6b460efc1c25a6ad50 Mon Sep 17 00:00:00 2001 From: Carrotzpc Date: Tue, 26 Mar 2024 14:21:36 +0800 Subject: [PATCH] feat(k8s-client): chant to dynamic module --- packages/k8s-client/package.json | 2 +- packages/k8s-client/src/index.ts | 1 + packages/k8s-client/src/kubernetes.module.ts | 26 +++++++--- packages/k8s-client/src/kubernetes.service.ts | 52 ++++++++----------- packages/server/.gitignore | 1 + packages/server/configs/config.default.yaml | 15 ++++-- packages/server/dolt/servercfg.d/config.yaml | 2 +- packages/server/src/app.module.ts | 4 +- packages/server/src/common/entities/index.ts | 4 +- packages/server/src/common/utils/tools.ts | 2 +- packages/server/src/config/server.config.ts | 4 +- packages/server/src/main.ts | 16 +++--- .../server/src/pipelines/pipelines.module.ts | 2 - .../server/src/pipelines/pipelines.service.ts | 6 ++- .../publish-records.service.ts | 8 ++- packages/server/tsconfig.json | 2 +- 16 files changed, 84 insertions(+), 63 deletions(-) diff --git a/packages/k8s-client/package.json b/packages/k8s-client/package.json index 1a56205..8701b60 100644 --- a/packages/k8s-client/package.json +++ b/packages/k8s-client/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "description": "该模块是对 [@kubernetes/client-node](https://github.com/kubernetes-client/javascript) 的二次封装，从资源的维度重新组织了 api。", "keywords": [], - "author": "zhangpc", + "author": "Carrotzpc ", "main": "dist/index.js", "module": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/k8s-client/src/index.ts b/packages/k8s-client/src/index.ts index 426c52d..d7bd601 100644 --- a/packages/k8s-client/src/index.ts +++ b/packages/k8s-client/src/index.ts @@ -1,2 +1,3 @@ export * from './kubernetes.module'; export * from './kubernetes.service'; +export * from './lib'; diff --git a/packages/k8s-client/src/kubernetes.module.ts b/packages/k8s-client/src/kubernetes.module.ts index ace2ccd..4e4a3c4 100644 --- a/packages/k8s-client/src/kubernetes.module.ts +++ b/packages/k8s-client/src/kubernetes.module.ts @@ -1,10 +1,22 @@ -import { Global, Module } from '@nestjs/common'; +import { DynamicModule, Global, Module } from '@nestjs/common'; -import { KubernetesService } from './kubernetes.service'; +import { K8S_CLIENT_CONFIG } from './kubernetes.constants'; +import { K8sConfig, KubernetesService } from './kubernetes.service'; @Global() -@Module({ - providers: [KubernetesService], - exports: [KubernetesService], -}) -export class KubernetesModule {} +@Module({}) +export class KubernetesModule { + static forRoot(config: K8sConfig): DynamicModule { + return { + module: KubernetesModule, + providers: [ + { + provide: K8S_CLIENT_CONFIG, + useValue: config, + }, + KubernetesService, + ], + exports: [KubernetesService], + }; + } +} diff --git a/packages/k8s-client/src/kubernetes.service.ts b/packages/k8s-client/src/kubernetes.service.ts index 36980a4..1d81cab 100644 --- a/packages/k8s-client/src/kubernetes.service.ts +++ b/packages/k8s-client/src/kubernetes.service.ts @@ -2,14 +2,13 @@ import * as K8s from '@kubernetes/client-node'; import { dumpYaml, loadAllYaml } from '@kubernetes/client-node'; import { WebSocketHandler } from '@kubernetes/client-node/dist/web-socket-handler'; -import { Injectable, Logger } from '@nestjs/common'; +import { Inject, Injectable, Logger } from '@nestjs/common'; import { WebSocket } from 'ws'; +import { K8S_CLIENT_CONFIG } from './kubernetes.constants'; import * as lib from './lib'; import { ClientOptions } from './lib/interfaces'; -import { IS_PROD } from './utils'; -const { env } = process; const K8S_WS_PROTOCOLS = [ 'base64.channel.k8s.io', // 增加了 base64 channel 'v4.channel.k8s.io', @@ -22,37 +21,36 @@ interface K8sUser extends K8s.User { ip?: string; } -const k8sApiTimeout = process.env.K8S_API_TIMEOUT || '12'; -const defaultInterceptor: K8s.Interceptor = requestOptions => { - requestOptions.timeout = Number.parseFloat(k8sApiTimeout) * 1000; -}; - type KubernetesClientBase = Awaited>; export interface KubernetesClient extends KubernetesClientBase { /** _sa 下的资源会使用 server 自己的 service account 来作为调用 k8s api 的凭证，且只能操作管理集群的资源*/ _sa?: Awaited; } -// @Todo: 需要改为注入 module 时传入 export interface K8sConfig { - cluster: K8s.Cluster; - saToken: string; + cluster?: K8s.Cluster; + /** ServiceAccount 类型的 token */ + saToken?: string; + options?: { + /** 超时时间，单位 s，默认为 12 */ + timeout?: number; + }; } @Injectable() export class KubernetesService { - k8sConfig: K8sConfig = { - cluster: { - name: env.K8S_SERVER_NAME || 'k8s-cluster', - server: env.K8S_OIDC_PROXY_URL || 'https://172.22.96.133:6443', - skipTLSVerify: true, - }, - saToken: - 'eyJhbGciOiJSUzI1NiIsImtpZCI6InhUWE1zQUJBd3lvUG1nYUlqc19DaHUtTUM0alVBOERVdWthX19rR2Z4NjQifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJ5dW50aS1zeXN0ZW0iLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlY3JldC5uYW1lIjoieXVudGktc2VydmVyIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6Inl1bnRpLXNlcnZlciIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjRiYTU1OGJhLTJlNDYtNDJjMi05MDRkLTdjY2EwZWRmMzdiYyIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDp5dW50aS1zeXN0ZW06eXVudGktc2VydmVyIn0.E6kwkN-g7vbRe2GKU3NsjkLMXjA8Ff7LMzNfcIP1Evb4wJBMWfqPgtFJpJCKTjlroZiRED-Xkjmw9rGoRTkCMjpgLNTnoWda6X5ECK_Od781mUMLnGVCJtzINTlVumxa9GekK7c9-JKdUD0HfFwm_pfyaKMVpBqT8rYSbSc4TX1QtoQ935FVcOts7o3U2vMXIEDDf1joeLHLHF-9Kszo04O6x1OlULOyUpcXYR7fGrFvmkbBZzhOdT6m7Y2jbEpMukwaRGpGhtQSmZuriFndtpJW-8HLGnkeNimaR8OfbW3Gifkr0J3xZspooRNjmBnwMN9KLvXDL9TUPfXux46YCQ', - }; + private readonly k8sConfig: K8sConfig; + private readonly defaultInterceptor: K8s.Interceptor; private logger = new Logger('KubernetesService'); + constructor(@Inject(K8S_CLIENT_CONFIG) private config: K8sConfig) { + this.k8sConfig = config; + this.defaultInterceptor = requestOptions => { + requestOptions.timeout = (this.k8sConfig.options?.timeout || 12) * 1000; + }; + } + createKubeConfig(cluster: K8s.Cluster, user: K8s.User) { const kubeConfig = new K8s.KubeConfig(); kubeConfig.loadFromClusterAndUser(cluster, user); @@ -85,14 +83,6 @@ export class KubernetesService { if (!options._sa) { return client; } - if (IS_PROD) { - try { - // @Todo: 暂时去掉 - // this.k8sConfig.saToken = readFileSync(K8S_SA_TOKEN_PATH).toString(); - } catch (error) { - this.logger.error('read service account token failed', error); - } - } const saClient = await this.getClientBase( { name: 'sa-client', token: this.k8sConfig.saToken, ip: user.ip }, // 只支持管理集群，不支持指定其他集群 @@ -109,6 +99,10 @@ export class KubernetesService { * */ async getSaClient() { + if (!this.k8sConfig.saToken) { + this.logger.warn('getSaClient', 'saToken is required'); + return; + } const saClient = await this.getClientBase({ name: 'sa-client', token: this.k8sConfig.saToken, @@ -167,7 +161,7 @@ export class KubernetesService { } private async getClientBase(user: K8sUser, options: ClientOptions = {}) { - const { cluster: specifiedCluster, interceptor = defaultInterceptor } = options; + const { cluster: specifiedCluster, interceptor = this.defaultInterceptor } = options; const defaultHeadersInterceptor: K8s.Interceptor = requestOptions => { this.logger.debug( `[${requestOptions.method}] ${(requestOptions as any).uri}`, diff --git a/packages/server/.gitignore b/packages/server/.gitignore index c250e83..41bf367 100644 --- a/packages/server/.gitignore +++ b/packages/server/.gitignore @@ -39,3 +39,4 @@ lerna-debug.log* # dev /dolt/databases /deploy/k8s +dolt/db/query.sql diff --git a/packages/server/configs/config.default.yaml b/packages/server/configs/config.default.yaml index 4dcb2df..0602cc8 100644 --- a/packages/server/configs/config.default.yaml +++ b/packages/server/configs/config.default.yaml @@ -24,7 +24,7 @@ db: host: 0.0.0.0 port: 13306 username: root - password: tenxcloud + password: yunti database: yunti logging: false timezone: Z @@ -98,9 +98,16 @@ npm: mirror: https://jsd.onmicrosoft.cn unpkg: http://dev-unpkg.tenxcloud.net kubernetes: - name: kube-oidc-proxy - server: https://172.22.96.133:6443 - skipTLSVerify: true + # k8s 集群配置 + cluster: + name: kube-oidc-proxy + server: https://172.22.96.133:6443 + skipTLSVerify: true + # k8s 集群 ServiceAccount token + saToken: '' + options: + # 超时时间，单位秒 + timeout: 10 minio: client: endPoint: '' diff --git a/packages/server/dolt/servercfg.d/config.yaml b/packages/server/dolt/servercfg.d/config.yaml index 230b679..3b1c69e 100644 --- a/packages/server/dolt/servercfg.d/config.yaml +++ b/packages/server/dolt/servercfg.d/config.yaml @@ -6,7 +6,7 @@ behavior: user: name: root - password: 'tenxcloud' + password: 'yunti' listener: host: 0.0.0.0 diff --git a/packages/server/src/app.module.ts b/packages/server/src/app.module.ts index a587674..83a8a2d 100644 --- a/packages/server/src/app.module.ts +++ b/packages/server/src/app.module.ts @@ -6,6 +6,7 @@ import { GraphQLModule } from '@nestjs/graphql'; import { ScheduleModule } from '@nestjs/schedule'; import { ServeStaticModule } from '@nestjs/serve-static'; import { TypeOrmModule } from '@nestjs/typeorm'; +import { KubernetesModule } from '@yuntijs/k8s-client'; import { Response } from 'express'; import { DirectiveLocation, GraphQLDirective } from 'graphql'; import { join } from 'node:path'; @@ -32,7 +33,7 @@ import { GRAPHQL_PATH } from './common/utils'; import { ComponentsMembersModule } from './components-members/components-members.module'; import { ComponentsVersionsModule } from './components-versions/components-versions.module'; import { ComponentsModule } from './components/components.module'; -import serverConfig from './config/server.config'; +import serverConfig, { SERVER_CONFIG } from './config/server.config'; import { GitModule } from './git/git.module'; import { MergeRequestModule } from './merge-requests/merge-requests.module'; import { MinioModule } from './minio/minio.module'; @@ -117,6 +118,7 @@ import { UsersModule } from './users/users.module'; }, }, }), + KubernetesModule.forRoot(SERVER_CONFIG.kubernetes), UsersModule, PagesModule, AppsModule, diff --git a/packages/server/src/common/entities/index.ts b/packages/server/src/common/entities/index.ts index 7a485f7..2efd1dc 100644 --- a/packages/server/src/common/entities/index.ts +++ b/packages/server/src/common/entities/index.ts @@ -1,7 +1,7 @@ import { registerAs } from '@nestjs/config'; import { TypeOrmModuleOptions } from '@nestjs/typeorm'; -import { serverConfig } from '../../config/server.config'; +import { SERVER_CONFIG } from '../../config/server.config'; import { AppMember } from './apps-members.entity'; import { App } from './apps.entity'; import { Block } from './blocks.entity'; @@ -22,7 +22,7 @@ import { PublishChannel } from './publish-channels.entity'; import { PublishRecord } from './publish-records.entity'; import { User } from './users.entity'; -const { maxDataSources: maxDs, ...config } = serverConfig.db; +const { maxDataSources: maxDs, ...config } = SERVER_CONFIG.db; /** yunti-server 与数据库建立的最大连接数 */ export const maxDataSources = maxDs; diff --git a/packages/server/src/common/utils/tools.ts b/packages/server/src/common/utils/tools.ts index 2f655c5..edb6244 100644 --- a/packages/server/src/common/utils/tools.ts +++ b/packages/server/src/common/utils/tools.ts @@ -1,4 +1,4 @@ -import { K8s } from '@yuntijs/k8s-client/lib'; +import { K8s } from '@yuntijs/k8s-client'; import { isObject } from 'lodash'; import { customAlphabet } from 'nanoid'; import { lowercase, numbers } from 'nanoid-dictionary'; diff --git a/packages/server/src/config/server.config.ts b/packages/server/src/config/server.config.ts index 9642e64..ea34cb0 100644 --- a/packages/server/src/config/server.config.ts +++ b/packages/server/src/config/server.config.ts @@ -12,6 +12,6 @@ function mergeCustomizer(objValue: any, srcValue: any) { const serverDefaultConfig = getConfigByPath(SERVER_DEFAULT_CONFIG_PATH); const serverRuntimeConfig = getConfigByPath(SERVER_CONFIG_PATH); -export const serverConfig = mergeWith(serverDefaultConfig, serverRuntimeConfig, mergeCustomizer); +export const SERVER_CONFIG = mergeWith(serverDefaultConfig, serverRuntimeConfig, mergeCustomizer); -export default registerAs('server', () => serverConfig); +export default registerAs('server', () => SERVER_CONFIG); diff --git a/packages/server/src/main.ts b/packages/server/src/main.ts index 3ab1444..ef5ffb5 100644 --- a/packages/server/src/main.ts +++ b/packages/server/src/main.ts @@ -12,13 +12,13 @@ import { join } from 'node:path'; import { AppModule } from './app.module'; import { LoggerMiddleware } from './common/middleware/logger.middleware'; import { IS_PROD } from './common/utils'; -import { serverConfig } from './config/server.config'; +import { SERVER_CONFIG } from './config/server.config'; import ejs = require('ejs'); async function bootstrap() { const app = await NestFactory.create(AppModule, { - logger: IS_PROD ? serverConfig.log.levels.split(',') : undefined, + logger: IS_PROD ? SERVER_CONFIG.log.levels.split(',') : undefined, }); app.enableCors({ @@ -32,8 +32,8 @@ async function bootstrap() { app.set('trust proxy', true); // 启用 gzip 压缩 - if (serverConfig.compression?.enabled) { - app.use(compression(serverConfig.compression)); + if (SERVER_CONFIG.compression?.enabled) { + app.use(compression(SERVER_CONFIG.compression)); } // ~ set ejs @@ -44,7 +44,7 @@ async function bootstrap() { app.setBaseViewsDir(join(__dirname, '../../../', 'public')); app.setViewEngine('html'); - app.use(json(serverConfig.bodyParser.json)); + app.use(json(SERVER_CONFIG.bodyParser.json)); app.use( helmet({ @@ -57,15 +57,15 @@ async function bootstrap() { app.use( session({ store: new RedisStore({ - client: new Redis(serverConfig.redis), + client: new Redis(SERVER_CONFIG.redis), }), - ...serverConfig.session, + ...SERVER_CONFIG.session, }) ); app.use(LoggerMiddleware); - await app.listen(serverConfig.web.port); + await app.listen(SERVER_CONFIG.web.port); // eslint-disable-next-line no-console console.log(`bff-server is running on: ${await app.getUrl()}`); diff --git a/packages/server/src/pipelines/pipelines.module.ts b/packages/server/src/pipelines/pipelines.module.ts index 8debc4e..b0dee2f 100644 --- a/packages/server/src/pipelines/pipelines.module.ts +++ b/packages/server/src/pipelines/pipelines.module.ts @@ -1,11 +1,9 @@ import { Module } from '@nestjs/common'; -import { KubernetesModule } from '@yuntijs/k8s-client/kubernetes.module'; import { PipelinesService } from './pipelines.service'; @Module({ providers: [PipelinesService], exports: [PipelinesService], - imports: [KubernetesModule], }) export class PipelinesModule {} diff --git a/packages/server/src/pipelines/pipelines.service.ts b/packages/server/src/pipelines/pipelines.service.ts index c254c23..9cfbbe4 100644 --- a/packages/server/src/pipelines/pipelines.service.ts +++ b/packages/server/src/pipelines/pipelines.service.ts @@ -1,7 +1,6 @@ import { Inject, Injectable, Logger } from '@nestjs/common'; import { ConfigType } from '@nestjs/config'; -import { KubernetesService } from '@yuntijs/k8s-client/kubernetes.service'; -import { CRD, K8s } from '@yuntijs/k8s-client/lib'; +import { type CRD, K8s, KubernetesService } from '@yuntijs/k8s-client'; import { extractPipelineRunStatus } from '@/common/utils'; import serverConfig from '@/config/server.config'; @@ -150,6 +149,9 @@ export class PipelinesService { async makePipeLineRunInformer(namespace: string, pipeline: string) { const method = 'makePipeLineRunInformer'; const k8s = await this.k8sService.getSaClient(); + if (!k8s) { + return; + } const labelSelector = `tekton.dev/pipeline=${pipeline}`; const listFn = () => k8s.pipelineRun.list(namespace, { labelSelector }); const { group, version, name } = k8s.pipelineRun; diff --git a/packages/server/src/publish-records/publish-records.service.ts b/packages/server/src/publish-records/publish-records.service.ts index 500036c..b22c3b0 100644 --- a/packages/server/src/publish-records/publish-records.service.ts +++ b/packages/server/src/publish-records/publish-records.service.ts @@ -1,7 +1,7 @@ import { Inject, Injectable, Logger } from '@nestjs/common'; import { ConfigType } from '@nestjs/config'; import { Timeout } from '@nestjs/schedule'; -import { CRD } from '@yuntijs/k8s-client/lib'; +import { CRD } from '@yuntijs/k8s-client'; import { Like } from 'typeorm'; import { AppsMembersService } from '@/apps-members/apps-members.service'; @@ -368,8 +368,12 @@ export class PublishRecordsService { pipeline?.namespace, pipeline?.publish?.name ); + if (!informer) { + this.logger.warn(`[${method}] ⚠ informer start failed ⚠`); + return; + } const startInformer = () => - informer.start().then(() => this.logger.log(`[${method}] informer stated ...`)); + informer.start().then(() => this.logger.log(`[${method}] informer started ...`)); // informer.on('add', (pr: CRD.PipelineRun) => { // console.log(`Added: ${pr.metadata!.name}`); diff --git a/packages/server/tsconfig.json b/packages/server/tsconfig.json index 6eebd9f..d7a0409 100644 --- a/packages/server/tsconfig.json +++ b/packages/server/tsconfig.json @@ -21,7 +21,7 @@ "resolveJsonModule": true, "paths": { "@/*": ["src/*"], - "@yuntijs/k8s-client/*": ["../k8s-client/src/*"] + "@yuntijs/k8s-client": ["../k8s-client/src"] } }, "exclude": ["dist"]