From ef31ebf18c3fc7866daa98267418bab0ea48f222 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Fri, 13 Dec 2024 14:39:24 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E4=BB=93=E5=BA=93?= =?UTF-8?q?=E5=A4=87=E4=BB=BD=E5=8A=9F=E8=83=BD#2837?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/apidoc/backup/backup.md | 218 +++++++++++++ .../com/tencent/bkrepo/job/Constants.kt | 9 +- .../job/backup/config/DataBackupConfig.kt | 15 + .../bkrepo/job/backup/dao/BackupTaskDao.kt | 103 ++++++ .../bkrepo/job/backup/model/TBackupTask.kt | 99 ++++++ .../bkrepo/job/backup/pojo/BackupTaskState.kt | 38 +++ .../pojo/query/BackupFileReferenceInfo.kt | 39 +++ .../backup/pojo/query/BackupMavenMetadata.kt | 41 +++ .../job/backup/pojo/query/BackupNodeInfo.kt | 63 ++++ .../backup/pojo/query/BackupPackageInfo.kt | 80 +++++ .../pojo/query/BackupPackageVersionInfo.kt | 80 +++++ .../BackupPackageVersionInfoWithKeyInfo.kt | 83 +++++ .../backup/pojo/query/BackupProjectInfo.kt | 49 +++ .../backup/pojo/query/BackupRepositoryInfo.kt | 61 ++++ .../pojo/query/BackupStorageCredentials.kt | 45 +++ .../backup/pojo/query/VersionBackupInfo.kt | 39 +++ .../backup/pojo/query/common/BackupAccount.kt | 53 +++ .../pojo/query/common/BackupConanMetadata.kt | 12 + .../pojo/query/common/BackupCredentialSet.kt | 48 +++ .../pojo/query/common/BackupPermission.kt | 52 +++ .../backup/pojo/query/common/BackupRole.kt | 47 +++ .../pojo/query/common/BackupScopeRule.kt | 40 +++ .../pojo/query/common/BackupTemporaryToken.kt | 52 +++ .../backup/pojo/query/common/BackupUser.kt | 24 ++ .../query/enums/BackupCredentialStatus.kt | 37 +++ .../backup/pojo/query/enums/BackupDataEnum.kt | 134 ++++++++ .../pojo/query/enums/BackupRoleSource.kt | 36 +++ .../backup/pojo/query/enums/BackupRoleType.kt | 37 +++ .../pojo/query/enums/BackupUserSource.kt | 34 ++ .../job/backup/pojo/record/BackupContext.kt | 55 ++++ .../job/backup/pojo/record/BackupProgress.kt | 42 +++ .../pojo/setting/BackupConflictStrategy.kt | 43 +++ .../job/backup/pojo/setting/BackupCronType.kt | 39 +++ .../pojo/setting/BackupErrorStrategy.kt | 43 +++ .../pojo/setting/BackupExecutionPlan.kt | 48 +++ .../pojo/setting/BackupExecutionStrategy.kt | 48 +++ .../job/backup/pojo/setting/BackupSetting.kt | 50 +++ .../job/backup/pojo/task/BackupContent.kt | 64 ++++ .../bkrepo/job/backup/pojo/task/BackupTask.kt | 79 +++++ .../job/backup/pojo/task/BackupTaskRequest.kt | 39 +++ .../backup/pojo/task/ProjectContentInfo.kt | 68 ++++ .../job/backup/service/BackupDataHandler.kt | 33 ++ .../service/BackupRepoSpecialDataService.kt | 53 +++ .../job/backup/service/DataBackupService.kt | 15 + .../service/DataRecordsBackupService.kt | 36 +++ .../service/DataRecordsRestoreService.kt | 36 +++ .../job/backup/service/impl/BaseService.kt | 132 ++++++++ .../service/impl/DataBackupServiceImpl.kt | 123 +++++++ .../impl/DataRecordsBackupServiceImpl.kt | 267 +++++++++++++++ .../impl/DataRecordsRestoreServiceImpl.kt | 217 +++++++++++++ .../impl/base/BackupAccountDataHandler.kt | 128 ++++++++ .../base/BackupConanMetadataDataHandler.kt | 65 ++++ .../service/impl/base/BackupDataMappings.kt | 99 ++++++ .../base/BackupMavenMetadataDataHandler.kt | 88 +++++ .../impl/base/BackupNodeDataHandler.kt | 303 ++++++++++++++++++ .../impl/base/BackupPackageDataHandler.kt | 96 ++++++ .../base/BackupPackageVersionDataHandler.kt | 146 +++++++++ .../impl/base/BackupPermissionDataHandler.kt | 99 ++++++ .../impl/base/BackupProjectDataHandler.kt | 83 +++++ .../impl/base/BackupRepositoryDataHandler.kt | 130 ++++++++ .../impl/base/BackupRoleDataHandler.kt | 74 +++++ .../base/BackupTemporaryTokenDataHandler.kt | 56 ++++ .../impl/base/BackupUserDataHandler.kt | 113 +++++++ .../impl/repo/BackupConanRepoHandler.kt | 56 ++++ .../impl/repo/BackupMavenRepoHandler.kt | 56 ++++ .../impl/repo/BackupRepoSpecialMappings.kt | 54 ++++ .../bkrepo/job/backup/util/ZipFileUtil.kt | 62 ++++ .../tencent/bkrepo/job/config/JobConfig.kt | 2 + .../controller/user/UserBackupController.kt | 75 +++++ 69 files changed, 4982 insertions(+), 1 deletion(-) create mode 100644 docs/apidoc/backup/backup.md create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/config/DataBackupConfig.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/dao/BackupTaskDao.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/model/TBackupTask.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/BackupTaskState.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupFileReferenceInfo.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupMavenMetadata.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupNodeInfo.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupPackageInfo.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupPackageVersionInfo.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupPackageVersionInfoWithKeyInfo.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupProjectInfo.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupRepositoryInfo.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupStorageCredentials.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/VersionBackupInfo.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupAccount.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupConanMetadata.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupCredentialSet.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupPermission.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupRole.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupScopeRule.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupTemporaryToken.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupUser.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/enums/BackupCredentialStatus.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/enums/BackupDataEnum.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/enums/BackupRoleSource.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/enums/BackupRoleType.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/enums/BackupUserSource.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/record/BackupContext.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/record/BackupProgress.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupConflictStrategy.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupCronType.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupErrorStrategy.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupExecutionPlan.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupExecutionStrategy.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupSetting.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupContent.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupTask.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupTaskRequest.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/ProjectContentInfo.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/BackupDataHandler.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/BackupRepoSpecialDataService.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataBackupService.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataRecordsBackupService.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataRecordsRestoreService.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/BaseService.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataBackupServiceImpl.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataRecordsBackupServiceImpl.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataRecordsRestoreServiceImpl.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupAccountDataHandler.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupConanMetadataDataHandler.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupDataMappings.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupMavenMetadataDataHandler.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupNodeDataHandler.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupPackageDataHandler.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupPackageVersionDataHandler.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupPermissionDataHandler.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupProjectDataHandler.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupRepositoryDataHandler.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupRoleDataHandler.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupTemporaryTokenDataHandler.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupUserDataHandler.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/repo/BackupConanRepoHandler.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/repo/BackupMavenRepoHandler.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/repo/BackupRepoSpecialMappings.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/util/ZipFileUtil.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/controller/user/UserBackupController.kt diff --git a/docs/apidoc/backup/backup.md b/docs/apidoc/backup/backup.md new file mode 100644 index 0000000000..f5cc66958a --- /dev/null +++ b/docs/apidoc/backup/backup.md @@ -0,0 +1,218 @@ +# 系统备份以及恢复接口 + +[toc] + +## 创建备份/恢复任务 + +- API: POST /job/api/job/backup +- API 名称: 创建备份/恢复任务 +- 功能说明： + - 中文：创建备份/恢复任务 + - English：create backup/restore task + +- 请求体 + ```json + 新建备份任务 + { + "name":"test", + "storeLocation":"F:\\backuptest", + "content":{ + "commonData": true, + "compression":true, + "projects":[{ + "projectRegex":"*" + }] + } + } + ``` + ```json + 新建恢复任务 + { + "name":"test1-restore", + "storeLocation":"F:\\backuptest\\test1\\test1-20241224.171354.zip", + "type":"DATA_RESTORE", + "content":{ + "commonData": true, + "compression":true, + "projects":[{ + "projectRegex":"*" + }] + } + } + ``` +- 请求字段说明 + + |字段|类型|是否必须|默认值|说明|Description| + |---|---|---|---|---|---| + |name|string|是|无|任务名|task name| + |storeLocation|string|是|无|本地存储地址|store folder| + |backupSetting|backupSetting|否|有|任务设置| task setting| + |content|BackupContent|是|无|任务内容| task content| + |type|string|否|有|任务类型：备份DATA_BACKUP/恢复DATA_RESTORE| task type: DATA_BACKUP/DATA_RESTORE | + + +- BackupContent对象说明 + + |字段|类型|是否必须|默认值|说明|Description| + |---|---|---|---|---|---| + |commonData|bool|否|false|是否备份或者恢复基础数据|backup/restore basic data| + |compression|bool|否|false|是否压缩最终备份文件目录|compress backup folder| + |increment|bool|否|null|是否增量备份|incremental backup| + |incrementDate|string|否|无|如开启增量备份，日期不指定则从任务执行日期前一天开始备份；如指定，则从指定日期开始|if increment is true and incrementDate is not set, then the default incrementDate is current date minus 1. Backup tasks will backup all the data after incrementDate| + |projects|arrays[ProjectContentInfo]|否|null|具体项目仓库信息|project or repo info| + +- ProjectContentInfo对象说明 + + |字段|类型|是否必须|默认值|说明|Description| + |---|---|---|---|---|---| + |projectId|string|否|null|指定备份/恢复项目|target projectId| + |projectRegex|string|否|null|备份/恢复项目正则|regex of target projectid| + |repoList|arrays|否|null|指定备份/恢复仓库列表|target repos| + |repoRegex|string|否|null|备份/恢复仓库正则|regex of target repo| + |excludeProjects|arrays|否|null|排除项目列表|exclude project list| + |excludeRepos|arrays|否|null|排除仓库列表|exclude repo list| + +- setting对象说明 + + |字段|类型|是否必须|默认值|说明|Description| + |---|---|---|---|---|---| + |conflictStrategy|enum|是|OVERWRITE|[SKIP,OVERWRITE]|conflict strategy| + |errorStrategy|enum|是|CONTINUE|[CONTINUE,FAST_FAIL]|error strategy| + |executionStrategy|enum|是|IMMEDIATELY|[IMMEDIATELY,SPECIFIED_TIME,CRON_EXPRESSION]|execution strategy| + |executionPlan|object|是|无|调度策略|execution plan| + +- executionPlan对象说明 + + |字段|类型|是否必须|默认值|说明|Description| + |---|---|---|---|---|---| + |executeImmediately|bool|是|true|立即执行|execute immediately| + |executeTime|time|否|无|执行时间执行|execute time| + |cronExpression|string|否|无|cron表达式执行|cron expression| + +- 响应体 + + ``` json + { + "code" : 0, + "message" : null, + "data" : "676a8abd60cc914e2a8a26f0", + "traceId" : null + } + ``` + +- data字段说明 + 返回任务id + + +## 执行任务 + +- API: POST /api/job/backup/execute/{taskId} +- API 名称: 执行任务 +- 功能说明： + - 中文：执行任务 + - English：execute task + +- 请求体 + 此接口请求体为空 + +- 请求字段说明 + + |字段|类型|是否必须|默认值|说明|Description| + |---|---|---|---|---|---| + |taskId|string|是|无|任务id|task id| + +- 请求头 + +- 响应头 + +- 响应体 + + ``` json + { + "code" : 0, + "message" : null, + "data" : null, + "traceId" : null + } + ``` + +## 分页获取任务列表 + +- API: GET /job/api/job/backup/tasks?state={state}&pageSize={pageSize}&pageNumber={pageNumber} +- API 名称: 获取任务列表 +- 功能说明： + - 中文：获取任务列表 + - English：get task list + +- 请求体 + 此接口请求体为空 + +- 请求字段说明 + + |字段|类型|是否必须|默认值|说明|Description| + |---|---|---|---|---|---| + |state|string|否|null|任务执行状态|task state: PENDING/RUNNING/FINISHED| + |pageNumber|int|否|1|页码|page number| + |pageSize|int|否|20|页大小|page size| + +- 请求头 + +- 响应头 + +- 响应体 + ``` json + { + "code": 0, + "message": null, + "data": { + "pageNumber": 1, + "pageSize": 1, + "totalRecords": 7, + "totalPages": 7, + "records": [ + { + "id": "675ab19767a452255ba84d8b", + "createdBy": "xxxx", + "createdDate": "xxxx", + "lastModifiedBy": "xxx", + "lastModifiedDate": "xxx", + "startDate": "xxx", + "endDate": "xxx", + "state": "FINISHED", + "content": { + "commonData": false, + "compression": false, + "projects": [ + { + "projectId": "backuptest", + "projectRegex": null, + "repoList": null, + "repoRegex": null, + "excludeProjects": null, + "excludeRepos": null + } + ], + "increment": null, + "incrementDate": null + }, + "storeLocation": "F:\\backuptest", + "backupSetting": { + "conflictStrategy": "SKIP", + "errorStrategy": "CONTINUE", + "executionStrategy": "IMMEDIATELY", + "executionPlan": { + "executeImmediately": true, + "executeTime": null, + "cronExpression": null + }, + "spaceCapCheck": true + }, + "type": "DATA_BACKUP" + } + ], + "count": 7, + "page": 1 + }, + "traceId": "xxx" + } + ``` json diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/Constants.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/Constants.kt index d5d4e75a2c..9a85772f1a 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/Constants.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/Constants.kt @@ -69,6 +69,7 @@ const val REPLICA_TYPE = "replicaType" const val NAME = "name" const val SIZE = "size" const val NODE_NUM = "nodeNum" +const val PACKAGE_ID = "packageId" /** @@ -114,4 +115,10 @@ const val PACKAGE_DOWNLOADS_COLLECTION_NAME = "package_downloads" const val SEPARATION_TASK_COLLECTION_NAME = "separation_task" const val PACKAGE_VERSION = "version" -const val PACKAGE_DOWNLOAD_DATE = "date" \ No newline at end of file +const val PACKAGE_DOWNLOAD_DATE = "date" + +/** + * 记录备份 + */ +const val DATA_RECORDS_BACKUP = "DATA_BACKUP" +const val DATA_RECORDS_RESTORE = "DATA_RESTORE" \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/config/DataBackupConfig.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/config/DataBackupConfig.kt new file mode 100644 index 0000000000..fb59ffdfd3 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/config/DataBackupConfig.kt @@ -0,0 +1,15 @@ +package com.tencent.bkrepo.job.backup.config + +import com.tencent.bkrepo.common.storage.credentials.InnerCosCredentials +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties("backup") +data class DataBackupConfig( + // 当磁盘已用容量比例大于大于阈值时,不进行备份 + var usageThreshold: Double = 0.8, + + /** + * 备份数据存储实例 + * */ + var cos: InnerCosCredentials = InnerCosCredentials(), +) \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/dao/BackupTaskDao.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/dao/BackupTaskDao.kt new file mode 100644 index 0000000000..6db4bf47f0 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/dao/BackupTaskDao.kt @@ -0,0 +1,103 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.dao + +import com.tencent.bkrepo.common.mongo.dao.simple.SimpleMongoDao +import com.tencent.bkrepo.job.backup.model.TBackupTask +import com.tencent.bkrepo.job.backup.pojo.BackupTaskState +import com.tencent.bkrepo.repository.constant.SYSTEM_USER +import org.springframework.data.domain.PageRequest +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Repository +import java.time.LocalDateTime + +@Repository +class BackupTaskDao : SimpleMongoDao() { + + fun findTasksById(taskId: String): TBackupTask? { + val criteria = Criteria().and(TBackupTask::id.name).isEqualTo(taskId) + return findOne(Query(criteria)) + } + + fun find(state: String?, pageRequest: PageRequest): List { + val criteria = Criteria() + state?.let { criteria.and(TBackupTask::state.name).isEqualTo(it) } + return find(Query(criteria).with(pageRequest)) + } + + fun count(state: String?): Long { + val criteria = Criteria() + state?.let { criteria.and(TBackupTask::state.name).isEqualTo(it) } + return count(Query(criteria)) + } + + fun updateState( + taskId: String, + state: BackupTaskState, + startDate: LocalDateTime? = null, + endDate: LocalDateTime? = null + ) { + val criteria = Criteria().and(ID).isEqualTo(taskId) + val update = Update.update(TBackupTask::lastModifiedBy.name, SYSTEM_USER) + .set(TBackupTask::lastModifiedDate.name, LocalDateTime.now()) + .set(TBackupTask::state.name, state.name) + startDate?.let { update.set(TBackupTask::startDate.name, startDate) } + endDate?.let { update.set(TBackupTask::endDate.name, endDate) } + this.updateFirst(Query(criteria), update) + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/model/TBackupTask.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/model/TBackupTask.kt new file mode 100644 index 0000000000..0dee9f2adb --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/model/TBackupTask.kt @@ -0,0 +1,99 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.model + +import com.tencent.bkrepo.job.DATA_RECORDS_BACKUP +import com.tencent.bkrepo.job.backup.model.TBackupTask.Companion.TASK_STATE_IDX +import com.tencent.bkrepo.job.backup.model.TBackupTask.Companion.TASK_STATE_IDX_DEF +import com.tencent.bkrepo.job.backup.model.TBackupTask.Companion.TASK_TYPE_IDX +import com.tencent.bkrepo.job.backup.model.TBackupTask.Companion.TASK_TYPE_IDX_DEF +import com.tencent.bkrepo.job.backup.pojo.task.BackupContent +import com.tencent.bkrepo.job.backup.pojo.BackupTaskState +import com.tencent.bkrepo.job.backup.pojo.setting.BackupSetting +import org.springframework.data.mongodb.core.index.CompoundIndex +import org.springframework.data.mongodb.core.index.CompoundIndexes +import org.springframework.data.mongodb.core.mapping.Document +import java.time.LocalDateTime + +/** + * 备份与恢复任务 + */ +@Document("backup_task") +@CompoundIndexes( + CompoundIndex(name = TASK_STATE_IDX, def = TASK_STATE_IDX_DEF, background = true), + CompoundIndex(name = TASK_TYPE_IDX, def = TASK_TYPE_IDX_DEF, background = true), +) +data class TBackupTask( + val id: String? = null, + val name: String, + val storeLocation: String, + val type: String = DATA_RECORDS_BACKUP, + var content: BackupContent? = null, + val backupSetting: BackupSetting, + val state: String = BackupTaskState.PENDING.name, + val createdBy: String, + val createdDate: LocalDateTime, + val lastModifiedBy: String, + val lastModifiedDate: LocalDateTime, + val startDate: LocalDateTime? = null, + val endDate: LocalDateTime? = null, +) { + companion object { + const val TASK_STATE_IDX = "state_idx" + const val TASK_STATE_IDX_DEF = "{'state': 1}" + const val TASK_TYPE_IDX = "type_idx" + const val TASK_TYPE_IDX_DEF = "{'type': 1}" + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/BackupTaskState.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/BackupTaskState.kt new file mode 100644 index 0000000000..89cb2e152e --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/BackupTaskState.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo + + +enum class BackupTaskState { + + PENDING, + + RUNNING, + + FINISHED, +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupFileReferenceInfo.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupFileReferenceInfo.kt new file mode 100644 index 0000000000..22649f57fb --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupFileReferenceInfo.kt @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query + +data class BackupFileReferenceInfo( + var id: String? = null, + var sha256: String, + var credentialsKey: String? = null, + var count: Long +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupMavenMetadata.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupMavenMetadata.kt new file mode 100644 index 0000000000..b917a05058 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupMavenMetadata.kt @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query + +data class BackupMavenMetadata( + var id: String?, + val projectId: String, + val repoName: String, + val groupId: String, + val artifactId: String, + val version: String, + val timestamp: String?, + val buildNo: Int, + val classifier: String?, + val extension: String, +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupNodeInfo.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupNodeInfo.kt new file mode 100644 index 0000000000..84cb18d420 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupNodeInfo.kt @@ -0,0 +1,63 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query + +import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel +import java.time.LocalDateTime + +data class BackupNodeInfo( + var id: String? = null, + var createdBy: String, + var createdDate: LocalDateTime, + var lastModifiedBy: String, + var lastModifiedDate: LocalDateTime, + var lastAccessDate: LocalDateTime? = null, + + var folder: Boolean, + var path: String, + var name: String, + var fullPath: String, + var size: Long, + var expireDate: LocalDateTime? = null, + var sha256: String? = null, + var md5: String? = null, + var deleted: LocalDateTime? = null, + var copyFromCredentialsKey: String? = null, + var copyIntoCredentialsKey: String? = null, + var metadata: MutableList? = null, + var clusterNames: Set? = null, + var nodeNum: Long? = null, + var archived: Boolean? = null, + var compressed: Boolean? = null, + var projectId: String, + var repoName: String, +) \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupPackageInfo.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupPackageInfo.kt new file mode 100644 index 0000000000..6cf4a62a3d --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupPackageInfo.kt @@ -0,0 +1,80 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query + +import com.tencent.bkrepo.repository.pojo.packages.PackageType +import java.time.LocalDateTime + +data class BackupPackageInfo( + var id: String? = null, + var createdBy: String, + var createdDate: LocalDateTime, + var lastModifiedBy: String, + var lastModifiedDate: LocalDateTime, + + var projectId: String, + var repoName: String, + var name: String, + var key: String, + var type: PackageType, + var latest: String? = null, + var downloads: Long, + var versions: Long, + var description: String? = null, + var versionTag: Map? = null, + var extension: Map? = null, + var historyVersion: Set = emptySet(), + var clusterNames: Set? = null, +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupPackageVersionInfo.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupPackageVersionInfo.kt new file mode 100644 index 0000000000..524d47d538 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupPackageVersionInfo.kt @@ -0,0 +1,80 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query + +import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel +import java.time.LocalDateTime + +data class BackupPackageVersionInfo( + var id: String? = null, + var createdBy: String, + var createdDate: LocalDateTime, + var lastModifiedBy: String, + var lastModifiedDate: LocalDateTime, + + var packageId: String, + var name: String, + var size: Long, + var ordinal: Long, + var downloads: Long, + var manifestPath: String? = null, + var artifactPath: String? = null, + var artifactPaths: MutableSet? = null, + var stageTag: List, + var metadata: List, + var tags: List? = null, + var extension: Map? = null, + var clusterNames: Set? = null, +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupPackageVersionInfoWithKeyInfo.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupPackageVersionInfoWithKeyInfo.kt new file mode 100644 index 0000000000..8ba74ea317 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupPackageVersionInfoWithKeyInfo.kt @@ -0,0 +1,83 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query + +import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel +import java.time.LocalDateTime + +data class BackupPackageVersionInfoWithKeyInfo( + var id: String? = null, + var createdBy: String, + var createdDate: LocalDateTime, + var lastModifiedBy: String, + var lastModifiedDate: LocalDateTime, + + var packageId: String, + var name: String, + var size: Long, + var ordinal: Long, + var downloads: Long, + var manifestPath: String? = null, + var artifactPath: String? = null, + var artifactPaths: MutableSet? = null, + var stageTag: List, + var metadata: List, + var tags: List? = null, + var extension: Map? = null, + var clusterNames: Set? = null, + var projectId: String? = null, + var repoName: String? = null, + var key: String? = null, +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupProjectInfo.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupProjectInfo.kt new file mode 100644 index 0000000000..b59bfe23cb --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupProjectInfo.kt @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query + +import com.tencent.bkrepo.repository.pojo.project.ProjectMetadata +import java.time.LocalDateTime + +data class BackupProjectInfo( + var id: String? = null, + var createdBy: String, + var createdDate: LocalDateTime, + var lastModifiedBy: String, + var lastModifiedDate: LocalDateTime, + + var name: String, + var displayName: String, + var description: String, + var metadata: List = emptyList(), + var credentialsKey: String? = null, +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupRepositoryInfo.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupRepositoryInfo.kt new file mode 100644 index 0000000000..8e8e5f1b96 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupRepositoryInfo.kt @@ -0,0 +1,61 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query + +import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import java.time.LocalDateTime + +data class BackupRepositoryInfo( + var id: String? = null, + var createdBy: String, + var createdDate: LocalDateTime, + var lastModifiedBy: String, + var lastModifiedDate: LocalDateTime, + + var name: String, + var type: RepositoryType, + var category: RepositoryCategory, + var public: Boolean, + var description: String? = null, + var configuration: String, + var credentialsKey: String? = null, + var oldCredentialsKey: String? = null, + var display: Boolean = true, + + var projectId: String, + + var quota: Long? = null, + var used: Long? = null, + var clusterNames: Set? = null, + var deleted: LocalDateTime? = null +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupStorageCredentials.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupStorageCredentials.kt new file mode 100644 index 0000000000..54006b6d62 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/BackupStorageCredentials.kt @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query + +import java.time.LocalDateTime + +data class BackupStorageCredentials( + var id: String? = null, + var createdBy: String, + var createdDate: LocalDateTime, + var lastModifiedBy: String, + var lastModifiedDate: LocalDateTime, + + var credentials: String, + var region: String? = null +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/VersionBackupInfo.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/VersionBackupInfo.kt new file mode 100644 index 0000000000..34a8df5037 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/VersionBackupInfo.kt @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query + +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType + +data class VersionBackupInfo( + val type: RepositoryType, + val projectId: String, + val repoName: String, + val packageKey: String, + val version: String, + val overwrite: Boolean = false +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupAccount.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupAccount.kt new file mode 100644 index 0000000000..39b779f049 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupAccount.kt @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query.common + +import com.tencent.bkrepo.auth.pojo.enums.ResourceType +import com.tencent.bkrepo.auth.pojo.oauth.AuthorizationGrantType +import java.time.LocalDateTime + +class BackupAccount( + var id: String? = null, + var appId: String, + var locked: Boolean, + var credentials: List, + var owner: String?, + var authorizationGrantTypes: Set?, + var homepageUrl: String?, + var redirectUri: String?, + var avatarUrl: String?, + var scope: Set?, + var scopeDesc: List?, + var description: String?, + var createdDate: LocalDateTime?, + var lastModifiedDate: LocalDateTime? +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupConanMetadata.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupConanMetadata.kt new file mode 100644 index 0000000000..3b8f1dcfe4 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupConanMetadata.kt @@ -0,0 +1,12 @@ +package com.tencent.bkrepo.job.backup.pojo.query.common + +data class BackupConanMetadata( + var id: String?, + val projectId: String, + val repoName: String, + val user: String, + val name: String, + val version: String, + val channel: String, + val recipe: String +) \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupCredentialSet.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupCredentialSet.kt new file mode 100644 index 0000000000..c4815cd9b8 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupCredentialSet.kt @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query.common + +import com.tencent.bkrepo.auth.pojo.oauth.AuthorizationGrantType +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupCredentialStatus +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty +import java.time.LocalDateTime + +@ApiModel("账户认证信息") +data class BackupCredentialSet( + @ApiModelProperty("accessKey") + val accessKey: String, + @ApiModelProperty("secretKey") + var secretKey: String, + @ApiModelProperty("创建时间") + val createdAt: LocalDateTime, + @ApiModelProperty("状态") + val status: BackupCredentialStatus, + @ApiModelProperty("认证授权方式") + val authorizationGrantType: AuthorizationGrantType? = AuthorizationGrantType.PLATFORM +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupPermission.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupPermission.kt new file mode 100644 index 0000000000..7fe0f5db5e --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupPermission.kt @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query.common + +import java.time.LocalDateTime + +data class BackupPermission( + var id: String? = null, + var resourceType: String, + var projectId: String? = null, + var permName: String, + var repos: List = emptyList(), + var includePattern: List = emptyList(), + var excludePattern: List = emptyList(), + var createBy: String, + val createAt: LocalDateTime, + var updatedBy: String, + val updateAt: LocalDateTime, + var users: List = emptyList(), + var roles: List = emptyList(), + var departments: List = emptyList(), + var actions: List = emptyList() +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupRole.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupRole.kt new file mode 100644 index 0000000000..cf16d74b3e --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupRole.kt @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query.common + +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupRoleSource +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupRoleType + +data class BackupRole( + var id: String? = null, + val roleId: String, + val type: BackupRoleType, + var name: String, + val projectId: String, + val repoName: String? = null, + val admin: Boolean = false, + var description: String? = null, + var source: BackupRoleSource? = null +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupScopeRule.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupScopeRule.kt new file mode 100644 index 0000000000..6f43af1b63 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupScopeRule.kt @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query.common + +import com.tencent.bkrepo.common.query.enums.OperationType +import io.swagger.annotations.ApiModelProperty + +data class BackupScopeRule( + @ApiModelProperty("字段名") + val field: String, + @ApiModelProperty("值") + val value: Any, + @ApiModelProperty("操作类型") + val operation: OperationType = OperationType.DEFAULT +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupTemporaryToken.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupTemporaryToken.kt new file mode 100644 index 0000000000..74a54ef266 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupTemporaryToken.kt @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query.common + +import com.tencent.bkrepo.auth.pojo.token.TokenType +import java.time.LocalDateTime + +data class BackupTemporaryToken( + var id: String? = null, + var createdBy: String, + var createdDate: LocalDateTime, + var lastModifiedBy: String, + var lastModifiedDate: LocalDateTime, + var projectId: String, + var repoName: String, + var fullPath: String, + var token: String, + var authorizedUserList: Set, + var authorizedIpList: Set, + var expireDate: LocalDateTime? = null, + var permits: Int? = null, + var type: TokenType +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupUser.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupUser.kt new file mode 100644 index 0000000000..509b2ad686 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/common/BackupUser.kt @@ -0,0 +1,24 @@ +package com.tencent.bkrepo.job.backup.pojo.query.common + +import com.tencent.bkrepo.auth.pojo.token.Token +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupUserSource +import java.time.LocalDateTime + +data class BackupUser( + var id: String?, + val userId: String, + val name: String, + val pwd: String, + val admin: Boolean = false, + val locked: Boolean = false, + val tokens: List = emptyList(), + val roles: List = emptyList(), + val asstUsers: List = emptyList(), + val group: Boolean = false, + val email: String? = null, + val phone: String? = null, + var accounts: List? = emptyList(), + val source: BackupUserSource = BackupUserSource.REPO, + val createdDate: LocalDateTime? = LocalDateTime.now(), + val lastModifiedDate: LocalDateTime? = LocalDateTime.now() +) \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/enums/BackupCredentialStatus.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/enums/BackupCredentialStatus.kt new file mode 100644 index 0000000000..45d2d71bd8 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/enums/BackupCredentialStatus.kt @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query.enums + +enum class BackupCredentialStatus { + ENABLE, + DISABLE +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/enums/BackupDataEnum.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/enums/BackupDataEnum.kt new file mode 100644 index 0000000000..0d80a5d653 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/enums/BackupDataEnum.kt @@ -0,0 +1,134 @@ +package com.tencent.bkrepo.job.backup.pojo.query.enums + +import com.tencent.bkrepo.job.backup.pojo.query.BackupMavenMetadata +import com.tencent.bkrepo.job.backup.pojo.query.BackupNodeInfo +import com.tencent.bkrepo.job.backup.pojo.query.BackupPackageInfo +import com.tencent.bkrepo.job.backup.pojo.query.BackupPackageVersionInfoWithKeyInfo +import com.tencent.bkrepo.job.backup.pojo.query.BackupProjectInfo +import com.tencent.bkrepo.job.backup.pojo.query.BackupRepositoryInfo +import com.tencent.bkrepo.job.backup.pojo.query.common.BackupAccount +import com.tencent.bkrepo.job.backup.pojo.query.common.BackupConanMetadata +import com.tencent.bkrepo.job.backup.pojo.query.common.BackupPermission +import com.tencent.bkrepo.job.backup.pojo.query.common.BackupRole +import com.tencent.bkrepo.job.backup.pojo.query.common.BackupTemporaryToken +import com.tencent.bkrepo.job.backup.pojo.query.common.BackupUser + +enum class BackupDataEnum( + val collectionName: String, + val fileName: String, + val backupClazz: Class<*>, + val type: String, + val parentData: Boolean = true, + val relatedData: String? = null, + val specialData: Boolean = false, +) { + USER_DATA( + "user", + "user.json", + BackupUser::class.java, + "PUBLIC" + ), + ROLE_DATA( + "role", + "role.json", + BackupRole::class.java, + "PUBLIC" + ), + TEMPORARY_TOKEN_DATA( + "temporary_token", + "temporary_token.json", + BackupTemporaryToken::class.java, + "PUBLIC" + ), + PERMISSION_DATA( + "permission", + "permission.json", + BackupPermission::class.java, + "PUBLIC" + ), + ACCOUNT_DATA( + "account", + "account.json", + BackupAccount::class.java, + "PUBLIC" + ), + PROJECT_DATA( + "project", + "project.json", + BackupProjectInfo::class.java, + "PRIVATE" + ), + REPOSITORY_DATA( + "repository", + "repository.json", + BackupRepositoryInfo::class.java, + "PRIVATE" + ), + PACKAGE_DATA( + "package", + "package.json", + BackupPackageInfo::class.java, + "PRIVATE", + relatedData = "package_version" + ), + PACKAGE_VERSION_DATA( + "package_version", + "package-version.json", + BackupPackageVersionInfoWithKeyInfo::class.java, + "PRIVATE", + false + ), + NODE_DATA( + "node", + "node.json", + BackupNodeInfo::class.java, + "PRIVATE" + ), + MAVEN_METADATA_DATA( + "maven_metadata", + "maven-metadata.json", + BackupMavenMetadata::class.java, + "PRIVATE", + false, + specialData = true + ), + CONAN_METADATA_DATA( + "conan_metadata", + "conan-metadata.json", + BackupConanMetadata::class.java, + "PRIVATE", + false, + specialData = true + ), + ; + + companion object { + fun get(backupClazz: Class<*>): BackupDataEnum { + values().forEach { + if (backupClazz == it.backupClazz) return it + } + throw IllegalArgumentException("No enum for constant $backupClazz") + } + + fun getByCollectionName(collectionName: String): BackupDataEnum { + values().forEach { + if (collectionName == it.collectionName) return it + } + throw IllegalArgumentException("No enum for constant $collectionName") + } + + fun getParentAndSpecialDataList(type: String): List { + return values().filter { + it.type == type && it.parentData == true + } + } + + fun getNonSpecialDataList(type: String): List { + return values().filter { it.type == type && it.specialData == false } + } + + const val PUBLIC_TYPE = "PUBLIC" + const val PRIVATE_TYPE = "PRIVATE" + + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/enums/BackupRoleSource.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/enums/BackupRoleSource.kt new file mode 100644 index 0000000000..7b99694300 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/enums/BackupRoleSource.kt @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query.enums + +import io.swagger.annotations.ApiModel + +@ApiModel("角色来源") +enum class BackupRoleSource { + DEVOPS, + TAI +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/enums/BackupRoleType.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/enums/BackupRoleType.kt new file mode 100644 index 0000000000..88aae111e5 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/enums/BackupRoleType.kt @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query.enums + +enum class BackupRoleType { + PROJECT, + REPO +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/enums/BackupUserSource.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/enums/BackupUserSource.kt new file mode 100644 index 0000000000..f65d376878 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/query/enums/BackupUserSource.kt @@ -0,0 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.query.enums; + +enum class BackupUserSource(val value: String) { + PIPELINE("pipeline"), + PAAS("paas"), + REPO("repo"); +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/record/BackupContext.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/record/BackupContext.kt new file mode 100644 index 0000000000..24a7e556bb --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/record/BackupContext.kt @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.record + +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import com.tencent.bkrepo.job.backup.model.TBackupTask +import com.tencent.bkrepo.job.backup.pojo.query.BackupNodeInfo +import java.nio.file.Path +import java.time.LocalDateTime + +class BackupContext( + val task: TBackupTask, +) { + lateinit var targertPath: Path + lateinit var startDate: LocalDateTime + var currentPath: Path? = null + var type: String = task.type + var taskId: String = task.id!! + var currentProjectId: String? = null + var currentRepoName: String? = null + var currentRepositoryType: RepositoryType? = null + var currentStorageCredentials: StorageCredentials? = null + var currentPackageId: String? = null + var currentPackageKey: String? = null + var currentVersionName: String? = null + var currentNode: BackupNodeInfo? = null + var currentFile: String? = null + var incrementDate: LocalDateTime? = null +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/record/BackupProgress.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/record/BackupProgress.kt new file mode 100644 index 0000000000..6ea05fe885 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/record/BackupProgress.kt @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.record + +data class BackupProgress( + var success: Long = 0, + var failed: Long = 0, + var skipped: Long = 0, + /** + * 最后执行id + */ + var projectId: String? = null, + var repoName: String? = null, + var packageId: String? = null, + var versionId: String? = null, + var nodeId: String? = null +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupConflictStrategy.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupConflictStrategy.kt new file mode 100644 index 0000000000..1cb18a6638 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupConflictStrategy.kt @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.setting + +/** + * 冲突处理策略 + */ +enum class BackupConflictStrategy { + /** + * 跳过文件 + */ + SKIP, + + /** + * 覆盖 + */ + OVERWRITE +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupCronType.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupCronType.kt new file mode 100644 index 0000000000..2e6b1a94ad --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupCronType.kt @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.setting + +/** + * 备份执行策略 + */ +enum class BackupCronType { + // 调度同步，指定时间/定时执行 + SCHEDULED, + + // 只执行一次，手动调用执行 + RUN_ONCE, +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupErrorStrategy.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupErrorStrategy.kt new file mode 100644 index 0000000000..5eded5eb09 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupErrorStrategy.kt @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.setting + +/** + * 出错时的处理策略 + */ +enum class BackupErrorStrategy { + /** + * 出错时继续执行 + */ + CONTINUE, + + /** + * 出错后本次任务失败 + */ + FAST_FAIL +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupExecutionPlan.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupExecutionPlan.kt new file mode 100644 index 0000000000..1e29ac845c --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupExecutionPlan.kt @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.setting + +import java.time.LocalDateTime + +/** + * 执行计划 + */ +data class BackupExecutionPlan( + /** + * 执行一次，创建后立即执行 + */ + val executeImmediately: Boolean = true, + /** + * 执行一次，指定时间执行 + */ + val executeTime: LocalDateTime? = null, + /** + * cron表达式定时执行 + */ + val cronExpression: String? = null +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupExecutionStrategy.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupExecutionStrategy.kt new file mode 100644 index 0000000000..30f882792a --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupExecutionStrategy.kt @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.setting + +/** + * 计划调度策略 + */ +enum class BackupExecutionStrategy { + /** + * 立即执行 + */ + IMMEDIATELY, + + /** + * 指定时间执行 + */ + SPECIFIED_TIME, + + /** + * cron表达式执行 + */ + CRON_EXPRESSION +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupSetting.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupSetting.kt new file mode 100644 index 0000000000..da330289a0 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/setting/BackupSetting.kt @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.setting + +/** + * 备份任务设置 + */ +data class BackupSetting( + /** + * 冲突解决策略 + */ + val conflictStrategy: BackupConflictStrategy = BackupConflictStrategy.OVERWRITE, + /** + * 错误处理策略 + */ + val errorStrategy: BackupErrorStrategy = BackupErrorStrategy.CONTINUE, + /** + * 执行计划策略 + */ + val executionStrategy: BackupExecutionStrategy = BackupExecutionStrategy.IMMEDIATELY, + /** + * 执行计划 + */ + val executionPlan: BackupExecutionPlan = BackupExecutionPlan(), +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupContent.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupContent.kt new file mode 100644 index 0000000000..f14e566137 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupContent.kt @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.task + +data class BackupContent( + var commonData: Boolean = false, + var compression: Boolean = false, + var projects: MutableList? = null, + var increment: Boolean? = null, + // 如开启增量备份，不指定日期则默认周期为为前一天凌晨到现在的增量 + var incrementDate: String? = null, +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupTask.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupTask.kt new file mode 100644 index 0000000000..180917a76e --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupTask.kt @@ -0,0 +1,79 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.task + +import com.tencent.bkrepo.job.backup.model.TBackupTask +import com.tencent.bkrepo.job.backup.pojo.setting.BackupSetting +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty +import java.time.LocalDateTime + +@ApiModel("数据备份任务") +data class BackupTask( + @ApiModelProperty("ID") + val id: String? = null, + @ApiModelProperty("创建人") + val createdBy: String, + @ApiModelProperty("创建时间") + val createdDate: LocalDateTime, + @ApiModelProperty("最后修改人") + val lastModifiedBy: String, + @ApiModelProperty("最后修改时间") + val lastModifiedDate: LocalDateTime, + @ApiModelProperty("任务开始执行的时间") + val startDate: LocalDateTime? = null, + @ApiModelProperty("任务结束执行的时间") + val endDate: LocalDateTime? = null, + @ApiModelProperty("任务状态") + val state: String, + @ApiModelProperty("任务内容") + val content: BackupContent?, + @ApiModelProperty("存储路径") + val storeLocation: String, + @ApiModelProperty("任务配置") + val backupSetting: BackupSetting, + @ApiModelProperty("任务类型") + val type: String, +) { + companion object { + fun TBackupTask.toDto() = BackupTask( + id = id, + createdBy = createdBy, + createdDate = createdDate, + lastModifiedBy = lastModifiedBy, + lastModifiedDate = lastModifiedDate, + startDate = startDate, + endDate = endDate, + state = state, + content = content, + storeLocation = storeLocation, + backupSetting = backupSetting, + type = type + ) + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupTaskRequest.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupTaskRequest.kt new file mode 100644 index 0000000000..87ea4af65d --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/BackupTaskRequest.kt @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.task + +import com.tencent.bkrepo.job.DATA_RECORDS_BACKUP +import com.tencent.bkrepo.job.backup.pojo.setting.BackupSetting + +data class BackupTaskRequest( + val name: String, + val storeLocation: String, + val type: String = DATA_RECORDS_BACKUP, + var content: BackupContent? = null, + val backupSetting: BackupSetting = BackupSetting(), +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/ProjectContentInfo.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/ProjectContentInfo.kt new file mode 100644 index 0000000000..c0493e6cb7 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/pojo/task/ProjectContentInfo.kt @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.pojo.task + +/** + * 项目备份配置 + */ +data class ProjectContentInfo( + val projectId: String? = null, + val projectRegex: String? = null, + val repoList: List? = null, + val repoRegex: String? = null, + val excludeProjects: List? = null, + val excludeRepos: List? = null, +) + diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/BackupDataHandler.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/BackupDataHandler.kt new file mode 100644 index 0000000000..1c8182add9 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/BackupDataHandler.kt @@ -0,0 +1,33 @@ +package com.tencent.bkrepo.job.backup.service + +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import org.springframework.data.mongodb.core.query.Criteria +import java.nio.file.Paths + +interface BackupDataHandler { + + fun dataType(): BackupDataEnum + + fun buildQueryCriteria(context: BackupContext): Criteria + + fun getCollectionName(backupDataEnum: BackupDataEnum, context: BackupContext): String { + return backupDataEnum.collectionName + } + + fun preBackupDataHandler(record: T, backupDataEnum: BackupDataEnum, context: BackupContext) {} + + fun postBackupDataHandler(context: BackupContext) {} + + fun returnLastId(data: T): String + + fun preRestoreDataHandler(backupDataEnum: BackupDataEnum, context: BackupContext) { + context.currentFile = Paths.get(context.currentPath.toString(), backupDataEnum.fileName).toString() + } + + fun storeRestoreDataHandler(record: T, backupDataEnum: BackupDataEnum, context: BackupContext) + + fun getSpecialDataEnum(context: BackupContext): List? { + return null + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/BackupRepoSpecialDataService.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/BackupRepoSpecialDataService.kt new file mode 100644 index 0000000000..a169773ada --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/BackupRepoSpecialDataService.kt @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.service + +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext + +/** + * 不同依赖源特有数据备份或者恢复 + */ +interface BackupRepoSpecialDataService { + /** + * 匹配仓库类型 + */ + fun type(): RepositoryType + + /** + * 额外兼容仓库类型 + * (现主要是因为使用oci替换docker，需要兼容) + */ + fun extraType(): RepositoryType? + + /** + * 查找仓库要恢复的特殊数据类型 + */ + fun getRepoSpecialDataEnum(context: BackupContext): List +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataBackupService.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataBackupService.kt new file mode 100644 index 0000000000..cb35c5d579 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataBackupService.kt @@ -0,0 +1,15 @@ +package com.tencent.bkrepo.job.backup.service + +import com.tencent.bkrepo.common.api.pojo.Page +import com.tencent.bkrepo.job.backup.pojo.task.BackupTask +import com.tencent.bkrepo.job.backup.pojo.task.BackupTaskRequest +import org.springframework.data.domain.PageRequest + +interface DataBackupService { + + fun createTask(taskRequest: BackupTaskRequest): String + + fun executeTask(taskId: String) + + fun findTasks(state: String?, pageRequest: PageRequest): Page +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataRecordsBackupService.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataRecordsBackupService.kt new file mode 100644 index 0000000000..26cf850c1f --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataRecordsBackupService.kt @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.service + +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext + +interface DataRecordsBackupService { + + fun projectDataBackup(context: BackupContext) + +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataRecordsRestoreService.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataRecordsRestoreService.kt new file mode 100644 index 0000000000..596bdba77e --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/DataRecordsRestoreService.kt @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.service + +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext + +interface DataRecordsRestoreService { + + fun projectDataRestore(context: BackupContext) + +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/BaseService.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/BaseService.kt new file mode 100644 index 0000000000..8198f05796 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/BaseService.kt @@ -0,0 +1,132 @@ +package com.tencent.bkrepo.job.backup.service.impl + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.exception.BadRequestException +import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.api.util.toJsonString +import com.tencent.bkrepo.common.storage.credentials.InnerCosCredentials +import com.tencent.bkrepo.common.storage.innercos.client.CosClient +import com.tencent.bkrepo.common.storage.message.StorageErrorException +import com.tencent.bkrepo.common.storage.message.StorageMessageCode +import com.tencent.bkrepo.common.storage.util.createFile +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.pojo.setting.BackupErrorStrategy +import org.slf4j.LoggerFactory +import java.io.BufferedOutputStream +import java.io.BufferedWriter +import java.io.File +import java.io.FileOutputStream +import java.io.FileWriter +import java.io.IOException +import java.io.InputStream +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths + + +open class BaseService { + + + fun freeSpaceCheck(context: BackupContext, usageThreshold: Double) { + val currentDirectory = File(context.task.storeLocation) + val freeSpace = currentDirectory.freeSpace + val totalSpace = currentDirectory.totalSpace + + val freeSpacePercentage = (freeSpace.toDouble() / totalSpace.toDouble()) + val usedSpacePercentage = 1 - freeSpacePercentage + if (usedSpacePercentage > usageThreshold) { + val msg = "disk usage reached limit $usageThreshold" + logger.error(msg) + throw BadRequestException(CommonMessageCode.PARAMETER_INVALID, msg) + } + } + + + // TODO 全存储在一个文件中，当数据过多会导致内容过大 + fun storeData(data: T, backupDataEnum: BackupDataEnum, context: BackupContext) { + val fileName = backupDataEnum.fileName + try { + val dir = if (context.currentProjectId.isNullOrEmpty()) { + StringPool.EMPTY + } else { + context.currentProjectId + StringPool.SLASH + context.currentRepoName + } + val filePath = buildPath(dir, fileName, context.targertPath) + touch(filePath) + val dataStr = data!!.toJsonString().replace(System.lineSeparator(), "") + appendToFile(dataStr, filePath.toString()) + logger.info("Success to append file [$fileName]") + } catch (exception: Exception) { + logger.error("Failed to create file", exception) + if (context.task.backupSetting.errorStrategy == BackupErrorStrategy.FAST_FAIL) { + throw StorageErrorException(StorageMessageCode.STORE_ERROR) + } + } + } + + fun onCreateClient(credentials: InnerCosCredentials): CosClient { + require(credentials.secretId.isNotBlank()) + require(credentials.secretKey.isNotBlank()) + require(credentials.region.isNotBlank()) + require(credentials.bucket.isNotBlank()) + return CosClient(credentials) + } + + fun deleteFolder(targetPath: Path) { + try { + // 最后需要删除目录 + deleteDirectory(targetPath) + } catch (e: Exception) { + logger.warn("delete temp folder error: ", e) + } + } + + fun buildPath(dir: String, fileName: String, root: Path): Path { + return Paths.get(root.toString(), dir, fileName) + } + + fun touch(filePath: Path): File { + filePath.createFile() + return filePath.toFile() + } + + @Throws(IOException::class) + fun appendToFile(content: String, filePath: String) { + BufferedWriter(FileWriter(filePath, true)).use { writer -> + writer.write(content) + writer.newLine() + } + } + + @Throws(IOException::class) + fun streamToFile(inputStream: InputStream, filePath: String) { + inputStream.use { inputStream -> + BufferedOutputStream(FileOutputStream(filePath)).use { outputStream -> + val buffer = ByteArray(DEFAULT_BUFFER_SIZE) + var bytesRead: Int + while (inputStream.read(buffer).also { bytesRead = it } != -1) { + outputStream.write(buffer, 0, bytesRead) + } + } + } + } + + fun exist(filePath: Path): Boolean { + return Files.isRegularFile(filePath) + } + + fun deleteDirectory(directory: Path) { + if (Files.exists(directory)) { + Files.walk(directory) + .sorted(Comparator.reverseOrder()) + .forEach { Files.delete(it) } + } + } + + companion object { + val logger = LoggerFactory.getLogger(BaseService::class.java) + const val ZIP_FILE_SUFFIX = ".zip" + const val FILE_STORE_FOLDER = "bk+store" + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataBackupServiceImpl.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataBackupServiceImpl.kt new file mode 100644 index 0000000000..d1b12b93b4 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataBackupServiceImpl.kt @@ -0,0 +1,123 @@ +package com.tencent.bkrepo.job.backup.service.impl + +import com.tencent.bkrepo.common.api.exception.BadRequestException +import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.api.pojo.Page +import com.tencent.bkrepo.common.mongo.util.Pages +import com.tencent.bkrepo.common.security.util.SecurityUtils +import com.tencent.bkrepo.job.DATA_RECORDS_BACKUP +import com.tencent.bkrepo.job.DATA_RECORDS_RESTORE +import com.tencent.bkrepo.job.backup.dao.BackupTaskDao +import com.tencent.bkrepo.job.backup.model.TBackupTask +import com.tencent.bkrepo.job.backup.pojo.BackupTaskState +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.pojo.task.BackupTask +import com.tencent.bkrepo.job.backup.pojo.task.BackupTask.Companion.toDto +import com.tencent.bkrepo.job.backup.pojo.task.BackupTaskRequest +import com.tencent.bkrepo.job.backup.service.DataBackupService +import com.tencent.bkrepo.job.backup.service.DataRecordsBackupService +import com.tencent.bkrepo.job.backup.service.DataRecordsRestoreService +import org.slf4j.LoggerFactory +import org.springframework.data.domain.PageRequest +import org.springframework.stereotype.Service +import java.io.FileNotFoundException +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.time.LocalDateTime + +@Service +class DataBackupServiceImpl( + private val backupTaskDao: BackupTaskDao, + private val dataRecordsBackupService: DataRecordsBackupService, + private val dataRecordsRestoreService: DataRecordsRestoreService +) : DataBackupService { + override fun createTask(taskRequest: BackupTaskRequest): String { + contentCheck(taskRequest) + val task = buildBackupTask(taskRequest) + return backupTaskDao.save(task).id!! + } + + override fun executeTask(taskId: String) { + val task = backupTaskDao.findTasksById(taskId) + ?: throw BadRequestException(CommonMessageCode.PARAMETER_INVALID, "taskId") + if (task.state != BackupTaskState.PENDING.name) + throw BadRequestException(CommonMessageCode.PARAMETER_INVALID, "state") + if (task.type == DATA_RECORDS_BACKUP) { + val context = BackupContext(task = task) + dataRecordsBackupService.projectDataBackup(context) + } else { + val context = BackupContext(task = task) + dataRecordsRestoreService.projectDataRestore(context) + } + } + + override fun findTasks(state: String?, pageRequest: PageRequest): Page { + val count = backupTaskDao.count(state) + val records = backupTaskDao.find(state, pageRequest).map { it.toDto() } + return Pages.ofResponse(pageRequest, count, records) + } + + private fun contentCheck(request: BackupTaskRequest) { + with(request) { + + if (content == null || content!!.projects.isNullOrEmpty()) { + logger.warn("backup content [$content] is illegal!") + throw BadRequestException(CommonMessageCode.PARAMETER_INVALID, BackupTaskRequest::content.name) + } + try { + val targetFile = Paths.get(storeLocation) + storeLocationHandler(request.type, targetFile) + if (!targetFile.toFile().exists()) throw FileNotFoundException(storeLocation) + } catch (e: Exception) { + logger.warn("backup store location [$storeLocation] is illegal!") + throw BadRequestException( + CommonMessageCode.PARAMETER_INVALID, + BackupTaskRequest::storeLocation.name + ) + } + } + } + + private fun storeLocationHandler(type: String, targetFile: Path) { + when (type) { + DATA_RECORDS_BACKUP -> { + try { + Files.createDirectories(targetFile) + } catch (e: Exception) { + logger.warn("backup store location [$targetFile] is illegal!") + throw BadRequestException( + CommonMessageCode.PARAMETER_INVALID, + BackupTaskRequest::storeLocation.name + ) + } + return + } + DATA_RECORDS_RESTORE -> { + return + } + else -> { + logger.warn("task type [$type] is illegal!") + throw BadRequestException(CommonMessageCode.PARAMETER_INVALID, BackupTaskRequest::type.name) + } + } + } + + private fun buildBackupTask(request: BackupTaskRequest): TBackupTask { + return TBackupTask( + name = request.name, + storeLocation = request.storeLocation, + type = request.type, + content = request.content, + createdBy = SecurityUtils.getUserId(), + createdDate = LocalDateTime.now(), + lastModifiedDate = LocalDateTime.now(), + lastModifiedBy = SecurityUtils.getUserId(), + backupSetting = request.backupSetting + ) + } + + companion object { + private val logger = LoggerFactory.getLogger(DataBackupServiceImpl::class.java) + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataRecordsBackupServiceImpl.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataRecordsBackupServiceImpl.kt new file mode 100644 index 0000000000..6294f52d97 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataRecordsBackupServiceImpl.kt @@ -0,0 +1,267 @@ +package com.tencent.bkrepo.job.backup.service.impl + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.util.EscapeUtils +import com.tencent.bkrepo.common.mongo.constant.ID +import com.tencent.bkrepo.common.mongo.constant.MIN_OBJECT_ID +import com.tencent.bkrepo.common.storage.message.StorageErrorException +import com.tencent.bkrepo.common.storage.message.StorageMessageCode +import com.tencent.bkrepo.job.BATCH_SIZE +import com.tencent.bkrepo.job.NAME +import com.tencent.bkrepo.job.PROJECT +import com.tencent.bkrepo.job.backup.config.DataBackupConfig +import com.tencent.bkrepo.job.backup.dao.BackupTaskDao +import com.tencent.bkrepo.job.backup.pojo.BackupTaskState +import com.tencent.bkrepo.job.backup.pojo.query.BackupRepositoryInfo +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum.Companion.PRIVATE_TYPE +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum.Companion.PUBLIC_TYPE +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.pojo.setting.BackupErrorStrategy +import com.tencent.bkrepo.job.backup.pojo.task.ProjectContentInfo +import com.tencent.bkrepo.job.backup.service.DataRecordsBackupService +import com.tencent.bkrepo.job.backup.service.impl.base.BackupDataMappings +import com.tencent.bkrepo.job.backup.util.ZipFileUtil +import org.bson.types.ObjectId +import org.slf4j.LoggerFactory +import org.springframework.data.domain.Sort +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Service +import java.io.File +import java.nio.file.Files +import java.nio.file.Paths +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter + +@Service +class DataRecordsBackupServiceImpl( + private val mongoTemplate: MongoTemplate, + private val backupTaskDao: BackupTaskDao, + private val dataBackupConfig: DataBackupConfig, +) : DataRecordsBackupService, BaseService() { + override fun projectDataBackup(context: BackupContext) { + with(context) { + logger.info("Start to run backup task ${context.task}.") + startDate = LocalDateTime.now() + backupTaskDao.updateState(taskId, BackupTaskState.RUNNING, startDate) + // 需要进行磁盘判断， 当达到磁盘容量大小的限定百分比时则停止 + freeSpaceCheck(context, dataBackupConfig.usageThreshold) + // TODO 需要进行仓库用量判断 + // TODO 异常需要捕获 + // TODO 历史备份保留周期，当超过该周期时需要进行删除 + if (task.content == null || task.content!!.projects.isNullOrEmpty()) return + init(context) + // 备份公共基础数据 + if (task.content!!.commonData) { + commonDataBackup(context) + } + // 备份业务数据 + customDataBackup(context) + // 最后进行压缩 + if (task.content!!.compression) { + // TODO 使用压缩需要关注对CPU的影响 + logger.info("start to compress file and upload to cos") + val zipFileName = buildZipFileName(context) + val zipFilePath = buildZipFilePath(context) + try { + ZipFileUtil.compressDirectory(targertPath.toString(), zipFilePath) + val zipFile = File(zipFilePath) + val cosClient = onCreateClient(dataBackupConfig.cos) + cosClient.putFileObject(zipFileName, zipFile) + } catch (e: Exception) { + logger.error("compress or upload zip file to cos error, $e") + throw StorageErrorException(StorageMessageCode.STORE_ERROR) + } + deleteFolder(targertPath) + } + backupTaskDao.updateState(taskId, BackupTaskState.FINISHED, endDate = LocalDateTime.now()) + logger.info("Backup task ${context.task} has been finished!") + } + } + + private fun init(context: BackupContext) { + val currentDateStr = context.startDate.format(DateTimeFormatter.ofPattern("yyyyMMdd.HHmmss")) + val path = Paths.get(context.task.storeLocation, context.task.name, currentDateStr) + if (!Files.exists(path)) { + Files.createDirectories(path) + } + context.targertPath = path + if (context.task.content?.increment != null) { + val date = if (context.task.content?.incrementDate != null) { + LocalDateTime.parse(context.task.content!!.incrementDate, DateTimeFormatter.ISO_DATE_TIME) + } else { + context.startDate.minusDays(1) + } + context.incrementDate = date + } + } + + private fun commonDataBackup(context: BackupContext) { + BackupDataEnum.getParentAndSpecialDataList(PUBLIC_TYPE).forEach { + logger.info("start to backup common data ${it.collectionName}.") + try { + queryResult(context, it) + } catch (e: Exception) { + logger.error("backup common data ${it.collectionName} error $e") + throw e + } + logger.info("common data ${it.collectionName} has been backed up!") + } + } + + private fun customDataBackup(context: BackupContext) { + with(context) { + for (projectFilterInfo in task.content!!.projects!!) { + if (!checkProjectParams(projectFilterInfo)) continue + val criteria = buildCriteria(projectFilterInfo) + queryResult(context, BackupDataEnum.REPOSITORY_DATA, criteria) + } + } + } + + private fun checkProjectParams(project: ProjectContentInfo?): Boolean { + return !(project != null && + project.projectRegex.isNullOrEmpty() && + project.projectId.isNullOrEmpty() && + project.excludeProjects.isNullOrEmpty()) + } + + private fun buildCriteria(project: ProjectContentInfo): Criteria { + return Criteria().andOperator(buildProjectCriteria(project), buildRepoCriteria(project)) + } + + private fun buildProjectCriteria(project: ProjectContentInfo): Criteria { + with(project) { + val criteria = Criteria() + if (projectId.isNullOrEmpty()) { + if (projectRegex.isNullOrEmpty()) { + criteria.and(PROJECT).nin(excludeProjects!!) + } else { + val escapeValue = EscapeUtils.escapeRegexExceptWildcard(projectRegex) + val regexPattern = escapeValue.replace("*", ".*") + criteria.and(PROJECT).regex("^$regexPattern") + } + } else { + criteria.and(PROJECT).isEqualTo(projectId) + } + return criteria + } + } + + private fun buildRepoCriteria(project: ProjectContentInfo): Criteria { + val criteria = Criteria() + if (project.repoList.isNullOrEmpty()) { + if (project.repoRegex.isNullOrEmpty()) { + if (!project.excludeRepos.isNullOrEmpty()) { + criteria.and(NAME).nin(project.excludeRepos) + } + } else { + val escapeValue = EscapeUtils.escapeRegexExceptWildcard(project.repoRegex) + val regexPattern = escapeValue.replace("*", ".*") + criteria.and(NAME).regex("^$regexPattern") + } + } else { + criteria.and(NAME).`in`(project.repoList) + } + return criteria + } + + private fun buildZipFileName(context: BackupContext): String { + val currentDateStr = context.startDate.format(DateTimeFormatter.ofPattern("yyyyMMdd.HHmmss")) + return context.task.name + StringPool.DASH + currentDateStr + ZIP_FILE_SUFFIX + } + + private fun buildZipFilePath(context: BackupContext): String { + val fileName = buildZipFileName(context) + return Paths.get(context.task.storeLocation, context.task.name, fileName).toString() + } + + private fun queryResult( + context: BackupContext, + backupDataEnum: BackupDataEnum, + customRepoCriteria: Criteria? = null + ) { + val pageSize = BATCH_SIZE + var querySize: Int + var lastId = ObjectId(MIN_OBJECT_ID) + val criteria = customRepoCriteria ?: BackupDataMappings.buildQueryCriteria(backupDataEnum, context) + val collectionName = BackupDataMappings.getCollectionName(backupDataEnum, context) + do { + val query = Query(criteria) + .addCriteria(Criteria.where(ID).gt(lastId)) + .limit(BATCH_SIZE) + .with(Sort.by(ID).ascending()) + val datas = mongoTemplate.find(query, backupDataEnum.backupClazz, collectionName) + if (datas.isEmpty()) { + break + } + if (customRepoCriteria == null) { + processData(datas, backupDataEnum, context) + } else { + processCustomRepoData(datas as List, context) + } + querySize = datas.size + lastId = ObjectId(BackupDataMappings.returnLastId(datas.last(), backupDataEnum)) + } while (querySize == pageSize) + } + + private fun processCustomRepoData(records: List, context: BackupContext) { + records.forEach { record -> + context.currentRepositoryType = record.type + context.currentProjectId = record.projectId + context.currentRepoName = record.name + BackupDataEnum.getParentAndSpecialDataList(PRIVATE_TYPE).forEach { + logger.info( + "start to backup custom data ${it.collectionName} " + + "for repo ${record.projectId}|${record.name}." + ) + try { + queryResult(context, it) + } catch (e: Exception) { + logger.error( + "backup custom data ${it.collectionName} " + + "error for repo ${record.projectId}|${record.name} $e" + ) + throw e + } + logger.info( + "custom data ${it.collectionName} of " + + "repo ${record.projectId}|${record.name} has been backed up!" + ) + } + } + } + + private fun processData(data: List, backupDataEnum: BackupDataEnum, context: BackupContext) { + data.forEach { record -> + try { + BackupDataMappings.preBackupDataHandler(record, backupDataEnum, context) + storeData(record, backupDataEnum, context) + BackupDataMappings.postBackupDataHandler(backupDataEnum, context) + if (!backupDataEnum.relatedData.isNullOrEmpty()) { + val relatedDataEnum = BackupDataEnum.getByCollectionName(backupDataEnum.relatedData) + queryResult(context, relatedDataEnum) + } + val specialData = BackupDataMappings.getSpecialDataEnum(backupDataEnum, context) + specialData?.forEach { + queryResult(context, it) + } + } catch (e: Exception) { + logger.error( + "Failed to process record $record with " + + "data of ${backupDataEnum.collectionName}, error is ${e.message}" + ) + if (context.task.backupSetting.errorStrategy == BackupErrorStrategy.FAST_FAIL) { + throw StorageErrorException(StorageMessageCode.STORE_ERROR) + } + } + } + } + + companion object { + private val logger = LoggerFactory.getLogger(DataRecordsBackupServiceImpl::class.java) + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataRecordsRestoreServiceImpl.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataRecordsRestoreServiceImpl.kt new file mode 100644 index 0000000000..74ff5b161c --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/DataRecordsRestoreServiceImpl.kt @@ -0,0 +1,217 @@ +package com.tencent.bkrepo.job.backup.service.impl + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.exception.BadRequestException +import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.api.util.JsonUtils +import com.tencent.bkrepo.job.backup.config.DataBackupConfig +import com.tencent.bkrepo.job.backup.dao.BackupTaskDao +import com.tencent.bkrepo.job.backup.pojo.BackupTaskState +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum.Companion.PRIVATE_TYPE +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum.Companion.PUBLIC_TYPE +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.pojo.task.BackupContent +import com.tencent.bkrepo.job.backup.pojo.task.BackupTaskRequest +import com.tencent.bkrepo.job.backup.pojo.task.ProjectContentInfo +import com.tencent.bkrepo.job.backup.service.DataRecordsRestoreService +import com.tencent.bkrepo.job.backup.service.impl.base.BackupDataMappings +import com.tencent.bkrepo.job.backup.util.ZipFileUtil +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Service +import java.io.File +import java.io.FileNotFoundException +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter +import kotlin.io.path.name +import kotlin.streams.toList + +@Service +class DataRecordsRestoreServiceImpl( + private val backupTaskDao: BackupTaskDao, + private val dataBackupConfig: DataBackupConfig, +) : DataRecordsRestoreService, BaseService() { + override fun projectDataRestore(context: BackupContext) { + with(context) { + logger.info("Start to run restore task ${context.task}.") + startDate = LocalDateTime.now() + backupTaskDao.updateState(taskId, BackupTaskState.RUNNING, startDate) + // TODO 需要进行磁盘判断 + // TODO 异常需要捕获 + if (task.content == null) return + preProcessFile(context) + // 恢复公共基础数据 + if (task.content!!.commonData) { + context.currentPath = context.targertPath + commonDataRestore(context) + } + // 备份业务数据 + findSecondLevelDirectories(context.targertPath).forEach { + if (filterFolder(it, task.content)) { + context.currentPath = it + customDataRestore(context) + } + } + if (task.content!!.compression) { + deleteFolder(targertPath) + } + backupTaskDao.updateState(taskId, BackupTaskState.FINISHED, endDate = LocalDateTime.now()) + logger.info("Restore task ${context.task} has been finished!") + } + } + + private fun preProcessFile(context: BackupContext) { + with(context) { + val path = Paths.get(context.task.storeLocation) + if (!Files.exists(path)) { + throw FileNotFoundException(context.task.storeLocation) + } + val targetFolder = if (task.content!!.compression) { + freeSpaceCheck(context, dataBackupConfig.usageThreshold) + try { + decompressZipFile(path, context) + } catch (e: Exception) { + logger.error("decompress zip file error: ${e.message}") + throw BadRequestException( + CommonMessageCode.PARAMETER_INVALID, + BackupTaskRequest::storeLocation.name + ) + } + } else { + path + } + logger.info("restore root folder is $targetFolder") + context.targertPath = targetFolder + } + } + + private fun decompressZipFile(sourcePath: Path, context: BackupContext): Path { + val currentDateStr = context.startDate.format(DateTimeFormatter.ofPattern("yyyyMMdd.HHmmss")) + val tempFolder = sourcePath.name.removeSuffix(ZIP_FILE_SUFFIX) + StringPool.DASH + currentDateStr + val unZipTempFolder = Paths.get(sourcePath.parent.toString(), tempFolder) + Files.createDirectories(unZipTempFolder) + ZipFileUtil.decompressFile(context.task.storeLocation, unZipTempFolder.toString()) + if (!Files.exists(unZipTempFolder)) { + throw FileNotFoundException(unZipTempFolder.toString()) + } + val subdirectories = Files.list(unZipTempFolder) + .filter { Files.isDirectory(it) } + .toList() + return subdirectories.firstOrNull() ?: throw FileNotFoundException(unZipTempFolder.toString()) + } + + fun findSecondLevelDirectories(directory: Path): List { + val secondLevelDirectories = mutableListOf() + + Files.walk(directory, 2) + .filter { + Files.isDirectory(it) && + it != directory && + it.parent.name != BaseService.FILE_STORE_FOLDER + && it.parent != directory + } + .forEach { secondLevelDirectories.add(it) } + + return secondLevelDirectories + } + + + private fun filterFolder(path: Path, content: BackupContent?): Boolean { + if (content == null) return false + val currentFolderName = path.name + val parentFolderName = path.parent.name + content.projects?.forEach { + if (projectFilter(it, parentFolderName) && repoFilter(it, currentFolderName)) return true + } + return false + } + + private fun projectFilter(project: ProjectContentInfo, projectFolder: String): Boolean { + var projectMatch = false + if (project.projectId.isNullOrEmpty()) { + if (project.projectRegex.isNullOrEmpty()) { + projectMatch = project.excludeProjects?.contains(projectFolder) != true + } else { + val regex = Regex(project.projectRegex.replace("*", ".*")) + projectMatch = regex.matches(projectFolder) + } + } else { + projectMatch = projectFolder == project.projectId + } + return projectMatch + } + + private fun repoFilter(project: ProjectContentInfo, repoFolder: String): Boolean { + var repoMatch = false + if (project.repoList.isNullOrEmpty()) { + if (project.repoRegex.isNullOrEmpty()) { + if (!project.excludeRepos.isNullOrEmpty()) { + repoMatch = project.excludeRepos.contains(repoFolder) != true + } else { + repoMatch = true + } + } else { + val regex = Regex(project.repoRegex.replace("*", ".*")) + repoMatch = regex.matches(repoFolder) + } + } else { + repoMatch = project.repoList.contains(repoFolder) + } + return repoMatch + } + + private fun commonDataRestore(context: BackupContext) { + BackupDataEnum.getNonSpecialDataList(PUBLIC_TYPE).forEach { + logger.info("start to restore common data ${it.collectionName}!") + try { + processFiles(context, it) + } catch (e: Exception) { + logger.error("restore common data ${it.collectionName} error $e") + } + logger.info("common data ${it.collectionName} has been restored!") + } + } + + private fun customDataRestore(context: BackupContext) { + BackupDataEnum.getNonSpecialDataList(PRIVATE_TYPE).forEach { + logger.info("start to restore custom data ${it.collectionName} with folder ${context.currentPath}!") + try { + processFiles(context, it) + } catch (e: Exception) { + logger.error("restore custom data ${it.collectionName} with folder ${context.currentPath} error $e") + throw e + } + logger.info("custom data ${it.collectionName} has been restored with folder ${context.currentPath}!") + } + } + + private fun processFiles(context: BackupContext, backupDataEnum: BackupDataEnum) { + BackupDataMappings.preRestoreDataHandler(backupDataEnum, context) + loadAndStoreRecord(backupDataEnum, context) + val specialData = BackupDataMappings.getSpecialDataEnum(backupDataEnum, context) ?: return + specialData.forEach { + loadAndStoreRecord(it, context) + } + } + + private fun loadAndStoreRecord(backupDataEnum: BackupDataEnum, context: BackupContext) { + with(context) { + if (!Files.exists(Paths.get(currentFile))) { + logger.warn("$currentFile not exist!") + return + } + val file = File(currentFile) + file.forEachLine { line -> + val record = JsonUtils.objectMapper.readValue(line, backupDataEnum.backupClazz) + BackupDataMappings.storeRestoreDataHandler(record, backupDataEnum, context) + } + } + } + + companion object { + private val logger = LoggerFactory.getLogger(DataRecordsRestoreServiceImpl::class.java) + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupAccountDataHandler.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupAccountDataHandler.kt new file mode 100644 index 0000000000..2d13a7df70 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupAccountDataHandler.kt @@ -0,0 +1,128 @@ +package com.tencent.bkrepo.job.backup.service.impl.base + +import com.tencent.bkrepo.auth.pojo.enums.ResourceType +import com.tencent.bkrepo.auth.pojo.oauth.AuthorizationGrantType +import com.tencent.bkrepo.job.backup.pojo.query.common.BackupAccount +import com.tencent.bkrepo.job.backup.pojo.query.common.BackupCredentialSet +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.pojo.setting.BackupConflictStrategy +import com.tencent.bkrepo.job.backup.service.BackupDataHandler +import org.slf4j.LoggerFactory +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.stereotype.Component + +@Component +class BackupAccountDataHandler( + private val mongoTemplate: MongoTemplate, +) : BackupDataHandler { + override fun dataType(): BackupDataEnum { + return BackupDataEnum.ACCOUNT_DATA + } + + override fun buildQueryCriteria(context: BackupContext): Criteria { + val criteria = Criteria() + if (context.incrementDate != null) { + criteria.and(BackupAccount::lastModifiedDate.name).gte(context.incrementDate) + } + return criteria + } + + override fun returnLastId(data: T): String { + return (data as BackupAccount).id!! + } + + override fun storeRestoreDataHandler(record: T, backupDataEnum: BackupDataEnum, context: BackupContext) { + val record = record as BackupAccount + val existRecord = findExistAccount(record) + if (existRecord != null) { + if (context.task.backupSetting.conflictStrategy == BackupConflictStrategy.SKIP) { + return + } + updateExistAccount(record, existRecord) + } else { + mongoTemplate.save(record, BackupDataEnum.ACCOUNT_DATA.collectionName) + logger.info("Create account ${record.appId} success!") + } + } + + private fun findExistAccount(record: BackupAccount): BackupAccount? { + val existAccountQuery = buildQuery(record) + return mongoTemplate.findOne( + existAccountQuery, + BackupAccount::class.java, + BackupDataEnum.ACCOUNT_DATA.collectionName + ) + } + + private fun updateExistAccount(accountInfo: BackupAccount, existRecord: BackupAccount) { + val accountQuery = buildQuery(accountInfo) + val mergedCredentials = mergeCredentialSet(accountInfo.credentials, existRecord.credentials) + val mergedAuthorizationGrantTypes = mergeAuthorizationGrantType( + accountInfo.authorizationGrantTypes, existRecord.authorizationGrantTypes + ) + val mergedScope = mergeResourceType(accountInfo.scope, existRecord.scope) + + val update = Update() + .set(BackupAccount::credentials.name, mergedCredentials) + .set(BackupAccount::authorizationGrantTypes.name, mergedAuthorizationGrantTypes) + .set(BackupAccount::scope.name, mergedScope) + .set(BackupAccount::homepageUrl.name, accountInfo.homepageUrl) + .set(BackupAccount::redirectUri.name, accountInfo.redirectUri) + .set(BackupAccount::avatarUrl.name, accountInfo.avatarUrl) + .set(BackupAccount::locked.name, accountInfo.locked) + .set(BackupAccount::description.name, accountInfo.description) + accountInfo.scopeDesc?.let { + update.addToSet(BackupAccount::scopeDesc.name, accountInfo.scopeDesc) + } + mongoTemplate.updateFirst(accountQuery, update, BackupDataEnum.ACCOUNT_DATA.collectionName) + logger.info("update exist account success with id ${accountInfo.appId}") + } + + private fun buildQuery(accountInfo: BackupAccount): Query { + return Query.query(Criteria.where(BackupAccount::appId.name).`is`(accountInfo.appId)) + } + + fun mergeCredentialSet( + oldData: List, + newData: List + ): MutableList { + val result = mutableListOf() + result.addAll(oldData) + newData.forEach { credentialSet -> + val existed = result.firstOrNull { it.accessKey == credentialSet.accessKey } + if (existed == null) { + result.add(credentialSet) + } + } + return result + } + + fun mergeAuthorizationGrantType( + oldData: Set?, + newData: Set? + ): MutableSet { + val result = mutableSetOf() + oldData?.let { result.addAll(oldData) } + newData?.let { result.addAll(newData) } + return result + } + + fun mergeResourceType( + oldData: Set?, + newData: Set? + ): MutableSet { + val result = mutableSetOf() + oldData?.let { result.addAll(oldData) } + newData?.let { result.addAll(newData) } + return result + } + + + companion object { + private val logger = LoggerFactory.getLogger(BackupAccountDataHandler::class.java) + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupConanMetadataDataHandler.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupConanMetadataDataHandler.kt new file mode 100644 index 0000000000..77305f9d92 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupConanMetadataDataHandler.kt @@ -0,0 +1,65 @@ +package com.tencent.bkrepo.job.backup.service.impl.base + +import com.tencent.bkrepo.common.artifact.constant.REPO_NAME +import com.tencent.bkrepo.job.PROJECT +import com.tencent.bkrepo.job.backup.pojo.query.common.BackupConanMetadata +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.service.BackupDataHandler +import org.slf4j.LoggerFactory +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Component + +@Component +class BackupConanMetadataDataHandler( + private val mongoTemplate: MongoTemplate, +) : BackupDataHandler { + override fun dataType(): BackupDataEnum { + return BackupDataEnum.CONAN_METADATA_DATA + } + + override fun buildQueryCriteria(context: BackupContext): Criteria { + return Criteria.where(PROJECT).isEqualTo(context.currentProjectId) + .and(REPO_NAME).isEqualTo(context.currentRepoName) + } + + override fun returnLastId(data: T): String { + return (data as BackupConanMetadata).id!! + } + + override fun storeRestoreDataHandler(record: T, backupDataEnum: BackupDataEnum, context: BackupContext) { + val record = record as BackupConanMetadata + val existRecord = findExistConanMetadata(record) + if (existRecord == null) { + mongoTemplate.save(record, BackupDataEnum.CONAN_METADATA_DATA.collectionName) + logger.info("Create metadata in ${record.projectId}|${record.repoName} success!") + } + } + + private fun findExistConanMetadata(record: BackupConanMetadata): BackupConanMetadata? { + val existMetadataQuery = buildQuery(record) + return mongoTemplate.findOne( + existMetadataQuery, + BackupConanMetadata::class.java, + BackupDataEnum.CONAN_METADATA_DATA.collectionName + ) + } + + private fun buildQuery(conanMetadata: BackupConanMetadata): Query { + return Query( + Criteria.where(BackupConanMetadata::projectId.name).isEqualTo(conanMetadata.projectId) + .and(BackupConanMetadata::repoName.name).isEqualTo(conanMetadata.repoName) + .and(BackupConanMetadata::user.name).isEqualTo(conanMetadata.user) + .and(BackupConanMetadata::name.name).isEqualTo(conanMetadata.name) + .and(BackupConanMetadata::version.name).isEqualTo(conanMetadata.version) + .and(BackupConanMetadata::channel.name).isEqualTo(conanMetadata.channel) + ) + } + + companion object { + private val logger = LoggerFactory.getLogger(BackupConanMetadataDataHandler::class.java) + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupDataMappings.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupDataMappings.kt new file mode 100644 index 0000000000..afa88c67cb --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupDataMappings.kt @@ -0,0 +1,99 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.service.impl.base + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.service.util.SpringContextUtils +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.service.BackupDataHandler +import org.springframework.data.mongodb.core.query.Criteria + +object BackupDataMappings { + + private val mappers = mutableMapOf() + + init { + addMapper(SpringContextUtils.getBean(BackupUserDataHandler::class.java)) + addMapper(SpringContextUtils.getBean(BackupAccountDataHandler::class.java)) + addMapper(SpringContextUtils.getBean(BackupRoleDataHandler::class.java)) + addMapper(SpringContextUtils.getBean(BackupTemporaryTokenDataHandler::class.java)) + addMapper(SpringContextUtils.getBean(BackupPermissionDataHandler::class.java)) + addMapper(SpringContextUtils.getBean(BackupProjectDataHandler::class.java)) + addMapper(SpringContextUtils.getBean(BackupRepositoryDataHandler::class.java)) + addMapper(SpringContextUtils.getBean(BackupPackageDataHandler::class.java)) + addMapper(SpringContextUtils.getBean(BackupPackageVersionDataHandler::class.java)) + addMapper(SpringContextUtils.getBean(BackupNodeDataHandler::class.java)) + addMapper(SpringContextUtils.getBean(BackupMavenMetadataDataHandler::class.java)) + addMapper(SpringContextUtils.getBean(BackupConanMetadataDataHandler::class.java)) + } + + private fun addMapper(mapper: BackupDataHandler) { + mappers[mapper.dataType()] = mapper + } + + fun buildQueryCriteria(backupDataEnum: BackupDataEnum, context: BackupContext): Criteria { + val mapper = mappers[backupDataEnum] ?: return Criteria() + return mapper.buildQueryCriteria(context) + } + + fun getCollectionName(backupDataEnum: BackupDataEnum, context: BackupContext): String { + val mapper = mappers[backupDataEnum] ?: return StringPool.EMPTY + return mapper.getCollectionName(backupDataEnum, context) + } + + fun preBackupDataHandler(record: T, backupDataEnum: BackupDataEnum, context: BackupContext) { + val mapper = mappers[backupDataEnum] ?: return + mapper.preBackupDataHandler(record, backupDataEnum, context) + } + + fun postBackupDataHandler(backupDataEnum: BackupDataEnum, context: BackupContext) { + val mapper = mappers[backupDataEnum] ?: return + mapper.postBackupDataHandler(context) + } + + fun returnLastId(data: T, backupDataEnum: BackupDataEnum): String { + val mapper = mappers[backupDataEnum] ?: return StringPool.EMPTY + return mapper.returnLastId(data) + } + + fun preRestoreDataHandler(backupDataEnum: BackupDataEnum, context: BackupContext) { + val mapper = mappers[backupDataEnum] ?: return + mapper.preRestoreDataHandler(backupDataEnum, context) + } + + fun storeRestoreDataHandler(record: T, backupDataEnum: BackupDataEnum, context: BackupContext) { + val mapper = mappers[backupDataEnum] ?: return + mapper.storeRestoreDataHandler(record, backupDataEnum, context) + } + + fun getSpecialDataEnum(backupDataEnum: BackupDataEnum, context: BackupContext): List? { + val mapper = mappers[backupDataEnum] ?: return null + return mapper.getSpecialDataEnum(context) + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupMavenMetadataDataHandler.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupMavenMetadataDataHandler.kt new file mode 100644 index 0000000000..1ffb157d6a --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupMavenMetadataDataHandler.kt @@ -0,0 +1,88 @@ +package com.tencent.bkrepo.job.backup.service.impl.base + +import com.tencent.bkrepo.common.artifact.constant.REPO_NAME +import com.tencent.bkrepo.job.PROJECT +import com.tencent.bkrepo.job.backup.pojo.query.BackupMavenMetadata +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.pojo.setting.BackupConflictStrategy +import com.tencent.bkrepo.job.backup.service.BackupDataHandler +import org.slf4j.LoggerFactory +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Component + +@Component +class BackupMavenMetadataDataHandler( + private val mongoTemplate: MongoTemplate, +) : BackupDataHandler { + override fun dataType(): BackupDataEnum { + return BackupDataEnum.MAVEN_METADATA_DATA + } + + override fun buildQueryCriteria(context: BackupContext): Criteria { + return Criteria.where(PROJECT).isEqualTo(context.currentProjectId) + .and(REPO_NAME).isEqualTo(context.currentRepoName) + } + + override fun returnLastId(data: T): String { + return (data as BackupMavenMetadata).id!! + } + + override fun storeRestoreDataHandler(record: T, backupDataEnum: BackupDataEnum, context: BackupContext) { + val record = record as BackupMavenMetadata + val existRecord = findExistMavenMetadata(record) + if (existRecord != null) { + if (context.task.backupSetting.conflictStrategy == BackupConflictStrategy.SKIP) { + return + } + updateExistMavenMetadata(record) + } else { + mongoTemplate.save(record, BackupDataEnum.MAVEN_METADATA_DATA.collectionName) + logger.info("Create metadata in ${record.projectId}|${record.repoName} success!") + } + } + + private fun findExistMavenMetadata(record: BackupMavenMetadata): BackupMavenMetadata? { + val existMetadataQuery = buildQuery(record) + return mongoTemplate.findOne( + existMetadataQuery, + BackupMavenMetadata::class.java, + BackupDataEnum.MAVEN_METADATA_DATA.collectionName + ) + } + + private fun updateExistMavenMetadata(mavenMetadata: BackupMavenMetadata) { + // TODO 记录更新时需要对比时间，保留最新的记录 + val metadataQuery = buildQuery(mavenMetadata) + val update = Update() + .set(BackupMavenMetadata::timestamp.name, mavenMetadata.timestamp) + .set(BackupMavenMetadata::buildNo.name, mavenMetadata.buildNo) + + mongoTemplate.updateFirst( + metadataQuery, + update, + BackupDataEnum.MAVEN_METADATA_DATA.collectionName + ) + logger.info("update exist metadata $mavenMetadata success") + } + + private fun buildQuery(mavenMetadata: BackupMavenMetadata): Query { + return Query( + Criteria.where(BackupMavenMetadata::projectId.name).isEqualTo(mavenMetadata.projectId) + .and(BackupMavenMetadata::repoName.name).isEqualTo(mavenMetadata.repoName) + .and(BackupMavenMetadata::groupId.name).isEqualTo(mavenMetadata.groupId) + .and(BackupMavenMetadata::artifactId.name).isEqualTo(mavenMetadata.artifactId) + .and(BackupMavenMetadata::version.name).isEqualTo(mavenMetadata.version) + .and(BackupMavenMetadata::classifier.name).isEqualTo(mavenMetadata.classifier) + .and(BackupMavenMetadata::extension.name).isEqualTo(mavenMetadata.extension) + ) + } + + companion object { + private val logger = LoggerFactory.getLogger(BackupMavenMetadataDataHandler::class.java) + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupNodeDataHandler.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupNodeDataHandler.kt new file mode 100644 index 0000000000..09a90009b7 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupNodeDataHandler.kt @@ -0,0 +1,303 @@ +package com.tencent.bkrepo.job.backup.service.impl.base + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.artifact.api.toArtifactFile +import com.tencent.bkrepo.common.artifact.constant.REPO_NAME +import com.tencent.bkrepo.common.artifact.manager.StorageManager +import com.tencent.bkrepo.common.mongo.constant.ID +import com.tencent.bkrepo.common.storage.core.StorageService +import com.tencent.bkrepo.common.storage.core.locator.HashFileLocator +import com.tencent.bkrepo.common.storage.message.StorageErrorException +import com.tencent.bkrepo.common.storage.message.StorageMessageCode +import com.tencent.bkrepo.job.PROJECT +import com.tencent.bkrepo.job.backup.pojo.query.BackupFileReferenceInfo +import com.tencent.bkrepo.job.backup.pojo.query.BackupNodeInfo +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.pojo.setting.BackupConflictStrategy +import com.tencent.bkrepo.job.backup.service.BackupDataHandler +import com.tencent.bkrepo.job.backup.service.impl.BaseService +import com.tencent.bkrepo.job.batch.utils.RepositoryCommonUtils +import com.tencent.bkrepo.job.separation.pojo.query.NodeDetailInfo +import com.tencent.bkrepo.job.separation.util.SeparationUtils +import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import com.tencent.bkrepo.repository.pojo.node.NodeInfo +import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail +import org.slf4j.LoggerFactory +import org.springframework.dao.DuplicateKeyException +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Component +import java.nio.file.Path +import java.nio.file.Paths +import java.time.format.DateTimeFormatter + +@Component +class BackupNodeDataHandler( + private val storageManager: StorageManager, + private val mongoTemplate: MongoTemplate, + private val storageService: StorageService, +) : BackupDataHandler, BaseService() { + override fun dataType(): BackupDataEnum { + return BackupDataEnum.NODE_DATA + } + + override fun buildQueryCriteria(context: BackupContext): Criteria { + val criteria = Criteria.where(PROJECT).isEqualTo(context.currentProjectId) + .and(REPO_NAME).isEqualTo(context.currentRepoName) + if (context.incrementDate != null) { + criteria.and(BackupNodeInfo::lastModifiedDate.name).gte(context.incrementDate) + } + return criteria + } + + override fun getCollectionName(backupDataEnum: BackupDataEnum, context: BackupContext): String { + if (context.currentProjectId == null) return StringPool.EMPTY + return SeparationUtils.getNodeCollectionName(context.currentProjectId!!) + } + + override fun preBackupDataHandler(record: T, backupDataEnum: BackupDataEnum, context: BackupContext) { + val node = record as BackupNodeInfo + context.currentNode = node + storeRealFile(context) + } + + override fun returnLastId(data: T): String { + return (data as BackupNodeInfo).id!! + } + + override fun storeRestoreDataHandler(record: T, backupDataEnum: BackupDataEnum, context: BackupContext) { + val record = record as BackupNodeInfo? + val collectionName = SeparationUtils.getNodeCollectionName(record!!.projectId) + val repo = RepositoryCommonUtils.getRepositoryDetail(record.projectId, record.repoName) + uploadFile(record, context, repo) + var existRecord = findExistNode(record) + if (existRecord != null) { + if (context.task.backupSetting.conflictStrategy == BackupConflictStrategy.SKIP) { + return + } + try { + updateExistNode(record, collectionName) + } catch (e: DuplicateKeyException) { + updateDuplicateNode(record, collectionName) + if (sha256Check(record.folder, record.sha256)) { + increment(record.sha256!!, repo.storageCredentials?.key) + increment(existRecord.sha256!!, repo.storageCredentials?.key, -1) + } + } + } else { + try { + mongoTemplate.save(record, collectionName) + if (sha256Check(record.folder, record.sha256)) { + increment(record.sha256!!, repo.storageCredentials?.key) + } + logger.info("Create node ${record.fullPath} in ${record.projectId}|${record.repoName} success!") + } catch (exception: DuplicateKeyException) { + if (context.task.backupSetting.conflictStrategy == BackupConflictStrategy.SKIP) { + return + } + existRecord = findExistNode(record, false) + // 可能存在已经上传的节点记录不在备份数据里 + updateDuplicateNode(record, collectionName) + if (sha256Check(record.folder, record.sha256)) { + increment(record.sha256!!, repo.storageCredentials?.key) + increment(existRecord!!.sha256!!, repo.storageCredentials?.key, -1) + } + } + } + } + + private fun storeRealFile(context: BackupContext) { + with(context) { + if (currentNode!!.folder || currentNode!!.sha256 == FAKE_SHA256 + || currentNode!!.sha256.isNullOrEmpty()) return + val nodeDetail = convertToDetail(currentNode) + val dir = FILE_STORE_FOLDER + generateRandomPath(currentNode!!.sha256!!) + val filePath = buildPath(dir, currentNode!!.sha256!!, context.targertPath) + if (exist(filePath)) { + logger.info("real file already exist [${currentNode!!.sha256}]") + return + } + // 存储异常如何处理 + storageManager.loadFullArtifactInputStream(nodeDetail, currentStorageCredentials)?.use { + try { + touch(filePath) + streamToFile(it, filePath.toString()) + logger.info("Success to store real file [${currentNode!!.sha256}]") + } catch (exception: Exception) { + logger.error("Failed to store real file", exception) + throw StorageErrorException(StorageMessageCode.STORE_ERROR) + } + } ?: { + logger.error("File of node $nodeDetail not exist!") + throw StorageErrorException(StorageMessageCode.STORE_ERROR) + } + } + } + + private fun generateRandomPath(sha256: String): String { + val fileLocator = HashFileLocator() + return fileLocator.locate(sha256) + } + + private fun convertToDetail(tNode: BackupNodeInfo?): NodeDetail? { + return convert(tNode)?.let { NodeDetail(it) } + } + + private fun convert(tNode: BackupNodeInfo?): NodeInfo? { + return tNode?.let { + val metadata = toMap(it.metadata) + NodeInfo( + id = it.id, + createdBy = it.createdBy, + createdDate = it.createdDate.format(DateTimeFormatter.ISO_DATE_TIME), + lastModifiedBy = it.lastModifiedBy, + lastModifiedDate = it.lastModifiedDate.format(DateTimeFormatter.ISO_DATE_TIME), + projectId = it.projectId, + repoName = it.repoName, + folder = it.folder, + path = it.path, + name = it.name, + fullPath = it.fullPath, + size = if (it.size < 0L) 0L else it.size, + nodeNum = it.nodeNum?.let { nodeNum -> + if (nodeNum < 0L) 0L else nodeNum + }, + sha256 = it.sha256, + md5 = it.md5, + metadata = metadata, + nodeMetadata = it.metadata, + copyFromCredentialsKey = it.copyFromCredentialsKey, + copyIntoCredentialsKey = it.copyIntoCredentialsKey, + deleted = it.deleted?.format(DateTimeFormatter.ISO_DATE_TIME), + lastAccessDate = it.lastAccessDate?.format(DateTimeFormatter.ISO_DATE_TIME), + clusterNames = it.clusterNames, + archived = it.archived, + compressed = it.compressed, + ) + } + } + + private fun toMap(metadataList: List?): Map { + return metadataList?.associate { it.key to it.value }.orEmpty() + } + + private fun findExistNode(record: BackupNodeInfo, findById: Boolean = true): BackupNodeInfo? { + val existNodeQuery = if (findById) { + buildIdQuery(record) + } else { + buildQuery(record) + } + val collectionName = SeparationUtils.getNodeCollectionName(record.projectId) + return mongoTemplate.findOne(existNodeQuery, BackupNodeInfo::class.java, collectionName) + } + + private fun updateExistNode(nodeInfo: BackupNodeInfo, nodeCollectionName: String) { + val nodeQuery = Query(Criteria.where(ID).isEqualTo(nodeInfo.id)) + // 逻辑删除， 同时删除索引 + val update = Update() + .set(NodeDetailInfo::lastModifiedBy.name, nodeInfo.lastModifiedBy) + .set(NodeDetailInfo::lastModifiedDate.name, nodeInfo.lastModifiedDate) + .set(NodeDetailInfo::lastAccessDate.name, nodeInfo.lastAccessDate) + .set(NodeDetailInfo::deleted.name, nodeInfo.deleted) + .set(NodeDetailInfo::expireDate.name, nodeInfo.expireDate) + .set(NodeDetailInfo::copyFromCredentialsKey.name, nodeInfo.copyFromCredentialsKey) + .set(NodeDetailInfo::copyIntoCredentialsKey.name, nodeInfo.copyIntoCredentialsKey) + .set(NodeDetailInfo::metadata.name, nodeInfo.metadata) + .set(NodeDetailInfo::archived.name, nodeInfo.archived) + .set(NodeDetailInfo::compressed.name, nodeInfo.compressed) + + mongoTemplate.updateFirst(nodeQuery, update, nodeCollectionName) + logger.info( + "update exist node success with id ${nodeInfo.id} " + + "and fullPath ${nodeInfo.fullPath} in ${nodeInfo.projectId}|${nodeInfo.repoName}" + ) + } + + + fun uploadFile(record: BackupNodeInfo, context: BackupContext, repo: RepositoryDetail) { + if (!sha256Check(record.folder, record.sha256)) return + val filePath = generateRandomPath(context.targertPath, record.sha256!!) + val artifactFile = filePath.toFile().toArtifactFile() + // TODO 增加重试以及异常捕获 + storageService.store(record.sha256!!, artifactFile, repo.storageCredentials) + } + + fun sha256Check(folder: Boolean, sha256: String?): Boolean { + // 增加对应cold node 的文件引用 + if (folder || sha256.isNullOrBlank() || sha256 == FAKE_SHA256) { + return false + } + return true + } + + fun increment(sha256: String, credentialsKey: String?, inc: Long = 1) { + val collectionName = SeparationUtils.getFileReferenceCollectionName(sha256) + val criteria = Criteria.where(BackupFileReferenceInfo::sha256.name).`is`(sha256) + .and(BackupFileReferenceInfo::credentialsKey.name).`is`(credentialsKey) + val query = Query(criteria) + val update = Update().inc(BackupFileReferenceInfo::count.name, inc) + try { + mongoTemplate.upsert(query, update, collectionName) + } catch (exception: DuplicateKeyException) { + // retry because upsert operation is not atomic + mongoTemplate.upsert(query, update, collectionName) + } + logger.info("Increment node reference of file [$sha256] on credentialsKey [$credentialsKey].") + } + + fun updateDuplicateNode(record: BackupNodeInfo, collectionName: String) { + val existNodeQuery = buildQuery(record) + val update = Update() + .set(NodeDetailInfo::createdBy.name, record.createdBy) + .set(NodeDetailInfo::createdDate.name, record.createdDate) + .set(NodeDetailInfo::lastModifiedBy.name, record.lastModifiedBy) + .set(NodeDetailInfo::lastModifiedDate.name, record.lastModifiedDate) + .set(NodeDetailInfo::lastAccessDate.name, record.lastAccessDate) + .set(NodeDetailInfo::expireDate.name, record.expireDate) + .set(NodeDetailInfo::copyFromCredentialsKey.name, record.copyFromCredentialsKey) + .set(NodeDetailInfo::copyIntoCredentialsKey.name, record.copyIntoCredentialsKey) + .set(NodeDetailInfo::metadata.name, record.metadata) + .set(NodeDetailInfo::archived.name, record.archived) + .set(NodeDetailInfo::compressed.name, record.compressed) + .set(NodeDetailInfo::sha256.name, record.sha256) + .set(NodeDetailInfo::md5.name, record.md5) + .set(NodeDetailInfo::size.name, record.size) + .set(NodeDetailInfo::id.name, record.id) + mongoTemplate.updateFirst(existNodeQuery, update, collectionName) + } + + private fun buildQuery(record: BackupNodeInfo): Query { + return Query( + Criteria.where(BackupNodeInfo::repoName.name).isEqualTo(record.repoName) + .and(BackupNodeInfo::projectId.name).isEqualTo(record.projectId) + .and(BackupNodeInfo::fullPath.name).isEqualTo(record.fullPath) + .and(BackupNodeInfo::deleted.name).isEqualTo(record.deleted) + ) + } + + private fun buildIdQuery(record: BackupNodeInfo): Query { + return Query( + Criteria.where(BackupNodeInfo::repoName.name).isEqualTo(record.repoName) + .and(BackupNodeInfo::projectId.name).isEqualTo(record.projectId) + .and(BackupNodeInfo::fullPath.name).isEqualTo(record.fullPath) + .and(BackupNodeInfo::id.name).isEqualTo(record.id) + ) + } + + /** + * 生成随机文件路径 + * */ + fun generateRandomPath(root: Path, sha256: String): Path { + return Paths.get(root.toFile().path, FILE_STORE_FOLDER, generateRandomPath(sha256), sha256) + } + + companion object { + private val logger = LoggerFactory.getLogger(BackupNodeDataHandler::class.java) + private const val FAKE_SHA256 = "0000000000000000000000000000000000000000000000000000000000000000" + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupPackageDataHandler.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupPackageDataHandler.kt new file mode 100644 index 0000000000..c22db7ea59 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupPackageDataHandler.kt @@ -0,0 +1,96 @@ +package com.tencent.bkrepo.job.backup.service.impl.base + +import com.tencent.bkrepo.common.artifact.constant.REPO_NAME +import com.tencent.bkrepo.job.PROJECT +import com.tencent.bkrepo.job.backup.pojo.query.BackupPackageInfo +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.pojo.setting.BackupConflictStrategy +import com.tencent.bkrepo.job.backup.service.BackupDataHandler +import org.slf4j.LoggerFactory +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Component + +@Component +class BackupPackageDataHandler( + private val mongoTemplate: MongoTemplate, +) : BackupDataHandler { + override fun dataType(): BackupDataEnum { + return BackupDataEnum.PACKAGE_DATA + } + + override fun buildQueryCriteria(context: BackupContext): Criteria { + val criteria = Criteria.where(PROJECT).isEqualTo(context.currentProjectId) + .and(REPO_NAME).isEqualTo(context.currentRepoName) + if (context.incrementDate != null) { + criteria.and(BackupPackageInfo::lastModifiedDate.name).gte(context.incrementDate) + } + return criteria + } + + override fun preBackupDataHandler(record: T, backupDataEnum: BackupDataEnum, context: BackupContext) { + val packageInfo = record as BackupPackageInfo + context.currentPackageId = record.id + context.currentPackageKey = record.key + } + + override fun returnLastId(data: T): String { + return (data as BackupPackageInfo).id!! + } + + override fun storeRestoreDataHandler(record: T, backupDataEnum: BackupDataEnum, context: BackupContext) { + val record = record as BackupPackageInfo + val existRecord = findExistPackage(record) + if (existRecord != null) { + if (context.task.backupSetting.conflictStrategy == BackupConflictStrategy.SKIP) { + return + } + updateExistPackage(record) + } else { + mongoTemplate.save(record, BackupDataEnum.PACKAGE_DATA.collectionName) + logger.info("Create package ${record.key} in ${record.projectId}|${record.name} success!") + } + } + + //TODO 依赖源节点恢复时需要考虑索引文件如何更新.(仓库索引/包索引等 ) + private fun findExistPackage(record: BackupPackageInfo): BackupPackageInfo? { + val existPackageQuery = buildQuery(record) + return mongoTemplate.findOne( + existPackageQuery, + BackupPackageInfo::class.java, + BackupDataEnum.PACKAGE_DATA.collectionName + ) + } + + private fun updateExistPackage(packageInfo: BackupPackageInfo) { + val packageQuery = buildQuery(packageInfo) + // 逻辑删除， 同时删除索引 + val update = Update() + .set(BackupPackageInfo::latest.name, packageInfo.latest) + .set(BackupPackageInfo::description.name, packageInfo.description) + .set(BackupPackageInfo::extension.name, packageInfo.extension) + .set(BackupPackageInfo::clusterNames.name, packageInfo.clusterNames) + + mongoTemplate.updateFirst(packageQuery, update, BackupDataEnum.PACKAGE_DATA.collectionName) + logger.info( + "update exist package ${packageInfo.key} " + + "success with name ${packageInfo.projectId}|${packageInfo.repoName}" + ) + } + + private fun buildQuery(packageInfo: BackupPackageInfo): Query { + return Query( + Criteria.where(BackupPackageInfo::projectId.name).isEqualTo(packageInfo.projectId) + .and(BackupPackageInfo::repoName.name).isEqualTo(packageInfo.repoName) + .and(BackupPackageInfo::key.name).isEqualTo(packageInfo.key) + ) + } + + companion object { + private val logger = LoggerFactory.getLogger(BackupPackageDataHandler::class.java) + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupPackageVersionDataHandler.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupPackageVersionDataHandler.kt new file mode 100644 index 0000000000..de8ec041e3 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupPackageVersionDataHandler.kt @@ -0,0 +1,146 @@ +package com.tencent.bkrepo.job.backup.service.impl.base + +import com.tencent.bkrepo.job.PACKAGE_ID +import com.tencent.bkrepo.job.backup.pojo.query.BackupPackageInfo +import com.tencent.bkrepo.job.backup.pojo.query.BackupPackageVersionInfo +import com.tencent.bkrepo.job.backup.pojo.query.BackupPackageVersionInfoWithKeyInfo +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.pojo.setting.BackupConflictStrategy +import com.tencent.bkrepo.job.backup.service.BackupDataHandler +import org.slf4j.LoggerFactory +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Component + +@Component +class BackupPackageVersionDataHandler( + private val mongoTemplate: MongoTemplate, +) : BackupDataHandler { + override fun dataType(): BackupDataEnum { + return BackupDataEnum.PACKAGE_VERSION_DATA + } + + override fun buildQueryCriteria(context: BackupContext): Criteria { + val criteria = Criteria.where(PACKAGE_ID).isEqualTo(context.currentPackageId) + if (context.incrementDate != null) { + criteria.and(BackupPackageVersionInfoWithKeyInfo::lastModifiedDate.name).gte(context.incrementDate) + } + return criteria + } + + override fun preBackupDataHandler(record: T, backupDataEnum: BackupDataEnum, context: BackupContext) { + val record = record as BackupPackageVersionInfoWithKeyInfo + context.currentVersionName = record.name + record.projectId = context.currentProjectId!! + record.repoName = context.currentRepoName!! + record.key = context.currentPackageKey!! + + } + + override fun returnLastId(data: T): String { + return (data as BackupPackageVersionInfoWithKeyInfo).id!! + } + + override fun storeRestoreDataHandler(record: T, backupDataEnum: BackupDataEnum, context: BackupContext) { + val record = record as BackupPackageVersionInfoWithKeyInfo + val (packageId, existRecord) = findExistPackageVersion(record) + record.packageId = packageId + if (existRecord != null) { + if (context.task.backupSetting.conflictStrategy == BackupConflictStrategy.SKIP) { + return + } + updateExistPackageVersion(record) + } else { + val storeRecord = convert(record) + mongoTemplate.save(storeRecord, BackupDataEnum.PACKAGE_VERSION_DATA.collectionName) + logger.info("Create version ${record.name} of packageId ${record.packageId} success!") + } + + } + + private fun findExistPackageVersion( + record: BackupPackageVersionInfoWithKeyInfo + ): Pair { + // package的id会在version表中使用 + val existPackageQuery = Query( + Criteria.where(BackupPackageVersionInfoWithKeyInfo::repoName.name).isEqualTo(record.repoName) + .and(BackupPackageVersionInfoWithKeyInfo::projectId.name).isEqualTo(record.projectId) + .and(BackupPackageVersionInfoWithKeyInfo::key.name).isEqualTo(record.key) + ) + // TODO 此处需要缓存，避免过多导致频繁查询 + val packageInfo = mongoTemplate.findOne( + existPackageQuery, + BackupPackageInfo::class.java, + BackupDataEnum.PACKAGE_DATA.collectionName + ) + val existVersionQuery = Query( + Criteria.where(BackupPackageVersionInfoWithKeyInfo::packageId.name).isEqualTo(packageInfo.id) + .and(BackupPackageVersionInfoWithKeyInfo::name.name).isEqualTo(record.name) + ) + val existVersion = mongoTemplate.findOne( + existVersionQuery, + BackupPackageVersionInfo::class.java, + BackupDataEnum.PACKAGE_VERSION_DATA.collectionName + ) + return Pair(packageInfo.id!!, existVersion) + } + + private fun updateExistPackageVersion(versionInfo: BackupPackageVersionInfoWithKeyInfo) { + val packageQuery = Query( + Criteria.where(BackupPackageVersionInfo::packageId.name).isEqualTo(versionInfo.packageId) + .and(BackupPackageVersionInfo::name.name).isEqualTo(versionInfo.name) + ) + val update = Update() + .set(BackupPackageVersionInfo::size.name, versionInfo.size) + .set(BackupPackageVersionInfo::ordinal.name, versionInfo.ordinal) + .set(BackupPackageVersionInfo::downloads.name, versionInfo.downloads) + .set(BackupPackageVersionInfo::manifestPath.name, versionInfo.manifestPath) + .set(BackupPackageVersionInfo::artifactPath.name, versionInfo.artifactPath) + .set(BackupPackageVersionInfo::artifactPaths.name, versionInfo.artifactPaths) + .set(BackupPackageVersionInfo::stageTag.name, versionInfo.stageTag) + .set(BackupPackageVersionInfo::metadata.name, versionInfo.metadata) + .set(BackupPackageVersionInfo::tags.name, versionInfo.tags) + .set(BackupPackageVersionInfo::extension.name, versionInfo.extension) + .set(BackupPackageVersionInfo::clusterNames.name, versionInfo.clusterNames) + + mongoTemplate.updateFirst( + packageQuery, + update, + BackupDataEnum.PACKAGE_VERSION_DATA.collectionName + ) + logger.info("update exist version ${versionInfo.name} of package ${versionInfo.packageId} success") + } + + fun convert(versionInfo: BackupPackageVersionInfoWithKeyInfo): BackupPackageVersionInfo { + with(versionInfo) { + return BackupPackageVersionInfo( + id = id, + createdBy = createdBy, + createdDate = createdDate, + lastModifiedBy = lastModifiedBy, + lastModifiedDate = lastModifiedDate, + packageId = packageId, + name = name, + size = size, + ordinal = ordinal, + downloads = downloads, + manifestPath = manifestPath, + artifactPath = artifactPath, + artifactPaths = artifactPaths, + stageTag = stageTag, + metadata = metadata, + tags = tags, + extension = extension, + clusterNames = clusterNames, + ) + } + } + + companion object { + private val logger = LoggerFactory.getLogger(BackupPackageVersionDataHandler::class.java) + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupPermissionDataHandler.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupPermissionDataHandler.kt new file mode 100644 index 0000000000..d71f91c7de --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupPermissionDataHandler.kt @@ -0,0 +1,99 @@ +package com.tencent.bkrepo.job.backup.service.impl.base + +import com.tencent.bkrepo.job.backup.pojo.query.common.BackupPermission +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.pojo.setting.BackupConflictStrategy +import com.tencent.bkrepo.job.backup.service.BackupDataHandler +import org.slf4j.LoggerFactory +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.stereotype.Component + +@Component +class BackupPermissionDataHandler( + private val mongoTemplate: MongoTemplate, +) : BackupDataHandler { + override fun dataType(): BackupDataEnum { + return BackupDataEnum.PERMISSION_DATA + } + + override fun buildQueryCriteria(context: BackupContext): Criteria { + val criteria = Criteria() + if (context.incrementDate != null) { + criteria.and(BackupPermission::updateAt.name).gte(context.incrementDate) + } + return criteria + } + + override fun returnLastId(data: T): String { + return (data as BackupPermission).id!! + } + + override fun storeRestoreDataHandler(record: T, backupDataEnum: BackupDataEnum, context: BackupContext) { + val record = record as BackupPermission + val existRecord = findExistPermission(record) + if (existRecord != null) { + if (context.task.backupSetting.conflictStrategy == BackupConflictStrategy.SKIP) { + return + } + updateExistPermission(record, existRecord) + } else { + mongoTemplate.save(record, BackupDataEnum.PERMISSION_DATA.collectionName) + logger.info("Create permission ${record.permName} success!") + } + } + + private fun findExistPermission(record: BackupPermission): BackupPermission? { + val existPermissionQuery = buildQuery(record) + return mongoTemplate.findOne( + existPermissionQuery, + BackupPermission::class.java, + BackupDataEnum.PERMISSION_DATA.collectionName + ) + } + + private fun updateExistPermission(permissionInfo: BackupPermission, existRecord: BackupPermission) { + val permissionQuery = buildQuery(permissionInfo) + val mergedIncludePattern = BackupUserDataHandler.merge( + permissionInfo.includePattern, existRecord.includePattern + ) + val mergedExcludePattern = BackupUserDataHandler.merge( + permissionInfo.excludePattern, existRecord.excludePattern + ) + val mergedUsers = BackupUserDataHandler.merge(permissionInfo.users, existRecord.users) + val mergedRoles = BackupUserDataHandler.merge(permissionInfo.roles, existRecord.roles) + val mergedDepartments = BackupUserDataHandler.merge(permissionInfo.departments, existRecord.departments) + val mergedActions = BackupUserDataHandler.merge(permissionInfo.actions, existRecord.actions) + + val update = Update() + .set(BackupPermission::includePattern.name, mergedIncludePattern) + .set(BackupPermission::excludePattern.name, mergedExcludePattern) + .set(BackupPermission::updatedBy.name, permissionInfo.updatedBy) + .set(BackupPermission::users.name, mergedUsers) + .set(BackupPermission::roles.name, mergedRoles) + .set(BackupPermission::departments.name, mergedDepartments) + .set(BackupPermission::actions.name, mergedActions) + mongoTemplate.updateFirst( + permissionQuery, + update, + BackupDataEnum.PERMISSION_DATA.collectionName + ) + logger.info("update exist permission success with id ${permissionInfo.id}") + } + + private fun buildQuery(permissionInfo: BackupPermission): Query { + return Query.query( + Criteria.where(BackupPermission::permName.name).`is`(permissionInfo.permName) + .and(BackupPermission::projectId.name).`is`(permissionInfo.projectId) + .and(BackupPermission::repos.name).`is`(permissionInfo.repos) + .and(BackupPermission::resourceType.name).`is`(permissionInfo.resourceType) + ) + } + + companion object { + private val logger = LoggerFactory.getLogger(BackupPermissionDataHandler::class.java) + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupProjectDataHandler.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupProjectDataHandler.kt new file mode 100644 index 0000000000..25ae19b8e6 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupProjectDataHandler.kt @@ -0,0 +1,83 @@ +package com.tencent.bkrepo.job.backup.service.impl.base + +import com.tencent.bkrepo.job.NAME +import com.tencent.bkrepo.job.backup.pojo.query.BackupProjectInfo +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.pojo.setting.BackupConflictStrategy +import com.tencent.bkrepo.job.backup.service.BackupDataHandler +import org.slf4j.LoggerFactory +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Component + +@Component +class BackupProjectDataHandler( + private val mongoTemplate: MongoTemplate, +) : BackupDataHandler { + override fun dataType(): BackupDataEnum { + return BackupDataEnum.PROJECT_DATA + } + + override fun buildQueryCriteria(context: BackupContext): Criteria { + val criteria = Criteria.where(NAME).isEqualTo(context.currentProjectId) + if (context.incrementDate != null) { + criteria.and(BackupProjectInfo::lastModifiedDate.name).gte(context.incrementDate) + } + return criteria + } + + override fun returnLastId(data: T): String { + return (data as BackupProjectInfo).id!! + } + + override fun storeRestoreDataHandler(record: T, backupDataEnum: BackupDataEnum, context: BackupContext) { + val record = record as BackupProjectInfo + val existRecord = findExistProject(record) + if (existRecord != null) { + if (context.task.backupSetting.conflictStrategy == BackupConflictStrategy.SKIP) { + return + } + updateExistProject(record) + } else { + mongoTemplate.save(record, BackupDataEnum.PROJECT_DATA.collectionName) + logger.info("Create project ${record.name} success!") + } + } + + private fun findExistProject(record: BackupProjectInfo): BackupProjectInfo? { + val existProjectQuery = buildQuery(record) + return mongoTemplate.findOne( + existProjectQuery, + BackupProjectInfo::class.java, + BackupDataEnum.PROJECT_DATA.collectionName + ) + } + + private fun updateExistProject(projectInfo: BackupProjectInfo) { + val projectQuery = buildQuery(projectInfo) + val update = Update() + .set(BackupProjectInfo::lastModifiedBy.name, projectInfo.lastModifiedBy) + .set(BackupProjectInfo::createdBy.name, projectInfo.createdBy) + .set(BackupProjectInfo::createdDate.name, projectInfo.createdDate) + .set(BackupProjectInfo::lastModifiedDate.name, projectInfo.lastModifiedDate) + .set(BackupProjectInfo::displayName.name, projectInfo.displayName) + .set(BackupProjectInfo::description.name, projectInfo.description) + .set(BackupProjectInfo::metadata.name, projectInfo.metadata) + .set(BackupProjectInfo::credentialsKey.name, projectInfo.credentialsKey) + + mongoTemplate.updateFirst(projectQuery, update, BackupDataEnum.PROJECT_DATA.collectionName) + logger.info("update exist project success with name ${projectInfo.name}") + } + + private fun buildQuery(projectInfo: BackupProjectInfo): Query { + return Query(Criteria.where(NAME).isEqualTo(projectInfo.name)) + } + + companion object { + private val logger = LoggerFactory.getLogger(BackupProjectDataHandler::class.java) + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupRepositoryDataHandler.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupRepositoryDataHandler.kt new file mode 100644 index 0000000000..145f97ae6e --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupRepositoryDataHandler.kt @@ -0,0 +1,130 @@ +package com.tencent.bkrepo.job.backup.service.impl.base + +import com.tencent.bkrepo.common.api.util.readJsonString +import com.tencent.bkrepo.common.mongo.constant.ID_IDX +import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import com.tencent.bkrepo.job.NAME +import com.tencent.bkrepo.job.PROJECT +import com.tencent.bkrepo.job.backup.pojo.query.BackupRepositoryInfo +import com.tencent.bkrepo.job.backup.pojo.query.BackupStorageCredentials +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.pojo.setting.BackupConflictStrategy +import com.tencent.bkrepo.job.backup.service.BackupDataHandler +import com.tencent.bkrepo.job.backup.service.impl.repo.BackupRepoSpecialMappings +import org.slf4j.LoggerFactory +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Component + +@Component +class BackupRepositoryDataHandler( + private val mongoTemplate: MongoTemplate, +) : BackupDataHandler { + override fun dataType(): BackupDataEnum { + return BackupDataEnum.REPOSITORY_DATA + } + + override fun buildQueryCriteria(context: BackupContext): Criteria { + val criteria = Criteria.where(PROJECT).isEqualTo(context.currentProjectId) + .and(NAME).isEqualTo(context.currentRepoName) + if (context.incrementDate != null) { + criteria.and(BackupRepositoryInfo::lastModifiedDate.name).gte(context.incrementDate) + } + return criteria + } + + override fun preBackupDataHandler(record: T, backupDataEnum: BackupDataEnum, context: BackupContext) { + val repo = record as BackupRepositoryInfo + context.currentRepositoryType = repo.type + context.currentStorageCredentials = findStorageCredentials(repo.credentialsKey) + } + + override fun returnLastId(data: T): String { + return (data as BackupRepositoryInfo).id!! + } + + override fun storeRestoreDataHandler(record: T, backupDataEnum: BackupDataEnum, context: BackupContext) { + val record = record as BackupRepositoryInfo + val existRecord = findExistRepository(record) + if (existRecord != null) { + if (context.task.backupSetting.conflictStrategy == BackupConflictStrategy.SKIP) { + return + } + // TODO 仓库涉及存储, 不能简单更新 + updateExistRepo(record) + } else { + mongoTemplate.save(record, BackupDataEnum.REPOSITORY_DATA.collectionName) + logger.info("Create repo ${record.projectId}|${record.name} success!") + } + } + + override fun getSpecialDataEnum(context: BackupContext): List? { + return BackupRepoSpecialMappings.getRepoSpecialDataEnum(context) + } + + private fun findStorageCredentials(currentCredentialsKey: String?): StorageCredentials? { + val backupStorageCredentials = currentCredentialsKey?.let { + mongoTemplate.findOne( + Query(Criteria.where(ID_IDX).isEqualTo(it)), + BackupStorageCredentials::class.java, + STORAGE_CREDENTIALS_COLLECTION_NAME + ) + } + return backupStorageCredentials?.let { convert(backupStorageCredentials) } + } + + private fun convert(credentials: BackupStorageCredentials): StorageCredentials { + return credentials.credentials.readJsonString().apply { this.key = credentials.id } + } + + private fun findExistRepository(record: BackupRepositoryInfo): BackupRepositoryInfo? { + val existRepoQuery = buildQuery(record) + return mongoTemplate.findOne( + existRepoQuery, + BackupRepositoryInfo::class.java, + BackupDataEnum.REPOSITORY_DATA.collectionName + ) + } + + private fun updateExistRepo(repoInfo: BackupRepositoryInfo) { + val repoQuery = buildQuery(repoInfo) + val update = Update() + .set(BackupRepositoryInfo::lastModifiedBy.name, repoInfo.lastModifiedBy) + .set(BackupRepositoryInfo::createdBy.name, repoInfo.createdBy) + .set(BackupRepositoryInfo::createdDate.name, repoInfo.createdDate) + .set(BackupRepositoryInfo::lastModifiedDate.name, repoInfo.lastModifiedDate) + .set(BackupRepositoryInfo::type.name, repoInfo.type) + .set(BackupRepositoryInfo::description.name, repoInfo.description) + .set(BackupRepositoryInfo::category.name, repoInfo.category) + .set(BackupRepositoryInfo::public.name, repoInfo.public) + .set(BackupRepositoryInfo::configuration.name, repoInfo.configuration) + .set(BackupRepositoryInfo::oldCredentialsKey.name, repoInfo.oldCredentialsKey) + .set(BackupRepositoryInfo::display.name, repoInfo.display) + .set(BackupRepositoryInfo::clusterNames.name, repoInfo.clusterNames) + .set(BackupRepositoryInfo::credentialsKey.name, repoInfo.credentialsKey) + .set(BackupRepositoryInfo::deleted.name, repoInfo.deleted) + .set(BackupRepositoryInfo::quota.name, repoInfo.quota) + .set(BackupRepositoryInfo::used.name, repoInfo.used) + // TODO quote和used需要进行事实计算更新 + + mongoTemplate.updateFirst(repoQuery, update, BackupDataEnum.REPOSITORY_DATA.collectionName) + logger.info("update exist repo success with name ${repoInfo.projectId}|${repoInfo.name}") + } + + private fun buildQuery(repoInfo: BackupRepositoryInfo): Query { + return Query( + Criteria.where(BackupRepositoryInfo::projectId.name).isEqualTo(repoInfo.projectId) + .and(BackupRepositoryInfo::name.name).isEqualTo(repoInfo.name) + .and(BackupRepositoryInfo::deleted.name).isEqualTo(repoInfo.deleted) + ) + } + + companion object { + private val logger = LoggerFactory.getLogger(BackupRepositoryDataHandler::class.java) + private const val STORAGE_CREDENTIALS_COLLECTION_NAME = "storage_credentials" + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupRoleDataHandler.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupRoleDataHandler.kt new file mode 100644 index 0000000000..95d3e013ae --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupRoleDataHandler.kt @@ -0,0 +1,74 @@ +package com.tencent.bkrepo.job.backup.service.impl.base + +import com.tencent.bkrepo.job.backup.pojo.query.common.BackupRole +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.pojo.setting.BackupConflictStrategy +import com.tencent.bkrepo.job.backup.service.BackupDataHandler +import org.slf4j.LoggerFactory +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Component + +@Component +class BackupRoleDataHandler( + private val mongoTemplate: MongoTemplate, +) : BackupDataHandler { + override fun dataType(): BackupDataEnum { + return BackupDataEnum.ROLE_DATA + } + + override fun buildQueryCriteria(context: BackupContext): Criteria { + return Criteria() + } + + override fun returnLastId(data: T): String { + return (data as BackupRole).id!! + } + + override fun storeRestoreDataHandler(record: T, backupDataEnum: BackupDataEnum, context: BackupContext) { + val record = record as BackupRole + val existRecord = findExistRole(record) + if (existRecord != null) { + if (context.task.backupSetting.conflictStrategy == BackupConflictStrategy.SKIP) { + return + } + updateExistRole(record) + } else { + mongoTemplate.save(record, BackupDataEnum.ROLE_DATA.collectionName) + logger.info("Create role ${record.name} success!") + } + } + + private fun findExistRole(record: BackupRole): BackupRole? { + val existRoleQuery = buildQuery(record) + return mongoTemplate.findOne(existRoleQuery, BackupRole::class.java, BackupDataEnum.ROLE_DATA.collectionName) + } + + private fun updateExistRole(roleInfo: BackupRole) { + val roleQuery = buildQuery(roleInfo) + val update = Update() + .set(BackupRole::name.name, roleInfo.name) + .set(BackupRole::admin.name, roleInfo.admin) + .set(BackupRole::description.name, roleInfo.description) + mongoTemplate.updateFirst(roleQuery, update, BackupDataEnum.ROLE_DATA.collectionName) + logger.info("update exist role success with name ${roleInfo.name}") + } + + private fun buildQuery(roleInfo: BackupRole): Query { + return Query( + Criteria.where(BackupRole::roleId.name).isEqualTo(roleInfo.roleId) + .and(BackupRole::projectId.name).isEqualTo(roleInfo.projectId) + .and(BackupRole::repoName.name).isEqualTo(roleInfo.repoName) + .and(BackupRole::type.name).isEqualTo(roleInfo.type) + .and(BackupRole::source.name).isEqualTo(roleInfo.source) + ) + } + + companion object { + private val logger = LoggerFactory.getLogger(BackupRoleDataHandler::class.java) + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupTemporaryTokenDataHandler.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupTemporaryTokenDataHandler.kt new file mode 100644 index 0000000000..a7e6b109cc --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupTemporaryTokenDataHandler.kt @@ -0,0 +1,56 @@ +package com.tencent.bkrepo.job.backup.service.impl.base + +import com.tencent.bkrepo.job.backup.pojo.query.common.BackupTemporaryToken +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.service.BackupDataHandler +import org.slf4j.LoggerFactory +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Component + +@Component +class BackupTemporaryTokenDataHandler( + private val mongoTemplate: MongoTemplate, +) : BackupDataHandler { + override fun dataType(): BackupDataEnum { + return BackupDataEnum.TEMPORARY_TOKEN_DATA + } + + override fun buildQueryCriteria(context: BackupContext): Criteria { + val criteria = Criteria() + if (context.incrementDate != null) { + criteria.and(BackupTemporaryToken::lastModifiedDate.name).gte(context.incrementDate) + } + return criteria + } + + override fun returnLastId(data: T): String { + return (data as BackupTemporaryToken).id!! + } + + override fun storeRestoreDataHandler(record: T, backupDataEnum: BackupDataEnum, context: BackupContext) { + val record = record as BackupTemporaryToken + val existedRecord = findExistTemporaryToken(record) + if (existedRecord == null) { + mongoTemplate.save(record, BackupDataEnum.TEMPORARY_TOKEN_DATA.collectionName) + logger.info("Create token ${record.id} success!") + } + } + + + private fun findExistTemporaryToken(record: BackupTemporaryToken): BackupTemporaryToken? { + val existTokenQuery = Query(Criteria.where(BackupTemporaryToken::id.name).isEqualTo(record.id)) + return mongoTemplate.findOne( + existTokenQuery, + BackupTemporaryToken::class.java, + BackupDataEnum.TEMPORARY_TOKEN_DATA.collectionName + ) + } + + companion object { + private val logger = LoggerFactory.getLogger(BackupTemporaryTokenDataHandler::class.java) + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupUserDataHandler.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupUserDataHandler.kt new file mode 100644 index 0000000000..b210a4e9f8 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/base/BackupUserDataHandler.kt @@ -0,0 +1,113 @@ +package com.tencent.bkrepo.job.backup.service.impl.base + +import com.tencent.bkrepo.auth.pojo.token.Token +import com.tencent.bkrepo.job.backup.pojo.query.common.BackupUser +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.pojo.setting.BackupConflictStrategy +import com.tencent.bkrepo.job.backup.service.BackupDataHandler +import org.slf4j.LoggerFactory +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Component + +@Component +class BackupUserDataHandler( + private val mongoTemplate: MongoTemplate, +) : BackupDataHandler { + override fun dataType(): BackupDataEnum { + return BackupDataEnum.USER_DATA + } + + override fun buildQueryCriteria(context: BackupContext): Criteria { + val criteria = Criteria() + if (context.incrementDate != null) { + criteria.and(BackupUser::lastModifiedDate.name).gte(context.incrementDate) + } + return criteria + } + + override fun returnLastId(data: T): String { + return (data as BackupUser).id!! + } + + override fun storeRestoreDataHandler(record: T, backupDataEnum: BackupDataEnum, context: BackupContext) { + val record = record as BackupUser + val existRecord = findExistUser(record) + if (existRecord != null) { + if (context.task.backupSetting.conflictStrategy == BackupConflictStrategy.SKIP) { + return + } + updateExistUser(record, existRecord) + } else { + mongoTemplate.save(record, BackupDataEnum.USER_DATA.collectionName) + logger.info("Create user ${record.name} success!") + } + } + + private fun findExistUser(record: BackupUser): BackupUser? { + val existUserQuery = buildQuery(record) + return mongoTemplate.findOne( + existUserQuery, + BackupUser::class.java, + BackupDataEnum.USER_DATA.collectionName + ) + } + + private fun updateExistUser(userInfo: BackupUser, existRecord: BackupUser) { + // TODO 部分字段更新可能会导致原有数据不能使用 + val roleQuery = buildQuery(userInfo) + val mergedRoles = merge(userInfo.roles, existRecord.roles) + val mergedAsstUsers = merge(userInfo.asstUsers, existRecord.asstUsers) + val mergedAccounts = merge(userInfo.accounts.orEmpty(), existRecord.accounts.orEmpty()) + val mergedTokens = mergeTokens(userInfo.tokens, existRecord.tokens) + + val update = Update() + .set(BackupUser::name.name, userInfo.name) + .set(BackupUser::admin.name, userInfo.admin) + .set(BackupUser::pwd.name, userInfo.pwd) + .set(BackupUser::locked.name, userInfo.locked) + .set(BackupUser::group.name, userInfo.group) + .set(BackupUser::pwd.name, userInfo.pwd) + .set(BackupUser::email.name, userInfo.email) + .set(BackupUser::pwd.name, userInfo.pwd) + .set(BackupUser::phone.name, userInfo.phone) + .set(BackupUser::source.name, userInfo.source) + .set(BackupUser::tokens.name, mergedTokens) + .set(BackupUser::roles.name, mergedRoles) + .set(BackupUser::asstUsers.name, mergedAsstUsers) + .set(BackupUser::accounts.name, mergedAccounts) + mongoTemplate.updateFirst(roleQuery, update, BackupDataEnum.USER_DATA.collectionName) + logger.info("update exist user success with name ${userInfo.userId}") + } + + private fun buildQuery(userInfo: BackupUser): Query { + return Query(Criteria.where(BackupUser::userId.name).isEqualTo(userInfo.userId)) + } + + fun mergeTokens(oldData: List, newData: List): MutableList { + val result = mutableListOf() + result.addAll(oldData) + newData.forEach { newToken -> + val existed = result.firstOrNull { it.id == newToken.id } + if (existed == null) { + result.add(newToken) + } + } + return result + } + + companion object { + private val logger = LoggerFactory.getLogger(BackupUserDataHandler::class.java) + + fun merge(oldData: List, newData: List): MutableList { + val result = mutableSetOf() + result.addAll(oldData) + result.addAll(newData) + return result.toMutableList() + } + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/repo/BackupConanRepoHandler.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/repo/BackupConanRepoHandler.kt new file mode 100644 index 0000000000..3bb98ab428 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/repo/BackupConanRepoHandler.kt @@ -0,0 +1,56 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.service.impl.repo + +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.service.BackupRepoSpecialDataService +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Component + + +@Component +class BackupConanRepoHandler : BackupRepoSpecialDataService { + + override fun type(): RepositoryType { + return RepositoryType.CONAN + } + + override fun extraType(): RepositoryType? { + return null + } + + override fun getRepoSpecialDataEnum(context: BackupContext): List { + return listOf(BackupDataEnum.CONAN_METADATA_DATA) + } + + companion object { + private val logger = LoggerFactory.getLogger(BackupConanRepoHandler::class.java) + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/repo/BackupMavenRepoHandler.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/repo/BackupMavenRepoHandler.kt new file mode 100644 index 0000000000..443713f46e --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/repo/BackupMavenRepoHandler.kt @@ -0,0 +1,56 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.service.impl.repo + +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.service.BackupRepoSpecialDataService +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Component + + +@Component +class BackupMavenRepoHandler : BackupRepoSpecialDataService { + + override fun type(): RepositoryType { + return RepositoryType.MAVEN + } + + override fun extraType(): RepositoryType? { + return null + } + + override fun getRepoSpecialDataEnum(context: BackupContext): List { + return listOf(BackupDataEnum.MAVEN_METADATA_DATA) + } + + companion object { + private val logger = LoggerFactory.getLogger(BackupMavenRepoHandler::class.java) + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/repo/BackupRepoSpecialMappings.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/repo/BackupRepoSpecialMappings.kt new file mode 100644 index 0000000000..af9a590858 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/service/impl/repo/BackupRepoSpecialMappings.kt @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.backup.service.impl.repo + +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.service.util.SpringContextUtils +import com.tencent.bkrepo.job.backup.pojo.query.enums.BackupDataEnum +import com.tencent.bkrepo.job.backup.pojo.record.BackupContext +import com.tencent.bkrepo.job.backup.service.BackupRepoSpecialDataService + +object BackupRepoSpecialMappings { + + private val mappers = mutableMapOf() + + init { + addMapper(SpringContextUtils.getBean(BackupMavenRepoHandler::class.java)) + addMapper(SpringContextUtils.getBean(BackupConanRepoHandler::class.java)) + } + + private fun addMapper(mapper: BackupRepoSpecialDataService) { + mappers[mapper.type()] = mapper + mapper.extraType()?.let { mappers[mapper.extraType()!!] = mapper } + } + + fun getRepoSpecialDataEnum(context: BackupContext): List? { + val mapper = mappers[context.currentRepositoryType] ?: return null + return mapper.getRepoSpecialDataEnum(context) + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/util/ZipFileUtil.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/util/ZipFileUtil.kt new file mode 100644 index 0000000000..88345ed2d4 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/util/ZipFileUtil.kt @@ -0,0 +1,62 @@ +package com.tencent.bkrepo.job.backup.util + +import java.io.File +import java.io.FileInputStream +import java.io.FileOutputStream +import java.util.zip.ZipEntry +import java.util.zip.ZipFile +import java.util.zip.ZipOutputStream + +object ZipFileUtil { + + fun compressDirectory(directoryPath: String, zipFilePath: String) { + val directory = File(directoryPath) + val zipFile = File(zipFilePath) + + val zipOutputStream = ZipOutputStream(FileOutputStream(zipFile)) + + compress(directory, directory.name, zipOutputStream) + + zipOutputStream.close() + } + + fun decompressFile(zipFilePath: String, destinationFolder: String) { + val zipFile = ZipFile(zipFilePath) + val entries = zipFile.entries() + while (entries.hasMoreElements()) { + val entry = entries.nextElement() + val entryDestination = File(destinationFolder, entry.name) + if (entry.isDirectory) { + entryDestination.mkdirs() + } else { + entryDestination.parentFile.mkdirs() + val inputStream = zipFile.getInputStream(entry) + val outputStream = FileOutputStream(entryDestination) + inputStream.copyTo(outputStream) + inputStream.close() + outputStream.close() + } + } + } + + private fun compress(file: File, fileName: String, zipOutputStream: ZipOutputStream) { + if (file.isDirectory) { + val files = file.listFiles() + for (childFile in files) { + compress(childFile, fileName + File.separator + childFile.name, zipOutputStream) + } + } else { + val buffer = ByteArray(1024) + val fileInputStream = FileInputStream(file) + + zipOutputStream.putNextEntry(ZipEntry(fileName)) + + var length: Int + while (fileInputStream.read(buffer).also { length = it } > 0) { + zipOutputStream.write(buffer, 0, length) + } + + fileInputStream.close() + } + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/JobConfig.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/JobConfig.kt index b0edcd330d..c2d8c6c85a 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/JobConfig.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/JobConfig.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.job.config import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent +import com.tencent.bkrepo.job.backup.config.DataBackupConfig import com.tencent.bkrepo.job.executor.BlockThreadPoolTaskExecutorDecorator import com.tencent.bkrepo.job.migrate.config.MigrateRepoStorageProperties import com.tencent.bkrepo.job.separation.config.DataSeparationConfig @@ -48,6 +49,7 @@ import java.util.function.Consumer JobProperties::class, MigrateRepoStorageProperties::class, DataSeparationConfig::class, + DataBackupConfig::class ) class JobConfig { @Bean diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/controller/user/UserBackupController.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/controller/user/UserBackupController.kt new file mode 100644 index 0000000000..8855f94b5b --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/controller/user/UserBackupController.kt @@ -0,0 +1,75 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.controller.user + +import com.tencent.bkrepo.common.api.constant.DEFAULT_PAGE_NUMBER +import com.tencent.bkrepo.common.api.constant.DEFAULT_PAGE_SIZE +import com.tencent.bkrepo.common.api.pojo.Page +import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.mongo.util.Pages +import com.tencent.bkrepo.common.security.permission.Principal +import com.tencent.bkrepo.common.security.permission.PrincipalType +import com.tencent.bkrepo.common.service.util.ResponseBuilder +import com.tencent.bkrepo.job.backup.pojo.task.BackupTask +import com.tencent.bkrepo.job.backup.pojo.task.BackupTaskRequest +import com.tencent.bkrepo.job.backup.service.DataBackupService +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping("/api/job/backup") +@Principal(type = PrincipalType.ADMIN) +class UserBackupController( + private val dataBackupService: DataBackupService, +) { + @PostMapping + fun createTask(@RequestBody request: BackupTaskRequest): Response { + return ResponseBuilder.success(dataBackupService.createTask(request)) + } + + @PostMapping("/execute/{taskId}") + fun executeTask(@PathVariable("taskId") taskId: String): Response { + dataBackupService.executeTask(taskId) + return ResponseBuilder.success() + } + + @GetMapping("/tasks") + fun tasks( + @RequestParam(required = false) state: String? = null, + @RequestParam(required = false, defaultValue = "$DEFAULT_PAGE_NUMBER") pageNumber: Int = DEFAULT_PAGE_NUMBER, + @RequestParam(required = false, defaultValue = "$DEFAULT_PAGE_SIZE") pageSize: Int = DEFAULT_PAGE_SIZE, + ): Response> { + val page = dataBackupService.findTasks(state, Pages.ofRequest(pageNumber, pageSize)) + return ResponseBuilder.success(page) + } +}