Merge pull request #2902 from zacYL/issue_2837_master

feat: 增加仓库备份功能#2837

docs/apidoc/backup/backup.md

@@ -0,0 +1,218 @@
+# 系统备份以及恢复接口
+
+[toc]
+
+## 创建备份/恢复任务
+
+- API: POST /job/api/job/backup
+- API 名称: 创建备份/恢复任务
+- 功能说明：
+	- 中文：创建备份/恢复任务
+	- English：create backup/restore task
+
+- 请求体
+  ```json
+  新建备份任务
+  {
+      "name":"test",
+      "storeLocation":"F:\\backuptest",
+      "content":{
+          "commonData": true,
+          "compression":true,
+          "projects":[{
+              "projectRegex":"*"
+          }]
+      }
+  }
+  ```
+  ```json
+  新建恢复任务
+  {
+      "name":"test1-restore",
+      "storeLocation":"F:\\backuptest\\test1\\test1-20241224.171354.zip",
+      "type":"DATA_RESTORE",
+      "content":{
+          "commonData": true,
+          "compression":true,
+          "projects":[{
+              "projectRegex":"*"
+          }]
+      }
+  }
+  ```
+- 请求字段说明
+
+  |字段|类型|是否必须|默认值|说明|Description|
+  |---|---|---|---|---|---|
+  |name|string|是|无|任务名|task name|
+  |storeLocation|string|是|无|本地存储地址|store folder|
+  |backupSetting|backupSetting|否|有|任务设置| task setting|
+  |content|BackupContent|是|无|任务内容| task content|
+  |type|string|否|有|任务类型：备份DATA_BACKUP/恢复DATA_RESTORE| task type: DATA_BACKUP/DATA_RESTORE |
+
+
+- BackupContent对象说明
+
+  |字段|类型|是否必须|默认值|说明|Description|
+  |---|---|---|---|---|---|
+  |commonData|bool|否|false|是否备份或者恢复基础数据|backup/restore basic data|
+  |compression|bool|否|false|是否压缩最终备份文件目录|compress backup folder|
+  |increment|bool|否|null|是否增量备份|incremental backup|
+  |incrementDate|string|否|无|如开启增量备份，日期不指定则从任务执行日期前一天开始备份；如指定，则从指定日期开始|if increment is true and incrementDate is not set, then the default incrementDate is current date minus 1. Backup tasks will backup all the data after incrementDate|
+  |projects|arrays[ProjectContentInfo]|否|null|具体项目仓库信息|project or repo info|
+
+- ProjectContentInfo对象说明
+
+  |字段|类型|是否必须|默认值|说明|Description|
+  |---|---|---|---|---|---|
+  |projectId|string|否|null|指定备份/恢复项目|target projectId|
+  |projectRegex|string|否|null|备份/恢复项目正则|regex of target projectid|
+  |repoList|arrays|否|null|指定备份/恢复仓库列表|target repos|
+  |repoRegex|string|否|null|备份/恢复仓库正则|regex of target repo|
+  |excludeProjects|arrays|否|null|排除项目列表|exclude project list|
+  |excludeRepos|arrays|否|null|排除仓库列表|exclude repo list|
+
+- setting对象说明
+
+  |字段|类型|是否必须|默认值|说明|Description|
+  |---|---|---|---|---|---|
+  |conflictStrategy|enum|是|OVERWRITE|[SKIP,OVERWRITE]|conflict strategy|
+  |errorStrategy|enum|是|CONTINUE|[CONTINUE,FAST_FAIL]|error strategy|
+  |executionStrategy|enum|是|IMMEDIATELY|[IMMEDIATELY,SPECIFIED_TIME,CRON_EXPRESSION]|execution strategy|
+  |executionPlan|object|是|无|调度策略|execution plan|
+
+- executionPlan对象说明
+
+  |字段|类型|是否必须|默认值|说明|Description|
+  |---|---|---|---|---|---|
+  |executeImmediately|bool|是|true|立即执行|execute immediately|
+  |executeTime|time|否|无|执行时间执行|execute time|
+  |cronExpression|string|否|无|cron表达式执行|cron expression|
+
+- 响应体
+
+  ``` json
+  {
+    "code" : 0,
+    "message" : null,
+    "data" : "676a8abd60cc914e2a8a26f0",
+    "traceId" : null
+  }
+  ```
+
+- data字段说明
+ 返回任务id
+
+
+## 执行任务
+
+- API: POST /api/job/backup/execute/{taskId}
+- API 名称: 执行任务
+- 功能说明：
+  - 中文：执行任务
+  - English：execute task
+
+- 请求体
+  此接口请求体为空
+
+- 请求字段说明
+
+  |字段|类型|是否必须|默认值|说明|Description|
+  |---|---|---|---|---|---|
+  |taskId|string|是|无|任务id|task id|
+
+- 请求头
+
+- 响应头
+
+- 响应体
+
+  ``` json
+  {
+    "code" : 0,
+    "message" : null,
+    "data" : null,
+    "traceId" : null
+  }
+  ```
+
+## 分页获取任务列表
+
+- API: GET /job/api/job/backup/tasks?state={state}&pageSize={pageSize}&pageNumber={pageNumber}
+- API 名称: 获取任务列表
+- 功能说明：
+  - 中文：获取任务列表
+  - English：get task list
+
+- 请求体
+  此接口请求体为空
+
+- 请求字段说明
+
+  |字段|类型|是否必须|默认值|说明|Description|
+  |---|---|---|---|---|---|
+  |state|string|否|null|任务执行状态|task state: PENDING/RUNNING/FINISHED|
+  |pageNumber|int|否|1|页码|page number|
+  |pageSize|int|否|20|页大小|page size|
+
+- 请求头
+
+- 响应头
+
+- 响应体
+  ``` json
+  {
+      "code": 0,
+      "message": null,
+      "data": {
+          "pageNumber": 1,
+          "pageSize": 1,
+          "totalRecords": 7,
+          "totalPages": 7,
+          "records": [
+              {
+                  "id": "675ab19767a452255ba84d8b",
+                  "createdBy": "xxxx",
+                  "createdDate": "xxxx",
+                  "lastModifiedBy": "xxx",
+                  "lastModifiedDate": "xxx",
+                  "startDate": "xxx",
+                  "endDate": "xxx",
+                  "state": "FINISHED",
+                  "content": {
+                      "commonData": false,
+                      "compression": false,
+                      "projects": [
+                          {
+                              "projectId": "backuptest",
+                              "projectRegex": null,
+                              "repoList": null,
+                              "repoRegex": null,
+                              "excludeProjects": null,
+                              "excludeRepos": null
+                          }
+                      ],
+                      "increment": null,
+                      "incrementDate": null
+                  },
+                  "storeLocation": "F:\\backuptest",
+                  "backupSetting": {
+                      "conflictStrategy": "SKIP",
+                      "errorStrategy": "CONTINUE",
+                      "executionStrategy": "IMMEDIATELY",
+                      "executionPlan": {
+                          "executeImmediately": true,
+                          "executeTime": null,
+                          "cronExpression": null
+                      },
+                      "spaceCapCheck": true
+                  },
+                  "type": "DATA_BACKUP"
+              }
+          ],
+          "count": 7,
+          "page": 1
+      },
+      "traceId": "xxx"
+  }
+  ``` json

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/Constants.kt

@@ -69,6 +69,7 @@ const val REPLICA_TYPE = "replicaType"
 const val NAME = "name"
 const val SIZE = "size"
 const val NODE_NUM = "nodeNum"
+const val PACKAGE_ID = "packageId"
 
 
 /**
@@ -114,4 +115,10 @@ const val PACKAGE_DOWNLOADS_COLLECTION_NAME = "package_downloads"
 const val SEPARATION_TASK_COLLECTION_NAME = "separation_task"
 
 const val PACKAGE_VERSION = "version"
-const val PACKAGE_DOWNLOAD_DATE = "date"
+const val PACKAGE_DOWNLOAD_DATE = "date"
+
+/**
+ * 记录备份
+ */
+const val DATA_RECORDS_BACKUP = "DATA_BACKUP"
+const val DATA_RECORDS_RESTORE = "DATA_RESTORE"

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/config/DataBackupConfig.kt

@@ -0,0 +1,15 @@
+package com.tencent.bkrepo.job.backup.config
+
+import com.tencent.bkrepo.common.storage.credentials.InnerCosCredentials
+import org.springframework.boot.context.properties.ConfigurationProperties
+
+@ConfigurationProperties("backup")
+data class DataBackupConfig(
+    // 当磁盘已用容量比例大于大于阈值时,不进行备份
+    var usageThreshold: Double = 0.8,
+
+    /**
+     * 备份数据存储实例
+     * */
+    var cos: InnerCosCredentials = InnerCosCredentials(),
+)

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/backup/dao/BackupTaskDao.kt

@@ -0,0 +1,103 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.job.backup.dao
+
+import com.tencent.bkrepo.common.mongo.dao.simple.SimpleMongoDao
+import com.tencent.bkrepo.job.backup.model.TBackupTask
+import com.tencent.bkrepo.job.backup.pojo.BackupTaskState
+import com.tencent.bkrepo.repository.constant.SYSTEM_USER
+import org.springframework.data.domain.PageRequest
+import org.springframework.data.mongodb.core.query.Criteria
+import org.springframework.data.mongodb.core.query.Query
+import org.springframework.data.mongodb.core.query.Update
+import org.springframework.data.mongodb.core.query.isEqualTo
+import org.springframework.stereotype.Repository
+import java.time.LocalDateTime
+
+@Repository
+class BackupTaskDao : SimpleMongoDao<TBackupTask>() {
+
+    fun findTasksById(taskId: String): TBackupTask? {
+        val criteria = Criteria().and(TBackupTask::id.name).isEqualTo(taskId)
+        return findOne(Query(criteria))
+    }
+
+    fun find(state: String?, pageRequest: PageRequest): List<TBackupTask> {
+        val criteria = Criteria()
+        state?.let { criteria.and(TBackupTask::state.name).isEqualTo(it) }
+        return find(Query(criteria).with(pageRequest))
+    }
+
+    fun count(state: String?): Long {
+        val criteria = Criteria()
+        state?.let { criteria.and(TBackupTask::state.name).isEqualTo(it) }
+        return count(Query(criteria))
+    }
+
+    fun updateState(
+        taskId: String,
+        state: BackupTaskState,
+        startDate: LocalDateTime? = null,
+        endDate: LocalDateTime? = null
+    ) {
+        val criteria = Criteria().and(ID).isEqualTo(taskId)
+        val update = Update.update(TBackupTask::lastModifiedBy.name, SYSTEM_USER)
+            .set(TBackupTask::lastModifiedDate.name, LocalDateTime.now())
+            .set(TBackupTask::state.name, state.name)
+        startDate?.let { update.set(TBackupTask::startDate.name, startDate) }
+        endDate?.let { update.set(TBackupTask::endDate.name, endDate) }
+        this.updateFirst(Query(criteria), update)
+    }
+}