impr: 优化archive下载器 #1696

* impr: 优化archive下载器 #1696

* bug: 修复签名文件被删除 #1696

src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/config/CompressProperties.kt

@@ -1,5 +1,6 @@
 package com.tencent.bkrepo.archive.config
 
+import org.springframework.util.unit.DataSize
 import java.time.Duration
 
 data class CompressProperties(
@@ -9,4 +10,6 @@ data class CompressProperties(
     var patchThreads: Int = 1, // 文件合并：IO
     var ratio: Float = 0.5f, // 重复率阈值
     var signFileCacheTime: Duration = Duration.ofHours(6), // 签名文件缓存事件
+    var lowWaterMark: DataSize = DataSize.ofGigabytes(10),
+    var highWaterMark: DataSize = DataSize.ofGigabytes(100),
 )

src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/FileStorageFileProvider.kt

@@ -16,7 +16,8 @@ import kotlin.system.measureNanoTime
 
 class FileStorageFileProvider(
     private val fileDir: Path,
-    private val diskFreeThreshold: Long,
+    private val highWaterMark: Long,
+    private val lowWaterMark: Long,
     private val executor: Executor,
 ) : FileProvider {
 
@@ -51,19 +52,20 @@ class FileStorageFileProvider(
 
     private fun checkDiskSpace() {
         var diskFreeInBytes = fileDir.toFile().usableSpace
-        while (diskFreeInBytes < diskFreeThreshold) {
-            logger.info(
-                "DFree disk space below threshold.Available:" +
-                    " $diskFreeInBytes bytes (threshold: $diskFreeThreshold).",
-            )
-            Thread.sleep(CHECK_INTERVAL)
-            diskFreeInBytes = fileDir.toFile().usableSpace
+        val msg = "Free disk space below threshold.Available: %s bytes (threshold: %s)."
+        if (diskFreeInBytes < lowWaterMark) {
+            logger.info("Free disk below low water mark. ${msg.format(diskFreeInBytes, lowWaterMark)}")
+            while (diskFreeInBytes < highWaterMark) {
+                logger.info(msg.format(diskFreeInBytes, highWaterMark))
+                Thread.sleep(CHECK_INTERVAL)
+                diskFreeInBytes = fileDir.toFile().usableSpace
+            }
         }
     }
 
     companion object {
         private val logger = LoggerFactory.getLogger(FileStorageFileProvider::class.java)
         private const val RETRY_TIMES = 3
-        private const val CHECK_INTERVAL = 60000L
+        private const val CHECK_INTERVAL = 10 * 60000L
     }
 }

src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/compress/BDCompressor.kt

@@ -35,7 +35,7 @@ class BDCompressor(
                 val start = System.nanoTime()
                 val file = BDUtils.deltaByChecksumFile(src, checksum, srcKey, destKey, workDir, ratio)
                 val nanos = System.nanoTime() - start
-                val throughput = Throughput(nanos, file.length())
+                val throughput = Throughput(src.length(), nanos)
                 logger.info("Success to bd compress $srcKey,$throughput.")
                 file
             } finally {

src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/compress/BDZipManager.kt

@@ -13,6 +13,7 @@ import com.tencent.bkrepo.archive.utils.ArchiveDaoUtils.optimisticLock
 import com.tencent.bkrepo.archive.utils.ArchiveUtils
 import com.tencent.bkrepo.common.artifact.api.toArtifactFile
 import com.tencent.bkrepo.common.artifact.stream.Range
+import com.tencent.bkrepo.common.bksync.transfer.exception.TooLowerReuseRateException
 import com.tencent.bkrepo.common.service.util.SpringContextUtils
 import com.tencent.bkrepo.common.storage.core.StorageService
 import com.tencent.bkrepo.common.storage.monitor.Throughput
@@ -56,7 +57,8 @@ class BDZipManager(
 
     private val fileProvider = FileStorageFileProvider(
         workDir.resolve(DOWNLOAD_DIR),
-        archiveProperties.threshold.toBytes(),
+        archiveProperties.compress.highWaterMark.toBytes(),
+        archiveProperties.compress.lowWaterMark.toBytes(),
         fileDownloadThreadPool,
     )
     private val checksumProvider = ChecksumFileProvider(
@@ -126,7 +128,9 @@ class BDZipManager(
                     logger.info("Success to compress file [$sha256] on $storageCredentialsKey.")
                 }
                 .doOnError {
-                    logger.info("Failed to compress file [$sha256].", it)
+                    if (it !is TooLowerReuseRateException) {
+                        logger.error("Failed to compress file [$sha256].", it)
+                    }
                     status = CompressStatus.COMPRESS_FAILED
                     fileReferenceClient.decrement(baseSha256, storageCredentialsKey)
                 }
@@ -189,7 +193,7 @@ class BDZipManager(
                     logger.info("Success to uncompress file [$sha256] on $storageCredentialsKey")
                 }
                 .doOnError {
-                    logger.info("Failed to uncompress file [$sha256] on $storageCredentialsKey", it)
+                    logger.error("Failed to uncompress file [$sha256] on $storageCredentialsKey", it)
                     file.status = CompressStatus.UNCOMPRESS_FAILED
                 }
                 .doFinally {

src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/compress/ChecksumFileProvider.kt

@@ -53,7 +53,13 @@ class ChecksumFileProvider(
     }
 
     private fun sign(file: File, checksumFilePath: Path, sha256: String, key: String?): File {
+        if (Files.exists(checksumFilePath)) {
+            return checksumFilePath.toFile()
+        }
         synchronized(sha256.intern()) {
+            if (Files.exists(checksumFilePath)) {
+                return checksumFilePath.toFile()
+            }
             Files.newOutputStream(checksumFilePath).use { out ->
                 val bkSync = BkSync()
                 val throughput = measureThroughput(file.length()) { bkSync.checksum(file, out) }