Skip to content

Commit

Permalink
Optimized the update logic during streaming download. First download …
Browse files Browse the repository at this point in the history 
…the list of files to be deleted, delete the files in the list, then download the files to a temporary folder, and finally move the files.

Signed-off-by: OsakaRuma <[email protected]>
  • Loading branch information
@iamscottxu
iamscottxu committed Oct 19, 2024
1 parent 533129e commit e698cb1
Show file tree
Hide file tree
Showing 3 changed files with 241 additions and 12 deletions.
129 changes: 126 additions & 3 deletions src/Starward.Core.ZipStreamDownload/FastZipStreamDownload.partial.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,16 @@ namespace Starward.Core.ZipStreamDownload;
/// <param name="tempDirectoryInfo">解压临时文件夹信息对象</param>
public partial class FastZipStreamDownload(DirectoryInfo targetDirectoryInfo, DirectoryInfo tempDirectoryInfo)
{
/// <summary>
/// 默认ZIP文件的文件名
/// </summary>
private const string DefaultZipFileName = "temp.zip";

/// <summary>
/// 中央目录数据文件的后缀名
/// </summary>
private const string CentralDirectoryDataFileNameExtension = "zipcdr";

/// <summary>
/// 进度改变报告接口
/// </summary>
Expand Down Expand Up @@ -179,14 +189,113 @@ public FastZipStreamDownload(string targetDirectorPath) :
{
}

/// <summary>
/// 下载并获取<see cref="ZipFile"/>的实例，使用该实例可获取实体文件的列表。
/// </summary>
/// <param name="zipFileDownloadFactory"><see cref="IZipFileDownloadFactory"/>的实例</param>
/// <param name="cancellationToken">取消令牌</param>
/// <returns>一个任务，返回一个<see cref="ZipFile"/>的实例。</returns>
private async Task<ZipFile> GetZipFileCentralDirectoryDataFileAsync(IZipFileDownloadFactory zipFileDownloadFactory,
CancellationToken cancellationToken = default)
{
//参数校验
ThrowException.ThrowDirectoryNotFoundExceptionIfDirectoryNotExists(TargetDirectoryInfo);
ThrowException.ThrowDirectoryNotFoundExceptionIfDirectoryNotExists(TempDirectoryInfo);

//不允许执行多次
if (Interlocked.Increment(ref _downloadZipFileIsRunning) > 1)
{
Interlocked.Decrement(ref _downloadZipFileIsRunning);
throw new InvalidOperationException();
}

_entriesExceptionDictionary.Clear();

try
{
var result = await CoreAsync().ConfigureAwait(false);
ProgressReport(ProcessingStageEnum.None, true); //报告全局进度
return result;
}
finally
{
Interlocked.Decrement(ref _downloadZipFileIsRunning);
}

async Task<ZipFile> CoreAsync()
{
var zipFileName =
(zipFileDownloadFactory.ZipFileUri == null ? null : zipFileDownloadFactory.ZipFileUri.GetFileName()) ??
DefaultZipFileName; //如果找不到URL中的文件名，则默认为temp.zip。

var centralDirectoryDataFileInfo =
new FileInfo(Path.Join(TempDirectoryInfo.FullName,
$"{zipFileName}.{CentralDirectoryDataFileNameExtension}"));

var zipFileDownload = zipFileDownloadFactory.GetInstance();

Exception? exception = null;
try
{
return await DownloadAndOpenCentralDirectoryDataFileAsync(zipFileDownload,
centralDirectoryDataFileInfo, (processingStage, completed,
progress, downloadByteCount) => ProgressReport(processingStage, completed,
progress, downloadByteCount) //报告实体进度
, cancellationToken).ConfigureAwait(false);
}
catch (Exception e)
{
exception = e;
centralDirectoryDataFileInfo.Delete(); //删除中央目录文件，下次重新获取
throw;
}
finally
{
ProgressReport(ProcessingStageEnum.None, true, exception: exception);
}
}
}

/// <summary>
/// 获取一个<see cref="ZipEntry"/>的列表，该列表表示一个文件实体的列表。
/// </summary>
/// <param name="zipFileDownloadFactory"><see cref="IZipFileDownloadFactory"/>的实例</param>
/// <param name="cancellationToken">取消令牌</param>
/// <returns>一个任务，返回一个<see cref="ZipEntry"/>的列表。</returns>
public async Task<List<ZipEntry>> GetZipFileFileEntriesAsync(IZipFileDownloadFactory zipFileDownloadFactory,
CancellationToken cancellationToken = default)
{
using var centralDirectoryDataFile =
await GetZipFileCentralDirectoryDataFileAsync(zipFileDownloadFactory, cancellationToken);
return await centralDirectoryDataFile.GetFileEntriesAsync(cancellationToken).ConfigureAwait(false);
}

/// <summary>
/// 获取一个<see cref="ZipEntry"/>的列表，该列表表示一个目录实体的列表。
/// </summary>
/// <param name="zipFileDownloadFactory"><see cref="IZipFileDownloadFactory"/>的实例</param>
/// <param name="cancellationToken">取消令牌</param>
/// <returns>一个任务，返回一个<see cref="ZipEntry"/>的列表。</returns>
public async Task<List<ZipEntry>> GetZipFileDirectoryEntriesAsync(IZipFileDownloadFactory zipFileDownloadFactory,
CancellationToken cancellationToken = default)
{
using var centralDirectoryDataFile =
await GetZipFileCentralDirectoryDataFileAsync(zipFileDownloadFactory, cancellationToken);
return await centralDirectoryDataFile.GetDirectoryEntriesAsync(cancellationToken).ConfigureAwait(false);
}

/// <summary>
/// 开始下载ZIP并解压文件（异步）
/// </summary>
/// <param name="zipFileDownloadFactory"><see cref="IZipFileDownloadFactory"/>的实例</param>
/// <param name="extractFiles">是否对下载好的文件进行解压（只对半流式下载模式生效）</param>
/// <param name="downloadFiles">需要下载或忽略（由<paramref name="ignoreFiles"/>决定）的文件列表，为空下载全部文件</param>
/// <param name="ignoreFiles">在进行文件名比较时是包含还是忽略</param>
/// <param name="ignoreCase">在进行文件名比较时是否忽略大小写</param>
/// <param name="cancellationToken">取消令牌</param>
/// <returns>一个任务。</returns>
public async Task DownloadZipFileAsync(IZipFileDownloadFactory zipFileDownloadFactory, bool extractFiles,
public async Task DownloadZipFileAsync(IZipFileDownloadFactory zipFileDownloadFactory, bool extractFiles = true,
List<string>? downloadFiles = null, bool ignoreFiles = false, bool ignoreCase = false,
CancellationToken cancellationToken = default)
{
//参数校验
Expand Down Expand Up @@ -217,10 +326,11 @@ async Task CoreAsync()
{
var zipFileName =
(zipFileDownloadFactory.ZipFileUri == null ? null : zipFileDownloadFactory.ZipFileUri.GetFileName()) ??
"temp.zip"; //如果找不到URL中的文件名，则默认为temp.zip。
DefaultZipFileName; //如果找不到URL中的文件名，则默认为temp.zip。

var centralDirectoryDataFileInfo =
new FileInfo(Path.Join(TempDirectoryInfo.FullName, $"{zipFileName}.zipcdr"));
new FileInfo(Path.Join(TempDirectoryInfo.FullName,
$"{zipFileName}.{CentralDirectoryDataFileNameExtension}"));

var zipFileDownload = zipFileDownloadFactory.GetInstance();

Expand Down Expand Up @@ -271,6 +381,19 @@ await DownloadAndOpenCentralDirectoryDataFileAsync(zipFileDownload, centralDirec
//下载和解压文件
var fileEntries =
await centralDirectoryDataFile.GetFileEntriesAsync(cancellationToken).ConfigureAwait(false);
if (downloadFiles is { Count: > 0 })
{
var stringComparison = ignoreCase
? StringComparison.InvariantCultureIgnoreCase
: StringComparison.InvariantCulture;
if (ignoreFiles)
fileEntries = fileEntries.Where(
e => downloadFiles.All(f => !f.Equals(ZipEntry.CleanName(e.Name), stringComparison))).ToList();
else
fileEntries = fileEntries.Where(
e => downloadFiles.Any(f => f.Equals(ZipEntry.CleanName(e.Name), stringComparison))).ToList();
}

ProgressReport(ProcessingStageEnum.DownloadingAndExtractingFile, false, entries: fileEntries); //报告全局进度

fileEntries.ForEach(AddEntryTask);
Expand Down
4 changes: 2 additions & 2 deletions src/Starward.Core.ZipStreamDownload/FastZipStreamDownloadTask.partial.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,9 +77,9 @@ private void AddEntryTask(ZipEntry entry)
var entryData = new EntryTaskData
{
Entry = entry,
CompressedFileDirectoryInfo = tempDirectoryInfo,
CompressedFileDirectoryInfo = TempDirectoryInfo,
CompressedFileInfo = compressedFileInfo,
ExtractedFileDirectoryInfo = targetDirectoryInfo,
ExtractedFileDirectoryInfo = TargetDirectoryInfo,
ExtractedFileInfo = targetFileInfo,
ExtractedFileTempInfo = new FileInfo($"{targetFileInfo.FullName}_tmp")
};
Expand Down
120 changes: 113 additions & 7 deletions src/Starward/Services/Download/InstallGameService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
using System.Net.Sockets;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Text.RegularExpressions;
using System.Threading;
Expand All @@ -28,6 +29,7 @@
using Starward.Models;
using Vanara.PInvoke;
using System.Web;
using ICSharpCode.SharpZipLib.Zip;

namespace Starward.Services.Download;

Expand Down Expand Up @@ -1436,6 +1438,10 @@ protected async Task DownloadItemAsync(InstallGameItem item, CancellationToken c

protected async Task StreamDownloadItemAsync(InstallGameItem item, CancellationToken cancellationToken = default)
{
const string deleteFileListFileName = "DeleteFiles.txt";
//const string hdiffFileListFileName = "HdiffFiles.txt";
const string pkgVersionFileName = "*pkg_version";

//不要尝试在执行此任务时执行其他任务，此任务本身为多线程任务，且不支持与其他任务并行。
IZipFileDownloadFactory zipFileDownloadFactory;

Expand Down Expand Up @@ -1464,16 +1470,28 @@ protected async Task StreamDownloadItemAsync(InstallGameItem item, CancellationT
var progress = new FastZipStreamDownloadProgressUtils();

var checkDateTimeVerifying = true;
var checkCrcVerifying = false;

if (InstallTask == InstallGameTask.Update) checkCrcVerifying = true;
else if (InstallTask == InstallGameTask.Repair) checkDateTimeVerifying = false;
DirectoryInfo decompressDirectoryInfo;
var isUpdate = false;
if (InstallTask == InstallGameTask.Update)
{
decompressDirectoryInfo = new DirectoryInfo(Path.Combine(item.DecompressPath,
Path.GetFileName(zipFileDownloadFactory.ZipFileUri!.LocalPath)));
isUpdate = true;
}
else
{
decompressDirectoryInfo = new DirectoryInfo(item.DecompressPath);
if (InstallTask == InstallGameTask.Repair)
checkDateTimeVerifying = false;
}
decompressDirectoryInfo.Create();

var fastZipStreamDownload = new FastZipStreamDownload(item.DecompressPath)
var fastZipStreamDownload = new FastZipStreamDownload(decompressDirectoryInfo)
{
EnableFullStreamDownload = true,
CheckDateTimeVerifyingExistingFile = checkDateTimeVerifying,
CheckCrcVerifyingExistingFile = checkCrcVerifying,
CheckCrcVerifyingExistingFile = false,
DownloadThreadCount = 20,
Progress = progress.Progress
};
Expand All @@ -1489,14 +1507,80 @@ protected async Task StreamDownloadItemAsync(InstallGameItem item, CancellationT
FastZipStreamDownload.ProcessingStageEnum.StreamExtractingFile
})
{
finishBytesBase += item.Size - progress.DownloadBytesTotal.Value;
// ReSharper disable once AccessToModifiedClosure
Interlocked.Add(ref finishBytesBase, item.Size - progress.DownloadBytesTotal.Value);
downloadingStarted = true;
}
_finishBytes = finishBytesBase + progress.DownloadBytesCompleted;
};

var directoryEntries = await fastZipStreamDownload.GetZipFileDirectoryEntriesAsync(zipFileDownloadFactory,
cancellationToken: cancellationToken).ConfigureAwait(false);
var fileEntries = await fastZipStreamDownload.GetZipFileFileEntriesAsync(zipFileDownloadFactory,
cancellationToken: cancellationToken).ConfigureAwait(false);
if (isUpdate && fileEntries.Any(fileEntry =>
{
var fileName = ZipEntry.CleanName(fileEntry.Name);
return fileName.Equals(deleteFileListFileName, StringComparison.OrdinalIgnoreCase);
}))
{
await fastZipStreamDownload.DownloadZipFileAsync(zipFileDownloadFactory, extractFiles: true,
downloadFiles: new List<string> {deleteFileListFileName},
ignoreCase: true, cancellationToken: cancellationToken).ConfigureAwait(false);
var deleteFileListFile = new FileInfo(Path.Combine(decompressDirectoryInfo.FullName,
deleteFileListFileName));
using var deleteFileListFileReader = deleteFileListFile.OpenText();
while (!deleteFileListFileReader.EndOfStream)
{
var file = await deleteFileListFileReader.ReadLineAsync(cancellationToken).ConfigureAwait(false);
if (!string.IsNullOrWhiteSpace(file)) File.Delete(Path.Combine(item.DecompressPath, file));
}
deleteFileListFileReader.Close();
deleteFileListFile.Delete();
var deleteFileListFileEntry = fileEntries.Find(e => ZipEntry.CleanName(e.Name).Equals(
deleteFileListFileName, StringComparison.OrdinalIgnoreCase));
Interlocked.Add(ref finishBytesBase, deleteFileListFileEntry!.CompressedSize);
}
else if (isUpdate)
{
var pkgVersionFiles = new DirectoryInfo(item.DecompressPath)
.GetFiles(pkgVersionFileName, new EnumerationOptions {
MatchCasing = MatchCasing.CaseInsensitive, MaxRecursionDepth = 0
})
.Where(fileInfo => fileEntries.Any(fileEntry =>
fileEntry.Name.Equals(fileInfo.Name, StringComparison.OrdinalIgnoreCase)))
.ToList();
foreach (var pkgVersionFile in pkgVersionFiles)
{
using var deleteFileListFileReader = pkgVersionFile.OpenText();
while (!deleteFileListFileReader.EndOfStream)
{
var gameFileInfoJson =
await deleteFileListFileReader.ReadLineAsync(cancellationToken).ConfigureAwait(false);
if (string.IsNullOrWhiteSpace(gameFileInfoJson)) continue;
string? gameFileRemoteName = null;
try
{
var gameFileInfo = JsonNode.Parse(gameFileInfoJson);
gameFileRemoteName = gameFileInfo?["remoteName"]?.GetValue<string>();
}
catch (JsonException) { /* nothing */ }
if (string.IsNullOrWhiteSpace(gameFileRemoteName)) continue;
File.Delete(Path.Combine(item.DecompressPath, gameFileRemoteName));
}
deleteFileListFileReader.Close();
}
}
if (isUpdate) fileEntries
.Select(fileEntry => ZipEntry.CleanName(fileEntry.Name))
.Where(fileName =>
!fileName.Equals(deleteFileListFileName, StringComparison.OrdinalIgnoreCase))
.Select(fileName => Path.Combine(item.DecompressPath, fileName))
.ToList().ForEach(File.Delete);

await fastZipStreamDownload.DownloadZipFileAsync(zipFileDownloadFactory, extractFiles: true,
await fastZipStreamDownload.DownloadZipFileAsync(zipFileDownloadFactory,
downloadFiles: new List<string> {deleteFileListFileName},
ignoreFiles: true, ignoreCase: true, extractFiles: true,
cancellationToken: cancellationToken).ConfigureAwait(false);

if (progress.DownloadBytesTotal != null) _finishBytes = finishBytesBase + progress.DownloadBytesTotal.Value;
Expand All @@ -1506,6 +1590,28 @@ await fastZipStreamDownload.DownloadZipFileAsync(zipFileDownloadFactory, extract
if (exceptions.Count == 1) throw exceptions.First();
if (exceptions.Count > 1) throw new AggregateException(exceptions);

if (isUpdate)
{
foreach (var directoryEntry in directoryEntries)
{
var directory =
new DirectoryInfo(Path.Combine(item.DecompressPath, ZipEntry.CleanName(directoryEntry.Name)));
directory.Create();
directory.CreationTimeUtc = directory.LastWriteTimeUtc = directoryEntry.DateTime;
}
foreach (var fileEntry in fileEntries)
{
var fileName = ZipEntry.CleanName(fileEntry.Name);
if (fileName.Equals(deleteFileListFileName, StringComparison.OrdinalIgnoreCase)) continue;
var targetPath = Path.Combine(item.DecompressPath, fileName);
var directoryName = Path.GetDirectoryName(targetPath);
if (directoryName != null) Directory.CreateDirectory(directoryName);
var file = new FileInfo(Path.Combine(decompressDirectoryInfo.FullName,
ZipEntry.CleanName(fileEntry.Name)));
file.MoveTo(targetPath, true);
}
decompressDirectoryInfo.Delete(recursive: true);
}

await ApplyDiffFilesAsync(item.DecompressPath).ConfigureAwait(false);
}
Expand Down

0 comments on commit e698cb1

Please sign in to comment.