Skip to content
This repository has been archived by the owner on Jul 5, 2024. It is now read-only.

Commit

Permalink
poc vcdiff
Browse files Browse the repository at this point in the history
  • Loading branch information
caesay committed Dec 24, 2023
1 parent f3d7b12 commit cff76e0
Show file tree
Hide file tree
Showing 33 changed files with 5,214 additions and 19 deletions.
8 changes: 7 additions & 1 deletion Squirrel.sln
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Squirrel.Deployment", "src\
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Squirrel.Packaging.Tests", "test\Squirrel.Packaging.Tests\Squirrel.Packaging.Tests.csproj", "{175B06A5-5C09-4DAB-A6AF-C8A2257BD1B6}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestApp", "test\TestApp\TestApp.csproj", "{784B5987-2E71-4AEE-81B9-E0CC7F1DBEB3}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestApp", "test\TestApp\TestApp.csproj", "{784B5987-2E71-4AEE-81B9-E0CC7F1DBEB3}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Squirrel.VCDiff", "src\Squirrel.VCDiff\Squirrel.VCDiff.csproj", "{302EA080-C3AE-42E5-BB19-E628E893C748}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Expand Down Expand Up @@ -80,6 +82,10 @@ Global
{784B5987-2E71-4AEE-81B9-E0CC7F1DBEB3}.Debug|Any CPU.Build.0 = Debug|Any CPU
{784B5987-2E71-4AEE-81B9-E0CC7F1DBEB3}.Release|Any CPU.ActiveCfg = Release|Any CPU
{784B5987-2E71-4AEE-81B9-E0CC7F1DBEB3}.Release|Any CPU.Build.0 = Release|Any CPU
{302EA080-C3AE-42E5-BB19-E628E893C748}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{302EA080-C3AE-42E5-BB19-E628E893C748}.Debug|Any CPU.Build.0 = Debug|Any CPU
{302EA080-C3AE-42E5-BB19-E628E893C748}.Release|Any CPU.ActiveCfg = Release|Any CPU
{302EA080-C3AE-42E5-BB19-E628E893C748}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down
30 changes: 14 additions & 16 deletions src/Squirrel.Packaging/DeltaPackageBuilder.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
using System.Text;
using Squirrel.Compression;
using Microsoft.Extensions.Logging;
using NuGet.Protocol.Plugins;
using VCDiff.Encoders;

namespace Squirrel.Packaging;

Expand Down Expand Up @@ -66,11 +68,6 @@ public ReleasePackageBuilder CreateDeltaPackage(ReleasePackageBuilder basePackag

int fNew = 0, fSame = 0, fChanged = 0, fWarnings = 0;

bool bytesAreIdentical(ReadOnlySpan<byte> a1, ReadOnlySpan<byte> a2)
{
return a1.SequenceEqual(a2);
}

void createDeltaForSingleFile(FileInfo targetFile, DirectoryInfo workingDirectory)
{
// NB: There are three cases here that we'll handle:
Expand All @@ -95,30 +92,31 @@ void createDeltaForSingleFile(FileInfo targetFile, DirectoryInfo workingDirector
var oldFilePath = baseLibFiles[relativePath];
_logger.Debug($"Delta patching {oldFilePath} => {targetFile.FullName}");

var oldData = File.ReadAllBytes(oldFilePath);
var newData = File.ReadAllBytes(targetFile.FullName);

if (bytesAreIdentical(oldData, newData)) {
var comparer = new FileComparer(oldFilePath, targetFile.FullName);
if (comparer.Compare()) {
// 2. exists in both, keep it the same
_logger.Debug($"{relativePath} hasn't changed, writing dummy file");
File.Create(targetFile.FullName + ".bsdiff").Dispose();
File.Create(targetFile.FullName + ".vcdiff").Dispose();
File.Create(targetFile.FullName + ".shasum").Dispose();
fSame++;
} else {
// 3. changed, write a delta in new
using (FileStream of = File.Create(targetFile.FullName + ".bsdiff")) {
BinaryPatchUtility.Create(oldData, newData, of);
}
var rl = ReleaseEntry.GenerateFromFile(new MemoryStream(newData), targetFile.Name + ".shasum");
const int FS_BUFFER_SIZE = 1000000; // 1mb
using var source = new FileStream(oldFilePath, FileMode.Open, FileAccess.Read, FileShare.Read, FS_BUFFER_SIZE);
using var target = new FileStream(targetFile.FullName, FileMode.Open, FileAccess.Read, FileShare.Read, FS_BUFFER_SIZE);
using var output = File.Create(targetFile.FullName + ".vcdiff");
using var encoder = new VcEncoder(source, target, output, 1, 32);
encoder.Encode();
target.Seek(0, SeekOrigin.Begin);
var rl = ReleaseEntry.GenerateFromFile(target, targetFile.Name + ".shasum");
File.WriteAllText(targetFile.FullName + ".shasum", rl.EntryAsString, Encoding.UTF8);
fChanged++;
}
targetFile.Delete();
baseLibFiles.Remove(relativePath);
} catch (Exception ex) {
_logger.Debug(ex, String.Format("Failed to create a delta for {0}", targetFile.Name));
Utility.DeleteFileOrDirectoryHard(targetFile.FullName + ".bsdiff", throwOnFailure: false);
Utility.DeleteFileOrDirectoryHard(targetFile.FullName + ".diff", throwOnFailure: false);
Utility.DeleteFileOrDirectoryHard(targetFile.FullName + ".vcdiff", throwOnFailure: false);
Utility.DeleteFileOrDirectoryHard(targetFile.FullName + ".shasum", throwOnFailure: false);
fWarnings++;
throw;
Expand Down
190 changes: 190 additions & 0 deletions src/Squirrel.Packaging/FileComparer.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.Intrinsics.X86;
using System.Runtime.Intrinsics;
using System.Text;
using System.Threading.Tasks;
using System.Runtime.CompilerServices;
using System.Numerics;

namespace Squirrel.Packaging
{
public class FileComparer
{
/// <summary>
/// Fileinfo for source file
/// </summary>
protected readonly FileInfo FileInfo1;

/// <summary>
/// Fileinfo for target file
/// </summary>
protected readonly FileInfo FileInfo2;

/// <summary>
/// Base class for creating a file comparer
/// </summary>
/// <param name="filePath01">Absolute path to source file</param>
/// <param name="filePath02">Absolute path to target file</param>
public FileComparer(string filePath01, string filePath02)
{
FileInfo1 = new FileInfo(filePath01);
FileInfo2 = new FileInfo(filePath02);
EnsureFilesExist();
}

const int FS_BUFFER_SIZE = 1000000; // 1mb
const int BUFFER_SIZE = 4096;
const int EQMASK = unchecked((int) (0b1111_1111_1111_1111_1111_1111_1111_1111));
const int AvxRegisterSize = 32;
const int SseRegisterSize = 16;

/// <summary>
/// Compares the two given files and returns true if the files are the same
/// </summary>
/// <returns>true if the files are the same, false otherwise</returns>
public unsafe bool Compare()
{
if (IsDifferentLength()) {
return false;
}

if (IsSameFile()) {
return true;
}

using var stream1 = new FileStream(FileInfo1.FullName, FileMode.Open, FileAccess.Read, FileShare.Read, FS_BUFFER_SIZE);
using var stream2 = new FileStream(FileInfo2.FullName, FileMode.Open, FileAccess.Read, FileShare.Read, FS_BUFFER_SIZE);

long length = FileInfo1.Length;
long totalRead = 0;
var buffer1 = new byte[BUFFER_SIZE];
var buffer2 = new byte[BUFFER_SIZE];
fixed (byte* oh1 = buffer1)
fixed (byte* oh2 = buffer2) {
while (totalRead < length) {
var count1 = ReadIntoBuffer(stream1, buffer1);
var count2 = ReadIntoBuffer(stream2, buffer2);
if (count1 != count2) {
return false;
}
if (count1 == 0) {
return true;
}
if (!BlockContentsMatch(oh1, oh2, count1)) {
return false;
}
totalRead += count1;
}
}
return true;
}

protected int ReadIntoBuffer(in Stream stream, in byte[] buffer)
{
var bytesRead = 0;
while (bytesRead < buffer.Length) {
var read = stream.Read(buffer, bytesRead, buffer.Length - bytesRead);
// Reached end of stream.
if (read == 0) {
return bytesRead;
}
bytesRead += read;
}
return bytesRead;
}

[SkipLocalsInit]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe bool BlockContentsMatch(byte* sourcePtr, byte* targetPtr, int length)
{
int sOffset = 0;
int tOffset = 0;
byte* sPtr = sourcePtr;
byte* tPtr = targetPtr;
int yetToExamine = length;

if (Avx2.IsSupported && yetToExamine >= AvxRegisterSize) {
while (yetToExamine >= AvxRegisterSize) {
Vector256<byte> lv = Avx.LoadVector256(&sPtr[sOffset]);
Vector256<byte> rv = Avx.LoadVector256(&tPtr[tOffset]);
if (Avx2.MoveMask(Avx2.CompareEqual(lv, rv)) != EQMASK)
return false;

sOffset += AvxRegisterSize;
tOffset += AvxRegisterSize;
yetToExamine -= AvxRegisterSize;
}
}

if (Sse2.IsSupported && yetToExamine >= SseRegisterSize) {
while (yetToExamine >= SseRegisterSize) {
Vector128<byte> lv = Sse2.LoadVector128(&sPtr[sOffset]);
Vector128<byte> rv = Sse2.LoadVector128(&tPtr[tOffset]);
if ((uint) Sse2.MoveMask(Sse2.CompareEqual(lv, rv)) != ushort.MaxValue)
return false;

sOffset += SseRegisterSize;
tOffset += SseRegisterSize;
yetToExamine -= SseRegisterSize;
}
}

int vectorSize = Vector<byte>.Count;
if (yetToExamine >= vectorSize) {
var sBuf = new Span<byte>(sourcePtr, length);
var tBuf = new Span<byte>(targetPtr, length);

while (yetToExamine >= vectorSize) {
Vector<byte> lv = new Vector<byte>(sBuf.Slice(sOffset));
Vector<byte> rv = new Vector<byte>(tBuf.Slice(tOffset));
if (!Vector.EqualsAll(lv, rv))
return false;

sOffset += vectorSize;
tOffset += vectorSize;
yetToExamine -= vectorSize;
}
}

while (yetToExamine > 0) {
if (sPtr[sOffset] != tPtr[tOffset])
return false;

--yetToExamine;
++sOffset;
++tOffset;
}

return true;
}

private bool IsSameFile()
{
return string.Equals(FileInfo1.FullName, FileInfo2.FullName, StringComparison.OrdinalIgnoreCase);
}

/// <summary>
/// Does an early comparison by checking files Length, if lengths are not the same, files are definetely different
/// </summary>
/// <returns>true if different length</returns>
private bool IsDifferentLength()
{
return FileInfo1.Length != FileInfo2.Length;
}

/// <summary>
/// Makes sure files exist
/// </summary>
private void EnsureFilesExist()
{
if (FileInfo1.Exists == false) {
throw new ArgumentNullException(nameof(FileInfo1));
}
if (FileInfo2.Exists == false) {
throw new ArgumentNullException(nameof(FileInfo2));
}
}
}
}
1 change: 1 addition & 0 deletions src/Squirrel.Packaging/Squirrel.Packaging.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<NoWarn>$(NoWarn);CA2007;CS8002</NoWarn>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>

<ItemGroup>
Expand Down
Loading

0 comments on commit cff76e0

Please sign in to comment.