Skip to content

Commit

Permalink
Debloat the libs subdirectory (#46)
Browse files Browse the repository at this point in the history
Deduplicate assemblies and share the same file across targets where it makes sense.
  • Loading branch information
MichalStrehovsky authored Nov 5, 2022
1 parent 39e49b7 commit 7edb4f4
Show file tree
Hide file tree
Showing 5 changed files with 142 additions and 14 deletions.
6 changes: 6 additions & 0 deletions .github/workflows/cicd.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,12 @@ jobs:
- name: Build (CD)
if: ${{ github.event.inputs.version != '' }}
run: dotnet build src/bflat/bflat.csproj -t:BuildLayouts -c:Release -p:Version=${{ github.event.inputs.version }}
- name: Debloat the libs
run: |
dotnet run --project src/debloat/debloat.csproj layouts/windows-x64/lib
dotnet run --project src/debloat/debloat.csproj layouts/windows-arm64/lib
dotnet run --project src/debloat/debloat.csproj layouts/linux-glibc-x64/lib
dotnet run --project src/debloat/debloat.csproj layouts/linux-glibc-arm64/lib
- name: Prepare symbols
run: |
mv layouts/windows-x64/bflat.pdb layouts/obj/windows-x64/
Expand Down
24 changes: 17 additions & 7 deletions src/bflat/BuildCommand.cs
Original file line number Diff line number Diff line change
Expand Up @@ -350,29 +350,39 @@ public override int Handle(ParseResult result)
{
char separator = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ';' : ':';

string currentLibPath = Path.Combine(homePath, "lib");

libPath = currentLibPath;

string osPart = targetOS switch
{
TargetOS.Linux when libc == "bionic" => "linux-bionic",
TargetOS.Linux => "linux-glibc",
TargetOS.Linux => "linux",
TargetOS.Windows => "windows",
_ => throw new Exception(targetOS.ToString()),
};
currentLibPath = Path.Combine(currentLibPath, osPart);
libPath = currentLibPath + separator + libPath;

string archPart = targetArchitecture switch
{
TargetArchitecture.ARM64 => "arm64",
TargetArchitecture.X64 => "x64",
_ => throw new Exception(targetArchitecture.ToString()),
};
currentLibPath = Path.Combine(currentLibPath, archPart);
libPath = currentLibPath + separator + libPath;

string osArchPath = Path.Combine(homePath, "lib", $"{osPart}-{archPart}");
if (!Directory.Exists(osArchPath))
if (targetOS == TargetOS.Linux)
{
Console.Error.WriteLine($"Directory '{osArchPath}' doesn't exist.");
return 1;
currentLibPath = Path.Combine(currentLibPath, libc ?? "glibc");
libPath = currentLibPath + separator + libPath;
}

libPath = String.Concat(osArchPath, separator.ToString(), Path.Combine(homePath, "lib"));
if (!Directory.Exists(currentLibPath))
{
Console.Error.WriteLine($"Directory '{currentLibPath}' doesn't exist.");
return 1;
}
}

if (!bare)
Expand Down
24 changes: 17 additions & 7 deletions src/bflat/bflat.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
<RuntimeVersion>7.0.0-rtm.22529.1</RuntimeVersion>
<RuntimeUri>https://github.com/bflattened/runtime/releases/download/</RuntimeUri>

<BlobsVersion>1.2</BlobsVersion>
<BlobsVersion>1.3</BlobsVersion>
<BlobsUri>https://github.com/bflattened/blobs/releases/download/</BlobsUri>

<MicrosoftCodeAnalysisCSharpVersion>4.4.0-1.final</MicrosoftCodeAnalysisCSharpVersion>
Expand Down Expand Up @@ -62,11 +62,21 @@
</ItemGroup>

<ItemGroup>
<SupportedTarget Include="windows-x64" />
<SupportedTarget Include="windows-arm64" />
<SupportedTarget Include="linux-glibc-x64" />
<SupportedTarget Include="linux-glibc-arm64" />
<SupportedTarget Include="linux-bionic-arm64" />
<SupportedTarget Include="windows-x64">
<LibPath>windows\x64\</LibPath>
</SupportedTarget>
<SupportedTarget Include="windows-arm64">
<LibPath>windows\arm64\</LibPath>
</SupportedTarget>
<SupportedTarget Include="linux-glibc-x64">
<LibPath>linux\x64\glibc\</LibPath>
</SupportedTarget>
<SupportedTarget Include="linux-glibc-arm64">
<LibPath>linux\arm64\glibc</LibPath>
</SupportedTarget>
<SupportedTarget Include="linux-bionic-arm64">
<LibPath>linux\arm64\bionic</LibPath>
</SupportedTarget>
</ItemGroup>

<ItemGroup>
Expand Down Expand Up @@ -146,7 +156,7 @@
Outputs="@(SupportedTarget->'$(IntermediateOutputPath)\bflat-libs-%(Identity)-$(RuntimeVersion).semaphore')">

<Unzip SourceFiles="$(DownloadCachePath)bflat-libs-%(SupportedTarget.Identity)-$(RuntimeVersion).zip"
DestinationFolder="$(OutputPath)lib\%(SupportedTarget.Identity)" />
DestinationFolder="$(OutputPath)lib\%(SupportedTarget.LibPath)" />

<Touch Files="$(IntermediateOutputPath)\bflat-libs-%(SupportedTarget.Identity)-$(RuntimeVersion).semaphore"
AlwaysCreate="true" />
Expand Down
94 changes: 94 additions & 0 deletions src/debloat/Program.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
// bflat C# compiler
// Copyright (C) 2021-2022 Michal Strehovsky
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Reflection.Metadata;
using System.Reflection.PortableExecutable;

Merge(args[0]);

static void Merge(string directory)
{
foreach (string subDirectory in Directory.EnumerateDirectories(directory))
Merge(subDirectory);

Dictionary<string, List<string>> candidates = new Dictionary<string, List<string>>();
int expectedFiles = 0;
foreach (string subDirectory in Directory.EnumerateDirectories(directory))
{
expectedFiles++;
foreach (var f in Directory.EnumerateFiles(subDirectory, "*.dll"))
{
string key = Path.GetFileName(f);
if (!candidates.TryGetValue(key, out List<string> list))
candidates.Add(key, list = new List<string>());
list.Add(f);
}
}

foreach (var maybeSameFiles in candidates.Values)
{
if (maybeSameFiles.Count != expectedFiles)
continue;

bool allSame = true;
ReadOnlySpan<byte> expected = SanitizedAssemblyBytes(maybeSameFiles[0]);
for (int i = 1; i < maybeSameFiles.Count; i++)
{
ReadOnlySpan<byte> actual = SanitizedAssemblyBytes(maybeSameFiles[i]);
if (!expected.SequenceEqual(actual))
{
allSame = false;
break;
}
}

if (allSame)
{
File.Move(maybeSameFiles[0], Path.Combine(directory, Path.GetFileName(maybeSameFiles[0])));
string pdbFile = Path.ChangeExtension(maybeSameFiles[0], "pdb");
if (File.Exists(pdbFile))
File.Move(pdbFile, Path.Combine(directory, Path.GetFileName(pdbFile)));

for (int i = 1; i < maybeSameFiles.Count; i++)
{
File.Delete(maybeSameFiles[i]);
pdbFile = Path.ChangeExtension(maybeSameFiles[i], "pdb");
if (File.Exists(pdbFile))
File.Delete(pdbFile);
}
}
}
}

static byte[] SanitizedAssemblyBytes(string fileName)
{
byte[] b = File.ReadAllBytes(fileName);
Span<byte> span = b;
PEReader perdr = new PEReader(ImmutableArray.Create(b));
span.Slice(perdr.PEHeaders.CoffHeaderStartOffset + 4, 4).Clear();
MetadataReader mdrdr = perdr.GetMetadataReader();
Guid mvid = mdrdr.GetGuid(mdrdr.GetModuleDefinition().Mvid);
span.Slice(span.IndexOf(mvid.ToByteArray()), 16).Clear();
foreach (var ddentry in perdr.ReadDebugDirectory())
span.Slice(ddentry.DataPointer, ddentry.DataSize).Clear();
if (perdr.PEHeaders.TryGetDirectoryOffset(perdr.PEHeaders.PEHeader.DebugTableDirectory, out int debugDir))
span.Slice(debugDir, perdr.PEHeaders.PEHeader.DebugTableDirectory.Size).Clear();
return b;
}
8 changes: 8 additions & 0 deletions src/debloat/debloat.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net6.0</TargetFramework>
</PropertyGroup>

</Project>

0 comments on commit 7edb4f4

Please sign in to comment.