From a189836006b8696377bc758fca80c6021f6db7c2 Mon Sep 17 00:00:00 2001
From: khhs167 <khhs1671@gmail.com>
Date: Mon, 19 Jun 2023 14:53:06 +0200
Subject: [PATCH 1/2] Added CSV stuff
---
ContentPipe/Content.cs | 71 +++++++++++++++++++++++++++++-
ContentPipe/ContentDirectory.cs | 76 +++++++++++++--------------------
ContentPipe/IContentProvider.cs | 58 +++++++++++++++++++++++--
3 files changed, 154 insertions(+), 51 deletions(-)
diff --git a/ContentPipe/Content.cs b/ContentPipe/Content.cs
index cbd1264..f84a530 100644
--- a/ContentPipe/Content.cs
+++ b/ContentPipe/Content.cs
@@ -1,4 +1,5 @@
using System.Text;
+using System.Text.RegularExpressions;
namespace ContentPipe;
@@ -8,6 +9,19 @@ namespace ContentPipe;
public static class Content
{
private static readonly Dictionary<string, IContentProvider> Providers = new Dictionary<string, IContentProvider>();
+
+ private static readonly Dictionary<string, int> LoadedContent = new Dictionary<string, int>();
+
+ /// <summary>
+ /// If Loads should be logged. This will slow down performance of loading by some amount!
+ /// </summary>
+ public static bool ShouldLogLoads = false;
+
+ /// <summary>
+ /// A filter to run on all log load registrations, in case you want to ignore something.
+ /// This only affects the output log!
+ /// </summary>
+ public static Regex LogLoadIgnoreFilter = new Regex("");
/// <summary>
/// Load a packed content directory(.cpkg file)
@@ -20,6 +34,20 @@ public static void LoadDirectory(string path)
Providers.Add(path, new PacketContentProvider(new ContentDirectory(path)));
}
+ /// <summary>
+ /// Load a packed ContentDirectory where all content has a prefix.
+ /// </summary>
+ /// <param name="path">The path to the cpkg directory, without extension</param>
+ /// <param name="prefix">The prefix to be used for said directory</param>
+ /// <returns>The string to be used when unloading, as prefixing slightly modifies the string</returns>
+ public static string LoadPrefixed(string path, string prefix)
+ {
+ string pfxPath = prefix + path;
+ if(!Providers.ContainsKey(pfxPath))
+ Providers.Add(pfxPath, new PrefixedContentProvider(prefix, new PacketContentProvider(new ContentDirectory(path))));
+ return pfxPath;
+ }
+
/// <summary>
/// Unload a packed content directory
/// </summary>
@@ -52,6 +80,14 @@ public static void LoadPhysicalDirectory(string path)
Providers.Add(path, new PhysicalContentProvider(path));
}
+ private static void RegisterLoad(string resource)
+ {
+ if(!ShouldLogLoads)
+ return;
+ LoadedContent.TryAdd(resource, 0);
+ LoadedContent[resource]++;
+ }
+
/// <summary>
/// Load/Fetch a content lump from loaded directories. Newer loaded directories are fetched from before other directories
/// </summary>
@@ -59,6 +95,7 @@ public static void LoadPhysicalDirectory(string path)
/// <returns>The content lump loaded, or null if it is not available</returns>
public static ContentLump? Load(string resource)
{
+ RegisterLoad(resource);
foreach (var provider in Providers.Values)
{
ContentLump? lump = provider.Load(resource);
@@ -68,7 +105,7 @@ public static void LoadPhysicalDirectory(string path)
}
return null;
}
-
+
/// <summary>
/// Load/Fetch a content lump from all loaded directories. Newer loaded directories are fetched from before other directories
/// </summary>
@@ -76,6 +113,7 @@ public static void LoadPhysicalDirectory(string path)
/// <returns>The content lumps loaded</returns>
public static ContentLump[] LoadAll(string resource)
{
+ RegisterLoad(resource);
List<ContentLump> contentLumps = new List<ContentLump>();
foreach (var provider in Providers.Values.Reverse())
{
@@ -178,4 +216,35 @@ public static string[] LoadAllStrings(string resource)
return strings.ToArray();
}
+
+ /// <summary>
+ /// Get the load log if one is collected, formatted as CSV
+ /// </summary>
+ /// <returns>The load log</returns>
+ public static string WriteLoadLog(bool includeDeadResources = false)
+ {
+ StringWriter stringWriter = new StringWriter();
+ stringWriter.WriteLine("File,Loads");
+
+ if (includeDeadResources)
+ {
+ foreach (var provider in Providers)
+ {
+ string[] resources = provider.Value.GetContent();
+ foreach (var res in resources)
+ {
+ if(!String.IsNullOrWhiteSpace(res))
+ LoadedContent.TryAdd(res, 0);
+ }
+ }
+ }
+ var lc = LoadedContent.OrderByDescending(l => l.Value);
+ foreach (var load in lc)
+ {
+ if(!LogLoadIgnoreFilter.IsMatch(load.Key))
+ stringWriter.WriteLine(load.Key + "," + load.Value);
+ }
+
+ return stringWriter.ToString();
+ }
}
diff --git a/ContentPipe/ContentDirectory.cs b/ContentPipe/ContentDirectory.cs
index 77e0537..5a01a67 100644
--- a/ContentPipe/ContentDirectory.cs
+++ b/ContentPipe/ContentDirectory.cs
@@ -8,8 +8,9 @@ namespace ContentPipe;
/// </summary>
public struct ContentDirectory
{
- private Dictionary<string, ContentLump> contentLumps = new Dictionary<string, ContentLump>();
+ private ZipArchive archive;
+
/// <summary>
/// Fetch a content lump, returns null if it is not available
/// </summary>
@@ -18,11 +19,24 @@ public ContentLump? this[string name]
{
get
{
- if (contentLumps.ContainsKey(name))
- return contentLumps[name];
- return null;
+ ZipArchiveEntry? entry = archive.GetEntry(name);
+ if (entry == null)
+ return null;
+
+ string tempFile = Directory.GetCurrentDirectory() + "/temp-read";
+
+ entry.ExtractToFile(tempFile);
+ ContentLump contentLump = new()
+ {
+ Name = name,
+ Data = File.ReadAllBytes(tempFile)
+ };
+ File.Delete(tempFile);
+
+ return contentLump;
}
}
+
/// <summary>
/// Compress a directory into a .cpkg file. It will be stored in the same directory as the source directory,
@@ -32,32 +46,9 @@ public ContentLump? this[string name]
/// <param name="path">The path to the directory to compress</param>
public static void CompressDirectory(string path)
{
-
- if (!Directory.Exists(path))
- return;
-
- string[] files = Directory.GetFiles(path, "*", SearchOption.AllDirectories);
- string filePath = Path.Combine(Path.GetDirectoryName(path)!, Path.GetFileName(path)) + ".cpkg";
-
- List<ContentLump> data = new List<ContentLump>();
-
- foreach (var file in files)
- {
- string localPath = Path.GetRelativePath(path, file);
- ContentLump lump = new ContentLump();
- lump.Name = localPath;
- lump.Data = File.ReadAllBytes(file);
- data.Add(lump);
- }
-
- byte[] compressedData = MessagePackSerializer.Serialize(data.ToArray());
-
- FileStream fileStream = File.Open(filePath, FileMode.OpenOrCreate);
- DeflateStream gZipStream = new DeflateStream(fileStream, CompressionLevel.SmallestSize, false);
- gZipStream.Write(compressedData);
- gZipStream.Close();
- fileStream.Close();
-
+ if(File.Exists(path + ".cpkg"))
+ File.Delete(path + ".cpkg");
+ ZipFile.CreateFromDirectory(path, path + ".cpkg");
}
/// <summary>
@@ -66,23 +57,16 @@ public static void CompressDirectory(string path)
/// <param name="path">The path to the file, excluding the extension</param>
public ContentDirectory(string path)
{
-
- FileStream fileStream = File.Open(path + ".cpkg", FileMode.Open);
- DeflateStream gZipStream = new DeflateStream(fileStream, CompressionMode.Decompress, false);
-
- MemoryStream memoryStream = new MemoryStream();
- gZipStream.CopyTo(memoryStream);
- gZipStream.Close();
- fileStream.Close();
-
- ContentLump[] data = MessagePackSerializer.Deserialize<ContentLump[]>(memoryStream.ToArray());
-
- foreach (var lump in data)
+ string fileName = path + ".cpkg";
+ archive = ZipFile.OpenRead(fileName);
+ Content = new string[archive.Entries.Count];
+ for (int i = 0; i < archive.Entries.Count; i++)
{
- contentLumps.Add(lump.Name, lump);
+ Content[i] = archive.Entries[i].FullName;
}
-
}
+
+ public readonly string[] Content;
}
/// <summary>
@@ -94,12 +78,12 @@ public struct ContentLump
/// <summary>
/// The name of the content lump
/// </summary>
- [Key(0)] public string Name;
+ public string Name;
/// <summary>
/// The data of the ContentLump
/// </summary>
- [Key(1)] public byte[] Data = Array.Empty<byte>();
+ public byte[] Data = Array.Empty<byte>();
/// <summary>
/// Create a content lump with 0:ed fields.
diff --git a/ContentPipe/IContentProvider.cs b/ContentPipe/IContentProvider.cs
index 10326ca..018fc04 100644
--- a/ContentPipe/IContentProvider.cs
+++ b/ContentPipe/IContentProvider.cs
@@ -3,11 +3,44 @@ namespace ContentPipe;
internal interface IContentProvider
{
public ContentLump? Load(string name);
+ public string[] GetContent();
}
-internal struct PacketContentProvider : IContentProvider
+internal readonly struct PrefixedContentProvider : IContentProvider
{
- private ContentDirectory directory;
+ private readonly IContentProvider provider;
+ private readonly string prefix;
+
+ public PrefixedContentProvider(string prefix, IContentProvider provider)
+ {
+ this.prefix = prefix;
+ this.provider = provider;
+ }
+
+ public ContentLump? Load(string name)
+ {
+ if (!name.StartsWith(prefix))
+ return null;
+
+ string deAliased = name[prefix.Length..];
+ return provider.Load(deAliased);
+ }
+
+ public string[] GetContent()
+ {
+ string[] content = provider.GetContent();
+ for (int i = 0; i < content.Length; i++)
+ {
+ content[i] = prefix + content[i];
+ }
+ return content;
+ }
+
+}
+
+internal readonly struct PacketContentProvider : IContentProvider
+{
+ private readonly ContentDirectory directory;
public PacketContentProvider(ContentDirectory directory)
{
this.directory = directory;
@@ -17,11 +50,16 @@ public PacketContentProvider(ContentDirectory directory)
{
return directory[name];
}
+
+ public string[] GetContent()
+ {
+ return directory.Content;
+ }
}
-internal struct PhysicalContentProvider : IContentProvider
+internal readonly struct PhysicalContentProvider : IContentProvider
{
- private string directory;
+ private readonly string directory;
public PhysicalContentProvider(string directory)
{
this.directory = directory;
@@ -39,4 +77,16 @@ public PhysicalContentProvider(string directory)
}
return null;
}
+
+ public string[] GetContent()
+ {
+ string[] files = Directory.GetFiles(directory, "*", SearchOption.AllDirectories);
+
+ for (int i = 0; i < files.Length; i++)
+ {
+ files[i] = Path.GetRelativePath(directory, files[i]);
+ }
+
+ return files;
+ }
}
\ No newline at end of file
From ec25440e70a96da9b571c9a80c7968a8cf2c9eab Mon Sep 17 00:00:00 2001
From: khhs167 <khhs1671@gmail.com>
Date: Fri, 30 Jun 2023 16:29:48 +0200
Subject: [PATCH 2/2] Minor logging changes afaik
---
ContentPipe/Extras.cs | 31 +++++++++++++++++++++++++++++++
1 file changed, 31 insertions(+)
create mode 100644 ContentPipe/Extras.cs
diff --git a/ContentPipe/Extras.cs b/ContentPipe/Extras.cs
new file mode 100644
index 0000000..67e68f9
--- /dev/null
+++ b/ContentPipe/Extras.cs
@@ -0,0 +1,31 @@
+using System.Security.Cryptography;
+using System.Text;
+
+namespace ContentPipe;
+
+internal static class Extras
+{
+ public static string ReadTerminatedString(this BinaryReader reader)
+ {
+ string s = "";
+ char c;
+ while ((c = reader.ReadChar()) != '\0')
+ s += c;
+ return s;
+ }
+
+ public static byte[] GetHash(this string inputString)
+ {
+ using (HashAlgorithm algorithm = SHA256.Create())
+ return algorithm.ComputeHash(Encoding.UTF8.GetBytes(inputString));
+ }
+
+ public static string GetHashString(this string inputString)
+ {
+ StringBuilder sb = new StringBuilder();
+ foreach (byte b in GetHash(inputString))
+ sb.Append(b.ToString("X2"));
+
+ return sb.ToString();
+ }
+}