add file sizes and hashes (closes #5)

QuiltMC · Feb 16, 2025 · 92300e1 · 92300e1
1 parent 6472173
commit 92300e1
Show file tree

Hide file tree

Showing 15 changed files with 135 additions and 72 deletions.
diff --git a/...main/java/org/quiltmc/meta/update/collector/v3/HashedMojmapAndGameVersionCollectorV3.java b/...main/java/org/quiltmc/meta/update/collector/v3/HashedMojmapAndGameVersionCollectorV3.java
@@ -32,7 +32,7 @@ public String getName() {
 
     @Override
     public CompletableFuture<Void> gatherData(BiConsumer<String, FileInfo> output, Executor executor) {
-        return CompletableFuture.supplyAsync(() -> getVersions(maven), executor).thenAcceptAsync(hashed -> {
+        return getVersions(maven, executor).thenAcceptAsync(hashed -> {
             var minecraftMeta = MinecraftMeta.get(Constants.GSON);
 
             hashed.forEach((gameVersion, version) -> {
@@ -55,8 +55,11 @@ public CompletableFuture<Void> gatherData(BiConsumer<String, FileInfo> output, E
         }, executor);
     }
 
-    public static Map<String, HashedMojmapVersionV3> getVersions(MavenRepository maven) {
-        return maven.getMetadata(Constants.QUILT_MAVEN_GROUP, "hashed").stream().map(artifact -> new HashedMojmapVersionV3(artifact.mavenId(), artifact.version)).collect(Collectors.toMap(HashedMojmapVersionV3::version, Function.identity()));
+    public static CompletableFuture<Map<String, HashedMojmapVersionV3>> getVersions(MavenRepository maven, Executor executor) {
+        return CompletableFuture.supplyAsync(() -> maven.getMetadata(Constants.QUILT_MAVEN_GROUP, "hashed"), executor).thenApply(metadata -> metadata.stream().map(artifact -> {
+            var fileInfo = artifact.fileInfo(executor).join();
+            return new HashedMojmapVersionV3(artifact.mavenId(), artifact.version, fileInfo.fileSize(), fileInfo.formatHashesForJson());
+        }).collect(Collectors.toMap(HashedMojmapVersionV3::version, Function.identity())));
     }
 
     public static List<GameVersionV3> getGameVersions(MinecraftMeta minecraftMeta, Predicate<String> isValidVersion) {

diff --git a/src/main/java/org/quiltmc/meta/update/collector/v3/LauncherProfileCollectorV3.java b/src/main/java/org/quiltmc/meta/update/collector/v3/LauncherProfileCollectorV3.java
@@ -38,7 +38,7 @@ public String getName() {
     public CompletableFuture<Void> gatherData(BiConsumer<String, FileInfo> output, Executor executor) {
         var time = Instant.now();
 
-        return QuiltLauncherMetadataCollectorV3.getVersions(quiltMaven, fabricMaven, executor).thenAcceptAsync(gameLauncherVersions -> {
+        return QuiltLauncherMetadataCollectorV3.getVersions(quiltMaven, fabricMaven, executor).thenAccept(gameLauncherVersions -> {
             gameLauncherVersions.forEach((gameVersion, launcherMetadata) -> {
                 var hashed = new LauncherProfileV3.Library(launcherMetadata.hashed().maven(), quiltMaven.url().toString());
                 var intermediary = new LauncherProfileV3.Library(launcherMetadata.intermediary().maven(), fabricMaven.url().toString());
@@ -50,12 +50,13 @@ public CompletableFuture<Void> gatherData(BiConsumer<String, FileInfo> output, E
 
                     var mainClass = Optional.ofNullable(launcherMetadata.launcherMeta().mainClass()).map(map -> map.get(profileType.getName())).orElse(null);
                     var serverLauncherMainClass = mainClass != null ? launcherMetadata.launcherMeta().mainClass().get("serverLauncher") : null;
+                    var profileId = "quilt-loader-%s-%s".formatted(launcherMetadata.loader().version(), gameVersion);
 
                     // need to compute the hash with constant timestamps, otherwise all profiles would be constantly flagged as needing updates
-                    var profileForCache = profileType.create("quilt-loader-%s-%s".formatted(launcherMetadata.loader().version(), gameVersion), gameVersion, LauncherProfileV3.TYPE_RELEASE, mainClass, serverLauncherMainClass, Map.of("game", List.of()), libraries, Instant.EPOCH, Instant.EPOCH);
+                    var profileForCache = profileType.create(profileId, gameVersion, LauncherProfileV3.TYPE_RELEASE, mainClass, serverLauncherMainClass, Map.of("game", List.of()), libraries, Instant.EPOCH, Instant.EPOCH);
                     var hash = HashFunction.SHA1.createHash(Constants.GSON.toJson(profileForCache).getBytes(StandardCharsets.UTF_8));
 
-                    var profile = profileType.create("quilt-loader-%s-%s".formatted(launcherMetadata.loader().version(), gameVersion), gameVersion, LauncherProfileV3.TYPE_RELEASE, mainClass, serverLauncherMainClass, Map.of("game", List.of()), libraries, time, time);
+                    var profile = profileType.create(profileId, gameVersion, LauncherProfileV3.TYPE_RELEASE, mainClass, serverLauncherMainClass, Map.of("game", List.of()), libraries, time, time);
                     var profileBytes = Constants.GSON.toJson(profile).getBytes(StandardCharsets.UTF_8);
 
                     //TODO change path in v4: profiles/<mcVer>/<loaderVer>/client + profiles/<mcVer>/<loaderVer>/server
@@ -64,7 +65,7 @@ public CompletableFuture<Void> gatherData(BiConsumer<String, FileInfo> output, E
             });
 
             LOGGER.info("Generated {} launch profiles", gameLauncherVersions.size() * ProfileType.values().length);
-        }, executor);
+        });
     }
 
     private enum ProfileType {

diff --git a/src/main/java/org/quiltmc/meta/update/collector/v3/QuiltInstallerVersionCollectorV3.java b/src/main/java/org/quiltmc/meta/update/collector/v3/QuiltInstallerVersionCollectorV3.java
@@ -26,14 +26,17 @@ public String getName() {
 
     @Override
     public CompletableFuture<Void> gatherData(BiConsumer<String, FileInfo> output, Executor executor) {
-        return CompletableFuture.supplyAsync(() -> getVersions(quiltMaven), executor).thenAcceptAsync(versions -> {
+        return getVersions(quiltMaven, executor).thenAccept(versions -> {
             output.accept("v3/versions/installer", FileInfo.jsonList(versions));
 
             LOGGER.info("Found {} installer versions", versions.size());
-        }, executor);
+        });
     }
 
-    public static List<QuiltInstallerVersionV3> getVersions(MavenRepository maven) {
-        return maven.getMetadata(Constants.QUILT_MAVEN_GROUP, "quilt-installer").stream().map(artifact -> new QuiltInstallerVersionV3(artifact.mavenId(), artifact.version, artifact.url().toString())).toList();
+    public static CompletableFuture<List<QuiltInstallerVersionV3>> getVersions(MavenRepository maven, Executor executor) {
+        return CompletableFuture.supplyAsync(() -> maven.getMetadata(Constants.QUILT_MAVEN_GROUP, "quilt-installer"), executor).thenApply(mavenMetadata -> mavenMetadata.stream().map(artifact -> {
+            var fileInfo = artifact.fileInfo(executor).join();
+            return new QuiltInstallerVersionV3(artifact.mavenId(), artifact.version, artifact.url().toString(), fileInfo.fileSize(), fileInfo.formatHashesForJson());
+        }).toList());
     }
 }
diff --git a/src/main/java/org/quiltmc/meta/update/collector/v3/QuiltLauncherMetadataCollectorV3.java b/src/main/java/org/quiltmc/meta/update/collector/v3/QuiltLauncherMetadataCollectorV3.java
@@ -60,15 +60,16 @@ public static CompletableFuture<Multimap<String, QuiltLauncherMetadataV3>> getVe
             var artifactUrl = artifact.url().toString();
             var metaUrl = URI.create(artifactUrl.substring(0, artifactUrl.lastIndexOf('.')) + ".json");
 
+            // TODO use httpclient
             try (var reader = new InputStreamReader(metaUrl.toURL().openStream())) {
                 var loaderMetadata = Constants.GSON.fromJson(reader, QuiltLoaderMetadataJson.class);
                 return Pair.of(artifact.version, loaderMetadata);
             } catch (IOException e) {
                 throw new UncheckedIOException("Connection error: " + metaUrl, e);
             }
         }, executor)).toList()).thenApplyAsync(futures -> {
-            Map<String, QuiltLoaderVersionV3> loaderVersions = QuiltLoaderVersionCollectorV3.getVersions(quiltMaven).stream().collect(Collectors.toMap(QuiltLoaderVersionV3::version, Function.identity()));
-            Map<String, HashedMojmapVersionV3> hashedVersions = HashedMojmapAndGameVersionCollectorV3.getVersions(quiltMaven);
+            Map<String, QuiltLoaderVersionV3> loaderVersions = QuiltLoaderVersionCollectorV3.getVersions(quiltMaven, executor).join().stream().collect(Collectors.toMap(QuiltLoaderVersionV3::version, Function.identity()));
+            Map<String, HashedMojmapVersionV3> hashedVersions = HashedMojmapAndGameVersionCollectorV3.getVersions(quiltMaven, executor).join();
             Map<String, IntermediaryVersionV3> intermediaryVersions = IntermediaryVersionCollectorV3.getVersions(fabricMaven);
 
             CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new)).join();

diff --git a/src/main/java/org/quiltmc/meta/update/collector/v3/QuiltLoaderVersionCollectorV3.java b/src/main/java/org/quiltmc/meta/update/collector/v3/QuiltLoaderVersionCollectorV3.java
@@ -26,20 +26,22 @@ public String getName() {
 
     @Override
     public CompletableFuture<Void> gatherData(BiConsumer<String, FileInfo> output, Executor executor) {
-        return CompletableFuture.supplyAsync(() -> getVersions(quiltMaven), executor).thenAccept(versions -> {
+        return getVersions(quiltMaven, executor).thenAccept(versions -> {
             output.accept("v3/versions/loader", FileInfo.jsonList(versions));
 
             LOGGER.info("Found {} quilt loader versions", versions.size());
         });
     }
 
-    public static List<QuiltLoaderVersionV3> getVersions(MavenRepository maven) {
-        return maven.getMetadata(Constants.QUILT_MAVEN_GROUP, "quilt-loader").stream().map(artifact -> {
+    public static CompletableFuture<List<QuiltLoaderVersionV3>> getVersions(MavenRepository maven, Executor executor) {
+        return CompletableFuture.supplyAsync(() -> maven.getMetadata(Constants.QUILT_MAVEN_GROUP, "quilt-loader"), executor).thenApply(mavenMetadata -> mavenMetadata.stream().map(artifact -> {
             var loaderVersion = artifact.version;
             var separator = loaderVersion.contains("+build.") ? "+build." : ".";
             var buildNumber = Integer.parseInt(loaderVersion.substring(loaderVersion.lastIndexOf(".") + 1));
 
-            return new QuiltLoaderVersionV3(artifact.mavenId(), loaderVersion, buildNumber, separator);
-        }).toList();
+            var fileInfo = artifact.fileInfo(executor).join();
+
+            return new QuiltLoaderVersionV3(artifact.mavenId(), loaderVersion, buildNumber, separator, fileInfo.fileSize(), fileInfo.formatHashesForJson());
+        }).toList());
     }
 }
diff --git a/src/main/java/org/quiltmc/meta/update/collector/v3/QuiltMappingsCollectorV3.java b/src/main/java/org/quiltmc/meta/update/collector/v3/QuiltMappingsCollectorV3.java
@@ -27,7 +27,7 @@ public String getName() {
 
     @Override
     public CompletableFuture<Void> gatherData(BiConsumer<String, FileInfo> output, Executor executor) {
-        return CompletableFuture.supplyAsync(() -> getVersions(quiltMaven)).thenAcceptAsync(gameQuiltMappings -> {
+        return getVersions(quiltMaven, executor).thenAccept(gameQuiltMappings -> {
             gameQuiltMappings.asMap().forEach((gameVersion, mappingsVersions) -> output.accept("v3/versions/quilt-mappings/" + gameVersion, FileInfo.jsonList(mappingsVersions)));
 
             output.accept("v3/versions/quilt-mappings", FileInfo.jsonList(gameQuiltMappings.values()));
@@ -36,7 +36,7 @@ public CompletableFuture<Void> gatherData(BiConsumer<String, FileInfo> output, E
             output.accept("v3/versions/game/quilt-mappings", FileInfo.jsonList(gameQuiltMappings.keys()));
 
             LOGGER.info("Found {} quilt mappings", gameQuiltMappings.size());
-        }, executor);
+        });
     }
 
     private static String stripInfo(String version) {
@@ -49,18 +49,22 @@ private static String stripInfo(String version) {
         }
     }
 
-    public static Multimap<String, QuiltMappingsVersionV3> getVersions(MavenRepository maven) {
-        Multimap<String, QuiltMappingsVersionV3> gameQuiltMappings = MultimapBuilder.linkedHashKeys().hashSetValues().build();
-        maven.getMetadata(Constants.QUILT_MAVEN_GROUP, "quilt-mappings").forEach(artifact -> {
-            var mappingsVersion = artifact.version;
-            var gameVersion = stripInfo(mappingsVersion);
-            var separator = mappingsVersion.contains("+build.") ? "+build." : ".";
-            var buildNumber = Integer.parseInt(mappingsVersion.substring(mappingsVersion.lastIndexOf(".") + 1));
+    public static CompletableFuture<Multimap<String, QuiltMappingsVersionV3>> getVersions(MavenRepository maven, Executor executor) {
+        return CompletableFuture.supplyAsync(() -> maven.getMetadata(Constants.QUILT_MAVEN_GROUP, "quilt-mappings"), executor).thenApply(mavenMetadata -> {
+            Multimap<String, QuiltMappingsVersionV3> gameQuiltMappings = MultimapBuilder.linkedHashKeys().hashSetValues().build();
+            mavenMetadata.forEach(artifact -> {
+                var mappingsVersion = artifact.version;
+                var gameVersion = stripInfo(mappingsVersion);
+                var separator = mappingsVersion.contains("+build.") ? "+build." : ".";
+                var buildNumber = Integer.parseInt(mappingsVersion.substring(mappingsVersion.lastIndexOf(".") + 1));
 
-            var version = new QuiltMappingsVersionV3(artifact.mavenId(), mappingsVersion, gameVersion, buildNumber, separator, gameVersion);
-            gameQuiltMappings.put(gameVersion, version);
-        });
+                var fileInfo = artifact.fileInfo(executor).join();
 
-        return gameQuiltMappings;
+                var version = new QuiltMappingsVersionV3(artifact.mavenId(), mappingsVersion, gameVersion, buildNumber, separator, gameVersion, fileInfo.fileSize(), fileInfo.formatHashesForJson());
+                gameQuiltMappings.put(gameVersion, version);
+            });
+
+            return gameQuiltMappings;
+        });
     }
 }
diff --git a/src/main/java/org/quiltmc/meta/update/collector/v3/RootVersionListCollectorV3.java b/src/main/java/org/quiltmc/meta/update/collector/v3/RootVersionListCollectorV3.java
@@ -31,16 +31,16 @@ public String getName() {
     @Override
     public CompletableFuture<Void> gatherData(BiConsumer<String, FileInfo> output, Executor executor) {
         return CompletableFuture.runAsync(() -> {
-            Map<String, HashedMojmapVersionV3> hashed = HashedMojmapAndGameVersionCollectorV3.getVersions(quiltMaven);
+            Map<String, HashedMojmapVersionV3> hashed = HashedMojmapAndGameVersionCollectorV3.getVersions(quiltMaven, executor).join();
             List<GameVersionV3> game = HashedMojmapAndGameVersionCollectorV3.getGameVersions(MinecraftMeta.get(Constants.GSON), hashed::containsKey);
-            Multimap<String, QuiltMappingsVersionV3> mappings = QuiltMappingsCollectorV3.getVersions(quiltMaven);
-            List<QuiltLoaderVersionV3> loader = QuiltLoaderVersionCollectorV3.getVersions(quiltMaven);
-            List<QuiltInstallerVersionV3> installer = QuiltInstallerVersionCollectorV3.getVersions(quiltMaven);
+            Multimap<String, QuiltMappingsVersionV3> mappings = QuiltMappingsCollectorV3.getVersions(quiltMaven, executor).join();
+            List<QuiltLoaderVersionV3> loader = QuiltLoaderVersionCollectorV3.getVersions(quiltMaven, executor).join();
+            List<QuiltInstallerVersionV3> installer = QuiltInstallerVersionCollectorV3.getVersions(quiltMaven, executor).join();
 
             var versionList = new VersionListV3(game, mappings.values(), hashed.values(), loader, installer);
             output.accept("v3/versions", FileInfo.json(versionList));
 
             LOGGER.info("Generated root version list");
-        });
+        }, executor);
     }
 }
diff --git a/src/main/java/org/quiltmc/meta/update/collector/v3/StaticFileCollectorV3.java b/src/main/java/org/quiltmc/meta/update/collector/v3/StaticFileCollectorV3.java
@@ -1,9 +1,9 @@
 package org.quiltmc.meta.update.collector.v3;
 
-import org.quiltmc.meta.update.util.http.ContentType;
 import org.quiltmc.meta.update.upload.FileInfo;
 import org.quiltmc.meta.update.upload.InputCollector;
 import org.quiltmc.meta.update.util.Pair;
+import org.quiltmc.meta.update.util.http.ContentType;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -36,7 +36,7 @@ public CompletableFuture<Void> gatherData(BiConsumer<String, FileInfo> output, E
         return CompletableFuture.runAsync(() -> {
             this.fileInfos.forEach((apiPath, pair) -> {
                 try (var stream = StaticFileCollectorV3.class.getClassLoader().getResourceAsStream("static/" + pair.first())) {
-                    if(stream == null) {
+                    if (stream == null) {
                         throw new FileNotFoundException("static://" + pair.first());
                     }
 

diff --git a/src/main/java/org/quiltmc/meta/update/model/v3/HashedMojmapVersionV3.java b/src/main/java/org/quiltmc/meta/update/model/v3/HashedMojmapVersionV3.java
@@ -1,4 +1,8 @@
 package org.quiltmc.meta.update.model.v3;
 
-public record HashedMojmapVersionV3(String maven, String version) {
+import com.google.gson.annotations.SerializedName;
+
+import java.util.Map;
+
+public record HashedMojmapVersionV3(String maven, String version, @SerializedName("file_size") long fileSize, Map<String, String> hashes) {
 }
diff --git a/src/main/java/org/quiltmc/meta/update/model/v3/IntermediaryVersionV3.java b/src/main/java/org/quiltmc/meta/update/model/v3/IntermediaryVersionV3.java
@@ -1,4 +1,7 @@
 package org.quiltmc.meta.update.model.v3;
 
+/**
+ * No hashes provided here because fabric likes to re-publish intermediary artifacts!
+ */
 public record IntermediaryVersionV3(String maven, String version) {
 }
diff --git a/src/main/java/org/quiltmc/meta/update/model/v3/QuiltInstallerVersionV3.java b/src/main/java/org/quiltmc/meta/update/model/v3/QuiltInstallerVersionV3.java
@@ -1,4 +1,8 @@
 package org.quiltmc.meta.update.model.v3;
 
-public record QuiltInstallerVersionV3(String maven, String version, String url) {
+import com.google.gson.annotations.SerializedName;
+
+import java.util.Map;
+
+public record QuiltInstallerVersionV3(String maven, String version, String url, @SerializedName("file_size") long fileSize, Map<String, String> hashes) {
 }
diff --git a/src/main/java/org/quiltmc/meta/update/model/v3/QuiltLoaderVersionV3.java b/src/main/java/org/quiltmc/meta/update/model/v3/QuiltLoaderVersionV3.java
@@ -1,4 +1,8 @@
 package org.quiltmc.meta.update.model.v3;
 
-public record QuiltLoaderVersionV3(String maven, String version, int build, String separator) {
+import com.google.gson.annotations.SerializedName;
+
+import java.util.Map;
+
+public record QuiltLoaderVersionV3(String maven, String version, int build, String separator, @SerializedName("file_size") long fileSize, Map<String, String> hashes) {
 }
diff --git a/src/main/java/org/quiltmc/meta/update/model/v3/QuiltMappingsVersionV3.java b/src/main/java/org/quiltmc/meta/update/model/v3/QuiltMappingsVersionV3.java
@@ -1,5 +1,9 @@
 package org.quiltmc.meta.update.model.v3;
 
+import com.google.gson.annotations.SerializedName;
+
+import java.util.Map;
+
 // TODO remove 'hashed' field in v4? it seems superfluous
-public record QuiltMappingsVersionV3(String maven, String version, String gameVersion, int build, String separator, String hashed) {
+public record QuiltMappingsVersionV3(String maven, String version, String gameVersion, int build, String separator, String hashed, @SerializedName("file_size") long fileSize, Map<String, String> hashes) {
 }
diff --git a/src/main/java/org/quiltmc/meta/update/util/function/ThrowingConsumer.java b/src/main/java/org/quiltmc/meta/update/util/function/ThrowingConsumer.java
@@ -0,0 +1,7 @@
+package org.quiltmc.meta.update.util.function;
+
+@FunctionalInterface
+public interface ThrowingConsumer<T, E extends Throwable> {
+
+    void accept(T t) throws E;
+}