From 13480dbcb9f83dd7f91e69311b494f62ab75186f Mon Sep 17 00:00:00 2001 From: Gegy Date: Sun, 16 Jul 2023 14:50:54 +0200 Subject: [PATCH] Fetch mod files with HttpClient --- .../serverpacklocator/ServerManifest.java | 4 +- .../client/ClientSidedPackHandler.java | 26 +-- .../client/SimpleHttpClient.java | 207 +++++++----------- 3 files changed, 95 insertions(+), 142 deletions(-) diff --git a/src/main/java/cpw/mods/forge/serverpacklocator/ServerManifest.java b/src/main/java/cpw/mods/forge/serverpacklocator/ServerManifest.java index dab886b..2d011cb 100644 --- a/src/main/java/cpw/mods/forge/serverpacklocator/ServerManifest.java +++ b/src/main/java/cpw/mods/forge/serverpacklocator/ServerManifest.java @@ -28,8 +28,8 @@ public record ServerManifest(String forgeVersion, List files) { ModFileData.CODEC.listOf().fieldOf("files").forGetter(ServerManifest::files) ).apply(i, ServerManifest::new)); - public static DataResult loadFromStream(final InputStream stream) { - JsonElement json = JsonParser.parseReader(new InputStreamReader(stream, StandardCharsets.UTF_8)); + public static DataResult parse(final String string) { + JsonElement json = JsonParser.parseString(string); return CODEC.parse(JsonOps.INSTANCE, json); } diff --git a/src/main/java/cpw/mods/forge/serverpacklocator/client/ClientSidedPackHandler.java b/src/main/java/cpw/mods/forge/serverpacklocator/client/ClientSidedPackHandler.java index 214c2e4..9c26f3c 100644 --- a/src/main/java/cpw/mods/forge/serverpacklocator/client/ClientSidedPackHandler.java +++ b/src/main/java/cpw/mods/forge/serverpacklocator/client/ClientSidedPackHandler.java @@ -1,7 +1,6 @@ package cpw.mods.forge.serverpacklocator.client; import com.electronwill.nightconfig.core.ConfigFormat; -import cpw.mods.forge.serverpacklocator.LaunchEnvironmentHandler; import cpw.mods.forge.serverpacklocator.ServerManifest; import cpw.mods.forge.serverpacklocator.SidedPackHandler; import net.minecraftforge.forgespi.locating.IModFile; @@ -9,16 +8,18 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import javax.annotation.Nullable; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; -import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; public class ClientSidedPackHandler extends SidedPackHandler { private static final Logger LOGGER = LogManager.getLogger(); private SimpleHttpClient clientDownloader; + @Nullable + private ServerManifest manifest; public ClientSidedPackHandler(final Path serverModsDir) { super(serverModsDir); @@ -44,7 +45,10 @@ protected boolean validateConfig() { @Override protected List processModList(List scannedMods) { - final Set manifestFileList = clientDownloader.getManifest().files() + if (manifest == null) { + return List.of(); + } + final Set manifestFileList = manifest.files() .stream() .map(ServerManifest.ModFileData::fileName) .collect(Collectors.toSet()); @@ -57,13 +61,9 @@ protected List processModList(List scannedMods) { protected boolean waitForDownload() { if (!isValid()) return false; - try { - if (!clientDownloader.waitForResult()) { - LOGGER.info("There was a problem with the connection, there will not be any server mods"); - return false; - } - } catch (ExecutionException e) { - LOGGER.error("Caught exception downloading mods from server", e); + manifest = clientDownloader.waitForResult(); + if (manifest == null) { + LOGGER.info("There was a problem with the connection, there will not be any server mods"); return false; } return true; @@ -71,9 +71,7 @@ protected boolean waitForDownload() { @Override public void initialize(final IModLocator dirLocator) { - clientDownloader = new SimpleHttpClient( - this, - getConfig().>getOptional("client.excludedModIds").orElse(Collections.emptyList()) - ); + List excludedModIds = getConfig().>getOptional("client.excludedModIds").orElse(List.of()); + clientDownloader = new SimpleHttpClient(this, Set.copyOf(excludedModIds)); } } diff --git a/src/main/java/cpw/mods/forge/serverpacklocator/client/SimpleHttpClient.java b/src/main/java/cpw/mods/forge/serverpacklocator/client/SimpleHttpClient.java index 8669432..5505eb8 100644 --- a/src/main/java/cpw/mods/forge/serverpacklocator/client/SimpleHttpClient.java +++ b/src/main/java/cpw/mods/forge/serverpacklocator/client/SimpleHttpClient.java @@ -1,130 +1,112 @@ package cpw.mods.forge.serverpacklocator.client; +import com.google.common.collect.Iterators; import com.google.common.hash.HashCode; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.mojang.serialization.DataResult; import cpw.mods.forge.serverpacklocator.FileChecksumValidator; import cpw.mods.forge.serverpacklocator.LaunchEnvironmentHandler; import cpw.mods.forge.serverpacklocator.ServerManifest; -import cpw.mods.modlauncher.api.LamdbaExceptionUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.net.URL; -import java.net.URLConnection; +import javax.annotation.Nullable; +import java.net.URI; import java.net.URLEncoder; -import java.nio.channels.Channels; -import java.nio.channels.FileChannel; -import java.nio.channels.ReadableByteChannel; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.*; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; +import java.util.concurrent.*; public class SimpleHttpClient { private static final Logger LOGGER = LogManager.getLogger(); + + private static final Executor EXECUTOR = Executors.newFixedThreadPool(2, new ThreadFactoryBuilder() + .setNameFormat("ServerPackLocator HTTP Client - %d") + .setDaemon(true) + .build()); + + private static final String USER_AGENT = "ServerPackLocator (https://github.com/LoveTropics/serverpacklocator)"; + + private final HttpClient client = HttpClient.newBuilder() + .executor(EXECUTOR) + .build(); + private final Path outputDir; - private ServerManifest serverManifest; - private Iterator fileDownloaderIterator; - private final Future downloadJob; - private final List excludedModIds; + private final CompletableFuture downloadJob; + private final Set excludedModIds; - public SimpleHttpClient(final ClientSidedPackHandler packHandler, final List excludedModIds) { + public SimpleHttpClient(final ClientSidedPackHandler packHandler, final Set excludedModIds) { this.outputDir = packHandler.getServerModsDir(); this.excludedModIds = excludedModIds; - final Optional remoteServer = packHandler.getConfig().getOptional("client.remoteServer"); - downloadJob = Executors.newSingleThreadExecutor().submit(() -> remoteServer - .map(server -> server.endsWith("/") ? server.substring(0, server.length() - 1) : server) - .map(this::connectAndDownload).orElse(false)); + final Optional remoteServer = packHandler.getConfig().getOptional("client.remoteServer") + .map(server -> server.endsWith("/") ? server.substring(0, server.length() - 1) : server); + downloadJob = remoteServer.map(this::connectAndDownload) + .orElse(CompletableFuture.completedFuture(null)); } - private boolean connectAndDownload(final String server) { - try { - downloadManifest(server); - downloadNextFile(server); - return true; - } catch (Exception ex) { - LOGGER.error("Failed to download modpack from server: " + server, ex); - return false; - } + private CompletableFuture connectAndDownload(final String host) { + return downloadManifest(host).thenCompose(manifest -> { + List filesToDownload = manifest.files().stream() + .filter(file -> !excludedModIds.contains(file.rootModId())) + .toList(); + LOGGER.debug("Downloading {} of {} files from manifest", filesToDownload.size(), manifest.files().size()); + + return sequential(Iterators.transform(filesToDownload.iterator(), file -> downloadFile(host, file))).thenApply(unused -> { + LOGGER.debug("Finished downloading files"); + return manifest; + }); + }); } - protected void downloadManifest(final String serverHost) throws IOException - { - var address = serverHost + "/servermanifest.json"; - - LOGGER.info("Requesting server manifest from: " + serverHost); - LaunchEnvironmentHandler.INSTANCE.addProgressMessage("Requesting server manifest from: " + serverHost); - - var url = new URL(address); - var connection = url.openConnection(); - - try (BufferedInputStream in = new BufferedInputStream(connection.getInputStream())) { - DataResult result = ServerManifest.loadFromStream(in); - this.serverManifest = result.result().orElseThrow(() -> new IllegalStateException("Manifest was malformed: " + result.error().orElseThrow())); - } catch (IOException e) { - throw new IllegalStateException("Failed to download manifest", e); + private static CompletableFuture sequential(final Iterator> iterator) { + if (iterator.hasNext()) { + return iterator.next().thenCompose(unused -> sequential(iterator)); } - LOGGER.debug("Received manifest"); - buildFileFetcher(); + return CompletableFuture.completedFuture(null); } - private void downloadFile(final String server, final ServerManifest.ModFileData next) throws IOException - { - final HashCode existingChecksum = FileChecksumValidator.computeChecksumFor(resolvePath(next)); - if (Objects.equals(next.checksum(), existingChecksum)) { - LOGGER.debug("Found existing file {} - skipping", next.fileName()); - downloadNextFile(server); - return; - } - - final String nextFile = next.fileName(); - LOGGER.info("Requesting file {}", nextFile); - LaunchEnvironmentHandler.INSTANCE.addProgressMessage("Requesting file "+nextFile); - final String requestUri = server + LamdbaExceptionUtils.rethrowFunction((String f) -> URLEncoder.encode(f, StandardCharsets.UTF_8)) - .andThen(s -> s.replaceAll("\\+", "%20")) - .andThen(s -> "/files/"+s) - .apply(nextFile); - - try - { - URLConnection connection = new URL(requestUri).openConnection(); - - File file = resolvePath(next).toFile(); - - FileChannel download = new FileOutputStream(file).getChannel(); - - long totalBytes = connection.getContentLengthLong(), time = System.nanoTime(), between, length; - int percent; - - ReadableByteChannel channel = Channels.newChannel(connection.getInputStream()); + private CompletableFuture downloadManifest(final String host) { + LOGGER.info("Requesting server manifest from: {}", host); + LaunchEnvironmentHandler.INSTANCE.addProgressMessage("Requesting server manifest from: " + host); - while (download.transferFrom(channel, file.length(), 1024) > 0) - { - between = System.nanoTime() - time; - - if (between < 1000000000) continue; - - length = file.length(); - - percent = (int) ((double) length / ((double) totalBytes == 0.0 ? 1.0 : (double) totalBytes) * 100.0); + HttpRequest request = HttpRequest.newBuilder(URI.create(host + "/servermanifest.json")) + .header("User-Agent", USER_AGENT) + .GET() + .build(); + return client.sendAsync(request, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)) + .thenApply(SimpleHttpClient::parseManifest); + } - LOGGER.info("Downloaded {}% of {}", percent, nextFile); - LaunchEnvironmentHandler.INSTANCE.addProgressMessage("Downloaded " + percent + "% of " + nextFile); + private static ServerManifest parseManifest(HttpResponse response) { + DataResult result = ServerManifest.parse(response.body()); + return result.result().orElseThrow(() -> new IllegalStateException("Manifest was malformed: " + result.error().orElseThrow())); + } - time = System.nanoTime(); - } + private CompletableFuture downloadFile(final String host, final ServerManifest.ModFileData modFile) { + final Path targetPath = resolvePath(modFile); - downloadNextFile(server); - } catch (Exception ex) { - throw new IllegalStateException("Failed to download file: " + nextFile, ex); + final HashCode existingChecksum = FileChecksumValidator.computeChecksumFor(targetPath); + if (Objects.equals(modFile.checksum(), existingChecksum)) { + LOGGER.debug("Found existing file {} - skipping", modFile.fileName()); + return CompletableFuture.completedFuture(null); } + + final String fileName = modFile.fileName(); + LOGGER.info("Requesting file: {}", fileName); + LaunchEnvironmentHandler.INSTANCE.addProgressMessage("Requesting file: " + fileName); + + final URI uri = URI.create(host + "/files/" + URLEncoder.encode(fileName, StandardCharsets.UTF_8).replaceAll("\\+", "%20")); + final HttpRequest request = HttpRequest.newBuilder(uri) + .header("User-Agent", USER_AGENT) + .GET() + .build(); + return client.sendAsync(request, HttpResponse.BodyHandlers.ofFile(targetPath)) + .thenAccept(response -> LaunchEnvironmentHandler.INSTANCE.addProgressMessage("Finished downloading file: " + fileName)); } private Path resolvePath(final ServerManifest.ModFileData modFile) { @@ -135,40 +117,13 @@ private Path resolvePath(final ServerManifest.ModFileData modFile) { return path; } - private void downloadNextFile(final String server) throws IOException - { - final Iterator fileDataIterator = fileDownloaderIterator; - if (fileDataIterator.hasNext()) { - downloadFile(server, fileDataIterator.next()); - } else { - LOGGER.info("Finished downloading closing channel"); - } - } - - private void buildFileFetcher() { - if (this.excludedModIds.isEmpty()) - { - fileDownloaderIterator = serverManifest.files().iterator(); - } - else - { - fileDownloaderIterator = serverManifest.files() - .stream() - .filter(modFileData -> !this.excludedModIds.contains(modFileData.rootModId())) - .iterator(); - } - - } - - boolean waitForResult() throws ExecutionException { + @Nullable + ServerManifest waitForResult() { try { - return downloadJob.get(); - } catch (InterruptedException e) { - return false; + return downloadJob.join(); + } catch (Throwable t) { + LOGGER.error("Encountered an exception while downloading server mods", t); + return null; } } - - public ServerManifest getManifest() { - return this.serverManifest; - } }