From 22eb03775cf919ac3e380822f9d07da09802db00 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Tue, 2 Jul 2024 15:22:49 +0300 Subject: [PATCH 1/3] allways pull latest changes for upstream and launcher repos Signed-off-by: Trial97 --- update.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/update.sh b/update.sh index 8e0fe38..5054301 100755 --- a/update.sh +++ b/update.sh @@ -33,6 +33,7 @@ function launcher_git() { currentDate=$(date -I) upstream_git reset --hard HEAD || exit 1 +upstream_git pull python -m meta.run.update_mojang || fail_in python -m meta.run.update_forge || fail_in @@ -57,6 +58,7 @@ if [ "${DEPLOY_TO_GIT}" = true ]; then fi launcher_git reset --hard HEAD || exit 1 +launcher_git pull python -m meta.run.generate_mojang || fail_out python -m meta.run.generate_forge || fail_out From 55e7ce1cfc93296697846286c28dd1907a97497d Mon Sep 17 00:00:00 2001 From: Trial97 Date: Fri, 5 Jul 2024 09:32:52 +0300 Subject: [PATCH 2/3] check sha1 for forge/neoforge maven Signed-off-by: Trial97 --- meta/common/__init__.py | 9 +++++++++ meta/run/update_forge.py | 20 +++++++++++++++++++- meta/run/update_neoforge.py | 20 +++++++++++++++++++- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/meta/common/__init__.py b/meta/common/__init__.py index d7ee49a..16a4f50 100644 --- a/meta/common/__init__.py +++ b/meta/common/__init__.py @@ -86,3 +86,12 @@ def default_session(): sess.headers.update({"User-Agent": "PrismLauncherMeta/1.0"}) return sess + + +def remove_files(file_paths): + for file_path in file_paths: + try: + if os.path.isfile(file_path): + os.remove(file_path) + except Exception as e: + print(e) diff --git a/meta/run/update_forge.py b/meta/run/update_forge.py index 8fc8920..5a86727 100755 --- a/meta/run/update_forge.py +++ b/meta/run/update_forge.py @@ -16,7 +16,12 @@ from pydantic import ValidationError -from meta.common import upstream_path, ensure_upstream_dir, default_session +from meta.common import ( + upstream_path, + ensure_upstream_dir, + default_session, + remove_files, +) from meta.common.forge import ( JARS_DIR, INSTALLER_INFO_DIR, @@ -292,6 +297,19 @@ def main(): UPSTREAM_DIR + "/forge/version_manifests/%s.json" % version.long_version ) + if not os.path.isfile(jar_path): + remove_files([profile_path, installer_info_path]) + else: + fileSha1 = filehash(jar_path, hashlib.sha1) + try: + rfile = sess.get(version.url() + ".sha1") + rfile.raise_for_status() + if fileSha1 != rfile.text.strip(): + remove_files([jar_path, profile_path, installer_info_path]) + except Exception as e: + eprint("Failed to check sha1 %s" % version.url()) + eprint("Error is %s" % e) + installer_refresh_required = not os.path.isfile( profile_path ) or not os.path.isfile(installer_info_path) diff --git a/meta/run/update_neoforge.py b/meta/run/update_neoforge.py index 60b94ec..c6eb0f3 100644 --- a/meta/run/update_neoforge.py +++ b/meta/run/update_neoforge.py @@ -17,7 +17,12 @@ from pydantic import ValidationError -from meta.common import upstream_path, ensure_upstream_dir, default_session +from meta.common import ( + upstream_path, + ensure_upstream_dir, + default_session, + remove_files, +) from meta.common.neoforge import ( JARS_DIR, INSTALLER_INFO_DIR, @@ -243,6 +248,19 @@ def main(): UPSTREAM_DIR + "/neoforge/version_manifests/%s.json" % version.long_version ) + if not os.path.isfile(jar_path): + remove_files([profile_path, installer_info_path]) + else: + fileSha1 = filehash(jar_path, hashlib.sha1) + try: + rfile = sess.get(version.url() + ".sha1") + rfile.raise_for_status() + if fileSha1 != rfile.text.strip(): + remove_files([jar_path, profile_path, installer_info_path]) + except Exception as e: + eprint("Failed to check sha1 %s" % version.url()) + eprint("Error is %s" % e) + installer_refresh_required = not os.path.isfile( profile_path ) or not os.path.isfile(installer_info_path) From 9b81dc7974545429855fa93a62447e4e993726b2 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Mon, 16 Dec 2024 21:00:29 +0200 Subject: [PATCH 3/3] Chache sha1 file Signed-off-by: Trial97 --- meta/common/__init__.py | 25 +++++++++++++++++ meta/run/generate_forge.py | 7 +---- meta/run/generate_neoforge.py | 7 +---- meta/run/update_forge.py | 51 +++++++++++++++++------------------ meta/run/update_neoforge.py | 45 ++++++++++++++++--------------- 5 files changed, 76 insertions(+), 59 deletions(-) diff --git a/meta/common/__init__.py b/meta/common/__init__.py index 16a4f50..8e42933 100644 --- a/meta/common/__init__.py +++ b/meta/common/__init__.py @@ -1,6 +1,8 @@ import os import os.path import datetime +import hashlib +import sys from urllib.parse import urlparse from typing import Any, Optional @@ -95,3 +97,26 @@ def remove_files(file_paths): os.remove(file_path) except Exception as e: print(e) + + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +def filehash(filename, hashtype, blocksize=65536): + hashtype = hashtype() + with open(filename, "rb") as f: + for block in iter(lambda: f.read(blocksize), b""): + hashtype.update(block) + return hashtype.hexdigest() + + +def get_file_sha1_from_file(file_name, sha1_file): + if os.path.isfile(sha1_file): + with open(sha1_file, "r") as file: + return file.read() + + new_sha1 = filehash(file_name, hashlib.sha1) + with open(sha1_file, "w") as file: + file.write(new_sha1) + return diff --git a/meta/run/generate_forge.py b/meta/run/generate_forge.py index 95b5ac3..a01f503 100755 --- a/meta/run/generate_forge.py +++ b/meta/run/generate_forge.py @@ -1,11 +1,10 @@ import os import re -import sys from packaging import version as pversion from operator import attrgetter from typing import Collection -from meta.common import ensure_component_dir, launcher_path, upstream_path +from meta.common import ensure_component_dir, launcher_path, upstream_path, eprint from meta.common.forge import ( FORGE_COMPONENT, INSTALLER_MANIFEST_DIR, @@ -44,10 +43,6 @@ ensure_component_dir(FORGE_COMPONENT) -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) - - # Construct a set of libraries out of a Minecraft version file, for filtering. mc_version_cache = {} diff --git a/meta/run/generate_neoforge.py b/meta/run/generate_neoforge.py index ee1e269..ff58bed 100644 --- a/meta/run/generate_neoforge.py +++ b/meta/run/generate_neoforge.py @@ -1,11 +1,10 @@ from copy import deepcopy import os import re -import sys from operator import attrgetter from typing import Collection -from meta.common import ensure_component_dir, launcher_path, upstream_path +from meta.common import ensure_component_dir, launcher_path, upstream_path, eprint from meta.common.neoforge import ( NEOFORGE_COMPONENT, INSTALLER_MANIFEST_DIR, @@ -38,10 +37,6 @@ ensure_component_dir(NEOFORGE_COMPONENT) -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) - - def version_from_build_system_installer( installer: MojangVersion, profile: NeoForgeInstallerProfileV2, diff --git a/meta/run/update_forge.py b/meta/run/update_forge.py index 5a86727..30c701f 100755 --- a/meta/run/update_forge.py +++ b/meta/run/update_forge.py @@ -7,7 +7,6 @@ import json import os import re -import sys import zipfile from contextlib import suppress from datetime import datetime @@ -21,6 +20,9 @@ ensure_upstream_dir, default_session, remove_files, + eprint, + filehash, + get_file_sha1_from_file, ) from meta.common.forge import ( JARS_DIR, @@ -43,6 +45,7 @@ InstallerInfo, ForgeLegacyInfo, ) +from meta.common.http import download_binary_file from meta.model.mojang import MojangVersion UPSTREAM_DIR = upstream_path() @@ -58,18 +61,6 @@ sess = default_session() -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) - - -def filehash(filename, hashtype, blocksize=65536): - hashtype = hashtype() - with open(filename, "rb") as f: - for block in iter(lambda: f.read(blocksize), b""): - hashtype.update(block) - return hashtype.hexdigest() - - def get_single_forge_files_manifest(longversion): print(f"Getting Forge manifest for {longversion}") path_thing = UPSTREAM_DIR + "/forge/files_manifests/%s.json" % longversion @@ -297,15 +288,20 @@ def main(): UPSTREAM_DIR + "/forge/version_manifests/%s.json" % version.long_version ) + new_sha1 = None + sha1_file = jar_path + ".sha1" if not os.path.isfile(jar_path): remove_files([profile_path, installer_info_path]) else: - fileSha1 = filehash(jar_path, hashlib.sha1) + fileSha1 = get_file_sha1_from_file(jar_path, sha1_file) try: rfile = sess.get(version.url() + ".sha1") rfile.raise_for_status() - if fileSha1 != rfile.text.strip(): - remove_files([jar_path, profile_path, installer_info_path]) + new_sha1 = rfile.text.strip() + if fileSha1 != new_sha1: + remove_files( + [jar_path, profile_path, installer_info_path, sha1_file] + ) except Exception as e: eprint("Failed to check sha1 %s" % version.url()) eprint("Error is %s" % e) @@ -318,11 +314,18 @@ def main(): # grab the installer if it's not there if not os.path.isfile(jar_path): eprint("Downloading %s" % version.url()) - rfile = sess.get(version.url(), stream=True) - rfile.raise_for_status() - with open(jar_path, "wb") as f: - for chunk in rfile.iter_content(chunk_size=128): - f.write(chunk) + download_binary_file(sess, jar_path, version.url()) + if new_sha1 is None: + try: + rfile = sess.get(version.url() + ".sha1") + rfile.raise_for_status() + new_sha1 = rfile.text.strip() + except Exception as e: + eprint("Failed to download new sha1 %s" % version.url()) + eprint("Error is %s" % e) + if new_sha1 is not None: # this is in case the fetch failed + with open(sha1_file, "w") as file: + file.write(new_sha1) eprint("Processing %s" % version.url()) # harvestables from the installer @@ -387,11 +390,7 @@ def main(): if not os.path.isfile(LEGACYINFO_PATH): # grab the jar/zip if it's not there if not os.path.isfile(jar_path): - rfile = sess.get(version.url(), stream=True) - rfile.raise_for_status() - with open(jar_path, "wb") as f: - for chunk in rfile.iter_content(chunk_size=128): - f.write(chunk) + download_binary_file(sess, jar_path, version.url()) # find the latest timestamp in the zip file tstamp = datetime.fromtimestamp(0) with zipfile.ZipFile(jar_path) as jar: diff --git a/meta/run/update_neoforge.py b/meta/run/update_neoforge.py index c6eb0f3..f6a8960 100644 --- a/meta/run/update_neoforge.py +++ b/meta/run/update_neoforge.py @@ -7,7 +7,6 @@ import json import os import re -import sys import zipfile from contextlib import suppress from datetime import datetime @@ -22,7 +21,11 @@ ensure_upstream_dir, default_session, remove_files, + eprint, + filehash, + get_file_sha1_from_file, ) +from meta.common.http import download_binary_file from meta.common.neoforge import ( JARS_DIR, INSTALLER_INFO_DIR, @@ -52,18 +55,6 @@ sess = default_session() -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) - - -def filehash(filename, hashtype, blocksize=65536): - hashtype = hashtype() - with open(filename, "rb") as f: - for block in iter(lambda: f.read(blocksize), b""): - hashtype.update(block) - return hashtype.hexdigest() - - def find_nth(haystack, needle, n): start = haystack.find(needle) while start >= 0 and n > 1: @@ -248,15 +239,20 @@ def main(): UPSTREAM_DIR + "/neoforge/version_manifests/%s.json" % version.long_version ) + new_sha1 = None + sha1_file = jar_path + ".sha1" if not os.path.isfile(jar_path): remove_files([profile_path, installer_info_path]) else: - fileSha1 = filehash(jar_path, hashlib.sha1) + fileSha1 = get_file_sha1_from_file(jar_path, sha1_file) try: rfile = sess.get(version.url() + ".sha1") rfile.raise_for_status() - if fileSha1 != rfile.text.strip(): - remove_files([jar_path, profile_path, installer_info_path]) + new_sha1 = rfile.text.strip() + if fileSha1 != new_sha1: + remove_files( + [jar_path, profile_path, installer_info_path, sha1_file] + ) except Exception as e: eprint("Failed to check sha1 %s" % version.url()) eprint("Error is %s" % e) @@ -270,16 +266,23 @@ def main(): if not os.path.isfile(jar_path): eprint("Downloading %s" % version.url()) try: - rfile = sess.get(version.url(), stream=True) - rfile.raise_for_status() Path(jar_path).parent.mkdir(parents=True, exist_ok=True) - with open(jar_path, "wb") as f: - for chunk in rfile.iter_content(chunk_size=128): - f.write(chunk) + download_binary_file(sess, jar_path, version.url()) except Exception as e: eprint("Failed to download %s" % version.url()) eprint("Error is %s" % e) continue + if new_sha1 is None: + try: + rfile = sess.get(version.url() + ".sha1") + rfile.raise_for_status() + new_sha1 = rfile.text.strip() + except Exception as e: + eprint("Failed to download new sha1 %s" % version.url()) + eprint("Error is %s" % e) + if new_sha1 is not None: # this is in case the fetch failed + with open(sha1_file, "w") as file: + file.write(new_sha1) eprint("Processing %s" % version.url()) # harvestables from the installer