diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml new file mode 100644 index 00000000000..07b5f4272dc --- /dev/null +++ b/.github/workflows/dependency-check.yml @@ -0,0 +1,61 @@ +name: Dependency check version + +on: + schedule: + - cron: '0 3 * * 1' # Every Monday at 03:00 AM UTC + workflow_dispatch: + +jobs: + dependency-check: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + cache-dependency-path: '**/dependency_requirements.txt' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r scripts/dependency_requirements.txt + + - name: Run version check + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: python scripts/dependency_updater.py --ci-check + + - name: Trigger subsequent action for each component + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + version_diff=$(cat version_diff.json) + count=0 # Initialize a counter + for row in $(echo "${version_diff}" | jq -r 'to_entries[] | @base64'); do + _jq() { + echo ${row} | base64 --decode | jq -r ${1} + } + + component=$(_jq '.key') + current_version=$(_jq '.value.current_version') + processed_latest_version=$(_jq '.value.processed_latest_version') + + echo "Triggering update for $component from $current_version to $processed_latest_version" + + gh workflow run dependency-pull-request.yml \ + -f component=$component \ + -f current_version=$current_version \ + -f latest_version=$processed_latest_version \ + + count=$((count + 1)) + + # Stop after triggering 30 actions + if [ "$count" -ge 30 ]; then + echo "Reached the limit of 30 triggered actions." + break + fi + done diff --git a/.github/workflows/dependency-pull-request.yml b/.github/workflows/dependency-pull-request.yml new file mode 100644 index 00000000000..1dc7dd8070e --- /dev/null +++ b/.github/workflows/dependency-pull-request.yml @@ -0,0 +1,71 @@ +name: Dependency create bump PR +run-name: Create bump PR for ${{ inputs.component }} from ${{ inputs.current_version }} to ${{ inputs.latest_version }} + +on: + workflow_dispatch: + inputs: + component: + description: "Component to update" + required: true + current_version: + description: "Current version of the component" + required: true + latest_version: + description: "Latest version of the component" + required: true + +jobs: + create-pull-request: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + cache-dependency-path: '**/dependency_requirements.txt' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r scripts/dependency_requirements.txt + + - name: Run version check to create version_diff.json + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: python scripts/dependency_updater.py --ci-check --component ${{ github.event.inputs.component }} + + - name: Update component versions and checksums + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: python scripts/dependency_updater.py --component ${{ github.event.inputs.component }} + + - name: Generate PR body + id: generate_pr_body + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + pr_body=$(python scripts/generate_pr_body.py --component ${{ github.event.inputs.component }}) + + # Escape any special characters (e.g., newlines) for the GITHUB_OUTPUT + echo "pr_body<> $GITHUB_OUTPUT + echo "$pr_body" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - name: Cleanup cache and version_diff.json + run: rm -r cache/ version_diff.json + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v5 + with: + branch: "update-dependency-${{ github.event.inputs.component }}" + commit-message: "Bump ${{ github.event.inputs.component }} from ${{ github.event.inputs.current_version }} to ${{ github.event.inputs.latest_version }}" + title: "Bump ${{ github.event.inputs.component }} from ${{ github.event.inputs.current_version }} to ${{ github.event.inputs.latest_version }}" + body: ${{ steps.generate_pr_body.outputs.pr_body }} + labels: | + dependencies + release-note-none diff --git a/scripts/dependency_config.py b/scripts/dependency_config.py new file mode 100644 index 00000000000..8ff5da237fb --- /dev/null +++ b/scripts/dependency_config.py @@ -0,0 +1,221 @@ +ARCHITECTURES = ['arm', 'arm64', 'amd64', 'ppc64le'] +OSES = ['darwin', 'linux', 'windows'] +README_COMPONENTS = ['etcd', 'containerd', 'crio', 'calicoctl', 'krew', 'helm'] +SHA256REGEX = r'(\b[a-f0-9]{64})\b' + +PATH_DOWNLOAD = 'roles/kubespray-defaults/defaults/main/download.yml' +PATH_CHECKSUM = 'roles/kubespray-defaults/defaults/main/checksums.yml' +PATH_MAIN = 'roles/kubespray-defaults/defaults/main/main.yml' +PATH_README = 'README.md' +PATH_VERSION_DIFF = 'version_diff.json' + +COMPONENT_INFO = { + 'calico_crds': { + 'owner': 'projectcalico', + 'repo': 'calico', + 'url_download': 'https://github.com/projectcalico/calico/archive/{version}.tar.gz', + 'placeholder_version': ['calico_version'], + 'placeholder_checksum' : 'calico_crds_archive_checksums', + 'checksum_structure' : 'simple', + 'sha_regex' : r'', # binary + }, + 'calicoctl': { + 'owner': 'projectcalico', + 'repo': 'calico', + 'url_download': 'https://github.com/projectcalico/calico/releases/download/{version}/SHA256SUMS', + 'placeholder_version': ['calico_version'], + 'placeholder_checksum' : 'calicoctl_binary_checksums', + 'checksum_structure' : 'arch', + 'sha_regex' : r'linux-{arch}\b', + }, + 'ciliumcli': { + 'owner': 'cilium', + 'repo': 'cilium-cli', + 'url_download': 'https://github.com/cilium/cilium-cli/releases/download/{version}/cilium-linux-{arch}.tar.gz.sha256sum', + 'placeholder_version': ['cilium_cli_version'], + 'placeholder_checksum' : 'ciliumcli_binary_checksums', + 'checksum_structure' : 'arch', + 'sha_regex' : r'{arch}', + }, + 'cni': { + 'owner': 'containernetworking', + 'repo': 'plugins', + 'url_download': 'https://github.com/containernetworking/plugins/releases/download/{version}/cni-plugins-linux-{arch}-{version}.tgz.sha256', + 'placeholder_version': ['cni_version'], + 'placeholder_checksum' : 'cni_binary_checksums', + 'checksum_structure' : 'arch', + 'sha_regex' : r'{arch}', + }, + 'containerd': { + 'owner': 'containerd', + 'repo': 'containerd', + 'url_download': 'https://github.com/containerd/containerd/releases/download/v{version}/containerd-{version}-linux-{arch}.tar.gz.sha256sum', + 'placeholder_version': ['containerd_version'], + 'placeholder_checksum' : 'containerd_archive_checksums', + 'checksum_structure' : 'arch', + 'sha_regex' : r'{arch}', + }, + 'crictl': { + 'owner': 'kubernetes-sigs', + 'repo': 'cri-tools', + 'url_download': 'https://github.com/kubernetes-sigs/cri-tools/releases/download/{version}/crictl-{version}-linux-{arch}.tar.gz.sha256', + 'placeholder_version': ['crictl_supported_versions', 'kube_major_version'], + 'placeholder_checksum' : 'crictl_checksums', + 'checksum_structure' : 'arch', + 'sha_regex' : r'simple', # only sha + }, + 'cri_dockerd': { + 'owner': 'Mirantis', + 'repo': 'cri-dockerd', + 'url_download': 'https://github.com/Mirantis/cri-dockerd/releases/download/v{version}/cri-dockerd-{version}.{arch}.tgz', + 'placeholder_version': ['cri_dockerd_version'], + 'placeholder_checksum' : 'cri_dockerd_archive_checksums', + 'checksum_structure' : 'arch', + 'sha_regex' : r'', # binary + }, + 'crio': { + 'owner': 'cri-o', + 'repo': 'cri-o', + 'url_download': 'https://storage.googleapis.com/cri-o/artifacts/cri-o.{arch}.{version}.tar.gz.sha256sum', + 'placeholder_version': ['crio_supported_versions', 'kube_major_version'], + 'placeholder_checksum' : 'crio_archive_checksums', + 'checksum_structure' : 'arch', + 'sha_regex' : r'{arch}', + }, + 'crun': { + 'owner': 'containers', + 'repo': 'crun', + 'url_download': 'https://github.com/containers/crun/releases/download/{version}/crun-{version}-linux-{arch}', + 'placeholder_version': ['crun_version'], + 'placeholder_checksum' : 'crun_checksums', + 'checksum_structure' : 'arch', + 'sha_regex' : r'', # binary + }, + 'etcd': { + 'owner': 'etcd-io', + 'repo': 'etcd', + 'url_download': 'https://github.com/etcd-io/etcd/releases/download/{version}/SHA256SUMS', + 'placeholder_version': ['etcd_supported_versions', 'kube_major_version'], + 'placeholder_checksum' : 'etcd_binary_checksums', + 'checksum_structure' : 'arch', + 'sha_regex' : r'linux-{arch}\.', + }, + 'gvisor_containerd_shim': { + 'owner': 'google', + 'repo': 'gvisor', + 'url_download': 'https://storage.googleapis.com/gvisor/releases/release/{version}/{arch}/containerd-shim-runsc-v1', + 'placeholder_version': ['gvisor_version'], + 'placeholder_checksum' : 'gvisor_containerd_shim_binary_checksums', + 'checksum_structure' : 'arch', + 'sha_regex' : r'', # binary + }, + 'gvisor_runsc': { + 'owner': 'google', + 'repo': 'gvisor', + 'url_download': 'https://storage.googleapis.com/gvisor/releases/release/{version}/{arch}/runsc', + 'placeholder_version': ['gvisor_version'], + 'placeholder_checksum' : 'gvisor_runsc_binary_checksums', + 'checksum_structure' : 'arch', + 'sha_regex' : r'', # binary + }, + 'helm': { + 'owner': 'helm', + 'repo': 'helm', + 'url_download': 'https://get.helm.sh/helm-{version}-linux-{arch}.tar.gz.sha256sum', + 'placeholder_version': ['helm_version'], + 'placeholder_checksum' : 'helm_archive_checksums', + 'checksum_structure' : 'arch', + 'sha_regex' : r'{arch}', + }, + + 'kata_containers': { + 'owner': 'kata-containers', + 'repo': 'kata-containers', + 'url_download': 'https://github.com/kata-containers/kata-containers/releases/download/{version}/kata-static-{version}-{arch}.tar.xz', + 'placeholder_version': ['kata_containers_version'], + 'placeholder_checksum' : 'kata_containers_binary_checksums', + 'checksum_structure' : 'arch', + 'sha_regex' : r'', # binary + }, + 'krew': { + 'owner': 'kubernetes-sigs', + 'repo': 'krew', + 'url_download': 'https://github.com/kubernetes-sigs/krew/releases/download/{version}/krew-{os_name}_{arch}.tar.gz.sha256', + 'placeholder_version': ['krew_version'], + 'placeholder_checksum' : 'krew_archive_checksums', + 'checksum_structure' : 'os_arch', + 'sha_regex' : r'simple', # only sha + }, + 'kubeadm': { + 'owner': 'kubernetes', + 'repo': 'kubernetes', + 'url_download': 'https://dl.k8s.io/release/{version}/bin/linux/{arch}/kubeadm.sha256', + 'placeholder_version': ['kube_version'], + 'placeholder_checksum' : 'kubeadm_checksums', + 'checksum_structure' : 'arch', + 'sha_regex' : r'simple', # only sha + }, + 'kubectl': { + 'owner': 'kubernetes', + 'repo': 'kubernetes', + 'url_download': 'https://dl.k8s.io/release/{version}/bin/linux/{arch}/kubectl.sha256', + 'placeholder_version': ['kube_version'], + 'placeholder_checksum' : 'kubectl_checksums', + 'checksum_structure' : 'arch', + 'sha_regex' : r'simple', # only sha + }, + 'kubelet': { + 'owner': 'kubernetes', + 'repo': 'kubernetes', + 'url_download': 'https://dl.k8s.io/release/{version}/bin/linux/{arch}/kubelet.sha256', + 'placeholder_version': ['kube_version'], + 'placeholder_checksum' : 'kubelet_checksums', + 'checksum_structure' : 'arch', + 'sha_regex' : r'simple', # only sha + }, + 'nerdctl': { + 'owner': 'containerd', + 'repo': 'nerdctl', + 'url_download': 'https://github.com/containerd/nerdctl/releases/download/v{version}/SHA256SUMS', + 'placeholder_version': ['nerdctl_version'], + 'placeholder_checksum' : 'nerdctl_archive_checksums', + 'checksum_structure' : 'arch', + 'sha_regex' : r'nerdctl-(?!full)[\w.-]+-linux-{arch}\.tar\.gz', + }, + 'runc': { + 'owner': 'opencontainers', + 'repo': 'runc', + 'url_download': 'https://github.com/opencontainers/runc/releases/download/{version}/runc.sha256sum', + 'placeholder_version': ['runc_version'], + 'placeholder_checksum' : 'runc_checksums', + 'checksum_structure' : 'arch', + 'sha_regex' : r'\.{arch}\b', + }, + 'skopeo': { + 'owner': 'containers', + 'repo': 'skopeo', + 'url_download': 'https://github.com/lework/skopeo-binary/releases/download/{version}/skopeo-linux-{arch}', + 'placeholder_version': ['skopeo_version'], + 'placeholder_checksum' : 'skopeo_binary_checksums', + 'checksum_structure' : 'arch', + 'sha_regex' : r'', # binary + }, + 'youki': { + 'owner': 'containers', + 'repo': 'youki', + 'url_download': 'https://github.com/containers/youki/releases/download/v{version}/youki-{version}-{arch}.tar.gz', + 'placeholder_version': ['youki_version'], + 'placeholder_checksum' : 'youki_checksums', + 'checksum_structure' : 'arch', + 'sha_regex' : r'', # binary + }, + 'yq': { + 'owner': 'mikefarah', + 'repo': 'yq', + 'url_download': 'https://github.com/mikefarah/yq/releases/download/{version}/checksums-bsd', + 'placeholder_version': ['yq_version'], + 'placeholder_checksum' : 'yq_checksums', + 'checksum_structure' : 'arch', + 'sha_regex' : r'SHA256 \([^)]+linux_{arch}\)', + }, +} diff --git a/scripts/dependency_requirements.txt b/scripts/dependency_requirements.txt new file mode 100644 index 00000000000..640e4717c97 --- /dev/null +++ b/scripts/dependency_requirements.txt @@ -0,0 +1,7 @@ +certifi==2024.8.30 +charset-normalizer==3.3.2 +idna==3.9 +requests==2.32.3 +ruamel.yaml==0.18.6 +ruamel.yaml.clib==0.2.8 +urllib3==2.2.3 diff --git a/scripts/dependency_updater.py b/scripts/dependency_updater.py new file mode 100644 index 00000000000..27d83390141 --- /dev/null +++ b/scripts/dependency_updater.py @@ -0,0 +1,536 @@ +import os +import re +import sys +import logging +import requests +import json +import argparse +import hashlib +from ruamel.yaml import YAML +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry +from concurrent.futures import ThreadPoolExecutor +from dependency_config import ARCHITECTURES, OSES, README_COMPONENTS, PATH_DOWNLOAD, PATH_CHECKSUM, PATH_MAIN, PATH_README, PATH_VERSION_DIFF, COMPONENT_INFO, SHA256REGEX + + +yaml = YAML() +yaml.explicit_start = True +yaml.preserve_quotes = True +yaml.width = 4096 +yaml.indent(mapping=2, sequence=4, offset=2) + + +pwd = os.getcwd() +cache_dir = './cache' +cache_expiry_seconds = 86400 +os.makedirs(cache_dir, exist_ok=True) + + +github_api_url = 'https://api.github.com/graphql' +gh_token = os.getenv('GH_TOKEN') +if not gh_token: + logging.error('GH_TOKEN is not set. You can set it via "export GH_TOKEN=". Exiting.') + sys.exit(1) + + +def setup_logging(loglevel): + log_format = '%(asctime)s - %(levelname)s - [%(threadName)s] - %(message)s' + numeric_level = getattr(logging, loglevel.upper(), None) + if not isinstance(numeric_level, int): + raise ValueError(f'Invalid log level: {loglevel}') + logging.basicConfig(level=numeric_level, format=log_format) + +def get_session_with_retries(): + session = requests.Session() + adapter = HTTPAdapter( + pool_connections=50, + pool_maxsize=50, + max_retries=Retry(total=3, backoff_factor=1) + ) + session.mount('http://', adapter) + session.mount('https://', adapter) + return session + +def get_current_version(component, component_data): + kube_major_version = component_data['kube_major_version'] + placeholder_version = [kube_major_version if item == 'kube_major_version' else item for item in component_data['placeholder_version']] + if component.startswith('kube'): + current_version = main_yaml_data + else: + current_version = download_yaml_data + for key in placeholder_version: + current_version = current_version.get(key) + return current_version + +def get_latest_version(component_repo_metadata): + releases = component_repo_metadata.get('releases', {}).get('nodes', []) + for release in releases: + if release.get('isLatest', False): + return release['tagName'] + tags = component_repo_metadata.get('refs', {}).get('nodes', []) # fallback on tags + if tags: + first_tag = tags[0]['name'] + return first_tag + return None + +def get_patch_versions(component, latest_version, component_repo_metadata): + if component in ['gvisor_runsc','gvisor_containerd_shim']: # hack for gvisor + return [latest_version] + match = re.match(r'v?(\d+)\.(\d+)', latest_version) + if not match: + logging.error(f'Invalid version format: {latest_version}') + return [] + major_version, minor_version = match.groups() + patch_versions = [] + stable_version_pattern = re.compile(rf'^v?{major_version}\.{minor_version}(\.\d+)?$') # no rc, alpha, dev, etc. + # Search releases + releases = component_repo_metadata.get('releases', {}).get('nodes', []) + for release in releases: + version = release.get('tagName', '') + if stable_version_pattern.match(version): + patch_versions.append(version) + # Fallback to tags + if not patch_versions: + tags = component_repo_metadata.get('refs', {}).get('nodes', []) + for tag in tags: + version = tag.get('name', '') + if stable_version_pattern.match(version): + patch_versions.append(version) + patch_versions.sort(key=lambda v: list(map(int, re.findall(r'\d+', v)))) # sort for checksum update + return patch_versions + +def get_repository_metadata(component_info, session): + query_parts = [] + for component, data in component_info.items(): + owner = data['owner'] + repo = data['repo'] + query_parts.append(f""" + {component}: repository(owner: "{owner}", name: "{repo}") {{ + releases(first: {args.graphql_number_of_entries}, orderBy: {{field: CREATED_AT, direction: DESC}}) {{ + nodes {{ + tagName + url + description + publishedAt + isLatest + }} + }} + refs(refPrefix: "refs/tags/", first: {args.graphql_number_of_entries}, orderBy: {{field: TAG_COMMIT_DATE, direction: DESC}}) {{ + nodes {{ + name + target {{ + ... on Tag {{ + target {{ + ... on Commit {{ + history(first: {args.graphql_number_of_commits}) {{ + edges {{ + node {{ + oid + message + url + }} + }} + }} + }} + }} + }} + ... on Commit {{ + # In case the tag directly points to a commit + history(first: {args.graphql_number_of_commits}) {{ + edges {{ + node {{ + oid + message + url + }} + }} + }} + }} + }} + }} + }} + }} + """) + + query = f"query {{ {''.join(query_parts)} }}" + headers = { + 'Authorization': f'Bearer {gh_token}', + 'Content-Type': 'application/json' + } + + try: + response = session.post(github_api_url, json={'query': query}, headers=headers) + response.raise_for_status() + json_data = response.json() + data = json_data.get('data') + if data is not None and bool(data): # Ensure 'data' is not None and not empty + logging.debug(f'GraphQL data response:\n{json.dumps(data, indent=2)}') + return data + else: + logging.error(f'GraphQL query returned errors: {json_data}') + return None + except Exception as e: + logging.error(f'Error fetching repository metadata: {e}') + return None + +def calculate_checksum(cachefile, sha_regex): + if sha_regex: + logging.debug(f'Searching with regex {sha_regex} in file {cachefile}') + with open(f'cache/{cachefile}', 'r') as f: + for line in f: + if sha_regex == 'simple': # Only sha is present in the file + pattern = re.compile(SHA256REGEX) + else: + pattern = re.compile(rf'(?:{SHA256REGEX}.*{sha_regex}|{sha_regex}.*{SHA256REGEX})') # Sha may be at start or end + match = pattern.search(line) + if match: + checksum = match.group(1) or match.group(2) + logging.debug(f'Matched line: {line.strip()}') + return checksum + else: # binary + sha256_hash = hashlib.sha256() + with open(f'cache/{cachefile}', 'rb') as f: + for byte_block in iter(lambda: f.read(4096), b''): + sha256_hash.update(byte_block) + checksum = sha256_hash.hexdigest() + return checksum + +def download_file_and_get_checksum(component, arch, url_download, version, sha_regex, session): + logging.info(f'Download URL {url_download}') + cache_file = f'{component}-{arch}-{version}' + if os.path.exists(f'cache/{cache_file}'): + logging.info(f'Using cached file for {url_download}') + return calculate_checksum(cache_file, sha_regex) + try: + response = session.get(url_download, timeout=10) + response.raise_for_status() + with open(f'cache/{cache_file}', 'wb') as f: + f.write(response.content) + logging.info(f'Downloaded and cached file for {url_download}') + return calculate_checksum(cache_file, sha_regex) + except Exception as e: + logging.warning(e) + return None + +def get_checksums(component, component_data, versions, session): + checksums = {} + for version in versions: + processed_version = process_version_string(component, version) + checksums[version] = {} + url_download_template = component_data.get('url_download') + if component_data['checksum_structure'] == 'os_arch': + # OS -> Arch -> Checksum + for os_name in OSES: + if os_name not in checksums[version]: + checksums[version][os_name] = {} + for arch in ARCHITECTURES: + url_download = url_download_template.format(arch=arch, os_name=os_name, version=processed_version) + sha_regex = component_data.get('sha_regex').format(arch=arch, os_name=os_name) + checksum = download_file_and_get_checksum(component, arch, url_download, processed_version, sha_regex, session) or 0 + checksums[version][os_name][arch] = checksum + elif component_data['checksum_structure'] == 'arch': + # Arch -> Checksum + for arch in ARCHITECTURES: + tmp_arch = arch + if component == 'youki': + tmp_arch = tmp_arch.replace('arm64', 'aarch64-gnu').replace('amd64', 'x86_64-gnu') + elif component in ['gvisor_containerd_shim','gvisor_runsc']: + tmp_arch = tmp_arch.replace("arm64", "aarch64").replace("amd64", "x86_64") + url_download = url_download_template.format(arch=tmp_arch, version=processed_version) + sha_regex = component_data.get('sha_regex').format(arch=tmp_arch) + checksum = download_file_and_get_checksum(component, arch, url_download, processed_version, sha_regex, session) or 0 + checksums[version][arch] = checksum + elif component_data['checksum_structure'] == 'simple': + # Checksum + url_download = url_download_template.format(version=processed_version) + sha_regex = component_data.get('sha_regex') + checksum = download_file_and_get_checksum(component, '', url_download, processed_version, sha_regex, session) or 0 + checksums[version] = checksum # Store checksum for the version + return checksums + +def update_checksum(component, component_data, checksums, version): + processed_version = process_version_string(component, version) + placeholder_checksum = component_data['placeholder_checksum'] + checksum_structure = component_data['checksum_structure'] + current = checksum_yaml_data[placeholder_checksum] + + if checksum_structure == 'simple': + # Simple structure (placeholder_checksum -> version -> checksum) + checksum_yaml_data[placeholder_checksum] = {processed_version: checksums, **current} + elif checksum_structure == 'os_arch': + # OS structure (placeholder_checksum -> os -> arch -> version -> checksum) + for os_name, arch_dict in checksums.items(): + os_current = current.setdefault(os_name, {}) + for arch, checksum in arch_dict.items(): + os_current[arch] = {(processed_version): checksum, **os_current.get(arch, {})} + elif checksum_structure == 'arch': + # Arch structure (placeholder_checksum -> arch -> version -> checksum) + for arch, checksum in checksums.items(): + current[arch] = {(processed_version): checksum, **current.get(arch, {})} + logging.info(f'Updated {placeholder_checksum} with version {processed_version} and checksums {checksums}') + +def resolve_kube_dependent_component_version(component, component_data, version): + kube_major_version = component_data['kube_major_version'] + if component in ['crictl', 'crio']: + try: + component_major_version = get_major_version(version) + if component_major_version == kube_major_version: + resolved_version = kube_major_version + else: + resolved_version = component_major_version + except (IndexError, AttributeError): + logging.error(f'Error parsing version for {component}: {version}') + return + else: + resolved_version = kube_major_version + return resolved_version + +def update_version(component, component_data, version): + placeholder_version = component_data['placeholder_version'] + resolved_version = resolve_kube_dependent_component_version(component, component_data, version) + updated_placeholder = [ + resolved_version if item == 'kube_major_version' else item + for item in placeholder_version + ] + current = download_yaml_data + if len(updated_placeholder) == 1: + current[updated_placeholder[0]] = version + else: + for key in updated_placeholder[:-1]: + current = current.setdefault(key, {}) + final_key = updated_placeholder[-1] + if final_key in current: + current[final_key] = version + else: + new_entry = {final_key: version, **current} + current.clear() + current.update(new_entry) + logging.info(f'Updated {updated_placeholder} to {version}') + +def update_readme(component, version): + for i, line in enumerate(readme_data): + if component in line and re.search(r'v\d+\.\d+\.\d+', line): + readme_data[i] = re.sub(r'v\d+\.\d+\.\d+', version, line) + logging.info(f'Updated {component} to {version} in README') + break + return readme_data + +def safe_save_files(file_path, data=None, save_func=None): + if not save_func(file_path, data): + logging.error(f'Failed to save file {file_path}') + sys.exit(1) + +def create_json_file(file_path): + new_data = {} + try: + with open(file_path, 'w') as f: + json.dump(new_data, f, indent=2) + return new_data + except Exception as e: + logging.error(f'Failed to create {file_path}: {e}') + return None + +def save_json_file(file_path, data): + try: + with open(file_path, 'w') as f: + json.dump(data, f, indent=2) + return True + except Exception as e: + logging.error(f'Failed to save {file_path}: {e}') + return False + +def load_yaml_file(yaml_file): + try: + with open(yaml_file, 'r') as f: + return yaml.load(f) + except Exception as e: + logging.error(f'Failed to load {yaml_file}: {e}') + return None + +def save_yaml_file(yaml_file, data): + try: + with open(yaml_file, 'w') as f: + yaml.dump(data, f) + return True + except Exception as e: + logging.error(f'Failed to save {yaml_file}: {e}') + return False + +def open_readme(path_readme): + try: + with open(path_readme, 'r') as f: + return f.readlines() + except Exception as e: + logging.error(f'Failed to load {path_readme}: {e}') + return None + +def save_readme(path_readme, data): + try: + with open(path_readme, 'w') as f: + f.writelines(data) + return True + except Exception as e: + logging.error(f'Failed to save {path_readme}: {e}') + return False + +def process_version_string(component, version): + if component in ['youki', 'nerdctl', 'cri_dockerd', 'containerd']: + if version.startswith('v'): + version = version[1:] + return version + match = re.search(r'release-(\d{8})', version) # gvisor + if match: + version = match.group(1) + return version + +def get_major_version(version): + match = re.match(r'^(v\d+)\.(\d+)', version) + if match: + return f'{match.group(1)}.{match.group(2)}' + return None + +def process_component(component, component_data, repo_metadata, session): + logging.info(f'Processing component: {component}') + component_repo_metada = repo_metadata.get(component, {}) + + # Get current kube version + kube_version = main_yaml_data.get('kube_version') + kube_major_version = get_major_version(kube_version) + component_data['kube_version'] = kube_version # needed for nested components + component_data['kube_major_version'] = kube_major_version # needed for nested components + + # Get current component version + current_version = get_current_version(component, component_data) + if not current_version: + logging.info(f'Stop processing component {component}, current version unknown') + return + + # Get latest component version + latest_version = get_latest_version(component_repo_metada) + if not latest_version: + logging.info(f'Stop processing component {component}, latest version unknown.') + return + # Kubespray version + processed_latest_version = process_version_string(component, latest_version) + + # Log version comparison + if current_version == processed_latest_version: + logging.info(f'Component {component}, version {current_version} is up to date') + else: + logging.info(f'Component {component} version discrepancy, current={current_version}, latest={processed_latest_version}') + + # CI - write data and return + if args.ci_check and current_version != latest_version: + version_diff[component] = { + # used in dependecy-check.yml workflow + 'current_version' : current_version, + 'latest_version' : latest_version, # used for PR name + # used in generate_pr_body.py script + 'processed_latest_version': processed_latest_version, # used for PR body + 'owner' : component_data['owner'], + 'repo' : component_data['repo'], + 'repo_metadata' : component_repo_metada, + } + return + + # Get patch versions + patch_versions = get_patch_versions(component, latest_version, component_repo_metada) + logging.info(f'Component {component} patch versions: {patch_versions}') + + # Get checksums for all patch versions + checksums = get_checksums(component, component_data, patch_versions, session) + # Update checksums + for version in patch_versions: + version_checksum = checksums.get(version) + update_checksum(component, component_data, version_checksum, version) + + # Update version in configuration + if component not in ['kubeadm', 'kubectl', 'kubelet']: # kubernetes dependent components + if component != 'calico_crds': # TODO double check if only calicoctl may change calico_version + update_version(component, component_data, processed_latest_version) + + # Update version in README + if component in README_COMPONENTS: + if component in ['crio', 'crictl']: + component_major_version = get_major_version(processed_latest_version) + if component_major_version != kube_major_version: # do not update README, we just added checksums + return + # replace component name to fit readme + component = component.replace('crio', 'cri-o').replace('calicoctl', 'calico') + update_readme(component, latest_version) + +def main(): + # Setup logging + setup_logging(args.loglevel) + # Setup session with retries + session = get_session_with_retries() + + # Load configuration files + global main_yaml_data, checksum_yaml_data, download_yaml_data, readme_data, version_diff + main_yaml_data = load_yaml_file(PATH_MAIN) + checksum_yaml_data = load_yaml_file(PATH_CHECKSUM) + download_yaml_data = load_yaml_file(PATH_DOWNLOAD) + readme_data = open_readme(PATH_README) + if not (main_yaml_data and checksum_yaml_data and download_yaml_data and readme_data): + logging.error(f'Failed to open one or more configuration files, current working directory is {pwd}. Exiting...') + sys.exit(1) + + # CI - create version_diff file + if args.ci_check: + version_diff = create_json_file(PATH_VERSION_DIFF) + if version_diff is None: + logging.error(f'Failed to create {PATH_VERSION_DIFF} file') + sys.exit(1) + + # Process single component + if args.component != 'all': + if args.component in COMPONENT_INFO: + specific_component_info = {args.component: COMPONENT_INFO[args.component]} + # Get repository metadata => releases, tags and commits + logging.info(f'Fetching repository metadata for the component {args.component}') + repo_metadata = get_repository_metadata(specific_component_info, session) + if not repo_metadata: + sys.exit(1) + process_component(args.component, COMPONENT_INFO[args.component], repo_metadata, session) + else: + logging.error(f'Component {args.component} not found in config.') + sys.exit(1) + # Process all components in the configuration file concurrently + else: + # Get repository metadata => releases, tags and commits + logging.info('Fetching repository metadata for all components') + repo_metadata = get_repository_metadata(COMPONENT_INFO, session) + if not repo_metadata: + sys.exit(1) + with ThreadPoolExecutor(max_workers=args.max_workers) as executor: + futures = [] + logging.info(f'Running with {executor._max_workers} executors') + for component, component_data in COMPONENT_INFO.items(): + futures.append(executor.submit(process_component, component, component_data, repo_metadata, session)) + for future in futures: + future.result() + + # CI - save JSON file + if args.ci_check: + safe_save_files(PATH_VERSION_DIFF, version_diff, save_json_file) + + # Save configurations + else: + safe_save_files(PATH_CHECKSUM, checksum_yaml_data, save_yaml_file) + safe_save_files(PATH_DOWNLOAD, download_yaml_data, save_yaml_file) + safe_save_files(PATH_README, readme_data, save_readme) + + logging.info('Finished.') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Kubespray version and checksum updater for dependencies') + parser.add_argument('--loglevel', default='INFO', help='Set the log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)') + parser.add_argument('--component', default='all', help='Specify a component to process, default is all components') + parser.add_argument('--max-workers', type=int, default=4, help='Maximum number of concurrent workers, use with caution(sometimes less is more)') + parser.add_argument('--ci-check', action='store_true', help='Check versions, store discrepancies in version_diff.json') + parser.add_argument('--graphql-number-of-entries', type=int, default=10, help='Number of releases/tags to retrieve from Github GraphQL per component (default: 10)') + parser.add_argument('--graphql-number-of-commits', type=int, default=5, help='Number of commits to retrieve from Github GraphQL per tag (default: 5)') + args = parser.parse_args() + + main() diff --git a/scripts/generate_pr_body.py b/scripts/generate_pr_body.py new file mode 100644 index 00000000000..9ec937a8bbf --- /dev/null +++ b/scripts/generate_pr_body.py @@ -0,0 +1,130 @@ +import sys +import json +import argparse + +# Do not commit any prints if the script doesn't exit with error code +# Otherwise it will be part of the PR body + + +def load_json(component): + try: + with open('version_diff.json', 'r') as f: + repo_metadata = json.load(f) + component_data = repo_metadata.get(component) + return component_data + except Exception as e: + return None + +def get_version_commits(version, component_metadata): + tags = component_metadata.get('refs', {}).get('nodes', []) + for tag in tags: + if tag['name'] == version: + target = tag.get('target', {}) + + # Check if the target is a Tag pointing to a Commit + if 'history' in target.get('target', {}): + commit_history = target['target']['history'].get('edges', []) + # Check if the target is directly a Commit object + elif 'history' in target: + commit_history = target['history'].get('edges', []) + else: + return None + + commits = [] + for commit in commit_history: + commit_node = commit.get('node', {}) + commit_info = { + 'oid': commit_node.get('oid'), + 'message': commit_node.get('message'), + 'url': commit_node.get('url') + } + commits.append(commit_info) + + if commits: + return commits + return None + +def get_version_description(version, repo_metadata): + if repo_metadata: + releases = repo_metadata.get('releases', {}).get('nodes', []) + for release in releases: + if release.get('tagName') == version: + description = release.get('description', None) + return format_description(description) + return None + +def handle_reference(input): + return input.replace('github.com', 'redirect.github.com') # Prevent reference in the sourced PR + +# Split description into visible and collapsed +def format_description(description): + description = handle_reference(description) + lines = description.splitlines() + if len(lines) > args.description_number_of_lines: + first_part = '\n'.join(lines[:args.description_number_of_lines]) + collapsed_part = '\n'.join(lines[args.description_number_of_lines:]) + formatted_description = f"""{first_part} + +
+ Show more + +{collapsed_part} + +
+""" + return formatted_description + else: + return description + +def main(): + component_data = load_json(args.component) + if not component_data: + print('Failed to load component data') + sys.exit(1) + owner = component_data.get('owner') + repo = component_data.get('repo') + latest_version = component_data.get('latest_version') + repo_metadata = component_data.get('repo_metadata') + release_url = f'https://github.com/{owner}/{repo}/releases/tag/{latest_version}' + commits = get_version_commits(latest_version, repo_metadata) + description = get_version_description(latest_version, repo_metadata) + + # General info + pr_body = f""" +### {latest_version} + +**URL**: [Release {latest_version}]({release_url}) + + """ + + # Description + if description: + + pr_body += f""" +#### Description: +{description} + """ + + # Commits + if commits: + pr_commits = '\n
\nCommits\n\n' + for commit in commits: + short_oid = commit.get('oid')[:7] + message = commit.get('message').split('\n')[0] + commit_message = handle_reference(message) + # commit_message = link_pull_requests(commit_message, repo_url) + commit_url = commit.get('url') + pr_commits += f'- [`{short_oid}`]({commit_url}) {commit_message} \n' + pr_commits += '\n
' + pr_body += pr_commits + + # Print body + print(pr_body) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Pull Request body generator') + parser.add_argument('--component', required=True, help='Specify the component to process') + parser.add_argument('--description-number-of-lines', type=int, default=20, help='Number of lines to include from the description') + args = parser.parse_args() + + main()