From d85c142cdd9bffeab1cd8160f1b021d3619e499e Mon Sep 17 00:00:00 2001 From: Florent Benoit Date: Sun, 7 Apr 2024 22:15:17 +0200 Subject: [PATCH] chore: add scripts to download podman 5 machines For Windows download from another repository on GitHub containers/podman-machine-wsl-os and for macOS, fetch OCI images from quay.io/v2/podman/machine-os related to https://github.com/containers/podman-desktop/issues/6360 Signed-off-by: Florent Benoit --- extensions/podman/scripts/download.ts | 7 +- .../podman/scripts/podman-download.spec.ts | 237 ++++++++++++++- extensions/podman/scripts/podman-download.ts | 273 ++++++++++++++++-- 3 files changed, 491 insertions(+), 26 deletions(-) diff --git a/extensions/podman/scripts/download.ts b/extensions/podman/scripts/download.ts index 35a2f5518b111..2245cccec4018 100644 --- a/extensions/podman/scripts/download.ts +++ b/extensions/podman/scripts/download.ts @@ -23,8 +23,7 @@ import * as podman4JSON from '../src/podman4.json'; import * as podman5JSON from '../src/podman5.json'; const podman4Download = new PodmanDownload(podman4JSON, true); -podman4Download.downloadBinaries(); +await podman4Download.downloadBinaries(); -// do not fetch for airgap mode now -const podman5Download = new PodmanDownload(podman5JSON, false); -podman5Download.downloadBinaries(); +const podman5Download = new PodmanDownload(podman5JSON, true); +await podman5Download.downloadBinaries(); diff --git a/extensions/podman/scripts/podman-download.spec.ts b/extensions/podman/scripts/podman-download.spec.ts index 60a41388b8b4e..bc526f9a88588 100644 --- a/extensions/podman/scripts/podman-download.spec.ts +++ b/extensions/podman/scripts/podman-download.spec.ts @@ -20,6 +20,7 @@ import { afterEach } from 'node:test'; import { beforeEach, describe, expect, test, vi } from 'vitest'; import { DownloadAndCheck, + Podman5DownloadMachineOS, PodmanDownload, PodmanDownloadFcosImage, PodmanDownloadFedoraImage, @@ -27,8 +28,13 @@ import { } from './podman-download'; import * as podman4JSON from '../src/podman4.json'; import nock from 'nock'; -import { appendFileSync, existsSync, mkdirSync } from 'node:fs'; +import { WriteStream, appendFileSync, createWriteStream, existsSync, mkdirSync } from 'node:fs'; import { Octokit } from 'octokit'; +import { PassThrough, Readable, Writable } from 'node:stream'; +import { WritableStream, WritableStreamDefaultWriter } from 'stream/web'; +import path from 'node:path'; +import { tmpdir } from 'node:os'; +import exp from 'node:constants'; const mockedPodman4 = { version: '4.5.0', @@ -256,6 +262,8 @@ test('downloadAndCheckSha', async () => { } as unknown as ShaCheck; // mock GitHub requests + vi.mocked(shaCheck.checkFile).mockResolvedValue(true); + const response = { name: 'vFakeVersion', assets: [ @@ -312,3 +320,230 @@ test('downloadAndCheckSha', async () => { // check the sha expect(shaCheck.checkFile).toHaveBeenCalledWith(expect.stringContaining('podman-fake-binary'), 'fake-sha'); }); + +describe('Podman5DownloadMachineOS', () => { + const shaCheck = { + checkFile: vi.fn(), + } as unknown as ShaCheck; + + beforeEach(() => { + vi.resetAllMocks(); + vi.mocked(shaCheck.checkFile).mockResolvedValue(true); + }); + + class TestPodman5DownloadMachineOS extends Podman5DownloadMachineOS { + public pipe( + title: string, + total: number, + stream: ReadableStream, + writableStream: globalThis.WritableStream, + ): Promise { + return super.pipe(title, total, stream, writableStream); + } + } + + test('download all the files and perform checks', async () => { + // spy Writable.toWeb + vi.spyOn(Writable, 'toWeb').mockResolvedValue({} as unknown as WritableStream); + + // mock manifests + const rootManifest = { + schemaVersion: 2, + mediaType: 'application/vnd.oci.image.index.v1+json', + manifests: [ + { + mediaType: 'application/vnd.oci.image.manifest.v1+json', + digest: 'sha256:123amd64', + size: 481, + annotations: { + disktype: 'applehv', + }, + platform: { + architecture: 'x86_64', + os: 'linux', + }, + }, + { + mediaType: 'application/vnd.oci.image.manifest.v1+json', + digest: 'sha256:456arm64', + size: 482, + annotations: { + disktype: 'applehv', + }, + platform: { + architecture: 'aarch64', + os: 'linux', + }, + }, + { + mediaType: 'application/vnd.oci.image.manifest.v1+json', + digest: 'sha256:ee45494db66e33525f50835af65c4099db4db7a066b1da9a85fba7e88f95f594', + size: 481, + annotations: { + disktype: 'hyperv', + }, + platform: { + architecture: 'x86_64', + os: 'linux', + }, + }, + { + mediaType: 'application/vnd.oci.image.manifest.v1+json', + digest: 'sha256:56bbdde7b2dc8714a0397f37ae1c8ac1353ed3e4de1b09c1db791d6fa5bc56fa', + size: 482, + annotations: { + disktype: 'hyperv', + }, + platform: { + architecture: 'aarch64', + os: 'linux', + }, + }, + { + mediaType: 'application/vnd.oci.image.manifest.v1+json', + digest: 'sha256:c25ce5ba618f870f88c418c7aa0f176af4a7b32ed39aa328a8e27eae5a497e11', + size: 480, + annotations: { + disktype: 'qemu', + }, + platform: { + architecture: 'x86_64', + os: 'linux', + }, + }, + { + mediaType: 'application/vnd.oci.image.manifest.v1+json', + digest: 'sha256:9a9285bd1a01e5b4c5b27467025cc5da281e250913432a9f92cf8fe1668fec19', + size: 481, + annotations: { + disktype: 'qemu', + }, + platform: { + architecture: 'aarch64', + os: 'linux', + }, + }, + { + mediaType: 'application/vnd.oci.image.manifest.v1+json', + digest: 'sha256:34f21a8b9b8b9ff13fc348f8eba72fa92e74e52caa9984a78b68a7f5822641c7', + size: 11003, + platform: { + architecture: 'aarch64', + os: 'linux', + }, + }, + { + mediaType: 'application/vnd.oci.image.manifest.v1+json', + digest: 'sha256:c7bbce32d96c44b0db6e05a3f78fa4cb11f578c7d18cec49d23a7d6b40843a05', + size: 11009, + platform: { + architecture: 'x86_644', + os: 'linux', + }, + }, + ], + }; + + nock('https://quay.io').get('/v2/podman/machine-os/manifests/1.0-fake').reply(200, rootManifest); + + // fake digest for amd64 + nock('https://quay.io') + .get('/v2/podman/machine-os/manifests/sha256:123amd64') + .reply(200, { + schemaVersion: 2, + mediaType: 'application/vnd.oci.image.manifest.v1+json', + config: { + mediaType: 'application/vnd.oci.empty.v1+json', + digest: 'sha256:1234', + size: 2, + data: 'e30=', + }, + layers: [ + { + mediaType: 'application/zstd', + digest: 'sha256:zstfakeamd64digest', + size: 1233263850, + annotations: { + 'org.opencontainers.image.title': 'podman-machine-daily.amd64.applehv.raw.zst', + }, + }, + ], + }); + + // fake digest for arm64 + nock('https://quay.io') + .get('/v2/podman/machine-os/manifests/sha256:456arm64') + .reply(200, { + schemaVersion: 2, + mediaType: 'application/vnd.oci.image.manifest.v1+json', + config: { + mediaType: 'application/vnd.oci.empty.v1+json', + digest: 'sha256:1234', + size: 2, + data: 'e30=', + }, + layers: [ + { + mediaType: 'application/zstd', + digest: 'sha256:zstfakearm64digest', + size: 1233263850, + annotations: { + 'org.opencontainers.image.title': 'podman-machine-daily.aarch64.applehv.raw.zst', + }, + }, + ], + }); + + // now do the digests for blobs + nock('https://quay.io') + .get('/v2/podman/machine-os/blobs/sha256:zstfakeamd64digest') + .reply(200, 'fake-amd64-content'); + + const zstdArchiveFakeContent = 'blablabla-ARM64\n'; + nock('https://quay.io') + .get('/v2/podman/machine-os/blobs/sha256:zstfakearm64digest') + .reply(200, zstdArchiveFakeContent, { + 'content-type': 'application/octet-stream', + 'content-length': `${zstdArchiveFakeContent.length}`, + 'content-disposition': 'attachment; filename=binary.zip', + }); + + const fakeContent = 'blablabla-ARM64\n'; + + const processArm64File = (): Buffer => { + return Buffer.from(fakeContent); + }; + + nock('https://quay.io') + .get('/v2/podman/machine-os/blobs/sha256:zstfakearm64digest') + .reply(200, processArm64File(), { + 'content-type': 'application/octet-stream', + 'content-length': `${fakeContent.length}`, + 'content-disposition': 'attachment; filename=foo.raw.std', + }); + + const podman5DownloadMachineOS = new TestPodman5DownloadMachineOS('1.0-fake', shaCheck, '/fake-directory'); + + vi.spyOn(podman5DownloadMachineOS, 'pipe').mockResolvedValue(); + + await podman5DownloadMachineOS.download(); + }); + + test('check pipe method', async () => { + const podman5DownloadMachineOS = new TestPodman5DownloadMachineOS('1.0-fake', shaCheck, '/fake-directory'); + + const myStream = Readable.from('Hello, World!'); + + const readableStream = Readable.toWeb(myStream) as ReadableStream; + + const writeMock = vi.fn(); + const writableStream = new WritableStream({ + write: writeMock, + }); + + await podman5DownloadMachineOS.pipe('fake-title', 100, readableStream, writableStream); + + // check we wrote the content + expect(writeMock).toHaveBeenCalledWith('Hello, World!', expect.anything()); + }); +}); diff --git a/extensions/podman/scripts/podman-download.ts b/extensions/podman/scripts/podman-download.ts index 4e4ca2ac9f566..bdef394644381 100644 --- a/extensions/podman/scripts/podman-download.ts +++ b/extensions/podman/scripts/podman-download.ts @@ -23,6 +23,7 @@ import { Octokit } from 'octokit'; import type { OctokitOptions } from '@octokit/core/dist-types/types'; import { hashFile } from 'hasha'; import { fileURLToPath } from 'node:url'; +import { Writable } from 'node:stream'; // to make this file a module export class PodmanDownload { @@ -31,6 +32,8 @@ export class PodmanDownload { #downloadAndCheck: DownloadAndCheck; #podmanDownloadFcosImage: PodmanDownloadFcosImage; #podmanDownloadFedoraImage: PodmanDownloadFedoraImage; + #podman5DownloadFedoraImage: Podman5DownloadFedoraImage | undefined; + #podman5DownloadMachineOS: Podman5DownloadMachineOS | undefined; #shaCheck: ShaCheck; @@ -121,6 +124,23 @@ export class PodmanDownload { if (!fs.existsSync(this.#assetsFolder)) { fs.mkdirSync(this.#assetsFolder); } + + if (podmanJSON.version.startsWith('5.')) { + // grab only first 2 digits from the version + const majorMinorVersion = podmanJSON.version.split('.').slice(0, 2).join('.'); + + this.#podman5DownloadFedoraImage = new Podman5DownloadFedoraImage( + majorMinorVersion, + this.#octokit, + this.#downloadAndCheck, + ); + + this.#podman5DownloadMachineOS = new Podman5DownloadMachineOS( + majorMinorVersion, + this.#shaCheck, + this.#assetsFolder, + ); + } } protected getPodmanDownloadFcosImage(): PodmanDownloadFcosImage { @@ -141,7 +161,7 @@ export class PodmanDownload { async downloadBinaries(): Promise { // fetch from GitHub releases for (const artifact of this.#artifactsToDownload) { - this.#downloadAndCheck.downloadAndCheckSha(artifact.version, artifact.downloadName, artifact.artifactName); + await this.#downloadAndCheck.downloadAndCheckSha(artifact.version, artifact.downloadName, artifact.artifactName); } // fetch optional binaries in case of AirGap @@ -155,13 +175,19 @@ export class PodmanDownload { if (this.#platform === 'win32') { // download the fedora image - this.#podmanDownloadFedoraImage.download('podman-wsl-fedora', 'x64'); - this.#podmanDownloadFedoraImage.download('podman-wsl-fedora-arm', 'arm64'); + + await this.#podman5DownloadFedoraImage?.download('x64'); + await this.#podman5DownloadFedoraImage?.download('arm64'); + + await this.#podmanDownloadFedoraImage.download('podman-wsl-fedora', 'x64'); + await this.#podmanDownloadFedoraImage.download('podman-wsl-fedora-arm', 'arm64'); } else if (this.#platform === 'darwin') { // download the fedora core os images for both arches - await this.#podmanDownloadFcosImage.download('x64'); await this.#podmanDownloadFcosImage.download('arm64'); + + // download the podman 5 machines OS + await this.#podman5DownloadMachineOS?.download(); } } } @@ -278,21 +304,21 @@ export class PodmanDownloadFcosImage { const destFile = path.resolve(this.#assetsFolder, filename); if (!fs.existsSync(destFile)) { // download the file from diskLocation - console.log(`Downloading Podman package from ${diskLocation}`); + console.log(`⚡️ Downloading Podman package from ${diskLocation}`); await this.httpsDownloader.downloadFile(diskLocation, destFile); - console.log(`Downloaded to ${destFile}`); + console.log(`📔 Downloaded to ${destFile}`); } else { - console.log(`Podman image ${filename} already downloaded.`); + console.log(`⏭️ Skipping podman image (already downloaded to ${filename})`); } if (!(await this.#shaCheck.checkFile(destFile, sha256))) { - console.warn(`Checksum for downloaded ${destFile} is not matching, downloading again...`); + console.warn(`❌ Invalid checksum for downloaded ${destFile} is not matching, downloading again...`); fs.rmSync(destFile); this.#downloadAttempt++; // call the loop again - this.download(arch); + await this.download(arch); } else { - console.log(`Checksum for ${filename} matched.`); + console.log(`✅ Valid checksum for ${filename}`); } } } @@ -338,11 +364,11 @@ export class PodmanDownloadFedoraImage { const destFile = path.resolve(this.#assetsFolder, filename); if (!fs.existsSync(destFile)) { // download the file from diskLocation - console.log(`Downloading Podman package from ${artifactRelease.browser_download_url}`); + console.log(`⚡️ Downloading Podman package from ${artifactRelease.browser_download_url}`); await this.#httpsDownloader.downloadFile(artifactRelease.browser_download_url, destFile); - console.log(`Downloaded to ${destFile}`); + console.log(`📔 Downloaded to ${destFile}`); } else { - console.log(`Podman image ${filename} already downloaded.`); + console.log(`⏭️ Skipping Windows podman image for ${arch} (already downloaded to ${filename})`); } } } @@ -423,7 +449,7 @@ export class DownloadAndCheck { const destFile = path.resolve(this.#assetsFolder, fileName); if (!fs.existsSync(destFile)) { - console.log(`Downloading artifact from ${artifactRelease.browser_download_url}`); + console.log(`⚡️ Downloading artifact from ${artifactRelease.browser_download_url}`); // await downloadFile(url, destFile); const artifactAsset = await this.#octokit.rest.repos.getReleaseAsset({ asset_id: artifactRelease.id, @@ -435,22 +461,227 @@ export class DownloadAndCheck { }); fs.appendFileSync(destFile, Buffer.from(artifactAsset.data as unknown as ArrayBuffer)); - console.log(`Downloaded to ${destFile}`); + console.log(`📔 Downloaded to ${destFile}`); } else { - console.log(`Artifact ${artifactRelease.browser_download_url} already downloaded.`); + console.log(`⏭️ Skipping ${artifactName} (already downloaded)`); } - console.log(`Verifying ${fileName}...`); - if (!(await this.#shaCheck.checkFile(destFile, msiSha))) { - console.warn(`Checksum for downloaded ${destFile} does not match, downloading again...`); + console.warn(`❌ Invalid checksum for ${fileName} downloading again...`); fs.rmSync(destFile); this.#downloadAttempt++; - this.downloadAndCheckSha(tagVersion, fileName, artifactName); + await this.downloadAndCheckSha(tagVersion, fileName, artifactName); } else { - console.log(`Checksum for ${fileName} is matching.`); + console.log(`✅ Valid checksum for ${fileName}`); } this.#downloadAttempt = 0; } } + +export class Podman5DownloadFedoraImage { + readonly MAX_DOWNLOAD_ATTEMPT = 3; + #downloadAttempt = 0; + #octokit: Octokit; + #version: string; + + #downloadAndCheck: DownloadAndCheck; + + constructor( + readonly version: string, + readonly octokit: Octokit, + readonly downloadAndCheck: DownloadAndCheck, + ) { + this.#version = version; + this.#octokit = octokit; + this.#downloadAndCheck = downloadAndCheck; + } + + // For Windows binaries, grab the latest release from GitHub repository + async download(arch: string): Promise { + if (this.#downloadAttempt >= this.MAX_DOWNLOAD_ATTEMPT) { + console.error('Max download attempt reached, exiting...'); + process.exit(1); + } + + const owner = 'containers'; + const repo = 'podman-machine-wsl-os'; + + // now, grab the files + const release = await this.#octokit.request('GET /repos/{owner}/{repo}/releases/latest', { + owner, + repo, + }); + + let artifactArch; + if (arch === 'x64') { + artifactArch = 'amd64'; + } else if (arch === 'arm64') { + artifactArch = 'arm64'; + } + + const artifactName = `${this.#version}-rootfs-${artifactArch}.tar.zst`; + const filename = `podman-image-${arch}.tar.zst`; + const artifactRelease = release.data.assets.find(asset => asset.name === artifactName); + + if (!artifactRelease) { + throw new Error( + `Can't find asset with name ${artifactName} to download and verify for podman image from repository ${repo}`, + ); + } + + // tag version + const tagVersion = release.data.tag_name; + await this.#downloadAndCheck.downloadAndCheckSha(tagVersion, filename, artifactName, owner, repo); + } +} + +export class Podman5DownloadMachineOS { + #version: string; + #shaCheck: ShaCheck; + #assetsFolder: string; + + constructor( + readonly version: string, + readonly shaCheck: ShaCheck, + readonly assetsFolder: string, + ) { + this.#version = version; + this.#shaCheck = shaCheck; + this.#assetsFolder = assetsFolder; + } + + async getManifest(manifestUrl: string): Promise { + const response = await fetch(manifestUrl, { + method: 'GET', + headers: { + 'docker-distribution-api-version': 'registry/2.0', + Accept: 'application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json', + }, + }); + return response.json(); + } + + protected async pipe( + title: string, + total: number, + stream: ReadableStream, + writableStream: WritableStream, + ) { + let loaded = 0; + + var progress = new TransformStream({ + transform(chunk, controller) { + loaded += chunk.length; + + // 20 chars = 100% + const i = Math.floor((loaded / total) * 20); + const dots = '.'.repeat(i); + const left = 20 - i; + const empty = ' '.repeat(left); + + process.stdout.write(`\r⚡️ Downloading ${title} [${dots}${empty}] ${i * 5}%`); + controller.enqueue(chunk); + }, + }); + + await stream.pipeThrough(progress).pipeTo(writableStream); + } + + async downloadZstdFromManifest( + title: string, + filename: string, + layer: { digest: string; size: number }, + ): Promise { + const blobURL = `https://quay.io/v2/podman/machine-os/blobs/${layer.digest}`; + + const blobResponse = await fetch(blobURL); + const total = layer.size; + const outputFile = path.resolve(this.#assetsFolder, filename); + // digest is using the format : sha256:checksum + // extract the checksum + const checksum = layer.digest.split(':')[1]; + + // check if the file exists and has the expected checksum + if (fs.existsSync(outputFile)) { + // check now the checksum + const valid = await this.#shaCheck.checkFile(outputFile, checksum); + if (valid) { + console.log(`⏭️ Skipping ${title} (already downloaded to ${filename})`); + return; + } + } + + const writer = fs.createWriteStream(outputFile); + const writableStream = Writable.toWeb(writer); + + if (!blobResponse.body) { + throw new Error(`❌ Cannot get blob for ${title}`); + } + + await this.pipe(title, total, blobResponse.body, writableStream); + + process.stdout.write(`\r📔 ${title} downloaded to ${filename}\n`); + + // verify the checksum + const valid = await this.#shaCheck.checkFile(outputFile, checksum); + if (valid) { + console.log(`✅ Valid checksum for ${filename}`); + } else { + throw new Error(`❌ Invalid checksum for ${filename}`); + } + } + + // For macOS, need to grab images from quay.io/podman/machine-os repository + async download(): Promise { + const manifestUrl = `https://quay.io/v2/podman/machine-os/manifests/${this.#version}`; + + // get first level of manifests + const rootManifest = await this.getManifest(manifestUrl); + + if (rootManifest.errors) { + console.error(`❌ Cannot get manifest for ${manifestUrl}`, rootManifest.errors); + throw new Error(`❌ Cannot get manifest for ${manifestUrl}`); + } + + const manifests = rootManifest.manifests; + + // grab applehv as annotations / disktype + const keepManifests = manifests.filter(manifest => { + const annotations = manifest.annotations; + return annotations && annotations.disktype === 'applehv'; + }); + + // should have aarch64 for arm64 and x86_64 for x64 + const amd64Manifest = keepManifests.find( + manifest => manifest.platform.architecture === 'x86_64' && manifest.platform.os === 'linux', + ); + const arm64Manifest = keepManifests.find( + manifest => manifest.platform.architecture === 'aarch64' && manifest.platform.os === 'linux', + ); + + if (!amd64Manifest || !arm64Manifest) { + throw new Error('❌ Cannot find amd64 or arm64 manifest'); + } + + // now get the zstd entry from the arch manifest + const amd64ZstdManifest = await this.getManifest( + `https://quay.io/v2/podman/machine-os/manifests/${amd64Manifest.digest}`, + ); + const arm64ZstdManifest = await this.getManifest( + `https://quay.io/v2/podman/machine-os/manifests/${arm64Manifest.digest}`, + ); + + // download the zstd layers + await this.downloadZstdFromManifest( + `${manifestUrl} for arm64`, + 'podman-image-arm64.zst', + arm64ZstdManifest.layers[0], + ); + await this.downloadZstdFromManifest( + `${manifestUrl} for amd64`, + 'podman-image-amd64.zst', + amd64ZstdManifest.layers[0], + ); + } +}