Improve scripts, add new iso build mode - "empty", update repositories (

#29) * Improve scripts * Add new iso build mode - "empty" * Update repositories
sasha0552 · Mar 26, 2024 · e0f9138 · e0f9138
1 parent 864a075
commit e0f9138
Show file tree

Hide file tree

Showing 34 changed files with 243 additions and 166 deletions.
diff --git a/.ci/actions.py b/.ci/actions.py
@@ -2,27 +2,28 @@
 
 import jinja2
 
-def main():
-  for filename in [ "gh-build-iso", "sh-build-iso" ]:
-    for type in [ "cuda", "rocm" ]:
-      # read input file
-      with open(f".ci/template/{filename}.yml.jinja2", "r") as file:
-        template = jinja2.Template(file.read())
+def render_template(input_path, output_path, **options):
+  # read input file
+  with open(input_path, "r") as file:
+    template = jinja2.Template(file.read(), lstrip_blocks=True, trim_blocks=True)
 
-      # render template
-      rendered = template.render(type=type)
+  # render template
+  rendered = template.render(**options)
 
-      # FIXME: skip if hosted and rocm
-      if filename == "gh-build-iso" and type == "rocm":
-        continue
+  # write output file
+  with open(output_path, "w") as file:
+    file.write(rendered)
 
-      # FIXME: skip if selfhosted and cuda
-      if filename == "sh-build-iso" and type == "cuda":
-        continue
+def main():
+  # define directories
+  i = ".ci/template"
+  o = ".github/workflows"
 
-      # write output file
-      with open(f".github/workflows/{filename}-{type}.yml", "w") as file:
-        file.write(rendered)
+  # render templates
+  render_template(f"{i}/gh-build-iso.yml.jinja2", f"{o}/gh-build-iso-cuda.yml"      , platform="cuda", type="normal")
+  render_template(f"{i}/gh-build-iso.yml.jinja2", f"{o}/gh-build-iso-cuda-empty.yml", platform="cuda", type="empty")
+  render_template(f"{i}/sh-build-iso.yml.jinja2", f"{o}/sh-build-iso-rocm.yml"      , platform="rocm", type="normal")
+  render_template(f"{i}/gh-build-iso.yml.jinja2", f"{o}/gh-build-iso-rocm-empty.yml", platform="rocm", type="empty")
 
 if __name__ == "__main__":
   main()
diff --git a/.ci/configure.py b/.ci/configure.py
@@ -8,7 +8,7 @@ def render_template(filepath, **options):
   if filepath.endswith(".jinja2"):
     # read input file
     with open(filepath, "r") as file:
-      template = jinja2.Template(file.read())
+      template = jinja2.Template(file.read(), lstrip_blocks=True, trim_blocks=True)
 
     # render template
     rendered = template.render(**options)
@@ -19,14 +19,18 @@ def render_template(filepath, **options):
 
 def main():
   # by default, use cuda
-  cuda = True
-  rocm = False
+  platform = "cuda"
 
-  # enable rocm if specified
-  if len(sys.argv) == 2:
-    if sys.argv[1] == "rocm":
-      cuda = False
-      rocm = True
+  # and normal iso
+  type = "normal"
+
+  # set platform
+  if len(sys.argv) >= 2:
+    platform = sys.argv[1]
+
+  # set type
+  if len(sys.argv) >= 3:
+    type = sys.argv[2]
 
   # list of rendered files
   rendered = []
@@ -37,10 +41,10 @@ def main():
     rendered.sort()
 
     # render file
-    render_template(filepath, CUDA=cuda, ROCm=rocm, rendered=rendered)
+    render_template(filepath, platform=platform, type=type, rendered=rendered)
 
     # add output file to rendered list
-    rendered.append(filepath[:-7])
+    rendered.append(filepath[:-7].replace("\\", "/"))
 
     # print status
     print(f"File '{filepath}' rendered successfully")

diff --git a/.ci/empty.sh b/.ci/empty.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+set -eu
+
+rm -fr "airootfs/home"
+rm -fr "airootfs/root/customize_airootfs"
diff --git a/.ci/template/gh-build-iso.yml.jinja2 b/.ci/template/gh-build-iso.yml.jinja2
@@ -1,4 +1,4 @@
-name: Build ISO on hosted runner ({{ "CUDA" if type == "cuda" else "ROCm" }})
+name: Build ISO on hosted runner ({{ "CUDA" if platform == "cuda" else "ROCm" }}, {{ type }})
 
 on:
   push:
@@ -14,23 +14,27 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+{% if type != "empty" or platform == "rocm" %}
       - name: Cleanup
         uses: rokibhasansagar/slimhub_actions@main
         with:
           retain: "docker_buildkit,docker_imgcache"
+{% endif %}
 
       - name: Checkout repository
         uses: actions/checkout@v4
+{% if type != "empty" %}
         with:
           submodules: recursive
+{% endif %}
 
       - name: Build image
         uses: addnab/docker-run-action@v3
         with:
           image: archlinux:latest
-{%- raw %}
+{% raw %}
           options: --privileged --volume ${{ github.workspace }}:/workspace
-{%- endraw %}
+{% endraw %}
           run: |
             # Exit on error
             set -eu
@@ -43,8 +47,13 @@ jobs:
               # Patch mkarchiso
               .ci/mkarchiso.sh
 
-              # Configure to use {{ "CUDA" if type == "cuda" else "ROCm" }}
-              .ci/configure.py {{ type }}
+              {% if type == "empty" %}
+              # Remove repositories
+              .ci/empty.sh
+              {% endif %}
+
+              # Configure to use {{ "CUDA" if platform == "cuda" else "ROCm" }}
+              .ci/configure.py {{ platform }} {{ type }}
             popd
 
             # Build image

diff --git a/.ci/template/sh-build-iso.yml.jinja2 b/.ci/template/sh-build-iso.yml.jinja2
@@ -1,4 +1,4 @@
-name: Build ISO on selfhosted runner ({{ "CUDA" if type == "cuda" else "ROCm" }})
+name: Build ISO on selfhosted runner ({{ "CUDA" if type == "cuda" else "ROCm" }}, {{ type }})
 
 on:
   push:
@@ -19,6 +19,7 @@ jobs:
         run: |
           # Create ssh directory
           mkdir ~/.ssh
+
 {% raw %}
           # Save ssh config
           echo "${{ secrets.SSH_CONFIG }}" | base64 -d > ~/.ssh/config
@@ -29,6 +30,7 @@ jobs:
           # Save ssh private key
           echo "${{ secrets.SSH_PRIVATE_KEY }}" | base64 -d > /tmp/private.key
 {% endraw %}
+
           # Fix permissions for private key
           chmod 600 /tmp/private.key
 
@@ -47,16 +49,18 @@ jobs:
 
       - name: Checkout repository
         uses: actions/checkout@v4
+{% if type != "empty" %}
         with:
           submodules: recursive
+{% endif %}
 
       - name: Build image
         uses: addnab/docker-run-action@v3
         with:
           image: archlinux:latest
-{%- raw %}
+{% raw %}
           options: --privileged --rm --volume ${{ github.workspace }}:/workspace
-{%- endraw %}
+{% endraw %}
           run: |
             # Exit on error
             set -eu
@@ -69,8 +73,13 @@ jobs:
               # Patch mkarchiso
               .ci/mkarchiso.sh
 
-              # Configure to use {{ "CUDA" if type == "cuda" else "ROCm" }}
-              .ci/configure.py {{ type }}
+              {% if type == "empty" %}
+              # Remove repositories
+              .ci/empty.sh
+              {% endif %}
+
+              # Configure to use {{ "CUDA" if platform == "cuda" else "ROCm" }}
+              .ci/configure.py {{ platform }} {{ type }}
             popd
 
             # Build image
@@ -100,6 +109,7 @@ jobs:
         run: |
           # Create ssh directory
           mkdir ~/.ssh
+
 {% raw %}
           # Save ssh config
           echo "${{ secrets.SSH_CONFIG }}" | base64 -d > ~/.ssh/config
@@ -110,6 +120,7 @@ jobs:
           # Save ssh private key
           echo "${{ secrets.SSH_PRIVATE_KEY }}" | base64 -d > /tmp/private.key
 {% endraw %}
+
           # Fix permissions for private key
           chmod 600 /tmp/private.key
 

diff --git a/.github/workflows/gh-build-iso-cuda-empty.yml b/.github/workflows/gh-build-iso-cuda-empty.yml
@@ -0,0 +1,61 @@
+name: Build ISO on hosted runner (CUDA, empty)
+
+on:
+  push:
+    branches:
+    - main
+
+  pull_request:
+    branches:
+    - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Build image
+        uses: addnab/docker-run-action@v3
+        with:
+          image: archlinux:latest
+
+          options: --privileged --volume ${{ github.workspace }}:/workspace
+          run: |
+            # Exit on error
+            set -eu
+
+            # Enter project directory
+            pushd /workspace
+              # Install dependencies
+              .ci/dependencies.sh
+
+              # Patch mkarchiso
+              .ci/mkarchiso.sh
+
+              # Remove repositories
+              .ci/empty.sh
+
+              # Configure to use CUDA
+              .ci/configure.py cuda empty
+            popd
+
+            # Build image
+            mkarchiso -v -m iso -w /_work -o /workspace/out /workspace
+
+      - name: Create summary
+        run: |
+          # Exit on error
+          set -eu
+
+          # Print checksums to summary
+          sha256sum out/* > "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: archiso-output
+          path: out/
diff --git a/.github/workflows/gh-build-iso-cuda.yml b/.github/workflows/gh-build-iso-cuda.yml
@@ -1,4 +1,4 @@
-name: Build ISO on hosted runner (CUDA)
+name: Build ISO on hosted runner (CUDA, normal)
 
 on:
   push:
@@ -28,6 +28,7 @@ jobs:
         uses: addnab/docker-run-action@v3
         with:
           image: archlinux:latest
+
           options: --privileged --volume ${{ github.workspace }}:/workspace
           run: |
             # Exit on error
@@ -41,8 +42,9 @@ jobs:
               # Patch mkarchiso
               .ci/mkarchiso.sh
 
+
               # Configure to use CUDA
-              .ci/configure.py cuda
+              .ci/configure.py cuda normal
             popd
 
             # Build image

diff --git a/.github/workflows/gh-build-iso-rocm-empty.yml b/.github/workflows/gh-build-iso-rocm-empty.yml
@@ -0,0 +1,65 @@
+name: Build ISO on hosted runner (ROCm, empty)
+
+on:
+  push:
+    branches:
+    - main
+
+  pull_request:
+    branches:
+    - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Cleanup
+        uses: rokibhasansagar/slimhub_actions@main
+        with:
+          retain: "docker_buildkit,docker_imgcache"
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Build image
+        uses: addnab/docker-run-action@v3
+        with:
+          image: archlinux:latest
+
+          options: --privileged --volume ${{ github.workspace }}:/workspace
+          run: |
+            # Exit on error
+            set -eu
+
+            # Enter project directory
+            pushd /workspace
+              # Install dependencies
+              .ci/dependencies.sh
+
+              # Patch mkarchiso
+              .ci/mkarchiso.sh
+
+              # Remove repositories
+              .ci/empty.sh
+
+              # Configure to use ROCm
+              .ci/configure.py rocm empty
+            popd
+
+            # Build image
+            mkarchiso -v -m iso -w /_work -o /workspace/out /workspace
+
+      - name: Create summary
+        run: |
+          # Exit on error
+          set -eu
+
+          # Print checksums to summary
+          sha256sum out/* > "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: archiso-output
+          path: out/