diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..2042a32 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,163 @@ +name: Check and Build +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + release: + types: [released] +permissions: read-all +jobs: + format: + name: Check formatting + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + python: ['3.11', '3.12'] + steps: + - name: Checkout the Git repository + uses: actions/checkout@v3 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + cache: 'pip' + - name: Check formatting + run: make format + lint: + name: Check for erroneous constructs + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + python: ['3.11', '3.12'] + steps: + - name: Checkout the Git repository + uses: actions/checkout@v3 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + cache: 'pip' + - name: Check for erroneous constructs + run: make lint + links: + name: Check Markdown links + runs-on: ubuntu-20.04 + steps: + - name: Checkout the Git repository + uses: actions/checkout@v3 + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version: '16' + - name: Check Markdown links + run: | + npm install -g markdown-link-check + make links + test: + name: Run tests + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + python: ['3.11', '3.12'] + steps: + - name: Checkout the Git repository + uses: actions/checkout@v3 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + cache: 'pip' + - name: Run tests + run: make test + build: + name: Make pip packages + runs-on: ubuntu-20.04 + needs: [format, lint, test] + steps: + - name: Checkout the Git repository + uses: actions/checkout@v3 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + cache: 'pip' + - name: Building toltecmk + run: make build + - uses: actions/upload-artifact@v4 + with: + name: pip + path: dist/* + if-no-files-found: error + standalone: + name: Make Standalone + runs-on: ubuntu-20.04 + needs: [format, lint, test] + steps: + - name: Checkout the Git repository + uses: actions/checkout@v3 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + cache: 'pip' + - name: Building toltecmk + run: make standalone + - name: Sanity check + run: ./toltecmk --help + - uses: actions/upload-artifact@v4 + with: + name: toltecmk + path: toltecmk + if-no-files-found: error + publish: + name: Publish to PyPi + runs-on: ubuntu-20.04 + needs: [build, links] + if: github.repository == 'toltec-dev/build' && github.event_name == 'release' && startsWith(github.ref, 'refs/tags') + permissions: + id-token: write + environment: + name: pypi + url: https://pypi.org/p/toltecmk + steps: + - name: Download pip packages + id: download + uses: actions/download-artifact@v4 + with: + name: pip + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: ${{ steps.download.outputs.download-path }} + release: + name: Add ${{ matrix.artifact }} to release + if: github.repository == 'toltec-dev/build' && github.event_name == 'release' && startsWith(github.ref, 'refs/tags') + needs: [standalone, build] + runs-on: ubuntu-latest + strategy: + matrix: + artifact: + - 'pip' + - 'toltecmk' + permissions: + contents: write + steps: + - name: Checkout the Git repository + uses: actions/checkout@v4 + - name: Download executable + id: download + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.artifact }} + path: dist + - name: Upload to release + run: + find . -type f | xargs -rI {} gh release upload "$TAG" {} --clobber + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.event.release.tag_name }} + working-directory: ${{ steps.download.outputs.download-path }} diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml deleted file mode 100644 index 5eaab90..0000000 --- a/.github/workflows/checks.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: checks -on: [push, pull_request, workflow_dispatch] -permissions: read-all -jobs: - format: - name: Check formatting - runs-on: ubuntu-20.04 - steps: - - name: Checkout the Git repository - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - cache: 'pip' - - name: Check formatting - run: make format - lint: - name: Check for erroneous constructs - runs-on: ubuntu-20.04 - steps: - - name: Checkout the Git repository - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - cache: 'pip' - - name: Check for erroneous constructs - run: make lint - links: - name: Check Markdown links - runs-on: ubuntu-20.04 - steps: - - name: Checkout the Git repository - uses: actions/checkout@v3 - - name: Setup Node - uses: actions/setup-node@v3 - with: - node-version: '16' - - name: Check Markdown links - run: | - npm install -g markdown-link-check - make links - test: - name: Run tests - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - python: ['3.11'] - steps: - - name: Checkout the Git repository - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - cache: 'pip' - - name: Run tests - run: make test - standalone: - name: Make Standalone - runs-on: ubuntu-20.04 - needs: [format, lint, test] - steps: - - name: Checkout the Git repository - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - cache: 'pip' - - name: Nuitka cache - uses: actions/cache@v3 - with: - path: nuitka-cache - key: ${{ runner.os }}-nuitka-cache - - name: Building toltecmk - run: NUITKA_CACHE_DIR=nuitka-cache make standalone - - uses: actions/upload-artifact@v3 - with: - name: toltecmk - path: toltecmk diff --git a/Makefile b/Makefile index 135f396..dca0371 100644 --- a/Makefile +++ b/Makefile @@ -33,13 +33,13 @@ export USAGE help: @echo "$$USAGE" -.venv-build/bin/activate: +.venv-build/bin/activate: requirements.build.txt @echo "Setting up development virtual env in .venv" python -m venv .venv-build; \ . .venv-build/bin/activate; \ python -m pip install -r requirements.build.txt -.venv-runtime/bin/activate: +.venv-runtime/bin/activate: requirements.txt @echo "Setting up development virtual env in .venv" python -m venv .venv-runtime; \ . .venv-runtime/bin/activate; \ @@ -54,13 +54,7 @@ build: .venv-build/bin/activate standalone: .venv-build/bin/activate . .venv-build/bin/activate; \ - python -m nuitka \ - --follow-imports --enable-plugin=anti-bloat \ - --enable-plugin=pylint-warnings \ - --onefile --linux-onefile-icon=media/overview.svg \ - --assume-yes-for-downloads \ - -o toltecmk \ - toltec + python -m PyInstaller --distpath . toltecmk.spec test: .venv-runtime/bin/activate . .venv-runtime/bin/activate; \ diff --git a/pyproject.toml b/pyproject.toml index a32f463..eaa8bb7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "toltecmk" -version = "0.2.1" +version = "0.3.0" authors = [ { name="Mattéo Delabre", email="git.matteo@delab.re" }, { name="Eeems", email="eeems@eeems.email" }, @@ -40,9 +40,9 @@ packages = [ ] [tool.setuptools.package-data] -toltec = ["*.py.typed"] -"toltec.recipe_parsers" = ["*.py.typed"] -"toltec.hooks" = ["*.py.typed"] +toltec = ["py.typed", "*.py.typed"] +"toltec.recipe_parsers" = ["py.typed", "*.py.typed"] +"toltec.hooks" = ["py.typed", "*.py.typed"] [tool.setuptools.dynamic] dependencies = {file = ["requirements.txt"]} diff --git a/requirements.build.txt b/requirements.build.txt index ccfbd55..aa68724 100644 --- a/requirements.build.txt +++ b/requirements.build.txt @@ -2,18 +2,17 @@ build==0.6.0.post1 packaging==23.1 pep517==0.13.0 -nuitka==1.7.10 +pyinstaller==6.3.0 black==23.7.0 -astroid==2.15.6 certifi==2023.7.22 charset-normalizer==3.2.0 idna==3.4 isort==5.12.0 lazy-object-proxy==1.9.0 mccabe==0.7.0 -mypy==1.5.1 +mypy==1.7.1 mypy-extensions==1.0.0 -pylint==2.17.5 +pylint==3.0.3 pyparsing==3.1.1 six==1.16.0 toml==0.10.2 diff --git a/requirements.txt b/requirements.txt index 365ffff..1884b4c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ docker==6.1.3 python-dateutil==2.8.2 -pyelftools==0.29 +pyelftools==0.30 diff --git a/tests/fixtures/hello/hello.c b/tests/fixtures/hello/hello.c new file mode 100644 index 0000000..11f148d --- /dev/null +++ b/tests/fixtures/hello/hello.c @@ -0,0 +1,9 @@ +// Copyright (c) 2023 The Toltec Contributors +// SPDX-License-Identifier: MIT + +#include + +int main() { + printf("Hello, World!"); + return 0; +} diff --git a/tests/fixtures/hello/package b/tests/fixtures/hello/package new file mode 100644 index 0000000..ca2b948 --- /dev/null +++ b/tests/fixtures/hello/package @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Copyright (c) 2023 The Toltec Contributors +# SPDX-License-Identifier: MIT + +pkgnames=(hello) +pkgdesc="Hello World!" +timestamp=2023-12-27T00:58Z +maintainer="Eeems " +url="https://github.com/toltec-dev/build" +license=MIT +pkgver=0.0.1-1 +section="utils" +flags=() + +image=base:v2.1 +source=(hello.c) +sha256sums=(SKIP) + +build() { + export CC="${CROSS_COMPILE}gcc" + "$CC" -o hello hello.c +} +package() { + install -D -m 755 "$srcdir"/hello "$pkgdir"/opt/bin/hello +} diff --git a/tests/test_strip.py b/tests/test_strip.py new file mode 100644 index 0000000..5b86895 --- /dev/null +++ b/tests/test_strip.py @@ -0,0 +1,92 @@ +# Copyright (c) 2023 The Toltec Contributors +# SPDX-License-Identifier: MIT + +import unittest +import subprocess +import shutil + +from os import path +from tempfile import TemporaryDirectory +from toltec.hooks.strip import walk_elfs +from elftools.elf.elffile import ELFFile + + +class TestBuild(unittest.TestCase): + def setUp(self) -> None: + self.dir = path.dirname(path.realpath(__file__)) + self.fixtures_dir = path.join(self.dir, "fixtures") + + def test_strip(self) -> None: + with TemporaryDirectory() as tmp_dir: + rec_dir = path.join(self.fixtures_dir, "hello") + work_dir = path.join(tmp_dir, "build") + dist_dir = path.join(tmp_dir, "dist") + + result = subprocess.run( + [ + "python3", + "-m", + "toltec", + "--work-dir", + work_dir, + "--dist-dir", + dist_dir, + "--", + rec_dir, + ], + capture_output=True, + check=False, + ) + self.assertEqual( + result.returncode, 0, result.stderr.decode("utf-8") + ) + self.assertEqual(result.stdout.decode("utf-8"), "") + walk_elfs( + work_dir, + lambda i, p: self.assertIsNone( + i.get_section_by_name(".symtab"), f"{p} is not stripped" + ), + ) + + with TemporaryDirectory() as tmp_dir: + src_dir = path.join(self.fixtures_dir, "hello") + rec_dir = path.join(tmp_dir, "src") + work_dir = path.join(tmp_dir, "build") + dist_dir = path.join(tmp_dir, "dist") + shutil.copytree(src_dir, rec_dir) + replacements = {"flags=()": "flags=(nostrip)"} + with open( + path.join(src_dir, "package"), "rt", encoding="utf-8" + ) as infile, open( + path.join(rec_dir, "package"), "wt", encoding="utf-8" + ) as outfile: + for line in infile: + for src, target in replacements.items(): + line = line.replace(src, target) + outfile.write(line) + + result = subprocess.run( + [ + "python3", + "-m", + "toltec", + "--work-dir", + work_dir, + "--dist-dir", + dist_dir, + "--", + rec_dir, + ], + capture_output=True, + check=False, + ) + self.assertEqual( + result.returncode, 0, result.stderr.decode("utf-8") + ) + self.assertEqual(result.stdout.decode("utf-8"), "") + walk_elfs( + work_dir, + lambda i, p: self.assertIsNotNone( + i.get_section_by_name(".symtab"), f"{p} is stripped" + ), + ) diff --git a/tests/test_toltec.py b/tests/test_toltec.py index 7e6f5b2..d43bbb5 100644 --- a/tests/test_toltec.py +++ b/tests/test_toltec.py @@ -39,9 +39,11 @@ def test_build_rmkit(self) -> None: rec_dir, ], capture_output=True, + check=False, + ) + self.assertEqual( + result.returncode, 0, result.stderr.decode("utf-8") ) - - self.assertEqual(result.returncode, 0) self.assertEqual(result.stdout.decode("utf-8"), "") self.assertEqual( result.stderr.decode("utf-8"), diff --git a/toltec/__main__.py b/toltec/__main__.py index 1e9b7e6..26f6968 100644 --- a/toltec/__main__.py +++ b/toltec/__main__.py @@ -14,7 +14,7 @@ from toltec import util -def main() -> int: +def main() -> int: # pylint:disable=too-many-branches """Execute requested commands and return appropriate exit code.""" parser = argparse.ArgumentParser(description=__doc__) diff --git a/toltec/bash.py b/toltec/bash.py index 1f6f094..b81d3aa 100644 --- a/toltec/bash.py +++ b/toltec/bash.py @@ -26,9 +26,8 @@ class ScriptError(Exception): # from the result of `get_declarations()`. Subset of the list at: # default_variables = { + "_", "BASH", - "BASHOPTS", - "BASHPID", "BASH_ALIASES", "BASH_ARGC", "BASH_ARGV", @@ -36,10 +35,13 @@ class ScriptError(Exception): "BASH_CMDS", "BASH_COMMAND", "BASH_LINENO", + "BASH_LOADABLES_PATH", "BASH_SOURCE", "BASH_SUBSHELL", "BASH_VERSINFO", "BASH_VERSION", + "BASHOPTS", + "BASHPID", "COLUMNS", "COMP_WORDBREAKS", "DIRSTACK", @@ -78,10 +80,38 @@ class ScriptError(Exception): "SRANDOM", "TERM", "UID", - "_", } +def _get_bash_stdout(src: str) -> str: + """ + Get the stdout from a bash script + + :param src: bash script to run + :returns: the stdout of the script + """ + env: Dict[str, str] = { + "PATH": os.environ["PATH"], + } + + subshell = subprocess.run( # pylint:disable=subprocess-run-check + ["/usr/bin/env", "bash"], + input=src.encode(), + capture_output=True, + env=env, + ) + + errors = subshell.stderr.decode() + + if subshell.returncode == 2 or "syntax error" in errors: + raise ScriptError(f"Bash syntax error\n{errors}") + + if subshell.returncode != 0 or errors: + raise ScriptError(f"Bash error\n{errors}") + + return subshell.stdout.decode() + + def get_declarations(src: str) -> Tuple[Variables, Functions]: """ Extract all variables and functions defined by a Bash script. @@ -97,28 +127,7 @@ def get_declarations(src: str) -> Tuple[Variables, Functions]: declare -f declare -p """ - env: Dict[str, str] = { - "PATH": os.environ["PATH"], - } - - declarations_subshell = ( - subprocess.run( # pylint:disable=subprocess-run-check - ["/usr/bin/env", "bash"], - input=src.encode(), - capture_output=True, - env=env, - ) - ) - - errors = declarations_subshell.stderr.decode() - - if declarations_subshell.returncode == 2 or "syntax error" in errors: - raise ScriptError(f"Bash syntax error\n{errors}") - - if declarations_subshell.returncode != 0 or errors: - raise ScriptError(f"Bash error\n{errors}") - - declarations = declarations_subshell.stdout.decode() + declarations = _get_bash_stdout(src) # Parse `declare` statements and function statements lexer = shlex.shlex(declarations, posix=True) @@ -319,7 +328,10 @@ def _parse_var(lexer: shlex.shlex) -> Tuple[str, Optional[Any]]: else: string_token = lexer.get_token() or "" if string_token == "$": - string_token = lexer.get_token() or "" + quoted_string = lexer.get_token() or "" + string_token = _get_bash_stdout( + "echo -n $" + shlex.quote(quoted_string) + ) var_value = _parse_string(string_token) else: lexer.push_token(lookahead) diff --git a/toltec/builder.py b/toltec/builder.py index 2799df2..bfd53ea 100644 --- a/toltec/builder.py +++ b/toltec/builder.py @@ -9,8 +9,10 @@ import os import logging import textwrap +from importlib.util import find_spec, module_from_spec import docker import requests +from . import hooks from . import bash, util, ipk from .recipe import RecipeBundle, Recipe, Package from .version import DependencyKind @@ -50,6 +52,17 @@ def __init__(self, work_dir: str, dist_dir: str) -> None: permissions." ) from err + for hook in hooks.__all__: + spec = find_spec(f"toltec.hooks.{hook}") + if spec: + module = module_from_spec(spec) + spec.loader.exec_module(module) # type: ignore + module.register(self) # type: ignore + else: + raise RuntimeError( + f"Hook module 'toltec.hooks.{hook}' couldn’t be loaded" + ) + def __enter__(self) -> "Builder": return self @@ -123,6 +136,7 @@ def make( self, recipe_bundle: RecipeBundle, build_matrix: Optional[Mapping[str, Optional[List[Package]]]] = None, + check_directory: bool = True, ) -> bool: """ Build packages defined by a recipe. @@ -132,7 +146,7 @@ def make( (default: all supported packages for each architecture) :returns: true if all the requested packages were built correctly """ - if not util.check_directory( + if check_directory and not util.check_directory( self.work_dir, f"The build directory '{self.work_dir}' \ already exists.\nWould you like to [c]ancel, [r]emove that directory, \ diff --git a/toltec/hooks/__init__.py b/toltec/hooks/__init__.py new file mode 100644 index 0000000..51d3150 --- /dev/null +++ b/toltec/hooks/__init__.py @@ -0,0 +1,10 @@ +""" +Built in hooks +""" +__all__ = [ + "patch_rm2fb", + "strip", + "reload_oxide_apps", # Depends on install_lib + # This hook needs to come after any hooks that may depend on it + "install_lib", +] diff --git a/toltec/hooks/install_lib.py b/toltec/hooks/install_lib.py new file mode 100644 index 0000000..695d167 --- /dev/null +++ b/toltec/hooks/install_lib.py @@ -0,0 +1,203 @@ +""" +Build hook to automatically add install-lib helper functions + +After the artifacts are packaged, this hook will look for known install-lib +methods and add them to scripts if found. +""" +import logging + +from typing import Set, Iterable +from toltec.builder import Builder +from toltec.recipe import Package +from toltec.util import listener + +logger = logging.getLogger(__name__) + +METHODS = {} + + +def add_method(name: str, src: str, *depends: Iterable[str]) -> None: + """Add a method to be automatically added to scripts that use it""" + METHODS[name] = ( + src, + depends, + ) + + +def register(builder: Builder) -> None: + """Register the hook""" + + @listener(builder.post_package) + def post_package( + builder: Builder, # pylint: disable=unused-argument + package: Package, + src_dir: str, # pylint: disable=unused-argument + pkg_dir: str, # pylint: disable=unused-argument + ) -> None: + for name in ( + "configure", + "postremove", + "postupgrade", + "preinstall", + "preremove", + "preupgrade", + ): + function = getattr(package, name) + methods: Set[str] = set() + for method in METHODS: # pylint: disable=consider-using-dict-items + if method in function: + methods.add(method) + src, depends = METHODS[method] + methods.update(depends) # type: ignore + + for method in methods: + src, depends = METHODS[method] + function = f""" +{method}() {{ + {src} +}} +{function} +""" + + setattr(package, name, function) + + add_method("is-enabled", 'systemctl --quiet is-enabled "$1" 2> /dev/null') + add_method( + "is-masked", + '[[ "$(systemctl is-enabled "$1" 2> /dev/null)" == "masked" ]]', + ) + add_method("is-active", 'systemctl --quiet is-active "$1" 2> /dev/null') + add_method( + "get-conflicts", + """ + # Find enabled units that have a conflicting name + for name in $(systemctl cat "$1" | awk -F'=' '/^Alias=/{print $2}'); do + local realname + if realname="$(basename "$(readlink "/etc/systemd/system/$name")")"; then + echo "$realname" + fi + done + + # Find units that are declared as conflicting + # (systemd automatically adds a conflict with "shutdown.target" to all + # service units see systemd.service(5), section "Automatic Dependencies") + systemctl show "$1" | awk -F'=' '/^Conflicts=/{print $2}' \ + | sed 's|\bshutdown.target\b||' + """, + ) + add_method( + "how-to-enable", + """ + for conflict in $(get-conflicts "$1"); do + if is-enabled "$conflict"; then + echo "$ systemctl disable --now ${conflict/.service/}" + fi + done + + echo "$ systemctl enable --now ${1/.service/}" + """, + "is-enabled", + "get-conflicts", + ) + add_method( + "reload-oxide-apps", + """ + if ! is-active tarnish.service; then + return + fi + echo -n "Reloading Oxide applications: " + local ret + if type update-desktop-database &> /dev/null; then + update-desktop-database --quiet + ret=$? + else + /opt/bin/rot apps call reload 2> /dev/null + ret=$? + fi + if [ $ret -eq 0 ]; then + echo "Done!" + else + echo "Failed!" + fi + """, + "is-active", + ) + add_method( + "add-bind-mount", + """ + local unit_name + local unit_path + unit_name="$(systemd-escape --path "$2").mount" + unit_path="/lib/systemd/system/$unit_name" + + if [[ -e $unit_path ]]; then + echo "Bind mount configuration for '$2' already exists, updating" + else + echo "Mounting '$1' over '$2'" + fi + + cat > "$unit_path" << UNIT + [Unit] + Description=Bind mount $1 over $2 + DefaultDependencies=no + Conflicts=umount.target + Before=local-fs.target umount.target + + [Mount] + What=$1 + Where=$2 + Type=none + Options=bind + + [Install] + WantedBy=local-fs.target + UNIT + + systemctl daemon-reload + systemctl enable "$unit_name" + systemctl restart "$unit_name" + """, + ) + add_method( + "remove-bind-mount", + """ + local unit_name + local unit_path + unit_name="$(systemd-escape --path "$1").mount" + unit_path="/lib/systemd/system/$unit_name" + + if [[ ! -e $unit_path ]]; then + echo "No existing bind mount for '$1'" + return 1 + fi + + echo "Removing mount over '$1'" + systemctl disable "$unit_name" + systemctl stop "$unit_name" + rm "$unit_path" + systemctl daemon-reload + """, + ) + add_method( + "unit-exists", + '[ "$(systemctl --quiet list-unit-files "${1}" | grep -c "${1}")" -eq 1 ]', + ) + add_method( + "disable-unit", + """ + if ! unit-exists "${1}"; then + return + fi + if is-active "$1"; then + echo "Stopping ${1}" + systemctl stop "${1}" + fi + if is-enabled "${1}"; then + echo "Disabling ${1}" + systemctl disable "${1}" + fi + """, + "unit-exists", + "is-active", + "is-enabled", + ) diff --git a/toltec/hooks/patch_rm2fb.py b/toltec/hooks/patch_rm2fb.py new file mode 100644 index 0000000..accc135 --- /dev/null +++ b/toltec/hooks/patch_rm2fb.py @@ -0,0 +1,81 @@ +""" +Build hook for patching all binary objects that access /dev/fb0 to depend on +librm2fb_client.so.1 after building a recipe. + +After the build() script is run, and before the artifacts are packaged, this +hook looks for ARM ELF-files in the build directory that access /dev/fb0. +It then uses patchelf to add a dependency on librm2fb_client.so.1 to the +binaries. This behavior is only enabled if the recipe declares the +'patch_rm2fb' flag. +""" +import os +import logging +import shlex +from elftools.elf.elffile import ELFFile +from toltec.builder import Builder +from toltec.recipe import Recipe +from toltec.util import listener +from toltec.hooks.strip import walk_elfs, run_in_container, MOUNT_SRC + +logger = logging.getLogger(__name__) + + +def register(builder: Builder) -> None: + """Register the hook""" + + @listener(builder.post_build) + def post_build( # pylint: disable=too-many-locals + builder: Builder, recipe: Recipe, src_dir: str + ) -> None: + if "patch_rm2fb" not in recipe.flags: + return + + logger.debug("Adding dependency to rm2fb ('patch_rm2fb' flag is set)") + # Search for binary objects that can be stripped + binaries = [] + + def filter_elfs(info: ELFFile, file_path: str) -> None: + symtab = info.get_section_by_name(".symtab") + if symtab is None or info.get_machine_arch() != "ARM": + return + + dynamic = info.get_section_by_name(".dynamic") + rodata = info.get_section_by_name(".rodata") + if dynamic and rodata and rodata.data().find(b"/dev/fb0") != -1: + binaries.append(file_path) + + walk_elfs(src_dir, filter_elfs) + + if not binaries: + logger.debug("Skipping, no arm binaries found") + return + + # Save original mtimes to restore them afterwards + # This will prevent any Makefile rules to be triggered again + # in packaging scripts that use `make install` + original_mtime = {} + + script = [ + "export DEBIAN_FRONTEND=noninteractive", + "apt-get update -qq", + "apt-get install -qq --no-install-recommends patchelf", + ] + + def docker_file_path(file_path: str) -> str: + return shlex.quote( + os.path.join(MOUNT_SRC, os.path.relpath(file_path, src_dir)) + ) + + for file_path in binaries: + original_mtime[file_path] = os.stat(file_path).st_mtime_ns + + script.append( + "patchelf --add-needed librm2fb_client.so.1 " + + " ".join(docker_file_path(file_path) for file_path in binaries) + ) + + run_in_container(builder, src_dir, logger, script) + + # Restore original mtimes + for file_path, mtime in original_mtime.items(): + os.utime(file_path, ns=(mtime, mtime)) diff --git a/toltec/hooks/reload_oxide_apps.py b/toltec/hooks/reload_oxide_apps.py new file mode 100644 index 0000000..2edd617 --- /dev/null +++ b/toltec/hooks/reload_oxide_apps.py @@ -0,0 +1,35 @@ +""" +Build hook for automatically reloading applications in oxide + +After the artifacts are packaged, this hook looks for files in either +/opt/etc/draft or /opt/usr/share/applications and adds reload-oxide-apps to +configure, postupgrade, and postremove +""" +import os +import logging + +from toltec.builder import Builder +from toltec.recipe import Package +from toltec.util import listener + +logger = logging.getLogger(__name__) + +OXIDE_HOOK = "\nreload-oxide-apps\n" + + +def register(builder: Builder) -> None: + """Register the hook""" + + @listener(builder.post_package) + def post_package( + builder: Builder, # pylint: disable=unused-argument + package: Package, + src_dir: str, # pylint: disable=unused-argument + pkg_dir: str, + ) -> None: + if os.path.exists( + os.path.join(pkg_dir, "opt/usr/share/applications") + ) or os.path.exists(os.path.join(pkg_dir, "opt/etc/draft")): + package.configure += OXIDE_HOOK + package.postupgrade += OXIDE_HOOK + package.postremove += OXIDE_HOOK diff --git a/toltec/hooks/strip.py b/toltec/hooks/strip.py index b2625ae..7cb0e0d 100644 --- a/toltec/hooks/strip.py +++ b/toltec/hooks/strip.py @@ -8,6 +8,7 @@ import os import logging import shlex +from typing import Callable, List import docker from elftools.elf.elffile import ELFFile, ELFError from toltec import bash @@ -17,38 +18,76 @@ logger = logging.getLogger(__name__) +MOUNT_SRC = "/src" +TOOLCHAIN = "toolchain:v1.3.1" + + +def walk_elfs(src_dir: str, for_each: Callable) -> None: + """Walk through all the ELF binaries in a directory and run a method for each of them""" + for directory, _, files in os.walk(src_dir): + for file_name in files: + file_path = os.path.join(directory, file_name) + + try: + with open(file_path, "rb") as file: + for_each(ELFFile(file), file_path) + except ELFError: + # Ignore non-ELF files + pass + except IsADirectoryError: + # Ignore directories + pass + + +def run_in_container( + builder: Builder, src_dir: str, _logger: logging.Logger, script: List[str] +) -> None: + """Run a script in a container and log output""" + logs = bash.run_script_in_container( + builder.docker, + image=builder.IMAGE_PREFIX + TOOLCHAIN, + mounts=[ + docker.types.Mount( + type="bind", + source=os.path.abspath(src_dir), + target=MOUNT_SRC, + ) + ], + variables={}, + script="\n".join(script), + ) + bash.pipe_logs(_logger, logs) + def register(builder: Builder) -> None: + """Register the hook""" + @listener(builder.post_build) - def post_build(builder: Builder, recipe: Recipe, src_dir: str) -> None: + def post_build( # pylint: disable=too-many-locals,too-many-branches + builder: Builder, recipe: Recipe, src_dir: str + ) -> None: if "nostrip" in recipe.flags: logger.debug("Skipping strip ('nostrip' flag is set)") return # Search for binary objects that can be stripped - strip_arm = [] - strip_x86 = [] - - for directory, _, files in os.walk(src_dir): - for file_name in files: - file_path = os.path.join(directory, file_name) - - try: - with open(file_path, "rb") as file: - info = ELFFile(file) - symtab = info.get_section_by_name(".symtab") - - if symtab: - if info.get_machine_arch() == "ARM": - strip_arm.append(file_path) - elif info.get_machine_arch() in ("x86", "x64"): - strip_x86.append(file_path) - except ELFError: - # Ignore non-ELF files - pass - except IsADirectoryError: - # Ignore directories - pass + strip_arm: List[str] = [] + strip_x86: List[str] = [] + + def filter_elfs(info: ELFFile, file_path: str) -> None: + symtab = info.get_section_by_name(".symtab") + if not symtab: + return + if info.get_machine_arch() == "ARM": + strip_arm.append(file_path) + elif info.get_machine_arch() in ("x86", "x64"): + strip_x86.append(file_path) + + walk_elfs(src_dir, filter_elfs) + + if not strip_arm and not strip_x86: + logger.debug("Skipping, no binaries found") + return # Save original mtimes to restore them afterwards # This will prevent any Makefile rules to be triggered again @@ -60,11 +99,11 @@ def post_build(builder: Builder, recipe: Recipe, src_dir: str) -> None: # Run strip on found binaries script = [] - mount_src = "/src" - docker_file_path = lambda file_path: shlex.quote( - os.path.join(mount_src, os.path.relpath(file_path, src_dir)) - ) + def docker_file_path(file_path: str) -> str: + return shlex.quote( + os.path.join(MOUNT_SRC, os.path.relpath(file_path, src_dir)) + ) # Strip debugging symbols and unneeded sections if strip_x86: @@ -99,20 +138,7 @@ def post_build(builder: Builder, recipe: Recipe, src_dir: str) -> None: os.path.relpath(file_path, src_dir), ) - logs = bash.run_script_in_container( - builder.docker, - image=builder.IMAGE_PREFIX + "toolchain:v2.1", - mounts=[ - docker.types.Mount( - type="bind", - source=os.path.abspath(src_dir), - target=mount_src, - ) - ], - variables={}, - script="\n".join(script), - ) - bash.pipe_logs(logger, logs) + run_in_container(builder, src_dir, logger, script) # Restore original mtimes for file_path, mtime in original_mtime.items(): diff --git a/toltec/recipe_parsers/bash.py b/toltec/recipe_parsers/bash.py index 0e775b8..8499d28 100644 --- a/toltec/recipe_parsers/bash.py +++ b/toltec/recipe_parsers/bash.py @@ -407,8 +407,7 @@ def _pop_field_string( if not isinstance(value, str): raise RecipeError( path, - f"Field '{name}' must be a string, \ -got a {type(value).__name__}", + f"Field '{name}' must be a string, got a {type(value).__name__}", ) return value @@ -428,10 +427,10 @@ def _pop_field_indexed( value = variables.pop(name) if not isinstance(value, list): + _name = type(value).__name__ raise RecipeError( path, - f"Field '{name}' must be an indexed array, \ -got a {type(value).__name__}", + f"Field '{name}' must be an indexed array, got a {_name}", ) return value diff --git a/toltecmk.spec b/toltecmk.spec new file mode 100644 index 0000000..6ac6c9e --- /dev/null +++ b/toltecmk.spec @@ -0,0 +1,38 @@ +# -*- mode: python ; coding: utf-8 -*- + +import toltec.hooks + +a = Analysis( + ["toltec/__main__.py"], + pathex=[], + binaries=[], + datas=[], + hiddenimports=[f"toltec.hooks.{x}" for x in toltec.hooks.__all__], + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=[], + noarchive=False, +) +pyz = PYZ(a.pure) + +exe = EXE( + pyz, + a.scripts, + a.binaries, + a.datas, + [], + name="toltecmk", + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + upx_exclude=[], + runtime_tmpdir=None, + console=True, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, +)