Shared tc dir

taskcluster/docker/REGISTRY

@@ -0,0 +1 @@
+mozilla-mobile

taskcluster/docker/base/Dockerfile

@@ -0,0 +1,64 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+FROM ubuntu:18.04
+
+MAINTAINER Tom Prince "[email protected]"
+
+# Add worker user
+RUN mkdir /builds && \
+    useradd -d /builds/worker -s /bin/bash -m worker && \
+    chown worker:worker /builds/worker && \
+    mkdir /builds/worker/artifacts && \
+    chown worker:worker /builds/worker/artifacts
+
+WORKDIR /builds/worker/
+
+#----------------------------------------------------------------------------------------------------------------------
+#-- Configuration -----------------------------------------------------------------------------------------------------
+#----------------------------------------------------------------------------------------------------------------------
+
+ENV LANG en_US.UTF-8
+
+# Do not use fancy output on taskcluster
+ENV TERM dumb
+
+#----------------------------------------------------------------------------------------------------------------------
+#-- System ------------------------------------------------------------------------------------------------------------
+#----------------------------------------------------------------------------------------------------------------------
+
+RUN apt-get update -qq \
+    # We need to install tzdata before all of the other packages. Otherwise it will show an interactive dialog that
+    # we cannot navigate while building the Docker image.
+    && apt-get install -y tzdata \
+    && apt-get install -y openjdk-8-jdk \
+                          wget \
+                          expect \
+                          git \
+                          curl \
+                          python \
+                          python-pip \
+                          python3 \
+                          locales \
+                          unzip \
+			  mercurial \
+    && apt-get clean
+
+RUN pip install --upgrade pip
+RUN pip install 'taskcluster>=4,<5'
+
+RUN locale-gen en_US.UTF-8
+
+# %include-run-task
+
+ENV SHELL=/bin/bash \
+    HOME=/builds/worker \
+    PATH=/builds/worker/.local/bin:$PATH
+
+
+VOLUME /builds/worker/checkouts
+VOLUME /builds/worker/.cache
+
+# run-task expects to run as root
+USER root

taskcluster/docker/node/Dockerfile

@@ -0,0 +1,38 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# %ARG           NODE_VERSION
+FROM          node:$NODE_VERSION
+MAINTAINER    Aki Sasaki <[email protected]>
+
+RUN echo "deb http://deb.debian.org/debian testing main" >> /etc/apt/sources.list
+RUN apt-get update -qq \
+    && apt-get remove -y python3 \
+    && apt-get install -y zip \
+    && apt-get install -y python3.7 \
+    && apt-get clean
+
+RUN rm /usr/bin/python3 && ln -s /usr/bin/python3.7 /usr/bin/python3
+
+# Add worker user
+RUN mkdir /builds && \
+    useradd -d /builds/worker -s /bin/bash -m worker -g 1000 -o -u 1000 && \
+    groupadd -g 1000 -o worker && \
+    mkdir /builds/worker/artifacts && \
+    chown worker:worker /builds/worker/artifacts
+
+# %include-run-task
+
+COPY build.py /usr/local/bin/build.py
+COPY test.py /usr/local/bin/test.py
+
+ENV SHELL=/bin/bash \
+    HOME=/builds/worker \
+    PATH=/builds/worker/.local/bin:$PATH
+
+VOLUME /builds/worker/checkouts
+VOLUME /builds/worker/.cache
+
+# Set a default command useful for debugging
+CMD ["/bin/bash", "--login"]

taskcluster/docker/node/build.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python
+from __future__ import print_function
+
+import functools
+import glob
+import hashlib
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+
+
+def test_is_subdir(parent_dir, target_dir):
+    p1 = Path(os.path.realpath(parent_dir))
+    p2 = Path(os.path.realpath(target_dir))
+    if p1 not in p2.parents:
+        raise Exception("{} is not under {}!".format(target_dir, parent_dir))
+
+
+def test_var_set(varnames):
+    """Test for `varnames` in `os.environ`"""
+    errors = []
+    for varname in varnames:
+        if varname not in os.environ:
+            errors.append(("error: {} is not set".format(varname)))
+    if errors:
+        print("\n".join(errors))
+        sys.exit(1)
+
+
+def run_command(command, **kwargs):
+    print("Running {} ...".format(command))
+    subprocess.check_call(command, **kwargs)
+
+
+def get_output(command, **kwargs):
+    print("Getting output from {} ...".format(command))
+    return subprocess.check_output(command, **kwargs)
+
+
+def get_package_info():
+    if not os.path.exists("package.json"):
+        raise Exception("Can't find package.json in {}!".format(os.getcwd()))
+    with open("package.json") as fh:
+        contents = json.load(fh)
+    return contents
+
+
+def cd(path):
+    print("Changing directory to {} ...".format(path))
+    os.chdir(path)
+
+
+def mkdir(path):
+    print("mkdir {}".format(path))
+    os.makedirs(path, exist_ok=True)
+
+
+def get_hash(path, hash_alg="sha256"):
+    h = hashlib.new(hash_alg)
+    with open(path, "rb") as fh:
+        for chunk in iter(functools.partial(fh.read, 4096), b''):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def main():
+    test_var_set([
+        "ARTIFACT_PREFIX",
+        "XPI_NAME",
+    ])
+
+    artifact_prefix = os.environ["ARTIFACT_PREFIX"]
+    xpi_name = os.environ["XPI_NAME"]
+    xpi_type = os.environ.get("XPI_TYPE")
+    repo_prefix = os.environ.get("REPO_PREFIX", "xpi")
+    head_repo_env_var = "{}_HEAD_REPOSITORY".format(repo_prefix.upper())
+    test_var_set([head_repo_env_var])
+
+    artifact_dir = "/builds/worker/artifacts"
+    base_src_dir = "/builds/worker/checkouts/src"
+
+    package_info = get_package_info()
+
+    revision = get_output(["git", "rev-parse", "HEAD"])
+
+    build_manifest = {
+        "name": xpi_name,
+        "addon-type": xpi_type,
+        "repo": os.environ[head_repo_env_var],
+        "revision": revision.decode("utf-8").rstrip(),
+        "directory": os.path.relpath(base_src_dir, os.getcwd()),
+        "version": package_info["version"],
+        "artifacts": [],
+    }
+
+    if os.environ.get("XPI_INSTALL_TYPE", "yarn") == "yarn":
+        run_command(["yarn", "install", "--frozen-lockfile"])
+    else:
+        run_command(["npm", "install"])
+
+    run_command(["yarn", "build"])
+
+    if 'XPI_ARTIFACTS' in os.environ:
+        xpi_artifacts = os.environ["XPI_ARTIFACTS"].split(";")
+    else:
+        xpi_artifacts = glob.glob('*.xpi')
+
+    for artifact in xpi_artifacts:
+        target_path = os.path.join(artifact_dir, os.path.basename(artifact))
+        if not os.path.exists(artifact):
+            raise Exception("Missing artifact {}".format(artifact))
+        test_is_subdir(os.getcwd(), artifact)
+        print("Copying {} to {}".format(artifact, target_path))
+        path = os.path.join(artifact_prefix, os.path.basename(target_path))
+        artifact_info = {
+            "path": os.path.join(artifact_prefix, os.path.basename(artifact)),
+            "filesize_bytes": int(os.path.getsize(artifact)),
+            "sha256": str(get_hash(artifact)),
+        }
+        build_manifest["artifacts"].append(artifact_info)
+        shutil.copyfile(artifact, target_path)
+
+    with open(os.path.join(artifact_dir, "manifest.json"), "w") as fh:
+        fh.write(json.dumps(build_manifest, indent=2, sort_keys=True))
+
+
+__name__ == '__main__' and main()

taskcluster/docker/node/test.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+from __future__ import print_function
+
+import functools
+import hashlib
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+
+
+def test_is_subdir(parent_dir, target_dir):
+    p1 = Path(os.path.realpath(parent_dir))
+    p2 = Path(os.path.realpath(target_dir))
+    if p1 not in p2.parents:
+        raise Exception("{} is not under {}!".format(target_dir, parent_dir))
+
+
+def test_var_set(varnames):
+    """Test for `varnames` in `os.environ`"""
+    errors = []
+    for varname in varnames:
+        if varname not in os.environ:
+            errors.append(("error: {} is not set".format(varname)))
+    if errors:
+        print("\n".join(errors))
+        sys.exit(1)
+
+
+def run_command(command, **kwargs):
+    print("Running {} ...".format(command))
+    subprocess.check_call(command, **kwargs)
+
+
+def get_output(command, **kwargs):
+    print("Getting output from {} ...".format(command))
+    return subprocess.check_output(command, **kwargs)
+
+
+def get_package_info():
+    if not os.path.exists("package.json"):
+        raise Exception("Can't find package.json in {}!".format(os.getcwd()))
+    with open("package.json") as fh:
+        contents = json.load(fh)
+    return contents
+
+
+def cd(path):
+    print("Changing directory to {} ...".format(path))
+    os.chdir(path)
+
+
+def get_hash(path, hash_alg="sha256"):
+    h = hashlib.new(hash_alg)
+    with open(path, "rb") as fh:
+        for chunk in iter(functools.partial(fh.read, 4096), b''):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def main():
+    if os.environ.get("XPI_INSTALL_TYPE", "yarn") == "yarn":
+        run_command(["yarn", "install", "--frozen-lockfile"])
+    else:
+        run_command(["npm", "install"])
+
+    commands = []
+    if len(sys.argv) != 1:
+        commands = sys.argv[1:]
+    else:
+        package_info = get_package_info()
+        if "test" in package_info.get("scripts", {}):
+            commands = ["test"]
+        else:
+            print("No `test` target in package.json; noop")
+
+    for command in commands:
+        run_command(["yarn", command])
+
+
+__name__ == '__main__' and main()