Initial release

LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2017-2019 Blade SAS
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

README.adoc

@@ -0,0 +1,123 @@
+= gitlab-ci helper scripts for OpenEmbedded builds
+
+This project contains gitlab-ci config snippets and associated script
+to help in building an OpenEmbedded-based project.
+
+This includes:
+
+- support for building for several platforms
+- support for building images or SDK
+- publication of package feeds
+- publication of shared-state for reuse by developpers
+- offload of build artifacts through rsync servers, as gitlab does not
+  support large-enough artifacts
+- support for platform-specific branches using a branch prefix
+  (eg. `board1/master`)
+- persistence of downloads and shared-state directories
+- different downloads directories between major poky versions
+- different shared-state directories when you decide it (typically on
+  toolchain change)
+- copy-on-write shared-state for use in dev branches to limit cache
+  polution and inflation (eg. `BUILD_GEN: 1.2+mytest`)
+- (non-optional) usage of buildhistory repo and PR server
+- buildhistory stored in seprate branches for each platform and each
+  source branch
+- archiving of interesting task logs as build artifacts (filtering out
+  setscene and rm_work)
+
+== assumptions on project setup
+
+- hosted by gitlab 12.0 or higher (for multiple `extends`)
+- this repository is included in your project repo as `/ci` (eg. using
+  git submodule or subtree)
+- yocto project's Poky repository is available in your project repo as
+  `/poky`
+- all other layers are available after checkout
+- `$BUILDHISTREPO` must be writable from your gitlab-ci user
+- you have set up a PR server, accessible from your runner (which
+  assumes both are located inside of a trusted network)
+
+This has been only used with a submodule-based setup, other setups may
+work too (eg. using git subtrees or Android's `repo`), examples and
+patches welcomed.
+
+== current limitations
+
+- assumes a single shell runner containing all tools required to build
+  your project
+- the shared-state and download dirs are kept in the runner
+- to have the CI build for a given target plaform from a given branch
+  of your source repository where it was never CI-built before, you
+  must first push the `$BRANCHPREFIX_ARCH/$BASEBRANCHNAME` first
+- lots of `siginfo` files in the shared-state get just their timestamp
+  modified, resulting in much more rsync work than really needed
+
+planned evolutions:
+
+- support docker-based runners
+
+== example use
+
+  include:
+    - local: "/ci/oe.yml"
+
+  variables:
+    GIT_SUBMODULE_STRATEGY: recursive
+    BUILD_GEN: "1.2"
+    POKY_BASE: "warrior"
+    B: "build"
+    BUILDHISTREPO: "ssh://[email protected]/oe/buildhistory.git"
+    PRSERV: "prserv.company.net:8585"
+
+  .template_variables_board1: &variables_board1
+    YOCTO_ARCH: "x86_64"
+    UPDATE_ARCH: "board1"
+    BRANCHPREFIX_ARCH: "board1"
+
+  .board1_exceptions:
+    except:
+      - /^board2\/.*$/
+
+  cache:
+    paths:
+    - $B/cache
+
+  stages:
+    - build
+
+  .my_oe_setup:
+    extends: .oe_setup
+    tags:
+      - oe
+    when: manual
+    only:
+      - branches
+
+  board1-image:
+    variables:
+      <<: *variables_board1
+    extends:
+      - .my_oe_setup
+      - .oe_deploy
+      - .oe_buildlogs_artifacts
+      - .board1_exceptions
+    allow_failure: false
+    stage: build
+    script:
+      - ./ci/run-bitbake --dir "${B}" -- -k core-image-x11 my-packagegroup
+  # save/deploy artifacts
+      - ./ci-my-images-deploy ...
+      - git -C "$B/buildhistory" push origin HEAD
+
+  board1-sdk:
+    variables:
+      <<: *variables_board1
+      IMGROOT: core-image-x11
+    extends:
+      - .my_oe_setup
+      - .oe_buildlogs_artifacts
+      - .board1_exceptions
+    stage: afterbuild
+    script:
+      - ./ci/run-bitbake --dir ${B} -- ${IMGROOT} -c populate_sdk
+      - tar -C $B/tmp/deploy/sdk/ -cvf - . --xform=s,^.,${YOCTO_ARCH}/${BASEBRANCHNAME}, | ssh -p ${SSH_SDK_PORT} ${SSH_SDK_SRV} tar -C ${SSH_SDK_DIR} -xf -

feed-publish

@@ -0,0 +1,59 @@
+#!/bin/sh
+set -e
+
+B=""
+ARCH=""
+DST=""
+PORT=""
+FEED=""
+PKGTYPE="ipk"
+while [ $# -gt 1 ]; do
+    case "$1" in
+        --dir)
+            B="$2"
+            shift
+            ;;
+        --arch)
+            ARCH="$2"
+            shift
+            ;;
+        --rsync-to)
+            DST="$2"
+            shift
+            ;;
+        --ssh-port)
+            PORT="$2"
+            shift
+            ;;
+        --feed)
+            FEED="$2"
+            shift
+            ;;
+        --pkg-type)
+            PKGTYPE="$2"
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+        -*)
+            die "unknown flag '$1'"
+            ;;
+        *)
+            break
+            ;;
+    esac
+    shift
+done
+
+[ -d "$B" ] || die "directory '$B' does not exist"
+[ -n "$ARCH" ] || die "missing --arch"
+[ -n "$DST" ] || die "missing --rsync-to"
+[ -n "$FEED" ] || die "missing --feed"
+
+if [ -n "$PORT" ]; then
+    export RSYNC_RSH="ssh -p '$PORT'"
+fi
+
+rsync -av --delete-after "${B}/tmp/deploy/${PKGTYPE}/" "${DST}/${ARCH}/${FEED}"

mkconf

@@ -0,0 +1,117 @@
+#!/bin/bash
+set -e
+
+die() {
+    echo >&2 "ERROR: $*"
+    exit 1
+}
+
+B=""
+TEMPLATECONF=""
+GEN=""
+DL_DIR=""
+HISTREPO=""
+HISTREF=""
+PRSERV=""
+while [ $# -gt 1 ]; do
+    case "$1" in
+        --dir)
+            B="$2"
+            shift
+            ;;
+        --template-conf)
+            TEMPLATECONF="$2"
+            shift
+            ;;
+        --sstate-gen)
+            GEN="$2"
+            shift
+            ;;
+        --download-dir)
+            DL_DIR="$2"
+            shift
+            ;;
+        --buildhistory-repo)
+            HISTREPO="$2"
+            shift
+            ;;
+        --buildhistory-ref)
+            HISTREF="$2"
+            shift
+            ;;
+        --prserv)
+            PRSERV="$2"
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+        -*)
+            die "unknown flag '$1'"
+            ;;
+        *)
+            break
+            ;;
+    esac
+    shift
+done
+
+[ -n "$B" ] || die "missing --dir"
+[ -n "$TEMPLATECONF" ] || die "missing --template-conf"
+[ -n "$GEN" ] || die "missing --sstate-gen"
+[ -n "$DL_DIR" ] || die "missing --download-dir"
+[ -n "$HISTREPO" ] || die "missing --buildhistory-repo"
+[ -n "$HISTREF" ] || die "missing --buildhistory-ref"
+[ -n "$PRSERV" ] || die "missing --prserv"
+
+
+SSTATE="${HOME}/sstate-cache-$GEN"
+
+case "$GEN" in
+    *+*)
+        BASEGEN=${GEN%+*}
+        BASESSTATE="${HOME}/sstate-cache-$BASEGEN"
+        if [ ! -e "${SSTATE}" ]; then
+            [ -e "${BASESSTATE}" ] || die "Could not find base sstate '${BASESSTATE}'"
+            cp -al "${BASESSTATE}" "${SSTATE}"
+        fi
+    ;;
+esac
+
+if [ -r "$B"/conf/local.conf ]; then
+    . poky/oe-init-build-env "$B"
+
+else
+    TEMPLATECONF="$TEMPLATECONF" . poky/oe-init-build-env "$B"
+
+    # make sure the branch is current so shadow-image identifies
+    # BRANCHNAME (unless CI_COMMIT_REF_NAME does not match the sha -
+    # maybe for rebuilds of older versions ?  Or should we just "git
+    # branch -f" all the time ?)
+    UPSTREAMSHA=$(git rev-parse "origin/${CI_COMMIT_REF_NAME}")
+    if [ "$UPSTREAMSHA" = "${CI_COMMIT_SHA}" ]; then
+        git branch -f "${CI_COMMIT_REF_NAME}" "origin/${CI_COMMIT_REF_NAME}"
+        git checkout "${CI_COMMIT_REF_NAME}"
+        # assert we did not change sha1
+        test $(git rev-parse HEAD) = ${CI_COMMIT_SHA}
+    fi
+
+    git clone "$HISTREPO" buildhistory
+
+    # adjust conf
+    cat >> "conf/local.conf" <<EOF
+
+DL_DIR = "${DL_DIR}"
+SSTATE_DIR = "${SSTATE}"
+RM_WORK_EXCLUDE = ""
+
+INHERIT += "buildhistory"
+BUILDHISTORY_COMMIT = "1"
+BUILDHISTORY_FEATURES = "image"
+EOF
+
+    sed -i 's/^PRSERV_HOST = .*/PRSERV_HOST = "'"$PRSERV"'"/' "conf/local.conf"
+fi
+
+git -C buildhistory checkout "$HISTREF"

oe.yml

@@ -0,0 +1,31 @@
+# needs project-level variables
+# - RSYNC_REPO_SRV: rsync url to publish package feeds
+# - RSYNC_REPO_PORT (optional): ssh port to join rsync server for package feeds
+# - RSYNC_SSTATE_SRV: rsync url to publish shared-state cache
+# - RSYNC_SSTATE_PORT (optional): ssh port to join rsync server for  shared-state cache
+# needs global variables:
+# - POKY_BASE: poky version nickname for separate download dirs
+# - B: oe build directory
+# - BUILD_GEN: generation for separate shared-states
+# - BUILDHISTREPO: git buildhistory repository
+# - PRSERV: suitable value for PRSERV_HOST in local.conf
+# needs platform variables:
+# - BRANCHPREFIX_ARCH: for conf template directory and buildhistory branch prefix
+# - UPDATE_ARCH: used to separate feeds for different architectures
+
+.oe_setup:
+  before_script:
+    - BASEBRANCHNAME="${CI_COMMIT_REF_NAME#${BRANCHPREFIX_ARCH}/}"
+    - ./ci/mkconf --dir "${B}" --sstate-gen "${BUILD_GEN}" --download-dir "${HOME}/downloads-${POKY_BASE}" --template-conf "$PWD/conf-${BRANCHPREFIX_ARCH}" --buildhistory-repo "$BUILDHISTREPO" --buildhistory-ref "${BRANCHPREFIX_ARCH}/$BASEBRANCHNAME" --prserv "$PRSERV"
+
+.oe_deploy:
+  after_script:
+    - ./ci/feed-publish --dir "${B}" --arch "${UPDATE_ARCH}" --feed "$BASEBRANCHNAME" --rsync-to "${RSYNC_REPO_SRV}" --ssh-port "${RSYNC_REPO_PORT}"
+    - ./ci/sstate-publish --dir "${B}" --sstate-gen "${BUILD_GEN}" --rsync-to "${RSYNC_SSTATE_SRV}" --ssh-port "${RSYNC_SSTATE_PORT}"
+
+.oe_buildlogs_artifacts:
+  artifacts:
+    name: "oe-logs-${UPDATE_ARCH}-${CI_JOB_ID}"
+    when: always
+    paths:
+      - "oelogs/*/*/*"

run

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+
+B=""
+while [ $# -gt 1 ]; do
+    case "$1" in
+        --dir)
+            B="$2"
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+        -*)
+            die "unknown flag '$1'"
+            ;;
+        *)
+            break
+            ;;
+    esac
+    shift
+done
+
+[ -d "$B" ] || die "directory '$B' does not exist"
+
+. poky/oe-init-build-env "$B"
+"$@"