From 2b03dbf0318ffeb1f9e1391b96181d360bb8482c Mon Sep 17 00:00:00 2001 From: David Xia Date: Sat, 6 Oct 2018 12:34:00 -0400 Subject: [PATCH] WIP: build with JDK11 and Docker on CircleCI * Upgrade docker-maven-plugin from 0.4.13 to 1.2.0 * latest version supports Java 11 * Upgrade ssh-agent-tls from 0.0.3 to 0.0.4 * Upgrade ssh-agent-proxy from 0.1.5 to 0.2.0 --- .circleci/Dockerfile | 25 +++++++++++++++++ .circleci/config.yml | 60 ++++++++++++++++++++++++++++++++++++++++- circle.sh | 22 +++++++++------ helios-services/pom.xml | 4 +-- pom.xml | 6 ++--- 5 files changed, 103 insertions(+), 14 deletions(-) create mode 100644 .circleci/Dockerfile diff --git a/.circleci/Dockerfile b/.circleci/Dockerfile new file mode 100644 index 0000000000..3ba6d627b3 --- /dev/null +++ b/.circleci/Dockerfile @@ -0,0 +1,25 @@ +# Dockerfile for the container CircleCI uses to build and test helios +FROM ubuntu:bionic + +RUN apt-get -qq update && apt-get -qq install \ + # Required tools for primary containers that aren't already in bionic + # https://circleci.com/docs/2.0/custom-images/#required-tools-for-primary-containers + git ssh \ + # tools we need + wget maven lsof jq python-minimal python-pip + +# Download and validate checksum of openjdk-11 +# Checksum from https://download.java.net/java/ga/jdk11/openjdk-11_linux-x64_bin.tar.gz.sha256 +RUN wget --quiet https://download.java.net/java/ga/jdk11/openjdk-11_linux-x64_bin.tar.gz && \ + echo '3784cfc4670f0d4c5482604c7c513beb1a92b005f569df9bf100e8bef6610f2e openjdk-11_linux-x64_bin.tar.gz' > openjdk-11-sha256sum.txt && \ + sha256sum -c openjdk-11-sha256sum.txt + +# Install openjdk-11 +RUN tar -xzf openjdk-11_linux-x64_bin.tar.gz && \ + mkdir -p /usr/lib/jvm && \ + mv jdk-11 /usr/lib/jvm/openjdk-11 && \ + update-alternatives --install /usr/bin/java java /usr/lib/jvm/openjdk-11/bin/java 20000 && \ + update-alternatives --install /usr/bin/javac javac /usr/lib/jvm/openjdk-11/bin/javac 20000 + +# Install codecov +RUN pip install codecov diff --git a/.circleci/config.yml b/.circleci/config.yml index 147b3b5b77..4ec106c853 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,6 +12,7 @@ jobs: environment: CIRCLE_ARTIFACTS: /tmp/circleci-artifacts CIRCLE_TEST_REPORTS: /tmp/circleci-test-results + CIRCLE_MACHINE_EXECUTOR: true MAVEN_OPTS: -Xmx128m JAVA_VERSION: *jdk_180_version @@ -37,7 +38,7 @@ jobs: if [[ $(python --version 2>&1) = *"2.7.10"* ]] || pyenv versions --bare | grep -x -q '2.7.10'; then pyenv global 2.7.10;else pyenv install --skip-existing 2.7.10 && pyenv global 2.7.10 && pyenv rehash && pip install virtualenv && pip install nose && pip install pep8 && pyenv rehash;fi # Configuring Docker to accept remote connections - - run: ./circle.sh pre_machine + - run: ./circle.sh pre_machine && ./circle.sh pre_machine_docker # Restarting docker for the changes to be applied - run: sudo service docker restart @@ -64,3 +65,60 @@ jobs: path: /var/log/upstart/docker.log - store_artifacts: path: /tmp/circleci-test-results + + build_java11: + docker: + - image: dxia/bionic-openjdk11:test + resource_class: large + working_directory: ~/spotify/helios + parallelism: 6 + shell: /bin/bash --login + environment: + CIRCLE_ARTIFACTS: /tmp/circleci-artifacts + CIRCLE_TEST_REPORTS: /tmp/circleci-test-results + MAVEN_OPTS: -Xmx128m + + steps: + # Machine Setup + # If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each + # The following `checkout` command checks out your code to your working directory. In 1.0 this is done implicitly. In 2.0 you can choose where in the course of a job your code should be checked out. + - checkout + + - setup_remote_docker + + # 'See docs on artifact collection here https://circleci.com/docs/2.0/artifacts/' + - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS + + - run: ./circle.sh pre_machine + + - run: + command: ./circle.sh dependencies + no_output_timeout: 20m + + - run: + command: ./circle.sh test + no_output_timeout: 20m + + - run: if [ "$CIRCLE_NODE_INDEX" == "0" ]; then ./circle.sh post_test; fi + + - run: ./circle.sh collect_test_reports + + - store_test_results: + path: /tmp/circleci-test-results + + # Save artifacts + - store_artifacts: + path: /tmp/circleci-artifacts + - store_artifacts: + path: artifacts + - store_artifacts: + path: /var/log/upstart/docker.log + - store_artifacts: + path: /tmp/circleci-test-results + +workflows: + version: 2 + build_and_test: + jobs: + - build + - build_java11 diff --git a/circle.sh b/circle.sh index 166523b338..6b78c2bd0a 100755 --- a/circle.sh +++ b/circle.sh @@ -2,6 +2,8 @@ case "$1" in pre_machine) + grep -c ^processor /proc/cpuinfo + # ensure correct level of parallelism expected_nodes=6 if [ "$CIRCLE_NODE_TOTAL" -ne "$expected_nodes" ] @@ -10,15 +12,17 @@ case "$1" in exit 1 fi + # Edit pom files to have correct version syntax + for i in $(find . -name pom.xml -not -path './.rvm*'); do sed -i 's/${revision}/0/g' $i; done + + ;; + + pre_machine_docker) # have docker bind to localhost docker_opts='DOCKER_OPTS="$DOCKER_OPTS -H tcp://0.0.0.0:2375"' sudo sh -c "echo '$docker_opts' >> /etc/default/docker" - cat /etc/default/docker - # Edit pom files to have correct version syntax - for i in $(find . -name pom.xml -not -path './.rvm*'); do sed -i 's/${revision}/0/g' $i; done - ;; post_machine) @@ -34,10 +38,12 @@ case "$1" in ;; test) - # fix DOCKER_HOST to be accessible from within containers - docker0_ip=$(/sbin/ifconfig docker0 | grep 'inet addr' | \ - awk -F: '{print $2}' | awk '{print $1}') - export DOCKER_HOST="tcp://$docker0_ip:2375" + if [ -n "$CIRCLE_MACHINE_EXECUTOR" ]; then + # fix DOCKER_HOST to be accessible from within containers + docker0_ip=$(/sbin/ifconfig docker0 | grep 'inet addr' | \ + awk -F: '{print $2}' | awk '{print $1}') + export DOCKER_HOST="tcp://$docker0_ip:2375" + fi case $CIRCLE_NODE_INDEX in 0) diff --git a/helios-services/pom.xml b/helios-services/pom.xml index 010027720c..64eebdf961 100644 --- a/helios-services/pom.xml +++ b/helios-services/pom.xml @@ -35,7 +35,7 @@ com.spotify docker-maven-plugin - 0.4.13 + 1.2.0-SNAPSHOT @@ -119,7 +119,7 @@ com.spotify docker-maven-plugin - 0.4.13 + 1.2.0-SNAPSHOT diff --git a/pom.xml b/pom.xml index cd1aafff69..9108dfcb2c 100644 --- a/pom.xml +++ b/pom.xml @@ -323,12 +323,12 @@ com.spotify ssh-agent-tls - 0.0.3 + 0.0.4 com.spotify ssh-agent-proxy - 0.1.5 + 0.2.0