From 657fcaf9d036fd4c39ded00c409c15e74eb2288b Mon Sep 17 00:00:00 2001 From: zyy17 Date: Mon, 24 Jul 2023 11:06:16 +0800 Subject: [PATCH] refactor: unify the greptime artifacts building (#2015) * refactor: unify the make targets of building images * refactor: make Dockerfile more clean 1. Add dev-builder image to build greptime binary easily; 2. Add 'docker/ci/Dockerfile-centos' to release centos image; 3. Delete Dockerfile of aarch64 and just need to use one Dockerfile; Signed-off-by: zyy17 --------- Signed-off-by: zyy17 --- .github/workflows/release.yml | 4 +- Makefile | 112 ++++++++++++++++-- docker/aarch64/Dockerfile | 60 ---------- docker/aarch64/compile-python.sh | 87 -------------- .../Dockerfile} | 31 ++++- docker/ci/Dockerfile | 2 +- docker/ci/Dockerfile-centos | 16 +++ docker/dev-builder/Dockerfile | 36 ++++++ docker/{ci => python}/requirements.txt | 0 docker/{ => ubuntu}/Dockerfile | 30 +++-- 10 files changed, 208 insertions(+), 170 deletions(-) delete mode 100644 docker/aarch64/Dockerfile delete mode 100644 docker/aarch64/compile-python.sh rename docker/{Dockerfile-centos7-builder => centos/Dockerfile} (51%) create mode 100644 docker/ci/Dockerfile-centos create mode 100644 docker/dev-builder/Dockerfile rename docker/{ci => python}/requirements.txt (100%) rename docker/{ => ubuntu}/Dockerfile (59%) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ea11585ffb13..eae7af690192 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -399,7 +399,7 @@ jobs: uses: docker/build-push-action@v3 if: success() || steps.unzip-arm64.conclusion == 'success' # Build and push all platform if unzip-arm64 succeeds with: - context: ./docker/ci/ + context: . file: ./docker/ci/Dockerfile push: true platforms: linux/amd64,linux/arm64 @@ -411,7 +411,7 @@ jobs: uses: docker/build-push-action@v3 if: success() || steps.download-arm64.conclusion == 'failure' # Only build and push amd64 platform if download-arm64 fails with: - context: ./docker/ci/ + context: . file: ./docker/ci/Dockerfile push: true platforms: linux/amd64 diff --git a/Makefile b/Makefile index c35715ffa783..9e6245c9a0b5 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,67 @@ -IMAGE_REGISTRY ?= greptimedb +# The arguments for building images. +CARGO_PROFILE ?= +FEATURES ?= +TARGET_DIR ?= +CARGO_BUILD_OPTS := --locked +IMAGE_REGISTRY ?= docker.io +IMAGE_NAMESPACE ?= greptime IMAGE_TAG ?= latest +BUILDX_MULTI_PLATFORM_BUILD ?= false +BUILDX_BUILDER_NAME ?= gtbuilder +BASE_IMAGE ?= ubuntu +RUST_TOOLCHAIN ?= $(shell cat rust-toolchain.toml | grep channel | cut -d'"' -f2) +CARGO_REGISTRY_CACHE ?= ${HOME}/.cargo/registry + +# The arguments for running integration tests. +ETCD_VERSION ?= v3.5.9 +ETCD_IMAGE ?= quay.io/coreos/etcd:${ETCD_VERSION} +RETRY_COUNT ?= 3 +BUILD_JOBS ?= $(shell expr $$(nproc) / 2) +ifeq ($(BUILD_JOBS), 0) # If the number of cores is less than 2, set the build jobs to 1. + BUILD_JOBS := 1 +endif + +ifdef CARGO_PROFILE + CARGO_BUILD_OPTS += --profile ${CARGO_PROFILE} +endif + +ifdef FEATURES + CARGO_BUILD_OPTS += --features ${FEATURES} +endif + +ifdef TARGET_DIR + CARGO_BUILD_OPTS += --target-dir ${TARGET_DIR} +endif + +ifeq ($(BUILDX_MULTI_PLATFORM_BUILD), true) + BUILDX_MULTI_PLATFORM_BUILD_OPTS := --platform linux/amd64,linux/arm64 --push +else + BUILDX_MULTI_PLATFORM_BUILD_OPTS := -o type=docker +endif ##@ Build .PHONY: build -build: ## Build debug version greptime. - cargo build +build: ## Build debug version greptime. If USE_DEV_BUILDER is true, the binary will be built in dev-builder. +ifeq ($(USE_DEV_BUILDER), true) + docker run --network=host \ + -v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry \ + -w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder:latest \ + make build CARGO_PROFILE=${CARGO_PROFILE} FEATURES=${FEATURES} TARGET_DIR=${TARGET_DIR} +else + cargo build ${CARGO_BUILD_OPTS} +endif .PHONY: release -release: ## Build release version greptime. - cargo build --release +release: ## Build release version greptime. If USE_DEV_BUILDER is true, the binary will be built in dev-builder. +ifeq ($(USE_DEV_BUILDER), true) + docker run --network=host \ + -v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry \ + -w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder:latest \ + make release CARGO_PROFILE=${CARGO_PROFILE} FEATURES=${FEATURES} TARGET_DIR=${TARGET_DIR} +else + cargo build --release ${CARGO_BUILD_OPTS} +endif .PHONY: clean clean: ## Clean the project. @@ -28,13 +80,38 @@ check-toml: ## Check all TOML files. taplo format --check --option "indent_string= " .PHONY: docker-image -docker-image: ## Build docker image. - docker build --network host -f docker/Dockerfile -t ${IMAGE_REGISTRY}:${IMAGE_TAG} . +docker-image: multi-platform-buildx ## Build docker image. + docker buildx build --builder ${BUILDX_BUILDER_NAME} \ + --build-arg="CARGO_PROFILE=${CARGO_PROFILE}" --build-arg="FEATURES=${FEATURES}" \ + -f docker/${BASE_IMAGE}/Dockerfile \ + -t ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/greptimedb:${IMAGE_TAG} ${BUILDX_MULTI_PLATFORM_BUILD_OPTS} . + +.PHONY: build-greptime-by-buildx +build-greptime-by-buildx: multi-platform-buildx ## Build greptime binary by docker buildx. The binary will be copied to the current directory. + docker buildx build --builder ${BUILDX_BUILDER_NAME} \ + --target=builder \ + --build-arg="CARGO_PROFILE=${CARGO_PROFILE}" --build-arg="FEATURES=${FEATURES}" \ + -f docker/${BASE_IMAGE}/Dockerfile \ + -t ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/greptimedb-builder:${IMAGE_TAG} ${BUILDX_MULTI_PLATFORM_BUILD_OPTS} . + + docker run --rm -v ${PWD}:/data \ + --entrypoint cp ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/greptimedb-builder:${IMAGE_TAG} \ + /out/target/${CARGO_PROFILE}/greptime /data/greptime + +.PHONY: dev-builder +dev-builder: multi-platform-buildx ## Build dev-builder image. + docker buildx build --builder ${BUILDX_BUILDER_NAME} \ + --build-arg="RUST_TOOLCHAIN=${RUST_TOOLCHAIN}" \ + -f docker/dev-builder/Dockerfile \ + -t ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder:${IMAGE_TAG} ${BUILDX_MULTI_PLATFORM_BUILD_OPTS} . + +.PHONY: multi-platform-buildx +multi-platform-buildx: ## Create buildx multi-platform builder. + docker buildx inspect ${BUILDX_BUILDER_NAME} || docker buildx create --name ${BUILDX_BUILDER_NAME} --driver docker-container --bootstrap --use ##@ Test - test: nextest ## Run unit and integration tests. - cargo nextest run --retries 3 + cargo nextest run --retries ${RETRY_COUNT} --build-jobs=${BUILD_JOBS} .PHONY: nextest ## Install nextest tools. nextest: @@ -56,6 +133,21 @@ clippy: ## Check clippy rules. fmt-check: ## Check code format. cargo fmt --all -- --check +.PHONY: start-etcd +start-etcd: ## Start single node etcd for testing purpose. + docker run --rm -d --network=host -p 2379-2380:2379-2380 ${ETCD_IMAGE} + +.PHONY: stop-etcd +stop-etcd: ## Stop single node etcd for testing purpose. + docker stop $$(docker ps -q --filter ancestor=${ETCD_IMAGE}) + +.PHONY: run-it-in-container +run-it-in-container: start-etcd ## Run integration tests in dev-builder. + docker run --network=host \ + -v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry -v /tmp:/tmp \ + -w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder:latest \ + make test sqlness-test BUILD_JOBS=${BUILD_JOBS} + ##@ General # The help target prints out all targets with their descriptions organized @@ -71,4 +163,4 @@ fmt-check: ## Check code format. .PHONY: help help: ## Display help messages. - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-30s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) diff --git a/docker/aarch64/Dockerfile b/docker/aarch64/Dockerfile deleted file mode 100644 index e10410bad840..000000000000 --- a/docker/aarch64/Dockerfile +++ /dev/null @@ -1,60 +0,0 @@ -FROM ubuntu:22.04 as builder - -ENV LANG en_US.utf8 -WORKDIR /greptimedb - -# Install dependencies. -RUN apt-get update && apt-get install -y \ - libssl-dev \ - protobuf-compiler \ - curl \ - git \ - build-essential \ - pkg-config \ - wget - -# Install Rust. -SHELL ["/bin/bash", "-c"] -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y -ENV PATH /root/.cargo/bin/:$PATH - -# Install cross platform toolchain -RUN apt-get -y update && \ - apt-get -y install g++-aarch64-linux-gnu gcc-aarch64-linux-gnu && \ - apt-get install binutils-aarch64-linux-gnu - -COPY ./docker/aarch64/compile-python.sh ./docker/aarch64/ -RUN chmod +x ./docker/aarch64/compile-python.sh && \ - ./docker/aarch64/compile-python.sh - -COPY ./rust-toolchain.toml . -# Install rustup target for cross compiling. -RUN rustup target add aarch64-unknown-linux-gnu -COPY . . -# Update dependency, using separate `RUN` to separate cache -RUN cargo fetch - -# This three env var is set in script, so I set it manually in dockerfile. -ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/ -ENV LIBRARY_PATH=$LIBRARY_PATH:/usr/local/lib/ -ENV PY_INSTALL_PATH=/greptimedb/python_arm64_build - -# Set the environment variable for cross compiling and compile it -# cross compiled python is `python3` in path, but pyo3 need `python` in path so alias it -# Build the project in release mode. -RUN export PYO3_CROSS_LIB_DIR=$PY_INSTALL_PATH/lib && \ - alias python=python3 && \ - cargo build --target aarch64-unknown-linux-gnu --release -F pyo3_backend - -# Exporting the binary to the clean image -FROM ubuntu:22.04 as base - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get \ - -y install ca-certificates \ - curl - -WORKDIR /greptime -COPY --from=builder /greptimedb/target/aarch64-unknown-linux-gnu/release/greptime /greptime/bin/ -ENV PATH /greptime/bin/:$PATH - -ENTRYPOINT ["greptime"] diff --git a/docker/aarch64/compile-python.sh b/docker/aarch64/compile-python.sh deleted file mode 100644 index ba0bef763de9..000000000000 --- a/docker/aarch64/compile-python.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# this script will download Python source code, compile it, and install it to /usr/local/lib -# then use this python to compile cross-compiled python for aarch64 -ARCH=$1 -PYTHON_VERSION=3.10.10 -PYTHON_SOURCE_DIR=Python-${PYTHON_VERSION} -PYTHON_INSTALL_PATH_AMD64=${PWD}/python-${PYTHON_VERSION}/amd64 -PYTHON_INSTALL_PATH_AARCH64=${PWD}/python-${PYTHON_VERSION}/aarch64 - -function download_python_source_code() { - wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz - tar -xvf Python-$PYTHON_VERSION.tgz -} - -function compile_for_amd64_platform() { - mkdir -p "$PYTHON_INSTALL_PATH_AMD64" - - echo "Compiling for amd64 platform..." - - ./configure \ - --prefix="$PYTHON_INSTALL_PATH_AMD64" \ - --enable-shared \ - ac_cv_pthread_is_default=no ac_cv_pthread=yes ac_cv_cxx_thread=yes \ - ac_cv_have_long_long_format=yes \ - --disable-ipv6 ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no - - make - make install -} - -# explain Python compile options here a bit:s -# --enable-shared: enable building a shared Python library (default is no) but we do need it for calling from rust -# CC, CXX, AR, LD, RANLIB: set the compiler, archiver, linker, and ranlib programs to use -# build: the machine you are building on, host: the machine you will run the compiled program on -# --with-system-ffi: build _ctypes module using an installed ffi library, see Doc/library/ctypes.rst, not used in here TODO: could remove -# ac_cv_pthread_is_default=no ac_cv_pthread=yes ac_cv_cxx_thread=yes: -# allow cross-compiled python to have -pthread set for CXX, see https://github.com/python/cpython/pull/22525 -# ac_cv_have_long_long_format=yes: target platform supports long long type -# disable-ipv6: disable ipv6 support, we don't need it in here -# ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no: disable pty support, we don't need it in here -function compile_for_aarch64_platform() { - export LD_LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LD_LIBRARY_PATH - export LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LIBRARY_PATH - export PATH=$PYTHON_INSTALL_PATH_AMD64/bin:$PATH - - mkdir -p "$PYTHON_INSTALL_PATH_AARCH64" - - echo "Compiling for aarch64 platform..." - echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH" - echo "LIBRARY_PATH: $LIBRARY_PATH" - echo "PATH: $PATH" - - ./configure --build=x86_64-linux-gnu --host=aarch64-linux-gnu \ - --prefix="$PYTHON_INSTALL_PATH_AARCH64" --enable-optimizations \ - CC=aarch64-linux-gnu-gcc \ - CXX=aarch64-linux-gnu-g++ \ - AR=aarch64-linux-gnu-ar \ - LD=aarch64-linux-gnu-ld \ - RANLIB=aarch64-linux-gnu-ranlib \ - --enable-shared \ - ac_cv_pthread_is_default=no ac_cv_pthread=yes ac_cv_cxx_thread=yes \ - ac_cv_have_long_long_format=yes \ - --disable-ipv6 ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no - - make - make altinstall -} - -# Main script starts here. -download_python_source_code - -# Enter the python source code directory. -cd $PYTHON_SOURCE_DIR || exit 1 - -# Build local python first, then build cross-compiled python. -compile_for_amd64_platform - -# Clean the build directory. -make clean && make distclean - -# Cross compile python for aarch64. -if [ "$ARCH" = "aarch64-unknown-linux-gnu" ]; then - compile_for_aarch64_platform -fi diff --git a/docker/Dockerfile-centos7-builder b/docker/centos/Dockerfile similarity index 51% rename from docker/Dockerfile-centos7-builder rename to docker/centos/Dockerfile index 5d31a52d1deb..ae42114593dc 100644 --- a/docker/Dockerfile-centos7-builder +++ b/docker/centos/Dockerfile @@ -1,4 +1,7 @@ -FROM centos:7 +FROM centos:7 as builder + +ARG CARGO_PROFILE +ARG FEATURES ENV LANG en_US.utf8 WORKDIR /greptimedb @@ -21,4 +24,28 @@ SHELL ["/bin/bash", "-c"] RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y ENV PATH /opt/rh/rh-python38/root/usr/bin:/usr/local/bin:/root/.cargo/bin/:$PATH -CMD ["cargo", "build", "--release"] +# Build the project in release mode. +RUN --mount=target=.,rw \ + --mount=type=cache,target=/usr/local/cargo/registry \ + make build \ + CARGO_PROFILE=${CARGO_PROFILE} \ + FEATURES=${FEATURES} \ + TARGET_DIR=/out/target + +# Export the binary to the clean image. +FROM centos:7 as base + +ARG CARGO_PROFILE + +RUN yum install -y epel-release \ + openssl \ + openssl-devel \ + centos-release-scl \ + rh-python38 \ + rh-python38-python-devel + +WORKDIR /greptime +COPY --from=builder /out/target/${CARGO_PROFILE}/greptime /greptime/bin/ +ENV PATH /greptime/bin/:$PATH + +ENTRYPOINT ["greptime"] diff --git a/docker/ci/Dockerfile b/docker/ci/Dockerfile index a55732f7b7e3..8907cf624c49 100644 --- a/docker/ci/Dockerfile +++ b/docker/ci/Dockerfile @@ -7,7 +7,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ python3-pip \ curl -COPY requirements.txt /etc/greptime/requirements.txt +COPY ./docker/python/requirements.txt /etc/greptime/requirements.txt RUN python3 -m pip install -r /etc/greptime/requirements.txt diff --git a/docker/ci/Dockerfile-centos b/docker/ci/Dockerfile-centos new file mode 100644 index 000000000000..1ca6f32acb3b --- /dev/null +++ b/docker/ci/Dockerfile-centos @@ -0,0 +1,16 @@ +FROM centos:7 + +RUN yum install -y epel-release \ + openssl \ + openssl-devel \ + centos-release-scl \ + rh-python38 \ + rh-python38-python-devel + +ARG TARGETARCH + +ADD $TARGETARCH/greptime /greptime/bin/ + +ENV PATH /greptime/bin/:$PATH + +ENTRYPOINT ["greptime"] diff --git a/docker/dev-builder/Dockerfile b/docker/dev-builder/Dockerfile new file mode 100644 index 000000000000..afefe2a20a93 --- /dev/null +++ b/docker/dev-builder/Dockerfile @@ -0,0 +1,36 @@ +FROM ubuntu:22.04 + +ENV LANG en_US.utf8 +WORKDIR /greptimedb + +# Install dependencies. +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + libssl-dev \ + tzdata \ + protobuf-compiler \ + curl \ + ca-certificates \ + git \ + build-essential \ + pkg-config \ + python3.10 \ + python3.10-dev \ + python3-pip + +RUN git config --global --add safe.directory /greptimedb + +# Install Python dependencies. +COPY ./docker/python/requirements.txt /etc/greptime/requirements.txt +RUN python3 -m pip install -r /etc/greptime/requirements.txt + +# Install Rust. +SHELL ["/bin/bash", "-c"] +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y +ENV PATH /root/.cargo/bin/:$PATH + +# Install Rust toolchains. +ARG RUST_TOOLCHAIN +RUN rustup toolchain install ${RUST_TOOLCHAIN} + +# Install nextest. +RUN cargo install cargo-nextest --locked diff --git a/docker/ci/requirements.txt b/docker/python/requirements.txt similarity index 100% rename from docker/ci/requirements.txt rename to docker/python/requirements.txt diff --git a/docker/Dockerfile b/docker/ubuntu/Dockerfile similarity index 59% rename from docker/Dockerfile rename to docker/ubuntu/Dockerfile index 8747926a9ca1..18cdd7e16890 100644 --- a/docker/Dockerfile +++ b/docker/ubuntu/Dockerfile @@ -1,5 +1,8 @@ FROM ubuntu:22.04 as builder +ARG CARGO_PROFILE +ARG FEATURES + ENV LANG en_US.utf8 WORKDIR /greptimedb @@ -11,11 +14,9 @@ RUN apt-get update && apt-get install -y \ git \ build-essential \ pkg-config \ - python3 \ - python3-dev \ - python3-pip \ - && pip3 install --upgrade pip \ - && pip3 install pyarrow + python3.10 \ + python3.10-dev \ + python3-pip # Install Rust. SHELL ["/bin/bash", "-c"] @@ -23,19 +24,32 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-mo ENV PATH /root/.cargo/bin/:$PATH # Build the project in release mode. -COPY . . -RUN cargo build --release +RUN --mount=target=.,rw \ + --mount=type=cache,target=/usr/local/cargo/registry \ + make build \ + CARGO_PROFILE=${CARGO_PROFILE} \ + FEATURES=${FEATURES} \ + TARGET_DIR=/out/target # Export the binary to the clean image. # TODO(zyy17): Maybe should use the more secure container image. FROM ubuntu:22.04 as base +ARG CARGO_PROFILE + RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get \ -y install ca-certificates \ + python3.10 \ + python3.10-dev \ + python3-pip \ curl +COPY ./docker/python/requirements.txt /etc/greptime/requirements.txt + +RUN python3 -m pip install -r /etc/greptime/requirements.txt + WORKDIR /greptime -COPY --from=builder /greptimedb/target/release/greptime /greptime/bin/ +COPY --from=builder /out/target/${CARGO_PROFILE}/greptime /greptime/bin/ ENV PATH /greptime/bin/:$PATH ENTRYPOINT ["greptime"]