Add CI for Python version

.github/workflows/benchmark.yml

@@ -0,0 +1,44 @@
+name: Benchmark
+
+on:
+    workflow_dispatch:
+    push:
+
+jobs:
+  benchmark:
+    name: Run benchmark
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        language:
+          # - "go"
+          # - "nodejs"
+          - "python"
+          # - "ruby"
+    env:
+      ISHOCON_APP_LANG: ${{ matrix.language }}
+      UNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Replace base image in docker-compose.yml with github actor name
+        run: |
+          make change-lang
+          sed -i 's/ishocon1-app-base/${{ env.UNAME }}\/ishocon1-app-base/g' ./docker-compose.yml
+          sed -i 's/ishocon1-app-${{ env.ISHOCON_APP_LANG }}/${{ env.UNAME }}\/ishocon1-app-${{ env.ISHOCON_APP_LANG }}/g' ./docker-compose.yml
+          cat ./docker-compose.yml
+
+      - name: Build images
+        run: |
+          make pull || true
+          make build
+        timeout-minutes: 20
+
+      - run: make bench-from-scratch
+        timeout-minutes: 10
+
+      - name: Dump docker logs
+        uses: jwalton/gh-docker-logs@v2
+        if: ${{ always() }}

.github/workflows/build_and_push_images.yml

@@ -0,0 +1,116 @@
+name: Build and push images
+
+on:
+    workflow_dispatch:
+    workflow_run:
+      workflows: ["Benchmark"]
+      branches:
+        - main
+      types:
+        - completed
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  build-base-image:
+    name: Build and push base images
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      UNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+    steps:
+      - run: echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV
+      - uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ env.UNAME }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
+      - name: Cache docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+      - name: Build and push base image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          file: ./docker/app/base/Dockerfile
+          tags: ${{ env.UNAME }}/ishocon1-app-base:latest,${{ env.UNAME }}/ishocon1-app-base:${{ env.DATE }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
+          platforms: linux/amd64,linux/arm64/v8
+      - name: Move new cache to the place where to be cached
+        run: |
+          echo "Temporary fix for cleaning up old cache."
+          echo "See isssues:
+            - https://github.com/docker/build-push-action/issues/252
+            - https://github.com/moby/buildkit/issues/1896
+          "
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+  build-app-images:
+    name: Build app images
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    needs: build-base-image
+    strategy:
+      fail-fast: false
+      matrix:
+        language:
+          # - "go"
+          # - "nodejs"
+          - "python"
+          # - "ruby"
+    env:
+      ISHOCON_APP_LANG: ${{ matrix.language }}
+      UNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+    steps:
+      - run: echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV
+      - uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ env.UNAME }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
+      - name: Cache docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ matrix.language }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-${{ matrix.language }}
+            ${{ runner.os }}-buildx
+      - name: Build and push app image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          file: ./docker/app/${{ env.ISHOCON_APP_LANG }}/Dockerfile
+          tags: ${{ env.UNAME }}/ishocon1-app-${{ env.ISHOCON_APP_LANG }}:latest,${{ env.UNAME }}/ishocon1-app-${{ env.ISHOCON_APP_LANG }}:${{ env.DATE }}
+          build-args: BASE_IMAGE=${{ env.UNAME }}/ishocon1-app-base:latest
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
+          platforms: linux/amd64,linux/arm64/v8
+      - name: Move new cache to the place where to be cached
+        run: |
+          echo "Temporary fix for cleaning up old cache."
+          echo "See isssues:
+            - https://github.com/docker/build-push-action/issues/252
+            - https://github.com/moby/buildkit/issues/1896
+          "
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

Makefile

@@ -0,0 +1,76 @@
+WORKLOAD = 3
+ifeq ($(UNAME),)
+	UNAME = $(shell whoami)
+endif
+
+ifeq ($(ARCH),)
+	ARCH = $(shell uname -m)
+endif
+
+LOCAL_ISHOCON_BASE_IMAGE = ishocon1-app-base:latest
+
+build-base:
+	docker build \
+	-f ./docker/app/base/Dockerfile \
+	-t $(LOCAL_ISHOCON_BASE_IMAGE) \
+	-t $(UNAME)/ishocon1-app-base:latest \
+	.;
+
+build: change-lang build-base
+	ISHOCON_APP_LANG=$(ISHOCON_APP_LANG:python)
+	docker build \
+	--build-arg BASE_IMAGE=$(LOCAL_ISHOCON_BASE_IMAGE) \
+	-f ./docker/app/$(ISHOCON_APP_LANG)/Dockerfile \
+	-t ishocon1-app-$(ISHOCON_APP_LANG):latest \
+	-t $(UNAME)/ishocon1-app-$(ISHOCON_APP_LANG):latest \
+	.;
+	@echo "Build done."
+
+pull-base:
+	docker pull $(UNAME)/ishocon1-app-base:latest;
+	docker tag $(UNAME)/ishocon1-app-base:latest $(LOCAL_ISHOCON_BASE_IMAGE);
+
+pull-app: check-lang
+	docker pull $(UNAME)/ishocon1-app-$(ISHOCON_APP_LANG):latest;
+	docker tag $(UNAME)/ishocon1-app-$(ISHOCON_APP_LANG):latest ishocon1-app-$(ISHOCON_APP_LANG):latest;
+
+pull: pull-base pull-app
+	@echo "Pull done."
+
+push:
+	docker push $(UNAME)/ishocon1-app-base:latest;
+	docker push $(UNAME)/ishocon1-app-$(ISHOCON_APP_LANG):latest;
+
+up:
+	docker compose up -d;
+
+up-nod:
+	docker compose up;
+
+down:
+	docker compose down;
+
+bench:
+	docker exec -i ishocon1-app-1 sh -c "./benchmark --workload ${WORKLOAD}"
+
+bench-from-scratch:
+	docker compose up -d;
+	sleep 30;
+	docker exec -i ishocon1-app-1 sh -c "./benchmark --workload ${WORKLOAD}"
+
+check-lang:
+	if echo "$(ISHOCON_APP_LANG)" | grep -qE '^(ruby|python|go|nodejs)$$'; then \
+        echo "ISHOCON_APP_LANG is valid."; \
+    else \
+        echo "Invalid ISHOCON_APP_LANG. It must be one of: ruby, python, go, nodejs."; \
+        exit 1; \
+    fi;
+
+change-lang: check-lang
+	if sed --version 2>&1 | grep -q GNU; then \
+		echo "GNU sed"; \
+		sed -i 's/\(ruby\|python\|go\|nodejs\)/'"$(ISHOCON_APP_LANG)"'/g' ./docker-compose.yml; \
+	else \
+		echo "BSD sed"; \
+		sed -i '' -E 's/(ruby|python|go|nodejs)/'"$(ISHOCON_APP_LANG)"'/g' ./docker-compose.yml; \
+	fi;

admin/README.md

@@ -3,6 +3,7 @@
 $ mysql -u root ishocon1 < init.sql
 $ ruby insert.rb
 ```
+
 # 初期データ
 * users: 5000件
 * products: 10000件

admin/benchmarker/go.mod

@@ -0,0 +1,9 @@
+module main
+
+go 1.13
+
+require (
+	github.com/PuerkitoBio/goquery v1.6.1
+	github.com/go-sql-driver/mysql v1.5.0
+	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
+)

admin/benchmarker/go.sum

@@ -0,0 +1,17 @@
+github.com/PuerkitoBio/goquery v1.6.1 h1:FgjbQZKl5HTmcn4sKBgvx8vv63nhyhIpv7lJpFGCWpk=
+github.com/PuerkitoBio/goquery v1.6.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

docker-compose.yml

@@ -7,7 +7,7 @@ services:
       args:
         BASE_IMAGE: ishocon1-app-base:latest
         # ローカルで base image をビルドしない場合は以下を利用すること
-        # BASE_IMAGE: showwin/ishocon2_app_base:latest
+        # BASE_IMAGE: showwin/ishocon1_app_base:latest
     image: ishocon1-app-python:latest
     environment:
       ISHOCON_APP_LANG: "${ISHOCON_APP_LANG-python}"
@@ -20,5 +20,5 @@ services:
     command: [/home/ishocon/run.sh]
     tty: true
     ports:
-      - "8080:8080"
+      - "80:80"
       - "3306:3306"

docker/app/base/Dockerfile

@@ -1,3 +1,30 @@
+### benchmarker builder
+FROM ubuntu:20.04 as benchmarker-builder
+
+ENV LANG en_US.UTF-8
+ENV LC_ALL=C.UTF-8
+ENV TZ Asia/Tokyo
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+RUN apt-get update && \
+    apt-get install -y wget tzdata && \
+    apt-get clean
+
+# Go のインストール
+ARG TARGETARCH
+RUN wget -q https://dl.google.com/go/go1.13.15.linux-${TARGETARCH}.tar.gz && \
+    tar -C /usr/local -xzf go1.13.15.linux-${TARGETARCH}.tar.gz && \
+    rm go1.13.15.linux-${TARGETARCH}.tar.gz
+ENV PATH=$PATH:/usr/local/go/bin \
+    GOROOT=/usr/local/go \
+    GOPATH=$HOME/.local/go
+
+# build benchmark
+COPY admin/benchmarker /root/admin/benchmarker
+RUN cd /root/admin/benchmarker &&  \
+    GOARCH=${TARGETARCH} go build -x -o ../../benchmark *.go
+
+### base image
 FROM ubuntu:20.04
 
 ENV LANG=en_US.UTF-8 \
@@ -17,6 +44,9 @@ RUN groupadd -g 1001 ishocon && \
     echo 'ishocon:ishocon' | chpasswd
 RUN echo 'ishocon ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 
+# benchmarker のコピー
+COPY --from=benchmarker-builder /root/benchmark /home/ishocon/benchmark
+
 # *env 系の設定が入った .bashrc のコピー
 COPY docker/app/base/.bashrc /home/ishocon/.bashrc
 
@@ -34,4 +64,4 @@ COPY admin/ishocon1.dump.tar.gz /home/ishocon/data/ishocon1.dump.tar.gz
 
 USER ishocon
 
-EXPOSE 3306 443
+EXPOSE 3306 80