diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..5d6d4ee --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,44 @@ +name: Benchmark + +on: + workflow_dispatch: + push: + +jobs: + benchmark: + name: Run benchmark + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + language: + # - "go" + # - "nodejs" + - "python" + # - "ruby" + env: + ISHOCON_APP_LANG: ${{ matrix.language }} + UNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + steps: + - uses: actions/checkout@v4 + + - name: Replace base image in docker-compose.yml with github actor name + run: | + make change-lang + sed -i 's/ishocon1-app-base/${{ env.UNAME }}\/ishocon1-app-base/g' ./docker-compose.yml + sed -i 's/ishocon1-app-${{ env.ISHOCON_APP_LANG }}/${{ env.UNAME }}\/ishocon1-app-${{ env.ISHOCON_APP_LANG }}/g' ./docker-compose.yml + cat ./docker-compose.yml + + - name: Build images + run: | + make pull || true + make build + timeout-minutes: 20 + + - run: make bench-from-scratch + timeout-minutes: 10 + + - name: Dump docker logs + uses: jwalton/gh-docker-logs@v2 + if: ${{ always() }} diff --git a/.github/workflows/build_and_push_images.yml b/.github/workflows/build_and_push_images.yml new file mode 100644 index 0000000..106fd9b --- /dev/null +++ b/.github/workflows/build_and_push_images.yml @@ -0,0 +1,116 @@ +name: Build and push images + +on: + workflow_dispatch: + workflow_run: + workflows: ["Benchmark"] + branches: + - main + types: + - completed + +concurrency: + group: ${{ github.workflow }}-${{ github.sha }} + cancel-in-progress: true + +jobs: + build-base-image: + name: Build and push base images + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + UNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + steps: + - run: echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV + - uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.UNAME }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + - name: Cache docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - name: Build and push base image + uses: docker/build-push-action@v5 + with: + context: . + push: true + file: ./docker/app/base/Dockerfile + tags: ${{ env.UNAME }}/ishocon1-app-base:latest,${{ env.UNAME }}/ishocon1-app-base:${{ env.DATE }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + platforms: linux/amd64,linux/arm64/v8 + - name: Move new cache to the place where to be cached + run: | + echo "Temporary fix for cleaning up old cache." + echo "See isssues: + - https://github.com/docker/build-push-action/issues/252 + - https://github.com/moby/buildkit/issues/1896 + " + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache + build-app-images: + name: Build app images + runs-on: ubuntu-latest + timeout-minutes: 60 + needs: build-base-image + strategy: + fail-fast: false + matrix: + language: + # - "go" + # - "nodejs" + - "python" + # - "ruby" + env: + ISHOCON_APP_LANG: ${{ matrix.language }} + UNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + steps: + - run: echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV + - uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.UNAME }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + - name: Cache docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ matrix.language }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-${{ matrix.language }} + ${{ runner.os }}-buildx + - name: Build and push app image + uses: docker/build-push-action@v5 + with: + context: . + push: true + file: ./docker/app/${{ env.ISHOCON_APP_LANG }}/Dockerfile + tags: ${{ env.UNAME }}/ishocon1-app-${{ env.ISHOCON_APP_LANG }}:latest,${{ env.UNAME }}/ishocon1-app-${{ env.ISHOCON_APP_LANG }}:${{ env.DATE }} + build-args: BASE_IMAGE=${{ env.UNAME }}/ishocon1-app-base:latest + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + platforms: linux/amd64,linux/arm64/v8 + - name: Move new cache to the place where to be cached + run: | + echo "Temporary fix for cleaning up old cache." + echo "See isssues: + - https://github.com/docker/build-push-action/issues/252 + - https://github.com/moby/buildkit/issues/1896 + " + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2d65d17 --- /dev/null +++ b/Makefile @@ -0,0 +1,76 @@ +WORKLOAD = 3 +ifeq ($(UNAME),) + UNAME = $(shell whoami) +endif + +ifeq ($(ARCH),) + ARCH = $(shell uname -m) +endif + +LOCAL_ISHOCON_BASE_IMAGE = ishocon1-app-base:latest + +build-base: + docker build \ + -f ./docker/app/base/Dockerfile \ + -t $(LOCAL_ISHOCON_BASE_IMAGE) \ + -t $(UNAME)/ishocon1-app-base:latest \ + .; + +build: change-lang build-base + ISHOCON_APP_LANG=$(ISHOCON_APP_LANG:python) + docker build \ + --build-arg BASE_IMAGE=$(LOCAL_ISHOCON_BASE_IMAGE) \ + -f ./docker/app/$(ISHOCON_APP_LANG)/Dockerfile \ + -t ishocon1-app-$(ISHOCON_APP_LANG):latest \ + -t $(UNAME)/ishocon1-app-$(ISHOCON_APP_LANG):latest \ + .; + @echo "Build done." + +pull-base: + docker pull $(UNAME)/ishocon1-app-base:latest; + docker tag $(UNAME)/ishocon1-app-base:latest $(LOCAL_ISHOCON_BASE_IMAGE); + +pull-app: check-lang + docker pull $(UNAME)/ishocon1-app-$(ISHOCON_APP_LANG):latest; + docker tag $(UNAME)/ishocon1-app-$(ISHOCON_APP_LANG):latest ishocon1-app-$(ISHOCON_APP_LANG):latest; + +pull: pull-base pull-app + @echo "Pull done." + +push: + docker push $(UNAME)/ishocon1-app-base:latest; + docker push $(UNAME)/ishocon1-app-$(ISHOCON_APP_LANG):latest; + +up: + docker compose up -d; + +up-nod: + docker compose up; + +down: + docker compose down; + +bench: + docker exec -i ishocon1-app-1 sh -c "./benchmark --workload ${WORKLOAD}" + +bench-from-scratch: + docker compose up -d; + sleep 30; + docker exec -i ishocon1-app-1 sh -c "./benchmark --workload ${WORKLOAD}" + +check-lang: + if echo "$(ISHOCON_APP_LANG)" | grep -qE '^(ruby|python|go|nodejs)$$'; then \ + echo "ISHOCON_APP_LANG is valid."; \ + else \ + echo "Invalid ISHOCON_APP_LANG. It must be one of: ruby, python, go, nodejs."; \ + exit 1; \ + fi; + +change-lang: check-lang + if sed --version 2>&1 | grep -q GNU; then \ + echo "GNU sed"; \ + sed -i 's/\(ruby\|python\|go\|nodejs\)/'"$(ISHOCON_APP_LANG)"'/g' ./docker-compose.yml; \ + else \ + echo "BSD sed"; \ + sed -i '' -E 's/(ruby|python|go|nodejs)/'"$(ISHOCON_APP_LANG)"'/g' ./docker-compose.yml; \ + fi; diff --git a/admin/README.md b/admin/README.md index 5570a04..c7cf8ac 100644 --- a/admin/README.md +++ b/admin/README.md @@ -3,6 +3,7 @@ $ mysql -u root ishocon1 < init.sql $ ruby insert.rb ``` + # 初期データ * users: 5000件 * products: 10000件 diff --git a/admin/benchmarker/go.mod b/admin/benchmarker/go.mod new file mode 100644 index 0000000..708aa34 --- /dev/null +++ b/admin/benchmarker/go.mod @@ -0,0 +1,9 @@ +module main + +go 1.13 + +require ( + github.com/PuerkitoBio/goquery v1.6.1 + github.com/go-sql-driver/mysql v1.5.0 + golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 +) diff --git a/admin/benchmarker/go.sum b/admin/benchmarker/go.sum new file mode 100644 index 0000000..0eeb46e --- /dev/null +++ b/admin/benchmarker/go.sum @@ -0,0 +1,17 @@ +github.com/PuerkitoBio/goquery v1.6.1 h1:FgjbQZKl5HTmcn4sKBgvx8vv63nhyhIpv7lJpFGCWpk= +github.com/PuerkitoBio/goquery v1.6.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/admin/benchmark.go b/admin/benchmarker/main.go similarity index 100% rename from admin/benchmark.go rename to admin/benchmarker/main.go diff --git a/admin/request.go b/admin/benchmarker/request.go similarity index 100% rename from admin/request.go rename to admin/benchmarker/request.go diff --git a/admin/scenario.go b/admin/benchmarker/scenario.go similarity index 100% rename from admin/scenario.go rename to admin/benchmarker/scenario.go diff --git a/admin/support.go b/admin/benchmarker/support.go similarity index 100% rename from admin/support.go rename to admin/benchmarker/support.go diff --git a/admin/validator.go b/admin/benchmarker/validator.go similarity index 100% rename from admin/validator.go rename to admin/benchmarker/validator.go diff --git a/docker-compose.yml b/docker-compose.yml index ad14ca8..7e87463 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: args: BASE_IMAGE: ishocon1-app-base:latest # ローカルで base image をビルドしない場合は以下を利用すること - # BASE_IMAGE: showwin/ishocon2_app_base:latest + # BASE_IMAGE: showwin/ishocon1_app_base:latest image: ishocon1-app-python:latest environment: ISHOCON_APP_LANG: "${ISHOCON_APP_LANG-python}" @@ -20,5 +20,5 @@ services: command: [/home/ishocon/run.sh] tty: true ports: - - "8080:8080" + - "80:80" - "3306:3306" diff --git a/docker/app/base/Dockerfile b/docker/app/base/Dockerfile index 5ecacac..ea4a7d0 100644 --- a/docker/app/base/Dockerfile +++ b/docker/app/base/Dockerfile @@ -1,3 +1,30 @@ +### benchmarker builder +FROM ubuntu:20.04 as benchmarker-builder + +ENV LANG en_US.UTF-8 +ENV LC_ALL=C.UTF-8 +ENV TZ Asia/Tokyo +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN apt-get update && \ + apt-get install -y wget tzdata && \ + apt-get clean + +# Go のインストール +ARG TARGETARCH +RUN wget -q https://dl.google.com/go/go1.13.15.linux-${TARGETARCH}.tar.gz && \ + tar -C /usr/local -xzf go1.13.15.linux-${TARGETARCH}.tar.gz && \ + rm go1.13.15.linux-${TARGETARCH}.tar.gz +ENV PATH=$PATH:/usr/local/go/bin \ + GOROOT=/usr/local/go \ + GOPATH=$HOME/.local/go + +# build benchmark +COPY admin/benchmarker /root/admin/benchmarker +RUN cd /root/admin/benchmarker && \ + GOARCH=${TARGETARCH} go build -x -o ../../benchmark *.go + +### base image FROM ubuntu:20.04 ENV LANG=en_US.UTF-8 \ @@ -17,6 +44,9 @@ RUN groupadd -g 1001 ishocon && \ echo 'ishocon:ishocon' | chpasswd RUN echo 'ishocon ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers +# benchmarker のコピー +COPY --from=benchmarker-builder /root/benchmark /home/ishocon/benchmark + # *env 系の設定が入った .bashrc のコピー COPY docker/app/base/.bashrc /home/ishocon/.bashrc @@ -34,4 +64,4 @@ COPY admin/ishocon1.dump.tar.gz /home/ishocon/data/ishocon1.dump.tar.gz USER ishocon -EXPOSE 3306 443 +EXPOSE 3306 80 diff --git a/webapp/python/app.py b/webapp/python/app.py index 13023a8..57527b0 100644 --- a/webapp/python/app.py +++ b/webapp/python/app.py @@ -33,21 +33,27 @@ def db(): if hasattr(request, "db"): return request.db else: - request.db = MySQLdb.connect( + db = MySQLdb.connect( host=config("db_host"), port=config("db_port"), user=config("db_username"), password=config("db_password"), database=config("db_database"), charset="utf8mb4", - conv={FIELD_TYPE.LONG: int}, + conv={ + FIELD_TYPE.TINY: int, + FIELD_TYPE.SHORT: int, + FIELD_TYPE.LONG: int, + FIELD_TYPE.INT24: int, + }, cursorclass=DictCursor, ) - cur = request.db.cursor() + cur = db.cursor() cur.execute( "SET SESSION sql_mode='TRADITIONAL,NO_AUTO_VALUE_ON_ZERO,ONLY_FULL_GROUP_BY'" ) cur.execute("SET NAMES utf8mb4") + request.db = db return request.db @@ -58,7 +64,9 @@ def close_db(exception=None): def to_jst(datetime_utc): - return datetime_utc + datetime.timedelta(hours=9) + return datetime.datetime.strptime( + datetime_utc, "%Y-%m-%d %H:%M:%S" + ) + datetime.timedelta(hours=9) def to_utc(datetime_jst): @@ -98,6 +106,7 @@ def update_last_login(user_id): user_id, ), ) + db().commit() def get_comments(product_id): @@ -124,7 +133,7 @@ def get_comments_count(product_id): product_id ) ) - return cur.fetchone()["count"] + return int(cur.fetchone()["count"]) def buy_product(product_id, user_id): @@ -136,6 +145,7 @@ def buy_product(product_id, user_id): to_utc(datetime.datetime.now()).strftime("%Y-%m-%d %H:%M:%S"), ) ) + db().commit() def already_bought(product_id): @@ -146,7 +156,7 @@ def already_bought(product_id): "SELECT count(*) as count FROM histories WHERE product_id = %s AND user_id = %s", (product_id, current_user()["id"]), ) - return cur.fetchone()["count"] > 0 + return int(cur.fetchone()["count"]) > 0 def create_comment(product_id, user_id, content): @@ -162,6 +172,7 @@ def create_comment(product_id, user_id, content): to_utc(datetime.datetime.now()).strftime("%Y-%m-%d %H:%M:%S"), ) ) + db().commit() @app.errorhandler(401) @@ -204,7 +215,7 @@ def get_index(): for product in products: product["description"] = product["description"][:70] - product["created_at"] = to_jst(product["created_at"]) + product["created_at"] = to_jst(product["created_at"].decode()) product["comments"] = get_comments(product["id"]) product["comments_count"] = get_comments_count(product["id"]) @@ -231,7 +242,7 @@ def get_mypage(user_id): for product in products: total_pay += product["price"] product["description"] = product["description"][:70] - product["created_at"] = to_jst(product["created_at"]) + product["created_at"] = to_jst(product["created_at"].decode()) cur = db().cursor() cur.execute("SELECT * FROM users WHERE id = {}".format(str(user_id))) @@ -287,6 +298,7 @@ def get_initialize(): cur.execute("DELETE FROM products WHERE id > 10000") cur.execute("DELETE FROM comments WHERE id > 200000") cur.execute("DELETE FROM histories WHERE id > 500000") + db().commit() return "Finish"