diff --git a/.github/workflows/run_comparative_benchmark.yml b/.github/workflows/run_comparative_benchmark.yml index 8cabfd16..5ed9ef98 100644 --- a/.github/workflows/run_comparative_benchmark.yml +++ b/.github/workflows/run_comparative_benchmark.yml @@ -13,6 +13,7 @@ on: # Scheduled to run at 09:00 UTC and 21:00 UTC. - cron: '0 09,21 * * *' workflow_dispatch: + pull_request: concurrency: # A PR number if a pull request and otherwise the commit hash. This cancels @@ -47,50 +48,51 @@ jobs: # `runner-group` a job output variable. echo "runner-group=${RUNNER_GROUP}" > "${GITHUB_OUTPUT}" - build_xla_tools: - needs: setup - runs-on: - - self-hosted # must come first - - runner-group=${{ needs.setup.outputs.runner-group }} - - environment=prod - - cpu - - os-family=Linux - env: - CUDA_VERSION: 11.8 - XLA_TOOLS_DIR: xla-tools-dir - outputs: - xla-tools-dir: ${{ env.XLA_TOOLS_DIR }} - xla-tools-dir-archive: ${{ steps.archive.outputs.xla-tools-dir-archive }} - xla-tools-dir-gcs-artifact: ${{ steps.upload.outputs.xla-tools-dir-gcs-artifact }} - steps: - - name: "Checking out PR repository" - uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0 - - name: "Building XLA" - run: | - mkdir -p "${XLA_TOOLS_DIR}" - docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ - "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \ - ./comparative_benchmark/xla_hlo/build_xla_tools.sh \ - "${XLA_TOOLS_DIR}" \ - "${CUDA_VERSION}" - - name: "Creating tool dir archive" - id: archive - env: - XLA_TOOLS_DIR_ARCHIVE: ${{ env.XLA_TOOLS_DIR }}.tgz - run: | - tar -zcvf ${XLA_TOOLS_DIR_ARCHIVE} ${XLA_TOOLS_DIR} - echo "xla-tools-dir-archive=${XLA_TOOLS_DIR_ARCHIVE}" >> "${GITHUB_OUTPUT}" - - name: "Uploading tool dir archive" - id: upload - env: - XLA_TOOLS_DIR_ARCHIVE: ${{ steps.archive.outputs.xla-tools-dir-archive }} - XLA_TOOLS_DIR_GCS_ARTIFACT: ${{ env.GCS_DIR }}/${{ steps.archive.outputs.xla-tools-dir-archive }} - run: | - gcloud storage cp "${XLA_TOOLS_DIR_ARCHIVE}" "${XLA_TOOLS_DIR_GCS_ARTIFACT}" - echo "xla-tools-dir-gcs-artifact=${XLA_TOOLS_DIR_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}" + # build_xla_tools: + # needs: setup + # runs-on: + # - self-hosted # must come first + # - runner-group=${{ needs.setup.outputs.runner-group }} + # - environment=prod + # - cpu + # - os-family=Linux + # env: + # CUDA_VERSION: 11.8 + # XLA_TOOLS_DIR: xla-tools-dir + # outputs: + # xla-tools-dir: ${{ env.XLA_TOOLS_DIR }} + # xla-tools-dir-archive: ${{ steps.archive.outputs.xla-tools-dir-archive }} + # xla-tools-dir-gcs-artifact: ${{ steps.upload.outputs.xla-tools-dir-gcs-artifact }} + # steps: + # - name: "Checking out PR repository" + # uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0 + # - name: "Building XLA" + # run: | + # mkdir -p "${XLA_TOOLS_DIR}" + # docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ + # "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \ + # ./comparative_benchmark/xla_hlo/build_xla_tools.sh \ + # "${XLA_TOOLS_DIR}" \ + # "${CUDA_VERSION}" + # - name: "Creating tool dir archive" + # id: archive + # env: + # XLA_TOOLS_DIR_ARCHIVE: ${{ env.XLA_TOOLS_DIR }}.tgz + # run: | + # tar -zcvf ${XLA_TOOLS_DIR_ARCHIVE} ${XLA_TOOLS_DIR} + # echo "xla-tools-dir-archive=${XLA_TOOLS_DIR_ARCHIVE}" >> "${GITHUB_OUTPUT}" + # - name: "Uploading tool dir archive" + # id: upload + # env: + # XLA_TOOLS_DIR_ARCHIVE: ${{ steps.archive.outputs.xla-tools-dir-archive }} + # XLA_TOOLS_DIR_GCS_ARTIFACT: ${{ env.GCS_DIR }}/${{ steps.archive.outputs.xla-tools-dir-archive }} + # run: | + # gcloud storage cp "${XLA_TOOLS_DIR_ARCHIVE}" "${XLA_TOOLS_DIR_GCS_ARTIFACT}" + # echo "xla-tools-dir-gcs-artifact=${XLA_TOOLS_DIR_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}" benchmark_on_a2-highgpu-1g: - needs: [setup, build_xla_tools] + # needs: [setup, build_xla_tools] + needs: [setups] runs-on: - self-hosted # must come first - runner-group=${{ needs.setup.outputs.runner-group }} @@ -110,47 +112,47 @@ jobs: run: | echo "results-gcs-dir=${GCS_DIR}/${TARGET_DEVICE}_$(date +'%Y-%m-%d').$(date +'%s')" >> "${GITHUB_OUTPUT}" mkdir "${RESULTS_DIR}" - - name: "Downloading and unpacking XLA tools" - run: | - gcloud storage cp "${XLA_TOOLS_DIR_GCS_ARTIFACT}" "${XLA_TOOLS_DIR_ARCHIVE}" - tar -xvf "${XLA_TOOLS_DIR_ARCHIVE}" - - name: "Benchmarking XLA-HLO:GPU" - env: - XLA_HLO_RESULTS_JSON: xla-hlo.json - RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }} - run: | - RESULTS_PATH="${RESULTS_DIR}/${XLA_HLO_RESULTS_JSON}" - docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ - --env "OOBI_XLA_TOOLS_DIR=${XLA_TOOLS_DIR}" \ - "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \ - ./comparative_benchmark/xla_hlo/benchmark_all.sh \ - "${TARGET_DEVICE}"\ - "${RESULTS_PATH}" - gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" - - name: "Benchmarking JAX-XLA:GPU" - env: - JAX_XLA_RESULTS_JSON: jax-xla.json - RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }} - run: | - RESULTS_PATH="${RESULTS_DIR}/${JAX_XLA_RESULTS_JSON}" - docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ - "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \ - ./comparative_benchmark/jax_xla/benchmark_all.sh \ - "${TARGET_DEVICE}"\ - "${RESULTS_PATH}" - gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" - - name: "Benchmarking TF-XLA:GPU" - env: - TF_XLA_RESULTS_JSON: tf-xla.json - RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }} - run: | - RESULTS_PATH="${RESULTS_DIR}/${TF_XLA_RESULTS_JSON}" - docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ - "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \ - ./comparative_benchmark/tf_xla/benchmark_all.sh \ - "${TARGET_DEVICE}"\ - "${RESULTS_PATH}" - gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" + # - name: "Downloading and unpacking XLA tools" + # run: | + # gcloud storage cp "${XLA_TOOLS_DIR_GCS_ARTIFACT}" "${XLA_TOOLS_DIR_ARCHIVE}" + # tar -xvf "${XLA_TOOLS_DIR_ARCHIVE}" + # - name: "Benchmarking XLA-HLO:GPU" + # env: + # XLA_HLO_RESULTS_JSON: xla-hlo.json + # RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }} + # run: | + # RESULTS_PATH="${RESULTS_DIR}/${XLA_HLO_RESULTS_JSON}" + # docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ + # --env "OOBI_XLA_TOOLS_DIR=${XLA_TOOLS_DIR}" \ + # "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \ + # ./comparative_benchmark/xla_hlo/benchmark_all.sh \ + # "${TARGET_DEVICE}"\ + # "${RESULTS_PATH}" + # gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" + # - name: "Benchmarking JAX-XLA:GPU" + # env: + # JAX_XLA_RESULTS_JSON: jax-xla.json + # RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }} + # run: | + # RESULTS_PATH="${RESULTS_DIR}/${JAX_XLA_RESULTS_JSON}" + # docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ + # "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \ + # ./comparative_benchmark/jax_xla/benchmark_all.sh \ + # "${TARGET_DEVICE}"\ + # "${RESULTS_PATH}" + # gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" + # - name: "Benchmarking TF-XLA:GPU" + # env: + # TF_XLA_RESULTS_JSON: tf-xla.json + # RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }} + # run: | + # RESULTS_PATH="${RESULTS_DIR}/${TF_XLA_RESULTS_JSON}" + # docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ + # "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \ + # ./comparative_benchmark/tf_xla/benchmark_all.sh \ + # "${TARGET_DEVICE}"\ + # "${RESULTS_PATH}" + # gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" - name: "Benchmarking PT-Inductor:GPU" env: PT_INDUCTOR_RESULTS_JSON: pt-inductor.json