Test in presubmit

iree-org · Jul 6, 2023 · 652c5e6 · 652c5e6
1 parent 3abe70a
commit 652c5e6
Showing 1 changed file with 85 additions and 83 deletions.
diff --git a/.github/workflows/run_comparative_benchmark.yml b/.github/workflows/run_comparative_benchmark.yml
@@ -13,6 +13,7 @@ on:
     # Scheduled to run at 09:00 UTC and 21:00 UTC.
     - cron: '0 09,21 * * *'
   workflow_dispatch:
+  pull_request:
 
 concurrency:
   # A PR number if a pull request and otherwise the commit hash. This cancels
@@ -47,50 +48,51 @@ jobs:
           # `runner-group` a job output variable.
           echo "runner-group=${RUNNER_GROUP}" > "${GITHUB_OUTPUT}"
 
-  build_xla_tools:
-    needs: setup
-    runs-on:
-      - self-hosted  # must come first
-      - runner-group=${{ needs.setup.outputs.runner-group }}
-      - environment=prod
-      - cpu
-      - os-family=Linux
-    env:
-      CUDA_VERSION: 11.8
-      XLA_TOOLS_DIR: xla-tools-dir
-    outputs:
-      xla-tools-dir: ${{ env.XLA_TOOLS_DIR }}
-      xla-tools-dir-archive: ${{ steps.archive.outputs.xla-tools-dir-archive }}
-      xla-tools-dir-gcs-artifact: ${{ steps.upload.outputs.xla-tools-dir-gcs-artifact }}
-    steps:
-      - name: "Checking out PR repository"
-        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791  # v2.5.0
-      - name: "Building XLA"
-        run: |
-          mkdir -p "${XLA_TOOLS_DIR}"
-          docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \
-            "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \
-            ./comparative_benchmark/xla_hlo/build_xla_tools.sh \
-              "${XLA_TOOLS_DIR}" \
-              "${CUDA_VERSION}"
-      - name: "Creating tool dir archive"
-        id: archive
-        env:
-          XLA_TOOLS_DIR_ARCHIVE: ${{ env.XLA_TOOLS_DIR }}.tgz
-        run: |
-          tar -zcvf ${XLA_TOOLS_DIR_ARCHIVE} ${XLA_TOOLS_DIR}
-          echo "xla-tools-dir-archive=${XLA_TOOLS_DIR_ARCHIVE}" >> "${GITHUB_OUTPUT}"
-      - name: "Uploading tool dir archive"
-        id: upload
-        env:
-          XLA_TOOLS_DIR_ARCHIVE: ${{ steps.archive.outputs.xla-tools-dir-archive }}
-          XLA_TOOLS_DIR_GCS_ARTIFACT: ${{ env.GCS_DIR }}/${{ steps.archive.outputs.xla-tools-dir-archive }}
-        run: |
-          gcloud storage cp "${XLA_TOOLS_DIR_ARCHIVE}" "${XLA_TOOLS_DIR_GCS_ARTIFACT}"
-          echo "xla-tools-dir-gcs-artifact=${XLA_TOOLS_DIR_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}"
+  # build_xla_tools:
+  #   needs: setup
+  #   runs-on:
+  #     - self-hosted  # must come first
+  #     - runner-group=${{ needs.setup.outputs.runner-group }}
+  #     - environment=prod
+  #     - cpu
+  #     - os-family=Linux
+  #   env:
+  #     CUDA_VERSION: 11.8
+  #     XLA_TOOLS_DIR: xla-tools-dir
+  #   outputs:
+  #     xla-tools-dir: ${{ env.XLA_TOOLS_DIR }}
+  #     xla-tools-dir-archive: ${{ steps.archive.outputs.xla-tools-dir-archive }}
+  #     xla-tools-dir-gcs-artifact: ${{ steps.upload.outputs.xla-tools-dir-gcs-artifact }}
+  #   steps:
+  #     - name: "Checking out PR repository"
+  #       uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791  # v2.5.0
+  #     - name: "Building XLA"
+  #       run: |
+  #         mkdir -p "${XLA_TOOLS_DIR}"
+  #         docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \
+  #           "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \
+  #           ./comparative_benchmark/xla_hlo/build_xla_tools.sh \
+  #             "${XLA_TOOLS_DIR}" \
+  #             "${CUDA_VERSION}"
+  #     - name: "Creating tool dir archive"
+  #       id: archive
+  #       env:
+  #         XLA_TOOLS_DIR_ARCHIVE: ${{ env.XLA_TOOLS_DIR }}.tgz
+  #       run: |
+  #         tar -zcvf ${XLA_TOOLS_DIR_ARCHIVE} ${XLA_TOOLS_DIR}
+  #         echo "xla-tools-dir-archive=${XLA_TOOLS_DIR_ARCHIVE}" >> "${GITHUB_OUTPUT}"
+  #     - name: "Uploading tool dir archive"
+  #       id: upload
+  #       env:
+  #         XLA_TOOLS_DIR_ARCHIVE: ${{ steps.archive.outputs.xla-tools-dir-archive }}
+  #         XLA_TOOLS_DIR_GCS_ARTIFACT: ${{ env.GCS_DIR }}/${{ steps.archive.outputs.xla-tools-dir-archive }}
+  #       run: |
+  #         gcloud storage cp "${XLA_TOOLS_DIR_ARCHIVE}" "${XLA_TOOLS_DIR_GCS_ARTIFACT}"
+  #         echo "xla-tools-dir-gcs-artifact=${XLA_TOOLS_DIR_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}"
 
   benchmark_on_a2-highgpu-1g:
-    needs: [setup, build_xla_tools]
+  #   needs: [setup, build_xla_tools]
+    needs: [setup]
     runs-on:
       - self-hosted  # must come first
       - runner-group=${{ needs.setup.outputs.runner-group }}
@@ -110,47 +112,47 @@ jobs:
         run: |
           echo "results-gcs-dir=${GCS_DIR}/${TARGET_DEVICE}_$(date +'%Y-%m-%d').$(date +'%s')" >> "${GITHUB_OUTPUT}"
           mkdir "${RESULTS_DIR}"
-      - name: "Downloading and unpacking XLA tools"
-        run: |
-          gcloud storage cp "${XLA_TOOLS_DIR_GCS_ARTIFACT}" "${XLA_TOOLS_DIR_ARCHIVE}"
-          tar -xvf "${XLA_TOOLS_DIR_ARCHIVE}"
-      - name: "Benchmarking XLA-HLO:GPU"
-        env:
-          XLA_HLO_RESULTS_JSON: xla-hlo.json
-          RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }}
-        run: |
-          RESULTS_PATH="${RESULTS_DIR}/${XLA_HLO_RESULTS_JSON}"
-          docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \
-            --env "OOBI_XLA_TOOLS_DIR=${XLA_TOOLS_DIR}" \
-            "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \
-            ./comparative_benchmark/xla_hlo/benchmark_all.sh \
-              "${TARGET_DEVICE}"\
-              "${RESULTS_PATH}"
-          gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/"
-      - name: "Benchmarking JAX-XLA:GPU"
-        env:
-          JAX_XLA_RESULTS_JSON: jax-xla.json
-          RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }}
-        run: |
-          RESULTS_PATH="${RESULTS_DIR}/${JAX_XLA_RESULTS_JSON}"
-          docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \
-            "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \
-            ./comparative_benchmark/jax_xla/benchmark_all.sh \
-              "${TARGET_DEVICE}"\
-              "${RESULTS_PATH}"
-          gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/"
-      - name: "Benchmarking TF-XLA:GPU"
-        env:
-          TF_XLA_RESULTS_JSON: tf-xla.json
-          RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }}
-        run: |
-          RESULTS_PATH="${RESULTS_DIR}/${TF_XLA_RESULTS_JSON}"
-          docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \
-            "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \
-            ./comparative_benchmark/tf_xla/benchmark_all.sh \
-              "${TARGET_DEVICE}"\
-              "${RESULTS_PATH}"
-          gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/"
+      # - name: "Downloading and unpacking XLA tools"
+      #   run: |
+      #     gcloud storage cp "${XLA_TOOLS_DIR_GCS_ARTIFACT}" "${XLA_TOOLS_DIR_ARCHIVE}"
+      #     tar -xvf "${XLA_TOOLS_DIR_ARCHIVE}"
+      # - name: "Benchmarking XLA-HLO:GPU"
+      #   env:
+      #     XLA_HLO_RESULTS_JSON: xla-hlo.json
+      #     RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }}
+      #   run: |
+      #     RESULTS_PATH="${RESULTS_DIR}/${XLA_HLO_RESULTS_JSON}"
+      #     docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \
+      #       --env "OOBI_XLA_TOOLS_DIR=${XLA_TOOLS_DIR}" \
+      #       "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \
+      #       ./comparative_benchmark/xla_hlo/benchmark_all.sh \
+      #         "${TARGET_DEVICE}"\
+      #         "${RESULTS_PATH}"
+      #     gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/"
+      # - name: "Benchmarking JAX-XLA:GPU"
+      #   env:
+      #     JAX_XLA_RESULTS_JSON: jax-xla.json
+      #     RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }}
+      #   run: |
+      #     RESULTS_PATH="${RESULTS_DIR}/${JAX_XLA_RESULTS_JSON}"
+      #     docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \
+      #       "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \
+      #       ./comparative_benchmark/jax_xla/benchmark_all.sh \
+      #         "${TARGET_DEVICE}"\
+      #         "${RESULTS_PATH}"
+      #     gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/"
+      # - name: "Benchmarking TF-XLA:GPU"
+      #   env:
+      #     TF_XLA_RESULTS_JSON: tf-xla.json
+      #     RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }}
+      #   run: |
+      #     RESULTS_PATH="${RESULTS_DIR}/${TF_XLA_RESULTS_JSON}"
+      #     docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \
+      #       "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \
+      #       ./comparative_benchmark/tf_xla/benchmark_all.sh \
+      #         "${TARGET_DEVICE}"\
+      #         "${RESULTS_PATH}"
+      #     gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/"
       - name: "Benchmarking PT-Inductor:GPU"
         env:
           PT_INDUCTOR_RESULTS_JSON: pt-inductor.json