From 3abe70abb52676d39ae5152930ccadb6251534b0 Mon Sep 17 00:00:00 2001
From: Jerry Wu <cheyuw@google.com>
Date: Thu, 6 Jul 2023 21:25:37 +0000
Subject: [PATCH] Add becnhmark workflow

---
 .../workflows/run_comparative_benchmark.yml   | 12 +++++
 .../pt_inductor/benchmark_all.sh              | 50 +++++++++++++++++++
 .../pt_inductor/setup_venv.sh                 | 28 +++++++++++
 3 files changed, 90 insertions(+)
 create mode 100755 comparative_benchmark/pt_inductor/benchmark_all.sh
 create mode 100755 comparative_benchmark/pt_inductor/setup_venv.sh

diff --git a/.github/workflows/run_comparative_benchmark.yml b/.github/workflows/run_comparative_benchmark.yml
index 10cb876f..8cabfd16 100644
--- a/.github/workflows/run_comparative_benchmark.yml
+++ b/.github/workflows/run_comparative_benchmark.yml
@@ -151,3 +151,15 @@ jobs:
               "${TARGET_DEVICE}"\
               "${RESULTS_PATH}"
           gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/"
+      - name: "Benchmarking PT-Inductor:GPU"
+        env:
+          PT_INDUCTOR_RESULTS_JSON: pt-inductor.json
+          RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }}
+        run: |
+          RESULTS_PATH="${RESULTS_DIR}/${PT_INDUCTOR_RESULTS_JSON}"
+          docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \
+            "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \
+            ./comparative_benchmark/pt_inductor/benchmark_all.sh \
+              "${TARGET_DEVICE}"\
+              "${RESULTS_PATH}"
+          gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/"
diff --git a/comparative_benchmark/pt_inductor/benchmark_all.sh b/comparative_benchmark/pt_inductor/benchmark_all.sh
new file mode 100755
index 00000000..6c513ae3
--- /dev/null
+++ b/comparative_benchmark/pt_inductor/benchmark_all.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Copyright 2023 The OpenXLA Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+set -xeuo pipefail
+
+VENV_DIR="${OOBI_VENV_DIR:-pt-benchmarks.venv}"
+PYTHON="${PYTHON:-/usr/bin/python3}"
+TARGET_DEVICE="${1:-${OOBI_TARGET_DEVICE}}"
+OUTPUT_PATH="${2:-${OOBI_OUTPUT}}"
+
+TD="$(cd $(dirname $0) && pwd)"
+
+VENV_DIR="${VENV_DIR}" PYTHON="${PYTHON}" source "${TD}/setup_venv.sh"
+
+declare -a GPU_BENCHMARK_NAMES=(
+  "models/BERT_LARGE_FP32_PT_.+/inputs/.+/expected_outputs/.+/target_devices/a2-highgpu-1g"
+  "models/BERT_LARGE_FP16_PT_.+/inputs/.+/expected_outputs/.+/target_devices/a2-highgpu-1g"
+)
+
+declare -a CPU_BENCHMARK_NAMES=(
+  "models/BERT_LARGE_FP32_PT_.+_BATCH1/inputs/.+/expected_outputs/.+/target_devices/c2-standard-16"
+  "models/BERT_LARGE_FP32_PT_.+_BATCH32/inputs/.+/expected_outputs/.+/target_devices/c2-standard-16"
+  "models/BERT_LARGE_FP32_PT_.+_BATCH64/inputs/.+/expected_outputs/.+/target_devices/c2-standard-16"
+)
+
+if [ "${TARGET_DEVICE}" = "a2-highgpu-1g" ]; then
+  BENCHMARK_NAMES=("${GPU_BENCHMARK_NAMES[@]}")
+  ITERATIONS=50
+elif [ "${TARGET_DEVICE}" = "c2-standard-16" ]; then
+  BENCHMARK_NAMES=("${CPU_BENCHMARK_NAMES[@]}")
+  ITERATIONS=20
+else
+  echo "Unsupported target device ${TARGET_DEVICE}."
+  exit 1
+fi
+
+"${TD}/../scripts/create_results_json.sh" "${OUTPUT_PATH}"
+
+for benchmark_name in "${BENCHMARK_NAMES[@]}"; do
+  "${TD}/run_benchmarks.py" \
+    --benchmark_name="${benchmark_name}" \
+    --output="${OUTPUT_PATH}" \
+    --iterations="${ITERATIONS}" \
+    --verbose
+done
diff --git a/comparative_benchmark/pt_inductor/setup_venv.sh b/comparative_benchmark/pt_inductor/setup_venv.sh
new file mode 100755
index 00000000..5ec38378
--- /dev/null
+++ b/comparative_benchmark/pt_inductor/setup_venv.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# Copyright 2023 The OpenXLA Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# Environment variables:
+#   VENV_DIR=tf-benchmarks.venv
+#   PYTHON=/usr/bin/python3.10
+
+set -xeuo pipefail
+
+TD="$(cd $(dirname $0) && pwd)"
+VENV_DIR=${VENV_DIR:-pt-benchmarks.venv}
+PYTHON="${PYTHON:-"$(which python3)"}"
+
+"${PYTHON}" -m venv "${VENV_DIR}" || echo "Could not create venv."
+source "${VENV_DIR}/bin/activate" || echo "Could not activate venv"
+
+# Upgrade pip and install requirements. 'python' is used here in order to
+# reference to the python executable from the venv.
+python -m pip install --upgrade pip || die "Could not upgrade pip"
+python -m pip install -r "${TD}/requirements.txt"
+
+echo "Activate venv with:"
+echo "  source ${VENV_DIR}/bin/activate"