[PyCDE] Move build and publishing pipeline into repo (#7062)

Previously was built and published internally. Resolves #1509.

.github/workflows/pycdePublish.yml

@@ -1,60 +1,195 @@
 name: PyCDE Test and Publish
 
-# Build the wheels for a series of Python versions and OSes and publish them to
-# PyPI.
-
 on:
+  push:
+    tags:
+      - pycde-*
   workflow_dispatch:
 
 jobs:
-  # Build CIRCT and run its tests using a Docker container with all the
-  # integration testing prerequisite installed.
-  build-circt:
+
+  # ---------------------------------------------------------------------------
+  #  Build and test Linux wheels. Run the CIRCT tests also.
+  # ---------------------------------------------------------------------------
+
+  build-linux:
     name: Build
-    runs-on: ubuntu-latest
+    # Run on an internal MSFT subscription. Please DO NOT use this for any other
+    # workflows without talking to John Demme ([email protected], GH
+    # teqdruid) first. We may lose funding for this if it ends up costing too
+    # much.
+    # If individual jobs fail due to timeouts or disconnects, please report to
+    # John and re-run the job.
+    runs-on:
+      - self-hosted
+      - 1ES.Pool=1ES-CIRCT-builds
+      - 1ES.ImageOverride=CIRCT-StdUbuntu2204
     strategy:
-      # Keep the 'matrix' strategy with one data point to make it obvious that
-      # this is one point in the overall matrix.
       matrix:
         python-env:
-          - cp38-win_amd64
+          - cp38-manylinux_x86_64
+          - cp39-manylinux_x86_64
+          - cp310-manylinux_x86_64
+          - cp311-manylinux_x86_64
+          - cp312-manylinux_x86_64
     steps:
-      # Clone the CIRCT repo and its submodules. Do shallow clone to save clone
-      # time.
-      - name: Get CIRCT
-        uses: actions/checkout@v3
+      - name: Get CIRCT (no submodules)
+        uses: actions/checkout@v4
         with:
-          submodules: true
+          fetch-depth: 0
+          fetch-tags: true
+          submodules: false
+
+      - name: Get shallow LLVM submodule
+        run: |
+          git submodule update --init --recursive --recommend-shallow --depth 1
 
-      # --------
-      # Build and test CIRCT
-      # --------
       - name: Install dependencies
         run: |
           set -o errexit
-          sudo apt install -y python3-pip
           python3 -m pip install --upgrade pip
           python3 -m pip install cibuildwheel twine
+
+      # Build CIRCT and run its tests using a Docker container with all the
+      # integration testing prerequisite installed.
       - name: Build wheel
         env:
-          CC: clang
-          CXX: clang++
           CIBW_BUILD: ${{ matrix.python-env }}
-          CIBW_ENVIRONMENT: CMAKE_GENERATOR=Ninja
+          CMAKE_GENERATOR: Ninja
           SETUPTOOLS_SCM_DEBUG: True
-          BUILD_TYPE: RelWithAsserts
+          BUILD_TYPE: Release
           RUN_TESTS: True
         run: |
+          set -o errexit
           export PATH=$PATH:$HOME/.local/bin
-          cd $(Pipeline.Workspace)/circt
-
-          echo "Fetching history and tags from CIRCT repo"
-          git fetch --depth=1000000 --tags --no-recurse-submodules
 
           echo "Building wheel"
           cibuildwheel --output-dir wheelhouse frontends/PyCDE
-      - name: Upload Binary (Non-Tag)
-        uses: actions/upload-artifact@v3
+
+      - name: Get wheel name
+        shell: bash
+        id: whl-name
+        run: |
+          cd wheelhouse
+          echo WHL=`ls *.whl` >> "$GITHUB_OUTPUT"
+
+      - name: Upload wheels as artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.whl-name.outputs.WHL }}
+          path: wheelhouse/${{ steps.whl-name.outputs.WHL }}
+          retention-days: 7
+          if-no-files-found: error
+
+
+  # ---------------------------------------------------------------------------
+  #  Build and test Windows wheels.
+  # ---------------------------------------------------------------------------
+
+  build-windows:
+    name: Build
+    # Run on an internal MSFT subscription. Please DO NOT use this for any other
+    # workflows without talking to John Demme ([email protected], GH
+    # teqdruid) first. We may lose funding for this if it ends up costing too
+    # much.
+    # If individual jobs fail due to timeouts or disconnects, please report to
+    # John and re-run the job.
+    runs-on:
+      - self-hosted
+      - 1ES.Pool=1ES-CIRCT-builds
+      - 1ES.ImageOverride=1esMMSWindows2022
+    strategy:
+      matrix:
+        python-env:
+          - cp38-win_amd64
+          - cp39-win_amd64
+          - cp310-win_amd64
+          - cp311-win_amd64
+          - cp312-win_amd64
+    steps:
+      # Since we don't use docker on Windows, we need to install the dependencies.
+      - name: Build additional c++ deps
+        shell: pwsh
+        run: |
+          & "${env:VCPKG_INSTALLATION_ROOT}/vcpkg" --triplet x64-windows install zlib capnproto
+
+      - name: Get CIRCT
+        uses: actions/checkout@v4
         with:
-          path: wheelhouse/*.whl
+          fetch-depth: 0
+          fetch-tags: true
+          submodules: false
+
+      - name: Get shallow LLVM submodule
+        run: |
+          git submodule update --init --recursive --recommend-shallow --depth 1
+
+      - name: Install dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install cibuildwheel twine
+          choco install ninja
+
+      - name: Build wheel
+        shell: cmd
+        env:
+          CIBW_BUILD: ${{ matrix.python-env }}
+          CMAKE_GENERATOR: Ninja
+          SETUPTOOLS_SCM_DEBUG: True
+          BUILD_TYPE: Release
+          # PyCDE integration tests are not yet implemented on Windows.
+          RUN_TESTS: False
+        run: |
+          echo "Building wheel"
+          "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=x64 -host_arch=x64 && python3 -m cibuildwheel --output-dir wheelhouse frontends/PyCDE
+
+      - name: Get wheel name
+        shell: bash
+        id: whl-name
+        run: |
+          cd wheelhouse
+          echo WHL=`ls *.whl` >> "$GITHUB_OUTPUT"
+
+      - name: Upload wheels as artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.whl-name.outputs.WHL }}
+          path: wheelhouse/${{ steps.whl-name.outputs.WHL }}
           retention-days: 7
+          if-no-files-found: error
+
+
+  # ---------------------------------------------------------------------------
+  #  If both the Linux and Windows builds are successful, push the wheels to
+  #  pypi. Only do this on the main branch or a tag.
+  # ---------------------------------------------------------------------------
+
+  push_wheels:
+    name: Push wheels (Tag or Weekly)
+    runs-on: ubuntu-20.04
+    if: github.repository == 'llvm/circt' && ( github.ref == 'refs/heads/main' || github.ref_type == 'tag' )
+    needs:
+      - build-linux
+      - build-windows
+    environment:
+      name: pypi
+      url: https://pypi.org/p/esiaccel
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download wheels
+        uses: actions/download-artifact@v4
+        with:
+          path: ./wheelhouse/
+          merge-multiple: true
+
+      - name: List downloaded wheels
+        run: ls -laR
+        working-directory: ./wheelhouse/
+
+      - name: Upload wheels to pypi
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: wheelhouse/
+          verify-metadata: false

frontends/PyCDE/pyproject.toml

@@ -31,6 +31,16 @@ manylinux-x86_64-image = "ghcr.io/circt/images/pycde-build"
 [tool.cibuildwheel.linux]
 # Use our internal auditwheel script so as to not mess up the collateral.
 repair-wheel-command = "frontends/PyCDE/auditwheel.sh {dest_dir} {wheel}"
+environment-pass = [
+  "SCCACHE_GHA_ENABLED",
+  "ACTIONS_CACHE_URL",
+  "ACTIONS_RUNTIME_TOKEN",
+  "CMAKE_GENERATOR",
+  "SETUPTOOLS_SCM_DEBUG",
+  "BUILD_TYPE",
+  "RUN_TESTS",
+  "COMPILER_LAUNCHER"
+]
 
 [project]
 name = "pycde"

frontends/PyCDE/setup.py

@@ -45,7 +45,6 @@ class CMakeBuild(build_py):
 
   def run(self):
     target_dir = self.build_lib
-    test = os.getenv("RUN_TESTS")
     cmake_build_dir = os.getenv("PYCDE_CMAKE_BUILD_DIR")
     if not cmake_build_dir:
       cmake_build_dir = os.path.join(target_dir, "..", "cmake_build")
@@ -54,6 +53,8 @@ def run(self):
         os.environ.get("CIRCT_DIRECTORY", os.path.join(_thisdir, "..", "..")))
     src_dir = os.path.abspath(os.path.join(circt_dir, "llvm", "llvm"))
     cfg = "Release"
+    if "BUILD_TYPE" in os.environ:
+      cfg = os.environ["BUILD_TYPE"]
     cmake_args = [
         "-DCMAKE_INSTALL_PREFIX={}".format(os.path.abspath(cmake_install_dir)),
         "-DPython3_EXECUTABLE={}".format(sys.executable.replace("\\", "/")),
@@ -66,9 +67,23 @@ def run(self):
         "-DCIRCT_ENABLE_FRONTENDS=PyCDE",
         "-DLLVM_EXTERNAL_PROJECTS=circt",
         "-DLLVM_EXTERNAL_CIRCT_SOURCE_DIR={}".format(circt_dir),
+        "-DESI_RUNTIME=ON",
     ]
+    if "COMPILER_LAUNCHER" in os.environ:
+      cmake_args += [
+          f"-DCMAKE_C_COMPILER_LAUNCHER={os.environ['COMPILER_LAUNCHER']}",
+          f"-DCMAKE_CXX_COMPILER_LAUNCHER={os.environ['COMPILER_LAUNCHER']}"
+      ]
+    if "CC" in os.environ:
+      cmake_args += [f"-DCMAKE_C_COMPILER={os.environ['CC']}"]
+    if "CXX" in os.environ:
+      cmake_args += [f"-DCMAKE_CXX_COMPILER={os.environ['CXX']}"]
     if "CIRCT_EXTRA_CMAKE_ARGS" in os.environ:
       cmake_args += os.environ["CIRCT_EXTRA_CMAKE_ARGS"].split(" ")
+    if "VCPKG_INSTALLATION_ROOT" in os.environ:
+      cmake_args += [
+          f"-DCMAKE_TOOLCHAIN_FILE={os.environ['VCPKG_INSTALLATION_ROOT']}/scripts/buildsystems/vcpkg.cmake"
+      ]
     build_args = []
     build_parallelism = os.getenv("CMAKE_PARALLELISM")
     if build_parallelism:
@@ -81,25 +96,22 @@ def run(self):
     cmake_cache_file = os.path.join(cmake_build_dir, "CMakeCache.txt")
     if os.path.exists(cmake_cache_file):
       os.remove(cmake_cache_file)
+    print(f"Running cmake with args: {cmake_args}", file=sys.stderr)
+    subprocess.check_call(["echo", "Running: cmake", src_dir] + cmake_args)
     subprocess.check_call(["cmake", src_dir] + cmake_args, cwd=cmake_build_dir)
+    targets = ["check-pycde"]
+    if "RUN_TESTS" in os.environ and os.environ["RUN_TESTS"] != "false":
+      # The pycde integration tests test both PyCDE and the ESIRuntime so
+      # failure shouldn't gate publishing PyCDE.
+      # targets.append("check-pycde-integration")
+      targets.append("check-circt")
     subprocess.check_call([
         "cmake",
         "--build",
         ".",
         "--target",
-        "check-pycde",
-    ] + build_args,
+    ] + targets + build_args,
                           cwd=cmake_build_dir)
-    if test.lower() == "true":
-      subprocess.check_call([
-          "cmake",
-          "--build",
-          ".",
-          "--target",
-          "check-pycde-integration",
-          "check-circt",
-      ] + build_args,
-                            cwd=cmake_build_dir)
     install_cmd = ["cmake", "--build", ".", "--target", "install-PyCDE"]
     subprocess.check_call(install_cmd + build_args, cwd=cmake_build_dir)
     shutil.copytree(os.path.join(cmake_install_dir, "python_packages"),

lib/Dialect/ESI/runtime/CMakeLists.txt

@@ -241,7 +241,9 @@ if (WHEEL_BUILD)
 endif()
 
 # Pybind11 is used to wrap the ESIRuntime APIs.
-find_package(Python3 COMPONENTS Interpreter Development)
+if(NOT DEFINED Python3_FOUND)
+  find_package(Python3 COMPONENTS Interpreter Development)
+endif()
 if(Python3_FOUND)
   IF(MSVC)
     # Work around an issue with pybind11 and cmake incompatibility on Windows in debug mode.