Switch to pants as build system (#199)

.buildkite/install-repo.sh

@@ -7,10 +7,4 @@ gcloud config set account [email protected]
 
 echo --- Installing dependencies
 
-eval "$(pyenv init --path)"
-eval "$(pyenv init -)"
-
-${PDM_COMMAND:1:-1} use ${PYTHON_VERSION:1:-1}
-${PDM_COMMAND:1:-1} install --plugins
-${PDM_COMMAND:1:-1} install -d -G ci
-${PDM_COMMAND:1:-1} torch install cpu
+./get-pants.sh

.buildkite/pipeline.yml

@@ -1,95 +1,86 @@
-plugin_base: &plugin_base
-    service-account-name: monorepo-ci
-    image: gcr.io/embark-shared/ml/ci-runner@sha256:dac3595ade7e3e92ed006f6c29f461b71bb3a6b0ade8d3afb88ba8e55b9601d6
-    default-secret-name: buildkite-k8s-plugin
-    always-pull: false
-    use-agent-node-affinity: true
-
-agents: &agent
-  cluster: builds-fi-2
-  queue: monorepo-ci
-  size: small
-
-tiny: &tiny
-  agents: *agent
-  plugins:
-  - EmbarkStudios/k8s#1.2.10:
-      << : *plugin_base
-      resources-limit-cpu: 3
-      resources-limit-memory: 10Gi
-
-  agents: *agent
-
-small: &small
-  agents: *agent
-  plugins:
-  - EmbarkStudios/k8s#1.2.10:
-      << : *plugin_base
-      resources-limit-cpu: 7
-      resources-limit-memory: 20Gi
-
-large: &large
-  agents: *agent
-  plugins:
-  - EmbarkStudios/k8s#1.2.10:
-      << : *plugin_base
-      resources-limit-cpu: 14
-      resources-limit-memory: 35Gi
-
+agent_kinds:
+  large_config: &large
+    agents:
+      cluster: builds-fi-2
+      queue: monorepo-ci
+      size: large
+    env:
+      PANTS_PROCESS_EXECUTION_LOCAL_PARALLELISM: 12
+      PANTS_RULE_THREADS_CORE: 6
+
+  small_config: &small
+    agents:
+      cluster: builds-fi-2
+      queue: monorepo-ci
+      size: small
+    env:
+      PANTS_PROCESS_EXECUTION_LOCAL_PARALLELISM: 6
+      PANTS_RULE_THREADS_CORE: 3
 
 env:
-  PDM_COMMAND: pdm210
   PYTHON_VERSION: '3.10'
-
+  PY: "python3.10"
+  PANTS_PYTHON_INTERPRETER_CONSTRAINTS: "['==3.10.*']"
+  PANTS_CONFIG_FILES: "pants.ci.toml"
+  PYTHON_BIN_NAME: "python3.10"
+  PEX_PYTHON: "python3.10"
 
 steps:
   - group: ":passport_control: Validating PR"
     steps:
       - label: ":hourglass: Validating branch age"
         command: bash .buildkite/validate-branch-age.sh
-        << : *tiny
+        <<: *small
 
       - label: ":straight_ruler: Checking line-endings"
         command: bash .buildkite/check-line-endings.sh
-        << : *tiny
-
-      - label: ":lock: Checking lockfile"
-        command: bash .buildkite/validate-lockfile.sh
-        << : *tiny
+        <<: *small
 
   - wait
 
   - group: ":vertical_traffic_light: Validating changes"
     steps:
-      - label: 📚 Publish docs
-        command: bash .buildkite/publish-docs.sh
-        << : *tiny
+      - label: ":package: Validating {{matrix}} packages"
+        command: |
+          . ci/init.sh
+          pants package --filter-address-regex='.*@resolve={{matrix}}' emote::
+
+        matrix:
+          - "cpu"
+          - "gpu"
+          - "base"
+
+        <<: *large
 
-      - label: ":python-black: Validate black"
-        command: bash .buildkite/run-black.sh
-        << : *tiny
+      - label: ":package: Packaging bases"
+        command: |
+          . ci/init.sh
+          pants package --filter-address-regex='-.*@resolve=.*' emote::
 
-      - label: ":isort: Validate isort"
-        command: bash .buildkite/run-isort.sh
-        << : *tiny
+        <<: *large
 
-      - label: ":bandit: Validate bandit"
-        command: bash .buildkite/run-bandit.sh
-        << : *tiny
+      - label: ":pants: Validating tailor"
+        command: |
+          . ci/init.sh
+          pants tailor --check ::
 
-      - label: ":lint-roller: Validate flake8"
-        command: bash .buildkite/run-flake8.sh
-        << : *tiny
+        <<: *small
+
+      - label: ":python-black: :isort: Check-and-lint "
+        command: |
+          . ci/init.sh
+          pants update-build-files --check lint ::
+
+        <<: *small
 
       - label: ":pytest: Run tests"
-        command: bash .buildkite/run-pytest.sh
-        << : *large
+        command: |
+          . ci/init.sh
+          pants test ::
 
-  - wait
+        <<: *small
 
-  - label: ":package: Validate packaging"
-    command: bash .buildkite/run-package.sh
-    << : *tiny
+  - wait
 
   - label: ":packagecloud: Triggering Cloud-Training build"
     trigger: erupt

.buildkite/publish-docs.sh

@@ -4,24 +4,17 @@ set -eo pipefail
 source .buildkite/install-repo.sh
 
 echo --- Building docs
-pushd docs
-EXIT_CODE=0
 
-export TZ=UTC
-
-PDM=${PDM_COMMAND:1:-1} ${PDM_COMMAND:1:-1} run make deploy  || EXIT_CODE=$?
-
-if [ $EXIT_CODE -ne 0 ]; then
-	cat << EOF | buildkite-agent annotate --style "error" --context "sphinx"
-:warning: Failed building documentation. Please check logs below, or build docs locally using `make deploy` to check for errors.
+if ! pants package //docs:book; then
+	cat << EOF | buildkite-agent annotate --style "error" --context "pants"
+:warning: Failed building documentation. Please check logs below, or build docs locally using \`pants package //docs:book\` to check for errors.
 EOF
 	exit 1
 else
 	if [[ "$BUILDKITE_BRANCH" = "main" ]]; then
-		gsutil rsync -r ./_build/dirhtml gs://embark-static/emote-docs
-		buildkite-agent annotate "✅ New documentation deployed at https://static.embark.net/emote-docs/" --style "success" --context "sphinx"
+		gsutil rsync -r ./dist/docs/book gs://embark-static/emote-docs
+		buildkite-agent annotate "✅ New documentation deployed at https://static.embark.net/emote-docs/" --style "success" --context "pants"
 	else
-		buildkite-agent annotate "✅ Documentation built succesfully" --style "success" --context "sphinx"
+		buildkite-agent annotate "✅ Documentation built succesfully" --style "success" --context "pants"
 	fi
 fi
-popd

.github/workflows/build-docs.yml

@@ -0,0 +1,19 @@
+name: Build docs
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+env:
+  PANTS_BACKEND_PACKAGES: '-["pants.backend.python.providers.experimental.pyenv"]'
+  PANTS_VERIFY_CONFIG: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Generate docs
+        uses: ./.github/workflows/docs