diff --git a/.github/actions/setup-dependencies/action.yml b/.github/actions/setup-dependencies/action.yml
index e3b22d5dd9..588a4b7dc3 100644
--- a/.github/actions/setup-dependencies/action.yml
+++ b/.github/actions/setup-dependencies/action.yml
@@ -3,13 +3,18 @@ inputs:
     description: "Python version to setup"
     required: false
     default: "3.9"
+  install-test-deps:
+    description: "Wether to install 3rd Party dependencies (for tests)"
+    required: false
+    default: "true" # unfortunately boolean variables are not supported
 
 name: "Setup dependencies"
 description: "Install all required dependencies for worflows to run."
 runs:
   using: "composite"
   steps:
-    - name: Install 3rd party dependencies
+    - if: inputs.install-test-deps == 'true'
+      name: Install 3rd party dependencies
       run: sudo ./install-deps.sh
       shell: bash
 
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
index ac7bc2a463..dc0293ef71 100644
--- a/.github/workflows/CI.yml
+++ b/.github/workflows/CI.yml
@@ -39,6 +39,8 @@ jobs:
 
       - name: Setup 3rd party dependencies
         uses: ./.github/actions/setup-dependencies
+        with:
+          install-test-deps: "false"
 
       - name: Setup Nix
         uses: cachix/install-nix-action@v30
@@ -256,6 +258,8 @@ jobs:
 
       - name: Setup 3rd party dependencies
         uses: ./.github/actions/setup-dependencies
+        with:
+          install-test-deps: "false"
 
       - name: Build sdist
         uses: PyO3/maturin-action@v1