diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 15e0ef4772b6..7b41d9581b20 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -233,7 +233,11 @@ jobs:
           key: ${{ steps.restore-vendor-bundle.outputs.cache-primary-key }}
       - name: Restore pre-build vendor/bundle to prevent 2nd build from scratch during push
         run: |
-          rm -rf vendor/bundle && mv vendor/bundle.bak vendor/bundle
+          rm -rf vendor/bundle
+
+          if [ -f vendor/bundle.bak ]; then
+            mv vendor/bundle.bak vendor/bundle
+          fi
       - name: Test
         # We only test the native container. If that fails the builds for the others
         # will be cancelled as well.