fix: readjusting nvidia-smi-test

.github/workflows/run-simulators.yml

@@ -83,13 +83,43 @@ jobs:
               rm -rf Scenic &&
               git clone --branch $(basename "${{ github.ref }}") --single-branch https://[email protected]/BerkeleyLearnVerify/Scenic.git &&
               cd Scenic &&
+              python3 -m venv venv &&
+              source venv/bin/activate &&
+              python3 -m pip install -e .[test-full] &&
               python3 .github/check_latest_simulators.py
             '
 
+    check_nvidia_smi:
+      name: check_nvidia_smi
+      runs-on: ubuntu-latest
+      needs: start_ec2_instance
+      continue-on-error: true
+      steps:
+        - name: Check NVIDIA SMI
+          env:
+            PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+            HOSTNAME: ${{ secrets.SSH_HOST}}
+            USER_NAME: ${{ secrets.SSH_USERNAME}}
+          run: |
+            echo "$PRIVATE_KEY" > private_key && chmod 600 private_key
+            ssh -o StrictHostKeyChecking=no -i private_key ${USER_NAME}@${HOSTNAME} '
+              output=$(nvidia-smi)
+              echo "$output"
+              if [ -z "$output" ]; then
+                echo "NVIDIA Driver is not set"
+                exit 1
+              fi
+            '  
+        - name: NVIDIA Driver is not set
+          if: ${{ failure() }}
+          run: |
+            echo "NVIDIA SMI is not working, please run the steps here on the instance:" 
+            echo "https://scenic-lang.atlassian.net/wiki/spaces/KAN/pages/2785287/Setting+Up+AWS+VM?parentProduct=JSW&initialAllowedFeatures=byline-contributors.byline-extensions.page-comments.delete.page-reactions.inline-comments.non-licensed-share&themeState=dark%253Adark%2520light%253Alight%2520spacing%253Aspacing%2520colorMode%253Alight&locale=en-US#Install-NVIDIA-Drivers"
+
     run_carla_simulators:
       name: run_carla_simulators
       runs-on: ubuntu-latest
-      needs: check_simulator_version_updates
+      needs: [check_simulator_version_updates, check_nvidia_smi]
       steps:
         - name: Run CARLA Tests
           env:
@@ -99,9 +129,8 @@ jobs:
           run: |
             echo "$PRIVATE_KEY" > private_key && chmod 600 private_key
             ssh -o StrictHostKeyChecking=no -i private_key ${USER_NAME}@${HOSTNAME} '
-              cd /home/ubuntu/actions &&
+              cd /home/ubuntu/actions/Scenic &&
               source venv/bin/activate &&
-              cd Scenic &&
               carla_versions=($(find /software -maxdepth 1 -type d -name 'carla*')) &&
               for version in "${carla_versions[@]}"; do
                 echo "============================= CARLA $version ============================="  
@@ -113,7 +142,7 @@ jobs:
     run_webots_simulators:
       name: run_webots_simulators
       runs-on: ubuntu-latest
-      needs: check_simulator_version_updates
+      needs: [check_simulator_version_updates, check_nvidia_smi]
       steps:
         - name: Run Webots Tests
           env:
@@ -124,9 +153,8 @@ jobs:
             echo "$PRIVATE_KEY" > private_key && chmod 600 private_key
             ssh -o StrictHostKeyChecking=no -i private_key ${USER_NAME}@${HOSTNAME} '
               Xvfb :99 -screen 0 1024x768x16 &
-              cd /home/ubuntu/actions &&
+              cd /home/ubuntu/actions/Scenic &&
               source venv/bin/activate &&
-              cd Scenic &&
               webots_versions=($(find /software -maxdepth 1 -type d -name 'webots*')) &&
               export DISPLAY=:99 &&
               for version in "${webots_versions[@]}"; do