run_simulators #1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: run_simulators | |
| on: | |
| # IMPORTANT: this workflow should only be triggered manually via the Actions | |
| # portal of the repo!!! Do not modify this workflow's trigger! | |
| workflow_dispatch: | |
| jobs: | |
| start_ec2_instance: | |
| name: start_ec2_instance | |
| runs-on: ubuntu-latest | |
| concurrency: | |
| group: sim | |
| steps: | |
| - name: Start EC2 Instance | |
| env: | |
| INSTANCE_ID: ${{ secrets.AWS_EC2_INSTANCE_ID }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} | |
| run: | | |
| # Get the instance state | |
| instance_state=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID | jq -r '.Reservations[].Instances[].State.Name') | |
| # If the machine is stopping wait for it to fully stop | |
| while [ "$instance_state" == "stopping" ]; do | |
| echo "Instance is stopping, waiting for it to fully stop..." | |
| sleep 10 | |
| instance_state=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID | jq -r '.Reservations[].Instances[].State.Name') | |
| done | |
| # Check if instance state is "stopped" | |
| if [[ "$instance_state" == "stopped" ]]; then | |
| echo "Instance is stopped, starting it..." | |
| aws ec2 start-instances --instance-ids $INSTANCE_ID | |
| elif [[ "$instance_state" == "pending" ]]; then | |
| echo "Instance startup is pending, continuing..." | |
| elif [[ "$instance_state" == "running" ]]; then | |
| echo "Instance is already running..." | |
| exit 0 | |
| else | |
| echo "Unknown instance state: $instance_state" | |
| exit 1 | |
| fi | |
| # wait for status checks to pass | |
| TIMEOUT=120 # Timeout in seconds | |
| START_TIME=$(date +%s) | |
| END_TIME=$((START_TIME + TIMEOUT)) | |
| while true; do | |
| response=$(aws ec2 describe-instance-status --instance-ids $INSTANCE_ID) | |
| system_status=$(echo "$response" | jq -r '.InstanceStatuses[0].SystemStatus.Status') | |
| instance_status=$(echo "$response" | jq -r '.InstanceStatuses[0].InstanceStatus.Status') | |
| if [[ "$system_status" == "ok" && "$instance_status" == "ok" ]]; then | |
| echo "Both SystemStatus and InstanceStatus are 'ok'" | |
| exit 0 | |
| fi | |
| CURRENT_TIME=$(date +%s) | |
| if [[ "$CURRENT_TIME" -ge "$END_TIME" ]]; then | |
| echo "Timeout: Both SystemStatus and InstanceStatus have not reached 'ok' state within $TIMEOUT seconds." | |
| exit 1 | |
| fi | |
| sleep 10 # Check status every 10 seconds | |
| done | |
| check_simulator_version_updates: | |
| name: check_simulator_version_updates | |
| runs-on: ubuntu-latest | |
| needs: start_ec2_instance | |
| steps: | |
| - name: Check for Simulator Version Updates | |
| env: | |
| PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} | |
| HOSTNAME: ${{ secrets.SSH_HOST }} | |
| USER_NAME: ${{ secrets.SSH_USERNAME }} | |
| GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} | |
| run: | | |
| echo "$PRIVATE_KEY" > private_key && chmod 600 private_key | |
| ssh -o StrictHostKeyChecking=no -i private_key ${USER_NAME}@${HOSTNAME} ' | |
| cd /home/ubuntu/actions/ && | |
| rm -rf Scenic && | |
| git clone --branch $(basename "${{ github.ref }}") --single-branch https://[email protected]/BerkeleyLearnVerify/Scenic.git && | |
| cd Scenic && | |
| python3 -m venv venv && | |
| source venv/bin/activate && | |
| python3 -m pip install -e .[test-full] && | |
| python3 .github/check_latest_simulators.py | |
| ' | |
| check_nvidia_smi: | |
| name: check_nvidia_smi | |
| runs-on: ubuntu-latest | |
| needs: start_ec2_instance | |
| continue-on-error: true | |
| steps: | |
| - name: Check NVIDIA SMI | |
| env: | |
| PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} | |
| HOSTNAME: ${{ secrets.SSH_HOST}} | |
| USER_NAME: ${{ secrets.SSH_USERNAME}} | |
| run: | | |
| echo "$PRIVATE_KEY" > private_key && chmod 600 private_key | |
| ssh -o StrictHostKeyChecking=no -i private_key ${USER_NAME}@${HOSTNAME} ' | |
| output=$(nvidia-smi) | |
| echo "$output" | |
| if [ -z "$output" ]; then | |
| echo "NVIDIA Driver is not set" | |
| exit 1 | |
| fi | |
| ' | |
| - name: NVIDIA Driver is not set | |
| if: ${{ failure() }} | |
| run: | | |
| echo "NVIDIA SMI is not working, please run the steps here on the instance:" | |
| echo "https://scenic-lang.atlassian.net/wiki/spaces/KAN/pages/2785287/Setting+Up+AWS+VM?parentProduct=JSW&initialAllowedFeatures=byline-contributors.byline-extensions.page-comments.delete.page-reactions.inline-comments.non-licensed-share&themeState=dark%253Adark%2520light%253Alight%2520spacing%253Aspacing%2520colorMode%253Alight&locale=en-US#Install-NVIDIA-Drivers" | |
| run_carla_simulators: | |
| name: run_carla_simulators | |
| runs-on: ubuntu-latest | |
| needs: [check_simulator_version_updates, check_nvidia_smi] | |
| steps: | |
| - name: Run CARLA Tests | |
| env: | |
| PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} | |
| HOSTNAME: ${{secrets.SSH_HOST}} | |
| USER_NAME: ${{secrets.SSH_USERNAME}} | |
| run: | | |
| echo "$PRIVATE_KEY" > private_key && chmod 600 private_key | |
| ssh -o StrictHostKeyChecking=no -i private_key ${USER_NAME}@${HOSTNAME} ' | |
| cd /home/ubuntu/actions/Scenic && | |
| source venv/bin/activate && | |
| carla_versions=($(find /software -maxdepth 1 -type d -name 'carla*')) && | |
| for version in "${carla_versions[@]}"; do | |
| echo "============================= CARLA $version =============================" | |
| export CARLA_ROOT="$version" | |
| pytest tests/simulators/carla/test_carla.py | |
| done | |
| ' | |
| run_webots_simulators: | |
| name: run_webots_simulators | |
| runs-on: ubuntu-latest | |
| needs: [check_simulator_version_updates, check_nvidia_smi] | |
| steps: | |
| - name: Run Webots Tests | |
| env: | |
| PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} | |
| HOSTNAME: ${{secrets.SSH_HOST}} | |
| USER_NAME: ${{secrets.SSH_USERNAME}} | |
| run: | | |
| echo "$PRIVATE_KEY" > private_key && chmod 600 private_key | |
| ssh -o StrictHostKeyChecking=no -i private_key ${USER_NAME}@${HOSTNAME} ' | |
| Xvfb :99 -screen 0 1024x768x16 & | |
| cd /home/ubuntu/actions/Scenic && | |
| source venv/bin/activate && | |
| webots_versions=($(find /software -maxdepth 1 -type d -name 'webots*')) && | |
| export DISPLAY=:99 && | |
| for version in "${webots_versions[@]}"; do | |
| echo "============================= Webots $version =============================" | |
| export WEBOTS_ROOT="$version" | |
| pytest tests/simulators/webots/test_webots.py | |
| done | |
| ' | |
| stop_ec2_instance: | |
| name: stop_ec2_instance | |
| runs-on: ubuntu-latest | |
| needs: [run_carla_simulators, run_webots_simulators] | |
| steps: | |
| - name: Stop EC2 Instance | |
| env: | |
| INSTANCE_ID: ${{ secrets.AWS_EC2_INSTANCE_ID }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} | |
| run: | | |
| # Get the instance state | |
| instance_state=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID | jq -r '.Reservations[].Instances[].State.Name') | |
| # If the machine is pending wait for it to fully start | |
| while [ "$instance_state" == "pending" ]; do | |
| echo "Instance is pending startup, waiting for it to fully start..." | |
| sleep 10 | |
| instance_state=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID | jq -r '.Reservations[].Instances[].State.Name') | |
| done | |
| # Check if instance state is "stopped" | |
| if [[ "$instance_state" == "running" ]]; then | |
| echo "Instance is running, stopping it..." | |
| aws ec2 stop-instances --instance-ids $INSTANCE_ID | |
| elif [[ "$instance_state" == "stopping" ]]; then | |
| echo "Instance is stopping..." | |
| elif [[ "$instance_state" == "stopped" ]]; then | |
| echo "Instance is already stopped..." | |
| exit 0 | |
| else | |
| echo "Unknown instance state: $instance_state" | |
| exit 1 | |
| fi |