From 18521b0ec390c8eda49f3d8913023c75866af06e Mon Sep 17 00:00:00 2001 From: Armando Banuelos Date: Fri, 12 Apr 2024 12:14:08 -0700 Subject: [PATCH] fix: add env variable for CARLA path and blueprint tests with manual skip --- pyproject.toml | 4 +- tests/conftest.py | 11 +- tests/simulators/carla/test_carla.py | 355 ++++++++++++++++++++++++++- 3 files changed, 365 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a9a356594..0f49cff75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,7 @@ test-full = [ # like 'test' but adds dependencies for optional features "scenic[test]", # all dependencies from 'test' extra above "scenic[guideways]", # for running guideways modules "astor >= 0.8.1", + "carla >= 0.9.12", "dill", "exceptiongroup", "inflect ~= 5.5", @@ -73,9 +74,6 @@ test-full = [ # like 'test' but adds dependencies for optional features "sphinx-tabs ~= 3.4.1", "verifai >= 2.1.0b1", ] -test-sim = [ - 'carla >= 0.9.12; python_version <= "3.8" and (platform_system == "Linux" or platform_system == "Windows")', -] dev = [ "scenic[test-full]", "black ~= 24.0", diff --git a/tests/conftest.py b/tests/conftest.py index 81c03155e..c9d9f04b7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ from contextlib import contextmanager +import os import os.path from pathlib import Path import re @@ -53,10 +54,18 @@ def loader(relpath, **kwargs): return loader +def checkCarlaPath(): + CARLA_ROOT = os.environment.get("CARLA_ROOT") + if not CARLA_ROOT: + pytest.skip("CARLA_ROOT env variable not set.") + return CARLA_ROOT + + @pytest.fixture def launchCarlaServer(): + CARLA_ROOT = checkCarlaPath() carla_process = subprocess.Popen( - "bash /opt/carla-simulator/CarlaUE4.sh -RenderOffScreen", shell=True + f"bash {CARLA_ROOT}/CarlaUE4.sh -RenderOffScreen", shell=True ) # NOTE: CARLA server takes time to start up time.sleep(3) diff --git a/tests/simulators/carla/test_carla.py b/tests/simulators/carla/test_carla.py index c88c19e1b..1fdf6c410 100644 --- a/tests/simulators/carla/test_carla.py +++ b/tests/simulators/carla/test_carla.py @@ -7,7 +7,355 @@ "ignore::scenic.core.simulators.SimulatorInterfaceWarning" ) -pytest.importorskip("carla") + +def preprocess_old_blueprint_names(original): + d = {} + for key, value in original.items(): + for sub_value in value: + d[sub_value] = key + return d + + +def model_blueprint(simulator, mapPath, town, modelType, modelName, newModelNames=None): + code = f""" + param map = r'{mapPath}' + param carla_map = '{town}' + param time_step = 1.0/10 + + model scenic.simulators.carla.model + ego = new {modelType} with blueprint '{modelName}' + terminate after 1 steps + """ + scenario = compileScenic(code, mode2D=True) + scene, _ = scenario.generate(maxIterations=10000) + simulation = simulator.simulate(scene) + obj = simulation.objects[0] + if newModelNames: + assert obj.blueprint == newModelNames[modelName] + else: + assert obj.blueprint == modelName + + +from scenic.simulators.carla.blueprints import carModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", carModels) +def test_car_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "Car", modelName) + + +from scenic.simulators.carla.blueprints import oldBlueprintNames + +oldToNew = preprocess_old_blueprint_names(oldBlueprintNames) +oldModels = oldToNew.keys() + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", oldModels) +def test_old_blue_prints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "Car", modelName, oldToNew) + + +from scenic.simulators.carla.blueprints import bicycleModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", bicycleModels) +def test_bicycle_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "Bicycle", modelName) + + +from scenic.simulators.carla.blueprints import motorcycleModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", motorcycleModels) +def test_motorcycle_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "Motorcycle", modelName) + + +from scenic.simulators.carla.blueprints import truckModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", truckModels) +def test_truck_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "Truck", modelName) + + +from scenic.simulators.carla.blueprints import trashModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", trashModels) +def test_trash_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "Trash", modelName) + + +from scenic.simulators.carla.blueprints import coneModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", coneModels) +def test_cone_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "Cone", modelName) + + +from scenic.simulators.carla.blueprints import debrisModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", debrisModels) +def test_debris_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "Debris", modelName) + + +from scenic.simulators.carla.blueprints import vendingMachineModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", vendingMachineModels) +def test_vending_machine_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "VendingMachine", modelName) + + +from scenic.simulators.carla.blueprints import chairModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", chairModels) +def test_chair_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "Chair", modelName) + + +from scenic.simulators.carla.blueprints import busStopModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", busStopModels) +def test_bus_stop_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "BusStop", modelName) + + +from scenic.simulators.carla.blueprints import advertisementModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", advertisementModels) +def test_advertisement_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "Advertisement", modelName) + + +from scenic.simulators.carla.blueprints import garbageModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", garbageModels) +def test_garbage_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "Garbage", modelName) + + +from scenic.simulators.carla.blueprints import containerModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", containerModels) +def test_container_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "Container", modelName) + + +from scenic.simulators.carla.blueprints import tableModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", tableModels) +def test_table_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "Table", modelName) + + +from scenic.simulators.carla.blueprints import barrierModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", barrierModels) +def test_barrier_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "Barrier", modelName) + + +from scenic.simulators.carla.blueprints import plantpotModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", plantpotModels) +def test_plant_pots_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "PlantPot", modelName) + + +from scenic.simulators.carla.blueprints import mailboxModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", mailboxModels) +def test_mailbox_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "Mailbox", modelName) + + +from scenic.simulators.carla.blueprints import gnomeModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", gnomeModels) +def test_gnome_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "Gnome", modelName) + + +from scenic.simulators.carla.blueprints import creasedboxModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", creasedboxModels) +def test_creased_box_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "CreasedBox", modelName) + + +from scenic.simulators.carla.blueprints import caseModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", caseModels) +def test_case_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "Case", modelName) + + +from scenic.simulators.carla.blueprints import boxModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", boxModels) +def test_box_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "Box", modelName) + + +from scenic.simulators.carla.blueprints import benchModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", benchModels) +def test_bench_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "Bench", modelName) + + +from scenic.simulators.carla.blueprints import barrelModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", barrelModels) +def test_barrel_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "Barrel", modelName) + + +from scenic.simulators.carla.blueprints import atmModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", atmModels) +def test_atm_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "ATM", modelName) + + +from scenic.simulators.carla.blueprints import kioskModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", kioskModels) +def test_kiosk_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "Kiosk", modelName) + + +from scenic.simulators.carla.blueprints import ironplateModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", ironplateModels) +def test_iron_plate_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "IronPlate", modelName) + + +from scenic.simulators.carla.blueprints import trafficwarningModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", trafficwarningModels) +def test_traffic_warning_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "TrafficWarning", modelName) + + +from scenic.simulators.carla.blueprints import walkerModels + + +@pytest.mark.skip(reason="Skipping test due to Carla memory leak issues") +@pytest.mark.parametrize("modelName", walkerModels) +def test_pedestrian_blueprints(getCarlaSimulator, launchCarlaServer, modelName): + pytest.importorskip("carla") + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, "Pedestrian", modelName) def test_throttle(getCarlaSimulator, launchCarlaServer): @@ -64,6 +412,11 @@ def test_brake(getCarlaSimulator, launchCarlaServer): assert int(records[-1][1]) < threshold +def test_missing_carla_root_env(launchCarlaServer): + with pytest.raises(pytest.skip.Exception): + assert True + + def test_basic(loadLocalScenario): scenario = loadLocalScenario("basic.scenic", mode2D=True) scenario.generate(maxIterations=1000)