Add external vehicle load from matrices in a performant way

[EwoutH]

I have to matrices with vehicle loads, od_ext_into_city and od_ext_out_city which contain OD-indexed vehicle loads for respectively traffic into the city and traffic out to the city. The shapes are (13, 21) and (21, 13) respectively, so 273 areas in total. Here's an example:

	14	32	35	37	38	39	40	53	54	55	56	57	58
1	69	431	1002	236	161	809	427	663	347	705	473	422	777
2	70	645	899	365	187	636	478	670	316	658	489	380	593
3	46	291	1406	239	97	294	460	687	233	434	356	278	309

That's the traffic for the full between each of those areas, I have a distribution trips_by_hour_chance which distributes that over the day, so that it's busier in the rush hours.

node_mrdh65_dict contains which nodes are in which area.

So currently I have the following loop to assign demand to these areas for each hour:

self.mrdh65s = list(set([a.mrdh65 for a in self.agents]))
self.mrdh65s_ext = data.od_ext_into_city.index.to_list()

# Convert to NumPy, int16
self.od_ext_into_city = data.od_ext_into_city * self.ext_vehicle_load / 10  # UXsim platoon size
self.od_ext_out_city = data.od_ext_out_city * self.ext_vehicle_load / 10

for hour in range(self.start_time, self.end_time):
    # Calculate the start and end times for this hour
    sim_hour = hour - self.start_time
    start_time = sim_hour * 3600
    end_time = (sim_hour + 1) * 3600

    # Get the trip multiplier for this hour
    hour_multiplier = self.trips_by_hour_chance[hour]

    for ext_area in self.mrdh65s_ext:
        for int_area in self.mrdh65s:
            volume_in = round(self.od_ext_into_city[int_area][ext_area] * hour_multiplier)
            volume_out = round(self.od_ext_out_city[ext_area][int_area] * hour_multiplier)

            print(f"Hour {hour}, ext {ext_area}, int {int_area}: in {volume_in}, out {volume_out}")

            ext_nodes = self.uw.node_mrdh65_dict[ext_area]
            int_nodes = self.uw.node_mrdh65_dict[int_area]

            # Add trips into the city
            self.uw.adddemand_nodes2nodes(
                origs=ext_nodes,
                dests=int_nodes,
                t_start=start_time,
                t_end=end_time,
                volume=volume_in
            )

            # Add trips out of the city
            self.uw.adddemand_nodes2nodes(
                origs=int_nodes,
                dests=ext_nodes,
                t_start=start_time,
                t_end=end_time,
                volume=volume_out
            )

However, this is quite slow. I'm calling adddemand_nodes2nodes for 273 OD-pairs, for 16 hours, twice, or a total of 8736 times. This takes a few minutes, which is about the same total time of the full simulation.

A profile looks like this:

What would be a faster way to do this?

CC @toruseo

[EwoutH]

I decided to skip adddemand_nodes2nodes and adddemand both, and just directly use addVehicle.

diff --git a/model/model.py b/model/model.py
index c223782..26fc7ab 100644
--- a/model/model.py
+++ b/model/model.py
@@ -119,14 +119,16 @@ def __init__(self, n_agents=simulated_population, step_time=1/12, start_time=6,
         # External vehicle load
         # Get a list of origin and destination areas for the external trips
         def add_external_vehicle_load():
             self.mrdh65s_ext = data.od_ext_into_city.index.to_list()
 
             # Convert to NumPy, int16
-            self.od_ext_into_city = data.od_ext_into_city * self.ext_vehicle_load
-            self.od_ext_out_city = data.od_ext_out_city * self.ext_vehicle_load
+            self.od_ext_into_city = data.od_ext_into_city * self.ext_vehicle_load / 10  # UXsim platoon size
+            self.od_ext_out_city = data.od_ext_out_city * self.ext_vehicle_load / 10
 
             for hour in range(self.start_time, self.end_time):
                 print(f"Hour {hour}")

                 # Calculate the start and end times for this hour
                 sim_hour = hour - self.start_time
                 start_time = sim_hour * 3600
@@ -145,24 +147,21 @@ def add_external_vehicle_load():
                         ext_nodes = self.uw.node_mrdh65_dict[ext_area]
                         int_nodes = self.uw.node_mrdh65_dict[int_area]
 
-                        # Add trips into the city
-                        # TODO: Double check if this is per platoon or per vehicle
-                        self.uw.adddemand_nodes2nodes(
-                            origs=ext_nodes,
-                            dests=int_nodes,
-                            t_start=start_time,
-                            t_end=end_time,
-                            volume=volume_in
-                        )
-
-                        # Add trips out of the city
-                        self.uw.adddemand_nodes2nodes(
-                            origs=int_nodes,
-                            dests=ext_nodes,
-                            t_start=start_time,
-                            t_end=end_time,
-                            volume=volume_out
-                        )
+                        def add_vehicle_load(volume, orig_nodes, dest_nodes):
+                            times = np.random.uniform(start_time, end_time, volume)
+                            os, ds = self.random.choices(orig_nodes, k=volume), self.random.choices(dest_nodes, k=volume)
+                            for time, o, d in zip(times, os, ds):
+                                self.uw.addVehicle(orig=o, dest=d, departure_time=time)
+
+                        if volume_in > 0:
+                            add_vehicle_load(volume_in, ext_nodes, int_nodes)
+                        if volume_out > 0:
+                            add_vehicle_load(volume_out, int_nodes, ext_nodes)

         if self.ext_vehicle_load:
             add_external_vehicle_load()

It's way faster, so it solves this problem for me.

Full new code:

        # External vehicle load
        # Get a list of origin and destination areas for the external trips
        def add_external_vehicle_load():
            print(f"Adding external vehicles to the simulation.")
            ext_vehicles = 0
            self.mrdh65s_ext = data.od_ext_into_city.index.to_list()

            # Convert to NumPy, int16
            self.od_ext_into_city = data.od_ext_into_city * self.ext_vehicle_load / self.uxsim_platoon_size
            self.od_ext_out_city = data.od_ext_out_city * self.ext_vehicle_load / self.uxsim_platoon_size

            for hour in range(self.start_time, self.end_time):

                # Calculate the start and end times for this hour
                sim_hour = hour - self.start_time
                start_time = sim_hour * 3600
                end_time = (sim_hour + 1) * 3600

                # Get the trip multiplier for this hour
                hour_multiplier = self.trips_by_hour_chance[hour]

                for ext_area in self.mrdh65s_ext:
                    for int_area in self.mrdh65s:
                        volume_in = round(self.od_ext_into_city[int_area][ext_area] * hour_multiplier)
                        volume_out = round(self.od_ext_out_city[ext_area][int_area] * hour_multiplier)

                        # print(f"Hour {hour}, ext {ext_area}, int {int_area}: in {volume_in}, out {volume_out}")

                        ext_nodes = self.uw.node_mrdh65_dict[ext_area]
                        int_nodes = self.uw.node_mrdh65_dict[int_area]

                        def add_vehicle_load(volume, orig_nodes, dest_nodes):
                            times = np.random.uniform(start_time, end_time, volume)
                            os, ds = self.random.choices(orig_nodes, k=volume), self.random.choices(dest_nodes, k=volume)
                            for time, o, d in zip(times, os, ds):
                                self.uw.addVehicle(orig=o, dest=d, departure_time=time)

                        if volume_in > 0:
                            add_vehicle_load(volume_in, ext_nodes, int_nodes)
                        if volume_out > 0:
                            add_vehicle_load(volume_out, int_nodes, ext_nodes)

                        ext_vehicles += volume_in + volume_out

            print(f"Added {ext_vehicles} external vehicles to the simulation.")

        if self.ext_vehicle_load:
            add_external_vehicle_load()

[toruseo]

This approach with zip is a good solution. I will implement it into the default functions. Thanks!