Fix more conflicts, all tests now pass

data_toolkit/common_methods.py

@@ -0,0 +1,38 @@
+# Copyright 2016-2025 Blue Marble Analytics LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+
+"""
+
+import os.path
+
+
+def create_csv_generic(
+    filename,
+    df,
+    overwrite,
+):
+    """ """
+    if not os.path.exists(filename) or overwrite:
+        df.to_csv(
+            filename,
+            mode="w",
+            index=False,
+        )
+    else:
+        raise ValueError(
+            f"The file {filename} already exists and overwrite has not been "
+            "indicated."
+        )

data_toolkit/project/create_monte_carlo_gen_input_csvs_common.py

@@ -16,6 +16,9 @@
 import os.path
 import pandas as pd
 
+from data_toolkit.project.opchar.var_profiles.common_methods import (
+    create_iterations_csv,
+)
 from db.common_functions import connect_to_database
 
 BINS_ID_DEFAULT = 1
@@ -201,6 +204,17 @@ def create_project_csv(
             index=False,
         )
 
+        # Add the iterations CSV
+        iterations_directory = os.path.join(output_directory, "iterations")
+        os.makedirs(iterations_directory, exist_ok=True)
+        create_iterations_csv(
+            iterations_directory=iterations_directory,
+            project=project,
+            profile_id=profile_scenario_id,
+            profile_name=profile_scenario_name,
+            overwrite=True,
+        )
+
 
 def create_project_csv_pool(pool_datum):
     [

data_toolkit/project/opchar/var_profiles/common_methods.py

@@ -0,0 +1,36 @@
+# Copyright 2016-2024 Blue Marble Analytics LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os.path
+import pandas as pd
+
+from data_toolkit.common_methods import create_csv_generic
+
+
+def create_iterations_csv(
+    iterations_directory, project, profile_id, profile_name, overwrite
+):
+    filename = os.path.join(
+        iterations_directory,
+        f"{project}-{profile_id}-{profile_name}.csv",
+    )
+
+    df = pd.DataFrame(
+        {
+            "varies_by_weather_iteration": [1],
+            "varies_by_hydro_iteration": [0],
+        }
+    )
+
+    create_csv_generic(filename=filename, df=df, overwrite=overwrite)

data_toolkit/project/opchar/var_profiles/create_sync_var_gen_input_csvs.py

@@ -50,6 +50,9 @@
 import os.path
 import sys
 
+from data_toolkit.project.opchar.var_profiles.common_methods import (
+    create_iterations_csv,
+)
 from db.common_functions import connect_to_database
 from data_toolkit.project.create_sync_gen_input_csvs_common import (
     create_profile_csvs,
@@ -138,6 +141,16 @@ def create_variable_profile_csvs_pool(pool_datum):
         raw_data_units_table_name="raw_data_var_project_units",
     )
 
+    iterations_directory = os.path.join(output_directory, "iterations")
+    os.makedirs(iterations_directory, exist_ok=True)
+    create_iterations_csv(
+        iterations_directory=iterations_directory,
+        project=project,
+        profile_id=variable_generator_profile_scenario_id,
+        profile_name=variable_generator_profile_scenario_name,
+        overwrite=True,
+    )
+
 
 def main(args=None):
     if args is None:

data_toolkit/system/common_methods.py

@@ -19,24 +19,7 @@
 import os.path
 import pandas as pd
 
-
-def create_csv_generic(
-    filename,
-    df,
-    overwrite,
-):
-    """ """
-    if not os.path.exists(filename) or overwrite:
-        df.to_csv(
-            filename,
-            mode="w",
-            index=False,
-        )
-    else:
-        raise ValueError(
-            f"The file {filename} already exists and overwrite has not been "
-            "indicated."
-        )
+from data_toolkit.common_methods import create_csv_generic
 
 
 def create_load_scenario_csv(

data_toolkit/system/create_monte_carlo_load_input_csvs.py

@@ -296,7 +296,9 @@ def create_load_levels_csv(
                 "load_levels",
                 f"{load_levels_scenario_id}_{load_levels_scenario_name}.csv",
             )
-            if not os.path.exists(filename) or (overwrite_load_levels_csv and draw_n == 0):
+            if not os.path.exists(filename) or (
+                overwrite_load_levels_csv and draw_n == 0
+            ):
                 mode = "w"
                 write_header = True
             else:

db/csvs_test_examples/project/project_availability/weather/iterations/Batteries-1-ra_toolkit.csv

@@ -1,2 +1,2 @@
-varies_by_weather_iteration,varies_by_hydro_iteration
-0,0
+varies_by_weather_iteration,varies_by_hydro_iteration
+1,0

db/csvs_test_examples/project/project_availability/weather/iterations/Gas-1-ra_toolkit.csv

@@ -1,2 +1,2 @@
-varies_by_weather_iteration,varies_by_hydro_iteration
-1,0
+varies_by_weather_iteration,varies_by_hydro_iteration
+1,0

db/csvs_test_examples/project/project_availability/weather/iterations/Hybrid-1-ra_toolkit.csv

@@ -1,2 +1,2 @@
-varies_by_weather_iteration,varies_by_hydro_iteration
-0,0
+varies_by_weather_iteration,varies_by_hydro_iteration
+1,0

db/csvs_test_examples/project/project_variable_generator_profiles/iterations/Hybrid-3-ra_toolkit.csv

@@ -1,2 +1,2 @@
-varies_by_weather_iteration,varies_by_hydro_iteration
-1,0
+varies_by_weather_iteration,varies_by_hydro_iteration
+1,0

db/csvs_test_examples/project/project_variable_generator_profiles/iterations/Solar-3-ra_toolkit.csv

@@ -1,2 +1,2 @@
-varies_by_weather_iteration,varies_by_hydro_iteration
-1,0
+varies_by_weather_iteration,varies_by_hydro_iteration
+1,0

db/csvs_test_examples/project/project_variable_generator_profiles/iterations/Solar_Zone1-1-open_data.csv

@@ -0,0 +1,2 @@
+varies_by_weather_iteration,varies_by_hydro_iteration
+1,0

db/csvs_test_examples/project/project_variable_generator_profiles/iterations/Solar_Zone2-1-open_data.csv

@@ -0,0 +1,2 @@
+varies_by_weather_iteration,varies_by_hydro_iteration
+1,0

db/csvs_test_examples/project/project_variable_generator_profiles/iterations/Wind-3-ra_toolkit.csv

@@ -1,2 +1,2 @@
-varies_by_weather_iteration,varies_by_hydro_iteration
-1,0
+varies_by_weather_iteration,varies_by_hydro_iteration
+1,0

db/csvs_test_examples/project/project_variable_generator_profiles/iterations/Wind_Zone1-1-open_data.csv

@@ -0,0 +1,2 @@
+varies_by_weather_iteration,varies_by_hydro_iteration
+1,0

db/csvs_test_examples/project/project_variable_generator_profiles/iterations/Wind_Zone2-1-open_data.csv

@@ -0,0 +1,2 @@
+varies_by_weather_iteration,varies_by_hydro_iteration
+1,0

examples/open_data/scenario_description.csv

@@ -1,4 +1,4 @@
-scenario_id,119
+scenario_id,138
 scenario_name,open_data
 of_transmission,1
 of_transmission_hurdle_rates,
@@ -26,8 +26,10 @@ of_deliverability,
 of_elcc_surface,
 of_local_capacity,
 of_markets,
+of_water,
 of_tuning,
-temporal_scenario_id,20
+of_policy,
+temporal_scenario_id,21
 load_zone_scenario_id,7
 lf_reserves_up_ba_scenario_id,NULL
 lf_reserves_down_ba_scenario_id,NULL
@@ -47,11 +49,14 @@ performance_standard_zones_carbon_credits_zones_scenario_id,NULL
 carbon_tax_zones_carbon_credits_zones_scenario_id,NULL
 carbon_credits_params_scenario_id,NULL
 fuel_burn_limit_ba_scenario_id,NULL
+policy_zone_scenario_id,NULL
 prm_zone_scenario_id,NULL
 local_capacity_zone_scenario_id,NULL
 market_scenario_id,NULL
-project_portfolio_scenario_id,24
-project_operational_chars_scenario_id,29
+water_system_params_scenario_id,NULL
+water_network_scenario_id,NULL
+project_portfolio_scenario_id,31
+project_operational_chars_scenario_id,38
 project_availability_scenario_id,3
 fuel_scenario_id,3
 project_load_zone_scenario_id,2
@@ -72,6 +77,7 @@ project_carbon_credits_generation_zone_scenario_id,NULL
 project_carbon_credits_purchase_zone_scenario_id,NULL
 project_carbon_credits_scenario_id,NULL
 project_fuel_burn_limit_ba_scenario_id,NULL
+project_policy_zone_scenario_id,NULL
 project_prm_zone_scenario_id,NULL
 prm_capacity_transfer_scenario_id,NULL
 prm_capacity_transfer_params_scenario_id,NULL
@@ -84,8 +90,8 @@ project_prm_deliverability_multipliers_scenario_id,NULL
 project_local_capacity_zone_scenario_id,NULL
 project_local_capacity_chars_scenario_id,NULL
 load_zone_market_scenario_id,NULL
-project_specified_capacity_scenario_id,6
-project_specified_fixed_cost_scenario_id,3
+project_specified_capacity_scenario_id,8
+project_specified_fixed_cost_scenario_id,5
 fuel_price_scenario_id,2
 project_new_cost_scenario_id,NULL
 project_new_potential_scenario_id,NULL
@@ -107,7 +113,7 @@ transmission_capacity_group_scenario_id,NULL
 transmission_carbon_cap_zone_scenario_id,NULL
 transmission_simultaneous_flow_limit_scenario_id,NULL
 transmission_simultaneous_flow_limit_line_group_scenario_id,NULL
-load_scenario_id,7
+load_scenario_id,8
 lf_reserves_up_scenario_id,NULL
 lf_reserves_down_scenario_id,NULL
 regulation_up_scenario_id,NULL
@@ -123,9 +129,14 @@ carbon_tax_scenario_id,NULL
 performance_standard_scenario_id,NULL
 fuel_burn_limit_scenario_id,NULL
 subsidy_scenario_id,NULL
+policy_requirement_scenario_id,NULL
 prm_requirement_scenario_id,NULL
 local_capacity_requirement_scenario_id,NULL
 elcc_surface_scenario_id,NULL
 market_price_scenario_id,NULL
 market_volume_scenario_id,NULL
+water_node_reservoir_scenario_id,NULL
+water_flow_scenario_id,NULL
+water_inflow_scenario_id,NULL
+water_powerhouse_scenario_id,NULL
 tuning_scenario_id,NULL

examples/open_data/weather_iteration_2010/hydro_iteration_2010/1/inputs/load_level_defaults.tab

@@ -0,0 +1,3 @@
+load_zone	load_component	load_level_default
+Zone1	all	.
+Zone2	all	.

examples/open_data/weather_iteration_2010/hydro_iteration_2010/1/inputs/load_mw.tab

@@ -1,49 +1,49 @@
-LOAD_ZONES	timepoint	load_mw
-Zone1	1	9001.0
-Zone1	2	8454.0
-Zone1	3	8074.0
-Zone1	4	7818.0
-Zone1	5	7767.0
-Zone1	6	7942.0
-Zone1	7	8233.0
-Zone1	8	8288.0
-Zone1	9	7940.0
-Zone1	10	7733.0
-Zone1	11	7640.0
-Zone1	12	7556.0
-Zone1	13	7736.0
-Zone1	14	8081.0
-Zone1	15	8544.0
-Zone1	16	9165.0
-Zone1	17	10237.0
-Zone1	18	11004.0
-Zone1	19	10793.0
-Zone1	20	10686.0
-Zone1	21	10485.0
-Zone1	22	10106.0
-Zone1	23	9468.0
-Zone1	24	8839.0
-Zone2	1	9001.0
-Zone2	2	8454.0
-Zone2	3	8074.0
-Zone2	4	7818.0
-Zone2	5	7767.0
-Zone2	6	7942.0
-Zone2	7	8233.0
-Zone2	8	8288.0
-Zone2	9	7940.0
-Zone2	10	7733.0
-Zone2	11	7640.0
-Zone2	12	7556.0
-Zone2	13	7736.0
-Zone2	14	8081.0
-Zone2	15	8544.0
-Zone2	16	9165.0
-Zone2	17	10237.0
-Zone2	18	11004.0
-Zone2	19	10793.0
-Zone2	20	10686.0
-Zone2	21	10485.0
-Zone2	22	10106.0
-Zone2	23	9468.0
-Zone2	24	8839.0
+LOAD_ZONES	timepoint	load_component	load_mw
+Zone1	1	all	9001.0
+Zone1	2	all	8454.0
+Zone1	3	all	8074.0
+Zone1	4	all	7818.0
+Zone1	5	all	7767.0
+Zone1	6	all	7942.0
+Zone1	7	all	8233.0
+Zone1	8	all	8288.0
+Zone1	9	all	7940.0
+Zone1	10	all	7733.0
+Zone1	11	all	7640.0
+Zone1	12	all	7556.0
+Zone1	13	all	7736.0
+Zone1	14	all	8081.0
+Zone1	15	all	8544.0
+Zone1	16	all	9165.0
+Zone1	17	all	10237.0
+Zone1	18	all	11004.0
+Zone1	19	all	10793.0
+Zone1	20	all	10686.0
+Zone1	21	all	10485.0
+Zone1	22	all	10106.0
+Zone1	23	all	9468.0
+Zone1	24	all	8839.0
+Zone2	1	all	9001.0
+Zone2	2	all	8454.0
+Zone2	3	all	8074.0
+Zone2	4	all	7818.0
+Zone2	5	all	7767.0
+Zone2	6	all	7942.0
+Zone2	7	all	8233.0
+Zone2	8	all	8288.0
+Zone2	9	all	7940.0
+Zone2	10	all	7733.0
+Zone2	11	all	7640.0
+Zone2	12	all	7556.0
+Zone2	13	all	7736.0
+Zone2	14	all	8081.0
+Zone2	15	all	8544.0
+Zone2	16	all	9165.0
+Zone2	17	all	10237.0
+Zone2	18	all	11004.0
+Zone2	19	all	10793.0
+Zone2	20	all	10686.0
+Zone2	21	all	10485.0
+Zone2	22	all	10106.0
+Zone2	23	all	9468.0
+Zone2	24	all	8839.0