Consolidate Tx operations results

blue-marble · Jul 6, 2023 · 09fe0e5 · 09fe0e5
1 parent 3c75ca5
commit 09fe0e5
Show file tree

Hide file tree

Showing 6 changed files with 204 additions and 339 deletions.
diff --git a/db/db_schema.sql b/db/db_schema.sql
@@ -4217,18 +4217,21 @@ DROP TABLE IF EXISTS results_transmission_operations;
 CREATE TABLE results_transmission_operations (
 scenario_id INTEGER,
 transmission_line VARCHAR(64),
-load_zone_from VARCHAR(64),
-load_zone_to VARCHAR(64),
+timepoint INTEGER,
 period INTEGER,
 subproblem_id INTEGER,
 stage_id INTEGER,
-timepoint INTEGER,
 timepoint_weight FLOAT,
 number_of_hours_in_timepoint FLOAT,
 spinup_or_lookahead INTEGER,
+tx_operational_type VARCHAR(16),
+load_zone_from VARCHAR(64),
+load_zone_to VARCHAR(64),
 transmission_flow_mw FLOAT,
 transmission_losses_lz_from FLOAT,
 transmission_losses_lz_to FLOAT,
+hurdle_cost_positive_direction FLOAT,
+hurdle_cost_negative_direction FLOAT,
 PRIMARY KEY (scenario_id, transmission_line, subproblem_id, stage_id, timepoint)
 );
 
@@ -4246,24 +4249,6 @@ PRIMARY KEY (scenario_id, subproblem_id, stage_id, period, load_zone,
 spinup_or_lookahead)
 );
 
-DROP TABLE IF EXISTS results_transmission_hurdle_costs;
-CREATE TABLE results_transmission_hurdle_costs (
-scenario_id INTEGER,
-transmission_line VARCHAR(64),
-load_zone_from VARCHAR(64),
-load_zone_to VARCHAR(64),
-period INTEGER,
-subproblem_id INTEGER,
-stage_id INTEGER,
-timepoint INTEGER,
-timepoint_weight FLOAT,
-number_of_hours_in_timepoint FLOAT,
-spinup_or_lookahead INTEGER,
-hurdle_cost_positive_direction FLOAT,
-hurdle_cost_negative_direction FLOAT,
-PRIMARY KEY (scenario_id, transmission_line, subproblem_id, stage_id, timepoint)
-);
-
 -- Transmission Costs - Aggregated
 DROP TABLE IF EXISTS results_transmission_hurdle_costs_agg;
 CREATE TABLE results_transmission_hurdle_costs_agg (

diff --git a/gridpath/auxiliary/module_list.py b/gridpath/auxiliary/module_list.py
@@ -124,9 +124,11 @@ def all_modules_list():
         "transmission.capacity.consolidate_results",
         "transmission.capacity.capacity_groups",
         "transmission.availability.availability",
+        "transmission.operations",
         "transmission.operations.operational_types",
         "transmission.operations.operations",
         "transmission.operations.transmission_flow_limits",
+        "transmission.operations.consolidate_results",
         "transmission.operations.hurdle_costs",
         "transmission.operations.simultaneous_flow_limits",
         "transmission.operations.carbon_emissions",
@@ -233,9 +235,11 @@ def optional_modules_list():
             "transmission.capacity.consolidate_results",
             "transmission.capacity.capacity_groups",
             "transmission.availability.availability",
+            "transmission.operations",
             "transmission.operations.operational_types",
             "transmission.operations.operations",
             "transmission.operations.transmission_flow_limits",
+            "transmission.operations.consolidate_results",
             "system.load_balance.aggregate_transmission_power",
             "transmission.operations.export_penalty_costs",
             "objective.transmission.aggregate_capacity_costs",

diff --git a/gridpath/transmission/operations/__init__.py b/gridpath/transmission/operations/__init__.py
@@ -0,0 +1,58 @@
+# Copyright 2016-2023 Blue Marble Analytics LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os.path
+import pandas as pd
+
+from db.common_functions import spin_on_database_lock_generic
+from gridpath.auxiliary.db_interface import setup_results_import
+
+
+def import_results_into_database(
+    scenario_id, subproblem, stage, c, db, results_directory, quiet
+):
+    """
+
+    :param scenario_id:
+    :param c:
+    :param db:
+    :param results_directory:
+    :param quiet:
+    :return:
+    """
+    if not quiet:
+        print("transmission operations")
+    # transmission_operations.csv
+    # Delete prior results and create temporary import table for ordering
+    setup_results_import(
+        conn=db,
+        cursor=c,
+        table="results_transmission_operations",
+        scenario_id=scenario_id,
+        subproblem=subproblem,
+        stage=stage,
+    )
+
+    df = pd.read_csv(os.path.join(results_directory, "transmission_operations.csv"))
+    df["scenario_id"] = scenario_id
+    df["subproblem_id"] = subproblem
+    df["stage_id"] = stage
+    spin_on_database_lock_generic(
+        command=df.to_sql(
+            name="results_transmission_operations",
+            con=db,
+            if_exists="append",
+            index=False,
+        )
+    )
diff --git a/gridpath/transmission/operations/consolidate_results.py b/gridpath/transmission/operations/consolidate_results.py
@@ -0,0 +1,39 @@
+# Copyright 2016-2023 Blue Marble Analytics LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+"""
+
+import os.path
+
+TX_OPERATIONS_DF = "tx_operations_df"
+
+
+def export_results(scenario_directory, subproblem, stage, m, d):
+    """
+    Export all results from the TX_OPERATIONS_DF that various modules
+    have added to
+    """
+    prj_opr_df = getattr(d, TX_OPERATIONS_DF)
+    prj_opr_df.to_csv(
+        os.path.join(
+            scenario_directory,
+            str(subproblem),
+            str(stage),
+            "results",
+            "transmission_operations.csv",
+        ),
+        sep=",",
+        index=True,
+    )
diff --git a/gridpath/transmission/operations/hurdle_costs.py b/gridpath/transmission/operations/hurdle_costs.py
@@ -34,6 +34,8 @@
     validate_values,
     validate_missing_inputs,
 )
+from gridpath.common_functions import create_results_df
+from gridpath.transmission.operations.consolidate_results import TX_OPERATIONS_DF
 
 
 def add_model_components(m, d, scenario_directory, subproblem, stage):
@@ -225,45 +227,30 @@ def export_results(scenario_directory, subproblem, stage, m, d):
     :param d: Dynamic components
     :return: Nothing
     """
-    with open(
-        os.path.join(
-            scenario_directory,
-            str(subproblem),
-            str(stage),
-            "results",
-            "costs_transmission_hurdle.csv",
-        ),
-        "w",
-        newline="",
-    ) as f:
-        writer = csv.writer(f)
-        writer.writerow(
-            [
-                "tx_line",
-                "period",
-                "timepoint",
-                "timepoint_weight",
-                "number_of_hours_in_timepoint",
-                "load_zone_from",
-                "load_zone_to",
-                "hurdle_cost_positive_direction",
-                "hurdle_cost_negative_direction",
-            ]
-        )
-        for tx, tmp in m.TX_OPR_TMPS:
-            writer.writerow(
-                [
-                    tx,
-                    m.period[tmp],
-                    tmp,
-                    m.tmp_weight[tmp],
-                    m.hrs_in_tmp[tmp],
-                    m.load_zone_from[tx],
-                    m.load_zone_to[tx],
-                    value(m.Hurdle_Cost_Pos_Dir[tx, tmp]),
-                    value(m.Hurdle_Cost_Neg_Dir[tx, tmp]),
-                ]
-            )
+
+    tx_opr_df = getattr(d, TX_OPERATIONS_DF)
+    results_columns = [
+        "hurdle_cost_positive_direction",
+        "hurdle_cost_negative_direction",
+    ]
+    data = [
+        [
+            tx,
+            tmp,
+        ]
+        for (tx, tmp) in m.TX_OPR_TMPS
+    ]
+    cost_df = create_results_df(
+        index_columns=["transmission_line", "timepoint"],
+        results_columns=results_columns,
+        data=data,
+    )
+
+    for c in results_columns:
+        tx_opr_df[c] = None
+    tx_opr_df.update(cost_df)
+
+    setattr(d, TX_OPERATIONS_DF, tx_opr_df)
 
 
 # Database
@@ -354,101 +341,6 @@ def write_model_inputs(
             writer.writerow(replace_nulls)
 
 
-def import_results_into_database(
-    scenario_id, subproblem, stage, c, db, results_directory, quiet
-):
-    """
-
-    :param scenario_id:
-    :param c:
-    :param db:
-    :param results_directory:
-    :param quiet:
-    :return:
-    """
-    # Hurdle costs
-    if not quiet:
-        print("transmission hurdle costs")
-
-    # Delete prior results and create temporary import table for ordering
-    setup_results_import(
-        conn=db,
-        cursor=c,
-        table="results_transmission_hurdle_costs",
-        scenario_id=scenario_id,
-        subproblem=subproblem,
-        stage=stage,
-    )
-
-    # Load results into the temporary table
-    results = []
-    with open(
-        os.path.join(results_directory, "costs_transmission_hurdle.csv"), "r"
-    ) as tx_op_file:
-        reader = csv.reader(tx_op_file)
-
-        next(reader)  # skip header
-        for row in reader:
-            tx_line = row[0]
-            period = row[1]
-            timepoint = row[2]
-            timepoint_weight = row[3]
-            number_of_hours_in_timepoint = row[4]
-            lz_from = row[5]
-            lz_to = row[6]
-            hurdle_cost_positve_direction = row[7]
-            hurdle_cost_negative_direction = row[8]
-
-            results.append(
-                (
-                    scenario_id,
-                    tx_line,
-                    period,
-                    subproblem,
-                    stage,
-                    timepoint,
-                    timepoint_weight,
-                    number_of_hours_in_timepoint,
-                    lz_from,
-                    lz_to,
-                    hurdle_cost_positve_direction,
-                    hurdle_cost_negative_direction,
-                )
-            )
-    insert_temp_sql = """
-        INSERT INTO temp_results_transmission_hurdle_costs{}
-        (scenario_id, transmission_line, period, subproblem_id, stage_id,
-        timepoint, timepoint_weight,
-        number_of_hours_in_timepoint,
-        load_zone_from, load_zone_to, 
-        hurdle_cost_positive_direction, hurdle_cost_negative_direction)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
-        """.format(
-        scenario_id
-    )
-    spin_on_database_lock(conn=db, cursor=c, sql=insert_temp_sql, data=results)
-
-    # Insert sorted results into permanent results table
-    insert_sql = """
-        INSERT INTO results_transmission_hurdle_costs
-        (scenario_id, transmission_line, period, subproblem_id, stage_id, 
-        timepoint, timepoint_weight, number_of_hours_in_timepoint,
-        load_zone_from, load_zone_to, hurdle_cost_positive_direction,
-        hurdle_cost_negative_direction)
-        SELECT
-        scenario_id, transmission_line, period, subproblem_id, stage_id,
-        timepoint, timepoint_weight, number_of_hours_in_timepoint,
-        load_zone_from, load_zone_to, hurdle_cost_positive_direction,
-        hurdle_cost_negative_direction
-        FROM temp_results_transmission_hurdle_costs{}
-         ORDER BY scenario_id, transmission_line, subproblem_id, stage_id, 
-        timepoint;
-        """.format(
-        scenario_id
-    )
-    spin_on_database_lock(conn=db, cursor=c, sql=insert_sql, data=(), many=False)
-
-
 def process_results(db, c, scenario_id, subscenarios, quiet):
     """
     Aggregate costs by zone and period. Costs are allocated to the destination
@@ -488,7 +380,7 @@ def process_results(db, c, scenario_id, subscenarios, quiet):
         load_zone_to AS load_zone, spinup_or_lookahead,
         SUM(hurdle_cost_positive_direction * timepoint_weight * 
         number_of_hours_in_timepoint) AS pos_dir_hurdle_cost
-        FROM results_transmission_hurdle_costs
+        FROM results_transmission_operations
         WHERE scenario_id = ?
         GROUP BY subproblem_id, stage_id, period, load_zone, spinup_or_lookahead
         ORDER BY subproblem_id, stage_id, period, load_zone, spinup_or_lookahead
@@ -500,7 +392,7 @@ def process_results(db, c, scenario_id, subscenarios, quiet):
         load_zone_from AS load_zone, spinup_or_lookahead,
         SUM(hurdle_cost_negative_direction * timepoint_weight * 
         number_of_hours_in_timepoint) AS neg_dir_hurdle_cost
-        FROM results_transmission_hurdle_costs
+        FROM results_transmission_operations
         WHERE scenario_id = ?
         GROUP BY subproblem_id, stage_id, period, load_zone, spinup_or_lookahead
         ORDER BY subproblem_id, stage_id, period, load_zone, spinup_or_lookahead