Merge pull request #746 from gridsingularity/bug/D3ASIM-2314

Bug/d3 asim 2314

integration_tests/integration_tests.feature

@@ -47,6 +47,7 @@ Feature: Run integration tests
   Examples: Settings
       |                scenario                |
       |         grid_fees.default_2a           |
+      |           jira.d3asim_2303             |
       |              default_2a                |
       |      two_sided_market.default_2a       |
       |    two_sided_pay_as_clear.default_2a   |

integration_tests/steps/integration_tests.py

@@ -35,7 +35,7 @@
     get_number_of_unmatched_loads
 
 TODAY_STR = today(tz=TIME_ZONE).format(DATE_FORMAT)
-ACCUMULATED_KEYS_LIST = ["Accumulated Trades", "External Trades", "totals_with_penalties"]
+ACCUMULATED_KEYS_LIST = ["Accumulated Trades", "External Trades", "Totals", "Market Fees"]
 
 
 @given('we have a scenario named {scenario}')
@@ -693,44 +693,37 @@ def test_output(context, scenario, sim_duration, slot_length, tick_length):
 @then('the energy bills report the correct accumulated traded energy price')
 def test_accumulated_energy_price(context):
     bills = context.simulation.endpoint_buffer.market_bills.bills_results
-    # cell_tower_bill = bills["Cell Tower"]["earned"] - bills["Cell Tower"]["spent"]
-    # net_traded_energy_price = cell_tower_bill
-    for house_key in ["House 1", "House 2"]:
-        extern_trades = bills[house_key]["External Trades"]
+    for bills_key in [c for c in bills.keys() if "Accumulated Trades" in c]:
+        extern_trades = bills[bills_key]["External Trades"]
         assert extern_trades["total_energy"] == extern_trades["bought"] - extern_trades["sold"]
-        assert extern_trades["total_cost"] == extern_trades["spent"] - extern_trades["earned"]
-        # house_bill = \
-        #     bills[house_key]["Totals"]["spent"] - \
-        #     bills[house_key]["Totals"]["earned"] + \
-        #     bills[house_key]["Accumulated Trades"]["earned"] - \
-        #     bills[house_key]["Accumulated Trades"]["spent"]
-        #
-        # area_net_traded_energy_price = \
-        #     sum([v["earned"] - v["spent"] for k, v in bills[house_key].items()
-        #         if k not in ACCUMULATED_KEYS_LIST])
-        # assert isclose(area_net_traded_energy_price, house_bill, rel_tol=1e-02), \
-        #     f"area: {area_net_traded_energy_price} house {house_bill}"
-        # net_traded_energy_price += area_net_traded_energy_price
-
-        for accumulated_section in ["Accumulated Trades", "External Trades", "Totals"]:
-            assert isclose(bills[house_key][accumulated_section]["spent"]
-                           + bills[house_key][accumulated_section]["market_fee"]
-                           - bills[house_key][accumulated_section]["earned"],
-                           bills[house_key][accumulated_section]["total_cost"],  abs_tol=1e-10)
+        # Checks if "Accumulated Trades" got accumulated correctly:
+        house_bill = bills[bills_key]["Accumulated Trades"]["earned"] - \
+            bills[bills_key]["Accumulated Trades"]["spent"]
+        area_net_traded_energy_price = \
+            sum([v["earned"] - v["spent"] for k, v in bills[bills_key].items()
+                if k not in ACCUMULATED_KEYS_LIST])
+        assert isclose(area_net_traded_energy_price, house_bill, rel_tol=1e-02), \
+            f"{bills_key} area: {area_net_traded_energy_price} house {house_bill}"
+        # Checks if spent+market_fee-earned=total_cost is true for all accumulated members
+        for accumulated_section in ACCUMULATED_KEYS_LIST:
+            assert isclose(bills[bills_key][accumulated_section]["spent"]
+                           + bills[bills_key][accumulated_section]["market_fee"]
+                           - bills[bills_key][accumulated_section]["earned"],
+                           bills[bills_key][accumulated_section]["total_cost"],  abs_tol=1e-10)
         #
-        # for key in ["spent", "earned", "total_cost", "sold", "bought", "total_energy"]:
-        #     assert isclose(bills[house_key]["Accumulated Trades"][key] +
-        #                    bills[house_key]["External Trades"][key] +
-        #                    bills[house_key]["Market Fees"][key],
-        #                    bills[house_key]["Totals"][key], abs_tol=1e-10)
-
-        assert isclose(bills[house_key]["Totals"]["total_cost"], 0, abs_tol=1e-10)
-    # assert isclose(net_traded_energy_price, 0, abs_tol=1e-10)
+        for key in ["spent", "earned", "total_cost", "sold", "bought", "total_energy"]:
+            assert isclose(bills[bills_key]["Accumulated Trades"][key] +
+                           bills[bills_key]["External Trades"][key] +
+                           bills[bills_key]["Market Fees"][key],
+                           bills[bills_key]["Totals"][key], abs_tol=1e-10)
+        assert isclose(bills[bills_key]["Totals"]["total_cost"], 0, abs_tol=1e-10)
 
 
 @then('the traded energy report the correct accumulated traded energy')
 def test_accumulated_energy(context):
     bills = context.simulation.endpoint_buffer.market_bills.bills_results
+    if "Cell Tower" not in bills:
+        return
     cell_tower_net = bills["Cell Tower"]["sold"] - bills["Cell Tower"]["bought"]
     net_energy = cell_tower_net
     for house_key in ["House 1", "House 2"]:
@@ -788,7 +781,7 @@ def assert_area_cumulative_bills(area):
             estimated_total = area_bills["spent_total"] - area_bills["earned"] + \
                               area_bills["penalties"]
             assert isclose(area_bills["total"], estimated_total, rel_tol=1e-2)
-            assert isclose(bills[area.uuid]["spent"],
+            assert isclose(bills[area.uuid]["spent"] + bills[area.uuid]["market_fee"],
                            cumulative_bills[area.uuid]["spent_total"], rel_tol=1e-2)
             return
         child_uuids = [child.uuid for child in area.children]

src/d3a/d3a_core/sim_results/stats.py

@@ -64,27 +64,62 @@ def __init__(self, is_spot_market=True):
         self.bills_results = {}
         self.bills_redis_results = {}
         self.market_fees = {}
-        self.external_trade_fees = {}
+        self.external_trades = {}
         self.cumulative_bills_results = {}
 
     def _store_bought_trade(self, result_dict, trade):
         # Division by 100 to convert cents to Euros
         fee_price = trade.fee_price / 100. if trade.fee_price is not None else 0.
         result_dict['bought'] += trade.offer.energy
-        result_dict['spent'] += trade.offer.price / 100.
+        if ConstSettings.IAASettings.MARKET_TYPE == 1:
+            result_dict['spent'] += trade.offer.price / 100.
+            result_dict['total_cost'] += trade.offer.price / 100. + fee_price
+        else:
+            result_dict['spent'] += trade.offer.price / 100. - fee_price
+            result_dict['total_cost'] += trade.offer.price / 100.
         result_dict['total_energy'] += trade.offer.energy
         result_dict['market_fee'] += fee_price
-        result_dict['total_cost'] += trade.offer.price / 100. + fee_price
 
-    def _store_sold_trade(self, result_dict, trade, is_internal_trade, area):
+    def _store_sold_trade(self, result_dict, trade):
         # Division by 100 to convert cents to Euros
         fee_price = trade.fee_price if trade.fee_price is not None else 0.
         result_dict['sold'] += trade.offer.energy
-        result_dict['earned'] += trade.offer.price / 100.
         result_dict['total_energy'] -= trade.offer.energy
-        result_dict['total_cost'] -= trade.offer.price / 100.
-        if not is_internal_trade:
-            self.external_trade_fees[area.name] += fee_price / 100.
+        if ConstSettings.IAASettings.MARKET_TYPE == 1:
+            trade_price = trade.offer.price
+        else:
+            trade_price = trade.offer.price - fee_price
+        result_dict['earned'] += trade_price / 100.
+        result_dict['total_cost'] -= trade_price / 100.
+
+    def _store_outgoing_external_trade(self, trade, area):
+        fee_price = trade.fee_price if trade.fee_price is not None else 0.
+        self.external_trades[area.name]['sold'] += trade.offer.energy
+        if ConstSettings.IAASettings.MARKET_TYPE == 1:
+            self.external_trades[area.name]['earned'] += trade.offer.price / 100.
+            self.external_trades[area.name]['total_cost'] -= trade.offer.price / 100.
+        else:
+            self.external_trades[area.name]['earned'] += \
+                (trade.offer.price - fee_price) / 100.
+            self.external_trades[area.name]['total_cost'] -= \
+                (trade.offer.price - fee_price) / 100.
+        self.external_trades[area.name]['total_energy'] -= trade.offer.energy
+        self.external_trades[area.name]['market_fee'] += fee_price / 100.
+
+    def _store_incoming_external_trade(self, trade, area):
+        fee_price = trade.fee_price / 100. if trade.fee_price is not None else 0.
+        self.external_trades[area.name]['bought'] += trade.offer.energy
+        if ConstSettings.IAASettings.MARKET_TYPE == 1:
+            self.external_trades[area.name]['spent'] += \
+                trade.offer.price / 100.
+            self.external_trades[area.name]['total_cost'] += \
+                trade.offer.price / 100. + fee_price
+        else:
+            self.external_trades[area.name]['spent'] += \
+                trade.offer.price / 100. - fee_price
+            self.external_trades[area.name]['total_cost'] += \
+                trade.offer.price / 100.
+        self.external_trades[area.name]['total_energy'] += trade.offer.energy
 
     @classmethod
     def _get_past_markets_from_area(cls, area, past_market_types):
@@ -173,12 +208,20 @@ def update_cumulative_bills(self, area):
                       for m in self._get_past_markets_from_area(area.parent, "past_markets")]
             trades = list(chain(*trades))
 
-            spent_total = sum(trade.offer.price
-                              for trade in trades
-                              if trade.buyer == area.name) / 100.0
-            earned = sum(trade.offer.price - trade.fee_price
-                         for trade in trades
-                         if trade.seller == area.name) / 100.0
+            if ConstSettings.IAASettings.MARKET_TYPE == 1:
+                spent_total = sum(trade.offer.price + trade.fee_price
+                                  for trade in trades
+                                  if trade.buyer == area.name) / 100.0
+                earned = sum(trade.offer.price
+                             for trade in trades
+                             if trade.seller == area.name) / 100.0
+            else:
+                spent_total = sum(trade.offer.price
+                                  for trade in trades
+                                  if trade.buyer == area.name) / 100.0
+                earned = sum(trade.offer.price - trade.fee_price
+                             for trade in trades
+                             if trade.seller == area.name) / 100.0
             penalty_energy = self._calculate_device_penalties(area)
             if penalty_energy is None:
                 penalty_energy = 0.0
@@ -198,9 +241,12 @@ def _energy_bills(self, area, past_market_types):
         if not area.children:
             return None
 
-        if area.name not in self.external_trade_fees or \
+        if area.name not in self.external_trades or \
                 ConstSettings.GeneralSettings.KEEP_PAST_MARKETS is True:
-            self.external_trade_fees[area.name] = 0.
+            self.external_trades[area.name] = dict(
+                bought=0.0, sold=0.0, spent=0.0, earned=0.0,
+                total_energy=0.0, total_cost=0.0, market_fee=0.0)
+
         result = self._get_child_data(area)
         for market in self._get_past_markets_from_area(area, past_market_types):
             for trade in market.trades:
@@ -209,9 +255,13 @@ def _energy_bills(self, area, past_market_types):
                 if buyer in result:
                     self._store_bought_trade(result[buyer], trade)
                 if seller in result:
-                    self._store_sold_trade(
-                        result[seller], trade,
-                        buyer != area_name_from_area_or_iaa_name(area.name), area)
+                    self._store_sold_trade(result[seller], trade)
+                # Outgoing external trades
+                if buyer == area_name_from_area_or_iaa_name(area.name) and seller in result:
+                    self._store_outgoing_external_trade(trade, area)
+                # Incoming external trades
+                if seller == area_name_from_area_or_iaa_name(area.name) and buyer in result:
+                    self._store_incoming_external_trade(trade, area)
         for child in area.children:
             child_result = self._energy_bills(child, past_market_types)
             if child_result is not None:
@@ -299,36 +349,35 @@ def _generate_external_and_total_bills(self, area, results, flattened):
 
         all_child_results = [v for v in results[area.name].values()]
         self._write_acculumated_stats(area, results, all_child_results, "Accumulated Trades")
-
-        total_external_fee = self.external_trade_fees[area.name]
-        if area.name in flattened:
+        total_market_fee = results[area.name]["Accumulated Trades"]["market_fee"]
+        if area.name in self.external_trades:
             # External trades are the trades of the parent area
-            external = flattened[area.name].copy()
+            external = self.external_trades[area.name].copy()
             # Should switch spent/earned and bought/sold, to match the perspective of the UI
             spent = external.pop("spent")
             earned = external.pop("earned")
             bought = external.pop("bought")
             sold = external.pop("sold")
+            total_external_fee = external.pop("market_fee")
             external.update(**{
-                "spent": earned, "earned": spent + total_external_fee, "bought": sold,
-                "sold": bought, "market_fee": 0})
+                "spent": earned, "earned": spent, "bought": sold,
+                "sold": bought, "market_fee": total_external_fee})
             external["total_energy"] = external["bought"] - external["sold"]
-            external["total_cost"] = external["spent"] - external["earned"]
+            external["total_cost"] = external["spent"] - external["earned"] + total_external_fee
             results[area.name].update({"External Trades": external})
 
+            total_market_fee += total_external_fee
+            results[area.name].update(self._market_fee_section(total_market_fee))
             totals_child_list = [results[area.name]["Accumulated Trades"],
-                                 results[area.name]["External Trades"]]
+                                 results[area.name]["External Trades"],
+                                 results[area.name]["Market Fees"]]
         else:
             # If root area, Accumulated Trades == Totals
-            totals_child_list = [results[area.name]["Accumulated Trades"]]
+            results[area.name].update(self._market_fee_section(total_market_fee))
+            totals_child_list = [results[area.name]["Accumulated Trades"],
+                                 results[area.name]["Market Fees"]]
 
         self._write_acculumated_stats(area, results, totals_child_list, "Totals")
-        total_market_fee = results[area.name]["Totals"]["market_fee"] + total_external_fee
-        results[area.name]["Totals"]["market_fee"] = 0.
-        results[area.name]["Totals"]["total_cost"] -= \
-            results[area.name]["Accumulated Trades"]["market_fee"]
-        # results[area.name]["Totals"]["total_cost"] -= total_external_fee
-        results[area.name].update(self._market_fee_section(total_market_fee))
         return results
 
     def _bills_for_redis(self, area, bills_results):

src/d3a/setup/jira/d3asim_2303.py

@@ -0,0 +1,62 @@
+"""
+Copyright 2018 Grid Singularity
+This file is part of D3A.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+from d3a.models.appliance.switchable import SwitchableAppliance
+from d3a.models.area import Area
+from d3a.models.strategy.finite_power_plant import FinitePowerPlant
+from d3a.models.strategy.market_maker_strategy import MarketMakerStrategy
+from d3a.models.strategy.load_hours import LoadHoursStrategy
+from d3a_interface.constants_limits import ConstSettings
+
+
+def get_setup(config):
+    ConstSettings.IAASettings.MARKET_TYPE = 2
+    area = Area(
+        'Grid',
+        [
+            Area(
+                'Community',
+                [
+                    Area('H1',
+                         [
+                            Area('Load', strategy=LoadHoursStrategy(avg_power_W=1000,
+                                                                    hrs_per_day=1,
+                                                                    hrs_of_day=list(
+                                                                       range(10, 11)),
+                                                                    initial_buying_rate=60,
+                                                                    final_buying_rate=60,
+                                                                    update_interval=1),
+                                 appliance=SwitchableAppliance()),
+                         ], transfer_fee_const=2)
+                ],
+                transfer_fee_const=3,
+            ),
+            Area(
+                'DSO',
+                [
+                    Area('Power Plant', strategy=FinitePowerPlant(energy_rate=30,
+                                                                  max_available_power_kW=1000),
+                         appliance=SwitchableAppliance()),
+                ],
+                transfer_fee_const=10,
+            ),
+            Area('Market Maker', strategy=MarketMakerStrategy(grid_connected=True, energy_rate=50),
+                 appliance=SwitchableAppliance()),
+        ], transfer_fee_const=10,
+        config=config
+    )
+    return area