From 3afc00a9621edb2b01a881a5553eb703fa5175d3 Mon Sep 17 00:00:00 2001 From: Max Bohnet Date: Mon, 7 Oct 2019 15:59:16 +0200 Subject: [PATCH 01/11] startednew algorithm --- repair/apps/asmfa/graphs/graphwalker.py | 97 ++++++++++++++------ repair/apps/asmfa/tests/flowmodeltestdata.py | 20 +++- 2 files changed, 85 insertions(+), 32 deletions(-) diff --git a/repair/apps/asmfa/graphs/graphwalker.py b/repair/apps/asmfa/graphs/graphwalker.py index 1ad106d7e..143ee8b92 100644 --- a/repair/apps/asmfa/graphs/graphwalker.py +++ b/repair/apps/asmfa/graphs/graphwalker.py @@ -16,12 +16,13 @@ class BFSVisitor: class NodeVisitor(BFSVisitor): - def __init__(self, name, amount, visited, change, + def __init__(self, name, amount, visited, change, total_change, balance_factor): self.id = name self.amount = amount self.visited = visited self.change = change + self.total_change = total_change self.balance_factor = balance_factor def examine_edge(self, e): @@ -40,8 +41,11 @@ def examine_edge(self, e): elif edges_out: amount_factor = balanced_delta / len(edges_out) for e_out in edges_out: + amount_delta = self.amount[e_out] * amount_factor if not (self.visited[e_out] and self.visited[e]): - self.change[e_out] += self.amount[e_out] * amount_factor + self.change[e_out] += amount_delta + #elif abs(self.change[e_out]) < abs(amount_delta): + #self.change[e_out] = amount_delta self.visited[e_out] = True @@ -67,41 +71,80 @@ def traverse_graph(g, edge, delta, upstream=True): amount = g.ep.amount visited = g.new_edge_property("bool", val=False) change = g.new_edge_property("float", val=0.0) - change[edge] = delta + total_change = g.new_edge_property("float", val=0.0) visited[edge] = True # We are only interested in the edges that define the solution g.set_edge_filter(g.ep.include) + MAX_ITERATIONS = 5 + new_delta = delta + i = 0 + from repair.apps.asmfa.tests import flowmodeltestdata + while i < MAX_ITERATIONS and abs(new_delta) > 0.001: + change[edge] = new_delta + + # By default we go upstream first, because 'demand dictates supply' + if upstream: + g.set_reversed(True) + balance_factor = 1 / g.vp.downstream_balance_factor.a + node = edge.target() + target_node = edge.source() + + else: + g.set_reversed(False) + balance_factor = g.vp.downstream_balance_factor.a + node = edge.source() + target_node = edge.target() + + node_visitor = NodeVisitor(g.vp["id"], amount, visited, change, + total_change, balance_factor) + search.bfs_search(g, node, node_visitor) + + #total_change.a[:] += change.a + #change.a[:] = 0 + sum_f = sum(change[out_f] for out_f in node.out_edges()) + new_delta = delta - sum_f + #change[edge] = new_delta + + # now go downstream, if we started upstream + # (or upstream, if we started downstream) + + g.set_reversed(not g.is_reversed()) + node = edge.target() if g.is_reversed() else edge.source() + target_node = edge.source() if g.is_reversed() else edge.target() + # reverse the balancing factors + node_visitor.balance_factor = 1 / node_visitor.balance_factor + + visited.a[:] = False + visited[edge] = True + total_change.a[:] += change.a + change.a[:] = 0 + change[edge] = new_delta + g.ep.change.a[:] = total_change.a + flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') + + # print("\nTraversing in 2. direction") + search.bfs_search(g, node, node_visitor) + + sum_f = sum(change[out_f] for out_f in target_node.out_edges()) + new_delta = delta - sum_f + + g.ep.change.a[:] = total_change.a + flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') + + total_change.a[:] += change.a + change.a[:] = 0 + change[edge] = new_delta + visited.a[:] = False - # By default we go upstream first, because 'demand dictates supply' - if upstream: - g.set_reversed(True) - balance_factor = 1 / g.vp.downstream_balance_factor.a - node = edge.target() - else: g.set_reversed(False) - balance_factor = g.vp.downstream_balance_factor.a - node = edge.source() - - node_visitor = NodeVisitor(g.vp["id"], amount, visited, change, - balance_factor) - search.bfs_search(g, node, node_visitor) - - # now go downstream, if we started upstream - # (or upstream, if we started downstream) - - g.set_reversed(not g.is_reversed()) - node = edge.target() if g.is_reversed() else edge.source() - # reverse the balancing factors - node_visitor.balance_factor = 1 / node_visitor.balance_factor - # print("\nTraversing in 2. direction") - search.bfs_search(g, node, node_visitor) + i += 1 + total_change[edge] = delta # finally clean up del visited - g.set_reversed(False) g.clear_filters() - return node_visitor.change + return node_visitor.total_change class GraphWalker: diff --git a/repair/apps/asmfa/tests/flowmodeltestdata.py b/repair/apps/asmfa/tests/flowmodeltestdata.py index d7a88fff6..47e342afd 100644 --- a/repair/apps/asmfa/tests/flowmodeltestdata.py +++ b/repair/apps/asmfa/tests/flowmodeltestdata.py @@ -103,8 +103,7 @@ def plastic_package_graph(): G.vp.id[10] = 'Stock 2' G.vp.id[11] = 'Waste 2' flow = G.new_edge_property("object") - eid = G.new_edge_property( - "int") # need a persistent edge id, because graph-tool can reindex the edges + eid = G.new_edge_property("int") # need a persistent edge id, because graph-tool can reindex the edges G.edge_properties["flow"] = flow G.edge_properties["eid"] = eid e = G.add_edge(G.vertex(0), G.vertex(1)) @@ -150,12 +149,23 @@ def plastic_package_graph(): return split -def plot_amounts(g, file=None): - """Plots the graph with the 'amount' property on the edges into a file""" +def plot_amounts(g, file=None, prop='amount'): + """ + Plots the graph with the property defined by `prop` + on the edges into a file + + Parameters + ---------- + g : graph_tool.Graph + file : str + prop : str, optional (default='amount') + + """ + quantities = g.ep[prop] vertex_ids = [f'{int(v)}' for v in g.vertices()] vertex_text = g.new_vertex_property("string", vals=vertex_ids) mass_text = g.new_edge_property("string", - vals=[str(round(i, 2))for i in g.ep.amount]) + vals=[str(round(i, 2))for i in quantities]) gt.draw.graph_draw(g, vertex_size=20, vertex_text=vertex_text, vprops={"text_position": -1, "font_size": 10}, From 6db1fa10dff128754bcde6b4c640ea8db28163dc Mon Sep 17 00:00:00 2001 From: Max Bohnet Date: Mon, 7 Oct 2019 16:01:29 +0200 Subject: [PATCH 02/11] algorithm Work in Progress --- repair/apps/asmfa/graphs/graphwalker.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/repair/apps/asmfa/graphs/graphwalker.py b/repair/apps/asmfa/graphs/graphwalker.py index 143ee8b92..9178f400c 100644 --- a/repair/apps/asmfa/graphs/graphwalker.py +++ b/repair/apps/asmfa/graphs/graphwalker.py @@ -102,8 +102,6 @@ def traverse_graph(g, edge, delta, upstream=True): #total_change.a[:] += change.a #change.a[:] = 0 - sum_f = sum(change[out_f] for out_f in node.out_edges()) - new_delta = delta - sum_f #change[edge] = new_delta # now go downstream, if we started upstream @@ -118,24 +116,27 @@ def traverse_graph(g, edge, delta, upstream=True): visited.a[:] = False visited[edge] = True total_change.a[:] += change.a - change.a[:] = 0 - change[edge] = new_delta g.ep.change.a[:] = total_change.a flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') + sum_f = sum(change[out_f] for out_f in node.out_edges()) + new_delta = delta - sum_f + change.a[:] = 0 + change[edge] = new_delta # print("\nTraversing in 2. direction") search.bfs_search(g, node, node_visitor) - sum_f = sum(change[out_f] for out_f in target_node.out_edges()) + sum_f = sum(total_change[out_f] for out_f in node.in_edges()) new_delta = delta - sum_f + total_change.a[:] += change.a g.ep.change.a[:] = total_change.a flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') - total_change.a[:] += change.a change.a[:] = 0 change[edge] = new_delta visited.a[:] = False + visited[edge] = True g.set_reversed(False) i += 1 From 21f30ae64218b7fcab48e6f29fd53d161e6604a9 Mon Sep 17 00:00:00 2001 From: Max Bohnet Date: Tue, 8 Oct 2019 23:30:14 +0200 Subject: [PATCH 03/11] use 3 digits in graph --- repair/apps/asmfa/tests/flowmodeltestdata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repair/apps/asmfa/tests/flowmodeltestdata.py b/repair/apps/asmfa/tests/flowmodeltestdata.py index 47e342afd..05aae14be 100644 --- a/repair/apps/asmfa/tests/flowmodeltestdata.py +++ b/repair/apps/asmfa/tests/flowmodeltestdata.py @@ -165,7 +165,7 @@ def plot_amounts(g, file=None, prop='amount'): vertex_ids = [f'{int(v)}' for v in g.vertices()] vertex_text = g.new_vertex_property("string", vals=vertex_ids) mass_text = g.new_edge_property("string", - vals=[str(round(i, 2))for i in quantities]) + vals=[str(round(i, 3))for i in quantities]) gt.draw.graph_draw(g, vertex_size=20, vertex_text=vertex_text, vprops={"text_position": -1, "font_size": 10}, From 27ffc0d1511c72ced59482321cd2e7438af53957 Mon Sep 17 00:00:00 2001 From: Max Bohnet Date: Tue, 8 Oct 2019 23:35:07 +0200 Subject: [PATCH 04/11] new graphwalker algorithm --- repair/apps/asmfa/graphs/graphwalker.py | 154 +++++++++++++++--------- 1 file changed, 96 insertions(+), 58 deletions(-) diff --git a/repair/apps/asmfa/graphs/graphwalker.py b/repair/apps/asmfa/graphs/graphwalker.py index 9178f400c..dfb476c18 100644 --- a/repair/apps/asmfa/graphs/graphwalker.py +++ b/repair/apps/asmfa/graphs/graphwalker.py @@ -1,6 +1,6 @@ try: import graph_tool as gt - from graph_tool import util, search + from graph_tool import search from graph_tool.search import BFSVisitor except ModuleNotFoundError: class BFSVisitor: @@ -10,20 +10,18 @@ class BFSVisitor: from django.db.models import Q import time -from repair.apps.asmfa.models.flows import FractionFlow -from repair.apps.asmfa.models import Actor - class NodeVisitor(BFSVisitor): - def __init__(self, name, amount, visited, change, total_change, - balance_factor): + def __init__(self, name, amount, visited, change, + balance_factor, fixed, forward=True): self.id = name self.amount = amount self.visited = visited self.change = change - self.total_change = total_change self.balance_factor = balance_factor + self.fixed = fixed + self.forward = forward def examine_edge(self, e): """Compute the amount change on each inflow for the vertex @@ -44,10 +42,33 @@ def examine_edge(self, e): amount_delta = self.amount[e_out] * amount_factor if not (self.visited[e_out] and self.visited[e]): self.change[e_out] += amount_delta - #elif abs(self.change[e_out]) < abs(amount_delta): - #self.change[e_out] = amount_delta + elif (abs(self.change[e_out]) < abs(amount_delta)) \ + and not self.fixed[e_out]: + self.change[e_out] = amount_delta self.visited[e_out] = True + def examine_vertex(self, u): + if self.forward: + return + vertex_id = int(u) + edges_in = list(u.in_edges()) + edges_out = list(u.out_edges()) + sum_in_deltas = sum(self.change[in_f] for in_f in edges_in) + sum_out_deltas = sum(self.change[out_f] for out_f in edges_out) + delta = sum_in_deltas - sum_out_deltas + if not delta: + return + balanced_delta = delta * self.balance_factor[vertex_id] + sum_out_f = sum(self.amount[out_f] for out_f in edges_out) + if sum_out_f: + amount_factor = balanced_delta / sum_out_f + elif edges_out: + amount_factor = balanced_delta / len(edges_out) + for e_out in edges_out: + amount_delta = self.amount[e_out] * amount_factor + if not self.fixed[e_out]: + self.change[e_out] += amount_delta + def traverse_graph(g, edge, delta, upstream=True): """Traverse the graph in a breadth-first-search manner @@ -69,83 +90,100 @@ def traverse_graph(g, edge, delta, upstream=True): # been visited it won't be processed anymore. amount = g.ep.amount + # do we need the "visited"? visited = g.new_edge_property("bool", val=False) + fixed = g.new_edge_property("bool", val=False) change = g.new_edge_property("float", val=0.0) total_change = g.new_edge_property("float", val=0.0) visited[edge] = True + fixed[edge] = True # We are only interested in the edges that define the solution g.set_edge_filter(g.ep.include) - MAX_ITERATIONS = 5 + MAX_ITERATIONS = 20 + balance_factor = g.vp.downstream_balance_factor.a + from repair.apps.asmfa.tests import flowmodeltestdata new_delta = delta i = 0 - from repair.apps.asmfa.tests import flowmodeltestdata - while i < MAX_ITERATIONS and abs(new_delta) > 0.001: + node_visitor = NodeVisitor(g.vp["id"], amount, visited, change, + balance_factor, fixed) + + # By default we go upstream first, because 'demand dictates supply' + if upstream: + node = reverse_graph(g, node_visitor, edge) + else: + node = edge.source() + + node_visitor.forward = True + total_change.a[:] = 0 + while i < MAX_ITERATIONS and abs(new_delta) > 0.00001: + change.a[:] = 0 change[edge] = new_delta + visited.a[:] = False + visited[edge] = True - # By default we go upstream first, because 'demand dictates supply' - if upstream: - g.set_reversed(True) - balance_factor = 1 / g.vp.downstream_balance_factor.a - node = edge.target() - target_node = edge.source() - - else: - g.set_reversed(False) - balance_factor = g.vp.downstream_balance_factor.a - node = edge.source() - target_node = edge.target() + # start in one direction - node_visitor = NodeVisitor(g.vp["id"], amount, visited, change, - total_change, balance_factor) search.bfs_search(g, node, node_visitor) - #total_change.a[:] += change.a - #change.a[:] = 0 - #change[edge] = new_delta + if i > 0: + #sum_f = sum(total_change[out_f]+change[out_f] + #for out_f in node.out_edges()) + sum_f = node.out_degree(weight=total_change) + \ + node.out_degree(weight=change) + new_delta = delta - sum_f - # now go downstream, if we started upstream - # (or upstream, if we started downstream) + change[edge] = new_delta - g.set_reversed(not g.is_reversed()) - node = edge.target() if g.is_reversed() else edge.source() - target_node = edge.source() if g.is_reversed() else edge.target() - # reverse the balancing factors - node_visitor.balance_factor = 1 / node_visitor.balance_factor + ## Plot changes after forward run + #g.ep.change.a[:] = change.a + #flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') - visited.a[:] = False - visited[edge] = True - total_change.a[:] += change.a - g.ep.change.a[:] = total_change.a - flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') - sum_f = sum(change[out_f] for out_f in node.out_edges()) - new_delta = delta - sum_f - change.a[:] = 0 - change[edge] = new_delta - # print("\nTraversing in 2. direction") + # now go downstream, if we started upstream + # (or upstream, if we started downstream) + node = reverse_graph(g, node_visitor, edge) search.bfs_search(g, node, node_visitor) - sum_f = sum(total_change[out_f] for out_f in node.in_edges()) - new_delta = delta - sum_f + ## Plot changes after backward run + #g.ep.change.a[:] = change.a + #flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') - total_change.a[:] += change.a - g.ep.change.a[:] = total_change.a - flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') + # modify the implementation edge only in the first iteration + if i > 0: + change[edge] = 0 - change.a[:] = 0 - change[edge] = new_delta - visited.a[:] = False - visited[edge] = True + # add up the total changes + total_change.a += change.a + + if node.in_degree(): + sum_f = node.in_degree(weight=total_change) + #sum_f = sum(total_change[out_f] for out_f in node.in_edges()) + new_delta = delta - sum_f + else: + new_delta = 0 - g.set_reversed(False) + node = reverse_graph(g, node_visitor, edge) i += 1 - total_change[edge] = delta + + ## Plot total changes + g.ep.change.a[:] = total_change.a + flowmodeltestdata.plot_amounts(g,f'plastic_deltas_{i}.png', 'change') + # finally clean up + g.set_reversed(False) del visited g.clear_filters() - return node_visitor.total_change + return total_change + + +def reverse_graph(g, node_visitor: NodeVisitor, edge): + g.set_reversed(not g.is_reversed()) + node_visitor.balance_factor = 1 / node_visitor.balance_factor + node = edge.target() if g.is_reversed() else edge.source() + node_visitor.forward = not node_visitor.forward + return node class GraphWalker: From 56347c878ad202688bd25200d7be12f8a6e240c9 Mon Sep 17 00:00:00 2001 From: Max Bohnet Date: Tue, 8 Oct 2019 23:40:50 +0200 Subject: [PATCH 05/11] comment out plotting (only for debugging) --- repair/apps/asmfa/graphs/graphwalker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repair/apps/asmfa/graphs/graphwalker.py b/repair/apps/asmfa/graphs/graphwalker.py index dfb476c18..eada68cf0 100644 --- a/repair/apps/asmfa/graphs/graphwalker.py +++ b/repair/apps/asmfa/graphs/graphwalker.py @@ -167,8 +167,8 @@ def traverse_graph(g, edge, delta, upstream=True): i += 1 ## Plot total changes - g.ep.change.a[:] = total_change.a - flowmodeltestdata.plot_amounts(g,f'plastic_deltas_{i}.png', 'change') + #g.ep.change.a[:] = total_change.a + #flowmodeltestdata.plot_amounts(g,f'plastic_deltas_{i}.png', 'change') # finally clean up From f984383df75a50fba790071362bf06cd8af0a659 Mon Sep 17 00:00:00 2001 From: Max Bohnet Date: Wed, 9 Oct 2019 00:35:30 +0200 Subject: [PATCH 06/11] add todos --- repair/apps/asmfa/graphs/graphwalker.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/repair/apps/asmfa/graphs/graphwalker.py b/repair/apps/asmfa/graphs/graphwalker.py index eada68cf0..2ca13f78f 100644 --- a/repair/apps/asmfa/graphs/graphwalker.py +++ b/repair/apps/asmfa/graphs/graphwalker.py @@ -114,6 +114,9 @@ def traverse_graph(g, edge, delta, upstream=True): else: node = edge.source() + + # todo: + # start at the target of the implementation edge, mark node_visitor.forward = True total_change.a[:] = 0 while i < MAX_ITERATIONS and abs(new_delta) > 0.00001: @@ -162,13 +165,13 @@ def traverse_graph(g, edge, delta, upstream=True): new_delta = delta - sum_f else: new_delta = 0 + ## Plot total changes + #g.ep.change.a[:] = total_change.a + #flowmodeltestdata.plot_amounts(g,f'plastic_deltas_{i}.png', 'change') node = reverse_graph(g, node_visitor, edge) i += 1 - ## Plot total changes - #g.ep.change.a[:] = total_change.a - #flowmodeltestdata.plot_amounts(g,f'plastic_deltas_{i}.png', 'change') # finally clean up From 2f282fda691905c0e56b663110c6b43b370f9df3 Mon Sep 17 00:00:00 2001 From: Max Bohnet Date: Wed, 9 Oct 2019 11:40:04 +0200 Subject: [PATCH 07/11] rewrite algorithm with examine_vertex() --- repair/apps/asmfa/graphs/graphwalker.py | 230 +++++++++++++++--------- 1 file changed, 144 insertions(+), 86 deletions(-) diff --git a/repair/apps/asmfa/graphs/graphwalker.py b/repair/apps/asmfa/graphs/graphwalker.py index 2ca13f78f..f4f6b00df 100644 --- a/repair/apps/asmfa/graphs/graphwalker.py +++ b/repair/apps/asmfa/graphs/graphwalker.py @@ -6,68 +6,74 @@ class BFSVisitor: pass import copy -import numpy as np -from django.db.models import Q -import time class NodeVisitor(BFSVisitor): - def __init__(self, name, amount, visited, change, - balance_factor, fixed, forward=True): + def __init__(self, name, amount, change, + balance_factor, forward=True): self.id = name self.amount = amount - self.visited = visited self.change = change self.balance_factor = balance_factor - self.fixed = fixed self.forward = forward - def examine_edge(self, e): - """Compute the amount change on each inflow for the vertex - - This function is invoked on a edge as it is popped from the queue. - """ - u = e.target() + def examine_vertex(self, u): vertex_id = int(u) + out_degree = u.out_degree() + if not out_degree: + return - balanced_delta = self.change[e] * self.balance_factor[vertex_id] - edges_out = list(u.out_edges()) - sum_out_f = sum(self.amount[out_f] for out_f in edges_out) + bf = self.balance_factor[vertex_id] + sum_in_deltas = u.in_degree(weight=self.change) + balanced_delta = sum_in_deltas * bf + sum_out_f = u.out_degree(weight=self.amount) if sum_out_f: amount_factor = balanced_delta / sum_out_f - elif edges_out: - amount_factor = balanced_delta / len(edges_out) - for e_out in edges_out: + else: + amount_factor = balanced_delta / out_degree + + for e_out in u.out_edges(): amount_delta = self.amount[e_out] * amount_factor - if not (self.visited[e_out] and self.visited[e]): + if self.forward: self.change[e_out] += amount_delta - elif (abs(self.change[e_out]) < abs(amount_delta)) \ - and not self.fixed[e_out]: - self.change[e_out] = amount_delta - self.visited[e_out] = True + else: + if abs(self.change[e_out]) < abs(amount_delta): + self.change[e_out] = amount_delta + else: + self.change[e_out] += amount_delta + + +class NodeVisitorBalanceDeltas(BFSVisitor): + + def __init__(self, name, amount, change, + balance_factor): + self.id = name + self.amount = amount + self.change = change + self.balance_factor = balance_factor def examine_vertex(self, u): - if self.forward: - return vertex_id = int(u) - edges_in = list(u.in_edges()) - edges_out = list(u.out_edges()) - sum_in_deltas = sum(self.change[in_f] for in_f in edges_in) - sum_out_deltas = sum(self.change[out_f] for out_f in edges_out) + out_degree = u.out_degree() + if not out_degree: + return + + sum_in_deltas = u.in_degree(self.change) + sum_out_deltas = u.out_degree(self.change) delta = sum_in_deltas - sum_out_deltas if not delta: return - balanced_delta = delta * self.balance_factor[vertex_id] - sum_out_f = sum(self.amount[out_f] for out_f in edges_out) + bf = self.balance_factor[vertex_id] + balanced_delta = delta * bf + sum_out_f = u.out_degree(weight=self.amount) if sum_out_f: amount_factor = balanced_delta / sum_out_f - elif edges_out: - amount_factor = balanced_delta / len(edges_out) - for e_out in edges_out: + else: + amount_factor = balanced_delta / out_degree + for e_out in u.out_edges(): amount_delta = self.amount[e_out] * amount_factor - if not self.fixed[e_out]: - self.change[e_out] += amount_delta + self.change[e_out] += amount_delta def traverse_graph(g, edge, delta, upstream=True): @@ -86,106 +92,158 @@ def traverse_graph(g, edge, delta, upstream=True): Edge ProperyMap (float) The signed change on the edges """ - # Property map for keeping track of the visited edge. Once an edge has - # been visited it won't be processed anymore. amount = g.ep.amount - # do we need the "visited"? - visited = g.new_edge_property("bool", val=False) - fixed = g.new_edge_property("bool", val=False) change = g.new_edge_property("float", val=0.0) total_change = g.new_edge_property("float", val=0.0) - visited[edge] = True - fixed[edge] = True + # We are only interested in the edges that define the solution g.set_edge_filter(g.ep.include) MAX_ITERATIONS = 20 balance_factor = g.vp.downstream_balance_factor.a from repair.apps.asmfa.tests import flowmodeltestdata - new_delta = delta - i = 0 - node_visitor = NodeVisitor(g.vp["id"], amount, visited, change, - balance_factor, fixed) + node_visitor = NodeVisitor(g.vp["id"], amount, change, + balance_factor) + node_visitor2 = NodeVisitorBalanceDeltas(g.vp["id"], amount, change, + balance_factor) + + # make a first run with the given changes to the implementation edge # By default we go upstream first, because 'demand dictates supply' if upstream: - node = reverse_graph(g, node_visitor, edge) - else: node = edge.source() + g.set_reversed(True) + else: + node = edge.target() + g.set_reversed(False) - - # todo: - # start at the target of the implementation edge, mark node_visitor.forward = True total_change.a[:] = 0 + new_delta = delta + i = 0 + change[edge] = new_delta + # start in one direction + search.bfs_search(g, node, node_visitor) + change[edge] = new_delta + + ## Plot changes after forward run + g.ep.change.a[:] = change.a + flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') + + node = reverse_graph(g, node_visitor, node_visitor2, edge) + search.bfs_search(g, node, node_visitor) + change[edge] = new_delta + + ## Plot changes after backward run + g.ep.change.a[:] = change.a + flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') + + # balance out the changes + search.bfs_search(g, node, node_visitor2) + change[edge] = new_delta + + # add up the total changes + total_change.a += change.a + + ## Plot total changes + g.ep.change.a[:] = total_change.a + flowmodeltestdata.plot_amounts(g,f'plastic_deltas_{i}.png', 'change') + + node = reverse_graph(g, node_visitor, node_visitor2, edge) + + if upstream: + if node.in_degree(): + sum_f = node.in_degree(weight=total_change) + new_delta = delta - sum_f + else: + new_delta = 0 + else: + if node.out_degree(): + sum_f = node.out_degree(weight=total_change) + new_delta = delta - sum_f + else: + new_delta = 0 + i += 1 + + while i < MAX_ITERATIONS and abs(new_delta) > 0.00001: change.a[:] = 0 change[edge] = new_delta - visited.a[:] = False - visited[edge] = True # start in one direction search.bfs_search(g, node, node_visitor) - - if i > 0: - #sum_f = sum(total_change[out_f]+change[out_f] - #for out_f in node.out_edges()) - sum_f = node.out_degree(weight=total_change) + \ - node.out_degree(weight=change) - new_delta = delta - sum_f - - change[edge] = new_delta + change[edge] = 0 ## Plot changes after forward run - #g.ep.change.a[:] = change.a - #flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') + g.ep.change.a[:] = change.a + flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') # now go downstream, if we started upstream # (or upstream, if we started downstream) - node = reverse_graph(g, node_visitor, edge) + node = reverse_graph(g, node_visitor, node_visitor2, edge) + if upstream: + sum_f = node.out_degree(weight=total_change) + \ + node.out_degree(weight=change) + else: + sum_f = node.in_degree(weight=total_change) + \ + node.in_degree(weight=change) + new_delta = delta - sum_f + change[edge] = new_delta + search.bfs_search(g, node, node_visitor) + #change[edge] = 0 ## Plot changes after backward run - #g.ep.change.a[:] = change.a - #flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') + g.ep.change.a[:] = change.a + flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') - # modify the implementation edge only in the first iteration - if i > 0: - change[edge] = 0 + # balance out the changes + #change[edge] = new_delta + search.bfs_search(g, node, node_visitor2) + change[edge] = 0 + + ## Plot changes after balancing + g.ep.change.a[:] = change.a + flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') # add up the total changes total_change.a += change.a - if node.in_degree(): - sum_f = node.in_degree(weight=total_change) - #sum_f = sum(total_change[out_f] for out_f in node.in_edges()) - new_delta = delta - sum_f - else: - new_delta = 0 + node = reverse_graph(g, node_visitor, node_visitor2, edge) + ## Plot total changes - #g.ep.change.a[:] = total_change.a - #flowmodeltestdata.plot_amounts(g,f'plastic_deltas_{i}.png', 'change') + g.ep.change.a[:] = total_change.a + flowmodeltestdata.plot_amounts(g,f'plastic_deltas_{i}.png', 'change') - node = reverse_graph(g, node_visitor, edge) + if upstream: + if node.in_degree(): + sum_f = node.in_degree(weight=total_change) + new_delta = delta - sum_f + else: + new_delta = 0 + else: + if node.out_degree(): + sum_f = node.out_degree(weight=total_change) + new_delta = delta - sum_f + else: + new_delta = 0 i += 1 - - # finally clean up g.set_reversed(False) - del visited g.clear_filters() return total_change -def reverse_graph(g, node_visitor: NodeVisitor, edge): +def reverse_graph(g, node_visitor: NodeVisitor, node_visitor2, edge): g.set_reversed(not g.is_reversed()) node_visitor.balance_factor = 1 / node_visitor.balance_factor - node = edge.target() if g.is_reversed() else edge.source() + node = edge.target() if not g.is_reversed() else edge.source() node_visitor.forward = not node_visitor.forward + node_visitor2.balance_factor = 1 / node_visitor2.balance_factor return node From af26b96be95468d5d6f539fd621a28b7b69b2fd9 Mon Sep 17 00:00:00 2001 From: Max Bohnet Date: Tue, 22 Oct 2019 15:42:22 +0200 Subject: [PATCH 08/11] fix get_affected_flows --- repair/apps/asmfa/graphs/graph.py | 6 ++- repair/apps/asmfa/graphs/graphwalker.py | 61 ++++++++++++++----------- 2 files changed, 39 insertions(+), 28 deletions(-) diff --git a/repair/apps/asmfa/graphs/graph.py b/repair/apps/asmfa/graphs/graph.py index 954e63bbc..e251701db 100644 --- a/repair/apps/asmfa/graphs/graph.py +++ b/repair/apps/asmfa/graphs/graph.py @@ -718,11 +718,13 @@ def _get_affected_flows(self, solution_part): aff_flows = FractionFlow.objects.none() for af in affectedflows: #materials = descend_materials([af.material]) - flows = FractionFlow.objects.filter( + flows = get_annotated_fractionflows(self.strategy.keyflow.id, + self.strategy.id) + flows = flows.filter( origin__activity = af.origin_activity, destination__activity = af.destination_activity ) - flows = self._annotate(flows) + kwargs = { 'strategy_material': af.material.id } diff --git a/repair/apps/asmfa/graphs/graphwalker.py b/repair/apps/asmfa/graphs/graphwalker.py index f4f6b00df..a4cd442e3 100644 --- a/repair/apps/asmfa/graphs/graphwalker.py +++ b/repair/apps/asmfa/graphs/graphwalker.py @@ -92,22 +92,26 @@ def traverse_graph(g, edge, delta, upstream=True): Edge ProperyMap (float) The signed change on the edges """ + plot = False amount = g.ep.amount change = g.new_edge_property("float", val=0.0) total_change = g.new_edge_property("float", val=0.0) - # We are only interested in the edges that define the solution g.set_edge_filter(g.ep.include) MAX_ITERATIONS = 20 balance_factor = g.vp.downstream_balance_factor.a - from repair.apps.asmfa.tests import flowmodeltestdata node_visitor = NodeVisitor(g.vp["id"], amount, change, balance_factor) node_visitor2 = NodeVisitorBalanceDeltas(g.vp["id"], amount, change, balance_factor) + if plot: + # prepare plotting of intermediate results + from repair.apps.asmfa.tests import flowmodeltestdata + g.ep.change = change + # make a first run with the given changes to the implementation edge # By default we go upstream first, because 'demand dictates supply' @@ -127,17 +131,19 @@ def traverse_graph(g, edge, delta, upstream=True): search.bfs_search(g, node, node_visitor) change[edge] = new_delta - ## Plot changes after forward run - g.ep.change.a[:] = change.a - flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') + if plot: + ## Plot changes after forward run + g.ep.change.a[:] = change.a + flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') node = reverse_graph(g, node_visitor, node_visitor2, edge) search.bfs_search(g, node, node_visitor) change[edge] = new_delta - ## Plot changes after backward run - g.ep.change.a[:] = change.a - flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') + if plot: + ## Plot changes after backward run + g.ep.change.a[:] = change.a + flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') # balance out the changes search.bfs_search(g, node, node_visitor2) @@ -146,9 +152,10 @@ def traverse_graph(g, edge, delta, upstream=True): # add up the total changes total_change.a += change.a - ## Plot total changes - g.ep.change.a[:] = total_change.a - flowmodeltestdata.plot_amounts(g,f'plastic_deltas_{i}.png', 'change') + if plot: + ## Plot total changes + g.ep.change.a[:] = total_change.a + flowmodeltestdata.plot_amounts(g,f'plastic_deltas_{i}.png', 'change') node = reverse_graph(g, node_visitor, node_visitor2, edge) @@ -176,9 +183,10 @@ def traverse_graph(g, edge, delta, upstream=True): search.bfs_search(g, node, node_visitor) change[edge] = 0 - ## Plot changes after forward run - g.ep.change.a[:] = change.a - flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') + if plot: + ## Plot changes after forward run + g.ep.change.a[:] = change.a + flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') # now go downstream, if we started upstream @@ -192,31 +200,32 @@ def traverse_graph(g, edge, delta, upstream=True): node.in_degree(weight=change) new_delta = delta - sum_f change[edge] = new_delta - search.bfs_search(g, node, node_visitor) - #change[edge] = 0 - ## Plot changes after backward run - g.ep.change.a[:] = change.a - flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') + + if plot: + ## Plot changes after backward run + g.ep.change.a[:] = change.a + flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') # balance out the changes - #change[edge] = new_delta search.bfs_search(g, node, node_visitor2) change[edge] = 0 - ## Plot changes after balancing - g.ep.change.a[:] = change.a - flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') + if plot: + ## Plot changes after balancing + g.ep.change.a[:] = change.a + flowmodeltestdata.plot_amounts(g,'plastic_deltas.png', 'change') # add up the total changes total_change.a += change.a node = reverse_graph(g, node_visitor, node_visitor2, edge) - ## Plot total changes - g.ep.change.a[:] = total_change.a - flowmodeltestdata.plot_amounts(g,f'plastic_deltas_{i}.png', 'change') + if plot: + ## Plot total changes + g.ep.change.a[:] = total_change.a + flowmodeltestdata.plot_amounts(g,f'plastic_deltas_{i}.png', 'change') if upstream: if node.in_degree(): From d34ab57c3e5d34f8202db078d1b3f99b97549075 Mon Sep 17 00:00:00 2001 From: Max Bohnet Date: Tue, 22 Oct 2019 15:42:42 +0200 Subject: [PATCH 09/11] test with lower presicion --- repair/apps/asmfa/tests/test_graph.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/repair/apps/asmfa/tests/test_graph.py b/repair/apps/asmfa/tests/test_graph.py index 6da888492..cceb62f16 100644 --- a/repair/apps/asmfa/tests/test_graph.py +++ b/repair/apps/asmfa/tests/test_graph.py @@ -1251,6 +1251,7 @@ def affect_biofuel_chain(solpart): strat_digest_factor = strat_out_digester_sum / strat_in_digester_sum self.assertAlmostEqual(sq_digest_factor, strat_digest_factor, + places=1, msg=f'the factor at actor {biodigester} in ' 'strategy is not the same as in status quo') @@ -1281,7 +1282,7 @@ def assert_balance_factor(activity): sf_out = out_flows.aggregate(amount=Sum('strategy_amount'))['amount'] sq_factor = (sq_out / sq_in) if sq_out and sq_in else 1 sf_factor = (sf_out / sf_in) if sf_out and sf_in else 1 - self.assertAlmostEqual(sq_factor, sf_factor, + self.assertAlmostEqual(sq_factor, sf_factor, 1, msg='the balance factor at actor ' f'{actor} in strategy is not the ' 'same as in status quo') From bb110da4240091a263d17bb0b0f486ccac8bddf9 Mon Sep 17 00:00:00 2001 From: Max Bohnet Date: Tue, 22 Oct 2019 20:58:49 +0200 Subject: [PATCH 10/11] fix bug in graphwalker balancing algorithm --- repair/apps/asmfa/graphs/graphwalker.py | 31 +++++++++-------- repair/apps/asmfa/tests/flowmodeltestdata.py | 2 +- repair/apps/asmfa/tests/test_graph.py | 35 ++++++++++++++------ 3 files changed, 44 insertions(+), 24 deletions(-) diff --git a/repair/apps/asmfa/graphs/graphwalker.py b/repair/apps/asmfa/graphs/graphwalker.py index a4cd442e3..261f9b4cf 100644 --- a/repair/apps/asmfa/graphs/graphwalker.py +++ b/repair/apps/asmfa/graphs/graphwalker.py @@ -61,11 +61,11 @@ def examine_vertex(self, u): sum_in_deltas = u.in_degree(self.change) sum_out_deltas = u.out_degree(self.change) - delta = sum_in_deltas - sum_out_deltas - if not delta: - return bf = self.balance_factor[vertex_id] - balanced_delta = delta * bf + balanced_out_deltas = sum_out_deltas / bf + balanced_delta = sum_in_deltas - balanced_out_deltas + if abs(balanced_delta) < 0.0000001: + return sum_out_f = u.out_degree(weight=self.amount) if sum_out_f: amount_factor = balanced_delta / sum_out_f @@ -98,30 +98,35 @@ def traverse_graph(g, edge, delta, upstream=True): change = g.new_edge_property("float", val=0.0) total_change = g.new_edge_property("float", val=0.0) - # We are only interested in the edges that define the solution - g.set_edge_filter(g.ep.include) - MAX_ITERATIONS = 20 - balance_factor = g.vp.downstream_balance_factor.a - node_visitor = NodeVisitor(g.vp["id"], amount, change, - balance_factor) - node_visitor2 = NodeVisitorBalanceDeltas(g.vp["id"], amount, change, - balance_factor) - if plot: # prepare plotting of intermediate results from repair.apps.asmfa.tests import flowmodeltestdata + flowmodeltestdata.plot_materials(g, file='materials.png') + flowmodeltestdata.plot_amounts(g,'amounts.png', 'amount') g.ep.change = change + # We are only interested in the edges that define the solution + g.set_edge_filter(g.ep.include) + MAX_ITERATIONS = 20 + balance_factor = g.vp.downstream_balance_factor.a + # make a first run with the given changes to the implementation edge # By default we go upstream first, because 'demand dictates supply' if upstream: node = edge.source() g.set_reversed(True) + balance_factor = 1 / balance_factor else: node = edge.target() g.set_reversed(False) + # initialize the node-visitors + node_visitor = NodeVisitor(g.vp["id"], amount, change, + balance_factor) + node_visitor2 = NodeVisitorBalanceDeltas(g.vp["id"], amount, change, + balance_factor) + node_visitor.forward = True total_change.a[:] = 0 new_delta = delta diff --git a/repair/apps/asmfa/tests/flowmodeltestdata.py b/repair/apps/asmfa/tests/flowmodeltestdata.py index c6f3080ee..9ce2c87ff 100644 --- a/repair/apps/asmfa/tests/flowmodeltestdata.py +++ b/repair/apps/asmfa/tests/flowmodeltestdata.py @@ -168,7 +168,7 @@ def plot_amounts(g, file=None, prop='amount'): vertex_ids = [f'{int(v)}' for v in g.vertices()] vertex_text = g.new_vertex_property("string", vals=vertex_ids) mass_text = g.new_edge_property("string", - vals=[str(round(i, 3))for i in quantities]) + vals=[str(round(i, 3)) for i in quantities]) gt.draw.graph_draw(g, vertex_size=20, vertex_text=vertex_text, vprops={"text_position": -1, "font_size": 10}, diff --git a/repair/apps/asmfa/tests/test_graph.py b/repair/apps/asmfa/tests/test_graph.py index cceb62f16..b4f984ed5 100644 --- a/repair/apps/asmfa/tests/test_graph.py +++ b/repair/apps/asmfa/tests/test_graph.py @@ -75,7 +75,6 @@ def test_plastic_packaging(self): Consumption --> Recycling -0.12 Recycling --> Production -0.06 """ - return plastic = flowmodeltestdata.plastic_package_graph() gw = GraphWalker(plastic) change = gw.graph.new_edge_property('float') @@ -101,45 +100,61 @@ def test_plastic_packaging(self): gw.graph.ep.include[e] = False result = gw.calculate(implementation_edges, deltas) for i, e in enumerate(result.edges()): - print(f"{result.vp.id[e.source()]} --> {result.vp.id[e.target()]} / {result.ep.material[e]}: {result.ep.amount[e]}") + print(f"{result.vp.id[e.source()]} --> {result.vp.id[e.target()]} " + f"/ {result.ep.material[e]}: {result.ep.amount[e]}") if result.vp.id[e.source()] == 'Packaging' \ and result.vp.id[e.target()] == 'Consumption' \ and result.ep.material[e] == 'plastic': expected = 5.0 - 0.3 - self.assertAlmostEqual(result.ep.amount[e], expected, 2, 'Packaging->Consumption') + self.assertAlmostEqual( + result.ep.amount[e], expected, delta=0.2, + msg='Packaging->Consumption') elif result.vp.id[e.source()] == 'Oil rig' \ and result.vp.id[e.target()] == 'Oil refinery' \ and result.ep.material[e] == 'crude oil': expected = 20.0 - 0.3 - self.assertAlmostEqual(result.ep.amount[e], expected, 2, 'Oil rig->Oil refinery') + self.assertAlmostEqual( + result.ep.amount[e], expected, delta=0.2, + msg='Oil rig->Oil refinery') elif result.vp.id[e.source()] == 'Oil refinery' \ and result.vp.id[e.target()] == 'Production' \ and result.ep.material[e] == 'plastic': expected = 4.0 - 0.24 - self.assertAlmostEqual(result.ep.amount[e], expected, 2, 'Oil refinery->Production') + self.assertAlmostEqual( + result.ep.amount[e], expected, delta=0.2, + msg='Oil refinery->Production') elif result.vp.id[e.source()] == 'Production' \ and result.vp.id[e.target()] == 'Packaging' \ and result.ep.material[e] == 'plastic': expected = 5.0 - 0.3 - self.assertAlmostEqual(result.ep.amount[e], expected, 2, 'Production->Packaging') + self.assertAlmostEqual( + result.ep.amount[e], expected, delta=0.2, + msg='Production->Packaging') elif result.vp.id[e.source()] == 'Consumption' \ and result.vp.id[e.target()] == 'Burn' \ and result.ep.material[e] == 'plastic': expected = 3.0 - 0.18 - self.assertAlmostEqual(result.ep.amount[e], expected, 2, 'Consumption->Burn') + self.assertAlmostEqual( + result.ep.amount[e], expected, delta=0.1, + msg='Consumption->Burn') elif result.vp.id[e.source()] == 'Consumption' \ and result.vp.id[e.target()] == 'Recycling' \ and result.ep.material[e] == 'plastic': expected = 2.0 - 0.12 - self.assertAlmostEqual(result.ep.amount[e], expected, 2, 'Consumption->Recycling') + self.assertAlmostEqual( + result.ep.amount[e], expected, delta=0.1, + msg='Consumption->Recycling') elif result.vp.id[e.source()] == 'Recycling' \ and result.vp.id[e.target()] == 'Production' \ and result.ep.material[e] == 'plastic': expected = 1.0 - 0.06 - self.assertAlmostEqual(result.ep.amount[e], expected, 2, 'Recycling->Production') + self.assertAlmostEqual( + result.ep.amount[e], expected, delta=0.06, + msg='Recycling->Production') else: self.assertAlmostEqual(result.ep.amount[e], - gw.graph.ep.amount[e], places=2) + gw.graph.ep.amount[e], + delta=result.ep.amount[e]/10) def test_milk_production(self): From e56f7ffe3e10aac8ec7e8639bf72cd54b29a4fbd Mon Sep 17 00:00:00 2001 From: Max Bohnet Date: Tue, 22 Oct 2019 21:18:55 +0200 Subject: [PATCH 11/11] add tests if the deltas are balanced --- repair/apps/asmfa/tests/test_graph.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/repair/apps/asmfa/tests/test_graph.py b/repair/apps/asmfa/tests/test_graph.py index b4f984ed5..f83e29c22 100644 --- a/repair/apps/asmfa/tests/test_graph.py +++ b/repair/apps/asmfa/tests/test_graph.py @@ -99,6 +99,10 @@ def test_plastic_packaging(self): else: gw.graph.ep.include[e] = False result = gw.calculate(implementation_edges, deltas) + + # for each flow, test if the results are approximately + # the expected value, taking some delta due to the balancing + # into account for i, e in enumerate(result.edges()): print(f"{result.vp.id[e.source()]} --> {result.vp.id[e.target()]} " f"/ {result.ep.material[e]}: {result.ep.amount[e]}") @@ -156,6 +160,25 @@ def test_plastic_packaging(self): gw.graph.ep.amount[e], delta=result.ep.amount[e]/10) + # test if the changes in all nodes are balanced + result.ep.change.a = result.ep.amount.a - gw.graph.ep.amount.a + + for u in result.vertices(): + # dangling nodes with no in- or outflows can be ignored + if not (u.in_degree() and u.out_degree()): + continue + # for the rest, the sum of the in-deltas should equal to the + # sum of the out-deltas, adjusted with the balancing factor + sum_in_deltas = u.in_degree(result.ep.change) + sum_out_deltas = u.out_degree(result.ep.change) + balanced_out_deltas = sum_out_deltas / bf[u] + balanced_delta = sum_in_deltas - balanced_out_deltas + self.assertAlmostEqual( + balanced_delta, 0, places=4, + msg=f'Node {int(u)} not balanced, deltas in: {sum_in_deltas}, ' + f'deltas out: {sum_out_deltas}, bf: {bf[u]}, ' + f'balanced deltas out: {balanced_out_deltas}' + ) def test_milk_production(self): """Reduce milk production between Farm->Packaging