From df6131dc7564a0456b63ebcf23e9d92b152fd0d0 Mon Sep 17 00:00:00 2001 From: Gabriel de Oliveira Ramos Date: Mon, 18 Dec 2017 13:47:28 -0200 Subject: [PATCH] New sumotl environment (and other improvements) by Liza Lunardi Lemos --- environment/__init__.py | 4 +- environment/sumo.py | 55 +- environment/sumotl.py | 554 + exploration/__init__.py | 4 +- exploration/epsilon_greedy.py | 27 +- learner/__init__.py | 7 +- learner/q_learning.py | 11 +- main.py | 49 +- nets/3x3grid/3x3Grid2lanes.net.xml | 1309 ++ nets/3x3grid/3x3grid.sumocfg | 13 + nets/3x3grid/routes14000.rou.xml | 21021 +++++++++++++++++++++++++++ 11 files changed, 23010 insertions(+), 44 deletions(-) create mode 100644 environment/sumotl.py create mode 100644 nets/3x3grid/3x3Grid2lanes.net.xml create mode 100644 nets/3x3grid/3x3grid.sumocfg create mode 100644 nets/3x3grid/routes14000.rou.xml diff --git a/environment/__init__.py b/environment/__init__.py index ffe1c03..835a416 100644 --- a/environment/__init__.py +++ b/environment/__init__.py @@ -52,7 +52,7 @@ def reset_episode(self): self._has_episode_ended = False for l in self._learners.values(): - l.reset_episodic() + l.reset_episodic(self._episodes) #register a learner within the environment def register_learner(self, learner): @@ -64,4 +64,4 @@ def register_learner(self, learner): def __str__(self): return "%s %s" % (self.__class__.__name__, self.__class__.__bases__[0].__name__) - \ No newline at end of file + diff --git a/environment/sumo.py b/environment/sumo.py index 04b91c7..ad3a36e 100644 --- a/environment/sumo.py +++ b/environment/sumo.py @@ -171,7 +171,6 @@ def __get_state(self, ID): def __close_connection(self): traci.close() #stop TraCI sys.stdout.flush() #clear standard output - self._sumo_process.wait() #wait for SUMO's subprocess termination def get_state_actions(self, state): return self.__env[state].keys() @@ -180,14 +179,9 @@ def reset_episode(self): super(SUMO, self).reset_episode() - #open a SUMO subprocess - self._sumo_process = subprocess.Popen([self._sumo_binary, "-c", self.__cfg_file, "--remote-port", "%i"%(self.__port)], stdout=open(os.devnull, 'wb'), stderr=open(os.devnull, 'wb')) - #initialise TraCI - traci.init(self.__port) + traci.start([self._sumo_binary , "-c", self.__cfg_file]) # SUMO 0.28 - #register commands to be performed upon normal termination - atexit.register(self.__close_connection) #------------------------------------ for vehID in self.get_vehicles_ID_list(): @@ -198,6 +192,8 @@ def reset_episode(self): self.__vehicles[vehID]['travel_time'] = -1.0 self.__vehicles[vehID]['time_last_link'] = -1.0 self.__vehicles[vehID]['route'] = [self.__vehicles[vehID]['origin']] + self.__vehicles[vehID]['initialized'] = False + self.__vehicles[vehID]['n_of_traversed_links'] = 0 @contextmanager def redirected(self): @@ -246,9 +242,12 @@ def run_episode(self, max_steps=-1): # create an initial route for vehicle vehID, consisting of the first action, only traci.route.add('R-%s'%vehID, [learner_state_action[vehID][1]]) + #print vehID, [learner_state_action[vehID][1]] with (self.redirected()): try: + #print '*** **',vehID, 'R-%s'%vehID, traci.vehicle.getRoute(vehID),'\n' traci.vehicle.setRouteID(vehID, 'R-%s'%vehID) + self.__vehicles[vehID]['initialized'] = True #print vehID , ': ' , 'R-%s'%vehID , ' = ' , traci.route.getEdges('R-%s'%vehID ) except: pass @@ -259,6 +258,7 @@ def run_episode(self, max_steps=-1): #main loop arrived=0 departed=0 + while ((max_steps > -1 and traci.simulation.getCurrentTime() < max_steps) or max_steps <= -1) and (traci.simulation.getMinExpectedNumber() > 0 or traci.simulation.getArrivedNumber() > 0): #if (traci.simulation.getCurrentTime()/1000) % 100 == 0: @@ -267,14 +267,15 @@ def run_episode(self, max_steps=-1): # loaded vehicles # the initial route must be set as soon as the vehicle is loaded, and BEFORE it enters the network for vehID in traci.simulation.getLoadedIDList(): - #rrraaab = traci.vehicle.getRoute(vehID) traci.vehicle.setRouteID(vehID, 'R-%s'%vehID) + self.__vehicles[vehID]['initialized'] = True #if vehID == vtest: # print "Route of %s was %s now is %s" % (vehID, rrraaab, traci.vehicle.getRoute(vehID)) # run a single simulation step traci.simulationStep() current_time = traci.simulation.getCurrentTime()/1000 + #if current_time % 500 == 0: # print '> %i, %i (%i departed, %i arrived) ' % (self._episodes, current_time, departed, arrived) @@ -282,6 +283,10 @@ def run_episode(self, max_steps=-1): for vehID in traci.simulation.getDepartedIDList(): self.__vehicles[vehID]["departure_time"] = current_time departed += 1 + + if not self.__vehicles[vehID]['initialized']: + traci.vehicle.setRouteID(vehID, 'R-%s'%vehID) + self.__vehicles[vehID]['initialized'] = True # arrived vehicles (those that have reached their destinations) vehicles_to_process_feedback = {} @@ -329,13 +334,13 @@ def run_episode(self, max_steps=-1): #update current_link self.__vehicles[vehID]['current_link'] = road + self.__vehicles[vehID]['n_of_traversed_links'] += 1 #get the next node, and add it to the route node = self.__get_edge_destination(self.__vehicles[vehID]["current_link"]) self.__vehicles[vehID]['route'].append(self.__get_edge_destination(self.__vehicles[vehID]['current_link'])) if node != self.__vehicles[vehID]['destination']: - vehicles_to_process_act[vehID] = [ node, #next state [x.getID().encode('utf-8') for x in self.__net.getEdge(self.__vehicles[vehID]['current_link']).getOutgoing()] #available actions @@ -354,8 +359,7 @@ def run_episode(self, max_steps=-1): sum_tt += self.__vehicles[vehID]['travel_time'] else: # for those vehicles that have not reached their destination sum_tt += current_time - self.__vehicles[vehID]["departure_time"] - #print '%i\t%s\t%s\t%s' % (self._episodes, vehID, self.__vehicles[vehID]['travel_time'], self.__vehicles[vehID]['route']) - print '%i\t%s\t%s' % (self._episodes, (sum_tt/len(self.get_vehicles_ID_list()))/60, time.time() - start) + print '%i\t%s\t%s\t%s' % (self._episodes, vehID, self.__vehicles[vehID]['travel_time'], self.__vehicles[vehID]['route']) self.__close_connection() @@ -399,7 +403,9 @@ def __process_vehicles_act(self, vehicles, current_time): # act_last for vehID in vehicles.keys(): + #~ print vehID, vehicles[vehID][1] _, action = self._learners[vehID].act_last(vehicles[vehID][0], vehicles[vehID][1]) + #~ print vehID, action #print "%s is in state %s and chosen action %s among %s" % (vehID, vehicles[vehID][0], action, vehicles[vehID][1]) if not vehicles[vehID][1]: @@ -410,11 +416,17 @@ def __process_vehicles_act(self, vehicles, current_time): #update route cur_route = traci.vehicle.getRoute(vehID) + #~ print 'route', traci.route.getEdges('R-%s'%vehID) + #~ print vehID, traci.vehicle.getRoute(vehID) cur_route.append(action) - #print 'route+ = %s'%cur_route - if cur_route[0] != self.__vehicles[vehID]['current_link']: - del cur_route[0] - #print 'route- = %s'%cur_route + #~ print 'current ', vehID, self.__vehicles[vehID]['current_link'] + + # remove traversed links from the route + # (this is necessary because otherwise the driver will try + # to reach the first link of such route from its current link) + cur_route = cur_route[self.__vehicles[vehID]['n_of_traversed_links']-1:] + + #~ print vehID, cur_route traci.vehicle.setRoute(vehID, cur_route) def __is_link(self, edge_id): @@ -601,8 +613,7 @@ def __get_state(self, ID): def __close_connection(self): traci.close() # stop TraCI sys.stdout.flush() # clear standard output - self._sumo_process.wait() # wait for SUMO's subprocess termination - + def get_state_actions(self, state): self.__check_env() return self.__env[state][2] @@ -611,14 +622,8 @@ def reset_episode(self): super(SUMORouteChoice, self).reset_episode() - # open a SUMO subprocess - self._sumo_process = subprocess.Popen([self._sumo_binary, "-c", self.__cfg_file, "--remote-port", "%i"%(self.__port)], stdout=open(os.devnull, 'wb'), stderr=open(os.devnull, 'wb')) - # initialise TraCI - traci.init(self.__port) - - # register commands to be performed upon normal termination - atexit.register(self.__close_connection) + traci.start([self._sumo_binary , "-c", self.__cfg_file]) # SUMO 0.28 # reset vehicles attributes for vehID in self.get_vehicles_ID_list(): @@ -720,7 +725,7 @@ def run_episode(self, max_steps=-1): sum_tt = 0 for vehID in self.get_vehicles_ID_list(): sum_tt += self.__vehicles[vehID]['travel_time'] - print '%i\t%s\t%s' % (self._episodes, (sum_tt/len(self.get_vehicles_ID_list()))/60, time.time() - start) + print '\n***%i\t%s\t%s***' % (self._episodes, (sum_tt/len(self.get_vehicles_ID_list()))/60, time.time() - start) self.__close_connection() diff --git a/environment/sumotl.py b/environment/sumotl.py new file mode 100644 index 0000000..d386703 --- /dev/null +++ b/environment/sumotl.py @@ -0,0 +1,554 @@ +''' +Created on 12/12/2017 + +@author: Liza L. Lemos +''' + +from environment import Environment +import traci +import sumolib +from xml.dom import minidom +import sys, os +import subprocess +import atexit +from contextlib import contextmanager +import time +from array import array +import numpy as np +import datetime +import math + + + +class SUMOTrafficLights(Environment): + + def __init__(self, cfg_file, port=8813, use_gui=False): + + super(SUMOTrafficLights, self).__init__() + + self.__create_env(cfg_file, port, use_gui) + + + ''' + Create the environment as a MDP. The MDP is modeled as follows: + * for each traffic light: + * the STATE is defined as a vector [current phase, elapsed time of current phase, queue length for each phase] + * for simplicity, the elapsed time is discretized in intervals of 5s + * and, the queue length is calculated according to the occupation of the link. + * The occupation is discretized in 4 intervals (equally distributed) + * The number of ACTIONS is equal to the number of phases + * Currentlly, there are only two phases thus the actions are either keep green time at the current phase or + * allow green to another phase. As usual, we call these actions 'keep' and 'change' + * At each junction, REWARD is defined as the difference between the current and the previous average queue length (AQL) + * at the approaching lanes, i.e., for each traffic light the reward is defined as $R(s,a,s')= AQL_{s} - AQL_{s'}$. + * the transitions between states are deterministic + ''' + def __create_env(self, cfg_file, port, use_gui): + + #check for SUMO's binaries + if use_gui: + self._sumo_binary = sumolib.checkBinary('sumo-gui') + else: + self._sumo_binary = sumolib.checkBinary('sumo') + + #register SUMO/TraCI parameters + self.__cfg_file = cfg_file + self.__net_file = self.__cfg_file[:self.__cfg_file.rfind("/")+1] + minidom.parse(self.__cfg_file).getElementsByTagName('net-file')[0].attributes['value'].value + + + #read the network file + self.__net = sumolib.net.readNet(self.__net_file) + + self.__env = {} + + d = ['keep', 'change'] + # d[0] = 'keep' + # d[1] = 'change' + + # to each state the actions are the same + # self.__env[state] has 160 possible variations + # [idPhase, elapsed time, queue NS, queue EW] = [2, 5, 4, 4] + # 2 * 5 * 4 * 4 = 160 + # idPhase: 2 phases - NS, EW + # elapsed time: 30s that are discretize in 5 intervals + # queue: 0 to 100% discretize in 4 intervals + # Note: to change the number of phases, it is necessary to change the number of states, e.g. 3 phases: [3, 5, 4, 4, 4] + # it is also necessary to change the method change_trafficlight + for x in range(0, 160): + self.__env[x] = d + + #create the set of traffic ligths + self.__create_trafficlights() + + self.__create_edges() + + + def __create_trafficlights(self): + # set of all traffic lights in the simulation + # each element in __trafficlights correspond to another in __learners + self.__trafficlights = {} + + # process all trafficlights entries + junctions_parse = minidom.parse(self.__net_file).getElementsByTagName('junction') + for element in junctions_parse: + if element.getAttribute('type') == "traffic_light": + tlID = element.getAttribute('id').encode('utf-8') + + # create the entry in the dictionary + self.__trafficlights[tlID] = { + 'greenTime': 0, + 'nextGreen': -1, + 'yellowTime': -1, + 'redTime': -1 + } + + def reset_episode(self): + + super(SUMOTrafficLights, self).reset_episode() + + # initialise TraCI + traci.start([self._sumo_binary , "-c", self.__cfg_file]) + + # reset traffic lights attributes + for tlID in self.get_trafficlights_ID_list(): + self.__trafficlights[tlID]['greenTime'] = 0 + self.__trafficlights[tlID]['nextGreen'] = -1 + self.__trafficlights[tlID]['yellowTime'] = -1 + self.__trafficlights[tlID]['redTime'] = -1 + + # define the edges/lanes that are controled for each traffic light + # the function getControlledLanes() from TRACI, returned the names of lanes doubled + # that's way is listed here + def __create_edges(self): + self._edgesNS = {} + self._edgesEW = {} + + self._edgesNS[0] = ['0Ni_0', '0Ni_1', '0Si_0', '0Si_1'] + self._edgesEW[0] = ['0Wi_0', '0Wi_1', '0Ei_0', '0Ei_1'] + self._edgesNS[1] = ['1Ni_0', '1Ni_1', '1Si_0', '1Si_1'] + self._edgesEW[1] = ['1Wi_0', '1Wi_1', '1Ei_0', '1Ei_1'] + self._edgesNS[2] = ['2Ni_0', '2Ni_1', '2Si_0', '2Si_1'] + self._edgesEW[2] = ['2Wi_0', '2Wi_1', '2Ei_0', '2Ei_1'] + self._edgesNS[3] = ['3Ni_0', '3Ni_1', '3Si_0', '3Si_1'] + self._edgesEW[3] = ['3Wi_0', '3Wi_1', '3Ei_0', '3Ei_1'] + self._edgesNS[4] = ['4Ni_0', '4Ni_1', '4Si_0', '4Si_1'] + self._edgesEW[4] = ['4Wi_0', '4Wi_1', '4Ei_0', '4Ei_1'] + self._edgesNS[5] = ['5Ni_0', '5Ni_1', '5Si_0', '5Si_1'] + self._edgesEW[5] = ['5Wi_0', '5Wi_1', '5Ei_0', '5Ei_1'] + self._edgesNS[6] = ['6Ni_0', '6Ni_1', '6Si_0', '6Si_1'] + self._edgesEW[6] = ['6Wi_0', '6Wi_1', '6Ei_0', '6Ei_1'] + self._edgesNS[7] = ['7Ni_0', '7Ni_1', '7Si_0', '7Si_1'] + self._edgesEW[7] = ['7Wi_0', '7Wi_1', '7Ei_0', '7Ei_1'] + self._edgesNS[8] = ['8Ni_0', '8Ni_1', '8Si_0', '8Si_1'] + self._edgesEW[8] = ['8Wi_0', '8Wi_1', '8Ei_0', '8Ei_1'] + + + # calculates the capacity for each queue of each traffic light + def __init_edges_capacity(self): + self._edgesNScapacity = {} + self._edgesEWcapacity = {} + + + for tlID in self.get_trafficlights_ID_list(): + #~ print '----' + #~ print 'tlID', tlID + lengthNS = 0 + lengthWE = 0 + for lane in self._edgesNS[int(tlID)]: + lengthNS += traci.lane.getLength(lane) + for lane in self._edgesEW[int(tlID)]: + lengthWE += traci.lane.getLength(lane) + lengthNS = lengthNS/7.5 # vehicle length 5m + 2.5m (minGap) + lengthWE = lengthWE/7.5 + self._edgesNScapacity[int(tlID)] = lengthNS + self._edgesEWcapacity[int(tlID)] = lengthWE + + + + # https://sourceforge.net/p/sumo/mailman/message/35824947/ + # It's necessary set a new logic, because we need more duration time. + # in SUMO the duration of the phases are set in .net file. + # but if in .net the phase duration is set to 30s and if we want 40s, the simulator will change phase in 30s + # thus, we set the duration with a high value + # also, the yellow and all red phase duration can be set here + # if prefer, this can be changed in .net file 'tllogic' tag + # obs: the duration is set in ms + def __create_tlogic(self): + phases = [] + phases.append(traci._trafficlights.Phase(200000, 200000, 200000, "GGGgrrrrGGGgrrrr")) # N-S + phases.append(traci._trafficlights.Phase(2000, 2000, 2000, "YYYYrrrrYYYYrrrr")) + phases.append(traci._trafficlights.Phase(1000, 1000, 1000, "rrrrrrrrrrrrrrrr")) + phases.append(traci._trafficlights.Phase(200000, 200000,200000, "rrrrGGGgrrrrGGGg")) # E-W + phases.append(traci._trafficlights.Phase(2000, 2000, 2000, "rrrrYYYYrrrrYYYY")) + phases.append(traci._trafficlights.Phase(1000, 1000, 1000, "rrrrrrrrrrrrrrrr")) + + logic = traci._trafficlights.Logic("new-program", 0, 0, 0, phases) + for tlID in self.get_trafficlights_ID_list(): + traci.trafficlights.setCompleteRedYellowGreenDefinition(tlID,logic) + + + def get_trafficlights_ID_list(self): + # return a list with the traffic lights' IDs + return self.__trafficlights.keys() + + # commands to be performed upon normal termination + def __close_connection(self): + traci.close() # stop TraCI + sys.stdout.flush() # clear standard output + + def get_state_actions(self, state): + self.__check_env() + # print state + # print self.__env[state] + return self.__env[state] + + # check whether the environment is ready to run + def __check_env(self): + # check whether the environment data structure was defined + if not self.__env: + raise Exception("The traffic lights must be set before running!") + + # discretize the queue occupation in 4 classes equally distributed + def discretize_queue(self, queue): + q_class = math.ceil((queue)/25) + if queue >= 75: + q_class = 3 + + # percentage + #~ if queue < 25: + #~ q_class = 0 # 0 - 25% + #~ if queue >= 25 and queue < 50: + #~ q_class = 1 # 25 - 50% + #~ if queue >= 50 and queue < 75: + #~ q_class = 2 # 50 - 75% + #~ if queue >= 75: + #~ q_class = 3 # 75 - 100% + + return int(q_class) + + + #http://stackoverflow.com/questions/759296/converting-a-decimal-to-a-mixed-radix-base-number + def mixed_radix_encode(self, idPhase, duration, queueNS, queueEW): + factors = [2, 5, 4, 4] + + queueNS = self.discretize_queue(queueNS) + queueEW = self.discretize_queue(queueEW) + + # the total elapsed time is 30s that are discretize in intervals + # discretize the duration time (elapsed time) in intervals of 5s (interv_action_selection), except the first interval + # the fisrt interval is 0 - minGreenTime + if duration > 0 and duration <= 10: # minGreenTime + duration = 0 + if duration > 10 and duration <= 15: + duration = 1 + if duration > 15 and duration <= 20: + duration = 2 + if duration > 20 and duration <= 25: + duration = 3 + if duration > 25: + duration = 4 + + # idPhase = 0 (NS green), idPhase = 3 (EW green), + # but for the mixed radix conversion idPhase can only assume 0 or 1 + if idPhase == 3: + idPhase = 1 + + # mixed radix conversion + values = [idPhase, duration, queueNS, queueEW] + res = 0 + for i in range(4): + res = res * factors[i] + values[i] + + return res + + # decode a mixed radix conversion + def mixed_radix_decode(self, value): + print 'value', value + factors = [2, 5, 4, 4] + res = [0,0,0,0] + for i in reversed(range(4)): + res[i] = value % factors[i] + value = value / factors[i] + + print 'reverse %s' % (res) + + # change the traffic light phase + # set yellow phase and save the next green + def change_trafficlight(self, tlID): + if traci.trafficlights.getPhase(tlID) == 0: # NS phase + traci.trafficlights.setPhase(tlID, 1) + self.__trafficlights[tlID]["nextGreen"] = 3 + elif traci.trafficlights.getPhase(tlID) == 3: # EW phase + traci.trafficlights.setPhase(tlID, 4) + self.__trafficlights[tlID]["nextGreen"] = 0 + + + # obs: traci.trafficlights.getPhaseDuration(tlID) + # it is the time defined in .net file, not the current elapsed time + def update_phaseTime(self, string, tlID): + self.__trafficlights[tlID][string] += 1 + + + #for states + def calculate_queue_size(self, tlID): + minSpeed = 2.8 # 10km/h - 2.78m/s + allVehicles = traci.vehicle.getIDList() + + for vehID in allVehicles: + traci.vehicle.subscribe(vehID, [traci.constants.VAR_LANE_ID, traci.constants.VAR_SPEED]) + + info_veh = traci.vehicle.getSubscriptionResults() + + # VAR_LANE_ID = 81 + # VAR_SPEED = 64 Returns the speed of the named vehicle within the last step [m/s]; error value: -1001 + + qNS = [] + qEW = [] + if len(info_veh) > 0: + for x in info_veh.keys(): + if info_veh[x][81] in self._edgesNS[int(tlID)]: + qNS.append(x) + if info_veh[x][81] in self._edgesEW[int(tlID)]: + qEW.append(x) + + return [qNS, qEW] + + #for the reward + def calculate_stopped_queue_length(self, tlID): + minSpeed = 2.8 # 10km/h - 2.78m/s + allVehicles = traci.vehicle.getIDList() + + for vehID in allVehicles: + traci.vehicle.subscribe(vehID, [traci.constants.VAR_LANE_ID, traci.constants.VAR_SPEED]) + + info_veh = traci.vehicle.getSubscriptionResults() + + # VAR_LANE_ID = 81 + # VAR_SPEED = 64 Returns the speed of the named vehicle within the last step [m/s]; error value: -1001 + + qNS = [] + qEW = [] + if len(info_veh) > 0: + for x in info_veh.keys(): + if info_veh[x][64] <= minSpeed: + if info_veh[x][81] in self._edgesNS[int(tlID)]: + qNS.append(x) + if info_veh[x][81] in self._edgesEW[int(tlID)]: + qEW.append(x) + + return [len(qNS), len(qEW)] + + + def calculate_new_state(self, tlID): + + # 1) index of the current phase + idPhase = traci.trafficlights.getPhase(tlID) + + # 2) the elapsed time in the current phase + # obs: duration = traci.trafficlights.getPhaseDuration(tlID) + # its the time defined in .net file, not the current elapsed time + duration = self.__trafficlights[tlID]["greenTime"] + + # 3) queue size + qNS_list, qEW_list = self.calculate_queue_size(tlID) + + qNS = len(qNS_list) + qEW = len(qEW_list) + + # vehicle / capacity + qNS_occupation = 0 + qEW_occupation = 0 + if qNS > 0: + qNS_occupation = (qNS*100)/self._edgesNScapacity[int(tlID)] + if qEW > 0: + qEW_occupation = (qEW*100)/self._edgesEWcapacity[int(tlID)] + + new_state = self.mixed_radix_encode(idPhase, duration ,qNS_occupation, qEW_occupation) + + return new_state + + def run_episode(self, max_steps=-1, arq_tl='saida_tl.txt', exp=None): + + self.__check_env() + + start = time.time() + + max_steps *= 1000 # traci returns steps in ms, not s + self._has_episode_ended = False + self._episodes += 1 + self.reset_episode() + + self.__init_edges_capacity() # initialize the queue capacity of each traffic light + self.__create_tlogic() + + #---------------------------------------------------------------------------------- + + current_time = 0 + previousNSqueue = [0] * len(self.get_trafficlights_ID_list()) + previousEWqueue = [0] * len(self.get_trafficlights_ID_list()) + currentNSqueue = [0] * len(self.get_trafficlights_ID_list()) + currentEWqueue = [0] * len(self.get_trafficlights_ID_list()) + new_state = [0] * len(self.get_trafficlights_ID_list()) + state = [0] * len(self.get_trafficlights_ID_list()) + choose = [0] * len(self.get_trafficlights_ID_list()) # flag: if choose an action + maxGreenTime = 180 # maximum green time, to prevent starvation + minGreenTime = 10 + interv_action_selection = 5 # interval for action selection + update_epsilon = maxGreenTime * 2 # maxGreenTime *2: to assure that the traffic ligth pass at least one time in each phase + + + # main loop + while ((max_steps > -1 and traci.simulation.getCurrentTime() < max_steps) or max_steps <= -1) and (traci.simulation.getMinExpectedNumber() > 0 or traci.simulation.getArrivedNumber() > 0): + + + queueNS = [0] * len(self.get_trafficlights_ID_list()) + queueEW = [0] * len(self.get_trafficlights_ID_list()) + + learner_state_action = {} + for tlID in self.get_trafficlights_ID_list(): + # A) LEARNER ACTION + # each traffic light makes a decision at each interv_action_selection (5s) + if self.__trafficlights[tlID]["greenTime"] > 9 and (self.__trafficlights[tlID]["greenTime"] % interv_action_selection) == 0 : + new_state[int(tlID)] = self.calculate_new_state(tlID) + state[int(tlID)], action = self._learners[tlID].act_last(new_state[int(tlID)]) + learner_state_action[tlID] = [state[int(tlID)], action] + # if green time is equal or more than maxGreenTime, change phase + if self.__trafficlights[tlID]["greenTime"] >= maxGreenTime: + learner_state_action[tlID] = [state[int(tlID)], 'change'] + choose[int(tlID)] = True # flag: if choose an action + else: + choose[int(tlID)] = False + + + # run a single simulation step + traci.simulationStep() + current_time = traci.simulation.getCurrentTime()/1000 + + # update epsilon manually - traffic lights are not a episodic task + # maxGreenTime *2: to assure that the traffic ligth pass at least one time in each phase + if update_epsilon == current_time: + update_epsilon = update_epsilon + (maxGreenTime*2) + exp.update_epsilon_manually() + + # before start needs 'change' or 'keep' the phase according to the selected action + for tlID in self.get_trafficlights_ID_list(): + + # green phase: idPhase = 0 or 3 (when have two phases) + # if yellow or all red phase - do nothing + if traci.trafficlights.getPhase(tlID) == 0 or traci.trafficlights.getPhase(tlID) == 3: + self.update_phaseTime('greenTime', tlID) + + # if choose == True: run the action (change, keep) + # else: just calculate the queue length (reward will be the average queue length) + if choose[int(tlID)] == True: + # B) RUN ACTION + if learner_state_action[tlID][1] == 'change': #TODO: more phases + self.__trafficlights[tlID]["greenTime"] = 0 + + # this method must set yellow phase and save the next green phase + self.change_trafficlight(tlID) + else: # if action = 'keep' just calculte queue size + # calculate queue size + queueNS[int(tlID)], queueEW[int(tlID)] = self.calculate_stopped_queue_length(tlID) + + currentNSqueue[int(tlID)] += queueNS[int(tlID)] + currentEWqueue[int(tlID)] += queueEW[int(tlID)] + + else: + # calculate queue size + queueNS[int(tlID)], queueEW[int(tlID)] = self.calculate_stopped_queue_length(tlID) + + currentNSqueue[int(tlID)] += queueNS[int(tlID)] + currentEWqueue[int(tlID)] += queueEW[int(tlID)] + + # if it will select action in the next step, + # in the previous you need to calculate the feedback and update Q-table + if self.__trafficlights[tlID]["greenTime"] > (minGreenTime - 1) and (self.__trafficlights[tlID]["greenTime"] % interv_action_selection) == 0 and current_time > 13: + # if current_time: it can enter in the beggining - 13 = 10 (minGreenTime) + 2 (yellow) + 1 (allRed) + + # calculate the average queue length + if self.__trafficlights[tlID]["greenTime"] == minGreenTime: # action 'change': stay minGreenTime before select new action + aver_currentNSqueue = currentNSqueue[int(tlID)]/float(minGreenTime) + aver_currentEWqueue = currentEWqueue[int(tlID)]/float(minGreenTime) + else: # action 'keep': stay interv_action_selection before select new action + aver_currentNSqueue = currentNSqueue[int(tlID)]/float(interv_action_selection) + aver_currentEWqueue = currentEWqueue[int(tlID)]/float(interv_action_selection) + + # C) CALCULATE REWARD + trafficlight_to_proces_feedback = {} + + # we define the reward as the difference between the previous and current average queue length (AQL) + # at the junction $R(s,a,s')= AQL_{s} - AQL_{s'}$ + reward = ((aver_currentEWqueue + aver_currentNSqueue) - (previousEWqueue[int(tlID)] + previousNSqueue[int(tlID)])) + reward *= -1 + + # D) PROCESS FEEDBACK + trafficlight_to_proces_feedback[tlID] = [ + reward, + new_state[int(tlID)], + state[int(tlID)] + ] + + self.__process_trafficlights_feedback(trafficlight_to_proces_feedback) + + # update previous queue + previousNSqueue[int(tlID)] = aver_currentNSqueue + previousEWqueue[int(tlID)] = aver_currentEWqueue + # clean current queue + currentNSqueue[int(tlID)] = 0 + currentEWqueue[int(tlID)] = 0 + + + self.metrics(arq_tl, current_time) + + + self.__close_connection() + self._has_episode_ended = True + + + def __process_trafficlights_feedback(self, traffic_lights): + # feedback_last + for tlID in traffic_lights.keys(): + self._learners[str(tlID)].feedback_last(traffic_lights[tlID][0], traffic_lights[tlID][1], traffic_lights[tlID][2]) + + def metrics(self, arquivo, current_time): + minSpeed = 2.8 # 10km/h - 2.78m/s + + # using subcriptions + allVehicles = traci.vehicle.getIDList() + for vehID in allVehicles: + traci.vehicle.subscribe(vehID, [traci.constants.VAR_LANE_ID, traci.constants.VAR_SPEED]) + + lanes = traci.vehicle.getSubscriptionResults() + + # VAR_LANE_ID = 81 + # VAR_SPEED = 64 Returns the speed of the named vehicle within the last step [m/s]; error value: -1001 + # VAR_WAITING_TIME = 122 Returns the waiting time [s] + + cont_veh_per_tl = [0] * len(self.get_trafficlights_ID_list()) + if len(lanes) > 0: + for x in lanes.keys(): + for tlID in self.get_trafficlights_ID_list(): + if lanes[x][64] <= minSpeed: + if (lanes[x][81] in self._edgesNS[int(tlID)]) or (lanes[x][81] in self._edgesEW[int(tlID)]): + cont_veh_per_tl[int(tlID)] += 1 + + # save in a file + # how many vehicles were in queue in each timestep + average_queue = 0 + for tlID in self.get_trafficlights_ID_list(): + average_queue = average_queue + cont_veh_per_tl[int(tlID)] + average_queue = average_queue/float(len(self.__trafficlights)) + arquivo.writelines('%d,%s,%.1f,%d\n' % (current_time, str(cont_veh_per_tl)[1:-1], average_queue, len(allVehicles))) + + def run_step(self): + raise Exception('run_step is not available in %s class' % self) + return + + def has_episode_ended(self): + return self._has_episode_ended + + def __calc_reward(self, state, action, new_state): + raise Exception('__calc_reward is not available in %s class' % self) + return + diff --git a/exploration/__init__.py b/exploration/__init__.py index 58e7ef1..810b218 100644 --- a/exploration/__init__.py +++ b/exploration/__init__.py @@ -11,7 +11,7 @@ class ExplorationStrategy: #Return an action, given an actions dictionary in the form action:Q-value @abc.abstractmethod - def choose(self, action_dict): + def choose(self, action_dict, episode): return #Called only in the beginning of each episode @@ -22,4 +22,4 @@ def reset_episodic(self): #Called in the beginning of the simulation @abc.abstractmethod def reset_all(self): - pass \ No newline at end of file + pass diff --git a/exploration/epsilon_greedy.py b/exploration/epsilon_greedy.py index f257006..b8e0bfd 100644 --- a/exploration/epsilon_greedy.py +++ b/exploration/epsilon_greedy.py @@ -9,23 +9,33 @@ class EpsilonGreedy(ExplorationStrategy): - def __init__(self, epsilon=1, decay_rate=0.99):#to avoid decay rate, set it to 0.0 + def __init__(self, epsilon=1, min_epsilon=0.0, decay_rate=0.99, manual_decay=False):#to avoid decay rate, set it to 0.0 self._epsilon_ini = epsilon self._epsilon = epsilon + self._min_epsilon = min_epsilon self._decay_rate = decay_rate + self._manual_decay = manual_decay + self._last_episode = 0 #Return an action, given an actions dictionary in the form action:Q-value - def choose(self, action_dict): - i = -1 + def choose(self, action_dict, episode): + + #update epsilon value + if self._manual_decay == False and self._last_episode != episode and self._decay_rate > 0.0 and self._epsilon > self._min_epsilon: + self._epsilon = self._epsilon * self._decay_rate + self._last_episode = episode #select an action r = random.random() + i = -1 if r < self._epsilon: #select an action uniformly at random (exploration) i = random.randint(0, len(action_dict)-1) + #~ print 'exploration', self._epsilon else: #select an action greedily (exploitation) i = sampling.reservoir_sampling(action_dict.values(), True) + #~ print 'exploitation', self._epsilon #update epsilon value if self._decay_rate > 0.0: @@ -33,10 +43,17 @@ def choose(self, action_dict): #return selected action return action_dict.keys()[i] - + + def update_epsilon_manually(self): + if self._manual_decay == False: + print '[WARNING] Manual decay should not be used; Epsilon was set with automatic decay!' + if self._epsilon > self._min_epsilon: + if self._decay_rate > 0.0: + self._epsilon = self._epsilon * self._decay_rate + def reset_episodic(self): #epsilon greedy is not episodic, so nothing to do here pass def reset_all(self): - self._epsilon = self._epsilon_ini \ No newline at end of file + self._epsilon = self._epsilon_ini diff --git a/learner/__init__.py b/learner/__init__.py index 8dde50f..8525d89 100644 --- a/learner/__init__.py +++ b/learner/__init__.py @@ -19,6 +19,8 @@ def __init__(self, name, env, child_instance): self._state = None self._action = None + self._episode = None + # subset of available actions in the current state # (used only when NOT ALL actions of the current state are available) # PS: when all actions are available, the value must be set to None @@ -30,9 +32,8 @@ def reset_all(self): pass # reset all episode-related attributes - @abc.abstractmethod - def reset_episodic(self): - pass + def reset_episodic(self, episode): + self._episode = episode # return the agent's current state and the action to be performed by him. # the state may be passed as parameter if the reasoning is being made in advance (SUMO env, eg). diff --git a/learner/q_learning.py b/learner/q_learning.py index f596916..6cd09be 100644 --- a/learner/q_learning.py +++ b/learner/q_learning.py @@ -21,7 +21,7 @@ def __init__(self, name, env, starting_state, goal_state, alpha, gamma, exp_stra self._initialise_Q_table() - self.reset_episodic() + self.reset_episodic(0) #initialize the Q-table. #in the beginning, only the entries corresponding to initial state @@ -35,7 +35,9 @@ def reset_all(self): #nothing to do here (instead of reset_all, the learner could be recreated) pass - def reset_episodic(self): + def reset_episodic(self, episode): + super(QLearner, self).reset_episodic(episode) + self._state = self._starting_state self._action = None self._accumulated_reward = 0.0 @@ -83,7 +85,7 @@ def act_last(self, state=None, available_actions=None): self._has_arrived = True else: #choose action according to the the exploration strategy - self._action = self._exp_strategy.choose(available) + self._action = self._exp_strategy.choose(available, self._episode) #print "Action %s taken in state %s" % (self._action, self._state) @@ -192,7 +194,8 @@ def _initialise_Q_table(self): self._QTable[self._starting_state] = dict({a:0 for a in self._get_actions_f(self._starting_state)}) - def reset_episodic(self): + def reset_episodic(self, episode): + super(QLearner, self).reset_episodic(episode) self._state = self._starting_state self._action = None self._accumulated_reward = 0.0 diff --git a/main.py b/main.py index 3315a78..f4aae67 100644 --- a/main.py +++ b/main.py @@ -10,6 +10,8 @@ from environment.cliffwalking import CliffWalking from environment.sumo import SUMO from environment.sumo import SUMORouteChoice +from environment.sumotl import SUMOTrafficLights + from environment.NFG.twoplayer_twoaction import TwoPlayerTwoAction from learner.q_learning import QLearner @@ -24,6 +26,9 @@ from itertools import *#@UnusedWildImport #chain, combinations +import datetime + + def test_cliff(): @@ -57,7 +62,7 @@ def test_SUMO(): env = SUMO('nets/OW/OW-traci.sumocfg', 8813, False) #an exploration strategy - exp = EpsilonGreedy(1, 0.925) + exp = EpsilonGreedy(epsilon=1, min_epsilon=0.1, decay_rate=0.99) #for each vehicle in the route file for vehID in env.get_vehicles_ID_list(): @@ -98,7 +103,7 @@ def test_SUMORouteChoice(): env.set_routes_OD_pair(origin, destination, routes) # an exploration strategy - exp = EpsilonGreedy(1, 0.925) + exp = EpsilonGreedy(epsilon=1, min_epsilon=0.1, decay_rate=0.99) # for each vehicle in the route file for vehID in env.get_vehicles_ID_list(): @@ -121,6 +126,43 @@ def test_SUMORouteChoice(): for _ in xrange(n_episodes): env.run_episode(50000) #print env._learners['1.0']._QTable + +def test_SUMOTrafficLights(): + + + print datetime.datetime.now().time() + print 'SUMO traffic lights' + + # a SUMO environment + env = SUMOTrafficLights('nets/3x3grid/3x3grid.sumocfg', 8813, False) + + # an exploration strategy + exp = EpsilonGreedy(epsilon=1, min_epsilon=0.0, decay_rate=0.95, manual_decay=True) + + # for each traffic light in the net file + for tlID in env.get_trafficlights_ID_list(): + # create a learner + _ = QLearner(tlID, env, 0, 0 , 0.1, 0.8, exp) + + # number of episodes + n_episodes = 100 + + + # for each episode + for i in xrange(n_episodes): + # print queue length + arq_avg_nome = 'tl_%d.txt' % (i) + arq_tl = open(arq_avg_nome, 'w') #para salvar saida em um arquivo + arq_tl.writelines('##%s## \n' % (datetime.datetime.now().time())) + arq_tl.write('step,tl0,tl1,tl2,tl3,tl4,tl5,tl6,tl7,tl8,average,all\n') + + env.run_episode(28800, arq_tl, exp) + + + arq_tl.close() + print datetime.datetime.now().time() + + def test_NFG(): @@ -351,7 +393,8 @@ def test_old(): #test_cliff() #test_SUMO() - test_SUMORouteChoice() + #test_SUMORouteChoice() + test_SUMOTrafficLights() #test_NFG() #test_OPPORTUNE() #test_OPPORTUNE_route_choice() diff --git a/nets/3x3grid/3x3Grid2lanes.net.xml b/nets/3x3grid/3x3Grid2lanes.net.xml new file mode 100644 index 0000000..0db14ab --- /dev/null +++ b/nets/3x3grid/3x3Grid2lanes.net.xml @@ -0,0 +1,1309 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/nets/3x3grid/3x3grid.sumocfg b/nets/3x3grid/3x3grid.sumocfg new file mode 100644 index 0000000..6b37718 --- /dev/null +++ b/nets/3x3grid/3x3grid.sumocfg @@ -0,0 +1,13 @@ + + + + + + + + + + diff --git a/nets/3x3grid/routes14000.rou.xml b/nets/3x3grid/routes14000.rou.xml new file mode 100644 index 0000000..09ef120 --- /dev/null +++ b/nets/3x3grid/routes14000.rou.xml @@ -0,0 +1,21021 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +