-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathModelTarget.py.05162009.working
158 lines (141 loc) · 6.88 KB
/
ModelTarget.py.05162009.working
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
import sys
import threading
import asyncio
import random
import importlib
import copy
import uuid
import collections
import time
import queue
from queue import Empty
from functools import lru_cache
import logging
class ModelTarget(object):
def __init__(self,*args,**kwargs):
#Setup log
self.logger = logging.getLogger(__name__)
super(ModelTarget,self).__init__()
self._id = uuid.uuid1().int
self._modelId = kwargs.get('modelId')
self._type = 'MODELTARGET'
self._moduleName = kwargs.get('module')
self._className = kwargs.get('class')
self._methodName = kwargs.get('method')
self._modelType = kwargs.get('type')
self._input_ports = kwargs.get('input_ports')
self._args = args
self._kwargs = kwargs
self._qIn = kwargs.get('q').getQIn()
self._qOut = kwargs.get('q').getQOut()
self._qErr = kwargs.get('q').getQErr()
self._qOnSuccess = kwargs.get('q').getQOnSuccess()
self._qOnFailure = kwargs.get('q').getQOnFailure()
self._module = self.importModule()
self.lock = threading.RLock()
#self._loop = asyncio.get_event_loop()
self._loop = asyncio.new_event_loop()
asyncio.set_event_loop(self._loop)
@lru_cache(maxsize=2048)
def importModule(self):
return importlib.import_module(self._moduleName)
def getRandomQs(self,qIO,method,filterEmpty=True,getAll=False):
_queue=[]
for model,ports in qIO.items():
if (_queue and method == 'any'): break
for port,queueInit in ports.items():
_queues = list(map(lambda x:x.getObj(),queueInit))
if _queues:
if filterEmpty: #Filter empty queues for get
filtered_q = list(filter(lambda x:x.qsize() > 0, _queues))
else: # Leave empty queues for put
filtered_q = list(filter(lambda x:x.qsize() >= 0, _queues))
if filtered_q:
_queue.append({'model':model,'port':port,'q':filtered_q[random.randint(0,len(filtered_q)-1)]})
if (method == 'all'):
if list(set(_queues) - set(filtered_q)):
_queue.append({'model':model,'port':port,'q':list(set(_queues) - set(filtered_q))[0]})
elif (_queue and method == 'any'): break
return _queue
def getPut(self,qIO,method=None,**kwargs):
result = {}
if (method == 'put'):
for _queue in self.getRandomQs(qIO,method,filterEmpty=False):
_queue.get('q').put({(_queue.get('model'),_queue.get('port')):kwargs['item']})
self.logger.debug('PUT: Model - %-10s, Port - %-10s, Data - %s', _queue.get('model'), _queue.get('port'), kwargs['item'])
else:
for _queue in self.getRandomQs(qIO,method,filterEmpty=True):
result.update(_queue.get('q').get(timeout=1).get((_queue.get('model'),_queue.get('port'))))
self.logger.debug('GET: Model - %-10s, Port - %-10s, Data - %s', _queue.get('model'), _queue.get('port'), result)
if (method == 'any'):
break
if not result: #This is done for input_ports = any, to wait on a queue
for _queue in self.getRandomQs(qIO,method,filterEmpty=False):
result.update(_queue.get('q').get(timeout=1).get((_queue.get('model'),_queue.get('port'))))
self.logger.debug('GET: Model - %-10s, Port - %-10s, Data - %s', _queue.get('model'), _queue.get('port'), result)
if (method == 'any'):
break
return result
def getQIn(self):
return self._qIn
def getQOut(self):
return self._qOut
def getQErr(self):
return self._qErr
def getQOnSuccess(self):
return self._qOnSuccess
def getQOnFailure(self):
return self._qOnFailure
async def consumer(self,*args,**kwargs):
e_complete = kwargs.get('process_complete')
while True:
try:
_result = kwargs.get('resultQ').get(timeout=1)
self.logger.debug('CONSUMER: GET: Class - %-10s, Method - %-10s, Data - %s', self._className, self._methodName, _result)
if _result:
if (self._modelType == 'connection'):
if _result.get('OnFailure'):
self.getPut(self.getQOnFailure(),method='put',item=_result)
else:
self.getPut(self.getQOnSuccess(),method='put',item=_result)
else:
self.getPut(self.getQOut(),method='put',item=_result)
except Empty:
if e_complete.wait(1):
e_complete.clear()
break
else:
continue
def producer_api(self,*args,**kwargs):
self.logger.debug('PRODUCER_API: Class - %-10s, Method - %-10s, Send - %s', self._className, self._methodName, kwargs.get('taskKwargs'))
_result = getattr(kwargs.get('taskObject'),self._methodName)(kwargs.get('resultQ'),*kwargs.get('taskArgs'),**kwargs.get('taskKwargs'))
self.logger.debug('PRODUCER_API: Class - %-10s, Method - %-10s, Recv - %s', self._className, self._methodName, kwargs.get('taskKwargs'))
async def producer(self,*args,**kwargs):
e_complete = kwargs.get('process_complete')
e_complete.clear()
self.producer_api(self,*args,**kwargs)
e_complete.set()
def target(self,*args,**kwargs):
while True:
try:
_args = ()
_kwargs = {}
_object = None
resultQ = queue.Queue()
process_complete = threading.Event()
with self.lock:
_object = getattr(self._module,self._className)(*self._args,**self._kwargs)
try:
_kwargs = self.getPut(self.getQIn(),method=self._input_ports)
# Controller
tasks = [asyncio.ensure_future(self.producer(process_complete=process_complete,resultQ=resultQ,taskObject=_object,taskArgs=_args,taskKwargs=_kwargs,*args,**kwargs),loop=self._loop),asyncio.ensure_future(self.consumer(process_complete=process_complete,resultQ=resultQ,taskObject=_object,taskArgs=_args,taskKwargs=_kwargs,*args,**kwargs),loop=self._loop)]
result_producer, result_consumer=self._loop.run_until_complete(asyncio.gather(*tasks,return_exceptions = True))
except Empty:
continue
if (self._modelType == 'generator'):
break
except:
self.logger.info(sys.exc_info())
self.getPut(self.getQErr(),method='put',item=sys.exc_info())
def getTarget(self,*args,**kwargs):
return self.target