-
Notifications
You must be signed in to change notification settings - Fork 7
/
DataDef.py
8784 lines (8260 loc) · 563 KB
/
DataDef.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
###############################################################################
# Copyright (C) 2013-2018 Jacob Barhak
# Copyright (C) 2009-2012 The Regents of the University of Michigan
#
# This file is part of the MIcroSimulation Tool (MIST).
# The MIcroSimulation Tool (MIST) is free software: you
# can redistribute it and/or modify it under the terms of the GNU General
# Public License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
#
# The MIcroSimulation Tool (MIST) is distributed in the
# hope that it will be useful, but WITHOUT ANY WARRANTY; without even the
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
###############################################################################
#
# ADDITIONAL CLARIFICATION
#
# The MIcroSimulation Tool (MIST) is distributed in the
# hope that it will be useful, but "as is" and WITHOUT ANY WARRANTY of any
# kind, including any warranty that it will not infringe on any property
# rights of another party or the IMPLIED WARRANTIES OF MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. THE AUTHORS assume no responsibilities
# with respect to the use of the MIcroSimulation Tool (MIST).
#
# The MIcroSimulation Tool (MIST) was derived from the Indirect Estimation
# and Simulation Tool (IEST) and uses code distributed under the IEST name.
# The change of the name signifies a split from the original design that
# focuses on microsimulation. For the sake of completeness, the copyright
# statement from the original tool developed by the University of Michigan
# is provided below and is also mentioned above.
#
###############################################################################
############################ Original Copyright ###############################
###############################################################################
# Copyright (C) 2009-2012 The Regents of the University of Michigan
# Initially developed by Deanna Isaman, Jacob Barhak, Morton Brown, Wen Ye
#
# This file is part of the Indirect Estimation and Simulation Tool (IEST).
# The Indirect Estimation and Simulation Tool (IEST) is free software: you
# can redistribute it and/or modify it under the terms of the GNU General
# Public License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
#
# The Indirect Estimation and Simulation Tool (IEST) is distributed in the
# hope that it will be useful, but WITHOUT ANY WARRANTY; without even the
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
###############################################################################
# This means that throughout the program the division operator will be treated
# as float division.
from __future__ import division
# Import these libraries
import datetime
import re
import math
import os
import sys
import glob
import pickle
import zipfile
import parser
import StringIO
import tokenize
import csv
import types
import copy
import string
import tempfile
import shutil
if sys.version_info[0:2]<(2,7):
raise ValueError, 'System Initialization Error: The python version installed on this system does not meet the current version requirement of at least python version 2.6. Please install Python 2.7 or above and associated versions of the supporting libraries that correspond to this python version.'
SystemSupportsProcesses = sys.version_info[0:2]!=(2,5) and sys.platform != 'win32'
if SystemSupportsProcesses:
import multiprocessing
import numpy
import scipy.stats
import inspyred
# Since used by generated scripts, this generates a warning to remove in Spyder
RemoveWarningNotification = [scipy.stats == None, inspyred == None]
# update the module names
LoadedModuleNames = dir()
# Define the version of the data definitions file
# The string at the end will allow the system to distinguish between
# different revisions of a similar data structure than may or may not be
# compatible with each other. It is the responsibility of the code to
# check what code versions and revisions are compatible with it.
Version = (0,92,5,0,'MIST')
# DEBUG variables
# DebugPrints options are ['Table','TBD','Matrix', 'ExprParse','ExprConstruct', 'PrepareSimulation', 'Load' , 'MultiProcess', 'CSV', 'TempDir']
DebugPrints = []
MyDebugVar = {}
MessagesTBD = []
# First, some useful constants
FloatCompareTolerance = 1e-13 # Tolerance for comparing some floats for equality
Inf = float('inf') # May be replaced by numpy later on
NaN = float('nan') # May be replaced by numpy later on
inf = Inf # Duality needed to support both upper and lower case versions
nan = NaN # Duality needed to support both upper and lower case versions
InitTime = datetime.datetime(datetime.MINYEAR,1,1)
LowerCaseAlphabet = string.ascii_lowercase
UpperCaseAlphabet = string.ascii_uppercase
NumericCharacters = string.digits
AlphabetNoNumeric = LowerCaseAlphabet + UpperCaseAlphabet
AlphabetAndNumeric = AlphabetNoNumeric + NumericCharacters
ParamTextMatchPattern = '^[' + AlphabetNoNumeric + ']\w*$'
ParamUnderscoreAllowedTextMatchPattern = '^[' + AlphabetNoNumeric + '_]\w*$'
WhiteSpaces = string.whitespace
PythonSuffix = '.py'
TextSuffix = '.txt'
TempSuffix = '.tmp'
DefaultTempPathName = 'Temp'
DefaultTemporaryFileNamePrefix = 'Temp'
DefaultSimulationScriptFileNamePrefix = 'Sim'
DefaultSimulationOutputFileNamePrefix = 'SimOut'
DefaultGenerationScriptFileNamePrefix = 'Gen'
DefaultGenerationOutputFileNamePrefix = 'GenOut'
DefaultRandomStateFileNamePrefix = 'Rand'
DefaultFullResultsOutputFileName = 'SimulationResultsFull.csv'
DefaultFinalResultsOutputFileName = 'SimulationResultsFinal.csv'
DefaultSystemOptions = {'ValidateDataInRuntime':2, 'NumberOfErrorsConsideredAsWarningsForSimulation':200, 'NumberOfErrorsConsideredAsWarningsForPopulationGeneration':400, 'NumberOfTriesToRecalculateSimulationStep':5, 'NumberOfTriesToRecalculateSimulationOfIndividualFromStart':2, 'NumberOfTriesToRecalculateIndividualDuringPopulationGeneration':5, 'SystemPrecisionForProbabilityBoundCheck':1e-14 , 'RepairPopulation':1 , 'VerboseLevel': 5 , 'RandomSeed': NaN, 'GeneticAlgorithmCandidatesPerSelectedIndividual': 10, 'GeneticAlgorithmMaxEvalsTerminator': 7500, 'GeneticAlgorithmMaxStableGenerationCountTerminator':6, 'GeneticAlgorithmTournamentSize': 5, 'GeneticAlgorithmNumberOfElitesToSurviveIfBetterThanWorstOffspring': 15, 'GeneticAlgorithmSolutionPopulationSize': 100, 'GeneticAlgorithmMutationRate': 0.002}
StateIndicatorNotePrefix = 'A State Indicator Automatically Generated for State: '
ParameterTypes = ['Number','Integer','Expression','State Indicator','System Option','System Reserved']
ParamNameExtensitons = ['','_Entered']
ReportCalculationMethods = ['Auto Detect', 'Sum Over All Records', 'Average Over All Records', 'STD Over All Records', 'Min Over All Records', 'Max Over All Records', 'Valid Count of All Records', 'Sum Over Demographics', 'Average Over Demographics', 'STD Over Demographics', 'Min Over Demographics', 'Max Over Demographics', 'Valid Count of Demographics', 'Sum Over Last Observations Carried Forward', 'Average Over Last Observations Carried Forward', 'STD Over Last Observations Carried Forward', 'Min Over Last Observations Carried Forward', 'Max Over Last Observations Carried Forward', 'Valid Count of Last Observations Carried Forward', 'Record Count', 'Demographic Count', 'Last Value Count', 'Interval Start', 'Interval End', 'Interval Length', 'No Summary' ]
ReportCalculationMethodShortTitles = ['', 'Sum All', 'Avg All', 'STD All', 'Min All', 'Max All', 'Valid All', 'Sum Dem.', 'Avg Dem.', 'STD Dem.', 'Min Dem.', 'Max Dem.', 'Valid Dem.', 'Sum LOCF', 'Avg LOCF', 'STD LOCF', 'Min LOCF', 'Max LOCF', 'Valid LOCF', 'Rec Count', 'Dem. Count', 'Last Count', 'Start Step', 'End Step', 'Interval Length', '']
ReportStratificationHeader = 'Stratification - '
ReportStratificationDescriptionDict = {-1:'None: ', 1:'By initial demographics for cell: ', 2:'By entry demographics to time interval for cell: ', 3:'By record for cell: '}
StatFunctions = ['MEAN', 'STD', 'MEDIAN', 'MIN', 'MAX', 'SUM', 'COUNT'] + ['PERCENT%0.2i'%(Entry) for Entry in range(1,100)]
# A list of reserved words not to be used as parameter names and related
# the builtin reserved words are generated by:
# filter (lambda Entry: Entry[0]!='_', dir(__builtins__))
BannedSymbolsInExpression = ["'",'"','`','~','!','@','#','$','%','^','&',':',';','?','<','>','=','{','}', '//', '\n', '\x0b','\x0c','\r']
PythonReservedWords = ['and','del','from','not','while','as','elif','global','or','with','assert','else','if','pass','yield','break','except','import','print','class','exec','in', 'raise','continue','finally','is','return','def','for','lambda','try','self']
ProgramReservedWords = ['division','DataDef','sys','pickle','tempfile','os','random','args']
BuiltinReservedWords = ['ArithmeticError', 'AssertionError', 'AttributeError', 'BaseException', 'DeprecationWarning', 'EOFError', 'Ellipsis', 'EnvironmentError', 'Exception', 'False', 'FloatingPointError', 'FutureWarning', 'GeneratorExit', 'IOError', 'ImportError', 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError', 'KeyboardInterrupt', 'LookupError', 'MemoryError', 'NameError', 'None', 'NotImplemented', 'NotImplementedError', 'OSError', 'OverflowError', 'PendingDeprecationWarning', 'ReferenceError', 'RuntimeError', 'RuntimeWarning', 'StandardError', 'StopIteration', 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit', 'TabError', 'True', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError', 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError', 'UnicodeWarning', 'UserWarning', 'ValueError', 'Warning', 'WindowsError', 'ZeroDivisionError', 'abs', 'all', 'any', 'apply', 'basestring', 'bool', 'buffer', 'callable', 'chr', 'classmethod', 'cmp', 'coerce', 'compile', 'complex', 'copyright', 'credits', 'delattr', 'dict', 'dir', 'divmod', 'enumerate', 'eval', 'execfile', 'exit', 'file', 'filter', 'float', 'frozenset', 'getattr', 'globals', 'hasattr', 'hash', 'help', 'hex', 'id', 'input', 'int', 'intern', 'isinstance', 'issubclass', 'iter', 'len', 'license', 'list', 'locals', 'long', 'map', 'max', 'min', 'object', 'oct', 'open', 'ord', 'pow', 'property', 'quit', 'range', 'raw_input', 'reduce', 'reload', 'repr', 'reversed', 'round', 'set', 'setattr', 'slice', 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', 'type', 'unichr', 'unicode', 'vars', 'xrange', 'zip']
OtherReservedWords = ['oo','pi','E','MyLn']
StatisticalContinuousDistributionNames = ['Uniform','Gaussian']
StatisticalDiscreteDistributionNames = ['Bernoulli','Binomial','Geometric']
StatisticalDistributionNames = StatisticalContinuousDistributionNames + StatisticalDiscreteDistributionNames
Expr2SympyFunctionMapping = {'Exp':'exp','Log':'log','Ln':'ln','Pow':'pow','Sqrt':'sqrt'}
Sympy2ExprFunctionMapping = dict(zip(Expr2SympyFunctionMapping.values(), Expr2SympyFunctionMapping.keys()))
ExpressionSupportedFunctionsNames = StatisticalDistributionNames + sorted(Expr2SympyFunctionMapping.keys()) + ['Eq','Ne','Gr','Ge','Ls','Le','Or','And','Not','IsTrue','IsInvalidNumber','IsInfiniteNumber','IsFiniteNumber','Iif','CostWizard','Log10','Pi','Mod','Abs','Floor','Ceil','Max','Min']
RuntimeFunctionNames = [('Table','TableRunTime')]
ExpressionSupportedFunctionsAndSpecialNames = ExpressionSupportedFunctionsNames + ['Inf','NaN','inf','nan'] + reduce(lambda TheList,Entry: TheList + [Entry[0]], RuntimeFunctionNames,[])
AllowedMathOperatorsInExpr = ['+','-','/','*','**']
##SystemReservedParametersToBeCreated = ['Time', 'IndividualID', 'Dummy', 'Repetition', 'AllCoefficients', 'BlankColumn', 'Inf', 'NaN','inf','nan']
SystemReservedParametersToBeCreated = ['Time', 'IndividualID', 'Dummy', 'Repetition', 'BlankColumn', 'Inf', 'NaN','inf','nan']
SystemReservedParametersAllowedInPopulationColumns = ['Dummy']
GlobalsInDB = ['Version', 'Params' , 'States' , 'StudyModels' , 'Transitions' , 'PopulationSets' , 'Projects' , 'SimulationResults']
ClassDescriptionDict = {'ParamsClass':'Parameters Collection' , 'StatesClass':'States Collection' , 'StudyModelsClass':'Studies/Models Collection' , 'TransitionsClass':'Transitions Collection' , 'PopulationSetsClass':'Population Sets Collection' , 'ProjectsClass':'Projects Collection' , 'SimulationResultsClass':'Simulation Results Collection', 'Param':'Parameter' , 'State':'State' , 'StudyModel':'Study/Model' , 'Transition':'Transition' , 'PopulationSet':'Population Set' , 'Project':'Project' , 'SimulationResult':'Simulation Result', 'Expr':'Expression' , 'SimulationRule':'Simulation Rule'}
AppFileNameExtension = '.txt'
EmptyEvalDict = {'__builtins__':'','Inf':Inf,'NaN':NaN,'inf':Inf,'nan':NaN}
# Define Load Options Evaluation Disctionary
# Start with the empty evaluation dictionary
LoadOptionsEvalDict = copy.deepcopy(EmptyEvalDict)
# Add True and False
LoadOptionsEvalDict['True']=True
LoadOptionsEvalDict['False']=False
# Define and Mark the database dirty status flag
# Careful do not access directly, use AccessDirtyStatus instead
DirtyStatusFlag = [False]
# Also some useful functions
IsStr = lambda x: isinstance(x,types.StringType)
IsList = lambda x: isinstance(x,types.ListType)
IsTuple = lambda x: isinstance(x,types.TupleType)
IsFloat = lambda x: isinstance(x,types.FloatType)
IsBoolean = lambda x: isinstance(x,types.BooleanType)
IsInt = lambda x: isinstance(x,types.IntType)
IsNumericType = lambda x: IsFloat(x) or IsInt(x)
IsReducedNumeric = lambda x: (IsFloat(x) or IsInt(x)) and (not IsBoolean(x))
IsDict = lambda x: isinstance(x,types.DictionaryType)
IsNone = lambda x: isinstance(x,types.NoneType)
IsNaN = lambda x: (x==x)==False
IsInf = lambda x: x==Inf or x ==-Inf
IsFinite = lambda x: IsNumericType(x) and not IsNaN(x) and not IsInf(x)
IsClass = lambda x: isinstance(x,types.ClassType)
IsInstance = lambda x: isinstance(x,types.InstanceType)
IsFunction = lambda x: isinstance(x,types.FunctionType)
IsMethod = lambda x: isinstance(x,types.MethodType)
IsInstanceOf = lambda x,y: x.__class__.__name__==y
IsInstanceOfLib = lambda x,y: x.__class__.__module__.startswith(y)
# Transfers ill defined value to zero
Ill2Zero = lambda x: Iif(IsNaN(x) or not IsNumericType(x), 0, x)
def RaiseNaN (InList):
""" Returns NaN if any member IsNaN, returns the first member otherwise """
# This function is used to detect NaN in a list of operators. It is useful
# for raising NaN if any of the inputs/result is NaN. Otherwise it returns
# the result that should be the first parameter in the list.
if any(map(IsNaN,InList)):
return NaN
else:
return InList[0]
# Useful functions for reduce
AndOp = lambda x,y: x and y
OrOp = lambda x,y: x or y
NotOp = lambda x: not x
XorOp = lambda x,y: (x or y) and (not (x and y))
SumOp = lambda x,y: x + y
SubOp = lambda x,y: x - y
MultOp = lambda x,y: x * y
PowOp = lambda x,y: x ** y
IsEqualOp = lambda x,y: x == y
IsHigherOp = lambda x,y: x > y
IsLowerOp = lambda x,y: x < y
IsHigherOrEqualOp = lambda x,y: x >= y
IsLowerOrEqualOp = lambda x,y: x <= y
ProdOp = lambda x: reduce(MultOp, x, 1)
# Useful list operators
NotList = lambda ListName: map ( NotOp , ListName )
SetDiff = lambda x,y: list(set(x)-set(y))
try:
FloatInfo = numpy.finfo(float)
except:
print 'Warning: some mathematical options do not work properly unless Numpy is installed and properly and loaded to the system.'
# Other important values
# Define the temp directory class
class TempDirectoryClass(str):
""" retains the temp directory name """
def __new__ ( cls, UseDefaultPathName = True, NamePrefix = None, NameSuffix = None, CreateUnderThisDirectory = None):
"""Constructor to allow creation of a non mutable object"""
# Upon creation of the instance, a new directory may require creation
# By default the system will use the DefaultTempPathName and will
# create this directory relative to the working directory if it does
# not exist yet.
# However, if UseDefaultPathName is set to False then a new temporary
# directory will be created and returned. This option is not currently
# used, yet may be a possible future use.
if CreateUnderThisDirectory == None:
CreateUnderThisDirectory = os.path.dirname(sys.argv[0])
if NamePrefix == None:
NamePrefix = DefaultTempPathName
if NameSuffix == None:
NameSuffix = ''
if UseDefaultPathName:
# This is the default option, where the system path will be the temp
# directory below the installation directory of the running program.
PathName = os.path.join(CreateUnderThisDirectory, DefaultTempPathName)
if not os.path.isdir(PathName):
try:
os.makedirs(PathName)
except:
# Check if this directory was not created by another process
# while trying to do this from this process. It actually
# does not matter who creates the directory as long as it
# is created. So if someone else created it, ignore the
# error
if not os.path.isdir(PathName):
raise ValueError, 'Return Path: Cannot recreate the directory ' + repr(PathName) + '. Please check if this path does not already exist and if there are sufficient privileges to create this path'
else:
# This is the non default option where a new temporary directory is
# created using Pythons tempfile library
PathName = tempfile.mkdtemp(NameSuffix, NamePrefix, CreateUnderThisDirectory)
return PathName
# Now actually create the temp directory for this session which is a global
# instance that will later be used in many places.
SessionTempDirecory = TempDirectoryClass()
if 'TempDir' in DebugPrints:
print 'Temp Directory is:'
print SessionTempDirecory
def RedirectOutputToValidFile(TheFile , ExchangeFileName = None):
""" Find redirect for TheFile to a real file if not suitable for output """
# The function should be called in the following manner:
# (TheFile, BackupOfOriginal) = RedirectOutputToValidFile(TheFile,...)
# The function returns a tuple
# (RetFile, BackupOfOriginal)
# if not redirected, RetFile = OldFile = TheFile
# If redirected, RetFile != OldFile both representing TheFile before and
# after redirection. Note that actual redirection does not happen within
# the function itself.
# Note that redirection back to the original file may be required
# using the following scheme:
# RedirectOutputBackwards(TheFile, RetFile, BackupOfOriginal)
# With proper use, this function allows programs that print to stdandard
# streams to be used when invoked in Idle, Python and Pythonw environments
# and work properly by internal redirection of the stream to file
(RetFile, BackupOfOriginal) = (TheFile, TheFile)
NeedToCreateFile = False
if not IsInstanceOfLib (TheFile, 'idlelib.rpc'):
# Check if the file is created by the idle environment
# if so, it is ok to use it
# Otherwise, check if the file is an actual os file.
# first check if it has a file open:
if not hasattr(TheFile,'fileno'):
# try to create the file
NeedToCreateFile = True
else:
try:
# try to access file status
os.fstat(TheFile.fileno())
except:
# if could not access statistics, this is an invalid file
NeedToCreateFile = True
if NeedToCreateFile:
# if there is a need to create a file
if ExchangeFileName == None:
# if not filename is given, create a temp file
(FileDescriptor, FileName) = tempfile.mkstemp ( TextSuffix, DefaultTemporaryFileNamePrefix , SessionTempDirecory , True)
RetFile = os.fdopen(FileDescriptor,'w')
else:
RetFile = open(ExchangeFileName,'w')
return (RetFile, BackupOfOriginal)
def RedirectOutputBackwards(TheFile, BackupOfOriginal):
""" Redirect file backwards using backup """
# This function should be called in the following manner:
# TheFile = RedirectOutputBackwards(TheFile, BackupOfOriginal)
# The function complements the following function call:
# (TheFile, BackupOfOriginal) = RedirectOutputToValidFile(TheFile,...)
# it restores TheFile to the original file and closes a newly created file
# if it was created. Note that acrual restoration does not take place
# within the function, it happens outside the call by the back arrignment.
# always return the backup
RetVal = BackupOfOriginal
if TheFile != BackupOfOriginal:
# if a file was open during redirection, close it
TheFile.close()
return RetVal
def FilePatternMatchOptimizedForNFS(FilePattern):
"An optimized version of glob optimized for NFS by skip dir list for file"
# This function behaves almost exactly like glob. However, if no wildcards
# are found in the pattern name, then the system skips retrieving the
# entire directory and just checks if the file exists. Otherwise the
# system will just use glob. This is important when accessing files over
# NFS in cases where there are many files in a directory and they need to
# be transferred to different machines. The problem intensifies over large
# cluster runs and becomes a bottleneck - therefore there is a need to
# avoid it.
# the return is always a list of filenames that match the pattern.
# The following wildcards in the file pattern will indicated that the slow
# version of glob has to used: *?[]!. Otherwise it is assumed that
# the file pattern has only a single file.
#
IsWildCardInPattern = False
WildCardList = '*?[]!'
for WildChar in WildCardList:
if WildChar in FilePattern:
IsWildCardInPattern = True
break
if IsWildCardInPattern:
# if wildcards found use glob
RetVal = glob.glob(FilePattern)
else:
# otherwise just check if the file exists and return a list
if os.path.exists(FilePattern):
# Note that os.path.exists is used rather than os.path.isfile
# so the behavior will match glob that may return a directory
# return the file name if found
RetVal = [FilePattern]
else:
# return an empty list if not found
RetVal = []
return RetVal
# Useful string functions
def RemoveChars(InStr, RemoveStr = None):
"""Removes InStr characters listed in RemoveStr / whitespaces by default"""
# If no characters are requested, use the whitespace list
if RemoveStr == None:
RemoveStr=WhiteSpaces
OutStr=InStr
# For each character in the to remove list, replace it with nothing
for Char in RemoveStr:
OutStr = OutStr.replace(Char,'')
return OutStr
def SmartStr(Value):
""" returns a descriptive str that recognizes values such as NaN and inf """
if IsStr(Value):
TempStr = Value
else:
TempStr = repr(Value)
for (StringToReplace) in ['Inf','NaN','-NaN']:
# Note that -NaN may have a different output and is therefore considered
# seperatly that NaN. In an attempt to be system independent, these
# strings are evaluated online.
ValueToReplace = repr(eval(StringToReplace, EmptyEvalDict))
TempStr = TempStr.replace(ValueToReplace, StringToReplace)
return TempStr
# Other useful query functions
def RepVal (Statement, ValueToReplace = None, ReplacementValue = 0):
""" Replace Statement with ReplacementValue if equals ValueToReplace """
if Statement == ValueToReplace:
return ReplacementValue
else:
return Statement
def FilterByAnother (DataSequence, BoolSequence):
""" Filter the DataSequence by BoolSequence , keeping True members"""
Combined = zip(DataSequence , BoolSequence)
FilteredTuple = filter (lambda (Data , Bool) : Bool , Combined)
Result = map (lambda (Data , Bool) : Data , FilteredTuple)
return Result
def FindDuplicatesInSequence(Sequence):
""" Returns a list of items that are duplicated in a sequence"""
Duplicates = []
for (ItemIndex, Item) in enumerate(Sequence):
if Item in Sequence[:(ItemIndex)] or Item in Sequence[(ItemIndex+1):]:
Duplicates = Duplicates + [Item]
Duplicates = list(set(Duplicates))
return Duplicates
def TBD( Message = "This code is yet to be constructed"):
"""Print a message that this code is yet to be constructed"""
global MessagesTBD
if Message == '':
for RecordedMessage in MessagesTBD:
print RecordedMessage
if Message not in MessagesTBD:
MessagesTBD = MessagesTBD + [Message]
if 'TBD' in DebugPrints:
print 'TBD: ' + Message
return
def DetermineFileNameAndPath(FileName):
"""Determines file name and path, uses current path if undefined"""
(OriginalPathOnly , FileNameOnly) = os.path.split (FileName)
# Make the path absolute to be exact
PathOnly = os.path.abspath (OriginalPathOnly)
FileNameFullPath = os.path.join(PathOnly , FileNameOnly)
return (PathOnly , FileNameOnly, FileNameFullPath)
# Programmatic support
class ClassFunctionWrapper():
"""Class wrapper for a function. Allows calling class without an instance"""
def __init__(self, FunctionToCall):
self.__call__ = FunctionToCall
return
# Create a temporary class to pass data around as if using a pipe
class PipeMock():
Data = None
def send(self,DataToSend):
self.Data = DataToSend
return
def recv(self):
return self.Data
def close(self):
self.Data = None
def RunFunctionAsProcess(FunctionToRun):
""" If possible, run the function as a process, return pipe"""
if not (SystemSupportsProcesses):
# Handle the serial executions case
OutputConnection = PipeMock()
if 'MultiProcess' in DebugPrints:
print 'running the function'
OutputConnection = PipeMock()
RetVal = FunctionToRun()
OutputConnection.send(RetVal)
if 'MultiProcess' in DebugPrints:
print 'finished running the function'
# store the result, errors will be handled later
PipeList = [OutputConnection]
ProcessList = None
else:
def RunAsProcess(ChildConnection):
RetVal = FunctionToRun()
ChildConnection.send(RetVal)
return RetVal
# Handle the parallel process execution case
# create connections
(ParentConnenction, ChildConnection) = multiprocessing.Pipe()
PipeList = [ParentConnenction]
# create processes
if 'MultiProcess' in DebugPrints:
print 'spawning a process'
TheProcess = multiprocessing.Process(target = RunAsProcess, args = (ChildConnection,))
ProcessList = [TheProcess]
if 'MultiProcess' in DebugPrints:
print 'process spawned'
# Now actually start running the process
TheProcess.start()
# return the Process and pipe list to be collected by CollectResults
return (ProcessList, PipeList)
def IsEqualDetailed (Arg1, Arg2, AttributesToSkip = None):
""" A generic function to compare data in complex structures """
if type(Arg1) != type(Arg2):
# Arguments of different types mean inequality
RetVal = False
elif IsMethod(Arg1) or IsFunction(Arg1) or IsClass(Arg1):
# If the attribute is a method, function or instance then
# no comparison is required
RetVal = True
elif IsInstance(Arg1):
# If these are instances, Check all attributes
# The default of attributes to skip are the book keeping attributes
if AttributesToSkip == None:
AttributesToSkip = ['ID', 'CreatedOn', 'LastModified']
# Assume instances are equal
RetVal = True
# Get a list of attributes of both objects
SelfAttributes = dir(Arg1)
OtherAttributes = dir(Arg2)
# If the attribute list is not the same then skip further comparison
if SelfAttributes == OtherAttributes:
# For every attribute that is not in the skip list
for Attr in SelfAttributes:
if Attr not in AttributesToSkip:
Element1 = getattr(Arg1,Attr)
Element2 = getattr(Arg2,Attr)
ElementEqual = IsEqualDetailed (Element1, Element2, AttributesToSkip)
# If comparison failed, skip checking more elements
if not ElementEqual:
RetVal = False
break
else:
RetVal = False
elif IsDict(Arg1):
# For dictionaries convert these into lists and continue comparison
ConvertArg1 = map(None, Arg1.iteritems())
ConvertArg2 = map(None, Arg2.iteritems())
RetVal = IsEqualDetailed (ConvertArg1, ConvertArg2, AttributesToSkip)
elif IsList(Arg1) or IsTuple(Arg1):
# If this is a list or a tuple, continue comparison per list element
if len(Arg1) != len(Arg2):
# Different sizes mean different lists
RetVal = False
else:
# Compare all elements by looping
RetVal = True
for Element1, Element2 in zip(Arg1, Arg2):
ElementEqual = IsEqualDetailed (Element1, Element2, AttributesToSkip)
# If comparison failed, skip checking more elements
if not ElementEqual:
RetVal = False
break
elif IsNumericType(Arg1) and IsNaN(Arg1):
# Special care should be taken if NaN are compared
RetVal = IsNaN(Arg2)
elif IsNumericType(Arg1) or IsStr(Arg1) or IsNone(Arg1):
# All other types just check equality
RetVal = Arg1 == Arg2
else:
raise ValueError, 'Assertion Error - Unsupported argument type encountered: ' + str(type(Arg1))
# Return the result
return RetVal
def CalcCopyName(Object, NewName = None, BaseName = None):
""" Calculates an appropriate name for copying """
if NewName == None:
# If no specific name was specified, find an appropriate name
# Before that collect the existing names
if IsStr(Object) and not IsInstanceOf(Object,'Param'):
CollectionName = Object
else:
CollectionName = Object.__class__.__name__ + 's'
Collection = globals()[CollectionName]
if CollectionName == 'Params':
# Treat parameters differently since they do not have a
# name attribute
Names = Collection.keys()
if BaseName == None:
BaseName = str(Object)
else:
# All other classes are the same
Names = map(lambda Entry: Entry.Name, Collection.values())
if BaseName == None:
BaseName = str(Object.Name)
# If no name is specified, select a name that does not
# already exist. For this loop until a name is found.
Index = 0
while True:
NewName = BaseName + '_' + str(Index)
if NewName in Names:
Index = Index + 1
else:
# This means a name has been found
break
# Return the calculated or specified name
return NewName
def DescribeReturnsName(self):
""" Return the Name attribute of the class"""
RetVal = self.Name
return RetVal
def DependancyErrorCheck(self, DependancyCollection, DependencyDefFunc, ErrorHeaderString):
DependancyCollectionItems = DependancyCollection.items()
IsDependant = map(DependencyDefFunc, DependancyCollectionItems)
if any(IsDependant):
DependentEntries = FilterByAnother(DependancyCollectionItems, IsDependant)
DependentKeys = map (lambda (EntryKey,Entry): EntryKey, DependentEntries )
ErrorDetailString = DependancyCollection.ID2Name(DependentKeys)
raise ValueError, ErrorHeaderString + ErrorDetailString
# Define system reserved functions
# Boolean and comparison Operators
# Although quite safe, avoid using Python True and False outcomes
# Also note that no special NaN treatment is added to comparison operations
def Eq(x,y):
""" Replaces the equality Operator x==y """
if x==y:
return 1
else:
return 0
def Ne(x,y):
""" Replaces the inequality Operator x!=y """
if x!=y:
return 1
else:
return 0
def Gr(x,y):
""" Replaces the Greater than Operator x>y """
if x>y:
return 1
else:
return 0
def Ge(x,y):
""" Replaces the Greater than or Equal Operator x>=y """
if x>=y:
return 1
else:
return 0
def Ls(x,y):
""" Replaces the Less than Operator x<y """
if x<y:
return 1
else:
return 0
def Le(x,y):
""" Replaces the Less than or Equal Operator x<=y """
if x<=y:
return 1
else:
return 0
# Define Boolean operations
# Note that Ill argument such as NaN are considered as zero
def Or(*ArgList):
""" Replaces the Python 'or' operator """
Res = False
ArgNum = len(ArgList)
if ArgNum < 2:
raise ValueError, 'The Or operator requires at least 2 parameters, whereas ' + str(ArgNum) + ' are provided'
for Arg in ArgList:
Res = Res or Ill2Zero(Arg)
if Res:
return 1
else:
return 0
def And(*ArgList):
""" Replaces the Python 'and' operator """
Res = True
ArgNum = len(ArgList)
if ArgNum < 2:
raise ValueError, 'The And operator requires at least 2 parameters, whereas ' + str(ArgNum) + ' are provided'
for Arg in ArgList:
Res = Res and Ill2Zero(Arg)
if Res:
return 1
else:
return 0
def Not(x):
""" Replaces the Python 'not' operator """
if not Ill2Zero(x):
return 1
else:
return 0
def IsTrue(x):
""" Returns 1 for a non zero number. Returns 0 otherwise """
if Ill2Zero(x):
return 1
else:
return 0
# Define Special Number functions exposed to the user:
# Note that non numeric types such as matrices have special consideration
def IsInvalidNumber(x):
""" Will return 1 for x=NaN or for a non numeric type, 0 otherwise """
if not IsNumericType(x) or IsNaN(x):
return 1
else:
return 0
def IsInfiniteNumber(x):
""" Will return 1 for x=-Inf or x=Inf, 0 otherwise """
if x==Inf or x==-Inf:
return 1
else:
return 0
def IsFiniteNumber(x):
""" Will return 0 if x is invalid or an Infinite number, 1 otherwise """
if not IsInvalidNumber(x) and not IsInfiniteNumber(x):
return 1
else:
return 0
# Control and Data Access
def Iif (Statement, TruePart, FalsePart):
"""Immidiate if: if statement is true return TruePart otherwise FalsePart"""
if Statement:
return TruePart
else:
return FalsePart
def TableParseOnly(*ArgList):
""" Parses a table definition - return the class """
# First see if the Arguments to the table are of 2 lists. if so, then
# the new method of calling is used and there is no need to reconstruct
# any string.
if len(ArgList)==2 and IsList(ArgList[0]) and IsList(ArgList[1]):
# if using the new format, just call the funcion with the arguments
TempTable = TableClass(DimensionsArray = ArgList[0], ValuesArray = ArgList[1])
else:
# Recostruct the Argument list string while replacing back Inf and NaN
# This is requires since the call to the function already replaced
# Inf and NaN with values and string conversion will create artifacts.
# Since the table class requires string representation, this processing
# is required to provide proper input. The function returns the constructed
# Table object
ArgString = str(ArgList)
ArgString = ArgString.replace(str(Inf),'Inf')
ArgString = ArgString.replace(str(-Inf),'(-Inf)')
ArgString = ArgString.replace(str(NaN),'NaN')
try:
# Try to create a table class with the input parameters. If unsuccessful
# raise an error
TempTable = TableClass( InitString = 'Table'+ArgString, ParseOnly = True)
except:
(ExceptType, ExceptValue, ExceptTraceback) = sys.exc_info()
raise ValueError, 'Table Parser Error: The table cannot be evaluated. Check the number and the validity of parameters. Here are additional details:' + str(ExceptValue)
return TempTable
def Table(*ArgList):
""" Parses a table definition - Validation Time version"""
# Just call the parser, an error will be raised if there is a problem
TableParseOnly(*ArgList)
# return a number
return 0
def TableRunTime(*ArgList):
""" Run Time version - parses parameters and returns a value """
# Call the parser to get a table object
TableObject = TableParseOnly(*ArgList)
# Evaluate the table value
Value = TableObject.Evaluate()
# return a number
return Value
def CostWizard (FunctionType, InitialValue, ParameterVector, CoefficientVector):
""" Calculate costs and quality of Life using this function """
# The function uses the approach presented in the following paper:
# Zhou H, Isaman DJM, Messinger S, Brown MB, Klein R, Brandle M, Herman WH:
# A Computer Simulation Model of Diabetes Progression, Quality of Life, and
# Cost. Diabetes Care 28:2856-2863, 2005
# The function Input is:
# FunctionType: For a Cost function it is 0, for Qualify of Life it is 1.
# InitialValue: The base value
# ParameterVector : A sequence of parameter values
# CoefficientVector : A sequence of coefficient values associated with the
# parameters in ParameterVector and should be the same
# length of the sequence.
# Check input validity first
if FunctionType not in [0,1]:
raise ValueError, 'Cost Wizard parameter 1 invalid: Unknown Function Type provided for the Cost Wizard'
if not IsFinite(InitialValue):
raise ValueError, 'Cost Wizard parameter 2 invalid: Initial value is not a valid finite number'
if not IsList(ParameterVector):
raise ValueError, 'Cost Wizard parameter 3 invalid: The Parameter Vector must be a List, i.e. enclosed in brackets'
if not IsList(CoefficientVector):
raise ValueError, 'Cost Wizard parameter 4 invalid: The Coefficient Vector must be a List, i.e. enclosed in brackets'
if len(ParameterVector) != len(CoefficientVector):
raise ValueError, 'Cost Wizard Parameters 3,4. It is required that the parameter vector and the coefficient vector will be the same size'
for Member in ParameterVector:
if not IsFinite(Member):
raise ValueError, 'Cost Wizard parameter 3: An invalid member detected in the parameter vector. This member is not a valid finite number.'
for Member in CoefficientVector:
if not IsFinite(Member):
raise ValueError, 'Cost Wizard parameter 4: An invalid member detected in the coefficient vector. The coefficient is not a valid finite number'
# Calculate the dot product of both vectors
SumOfMult = reduce(SumOp,map(MultOp,ParameterVector,CoefficientVector),0.0)
if FunctionType == 0:
# Cost functions
Result = InitialValue * 10.0**SumOfMult
elif FunctionType == 1:
# QOL function
Result = InitialValue + SumOfMult
else:
raise ValueError, 'Cost Wizard Error: The cost wizard does not support this type of cost function.'
return Result
def Exp(x):
""" Exponent wrapper for expression resolution"""
try:
RetVal = math.exp(x)
except:
# Return NaN in case of an error
RetVal = NaN
return RetVal
def Log(x,n):
""" Logarithm wrapper for expression resolution"""
try:
RetVal = math.log(x,n)
except:
# Return NaN in case of an error
RetVal = NaN
return RetVal
def Ln(x):
""" Natural Logarithm wrapper for expression resolution"""
try:
RetVal = math.log(x)
except:
# Return NaN in case of an error
RetVal = NaN
return RetVal
def Log10(x):
""" Logarithm base 10 wrapper for expression resolution"""
try:
RetVal = math.log10(x)
except:
# Return NaN in case of an error
RetVal = NaN
return RetVal
def Pow(x,n):
""" Power wrapper for expression resolution"""
try:
RetVal = math.pow(x,n)
except:
# Return NaN in case of an error
RetVal = NaN
return RetVal
def Sqrt(x):
""" Square root wrapper for expression resolution"""
try:
RetVal = math.sqrt(x)
except:
# Return NaN in case of an error
RetVal = NaN
return RetVal
def Pi():
""" Pi wrapper for expression resolution"""
try:
RetVal = math.pi
except:
# Return NaN in case of an error
RetVal = NaN
return RetVal
def Mod(x,n):
""" Modulus wrapper for expression resolution"""
try:
RetVal = math.fmod(x,n)
except:
# Return NaN in case of an error
RetVal = NaN
return RetVal
def Abs(x):
""" Absolute value wrapper for expression resolution"""
try:
RetVal = math.fabs(x)
except:
# Return NaN in case of an error
RetVal = NaN
return RetVal
def Floor(x):
""" Floor wrapper for expression resolution"""
try:
RetVal = math.floor(x)
except:
# Return NaN in case of an error
RetVal = NaN
return RetVal
def Ceil(x):
""" Ceiling wrapper for expression resolution"""
try:
RetVal = math.ceil(x)
except:
# Return NaN in case of an error
RetVal = NaN
return RetVal
def Max(*ArgList):
""" Maximum wrapper for expression resolution"""
# Note that keyword argument key prohibits from using the max option
# of using a keyword key to define a function
if len (ArgList) < 2:
raise ValueError, 'Max function: At least two input arguments are required to calculate a maximum value'
try:
# Note that NaNs will cause returning NaN
if any(map(IsInvalidNumber,ArgList)):
RetVal = NaN
else:
RetVal = max(ArgList)
except:
# Return NaN in case of an error
RetVal = NaN
return RetVal
def Min(*ArgList):
""" Minimum wrapper for expression resolution"""
# Note that keyword argument key prohibits from using the min option
# of using a keyword key to define a function
if len (ArgList) < 2:
raise ValueError, 'Min function: At least two input arguments are required to calculate a minimum value'
try:
# Note that NaNs will cause returning NaN
if any(map(IsInvalidNumber,ArgList)):
RetVal = NaN
else:
RetVal = min(ArgList)
except:
# Return NaN in case of an error
RetVal = NaN
return RetVal
# Statistical distributions
# Statistical distribution functions have two usage modes, if the value x is
# provided as input then the function will return the cdf for the given x value.
# When x is not provided, then function returns a random value according to the
# specified distribution and its parameters.
# Statistical functions rely on the Scipy module being loaded.
def Bernoulli(p):
""" Bernoulli distribution with chance p """
try:
# Random Generator