diff --git a/bin/ma5 b/bin/ma5 index f7443f59..518c59da 100755 --- a/bin/ma5 +++ b/bin/ma5 @@ -33,7 +33,7 @@ This is the main executable, a simple frontend to set up the PYTHONPATH and call immediately the command line interface scripts """ -import importlib +import importlib.util import os import sys diff --git a/madanalysis/IOinterface/job_writer.py b/madanalysis/IOinterface/job_writer.py index c5fae488..8350776a 100644 --- a/madanalysis/IOinterface/job_writer.py +++ b/madanalysis/IOinterface/job_writer.py @@ -705,6 +705,9 @@ def WriteSampleAnalyzerMakefile(self,option=""): # Options option.has_commons = True options.has_process = True + if self.main.archi_info.has_onnx: + options.has_onnx_inc = True + options.has_onnx_lib = True if self.main.archi_info.has_root: options.has_root_inc = True options.has_root_lib = True @@ -740,6 +743,9 @@ def WriteMakefiles(self,option=""): # Options options.has_commons = True options.has_process = True + if self.main.archi_info.has_onnx: + options.has_onnx_inc = True + options.has_onnx_lib = True if self.main.archi_info.has_root: options.has_root_inc = True options.has_root_lib = True diff --git a/madanalysis/IOinterface/library_writer.py b/madanalysis/IOinterface/library_writer.py index eb3ff3ef..5f3fd716 100644 --- a/madanalysis/IOinterface/library_writer.py +++ b/madanalysis/IOinterface/library_writer.py @@ -132,6 +132,8 @@ def WriteMakefileForInterfaces(self,package): filename = self.path+"/SampleAnalyzer/Test/Makefile_delphesMA5tune" elif package=='test_root': filename = self.path+"/SampleAnalyzer/Test/Makefile_root" + elif package=='test_onnx': + filename = self.path+"/SampleAnalyzer/Test/Makefile_onnx" # Header title='' @@ -155,6 +157,8 @@ def WriteMakefileForInterfaces(self,package): title='*delphesMA5tune-interface* test' elif package=='test_root': title='*root-interface* test' + elif package=='test_onnx': + title='*onnx-interface* test' else: title='interface to '+package @@ -190,6 +194,18 @@ def WriteMakefileForInterfaces(self,package): # options.has_zlib_lib = True toRemove.extend(['compilation_zlib.log','linking_zlib.log','cleanup_zlib.log',\ 'mrproper_zlib.log','../Bin/TestZlib.log']) + elif package=='onnx': + options.has_commons = True + options.has_onnx_inc = True + options.has_onnx_lib = True + toRemove.extend(['compilation_onnx.log','linking_onnx.log','cleanup_onnx.log','mrproper_onnx.log']) + elif package=='test_onnx': + options.has_commons = True + #options.has_onnx_ma5lib = True + options.has_onnx_inc = True + options.has_onnx_lib = True + toRemove.extend(['compilation_onnx.log','linking_onnx.log','cleanup_onnx.log',\ + 'mrproper_onnx.log','../Bin/TestOnnx.log']) elif package=='delphes': options.has_commons = True options.has_delphes_inc = True @@ -260,6 +276,8 @@ def WriteMakefileForInterfaces(self,package): options.has_zlib_tag = self.main.archi_info.has_zlib options.has_root_tag = self.main.archi_info.has_root options.has_root_ma5lib = self.main.archi_info.has_root + options.has_onnx_tag = self.main.archi_info.has_onnx + options.has_onnx_ma5lib = self.main.archi_info.has_onnx toRemove.extend(['compilation.log','linking.log','cleanup.log','mrproper.log']) elif package=='test_process': options.has_commons = True @@ -304,6 +322,9 @@ def WriteMakefileForInterfaces(self,package): elif package=='test_root': cppfiles = ['Root/*.cpp'] hfiles = ['Root/*.h'] + elif package=='test_onnx': + cppfiles = ['Onnx/*.cpp'] + hfiles = ['Onnx/*.h'] else: cppfiles = [package+'/*.cpp'] hfiles = [package+'/*.h'] @@ -333,6 +354,10 @@ def WriteMakefileForInterfaces(self,package): isLibrary=False ProductName='TestRoot' ProductPath='../Bin/' + elif package=='test_onnx': + isLibrary=False + ProductName='TestOnnx' + ProductPath='../Bin/' elif package=='test_delphes': isLibrary=False ProductName='TestDelphes' @@ -364,7 +389,7 @@ def Compile(self,ncores,package,folder): if package in ['process','commons','test','configuration']: logfile = folder+'/compilation.log' elif package in ['test_process','test_commons','test_zlib','test_fastjet',\ - 'test_root','test_delphes','test_delphesMA5tune']: + 'test_root','test_delphes','test_delphesMA5tune','test_onnx']: logfile = folder+'/compilation_'+package[5:]+'.log' else: logfile = folder+'/compilation_'+package+'.log' @@ -373,7 +398,7 @@ def Compile(self,ncores,package,folder): if package in ['process','commons','test','configuration']: makefile = 'Makefile' elif package in ['test_process','test_commons','test_zlib','test_fastjet',\ - 'test_root','test_delphes','test_delphesMA5tune']: + 'test_root','test_delphes','test_delphesMA5tune','test_onnx']: makefile = 'Makefile_'+package[5:] else: makefile = 'Makefile_'+package @@ -401,7 +426,7 @@ def Link(self,package,folder): if package in ['process','commons','test','configuration']: logfile = folder+'/linking.log' elif package in ['test_process','test_commons','test_zlib','test_fastjet',\ - 'test_root','test_delphes','test_delphesMA5tune']: + 'test_root','test_delphes','test_delphesMA5tune','test_onnx']: logfile = folder+'/linking_'+package[5:]+'.log' else: logfile = folder+'/linking_'+package+'.log' @@ -410,7 +435,7 @@ def Link(self,package,folder): if package in ['process','commons','test','configuration']: makefile = 'Makefile' elif package in ['test_process','test_commons','test_zlib','test_fastjet',\ - 'test_root','test_delphes','test_delphesMA5tune']: + 'test_root','test_delphes','test_delphesMA5tune','test_onnx']: makefile = 'Makefile_'+package[5:] else: makefile = 'Makefile_'+package @@ -435,7 +460,7 @@ def Clean(self,package,folder): if package in ['process','commons','configuration','test']: logfile = folder+'/cleanup.log' elif package in ['test_process','test_commons','test_zlib','test_fastjet',\ - 'test_root','test_delphes','test_delphesMA5tune']: + 'test_root','test_delphes','test_delphesMA5tune','test_onnx']: logfile = folder+'/cleanup_'+package[5:]+'.log' else: logfile = folder+'/cleanup_'+package+'.log' @@ -444,7 +469,7 @@ def Clean(self,package,folder): if package in ['process','commons','test','configuration']: makefile = 'Makefile' elif package in ['test_process','test_commons','test_zlib','test_fastjet',\ - 'test_root','test_delphes','test_delphesMA5tune']: + 'test_root','test_delphes','test_delphesMA5tune','test_onnx']: makefile = 'Makefile_'+package[5:] else: makefile = 'Makefile_'+package @@ -469,7 +494,7 @@ def MrProper(self,package,folder): if package in ['process','commons','configuration']: logfile = folder+'/mrproper.log' elif package in ['test_process','test_commons','test_zlib','test_root','test_fastjet',\ - 'test_delphes','test_delphesMA5tune']: + 'test_delphes','test_delphesMA5tune','test_onnx']: logfile = folder+'/mrproper_'+package[5:]+'.log' else: logfile = folder+'/mrproper_'+package+'.log' @@ -480,7 +505,7 @@ def MrProper(self,package,folder): if package in ['process','commons','test','configuration']: makefile = 'Makefile' elif package in ['test_process','test_commons','test_zlib','test_root','test_fastjet',\ - 'test_delphes','test_delphesMA5tune']: + 'test_delphes','test_delphesMA5tune','test_onnx']: makefile = 'Makefile_'+package[5:] else: makefile = 'Makefile_'+package diff --git a/madanalysis/build/makefile_writer.py b/madanalysis/build/makefile_writer.py index 1c1513ba..e4a2c8a5 100644 --- a/madanalysis/build/makefile_writer.py +++ b/madanalysis/build/makefile_writer.py @@ -40,6 +40,7 @@ def __init__(self): self.has_fastjet = False self.has_delphes = False self.has_delphesMA5tune = False + self.has_onnx = False @staticmethod @@ -95,6 +96,9 @@ def UserfriendlyMakefileForSampleAnalyzer(filename,options): if options.has_delphesMA5tune: file.write('\tcd Interfaces && $(MAKE) -f Makefile_delphesMA5tune\n') file.write('\tcd Test && $(MAKE) -f Makefile_delphesMA5tune\n') + if options.has_onnx: + file.write('\tcd Interfaces && $(MAKE) -f Makefile_onnx\n') + file.write('\tcd Test && $(MAKE) -f Makefile_onnx\n') if options.has_process: file.write('\tcd Process && $(MAKE) -f Makefile\n') file.write('\tcd Test && $(MAKE) -f Makefile_process\n') @@ -121,6 +125,9 @@ def UserfriendlyMakefileForSampleAnalyzer(filename,options): if options.has_delphesMA5tune: file.write('\tcd Interfaces && $(MAKE) -f Makefile_delphesMA5tune clean\n') file.write('\tcd Test && $(MAKE) -f Makefile_delphesMA5tune clean\n') + if options.has_onnx: + file.write('\tcd Interfaces && $(MAKE) -f Makefile_onnx clean\n') + file.write('\tcd Test && $(MAKE) -f Makefile_onnx clean\n') if options.has_process: file.write('\tcd Process && $(MAKE) -f Makefile clean\n') file.write('\tcd Test && $(MAKE) -f Makefile_process clean\n') @@ -148,6 +155,9 @@ def UserfriendlyMakefileForSampleAnalyzer(filename,options): if options.has_delphesMA5tune: file.write('\tcd Interfaces && $(MAKE) -f Makefile_delphesMA5tune mrproper\n') file.write('\tcd Test && $(MAKE) -f Makefile_delphesMA5tune mrproper\n') + if options.has_onnx: + file.write('\tcd Interfaces && $(MAKE) -f Makefile_onnx mrproper\n') + file.write('\tcd Test && $(MAKE) -f Makefile_onnx mrproper\n') if options.has_process: file.write('\tcd Process && $(MAKE) -f Makefile mrproper\n') file.write('\tcd Test && $(MAKE) -f Makefile_process mrproper\n') @@ -186,6 +196,10 @@ def __init__(self): self.has_root_tag = False self.has_root_lib = False self.has_root_ma5lib = False + self.has_onnx_tag = False + self.has_onnx_inc = False + self.has_onnx_lib = False + self.has_onnx_ma5lib = False @staticmethod @@ -218,14 +232,14 @@ def Makefile(MakefileName,title,ProductName,ProductPath,isLibrary,cppfiles,hfile # - general cxxflags=[] - cxxflags.extend(['-Wall','-std=c++11','-O3','-fPIC', '-I$(MA5_BASE)/tools/']) # general + cxxflags.extend(['-Wall','-std=c++14','-O3','-fPIC', '-I$(MA5_BASE)/tools/']) # general file.write('CXXFLAGS = '+' '.join(cxxflags)+'\n') for item in moreIncludes: file.write('CXXFLAGS += '+' -I'+item+'\n') # - compilation severity if not options.has_root_inc and not options.has_fastjet_inc and \ - not options.has_zlib_inc and \ + not options.has_zlib_inc and not options.has_onnx_inc and \ not options.has_delphes_inc and not options.has_delphesMA5tune_inc: # - compilation severity level 1 @@ -297,6 +311,12 @@ def Makefile(MakefileName,title,ProductName,ProductPath,isLibrary,cppfiles,hfile cxxflags.extend(['-I'+archi_info.zlib_inc_path]) file.write('CXXFLAGS += '+' '.join(cxxflags)+'\n') + # - onnx + if options.has_onnx_inc: + cxxflags=[] + cxxflags.extend(['-I'+archi_info.onnx_inc_path]) + file.write('CXXFLAGS += '+' '.join(cxxflags)+'\n') + # - delphes if options.has_delphes_inc: cxxflags=[] @@ -319,6 +339,8 @@ def Makefile(MakefileName,title,ProductName,ProductPath,isLibrary,cppfiles,hfile cxxflags.extend(['-DFASTJET_USE']) if options.has_zlib_tag: cxxflags.extend(['-DZIP_USE']) + if options.has_onnx_tag: + cxxflags.extend(['-DONNX_USE']) if options.has_delphes_tag: cxxflags.extend(['-DDELPHES_USE']) if options.has_delphesMA5tune_tag: @@ -357,6 +379,17 @@ def Makefile(MakefileName,title,ProductName,ProductPath,isLibrary,cppfiles,hfile libs.extend(['-lz']) file.write('LIBFLAGS += '+' '.join(libs)+'\n') + # - onnx + if options.has_onnx_ma5lib or options.has_onnx_lib: + libs=[] + if options.has_onnx_ma5lib: + libs.extend(['-lonnx_for_ma5']) + if options.has_onnx_lib: + libs.extend(['-L'+archi_info.onnx_lib_path,'-lonnxruntime']) + libs.extend(['-Wl,-rpath,'+archi_info.onnx_lib_path]) + + file.write('LIBFLAGS += '+' '.join(libs)+'\n') + # - root if options.has_root_ma5lib: libs=[] @@ -435,6 +468,8 @@ def Makefile(MakefileName,title,ProductName,ProductPath,isLibrary,cppfiles,hfile libs.append('$(MA5_BASE)/tools/SampleAnalyzer/Lib/libdelphesMA5tune_for_ma5.so') if options.has_fastjet_ma5lib: libs.append('$(MA5_BASE)/tools/SampleAnalyzer/Lib/libfastjet_for_ma5.so') + if options.has_onnx_ma5lib: + libs.append('$(MA5_BASE)/tools/SampleAnalyzer/Lib/libonnx_for_ma5.so') if len(libs)!=0: file.write('# Requirements to check before building\n') for ind in range(0,len(libs)): diff --git a/madanalysis/core/launcher.py b/madanalysis/core/launcher.py index 3d6aefc0..fa6c1eb7 100644 --- a/madanalysis/core/launcher.py +++ b/madanalysis/core/launcher.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2012-2023 Jack Araz, Eric Conte & Benjamin Fuks +# Copyright (C) 2012-2022 Jack Araz, Eric Conte & Benjamin Fuks # The MadAnalysis development team, email: # # This file is part of MadAnalysis 5. @@ -91,6 +91,10 @@ def DefaultInstallCard(): output.write('# fastjet_veto = 0 # 0=No, 1=Yes\n') output.write('# fastjet_bin_path = /home/fastjet/build/bin/\n') output.write('\n') + output.write('# -----ONNX-----\n') + output.write('# onnx_veto = 0 # 0=No, 1=Yes\n') + output.write('# onnx_bin_path = /home/onnx/build/bin/\n') + output.write('\n') output.write('# -----PAD-----\n') output.write('# pad_veto = 0 # 0=No, 1=Yes\n') output.write('# pad_build_path = /home/PAD/build/\n') @@ -241,8 +245,7 @@ def MainSession(mode,arglist,ma5dir,version,date): logging.getLogger('MA5').info("* \ \_\\\ \_\ \_\ \_\ \____/ *") logging.getLogger('MA5').info("* \/_/ \/_/\/_/\/_/\/___/ *") logging.getLogger('MA5').info("* *") - logging.getLogger('MA5').info("* MA5 release : " + "%-24s" % main.archi_info.ma5_version + \ - "%+15s" % main.archi_info.ma5_date + " *") + logging.getLogger('MA5').info("* MA5 release : " + "%-24s" % main.archi_info.ma5_version + "%+15s" % main.archi_info.ma5_date + " *") logging.getLogger('MA5').info("* *") logging.getLogger('MA5').info("* Comput. Phys. Commun. 184 (2013) 222-256 *") logging.getLogger('MA5').info("* Eur. Phys. J. C74 (2014) 3103 *") @@ -362,8 +365,7 @@ def Usage(): logging.getLogger('MA5').info(" -v or --version") logging.getLogger('MA5').info(" or --release : display the version number of MadAnalysis") logging.getLogger('MA5').info(" -b or --build : rebuild the SampleAnalyzer static library") - logging.getLogger('MA5').info(" -f or --forced : do not ask for confirmation when MA5 removes "+\ - "a directory or overwrites an object") + logging.getLogger('MA5').info(" -f or --forced : do not ask for confirmation when MA5 removes a directory or overwrites an object") logging.getLogger('MA5').info(" -s or --script : quit automatically MA5 when the script is loaded") logging.getLogger('MA5').info(" -h or --help : dump this help") logging.getLogger('MA5').info(" -i or --installcard : produce the default installation card in installation_card.dat") diff --git a/madanalysis/core/library_builder.py b/madanalysis/core/library_builder.py index cafab529..df22145f 100644 --- a/madanalysis/core/library_builder.py +++ b/madanalysis/core/library_builder.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2012-2023 Jack Araz, Eric Conte & Benjamin Fuks +# Copyright (C) 2012-2022 Jack Araz, Eric Conte & Benjamin Fuks # The MadAnalysis development team, email: # # This file is part of MadAnalysis 5. @@ -80,6 +80,8 @@ def checkMA5(self): libraries.append(self.archi_info.ma5dir+'/tools/SampleAnalyzer/Lib/libdelphes_for_ma5.so') if self.archi_info.has_delphesMA5tune: libraries.append(self.archi_info.ma5dir+'/tools/SampleAnalyzer/Lib/libdelphesMA5tune_for_ma5.so') + if self.archi_info.has_onnx: + libraries.append(self.archi_info.ma5dir+'/tools/SampleAnalyzer/Lib/libonnx_for_ma5.so') for library in libraries: if not os.path.isfile(library): self.logger.debug('\t-> library '+ library + " not found.") diff --git a/madanalysis/core/main.py b/madanalysis/core/main.py index 8d417c65..a8b9455d 100644 --- a/madanalysis/core/main.py +++ b/madanalysis/core/main.py @@ -572,6 +572,15 @@ def BuildLibrary(self,forced=False): self.archi_info.ma5dir+'/tools/SampleAnalyzer/Bin/TestZlib',\ self.archi_info.ma5dir+'/tools/SampleAnalyzer/Test/',True]) + # ONNX + if self.archi_info.has_onnx: + libraries.append(['onnx', 'interface to onnx', 'onnx',\ + self.archi_info.ma5dir+'/tools/SampleAnalyzer/Lib/libonnx_for_ma5.so',\ + self.archi_info.ma5dir+'/tools/SampleAnalyzer/Interfaces',False]) + libraries.append(['test_onnx','interface to onnx', 'test_onnx',\ + self.archi_info.ma5dir+'/tools/SampleAnalyzer/Bin/TestOnnx',\ + self.archi_info.ma5dir+'/tools/SampleAnalyzer/Test/',True]) + # Fastjet if self.archi_info.has_fastjet: libraries.append(['FastJet', 'interface to FastJet', 'fastjet',\ @@ -637,6 +646,7 @@ def BuildLibrary(self,forced=False): options.has_process = True options.has_test = True options.has_zlib = self.archi_info.has_zlib + options.has_onnx = self.archi_info.has_onnx options.has_fastjet = self.archi_info.has_fastjet options.has_delphes = self.archi_info.has_delphes options.has_delphesMA5tune = self.archi_info.has_delphesMA5tune diff --git a/madanalysis/input/installation_options.dat b/madanalysis/input/installation_options.dat index c6350f23..9da2a65f 100644 --- a/madanalysis/input/installation_options.dat +++ b/madanalysis/input/installation_options.dat @@ -54,3 +54,8 @@ # -----SCIPY----- # scipy_veto = 0 # 0=No, 1=Yes +# -----ONNX----- +# onnx_veto = 0 # 0=No, 1=Yes +# onnx_includes = /home/onnx/include/ +# onnx_libs = /home/onnx/lib/ + diff --git a/madanalysis/install/install_manager.py b/madanalysis/install/install_manager.py index 17651f52..3aa690ef 100644 --- a/madanalysis/install/install_manager.py +++ b/madanalysis/install/install_manager.py @@ -85,6 +85,9 @@ def Execute(self, rawpackage): elif package=='simplify': from madanalysis.install.install_simplify import InstallSimplify installer=InstallSimplify(self.main) + elif package=='onnx': + from madanalysis.install.install_onnx import InstallOnnx + installer=InstallOnnx(self.main) else: self.logger.error('the package "'+rawpackage+'" is unknown') return False diff --git a/madanalysis/install/install_onnx.py b/madanalysis/install/install_onnx.py new file mode 100644 index 00000000..31e1dea5 --- /dev/null +++ b/madanalysis/install/install_onnx.py @@ -0,0 +1,152 @@ +################################################################################ +# +# Copyright (C) 2012-2023 Jack Araz, Eric Conte & Benjamin Fuks +# The MadAnalysis development team, email: +# +# This file is part of MadAnalysis 5. +# Official website: +# +# MadAnalysis 5 is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# MadAnalysis 5 is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with MadAnalysis 5. If not, see +# +################################################################################ + + +from __future__ import absolute_import + +import logging +import os +import sys + +from shell_command import ShellCommand + +from madanalysis.install.install_service import InstallService + + +class InstallOnnx: + def __init__(self, main): + self.main = main + self.installdir = os.path.normpath(self.main.archi_info.ma5dir + "/tools/onnx/") + self.toolsdir = os.path.normpath(self.main.archi_info.ma5dir + "/tools") + self.tmpdir = self.main.session_info.tmpdir + self.downloaddir = self.main.session_info.downloaddir + self.untardir = os.path.normpath(self.tmpdir + "/MA5_onnx/") + self.ncores = 1 + self.version = "1.17.1" + if self.main.archi_info.isMac : + if self.main.archi_info.isARM64: + self.ver_name = "onnxruntime-osx-arm64-"+self.version + else : + self.ver_name = "onnxruntime-osx-x86_64-"+self.version + self.lib_name = "libonnxruntime."+self.version+".dylib" + else : #if not mac is linux + self.ver_name = "onnxruntime-linux-x64-"+self.version + self.lib_name = "libonnxruntime.so."+self.version + + self.files = {self.ver_name+".tgz": "https://github.com/microsoft/onnxruntime/releases/download/v"+self.version+"/"+self.ver_name+".tgz"} + + def Detect(self): + if not os.path.isdir(self.toolsdir): + logging.getLogger("MA5").debug( + "The folder '" + self.toolsdir + "' is not found" + ) + return False + if not os.path.isdir(self.installdir): + logging.getLogger("MA5").debug( + "The folder " + self.installdir + "' is not found" + ) + return False + return True + + def Remove(self, question=True): + from madanalysis.IOinterface.folder_writer import FolderWriter + return FolderWriter.RemoveDirectory(self.installdir, question) + + def GetNcores(self): + self.ncores = InstallService.get_ncores( + self.main.archi_info.ncores, self.main.forced + ) + + def CreatePackageFolder(self): + if not InstallService.create_tools_folder(self.toolsdir): + return False + if not InstallService.create_package_folder(self.toolsdir, "onnx"): + return False + return True + + def CreateTmpFolder(self): + ok = InstallService.prepare_tmp(self.untardir, self.downloaddir) + if ok: + self.tmpdir = self.untardir + return ok + + def Download(self): + # Checking connection with MA5 web site + if not InstallService.check_ma5site(): + return False + # Launching wget + logname = os.path.normpath(self.installdir + "/wget.log") + if not InstallService.wget(self.files, logname, self.downloaddir): + return False + # Ok + return True + + def Unpack(self): + # Logname + logname = os.path.normpath(self.installdir + "/unpack.log") + # Unpacking the tarball + ok, packagedir = InstallService.untar( + logname, self.downloaddir, self.installdir, self.ver_name+".tgz" + ) + if not ok: + return False + # Ok: returning the good folder + self.tmpdir = packagedir + return True + + def Check(self): + # Check folders + dirs = [self.installdir +"/" +self.ver_name +"/include", self.installdir +"/" +self.ver_name + "/lib"] + + for dir in dirs: + if not os.path.isdir(dir): + logging.getLogger("MA5").error("folder " + dir + " is missing.") + self.display_log() + return False + + # Check one header file + if not os.path.isfile(self.installdir +"/" +self.ver_name +"/include/onnxruntime_cxx_api.h"): + logging.getLogger("MA5").error("header labeled 'include/onnx.h' is missing.") + self.display_log() + return False + + if (not os.path.isfile(self.installdir +"/" +self.ver_name +"/lib/"+self.lib_name)): + logging.getLogger("MA5").error( + "libonnxruntime is missing." + ) + self.display_log() + return False + + return True + + def display_log(self): + logging.getLogger("MA5").error("More details can be found into the log files:") + logging.getLogger("MA5").error( + " - " + os.path.normpath(self.installdir + "/wget.log") + ) + logging.getLogger("MA5").error( + " - " + os.path.normpath(self.installdir + "/unpack.log") + ) + + def NeedToRestart(self): + return True diff --git a/madanalysis/interpreter/cmd_install.py b/madanalysis/interpreter/cmd_install.py index 9168bd29..d97e321f 100644 --- a/madanalysis/interpreter/cmd_install.py +++ b/madanalysis/interpreter/cmd_install.py @@ -118,6 +118,8 @@ def UpdatePaths(): return installer.Execute('samples') elif args[0]=='zlib': return installer.Execute('zlib') + elif args[0]=='onnx': + return installer.Execute('onnx') elif args[0] in ['delphes', 'delphesMA5tune']: return inst_delphes(self.main,installer,args[0]) elif args[0]=='fastjet': @@ -210,7 +212,7 @@ def UpdatePaths(): def help(self): self.logger.info(" Syntax: install ") self.logger.info(" Download and install a MadAnalysis component from the official site.") - self.logger.info(" List of available components: samples zlib fastjet delphes delphesMA5tune PAD PADForMA5tune PADForSFS") + self.logger.info(" List of available components: samples zlib fastjet delphes delphesMA5tune PAD PADForMA5tune PADForSFS onnx") def complete(self,text,args,begidx,endidx): @@ -224,7 +226,7 @@ def complete(self,text,args,begidx,endidx): else: output = ["samples","zlib","fastjet", "delphes", "delphesMA5tune",\ "gnuplot", "matplotlib", "root" , "numpy", "PAD", "PADForMA5tune",\ - "PADForSFS", "pyhf"] + "PADForSFS", "pyhf", "onnx"] return self.finalize_complete(text,output) diff --git a/madanalysis/interpreter/ma5_interpreter.py b/madanalysis/interpreter/ma5_interpreter.py index b0d486c0..bfb89440 100644 --- a/madanalysis/interpreter/ma5_interpreter.py +++ b/madanalysis/interpreter/ma5_interpreter.py @@ -361,6 +361,14 @@ def config_update(checkup): if not installer.Execute('zlib'): self.logger.error('Impossible to install zlib.') return False + + # If not onnx -> install onnx + if not self.main.archi_info.has_onnx: + self.logger.info('The onnx package has not been found. Proceeding with its local installation.') + installer=InstallManager(self.main) + if not installer.Execute('onnx'): + self.logger.error('Impossible to install onnx.') + return False # If not fastjet -> install fastjet if not self.main.archi_info.has_fastjet: diff --git a/madanalysis/system/architecture_info.py b/madanalysis/system/architecture_info.py index 6106ad05..474781b5 100644 --- a/madanalysis/system/architecture_info.py +++ b/madanalysis/system/architecture_info.py @@ -37,6 +37,7 @@ def __init__(self): # Main flags self.isMac = False + self.isARM64 = False # Is there optional package? self.has_root = False @@ -44,6 +45,7 @@ def __init__(self): self.has_zlib = False self.has_delphes = False self.has_delphesMA5tune = False + self.has_onnx = False # Library to put before all the others? self.root_priority = False @@ -51,6 +53,7 @@ def __init__(self): self.delphes_priority = False self.delphesMA5tune_priority = False self.fastjet_priority = False + self.onnx_priority = False # Library files self.zlib_original_libs = [] @@ -58,6 +61,8 @@ def __init__(self): self.root_original_bins = [] self.delphes_original_libs = [] self.delphesMA5tune_original_libs = [] + self.onnx_original_libs = [] + # Version self.python_version = "" @@ -88,6 +93,9 @@ def __init__(self): self.zlib_inc_path="" self.zlib_lib_path="" self.zlib_lib="" + self.onnx_inc_path="" + self.onnx_lib_path="" + self.onnx_lib="" self.delphes_inc_paths=[] self.delphes_lib_paths=[] self.delphes_lib="" diff --git a/madanalysis/system/checkup.py b/madanalysis/system/checkup.py index b12c7e94..902afe3f 100644 --- a/madanalysis/system/checkup.py +++ b/madanalysis/system/checkup.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2012-2023 Jack Araz, Eric Conte & Benjamin Fuks +# Copyright (C) 2012-2022 Jack Araz, Eric Conte & Benjamin Fuks # The MadAnalysis development team, email: # # This file is part of MadAnalysis 5. @@ -63,6 +63,8 @@ def CheckArchitecture(self): if self.archi_info.platform.lower() in ['darwin','mac','macosx']: self.archi_info.isMac = True platform_text+='\x1b[32m'+'[MAC/OSX mode]'+'\x1b[0m' + if platform.machine().lower() != "x86_64" : + self.archi_info.isARM64 = True else: self.archi_info.isMac = False platform_text+='\x1b[32m'+'[Linux mode]'+'\x1b[0m' @@ -324,6 +326,8 @@ def CheckOptionalProcessingPackages(self): return False if not self.checker.Execute('root'): return False + if not self.checker.Execute('onnx'): + return False self.archi_info.has_delphes = checker2.checkDelphes() self.archi_info.has_delphesMA5tune = checker2.checkDelphesMA5tune() @@ -346,6 +350,8 @@ def CheckOptionalReinterpretationPackages(self): return False if not self.checker.Execute('simplify'): return False + if not self.checker.Execute('onnx'): + return False return True @@ -532,13 +538,6 @@ def SetFolder(self): self.logger.debug('after DYLD_LIBRARY_PATH='+str(os.environ['DYLD_LIBRARY_PATH'])) self.logger.debug('--------') - # ROOT INCLUDE PATH - if self.archi_info.has_delphes: - os.environ['ROOT_INCLUDE_PATH']=os.path.join(self.archi_info.ma5dir,'tools','delphes','external'); - if self.archi_info.has_delphesMA5tune: - os.environ['ROOT_INCLUDE_PATH']=os.path.join(self.archi_info.ma5dir,'tools','delphesMA5tune','external'); - - self.logger.debug('-------- END: set environment variables --------') return True diff --git a/madanalysis/system/detect_manager.py b/madanalysis/system/detect_manager.py index 6b1edd9a..80582d1e 100644 --- a/madanalysis/system/detect_manager.py +++ b/madanalysis/system/detect_manager.py @@ -54,6 +54,9 @@ def Execute(self, rawpackage): if package=='zlib': from madanalysis.system.detect_zlib import DetectZlib checker=DetectZlib(self.archi_info, self.user_info, self.session_info, self.debug) + elif package=='onnx': + from madanalysis.system.detect_onnx import DetectONNX + checker=DetectONNX(self.archi_info, self.user_info, self.session_info, self.debug) elif package=='fastjet': from madanalysis.system.detect_fastjet import DetectFastjet checker=DetectFastjet(self.archi_info, self.user_info, self.session_info, self.debug) diff --git a/madanalysis/system/detect_onnx.py b/madanalysis/system/detect_onnx.py new file mode 100644 index 00000000..5ca934b3 --- /dev/null +++ b/madanalysis/system/detect_onnx.py @@ -0,0 +1,133 @@ +################################################################################ +# +# Copyright (C) 2012-2022 Jack Araz, Eric Conte & Benjamin Fuks +# The MadAnalysis development team, email: +# +# This file is part of MadAnalysis 5. +# Official website: +# +# MadAnalysis 5 is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# MadAnalysis 5 is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with MadAnalysis 5. If not, see +# +################################################################################ + + +from __future__ import absolute_import +import logging +import glob +import os +import sys +import re +import platform +from shell_command import ShellCommand +from madanalysis.enumeration.detect_status_type import DetectStatusType + + +class DetectONNX: + + def __init__(self, archi_info, user_info, session_info, debug): + # mandatory options + self.archi_info = archi_info + self.user_info = user_info + self.session_info = session_info + self.debug = debug + self.name = 'ONNXRunTime' + self.mandatory = False + self.force = False + self.log = [] + self.logger = logging.getLogger('MA5') + self.version = "1.17.1" + if self.archi_info.isMac : + if self.archi_info.isARM64: + self.ver_name = "onnxruntime-osx-arm64-"+self.version + else : + self.ver_name = "onnxruntime-osx-x86_64-"+self.version + self.lib_name = "libonnxruntime."+self.version+".dylib" + else : #if not mac is linux + self.ver_name = "onnxruntime-linux-x64-"+self.version + self.lib_name = "libonnxruntime.so."+self.version + # NAme of the header + self.headernames=['onnxruntime_cxx_api.h'] + + # Name of the dynamic lib + self.libnames=[self.lib_name] + + # adding what you want here + + + def PrintDisableMessage(self): + self.logger.warning("ONNXRunTime is disabled. Cannot use .onnx files.") + + + def IsItVetoed(self): + if self.user_info.onnx_veto: + self.logger.debug("user setting: veto on ONNX") + return True + else: + self.logger.debug("no user veto") + return False + + + def ToolsDetection(self): + + # Check + pathname = os.path.normpath(self.archi_info.ma5dir+'/tools/onnx/'+self.ver_name) + self.logger.debug("Look for onnx folder in path "+pathname+" ...") + if not os.path.isdir(pathname): + self.logger.debug("-> not found") + return DetectStatusType.UNFOUND, '' + else: + self.logger.debug("-> found") + + # headers + result1 = self.LookForPattern(pathname+'/include/',self.headernames) + test1 = (len(result1)!=0) + + # libs + result2 = self.LookForPattern(pathname+'/lib/',self.libnames) + test2 = (len(result2)!=0) + + # Return + if test1 and test2: + self.header_files = result1 + self.library_files = result2 + return DetectStatusType.FOUND, '' + else: + return DetectStatusType.UNFOUND, '' + + def LookForPattern(self,path,patterns): + result=[] + for pattern in patterns: + filename=os.path.normpath(path+'/'+pattern) + self.logger.debug('look for pattern '+filename+' ...') + thefiles = glob.glob(filename) + for thefile in thefiles: + if thefile not in result: + self.logger.debug('-> found: '+thefile) + result.append(thefile) + if len(result)==0: + self.logger.debug('-> no file found') + return result + + def SaveInfo(self): + self.session_info.has_onnx = True + self.archi_info.has_onnx = True + self.archi_info.onnx_priority = self.force + self.archi_info.onnx_lib = self.library_files[0] + self.archi_info.onnx_inc_path = os.path.dirname(self.header_files[0]) + self.archi_info.onnx_lib_path = os.path.dirname(self.library_files[0]) + self.archi_info.onnx_original_libs.extend([fl for fl in self.library_files if not fl in self.archi_info.onnx_original_libs]) + + return True + + diff --git a/madanalysis/system/session_info.py b/madanalysis/system/session_info.py index e43adf2a..4db82854 100644 --- a/madanalysis/system/session_info.py +++ b/madanalysis/system/session_info.py @@ -39,6 +39,7 @@ def __init__(self): self.has_gnuplot = False self.has_pdflatex = False self.has_latex = False + self.has_onnx = False self.has_dvipdf = False self.has_web = True self.has_pad = False diff --git a/madanalysis/system/user_info.py b/madanalysis/system/user_info.py index 1ee5ec9c..fb2c703d 100644 --- a/madanalysis/system/user_info.py +++ b/madanalysis/system/user_info.py @@ -63,6 +63,11 @@ def __init__(self): self.zlib_includes = None self.zlib_libs = None + # Onnx + self.onnx_veto = None + self.onnx_veto_includes = None + self.onnx_libs = None + # Fastjet self.fastjet_veto = None self.fastjet_bin_path = None diff --git a/tools/SampleAnalyzer/Test/Onnx/Test.cpp b/tools/SampleAnalyzer/Test/Onnx/Test.cpp new file mode 100644 index 00000000..3499c3f6 --- /dev/null +++ b/tools/SampleAnalyzer/Test/Onnx/Test.cpp @@ -0,0 +1,47 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2012-2023 Jack Araz, Eric Conte & Benjamin Fuks +// The MadAnalysis development team, email: +// +// This file is part of MadAnalysis 5. +// Official website: +// +// MadAnalysis 5 is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// MadAnalysis 5 is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with MadAnalysis 5. If not, see +// +//////////////////////////////////////////////////////////////////////////////// + + +// SampleHeader headers +#include "SampleAnalyzer/Commons/DataFormat/EventFormat.h" +#include "SampleAnalyzer/Commons/DataFormat/SampleFormat.h" +#include "onnxruntime_cxx_api.h" + + +using namespace MA5; + +// ----------------------------------------------------------------------- +// main program +// ----------------------------------------------------------------------- +MAint32 main(MAint32 argc, MAchar *argv[]) +{ + std::cout << "BEGIN-SAMPLEANALYZER-TEST" << std::endl; + std::cout << std::endl; + Ort::Env env; + + EventFormat event; + SampleFormat sample; + + std::cout << "END-SAMPLEANALYZER-TEST" << std::endl; + return 0; +}