Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ONNX Runtime API #259

Open
wants to merge 12 commits into
base: main
Choose a base branch
from
+473 −27
Open

ONNX Runtime API #259

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
509430d
install and checker working for onnx
Apr 2, 2024
87d7c5d
adding library linking functions
Apr 3, 2024
7b117eb
adding rpath and Test for ONNX
Apr 22, 2024
64a56a7
adding options in job writer
Apr 24, 2024
55e0cfd
Added includes to test program and correct makefile options for testing
Apr 26, 2024
7dc718a
adding option for linux
May 13, 2024
f45e77f
modifying check function with custom lib name
May 17, 2024
b52a463
add detection and library saving to archi info
May 17, 2024
dd7661a
chaging include to match all platform paths
May 21, 2024
6cfabc2
adding .util extension in importlib for linux systems
May 21, 2024
76b63f9
adding ARM64 flags and ONNX tarball
May 21, 2024
7a95509
changing compiler ver for linux platform issues with onnx
May 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion bin/ma5
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ This is the main executable, a simple frontend to set up the PYTHONPATH
and call immediately the command line interface scripts
"""

import importlib
import importlib.util
import os
import sys

Expand Down
6 changes: 6 additions & 0 deletions madanalysis/IOinterface/job_writer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -705,6 +705,9 @@ def WriteSampleAnalyzerMakefile(self,option=""):
# Options
option.has_commons = True
options.has_process = True
if self.main.archi_info.has_onnx:
options.has_onnx_inc = True
options.has_onnx_lib = True
if self.main.archi_info.has_root:
options.has_root_inc = True
options.has_root_lib = True
Expand Down Expand Up @@ -740,6 +743,9 @@ def WriteMakefiles(self,option=""):
# Options
options.has_commons = True
options.has_process = True
if self.main.archi_info.has_onnx:
options.has_onnx_inc = True
options.has_onnx_lib = True
if self.main.archi_info.has_root:
options.has_root_inc = True
options.has_root_lib = True
Expand Down
41 changes: 33 additions & 8 deletions madanalysis/IOinterface/library_writer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,8 @@ def WriteMakefileForInterfaces(self,package):
filename = self.path+"/SampleAnalyzer/Test/Makefile_delphesMA5tune"
elif package=='test_root':
filename = self.path+"/SampleAnalyzer/Test/Makefile_root"
elif package=='test_onnx':
filename = self.path+"/SampleAnalyzer/Test/Makefile_onnx"

# Header
title=''
Expand All @@ -155,6 +157,8 @@ def WriteMakefileForInterfaces(self,package):
title='*delphesMA5tune-interface* test'
elif package=='test_root':
title='*root-interface* test'
elif package=='test_onnx':
title='*onnx-interface* test'
else:
title='interface to '+package

Expand Down Expand Up @@ -190,6 +194,18 @@ def WriteMakefileForInterfaces(self,package):
# options.has_zlib_lib = True
toRemove.extend(['compilation_zlib.log','linking_zlib.log','cleanup_zlib.log',\
'mrproper_zlib.log','../Bin/TestZlib.log'])
elif package=='onnx':
options.has_commons = True
options.has_onnx_inc = True
options.has_onnx_lib = True
toRemove.extend(['compilation_onnx.log','linking_onnx.log','cleanup_onnx.log','mrproper_onnx.log'])
elif package=='test_onnx':
options.has_commons = True
#options.has_onnx_ma5lib = True
options.has_onnx_inc = True
options.has_onnx_lib = True
toRemove.extend(['compilation_onnx.log','linking_onnx.log','cleanup_onnx.log',\
'mrproper_onnx.log','../Bin/TestOnnx.log'])
elif package=='delphes':
options.has_commons = True
options.has_delphes_inc = True
Expand Down Expand Up @@ -260,6 +276,8 @@ def WriteMakefileForInterfaces(self,package):
options.has_zlib_tag = self.main.archi_info.has_zlib
options.has_root_tag = self.main.archi_info.has_root
options.has_root_ma5lib = self.main.archi_info.has_root
options.has_onnx_tag = self.main.archi_info.has_onnx
options.has_onnx_ma5lib = self.main.archi_info.has_onnx
toRemove.extend(['compilation.log','linking.log','cleanup.log','mrproper.log'])
elif package=='test_process':
options.has_commons = True
Expand Down Expand Up @@ -304,6 +322,9 @@ def WriteMakefileForInterfaces(self,package):
elif package=='test_root':
cppfiles = ['Root/*.cpp']
hfiles = ['Root/*.h']
elif package=='test_onnx':
cppfiles = ['Onnx/*.cpp']
hfiles = ['Onnx/*.h']
else:
cppfiles = [package+'/*.cpp']
hfiles = [package+'/*.h']
Expand Down Expand Up @@ -333,6 +354,10 @@ def WriteMakefileForInterfaces(self,package):
isLibrary=False
ProductName='TestRoot'
ProductPath='../Bin/'
elif package=='test_onnx':
isLibrary=False
ProductName='TestOnnx'
ProductPath='../Bin/'
elif package=='test_delphes':
isLibrary=False
ProductName='TestDelphes'
Expand Down Expand Up @@ -364,7 +389,7 @@ def Compile(self,ncores,package,folder):
if package in ['process','commons','test','configuration']:
logfile = folder+'/compilation.log'
elif package in ['test_process','test_commons','test_zlib','test_fastjet',\
'test_root','test_delphes','test_delphesMA5tune']:
'test_root','test_delphes','test_delphesMA5tune','test_onnx']:
logfile = folder+'/compilation_'+package[5:]+'.log'
else:
logfile = folder+'/compilation_'+package+'.log'
Expand All @@ -373,7 +398,7 @@ def Compile(self,ncores,package,folder):
if package in ['process','commons','test','configuration']:
makefile = 'Makefile'
elif package in ['test_process','test_commons','test_zlib','test_fastjet',\
'test_root','test_delphes','test_delphesMA5tune']:
'test_root','test_delphes','test_delphesMA5tune','test_onnx']:
makefile = 'Makefile_'+package[5:]
else:
makefile = 'Makefile_'+package
Expand Down Expand Up @@ -401,7 +426,7 @@ def Link(self,package,folder):
if package in ['process','commons','test','configuration']:
logfile = folder+'/linking.log'
elif package in ['test_process','test_commons','test_zlib','test_fastjet',\
'test_root','test_delphes','test_delphesMA5tune']:
'test_root','test_delphes','test_delphesMA5tune','test_onnx']:
logfile = folder+'/linking_'+package[5:]+'.log'
else:
logfile = folder+'/linking_'+package+'.log'
Expand All @@ -410,7 +435,7 @@ def Link(self,package,folder):
if package in ['process','commons','test','configuration']:
makefile = 'Makefile'
elif package in ['test_process','test_commons','test_zlib','test_fastjet',\
'test_root','test_delphes','test_delphesMA5tune']:
'test_root','test_delphes','test_delphesMA5tune','test_onnx']:
makefile = 'Makefile_'+package[5:]
else:
makefile = 'Makefile_'+package
Expand All @@ -435,7 +460,7 @@ def Clean(self,package,folder):
if package in ['process','commons','configuration','test']:
logfile = folder+'/cleanup.log'
elif package in ['test_process','test_commons','test_zlib','test_fastjet',\
'test_root','test_delphes','test_delphesMA5tune']:
'test_root','test_delphes','test_delphesMA5tune','test_onnx']:
logfile = folder+'/cleanup_'+package[5:]+'.log'
else:
logfile = folder+'/cleanup_'+package+'.log'
Expand All @@ -444,7 +469,7 @@ def Clean(self,package,folder):
if package in ['process','commons','test','configuration']:
makefile = 'Makefile'
elif package in ['test_process','test_commons','test_zlib','test_fastjet',\
'test_root','test_delphes','test_delphesMA5tune']:
'test_root','test_delphes','test_delphesMA5tune','test_onnx']:
makefile = 'Makefile_'+package[5:]
else:
makefile = 'Makefile_'+package
Expand All @@ -469,7 +494,7 @@ def MrProper(self,package,folder):
if package in ['process','commons','configuration']:
logfile = folder+'/mrproper.log'
elif package in ['test_process','test_commons','test_zlib','test_root','test_fastjet',\
'test_delphes','test_delphesMA5tune']:
'test_delphes','test_delphesMA5tune','test_onnx']:
logfile = folder+'/mrproper_'+package[5:]+'.log'
else:
logfile = folder+'/mrproper_'+package+'.log'
Expand All @@ -480,7 +505,7 @@ def MrProper(self,package,folder):
if package in ['process','commons','test','configuration']:
makefile = 'Makefile'
elif package in ['test_process','test_commons','test_zlib','test_root','test_fastjet',\
'test_delphes','test_delphesMA5tune']:
'test_delphes','test_delphesMA5tune','test_onnx']:
makefile = 'Makefile_'+package[5:]
else:
makefile = 'Makefile_'+package
Expand Down
39 changes: 37 additions & 2 deletions madanalysis/build/makefile_writer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ def __init__(self):
self.has_fastjet = False
self.has_delphes = False
self.has_delphesMA5tune = False
self.has_onnx = False


@staticmethod
Expand Down Expand Up @@ -95,6 +96,9 @@ def UserfriendlyMakefileForSampleAnalyzer(filename,options):
if options.has_delphesMA5tune:
file.write('\tcd Interfaces && $(MAKE) -f Makefile_delphesMA5tune\n')
file.write('\tcd Test && $(MAKE) -f Makefile_delphesMA5tune\n')
if options.has_onnx:
file.write('\tcd Interfaces && $(MAKE) -f Makefile_onnx\n')
file.write('\tcd Test && $(MAKE) -f Makefile_onnx\n')
if options.has_process:
file.write('\tcd Process && $(MAKE) -f Makefile\n')
file.write('\tcd Test && $(MAKE) -f Makefile_process\n')
Expand All @@ -121,6 +125,9 @@ def UserfriendlyMakefileForSampleAnalyzer(filename,options):
if options.has_delphesMA5tune:
file.write('\tcd Interfaces && $(MAKE) -f Makefile_delphesMA5tune clean\n')
file.write('\tcd Test && $(MAKE) -f Makefile_delphesMA5tune clean\n')
if options.has_onnx:
file.write('\tcd Interfaces && $(MAKE) -f Makefile_onnx clean\n')
file.write('\tcd Test && $(MAKE) -f Makefile_onnx clean\n')
if options.has_process:
file.write('\tcd Process && $(MAKE) -f Makefile clean\n')
file.write('\tcd Test && $(MAKE) -f Makefile_process clean\n')
Expand Down Expand Up @@ -148,6 +155,9 @@ def UserfriendlyMakefileForSampleAnalyzer(filename,options):
if options.has_delphesMA5tune:
file.write('\tcd Interfaces && $(MAKE) -f Makefile_delphesMA5tune mrproper\n')
file.write('\tcd Test && $(MAKE) -f Makefile_delphesMA5tune mrproper\n')
if options.has_onnx:
file.write('\tcd Interfaces && $(MAKE) -f Makefile_onnx mrproper\n')
file.write('\tcd Test && $(MAKE) -f Makefile_onnx mrproper\n')
if options.has_process:
file.write('\tcd Process && $(MAKE) -f Makefile mrproper\n')
file.write('\tcd Test && $(MAKE) -f Makefile_process mrproper\n')
Expand Down Expand Up @@ -186,6 +196,10 @@ def __init__(self):
self.has_root_tag = False
self.has_root_lib = False
self.has_root_ma5lib = False
self.has_onnx_tag = False
self.has_onnx_inc = False
self.has_onnx_lib = False
self.has_onnx_ma5lib = False


@staticmethod
Expand Down Expand Up @@ -218,14 +232,14 @@ def Makefile(MakefileName,title,ProductName,ProductPath,isLibrary,cppfiles,hfile

# - general
cxxflags=[]
cxxflags.extend(['-Wall','-std=c++11','-O3','-fPIC', '-I$(MA5_BASE)/tools/']) # general
cxxflags.extend(['-Wall','-std=c++14','-O3','-fPIC', '-I$(MA5_BASE)/tools/']) # general
file.write('CXXFLAGS = '+' '.join(cxxflags)+'\n')
for item in moreIncludes:
file.write('CXXFLAGS += '+' -I'+item+'\n')

# - compilation severity
if not options.has_root_inc and not options.has_fastjet_inc and \
not options.has_zlib_inc and \
not options.has_zlib_inc and not options.has_onnx_inc and \
not options.has_delphes_inc and not options.has_delphesMA5tune_inc:

# - compilation severity level 1
Expand Down Expand Up @@ -297,6 +311,12 @@ def Makefile(MakefileName,title,ProductName,ProductPath,isLibrary,cppfiles,hfile
cxxflags.extend(['-I'+archi_info.zlib_inc_path])
file.write('CXXFLAGS += '+' '.join(cxxflags)+'\n')

# - onnx
if options.has_onnx_inc:
cxxflags=[]
cxxflags.extend(['-I'+archi_info.onnx_inc_path])
file.write('CXXFLAGS += '+' '.join(cxxflags)+'\n')

# - delphes
if options.has_delphes_inc:
cxxflags=[]
Expand All @@ -319,6 +339,8 @@ def Makefile(MakefileName,title,ProductName,ProductPath,isLibrary,cppfiles,hfile
cxxflags.extend(['-DFASTJET_USE'])
if options.has_zlib_tag:
cxxflags.extend(['-DZIP_USE'])
if options.has_onnx_tag:
cxxflags.extend(['-DONNX_USE'])
if options.has_delphes_tag:
cxxflags.extend(['-DDELPHES_USE'])
if options.has_delphesMA5tune_tag:
Expand Down Expand Up @@ -357,6 +379,17 @@ def Makefile(MakefileName,title,ProductName,ProductPath,isLibrary,cppfiles,hfile
libs.extend(['-lz'])
file.write('LIBFLAGS += '+' '.join(libs)+'\n')

# - onnx
if options.has_onnx_ma5lib or options.has_onnx_lib:
libs=[]
if options.has_onnx_ma5lib:
libs.extend(['-lonnx_for_ma5'])
if options.has_onnx_lib:
libs.extend(['-L'+archi_info.onnx_lib_path,'-lonnxruntime'])
libs.extend(['-Wl,-rpath,'+archi_info.onnx_lib_path])

file.write('LIBFLAGS += '+' '.join(libs)+'\n')

# - root
if options.has_root_ma5lib:
libs=[]
Expand Down Expand Up @@ -435,6 +468,8 @@ def Makefile(MakefileName,title,ProductName,ProductPath,isLibrary,cppfiles,hfile
libs.append('$(MA5_BASE)/tools/SampleAnalyzer/Lib/libdelphesMA5tune_for_ma5.so')
if options.has_fastjet_ma5lib:
libs.append('$(MA5_BASE)/tools/SampleAnalyzer/Lib/libfastjet_for_ma5.so')
if options.has_onnx_ma5lib:
libs.append('$(MA5_BASE)/tools/SampleAnalyzer/Lib/libonnx_for_ma5.so')
if len(libs)!=0:
file.write('# Requirements to check before building\n')
for ind in range(0,len(libs)):
Expand Down
12 changes: 7 additions & 5 deletions madanalysis/core/launcher.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
################################################################################
#
# Copyright (C) 2012-2023 Jack Araz, Eric Conte & Benjamin Fuks
# Copyright (C) 2012-2022 Jack Araz, Eric Conte & Benjamin Fuks
# The MadAnalysis development team, email: <[email protected]>
#
# This file is part of MadAnalysis 5.
Expand Down Expand Up @@ -91,6 +91,10 @@ def DefaultInstallCard():
output.write('# fastjet_veto = 0 # 0=No, 1=Yes\n')
output.write('# fastjet_bin_path = /home/fastjet/build/bin/\n')
output.write('\n')
output.write('# -----ONNX-----\n')
output.write('# onnx_veto = 0 # 0=No, 1=Yes\n')
output.write('# onnx_bin_path = /home/onnx/build/bin/\n')
output.write('\n')
output.write('# -----PAD-----\n')
output.write('# pad_veto = 0 # 0=No, 1=Yes\n')
output.write('# pad_build_path = /home/PAD/build/\n')
Expand Down Expand Up @@ -241,8 +245,7 @@ def MainSession(mode,arglist,ma5dir,version,date):
logging.getLogger('MA5').info("* \ \_\\\ \_\ \_\ \_\ \____/ *")
logging.getLogger('MA5').info("* \/_/ \/_/\/_/\/_/\/___/ *")
logging.getLogger('MA5').info("* *")
logging.getLogger('MA5').info("* MA5 release : " + "%-24s" % main.archi_info.ma5_version + \
"%+15s" % main.archi_info.ma5_date + " *")
logging.getLogger('MA5').info("* MA5 release : " + "%-24s" % main.archi_info.ma5_version + "%+15s" % main.archi_info.ma5_date + " *")
logging.getLogger('MA5').info("* *")
logging.getLogger('MA5').info("* Comput. Phys. Commun. 184 (2013) 222-256 *")
logging.getLogger('MA5').info("* Eur. Phys. J. C74 (2014) 3103 *")
Expand Down Expand Up @@ -362,8 +365,7 @@ def Usage():
logging.getLogger('MA5').info(" -v or --version")
logging.getLogger('MA5').info(" or --release : display the version number of MadAnalysis")
logging.getLogger('MA5').info(" -b or --build : rebuild the SampleAnalyzer static library")
logging.getLogger('MA5').info(" -f or --forced : do not ask for confirmation when MA5 removes "+\
"a directory or overwrites an object")
logging.getLogger('MA5').info(" -f or --forced : do not ask for confirmation when MA5 removes a directory or overwrites an object")
logging.getLogger('MA5').info(" -s or --script : quit automatically MA5 when the script is loaded")
logging.getLogger('MA5').info(" -h or --help : dump this help")
logging.getLogger('MA5').info(" -i or --installcard : produce the default installation card in installation_card.dat")
Expand Down
4 changes: 3 additions & 1 deletion madanalysis/core/library_builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
################################################################################
#
# Copyright (C) 2012-2023 Jack Araz, Eric Conte & Benjamin Fuks
# Copyright (C) 2012-2022 Jack Araz, Eric Conte & Benjamin Fuks
# The MadAnalysis development team, email: <[email protected]>
#
# This file is part of MadAnalysis 5.
Expand Down Expand Up @@ -80,6 +80,8 @@ def checkMA5(self):
libraries.append(self.archi_info.ma5dir+'/tools/SampleAnalyzer/Lib/libdelphes_for_ma5.so')
if self.archi_info.has_delphesMA5tune:
libraries.append(self.archi_info.ma5dir+'/tools/SampleAnalyzer/Lib/libdelphesMA5tune_for_ma5.so')
if self.archi_info.has_onnx:
libraries.append(self.archi_info.ma5dir+'/tools/SampleAnalyzer/Lib/libonnx_for_ma5.so')
for library in libraries:
if not os.path.isfile(library):
self.logger.debug('\t-> library '+ library + " not found.")
Expand Down
10 changes: 10 additions & 0 deletions madanalysis/core/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -572,6 +572,15 @@ def BuildLibrary(self,forced=False):
self.archi_info.ma5dir+'/tools/SampleAnalyzer/Bin/TestZlib',\
self.archi_info.ma5dir+'/tools/SampleAnalyzer/Test/',True])

# ONNX
if self.archi_info.has_onnx:
libraries.append(['onnx', 'interface to onnx', 'onnx',\
self.archi_info.ma5dir+'/tools/SampleAnalyzer/Lib/libonnx_for_ma5.so',\
self.archi_info.ma5dir+'/tools/SampleAnalyzer/Interfaces',False])
libraries.append(['test_onnx','interface to onnx', 'test_onnx',\
self.archi_info.ma5dir+'/tools/SampleAnalyzer/Bin/TestOnnx',\
self.archi_info.ma5dir+'/tools/SampleAnalyzer/Test/',True])

# Fastjet
if self.archi_info.has_fastjet:
libraries.append(['FastJet', 'interface to FastJet', 'fastjet',\
Expand Down Expand Up @@ -637,6 +646,7 @@ def BuildLibrary(self,forced=False):
options.has_process = True
options.has_test = True
options.has_zlib = self.archi_info.has_zlib
options.has_onnx = self.archi_info.has_onnx
options.has_fastjet = self.archi_info.has_fastjet
options.has_delphes = self.archi_info.has_delphes
options.has_delphesMA5tune = self.archi_info.has_delphesMA5tune
Expand Down
5 changes: 5 additions & 0 deletions madanalysis/input/installation_options.dat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,3 +54,8 @@
# -----SCIPY-----
# scipy_veto = 0 # 0=No, 1=Yes

# -----ONNX-----
# onnx_veto = 0 # 0=No, 1=Yes
# onnx_includes = /home/onnx/include/
# onnx_libs = /home/onnx/lib/

3 changes: 3 additions & 0 deletions madanalysis/install/install_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,9 @@ def Execute(self, rawpackage):
elif package=='simplify':
from madanalysis.install.install_simplify import InstallSimplify
installer=InstallSimplify(self.main)
elif package=='onnx':
from madanalysis.install.install_onnx import InstallOnnx
installer=InstallOnnx(self.main)
else:
self.logger.error('the package "'+rawpackage+'" is unknown')
return False
Expand Down
Loading