Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
make it work on win, MPI having issues still
  • Loading branch information
fschlimb authored and GitHub Enterprise committed Sep 14, 2018
1 parent 8f3ce30 commit 130a127
Show file tree
Hide file tree
Showing 14 changed files with 286 additions and 187 deletions.
12 changes: 12 additions & 0 deletions conda-recipe/bld.bat
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
if "%PY3K%"=="1" (
set ARGS=""
) else {
set ARGS="--old-and-unmanageable"
}

set DAAL4PY_VERSION=%PKG_VERSION%
set CNCROOT=%PREFIX%
set TBBROOT=%PREFIX%
set DAALROOT=%PREFIX%

%PYTHON% setup.py install %ARGS%
9 changes: 5 additions & 4 deletions conda-recipe/build.sh
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
#!/bin/bash

if [ -n "$PY_VER" ] && [ "${PY_VER:0:1}" -lt "3" ]; then
ARG="--old-and-unmanageable"
if [ "$PY3K" == "1" ]; then
ARGS=""
else
ARG=""
ARGS="--old-and-unmanageable"
fi
DAAL4PY_VERSION=$PKG_VERSION CNCROOT=${PREFIX} TBBROOT=${PREFIX} DAALROOT=${PREFIX} ${PYTHON} setup.py install $ARG

DAAL4PY_VERSION=$PKG_VERSION CNCROOT=${PREFIX} TBBROOT=${PREFIX} DAALROOT=${PREFIX} ${PYTHON} setup.py install $ARGS
4 changes: 3 additions & 1 deletion conda-recipe/meta.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ requirements:
- tbb-devel
- cnc >=1.2.300
- daal-devel >=2018.0.1
- daal-static >=2018.0.1 [win]
- cython
- jinja2
run:
Expand All @@ -27,7 +28,8 @@ requirements:
- daal >=2018.0.1

build:
number: {{buildnumber}}
number: {{buildnumber}}
include_recipe: False

test:
imports:
Expand Down
75 changes: 75 additions & 0 deletions examples/run_examples.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
#*******************************************************************************
# Copyright 2014-2017 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#******************************************************************************/

import os
import platform
import struct
import subprocess
import sys

from os.path import join as jp
from time import gmtime, strftime

exdir = os.path.dirname(os.path.realpath(__file__))

IS_WIN = platform.system() == 'Windows'

assert 8 * struct.calcsize('P') in [32, 64]

if 8 * struct.calcsize('P') == 32:
logdir = jp(exdir, '_results', 'ia32')
else:
logdir = jp(exdir, '_results', 'intel64')

def get_exe_cmd(ex):
if 'batch' in ex:
return '"' + sys.executable + '" "' + ex + '"'
else:
return 'mpirun -n 4 -genv DIST_CNC=MPI "' + sys.executable + '" "' + ex + '"'

def run_all():
success = 0
n = 0
if not os.path.exists(logdir):
os.makedirs(logdir)
for (dirpath, dirnames, filenames) in os.walk(exdir):
for script in filenames:
if script.endswith('.py') and script not in ['run_examples.py', '__init__.py']:
n += 1
logfn = jp(logdir, script.replace('.py', '.res'))
with open(logfn, 'w') as logfile:
print('\n##### ' + jp(dirpath, script))
execute_string = get_exe_cmd(jp(dirpath, script))
proc = subprocess.Popen(execute_string if IS_WIN else ['/bin/bash', '-c', execute_string],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=(True if IS_WIN else False))
out = proc.communicate()[0]
logfile.write(out.decode('ascii'))
if proc.returncode:
print(out)
print(strftime("%H:%M:%S", gmtime()) + '\tFAILED\t' + script + '\twith errno\t' + str(proc.returncode))
else:
success += 1
print(strftime("%H:%M:%S", gmtime()) + '\tPASSED\t' + script)
if success != n:
print('{}/{} examples passed, {} failed'.format(success,n, n - success))
print('Error(s) occured. Logs can be found in ' + logdir)
else:
print('{}/{} examples passed'.format(success,n))

if __name__ == '__main__':
run_all()
19 changes: 11 additions & 8 deletions generator/gen_daal4py.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ def read(self):
what namespace it is affiliated with. Once done, we have a dictionary where the key is the namespace
and the values are namespace class objects. These objects carry all information as extracted by parse.py.
"""
print('reading headers from ' + self.include_root)
for (dirpath, dirnames, filenames) in os.walk(self.include_root):
for filename in filenames:
if filename.endswith('.h') and not 'neural_networks' in dirpath and not any(filename.endswith(x) for x in cython_interface.ignore_files):
Expand Down Expand Up @@ -288,7 +289,7 @@ def to_hltype(self, ns, t):
if t.endswith('ModelPtr'):
thens = self.get_ns(ns, t, attrs=['typedefs'])
return ('daal::' + thens + '::ModelPtr', 'class', tns)
if t in ['data_management::NumericTablePtr',] or t in ifaces.values():
if t in ['data_management::NumericTablePtr'] or t in ifaces.values():
return ('daal::' + t, 'class', tns)
if 'Batch' in self.namespace_dict[ns].classes and t in self.namespace_dict[ns].classes['Batch'].typedefs:
tns, tname = splitns(self.namespace_dict[ns].classes['Batch'].typedefs[t])
Expand Down Expand Up @@ -325,7 +326,7 @@ def get_values(self, ns, n):
if nns:
nn = splitns(n)[1]
if nn in self.namespace_dict[nns].enums:
return [nns + '::' + x for x in self.namespace_dict[nns].enums[nn]]
return [re.sub(r'(?<!daal::)algorithms::', r'daal::algorithms::', nns + '::' + x) for x in self.namespace_dict[nns].enums[nn]]
return ['unknown_' + nns + '_class_'+n]
return ['unknown_'+n]

Expand All @@ -341,7 +342,7 @@ def get_tmplarg(self, ns, n):
if nns:
nn = splitns(n)[1]
if nn in self.namespace_dict[nns].enums:
return nns + '::' + nn
return re.sub(r'(?<!daal::)algorithms::', r'daal::algorithms::', nns + '::' + nn)
return 'unknown_' + nns + '_class_'+n
return 'unknown_'+n

Expand Down Expand Up @@ -460,7 +461,8 @@ def prepare_modelmaps(self, ns, mname='Model'):
if not any(g.endswith(x) for x in ['SerializationTag',]):
gn = splitns(g)[1].replace('get', '')
if not any(gn == x[1] for x in jparams['named_gets']):
jparams['named_gets'].append((huhu[g], gn))
typ = re.sub(r'(?<!daal::)data_management', r'daal::data_management', huhu[g])
jparams['named_gets'].append((typ, gn))
return jparams


Expand Down Expand Up @@ -555,7 +557,8 @@ def prepare_hlwrapper(self, ns, mode, func):
'template_args': None,
'pargs': None})
for s in v['specs']:
tdecl.append({'template_decl': OrderedDict([(x, v['tmpl_decl'][x]) for x in s['template_decl']]),
tdecl.append({'template_decl': OrderedDict([(re.sub(r'(?<!daal::)algorithms::', r'daal::algorithms::', x),
v['tmpl_decl'][x]) for x in s['template_decl']]),
'template_args': [s['expl'][x] if x in s['expl'] else x for x in v['tmpl_decl']],
'pargs': [s['expl'][x] for x in s['expl']]})
else:
Expand Down Expand Up @@ -624,6 +627,7 @@ def prepare_hlwrapper(self, ns, mode, func):
pval = tmp
if pval != None:
thetype = (hlt if hlt else parms[p])
pval = re.sub(r'(?<!daal::)algorithms::', r'daal::algorithms::', pval)
if tmp in jparams['params_req']:
td['params_req'][tmp] = pval
decl_req.append('const ' + thetype + ' ' + tmp)
Expand All @@ -645,7 +649,6 @@ def prepare_hlwrapper(self, ns, mode, func):
tmp_iargs_decl = []
tmp_iargs_call = []
tmp_input_args = []
setinputs = ''
inp = self.get_class_for_typedef(ns, 'Batch', 'InputType')
if not inp and 'Input' in self.namespace_dict[ns].classes:
inp = (ns, 'Input')
Expand All @@ -665,6 +668,7 @@ def prepare_hlwrapper(self, ns, mode, func):
if 'NumericTablePtr' in itype:
#ns in has_dist and iname in has_dist[ns]['step_specs'][0].inputnames or iname in ['data', 'labels', 'dependentVariable', 'tableToFill']:
itype = 'TableOrFList *'
ins = re.sub(r'(?<!daal::)algorithms::', r'daal::algorithms::', ins)
tmp_iargs_decl.insert(i, 'const ' + itype + ' ' + iname + dflt)
tmp_iargs_call.insert(i, iname)
tmp_input_args.insert(i, (ins + '::' + iname, iname, itype))
Expand Down Expand Up @@ -752,7 +756,7 @@ def hlapi(self, algo_patterns):
for e in self.namespace_dict[ns].enums:
for v in self.namespace_dict[ns].enums[e]:
vv = ns + '::' + v
cpp_begin += ' '*4 +'{"' + v + '", ' + vv + '},\n'
cpp_begin += ' '*4 +'{"' + v + '", daal::' + vv + '},\n'
cpp_begin += '};\n\n'

hlargs = {}
Expand Down Expand Up @@ -798,7 +802,6 @@ def gen_daal4py(daalroot, outdir, warn_all=False):
global no_warn
if warn_all:
no_warn = {}

iface = cython_interface(jp(daalroot, 'include', 'algorithms'))
iface.read()
cpp_h, cpp_cpp, pyx_file = iface.hlapi(['kmeans',
Expand Down
21 changes: 11 additions & 10 deletions generator/wrapper_gen.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ def my_procid():
{% for m in enum_gets %}
extern "C" {{m[2]}} {{'*' if 'Ptr' in m[2] else ''}} get_{{flatname}}_{{m[1]}}({{class_type}} * obj_)
{
return RAW< {{m[2]}} >()((*obj_)->get({{m[0]}}::{{m[1]}}));
return RAW< {{m[2]}} >()((*obj_)->get(daal::{{m[0]}}::{{m[1]}}));
}
{% endfor %}
{% for m in named_gets %}
Expand Down Expand Up @@ -280,17 +280,17 @@ def __dealloc__(self):
{% macro gen_typedefs(ns, template_decl, template_args, mode="Batch", suffix="b", step_spec=None) %}
{% set disttarg = (step_spec.name.rsplit('__', 1)[0] + ', ') if step_spec.name else "" %}
{% if template_decl|length > 0 %}
typedef {{ns}}::{{mode}}<{{disttarg + ', '.join(template_args)}}> algo{{suffix}}_type;
typedef daal::{{ns}}::{{mode}}<{{disttarg + ', '.join(template_args)}}> algo{{suffix}}_type;
{% else %}
typedef {{ns}}::{{mode}} algo{{suffix}}_type;
typedef daal::{{ns}}::{{mode}} algo{{suffix}}_type;
{% endif %}
{% if step_spec %}
typedef {{step_spec.iomanager}}< algo{{suffix}}_type, {{', '.join(step_spec.input)}}, {{step_spec.output}}{{(","+",".join(step_spec.iomargs)) if step_spec.iomargs else ""}} > iom{{suffix}}_type;
{% else %}
{% if iombatch %}
typedef {{iombatch}} iom{{suffix}}_type;
{% else %}
typedef IOManager< algo{{suffix}}_type, services::SharedPtr< typename algo{{suffix}}_type::InputType >, services::SharedPtr< typename algo{{suffix}}_type::ResultType > > iom{{suffix}}_type;
typedef IOManager< algo{{suffix}}_type, daal::services::SharedPtr< typename algo{{suffix}}_type::InputType >, daal::services::SharedPtr< typename algo{{suffix}}_type::ResultType > > iom{{suffix}}_type;
{% endif %}
{% endif %}
{%- endmacro %}
Expand Down Expand Up @@ -397,7 +397,7 @@ def __dealloc__(self):
{{algo}}__iface__(bool d=false) : _distributed(d) {}
{% set indent = 23+(result_map.class_type|length) %}
virtual {{result_map.class_type}} * compute({{(',\n'+' '*indent).join(iargs_decl|cppdecl)}},
{{' '*indent}}bool setup_only = false) {assert(false);}
{{' '*indent}}bool setup_only = false) {assert(false); return NULL;}
};
"""

Expand Down Expand Up @@ -622,11 +622,12 @@ def setup(self,
{% endfor %}
{%- endmacro %}
extern "C" {{algo}}__iface__ * mk_{{algo}}({{pargs_decl|cpp_decl(pargs_call, template_decl, 56+2*(algo|length))}})
extern "C" {{algo}}__iface__ * mk_{{algo}}({{pargs_decl|cpp_decl(pargs_call, template_decl, 37+2*(algo|length))}})
{
{% if template_decl %}
{{tfactory(template_decl.items()|list, algo+'_manager', pargs_call, dist=dist)}}
throw std::invalid_argument("no equivalent(s) for C++ template argument(s) in mk_{{algo}}");
std::cerr << "no equivalent(s) for C++ template argument(s) exist in mk_{{algo}}" << std::endl;
return NULL;
{% else %}
return new {{algo}}_manager({{', '.join(pargs_call)}}, distributed);
{% endif %}
Expand Down Expand Up @@ -781,9 +782,9 @@ def __flat(typ):
else:
r = '_'.join(nn)
return ('c_' if cpp and typ.endswith('__iface__') else '') + r + (' *' if cpp and any(typ.endswith(x) for x in ['__iface__', 'Ptr']) else '')
ty = ty.replace('daal::algorithms::kernel_function::KernelIfacePtr', 'services::SharedPtr<kernel_function::KernelIface>')
ty = re.sub(r'(daal::)?(algorithms::)?(engines::)?EnginePtr', r'services::SharedPtr<engines::BatchBase>', ty)
ty = re.sub(r'(?:daal::)?(?:algorithms::)?([^:]+::)BatchPtr', r'services::SharedPtr<\1Batch>', ty)
ty = ty.replace('daal::algorithms::kernel_function::KernelIfacePtr', 'daal::services::SharedPtr<kernel_function::KernelIface>')
ty = re.sub(r'(daal::)?(algorithms::)?(engines::)?EnginePtr', r'daal::services::SharedPtr<engines::BatchBase>', ty)
ty = re.sub(r'(?:daal::)?(?:algorithms::)?([^:]+::)BatchPtr', r'daal::services::SharedPtr<\1Batch>', ty)
ty = re.sub(r'(daal::)?services::SharedPtr<([^>]+)>', r'\2__iface__', ty)
return ' '.join([__flat(x).replace('const', '') for x in ty.split(' ')])
return [_flat(x) for x in t] if isinstance(t,list) else _flat(t)
Expand Down
Loading

0 comments on commit 130a127

Please sign in to comment.