diff --git a/.azure-pipelines/linux-CI-nightly.yml b/.azure-pipelines/linux-CI-nightly.yml deleted file mode 100644 index c232ae702..000000000 --- a/.azure-pipelines/linux-CI-nightly.yml +++ /dev/null @@ -1,127 +0,0 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python - -trigger: -- master - -jobs: - -- job: 'Test' - pool: - vmImage: 'ubuntu-latest' - strategy: - matrix: - Py39-Onnx1102-SkLatest: - python.version: '3.9' - numpy.version: '>=1.18.3' - onnx.version: '==1.10.2' - onnxrt.version: '-i https://test.pypi.org/simple/ ort-nightly' - sklearn.version: '>=1.0' - Py39-Onnx190-SkLatest: - python.version: '3.9' - numpy.version: '>=1.18.3' - onnx.version: '==1.9.0' - onnxrt.version: '-i https://test.pypi.org/simple/ ort-nightly' - sklearn.version: '>=1.0' - Py39-Onnx181-SkLatest: - python.version: '3.9' - numpy.version: '>=1.18.3' - onnx.version: '==1.8.1' - onnxrt.version: '-i https://test.pypi.org/simple/ ort-nightly' - sklearn.version: '>=1.0' - Py38-Onnx170-Sk0242: - python.version: '3.8' - numpy.version: '>=1.18.3' - onnx.version: '==1.7.0' - onnxrt.version: '-i https://test.pypi.org/simple/ ort-nightly' - sklearn.version: '==0.24.2' - Py37-Onnx160-Sk0232: - python.version: '3.7' - numpy.version: '==1.18.1' - onnx.version: '==1.6.0' - onnxrt.version: '-i https://test.pypi.org/simple/ ort-nightly' - sklearn.version: '==0.23.2' - maxParallel: 3 - - steps: - - script: | - sudo install -d -m 0777 /home/vsts/.conda/envs - displayName: Fix Conda permissions - - - task: CondaEnvironment@1 - inputs: - createCustomEnvironment: true - environmentName: 'py$(python.version)' - packageSpecs: 'python=$(python.version)' - - - script: | - conda config --set always_yes yes --set changeps1 no - conda install -c conda-forge numpy$(numpy.version) - conda install protobuf - python -m pip install --upgrade pip - displayName: 'Install environment' - - - script: | - conda install numpy$(numpy.version) - pip install openpyxl cerberus - displayName: 'install numpy' - - - script: | - conda install scikit-learn$(sklearn.version) - displayName: 'install scikit-learn' - - - script: | - pip install onnx$(onnx.version) - displayName: 'install onnx' - - - script: | - pip install flatbuffers - pip install $(onnxrt.version) - displayName: 'install onnxruntime' - - - script: | - pip install -r requirements.txt - git clone https://github.com/microsoft/onnxconverter-common - cd onnxconverter-common - pip install -e . - cd .. - pip install -r requirements-dev.txt - pip install pytest - displayName: 'install requirements' - - - script: | - pip install -e . - displayName: 'install' - - - script: | - echo "---------------" - pip show numpy - echo "---------------" - pip show onnx - echo "---------------" - pip show onnxruntime - echo "---------------" - pip show onnxconverter-common - echo "---------------" - pip show scikit-learn - echo "---------------" - displayName: 'version' - - - script: | - pytest tests --durations=0 --basetemp=temp --doctest-modules --junitxml=junit/test-results.xml - displayName: 'pytest' - - - script: | - python -m pip install openpyxl - python tests/benchmark.py - displayName: 'benchmark' - - - task: PublishTestResults@2 - inputs: - testResultsFiles: '**/test-results.xml' - testCoverageFiles: 'htmlcov/**' - testBenchmarkFiles: 'TESTDUMP/*.xlsx' - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() diff --git a/.azure-pipelines/linux-conda-CI.yml b/.azure-pipelines/linux-conda-CI.yml deleted file mode 100644 index f5bdae960..000000000 --- a/.azure-pipelines/linux-conda-CI.yml +++ /dev/null @@ -1,395 +0,0 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python - -trigger: -- master - -jobs: - -- job: 'CI_Linux' - pool: - vmImage: 'ubuntu-latest' - strategy: - matrix: - - Py311-Onnx160-Rt171-Skl140: - do.bench: '0' - python.version: '3.11' # onnxruntime-extensions needs to be update to switch to 3.12 - numpy.version: '>=1.21.1' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.16.0' # -i https://test.pypi.org/simple/ onnx==1.15.0rc2' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.17.1' - sklearn.version: '>=1.4.0' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '1' - - Py311-Onnx150-Rt170-Skl140: - do.bench: '0' - python.version: '3.11' # onnxruntime-extensions needs to be update to switch to 3.12 - numpy.version: '>=1.21.1' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.15.0' # -i https://test.pypi.org/simple/ onnx==1.15.0rc2' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.17.0' - sklearn.version: '>=1.4.0' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - Py311-Onnx150-Rt163-Skl140: - do.bench: '0' - python.version: '3.11' - numpy.version: '>=1.21.1' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.15.0' # -i https://test.pypi.org/simple/ onnx==1.15.0rc2' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.16.3' - sklearn.version: '==1.4.0' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - Py311-Onnx150-Rt161-Skl132: - do.bench: '0' - python.version: '3.11' - numpy.version: '>=1.21.1' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.15.0' # -i https://test.pypi.org/simple/ onnx==1.15.0rc2' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.16.1' - sklearn.version: '==1.3.2' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - Py311-Onnx141-Rt160-Skl131: - do.bench: '0' - python.version: '3.11' - numpy.version: '>=1.21.1' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.14.1' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.16.0' - sklearn.version: '==1.3.1' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - Py311-Onnx140-Rt151-Skl130: - do.bench: '0' - python.version: '3.11' - numpy.version: '>=1.21.1' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.14.0' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.15.1' - sklearn.version: '==1.3.0' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - Py310-Onnx140-Rt140-Skl122: - do.bench: '0' - python.version: '3.10' - numpy.version: '>=1.21.1' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.14.1' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.14.0' - sklearn.version: '==1.2.2' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - Py310-Onnx130-Rt140-Skl121: - do.bench: '0' - python.version: '3.10' - numpy.version: '>=1.21.2' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.13.0' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.14.0' - sklearn.version: '==1.2.1' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - Py310-Onnx130-Rt131-Skl120: - do.bench: '0' - python.version: '3.10' - numpy.version: '>=1.21.0' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.13.0' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.13.1' - sklearn.version: '==1.2.0' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - Py310-Onnx120-Rt131-Skl120: - do.bench: '0' - python.version: '3.10' - numpy.version: '>=1.21.0' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.12.0' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.13.1' - sklearn.version: '==1.2.0' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - Py310-Onnx120-Rt1201-Skl11: - do.bench: '0' - python.version: '3.10' - numpy.version: '>=1.21.0' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.12.0' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.12.1' - sklearn.version: '==1.1.3' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - Py39-Onnx120-Rt1111-Skl11: - do.bench: '0' - python.version: '3.9' - numpy.version: '>=1.21.0,<1.23.0' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.12.0' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.11.1' - sklearn.version: '==1.1.3' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - Py39-Onnx1110-Rt1111-Skl11: - do.bench: '0' - python.version: '3.9' - numpy.version: '>=1.21.0,<1.23.0' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.11.0' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.11.1' - sklearn.version: '==1.1.3' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - maxParallel: 3 - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - addToPath: true - architecture: 'x64' - displayName: 'Use Python $(python.version)' - - - script: | - sudo apt-get install -y language-pack-en - sudo locale-gen en_US.UTF-8 - sudo update-locale LANG=en_US.UTF-8 - displayName: 'Install packages' - - - script: | - pip install "numpy$(numpy.version)" - pip install "scipy$(scipy.version)" - pip install protobuf - python -m pip install --upgrade pip - displayName: 'Install environment' - - - script: | - if [ '$(sklearn.version)' == 'git' ] - then - git clone https://github.com/scikit-learn/scikit-learn.git --recursive - python setup.py install - else - pip install "scikit-learn$(sklearn.version)" - fi - displayName: 'install scikit-learn' - - - script: | - pip install $(onnx.version) $(onnxrt.version) - displayName: 'install onnxruntime, onnx' - - - script: | - if [ '$(onnxcc.version)' == 'git' ] - then - pip install git+https://github.com/microsoft/onnxconverter-common --no-deps - else - pip install onnxconverter-common$(onnxcc.version) --no-deps - fi - displayName: 'install onnxconverter-common' - - - script: | - pip install -r requirements.txt - pip install -r requirements-dev.txt - displayName: 'install requirements' - - - script: | - pip show onnx - displayName: 'onnx version' - - - script: | - pip install $(onnx.version) $(onnxrt.version) "numpy$(numpy.version)" scikit-learn$(sklearn.version) scipy$(scipy.version) "pandas$(pandas.version)" - displayName: install onnxruntime, onnx, numpy, scikit-learn, pandas again # to make sure the proper version is installed - - - script: | - pip install -e . - displayName: 'install' - - - script: | - echo "---------------" - pip show numpy - echo "---------------" - pip show pandas - echo "---------------" - pip show scipy - echo "---------------" - pip show pandas - echo "---------------" - pip show onnx - echo "---------------" - pip show onnxruntime - echo "---------------" - pip show onnxconverter-common - echo "---------------" - pip show scikit-learn - echo "---------------" - pip show onnxmltools - echo "---------------" - displayName: 'version' - - - script: | - if [ '$(onnx.target_opset)' != '' ] - then - export TEST_TARGET_OPSET="$(onnx.target_opset)" - fi - cd tests - python -c "from numpy import __version__;print('numpy', __version__)" - python -c "from pandas import __version__;print('pandas', __version__)" - python -c "from scipy import __version__;print('scipy', __version__)" - python -c "from sklearn import __version__;print('sklearn', __version__)" - python -c "from onnxruntime import __version__;print('onnxruntime', __version__)" - python -c "from onnx import __version__;print('onnx', __version__)" - python -c "from skl2onnx import __max_supported_opset__;print('__max_supported_opset__', __max_supported_opset__)" - python -c "import test_utils;print('max_onnxruntime_opset', test_utils.max_onnxruntime_opset())" - python -c "import test_utils;print('TARGET OPSET, IR', test_utils.TARGET_OPSET, test_utils.TARGET_IR)" - python -c "import onnx.defs;print('onnx_opset_version', onnx.defs.onnx_opset_version())" - cd .. - displayName: 'target_opset' - - - script: | - if [ '$(onnx.target_opset)' != '' ] - then - export TEST_TARGET_OPSET="$(onnx.target_opset)" - fi - pytest tests --maxfail=10 --durations=0 --basetemp=temp --doctest-modules --junitxml=junit/test-results.xml - displayName: 'pytest' - - - script: | - # some of this is triggering the following error when importing scipy on python 3.10 - # ImportError: /lib/x86_64-linux-gnu/libstdc++.so.6: version `GLIBCXX_3.4.29' - conda install -c conda-forge "lightgbm${lgbm.version}" "xgboost<2" --no-deps - pip install hummingbird-ml hummingbird "xgboost<2" lightgbm - pip install --no-deps git+https://github.com/microsoft/onnxconverter-common.git - pip install onnxmltools - displayName: 'install onnxmltools' - condition: eq(variables['run.example'], '1') - - - script: | - if [ '$(onnx.target_opset)' != '' ] - then - export TEST_TARGET_OPSET="$(onnx.target_opset)" - fi - pytest tests_onnxmltools --durations=0 --basetemp=temp --maxfail=5 - displayName: 'pytest-onnxmltools' - condition: eq(variables['run.example'], '1') - - - script: | - python -m pip install ruff - ruff skl2onnx tests tests_onnxmltools - displayName: 'ruff' - - - script: | - if [ '$(onnx.target_opset)' != '' ] - then - export TEST_TARGET_OPSET="$(onnx.target_opset)" - fi - if [ '$(do.bench)' == '1' ] - then - pip install openpyxl - coverage run --include=skl2onnx/** tests/benchmark.py - coverage report -m - coverage html - fi - displayName: 'coverage' - - - script: | - pip install -r docs/requirements.txt - pip uninstall -y skl2onnx - python setup.py install - pip install --no-deps git+https://github.com/microsoft/onnxconverter-common.git - pip install --no-deps git+https://github.com/onnx/onnxmltools.git - displayName: 'install for documentation' - condition: eq(variables['run.example'], '1') - - - script: | - if [ '$(onnx.target_opset)' != '' ] - then - export TEST_TARGET_OPSET="$(onnx.target_opset)" - fi - pytest docs/tests --durations=0 --basetemp=temp --doctest-modules -v -v --maxfail=5 - displayName: 'run documentation examples' - condition: eq(variables['run.example'], '1') - - # dot cannot be found but is missing to build the documentation - #- script: | - # if [ '$(onnx.target_opset)' != '' ] - # then - # export TEST_TARGET_OPSET="$(onnx.target_opset)" - # fi - # conda install -c anaconda graphviz - # python -m sphinx -j2 -v -N -T -b html -d build/doctrees docs dist/html - # displayName: 'build documentation' - # condition: eq(variables['run.example'], '1') - - - script: | - if [ '$(onnx.target_opset)' != '' ] - then - export TEST_TARGET_OPSET="$(onnx.target_opset)" - fi - if [ '$(do.bench)' == '1' ] - then - python tests/benchmark.py - fi - displayName: 'benchmark' - condition: eq(variables['run.example'], '0') - - - task: PublishTestResults@2 - inputs: - testResultsFiles: '**/test-results.xml' - testCoverageFiles: 'htmlcov/**' - testBenchmarkFiles: 'TESTDUMP/*.xlsx' - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() diff --git a/.azure-pipelines/win32-CI-nightly.yml b/.azure-pipelines/win32-CI-nightly.yml deleted file mode 100644 index 3206260fc..000000000 --- a/.azure-pipelines/win32-CI-nightly.yml +++ /dev/null @@ -1,128 +0,0 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python - -trigger: -- master - -jobs: - -- job: 'Test' - pool: - vmImage: 'windows-latest' - strategy: - matrix: - Py39-Onnx1102-nightlyRT-SkLatest: - python.version: '3.9' - onnx.version: 'onnx==1.10.2' - numpy.version: 'numpy>=1.18.1' - scipy.version: 'scipy' - onnxrt.version: '-i https://test.pypi.org/simple/ ort-nightly' - sklearn.version: '>=1.0' - Py39-Onnx190-nightlyRT-SkLatest: - python.version: '3.9' - onnx.version: 'onnx==1.9.1' - numpy.version: 'numpy>=1.18.1' - scipy.version: 'scipy' - onnxrt.version: '-i https://test.pypi.org/simple/ ort-nightly' - sklearn.version: '>=1.0' - Py39-Onnx180-nightlyRT-SkLatest: - python.version: '3.9' - onnx.version: 'onnx==1.8.1' - numpy.version: 'numpy>=1.18.1' - scipy.version: 'scipy' - onnxrt.version: '-i https://test.pypi.org/simple/ ort-nightly' - sklearn.version: '>=1.0' - Py38-Onnx170-nightlyRT-Sk0242: - python.version: '3.8' - onnx.version: 'onnx==1.7.0' - numpy.version: 'numpy>=1.18.1' - scipy.version: 'scipy' - onnxrt.version: '-i https://test.pypi.org/simple/ ort-nightly' - sklearn.version: '==0.24.2' - Py37-Onnx160-nightlyRT-Npy1180-Sk0232: - python.version: '3.7' - onnx.version: 'onnx==1.6.0' - numpy.version: 'numpy==1.18.1' - scipy.version: 'scipy' - onnxrt.version: '-i https://test.pypi.org/simple/ ort-nightly' - sklearn.version: '==0.23.2' - maxParallel: 3 - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - architecture: 'x64' - - - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" - displayName: Add conda to PATH - - - script: conda create --yes --quiet --name py$(python.version) -c conda-forge python=$(python.version) protobuf - displayName: Create Anaconda environment - - - script: | - call activate py$(python.version) - python -m pip install --upgrade pip - conda install $(numpy.version) $(scipy.version) - pip install $(onnx.version) - pip install openpyxl cerberus - displayName: 'Install numpy, scipy, onnx' - - - script: | - call activate py$(python.version) - pip install -r requirements.txt - git clone https://github.com/microsoft/onnxconverter-common - cd onnxconverter-common - pip install -e . - cd .. - pip install -r requirements-dev.txt - pip install pytest - displayName: 'install requirements' - - - script: | - call activate py$(python.version) - pip install $(onnx.version) - pip install flatbuffers - pip install $(onnxrt.version) - displayName: 'install onnx, onnxruntime' - - - script: | - call activate py$(python.version) - conda install scikit-learn$(sklearn.version) - displayName: 'install scikit-learn' - - - script: | - call activate py$(python.version) - pip install -e . - displayName: 'install skl2onnx' - - - script: | - call activate py$(python.version) - python -c "import numpy;print('numpy:',numpy.__version__)" - python -c "import scipy;print('scipy:',scipy.__version__)" - python -c "import sklearn;print('sklearn:',sklearn.__version__)" - python -c "import onnx;print('onnx:',onnx.__version__)" - python -c "import onnxconverter_common;print('onnxconverter-common:',onnxconverter_common.__version__)" - python -c "import onnxruntime;print('onnxruntime:',onnxruntime.__version__)" - displayName: 'version' - - - script: | - call activate py$(python.version) - pytest tests --durations=0 --basetemp=temp --doctest-modules --junitxml=junit/test-results.xml - displayName: 'pytest' - - - script: | - call activate py$(python.version) - python -m pip install openpyxl - python tests/benchmark.py - displayName: 'benchmark' - - - task: PublishTestResults@2 - inputs: - testResultsFiles: '**/test-results.xml' - testCoverageFiles: 'htmlcov/**' - testBenchmarkFiles: 'TESTDUMP/*.xlsx' - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() diff --git a/.azure-pipelines/win32-conda-CI.yml b/.azure-pipelines/win32-conda-CI.yml deleted file mode 100644 index 6d9952f9f..000000000 --- a/.azure-pipelines/win32-conda-CI.yml +++ /dev/null @@ -1,230 +0,0 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python - -trigger: -- master - -jobs: - -- job: 'CI_Windows' - pool: - vmImage: 'windows-latest' - strategy: - matrix: - Py311-Onnx160-Rt171-Skl140: - python.version: '3.11' # onnxruntime-extensions needs to be update to switch to 3.12 - onnx.version: 'onnx==1.16.0' # '-i https://test.pypi.org/simple/ onnx==1.15.0rc2' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.17.1' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '>=1.4.0' - Py311-Onnx150-Rt170-Skl140: - python.version: '3.11' # onnxruntime-extensions needs to be update to switch to 3.12 - onnx.version: 'onnx==1.15.0' # '-i https://test.pypi.org/simple/ onnx==1.15.0rc2' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.17.0' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '>=1.4.0' - Py311-Onnx150-Rt163-Skl140: - python.version: '3.11' - onnx.version: 'onnx==1.15.0' # '-i https://test.pypi.org/simple/ onnx==1.15.0rc2' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.16.3' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '==1.4.0' - Py311-Onnx150-Rt163-Skl131: - python.version: '3.11' - onnx.version: 'onnx==1.15.0' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.16.1' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '==1.3.1' - Py311-Onnx141-Rt160-Skl131: - python.version: '3.11' - onnx.version: 'onnx==1.14.1' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.16.0' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '>=1.3.1' - Py311-Onnx140-Rt151-Skl130: - python.version: '3.11' - onnx.version: 'onnx==1.14.0' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.15.1' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '==1.3.0' - Py310-Onnx140-Rt140-Skl122: - python.version: '3.10' - onnx.version: 'onnx==1.14.1' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.14.0' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '==1.2.2' - Py310-Onnx130-Rt140-Skl121: - python.version: '3.10' - onnx.version: 'onnx==1.13.0' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.14.0' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '==1.2.2' - Py310-Onnx130-Rt131-Skl120: - python.version: '3.10' - onnx.version: 'onnx==1.13.0' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.13.1' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '==1.2.0' - Py310-Onnx120-Rt131-Skl120: - python.version: '3.10' - onnx.version: 'onnx==1.12.0' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.13.1' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '==1.2.0' - Py310-Onnx120-Rt120-Skl11: - python.version: '3.10' - onnx.version: 'onnx==1.12.0' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.12.0' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '==1.1.3' - Py39-Onnx120-Rt1111-Skl11: - python.version: '3.9' - onnx.version: 'onnx==1.12.0' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.11.1' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '==1.1.3' - Py39-Onnx1110-Rt1111-Skl11: - python.version: '3.9' - onnx.version: 'onnx==1.11.0' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.11.1' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '==1.1.3' - - maxParallel: 3 - - steps: - - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" - displayName: Add conda to PATH - - - script: conda create --yes --quiet --name skl2onnxEnvironment $(numpy.version) "$(scipy.version)" pandas python=$(python.version) pytest - displayName: Create Anaconda environment - - - script: | - call activate skl2onnxEnvironment - python -m pip install --upgrade pip - conda install -y -c conda-forge protobuf - displayName: 'Install protobuf' - - - script: | - call activate skl2onnxEnvironment - python -m pip install typing-extensions>=3.6.2.1 - python -m pip install $(onnx.version) - displayName: 'Install onnx' - - - script: | - call activate skl2onnxEnvironment - python -m pip install $(onnxrt.version) - displayName: 'Install onnxruntime' - - - script: | - call activate skl2onnxEnvironment - python -c "import numpy" - displayName: 'check numpy' - - - script: | - call activate skl2onnxEnvironment - # latest version of scikit-learn are not available on conda - pip install scikit-learn$(sklearn.version) - displayName: 'install scikit-learn' - - - script: | - call activate skl2onnxEnvironment - python -c "import numpy" - displayName: 'check numpy' - - - script: | - call activate skl2onnxEnvironment - python -m pip install ruff - ruff skl2onnx tests tests_onnxmltools - displayName: 'ruff' - - - script: | - call activate skl2onnxEnvironment - pip install $(onnxcc.version) --no-deps - displayName: 'install onnxconverter-common' - - - script: | - call activate skl2onnxEnvironment - pip install -r requirements.txt - pip install -r requirements-dev.txt - displayName: 'install requirements' - - - script: | - call activate skl2onnxEnvironment - pip install -e . - displayName: 'install skl2onnx' - - - script: | - call activate skl2onnxEnvironment - python -c "import numpy" - displayName: 'check numpy' - - - script: | - call activate skl2onnxEnvironment - if "$(onnx.target_opset)" neq "" set TEST_TARGET_OPSET=$(onnx.target_opset) - cd tests - python -c "from numpy import __version__;print('numpy', __version__)" - python -c "from scipy import __version__;print('scipy', __version__)" - python -c "from sklearn import __version__;print('sklearn', __version__)" - python -c "from onnxruntime import __version__;print('onnxruntime', __version__)" - python -c "from onnx import __version__;print('onnx', __version__)" - python -c "from skl2onnx import __max_supported_opset__;print('__max_supported_opset__', __max_supported_opset__)" - python -c "import test_utils;print('max_onnxruntime_opset', test_utils.max_onnxruntime_opset())" - python -c "import test_utils;print('TARGET OPSET, IR', test_utils.TARGET_OPSET, test_utils.TARGET_IR)" - python -c "import onnx.defs;print('onnx_opset_version', onnx.defs.onnx_opset_version())" - cd .. - displayName: 'target_opset' - - - script: | - call activate skl2onnxEnvironment - if "$(onnx.target_opset)" neq "" set TEST_TARGET_OPSET=$(onnx.target_opset) - python -m pytest tests --durations=0 --basetemp=temp --doctest-modules --junitxml=junit/test-results.xml --maxfail=10 - displayName: 'pytest' - - - task: PublishTestResults@2 - inputs: - testResultsFiles: '**/test-results.xml' - testCoverageFiles: 'htmlcov/**' - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml new file mode 100644 index 000000000..d86d794e7 --- /dev/null +++ b/.github/workflows/linux-ci.yml @@ -0,0 +1,104 @@ +name: CI Linux +on: [push, pull_request] +jobs: + run: + name: Py==${{ matrix.python_version }} - sklearn${{ matrix.sklearn_version }} - ${{ matrix.onnxrt_version }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + python_version: ['3.12', '3.11', '3.10', '3.9'] + include: + - python_version: '3.12' + documentation: 0 + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx==1.16.0' + onnxrt_version: 'onnxruntime==1.18.0' + sklearn_version: '==1.4.2' + - python_version: '3.11' + documentation: 1 + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx<1.16.0' + onnxrt_version: 'onnxruntime==1.17.3' + sklearn_version: '==1.3.2' + - python_version: '3.10' + documentation: 0 + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx==1.14.1' + onnxrt_version: 'onnxruntime==1.16.0' + sklearn_version: '==1.2.2' + - python_version: '3.9' + documentation: 0 + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx==1.13.0' + onnxrt_version: 'onnxruntime==1.14.0' + sklearn_version: '==1.1.3' + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python_version }} + + - uses: tlylt/install-graphviz@v1 + + - name: Install pandoc + if: matrix.documentation == 1 + run: sudo apt-get install -y pandoc + + - name: Install requirements + run: python -m pip install -r requirements.txt + + - name: Install requirements dev + run: python -m pip install -r requirements-dev.txt + + - name: Cache pip + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- + + - name: Install specific versions + run: pip install "${{ matrix.onnx_version }}" "${{ matrix.onnxrt_version }}" "numpy${{ matrix.numpy_version }}" "scikit-learn${{ matrix.sklearn_version}}" "scipy${{ matrix.scipy_version }}" + + - name: Install + run: pip install -e . + + - name: versions + run: | + python -c "from numpy import __version__;print('numpy', __version__)" + python -c "from pandas import __version__;print('pandas', __version__)" + python -c "from scipy import __version__;print('scipy', __version__)" + python -c "from sklearn import __version__;print('sklearn', __version__)" + python -c "from onnxruntime import __version__;print('onnxruntime', __version__)" + python -c "from onnx import __version__;print('onnx', __version__)" + python -c "from skl2onnx import __max_supported_opset__;print('__max_supported_opset__', __max_supported_opset__)" + python -c "import onnx.defs;print('onnx_opset_version', onnx.defs.onnx_opset_version())" + + - name: Run tests + run: | + pytest tests --maxfail=10 --durations=10 + + - name: Run documentation tests + if: matrix.documentation == 1 + run: | + pip install -r docs/requirements.txt + pytest docs/tests --durations=10 --maxfail=5 + # python tests/benchmark.py + + - name: build + run: pip wheel . + + - uses: actions/upload-artifact@v4 + with: + path: ./dist/** diff --git a/.github/workflows/windows-macos-ci.yml b/.github/workflows/windows-macos-ci.yml new file mode 100644 index 000000000..2916a3cbe --- /dev/null +++ b/.github/workflows/windows-macos-ci.yml @@ -0,0 +1,83 @@ +name: CI Win/Mac +on: [push, pull_request] +jobs: + run: + name: ${{ matrix.os }} py==${{ matrix.python_version }} - sklearn==${{ matrix.sklearn_version }} - ${{ matrix.onnxrt_version }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest, macos-latest] + python_version: ['3.11', '3.10', '3.9'] + include: + - python_version: '3.11' + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx<1.16.0' + onnxrt_version: 'onnxruntime==1.17.3' + sklearn_version: '==1.3.2' + - python_version: '3.10' + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx<1.15' + onnxrt_version: 'onnxruntime<1.17.0' + sklearn_version: '==1.2.2' + - python_version: '3.9' + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx<1.14' + onnxrt_version: 'onnxruntime<1.16.0' + sklearn_version: '==1.2.2' + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python_version }} + + - uses: tlylt/install-graphviz@v1 + + - name: Install requirements + run: python -m pip install -r requirements.txt + + - name: Install requirements dev + run: python -m pip install -r requirements-dev.txt + + - name: Cache pip + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- + + - name: Install specific versions + run: pip install "${{ matrix.onnx_version }}" "${{ matrix.onnxrt_version }}" "numpy${{ matrix.numpy_version }}" "scikit-learn${{ matrix.sklearn_version}}" "scipy${{ matrix.scipy_version }}" + + - name: Install + run: pip install -e . + + - name: versions + run: | + python -c "from numpy import __version__;print('numpy', __version__)" + python -c "from pandas import __version__;print('pandas', __version__)" + python -c "from scipy import __version__;print('scipy', __version__)" + python -c "from sklearn import __version__;print('sklearn', __version__)" + python -c "from onnxruntime import __version__;print('onnxruntime', __version__)" + python -c "from onnx import __version__;print('onnx', __version__)" + python -c "from skl2onnx import __max_supported_opset__;print('__max_supported_opset__', __max_supported_opset__)" + python -c "import onnx.defs;print('onnx_opset_version', onnx.defs.onnx_opset_version())" + + - name: Run tests + run: | + pytest tests --maxfail=10 --durations=10 + + - name: build + run: pip wheel . + + - uses: actions/upload-artifact@v4 + with: + path: ./dist/** diff --git a/.gitignore b/.gitignore index f2ab678fb..8c9a8ba76 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ build/ # test generated files *.onnx +*.whl .pytest_cache .cache htmlcov diff --git a/README.md b/README.md index 6ac428186..2b032c3c3 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@

-[![Build Status](https://dev.azure.com/onnxmltools/sklearn-onnx/_apis/build/status%2Fonnx.sklearn-onnx.linux.CI?branchName=refs%2Fpull%2F1020%2Fmerge)](https://dev.azure.com/onnxmltools/sklearn-onnx/_build/latest?definitionId=21&branchName=refs%2Fpull%2F1020%2Fmerge) +[![Linux](https://github.com/onnx/sklearn-onnx/actions/workflows/linux-ci.yml/badge.svg)](https://github.com/onnx/sklearn-onnx/actions/workflows/linux-ci.yml) -[![Build Status](https://dev.azure.com/onnxmltools/sklearn-onnx/_apis/build/status%2Fonnx.sklearn-onnx.win.CI?branchName=refs%2Fpull%2F1020%2Fmerge)](https://dev.azure.com/onnxmltools/sklearn-onnx/_build/latest?definitionId=22&branchName=refs%2Fpull%2F1020%2Fmerge) +[![Windows/Macos](https://github.com/onnx/sklearn-onnx/actions/workflows/windows-macos-ci.yml/badge.svg)](https://github.com/onnx/sklearn-onnx/actions/workflows/windows-macos-ci.yml) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) diff --git a/docs/examples/plot_backend.py b/docs/examples/plot_backend.py index 3599ff5c4..3b7643b33 100644 --- a/docs/examples/plot_backend.py +++ b/docs/examples/plot_backend.py @@ -11,7 +11,7 @@ .. index:: backend *ONNX Runtime* extends the -`onnx backend API `_ to run predictions using this runtime. Let's use the API to compute the prediction diff --git a/docs/examples/plot_benchmark_pipeline.py b/docs/examples/plot_benchmark_pipeline.py index 5634cfcf2..4428d2ea7 100644 --- a/docs/examples/plot_benchmark_pipeline.py +++ b/docs/examples/plot_benchmark_pipeline.py @@ -15,7 +15,7 @@ `Pipelining: chaining a PCA and a logistic regression `_. There is one change because -`ONNX-ML Imputer `_ does not handle string type. This cannot be part of the final ONNX pipeline and must be removed. Look for comment starting with ``---`` below. diff --git a/docs/examples/plot_cast_transformer.py b/docs/examples/plot_cast_transformer.py index 82cfef35e..66439329b 100644 --- a/docs/examples/plot_cast_transformer.py +++ b/docs/examples/plot_cast_transformer.py @@ -113,7 +113,7 @@ def maxdiff(a1, a2): # by ``(x / y)`` and this division must happen in double. # By default, the *sklearn-onnx* assumes every # computer should happen in float. `ONNX 1.7 specifications -# `_ # does not support double scaling (input and output does, # but not the parameters). The solution needs to diff --git a/docs/examples/plot_complex_pipeline.py b/docs/examples/plot_complex_pipeline.py index f9404d944..8457bdf11 100644 --- a/docs/examples/plot_complex_pipeline.py +++ b/docs/examples/plot_complex_pipeline.py @@ -24,7 +24,7 @@ `_. There is one change because `ONNX-ML Imputer -`_ does not handle string type. This cannot be part of the final ONNX pipeline and must be removed. Look for comment starting with ``---`` below. diff --git a/docs/examples/plot_custom_model.py b/docs/examples/plot_custom_model.py index 5a35811c0..94e7faf19 100644 --- a/docs/examples/plot_custom_model.py +++ b/docs/examples/plot_custom_model.py @@ -354,7 +354,7 @@ def predictable_tsne_converter(scope, operator, container): name = scope.get_unique_operator_name("Scaler") # The parameter follows the specifications of ONNX - # https://github.com/onnx/onnx/blob/master/docs/Operators-ml.md#ai.onnx.ml.Scaler + # https://github.com/onnx/onnx/blob/main/docs/Operators-ml.md#ai.onnx.ml.Scaler attrs = dict( name=name, scale=op.inv_std_.ravel().astype(numpy.float32), diff --git a/docs/examples/plot_intermediate_outputs.py b/docs/examples/plot_intermediate_outputs.py index 5c3b6493f..2408ca659 100644 --- a/docs/examples/plot_intermediate_outputs.py +++ b/docs/examples/plot_intermediate_outputs.py @@ -20,7 +20,7 @@ `_. There is one change because `ONNX-ML Imputer -`_ does not handle string type. This cannot be part of the final ONNX pipeline and must be removed. Look for comment starting with ``---`` below. diff --git a/docs/examples/plot_investigate_pipeline.py b/docs/examples/plot_investigate_pipeline.py index 2f532d139..1ca787e27 100644 --- a/docs/examples/plot_investigate_pipeline.py +++ b/docs/examples/plot_investigate_pipeline.py @@ -17,7 +17,7 @@ compose/plot_digits_pipe.html>`_. There is one change because `ONNX-ML Imputer -`_ does not handle string type. This cannot be part of the final ONNX pipeline and must be removed. Look for comment starting with ``---`` below. diff --git a/docs/examples/plot_onnx_operators.py b/docs/examples/plot_onnx_operators.py index cd7d7af1b..48fa0fbc9 100644 --- a/docs/examples/plot_onnx_operators.py +++ b/docs/examples/plot_onnx_operators.py @@ -15,12 +15,12 @@ the backend used to compute the predictions. It is better to look first if the existing operators can be used. The list is available on *github* and gives the `basic operators -`_ +`_ and others `dedicated to machine learning -`_. +`_. *ONNX* has a Python API which can be used to define an *ONNX* graph: `PythonAPIOverview.md -`_. +`_. But it is quite verbose and makes it difficult to describe big graphs. *sklearn-onnx* implements a nicer way to test *ONNX* operators. @@ -29,11 +29,11 @@ Let's try the example given by ONNX documentation: `ONNX Model Using Helper Functions -`_. It relies on *protobuf* whose definition can be found on github `onnx.proto -`_. +`_. """ import onnxruntime import numpy diff --git a/docs/examples/plot_pipeline.py b/docs/examples/plot_pipeline.py index 6eae7ed14..1f230dd8e 100644 --- a/docs/examples/plot_pipeline.py +++ b/docs/examples/plot_pipeline.py @@ -48,7 +48,7 @@ # Draw a model with ONNX # ++++++++++++++++++++++ # We use `net_drawer.py -# `_ +# `_ # included in *onnx* package. # We use *onnx* to load the model # in a different way than before. diff --git a/docs/examples/plot_pipeline_lightgbm.py b/docs/examples/plot_pipeline_lightgbm.py index fe89480a6..63b8df391 100644 --- a/docs/examples/plot_pipeline_lightgbm.py +++ b/docs/examples/plot_pipeline_lightgbm.py @@ -64,11 +64,11 @@ # # The converter is implemented in *onnxmltools*: # `onnxmltools...LightGbm.py -# `_. # and the shape calculator: # `onnxmltools...Classifier.py -# `_. ############################################## diff --git a/docs/examples/plot_pipeline_xgboost.py b/docs/examples/plot_pipeline_xgboost.py index 272369cc2..720373aa3 100644 --- a/docs/examples/plot_pipeline_xgboost.py +++ b/docs/examples/plot_pipeline_xgboost.py @@ -79,11 +79,11 @@ # # The converter is implemented in *onnxmltools*: # `onnxmltools...XGBoost.py -# `_. # and the shape calculator: # `onnxmltools...Classifier.py -# `_. ############################################## diff --git a/docs/exts/github_link.py b/docs/exts/github_link.py index 9d87a68de..a57d24dd1 100644 --- a/docs/exts/github_link.py +++ b/docs/exts/github_link.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # Source: https://github.com/scikit-learn/scikit-learn/blob/ -# master/doc/sphinxext/github_link.py +# main/doc/sphinxext/github_link.py from operator import attrgetter import inspect import subprocess diff --git a/docs/index.rst b/docs/index.rst index 8d49467ab..4fa04e447 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,10 +9,10 @@ sklearn-onnx: Convert your scikit-learn model into ONNX :widths: 5 5 * - Linux - Windows - * - .. image:: https://dev.azure.com/onnxmltools/sklearn-onnx/_apis/build/status/sklearn-onnx-linux-conda-ci?branchName=master - :target: https://dev.azure.com/onnxmltools/sklearn-onnx/_build/latest?definitionId=5?branchName=master - - .. image:: https://dev.azure.com/onnxmltools/sklearn-onnx/_apis/build/status/sklearn-onnx-win32-conda-ci?branchName=master - :target: https://dev.azure.com/onnxmltools/sklearn-onnx/_build/latest?definitionId=5?branchName=master + * - .. image:: https://github.com/onnx/sklearn-onnx/actions/workflows/linux-ci.yml/badge.svg + :target: https://github.com/onnx/sklearn-onnx/actions/workflows/linux-ci.yml + - .. image:: https://github.com/onnx/sklearn-onnx/actions/workflows/windows-macos-ci.yml/badge.svg + :target: https://github.com/onnx/sklearn-onnx/actions/workflows/windows-macos-ci.yml *sklearn-onnx* enables you to convert models from @@ -43,7 +43,7 @@ or submit a new one. Sources are available on The converter can convert a model for a specific version of ONNX. Every ONNX release is labelled with an opset number returned by function `onnx_opset_version -`_. +`_. This function returns the default value for parameter target opset (parameter *target_opset*) if it is not specified when converting the model. Every operator is versioned. diff --git a/docs/pipeline.rst b/docs/pipeline.rst index 8fbb42ce9..7b3b3e02f 100644 --- a/docs/pipeline.rst +++ b/docs/pipeline.rst @@ -8,7 +8,7 @@ Convert a pipeline *skl2onnx* converts any machine learning pipeline into *ONNX* pipelines. Every transformer or predictors is converted into one or multiple nodes into the *ONNX* graph. -Any `ONNX backend `_ +Any `ONNX backend `_ can then use this graph to compute equivalent outputs for the same inputs. .. _l-complex-pipeline: @@ -148,8 +148,8 @@ order: * **converter(scope, operator, container):** The converter converts the transformers or predictors into *ONNX* nodes. Each node can an *ONNX* - `operator `_ or - `ML operator `_ or + `operator `_ or + `ML operator `_ or custom *ONNX* operators. As *sklearn-onnx* may convert pipelines with model coming from other libraries, @@ -228,7 +228,7 @@ isolate the source of the differences. The function may then be used to investigate each component independently. The following piece of code is extracted from unit test `test_investigate.py `_ and converts +blob/main/tests/test_investigate.py>`_ and converts a pipeline and each of its components independently. :: diff --git a/docs/requirements.txt b/docs/requirements.txt index 58c0069ce..a14fffedd 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -5,12 +5,13 @@ coverage flake8 furo joblib -lightgbm<4.0 +lightgbm; sys_platform != 'darwin' loky matplotlib mlinsights>=0.3.631 nbsphinx onnx +onnxmltools onnxruntime pillow py-spy @@ -20,7 +21,6 @@ pyinstrument pyod pytest pytest-cov -scikit-learn>=1.1 skl2onnx sphinx sphinxcontrib-blockdiag @@ -29,4 +29,4 @@ sphinx-runpython tabulate tqdm wheel -xgboost<2 +xgboost diff --git a/docs/supported.rst b/docs/supported.rst index bda1942cb..147387343 100644 --- a/docs/supported.rst +++ b/docs/supported.rst @@ -46,9 +46,9 @@ easy to insert into a graph. These operators get dynamically added and the list depends on the installed *ONNX* package. The documentation for these operators can be found on github: `ONNX Operators.md -`_ +`_ and `ONNX-ML Operators -`_. +`_. Associated to `onnxruntime `_, the mapping makes it easier to easily check the output of the *ONNX* operators on any data as shown diff --git a/docs/tutorial/plot_cbegin_opset.py b/docs/tutorial/plot_cbegin_opset.py index b0c0bfb38..b5341a7bc 100644 --- a/docs/tutorial/plot_cbegin_opset.py +++ b/docs/tutorial/plot_cbegin_opset.py @@ -103,7 +103,7 @@ def get_domain_opset(onx): ######################################## # It shows that the model cannot be converted for opset # below 5. Operator `Reshape `_ changed in +# onnx/blob/main/docs/Operators.md#Reshape>`_ changed in # opset 5: a parameter became an input. The converter # does not support *opset < 5* because runtimes usually do not. # diff --git a/docs/tutorial/plot_gexternal_lightgbm.py b/docs/tutorial/plot_gexternal_lightgbm.py index 30a3db914..afdfc0eda 100644 --- a/docs/tutorial/plot_gexternal_lightgbm.py +++ b/docs/tutorial/plot_gexternal_lightgbm.py @@ -54,11 +54,11 @@ # # The converter is implemented in :epkg:`onnxmltools`: # `onnxmltools...LightGbm.py -# `_. # and the shape calculator: # `onnxmltools...Classifier.py -# `_. update_registered_converter( diff --git a/docs/tutorial/plot_gexternal_lightgbm_reg.py b/docs/tutorial/plot_gexternal_lightgbm_reg.py index 19f4d4e03..000958584 100644 --- a/docs/tutorial/plot_gexternal_lightgbm_reg.py +++ b/docs/tutorial/plot_gexternal_lightgbm_reg.py @@ -69,11 +69,11 @@ # # The converter is implemented in :epkg:`onnxmltools`: # `onnxmltools...LightGbm.py -# `_. # and the shape calculator: # `onnxmltools...Regressor.py -# `_. diff --git a/docs/tutorial/plot_gexternal_xgboost.py b/docs/tutorial/plot_gexternal_xgboost.py index adde4cf15..d05d2d158 100644 --- a/docs/tutorial/plot_gexternal_xgboost.py +++ b/docs/tutorial/plot_gexternal_xgboost.py @@ -73,11 +73,11 @@ # # The converter is implemented in :epkg:`onnxmltools`: # `onnxmltools...XGBoost.py -# `_. # and the shape calculator: # `onnxmltools...Classifier.py -# `_. update_registered_converter( diff --git a/docs/tutorial/plot_jcustom_syntax.py b/docs/tutorial/plot_jcustom_syntax.py index e56fc8912..595526c77 100644 --- a/docs/tutorial/plot_jcustom_syntax.py +++ b/docs/tutorial/plot_jcustom_syntax.py @@ -9,9 +9,9 @@ There are two ways to write a converter. The first one is less verbose and easier to understand (see `k_means.py `_). +main/skl2onnx/operator_converters/k_means.py>`_). The other is very verbose (see `ada_boost.py `_ +sklearn-onnx/blob/main/skl2onnx/operator_converters/ada_boost.py>`_ for an example). The first way is used in :ref:`l-plot-custom-converter`. diff --git a/requirements-dev.txt b/requirements-dev.txt index 7d137d5d0..0da5e225f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,7 @@ # tests black onnxruntime-extensions +onnxscript pandas py-cpuinfo pybind11 @@ -9,13 +10,7 @@ pytest-cov ruff wheel -# docs -furo -onnxscript -sphinx -sphinxcontrib-blockdiag -tqdm - # docs/examples -lightgbm<4.0 +xgboost +lightgbm; sys_platform != 'darwin' matplotlib diff --git a/skl2onnx/algebra/automation.py b/skl2onnx/algebra/automation.py index 6f8c47a74..187be25c5 100644 --- a/skl2onnx/algebra/automation.py +++ b/skl2onnx/algebra/automation.py @@ -192,7 +192,7 @@ def process_documentation(doc): if doc is None: doc = "" doc = textwrap.dedent(doc) - main_docs_url = "https://github.com/onnx/onnx/blob/master/" + main_docs_url = "https://github.com/onnx/onnx/blob/main/" rep = { "[the doc](IR.md)": "`ONNX <{0}docs/IR.md>`_", "[the doc](Broadcasting.md)": ( @@ -227,7 +227,7 @@ def process_documentation(doc): return "\n".join(lines) def build_doc_url(sch): - doc_url = "https://github.com/onnx/onnx/blob/master/docs/Operators" + doc_url = "https://github.com/onnx/onnx/blob/main/docs/Operators" if "ml" in sch.domain: doc_url += "-ml" doc_url += ".md" diff --git a/skl2onnx/algebra/onnx_ops.py b/skl2onnx/algebra/onnx_ops.py index 0eb8db69d..4517e1963 100644 --- a/skl2onnx/algebra/onnx_ops.py +++ b/skl2onnx/algebra/onnx_ops.py @@ -164,9 +164,9 @@ def dynamic_class_creation(cache=False): Automatically generates classes for each of the operators module *onnx* defines and described at `Operators - `_ + `_ and `Operators - `_. """ cache_dir = cache_folder() diff --git a/skl2onnx/common/_topology.py b/skl2onnx/common/_topology.py index 1197cb8b4..1be1db166 100644 --- a/skl2onnx/common/_topology.py +++ b/skl2onnx/common/_topology.py @@ -1507,7 +1507,7 @@ def convert_topology( raise RuntimeError( "Parameter target_opset {} > {} is higher than the " "version of the installed onnx package. See " - "https://github.com/onnx/onnx/blob/master/docs/" + "https://github.com/onnx/onnx/blob/main/docs/" "Versioning.md#released-versions" ".".format(onnx_target_opset, found) ) diff --git a/skl2onnx/operator_converters/calibrated_classifier_cv.py b/skl2onnx/operator_converters/calibrated_classifier_cv.py index 808958b95..3ca087df4 100644 --- a/skl2onnx/operator_converters/calibrated_classifier_cv.py +++ b/skl2onnx/operator_converters/calibrated_classifier_cv.py @@ -242,7 +242,7 @@ def convert_calibrated_classifier_base_estimator( # In the following graph, variable names are in lower case characters only # and operator names are in upper case characters. We borrow operator names # from the official ONNX spec: - # https://github.com/onnx/onnx/blob/master/docs/Operators.md + # https://github.com/onnx/onnx/blob/main/docs/Operators.md # All variables are followed by their shape in []. # # Symbols: @@ -473,7 +473,7 @@ def convert_sklearn_calibrated_classifier_cv( # In the following graph, variable names are in lower case characters only # and operator names are in upper case characters. We borrow operator names # from the official ONNX spec: - # https://github.com/onnx/onnx/blob/master/docs/Operators.md + # https://github.com/onnx/onnx/blob/main/docs/Operators.md # All variables are followed by their shape in []. # # Symbols: diff --git a/skl2onnx/operator_converters/local_outlier_factor.py b/skl2onnx/operator_converters/local_outlier_factor.py index 2dcd0398a..5a5beda8d 100644 --- a/skl2onnx/operator_converters/local_outlier_factor.py +++ b/skl2onnx/operator_converters/local_outlier_factor.py @@ -55,7 +55,7 @@ def convert_sklearn_local_outlier_factor( if options["optim"] == "cdist": warnings.warn( "Option p=%r may not be compatible with the runtime. " - "See https://github.com/microsoft/onnxruntime/blob/master/" + "See https://github.com/microsoft/onnxruntime/blob/main/" "docs/ContribOperators.md#com.microsoft.CDist." ) kwargs["p"] = op.p diff --git a/skl2onnx/operator_converters/naive_bayes.py b/skl2onnx/operator_converters/naive_bayes.py index e1a63d3ae..1e1af6a80 100644 --- a/skl2onnx/operator_converters/naive_bayes.py +++ b/skl2onnx/operator_converters/naive_bayes.py @@ -319,7 +319,7 @@ def convert_sklearn_naive_bayes( # Note: In the following graph, variable names are in lower case # characters only and operator names are in upper case characters. # We borrow operator names from the official ONNX spec: - # https://github.com/onnx/onnx/blob/master/docs/Operators.md + # https://github.com/onnx/onnx/blob/main/docs/Operators.md # All variables are followed by their shape in []. # # Symbols: diff --git a/skl2onnx/operator_converters/ovr_decision_function.py b/skl2onnx/operator_converters/ovr_decision_function.py index a45834598..fb014c8b3 100644 --- a/skl2onnx/operator_converters/ovr_decision_function.py +++ b/skl2onnx/operator_converters/ovr_decision_function.py @@ -25,7 +25,7 @@ def convert_sklearn_ovr_decision_function( ): # Applies _ovr_decision_function. # See https://github.com/scikit-learn/scikit-learn/blob/ - # master/sklearn/utils/multiclass.py#L407: + # main/sklearn/utils/multiclass.py#L407: # :: # def _ovr_decision_function(predictions, confidences, n_classes): # diff --git a/skl2onnx/operator_converters/support_vector_machines.py b/skl2onnx/operator_converters/support_vector_machines.py index 6f57771d2..ea5160123 100644 --- a/skl2onnx/operator_converters/support_vector_machines.py +++ b/skl2onnx/operator_converters/support_vector_machines.py @@ -39,7 +39,7 @@ def convert_sklearn_svm_regressor( *onnxruntime* and *scikit-learn* do not return the same raw scores. *scikit-learn* returns aggregated scores as a *matrix[N, C]* coming from `_ovr_decision_function - `_. *onnxruntime* returns the raw score from *svm* algorithm as a *matrix[N, (C(C-1)/2]*. """ @@ -167,7 +167,7 @@ def convert_sklearn_svm_classifier( *onnxruntime* and *scikit-learn* do not return the same raw scores. *scikit-learn* returns aggregated scores as a *matrix[N, C]* coming from `_ovr_decision_function - `_. *onnxruntime* returns the raw score from *svm* algorithm as a *matrix[N, (C(C-1)/2]*. """ @@ -285,7 +285,7 @@ def convert_sklearn_svm_classifier( ): # Applies _ovr_decision_function. # See https://github.com/scikit-learn/scikit-learn/blob/ - # master/sklearn/utils/multiclass.py#L407: + # main/sklearn/utils/multiclass.py#L407: # :: # _ovr_decision_function(dec < 0, -dec, len(self.classes_)) diff --git a/skl2onnx/operator_converters/text_vectoriser.py b/skl2onnx/operator_converters/text_vectoriser.py index 2601f4812..e00c819a8 100644 --- a/skl2onnx/operator_converters/text_vectoriser.py +++ b/skl2onnx/operator_converters/text_vectoriser.py @@ -20,7 +20,7 @@ def _intelligent_split(text, op, tokenizer, existing): to name ngrams. ``'a b'`` could be ``('a ', 'b')`` or ``('a', ' b')``. See `ngram sequence - `_. """ if op.analyzer == "word": @@ -173,7 +173,7 @@ def convert_sklearn_text_vectorizer( to a custom tokenizer based on `python wrapper for re2 `_ or its sources `pyre2 `_ - (`syntax `_). + (`syntax `_). If the regular expression is not specified and if the instance of TfidfVectorizer is using the default pattern ``(?u)\\\\b\\\\w\\\\w+\\\\b``, it is replaced by diff --git a/skl2onnx/sklapi/sklearn_text.py b/skl2onnx/sklapi/sklearn_text.py index 6f81caae0..b6b1b90fe 100644 --- a/skl2onnx/sklapi/sklearn_text.py +++ b/skl2onnx/sklapi/sklearn_text.py @@ -15,7 +15,7 @@ class NGramsMixin(VectorizerMixin): """ Overloads method `_word_ngrams - `_ + `_ to get tuples instead of string in member `vocabulary_ `_. of `TfidfVectorizer` or :epkg:`CountVectorizer`. @@ -104,7 +104,7 @@ def _fix_vocabulary(expected, new_voc): class TraceableCountVectorizer(CountVectorizer, NGramsMixin): """ Inherits from :class:`NGramsMixin` which overloads method `_word_ngrams - `_ + `_ to keep more information about n-grams but still produces the same outputs than `CountVectorizer`. @@ -178,7 +178,7 @@ def fit_transform(self, X, y=None): class TraceableTfidfVectorizer(TfidfVectorizer, NGramsMixin): """ Inherits from :class:`NGramsMixin` which overloads method `_word_ngrams - `_ + `_ to keep more information about n-grams but still produces the same outputs than `TfidfVectorizer`. diff --git a/skl2onnx/sklapi/woe_transformer_onnx.py b/skl2onnx/sklapi/woe_transformer_onnx.py index 18c966318..231e22ade 100644 --- a/skl2onnx/sklapi/woe_transformer_onnx.py +++ b/skl2onnx/sklapi/woe_transformer_onnx.py @@ -224,7 +224,7 @@ def _update_feature(self, feature): def onnx_attributes(self): """ See `TreeEnsembleRegressor - `_. """ atts = dict( diff --git a/tests/test_algebra_onnx_doc.py b/tests/test_algebra_onnx_doc.py index 1a2d8b722..da70a236c 100644 --- a/tests/test_algebra_onnx_doc.py +++ b/tests/test_algebra_onnx_doc.py @@ -46,6 +46,7 @@ def test_transpose2(self): @unittest.skipIf( sys.platform.startswith("win"), reason="onnx schema are incorrect on Windows" ) + @unittest.skipIf(TARGET_OPSET <= 20, reason="not available") def test_doc_onnx(self): rst = get_rst_doc() assert "**Summary**" in rst @@ -53,6 +54,7 @@ def test_doc_onnx(self): @unittest.skipIf( sys.platform.startswith("win"), reason="onnx schema are incorrect on Windows" ) + @unittest.skipIf(TARGET_OPSET <= 20, reason="not available") def test_doc_sklearn(self): try: rst = get_rst_doc_sklearn() diff --git a/tests/test_issues_2024.py b/tests/test_issues_2024.py index 569e6a027..1bdc8abef 100644 --- a/tests/test_issues_2024.py +++ b/tests/test_issues_2024.py @@ -115,7 +115,7 @@ def test_issue_1055(self): assert_almost_equal(expected, got[1], decimal=2) @unittest.skipIf( - pv.Version(ort_version) < pv.Version("1.16.0"), + pv.Version(ort_version) < pv.Version("1.17.3"), reason="opset 19 not implemented", ) def test_issue_1069(self): diff --git a/tests/test_sklearn_one_hot_encoder_converter.py b/tests/test_sklearn_one_hot_encoder_converter.py index 58f72100f..5073c4c1d 100644 --- a/tests/test_sklearn_one_hot_encoder_converter.py +++ b/tests/test_sklearn_one_hot_encoder_converter.py @@ -2,6 +2,7 @@ """Tests scikit-learn's OneHotEncoder converter.""" import unittest +import sys import packaging.version as pv import numpy from numpy.testing import assert_almost_equal @@ -373,18 +374,9 @@ def test_one_hot_encoder_string_drop_first_2(self): data, model, model_onnx, basename="SklearnOneHotEncoderStringDropFirst2" ) - @unittest.skipIf( - onnx_opset_version() < 19, reason="missing ops in reference implementation" - ) - @ignore_warnings(category=RuntimeWarning) - def test_shape_inference(self): + def _shape_inference(self, engine): cat_columns_openings = ["cat_1", "cat_2"] - num_columns_openings = [ - "num_1", - "num_2", - "num_3", - "num_4", - ] + num_columns_openings = ["num_1", "num_2", "num_3", "num_4"] regression_aperturas = LinearRegression() @@ -450,23 +442,34 @@ def test_shape_inference(self): ] ) - # ReferenceEvaluator - with self.subTest(engine="onnx"): - ref = ReferenceEvaluator(model_onnx) - res = ref.run(None, feeds) - self.assertEqual(1, len(res)) - self.assertEqual(expected.shape, res[0].shape) - assert_almost_equal(expected, res[0]) - # onnxruntime - with self.subTest(engine="onnxruntime"): - sess = InferenceSession( + if engine == "onnxruntime": + ref = InferenceSession( model_onnx.SerializeToString(), providers=["CPUExecutionProvider"] ) - res = sess.run(None, feeds) - self.assertEqual(1, len(res)) - self.assertEqual(expected.shape, res[0].shape) - assert_almost_equal(expected, res[0]) + else: + ref = ReferenceEvaluator(model_onnx) + + res = ref.run(None, feeds) + self.assertEqual(1, len(res)) + self.assertEqual(expected.shape, res[0].shape) + assert_almost_equal(expected, res[0]) + + @unittest.skipIf( + onnx_opset_version() < 19, reason="missing ops in reference implementation" + ) + @ignore_warnings(category=RuntimeWarning) + @unittest.skipIf(sys.platform == "darwin", "interesting discrepancy") + def test_shape_inference_onnx(self): + self._shape_inference("onnx") + + @unittest.skipIf( + onnx_opset_version() < 19, reason="missing ops in reference implementation" + ) + @ignore_warnings(category=RuntimeWarning) + @unittest.skipIf(sys.platform == "darwin", "interesting discrepancy") + def test_shape_inference_onnxruntime(self): + self._shape_inference("onnxruntime") @unittest.skipIf( not one_hot_encoder_supports_drop(), diff --git a/tests/test_utils/reference_implementation_afe.py b/tests/test_utils/reference_implementation_afe.py index 2275ea67a..499f4e399 100644 --- a/tests/test_utils/reference_implementation_afe.py +++ b/tests/test_utils/reference_implementation_afe.py @@ -49,7 +49,7 @@ def _run(self, data, indices): the output has still two like a matrix with one row. The implementation follows what :epkg:`onnxruntime` does in `array_feature_extractor.cc - `_. """ res = _array_feature_extrator(data, indices)