Fix #400, support multi:softmax objective

.azure-pipelines/linux-conda-CI.yml

@@ -49,6 +49,12 @@ jobs:
         ONNXRT_PATH: onnxruntime==1.6.0
         COREML_PATH: git+https://github.com/apple/coremltools@3.1
         xgboost.version: '>=1.0'
+      Python37-180-RT160-xgb11:
+        python.version: '3.7'
+        ONNX_PATH: onnx==1.8.0
+        ONNXRT_PATH: onnxruntime==1.6.0
+        COREML_PATH: git+https://github.com/apple/coremltools@3.1
+        xgboost.version: '<1.2'
     maxParallel: 3
 
   steps:

.azure-pipelines/win32-conda-CI.yml

@@ -1,113 +1,113 @@
-# Python package
-# Create and test a Python package on multiple Python versions.
-# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
-# https://docs.microsoft.com/azure/devops/pipelines/languages/python
-
-trigger:
-- master
-
-jobs:
-
-- job: 'Test'
-  pool:
-    vmImage: 'vs2017-win2016'
-  strategy:
-    matrix:
-      Python35-141-RT030:
-        python.version: '3.5'
-        ONNX_PATH: onnx==1.4.1
-        ONNXRT_PATH: onnxruntime==0.3.0
-        COREML_PATH: https://github.com/apple/coremltools/archive/v2.0.zip
-        sklearn.version: '==0.19.1'
-
-      Python36-141-RT030:
-        python.version: '3.6'
-        ONNX_PATH: onnx==1.4.1
-        ONNXRT_PATH: onnxruntime==0.3.0
-        COREML_PATH: git+https://github.com/apple/coremltools@3.1
-        sklearn.version: ''
-
-      Python37-150-RT040:
-        python.version: '3.7'
-        ONNX_PATH: onnx==1.5.0
-        ONNXRT_PATH: onnxruntime==0.4.0
-        COREML_PATH: git+https://github.com/apple/coremltools@3.1
-        sklearn.version: ''
-
-      Python37-160-RT111:
-        python.version: '3.7'
-        ONNX_PATH: onnx==1.6.0
-        ONNXRT_PATH: onnxruntime==1.1.1
-        COREML_PATH: git+https://github.com/apple/coremltools@3.1
-        sklearn.version: ''
-
-      Python37-170-RT130:
-        python.version: '3.7'
-        ONNX_PATH: onnx==1.7.0
-        ONNXRT_PATH: onnxruntime==1.3.0
-        COREML_PATH: git+https://github.com/apple/coremltools@3.1
-        sklearn.version: ''
-
-      Python37-180-RT160:
-        python.version: '3.7'
-        ONNX_PATH: onnx==1.8.0
-        ONNXRT_PATH: onnxruntime==1.6.0
-        COREML_PATH: git+https://github.com/apple/coremltools@3.1
-        sklearn.version: ''
-
-    maxParallel: 3
-
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '$(python.version)'
-      architecture: 'x64'
-
-  - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
-    displayName: Add conda to PATH
-
-  - script: conda create --yes --quiet --name py$(python.version) -c conda-forge python=$(python.version) numpy protobuf
-    displayName: Create Anaconda environment
-
-  - script: |
-      call activate py$(python.version)
-      python -m pip install --upgrade pip numpy
-      echo Test numpy installation... && python -c "import numpy"
-      python -m pip install %COREML_PATH% %ONNX_PATH%
-      python -m pip install tensorflow-cpu==1.15.0
-      python -m pip install tf2onnx==1.5.6
-      python -m pip install git+https://github.com/microsoft/onnxconverter-common
-      python -m pip install git+https://github.com/onnx/keras-onnx
-      echo Test onnxconverter-common installation... && python -c "import onnxconverter_common"
-      python -m pip install -r requirements.txt
-      python -m pip install -r requirements-dev.txt
-      python -m pip install %ONNXRT_PATH%
-      python -m pip install scikit-learn$(sklearn.version)
-      echo Test onnxruntime installation... && python -c "import onnxruntime"
-      REM install libsvm from github
-      git clone --recursive https://github.com/cjlin1/libsvm libsvm
-      copy libsvm\windows\*.dll libsvm\python
-      set PYTHONPATH=libsvm\python;%PYTHONPATH%
-      dir libsvm\python
-      echo Test libsvm installation... && python -c "import svmutil"
-      echo "debug environment" && path
-      python -m pip show pytest
-    displayName: 'Install dependencies'
-
-  - script: |
-      call activate py$(python.version)
-      python -m flake8 ./onnxmltools
-    displayName: 'run flake8 check'
-
-  - script: |
-      call activate py$(python.version)
-      set PYTHONPATH=libsvm\python;%PYTHONPATH%
-      python -m pip install -e .
-      python -m pytest tests --ignore=tests/sparkml --doctest-modules --junitxml=junit/test-results.xml
-    displayName: 'pytest - onnxmltools'
-
-  - task: PublishTestResults@2
-    inputs:
-      testResultsFiles: '**/test-results.xml'
-      testRunTitle: 'Python $(python.version)'
-    condition: succeededOrFailed()
+# Python package
+# Create and test a Python package on multiple Python versions.
+# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
+# https://docs.microsoft.com/azure/devops/pipelines/languages/python
+
+trigger:
+- master
+
+jobs:
+
+- job: 'Test'
+  pool:
+    vmImage: 'vs2017-win2016'
+  strategy:
+    matrix:
+      Python35-141-RT030:
+        python.version: '3.5'
+        ONNX_PATH: onnx==1.4.1
+        ONNXRT_PATH: onnxruntime==0.3.0
+        COREML_PATH: https://github.com/apple/coremltools/archive/v2.0.zip
+        sklearn.version: '==0.19.1'
+
+      Python36-141-RT030:
+        python.version: '3.6'
+        ONNX_PATH: onnx==1.4.1
+        ONNXRT_PATH: onnxruntime==0.3.0
+        COREML_PATH: git+https://github.com/apple/coremltools@3.1
+        sklearn.version: ''
+
+      Python37-150-RT040:
+        python.version: '3.7'
+        ONNX_PATH: onnx==1.5.0
+        ONNXRT_PATH: onnxruntime==0.4.0
+        COREML_PATH: git+https://github.com/apple/coremltools@3.1
+        sklearn.version: ''
+
+      Python37-160-RT111:
+        python.version: '3.7'
+        ONNX_PATH: onnx==1.6.0
+        ONNXRT_PATH: onnxruntime==1.1.1
+        COREML_PATH: git+https://github.com/apple/coremltools@3.1
+        sklearn.version: ''
+
+      Python37-170-RT130:
+        python.version: '3.7'
+        ONNX_PATH: onnx==1.7.0
+        ONNXRT_PATH: onnxruntime==1.3.0
+        COREML_PATH: git+https://github.com/apple/coremltools@3.1
+        sklearn.version: ''
+
+      Python37-180-RT160:
+        python.version: '3.7'
+        ONNX_PATH: onnx==1.8.0
+        ONNXRT_PATH: onnxruntime==1.6.0
+        COREML_PATH: git+https://github.com/apple/coremltools@3.1
+        sklearn.version: ''
+
+    maxParallel: 3
+
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '$(python.version)'
+      architecture: 'x64'
+
+  - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
+    displayName: Add conda to PATH
+
+  - script: conda create --yes --quiet --name py$(python.version) -c conda-forge python=$(python.version) numpy protobuf
+    displayName: Create Anaconda environment
+
+  - script: |
+      call activate py$(python.version)
+      python -m pip install --upgrade pip numpy
+      echo Test numpy installation... && python -c "import numpy"
+      python -m pip install %COREML_PATH% %ONNX_PATH%
+      python -m pip install tensorflow-cpu==1.15.0
+      python -m pip install tf2onnx==1.5.6
+      python -m pip install git+https://github.com/microsoft/onnxconverter-common
+      python -m pip install git+https://github.com/onnx/keras-onnx
+      echo Test onnxconverter-common installation... && python -c "import onnxconverter_common"
+      python -m pip install -r requirements.txt
+      python -m pip install -r requirements-dev.txt
+      python -m pip install %ONNXRT_PATH%
+      python -m pip install scikit-learn$(sklearn.version)
+      echo Test onnxruntime installation... && python -c "import onnxruntime"
+      REM install libsvm from github
+      git clone --recursive https://github.com/cjlin1/libsvm libsvm
+      copy libsvm\windows\*.dll libsvm\python
+      set PYTHONPATH=libsvm\python;%PYTHONPATH%
+      dir libsvm\python
+      echo Test libsvm installation... && python -c "import svmutil"
+      echo "debug environment" && path
+      python -m pip show pytest
+    displayName: 'Install dependencies'
+
+  - script: |
+      call activate py$(python.version)
+      python -m flake8 ./onnxmltools
+    displayName: 'run flake8 check'
+
+  - script: |
+      call activate py$(python.version)
+      set PYTHONPATH=libsvm\python;%PYTHONPATH%
+      python -m pip install -e .
+      python -m pytest tests --ignore=tests/sparkml --doctest-modules --junitxml=junit/test-results.xml
+    displayName: 'pytest - onnxmltools'
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFiles: '**/test-results.xml'
+      testRunTitle: 'Python $(python.version)'
+    condition: succeededOrFailed()

onnxmltools/convert/xgboost/_parse.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import json
+import re
 import numpy as np
 from xgboost import XGBRegressor, XGBClassifier
 from onnxconverter_common.data_types import FloatTensorType
@@ -53,8 +54,15 @@ def _get_attributes(booster):
         # classification
         kwargs['num_target'] = 0
         if trees > ntrees > 0:
+            state = booster.__getstate__()
+            bstate = bytes(state['handle'])
+            reg = re.compile(b'(multi:[a-z]{1,15})')
+            objs = list(set(reg.findall(bstate)))
+            if len(objs) != 1:
+                raise RuntimeError(
+                    "Unable to guess objective in {}.".format(objs))
             kwargs['num_class'] = trees // ntrees
-            kwargs["objective"] = "multi:softprob"
+            kwargs["objective"] = objs[0].decode('ascii')
         else:
             kwargs['num_class'] = 1
             kwargs["objective"] = "binary:logistic"

onnxmltools/convert/xgboost/operator_converters/XGBoost.py

@@ -245,7 +245,7 @@ def convert(scope, operator, container):
         else:
             # See https://github.com/dmlc/xgboost/blob/master/src/common/math.h#L35.
             attr_pairs['post_transform'] = "SOFTMAX"
-            # attr_pairs['base_values'] = [base_score for n in range(ncl)]
+            attr_pairs['base_values'] = [base_score for n in range(ncl)]
             attr_pairs['class_ids'] = [v % ncl for v in attr_pairs['class_treeids']]
 
         classes = xgb_node.classes_
@@ -264,8 +264,10 @@ def convert(scope, operator, container):
                                op_domain='ai.onnx.ml',
                                name=scope.get_unique_operator_name('TreeEnsembleClassifier'),
                                **attr_pairs)
-        elif objective == "multi:softprob":
+        elif objective in ("multi:softprob", "multi:softmax"):
             ncl = len(js_trees) // params['n_estimators']
+            if objective == 'multi:softmax':
+                attr_pairs['post_transform'] = 'NONE'
             container.add_node('TreeEnsembleClassifier', operator.input_full_names,
                                operator.output_full_names,
                                op_domain='ai.onnx.ml',

onnxmltools/utils/tests_helper.py

@@ -97,9 +97,12 @@ def dump_data_and_model(data, model, onnx=None, basename="model", folder=None,
             if model_dict['objective'].startswith('binary'):
                 score = model.predict(datax)
                 prediction = [score > 0.5, numpy.vstack([1-score, score]).T]
-            elif model_dict['objective'].startswith('multi'):
+            elif model_dict['objective'].startswith('multi:softprob'):
                 score = model.predict(datax)
                 prediction = [score.argmax(axis=1), score]
+            elif model_dict['objective'].startswith('multi:softmax'):
+                score = model.predict(datax, output_margin=True)
+                prediction = [score.argmax(axis=1), score]
             else:
                 prediction = [model.predict(datax)]
         elif hasattr(model, "predict_proba"):

tests/xgboost/test_xgboost_converters.py

@@ -4,7 +4,6 @@
 Tests scilit-learn's tree-based methods' converters.
 """
 import os
-import sys
 import unittest
 import numpy as np
 import pandas
@@ -192,7 +191,7 @@ def test_xgboost_booster_classifier_bin(self):
                             allow_failure="StrictVersion(onnx.__version__) < StrictVersion('1.3.0')",
                             basename="XGBBoosterMCl")
 
-    def test_xgboost_booster_classifier_multiclass(self):
+    def test_xgboost_booster_classifier_multiclass_softprob(self):
         x, y = make_classification(n_classes=3, n_features=5,
                                    n_samples=100,
                                    random_state=42, n_informative=3)
@@ -208,7 +207,25 @@ def test_xgboost_booster_classifier_multiclass(self):
         dump_data_and_model(x_test.astype(np.float32),
                             model, model_onnx,
                             allow_failure="StrictVersion(onnx.__version__) < StrictVersion('1.3.0')",
-                            basename="XGBBoosterMCl")
+                            basename="XGBBoosterMClSoftProb")
+
+    def test_xgboost_booster_classifier_multiclass_softmax(self):
+        x, y = make_classification(n_classes=3, n_features=5,
+                                   n_samples=100,
+                                   random_state=42, n_informative=3)
+        x_train, x_test, y_train, _ = train_test_split(x, y, test_size=0.5,
+                                                       random_state=42)
+
+        data = DMatrix(x_train, label=y_train)
+        model = train({'objective': 'multi:softmax',
+                       'n_estimators': 3, 'min_child_samples': 1,
+                       'num_class': 3}, data)
+        model_onnx = convert_xgboost(model, 'tree-based classifier',
+                                     [('input', FloatTensorType([None, x.shape[1]]))])
+        dump_data_and_model(x_test.astype(np.float32),
+                            model, model_onnx,
+                            allow_failure="StrictVersion(onnx.__version__) < StrictVersion('1.3.0')",
+                            basename="XGBBoosterMClSoftMax")
 
     def test_xgboost_booster_classifier_reg(self):
         x, y = make_classification(n_classes=2, n_features=5,