[WIP] ENH: Hellinger distance tree split criterion for imbalanced data classification

.gitignore

@@ -119,4 +119,5 @@ cythonize.dat
 doc/_build/
 doc/auto_examples/
 doc/generated/
-doc/bibtex/auto
+doc/bibtex/auto
+/imblearn/tree/*.c

.travis.yml

@@ -33,13 +33,13 @@ matrix:
     - env: DISTRIB="ubuntu" TEST_DOC="true" TEST_NUMPYDOC="false"
     # Latest release
     - env: DISTRIB="conda" PYTHON_VERSION="3.7"
-           NUMPY_VERSION="*" SCIPY_VERSION="*" SKLEARN_VERSION="master"
+           NUMPY_VERSION="*" SCIPY_VERSION="*" SKLEARN_VERSION="master" CYTHON_VERSION="*"
            OPTIONAL_DEPS="keras" TEST_DOC="true" TEST_NUMPYDOC="false"
     - env: DISTRIB="conda" PYTHON_VERSION="3.7"
-           NUMPY_VERSION="*" SCIPY_VERSION="*" SKLEARN_VERSION="master"
+           NUMPY_VERSION="*" SCIPY_VERSION="*" SKLEARN_VERSION="master" CYTHON_VERSION="*"
            OPTIONAL_DEPS="tensorflow" TEST_DOC="true" TEST_NUMPYDOC="false"
     - env: DISTRIB="conda" PYTHON_VERSION="3.7"
-           NUMPY_VERSION="*" SCIPY_VERSION="*" SKLEARN_VERSION="master"
+           NUMPY_VERSION="*" SCIPY_VERSION="*" SKLEARN_VERSION="master" CYTHON_VERSION="*"
            OPTIONAL_DEPS="false" TEST_DOC="false" TEST_NUMPYDOC="true"
 
 install: source build_tools/travis/install.sh

MANIFEST.in

@@ -1,6 +1,7 @@
-
 recursive-include doc *
 recursive-include examples *
+include imblearn/tree *.pyx
+include imblearn/tree *.pyd
 include AUTHORS.rst
 include CONTRIBUTING.ms
 include LICENSE

appveyor.yml

@@ -37,6 +37,7 @@ install:
   - activate testenv
   - conda install scipy numpy joblib -y -q
   - pip install --pre -f https://sklearn-nightly.scdn8.secure.raxcdn.com scikit-learn
+  - conda install -c anaconda cython -y -q
   - conda install %OPTIONAL_DEP% -y -q
   - conda install pytest pytest-cov -y -q
   - pip install codecov

build_tools/travis/install.sh

@@ -33,7 +33,7 @@ if [[ "$DISTRIB" == "conda" ]]; then
     # provided versions
     conda create -n testenv --yes python=$PYTHON_VERSION pip
     source activate testenv
-    conda install --yes numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION
+    conda install --yes numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION cython=$CYTHON_VERSION
 
     if [[ "$OPTIONAL_DEPS" == "keras" ]]; then
         conda install --yes pandas keras tensorflow=1
@@ -66,12 +66,14 @@ elif [[ "$DISTRIB" == "ubuntu" ]]; then
     pip install --pre -f https://sklearn-nightly.scdn8.secure.raxcdn.com scikit-learn
     pip3 install pandas
     pip3 install pytest pytest-cov codecov sphinx numpydoc
+    pip3 install cython
 
 fi
 
 python --version
 python -c "import numpy; print('numpy %s' % numpy.__version__)"
 python -c "import scipy; print('scipy %s' % scipy.__version__)"
+python -c "import Cython; print('Cython %s' % Cython.__version__)"
 
 pip install -e .
 ccache --show-stats

build_tools/travis/test_script.sh

@@ -19,6 +19,7 @@ run_tests(){
     python --version
     python -c "import numpy; print('numpy %s' % numpy.__version__)"
     python -c "import scipy; print('scipy %s' % scipy.__version__)"
+    python -c "import Cython; print('Cython %s' % Cython.__version__)"
     python -c "import multiprocessing as mp; print('%d CPUs' % mp.cpu_count())"
 
     pytest --cov=$MODULE -r sx --pyargs $MODULE

doc/api.rst

@@ -196,6 +196,29 @@ Imbalance-learn provides some fast-prototyping tools.
 
 .. _metrics_ref:
 
+:mod:`imblearn.tree`: Tree split criterion
+==================================
+
+.. automodule:: imblearn.tree
+    :no-members:
+    :no-inherited-members:
+
+.. currentmodule:: imblearn
+
+.. autosummary::
+   :toctree: generated/
+   :template: class.rst
+
+   tree.criterion.HellingerDistanceCriterion
+
+.. autosummary::
+   :toctree: generated/
+   :template: function.rst
+
+   pipeline.make_pipeline
+
+.. _metrics_ref:
+
 :mod:`imblearn.metrics`: Metrics
 ================================
 

doc/miscellaneous.rst

@@ -169,4 +169,4 @@ will be passed to ``fit_generator``::
 
 .. topic:: References
 
-  * :ref:`sphx_glr_auto_examples_applications_porto_seguro_keras_under_sampling.py`
+  * :ref:`sphx_glr_auto_examples_applications_porto_seguro_keras_under_sampling.py`

doc/tree.rst

@@ -0,0 +1,26 @@
+.. _tree-split:
+
+==============
+Tree-split
+==============
+
+.. currentmodule:: imblearn.tree
+
+.. _cluster_centroids:
+
+
+Hellinger Distance split
+====================
+
+Hellinger Distance is used to quantify the similarity between two probability distributions.
+When used as split criterion in Decision Tree Classifier it makes it skew insensitive and helps tackle the imbalance problem.
+
+  >>> import numpy as np
+  >>> from sklearn.ensemble import RandomForestClassifier
+  >>> from imblearn.tree.criterion import HellingerDistanceCriterion
+
+  >>> hdc = HellingerDistanceCriterion(1, np.array([2],dtype='int64'))
+  >>> clf = RandomForestClassifier(criterion=hdc)
+
+:class:`HellingerDistanceCriterion` offers a Cython implementation of Hellinger Distance
+as a criterion for decision tree split compatible with sklearn tree based classification models.

doc/user_guide.rst

@@ -12,6 +12,7 @@ User Guide
    introduction.rst
    over_sampling.rst
    under_sampling.rst
+   tree.rst
    combine.rst
    ensemble.rst
    miscellaneous.rst

doc/whats_new/v0.4.rst

@@ -129,6 +129,9 @@ Enhancement
   :class:`BorderlineSMOTE` and :class:`SVMSMOTE`.
   :issue:`440` by :user:`Guillaume Lemaitre <glemaitre>`.
 
+- Add support for Hellinger Distance as sklearn classification tree split criterion.
+  By :user: `Evgeni Dubov <EvgeniDubov>`.
+
 - Allow :class:`imblearn.over_sampling.RandomOverSampler` can return indices
   using the attributes ``return_indices``.
   :issue:`439` by :user:`Hugo Gascon<hgascon>` and

examples/tree/README.txt

@@ -0,0 +1,9 @@
+.. _tree_examples:
+
+Example using Hellinger Distance as tree split criterion 
+========================================================
+
+Hellinger Distance is used to quantify the similarity between two probability distributions.
+When used as split criterion in Decision Tree Classifier it makes it skew insensitive and helps tackle the imbalance problem.
+This is Cython implementation of Hellinger Distance as a criterion for decision tree split compatible with sklearn tree based classification models.
+

examples/tree/train_model_with_hellinger_distance_criterion.py

@@ -0,0 +1,17 @@
+import numpy as np
+
+from sklearn.datasets import make_classification
+from sklearn.model_selection import train_test_split
+from sklearn.ensemble import RandomForestClassifier
+
+from imblearn.tree.criterion import HellingerDistanceCriterion
+
+X, y = make_classification(
+    n_samples=10000, n_features=40, n_informative=5,
+    n_classes=2, weights=[0.05, 0.95], random_state=1)
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4)
+
+hdc = HellingerDistanceCriterion(1, np.array([2], dtype='int64'))
+clf = RandomForestClassifier(criterion=hdc, max_depth=4, n_estimators=100)
+clf.fit(X_train, y_train)
+print(clf.score(X_test, y_test))

imblearn/tree/criterion.pyx

@@ -0,0 +1,92 @@
+# Author: Evgeni Dubov <[email protected]>
+#
+# License: BSD 3 clause
+
+#cython: language_level=3, boundscheck=False
+
+from libc.math cimport sqrt, pow
+from libc.math cimport abs
+
+import numpy as np
+
+from sklearn.tree._criterion cimport ClassificationCriterion
+from sklearn.tree._criterion cimport SIZE_t
+
+cdef double INFINITY = np.inf
+
+
+cdef class HellingerDistanceCriterion(ClassificationCriterion):
+    """Hellinger distance criterion.
+
+
+    """
+
+    cdef double proxy_impurity_improvement(self) nogil:
+        cdef:
+            double impurity_left
+            double impurity_right
+
+        self.children_impurity(&impurity_left, &impurity_right)
+
+        return impurity_right + impurity_left
+
+    cdef double impurity_improvement(self, double impurity) nogil:
+        cdef:
+            double impurity_left
+            double impurity_right
+
+        self.children_impurity(&impurity_left, &impurity_right)
+
+        return impurity_right + impurity_left
+
+    cdef double node_impurity(self) nogil:
+        cdef:
+            SIZE_t* n_classes = self.n_classes
+            double* sum_total = self.sum_total
+            double hellinger = 0.0
+            double sq_count
+            double count_k
+            SIZE_t k, c
+
+        for k in range(self.n_outputs):
+            for c in range(n_classes[k]):
+                hellinger += 1.0
+
+        return hellinger / self.n_outputs
+
+    cdef void children_impurity(self, double* impurity_left,
+                                double* impurity_right) nogil:
+        cdef:
+            SIZE_t* n_classes = self.n_classes
+            double* sum_left = self.sum_left
+            double* sum_right = self.sum_right
+            double hellinger_left = 0.0
+            double hellinger_right = 0.0
+            double count_k1 = 0.0
+            double count_k2 = 0.0
+            SIZE_t k, c
+
+        # stop splitting in case reached pure node with 0 samples of second
+        # class
+        if sum_left[1] + sum_right[1] == 0:
+            impurity_left[0] = -INFINITY
+            impurity_right[0] = -INFINITY
+            return
+
+        for k in range(self.n_outputs):
+            if(sum_left[0] + sum_right[0] > 0):
+                count_k1 = sqrt(sum_left[0] / (sum_left[0] + sum_right[0]))
+            if(sum_left[1] + sum_right[1] > 0):
+                count_k2 = sqrt(sum_left[1] / (sum_left[1] + sum_right[1]))
+
+            hellinger_left += pow((count_k1  - count_k2), 2)
+
+            if(sum_left[0] + sum_right[0] > 0):
+                count_k1 = sqrt(sum_right[0] / (sum_left[0] + sum_right[0]))
+            if(sum_left[1] + sum_right[1] > 0):
+                count_k2 = sqrt(sum_right[1] / (sum_left[1] + sum_right[1]))
+
+            hellinger_right += pow((count_k1  - count_k2), 2)
+
+        impurity_left[0]  = hellinger_left  / self.n_outputs
+        impurity_right[0] = hellinger_right / self.n_outputs

imblearn/tree/setup.py

@@ -0,0 +1,18 @@
+import numpy
+
+
+def configuration(parent_package='', top_path=None):
+    from numpy.distutils.misc_util import Configuration
+    config = Configuration('tree', parent_package, top_path)
+    libraries = []
+    config.add_extension('criterion',
+                         sources=['criterion.c'],
+                         include_dirs=[numpy.get_include()],
+                         libraries=libraries)
+
+    return config
+
+
+if __name__ == "__main__":
+    from numpy.distutils.core import setup
+    setup(**configuration().todict())