HistogramOptions now named to HistogramAndQuantilesOptions, impleme…

…nt `num_quantiles` option to `HistogramAndQuantilesOptions` (#956) * Implemented options for `num_quantiles` feature in `NumericStatsMixin` * Implemented options for `num_quantiles.py`. * Added tests for `num_quantiles`. * Fixed tests for num_quantiles option in column profiles. * Added tests for num_quantiles option. * Update dataprofiler/tests/profilers/profiler_options/test_num_quantiles_option.py Co-authored-by: Taylor Turner <[email protected]> * Minor edits to tests * Empty commit * Precommits * Minor change to Makefile. * Got rid of print statements. * "empty commit" * Added deprecation for `HistogramOption`. * Cleaner deprecation: removed `HistogramOption` class. * json_decoder, test_h_a_q changes. * Fixed string bug for testing in test_histogram_and_quantiles * Explicitfied deprecation warning, validated profile conversion from histogramoption to histogramandquantilesoption in decoding phase. * Minor deprecation warning wording change. * Update dataprofiler/tests/profilers/profiler_options/test_histogram_and_quantiles_option.py Co-authored-by: Taylor Turner <[email protected]> * Changed deprecation warning. * Changed quantiles option to initialize to None. * Minor change * Minor change again D: * Minor change again again D: sadness --------- Co-authored-by: Taylor Turner <[email protected]>
capitalone · Aug 1, 2023 · 01113a2 · 01113a2
1 parent f0ecbac
commit 01113a2
Show file tree

Hide file tree

Showing 13 changed files with 354 additions and 205 deletions.
diff --git a/dataprofiler/profilers/__init__.py b/dataprofiler/profilers/__init__.py
@@ -28,7 +28,7 @@
     DataLabelerOptions,
     DateTimeOptions,
     FloatOptions,
-    HistogramOption,
+    HistogramAndQuantilesOption,
     HyperLogLogOptions,
     IntOptions,
     ModeOption,
@@ -66,7 +66,8 @@
 
 json_decoder._options = {
     BooleanOption.__name__: BooleanOption,
-    HistogramOption.__name__: HistogramOption,
+    "HistogramOption": HistogramAndQuantilesOption,
+    HistogramAndQuantilesOption.__name__: HistogramAndQuantilesOption,
     ModeOption.__name__: ModeOption,
     BaseInspectorOptions.__name__: BaseInspectorOptions,
     NumericalOptions.__name__: NumericalOptions,

diff --git a/dataprofiler/profilers/json_decoder.py b/dataprofiler/profilers/json_decoder.py
@@ -1,6 +1,7 @@
 """Contains methods to decode components of a Profiler."""
 from __future__ import annotations
 
+import warnings
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
@@ -72,6 +73,14 @@ def get_option_class(class_name: str) -> type[BaseOption]:
     options_class: type[BaseOption] | None = _options.get(class_name)
     if options_class is None:
         raise ValueError(f"Invalid option class {class_name} " f"failed to load.")
+
+    if class_name == "HistogramOption":
+        warnings.warn(
+            f"{class_name} will be deprecated in the future. During the JSON encode "
+            "process, HistogramOption is mapped to HistogramAndQuantilesOption. "
+            "Please begin utilizing the new HistogramAndQuantilesOption class.",
+            DeprecationWarning,
+        )
     return options_class
 
 

diff --git a/dataprofiler/profilers/numerical_column_stats.py b/dataprofiler/profilers/numerical_column_stats.py
@@ -82,14 +82,15 @@ def __init__(self, options: NumericalOptions = None) -> None:
         self._mode_is_enabled: bool = True
         self.num_zeros: int | np.int64 = np.int64(0)
         self.num_negatives: int | np.int64 = np.int64(0)
-        self._num_quantiles: int = 1000  # TODO: add to options
+        self._num_quantiles: int = 1000  # By default, we use 1000 quantiles
 
         if options:
             self.bias_correction = options.bias_correction.is_enabled
             self._top_k_modes = options.mode.top_k_modes
             self._median_is_enabled = options.median.is_enabled
             self._median_abs_dev_is_enabled = options.median_abs_deviation.is_enabled
             self._mode_is_enabled = options.mode.is_enabled
+            self._num_quantiles = options.histogram_and_quantiles.num_quantiles
             bin_count_or_method = options.histogram_and_quantiles.bin_count_or_method
             if isinstance(bin_count_or_method, str):
                 self.histogram_bin_method_names = [bin_count_or_method]

diff --git a/dataprofiler/profilers/profiler_options.py b/dataprofiler/profilers/profiler_options.py
@@ -210,13 +210,14 @@ def _validate_helper(self, variable_path: str = "BooleanOption") -> list[str]:
         return errors
 
 
-class HistogramOption(BooleanOption["HistogramOption"]):
+class HistogramAndQuantilesOption(BooleanOption["HistogramAndQuantilesOption"]):
     """For setting histogram options."""
 
     def __init__(
         self,
         is_enabled: bool = True,
         bin_count_or_method: str | int | list[str] = "auto",
+        num_quantiles: int = 1000,
     ) -> None:
         """
         Initialize Options for histograms.
@@ -226,11 +227,17 @@ def __init__(
         :ivar bin_count_or_method: bin count or the method with which to
             calculate histograms
         :vartype bin_count_or_method: Union[str, int, list(str)]
+        :ivar num_quantiles: boolean option to enable/disable num_quantiles
+            and set the number of quantiles
+        :vartype num_quantiles: int
         """
         self.bin_count_or_method = bin_count_or_method
+        self.num_quantiles = num_quantiles
         super().__init__(is_enabled=is_enabled)
 
-    def _validate_helper(self, variable_path: str = "HistogramOption") -> list[str]:
+    def _validate_helper(
+        self, variable_path: str = "HistogramAndQuantilesOption"
+    ) -> list[str]:
         """
         Validate the options do not conflict and cause errors.
 
@@ -260,6 +267,12 @@ def _validate_helper(self, variable_path: str = "HistogramOption") -> list[str]:
                     "than 1, a string, or list of strings from the "
                     "following: {}.".format(variable_path, valid_methods)
                 )
+
+        if self.num_quantiles is not None and (
+            not isinstance(self.num_quantiles, int) or self.num_quantiles < 1
+        ):
+            errors.append(f"{variable_path}.num_quantiles must be a positive integer.")
+
         return errors
 
 
@@ -396,7 +409,9 @@ def __init__(self) -> None:
         self.median_abs_deviation: BooleanOption = BooleanOption(is_enabled=True)
         self.num_zeros: BooleanOption = BooleanOption(is_enabled=True)
         self.num_negatives: BooleanOption = BooleanOption(is_enabled=True)
-        self.histogram_and_quantiles: HistogramOption = HistogramOption()
+        self.histogram_and_quantiles: HistogramAndQuantilesOption = (
+            HistogramAndQuantilesOption()
+        )
         # By default, we correct for bias
         self.bias_correction: BooleanOption = BooleanOption(is_enabled=True)
         BaseInspectorOptions.__init__(self)

diff --git a/dataprofiler/tests/profilers/profiler_options/test_float_options.py b/dataprofiler/tests/profilers/profiler_options/test_float_options.py
@@ -94,7 +94,7 @@ def test_json_encode(self):
                     "data": {"is_enabled": True},
                 },
                 "histogram_and_quantiles": {
-                    "class": "HistogramOption",
+                    "class": "HistogramAndQuantilesOption",
                     "data": mock.ANY,
                 },
                 "bias_correction": {