From 7b3d18b52d23beab0ecf670aa991faed30123ced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pelle=20Sillre=CC=81n?= Date: Tue, 18 Apr 2023 16:37:12 +0200 Subject: [PATCH 1/5] Added reference level point estimate to hover box of difference plots --- .../analysis/frequentist/chartify_grapher.py | 9 +++++++++ spotify_confidence/analysis/frequentist/experiment.py | 4 +++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/spotify_confidence/analysis/frequentist/chartify_grapher.py b/spotify_confidence/analysis/frequentist/chartify_grapher.py index 035422a..81c56cc 100644 --- a/spotify_confidence/analysis/frequentist/chartify_grapher.py +++ b/spotify_confidence/analysis/frequentist/chartify_grapher.py @@ -32,6 +32,7 @@ ) from ..constants import ( POINT_ESTIMATE, + ORIGINAL_POINT_ESTIMATE, DIFFERENCE, CI_LOWER, CI_UPPER, @@ -43,6 +44,7 @@ NIM, NIM_TYPE, PREFERENCE, + SFX1, ) from ...chartgrid import ChartGrid @@ -556,6 +558,7 @@ def add_ci_to_chart_datasources( data[DIFFERENCE] = np.array(df[DIFFERENCE][index]) data["p_value"] = np.array(df[P_VALUE][index]) data["adjusted_p"] = np.array(df[ADJUSTED_P][index]) + data["reference_level_avg"] = np.array(df[ORIGINAL_POINT_ESTIMATE + SFX1][index]) if NULL_HYPOTHESIS in df.columns: data["null_hyp"] = np.array(df[NULL_HYPOTHESIS][index]) @@ -590,8 +593,14 @@ def add_tools( else [] ) nim_tool_tip = [("null hypothesis", f"@null_hyp{{{axis_format}}}")] if NULL_HYPOTHESIS in df.columns else [] + reference_level_tool_tip = ( + [(f"{df['level_1'].values[0]} avg", f"@reference_level_avg{{{axis_format}}}")] + if "level_1" in df.columns + else [] + ) tooltips = ( [("group", "@color")] + + reference_level_tool_tip + ordinal_tool_tip + [(f"{center_name}", f"@{center_name}{{{axis_format}}}")] + [ diff --git a/spotify_confidence/analysis/frequentist/experiment.py b/spotify_confidence/analysis/frequentist/experiment.py index 8b55906..6fae40f 100644 --- a/spotify_confidence/analysis/frequentist/experiment.py +++ b/spotify_confidence/analysis/frequentist/experiment.py @@ -200,6 +200,7 @@ def difference_plot( groupby=groupby, non_inferiority_margins=non_inferiority_margins, final_expected_sample_size_column=final_expected_sample_size_column, + verbose=True, ) chartgrid = self._confidence_grapher.plot_difference( difference_df, absolute, groupby, non_inferiority_margins, use_adjusted_intervals, split_plot_by_groups @@ -217,7 +218,7 @@ def differences_plot( split_plot_by_groups: bool = False, ) -> ChartGrid: difference_df = self.differences( - levels, absolute, groupby, non_inferiority_margins, final_expected_sample_size_column + levels, absolute, groupby, non_inferiority_margins, final_expected_sample_size_column, verbose=True ) chartgrid = self._confidence_grapher.plot_differences( difference_df, absolute, groupby, non_inferiority_margins, use_adjusted_intervals, split_plot_by_groups @@ -242,6 +243,7 @@ def multiple_difference_plot( level_as_reference=level_as_reference, non_inferiority_margins=non_inferiority_margins, final_expected_sample_size_column=final_expected_sample_size_column, + verbose=True, ) chartgrid = self._confidence_grapher.plot_multiple_difference( difference_df, From 28c05cf3cda7807d9a8556db860947272fe7bf70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pelle=20Sillre=CC=81n?= Date: Tue, 18 Apr 2023 16:40:11 +0200 Subject: [PATCH 2/5] Updated version and history --- HISTORY.rst | 4 ++++ README.md | 4 ++-- setup.cfg | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 116375a..2c7848b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,10 @@ History ======= +3.0.1 (2023-04-20) +------------------ +* Added reference level point estimate to the hover box of difference plots + 3.0.0 (2023-03-24) ------------------ * Dropped support for python 3.6 diff --git a/README.md b/README.md index c8f64df..3677894 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,11 @@ Spotify Confidence ======== ![Status](https://img.shields.io/badge/Status-Beta-blue.svg) -![Latest release](https://img.shields.io/badge/release-3.0.0-green.svg "Latest release: 3.0.0") -![Python](https://img.shields.io/badge/Python-3.6-blue.svg "Python") +![Latest release](https://img.shields.io/badge/release-3.0.1-green.svg "Latest release: 3.0.1") ![Python](https://img.shields.io/badge/Python-3.7-blue.svg "Python") ![Python](https://img.shields.io/badge/Python-3.8-blue.svg "Python") ![Python](https://img.shields.io/badge/Python-3.9-blue.svg "Python") +![Python](https://img.shields.io/badge/Python-3.10-blue.svg "Python") Python library for AB test analysis. diff --git a/setup.cfg b/setup.cfg index 635c8a0..75156f0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = spotify-confidence -version = 3.0.0 +version = 3.0.1 author = Per Sillren author_email = pers@spotify.com description = Package for calculating and visualising confidence intervals, e.g. for A/B test analysis. From 7829b4c40798e32b5768335ec8d3e36bf0e9c939 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pelle=20Sillre=CC=81n?= Date: Fri, 21 Apr 2023 11:34:07 +0200 Subject: [PATCH 3/5] Made sure tooltips are also shown for ordinal plots --- .../analysis/frequentist/chartify_grapher.py | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/spotify_confidence/analysis/frequentist/chartify_grapher.py b/spotify_confidence/analysis/frequentist/chartify_grapher.py index 81c56cc..13d26d3 100644 --- a/spotify_confidence/analysis/frequentist/chartify_grapher.py +++ b/spotify_confidence/analysis/frequentist/chartify_grapher.py @@ -186,7 +186,7 @@ def _ordinal_difference_plot( y_axis_label = self._get_difference_plot_label(absolute) ch = self._ordinal_plot( - "difference", + DIFFERENCE, difference_df, groupby=None, level_name="", @@ -383,6 +383,8 @@ def _ordinal_plot( y_column=center_name, color_column=colors, ) + # Also plot transparent circles, just to be able to show hover box + ch.figure.line(source=df, x=self._ordinal_group_column, y=center_name, name="center", line_alpha=0) ch.style.color_palette.reset_palette_order() ch.plot.area( data_frame=( @@ -405,6 +407,15 @@ def _ordinal_plot( line_dash="dashed", line_width=1, ) + # Also plot named transparent line, just to be able to show hover box + ch.figure.line( + source=df.sort_values(self._ordinal_group_column), + x=self._ordinal_group_column, + y=NULL_HYPOTHESIS, + line_width=3, + line_alpha=0, + name="nim", + ) ch.axes.set_yaxis_label(y_axis_label) ch.axes.set_xaxis_label(self._ordinal_group_column) ch.set_source_label("") @@ -555,6 +566,7 @@ def add_ci_to_chart_datasources( data["color"] = np.array(df[group_col][index]) if DIFFERENCE in data.keys() or NULL_HYPOTHESIS in data.keys(): index = data["index"] + data["reference_level"] = np.array(df["level_1"][index]) data[DIFFERENCE] = np.array(df[DIFFERENCE][index]) data["p_value"] = np.array(df[P_VALUE][index]) data["adjusted_p"] = np.array(df[ADJUSTED_P][index]) @@ -583,6 +595,13 @@ def add_tools( absolute=absolute, extra_zeros=2, ) + axis_format_reference_level, _, _ = axis_format_precision( + numbers=concat( + [df[LOWER], df[center_name], df[UPPER], df[NULL_HYPOTHESIS] if NULL_HYPOTHESIS in df.columns else None] + ), + absolute=True, + extra_zeros=2, + ) ordinal_tool_tip = [] if not ordinal else [(self._ordinal_group_column, f"@{self._ordinal_group_column}")] p_value_tool_tip = ( ( @@ -594,7 +613,7 @@ def add_tools( ) nim_tool_tip = [("null hypothesis", f"@null_hyp{{{axis_format}}}")] if NULL_HYPOTHESIS in df.columns else [] reference_level_tool_tip = ( - [(f"{df['level_1'].values[0]} avg", f"@reference_level_avg{{{axis_format}}}")] + [(f"reference level", f"@reference_level: @reference_level_avg{{{axis_format_reference_level}}}")] if "level_1" in df.columns else [] ) @@ -612,7 +631,7 @@ def add_tools( + p_value_tool_tip + nim_tool_tip ) - lines_with_hover = [] if ordinal else ["center", "nim"] + lines_with_hover = ["center", "nim"] renderers = [r for r in chart.figure.renderers if r.name in lines_with_hover] hover = tools.HoverTool(tooltips=tooltips, renderers=renderers) From 1ed64706c82c85193f8a86f9426f356203043d84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pelle=20Sillre=CC=81n?= Date: Fri, 21 Apr 2023 11:43:57 +0200 Subject: [PATCH 4/5] Updated Chartify requirement to automatically get jupyter_bokeh installed to enable html output in notebooks --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 75156f0..23d452e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,7 +24,7 @@ install_requires = scipy>=1.6.0,<1.8.0 pandas>=1.2.0,<2.0.0 statsmodels>=0.13.0,<1.0.0 - chartify>=4.0.2 + chartify>=4.0.3 ipywidgets>=8.0.0 [options.packages.find] From c8210561ace2c7a6d53832ad76000847c504121e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pelle=20Sillre=CC=81n?= Date: Fri, 21 Apr 2023 11:58:49 +0200 Subject: [PATCH 5/5] Convert f-string to normal string to make flake8 happy --- spotify_confidence/analysis/frequentist/chartify_grapher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spotify_confidence/analysis/frequentist/chartify_grapher.py b/spotify_confidence/analysis/frequentist/chartify_grapher.py index 13d26d3..3a297c1 100644 --- a/spotify_confidence/analysis/frequentist/chartify_grapher.py +++ b/spotify_confidence/analysis/frequentist/chartify_grapher.py @@ -613,7 +613,7 @@ def add_tools( ) nim_tool_tip = [("null hypothesis", f"@null_hyp{{{axis_format}}}")] if NULL_HYPOTHESIS in df.columns else [] reference_level_tool_tip = ( - [(f"reference level", f"@reference_level: @reference_level_avg{{{axis_format_reference_level}}}")] + [("reference level", f"@reference_level: @reference_level_avg{{{axis_format_reference_level}}}")] if "level_1" in df.columns else [] )