From 0c7285661a7a14ef09399b77e8d1dc925e5b6a23 Mon Sep 17 00:00:00 2001
From: Jamie Broomall <88007022+jamie256@users.noreply.github.com>
Date: Wed, 13 Mar 2024 18:47:55 -0500
Subject: [PATCH] Update Ranking metrics notebook example to pin whylogs
version install (#1483)
## Description
The ranking metrics example notebook has a bug when used with latest
whylogs version, while those bugs are being addressed let's pin the
install to the version this notebook example was based on to mitigate.
## Changes
- specify version 1.3.21 when installing whylogs in the notebook
example.
## Related
Avoids the error but does not address underlying root cause of #1480
- [x] I have reviewed the [Guidelines for Contributing](CONTRIBUTING.md)
and the [Code of Conduct](CODE_OF_CONDUCT.md).
---
...nking_Performance_Metrics_to_WhyLabs.ipynb | 722 +++++++-----------
1 file changed, 260 insertions(+), 462 deletions(-)
diff --git a/python/examples/experimental/Writing_Ranking_Performance_Metrics_to_WhyLabs.ipynb b/python/examples/experimental/Writing_Ranking_Performance_Metrics_to_WhyLabs.ipynb
index efb56021bb..bf5dc5ae4b 100644
--- a/python/examples/experimental/Writing_Ranking_Performance_Metrics_to_WhyLabs.ipynb
+++ b/python/examples/experimental/Writing_Ranking_Performance_Metrics_to_WhyLabs.ipynb
@@ -48,12 +48,12 @@
},
"outputs": [],
"source": [
- "%pip install whylogs"
+ "%pip install whylogs==1.3.21"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 1,
"metadata": {
"id": "REwojoWgwbjn"
},
@@ -80,7 +80,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 2,
"metadata": {
"id": "ZxTd2UZ7wbjo"
},
@@ -117,18 +117,18 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 3,
"metadata": {
- "id": "Hvhrr3Xuwbjp",
- "outputId": "9e8ff52c-9225-4980-d660-5c9365b21fb2",
"colab": {
"base_uri": "https://localhost:8080/"
- }
+ },
+ "id": "Hvhrr3Xuwbjp",
+ "outputId": "9e8ff52c-9225-4980-d660-5c9365b21fb2"
},
"outputs": [
{
- "output_type": "stream",
"name": "stdout",
+ "output_type": "stream",
"text": [
"⚠️ No session found. Call whylogs.init() to initialize a session and authenticate. See https://docs.whylabs.ai/docs/whylabs-whylogs-init for more information.\n"
]
@@ -151,255 +151,20 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 4,
"metadata": {
- "id": "InKyepzFwbjp",
- "outputId": "aef25d33-ae9e-4b4f-f3be-7da902429ed8",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 1000
- }
+ },
+ "id": "InKyepzFwbjp",
+ "outputId": "aef25d33-ae9e-4b4f-f3be-7da902429ed8"
},
"outputs": [
{
- "output_type": "execute_result",
"data": {
- "text/plain": [
- "column accuracy_k_2 average_precision_k_2 \\\n",
- "cardinality/est 1.0 3.0 \n",
- "cardinality/lower_1 1.0 3.0 \n",
- "cardinality/upper_1 1.00005 3.00015 \n",
- "counts/inf 0 0 \n",
- "counts/n 1 4 \n",
- "counts/nan 0 0 \n",
- "counts/null 0 0 \n",
- "distribution/max 0.5 0.75 \n",
- "distribution/mean 0.5 0.25 \n",
- "distribution/median 0.5 0.25 \n",
- "distribution/min 0.5 0.0 \n",
- "distribution/n 1 4 \n",
- "distribution/q_01 0.5 0.0 \n",
- "distribution/q_05 0.5 0.0 \n",
- "distribution/q_10 0.5 0.0 \n",
- "distribution/q_25 0.5 0.0 \n",
- "distribution/q_75 0.5 0.75 \n",
- "distribution/q_90 0.5 0.75 \n",
- "distribution/q_95 0.5 0.75 \n",
- "distribution/q_99 0.5 0.75 \n",
- "distribution/stddev 0.0 0.353553 \n",
- "type SummaryType.COLUMN SummaryType.COLUMN \n",
- "types/boolean 0 0 \n",
- "types/fractional 1 4 \n",
- "types/integral 0 0 \n",
- "types/object 0 0 \n",
- "types/string 0 0 \n",
- "types/tensor 0 0 \n",
- "frequent_items/frequent_strings NaN NaN \n",
- "ints/max NaN NaN \n",
- "ints/min NaN NaN \n",
- "\n",
- "column mean_average_precision_k_2 \\\n",
- "cardinality/est 1.0 \n",
- "cardinality/lower_1 1.0 \n",
- "cardinality/upper_1 1.00005 \n",
- "counts/inf 0 \n",
- "counts/n 1 \n",
- "counts/nan 0 \n",
- "counts/null 0 \n",
- "distribution/max 0.25 \n",
- "distribution/mean 0.25 \n",
- "distribution/median 0.25 \n",
- "distribution/min 0.25 \n",
- "distribution/n 1 \n",
- "distribution/q_01 0.25 \n",
- "distribution/q_05 0.25 \n",
- "distribution/q_10 0.25 \n",
- "distribution/q_25 0.25 \n",
- "distribution/q_75 0.25 \n",
- "distribution/q_90 0.25 \n",
- "distribution/q_95 0.25 \n",
- "distribution/q_99 0.25 \n",
- "distribution/stddev 0.0 \n",
- "type SummaryType.COLUMN \n",
- "types/boolean 0 \n",
- "types/fractional 1 \n",
- "types/integral 0 \n",
- "types/object 0 \n",
- "types/string 0 \n",
- "types/tensor 0 \n",
- "frequent_items/frequent_strings NaN \n",
- "ints/max NaN \n",
- "ints/min NaN \n",
- "\n",
- "column mean_reciprocal_rank precision_k_2 \\\n",
- "cardinality/est 1.0 2.0 \n",
- "cardinality/lower_1 1.0 2.0 \n",
- "cardinality/upper_1 1.00005 2.0001 \n",
- "counts/inf 0 0 \n",
- "counts/n 1 4 \n",
- "counts/nan 0 0 \n",
- "counts/null 0 0 \n",
- "distribution/max 0.611111 0.5 \n",
- "distribution/mean 0.611111 0.25 \n",
- "distribution/median 0.611111 0.5 \n",
- "distribution/min 0.611111 0.0 \n",
- "distribution/n 1 4 \n",
- "distribution/q_01 0.611111 0.0 \n",
- "distribution/q_05 0.611111 0.0 \n",
- "distribution/q_10 0.611111 0.0 \n",
- "distribution/q_25 0.611111 0.0 \n",
- "distribution/q_75 0.611111 0.5 \n",
- "distribution/q_90 0.611111 0.5 \n",
- "distribution/q_95 0.611111 0.5 \n",
- "distribution/q_99 0.611111 0.5 \n",
- "distribution/stddev 0.0 0.288675 \n",
- "type SummaryType.COLUMN SummaryType.COLUMN \n",
- "types/boolean 0 0 \n",
- "types/fractional 1 4 \n",
- "types/integral 0 0 \n",
- "types/object 0 0 \n",
- "types/string 0 0 \n",
- "types/tensor 0 0 \n",
- "frequent_items/frequent_strings NaN NaN \n",
- "ints/max NaN NaN \n",
- "ints/min NaN NaN \n",
- "\n",
- "column raw_predictions \\\n",
- "cardinality/est 0.0 \n",
- "cardinality/lower_1 0.0 \n",
- "cardinality/upper_1 0.0 \n",
- "counts/inf 0 \n",
- "counts/n 4 \n",
- "counts/nan 0 \n",
- "counts/null 0 \n",
- "distribution/max NaN \n",
- "distribution/mean 0.0 \n",
- "distribution/median NaN \n",
- "distribution/min NaN \n",
- "distribution/n 0 \n",
- "distribution/q_01 NaN \n",
- "distribution/q_05 NaN \n",
- "distribution/q_10 NaN \n",
- "distribution/q_25 NaN \n",
- "distribution/q_75 NaN \n",
- "distribution/q_90 NaN \n",
- "distribution/q_95 NaN \n",
- "distribution/q_99 NaN \n",
- "distribution/stddev 0.0 \n",
- "type SummaryType.COLUMN \n",
- "types/boolean 0 \n",
- "types/fractional 0 \n",
- "types/integral 0 \n",
- "types/object 4 \n",
- "types/string 0 \n",
- "types/tensor 0 \n",
- "frequent_items/frequent_strings [] \n",
- "ints/max NaN \n",
- "ints/min NaN \n",
- "\n",
- "column raw_targets \\\n",
- "cardinality/est 4.0 \n",
- "cardinality/lower_1 4.0 \n",
- "cardinality/upper_1 4.0002 \n",
- "counts/inf 0 \n",
- "counts/n 4 \n",
- "counts/nan 0 \n",
- "counts/null 0 \n",
- "distribution/max NaN \n",
- "distribution/mean 0.0 \n",
- "distribution/median NaN \n",
- "distribution/min NaN \n",
- "distribution/n 0 \n",
- "distribution/q_01 NaN \n",
- "distribution/q_05 NaN \n",
- "distribution/q_10 NaN \n",
- "distribution/q_25 NaN \n",
- "distribution/q_75 NaN \n",
- "distribution/q_90 NaN \n",
- "distribution/q_95 NaN \n",
- "distribution/q_99 NaN \n",
- "distribution/stddev 0.0 \n",
- "type SummaryType.COLUMN \n",
- "types/boolean 0 \n",
- "types/fractional 0 \n",
- "types/integral 0 \n",
- "types/object 0 \n",
- "types/string 4 \n",
- "types/tensor 0 \n",
- "frequent_items/frequent_strings [FrequentItem(value='cat', est=1, upper=1, low... \n",
- "ints/max NaN \n",
- "ints/min NaN \n",
- "\n",
- "column recall_k_2 \\\n",
- "cardinality/est 2.0 \n",
- "cardinality/lower_1 2.0 \n",
- "cardinality/upper_1 2.0001 \n",
- "counts/inf 0 \n",
- "counts/n 4 \n",
- "counts/nan 1 \n",
- "counts/null 1 \n",
- "distribution/max 1.0 \n",
- "distribution/mean 0.666667 \n",
- "distribution/median 1.0 \n",
- "distribution/min 0.0 \n",
- "distribution/n 3 \n",
- "distribution/q_01 0.0 \n",
- "distribution/q_05 0.0 \n",
- "distribution/q_10 0.0 \n",
- "distribution/q_25 0.0 \n",
- "distribution/q_75 1.0 \n",
- "distribution/q_90 1.0 \n",
- "distribution/q_95 1.0 \n",
- "distribution/q_99 1.0 \n",
- "distribution/stddev 0.57735 \n",
- "type SummaryType.COLUMN \n",
- "types/boolean 0 \n",
- "types/fractional 3 \n",
- "types/integral 0 \n",
- "types/object 0 \n",
- "types/string 0 \n",
- "types/tensor 0 \n",
- "frequent_items/frequent_strings NaN \n",
- "ints/max NaN \n",
- "ints/min NaN \n",
- "\n",
- "column top_rank \n",
- "cardinality/est 4.0 \n",
- "cardinality/lower_1 4.0 \n",
- "cardinality/upper_1 4.0002 \n",
- "counts/inf 0 \n",
- "counts/n 4 \n",
- "counts/nan 0 \n",
- "counts/null 0 \n",
- "distribution/max 3.0 \n",
- "distribution/mean 1.5 \n",
- "distribution/median 2.0 \n",
- "distribution/min 0.0 \n",
- "distribution/n 4 \n",
- "distribution/q_01 0.0 \n",
- "distribution/q_05 0.0 \n",
- "distribution/q_10 0.0 \n",
- "distribution/q_25 1.0 \n",
- "distribution/q_75 3.0 \n",
- "distribution/q_90 3.0 \n",
- "distribution/q_95 3.0 \n",
- "distribution/q_99 3.0 \n",
- "distribution/stddev 1.290994 \n",
- "type SummaryType.COLUMN \n",
- "types/boolean 0 \n",
- "types/fractional 0 \n",
- "types/integral 4 \n",
- "types/object 0 \n",
- "types/string 0 \n",
- "types/tensor 0 \n",
- "frequent_items/frequent_strings [FrequentItem(value='1', est=1, upper=1, lower... \n",
- "ints/max 3.0 \n",
- "ints/min 0.0 "
- ],
"text/html": [
- "\n",
- "
\n",
- "
\n",
+ "
\n",
"\n",
- "\n",
- " \n",
- "
\n",
- "\n",
- "\n",
- "
\n",
- "
\n",
- "\n",
- "\n",
+ "column recall_k_2 \\\n",
+ "cardinality/est 2.0 \n",
+ "cardinality/lower_1 2.0 \n",
+ "cardinality/upper_1 2.0001 \n",
+ "counts/inf 0 \n",
+ "counts/n 4 \n",
+ "counts/nan 1 \n",
+ "counts/null 1 \n",
+ "distribution/max 1.0 \n",
+ "distribution/mean 0.666667 \n",
+ "distribution/median 1.0 \n",
+ "distribution/min 0.0 \n",
+ "distribution/n 3 \n",
+ "distribution/q_01 0.0 \n",
+ "distribution/q_05 0.0 \n",
+ "distribution/q_10 0.0 \n",
+ "distribution/q_25 0.0 \n",
+ "distribution/q_75 1.0 \n",
+ "distribution/q_90 1.0 \n",
+ "distribution/q_95 1.0 \n",
+ "distribution/q_99 1.0 \n",
+ "distribution/stddev 0.57735 \n",
+ "type SummaryType.COLUMN \n",
+ "types/boolean 0 \n",
+ "types/fractional 3 \n",
+ "types/integral 0 \n",
+ "types/object 0 \n",
+ "types/string 0 \n",
+ "types/tensor 0 \n",
+ "frequent_items/frequent_strings NaN \n",
+ "ints/max NaN \n",
+ "ints/min NaN \n",
"\n",
- " \n",
- "
\n",
- "
\n",
- "
\n"
+ "column top_rank \n",
+ "cardinality/est 4.0 \n",
+ "cardinality/lower_1 4.0 \n",
+ "cardinality/upper_1 4.0002 \n",
+ "counts/inf 0 \n",
+ "counts/n 4 \n",
+ "counts/nan 0 \n",
+ "counts/null 0 \n",
+ "distribution/max 3.0 \n",
+ "distribution/mean 1.5 \n",
+ "distribution/median 2.0 \n",
+ "distribution/min 0.0 \n",
+ "distribution/n 4 \n",
+ "distribution/q_01 0.0 \n",
+ "distribution/q_05 0.0 \n",
+ "distribution/q_10 0.0 \n",
+ "distribution/q_25 1.0 \n",
+ "distribution/q_75 3.0 \n",
+ "distribution/q_90 3.0 \n",
+ "distribution/q_95 3.0 \n",
+ "distribution/q_99 3.0 \n",
+ "distribution/stddev 1.290994 \n",
+ "type SummaryType.COLUMN \n",
+ "types/boolean 0 \n",
+ "types/fractional 0 \n",
+ "types/integral 4 \n",
+ "types/object 0 \n",
+ "types/string 0 \n",
+ "types/tensor 0 \n",
+ "frequent_items/frequent_strings [FrequentItem(value='1', est=1, upper=1, lower... \n",
+ "ints/max 3.0 \n",
+ "ints/min 0.0 "
]
},
+ "execution_count": 4,
"metadata": {},
- "execution_count": 5
+ "output_type": "execute_result"
}
],
"source": [
@@ -1037,11 +827,19 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 6,
"metadata": {
"id": "GtIW3274wbjq"
},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Using API Key ID: \n"
+ ]
+ }
+ ],
"source": [
"# Configure WhyLabs info, if needed\n",
"import getpass\n",
@@ -1067,15 +865,18 @@
},
{
"cell_type": "markdown",
- "source": [
- "The ranking metrics should be available in the WhyLabs platform, under the Performance tab."
- ],
"metadata": {
"id": "lSAG0mcwLL5d"
- }
+ },
+ "source": [
+ "The ranking metrics should be available in the WhyLabs platform, under the Performance tab."
+ ]
}
],
"metadata": {
+ "colab": {
+ "provenance": []
+ },
"kernelspec": {
"display_name": "wl-public",
"language": "python",
@@ -1093,10 +894,7 @@
"pygments_lexer": "ipython3",
"version": "3.8.10"
},
- "orig_nbformat": 4,
- "colab": {
- "provenance": []
- }
+ "orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 0