Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ranking Metrics - Better Precision/Recall/MRR calculation #1492

Merged
merged 11 commits into from
Mar 26, 2024
Merged

Ranking Metrics - Better Precision/Recall/MRR calculation #1492

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
53913eb
ranking example rewrite
Mar 18, 2024
ddc7faf
Merge branch 'mainline' into dev/felipe/ranking-example
Mar 19, 2024
53c8059
rewrite
Mar 19, 2024
35abcf4
wip
Mar 19, 2024
8721056
wip
Mar 19, 2024
754327b
fix other metrics
Mar 20, 2024
d380cf8
Merge branch 'mainline' into dev/felipe/precision-recall-ranking
Mar 20, 2024
4889a66
update notebook results
Mar 20, 2024
f32a356
add tests for precision and recall at k
Mar 20, 2024
e7d8fe0
fix several metrics
Mar 20, 2024
e0a4201
add sum_gain_k and tests and type annotations
Mar 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions python/examples/experimental/Writing_Ranking_Performance_Metrics_to_WhyLabs.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,19 +206,14 @@
" <td>1.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>mean_reciprocal_rank</th>\n",
" <td>1</td>\n",
" <td>0.333333</td>\n",
" </tr>\n",
" <tr>\n",
" <th>norm_dis_cumul_gain_k_3</th>\n",
" <td>1</td>\n",
" <td>1.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>precision_k_3</th>\n",
" <td>1</td>\n",
" <td>0.333333</td>\n",
" <td>0.666667</td>\n",
" </tr>\n",
" <tr>\n",
" <th>predictions</th>\n",
Expand All @@ -231,14 +226,19 @@
" <td>1.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>reciprocal_rank</th>\n",
" <td>1</td>\n",
" <td>1.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>targets</th>\n",
" <td>1</td>\n",
" <td>0.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>top_rank</th>\n",
" <td>1</td>\n",
" <td>3.000000</td>\n",
" <td>1.000000</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
Expand All @@ -249,13 +249,13 @@
"column \n",
"accuracy_k_3 1 1.000000\n",
"average_precision_k_3 1 1.000000\n",
"mean_reciprocal_rank 1 0.333333\n",
"norm_dis_cumul_gain_k_3 1 1.000000\n",
"precision_k_3 1 0.333333\n",
"precision_k_3 1 0.666667\n",
"predictions 1 0.000000\n",
"recall_k_3 1 1.000000\n",
"reciprocal_rank 1 1.000000\n",
"targets 1 0.000000\n",
"top_rank 1 3.000000"
"top_rank 1 1.000000"
]
},
"execution_count": 4,
Expand Down
40 changes: 27 additions & 13 deletions python/tests/experimental/api/test_logger.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ def test_log_batch_ranking_metrics_single_simple():

column_names = [
"accuracy_k_3",
"mean_reciprocal_rank",
"reciprocal_rank",
"precision_k_3",
"recall_k_3",
"top_rank",
Expand All @@ -33,17 +33,21 @@ def test_log_batch_ranking_metrics_single_simple():
for col in column_names:
assert col in pandas_summary.index
assert pandas_summary.loc["accuracy_k_3", "counts/n"] == 1
assert pandas_summary.loc["mean_reciprocal_rank", "counts/n"] == 1
assert pandas_summary.loc["reciprocal_rank", "counts/n"] == 4
assert pandas_summary.loc["precision_k_3", "counts/n"] == 4
assert pandas_summary.loc["recall_k_3", "counts/n"] == 4
assert pandas_summary.loc["top_rank", "counts/n"] == 4
assert pandas_summary.loc["average_precision_k_3", "counts/n"] == 4
assert pandas_summary.loc["norm_dis_cumul_gain_k_3", "counts/n"] == 1
assert pandas_summary.loc["average_precision_k_3", "counts/n"] == 4
assert pandas_summary.loc["norm_dis_cumul_gain_k_3", "counts/n"] == 1
assert pandas_summary.loc["norm_dis_cumul_gain_k_3", "counts/n"] == 4
# ndcg = [1, 0, 0.63, 0.5]
assert isclose(pandas_summary.loc["norm_dis_cumul_gain_k_3", "distribution/mean"], 0.53273, abs_tol=0.00001)
assert isclose(pandas_summary.loc["average_precision_k_3", "distribution/mean"], 0.45833, abs_tol=0.00001)
assert isclose(pandas_summary.loc["precision_k_3", "distribution/mean"], 0.25, abs_tol=0.00001)
assert isclose(pandas_summary.loc["recall_k_3", "distribution/mean"], 1.0, abs_tol=0.00001)
# rr = [1, 0, 0.5, 0.33333]
assert isclose(pandas_summary.loc["reciprocal_rank", "distribution/mean"], 0.45833, abs_tol=0.00001)
assert isclose(pandas_summary.loc["accuracy_k_3", "distribution/mean"], 0.75, abs_tol=0.00001)


def test_log_batch_ranking_metrics_binary_simple():
Expand All @@ -57,7 +61,7 @@ def test_log_batch_ranking_metrics_binary_simple():
k = 2
column_names = [
"accuracy_k_" + str(k),
"mean_reciprocal_rank",
"reciprocal_rank",
"precision_k_" + str(k),
"recall_k_" + str(k),
"top_rank",
Expand All @@ -67,16 +71,21 @@ def test_log_batch_ranking_metrics_binary_simple():
for col in column_names:
assert col in pandas_summary.index
assert pandas_summary.loc["accuracy_k_" + str(k), "counts/n"] == 1
assert pandas_summary.loc["mean_reciprocal_rank", "counts/n"] == 1
assert pandas_summary.loc["reciprocal_rank", "counts/n"] == 4
assert pandas_summary.loc["precision_k_" + str(k), "counts/n"] == 4
assert pandas_summary.loc["recall_k_" + str(k), "counts/n"] == 4
assert pandas_summary.loc["top_rank", "counts/n"] == 4
assert pandas_summary.loc["average_precision_k_" + str(k), "counts/n"] == 4
assert pandas_summary.loc["norm_dis_cumul_gain_k_" + str(k), "counts/n"] == 1
assert pandas_summary.loc["norm_dis_cumul_gain_k_" + str(k), "counts/n"] == 4
# ndcg@2 = [0.613147, 1.0, 1.0, 0.63093]
# average_precision_k_2 = [1.0, 0.0, 1.0, 0.5]
assert isclose(pandas_summary.loc["norm_dis_cumul_gain_k_" + str(k), "distribution/mean"], 0.81101, abs_tol=0.00001)
assert isclose(pandas_summary.loc["average_precision_k_" + str(k), "distribution/mean"], 0.62500, abs_tol=0.00001)
assert isclose(pandas_summary.loc["precision_k_" + str(k), "distribution/mean"], 0.5, abs_tol=0.00001)
assert isclose(pandas_summary.loc["recall_k_" + str(k), "distribution/mean"], 0.83333, abs_tol=0.00001)
# rr = [1, 0, 1, 0.5]
assert isclose(pandas_summary.loc["reciprocal_rank", "distribution/mean"], 0.625, abs_tol=0.00001)
assert isclose(pandas_summary.loc["accuracy_k_2", "distribution/mean"], 0.75, abs_tol=0.00001)


def test_log_batch_ranking_metrics_multiple_simple():
Expand Down Expand Up @@ -104,7 +113,7 @@ def test_log_batch_ranking_metrics_multiple_simple():

column_names = [
"accuracy_k_" + str(k),
"mean_reciprocal_rank",
"reciprocal_rank",
"precision_k_" + str(k),
"recall_k_" + str(k),
"top_rank",
Expand All @@ -114,15 +123,15 @@ def test_log_batch_ranking_metrics_multiple_simple():
for col in column_names:
assert col in pandas_summary.index
assert pandas_summary.loc["accuracy_k_" + str(k), "counts/n"] == 1
assert pandas_summary.loc["mean_reciprocal_rank", "counts/n"] == 1
assert pandas_summary.loc["reciprocal_rank", "counts/n"] == 4
assert pandas_summary.loc["precision_k_" + str(k), "counts/n"] == 4
assert pandas_summary.loc["recall_k_" + str(k), "counts/n"] == 4
assert pandas_summary.loc["top_rank", "counts/n"] == 4
assert pandas_summary.loc["average_precision_k_" + str(k), "counts/n"] == 4
assert pandas_summary.loc["norm_dis_cumul_gain_k_" + str(k), "counts/n"] == 1
assert pandas_summary.loc["norm_dis_cumul_gain_k_" + str(k), "counts/n"] == 4
# ndcg@3 = [0.9197, 0.0, 1.0, 0.386853]
# average_precision_k_3 = [0.83, 0.0, 1.0, 0.5]
assert isclose(pandas_summary.loc[f"norm_dis_cumul_gain_k_{k}", "distribution/median"], 0.57664, abs_tol=0.00001)
assert isclose(pandas_summary.loc[f"norm_dis_cumul_gain_k_{k}", "distribution/mean"], 0.57664, abs_tol=0.00001)
assert isclose(pandas_summary.loc["average_precision_k_" + str(k), "distribution/mean"], 0.58333, abs_tol=0.00001)


Expand All @@ -135,7 +144,7 @@ def test_log_batch_ranking_metrics_default_target():
k = 3
column_names = [
"accuracy_k_" + str(k),
"mean_reciprocal_rank",
"reciprocal_rank",
"precision_k_" + str(k),
"recall_k_" + str(k),
"top_rank",
Expand All @@ -145,7 +154,7 @@ def test_log_batch_ranking_metrics_default_target():
for col in column_names:
assert col in pandas_summary.index
assert pandas_summary.loc["accuracy_k_" + str(k), "counts/n"] == 1
assert pandas_summary.loc["mean_reciprocal_rank", "counts/n"] == 1
assert pandas_summary.loc["reciprocal_rank", "counts/n"] == 1
assert pandas_summary.loc["precision_k_" + str(k), "counts/n"] == 1
assert pandas_summary.loc["recall_k_" + str(k), "counts/n"] == 1
assert pandas_summary.loc["top_rank", "counts/n"] == 1
Expand All @@ -155,6 +164,7 @@ def test_log_batch_ranking_metrics_default_target():
assert isclose(pandas_summary.loc[f"norm_dis_cumul_gain_k_{k}", "distribution/median"], 0.90130, abs_tol=0.00001)
# AP assumes binary relevance - this case doesn't raise an error, just a warning, but the result is not meaningful
assert isclose(pandas_summary.loc["average_precision_k_" + str(k), "distribution/mean"], 1.00000, abs_tol=0.00001)
assert isclose(pandas_summary.loc["accuracy_k_3", "distribution/mean"], 1.0, abs_tol=0.00001)


def test_log_batch_ranking_metrics_ranking_ndcg_wikipedia():
Expand Down Expand Up @@ -195,6 +205,9 @@ def test_log_batch_ranking_metrics_average_precision_sklearn_example():
pandas_summary = result.view().to_pandas()

assert isclose(pandas_summary.loc["average_precision_k_" + str(k), "distribution/mean"], 0.83333, abs_tol=0.00001)
assert isclose(pandas_summary.loc["precision_k_" + str(k), "distribution/mean"], 0.5, abs_tol=0.00001)
assert isclose(pandas_summary.loc["recall_k_" + str(k), "distribution/mean"], 1.0, abs_tol=0.00001)
assert isclose(pandas_summary.loc["reciprocal_rank", "distribution/mean"], 1.0, abs_tol=0.00001)


def test_log_batch_ranking_metrics_average_precision():
Expand All @@ -215,3 +228,4 @@ def test_log_batch_ranking_metrics_average_precision():
assert isclose(
pandas_summary.loc["average_precision_k_" + str(k), "distribution/mean"], res[1], abs_tol=0.00001
)
assert isclose(pandas_summary.loc["reciprocal_rank", "distribution/mean"], 0.45833, abs_tol=0.00001)
154 changes: 84 additions & 70 deletions python/whylogs/experimental/api/logger/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,74 @@ def _convert_to_int_if_bool(data: pd.core.frame.DataFrame, *columns: str) -> pd.
return data


class RowWiseMetrics:
def __init__(
self,
target_column: str,
prediction_column: str,
convert_non_numeric: bool = False,
k: Optional[int] = None,
max_k: Optional[int] = None,
):
self.target_column = target_column
self.prediction_column = prediction_column
self.convert_non_numeric = convert_non_numeric

def relevant_counter(self, row, k):
if self.convert_non_numeric:
return sum(
[1 if pred_val in row[self.target_column] else 0 for pred_val in row[self.prediction_column][:k]]
)
else:
paired_sorted = sorted(zip(row[self.prediction_column], row[self.target_column]))
sorted_predictions, sorted_targets = zip(*paired_sorted)
sorted_predictions, sorted_targets = list(sorted_predictions), list(sorted_targets)
return sum([1 if target_val else 0 for target_val in sorted_targets[:k]])

def is_k_item_relevant(self, row, k):
if self.convert_non_numeric:
return 1 if row[self.prediction_column][k - 1] in row[self.target_column] else 0
else:
index_ki = row[self.prediction_column].index(k)
return 1 if row[self.target_column][index_ki] else 0

def get_top_rank(self, row, k):
for ki in range(1, k + 1):
if self.is_k_item_relevant(row, ki):
return ki
return None

def calc_non_numeric_relevance(self, row_dict):
prediction_relevance = []
ideal_relevance = []
for target_val in row_dict[self.prediction_column]:
ideal_relevance.append(1 if target_val in row_dict[self.target_column] else 0)
prediction_relevance.append(1 if target_val in row_dict[self.target_column] else 0)
for target_val in row_dict[self.target_column]:
if target_val not in row_dict[self.prediction_column]:
ideal_relevance.append(1)
return (prediction_relevance, sorted(ideal_relevance, reverse=True))

def calculate_row_ndcg(self, row_dict, k):
if not self.convert_non_numeric:
dcg_vals = [
rel / math.log2(pos + 1)
for rel, pos in zip(row_dict[self.target_column], row_dict[self.prediction_column])
if pos <= k
]
idcg_vals = [
rel / math.log2(pos + 2)
for pos, rel in enumerate(sorted(row_dict[self.target_column], reverse=True)[:k])
]
else:
predicted_relevances, ideal_relevances = self.calc_non_numeric_relevance(row_dict)
dcg_vals = [(rel / math.log(i + 2, 2)) for i, rel in enumerate(predicted_relevances[:k])]
idcg_vals = [(rel / math.log(i + 2, 2)) for i, rel in enumerate(ideal_relevances[:k])]
if sum(idcg_vals) == 0:
return 1 # if there is no relevant data, not much the recommender can do
return sum(dcg_vals) / sum(idcg_vals)


def _calculate_average_precisions(
formatted_data: pd.core.frame.DataFrame,
target_column: str,
Expand All @@ -26,32 +94,18 @@ def _calculate_average_precisions(
) -> np.ndarray:
ki_dict: pd.DataFrame = None
last_item_relevant_dict: pd.DataFrame = None

def relevant_counter(row):
if convert_non_numeric:
return sum([1 if pred_val in row[target_column] else 0 for pred_val in row[prediction_column][:ki]])
else:
paired_sorted = sorted(zip(row[prediction_column], row[target_column]))
sorted_predictions, sorted_targets = zip(*paired_sorted)
sorted_predictions, sorted_targets = list(sorted_predictions), list(sorted_targets)
return sum([1 if target_val else 0 for target_val in sorted_targets[:ki]])

def is_last_item_relevant(row):
if convert_non_numeric:
return 1 if row[prediction_column][ki - 1] in row[target_column] else 0
else:
index_ki = row[prediction_column].index(ki)
return 1 if row[target_column][index_ki] else 0
row_metrics_functions = RowWiseMetrics(target_column, prediction_column, convert_non_numeric)

for ki in range(1, k + 1):
ki_result = (
formatted_data.apply(
relevant_counter,
row_metrics_functions.relevant_counter,
args=(ki,),
axis=1,
)
/ ki
)
last_item_result = formatted_data.apply(is_last_item_relevant, axis=1)
last_item_result = formatted_data.apply(row_metrics_functions.is_k_item_relevant, args=(ki,), axis=1)
if ki == 1:
ki_dict = ki_result.to_frame()
ki_dict.columns = ["p@" + str(ki)]
Expand Down Expand Up @@ -121,72 +175,32 @@ def log_batch_ranking_metrics(
if k and k < 1:
raise ValueError("k must be a positive integer")

formatted_data["count_at_k"] = formatted_data[relevant_cols].apply(
lambda row: sum([1 if pred_val in row[target_column] else 0 for pred_val in row[prediction_column][:k]]), axis=1
)
formatted_data["count_all"] = formatted_data[relevant_cols].apply(
lambda row: sum([1 if pred_val in row[target_column] else 0 for pred_val in row[prediction_column]]), axis=1
row_wise_functions = RowWiseMetrics(target_column, prediction_column, convert_non_numeric)
formatted_data["count_at_k"] = formatted_data.apply(row_wise_functions.relevant_counter, args=(k,), axis=1)
formatted_data["count_all"] = formatted_data.apply(row_wise_functions.relevant_counter, args=(_max_k,), axis=1)
formatted_data["top_rank"] = formatted_data[relevant_cols].apply(
row_wise_functions.get_top_rank, args=(_max_k,), axis=1
)

def get_top_rank(row):
matches = [i + 1 for i, pred_val in enumerate(row[prediction_column]) if pred_val in row[target_column]]
if not matches:
return 0
else:
return matches[0]

formatted_data["top_rank"] = formatted_data[relevant_cols].apply(get_top_rank, axis=1)
output_data = (formatted_data["count_at_k"] / (k if k else 1)).to_frame()
output_data.columns = ["precision" + ("_k_" + str(k) if k else "")]
output_data["recall" + ("_k_" + str(k) if k else "")] = formatted_data["count_at_k"] / formatted_data["count_all"]
output_data = pd.DataFrame()
output_data[f"recall_k_{k}"] = formatted_data["count_at_k"] / formatted_data["count_all"]
output_data[f"precision_k_{k}"] = formatted_data["count_at_k"] / (k if k else 1)
output_data["top_rank"] = formatted_data["top_rank"]

output_data["average_precision" + ("_k_" + str(k) if k else "")] = _calculate_average_precisions(
formatted_data, target_column, prediction_column, convert_non_numeric=convert_non_numeric, k=k # type: ignore
)

def _calc_non_numeric_relevance(row_dict):
prediction_relevance = []
ideal_relevance = []
for target_val in row_dict[prediction_column]:
ideal_relevance.append(1 if target_val in row_dict[target_column] else 0)
prediction_relevance.append(1 if target_val in row_dict[target_column] else 0)
for target_val in row_dict[target_column]:
if target_val not in row_dict[prediction_column]:
ideal_relevance.append(1)
return (prediction_relevance, sorted(ideal_relevance, reverse=True))

def _calculate_row_ndcg(row_dict, k):
if not convert_non_numeric:
dcg_vals = [
rel / math.log2(pos + 1)
for rel, pos in zip(row_dict[target_column], row_dict[prediction_column])
if pos <= k
]
idcg_vals = [
rel / math.log2(pos + 2) for pos, rel in enumerate(sorted(row_dict[target_column], reverse=True)[:k])
]
else:
predicted_relevances, ideal_relevances = _calc_non_numeric_relevance(row_dict)
dcg_vals = [(rel / math.log(i + 2, 2)) for i, rel in enumerate(predicted_relevances[:k])]
idcg_vals = [(rel / math.log(i + 2, 2)) for i, rel in enumerate(ideal_relevances[:k])]
if sum(idcg_vals) == 0:
return 1 # if there is no relevant data, not much the recommender can do
return sum(dcg_vals) / sum(idcg_vals)

formatted_data["norm_dis_cumul_gain" + ("_k_" + str(k) if k else "")] = formatted_data.apply(
_calculate_row_ndcg, args=(k,), axis=1
output_data["norm_dis_cumul_gain" + ("_k_" + str(k) if k else "")] = formatted_data.apply(
row_wise_functions.calculate_row_ndcg, args=(k,), axis=1
)
hit_ratio = formatted_data["count_at_k"].apply(lambda x: bool(x)).sum() / len(formatted_data)
mrr = (1 / formatted_data["top_rank"]).replace([np.inf], np.nan).mean()
ndcg = formatted_data["norm_dis_cumul_gain" + ("_k_" + str(k) if k else "")].mean()
mrr = (1 / formatted_data["top_rank"]).replace([np.inf, np.nan], 0)
output_data["reciprocal_rank"] = mrr
result = log(pandas=output_data, schema=schema)
result = result.merge(
log(
row={
"accuracy" + ("_k_" + str(k) if k else ""): hit_ratio,
"mean_reciprocal_rank": mrr,
"norm_dis_cumul_gain" + ("_k_" + str(k) if k else ""): ndcg,
},
schema=schema,
)
Expand Down
Loading