From 5e37743238addbc00d3293eef26157d80805a7d5 Mon Sep 17 00:00:00 2001
From: Ivan Leo <ivanleomk@gmail.com>
Date: Thu, 7 Nov 2024 15:22:58 +0800
Subject: [PATCH] feat: added initial metrics

---
 .github/workflows/release.yml | 32 ++++++++++++++++++++++++++++++++
 .python-version               |  1 +
 README.md                     |  0
 pyproject.toml                | 14 ++++++++++++++
 rag_metrics/__init__.py       |  4 ++++
 rag_metrics/metrics.py        | 12 ++++++++++++
 6 files changed, 63 insertions(+)
 create mode 100644 .github/workflows/release.yml
 create mode 100644 .python-version
 create mode 100644 README.md
 create mode 100644 pyproject.toml
 create mode 100644 rag_metrics/__init__.py
 create mode 100644 rag_metrics/metrics.py

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
new file mode 100644
index 0000000..eb9030f
--- /dev/null
+++ b/.github/workflows/release.yml
@@ -0,0 +1,32 @@
+name: Upload Python Package to PyPI when a Release is Created
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  pypi-publish:
+    name: Publish release to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/rag-metrics
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+
+      - name: Set up Python
+        run: uv python install
+
+      - name: Install the project
+        run: uv sync --all-extras --dev
+
+      - name: Build Package
+        run: uv build
+
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
diff --git a/.python-version b/.python-version
new file mode 100644
index 0000000..2c07333
--- /dev/null
+++ b/.python-version
@@ -0,0 +1 @@
+3.11
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..e69de29
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..bce3e9e
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,14 @@
+[project]
+name = "rag-metrics"
+version = "0.1.0"
+description = "Metrics for RAG systems"
+readme = "README.md"
+authors = [
+    { name = "Ivan Leo", email = "ivanleomk@gmail.com" } 
+]
+requires-python = ">=3.11"
+dependencies = []
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
diff --git a/rag_metrics/__init__.py b/rag_metrics/__init__.py
new file mode 100644
index 0000000..26536d4
--- /dev/null
+++ b/rag_metrics/__init__.py
@@ -0,0 +1,4 @@
+from rag_metrics.metrics import calculate_mrr, calculate_recall
+
+
+__all__ = ["calculate_mrr", "calculate_recall"]
diff --git a/rag_metrics/metrics.py b/rag_metrics/metrics.py
new file mode 100644
index 0000000..544b7d0
--- /dev/null
+++ b/rag_metrics/metrics.py
@@ -0,0 +1,12 @@
+def calculate_mrr(predictions: list[str], gt: list[str]):
+    mrr = 0
+    for label in gt:
+        if label in predictions:
+            # Find the relevant item that has the smallest index
+            mrr = max(mrr, 1 / (predictions.index(label) + 1))
+    return mrr
+
+
+def calculate_recall(predictions: list[str], gt: list[str]):
+    # Calculate the proportion of relevant items that were retrieved
+    return len([label for label in gt if label in predictions]) / len(gt)