feat: added initial metrics

.github/workflows/release.yml

@@ -0,0 +1,32 @@
+name: Upload Python Package to PyPI when a Release is Created
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  pypi-publish:
+    name: Publish release to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/rag-metrics
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+
+      - name: Set up Python
+        run: uv python install
+
+      - name: Install the project
+        run: uv sync --all-extras --dev
+
+      - name: Build Package
+        run: uv build
+
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

.python-version

@@ -0,0 +1 @@
+3.11

pyproject.toml

@@ -0,0 +1,14 @@
+[project]
+name = "rag-metrics"
+version = "0.1.0"
+description = "Metrics for RAG systems"
+readme = "README.md"
+authors = [
+    { name = "Ivan Leo", email = "[email protected]" } 
+]
+requires-python = ">=3.11"
+dependencies = []
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"

rag_metrics/__init__.py

@@ -0,0 +1,4 @@
+from rag_metrics.metrics import calculate_mrr, calculate_recall
+
+
+__all__ = ["calculate_mrr", "calculate_recall"]

rag_metrics/metrics.py

@@ -0,0 +1,12 @@
+def calculate_mrr(predictions: list[str], gt: list[str]):
+    mrr = 0
+    for label in gt:
+        if label in predictions:
+            # Find the relevant item that has the smallest index
+            mrr = max(mrr, 1 / (predictions.index(label) + 1))
+    return mrr
+
+
+def calculate_recall(predictions: list[str], gt: list[str]):
+    # Calculate the proportion of relevant items that were retrieved
+    return len([label for label in gt if label in predictions]) / len(gt)