feat: added analytics report generator (#2641)

.eslintignore

@@ -2,4 +2,6 @@
 **/out/*
 **/.next/*
 
-build
+build
+
+analytics

.gitignore

@@ -40,3 +40,8 @@ yarn-error.log*
 # typescript
 *.tsbuildinfo
 next-env.d.ts
+
+# python
+__pycache__
+.ipynb_checkpoints
+venv

.prettierignore

@@ -17,3 +17,6 @@ node_modules
 
 # MDX
 *.mdx
+
+# python
+analytics

analytics/constants.py

@@ -0,0 +1,20 @@
+# CHANGE THESE VALUES TO GENERATE NEW REPORTS
+# The start and end dates of the current month (yyyy-mm-dd)
+START_DATE_CURRENT = "2024-12-01"
+END_DATE_CURRENT = "2024-12-30"
+# The start and end dates of the prior months
+START_DATE_PRIOR = "2024-11-01"
+END_DATE_PRIOR = "2024-11-30"
+# The name of the folder in which to save the report
+PARENT_FOLDER_NAME = "December 2024 (demos)"
+
+# The name of the spreadsheet with the report
+SHEET_NAME = "HCA Portal"
+
+HCA_PORTAL_ID = "361323030"
+# Filter to exclud the Data Explorer
+HCA_BROWSER_EXCLUDE_FILTER = {"filter": {"fieldName": "hostName", "stringFilter": {"matchType": "EXACT", "value": "data.humancellatlas.org"}}}
+SECRET_NAME = "HCA_ANALYTICS_REPORTING_CLIENT_SECRET_PATH"
+ANALYTICS_START = "2021-01-01"
+
+OAUTH_PORT = 8082

analytics/generate_sheets_report.ipynb

@@ -0,0 +1,112 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import analytics.api as ga\n",
+    "import analytics.sheets_api as sheets\n",
+    "import analytics.sheets_elements as elements\n",
+    "import pandas as pd\n",
+    "from constants import *"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%env HCA_ANALYTICS_REPORTING_CLIENT_SECRET_PATH=../../do_not_commit_ga4_credentials.json"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ga_authentication, drive_authentication = ga.authenticate(\n",
+    "    SECRET_NAME,\n",
+    "    ga.ga4_service_params,\n",
+    "    ga.drive_service_params,\n",
+    "    port=OAUTH_PORT\n",
+    ")\n",
+    "\n",
+    "date_string = f\"{START_DATE_CURRENT} - {END_DATE_CURRENT}\"\n",
+    "\n",
+    "default_params = {\n",
+    "    \"service_system\": ga_authentication,\n",
+    "    \"start_date\": START_DATE_CURRENT,\n",
+    "    \"end_date\": END_DATE_CURRENT,\n",
+    "}\n",
+    "\n",
+    "anvil_portal_params = {\n",
+    "    **default_params,\n",
+    "    \"base_dimension_filter\": HCA_BROWSER_EXCLUDE_FILTER,\n",
+    "    \"property\": HCA_PORTAL_ID,\n",
+    "}\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_outbound = elements.get_outbound_links_change(anvil_portal_params, START_DATE_CURRENT, END_DATE_CURRENT, START_DATE_PRIOR, END_DATE_PRIOR)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dict_spreadsheet = {\n",
+    "    \"Outbound Links\": df_outbound,\n",
+    "}\n",
+    "sheets.fill_spreadsheet_with_df_dict(\n",
+    "    sheets.create_sheet_in_folder(\n",
+    "        drive_authentication,\n",
+    "        SHEET_NAME,\n",
+    "        PARENT_FOLDER_NAME,\n",
+    "        override_behavior=sheets.FILE_OVERRIDE_BEHAVIORS.OVERRIDE_IF_IN_SAME_PLACE\n",
+    "    ),\n",
+    "    dict_spreadsheet,\n",
+    "    sheets.FILE_OVERRIDE_BEHAVIORS.OVERRIDE_IF_IN_SAME_PLACE,\n",
+    "    column_formatting_options={\n",
+    "        \"Outbound Links\": {\n",
+    "            \"Total Clicks Percent Change\": sheets.COLUMN_FORMAT_OPTIONS.PERCENT_COLORED,\n",
+    "            \"Total Users Percent Change\": sheets.COLUMN_FORMAT_OPTIONS.PERCENT_COLORED,\n",
+    "        }\n",
+    "    }\n",
+    ")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

analytics/readme.md

@@ -0,0 +1,15 @@
+## Installing the environment
+- Use Python 3.12.4
+- Run `python -m venv ./venv` to create a new environment under `./venv`
+- Run `source ./venv/bin/activate` to activate the environment
+- Run `pip install -r ./requirements.txt` to install requirements
+
+## Deactivating/reactivating
+- To deactivate the environment, run `deactivate`
+- To activate the environment again, run `source ./venv/bin/activate`
+
+## Generating Reports
+- Update `constants.py` to reflect the date ranges and file name you would like for the report
+- Open `./generate_sheets_report.ipynb` using your favorite IDE or by running `jupyter notebook` and selecting it from the browser window that appears
+- Run all cells in the Jupyter notebook by pressing the button with two arrows at the top. You will be prompted to log in to your Google Account, which must have access to the relevant analytics property
+- Check your Google Drive to ensure that the desired spreadsheet is present

analytics/requirements.txt

@@ -0,0 +1,126 @@
+-e git+https://github.com/DataBiosphere/data-browser.git@eef1cee33a84853c9e15b7dc958aa07feedacfb7#egg=analytics&subdirectory=analytics/analytics_package
+anyio==4.8.0
+appnope==0.1.4
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+arrow==1.3.0
+asttokens==3.0.0
+async-lru==2.0.4
+attrs==24.3.0
+babel==2.16.0
+beautifulsoup4==4.12.3
+bleach==6.2.0
+cachetools==5.5.1
+certifi==2024.12.14
+cffi==1.17.1
+charset-normalizer==3.4.1
+comm==0.2.2
+contourpy==1.3.1
+cycler==0.12.1
+debugpy==1.8.12
+decorator==5.1.1
+defusedxml==0.7.1
+executing==2.2.0
+fastjsonschema==2.21.1
+fonttools==4.55.4
+fqdn==1.5.1
+google-api-core==2.24.0
+google-api-python-client==2.159.0
+google-auth==2.37.0
+google-auth-httplib2==0.2.0
+google-auth-oauthlib==1.2.1
+googleapis-common-protos==1.66.0
+gspread==6.1.4
+gspread-formatting==1.2.0
+h11==0.14.0
+httpcore==1.0.7
+httplib2==0.22.0
+httpx==0.28.1
+idna==3.10
+ipykernel==6.29.5
+ipython==8.31.0
+ipywidgets==8.1.5
+isoduration==20.11.0
+jedi==0.19.2
+Jinja2==3.1.5
+json5==0.10.0
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+jupyter==1.1.1
+jupyter-console==6.6.3
+jupyter-events==0.11.0
+jupyter-lsp==2.2.5
+jupyter_client==8.6.3
+jupyter_core==5.7.2
+jupyter_server==2.15.0
+jupyter_server_terminals==0.5.3
+jupyterlab==4.3.4
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.27.3
+jupyterlab_widgets==3.0.13
+kiwisolver==1.4.8
+MarkupSafe==3.0.2
+matplotlib==3.10.0
+matplotlib-inline==0.1.7
+mistune==3.1.0
+nbclient==0.10.2
+nbconvert==7.16.5
+nbformat==5.10.4
+nest-asyncio==1.6.0
+notebook==7.3.2
+notebook_shim==0.2.4
+numpy==2.2.2
+oauthlib==3.2.2
+overrides==7.7.0
+packaging==24.2
+pandas==2.2.3
+pandocfilters==1.5.1
+parso==0.8.4
+pexpect==4.9.0
+pillow==11.1.0
+platformdirs==4.3.6
+prometheus_client==0.21.1
+prompt_toolkit==3.0.50
+proto-plus==1.25.0
+protobuf==5.29.3
+psutil==6.1.1
+ptyprocess==0.7.0
+pure_eval==0.2.3
+pyasn1==0.6.1
+pyasn1_modules==0.4.1
+pycparser==2.22
+Pygments==2.19.1
+pyparsing==3.2.1
+python-dateutil==2.9.0.post0
+python-json-logger==3.2.1
+pytz==2024.2
+PyYAML==6.0.2
+pyzmq==26.2.0
+referencing==0.36.1
+requests==2.32.3
+requests-oauthlib==2.0.0
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rpds-py==0.22.3
+rsa==4.9
+Send2Trash==1.8.3
+setuptools==75.8.0
+six==1.17.0
+sniffio==1.3.1
+soupsieve==2.6
+stack-data==0.6.3
+terminado==0.18.1
+tinycss2==1.4.0
+tornado==6.4.2
+traitlets==5.14.3
+types-python-dateutil==2.9.0.20241206
+tzdata==2025.1
+uri-template==1.3.0
+uritemplate==4.1.1
+urllib3==2.3.0
+wcwidth==0.2.13
+webcolors==24.11.1
+webencodings==0.5.1
+websocket-client==1.8.0
+widgetsnbextension==4.0.13

tsconfig.json

@@ -21,5 +21,5 @@
     }
   },
   "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"],
-  "exclude": ["node_modules"]
+  "exclude": ["node_modules", "./analytics"]
 }