SpeziDataPipelineTemplate.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1b4070c1-59f7-474e-8ea1-156cd42ef811",
   "metadata": {},
   "source": [
    "## Import modules\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3da82a85-6035-4757-a666-e56b5f270513",
   "metadata": {},
   "outputs": [],
   "source": [
    "from spezi_data_pipeline.data_access.firebase_fhir_data_access import FirebaseFHIRAccess\n",
    "from spezi_data_pipeline.data_flattening.fhir_resources_flattener import flatten_fhir_resources, FHIRDataFrame\n",
    "from spezi_data_pipeline.data_processing.data_processor import FHIRDataProcessor\n",
    "from spezi_data_pipeline.data_processing.observation_processor import calculate_activity_index\n",
    "from spezi_data_pipeline.data_exploration.data_explorer import DataExplorer, visualizer_factory, explore_total_records_number\n",
    "from spezi_data_pipeline.data_export.data_exporter import DataExporter"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3623d09-656a-4abe-a4b9-a433ec86a91c",
   "metadata": {},
   "source": [
    "## Define credential files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e844a4e1-1d9e-4c4a-a6a5-46da5b08f17c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Define your Firebase project ID\n",
    "project_id = \"spezi-data-pipeline\"  # Replace with your Firebase project ID\n",
    "\n",
    "# Define the service account key file\n",
    "service_account_key_file = \"path_to_service_account_key_file.json\"  # Replace with your service account key file\n",
    "\n",
    "# Define the collection name where your FHIR observations are stored and the input code if filtering is needed\n",
    "collection_name = \"users\"\n",
    "subcollection_name = \"HealthKit\"\n",
    "\n",
    "# Define the list of HealthKit quantities to query from Firebase Firestore\n",
    "loinc_codes = [\"55423-8\", \"8867-4\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dee3e7e7-e273-4367-b36b-6a9148ad898f",
   "metadata": {},
   "source": [
    "## Initialize the FirebaseFHIRAccess class using your Firebase credentials"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62b89662-db30-4976-8ecf-da907b0721ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize the FirebaseFHIRAccess class using your Firebase credentials\n",
    "firebase_access = FirebaseFHIRAccess(project_id, service_account_key_file)\n",
    "firebase_access.connect()\n",
    "\n",
    "fhir_observations = firebase_access.fetch_data(collection_name, subcollection_name, loinc_codes)\n",
    "\n",
    "flattened_fhir_dataframe = flatten_fhir_resources(fhir_observations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "495b1d17-6414-4718-870a-2f12d6c9c5b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "flattened_fhir_dataframe.df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01210300-19f1-4cd6-a820-9864928d2b90",
   "metadata": {},
   "source": [
    "## Explore the number of recordings in the database by LOINC code  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9de99b98-4b1d-4b35-8e0c-79925b830f60",
   "metadata": {},
   "outputs": [],
   "source": [
    "explore_total_records_number(flattened_fhir_dataframe.df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43616584-8c5d-4580-baaa-59f25e5f334f",
   "metadata": {},
   "source": [
    "## Apply basic processing for convenient data readability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d878a15-3382-4952-83d2-0507fcff471a",
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_fhir_dataframe = FHIRDataProcessor().process_fhir_data(flattened_fhir_dataframe)\n",
    "processed_fhir_dataframe.df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e91ed96-c86e-4486-ab28-ef55fdd1947e",
   "metadata": {},
   "source": [
    "## Explore Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8ddd7a7-25a2-40e1-ac34-c6de3adeb204",
   "metadata": {},
   "source": [
    "### HealthKit Quantity Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a09ab04-4c93-44dd-b95c-1b825f645b58",
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_users = [\"3EUoHxIuYkWMKcnLfK38nTGOqHn1\",\"7uMKVmPZdwgtb9hc6r9YZyYXnwc2\", \"sgsxyilwB3T3xf3LIvkpSajN3NW2\"]\n",
    "selected_start_date = \"2024-02-22\"\n",
    "selected_end_date = \"2024-04-02\"\n",
    "explorer = visualizer_factory(processed_fhir_dataframe)\n",
    "\n",
    "# explorer.set_user_ids(selected_users)\n",
    "# explorer.set_date_range(selected_start_date, selected_end_date)\n",
    "# explorer.set_y_bounds(50, 50000)\n",
    "\n",
    "figs = explorer.create_static_plot(processed_fhir_dataframe)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab1b9dfe-1cfa-4838-bb5a-28989b6211a0",
   "metadata": {},
   "source": [
    "### Calculate Activity Index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93970c18-acc4-45cf-9d58-495a6c0eb280",
   "metadata": {},
   "outputs": [],
   "source": [
    "activity_index_fhir_dataframe = calculate_activity_index(processed_fhir_dataframe)\n",
    "\n",
    "selected_users = [\"XrftRMc358NndzcRWEQ7P2MxvabZ\"]\n",
    "explorer = visualizer_factory(activity_index_fhir_dataframe)\n",
    "\n",
    "explorer.set_user_ids(selected_users)\n",
    "# explorer.set_date_range(selected_start_date, selected_end_date)\n",
    "\n",
    "figs = explorer.create_static_plot(activity_index_fhir_dataframe)\n",
    "activity_index_fhir_dataframe.df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6558f0c-8035-40ae-b45c-429109398e44",
   "metadata": {},
   "source": [
    "### ECG Recording Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b908976-582f-42cd-8f86-dd588202ec79",
   "metadata": {},
   "outputs": [],
   "source": [
    "# selected_users = [\"k3BnzOGAO0fIaxkDVXTZKlj3LAu2\", \"3EUoHxIuYkWMKcnLfK38nTGOqHn1\"]\n",
    "\n",
    "# selected_start_date = \"2023-03-13\"\n",
    "# selected_end_date = \"2023-03-13\"\n",
    "\n",
    "# visualizer = visualizer_factory(processed_fhir_dataframe)\n",
    "# visualizer.set_user_ids(selected_users)\n",
    "# visualizer.set_date_range(selected_start_date, selected_end_date)\n",
    "# figs = visualizer.plot_ecg_subplots(processed_fhir_dataframe) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a91e9dce-e956-4694-8880-cf4508ccd8ae",
   "metadata": {},
   "source": [
    "## Export data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1bd3d6a-1aec-4bdc-9325-7c85efba0ff3",
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_users = [\"sEmijWpn0vXe1cj60GO5kkjkrdT4\"]\n",
    "\n",
    "exporter = DataExporter(processed_fhir_dataframe)\n",
    "# exporter.set_user_ids(selected_users)\n",
    "# exporter.set_date_range(selected_start_date, selected_end_date)\n",
    "# exporter.set_date_range(selected_start_date, selected_start_date)\n",
    "exporter.create_and_save_plot(\"data_plot\") "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}