pfp_regression_tests.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Regression tests through LDT to verify PFP API integrity"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Main Script Definitions\n",
    "Below are the main definitions for functions that will then be called to execute API tests.  \n",
    "This cell must be run before attempting to run a test cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "import subprocess\n",
    "import sys\n",
    "import os\n",
    "import pytest\n",
    "from loguru import logger\n",
    "\n",
    "#### Basal Functions for All Verification Tests  ######################################################\n",
    "logger.remove()     # comment out for higher trace specificity than loglvl trace provides\n",
    "logger.add(sys.stdout, level='INFO', format=\"<b><level>{level}</></> \\t{message}\")\n",
    "\n",
    "\n",
    "## Run any generic LDT test and capture output\n",
    "def run_ldt_test(ldt_command):\n",
    "    logger.trace('Running run_ldt_test()...')\n",
    "    # Runs LDT as a subprocess\n",
    "    result = subprocess.run(ldt_command, capture_output=True, text=True, shell=True)\n",
    "    return result.stdout\n",
    "\n",
    "\n",
    "## Strip LDT log apart and parse as JSON to search through for keys to match on\n",
    "def strip_ldt_log(log):\n",
    "    logger.trace('Running strip_ldt_log()...')\n",
    "    try:\n",
    "        # Find the position of the last common trace line for all successful requests\n",
    "        start_position = log.find(\"TRACE \tRunning 'response_handling.handle_response'...\")\n",
    "\n",
    "        # Extract the relevant part of the output (from the common trace line to the end)\n",
    "        response_json = log[start_position:].strip()\n",
    "        return response_json\n",
    "    \n",
    "    # Error handlers\n",
    "    except ValueError as json_error:\n",
    "        logger.error(f\"Error decoding JSON: {json_error}\")\n",
    "        return None\n",
    "\n",
    "    except Exception as e:\n",
    "        logger.error(f\"An unexpected error occurred: {e}\")\n",
    "        return None\n",
    "#######################################################################################################\n",
    "\n",
    "## Test a generic causal pathway's functionality\n",
    "def causal_pathway_test(pathway, measure, acceptable_by):\n",
    "    logger.info(f'Running causal_pathway_test for {pathway}...')\n",
    "    ldt_command = f\"python3 ldt.py --loglvl trace --target local --CP {pathway} --respond\"\n",
    "    \n",
    "    test_output = run_ldt_test(ldt_command)\n",
    "    logger.info('LDT subprocess finished')\n",
    "\n",
    "    # Call func to strip end of response log to use for expeced outcome matching\n",
    "    output_data = strip_ldt_log(test_output)\n",
    "\n",
    "    ## Assert expected outcome is being seen in output\n",
    "    if output_data != None:\n",
    "        logger.debug(output_data)\n",
    "        assert f\"selected_candidate.measure:\t{measure}\" in output_data\n",
    "        assert f\"selected_candidate.acceptable_by:\t['{acceptable_by}']\" in output_data\n",
    " \n",
    "    else:\n",
    "        logger.error(\"An error occured. Verification could not be completed.\") \n",
    "\n",
    "\n",
    "    \n",
    "## Test student_t_cleaner functionality\n",
    "def cleaner_test():\n",
    "    logger.info('Running cleaner_test...')\n",
    "    ldt_command = \"python3 ldt.py --loglvl trace --target local --sendLocals 1 --localPath Regression_tests/PFP/no-significant-data.json\"\n",
    "    \n",
    "    test_output = run_ldt_test(ldt_command)\n",
    "    logger.info('LDT subprocess finished')\n",
    "\n",
    "    # Call func to strip end of response log to use for expeced outcome matching\n",
    "    output_data = strip_ldt_log(test_output)\n",
    "\n",
    "    ## Assert expected outcome is being seen in output\n",
    "    if output_data != None:\n",
    "        logger.debug(output_data)\n",
    "        assert 'detail' in output_data\n",
    "        assert '400-error' in output_data\n",
    "        assert 'Status Code:\t400' in output_data\n",
    " \n",
    "    else:\n",
    "        logger.error(\"An error occured. Verification could not be completed.\") \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Causal Pathway Verification\n",
    "By manipulating pathway, measure, and acceptable_by arguments, we can test any of the causal pathway testing suite responses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mINFO\u001b[0m\u001b[1m\u001b[0m \tRunning causal_pathway_test for social_worse...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[1mINFO\u001b[0m\u001b[1m\u001b[0m \tLDT subprocess finished\n"
     ]
    }
   ],
   "source": [
    "# Social Worse\n",
    "## Data pulled from:     https://github.com/Display-Lab/knowledge-base/blob/main/vignettes/dev_templates/causal_pathway_test_suite/social_worse_cptest.json\n",
    "causal_pathway_test('social_worse', 'GLU01', 'Social Worse')\n",
    "\n",
    "# Social Better\n",
    "## Data pulled from:     https://github.com/Display-Lab/knowledge-base/blob/main/vignettes/dev_templates/causal_pathway_test_suite/social_better_cptest.json\n",
    "causal_pathway_test('social_better', 'SUS04', 'Social better')\n",
    "\n",
    "# Improving\n",
    "## Data pulled from:     https://github.com/Display-Lab/knowledge-base/blob/main/vignettes/dev_templates/causal_pathway_test_suite/improving_cptest.json\n",
    "causal_pathway_test('improving', 'BP03', 'Improving')\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### No Significant Data Test\n",
    "Tests input message in `Regression_tests/PFP/no-significant-data.json`   \n",
    "- Verifies data cleaning in `PFP/bit_stomach/student_t_cleaner.py` fully functional"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaner_test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "LDTvenv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}