docs/devcontainer/mpi4py-tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ExaGO mpi4py Jupyter Tutorial\n",
    "\n",
    "We have a Python wrapper for ExaGO installed in this container through spack. You can view the Dockerfile in `.devcontainer/Dockerfile`. See the Appendix for more details.\n",
    "\n",
    "This is different to the base tutorial in that we include some additional information about MPI configuration."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ExaGO 1.6.0 built on Nov  9 2023\n",
      "ExaGO 1.6.0 built on Nov  9 2023\n",
      "ExaGO 1.6.0 built on Nov  9 2023\n",
      "ExaGO 1.6.0 built on Nov  9 2023\n",
      "ExaGO 1.6.0 built on Nov  9 2023\n"
     ]
    }
   ],
   "source": [
    "!mpirun -n 5 opflow --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello\n",
      "<mpi4py.MPI.Intracomm object at 0x7fd664469910> from rank 0 out of 1 total ranks\n"
     ]
    }
   ],
   "source": [
    "# Unique to MPI configuration\n",
    "print(f'Hello')\n",
    "import mpi4py\n",
    "mpi4py.rc.threads = False\n",
    "from mpi4py import MPI\n",
    "comm = MPI.COMM_WORLD\n",
    "rank = comm.Get_rank()\n",
    "size = comm.Get_size()\n",
    "print(f'{comm} from rank {rank} out of {size} total ranks')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/app/docs/devcontainer\n",
      "/home/app\n"
     ]
    }
   ],
   "source": [
    "# Get pwd and change directory to where datafiles are\n",
    "import os\n",
    "print(f'{os.getcwd()}')\n",
    "os.chdir('../..')\n",
    "print(f'{os.getcwd()}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We need to run in a parallel cluster, save file to disk (or return in place), and then do something with the solution..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting 6 engines with <class 'ipyparallel.cluster.launcher.MPIEngineSetLauncher'>\n",
      "100%|██████████| 6/6 [00:05<00:00,  1.03engine/s]\n",
      "Hello World from rank 0. total ranks=6. host=dfdb9932ea09 out of /home/app\n",
      "Hello World from rank 1. total ranks=6. host=dfdb9932ea09 out of /home/app\n",
      "Hello World from rank 2. total ranks=6. host=dfdb9932ea09 out of /home/app\n",
      "Hello World from rank 3. total ranks=6. host=dfdb9932ea09 out of /home/app\n",
      "Hello World from rank 4. total ranks=6. host=dfdb9932ea09 out of /home/app\n",
      "Hello World from rank 5. total ranks=6. host=dfdb9932ea09 out of /home/app\n",
      "Stopping engine(s): 1699501339\n",
      "engine set stopped 1699501339: {'exit_code': 0, 'pid': 35617, 'identifier': 'ipengine-1699501338-ciod-1699501339-35554'}\n",
      "Stopping controller\n",
      "Controller stopped: {'exit_code': 0, 'pid': 35587, 'identifier': 'ipcontroller-1699501338-ciod-35554'}\n"
     ]
    }
   ],
   "source": [
    "# Hello world first:\n",
    "import ipyparallel as ipp\n",
    "\n",
    "def mpi_example():\n",
    "    import os\n",
    "    from mpi4py import MPI\n",
    "    comm = MPI.COMM_WORLD\n",
    "    return f\"Hello World from rank {comm.Get_rank()}. total ranks={comm.Get_size()}. host={MPI.Get_processor_name()} out of {os.getcwd()}\"\n",
    "\n",
    "# request an MPI cluster with 24 engines\n",
    "with ipp.Cluster(controller_ip=\"*\", engines=\"mpi\", n=6) as rc:\n",
    "    # get a broadcast_view on the cluster which is best\n",
    "    # suited for MPI style computation\n",
    "    view = rc.broadcast_view()\n",
    "    # run the mpi_example function on all engines in parallel\n",
    "    r = view.apply_sync(mpi_example)\n",
    "    # Retrieve and print the result from the engines\n",
    "    print(\"\\n\".join(r))\n",
    "# at this point, the cluster processes have been shutdown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/app\n",
      "/home/app\n",
      "Starting 1 engines with <class 'ipyparallel.cluster.launcher.MPIEngineSetLauncher'>\n",
      "100%|██████████| 1/1 [00:06<00:00,  6.02s/engine]\n",
      "Stopping engine(s): 1699625249\n",
      "engine set stopped 1699625249: {'exit_code': 15, 'pid': 4000, 'identifier': 'ipengine-1699625248-oea1-1699625249-2295'}\n",
      "Stopping controller\n",
      "Controller stopped: {'exit_code': 0, 'pid': 3989, 'identifier': 'ipcontroller-1699625248-oea1-2295'}\n"
     ]
    }
   ],
   "source": [
    "import ipyparallel as ipp\n",
    "import os\n",
    "print(f'{os.getcwd()}')\n",
    "os.chdir('/home/app')\n",
    "print(f'{os.getcwd()}')\n",
    "\n",
    "def run_exago():\n",
    "    from mpi4py import MPI\n",
    "    import exago\n",
    "    import os\n",
    "    comm = MPI.COMM_WORLD\n",
    "    rank = comm.Get_rank()\n",
    "    exago.initialize(\"app\", comm)\n",
    "    scopf = exago.SCOPFLOW()\n",
    "    path = exago.prefix()\n",
    "    scopf.set_network_data(\n",
    "        os.path.join(path, 'share', 'exago', 'datafiles', 'case9', 'case9mod.m'))\n",
    "    scopf.set_contingency_data(\n",
    "        os.path.join(path, 'share', 'exago', 'datafiles', 'case9', 'case9.cont'),\n",
    "        exago.ContingencyFileInputFormat.NATIVE)\n",
    "    scopf.set_num_contingencies(4)\n",
    "    # set solver HIOP\n",
    "    # subproblem solver IPOPT\n",
    "    scopf.set_solver(\"IPOPT\")\n",
    "    scopf.solve()\n",
    "\n",
    "\n",
    "    comm.barrier()\n",
    "\n",
    "    scopf.save_solution_all(exago.OutputFormat.JSON, 'solution')\n",
    "\n",
    "    # exago.finalize()\n",
    "    comm.barrier()\n",
    "\n",
    "# request an MPI cluster with 24 engines\n",
    "with ipp.Cluster(controller_ip=\"*\", engines=\"mpi\", n=1) as rc:\n",
    "    # get a broadcast_view on the cluster which is best\n",
    "    # suited for MPI style computation\n",
    "    view = rc.broadcast_view()\n",
    "    # run the mpi_example function on all engines in parallel\n",
    "    r = view.apply_sync(run_exago)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "\t\"casefile\": \"./datafiles/case9/case9mod.m\",\n",
      "\t\"gicfile\": \"not given\",\n",
      "\t\"nbranch\": 9,\n",
      "\t\"ngen\": 3,\n",
      "\t\"nbus\": 9,\n",
      "\t\"KVlevels\": [\n",
      "\t\t345.000000\n",
      "\t],\n",
      "\t\"casejsonfile\": \"solution.json\",\n",
      "\t\"geojsondata\": {\n",
      "\t\t\"type\": \"FeatureCollection\",\n",
      "\t\t\"features\": [\n",
      "\t\t\t{\n",
      "\t\t\t\t\"type\": \"Feature\",\n",
      "\t\t\t\t\"geometry\": {\n",
      "\t\t\t\t\t\"type\": \"Point\",\n",
      "\t\t\t\t\t\"coordinates\": [\n",
      "\t\t\t\t\t\t-97.508469,\n",
      "\t\t\t\t\t\t35.981918\n",
      "\t\t\t\t\t]\n",
      "\t\t\t\t},\n",
      "\t\t\t\t\"properties\": {\n",
      "\t\t\t\t\t\"elementtype\": \"SUBSTATION\",\n",
      "\t\t\t\t\t\"NAME\": \"1\",\n",
      "\t\t\t\t\t\"nbus\": 1,\n",
      "\t\t\t\t\t\"KVlevels\": [\n",
      "\t\t\t\t\t\t345.000000\n",
      "\t\t\t\t\t],\n",
      "\t\t\t\t\t\"bus\": [\n",
      "\t\t\t\t\t\t{\n",
      "\t\t\t\t\t\t\t\"elementtype\": \"bus\",\n",
      "\t\t\t\t\t\t\t\"BUS_I\": 1,\n",
      "\t\t\t\t\t\t\t\"VA\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"VM\": 1.099983,\n",
      "\t\t\t\t\t\t\t\"BUS_NAME\": \" 1\",\n",
      "\t\t\t\t\t\t\t\"VMIN\": 0.900000,\n",
      "\t\t\t\t\t\t\t\"VMAX\": 1.100000,\n",
      "\t\t\t\t\t\t\t\"BASE_KV\": 345.000000,\n",
      "\t\t\t\t\t\t\t\"PD\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"QD\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"PDloss\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"QDloss\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"LAM_P\": 2102.907811,\n",
      "\t\t\t\t\t\t\t\"LAM_Q\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"ngen\": 1,\n",
      "\t\t\t\t\t\t\t\"gen\": [\n",
      "\t\t\t\t\t\t\t\t{\n",
      "\t\t\t\t\t\t\t\t\t\"GEN_BUS\": 1,\n",
      "\t\t\t\t\t\t\t\t\t\"GEN_FUEL\": \"UNDEFINED\",\n",
      "\t\t\t\t\t\t\t\t\t\"PG\": 72.859446,\n",
      "\t\t\t\t\t\t\t\t\t\"QG\": 6.790092,\n",
      "\t\t\t\t\t\t\t\t\t\"GEN_STATUS\": 1,\n",
      "\t\t\t\t\t\t\t\t\t\"PMAX\": 350.000000,\n",
      "\t\t\t\t\t\t\t\t\t\"PMIN\": 10.000000,\n",
      "\t\t\t\t\t\t\t\t\t\"QMAX\": 300.000000,\n",
      "\t\t\t\t\t\t\t\t\t\"QMIN\": -300.000000\n",
      "\t\t\t\t\t\t\t\t}\n",
      "\t\t\t\t\t\t\t]\n",
      "\t\t\t\t\t\t}\n",
      "\t\t\t\t\t],\n",
      "\t\t\t\t\t\"Vm\": 1.099983\n",
      "\t\t\t\t}\n",
      "\t\t\t},\n",
      "\t\t\t{\n",
      "\t\t\t\t\"type\": \"Feature\",\n",
      "\t\t\t\t\"geometry\": {\n",
      "\t\t\t\t\t\"type\": \"Point\",\n",
      "\t\t\t\t\t\"coordinates\": [\n",
      "\t\t\t\t\t\t-97.187075,\n",
      "\t\t\t\t\t\t35.864940\n",
      "\t\t\t\t\t]\n",
      "\t\t\t\t},\n",
      "\t\t\t\t\"properties\": {\n",
      "\t\t\t\t\t\"elementtype\": \"SUBSTATION\",\n",
      "\t\t\t\t\t\"NAME\": \"2\",\n",
      "\t\t\t\t\t\"nbus\": 1,\n",
      "\t\t\t\t\t\"KVlevels\": [\n",
      "\t\t\t\t\t\t345.000000\n",
      "\t\t\t\t\t],\n",
      "\t\t\t\t\t\"bus\": [\n",
      "\t\t\t\t\t\t{\n",
      "\t\t\t\t\t\t\t\"elementtype\": \"bus\",\n",
      "\t\t\t\t\t\t\t\"BUS_I\": 2,\n",
      "\t\t\t\t\t\t\t\"VA\": 3.927539,\n",
      "\t\t\t\t\t\t\t\"VM\": 1.095254,\n",
      "\t\t\t\t\t\t\t\"BUS_NAME\": \" 1\",\n",
      "\t\t\t\t\t\t\t\"VMIN\": 0.900000,\n",
      "\t\t\t\t\t\t\t\"VMAX\": 1.100000,\n",
      "\t\t\t\t\t\t\t\"BASE_KV\": 345.000000,\n",
      "\t\t\t\t\t\t\t\"PD\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"QD\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"PDloss\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"QDloss\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"LAM_P\": 2059.179811,\n",
      "\t\t\t\t\t\t\t\"LAM_Q\": -0.000000,\n",
      "\t\t\t\t\t\t\t\"ngen\": 1,\n",
      "\t\t\t\t\t\t\t\"gen\": [\n",
      "\t\t\t\t\t\t\t\t{\n",
      "\t\t\t\t\t\t\t\t\t\"GEN_BUS\": 2,\n",
      "\t\t\t\t\t\t\t\t\t\"GEN_FUEL\": \"UNDEFINED\",\n",
      "\t\t\t\t\t\t\t\t\t\"PG\": 114.069401,\n",
      "\t\t\t\t\t\t\t\t\t\"QG\": -5.128376,\n",
      "\t\t\t\t\t\t\t\t\t\"GEN_STATUS\": 1,\n",
      "\t\t\t\t\t\t\t\t\t\"PMAX\": 300.000000,\n",
      "\t\t\t\t\t\t\t\t\t\"PMIN\": 10.000000,\n",
      "\t\t\t\t\t\t\t\t\t\"QMAX\": 300.000000,\n",
      "\t\t\t\t\t\t\t\t\t\"QMIN\": -300.000000\n",
      "\t\t\t\t\t\t\t\t}\n",
      "\t\t\t\t\t\t\t]\n",
      "\t\t\t\t\t\t}\n",
      "\t\t\t\t\t],\n",
      "\t\t\t\t\t\"Vm\": 1.095254\n",
      "\t\t\t\t}\n",
      "\t\t\t},\n",
      "\t\t\t{\n",
      "\t\t\t\t\"type\": \"Feature\",\n",
      "\t\t\t\t\"geometry\": {\n",
      "\t\t\t\t\t\"type\": \"Point\",\n",
      "\t\t\t\t\t\"coordinates\": [\n",
      "\t\t\t\t\t\t-97.016065,\n",
      "\t\t\t\t\t\t35.568742\n",
      "\t\t\t\t\t]\n",
      "\t\t\t\t},\n",
      "\t\t\t\t\"properties\": {\n",
      "\t\t\t\t\t\"elementtype\": \"SUBSTATION\",\n",
      "\t\t\t\t\t\"NAME\": \"3\",\n",
      "\t\t\t\t\t\"nbus\": 1,\n",
      "\t\t\t\t\t\"KVlevels\": [\n",
      "\t\t\t\t\t\t345.000000\n",
      "\t\t\t\t\t],\n",
      "\t\t\t\t\t\"bus\": [\n",
      "\t\t\t\t\t\t{\n",
      "\t\t\t\t\t\t\t\"elementtype\": \"bus\",\n",
      "\t\t\t\t\t\t\t\"BUS_I\": 3,\n",
      "\t\t\t\t\t\t\t\"VA\": 2.120068,\n",
      "\t\t\t\t\t\t\t\"VM\": 1.086566,\n",
      "\t\t\t\t\t\t\t\"BUS_NAME\": \" 1\",\n",
      "\t\t\t\t\t\t\t\"VMIN\": 0.900000,\n",
      "\t\t\t\t\t\t\t\"VMAX\": 1.100000,\n",
      "\t\t\t\t\t\t\t\"BASE_KV\": 345.000000,\n",
      "\t\t\t\t\t\t\t\"PD\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"QD\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"PDloss\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"QDloss\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"LAM_P\": 2065.154806,\n",
      "\t\t\t\t\t\t\t\"LAM_Q\": -0.000000,\n",
      "\t\t\t\t\t\t\t\"ngen\": 1,\n",
      "\t\t\t\t\t\t\t\"gen\": [\n",
      "\t\t\t\t\t\t\t\t{\n",
      "\t\t\t\t\t\t\t\t\t\"GEN_BUS\": 3,\n",
      "\t\t\t\t\t\t\t\t\t\"GEN_FUEL\": \"UNDEFINED\",\n",
      "\t\t\t\t\t\t\t\t\t\"PG\": 80.210400,\n",
      "\t\t\t\t\t\t\t\t\t\"QG\": -23.474118,\n",
      "\t\t\t\t\t\t\t\t\t\"GEN_STATUS\": 1,\n",
      "\t\t\t\t\t\t\t\t\t\"PMAX\": 270.000000,\n",
      "\t\t\t\t\t\t\t\t\t\"PMIN\": 10.000000,\n",
      "\t\t\t\t\t\t\t\t\t\"QMAX\": 300.000000,\n",
      "\t\t\t\t\t\t\t\t\t\"QMIN\": -300.000000\n",
      "\t\t\t\t\t\t\t\t}\n",
      "\t\t\t\t\t\t\t]\n",
      "\t\t\t\t\t\t}\n",
      "\t\t\t\t\t],\n",
      "\t\t\t\t\t\"Vm\": 1.086566\n",
      "\t\t\t\t}\n",
      "\t\t\t},\n",
      "\t\t\t{\n",
      "\t\t\t\t\"type\": \"Feature\",\n",
      "\t\t\t\t\"geometry\": {\n",
      "\t\t\t\t\t\"type\": \"Point\",\n",
      "\t\t\t\t\t\"coordinates\": [\n",
      "\t\t\t\t\t\t-97.075456,\n",
      "\t\t\t\t\t\t35.231918\n",
      "\t\t\t\t\t]\n",
      "\t\t\t\t},\n",
      "\t\t\t\t\"properties\": {\n",
      "\t\t\t\t\t\"elementtype\": \"SUBSTATION\",\n",
      "\t\t\t\t\t\"NAME\": \"4\",\n",
      "\t\t\t\t\t\"nbus\": 1,\n",
      "\t\t\t\t\t\"KVlevels\": [\n",
      "\t\t\t\t\t\t345.000000\n",
      "\t\t\t\t\t],\n",
      "\t\t\t\t\t\"bus\": [\n",
      "\t\t\t\t\t\t{\n",
      "\t\t\t\t\t\t\t\"elementtype\": \"bus\",\n",
      "\t\t\t\t\t\t\t\"BUS_I\": 4,\n",
      "\t\t\t\t\t\t\t\"VA\": -1.992724,\n",
      "\t\t\t\t\t\t\t\"VM\": 1.097021,\n",
      "\t\t\t\t\t\t\t\"BUS_NAME\": \" 1\",\n",
      "\t\t\t\t\t\t\t\"VMIN\": 0.900000,\n",
      "\t\t\t\t\t\t\t\"VMAX\": 1.100000,\n",
      "\t\t\t\t\t\t\t\"BASE_KV\": 345.000000,\n",
      "\t\t\t\t\t\t\t\"PD\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"QD\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"PDloss\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"QDloss\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"LAM_P\": 2103.164667,\n",
      "\t\t\t\t\t\t\t\"LAM_Q\": 0.076862,\n",
      "\t\t\t\t\t\t\t\"ngen\": 0,\n",
      "\t\t\t\t\t\t\t\"gen\": [\n",
      "\t\t\t\t\t\t\t]\n",
      "\t\t\t\t\t\t}\n",
      "\t\t\t\t\t],\n",
      "\t\t\t\t\t\"Vm\": 1.097021\n",
      "\t\t\t\t}\n",
      "\t\t\t},\n",
      "\t\t\t{\n",
      "\t\t\t\t\"type\": \"Feature\",\n",
      "\t\t\t\t\"geometry\": {\n",
      "\t\t\t\t\t\"type\": \"Point\",\n",
      "\t\t\t\t\t\"coordinates\": [\n",
      "\t\t\t\t\t\t-97.337459,\n",
      "\t\t\t\t\t\t35.012072\n",
      "\t\t\t\t\t]\n",
      "\t\t\t\t},\n",
      "\t\t\t\t\"properties\": {\n",
      "\t\t\t\t\t\"elementtype\": \"SUBSTATION\",\n",
      "\t\t\t\t\t\"NAME\": \"5\",\n",
      "\t\t\t\t\t\"nbus\": 1,\n",
      "\t\t\t\t\t\"KVlevels\": [\n",
      "\t\t\t\t\t\t345.000000\n",
      "\t\t\t\t\t],\n",
      "\t\t\t\t\t\"bus\": [\n",
      "\t\t\t\t\t\t{\n",
      "\t\t\t\t\t\t\t\"elementtype\": \"bus\",\n",
      "\t\t\t\t\t\t\t\"BUS_I\": 5,\n",
      "\t\t\t\t\t\t\t\"VA\": -3.059611,\n",
      "\t\t\t\t\t\t\t\"VM\": 1.079374,\n",
      "\t\t\t\t\t\t\t\"BUS_NAME\": \" 1\",\n",
      "\t\t\t\t\t\t\t\"VMIN\": 0.900000,\n",
      "\t\t\t\t\t\t\t\"VMAX\": 1.100000,\n",
      "\t\t\t\t\t\t\t\"BASE_KV\": 345.000000,\n",
      "\t\t\t\t\t\t\t\"PD\": 75.000000,\n",
      "\t\t\t\t\t\t\t\"QD\": 50.000000,\n",
      "\t\t\t\t\t\t\t\"PDloss\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"QDloss\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"LAM_P\": 2113.454996,\n",
      "\t\t\t\t\t\t\t\"LAM_Q\": 7.292915,\n",
      "\t\t\t\t\t\t\t\"ngen\": 0,\n",
      "\t\t\t\t\t\t\t\"gen\": [\n",
      "\t\t\t\t\t\t\t]\n",
      "\t\t\t\t\t\t}\n",
      "\t\t\t\t\t],\n",
      "\t\t\t\t\t\"Vm\": 1.079374\n",
      "\t\t\t\t}\n",
      "\t\t\t},\n",
      "\t\t\t{\n",
      "\t\t\t\t\"type\": \"Feature\",\n",
      "\t\t\t\t\"geometry\": {\n",
      "\t\t\t\t\t\"type\": \"Point\",\n",
      "\t\t\t\t\t\"coordinates\": [\n",
      "\t\t\t\t\t\t-97.679479,\n",
      "\t\t\t\t\t\t35.012072\n",
      "\t\t\t\t\t]\n",
      "\t\t\t\t},\n",
      "\t\t\t\t\"properties\": {\n",
      "\t\t\t\t\t\"elementtype\": \"SUBSTATION\",\n",
      "\t\t\t\t\t\"NAME\": \"6\",\n",
      "\t\t\t\t\t\"nbus\": 1,\n",
      "\t\t\t\t\t\"KVlevels\": [\n",
      "\t\t\t\t\t\t345.000000\n",
      "\t\t\t\t\t],\n",
      "\t\t\t\t\t\"bus\": [\n",
      "\t\t\t\t\t\t{\n",
      "\t\t\t\t\t\t\t\"elementtype\": \"bus\",\n",
      "\t\t\t\t\t\t\t\"BUS_I\": 6,\n",
      "\t\t\t\t\t\t\t\"VA\": -3.926879,\n",
      "\t\t\t\t\t\t\t\"VM\": 1.086823,\n",
      "\t\t\t\t\t\t\t\"BUS_NAME\": \" 1\",\n",
      "\t\t\t\t\t\t\t\"VMIN\": 0.900000,\n",
      "\t\t\t\t\t\t\t\"VMAX\": 1.100000,\n",
      "\t\t\t\t\t\t\t\"BASE_KV\": 345.000000,\n",
      "\t\t\t\t\t\t\t\"PD\": 90.000000,\n",
      "\t\t\t\t\t\t\t\"QD\": 30.000000,\n",
      "\t\t\t\t\t\t\t\"PDloss\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"QDloss\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"LAM_P\": 2129.849392,\n",
      "\t\t\t\t\t\t\t\"LAM_Q\": 1.619742,\n",
      "\t\t\t\t\t\t\t\"ngen\": 0,\n",
      "\t\t\t\t\t\t\t\"gen\": [\n",
      "\t\t\t\t\t\t\t]\n",
      "\t\t\t\t\t\t}\n",
      "\t\t\t\t\t],\n",
      "\t\t\t\t\t\"Vm\": 1.086823\n",
      "\t\t\t\t}\n",
      "\t\t\t},\n",
      "\t\t\t{\n",
      "\t\t\t\t\"type\": \"Feature\",\n",
      "\t\t\t\t\"geometry\": {\n",
      "\t\t\t\t\t\"type\": \"Point\",\n",
      "\t\t\t\t\t\"coordinates\": [\n",
      "\t\t\t\t\t\t-97.941482,\n",
      "\t\t\t\t\t\t35.231918\n",
      "\t\t\t\t\t]\n",
      "\t\t\t\t},\n",
      "\t\t\t\t\"properties\": {\n",
      "\t\t\t\t\t\"elementtype\": \"SUBSTATION\",\n",
      "\t\t\t\t\t\"NAME\": \"7\",\n",
      "\t\t\t\t\t\"nbus\": 1,\n",
      "\t\t\t\t\t\"KVlevels\": [\n",
      "\t\t\t\t\t\t345.000000\n",
      "\t\t\t\t\t],\n",
      "\t\t\t\t\t\"bus\": [\n",
      "\t\t\t\t\t\t{\n",
      "\t\t\t\t\t\t\t\"elementtype\": \"bus\",\n",
      "\t\t\t\t\t\t\t\"BUS_I\": 7,\n",
      "\t\t\t\t\t\t\t\"VA\": 0.534813,\n",
      "\t\t\t\t\t\t\t\"VM\": 1.100000,\n",
      "\t\t\t\t\t\t\t\"BUS_NAME\": \" 1\",\n",
      "\t\t\t\t\t\t\t\"VMIN\": 0.900000,\n",
      "\t\t\t\t\t\t\t\"VMAX\": 1.100000,\n",
      "\t\t\t\t\t\t\t\"BASE_KV\": 345.000000,\n",
      "\t\t\t\t\t\t\t\"PD\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"QD\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"PDloss\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"QDloss\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"LAM_P\": 2059.568013,\n",
      "\t\t\t\t\t\t\t\"LAM_Q\": -0.040507,\n",
      "\t\t\t\t\t\t\t\"ngen\": 0,\n",
      "\t\t\t\t\t\t\t\"gen\": [\n",
      "\t\t\t\t\t\t\t]\n",
      "\t\t\t\t\t\t}\n",
      "\t\t\t\t\t],\n",
      "\t\t\t\t\t\"Vm\": 1.100000\n",
      "\t\t\t\t}\n",
      "\t\t\t},\n",
      "\t\t\t{\n",
      "\t\t\t\t\"type\": \"Feature\",\n",
      "\t\t\t\t\"geometry\": {\n",
      "\t\t\t\t\t\"type\": \"Point\",\n",
      "\t\t\t\t\t\"coordinates\": [\n",
      "\t\t\t\t\t\t-98.000873,\n",
      "\t\t\t\t\t\t35.568742\n",
      "\t\t\t\t\t]\n",
      "\t\t\t\t},\n",
      "\t\t\t\t\"properties\": {\n",
      "\t\t\t\t\t\"elementtype\": \"SUBSTATION\",\n",
      "\t\t\t\t\t\"NAME\": \"8\",\n",
      "\t\t\t\t\t\"nbus\": 1,\n",
      "\t\t\t\t\t\"KVlevels\": [\n",
      "\t\t\t\t\t\t345.000000\n",
      "\t\t\t\t\t],\n",
      "\t\t\t\t\t\"bus\": [\n",
      "\t\t\t\t\t\t{\n",
      "\t\t\t\t\t\t\t\"elementtype\": \"bus\",\n",
      "\t\t\t\t\t\t\t\"BUS_I\": 8,\n",
      "\t\t\t\t\t\t\t\"VA\": -1.720022,\n",
      "\t\t\t\t\t\t\t\"VM\": 1.089476,\n",
      "\t\t\t\t\t\t\t\"BUS_NAME\": \" 1\",\n",
      "\t\t\t\t\t\t\t\"VMIN\": 0.900000,\n",
      "\t\t\t\t\t\t\t\"VMAX\": 1.100000,\n",
      "\t\t\t\t\t\t\t\"BASE_KV\": 345.000000,\n",
      "\t\t\t\t\t\t\t\"PD\": 100.000000,\n",
      "\t\t\t\t\t\t\t\"QD\": 35.000000,\n",
      "\t\t\t\t\t\t\t\"PDloss\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"QDloss\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"LAM_P\": 2079.336928,\n",
      "\t\t\t\t\t\t\t\"LAM_Q\": 2.993805,\n",
      "\t\t\t\t\t\t\t\"ngen\": 0,\n",
      "\t\t\t\t\t\t\t\"gen\": [\n",
      "\t\t\t\t\t\t\t]\n",
      "\t\t\t\t\t\t}\n",
      "\t\t\t\t\t],\n",
      "\t\t\t\t\t\"Vm\": 1.089476\n",
      "\t\t\t\t}\n",
      "\t\t\t},\n",
      "\t\t\t{\n",
      "\t\t\t\t\"type\": \"Feature\",\n",
      "\t\t\t\t\"geometry\": {\n",
      "\t\t\t\t\t\"type\": \"Point\",\n",
      "\t\t\t\t\t\"coordinates\": [\n",
      "\t\t\t\t\t\t-97.829863,\n",
      "\t\t\t\t\t\t35.864940\n",
      "\t\t\t\t\t]\n",
      "\t\t\t\t},\n",
      "\t\t\t\t\"properties\": {\n",
      "\t\t\t\t\t\"elementtype\": \"SUBSTATION\",\n",
      "\t\t\t\t\t\"NAME\": \"9\",\n",
      "\t\t\t\t\t\"nbus\": 1,\n",
      "\t\t\t\t\t\"KVlevels\": [\n",
      "\t\t\t\t\t\t345.000000\n",
      "\t\t\t\t\t],\n",
      "\t\t\t\t\t\"bus\": [\n",
      "\t\t\t\t\t\t{\n",
      "\t\t\t\t\t\t\t\"elementtype\": \"bus\",\n",
      "\t\t\t\t\t\t\t\"BUS_I\": 9,\n",
      "\t\t\t\t\t\t\t\"VA\": -0.134848,\n",
      "\t\t\t\t\t\t\t\"VM\": 1.100000,\n",
      "\t\t\t\t\t\t\t\"BUS_NAME\": \" 1\",\n",
      "\t\t\t\t\t\t\t\"VMIN\": 0.900000,\n",
      "\t\t\t\t\t\t\t\"VMAX\": 1.100000,\n",
      "\t\t\t\t\t\t\t\"BASE_KV\": 345.000000,\n",
      "\t\t\t\t\t\t\t\"PD\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"QD\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"PDloss\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"QDloss\": 0.000000,\n",
      "\t\t\t\t\t\t\t\"LAM_P\": 2065.428566,\n",
      "\t\t\t\t\t\t\t\"LAM_Q\": -0.091922,\n",
      "\t\t\t\t\t\t\t\"ngen\": 0,\n",
      "\t\t\t\t\t\t\t\"gen\": [\n",
      "\t\t\t\t\t\t\t]\n",
      "\t\t\t\t\t\t}\n",
      "\t\t\t\t\t],\n",
      "\t\t\t\t\t\"Vm\": 1.100000\n",
      "\t\t\t\t}\n",
      "\t\t\t},\n",
      "\t\t\t{\n",
      "\t\t\t\t\"type\": \"Feature\",\n",
      "\t\t\t\t\"geometry\": {\n",
      "\t\t\t\t\t\"type\": \"LineString\",\n",
      "\t\t\t\t\t\"coordinates\": [\n",
      "\t\t\t\t\t\t[\n",
      "\t\t\t\t\t\t\t-97.508469,\n",
      "\t\t\t\t\t\t\t35.981918\n",
      "\t\t\t\t\t\t],\n",
      "\t\t\t\t\t\t[\n",
      "\t\t\t\t\t\t\t-97.075456,\n",
      "\t\t\t\t\t\t\t35.231918\n",
      "\t\t\t\t\t\t]\n",
      "\t\t\t\t\t]\n",
      "\t\t\t\t},\n",
      "\t\t\t\t\"properties\": {\n",
      "\t\t\t\t\t\"elementtype\": \"Branch\",\n",
      "\t\t\t\t\t\"NAME\": \"1 -- 4\",\n",
      "\t\t\t\t\t\"F_BUS\": 1,\n",
      "\t\t\t\t\t\"T_BUS\": 4,\n",
      "\t\t\t\t\t\"BR_STATUS\": 1,\n",
      "\t\t\t\t\t\"KV\": 345.000000,\n",
      "\t\t\t\t\t\"RATE_A\": 380.000000,\n",
      "\t\t\t\t\t\"PF\": 72.859445,\n",
      "\t\t\t\t\t\"QF\": 6.790095,\n",
      "\t\t\t\t\t\"PT\": -72.855020,\n",
      "\t\t\t\t\t\"QT\": -4.253076\n",
      "\t\t\t\t}\n",
      "\t\t\t},\n",
      "\t\t\t{\n",
      "\t\t\t\t\"type\": \"Feature\",\n",
      "\t\t\t\t\"geometry\": {\n",
      "\t\t\t\t\t\"type\": \"LineString\",\n",
      "\t\t\t\t\t\"coordinates\": [\n",
      "\t\t\t\t\t\t[\n",
      "\t\t\t\t\t\t\t-97.187075,\n",
      "\t\t\t\t\t\t\t35.864940\n",
      "\t\t\t\t\t\t],\n",
      "\t\t\t\t\t\t[\n",
      "\t\t\t\t\t\t\t-97.941482,\n",
      "\t\t\t\t\t\t\t35.231918\n",
      "\t\t\t\t\t\t]\n",
      "\t\t\t\t\t]\n",
      "\t\t\t\t},\n",
      "\t\t\t\t\"properties\": {\n",
      "\t\t\t\t\t\"elementtype\": \"Branch\",\n",
      "\t\t\t\t\t\"NAME\": \"2 -- 7\",\n",
      "\t\t\t\t\t\"F_BUS\": 2,\n",
      "\t\t\t\t\t\"T_BUS\": 7,\n",
      "\t\t\t\t\t\"BR_STATUS\": 1,\n",
      "\t\t\t\t\t\"KV\": 345.000000,\n",
      "\t\t\t\t\t\"RATE_A\": 250.000000,\n",
      "\t\t\t\t\t\"PF\": 114.069299,\n",
      "\t\t\t\t\t\"QF\": -5.126619,\n",
      "\t\t\t\t\t\"PT\": -114.058431,\n",
      "\t\t\t\t\t\"QT\": 11.907578\n",
      "\t\t\t\t}\n",
      "\t\t\t},\n",
      "\t\t\t{\n",
      "\t\t\t\t\"type\": \"Feature\",\n",
      "\t\t\t\t\"geometry\": {\n",
      "\t\t\t\t\t\"type\": \"LineString\",\n",
      "\t\t\t\t\t\"coordinates\": [\n",
      "\t\t\t\t\t\t[\n",
      "\t\t\t\t\t\t\t-97.016065,\n",
      "\t\t\t\t\t\t\t35.568742\n",
      "\t\t\t\t\t\t],\n",
      "\t\t\t\t\t\t[\n",
      "\t\t\t\t\t\t\t-97.829863,\n",
      "\t\t\t\t\t\t\t35.864940\n",
      "\t\t\t\t\t\t]\n",
      "\t\t\t\t\t]\n",
      "\t\t\t\t},\n",
      "\t\t\t\t\"properties\": {\n",
      "\t\t\t\t\t\"elementtype\": \"Branch\",\n",
      "\t\t\t\t\t\"NAME\": \"3 -- 9\",\n",
      "\t\t\t\t\t\"F_BUS\": 3,\n",
      "\t\t\t\t\t\"T_BUS\": 9,\n",
      "\t\t\t\t\t\"BR_STATUS\": 1,\n",
      "\t\t\t\t\t\"KV\": 345.000000,\n",
      "\t\t\t\t\t\"RATE_A\": 300.000000,\n",
      "\t\t\t\t\t\"PF\": 80.210330,\n",
      "\t\t\t\t\t\"QF\": -23.472244,\n",
      "\t\t\t\t\t\"PT\": -80.204414,\n",
      "\t\t\t\t\t\"QT\": 26.926958\n",
      "\t\t\t\t}\n",
      "\t\t\t},\n",
      "\t\t\t{\n",
      "\t\t\t\t\"type\": \"Feature\",\n",
      "\t\t\t\t\"geometry\": {\n",
      "\t\t\t\t\t\"type\": \"LineString\",\n",
      "\t\t\t\t\t\"coordinates\": [\n",
      "\t\t\t\t\t\t[\n",
      "\t\t\t\t\t\t\t-97.075456,\n",
      "\t\t\t\t\t\t\t35.231918\n",
      "\t\t\t\t\t\t],\n",
      "\t\t\t\t\t\t[\n",
      "\t\t\t\t\t\t\t-97.337459,\n",
      "\t\t\t\t\t\t\t35.012072\n",
      "\t\t\t\t\t\t]\n",
      "\t\t\t\t\t]\n",
      "\t\t\t\t},\n",
      "\t\t\t\t\"properties\": {\n",
      "\t\t\t\t\t\"elementtype\": \"Branch\",\n",
      "\t\t\t\t\t\"NAME\": \"4 -- 5\",\n",
      "\t\t\t\t\t\"F_BUS\": 4,\n",
      "\t\t\t\t\t\"T_BUS\": 5,\n",
      "\t\t\t\t\t\"BR_STATUS\": 1,\n",
      "\t\t\t\t\t\"KV\": 345.000000,\n",
      "\t\t\t\t\t\"RATE_A\": 250.000000,\n",
      "\t\t\t\t\t\"PF\": 28.254972,\n",
      "\t\t\t\t\t\"QF\": 9.102475,\n",
      "\t\t\t\t\t\"PT\": -28.156409,\n",
      "\t\t\t\t\t\"QT\": -29.107542\n",
      "\t\t\t\t}\n",
      "\t\t\t},\n",
      "\t\t\t{\n",
      "\t\t\t\t\"type\": \"Feature\",\n",
      "\t\t\t\t\"geometry\": {\n",
      "\t\t\t\t\t\"type\": \"LineString\",\n",
      "\t\t\t\t\t\"coordinates\": [\n",
      "\t\t\t\t\t\t[\n",
      "\t\t\t\t\t\t\t-97.075456,\n",
      "\t\t\t\t\t\t\t35.231918\n",
      "\t\t\t\t\t\t],\n",
      "\t\t\t\t\t\t[\n",
      "\t\t\t\t\t\t\t-97.679479,\n",
      "\t\t\t\t\t\t\t35.012072\n",
      "\t\t\t\t\t\t]\n",
      "\t\t\t\t\t]\n",
      "\t\t\t\t},\n",
      "\t\t\t\t\"properties\": {\n",
      "\t\t\t\t\t\"elementtype\": \"Branch\",\n",
      "\t\t\t\t\t\"NAME\": \"4 -- 6\",\n",
      "\t\t\t\t\t\"F_BUS\": 4,\n",
      "\t\t\t\t\t\"T_BUS\": 6,\n",
      "\t\t\t\t\t\"BR_STATUS\": 1,\n",
      "\t\t\t\t\t\"KV\": 345.000000,\n",
      "\t\t\t\t\t\"RATE_A\": 250.000000,\n",
      "\t\t\t\t\t\"PF\": 44.600049,\n",
      "\t\t\t\t\t\"QF\": -4.849399,\n",
      "\t\t\t\t\t\"PT\": -44.315995,\n",
      "\t\t\t\t\t\"QT\": -12.452026\n",
      "\t\t\t\t}\n",
      "\t\t\t},\n",
      "\t\t\t{\n",
      "\t\t\t\t\"type\": \"Feature\",\n",
      "\t\t\t\t\"geometry\": {\n",
      "\t\t\t\t\t\"type\": \"LineString\",\n",
      "\t\t\t\t\t\"coordinates\": [\n",
      "\t\t\t\t\t\t[\n",
      "\t\t\t\t\t\t\t-97.337459,\n",
      "\t\t\t\t\t\t\t35.012072\n",
      "\t\t\t\t\t\t],\n",
      "\t\t\t\t\t\t[\n",
      "\t\t\t\t\t\t\t-97.941482,\n",
      "\t\t\t\t\t\t\t35.231918\n",
      "\t\t\t\t\t\t]\n",
      "\t\t\t\t\t]\n",
      "\t\t\t\t},\n",
      "\t\t\t\t\"properties\": {\n",
      "\t\t\t\t\t\"elementtype\": \"Branch\",\n",
      "\t\t\t\t\t\"NAME\": \"5 -- 7\",\n",
      "\t\t\t\t\t\"F_BUS\": 5,\n",
      "\t\t\t\t\t\"T_BUS\": 7,\n",
      "\t\t\t\t\t\"BR_STATUS\": 1,\n",
      "\t\t\t\t\t\"KV\": 345.000000,\n",
      "\t\t\t\t\t\"RATE_A\": 250.000000,\n",
      "\t\t\t\t\t\"PF\": -46.843421,\n",
      "\t\t\t\t\t\"QF\": -20.891819,\n",
      "\t\t\t\t\t\"PT\": 47.448707,\n",
      "\t\t\t\t\t\"QT\": -12.401091\n",
      "\t\t\t\t}\n",
      "\t\t\t},\n",
      "\t\t\t{\n",
      "\t\t\t\t\"type\": \"Feature\",\n",
      "\t\t\t\t\"geometry\": {\n",
      "\t\t\t\t\t\"type\": \"LineString\",\n",
      "\t\t\t\t\t\"coordinates\": [\n",
      "\t\t\t\t\t\t[\n",
      "\t\t\t\t\t\t\t-97.679479,\n",
      "\t\t\t\t\t\t\t35.012072\n",
      "\t\t\t\t\t\t],\n",
      "\t\t\t\t\t\t[\n",
      "\t\t\t\t\t\t\t-97.829863,\n",
      "\t\t\t\t\t\t\t35.864940\n",
      "\t\t\t\t\t\t]\n",
      "\t\t\t\t\t]\n",
      "\t\t\t\t},\n",
      "\t\t\t\t\"properties\": {\n",
      "\t\t\t\t\t\"elementtype\": \"Branch\",\n",
      "\t\t\t\t\t\"NAME\": \"6 -- 9\",\n",
      "\t\t\t\t\t\"F_BUS\": 6,\n",
      "\t\t\t\t\t\"T_BUS\": 9,\n",
      "\t\t\t\t\t\"BR_STATUS\": 1,\n",
      "\t\t\t\t\t\"KV\": 345.000000,\n",
      "\t\t\t\t\t\"RATE_A\": 150.000000,\n",
      "\t\t\t\t\t\"PF\": -45.683824,\n",
      "\t\t\t\t\t\"QF\": -17.547371,\n",
      "\t\t\t\t\t\"PT\": 46.377176,\n",
      "\t\t\t\t\t\"QT\": -22.232518\n",
      "\t\t\t\t}\n",
      "\t\t\t},\n",
      "\t\t\t{\n",
      "\t\t\t\t\"type\": \"Feature\",\n",
      "\t\t\t\t\"geometry\": {\n",
      "\t\t\t\t\t\"type\": \"LineString\",\n",
      "\t\t\t\t\t\"coordinates\": [\n",
      "\t\t\t\t\t\t[\n",
      "\t\t\t\t\t\t\t-97.941482,\n",
      "\t\t\t\t\t\t\t35.231918\n",
      "\t\t\t\t\t\t],\n",
      "\t\t\t\t\t\t[\n",
      "\t\t\t\t\t\t\t-98.000873,\n",
      "\t\t\t\t\t\t\t35.568742\n",
      "\t\t\t\t\t\t]\n",
      "\t\t\t\t\t]\n",
      "\t\t\t\t},\n",
      "\t\t\t\t\"properties\": {\n",
      "\t\t\t\t\t\"elementtype\": \"Branch\",\n",
      "\t\t\t\t\t\"NAME\": \"7 -- 8\",\n",
      "\t\t\t\t\t\"F_BUS\": 7,\n",
      "\t\t\t\t\t\"T_BUS\": 8,\n",
      "\t\t\t\t\t\"BR_STATUS\": 1,\n",
      "\t\t\t\t\t\"KV\": 345.000000,\n",
      "\t\t\t\t\t\"RATE_A\": 250.000000,\n",
      "\t\t\t\t\t\"PF\": 66.609412,\n",
      "\t\t\t\t\t\"QF\": 0.489599,\n",
      "\t\t\t\t\t\"PT\": -66.291389,\n",
      "\t\t\t\t\t\"QT\": -15.653093\n",
      "\t\t\t\t}\n",
      "\t\t\t},\n",
      "\t\t\t{\n",
      "\t\t\t\t\"type\": \"Feature\",\n",
      "\t\t\t\t\"geometry\": {\n",
      "\t\t\t\t\t\"type\": \"LineString\",\n",
      "\t\t\t\t\t\"coordinates\": [\n",
      "\t\t\t\t\t\t[\n",
      "\t\t\t\t\t\t\t-98.000873,\n",
      "\t\t\t\t\t\t\t35.568742\n",
      "\t\t\t\t\t\t],\n",
      "\t\t\t\t\t\t[\n",
      "\t\t\t\t\t\t\t-97.829863,\n",
      "\t\t\t\t\t\t\t35.864940\n",
      "\t\t\t\t\t\t]\n",
      "\t\t\t\t\t]\n",
      "\t\t\t\t},\n",
      "\t\t\t\t\"properties\": {\n",
      "\t\t\t\t\t\"elementtype\": \"Branch\",\n",
      "\t\t\t\t\t\"NAME\": \"8 -- 9\",\n",
      "\t\t\t\t\t\"F_BUS\": 8,\n",
      "\t\t\t\t\t\"T_BUS\": 9,\n",
      "\t\t\t\t\t\"BR_STATUS\": 1,\n",
      "\t\t\t\t\t\"KV\": 345.000000,\n",
      "\t\t\t\t\t\"RATE_A\": 150.000000,\n",
      "\t\t\t\t\t\"PF\": -33.708218,\n",
      "\t\t\t\t\t\"QF\": -19.344343,\n",
      "\t\t\t\t\t\"PT\": 33.826963,\n",
      "\t\t\t\t\t\"QT\": -4.698016\n",
      "\t\t\t\t}\n",
      "\t\t\t}\n",
      "\t\t]\n",
      "\t},\n",
      "\t\"summary\": {\n",
      "\t\t\"NBUS\": 9,\n",
      "\t\t\"NGEN\": 3,\n",
      "\t\t\"NGENON\": 3,\n",
      "\t\t\"NLINE\": 9,\n",
      "\t\t\"NLINEON\": 9,\n",
      "\t\t\"NLOAD\": 3,\n",
      "\t\t\"GENCAP\": 920.000000,\n",
      "\t\t\"GENON\": [\n",
      "\t\t\t267.139247,\n",
      "\t\t\t-21.812401\n",
      "\t\t],\n",
      "\t\t\"GENCAPON\": 920.000000,\n",
      "\t\t\"LOAD\": [\n",
      "\t\t\t265.000000,\n",
      "\t\t\t115.000000\n",
      "\t\t],\n",
      "\t\t\"LOADSHED\": [\n",
      "\t\t\t0.000000,\n",
      "\t\t\t0.000000\n",
      "\t\t],\n",
      "\t\t\"SolveRealTime\": 0.013155,\n",
      "\t\t\"SolveCPUTime\": 0.000001,\n",
      "\t}\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "!cat solution.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The output to stdout isn't quite right, and we noteably don't call `exago.finalize()` which is a little concerning, but this seems to work. Should be fine to run SCOPFLOW and more complex applications, and then you can loop over ExaGO runs like this all natively to Python!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MPI Execution Options\n",
    "\n",
    "There are quite a few ways to run MPI backed code, and here are some notes on how I arrived at the above implementation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      " \"argv\": [\n",
      "  \"mpiexec\", \"-n\", \"1\", \"/opt/views/view/bin/python3\",\n",
      "  \"-m\",\n",
      "  \"ipykernel_launcher\",\n",
      "  \"-f\",\n",
      "  \"{connection_file}\"\n",
      " ],\n",
      " \"display_name\": \"ExaGO w/ MPI\",\n",
      " \"language\": \"python\",\n",
      " \"metadata\": {\n",
      "  \"debugger\": true\n",
      " }\n",
      "}"
     ]
    }
   ],
   "source": [
    "!cat /usr/local/share/jupyter/kernels/py311-mpi4py-exago/kernel.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Firstly we execute the ipykernel backend using MPI. This means all kernels are launched within a single rank MPI context, and MPI basics work.\n",
    "\n",
    "To run with more processes, we can either:\n",
    "1. Use `!` in a Python block and just run with `mpiexec -n N` for the number of ranks\n",
    "    - This works, but I/O is clunky\n",
    "    - It becomes cumbersome to code dynamic high-level applications that re-run different sims based on returned output\n",
    "1. Spawn an ipyparallel cluster\n",
    "    - https://ipyparallel.readthedocs.io/en/latest/reference/mpi.html \n",
    "        - you can start the cluster and connect to it, but these docs are a little confusing, as you start many processes\n",
    "        - Would be better to have it all be jupyter native\n",
    "    - https://kb.oakland.edu/uts/HPCJupyterMPI\n",
    "        - Inlcudes a much better Jupyter focused bit of docs around this\n",
    "1. Use a different workflow engine, and find a tool\n",
    "\n",
    "While a workflow tool is a strong consideration for most high level applications, we are going to stick with 2. 1 and 3 would likely be custom to your use-case"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Appendix\n",
    "\n",
    "From https://github.com/mpi4py/mpi4py/issues/15#issuecomment-765658147, had to modify mpi4py init routines by hand - this was all due to a red-herring, but is interesting discussion."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.00s - Debugger warning: It seems that frozen modules are being used, which may\n",
      "0.00s - make the debugger miss breakpoints. Please pass -Xfrozen_modules=off\n",
      "0.00s - to python to disable frozen modules.\n",
      "0.00s - Note: Debugging will proceed. Set PYDEVD_DISABLE_FILE_VALIDATION=1 to disable this validation.\n",
      "Available kernels:\n",
      "  python3               /opt/views/view/share/jupyter/kernels/python3\n",
      "  py311-exago           /usr/local/share/jupyter/kernels/py311-exago\n",
      "  py311-mpi4py-exago    /usr/local/share/jupyter/kernels/py311-mpi4py-exago\n"
     ]
    }
   ],
   "source": [
    "!jupyter kernelspec list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      " \"argv\": [\n",
      "  \"/opt/views/view/bin/python3\",\n",
      "  \"-m\",\n",
      "  \"ipykernel_launcher\",\n",
      "  \"-f\",\n",
      "  \"{connection_file}\"\n",
      " ],\n",
      " \"display_name\": \"ExaGO\",\n",
      " \"language\": \"python\",\n",
      " \"metadata\": {\n",
      "  \"debugger\": true\n",
      " }\n",
      "}"
     ]
    }
   ],
   "source": [
    "!cat /usr/local/share/jupyter/kernels/py311-exago/kernel.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As you can see, we have some extra script in our dockerfile that slightly modifies the installed Jupyter kernel to use mpiexec when launching!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Package                 Version\n",
      "----------------------- ---------\n",
      "aiohttp                 3.8.4\n",
      "aiosignal               1.3.1\n",
      "anyio                   3.6.2\n",
      "argon2-cffi             21.3.0\n",
      "argon2-cffi-bindings    21.2.0\n",
      "asttokens               2.4.0\n",
      "async-timeout           4.0.2\n",
      "attrs                   23.1.0\n",
      "Babel                   2.12.1\n",
      "backcall                0.2.0\n",
      "beautifulsoup4          4.12.2\n",
      "bleach                  6.0.0\n",
      "blinker                 1.6.2\n",
      "certifi                 2023.7.22\n",
      "cffi                    1.15.1\n",
      "charset-normalizer      3.3.0\n",
      "click                   8.1.5\n",
      "colorama                0.4.6\n",
      "comm                    0.1.4\n",
      "config                  0.5.1\n",
      "dataclasses-json        0.5.9\n",
      "debugpy                 1.6.7\n",
      "decorator               5.1.1\n",
      "defusedxml              0.7.1\n",
      "entrypoints             0.4\n",
      "executing               1.2.0\n",
      "fastjsonschema          2.16.3\n",
      "Flask                   2.3.2\n",
      "Flask-Cors              4.0.0\n",
      "frozenlist              1.4.0\n",
      "gevent                  23.7.0\n",
      "greenlet                2.0.2\n",
      "idna                    3.4\n",
      "iniconfig               2.0.0\n",
      "ipykernel               6.23.1\n",
      "ipyparallel             8.4.1\n",
      "ipython                 8.14.0\n",
      "ipython-genutils        0.2.0\n",
      "itsdangerous            2.1.2\n",
      "jedi                    0.18.2\n",
      "jellyfish               0.11.2\n",
      "Jinja2                  3.1.2\n",
      "json5                   0.9.14\n",
      "jsonschema              4.17.3\n",
      "jupyter_client          8.2.0\n",
      "jupyter_core            5.3.0\n",
      "jupyter-server          1.21.0\n",
      "jupyterlab              3.4.8\n",
      "jupyterlab-pygments     0.2.2\n",
      "jupyterlab_server       2.22.1\n",
      "langchain               0.0.233\n",
      "langsmith               0.0.5\n",
      "MarkupSafe              2.1.3\n",
      "marshmallow             3.19.0\n",
      "marshmallow-enum        1.5.1\n",
      "matplotlib-inline       0.1.6\n",
      "mistune                 2.0.5\n",
      "mpi4py                  3.1.4\n",
      "multidict               6.0.4\n",
      "mypy-extensions         1.0.0\n",
      "nbclassic               1.0.0\n",
      "nbclient                0.8.0\n",
      "nbconvert               7.4.0\n",
      "nbformat                5.8.0\n",
      "nest-asyncio            1.5.6\n",
      "notebook                6.5.4\n",
      "notebook_shim           0.2.3\n",
      "numexpr                 2.8.4\n",
      "numpy                   1.25.0\n",
      "openai                  0.27.8\n",
      "openapi-schema-pydantic 1.2.4\n",
      "packaging               23.1\n",
      "pandas                  2.0.2\n",
      "pandocfilters           1.5.0\n",
      "parso                   0.8.3\n",
      "pexpect                 4.8.0\n",
      "pickleshare             0.7.5\n",
      "pip                     23.3.1\n",
      "platformdirs            3.10.0\n",
      "pluggy                  1.0.0\n",
      "prometheus-client       0.17.0\n",
      "prompt-toolkit          3.0.38\n",
      "psutil                  5.9.5\n",
      "psycopg2                2.9.6\n",
      "ptyprocess              0.7.0\n",
      "pure-eval               0.2.2\n",
      "pycparser               2.21\n",
      "pydantic                1.10.11\n",
      "Pygments                2.16.1\n",
      "pyrsistent              0.19.3\n",
      "pytest                  7.3.2\n",
      "python-dateutil         2.8.2\n",
      "pytz                    2023.3\n",
      "PyYAML                  6.0\n",
      "pyzmq                   25.0.2\n",
      "requests                2.31.0\n",
      "Send2Trash              1.8.0\n",
      "setuptools              68.0.0\n",
      "shapely                 2.0.1\n",
      "six                     1.16.0\n",
      "sniffio                 1.3.0\n",
      "soupsieve               2.4.1\n",
      "SQLAlchemy              2.0.18\n",
      "stack-data              0.6.2\n",
      "tenacity                8.2.2\n",
      "terminado               0.17.1\n",
      "tinycss2                1.2.1\n",
      "tornado                 6.3.3\n",
      "tqdm                    4.66.1\n",
      "traitlets               5.9.0\n",
      "typing_extensions       4.7.1\n",
      "typing-inspect          0.9.0\n",
      "tzdata                  2023.3\n",
      "urllib3                 2.0.6\n",
      "us                      3.1.1\n",
      "wcwidth                 0.2.7\n",
      "webencodings            0.5.1\n",
      "websocket-client        1.6.3\n",
      "Werkzeug                2.3.6\n",
      "wheel                   0.41.3\n",
      "yarl                    1.9.2\n",
      "zope.event              4.6\n",
      "zope.interface          5.4.0\n"
     ]
    }
   ],
   "source": [
    "!pip3 list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "That's odd - ExaGO can import and run, but doesn't show up in pip. That's because we installed using Spack, and ExaGO is available directly through the `PYTHONPATH` environment variable.\n",
    "\n",
    "This is configured through `entrypoint.sh`, which is the last thing that happens in the configured `~/.bashrc`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "            fi; \\\n",
      "        fi`'\n",
      "    local lightblue='\\[\\033[1;34m\\]'\n",
      "    local removecolor='\\[\\033[0m\\]'\n",
      "    PS1=\"${userpart} ${lightblue}\\w ${gitbranch}${removecolor}\\$ \"\n",
      "    unset -f __bash_prompt\n",
      "}\n",
      "__bash_prompt\n",
      "export PROMPT_DIRTRIM=4\n",
      "source /entrypoint.sh\n"
     ]
    }
   ],
   "source": [
    "!cat ~/.bashrc | tail -n 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#!/bin/sh\n",
      ". /opt/spack-environment/activate.sh\n",
      "exec \"$@\"\n"
     ]
    }
   ],
   "source": [
    "!cat /entrypoint.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "export SPACK_ENV=/opt/spack-environment;\n",
      "export SPACK_ENV_VIEW=default;\n",
      "alias despacktivate='spack env deactivate';\n",
      "export ACLOCAL_PATH=/opt/views/view/share/aclocal;\n",
      "export CMAKE_PREFIX_PATH=/opt/views/view;\n",
      "export C_INCLUDE_PATH=/opt/views/view/include;\n",
      "export JUPYTERLAB_DIR=/opt/views/view/share/jupyter/lab;\n",
      "export JUPYTER_PATH=/opt/views/view/share/jupyter;\n",
      "export LD_LIBRARY_PATH=/opt/views/view/lib64:/opt/views/view/lib;\n",
      "export LIBRARY_PATH=/opt/views/view/lib;\n",
      "export MANPATH=/opt/views/view/share/man:/opt/views/view/man:;\n",
      "export MPICC=/opt/views/view/bin/mpicc;\n",
      "export MPICXX=/opt/views/view/bin/mpic++;\n",
      "export MPIF77=/opt/views/view/bin/mpif77;\n",
      "export MPIF90=/opt/views/view/bin/mpif90;\n",
      "export PATH=/opt/views/view/bin:/opt/spack/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin;\n",
      "export PETSC_ARCH=\"\";\n",
      "export PETSC_DIR=/opt/views/view;\n",
      "export PKG_CONFIG_PATH=/opt/views/view/lib64/pkgconfig:/opt/views/view/lib/pkgconfig:/opt/views/view/share/pkgconfig;\n",
      "export PYTHONPATH=/opt/views/view/lib/python3.11/site-packages;\n"
     ]
    }
   ],
   "source": [
    "!cat /opt/spack-environment/activate.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/opt/views/view/lib/python3.11/site-packages\n"
     ]
    }
   ],
   "source": [
    "!echo $PYTHONPATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 748\n",
      "drwxr-xr-x 168 root root 12288 Nov  9 01:24 .\n",
      "drwxr-xr-x  39 root root  4096 Nov  9 01:24 ..\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 Babel-2.12.1.dist-info\n",
      "drwxr-xr-x  11 root root  4096 Nov  9 01:24 IPython\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 Jinja2-3.1.2.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 MarkupSafe-2.1.3.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 Pygments-2.16.1.dist-info\n",
      "lrwxrwxrwx   1 root root   139 Nov  9 01:24 README.txt -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/python-3.11.6-ybsi7tlnssr3m4h3eqrzq6tyxdcsr2n7/lib/python3.11/site-packages/README.txt\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 Send2Trash-1.8.0.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 __pycache__\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 _argon2_cffi_bindings\n",
      "lrwxrwxrwx   1 root root   175 Nov  9 01:24 _cffi_backend.cpython-311-x86_64-linux-gnu.so -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-cffi-1.15.1-ns7kiujztzymljhmkwceej66qntnjlcr/lib/python3.11/site-packages/_cffi_backend.cpython-311-x86_64-linux-gnu.so\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 _distutils_hack\n",
      "lrwxrwxrwx   1 root root   158 Nov  9 01:24 _pyrsistent_version.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-pyrsistent-0.19.3-ptruwnt5v52i3udbb6cbvhq5oeqn7dox/lib/python3.11/site-packages/_pyrsistent_version.py\n",
      "drwxr-xr-x   9 root root  4096 Nov  9 01:24 _pytest\n",
      "drwxr-xr-x   7 root root  4096 Nov  9 01:24 anyio\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 anyio-3.6.2.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 argon2\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 argon2_cffi-21.3.0.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 argon2_cffi_bindings-21.2.0.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 asttokens\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 asttokens-2.4.0.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 attr\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 attrs\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 attrs-23.1.0.dist-info\n",
      "drwxr-xr-x   6 root root  4096 Nov  9 01:24 babel\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 backcall\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 backcall-0.2.0.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 beautifulsoup4-4.12.2.dist-info\n",
      "drwxr-xr-x   4 root root  4096 Nov  9 01:24 bleach\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 bleach-6.0.0.dist-info\n",
      "drwxr-xr-x   5 root root  4096 Nov  9 01:24 bs4\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 certifi\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 certifi-2023.7.22.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 cffi\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 cffi-1.15.1.dist-info\n",
      "drwxr-xr-x   4 root root  4096 Nov  9 01:24 charset_normalizer\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 charset_normalizer-3.3.0.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 comm\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 comm-0.1.4.dist-info\n",
      "drwxr-xr-x   6 root root  4096 Nov  9 01:24 dateutil\n",
      "drwxr-xr-x   8 root root  4096 Nov  9 01:24 debugpy\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 debugpy-1.6.7.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 decorator-5.1.1.dist-info\n",
      "lrwxrwxrwx   1 root root   146 Nov  9 01:24 decorator.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-decorator-5.1.1-y6omagu7er3rjtyisfdajw5upayicgxe/lib/python3.11/site-packages/decorator.py\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 defusedxml\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 defusedxml-0.7.1.dist-info\n",
      "lrwxrwxrwx   1 root root   160 Nov  9 01:24 distutils-precedence.pth -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-setuptools-68.0.0-ceuwjz4g6i5jxk4babjsbh6oxmqggbba/lib/python3.11/site-packages/distutils-precedence.pth\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 entrypoints-0.4.dist-info\n",
      "lrwxrwxrwx   1 root root   148 Nov  9 01:24 entrypoints.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-entrypoints-0.4-d7yae57isrvmfonytpxzfwt2zqj3dz3r/lib/python3.11/site-packages/entrypoints.py\n",
      "lrwxrwxrwx   1 root root   164 Nov  9 01:24 exago.cpython-311-x86_64-linux-gnu.so -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/exago-1.6.0-odeio72qx5b7od5yh44oxq52q4otbuz7/lib/python3.11/site-packages/exago.cpython-311-x86_64-linux-gnu.so\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 executing\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 executing-1.2.0.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 fastjsonschema\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 fastjsonschema-2.16.3.dist-info\n",
      "drwxr-xr-x   9 root root  4096 Nov  9 01:24 gevent\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 gevent-23.7.0.dist-info\n",
      "drwxr-xr-x   5 root root  4096 Nov  9 01:24 greenlet\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 greenlet-2.0.2.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 idna\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 idna-3.4.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 iniconfig\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 iniconfig-2.0.0.dist-info\n",
      "drwxr-xr-x   9 root root  4096 Nov  9 01:24 ipykernel\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 ipykernel-6.23.1.dist-info\n",
      "lrwxrwxrwx   1 root root   156 Nov  9 01:24 ipykernel_launcher.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-ipykernel-6.23.1-lkt3rawpdvlxrgc6gzcb7ikhvnfe6mq2/lib/python3.11/site-packages/ipykernel_launcher.py\n",
      "drwxr-xr-x  12 root root  4096 Nov  9 01:24 ipyparallel\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 ipyparallel-8.4.1.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 ipython-8.14.0.dist-info\n",
      "drwxr-xr-x   5 root root  4096 Nov  9 01:24 ipython_genutils\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 ipython_genutils-0.2.0.dist-info\n",
      "drwxr-xr-x   7 root root  4096 Nov  9 01:24 jedi\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 jedi-0.18.2.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 jinja2\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 json5\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 json5-0.9.14.dist-info\n",
      "drwxr-xr-x   6 root root  4096 Nov  9 01:24 jsonschema\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 jsonschema-4.17.3.dist-info\n",
      "lrwxrwxrwx   1 root root   147 Nov  9 01:24 jupyter.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-jupyter-core-5.3.0-if7k3k3yqs5n2xy3ojop3fpnefqktfwy/lib/python3.11/site-packages/jupyter.py\n",
      "drwxr-xr-x   8 root root  4096 Nov  9 01:24 jupyter_client\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 jupyter_client-8.2.0.dist-info\n",
      "drwxr-xr-x   5 root root  4096 Nov  9 01:24 jupyter_core\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 jupyter_core-5.3.0.dist-info\n",
      "drwxr-xr-x  17 root root  4096 Nov  9 01:24 jupyter_server\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 jupyter_server-1.21.0.dist-info\n",
      "drwxr-xr-x   9 root root  4096 Nov  9 01:24 jupyterlab\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 jupyterlab-3.4.8.dist-info\n",
      "drwxr-xr-x   4 root root  4096 Nov  9 01:24 jupyterlab_pygments\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 jupyterlab_pygments-0.2.2.dist-info\n",
      "drwxr-xr-x   5 root root  4096 Nov  9 01:24 jupyterlab_server\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 jupyterlab_server-2.22.1.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 markupsafe\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 matplotlib_inline\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 matplotlib_inline-0.1.6.dist-info\n",
      "drwxr-xr-x   5 root root  4096 Nov  9 01:24 mistune\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 mistune-2.0.5.dist-info\n",
      "drwxr-xr-x   7 root root  4096 Nov  9 01:24 mpi4py\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 mpi4py-3.1.4.dist-info\n",
      "drwxr-xr-x  12 root root  4096 Nov  9 01:24 nbclassic\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 nbclassic-1.0.0.dist-info\n",
      "drwxr-xr-x   4 root root  4096 Nov  9 01:24 nbclient\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 nbclient-0.8.0.dist-info\n",
      "drwxr-xr-x  11 root root  4096 Nov  9 01:24 nbconvert\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 nbconvert-7.4.0.dist-info\n",
      "drwxr-xr-x   8 root root  4096 Nov  9 01:24 nbformat\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 nbformat-5.8.0.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 nest_asyncio-1.5.6.dist-info\n",
      "lrwxrwxrwx   1 root root   152 Nov  9 01:24 nest_asyncio.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-nest-asyncio-1.5.6-uphpwzfjvy2o4gjf7zyscvvb3sjujtuh/lib/python3.11/site-packages/nest_asyncio.py\n",
      "drwxr-xr-x  20 root root  4096 Nov  9 01:24 notebook\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 notebook-6.5.4.dist-info\n",
      "drwxr-xr-x   4 root root  4096 Nov  9 01:24 notebook_shim\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 notebook_shim-0.2.3.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 packaging\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 packaging-23.1.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 pandocfilters-1.5.0.dist-info\n",
      "lrwxrwxrwx   1 root root   154 Nov  9 01:24 pandocfilters.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-pandocfilters-1.5.0-wseldytxe3leiottrhb5ogloooxslfd5/lib/python3.11/site-packages/pandocfilters.py\n",
      "drwxr-xr-x   5 root root  4096 Nov  9 01:24 parso\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 parso-0.8.3.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 pexpect\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 pexpect-4.8.0.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 pickleshare-0.7.5.dist-info\n",
      "lrwxrwxrwx   1 root root   150 Nov  9 01:24 pickleshare.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-pickleshare-0.7.5-wty2ecbqmknacpwe6nwpiyxbq5oqhcza/lib/python3.11/site-packages/pickleshare.py\n",
      "drwxr-xr-x   5 root root  4096 Nov  9 01:24 pkg_resources\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 platformdirs\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 platformdirs-3.10.0.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 pluggy\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 pluggy-1.0.0.dist-info\n",
      "drwxr-xr-x   6 root root  4096 Nov  9 01:24 prometheus_client\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 prometheus_client-0.17.0.dist-info\n",
      "drwxr-xr-x  18 root root  4096 Nov  9 01:24 prompt_toolkit\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 prompt_toolkit-3.0.38.dist-info\n",
      "drwxr-xr-x   4 root root  4096 Nov  9 01:24 psutil\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 psutil-5.9.5.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 ptyprocess\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 ptyprocess-0.7.0.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 pure_eval\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 pure_eval-0.2.2.dist-info\n",
      "lrwxrwxrwx   1 root root   176 Nov  9 01:24 pvectorc.cpython-311-x86_64-linux-gnu.so -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-pyrsistent-0.19.3-ptruwnt5v52i3udbb6cbvhq5oeqn7dox/lib/python3.11/site-packages/pvectorc.cpython-311-x86_64-linux-gnu.so\n",
      "lrwxrwxrwx   1 root root   136 Nov  9 01:24 py.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-pytest-7.3.2-arsygg5grgdxn2ydomwdcv6qg66wbgff/lib/python3.11/site-packages/py.py\n",
      "drwxr-xr-x   4 root root  4096 Nov  9 01:24 pycparser\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 pycparser-2.21.dist-info\n",
      "drwxr-xr-x   7 root root  4096 Nov  9 01:24 pygments\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 pyrsistent\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 pyrsistent-0.19.3.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 pytest\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 pytest-7.3.2.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 python_dateutil-2.8.2.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 pyzmq-25.0.2.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 requests\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 requests-2.31.0.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 send2trash\n",
      "drwxr-xr-x   8 root root  4096 Nov  9 01:24 setuptools\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 setuptools-68.0.0.dist-info\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 six-1.16.0.dist-info\n",
      "lrwxrwxrwx   1 root root   135 Nov  9 01:24 six.py -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-six-1.16.0-nbnfdrflk6rrh3fu3pengfnlmslekoqs/lib/python3.11/site-packages/six.py\n",
      "drwxr-xr-x   4 root root  4096 Nov  9 01:24 sniffio\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 sniffio-1.3.0.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 soupsieve\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 soupsieve-2.4.1.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 stack_data\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 stack_data-0.6.2.dist-info\n",
      "drwxr-xr-x   5 root root  4096 Nov  9 01:24 terminado\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 terminado-0.17.1.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 tinycss2\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 tinycss2-1.2.1.dist-info\n",
      "drwxr-xr-x   5 root root  4096 Nov  9 01:24 tornado\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 tornado-6.3.3.dist-info\n",
      "drwxr-xr-x   4 root root  4096 Nov  9 01:24 tqdm\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 tqdm-4.66.1.dist-info\n",
      "drwxr-xr-x   6 root root  4096 Nov  9 01:24 traitlets\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 traitlets-5.9.0.dist-info\n",
      "drwxr-xr-x   5 root root  4096 Nov  9 01:24 urllib3\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 urllib3-2.0.6.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 wcwidth\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 wcwidth-0.2.7.dist-info\n",
      "drwxr-xr-x   3 root root  4096 Nov  9 01:24 webencodings\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 webencodings-0.5.1.dist-info\n",
      "drwxr-xr-x   4 root root  4096 Nov  9 01:24 websocket\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 websocket_client-1.6.3.dist-info\n",
      "drwxr-xr-x  13 root root  4096 Nov  9 01:24 zmq\n",
      "drwxr-xr-x   4 root root  4096 Nov  9 01:24 zope\n",
      "lrwxrwxrwx   1 root root   164 Nov  9 01:24 zope.event-4.6-py3.11-nspkg.pth -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-zope-event-4.6-mlah5dql76d33hl34zeysun66ik2nmjk/lib/python3.11/site-packages/zope.event-4.6-py3.11-nspkg.pth\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 zope.event-4.6.dist-info\n",
      "lrwxrwxrwx   1 root root   176 Nov  9 01:24 zope.interface-5.4.0-py3.11-nspkg.pth -> /opt/software/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/py-zope-interface-5.4.0-6jsmi36ohggmpihuchqcd3r3s2eiqaan/lib/python3.11/site-packages/zope.interface-5.4.0-py3.11-nspkg.pth\n",
      "drwxr-xr-x   2 root root  4096 Nov  9 01:24 zope.interface-5.4.0.dist-info\n"
     ]
    }
   ],
   "source": [
    "!ls -al /opt/views/view/lib/python3.11/site-packages"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here you can see the full list of spack configured python packages, including exago."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Some info about this container"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PRETTY_NAME=\"Debian GNU/Linux 12 (bookworm)\"\n",
      "NAME=\"Debian GNU/Linux\"\n",
      "VERSION_ID=\"12\"\n",
      "VERSION=\"12 (bookworm)\"\n",
      "VERSION_CODENAME=bookworm\n",
      "ID=debian\n",
      "HOME_URL=\"https://www.debian.org/\"\n",
      "SUPPORT_URL=\"https://www.debian.org/support\"\n",
      "BUG_REPORT_URL=\"https://bugs.debian.org/\"\n"
     ]
    }
   ],
   "source": [
    "!cat /etc/os-release"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ExaGO w/ MPI",
   "language": "python",
   "name": "py311-mpi4py-exago"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}