From 357d8e1752abce3f99cbcf500cd68c31abc26568 Mon Sep 17 00:00:00 2001 From: Martin Hirzel Date: Fri, 12 Aug 2022 16:30:24 -0400 Subject: [PATCH] =?UTF-8?q?In=2006=5Fmultiobj,=20trying=20to=20exit=20more?= =?UTF-8?q?=20cleanly,=20and=20using=20symmetric=20disp=E2=80=A6=20(#1164)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * In 06_multiobj, trying to exit more cleanly, and using symmetric disparate impact. In 05_bias, minor wording tweaks. Signed-off-by: Martin Hirzel * Trying sys.exit(0) to exit 06_multobj "successfully" during tests. Signed-off-by: Martin Hirzel * Using if-statements to skip instead of exiting. Signed-off-by: Martin Hirzel Signed-off-by: Martin Hirzel --- .github/workflows/build.yml | 8 +- examples/kdd22/05_bias.ipynb | 554 ++++++----- examples/kdd22/06_multobj.ipynb | 1641 ++++++++++++++++++------------- 3 files changed, 1251 insertions(+), 952 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 23671539f..305965451 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -320,7 +320,7 @@ jobs: python-version: 3.8 setup-target: '.[tutorial,test]' test-case: 'test/test_notebooks.py' - nbexcludes: '06_multobj.ipynb' + # nbexcludes: '06_multobj.ipynb' steps: - uses: actions/checkout@v2 @@ -377,17 +377,17 @@ jobs: python-version: 3.7 setup-target: '.[tutorial,test]' test-case: 'test/test_notebooks.py' - nbexcludes: '06_multobj.ipynb' + # nbexcludes: '06_multobj.ipynb' - dir: 'examples/kdd22' python-version: 3.8 setup-target: '.[tutorial,test]' test-case: 'test/test_notebooks.py' - nbexcludes: '06_multobj.ipynb' + # nbexcludes: '06_multobj.ipynb' - dir: 'examples/kdd22' python-version: 3.9 setup-target: '.[tutorial,test]' test-case: 'test/test_notebooks.py' - nbexcludes: '06_multobj.ipynb' + # nbexcludes: '06_multobj.ipynb' steps: - uses: actions/checkout@v2 diff --git a/examples/kdd22/05_bias.ipynb b/examples/kdd22/05_bias.ipynb index e5aed167c..8d9f73bdf 100644 --- a/examples/kdd22/05_bias.ipynb +++ b/examples/kdd22/05_bias.ipynb @@ -20,7 +20,7 @@ "distributed with Lale, and shows how to use them either with manual\n", "machine learning or with AutoML.\n", "\n", - "This notebook has following sections:\n", + "This notebook has the following sections:\n", "\n", "- [5.1 Datasets and Fairness Information](#5.1-Dataset-and-Fairness-Information)\n", "- [5.2 Metrics](#5.2-Metrics)\n", @@ -834,8 +834,8 @@ "privileged group.\n", "This is unfair; for instance, it falls short of the 80% rule in US Law.\n", "We can train a model that has less bias than the dataset.\n", - "For instance, a `DummyClassifier` ignores its input and always returns\n", - "the majority label." + "For instance, a [DummyClassifier](https://lale.readthedocs.io/en/latest/modules/lale.lib.sklearn.dummy_classifier.html#lale.lib.sklearn.dummy_classifier.DummyClassifier)\n", + "ignores its input and always returns the majority label." ] }, { @@ -866,7 +866,7 @@ "The `DummyClassifier` is trivially perfectly fair: irrespective of\n", "whether or not an individual is a member of the privileged group, the\n", "predicted label is always `'good'`.\n", - "On the other hand it is poor at predicting the ground truth labels.\n", + "On the other hand, it is poor at predicting the ground truth labels.\n", "To better evaluate `DummyClassifier` as well as other models, we\n", "create a few more metrics directly from scikit-learn." ] @@ -966,8 +966,8 @@ " \n", " \n", " 0\n", - " 0.01\n", - " 0.06\n", + " 0.00\n", + " 0.02\n", " 1.00\n", " 0.50\n", " 0.00\n", @@ -977,7 +977,7 @@ " \n", " 1\n", " 0.00\n", - " 0.06\n", + " 0.02\n", " 1.00\n", " 0.50\n", " 0.00\n", @@ -987,7 +987,7 @@ " \n", " 2\n", " 0.00\n", - " 0.04\n", + " 0.02\n", " 1.00\n", " 0.50\n", " 0.00\n", @@ -997,7 +997,7 @@ " \n", " mean\n", " 0.00\n", - " 0.05\n", + " 0.02\n", " 1.00\n", " 0.50\n", " 0.00\n", @@ -1007,7 +1007,7 @@ " \n", " std\n", " 0.00\n", - " 0.01\n", + " 0.00\n", " 0.00\n", " 0.00\n", " 0.00\n", @@ -1020,11 +1020,11 @@ ], "text/plain": [ " fit_time score_time test_disparate impact test_balanced accuracy \\\n", - "0 0.01 0.06 1.00 0.50 \n", - "1 0.00 0.06 1.00 0.50 \n", - "2 0.00 0.04 1.00 0.50 \n", - "mean 0.00 0.05 1.00 0.50 \n", - "std 0.00 0.01 0.00 0.00 \n", + "0 0.00 0.02 1.00 0.50 \n", + "1 0.00 0.02 1.00 0.50 \n", + "2 0.00 0.02 1.00 0.50 \n", + "mean 0.00 0.02 1.00 0.50 \n", + "std 0.00 0.00 0.00 0.00 \n", "\n", " test_recall (unfavorable) test_recall (favorable) test_accuracy \n", "0 0.00 1.00 0.70 \n", @@ -1068,7 +1068,7 @@ "\n", "How well does a simple pipeline perform on this dataset if it does not\n", "use any algorithmic bias mitigators?\n", - "First, let's create a preprocessing pipeline to ensure all columns are\n", + "First, let's create a preprocessing sub-pipeline to ensure all columns are\n", "numeric." ] }, @@ -1083,72 +1083,79 @@ "\n", "\n", - "\n", "\n", - "\n", - "\n", + "\n", + "\n", "cluster:(root)\n", "\n", - "\n", + "\n", "\n", "\n", "\n", - "project_0\n", + "\n", + "project_0\n", "\n", - "\n", - "Project\n", + "\n", + "Project\n", "\n", "\n", "\n", "\n", - "concat_features\n", + "\n", + "concat_features\n", "\n", - "\n", - "Concat-\n", - "Features\n", + "\n", + "Concat-\n", + "Features\n", "\n", "\n", "\n", "\n", - "project_0->concat_features\n", - "\n", - "\n", + "\n", + "project_0->concat_features\n", + "\n", + "\n", "\n", "\n", - "project_1\n", + "\n", + "project_1\n", "\n", - "\n", - "Project\n", + "\n", + "Project\n", "\n", "\n", "\n", "\n", - "one_hot_encoder\n", + "\n", + "one_hot_encoder\n", "\n", - "\n", - "One-\n", - "Hot-\n", - "Encoder\n", + "\n", + "One-\n", + "Hot-\n", + "Encoder\n", "\n", "\n", "\n", "\n", - "project_1->one_hot_encoder\n", - "\n", - "\n", + "\n", + "project_1->one_hot_encoder\n", + "\n", + "\n", "\n", "\n", - "one_hot_encoder->concat_features\n", - "\n", - "\n", + "\n", + "one_hot_encoder->concat_features\n", + "\n", + "\n", "\n", "\n", "\n" ], "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -1215,39 +1222,39 @@ " \n", "
\n", " 0\n", - " 0.60\n", - " 0.13\n", - " 0.85\n", - " 0.69\n", - " 0.46\n", - " 0.92\n", + " 0.34\n", + " 0.07\n", + " 0.81\n", + " 0.68\n", + " 0.44\n", + " 0.93\n", " 0.78\n", "
\n", "
\n", " 1\n", - " 0.46\n", - " 0.19\n", - " 0.82\n", - " 0.67\n", - " 0.38\n", - " 0.95\n", - " 0.78\n", + " 0.24\n", + " 0.08\n", + " 0.68\n", + " 0.65\n", + " 0.36\n", + " 0.94\n", + " 0.77\n", "
\n", "
\n", " 2\n", - " 0.48\n", + " 0.32\n", " 0.13\n", - " 0.79\n", - " 0.64\n", - " 0.37\n", - " 0.91\n", - " 0.75\n", + " 0.77\n", + " 0.66\n", + " 0.40\n", + " 0.93\n", + " 0.77\n", "
\n", "
\n", " mean\n", - " 0.51\n", - " 0.15\n", - " 0.82\n", + " 0.30\n", + " 0.09\n", + " 0.75\n", " 0.67\n", " 0.40\n", " 0.93\n", @@ -1255,13 +1262,13 @@ "
\n", "
\n", " std\n", - " 0.08\n", - " 0.04\n", - " 0.03\n", - " 0.02\n", " 0.05\n", + " 0.03\n", + " 0.06\n", " 0.02\n", - " 0.02\n", + " 0.04\n", + " 0.01\n", + " 0.01\n", "
\n", " \n", "\n", @@ -1269,18 +1276,18 @@ ], "text/plain": [ " fit_time score_time test_disparate impact test_balanced accuracy \\\n", - "0 0.60 0.13 0.85 0.69 \n", - "1 0.46 0.19 0.82 0.67 \n", - "2 0.48 0.13 0.79 0.64 \n", - "mean 0.51 0.15 0.82 0.67 \n", - "std 0.08 0.04 0.03 0.02 \n", + "0 0.34 0.07 0.81 0.68 \n", + "1 0.24 0.08 0.68 0.65 \n", + "2 0.32 0.13 0.77 0.66 \n", + "mean 0.30 0.09 0.75 0.67 \n", + "std 0.05 0.03 0.06 0.02 \n", "\n", " test_recall (unfavorable) test_recall (favorable) test_accuracy \n", - "0 0.46 0.92 0.78 \n", - "1 0.38 0.95 0.78 \n", - "2 0.37 0.91 0.75 \n", + "0 0.44 0.93 0.78 \n", + "1 0.36 0.94 0.77 \n", + "2 0.40 0.93 0.77 \n", "mean 0.40 0.93 0.77 \n", - "std 0.05 0.02 0.02 " + "std 0.04 0.01 0.01 " ] }, "execution_count": 14, @@ -1310,8 +1317,8 @@ "\n", "Lale provides several [bias mitigators](https://lale.readthedocs.io/en/latest/modules/lale.lib.aif360.html#pre-estimator-mitigation-operators) from AIF360.\n", "This notebook demonstrates three representative ones.\n", - "The first one, `GerryFairClassifier`, is an in-estimator mitigator,\n", - "meaning that it has its own built-in estimator.\n", + "The first one, [GerryFairClassifier](https://lale.readthedocs.io/en/latest/modules/lale.lib.aif360.gerry_fair_classifier.html#lale.lib.aif360.gerry_fair_classifier.GerryFairClassifier),\n", + "is an in-estimator mitigator, meaning that it has its own built-in estimator.\n", "We configure it with the previously defined `fairness_info` and\n", "`prefix`, which is a data preparation sub-pipeline." ] @@ -1354,8 +1361,8 @@ " \n", "
\n", " 0\n", - " 1.31\n", - " 0.39\n", + " 1.18\n", + " 0.32\n", " 0.76\n", " 0.74\n", " 0.71\n", @@ -1364,7 +1371,7 @@ "
\n", "
\n", " 1\n", - " 0.81\n", + " 2.00\n", " 0.33\n", " 0.63\n", " 0.65\n", @@ -1374,8 +1381,8 @@ "
\n", "
\n", " 2\n", - " 0.81\n", - " 0.31\n", + " 0.95\n", + " 0.44\n", " 0.64\n", " 0.70\n", " 0.52\n", @@ -1384,8 +1391,8 @@ "
\n", "
\n", " mean\n", - " 0.97\n", - " 0.34\n", + " 1.38\n", + " 0.36\n", " 0.68\n", " 0.70\n", " 0.54\n", @@ -1394,8 +1401,8 @@ "
\n", "
\n", " std\n", - " 0.29\n", - " 0.04\n", + " 0.55\n", + " 0.07\n", " 0.07\n", " 0.04\n", " 0.16\n", @@ -1408,11 +1415,11 @@ ], "text/plain": [ " fit_time score_time test_disparate impact test_balanced accuracy \\\n", - "0 1.31 0.39 0.76 0.74 \n", - "1 0.81 0.33 0.63 0.65 \n", - "2 0.81 0.31 0.64 0.70 \n", - "mean 0.97 0.34 0.68 0.70 \n", - "std 0.29 0.04 0.07 0.04 \n", + "0 1.18 0.32 0.76 0.74 \n", + "1 2.00 0.33 0.63 0.65 \n", + "2 0.95 0.44 0.64 0.70 \n", + "mean 1.38 0.36 0.68 0.70 \n", + "std 0.55 0.07 0.07 0.04 \n", "\n", " test_recall (unfavorable) test_recall (favorable) test_accuracy \n", "0 0.71 0.76 0.75 \n", @@ -1437,9 +1444,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The second example, `DisparateImpactRemover`, is a pre-estimator\n", - "mitigator, meaning that it transforms the data to make it more fair\n", - "before it is piped to an estimator." + "The second example, [DisparateImpactRemover](https://lale.readthedocs.io/en/latest/modules/lale.lib.aif360.disparate_impact_remover.html#lale.lib.aif360.disparate_impact_remover.DisparateImpactRemover),\n", + "is a pre-estimator mitigator, meaning that it transforms the data to\n", + "make it more fair before it is piped to an estimator." ] }, { @@ -1480,53 +1487,53 @@ " \n", "
\n", " 0\n", - " 0.88\n", - " 0.46\n", - " 0.94\n", - " 0.66\n", + " 0.59\n", + " 0.28\n", + " 0.99\n", + " 0.67\n", " 0.41\n", - " 0.91\n", - " 0.76\n", + " 0.92\n", + " 0.77\n", "
\n", "
\n", " 1\n", - " 0.89\n", - " 0.40\n", - " 0.93\n", - " 0.67\n", - " 0.37\n", - " 0.97\n", - " 0.79\n", + " 0.49\n", + " 0.24\n", + " 0.92\n", + " 0.64\n", + " 0.35\n", + " 0.94\n", + " 0.76\n", "
\n", "
\n", " 2\n", - " 0.83\n", - " 0.45\n", - " 0.91\n", + " 0.53\n", + " 0.23\n", + " 1.02\n", " 0.65\n", - " 0.38\n", - " 0.93\n", - " 0.76\n", + " 0.37\n", + " 0.94\n", + " 0.77\n", "
\n", "
\n", " mean\n", - " 0.87\n", - " 0.44\n", - " 0.93\n", - " 0.66\n", - " 0.39\n", + " 0.54\n", + " 0.25\n", + " 0.97\n", + " 0.65\n", + " 0.38\n", " 0.93\n", - " 0.77\n", + " 0.76\n", "
\n", "
\n", " std\n", + " 0.05\n", " 0.03\n", - " 0.03\n", - " 0.02\n", + " 0.05\n", " 0.01\n", - " 0.02\n", " 0.03\n", " 0.01\n", + " 0.00\n", "
\n", " \n", "\n", @@ -1534,18 +1541,18 @@ ], "text/plain": [ " fit_time score_time test_disparate impact test_balanced accuracy \\\n", - "0 0.88 0.46 0.94 0.66 \n", - "1 0.89 0.40 0.93 0.67 \n", - "2 0.83 0.45 0.91 0.65 \n", - "mean 0.87 0.44 0.93 0.66 \n", - "std 0.03 0.03 0.02 0.01 \n", + "0 0.59 0.28 0.99 0.67 \n", + "1 0.49 0.24 0.92 0.64 \n", + "2 0.53 0.23 1.02 0.65 \n", + "mean 0.54 0.25 0.97 0.65 \n", + "std 0.05 0.03 0.05 0.01 \n", "\n", " test_recall (unfavorable) test_recall (favorable) test_accuracy \n", - "0 0.41 0.91 0.76 \n", - "1 0.37 0.97 0.79 \n", - "2 0.38 0.93 0.76 \n", - "mean 0.39 0.93 0.77 \n", - "std 0.02 0.03 0.01 " + "0 0.41 0.92 0.77 \n", + "1 0.35 0.94 0.76 \n", + "2 0.37 0.94 0.77 \n", + "mean 0.38 0.93 0.76 \n", + "std 0.03 0.01 0.00 " ] }, "execution_count": 16, @@ -1563,8 +1570,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The third example, `CalibratedEqOddsPostprocessing`, is a\n", - "post-estimator mitigator, meaning that it transforms the predictions\n", + "The third example, [CalibratedEqOddsPostprocessing](https://lale.readthedocs.io/en/latest/modules/lale.lib.aif360.calibrated_eq_odds_postprocessing.html#lale.lib.aif360.calibrated_eq_odds_postprocessing.CalibratedEqOddsPostprocessing),\n", + "is a post-estimator mitigator, meaning that it transforms the predictions\n", "after they are returned from an estimator." ] }, @@ -1606,53 +1613,53 @@ " \n", "
\n", " 0\n", - " 0.83\n", - " 0.45\n", - " 0.89\n", - " 0.60\n", - " 0.52\n", - " 0.68\n", + " 0.54\n", + " 0.21\n", + " 0.91\n", " 0.63\n", + " 0.58\n", + " 0.67\n", + " 0.64\n", "
\n", "
\n", " 1\n", - " 0.72\n", - " 0.36\n", - " 0.91\n", + " 0.46\n", + " 0.29\n", + " 0.88\n", " 0.62\n", " 0.56\n", - " 0.67\n", - " 0.64\n", + " 0.68\n", + " 0.65\n", "
\n", "
\n", " 2\n", - " 0.96\n", - " 0.30\n", - " 0.88\n", - " 0.61\n", - " 0.55\n", - " 0.67\n", - " 0.64\n", + " 0.39\n", + " 0.21\n", + " 0.84\n", + " 0.60\n", + " 0.52\n", + " 0.68\n", + " 0.63\n", "
\n", "
\n", " mean\n", - " 0.84\n", - " 0.37\n", - " 0.89\n", - " 0.61\n", - " 0.54\n", + " 0.46\n", + " 0.24\n", + " 0.88\n", + " 0.62\n", + " 0.55\n", " 0.68\n", " 0.64\n", "
\n", "
\n", " std\n", - " 0.12\n", - " 0.07\n", + " 0.08\n", + " 0.05\n", + " 0.04\n", + " 0.01\n", + " 0.03\n", " 0.01\n", " 0.01\n", - " 0.02\n", - " 0.00\n", - " 0.00\n", "
\n", " \n", "\n", @@ -1660,18 +1667,18 @@ ], "text/plain": [ " fit_time score_time test_disparate impact test_balanced accuracy \\\n", - "0 0.83 0.45 0.89 0.60 \n", - "1 0.72 0.36 0.91 0.62 \n", - "2 0.96 0.30 0.88 0.61 \n", - "mean 0.84 0.37 0.89 0.61 \n", - "std 0.12 0.07 0.01 0.01 \n", + "0 0.54 0.21 0.91 0.63 \n", + "1 0.46 0.29 0.88 0.62 \n", + "2 0.39 0.21 0.84 0.60 \n", + "mean 0.46 0.24 0.88 0.62 \n", + "std 0.08 0.05 0.04 0.01 \n", "\n", " test_recall (unfavorable) test_recall (favorable) test_accuracy \n", - "0 0.52 0.68 0.63 \n", - "1 0.56 0.67 0.64 \n", - "2 0.55 0.67 0.64 \n", - "mean 0.54 0.68 0.64 \n", - "std 0.02 0.00 0.00 " + "0 0.58 0.67 0.64 \n", + "1 0.56 0.68 0.65 \n", + "2 0.52 0.68 0.63 \n", + "mean 0.55 0.68 0.64 \n", + "std 0.03 0.01 0.01 " ] }, "execution_count": 17, @@ -1736,7 +1743,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "100%|████████| 10/10 [00:41<00:00, 4.19s/trial, best loss: -0.7634963346619189]\n" + "100%|████████| 10/10 [00:28<00:00, 2.88s/trial, best loss: -0.7595250541065992]\n" ] } ], @@ -1768,97 +1775,108 @@ "\n", "\n", - "\n", "\n", - "\n", + "\n", "\n", "cluster:(root)\n", - "\n", - "cluster:(root)\n", - "\n", - "\n", - "PostMit\n", + "\n", + "\n", + "cluster:(root)\n", + "\n", + "\n", + "PostMit\n", "\n", "\n", "\n", - "cluster:pipeline\n", + "\n", + "cluster:pipeline\n", "\n", - "\n", + "\n", "\n", "\n", "\n", "\n", - "project\n", + "\n", + "project\n", "\n", - "\n", - "Project\n", + "\n", + "Project\n", "\n", "\n", "\n", "\n", - "concat_features\n", + "\n", + "concat_features\n", "\n", - "\n", - "Concat-\n", - "Features\n", + "\n", + "Concat-\n", + "Features\n", "\n", "\n", "\n", "\n", - "project->concat_features\n", - "\n", - "\n", + "\n", + "project->concat_features\n", + "\n", + "\n", "\n", "\n", - "project_0\n", + "\n", + "project_0\n", "\n", - "\n", - "Project\n", + "\n", + "Project\n", "\n", "\n", "\n", "\n", - "one_hot_encoder\n", + "\n", + "one_hot_encoder\n", "\n", - "\n", - "One-\n", - "Hot-\n", - "Encoder\n", + "\n", + "One-\n", + "Hot-\n", + "Encoder\n", "\n", "\n", "\n", "\n", - "project_0->one_hot_encoder\n", - "\n", - "\n", + "\n", + "project_0->one_hot_encoder\n", + "\n", + "\n", "\n", "\n", - "one_hot_encoder->concat_features\n", - "\n", - "\n", + "\n", + "one_hot_encoder->concat_features\n", + "\n", + "\n", "\n", "\n", - "random_forest_classifier\n", - "\n", - "\n", - "Random-\n", - "Forest-\n", - "Classifier\n", + "\n", + "random_forest_classifier\n", + "\n", + "\n", + "Random-\n", + "Forest-\n", + "Classifier\n", "\n", "\n", "\n", "\n", - "concat_features->random_forest_classifier\n", - "\n", - "\n", + "\n", + "concat_features->random_forest_classifier\n", + "\n", + "\n", "\n", "\n", "\n" ], "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -1897,53 +1915,53 @@ " \n", "
\n", " 0\n", - " 0.99\n", - " 0.45\n", - " 0.96\n", - " 0.59\n", - " 0.50\n", - " 0.68\n", - " 0.63\n", + " 0.30\n", + " 0.18\n", + " 1.00\n", + " 0.51\n", + " 0.31\n", + " 0.72\n", + " 0.60\n", "
\n", "
\n", " 1\n", - " 0.80\n", - " 0.42\n", - " 1.02\n", - " 0.55\n", - " 0.40\n", - " 0.70\n", - " 0.61\n", + " 0.38\n", + " 0.17\n", + " 1.00\n", + " 0.52\n", + " 0.32\n", + " 0.73\n", + " 0.60\n", "
\n", "
\n", " 2\n", - " 0.71\n", - " 0.37\n", - " 1.07\n", - " 0.59\n", - " 0.48\n", - " 0.69\n", - " 0.63\n", + " 0.27\n", + " 0.16\n", + " 1.00\n", + " 0.52\n", + " 0.32\n", + " 0.72\n", + " 0.60\n", "
\n", "
\n", " mean\n", - " 0.84\n", - " 0.41\n", - " 1.02\n", - " 0.58\n", - " 0.46\n", - " 0.69\n", - " 0.62\n", + " 0.32\n", + " 0.17\n", + " 1.00\n", + " 0.52\n", + " 0.32\n", + " 0.72\n", + " 0.60\n", "
\n", "
\n", " std\n", - " 0.14\n", - " 0.04\n", - " 0.05\n", - " 0.02\n", - " 0.05\n", + " 0.06\n", " 0.01\n", + " 0.00\n", + " 0.00\n", " 0.01\n", + " 0.00\n", + " 0.00\n", "
\n", " \n", "\n", @@ -1951,18 +1969,18 @@ ], "text/plain": [ " fit_time score_time test_disparate impact test_balanced accuracy \\\n", - "0 0.99 0.45 0.96 0.59 \n", - "1 0.80 0.42 1.02 0.55 \n", - "2 0.71 0.37 1.07 0.59 \n", - "mean 0.84 0.41 1.02 0.58 \n", - "std 0.14 0.04 0.05 0.02 \n", + "0 0.30 0.18 1.00 0.51 \n", + "1 0.38 0.17 1.00 0.52 \n", + "2 0.27 0.16 1.00 0.52 \n", + "mean 0.32 0.17 1.00 0.52 \n", + "std 0.06 0.01 0.00 0.00 \n", "\n", " test_recall (unfavorable) test_recall (favorable) test_accuracy \n", - "0 0.50 0.68 0.63 \n", - "1 0.40 0.70 0.61 \n", - "2 0.48 0.69 0.63 \n", - "mean 0.46 0.69 0.62 \n", - "std 0.05 0.01 0.01 " + "0 0.31 0.72 0.60 \n", + "1 0.32 0.73 0.60 \n", + "2 0.32 0.72 0.60 \n", + "mean 0.32 0.72 0.60 \n", + "std 0.01 0.00 0.00 " ] }, "execution_count": 20, @@ -2025,7 +2043,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.3" + "version": "3.7.13" } }, "nbformat": 4, diff --git a/examples/kdd22/06_multobj.ipynb b/examples/kdd22/06_multobj.ipynb index 48ac1e708..2b95cf054 100644 --- a/examples/kdd22/06_multobj.ipynb +++ b/examples/kdd22/06_multobj.ipynb @@ -8,7 +8,11 @@ "\n", "# 6. Multi-objective AutoML with Lale\n", "\n", - "In this notebook, we will present how we can perform a multi-objective refinement of a pipeline found by AutoML utilizing a multi-objective optimizer. First, we will find a pipeline via single-objective AutoML. Then we will optimize the final estimator in the pipeline with a multi-objective optimizer to get a Pareto-front of pipelines.\n", + "This notebook presents how we can perform a multi-objective refinement of a pipeline found by AutoML utilizing a multi-objective optimizer.\n", + "First, we will find a pipeline via single-objective AutoML.\n", + "Then we will optimize the final estimator in the pipeline with a multi-objective optimizer to get a Pareto-front of pipelines.\n", + "\n", + "This notebook has the following sections:\n", "\n", "- [6.1 Pre-requisites](#6.1-Pre-requisites)\n", "- [6.2 Dataset](#6.1-Dataset)\n", @@ -31,7 +35,12 @@ "pip install git+https://github.com/IBM/lale-gpl.git@master\n", "\n", "```\n", - "- `Platypus-opt==1.0.4`: This package provides an implementation of the multi-objective **NSGA2** algorithm." + "- `Platypus-opt==1.0.4`: This package provides an implementation of the multi-objective **NSGA2** algorithm.\n", + "\n", + "```\n", + "pip install Platypus-opt==1.0.4\n", + "\n", + "```" ] }, { @@ -79,7 +88,7 @@ "text/html": [ "\n", - "
\n", + "
\n", " \n", " \n", " \n", @@ -108,202 +117,202 @@ " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
 
359bad<030.000000existing paidfurniture/equipment2406.000000<1004<=X<74.000000female div/dep/marnone4.000000real estate23.000000nonerent1.000000skilled1.000000noneyes359bad<030.000000existing paidfurniture/equipment2406.000000<1004<=X<74.000000female div/dep/marnone4.000000real estate23.000000nonerent1.000000skilled1.000000noneyes
707bad0<=X<20012.000000no credits/all paidfurniture/equipment2969.000000<100<14.000000female div/dep/marnone3.000000life insurance25.000000nonerent2.000000skilled1.000000noneyes707bad0<=X<20012.000000no credits/all paidfurniture/equipment2969.000000<100<14.000000female div/dep/marnone3.000000life insurance25.000000nonerent2.000000skilled1.000000noneyes
763badno checking21.000000critical/other existing creditnew car12680.000000no known savings>=74.000000male singlenone4.000000no known property30.000000nonefor free1.000000high qualif/self emp/mgmt1.000000yesyes763badno checking21.000000critical/other existing creditnew car12680.000000no known savings>=74.000000male singlenone4.000000no known property30.000000nonefor free1.000000high qualif/self emp/mgmt1.000000yesyes
835bad<012.000000no credits/all paidnew car1082.000000<1001<=X<44.000000male singlenone4.000000car48.000000bankown2.000000skilled1.000000noneyes835bad<012.000000no credits/all paidnew car1082.000000<1001<=X<44.000000male singlenone4.000000car48.000000bankown2.000000skilled1.000000noneyes
192bad0<=X<20027.000000existing paidbusiness3915.000000<1001<=X<44.000000male singlenone2.000000car36.000000noneown1.000000skilled2.000000yesyes192bad0<=X<20027.000000existing paidbusiness3915.000000<1001<=X<44.000000male singlenone2.000000car36.000000noneown1.000000skilled2.000000yesyes
629goodno checking9.000000existing paideducation3832.000000no known savings>=71.000000male singlenone4.000000real estate64.000000noneown1.000000unskilled resident1.000000noneyes629goodno checking9.000000existing paideducation3832.000000no known savings>=71.000000male singlenone4.000000real estate64.000000noneown1.000000unskilled resident1.000000noneyes
559bad0<=X<20018.000000critical/other existing creditfurniture/equipment1928.000000<100<12.000000male singlenone2.000000real estate31.000000noneown2.000000unskilled resident1.000000noneyes559bad0<=X<20018.000000critical/other existing creditfurniture/equipment1928.000000<100<12.000000male singlenone2.000000real estate31.000000noneown2.000000unskilled resident1.000000noneyes
684good0<=X<20036.000000delayed previouslybusiness9857.000000100<=X<5004<=X<71.000000male singlenone3.000000life insurance31.000000noneown2.000000unskilled resident2.000000yesyes684good0<=X<20036.000000delayed previouslybusiness9857.000000100<=X<5004<=X<71.000000male singlenone3.000000life insurance31.000000noneown2.000000unskilled resident2.000000yesyes
\n" ], "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -325,7 +334,7 @@ "source": [ "## 6.3 Single-objective AutoML\n", "\n", - "In this subsection, we will create a small pipeline search space and auto-configure the pipeline by optimizing for the balanced accuracy." + "In this subsection, we will create a small pipeline search space and auto-configure the pipeline by optimizing for the [balanced accuracy](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.balanced_accuracy_score.html)." ] }, { @@ -355,30 +364,30 @@ "\n", "\n", - "\n", "\n", - "\n", + "\n", "\n", "cluster:(root)\n", "\n", - "\n", + "\n", "\n", "\n", "\n", "cluster:choice_0\n", "\n", - "\n", - "Choice\n", + "\n", + "Choice\n", "\n", "\n", "\n", "\n", "cluster:choice_1\n", "\n", - "\n", - "Choice\n", + "\n", + "Choice\n", "\n", "\n", "\n", @@ -386,8 +395,8 @@ "\n", "project_0\n", "\n", - "\n", - "Project\n", + "\n", + "Project\n", "\n", "\n", "\n", @@ -395,56 +404,56 @@ "\n", "simple_imputer\n", "\n", - "\n", - "Simple-\n", - "Imputer\n", + "\n", + "Simple-\n", + "Imputer\n", "\n", "\n", "\n", "\n", "\n", "project_0->simple_imputer\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "norm\n", "\n", - "\n", - "Norm\n", + "\n", + "Norm\n", "\n", "\n", "\n", "\n", "\n", "simple_imputer->norm\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "concat_features\n", "\n", - "\n", - "Concat-\n", - "Features\n", + "\n", + "Concat-\n", + "Features\n", "\n", "\n", "\n", "\n", "\n", "norm->concat_features\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "no_op\n", "\n", - "\n", - "No-\n", - "Op\n", + "\n", + "No-\n", + "Op\n", "\n", "\n", "\n", @@ -452,8 +461,8 @@ "\n", "project_1\n", "\n", - "\n", - "Project\n", + "\n", + "Project\n", "\n", "\n", "\n", @@ -461,46 +470,46 @@ "\n", "one_hot\n", "\n", - "\n", - "One-\n", - "Hot\n", + "\n", + "One-\n", + "Hot\n", "\n", "\n", "\n", "\n", "\n", "project_1->one_hot\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "one_hot->concat_features\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "rf\n", "\n", - "\n", - "RF\n", + "\n", + "RF\n", "\n", "\n", "\n", "\n", "\n", "concat_features->rf\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "xg_boost\n", "\n", - "\n", - "XG-\n", - "Boost\n", + "\n", + "XG-\n", + "Boost\n", "\n", "\n", "\n", @@ -508,7 +517,7 @@ "\n" ], "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -519,10 +528,13 @@ "project_nums = Project(columns={'type': 'number'})\n", "project_cats = Project(columns={'type': 'string'})\n", "planned_pipeline = (\n", - " (project_nums >> SimpleImputer >> (Norm | NoOp) & project_cats >> OneHot(handle_unknown='ignore'))\n", + " (\n", + " (project_nums >> SimpleImputer >> (Norm | NoOp))\n", + " & (project_cats >> OneHot(handle_unknown='ignore'))\n", + " )\n", " >> ConcatFeatures\n", - " >> (RF | XGBoost(objective='binary:hinge')))\n", - "\n", + " >> (RF | XGBoost(objective='binary:hinge'))\n", + ")\n", "planned_pipeline.visualize()" ] }, @@ -535,7 +547,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [00:05<00:00, 1.02s/trial, best loss: -0.6405579399141631]\n", + "100%|██████████| 5/5 [00:06<00:00, 1.23s/trial, best loss: -0.6405579399141631]\n", "balanced accuracy 68.8%\n" ] } @@ -566,23 +578,23 @@ "\n", "\n", - "\n", "\n", - "\n", - "\n", + "\n", + "\n", "cluster:(root)\n", "\n", - "\n", + "\n", "\n", "\n", "\n", "\n", "project_0\n", "\n", - "\n", - "Project\n", + "\n", + "Project\n", "\n", "\n", "\n", @@ -590,56 +602,56 @@ "\n", "simple_imputer\n", "\n", - "\n", - "Simple-\n", - "Imputer\n", + "\n", + "Simple-\n", + "Imputer\n", "\n", "\n", "\n", "\n", "\n", "project_0->simple_imputer\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "no_op\n", "\n", - "\n", - "No-\n", - "Op\n", + "\n", + "No-\n", + "Op\n", "\n", "\n", "\n", "\n", "\n", "simple_imputer->no_op\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "concat_features\n", "\n", - "\n", - "Concat-\n", - "Features\n", + "\n", + "Concat-\n", + "Features\n", "\n", "\n", "\n", "\n", "\n", "no_op->concat_features\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "project_1\n", "\n", - "\n", - "Project\n", + "\n", + "Project\n", "\n", "\n", "\n", @@ -647,45 +659,45 @@ "\n", "one_hot\n", "\n", - "\n", - "One-\n", - "Hot\n", + "\n", + "One-\n", + "Hot\n", "\n", "\n", "\n", "\n", "\n", "project_1->one_hot\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "one_hot->concat_features\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "xg_boost\n", "\n", - "\n", - "XG-\n", - "Boost\n", + "\n", + "XG-\n", + "Boost\n", "\n", "\n", "\n", "\n", "\n", "concat_features->xg_boost\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n" ], "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -702,9 +714,8 @@ "source": [ "## 6.4 Specifying Objectives for Multi-objective AutoML\n", "\n", - "While we obtaine a pipeline by optimizing for balanced accuracy, there might be an application where we wish to simultaneously optimize for balanced accuracy and false positive rate, and select a pipeline from the set of pipelines with the best tradeoff between balanced accuracy and false positive rate.\n", - "\n", - "Here we define a `sklearn` scorer for false positive rate and evaluate the multiple objectives or scores (balanced accuracy and false positive rate) of the pipeline found above." + "While we obtained a pipeline by optimizing for balanced accuracy, there might be an application where we wish to simultaneously optimize for balanced accuracy and [symmetric disparate impact](https://lale.readthedocs.io/en/latest/modules/lale.lib.aif360.util.html#lale.lib.aif360.util.symmetric_disparate_impact), and select a pipeline from the set of pipelines with the best tradeoff between those two.\n", + "Here we define a scikit-learn scorer for symmetric disparate impact and evaluate the multiple objectives or scores (balanced accuracy and symmetric disparate impact) of the pipeline found above." ] }, { @@ -712,35 +723,42 @@ "execution_count": 7, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:No module named 'tensorflow': AdversarialDebiasing will be unavailable. To install, run:\n", + "pip install 'aif360[AdversarialDebiasing]'\n", + "WARNING:root:No module named 'numba': LFR will be unavailable. To install, run:\n", + "pip install 'aif360[LFR]'\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "Metric values using single obj best pipeline 0.688 0.479\n" + "Metric values using single-objective best pipeline: 0.688, 0.848\n" ] } ], "source": [ "from sklearn.metrics import make_scorer, get_scorer\n", + "from lale.lib.aif360 import symmetric_disparate_impact\n", "\n", - "# Define sklearn scorer for computing False Positive Rate (FPR)\n", - "def compute_fpr(y_true, y_pred):\n", - " from sklearn.metrics import confusion_matrix\n", - " tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()\n", - " fpr = round(fp / (fp + tn), 4)\n", - " return fpr\n", - "\n", - "fpr_scorer = make_scorer(compute_fpr, greater_is_better=False)\n", + "# Only use age, because personal status changes after one-hot encoding\n", + "di_scorer = symmetric_disparate_impact(\n", + " favorable_labels=['good'],\n", + " protected_attributes=[{'feature': 'age', 'reference_group': [[26, 1000]]}]\n", + ")\n", "\n", - "# Specify scoring params passed to Multi-objective Optimizer\n", - "scoring = ['balanced_accuracy', fpr_scorer] \n", - "best_score = [1, 0]\n", + "# Specify scoring arguments passed to Multi-objective Optimizer\n", + "scoring = ['balanced_accuracy', di_scorer] \n", + "best_score = [1, 1]\n", "\n", "# Let's also get scorers to compute the metrics value on optimizer's outputs\n", - "scorer = [get_scorer(scorer) for scorer in scoring]\n", - "print('Metric values using single obj best pipeline %.3f %.3f' \\\n", - " %(scorer[0](auto_trained, test_X, test_y), \\\n", - " -scorer[1](auto_trained, test_X, test_y)))" + "scorers = [get_scorer(scorer) for scorer in scoring]\n", + "print(\"Metric values using single-objective best pipeline: \" +\n", + " \", \".join(f\"{s(auto_trained, test_X, test_y):.3f}\" for s in scorers))" ] }, { @@ -760,11 +778,14 @@ "metadata": {}, "outputs": [], "source": [ + "from lale.lib.lale import OptimizeLast\n", "try:\n", " from lalegpl.lib.lale import NSGA2\n", + " lalegpl_installed = True\n", "except ImportError:\n", - " raise SystemExit('lale-gpl not installed, stopping execution here!')\n", - "from lale.lib.lale import OptimizeLast" + " lalegpl_installed = False\n", + " import sys\n", + " print(\"lale-gpl not installed, the remaining notebook will be skipped\", file=sys.stderr)" ] }, { @@ -782,21 +803,25 @@ "metadata": {}, "outputs": [], "source": [ - "# Parameters used when creating instance of Multi-objective optimizer \n", - "optimizer_args = {\n", - " 'scoring' : scoring,\n", - " 'best_score': best_score,\n", - " 'cv' : 2,\n", - " 'max_evals' : 40,\n", - " 'population_size' : 15,\n", - "}\n", - "\n", - "# Create operator for performing MOO on final estimator of best pipeline from single obj optimizer \n", - "optFinalEstimator = OptimizeLast(\n", - " estimator = auto_trained,\n", - " last_optimizer=NSGA2,\n", - " optimizer_args = optimizer_args\n", - ")" + "if lalegpl_installed:\n", + " optFinalEstimator = OptimizeLast(\n", + " estimator=auto_trained,\n", + " last_optimizer=NSGA2,\n", + " optimizer_args={\n", + " 'scoring' : scoring,\n", + " 'best_score': best_score,\n", + " 'cv' : 2,\n", + " 'max_evals' : 40,\n", + " 'population_size' : 15,\n", + " }\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `OptimizeLast` operator has a `fit` method that executes the multi-objective AutoML." ] }, { @@ -804,41 +829,140 @@ "execution_count": 10, "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 1min 35s, sys: 173 ms, total: 1min 35s\n", + "Wall time: 1min 35s\n" + ] + } + ], + "source": [ + "%%time\n", + "if lalegpl_installed:\n", + " opt_trained = optFinalEstimator.fit(train_X, train_y)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that the above optimization does not return a single pipeline but rather a set of Pareto-optimal pipelines that span the tradeoff between the multiple objectives (balanced accuracy and false positive rate in this case).\n", + "Calling `predict` on the `opt_trained` object uses the first pipeline found on the Pareto-front to generate the predictions. " + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "balanced accuracy 50.0%\n" + ] + } + ], + "source": [ + "if lalegpl_installed:\n", + " pred_y = opt_trained.predict(test_X)\n", + " acc = sklearn.metrics.balanced_accuracy_score(test_y, pred_y)\n", + " print(f'balanced accuracy {acc:.1%}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can also access specific pipelines from the Pareto front with the `get_pipeline` function, where we can specify a string name; by default `get_pipeline` returns the first pipeline.\n", + "Here we evaluate the multiple objectives for the first pipeline from the Pareto-front" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Metric values using the first multi-objective pareto pipeline: 0.500, 1.000\n" + ] + } + ], + "source": [ + "if lalegpl_installed:\n", + " pareto_pipeline = opt_trained.get_pipeline()\n", + " print(\"Metric values using the first multi-objective pareto pipeline: \" +\n", + " \", \".join(f\"{s(pareto_pipeline, test_X, test_y):.3f}\" for s in scorers))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6.6 Pareto-front of Multi-objective AutoML\n", + "\n", + "We can also view the complete Pareto-front of pipelines found with the multi-objective AutoML with the `summary` function." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "if lalegpl_installed:\n", + " opt_trained.summary()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here we visualize the different pipelines and their hyperparameters on the Pareto-front." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Pareto optimal pipeline p0:\n" + ] + }, { "data": { "image/svg+xml": [ "\n", "\n", - "\n", "\n", - "\n", - "\n", - "cluster:(root)\n", - "\n", - "\n", + "\n", + "\n", "cluster:(root)\n", - "\n", - "\n", - "OptimizeLast\n", - "\n", - "\n", - "\n", - "\n", - "cluster:pipeline\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", - "\n", - "\n", + "\n", "\n", - "project\n", - "\n", - "\n", - "Project\n", + "project_0\n", + "\n", + "\n", + "Project\n", "\n", "\n", "\n", @@ -846,56 +970,56 @@ "\n", "simple_imputer\n", "\n", - "\n", - "Simple-\n", - "Imputer\n", + "\n", + "Simple-\n", + "Imputer\n", "\n", "\n", "\n", - "\n", + "\n", "\n", - "project->simple_imputer\n", - "\n", - "\n", + "project_0->simple_imputer\n", + "\n", + "\n", "\n", "\n", "\n", "no_op\n", "\n", - "\n", - "No-\n", - "Op\n", + "\n", + "No-\n", + "Op\n", "\n", "\n", "\n", "\n", "\n", "simple_imputer->no_op\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "concat_features\n", "\n", - "\n", - "Concat-\n", - "Features\n", + "\n", + "Concat-\n", + "Features\n", "\n", "\n", "\n", "\n", "\n", "no_op->concat_features\n", - "\n", - "\n", + "\n", + "\n", "\n", - "\n", + "\n", "\n", - "project_0\n", - "\n", - "\n", - "Project\n", + "project_1\n", + "\n", + "\n", + "Project\n", "\n", "\n", "\n", @@ -903,249 +1027,403 @@ "\n", "one_hot\n", "\n", - "\n", - "One-\n", - "Hot\n", + "\n", + "One-\n", + "Hot\n", "\n", "\n", "\n", - "\n", + "\n", "\n", - "project_0->one_hot\n", - "\n", - "\n", + "project_1->one_hot\n", + "\n", + "\n", "\n", "\n", "\n", "one_hot->concat_features\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "xg_boost\n", - "\n", - "\n", - "XG-\n", - "Boost\n", + "\n", + "\n", + "XG-\n", + "Boost\n", "\n", "\n", "\n", "\n", "\n", "concat_features->xg_boost\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "nsga2\n", - "\n", - "\n", - "NSGA2\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n" ], "text/plain": [ - "" + "" ] }, "metadata": {}, "output_type": "display_data" - } - ], - "source": [ - "optFinalEstimator.visualize()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `OptimizeLast` operator has the `fit` operator that executes the multi-objective AutoML." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ + }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1min 11s, sys: 149 ms, total: 1min 12s\n", - "Wall time: 1min 12s\n" - ] - } - ], - "source": [ - "%%time\n", - "opt_trained = optFinalEstimator.fit(train_X, train_y)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note that the above optimization does not return a single pipeline but rather a set of Pareto-optimal pipelines that span the tradeoff between the multiple objectives (balanced accuracy and false positive rate in this case).\n", - "\n", - "We can still call `predict` on the `opt_trained` object and that uses the first pipeline found on the Pareto-front to generate the predictions. " - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ + "data": { + "text/markdown": [ + "```python\n", + "pipeline = XGBoost(\n", + " objective=\"binary:hinge\",\n", + " gamma=0.859189857095001,\n", + " learning_rate=0.10974219876135785,\n", + " max_depth=4,\n", + " min_child_weight=9,\n", + " n_estimators=435,\n", + " reg_alpha=0.35061976504895453,\n", + " reg_lambda=0.2720394745576489,\n", + " subsample=0.03310406188598723,\n", + ")\n", + "```" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, { "name": "stdout", "output_type": "stream", "text": [ - "balanced accuracy 67.3%\n" + "Pareto optimal pipeline p1:\n" ] - } - ], - "source": [ - "# Currently predict uses first pipeline in pareto-front\n", - "pred_y = opt_trained.predict(test_X)\n", - "acc = sklearn.metrics.balanced_accuracy_score(test_y, pred_y)\n", - "print(f'balanced accuracy {acc:.1%}')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can also access specific pipelines from the Pareto front with the `get_pipeline` function where we can specify a string name; by default `get_pipeline` returns the first pipeline.\n", - "\n", - "Here we evaluate the multiple objectives for the first pipeline from the Pareto-front" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "cluster:(root)\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "project_0\n", + "\n", + "\n", + "Project\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "simple_imputer\n", + "\n", + "\n", + "Simple-\n", + "Imputer\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "project_0->simple_imputer\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "no_op\n", + "\n", + "\n", + "No-\n", + "Op\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "simple_imputer->no_op\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "concat_features\n", + "\n", + "\n", + "Concat-\n", + "Features\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "no_op->concat_features\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "project_1\n", + "\n", + "\n", + "Project\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "one_hot\n", + "\n", + "\n", + "One-\n", + "Hot\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "project_1->one_hot\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "one_hot->concat_features\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "xg_boost\n", + "\n", + "\n", + "XG-\n", + "Boost\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "concat_features->xg_boost\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "```python\n", + "pipeline = XGBoost(\n", + " objective=\"binary:hinge\",\n", + " gamma=0.2530292257269834,\n", + " learning_rate=0.7562590526098265,\n", + " max_depth=6,\n", + " min_child_weight=14,\n", + " n_estimators=538,\n", + " reg_alpha=0.9097217358808588,\n", + " reg_lambda=0.6764854766703405,\n", + " subsample=0.44405647669435916,\n", + ")\n", + "```" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, { "name": "stdout", "output_type": "stream", "text": [ - "scorer values 0.673 0.448\n" + "Pareto optimal pipeline p2:\n" ] - } - ], - "source": [ - "# get_pipeline() currently returns first pipeline by default\n", - "pareto_pipeline = opt_trained.get_pipeline()\n", - "\n", - "print('scorer values %.3f %.3f' \\\n", - " %(scorer[0](pareto_pipeline, test_X, test_y), \\\n", - " -scorer[1](pareto_pipeline, test_X, test_y)))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 6.6 Pareto-front of Multi-objective AutoML\n", - "\n", - "We can also view the complete Pareto-front of pipelines found with the multi-objective AutoML with the `summary` function." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ + }, { "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
idloss1loss2
name
p000.3565280.5049
p110.3721700.4804
\n", - "
" + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "cluster:(root)\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "project_0\n", + "\n", + "\n", + "Project\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "simple_imputer\n", + "\n", + "\n", + "Simple-\n", + "Imputer\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "project_0->simple_imputer\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "no_op\n", + "\n", + "\n", + "No-\n", + "Op\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "simple_imputer->no_op\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "concat_features\n", + "\n", + "\n", + "Concat-\n", + "Features\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "no_op->concat_features\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "project_1\n", + "\n", + "\n", + "Project\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "one_hot\n", + "\n", + "\n", + "One-\n", + "Hot\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "project_1->one_hot\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "one_hot->concat_features\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "xg_boost\n", + "\n", + "\n", + "XG-\n", + "Boost\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "concat_features->xg_boost\n", + "\n", + "\n", + "\n", + "\n", + "\n" ], "text/plain": [ - " id loss1 loss2\n", - "name \n", - "p0 0 0.356528 0.5049\n", - "p1 1 0.372170 0.4804" + "" ] }, - "execution_count": 14, "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Call summary() routine to list the pareto-optimal solutions with corresponding loss values\n", - "opt_trained.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here we visualize the different pipelines and their hyperparameters on the Pareto-front." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "scrolled": false - }, - "outputs": [ + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "```python\n", + "pipeline = XGBoost(\n", + " objective=\"binary:hinge\",\n", + " gamma=0.13466037305156175,\n", + " learning_rate=0.45227009222474834,\n", + " max_depth=2,\n", + " min_child_weight=19,\n", + " n_estimators=841,\n", + " reg_alpha=0.09716104404748971,\n", + " reg_lambda=0.5899713576903844,\n", + " subsample=0.45730129547152193,\n", + ")\n", + "```" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, { "name": "stdout", "output_type": "stream", "text": [ - "Pareto optimal pipeline p0:\n" + "Pareto optimal pipeline p3:\n" ] }, { @@ -1154,23 +1432,23 @@ "\n", "\n", - "\n", "\n", - "\n", - "\n", + "\n", + "\n", "cluster:(root)\n", "\n", - "\n", + "\n", "\n", "\n", "\n", "\n", "project_0\n", "\n", - "\n", - "Project\n", + "\n", + "Project\n", "\n", "\n", "\n", @@ -1178,56 +1456,56 @@ "\n", "simple_imputer\n", "\n", - "\n", - "Simple-\n", - "Imputer\n", + "\n", + "Simple-\n", + "Imputer\n", "\n", "\n", "\n", "\n", "\n", "project_0->simple_imputer\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "no_op\n", "\n", - "\n", - "No-\n", - "Op\n", + "\n", + "No-\n", + "Op\n", "\n", "\n", "\n", "\n", "\n", "simple_imputer->no_op\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "concat_features\n", "\n", - "\n", - "Concat-\n", - "Features\n", + "\n", + "Concat-\n", + "Features\n", "\n", "\n", "\n", "\n", "\n", "no_op->concat_features\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "project_1\n", "\n", - "\n", - "Project\n", + "\n", + "Project\n", "\n", "\n", "\n", @@ -1235,45 +1513,45 @@ "\n", "one_hot\n", "\n", - "\n", - "One-\n", - "Hot\n", + "\n", + "One-\n", + "Hot\n", "\n", "\n", "\n", "\n", "\n", "project_1->one_hot\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "one_hot->concat_features\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "xg_boost\n", - "\n", - "\n", - "XG-\n", - "Boost\n", + "\n", + "\n", + "XG-\n", + "Boost\n", "\n", "\n", "\n", "\n", "\n", "concat_features->xg_boost\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n" ], "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -1285,14 +1563,14 @@ "```python\n", "pipeline = XGBoost(\n", " objective=\"binary:hinge\",\n", - " gamma=0.18817063046716287,\n", - " learning_rate=0.1326670086631808,\n", - " max_depth=3,\n", - " min_child_weight=4,\n", - " n_estimators=721,\n", - " reg_alpha=0.2846348240483074,\n", - " reg_lambda=0.8623403429578169,\n", - " subsample=0.40876592698273684,\n", + " gamma=0.7628866972001157,\n", + " learning_rate=0.11289406635035952,\n", + " max_depth=2,\n", + " min_child_weight=13,\n", + " n_estimators=793,\n", + " reg_alpha=0.8916440115567629,\n", + " reg_lambda=0.24964146334134232,\n", + " subsample=0.23733262975737876,\n", ")\n", "```" ], @@ -1307,7 +1585,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Pareto optimal pipeline p1:\n" + "Pareto optimal pipeline p4:\n" ] }, { @@ -1316,23 +1594,23 @@ "\n", "\n", - "\n", "\n", - "\n", - "\n", + "\n", + "\n", "cluster:(root)\n", "\n", - "\n", + "\n", "\n", "\n", "\n", "\n", "project_0\n", "\n", - "\n", - "Project\n", + "\n", + "Project\n", "\n", "\n", "\n", @@ -1340,56 +1618,56 @@ "\n", "simple_imputer\n", "\n", - "\n", - "Simple-\n", - "Imputer\n", + "\n", + "Simple-\n", + "Imputer\n", "\n", "\n", "\n", "\n", "\n", "project_0->simple_imputer\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "no_op\n", "\n", - "\n", - "No-\n", - "Op\n", + "\n", + "No-\n", + "Op\n", "\n", "\n", "\n", "\n", "\n", "simple_imputer->no_op\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "concat_features\n", "\n", - "\n", - "Concat-\n", - "Features\n", + "\n", + "Concat-\n", + "Features\n", "\n", "\n", "\n", "\n", "\n", "no_op->concat_features\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "project_1\n", "\n", - "\n", - "Project\n", + "\n", + "Project\n", "\n", "\n", "\n", @@ -1397,45 +1675,45 @@ "\n", "one_hot\n", "\n", - "\n", - "One-\n", - "Hot\n", + "\n", + "One-\n", + "Hot\n", "\n", "\n", "\n", "\n", "\n", "project_1->one_hot\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "one_hot->concat_features\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n", "xg_boost\n", - "\n", - "\n", - "XG-\n", - "Boost\n", + "\n", + "\n", + "XG-\n", + "Boost\n", "\n", "\n", "\n", "\n", "\n", "concat_features->xg_boost\n", - "\n", - "\n", + "\n", + "\n", "\n", "\n", "\n" ], "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -1447,14 +1725,14 @@ "```python\n", "pipeline = XGBoost(\n", " objective=\"binary:hinge\",\n", - " gamma=0.21366688493747565,\n", - " learning_rate=0.8910831102827533,\n", - " max_depth=1,\n", - " min_child_weight=6,\n", - " n_estimators=820,\n", - " reg_alpha=0.4051207340038795,\n", - " reg_lambda=0.3234419809813596,\n", - " subsample=0.7517190652925614,\n", + " gamma=0.24444274758583728,\n", + " learning_rate=0.7562590526098265,\n", + " max_depth=3,\n", + " min_child_weight=14,\n", + " n_estimators=533,\n", + " reg_alpha=0.9097217358808588,\n", + " reg_lambda=0.22943888113151426,\n", + " subsample=0.48633961754946003,\n", ")\n", "```" ], @@ -1467,11 +1745,12 @@ } ], "source": [ - "for pname in opt_trained.summary().index:\n", - " print(f'Pareto optimal pipeline {pname}:')\n", - " p = opt_trained.get_pipeline(pname)\n", - " p.visualize()\n", - " p.get_last().pretty_print(ipython_display=True, show_imports=False) " + "if lalegpl_installed:\n", + " for pname in opt_trained.summary().index:\n", + " print(f\"Pareto optimal pipeline {pname}:\")\n", + " p = opt_trained.get_pipeline(pname)\n", + " p.visualize()\n", + " p.get_last().pretty_print(ipython_display=True, show_imports=False)" ] }, { @@ -1488,12 +1767,14 @@ }, { "cell_type": "code", - "execution_count": 16, - "metadata": {}, + "execution_count": 15, + "metadata": { + "scrolled": false + }, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsgAAAE8CAYAAADdQEfkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAABOMUlEQVR4nO3deZwUxf3/8dcHBEFAjKAEwQhRNEFULpEED8BbCZp4x6gYjTGBxCMa9WtEfkSNiSQxwSNGI6hR8YqKiPFkPYgaDldAVERFBYwiKpccC3x+f1TP0jvM7Pbuzs4Ms+/n4zGPma7urv7UdE9tbXd1tbk7IiIiIiISNCl0ACIiIiIixUQNZBERERGRGDWQRURERERi1EAWEREREYlRA1lEREREJEYNZBERERGRGDWQS5SZbWNmN5nZIjNzM3uk0DHVxMxGRbF2LnQstWVmC8xsfKHjSDGz8Wa2vp559DSz581sebRfjq3Ful2idYYlWHZYtGyXeoQrskVQ3ZxfxVY3b4lS+7/QceRbo2ggx/4Ap14bzOx/ZjbBzHYvUEwDo4Nu2wbaxK+Bc4E7gNOAPzfQdrKKGlijzOwb+d621I+ZNQUeBHYGLiEcQ9MLGlTEzLY3s9+a2etmtsLMVpvZW2b2VzPrFi0z08w+M7Nm1eTzp6g+2Ct/0Uuc6mbVzY1F7KRBktewHG2zoY/lLSKGutqq0AHk2W+BecDWQB/gLOAQM9vL3T/OcywDgSuB24DlDZD/IcAsd/+/Bsg7qZ6EMj4DfFjAOKT2ugK7Ahe4+82FDibFzPYBngDaAfcBtwAVQHfgJELDozlwF/An4AjgsQz5NAVOAV5399l5CV6qo7o5v3qiujnflhD+IYq7HPgacFFa+n9ytM2BNOyxvKXEUCeNrYH8lLu/FH3+h5m9DVwPDAN+V5+MzayVu6+qX3g5tSOwuKaFzKwJ0Nzd1zR8SLIF2TF6/7KQQcRFZyAmAs2Afd19Vtr8/wOujibvBa4DfkSGBjKhkfJ1YEyDBZwjjeQ3qro5TSPZ741GdAz+M55mZmcDzdz9n5nXkoJy95J/ESpZB/ZPS98zSv97bLmngY+BdcB7hMp567T1xgPrCZef/0VoRLwWm38C8ArwFeE/pseBvdLW9wyvLtH8JoTLcG8DawmV6Y3AdgnKOjBL3gOBLtHn3wA/Bd4inH07NvZ9PBqV56uoDEOy5H8a8CvgA2BNtGzv2HKjssVRTeypdXoQLj9+EX1/9wI7pi17ADABWBB9R58SKp/Oacs1BS6LyvpVal8BP0tbrg3wB+D9aN9/SGhgtUxbrjnwe+B/wCrCGZhvRXGMT7B/Eu1boAyYD+wGPBlt61PgWqBJgu2MB9anpe0CvEM4U/eNGtZN328L0n43NR0nqWNtWFp6f8LZkTXRd3wpcCax47+auC6Kljsj4e/+iSi+bTPMu4vwG+5YQx47Es5SfxDtryXRvhmYttyuUZ4fR8u9D9wKtIkts3N0jC6Jyv96hu8n9b1l+412iOJZHB2n8wnHd5O0fI4DXgWWRcfOfODmJN9bPl+oblbd3Mjq5kx5ZUiv9jhNUjdRw7FcTUxHE+qmNYS/FWel9n+G3261v8maYkiSRyFfje0McrrdovfPovfhhB/Hk8BK4LuEH8w3gFPT1rVouVmEP/JNAMzsIsKP92HCH8zWwM+AqWbW193nEQ7qbYHvA78kVDYQDnCAmwiV5CTgr4TLx+cC/c3sO+6+rpoyvUmoIMdE+V4dS28ZfT4ZaBvF8TmwIOrv9x9CpXw94Uc5DJhoZie5+wNp2zmPcDn0r4QrERcDD5vZbu5eQfjj1BE4h02XT1Nx1OSfwFLgCqAb8HPg22bWL1b2EwiX2W8DPomW+ymwn5nt7e6ro+VGRq/bgT9G30F3YH/gZgAzawE8F+Xxd0LltzdwPtDDzI7y6NccfWfDgIeidfYl/MBbJCgX1G7ftiFU8k8QjqfDCf2B34/iSMzM9oji/BI41N0/qWbxWwh/hK4gfB8vEn4P1OE4icfQPSrPCuAqQoV4TirvBI4l/CGYkHD5uwhdLH5AqKhTcWwT5fWs13z5/gHC5egbgXeB7YH9gF6EP0aY2beBqYTfwd8Jx3pnwu+7HbDCzNoTvrd2wFhgEXAiMM7M2rt7+pnsTL/R9oQ/mi2i7SwGBgDXEP75OTeK5+Ao7jLCJdwK4JvAkTWUtZioblbd3Cjq5nQJj1OouW6q6VjOtO3BhH/E3iPs4xaE+iVTPZnkN1lTDLX5XedfoVvo+Xix6SzF0UB7YCfge4T/LDcQ/XcNbJNh3d8AG4n998um/4r+mrbszoRK7Oq09A6Eyu7uWNqoKI/0/6p7ROn3pKUPj9J/nrDM84GytLQuUR5fZdjug4QzL3vG0toQfiiLgK2itIFRHu8R+w+e0OBw4OgM3/v+CWNOfSfPAU1j6T+J0n8aS8u0r/aPljs1lvYa8HgN272M8N9y+n/o50T5HRpN7xVN35G23DVR+vgatpN43xIqOAfOTVu2HJiW4LscT3QGGdiH8IfqVeBrCfdF6rscVsfjpEv6+oQ/XBXA7rG0HQiN9iRnNj4n9BlO+rvfhtAYfyYt/dRoez+qYf220XIX17Dcc8Bq4FsZ5ln0PibK6/DYvGaEhs9qoF2C3+gthAZjx7T0awh11O7R9J8JZ46bVhd3MbxQ3ZxKq26/q24uobo5bZ0yYmeQkx6nJK+bMh7L1Sw/g/APULtY2rej48/Tlk36m8waQ9I8CvVqFKNYxEwi/OeyiNCXsQVwmrvPBHD3ryD0/TKz7aIzNi8Qzkj0zpDfTWnTxxH+Y7/XzNqnXoSK/mVgcIIYh0Tv6WeUbiU0JIZQfxPdfWFqIrph6Qhgsru/kUp39xWE/+R3YvPyj/dNZwIAno/ev5mD+Ma6+4b4tgh/8CvLntpXUfxtou/5LcJ31Ce27pfAntFZvmxOIuyfj9P22zPR/NR+S23/+rT1/1xzkaqsn3TfVhDOwsQ9Ty2+YzP7DqESngsc4u5fJF03Q151OU4yrTsvtu4S4O6EIWxLLW7yiI6Rh4BBZrZTbNaPCJdFH64hi9WEs9wDo+NhM1H6IOAud38rQwwefRwCzHH3J2PzKgjHTgvg4LRV03+jRjgzNxmoSDtOnyTUUYOixb8EWgFHRuttCVQ3B6qbqyrZujmLpMdpjXVTbZlZR8Kx9E93X5pKd/c3CXVMFXX4TW4mF3k0pMbWxeICYA7hYFsCvBn/sZtZf8J/nN8lXKKK2y5Dfu+lTaeGJcp2V/zGBDF2id6r/LF193VmNp8wugAAZvb1tHVXunuSy9Xvpk3vQPiDutkfeELDimi7/42lf5AW3xfR3+LtE2y/Jm+n5V1hZu9Rtew7EfqlDSH8Nx23XezzbwiXjOaa2TxCxfqAu5fFltmdcHkv26Wn1A1ru2SJb4mZJWl4donea9y3kUXunj6W8Rck/46bAE8RjtMjPe1mHzNryebf3ZK0P4BxdTlO4utuQ9p3F8mUlslywpmz2rgLOAP4ITDGzDoAhxLOFFV741a0X35F+CP7PzObAfw7WjcV867Re00jYXQhNPzSxb+3uEy/0a8RLtGn3wmfkjpObyL8oX0M+MTMpkTbfjBqlBcj1c2B6ubGUTdnk+g4TVg31VbG7zCWdlQ8oQ6/yc3kIo+G1NgayNN9053SVZhZV8Llo3epeoNDJ8J/yeln2zf45v3NUssMIfSVbGjp/YL+H+FyRk1W17xIjbI1ohr8jFV0d/fThFEIriP8oVhJuIwzgdi+cvepZrYr4RLuIYTLjT83s1vc/dxosSaE//6vyrLJGu84byDZvuOkNhLOkv6I0N/1zrT5JwHj0tK6Ei5vF6M3gd5mtrW7J/19TSGclfwR4ezQKYSbg+5KsrK732BmE4GhhLO8FwKXmdlZ7p4ojzpK/42mjun72PzMVcp7UNko6E0423QE4R+Ck4GLzWz/+Bm+IqK6OVDd3Djq5mwSH6cFrJvq8ptskDwaWmNrIFdnKOE/1SHuXvkfuJkdVos85kfvH3naEFQZeJb0BdH7t4CZsTiaEc5WvRJb9tC0ddPPmiS1hHDJ+VsZ5qUuf71fh3yzlbEme7Dp7Eiq7F2B1B/QvQg3UAxz9ztiy7UknGWrGoT7MuAe4B4z24rw4/upmf0u2tfzCSMdPJO+bprUcbEHof9carups3s1WRC9J9m3uXIm4XL17Wa2xt3vj817ks2Pof9Vk1d9jpMlhP6Ve2SYlyktk0cJZxpOJHkDd6OZ3Q382sz2JDSUFwPPJtwm7v4hcANwg5l9jXCpc3QUQ+qMX00PG1lA/X5fSwhn0JsnOE6Jzm49Fb0ws58RziyfQBiFYEuiull1cynWzZnU5jitqW6C2u3n+HeYLj2tNr/JbDHk4nfdoAreQi8iqUtsld9J9N/wr2qRx0OEzuz/L1q3iujHmpK6vLtd2mKTovcL09LPJvzQK8d0dfdn0l51qoSjS5lPEPosVvYHM7PU3bOLiVUatZCtjDX5RdT3LmVYlMfj0fRm+yryq/Q0M2sXn44aDnPS4poA9DKz76cHYmYtzCx1WT+1/fPTFrsgczE2k3jf5kq0b08lxH63mR0Tm/dxhmMo65ir9TlOonWfBI6y2BPSot9E0ruVbwEWAn80sx7pM6N99acM66X+WFxF6AN5j7vXeEndwiOBW8bToj7cC4iOHXf/jHCW+jQz26wRE+sD/Biwl5kdGpu3FeFYWsOmPpUZRd/fA8BQM9s3w3bamNnW0ed26fPZ1GjYrrrtFCnVzaqbS65uziLRcZqkbook3s8eRvR5DfhRfN9Ex93haYvX5jeZLYZc/K4blM4gb/JvwiWNx83sFsJOO5HN+8Vk5e7vm9mvCU/w+q+Z/Ytw1/k3CJc65xAqFAh3iwJcY2YPEDr9P+buc6Lt/9TCgxH+zabhZmYC/6hXKbO7HDgMeMHMbmDTUEJdgZMy9LdKYibhv8fLoh/cWuA5d/+0hvW2B54ys4cJwz0NJ/TJuj2a/yZhaKI/WnhU6ieEO7i/S7gDN+5NM3sJmBYttwcwgrAvUv28xhAuaT1oZv8k9OdrRugPdiJwPOGu81lmdhdwupm1YtNQQoeyaTiqrAq1b6N+gicSzsDeb2ZD4zeL1VJ9jpORhIr2+WjdCsLd6AsII23UVI5lUQN/MjDDzO4ljMxRQTjzcxKhT+KFaevNMbNywiVcSHj2mbD/p5jZg2y6VHxgVIb40wV/QRjm7b9m9ndCf72OhH7Ax0Tl+z2hm8MjZpYa5u0EwjBtF7v75wniuYxwnL9oZrcThjFrTRgj93jC2bsFwG1mtiPhLPmHhNEhziX8ocrUD7rYqW5W3VySdXOGOJIep0nrpmzHcrb7Ly4hlP3lqC5ryaZ9Eq+ja/ObzBhDLfMoDC/wMBr5eJFwSBtCJTSdcCn4f4RLF6nhX4bFlhtP2kMY0vI5mvADXR7lNT9ap3/aclcRzgBsoOrg2akBy+cR7lT9mHB5NNEQXVEe1Q0l9Jss6+xJ+AO6jNAXrrrB6DcbIitKH5WWNoJwCXB9NH9gNTGPipbpQegv+wVhmK77gK+nLduNcNbgyyjeiYRLYQuIDelDaFT8h1DJrCFcEv8Lmw9uv020/bcIP9ql0bEwCtg+tlxzQt+6T6nfYPTV7luyDyA/irThdrJsZ7NjlFDZPRcdk4NqWD/jMG+1OE66ZFqf8IfyZerwoJBYHu0JY8jOjvbBGsIf5uuBb2ZZ58JoG7Nq8RtqRxgPdQ7ht7yS0Ci9kGhordiyuxMempB6CMh7hDPerWPLfIMwYsdn0TE2Czizlr/RdoQbc96Ljp9PCY3zi4EW0TLHEf6JSD20ZBFhqLB9kpY9Xy9UNyfd76qbS6RuTphXtccptaubMh7L1cT0vSivtVT/oJBEv8kafk+J8yjEKzVGp4iIiIiIoD7IIiIiIiJVqIEsIiIiIhKjBrKIiIiISIwayCIiIiIiMY12mLf27dt7ly5dCh1GtVatWkWrVq0KHUbOqDzFrZTKU0plgbqXZ8aMGcuBl939iNquqzoy/1Se4lZK5SmlskD9yjNjxozP3H2H9PRG20Du0qUL06dPL3QY1SorK2PgwIGFDiNnVJ7iVkrlKaWyQN3LY2bv1KVxDKojC0HlKW6lVJ5SKgvUrzxm9kGmdHWxEBERERGJUQNZRERERCRGDWQRERERkZhG2wdZpDGrqKhg4cKFrFmzpjKtbdu2vPnmmwWMKndKqSxQc3latGhB586dadasWYPGkem4KaTGtp/zLV/HlUgxUgNZpBFauHAhbdq0oUuXLpgZACtWrKBNmzYFjiw3SqksUH153J2lS5eycOFCunbt2qBxZDpuCqkx7ed8y+dxJVKM1MVCpBFas2YN7dq1K4pGjtSPmdGuXbu8nNXVcdN45PO4EilGeW8gm9kRZva2mc03s0szzB9mZkvMrDx6nR2bd4aZvRO9zoil9zGz2VGefzXV3iI10s+kdORzX+q4aTy0r6Uxy2sD2cyaAjcCRwLdgVPMrHuGRe9z957R67Zo3e2BK4H9gH7AlWb2tWj5m4GfAN2iV53G/RQRERGR4rfoy9Xc8Nw7uDsQugXd8Nw7LPpydU7yz/cZ5H7AfHd/z93XAROAYxKuezjwtLt/7u5fAE8DR5hZR2Bbd3/Fw7d0J3BsA8QuIjnUtGlTevbsSY8ePTjhhBP46quv6p1nWVkZ//nPf2q93qRJk+jVqxf77LMP3bt355Zbbql2+VGjRjFmzJhqlykvL2fy5MmV0xMnTuTaa6+tdWxSVfy4Of3003XciDRSD89cyJin5jF60lwARk+ay5in5vHwzIU5yT/fDeROwEex6YVRWrrjzGyWmT1oZjvXsG6n6HNNeYpIEWnZsiXl5eXMmTOH5s2b87e//S3ReuvXr886ry4NnYqKCs455xwee+wxXn/9dV577bWcPGEqvaEzdOhQLr10s15lUkvx46ZZs2Y6bkQaqeGDduPMAV0YN3UBsxctY9zUBZw5oAvDB+2Wk/yLcRSLx4B73X2tmf0UuAMYnIuMzewc4ByADh06UFZWlotsG8zKlSuLPsbaUHmKR9u2bVmxYkWVtA0bNmyWBvDxsjU8NvtTfjJgZ8wMd+fWqR/xvb12pGPbFvWKI7W9fffdlzlz5nD//ffzhz/8gYqKCrbffntuu+02dtxxR6655href/99FixYQOfOnbnuuus4//zz+eij8D/z73//ezp27MjNN99M06ZNueOOOxgzZgydOnVi+PDhLF26lPbt23PTTTex8847V4nh888/p6KigubNm1fGs9NOO7FixQo++OCDjOuvXbuWZs2asWLFCo466iiuuuoqevfuzdKlSznooIOYOXMmV1xxBatXr+aFF17gwgsvZM2aNcycOZM//vGPWfM999xzadOmDa+99hqffvopo0eP5thjj2XRokWcddZZrFixgvXr1/PnP/+Z7373u1XKsWbNmnofjzXVkZmOm2zycdz079+fuXPn5uy4ufPOO7nuuusKdtxMmzYtp8fN//73P4YNG1btcVOT+hxXW3IdmUkpladUynJQG9h+r/V0aAm/2ms9e7VZwvPPP5+bzN09by/gO8CTsenLgMuqWb4psCz6fApwS2zeLVFaR+CtWHqV5bK9+vTp48VuypQphQ4hp1Se4jF37tzN0pYvX55x2bHPzvNdLpnkoybO8Y0bN/qoiXN8l0sm+dhn59UrhlatWrm7e0VFhQ8dOtRvuukm//zzz33jxo3u7n7rrbf6hRde6O7uV155pffu3du/+uord3c/5ZRT/MUXX3R39w8++MC/9a1vVS533XXXVZZlyJAhPn78eHd3/8c//uHHHHNMxljOOuss32GHHfzkk0/2f/7zn75hw4Zq109tx939oIMO8mnTprm7+5IlS3yXXXZxd/dx48b58OHDK7cRn86W7xlnnOHHH3+8b9iwwd944w3fdddd3d396quv9quuusrd3devX59xX2Xap8B0r2N9namOzLSNbPJx3Bx11FE5PW5SCnXcLF++PKfHzZgxY2o8bmpSm32ebkuuIzMppfKUQlni9cpf//lIlfqmNrLVk/k+gzwN6GZmXYFFwMnAD+MLmFlHd/84mhwKpEZNfxK4JnZj3mGExvXnZrbczPoDrwKnA2MbuBwijcbwQbuxdNU6xk1dwLipCwBychlr9erV9OzZE4ADDjiAs846i7fffpuTTjqJjz/+mHXr1lUZf3Xo0KG0bNkSgGeeeYa5c+dWzlu+fDkrV67cbBsvv/wy//rXvwA47bTT+PWvf50xlttuu43Zs2fzzDPPMGbMGJ5++mnGjx+feP3aqi7fY489liZNmtC9e3c++eQTAHr37s2IESOoqKjg2GOPrfzeilk+jpv99ttPx00k03Gz77778uMf/3iLOm5EkrpxyvzKbhV7tVnCmdvuwLipC2jXqjkjBnerd/557YPs7uuBEYTG7pvA/e7+hpmNNrOh0WK/NLM3zOx14JfAsGjdz4HfEhrZ04DRURrAz4HbgPnAu8ATeSqSSMkzM0YOqTrYzMgh3es9BFSqL2l5eTljx46lefPm/OIXv2DEiBHMnj2bW265pcoYrK1atar8vHHjRl555ZXK9RctWkTr1q0Tb/vwww+nZ8+enH125SiS7LXXXlxwwQU8/fTTPPTQQ4nz2mqrrdi4cSNATsaM3XrrrSs/e3R39oABA3jhhRfo1KkTw4YN484776z3dhpaPo6bMWPG6LiJZDpuDjzwwC3uuBFJ6vu9O3PRYbtX1jMjh3TnosN25/u9O+ck/7yPg+zuk919d3ff1d2vjtJGuvvE6PNl7r6nu+/j7oPc/a3Yure7+27Ra1wsfbq794jyHOGp2kFE6s3dK+8SThk9aS4N8TNbtmwZnTqFe2zvuOOOrMsddthhjB276UJReXk5AG3atKnSR/a73/0uEyZMAODuu+/mgAMOAODJJ5+kvLyc2267bbO+eOXl5eyyyy7Vrh/XpUsXZsyYAcCDDz5YmZ4eS1ySfOM+/PBDOnTowE9+8hPOPvtsZs6cWe3yxUDHTeGPmw8++GCLO25Ekuq0XUtGDO5W+U+3mTFicDc6bdcyJ/nrSXoiUq34Zaz3f3dU5V3DN06Zn/NtjRo1ihNOOIE+ffrQvn37rMv99a9/Zfr06ey999507969ciSD733vezz88MMMGDCAF198kbFjxzJu3Dj23ntv7rrrLv7yl79slpe784c//IE99tiDnj17cuWVVzJ+/HiAROtfdNFF3HzzzfTq1YvPPvusMn3QoEHMnTuXnj17ct9991VZJ0m+cS+++CL77LMPvXr14r777uO8886rdvlisCUeNz179iyp46asrGyLO25EikamjsmN4aWb9PJP5SketblJb+EXX/nYZ+dV3viwceNGH/vsPF/4xVcNGmN91OVmpGKWpDzFdpNePo6bxrif80036W1SSuUppbK416882erJYhzmTUSKSOoyVkrqMpZIdXTciMiWTF0sRERERERi1EAWaaRc97KWDO1LEZHcUgNZpBFq0aIFS5cuVcOqBLg7S5cupUWL+j2dTkRENlEfZJFGqHPnzixcuJAlS5ZUpq1Zs6ZkGlmlVBaouTwtWrSgc+fcjP0pIiJqIIs0Ss2aNavytDEIQ0L16tWrQBHlVimVBUqvPCIixU5dLEREZIuR/uS78ePHM2LEiAJFk9mCBQu45557Ms7buHEjv/zlL+nRowd77bUX++67L++//361+XXp0qXKWMmZjB8/nsWLF1dOn3322VUeqy0itaMzyCIiIlmsX7+erbaq3Z/KVAP5hz/84Wbz7rvvPhYvXsysWbNo0qQJCxcurPI47LoaP348PXr0YKeddgLgtttuq3eeIo2ZziCLiEjDWLYQHr8I/j4ovC9b2GCbWrFiBV27dqWiogKA5cuXV04PHDiQ8847j549e9KjRw/++9//ArBq1Sp+/OMf069fP3r16sWjjz4KhMbm0KFDGTx4MAcffDCff/45xx57LHvvvTf9+/dn1qxZQHiC32mnncZ3vvMdunXrxq233grApZdeyosvvkjPnj3585//XCXOjz/+mI4dO9KkSfjz27lzZ772ta8B8MADD7DXXnvRo0cPLrnkks3KuGDBAnr06FE5PWbMGEaNGsWDDz7I9OnTOfXUU+nZsyerV69m4MCBTJ8+HYB77703Y76tW7fm8ssvZ5999qF///588skn9d8RIiVCDWQREcm9ZQvh5gEwYzwsnhnebx5Q70by6tWr6dmzJz179mTAgAGMHDkSgDZt2jBw4EAef/xxACZMmMAPfvADmjVrBsBXX31FeXk5N910Ez/+8Y8BuPrqqxk8eDD//e9/mTJlChdffDGrVq0CYObMmTz44IM8//zzXHnllfTq1YtZs2ZxzTXXcPrpp1fGM2vWLJ577jlefvllRo8ezeLFi7n22ms54IADKC8v54ILLqgS/4knnshjjz1Gz549+dWvfsVrr70GwOLFi7nyyit57rnnKC8vZ9q0aTzyyCOJvpPjjz+evn37cvfdd1NeXk7Lli0r5y1evJhLLrkkY76rVq2if//+vP766xx44IGVDXwRUQNZREQawkvXw7pVsDGc0WVjRZh+6fp6ZduyZUvKy8spLy9n6tSpjB49unLe2Wefzbhx4wAYN24cZ555ZuW8U045BYADDzyQ5cuX8+WXX/LUU09x7bXX0rNnTwYOHMiaNWv48MMPATj00EPZfvvtQ1FeeonTTjsNgMGDB7N06VKWL18OwDHHHEPLli1p3749gwYNqjw7nU3nzp15++23+d3vfkeTJk04+OCDefbZZ5k2bRr7778/O+ywA1tttRWnnnoqL7zwQr2+K4Bp06YxcODAjPk2b96cIUOGANCnTx8WLFhQ7+2JlAr1QRYRkdxbNGNT4zhlYwUsmtlgmxwwYAALFiygrKyMDRs2VOmOYGZVljUz3J2HHnqIPfbYo8q8V199NXG/4Ez51mTrrbfmyCOP5Mgjj6RDhw488sgjHHLIITWut9VWW7Fx48bK6TVr1iSKMZtmzZpVxtu0aVPWr19fr/xESonOIIuISO516gNNmlVNa9IMOvVu0M2efvrp/PCHP6xy9hjCzXEQzga3bduWtm3bcvjhhzN27NjKB+akujukO+CAA7j77ruBMORe+/bt2XbbbQF49NFHWbNmDUuXLqWsrIx9992XNm3asGLFiox5zZw5s3K0iY0bNzJr1ix22WUX+vXrx9SpU/nss8/YsGED9957LwcddFCVdTt06MCnn37K0qVLWbt2LZMmTaqcl22b/fr14/nnn682XxHZnM4gi4hI7u1/Psx+YFM3iybNoHmrkN6ATj31VH7zm99UdqlIadGiBb169aKiooLbb78dgCuuuILzzz+fvffem40bN9K1a9cqjc6UUaNG8eMf/5i9996bbbbZhjvuuKNy3t57782gQYP47LPPuOKKK9hpp53YYYcdaNq0Kfvssw/Dhg2r0g/5008/5Sc/+Qlr164FQgN2xIgRtGjRglGjRjFo0CDcnaOPPppjjjmmShzNmjVj5MiR9OvXj06dOvGtb32rct6wYcM499xzadmyJS+//HJleseOHbn22murzVdEMnD3Rvnq06ePF7spU6YUOoScUnmKWymVp5TK4l738gDTPYd15Ny5c2sXwJcfuU/6lfstg8L7lx/VqRzZLF++fLO0Bx54wH/0ox9VSTvooIN82rRpOd22u/uVV17p1113Xc7yy1SeQqv1Po/R77B4lVJZ3OtXnmz1pM4gi4hIw2jbGY4ek7fN/eIXv+CJJ55g8uTJedumiJQmNZBFRKQkjB07NmN6WVlZg2xv1KhRDZKviBSebtITEZHEPLqhTUqf9rU0Zmogi4hIIi1atGDp0qVqODUC7s7SpUtp0aJFoUMRKYi8d7EwsyOAvwBNgdvc/dosyx0HPAjs6+7TzexU4OLYInsDvd293MzKgI7A6mjeYe7+aUOVQUSkMercuTMLFy5kyZIlhQ4FCOMAl1IDrtjK06JFCzp37lzoMEQKIq8NZDNrCtwIHAosBKaZ2UR3n5u2XBvgPODVVJq73w3cHc3fC3jE3ctjq53q7tMbtgQiIo1Xs2bN6Nq1a6HDqFRWVkavXr0KHUbOlFp5RLZk+e5i0Q+Y7+7vufs6YAKQaUDG3wK/B7I9JuiUaF0RERERkZyyJH3JzGwvd59d742ZHQ8c4e5nR9OnAfu5+4jYMr2By939uKjrxEXpZ4bN7F3gGHefE02XAe2ADcBDwFWeoWBmdg5wDkCHDh36TJhQ3G3slStX0rp160KHkTMqT3ErpfKUUlmg7uUZNGjQDHfvm3R51ZGFpfIUt1IqTymVBepXnqz1ZKbBkdNfwEZgGvAzYLsk62TJ53hCv+PU9GnADbHpJkAZ0CWaLgP6puWxHzA7La1T9N4GeAo4vaZY9KCQ/FN5ilsplaeUyuJePA8KKTbaz8VN5SlepVQW94Z5UEjSLhaDgbnAH4DFZnavmR1qZlabVjqwCNg5Nt05SktpA/QAysxsAdAfmGhm8Zb9ycC98UzdfVH0vgK4h9CVQ0RERESk1hI1kN29zN3PAL4OjAA6AU8CH5jZb81s14TbmwZ0M7OuZtac0NidGNvOMndv7+5d3L0L8Aow1KMuFmbWBDiRWP9jM9vKzNpHn5sBQ4A5CeMREREREamiVjfpufsqd7/d3Q8E9gAWAP8HzDOz583s+zWsv57QwH4SeBO4393fMLPRZjY0QQgHAh+5+3uxtK2BJ81sFlBOOCN9a23KJSIiIiKSUuth3sysCzAMOJ3QXWIy8AhwOHCfmd3o7hdkW9/dJ0frxNNGZll2YNp0GaHbRTxtFdCnVoUQEREREcki0RlkM9vGzE43synAfOBUwlnab7j799z9H+5+IvBT4KyGC1dEREREpGElPYP8CaEx/S/gkOhMbibTgKU5iEtEREREpCCSNpB/Ddzj7suqW8jDuMTF85glEREREZFaStRAdvebGzoQEREREZFikLQP8u1mlvGRStGYyBo1QkRERERKQtJh3g4lPMI5k4cII1iIiIiIiGzxkjaQdwA+zzLvC2DH3IQjIiIiIlJYSRvIHxAe0pHJgcDC3IQjIiIiIlJYSRvI44FLzGy4mbUGMLPWZvZzwggXtzVQfCIiIiIieZV0mLffA7sCY4G/mtkqoBVgwN+j+SIiIiIiW7ykw7xtBM42s+uAwcD2hAeCPOfu8xowPhERERGRvEp6BhkAd38beLuBYhERERERKbhaNZDNrDOwO9AifZ67T85VUCIiIiIihZKogWxmbYD7gcNSSdG7xxZrmsO4REREREQKIukoFr8DvgEcQGgcfx8YCPwDeB/o3xDBiYiIiIjkW9IG8lHA1cCr0fRid3/B3c8BHgUubojgRERERETyLWkDuQPwkbtvAFYRRrFImcymrhciIiIiIlu0pA3kj4D20ed3gCGxefsBa3IZlIiIiIhIoSQdxeJp4BDgYeDPwB1m1gdYS3jU9B8bJjwRERERkfxK2kC+BNgGwN3vMrOVwPFAS2AEcEvDhCciIiIikl81NpDNbGvCqBX/BT4DcPeHCWeTRURERERKSo19kN19LXAbsFMuNmhmR5jZ22Y238wurWa548zMzaxvNN3FzFabWXn0+lts2T5mNjvK869mZtnyFRERERGpTtKb9GYTnqBXL2bWFLgROBLoDpxiZt0zLNcGOI9Nw8qlvOvuPaPXubH0m4GfAN2i1xH1jVVEREREGqekDeQLgF+b2RAzq9XjqdP0A+a7+3vuvg6YAByTYbnfAr8nwegYZtYR2NbdX3F3B+4Ejq1HjCIiIiLSiFloU9awkNkSwk16LQiPl/6Cqo+Zxt13TJDP8cAR7n52NH0asJ+7j4gt0xu43N2PM7My4CJ3n25mXYA3gHnAcuA37v5i1AXjWnc/JFr/AOASd48PRZfK+xzgHIAOHTr0mTBhQo1lL6SVK1fSunXrQoeRMypPcSul8pRSWaDu5Rk0aNAMd++bdHnVkYWl8hS3UipPKZUF6leebPVk0rPBN5LWIG4IZtYE+BMwLMPsj4FvuPvSaIi5R8xsz9rk7+5/B/4O0LdvXx84cGD9Am5gZWVlFHuMtaHyFLdSKk8plQXyVx7VkYWl8hS3UipPKZUFGqY8iRrI7j4qR9tbBOwcm+4cpaW0AXoAZdF9dl8HJprZUHefThh3GXefYWbvEvpFL4ryyZaniIiIiEhiSfsg58o0oJuZdTWz5sDJwMTUTHdf5u7t3b2Lu3cBXgGGRl0sdohu8sPMvkm4Ge89d/8YWG5m/aPRK04HHs1zuURERESkRCQ6g2xm06ihi4W796spH3dfb2YjgCeBpsDt7v6GmY0Gprv7xGpWPxAYbWYVwEbgXHf/PJr3c2A84cElT0QvEREREZFaS9oH+Q02byB/DfgusBp4NukG3X0yMDktbWSWZQfGPj8EPJRluemErhkiIiIiIvWStA/ysEzpZtaa0EXiPzmMSURERESkYOrVB9ndVwJ/BC7PTTgiIiIiIoWVi5v0tiN0txARERER2eIlvUnvqAzJzYFvE56yNyWXQYmIiIiIFErSm/QmEW7Ss7T0CsKQaiM2W0NEREREZAuUtIHcNUPaGuBTT/KsahERERGRLUTSUSw+aOhARERERESKQaKb9Mzsl2Z2bZZ5v4se/iEiIiIissVLOorFz4H5WebNi+aLiIiIiGzxkjaQdyF7A/l9oEtOohERERERKbCkDeQvgD2yzNsDWJ6bcERERERECitpA/kxYJSZ7RVPNLMewJWEod5ERERERLZ4SYd5uwz4LvCamb0GfAx0BHoBc4BLGyY8EREREZH8SnQG2d0/B/YFhgPvAi2j958B+7n7Fw0WoYiIiIhIHiU9g4y7rwFuiV4iIiIiIiUp6TjIB5vZsCzzhpnZoJxGJSIiIiJSIElv0rsa6JBlXnvgmtyEIyIiIiJSWEkbyHsC07PMew3onptwREREREQKK2kDeT2wfZZ57XIUi4iIiIhIwSVtIL8EXGxmzeOJ0fSvgBdzHZiIiIiISCEkHcXickIjeb6Z3cemcZBPBNoCZzVMeCIiIiIi+ZV0HORZhHGQpwKnAb+P3l8C+rn7nKQbNLMjzOxtM5tvZlkfMGJmx5mZm1nfaPpQM5thZrOj98GxZcuiPMuj145J4xERERERiavNOMhvA6dkmmdmHdz9k5ryMLOmwI3AocBCYJqZTXT3uWnLtQHOA16NJX8GfM/dF0ePuH4S6BSbf6q7Z7uRUEREREQkkaR9kDdjZtuZ2Vlm9gyhsZtEP2C+u7/n7uuACcAxGZb7LeEs9ZpUgru/5u6Lo8k3gJZmtnVd4xcRERERycTcPfnCZi0JDdpTgMOBZsAc4J/ufl2C9Y8HjnD3s6Pp0wiPqh4RW6Y3cLm7H2dmZcBF6WeGo3zOdfdDoukywmgaG4CHgKs8Q8HM7BzgHIAOHTr0mTBhQuKyF8LKlStp3bp1ocPIGZWnuJVSeUqpLFD38gwaNGiGu/dNurzqyMJSeYpbKZWnlMoC9StP1nrS3at9EbphfA+4B1hBaIQuit5Pqmn9tLyOB26LTZ8G3BCbbgKUAV2i6TKgb1oeewLvArvG0jpF722Ap4DTa4qlT58+XuymTJlS6BBySuUpbqVUnlIqi3vdywNM91rU0a46sqBUnuJWSuUppbK416882erJrF0szGywmd0KfAI8CgwGxgMHAj0AI4xmURuLgJ1j052jtJQ2Ud5lZrYA6A9MjN2o1xl4OGoAv5tayd0XRe8rCA35frWMS0REREQEqP4mvWcAB54j9Ad+zt03AphZ2zpubxrQzcy6EhrGJwM/TM1092WER1cTbaeMqIuFmW0HPA5c6u5TY8tsBWzn7p+ZWTNgSBS7iIiIiEitVXeT3guEBvIgYBTws/oOn+bu64ERhBEo3gTud/c3zGy0mQ2tYfURwG7AyLTh3LYGnjSzWUA5oeF9a33iFBEREZHGK+sZZHcfaGY7Ec7yngyMBa6Pzuo+Tmg815q7TwYmp6WNzBZD7PNVwFVZsu1Tl1hERERERNJVO8ybuy929z+5ez+gGzAa2An4E6EP8kgzO17DrYmIiIhIqUg8DrK7v+vuv3X3PYGewB+AXYH7qf3NeiIiIiIiRalODwpx91nufqm7dwX2B/6Z27BERERERAoj8aOms3H3/wD/yUEsIiIiIiIFV+dHTYuIiIiIlCI1kEVEREREYtRAFhERERGJUQNZRERERCRGDWQRERERkZiso1iY2UZq8bQ8d2+ak4hERERERAqoumHefsmmBnIz4FfASuBR4FOgA3AM0Ar4YwPGKCIiIiKSN1kbyO5+Q+qzmf0JeBU4wd09ln4p8ADQtSGDFBERERHJl6R9kE8Hbo03jgGi6VuBH+U6MBERERGRQkjaQG4KfDvLvD1rkY+IiIiISFFL+qjpu4FrzGwrYCKhD/KOhD7Io4F/NEx4IiIiIiL5lbSBfCFQQWgM/z6Wvha4Bfh1juMSERERESmIRA1kd18HXGBmvwX2Joxg8T9gtrt/3oDxiYiIiIjkVdIzyABEjeGyhglFRERERKTwEt9cZ2Z7m9l9Zvauma01s95R+tVmdmTDhSgiIiIikj+JGshRA3gG8HXgTsKDQ1LWAr/IfWgiIiIiIvmX9Azy74Dx7n4QcHXavHKgZw5jEhEREREpmKQN5G8B90WfPW3ecmD7pBs0syPM7G0zmx89iS/bcseZmZtZ31jaZdF6b5vZ4bXNU0RERESkJklv0vsU+GaWeXsCHybJxMyaAjcChwILgWlmNtHd56Yt1wY4j/B461Rad+DkaHs7Ac+Y2e7R7BrzFBERERFJIukZ5AnAaDPbP5bmUQP1EsKDRJLoB8x39/eioeMmEB42ku63hPGW18TSjgEmuPtad38fmB/llzRPEREREZEaJT2DfAXQHXieMP4xwKOEm/aeAq5JmE8n4KPY9EJgv/gC0egYO7v742Z2cdq6r6St2yn6XG2esbzPAc4B6NChA2VlZQnDLoyVK1cWfYy1ofIUt1IqTymVBfJXHtWRhaXyFLdSKk8plQUapjxJHxSyFhhiZgcDBwPtgc+BZ9396VwFY2ZNgD8Bw3KVZ5y7/x34O0Dfvn194MCBDbGZnCkrK6PYY6wNlae4lVJ5SqkskL/yqI4sLJWnuJVSeUqpLNAw5antg0KeBZ6tx/YWATvHpjtHaSltgB5AmZlBOEM90cyG1rBudXmKiIiIiCSWdBzkk9O6O8TnXWRmJybc3jSgm5l1NbPmhJvuJqZmuvsyd2/v7l3cvQuhS8VQd58eLXeymW1tZl2BbsB/a8pTRERERKQ2kt6kdylVb5iL+wq4LEkm7r4eGAE8CbwJ3O/ub5jZ6OgscXXrvgHcD8wF/g0Md/cN2fJMEo+IiIiISLqkXSy6AXOyzHszmp+Iu08GJqeljcyy7MC06avZ/EElGfMUEREREamLpGeQvyL07c1kZ8LjpkVEREREtnhJG8jPAFeY2Y7xRDPbAbicMNSbiIiIiMgWL2kXi0sIN8y9a2b/Bj4GOgKHA18Cv26Q6ERERERE8izRGWR3/xDYB7iB0KXiyOh9LNDb3T+qZnURERERkS1G4nGQ3X0JCUerEBERERHZUiXtgywiIiIi0igkOoNsZs2A84AfEEazaJG+jLvvmJ4mIiIiIrKlSdrF4s/AT4FJwBRgXYNFJCIiIiJSQEkbyCcAl7r7HxsyGBERERGRQkvaB9mAWQ0ZiIiIiIhIMUjaQL4VOKUhAxERERERKQZJu1h8ApxqZlOApwkPB4lzd785l4GJiIiIiBRC0gby9dH7N4CDMsx3QA1kEREREdniJWogu7vGSxYRERGRRkENXxERERGRmMSPmgYws87A7mR+UMjkXAUlIiIiIlIoSZ+k1wa4HzgslRS9e2yxpjmMS0RERESkIJJ2sfgd4Qa9AwiN4+8DA4F/AO8D/RsiOBERERGRfEvaQD4KuBp4NZpe7O4vuPs5wKPAxQ0RnIiIiIhIviVtIHcAPnL3DcAqYPvYvMls6nohIiIiIrJFS9pA/ghoH31+BxgSm7cfsCaXQYmIiIiIFErSBvLTwCHR5z8Dw83sP9GT9X4L3Jl0g2Z2hJm9bWbzzezSDPPPNbPZZlZuZi+ZWfco/dQoLfXaaGY9o3llUZ6peTsmjUdEREREJC7pMG+XANsAuPtdZrYSOB5oCYwAbkmSiZk1BW4EDgUWAtPMbKK7z40tdo+7/y1afijwJ+AId78buDtK3wt4xN3LY+ud6u7TE5ZHRERERCSjpE/S+wr4Kjb9MPBwHbbXD5jv7u8BmNkE4BigsoHs7stjy7ei6lByKacAE+qwfRERERGRapl7pvZnA23M7HjC2eCzo+nTgP3cfUTacsOBC4HmwGB3fydt/rvAMe4+J5ouA9oBG4CHgKs8Q8HM7BzgHIAOHTr0mTAhf23sig3Ol1+tY4c2W1emLVmxlu22aU6zppZxnZUrV9K6det8hdjgVJ7iVkrlKaWyQN3LM2jQoBnu3jfp8oWsI+tC+7m4qTzFq5TKAvUrT9Z60t0zvoAlwKdJX9nyScvzeOC22PRpwA3VLP9D4I60tP2A2WlpnaL3NsBTwOk1xdKnTx/Pp7HPzvNdLpnkoybO8Y0bN/qoiXN8l0sm+dhn52VdZ8qUKfkLMA9UnuJWSuUppbK41708wHRPUDdneuW7jqwL7efipvIUr1Iqi3v9ypOtnqyui8WNZO7eUB+LgJ1j052jtGwmADenpZ0M3BtPcPdF0fsKM7uH0JUj8Y2D+TB80G4sXbWOcVMXMG7qAgDOHNCF4YN2K2xgIiIiIlJF1gayu49qgO1NA7qZWVdCw/hkwlniSmbWzTd1qTiaMKxcal4T4ETCE/1SaVsB27n7Z2bWjDAE3TMNEHu9mBkjh3SvbBwDjBzSHbPM3StEREREpDCSDvOWE+6+njDqxZPAm8D97v6GmY2ORqwAGGFmb5hZOaEf8hmxLA4kPLDkvVja1sCTZjYLKCc0vG9t2JLUnrszetLcKmmjJ81NdRERERERkSKRdJg3zOw7wFnA7kCL9Pnu3i9JPu4+mfD0vXjayNjn86pZtwzon5a2CuiTZNuFdOOU+YybuoAzB3Rh5JDujJ40l3FTF9CuVXNGDO5W6PBEREREJJKogWxmhxIatc8C+wNPEMZAHkAYz/j5hgqwVHy/d2cg9EVOdbdo16p5ZbqIiIiIFIekXSxGA38h9AkGuMLdBxPOJlcAZbkPrbR02q4lIwZ3q+xzbGaMGNyNTtu1LHBkIiIiIhKXtIHcnXDWeCNhZItWAO7+ATAKuLwhghMRERERybekDeQ1QJNovLiPgV1j85YThmsTEREREdniJb1J73VgD+BpQj/ky8xsEbCO0P1idsOEJyIiIiKSX0nPIF/PpoeG/B+wijBU2xRgR2B4ziMTERERESmARGeQo6HZUp8XmVkfYDfCSBZvufu6BopPRERERCSvEo+DHBf1RX6nxgVFRERERLYw1TaQo0c77w40A+a4u5tZR+AC4NvAR8A/3H1Gg0cqIiIiIpIHWfsgm1kXYBbwBuERznPNbE/gVWAYsANwAvAfMzuwoQMVEREREcmH6m7SuxZoDgwBBgKLCU/TKwc6u3t/YGfCjXpXNmiUIiIiIiJ5Ul0XiwOAX7n7EwBm9jPgLeCnqZvy3H2Nmd0A3N7gkYqIiIiI5EF1Z5C/Drwfm34vev80bbklQLtcBiUiIiIiUijVNZCNTWMfk/ZZ8qBig3PDc+8QBg0B9zC96MvVBY5MREREpHTVNMzbcWbWN/rchNBIPsHM+seW6dIQgQl8+dU6xkydx9JV6xg5pDujJ81l3NQFAIwY3K2wwYmIiIiUqJoayBdnSLskQ5rOLjeAHdpszZkDOjNu6oLKhvGZA7owfNBuhQ1MREREpIRl7WLh7k1q8Wqaz6Abk5FDum82bWYFikZERESk9FXXB1mKwOhJczebTvVJFhEREZHcUwO5iC1ZsZZxUxdw5oAuvP+7ozhzQBfGTV3AjVPmFzo0ERERkZJVUx9kKaDttmnORYd9k+GDdsPMGDmkO+1aNef7vTsXOjQRERGRkpX3M8hmdoSZvW1m883s0gzzzzWz2WZWbmYvmVn3KL2Lma2O0svN7G+xdfpE68w3s79aiXTSbdbUGDG4W2WfY7Mw3Wm7lgWOTERERKR05bWBbGZNgRuBI4HuwCmpBnDMPe6+l7v3BP4A/Ck271137xm9zo2l3wz8BOgWvY5oqDKIiIiISGnL9xnkfsB8d38velz1BOCY+ALuvjw22YoahpAzs47Atu7+ioe71+4Ejs1p1CIiIiLSaNS6gWxmTc1sg5n1rsP2OgEfxaYXRmnp2xhuZu8SziD/Mjarq5m9ZmbPm9kBsTwX1pSniIiIiEgSVtshw6JuEhVAX3efWct1jweOcPezo+nTgP3cfUSW5X8IHO7uZ5jZ1kBrd19qZn2AR4A9gd2Ba939kGidA4BL3H1IhvzOAc4B6NChQ58JEybUJvy8W7lyJa1bty50GDmj8hS3UipPKZUF6l6eQYMGzXD3vjUvGaiOLCyVp7iVUnlKqSxQv/JkrSfdvVYvoCmwAehdh3W/AzwZm74MuKya5ZsAy7LMKwP6Ah2Bt2LppwC31BRLnz59vNhNmTKl0CHklMpT3EqpPKVUFve6lweY7rWsp111ZMGoPMWtlMpTSmVxr195stWTde2DXNdRIqYB3cysq5k1B04GJlbJ2KxbbPJo4J0ofYfo7DVm9k3CzXjvufvHwHIz6x+NXnE68Ggd4xMRERGRRq7W4yC7+wbqeHOfu683sxHAk4Qz0be7+xtmNprQgp8IjDCzQwjdOL4AzohWPxAYbWYVwEbgXHf/PJr3c2A80BJ4InqJiIiIiNRa3h8U4u6TgclpaSNjn8/Lst5DwENZ5k0HeuQwTBERERFppPSoaRERERGRGDWQRURERERi1EAWEREREYlRA1lEREREJKbeDWQza2Fm38hFMCIiIiIihZaLM8hHA+/nIB8RERERkYJTFwsRERERkZis4yCb2XMJ89ghR7GIiIiIiBRcdQ8KORB4G5hbQx4tcheOiIiIiEhhVddAfgN4y91Pqi4DMzseuC+nUYmIiIiIFEh1fZBfAfonyMMBy004IiIiIiKFVd0Z5D8AjyfIYzLQNTfhiIiIiIgUVtYGsru/C7xbUwbuvhr4IJdBiYiIiIgUioZ5ExERERGJUQNZRERERCRGDWQRERERkRg1kEVEREREYtRAFhERERGJUQNZRERERCRGDWQRERERkRg1kEVEpF4WfbmaG557B3cHwN254bl3WPTl6gJHJiJSN3lvIJvZEWb2tpnNN7NLM8w/18xmm1m5mb1kZt2j9EPNbEY0b4aZDY6tUxblWR69dsxnmUREGrOHZy5kzFPzGD1pLu7O6ElzGfPUPB6eubDQoYmI1El1j5rOOTNrCtwIHAosBKaZ2UR3nxtb7B53/1u0/FDgT8ARwGfA99x9sZn1AJ4EOsXWO9Xdp+ejHCIissnwQbuxdNU6xk1dwLipCwA4c0AXhg/arbCBiYjUUb7PIPcD5rv7e+6+DpgAHBNfwN2XxyZbAR6lv+bui6P0N4CWZrZ1HmIWEZFqmBkjh3SvkjZySHfMrEARiYjUj6X6jOVlY2bHA0e4+9nR9GnAfu4+Im254cCFQHNgsLu/kyGfc939kGi6DGgHbAAeAq7yDAUzs3OAcwA6dOjQZ8KECbktYI6tXLmS1q1bFzqMnFF5ilsplaeUygJ1L8+gQYNmuHvfpMvXp478eNkaPlu5tnK6feut6di2RS2irT3t5+Km8hSvUioL1K88WetJd8/bCzgeuC02fRpwQzXL/xC4Iy1tT+BdYNdYWqfovQ3wFHB6TbH06dPHi92UKVMKHUJOqTzFrZTKU0plca97eYDpXsf6ujZ15Nhn5/kul0zyURPn+MaNG33UxDm+yyWTfOyz8+oUd1Laz8VN5SlepVQW9/qVJ1s9mdc+yMAiYOfYdOcoLZsJwM2pCTPrDDxMaAC/m0p390XR+wozu4fQlePOHMYtIiJZfL93ZyD0RU51t2jXqnlluojIlibffZCnAd3MrKuZNQdOBibGFzCzbrHJo4F3ovTtgMeBS919amz5rcysffS5GTAEmNOQhRARkU06bdeSEYO7VfY5NjNGDO5Gp+1aFjgyEZG6yesZZHdfb2YjCCNQNAVud/c3zGw04RT3RGCEmR0CVABfAGdEq48AdgNGmtnIKO0wYBXwZNQ4bgo8A9yat0KJiIiISEnJdxcL3H0yMDktbWTs83lZ1rsKuCpLtn1yFqCIiIiINGp6kp6IiIiISIwayCIiIiIiMWogi4iIiIjEqIEsIiIiIhKjBrKIiIiISIwayCIiIiIiMWogi4iIiIjE5H0cZBERkc0sWwgvXQ+LZkCnPrD/+dBWj6oWkcJQA1lERApr2UK4eQCsWwUbK+B/s2H2A/CzqWoki0hBqIuFiIgU1kvXb2ocQ3hftyqki4gUgBrIIiJSWItmbGocp2ysgEUzCxOPiDR6aiCLiEhhdeoDTZpVTWvSDDr1Lkw8ItLoqYEsIiKFtf/50LzVpkZyk2Zhev/zCxmViDRiuklPREQKq23ncEPeS9eHbhWdemsUCxEpKDWQRUSk8Np2hqPHFDoKERFAXSxERERERKpQA1lEREREJEYNZBERERGRGDWQRURERERi1EAWEREREYlRA1lEREREJMbcvdAxFISZLQE+KHQcNWgPfFboIHJI5SlupVSeUioL1L083YCX3f2I2q6oOrIgVJ7iVkrlKaWyQP3Ks4u775Ce2GgbyFsCM5vu7n0LHUeuqDzFrZTKU0plgdIrT66U2vei8hS3UipPKZUFGqY86mIhIiIiIhKjBrKIiIiISIwayMXt74UOIMdUnuJWSuUppbJA6ZUnV0rte1F5ilsplaeUygINUB71QRYRERERidEZZBERERGRGDWQRURERERi1EAuAmZ2hJm9bWbzzezSDPOHmdkSMyuPXmcXIs6kaipPtMyJZjbXzN4ws3vyHWNSCfbNn2P7ZZ6ZfVmAMBNLUJ5vmNkUM3vNzGaZ2VGFiDOpBOXZxcyejcpSZmadCxFnEmZ2u5l9amZzssw3M/trVNZZZtY73zEWiupI1ZH5ojpSdWQld9ergC+gKfAu8E2gOfA60D1tmWHADYWONYfl6Qa8Bnwtmt6x0HHXtSxpy/8CuL3Qcddz3/wd+Fn0uTuwoNBx17M8DwBnRJ8HA3cVOu5qynMg0BuYk2X+UcATgAH9gVcLHXMR7WfVkUValrTlVUcWX3lUR2Z56Qxy4fUD5rv7e+6+DpgAHFPgmOojSXl+Atzo7l8AuPuneY4xqdrum1OAe/MSWd0kKY8D20af2wKL8xhfbSUpT3fguejzlAzzi4a7vwB8Xs0ixwB3evAKsJ2ZdcxPdAWlOlJ1ZL6ojizi31a+60g1kAuvE/BRbHphlJbuuOiSwYNmtnN+QquTJOXZHdjdzKaa2StmVuvH4OZJ0n2Dme0CdGVTRVOMkpRnFPAjM1sITCac8SlWScrzOvCD6PP3gTZm1i4PsTWExMdjiVEdqToyX1RHqo6spAbyluExoIu77w08DdxR4HjqayvCJcSBhDMKt5rZdoUMKAdOBh509w2FDqSeTgHGu3tnwuWqu8xsS64nLgIOMrPXgIOARcCWvo9kc6oji5/qyOKkOjKLLXmnlopFQPxsR+corZK7L3X3tdHkbUCfPMVWFzWWh/Bf3UR3r3D394F5hD8GxSZJWVJOprgvHUKy8pwF3A/g7i8DLYD2eYmu9pL8dha7+w/cvRdweZT2Zd4izK3aHI+lRHWk6sh8UR2J6sgUNZALbxrQzcy6mllzQiUyMb5AWh+aocCbeYyvtmosD/AI4cwIZtaecDnxvTzGmFSSsmBm3wK+Bryc5/hqK0l5PgQOBjCzbxMq/yV5jTK5JL+d9rGzO5cBt+c5xlyaCJwe3andH1jm7h8XOqg8UB2pOjJfVEeqjtyk0Hcl6lV55+U8wt2ml0dpo4Gh0effAW8Q+gpNAb5V6JjrWR4D/gTMBWYDJxc65rqWJZoeBVxb6FhztG+6A1OjY60cOKzQMdezPMcD70TL3AZsXeiYqynLvcDHQAXhDOJZwLnAudF8A26Myjob6FvomItoP6uOLNKyRNOqI4u3PKojs7z0qGkRERERkRh1sRARERERiVEDWUREREQkRg1kEREREZEYNZBFRERERGLUQBYRERERiVEDWRqEmY0yM8/weibh+l2i5YfkIdYFsfjWmdlbZnZFNG5krrYxLMq/dTS9Y/QddUlbbmC0XI9cbbuGuOL7ZrWZvWlml5jZVnXI69dmNjD3UYqUHtWRm21DdaQUlVrvYJFaWAYckSGtGN0DjAW2BgYBVwJtCY/hzIXHge8AX0XTO0bbKAMWxJabGS33bo62m8QfgQeBlsAQ4FqgGXBVLfP5NXADoUwiUjPVkZuojpSiogayNKT17v5KoYNI6ONYrM+bWWfgXDO72HMwWLi7LyHB05bcfTmQ7+9sQazsU8xsT+B0al/5i0jtqI6MqI6UYqMuFpJ3ZtbRzG43s/eiS1bzzOyqmi7XmdlQM5thZqvM7Asze9XMDorNb2Jml5rZfDNbG+V7Rh3DnAG0AtpHeQ+OtrfGzD4xs5tSlwKj+c3MbIyZfRhte7GZPZwqU/zyYXTJcHa06pTU5btouSqXD82szMweyPBdXBdty6LpFmb2BzP7KNr+62Z2VB3L/jpVn2ePmV1rZrPNbKWZLTSzu83s67H5C4B2wJWxy5EDo3m53C8iJU91pOpI1ZGFpzPI0qBs835aGwgV6ufAhcAXwO6ER5HuAPw0Sz67Ei5x/QW4GGgB9AG2jy02FjiD8BjNmcChwO1mttTdJ9Uy9C7AOuBzC2cL/g08DRxHqBivBb7JpsujlwGnApcC7wNfJzzis2mGvD+Olr0bGB7Fms19wBgza+XuqwCiCv9E4P7YmZsHgX6ES5LvRvMnmllfdy+vZdm/EZUhbkfgGmAxYT/9CnjOzHq4+0bg+4RH/D5IeFwphMfkQm73i0hJUR2pOhLVkcWp0M/W1qs0X4TK3DO8Dsmw7FbAD4E1QPMorUu0/JBo+nhgaTXb2w3YCJyRln4nMK2GWBcQ+phtBWxD6GO2DHgwmj+B8Kz6prF1Tozi+040PQn4YzXbGBYt3zqa7hFND0xbbmCU3iOa3gFYD5wcW+Y70TJ9o+mDo+mD0vJ6AXighrI78Muo7G2AU4C18e1lWKcp0Cla98BY+mfAqFztF730KuWX6sjNtqE6spb7Ra+GfamLhTSkZcC+aa9XLTjfzOaa2WqggnCmYGvCf+aZzAbamtkdZnaYmbVKm38woZJ52My2Sr2AZ4GeZpbpLEXchVEcq4DHCBXn8GheP+Bhd98QW/4hQqW8fzRdDgyzcJfy3qnLevXloV/ec8BJseSTgHfdfXo0fQjwP2BqhrL3TbCZvxDKvpxwI86N7j4hvoCZHWlm/zGzZYRyL4xm7V5D3vXdLyKlTHVkPamOlIaiLhbSkNbHKqhKZnYBcB3we+B5wiXEfYEbCZcFN+Pub5vZMYTLc5OBCjN7GDgvqiDbE/5rz3YHeEc2VViZ/JNQCa4l3JCxIm3dT9Li2WBmS9l0+fIqQiX386hci8zsOnf/SzXbTGoCcJOZbQusBE4AxsfmtydcrqzIsO6GDGnprgPuJ9yRfj5wgZk94+6TAcxsX2Ai8DDhsumnhDMjr5Blf6XFVp/9IlLKVEeqjlQdWaTUQJZCOIFwae7yVIKZda9pJXd/HHjczNoCRwPXE/punUzor7ceGECohNN9WkP2n2T6QxX5mNC/rFL0X327aLu4+xpgJDDSzLoB5wLXm9nb7v7vmspWg4eBm4FjgA+AnQj97lI+BxYBx9Yx/w9TZTezFwhnoq4zsyfc3Ql955YAJ0XTmNkuCfOu734RaYxUR9aO6kjJOTWQpRBaEs5CxJ2adGV3XwbcY+Hu7O9Eyc8R/gtv6+5P5yTKTV4Fvm9m/xe7hPgDwu/npQzxvWNmFxEuP3Yn3LySbl30XtPZBdz9CzN7inDZ8APgTXefFVvkWcINISvd/a2EZcq2rQozu4JwtuR7hLMiLYGKVMUfybS/1rF5eRpyv4iUKtWRqiOlwNRAlkJ4Gvilmb1KuJv4VMKNClmZ2U8JFf2/CXcJdyOcZbkTKi8v/g2YYGZ/AKYTKqI9gd3d/ex6xHsV8BrwiJndDHQmXCJ80t1fjuJ7mDDs0WvAasINM1sR+ull8mG03BlRn7WKas7OQDgbcjvhMtwNafOeBp4Enjaz3wNvANsCPYEW7n5ZrUob+g6+RbgTfmKU//lmdj2h7+F3gR9lWO8t4Ggz+zfhMufbDbxfREqV6kjVkaojC63QdwnqVZovwh3an2WZ1xoYR7i09DlhyJshVL0zuQtV79D+DuFJS4sJd3K/T6iAt47la4T+YW8Qzr4sIfTfO72GWBcAY2pY5mDCWZI1hEteNxHdbR3Nv5hQsS0DVkTLHhObP4zYHdpR2qnAPMJZBY/SBsa/h9iybQhPmHJgjwzxbQ38P2B+lN//CH8oj66hXA6MyJB+ejSvfzT9a+Ajwg06zxD++FZZlzCk1CvRMpV3n9d1v+ilVym/VEeqjlQdWdwvi3aOiIiIiIigJ+mJiIiIiFShBrKIiIiISIwayCIiIiIiMWogi4iIiIjEqIEsIiIiIhKjBrKIiIiISIwayCIiIiIiMWogi4iIiIjE/H/PrWSPEPIb1wAAAABJRU5ErkJggg==\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsYAAAE8CAYAAADDiXdXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAABU8ElEQVR4nO3deZwU1bn/8c8DgiAoKKCXgAGNYIKIIC5R1IAriV41V+MSJY5xiQlo1GiUXyISkqgxGo3KNSZGUOOGejFIMG4MLkQNiLhhRIKjghgQRQTZ5/n9cU4PNU3PTA0zvQzzfb9e9equU6eqnqquPn361Kkqc3dERERERJq7FsUOQERERESkFKhiLCIiIiKCKsYiIiIiIoAqxiIiIiIigCrGIiIiIiKAKsYiIiIiIoAqxlskM9vGzP7XzBaamZvZI8WOqS5mNjrG2r3YsdSXmVWY2fhix5FhZuPNbH0Dl9HfzJ4xs+Xxczm+HvP2jPOUpchbFvP2bEC4Ik2CyubCKrWyuSnKfP7FjqOQtviKceKHNzNsMLOPzOx+M+tdpJgGx4Ntuzyt4qfAecCdwDDghjytp0axYjXazL5c6HVLw5hZS+AhYGfgMsIxNLOoQUVmtoOZ/dLMXjWzz81slZn9y8xuMrNeMc8sM/vYzFrVspzfxfJgz8JFL0kqm1U2NxeJxoI0Q1kjrTPfx3KTiGFzbFXsAArol8BcYGtgIHAWcLiZ7enuiwocy2DgSuB2YHkeln848Jq7/788LDut/oRtfAp4v4hxSP3tAnwFuMjdby12MBlmthfwGNAJeAC4DVgH9AFOJlQ4WgN3A78DhgKP5lhOS+BU4FV3f70gwUttVDYXVn9UNhfaEsIfoaSfAdsDl2Sl/6OR1jmY/B7LTSWGemtOFeMn3P35+P7PZvY2cCNQBlzdkAWbWTt3X9mw8BrVjsCHdWUysxZAa3dfnf+QpAnZMb4uK2YQSbHFYRLQCtjX3V/Lmv7/gF/H0fuA3wKnk6NiTKic/BdwXd4CbiTN5DuqsjlLM/ncm414DP4lmWZmZwOt3P0vueeSonH3LXogFK4OHJSVvkdM/2Mi35PAImAtMJ9QKG+dNd94YD3hNPP/ESoPrySmfwd4EfiC8A/pb8CeWfN7jqFnnN6CcLrtbWANoRAdC3RMsa2Da1j2YKBnfP9z4AfAvwitbccn9sdf4/Z8EbfhmBqWPwz4CfAesDrm3TuRb3RNcdQSe2aevoTTjJ/G/XcfsGNW3oOB+4GKuI8WEwqd7ln5WgIj47Z+kfmsgB9m5dsWuBZ4N3727xMqVm2z8rUGfgN8BKwktLh8NcYxPsXnk+qzBaYB84DdgMfjuhYD1wAtUqxnPLA+K60H8A6hZe7Ldcyb/blVZH1v6jpOMsdaWVb61wmtIavjPr4cOJPE8V9LXJfEfGek/N4/FuPbLse0uwnf4a51LGNHQqv0e/HzWhI/m8FZ+b4Sl7ko5nsX+BOwbSLPzvEYXRK3/9Uc+yez32r6ju4U4/kwHqfzCMd3i6zlnAC8BHwWj515wK1p9lshB1Q2q2xuZmVzrmXlSK/1OE1TNlHHsVxLTEcTyqbVhN+KszKff47vbq3fybpiSLOMYg3NqcU4227x9eP4OpzwpXgcWAEcSPiifBk4LWtei/leI/y4twAws0sIX9qJhB/K9sAPgelmto+7zyUczNsB3wYuIBQyEA5sgP8lFI6TgZsIp4nPA75uZge4+9patuktQsF4XVzurxPpbeP7U4AOMY5PgIrYn+8fhML4RsKXsQyYZGYnu/uDWev5MeG0502Esw6XAhPNbDd3X0f4UeoKnMvG06SZOOryF2ApcAXQC/gR8DUz2y+x7d8hnE6/HfhPzPcDYH8z6+fuq2K+UXG4A7g+7oM+wEHArQBm1gaYGpfxR0Kh1w+4EOhrZt/y+C2O+6wMeDjOsy/hi90mxXZB/T7bbQmF+2OE4+koQn/fd2McqZnZ7jHOZcAR7v6fWrLfRvjxuYKwP54jfB/YjOMkGUOfuD2fA78iFITnZpadwvGEH4D7U+a/m9CV4n8IBXQmjm3isp72uk/TP0g47TwW+DewA7A/MIDwI4SZfQ2YTvge/JFwrHcnfL87AZ+bWWfCfusE3AwsBE4CxplZZ3fPbrnO9R3tTPixbBPX8yEwCLiK8KfnvBjPYTHuaYRTteuAXYFv1rGtpURls8rmZlE2Z0t5nELdZVNdx3KudR9K+AM2n/AZtyGUL7nKyTTfybpiqM/3urCKXTPP98DGVomjgc7Al4D/JvyT3ED8Nw1sk2PenwOVJP7tsvFf0E1ZeXcmFF6/zkrfiVDI3ZNIGx2Xkf0vum9MvzcrfXhM/1HKbZ4HTMtK6xmX8UWO9T5EaGnZI5G2LeELshDYKqYNjsuYT+IfO6Gi4cDROfb7QSljzuyTqUDLRPo5Mf0HibRcn9VBMd9pibRXgL/Vsd6RhH/H2f/Iz43LOyKO7xnH78zKd1VMH1/HelJ/toSCzYHzsvLOBmak2JfjiS3GwF6EH6iXgO1TfhaZfVm2mcdJz+z5CT9Y64DeibQuhMp6mpaMTwh9gtN+77chVMKfyko/La7v9Drm7xDzXVpHvqnAKuCrOaZZfL0uLuuoxLRWhArPKqBTiu/obYSKYtes9KsIZVTvOH4DoaW4ZW1xl8KAyuZMWm2fu8rmLahszppnGokW47THKenLppzHci35Xyb88emUSPtaPP48K2/a72SNMaRdRjGGLf6uFAmTCf9UFhL6KrYBhrn7LAB3/wJC3y4z6xhbaJ4ltEDsnWN5/5s1fgLhH/p9ZtY5MxAK+BeAQ1PEeEx8zW5B+hOhAnEMDTfJ3RdkRuKFSEOBKe7+Zibd3T8n/HP/Eptu/3jf+M8f4Jn4umsjxHezu29IrovwQ1+17ZnPKsa/bdzP/yLso4GJeZcBe8RWvZqcTPh8FmV9bk/F6ZnPLbP+G7Pmv6HuTao2f9rPdh2h1SXpGeqxj83sAELhOwc43N0/TTtvjmVtznGSa965iXmXAPekDGE76nHxRjxGHgaGmNmXEpNOJ5z+nFjHIlYRWrUHx+NhEzF9CHC3u/8rRwwe3x4DvOHujyemrSMcO22Aw7Jmzf6OGqElbgqwLus4fZxQRg2J2ZcB7YBvxvmaApXNgcrm6rbYsrkGaY/TOsum+jKzroRj6S/uvjST7u5vEcqYajbjO7mJxlhGvjSnrhQXAW8QDrIlwFvJL7mZfZ3wD/NAwqmopI45ljc/azxze6GarnKvTBFjz/ha7UfW3dea2TzC3QIAMLP/ypp3hbunOS3976zxLoQf0k1+2AkVKuJ6/5lIfy8rvk/jb/AOKdZfl7ezlr3OzOZTfdu/ROh3dgzh33NSx8T7nxNODc0xs7mEAvVBd5+WyNObcBqvplNMmQvRetQQ3xIzS1Ph7Blf6/xso4Xunn0v4k9Jv49bAE8QjtNvetZFPGbWlk333ZKsH76kzTlOkvNuQ9a+i3Kl5bKc0FJWH3cDZwDfBa4zs52AIwgtQ7VekBU/l58Qflw/MrOXgb/HeTMxfyW+1nVni56ECl+25H5LyvUd3Z5wKj77yvaMzHH6v4Qf2EeB/5hZeVz3Q7EyXopUNgcqm5tH2VyTVMdpyrKpvnLuw0Tat5IJm/Gd3ERjLCNfmlPFeKZvvPK5GjPbhXCa6N9Uv3ChG+FfcXbL+gbftD9ZJs8xhL6Q+Zbd7+cXhNMWdVlVd5Y61VR5ynsLVbxa+0nCXQV+S/iBWEE4XXM/ic/K3aeb2VcIp2oPJ5xW/JGZ3ebu58VsLQj/9n9VwyrrvII8T2rax2lVElpFTyf0Z70ra/rJwListF0Ip7FL0VvA3ma2tbun/X6VE1ohTye0Bp1KuOjn7jQzu/stZjYJOJbQqnsxMNLMznL3VMvYTNnf0cwx/QCbtlRlzIeqysDehNaloYQ/AqcAl5rZQckWvRKisjlQ2dw8yuaapD5Oi1g2bc53Mi/LyKfmVDGuzbGEf6bHuHvVP24zO7Iey5gXXz/wrFtJ5eA1pFfE168CsxJxtCK0Tr2YyHtE1rzZrSRpLSGcWv5qjmmZ01zvbsZya9rGuuzOxtaQzLbvAmR+OPckXBhR5u53JvK1JbSqVQ/C/TPgXuBeM9uK8KX7gZldHT/reYQ7FzyVPW+WzHGxO6F/XGa9mda8ulTE1zSfbWM5k3Ba+g4zW+3uExLTHmfTY+ijWpbVkONkCaH/5O45puVKy+WvhJaFk0hfsa00s3uAn5rZHoQK8ofA0ynXibu/D9wC3GJm2xNOaY6JMWRa+Op6SEgFDft+LSG0mLdOcZwSW7OeiANm9kNCS/J3CHcVaEpUNqts3hLL5lzqc5zWVTZB/T7n5D7Mlp1Wn+9kTTE0xvc6b5pTH+PaZE6lVe2P+O/3J/VYxsOETuq/iPNWE7+kGZnTuB2zsk2OrxdnpZ9N+IJX3ZPV3Z/KGjar8I2nLB8j9Ems6u9lZpmrYT8kUVjUQ03bWJfzY9+6jLK4jL/F8U0+q+gn2Wlm1ik5HisMb2TFdT8wwMy+nR2ImbUxs8zp+8z6L8zKdlHuzdhE6s+2scTP9jRC7PeY2XGJaYtyHEM13jO1IcdJnPdx4FuWeKJZ/E6kvfr4NmABcL2Z9c2eGD+r3+WYL/Mj8StCH8d73b3OU+cWHt3bNpkW+2hXEI8dd/+Y0Co9zMw2qbwk+vg+CuxpZkckpm1FOJZWs7HPZE5x/z0IHGtm++ZYz7ZmtnV83yl7OhsrCx1rW0+JUtmssnmLK5trkOo4TVM2Rak/Zw936HkFOD352cTj7qis7PX5TtYUQ2N8r/NGLcbB3wmnLv5mZrcRPqyT2LTfS43c/V0z+ynhiVv/NLP/I1xF/mXCKc03CAUJhKs/Aa4yswcJnfkfdfc34vp/YOGBBn9n421jZgF/btBW1uxnwJHAs2Z2CxtvCbQLcHKO/lRpzCL8WxwZv2hrgKnuvriO+XYAnjCziYTbNg0n9Lm6I05/i3CLoestPNL0P4Qrsg8kXFGb9JaZPQ/MiPl2B0YQPotMP67rCKeuHjKzvxD667Ui9Pc6CTiRcBX5a2Z2N/A9M2vHxlsCHcHG20rVqFifbewHeBKhxXWCmR2bvAisnhpynIwiFLDPxHnXEa4uryDcOaOu7fgsVuynAC+b2X2EO22sI7T0nEzoc3hx1nxvmNlswqlaSNnaTPj8y83sITaeEj4kbkPyaYDnE27X9k8z+yOhP15XQj/f4+L2/YbQneERM8vcru07hNutXerun6SIZyThOH/OzO4g3I6sPeEetycSWusqgNvNbEdCq/j7hLs9nEf4gcrVz7nUqWxW2bxFls054kh7nKYtm2o6lmu6vuIywra/EMuytmz8TJJldH2+kzljqOcyCs+LeEuMQgykvDUNofCZSTjl+xHhFEXmNi5liXzjyXp4QtZyjiZ8MZfHZc2L83w9K9+vCP/4N1D9pteZG43PJVx5uohwGjTVrbbiMmq7JdDPa5hnD8IP52eEvm613UR+k1tdxfTRWWkjCKf61sfpg2uJeXTM05fQH/ZTwu22HgD+KytvL0IrwbIY7yTCKa8KErfmIVQm/kEoXFYTTn3/nk1vSr9NXP+/CF/WpfFYGA3skMjXmtB3bjENu4l8rZ8tNd/4fTRZt82pYT2bHKOEQm5qPCaH1DF/ztu11eM46ZlrfsIP5AtsxgM+EsvoTLgH7OvxM1hN+EG+Edi1hnkujut4rR7foU6E+5m+QfguryBURi8m3iIrkbc34WEHmYd3zCe0cLdP5Pky4Q4cH8dj7DXgzHp+RzsRLriZH4+fxYRK+aVAm5jnBMKfh8zDRhYSbvm1V9ptL9SAyua0n7vK5i2kbE65rFqPU+pXNuU8lmuJ6b/jstZQ+wM+Un0n6/g+pV5GoYfMfTZFRERERJo19TEWEREREUEVYxERERERQBVjERERERFAFWMREREREaAZ366tY8eOvttuuxU7jGpWrlxJu3btih1GNaUYE5RmXIopHcWU3ubG9fLLL3/s7l3qzlkzlZHplGJMUJpxKaZ0SjEmKM248lJGFvOWGMUcevfu7aWmvLy82CFsohRjci/NuBRTOoopvc2Ni/CYZZWRBVCKMbmXZlyKKZ1SjMm9NOPKRxmprhQiIiIiIqiPsYiIiIgIoIqxiIiIiAjQjC++E2mO1q1bx4IFC1i9enXRYujQoQNvvfVW0dafSynGBHXH1aZNG7p3706rVq0KGJVI05Sr/CvF734pxgSlGVc+ykhVjEWakQULFrDtttvSs2dPzKwoMXz++edsu+22RVl3TUoxJqg9Lndn6dKlLFiwgF122aXAkYk0PbnKv1L87pdiTFCaceWjjFRXCpFmZPXq1XTq1KlolWJpPGZGp06ditr6L9KUqPxrXja3jFTFWKSZ0Y/ClkOfpUj96DvTvGzO562uFCIi0nCfLYDnb4SFL0O3gXDQhdChe7GjEhGpF7UYi0hBdezYkf79+9O3b1++853v8MUXXzR4mdOmTeMf//hHveebPHkyAwYM4MADD6RPnz7cdtttteYfPXo01113Xa15Zs+ezZQpU6rGJ02axDXXXFPv2JqUzxbArYPg5fHw4azweuugkC4iVVq2bFly5d9ee+2Vqvy76qqrmkX5p4qxiBRU27ZtmT17Nm+88QatW7fmD3/4Q6r51q9fX+O0zflhWLduHeeeey6PPvoo//jHP3jllVcYPHhwvZaRS/YPw7HHHsvll1/e4OWWtOdvhLUroXJdGK9cF8afv7GYUYmUnFIs/1599VWVfwmqGItITguXreKWqe8Qnp4ZrvC9Zeo7LFy2qtHWcfDBBzNv3jweffRR9t9/fwYMGMDhhx/Of/7zHyC00A4bNoxBgwYxbNgwlixZwgknnMC+++7Lvvvuy/Tp06moqOAPf/gDN9xwA/379+e5556joqKCQw89lH79+nHYYYfx/vvvb7Luzz//nPXr19OpUycAtt56a3bffXeAVPMPHjyYmTNnAvDxxx/Ts2dP1q5dy6hRo3jggQfo378/DzzwAOPHj2fEiBG1LresrIwLLriAAw88kF133ZWHHnoIgI8++ohDDjmkqoXpueeea7R936gWvryxUpxRuQ4WzipOPCINVMzyb/HixYDKv0WLFhWl/FPFWERymjhrAdc9MZcxk+fg7oyZPIfrnpjLxFmNc3p8/fr1PPbYY+y5554cdNBBvPjii7zyyiuccsopXHvttVX55syZw1NPPcV9993Hj3/8Yy666CJmzJjBww8/zNlnn03Pnj0577zzuOiii5g9ezYHH3ww559/PmeccQavvfYap512GhdccMEm699hhx049thj6dGjB2eeeSb33HMPlZWVAKnmz6V169aMGTOGk08+mdmzZ3PyySdXm17bchctWsTzzz/P5MmTq1pYHnzwQY466ihmz57Nq6++Sv/+/eu7mwuj20BokXWf0BatoNvexYlHpIGKWf7deOONVfkKUf6deuqpJVn+3XvvvUUp/3TxnYjkNHzIbixduZZx0ysYN70CgDMH9WT4kN0atNxVq1ZVFXAHH3wwZ511Fm+//TYnn3wyixYtYu3atdXuOXnsscfStm1bAJ566inmzJlTNW358uWsWLFik3W88MIL/N///R8Aw4YN46c//WnOWG6//XZef/11Jk+ezHXXXceTTz7J+PHjU89fX7Ut9/jjj6dFixb06dOnqsV87733ZsSIEaxbt47jjz++dCvGB10Irz+4sTtFi1bQul1IF2mCiln+7bzzzlX5C1H+PfXUUyVZ/u277758//vfL3j5pxZjEcnJzBh1TJ9qaaOO6dPg2x1l+tjNnj2bm2++mdatW3P++eczYsQIXn/9dW677bZq951s165d1fvKykpefPHFqvkXLlxI+/btU6/7qKOOon///px99tlVaXvuuScjRozgySef5OGHH069rK222qqqhaUx7iW89dZbV73PnL4dNGgQzz77LN26daOsrIy77rqrwevJiw7d4YfTYWAZfGlgeP3hdN2VQpqsYpZ/a9asqcpfiPLvoosuKsny75BDDilK+aeKsYjklDl9mJQ5rdjYPvvsM7p16wbAnXfeWWO+I488kptvvrlqfPbs2QBsu+22fP7551XpBx54IPfffz8A99xzDwcffDAAjz/+OLNnz+b2229nxYoVTJs2rdqyevToUev8ST179uTll18GqOoTlyuWpDTLTXr//ffZaaedOOecczj77LOZNauE++x26A5HXwfnTg2vqhRLE6byr/jl33vvvVeU8k8VYxHJaWz5PMZNr+DMQT159+pvceagnoybXsHY8nmNvq7Ro0fzne98h4EDB9K5c+ca8910003MnDmTfv360adPn6oruv/7v/+biRMnVl18cvPNNzNu3Dj69evH3Xffze9///tNluXuXHvttey+++4MGjSIK6+8kvHjxwOkmv+SSy7h1ltvZcCAAXz88cdV6UOGDGHOnDlVF58kpVlu0nPPPcdee+3FgAEDeOCBB/jxj39ca34RaRzNqfzr379/SZZ/06ZNK0755+7Ncujdu7eXmvLy8mKHsIlSjMm9NONqCjHNmTMn9bwLPv3Cb356rldWVrq7e2Vlpd/89Fxf8OkXDYpp+fLlDZo/H0oxJvd0ceX6TIGZrjKyIEoxJvfSjKvYMeX6rtT0HctX+ZdGUy6PCi0fZaQuvhORnLp1bMuIQ3tVjZtZtXERkS2Vyr/mS10pRERERERQxVik2fE8XDwixaHPUkSkcaliLNKMtGnThqVLl6pCtQVwd5YuXUqbNm2KHYqIyBZDfYxFmpHu3buzYMEClixZUrQYVq9eXXKVuVKMCeqOq02bNnTvrtuiiYg0loJXjM1sKPB7oCVwu7tfkyPPScBowIFX3f27ZjYEuCGR7avAKe7+iJmNB74BfBanlbn77LxthEgT1apVq2pPlSuGadOmMWDAgKLGkK0UY4LSjUtEZEtV0K4UZtYSGAt8E+gDnGpmfbLy9AJGAoPcfQ/gQgB3L3f3/u7eHzgU+AJ4IjHrpZnpqhSLiIhIqcl+Ut348eMZMWJEkaLJraKignvvvTfntMrKSi644AL69u3Lnnvuyb777su7775b6/J69uxZ7V7HuYwfP54PP/ywavzss8+u9vjrQip0H+P9gHnuPt/d1wL3A8dl5TkHGOvunwK4++IcyzkReMzdv8hrtCIiIiJN1Pr16+s9T20V4wceeIAPP/yQ1157jddff52JEyfSsWPHBka5acX49ttvp0+fPrXMkT+F7krRDfggMb4A2D8rT28AM5tO6G4x2t3/npXnFOB3WWm/NrNRwNPA5e6+Jms6ZnYucC5Aly5dqj0OsRRkP6KxFJRiTFCacSmmdBRTeoWOS2Vk/ZViTFCacRU7pg4dOmzyuOINGzbU+AhjW/4hrWf8Ly0/ms2G/+rP2n1/hG/3pQbHkVzf6tWrWbt2LR9++CEHHnggs2bNokWLFixcuJBBgwYxa9YsjjvuOPr27cv06dNZv349Y8eOZZ999mHlypVceumlzJkzh/Xr1zNy5EiOPvpo7rnnHiZNmsTKlSvZsGED99xzD8OHD6eiooK2bdty00030bdvX6666ireffdd5s+fz9KlS7nwwgspKyvj0ksvZe7cufTr149TTz21qkV7w4YNVFRU0KlTJ1auXFm1TzPb9OCDD3L99dfj7hx11FGMGTMGCBcKr1ixgo8++oiTTjqJl156CQhP8luxYgV9+vRh5syZnHrqqbRt25annnqKE044gV/96lfsvffeNS63a9eu/OAHP+CJJ56gTZs23H///ey4446b7O/Vq1fX77ir6ckf+RgILb23J8aHAbdk5ZkMTARaAbsQKtIdE9O7AkuAVllpBmwN3AmMqisWPdUpnVKMyb0041JM6Sim9DY3LvTku4IpxZjcSzOuYsdUnyff+bIP3K/+svsvOrlfuV14vfrLIb0BWrRo4XvttVfVsPPOO/vw4cPd3b2srMwnTpzoy5cv99tuu80vvvhid3f/xje+4Weffba7uz/zzDO+xx57uLv7yJEj/e6773Z3908//dR79erlK1as8HHjxnm3bt186dKl7u4+YsQIHz16tLu7P/30077XXnu5u/uVV17p/fr18y+++MKXLFni3bt394ULF3p5ebkfffTROffVBx984D169PC99trLL774Yp81a5a7uy9cuNB33nlnX7x4sa9bt86HDBniEydOdHf3Hj16+JIlS/zdd9+tit3d/be//a1feeWVVds4Y8aMqmmZ8dqWC/gDDzzg7u6XXnqp//KXv8y5z+v75LtCd6VYCOycGO8e05IWAJPcfZ27vwvMBZKPmzkJmOju6zIJ7r4obusaYByhy4aIiIhI/T1/I6xdCZWxqlG5Low/f2ODFtu2bVtmz55dNWRaPyH0qx03bhwA48aN48wzz6yaduqppwJwyCGHsHz5cpYtW8YTTzzBNddcQ//+/Rk8eDCrV6/m/fffB+CII45ghx12CJvy/PMMGzYMgEMPPZSlS5eyfPlyAI477jjatm1L586dGTJkCP/85z9rjb979+68/fbbXH311bRo0YLDDjuMp59+mhkzZjB48GC6dOnCVlttxWmnncazzz7boH0F1Lrc1q1bM3ToUAAGDhxIRUVFg9cHhe9KMQPoZWa7ECrEpwDfzcrzCHAqMM7MOhO6VsxPTD+VcHFeFTPr6u6LzMyA44E38hK9iIiIbPkWvryxUpxRuQ4WzsrbKgcNGkRFRQXPPfccGzZsoG/fvlXTQvWGauPuzsMPP8zuu+9ebdpLL71Eu3btUq0z13LrsvXWW/PNb36Tb37zm+y000488sgjHH744XXOt9VWW1FZWVk1vnr16lQx1qRVq1ZV8bZs2XKz+lPnUtAWY3dfD4wAHgfeAia4+5tmNsbMjo3ZHgeWmtkcoJxwt4mlAGbWk9Di/EzWou8xs9eB14HOwK/yvjEiIiKyZeo2EFq0qp7WohV02zuvq/3e977HWWedVa21GMJFbxBafzt06ECHDh046qijuPnmm6se2PTKK6/kXObBBx/MPffcA4RbQHbu3JntttsOgL/+9a+sXr2apUuXMm3aNPbdd1+23XbbGvtdz5o1q+oiucrKSl577TV69OjBfvvtxzPPPMPHH3/Mhg0buO+++/jGN75Rbd6ddtqJxYsXs3TpUtasWcPkyZOrptW0zjTLbWwFv4+xu08BpmSljUq8d+DiOGTPW0G4gC87/dBGD1RERESap4MuhNcf3NidokUraN0upOfRaaedxs9//vOqrhMZbdq0YcCAAaxbt4477rgDgCuuuIILL7yQfv36UVlZyS677FKtspkxevRovv/979OvXz+22WYb7rzzzqpp/fr1Y8iQIXz88cdcccUVfOlLX6JLly60bNmSvfbai7KyMi666KKq/IsXL+acc85hzZpwf4P99tuPESNG0KZNG6655hqGDBmCu3P00Udz3HHVbzrWqlUrRo0axX777Ue3bt346le/WjWtrKyM8847j7Zt2/LCCy9UpXft2rXO5Ta6mjofb+mDLixJpxRjci/NuBRTOoopPV18V12+PqcFn37hNz891ysrK93dvbKy0m9+eq4v+PSLosXUUKUYV7FjqtfFd+7hQrvJP3G/bUh4beCFd2k8+OCDfvLJJ1dLy74wrbFceeWV/tvf/jZ1/lr3VZGkiam+F9/pkdAiItKsTZy1gOuemMvSlWsZdUwfxkyew7jpFQCMOLRX7TPLlqtDdzj6uoKt7vzzz+exxx5jwoQJBVunbEoVYxERadaGD9mNpSvXMm56RVWF+MxBPRk+ZLfiBibNys033wywSV/bfN37efTo0XlZblNX6Nu1iYiIlBQzY9Qx1Z+yNeqYPqmu0JemxeOFatI8bM7nrYqxiIg0a+7OmMlzqqWNmTxHlagtTJs2bVi6dKk+12bC3Vm6dClt2rSp13zqSiEiIs3a2PJ5jJtewZmDelbrY9ypXWv1Md6CdO/enQULFrBkyZKqtNWrV9e74pRvpRgTlGZcdcXUpk0bunfvXq9lqmIsIiLN2rf3Dj+cw4fsVtWtolO71lXpsmVo1aoVu+yyS7W0adOmMWDAgCJFlFspxgSlGVc+YlLFWEREmrVuHdtWaxk2M7UUizRT6mMsIiIiIoIqxiIiIiIigCrGIiIiIiKAKsYiIiIiIoAqxiIiIiIigCrGIiIiIiKAKsYiIiIiIoAqxiIiIiIigCrGIiIiIiKAKsYiIiIiIoAqxiIiIiIigCrGIiIiIiJAESrGZjbUzN42s3lmdnkNeU4yszlm9qaZ3ZtI32Bms+MwKZG+i5m9FJf5gJm1LsS2iIiIiMiWI1XF2MweNrNvmVmDKtJm1hIYC3wT6AOcamZ9svL0AkYCg9x9D+DCxORV7t4/Dscm0n8D3ODuuwGfAmc1JE4REZFSsXDZKm6Z+g7uDoC7c8vUd1i4bFWRIxPZ8qSt6HYCHgUWmNk1Zrb7Zq5vP2Ceu89397XA/cBxWXnOAca6+6cA7r64tgWamQGHAg/FpDuB4zczPhERkZIycdYCrntiLmMmz8HdGTN5Dtc9MZeJsxYUOzSRLc5WaTK5+2Az2xUoA4YBl5rZS8AdwAPu/nnK9XUDPkiMLwD2z8rTG8DMpgMtgdHu/vc4rY2ZzQTWA9e4+yOESvsyd1+fWGa3lPGIiIiUtOFDdmPpyrWMm17BuOkVAJw5qCfDh+xW3MBEtkCWOTVTr5nMDgPOAL4NGPAwMM7dp9Ux34nAUHc/O44PA/Z39xGJPJOBdcBJQHfgWWBPd19mZt3cfWGspE8FDgM+A16M3Sgws52Bx9y9b471nwucC9ClS5eBEyZMqPe259OKFSto3759scOophRjgtKMSzGlo5jS29y4hgwZ8rK771Pf+VRG1l8hY3p94WdV7/fs1qHWvM19X6WlmNIrxbjyUka6e70HYBvgTGAmUEloBa4EZgMDapnvAODxxPhIYGRWnj8AZybGnwb2zbGs8cCJhIr5x8BWudZR09C7d28vNeXl5cUOYROlGJN7acalmNJRTOltblzATN+Mst1VRtZbIWKqrKz00ZPe8B6XTa4aRk96wysrK4saV30ppnRKMSb30owrH2VkvS6mM7NvmNk44CPgeuCfsdK6M9AXWArcVcsiZgC94l0kWgOnAJOy8jwCDI7r60zoWjHfzLY3s60T6YOAOXEDy2MlGUJL9l/rs10iIiKlamz5PMZNr+DMQT159+pvceagnoybXsHY8nnFDk1ki5Oqj7GZjQK+B+xK6NowHHjQ3Vdn8rj7HDO7AniupuW4+3ozGwE8Tug/fIe7v2lmYwi190lx2pFmNgfYAFzq7kvN7EDgNjOrJFw0eI27z4mLvgy438x+BbwC/Lke+6DeFi5bxcRZCxg+ZDfMDHdnbPk8vr13d7p1bJvPVYuISDPz7b27A1T95ow6pg+d2rWuSheRxpOqYgz8gHC3hzvcvba/qP8Cvl/bgtx9CjAlK21U4r0DF8chmecfwJ41LHM+4Y4XBZG5QnjpyrWMOqYPYybPqbogYsShvQoVhoiINAPdOrat9ttiZvqtEcmTtBXjnd29sq5M7v4JoQK9RdMVwiIiIiJbnrR9jIeYWVmuCWZWZmZDGi+k0pc5lZU06pg+hFsqi4iIiEhTlLZi/GtgpxqmdQauapxwmgaPN1hPytx4XURERESaprQV4z0It2bL5RXC452bDV0hLCIiIrLlSdvHeD2wQw3TOjVSLE2GrhAWERER2fKkbTF+nvAY6NbJxDj+E2q5RduWKHOFcKZPceYKYd2qTURERKTpStti/DNC5XiemT0ALAK6Eh7b3AE4Kz/hiYiIiIgURqoWY3d/DdgXmA4MA34TX58H9nP3N/IWobBw2SpumfpO1cV97s4tU99h4bJVRY5MREREZMuRtsUYd38bODWPsUgN9EARERERkfxLXTGW4tEDRURERETyL3XF2MwOIPQl7g20yZ7u7gV7JHNzk7nzRaZSDHqgiIiIiEhjS9XH2MyOAJ4FugMHAUuAFcBehNu1qY9xHumBIiIiIiL5l/Z2bWOA3wNHx/Er3P1QQuvxOmBa44cmGXqgiIiIiEj+pe1K0Qf4OVAJONAOwN3fM7PRwC+Au/IRoOiBIiIiIiKFkLbFeDXQwsO5+0XAVxLTlhO6WEie6IEiIiIiIvmXtsX4VWB34EngaWCkmS0E1hK6Wbyen/BERERERAojbYvxjYQuFAD/D1gJPA6UAzsCwxs9MhERERGRAkr75Lsp7j42vl8IDCS0IPcHdnP3l/MWoYiIiEgzpifgFk6dFWMza2Nmc81saCbNg3fc/TV3X5vfEEVERESar8wTcDO3ah0zeQ7XPTGXibMWFDu0LU6dfYzdfbWZdSTckUJERERECkhPwC2ctH2M7wHOzGcgIiIiTZFOc0u+ZW7VmqQn4OZH2orx+8AhZjbDzMaY2XAz+1Fi+GHaFZrZUDN728zmmdnlNeQ5yczmmNmbZnZvTOtvZi/EtNfM7ORE/vFm9q6ZzY5D/7TxiIiINIROc0u+6Qm4hZP2dm3Xx9euhAvvsjlwa10LMbOWwFjgCGABMMPMJrn7nESeXsBIYJC7f2pmO8ZJXwDfc/d3zOxLwMtm9ri7L4vTL3X3h1Juj4iISKPQaW7Jt+QTcEcd04cxk+cwbnoFndq1ZsShvYod3hYlVcXY3dO2LNdlP2Ceu88HMLP7geOA5N+gc4Cx7v5pXPfi+Do3Ec+HZrYY6AIsa6TYRERE6i1zmjtTKQad5pbGpSfgFo4VshnezE4Ehrr72XF8GLC/u49I5HkEmAsMAloCo93971nL2Q+4E9jD3SvNbDxwALCG8ACSy919TY71nwucC9ClS5eBEyZMaPRtbIgVK1bQvn37YodRTSnGBKUZl2JKRzGlt7lxDRky5GV336e+86mMrL9MTIs+W83HKzb+7HRuvzVdO7QpelylRDGlU4oxQWnGlZcy0t3rHIBv1TWkXM6JwO2J8WHALVl5JgMTgVbALsAHQMfE9K7A28DXs9IM2JpQYR5VVyy9e/f2UlNeXl7sEDZRijG5l2ZciikdxZTe5sYFzPQUZXJtg8rIdMrLy/3mp+d6j8sm++hJb3hlZaWPnvSG97hsst/89NyixlVqFFM6pRiTe2nGlY8yMm0f48mEfsTZ54WSzc0tUyxnIbBzYrx7TEtaALzk7uuAd81sLtCL0B95O+BvwM/c/cWqINwXxbdrzGwccEmKWKSZWbhsFRNnLag6FeXujC2fx7f37k63jm2LHZ6INFE6zS2y5Ujbd3gXYNf4mhn2Jjwe+h1Ct4c0ZgC9zGwXM2sNnAJMysrzCDAYwMw6A72B+TH/ROAuz7rIzsy6xlcDjgfeSBmPNCO6clxE8qFbx7aMOLRXVZ9iM2PEob30h1ukCUp78d17OZLfA2ab2QZCBfnYFMtZb2YjgMcJLcx3uPubZjaG0Kw9KU470szmABsId5tYamanA4cAncysLC6yzN1nA/eYWRdCi/Zs4Lw02yXNi64cFxERkdqk7UpRm1eA0Wkzu/sUYEpW2qjEewcujkMyz1+Av9SwzEPThyvNla4cFxERkdo06DZssXtDGbCojqwiRZfpPpGkG6SLiIhIRqoWYzObQfUL7QBaAz2BbdHjoqUJ0A3SRUREpDZpu1K8yaYV49XAg8Aj7v5mo0Ylkge6clxERERqk/biu7I8xyGSd5krxzMyV46LiIiIQMo+xma2s5ntXcO0vc1s51zTRERERESairQX390KnF7DtO8C/9s44YiIiIiIFEfaivHXgak1TCuP00VEREREmqy0FeNt2PTiu6R2jRCLSEEtXLaKW6a+U3W7NnfnlqnvsHDZqiJHJiIiIsWQtmL8OnBqDdNOJdy1QqRJ0SOiRUREJCnt7dquAR42s62B8YQHenQFzgBOiINIk6JHRIuIiEhSqhZjd59IqAQfADwKzIivBwCnu/sj+QpQJF8y9zJO0iOiRUREmq/Uj4R297uBnYE+wCHx9cvufl+eYhPJKz0iWkRERJLSdqUAwEON4V95ikWkoPSIaBEREUlKVTE2szuAbdz9lBzT7gNWuPs5jR2cSD7pEdEiIiKSlLYrxRHAwzVMexg4qnHCESmczCOiM32KM4+I7taxbZEjExERkWJIWzHuAnxSw7RPgR0bJxwRERERkeJIWzF+j3DBXS6HALrxq4iIiIg0aWkrxuOBy8xsuJm1BzCz9mb2I+CnwO15ik9EREREpCDS3pXiN8BXgJuBm8xsJeEx0Ab8Ebg2P+GJiIiIiBRGqoqxu1cCZ5vZb4EhQCdgKTDV3efmMT4RERERkYKo732M3wbeTqaZ2UHAqe4+vDEDExEREREppNRPvksyswFmdq2ZvQc8C2xyf+Na5h1qZm+b2Twzu7yGPCeZ2Rwze9PM7k2kn2Fm78ThjET6QDN7PS7zJtMzfUVERESknlK3GJtZb+DUOGQeC/YkcCHwt5TLaAmMJdwXeQEww8wmufucRJ5ewEhgkLt/amY7xvQdgCuBfQAHXo7zfgrcCpwDvARMAYYCj6XdNhERERGRWluMzay7mV1iZi8DbwE/Bz4ALiZceHeVu09097Up17cfMM/d58d57geOy8pzDjA2Vnhx98Ux/SjgSXf/JE57EhhqZl2B7dz9xfjI6ruA41PGIyIiIiICgIW6ZI4JZs8CBxIqwC8C9wET3H2xmXUgPNhjsLs/m3plZicCQ9397Dg+DNjf3Uck8jwCzAUGAS2B0e7+dzO7BGjj7r+K+a4AVgHTgGvc/fCYfjBwmbsfk2P95wLnAnTp0mXghAkT0oZeECtWrKB9+/bFDqOaUowJSjMuxZSOYkpvc+MaMmTIy+6+T33nUxlZf6UYE5RmXIopnVKMCUozrryUke6ecwAq4/AkcDixEh2ndYjTDqlp/hqWeSJwe2J8GHBLVp7JwESgFbALoYW6I3AJ8PNEviti2j7AU4n0g4HJdcXSu3dvLzXl5eXFDmETpRiTe2nGpZjSUUzpbW5cwEyvR9mca1AZmU4pxuRemnEppnRKMSb30owrH2VkbV0phhDuUTwAeBxYZGY3m9kgQivy5lgI7JwY7x7TkhYAk9x9nbu/S2g97lXLvAvj+9qWKSIiIiJSqxorxu7+jLufB/wXcCzwFHAG4S4UcwgXwO1c0/w1mAH0MrNdzKw14W4Wk7LyPAIMBjCzzkBvYD6hcn6kmW1vZtsDRwKPu/siYLmZfT3ejeJ7wF/rGZeIiIiINHN13q7N3de7+9/c/XRgR8JdKV4C1gJ3xVuqXZZmZe6+HhhBqOS+Reiz/KaZjTGzY2O2x4GlZjYHKAcudfel7v4J8EtC5XoGMCamAfyI8FjqecC/0R0pRERERKSe6vuAj9XABGCCmW0HnECoKP+S8NjoNMuYQrilWjJtVOK9E+56cXGOee8A7siRPhPom3pDRERERESybNYDPgDcfbm7j3P3I6nex1dEREREpMnZ7Ipxkm+817CIiIiISJPUKBVjEREREZGmThVjERERERFUMRYRERERAVQxFhEREREBarldm5mNqmlaLu4+puHhiIiIiIgUR233MT4/a7wtsE18vwJoH99/EQdVjEVERESkyartkdBdMgPhkdCLgdOBdu6+HdAOGBbTjytEsCIiIiIi+ZL2yXc3AVe5+72ZBHdfBdxjZu2AscDeeYhPRERERKQg0l581xf4sIZpC4GvNU44IiIiIiLFkbZiPBe42My2TiaaWRvgYuDtxg5MRERERKSQ0nalOB+YAiwwsycJ/Yp3BI4gXJD3zfyEJyIiIiJSGKlajN39WaAXMA7oChwVX8cBveJ0EREREZEmK22LMe6+CPhpHmMRERERESma1BVjADPrAwwEdgbucPePzGw34D/u/nk+AhQRERERKYRUFWMzaw/cAZwIrIvz/R34CLgKeB+4JE8xioiIiIjkXdq7UvwOOBA4DNgWsMS0KcDQRo5LRERERKSg0nal+B/gx+5ebmYts6a9B/Ro3LBERERERAorbYtxW2BpDdO2BTY0TjgiIiIiIsWRtmI8A/heDdNOBP7ROOGIiIiIiBRH2orxFcD/mNlTwNmAA98ys7uB7wBXpl2hmQ01s7fNbJ6ZXZ5jepmZLTGz2XE4O6YPSaTNNrPVZnZ8nDbezN5NTOufNh4REREREUjZx9jdnzOzw4BrgFsIF9/9AngRONzdZ6RZTuyfPJbwxLwFwAwzm+Tuc7KyPuDuI7JiKAf6x+XsAMwDnkhkudTdH0oTh4iIiIhItvo84GM6cLCZtQW2B5a5+xf1XN9+wDx3nw9gZvcDxwHZFeO6nAg8thnrFxERERHJydy97kxm2wLt49Pvsqd1BT539xUplnMiMNTdM90jhgH7J1uHzawMuBpYAswFLnL3D7KWMxX4nbtPjuPjgQOANcDTwOXuvibH+s8FzgXo0qXLwAkTJtS57YW0YsUK2rdvX+wwqinFmKA041JM6Sim9DY3riFDhrzs7vvUdz6VkfVXijFBacalmNIpxZigNOPKSxnp7nUOwATgTzVMuw24P+VyTgRuT4wPA27JytMJ2Dq+/wEwNWt6V0KluVVWmgFbA3cCo+qKpXfv3l5qysvLix3CJkoxJvfSjEsxpaOY0tvcuICZnqJMrm1QGZlOKcbkXppxKaZ0SjEm99KMKx9lZNqL7w4B/lbDtClxehoLCY+Tzuge06q4+1Lf2Np7O+ER1EknARPdfV1inkVxW9cA4whdNkREREREUktbMe4A1NSfdzWhz3EaM4BeZraLmbUGTgEmJTPErhkZxwJvZS3jVOC+XPOYmQHHA2+kjEdEREREBEh/8d07wNFUvwtExreAf6dZiLuvN7MRwONAS+AOd3/TzMYQmrUnAReY2bHAeuAToCwzv5n1JLQ4P5O16HvMrAuhO8Vs4LyU2yUiIiIiAqSvGN8M/MHM1gLjgUWEfr1nAMOBH6ZdobtPIXS/SKaNSrwfCYysYd4KoFuO9EPTrl9EREREJJe09zH+k5ntRKiwXpyYtBr4ubv/KR/BiYiIiIgUSn3uY/wrM7uZcFu0TsBS4AV3/yxfwYmIiIiIFErqijFArAT/PU+xiIiIiIgUTeqKsZm1IdyWrTvQJmuyu/utjRmYiIiIiEghpaoYm9lBwMNAlxqyOKCKsYiIiIg0WWnvY3wTMB8YQHgqXYusoWX+QhQRERERyb+0XSl2B/7H3V/NZzAiIiIiIsWStsX4NeC/8hmIiIiIiEgxpa0Y/xC4yMy+kc9gRERERESKJW1XiieBbYCp8el3n2dncPcdGzMwEREREZFCSlsxHku484SIiIiIyBYp7SOhR+c5DhERERGRokrbx1hEREREZItWnyffHQCcBfRm0yff4e77NWJcIiIiIiIFlarF2MyOAJ4lPA76IGAJsALYC+gEvJGvAEVERERECiFtV4oxwO+Bo+P4Fe5+KKH1eB0wrfFDExEREREpnLQV4z7AY0Al4e4U7QDc/T1gNPCzfAQnIiIiIlIoaSvGq4EW7u7AIuAriWnLCV0sRERERESarLQX370K7E540MfTwEgzWwisJXSzeD0/4YmIiIiIFEbaFuMb2fiAj/8HrAQeB8qBHYHhjR6ZiIiIiEgBpX3Ax5TE+4VmNhDYDWgL/Mvd1+YpPhERERGRgtisB3x48I67v1bfSrGZDTWzt81snpldnmN6mZktMbPZcTg7MW1DIn1SIn0XM3spLvMBM2u9OdslIiIiIs1XjS3GZvajeizH3f3WujKZWUtgLHAEsACYYWaT3H1OVtYH3H1EjkWscvf+OdJ/A9zg7veb2R8IDyKpMx4RERERkYzaulLcUo/lOOkqovsB89x9PoCZ3Q8cB2RXjFMzMwMOBb4bk+4k3EJOFWMRERERSc3CHdgKtDKzE4Gh7n52HB8G7J9sHTazMuBqwtP15gIXufsHcdp6YDawHrjG3R8xs87Ai+6+W8yzM/CYu/fNsf5zgXMBunTpMnDChAn52tTNsmLFCtq3b1/sMKopxZigNONSTOkopvQ2N64hQ4a87O771Hc+lZH1V4oxQWnGpZjSKcWYoDTjyksZ6e4FG4ATgdsT48OAW7LydAK2ju9/AExNTOsWX3cFKgj3U+5MaIXO5NkZeKOuWHr37u2lpry8vNghbKIUY3IvzbgUUzqKKb3NjQuY6Q0sr1VGplOKMbmXZlyKKZ1SjMm9NOPKRxmZ9j7GAJhZd8JjoNvkqGBP2XSOTSyMFdeM7jEtuZylidHbgWsT0xbG1/lmNg0YADwMdDSzrdx9fa5lioiIiIjUJVXF2My2BSYAR2aS4muyH0bLFIuaAfQys10IlddT2Ng3OLOuru6+KI4eC7wV07cHvnD3NbH7xCDgWnd3MysntEbfD5wB/DXNdomIiIiIZKS9XdvVwJeBgwmV4m8Dg4E/A+8CX0+zkNiiO4LwcJC3gAnu/qaZjTGzY2O2C8zsTTN7FbgAKIvpXwNmxvRyQh/jzEV7lwEXm9k8QleMP6fcLhERERERIP0job8F/Bx4KY5/6O4zgGfN7HrgUuCkNAuKXS6mZKWNSrwfCYzMMd8/gD1rWOZ8wh0vREREREQ2S9oW452AD9x9A+Fx0Dskpk1hYxcLEREREZEmKW3F+APC3R8A3gGOSUzbH1jdmEGJiIiIiBRa2q4UTwKHAxOBG4A7zWwgsAY4BLg+P+GJiIiIiBRG2orxZcA2AO5+t5mtINwFoi3hYrrb8hOeiIiIiEhhpKoYu/sXwBeJ8YmE1mMRERERkS1C2j7G1ZhZRzMbaGY7NnZAIiIiIiLFUGvF2MxOMbP7zexhMzstpl0BLAL+CSyK09oVIFYRERERkbypsWJsZucA9wK7AB2AcWZ2A3Ax8P+Ao4HLgcOAn+U/VBERERGR/Kmtj/H5wI3ufjGAmZ0O3An82N1viXn+bmbrgfMIlWURERERkSaptq4UXwEeTYz/lfA46Jez8s0EejRyXCIiIiIiBVVbxbgt4Sl3GZm7UqzJyrcWaNWYQYmIiIiIFFpdd6XwlGkiIiIiIk1aXfcxfjz2IU56Oist7UNCRERERERKVm2V2l8ULAoRERERkSKrsWLs7qoYi4iIiEizsVlPvhMRERER2dKoYiwiIiIigirGIiIiIiKAKsYiIiIiIoAqxiIiIiIigCrGIiIiIiJAESrGZjbUzN42s3lmdnmO6WVmtsTMZsfh7Jje38xeMLM3zew1Mzs5Mc94M3s3MU//Am6SiIiIiGwBCvrUOjNrCYwFjgAWADPMbJK7z8nK+oC7j8hK+wL4nru/Y2ZfAl42s8fdfVmcfqm7P5TP+EVERERky1XoFuP9gHnuPt/d1wL3A8elmdHd57r7O/H9h8BioEveIhURERGRZsXcvXArMzsRGOrume4Rw4D9k63DZlYGXA0sAeYCF7n7B1nL2Q+4E9jD3SvNbDxwALAGeBq43N3X5Fj/ucC5AF26dBk4YcKERt/GhlixYgXt27cvdhjVlGJMUJpxKaZ0FFN6mxvXkCFDXnb3feo7n8rI+ivFmKA041JM6ZRiTFCaceWljHT3gg3AicDtifFhwC1ZeToBW8f3PwCmZk3vCrwNfD0rzYCtCRXmUXXF0rt3by815eXlxQ5hE6UYk3tpxqWY0lFM6W1uXMBMb2B5rTIynVKMyb0041JM6ZRiTO6lGVc+yshCd6VYCOycGO8e06q4+1Lf2Np7OzAwM83MtgP+BvzM3V9MzLMobusaYByhy4aIiIiISGqFrhjPAHqZ2S5m1ho4BZiUzGBmXROjxwJvxfTWwETgLs+6yC4zj5kZcDzwRr42QERERES2TAW9K4W7rzezEcDjQEvgDnd/08zGEJq1JwEXmNmxwHrgE6Aszn4ScAjQKfZDBihz99nAPWbWhdCdYjZwXmG2SERERES2FAWtGAO4+xRgSlbaqMT7kcDIHPP9BfhLDcs8tJHDFBEREZFmRk++ExERERFBFWMREREREUAVYxERERERQBVjERERERFAFWMREREREUAVYxERERERQBVjERERERFAFWMREREREUAVYxERERERQBVjERERERFAFWMREREREUAVYxERERERQBVjERERERFAFWMREREREUAVYxERERERQBVjERERERFAFWMREREREUAVYxERERERQBVjERERERFAFWMREREREaAIFWMzG2pmb5vZPDO7PMf0MjNbYmaz43B2YtoZZvZOHM5IpA80s9fjMm8yMyvU9oiIiIjIlqGgFWMzawmMBb4J9AFONbM+ObI+4O7943B7nHcH4Epgf2A/4Eoz2z7mvxU4B+gVh6H53RIRkfxYuGwVt0x9B3cHwN25Zeo7LFy2qsiRiYgUX77LyK0aZSnp7QfMc/f5AGZ2P3AcMCfFvEcBT7r7J3HeJ4GhZjYN2M7dX4zpdwHHA481evQiInk2cdYCrntiLktXruUb28KYyXMYN70CgBGH9ipucCIiRZbvMrLQFeNuwAeJ8QWEFuBsJ5jZIcBc4CJ3/6CGebvFYUGOdBGRJmf4kN1YunIt46ZXsMOe6xn3+krOHNST4UN2K3ZoIiJFl+8y0jJN0YVgZicCQ9397Dg+DNjf3Uck8nQCVrj7GjP7AXCyux9qZpcAbdz9VzHfFcAqYBpwjbsfHtMPBi5z92NyrP9c4FyALl26DJwwYUIet7b+VqxYQfv27YsdRjWlGBOUZlyKKR3FlM7rCz9jp7bwn1WwZ7cO9Zp3yJAhL7v7PvVdp8rI+ivFmKA041JM6ZRiTFB6ceWtjHT3gg3AAcDjifGRwMha8rcEPovvTwVuS0y7LaZ1Bf6VSK+Wr6ahd+/eXmrKy8uLHcImSjEm99KMSzGlo5hqV1lZ6aMnveE9LpvsN/3lEe9x2WQfPekNr6ysTL0MYKY3sLxWGZlOKcbkXppxKaZ0SjEm99KJK99lZKHvSjED6GVmu5hZa+AUYFIyg5l1TYweC7wV3z8OHGlm28eL7o4kVLIXAcvN7OvxbhTfA/6a7w0REcmHseXzGDe9gjMH9WTPbh04c1BPxk2vYGz5vGKHJiJSdPkuIwvax9jd15vZCEIltyVwh7u/aWZjCLX3ScAFZnYssB74BCiL835iZr8kVK4Bxni8EA/4ETAeaEu46E4X3olIk/TtvbsDoR/dM888w6hj+tCpXeuqdBGR5izfZWShL77D3acAU7LSRiXejyR0scg17x3AHTnSZwJ9GzdSEZHC69axbbUrq81Md6MQEYnyXUbqyXciIiIiIqhiLCIiIiICqGIsIiIiIgKoYiwiIiIiAqhiLCIiIiICqGIsIiIiIgIU+JHQpcTMPgfeLnYcWToDHxc7iCylGBOUZlyKKR3FlN7mxtXD3bs0ZMUqI1MrxZigNONSTOmUYkxQmnE1ehlZ8PsYl5C3vabnZBeJmc1UTOmUYlyKKR3FlF6R41IZmUIpxgSlGZdiSqcUY4LSjCsfMakrhYiIiIgIqhiLiIiIiADNu2L8x2IHkINiSq8U41JM6Sim9IoZVynuE8WUXinGpZjSKcWYoDTjavSYmu3FdyIiIiIiSc25xVhEREREpIoqxiIiIiIibIEVYzMbamZvm9k8M7s8x/Qvm1m5mb1iZq+Z2bdyTF9hZpeUSlxm1s/MXjCzN83sdTNrU8yYzKyVmd0ZY3nLzEY2RjwpY+phZk/HeKaZWffEtDPM7J04nFHsmMysf+Jze83MTm6smBoSV2L6dma2wMxuKYWY4vH2RDym5phZzxKI6dr4+b1lZjeZmTVSTHeY2WIze6OG6RbXNy/GtXdiWoOOc5WR+Y9JZWS6mFRGqoysJaailZG4+xYzAC2BfwO7Aq2BV4E+WXn+CPwwvu8DVGRNfwh4ELikFOIi3Gv6NWCvON4JaFnkmL4L3B/fbwNUAD0LFNODwBnx/aHA3fH9DsD8+Lp9fL99kWPqDfSK778ELAI6FvCYyhlXYvrvgXuBW0ohJmAacER83x7Ypsif34HA9LiMlsALwOBG2leHAHsDb9Qw/VvAY4ABXwdeaozjPOX+UBnZ8JhURqaLSWWkysia4ipKGenuW1yL8X7APHef7+5rgfuB47LyOLBdfN8B+DAzwcyOB94F3iyhuI4EXnP3VwHcfam7byhyTA60M7OtgLbAWmB5gWLqA0yN78sT048CnnT3T9z9U+BJYGgxY3L3ue7+Tnz/IbAYaNDTyBojLgAzGwjsBDzRSPE0KCYz6wNs5e5PArj7Cnf/opgxEY7zNoQfi62BVsB/GiEm3P1Z4JNashwH3OXBi0BHM+tKw49zlZGFiUllZIqYVEaqjKxJEcvILa5i3A34IDG+IKYljQZON7MFwBTgfAAzaw9cBvyilOIi/KN2M3vczGaZ2U9LIKaHgJWEf/fvA9e5e20HcGPG9CrwP/H9t4FtzaxTynkLHVMVM9uPUHj8uxFialBcZtYCuB5o1FPhDYmJcJwvM7P/i6elf2tmLYsZk7u/QPgRWBSHx939rUaIKY2a4m7oca4ysjAxqYxMF1MVlZG1x4TKyGz5KiO3uIpxGqcC4929O6Ep/u74JRgN3ODuK0osrq2Ag4DT4uu3zeywIse0H7CBcOprF+AnZrZrgWK6BPiGmb0CfANYGGMpplpjiv9i7wbOdPfKEojrR8AUd19QwFjqimkr4OA4fV/Cab2yYsZkZrsBXwO6EwrWQ83s4ALFVEwqIxsek8rI6lRGNjwmlZEFslWxA2hkC4GdE+PdY1rSWcRmdXd/wcJFGp2B/YETzexaoCNQaWar3b0xOt03JK4FwLPu/jGAmU0h9Lt5uogxfRf4u7uvAxab2XRgH0JfnrzGFE+3/Q9UtWCd4O7LzGwhMDhr3mkNjKdBMcXx7YC/AT+Lp3saS0P21QHAwWb2I0I/tdZmtsLdN7noooAxLQBmu/v8OO0RQr+xPxcxpnOAFzMVQTN7DDgAeK6BMTUk7oYe5yojCxOTysgUMcVxlZHpYlIZmS7uhh/n3gidpEtlIFT05xP+oWc6ke+RlecxoCy+/xqhT5hl5RlN415YstlxETqPzyJcwLEV8BRwdJFjugwYF9PbAXOAfgWKqTPQIr7/NTAmvt+B0Pdx+zi8C+xQ5JhaE36cLyzSsZ4zrqw8ZTTehSUN2VctY/4ucXwcMLzIMZ0cv29bEfrOPQ38dyN+hj2p+cKSo6l+Yck/G+M4T7k/VEY2PCaVkeliUhmpMrK22HpS4DLS3besinHcKd8C5hL6Kf0spo0Bjo3v+xCuonwVmA0cmWMZo2nEQr+hcQGnEy52eQO4ttgxEf5BPxhjmgNcWsCYTgTeiXluB7ZOzPt9YF4czix2TPFzWxf3XWboX+y4spZRRiMV+o3w+R1BuLvA68B4oHWRP7+WwG3AW/E4/10j7qf7CH3y1hFaPM8CzgPOi9MNGBtjfh3Yp7GO8xT7Q2VkA2NCZWTa75jKyPp9fiojC1BG6pHQIiIiIiI0z4vvREREREQ2oYqxiIiIiAiqGIuIiIiIAKoYi4iIiIgAqhiLiIiIiACqGDd5ZlZmZi+b2edm9ml8VOTvih1XYzGzc83s+HrkH29mMxu4zsFm5onhczP7l5n9ycz2ypG/wsyua8g6i8HM9jOz0Y24vLK4v9o31jILobH3g5QWlZGb5FcZmZLKyKC5lZGqGDdhZjaScE/BxwlPpfke8Ffg2GLG1cjOBY6vR/5f0niPyTyN8BSf44AbgD2BmWZ2Vla+bwM3NdI6C2k/4MpiB1ECtB+2UCojc1IZmZ7KhqBZ7Yct7ZHQzc0I4DZ3/3+JtEfN7BfFCqhYzKytu69y93834mJfc/c34vupZvYn4A7gVjN7xt3nAbj7K424zgbJ7IdixyFSIlRGRiojA5WRUhe1GDdtHYGPshM98dQWM/unmY3PzhNPp70S32dOix1mZn81s5Vm9o6ZHWlmLc3st2b2sZktNLOLcyxnppkdbWZzzOwLM/ubme1gZruZWXlc3kwz65c1bwszu9zM5pnZGjOba2ZnJKZPAwYCZyRO2ZXFaRVmdr2ZXRGfIb88GU/WenqY2X1xG74ws9fM7Lv129Xg7pXARcAG4OzE8qudJjSzPczs72b2Sdz2t8xseHK7zOyheAq0wsxWxX3WLSvua8zsdTNbYWYLzOweM/uvrDw17YcDzGySmS2KMcw2s9MS85UBN8f3mX07LTG9b4zp8zg8mL3uuphZz7jcU8xsnJktj9txepz+UzP70MyWmNlvzKxFYt7R8fMaZGazzGx13IaDstbxPTN7Pu7rT+Pxtk+OWA6J01aY2WfxMxhQ136QJq8jKiNVRqqMVBlZD2oxbtpmAeeb2fvAZHdfmiPPn4HrzWyEu68AsNC/6URgZFbe2+IwFvgp8BBwD+HRi98lPJv8ejOb7u4vJeb7MuHxkT8HtiF8if5IeM75n4BrgauB+81sj8SP0s3AGXHeWYTHXd5hZkvdfTLwI+BhwnPcfxnnSbZ2fJfwyNUfUcOxbGY7Ai8AXwCXAB8AfYGdc+Wvi7t/Gn9Uvl5LtkcJj8g8HVgD7A5sl5XngJh+MdAG+A3wCLBvIs+OwFXAh0AX4CeEVpm+8QcoI9d+6EF4VO0fgNXAIGCcmVW6+33A34Dr4zIPiPNkfjB2i/POjNuwFWH/P2pm+3n9H5f5G8JxdALhUZ13mtmAGOP3CT/svwJeAe5PzLcN8BfCsbMoxvqYmfVy90xlpydwF+G4aA2cCjwXj7P5cXsGA08C5YTjbWXcH91q2w+yRVAZqTISVEaqjKyPxnqutYbCD0A/QoHoQCXhiz8G2C6RZzvCQX5mIu37hMKoUxwfHJdxZSJPn5g2NZHWgtD68ptE2nhgPfCVRNq1cd7vJdK+FdO+Fsd3izGfkbVNdwEzEuMzgfE5tr2CUBC0yUofD8xMjF8dt79rPfZrZn/0rWH6fcBbWbFcF993jvPuWcvypxGe//7lRNqgON/QGuZpSSikHDikrv2QNa8RCu7bsj7PEcTGs6z8dwNvA60Tab0IrUBH17Keshhf+zjeM46Pyzoe1wHvAC0T6f8EHkiMj47zfjeR1h74BLimhvW3iNv5L2BUIv2FeBxZDfPl3A8amv6AykiVkSojk+tXGZliUFeKJszdXwO+RriQ5H8JX+4rCBc/tI95lhNaNcoSs5YBk3zT1pOnE+/nxdepifVVEn5kqp3OAiq8er+1TeZNpGXmPYxQ6E80s60yQ4yhv5m1rGGzq8Xr7qvryHMo8Hd3X5RieWlZLdM+IbS4/MHMTo6tMbnMcvf3MyPuPh1YTLjIIazE7Jtm9g8z+4zww7ogTuqdtaxN9oOZbW9mN5nZe4RCdh3hIp3seXM5HJgIVCY+l3cJPzCbnIJLoeq4isfjEuAZd9+QyDOPTY8rYhyZeVcQWjWS++hrZjbRzP5D+FFaR2hl6h2ntwP2B+70WMJL86EyUmVkpDJSZWRqqhg3ce6+xt0fdfcR7t6H0K+rF5C8KvjPwMFmtquZfQU4mHCBRLZlieWuzU6L1hJOa+WcL5EnOz2Tlpm3M+Ef/mdsLJTWEVoztgK65ogv239S5OlEaC1oTN1qWnf8YTyS0Gp0B/CRmT0XT4slLc4x+2LidpvZvsAkQkE/jHAKK3NqMnv/54plPHAy8NsYz74xnux5c+kMXEb1z2UdsCubd3p1Wdb42hrSsmNb4ZteJJPcR9sCT8SYLiYc1/sCryaWtT3hR7qxjwFpIlRG1kllpMpIlZEJ6mO8hXH3P5vZtcBXE2nPmtk7hFYQI/THeqI4EVb5hPAPfxChVSRbrkIxW5p/t0tJ9wOSipltT2gRuLGmPO7+L+AEM2tFKIh+A/zNzLr7xn5vuVpJdmRj4fRtQqvByZl/8WbWo6ZVZsXYBjgGGO7uf0ikp/0j/AmhFeL2HNM+TrmMxtDeNr2CPLmPDgC6A0fEfQ6AmXVI5P+UcHw12jEgTZvKyE2ojNyYrjJSVDFuysxsR3dfnJXWBejApv+Q7yBceABwV9YpmmKYSmgN6eDuT9aSL9e/5Pp4GrjAzHZy9zStJzWKheYNhLj/XFd+d19HuBDkd8C9hCvkP4mT9zazL2dOFZrZIEKB9s84vS2wLuvU1mmkszXhbNCaROzbEk4nJ5e3Nk5rk3Wa8WlgD+DlEji19m3CvstcEHUE4aIlCPsIqm/ngYR+ey8DuPtKM3sJ+J6Z3VLD9tS0H6SJUxmZispIVEaiMrKKKsZN2+tm9ldCy8ZiwhWslxCuLr4zK++dhKtatwLGFTLIXNz9bTP7A+Eq7GsJHf/bEAqb3u6eudXPv4CjzOwoQsvGuzn6/dXmBsJN/Z8zs18T+rZ9DWjn7tfWMW+/WNC0IfTHOpPQEvIDj/fnzGbhdkvXAQ8Q+hpuTzjl9qq7f5LIuoTQQnIlG6+4nuXuf4/TnwQuNLMbCVdwH0i4+rlO7v6Zmc0ARpnZckJrwOWEU7LJK78zLQg/NrOpwHJ3f5twUcc/Y3x3EFpAuhEK3PHuPi1NHI1gFfDr+Bl8SDi2WwO/j9NfBFYAf4rHUPcY+8Ks5VwOPEW4WvuPhAuNDiBcgDSZmveDNH0qI+umMlJlpMrIpLquztNQugMwnFDgf0i43UwF4Z/jV2vI/zzwfI70weS4wjimjchKmwY8lBgfT+IK55hWRuLK25jWM6Ydk0gz4ELCleJriBccUP1K7V0JX9jP4vxlMb2CeJVz1rpzxdODUAh/SvhBfBU4pZb9mtkfmWEl4QrkPwF75chfFQuhReNuQoG/mtCP7j6qX109jXCxz3nA+4TC7TFg56zl/pTwI7Uy7oNe2Z9JLfthN0Krxsq4jp8SCsSPs/b/tfH4qQSmJaZ9Ncb4SYxvHuGK7e617Ldqn3uuz7ymmLM/t0yshNOss+Px8SqJq81jvqHAGzHG1whX9k8jcYzGfN8Ano2f/zLCbYn617UfNDTtAZWRKiNVRqqMrOdgcaNlC2dmOxD+JY5w9zpPcUn+WLg5+sfufmKxYylVZjaacKx2LnYs0jyojCwdKiPrpjIyf9SVYgsX+031AX4MfE74Zy4iIqiMFJHqVDHe8g0knBJ5j3D67YsixyMiUkpURopIFXWlEBERERFBD/gQEREREQFUMRYRERERAVQxFhEREREBVDEWEREREQFUMRYRERERAeD/AxLjWrMJZtIEAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] @@ -1505,50 +1786,50 @@ } ], "source": [ - "import matplotlib.pyplot as plt\n", - "figs, axs = plt.subplots(1, 2, sharex=True, sharey=True, figsize=(10, 4.5), squeeze=True)\n", + "if lalegpl_installed:\n", + " import matplotlib.pyplot as plt\n", + " figs, axs = plt.subplots(1, 2, sharex=True, sharey=True, figsize=(10, 4.5), squeeze=True)\n", "\n", - "pareto_pipelines = [opt_trained.get_pipeline(pipeline_name=id) for id in opt_trained.summary().index.tolist()]\n", - "df_cv = opt_trained.summary()\n", + " pareto_pipelines = [\n", + " opt_trained.get_pipeline(pipeline_name=id)\n", + " for id in opt_trained.summary().index.tolist()\n", + " ]\n", + " df_cv = opt_trained.summary()\n", "\n", - "cols = ['loss1' , 'loss2']\n", - "label_dict = {'loss1' : '1 - Balanced Accuracy', 'loss2' : 'False Positive Rate'}\n", + " cols = ['loss1' , 'loss2']\n", + " label_dict = {'loss1' : 'Balanced Accuracy', 'loss2' : 'Symmetric Disparate Impact'}\n", "\n", - "# Let's put loss2 on X-axis\n", - "ax = axs[0]\n", - "ax.scatter(df_cv['loss2'], df_cv['loss1'], s=30, marker='x', label='Pareto-Solutions')\n", - "ax.set_xlabel(label_dict['loss2'], fontsize=15)\n", - "ax.set_ylabel(label_dict['loss1'], fontsize=15)\n", - "ax.grid()\n", - "ax.legend()\n", - "ax.set_title(\"Pareto-front based on k-fold CV scores\", fontsize=17)\n", + " # Let's put loss2 on X-axis\n", + " ax = axs[0]\n", + " ax.scatter(df_cv['loss2'] * -1 + 1, df_cv['loss1'] * -1 + 1, s=30, marker='x', label='Pareto-Solutions')\n", + " ax.set_xlabel(label_dict['loss2'], fontsize=15)\n", + " ax.set_ylabel(label_dict['loss1'], fontsize=15)\n", + " ax.grid()\n", + " ax.legend()\n", + " ax.set_title(\"Pareto-front based on k-fold CV scores\", fontsize=17)\n", "\n", - "# Let's evaluate the pareto-optimal pipelines on test data\n", - "pX, pY = [], []\n", - "for pipeline in pareto_pipelines:\n", - " obj1_value = best_score[0] - scorer[0](pipeline, test_X, test_y)\n", - " obj2_value = best_score[1] - scorer[1](pipeline, test_X, test_y)\n", - " \n", - " pX.append(obj1_value)\n", - " pY.append(obj2_value)\n", + " # Let's evaluate the pareto-optimal pipelines on test data\n", + " pX, pY = [], []\n", + " for pipeline in pareto_pipelines:\n", + " pX.append(scorers[0](pipeline, test_X, test_y))\n", + " pY.append(scorers[1](pipeline, test_X, test_y))\n", "\n", - "dict = {'loss1' : pX, 'loss2' : pY}\n", - "df = pd.DataFrame(dict) \n", + " dict = {'loss1' : pX, 'loss2' : pY}\n", + " df = pd.DataFrame(dict) \n", " \n", - "# Metric values using single objective best pipeline \n", - "sopt_err = best_score[0] - scorer[0](auto_trained, test_X, test_y)\n", - "sopt_fpr = best_score[1] - scorer[1](auto_trained, test_X, test_y)\n", + " # Metric values using single objective best pipeline \n", + " sopt_err = scorers[0](auto_trained, test_X, test_y)\n", + " sopt_fpr = scorers[1](auto_trained, test_X, test_y)\n", " \n", - "\n", - "ax = axs[1]\n", - "ax.scatter(df['loss2'], df['loss1'], s=30, marker='x', label='Pareto-Solutions')\n", - "ax.scatter(sopt_fpr, sopt_err, s=30, marker='o', label='Hyperopt Solution')\n", - "ax.set_xlabel(label_dict['loss2'], fontsize=15)\n", - "ax.grid()\n", - "ax.legend()\n", - "ax.set_title(\"Pareto-front based on Test data\", fontsize=17)\n", - "plt.tight_layout()\n", - "plt.show()\n" + " ax = axs[1]\n", + " ax.scatter(df['loss2'], df['loss1'], s=30, marker='x', label='Pareto-Solutions')\n", + " ax.scatter(sopt_fpr, sopt_err, s=30, marker='o', label='Hyperopt Solution')\n", + " ax.set_xlabel(label_dict['loss2'], fontsize=15)\n", + " ax.grid()\n", + " ax.legend()\n", + " ax.set_title(\"Pareto-front based on Test data\", fontsize=17)\n", + " plt.tight_layout()\n", + " plt.show()" ] } ], @@ -1568,7 +1849,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.12" + "version": "3.7.13" } }, "nbformat": 4,