From ad149b439dd0cb1e4e0473930a8363d649e5bbb7 Mon Sep 17 00:00:00 2001 From: Jimmy Royer Date: Wed, 2 Oct 2024 17:48:45 -0400 Subject: [PATCH 1/9] Added timer for init --- application.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/application.py b/application.py index 7235426b1e..401d8bf024 100644 --- a/application.py +++ b/application.py @@ -2,6 +2,9 @@ from __future__ import print_function import os +import time + +start_time = time.time() import newrelic.agent # See https://bit.ly/2xBVKBH from apig_wsgi import make_lambda_handler @@ -40,6 +43,9 @@ print("========================================================") print("") +end_time = time.time() +elapsed_time = end_time - start_time +print(f"Elapsed time: {elapsed_time:.2f}s") def handler(event, context): newrelic.agent.initialize() # noqa: E402 From 24fd2cdb4fc2032512f03e472218e3d296af8671 Mon Sep 17 00:00:00 2001 From: Jimmy Royer Date: Wed, 2 Oct 2024 17:53:10 -0400 Subject: [PATCH 2/9] Added profiling code around library init --- application.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/application.py b/application.py index 401d8bf024..fa4ad43401 100644 --- a/application.py +++ b/application.py @@ -4,6 +4,21 @@ import os import time +# Check if profiling should be enabled +enable_profiling = os.getenv('NOTIFY_PROFILE') is not None + +if enable_profiling: + print("Profiling enabled") + import cProfile + import pstats + from pstats import SortKey + + # Create a cProfile.Profile object + profiler = cProfile.Profile() + # Start profiling + profiler.enable() + +# Timer start for initialization. start_time = time.time() import newrelic.agent # See https://bit.ly/2xBVKBH @@ -43,10 +58,22 @@ print("========================================================") print("") +# Timer end for initialization. end_time = time.time() elapsed_time = end_time - start_time print(f"Elapsed time: {elapsed_time:.2f}s") +if enable_profiling: + # Stop profiling + profiler.disable() + # Dump profiling results to a file + profiler.dump_stats('profile_results.prof') + # Analyze profiling results + with open('profile_report.txt', 'w') as f: + stats = pstats.Stats('profile_results.prof', stream=f) + stats.sort_stats(SortKey.CUMULATIVE) + stats.print_stats() + def handler(event, context): newrelic.agent.initialize() # noqa: E402 newrelic.agent.register_application(timeout=20.0) From 1e9e1545023ddb59860ab0206e7628c403013e05 Mon Sep 17 00:00:00 2001 From: Jimmy Royer Date: Wed, 2 Oct 2024 18:15:43 -0400 Subject: [PATCH 3/9] Reduced # of gunicorn workers --- gunicorn_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn_config.py b/gunicorn_config.py index e43a7cb288..6fa3141605 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -7,7 +7,7 @@ newrelic.agent.initialize() # noqa: E402 -workers = 4 +workers = 1 worker_class = "gevent" worker_connections = 256 bind = "0.0.0.0:{}".format(os.getenv("PORT")) From 1175bca8afa263e41074beb43c7a7513bd475d7b Mon Sep 17 00:00:00 2001 From: Jimmy Royer Date: Wed, 2 Oct 2024 18:20:32 -0400 Subject: [PATCH 4/9] Upgrade newrelic to version 8.11.0 with active profiler --- poetry.lock | 34 +++++++++++++++++----------------- pyproject.toml | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/poetry.lock b/poetry.lock index 2a806ae3c5..cee405c7a5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2562,26 +2562,26 @@ test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] [[package]] name = "newrelic" -version = "8.10.0" +version = "8.11.0" description = "New Relic Python Agent" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" files = [ - {file = "newrelic-8.10.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:cf3b67327e64d2b50aec855821199b2bc46bc0c2d142df269d420748dd49b31b"}, - {file = "newrelic-8.10.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:9601d886669fe1e0c23bbf91fb68ab23086011816ba96c6dd714c60dc0a74088"}, - {file = "newrelic-8.10.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:55a64d2abadf69bbc7bb01178332c4f25247689a97b01a62125d162ea7ec8974"}, - {file = "newrelic-8.10.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:b6cddd869ac8f7f32f6de8212ae878a21c9e63f2183601d239a76d38c5d5a366"}, - {file = "newrelic-8.10.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9af0130e1f1ca032c606d15a6d5558d27273a063b7c53702218b3beccd50b23"}, - {file = "newrelic-8.10.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2fd24b32dbf510e4e3fe40b71ad395dd73a4bb9f5eaf59eb5ff22ed76ba2d41"}, - {file = "newrelic-8.10.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2567ba9e29fd7b9f4c23cf16a5a149097eb0e5da587734c5a40732d75aaec189"}, - {file = "newrelic-8.10.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9c9f7842234a51e4a2fdafe42c42ebe0b6b1966279f2f91ec8a9c16480c2236"}, - {file = "newrelic-8.10.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:365d3b1a10d1021217beeb28a93c1356a9feb94bd24f02972691dc71227e40dc"}, - {file = "newrelic-8.10.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecd0666557419dbe11b04e3b38480b3113b3c4670d42619420d60352a1956dd8"}, - {file = "newrelic-8.10.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:722072d57e2d416de68b650235878583a2a8809ea39c7dd5c8c11a19089b7665"}, - {file = "newrelic-8.10.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dbda843100c99ac3291701c0a70fedb705c0b0707800c60b93657d3985aae357"}, - {file = "newrelic-8.10.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ed36fb91f152128825459eae9a52da364352ea95bcd78b405b0a5b8057b2ed7"}, - {file = "newrelic-8.10.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc975c29548e25805ead794d9de7ab3cb8ba4a6a106098646e1ab03112d1432e"}, - {file = "newrelic-8.10.0.tar.gz", hash = "sha256:8a2271b76ea684a63936302579d6085d46a2b54042cb91dc9b0d71a0cd4dd38b"}, + {file = "newrelic-8.11.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:41cf36017b1b54187c5b422fff32528abc4f2043227397f7a242864c31939c37"}, + {file = "newrelic-8.11.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:2e70371c8b5fcfca6e7f57395f205ed6b38228b4a5594f49f7aae4d978aa9eef"}, + {file = "newrelic-8.11.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:18dae1606f79a92c21967d11cbd4f42c03eb5b04b2f6b86a954b5e6a736113d7"}, + {file = "newrelic-8.11.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:cc169ad19cb3b44ec028df08d172327b72d0137d62935c87cbbef173f710c2ed"}, + {file = "newrelic-8.11.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d95eade883fb3aa0a2a2d04f1dd85f0df04d31bfc32f78139fe91c0362c1987e"}, + {file = "newrelic-8.11.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:313aa30918e54bfc226229d3b4fa7f8879fa5beab98ed46218ef36265b7205e5"}, + {file = "newrelic-8.11.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a649068a0cf5c4a7ba0f9739e52f1ad9d2882aafcfe3361fd264cbb975c5318"}, + {file = "newrelic-8.11.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c9463dd1b1407ade27a4c7ebb2137b77c51e23498cec7ee2671fff3fd33fd73"}, + {file = "newrelic-8.11.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc4e0fe04ad4f1fc19cb86844d73e834d99dd613dd8a4f2e7be204193d5b8dc8"}, + {file = "newrelic-8.11.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d5a3921c73913ea91407a6b2cda4e68aa3e52d7ceba77365d9b090b184f7e15"}, + {file = "newrelic-8.11.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8954dcfa78cb5d0398490938f3a84f29d53554bb498bb50bf232715a3577a68"}, + {file = "newrelic-8.11.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e32b7c92ed8922df599769270dbc5a322ca78e29b5c89083b2b39af80a44361"}, + {file = "newrelic-8.11.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68fec20006b312a3c93de82e91cb39be44ac73ba08d15552fff5206dfabda227"}, + {file = "newrelic-8.11.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:16761522dde9c05146078f7729aefbf05cea7fc42c703c0b756618168939f872"}, + {file = "newrelic-8.11.0.tar.gz", hash = "sha256:59cb3da7e5dd526e7f8696a5ed704d06c43bc2d4da840897534e9a51da7266eb"}, ] [package.extras] @@ -4581,4 +4581,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "~3.10.9" -content-hash = "caea27c85fc7d2332370c9708b2b0531324946b0e5018c802e7f8398ed8176d1" +content-hash = "4e5d51b76ace625f74291aa9cb4cdc5e985d240fffe0eda081e0151093e9ca16" diff --git a/pyproject.toml b/pyproject.toml index 137d988fae..2bf2bae8ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ PyYAML = "6.0.1" cachelib = "0.12.0" SQLAlchemy = "1.4.52" -newrelic = "8.10.0" +newrelic = "8.11.0" notifications-python-client = "6.4.1" python-dotenv = "1.0.1" pwnedpasswords = "2.0.0" From d4eee695b85ddc7cb01b824dfe18f8d43dddb3b4 Mon Sep 17 00:00:00 2001 From: Jimmy Royer Date: Wed, 2 Oct 2024 18:25:42 -0400 Subject: [PATCH 5/9] Disable the flake8 code style check --- application.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/application.py b/application.py index fa4ad43401..1b626e2fbd 100644 --- a/application.py +++ b/application.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# flake8: noqa from __future__ import print_function import os @@ -74,6 +75,7 @@ stats.sort_stats(SortKey.CUMULATIVE) stats.print_stats() + def handler(event, context): newrelic.agent.initialize() # noqa: E402 newrelic.agent.register_application(timeout=20.0) From 160072a8cb67bcdf50e034a9e3529daefe794269 Mon Sep 17 00:00:00 2001 From: Jimmy Royer Date: Wed, 16 Oct 2024 17:33:30 -0400 Subject: [PATCH 6/9] Added profiling tools and better profiling wrap --- .devcontainer/Dockerfile | 1 + .devcontainer/devcontainer.json | 3 ++- .devcontainer/docker-compose.yml | 4 ---- application.py | 41 +++----------------------------- gunicorn_config.py | 31 ++++++++++++++++++++++++ newrelic.ini | 5 ++-- poetry.lock | 13 +++++++++- pyproject.toml | 1 + 8 files changed, 53 insertions(+), 46 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 49963dcbfa..2f1c508b96 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -17,6 +17,7 @@ RUN apt-get update \ exa \ fd-find \ git \ + graphviz \ iproute2 \ less \ libsodium-dev \ diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 90716e9912..acd8fcb13e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -24,7 +24,8 @@ "donjayamanne.python-extension-pack", "eamodio.gitlens", "GitHub.copilot", - "GitHub.copilot-labs", + "github.copilot-chat", + "github.vscode-pull-request-github", "googlecloudtools.cloudcode", "kaiwood.center-editor-window", "matangover.mypy", diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index a012195183..81fc84924f 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -12,10 +12,6 @@ services: volumes: - ..:/workspace:cached command: sleep infinity - ports: - - 8000:8000 - - 8001:8001 - - 6011:6011 links: - db diff --git a/application.py b/application.py index 1b626e2fbd..01dcb2afcc 100644 --- a/application.py +++ b/application.py @@ -1,26 +1,7 @@ #!/usr/bin/env python -# flake8: noqa from __future__ import print_function import os -import time - -# Check if profiling should be enabled -enable_profiling = os.getenv('NOTIFY_PROFILE') is not None - -if enable_profiling: - print("Profiling enabled") - import cProfile - import pstats - from pstats import SortKey - - # Create a cProfile.Profile object - profiler = cProfile.Profile() - # Start profiling - profiler.enable() - -# Timer start for initialization. -start_time = time.time() import newrelic.agent # See https://bit.ly/2xBVKBH from apig_wsgi import make_lambda_handler @@ -44,8 +25,8 @@ app = create_app(application) -xray_recorder.configure(service='Notify-API', context=NotifyContext()) -XRayMiddleware(app, xray_recorder) +#xray_recorder.configure(service="Notify-API", context=NotifyContext()) +#XRayMiddleware(app, xray_recorder) apig_wsgi_handler = make_lambda_handler(app, binary_support=True) @@ -59,24 +40,8 @@ print("========================================================") print("") -# Timer end for initialization. -end_time = time.time() -elapsed_time = end_time - start_time -print(f"Elapsed time: {elapsed_time:.2f}s") - -if enable_profiling: - # Stop profiling - profiler.disable() - # Dump profiling results to a file - profiler.dump_stats('profile_results.prof') - # Analyze profiling results - with open('profile_report.txt', 'w') as f: - stats = pstats.Stats('profile_results.prof', stream=f) - stats.sort_stats(SortKey.CUMULATIVE) - stats.print_stats() - def handler(event, context): - newrelic.agent.initialize() # noqa: E402 + newrelic.agent.initialize(environment=app.config["NOTIFY_ENVIRONMENT"]) # noqa: E402 newrelic.agent.register_application(timeout=20.0) return apig_wsgi_handler(event, context) diff --git a/gunicorn_config.py b/gunicorn_config.py index 6fa3141605..9a4f84007d 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -1,11 +1,27 @@ +# flake8: noqa import os import sys +import time import traceback +import cProfile +import pstats +from pstats import SortKey + import gunicorn # type: ignore import newrelic.agent # See https://bit.ly/2xBVKBH +# Check if profiling should be enabled +enable_profiling = os.getenv("NOTIFY_PROFILE") is not None +if enable_profiling: + profiler = cProfile.Profile() + +print("Initializing New Relic agent") +start_time = time.time() newrelic.agent.initialize() # noqa: E402 +end_time = time.time() +elapsed_time = end_time - start_time +print(f"Elapsed time: {elapsed_time:.2f}s") workers = 1 worker_class = "gevent" @@ -38,6 +54,10 @@ def on_starting(server): server.log.info("Starting Notifications API") + if enable_profiling: + print("Profiling enabled") + global profiler + profiler.enable() def worker_abort(worker): @@ -48,6 +68,17 @@ def worker_abort(worker): def on_exit(server): server.log.info("Stopping Notifications API") + if enable_profiling: + # Stop profiling + global profiler + profiler.disable() + # Dump profiling results to a file + profiler.dump_stats("profile_results.prof") + # Analyze profiling results + with open("profile_report.txt", "w") as f: + stats = pstats.Stats("profile_results.prof", stream=f) + stats.sort_stats(SortKey.CUMULATIVE) + stats.print_stats() def worker_int(worker): diff --git a/newrelic.ini b/newrelic.ini index eddd873bfd..7d776d4e43 100644 --- a/newrelic.ini +++ b/newrelic.ini @@ -191,7 +191,8 @@ error_collector.ignore_errors = app.v2.errors:BadRequestError jsonschema.excepti # [newrelic:development] -# monitor_mode = false +monitor_mode = true +log_level = debug [newrelic:staging] # app_name = Python Application (Staging) @@ -204,6 +205,6 @@ error_collector.ignore_errors = app.v2.errors:BadRequestError jsonschema.excepti # monitor_mode = false [newrelic:dev] -# monitor_mode = false +# monitor_mode = true # --------------------------------------------------------------------------- diff --git a/poetry.lock b/poetry.lock index 2a806ae3c5..8b617052bb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1605,6 +1605,17 @@ benchmarks = ["httplib2", "httpx", "requests", "urllib3"] dev = ["dpkt", "pytest", "requests"] examples = ["oauth2"] +[[package]] +name = "gprof2dot" +version = "2024.6.6" +description = "Generate a dot graph from the output of several profilers." +optional = false +python-versions = ">=3.8" +files = [ + {file = "gprof2dot-2024.6.6-py2.py3-none-any.whl", hash = "sha256:45b14ad7ce64e299c8f526881007b9eb2c6b75505d5613e96e66ee4d5ab33696"}, + {file = "gprof2dot-2024.6.6.tar.gz", hash = "sha256:fa1420c60025a9eb7734f65225b4da02a10fc6dd741b37fa129bc6b41951e5ab"}, +] + [[package]] name = "greenlet" version = "2.0.2" @@ -4581,4 +4592,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "~3.10.9" -content-hash = "caea27c85fc7d2332370c9708b2b0531324946b0e5018c802e7f8398ed8176d1" +content-hash = "5a2a8c266be9de4817a7c55eca66f530e727dd3bd716975bd1585d28d01aff4c" diff --git a/pyproject.toml b/pyproject.toml index 137d988fae..78f6d3e500 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,7 @@ aws-xray-sdk = "2.14.0" [tool.poetry.group.test.dependencies] flake8 = "6.1.0" +gprof2dot = "2024.6.6" isort = "5.13.2" moto = "4.2.14" idna = "2.10" From e0b1a6a710fd4c9559c0db06d8c22cdeb08218ba Mon Sep 17 00:00:00 2001 From: Jimmy Royer Date: Wed, 16 Oct 2024 18:48:54 -0400 Subject: [PATCH 7/9] Adding snakeviz dependency --- poetry.lock | 36 +++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 8b617052bb..7f69b585d3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3873,6 +3873,20 @@ files = [ {file = "smartypants-2.0.1-py2.py3-none-any.whl", hash = "sha256:8db97f7cbdf08d15b158a86037cd9e116b4cf37703d24e0419a0d64ca5808f0d"}, ] +[[package]] +name = "snakeviz" +version = "2.2.0" +description = "A web-based viewer for Python profiler output" +optional = false +python-versions = ">=3.7" +files = [ + {file = "snakeviz-2.2.0-py2.py3-none-any.whl", hash = "sha256:569e2d71c47f80a886aa6e70d6405cb6d30aa3520969ad956b06f824c5f02b8e"}, + {file = "snakeviz-2.2.0.tar.gz", hash = "sha256:7bfd00be7ae147eb4a170a471578e1cd3f41f803238958b6b8efcf2c698a6aa9"}, +] + +[package.dependencies] +tornado = ">=2.0" + [[package]] name = "sqlalchemy" version = "1.4.52" @@ -4030,6 +4044,26 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "tornado" +version = "6.4.1" +description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." +optional = false +python-versions = ">=3.8" +files = [ + {file = "tornado-6.4.1-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:163b0aafc8e23d8cdc3c9dfb24c5368af84a81e3364745ccb4427669bf84aec8"}, + {file = "tornado-6.4.1-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6d5ce3437e18a2b66fbadb183c1d3364fb03f2be71299e7d10dbeeb69f4b2a14"}, + {file = "tornado-6.4.1-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2e20b9113cd7293f164dc46fffb13535266e713cdb87bd2d15ddb336e96cfc4"}, + {file = "tornado-6.4.1-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ae50a504a740365267b2a8d1a90c9fbc86b780a39170feca9bcc1787ff80842"}, + {file = "tornado-6.4.1-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:613bf4ddf5c7a95509218b149b555621497a6cc0d46ac341b30bd9ec19eac7f3"}, + {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25486eb223babe3eed4b8aecbac33b37e3dd6d776bc730ca14e1bf93888b979f"}, + {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:454db8a7ecfcf2ff6042dde58404164d969b6f5d58b926da15e6b23817950fc4"}, + {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a02a08cc7a9314b006f653ce40483b9b3c12cda222d6a46d4ac63bb6c9057698"}, + {file = "tornado-6.4.1-cp38-abi3-win32.whl", hash = "sha256:d9a566c40b89757c9aa8e6f032bcdb8ca8795d7c1a9762910c722b1635c9de4d"}, + {file = "tornado-6.4.1-cp38-abi3-win_amd64.whl", hash = "sha256:b24b8982ed444378d7f21d563f4180a2de31ced9d8d84443907a0a64da2072e7"}, + {file = "tornado-6.4.1.tar.gz", hash = "sha256:92d3ab53183d8c50f8204a51e6f91d18a15d5ef261e84d452800d4ff6fc504e9"}, +] + [[package]] name = "tqdm" version = "4.66.5" @@ -4592,4 +4626,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "~3.10.9" -content-hash = "5a2a8c266be9de4817a7c55eca66f530e727dd3bd716975bd1585d28d01aff4c" +content-hash = "f5d28439fddff44d055d3a9f691b05422212655432cd3bc4e49ed33ac9e4a3d7" diff --git a/pyproject.toml b/pyproject.toml index 78f6d3e500..9037c5272f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,6 +92,7 @@ coveralls = "3.3.1" pytest-xdist = "2.5.0" freezegun = "1.5.1" requests-mock = "1.12.1" +snakeviz = "2.2.0" # optional requirements for jsonschema strict-rfc3339 = "0.7" rfc3987 = "1.3.8" From 0589d60d428e3254eda529f0cd20775a44d69452 Mon Sep 17 00:00:00 2001 From: Jimmy Royer Date: Wed, 16 Oct 2024 18:54:29 -0400 Subject: [PATCH 8/9] Changed profile report names running under gunicorn process. --- gunicorn_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gunicorn_config.py b/gunicorn_config.py index 9a4f84007d..af539cebd3 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -75,8 +75,8 @@ def on_exit(server): # Dump profiling results to a file profiler.dump_stats("profile_results.prof") # Analyze profiling results - with open("profile_report.txt", "w") as f: - stats = pstats.Stats("profile_results.prof", stream=f) + with open("profile_report-gcrn-nr8xx.txt", "w") as f: + stats = pstats.Stats("profile_results-gcrn-nr8xx.prof", stream=f) stats.sort_stats(SortKey.CUMULATIVE) stats.print_stats() From 5fd9ec5b82106262c58b370ab07de2f1a99eb85c Mon Sep 17 00:00:00 2001 From: Jimmy Royer Date: Tue, 29 Oct 2024 11:54:48 -0400 Subject: [PATCH 9/9] Added back profiling on the app side --- .devcontainer/devcontainer.json | 7 +++--- application.py | 42 +++++++++++++++++++++++++++++++-- gunicorn_config.py | 2 +- 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index acd8fcb13e..fdcd8537c2 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -23,6 +23,7 @@ "bungcip.better-toml", "donjayamanne.python-extension-pack", "eamodio.gitlens", + "fill-labs.dependi", "GitHub.copilot", "github.copilot-chat", "github.vscode-pull-request-github", @@ -48,11 +49,11 @@ }, "features": { "docker-from-docker": { - "version": "latest", + "version": "1.6.0", "moby": true }, "kubectl-helm-minikube": { - "version": "latest", + "version": "1.2.0", "helm": "latest", "minikube": "none" }, @@ -61,6 +62,6 @@ } }, "postCreateCommand": "notify-dev-entrypoint.sh", - "remoteUser": "vscode", + "remoteUser": "vscode" } \ No newline at end of file diff --git a/application.py b/application.py index 01dcb2afcc..3a846f5e24 100644 --- a/application.py +++ b/application.py @@ -2,6 +2,44 @@ from __future__ import print_function import os +import time + +# Check if profiling should be enabled +enable_profiling = os.getenv("NOTIFY_PROFILE") is not None + +if enable_profiling: + print("Profiling enabled") + import atexit + import cProfile + import pstats + from pstats import SortKey + + # Create a cProfile.Profile object + profiler = cProfile.Profile() + # Start profiling + profiler.enable() + + def close_profiling(): + # Timer end for initialization. + end_time = time.time() + elapsed_time = end_time - start_time + print(f"Elapsed time: {elapsed_time:.2f}s") + + # Stop profiling + profiler.disable() + # Dump profiling results to a file + profiler.dump_stats("profile_results-app-nr8xx.prof") + # Analyze profiling results + with open("profile_report-app-nr8xx.txt", "w") as f: + stats = pstats.Stats("profile_results-app-nr8xx.prof", stream=f) + stats.sort_stats(SortKey.CUMULATIVE) + stats.print_stats() + + atexit.register(close_profiling) + + +# Timer start for initialization. +start_time = time.time() import newrelic.agent # See https://bit.ly/2xBVKBH from apig_wsgi import make_lambda_handler @@ -25,8 +63,8 @@ app = create_app(application) -#xray_recorder.configure(service="Notify-API", context=NotifyContext()) -#XRayMiddleware(app, xray_recorder) +# xray_recorder.configure(service="Notify-API", context=NotifyContext()) +# XRayMiddleware(app, xray_recorder) apig_wsgi_handler = make_lambda_handler(app, binary_support=True) diff --git a/gunicorn_config.py b/gunicorn_config.py index af539cebd3..72ed27ce47 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -73,7 +73,7 @@ def on_exit(server): global profiler profiler.disable() # Dump profiling results to a file - profiler.dump_stats("profile_results.prof") + profiler.dump_stats("profile_results-gcrn-nr8xx.prof") # Analyze profiling results with open("profile_report-gcrn-nr8xx.txt", "w") as f: stats = pstats.Stats("profile_results-gcrn-nr8xx.prof", stream=f)