From 4dd9449c98dcbf1a71569424bdd7357ad0ada753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Loipf=C3=BChrer?= Date: Fri, 31 Jan 2025 16:32:20 +0100 Subject: [PATCH 1/3] chore: add pre-commit for ruff --- .pre-commit-config.yaml | 16 ++++++++++++++++ .readthedocs.yaml | 2 +- .vscode/settings.json | 2 +- .../database/revisions/0003_purchase_items.sql | 2 +- .../revisions/0007_invite_join_as_editor.sql | 2 +- .../revisions/0009_robust_notifications.sql | 2 +- .../revisions/0013_register_with_invite.sql | 1 - ...0015_metadata_fields_and_revision_changed.sql | 1 - .../0017_clean_non_nullable_strings.sql | 2 +- config/nginx/abrechnung | 2 +- debian/.gitignore | 2 +- debian/abrechnung-api.service | 2 +- debian/abrechnung.dirs | 2 +- debian/abrechnung.links | 2 +- debian/changelog | 4 ++-- debian/compat | 2 +- docker/Dockerfile-api | 2 +- docker/Dockerfile-devel | 2 +- docker/build_debian.sh | 2 +- docs/development/contributing.rst | 2 +- docs/requires.txt | 2 +- docs/usage/installation.rst | 6 +++--- frontend/.prettierignore | 2 +- frontend/apps/web/.browserslistrc | 2 +- frontend/apps/web/src/assets/.gitignore | 2 +- pyproject.toml | 1 + systemd/abrechnung-api.service | 2 +- 27 files changed, 43 insertions(+), 28 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..9ae4e244 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,16 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace +- repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.9.4 + hooks: + # Run the linter. + - id: ruff + args: [ --fix ] + # Run the formatter. + - id: ruff-format diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 3d59c895..d8b2df50 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -22,4 +22,4 @@ formats: # Optionally declare the Python requirements required to build your docs python: install: - - requirements: docs/requires.txt \ No newline at end of file + - requirements: docs/requires.txt diff --git a/.vscode/settings.json b/.vscode/settings.json index de288e1e..163c9840 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,3 +1,3 @@ { "python.formatting.provider": "black" -} \ No newline at end of file +} diff --git a/abrechnung/database/revisions/0003_purchase_items.sql b/abrechnung/database/revisions/0003_purchase_items.sql index 53e5d8b1..8d2b10c6 100644 --- a/abrechnung/database/revisions/0003_purchase_items.sql +++ b/abrechnung/database/revisions/0003_purchase_items.sql @@ -34,4 +34,4 @@ create table if not exists purchase_item_usage ( -- amount of shares this account has from the purchase item share_amount double precision not null -); \ No newline at end of file +); diff --git a/abrechnung/database/revisions/0007_invite_join_as_editor.sql b/abrechnung/database/revisions/0007_invite_join_as_editor.sql index 2bac0d7d..eca370c8 100644 --- a/abrechnung/database/revisions/0007_invite_join_as_editor.sql +++ b/abrechnung/database/revisions/0007_invite_join_as_editor.sql @@ -1,4 +1,4 @@ -- migration: dbcccb58 -- requires: a77c9b57 -alter table group_invite add column join_as_editor bool default false; \ No newline at end of file +alter table group_invite add column join_as_editor bool default false; diff --git a/abrechnung/database/revisions/0009_robust_notifications.sql b/abrechnung/database/revisions/0009_robust_notifications.sql index 06c26b92..fa107b43 100644 --- a/abrechnung/database/revisions/0009_robust_notifications.sql +++ b/abrechnung/database/revisions/0009_robust_notifications.sql @@ -2,4 +2,4 @@ -- requires: 156aef63 alter table transaction_revision add column version int default 0; -alter table account_revision add column version int default 0; \ No newline at end of file +alter table account_revision add column version int default 0; diff --git a/abrechnung/database/revisions/0013_register_with_invite.sql b/abrechnung/database/revisions/0013_register_with_invite.sql index 062925bc..bfac9218 100644 --- a/abrechnung/database/revisions/0013_register_with_invite.sql +++ b/abrechnung/database/revisions/0013_register_with_invite.sql @@ -2,4 +2,3 @@ -- requires: 5b333d87 alter table usr add column is_guest_user bool default false not null; - diff --git a/abrechnung/database/revisions/0015_metadata_fields_and_revision_changed.sql b/abrechnung/database/revisions/0015_metadata_fields_and_revision_changed.sql index 59c502c4..0f349df9 100644 --- a/abrechnung/database/revisions/0015_metadata_fields_and_revision_changed.sql +++ b/abrechnung/database/revisions/0015_metadata_fields_and_revision_changed.sql @@ -53,4 +53,3 @@ create table account_to_tag ( tag_id integer not null references tag (id), primary key (account_id, revision_id, tag_id) ); - diff --git a/abrechnung/database/revisions/0017_clean_non_nullable_strings.sql b/abrechnung/database/revisions/0017_clean_non_nullable_strings.sql index 629db27d..98e1c562 100644 --- a/abrechnung/database/revisions/0017_clean_non_nullable_strings.sql +++ b/abrechnung/database/revisions/0017_clean_non_nullable_strings.sql @@ -4,4 +4,4 @@ alter table transaction_history alter column description set default ''; alter table transaction_history alter column description set not null; -alter table grp alter column created_by set not null; \ No newline at end of file +alter table grp alter column created_by set not null; diff --git a/config/nginx/abrechnung b/config/nginx/abrechnung index 6d2c25b5..0d58eaa0 100644 --- a/config/nginx/abrechnung +++ b/config/nginx/abrechnung @@ -39,4 +39,4 @@ server { location / { try_files $uri /index.html =404; } -} \ No newline at end of file +} diff --git a/debian/.gitignore b/debian/.gitignore index 7526a8c7..80f02ab8 100644 --- a/debian/.gitignore +++ b/debian/.gitignore @@ -5,4 +5,4 @@ python3-abrechnung debhelper-build-stamp files *.substvars -tmp \ No newline at end of file +tmp diff --git a/debian/abrechnung-api.service b/debian/abrechnung-api.service index 5204284e..eb0ba4a6 100644 --- a/debian/abrechnung-api.service +++ b/debian/abrechnung-api.service @@ -8,4 +8,4 @@ User=abrechnung Group=abrechnung [Install] -WantedBy=multi-user.target \ No newline at end of file +WantedBy=multi-user.target diff --git a/debian/abrechnung.dirs b/debian/abrechnung.dirs index 1923a162..b151f19e 100644 --- a/debian/abrechnung.dirs +++ b/debian/abrechnung.dirs @@ -1 +1 @@ -etc/abrechnung \ No newline at end of file +etc/abrechnung diff --git a/debian/abrechnung.links b/debian/abrechnung.links index 76a06be3..467dd43c 100644 --- a/debian/abrechnung.links +++ b/debian/abrechnung.links @@ -1 +1 @@ -opt/venvs/abrechnung/bin/abrechnung usr/bin/abrechnung \ No newline at end of file +opt/venvs/abrechnung/bin/abrechnung usr/bin/abrechnung diff --git a/debian/changelog b/debian/changelog index 7955145f..d85d9f0d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,13 +2,13 @@ abrechnung (1.0.0) stable; urgency=medium * Abrechnung release 1.0.0 - -- Michael Loipführer Fri, 3 Jan 2025 16:56:40 + -- Michael Loipführer Fri, 3 Jan 2025 16:56:40 abrechnung (0.14.0) stable; urgency=medium * Abrechnung release 0.14.0 - -- Michael Loipführer Fri, 16 Aug 2024 23:57:11 + -- Michael Loipführer Fri, 16 Aug 2024 23:57:11 abrechnung (0.13.3) stable; urgency=medium diff --git a/debian/compat b/debian/compat index 9d607966..b4de3947 100644 --- a/debian/compat +++ b/debian/compat @@ -1 +1 @@ -11 \ No newline at end of file +11 diff --git a/docker/Dockerfile-api b/docker/Dockerfile-api index a635cfae..11b9b4c9 100644 --- a/docker/Dockerfile-api +++ b/docker/Dockerfile-api @@ -12,4 +12,4 @@ ADD --chmod=644 --chown=abrechnung:abrechnung docker/abrechnung.yaml /etc/abrech ADD --chmod=755 ./docker/entrypoint.py / COPY --chown=abrechnung:abrechnung ./docker/crontab /var/spool/cron/crontabs/abrechnung USER abrechnung -ENTRYPOINT ["/opt/abrechnung-venv/bin/python3", "-u", "/entrypoint.py"] \ No newline at end of file +ENTRYPOINT ["/opt/abrechnung-venv/bin/python3", "-u", "/entrypoint.py"] diff --git a/docker/Dockerfile-devel b/docker/Dockerfile-devel index 63fead8f..0f967d7d 100644 --- a/docker/Dockerfile-devel +++ b/docker/Dockerfile-devel @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1.3 FROM node:lts as build ADD frontend/ /build/ -RUN cd /build/ && npm install && npx nx build web +RUN cd /build/ && npm install && npx nx build web FROM node:lts as frontend-dev COPY --chown=node:node frontend/ /frontend/ diff --git a/docker/build_debian.sh b/docker/build_debian.sh index ceb910d9..43024822 100755 --- a/docker/build_debian.sh +++ b/docker/build_debian.sh @@ -39,4 +39,4 @@ for i in ../*.deb ../*.dsc ../*.tar.xz ../*.changes ../*.buildinfo; do [ -z "$TARGET_USERID" ] || chown "$TARGET_USERID" "$i" [ -z "$TARGET_GROUPID" ] || chgrp "$TARGET_GROUPID" "$i" mv "$i" /debs -done \ No newline at end of file +done diff --git a/docs/development/contributing.rst b/docs/development/contributing.rst index b503ce46..6df1a56b 100644 --- a/docs/development/contributing.rst +++ b/docs/development/contributing.rst @@ -15,4 +15,4 @@ As Developers Get started by setting up a local :ref:`development environment ` and reading through the :ref:`abrechnung-dev-architecture` overview and then start hacking. -`Pull requests `_ are always welcome! \ No newline at end of file +`Pull requests `_ are always welcome! diff --git a/docs/requires.txt b/docs/requires.txt index 8c10fa58..b720afa9 100644 --- a/docs/requires.txt +++ b/docs/requires.txt @@ -2,4 +2,4 @@ sphinx-autobuild sphinx-rtd-theme sphinx-autodoc-typehints -sphinxcontrib-openapi \ No newline at end of file +sphinxcontrib-openapi diff --git a/docs/usage/installation.rst b/docs/usage/installation.rst index 359e708a..bae93415 100644 --- a/docs/usage/installation.rst +++ b/docs/usage/installation.rst @@ -47,14 +47,14 @@ Then copy the ``.env.example`` file to ``.env`` and configure it to your liking cp .env.example .env vim .env -For production setups we recommend running an external postgres database but if you feel adventurous you +For production setups we recommend running an external postgres database but if you feel adventurous you can adapt the docker-compose file to also run a postgres container (which we definitely do not recommend). -In case of using an external postgres database make sure to +In case of using an external postgres database make sure to follow :ref:`the configuration instructions ` on how to create a database. Afterwards make sure to include the database configuration parameters in the ``.env`` configuration file. Then a simple simple :: - docker-compose -f docker-compose.prod.yaml up + docker-compose -f docker-compose.prod.yaml up Should suffice to get you up and running. diff --git a/frontend/.prettierignore b/frontend/.prettierignore index daffe92c..ccb5e57d 100644 --- a/frontend/.prettierignore +++ b/frontend/.prettierignore @@ -4,4 +4,4 @@ /coverage /.nx/cache -/.nx/workspace-data \ No newline at end of file +/.nx/workspace-data diff --git a/frontend/apps/web/.browserslistrc b/frontend/apps/web/.browserslistrc index f1d12df4..83f282df 100644 --- a/frontend/apps/web/.browserslistrc +++ b/frontend/apps/web/.browserslistrc @@ -13,4 +13,4 @@ last 2 Edge major versions last 2 Safari major version last 2 iOS major versions Firefox ESR -not IE 9-11 # For IE 9-11 support, remove 'not'. \ No newline at end of file +not IE 9-11 # For IE 9-11 support, remove 'not'. diff --git a/frontend/apps/web/src/assets/.gitignore b/frontend/apps/web/src/assets/.gitignore index 8e627b2f..fcd811a2 100644 --- a/frontend/apps/web/src/assets/.gitignore +++ b/frontend/apps/web/src/assets/.gitignore @@ -1,3 +1,3 @@ *.png *.svg -locales \ No newline at end of file +locales diff --git a/pyproject.toml b/pyproject.toml index d916e5c0..c645e244 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ test = [ ] dev = [ "ruff", + "pre-commit", "mypy==1.13.0", "types-PyYAML~=6.0", "pylint==3.3.2", diff --git a/systemd/abrechnung-api.service b/systemd/abrechnung-api.service index 083f174e..67f8f9e7 100644 --- a/systemd/abrechnung-api.service +++ b/systemd/abrechnung-api.service @@ -8,4 +8,4 @@ User=abrechnung Group=abrechnung [Install] -WantedBy=multi-user.target \ No newline at end of file +WantedBy=multi-user.target From c968eb68a7bdf613ad0065cbba5f536080bb46df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Loipf=C3=BChrer?= Date: Fri, 31 Jan 2025 16:34:27 +0100 Subject: [PATCH 2/3] chore(deb): remove obsolete db-clean service, remove obsolete systemd folder see #240 --- debian/abrechnung-db-clean.service | 11 ----------- debian/abrechnung-db-clean.timer | 8 -------- systemd/abrechnung-api.service | 11 ----------- systemd/abrechnung-db-clean.service | 11 ----------- systemd/abrechnung-db-clean.timer | 8 -------- systemd/abrechnung-mailer.service | 11 ----------- 6 files changed, 60 deletions(-) delete mode 100644 debian/abrechnung-db-clean.service delete mode 100644 debian/abrechnung-db-clean.timer delete mode 100644 systemd/abrechnung-api.service delete mode 100644 systemd/abrechnung-db-clean.service delete mode 100644 systemd/abrechnung-db-clean.timer delete mode 100644 systemd/abrechnung-mailer.service diff --git a/debian/abrechnung-db-clean.service b/debian/abrechnung-db-clean.service deleted file mode 100644 index eefabb52..00000000 --- a/debian/abrechnung-db-clean.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Abrechnung Database Cleanup Service -After=postgresql.service - -[Service] -ExecStart=/opt/venvs/abrechnung/bin/abrechnung db clean -User=abrechnung -Group=abrechnung - -[Install] -WantedBy=multi-user.target diff --git a/debian/abrechnung-db-clean.timer b/debian/abrechnung-db-clean.timer deleted file mode 100644 index 67e951a3..00000000 --- a/debian/abrechnung-db-clean.timer +++ /dev/null @@ -1,8 +0,0 @@ -[Unit] -Description=Abrechnung Database Cleanup Timer - -[Timer] -OnCalendar=*-*-* 3:00:00 - -[Install] -WantedBy=timers.target diff --git a/systemd/abrechnung-api.service b/systemd/abrechnung-api.service deleted file mode 100644 index 67f8f9e7..00000000 --- a/systemd/abrechnung-api.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Abrechnung API Service -After=postgresql.service - -[Service] -ExecStart=/usr/bin/abrechnung api -User=abrechnung -Group=abrechnung - -[Install] -WantedBy=multi-user.target diff --git a/systemd/abrechnung-db-clean.service b/systemd/abrechnung-db-clean.service deleted file mode 100644 index 605667f6..00000000 --- a/systemd/abrechnung-db-clean.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Abrechnung Database Cleanup Service -After=postgresql.service - -[Service] -ExecStart=/usr/bin/abrechnung db clean -User=abrechnung -Group=abrechnung - -[Install] -WantedBy=multi-user.target diff --git a/systemd/abrechnung-db-clean.timer b/systemd/abrechnung-db-clean.timer deleted file mode 100644 index 67e951a3..00000000 --- a/systemd/abrechnung-db-clean.timer +++ /dev/null @@ -1,8 +0,0 @@ -[Unit] -Description=Abrechnung Database Cleanup Timer - -[Timer] -OnCalendar=*-*-* 3:00:00 - -[Install] -WantedBy=timers.target diff --git a/systemd/abrechnung-mailer.service b/systemd/abrechnung-mailer.service deleted file mode 100644 index 489e4a66..00000000 --- a/systemd/abrechnung-mailer.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Abrechnung Mail Delivery Service -After=postgresql.service - -[Service] -ExecStart=/usr/bin/abrechnung mailer -User=abrechnung -Group=abrechnung - -[Install] -WantedBy=multi-user.target From 205eb095e2da9bea01bbbf04c2cd813e92dd22ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Loipf=C3=BChrer?= Date: Fri, 31 Jan 2025 17:40:37 +0100 Subject: [PATCH 3/3] feat(core): implement prometheus metrics see #231 --- abrechnung/application/groups.py | 10 +++- abrechnung/application/transactions.py | 22 ++++++++- abrechnung/config.py | 6 +++ abrechnung/http/api.py | 13 ++++++ abrechnung/http/metrics.py | 46 ++++++++++++++++++ abrechnung/util.py | 21 +++++++++ docs/development/setup.rst | 65 ++++++++++++++++---------- docs/usage/configuration.rst | 42 ++++++++++++++--- pyproject.toml | 1 + 9 files changed, 192 insertions(+), 34 deletions(-) create mode 100644 abrechnung/http/metrics.py diff --git a/abrechnung/application/groups.py b/abrechnung/application/groups.py index a519d625..18cbba8f 100644 --- a/abrechnung/application/groups.py +++ b/abrechnung/application/groups.py @@ -1,9 +1,9 @@ -from datetime import datetime +from datetime import datetime, timedelta import asyncpg from sftkit.database import Connection from sftkit.error import InvalidArgument -from sftkit.service import Service, with_db_transaction +from sftkit.service import Service, with_db_connection, with_db_transaction from abrechnung.config import Config from abrechnung.core.auth import create_group_log @@ -20,6 +20,7 @@ GroupPreview, ) from abrechnung.domain.users import User +from abrechnung.util import timed_cache class GroupService(Service[Config]): @@ -461,3 +462,8 @@ async def unarchive_group(self, *, conn: Connection, user: User, group_id: int): "update grp set archived = false where id = $1", group_id, ) + + @with_db_connection + @timed_cache(timedelta(minutes=5)) + async def total_number_of_groups(self, conn: Connection): + return await conn.fetchval("select count(*) from grp") diff --git a/abrechnung/application/transactions.py b/abrechnung/application/transactions.py index 468fe10b..e1e68c57 100644 --- a/abrechnung/application/transactions.py +++ b/abrechnung/application/transactions.py @@ -1,11 +1,11 @@ import base64 -from datetime import datetime +from datetime import datetime, timedelta from typing import Optional, Union import asyncpg from sftkit.database import Connection from sftkit.error import InvalidArgument -from sftkit.service import Service, with_db_transaction +from sftkit.service import Service, with_db_connection, with_db_transaction from abrechnung.application.common import _get_or_create_tag_ids from abrechnung.config import Config @@ -25,6 +25,7 @@ UpdateTransaction, ) from abrechnung.domain.users import User +from abrechnung.util import timed_cache class TransactionService(Service[Config]): @@ -721,3 +722,20 @@ async def _create_pending_transaction_change( ) return revision_id + + @with_db_connection + @timed_cache(timedelta(minutes=5)) + async def total_number_of_transactions(self, conn: Connection): + return await conn.fetchval("select count(*) from transaction_state_valid_at(now()) where not deleted") + + @with_db_connection + @timed_cache(timedelta(minutes=5)) + async def total_amount_of_money_per_currency(self, conn: Connection) -> dict[str, float]: + result = await conn.fetch( + "select t.currency_symbol, sum(t.value) as total_value " + "from transaction_state_valid_at(now()) as t where not t.deleted group by t.currency_symbol" + ) + aggregated = {} + for row in result: + aggregated[row["currency_symbol"]] = row["total_value"] + return aggregated diff --git a/abrechnung/config.py b/abrechnung/config.py index 6b7eaf1a..3d8a9135 100644 --- a/abrechnung/config.py +++ b/abrechnung/config.py @@ -67,6 +67,11 @@ class AuthConfig(BaseModel): auth: Optional[AuthConfig] = None +class MetricsConfig(BaseModel): + enabled: bool = False + expose_money_amounts: bool = False + + class Config(BaseSettings): model_config = SettingsConfigDict(env_prefix="ABRECHNUNG_", env_nested_delimiter="__") @@ -77,6 +82,7 @@ class Config(BaseSettings): # in case all params are optional this is needed to make the whole section optional demo: DemoConfig = DemoConfig() registration: RegistrationConfig = RegistrationConfig() + metrics: MetricsConfig = MetricsConfig() @classmethod def settings_customise_sources( diff --git a/abrechnung/http/api.py b/abrechnung/http/api.py index 51b3e611..1506167a 100644 --- a/abrechnung/http/api.py +++ b/abrechnung/http/api.py @@ -1,6 +1,7 @@ import json import logging +from prometheus_fastapi_instrumentator import Instrumentator from sftkit.http import Server from abrechnung import __version__ @@ -11,6 +12,7 @@ from abrechnung.config import Config from abrechnung.database.migrations import check_revision_version, get_database +from . import metrics from .context import Context from .routers import accounts, auth, common, groups, transactions @@ -64,9 +66,20 @@ async def _setup(self): async def _teardown(self): await self.db_pool.close() + def _instrument_api(self): + if not self.cfg.metrics.enabled: + return + instrumentor = Instrumentator() + instrumentor.add(metrics.abrechnung_number_of_groups_total(self.group_service)) + instrumentor.add(metrics.abrechnung_number_of_transactions_total(self.transaction_service)) + if self.cfg.metrics.expose_money_amounts: + instrumentor.add(metrics.abrechnung_total_amount_of_money(self.transaction_service)) + instrumentor.instrument(self.server.api).expose(self.server.api, endpoint="/api/metrics") + async def run(self): await self._setup() try: + self._instrument_api() await self.server.run(self.context) finally: await self._teardown() diff --git a/abrechnung/http/metrics.py b/abrechnung/http/metrics.py new file mode 100644 index 00000000..59ba0b5e --- /dev/null +++ b/abrechnung/http/metrics.py @@ -0,0 +1,46 @@ +from prometheus_client import Gauge +from prometheus_fastapi_instrumentator.metrics import Info + +from abrechnung.application.groups import GroupService +from abrechnung.application.transactions import TransactionService + + +def abrechnung_number_of_groups_total(group_service: GroupService): + metric = Gauge( + "abrechnung_number_of_groups_total", + "Number of groups created on this Abrechnung instance..", + ) + + async def instrumentation(info: Info) -> None: + total = await group_service.total_number_of_groups() + metric.set(total) + + return instrumentation + + +def abrechnung_number_of_transactions_total(transaction_service: TransactionService): + metric = Gauge( + "abrechnung_number_of_transactions_total", + "Number of transactions created on this Abrechnung instance..", + ) + + async def instrumentation(info: Info) -> None: + total = await transaction_service.total_number_of_transactions() + metric.set(total) + + return instrumentation + + +def abrechnung_total_amount_of_money(transaction_service: TransactionService): + metric = Gauge( + "abrechnung_total_amount_of_money", + "Total amount of money per currency cleared via thisthis Abrechnung instance..", + labelnames=("currency_symbol",), + ) + + async def instrumentation(info: Info) -> None: + total = await transaction_service.total_amount_of_money_per_currency() + for currency, value in total.items(): + metric.labels(currency).set(value) + + return instrumentation diff --git a/abrechnung/util.py b/abrechnung/util.py index 5c068247..d261c90b 100644 --- a/abrechnung/util.py +++ b/abrechnung/util.py @@ -1,3 +1,4 @@ +import functools import logging import re import uuid @@ -72,3 +73,23 @@ def is_valid_uuid(val: str): return True except ValueError: return False + + +def timed_cache(ttl: timedelta): + def wrapper(func): + last_execution: datetime | None = None + last_value = None + + @functools.wraps(func) + async def wrapped(*args, **kwargs): + nonlocal last_execution, last_value + current_execution = datetime.now() + if last_execution is None or last_value is None or current_execution - last_execution > ttl: + last_value = await func(*args, **kwargs) + last_execution = current_execution + + return last_value + + return wrapped + + return wrapper diff --git a/docs/development/setup.rst b/docs/development/setup.rst index f0cc7866..3464458d 100644 --- a/docs/development/setup.rst +++ b/docs/development/setup.rst @@ -11,32 +11,28 @@ Setup Installation ------------ -Fork and clone the repository :: +Fork and clone the repository + +.. code-block:: shell git clone https://github.com/SFTtech/abrechnung.git cd abrechnung -Then install the package in local development mode as well as all required dependencies. Make sure to have -`flit `_ installed first. Installing the dependencies can be two ways: +Then install the package in local development mode as well as all required dependencies. -Setup a virtual environment and install the packages via pip (straightforward way) :: +Setup a virtual environment and install the packages via pip - virtualenv -p python3 venv - source venv/bin/activate - pip install flit - flit install -s --deps develop +.. code-block:: shell -Or install the dependencies through your package manager (useful for distribution packaging) + virtualenv -p python3 .venv + source .venv/bin/activate + pip install -e . '[dev,test]' -* arch linux (slight chance some dependencies may be missing here) +Additionally you probably will want to activate the git pre-commit hooks (for formatting and linting) by running .. code-block:: shell - sudo pacman -S python-flit python-yaml python-aiohttp python-aiohttp-cors python-asyncpg python-sphinx python-schema python-email-validator python-bcrypt python-pyjwt python-aiosmtpd python-pytest python-pytest-cov python-black python-mypy python-pylint python-apispec python-marshmallow python-webargs - -Afterwards install the package without dependencies :: - - flit install -s --deps none + pre-commit install Database Setup -------------- @@ -60,7 +56,9 @@ Create the database (in a psql prompt): * Launch ``abrechnung -c abrechnung.yaml api`` * Launch ``abrechnung -c abrechnung.yaml mailer`` to get mail delivery (working mail server in config file required!) -It is always possible wipe and rebuild the database with :: +It is always possible wipe and rebuild the database with + +.. code-block:: shell abrechnung -c abrechnung.yaml db rebuild @@ -71,7 +69,7 @@ In case a new features requires changes to the database schema create a new migr .. code-block:: shell - ./tools/create_revision.py + sftkit create-migration In case you did not install the abrechnung in development mode it might be necessary to add the project root folder to your ``PYTHONPATH``. @@ -100,30 +98,49 @@ is used as a means to wipe and repopulate the database between tests. alter schema public owner to "" -Finally run the tests via :: +Finally run the tests via + +.. code-block:: shell make test -Run the linters via :: +Run the linters via + +.. code-block:: shell make lint +Run the formatters via + +.. code-block:: shell + + make format + Frontend Development -------------------- -Working on the frontend is quite easy, simply :: +Working on the frontend is quite easy, simply + +.. code-block:: shell cd web - yarn install - yarn start + npm install + npx nx serve web and you are good to go! Documentation ------------- -To build the documentation locally simply run :: +To build the documentation locally simply run +.. code-block:: shell + + pip install -r docs/requires.txt make docs -The html docs can then be found in ``docs/_build``. +The html docs can then be found in ``docs/_build`` or served locally with + +.. code-block:: shell + + make serve-docs diff --git a/docs/usage/configuration.rst b/docs/usage/configuration.rst index c860f1ca..68ead4dd 100644 --- a/docs/usage/configuration.rst +++ b/docs/usage/configuration.rst @@ -15,11 +15,13 @@ Database The first step after installing the **abrechnung** is to setup the database. Due to the use of database specific features we only support **PostgreSQL** with versions >= 13. Other versions might work but are untested. -First create a database with an associated user :: +First create a database with an associated user - $ sudo -u postgres psql - > create user abrechnung with password ''; - > create database abrechnung owner abrechnung; +.. code-block:: shell + + sudo -u postgres psql + create user abrechnung with password ''; + create database abrechnung owner abrechnung; Enter the information into the config file in ``/etc/abrechnung/abrechnung.yaml`` under the section database as @@ -31,7 +33,9 @@ Enter the information into the config file in ``/etc/abrechnung/abrechnung.yaml` dbname: "abrechnung" password: "" -Apply all database migrations with :: +Apply all database migrations with + +.. code-block:: shell abrechnung db migrate @@ -48,7 +52,9 @@ The ``name`` is used to populate the email subjects as ``[] ``. API Config --------------- Typically the config for the http API does not need to be changed much apart from two important settings! -In the ``api`` section make sure to insert a newly generated secret key, e.g. with :: +In the ``api`` section make sure to insert a newly generated secret key, e.g. with + +.. code-block:: shell pwgen -S 64 1 @@ -124,6 +130,30 @@ Guest users will not be able to create new groups themselves but can take part i valid_email_domains: ["some-domain.com"] allow_guest_users: true +Prometheus Metrics +------------------ + +Abrechnung also provides prometheus metrics which are disabled by default. +This includes some general metrics about the abrechnung instance such as + +- http request durations and groupings of error codes +- general python environment metrics such as process utilization and garbage collection performance + +Additionally it currently includes the following set of abrechnung specific metrics + +- number of groups created on the instance +- number of transactions created on the instance +- total amount of money by currency which was cleared via the instance, i.e. the total sum of transaction values per currency over all groups. + This is disabled by default as it may expose private data on very small abrechnung instances. + +To enable metrics under the api endpoint ``/api/metrics`` simply add the following to the config file + +.. code-block:: yaml + + metrics: + enabled: true + expose_money_amounts: false # disabled by default + Configuration via Environment Variables --------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index c645e244..c329d367 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ classifiers = [ requires-python = ">=3.10" dependencies = [ "sftkit==0.3.0", + "prometheus-fastapi-instrumentator==7.0.2", "pydantic[email]~=2.10.3", "pydantic-settings==2.6.1", "python-jose[cryptography]~=3.3.0",