From 7132586ef26c4a16d22bb6b3e9ee04e2f779cf2f Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Thu, 12 Sep 2024 23:12:25 +0200 Subject: [PATCH] add support for Beancount v3 This uses beanquery to replace the functionality in beancount.query, which works with both Beancount v2 and v3 - however there are some differences on columns in beanquery compared to beancount.query. For importers, this sticks to beancount.ingest on v2 and used beangulp for v3. It still expectes importers to conform to the v2 importer protocol for now. Due to changes related to duplicate detection, duplicate detection is not automatically done by Fava here but can still be manually specified as hooks. --- CHANGES | 8 + Makefile | 1 + constraints-old.txt | 31 +- constraints.txt | 85 ++-- contrib/scripts.py | 15 +- frontend/src/codemirror/bql-grammar.ts | 38 +- frontend/src/reports/query/query_table.ts | 8 +- pyproject.toml | 6 +- src/fava/beans/__init__.py | 9 + src/fava/beans/create.py | 19 +- src/fava/beans/funcs.py | 21 +- src/fava/core/ingest.py | 148 ++++++- src/fava/core/query.py | 168 ++++++++ src/fava/core/query_shell.py | 366 +++++------------- src/fava/ext/fava_ext_test/__init__.py | 3 +- src/fava/ext/portfolio_list/__init__.py | 3 +- src/fava/internal_api.py | 3 +- src/fava/json_api.py | 7 +- src/fava/util/excel.py | 4 +- stubs/beancount/core/data.pyi | 2 + ...t_core_ingest-test_ingest_examplefile.json | 5 +- .../test_core_query_shell-test_query | 9 - .../test_core_query_shell-test_query-2 | 43 -- .../test_core_query_shell-test_query_balances | 69 ++++ .../test_core_query_shell-test_query_to_file | 54 +-- .../test_core_query_shell-test_text_queries | 6 + .../test_json_api-test_api_imports-2.json | 5 +- ...test_json_api-test_api_query_result-2.json | 2 +- ...test_json_api-test_api_query_result-3.json | 56 +-- ...test_json_api-test_api_query_result-4.json | 54 +-- ...test_json_api-test_api_query_result-5.json | 2 +- .../test_json_api-test_api_query_result.json | 2 +- ...sation-test_deserialise_posting_and_format | 2 +- ...ialisation-test_serialise_entry_types.json | 4 +- tests/test_core_ingest.py | 15 +- tests/test_core_query_shell.py | 107 +++-- tests/test_json_api.py | 4 +- tox.ini | 1 + 38 files changed, 790 insertions(+), 595 deletions(-) create mode 100644 src/fava/core/query.py delete mode 100644 tests/__snapshots__/test_core_query_shell-test_query delete mode 100644 tests/__snapshots__/test_core_query_shell-test_query-2 create mode 100644 tests/__snapshots__/test_core_query_shell-test_query_balances create mode 100644 tests/__snapshots__/test_core_query_shell-test_text_queries diff --git a/CHANGES b/CHANGES index 827ae4dfe..b806d256a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,14 @@ Changelog ========= +Unreleased (Beancount v3 support) +--------------------------------- + +Support for Beancount version 3 was added. Using Beancount 2 is still +supported. Beancount query support is now provided by the beanquery package, +which has some minor differences in syntax, the provided columns and functions +to the bean-query functionality in Beancount version 2. + Unreleased ---------- diff --git a/Makefile b/Makefile index 9d19af244..f0b277c47 100644 --- a/Makefile +++ b/Makefile @@ -95,6 +95,7 @@ run-example: .PHONY: bql-grammar bql-grammar: tox exec -e lint -- python contrib/scripts.py generate-bql-grammar-json + -pre-commit run -a prettier # Build the distribution (sdist and wheel). .PHONY: dist diff --git a/constraints-old.txt b/constraints-old.txt index 1177b28d2..b5bac0b80 100644 --- a/constraints-old.txt +++ b/constraints-old.txt @@ -9,20 +9,33 @@ babel==2.7.0 # fava (pyproject.toml) # flask-babel beancount==2.3.5 + # via + # fava (pyproject.toml) + # beangulp + # beanquery +beangulp==0.1.0 + # via fava (pyproject.toml) +beanquery==0.1.dev0 # via fava (pyproject.toml) beautifulsoup4==4.0.1 - # via beancount + # via + # beancount + # beangulp bottle==0.12.20 # via # fava (pyproject.toml) # beancount chardet==1.0 - # via beancount + # via + # beancount + # beangulp cheroot==8.0.0 # via fava (pyproject.toml) -click==8.0.0 +click==8.0.1 # via # fava (pyproject.toml) + # beangulp + # beanquery # flask et-xmlfile==1.0.0 # via openpyxl @@ -61,10 +74,11 @@ lml==0.0.1 # via # pyexcel # pyexcel-io -lxml==5.0.0 +lxml==5.1.0 # via # fava (pyproject.toml) # beancount + # beangulp # pyexcel-ezodf # pyexcel-ods3 markdown2==2.3.0 @@ -105,14 +119,17 @@ pytest==7.2.0 # via # fava (pyproject.toml) # beancount -python-dateutil==2.1 +python-dateutil==2.6.0 # via # fava (pyproject.toml) # beancount + # beanquery python-gflags==1.3 # via google-api-python-client python-magic==0.4.12 - # via beancount + # via + # beancount + # beangulp pytz==2020.1 # via # fava (pyproject.toml) @@ -131,6 +148,8 @@ six==1.16.0 # python-dateutil sniffio==1.1.0 # via anyio +tatsu==5.7.4 + # via beanquery texttable==0.8.3 # via pyexcel watchfiles==0.20.0 diff --git a/constraints.txt b/constraints.txt index 77ce8a090..74de6d8df 100644 --- a/constraints.txt +++ b/constraints.txt @@ -13,22 +13,25 @@ babel==2.16.0 # fava (pyproject.toml) # flask-babel # sphinx -beancount==2.3.6 +beancount==3.0.0 + # via + # fava (pyproject.toml) + # beangulp + # beanquery +beangulp==0.1.1 + # via fava (pyproject.toml) +beanquery==0.1.dev0 # via fava (pyproject.toml) beautifulsoup4==4.12.3 # via - # beancount + # beangulp # furo blinker==1.8.2 # via flask -bottle==0.13.1 - # via beancount build==1.2.2 # via fava (pyproject.toml) cachetools==5.5.0 - # via - # google-auth - # tox + # via tox certifi==2024.8.30 # via requests cffi==1.17.1 @@ -37,7 +40,7 @@ cfgv==3.4.0 # via pre-commit chardet==5.2.0 # via - # beancount + # beangulp # pyexcel # tox charset-normalizer==3.3.2 @@ -47,6 +50,9 @@ cheroot==10.0.1 click==8.1.7 # via # fava (pyproject.toml) + # beancount + # beangulp + # beanquery # flask colorama==0.4.6 # via tox @@ -76,23 +82,6 @@ flask-babel==4.0.0 # via fava (pyproject.toml) furo==2024.8.6 # via fava (pyproject.toml) -google-api-core==2.19.2 - # via google-api-python-client -google-api-python-client==2.145.0 - # via beancount -google-auth==2.34.0 - # via - # google-api-core - # google-api-python-client - # google-auth-httplib2 -google-auth-httplib2==0.2.0 - # via google-api-python-client -googleapis-common-protos==1.65.0 - # via google-api-core -httplib2==0.22.0 - # via - # google-api-python-client - # google-auth-httplib2 identify==2.6.1 # via pre-commit idna==3.10 @@ -135,7 +124,8 @@ lml==0.1.0 # pyexcel-io lxml==5.3.0 # via - # beancount + # fava (pyproject.toml) + # beangulp # pyexcel-ezodf # pyexcel-ods3 markdown-it-py==3.0.0 @@ -175,8 +165,6 @@ packaging==24.1 # sphinx # tox # tox-uv -pdfminer2==20151206 - # via beancount pkginfo==1.10.0 # via twine platformdirs==4.3.3 @@ -189,24 +177,9 @@ pluggy==1.5.0 # pytest # tox ply==3.11 - # via - # fava (pyproject.toml) - # beancount + # via fava (pyproject.toml) pre-commit==3.8.0 # via fava (pyproject.toml) -proto-plus==1.24.0 - # via google-api-core -protobuf==5.28.1 - # via - # google-api-core - # googleapis-common-protos - # proto-plus -pyasn1==0.6.1 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.4.1 - # via google-auth pycparser==2.22 # via cffi pyexcel==0.7.0 @@ -234,8 +207,6 @@ pyinstaller-hooks-contrib==2024.8 # via pyinstaller pylint==3.2.7 # via fava (pyproject.toml) -pyparsing==3.1.4 - # via httplib2 pyproject-api==1.7.1 # via tox pyproject-hooks==1.1.0 @@ -243,24 +214,26 @@ pyproject-hooks==1.1.0 pytest==8.3.3 # via # fava (pyproject.toml) - # beancount # pytest-cov pytest-cov==5.0.0 # via fava (pyproject.toml) python-dateutil==2.9.0.post0 - # via beancount + # via + # beancount + # beanquery python-magic==0.4.27 - # via beancount + # via beangulp pytz==2024.2 # via flask-babel pyyaml==6.0.2 # via pre-commit readme-renderer==44.0 # via twine +regex==2024.9.11 + # via beancount requests==2.32.3 # via - # beancount - # google-api-core + # fava (pyproject.toml) # requests-toolbelt # sphinx # twine @@ -270,8 +243,6 @@ rfc3986==2.0.0 # via twine rich==13.8.1 # via twine -rsa==4.9 - # via google-auth secretstorage==3.3.3 # via keyring setuptools==75.0.0 @@ -282,9 +253,7 @@ setuptools==75.0.0 simplejson==3.19.3 # via fava (pyproject.toml) six==1.16.0 - # via - # pdfminer2 - # python-dateutil + # via python-dateutil sniffio==1.3.1 # via anyio snowballstemmer==2.2.0 @@ -313,6 +282,8 @@ sphinxcontrib-qthelp==2.0.0 # via sphinx sphinxcontrib-serializinghtml==2.0.0 # via sphinx +tatsu==5.7.4 + # via beanquery texttable==1.7.0 # via pyexcel tomlkit==0.13.2 @@ -331,8 +302,6 @@ types-simplejson==3.19.0.20240801 # via fava (pyproject.toml) typing-extensions==4.12.2 # via mypy -uritemplate==4.1.1 - # via google-api-python-client urllib3==2.2.3 # via # requests diff --git a/contrib/scripts.py b/contrib/scripts.py index 664337100..3b17e02c5 100755 --- a/contrib/scripts.py +++ b/contrib/scripts.py @@ -9,8 +9,9 @@ from typing import Iterable import requests -from beancount.query import query_env -from beancount.query import query_parser +from beanquery import query_compile +from beanquery.parser.parser import KEYWORDS +from beanquery.sources.beancount import TABLES from click import echo from click import group from click import UsageError @@ -42,14 +43,14 @@ def generate_bql_grammar_json() -> None: Should be run whenever the BQL changes.""" - target_env = query_env.TargetsEnvironment() + columns = {column for table in TABLES for column in table.columns} data = { - "columns": sorted(set(_env_to_list(target_env.columns))), - "functions": sorted(set(_env_to_list(target_env.functions))), - "keywords": sorted({kw.lower() for kw in query_parser.Lexer.keywords}), + "columns": sorted(columns), + "functions": sorted(query_compile.FUNCTIONS.keys()), + "keywords": sorted({kw.lower() for kw in KEYWORDS}), } path = BASE_PATH / "frontend" / "src" / "codemirror" / "bql-grammar.ts" - path.write_text("export default " + json.dumps(data)) + path.write_text("export default " + json.dumps(data, indent=" ")) class MissingPoeditorTokenError(UsageError): diff --git a/frontend/src/codemirror/bql-grammar.ts b/frontend/src/codemirror/bql-grammar.ts index dbaa4bed7..60463bbb8 100644 --- a/frontend/src/codemirror/bql-grammar.ts +++ b/frontend/src/codemirror/bql-grammar.ts @@ -1,8 +1,10 @@ export default { columns: [ "account", + "amount", "balance", - "change", + "close", + "comment", "cost_currency", "cost_date", "cost_label", @@ -11,21 +13,27 @@ export default { "date", "day", "description", + "discrepancy", + "entry", "filename", "flag", "id", "lineno", "links", "location", + "meta", "month", + "name", "narration", "number", + "open", "other_accounts", "payee", "position", "posting_flag", "price", "tags", + "tolerance", "type", "weight", "year", @@ -34,8 +42,8 @@ export default { "abs", "account_sortkey", "any_meta", + "bool", "close_date", - "coalesce", "commodity", "commodity_meta", "convert", @@ -46,7 +54,11 @@ export default { "date", "date_add", "date_diff", + "date_part", + "date_trunc", "day", + "decimal", + "empty", "entry_meta", "filter_currency", "findfirst", @@ -55,6 +67,9 @@ export default { "getprice", "grep", "grepn", + "has_account", + "int", + "interval", "joinstr", "last", "leaf", @@ -71,12 +86,17 @@ export default { "open_date", "open_meta", "parent", + "parse_date", "possign", "quarter", + "repr", "root", + "round", "safediv", + "splitcomp", "str", "subst", + "substr", "sum", "today", "units", @@ -84,39 +104,29 @@ export default { "value", "weekday", "year", - "ymonth", + "yearmonth", ], keywords: [ "and", "as", "asc", - "at", "balances", "by", - "clear", - "close", "desc", "distinct", - "errors", - "explain", "false", - "flatten", "from", "group", "having", "in", + "is", "journal", "limit", "not", - "null", - "on", - "open", "or", "order", "pivot", "print", - "reload", - "run", "select", "true", "where", diff --git a/frontend/src/reports/query/query_table.ts b/frontend/src/reports/query/query_table.ts index c98ff2599..e43012d51 100644 --- a/frontend/src/reports/query/query_table.ts +++ b/frontend/src/reports/query/query_table.ts @@ -9,6 +9,7 @@ import { constant, constants, date, + defaultValue, number, object, optional, @@ -115,7 +116,12 @@ function get_query_column(type: QueryType, index: number) { ); case "object": case "str": - return new StringSortedQueryColumn(type, index, string, (v) => v); + return new StringSortedQueryColumn( + type, + index, + defaultValue(string, () => ""), + (v) => v, + ); case "Amount": return new NumberSortedQueryColumn( type, diff --git a/pyproject.toml b/pyproject.toml index 6389d1fda..f5431efcb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,9 +39,12 @@ dependencies = [ "Flask>=2.2,<4", "Jinja2>=3,<4", "Werkzeug>=2.2,<4", - "beancount>=2.3.5,<3", + "beancount>=2,<4", + "beanquery", + "beangulp", "cheroot>=8,<11", "click>=7,<9", + "lxml>=5.1.0", # set a lower bound for the old_deps env "markdown2>=2.3.0,<3", "ply", "simplejson>=3.16.0,<4", @@ -75,6 +78,7 @@ dev = [ "pylint", "pytest", "pytest-cov", + "requests", "setuptools", "sphinx", "sphinx-autodoc-typehints", diff --git a/src/fava/beans/__init__.py b/src/fava/beans/__init__.py index 867977f9f..068ce4b85 100644 --- a/src/fava/beans/__init__.py +++ b/src/fava/beans/__init__.py @@ -1 +1,10 @@ """Types, functions and wrappers to deal with Beancount types.""" + +from __future__ import annotations + +try: + from beancount import query # noqa: F401 + + BEANCOUNT_V3 = False +except ImportError: + BEANCOUNT_V3 = True diff --git a/src/fava/beans/create.py b/src/fava/beans/create.py index cbc20c42c..6757c993a 100644 --- a/src/fava/beans/create.py +++ b/src/fava/beans/create.py @@ -12,6 +12,7 @@ from beancount.core.amount import Amount as BeancountAmount from beancount.core.position import Position as BeancountPosition +from fava.beans import BEANCOUNT_V3 from fava.beans.abc import Amount if TYPE_CHECKING: # pragma: no cover @@ -53,7 +54,10 @@ def amount(amt: Amount | Decimal | str, currency: str | None = None) -> Amount: def position(units: Amount, cost: Cost | None) -> Position: """Create a position.""" - return BeancountPosition(units, cost) # type: ignore[arg-type,return-value] + return BeancountPosition( # type: ignore[return-value] + units, # type: ignore[arg-type] + cost, # type: ignore[arg-type] + ) def posting( @@ -124,11 +128,14 @@ def note( date: datetime.date, account: str, comment: str, + tags: TagsOrLinks | None = None, + links: TagsOrLinks | None = None, ) -> Balance: """Create a Beancount Note.""" - return data.Note( # type: ignore[return-value] - meta, - date, - account, - comment, + if BEANCOUNT_V3: + return data.Note( # type: ignore[return-value] + meta, date, account, comment, tags, links + ) + return data.Note( # type: ignore[call-arg,return-value] + meta, date, account, comment ) diff --git a/src/fava/beans/funcs.py b/src/fava/beans/funcs.py index b94cb2240..c79da7684 100644 --- a/src/fava/beans/funcs.py +++ b/src/fava/beans/funcs.py @@ -6,8 +6,8 @@ from typing import TYPE_CHECKING from beancount.core import compare # type: ignore[attr-defined] -from beancount.query import query # type: ignore[attr-defined] -from beancount.query import query_execute # type: ignore[attr-defined] +from beanquery import query # type: ignore[import-untyped] +from beanquery import query_execute if TYPE_CHECKING: # pragma: no cover from typing import TypeAlias @@ -15,7 +15,12 @@ from fava.beans.abc import Directive from fava.beans.types import BeancountOptions - ResultType: TypeAlias = tuple[str, type[Any]] + class ResultType: + """The Column type from beanquery (which has more attributes).""" + + name: str + datatype: type[Any] + ResultRow: TypeAlias = tuple[Any, ...] QueryResult: TypeAlias = tuple[list[ResultType], list[ResultRow]] @@ -38,16 +43,10 @@ def get_position(entry: Directive) -> tuple[str, int]: raise ValueError(msg) -def execute_query( - query_: str, - entries: list[Directive], - options_map: BeancountOptions, -) -> QueryResult: +def execute_query(query_: str) -> QueryResult: """Execture a query.""" return query_execute.execute_query( # type: ignore[no-any-return] - query_, - entries, - options_map, + query_ ) diff --git a/src/fava/core/ingest.py b/src/fava/core/ingest.py index feb9d2f7c..6823f3a1e 100644 --- a/src/fava/core/ingest.py +++ b/src/fava/core/ingest.py @@ -2,21 +2,33 @@ from __future__ import annotations +import os import sys import traceback from dataclasses import dataclass +from inspect import signature from os import altsep from os import sep from pathlib import Path from runpy import run_path from typing import Any +from typing import Iterable from typing import TYPE_CHECKING -from beancount.ingest import cache # type: ignore[import-untyped] -from beancount.ingest import extract -from beancount.ingest import identify +try: # pragma: no cover + from beancount.ingest import cache # type: ignore[import-untyped] + from beancount.ingest import extract + + DEFAULT_HOOKS = [extract.find_duplicate_entries] + BEANGULP = False +except ImportError: + from beangulp import cache # type: ignore[import-untyped] + + DEFAULT_HOOKS = [] + BEANGULP = True from fava.beans.ingest import BeanImporterProtocol +from fava.core.file import _incomplete_sortkey from fava.core.module_base import FavaModule from fava.helpers import BeancountError from fava.helpers import FavaAPIError @@ -26,6 +38,7 @@ import datetime from fava.beans.abc import Directive + from fava.beans.ingest import FileMemo from fava.core import FavaLedger @@ -61,6 +74,57 @@ def __init__(self) -> None: super().__init__("You need to set at least one imports-dir.") +IGNORE_DIRS = { + ".cache", + ".git", + ".hg", + ".idea", + ".svn", + ".tox", + ".venv", + "__pycache__", + "node_modules", +} + + +def walk_dir(directory: Path) -> Iterable[Path]: + """Walk through all files in dir. + + Args: + directory: The directory to start in. + + Yields: + All full paths under directory, ignoring some directories. + """ + for root, dirs, filenames in os.walk(directory): + dirs[:] = sorted(d for d in dirs if d not in IGNORE_DIRS) + root_path = Path(root) + for filename in sorted(filenames): + yield root_path / filename + + +# Keep our own cache to also keep track of file mtimes +_CACHE: dict[str, tuple[int, FileMemo]] = {} + + +def get_cached_file(filename: str) -> FileMemo: + """Get a cached FileMemo. + + This checks the file's mtime before getting it from the Cache. + In addition to using the beangulp / + """ + mtime = Path(filename).stat().st_mtime_ns + cached = _CACHE.get(filename) + if cached: + mtime_cached, memo_cached = cached + if mtime <= mtime_cached: + return memo_cached + memo: FileMemo = cache._FileMemo(filename) # noqa: SLF001 + cache._CACHE[filename] = memo # noqa: SLF001 + _CACHE[filename] = (mtime, memo) + return memo + + @dataclass(frozen=True) class FileImportInfo: """Info about one file/importer combination.""" @@ -85,7 +149,7 @@ def file_import_info( importer: BeanImporterProtocol, ) -> FileImportInfo: """Generate info about a file with an importer.""" - file = cache.get_file(filename) + file = get_cached_file(filename) try: account = importer.file_account(file) date = importer.file_date(file) @@ -101,6 +165,64 @@ def file_import_info( ) +# Copied here from beangulp to minimise the imports. +_FILE_TOO_LARGE_THRESHOLD = 8 * 1024 * 1024 + + +def find_imports( + config: list[Any], directory: Path +) -> Iterable[FileImporters]: + """Pair files and matching importers. + + Yields: + For each file in directory, a pair of its filename and the matching + importers. + """ + for filename in walk_dir(directory): + stat = filename.stat() + if stat.st_size > _FILE_TOO_LARGE_THRESHOLD: + continue + + file = get_cached_file(str(filename)) + importers = [ + file_import_info(str(filename), importer) + for importer in config + if importer.identify(file) + ] + yield FileImporters( + name=str(filename), basename=filename.name, importers=importers + ) + + +def extract_from_file( + importer: Any, filename: str, existing_entries: list[Directive] +) -> list[Directive]: + """Import entries from a document. + + Args: + importer: The importer instance to handle the document. + filename: Filesystem path to the document. + existing_entries: Existing entries. + + Returns: + The list of imported entries. + """ + file = get_cached_file(filename) + entries = ( + importer.extract(file, existing_entries=existing_entries) + if "existing_entries" in signature(importer.extract).parameters + else importer.extract(file) + ) + if not entries: + return [] + + if hasattr(importer, "sort"): + importer.sort(entries) + else: + entries.sort(key=_incomplete_sortkey) + return entries # type: ignore[no-any-return] + + class IngestModule(FavaModule): """Exposes ingest functionality.""" @@ -149,7 +271,7 @@ def load_file(self) -> None: # noqa: D102 self.mtime = module_path.stat().st_mtime_ns self.config = mod["CONFIG"] - self.hooks = [extract.find_duplicate_entries] + self.hooks = list(DEFAULT_HOOKS) if "HOOKS" in mod: hooks = mod["HOOKS"] if not isinstance(hooks, list) or not all( @@ -179,18 +301,10 @@ def import_data(self) -> list[FileImporters]: if not self.config: return [] - ret = [] - + ret: list[FileImporters] = [] for directory in self.ledger.fava_options.import_dirs: full_path = self.ledger.join_path(directory) - files = list(identify.find_imports(self.config, str(full_path))) - for filename, importers in files: - base = Path(filename).name - infos = [ - file_import_info(filename, importer) - for importer in importers - ] - ret.append(FileImporters(filename, base, infos)) + ret.extend(find_imports(self.config, full_path)) return ret @@ -214,9 +328,9 @@ def extract(self, filename: str, importer_name: str) -> list[Directive]: self.load_file() try: - new_entries = extract.extract_from_file( - filename, + new_entries = extract_from_file( self.importers.get(importer_name), + filename, existing_entries=self.ledger.all_entries, ) except Exception as exc: diff --git a/src/fava/core/query.py b/src/fava/core/query.py new file mode 100644 index 000000000..8bc054638 --- /dev/null +++ b/src/fava/core/query.py @@ -0,0 +1,168 @@ +"""Query result types.""" + +from __future__ import annotations + +import datetime +from dataclasses import dataclass +from decimal import Decimal +from typing import TYPE_CHECKING + +from beancount.core.amount import Amount +from beancount.core.inventory import Inventory +from beancount.core.position import Position + +from fava.core.conversion import simple_units + +if TYPE_CHECKING: # pragma: no cover + from typing import Literal + from typing import TypeAlias + from typing import TypeVar + + from fava.core.inventory import SimpleCounterInventory + + T = TypeVar("T") + + QueryRowValue = ( + bool | int | str | datetime.date | Decimal | Position | Inventory + ) + SerialisedQueryRowValue = ( + bool + | int + | str + | datetime.date + | Decimal + | Position + | SimpleCounterInventory + ) + + +@dataclass(frozen=True) +class QueryResultTable: + """Table query result.""" + + types: list[BaseColumn] + rows: list[tuple[SerialisedQueryRowValue, ...]] + t: Literal["table"] = "table" + + +@dataclass(frozen=True) +class QueryResultText: + """Text query result.""" + + contents: str + t: Literal["string"] = "string" + + +if TYPE_CHECKING: # pragma: no cover + QueryResult: TypeAlias = QueryResultTable | QueryResultText + + +@dataclass(frozen=True) +class BaseColumn: + """A query column.""" + + name: str + dtype: str + + @staticmethod + def serialise( + val: QueryRowValue, + ) -> SerialisedQueryRowValue: + """Serialiseable version of the column value.""" + return val # type: ignore[return-value] + + +@dataclass(frozen=True) +class BoolColumn(BaseColumn): + """A boolean query column.""" + + dtype: str = "bool" + + +@dataclass(frozen=True) +class DecimalColumn(BaseColumn): + """A Decimal query column.""" + + dtype: str = "Decimal" + + +@dataclass(frozen=True) +class IntColumn(BaseColumn): + """A int query column.""" + + dtype: str = "int" + + +@dataclass(frozen=True) +class StrColumn(BaseColumn): + """A str query column.""" + + dtype: str = "str" + + +@dataclass(frozen=True) +class DateColumn(BaseColumn): + """A date query column.""" + + dtype: str = "date" + + +@dataclass(frozen=True) +class PositionColumn(BaseColumn): + """A Position query column.""" + + dtype: str = "Position" + + +@dataclass(frozen=True) +class SetColumn(BaseColumn): + """A set query column.""" + + dtype: str = "set" + + +@dataclass(frozen=True) +class AmountColumn(BaseColumn): + """An amount query column.""" + + dtype: str = "Amount" + + +@dataclass(frozen=True) +class ObjectColumn(BaseColumn): + """An object query column.""" + + dtype: str = "object" + + @staticmethod + def serialise(val: object) -> str: + """Serialise an object of unknown type to a string.""" + return str(val) + + +@dataclass(frozen=True) +class InventoryColumn(BaseColumn): + """A str query column.""" + + dtype: str = "Inventory" + + @staticmethod + def serialise( # type: ignore[override] + val: Inventory, + ) -> SimpleCounterInventory: + """Serialise an inventory.""" + return simple_units(val) + + +COLUMNS = { + Amount: AmountColumn, + Decimal: DecimalColumn, + Inventory: InventoryColumn, + Position: PositionColumn, + bool: BoolColumn, + datetime.date: DateColumn, + int: IntColumn, + set: SetColumn, + str: StrColumn, + object: ObjectColumn, +} diff --git a/src/fava/core/query_shell.py b/src/fava/core/query_shell.py index 0b02559bc..073dd9e0a 100644 --- a/src/fava/core/query_shell.py +++ b/src/fava/core/query_shell.py @@ -2,110 +2,95 @@ from __future__ import annotations -import contextlib -import datetime import io +import shlex import textwrap -from dataclasses import dataclass -from decimal import Decimal from typing import TYPE_CHECKING -from beancount.core.amount import Amount -from beancount.core.inventory import Inventory -from beancount.core.position import Position -from beancount.parser.options import OPTIONS_DEFAULTS -from beancount.query import query_compile -from beancount.query.query_compile import CompilationError -from beancount.query.query_parser import ParseError -from beancount.query.query_parser import RunCustom -from beancount.query.shell import BQLShell # type: ignore[import-untyped] +from beanquery import compiler # type: ignore[import-untyped] +from beanquery.compiler import CompilationError # type: ignore[import-untyped] +from beanquery.parser import ParseError # type: ignore[import-untyped] +from beanquery.shell import BQLShell # type: ignore[import-untyped] +from beanquery.sources.beancount import ( # type: ignore[import-untyped] + add_beancount_tables, +) from fava.beans.funcs import execute_query from fava.beans.funcs import run_query -from fava.core.conversion import simple_units from fava.core.module_base import FavaModule +from fava.core.query import COLUMNS +from fava.core.query import QueryResultTable +from fava.core.query import QueryResultText from fava.helpers import FavaAPIError from fava.util.excel import HAVE_EXCEL from fava.util.excel import to_csv from fava.util.excel import to_excel if TYPE_CHECKING: # pragma: no cover - from typing import Literal from typing import TypeVar from fava.beans.abc import Directive - from fava.beans.abc import Query from fava.beans.funcs import QueryResult from fava.beans.funcs import ResultRow from fava.beans.funcs import ResultType from fava.core import FavaLedger - from fava.core.inventory import SimpleCounterInventory - from fava.helpers import BeancountError + from fava.core.query import BaseColumn + from fava.core.query import SerialisedQueryRowValue T = TypeVar("T") - QueryRowValue = ( - bool | int | str | datetime.date | Decimal | Position | Inventory - ) - SerialisedQueryRowValue = ( - bool - | int - | str - | datetime.date - | Decimal - | Position - | SimpleCounterInventory - ) -# This is to limit the size of the history file. Fava is not using readline at -# all, but Beancount somehow still is... -try: - import readline +class FavaShellError(FavaAPIError): + """An error in the Fava BQL shell, will be turned into a string.""" - readline.set_history_length(1000) -except ImportError: # pragma: no cover - pass +class QueryNotFoundError(FavaShellError): + """Query '{name}' not found.""" -@dataclass(frozen=True) -class QueryResultTable: - """Table query result.""" + def __init__(self, name: str) -> None: + super().__init__(f"Query '{name}' not found.") - types: list[BaseColumn] - rows: list[tuple[SerialisedQueryRowValue, ...]] - t: Literal["table"] = "table" +class TooManyRunArgsError(FavaShellError): + """Too many args to run: '{args}'.""" -@dataclass(frozen=True) -class QueryResultText: - """Text query result.""" + def __init__(self, args: str) -> None: + super().__init__(f"Too many args to run: '{args}'.") - contents: str - t: Literal["string"] = "string" - -class QueryShell(BQLShell, FavaModule): # type: ignore[misc] +class FavaBQLShell(BQLShell): # type: ignore[misc] """A light wrapper around Beancount's shell.""" + outfile: io.StringIO + def __init__(self, ledger: FavaLedger) -> None: - self.buffer = io.StringIO() - BQLShell.__init__( - self, - is_interactive=False, - loadfun=None, - outfile=self.buffer, + super().__init__( + filename="", + outfile=io.StringIO(), + interactive=False, + ) + self.ledger = ledger + self.stdout = self.outfile + + def run(self, entries: list[Directive], query: str) -> QueryResult | str: + """Run a query, capturing output as string or returning the result.""" + add_beancount_tables( + self.context, + entries, + self.ledger.errors, + self.ledger.options, ) - FavaModule.__init__(self, ledger) - self.result: QueryResult | None = None - self.stdout = self.buffer - self.entries: list[Directive] = [] - self.errors: list[BeancountError] = [] - self.options_map = OPTIONS_DEFAULTS - - @property - def queries(self) -> list[Query]: - """All queries in the ledger.""" - return self.ledger.all_entries_by_type.Query + try: + result = self.onecmd(query) + except (FavaShellError, CompilationError, ParseError) as exc: + return f"ERROR: {exc!s}." + + if result: + types, rows = result + return (types, rows) + contents = self.outfile.getvalue() + self.outfile.truncate(0) + return contents.strip().strip("\x00") def add_help(self) -> None: """Attach help functions for each of the parsed token handlers.""" @@ -118,71 +103,52 @@ def add_help(self) -> None: f"help_{command_name.lower()}", lambda _, fun=func: print( textwrap.dedent(fun.__doc__).strip(), - file=self.buffer, + file=self.outfile, ), ) - def get_pager(self) -> io.StringIO: - """No real pager, just self.buffer to print to.""" - raise NotImplementedError - # maybe we should return self.buffer - def noop(self, _: T) -> None: """Doesn't do anything in Fava's query shell.""" - print(self.noop.__doc__, file=self.buffer) + print(self.noop.__doc__, file=self.outfile) on_Reload = noop # noqa: N815 do_exit = noop do_quit = noop do_EOF = noop # noqa: N815 - def on_Select(self, statement: str) -> None: # noqa: D102, N802 - try: - c_query = query_compile.compile( # type: ignore[attr-defined] - statement, - self.env_targets, - self.env_postings, - self.env_entries, - ) - except CompilationError as exc: - print(f"ERROR: {str(exc).rstrip('.')}.", file=self.buffer) - return - rtypes, rrows = execute_query(c_query, self.entries, self.options_map) + def on_Select(self, statement: str) -> QueryResult: # noqa: D102, N802 + c_query = compiler.compile(self.context, statement) + return execute_query(c_query) + + def do_run(self, arg: str) -> QueryResult | None: + """Run a custom query.""" + queries = self.ledger.all_entries_by_type.Query + stripped_arg = arg.rstrip("; \t") + if not stripped_arg: + # List the available queries. + for q in queries: + print(q.name, file=self.outfile) + return None - if not rrows: - print("(empty)", file=self.buffer) + name, *more = shlex.split(stripped_arg) + if more: + raise TooManyRunArgsError(stripped_arg) - self.result = rtypes, rrows + query = next((q for q in queries if q.name == name), None) + if query is None: + raise QueryNotFoundError(name) + return self.execute(query.query_string) # type: ignore[no-any-return] - def execute_query( - self, entries: list[Directive], query: str - ) -> ( - tuple[str, None, None] | tuple[None, list[ResultType], list[ResultRow]] - ): - """Run a query. - Arguments: - entries: The entries to run the query on. - query: A query string. +FavaBQLShell.on_Select.__doc__ = BQLShell.on_Select.__doc__ - Returns: - A tuple (contents, types, rows) where either the first or the last - two entries are None. If the query result is a table, it will be - contained in ``types`` and ``rows``, otherwise the result will be - contained in ``contents`` (as a string). - """ - self.entries = entries - self.errors = self.ledger.errors - self.options_map = self.ledger.options - with contextlib.redirect_stdout(self.buffer): - self.onecmd(query) - contents = self.buffer.getvalue() - self.buffer.truncate(0) - if self.result is None: - return (contents.strip().strip("\x00"), None, None) - types, rows = self.result - self.result = None - return (None, types, rows) + +class QueryShell(FavaModule): + """A Fava module to run BQL queries.""" + + def __init__(self, ledger: FavaLedger) -> None: + super().__init__(ledger) + self.shell = FavaBQLShell(ledger) def execute_query_serialised( self, entries: list[Directive], query: str @@ -196,32 +162,13 @@ def execute_query_serialised( Returns: Either a table or a text result (depending on the query). """ - contents, types, rows = self.execute_query(entries, query) - if contents and "ERROR" in contents: - raise FavaAPIError(contents) - - if not types or not rows: - return QueryResultText(contents or "") + res = self.shell.run(entries, query) + if isinstance(res, str): + if res.startswith("ERROR"): + raise FavaAPIError(res) + return QueryResultText(res) - return QueryResultTable(*serialise_query_result(types, rows)) - - def on_RunCustom(self, run_stmt: RunCustom) -> None: # noqa: N802 - """Run a custom query.""" - name = run_stmt.query_name - if name is None: - # List the available queries. - for query in self.queries: - print(query.name, file=self.buffer) - else: - try: - query = next( - query for query in self.queries if query.name == name - ) - except StopIteration: - print(f"ERROR: Query '{name}' not found", file=self.buffer) - else: - statement = self.parser.parse(query.query_string) - self.dispatch(statement) + return QueryResultTable(*serialise_query_result(*res)) def query_to_file( self, @@ -247,21 +194,14 @@ def query_to_file( """ name = "query_result" - try: - statement = self.parser.parse(query_string) - except ParseError as exception: - raise FavaAPIError(str(exception)) from exception - - if isinstance(statement, RunCustom): - name = statement.query_name - - try: - query = next( - query for query in self.queries if query.name == name - ) - except StopIteration as exc: - msg = f'Query "{name}" not found.' - raise FavaAPIError(msg) from exc + if query_string.startswith((".run", "run")): + _run, name, *more = shlex.split(query_string) + if more: + raise TooManyRunArgsError(query_string) + queries = self.ledger.all_entries_by_type.Query + query = next((q for q in queries if q.name == name), None) + if query is None: + raise QueryNotFoundError(name) query_string = query.query_string try: @@ -284,123 +224,11 @@ def query_to_file( return name, data -QueryShell.on_Select.__doc__ = BQLShell.on_Select.__doc__ - - -@dataclass(frozen=True) -class BaseColumn: - """A query column.""" - - name: str - dtype: str - - @staticmethod - def serialise( - val: QueryRowValue, - ) -> SerialisedQueryRowValue: - """Serialiseable version of the column value.""" - return val # type: ignore[return-value] - - -@dataclass(frozen=True) -class BoolColumn(BaseColumn): - """A boolean query column.""" - - dtype: str = "bool" - - -@dataclass(frozen=True) -class DecimalColumn(BaseColumn): - """A Decimal query column.""" - - dtype: str = "Decimal" - - -@dataclass(frozen=True) -class IntColumn(BaseColumn): - """A int query column.""" - - dtype: str = "int" - - -@dataclass(frozen=True) -class StrColumn(BaseColumn): - """A str query column.""" - - dtype: str = "str" - - -@dataclass(frozen=True) -class DateColumn(BaseColumn): - """A date query column.""" - - dtype: str = "date" - - -@dataclass(frozen=True) -class PositionColumn(BaseColumn): - """A Position query column.""" - - dtype: str = "Position" - - -@dataclass(frozen=True) -class SetColumn(BaseColumn): - """A set query column.""" - - dtype: str = "set" - - -@dataclass(frozen=True) -class AmountColumn(BaseColumn): - """An amount query column.""" - - dtype: str = "Amount" - - -@dataclass(frozen=True) -class ObjectColumn(BaseColumn): - """An object query column.""" - - dtype: str = "object" - - @staticmethod - def serialise(val: object) -> str: - """Serialise an object of unknown type to a string.""" - return str(val) - - -@dataclass(frozen=True) -class InventoryColumn(BaseColumn): - """A str query column.""" - - dtype: str = "Inventory" - - @staticmethod - def serialise(val: Inventory) -> SimpleCounterInventory: # type: ignore[override] - """Serialise an inventory.""" - return simple_units(val) - - -COLUMNS = { - Amount: AmountColumn, - Decimal: DecimalColumn, - Inventory: InventoryColumn, - Position: PositionColumn, - bool: BoolColumn, - datetime.date: DateColumn, - int: IntColumn, - set: SetColumn, - str: StrColumn, - object: ObjectColumn, -} - - def serialise_query_result( types: list[ResultType], rows: list[ResultRow] ) -> tuple[list[BaseColumn], list[tuple[SerialisedQueryRowValue, ...]]]: """Serialise the query result.""" - dtypes = [COLUMNS[dtype](name) for name, dtype in types] + dtypes = [COLUMNS[c.datatype](c.name) for c in types] mappers = [d.serialise for d in dtypes] mapped_rows = [ tuple(mapper(row[i]) for i, mapper in enumerate(mappers)) diff --git a/src/fava/ext/fava_ext_test/__init__.py b/src/fava/ext/fava_ext_test/__init__.py index 6e3f32615..ce2094a8e 100644 --- a/src/fava/ext/fava_ext_test/__init__.py +++ b/src/fava/ext/fava_ext_test/__init__.py @@ -27,7 +27,6 @@ if TYPE_CHECKING: # pragma: no cover from flask.wrappers import Response - from fava.beans.funcs import ResultType from fava.core.tree import Tree from fava.core.tree import TreeNode @@ -46,7 +45,7 @@ class Portfolio: title: str rows: list[Row] - types: tuple[ResultType, ...] = ( + types: tuple[tuple[str, type[str | Decimal]], ...] = ( ("account", str), ("balance", Decimal), ("allocation", Decimal), diff --git a/src/fava/ext/portfolio_list/__init__.py b/src/fava/ext/portfolio_list/__init__.py index a920f6285..d0fbb5096 100644 --- a/src/fava/ext/portfolio_list/__init__.py +++ b/src/fava/ext/portfolio_list/__init__.py @@ -17,7 +17,6 @@ from fava.helpers import FavaAPIError if TYPE_CHECKING: # pragma: no cover - from fava.beans.funcs import ResultType from fava.core.tree import Tree from fava.core.tree import TreeNode @@ -36,7 +35,7 @@ class Portfolio: title: str rows: list[Row] - types: tuple[ResultType, ...] = ( + types: tuple[tuple[str, type[str | Decimal]], ...] = ( ("account", str), ("balance", Decimal), ("allocation", Decimal), diff --git a/src/fava/internal_api.py b/src/fava/internal_api.py index d19bf4334..67022fff7 100644 --- a/src/fava/internal_api.py +++ b/src/fava/internal_api.py @@ -100,6 +100,7 @@ def _get_options() -> dict[str, str | list[str]]: def get_ledger_data() -> LedgerData: """Get the report-independent ledger data.""" ledger = g.ledger + all_queries = ledger.all_entries_by_type.Query return LedgerData( ledger.attributes.accounts, @@ -117,7 +118,7 @@ def get_ledger_data() -> LedgerData: ledger.format_decimal.precisions, ledger.attributes.tags, ledger.attributes.years, - ledger.query_shell.queries[: ledger.fava_options.sidebar_show_queries], + all_queries[: ledger.fava_options.sidebar_show_queries], len(ledger.misc.upcoming_events), ledger.extensions.extension_details, ledger.misc.sidebar_links, diff --git a/src/fava/json_api.py b/src/fava/json_api.py index 68e004a73..494202d6a 100644 --- a/src/fava/json_api.py +++ b/src/fava/json_api.py @@ -44,8 +44,8 @@ from flask.wrappers import Response from fava.core.ingest import FileImporters - from fava.core.query_shell import QueryResultTable - from fava.core.query_shell import QueryResultText + from fava.core.query import QueryResultTable + from fava.core.query import QueryResultText from fava.core.tree import SerialisedTreeNode from fava.internal_api import ChartData from fava.util.date import DateRange @@ -241,8 +241,7 @@ def get_payee_accounts(payee: str) -> list[str]: def get_query(query_string: str) -> QueryResultTable | QueryResultText: """Run a Beancount query.""" return g.ledger.query_shell.execute_query_serialised( - g.filtered.entries, - query_string, + g.filtered.entries, query_string ) diff --git a/src/fava/util/excel.py b/src/fava/util/excel.py index adea799b7..657443279 100644 --- a/src/fava/util/excel.py +++ b/src/fava/util/excel.py @@ -74,7 +74,7 @@ def _result_array( types: list[ResultType], rows: list[ResultRow], ) -> list[list[str]]: - result_array = [[name for name, t in types]] + result_array = [[t.name for t in types]] result_array.extend(_row_to_pyexcel(row, types) for row in rows) return result_array @@ -86,7 +86,7 @@ def _row_to_pyexcel(row: ResultRow, header: list[ResultType]) -> list[str]: if not value: result.append(value) continue - type_ = column[1] + type_ = column.datatype if type_ is Decimal: result.append(float(value)) elif type_ is int: diff --git a/stubs/beancount/core/data.pyi b/stubs/beancount/core/data.pyi index 2535b5d05..a6c3e4a1a 100644 --- a/stubs/beancount/core/data.pyi +++ b/stubs/beancount/core/data.pyi @@ -88,6 +88,8 @@ class Note(NamedTuple): date: datetime.date account: Account comment: str + tags: Tags | None + links: Links | None class Event(NamedTuple): meta: Meta diff --git a/tests/__snapshots__/test_core_ingest-test_ingest_examplefile.json b/tests/__snapshots__/test_core_ingest-test_ingest_examplefile.json index c590e32e8..2488af4e4 100644 --- a/tests/__snapshots__/test_core_ingest-test_ingest_examplefile.json +++ b/tests/__snapshots__/test_core_ingest-test_ingest_examplefile.json @@ -3,12 +3,14 @@ "account": "Assets:Checking", "comment": "Hinweis: Zinssatz auf 0,15% ge\u00e4ndert", "date": "2017-02-12", + "links": null, "meta": { "__source__": ";;2017-02-12;;;;;;;Hinweis: Zinssatz auf 0,15% ge\u00e4ndert;", "filename": "TEST_DATA_DIR/import.csv", "lineno": 2 }, - "t": "Note" + "t": "Note", + "tags": null }, { "date": "2017-02-13", @@ -39,7 +41,6 @@ "flag": "*", "links": [], "meta": { - "__duplicate__": true, "__source__": "ATXYZ123400000;1;2017-02-14;2017-02-14;2017-02-14-11.57.44.235044;;EUR;-100,00;Bankomat; BANKOMAT 00000483 K2 UM 11:56;4.467,89", "filename": "TEST_DATA_DIR/import.csv", "lineno": 0 diff --git a/tests/__snapshots__/test_core_query_shell-test_query b/tests/__snapshots__/test_core_query_shell-test_query deleted file mode 100644 index c79cead3d..000000000 --- a/tests/__snapshots__/test_core_query_shell-test_query +++ /dev/null @@ -1,9 +0,0 @@ -Parsed statement: - Select(targets=[Target(expression=Column(name='date'), name=None), Target(expression=Column(name='balance'), name=None)], from_clause=None, where_clause=None, group_by=None, order_by=None, pivot_by=None, limit=None, distinct=None, flatten=None) - -Compiled query: - EvalQuery(c_targets=[EvalTarget(c_expr=DateColumn(), name='date', is_aggregate=False), EvalTarget(c_expr=BalanceColumn(), name='balance', is_aggregate=False)], c_from=None, c_where=None, group_indexes=None, order_indexes=None, ordering=None, limit=None, distinct=None, flatten=None) - -Targets: - 'date': date - 'balance': Inventory \ No newline at end of file diff --git a/tests/__snapshots__/test_core_query_shell-test_query-2 b/tests/__snapshots__/test_core_query_shell-test_query-2 deleted file mode 100644 index 1a5388950..000000000 --- a/tests/__snapshots__/test_core_query_shell-test_query-2 +++ /dev/null @@ -1,43 +0,0 @@ -(None, - [('account', ), - ('sum_position', )], - [ResultRow(account='Assets:US:BofA:Checking', sum_position=(502.25 USD)), - ResultRow(account='Assets:US:ETrade:Cash', sum_position=(8689.74 USD)), - ResultRow(account='Assets:US:ETrade:GLD', sum_position=(1 GLD {191.22 USD, 2010-09-09}, 3 GLD {199.42 USD, 2010-11-06}, 5 GLD {201.02 USD, 2010-11-18}, 1 GLD {204.08 USD, 2010-12-07})), - ResultRow(account='Assets:US:ETrade:ITOT', sum_position=(11 ITOT {99.26 USD, 2010-11-18}, 6 ITOT {101.17 USD, 2010-11-06}, 2 ITOT {103.06 USD, 2010-12-07}, 39 ITOT {103.96 USD, 2010-09-20}, 2 ITOT {105.43 USD, 2010-09-09})), - ResultRow(account='Assets:US:ETrade:VEA', sum_position=(2 VEA {120.61 USD, 2010-09-09}, 2 VEA {126.02 USD, 2010-12-07})), - ResultRow(account='Assets:US:ETrade:VHT', sum_position=(4 VHT {69.34 USD, 2010-09-09}, 15 VHT {69.88 USD, 2010-11-18}, 4 VHT {72.52 USD, 2010-12-07})), - ResultRow(account='Assets:US:Federal:PreTax401k', sum_position=()), - ResultRow(account='Assets:US:Hoogle:Vacation', sum_position=(130 VACHR)), - ResultRow(account='Assets:US:Vanguard:Cash', sum_position=(0.04 USD)), - ResultRow(account='Assets:US:Vanguard:RGAGX', sum_position=(7.939 RGAGX {136.03 USD, 2010-05-17}, 7.875 RGAGX {137.13 USD, 2010-06-14}, 7.786 RGAGX {138.71 USD, 2010-06-28}, 7.752 RGAGX {139.33 USD, 2010-05-31}, 5.710 RGAGX {141.84 USD, 2010-07-12}, 7.515 RGAGX {143.72 USD, 2010-05-03}, 7.483 RGAGX {144.32 USD, 2010-04-19}, 7.457 RGAGX {144.84 USD, 2010-03-08}, 7.448 RGAGX {145.02 USD, 2010-01-11}, 7.425 RGAGX {145.44 USD, 2010-01-25}, 7.304 RGAGX {147.87 USD, 2010-02-22}, 7.229 RGAGX {149.40 USD, 2010-02-08}, 7.211 RGAGX {149.78 USD, 2010-04-05}, 7.188 RGAGX {150.25 USD, 2010-03-22})), - ResultRow(account='Assets:US:Vanguard:VBMPX', sum_position=(4.704 VBMPX {153.07 USD, 2010-01-25}, 4.695 VBMPX {153.36 USD, 2010-02-22}, 4.659 VBMPX {154.54 USD, 2010-02-08}, 4.639 VBMPX {155.18 USD, 2010-05-17}, 4.627 VBMPX {155.58 USD, 2010-04-19}, 4.605 VBMPX {156.33 USD, 2010-03-08}, 4.596 VBMPX {156.64 USD, 2010-01-11}, 4.593 VBMPX {156.79 USD, 2010-05-03}, 4.591 VBMPX {156.82 USD, 2010-03-22}, 4.565 VBMPX {157.74 USD, 2010-04-05}, 4.559 VBMPX {157.94 USD, 2010-05-31}, 4.552 VBMPX {158.17 USD, 2010-06-14}, 4.513 VBMPX {159.55 USD, 2010-06-28}, 3.378 VBMPX {159.88 USD, 2010-07-12})), - ResultRow(account='Liabilities:US:Chase:Slate', sum_position=(-1144.44 USD)), - ResultRow(account='Equity:Opening-Balances', sum_position=(-4363.28 USD)), - ResultRow(account='Income:US:ETrade:Dividends', sum_position=(-70.55 USD)), - ResultRow(account='Income:US:ETrade:Gains', sum_position=(-76.05 USD)), - ResultRow(account='Income:US:Federal:PreTax401k', sum_position=(-16500 IRAUSD)), - ResultRow(account='Income:US:Hoogle:GroupTermLife', sum_position=(-632.32 USD)), - ResultRow(account='Income:US:Hoogle:Match401k', sum_position=(-8250.00 USD)), - ResultRow(account='Income:US:Hoogle:Salary', sum_position=(-119999.88 USD)), - ResultRow(account='Income:US:Hoogle:Vacation', sum_position=(-130 VACHR)), - ResultRow(account='Expenses:Financial:Commissions', sum_position=(179.00 USD)), - ResultRow(account='Expenses:Financial:Fees', sum_position=(48.00 USD)), - ResultRow(account='Expenses:Food:Groceries', sum_position=(2196.80 USD)), - ResultRow(account='Expenses:Food:Restaurant', sum_position=(3795.18 USD)), - ResultRow(account='Expenses:Health:Dental:Insurance', sum_position=(75.40 USD)), - ResultRow(account='Expenses:Health:Life:GroupTermLife', sum_position=(632.32 USD)), - ResultRow(account='Expenses:Health:Medical:Insurance', sum_position=(711.88 USD)), - ResultRow(account='Expenses:Health:Vision:Insurance', sum_position=(1099.80 USD)), - ResultRow(account='Expenses:Home:Electricity', sum_position=(715.00 USD)), - ResultRow(account='Expenses:Home:Internet', sum_position=(879.86 USD)), - ResultRow(account='Expenses:Home:Phone', sum_position=(666.23 USD)), - ResultRow(account='Expenses:Home:Rent', sum_position=(26400.00 USD)), - ResultRow(account='Expenses:Taxes:Y2010:US:CityNYC', sum_position=(4547.92 USD)), - ResultRow(account='Expenses:Taxes:Y2010:US:Federal', sum_position=(27635.92 USD)), - ResultRow(account='Expenses:Taxes:Y2010:US:Federal:PreTax401k', sum_position=(16500.00 IRAUSD)), - ResultRow(account='Expenses:Taxes:Y2010:US:Medicare', sum_position=(2772.12 USD)), - ResultRow(account='Expenses:Taxes:Y2010:US:SDI', sum_position=(29.12 USD)), - ResultRow(account='Expenses:Taxes:Y2010:US:SocSec', sum_position=(7000.04 USD)), - ResultRow(account='Expenses:Taxes:Y2010:US:State', sum_position=(9492.08 USD)), - ResultRow(account='Expenses:Transport:Tram', sum_position=(1440.00 USD))]) \ No newline at end of file diff --git a/tests/__snapshots__/test_core_query_shell-test_query_balances b/tests/__snapshots__/test_core_query_shell-test_query_balances new file mode 100644 index 000000000..d75a77949 --- /dev/null +++ b/tests/__snapshots__/test_core_query_shell-test_query_balances @@ -0,0 +1,69 @@ +QueryResultTable(types=[StrColumn(name='account', dtype='str'), + InventoryColumn(name='SUM((position))', + dtype='Inventory')], + rows=[('Assets:US:BofA:Checking', {'USD': Decimal('502.25')}), + ('Assets:US:ETrade:Cash', {'USD': Decimal('8689.74')}), + ('Assets:US:ETrade:GLD', {'GLD': Decimal('10')}), + ('Assets:US:ETrade:ITOT', {'ITOT': Decimal('60')}), + ('Assets:US:ETrade:VEA', {'VEA': Decimal('4')}), + ('Assets:US:ETrade:VHT', {'VHT': Decimal('23')}), + ('Assets:US:Federal:PreTax401k', {}), + ('Assets:US:Hoogle:Vacation', {'VACHR': Decimal('130')}), + ('Assets:US:Vanguard:Cash', {'USD': Decimal('0.04')}), + ('Assets:US:Vanguard:RGAGX', + {'RGAGX': Decimal('103.322')}), + ('Assets:US:Vanguard:VBMPX', + {'VBMPX': Decimal('63.276')}), + ('Liabilities:US:Chase:Slate', + {'USD': Decimal('-1144.44')}), + ('Equity:Opening-Balances', + {'USD': Decimal('-4363.28')}), + ('Income:US:ETrade:Dividends', + {'USD': Decimal('-70.55')}), + ('Income:US:ETrade:Gains', {'USD': Decimal('-76.05')}), + ('Income:US:Federal:PreTax401k', + {'IRAUSD': Decimal('-16500')}), + ('Income:US:Hoogle:GroupTermLife', + {'USD': Decimal('-632.32')}), + ('Income:US:Hoogle:Match401k', + {'USD': Decimal('-8250.00')}), + ('Income:US:Hoogle:Salary', + {'USD': Decimal('-119999.88')}), + ('Income:US:Hoogle:Vacation', + {'VACHR': Decimal('-130')}), + ('Expenses:Financial:Commissions', + {'USD': Decimal('179.00')}), + ('Expenses:Financial:Fees', {'USD': Decimal('48.00')}), + ('Expenses:Food:Groceries', {'USD': Decimal('2196.80')}), + ('Expenses:Food:Restaurant', + {'USD': Decimal('3795.18')}), + ('Expenses:Health:Dental:Insurance', + {'USD': Decimal('75.40')}), + ('Expenses:Health:Life:GroupTermLife', + {'USD': Decimal('632.32')}), + ('Expenses:Health:Medical:Insurance', + {'USD': Decimal('711.88')}), + ('Expenses:Health:Vision:Insurance', + {'USD': Decimal('1099.80')}), + ('Expenses:Home:Electricity', + {'USD': Decimal('715.00')}), + ('Expenses:Home:Internet', {'USD': Decimal('879.86')}), + ('Expenses:Home:Phone', {'USD': Decimal('666.23')}), + ('Expenses:Home:Rent', {'USD': Decimal('26400.00')}), + ('Expenses:Taxes:Y2010:US:CityNYC', + {'USD': Decimal('4547.92')}), + ('Expenses:Taxes:Y2010:US:Federal', + {'USD': Decimal('27635.92')}), + ('Expenses:Taxes:Y2010:US:Federal:PreTax401k', + {'IRAUSD': Decimal('16500.00')}), + ('Expenses:Taxes:Y2010:US:Medicare', + {'USD': Decimal('2772.12')}), + ('Expenses:Taxes:Y2010:US:SDI', + {'USD': Decimal('29.12')}), + ('Expenses:Taxes:Y2010:US:SocSec', + {'USD': Decimal('7000.04')}), + ('Expenses:Taxes:Y2010:US:State', + {'USD': Decimal('9492.08')}), + ('Expenses:Transport:Tram', + {'USD': Decimal('1440.00')})], + t='table') \ No newline at end of file diff --git a/tests/__snapshots__/test_core_query_shell-test_query_to_file b/tests/__snapshots__/test_core_query_shell-test_query_to_file index 0043c6283..83028c732 100644 --- a/tests/__snapshots__/test_core_query_shell-test_query_to_file +++ b/tests/__snapshots__/test_core_query_shell-test_query_to_file @@ -1,27 +1,27 @@ -(b'account,sum_position (USD),sum_position (VACHR),sum_position (IRAUSD),sum_po' - b'sition (VHT),sum_position (VEA),sum_position (VBMPX),sum_position (RGAGX),su' - b'm_position (ITOT),sum_position (GLD)\r\nAssets:US:BofA:Checking,502.25,,,,' - b',,,,\r\nAssets:US:ETrade:Cash,8689.74,,,,,,,,\r\nAssets:US:ETrade:GLD,,,,,,,' - b',,10.0\r\nAssets:US:ETrade:ITOT,,,,,,,,60.0,\r\nAssets:US:ETrade:VEA,,,,,4.0' - b',,,,\r\nAssets:US:ETrade:VHT,,,,23.0,,,,,\r\nAssets:US:Federal:PreTax401k,,,' - b',,,,,,\r\nAssets:US:Hoogle:Vacation,,130.0,,,,,,,\r\nAssets:US:Vanguard:Cash' - b',0.04,,,,,,,,\r\nAssets:US:Vanguard:RGAGX,,,,,,,103.322,,\r\nAssets:US:Vangu' - b'ard:VBMPX,,,,,,63.276,,,\r\nLiabilities:US:Chase:Slate,-1144.44,,,,,,,,\r\nE' - b'quity:Opening-Balances,-4363.28,,,,,,,,\r\nIncome:US:ETrade:Dividends,-70.' - b'55,,,,,,,,\r\nIncome:US:ETrade:Gains,-76.05,,,,,,,,\r\nIncome:US:Federal:Pre' - b'Tax401k,,,-16500.0,,,,,,\r\nIncome:US:Hoogle:GroupTermLife,-632.32,,,,,,,,' - b'\r\nIncome:US:Hoogle:Match401k,-8250.0,,,,,,,,\r\nIncome:US:Hoogle:Salary,-1' - b'19999.88,,,,,,,,\r\nIncome:US:Hoogle:Vacation,,-130.0,,,,,,,\r\nExpenses:Fin' - b'ancial:Commissions,179.0,,,,,,,,\r\nExpenses:Financial:Fees,48.0,,,,,,,,\r\n' - b'Expenses:Food:Groceries,2196.8,,,,,,,,\r\nExpenses:Food:Restaurant,3795.18' - b',,,,,,,,\r\nExpenses:Health:Dental:Insurance,75.4,,,,,,,,\r\nExpenses:Health' - b':Life:GroupTermLife,632.32,,,,,,,,\r\nExpenses:Health:Medical:Insurance,71' - b'1.88,,,,,,,,\r\nExpenses:Health:Vision:Insurance,1099.8,,,,,,,,\r\nExpenses:' - b'Home:Electricity,715.0,,,,,,,,\r\nExpenses:Home:Internet,879.86,,,,,,,,\r\nE' - b'xpenses:Home:Phone,666.23,,,,,,,,\r\nExpenses:Home:Rent,26400.0,,,,,,,,\r\nE' - b'xpenses:Taxes:Y2010:US:CityNYC,4547.92,,,,,,,,\r\nExpenses:Taxes:Y2010:US:' - b'Federal,27635.92,,,,,,,,\r\nExpenses:Taxes:Y2010:US:Federal:PreTax401k,,,1' - b'6500.0,,,,,,\r\nExpenses:Taxes:Y2010:US:Medicare,2772.12,,,,,,,,\r\nExpenses' - b':Taxes:Y2010:US:SDI,29.12,,,,,,,,\r\nExpenses:Taxes:Y2010:US:SocSec,7000.0' - b'4,,,,,,,,\r\nExpenses:Taxes:Y2010:US:State,9492.08,,,,,,,,\r\nExpenses:Trans' - b'port:Tram,1440.0,,,,,,,,\r\n') \ No newline at end of file +(b'account,SUM((position)) (USD),SUM((position)) (VACHR),SUM((position)) (IRAUS' + b'D),SUM((position)) (VHT),SUM((position)) (VEA),SUM((position)) (VBMPX),SUM((' + b'position)) (RGAGX),SUM((position)) (ITOT),SUM((position)) (GLD)\r\nAssets:' + b'US:BofA:Checking,502.25,,,,,,,,\r\nAssets:US:ETrade:Cash,8689.74,,,,,,,,\r\n' + b'Assets:US:ETrade:GLD,,,,,,,,,10.0\r\nAssets:US:ETrade:ITOT,,,,,,,,60.0,\r\nA' + b'ssets:US:ETrade:VEA,,,,,4.0,,,,\r\nAssets:US:ETrade:VHT,,,,23.0,,,,,\r\nAsse' + b'ts:US:Federal:PreTax401k,,,,,,,,,\r\nAssets:US:Hoogle:Vacation,,130.0,,,,,' + b',,\r\nAssets:US:Vanguard:Cash,0.04,,,,,,,,\r\nAssets:US:Vanguard:RGAGX,,,,,,' + b',103.322,,\r\nAssets:US:Vanguard:VBMPX,,,,,,63.276,,,\r\nLiabilities:US:Chas' + b'e:Slate,-1144.44,,,,,,,,\r\nEquity:Opening-Balances,-4363.28,,,,,,,,\r\nInco' + b'me:US:ETrade:Dividends,-70.55,,,,,,,,\r\nIncome:US:ETrade:Gains,-76.05,,,,' + b',,,,\r\nIncome:US:Federal:PreTax401k,,,-16500.0,,,,,,\r\nIncome:US:Hoogle:Gr' + b'oupTermLife,-632.32,,,,,,,,\r\nIncome:US:Hoogle:Match401k,-8250.0,,,,,,,,\r' + b'\nIncome:US:Hoogle:Salary,-119999.88,,,,,,,,\r\nIncome:US:Hoogle:Vacation,,' + b'-130.0,,,,,,,\r\nExpenses:Financial:Commissions,179.0,,,,,,,,\r\nExpenses:Fi' + b'nancial:Fees,48.0,,,,,,,,\r\nExpenses:Food:Groceries,2196.8,,,,,,,,\r\nExpen' + b'ses:Food:Restaurant,3795.18,,,,,,,,\r\nExpenses:Health:Dental:Insurance,75' + b'.4,,,,,,,,\r\nExpenses:Health:Life:GroupTermLife,632.32,,,,,,,,\r\nExpenses:' + b'Health:Medical:Insurance,711.88,,,,,,,,\r\nExpenses:Health:Vision:Insuranc' + b'e,1099.8,,,,,,,,\r\nExpenses:Home:Electricity,715.0,,,,,,,,\r\nExpenses:Home' + b':Internet,879.86,,,,,,,,\r\nExpenses:Home:Phone,666.23,,,,,,,,\r\nExpenses:H' + b'ome:Rent,26400.0,,,,,,,,\r\nExpenses:Taxes:Y2010:US:CityNYC,4547.92,,,,,,,' + b',\r\nExpenses:Taxes:Y2010:US:Federal,27635.92,,,,,,,,\r\nExpenses:Taxes:Y201' + b'0:US:Federal:PreTax401k,,,16500.0,,,,,,\r\nExpenses:Taxes:Y2010:US:Medicar' + b'e,2772.12,,,,,,,,\r\nExpenses:Taxes:Y2010:US:SDI,29.12,,,,,,,,\r\nExpenses:T' + b'axes:Y2010:US:SocSec,7000.04,,,,,,,,\r\nExpenses:Taxes:Y2010:US:State,9492' + b'.08,,,,,,,,\r\nExpenses:Transport:Tram,1440.0,,,,,,,,\r\n') \ No newline at end of file diff --git a/tests/__snapshots__/test_core_query_shell-test_text_queries b/tests/__snapshots__/test_core_query_shell-test_text_queries new file mode 100644 index 000000000..7fc9e0ba7 --- /dev/null +++ b/tests/__snapshots__/test_core_query_shell-test_text_queries @@ -0,0 +1,6 @@ +parsed statement +---------------- + (select + targets: ( + (target + expression: (column \ No newline at end of file diff --git a/tests/__snapshots__/test_json_api-test_api_imports-2.json b/tests/__snapshots__/test_json_api-test_api_imports-2.json index c590e32e8..2488af4e4 100644 --- a/tests/__snapshots__/test_json_api-test_api_imports-2.json +++ b/tests/__snapshots__/test_json_api-test_api_imports-2.json @@ -3,12 +3,14 @@ "account": "Assets:Checking", "comment": "Hinweis: Zinssatz auf 0,15% ge\u00e4ndert", "date": "2017-02-12", + "links": null, "meta": { "__source__": ";;2017-02-12;;;;;;;Hinweis: Zinssatz auf 0,15% ge\u00e4ndert;", "filename": "TEST_DATA_DIR/import.csv", "lineno": 2 }, - "t": "Note" + "t": "Note", + "tags": null }, { "date": "2017-02-13", @@ -39,7 +41,6 @@ "flag": "*", "links": [], "meta": { - "__duplicate__": true, "__source__": "ATXYZ123400000;1;2017-02-14;2017-02-14;2017-02-14-11.57.44.235044;;EUR;-100,00;Bankomat; BANKOMAT 00000483 K2 UM 11:56;4.467,89", "filename": "TEST_DATA_DIR/import.csv", "lineno": 0 diff --git a/tests/__snapshots__/test_json_api-test_api_query_result-2.json b/tests/__snapshots__/test_json_api-test_api_query_result-2.json index f7673729a..235fe1780 100644 --- a/tests/__snapshots__/test_json_api-test_api_query_result-2.json +++ b/tests/__snapshots__/test_json_api-test_api_query_result-2.json @@ -8,7 +8,7 @@ "types": [ { "dtype": "int", - "name": "sum_day" + "name": "sum(day)" } ] } \ No newline at end of file diff --git a/tests/__snapshots__/test_json_api-test_api_query_result-3.json b/tests/__snapshots__/test_json_api-test_api_query_result-3.json index 7c8fa9024..04be0f397 100644 --- a/tests/__snapshots__/test_json_api-test_api_query_result-3.json +++ b/tests/__snapshots__/test_json_api-test_api_query_result-3.json @@ -3,7 +3,7 @@ [ "2014-01-01", "*", - "", + null, "Opening Balance for checking account", "Assets:US:BofA:Checking", { @@ -20,7 +20,7 @@ [ "2014-01-01", "*", - "", + null, "Opening Balance for checking account", "Equity:Opening-Balances", { @@ -35,7 +35,7 @@ [ "2014-01-01", "*", - "", + null, "Allowed contributions for one year", "Income:US:Federal:PreTax401k", { @@ -52,7 +52,7 @@ [ "2014-01-01", "*", - "", + null, "Allowed contributions for one year", "Assets:US:Federal:PreTax401k", { @@ -367,7 +367,7 @@ [ "2014-01-03", "*", - "", + null, "Employer match for contribution", "Assets:US:Vanguard:Cash", { @@ -384,7 +384,7 @@ [ "2014-01-03", "*", - "", + null, "Employer match for contribution", "Income:US:BayBook:Match401k", { @@ -495,7 +495,7 @@ [ "2014-01-06", "*", - "", + null, "Investing 40% of cash in VBMPX", "Assets:US:Vanguard:VBMPX", { @@ -517,7 +517,7 @@ [ "2014-01-06", "*", - "", + null, "Investing 40% of cash in VBMPX", "Assets:US:Vanguard:Cash", { @@ -535,7 +535,7 @@ [ "2014-01-06", "*", - "", + null, "Investing 60% of cash in RGAGX", "Assets:US:Vanguard:RGAGX", { @@ -559,7 +559,7 @@ [ "2014-01-06", "*", - "", + null, "Investing 60% of cash in RGAGX", "Assets:US:Vanguard:Cash", { @@ -578,7 +578,7 @@ [ "2014-01-06", "*", - "", + null, "Investing 40% of cash in VBMPX", "Assets:US:Vanguard:VBMPX", { @@ -602,7 +602,7 @@ [ "2014-01-06", "*", - "", + null, "Investing 40% of cash in VBMPX", "Assets:US:Vanguard:Cash", { @@ -621,7 +621,7 @@ [ "2014-01-06", "*", - "", + null, "Investing 60% of cash in RGAGX", "Assets:US:Vanguard:RGAGX", { @@ -645,7 +645,7 @@ [ "2014-01-06", "*", - "", + null, "Investing 60% of cash in RGAGX", "Assets:US:Vanguard:Cash", { @@ -1350,7 +1350,7 @@ [ "2014-01-17", "*", - "", + null, "Employer match for contribution", "Assets:US:Vanguard:Cash", { @@ -1369,7 +1369,7 @@ [ "2014-01-17", "*", - "", + null, "Employer match for contribution", "Income:US:BayBook:Match401k", { @@ -1464,7 +1464,7 @@ [ "2014-01-20", "*", - "", + null, "Investing 40% of cash in VBMPX", "Assets:US:Vanguard:VBMPX", { @@ -1488,7 +1488,7 @@ [ "2014-01-20", "*", - "", + null, "Investing 40% of cash in VBMPX", "Assets:US:Vanguard:Cash", { @@ -1507,7 +1507,7 @@ [ "2014-01-20", "*", - "", + null, "Investing 60% of cash in RGAGX", "Assets:US:Vanguard:RGAGX", { @@ -1531,7 +1531,7 @@ [ "2014-01-20", "*", - "", + null, "Investing 60% of cash in RGAGX", "Assets:US:Vanguard:Cash", { @@ -1550,7 +1550,7 @@ [ "2014-01-20", "*", - "", + null, "Investing 40% of cash in VBMPX", "Assets:US:Vanguard:VBMPX", { @@ -1574,7 +1574,7 @@ [ "2014-01-20", "*", - "", + null, "Investing 40% of cash in VBMPX", "Assets:US:Vanguard:Cash", { @@ -1593,7 +1593,7 @@ [ "2014-01-20", "*", - "", + null, "Investing 60% of cash in RGAGX", "Assets:US:Vanguard:RGAGX", { @@ -1617,7 +1617,7 @@ [ "2014-01-20", "*", - "", + null, "Investing 60% of cash in RGAGX", "Assets:US:Vanguard:Cash", { @@ -2246,7 +2246,7 @@ [ "2014-01-31", "*", - "", + null, "Employer match for contribution", "Assets:US:Vanguard:Cash", { @@ -2265,7 +2265,7 @@ [ "2014-01-31", "*", - "", + null, "Employer match for contribution", "Income:US:BayBook:Match401k", { @@ -2294,11 +2294,11 @@ }, { "dtype": "str", - "name": "maxwidth_payee_c48" + "name": "MAXWIDTH(payee, 48)" }, { "dtype": "str", - "name": "maxwidth_narration_c80" + "name": "MAXWIDTH(narration, 80)" }, { "dtype": "str", diff --git a/tests/__snapshots__/test_json_api-test_api_query_result-4.json b/tests/__snapshots__/test_json_api-test_api_query_result-4.json index e31f6d4de..52a55de53 100644 --- a/tests/__snapshots__/test_json_api-test_api_query_result-4.json +++ b/tests/__snapshots__/test_json_api-test_api_query_result-4.json @@ -16,7 +16,7 @@ { "USD": 3453.18 }, - "", + null, [] ], [ @@ -33,7 +33,7 @@ "number": -3453.18 }, {}, - "", + null, [] ], [ @@ -52,7 +52,7 @@ { "IRAUSD": -17500 }, - "", + null, [] ], [ @@ -69,7 +69,7 @@ "number": 17500 }, {}, - "", + null, [] ], [ @@ -424,7 +424,7 @@ { "USD": 600.0 }, - "", + null, [] ], [ @@ -441,7 +441,7 @@ "number": -600.0 }, {}, - "", + null, [] ], [ @@ -573,7 +573,7 @@ { "VBMPX": 14.643 }, - "", + null, [] ], [ @@ -593,7 +593,7 @@ "USD": -480.0, "VBMPX": 14.643 }, - "", + null, [] ], [ @@ -619,7 +619,7 @@ "USD": -480.0, "VBMPX": 14.643 }, - "", + null, [] ], [ @@ -640,7 +640,7 @@ "USD": -1200.02, "VBMPX": 14.643 }, - "", + null, [] ], [ @@ -666,7 +666,7 @@ "USD": -1200.02, "VBMPX": 21.964 }, - "", + null, [] ], [ @@ -687,7 +687,7 @@ "USD": -1440.0, "VBMPX": 21.964 }, - "", + null, [] ], [ @@ -713,7 +713,7 @@ "USD": -1440.0, "VBMPX": 21.964 }, - "", + null, [] ], [ @@ -734,7 +734,7 @@ "USD": -1799.96, "VBMPX": 21.964 }, - "", + null, [] ], [ @@ -1513,7 +1513,7 @@ "USD": -1199.96, "VBMPX": 21.964 }, - "", + null, [] ], [ @@ -1534,7 +1534,7 @@ "USD": -1799.96, "VBMPX": 21.964 }, - "", + null, [] ], [ @@ -1644,7 +1644,7 @@ "USD": -1799.96, "VBMPX": 36.444 }, - "", + null, [] ], [ @@ -1665,7 +1665,7 @@ "USD": -2279.97, "VBMPX": 36.444 }, - "", + null, [] ], [ @@ -1691,7 +1691,7 @@ "USD": -2279.97, "VBMPX": 36.444 }, - "", + null, [] ], [ @@ -1712,7 +1712,7 @@ "USD": -2999.98, "VBMPX": 36.444 }, - "", + null, [] ], [ @@ -1738,7 +1738,7 @@ "USD": -2999.98, "VBMPX": 43.684 }, - "", + null, [] ], [ @@ -1759,7 +1759,7 @@ "USD": -3239.99, "VBMPX": 43.684 }, - "", + null, [] ], [ @@ -1785,7 +1785,7 @@ "USD": -3239.99, "VBMPX": 43.684 }, - "", + null, [] ], [ @@ -1806,7 +1806,7 @@ "USD": -3599.99, "VBMPX": 43.684 }, - "", + null, [] ], [ @@ -2501,7 +2501,7 @@ "USD": -2999.99, "VBMPX": 43.684 }, - "", + null, [] ], [ @@ -2522,7 +2522,7 @@ "USD": -3599.99, "VBMPX": 43.684 }, - "", + null, [] ] ], @@ -2538,7 +2538,7 @@ }, { "dtype": "Amount", - "name": "units_position" + "name": "units(position)" }, { "dtype": "Inventory", diff --git a/tests/__snapshots__/test_json_api-test_api_query_result-5.json b/tests/__snapshots__/test_json_api-test_api_query_result-5.json index 2be79864a..2846b5c10 100644 --- a/tests/__snapshots__/test_json_api-test_api_query_result-5.json +++ b/tests/__snapshots__/test_json_api-test_api_query_result-5.json @@ -1,4 +1,4 @@ { - "contents": "\nShell utility commands (type help ):\n===========================================\nEOF clear exit help history lex parse quit set tokenize\n\nBeancount query commands:\n=========================\nattributes errors from print runcustom targets\nbalances explain journal reload select where", + "contents": "Shell utility commands (type help ):\n===========================================\nEOF describe exit help parse reload set \nclear errors explain history quit run tables\n\nBeancount query commands:\n=========================\nbalances from journal print select targets where", "t": "string" } \ No newline at end of file diff --git a/tests/__snapshots__/test_json_api-test_api_query_result.json b/tests/__snapshots__/test_json_api-test_api_query_result.json index 0a70c48eb..e73adc7f6 100644 --- a/tests/__snapshots__/test_json_api-test_api_query_result.json +++ b/tests/__snapshots__/test_json_api-test_api_query_result.json @@ -247,7 +247,7 @@ }, { "dtype": "Inventory", - "name": "sum_position" + "name": "SUM((position))" } ] } \ No newline at end of file diff --git a/tests/__snapshots__/test_serialisation-test_deserialise_posting_and_format b/tests/__snapshots__/test_serialisation-test_deserialise_posting_and_format index c39a7fcfe..730fff95f 100644 --- a/tests/__snapshots__/test_serialisation-test_deserialise_posting_and_format +++ b/tests/__snapshots__/test_serialisation-test_deserialise_posting_and_format @@ -1,3 +1,3 @@ 2017-12-12 * "Test3" "asdfasd" #tag ^link Assets 10 - Assets 10 EUR @ + Assets 10 EUR diff --git a/tests/__snapshots__/test_serialisation-test_serialise_entry_types.json b/tests/__snapshots__/test_serialisation-test_serialise_entry_types.json index d39c251ec..31a9cf711 100644 --- a/tests/__snapshots__/test_serialisation-test_serialise_entry_types.json +++ b/tests/__snapshots__/test_serialisation-test_serialise_entry_types.json @@ -85,11 +85,13 @@ "account": "Assets:Cash", "comment": "This is some comment or note", "date": "2017-12-20", + "links": [], "meta": { "filename": "", "lineno": 14 }, - "t": "Note" + "t": "Note", + "tags": [] }, { "account": "Assets:Cash", diff --git a/tests/test_core_ingest.py b/tests/test_core_ingest.py index 3f0ca44db..befe5a839 100644 --- a/tests/test_core_ingest.py +++ b/tests/test_core_ingest.py @@ -6,6 +6,7 @@ import pytest +from fava.beans import BEANCOUNT_V3 from fava.beans.abc import Amount from fava.beans.abc import Note from fava.beans.abc import Transaction @@ -61,20 +62,21 @@ def file_account(self, _file: FileMemo) -> str: importer = next(iter(ingest_ledger.ingest.importers.values())) assert importer - info = file_import_info(str(test_data_dir / "import.csv"), importer) + csv_path = test_data_dir / "import.csv" + info = file_import_info(str(csv_path), importer) assert info.account == "Assets:Checking" - info2 = file_import_info("/asdf/basename", Imp("rawfile")) + info2 = file_import_info(str(csv_path), Imp("rawfile")) assert isinstance(info2.account, str) assert info2 == FileImportInfo( "rawfile", "rawfile", local_today(), - "basename", + "import.csv", ) with pytest.raises(FavaAPIError) as err: - file_import_info("/asdf/basename", Invalid("rawfile")) + file_import_info(str(csv_path), Invalid("rawfile")) assert "Some error reason..." in err.value.message @@ -132,8 +134,9 @@ def test_ingest_examplefile( assert isinstance(entries[1].postings[1].units, Amount) assert entries[1].postings[1].units.number == -50.00 assert entries[1].postings[1].units.currency == "EUR" - assert "__duplicate__" not in entries[1].meta - assert "__duplicate__" in entries[2].meta + if not BEANCOUNT_V3: + assert "__duplicate__" not in entries[1].meta + assert "__duplicate__" in entries[2].meta def test_filepath_in_primary_imports_folder( diff --git a/tests/test_core_query_shell.py b/tests/test_core_query_shell.py index 623e577f1..8a6d00f33 100644 --- a/tests/test_core_query_shell.py +++ b/tests/test_core_query_shell.py @@ -1,69 +1,90 @@ from __future__ import annotations -from typing import Any +import sys from typing import TYPE_CHECKING import pytest -from fava.beans.funcs import run_query +from fava.core.query import QueryResultTable +from fava.core.query import QueryResultText from fava.helpers import FavaAPIError if TYPE_CHECKING: # pragma: no cover + from typing import Callable + + from fava.core.query import QueryResult + from .conftest import GetFavaLedger from .conftest import SnapshotFunc -def test_query(snapshot: SnapshotFunc, get_ledger: GetFavaLedger) -> None: +@pytest.fixture +def run_query(get_ledger: GetFavaLedger) -> Callable[[str], QueryResult]: query_ledger = get_ledger("query-example") - def run(query_string: str) -> Any: - return query_ledger.query_shell.execute_query( + def _run_query(query_string: str) -> QueryResult: + return query_ledger.query_shell.execute_query_serialised( query_ledger.all_entries, query_string, ) - def run_text(query_string: str) -> str: + return _run_query + + +@pytest.fixture +def run_text_query( + run_query: Callable[[str], QueryResult], +) -> Callable[[str], str]: + def _run_text_query(query_string: str) -> str: """Run a query that should only return string contents.""" - contents, types, result = run(query_string) - assert types is None - assert result is None - assert isinstance(contents, str) - return contents - - assert run_text("help") - assert run_text("print") - assert run_text("exit") == "Doesn't do anything in Fava's query shell." - assert ( - run_text("help exit") == "Doesn't do anything in Fava's query shell." - ) - snapshot(run_text("explain select date, balance")) - assert ( - run("lex select date, balance")[0] - == "LexToken(SELECT,'SELECT',1,0)\nLexToken(ID,'date',1,7)\nL" - "exToken(COMMA,',',1,11)\nLexToken(ID,'balance',1,13)" - ) + result = run_query(query_string) + assert isinstance(result, QueryResultText) + return result.contents - assert run_text("run") == "custom_query\ncustom query with space" - bal = run("balances") - snapshot(bal) + return _run_text_query - various_types = run( - "select date, payee, weight, change, balance, " - "cost_number, coalesce(cost_number, 1), tags" - ) - assert len(various_types[1]) == 8 - - assert run("run custom_query") == bal - assert run("run 'custom query with space'") == bal - assert run("balances")[1:] == run_query( - query_ledger.all_entries, - query_ledger.options, - "balances", - ) - assert ( - run_text("asdf") - == "ERROR: Syntax error near 'asdf' (at 0)\n asdf\n ^" + +def test_text_queries( + snapshot: SnapshotFunc, run_text_query: Callable[[str], str] +) -> None: + assert run_text_query(".help") + assert run_text_query("print") + + noop_doc = "Doesn't do anything in Fava's query shell." + assert run_text_query(".exit") == noop_doc + assert run_text_query(".help exit") == noop_doc + snapshot(run_text_query(".explain select date, balance")[:100]) + + assert run_text_query(".run") == "custom_query\ncustom query with space" + + +def test_query_balances( + snapshot: SnapshotFunc, run_query: Callable[[str], QueryResult] +) -> None: + assert isinstance(run_query(".run custom_query"), QueryResultTable) + bal = run_query("balances") + if sys.version_info >= (3, 12): + # This fails for some reason on older Pythons, probably some minor + # difference there. + snapshot(bal) + assert run_query(".run custom_query") == bal + assert run_query(".run 'custom query with space'") == bal + + +def test_query_types(run_query: Callable[[str], QueryResult]) -> None: + various_types = run_query( + "select date, payee, weight, position, balance, " + "cost_number, cost_number, tags" ) + assert isinstance(various_types, QueryResultTable) + assert len(various_types.types) == 8 + + +def test_query_errors(run_query: Callable[[str], QueryResult]) -> None: + with pytest.raises(FavaAPIError): + assert run_query(".run custom_query other") + with pytest.raises(FavaAPIError): + assert run_query("asdf") def test_query_to_file( diff --git a/tests/test_json_api.py b/tests/test_json_api.py index 78c4e6adb..8f9c98183 100644 --- a/tests/test_json_api.py +++ b/tests/test_json_api.py @@ -481,7 +481,7 @@ def test_api_add_entries( "select day, position, units(position), balance, payee, tags" " from year = 2014 and month = 1" ), - ("help"), + (".help"), ], ) def test_api_query_result( @@ -503,7 +503,7 @@ def test_api_query_result_error(test_client: FlaskClient) -> None: query_string={"query_string": "nononono"}, ) assert response.status_code == 200 - assert "ERROR: Syntax error near" in response.get_data(as_text=True) + assert "ERROR: syntax error" in response.get_data(as_text=True) def test_api_commodities_empty( diff --git a/tox.ini b/tox.ini index d467b7355..053cf1a3a 100644 --- a/tox.ini +++ b/tox.ini @@ -44,6 +44,7 @@ deps = mypy pylint pytest + requests setuptools types-setuptools types-simplejson