add support for Beancount v3

This uses beanquery to replace the functionality in beancount.query, which works with both Beancount v2 and v3 - however there are some differences on columns in beanquery compared to beancount.query. For importers, this sticks to beancount.ingest on v2 and used beangulp for v3. It still expectes importers to conform to the v2 importer protocol for now. Due to changes related to duplicate detection, duplicate detection is not automatically done by Fava here but can still be manually specified as hooks.
beancount · Sep 15, 2024 · 7132586 · 7132586
1 parent 78de554
commit 7132586
Show file tree

Hide file tree

Showing 38 changed files with 790 additions and 595 deletions.
diff --git a/CHANGES b/CHANGES
@@ -1,6 +1,14 @@
 Changelog
 =========
 
+Unreleased (Beancount v3 support)
+---------------------------------
+
+Support for Beancount version 3 was added. Using Beancount 2 is still
+supported. Beancount query support is now provided by the beanquery package,
+which has some minor differences in syntax, the provided columns and functions
+to the bean-query functionality in Beancount version 2.
+
 Unreleased
 ----------
 

diff --git a/Makefile b/Makefile
@@ -95,6 +95,7 @@ run-example:
 .PHONY: bql-grammar
 bql-grammar:
 	tox exec -e lint -- python contrib/scripts.py generate-bql-grammar-json
+	-pre-commit run -a prettier
 
 # Build the distribution (sdist and wheel).
 .PHONY: dist

diff --git a/constraints-old.txt b/constraints-old.txt
@@ -9,20 +9,33 @@ babel==2.7.0
     #   fava (pyproject.toml)
     #   flask-babel
 beancount==2.3.5
+    # via
+    #   fava (pyproject.toml)
+    #   beangulp
+    #   beanquery
+beangulp==0.1.0
+    # via fava (pyproject.toml)
+beanquery==0.1.dev0
     # via fava (pyproject.toml)
 beautifulsoup4==4.0.1
-    # via beancount
+    # via
+    #   beancount
+    #   beangulp
 bottle==0.12.20
     # via
     #   fava (pyproject.toml)
     #   beancount
 chardet==1.0
-    # via beancount
+    # via
+    #   beancount
+    #   beangulp
 cheroot==8.0.0
     # via fava (pyproject.toml)
-click==8.0.0
+click==8.0.1
     # via
     #   fava (pyproject.toml)
+    #   beangulp
+    #   beanquery
     #   flask
 et-xmlfile==1.0.0
     # via openpyxl
@@ -61,10 +74,11 @@ lml==0.0.1
     # via
     #   pyexcel
     #   pyexcel-io
-lxml==5.0.0
+lxml==5.1.0
     # via
     #   fava (pyproject.toml)
     #   beancount
+    #   beangulp
     #   pyexcel-ezodf
     #   pyexcel-ods3
 markdown2==2.3.0
@@ -105,14 +119,17 @@ pytest==7.2.0
     # via
     #   fava (pyproject.toml)
     #   beancount
-python-dateutil==2.1
+python-dateutil==2.6.0
     # via
     #   fava (pyproject.toml)
     #   beancount
+    #   beanquery
 python-gflags==1.3
     # via google-api-python-client
 python-magic==0.4.12
-    # via beancount
+    # via
+    #   beancount
+    #   beangulp
 pytz==2020.1
     # via
     #   fava (pyproject.toml)
@@ -131,6 +148,8 @@ six==1.16.0
     #   python-dateutil
 sniffio==1.1.0
     # via anyio
+tatsu==5.7.4
+    # via beanquery
 texttable==0.8.3
     # via pyexcel
 watchfiles==0.20.0

diff --git a/constraints.txt b/constraints.txt
@@ -13,22 +13,25 @@ babel==2.16.0
     #   fava (pyproject.toml)
     #   flask-babel
     #   sphinx
-beancount==2.3.6
+beancount==3.0.0
+    # via
+    #   fava (pyproject.toml)
+    #   beangulp
+    #   beanquery
+beangulp==0.1.1
+    # via fava (pyproject.toml)
+beanquery==0.1.dev0
     # via fava (pyproject.toml)
 beautifulsoup4==4.12.3
     # via
-    #   beancount
+    #   beangulp
     #   furo
 blinker==1.8.2
     # via flask
-bottle==0.13.1
-    # via beancount
 build==1.2.2
     # via fava (pyproject.toml)
 cachetools==5.5.0
-    # via
-    #   google-auth
-    #   tox
+    # via tox
 certifi==2024.8.30
     # via requests
 cffi==1.17.1
@@ -37,7 +40,7 @@ cfgv==3.4.0
     # via pre-commit
 chardet==5.2.0
     # via
-    #   beancount
+    #   beangulp
     #   pyexcel
     #   tox
 charset-normalizer==3.3.2
@@ -47,6 +50,9 @@ cheroot==10.0.1
 click==8.1.7
     # via
     #   fava (pyproject.toml)
+    #   beancount
+    #   beangulp
+    #   beanquery
     #   flask
 colorama==0.4.6
     # via tox
@@ -76,23 +82,6 @@ flask-babel==4.0.0
     # via fava (pyproject.toml)
 furo==2024.8.6
     # via fava (pyproject.toml)
-google-api-core==2.19.2
-    # via google-api-python-client
-google-api-python-client==2.145.0
-    # via beancount
-google-auth==2.34.0
-    # via
-    #   google-api-core
-    #   google-api-python-client
-    #   google-auth-httplib2
-google-auth-httplib2==0.2.0
-    # via google-api-python-client
-googleapis-common-protos==1.65.0
-    # via google-api-core
-httplib2==0.22.0
-    # via
-    #   google-api-python-client
-    #   google-auth-httplib2
 identify==2.6.1
     # via pre-commit
 idna==3.10
@@ -135,7 +124,8 @@ lml==0.1.0
     #   pyexcel-io
 lxml==5.3.0
     # via
-    #   beancount
+    #   fava (pyproject.toml)
+    #   beangulp
     #   pyexcel-ezodf
     #   pyexcel-ods3
 markdown-it-py==3.0.0
@@ -175,8 +165,6 @@ packaging==24.1
     #   sphinx
     #   tox
     #   tox-uv
-pdfminer2==20151206
-    # via beancount
 pkginfo==1.10.0
     # via twine
 platformdirs==4.3.3
@@ -189,24 +177,9 @@ pluggy==1.5.0
     #   pytest
     #   tox
 ply==3.11
-    # via
-    #   fava (pyproject.toml)
-    #   beancount
+    # via fava (pyproject.toml)
 pre-commit==3.8.0
     # via fava (pyproject.toml)
-proto-plus==1.24.0
-    # via google-api-core
-protobuf==5.28.1
-    # via
-    #   google-api-core
-    #   googleapis-common-protos
-    #   proto-plus
-pyasn1==0.6.1
-    # via
-    #   pyasn1-modules
-    #   rsa
-pyasn1-modules==0.4.1
-    # via google-auth
 pycparser==2.22
     # via cffi
 pyexcel==0.7.0
@@ -234,33 +207,33 @@ pyinstaller-hooks-contrib==2024.8
     # via pyinstaller
 pylint==3.2.7
     # via fava (pyproject.toml)
-pyparsing==3.1.4
-    # via httplib2
 pyproject-api==1.7.1
     # via tox
 pyproject-hooks==1.1.0
     # via build
 pytest==8.3.3
     # via
     #   fava (pyproject.toml)
-    #   beancount
     #   pytest-cov
 pytest-cov==5.0.0
     # via fava (pyproject.toml)
 python-dateutil==2.9.0.post0
-    # via beancount
+    # via
+    #   beancount
+    #   beanquery
 python-magic==0.4.27
-    # via beancount
+    # via beangulp
 pytz==2024.2
     # via flask-babel
 pyyaml==6.0.2
     # via pre-commit
 readme-renderer==44.0
     # via twine
+regex==2024.9.11
+    # via beancount
 requests==2.32.3
     # via
-    #   beancount
-    #   google-api-core
+    #   fava (pyproject.toml)
     #   requests-toolbelt
     #   sphinx
     #   twine
@@ -270,8 +243,6 @@ rfc3986==2.0.0
     # via twine
 rich==13.8.1
     # via twine
-rsa==4.9
-    # via google-auth
 secretstorage==3.3.3
     # via keyring
 setuptools==75.0.0
@@ -282,9 +253,7 @@ setuptools==75.0.0
 simplejson==3.19.3
     # via fava (pyproject.toml)
 six==1.16.0
-    # via
-    #   pdfminer2
-    #   python-dateutil
+    # via python-dateutil
 sniffio==1.3.1
     # via anyio
 snowballstemmer==2.2.0
@@ -313,6 +282,8 @@ sphinxcontrib-qthelp==2.0.0
     # via sphinx
 sphinxcontrib-serializinghtml==2.0.0
     # via sphinx
+tatsu==5.7.4
+    # via beanquery
 texttable==1.7.0
     # via pyexcel
 tomlkit==0.13.2
@@ -331,8 +302,6 @@ types-simplejson==3.19.0.20240801
     # via fava (pyproject.toml)
 typing-extensions==4.12.2
     # via mypy
-uritemplate==4.1.1
-    # via google-api-python-client
 urllib3==2.2.3
     # via
     #   requests

diff --git a/contrib/scripts.py b/contrib/scripts.py
@@ -9,8 +9,9 @@
 from typing import Iterable
 
 import requests
-from beancount.query import query_env
-from beancount.query import query_parser
+from beanquery import query_compile
+from beanquery.parser.parser import KEYWORDS
+from beanquery.sources.beancount import TABLES
 from click import echo
 from click import group
 from click import UsageError
@@ -42,14 +43,14 @@ def generate_bql_grammar_json() -> None:
 
     Should be run whenever the BQL changes."""
 
-    target_env = query_env.TargetsEnvironment()
+    columns = {column for table in TABLES for column in table.columns}
     data = {
-        "columns": sorted(set(_env_to_list(target_env.columns))),
-        "functions": sorted(set(_env_to_list(target_env.functions))),
-        "keywords": sorted({kw.lower() for kw in query_parser.Lexer.keywords}),
+        "columns": sorted(columns),
+        "functions": sorted(query_compile.FUNCTIONS.keys()),
+        "keywords": sorted({kw.lower() for kw in KEYWORDS}),
     }
     path = BASE_PATH / "frontend" / "src" / "codemirror" / "bql-grammar.ts"
-    path.write_text("export default " + json.dumps(data))
+    path.write_text("export default " + json.dumps(data, indent="  "))
 
 
 class MissingPoeditorTokenError(UsageError):