From b24f21ff245400693640517f5a923de309682a46 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 9 Feb 2023 21:21:12 +0530 Subject: [PATCH 01/20] tests: add tests for importing and uploading taxonomy --- backend/editor/api.py | 1 - backend/requirements-test.txt | 3 ++- backend/requirements.txt | 2 -- backend/tests/data/test.txt | 37 +++++++++++++++++++++++++++++++++++ backend/tests/test_api.py | 30 +++++++++++++++++++++++++++- 5 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 backend/tests/data/test.txt diff --git a/backend/editor/api.py b/backend/editor/api.py index ae236e24..7c9354e9 100644 --- a/backend/editor/api.py +++ b/backend/editor/api.py @@ -387,7 +387,6 @@ async def upload_taxonomy( """ Upload taxonomy file to be parsed """ - # use the file name as the taxonomy name taxonomy = TaxonomyGraph(branch, taxonomy_name) if not taxonomy.is_valid_branch_name(): raise HTTPException(status_code=400, detail="branch_name: Enter a valid branch name!") diff --git a/backend/requirements-test.txt b/backend/requirements-test.txt index 6a3d7bca..4cec037c 100644 --- a/backend/requirements-test.txt +++ b/backend/requirements-test.txt @@ -1 +1,2 @@ -pytest==7.1.2 \ No newline at end of file +pytest==7.1.2 +httpx==0.23.3 \ No newline at end of file diff --git a/backend/requirements.txt b/backend/requirements.txt index bc0c2904..1bf37742 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -18,6 +18,4 @@ watchfiles==0.18.1 websockets==10.3 PyGithub==1.56 python-multipart==0.0.5 -httpx==0.23.3 -pytest==7.1.2 -e ../parser/ \ No newline at end of file diff --git a/backend/tests/data/test.txt b/backend/tests/data/test.txt new file mode 100644 index 00000000..5ad4eba3 --- /dev/null +++ b/backend/tests/data/test.txt @@ -0,0 +1,37 @@ +# test taxonomy + +stopwords:fr: aux,au,de,le,du,la,a,et + +synonyms:en:passion fruit, passionfruit + +synonyms:fr:fruit de la passion, maracuja, passion + +en:yogurts, yoghurts +fr:yaourts, yoghourts, yogourts + + Date: Thu, 9 Feb 2023 21:47:32 +0530 Subject: [PATCH 02/20] test the tests --- Makefile | 4 ++++ backend/editor/settings.py | 2 +- backend/tests/test_api.py | 35 ++++++++++++++++++++++++++++++++++- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 40e6d0cc..c8e908a5 100644 --- a/Makefile +++ b/Makefile @@ -87,6 +87,10 @@ backend_tests: ${DOCKER_COMPOSE_TEST} run --rm taxonomy_api pytest /code/tests ${DOCKER_COMPOSE_TEST} stop neo4j +delete_test_db: + @echo "🍜 Deleting test database" + ${DOCKER_COMPOSE_TEST} down -v + checks: quality tests diff --git a/backend/editor/settings.py b/backend/editor/settings.py index c6067462..659f01a9 100644 --- a/backend/editor/settings.py +++ b/backend/editor/settings.py @@ -4,5 +4,5 @@ import os uri = os.environ.get("NEO4J_URI", "bolt://localhost:7687") -access_token = os.environ.get("GITHUB_PAT") +access_token = "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" repo_uri = os.environ.get("REPO_URI", "openfoodfacts/openfoodfacts-server") diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 0b63f9f4..198c5ebc 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -28,7 +28,7 @@ def test_ping(client): def test_import_from_github(client): response = client.post( - "/states/testing_branch/import", + "/test/testing_branch/import", json={"description": "test_description"}, ) assert response.status_code == 200 @@ -46,6 +46,39 @@ def test_upload_taxonomy(client): assert response.json() == True +def test_add_taxonomy_duplicate_branch_name(client): + with open("tests/data/test.txt", "rb") as f: + response = client.post( + "/test_taxonomy_2/test_branch/upload", + files={"file": f}, + data={"description": "test_description"}, + ) + assert response.status_code == 409 + assert response.json() == {"detail": "branch_name: Branch name should be unique!"} + + +def test_add_taxonomy_invalid_branch_name(client): + with open("tests/data/test.txt", "rb") as f: + response = client.post( + "/test_taxonomy/invalid-branch-name/upload", + files={"file": f}, + data={"description": "test_description"}, + ) + assert response.status_code == 400 + assert response.json() == {"detail": "branch_name: Enter a valid branch name!"} + + +def test_add_taxonomy_duplicate_project_name(client): + with open("tests/data/test.txt", "rb") as f: + response = client.post( + "/test_taxonomy/test_branch/upload", + files={"file": f}, + data={"description": "test_description"}, + ) + assert response.status_code == 409 + assert response.json() == {"detail": "Project already exists!"} + + def test_delete_project(neo4j, client): session = neo4j.session() create_project = """ From a9e2e44234747c38f60c68b45a94405952247518 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 9 Feb 2023 21:51:03 +0530 Subject: [PATCH 03/20] remove and revoke ghp --- backend/editor/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/editor/settings.py b/backend/editor/settings.py index 659f01a9..c6067462 100644 --- a/backend/editor/settings.py +++ b/backend/editor/settings.py @@ -4,5 +4,5 @@ import os uri = os.environ.get("NEO4J_URI", "bolt://localhost:7687") -access_token = "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" +access_token = os.environ.get("GITHUB_PAT") repo_uri = os.environ.get("REPO_URI", "openfoodfacts/openfoodfacts-server") From 81346d32dca94d31d956ed8b67f0689eef7f98d5 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 9 Feb 2023 22:03:51 +0530 Subject: [PATCH 04/20] use is instead of == --- backend/tests/test_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 198c5ebc..98aa9159 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -32,7 +32,7 @@ def test_import_from_github(client): json={"description": "test_description"}, ) assert response.status_code == 200 - assert response.json() == True + assert response.json() is True def test_upload_taxonomy(client): @@ -43,7 +43,7 @@ def test_upload_taxonomy(client): data={"description": "test_description"}, ) assert response.status_code == 200 - assert response.json() == True + assert response.json() is True def test_add_taxonomy_duplicate_branch_name(client): From 566d64940caa9bbab82188244b24bfbe92b02283 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 9 Feb 2023 22:15:15 +0530 Subject: [PATCH 05/20] clean db before running tests --- Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Makefile b/Makefile index c8e908a5..8dbd29a5 100644 --- a/Makefile +++ b/Makefile @@ -87,9 +87,6 @@ backend_tests: ${DOCKER_COMPOSE_TEST} run --rm taxonomy_api pytest /code/tests ${DOCKER_COMPOSE_TEST} stop neo4j -delete_test_db: - @echo "🍜 Deleting test database" - ${DOCKER_COMPOSE_TEST} down -v checks: quality tests From 659ff43e249920e710e0d9839824080f2c062241 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 9 Feb 2023 22:17:26 +0530 Subject: [PATCH 06/20] remove unnecessary change --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 8dbd29a5..40e6d0cc 100644 --- a/Makefile +++ b/Makefile @@ -87,7 +87,6 @@ backend_tests: ${DOCKER_COMPOSE_TEST} run --rm taxonomy_api pytest /code/tests ${DOCKER_COMPOSE_TEST} stop neo4j - checks: quality tests From 8bb510a27a2c6088aaf791aa06b9620ac0eaa969 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 9 Feb 2023 22:20:54 +0530 Subject: [PATCH 07/20] Stopped tracking .env File --- .env | 41 ----------------------------------------- 1 file changed, 41 deletions(-) delete mode 100644 .env diff --git a/.env b/.env deleted file mode 100644 index e27aca46..00000000 --- a/.env +++ /dev/null @@ -1,41 +0,0 @@ -# use windows path separator for compat -COMPOSE_PATH_SEPARATOR=; -COMPOSE_FILE=docker-compose.yml;docker/dev.yml - -# version -DOCKER_TAG=dev - -# domain name -TAXONOMY_EDITOR_DOMAIN=taxonomy.localhost -# exposition -TAXONOMY_EDITOR_EXPOSE=127.0.0.1:8091 -# this one is needed only in dev, to tell nginx and fastapi, which port urls should include -# it must either start with : or be empty -PUBLIC_TAXONOMY_EDITOR_PORT=:8091 -# this one is used to expose the websocket in dev and shoudl match PUBLIC_TAXONOMY_EDITOR_PORT but without leading ":" -WDS_SOCKET_PORT=8091 -# API scheme is useful because, in prod, we have to proxy and already proxied request -# and loose the original scheme -API_SCHEME=http - -# This is the PAT (Personal Access Token) -# to create PRs on openfoodfacts-server github project (must be able to read-write PRs) -# you may leave blank in tests… -GITHUB_PAT= - -# repository URI on github. If you want to test PR creation, use a fork -REPO_URI=openfoodfacts/openfoodfacts-server - -# eventually set this to your local user id to avoid permissions errors -# USER_UID=1000 -# USER_GID=1000 - -# Neo4J configurations -NEO4J_BOLT_EXPOSE=127.0.0.1:7687 -NEO4J_ADMIN_EXPOSE=127.0.0.1:7474 -# note: in prod, heap_initial__size and max__size should match, but it's ok like that for dev -NEO4J_server_memory_heap_initial__size=512M -NEO4J_server_memory_heap_max__size=2G -NEO4J_server_memory_pagecache_size=1G -NEO4J_db_memory_transaction_total_max=512M - From a5d2f8df6f34d6974e82c45bcb22e6a09095d5d3 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Tue, 21 Feb 2023 00:37:28 +0530 Subject: [PATCH 08/20] add dump.py --- backend/sample/dump.py | 66 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 backend/sample/dump.py diff --git a/backend/sample/dump.py b/backend/sample/dump.py new file mode 100644 index 00000000..5b71581e --- /dev/null +++ b/backend/sample/dump.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""A script to dump a Neo4J database to a JSON file.""" +import argparse +import json + +from neo4j import GraphDatabase + + +DEFAULT_URL = "bolt://localhost:7687" + + +def get_session(uri=DEFAULT_URL): + """Get a session object for the Neo4J database.""" + return GraphDatabase.driver(uri).session() + + +def dump_nodes(session, file): + """Dump all nodes from the database to a JSON file.""" + node_count = session.run("MATCH (n) RETURN count(n)").single()[0] + for i, node in enumerate(session.run("MATCH (n) RETURN n")): + node_dict = dict(node['n']) + print(node_dict) + labels_list = list(node['n'].labels) + node_dict['labels'] = labels_list + if i < node_count - 1: + f.write(json.dumps(node_dict, ensure_ascii=False) + ',') + else: + f.write(json.dumps(node_dict, ensure_ascii=False)) + + +def dump_relations(session, file): + """Dump all relationships from the database to a JSON file.""" + rels_count = session.run("MATCH (n)-[r]->(m) RETURN count(r)").single()[0] + for i, rel in enumerate(session.run("MATCH (n)-[r]->(m) RETURN r")): + start_node_id = rel['r'].nodes[0].id + end_node_id = rel['r'].nodes[1].id + start_node = session.run( + "MATCH (n) WHERE id(n) = $id RETURN n", {"id": start_node_id} + ).single()["n"]["id"] + end_node = session.run( + "MATCH (n) WHERE id(n) = $id RETURN n", {"id": end_node_id} + ).single()["n"]["id"] + rel_dict = {rel['r'].type: [start_node, end_node]} + if i < rels_count - 1: + f.write(json.dumps(rel_dict, ensure_ascii=False) + ',') + else: + f.write(json.dumps(rel_dict, ensure_ascii=False)) + + +def get_options(args=None): + """Parse command line arguments.""" + parser = argparse.ArgumentParser(description='Dump Neo4J database to JSON file') + parser.add_argument('--url', default=DEFAULT_URL, help='Neo4J database bolt URL') + parser.add_argument('file', help='JSON file name to dump') + return parser.parse_args(args) + + +if __name__ == "__main__": + options = get_options() + session = get_session(options.url) + with open(options.file, 'w') as f: + f.write('{"nodes": [') + dump_nodes(session, f) + f.write('], "relations": [') + dump_relations(session, f) + f.write(']}') From 2c00e3af13391fb3401903154275d5b45d93d353 Mon Sep 17 00:00:00 2001 From: Charles Perier Date: Thu, 26 Oct 2023 12:58:07 +0200 Subject: [PATCH 09/20] track again .env and update test taxonomy --- .env | 41 ++++++++++++++++ backend/tests/data/test.txt | 95 +++++++++++++++++++++++++++++++++++-- 2 files changed, 133 insertions(+), 3 deletions(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 00000000..e27aca46 --- /dev/null +++ b/.env @@ -0,0 +1,41 @@ +# use windows path separator for compat +COMPOSE_PATH_SEPARATOR=; +COMPOSE_FILE=docker-compose.yml;docker/dev.yml + +# version +DOCKER_TAG=dev + +# domain name +TAXONOMY_EDITOR_DOMAIN=taxonomy.localhost +# exposition +TAXONOMY_EDITOR_EXPOSE=127.0.0.1:8091 +# this one is needed only in dev, to tell nginx and fastapi, which port urls should include +# it must either start with : or be empty +PUBLIC_TAXONOMY_EDITOR_PORT=:8091 +# this one is used to expose the websocket in dev and shoudl match PUBLIC_TAXONOMY_EDITOR_PORT but without leading ":" +WDS_SOCKET_PORT=8091 +# API scheme is useful because, in prod, we have to proxy and already proxied request +# and loose the original scheme +API_SCHEME=http + +# This is the PAT (Personal Access Token) +# to create PRs on openfoodfacts-server github project (must be able to read-write PRs) +# you may leave blank in tests… +GITHUB_PAT= + +# repository URI on github. If you want to test PR creation, use a fork +REPO_URI=openfoodfacts/openfoodfacts-server + +# eventually set this to your local user id to avoid permissions errors +# USER_UID=1000 +# USER_GID=1000 + +# Neo4J configurations +NEO4J_BOLT_EXPOSE=127.0.0.1:7687 +NEO4J_ADMIN_EXPOSE=127.0.0.1:7474 +# note: in prod, heap_initial__size and max__size should match, but it's ok like that for dev +NEO4J_server_memory_heap_initial__size=512M +NEO4J_server_memory_heap_max__size=2G +NEO4J_server_memory_pagecache_size=1G +NEO4J_db_memory_transaction_total_max=512M + diff --git a/backend/tests/data/test.txt b/backend/tests/data/test.txt index 5ad4eba3..30f45d5b 100644 --- a/backend/tests/data/test.txt +++ b/backend/tests/data/test.txt @@ -4,34 +4,123 @@ stopwords:fr: aux,au,de,le,du,la,a,et synonyms:en:passion fruit, passionfruit -synonyms:fr:fruit de la passion, maracuja, passion +synonyms:fr:fruit de la passion, fruits de la passion, maracuja, passion en:yogurts, yoghurts fr:yaourts, yoghourts, yogourts +nl:yoghurts +description:en: a yogurts of whatever type +description:fr: un yaourt de n'importe quel type +color:en: white +flavour:en: undef Date: Thu, 2 Nov 2023 10:35:18 +0100 Subject: [PATCH 10/20] clean db before each test --- backend/tests/conftest.py | 6 ++++-- backend/tests/test_api.py | 16 ++++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 7addc9e4..9e39490a 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -16,7 +16,7 @@ def client(): @pytest.fixture -def neo4j(): +def neo4j(scope="session"): """waiting for neo4j to be ready""" uri = os.environ.get("NEO4J_URI", "bolt://localhost:7687") driver = GraphDatabase.driver(uri) @@ -29,4 +29,6 @@ def neo4j(): time.sleep(1) else: connected = True - return driver + session.close() + yield driver + driver.close() diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 98aa9159..886363a9 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -4,20 +4,20 @@ @pytest.fixture(autouse=True) def test_setup(neo4j): # delete all the nodes and relations in the database + session = neo4j.session() query = "MATCH (n) DETACH DELETE n" - neo4j.session().run(query) + session.run(query) query = "DROP INDEX p_test_branch_SearchIds IF EXISTS" - neo4j.session().run(query) + session.run(query) query = "DROP INDEX p_test_branch_SearchTags IF EXISTS" - neo4j.session().run(query) + session.run(query) + session.close() def test_hello(client): response = client.get("/") assert response.status_code == 200 - assert response.json() == { - "message": "Hello user! Tip: open /docs or /redoc for documentation" - } + assert response.json() == {"message": "Hello user! Tip: open /docs or /redoc for documentation"} def test_ping(client): @@ -47,6 +47,8 @@ def test_upload_taxonomy(client): def test_add_taxonomy_duplicate_branch_name(client): + test_upload_taxonomy(client) + with open("tests/data/test.txt", "rb") as f: response = client.post( "/test_taxonomy_2/test_branch/upload", @@ -69,6 +71,8 @@ def test_add_taxonomy_invalid_branch_name(client): def test_add_taxonomy_duplicate_project_name(client): + test_upload_taxonomy(client) + with open("tests/data/test.txt", "rb") as f: response = client.post( "/test_taxonomy/test_branch/upload", From bed219cc2339b533ed4aae8e4c786a9bb5b2e504 Mon Sep 17 00:00:00 2001 From: Charles Perier Date: Tue, 7 Nov 2023 14:27:31 +0100 Subject: [PATCH 11/20] use with for driver, and fix typo on an error name --- backend/editor/entries.py | 6 +++--- backend/editor/exceptions.py | 2 +- backend/tests/conftest.py | 24 +++++++++++------------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/backend/editor/entries.py b/backend/editor/entries.py index f3315118..ffeb7aa0 100644 --- a/backend/editor/entries.py +++ b/backend/editor/entries.py @@ -12,7 +12,7 @@ from .exceptions import GithubBranchExistsError # Custom exceptions from .exceptions import ( GithubUploadError, - TaxnonomyImportError, + TaxonomyImportError, TaxonomyParsingError, TaxonomyUnparsingError, ) @@ -125,7 +125,7 @@ async def import_from_github(self, description): return status except Exception as e: - raise TaxnonomyImportError() from e + raise TaxonomyImportError() from e async def upload_taxonomy(self, filepath, description): """ @@ -137,7 +137,7 @@ async def upload_taxonomy(self, filepath, description): await self.create_project(description) return status except Exception as e: - raise TaxnonomyImportError() from e + raise TaxonomyImportError() from e def dump_taxonomy(self): """ diff --git a/backend/editor/exceptions.py b/backend/editor/exceptions.py index e3f7c453..92a73387 100644 --- a/backend/editor/exceptions.py +++ b/backend/editor/exceptions.py @@ -23,7 +23,7 @@ def __init__(self): return super().__init__(exception_message) -class TaxnonomyImportError(RuntimeError): +class TaxonomyImportError(RuntimeError): """ Raised when attempting to fetch a taxonomy from GitHub """ diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 9e39490a..a795bd51 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -19,16 +19,14 @@ def client(): def neo4j(scope="session"): """waiting for neo4j to be ready""" uri = os.environ.get("NEO4J_URI", "bolt://localhost:7687") - driver = GraphDatabase.driver(uri) - session = driver.session() - connected = False - while not connected: - try: - session.run("MATCH () return 1 limit 1") - except ServiceUnavailable: - time.sleep(1) - else: - connected = True - session.close() - yield driver - driver.close() + with GraphDatabase.driver(uri) as driver: + with driver.session() as session: + connected = False + while not connected: + try: + session.run("MATCH () return 1 limit 1") + except ServiceUnavailable: + time.sleep(1) + else: + connected = True + yield driver From f2cd70f607e30f51b6b5cd29235054b0e7dda3a5 Mon Sep 17 00:00:00 2001 From: Charles Perier Date: Tue, 7 Nov 2023 22:22:44 +0100 Subject: [PATCH 12/20] add pytest-mock to requirements and mock methods using Github servers --- backend/requirements-test.txt | 1 + backend/tests/conftest.py | 20 ++++++++++---------- backend/tests/test_api.py | 18 ++++++++++-------- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/backend/requirements-test.txt b/backend/requirements-test.txt index 4cec037c..d6869fb6 100644 --- a/backend/requirements-test.txt +++ b/backend/requirements-test.txt @@ -1,2 +1,3 @@ pytest==7.1.2 +pytest-mock==3.12.0 httpx==0.23.3 \ No newline at end of file diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index a795bd51..5519b224 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -20,13 +20,13 @@ def neo4j(scope="session"): """waiting for neo4j to be ready""" uri = os.environ.get("NEO4J_URI", "bolt://localhost:7687") with GraphDatabase.driver(uri) as driver: - with driver.session() as session: - connected = False - while not connected: - try: - session.run("MATCH () return 1 limit 1") - except ServiceUnavailable: - time.sleep(1) - else: - connected = True - yield driver + with driver.session() as session: + connected = False + while not connected: + try: + session.run("MATCH () return 1 limit 1") + except ServiceUnavailable: + time.sleep(1) + else: + connected = True + yield driver diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 886363a9..784981b0 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -26,7 +26,9 @@ def test_ping(client): assert response.json().get("ping").startswith("pong @") -def test_import_from_github(client): +def test_import_from_github(client, mocker): + mocker.patch("editor.api.TaxonomyGraph.is_branch_unique", return_value=True) + mocker.patch("editor.api.TaxonomyGraph.import_from_github", return_value=True) response = client.post( "/test/testing_branch/import", json={"description": "test_description"}, @@ -35,7 +37,9 @@ def test_import_from_github(client): assert response.json() is True -def test_upload_taxonomy(client): +def test_upload_taxonomy(client, mocker): + mocker.patch("editor.api.TaxonomyGraph.is_branch_unique", return_value=True) + mocker.patch("editor.api.TaxonomyGraph.upload_taxonomy", return_value=True) with open("tests/data/test.txt", "rb") as f: response = client.post( "/test_taxonomy/test_branch/upload", @@ -46,9 +50,8 @@ def test_upload_taxonomy(client): assert response.json() is True -def test_add_taxonomy_duplicate_branch_name(client): - test_upload_taxonomy(client) - +def test_add_taxonomy_duplicate_branch_name(client, mocker): + mocker.patch("editor.api.TaxonomyGraph.is_branch_unique", return_value=False) with open("tests/data/test.txt", "rb") as f: response = client.post( "/test_taxonomy_2/test_branch/upload", @@ -70,9 +73,8 @@ def test_add_taxonomy_invalid_branch_name(client): assert response.json() == {"detail": "branch_name: Enter a valid branch name!"} -def test_add_taxonomy_duplicate_project_name(client): - test_upload_taxonomy(client) - +def test_add_taxonomy_duplicate_project_name(client, mocker): + mocker.patch("editor.api.TaxonomyGraph.does_project_exist", return_value=True) with open("tests/data/test.txt", "rb") as f: response = client.post( "/test_taxonomy/test_branch/upload", From 373ef6ce8550cd5572de11012bc41ef82dc842db Mon Sep 17 00:00:00 2001 From: Charles Perier Date: Tue, 7 Nov 2023 22:27:26 +0100 Subject: [PATCH 13/20] use with statement for session --- backend/tests/test_api.py | 80 +++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 41 deletions(-) diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 784981b0..4e185ecd 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -4,14 +4,13 @@ @pytest.fixture(autouse=True) def test_setup(neo4j): # delete all the nodes and relations in the database - session = neo4j.session() - query = "MATCH (n) DETACH DELETE n" - session.run(query) - query = "DROP INDEX p_test_branch_SearchIds IF EXISTS" - session.run(query) - query = "DROP INDEX p_test_branch_SearchTags IF EXISTS" - session.run(query) - session.close() + with neo4j.session() as session: + query = "MATCH (n) DETACH DELETE n" + session.run(query) + query = "DROP INDEX p_test_branch_SearchIds IF EXISTS" + session.run(query) + query = "DROP INDEX p_test_branch_SearchTags IF EXISTS" + session.run(query) def test_hello(client): @@ -86,36 +85,35 @@ def test_add_taxonomy_duplicate_project_name(client, mocker): def test_delete_project(neo4j, client): - session = neo4j.session() - create_project = """ - CREATE (n:PROJECT) - SET n.id = 'test_project' - SET n.taxonomy_name = 'test_taxonomy_name' - SET n.branch = 'test_branch' - SET n.description = 'test_description' - SET n.status = 'OPEN' - SET n.project_name = 'p_test_taxonomy_name_test_branch_name' - SET n.created_at = datetime() - """ - - create_project2 = """ - CREATE (n:PROJECT) - SET n.id = 'test_project2' - SET n.taxonomy_name = 'test_taxonomy_name2' - SET n.branch = 'test_branch2' - SET n.description = 'test_description2' - SET n.status = 'OPEN' - SET n.project_name = 'p_test_taxonomy_name_test_branch_name2' - SET n.created_at = datetime() - """ - session.run(create_project) - session.run(create_project2) - - response = client.delete("/test_taxonomy_name/test_branch/delete") - assert response.status_code == 200 - assert response.json() == {"message": "Deleted 1 projects"} - - response = client.delete("/test_taxonomy_name2/test_branch2/delete") - assert response.status_code == 200 - assert response.json() == {"message": "Deleted 1 projects"} - session.close() + with neo4j.session() as session: + create_project = """ + CREATE (n:PROJECT) + SET n.id = 'test_project' + SET n.taxonomy_name = 'test_taxonomy_name' + SET n.branch = 'test_branch' + SET n.description = 'test_description' + SET n.status = 'OPEN' + SET n.project_name = 'p_test_taxonomy_name_test_branch_name' + SET n.created_at = datetime() + """ + + create_project2 = """ + CREATE (n:PROJECT) + SET n.id = 'test_project2' + SET n.taxonomy_name = 'test_taxonomy_name2' + SET n.branch = 'test_branch2' + SET n.description = 'test_description2' + SET n.status = 'OPEN' + SET n.project_name = 'p_test_taxonomy_name_test_branch_name2' + SET n.created_at = datetime() + """ + session.run(create_project) + session.run(create_project2) + + response = client.delete("/test_taxonomy_name/test_branch/delete") + assert response.status_code == 200 + assert response.json() == {"message": "Deleted 1 projects"} + + response = client.delete("/test_taxonomy_name2/test_branch2/delete") + assert response.status_code == 200 + assert response.json() == {"message": "Deleted 1 projects"} From 61ecbdeb773bc28440c5eadfbb503cc4bc5750ae Mon Sep 17 00:00:00 2001 From: Charles Perier Date: Thu, 9 Nov 2023 13:38:55 +0100 Subject: [PATCH 14/20] improve github mock, improve deletion test and fix deletion endpoint --- backend/editor/entries.py | 4 +- backend/editor/github_functions.py | 4 +- backend/sample/dump.py | 26 +++++------ backend/sample/load.py | 26 ++++++----- backend/tests/test_api.py | 74 +++++++++++++----------------- 5 files changed, 64 insertions(+), 70 deletions(-) diff --git a/backend/editor/entries.py b/backend/editor/entries.py index ffeb7aa0..84c9c431 100644 --- a/backend/editor/entries.py +++ b/backend/editor/entries.py @@ -615,11 +615,11 @@ async def delete_taxonomy_project(self, branch, taxonomy_name): """ delete_query = """ - MATCH (n:PROJECT {taxonomy_name: $taxonomy_name, branch: $branch}) + MATCH (n:PROJECT {taxonomy_name: $taxonomy_name, branch_name: $branch_name}) DELETE n """ result = await get_current_transaction().run( - delete_query, taxonomy_name=taxonomy_name, branch=branch + delete_query, taxonomy_name=taxonomy_name, branch_name=branch ) summary = await result.consume() count = summary.counters.nodes_deleted diff --git a/backend/editor/github_functions.py b/backend/editor/github_functions.py index 59bfcff9..85656cb2 100644 --- a/backend/editor/github_functions.py +++ b/backend/editor/github_functions.py @@ -3,7 +3,7 @@ """ from textwrap import dedent -from github import Github +import github from . import settings @@ -21,7 +21,7 @@ def init_driver_and_repo(self): """ Initalize connection to Github with an access token """ - github_driver = Github(settings.access_token) + github_driver = github.Github(settings.access_token) repo = github_driver.get_repo(settings.repo_uri) return repo diff --git a/backend/sample/dump.py b/backend/sample/dump.py index 5b71581e..35de28e6 100644 --- a/backend/sample/dump.py +++ b/backend/sample/dump.py @@ -18,12 +18,12 @@ def dump_nodes(session, file): """Dump all nodes from the database to a JSON file.""" node_count = session.run("MATCH (n) RETURN count(n)").single()[0] for i, node in enumerate(session.run("MATCH (n) RETURN n")): - node_dict = dict(node['n']) + node_dict = dict(node["n"]) print(node_dict) - labels_list = list(node['n'].labels) - node_dict['labels'] = labels_list + labels_list = list(node["n"].labels) + node_dict["labels"] = labels_list if i < node_count - 1: - f.write(json.dumps(node_dict, ensure_ascii=False) + ',') + f.write(json.dumps(node_dict, ensure_ascii=False) + ",") else: f.write(json.dumps(node_dict, ensure_ascii=False)) @@ -32,35 +32,35 @@ def dump_relations(session, file): """Dump all relationships from the database to a JSON file.""" rels_count = session.run("MATCH (n)-[r]->(m) RETURN count(r)").single()[0] for i, rel in enumerate(session.run("MATCH (n)-[r]->(m) RETURN r")): - start_node_id = rel['r'].nodes[0].id - end_node_id = rel['r'].nodes[1].id + start_node_id = rel["r"].nodes[0].id + end_node_id = rel["r"].nodes[1].id start_node = session.run( "MATCH (n) WHERE id(n) = $id RETURN n", {"id": start_node_id} ).single()["n"]["id"] end_node = session.run( "MATCH (n) WHERE id(n) = $id RETURN n", {"id": end_node_id} ).single()["n"]["id"] - rel_dict = {rel['r'].type: [start_node, end_node]} + rel_dict = {rel["r"].type: [start_node, end_node]} if i < rels_count - 1: - f.write(json.dumps(rel_dict, ensure_ascii=False) + ',') + f.write(json.dumps(rel_dict, ensure_ascii=False) + ",") else: f.write(json.dumps(rel_dict, ensure_ascii=False)) def get_options(args=None): """Parse command line arguments.""" - parser = argparse.ArgumentParser(description='Dump Neo4J database to JSON file') - parser.add_argument('--url', default=DEFAULT_URL, help='Neo4J database bolt URL') - parser.add_argument('file', help='JSON file name to dump') + parser = argparse.ArgumentParser(description="Dump Neo4J database to JSON file") + parser.add_argument("--url", default=DEFAULT_URL, help="Neo4J database bolt URL") + parser.add_argument("file", help="JSON file name to dump") return parser.parse_args(args) if __name__ == "__main__": options = get_options() session = get_session(options.url) - with open(options.file, 'w') as f: + with open(options.file, "w") as f: f.write('{"nodes": [') dump_nodes(session, f) f.write('], "relations": [') dump_relations(session, f) - f.write(']}') + f.write("]}") diff --git a/backend/sample/load.py b/backend/sample/load.py index fe2c3ec6..7fd9668d 100644 --- a/backend/sample/load.py +++ b/backend/sample/load.py @@ -13,19 +13,24 @@ def get_session(uri=DEFAULT_URL): return GraphDatabase.driver(uri).session() + def clean_db(session): session.run("""match (p) detach delete(p)""") + def add_node(node, session): labels = node.pop("labels", []) query = f"CREATE (n:{','.join(labels)} $data)" session.run(query, data=node) + def add_link(rel, session): if len(rel) != 1 and len(next(iter(rel.values()))) != 2: - raise ValueError(f""" + raise ValueError( + f""" Expecting relations to by dict like {{"rel_name": ["node1", "node2"]}}, got {rel} - """.trim()) + """.trim() + ) for rel_name, (from_id, to_id) in rel.items(): query = f""" MATCH(source) WHERE source.id = $from_id @@ -46,15 +51,14 @@ def load_jsonl(file_path, session): def get_options(args=None): - parser = argparse.ArgumentParser(description='Import json file to Neo4J database') - parser.add_argument('--url', default=DEFAULT_URL, help='Neo4J database bolt URL') - parser.add_argument('file', help='Json file to import') + parser = argparse.ArgumentParser(description="Import json file to Neo4J database") + parser.add_argument("--url", default=DEFAULT_URL, help="Neo4J database bolt URL") + parser.add_argument("file", help="Json file to import") parser.add_argument( - '--reset', default=False, action="store_true", - help='Clean all database before importing' + "--reset", default=False, action="store_true", help="Clean all database before importing" ) - parser.add_argument('--yes', default=False, action="store_true", - help='Assume yes to all questions' + parser.add_argument( + "--yes", default=False, action="store_true", help="Assume yes to all questions" ) return parser.parse_args(args) @@ -66,7 +70,7 @@ def confirm_clean_db(session): response = input(f"You are about to remove {num_nodes} nodes, are you sure ? [y/N]: ") return response.lower() in ("y", "yes") - + if __name__ == "__main__": options = get_options() session = get_session(options.url) @@ -76,4 +80,4 @@ def confirm_clean_db(session): sys.exit(1) else: clean_db(session) - load_jsonl(options.file, session) \ No newline at end of file + load_jsonl(options.file, session) diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 4e185ecd..915b68c8 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -13,50 +13,63 @@ def test_setup(neo4j): session.run(query) +@pytest.fixture +def github_mock(mocker): + github_mock = mocker.patch("github.Github") + github_mock.return_value.get_repo.return_value.get_branches.return_value = [mocker.Mock()] + return github_mock + + def test_hello(client): response = client.get("/") + assert response.status_code == 200 assert response.json() == {"message": "Hello user! Tip: open /docs or /redoc for documentation"} def test_ping(client): response = client.get("/ping") + assert response.status_code == 200 assert response.json().get("ping").startswith("pong @") -def test_import_from_github(client, mocker): - mocker.patch("editor.api.TaxonomyGraph.is_branch_unique", return_value=True) +def test_import_from_github(client, github_mock, mocker): + # We mock the TaxonomyGraph.import_from_github method, + # which downloads the taxonomy file from a Github URL mocker.patch("editor.api.TaxonomyGraph.import_from_github", return_value=True) + response = client.post( "/test/testing_branch/import", json={"description": "test_description"}, ) + assert response.status_code == 200 assert response.json() is True -def test_upload_taxonomy(client, mocker): - mocker.patch("editor.api.TaxonomyGraph.is_branch_unique", return_value=True) - mocker.patch("editor.api.TaxonomyGraph.upload_taxonomy", return_value=True) +def test_upload_taxonomy(client, github_mock): with open("tests/data/test.txt", "rb") as f: response = client.post( "/test_taxonomy/test_branch/upload", files={"file": f}, data={"description": "test_description"}, ) + assert response.status_code == 200 assert response.json() is True -def test_add_taxonomy_duplicate_branch_name(client, mocker): - mocker.patch("editor.api.TaxonomyGraph.is_branch_unique", return_value=False) +def test_add_taxonomy_duplicate_branch_name(client, github_mock): + github_mock.return_value.get_repo.return_value.get_branches.return_value[0].name = "test_branch" + with open("tests/data/test.txt", "rb") as f: response = client.post( "/test_taxonomy_2/test_branch/upload", files={"file": f}, data={"description": "test_description"}, ) + assert response.status_code == 409 assert response.json() == {"detail": "branch_name: Branch name should be unique!"} @@ -68,52 +81,29 @@ def test_add_taxonomy_invalid_branch_name(client): files={"file": f}, data={"description": "test_description"}, ) + assert response.status_code == 400 assert response.json() == {"detail": "branch_name: Enter a valid branch name!"} -def test_add_taxonomy_duplicate_project_name(client, mocker): - mocker.patch("editor.api.TaxonomyGraph.does_project_exist", return_value=True) +def test_add_taxonomy_duplicate_project_name(client, github_mock): + test_upload_taxonomy(client, github_mock) + with open("tests/data/test.txt", "rb") as f: response = client.post( "/test_taxonomy/test_branch/upload", files={"file": f}, data={"description": "test_description"}, ) + assert response.status_code == 409 assert response.json() == {"detail": "Project already exists!"} -def test_delete_project(neo4j, client): - with neo4j.session() as session: - create_project = """ - CREATE (n:PROJECT) - SET n.id = 'test_project' - SET n.taxonomy_name = 'test_taxonomy_name' - SET n.branch = 'test_branch' - SET n.description = 'test_description' - SET n.status = 'OPEN' - SET n.project_name = 'p_test_taxonomy_name_test_branch_name' - SET n.created_at = datetime() - """ - - create_project2 = """ - CREATE (n:PROJECT) - SET n.id = 'test_project2' - SET n.taxonomy_name = 'test_taxonomy_name2' - SET n.branch = 'test_branch2' - SET n.description = 'test_description2' - SET n.status = 'OPEN' - SET n.project_name = 'p_test_taxonomy_name_test_branch_name2' - SET n.created_at = datetime() - """ - session.run(create_project) - session.run(create_project2) - - response = client.delete("/test_taxonomy_name/test_branch/delete") - assert response.status_code == 200 - assert response.json() == {"message": "Deleted 1 projects"} - - response = client.delete("/test_taxonomy_name2/test_branch2/delete") - assert response.status_code == 200 - assert response.json() == {"message": "Deleted 1 projects"} +def test_delete_project(client, github_mock): + test_upload_taxonomy(client, github_mock) + + response = client.delete("/test_taxonomy/test_branch/delete") + + assert response.status_code == 200 + assert response.json() == {"message": "Deleted 1 projects"} From 9bf6c4470c112b50ec100f5456c93f48d277c6f2 Mon Sep 17 00:00:00 2001 From: Charles Perier Date: Thu, 9 Nov 2023 16:26:51 +0100 Subject: [PATCH 15/20] fix dump and load scripts to make them work in docker --- backend/Dockerfile | 2 ++ backend/sample/dump.py | 3 ++- backend/sample/load.py | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 31e5d8ff..80e51b1a 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -28,5 +28,7 @@ RUN --mount=type=cache,id=pip-cache,target=/root/.cache/pip \ USER off:off COPY --chown=off:off ./backend/editor /code/editor +COPY --chown=off:off ./backend/sample /code/sample +RUN find /code/sample -type f -name '*.py' -exec chmod +x {} \; CMD ["uvicorn", "editor.api:app", "--host", "0.0.0.0", "--port", "80"] diff --git a/backend/sample/dump.py b/backend/sample/dump.py index 35de28e6..96717512 100644 --- a/backend/sample/dump.py +++ b/backend/sample/dump.py @@ -2,11 +2,12 @@ """A script to dump a Neo4J database to a JSON file.""" import argparse import json +import os from neo4j import GraphDatabase -DEFAULT_URL = "bolt://localhost:7687" +DEFAULT_URL = os.environ.get("NEO4J_URI", "bolt://localhost:7687") def get_session(uri=DEFAULT_URL): diff --git a/backend/sample/load.py b/backend/sample/load.py index 7fd9668d..c57a67c3 100644 --- a/backend/sample/load.py +++ b/backend/sample/load.py @@ -3,11 +3,12 @@ """ import argparse import json +import os import sys from neo4j import GraphDatabase -DEFAULT_URL = "bolt://localhost:7687" +DEFAULT_URL = os.environ.get("NEO4J_URI", "bolt://localhost:7687") def get_session(uri=DEFAULT_URL): From fb37f289f12493e4e79ded04faa2b457b4c75ed1 Mon Sep 17 00:00:00 2001 From: Charles Perier Date: Thu, 9 Nov 2023 16:56:32 +0100 Subject: [PATCH 16/20] add test for loading and dumping scripts --- backend/sample/dump.py | 1 - backend/tests/test_api.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/backend/sample/dump.py b/backend/sample/dump.py index 96717512..f4b0fc61 100644 --- a/backend/sample/dump.py +++ b/backend/sample/dump.py @@ -20,7 +20,6 @@ def dump_nodes(session, file): node_count = session.run("MATCH (n) RETURN count(n)").single()[0] for i, node in enumerate(session.run("MATCH (n) RETURN n")): node_dict = dict(node["n"]) - print(node_dict) labels_list = list(node["n"].labels) node_dict["labels"] = labels_list if i < node_count - 1: diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 915b68c8..ad3c8de3 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -1,3 +1,6 @@ +import os +import json +import subprocess import pytest @@ -107,3 +110,30 @@ def test_delete_project(client, github_mock): assert response.status_code == 200 assert response.json() == {"message": "Deleted 1 projects"} + + +def test_load_and_dump(): + # Path to the test data JSON file + test_data_path = "sample/test-neo4j.json" + + # Run load.py to import data into Neo4j database + subprocess.run(["sample/load.py", test_data_path]) + + # Run dump.py to dump the Neo4j database into a JSON file + dumped_file_path = "sample/dumped_test-neo4j.json" + subprocess.run(["sample/dump.py", dumped_file_path]) + + try: + # Read the original and dumped JSON files + with open(test_data_path, "r") as original_file: + original_data = json.load(original_file) + + with open(dumped_file_path, "r") as dumped_file: + dumped_data = json.load(dumped_file) + + # Perform assertions to compare the JSON contents (order-insensitive) + assert original_data == dumped_data + + finally: + # Clean up: remove the dumped file + os.remove(dumped_file_path) From 207e397cdfcfc6bc2e885aeeb645117d32c1e878 Mon Sep 17 00:00:00 2001 From: Charles Perier Date: Thu, 9 Nov 2023 17:26:47 +0100 Subject: [PATCH 17/20] fix import order --- backend/sample/dump.py | 1 - backend/tests/test_api.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/sample/dump.py b/backend/sample/dump.py index f4b0fc61..04dd1283 100644 --- a/backend/sample/dump.py +++ b/backend/sample/dump.py @@ -6,7 +6,6 @@ from neo4j import GraphDatabase - DEFAULT_URL = os.environ.get("NEO4J_URI", "bolt://localhost:7687") diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index ad3c8de3..709ca876 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -1,6 +1,7 @@ -import os import json +import os import subprocess + import pytest From bf9eea401cfa872c60a896fcacc9c0c2eccd5148 Mon Sep 17 00:00:00 2001 From: Charles Perier Date: Thu, 16 Nov 2023 14:32:01 +0100 Subject: [PATCH 18/20] fix dump and load scripts and tests using test taxonomy --- backend/sample/dump.py | 8 +- backend/sample/dumped-test-taxonomy.json | 501 +++++++++++++++++++++++ backend/sample/load.py | 7 +- backend/tests/test_api.py | 16 +- 4 files changed, 524 insertions(+), 8 deletions(-) create mode 100644 backend/sample/dumped-test-taxonomy.json diff --git a/backend/sample/dump.py b/backend/sample/dump.py index 04dd1283..0b7b8e7d 100644 --- a/backend/sample/dump.py +++ b/backend/sample/dump.py @@ -22,9 +22,9 @@ def dump_nodes(session, file): labels_list = list(node["n"].labels) node_dict["labels"] = labels_list if i < node_count - 1: - f.write(json.dumps(node_dict, ensure_ascii=False) + ",") + file.write(json.dumps(node_dict, ensure_ascii=False, default=str) + ",") else: - f.write(json.dumps(node_dict, ensure_ascii=False)) + file.write(json.dumps(node_dict, ensure_ascii=False, default=str)) def dump_relations(session, file): @@ -41,9 +41,9 @@ def dump_relations(session, file): ).single()["n"]["id"] rel_dict = {rel["r"].type: [start_node, end_node]} if i < rels_count - 1: - f.write(json.dumps(rel_dict, ensure_ascii=False) + ",") + file.write(json.dumps(rel_dict, ensure_ascii=False) + ",") else: - f.write(json.dumps(rel_dict, ensure_ascii=False)) + file.write(json.dumps(rel_dict, ensure_ascii=False)) def get_options(args=None): diff --git a/backend/sample/dumped-test-taxonomy.json b/backend/sample/dumped-test-taxonomy.json new file mode 100644 index 00000000..bc54cfbb --- /dev/null +++ b/backend/sample/dumped-test-taxonomy.json @@ -0,0 +1,501 @@ +{ + "nodes": [ + { + "preceding_lines": ["# test taxonomy"], + "src_position": 1, + "id": "__header__", + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "TEXT"] + }, + { + "is_before": "__header__", + "preceding_lines": [], + "src_position": 3, + "tags_fr": ["aux", "au", "de", "le", "du", "la", "a", "et"], + "id": "stopwords:0", + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "STOPWORDS"] + }, + { + "tags_en": ["passion fruit", "passionfruit"], + "is_before": "stopwords:0", + "preceding_lines": [], + "src_position": 5, + "id": "synonyms:0", + "tags_ids_en": ["passion-fruit", "passionfruit"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "SYNONYMS"] + }, + { + "tags_ids_fr": ["fruit-passion", "fruits-passion", "maracuja", "passion"], + "is_before": "synonyms:0", + "preceding_lines": [""], + "src_position": 7, + "tags_fr": [ + "fruit de la passion", + "fruits de la passion", + "maracuja", + "passion" + ], + "id": "synonyms:1", + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "SYNONYMS"] + }, + { + "tags_ids_fr": ["yaourts", "yoghourts", "yogourts"], + "prop_color_en": " white", + "preceding_lines": [], + "src_position": 9, + "tags_fr": ["yaourts", "yoghourts", "yogourts"], + "tags_ids_en": ["yogurts", "yoghurts"], + "tags_ids_nl": ["yoghurts"], + "main_language": "en", + "tags_en": ["yogurts", "yoghurts"], + "is_before": "synonyms:1", + "prop_flavour_en": " undef", + "prop_description_en": " a yogurts of whatever type", + "prop_description_fr": " un yaourt de n'importe quel type", + "id": "en:yogurts", + "tags_nl": ["yoghurts"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "tags_ids_fr": ["yaourts-banane"], + "prop_color_en": " yellow", + "preceding_lines": [], + "src_position": 17, + "tags_fr": ["yaourts à la banane"], + "tags_ids_en": ["banana-yogurts"], + "tags_ids_nl": ["bananenyoghurt"], + "main_language": "en", + "tags_en": ["banana yogurts"], + "is_before": "en:yogurts", + "prop_flavour_en": " banana", + "prop_description_en": " a banana yogurt", + "prop_description_fr": " un yaourt à la banane", + "id": "en:banana-yogurts", + "tags_nl": ["bananenyoghurt"], + "parents": ["en:yogurts"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "tags_ids_fr": ["yaourts-fruit-passion"], + "prop_color_en": " undef", + "preceding_lines": [], + "src_position": 26, + "tags_fr": ["yaourts au fruit de la passion"], + "tags_ids_en": ["passion-fruit-yogurts"], + "tags_ids_nl": ["yoghurts-met-passievrucht"], + "main_language": "en", + "tags_en": ["Passion fruit yogurts"], + "is_before": "en:banana-yogurts", + "prop_flavour_en": " passion fruit", + "id": "en:passion-fruit-yogurts", + "tags_nl": ["yoghurts met passievrucht"], + "parents": ["en:yogurts"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "tags_ids_fr": ["yaourts-alleges"], + "main_language": "fr", + "is_before": "en:passion-fruit-yogurts", + "preceding_lines": [], + "src_position": 33, + "tags_fr": ["yaourts allégés"], + "id": "fr:yaourts-alleges", + "parents": ["en:yogurts"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "tags_ids_fr": ["yaourts-citron"], + "prop_color_en": " yellow", + "preceding_lines": [], + "src_position": 36, + "tags_fr": ["yaourts au citron"], + "tags_ids_en": ["lemon-yogurts"], + "main_language": "en", + "tags_ids_nl": ["yoghurts-met-citroen"], + "tags_en": ["lemon yogurts"], + "is_before": "fr:yaourts-alleges", + "prop_flavour_en": " lemon", + "prop_description_en": " a yogurts with lemon inside", + "prop_description_fr": " un yaourt avec du citron", + "id": "en:lemon-yogurts", + "tags_nl": ["yoghurts met citroen"], + "parents": ["fr:yoghourts"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "tags_ids_fr": ["yaourts-fruit-passion-alleges"], + "main_language": "fr", + "tags_ids_nl": ["magere-yoghurts-met-passievrucht"], + "is_before": "en:lemon-yogurts", + "preceding_lines": [], + "src_position": 45, + "tags_fr": ["yaourts au fruit de la passion allégés"], + "id": "fr:yaourts-fruit-passion-alleges", + "tags_nl": ["magere yoghurts met passievrucht"], + "parents": ["fr:yaourts-fruit-passion", "fr:yaourts-alleges"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "tags_ids_fr": ["yaourts-citron-alleges"], + "main_language": "fr", + "tags_ids_nl": ["magere-citroenyoghurt"], + "is_before": "fr:yaourts-fruit-passion-alleges", + "preceding_lines": [""], + "src_position": 51, + "tags_fr": ["yaourts au citron allégés"], + "prop_description_en": " for light yogurts with lemon", + "id": "fr:yaourts-citron-alleges", + "tags_nl": ["magere citroenyoghurt"], + "parents": ["fr:yaourts-citron", "fr:yaourts-alleges"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "tags_ids_fr": ["yaourts-myrtille"], + "main_language": "fr", + "tags_ids_nl": ["bosbessenyoghurt"], + "is_before": "fr:yaourts-citron-alleges", + "preceding_lines": [], + "src_position": 57, + "prop_flavour_en": " blueberry", + "tags_fr": ["yaourts à la myrtille"], + "id": "fr:yaourts-myrtille", + "prop_flavour_fr": " myrtille", + "tags_nl": ["bosbessenyoghurt"], + "parents": ["fr:yaourt"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "main_language": "en", + "prop_vegan_en": "no", + "tags_en": ["meat"], + "is_before": "fr:yaourts-myrtille", + "preceding_lines": [], + "src_position": 63, + "id": "en:meat", + "tags_ids_en": ["meat"], + "prop_carbon_footprint_fr_foodges_value_fr": "10", + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "main_language": "en", + "tags_en": ["beef"], + "is_before": "en:meat", + "preceding_lines": [], + "src_position": 67, + "id": "en:beef", + "tags_ids_en": ["beef"], + "parents": ["en:meat"], + "prop_carbon_footprint_fr_foodges_value_fr": "15", + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "main_language": "en", + "tags_en": ["roast-beef"], + "is_before": "en:beef", + "preceding_lines": [], + "src_position": 71, + "id": "en:roast-beef", + "tags_ids_en": ["roast-beef"], + "parents": ["en:beef"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "main_language": "en", + "prop_vegan_en": "yes", + "tags_en": ["fake-meat"], + "is_before": "en:roast-beef", + "preceding_lines": [ + "# undef will stop parents from transmitting a value" + ], + "src_position": 74, + "id": "en:fake-meat", + "tags_ids_en": ["fake-meat"], + "parents": ["en:meat"], + "prop_carbon_footprint_fr_foodges_value_fr": "undef", + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "main_language": "en", + "tags_en": ["fake-stuff"], + "preceding_lines": [], + "is_before": "en:fake-meat", + "src_position": 80, + "id": "en:fake-stuff", + "tags_ids_en": ["fake-stuff"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "main_language": "en", + "tags_en": ["fake-duck-meat"], + "is_before": "en:fake-stuff", + "preceding_lines": [], + "src_position": 82, + "id": "en:fake-duck-meat", + "tags_ids_en": ["fake-duck-meat"], + "parents": ["en:fake-stuff", "en:fake-meat"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "prop_vegan_en": "yes", + "main_language": "en", + "tags_en": ["vegetable"], + "is_before": "en:fake-duck-meat", + "preceding_lines": [], + "src_position": 86, + "id": "en:vegetable", + "tags_ids_en": ["vegetable"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "main_language": "en", + "prop_vegan_en": "maybe", + "tags_ids_xx": [ + "something-that-means-soup-in-every-language", + "something-else-that-means-soup-in-every-language" + ], + "tags_en": ["soup"], + "is_before": "en:vegetable", + "preceding_lines": [ + "# the soup yogourt synonym is used to test suggestions matching xx: synonyms" + ], + "src_position": 90, + "id": "en:soup", + "tags_xx": [ + "something that means soup in every language", + "something else that means soup in every language" + ], + "tags_ids_en": ["soup"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "main_language": "en", + "prop_vegan_en": "yes", + "tags_en": ["vegan-soup"], + "is_before": "en:soup", + "preceding_lines": [], + "src_position": 94, + "id": "en:vegan-soup", + "tags_ids_en": ["vegan-soup"], + "parents": ["en:soup"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "main_language": "en", + "prop_vegan_en": "no", + "tags_en": ["fish-soup"], + "is_before": "en:vegan-soup", + "preceding_lines": [], + "src_position": 98, + "id": "en:fish-soup", + "tags_ids_en": ["fish-soup"], + "parents": ["en:soup"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "main_language": "de", + "preceding_lines": [], + "is_before": "en:fish-soup", + "tags_ids_de": ["spätzle"], + "src_position": 102, + "id": "de:spätzle", + "tags_de": ["Spätzle"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "main_language": "en", + "tags_en": ["Kale"], + "is_before": "de:spätzle", + "tags_ids_de": ["grünkohl"], + "preceding_lines": [], + "src_position": 104, + "id": "en:kale", + "tags_de": ["Grünkohl"], + "tags_ids_en": ["kale"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "tags_ids_fr": ["kefir-2-5"], + "preceding_lines": [], + "src_position": 107, + "tags_fr": ["Kéfir 2‚5 %"], + "tags_ids_ru": ["кефир-2.5"], + "tags_ids_en": ["kefir-2.5"], + "main_language": "en", + "tags_ru": ["Кефир 2.5 %", "Кефир 2.5%"], + "tags_en": ["Kefir 2.5 %"], + "is_before": "en:kale", + "tags_ids_de": ["kefir-2.5"], + "id": "en:kefir-2.5", + "tags_de": ["Kefir 2.5 %"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "tags_ids_fr": ["french-entry"], + "main_language": "fr", + "is_before": "en:kefir-2.5", + "tags_ids_de": ["special-value-for-german"], + "preceding_lines": [], + "src_position": 112, + "tags_fr": ["French entry"], + "id": "fr:french-entry", + "tags_de": ["Special value for German"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "tags_ids_fr": ["french-entry-with-default-value"], + "main_language": "fr", + "tags_ids_xx": ["french-entry-with-default-value"], + "is_before": "fr:french-entry", + "tags_ids_de": ["special-value-for-german-2"], + "preceding_lines": [], + "src_position": 115, + "tags_fr": ["French entry with default value"], + "id": "fr:french-entry-with-default-value", + "tags_xx": ["French entry with default value"], + "tags_de": ["Special value for German 2"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "main_language": "xx", + "tags_ids_xx": ["language-less-entry"], + "is_before": "fr:french-entry-with-default-value", + "preceding_lines": [], + "tags_ids_de": ["special-value-for-german-3"], + "src_position": 119, + "id": "xx:language-less-entry", + "tags_xx": ["Language-less entry"], + "tags_de": ["Special value for German 3"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "main_language": "sv", + "tags_sv": ["Ä-märket"], + "tags_ids_xx": ["ä-märket"], + "is_before": "xx:language-less-entry", + "preceding_lines": [ + "# xx: entry with accents, need to match unaccented version" + ], + "src_position": 123, + "tags_ids_sv": ["ä-märket"], + "id": "sv:ä-märket", + "tags_xx": ["Ä-märket"], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "main_language": "en", + "tags_en": [ + "Entry with (parentheses) and some *!#{}@$ characters", + "synonym with *%@$(]% chars" + ], + "is_before": "sv:ä-märket", + "preceding_lines": [], + "src_position": 126, + "id": "en:entry-with-parentheses-and-some-characters", + "tags_ids_en": [ + "entry-with-parentheses-and-some-characters", + "synonym-with-chars" + ], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ENTRY"] + }, + { + "is_before": "en:entry-with-parentheses-and-some-characters", + "preceding_lines": [], + "src_position": 126, + "id": "__footer__", + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "TEXT"] + }, + { + "warnings": [ + "parent not found for child fr:yaourts-myrtille with parent yaourt" + ], + "branch_name": "testbranch", + "created_at": "2023-11-16T00:36:22.663000000+00:00", + "id": "p_test_testbranch", + "taxonomy_name": "test", + "errors": [], + "labels": ["p_test_testbranch", "b_testbranch", "t_test", "ERRORS"] + }, + { + "branch_name": "testbranch", + "description": "just a test branch", + "created_at": "2023-11-16T00:36:22.686000000+00:00", + "id": "p_test_testbranch", + "taxonomy_name": "test", + "status": "OPEN", + "labels": ["PROJECT"] + } + ], + "relations": [ + { "is_child_of": ["en:banana-yogurts", "en:yogurts"] }, + { "is_child_of": ["en:passion-fruit-yogurts", "en:yogurts"] }, + { "is_child_of": ["fr:yaourts-alleges", "en:yogurts"] }, + { "is_child_of": ["en:lemon-yogurts", "en:yogurts"] }, + { + "is_child_of": [ + "fr:yaourts-fruit-passion-alleges", + "en:passion-fruit-yogurts" + ] + }, + { + "is_child_of": ["fr:yaourts-fruit-passion-alleges", "fr:yaourts-alleges"] + }, + { "is_child_of": ["fr:yaourts-citron-alleges", "en:lemon-yogurts"] }, + { "is_child_of": ["fr:yaourts-citron-alleges", "fr:yaourts-alleges"] }, + { "is_child_of": ["en:beef", "en:meat"] }, + { "is_child_of": ["en:roast-beef", "en:beef"] }, + { "is_child_of": ["en:fake-meat", "en:meat"] }, + { "is_child_of": ["en:fake-duck-meat", "en:fake-stuff"] }, + { "is_child_of": ["en:fake-duck-meat", "en:fake-meat"] }, + { "is_child_of": ["en:vegan-soup", "en:soup"] }, + { "is_child_of": ["en:fish-soup", "en:soup"] }, + { "is_before": ["__header__", "stopwords:0"] }, + { "is_before": ["stopwords:0", "synonyms:0"] }, + { "is_before": ["synonyms:0", "synonyms:1"] }, + { "is_before": ["synonyms:1", "en:yogurts"] }, + { "is_before": ["en:yogurts", "en:banana-yogurts"] }, + { "is_before": ["en:banana-yogurts", "en:passion-fruit-yogurts"] }, + { "is_before": ["en:passion-fruit-yogurts", "fr:yaourts-alleges"] }, + { "is_before": ["fr:yaourts-alleges", "en:lemon-yogurts"] }, + { "is_before": ["en:lemon-yogurts", "fr:yaourts-fruit-passion-alleges"] }, + { + "is_before": [ + "fr:yaourts-fruit-passion-alleges", + "fr:yaourts-citron-alleges" + ] + }, + { "is_before": ["fr:yaourts-citron-alleges", "fr:yaourts-myrtille"] }, + { "is_before": ["fr:yaourts-myrtille", "en:meat"] }, + { "is_before": ["en:meat", "en:beef"] }, + { "is_before": ["en:beef", "en:roast-beef"] }, + { "is_before": ["en:roast-beef", "en:fake-meat"] }, + { "is_before": ["en:fake-meat", "en:fake-stuff"] }, + { "is_before": ["en:fake-stuff", "en:fake-duck-meat"] }, + { "is_before": ["en:fake-duck-meat", "en:vegetable"] }, + { "is_before": ["en:vegetable", "en:soup"] }, + { "is_before": ["en:soup", "en:vegan-soup"] }, + { "is_before": ["en:vegan-soup", "en:fish-soup"] }, + { "is_before": ["en:fish-soup", "de:spätzle"] }, + { "is_before": ["de:spätzle", "en:kale"] }, + { "is_before": ["en:kale", "en:kefir-2.5"] }, + { "is_before": ["en:kefir-2.5", "fr:french-entry"] }, + { "is_before": ["fr:french-entry", "fr:french-entry-with-default-value"] }, + { + "is_before": [ + "fr:french-entry-with-default-value", + "xx:language-less-entry" + ] + }, + { "is_before": ["xx:language-less-entry", "sv:ä-märket"] }, + { + "is_before": [ + "sv:ä-märket", + "en:entry-with-parentheses-and-some-characters" + ] + }, + { + "is_before": [ + "en:entry-with-parentheses-and-some-characters", + "__footer__" + ] + } + ] +} diff --git a/backend/sample/load.py b/backend/sample/load.py index c57a67c3..ec591e63 100644 --- a/backend/sample/load.py +++ b/backend/sample/load.py @@ -6,6 +6,7 @@ import os import sys +from datetime import datetime from neo4j import GraphDatabase DEFAULT_URL = os.environ.get("NEO4J_URI", "bolt://localhost:7687") @@ -21,7 +22,11 @@ def clean_db(session): def add_node(node, session): labels = node.pop("labels", []) - query = f"CREATE (n:{','.join(labels)} $data)" + if "created_at" in node: + # Truncate the microseconds to six digits to match the format string + stringified_datetime = node["created_at"][:26] + node["created_at"][29:] + node["created_at"] = datetime.strptime(stringified_datetime, "%Y-%m-%dT%H:%M:%S.%f%z") + query = f"CREATE (n:{':'.join(labels)} $data)" session.run(query, data=node) diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 709ca876..55222f0e 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -115,13 +115,13 @@ def test_delete_project(client, github_mock): def test_load_and_dump(): # Path to the test data JSON file - test_data_path = "sample/test-neo4j.json" + test_data_path = "sample/dumped-test-taxonomy.json" # Run load.py to import data into Neo4j database subprocess.run(["sample/load.py", test_data_path]) # Run dump.py to dump the Neo4j database into a JSON file - dumped_file_path = "sample/dumped_test-neo4j.json" + dumped_file_path = "sample/dump.json" subprocess.run(["sample/dump.py", dumped_file_path]) try: @@ -132,7 +132,17 @@ def test_load_and_dump(): with open(dumped_file_path, "r") as dumped_file: dumped_data = json.load(dumped_file) - # Perform assertions to compare the JSON contents (order-insensitive) + # Label order does not matter: make it a set + for node in original_data["nodes"]: + node["labels"] = set(node["labels"]) + for node in dumped_data["nodes"]: + node["labels"] = set(node["labels"]) + + # Relation order does not matter: sort relations + original_data["relations"].sort(key=json.dumps) + dumped_data["relations"].sort(key=json.dumps) + + # Perform assertions to compare the JSON contents assert original_data == dumped_data finally: From 6036618ae4723bce7d2d8c5e289daa21c6f9bda8 Mon Sep 17 00:00:00 2001 From: Charles Perier Date: Thu, 16 Nov 2023 22:59:58 +0100 Subject: [PATCH 19/20] fix linting and mock access token --- backend/editor/github_functions.py | 2 +- backend/tests/test_api.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/editor/github_functions.py b/backend/editor/github_functions.py index 91e0917a..430c7975 100644 --- a/backend/editor/github_functions.py +++ b/backend/editor/github_functions.py @@ -3,8 +3,8 @@ """ from textwrap import dedent -from fastapi import HTTPException import github +from fastapi import HTTPException from . import settings diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 420f4952..0bfeb08d 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -21,6 +21,7 @@ def test_setup(neo4j): def github_mock(mocker): github_mock = mocker.patch("github.Github") github_mock.return_value.get_repo.return_value.get_branches.return_value = [mocker.Mock()] + mocker.patch("editor.settings.access_token", return_value="mock_access_token") return github_mock From 3bd3b9d334708b8d26c68785114ca1dd93ef58be Mon Sep 17 00:00:00 2001 From: Charles Perier Date: Thu, 16 Nov 2023 23:10:47 +0100 Subject: [PATCH 20/20] fix imports --- backend/sample/load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/sample/load.py b/backend/sample/load.py index ec591e63..3f4063c1 100644 --- a/backend/sample/load.py +++ b/backend/sample/load.py @@ -5,8 +5,8 @@ import json import os import sys - from datetime import datetime + from neo4j import GraphDatabase DEFAULT_URL = os.environ.get("NEO4J_URI", "bolt://localhost:7687")