chore: Add Poetry for Python dependency management (#278)

* chore: add poetry to python projects

* docs: update local development documentation

* chore: update dockerfile for backend

* chore: update labels for backend and parser

* chore: update to pass make checks

* chore: update env var parsing in frontend

* style: run linting

* chore: update docker-compose file

.github/labeler.yml

@@ -2,82 +2,81 @@
 # for reference, see https://github.com/actions/labeler
 
 github:
-- .github/**/*
+  - .github/**/*
 
 documentation:
-- CONTRIBUTING.md
-- LICENSE
-- README.md
-- backend/README.md
-- doc/how-to-guides/develop-with-neo4j.md
-- doc/how-to-guides/docker-compose-setup.md
-- doc/introduction/setup-dev.md
-- doc/references/taxonomy-parser.md
-- doc/assets/.empty
-- doc/**/*
+  - CONTRIBUTING.md
+  - LICENSE
+  - README.md
+  - backend/README.md
+  - doc/how-to-guides/develop-with-neo4j.md
+  - doc/how-to-guides/docker-compose-setup.md
+  - doc/introduction/setup-dev.md
+  - doc/references/taxonomy-parser.md
+  - doc/assets/.empty
+  - doc/**/*
 
 docker:
-- backend/Dockerfile
-- taxonomy-editor-frontend/Dockerfile
-- docker-compose.yml
-- docker/dev.yml
+  - backend/Dockerfile
+  - taxonomy-editor-frontend/Dockerfile
+  - docker-compose.yml
+  - docker/dev.yml
 
 backend:
-- backend/editor/__init__.py
-- backend/editor/api.py
-- backend/editor/entries.py
-- backend/editor/exceptions.py
-- backend/editor/graph_db.py
-- backend/editor/models.py
-- backend/editor/normalizer.py
-- backend/editor/settings.py
-- backend/requirements.txt
-- backend/sample/load.py
-- backend/sample/schema.json
-- backend/sample/test-neo4j.json
-- backend/sample/test.json
-- backend/sample/test.txt
-- backend/**/*
+  - backend/editor/__init__.py
+  - backend/editor/api.py
+  - backend/editor/entries.py
+  - backend/editor/exceptions.py
+  - backend/editor/graph_db.py
+  - backend/editor/models.py
+  - backend/editor/normalizer.py
+  - backend/editor/settings.py
+  - backend/pyproject.toml
+  - backend/poetry.lock
+  - backend/sample/load.py
+  - backend/sample/schema.json
+  - backend/sample/test-neo4j.json
+  - backend/sample/test.json
+  - backend/sample/test.txt
+  - backend/**/*
 
 frontend:
-- taxonomy-editor-frontend/**/*
+  - taxonomy-editor-frontend/**/*
 
 dependencies:
-- backend/requirements.txt
-- parser/requirements-dev.txt
-- parser/requirements-test.txt
-- parser/requirements.txt
-- taxonomy-editor-frontend/package.json
+  - backend/pyproject.toml
+  - backend/poetry.lock
+  - parser/pyproject.toml
+  - parser/poetry.lock
+  - taxonomy-editor-frontend/package.json
 
 api:
-- backend/editor/api.py
+  - backend/editor/api.py
 
 nginx:
-- conf/nginx.conf
+  - conf/nginx.conf
 
 neo4j:
-- neo4j/**/*
-  
+  - neo4j/**/*
+
 parser:
-- parser/Makefile
-- parser/openfoodfacts_taxonomy_parser/__init__.py
-- parser/openfoodfacts_taxonomy_parser/exception.py
-- parser/openfoodfacts_taxonomy_parser/parser.py
-- parser/openfoodfacts_taxonomy_parser/unparser.py
-- parser/pyproject.toml
-- parser/requirements-dev.txt
-- parser/requirements-test.txt
-- parser/requirements.txt
-- parser/setup.cfg
-- parser/tests/__init__.py
-- parser/tests/conftest.py
-- parser/tests/data/test.txt
-- parser/tests/integration/__init__.py
-- parser/**/*
+  - parser/Makefile
+  - parser/openfoodfacts_taxonomy_parser/__init__.py
+  - parser/openfoodfacts_taxonomy_parser/exception.py
+  - parser/openfoodfacts_taxonomy_parser/parser.py
+  - parser/openfoodfacts_taxonomy_parser/unparser.py
+  - parser/pyproject.toml
+  - parser/poetry.lock
+  - parser/setup.cfg
+  - parser/tests/__init__.py
+  - parser/tests/conftest.py
+  - parser/tests/data/test.txt
+  - parser/tests/integration/__init__.py
+  - parser/**/*
 
 tests:
-- parser/tests/__init__.py
-- parser/tests/conftest.py
-- parser/tests/data/test.txt
-- parser/tests/integration/__init__.py
-- parser/tests/**/*
+  - parser/tests/__init__.py
+  - parser/tests/conftest.py
+  - parser/tests/data/test.txt
+  - parser/tests/integration/__init__.py
+  - parser/tests/**/*

Makefile

@@ -26,6 +26,35 @@ DOCKER_COMPOSE_TEST=COMPOSE_PROJECT_NAME=test_taxonomy NEO4J_ADMIN_EXPOSE=127.0.
 
 
 
+
+#------------#
+# local dev  #
+#------------#
+
+install: ## Install dependencies
+	@echo "🍜 Installing dependencies"
+	cd taxonomy-editor-frontend && npm install
+	cd backend && poetry install
+	cd parser && poetry install
+	@echo "🍜 Project setup done"
+
+
+local-frontend: ## Run the frontend locally
+	@echo "🍜 Running frontend (ctrl+C to stop)"
+	cd taxonomy-editor-frontend && REACT_APP_API_URL="http://localhost:8080/" npm start
+
+local-backend: ## Run the backend locally
+	@echo "🍜 Running backend (ctrl+C to stop)"
+	cd backend && poetry run uvicorn editor.api:app --host 127.0.0.1 --port 8080 --env-file=../.env --reload
+
+databases: ## Start the databases Docker container for local development
+	@echo "🍜 Running neo4j (ctrl+C to stop)"
+	${DOCKER_COMPOSE} up --build neo4j
+
+add-local-test-data: ## Add test data to the local database
+	@echo "🍜 Adding test data to the database"
+	cd backend && poetry run python sample/load.py sample/test-neo4j.json
+
 #------------#
 # dev setup  #
 #------------#
@@ -68,9 +97,9 @@ quality: backend_quality frontend_quality ## Run all quality checks
 
 backend_quality: ## Run quality checks on backend code
 	@echo "🍜 Quality checks python"
-	${DOCKER_COMPOSE} run --rm taxonomy_api flake8 .
-	${DOCKER_COMPOSE} run --rm taxonomy_api isort --check-only .
-	${DOCKER_COMPOSE} run --rm taxonomy_api black --check .
+	${DOCKER_COMPOSE} run --rm taxonomy_api flake8 --exclude=.venv .
+	${DOCKER_COMPOSE} run --rm taxonomy_api isort --check-only --skip .venv .
+	${DOCKER_COMPOSE} run --rm taxonomy_api black --check --exclude=.venv .
 
 frontend_quality: ## Run quality checks on frontend code
 	@echo "🍜 Quality checks JS"

backend/Dockerfile

@@ -2,31 +2,52 @@
 # Base user uid / gid keep 1000 on prod, align with your user on dev
 ARG USER_UID=1000
 ARG USER_GID=1000
-ARG REQUIREMENTS_FILE=requirements.txt
+ARG DEVELOPMENT=false
 
-FROM python:3.9
+FROM python:3.11-buster as builder
+
+ARG DEVELOPMENT
+RUN pip install poetry==1.4.2
+
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache
+
+WORKDIR /code
+
+COPY backend/pyproject.toml backend/poetry.lock ./
+COPY parser /parser
+
+RUN if [ "$DEVELOPMENT" = "false" ] ; then \
+    poetry install --without dev --no-root ; \
+    else \
+    poetry install --no-root ; \
+    fi && rm -rf $POETRY_CACHE_DIR
+
+
+# The runtime image, used to just run the code provided its virtual environment
+FROM python:3.11-slim-buster as runtime
 
 ARG USER_UID
 ARG USER_GID
-ARG REQUIREMENTS_FILE
 
 WORKDIR /code
+
 RUN groupadd -g $USER_GID off && \
     useradd -u $USER_UID -g off -m off && \
     mkdir -p /home/off && \
     chown off:off -R /code /home/off
 
-COPY backend/requirements*.txt /code/
-COPY backend/editor /code/editor
-COPY parser  /parser
+ENV VIRTUAL_ENV=/code/.venv \
+    PATH="/code/.venv/bin:$PATH"
+
+COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
 
-RUN pip3 install --no-cache-dir --upgrade -r /code/$REQUIREMENTS_FILE
-# this is purely cosmetic
-RUN     ln -s /parser/openfoodfacts_taxonomy_parser /code/openfoodfacts_taxonomy_parser
-RUN --mount=type=cache,id=pip-cache,target=/root/.cache/pip \
-    pip3 install --upgrade -r /code/requirements.txt
+COPY backend/editor /code/editor
+COPY parser /parser
 
 USER off:off
 COPY --chown=off:off ./backend/editor /code/editor
 
-CMD ["uvicorn", "editor.api:app", "--host", "0.0.0.0", "--port", "80"]
+CMD ["uvicorn", "editor.api:app", "--host", "0.0.0.0", "--port", "80"]

backend/editor/entries.py

@@ -111,7 +111,6 @@ async def import_from_github(self, description):
         base_url += filename
         try:
             with tempfile.TemporaryDirectory(prefix="taxonomy-") as tmpdir:
-
                 # File to save the downloaded taxonomy
                 filepath = f"{tmpdir}/{filename}"
 
@@ -216,7 +215,7 @@ async def is_branch_unique(self):
         github_object = GithubOperations(self.taxonomy_name, self.branch_name)
         current_branches = github_object.list_all_branches()
 
-        if (await (result.value()) == []) and (self.branch_name not in current_branches):
+        if (await result.value() == []) and (self.branch_name not in current_branches):
             return True
         else:
             return False
@@ -325,7 +324,7 @@ async def add_node_to_beginning(self, label, entry):
             RETURN first_node
         """
         result = await get_current_transaction().run(query)
-        start_node = await (result.data())[0]["first_node"]
+        start_node = await result.data()[0]["first_node"]
         start_node_label = self.get_label(start_node["id"])  # Get current first node ID
 
         # Rebuild relationships by inserting incoming node at the beginning