diff --git a/.github/workflows/full_test.yml b/.github/workflows/full_test.yml index 97f69b1..bf16632 100644 --- a/.github/workflows/full_test.yml +++ b/.github/workflows/full_test.yml @@ -36,7 +36,7 @@ jobs: - name: unit_test run: | source .venv/bin/activate - pytest -vvv -cov + make unit-test make clean - name: integration_test run: | @@ -46,4 +46,4 @@ jobs: precis load-feeds precis check-feeds make run-ci - make test + make integration-test diff --git a/Makefile b/Makefile index 40d3ec5..81304b7 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,10 @@ build: clean: rm -r ${DATA_DIR} -.PHONY: test -test: +.PHONY: integration-test +integration-test: go test tests/integration/*.go -v + +.PHONY: unit-test +unit-test: + pytest -vvv -cov diff --git a/README.md b/README.md index 52b3d10..28e5530 100644 --- a/README.md +++ b/README.md @@ -54,12 +54,17 @@ Then to develop, in one terminal start tailwind by doing `make tw`. Then, in oth If you use `nix` or `nixos`, do `nix develop` to assumme a dev shell and then follow the install instructions above (including creating and activating the venv). +### Unit Tests +Precis has unit tests written against `pytest`. They are automated to run during the pull request pipeline, but you can also run them locally. + +Simply do `make unit-test` to run unit tests. + ### Integration Tests Precis has integration tests that are written in Go. They are automated to run during the pull request pipeline, but they also be run locally. First, install the version of Go specified in `go.mod`. I recommend to use a Golang version manager such as `gvm` or `g`. -Then, start the application using `make run`. Finally, run the integration tests with `make test`. +Then, start the application using `make run`. Finally, run the integration tests with `make integration-test`. # Features ## OPML Import/Export diff --git a/app/app.py b/app/app.py index e53f51f..f8dbe10 100644 --- a/app/app.py +++ b/app/app.py @@ -324,6 +324,7 @@ async def update_feed( preview_only: Annotated[bool, Form()] = False, refresh_enabled: Annotated[bool, Form()] = False, use_script: Annotated[bool, Form()] = False, + retrieve_content: Annotated[bool, Form()] = False, ): try: feed = Feed( @@ -335,6 +336,7 @@ async def update_feed( preview_only=preview_only, refresh_enabled=refresh_enabled, use_script=use_script, + retrieve_content=retrieve_content, ) await bk.update_feed(feed=feed) diff --git a/app/handlers.py b/app/handlers.py index d8e7c62..9310c8b 100644 --- a/app/handlers.py +++ b/app/handlers.py @@ -38,7 +38,14 @@ async def get_content( return EntryContent(url=entry.url, banned=True) try: - html = await self.get_html(url=entry.url, use_script=feed.use_script) + if not feed.retrieve_content: + logger.info( + "Feed is configured to not retrieve content, using rss content" + ) + html = entry.content + else: + html = await self.get_html(url=entry.url, use_script=feed.use_script) + content = self.get_main_content(content=html) if not html or not content: return EntryContent(url=entry.url, unretrievable=True) diff --git a/app/models.py b/app/models.py index 13f6835..70eb37e 100644 --- a/app/models.py +++ b/app/models.py @@ -15,6 +15,7 @@ class Feed(BaseModel): preview_only: bool = False refresh_enabled: bool = True use_script: bool = False + retrieve_content: bool = True @property def rss(self) -> Type[FeedParserDict]: @@ -37,6 +38,7 @@ class FeedEntry(BaseModel): url: str published_at: int updated_at: int + content: str = None authors: list[str] = [] preview: str = None diff --git a/app/rss.py b/app/rss.py index 8c71b39..ee4c8b0 100644 --- a/app/rss.py +++ b/app/rss.py @@ -4,7 +4,7 @@ from logging import getLogger from pathlib import Path from tempfile import SpooledTemporaryFile -from typing import List, Mapping, Type +from typing import List, Mapping from opml import OpmlDocument, OpmlOutline from ruamel.yaml import YAML @@ -54,6 +54,7 @@ async def _process_feed_entry( self, entry: Mapping, feed: Feed, start_ts: int ) -> True: published_time = timegm(entry.published_parsed) + content = "".join(i.get("value", "") for i in entry.get("content", [])) feed_entry = FeedEntry( **{ "title": entry.title, @@ -61,6 +62,7 @@ async def _process_feed_entry( "published_at": timegm(entry.published_parsed), "updated_at": timegm(entry.updated_parsed), "preview": entry.summary, + "content": content if content != "" else None, "feed_id": feed.id, "authors": ( [i["name"] for i in entry.authors] if "authors" in entry else [] @@ -105,7 +107,6 @@ async def _check_feed(self, feed: Feed): logger.info(f"Found {counter} new item(s) for feed {feed.name}") async def check_feeds(self) -> List: - now = int(datetime.now(tz=timezone.utc).timestamp()) logger.info(f"Checking feeds starting at time {now}") @@ -115,7 +116,6 @@ async def check_feeds(self) -> List: await self._check_feed(feed=feed) async def check_feed_by_id(self, id: str) -> List: - feed = self.db.get_feed(id=id) logger.info(f"Manual refresh requested for feed {feed.name}") @@ -123,7 +123,6 @@ async def check_feed_by_id(self, id: str) -> List: await self._check_feed(feed=feed) async def add_feed_entry(self, feed: Feed, entry: FeedEntry) -> None: - logger.info(f"Upserting entry from {feed.name}: {entry.title} - id {entry.id}") self.db.upsert_feed_entry(feed=feed, entry=entry) @@ -170,7 +169,6 @@ async def feeds_to_opml(self) -> OpmlDocument: return out_path, file_name async def opml_to_feeds(self, file: SpooledTemporaryFile): - opml = OpmlDocument.load(fp=file) feeds = [] @@ -193,7 +191,6 @@ async def opml_to_feeds(self, file: SpooledTemporaryFile): self.db.upsert_settings(settings=settings) async def backup(self): - feeds = self.db.get_feeds() settings: GlobalSettings = self.db.get_settings() handlers = self.db.get_handlers() @@ -229,7 +226,6 @@ async def backup(self): return out_path, file_name async def restore(self, file: SpooledTemporaryFile): - bk = load(file) settings = GlobalSettings(db=self.db, **bk.get("settings", {})) diff --git a/app/templates/feed_config.html b/app/templates/feed_config.html index 5872da8..9325adc 100644 --- a/app/templates/feed_config.html +++ b/app/templates/feed_config.html @@ -97,6 +97,11 @@

+