Merge pull request #47 from leozqin/backups

feat: add backups

Makefile

@@ -20,3 +20,7 @@ dev:
 .PHONY: build
 build:
 	docker compose up --build
+
+.PHONY: clean
+clean:
+	rm *.mdb db.json

app/app.py

@@ -89,11 +89,19 @@ async def root(request: Request):
 
 
 @app.get("/about", response_class=HTMLResponse)
-async def about(request: Request):
+async def about(
+    request: Request, update_status: bool = False, update_exception: str = None
+):
 
     return templates.TemplateResponse(
         "about.html",
-        {"request": request, "settings": await bk.get_settings(), **bk.about()},
+        {
+            "request": request,
+            "settings": await bk.get_settings(),
+            "update_status": update_status,
+            "update_exception": update_exception,
+            **bk.about(),
+        },
     )
 
 
@@ -316,6 +324,31 @@ async def export_opml(request: Request):
     return FileResponse(path=write_path, filename=file_name)
 
 
+@app.get("/api/backup/", status_code=status.HTTP_200_OK)
+async def backup(request: Request):
+
+    write_path, file_name = await rss.backup()
+
+    return FileResponse(path=write_path, filename=file_name)
+
+
+@app.post("/api/restore/", status_code=status.HTTP_200_OK)
+async def restore(request: Request, file: UploadFile):
+
+    try:
+        await rss.restore(file=file.file)
+
+        return RedirectResponse(
+            request.url_for("about").include_query_params(update_status=True),
+            status_code=status.HTTP_303_SEE_OTHER,
+        )
+    except Exception as e:
+        return RedirectResponse(
+            request.url_for("about").include_query_params(update_exception=e),
+            status_code=status.HTTP_303_SEE_OTHER,
+        )
+
+
 @app.post("/api/import_opml/", status_code=status.HTTP_200_OK)
 async def import_opml(request: Request, file: UploadFile):
 

app/backend.py

@@ -1,6 +1,7 @@
 from importlib.metadata import version
 from json import dumps, loads
 from logging import getLogger
+from sys import version as py_version
 from time import localtime, strftime
 from typing import List, Mapping, Type
 
@@ -30,6 +31,8 @@ def about(self):
 
         return {
             "version": version("precis"),
+            "python_version": py_version,
+            "fastapi_version": version("fastapi"),
             "docker": IS_DOCKER,
             "storage_handler": type(self.db).__name__,
             "github": GITHUB_LINK,

app/context.py

@@ -95,7 +95,7 @@ class StorageHandler(ABC):
         **{k: "content" for k in content_retrieval_handlers.keys()},
     }
 
-    def reconfigure_handler(self, id: str, config: Mapping):
+    def reconfigure_handler(self, id: str, config: Mapping) -> Type[HandlerBase]:
         return self.handler_map[id](**config)
 
     @abstractmethod
@@ -175,10 +175,11 @@ def upsert_feed_entry(self, feed: Feed, entry: FeedEntry) -> None:
         pass
 
     @abstractmethod
-    def get_entries(self, feed: Feed) -> Mapping[str, FeedEntry | str]:
+    def get_entries(self, feed: Feed) -> List[Mapping[str, str]]:
         """
-        Given a feed, retrieve the entries for that feed and return a mapping
-        of the feed entry ID to the feed entry object
+        Given a feed, retrieve the entries for that feed and return a list of
+        dicts, where each dict has key entry = FeedEntry object, feed_id = the
+        id of the feed for which the entry exists, and id = the entry ID.
         """
         pass
 
@@ -203,12 +204,19 @@ async def get_entry_content(
         """
         Given a feed entry, return the EntryContent object for that entry
         if one exists. If the redrive argument is true or if none exists,
-        create a new one using the URL of the feed entry. Use the get_main_content
+        create a new one using the URL of the feed entry and add it to the
+        database using upsert_entry_content. Use the get_main_content
         static method for the class to clean the content as needed. Use the
         summarize static method for the class to build the summary.
         """
         pass
 
+    @abstractmethod
+    async def upsert_entry_content(self, content: EntryContent):
+        """
+        Given an EntryContent object, insert it into the database.
+        """
+
     @abstractmethod
     def upsert_handler(
         self,

app/rss.py

@@ -1,5 +1,6 @@
 from calendar import timegm
 from datetime import datetime, timezone
+from json import dump, load
 from logging import getLogger
 from pathlib import Path
 from tempfile import SpooledTemporaryFile
@@ -10,7 +11,7 @@
 
 from app.constants import CONFIG_DIR, DATA_DIR
 from app.context import GlobalSettings, StorageHandler
-from app.models import Feed, FeedEntry
+from app.models import EntryContent, Feed, FeedEntry
 
 logger = getLogger("uvicorn.error")
 
@@ -171,3 +172,73 @@ async def opml_to_feeds(self, file: SpooledTemporaryFile):
         if not settings.finished_onboarding:
             settings.finished_onboarding = True
             self.db.upsert_settings(settings=settings)
+
+    async def backup(self):
+
+        feeds = self.db.get_feeds()
+        settings: GlobalSettings = self.db.get_settings()
+        handlers = self.db.get_handlers()
+
+        backup = {
+            "settings": settings.dict(exclude={"db"}),
+            "handlers": {k: v.dict() for k, v in handlers.items() if v},
+            "feeds": [i.dict() for i in feeds],
+            "feed_entries": {},
+            "entry_content": {},
+            "poll_state": {},
+        }
+
+        for feed in feeds:
+            feed: Feed
+            backup["poll_state"][feed.id] = self.db.get_poll_state(feed)
+            entries = [i["entry"] for i in self.db.get_entries(feed)]
+            entry_content = {i.id: await self.db.get_entry_content(i) for i in entries}
+            backup["feed_entries"][feed.id] = [i.dict() for i in entries]
+            backup["entry_content"][feed.id] = {
+                k: v.dict() for k, v in entry_content.items()
+            }
+
+        str_now = datetime.now().strftime("%Y%m%d%H%M%S")
+        file_name = f"precis_backup_{str_now}.json"
+        out_path = Path(DATA_DIR, file_name).resolve()
+
+        logger.info(f"writing backup to {out_path}")
+
+        with open(out_path, "w+") as fp:
+            dump(backup, fp)
+
+        return out_path, file_name
+
+    async def restore(self, file: SpooledTemporaryFile):
+
+        bk = load(file)
+
+        settings = GlobalSettings(db=self.db, **bk.get("settings", {}))
+        settings.finished_onboarding = True
+
+        handlers = [
+            self.db.reconfigure_handler(id=k, config=v)
+            for k, v in bk.get("handlers", {}).items()
+        ]
+        feeds = [Feed(**i) for i in bk.get("feeds", [])]
+
+        for handler in handlers:
+            self.db.upsert_handler(handler=handler)
+
+        self.db.upsert_settings(settings)
+
+        for feed in feeds:
+            self.db.upsert_feed(feed)
+
+        feed_entries: dict = bk.get("feed_entries", {})
+        for feed, entries in feed_entries.items():
+            feed_obj = self.db.get_feed(id=feed)
+            for entry in entries:
+                entry_obj = FeedEntry(**entry)
+                self.db.upsert_feed_entry(feed=feed_obj, entry=entry_obj)
+
+        content: dict = bk.get("entry_content", {})
+        for contents in content.values():
+            for i in contents.values():
+                content_obj = EntryContent(**i)
+                self.db.upsert_entry_content(content=content_obj)

app/storage/lmdb.py

@@ -227,14 +227,18 @@ async def get_entry_content(
                 summary=summary if summary else None,
             )
 
-            with self.db.begin(db=self._db(Named.entry_content), write=True) as txn:
-                txn.replace(
-                    self._serialize(entry_content.id),
-                    self._serialize(entry_content),
-                )
+            await self.upsert_entry_content(entry_content)
 
             return entry_content
 
+    async def upsert_entry_content(self, content: EntryContent):
+
+        with self.db.begin(db=self._db(Named.entry_content), write=True) as txn:
+            txn.replace(
+                self._serialize(content.id),
+                self._serialize(content),
+            )
+
     def upsert_handler(self, handler: type[HandlerBase]) -> None:
 
         with self.db.begin(self._db(Named.handler), write=True) as txn:

app/storage/tinydb.py

@@ -160,14 +160,19 @@ async def get_entry_content(
                 summary=summary if summary else None,
             )
 
-            query = Query().id.matches(entry.id)
-            table.upsert(
-                {"id": entry_content.id, "entry_contents": entry_content.dict()},
-                cond=query,
-            )
+            await self.upsert_entry_content(content=entry_content)
 
             return entry_content
 
+    async def upsert_entry_content(self, content: EntryContent):
+        table = self.db.table("entry_contents")
+        query = Query().id.matches(content.id)
+
+        table.upsert(
+            {"id": content.id, "entry_contents": content.dict()},
+            cond=query,
+        )
+
     def upsert_handler(
         self,
         handler: Type[

app/templates/about.html

@@ -34,10 +34,10 @@ <h1 class="text-5xl lg:text-3xl text-neutral">
               </a>
             </li>
             <li>
-                <a href="{{ url_for('settings') }}" class="text-4xl lg:text-xl text-nowrap no-underline">
-                  Edit Global Settings
-                </a>
-              </li>
+              <a href="{{ url_for('settings') }}" class="text-4xl lg:text-xl text-nowrap no-underline">
+                Edit Global Settings
+              </a>
+            </li>
             <li>
               <a href="{{ url_for('root') }}" class="text-4xl lg:text-xl text-nowrap no-underline">
                 Return to All Feeds
@@ -47,23 +47,74 @@ <h1 class="text-5xl lg:text-3xl text-neutral">
         </div>
       </div>
   </section>
-  <section class="w-5/6 md:w-1/2 justify-center mx-auto">
-    <img src="../../assets/logo-with-name-light.svg" />
+  <section class="w-5/6 md:w-2/3 justify-center mx-auto">
+    <img src="../../assets/logo-with-name-light.svg" class="justify-center mx-auto"/>
     <div class="py-5 text-2xl m3:text-xl">
-        <h3>
-            Version: {{ version }}
-        </h3>
-        <h3>
-            Running on: {% if docker %}Docker{% else %}Python{% endif %}
-        </h3>
-        <h3>
-            Storage Handler: {{ storage_handler }}
-        </h3>
-        <h3>
-            GitHub: <a class="link" href="{{ github }}">{{ github }}</a>
-        </h3>
+      <h3 class="py-1">
+        Version: {{ version }}
+      </h3>
+      <h3 class="py-1">
+        Running on: {% if docker %}Docker{% else %}Python{% endif %}
+      </h3>
+      <h3 class="py-1">
+        Python Version: {{ python_version }}
+      </h3>
+      <h3 class="py-1">
+        FastAPI Version: {{ fastapi_version }}
+      </h3>
+      <h3 class="py-1">
+        Storage Handler: {{ storage_handler }}
+      </h3>
+      <h3 class="py-1">
+        Active Theme: {{settings.theme.value}}
+      </h3>
+      <h3 class="py-1">
+        GitHub: <a class="link" href="{{ github }}">{{ github }}</a>
+      </h3>
+    </div>
+  </section>
+  <div class="divider"></div>
+  <section class="w-5/6 md:w-3/5 justify-center mx-auto">
+    <div>
+      <h2 class="text-4xl lg:text-2xl my-5 justify-center flex">
+        Create Backup
+      </h2>
+      <p class="text-2xl lg:text-xl my-5 justify-center flex">
+        A backup is a json file that contains a point-in-time snapshot of all of the data in your instance of Precis.
+        This includes API credentials, if you've configured them, so treat this file as you would a secret!
+        Backups can be restored across storage handlers.
+      </p>
+      <a href="{{ url_for('backup') }}" role="button"
+        class="btn btn-outline w-1/2 lg:w-2/5 mx-auto flex justify-center my-5 text-3xl lg:text-xl">
+        Backup
+      </a>
+    </div>
+  </section>
+  <section class="py-10">
+    <h2 class="text-4xl lg:text-2xl my-5 justify-center flex">
+      Restore from Backup
+    </h2>
+    <form id="opml_import" method="post" action="/api/restore" enctype="multipart/form-data">
+      <input type="file" name="file"
+        class="file-input file-input-bordered file-input-primary w-1/2 lg:w-2/5 mx-auto flex justify-center my-5 text-3xl lg:text-xl" />
+    </form>
+    <div class="w-1/2 lg:w-2/5 justify-center my-5 mx-auto">
+      <button type="submit" form="opml_import"
+        class="btn btn-outline w-full mx-auto flex justify-center my-5 text-3xl lg:text-xl">
+        Restore
+        {% if update_status %} &#9989 {% endif %}
+        {% if update_exception %} &#10060 {% endif %}
+      </button>
+      {% if update_exception %}
+      <div role="alert" class="alert alert-error">
+        <svg xmlns="http://www.w3.org/2000/svg" class="stroke-current shrink-0 h-6 w-6" fill="none" viewBox="0 0 24 24">
+          <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
+            d="M10 14l2-2m0 0l2-2m-2 2l-2-2m2 2l2 2m7-2a9 9 0 11-18 0 9 9 0 0118 0z" />
+        </svg>
+        <span>{{ update_exception }}</span>
+      </div>
+      {% endif %}
     </div>
-
   </section>
 
 </body>

app/templates/entries.html

@@ -39,6 +39,11 @@ <h1 class="text-5xl lg:text-3xl text-neutral">
                 Return to All Feeds
               </a>
             </li>
+            <li>
+              <a href="{{ url_for('about') }}" class="text-4xl lg:text-xl text-nowrap no-underline">
+                About Precis
+              </a>
+            </li>
           </ul>
         </div>
       </div>

app/templates/feed_config.html

@@ -38,6 +38,11 @@ <h1 class="text-5xl lg:text-3xl text-neutral">
                 Return to All Feeds
               </a>
             </li>
+            <li>
+              <a href="{{ url_for('about') }}" class="text-4xl lg:text-xl text-nowrap no-underline">
+                About Precis
+              </a>
+            </li>
           </ul>
         </div>
       </div>