Added DB auto-migrations feature (#2)

* Bump version: 0.2.1 → 0.3.0 - Added `db_last_updated` column to the schema. All older DBs can be upgraded using the `--migrate-db` flag which will open up the migration menu and the users need to migrate sequentially i.e. 1 → 2 ... since each migration overwrites the column with NULL values - Added `--verbose` flag which can be used to show fic stats during download/update * Added `params` to GET request for `--fetch-urls` * Added 429s error handling and Bot UA - If HTTP 429 error: TooManyRequests is encountered, the CLI will sleep for 30s then retry once more. - Added user-agent with contact mail for users so that they can prevent themselves from getting IP banned since AO3 might ban you if they can't find a way to contact you. * Edited the prompt_user_contact() message * Improve the manual migration * Removed manual db migration * Added auto-migrations feature If any database schema changes are detected, the db is migrated to the new schema.
fichub-cli-contrib · Mar 22, 2022 · af26258 · af26258
1 parent 07c6240
commit af26258
Show file tree

Hide file tree

Showing 13 changed files with 195 additions and 117 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.2.1
+current_version = 0.3.0
 commit = True
 tag = False
 parse = ^

diff --git a/README.md b/README.md
@@ -10,13 +10,13 @@ To report issues for the CLI, open an issue at https://github.com/FicHub/fichub-
 
 # Installation
 
-## Using pip (Recommended)
+## From pip (Stable, recommended)
 
 ```
 pip install -U fichub-cli-metadata
 ```
 
-## From Source (Might have bugs, for testing only)
+## From Github Source (Pre-release, for testing new features by Beta testers)
 
 ```
 pip install git+https://github.com/fichub-cli-contrib/fichub-cli-metadata@main
@@ -36,7 +36,6 @@ Options:
   --update-db               Self-Update existing db (--input-db required)
   --export-db               Export the existing db as json (--input-db
                             required)
-  --migrate-db              Migrate to new db schema (--input-db required)
   -o, --out-dir TEXT        Path to the Output directory (default: Current
                             Directory)
   --download-ebook TEXT     Download the ebook as well. Specify the format:
@@ -101,12 +100,6 @@ fichub_cli metadata --input-db "urls - 2022-01-29 T000558.sqlite" --update-db
 fichub_cli metadata --input-db "urls - 2022-01-29 T000558.sqlite" --export-db
 ```
 
-- To migrate an existing db to the new schema
-
-```
-fichub_cli metadata --input-db "urls - 2022-01-29 T000558.sqlite" --migrate-db
-```
-
 - To download the ebook along with the metadata
 
 ```
@@ -119,6 +112,12 @@ fichub_cli metadata -i urls.txt --download-ebook epub
 fichub_cli metadata --fetch-urls https://archiveofourown.org/users/flamethrower/
 ```
 
+---
+
+Note: If there are any database schema changes, the CLI will automatically migrate the db. A `.pre.migration` sqlite file will be created which would be your original db before any migrations as backup.
+
+---
+
 # Links
 
 - [Fichub-cli](https://github.com/FicHub/fichub-cli/)

diff --git a/fichub_cli_metadata/__init__.py b/fichub_cli_metadata/__init__.py
@@ -13,6 +13,6 @@
 # limitations under the License.
 
 # __version__ at the top to prevent ImportError: ... partially initialized module ...
-__version__ = "0.2.1"
+__version__ = "0.3.0"
 
 from .cli import app  # entry_point
diff --git a/fichub_cli_metadata/cli.py b/fichub_cli_metadata/cli.py
@@ -22,9 +22,8 @@
 from .utils.fetch_data import FetchData
 from fichub_cli.utils.processing import get_format_type
 
-init(autoreset=True)  # colorama init
-timestamp = datetime.now().strftime("%Y-%m-%d T%H%M%S")
 
+init(autoreset=True)  # colorama init
 app = typer.Typer(add_completion=False)
 
 
@@ -44,9 +43,6 @@ def metadata(
     export_db: bool = typer.Option(
         False, "--export-db", help="Export the existing db as json (--input-db required)", is_flag=True),
 
-    migrate_db: bool = typer.Option(
-        False, "--migrate-db", help="Migrate to new db schema (--input-db required)", is_flag=True),
-
     out_dir: str = typer.Option(
         "", "-o", "--out-dir", help="Path to the Output directory (default: Current Directory)"),
 
@@ -56,6 +52,9 @@ def metadata(
     fetch_urls: str = typer.Option(
         "", help="Fetch all story urls found from a page. Currently supports archiveofourown.org only"),
 
+    verbose: bool = typer.Option(
+        False, "-v", "--verbose", help="Show fic stats", is_flag=True),
+
     force: bool = typer.Option(
         False, "--force", help="Force update the metadata", is_flag=True),
 
@@ -84,7 +83,9 @@ def metadata(
 
     Failed downloads will be saved in the `err.log` file in the current directory
     """
+
     if log is True:
+        timestamp = datetime.now().strftime("%Y-%m-%d T%H%M%S")
         logger.remove()  # remove all existing handlers
         logger.add(f"fichub_cli - {timestamp}.log")
         debug = True
@@ -101,27 +102,22 @@ def metadata(
     if input and not update_db:
         fic = FetchData(debug=debug, automated=automated, format_type=format_type,
                         out_dir=out_dir, input_db=input_db, update_db=update_db,
-                        export_db=export_db, force=force)
+                        export_db=export_db, force=force, verbose=verbose)
         fic.save_metadata(input)
 
     if input_db and update_db:
+
         fic = FetchData(debug=debug, automated=automated, format_type=format_type,
                         out_dir=out_dir, input_db=input_db, update_db=update_db,
-                        export_db=export_db, force=force)
+                        export_db=export_db, force=force, verbose=verbose)
         fic.update_metadata()
 
     if export_db:
         fic = FetchData(debug=debug, automated=automated,
                         out_dir=out_dir, input_db=input_db, update_db=update_db,
-                        export_db=export_db, force=force)
+                        export_db=export_db, force=force, verbose=verbose)
         fic.export_db_as_json()
 
-    if input_db and migrate_db:
-        fic = FetchData(debug=debug, automated=automated,
-                        out_dir=out_dir, input_db=input_db, update_db=update_db,
-                        export_db=export_db, force=force)
-        fic.migrate_db()
-
     if fetch_urls:
         fic = FetchData(debug=debug)
         fic.fetch_urls_from_page(fetch_urls)

diff --git a/fichub_cli_metadata/utils/crud.py b/fichub_cli_metadata/utils/crud.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from datetime import datetime
 from tqdm import tqdm
 import sys
 from colorama import Fore
@@ -82,8 +83,9 @@ def update_data(db: Session, item: dict, debug: bool):
                 models.Metadata.favs: favs,
                 models.Metadata.follows: follows,
                 models.Metadata.status: item['status'],
-                models.Metadata.last_updated: item['updated'],
                 models.Metadata.words: item['words'],
+                models.Metadata.fic_last_updated: item['updated'],
+                models.Metadata.db_last_updated: datetime.now().astimezone().strftime('%Y-%m-%dT%H:%M:%S%z'),
                 models.Metadata.source: item['source']
             }
         )
@@ -105,7 +107,9 @@ def dump_json(db: Session, input_db, json_file: str, debug: bool):
     tqdm.write(Fore.GREEN + "Getting all rows from database.")
     try:
         all_rows = get_all_rows(db)
-    except OperationalError:
+    except OperationalError as e:
+        if debug:
+            logger.info(Fore.RED + str(e))
         db_not_found_log(debug, input_db)
         sys.exit()
 
@@ -117,11 +121,68 @@ def get_all_rows(db: Session):
     return db.query(models.Metadata).all()
 
 
-def add_column(db: Session):
+def add_fichub_id_column(db: Session, db_backup, debug: bool):
     """ To add a column AFTER an existing column
     """
-    db.execute("ALTER TABLE fichub_metadata RENAME TO TempOldTable;")
-    db.execute("CREATE TABLE fichub_metadata(id INTEGER NOT NULL, fichub_id VARCHAR(255),title VARCHAR(255), author VARCHAR(255), chapters INTEGER, created VARCHAR(255), description VARCHAR(255), rated VARCHAR(255), language VARCHAR(255), genre VARCHAR(255), characters VARCHAR(255), reviews INTEGER, favs INTEGER, follows INTEGER, status VARCHAR(255), last_updated VARCHAR(255), words INTEGER, source VARCHAR(255), PRIMARY KEY(id))")
-    db.execute("INSERT INTO fichub_metadata (id, title , author , chapters , created , description , rated , language , genre , characters , reviews , favs , follows , status , last_updated , words , source ) SELECT id, title , author , chapters , created , description , rated , language , genre , characters , reviews , favs , follows , status , last_updated , words , source FROM TempOldTable;")
-    db.execute("DROP TABLE TempOldTable;")
-    db.commit()
+
+    drop_TempFichubMetadata(db)
+    col_exists = False
+    try:
+        col_exists = db.execute("SELECT fichub_id from fichub_metadata;")
+        col_exists = True
+    except OperationalError as e:
+        if debug:
+            logger.error(e)
+        pass
+    if not col_exists:
+        tqdm.write(
+            Fore.GREEN + "Database Schema changes detected! Migrating the database.")
+        # backup the db before migrating the data
+        db_backup("pre.migration")
+
+        if debug:
+            logger.info("Migration: adding fichub_id column")
+        tqdm.write(Fore.GREEN + "Migration: adding fichub_id column")
+
+        db.execute("ALTER TABLE fichub_metadata RENAME TO TempFichubMetadata;")
+        db.execute("CREATE TABLE fichub_metadata(id INTEGER NOT NULL, fichub_id VARCHAR(255),title VARCHAR(255), author VARCHAR(255), chapters INTEGER, created VARCHAR(255), description VARCHAR(255), rated VARCHAR(255), language VARCHAR(255), genre VARCHAR(255), characters VARCHAR(255), reviews INTEGER, favs INTEGER, follows INTEGER, status VARCHAR(255), words INTEGER, last_updated VARCHAR(255), source VARCHAR(255), PRIMARY KEY(id))")
+        db.execute("INSERT INTO fichub_metadata (id, title, author, chapters, created, description, rated, language, genre, characters, reviews, favs, follows, status, words, last_updated, source ) SELECT id, title, author, chapters, created, description, rated, language, genre, characters, reviews, favs, follows, status, words,last_updated, source FROM TempFichubMetadata;")
+        db.execute("DROP TABLE TempFichubMetadata;")
+        db.commit()
+
+
+def add_db_last_updated_column(db: Session, db_backup, debug: bool):
+    """ To add a column AFTER an existing column
+    """
+
+    drop_TempFichubMetadata(db)
+    col_exists = False
+    try:
+        db.execute("SELECT db_last_updated from fichub_metadata;")
+        col_exists = True
+    except OperationalError as e:
+        if debug:
+            logger.error(e)
+        pass
+    if not col_exists:
+        tqdm.write(
+            Fore.GREEN + "Database Schema changes detected! Migrating the database.")
+        # backup the db before migrating the data
+        db_backup("pre.migration")
+
+        if debug:
+            logger.info("Migration: adding db_last_updated column")
+        tqdm.write(Fore.GREEN + "Migration: adding db_last_updated column")
+
+        db.execute("ALTER TABLE fichub_metadata RENAME TO TempFichubMetadata;")
+        db.execute("CREATE TABLE fichub_metadata(id INTEGER NOT NULL, fichub_id VARCHAR(255), title VARCHAR(255), author VARCHAR(255), chapters INTEGER, created VARCHAR(255), description VARCHAR(255), rated VARCHAR(255), language VARCHAR(255), genre VARCHAR(255), characters VARCHAR(255), reviews INTEGER, favs INTEGER, follows INTEGER, status VARCHAR(255), words INTEGER, fic_last_updated VARCHAR(255), db_last_updated VARCHAR(255), source VARCHAR(255), PRIMARY KEY(id))")
+        db.execute("INSERT INTO fichub_metadata (id, fichub_id, title, author, chapters, created, description, rated, language, genre, characters, reviews, favs, follows, status,  words, fic_last_updated, source ) SELECT id, fichub_id, title, author, chapters, created, description, rated, language, genre, characters, reviews, favs, follows, status, words, last_updated, source FROM TempFichubMetadata;")
+        db.execute("DROP TABLE TempFichubMetadata;")
+        db.commit()
+
+
+def drop_TempFichubMetadata(db: Session):
+    try:
+        db.execute("DROP TABLE TempFichubMetadata;")
+    except OperationalError:
+        pass