Multiple ebook formats & ebook_name config (#20)

* Bump version: 0.8.1 → 0.8.2

Added the feature to download multiple ebook formats simultaneously

* fix: Plugins not being added

* fix: URL encoding issue

* fix: File hash issue

Refactored the code in `fichub.py` to save the hash & download_url
for the fic correctly

* chore: Added `meta` props to the files dictionary

* Bump version: 0.8.2 → 0.8.3

Added `filename_format` feature where users can configure the file
name for the book as per the props mentioned in the README

* refactor: Exception Handler

Changed the Exception Handler from catching only Attribute Errors
to all Errors

* fix: Remove extraMeta prop from the filename_format options

* fix: Rename meta key's id to ichub_id to avoid conflict

* fix: Hardcoded filename_formats to ensure backwards compatibility with future API updates

* fix: Remove unsafe chars from the filename

* Bump version: 0.8.3 → 0.8.4

Added priority based processing for the rawExtendedMeta & extraMeta
where it will process it in this order:
rawExtendedMeta >> extraMeta >> None

* refactor: extraMeta processing

* Bump version: 0.8.4 → 0.9.0

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.8.1
+current_version = 0.9.0
 commit = True
 tag = False
 parse = ^

.gitignore

@@ -5,6 +5,8 @@ __pycache__/
 
 # ebooks
 *.epub
+*.mobi
+*.pdf
 
 # C extensions
 *.so

README.md

@@ -45,8 +45,7 @@ Options:
   -v, --verbose           Show fic stats
   -o,  --out-dir TEXT     Path to the Output directory for files (default:
                           Current Directory)
-  --format TEXT           Download Format: epub (default), mobi, pdf or html
-                          [default: epub]
+  --format TEXT           Download Formats, comma separated if multiple: epub (default), mobi, pdf or html
   --force                 Force overwrite of an existing file
   -ss, --supported-sites  List of supported sites
   -d,  --debug            Show the log in the console for debugging
@@ -60,7 +59,7 @@ Options:
 
 # Default Configuration
 
-- The fanfiction will be downloaded in epub format. To change it, use `-f` followed by the format.
+- The fanfiction will be downloaded in epub format. To change it, use `--format` followed by the format. Multiple formats can be selected by separating them by commas.
 - The fanfiction will be downloaded in the current directory. To change it, use `-o` followed by the path to the directory.
 - Failed downloads will be saved in the `err.log` file in the current directory.
 
@@ -86,6 +85,12 @@ fichub_cli -i urls.txt
 fichub_cli -l "https://www.fanfiction.net/s/11191235/1/Harry-Potter-and-the-Prince-of-Slytherin,https://www.fanfiction.net/s/13720575/1/A-Cadmean-Victory-Remastered"
 ```
 
+- To download multiple formats
+
+```
+fichub_cli -u "https://www.fanfiction.net/s/13720575/1/A-Cadmean-Victory-Remastered" --format epub,mobi
+```
+
 - To generate a changelog of the download
 
 ```
@@ -106,7 +111,17 @@ fichub_cli -i urls.txt --changelog
 
 # Configuration
 
-- Users can configure centain things like `db_up_time_format`, `fic_up_time_format` & `delete_output_log` etc by editing the `config.json` file in the app directory.
+- Users can configure centain things like `db_up_time_format`, `fic_up_time_format`, `delete_output_log` & `filename_format` by editing the `config.json` file in the app directory.
+
+- Filename format props (case-sensitive):
+  `author, fichubAuthorId, authorId, chapters, created, fichubId, genres, id, language, rated, fandom, status, updated, title`
+
+  Example:
+
+  ```
+  "filename_format": "[title] by [author]"
+  ```
+
 - To locate the config file, run `fichub_cli --config-info` and open the `config.json` file in an editor and make the necessary changes.
 
 ## Notes

fichub_cli/__init__.py

@@ -1 +1 @@
-__version__ = "0.8.1"
+__version__ = "0.9.0"

fichub_cli/cli.py

@@ -34,16 +34,15 @@
 
 app = typer.Typer(add_completion=False)
 app_dirs = PlatformDirs("fichub_cli", "fichub")
-
 discovered_plugins = {
     name: importlib.import_module(name)
     for finder, name, ispkg
     in pkgutil.iter_modules()
     if name.startswith('fichub_cli_')
 }
-
 for plugin in discovered_plugins.values():
-    app.add_typer(plugin.app)
+    if not plugin.__name__.endswith("-script"):
+        app.add_typer(plugin.app)
 
 # build/update the app directory & the config file
 appdir_builder(app_dirs)
@@ -72,7 +71,7 @@ def default(
         "", "-o", " --out-dir", help="Path to the Output directory for files (default: Current Directory)"),
 
     format: str = typer.Option(
-        "epub", help="Download Format: epub (default), mobi, pdf or html"),
+        "epub", help="Download Formats, comma separated if multiple: epub (default), mobi, pdf or html"),
 
     force: bool = typer.Option(
         False, "--force", help="Force overwrite of an existing file", is_flag=True),
@@ -191,9 +190,7 @@ def default(
         if fic.exit_status == 1:
             typer.echo(
                 Fore.RED +
-                "\nThe CLI ran into some errors! Check " + Style.RESET_ALL +
-                Fore.YELLOW + "err.log" + Style.RESET_ALL + Fore.RED +
-                " in the current directory!" + Style.RESET_ALL)
+                "\nThe CLI ran into some errors! Check the console for the log messages!" + Style.RESET_ALL)
 
         if os.path.exists("output.log"):
             with open(os.path.join(app_dirs.user_data_dir, "config.json"), 'r') as f:

fichub_cli/utils/fetch_data.py

@@ -16,18 +16,19 @@
 from tqdm import tqdm
 from colorama import Fore
 from loguru import logger
+import traceback
 
 from .fichub import FicHub
 from .logging import init_log, download_processing_log, \
     verbose_log
 from .processing import check_url, save_data, \
-    urls_preprocessing, check_output_log, build_changelog
+    urls_preprocessing, build_changelog
 
 bar_format = "{l_bar}{bar}| {n_fmt}/{total_fmt}, {rate_fmt}{postfix}, ETA: {remaining}"
 
 
 class FetchData:
-    def __init__(self, format_type=0, out_dir="", force=False,
+    def __init__(self, format_type=[0], out_dir="", force=False,
                  debug=False, changelog=False, automated=False, verbose=False):
         self.format_type = format_type
         self.out_dir = out_dir
@@ -84,14 +85,14 @@ def get_fic_with_infile(self, infile: str):
                                 # update the exit status
                                 self.exit_status = fic.exit_status
 
-                                if fic.file_name is None:
+                                if not fic.files:
                                     self.exit_status = 1
 
                                 else:
                                     self.exit_status, url_exit_status = save_data(
-                                        self.out_dir, fic.file_name,
-                                        fic.download_url, self.debug, self.force,
-                                        fic.cache_hash, self.exit_status,
+                                        self.out_dir, fic.files,
+                                        self.debug, self.force,
+                                         self.exit_status,
                                         self.automated)
 
                                     with open("output.log", "a") as file:
@@ -103,9 +104,11 @@ def get_fic_with_infile(self, infile: str):
                                         no_updates_urls.append(url)
                                 pbar.update(1)
 
-                            # Error: 'FicHub' object has no attribute 'file_name'
+                            # Error: 'FicHub' object has no attribute 'files'
                             # Reason: Unsupported URL
-                            except AttributeError:
+                            except Exception as e:
+                                if self.debug:
+                                    logger.error(str(traceback.format_exc()))
                                 with open("err.log", "a") as file:
                                     file.write(url.strip()+"\n")
                                 err_urls.append(url)
@@ -165,14 +168,14 @@ def get_fic_with_list(self, list_url: str):
                                 # update the exit status
                                 self.exit_status = fic.exit_status
 
-                                if fic.file_name is None:
+                                if not fic.files:
                                     self.exit_status = 1
 
                                 else:
                                     self.exit_status, url_exit_status = save_data(
-                                        self.out_dir, fic.file_name,
-                                        fic.download_url, self.debug, self.force,
-                                        fic.cache_hash, self.exit_status, self.automated)
+                                        self.out_dir, fic.files,
+                                        self.debug, self.force,
+                                         self.exit_status, self.automated)
 
                                     with open("output.log", "a") as file:
                                         file.write(f"{url}\n")
@@ -184,9 +187,11 @@ def get_fic_with_list(self, list_url: str):
 
                                 pbar.update(1)
 
-                            # Error: 'FicHub' object has no attribute 'file_name'
+                            # Error: 'FicHub' object has no attribute 'files'
                             # Reason: Unsupported URL
-                            except AttributeError:
+                            except Exception as e:
+                                if self.debug:
+                                    logger.error(str(traceback.format_exc()))
                                 with open("err.log", "a") as file:
                                     file.write(url.strip()+"\n")
                                 err_urls.append(url)
@@ -217,54 +222,56 @@ def get_fic_with_url(self, url_input: str):
         if self.debug:
             logger.info("-u flag used!")
 
-        url = check_output_log([url_input], self.debug)
-
+        url, _ = urls_preprocessing([url_input], self.debug)
         if url:
-            init_log(self.debug, self.force)
-            with tqdm(total=1, ascii=False,
-                      unit="file", bar_format=bar_format) as pbar:
+            if url[0]:
+                init_log(self.debug, self.force)
+                with tqdm(total=1, ascii=False,
+                        unit="file", bar_format=bar_format) as pbar:
 
-                download_processing_log(self.debug, url[0])
-                supported_url, self.exit_status = check_url(
-                    url[0], self.debug, self.exit_status)
+                    download_processing_log(self.debug, url[0])
+                    supported_url, self.exit_status = check_url(
+                        url[0], self.debug, self.exit_status)
 
-                if supported_url:
-                    try:
-                        fic = FicHub(self.debug, self.automated,
-                                     self.exit_status)
-                        fic.get_fic_metadata(url[0], self.format_type)
+                    if supported_url:
+                        try:
+                            fic = FicHub(self.debug, self.automated,
+                                        self.exit_status)
+                            fic.get_fic_metadata(url[0], self.format_type)
 
-                        if self.verbose:
-                            verbose_log(self.debug, fic)
+                            if self.verbose:
+                                verbose_log(self.debug, fic)
 
-                        # update the exit status
-                        self.exit_status = fic.exit_status
+                            # update the exit status
+                            self.exit_status = fic.exit_status
 
-                        if fic.file_name is None:
-                            self.exit_status = 1
+                            if not fic.files:
+                                self.exit_status = 1
 
-                        else:
-                            self.exit_status, _ = save_data(
-                                self.out_dir, fic.file_name,
-                                fic.download_url, self.debug, self.force,
-                                fic.cache_hash, self.exit_status, self.automated)
-                            with open("output.log", "a") as file:
-                                file.write(f"{url[0]}\n")
-                        pbar.update(1)
+                            else:
+                                self.exit_status, _ = save_data(
+                                    self.out_dir, fic.files,
+                                    self.debug, self.force,
+                                    self.exit_status, self.automated)
+                                with open("output.log", "a") as file:
+                                    file.write(f"{url[0]}\n")
+                            pbar.update(1)
 
-                    # Error: 'FicHub' object has no attribute 'file_name'
-                    # Reason: Unsupported URL
-                    except AttributeError:
+                        # Error: 'FicHub' object has no attribute 'files'
+                        # Reason: Unsupported URL
+                        except Exception:
+                            if self.debug:
+                                logger.error(str(traceback.format_exc()))
+                            with open("err.log", "a") as file:
+                                file.write(url[0].strip()+"\n")
+                            pbar.update(1)
+                            self.exit_status = 1
+                            pass  # skip the unsupported url
+
+                    else:  # skip the unsupported url
                         with open("err.log", "a") as file:
                             file.write(url[0].strip()+"\n")
                         pbar.update(1)
-                        self.exit_status = 1
-                        pass  # skip the unsupported url
-
-                else:  # skip the unsupported url
-                    with open("err.log", "a") as file:
-                        file.write(url[0].strip()+"\n")
-                    pbar.update(1)
-        else:
-            typer.echo(Fore.RED +
-                       "No new urls found! If output.log exists, please clear it.")
+            else:
+                typer.echo(Fore.RED +
+                        "No new urls found! If output.log exists, please clear it.")

fichub_cli/utils/fichub.py

@@ -44,7 +44,7 @@ def __init__(self, debug, automated, exit_status):
         self.http.mount("https://", adapter)
         self.http.mount("http://", adapter)
 
-    def get_fic_metadata(self, url: str, format_type: int):
+    def get_fic_metadata(self, url: str, format_type: list):
         """
         Sends GET request to Fichub API to fetch the metadata
         """
@@ -70,44 +70,47 @@ def get_fic_metadata(self, url: str, format_type: int):
                 break
             except (ConnectionError, TimeoutError, Exception) as e:
                 if self.debug:
-                    logger.error(str(e))
+                    logger.error(str(traceback.format_exc()))
                 tqdm.write("\n" + Fore.RED + str(e) + Style.RESET_ALL +
                            Fore.GREEN + "\nWill retry in 3s!" +
                            Style.RESET_ALL)
                 time.sleep(3)
 
         try:
             self.response = response.json()
-
-            if format_type == 0:
-                cache_url = self.response['epub_url']
-                self.cache_hash = (
-                    re.search(r"\?h=(.*)", self.response['epub_url'])).group(1)
-                self.file_format = ".epub"
-
-            elif format_type == 1:
-                cache_url = self.response['mobi_url']
-                self.cache_hash = (
-                    re.search(r"\?h=(.*)", self.response['epub_url'])).group(1)
-                self.file_format = ".mobi"
-
-            elif format_type == 2:
-                cache_url = self.response['pdf_url']
-                self.cache_hash = (
-                    re.search(r"\?h=(.*)", self.response['epub_url'])).group(1)
-                self.file_format = ".pdf"
-
-            elif format_type == 3:
-                cache_url = self.response['html_url']
-                self.cache_hash = (
-                    re.search(r"\?h=(.*)", self.response['epub_url'])).group(1)
-                self.file_format = ".zip"
-
-            self.file_name = self.response['epub_url'].split(
-                "/")[4].split("?")[0]
-            self.file_name = self.file_name.replace(".epub", self.file_format)
-            self.download_url = "https://fichub.net"+cache_url
-
+            self.file_format =[]
+            self.cache_hash = {}
+            cache_urls= {}
+
+            for format in format_type:
+                if format == 0:
+                    cache_urls['epub'] = self.response['urls']['epub']
+                    self.cache_hash['epub'] = self.response['hashes']['epub']
+                    self.file_format.append(".epub")
+
+                elif format == 1:
+                    cache_urls['mobi'] = self.response['urls']['mobi']
+                    self.cache_hash['mobi'] = self.response['hashes']['epub']
+                    self.file_format.append(".mobi")
+
+                elif format == 2:
+                    cache_urls['pdf'] = self.response['urls']['pdf']
+                    self.cache_hash['pdf'] = self.response['hashes']['epub']
+                    self.file_format.append(".pdf")
+
+                elif format == 3:
+                    cache_urls['zip'] =self.response['urls']['html']
+                    self.cache_hash['zip'] = self.response['hashes']['epub']
+                    self.file_format.append(".zip")
+
+            self.files = {}
+            self.files["meta"] = self.response['meta']
+            for file_format in self.file_format:
+                self.files[self.response['urls']['epub'].split(
+                "/")[4].split("?")[0].replace(".epub", file_format)] = {
+                "hash": self.cache_hash[file_format.replace(".","")],
+                "download_url": "https://fichub.net"+cache_urls[file_format.replace(".","")]
+                }
         # Error: 'epub_url'
         # Reason: Unsupported URL
         except (KeyError, UnboundLocalError) as e:
@@ -144,7 +147,7 @@ def get_fic_data(self, download_url: str):
                 break
             except (ConnectionError, TimeoutError, Exception) as e:
                 if self.debug:
-                    logger.error(str(e))
+                    logger.error(str(traceback.format_exc()))
                 tqdm.write("\n" + Fore.RED + str(e) + Style.RESET_ALL +
                            Fore.GREEN + "\nWill retry in 3s!" +
                            Style.RESET_ALL)