From 30ca172d0e522551d3e9c2dcbf81349d44a4884f Mon Sep 17 00:00:00 2001 From: Egezenn Date: Thu, 19 Dec 2024 03:45:01 +0300 Subject: [PATCH] Decluttered readme, todo updates, removal of unncecessary scripts, help texts for cli settings, fix for update_library_with_manual_changes_on_files files to do what it is supposed to do --- readme.md | 63 ++++++++++----------------------- scripts/linuxpkgRequirements.sh | 1 - scripts/msys2Requirements.sh | 1 - scripts/wingetRequirements.bat | 1 - todo.md | 63 ++++++++++++++++----------------- ytmasc/intermediates.py | 17 ++++----- ytmasc/intermediates_cli.py | 36 +++++++++++++------ 7 files changed, 82 insertions(+), 100 deletions(-) delete mode 100644 scripts/linuxpkgRequirements.sh delete mode 100644 scripts/msys2Requirements.sh delete mode 100644 scripts/wingetRequirements.bat diff --git a/readme.md b/readme.md index 4c9411e..a2bad07 100644 --- a/readme.md +++ b/readme.md @@ -1,6 +1,6 @@ # YTMASC -horrible orange triangle +horrible orange triangle YTMASC(**Y**ou**T**ube **M**usic **A**udio **S**craper & syn**C**hronizer) in a nutshell, aims to get your music library off of YouTube and provide you an offline backup of it along with other maintenance niceties. @@ -18,32 +18,35 @@ It's features are: The project just keeps expanding as I learn more stuff and want to implement niche things. So it's currently in Alpha stages, you may see new features come and go every now and then. -## I Requirements and notes +## CLI Usage Examples -### I.A Windows +`ytmasc` | `ytmasc gui`: launches the deprecated gui -- You need `python3.11` and `ffmpeg` packages. - - You can also run `msys2Requirements.sh` get these from MSYS2. - - You can run `wingetRequirements.bat` if you have winget installed. - - `ffmpeg` from wasn't working for me, just a heads up. +`ytmasc -h`: shows the help -### I.B GNU/Linux +`ytmasc gui`: will run the tkinter gui (it may get broken in the future) -- You need `python3.11`, `ffmpeg` and `python3-tk` packages. - - You can run `linuxpkgRequirements.sh` via the terminal. +`ytmasc run`: runs tasks according to the current configuration -### I.C Required python packages to run from source +`ytmasc set`: will show you the state of the config -- Run `pip install -r requirements.txt` command via the terminal. +`ytmasc set parser run_fetcher 1`: sets the fetcher to run -### I.D Side notes +`ytmasc --export_library_as_csv`: exports the library as csv to the `data` directory +## Requirements to run from source or build + +- You need `~=python3.11` (also `python3-tk` for the GUI on Linux) and `ffmpeg` packages. + +## Side notes + +- You need `ffmpeg` binaries for conversion. - YouTube blocks API requests if you exceed the amount they classify you as a bot (around 200 requests). You can either use a VPN, proxy or just wait to bypass this. See related `yt-dlp` [issue](https://github.com/yt-dlp/yt-dlp/issues/10128). - You might want to change your YouTube language while the fetcher is running as YouTube has rare concatenation of artists which gives you the word "and" in your own language. - While downloading, some changes may occur in YouTube which results in an error. Find out the music via `https://music.youtube.com/watch?v=`, delete the json entry and add the changed one into your likes to continue. At least your music is not lost by YouTube for some unknown reason! :\) -### I.E `fetcher.py` +### `fetcher.py` This part is a little duct taped, I couldn't find a good way to get the `libraryPage` formerly known as `likesPage` so I just emulated user input. It's written for a Windows computer that has `firefox` and `file explorer`. Shouldn't be hard to tinker and get it to work for your configuration. @@ -55,38 +58,10 @@ Change `savePageAsIndexOnRightClick` to which index your save as is on your brow The rest is fine if you don't have a really old computer. -## II CLI Usage Examples - -`python -m ytmasc` - -`-h` - -`gui`: will run the tkinter gui (it may get broken in the future) - -`run`: runs tasks according to the current configuration - -`set`: will show you the state of the config - -`set parser run_fetcher 1`: sets the fetcher to run - -`--export_library_as_csv`: exports the library as csv to the `data` directory - -## III Origin - -Back in highschool I didn't have a data plan for my phone \(because I mostly used it to receive phone calls\), so I just downloaded songs and tagged them to listen on the way. From creating dumb scripts for games using AutoHotkey, I botched together some lines to semi-automize the stuff I do manually. It was horrendous. But managable with the size of my library at that time. - -But as time passed, this library has expanded to 4 digits and so by being lazy and having a better programming knowledge I wanted to download my music library from YouTube *in a more convenient fashion*, but the existing tools can't do this. So, I started by parsing the library page on YouTube. Then formatting and outputting that into `yt-dl` to download the files. And realizing that I could do all this and other tedious maintenance with Python. - -### But why would you still do all this work when you have easy ways to get the music you want? - -Well, in the current era where companies became too restrictive of **how** you listen to your music and **enshittify** your experience in any way possible to justify their gains, this project is a callback to the serene days of maintaining your own library which is detached from any sort of service. - -Music should not be paywalled. This era of short-form, disposable media demolished the sense of attachment to the artists. Music is part of our culture and it's our responsibility as an individual to shoutout what we like. - -## IV Credits +## Credits I'd like to personally thank all the people that have developed\/maintained\/poured their hearts into the packages\/tools this *thing* is using and the artists that pushed me to keep a copy of their music in my library. -## V Disclaimer +## Disclaimer This project is not in any way, shape or form affiliated with YouTube, Google or any of their subsidiaries and affiliates. diff --git a/scripts/linuxpkgRequirements.sh b/scripts/linuxpkgRequirements.sh deleted file mode 100644 index 0d153c5..0000000 --- a/scripts/linuxpkgRequirements.sh +++ /dev/null @@ -1 +0,0 @@ -apt install python3.11 ffmpeg python3-tk \ No newline at end of file diff --git a/scripts/msys2Requirements.sh b/scripts/msys2Requirements.sh deleted file mode 100644 index b09e512..0000000 --- a/scripts/msys2Requirements.sh +++ /dev/null @@ -1 +0,0 @@ -pacman -S ucrt64/mingw-w64-ucrt-x86_64-ffmpeg ucrt64/mingw-w64-x86_64-python \ No newline at end of file diff --git a/scripts/wingetRequirements.bat b/scripts/wingetRequirements.bat deleted file mode 100644 index 0603fa5..0000000 --- a/scripts/wingetRequirements.bat +++ /dev/null @@ -1 +0,0 @@ -winget install python3.11 ffmpeg \ No newline at end of file diff --git a/todo.md b/todo.md index 6d073d8..c209129 100644 --- a/todo.md +++ b/todo.md @@ -1,35 +1,31 @@ # TODO - [TODO](#todo) - - [I Working on](#i-working-on) - - [II Highest Importance](#ii-highest-importance) - - [III Requires Research](#iii-requires-research) - - [IV Lowest Importance](#iv-lowest-importance) - - [V Doubts and Head Scratchers](#v-doubts-and-head-scratchers) - - [VI TK Gui Deprecation](#vi-tk-gui-deprecation) + - [I Highest Importance](#i-highest-importance) + - [II Requires Research](#ii-requires-research) + - [III Lowest Importance](#iii-lowest-importance) + - [IV Doubts and Head Scratchers](#iv-doubts-and-head-scratchers) - [Latest Implementations \& Fixes](#latest-implementations--fixes) -## I Working on +## I Highest Importance -## II Highest Importance +- [ ] CSV "import" currently overwrites the file :facepalm: + - [ ] Convert the provided CSV to JSON. check if key exists, if not add it yada yada + - [ ] Move it to intermediates - [ ] Make an executable for Linux aswell + - [x] Need the user to download ffmpeg binaries as `ffmpeg-python` is just a wrapper, should mention that in the readme - [ ] Addition of `database_tools` - [x] Write a utility script to check if there are missing pairs - [x] `matcher.py`, helper script to make the switch from your old archive. - - [ ] Script to migrate to youtubes metadata using previously created metadata. + - [ ] Script to migrate to youtubes metadata using previously created user metadata. + - [ ] Use a headless browser to get links under Songs if it doesn't exist as a Song fallback to Video + - `https://music.youtube.com/search?q=example+song` - [ ] Script to remove duplicate entries that only differ with the key. -- [ ] Manual editing logging - - [ ] Use metadata from user data if key exists - - [ ] Use fallback keys provided by user - - [ ] Make use of database_tools' finder - -- [x] Record any manual changes to songs \[did do this, but it's instead checking if json has been manually edited\] - - [ ] Check the downloads directory if any of the song tags has been manually changed - - If it is changed, update tags on `library.json` - - Else skip +- [ ] Provide the user with a fallback json that consists of replacement keys \(for the ID's that fail or preference\) + - [ ] Use the dbtools utility to be written to prompt the user for the replacement key - [x] Log unavailable videos into a file. - [ ] Could be better @@ -40,7 +36,7 @@ - Piped - [Invidious](https://github.com/grqz/yt-dlp-invidious) -## III Requires Research +## II Requires Research - [ ] Find out how to launch gui without a console @@ -57,11 +53,14 @@ - [ ] InnerTube -## IV Lowest Importance +## III Lowest Importance + +- [ ] Make settings interactable (e.g import the file named x) - [ ] Log bad thumbnails - [ ] `hqdefault` - [ ] Ones that are available as `hqdefault` but aren't a square (`720x720`) + - OpenCV hsl similiarity? the fills in those files aren't just 1 color last I tried & checked - [ ] Remove quirky file finding method for the `temp` directory in `download` function as it's not required anymore? @@ -82,42 +81,38 @@ - [ ] Add a function to delete entries in `library.json` that aren't in `downloads` -- [ ] Give some use to CSV <-> JSON conversion functions - - [ ] Fetch metadata from YouTube instead of using the existing data -- [ ] remove exceptions/cases/if statements that shouldn't naturally occur - - [ ] Implement `yt-dlp --get-filename -o "%(uploader)s hsfOqb5r8m0VbV31 %(title)s" VIDEO_URL` for null artists, which is caused by ViMusic's search function returning empty - [ ] Write `PermissionError` exceptions - [ ] Some languages have characters that are indexed inside the latin alphabet and by default I think Python just looks for its unicode index, so fixing this -- [ ] Rename `linuxpkgRequirements.sh` to `wingetRequirements.sh` and use the available package manager to install the packages. - - `pacman(Arch) -S`, `dnf(Red Hat/Fedora) install`, `apt(Debian/Ubuntu) install`, `zypper(SLES/openSUSE) install`, `emerge(Gentoo)` - -## V Doubts and Head Scratchers +## IV Doubts and Head Scratchers - [ ] Find something that doesn't disrupt the user's workflow while getting the `likesPage` - cookie injection with selenium? - undetected chromedriver? - user script? + - maybe google provides user data? (lame) - RiMusic database adds all artists text whereas parser just gets the first one - look into artists text and separate the text if it has a comma or an ampersand - while this is pretty easy to do it might produce false artist names - - see if `yt-dlp` can provide any information about this + - see if `yt-dlp` or InnerTune API can provide any information about this - leave it, the parsed data will never be perfect and require some manual changes for it to be pristine anyway -## VI TK Gui Deprecation - -Current TK GUI will be replaced with something like ImGui. - ## Latest Implementations & Fixes NOTE: These will be removed after a while. +- [x] Record any manual changes to songs ~~\[did do this, but it's instead checking if json has been manually edited\]~~ + - why did i do it for the json? probably i hadn't done the export import at that time and was doing something wacky + - [x] Check the downloads directory if any of the song tags has been manually changed + - If it is changed, update tags on `library.json` + - Else skip + - [x] Break when there is no internet connection in download loop - ERROR: [youtube] Fj7SH-YfH6A: Failed to extract any player response; please report this issue on , filling out the appropriate issue template. Confirm you are on the latest version using yt-dlp -U @@ -134,3 +129,5 @@ NOTE: These will be removed after a while. - [x] Ignore user config, copy the `defaultConfig.yaml` if there isn't a config file yet. - [x] Write intermediate functions, i.e functions that are chained together in `tkgui.py` right now to use them both in CLI&GUI + +- [x] Give some use to CSV <-> JSON conversion functions diff --git a/ytmasc/intermediates.py b/ytmasc/intermediates.py index c7a4d1b..215b97c 100644 --- a/ytmasc/intermediates.py +++ b/ytmasc/intermediates.py @@ -29,7 +29,6 @@ def delete_library_page_files(fetcher_is_going_to_run: bool): - try: remove(library_page_path) rmtree(f"{library_page_path[:-4]}_files") @@ -68,29 +67,27 @@ def find_newest_ri_music_export(): def update_library_with_manual_changes_on_files(): - existing_data = read_json(library_data_path) + modified_data = existing_data for key, value in existing_data.items(): song = loadmp3(path.join(download_path, key + audio_conversion_ext)) - if not ( - value["title"] == song.tag.title and value["artist"] == song.tag.artist - ): + if not (value["title"] == song.tag.title or value["artist"] == song.tag.artist): logger.info( f"Manual change detected on {key}, updating {library_data} with changes:\n" f"artist:\t{song.tag.artist} -> {value['artist']}\n" f"title:\t{song.tag.title} -> {value['title']}\n", ) - song.tag.artist = value["artist"] - song.tag.title = value["title"] - song.tag.save() + modified_data[key] = { + "artist": song.tag.artist, + "title": song.tag.title, + } - json = sort_dictionary_based_on_value_inside_nested_dictionary(existing_data) + json = sort_dictionary_based_on_value_inside_nested_dictionary(modified_data) write_json(library_data_path, json) def run_tasks(download: bool, convert: bool, tag: bool): - if not path.exists(library_data_path) or not path.getsize(library_data_path) > 0: logger.error( f"[FileNotFoundError] {library_data} doesn't exist or is empty. Build {library_data} by running a parse." diff --git a/ytmasc/intermediates_cli.py b/ytmasc/intermediates_cli.py index 8bdb9d4..d35ed88 100644 --- a/ytmasc/intermediates_cli.py +++ b/ytmasc/intermediates_cli.py @@ -42,15 +42,34 @@ def get_cli_args(): ) parser.add_argument( - "--update_library_with_manual_changes_on_files", action="store_true", help="" + "--update_library_with_manual_changes_on_files", + action="store_true", + help="Updates library with tag changes you've made to the files", ) - parser.add_argument("--export_library_as_csv", action="store_true", help="") - parser.add_argument("--import_csv_to_library", action="store_true", help="") - parser.add_argument("--update_tags", action="store_true", help="") - parser.add_argument("--db_compare", action="store_true", help="") - parser.add_argument("--db_find_unpaired", action="store_true", help="") parser.add_argument( - "-v", "--verbosity", default="w", action="store", help="d|i|w|e|c" + "--export_library_as_csv", + action="store_true", + help="Exports the library as a CSV file", + ) + parser.add_argument( + "--import_csv_to_library", + action="store_true", + help="Imports a CSV of 3 columns [ID, artist, title]", + ) + parser.add_argument( + "--db_compare", + action="store_true", + help="Helps you migrate your old library by checking if any of them exist in your current library to avoid duplication", + ) + parser.add_argument( + "--db_find_unpaired", action="store_true", help="Find unpaired items" + ) + parser.add_argument( + "-v", + "--verbosity", + default="w", + action="store", + help="Set the log verbosity d | i | w | e | c", ) return parser.parse_args() @@ -86,9 +105,6 @@ def handle_cli(args: classmethod): convert_csv_to_json(csv_library_data_path, library_data_path) # should work properly now with fillna() - if args.update_tags: - pass - if args.db_compare: compare()