diff --git a/README.md b/README.md index 33f477b..785175b 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,12 @@ or by ID twitch-dl download 1418494769 ``` +or all videos from a channel + +``` +twitch-dl download bananasaurus_rex --all +``` + Download a clip by URL ``` diff --git a/docs/advanced.md b/docs/advanced.md index 8277537..0cb3fff 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -9,3 +9,9 @@ the `TMP` environment variable, e.g. ``` TMP=/my/tmp/path/ twitch-dl download 221837124 ``` + +You can also specify the `--tempdir` argument to the `download` command without having to modify your environment variables. For example: + +``` +twitch-dl download 221837124 --tempdir /my/tmp/path/ +``` \ No newline at end of file diff --git a/docs/commands/clips.md b/docs/commands/clips.md index 3dc8b95..5c8baf0 100644 --- a/docs/commands/clips.md +++ b/docs/commands/clips.md @@ -33,11 +33,6 @@ twitch-dl clips [FLAGS] [OPTIONS] -j, --json Show results as JSON. Ignores --pager. - - - -d, --download - Download all videos in given period (in source quality) - diff --git a/docs/commands/download.md b/docs/commands/download.md index adba295..f1baad5 100644 --- a/docs/commands/download.md +++ b/docs/commands/download.md @@ -34,10 +34,25 @@ twitch-dl download [FLAGS] [OPTIONS] Don't run ffmpeg to join the downloaded vods, implies --keep. + + --skipall + Skip the current file if it already exists without prompting. + + --overwrite Overwrite the target file if it already exists without prompting. + + + -x, --all + Download all videos on the channel. Overrides all other arguments. Pass in the channel name as the 'videos' argument. + + + + -y, --skip-latest + Skip downloading the latest video. Only makes sense with the --all flag. + @@ -67,7 +82,7 @@ twitch-dl download [FLAGS] [OPTIONS] -q, --quality - Video quality, e.g. 720p. Set to 'source' to get best quality. + Video quality, e.g. 720p30 or 720p60. Set to 'source' to get best quality. @@ -89,6 +104,26 @@ twitch-dl download [FLAGS] [OPTIONS] -c, --chapter Download a single chapter of the video. Specify the chapter number or use the flag without a number to display a chapter select prompt. + + + -t, --tempdir + Override the temp dir path. + + + + -d, --output-dir + Customize location of the output directory. Defaults to the current directory. + + + + -u, --execute-after + Run a CLI command after each file is downloaded and processed. In your command, use ^p for the absolute path to the file that was downloaded, and ^f for just the file name. + + + + -z, --execute-before + Run a CLI command before each file is downloaded. Return an exit code of 0 to indicate you want to download the file, or nonzero to indicate you want to skip the file. In your command, use ^p for the absolute path to the file that was downloaded, and ^f for just the file name. + diff --git a/examples/s3_search.sh b/examples/s3_search.sh new file mode 100755 index 0000000..b6388b3 --- /dev/null +++ b/examples/s3_search.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Check if AWS CLI is installed +if ! command -v aws &> /dev/null +then + echo "AWS CLI not installed. Please install and configure it." + exit 2 +fi + +# Check for correct number of arguments +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit 2 +fi + +BUCKET=$1 +FILE=$2 + +# Check if file exists in the S3 bucket +if aws s3 ls "s3://$BUCKET/$FILE" > /dev/null; then + echo "File $FILE exists in bucket $BUCKET." + exit 1 +else + echo "File $FILE does not exist in bucket $BUCKET." + exit 0 +fi diff --git a/examples/s3_upload_nuke.sh b/examples/s3_upload_nuke.sh new file mode 100755 index 0000000..8933d73 --- /dev/null +++ b/examples/s3_upload_nuke.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Check if AWS CLI is installed +if ! command -v aws &> /dev/null +then + echo "AWS CLI not installed. Please install and configure it." + exit 2 +fi + +# Check for correct number of arguments +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit 2 +fi + +BUCKET=$1 +FILE=$2 + +# Upload the file to S3, then delete the local copy +set -e +aws s3 cp "${FILE}" "s3://$BUCKET/" --storage-class GLACIER +rm -f "${FILE}" + +exit 0 \ No newline at end of file diff --git a/twitchdl/commands/download.py b/twitchdl/commands/download.py index 192b053..8f15352 100644 --- a/twitchdl/commands/download.py +++ b/twitchdl/commands/download.py @@ -1,4 +1,5 @@ import asyncio +import sys import httpx import m3u8 import os @@ -6,6 +7,7 @@ import shutil import subprocess import tempfile +import shlex from os import path from pathlib import Path @@ -16,7 +18,18 @@ from twitchdl.download import download_file from twitchdl.exceptions import ConsoleError from twitchdl.http import download_all -from twitchdl.output import print_out +from twitchdl.output import print_err, print_out + +def _execute_download_command(file_path: str, command_template: str) -> bool: + print_out(f"file_path: {file_path}") + file_name = os.path.basename(file_path) + qfp = shlex.quote(str(file_path)) + qfn = shlex.quote(file_name) + command = command_template.replace("^p", qfp).replace("^f", qfn) + + compl = subprocess.run(command, shell=True, check=False) + print_out(f"Executed command: {command}") + return compl.returncode == 0 def _parse_playlists(playlists_m3u8): @@ -101,7 +114,8 @@ def _video_target_filename(video, args): } try: - return args.output.format(**subs) + target = args.output.format(**subs) + return Path(args.output_dir, target) if args.output_dir else target except KeyError as e: supported = ", ".join(subs.keys()) raise ConsoleError("Invalid key {} used in --output. Supported keys are: {}".format(e, supported)) @@ -131,7 +145,8 @@ def _clip_target_filename(clip, args): } try: - return args.output.format(**subs) + target = args.output.format(**subs) + return Path(args.output_dir, target) if args.output_dir else target except KeyError as e: supported = ", ".join(subs.keys()) raise ConsoleError("Invalid key {} used in --output. Supported keys are: {}".format(e, supported)) @@ -157,17 +172,36 @@ def _get_vod_paths(playlist, start: Optional[int], end: Optional[int]) -> List[s return files -def _crete_temp_dir(base_uri: str) -> str: +def _crete_temp_dir(base_uri: str, args) -> str: """Create a temp dir to store downloads if it doesn't exist.""" path = urlparse(base_uri).path.lstrip("/") - temp_dir = Path(tempfile.gettempdir(), "twitch-dl", path) + temp_dir = Path(args.tempdir if args.tempdir else tempfile.gettempdir(), "twitch-dl", path) temp_dir.mkdir(parents=True, exist_ok=True) return str(temp_dir) +def _download_all_videos(channel_name, args): + print_out(f"Fetching all videos for channel: {channel_name}...") + total_count, video_generator = twitch.channel_videos_generator(channel_name, sys.maxsize, 'time', 'archive') + print_out(f"Found {total_count} videos to download...") + + if args.skip_latest: + next(video_generator) # Skip the latest video + + for video in video_generator: + #Skip execution if the pre-execute command returns non-zero status + target_filename = _video_target_filename(video, args) + if not args.execute_before or (args.execute_before and _execute_download_command(target_filename, args.execute_before)): + _download_video(video['id'], args) + else: + print_out(f"Skipping video due to pre-execute returning nonzero: {video['id']}...") def download(args): - for video_id in args.videos: - download_one(video_id, args) + if args.all: + # Assuming the channel name is passed in args.videos[0] + _download_all_videos(args.videos[0], args) + else: + for video_id in args.videos: + download_one(video_id, args) def download_one(video: str, args): @@ -245,11 +279,30 @@ def _download_clip(slug: str, args) -> None: target = _clip_target_filename(clip, args) print_out("Target: {}".format(target)) - if not args.overwrite and path.exists(target): - response = input("File exists. Overwrite? [Y/n]: ") - if response.lower().strip() not in ["", "y"]: - raise ConsoleError("Aborted") - args.overwrite = True + if path.exists(target): + if args.skipall: + print("Target file exists. Skipping.") + return + if not args.overwrite: + while True: + response = input("File exists. Overwrite? [ \033[4mY\033[0mes, \033[4ma\033[0mlways yes, \033[4ms\033[0mkip, always s\033[4mk\033[0mip, a\033[4mb\033[0mort ]: ") + match response.lower().strip(): + case "y": + break # Just continue + case "a": + args.overwrite = True + break + case "s": + print("Skipping.") + return + case "k": + print("Skipping.") + args.skipall = True + return + case "b": + raise ConsoleError("Aborted") + case _: + print("Invalid input.") url = get_clip_authenticated_url(slug, args.quality) print_out("Selected URL: {}".format(url)) @@ -259,6 +312,9 @@ def _download_clip(slug: str, args) -> None: print_out("Downloaded: {}".format(target)) + if args.execute_after: + _execute_download_command(target, args.execute_after) + def _download_video(video_id, args) -> None: if args.start and args.end and args.end <= args.start: @@ -276,11 +332,28 @@ def _download_video(video_id, args) -> None: target = _video_target_filename(video, args) print_out("Output: {}".format(target)) - if not args.overwrite and path.exists(target): - response = input("File exists. Overwrite? [Y/n]: ") - if response.lower().strip() not in ["", "y"]: - raise ConsoleError("Aborted") - args.overwrite = True + if path.exists(target): + if args.skipall: + print("Target file exists. Skipping.") + return + if not args.overwrite: + while True: + response = input("File exists. Overwrite? [ \033[4mY\033[0mes, \033[4ma\033[0mlways yes, \033[4ms\033[0mkip, always s\033[4mk\033[0mip, a\033[4mb\033[0mort ]: ") + match response.lower().strip(): + case "y": + break # Just continue + case "a": + args.overwrite = True + break + case "s": + return + case "k": + args.skipall = True + return + case "b": + raise ConsoleError("Aborted") + case _: + print("Invalid input.") # Chapter select or manual offset start, end = _determine_time_range(video_id, args) @@ -300,7 +373,7 @@ def _download_video(video_id, args) -> None: playlist = m3u8.loads(response.text) base_uri = re.sub("/[^/]+$", "/", playlist_uri) - target_dir = _crete_temp_dir(base_uri) + target_dir = _crete_temp_dir(base_uri, args) vod_paths = _get_vod_paths(playlist, start, end) # Save playlists for debugging purposes @@ -345,6 +418,9 @@ def _download_video(video_id, args) -> None: print_out("\nDownloaded: {}".format(target)) + if args.execute_after: + _execute_download_command(target, args.execute_after) + def _determine_time_range(video_id, args): if args.start or args.end: diff --git a/twitchdl/console.py b/twitchdl/console.py index 92ee67b..daf3479 100644 --- a/twitchdl/console.py +++ b/twitchdl/console.py @@ -160,11 +160,6 @@ def rate(value: str) -> int: "nargs": "?", "const": 10, }), - (["-d", "--download"], { - "help": "Download all videos in given period (in source quality)", - "action": "store_true", - "default": False, - }), ], ), Command( @@ -203,7 +198,7 @@ def rate(value: str) -> int: "default": False, }), (["-q", "--quality"], { - "help": "Video quality, e.g. 720p. Set to 'source' to get best quality.", + "help": "Video quality, e.g. 720p30 or 720p60. Set to 'source' to get best quality.", "type": str, }), (["-a", "--auth-token"], { @@ -218,6 +213,11 @@ def rate(value: str) -> int: "action": "store_true", "default": False, }), + (["--skipall"], { + "help": "Skip the current file if it already exists without prompting.", + "action": "store_true", + "default": False, + }), (["--overwrite"], { "help": "Overwrite the target file if it already exists without prompting.", "action": "store_true", @@ -240,6 +240,36 @@ def rate(value: str) -> int: "nargs": "?", "const": 0 }), + (["-x", "--all"], { + "help": "Download all videos on the channel. Overrides all other arguments. Pass in the channel name as the 'videos' argument.", + "action": "store_true", + "default": False, + }), + (["-y", "--skip-latest"], { + "help": "Skip downloading the latest video. Only makes sense with the --all flag.", + "action": "store_true", + "default": False, + }), + (["-t", "--tempdir"], { + "help": "Override the temp dir path.", + "type": str, + "default": "" + }), + (["-d", "--output-dir"], { + "help": "Customize location of the output directory. Defaults to the current directory.", + "type": str, + "default": "." + }), + (["-u", "--execute-after"], { + "help": "Run a CLI command after each file is downloaded and processed. In your command, use ^p for the absolute path to the file that was downloaded, and ^f for just the file name. An example use case is in examples/s3_upload_nuke.sh in the GitHub repository.", + "type": str, + "default": None + }), + (["-z", "--execute-before"], { + "help": "Run a CLI command before each file is downloaded. Return an exit code of 0 to indicate you want to download the file, or nonzero to indicate you want to skip the file. In your command, use ^p for the absolute path to the file that was downloaded, and ^f for just the file name. An example use case is in examples/s3_search.sh in the GitHub repository.", + "type": str, + "default": None + }), ], ), Command( diff --git a/twitchdl/utils.py b/twitchdl/utils.py index 771435a..84df7cb 100644 --- a/twitchdl/utils.py +++ b/twitchdl/utils.py @@ -92,7 +92,7 @@ def titlify(value): CLIP_PATTERNS = [ r"^(?P[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)$", r"^https://(www.)?twitch.tv/\w+/clip/(?P[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)(\?.+)?$", - r"^https://clips.twitch.tv/(?P[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)(\?.+)?$", + r"^https://clips.twitch.tv/(?P[A-Za-z0-9-]+(?:-[A-Za-z0-9_-]{16})?)(\?.+)?$", ]