diff --git a/scripts/README.md b/scripts/README.md index e8a28e29ec..7b7b990a65 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -39,6 +39,7 @@ Two commands exist for downloading logs. |-z , --local-zone|Specify times for `-s` and `-e` in your local time zone. If this is not set, times are assumed to be in UTC|unset/false| |-p, --file-pattern|Should match the executor.s3.uploader.pattern setting, determines if we can match on file name for s3 logs|`%requestId/%Y/%m/%taskId_%index-%s-%filename`| |-N, --no-name-fetch-off|If a logtype matcher is specified, but the s3 log pattern does not include file name, don't download any s3 files| None (fetch all)| +|-D, --download-only|Only download logs in their current state, don't unzip or grep|| |-g, --grep|Grep string for searching log files(Only for `logfetch`)|| |-l, --logtype|Glob matcher for type of log file to download| None (match all)| |-S, --skip-s3|Don't search/download s3 logs|false| diff --git a/scripts/logfetch/entrypoint.py b/scripts/logfetch/entrypoint.py index 6063ffcfe9..a048b75bc6 100644 --- a/scripts/logfetch/entrypoint.py +++ b/scripts/logfetch/entrypoint.py @@ -52,7 +52,8 @@ def fetch_logs(args): all_logs += download_s3_logs(args) if not args.skip_live: all_logs += download_live_logs(args) - grep_files(args, all_logs) + if not args.download_only: + grep_files(args, all_logs) except KeyboardInterrupt: exit('Stopping logfetch...', 'magenta') @@ -64,7 +65,8 @@ def cat_logs(args): all_logs += download_s3_logs(args) if not args.skip_live: all_logs += download_live_logs(args) - cat_files(args, all_logs) + if not args.download_only: + cat_files(args, all_logs) except KeyboardInterrupt: exit('Stopping logcat...', 'magenta') @@ -148,6 +150,7 @@ def fetch(): parser.add_argument("--search", dest="search", help="run logsearch on the local cache of downloaded files", action='store_true') parser.add_argument("-V", "--verbose", dest="verbose", help="Print more verbose output", action='store_true') parser.add_argument("--silent", dest="silent", help="No stderr (progress, file names, etc) output", action='store_true') + parser.add_argument("-D" ,"--download-only", dest="download_only", help="Only download files, don't unzip or grep", action='store_true') args = parser.parse_args(remaining_argv) @@ -277,6 +280,7 @@ def cat(): parser.add_argument("-U", "--use-cache", dest="use_cache", help="Use cache for live logs, don't re-download them", action='store_true') parser.add_argument("-V", "--verbose", dest="verbose", help="Print more verbose output", action='store_true') parser.add_argument("--silent", dest="silent", help="No stderr (progress, file names, etc) output", action='store_true') + parser.add_argument("-D" ,"--download-only", dest="download_only", help="Only download files, don't unzip or grep", action='store_true') args = parser.parse_args(remaining_argv) diff --git a/scripts/logfetch/live_logs.py b/scripts/logfetch/live_logs.py index e8338036a2..87cdb58f4b 100644 --- a/scripts/logfetch/live_logs.py +++ b/scripts/logfetch/live_logs.py @@ -72,7 +72,7 @@ def download_live_logs(args): sys.stderr.write(colored('\nStarting {0} live logs downloads\n'.format(len(async_requests)), 'cyan')) callbacks.goal = len(async_requests) grequests.map(async_requests, stream=True, size=args.num_parallel_fetches) - if zipped_files: + if zipped_files and not args.download_only: if not args.silent: sys.stderr.write(colored('\nUnpacking {0} log(s)\n'.format(len(zipped_files)), 'cyan')) all_logs = all_logs + logfetch_base.unpack_logs(args, zipped_files) diff --git a/scripts/logfetch/s3_logs.py b/scripts/logfetch/s3_logs.py index 2f53757632..2b7091aa9a 100644 --- a/scripts/logfetch/s3_logs.py +++ b/scripts/logfetch/s3_logs.py @@ -50,12 +50,13 @@ def download_s3_logs(args): sys.stderr.write(colored('Starting {0} S3 Downloads with {1} parallel fetches\n'.format(len(async_requests), args.num_parallel_fetches), 'cyan')) callbacks.goal = len(async_requests) grequests.map(async_requests, stream=True, size=args.num_parallel_fetches) - if not args.silent: + if not args.silent and not args.download_only: sys.stderr.write(colored('\nUnpacking {0} S3 log(s)\n'.format(len(async_requests)), 'cyan')) else: if not args.silent: sys.stderr.write(colored('No S3 logs to download\n', 'cyan')) - all_logs = all_logs + logfetch_base.unpack_logs(args, zipped_files) + if not args.download_only: + all_logs = all_logs + logfetch_base.unpack_logs(args, zipped_files) if not args.silent: sys.stderr.write(colored('All S3 logs up to date\n', 'cyan')) return all_logs diff --git a/scripts/setup.py b/scripts/setup.py index 8480eb89b3..5a6df8e6af 100644 --- a/scripts/setup.py +++ b/scripts/setup.py @@ -11,7 +11,7 @@ setup( name='singularity-logfetch', - version='0.22.1', + version='0.23.0', description='Singularity log fetching and searching', author="HubSpot", author_email='singularity-users@googlegroups.com',