From ba91bcb1eb22d174d53a86e42ca87927cd5d0e96 Mon Sep 17 00:00:00 2001 From: Quentin Schulz Date: Sat, 9 Jan 2021 13:43:48 +0100 Subject: [PATCH] prosopopee: allow selecting number of threads used for thumbnail generation Generating thumbnails is done in parallel threads via multiprocessing.Pool. By default, Pool schedules tasks on as many threads as there are cpu threads on the host machine. Let's allow users to select the number of threads Pool can use. Signed-off-by: Quentin Schulz --- prosopopee/prosopopee.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/prosopopee/prosopopee.py b/prosopopee/prosopopee.py index 4be55d5..2522e43 100644 --- a/prosopopee/prosopopee.py +++ b/prosopopee/prosopopee.py @@ -47,7 +47,15 @@ def loglevel(string): help="Configure the logging level", ) subparser = parser.add_subparsers(dest="cmd") -subparser.add_parser("build", help="Generate static site") +parser_build = subparser.add_parser("build", help="Generate static site") +parser_build.add_argument( + "-j", + "--jobs", + default=None, + type=int, + help="Specifies number of jobs (thumbnail generations) to run simultaneously. Default: number " + "of threads available on the system", +) subparser.add_parser("test", help="Verify all your yaml data") subparser.add_parser("preview", help="Start preview webserver on port 9000") subparser.add_parser("deploy", help="Deploy your website") @@ -923,7 +931,12 @@ def main(): logger.info("Success: HTML file building without error") sys.exit(0) - with Pool() as pool: + # If prosopopee is started without any argument, 'build' is assumed but the jobs parameter + # is not part of the namespace, so set its default to None (or 'number of available CPU + # treads') + jobs = args.jobs if args.cmd else None + + with Pool(jobs) as pool: # Pool splits the iterable into pre-defined chunks which are then assigned to processes. # There is no other scheduling in play after that. This is an issue when chunks are # outrageously unbalanced in terms of CPU time which happens when most galleries are