From 3fd66420ca1f549ee7d5cd18e0de83cf8486326b Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Sun, 23 Jun 2024 14:10:48 -0700 Subject: [PATCH] Add emerge jobs tmpdir blocks and files threshold options --jobs-tmpdir-blocks-threshold[=RATIO] Specifies the maximum ratio of used blocks allowed (a floating-point number between 0.0 and 1.0) in PORTAGE_TMPDIR when starting a new job. With no argument, removes a previous blocks ratio threshold. For example, use a ratio of 0.50 to stop starting new jobs when the blocks usage in PORTAGE_TMPDIR exceeds 50%. This option conflicts with FEATURES="keepwork". WARNING: Since the job scheduler is unable to predict the future consumption of jobs that it has scheduled, users are advised to set a threshold that provides a significant amount of headroom, in order to decrease the probability that jobs will fail due to ENOSPC errors. --jobs-tmpdir-files-threshold[=RATIO] Specifies the maximum ratio of used files (inodes) allowed (a floating-point number between 0.0 and 1.0) in PORTAGE_TMPDIR when starting a new job. With no argument, removes a previous files ratio threshold. For example, use a ratio of 0.50 to stop starting new jobs when the files usage in PORTAGE_TMPDIR exceeds 50%. This option conflicts with FEATURES="keepwork". WARNING: Since the job scheduler is unable to predict the future consumption of jobs that it has scheduled, users are advised to set a threshold that provides a significant amount of headroom, in order to decrease the probability that jobs will fail due to ENOSPC errors. Bug: https://bugs.gentoo.org/934382 Signed-off-by: Zac Medico --- lib/_emerge/Scheduler.py | 60 ++++++++++++++++++++++++++++++++-- lib/_emerge/main.py | 69 +++++++++++++++++++++++++++++++++++++++- man/emerge.1 | 28 +++++++++++++++- 3 files changed, 153 insertions(+), 4 deletions(-) diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py index 614df9e783..977c201217 100644 --- a/lib/_emerge/Scheduler.py +++ b/lib/_emerge/Scheduler.py @@ -26,6 +26,7 @@ from portage._sets.base import InternalPackageSet from portage.util import ensure_dirs, writemsg, writemsg_level from portage.util.futures import asyncio +from portage.util.path import first_existing from portage.util.SlotObject import SlotObject from portage.util._async.SchedulerInterface import SchedulerInterface from portage.package.ebuild.digestcheck import digestcheck @@ -64,7 +65,7 @@ class Scheduler(PollScheduler): - # max time between loadavg checks (seconds) + # max time between loadavg and tmpdir statvfs checks (seconds) _loadavg_latency = 30 # max time between display status updates (seconds) @@ -229,6 +230,10 @@ def __init__( if max_jobs is None: max_jobs = 1 self._set_max_jobs(max_jobs) + self._jobs_tmpdir_blocks_threshold = myopts.get( + "--jobs-tmpdir-blocks-threshold" + ) + self._jobs_tmpdir_files_threshold = myopts.get("--jobs-tmpdir-files-threshold") self._running_root = trees[trees._running_eroot]["root_config"] self.edebug = 0 if settings.get("PORTAGE_DEBUG", "") == "1": @@ -1573,7 +1578,11 @@ def _main_loop(self): self._main_exit = self._event_loop.create_future() if ( - self._max_load is not None + ( + self._max_load is not None + or self._jobs_tmpdir_blocks_threshold is not None + or self._jobs_tmpdir_files_threshold is not None + ) and self._loadavg_latency is not None and (self._max_jobs is True or self._max_jobs > 1) ): @@ -1792,6 +1801,53 @@ def _is_work_scheduled(self): def _running_job_count(self): return self._jobs + def _can_add_job(self): + if not super()._can_add_job(): + return False + + running_job_count = self._running_job_count() + if running_job_count == 0 and not self._merge_wait_queue: + # Ensure there is forward progress if there are no running + # jobs and no jobs in the _merge_wait_queue. + return True + + if ( + self._jobs_tmpdir_blocks_threshold is not None + or self._jobs_tmpdir_files_threshold is not None + ) and hasattr(os, "statvfs"): + tmpdirs = set() + for root in self.trees: + settings = self.trees[root]["root_config"].settings + if settings["PORTAGE_TMPDIR"] in tmpdirs: + continue + tmpdirs.add(settings["PORTAGE_TMPDIR"]) + tmpdir = first_existing( + os.path.join(settings["PORTAGE_TMPDIR"], "portage") + ) + try: + vfs_stat = os.statvfs(tmpdir) + except OSError as e: + writemsg_level( + f"!!! statvfs('{tmpdir}'): {e}\n", + noiselevel=-1, + level=logging.ERROR, + ) + else: + if ( + self._jobs_tmpdir_blocks_threshold is not None + and (vfs_stat.f_blocks - vfs_stat.f_bavail) / vfs_stat.f_blocks + ) >= self._jobs_tmpdir_blocks_threshold: + return False + if ( + self._jobs_tmpdir_files_threshold is not None + and vfs_stat.f_files > 0 + and ((vfs_stat.f_files - vfs_stat.f_favail) / vfs_stat.f_files) + >= self._jobs_tmpdir_files_threshold + ): + return False + + return True + def _schedule_tasks(self): while True: state_change = 0 diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py index 465e20163d..6183650d53 100644 --- a/lib/_emerge/main.py +++ b/lib/_emerge/main.py @@ -1,4 +1,4 @@ -# Copyright 1999-2023 Gentoo Authors +# Copyright 1999-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import argparse @@ -165,6 +165,8 @@ def __contains__(self, s): "--getbinpkgonly": y_or_n, "--ignore-world": y_or_n, "--jobs": valid_integers, + "--jobs-tmpdir-blocks-threshold": valid_floats, + "--jobs-tmpdir-files-threshold": valid_floats, "--keep-going": y_or_n, "--load-average": valid_floats, "--onlydeps-with-ideps": y_or_n, @@ -523,6 +525,14 @@ def parse_opts(tmpcmdline, silent=False): "help": "Specifies the number of packages to build " + "simultaneously.", "action": "store", }, + "--jobs-tmpdir-blocks-threshold": { + "help": "Specifies maximum used blocks ratio when starting a new job.", + "action": "store", + }, + "--jobs-tmpdir-files-threshold": { + "help": "Specifies maximum used files ratio when starting a new job.", + "action": "store", + }, "--keep-going": { "help": "continue as much as possible after an error", "choices": true_y_or_n, @@ -1033,6 +1043,42 @@ def parse_opts(tmpcmdline, silent=False): myoptions.jobs = jobs + if myoptions.jobs_tmpdir_blocks_threshold == "True": + myoptions.jobs_tmpdir_blocks_threshold = None + + if myoptions.jobs_tmpdir_blocks_threshold: + try: + jobs_tmpdir_blocks_threshold = float(myoptions.jobs_tmpdir_blocks_threshold) + except ValueError: + jobs_tmpdir_blocks_threshold = 0.0 + + if jobs_tmpdir_blocks_threshold <= 0.0 or jobs_tmpdir_blocks_threshold > 1.0: + jobs_tmpdir_blocks_threshold = None + if not silent: + parser.error( + f"Invalid --jobs-tmpdir-blocks-threshold: '{myoptions.jobs_tmpdir_blocks_threshold}'\n" + ) + + myoptions.jobs_tmpdir_blocks_threshold = jobs_tmpdir_blocks_threshold + + if myoptions.jobs_tmpdir_files_threshold == "True": + myoptions.jobs_tmpdir_files_threshold = None + + if myoptions.jobs_tmpdir_files_threshold: + try: + jobs_tmpdir_files_threshold = float(myoptions.jobs_tmpdir_files_threshold) + except ValueError: + jobs_tmpdir_files_threshold = 0.0 + + if jobs_tmpdir_files_threshold <= 0.0 or jobs_tmpdir_files_threshold > 1.0: + jobs_tmpdir_files_threshold = None + if not silent: + parser.error( + f"Invalid --jobs-tmpdir-files-threshold: '{myoptions.jobs_tmpdir_files_threshold}'\n" + ) + + myoptions.jobs_tmpdir_files_threshold = jobs_tmpdir_files_threshold + if myoptions.load_average == "True": myoptions.load_average = None @@ -1304,6 +1350,27 @@ def emerge_main(args: Optional[list[str]] = None): emerge_config.action, emerge_config.opts, emerge_config.args = parse_opts( tmpcmdline ) + if ( + "--jobs-tmpdir-blocks-threshold" in emerge_config.opts + and "keepwork" in emerge_config.running_config.settings.features + ): + writemsg_level( + "--jobs-tmpdir-blocks-threshold conflicts with FEATURES=keepwork\n", + level=logging.ERROR, + noiselevel=-1, + ) + return 1 + + if ( + "--jobs-tmpdir-files-threshold" in emerge_config.opts + and "keepwork" in emerge_config.running_config.settings.features + ): + writemsg_level( + "--jobs-tmpdir-files-threshold conflicts with FEATURES=keepwork\n", + level=logging.ERROR, + noiselevel=-1, + ) + return 1 try: return run_action(emerge_config) diff --git a/man/emerge.1 b/man/emerge.1 index e30f5f813e..1e6f0af877 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -1,4 +1,4 @@ -.TH "EMERGE" "1" "May 2024" "Portage @VERSION@" "Portage" +.TH "EMERGE" "1" "Jun 2024" "Portage @VERSION@" "Portage" .SH "NAME" emerge \- Command\-line interface to the Portage system .SH "SYNOPSIS" @@ -693,6 +693,32 @@ Note that interactive packages currently force a setting of \fI\-\-jobs=1\fR. This issue can be temporarily avoided by specifying \fI\-\-accept\-properties=\-interactive\fR. .TP +.BR \-\-jobs\-tmpdir\-blocks\-threshold[=RATIO] +Specifies the maximum ratio of used blocks allowed (a floating\-point +number between \fI0.0\fR and \fI1.0\fR) in \fBPORTAGE_TMPDIR\fR when +starting a new job. With noargument, removes a previous blocks ratio +threshold. For example, use a ratio of \fI0.50\fR to stop starting new +jobs when the blocks usage in \fBPORTAGE_TMPDIR\fR exceeds \fI50%\fR. +This option conflicts with \fBFEATURES="keepwork"\fR. +\fBWARNING:\fR Since the job scheduler is unable to predict the future +consumption of jobs that it has scheduled, users are advised to set a +threshold that provides a significant amount of headroom, in order to +decrease the probability that jobs will fail due to \fIENOSPC\fR +errors. +.TP +.BR \-\-jobs\-tmpdir\-files\-threshold[=RATIO] +Specifies the maximum ratio of used files (inodes) allowed (a +floating\-point number between \fI0.0\fR and \fI1.0\fR) in +\fBPORTAGE_TMPDIR\fR when starting a new job. With no argument, removes +a previous files ratio threshold. For example, use a ratio of \fI0.85\fR +to stop starting new jobs when the files usage in \fBPORTAGE_TMPDIR\fR +exceeds \fI85%\fR. This option conflicts with \fBFEATURES="keepwork"\fR. +\fBWARNING:\fR Since the job scheduler is unable to predict the future +consumption of jobs that it has scheduled, users are advised to set a +threshold that provides a significant amount of headroom, in order to +decrease the probability that jobs will fail due to \fIENOSPC\fR +errors. +.TP .BR "\-\-keep\-going [ y | n ]" Continue as much as possible after an error. When an error occurs, dependencies are recalculated for remaining packages and any with