Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add emerge jobs tmpdir blocks and files threshold options #1351

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add emerge jobs tmpdir blocks and files threshold options #1351

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 58 additions & 2 deletions lib/_emerge/Scheduler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from portage._sets.base import InternalPackageSet
from portage.util import ensure_dirs, writemsg, writemsg_level
from portage.util.futures import asyncio
from portage.util.path import first_existing
from portage.util.SlotObject import SlotObject
from portage.util._async.SchedulerInterface import SchedulerInterface
from portage.package.ebuild.digestcheck import digestcheck
Expand Down Expand Up @@ -64,7 +65,7 @@


class Scheduler(PollScheduler):
# max time between loadavg checks (seconds)
# max time between loadavg and tmpdir statvfs checks (seconds)
_loadavg_latency = 30

# max time between display status updates (seconds)
Expand Down Expand Up @@ -229,6 +230,10 @@ def __init__(
if max_jobs is None:
max_jobs = 1
self._set_max_jobs(max_jobs)
self._jobs_tmpdir_blocks_threshold = myopts.get(
"--jobs-tmpdir-blocks-threshold"
)
self._jobs_tmpdir_files_threshold = myopts.get("--jobs-tmpdir-files-threshold")
self._running_root = trees[trees._running_eroot]["root_config"]
self.edebug = 0
if settings.get("PORTAGE_DEBUG", "") == "1":
Expand Down Expand Up @@ -1573,7 +1578,11 @@ def _main_loop(self):
self._main_exit = self._event_loop.create_future()

if (
self._max_load is not None
(
self._max_load is not None
or self._jobs_tmpdir_blocks_threshold is not None
or self._jobs_tmpdir_files_threshold is not None
)
and self._loadavg_latency is not None
and (self._max_jobs is True or self._max_jobs > 1)
):
Expand Down Expand Up @@ -1792,6 +1801,53 @@ def _is_work_scheduled(self):
def _running_job_count(self):
return self._jobs

def _can_add_job(self):
if not super()._can_add_job():
return False

zmedico marked this conversation as resolved.
Show resolved Hide resolved
running_job_count = self._running_job_count()
if running_job_count == 0 and not self._merge_wait_queue:
# Ensure there is forward progress if there are no running
# jobs and no jobs in the _merge_wait_queue.
return True

if (
self._jobs_tmpdir_blocks_threshold is not None
or self._jobs_tmpdir_files_threshold is not None
) and hasattr(os, "statvfs"):
tmpdirs = set()
for root in self.trees:
settings = self.trees[root]["root_config"].settings
if settings["PORTAGE_TMPDIR"] in tmpdirs:
continue
tmpdirs.add(settings["PORTAGE_TMPDIR"])
tmpdir = first_existing(
os.path.join(settings["PORTAGE_TMPDIR"], "portage")
)
try:
vfs_stat = os.statvfs(tmpdir)
except OSError as e:
writemsg_level(
f"!!! statvfs('{tmpdir}'): {e}\n",
noiselevel=-1,
level=logging.ERROR,
)
else:
if (
self._jobs_tmpdir_blocks_threshold is not None
and (vfs_stat.f_blocks - vfs_stat.f_bavail) / vfs_stat.f_blocks
) >= self._jobs_tmpdir_blocks_threshold:
return False
if (
self._jobs_tmpdir_files_threshold is not None
and vfs_stat.f_files > 0
and ((vfs_stat.f_files - vfs_stat.f_favail) / vfs_stat.f_files)
>= self._jobs_tmpdir_files_threshold
):
return False

return True

def _schedule_tasks(self):
while True:
state_change = 0
Expand Down
69 changes: 68 additions & 1 deletion lib/_emerge/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 1999-2023 Gentoo Authors
# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2

import argparse
Expand Down Expand Up @@ -165,6 +165,8 @@ def __contains__(self, s):
"--getbinpkgonly": y_or_n,
"--ignore-world": y_or_n,
"--jobs": valid_integers,
"--jobs-tmpdir-blocks-threshold": valid_floats,
"--jobs-tmpdir-files-threshold": valid_floats,
"--keep-going": y_or_n,
"--load-average": valid_floats,
"--onlydeps-with-ideps": y_or_n,
Expand Down Expand Up @@ -523,6 +525,14 @@ def parse_opts(tmpcmdline, silent=False):
"help": "Specifies the number of packages to build " + "simultaneously.",
"action": "store",
},
"--jobs-tmpdir-blocks-threshold": {
"help": "Specifies maximum used blocks ratio when starting a new job.",
"action": "store",
},
"--jobs-tmpdir-files-threshold": {
"help": "Specifies maximum used files ratio when starting a new job.",
"action": "store",
},
"--keep-going": {
"help": "continue as much as possible after an error",
"choices": true_y_or_n,
Expand Down Expand Up @@ -1033,6 +1043,42 @@ def parse_opts(tmpcmdline, silent=False):

myoptions.jobs = jobs

if myoptions.jobs_tmpdir_blocks_threshold == "True":
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just coming back to this: all of this really makes me think we need to revisit the option handling :(

But it's of course fine for now

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

True here indicates a missing argument, and this can mean different things for different options. For example, for --jobs a missing argument means unlimited jobs, and so it's handled like this instead:

        if myoptions.jobs == "True":
            jobs = True

myoptions.jobs_tmpdir_blocks_threshold = None

if myoptions.jobs_tmpdir_blocks_threshold:
try:
jobs_tmpdir_blocks_threshold = float(myoptions.jobs_tmpdir_blocks_threshold)
except ValueError:
jobs_tmpdir_blocks_threshold = 0.0

if jobs_tmpdir_blocks_threshold <= 0.0 or jobs_tmpdir_blocks_threshold > 1.0:
jobs_tmpdir_blocks_threshold = None
if not silent:
parser.error(
f"Invalid --jobs-tmpdir-blocks-threshold: '{myoptions.jobs_tmpdir_blocks_threshold}'\n"
)

myoptions.jobs_tmpdir_blocks_threshold = jobs_tmpdir_blocks_threshold

if myoptions.jobs_tmpdir_files_threshold == "True":
myoptions.jobs_tmpdir_files_threshold = None

if myoptions.jobs_tmpdir_files_threshold:
try:
jobs_tmpdir_files_threshold = float(myoptions.jobs_tmpdir_files_threshold)
except ValueError:
jobs_tmpdir_files_threshold = 0.0

if jobs_tmpdir_files_threshold <= 0.0 or jobs_tmpdir_files_threshold > 1.0:
jobs_tmpdir_files_threshold = None
if not silent:
parser.error(
f"Invalid --jobs-tmpdir-files-threshold: '{myoptions.jobs_tmpdir_files_threshold}'\n"
)

myoptions.jobs_tmpdir_files_threshold = jobs_tmpdir_files_threshold

if myoptions.load_average == "True":
myoptions.load_average = None

Expand Down Expand Up @@ -1304,6 +1350,27 @@ def emerge_main(args: Optional[list[str]] = None):
emerge_config.action, emerge_config.opts, emerge_config.args = parse_opts(
tmpcmdline
)
if (
"--jobs-tmpdir-blocks-threshold" in emerge_config.opts
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I suppose we should check for merge-wait too?

Copy link
Member Author

@zmedico zmedico Jun 27, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's valid to use this option with or without merge-wait because either way the user could desire to use disk pressure to limit starting of new jobs.

On the other hand, #1349 only makes sense with merge-wait.

and "keepwork" in emerge_config.running_config.settings.features
):
writemsg_level(
"--jobs-tmpdir-blocks-threshold conflicts with FEATURES=keepwork\n",
level=logging.ERROR,
noiselevel=-1,
)
return 1

if (
"--jobs-tmpdir-files-threshold" in emerge_config.opts
and "keepwork" in emerge_config.running_config.settings.features
):
writemsg_level(
"--jobs-tmpdir-files-threshold conflicts with FEATURES=keepwork\n",
level=logging.ERROR,
noiselevel=-1,
)
return 1

try:
return run_action(emerge_config)
Expand Down
28 changes: 27 additions & 1 deletion man/emerge.1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
.TH "EMERGE" "1" "May 2024" "Portage @VERSION@" "Portage"
.TH "EMERGE" "1" "Jun 2024" "Portage @VERSION@" "Portage"
.SH "NAME"
emerge \- Command\-line interface to the Portage system
.SH "SYNOPSIS"
Expand Down Expand Up @@ -693,6 +693,32 @@ Note that interactive packages currently force a setting
of \fI\-\-jobs=1\fR. This issue can be temporarily avoided
by specifying \fI\-\-accept\-properties=\-interactive\fR.
.TP
.BR \-\-jobs\-tmpdir\-blocks\-threshold[=RATIO]
Specifies the maximum ratio of used blocks allowed (a floating\-point
number between \fI0.0\fR and \fI1.0\fR) in \fBPORTAGE_TMPDIR\fR when
starting a new job. With noargument, removes a previous blocks ratio
threshold. For example, use a ratio of \fI0.50\fR to stop starting new
jobs when the blocks usage in \fBPORTAGE_TMPDIR\fR exceeds \fI50%\fR.
This option conflicts with \fBFEATURES="keepwork"\fR.
\fBWARNING:\fR Since the job scheduler is unable to predict the future
consumption of jobs that it has scheduled, users are advised to set a
threshold that provides a significant amount of headroom, in order to
decrease the probability that jobs will fail due to \fIENOSPC\fR
errors.
.TP
.BR \-\-jobs\-tmpdir\-files\-threshold[=RATIO]
Specifies the maximum ratio of used files (inodes) allowed (a
floating\-point number between \fI0.0\fR and \fI1.0\fR) in
\fBPORTAGE_TMPDIR\fR when starting a new job. With no argument, removes
a previous files ratio threshold. For example, use a ratio of \fI0.85\fR
to stop starting new jobs when the files usage in \fBPORTAGE_TMPDIR\fR
exceeds \fI85%\fR. This option conflicts with \fBFEATURES="keepwork"\fR.
\fBWARNING:\fR Since the job scheduler is unable to predict the future
consumption of jobs that it has scheduled, users are advised to set a
threshold that provides a significant amount of headroom, in order to
decrease the probability that jobs will fail due to \fIENOSPC\fR
errors.
.TP
.BR "\-\-keep\-going [ y | n ]"
Continue as much as possible after an error. When an error occurs,
dependencies are recalculated for remaining packages and any with
Expand Down