From 2fa097b60e5da562471f5af65658b740044039fd Mon Sep 17 00:00:00 2001 From: "Tony T. Wang" Date: Sun, 26 Jan 2025 08:26:47 +0000 Subject: [PATCH 1/2] Allow configurable wait time for compose up via environment variable --- src/inspect_ai/util/_sandbox/docker/compose.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/inspect_ai/util/_sandbox/docker/compose.py b/src/inspect_ai/util/_sandbox/docker/compose.py index 612d3ac45..1945a77cc 100644 --- a/src/inspect_ai/util/_sandbox/docker/compose.py +++ b/src/inspect_ai/util/_sandbox/docker/compose.py @@ -30,11 +30,14 @@ async def compose_up(project: ComposeProject) -> None: # passing the --wait flag (see https://github.com/docker/compose/issues/10596). # In practice, we will catch any errors when calling compose_check_running() # immediately after we call compose_up(). + seconds_to_wait_str: str = os.environ.get( + "INSPECT_EVAL_COMPOSE_UP_WAIT", COMPOSE_WAIT + ) await compose_command( - ["up", "--detach", "--wait", "--wait-timeout", COMPOSE_WAIT], + ["up", "--detach", "--wait", "--wait-timeout", seconds_to_wait_str], project=project, - # wait up to 5 minutes for container to go up (compose wait + 3 minutes) - timeout=300, + # wait up to (compose wait + 3 minutes) for container to go up + timeout=int(seconds_to_wait_str) + 180, ) From f566d0df84d66527da8bee872af9a026a03536f7 Mon Sep 17 00:00:00 2001 From: "Tony T. Wang" Date: Fri, 7 Feb 2025 15:45:08 +0000 Subject: [PATCH 2/2] Add documentation on INSPECT_EVAL_COMPOSE_UP_WAIT feature --- docs/sandboxing.qmd | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/sandboxing.qmd b/docs/sandboxing.qmd index df8854504..dc367fd46 100644 --- a/docs/sandboxing.qmd +++ b/docs/sandboxing.qmd @@ -336,3 +336,8 @@ services: To diagnose sandbox execution issues (e.g. commands that don't terminate properly, contianer lifecylce issues, etc.) you should use Inspect's [Tracing](tracing.qmd) facility. Trace logs record the beginning and end of calls to `subprocess()` (e.g. tool calls that run commands in sandboxes) as well as control commands sent to Docker Compose. The `inspect trace anomalies` subcommand then enables you to query for commands that don't terminate, timeout, or have errors. See the article on [Tracing](tracing.qmd) for additional details. + +### Sandbox container startup wait time +For complicated sandboxes that take multiple minutes to fully spin up, you may run into an error of the form "One or more docker containers failed to start from [...]". This error occurs because by default, Inspect will only wait 120 seconds for all sandbox containers to start. + +To override this 120 second default wait time, you can set the environment variable `INSPECT_EVAL_COMPOSE_UP_WAIT` which specifies in seconds how long inspect should wait for sandbox containers to spin up.