diff --git a/etc/openqa/openqa.ini b/etc/openqa/openqa.ini index 0b343066007..deca516e433 100644 --- a/etc/openqa/openqa.ini +++ b/etc/openqa/openqa.ini @@ -115,12 +115,16 @@ #do_push = no ## whether to clone CASEDIR or NEEDLES_DIR on the web UI if that var points to a Git repo #git_auto_clone = yes -## enable automatic updates of all test code and needles managed via Git (still experimental, currently still breaks scheduling parallel clusters) -#git_auto_update = no +## enable automatic updates of all test code and needles managed via Git +#git_auto_update = yes ## specifies how to handle errors on automatic updates via git_auto_update ## - when set to "best-effort" openQA jobs are started even if the update failed ## - when set to "strict" openQA jobs will be blocked until the update succeeded or set to incomplete when the retries for updating are exhausted #git_auto_update_method = best-effort +# whether openQA should attempt to display needles of the correct version in the web UI +#checkout_needles_sha = no +# retention for storing temporary needle refs in minutes +#temp_needle_refs_retention = 120 ## Authentication method to use for user management [auth] diff --git a/lib/OpenQA/Git.pm b/lib/OpenQA/Git.pm index da4a83d1fdd..1d0ce5224d3 100644 --- a/lib/OpenQA/Git.pm +++ b/lib/OpenQA/Git.pm @@ -6,6 +6,7 @@ package OpenQA::Git; use Mojo::Base -base, -signatures; use Mojo::Util 'trim'; use Cwd 'abs_path'; +use Mojo::File 'path'; use OpenQA::Utils qw(run_cmd_with_log_return_error); has 'app'; @@ -157,4 +158,16 @@ sub is_workdir_clean ($self) { return $r->{status}; } +sub cache_ref ($self, $ref, $relative_path, $output_file) { + if (-f $output_file) { + eval { path($output_file)->touch }; + return $@ ? $@ : undef; + } + my @git = $self->_prepare_git_command; + my $res = run_cmd_with_log_return_error [@git, 'show', "$ref:./$relative_path"], output_file => $output_file; + return undef if $res->{status}; + unlink $output_file; + return _format_git_error($res, 'Unable to cache Git ref'); +} + 1; diff --git a/lib/OpenQA/Needles.pm b/lib/OpenQA/Needles.pm new file mode 100644 index 00000000000..25a2990a111 --- /dev/null +++ b/lib/OpenQA/Needles.pm @@ -0,0 +1,57 @@ +# Copyright SUSE LLC +# SPDX-License-Identifier: GPL-2.0-or-later + +package OpenQA::Needles; +use Mojo::Base -strict, -signatures; + +use Exporter qw(import); +use File::Basename; +use File::Spec; +use File::Spec::Functions qw(catdir); +use OpenQA::Git; +use OpenQA::Log qw(log_error); +use OpenQA::Utils qw(prjdir sharedir); +use Mojo::File qw(path); + +our @EXPORT = qw(temp_dir is_in_temp_dir needle_temp_dir locate_needle); + +my $tmp_dir = prjdir() . '/webui/cache/needle-refs'; + +sub temp_dir () { $tmp_dir } + +sub is_in_temp_dir ($file_path) { index($file_path, $tmp_dir) == 0 } + +sub needle_temp_dir ($dir, $ref) { path($tmp_dir, basename(dirname($dir)), $ref, 'needles') } + +sub _locate_needle_for_ref ($relative_needle_path, $needles_dir, $needles_ref) { + return undef unless defined $needles_ref; + + my $temp_needles_dir = needle_temp_dir($needles_dir, $needles_ref); + my $subdir = dirname($relative_needle_path); + path($temp_needles_dir, $subdir)->make_path if File::Spec->splitdir($relative_needle_path) > 1; + + my $git = OpenQA::Git->new(dir => $needles_dir); + my $temp_json_path = "$temp_needles_dir/$relative_needle_path"; + my $basename = basename($relative_needle_path, '.json'); + my $relative_png_path = "$subdir/$basename.png"; + my $temp_png_path = "$temp_needles_dir/$relative_png_path"; + my $error = $git->cache_ref($needles_ref, $relative_needle_path, $temp_json_path) + // $git->cache_ref($needles_ref, $relative_png_path, $temp_png_path); + return $temp_json_path unless defined $error; + log_error "An error occurred when looking for ref '$needles_ref' of '$relative_needle_path': $error"; + return undef; +} + +sub locate_needle ($relative_needle_path, $needles_dir, $needles_ref = undef) { + my $location_for_ref = _locate_needle_for_ref($relative_needle_path, $needles_dir, $needles_ref); + return $location_for_ref if defined $location_for_ref; + my $absolute_filename = catdir($needles_dir, $relative_needle_path); + my $needle_exists = -f $absolute_filename; + if (!$needle_exists) { + $absolute_filename = catdir(sharedir(), $relative_needle_path); + $needle_exists = -f $absolute_filename; + } + return $absolute_filename if $needle_exists; + log_error "Needle file $relative_needle_path not found within $needles_dir."; + return undef; +} diff --git a/lib/OpenQA/Schema/Result/Needles.pm b/lib/OpenQA/Schema/Result/Needles.pm index 5af2980cc67..8ae89ab5463 100644 --- a/lib/OpenQA/Schema/Result/Needles.pm +++ b/lib/OpenQA/Schema/Result/Needles.pm @@ -16,7 +16,7 @@ use OpenQA::App; use OpenQA::Git; use OpenQA::Jobs::Constants; use OpenQA::Schema::Result::Jobs; -use OpenQA::Utils qw(locate_needle); +use OpenQA::Needles qw(locate_needle); __PACKAGE__->table('needles'); __PACKAGE__->load_components(qw(InflateColumn::DateTime Timestamps)); diff --git a/lib/OpenQA/Setup.pm b/lib/OpenQA/Setup.pm index e90b6646dd6..2fc7166175b 100644 --- a/lib/OpenQA/Setup.pm +++ b/lib/OpenQA/Setup.pm @@ -69,8 +69,10 @@ sub read_config ($app) { do_push => 'no', do_cleanup => 'no', git_auto_clone => 'yes', - git_auto_update => 'no', + git_auto_update => 'yes', git_auto_update_method => 'best-effort', + checkout_needles_sha => 'no', + temp_needle_refs_retention => 120, }, scheduler => { max_job_scheduled_time => 7, diff --git a/lib/OpenQA/Shared/Plugin/Gru.pm b/lib/OpenQA/Shared/Plugin/Gru.pm index 5ac31a611bd..9984e780713 100644 --- a/lib/OpenQA/Shared/Plugin/Gru.pm +++ b/lib/OpenQA/Shared/Plugin/Gru.pm @@ -31,7 +31,10 @@ sub register_tasks ($self) { OpenQA::Task::Asset::Download OpenQA::Task::Asset::Limit OpenQA::Task::Git::Clone - OpenQA::Task::Needle::Scan OpenQA::Task::Needle::Save OpenQA::Task::Needle::Delete + OpenQA::Task::Needle::Scan + OpenQA::Task::Needle::Save + OpenQA::Task::Needle::Delete + OpenQA::Task::Needle::LimitTempRefs OpenQA::Task::Job::Limit OpenQA::Task::Job::ArchiveResults OpenQA::Task::Job::FinalizeResults diff --git a/lib/OpenQA/Task/Needle/LimitTempRefs.pm b/lib/OpenQA/Task/Needle/LimitTempRefs.pm new file mode 100644 index 00000000000..cfbce1eb7ba --- /dev/null +++ b/lib/OpenQA/Task/Needle/LimitTempRefs.pm @@ -0,0 +1,39 @@ +# Copyright SUSE LLC +# SPDX-License-Identifier: GPL-2.0-or-later + +package OpenQA::Task::Needle::LimitTempRefs; +use Mojo::Base 'Mojolicious::Plugin', -signatures; + +use File::Find; +use File::stat; +use Fcntl qw(S_ISDIR); +use OpenQA::Needles; +use OpenQA::Task::SignalGuard; +use Time::Seconds; + +my $retention; + +sub register ($self, $app, $job) { + $retention = $app->config->{'scm git'}->{temp_needle_refs_retention} * ONE_MINUTE; + $app->minion->add_task(limit_temp_needle_refs => sub ($job) { _limit($app, $job) }); +} + +sub _limit ($app, $job) { + my $ensure_task_retry_on_termination_signal_guard = OpenQA::Task::SignalGuard->new($job); + + return $job->finish({error => 'Another job to remove needle versions is running. Try again later.'}) + unless my $guard = $app->minion->guard('limit_needle_versions_task', 7200); + + # remove all temporary needles which haven't been accessed in time period specified in config + my $temp_dir = OpenQA::Needles::temp_dir; + return undef unless -d $temp_dir; + my $now = time; + my $wanted = sub { + return undef unless my $lstat = lstat $File::Find::name; + return rmdir $File::Find::name if S_ISDIR($lstat->mode); # remove all empty dirs + return unlink $File::Find::name if ($now - $lstat->mtime) > $retention; + }; + find({no_chdir => 1, bydepth => 1, wanted => $wanted}, $temp_dir); +} + +1; diff --git a/lib/OpenQA/Utils.pm b/lib/OpenQA/Utils.pm index 0a3a2d1d404..258b31ca9d8 100644 --- a/lib/OpenQA/Utils.pm +++ b/lib/OpenQA/Utils.pm @@ -26,6 +26,7 @@ use OpenQA::Log qw(log_info log_debug log_warning log_error); use Config::Tiny; use Time::HiRes qw(tv_interval); use File::Basename; +use File::Path qw(make_path); use File::Spec; use File::Spec::Functions qw(catfile catdir); use Fcntl; @@ -92,7 +93,6 @@ our @EXPORT = qw( BUGREF_REGEX LABEL_REGEX FLAG_REGEX - locate_needle needledir productdir testcasedir @@ -254,22 +254,6 @@ sub is_in_tests { sub needledir { productdir(@_) . '/needles' } -sub locate_needle { - my ($relative_needle_path, $needles_dir) = @_; - - my $absolute_filename = catdir($needles_dir, $relative_needle_path); - my $needle_exists = -f $absolute_filename; - - if (!$needle_exists) { - $absolute_filename = catdir(sharedir(), $relative_needle_path); - $needle_exists = -f $absolute_filename; - } - return $absolute_filename if $needle_exists; - - log_error("Needle file $relative_needle_path not found within $needles_dir."); - return undef; -} - # Adds a timestamp to a string (eg. needle name) or replace the already present timestamp sub ensure_timestamp_appended { my ($str) = @_; @@ -333,10 +317,12 @@ sub run_cmd_with_log { sub run_cmd_with_log_return_error ($cmd, %args) { my $stdout_level = $args{stdout} // 'debug'; my $stderr_level = $args{stderr} // 'debug'; + my $output_file = $args{output_file}; log_info('Running cmd: ' . join(' ', @$cmd)); try { my ($stdin, $stdout_err, $stdout, $stderr) = ('') x 4; - my $ipc_run_succeeded = IPC::Run::run($cmd, \$stdin, \$stdout, \$stderr); + my @out_args = defined $output_file ? ('>', $output_file, '2>', \$stderr) : (\$stdout, \$stderr); + my $ipc_run_succeeded = IPC::Run::run($cmd, \$stdin, @out_args); my $return_code = $?; chomp $stderr; if ($ipc_run_succeeded) { diff --git a/lib/OpenQA/WebAPI/Controller/File.pm b/lib/OpenQA/WebAPI/Controller/File.pm index 91a867916f8..67658f69dc7 100644 --- a/lib/OpenQA/WebAPI/Controller/File.pm +++ b/lib/OpenQA/WebAPI/Controller/File.pm @@ -6,6 +6,7 @@ use Mojo::Base 'Mojolicious::Controller', -signatures; BEGIN { $ENV{MAGICK_THREAD_LIMIT} = 1; } +use OpenQA::Needles; use OpenQA::Utils qw(:DEFAULT prjdir assetdir imagesdir); use File::Basename; use File::Spec; @@ -29,11 +30,22 @@ sub needle ($self) { # make sure the directory of the file parameter is a real subdir of testcasedir before # using it to find needle subdirectory, to prevent access outside of the zoo - if ($jsonfile && !is_in_tests($jsonfile)) { + if ($jsonfile && !is_in_tests($jsonfile) && !OpenQA::Needles::is_in_temp_dir($jsonfile)) { my $prjdir = prjdir(); warn "$jsonfile is not in a subdir of $prjdir/share/tests or $prjdir/tests"; return $self->render(text => 'Forbidden', status => 403); } + # If the json file in not in the tests we may be using a temporary + # directory for needles from a different git SHA + my $jsonfile_in_temp_dir = $jsonfile && OpenQA::Needles::is_in_temp_dir($jsonfile); + if ($jsonfile_in_temp_dir) { + $needledir = dirname($jsonfile); + # In case we're in a subdirectory, keep taking the dirname until we + # have the path of the `needles` directory + while (basename($needledir) ne 'needles') { + $needledir = dirname($needledir); + } + } # Reject directory traversal breakouts here... if (index($jsonfile, '..') != -1) { warn "jsonfile value $jsonfile is invalid, cannot contain .."; diff --git a/lib/OpenQA/WebAPI/Controller/Step.pm b/lib/OpenQA/WebAPI/Controller/Step.pm index 4542ff72941..f51dc2564fb 100644 --- a/lib/OpenQA/WebAPI/Controller/Step.pm +++ b/lib/OpenQA/WebAPI/Controller/Step.pm @@ -9,9 +9,12 @@ use Encode 'decode_utf8'; use Mojo::File 'path'; use Mojo::URL; use Mojo::Util 'decode'; -use OpenQA::Utils qw(ensure_timestamp_appended find_bug_number locate_needle needledir testcasedir); +use OpenQA::Needles qw(needle_temp_dir locate_needle); +use OpenQA::Utils qw(ensure_timestamp_appended find_bug_number needledir testcasedir + run_cmd_with_log run_cmd_with_log_return_error); use OpenQA::Jobs::Constants; use File::Basename; +use File::Path 'make_path'; use File::Which 'which'; use POSIX 'strftime'; use Mojo::JSON 'decode_json'; @@ -91,6 +94,34 @@ sub view ($self) { $self->viewimg; } +sub _determine_needles_dir_for_job ($self, $job) { + return undef unless $self->app->config->{'scm git'}->{checkout_needles_sha} eq 'yes'; + return undef unless my $needle_dirs = realpath($job->needle_dir); + my $settings = $job->settings; + return ($needle_dirs, $settings->single({key => 'NEEDLES_DIR'}) // $settings->single({key => 'CASEDIR'})); +} + +sub _create_tmpdir_for_needles_refspec ($self, $job) { + my ($needle_dirs, $needles_dir_var) = $self->_determine_needles_dir_for_job($job); + return undef unless $needles_dir_var; + my $needles_url = Mojo::URL->new($needles_dir_var->value); + return undef unless $needles_url->scheme; + my $needles_ref = $needles_url->fragment; + eval { + my $vars = decode_json(path($job->result_dir, 'vars.json')->slurp); + $needles_ref = $vars->{NEEDLES_GIT_HASH} if ref $vars eq 'HASH'; + }; + chomp $needles_ref; + return undef unless $needles_ref; + return undef unless run_cmd_with_log ['git', '-C', $needle_dirs, 'fetch', '--depth', 1, 'origin', $needles_ref]; + my $rev_parse_res = run_cmd_with_log_return_error ['git', '-C', $needle_dirs, 'rev-parse', 'FETCH_HEAD']; + return undef unless $rev_parse_res->{status}; + $needles_ref = $rev_parse_res->{stdout}; + chomp $needles_ref; + needle_temp_dir($needle_dirs, $needles_ref)->make_path; + return $needles_ref; +} + # Needle editor sub edit ($self) { return $self->reply->not_found unless $self->_init && $self->check_tabmode(); @@ -101,6 +132,7 @@ sub edit ($self) { my $distri = $job->DISTRI; my $dversion = $job->VERSION || ''; my $needle_dir = $job->needle_dir; + my $needle_ref = $self->_create_tmpdir_for_needles_refspec($job); my $app = $self->app; my $needles_rs = $app->schema->resultset('Needles'); @@ -130,7 +162,7 @@ sub edit ($self) { # Second position: the only needle (with the same matches) my $needle_info = $self->_extended_needle_info($needle_dir, $needle_name, \%basic_needle_data, $module_detail->{json}, - 0, \@error_messages); + 0, \@error_messages, $needle_ref); if ($needle_info) { $needle_info->{matches} = $screenshot->{matches}; push(@needles, $needle_info); @@ -144,10 +176,10 @@ sub edit ($self) { # $needle contains information from result, in which 'areas' refers to the best matches. # We also use $area for transforming the match information into a real area for my $needle (@$module_detail_needles) { - my $needle_info = $self->_extended_needle_info( - $needle_dir, $needle->{name}, \%basic_needle_data, - $needle->{json}, $needle->{error}, \@error_messages - ) || next; + my $needle_info + = $self->_extended_needle_info($needle_dir, $needle->{name}, \%basic_needle_data, + $needle->{json}, $needle->{error}, \@error_messages, $needle_ref) + || next; my $matches = $needle_info->{matches}; for my $match (@{$needle->{area}}) { my %area = ( @@ -188,7 +220,7 @@ sub edit ($self) { # get needle info to show the needle also in selection my $needle_info = $self->_extended_needle_info($needle_dir, $new_needle->name, \%basic_needle_data, $new_needle->path, - undef, \@error_messages) + undef, \@error_messages, $needle_ref) || next; $needle_info->{title} = 'new: ' . $needle_info->{title}; push(@needles, $needle_info); @@ -274,9 +306,9 @@ sub _new_screenshot ($self, $tags, $image_name, $matches = undef) { return \%screenshot; } -sub _basic_needle_info ($self, $name, $distri, $version, $file_name, $needles_dir) { +sub _basic_needle_info ($self, $name, $distri, $version, $file_name, $needles_dir, $needle_ref) { $file_name //= "$name.json"; - $file_name = locate_needle($file_name, $needles_dir) if !-f $file_name; + $file_name = locate_needle($file_name, $needles_dir, $needle_ref) if !-f $file_name; return (undef, 'File not found') unless defined $file_name; my $needle; @@ -303,11 +335,14 @@ sub _basic_needle_info ($self, $name, $distri, $version, $file_name, $needles_di return ($needle, undef); } -sub _extended_needle_info ($self, $needle_dir, $needle_name, $basic_needle_data, $file_name, $error, $error_messages) { +sub _extended_needle_info ($self, $needle_dir, $needle_name, $basic_needle_data, $file_name, $error, $error_messages, + $needle_ref) +{ my $overall_list_of_tags = $basic_needle_data->{tags}; my $distri = $basic_needle_data->{distri}; my $version = $basic_needle_data->{version}; - my ($needle_info, $err) = $self->_basic_needle_info($needle_name, $distri, $version, $file_name, $needle_dir); + my ($needle_info, $err) + = $self->_basic_needle_info($needle_name, $distri, $version, $file_name, $needle_dir, $needle_ref); unless (defined $needle_info) { push(@$error_messages, "Could not parse needle $needle_name for $distri $version: $err"); return undef; @@ -467,6 +502,7 @@ sub viewimg ($self) { my $distri = $job->DISTRI; my $dversion = $job->VERSION || ''; my $needle_dir = $job->needle_dir; + my $needle_ref = $self->_create_tmpdir_for_needles_refspec($job); my $real_needle_dir = realpath($needle_dir) // $needle_dir; my $needles_rs = $self->app->schema->resultset('Needles'); @@ -502,7 +538,8 @@ sub viewimg ($self) { # load primary needle match my $primary_match; if (my $needle = $module_detail->{needle}) { - my ($needleinfo) = $self->_basic_needle_info($needle, $distri, $dversion, $module_detail->{json}, $needle_dir); + my ($needleinfo) + = $self->_basic_needle_info($needle, $distri, $dversion, $module_detail->{json}, $needle_dir, $needle_ref); if ($needleinfo) { my $info = { name => $needle, @@ -524,7 +561,8 @@ sub viewimg ($self) { if ($module_detail->{needles}) { for my $needle (@{$module_detail->{needles}}) { my $needlename = $needle->{name}; - my ($needleinfo) = $self->_basic_needle_info($needlename, $distri, $dversion, $needle->{json}, $needle_dir); + my ($needleinfo) + = $self->_basic_needle_info($needlename, $distri, $dversion, $needle->{json}, $needle_dir, $needle_ref); next unless $needleinfo; my $info = { name => $needlename, diff --git a/script/openqa-enqueue-needle-ref-cleanup b/script/openqa-enqueue-needle-ref-cleanup new file mode 100755 index 00000000000..bad166b2c6b --- /dev/null +++ b/script/openqa-enqueue-needle-ref-cleanup @@ -0,0 +1,2 @@ +#!/bin/sh -e +exec "$(dirname "$0")"/openqa eval -m production -V 'app->gru->enqueue(limit_temp_needle_refs => [], {priority => 5, ttl => 172800, limit => 1})' "$@" diff --git a/systemd/openqa-enqueue-needle-ref-cleanup.service b/systemd/openqa-enqueue-needle-ref-cleanup.service new file mode 100644 index 00000000000..153ff2b435e --- /dev/null +++ b/systemd/openqa-enqueue-needle-ref-cleanup.service @@ -0,0 +1,9 @@ +[Unit] +Description=Enqueues a needle ref cleanup task for openQA. +After=postgresql.service openqa-setup-db.service +Wants=openqa-setup-db.service + +[Service] +Type=oneshot +User=geekotest +ExecStart=/usr/share/openqa/script/openqa-enqueue-needle-ref-cleanup diff --git a/systemd/openqa-enqueue-needle-ref-cleanup.timer b/systemd/openqa-enqueue-needle-ref-cleanup.timer new file mode 100644 index 00000000000..468c68ec055 --- /dev/null +++ b/systemd/openqa-enqueue-needle-ref-cleanup.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Enqueues a needle refs task for openQA every hour. + +[Timer] +OnCalendar=hourly +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/t/14-grutasks.t b/t/14-grutasks.t index 840b147a36b..c76e2e3aea0 100644 --- a/t/14-grutasks.t +++ b/t/14-grutasks.t @@ -12,6 +12,7 @@ use OpenQA::Jobs::Constants; use OpenQA::JobDependencies::Constants; use OpenQA::JobGroupDefaults; use OpenQA::Schema::Result::Jobs; +use OpenQA::Needles; use File::Copy; require OpenQA::Test::Database; use OpenQA::Test::Utils qw(run_gru_job perform_minion_jobs); @@ -410,6 +411,39 @@ subtest 'limit_results_and_logs gru task cleans up logs' => sub { ok !-e $log_file_for_groupless_job, 'log file for groupless job got cleaned'; }; +subtest 'limit_temp_needle_refs task cleans up temp needle refs exceeding retention' => sub { + my $temp_dir = path(OpenQA::Needles::temp_dir); + is $temp_dir, 't/data/openqa/webui/cache/needle-refs', 'needle temp dir determined as expected'; + $temp_dir->child($_)->make_path for qw(ref1 ref2); + my @old_needle_files = ("$temp_dir/ref1/needle_old.png", "$temp_dir/ref1/needle_old.json"); + my @new_needle_files = ("$temp_dir/ref2/needle_new.png", "$temp_dir/ref2/needle_new.json"); + my $now = time; + my $old_timestamp = $now - (120 * ONE_MINUTE + 1); + my $new_timestamp = $now - ONE_MINUTE + 1; + foreach my $file (@old_needle_files) { + path($file)->touch; + utime $old_timestamp, $old_timestamp, $file; + } + foreach my $file (@new_needle_files) { + path($file)->touch; + utime $new_timestamp, $new_timestamp, $file; + } + + # enqueue and run cleanup + my $minion = $t->app->minion; + my $id = $minion->enqueue('limit_temp_needle_refs'); + ok defined $id, 'job enqueued'; + perform_minion_jobs $minion; + my $job_info = $minion->job($id)->info; + + subtest 'cleanup result' => sub { + is $job_info->{state}, 'finished', 'job finished'; + ok !-e $_, "old file '$_' cleaned up" for @old_needle_files; + ok !-e "$temp_dir/ref1", 'empty directory removed'; + ok -e $_, "new file '$_' preserved" for @new_needle_files; + } or diag explain $job_info; +}; + subtest 'limit audit events' => sub { my $app = $t->app; my $audit_events = $app->schema->resultset('AuditEvents'); diff --git a/t/config.t b/t/config.t index e12350db832..4e181b572a3 100644 --- a/t/config.t +++ b/t/config.t @@ -67,8 +67,10 @@ subtest 'Test configuration default modes' => sub { do_push => 'no', do_cleanup => 'no', git_auto_clone => 'yes', - git_auto_update => 'no', + git_auto_update => 'yes', git_auto_update_method => 'best-effort', + checkout_needles_sha => 'no', + temp_needle_refs_retention => 120, }, 'scheduler' => { max_job_scheduled_time => 7,