Use needles from correct ref of NEEDLES_DIR rebased version

etc/openqa/openqa.ini

@@ -115,12 +115,16 @@
 #do_push = no
 ## whether to clone CASEDIR or NEEDLES_DIR on the web UI if that var points to a Git repo
 #git_auto_clone = yes
-## enable automatic updates of all test code and needles managed via Git (still experimental, currently still breaks scheduling parallel clusters)
-#git_auto_update = no
+## enable automatic updates of all test code and needles managed via Git
+#git_auto_update = yes
 ## specifies how to handle errors on automatic updates via git_auto_update
 ## - when set to "best-effort" openQA jobs are started even if the update failed
 ## - when set to "strict" openQA jobs will be blocked until the update succeeded or set to incomplete when the retries for updating are exhausted
 #git_auto_update_method = best-effort
+# whether openQA should attempt to display needles of the correct version in the web UI
+#checkout_needles_sha = no
+# retention for storing temporary needle refs in minutes
+#temp_needle_refs_retention = 120
 
 ## Authentication method to use for user management
 [auth]

lib/OpenQA/Git.pm

@@ -6,6 +6,7 @@
 use Mojo::Base -base, -signatures;
 use Mojo::Util 'trim';
 use Cwd 'abs_path';
+use Mojo::File 'path';
 use OpenQA::Utils qw(run_cmd_with_log_return_error);
 
 has 'app';
@@ -157,4 +158,16 @@
     return $r->{status};
 }
 
+sub cache_ref ($self, $ref, $relative_path, $output_file) {
+    if (-f $output_file) {
+        eval { path($output_file)->touch };
+        return $@ ? $@ : undef;
+    }
+    my @git = $self->_prepare_git_command;
+    my $res = run_cmd_with_log_return_error [@git, 'show', "$ref:./$relative_path"], output_file => $output_file;
+    return undef if $res->{status};
+    unlink $output_file;
+    return _format_git_error($res, 'Unable to cache Git ref');
+}
+
 1;

lib/OpenQA/Needles.pm

@@ -0,0 +1,57 @@
+# Copyright SUSE LLC
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+package OpenQA::Needles;
+use Mojo::Base -strict, -signatures;
+
+use Exporter qw(import);
+use File::Basename;
+use File::Spec;
+use File::Spec::Functions qw(catdir);
+use OpenQA::Git;
+use OpenQA::Log qw(log_error);
+use OpenQA::Utils qw(prjdir sharedir);
+use Mojo::File qw(path);
+
+our @EXPORT = qw(temp_dir is_in_temp_dir needle_temp_dir locate_needle);
+
+my $tmp_dir = prjdir() . '/webui/cache/needle-refs';
+
+sub temp_dir () { $tmp_dir }
+
+sub is_in_temp_dir ($file_path) { index($file_path, $tmp_dir) == 0 }
+
+sub needle_temp_dir ($dir, $ref) { path($tmp_dir, basename(dirname($dir)), $ref, 'needles') }
+
+sub _locate_needle_for_ref ($relative_needle_path, $needles_dir, $needles_ref) {
+    return undef unless defined $needles_ref;
+
+    my $temp_needles_dir = needle_temp_dir($needles_dir, $needles_ref);
+    my $subdir = dirname($relative_needle_path);
+    path($temp_needles_dir, $subdir)->make_path if File::Spec->splitdir($relative_needle_path) > 1;
+
+    my $git = OpenQA::Git->new(dir => $needles_dir);
+    my $temp_json_path = "$temp_needles_dir/$relative_needle_path";
+    my $basename = basename($relative_needle_path, '.json');
+    my $relative_png_path = "$subdir/$basename.png";
+    my $temp_png_path = "$temp_needles_dir/$relative_png_path";
+    my $error = $git->cache_ref($needles_ref, $relative_needle_path, $temp_json_path)
+      // $git->cache_ref($needles_ref, $relative_png_path, $temp_png_path);
+    return $temp_json_path unless defined $error;
+    log_error "An error occurred when looking for ref '$needles_ref' of '$relative_needle_path': $error";
+    return undef;
+}
+
+sub locate_needle ($relative_needle_path, $needles_dir, $needles_ref = undef) {
+    my $location_for_ref = _locate_needle_for_ref($relative_needle_path, $needles_dir, $needles_ref);
+    return $location_for_ref if defined $location_for_ref;
+    my $absolute_filename = catdir($needles_dir, $relative_needle_path);
+    my $needle_exists = -f $absolute_filename;
+    if (!$needle_exists) {
+        $absolute_filename = catdir(sharedir(), $relative_needle_path);
+        $needle_exists = -f $absolute_filename;
+    }
+    return $absolute_filename if $needle_exists;
+    log_error "Needle file $relative_needle_path not found within $needles_dir.";
+    return undef;
+}

lib/OpenQA/Schema/Result/Needles.pm

@@ -16,7 +16,7 @@ use OpenQA::App;
 use OpenQA::Git;
 use OpenQA::Jobs::Constants;
 use OpenQA::Schema::Result::Jobs;
-use OpenQA::Utils qw(locate_needle);
+use OpenQA::Needles qw(locate_needle);
 
 __PACKAGE__->table('needles');
 __PACKAGE__->load_components(qw(InflateColumn::DateTime Timestamps));

lib/OpenQA/Setup.pm

@@ -69,8 +69,10 @@ sub read_config ($app) {
             do_push => 'no',
             do_cleanup => 'no',
             git_auto_clone => 'yes',
-            git_auto_update => 'no',
+            git_auto_update => 'yes',
             git_auto_update_method => 'best-effort',
+            checkout_needles_sha => 'no',
+            temp_needle_refs_retention => 120,
         },
         scheduler => {
             max_job_scheduled_time => 7,

lib/OpenQA/Shared/Plugin/Gru.pm

@@ -31,7 +31,10 @@ sub register_tasks ($self) {
       OpenQA::Task::Asset::Download
       OpenQA::Task::Asset::Limit
       OpenQA::Task::Git::Clone
-      OpenQA::Task::Needle::Scan OpenQA::Task::Needle::Save OpenQA::Task::Needle::Delete
+      OpenQA::Task::Needle::Scan
+      OpenQA::Task::Needle::Save
+      OpenQA::Task::Needle::Delete
+      OpenQA::Task::Needle::LimitTempRefs
       OpenQA::Task::Job::Limit
       OpenQA::Task::Job::ArchiveResults
       OpenQA::Task::Job::FinalizeResults

lib/OpenQA/Task/Needle/LimitTempRefs.pm

@@ -0,0 +1,39 @@
+# Copyright SUSE LLC
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+package OpenQA::Task::Needle::LimitTempRefs;
+use Mojo::Base 'Mojolicious::Plugin', -signatures;
+
+use File::Find;
+use File::stat;
+use Fcntl qw(S_ISDIR);
+use OpenQA::Needles;
+use OpenQA::Task::SignalGuard;
+use Time::Seconds;
+
+my $retention;
+
+sub register ($self, $app, $job) {
+    $retention = $app->config->{'scm git'}->{temp_needle_refs_retention} * ONE_MINUTE;
+    $app->minion->add_task(limit_temp_needle_refs => sub ($job) { _limit($app, $job) });
+}
+
+sub _limit ($app, $job) {
+    my $ensure_task_retry_on_termination_signal_guard = OpenQA::Task::SignalGuard->new($job);
+
+    return $job->finish({error => 'Another job to remove needle versions is running. Try again later.'})
+      unless my $guard = $app->minion->guard('limit_needle_versions_task', 7200);
+
+    # remove all temporary needles which haven't been accessed in time period specified in config
+    my $temp_dir = OpenQA::Needles::temp_dir;
+    return undef unless -d $temp_dir;
+    my $now = time;
+    my $wanted = sub {
+        return undef unless my $lstat = lstat $File::Find::name;
+        return rmdir $File::Find::name if S_ISDIR($lstat->mode);    # remove all empty dirs
+        return unlink $File::Find::name if ($now - $lstat->mtime) > $retention;
+    };
+    find({no_chdir => 1, bydepth => 1, wanted => $wanted}, $temp_dir);
+}
+
+1;

lib/OpenQA/Utils.pm

@@ -26,6 +26,7 @@ use OpenQA::Log qw(log_info log_debug log_warning log_error);
 use Config::Tiny;
 use Time::HiRes qw(tv_interval);
 use File::Basename;
+use File::Path qw(make_path);
 use File::Spec;
 use File::Spec::Functions qw(catfile catdir);
 use Fcntl;
@@ -92,7 +93,6 @@ our @EXPORT = qw(
   BUGREF_REGEX
   LABEL_REGEX
   FLAG_REGEX
-  locate_needle
   needledir
   productdir
   testcasedir
@@ -254,22 +254,6 @@ sub is_in_tests {
 
 sub needledir { productdir(@_) . '/needles' }
 
-sub locate_needle {
-    my ($relative_needle_path, $needles_dir) = @_;
-
-    my $absolute_filename = catdir($needles_dir, $relative_needle_path);
-    my $needle_exists = -f $absolute_filename;
-
-    if (!$needle_exists) {
-        $absolute_filename = catdir(sharedir(), $relative_needle_path);
-        $needle_exists = -f $absolute_filename;
-    }
-    return $absolute_filename if $needle_exists;
-
-    log_error("Needle file $relative_needle_path not found within $needles_dir.");
-    return undef;
-}
-
 # Adds a timestamp to a string (eg. needle name) or replace the already present timestamp
 sub ensure_timestamp_appended {
     my ($str) = @_;
@@ -333,10 +317,12 @@ sub run_cmd_with_log {
 sub run_cmd_with_log_return_error ($cmd, %args) {
     my $stdout_level = $args{stdout} // 'debug';
     my $stderr_level = $args{stderr} // 'debug';
+    my $output_file = $args{output_file};
     log_info('Running cmd: ' . join(' ', @$cmd));
     try {
         my ($stdin, $stdout_err, $stdout, $stderr) = ('') x 4;
-        my $ipc_run_succeeded = IPC::Run::run($cmd, \$stdin, \$stdout, \$stderr);
+        my @out_args = defined $output_file ? ('>', $output_file, '2>', \$stderr) : (\$stdout, \$stderr);
+        my $ipc_run_succeeded = IPC::Run::run($cmd, \$stdin, @out_args);
         my $return_code = $?;
         chomp $stderr;
         if ($ipc_run_succeeded) {

lib/OpenQA/WebAPI/Controller/File.pm

@@ -6,6 +6,7 @@
 
 BEGIN { $ENV{MAGICK_THREAD_LIMIT} = 1; }
 
+use OpenQA::Needles;
 use OpenQA::Utils qw(:DEFAULT prjdir assetdir imagesdir);
 use File::Basename;
 use File::Spec;
@@ -29,11 +30,22 @@
 
     # make sure the directory of the file parameter is a real subdir of testcasedir before
     # using it to find needle subdirectory, to prevent access outside of the zoo
-    if ($jsonfile && !is_in_tests($jsonfile)) {
+    if ($jsonfile && !is_in_tests($jsonfile) && !OpenQA::Needles::is_in_temp_dir($jsonfile)) {
         my $prjdir = prjdir();
         warn "$jsonfile is not in a subdir of $prjdir/share/tests or $prjdir/tests";
         return $self->render(text => 'Forbidden', status => 403);
     }
+    # If the json file in not in the tests we may be using a temporary
+    # directory for needles from a different git SHA
+    my $jsonfile_in_temp_dir = $jsonfile && OpenQA::Needles::is_in_temp_dir($jsonfile);
+    if ($jsonfile_in_temp_dir) {
+        $needledir = dirname($jsonfile);
+        # In case we're in a subdirectory, keep taking the dirname until we
+        # have the path of the `needles` directory
+        while (basename($needledir) ne 'needles') {
+            $needledir = dirname($needledir);
+        }
+    }
     # Reject directory traversal breakouts here...
     if (index($jsonfile, '..') != -1) {
         warn "jsonfile value $jsonfile is invalid, cannot contain ..";