diff --git a/lib/xfstests_utils.pm b/lib/xfstests_utils.pm new file mode 100644 index 000000000000..b76c7d9bc80c --- /dev/null +++ b/lib/xfstests_utils.pm @@ -0,0 +1,638 @@ +# SUSE's openQA tests +# +# Copyright 2024 SUSE LLC +# SPDX-License-Identifier: FSFAP +# +# Summary: Some functions to run test for xfstests +# Maintainer: Yong Sun +package xfstests_utils; + +use base Exporter; +use Exporter; +use 5.018; +use strict; +use warnings; +use utils; +use testapi; +use base 'opensusebasetest'; +use File::Basename; +use testapi; +use utils; +use Utils::Backends 'is_pvm'; +use serial_terminal 'select_serial_terminal'; +use power_action_utils qw(power_action prepare_system_shutdown); +use filesystem_utils qw(format_partition generate_xfstests_list); +use lockapi; +use mmapi; +use version_utils 'is_public_cloud'; +use LTP::utils; +use LTP::WhiteList; + +# Heartbeat variables, need to sync with tests/xfstests/run.pm +my $HB_PATN = ''; #shorter label to getting stable under heavy stress +my $HB_DONE = ''; #shorter label to getting stable under heavy stress +my $HB_DONE_FILE = '/opt/test.done'; +my $HB_EXIT_FILE = '/opt/test.exit'; +my $HB_SCRIPT = '/opt/heartbeat.sh'; + +# None heartbeat variables +my ($test_status, $test_start, $test_duration); + +# General variables, need to sync with tests/xfstests/run.pm +my $TEST_WRAPPER = '/opt/wrapper.sh'; +my $STATUS_LOG = '/opt/status.log'; +my $INST_DIR = '/opt/xfstests'; +my $LOG_DIR = '/opt/log'; +my $KDUMP_DIR = '/opt/kdump'; +my $TEST_FOLDER = '/opt/test'; +my $SCRATCH_FOLDER = '/opt/scratch'; + +our @EXPORT = qw( + heartbeat_prepare + heartbeat_start + heartbeat_stop + heartbeat_wait + test_wait + test_name + log_add + tests_from_category + exclude_grouplist + include_grouplist + tests_from_ranges + test_run + save_kdump + shuffle + copy_log + copy_fsxops + dump_btrfs_img + raw_dump + collect_fs_status + copy_all_log + reload_loop_device + umount_xfstests_dev + config_debug_option + test_run_without_heartb); + +=head2 heartbeat_prepare + +Create heartbeat script and directories (Call it only once) + +=cut + +sub heartbeat_prepare { + my $hb_intvl = shift; + my $redir = " >> /dev/$serialdev"; + my $script = < $HB_SCRIPT <<'END'\n$script\nEND\n( exit \$?)"); +} + +=head2 heartbeat_start + +Start heartbeat, setup environment variables (Call it everytime SUT reboots) + +=cut + +sub heartbeat_start { + enter_cmd(". ~/.xfstests; nohup sh $HB_SCRIPT &"); +} + +=head2 heartbeat_stop + +Stop heartbeat + +=cut + +sub heartbeat_stop { + check_var('VIRTIO_CONSOLE', '1') ? type_string "\n" : send_key 'ret'; + assert_script_run("touch $HB_EXIT_FILE"); +} + +=head2 heartbeat_wait + +Wait for heartbeat + +=cut + +sub heartbeat_wait { + # When under heavy load, the SUT might be unable to send + # heartbeat messages to serial console. That's why HB_TIMEOUT + # is set to 200 by default: waiting for such tests to finish. + my $hb_timeout = shift; + my $ret = wait_serial([$HB_PATN, $HB_DONE], $hb_timeout); + if ($ret) { + if ($ret =~ /$HB_PATN/) { + return ($HB_PATN, ''); + } + else { + my $status; + check_var('VIRTIO_CONSOLE', '1') ? type_string "\n" : send_key 'ret'; + my $ret = script_output("cat $HB_DONE_FILE; rm -f $HB_DONE_FILE"); + $ret =~ s/^\s+|\s+$//g; + if ($ret == 0) { + $status = 'PASSED'; + } + elsif ($ret == 22) { + $status = 'SKIPPED'; + } + else { + $status = 'FAILED'; + } + return ($HB_DONE, $status); + } + } + return ('', 'FAILED'); +} + +=head2 test_wait + +Wait for test to finish + +=cut + +sub test_wait { + my ($subtest_timeout, $hb_timeout) = @_; + my $begin = time(); + my ($type, $status) = heartbeat_wait($hb_timeout); + my $delta = time() - $begin; + # In case under heavy stress, only match first 2 words in label is enough + my $hb_label = substr($HB_PATN, 0, 2); + while ($type =~ /$hb_label/ and $delta < $subtest_timeout) { + ($type, $status) = heartbeat_wait($hb_timeout); + $delta = time() - $begin; + } + if ($type eq $HB_PATN) { + return ('', 'FAILED', $delta); + } + return ($type, $status, $delta); +} + +=head2 test_name + +Format the name of a subtest, because we don't want / to be seen like directory (e.g. xfs-005) +test - specific test (e.g. xfs/005) + +=cut + +sub test_name { + my $test = shift; + return $test =~ s/\//-/gr; +} + +=head2 log_add + +Add one test result to log file +file - log file +test - specific test (e.g. xfs/008) +status - test status +time - time consumed + +=cut + +sub log_add { + my ($file, $test, $status, $time) = @_; + my $name = test_name($test); + unless ($name and $status) { return; } + my $cmd = "echo '$name ... ... $status (${time}s)' | tee -a $file"; + my $ret = script_output($cmd); + return $ret; +} + +=head2 tests_from_category + +Return all the tests of a specific xfstests category +category - xfstests category (e.g. generic) +dir - xfstests installation dir (e.g. /opt/xfstests) + +=cut + +sub tests_from_category { + my ($category, $dir) = @_; + my $cmd = "find '$dir/tests/$category' -regex '.*/[0-9]+'"; + my $output = script_output($cmd, 60); + my @tests = split(/\n/, $output); + foreach my $test (@tests) { + $test = basename($test); + } + return @tests; +} + +=head2 exclude_grouplist + +Return matched exclude tests from groups in XFSTESTS_GROUPLIST +return structure - hash +Group name start with ! will exclude in test, and expected to use to update blacklist +If TEST_RANGES contain generic tests, then exclude tests from generic folder, else will exclude tests from filesystem type folder + +=cut + +sub exclude_grouplist { + my ($test_ranges, $fstype) = @_; + my %tests_list = (); + return unless get_var('XFSTESTS_GROUPLIST'); + my $test_folder = $test_ranges =~ /generic/ ? "generic" : $fstype; + my @group_list = split(/,/, get_var('XFSTESTS_GROUPLIST')); + foreach my $group_name (@group_list) { + next if ($group_name !~ /^\!/); + $group_name = substr($group_name, 1); + my $cmd = "awk '/$group_name/' $INST_DIR/tests/$test_folder/group.list | awk '{printf \"$test_folder/\"}{printf \$1}{printf \",\"}' > tmp.group"; + script_run($cmd); + $cmd = "awk '/$group_name/' $INST_DIR/tests/$fstype/group.list | awk '{printf \"$fstype/\"}{printf \$1}{printf \",\"}' >> tmp.group"; + script_run($cmd) if ($test_folder eq "generic" and $test_ranges =~ /$fstype/); + $cmd = "cat tmp.group"; + my %tmp_list = map { $_ => 1 } split(/,/, substr(script_output($cmd), 0, -1)); + %tests_list = (%tests_list, %tmp_list); + } + return %tests_list; +} + +=head2 include_grouplist + +Return matched include tests from groups in XFSTESTS_GROUPLIST +return structure - array +Group name start without ! will include in test, and expected to use to update test ranges +If TEST_RANGES contain generic tests, then include tests from generic folder, else will include tests from filesystem type folder + +=cut + +sub include_grouplist { + my ($test_ranges, $grouplist, $fstype) = _ @; + my @tests_list; + return unless $grouplist; + my $test_folder = $test_ranges =~ /generic/ ? "generic" : $fstype; + my @group_list = split(/,/, $grouplist); + foreach my $group_name (@group_list) { + next if ($group_name =~ /^\!/); + my $cmd = "awk '/$group_name/' $INST_DIR/tests/$test_folder/group.list | awk '{printf \"$test_folder/\"}{printf \$1}{printf \",\"}' > tmp.group"; + script_run($cmd); + $cmd = "awk '/$group_name/' $INST_DIR/tests/$fstype/group.list | awk '{printf \"$fstype/\"}{printf \$1}{printf \",\"}' >> tmp.group"; + script_run($cmd) if ($test_folder eq "generic" and $test_ranges =~ /$fstype/); + $cmd = "cat tmp.group"; + my $tests = substr(script_output($cmd), 0, -1); + foreach my $single_test (split(/,/, $tests)) { + push(@tests_list, $single_test); + } + } + return @tests_list; +} + +=head2 tests_from_ranges + +Return a list of tests to run from given test ranges +ranges - test ranges (e.g. xfs/001-100,btrfs/100-159) +dir - xfstests installation dir (e.g. /opt/xfstests) + +=cut + +sub tests_from_ranges { + my ($ranges, $dir) = @_; + if ($ranges !~ /\w+(\/\d+-\d+)?(,\w+(\/\d+-\d+)?)*/) { + die "Invalid test ranges: $ranges"; + } + my %cache; + my @tests; + foreach my $range (split(/,/, $ranges)) { + my ($min, $max) = (0, 99999); + my ($category, $min_max) = split(/\//, $range); + unless (defined($min_max)) { + next; + } + if ($min_max =~ /\d+-\d+/) { + ($min, $max) = split(/-/, $min_max); + } + else { + $min = $max = $min_max; + } + unless (exists($cache{$category})) { + $cache{$category} = [tests_from_category($category, $dir)]; + assert_script_run("mkdir -p $LOG_DIR/$category"); + } + foreach my $num (@{$cache{$category}}) { + if ($num >= $min and $num <= $max) { + push(@tests, "$category/$num"); + } + } + } + return @tests; +} + +=head2 test_run + +# Run a single test and write log to file +# test - test to run (e.g. xfs/001) + +=cut + +sub test_run { + my $test = shift; + my ($category, $num) = split(/\//, $test); + my $run_options = ''; + if (check_var('XFSTESTS', 'nfs')) { + $run_options = '-nfs'; + } + elsif (check_var('XFSTESTS', 'overlay')) { + $run_options = '-overlay'; + } + my $inject_code = get_var('INJECT_INFO', ''); + my $cmd = "\n$TEST_WRAPPER '$test' $run_options $inject_code | tee $LOG_DIR/$category/$num; "; + $cmd .= "echo \${PIPESTATUS[0]} > $HB_DONE_FILE\n"; + type_string($cmd); +} + +=head2 save_kdump + +Save kdump data for further uploading +test - corresponding test(e.g. xfs/009) +dir - Save kdump data to this dir +vmcore - include vmcore file +kernel - include kernel + +=cut + +sub save_kdump { + my ($test, $dir, %args) = @_; + $args{vmcore} ||= 0; + $args{kernel} ||= 0; + $args{debug} ||= 0; + my $name = test_name($test); + my $ret = script_run("mv /var/crash/* $dir/$name"); + if ($args{debug}) { + $ret += script_run("if [ -e /usr/lib/debug/boot ]; then tar zcvf $dir/$name/vmcore-debug.tar.gz --absolute-names /usr/lib/debug/boot; fi"); + } + return 0 if $ret != 0; + + my $removed = ""; + unless ($args{vmcore}) { + $removed .= " $dir/$name/vmcore"; + } + unless ($args{kernel}) { + $removed .= " $dir/$name/*.{gz,bz2,xz}"; + } + if ($removed) { + assert_script_run("rm -f $removed"); + } + return 1; +} + +=head2 shuffle + +mess up the order of test queue + +=cut + +sub shuffle { + my @arr = @_; + srand(time()); + for (my $i = $#arr; $i > 0; $i--) { + my $j = int(rand($i + 1)); + ($arr[$i], $arr[$j]) = ($arr[$j], $arr[$i]); + } + return @arr; +} + +=head2 copy_log + +Log & Must included: Copy log to ready to save + +=cut + +sub copy_log { + my ($category, $num, $log_type) = @_; + my $cmd = "if [ -e /opt/xfstests/results/$category/$num.$log_type ]; then cat /opt/xfstests/results/$category/$num.$log_type | tee $LOG_DIR/$category/$num.$log_type; fi"; + script_run($cmd); +} + +=head2 copy_fsxops + +Log: Copy junk.fsxops for fails fsx tests included in subtests + +=cut + +sub copy_fsxops { + my ($category, $num) = @_; + my $cmd = "if [ -e $TEST_FOLDER/junk.fsxops ]; then cp $TEST_FOLDER/junk.fsxops $LOG_DIR/$category/$num.junk.fsxops; fi"; + script_run($cmd); +} + +=head2 dump_btrfs_img + +Log: Only run in test Btrfs, collect image dump for inconsistent error + +=cut + +sub dump_btrfs_img { + my ($category, $num) = @_; + my $cmd = "echo \"no inconsistent error, skip btrfs image dump\""; + my $ret = script_output("grep -E -m 1 \"filesystem on .+ is inconsistent\" $LOG_DIR/$category/$num"); + if ($ret =~ /filesystem on (.+) is inconsistent/) { $cmd = "umount $1;btrfs-image $1 $LOG_DIR/$category/$num.img"; } + script_run($cmd); +} + +=head2 raw_dump + +Log: Raw dump from SCRATCH_DEV via dd + +=cut + +sub raw_dump { + my ($category, $num) = @_; + my $dev = get_var('XFSTESTS_SCRATCH_DEV') ? get_var('XFSTESTS_SCRATCH_DEV') : (split(/ /, get_var("XFSTESTS_SCRATCH_DEV_POOL")))[0]; + assert_script_run("umount $dev;dd if=$dev of=$LOG_DIR/$category/$num.raw bs=512 count=1000"); +} + +=head2 collect_fs_status + +Log: Collect fs runtime status for XFS, Btrfs and Ext4 + +=cut + +sub collect_fs_status { + my ($category, $num, $fstype) = @_; + my $cmd = < /dev/null +[ -n "\$SCRATCH_DEV" ] && mount \$SCRATCH_DEV $SCRATCH_FOLDER &> /dev/null +END_CMD + if ($fstype eq 'xfs') { + $cmd = < /sys/fs/$fstype/stats/stats <==" > $LOG_DIR/$category/$num.fs_stat +cat /sys/fs/$fstype/stats/stats >> $LOG_DIR/$category/$num.fs_stat +tail -n +1 /sys/fs/$fstype/*/log/* >> $LOG_DIR/$category/$num.fs_stat +tail -n +1 /sys/fs/$fstype/*/stats/stats >> $LOG_DIR/$category/$num.fs_stat +xfs_info $TEST_FOLDER > $LOG_DIR/$category/$num.xfsinfo +xfs_info $SCRATCH_FOLDER >> $LOG_DIR/$category/$num.xfsinfo +END_CMD + } + elsif ($fstype eq 'btrfs') { + $cmd = <> $LOG_DIR/$category/$num.fs_stat +tail -n +1 /sys/fs/$fstype/*/allocation/metadata/[bdft]* >> $LOG_DIR/$category/$num.fs_stat +tail -n +1 /sys/fs/$fstype/*/allocation/metadata/dup/* >> $LOG_DIR/$category/$num.fs_stat +tail -n +1 /sys/fs/$fstype/*/allocation/*/single/* >> $LOG_DIR/$category/$num.fs_stat +END_CMD + } + elsif ($fstype eq 'ext4') { + $cmd = <> $LOG_DIR/$category/$num.fs_stat +END_CMD + } + $cmd = < /dev/null +[ -n "\$SCRATCH_DEV" ] && umount \$SCRATCH_DEV &> /dev/null +END_CMD + enter_cmd("$cmd"); +} + +=head2 copy_all_log + +Add all above logs + +=cut + +sub copy_all_log { + my ($category, $num, $fstype) = @_; + copy_log($category, $num, 'out.bad'); + copy_log($category, $num, 'full'); + copy_log($category, $num, 'dmesg'); + copy_fsxops($category, $num); + collect_fs_status($category, $num, $fstype); + if (get_var('BTRFS_DUMP', 0) && (check_var 'XFSTESTS', 'btrfs')) { dump_btrfs_img($category, $num); } + if (get_var('RAW_DUMP', 0)) { raw_dump($category, $num); } +} + +=head2 reload_loop_device + +Reload loop device for xfstests + +=cut + +sub reload_loop_device { + my ($self, $fstype) = @_; + assert_script_run("losetup -fP $INST_DIR/test_dev"); + my $scratch_amount = script_output("ls $INST_DIR/scratch_dev* | wc -l"); + my $scratch_num = 1; + while ($scratch_amount >= $scratch_num) { + assert_script_run("losetup -fP $INST_DIR/scratch_dev$scratch_num", 300); + $scratch_num += 1; + } + script_run('losetup -a'); + format_partition("$INST_DIR/test_dev", $fstype); +} + +=head2 umount_xfstests_dev + +Umount TEST_DEV and SCRATCH_DEV + +=cut + +sub umount_xfstests_dev { + script_run('umount ' . get_var('XFSTESTS_TEST_DEV') . ' &> /dev/null') if get_var('XFSTESTS_TEST_DEV'); + script_run('umount ' . get_var('XFSTESTS_SCRATCH_DEV') . ' &> /dev/null') if get_var('XFSTESTS_SCRATCH_DEV'); + if (get_var('XFSTESTS_SCRATCH_DEV_POOL')) { + script_run("umount $_ &> /dev/null") foreach (split ' ', get_var('XFSTESTS_SCRATCH_DEV_POOL')); + } +} + +=head2 config_debug_option + +Enable softlockup panic collection and could manually enable other setting by XFSTESTS_DEBUG + +=cut + +sub config_debug_option { + script_run('echo 1 > /proc/sys/kernel/softlockup_all_cpu_backtrace'); # on detection capture more debug information + script_run('echo 1 > /proc/sys/kernel/softlockup_panic'); # panic when softlockup + if (get_var('XFSTESTS_DEBUG')) { + # e.g. XFSTESTS_DEBUG could be one or more parameter in following + # [hardlockup_panic hung_task_panic panic_on_io_nmi panic_on_oops panic_on_rcu_stall...] + script_run("echo 1 > /proc/sys/kernel/$_ ") foreach (split ' ', get_var('XFSTESTS_DEBUG')); + } +} + +=head2 test_run_without_heartbeat + +Run a single test and write log to file but without heartbeat, return log_add output + +=cut + +sub test_run_without_heartbeat { + my ($self, $test, $timeout, $fstype) = @_; + my ($category, $num) = split(/\//, $test); + my $run_options = ''; + my $status_num = 1; + if (check_var('XFSTESTS', 'nfs')) { + $run_options = '-nfs'; + } + elsif (check_var('XFSTESTS', 'overlay')) { + $run_options = '-overlay'; + } + my $inject_code = get_var('INJECT_INFO', ''); + eval { + $test_start = time(); + # Send kill signal 3 seconds after sending the default SIGTERM to avoid some tests refuse to stop after timeout + assert_script_run("timeout -k 3 " . ($timeout - 5) . " $TEST_WRAPPER '$test' $run_options $inject_code | tee $LOG_DIR/$category/$num; echo \${PIPESTATUS[0]} > $LOG_DIR/subtest_result_num", $timeout); + $status_num = script_output("tail -n 1 $LOG_DIR/subtest_result_num"); + $test_duration = time() - $test_start; + }; + if ($@) { + $test_status = 'FAILED'; + $test_duration = time() - $test_start; + sleep(2); + copy_all_log($category, $num, $fstype); + + prepare_system_shutdown; + check_var('VIRTIO_CONSOLE', '1') ? power('reset') : send_key 'alt-sysrq-b'; + reconnect_mgmt_console if is_pvm; + $self->wait_boot; + + sleep 1; + select_console('root-console'); + # Save kdump data to KDUMP_DIR if not set "NO_KDUMP=1" + unless (check_var('NO_KDUMP', '1')) { + unless (save_kdump($test, $KDUMP_DIR, vmcore => 1, kernel => 1, debug => 1)) { + # If no kdump data found, write warning to log + my $msg = "Warning: $test crashed SUT but has no kdump data"; + script_run("echo '$msg' >> $LOG_DIR/$category/$num"); + } + } + + # Reload loop device after a reboot + reload_loop_device($self, $fstype) if get_var('XFSTESTS_LOOP_DEVICE'); + } + else { + $status_num =~ s/^\s+|\s+$//g; + if ($status_num == 0) { + $test_status = 'PASSED'; + } + elsif ($status_num == 22) { + $test_status = 'SKIPPED'; + } + else { + $test_status = 'FAILED'; + copy_all_log($category, $num, $fstype); + } + } + # Add test status to STATUS_LOG file + return log_add($STATUS_LOG, $test, $test_status, $test_duration); +} + +1; diff --git a/tests/xfstests/run.pm b/tests/xfstests/run.pm index 11febc1f1389..e104bd032f2c 100644 --- a/tests/xfstests/run.pm +++ b/tests/xfstests/run.pm @@ -33,17 +33,10 @@ use mmapi; use version_utils 'is_public_cloud'; use LTP::utils; use LTP::WhiteList; +use xfstests_utils; -# Heartbeat variables -my $HB_INTVL = get_var('XFSTESTS_HEARTBEAT_INTERVAL') || 30; -my $HB_TIMEOUT = get_var('XFSTESTS_HEARTBEAT_TIMEOUT') || 200; -my $HB_PATN = ''; #shorter label to getting stable under heavy stress -my $HB_DONE = ''; #shorter label to getting stable under heavy stress -my $HB_DONE_FILE = '/opt/test.done'; -my $HB_EXIT_FILE = '/opt/test.exit'; -my $HB_SCRIPT = '/opt/heartbeat.sh'; -# xfstests variables +# xfstests common variables # - XFSTESTS_RANGES: Set sub tests ranges. e.g. XFSTESTS_RANGES=xfs/100-199 or XFSTESTS_RANGES=generic/010,generic/019,generic/038 # - XFSTESTS_BLACKLIST: Set sub tests not run in XFSTESTS_RANGES. e.g. XFSTESTS_BLACKLIST=generic/010,generic/019,generic/038 # - XFSTESTS_GROUPLIST: Include/Exclude tests in group(a classification by upstream). e.g. XFSTESTS_GROUPLIST='auto,!dangerous_online_repair' @@ -52,456 +45,26 @@ my $HB_SCRIPT = '/opt/heartbeat.sh'; my $TEST_RANGES = get_required_var('XFSTESTS_RANGES'); my $TEST_WRAPPER = '/opt/wrapper.sh'; my $BLACKLIST = get_var('XFSTESTS_BLACKLIST'); +my $GROUPLIST = get_var('XFSTESTS_GROUPLIST'); my $STATUS_LOG = '/opt/status.log'; my $INST_DIR = '/opt/xfstests'; my $LOG_DIR = '/opt/log'; my $KDUMP_DIR = '/opt/kdump'; -my $MAX_TIME = get_var('XFSTESTS_SUBTEST_MAXTIME') || 2400; +my $SUBTEST_MAX_TIME = get_var('XFSTESTS_SUBTEST_MAXTIME') || 2400; my $FSTYPE = get_required_var('XFSTESTS'); my $TEST_SUITE = get_var('TEST'); -# Variables use for no heartbeat mode -my $TIMEOUT_NO_HEARTBEAT = get_var('XFSTESTS_TIMEOUT', 2000); -my ($test_status, $test_start, $test_duration); - -my $TEST_FOLDER = '/opt/test'; -my $SCRATCH_FOLDER = '/opt/scratch'; - -# Create heartbeat script, directories(Call it only once) -sub heartbeat_prepare { - my $redir = " >> /dev/$serialdev"; - my $script = < $HB_SCRIPT <<'END'\n$script\nEND\n( exit \$?)"); -} - -# Start heartbeat, setup environment variables(Call it everytime SUT reboots) -sub heartbeat_start { - enter_cmd(". ~/.xfstests; nohup sh $HB_SCRIPT &"); -} - -# Stop heartbeat -sub heartbeat_stop { - check_var('VIRTIO_CONSOLE', '1') ? type_string "\n" : send_key 'ret'; - assert_script_run("touch $HB_EXIT_FILE"); -} - -# Wait for heartbeat -sub heartbeat_wait { - # When under heavy load, the SUT might be unable to send - # heartbeat messages to serial console. That's why HB_TIMEOUT - # is set to 200 by default: waiting for such tests to finish. - my $ret = wait_serial([$HB_PATN, $HB_DONE], $HB_TIMEOUT); - if ($ret) { - if ($ret =~ /$HB_PATN/) { - return ($HB_PATN, ''); - } - else { - my $status; - check_var('VIRTIO_CONSOLE', '1') ? type_string "\n" : send_key 'ret'; - my $ret = script_output("cat $HB_DONE_FILE; rm -f $HB_DONE_FILE"); - $ret =~ s/^\s+|\s+$//g; - if ($ret == 0) { - $status = 'PASSED'; - } - elsif ($ret == 22) { - $status = 'SKIPPED'; - } - else { - $status = 'FAILED'; - } - return ($HB_DONE, $status); - } - } - return ('', 'FAILED'); -} - -# Wait for test to finish -sub test_wait { - my $timeout = shift; - my $begin = time(); - my ($type, $status) = heartbeat_wait; - my $delta = time() - $begin; - # In case under heavy stress, only match first 2 words in label is enough - my $hb_label = substr($HB_PATN, 0, 2); - while ($type =~ /$hb_label/ and $delta < $timeout) { - ($type, $status) = heartbeat_wait; - $delta = time() - $begin; - } - if ($type eq $HB_PATN) { - return ('', 'FAILED', $delta); - } - return ($type, $status, $delta); -} - -# Return the name of a test(e.g. xfs-005) -# test - specific test(e.g. xfs/005) -sub test_name { - my $test = shift; - return $test =~ s/\//-/gr; -} - -# Add one test result to log file -# file - log file -# test - specific test(e.g. xfs/008) -# status - test status -# time - time consumed -sub log_add { - my ($file, $test, $status, $time) = @_; - my $name = test_name($test); - unless ($name and $status) { return; } - my $cmd = "echo '$name ... ... $status (${time}s)' | tee -a $file"; - my $ret = script_output($cmd); - return $ret; -} - -# Return all the tests of a specific xfstests category -# category - xfstests category(e.g. generic) -# dir - xfstests installation dir(e.g. /opt/xfstests) -sub tests_from_category { - my ($category, $dir) = @_; - my $cmd = "find '$dir/tests/$category' -regex '.*/[0-9]+'"; - my $output = script_output($cmd, 60); - my @tests = split(/\n/, $output); - foreach my $test (@tests) { - $test = basename($test); - } - return @tests; -} - -# Return matched exclude tests from groups in XFSTESTS_GROUPLIST -# return structure - hash -# Group name start with ! will exclude in test, and expected to use to update blacklist -# If TEST_RANGES contain generic tests, then exclude tests from generic folder, else will exclude tests from filesystem type folder -sub exclude_grouplist { - my %tests_list = (); - return unless get_var('XFSTESTS_GROUPLIST'); - my $test_folder = $TEST_RANGES =~ /generic/ ? "generic" : $FSTYPE; - my @group_list = split(/,/, get_var('XFSTESTS_GROUPLIST')); - foreach my $group_name (@group_list) { - next if ($group_name !~ /^\!/); - $group_name = substr($group_name, 1); - my $cmd = "awk '/$group_name/' $INST_DIR/tests/$test_folder/group.list | awk '{printf \"$test_folder/\"}{printf \$1}{printf \",\"}' > tmp.group"; - script_run($cmd); - $cmd = "awk '/$group_name/' $INST_DIR/tests/$FSTYPE/group.list | awk '{printf \"$FSTYPE/\"}{printf \$1}{printf \",\"}' >> tmp.group"; - script_run($cmd) if ($test_folder eq "generic" and $TEST_RANGES =~ /$FSTYPE/); - $cmd = "cat tmp.group"; - my %tmp_list = map { $_ => 1 } split(/,/, substr(script_output($cmd), 0, -1)); - %tests_list = (%tests_list, %tmp_list); - } - return %tests_list; -} - -# Return matched include tests from groups in XFSTESTS_GROUPLIST -# return structure - array -# Group name start without ! will include in test, and expected to use to update test ranges -# If TEST_RANGES contain generic tests, then include tests from generic folder, else will include tests from filesystem type folder -sub include_grouplist { - my @tests_list; - return unless get_var('XFSTESTS_GROUPLIST'); - my $test_folder = $TEST_RANGES =~ /generic/ ? "generic" : $FSTYPE; - my @group_list = split(/,/, get_var('XFSTESTS_GROUPLIST')); - foreach my $group_name (@group_list) { - next if ($group_name =~ /^\!/); - my $cmd = "awk '/$group_name/' $INST_DIR/tests/$test_folder/group.list | awk '{printf \"$test_folder/\"}{printf \$1}{printf \",\"}' > tmp.group"; - script_run($cmd); - $cmd = "awk '/$group_name/' $INST_DIR/tests/$FSTYPE/group.list | awk '{printf \"$FSTYPE/\"}{printf \$1}{printf \",\"}' >> tmp.group"; - script_run($cmd) if ($test_folder eq "generic" and $TEST_RANGES =~ /$FSTYPE/); - $cmd = "cat tmp.group"; - my $tests = substr(script_output($cmd), 0, -1); - foreach my $single_test (split(/,/, $tests)) { - push(@tests_list, $single_test); - } - } - return @tests_list; -} - -# Return a list of tests to run from given test ranges -# ranges - test ranges(e.g. xfs/001-100,btrfs/100-159) -# dir - xfstests installation dir(e.g. /opt/xfstests) -sub tests_from_ranges { - my ($ranges, $dir) = @_; - if ($ranges !~ /\w+(\/\d+-\d+)?(,\w+(\/\d+-\d+)?)*/) { - die "Invalid test ranges: $ranges"; - } - my %cache; - my @tests; - foreach my $range (split(/,/, $ranges)) { - my ($min, $max) = (0, 99999); - my ($category, $min_max) = split(/\//, $range); - unless (defined($min_max)) { - next; - } - if ($min_max =~ /\d+-\d+/) { - ($min, $max) = split(/-/, $min_max); - } - else { - $min = $max = $min_max; - } - unless (exists($cache{$category})) { - $cache{$category} = [tests_from_category($category, $dir)]; - assert_script_run("mkdir -p $LOG_DIR/$category"); - } - foreach my $num (@{$cache{$category}}) { - if ($num >= $min and $num <= $max) { - push(@tests, "$category/$num"); - } - } - } - return @tests; -} - -# Run a single test and write log to file -# test - test to run(e.g. xfs/001) -sub test_run { - my $test = shift; - my ($category, $num) = split(/\//, $test); - my $run_options = ''; - if (check_var('XFSTESTS', 'nfs')) { - $run_options = '-nfs'; - } - elsif (check_var('XFSTESTS', 'overlay')) { - $run_options = '-overlay'; - } - my $inject_code = get_var('INJECT_INFO', ''); - my $cmd = "\n$TEST_WRAPPER '$test' $run_options $inject_code | tee $LOG_DIR/$category/$num; "; - $cmd .= "echo \${PIPESTATUS[0]} > $HB_DONE_FILE\n"; - type_string($cmd); -} - -# Save kdump data for further uploading -# test - corresponding test(e.g. xfs/009) -# dir - Save kdump data to this dir -# vmcore - include vmcore file -# kernel - include kernel -sub save_kdump { - my ($test, $dir, %args) = @_; - $args{vmcore} ||= 0; - $args{kernel} ||= 0; - $args{debug} ||= 0; - my $name = test_name($test); - my $ret = script_run("mv /var/crash/* $dir/$name"); - if ($args{debug}) { - $ret += script_run("if [ -e /usr/lib/debug/boot ]; then tar zcvf $dir/$name/vmcore-debug.tar.gz --absolute-names /usr/lib/debug/boot; fi"); - } - return 0 if $ret != 0; - - my $removed = ""; - unless ($args{vmcore}) { - $removed .= " $dir/$name/vmcore"; - } - unless ($args{kernel}) { - $removed .= " $dir/$name/*.{gz,bz2,xz}"; - } - if ($removed) { - assert_script_run("rm -f $removed"); - } - return 1; -} - -# Kunth shuffle -sub shuffle { - my @arr = @_; - srand(time()); - for (my $i = $#arr; $i > 0; $i--) { - my $j = int(rand($i + 1)); - ($arr[$i], $arr[$j]) = ($arr[$j], $arr[$i]); - } - return @arr; -} - -# Log & Must included: Copy log to ready to save -sub copy_log { - my ($category, $num, $log_type) = @_; - my $cmd = "if [ -e /opt/xfstests/results/$category/$num.$log_type ]; then cat /opt/xfstests/results/$category/$num.$log_type | tee $LOG_DIR/$category/$num.$log_type; fi"; - script_run($cmd); -} - -# Log: Copy junk.fsxops for fails fsx tests included in subtests -sub copy_fsxops { - my ($category, $num) = @_; - my $cmd = "if [ -e $TEST_FOLDER/junk.fsxops ]; then cp $TEST_FOLDER/junk.fsxops $LOG_DIR/$category/$num.junk.fsxops; fi"; - script_run($cmd); -} - -# Log: Only run in test Btrfs, collect image dump for inconsistent error -sub dump_btrfs_img { - my ($category, $num) = @_; - my $cmd = "echo \"no inconsistent error, skip btrfs image dump\""; - my $ret = script_output("grep -E -m 1 \"filesystem on .+ is inconsistent\" $LOG_DIR/$category/$num"); - if ($ret =~ /filesystem on (.+) is inconsistent/) { $cmd = "umount $1;btrfs-image $1 $LOG_DIR/$category/$num.img"; } - script_run($cmd); -} - -# Log: Raw dump from SCRATCH_DEV via dd -sub raw_dump { - my ($category, $num) = @_; - my $dev = get_var('XFSTESTS_SCRATCH_DEV') ? get_var('XFSTESTS_SCRATCH_DEV') : (split(/ /, get_var("XFSTESTS_SCRATCH_DEV_POOL")))[0]; - assert_script_run("umount $dev;dd if=$dev of=$LOG_DIR/$category/$num.raw bs=512 count=1000"); -} - -# Log: Collect fs runtime status for XFS, Btrfs and Ext4 -sub collect_fs_status { - my ($category, $num) = @_; - my $cmd = < /dev/null -[ -n "\$SCRATCH_DEV" ] && mount \$SCRATCH_DEV $SCRATCH_FOLDER &> /dev/null -END_CMD - if ($FSTYPE eq 'xfs') { - $cmd = < /sys/fs/$FSTYPE/stats/stats <==" > $LOG_DIR/$category/$num.fs_stat -cat /sys/fs/$FSTYPE/stats/stats >> $LOG_DIR/$category/$num.fs_stat -tail -n +1 /sys/fs/$FSTYPE/*/log/* >> $LOG_DIR/$category/$num.fs_stat -tail -n +1 /sys/fs/$FSTYPE/*/stats/stats >> $LOG_DIR/$category/$num.fs_stat -xfs_info $TEST_FOLDER > $LOG_DIR/$category/$num.xfsinfo -xfs_info $SCRATCH_FOLDER >> $LOG_DIR/$category/$num.xfsinfo -END_CMD - } - elsif ($FSTYPE eq 'btrfs') { - $cmd = <> $LOG_DIR/$category/$num.fs_stat -tail -n +1 /sys/fs/$FSTYPE/*/allocation/metadata/[bdft]* >> $LOG_DIR/$category/$num.fs_stat -tail -n +1 /sys/fs/$FSTYPE/*/allocation/metadata/dup/* >> $LOG_DIR/$category/$num.fs_stat -tail -n +1 /sys/fs/$FSTYPE/*/allocation/*/single/* >> $LOG_DIR/$category/$num.fs_stat -END_CMD - } - elsif ($FSTYPE eq 'ext4') { - $cmd = <> $LOG_DIR/$category/$num.fs_stat -END_CMD - } - $cmd = < /dev/null -[ -n "\$SCRATCH_DEV" ] && umount \$SCRATCH_DEV &> /dev/null -END_CMD - enter_cmd("$cmd"); -} - -# Add all above logs -sub copy_all_log { - my ($category, $num) = @_; - copy_log($category, $num, 'out.bad'); - copy_log($category, $num, 'full'); - copy_log($category, $num, 'dmesg'); - copy_fsxops($category, $num); - collect_fs_status($category, $num); - if (get_var('BTRFS_DUMP', 0) && (check_var 'XFSTESTS', 'btrfs')) { dump_btrfs_img($category, $num); } - if (get_var('RAW_DUMP', 0)) { raw_dump($category, $num); } -} - -sub reload_loop_device { - my $self = shift; - assert_script_run("losetup -fP $INST_DIR/test_dev"); - my $scratch_amount = script_output("ls $INST_DIR/scratch_dev* | wc -l"); - my $scratch_num = 1; - while ($scratch_amount >= $scratch_num) { - assert_script_run("losetup -fP $INST_DIR/scratch_dev$scratch_num", 300); - $scratch_num += 1; - } - script_run('losetup -a'); - format_partition("$INST_DIR/test_dev", $FSTYPE); -} - -# Umount TEST_DEV and SCRATCH_DEV -sub umount_xfstests_dev { - script_run('umount ' . get_var('XFSTESTS_TEST_DEV') . ' &> /dev/null') if get_var('XFSTESTS_TEST_DEV'); - script_run('umount ' . get_var('XFSTESTS_SCRATCH_DEV') . ' &> /dev/null') if get_var('XFSTESTS_SCRATCH_DEV'); - if (get_var('XFSTESTS_SCRATCH_DEV_POOL')) { - script_run("umount $_ &> /dev/null") foreach (split ' ', get_var('XFSTESTS_SCRATCH_DEV_POOL')); - } -} - -sub config_debug_option { - script_run('echo 1 > /proc/sys/kernel/softlockup_all_cpu_backtrace'); # on detection capture more debug information - script_run('echo 1 > /proc/sys/kernel/softlockup_panic'); # panic when softlockup - if (get_var('XFSTESTS_DEBUG')) { - # e.g. XFSTESTS_DEBUG could be one or more parameter in following - # [hardlockup_panic hung_task_panic panic_on_io_nmi panic_on_oops panic_on_rcu_stall...] - script_run("echo 1 > /proc/sys/kernel/$_ ") foreach (split ' ', get_var('XFSTESTS_DEBUG')); - } -} - -# Run a single test and write log to file but without heartbeat -sub test_run_without_heartbeat { - my ($self, $test) = @_; - my ($category, $num) = split(/\//, $test); - my $run_options = ''; - my $status_num = 1; - if (check_var('XFSTESTS', 'nfs')) { - $run_options = '-nfs'; - } - elsif (check_var('XFSTESTS', 'overlay')) { - $run_options = '-overlay'; - } - my $inject_code = get_var('INJECT_INFO', ''); - eval { - $test_start = time(); - # Send kill signal 3 seconds after sending the default SIGTERM to avoid some tests refuse to stop after timeout - assert_script_run("timeout -k 3 " . ($TIMEOUT_NO_HEARTBEAT - 5) . " $TEST_WRAPPER '$test' $run_options $inject_code | tee $LOG_DIR/$category/$num; echo \${PIPESTATUS[0]} > $LOG_DIR/subtest_result_num", $TIMEOUT_NO_HEARTBEAT); - $status_num = script_output("tail -n 1 $LOG_DIR/subtest_result_num"); - $test_duration = time() - $test_start; - }; - if ($@) { - $test_status = 'FAILED'; - $test_duration = time() - $test_start; - sleep(2); - copy_all_log($category, $num); - - prepare_system_shutdown; - check_var('VIRTIO_CONSOLE', '1') ? power('reset') : send_key 'alt-sysrq-b'; - reconnect_mgmt_console if is_pvm; - $self->wait_boot; +# Heartbeat mode variables +# - XFSTESTS_HEARTBEAT_INTERVAL: Set how long to send/receive a heartbeat +# - XFSTESTS_HEARTBEAT_TIMEOUT: Set the threshold to decide lose heartbeat +my $HB_INTVL = get_var('XFSTESTS_HEARTBEAT_INTERVAL') || 30; +my $HB_TIMEOUT = get_var('XFSTESTS_HEARTBEAT_TIMEOUT') || 200; +my $HB_DONE = ''; - sleep 1; - select_console('root-console'); - # Save kdump data to KDUMP_DIR if not set "NO_KDUMP=1" - unless (check_var('NO_KDUMP', '1')) { - unless (save_kdump($test, $KDUMP_DIR, vmcore => 1, kernel => 1, debug => 1)) { - # If no kdump data found, write warning to log - my $msg = "Warning: $test crashed SUT but has no kdump data"; - script_run("echo '$msg' >> $LOG_DIR/$category/$num"); - } - } +# None heartbeat mode variables +# - XFSTESTS_TIMEOUT: Set the sub-test timeout threshold +my $TIMEOUT_NO_HEARTBEAT = get_var('XFSTESTS_TIMEOUT', 2000); - # Reload loop device after a reboot - reload_loop_device if get_var('XFSTESTS_LOOP_DEVICE'); - } - else { - $status_num =~ s/^\s+|\s+$//g; - if ($status_num == 0) { - $test_status = 'PASSED'; - } - elsif ($status_num == 22) { - $test_status = 'SKIPPED'; - } - else { - $test_status = 'FAILED'; - copy_all_log($category, $num); - } - } - # Add test status to STATUS_LOG file - return log_add($STATUS_LOG, $test, $test_status, $test_duration); -} sub run { my $self = shift; @@ -522,7 +85,7 @@ sub run { # Get test list my @tests = tests_from_ranges($TEST_RANGES, $INST_DIR); my %uniq; - @tests = (@tests, include_grouplist); + @tests = (@tests, include_grouplist($TEST_RANGES, $GROUPLIST, $FSTYPE)); @tests = grep { ++$uniq{$_} < 2; } @tests; # Shuffle tests list @@ -530,7 +93,7 @@ sub run { @tests = shuffle(@tests); } - heartbeat_prepare if $enable_heartbeat == 1; + heartbeat_prepare($HB_INTVL) if $enable_heartbeat == 1; assert_script_run("mkdir -p $KDUMP_DIR $LOG_DIR"); # wait until nfs service is ready @@ -542,7 +105,7 @@ sub run { heartbeat_start if $enable_heartbeat == 1; # Generate xfstests blacklist - my %black_list = (generate_xfstests_list($BLACKLIST), exclude_grouplist); + my %black_list = (generate_xfstests_list($BLACKLIST), exclude_grouplist($TEST_RANGES, $FSTYPE)); if (my $issues = get_var('XFSTESTS_KNOWN_ISSUES')) { my $whitelist = LTP::WhiteList->new($issues); my %skipped = map { $_ => 1 } $whitelist->list_skipped_tests($whitelist_env, $TEST_SUITE); @@ -564,11 +127,11 @@ sub run { my ($category, $num) = split(/\//, $test); enter_cmd("echo $test > /dev/$serialdev"); if ($enable_heartbeat == 0) { - $status_log_content = test_run_without_heartbeat($self, $test); + $status_log_content = test_run_without_heartbeat($self, $test, $TIMEOUT_NO_HEARTBEAT, $FSTYPE); next; } test_run($test); - my ($type, $status, $time) = test_wait($MAX_TIME); + my ($type, $status, $time) = test_wait($SUBTEST_MAX_TIME, $HB_TIMEOUT); if ($type eq $HB_DONE) { # Test finished without crashing SUT $status_log_content = log_add($STATUS_LOG, $test, $status, $time); @@ -603,7 +166,7 @@ sub run { log_add($STATUS_LOG, $test, $status, $time); # Reload loop device after a reboot - reload_loop_device if get_var('XFSTESTS_LOOP_DEVICE'); + reload_loop_device($self, $FSTYPE) if get_var('XFSTESTS_LOOP_DEVICE'); # Prepare for the next test heartbeat_start;