Skip to content

Commit

Permalink
Merge pull request rhinstaller#595 from KKoukiou/INSTALLER-4108
Browse files Browse the repository at this point in the history
tests: generate report.json for TEST_SCENARIO=compose-XYZ
  • Loading branch information
KKoukiou authored Jan 21, 2025
2 parents ffca829 + b9ede60 commit 6d689ee
Show file tree
Hide file tree
Showing 4 changed files with 80 additions and 0 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,4 @@
/test/common/
/test/images
/test/ks*.cfg
/test/report.json
31 changes: 31 additions & 0 deletions test/anacondalib.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
# You should have received a copy of the GNU Lesser General Public License
# along with this program; If not, see <http://www.gnu.org/licenses/>.

import json
import os
import subprocess
import sys
Expand Down Expand Up @@ -44,7 +45,9 @@ class VirtInstallMachineCase(MachineCase):
disk_image = ""
disk_size = 15
is_efi = os.environ.get("TEST_FIRMWARE", "bios") == "efi"
is_compose = bool(os.environ.get("TEST_COMPOSE", None))
MachineCase.machine_class = VirtInstallMachine
report_file = os.path.join(TEST_DIR, "report.json")

@property
def temp_dir(self):
Expand All @@ -67,6 +70,7 @@ def setUpClass(cls):

def setUp(self):
method = getattr(self, self._testMethodName)
test_plan = getattr(method, "test_plan", "")
boot_modes = getattr(method, "boot_modes", [])

if self.is_efi and "efi" not in boot_modes:
Expand All @@ -81,6 +85,8 @@ def setUp(self):
self.addCleanup(self.resetLanguage)
self.addCleanup(self.resetMisc)

self.test_plan = test_plan

super().setUp()

# Add installation target disk
Expand Down Expand Up @@ -237,14 +243,39 @@ def install(self, needs_confirmation, button_text="Install"):

self.handleReboot()

def appendResultsToReport(self):
with open(self.report_file, "r+") as f:
test_name = f"{self.__class__.__name__}.{self._testMethodName}"
firmware = "uefi" if self.is_efi else "bios"
error = super().getError()
status = "fail" if error else "pass"
# Add the new entry in the "tests" array in the JSON report file
data = json.load(f)
new_entry = {
"test_name": test_name,
"firmware": firmware,
"status": status,
"error": error,
"openqa_test": self.test_plan
}
data["tests"].append(new_entry)
f.seek(0)
json.dump(data, f, indent=4)
f.truncate()

def tearDown(self):
if not self.installation_finished:
self.downloadLogs()

if self.is_compose:
self.appendResultsToReport()

super().tearDown()


def test_plan(_url):
def decorator(func):
func.test_plan = _url
return func
return decorator

Expand Down
43 changes: 43 additions & 0 deletions test/prepare-report
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
#!/bin/bash

set -eu

# This script creates the report.json which will be used by the CI run to report the test results to OpenQA.
# Here is only fills in the metadata section. The tests section will be filled by the test run.
#
# Structure of report.json
# {
# "metadata": {
# "compose": "rawhide/Fedora-Rawhide-20250117.n.0",
# "test_env": "qemu-x86_64",
# },
# "tests": [
# {
# "test_name": "TestClassX1.test_example_feature",
# "firmware": "uefi",
# "openqa_test": "https://fedoraproject.org/wiki/QA:Testcase_webui_partitioning_guided_multi_select",
# "status": "pass",
# },
# {
# "test_name": "TestClassX2.test_another_feature",
# "firmware": "bios",
# "openqa_test": "https://fedoraproject.org/wiki/QA:Testcase_webui_partitioning_guided_free_space",
# "status": "fail",
# }
# ],
# "timestamp": "2025-01-17T10:30:00Z"
# }

COMPOSE=$TEST_COMPOSE
TEST_ENV="qemu-x86_64"

cat <<EOF > ./test/report.json
{
"metadata": {
"compose": "$COMPOSE",
"test_env": "$TEST_ENV"
},
"tests": [],
"timestamp": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
}
EOF
5 changes: 5 additions & 0 deletions test/run
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,11 @@ esac
# We need to know if a TEST_COMPOSE is specified before we start downloading the test images
make create-updates.img

# If TEST_COMPOSE is defined prepare the test report
if [ -n "${TEST_COMPOSE-}" ]; then
test/prepare-report
fi

# test runs in kernel_t context and triggers massive amounts of SELinux
# denials; SELinux gets disabled, but would still trigger unexpected messages
# we create huge VMs, so we need to reduce parallelism on CI
Expand Down

0 comments on commit 6d689ee

Please sign in to comment.