Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Core: Generate xUnit Reports for e2e tests #962

Open
wants to merge 36 commits into
base: master
Choose a base branch
from
Open

Core: Generate xUnit Reports for e2e tests #962

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
bba8838
added a way to generate reports
sawood14012 May 3, 2024
754a54b
install go-unit-report
sawood14012 May 3, 2024
45853c2
Merge branch 'master' into add-reporting
sawood14012 May 3, 2024
b62d260
Merge branch 'codeready-toolchain:master' into add-reporting
sawood14012 May 13, 2024
994ee26
check and install go-junit-report
sawood14012 May 13, 2024
9a4f036
another try
sawood14012 May 13, 2024
3608890
another try 2
sawood14012 May 13, 2024
cfc77de
add another path
sawood14012 May 13, 2024
ba52cf0
changed the base image
sawood14012 May 13, 2024
89aa8f9
Merge branch 'master' into add-reporting
sawood14012 May 15, 2024
54d88be
another try with path
sawood14012 May 15, 2024
66d7085
try again with different base
sawood14012 May 15, 2024
5e78253
Update make/test.mk
sawood14012 May 15, 2024
dd88f5b
Merge branch 'master' into add-reporting
sawood14012 May 15, 2024
c6b6145
added path
sawood14012 May 15, 2024
38e2527
Update make/test.mk
sawood14012 May 16, 2024
1261cef
Update test.mk
sawood14012 May 16, 2024
c4c1e1f
Merge branch 'master' into add-reporting
sawood14012 May 16, 2024
c566e6c
remove check go-junit-report
sawood14012 May 16, 2024
395097d
reinstall before trigger
sawood14012 May 16, 2024
8f42fb8
Update make/test.mk
sawood14012 May 21, 2024
bb08a5c
Merge branch 'master' into add-reporting
sawood14012 May 21, 2024
bd89da5
add metrics report
sawood14012 May 21, 2024
b6edfad
tidy up
sawood14012 May 22, 2024
a176b82
Revert Dockerfile change
sawood14012 May 22, 2024
1202053
Merge branch 'master' into add-reporting
sawood14012 May 23, 2024
1162d6d
Merge branch 'master' into add-reporting
sawood14012 Jun 14, 2024
da34df6
move the binary to /tmp
sawood14012 Jun 19, 2024
40c634c
run only on openshift ci
sawood14012 Jun 19, 2024
9768327
run only on openshift ci
sawood14012 Jun 20, 2024
1ecbfe3
Fix Pipe everyting
sawood14012 Jun 20, 2024
113cd51
Merge branch 'master' into add-reporting
sawood14012 Jun 20, 2024
466d6a5
Merge branch 'master' into add-reporting
sawood14012 Jun 21, 2024
0438d14
move test output to /tmp
sawood14012 Jun 24, 2024
0519d8e
move test output to /tmp 2
sawood14012 Jun 24, 2024
8732398
generate rp if failed
sawood14012 Jun 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 27 additions & 5 deletions make/test.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ E2E_PARALLELISM=1

TESTS_RUN_FILTER_REGEXP ?= ""

REPORT_PORTAL_DIR := rp_preproc/results

.PHONY: test-e2e
## Run the e2e tests
test-e2e: INSTALL_OPERATOR=true
Expand Down Expand Up @@ -66,13 +68,13 @@ verify-migration-and-deploy-e2e: prepare-projects e2e-deploy-latest e2e-migratio
.PHONY: e2e-migration-setup
e2e-migration-setup:
@echo "Setting up the environment before testing the operator migration..."
$(MAKE) execute-tests MEMBER_NS=${MEMBER_NS} MEMBER_NS_2=${MEMBER_NS_2} HOST_NS=${HOST_NS} REGISTRATION_SERVICE_NS=${REGISTRATION_SERVICE_NS} TESTS_TO_EXECUTE="./test/migration/setup"
$(MAKE) execute-tests MEMBER_NS=${MEMBER_NS} MEMBER_NS_2=${MEMBER_NS_2} HOST_NS=${HOST_NS} REGISTRATION_SERVICE_NS=${REGISTRATION_SERVICE_NS} TESTS_TO_EXECUTE="./test/migration/setup" REPORT_NAME="xunit_e2e_migration_setup.xml"
@echo "Environment successfully setup."

.PHONY: e2e-migration-verify
e2e-migration-verify:
@echo "Updating operators and verifying resources..."
$(MAKE) execute-tests MEMBER_NS=${MEMBER_NS} MEMBER_NS_2=${MEMBER_NS_2} HOST_NS=${HOST_NS} REGISTRATION_SERVICE_NS=${REGISTRATION_SERVICE_NS} TESTS_TO_EXECUTE="./test/migration/verify"
$(MAKE) execute-tests MEMBER_NS=${MEMBER_NS} MEMBER_NS_2=${MEMBER_NS_2} HOST_NS=${HOST_NS} REGISTRATION_SERVICE_NS=${REGISTRATION_SERVICE_NS} TESTS_TO_EXECUTE="./test/migration/verify" REPORT_NAME="xunit_e2e_migration_verify.xml"
@echo "Migration tests successfully finished"

.PHONY: e2e-deploy-latest
Expand Down Expand Up @@ -141,13 +143,13 @@ test-e2e-registration-local:
.PHONY: e2e-run-parallel
e2e-run-parallel:
@echo "Running e2e tests in parallel..."
$(MAKE) execute-tests MEMBER_NS=${MEMBER_NS} MEMBER_NS_2=${MEMBER_NS_2} HOST_NS=${HOST_NS} REGISTRATION_SERVICE_NS=${REGISTRATION_SERVICE_NS} TESTS_TO_EXECUTE="./test/e2e/parallel" E2E_PARALLELISM=100
$(MAKE) execute-tests MEMBER_NS=${MEMBER_NS} MEMBER_NS_2=${MEMBER_NS_2} HOST_NS=${HOST_NS} REGISTRATION_SERVICE_NS=${REGISTRATION_SERVICE_NS} TESTS_TO_EXECUTE="./test/e2e/parallel" E2E_PARALLELISM=100 REPORT_NAME="xunit_e2e_parallel.xml"
@echo "The parallel e2e tests successfully finished"

.PHONY: e2e-run
e2e-run:
@echo "Running e2e tests..."
$(MAKE) execute-tests MEMBER_NS=${MEMBER_NS} MEMBER_NS_2=${MEMBER_NS_2} HOST_NS=${HOST_NS} REGISTRATION_SERVICE_NS=${REGISTRATION_SERVICE_NS} TESTS_TO_EXECUTE="./test/e2e ./test/metrics"
$(MAKE) execute-tests MEMBER_NS=${MEMBER_NS} MEMBER_NS_2=${MEMBER_NS_2} HOST_NS=${HOST_NS} REGISTRATION_SERVICE_NS=${REGISTRATION_SERVICE_NS} TESTS_TO_EXECUTE="./test/e2e ./test/metrics" REPORT_NAME="xunit_e2e.xml"
@echo "The e2e tests successfully finished"

.PHONY: execute-tests
Expand All @@ -156,8 +158,10 @@ execute-tests:
-oc get Space -n ${HOST_NS}
@echo "Status of ToolchainStatus"
-oc get ToolchainStatus -n ${HOST_NS} -o yaml
@echo "Check Reporting"
$(MAKE) check-go-junit-report
MatousJobanek marked this conversation as resolved.
Show resolved Hide resolved
@echo "Starting test $(shell date)"
MEMBER_NS=${MEMBER_NS} MEMBER_NS_2=${MEMBER_NS_2} HOST_NS=${HOST_NS} REGISTRATION_SERVICE_NS=${REGISTRATION_SERVICE_NS} go test ${TESTS_TO_EXECUTE} -run ${TESTS_RUN_FILTER_REGEXP} -p 1 -parallel ${E2E_PARALLELISM} -v -timeout=90m -failfast || \
MEMBER_NS=${MEMBER_NS} MEMBER_NS_2=${MEMBER_NS_2} HOST_NS=${HOST_NS} REGISTRATION_SERVICE_NS=${REGISTRATION_SERVICE_NS} go test ${TESTS_TO_EXECUTE} -run ${TESTS_RUN_FILTER_REGEXP} -p 1 -parallel ${E2E_PARALLELISM} -v -timeout=90m -failfast 2>&1 | $(MAKE) generate-report REPORT_NAME=${REPORT_NAME} || \
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you cannot use the pipe for the whole output; otherwise, we will lose the content/output of the tests when running it locally as well as in openshift-ci

($(MAKE) print-logs HOST_NS=${HOST_NS} MEMBER_NS=${MEMBER_NS} MEMBER_NS_2=${MEMBER_NS_2} REGISTRATION_SERVICE_NS=${REGISTRATION_SERVICE_NS} && exit 1)

.PHONY: print-logs
Expand All @@ -175,6 +179,24 @@ else
$(MAKE) print-local-debug-info HOST_NS=${HOST_NS} MEMBER_NS=${MEMBER_NS} MEMBER_NS_2=${MEMBER_NS_2} REGISTRATION_SERVICE_NS=${REGISTRATION_SERVICE_NS}
endif

.PHONY: check-go-junit-report

check-go-junit-report:
go env
@command -v go-junit-report >/dev/null 2>&1 || { echo "go-junit-report is not installed. Installing..."; go install github.com/jstemmer/go-junit-report/v2@latest; }
@echo "go-junit-report version:" && go-junit-report -version

.PHONY: generate-report
generate-report:
@echo "Generating report"
ifneq ($(OPENSHIFT_BUILD_NAMESPACE),)
mkdir -p ${ARTIFACT_DIR}/${REPORT_PORTAL_DIR}
go-junit-report > ${ARTIFACT_DIR}/${REPORT_PORTAL_DIR}/${REPORT_NAME}
sawood14012 marked this conversation as resolved.
Show resolved Hide resolved
@echo "xUnit Report Generation Successful"
else
@echo "Skipping Report as its a local run"
endif

.PHONY: deploy-e2e-local-and-print-local-debug
deploy-e2e-local-and-print-local-debug: deploy-e2e-local print-local-debug-info

Expand Down
31 changes: 17 additions & 14 deletions openshift-ci/Dockerfile.tools
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM registry.access.redhat.com/ubi8/ubi:latest as build-tools
FROM registry.access.redhat.com/ubi8/go-toolset:1.20.12-5.1713833129 as build-tools

LABEL maintainer "Devtools <[email protected]>"
LABEL author "Devtools <[email protected]>"
Expand All @@ -8,12 +8,13 @@ ENV LANG=en_US.utf8 \
GOCACHE=/tmp/.cache \
PATH=$PATH:$GOPATH/bin \
GIT_COMMITTER_NAME=devtools \
[email protected] \
GOLANG_VERSION=go1.20.11 \
GOLANG_SHA256=ef79a11aa095a08772d2a69e4f152f897c4e96ee297b0dc20264b7dec2961abe
[email protected]

ARG GO_PACKAGE_PATH=github.com/codeready-toolchain/toolchain-e2e

# Switch to root user to install packages
USER root

RUN yum install -y \
findutils \
git \
Expand All @@ -27,20 +28,22 @@ RUN yum install -y \
gcc \
&& yum clean all

WORKDIR /tmp
# Switch back to the default user
USER 1001

# download, verify and install golang
ENV PATH=$PATH:/usr/local/go/bin
RUN curl -Lo ${GOLANG_VERSION}.linux-amd64.tar.gz https://dl.google.com/go/${GOLANG_VERSION}.linux-amd64.tar.gz \
&& echo "${GOLANG_SHA256} ${GOLANG_VERSION}.linux-amd64.tar.gz" > ${GOLANG_VERSION}.linux-amd64.sha256 \
&& sha256sum -c ${GOLANG_VERSION}.linux-amd64.sha256 \
&& tar xzf ${GOLANG_VERSION}.linux-amd64.tar.gz \
&& tar -C /usr/local -xzf ${GOLANG_VERSION}.linux-amd64.tar.gz \
&& rm -f ${GOLANG_VERSION}.linux-amd64.tar.gz \
&& go version
WORKDIR /tmp

RUN mkdir -p ${GOPATH}/src/${GO_PACKAGE_PATH}/

# Install the go-junit-report package
RUN go install github.com/jstemmer/go-junit-report/v2@latest

# Ensure the installed Go package is in the PATH
ENV PATH=$PATH:$GOPATH/bin

# Verify the installation
RUN go-junit-report --version

WORKDIR ${GOPATH}/src/${GO_PACKAGE_PATH}

ENTRYPOINT [ "/bin/bash" ]
Loading