From d9db87592c5747e24f05e6104ff06ec0cb7d318a Mon Sep 17 00:00:00 2001 From: Ella Shulman Date: Tue, 24 Sep 2024 05:58:00 -0400 Subject: [PATCH] Allow setting order and hooks for test role NOTE: this patch will require setting order for all jobs running not only tempest - Allow passing hook list directly to hook role - Allow setting up stages and hooks for test operator role Jira: https://issues.redhat.com/browse/OSPRH-10106 --- docs/source/usage/01_usage.md | 2 - roles/shiftstack/README.md | 4 +- roles/test_operator/README.md | 15 ++++- roles/test_operator/defaults/main.yml | 5 +- roles/test_operator/tasks/main.yml | 59 ++----------------- .../tasks/runners/ansibletest_runner.yml | 10 ++++ .../tasks/runners/horizontest_runner.yml | 10 ++++ .../tasks/runners/tempest_runner.yml | 11 ++++ .../tasks/runners/tobiko_runner.yml | 12 ++++ roles/test_operator/tasks/stages.yml | 33 +++++++++++ scenarios/reproducers/bgp-4-racks-3-ocps.yml | 1 - scenarios/reproducers/va-common.yml | 1 - zuul.d/architecture-jobs.yaml | 1 - 13 files changed, 101 insertions(+), 63 deletions(-) create mode 100644 roles/test_operator/tasks/runners/ansibletest_runner.yml create mode 100644 roles/test_operator/tasks/runners/horizontest_runner.yml create mode 100644 roles/test_operator/tasks/runners/tempest_runner.yml create mode 100644 roles/test_operator/tasks/runners/tobiko_runner.yml create mode 100644 roles/test_operator/tasks/stages.yml diff --git a/docs/source/usage/01_usage.md b/docs/source/usage/01_usage.md index 70707e7f17..1c0698a024 100644 --- a/docs/source/usage/01_usage.md +++ b/docs/source/usage/01_usage.md @@ -52,8 +52,6 @@ are shared among multiple roles: - `cifmw_run_tests`: (Bool) Specifies whether tests should be executed. Defaults to false. - `cifmw_run_test_role`: (String) Specifies which ci-framework role will be used to run tests. Allowed options are `test_operator`, `tempest` and `shiftstack`. Defaults to `tempest`. -- `cifmw_run_tempest`: (Bool) Specifies whether tempest tests should be run. Notice tempest tests can be executed with either `test_operator` or `tempest` roles. Defaults to true. -- `cifmw_run_tobiko`: (Bool) Specifies whether tobiko tests should be run. Notice tobiko tests can only be executed with the `test_operator` role. Defaults to false. - `cifmw_edpm_deploy_nfs`: (Bool) Specifies whether an nfs server should be deployed. - `cifmw_nfs_target`: (String) The Ansible inventory group where the nfs server is deployed. Defaults to `computes`. Only has an effect if `cifmw_edpm_deploy_nfs` is set to `true`. - `cifmw_nfs_network`: (String) The network the deployed nfs server will be accessible from. Defaults to `storage`. Only has an effect if `cifmw_edpm_deploy_nfs` is set to `true`. diff --git a/roles/shiftstack/README.md b/roles/shiftstack/README.md index c01aa2b6f9..72246f738e 100644 --- a/roles/shiftstack/README.md +++ b/roles/shiftstack/README.md @@ -22,10 +22,10 @@ Role for triggering Openshift on Openstack QA automation (installation and tests The role is imported in the test playbook, i.e. when: ``` cifmw_run_tests: true -cifmw_run_tempest: false +cifmw_test_operator_stages: [] cifmw_run_test_role: shiftstack cifmw_run_test_shiftstack_testconfig: 4.15_ovnkubernetes_ipi_va1.yaml cifmw_shiftstack_qa_gerrithub_change: refs/changes/29/1188429/50 #optional -$ ansible-playbook deploy-edpm.yml --extra-vars "cifmw_run_tests=true cifmw_run_tempest=false cifmw_run_test_role=shiftstack cifmw_openshift_kubeconfig={{ ansible_user_dir }}/.kube/config cifmw_run_test_shiftstack_testconfig=4.15_ovnkubernetes_ipi_va1.yaml cifmw_shiftstack_qa_gerrithub_change=refs/changes/29/1188429/50" +$ ansible-playbook deploy-edpm.yml --extra-vars "cifmw_run_tests=true cifmw_run_test_role=shiftstack cifmw_test_operator_stages=[] cifmw_openshift_kubeconfig={{ ansible_user_dir }}/.kube/config cifmw_run_test_shiftstack_testconfig=4.15_ovnkubernetes_ipi_va1.yaml cifmw_shiftstack_qa_gerrithub_change=refs/changes/29/1188429/50" ``` diff --git a/roles/test_operator/README.md b/roles/test_operator/README.md index 83adf6cf09..21694e162a 100644 --- a/roles/test_operator/README.md +++ b/roles/test_operator/README.md @@ -3,7 +3,6 @@ Execute tests via the [test-operator](https://openstack-k8s-operators.github.io/test-operator/). ## Parameters - * `cifmw_test_operator_artifacts_basedir`: (String) Directory where we will have all test-operator related files. Default value: `{{ cifmw_basedir }}/tests/test_operator` which defaults to `~/ci-framework-data/tests/test_operator` * `cifmw_test_operator_namespace`: (String) Namespace inside which all the resources are created. Default value: `openstack` * `cifmw_test_operator_index`: (String) Full name of container image with index that contains the test-operator. Default value: `quay.io/openstack-k8s-operators/test-operator-index:latest` @@ -21,8 +20,20 @@ Execute tests via the [test-operator](https://openstack-k8s-operators.github.io/ * `cifmw_test_operator_node_selector`: (Dict) `nodeSelector` value that is applied to all pods spawned by the test-operator and to the test-operator-controller-manager and test-operator-logs pods. Default value: `{}` * `cifmw_test_operator_storage_class_prefix`: (String) Prefix for `storageClass` in generated Tempest CRD file. Defaults to `"lvms-"` only if `cifmw_use_lvms` is True, otherwise it defaults to `""`. The prefix is prepended to the `cifmw_test_operator_storage_class`. It is not recommended to override this value, instead set `cifmw_use_lvms` True or False. * `cifmw_test_operator_storage_class`: (String) Value for `storageClass` in generated Tempest or Tobiko CRD file. Defaults to `"lvms-local-storage"` only if `cifmw_use_lvms` is True, otherwise it defaults to `"local-storage"`. -* `cifmw_test_operator_delete_logs_pod`: (Boolean) Delete tempest log pod created by the role at the end of the testing. Default value: `false`. +* `cifmw_test_operator_delete_logs_pod`: (Boolean) Delete tempest log pod created by the role at the end of the testing. Default value: `true`. * `cifmw_test_operator_privileged`: (Boolean) Spawn the test pods with `allowPrivilegedEscalation: true` and default linux capabilities. This is required for certain test-operator functionalities to work properly (e.g.: `extraRPMs`, certain set of tobiko tests). Default value: `true` +* `cifmw_test_operator_stages`: (List) List of dictionaries defining the stages that should be used in the test operator role. List items options are: + * `name`: (String) The name of the stage. The name must be unique. + * `type`: (String) The framework name you would like to call, currently the options are: tempest, ansibletest, horizontest, tobiko. + * `test_vars_file`: (String) Path to the file used for testing, this file should contain the testing params for this stage. + * `pre_test_stage_hooks`: (List) List of pre hooks to run as described [hooks README](https://github.com/openstack-k8s-operators/ci-framework/tree/main/roles/run_hook#hooks-expected-format). + * `post_test_stage_hooks`: (List) List of post hooks to run as described [hooks README](https://github.com/openstack-k8s-operators/ci-framework/tree/main/roles/run_hook#hooks-expected-format). + Default value: +``` +cifmw_test_operator_stages: + - name: tempest + type: tempest +``` ## Tempest specific parameters * `cifmw_test_operator_tempest_registry`: (String) The registry where to pull tempest container. Default value: `quay.io` diff --git a/roles/test_operator/defaults/main.yml b/roles/test_operator/defaults/main.yml index da88f8ab44..5615cfbafa 100644 --- a/roles/test_operator/defaults/main.yml +++ b/roles/test_operator/defaults/main.yml @@ -19,6 +19,9 @@ # All variables within this role should have a prefix of "cifmw_test_operator" # Section 1: generic parameters (applied to all supported test frameworks) +cifmw_test_operator_stages: + - name: tempest + type: tempest cifmw_test_operator_fail_on_test_failure: true cifmw_test_operator_artifacts_basedir: "{{ cifmw_basedir | default(ansible_user_dir ~ '/ci-framework-data') }}/tests/test_operator" cifmw_test_operator_namespace: openstack @@ -35,7 +38,7 @@ cifmw_test_operator_default_jobs: cifmw_test_operator_fail_fast: false cifmw_test_operator_storage_class_prefix: "{{ 'lvms-' if cifmw_use_lvms | default(false) | bool else '' }}" cifmw_test_operator_storage_class: "{{ cifmw_test_operator_storage_class_prefix }}local-storage" -cifmw_test_operator_delete_logs_pod: false +cifmw_test_operator_delete_logs_pod: true cifmw_test_operator_privileged: true # Section 2: tempest parameters - used when run_test_fw is 'tempest' diff --git a/roles/test_operator/tasks/main.yml b/roles/test_operator/tasks/main.yml index b745d7da2c..ca266d84bb 100644 --- a/roles/test_operator/tasks/main.yml +++ b/roles/test_operator/tasks/main.yml @@ -23,13 +23,6 @@ owner: "{{ ansible_user | default(lookup('env', 'USER')) }}" group: "{{ ansible_user | default(lookup('env', 'USER')) }}" -- name: Set run_tempest and run_tobiko bool values - ansible.builtin.set_fact: - run_tempest: "{{ cifmw_run_tempest | default('true') | bool }}" - run_tobiko: "{{ cifmw_run_tobiko | default('false') | bool }}" - run_ansibletest: "{{ cifmw_run_ansibletest | default('false') | bool }}" - run_horizontest: "{{ cifmw_run_horizontest | default('false') | bool }}" - - name: Ensure OperatorGroup for the test-operator is present kubernetes.core.k8s: kubeconfig: "{{ cifmw_openshift_kubeconfig }}" @@ -132,52 +125,12 @@ until: csv.resources[0].status.phase | default(omit) == "Succeeded" when: not cifmw_test_operator_dry_run | bool -- name: Run tempest job - vars: - run_test_fw: tempest - test_operator_config: "{{ cifmw_test_operator_tempest_config }}" - test_operator_job_name: "{{ cifmw_test_operator_tempest_name }}" - test_operator_kind_name: "{{ cifmw_test_operator_tempest_kind_name }}" - test_operator_crd_name: "{{ cifmw_test_operator_tempest_crd_name }}" - test_operator_workflow: "{{ cifmw_test_operator_tempest_workflow }}" - test_operator_config_playbook: tempest-tests.yml - ansible.builtin.include_tasks: run-test-operator-job.yml - when: run_tempest - -- name: Run ansibletest job - vars: - run_test_fw: ansibletest - test_operator_config: "{{ cifmw_test_operator_ansibletest_config }}" - test_operator_job_name: "{{ cifmw_test_operator_ansibletest_name }}" - test_operator_kind_name: "{{ cifmw_test_operator_ansibletest_kind_name }}" - test_operator_crd_name: "{{ cifmw_test_operator_ansibletest_crd_name }}" - test_operator_workflow: "{{ cifmw_test_operator_ansibletest_workflow }}" - ansible.builtin.include_tasks: run-test-operator-job.yml - when: run_ansibletest - -- name: Run horizontest job - vars: - run_test_fw: horizontest - test_operator_config: "{{ cifmw_test_operator_horizontest_config }}" - test_operator_job_name: "{{ cifmw_test_operator_horizontest_name }}" - test_operator_kind_name: "{{ cifmw_test_operator_horizontest_kind_name }}" - test_operator_crd_name: "{{ cifmw_test_operator_horizontest_crd_name }}" - test_operator_workflow: [] - ansible.builtin.include_tasks: run-test-operator-job.yml - when: run_horizontest - -# Since Tobiko may include disruptive tests (faults), it is better to execute it at the end -- name: Run tobiko job - vars: - run_test_fw: tobiko - test_operator_config: "{{ cifmw_test_operator_tobiko_config }}" - test_operator_job_name: "{{ cifmw_test_operator_tobiko_name }}" - test_operator_kind_name: "{{ cifmw_test_operator_tobiko_kind_name }}" - test_operator_crd_name: "{{ cifmw_test_operator_tobiko_crd_name }}" - test_operator_workflow: "{{ cifmw_test_operator_tobiko_workflow }}" - test_operator_config_playbook: tobiko-tests.yml - ansible.builtin.include_tasks: run-test-operator-job.yml - when: run_tobiko +- name: Call test stages loop + when: not cifmw_test_operator_dry_run | bool + ansible.builtin.include_tasks: stages.yml + loop: "{{ cifmw_test_operator_stages }}" + loop_control: + loop_var: _stage_vars - name: Delete all resources created by the role ansible.builtin.include_tasks: cleanup.yml diff --git a/roles/test_operator/tasks/runners/ansibletest_runner.yml b/roles/test_operator/tasks/runners/ansibletest_runner.yml new file mode 100644 index 0000000000..b0efffa5b8 --- /dev/null +++ b/roles/test_operator/tasks/runners/ansibletest_runner.yml @@ -0,0 +1,10 @@ +--- +- name: Run ansibletest job + vars: + run_test_fw: ansibletest + test_operator_config: "{{ _stage_var_file.cifmw_test_operator_ansibletest_config | default(cifmw_test_operator_ansibletest_config) }}" + test_operator_job_name: "{{ _stage_var_file.cifmw_test_operator_ansibletest_name | default(cifmw_test_operator_ansibletest_name) }}" + test_operator_kind_name: "{{ _stage_var_file.cifmw_test_operator_ansibletest_kind_name | default(cifmw_test_operator_ansibletest_kind_name) }}" + test_operator_crd_name: "{{ _stage_var_file.cifmw_test_operator_ansibletest_crd_name | default(cifmw_test_operator_ansibletest_crd_name) }}" + test_operator_workflow: "{{ _stage_var_file.cifmw_test_operator_ansibletest_workflow | default(cifmw_test_operator_ansibletest_workflow) }}" + ansible.builtin.include_tasks: run-test-operator-job.yml diff --git a/roles/test_operator/tasks/runners/horizontest_runner.yml b/roles/test_operator/tasks/runners/horizontest_runner.yml new file mode 100644 index 0000000000..daac80c69c --- /dev/null +++ b/roles/test_operator/tasks/runners/horizontest_runner.yml @@ -0,0 +1,10 @@ +--- +- name: Run horizontest job + vars: + run_test_fw: horizontest + test_operator_config: "{{ _stage_var_file.cifmw_test_operator_horizontest_config | default(cifmw_test_operator_horizontest_config) }}" + test_operator_job_name: "{{ _stage_var_file.cifmw_test_operator_horizontest_name | default(cifmw_test_operator_horizontest_name) }}" + test_operator_kind_name: "{{ _stage_var_file.cifmw_test_operator_horizontest_kind_name | default(cifmw_test_operator_horizontest_kind_name) }}" + test_operator_crd_name: "{{ _stage_var_file.cifmw_test_operator_horizontest_crd_name | default(cifmw_test_operator_horizontest_crd_name) }}" + test_operator_workflow: [] + ansible.builtin.include_tasks: run-test-operator-job.yml diff --git a/roles/test_operator/tasks/runners/tempest_runner.yml b/roles/test_operator/tasks/runners/tempest_runner.yml new file mode 100644 index 0000000000..b0ae7de882 --- /dev/null +++ b/roles/test_operator/tasks/runners/tempest_runner.yml @@ -0,0 +1,11 @@ +--- +- name: Run tempest job + vars: + run_test_fw: tempest + test_operator_config: "{{ _stage_var_file.cifmw_test_operator_tempest_config | default(cifmw_test_operator_tempest_config) }}" + test_operator_job_name: "{{ _stage_var_file.cifmw_test_operator_tempest_name | default(cifmw_test_operator_tempest_name) }}" + test_operator_kind_name: "{{ _stage_var_file.cifmw_test_operator_tempest_kind_name | default(cifmw_test_operator_tempest_kind_name) }}" + test_operator_crd_name: "{{ _stage_var_file.cifmw_test_operator_tempest_crd_name | default(cifmw_test_operator_tempest_crd_name) }}" + test_operator_workflow: "{{ _stage_var_file.cifmw_test_operator_tempest_workflow | default(cifmw_test_operator_tempest_workflow) }}" + test_operator_config_playbook: tempest-tests.yml + ansible.builtin.include_tasks: run-test-operator-job.yml diff --git a/roles/test_operator/tasks/runners/tobiko_runner.yml b/roles/test_operator/tasks/runners/tobiko_runner.yml new file mode 100644 index 0000000000..e3f89737ee --- /dev/null +++ b/roles/test_operator/tasks/runners/tobiko_runner.yml @@ -0,0 +1,12 @@ +--- +# Since Tobiko may include disruptive tests (faults), it is better to execute it at the end +- name: Run tobiko job + vars: + run_test_fw: tobiko + test_operator_config: "{{ _stage_var_file.cifmw_test_operator_tobiko_config | default(cifmw_test_operator_tobiko_config) }}" + test_operator_job_name: "{{ _stage_var_file.cifmw_test_operator_tobiko_name | default(cifmw_test_operator_tobiko_name) }}" + test_operator_kind_name: "{{ _stage_var_file.cifmw_test_operator_tobiko_kind_name | default(cifmw_test_operator_tobiko_kind_name) }}" + test_operator_crd_name: "{{ _stage_var_file.cifmw_test_operator_tobiko_crd_name | default(cifmw_test_operator_tobiko_crd_name) }}" + test_operator_workflow: "{{ _stage_var_file.cifmw_test_operator_tobiko_workflow | default(cifmw_test_operator_tobiko_workflow) }}" + test_operator_config_playbook: tobiko-tests.yml + ansible.builtin.include_tasks: run-test-operator-job.yml diff --git a/roles/test_operator/tasks/stages.yml b/roles/test_operator/tasks/stages.yml new file mode 100644 index 0000000000..5c1c0208d4 --- /dev/null +++ b/roles/test_operator/tasks/stages.yml @@ -0,0 +1,33 @@ +--- +- name: "Run stage {{ _stage_vars.name }}" + ansible.builtin.debug: + msg: "Run stage {{ _stage_vars.name }}" + +- name: "Call pre stage hooks for test operator stage {{ _stage_vars.name }}" + when: + - not cifmw_test_operator_dry_run | bool + - _stage_vars.pre_test_stage_hooks is defined + vars: + step: pre_test_hooks + hooks: "{{ _stage_vars.pre_test_stage_hooks }}" + ansible.builtin.import_role: + name: run_hook + +- name: Include stage var file + when: _stage_vars.test_vars_file is defined + ansible.builtin.include_vars: + file: "{{ _stage_vars.test_vars_file }}" + name: _stage_var_file + +- name: "Call runner {{ _stage_vars.type }}" + ansible.builtin.include_tasks: "runners/{{ _stage_vars.type }}_runner.yml" + +- name: "Call post stage hooks for test operator stage {{ _stage_vars.name }}" + when: + - not cifmw_test_operator_dry_run | bool + - _stage_vars.post_test_stage_hooks is defined + vars: + step: post_test_hooks + hooks: "{{ _stage_vars.post_test_stage_hooks }}" + ansible.builtin.import_role: + name: run_hook diff --git a/scenarios/reproducers/bgp-4-racks-3-ocps.yml b/scenarios/reproducers/bgp-4-racks-3-ocps.yml index 7cb1204ca5..3b73dd4d93 100644 --- a/scenarios/reproducers/bgp-4-racks-3-ocps.yml +++ b/scenarios/reproducers/bgp-4-racks-3-ocps.yml @@ -1,6 +1,5 @@ --- cifmw_run_tests: true -cifmw_run_tempest: true cifmw_run_test_role: test_operator cifmw_test_operator_timeout: 7200 cifmw_test_operator_tempest_include_list: | diff --git a/scenarios/reproducers/va-common.yml b/scenarios/reproducers/va-common.yml index b23cd96a4a..5b92bcb41c 100644 --- a/scenarios/reproducers/va-common.yml +++ b/scenarios/reproducers/va-common.yml @@ -29,7 +29,6 @@ cifmw_libvirt_manager_pub_net: ocpbm # Run tests cifmw_run_tests: true -cifmw_run_tempest: true cifmw_run_test_role: test_operator cifmw_test_operator_timeout: 7200 cifmw_test_operator_tempest_include_list: | diff --git a/zuul.d/architecture-jobs.yaml b/zuul.d/architecture-jobs.yaml index 4edcdaf601..08bbcc5c8d 100644 --- a/zuul.d/architecture-jobs.yaml +++ b/zuul.d/architecture-jobs.yaml @@ -53,7 +53,6 @@ cifmw_openshift_setup_skip_internal_registry_tls_verify: true cifmw_networking_env_def_file: "networking-env-definition.yml" cifmw_networking_mapper_networking_env_def_path: "/etc/ci/env/networking-environment-definition.yml" - cifmw_run_tempest: true cifmw_run_test_role: test_operator cifmw_run_tests: true cifmw_test_operator_concurrency: 4