Merge Zuul Logging job

.zuul.yaml

@@ -12,7 +12,7 @@
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-functional-test.yml"
     roles:
       - zuul: github.com/openstack-k8s-operators/ci-framework
-    required-projects:
+    required-projects: &required_projects
       - name: github.com/infrawatch/service-telemetry-operator
       - name: openstack-k8s-operators/ci-framework
         override-checkout: main
@@ -35,6 +35,21 @@
       - README*
       - .*/*.md
 
+- job:
+    name: functional-logging-tests-osp18
+    parent: telemetry-operator-multinode-logging
+    description: |
+      Run the logging functional test on osp18+patched version
+    vars:
+      cifmw_extras:
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir  }}/scenarios/centos-9/multinode-ci.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-logging.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-logging-test.yml"
+    roles:
+      - zuul: github.com/openstack-k8s-operators/ci-framework
+    required-projects: *required_projects
+    irrelevant-files: *irrelevant_files
+
 - job:
     name: feature-verification-tests-noop
     parent: noop
@@ -45,12 +60,12 @@
       - ci/noop.yml
     files: *irrelevant_files
 
-
 - project:
     name: infrawatch/feature-verification-tests
     github-check:
       jobs:
-        - feature-verification-tests-noop
+        #- feature-verification-tests-noop
         - openstack-k8s-operators-content-provider:
             override-checkout: main
-        - functional-tests-on-osp18
+        #- functional-tests-on-osp18
+        - functional-logging-tests-osp18

callback_plugins/custom_logger.py

@@ -4,6 +4,9 @@
 import os
 import re
 
+
+from ansible.errors import AnsibleCallbackError
+from ansible.errors import AnsibleError
 from ansible.plugins.callback import CallbackBase
 
 DOCUMENTATION = '''
@@ -34,13 +37,14 @@ def __init__(self):
         self.output_dir =  os.path.expanduser("~/")
         self.results = {}
 
-    def playbook_on_stats(self, stats):
+    def v2_playbook_on_stats(self, stats):
         #Log results for each host
         hosts= stats.processed
         for host in hosts:
             self.log_summary_results(host)
 
     def log_task_result(self, host, result, task_name):
+        print("logging task result: %s - %s - %s" % (host, result, task_name))
         # test_run_result.out only interested in the test tasks, not setup or debug.
         if "RHELOSP" in task_name or "RHOSO" in task_name:
             if "RHELOSP" in task_name:
@@ -55,6 +59,7 @@ def log_task_result(self, host, result, task_name):
 
             # Gather the result data to be used in the summary log.
             if host not in self.results:
+                print("host was not in results, adding a new value to self.results")
                 self.results[host] = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_task_names':[], 'ok_task_names':[] }
             if result == 'failed':
                 self.results[host]['failed_task_names'].append(task_name)
@@ -63,18 +68,36 @@ def log_task_result(self, host, result, task_name):
             self.results[host][result] += 1
 
     def log_summary_results(self, host):
+        # The issue is presenting itself in this method
         file_path = os.path.join(self.output_dir, f"summary_results.log")
+        print("Writing results to %s" % file_path)
+        # temperorily add more detail to the output
+        print(self.results)
+
         with open(file_path, 'w') as f:
             f.write(f"Host: {host}\n")
-            f.write(f"Tasks Succeeded: {self.results[host]['passed']}\n")
-            f.write(f"Tasks Failed: {self.results[host]['failed']}\n")
-            f.write(f"Tasks Skipped: {self.results[host]['skipped']}\n")
-            f.write("Failed Tasks:\n")
-            for task_name in self.results[host]['failed_task_names']:
-                f.write(f"  - {task_name}\n")
-            f.write("Succeeded Tasks:\n")
-            for task_name in self.results[host]['ok_task_names']:
-                f.write(f"  - {task_name}\n")
+            # This is the failed issue.
+            # Why is it happening?
+            # Unknown. But does it happen for the other values.
+            # It is happening for compute-0
+            # I need to see the partial log file, so this should NOT cause a failure.
+            # try/catch ANY exception
+            # It could be a nice upgrade to use jinja2 template to render this file.
+            # IS the issue that there were 'f's preceeding the strings?
+            try:
+                f.write("Tasks Succeeded: {self.results[host]['passed']}\n")
+                f.write("Tasks Failed: {self.results[host]['failed']}\n")
+                f.write("Tasks Skipped: {self.results[host]['skipped']}\n")
+                f.write("Failed Tasks:\n")
+                for task_name in self.results[host]['failed_task_names']:
+                    f.write(f"  - {task_name}\n")
+                f.write("Succeeded Tasks:\n")
+                for task_name in self.results[host]['ok_task_names']:
+                    f.write(f"  - {task_name}\n")
+            except AnsibleError as e:
+                # this is probably an AnsibleCallbackError
+                print("Ooops, there was an error")
+                print(e)
 
     def v2_runner_on_ok(self, result):
         host = result._host.get_name()
@@ -89,4 +112,4 @@ def v2_runner_on_failed(self, result, ignore_errors=False):
     def v2_runner_on_skipped(self, result):
         host = result._host.get_name()
         task_name = result._task.get_name()
-        self.log_task_result(host, 'skipped', task_name)
+        self.log_task_result(host, 'skipped', task_name)

ci/ansible.cfg

@@ -3,3 +3,6 @@ callbacks_enabled = custom_logger
 callback_plugins = ../callback_plugins
 # additional paths to search for roles
 roles_path = ../roles:/usr/share/ansible/roles:/etc/ansible/roles:~/.ansible/roles:../../ci-framework/roles
+
+# temp update to get more information from the playbooks running functional tests
+verbosity = 5

ci/logging_tests_computes.yml

@@ -0,0 +1,63 @@
+---
+- name: Prepare Logging Tests
+  hosts: computes
+  gather_facts: false
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path }}"
+  vars_files:
+    - vars/common.yml
+  tasks:
+    - name: Create log dir
+      ansible.builtin.file:
+        path: "{{ logs_dir }}/logging_compute"
+        state: directory
+        mode: "0755"
+
+- name: "Verify ctlplane logging containers and files"
+  hosts: computes
+  gather_facts: no
+  ignore_errors: true
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path }}"
+  vars:
+    container_test_id: "RHOSO-12753"
+    container_list:
+      - ceilometer_agent_compute
+      - ceilometer_agent_ipmi
+      - node_exporter
+
+    file_test_id: "RHOSO-12754"
+    file_list:
+      - /etc/rsyslog.d/10-telemetry.conf
+  tasks:
+    - name: "Run file and container tests"
+      ansible.builtin.import_role:
+        name: common
+
+
+- name: "Verify logging journalctl identifiers"
+  hosts: computes
+  gather_facts: no
+  ignore_errors: true
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path }}"
+  vars:
+    journal_test_id: "RHOSO-12681"
+    journal_list:
+      - ceilometer_agent_compute
+      - nova_compute
+  tasks:
+    - name: "Verify journalctl logging identifiers"
+      ansible.builtin.import_role:
+        name: telemetry_logging
+
+- name: Include report result
+  hosts: computes
+  tasks:
+    - ansible.builtin.include_tasks:
+        file: report_result.yml
+      vars:
+        test_dir: "{{ logs_dir }}/logging_local"