DNM - Test CI when configuring Manila with an NFS network

THIS IS A DNM PATCH BASED ON 2273 TO INVESTIGATE A POTENTIAL KEYSTONE ISSUE IN UPSTREAM CI
openstack-k8s-operators · Sep 30, 2024 · 8a1d8b2 · 8a1d8b2
1 parent 19fa476
commit 8a1d8b2
Show file tree

Hide file tree

Showing 9 changed files with 206 additions and 47 deletions.
diff --git a/hooks/playbooks/manila_create_default_resources.yml b/hooks/playbooks/manila_create_default_resources.yml
@@ -8,7 +8,31 @@
     extra_specs:
       snapshot_support: "True"
       create_share_from_snapshot_support: "True"
+    manila_provider_network_name: storage
+    manila_provider_network_vlan: 21
+    manila_provider_network_start: 172.18.0.150
+    manila_provider_network_end: 172.18.0.200
+    manila_provider_network_range: 172.18.0.0/24
   tasks:
+    - name: Create Manila provider network with Neutron for instance to access Manila
+      environment:
+        KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+        PATH: "{{ cifmw_path }}"
+      ansible.builtin.command: |
+        oc -n {{ namespace }} exec -it pod/openstackclient -- {{ item }}
+      loop:
+        - "openstack network create {{ manila_provider_network_name }} --share --provider-network-type vlan --provider-physical-network datacentre --provider-segment {{ manila_provider_network_vlan }}"
+        - "openstack subnet create --allocation-pool start={{ manila_provider_network_start }},end={{ manila_provider_network_end }} --dhcp --network {{ manila_provider_network_name }} --subnet-range {{ manila_provider_network_range }} --gateway none {{ manila_provider_network_name }}-subnet"
+      register: _manila_provider_network_creation
+      failed_when: >-
+        ( _manila_provider_network_creation.rc | int ) != 0
+      when:
+        - manila_provider_network_name | length > 0
+        - (manila_provider_network_vlan | string) | length > 0
+        - manila_provider_network_start | length > 0
+        - manila_provider_network_end | length > 0
+        - manila_provider_network_range | length > 0
+
     - name: Create share type default for manila tempest plugin tests
       environment:
         KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"

diff --git a/playbooks/ceph.yml b/playbooks/ceph.yml
@@ -346,13 +346,6 @@
       ansible.builtin.set_fact:
         cifmw_cephadm_storage_network: "{{ lookup('ansible.builtin.ini', 'public_network section=global file=' ~ cifmw_cephadm_bootstrap_conf) }}"
 
-    - name: Set RGW network range to storage network only if it was not provided
-      ansible.builtin.set_fact:
-        cifmw_cephadm_rgw_network: "{{ cifmw_cephadm_storage_network }}"
-      when:
-        - cifmw_cephadm_rgw_network is not defined or
-          cifmw_cephadm_rgw_network | length == 0
-
     - name: Set IP address of first monitor
       ansible.builtin.set_fact:
         cifmw_cephadm_first_mon_ip: "{{ hostvars[this_host][all_addresses] | ansible.utils.ipaddr(cifmw_cephadm_storage_network) | first }}"
@@ -366,30 +359,99 @@
         fail_msg: "node {{ item }} doesn't have any interface connected to network {{ cifmw_cephadm_storage_network }}"
       loop: "{{ _target_hosts }}"
 
-    - name: Assert if any EDPM nodes n/w interface is missing in RGW network
-      ansible.builtin.assert:
-        that:
-          - hostvars[item][all_addresses] | ansible.utils.ipaddr(cifmw_cephadm_rgw_network) | length > 0
-        fail_msg: "node {{ item }} doesn't have any interface connected to network {{ cifmw_cephadm_rgw_network }}"
-      loop: "{{ _target_hosts }}"
+    - name: Set RGW Network Properties
       when:
-        - cifmw_cephadm_rgw_network != cifmw_cephadm_storage_network
+        - cifmw_ceph_daemons_layout.rgw_enabled | default(true) | bool
+      block:
+        - name: Set RGW network range to storage network only if it was not provided
+          ansible.builtin.set_fact:
+            cifmw_cephadm_rgw_network: "{{ cifmw_cephadm_storage_network }}"
+          when:
+            - cifmw_cephadm_rgw_network is not defined or
+              cifmw_cephadm_rgw_network | length == 0
+
+        - name: Assert if any EDPM nodes n/w interface is missing in RGW network
+          ansible.builtin.assert:
+            that:
+              - hostvars[item][all_addresses] | ansible.utils.ipaddr(cifmw_cephadm_rgw_network) | length > 0
+            fail_msg: "node {{ item }} doesn't have any interface connected to network {{ cifmw_cephadm_rgw_network }}"
+          loop: "{{ _target_hosts }}"
+          when:
+            - cifmw_cephadm_rgw_network != cifmw_cephadm_storage_network
 
-    - name: Get already assigned RGW IP addresses
-      ansible.builtin.set_fact:
-        ips: "{{ ips | default([]) + [ hostvars[item][all_addresses] | ansible.utils.ipaddr(cifmw_cephadm_rgw_network) | first ] }}"
-      loop: "{{ _target_hosts }}"
+        - name: Get already assigned RGW IP addresses
+          ansible.builtin.set_fact:
+            ips: "{{ ips | default([]) + [ hostvars[item][all_addresses] | ansible.utils.ipaddr(cifmw_cephadm_rgw_network) | first ] }}"
+          loop: "{{ _target_hosts }}"
 
-    # cifmw_cephadm_vip is the VIP reserved in the RGW network
-    - name: Set VIP var as empty string
-      ansible.builtin.set_fact:
-        cifmw_cephadm_vip: ""
+        - name: Set VIP var as empty string
+          ansible.builtin.set_fact:
+            cifmw_cephadm_vip: ""
+          when:
+            - cifmw_cephadm_rgw_vip is undefined
 
-    - name: Process VIP
-      ansible.builtin.include_role:
-        name: cifmw_cephadm
-        tasks_from: check_vip
-      loop: "{{ range(1, (ips | length) + 1) | list }}"
+        - name: Get RGW VIP
+          ansible.builtin.include_role:
+            name: cifmw_cephadm
+            tasks_from: check_vip
+          loop: "{{ range(1, (ips | length) + 1) | list }}"
+          vars:
+            cifmw_cephadm_vip_network: "{{ cifmw_cephadm_rgw_network | default(storage_network_range, true) | default(ssh_network_range, true) }}"
+          when:
+            - cifmw_cephadm_rgw_vip is undefined
+
+        - name: Set RGW VIP
+          ansible.builtin.set_fact:
+            cifmw_cephadm_rgw_vip: "{{ cifmw_cephadm_vip }}/{{ cidr }}"
+          when:
+            - cifmw_cephadm_rgw_vip is undefined
+
+    - name: Set NFS Network Properties
+      when:
+        - cifmw_ceph_daemons_layout.ceph_nfs_enabled | default(false) | bool
+      block:
+        - name: Set NFS network range to storage network only if it was not provided
+          ansible.builtin.set_fact:
+            cifmw_cephadm_nfs_network: "{{ cifmw_cephadm_storage_network }}"
+          when:
+            - cifmw_cephadm_nfs_network is not defined or
+              cifmw_cephadm_nfs_network | length == 0
+
+        - name: Assert if any EDPM nodes n/w interface is missing in NFS network
+          ansible.builtin.assert:
+            that:
+              - hostvars[item][all_addresses] | ansible.utils.ipaddr(cifmw_cephadm_nfs_network) | length > 0
+            fail_msg: "node {{ item }} doesn't have any interface connected to network {{ cifmw_cephadm_nfs_network }}"
+          loop: "{{ _target_hosts }}"
+          when:
+            - cifmw_cephadm_nfs_network != cifmw_cephadm_storage_network
+
+        - name: Get already assigned NFS IP addresses
+          ansible.builtin.set_fact:
+            ips: "{{ ips | default([]) + [ hostvars[item][all_addresses] | ansible.utils.ipaddr(cifmw_cephadm_nfs_network) | first ] }}"
+          loop: "{{ _target_hosts }}"
+
+        - name: Set VIP var as empty string
+          ansible.builtin.set_fact:
+            cifmw_cephadm_vip: ""
+          when:
+            - cifmw_cephadm_nfs_vip is undefined
+
+        - name: Get NFS VIP
+          ansible.builtin.include_role:
+            name: cifmw_cephadm
+            tasks_from: check_vip
+          loop: "{{ range(1, (ips | length) + 1) | list }}"
+          vars:
+            cifmw_cephadm_vip_network: "{{ cifmw_cephadm_nfs_network | default(storage_network_range, true) | default(ssh_network_range, true) }}"
+          when:
+            - cifmw_cephadm_nfs_vip is undefined
+
+        - name: Set NFS VIP
+          ansible.builtin.set_fact:
+            cifmw_cephadm_nfs_vip: "{{ cifmw_cephadm_vip }}/{{ cidr }}"
+          when:
+            - cifmw_cephadm_nfs_vip is undefined
 
   tasks:
     - name: Satisfy Ceph prerequisites
@@ -422,9 +484,6 @@
       ansible.builtin.import_role:
         name: cifmw_cephadm
         tasks_from: rgw
-      vars:
-        # cifmw_cephadm_vip is computed or passed as an override via -e @extra.yml
-        cifmw_cephadm_rgw_vip: "{{ cifmw_cephadm_vip }}/{{ cidr }}"
 
     - name: Configure Monitoring Stack
       when: cifmw_ceph_daemons_layout.dashboard_enabled  | default(false) | bool
@@ -448,9 +507,6 @@
       ansible.builtin.import_role:
         name: cifmw_cephadm
         tasks_from: cephnfs
-      vars:
-        # we reuse the same VIP reserved for rgw
-        cifmw_cephadm_nfs_vip: "{{ cifmw_cephadm_vip }}/{{ cidr }}"
 
     - name: Create Cephx Keys for OpenStack
       ansible.builtin.import_role:

diff --git a/roles/ci_gen_kustomize_values/molecule/default/files/3-ocp-net-def.yml b/roles/ci_gen_kustomize_values/molecule/default/files/3-ocp-net-def.yml
@@ -86,6 +86,15 @@ instances:
                 parent_interface: enp6s0
                 skip_nm: false
                 vlan_id: 22
+            nfs:
+                interface_name: eth1.24
+                ip_v4: 172.21.0.5
+                mac_addr: '52:54:00:0c:a1:d9'
+                mtu: 1500
+                network_name: nfs
+                parent_interface: eth1
+                skip_nm: false
+                vlan_id: 24
             swift:
                 interface_name: eth1.25
                 ip_v4: 172.22.0.5
@@ -133,6 +142,15 @@ instances:
                 parent_interface: enp6s0
                 skip_nm: false
                 vlan_id: 22
+            nfs:
+                interface_name: eth1.24
+                ip_v4: 172.21.0.6
+                mac_addr: '52:54:00:0c:a0:d9'
+                mtu: 1500
+                network_name: nfs
+                parent_interface: eth1
+                skip_nm: false
+                vlan_id: 24
             swift:
                 interface_name: eth1.25
                 ip_v4: 172.22.0.6
@@ -180,6 +198,15 @@ instances:
                 parent_interface: enp6s0
                 skip_nm: false
                 vlan_id: 22
+            nfs:
+                interface_name: eth1.24
+                ip_v4: 172.21.0.7
+                mac_addr: '52:54:00:0b:a1:d9'
+                mtu: 1500
+                network_name: nfs
+                parent_interface: eth1
+                skip_nm: false
+                vlan_id: 24
             swift:
                 interface_name: eth1.25
                 ip_v4: 172.22.0.7
@@ -362,6 +389,38 @@ networks:
                     start_host: 100
                 ipv6_ranges: []
         vlan_id: 22
+    nfs:
+        dns_v4: []
+        dns_v6: []
+        mtu: 1496
+        network_name: nfs
+        network_v4:  172.21.0.0/24
+        search_domain: nfs.example.com
+        tools:
+            metallb:
+                ipv4_ranges:
+                -   end:  172.21.0..90
+                    end_host: 90
+                    length: 11
+                    start:  172.21.0.80
+                    start_host: 80
+                ipv6_ranges: []
+            multus:
+                ipv4_ranges:
+                -   end:  172.21.0.70
+                    end_host: 70
+                    length: 41
+                    start:  172.21.0.30
+                    start_host: 30
+                ipv6_ranges: []
+            netconfig:
+                ipv4_ranges:
+                -   end: 72.22.0.250
+                    end_host: 250
+                    length: 151
+                    start: 172.21.0.100
+                    start_host: 100
+        vlan_id: 24
     swift:
         dns_v4: []
         dns_v6: []
@@ -388,7 +447,7 @@ networks:
                 ipv6_ranges: []
             netconfig:
                 ipv4_ranges:
-                -   end: 72.22.0.250
+                -   end: 172.22.0.250
                     end_host: 250
                     length: 151
                     start: 172.22.0.100

diff --git a/roles/cifmw_cephadm/README.md b/roles/cifmw_cephadm/README.md
@@ -77,18 +77,21 @@ need to be changed for a typical EDPM deployment.
    is gathered from the `cifmw_cephadm_bootstrap_conf` file, which represents
    the initial Ceph configuration file passed at bootstrap time.
 
-* `cifmw_cephadm_rgw_network`: the Ceph `public_network` where the `radosgw`
-   instances should be bound. The network range is gathered from the
-   `cifmw_cephadm_bootstrap_conf` file, which represents the initial Ceph
-   configuration file passed at bootstrap time.
+* `cifmw_cephadm_rgw_network`: The network where the `radosgw`
+   instances will be bound. If this value is not passed then the Ceph
+   `public_network` which represents the initial Ceph configuration
+   file passed at bootstrap time.
+
+* `cifmw_cephadm_nfs_network`: The network for NFS `ganesha`. If this
+   value is not passed then the Ceph `public_network` which represents
+   the initial Ceph configuration file passed at bootstrap time.
 
 * `cifmw_cephadm_rgw_vip`: the ingress daemon deployed along with `radosgw`
    requires a `VIP` that will be owned by `keepalived`. This IP address will
    be used as entry point to reach the `radosgw backends` through `haproxy`.
 
 * `cifmw_cephadm_nfs_vip`: the ingress daemon deployed along with the `nfs`
-   cluster requires a `VIP` that will be owned by `keepalived`. This IP
-   address is the same used for rgw unless an override is passed, and it's
+   cluster requires a `VIP` that will be owned by `keepalived`. This IP is
    used as entry point to reach the `ganesha backends` through an `haproxy`
    instance where proxy-protocol is enabled.
 

diff --git a/roles/cifmw_cephadm/tasks/check_vip.yml b/roles/cifmw_cephadm/tasks/check_vip.yml
@@ -22,9 +22,9 @@
       ansible.builtin.set_fact:
         count: "{{ 2 if count is undefined else count | int + 2 }}"
 
-    - name: Get an IP address from the RGW network
+    - name: Get an IP address from the VIP network
       ansible.builtin.set_fact:
-        cur_ip: "{{ cifmw_cephadm_rgw_network | ansible.utils.next_nth_usable(count) }}"
+        cur_ip: "{{ cifmw_cephadm_vip_network | ansible.utils.next_nth_usable(count) }}"
 
     - name: Reserve VIP if the address is available
       ansible.builtin.set_fact:

diff --git a/roles/cifmw_cephadm/tasks/configure_object.yml b/roles/cifmw_cephadm/tasks/configure_object.yml
@@ -92,8 +92,8 @@
     script: |-
       oc -n {{ cifmw_cephadm_ns }} rsh openstackclient openstack role add --user {{ all_uuids.results.0.stdout }} --project {{ project_service_uuid.stdout }} {{ all_uuids.results.2.stdout }}
       oc -n {{ cifmw_cephadm_ns }} rsh openstackclient openstack role add --user {{ all_uuids.results.0.stdout }} --project {{ project_service_uuid.stdout }} {{ all_uuids.results.3.stdout }}
-      oc -n {{ cifmw_cephadm_ns }} rsh openstackclient openstack endpoint create --region regionOne {{ all_uuids.results.1.stdout }} public {{ cifmw_cephadm_urischeme }}://{{ cifmw_external_dns_vip_ext.values() | first if cifmw_external_dns_vip_ext is defined else cifmw_cephadm_vip }}:8080/swift/v1/AUTH_%\(tenant_id\)s
-      oc -n {{ cifmw_cephadm_ns }} rsh openstackclient openstack endpoint create --region regionOne {{ all_uuids.results.1.stdout }} internal {{ cifmw_cephadm_urischeme }}://{{ cifmw_external_dns_vip_int.values() | first if cifmw_external_dns_vip_int is defined else cifmw_cephadm_vip }}:8080/swift/v1/AUTH_%\(tenant_id\)s
+      oc -n {{ cifmw_cephadm_ns }} rsh openstackclient openstack endpoint create --region regionOne {{ all_uuids.results.1.stdout }} public {{ cifmw_cephadm_urischeme }}://{{ cifmw_external_dns_vip_ext.values() | first if cifmw_external_dns_vip_ext is defined else cifmw_cephadm_rgw_vip }}:8080/swift/v1/AUTH_%\(tenant_id\)s
+      oc -n {{ cifmw_cephadm_ns }} rsh openstackclient openstack endpoint create --region regionOne {{ all_uuids.results.1.stdout }} internal {{ cifmw_cephadm_urischeme }}://{{ cifmw_external_dns_vip_int.values() | first if cifmw_external_dns_vip_int is defined else cifmw_cephadm_rgw_vip }}:8080/swift/v1/AUTH_%\(tenant_id\)s
       oc -n {{ cifmw_cephadm_ns }} rsh openstackclient openstack role add --project {{ all_uuids.results.4.stdout }} --user {{ all_uuids.results.5.stdout }} {{ all_uuids.results.6.stdout }}
   delegate_to: localhost
   when:

diff --git a/roles/cifmw_cephadm/tasks/rgw.yml b/roles/cifmw_cephadm/tasks/rgw.yml
@@ -27,12 +27,12 @@
     - name: Define cifmw_external_dns_vip_ext
       ansible.builtin.set_fact:
         cifmw_external_dns_vip_ext: "{{ cifmw_external_dns_vip_ext | default({}) |
-          combine({ (cifmw_cephadm_vip): 'rgw-external.ceph.local' }) }}"
+          combine({ (cifmw_cephadm_rgw_vip | ansible.utils.ipaddr('address')): 'rgw-external.ceph.local' }) }}"
 
     - name: Define cifmw_external_dns_vip_int
       ansible.builtin.set_fact:
         cifmw_external_dns_vip_int: "{{ cifmw_external_dns_vip_ext | default({}) |
-          combine({ (cifmw_cephadm_vip): 'rgw-internal.ceph.local' }) }}"
+          combine({ (cifmw_cephadm_rgw_vip | ansible.utils.ipaddr('address')): 'rgw-internal.ceph.local' }) }}"
 
     - name: Create DNS domain and certificate
       ansible.builtin.include_role:

diff --git a/roles/cifmw_cephadm/templates/ceph_rgw.yml.j2 b/roles/cifmw_cephadm/templates/ceph_rgw.yml.j2
@@ -31,7 +31,7 @@ spec:
     monitor_port: 8999
     virtual_interface_networks:
     - {{ cifmw_cephadm_rgw_network }}
-    virtual_ip: {{ cifmw_cephadm_vip }}
+    virtual_ip: {{ cifmw_cephadm_rgw_vip }}
 {% if rgw_frontend_cert is defined %}
     ssl_cert: |
       {{ rgw_frontend_cert | indent( width=6 ) }}

diff --git a/scenarios/reproducers/networking-definition.yml b/scenarios/reproducers/networking-definition.yml
@@ -91,6 +91,19 @@ cifmw_networking_definition:
               end: 250
       vlan: 23
       mtu: 1500
+    nfs:
+      network: "172.21.0.0/24"
+      tools:
+        netconfig:
+          ranges:
+            - start: 100
+              end: 250
+        multus:
+          ranges:
+            - start: 30
+              end: 70
+      vlan: 24
+      mtu: 1500
     swift:
       network: "172.22.0.0/24"
       tools:
@@ -119,6 +132,8 @@ cifmw_networking_definition:
           trunk-parent: ctlplane
         storage:
           trunk-parent: ctlplane
+        nfs:
+          trunk-parent: ctlplane
         swift:
           trunk-parent: ctlplane
     ocp_workers:
@@ -142,6 +157,8 @@ cifmw_networking_definition:
           trunk-parent: ctlplane
         storagemgmt:
           trunk-parent: ctlplane
+        nfs:
+          trunk-parent: ctlplane
         swift:
           trunk-parent: ctlplane
     cephs: