sno.yml

---
- name: Single-Node OpenShift (SNO) with NVIDIA vGPU on VMware
  hosts: localhost
  gather_facts: false

  pre_tasks:

    - name: Ensure local temp directory '{{ local_temp_dir }}' exists
      ansible.builtin.file:
        path: "{{ local_temp_dir }}"
        state: directory

  roles:
    - name: vsphere
      vars:
        private_networks: "{{ use_private_networks | default(false) | bool }}"
        temp_directory: "{{ local_temp_dir }}"

- hosts: "{{ 'bastion' if use_private_networks | default(false) | bool else 'localhost' }}"
  gather_facts: false

  vars:
    private_networks: "{{ use_private_networks | default(false) | bool }}"
    remote_temp_dir: "{{ '/root/openshift-cluster' if private_networks else local_temp_dir }}"
    vgpu_profile: "grid_v100dx-32c" # Must be a CUDA-supported vGPU type
    oc_client_binary: https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/stable/openshift-client-linux.tar.gz
    vcenter_ip: "{{ hostvars.localhost.vcenter_ip }}"
    vcenter_username: "{{ hostvars.localhost.vcenter_username }}"
    vcenter_password: "{{ hostvars.localhost.vcenter_password }}"

  pre_tasks:

    - name: Ensure remote temp directory '{{ remote_temp_dir }}' exists
      ansible.builtin.file:
        path: "{{ remote_temp_dir }}"
        state: directory
      when: private_networks

  tasks:

    # Some tasks must always run on the bastion

    - ansible.builtin.include_role:
        name: esxi_connections
        apply:
          delegate_to: bastion

    - ansible.builtin.include_role:
        name: vgpu_driver
        apply:
          delegate_to: bastion

    - ansible.builtin.include_role:
        name: host_graphics
        apply:
          delegate_to: bastion

    - name: Install Python prerequisites
      ansible.builtin.pip:
        name:
        - aicli>=99.0.202210301623.202103111306
        - assisted-service-client>=2.9.0.post12
        state: present

    - name: Generate a cluster SSH key pair
      ansible.builtin.openssh_keypair:
        path: "{{ remote_temp_dir }}/{{openshift_cluster_name}}_ssh_rsa"
        size: 2048
      register: ssh_key_pair

    - name: Create an assisted SNO cluster
      karmab.aicli.ai_cluster:
        name: "{{ openshift_cluster_name }}"
        state: present
        parameters:
          openshift_version: "{{ openshift_version }}"
          sno: true
          minimal: true
          pull_secret: "{{ lookup('file', pull_secret_path) | string }}"
          base_dns_domain: "{{ openshift_base_domain }}"
          ssh_public_key: "{{ ssh_key_pair.public_key }}"
          tags: "{{ cluster_tags }}"
        offlinetoken: "{{ ocm_offline_token }}"

    - name: Read SNO cluster's InfraEnv
      karmab.aicli.ai_infraenv_info:
        name: "{{ openshift_cluster_name }}"
        offlinetoken: "{{ ocm_offline_token }}"
      register: sno_infraenv

    - name: Copy script for downloading assisted installer ISO
      ansible.builtin.copy:
        src: download_iso.py
        dest: bootstrap/download_iso.py
        mode: u=xwr,g=r,o=r
      delegate_to: bastion

    # TODO: Avoid downloading if the file is busy - used by an existing VM
    - name: Download assisted installer ISO for SNO cluster
      ansible.builtin.command: "python3 $HOME/bootstrap/download_iso.py {{ sno_infraenv.download_url }} {{ openshift_cluster_name }}.iso"
      args:
        chdir: bootstrap
      register: copy_result
      failed_when: "copy_result.rc != 0 and 'Device or resource busy' not in copy_result.stderr" # ignore when the file is locked by an existing VM
      delegate_to: bastion

    - name: Create a VM for SNO
      community.vmware.vmware_guest:
        hostname: "{{ vcenter_ip }}"
        username: "{{ vcenter_username }}"
        password: "{{ vcenter_password }}"
        folder: /
        datacenter: Metal
        name: "{{ vm_name }}"
        state: present
        guest_id: rhel8_64Guest
        disk:
          - size_gb: 180
            type: thin
            datastore: datastore1
        networks:
          - name: "{{ 'VM Private Net' if private_networks else 'VM Public Net 1' }}"
            start_connected: yes
        cdrom:
          - controller_number: 0
            unit_number: 0
            state: present
            type: iso
            iso_path: "[datastore1] {{ openshift_cluster_name }}.iso"
        hardware:
          memory_mb: 32768
          num_cpus: 8
          scsi: paravirtual
          boot_firmware: "efi"
        advanced_settings:
          - key: "pciPassthru.use64bitMMIO"
            value: "TRUE"
          - key: "pciPassthru.64bitMMIOSizeGB"
            value: "512"
        wait_for_ip_address: yes
        validate_certs: no
      register: vm

    - name: Power off the VM before adding vGPU
      community.vmware.vmware_guest_powerstate:
        hostname: "{{ vcenter_ip }}"
        username: "{{ vcenter_username }}"
        password: "{{ vcenter_password }}"
        folder: /
        datacenter: Metal
        name: "{{ vm_name }}"
        state: powered-off
        validate_certs: no

    - name: Add a vGPU profile to the VM
      community.vmware.vmware_guest_vgpu:
        hostname: "{{ vcenter_ip }}"
        username: "{{ vcenter_username }}"
        password: "{{ vcenter_password }}"
        folder: /
        datacenter: Metal
        name: "{{ vm_name }}"
        state: present
        vgpu: "{{ vgpu_profile }}"
        validate_certs: no

    - name: Print SNO host connection parameters
      ansible.builtin.debug:
        msg:
        - "You can now SSH to the OpenShift node: ssh -i {{ ssh_key_pair.filename }} core@{{ vm.instance.hw_eth0.ipaddresses[0] }}"

    - name: Wait for exactly one disconnected (down) host in the cluster
      karmab.aicli.ai_cluster_info:
        name: "{{ openshift_cluster_name }}"
        offlinetoken: "{{ ocm_offline_token }}"
      register: assisted_cluster
      until: assisted_cluster.status == 'installed' or (assisted_cluster.hosts | length == 1 and assisted_cluster.hosts[0].status == 'disconnected')
      retries: 10
      delay: 60

    - name: Power on the VM after adding vGPU
      community.vmware.vmware_guest_powerstate:
        hostname: "{{ vcenter_ip }}"
        username: "{{ vcenter_username }}"
        password: "{{ vcenter_password }}"
        folder: /
        datacenter: Metal
        name: "{{ vm_name }}"
        state: powered-on
        validate_certs: no

    - name: Set hostname (instead of localhost)
      karmab.aicli.ai_host:
        name: "{{ assisted_cluster.hosts[0].id }}"
        state: present
        parameters:
          requested_hostname: "{{ openshift_node_hostname }}"
        offlinetoken: "{{ ocm_offline_token }}"
      when: assisted_cluster.status != 'installed'

    - name: Install OpenShift cluster
      ansible.builtin.include_role:
        name: empovit.assisted_openshift.cluster_installation
      vars:
        cluster_name: "{{ openshift_cluster_name }}"
        temp_directory: "{{ remote_temp_dir }}"

    - name: Update DNS entries
      ansible.builtin.include_role:
        name: empovit.assisted_openshift.etc_hosts
      vars:
        api_vip: "{{ assisted_cluster.api_vip }}"
        ingress_vip: "{{ assisted_cluster.ingress_vip }}"
        cluster_name: "{{ openshift_cluster_name }}"

    # conflicts with pip-installed PyYAML
    - name: Uninstall PyYAML installed by distutils
      ansible.builtin.package:
        name: python3-yaml
        use: apt
        state: absent
      when: private_networks

    - name: Install oc binary on the bastion
      ansible.builtin.shell: "curl -L -o - {{ oc_client_binary }} | tar -C /usr/local/bin -xvzf - oc"
      when: private_networks

    - name: Install GPU Operator
      ansible.builtin.include_role:
        name: empovit.gpu_operator
      vars:
        openshift_api_vip: "{{ assisted_cluster.api_vip }}"
        gpu_type: vgpu