From cfc69a8ab850cf0529be8077bb40c17539a8b44b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Dost=C3=A1l?= Date: Fri, 9 Aug 2024 20:34:30 +0200 Subject: [PATCH] Test Azure cloud-netconfig addressing --- .../terraform/azure_cloud-netconfig.tf | 289 ++++++++++++++++++ tests/publiccloud/cloud_netconfig.pm | 100 ++++-- 2 files changed, 370 insertions(+), 19 deletions(-) create mode 100644 data/publiccloud/terraform/azure_cloud-netconfig.tf diff --git a/data/publiccloud/terraform/azure_cloud-netconfig.tf b/data/publiccloud/terraform/azure_cloud-netconfig.tf new file mode 100644 index 000000000000..135c5cdaad9c --- /dev/null +++ b/data/publiccloud/terraform/azure_cloud-netconfig.tf @@ -0,0 +1,289 @@ +terraform { + required_providers { + azurerm = { + version = "= 3.48.0" + source = "hashicorp/azurerm" + } + random = { + version = "= 3.1.0" + source = "hashicorp/random" + } + } +} + +provider "azurerm" { + features {} +} + +variable "instance_count" { + default = "1" +} + +variable "secondary_address" { + default = "2" +} + +variable "name" { + default = "openqa-vm" +} + +variable "type" { + default = "Standard_B2s" +} + +variable "region" { + default = "westeurope" +} + +variable "image_id" { + default = "" +} + +variable "image_uri" { + default = "" +} + +variable "publisher" { + default = "SUSE" +} + +variable "offer" { + default = "" +} + +variable "sku" { + default = "gen2" +} + +variable "extra-disk-size" { + default = "100" +} + +variable "extra-disk-type" { + default = "Premium_LRS" +} + +variable "create-extra-disk" { + default = false +} + +variable "cloud_init" { + default = "" +} + +variable "storage-account" { + # Note: Don't delete the default value!!! + # Not all of our `terraform destroy` calls pass this variable and neither is it necessary. + # However removing the default value might cause `terraform destroy` to fail in corner cases, + # resulting effectively in leaking resources due to failed cleanups. + default = "eisleqaopenqa" +} + +variable "tags" { + type = map(string) + default = {} +} + +variable "vm_create_timeout" { + default = "20m" +} + +variable "subnet_id" { + default = "" +} + +variable "ssh_public_key" { + default = "/root/.ssh/id_rsa.pub" +} + +resource "random_id" "service" { + count = var.instance_count + keepers = { + name = var.name + } + byte_length = 8 +} + + +resource "azurerm_resource_group" "openqa-group" { + name = "${var.name}-${element(random_id.service.*.hex, 0)}" + location = var.region + + tags = merge({ + openqa_created_by = var.name + openqa_created_date = timestamp() + openqa_created_id = element(random_id.service.*.hex, 0) + }, var.tags) +} + +resource "azurerm_public_ip" "openqa-publicip" { + name = "${var.name}-${element(random_id.service.*.hex, count.index)}-public-ip" + location = var.region + resource_group_name = azurerm_resource_group.openqa-group.name + allocation_method = "Dynamic" + count = var.instance_count +} + +resource "azurerm_network_interface" "openqa-nic" { + name = "${var.name}-${element(random_id.service.*.hex, count.index)}-nic" + location = var.region + resource_group_name = azurerm_resource_group.openqa-group.name + count = var.instance_count + + ip_configuration { + name = "${element(random_id.service.*.hex, count.index)}-nic-config" + subnet_id = var.subnet_id + private_ip_address_allocation = "Dynamic" + public_ip_address_id = element(azurerm_public_ip.openqa-publicip.*.id, count.index) + primary = true + } + ip_configuration { + name = "${element(random_id.service.*.hex, count.index)}-nic-secondary-config" + subnet_id = var.subnet_id + private_ip_address_version = "IPv4" + private_ip_address_allocation = "Dynamic" + } +} + +resource "azurerm_public_ip" "openqa-secondary-publicip" { + name = "${var.name}-${element(random_id.service.*.hex, count.index)}-secondary-public-ip" + location = var.region + resource_group_name = azurerm_resource_group.openqa-group.name + allocation_method = "Dynamic" + count = var.instance_count +} + +resource "azurerm_network_interface" "openqa-secondary-nic" { + name = "${var.name}-${element(random_id.service.*.hex, count.index)}-secondary-nic" + location = var.region + resource_group_name = azurerm_resource_group.openqa-group.name + count = var.instance_count + + ip_configuration { + name = "${element(random_id.service.*.hex, count.index)}-secondary-nic-config" + subnet_id = var.subnet_id + private_ip_address_allocation = "Dynamic" + public_ip_address_id = element(azurerm_public_ip.openqa-secondary-publicip.*.id, count.index) + primary = true + } + ip_configuration { + name = "${element(random_id.service.*.hex, count.index)}-secondary-nic-secondary-config" + subnet_id = var.subnet_id + private_ip_address_version = "IPv4" + private_ip_address_allocation = "Dynamic" + } +} + +resource "azurerm_image" "image" { + name = "${azurerm_resource_group.openqa-group.name}-disk1" + location = var.region + resource_group_name = azurerm_resource_group.openqa-group.name + hyper_v_generation = var.sku == "gen1" ? "V1" : "V2" + count = var.image_id != "" ? 1 : 0 + + os_disk { + os_type = "Linux" + os_state = "Generalized" + blob_uri = "https://${var.storage-account}.blob.core.windows.net/sle-images/${var.image_id}" + size_gb = 30 + } +} + +resource "azurerm_linux_virtual_machine" "openqa-vm" { + name = "${var.name}-${element(random_id.service.*.hex, count.index)}" + resource_group_name = azurerm_resource_group.openqa-group.name + location = var.region + size = var.type + computer_name = "${var.name}-${element(random_id.service.*.hex, count.index)}" + admin_username = "azureuser" + disable_password_authentication = true + + count = var.instance_count + + network_interface_ids = [azurerm_network_interface.openqa-nic[count.index].id, azurerm_network_interface.openqa-secondary-nic[count.index].id] + + tags = merge({ + openqa_created_by = var.name + openqa_created_date = timestamp() + openqa_created_id = element(random_id.service.*.hex, count.index) + }, var.tags) + + admin_ssh_key { + username = "azureuser" + public_key = file("${var.ssh_public_key}") + } + + os_disk { + name = "${var.name}-${element(random_id.service.*.hex, count.index)}-osdisk" + caching = "ReadWrite" + storage_account_type = "Standard_LRS" + # SLE images are 30G by default. Uncomment this line in case we need to increase the disk size + # note: value can not be decreased because 30 GB is minimum allowed by Azure + # disk_size_gb = 30 + } + + source_image_id = var.image_uri != "" ? var.image_uri : (var.image_id != "" ? azurerm_image.image.0.id : null) + dynamic "source_image_reference" { + for_each = range(var.image_id == "" && var.image_uri == "" ? 1 : 0) + content { + publisher = var.image_id != "" ? "" : var.publisher + offer = var.image_id != "" ? "" : var.offer + sku = var.image_id != "" ? "" : var.sku + version = var.image_id != "" ? "" : "latest" + } + } + + boot_diagnostics { + /* Passing a null value will utilize a Managed Storage Account to store Boot Diagnostics */ + storage_account_uri = null + } + + timeouts { + create = var.vm_create_timeout + } + + custom_data = var.cloud_init != "" ? filebase64(var.cloud_init) : null +} + +resource "azurerm_virtual_machine_data_disk_attachment" "default" { + count = var.create-extra-disk ? var.instance_count : 0 + managed_disk_id = element(azurerm_managed_disk.ssd_disk.*.id, count.index) + virtual_machine_id = element(azurerm_linux_virtual_machine.openqa-vm.*.id, count.index) + lun = "1" + caching = "ReadWrite" +} + +resource "azurerm_managed_disk" "ssd_disk" { + count = var.create-extra-disk ? var.instance_count : 0 + name = "ssd-disk-${element(random_id.service.*.hex, count.index)}" + location = azurerm_resource_group.openqa-group.location + resource_group_name = azurerm_resource_group.openqa-group.name + storage_account_type = var.extra-disk-type + create_option = "Empty" + disk_size_gb = var.extra-disk-size +} + + +output "vm_name" { + value = azurerm_linux_virtual_machine.openqa-vm.*.id +} + +data "azurerm_public_ip" "openqa-publicip" { + name = azurerm_public_ip.openqa-publicip[count.index].name + resource_group_name = azurerm_linux_virtual_machine.openqa-vm.0.resource_group_name + count = var.instance_count +} + +output "public_ip" { + value = data.azurerm_public_ip.openqa-publicip.*.ip_address +} + +data "azurerm_public_ip" "openqa-secondary-publicip" { + name = azurerm_public_ip.openqa-secondary-publicip[count.index].name + resource_group_name = azurerm_linux_virtual_machine.openqa-vm.0.resource_group_name + count = var.instance_count +} + +output "secondary_public_ip" { + value = data.azurerm_public_ip.openqa-secondary-publicip.*.ip_address +} diff --git a/tests/publiccloud/cloud_netconfig.pm b/tests/publiccloud/cloud_netconfig.pm index 9d83bf7c16a9..ac80b42d3d2f 100644 --- a/tests/publiccloud/cloud_netconfig.pm +++ b/tests/publiccloud/cloud_netconfig.pm @@ -16,30 +16,20 @@ use Test::Assert 'assert_equals'; use testapi; use publiccloud::utils qw(is_azure is_ec2 is_gce); -sub run { - my ($self, $args) = @_; - my $instance = $args->{my_instance}; - my $pers_net_rules = '/etc/udev/rules.d/75-persistent-net-generator.rules'; +sub query_metadata { + my ($self, $instance, %args) = @_; + my $ifNum = $args{ifNum}; + my $addrCount = $args{addrCount}; # Cloud metadata service API is reachable at local destination # 169.254.169.254 in case of all public cloud providers. my $pc_meta_api_ip = '169.254.169.254'; - $instance->ssh_assert_script_run('systemctl is-enabled cloud-netconfig.service'); - - $instance->ssh_assert_script_run('systemctl status cloud-netconfig.timer'); - - # 75-persistent-net-generator.rules is usually a symlink to /dev/null in SLES 12+ - $instance->ssh_assert_script_run("test -L $pers_net_rules"); - - my $local_eth0_ip = $instance->ssh_script_output(qq(ip -4 -o a s eth0 | grep -Po "inet \\K[\\d.]+")); - chomp($local_eth0_ip); - my $query_meta_ipv4_cmd = ""; if (is_azure()) { - $query_meta_ipv4_cmd = qq(curl -H Metadata:true "http://$pc_meta_api_ip/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2023-07-01&format=text"); + $query_meta_ipv4_cmd = qq(curl -H Metadata:true "http://$pc_meta_api_ip/metadata/instance/network/interface/$ifNum/ipv4/ipAddress/$addrCount/privateIpAddress?api-version=2023-07-01&format=text"); } elsif (is_ec2()) { - my $access_token = $instance->ssh_script_output(qq(curl -X PUT http://$pc_meta_api_ip/latest/api/token -H "X-aws-ec2-metadata-token-ttl-seconds: 60")); + my $access_token = $instance->ssh_script_output(qq(curl -X PUT http://$pc_meta_api_ip/latest/api/token -H "X-aws-ec2-metadata-token-ttl-seconds:60")); record_info("DEBUG", $access_token); $query_meta_ipv4_cmd = qq(curl -H "X-aws-ec2-metadata-token: $access_token" "http://$pc_meta_api_ip/latest/meta-data/local-ipv4"); } elsif (is_gce()) { @@ -47,9 +37,81 @@ sub run { } else { die("Unsupported public cloud provider"); } - my $metadata_eth0_ip = $instance->ssh_script_output($query_meta_ipv4_cmd); - die("Failed to get IP from metadata server") unless length($metadata_eth0_ip); - assert_equals($metadata_eth0_ip, $local_eth0_ip, 'Locally assigned IP does not equal the IP retrieved from CSP metadata service.'); + + my $data = $instance->ssh_script_output($query_meta_ipv4_cmd); + + return $data; +} + +sub run { + my ($self, $args) = @_; + my $instance = $args->{my_instance}; + $self->{instance} = $args->{my_instance}; + my $pers_net_rules = '/etc/udev/rules.d/75-persistent-net-generator.rules'; + + $instance->ssh_assert_script_run('systemctl is-enabled cloud-netconfig.service'); + + $instance->ssh_assert_script_run('systemctl status cloud-netconfig.timer'); + + # 75-persistent-net-generator.rules is usually a symlink to /dev/null in SLES 12+ + $instance->ssh_assert_script_run("test -L $pers_net_rules"); + + record_info('eth0'); + my $local_eth0_ip = $instance->ssh_script_output(qq(ip -4 -o a s eth0 | grep -Po "inet \\K[\\d.]+" | head -n1)); + chomp($local_eth0_ip); + my $metadata_eth0_ip = $self->query_metadata($instance, ifNum => '0', addrCount => '0'); + die("Failed to get interface 0 IP from metadata server") unless length($metadata_eth0_ip); + assert_equals($metadata_eth0_ip, $local_eth0_ip, 'Locally assigned eth1 IP does not equal the IP retrieved from CSP metadata service.'); + + if (is_azure) { + record_info('eth1'); + my $local_eth1_ip = $instance->ssh_script_output(qq(ip -4 -o a s eth1 | grep -Po "inet \\K[\\d.]+" | head -n1)); + chomp($local_eth1_ip); + my $metadata_eth1_ip = $self->query_metadata($instance, ifNum => '1', addrCount => '0'); + die("Failed to get interface 1 IP from metadata server") unless length($metadata_eth1_ip); + assert_equals($metadata_eth1_ip, $local_eth1_ip, 'Locally assigned eth1 IP does not equal the IP retrieved from CSP metadata service.'); + + record_info('secondary'); + $instance->ssh_assert_script_run('ip a s dev eth0 secondary'); + $instance->ssh_assert_script_run('ip a s dev eth1 secondary'); + + my $eth0_public_ip = $instance->ssh_script_output('curl --interface eth0 -s https://wtfismyip.com/text'); + my $eth1_public_ip = $instance->ssh_script_output('curl --interface eth1 -s https://wtfismyip.com/text'); + assert_not_equals($eth0_public_ip, $eth1_public_ip, "The connection from eth0 shouldn't have the same public IPv4 address as connection from eth1"); + } +} + +sub post_fail_hook { + my $self = shift; + + debug($self->{instance}); + $self->SUPER::post_fail_hook; +} + +sub post_run_hook { + my $self = shift; + + debug($self->{instance}); + $self->SUPER::post_run_hook; +} + +sub debug { + my $instance = shift; + record_info('DEBUG'); + + $instance->ssh_script_run("ip -j a s | jq -r '.'"); + + $instance->ssh_script_run("ip a s"); + $instance->ssh_script_run("ip -6 a s"); + + $instance->ssh_script_run("ip route list table all"); + $instance->ssh_script_run("ip -6 route list table all"); + + $instance->ssh_script_run("ip rule list all"); + $instance->ssh_script_run("ip -6 rule list all"); + + $instance->ssh_script_run("ip neighbor show"); + $instance->ssh_script_run("ip -6 neighbor show"); } 1;