DE19227:Terraform replace of a host fails when the host is in Failed …

internal/acceptance_test/resource_host_test.go

@@ -1,4 +1,4 @@
-// (C) Copyright 2020-2023 Hewlett Packard Enterprise Development LP
+// (C) Copyright 2020-2024 Hewlett Packard Enterprise Development LP
 
 package acceptance_test
 
@@ -18,7 +18,7 @@ import (
 
 const (
 	hostStateReadyWait = 30 * time.Second
-	hostStatePollCount = 4
+	hostStatePollCount = 10
 	isAsync            = true
 	isNotAsync         = false
 )

internal/resources/resource_host.go

@@ -3,11 +3,12 @@
 package resources
 
 import (
+	"context"
 	"fmt"
 	"strings"
 	"time"
 
-	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/retry"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 
 	rest "github.com/hewlettpackard/hpegl-metal-client/v1/pkg/client"
@@ -494,7 +495,7 @@ func resourceMetalHostCreate(d *schema.ResourceData, meta interface{}) (err erro
 	}
 
 	// host create is asynchronous in Metal svc. Wait until host state is Ready.
-	createStateConf := &resource.StateChangeConf{
+	createStateConf := &retry.StateChangeConf{
 		Pending: []string{
 			string(rest.HOSTSTATE_NEW),
 			string(rest.HOSTSTATE_IMAGING_PREP),
@@ -789,7 +790,7 @@ func resourceMetalHostUpdate(d *schema.ResourceData, meta interface{}) (err erro
 	}
 
 	// host update is asynchronous in Metal svc. Wait until host state is Ready.
-	updateStateConf := &resource.StateChangeConf{
+	updateStateConf := &retry.StateChangeConf{
 		Pending: []string{
 			string(rest.HOSTSTATE_UPDATING_CONNECTIONS),
 			string(rest.HOSTSTATE_CONNECTING),
@@ -826,7 +827,6 @@ func resourceMetalHostDelete(d *schema.ResourceData, meta interface{}) (err erro
 	if err != nil {
 		return err
 	}
-	var host rest.Host
 
 	defer func() {
 		// This is the last in the deferred chain to fire. If there has been no
@@ -839,7 +839,8 @@ func resourceMetalHostDelete(d *schema.ResourceData, meta interface{}) (err erro
 	}()
 
 	ctx := p.GetContext()
-	host, _, err = p.Client.HostsApi.GetByID(ctx, d.Id())
+
+	host, _, err := p.Client.HostsApi.GetByID(ctx, d.Id())
 	if err != nil {
 		return err
 	}
@@ -848,38 +849,14 @@ func resourceMetalHostDelete(d *schema.ResourceData, meta interface{}) (err erro
 		return nil
 	}
 
-	if host.State != rest.HOSTSTATE_READY {
-		// Hosts that are still prvisioning can be
-		// deleted immediately.
-		_, err = p.Client.HostsApi.Delete(ctx, d.Id())
-
-		return err
-	}
-
-	// Hosts that are powered-on can not be deleted directly, so flip the power.
-	if host.PowerStatus == rest.HOSTPOWERSTATE_ON {
-		ctx = p.GetContext()
-		_, _, err = p.Client.HostsApi.PowerOff(ctx, d.Id())
-		if err != nil {
+	// Hosts that are in the Ready state and powered-on can not be deleted while the
+	// power is on, so turn off the power.
+	if host.State == rest.HOSTSTATE_READY && host.PowerStatus == rest.HOSTPOWERSTATE_ON {
+		if err := powerOffHost(ctx, p.Client.HostsApi, d.Id(), d.Timeout(schema.TimeoutDefault)); err != nil {
 			return err
 		}
-		// The call is asynchronous so wait for Metal svc to complete the request.
-		for host.PowerStatus != rest.HOSTPOWERSTATE_OFF {
-			time.Sleep(pollInterval)
-
-			host, _, err = p.Client.HostsApi.GetByID(ctx, d.Id())
-			if err != nil {
-				return err
-			}
-
-			if host.State == rest.HOSTSTATE_FAILED {
-				return fmt.Errorf("failed to turn off host power")
-			}
-		}
 	}
 
-	ctx = p.GetContext()
-
 	if _, err := p.Client.HostsApi.Delete(ctx, d.Id()); err != nil {
 		//nolint:wrapcheck // defer func is wrapping the error.
 		return err
@@ -889,7 +866,7 @@ func resourceMetalHostDelete(d *schema.ResourceData, meta interface{}) (err erro
 	// reference to the host until it has really gone from Metal svc. If we delete the
 	// reference too early, or in the presence of errors, we will never be able to retry
 	// the delete operation from Terraform (since it has no reference to the resource).
-	deleteStateConf := &resource.StateChangeConf{
+	deleteStateConf := &retry.StateChangeConf{
 		Pending: []string{
 			string(rest.HOSTSTATE_DELETING),
 		},
@@ -916,6 +893,41 @@ func resourceMetalHostDelete(d *schema.ResourceData, meta interface{}) (err erro
 	return nil
 }
 
+func powerOffHost(ctx context.Context, hostAPI rest.HostsAPI, hostID string, timeout time.Duration) error {
+	_, _, err := hostAPI.PowerOff(ctx, hostID)
+	if err != nil {
+		return fmt.Errorf("power off host %v: %v", hostID, err)
+	}
+
+	// The power-off call is asynchronous so wait for Metal svc to complete the request.
+	powerOffStateConf := &retry.StateChangeConf{
+		Pending: []string{
+			string(rest.HOSTPOWERSTATE_UNKNOWN),
+			string(rest.HOSTPOWERSTATE_ON),
+		},
+		Target: []string{
+			string(rest.HOSTPOWERSTATE_OFF),
+		},
+		Refresh: func() (interface{}, string, error) {
+			host, _, err := hostAPI.GetByID(ctx, hostID)
+			if err != nil {
+				return nil, "", fmt.Errorf("get host %v", hostID)
+			}
+
+			return host, string(host.PowerStatus), nil
+		},
+		Timeout:    timeout,
+		Delay:      mediumTimeout,
+		MinTimeout: shortTimeout,
+	}
+
+	if _, err := powerOffStateConf.WaitForStateContext(ctx); err != nil {
+		return fmt.Errorf("waiting for host instance (%v) to be powered off: %v", hostID, err)
+	}
+
+	return nil
+}
+
 // volumeExists returns true & the volume ID, if the input matches
 // either the ID or the name from existing volumes.
 func volumeExists(vID string, volumes []rest.Volume) (string, bool) {