fix: fallback from bulk delete to delete single object

This is useful in case if one object deletion fails, in bulkdelete also all others fail, but when deleting single object, only the one fails

cmd/netbox-ssot/main.go

@@ -107,7 +107,7 @@ func main() {
 	}
 	wg.Wait()
 
-	// Orphan manager cleanup
+	// Orphan manager cleanup on successful run and if enabled
 	if config.Netbox.RemoveOrphans && successfullRun {
 		ssotLogger.Info(mainCtx, "Cleaning up orphaned objects...")
 		err = netboxInventory.DeleteOrphans(mainCtx)

go.sum

@@ -2,8 +2,6 @@ github.com/PaloAltoNetworks/pango v0.10.2 h1:Tjn6vIzzAq6Dd7N0mDuiP8w8pz8k5W9zz/T
 github.com/PaloAltoNetworks/pango v0.10.2/go.mod h1:GztcRnVLur7G+VFG7Z5ZKNFgScLtsycwPMp1qVebE5g=
 github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY=
 github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE=
-github.com/cisco-en-programmability/dnacenter-go-sdk/v5 v5.0.25 h1:H9i9u7ADeiuMzfkJIzQUg/noTWA82DmAehcAwlIfBPE=
-github.com/cisco-en-programmability/dnacenter-go-sdk/v5 v5.0.25/go.mod h1:4Km+JuiyL/LsNRvO4dMWUSUVbnNBRmbwzJMU1oUbn0E=
 github.com/cisco-en-programmability/dnacenter-go-sdk/v5 v5.0.26 h1:qLIr8VW60CKwjNJydytrnukETVb0FSZKTO/QKqE9STA=
 github.com/cisco-en-programmability/dnacenter-go-sdk/v5 v5.0.26/go.mod h1:4Km+JuiyL/LsNRvO4dMWUSUVbnNBRmbwzJMU1oUbn0E=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=

internal/netbox/inventory/delete_items.go

@@ -15,9 +15,11 @@ func (nbi *NetboxInventory) DeleteOrphans(ctx context.Context) error {
 		if len(ids) != 0 {
 			nbi.Logger.Infof(ctx, "Deleting orphaned objects of type %s", objectAPIPath)
 			nbi.Logger.Debugf(ctx, "Ids of objects to be deleted: %v", ids)
-			err := nbi.NetboxAPI.BulkDeleteObjects(ctx, objectAPIPath, ids)
-			if err != nil {
-				return err
+			for id := range ids {
+				err := nbi.NetboxAPI.DeleteObject(ctx, objectAPIPath, id)
+				if err != nil {
+					nbi.Logger.Errorf(nbi.Ctx, "delete objects: %s", err)
+				}
 			}
 		}
 	}

internal/netbox/service/rest.go

@@ -201,3 +201,20 @@ func (api *NetboxClient) BulkDeleteObjects(ctx context.Context, objectPath strin
 
 	return nil
 }
+
+// Function that deletes objectas on path objectPath.
+// It deletes a single object at a time. It is alternative to bulk delete
+// because if one delete fails other still go.
+func (api *NetboxClient) DeleteObject(ctx context.Context, objectPath string, id int) error {
+	api.Logger.Debugf(ctx, "Deleting object with id %d on route %s", id, objectPath)
+
+	response, err := api.doRequest(MethodDelete, fmt.Sprintf("%s%d/", objectPath, id), nil)
+	if err != nil {
+		return err
+	}
+
+	if response.StatusCode != http.StatusNoContent {
+		return fmt.Errorf("unexpected status code: %d: %s", response.StatusCode, response.Body)
+	}
+	return nil
+}

internal/source/fmc/fmc_client.go

@@ -7,24 +7,29 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"time"
+
+	"github.com/bl4ko/netbox-ssot/internal/constants"
 )
 
 type fmcClient struct {
-	HTTPClient   *http.Client
-	BaseURL      string
-	Username     string
-	Password     string
-	AccessToken  string
-	RefreshToken string
+	HTTPClient     *http.Client
+	BaseURL        string
+	Username       string
+	Password       string
+	AccessToken    string
+	RefreshToken   string
+	DefaultTimeout time.Duration
 }
 
 func newFMCClient(username string, password string, httpScheme string, hostname string, port int, httpClient *http.Client) (*fmcClient, error) {
 	// First we obtain access and refresh token
 	c := &fmcClient{
-		HTTPClient: httpClient,
-		BaseURL:    fmt.Sprintf("%s://%s:%d/api", httpScheme, hostname, port),
-		Username:   username,
-		Password:   password,
+		HTTPClient:     httpClient,
+		BaseURL:        fmt.Sprintf("%s://%s:%d/api", httpScheme, hostname, port),
+		Username:       username,
+		Password:       password,
+		DefaultTimeout: time.Second * constants.DefaultAPITimeout,
 	}
 
 	aToken, rToken, err := c.Authenticate()
@@ -40,8 +45,8 @@ func newFMCClient(username string, password string, httpScheme string, hostname
 
 // Authenticate performs authentication on FMC API. If successful it returns access and refresh tokens.
 func (fmcc fmcClient) Authenticate() (string, string, error) {
-	ctx := context.Background()
-	defer ctx.Done()
+	ctx, cancel := context.WithTimeout(context.Background(), fmcc.DefaultTimeout)
+	defer cancel()
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("%s/fmc_platform/v1/auth/generatetoken", fmcc.BaseURL), nil)
 	if err != nil {
 		return "", "", fmt.Errorf("new request with context: %w", err)
@@ -97,6 +102,8 @@ type Device struct {
 }
 
 func (fmcc *fmcClient) MakeRequest(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) {
+	ctx, cancel := context.WithTimeout(ctx, fmcc.DefaultTimeout)
+	defer cancel()
 	req, err := http.NewRequestWithContext(ctx, method, fmt.Sprintf("%s/%s", fmcc.BaseURL, path), body)
 	if err != nil {
 		return nil, err
@@ -112,7 +119,7 @@ func (fmcc *fmcClient) GetDomains() ([]Domain, error) {
 	domains := []Domain{}
 	ctx := context.Background()
 	for {
-		apiResponse, err := fmcc.MakeRequest(ctx, http.MethodGet, "fmc_platform/v1/info/domain", nil)
+		apiResponse, err := fmcc.MakeRequest(ctx, http.MethodGet, fmt.Sprintf("fmc_platform/v1/info/domain?offset=%d&limit=%d", offset, limit), nil)
 		if err != nil {
 			return nil, fmt.Errorf("make request for domains: %w", err)
 		}
@@ -134,7 +141,7 @@ func (fmcc *fmcClient) GetDomains() ([]Domain, error) {
 			domains = append(domains, marshaledResponse.Items...)
 		}
 
-		if marshaledResponse.Paging.Count < marshaledResponse.Paging.Limit {
+		if len(marshaledResponse.Items) < limit {
 			break
 		}
 		offset += limit
@@ -147,7 +154,7 @@ func (fmcc *fmcClient) GetDevices(domainUUID string) ([]Device, error) {
 	limit := 25
 	devices := []Device{}
 	ctx := context.Background()
-	devicesURL := fmt.Sprintf("fmc_config/v1/domain/%s/devices/devicerecords", domainUUID)
+	devicesURL := fmt.Sprintf("fmc_config/v1/domain/%s/devices/devicerecords?offset=%d&limit=%d", domainUUID, offset, limit)
 	for {
 		apiResponse, err := fmcc.MakeRequest(ctx, http.MethodGet, devicesURL, nil)
 		if err != nil {
@@ -171,7 +178,7 @@ func (fmcc *fmcClient) GetDevices(domainUUID string) ([]Device, error) {
 			devices = append(devices, marshaledResponse.Items...)
 		}
 
-		if marshaledResponse.Paging.Count < marshaledResponse.Paging.Limit {
+		if len(marshaledResponse.Items) < limit {
 			break
 		}
 		offset += limit
@@ -190,7 +197,7 @@ func (fmcc *fmcClient) GetDevicePhysicalInterfaces(domainUUID string, deviceID s
 	limit := 25
 	pIfaces := []PhysicalInterface{}
 	ctx := context.Background()
-	pInterfacesURL := fmt.Sprintf("fmc_config/v1/domain/%s/devices/devicerecords/%s/physicalinterfaces", domainUUID, deviceID)
+	pInterfacesURL := fmt.Sprintf("fmc_config/v1/domain/%s/devices/devicerecords/%s/physicalinterfaces?offset=%d&limit=%d", domainUUID, deviceID, offset, limit)
 	for {
 		apiResponse, err := fmcc.MakeRequest(ctx, http.MethodGet, pInterfacesURL, nil)
 		if err != nil {
@@ -214,7 +221,7 @@ func (fmcc *fmcClient) GetDevicePhysicalInterfaces(domainUUID string, deviceID s
 			pIfaces = append(pIfaces, marshaledResponse.Items...)
 		}
 
-		if marshaledResponse.Paging.Count < marshaledResponse.Paging.Limit {
+		if len(marshaledResponse.Items) < limit {
 			break
 		}
 		offset += limit
@@ -233,7 +240,7 @@ func (fmcc *fmcClient) GetDeviceVLANInterfaces(domainUUID string, deviceID strin
 	limit := 25
 	vlanIfaces := []VlanInterface{}
 	ctx := context.Background()
-	pInterfacesURL := fmt.Sprintf("fmc_config/v1/domain/%s/devices/devicerecords/%s/vlaninterfaces", domainUUID, deviceID)
+	pInterfacesURL := fmt.Sprintf("fmc_config/v1/domain/%s/devices/devicerecords/%s/vlaninterfaces?offset=%d&limit=%d", domainUUID, deviceID, offset, limit)
 	for {
 		apiResponse, err := fmcc.MakeRequest(ctx, http.MethodGet, pInterfacesURL, nil)
 		if err != nil {
@@ -257,7 +264,7 @@ func (fmcc *fmcClient) GetDeviceVLANInterfaces(domainUUID string, deviceID strin
 			vlanIfaces = append(vlanIfaces, marshaledResponse.Items...)
 		}
 
-		if marshaledResponse.Paging.Count < marshaledResponse.Paging.Limit {
+		if len(marshaledResponse.Items) < limit {
 			break
 		}
 		offset += limit