diff --git a/cmd/netbox-ssot/main.go b/cmd/netbox-ssot/main.go index f04587cd..198bae52 100644 --- a/cmd/netbox-ssot/main.go +++ b/cmd/netbox-ssot/main.go @@ -107,7 +107,7 @@ func main() { } wg.Wait() - // Orphan manager cleanup + // Orphan manager cleanup on successful run and if enabled if config.Netbox.RemoveOrphans && successfullRun { ssotLogger.Info(mainCtx, "Cleaning up orphaned objects...") err = netboxInventory.DeleteOrphans(mainCtx) diff --git a/go.sum b/go.sum index 5a349287..415dffaf 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,6 @@ github.com/PaloAltoNetworks/pango v0.10.2 h1:Tjn6vIzzAq6Dd7N0mDuiP8w8pz8k5W9zz/T github.com/PaloAltoNetworks/pango v0.10.2/go.mod h1:GztcRnVLur7G+VFG7Z5ZKNFgScLtsycwPMp1qVebE5g= github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY= github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE= -github.com/cisco-en-programmability/dnacenter-go-sdk/v5 v5.0.25 h1:H9i9u7ADeiuMzfkJIzQUg/noTWA82DmAehcAwlIfBPE= -github.com/cisco-en-programmability/dnacenter-go-sdk/v5 v5.0.25/go.mod h1:4Km+JuiyL/LsNRvO4dMWUSUVbnNBRmbwzJMU1oUbn0E= github.com/cisco-en-programmability/dnacenter-go-sdk/v5 v5.0.26 h1:qLIr8VW60CKwjNJydytrnukETVb0FSZKTO/QKqE9STA= github.com/cisco-en-programmability/dnacenter-go-sdk/v5 v5.0.26/go.mod h1:4Km+JuiyL/LsNRvO4dMWUSUVbnNBRmbwzJMU1oUbn0E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/internal/netbox/inventory/delete_items.go b/internal/netbox/inventory/delete_items.go index 05a9a681..da9301f3 100644 --- a/internal/netbox/inventory/delete_items.go +++ b/internal/netbox/inventory/delete_items.go @@ -15,9 +15,11 @@ func (nbi *NetboxInventory) DeleteOrphans(ctx context.Context) error { if len(ids) != 0 { nbi.Logger.Infof(ctx, "Deleting orphaned objects of type %s", objectAPIPath) nbi.Logger.Debugf(ctx, "Ids of objects to be deleted: %v", ids) - err := nbi.NetboxAPI.BulkDeleteObjects(ctx, objectAPIPath, ids) - if err != nil { - return err + for id := range ids { + err := nbi.NetboxAPI.DeleteObject(ctx, objectAPIPath, id) + if err != nil { + nbi.Logger.Errorf(nbi.Ctx, "delete objects: %s", err) + } } } } diff --git a/internal/netbox/service/rest.go b/internal/netbox/service/rest.go index 1186b51a..16687093 100644 --- a/internal/netbox/service/rest.go +++ b/internal/netbox/service/rest.go @@ -201,3 +201,20 @@ func (api *NetboxClient) BulkDeleteObjects(ctx context.Context, objectPath strin return nil } + +// Function that deletes objectas on path objectPath. +// It deletes a single object at a time. It is alternative to bulk delete +// because if one delete fails other still go. +func (api *NetboxClient) DeleteObject(ctx context.Context, objectPath string, id int) error { + api.Logger.Debugf(ctx, "Deleting object with id %d on route %s", id, objectPath) + + response, err := api.doRequest(MethodDelete, fmt.Sprintf("%s%d/", objectPath, id), nil) + if err != nil { + return err + } + + if response.StatusCode != http.StatusNoContent { + return fmt.Errorf("unexpected status code: %d: %s", response.StatusCode, response.Body) + } + return nil +} diff --git a/internal/source/fmc/fmc_client.go b/internal/source/fmc/fmc_client.go index 295dab70..119c6f86 100644 --- a/internal/source/fmc/fmc_client.go +++ b/internal/source/fmc/fmc_client.go @@ -7,24 +7,29 @@ import ( "fmt" "io" "net/http" + "time" + + "github.com/bl4ko/netbox-ssot/internal/constants" ) type fmcClient struct { - HTTPClient *http.Client - BaseURL string - Username string - Password string - AccessToken string - RefreshToken string + HTTPClient *http.Client + BaseURL string + Username string + Password string + AccessToken string + RefreshToken string + DefaultTimeout time.Duration } func newFMCClient(username string, password string, httpScheme string, hostname string, port int, httpClient *http.Client) (*fmcClient, error) { // First we obtain access and refresh token c := &fmcClient{ - HTTPClient: httpClient, - BaseURL: fmt.Sprintf("%s://%s:%d/api", httpScheme, hostname, port), - Username: username, - Password: password, + HTTPClient: httpClient, + BaseURL: fmt.Sprintf("%s://%s:%d/api", httpScheme, hostname, port), + Username: username, + Password: password, + DefaultTimeout: time.Second * constants.DefaultAPITimeout, } aToken, rToken, err := c.Authenticate() @@ -40,8 +45,8 @@ func newFMCClient(username string, password string, httpScheme string, hostname // Authenticate performs authentication on FMC API. If successful it returns access and refresh tokens. func (fmcc fmcClient) Authenticate() (string, string, error) { - ctx := context.Background() - defer ctx.Done() + ctx, cancel := context.WithTimeout(context.Background(), fmcc.DefaultTimeout) + defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("%s/fmc_platform/v1/auth/generatetoken", fmcc.BaseURL), nil) if err != nil { return "", "", fmt.Errorf("new request with context: %w", err) @@ -97,6 +102,8 @@ type Device struct { } func (fmcc *fmcClient) MakeRequest(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) { + ctx, cancel := context.WithTimeout(ctx, fmcc.DefaultTimeout) + defer cancel() req, err := http.NewRequestWithContext(ctx, method, fmt.Sprintf("%s/%s", fmcc.BaseURL, path), body) if err != nil { return nil, err @@ -112,7 +119,7 @@ func (fmcc *fmcClient) GetDomains() ([]Domain, error) { domains := []Domain{} ctx := context.Background() for { - apiResponse, err := fmcc.MakeRequest(ctx, http.MethodGet, "fmc_platform/v1/info/domain", nil) + apiResponse, err := fmcc.MakeRequest(ctx, http.MethodGet, fmt.Sprintf("fmc_platform/v1/info/domain?offset=%d&limit=%d", offset, limit), nil) if err != nil { return nil, fmt.Errorf("make request for domains: %w", err) } @@ -134,7 +141,7 @@ func (fmcc *fmcClient) GetDomains() ([]Domain, error) { domains = append(domains, marshaledResponse.Items...) } - if marshaledResponse.Paging.Count < marshaledResponse.Paging.Limit { + if len(marshaledResponse.Items) < limit { break } offset += limit @@ -147,7 +154,7 @@ func (fmcc *fmcClient) GetDevices(domainUUID string) ([]Device, error) { limit := 25 devices := []Device{} ctx := context.Background() - devicesURL := fmt.Sprintf("fmc_config/v1/domain/%s/devices/devicerecords", domainUUID) + devicesURL := fmt.Sprintf("fmc_config/v1/domain/%s/devices/devicerecords?offset=%d&limit=%d", domainUUID, offset, limit) for { apiResponse, err := fmcc.MakeRequest(ctx, http.MethodGet, devicesURL, nil) if err != nil { @@ -171,7 +178,7 @@ func (fmcc *fmcClient) GetDevices(domainUUID string) ([]Device, error) { devices = append(devices, marshaledResponse.Items...) } - if marshaledResponse.Paging.Count < marshaledResponse.Paging.Limit { + if len(marshaledResponse.Items) < limit { break } offset += limit @@ -190,7 +197,7 @@ func (fmcc *fmcClient) GetDevicePhysicalInterfaces(domainUUID string, deviceID s limit := 25 pIfaces := []PhysicalInterface{} ctx := context.Background() - pInterfacesURL := fmt.Sprintf("fmc_config/v1/domain/%s/devices/devicerecords/%s/physicalinterfaces", domainUUID, deviceID) + pInterfacesURL := fmt.Sprintf("fmc_config/v1/domain/%s/devices/devicerecords/%s/physicalinterfaces?offset=%d&limit=%d", domainUUID, deviceID, offset, limit) for { apiResponse, err := fmcc.MakeRequest(ctx, http.MethodGet, pInterfacesURL, nil) if err != nil { @@ -214,7 +221,7 @@ func (fmcc *fmcClient) GetDevicePhysicalInterfaces(domainUUID string, deviceID s pIfaces = append(pIfaces, marshaledResponse.Items...) } - if marshaledResponse.Paging.Count < marshaledResponse.Paging.Limit { + if len(marshaledResponse.Items) < limit { break } offset += limit @@ -233,7 +240,7 @@ func (fmcc *fmcClient) GetDeviceVLANInterfaces(domainUUID string, deviceID strin limit := 25 vlanIfaces := []VlanInterface{} ctx := context.Background() - pInterfacesURL := fmt.Sprintf("fmc_config/v1/domain/%s/devices/devicerecords/%s/vlaninterfaces", domainUUID, deviceID) + pInterfacesURL := fmt.Sprintf("fmc_config/v1/domain/%s/devices/devicerecords/%s/vlaninterfaces?offset=%d&limit=%d", domainUUID, deviceID, offset, limit) for { apiResponse, err := fmcc.MakeRequest(ctx, http.MethodGet, pInterfacesURL, nil) if err != nil { @@ -257,7 +264,7 @@ func (fmcc *fmcClient) GetDeviceVLANInterfaces(domainUUID string, deviceID strin vlanIfaces = append(vlanIfaces, marshaledResponse.Items...) } - if marshaledResponse.Paging.Count < marshaledResponse.Paging.Limit { + if len(marshaledResponse.Items) < limit { break } offset += limit