diff --git a/README.md b/README.md index 82cebcce7..3e87e1029 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Content Sources ## What is it? -Content Sources is an application for storing information about external content (currently YUM repositories) in a central location as well as creating snapshots of those repositories, backed by a Pulp server. +Content Sources is an application for storing information about external content (currently YUM repositories) in a central location as well as creating snapshots of those repositories, backed by a Pulp server. ## Developing @@ -16,27 +16,27 @@ Content Sources is an application for storing information about external content Create a config file from the example: - ```sh - $ cp ./configs/config.yaml.example ./configs/config.yaml - ``` +```sh +$ cp ./configs/config.yaml.example ./configs/config.yaml +``` ### Build needed kafka container - ```sh - $ make compose-build - ``` +```sh +$ make compose-build +``` ### Start dependency containers - ```sh - $ make compose-up - ``` +```sh +$ make compose-up +``` ### Run the server! - ```sh - $ make run - ``` +```sh +$ make run +``` ### @@ -44,39 +44,40 @@ Hit the API: ```sh $ curl -H "$( ./scripts/header.sh 9999 1111 )" http://localhost:8000/api/content-sources/v1.0/repositories/ - ``` +``` ### Stop dependency containers + When its time to shut down the running containers: - ```sh - $ make compose-down - ``` +```sh +$ make compose-down +``` And clean the volume that it uses by (this stops the container before doing it if it were running): - ```sh - $ make compose-clean - ``` - -> There are other make rules that could be helpful, run `make help` to list them. Some are highlighted below +```sh +$ make compose-clean +``` +> There are other make rules that could be helpful, run `make help` to list them. Some are highlighted below ### HOW TO ADD NEW MIGRATION FILES -You can add new migration files, with the prefixed date attached to the file name, by running the following: + +You can add new migration files, with the prefixed date attached to the file name, by running the following: ``` $ go run cmd/dbmigrate/main.go new ``` ### Database Commands + Migrate the Database ```sh $ make db-migrate-up ``` - Seed the database ```sh @@ -85,9 +86,9 @@ $ make db-migrate-seed Get an interactive shell: - ```sh - $ make db-shell - ``` +```sh +$ make db-shell +``` Or open directly a postgres client by running: @@ -125,7 +126,6 @@ Update the `configs/prometheus.yaml` file to set your hostname instead of `local $ cat ./configs/prometheus.example.yaml | sed "s/localhost/$(hostname)/g" > ./configs/prometheus.yaml ``` - To start prometheus run: ```sh @@ -157,8 +157,8 @@ $ make prometheus-ui rbac_timeout: 30 mocks: rbac: - user_read_write: ["jdoe@example.com","jdoe"] - user_read: ["tdoe@example.com","tdoe"] + user_read_write: ["jdoe@example.com", "jdoe"] + user_read: ["tdoe@example.com", "tdoe"] ``` **Running it** @@ -200,13 +200,24 @@ $ curl -H "$( ./scripts/header.sh 9999 1111 )" http://localhost:8000/api/content $ make openapi ``` +### Generating/Update mocks: + +Ensure [mockery](https://vektra.github.io/mockery/latest/installation/) is installed. + +Then: + +```sh +$ go generate ./... +``` + ### Configuration -The default configuration file in ./configs/config.yaml.example shows all available config options. Any of these can be overridden with an environment variable. For example "database.name" can be passed in via an environment variable named "DATABASE_NAME". +The default configuration file in ./configs/config.yaml.example shows all available config options. Any of these can be overridden with an environment variable. For example "database.name" can be passed in via an environment variable named "DATABASE_NAME". ### Linting To use golangci-lint: + 1. `make install-golangci-lint` 2. `make lint` @@ -214,21 +225,21 @@ To use pre-commit linter: `make install-pre-commit` ### Code Layout -| Path | Description | -|-----------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [api](./api/) | Openapi docs and doc generation code | -| [db/migrations](./db/migrations/) | Database Migrations | | -| [pkg/api](./pkg/api) | API Structures that are used for handling data within our API Handlers | -| [pkg/config](./pkg/config) | Config loading and application bootstrapping code | -| [pkg/dao](./pkg/dao) | Database Access Object. Abstraction layer that provides an interface and implements it for our default database provider (postgresql). It is separated out for abstraction and easier testing | -| [pkg/db](./pkg/db) | Database connection and migration related code | -| [pkg/handler](./pkg/handler) | Methods that directly handle API requests | -| [pkg/middleware](./pkg/middleware)| Hold all the middleware components created for the service. | -| [pkg/event](./pkg/event) | Event message logic. Mre info [here](./pkg/event/README.md). | -| [pkg/models](./pkg/models) | Structs that represent database models (Gorm) | -| [pkg/seeds](./pkg/seeds) | Code to help seed the database for both development and testing | +| Path | Description | +| ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --- | +| [api](./api/) | Openapi docs and doc generation code | +| [db/migrations](./db/migrations/) | Database Migrations | | +| [pkg/api](./pkg/api) | API Structures that are used for handling data within our API Handlers | +| [pkg/config](./pkg/config) | Config loading and application bootstrapping code | +| [pkg/dao](./pkg/dao) | Database Access Object. Abstraction layer that provides an interface and implements it for our default database provider (postgresql). It is separated out for abstraction and easier testing | +| [pkg/db](./pkg/db) | Database connection and migration related code | +| [pkg/handler](./pkg/handler) | Methods that directly handle API requests | +| [pkg/middleware](./pkg/middleware) | Hold all the middleware components created for the service. | +| [pkg/event](./pkg/event) | Event message logic. Mre info [here](./pkg/event/README.md). | +| [pkg/models](./pkg/models) | Structs that represent database models (Gorm) | +| [pkg/seeds](./pkg/seeds) | Code to help seed the database for both development and testing | ## More info - * [Architecture](docs/architecture.md) - * [OpenApi Docs](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/content-services/content-sources-backend/main/api/openapi.json) +- [Architecture](docs/architecture.md) +- [OpenApi Docs](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/content-services/content-sources-backend/main/api/openapi.json) diff --git a/cmd/external-repos/main.go b/cmd/external-repos/main.go index 45574400b..c7db89afa 100644 --- a/cmd/external-repos/main.go +++ b/cmd/external-repos/main.go @@ -5,16 +5,19 @@ import ( "fmt" "os" "sort" + "time" "github.com/content-services/content-sources-backend/pkg/config" "github.com/content-services/content-sources-backend/pkg/dao" "github.com/content-services/content-sources-backend/pkg/db" "github.com/content-services/content-sources-backend/pkg/external_repos" + "github.com/content-services/content-sources-backend/pkg/pulp_client" "github.com/content-services/content-sources-backend/pkg/tasks/client" "github.com/content-services/content-sources-backend/pkg/tasks/payloads" "github.com/content-services/content-sources-backend/pkg/tasks/queue" _ "github.com/golang-migrate/migrate/v4/source/file" _ "github.com/lib/pq" + "github.com/openlyinc/pointy" "github.com/rs/zerolog/log" "gorm.io/gorm" ) @@ -33,7 +36,7 @@ func main() { } if len(args) < 2 { - log.Fatal().Msg("Requires arguments: download, import, introspect, nightly-jobs") + log.Fatal().Msg("Requires arguments: download, import, introspect, snapshot, nightly-jobs") } if args[1] == "download" { if len(args) < 3 { @@ -73,13 +76,31 @@ func main() { log.Panic().Err(errors[i]).Msg("Failed to introspect repository due to fatal errors") } log.Debug().Msgf("Inserted %d packages", count) + } else if args[1] == "snapshot" { + if len(args) < 3 { + log.Error().Msg("Usage: ./external_repos snapshot URL [URL2]...") + os.Exit(1) + } + var urls []string + for i := 2; i < len(args); i++ { + urls = append(urls, args[i]) + } + if config.Get().Features.Snapshots.Enabled { + waitForPulp() + err := enqueueSnapshotRepos(&urls) + if err != nil { + log.Warn().Msgf("Error enqueuing snapshot tasks: %v", err) + } + } else { + log.Warn().Msg("Snapshotting disabled") + } } else if args[1] == "nightly-jobs" { err = enqueueIntrospectAllRepos() if err != nil { log.Error().Err(err).Msg("error queueing introspection tasks") } if config.Get().Features.Snapshots.Enabled { - err = enqueueSyncAllRepos() + err = enqueueSnapshotRepos(nil) if err != nil { log.Error().Err(err).Msg("error queueing snapshot tasks") } @@ -110,6 +131,18 @@ func saveToDB(db *gorm.DB) error { return err } +func waitForPulp() { + for { + client := pulp_client.GetPulpClientWithDomain(context.Background(), pulp_client.DefaultDomain) + _, err := client.GetRpmRemoteList() + if err == nil { + return + } + log.Warn().Err(err).Msg("Pulp isn't up yet, waiting 5s.") + time.Sleep(5 * time.Second) + } +} + func scanForExternalRepos(path string) { urls, err := external_repos.IBUrlsFromDir(path) if err != nil { @@ -161,7 +194,7 @@ func enqueueIntrospectAllRepos() error { return nil } -func enqueueSyncAllRepos() error { +func enqueueSnapshotRepos(urls *[]string) error { q, err := queue.NewPgQueue(db.GetUrl()) if err != nil { return fmt.Errorf("error getting new task queue: %w", err) @@ -169,7 +202,15 @@ func enqueueSyncAllRepos() error { c := client.NewTaskClient(&q) repoConfigDao := dao.GetRepositoryConfigDao(db.DB) - repoConfigs, err := repoConfigDao.InternalOnly_ListReposToSnapshot() + var filter *dao.ListRepoFilter + if urls != nil { + filter = &dao.ListRepoFilter{ + URLs: urls, + RedhatOnly: pointy.Pointer(true), + } + } + repoConfigs, err := repoConfigDao.InternalOnly_ListReposToSnapshot(filter) + if err != nil { return fmt.Errorf("error getting repository configurations: %w", err) } diff --git a/deployments/deployment.yaml b/deployments/deployment.yaml index 02b1845b7..8fd53757c 100644 --- a/deployments/deployment.yaml +++ b/deployments/deployment.yaml @@ -52,6 +52,12 @@ objects: - introspect - https://cdn.redhat.com/content/dist/layered/rhel8/x86_64/ansible/2/os - https://cdn.redhat.com/content/dist/rhel8/8.8/x86_64/baseos/os + - name: snapshot-single-repo + inheritEnv: true + args: + - /external-repos + - snapshot + - https://cdn.redhat.com/content/dist/layered/rhel8/x86_64/ansible/2/os/ image: ${IMAGE}:${IMAGE_TAG} livenessProbe: failureThreshold: 3 diff --git a/pkg/cache/cache_mock.go b/pkg/cache/cache_mock.go index 6d2437342..f5dd3053c 100644 --- a/pkg/cache/cache_mock.go +++ b/pkg/cache/cache_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package cache diff --git a/pkg/dao/domain_dao_mock.go b/pkg/dao/domain_dao_mock.go index 07d556779..25fe99c3e 100644 --- a/pkg/dao/domain_dao_mock.go +++ b/pkg/dao/domain_dao_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package dao @@ -57,13 +57,12 @@ func (_m *MockDomainDao) FetchOrCreateDomain(orgId string) (string, error) { return r0, r1 } -type mockConstructorTestingTNewMockDomainDao interface { +// NewMockDomainDao creates a new instance of MockDomainDao. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockDomainDao(t interface { mock.TestingT Cleanup(func()) -} - -// NewMockDomainDao creates a new instance of MockDomainDao. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewMockDomainDao(t mockConstructorTestingTNewMockDomainDao) *MockDomainDao { +}) *MockDomainDao { mock := &MockDomainDao{} mock.Mock.Test(t) diff --git a/pkg/dao/interfaces.go b/pkg/dao/interfaces.go index 188f3581d..004167f5b 100644 --- a/pkg/dao/interfaces.go +++ b/pkg/dao/interfaces.go @@ -44,7 +44,7 @@ type RepositoryConfigDao interface { BulkCreate(newRepositories []api.RepositoryRequest) ([]api.RepositoryResponse, []error) Update(orgID, uuid string, repoParams api.RepositoryRequest) (bool, error) Fetch(orgID string, uuid string) (api.RepositoryResponse, error) - InternalOnly_ListReposToSnapshot() ([]models.RepositoryConfiguration, error) + InternalOnly_ListReposToSnapshot(filter *ListRepoFilter) ([]models.RepositoryConfiguration, error) List(orgID string, paginationData api.PaginationData, filterData api.FilterData) (api.RepositoryCollectionResponse, int64, error) Delete(orgID string, uuid string) error SoftDelete(orgID string, uuid string) error @@ -79,7 +79,7 @@ type RepositoryDao interface { //go:generate mockery --name SnapshotDao --filename snapshots_mock.go --inpackage type SnapshotDao interface { Create(snap *models.Snapshot) error - List(repoConfigUuid string, paginationData api.PaginationData, _ api.FilterData) (api.SnapshotCollectionResponse, int64, error) + List(orgID string, repoConfigUuid string, paginationData api.PaginationData, filterData api.FilterData) (api.SnapshotCollectionResponse, int64, error) FetchForRepoConfigUUID(repoConfigUUID string) ([]models.Snapshot, error) Delete(snapUUID string) error FetchLatestSnapshot(repoConfigUUID string) (api.SnapshotResponse, error) diff --git a/pkg/dao/metrics_mock.go b/pkg/dao/metrics_mock.go index 13bd67949..866df9ab7 100644 --- a/pkg/dao/metrics_mock.go +++ b/pkg/dao/metrics_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.20.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package dao @@ -79,13 +79,12 @@ func (_m *MockMetricsDao) RepositoryConfigsCount() int { return r0 } -type mockConstructorTestingTNewMockMetricsDao interface { +// NewMockMetricsDao creates a new instance of MockMetricsDao. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockMetricsDao(t interface { mock.TestingT Cleanup(func()) -} - -// NewMockMetricsDao creates a new instance of MockMetricsDao. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewMockMetricsDao(t mockConstructorTestingTNewMockMetricsDao) *MockMetricsDao { +}) *MockMetricsDao { mock := &MockMetricsDao{} mock.Mock.Test(t) diff --git a/pkg/dao/repositories_mock.go b/pkg/dao/repositories_mock.go index 987a0d570..33dafbf35 100644 --- a/pkg/dao/repositories_mock.go +++ b/pkg/dao/repositories_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.20.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package dao @@ -145,13 +145,12 @@ func (_m *MockRepositoryDao) Update(repo RepositoryUpdate) error { return r0 } -type mockConstructorTestingTNewMockRepositoryDao interface { +// NewMockRepositoryDao creates a new instance of MockRepositoryDao. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockRepositoryDao(t interface { mock.TestingT Cleanup(func()) -} - -// NewMockRepositoryDao creates a new instance of MockRepositoryDao. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewMockRepositoryDao(t mockConstructorTestingTNewMockRepositoryDao) *MockRepositoryDao { +}) *MockRepositoryDao { mock := &MockRepositoryDao{} mock.Mock.Test(t) diff --git a/pkg/dao/repository_configs.go b/pkg/dao/repository_configs.go index 8eac30d17..f0859ef5f 100644 --- a/pkg/dao/repository_configs.go +++ b/pkg/dao/repository_configs.go @@ -72,7 +72,11 @@ func DBErrorToApi(e error) *ce.DaoError { if ok { return &ce.DaoError{BadValidation: dbError.Validation, Message: dbError.Message} } - return &ce.DaoError{Message: e.Error()} + + return &ce.DaoError{ + Message: e.Error(), + NotFound: ce.HttpCodeForDaoError(e) == 404, //Check if isNotFoundError + } } func (r *repositoryConfigDaoImpl) InitializePulpClient(ctx context.Context, orgID string) error { @@ -94,6 +98,11 @@ func (r *repositoryConfigDaoImpl) InitializePulpClient(ctx context.Context, orgI func (r repositoryConfigDaoImpl) Create(newRepoReq api.RepositoryRequest) (api.RepositoryResponse, error) { var newRepo models.Repository var newRepoConfig models.RepositoryConfiguration + + if *newRepoReq.OrgID == config.RedHatOrg { + return api.RepositoryResponse{}, errors.New("Creating of Red Hat repositories is not permitted") + } + ApiFieldsToModel(newRepoReq, &newRepoConfig, &newRepo) cleanedUrl := models.CleanupURL(newRepo.URL) @@ -114,7 +123,7 @@ func (r repositoryConfigDaoImpl) Create(newRepoReq api.RepositoryRequest) (api.R } // reload the repoConfig to fetch repository info too - newRepoConfig, err := r.fetchRepoConfig(newRepoConfig.OrgID, newRepoConfig.UUID) + newRepoConfig, err := r.fetchRepoConfig(newRepoConfig.OrgID, newRepoConfig.UUID, false) if err != nil { return api.RepositoryResponse{}, DBErrorToApi(err) } @@ -162,23 +171,31 @@ func (r repositoryConfigDaoImpl) bulkCreate(tx *gorm.DB, newRepositories []api.R newRepoConfigs := make([]models.RepositoryConfiguration, size) newRepos := make([]models.Repository, size) responses := make([]api.RepositoryResponse, size) - errors := make([]error, size) + errorList := make([]error, size) tx.SavePoint("beforecreate") for i := 0; i < size; i++ { + if newRepoConfigs[i].OrgID == config.RedHatOrg { + errorList[i] = errors.New("Creating of Red Hat repositories is not permitted") + tx.RollbackTo("beforecreate") + continue + } + if newRepositories[i].OrgID != nil { newRepoConfigs[i].OrgID = *(newRepositories[i].OrgID) } + if newRepositories[i].AccountID != nil { newRepoConfigs[i].AccountID = *(newRepositories[i].AccountID) } + ApiFieldsToModel(newRepositories[i], &newRepoConfigs[i], &newRepos[i]) newRepos[i].Status = "Pending" cleanedUrl := models.CleanupURL(newRepos[i].URL) create := tx.Where("url = ?", cleanedUrl).FirstOrCreate(&newRepos[i]) if err := create.Error; err != nil { dbErr = DBErrorToApi(err) - errors[i] = dbErr + errorList[i] = dbErr tx.RollbackTo("beforecreate") continue } @@ -186,7 +203,7 @@ func (r repositoryConfigDaoImpl) bulkCreate(tx *gorm.DB, newRepositories []api.R newRepoConfigs[i].RepositoryUUID = newRepos[i].UUID if err := tx.Create(&newRepoConfigs[i]).Error; err != nil { dbErr = DBErrorToApi(err) - errors[i] = dbErr + errorList[i] = dbErr tx.RollbackTo("beforecreate") continue } @@ -203,24 +220,37 @@ func (r repositoryConfigDaoImpl) bulkCreate(tx *gorm.DB, newRepositories []api.R // If there is at least 1 error, return empty response slice. if dbErr == nil { return responses, []error{} - } else { - return []api.RepositoryResponse{}, errors } + return []api.RepositoryResponse{}, errorList +} + +type ListRepoFilter struct { + URLs *[]string + RedhatOnly *bool } -func (p repositoryConfigDaoImpl) InternalOnly_ListReposToSnapshot() ([]models.RepositoryConfiguration, error) { +func (p repositoryConfigDaoImpl) InternalOnly_ListReposToSnapshot(filter *ListRepoFilter) ([]models.RepositoryConfiguration, error) { var dbRepos []models.RepositoryConfiguration - var result *gorm.DB + var query *gorm.DB interval := fmt.Sprintf("%v hours", config.SnapshotInterval) if config.Get().Options.AlwaysRunCronTasks { - result = p.db.Where("snapshot IS TRUE").Find(&dbRepos) + query = p.db.Where("snapshot IS TRUE") } else { - result = p.db.Where("snapshot IS TRUE").Joins("LEFT JOIN tasks on last_snapshot_task_uuid = tasks.id"). + query = p.db.Where("snapshot IS TRUE").Joins("LEFT JOIN tasks on last_snapshot_task_uuid = tasks.id"). Where(p.db.Where("tasks.queued_at <= (current_date - cast(? as interval))", interval). Or("tasks.status NOT IN ?", []string{config.TaskStatusCompleted, config.TaskStatusPending, config.TaskStatusRunning}). - Or("last_snapshot_task_uuid is NULL")). - Find(&dbRepos) + Or("last_snapshot_task_uuid is NULL")) + } + if filter != nil { + query = query.Joins("INNER JOIN repositories r on r.uuid = repository_configurations.repository_uuid") + if filter.RedhatOnly != nil && *filter.RedhatOnly { + query = query.Where("r.origin = ?", config.OriginRedHat) + } + if filter.URLs != nil { + query = query.Where("r.url in ?", *filter.URLs) + } } + result := query.Find(&dbRepos) if result.Error != nil { return dbRepos, result.Error @@ -366,7 +396,7 @@ func (r repositoryConfigDaoImpl) InternalOnly_FetchRepoConfigsForRepoUUID(uuid s func (r repositoryConfigDaoImpl) Fetch(orgID string, uuid string) (api.RepositoryResponse, error) { var repo api.RepositoryResponse - repoConfig, err := r.fetchRepoConfig(orgID, uuid) + repoConfig, err := r.fetchRepoConfig(orgID, uuid, true) if err != nil { return api.RepositoryResponse{}, err } @@ -387,19 +417,26 @@ func (r repositoryConfigDaoImpl) Fetch(orgID string, uuid string) (api.Repositor return repo, nil } -func (r repositoryConfigDaoImpl) fetchRepoConfig(orgID string, uuid string) (models.RepositoryConfiguration, error) { +// fetchRepConfig: "includeRedHatRepos" allows the fetching of red_hat repositories +func (r repositoryConfigDaoImpl) fetchRepoConfig(orgID string, uuid string, includeRedHatRepos bool) (models.RepositoryConfiguration, error) { found := models.RepositoryConfiguration{} + + orgIdsToCheck := []string{orgID} + + if includeRedHatRepos { + orgIdsToCheck = append(orgIdsToCheck, config.RedHatOrg) + } + result := r.db. Preload("Repository").Preload("LastSnapshot"). - Where("UUID = ? AND ORG_ID = ?", UuidifyString(uuid), orgID). + Where("UUID = ? AND ORG_ID IN ?", UuidifyString(uuid), orgIdsToCheck). First(&found) if result.Error != nil { if result.Error == gorm.ErrRecordNotFound { return found, &ce.DaoError{NotFound: true, Message: "Could not find repository with UUID " + uuid} - } else { - return found, DBErrorToApi(result.Error) } + return found, DBErrorToApi(result.Error) } return found, nil } @@ -417,9 +454,8 @@ func (r repositoryConfigDaoImpl) FetchByRepoUuid(orgID string, repoUuid string) if result.Error != nil { if result.Error == gorm.ErrRecordNotFound { return repo, &ce.DaoError{NotFound: true, Message: "Could not find repository with UUID " + repoUuid} - } else { - return repo, DBErrorToApi(result.Error) } + return repo, DBErrorToApi(result.Error) } ModelToApiFields(repoConfig, &repo) @@ -435,7 +471,8 @@ func (r repositoryConfigDaoImpl) Update(orgID, uuid string, repoParams api.Repos // We are updating the repo config & snapshots, so bundle in a transaction err = r.db.Transaction(func(tx *gorm.DB) error { - if repoConfig, err = r.fetchRepoConfig(orgID, uuid); err != nil { + // Setting "includeRedHatRepos" to false here to prevent updating red_hat repositories + if repoConfig, err = r.fetchRepoConfig(orgID, uuid, false); err != nil { return err } ApiFieldsToModel(repoParams, &repoConfig, &repo) @@ -526,7 +563,7 @@ func (r repositoryConfigDaoImpl) SoftDelete(orgID string, uuid string) error { var repoConfig models.RepositoryConfiguration var err error - if repoConfig, err = r.fetchRepoConfig(orgID, uuid); err != nil { + if repoConfig, err = r.fetchRepoConfig(orgID, uuid, false); err != nil { return err } @@ -587,7 +624,7 @@ func (r repositoryConfigDaoImpl) bulkDelete(tx *gorm.DB, orgID string, uuids []s var err error var repoConfig models.RepositoryConfiguration - if repoConfig, err = r.fetchRepoConfig(orgID, uuids[i]); err != nil { + if repoConfig, err = r.fetchRepoConfig(orgID, uuids[i], false); err != nil { dbErr = DBErrorToApi(err) errors[i] = dbErr tx.RollbackTo(save) diff --git a/pkg/dao/repository_configs_mock.go b/pkg/dao/repository_configs_mock.go index 62393be53..9fc4891e7 100644 --- a/pkg/dao/repository_configs_mock.go +++ b/pkg/dao/repository_configs_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.36.1. DO NOT EDIT. package dao @@ -177,25 +177,25 @@ func (_m *MockRepositoryConfigDao) InternalOnly_FetchRepoConfigsForRepoUUID(uuid return r0 } -// InternalOnly_ListReposToSnapshot provides a mock function with given fields: -func (_m *MockRepositoryConfigDao) InternalOnly_ListReposToSnapshot() ([]models.RepositoryConfiguration, error) { - ret := _m.Called() +// InternalOnly_ListReposToSnapshot provides a mock function with given fields: filter +func (_m *MockRepositoryConfigDao) InternalOnly_ListReposToSnapshot(filter *ListRepoFilter) ([]models.RepositoryConfiguration, error) { + ret := _m.Called(filter) var r0 []models.RepositoryConfiguration var r1 error - if rf, ok := ret.Get(0).(func() ([]models.RepositoryConfiguration, error)); ok { - return rf() + if rf, ok := ret.Get(0).(func(*ListRepoFilter) ([]models.RepositoryConfiguration, error)); ok { + return rf(filter) } - if rf, ok := ret.Get(0).(func() []models.RepositoryConfiguration); ok { - r0 = rf() + if rf, ok := ret.Get(0).(func(*ListRepoFilter) []models.RepositoryConfiguration); ok { + r0 = rf(filter) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]models.RepositoryConfiguration) } } - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() + if rf, ok := ret.Get(1).(func(*ListRepoFilter) error); ok { + r1 = rf(filter) } else { r1 = ret.Error(1) } diff --git a/pkg/dao/repository_configs_test.go b/pkg/dao/repository_configs_test.go index 0912cdd19..3e8751b15 100644 --- a/pkg/dao/repository_configs_test.go +++ b/pkg/dao/repository_configs_test.go @@ -109,6 +109,22 @@ func (suite *RepositoryConfigSuite) TestCreateTwiceWithNoSlash() { assert.ErrorContains(suite.T(), err, "Name cannot be blank") } +func (suite *RepositoryConfigSuite) TestCreateRedHatRepository() { + toCreate := api.RepositoryRequest{ + Name: pointy.String(""), + URL: pointy.String("something-no-slash"), + OrgID: pointy.String(config.RedHatOrg), + AccountID: pointy.String("123"), + DistributionArch: pointy.String(""), + DistributionVersions: &[]string{ + config.El9, + }, + } + dao := GetRepositoryConfigDao(suite.tx) + _, err := dao.Create(toCreate) + assert.ErrorContains(suite.T(), err, "Creating of Red Hat repositories is not permitted") +} + func (suite *RepositoryConfigSuite) TestRepositoryCreateAlreadyExists() { t := suite.T() tx := suite.tx @@ -1384,6 +1400,25 @@ func (suite *RepositoryConfigSuite) TestBulkDeleteOneNotFound() { assert.Len(t, found, repoConfigCount) } +func (suite *RepositoryConfigSuite) TestBulkDeleteRedhatRepository() { + t := suite.T() + dao := GetRepositoryConfigDao(suite.tx) + orgID := config.RedHatOrg + repoConfigCount := 5 + + err := seeds.SeedRepositoryConfigurations(suite.tx, repoConfigCount, seeds.SeedOptions{OrgID: orgID}) + assert.Nil(t, err) + + errs := dao.BulkDelete(orgID, []string{"doesn't matter"}) + assert.Len(t, errs, 1) + assert.Equal(t, ce.HttpCodeForDaoError(errs[0]), 404) + + var found []models.RepositoryConfiguration + err = suite.tx.Where("org_id = ?", orgID).Find(&found).Error + assert.NoError(t, err) + assert.Len(t, found, repoConfigCount) +} + func (suite *RepositoryConfigSuite) TestBulkDeleteMultipleNotFound() { t := suite.T() dao := GetRepositoryConfigDao(suite.tx) @@ -1679,6 +1714,7 @@ type RepoToSnapshotTest struct { Opts *seeds.TaskSeedOptions Included bool OptionAlwaysRunCronTasks bool + Filter *ListRepoFilter } func (suite *RepositoryConfigSuite) TestListReposToSnapshot() { @@ -1718,6 +1754,18 @@ func (suite *RepositoryConfigSuite) TestListReposToSnapshot() { Opts: &seeds.TaskSeedOptions{RepoConfigUUID: repo.UUID, OrgID: repo.OrgID, Status: config.TaskStatusFailed}, Included: true, }, + { + Name: "Previous Snapshot Failed, and url specified", + Opts: &seeds.TaskSeedOptions{RepoConfigUUID: repo.UUID, OrgID: repo.OrgID, Status: config.TaskStatusFailed}, + Included: true, + Filter: &ListRepoFilter{URLs: &[]string{repo.URL}}, + }, + { + Name: "Previous Snapshot Failed, and url specified", + Opts: &seeds.TaskSeedOptions{RepoConfigUUID: repo.UUID, OrgID: repo.OrgID, Status: config.TaskStatusFailed}, + Included: false, + Filter: &ListRepoFilter{RedhatOnly: pointy.Pointer(true)}, + }, { Name: "Previous Snapshot was successful and recent", Opts: &seeds.TaskSeedOptions{RepoConfigUUID: repo.UUID, OrgID: repo.OrgID, Status: config.TaskStatusCompleted}, @@ -1747,7 +1795,7 @@ func (suite *RepositoryConfigSuite) TestListReposToSnapshot() { config.Get().Options.AlwaysRunCronTasks = testCase.OptionAlwaysRunCronTasks - afterRepos, err := dao.InternalOnly_ListReposToSnapshot() + afterRepos, err := dao.InternalOnly_ListReposToSnapshot(testCase.Filter) assert.NoError(t, err) for i := range afterRepos { if repo.UUID == afterRepos[i].UUID { diff --git a/pkg/dao/rpms.go b/pkg/dao/rpms.go index 2aa5d23c2..2d09dbd4d 100644 --- a/pkg/dao/rpms.go +++ b/pkg/dao/rpms.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/content-services/content-sources-backend/pkg/api" + "github.com/content-services/content-sources-backend/pkg/config" ce "github.com/content-services/content-sources-backend/pkg/errors" "github.com/content-services/content-sources-backend/pkg/models" "github.com/content-services/yummy/pkg/yum" @@ -28,7 +29,7 @@ func (r rpmDaoImpl) isOwnedRepository(orgID string, repositoryConfigUUID string) var repoConfigs []models.RepositoryConfiguration var count int64 if err := r.db. - Where("org_id = ? and uuid = ?", orgID, UuidifyString(repositoryConfigUUID)). + Where("org_id IN (?, ?) AND uuid = ?", orgID, config.RedHatOrg, UuidifyString(repositoryConfigUUID)). Find(&repoConfigs). Count(&count). Error; err != nil { @@ -40,7 +41,13 @@ func (r rpmDaoImpl) isOwnedRepository(orgID string, repositoryConfigUUID string) return true, nil } -func (r rpmDaoImpl) List(orgID string, repositoryConfigUUID string, limit int, offset int, search string, sortBy string) (api.RepositoryRpmCollectionResponse, int64, error) { +func (r rpmDaoImpl) List( + orgID string, + repositoryConfigUUID string, + limit int, offset int, + search string, + sortBy string, +) (api.RepositoryRpmCollectionResponse, int64, error) { // Check arguments if orgID == "" { return api.RepositoryRpmCollectionResponse{}, 0, fmt.Errorf("orgID can not be an empty string") diff --git a/pkg/dao/rpms_mock.go b/pkg/dao/rpms_mock.go index 155968833..e64417858 100644 --- a/pkg/dao/rpms_mock.go +++ b/pkg/dao/rpms_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.20.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package dao @@ -109,13 +109,12 @@ func (_m *MockRpmDao) Search(orgID string, request api.SearchRpmRequest) ([]api. return r0, r1 } -type mockConstructorTestingTNewMockRpmDao interface { +// NewMockRpmDao creates a new instance of MockRpmDao. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockRpmDao(t interface { mock.TestingT Cleanup(func()) -} - -// NewMockRpmDao creates a new instance of MockRpmDao. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewMockRpmDao(t mockConstructorTestingTNewMockRpmDao) *MockRpmDao { +}) *MockRpmDao { mock := &MockRpmDao{} mock.Mock.Test(t) diff --git a/pkg/dao/rpms_test.go b/pkg/dao/rpms_test.go index 5815a1a94..9707a8f3e 100644 --- a/pkg/dao/rpms_test.go +++ b/pkg/dao/rpms_test.go @@ -109,6 +109,67 @@ func (s *RpmSuite) TestRpmList() { assert.Equal(t, count, int64(0)) } +func (s *RpmSuite) TestRpmListRedHatRepositories() { + var err error + t := s.Suite.T() + + redHatRepo := repoPublicTest.DeepCopy() + redHatRepo.URL = "https://www.public.redhat.com" + if err := s.tx.Create(redHatRepo).Error; err != nil { + s.FailNow("Preparing Repository record: %w", err) + } + + redhatRepoConfig := repoConfigTest1.DeepCopy() + redhatRepoConfig.OrgID = config.RedHatOrg + redhatRepoConfig.Name = "Demo Redhat Repository Config" + redhatRepoConfig.RepositoryUUID = redHatRepo.Base.UUID + if err := s.tx.Create(redhatRepoConfig).Error; err != nil { + s.FailNow("Preparing RepositoryConfiguration record: %w", err) + } + + // Prepare RepositoryRpm records + rpm1 := repoRpmTest1.DeepCopy() + rpm2 := repoRpmTest2.DeepCopy() + dao := GetRpmDao(s.tx) + + err = s.tx.Create(&rpm1).Error + assert.NoError(t, err) + err = s.tx.Create(&rpm2).Error + assert.NoError(t, err) + + // Add one red hat repo + err = s.tx.Create(&models.RepositoryRpm{ + RepositoryUUID: redHatRepo.Base.UUID, + RpmUUID: rpm1.Base.UUID, + }).Error + assert.NoError(t, err) + + //Add one regular repository + err = s.tx.Create(&models.RepositoryRpm{ + RepositoryUUID: s.repo.Base.UUID, + RpmUUID: rpm2.Base.UUID, + }).Error + + assert.NoError(t, err) + + var repoRpmList api.RepositoryRpmCollectionResponse + var count int64 + + // Check red hat repo package (matched "-1" orgID) + repoRpmList, count, err = dao.List("ThisOrgIdWontMatter", redhatRepoConfig.Base.UUID, 10, 0, "", "") + assert.NoError(t, err) + assert.Equal(t, int64(1), count) + assert.Equal(t, repoRpmList.Meta.Count, count) + assert.Equal(t, repoRpmTest1.Name, repoRpmList.Data[0].Name) // Asserts name:asc by default + + // Check custom repo package (checks orgId) + repoRpmList, count, err = dao.List(orgIDTest, s.repoConfig.Base.UUID, 10, 0, "", "") + assert.NoError(t, err) + assert.Equal(t, int64(1), count) + assert.Equal(t, repoRpmList.Meta.Count, count) + assert.Equal(t, repoRpmTest2.Name, repoRpmList.Data[0].Name) // Asserts name:asc by default +} + func (s *RpmSuite) TestRpmListRepoNotFound() { t := s.Suite.T() dao := GetRpmDao(s.tx) diff --git a/pkg/dao/snapshots.go b/pkg/dao/snapshots.go index a0dfd463d..eaa3d2827 100644 --- a/pkg/dao/snapshots.go +++ b/pkg/dao/snapshots.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/content-services/content-sources-backend/pkg/api" + "github.com/content-services/content-sources-backend/pkg/config" ce "github.com/content-services/content-sources-backend/pkg/errors" "github.com/content-services/content-sources-backend/pkg/models" "github.com/content-services/content-sources-backend/pkg/pulp_client" @@ -48,17 +49,28 @@ func (sDao *snapshotDaoImpl) Create(s *models.Snapshot) error { } // List the snapshots for a given repository config -func (sDao *snapshotDaoImpl) List(repoConfigUuid string, paginationData api.PaginationData, _ api.FilterData) (api.SnapshotCollectionResponse, int64, error) { +func (sDao *snapshotDaoImpl) List( + orgID string, + repoConfigUUID string, + paginationData api.PaginationData, + _ api.FilterData, +) (api.SnapshotCollectionResponse, int64, error) { var snaps []models.Snapshot var totalSnaps int64 var repoConfig models.RepositoryConfiguration // First check if repo config exists - result := sDao.db.Where("uuid = ?", UuidifyString(repoConfigUuid)).First(&repoConfig) + result := sDao.db.Where( + "repository_configurations.org_id IN (?,?) AND uuid = ?", + orgID, + config.RedHatOrg, + UuidifyString(repoConfigUUID)). + First(&repoConfig) + if result.Error != nil { if result.Error == gorm.ErrRecordNotFound { return api.SnapshotCollectionResponse{}, totalSnaps, &ce.DaoError{ - Message: "Could not find repository with UUID " + repoConfigUuid, + Message: "Could not find repository with UUID " + repoConfigUUID, NotFound: true, } } @@ -71,12 +83,13 @@ func (sDao *snapshotDaoImpl) List(repoConfigUuid string, paginationData api.Pagi order := convertSortByToSQL(paginationData.SortBy, sortMap, "created_at asc") filteredDB := sDao.db. - Where("snapshots.repository_configuration_uuid = ?", UuidifyString(repoConfigUuid)) + Model(&models.Snapshot{}). + Joins("JOIN repository_configurations ON repository_configuration_uuid = repository_configurations.uuid"). + Where("repository_configuration_uuid = ?", UuidifyString(repoConfigUUID)). + Where("repository_configurations.org_id IN (?,?)", orgID, config.RedHatOrg) // Get count - filteredDB. - Model(&snaps). - Count(&totalSnaps) + filteredDB.Count(&totalSnaps) if filteredDB.Error != nil { return api.SnapshotCollectionResponse{}, 0, filteredDB.Error @@ -123,7 +136,7 @@ func (sDao *snapshotDaoImpl) Fetch(uuid string) (models.Snapshot, error) { func (sDao *snapshotDaoImpl) GetRepositoryConfigurationFile(orgID, snapshotUUID, repoConfigUUID string) (string, error) { rcDao := repositoryConfigDaoImpl{db: sDao.db} - repoConfig, err := rcDao.fetchRepoConfig(orgID, repoConfigUUID) + repoConfig, err := rcDao.fetchRepoConfig(orgID, repoConfigUUID, true) if err != nil { return "", err } diff --git a/pkg/dao/snapshots_mock.go b/pkg/dao/snapshots_mock.go index 476066758..c370f1c83 100644 --- a/pkg/dao/snapshots_mock.go +++ b/pkg/dao/snapshots_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package dao @@ -133,30 +133,30 @@ func (_m *MockSnapshotDao) InitializePulpClient(ctx context.Context, orgID strin return r0 } -// List provides a mock function with given fields: repoConfigUuid, paginationData, _a2 -func (_m *MockSnapshotDao) List(repoConfigUuid string, paginationData api.PaginationData, _a2 api.FilterData) (api.SnapshotCollectionResponse, int64, error) { - ret := _m.Called(repoConfigUuid, paginationData, _a2) +// List provides a mock function with given fields: orgID, repoConfigUuid, paginationData, filterData +func (_m *MockSnapshotDao) List(orgID string, repoConfigUuid string, paginationData api.PaginationData, filterData api.FilterData) (api.SnapshotCollectionResponse, int64, error) { + ret := _m.Called(orgID, repoConfigUuid, paginationData, filterData) var r0 api.SnapshotCollectionResponse var r1 int64 var r2 error - if rf, ok := ret.Get(0).(func(string, api.PaginationData, api.FilterData) (api.SnapshotCollectionResponse, int64, error)); ok { - return rf(repoConfigUuid, paginationData, _a2) + if rf, ok := ret.Get(0).(func(string, string, api.PaginationData, api.FilterData) (api.SnapshotCollectionResponse, int64, error)); ok { + return rf(orgID, repoConfigUuid, paginationData, filterData) } - if rf, ok := ret.Get(0).(func(string, api.PaginationData, api.FilterData) api.SnapshotCollectionResponse); ok { - r0 = rf(repoConfigUuid, paginationData, _a2) + if rf, ok := ret.Get(0).(func(string, string, api.PaginationData, api.FilterData) api.SnapshotCollectionResponse); ok { + r0 = rf(orgID, repoConfigUuid, paginationData, filterData) } else { r0 = ret.Get(0).(api.SnapshotCollectionResponse) } - if rf, ok := ret.Get(1).(func(string, api.PaginationData, api.FilterData) int64); ok { - r1 = rf(repoConfigUuid, paginationData, _a2) + if rf, ok := ret.Get(1).(func(string, string, api.PaginationData, api.FilterData) int64); ok { + r1 = rf(orgID, repoConfigUuid, paginationData, filterData) } else { r1 = ret.Get(1).(int64) } - if rf, ok := ret.Get(2).(func(string, api.PaginationData, api.FilterData) error); ok { - r2 = rf(repoConfigUuid, paginationData, _a2) + if rf, ok := ret.Get(2).(func(string, string, api.PaginationData, api.FilterData) error); ok { + r2 = rf(orgID, repoConfigUuid, paginationData, filterData) } else { r2 = ret.Error(2) } diff --git a/pkg/dao/snapshots_test.go b/pkg/dao/snapshots_test.go index 33303075c..7f69d201f 100644 --- a/pkg/dao/snapshots_test.go +++ b/pkg/dao/snapshots_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/content-services/content-sources-backend/pkg/api" + "github.com/content-services/content-sources-backend/pkg/config" ce "github.com/content-services/content-sources-backend/pkg/errors" "github.com/content-services/content-sources-backend/pkg/models" "github.com/content-services/content-sources-backend/pkg/pulp_client" @@ -59,6 +60,29 @@ func (s *SnapshotsSuite) createRepository() models.RepositoryConfiguration { return rConfig } +func (s *SnapshotsSuite) createRedhatRepository() models.RepositoryConfiguration { + t := s.T() + tx := s.tx + + testRepository := models.Repository{ + URL: "https://example.redhat.com", + LastIntrospectionTime: nil, + LastIntrospectionError: nil, + } + err := tx.Create(&testRepository).Error + assert.NoError(t, err) + + rConfig := models.RepositoryConfiguration{ + Name: "redhatSnapshot", + OrgID: config.RedHatOrg, + RepositoryUUID: testRepository.UUID, + } + + err = tx.Create(&rConfig).Error + assert.NoError(t, err) + return rConfig +} + func (s *SnapshotsSuite) createSnapshot(rConfig models.RepositoryConfiguration) models.Snapshot { t := s.T() tx := s.tx @@ -90,6 +114,7 @@ func (s *SnapshotsSuite) TestCreateAndList() { repoDao := repositoryConfigDaoImpl{db: tx, yumRepo: &mockExt.YumRepositoryMock{}} rConfig := s.createRepository() + pageData := api.PaginationData{ Limit: 100, Offset: 0, @@ -102,9 +127,9 @@ func (s *SnapshotsSuite) TestCreateAndList() { snap := s.createSnapshot(rConfig) - collection, total, err := sDao.List(rConfig.UUID, pageData, filterData) + collection, total, err := sDao.List(rConfig.OrgID, rConfig.UUID, pageData, filterData) - repository, _ := repoDao.fetchRepoConfig(rConfig.OrgID, rConfig.UUID) + repository, _ := repoDao.fetchRepoConfig(rConfig.OrgID, rConfig.UUID, false) repositoryList, repoCount, _ := repoDao.List(rConfig.OrgID, api.PaginationData{Limit: -1}, api.FilterData{}) assert.NoError(t, err) @@ -128,6 +153,55 @@ func (s *SnapshotsSuite) TestCreateAndList() { } } +func (s *SnapshotsSuite) TestCreateAndListRedHatRepo() { + t := s.T() + tx := s.tx + + mockPulpClient := pulp_client.NewMockPulpClient(t) + sDao := snapshotDaoImpl{db: tx, pulpClient: mockPulpClient} + mockPulpClient.On("GetContentPath").Return(testContentPath, nil) + + repoDao := repositoryConfigDaoImpl{db: tx, yumRepo: &mockExt.YumRepositoryMock{}} + + redhatRepositoryConfig := s.createRedhatRepository() + redhatSnap := s.createSnapshot(redhatRepositoryConfig) + + pageData := api.PaginationData{ + Limit: 100, + Offset: 0, + } + filterData := api.FilterData{ + Search: "", + Arch: "", + Version: "", + } + + collection, total, err := sDao.List("ShouldNotMatter", redhatRepositoryConfig.UUID, pageData, filterData) + + repository, _ := repoDao.fetchRepoConfig("ShouldNotMatter", redhatRepositoryConfig.UUID, true) + repositoryList, repoCount, _ := repoDao.List("ShouldNotMatter", api.PaginationData{Limit: -1}, api.FilterData{}) + + assert.NoError(t, err) + assert.Equal(t, int64(1), total) + assert.Equal(t, 1, len(collection.Data)) + if len(collection.Data) > 0 { + assert.Equal(t, redhatSnap.RepositoryPath, collection.Data[0].RepositoryPath) + assert.Equal(t, redhatSnap.ContentCounts, models.ContentCountsType(collection.Data[0].ContentCounts)) + assert.Equal(t, redhatSnap.AddedCounts, models.ContentCountsType(collection.Data[0].AddedCounts)) + assert.Equal(t, redhatSnap.RemovedCounts, models.ContentCountsType(collection.Data[0].RemovedCounts)) + assert.False(t, collection.Data[0].CreatedAt.IsZero()) + // Check that the repositoryConfig has the appropriate values + assert.Equal(t, redhatSnap.UUID, repository.LastSnapshotUUID) + assert.EqualValues(t, redhatSnap.AddedCounts, repository.LastSnapshot.AddedCounts) + assert.EqualValues(t, redhatSnap.RemovedCounts, repository.LastSnapshot.RemovedCounts) + // Check that the list repositoryConfig has the appropriate values + assert.Equal(t, int64(1), repoCount) + assert.Equal(t, redhatSnap.UUID, repositoryList.Data[0].LastSnapshotUUID) + assert.EqualValues(t, redhatSnap.AddedCounts, repositoryList.Data[0].LastSnapshot.AddedCounts) + assert.EqualValues(t, redhatSnap.RemovedCounts, repositoryList.Data[0].LastSnapshot.RemovedCounts) + } +} + func (s *SnapshotsSuite) TestListNoSnapshots() { t := s.T() tx := s.tx @@ -161,7 +235,7 @@ func (s *SnapshotsSuite) TestListNoSnapshots() { err = tx.Create(&rConfig).Error assert.NoError(t, err) - collection, total, err := sDao.List(rConfig.UUID, pageData, filterData) + collection, total, err := sDao.List(rConfig.OrgID, rConfig.UUID, pageData, filterData) assert.NoError(t, err) assert.Equal(t, int64(0), total) assert.Equal(t, 0, len(collection.Data)) @@ -190,7 +264,7 @@ func (s *SnapshotsSuite) TestListPageLimit() { s.createSnapshot(rConfig) } - collection, total, err := sDao.List(rConfig.UUID, pageData, filterData) + collection, total, err := sDao.List(rConfig.OrgID, rConfig.UUID, pageData, filterData) assert.NoError(t, err) assert.Equal(t, int64(11), total) assert.Equal(t, 10, len(collection.Data)) @@ -215,13 +289,58 @@ func (s *SnapshotsSuite) TestListNotFound() { s.createSnapshot(rConfig) - collection, total, err := sDao.List("bad-uuid", pageData, filterData) + collection, total, err := sDao.List(rConfig.OrgID, "bad-uuid", pageData, filterData) + assert.Error(t, err) + daoError, ok := err.(*ce.DaoError) + assert.True(t, ok) + assert.True(t, daoError.NotFound) + assert.Equal(t, int64(0), total) + assert.Equal(t, 0, len(collection.Data)) +} + +func (s *SnapshotsSuite) TestListNotFoundBadOrgId() { + t := s.T() + tx := s.tx + + sDao := snapshotDaoImpl{db: tx} + + testRepository := models.Repository{ + URL: "https://example.com", + LastIntrospectionTime: nil, + LastIntrospectionError: nil, + } + err := tx.Create(&testRepository).Error + assert.NoError(t, err) + + rConfig := models.RepositoryConfiguration{ + Name: "toSnapshot", + OrgID: "not-banana-id", + RepositoryUUID: testRepository.UUID, + } + + err = tx.Create(&rConfig).Error + assert.NoError(t, err) + + pageData := api.PaginationData{ + Limit: 100, + Offset: 0, + } + filterData := api.FilterData{ + Search: "", + Arch: "", + Version: "", + } + + s.createSnapshot(rConfig) + + collection, total, err := sDao.List("bad-banana-id", rConfig.UUID, pageData, filterData) assert.Error(t, err) daoError, ok := err.(*ce.DaoError) assert.True(t, ok) assert.True(t, daoError.NotFound) assert.Equal(t, int64(0), total) assert.Equal(t, 0, len(collection.Data)) + assert.ErrorContains(t, err, "Could not find repository with UUID "+rConfig.UUID) } func (s *SnapshotsSuite) TestFetchForRepoUUID() { diff --git a/pkg/dao/task_info_mock.go b/pkg/dao/task_info_mock.go index fed4d0a88..c75e6ed37 100644 --- a/pkg/dao/task_info_mock.go +++ b/pkg/dao/task_info_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.20.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package dao @@ -105,13 +105,12 @@ func (_m *MockTaskInfoDao) List(OrgID string, pageData api.PaginationData, filte return r0, r1, r2 } -type mockConstructorTestingTNewMockTaskInfoDao interface { +// NewMockTaskInfoDao creates a new instance of MockTaskInfoDao. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockTaskInfoDao(t interface { mock.TestingT Cleanup(func()) -} - -// NewMockTaskInfoDao creates a new instance of MockTaskInfoDao. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewMockTaskInfoDao(t mockConstructorTestingTNewMockTaskInfoDao) *MockTaskInfoDao { +}) *MockTaskInfoDao { mock := &MockTaskInfoDao{} mock.Mock.Test(t) diff --git a/pkg/handler/snapshots.go b/pkg/handler/snapshots.go index 6193d2102..fe4240e9f 100644 --- a/pkg/handler/snapshots.go +++ b/pkg/handler/snapshots.go @@ -51,7 +51,7 @@ func (sh *SnapshotHandler) listSnapshots(c echo.Context) error { return ce.NewErrorResponse(ce.HttpCodeForDaoError(err), "Error initializing pulp client", err.Error()) } - snapshots, totalSnaps, err := sh.DaoRegistry.Snapshot.List(uuid, pageData, filterData) + snapshots, totalSnaps, err := sh.DaoRegistry.Snapshot.List(orgID, uuid, pageData, filterData) if err != nil { return ce.NewErrorResponse(ce.HttpCodeForDaoError(err), "Error listing repository snapshots", err.Error()) } diff --git a/pkg/handler/snapshots_test.go b/pkg/handler/snapshots_test.go index 77f3bc5e1..1f0bcf455 100644 --- a/pkg/handler/snapshots_test.go +++ b/pkg/handler/snapshots_test.go @@ -66,7 +66,7 @@ func (suite *SnapshotSuite) TestSnapshotList() { uuid := "abcadaba" orgID := test_handler.MockOrgId suite.reg.Snapshot.On("InitializePulpClient", mock.AnythingOfType("*context.valueCtx"), orgID).Return(nil).Once() - suite.reg.Snapshot.On("List", uuid, paginationData, api.FilterData{}).Return(collection, int64(1), nil) + suite.reg.Snapshot.On("List", test_handler.MockOrgId, uuid, paginationData, api.FilterData{}).Return(collection, int64(1), nil) path := fmt.Sprintf("%s/repositories/%s/snapshots/?limit=%d", fullRootPath(), uuid, 10) req := httptest.NewRequest(http.MethodGet, path, nil) diff --git a/pkg/middleware/metrics.go b/pkg/middleware/metrics.go index 9117336d8..b440fe70b 100644 --- a/pkg/middleware/metrics.go +++ b/pkg/middleware/metrics.go @@ -56,8 +56,9 @@ func MetricsMiddlewareWithConfig(config *MetricsConfig) echo.MiddlewareFunc { method := ctx.Request().Method path := MatchedRoute(ctx) err := next(ctx) + timeStart := time.Since(start) status := mapStatus(ctx.Response().Status) - defer config.Metrics.HttpStatusHistogram.WithLabelValues(status, method, path).Observe(time.Since(start).Seconds()) + defer config.Metrics.HttpStatusHistogram.WithLabelValues(status, method, path).Observe(timeStart.Seconds()) return err } } diff --git a/pkg/pulp_client/pulp_client_mock.go b/pkg/pulp_client/pulp_client_mock.go index c81f27b1a..ab4d8bef0 100644 --- a/pkg/pulp_client/pulp_client_mock.go +++ b/pkg/pulp_client/pulp_client_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package pulp_client diff --git a/pkg/pulp_client/pulp_global_client_mock.go b/pkg/pulp_client/pulp_global_client_mock.go index 7076d3667..3d12412f5 100644 --- a/pkg/pulp_client/pulp_global_client_mock.go +++ b/pkg/pulp_client/pulp_global_client_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package pulp_client diff --git a/pkg/tasks/client/client_mock.go b/pkg/tasks/client/client_mock.go index 24f083d24..48fc493d5 100644 --- a/pkg/tasks/client/client_mock.go +++ b/pkg/tasks/client/client_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package client diff --git a/pkg/tasks/queue/queue_mock.go b/pkg/tasks/queue/queue_mock.go index 522917c77..d8832cc7c 100644 --- a/pkg/tasks/queue/queue_mock.go +++ b/pkg/tasks/queue/queue_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package queue diff --git a/test/integration/snapshot_test.go b/test/integration/snapshot_test.go index 10f8132cb..e59d885bf 100644 --- a/test/integration/snapshot_test.go +++ b/test/integration/snapshot_test.go @@ -87,7 +87,7 @@ func (s *SnapshotSuite) TestSnapshot() { s.snapshotAndWait(taskClient, repo, repoUuid, accountId) // Verify the snapshot was created - snaps, _, err := s.dao.Snapshot.List(repo.UUID, api.PaginationData{Limit: -1}, api.FilterData{}) + snaps, _, err := s.dao.Snapshot.List(repo.OrgID, repo.UUID, api.PaginationData{Limit: -1}, api.FilterData{}) assert.NoError(s.T(), err) assert.NotEmpty(s.T(), snaps) time.Sleep(5 * time.Second) @@ -138,7 +138,7 @@ func (s *SnapshotSuite) TestSnapshot() { s.WaitOnTask(taskUuid) // Verify the snapshot was deleted - snaps, _, err = s.dao.Snapshot.List(repo.UUID, api.PaginationData{Limit: -1}, api.FilterData{}) + snaps, _, err = s.dao.Snapshot.List(repo.OrgID, repo.UUID, api.PaginationData{Limit: -1}, api.FilterData{}) assert.Error(s.T(), err) assert.Empty(s.T(), snaps.Data) time.Sleep(5 * time.Second) @@ -185,7 +185,7 @@ func (s *SnapshotSuite) snapshotAndWait(taskClient client.TaskClient, repo api.R s.WaitOnTask(taskUuid) // Verify the snapshot was created - snaps, _, err := s.dao.Snapshot.List(repo.UUID, api.PaginationData{Limit: -1}, api.FilterData{}) + snaps, _, err := s.dao.Snapshot.List(repo.OrgID, repo.UUID, api.PaginationData{Limit: -1}, api.FilterData{}) assert.NoError(s.T(), err) assert.NotEmpty(s.T(), snaps) time.Sleep(5 * time.Second) @@ -211,7 +211,7 @@ func (s *SnapshotSuite) cancelAndWait(taskClient client.TaskClient, taskUUID uui s.WaitOnCanceledTask(taskUUID) // Verify the snapshot was not created - snaps, _, err := s.dao.Snapshot.List(repo.UUID, api.PaginationData{}, api.FilterData{}) + snaps, _, err := s.dao.Snapshot.List(repo.OrgID, repo.UUID, api.PaginationData{}, api.FilterData{}) assert.NoError(s.T(), err) assert.Equal(s.T(), api.SnapshotCollectionResponse{Data: []api.SnapshotResponse{}}, snaps) }