diff --git a/README.md b/README.md index 82cebcce7..3e87e1029 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Content Sources ## What is it? -Content Sources is an application for storing information about external content (currently YUM repositories) in a central location as well as creating snapshots of those repositories, backed by a Pulp server. +Content Sources is an application for storing information about external content (currently YUM repositories) in a central location as well as creating snapshots of those repositories, backed by a Pulp server. ## Developing @@ -16,27 +16,27 @@ Content Sources is an application for storing information about external content Create a config file from the example: - ```sh - $ cp ./configs/config.yaml.example ./configs/config.yaml - ``` +```sh +$ cp ./configs/config.yaml.example ./configs/config.yaml +``` ### Build needed kafka container - ```sh - $ make compose-build - ``` +```sh +$ make compose-build +``` ### Start dependency containers - ```sh - $ make compose-up - ``` +```sh +$ make compose-up +``` ### Run the server! - ```sh - $ make run - ``` +```sh +$ make run +``` ### @@ -44,39 +44,40 @@ Hit the API: ```sh $ curl -H "$( ./scripts/header.sh 9999 1111 )" http://localhost:8000/api/content-sources/v1.0/repositories/ - ``` +``` ### Stop dependency containers + When its time to shut down the running containers: - ```sh - $ make compose-down - ``` +```sh +$ make compose-down +``` And clean the volume that it uses by (this stops the container before doing it if it were running): - ```sh - $ make compose-clean - ``` - -> There are other make rules that could be helpful, run `make help` to list them. Some are highlighted below +```sh +$ make compose-clean +``` +> There are other make rules that could be helpful, run `make help` to list them. Some are highlighted below ### HOW TO ADD NEW MIGRATION FILES -You can add new migration files, with the prefixed date attached to the file name, by running the following: + +You can add new migration files, with the prefixed date attached to the file name, by running the following: ``` $ go run cmd/dbmigrate/main.go new ``` ### Database Commands + Migrate the Database ```sh $ make db-migrate-up ``` - Seed the database ```sh @@ -85,9 +86,9 @@ $ make db-migrate-seed Get an interactive shell: - ```sh - $ make db-shell - ``` +```sh +$ make db-shell +``` Or open directly a postgres client by running: @@ -125,7 +126,6 @@ Update the `configs/prometheus.yaml` file to set your hostname instead of `local $ cat ./configs/prometheus.example.yaml | sed "s/localhost/$(hostname)/g" > ./configs/prometheus.yaml ``` - To start prometheus run: ```sh @@ -157,8 +157,8 @@ $ make prometheus-ui rbac_timeout: 30 mocks: rbac: - user_read_write: ["jdoe@example.com","jdoe"] - user_read: ["tdoe@example.com","tdoe"] + user_read_write: ["jdoe@example.com", "jdoe"] + user_read: ["tdoe@example.com", "tdoe"] ``` **Running it** @@ -200,13 +200,24 @@ $ curl -H "$( ./scripts/header.sh 9999 1111 )" http://localhost:8000/api/content $ make openapi ``` +### Generating/Update mocks: + +Ensure [mockery](https://vektra.github.io/mockery/latest/installation/) is installed. + +Then: + +```sh +$ go generate ./... +``` + ### Configuration -The default configuration file in ./configs/config.yaml.example shows all available config options. Any of these can be overridden with an environment variable. For example "database.name" can be passed in via an environment variable named "DATABASE_NAME". +The default configuration file in ./configs/config.yaml.example shows all available config options. Any of these can be overridden with an environment variable. For example "database.name" can be passed in via an environment variable named "DATABASE_NAME". ### Linting To use golangci-lint: + 1. `make install-golangci-lint` 2. `make lint` @@ -214,21 +225,21 @@ To use pre-commit linter: `make install-pre-commit` ### Code Layout -| Path | Description | -|-----------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [api](./api/) | Openapi docs and doc generation code | -| [db/migrations](./db/migrations/) | Database Migrations | | -| [pkg/api](./pkg/api) | API Structures that are used for handling data within our API Handlers | -| [pkg/config](./pkg/config) | Config loading and application bootstrapping code | -| [pkg/dao](./pkg/dao) | Database Access Object. Abstraction layer that provides an interface and implements it for our default database provider (postgresql). It is separated out for abstraction and easier testing | -| [pkg/db](./pkg/db) | Database connection and migration related code | -| [pkg/handler](./pkg/handler) | Methods that directly handle API requests | -| [pkg/middleware](./pkg/middleware)| Hold all the middleware components created for the service. | -| [pkg/event](./pkg/event) | Event message logic. Mre info [here](./pkg/event/README.md). | -| [pkg/models](./pkg/models) | Structs that represent database models (Gorm) | -| [pkg/seeds](./pkg/seeds) | Code to help seed the database for both development and testing | +| Path | Description | +| ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --- | +| [api](./api/) | Openapi docs and doc generation code | +| [db/migrations](./db/migrations/) | Database Migrations | | +| [pkg/api](./pkg/api) | API Structures that are used for handling data within our API Handlers | +| [pkg/config](./pkg/config) | Config loading and application bootstrapping code | +| [pkg/dao](./pkg/dao) | Database Access Object. Abstraction layer that provides an interface and implements it for our default database provider (postgresql). It is separated out for abstraction and easier testing | +| [pkg/db](./pkg/db) | Database connection and migration related code | +| [pkg/handler](./pkg/handler) | Methods that directly handle API requests | +| [pkg/middleware](./pkg/middleware) | Hold all the middleware components created for the service. | +| [pkg/event](./pkg/event) | Event message logic. Mre info [here](./pkg/event/README.md). | +| [pkg/models](./pkg/models) | Structs that represent database models (Gorm) | +| [pkg/seeds](./pkg/seeds) | Code to help seed the database for both development and testing | ## More info - * [Architecture](docs/architecture.md) - * [OpenApi Docs](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/content-services/content-sources-backend/main/api/openapi.json) +- [Architecture](docs/architecture.md) +- [OpenApi Docs](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/content-services/content-sources-backend/main/api/openapi.json) diff --git a/pkg/cache/cache_mock.go b/pkg/cache/cache_mock.go index 6d2437342..f5dd3053c 100644 --- a/pkg/cache/cache_mock.go +++ b/pkg/cache/cache_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package cache diff --git a/pkg/dao/domain_dao_mock.go b/pkg/dao/domain_dao_mock.go index 07d556779..25fe99c3e 100644 --- a/pkg/dao/domain_dao_mock.go +++ b/pkg/dao/domain_dao_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package dao @@ -57,13 +57,12 @@ func (_m *MockDomainDao) FetchOrCreateDomain(orgId string) (string, error) { return r0, r1 } -type mockConstructorTestingTNewMockDomainDao interface { +// NewMockDomainDao creates a new instance of MockDomainDao. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockDomainDao(t interface { mock.TestingT Cleanup(func()) -} - -// NewMockDomainDao creates a new instance of MockDomainDao. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewMockDomainDao(t mockConstructorTestingTNewMockDomainDao) *MockDomainDao { +}) *MockDomainDao { mock := &MockDomainDao{} mock.Mock.Test(t) diff --git a/pkg/dao/interfaces.go b/pkg/dao/interfaces.go index 188f3581d..2f1747f69 100644 --- a/pkg/dao/interfaces.go +++ b/pkg/dao/interfaces.go @@ -79,7 +79,7 @@ type RepositoryDao interface { //go:generate mockery --name SnapshotDao --filename snapshots_mock.go --inpackage type SnapshotDao interface { Create(snap *models.Snapshot) error - List(repoConfigUuid string, paginationData api.PaginationData, _ api.FilterData) (api.SnapshotCollectionResponse, int64, error) + List(orgID string, repoConfigUuid string, paginationData api.PaginationData, filterData api.FilterData) (api.SnapshotCollectionResponse, int64, error) FetchForRepoConfigUUID(repoConfigUUID string) ([]models.Snapshot, error) Delete(snapUUID string) error FetchLatestSnapshot(repoConfigUUID string) (api.SnapshotResponse, error) diff --git a/pkg/dao/metrics_mock.go b/pkg/dao/metrics_mock.go index 13bd67949..866df9ab7 100644 --- a/pkg/dao/metrics_mock.go +++ b/pkg/dao/metrics_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.20.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package dao @@ -79,13 +79,12 @@ func (_m *MockMetricsDao) RepositoryConfigsCount() int { return r0 } -type mockConstructorTestingTNewMockMetricsDao interface { +// NewMockMetricsDao creates a new instance of MockMetricsDao. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockMetricsDao(t interface { mock.TestingT Cleanup(func()) -} - -// NewMockMetricsDao creates a new instance of MockMetricsDao. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewMockMetricsDao(t mockConstructorTestingTNewMockMetricsDao) *MockMetricsDao { +}) *MockMetricsDao { mock := &MockMetricsDao{} mock.Mock.Test(t) diff --git a/pkg/dao/repositories_mock.go b/pkg/dao/repositories_mock.go index 987a0d570..33dafbf35 100644 --- a/pkg/dao/repositories_mock.go +++ b/pkg/dao/repositories_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.20.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package dao @@ -145,13 +145,12 @@ func (_m *MockRepositoryDao) Update(repo RepositoryUpdate) error { return r0 } -type mockConstructorTestingTNewMockRepositoryDao interface { +// NewMockRepositoryDao creates a new instance of MockRepositoryDao. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockRepositoryDao(t interface { mock.TestingT Cleanup(func()) -} - -// NewMockRepositoryDao creates a new instance of MockRepositoryDao. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewMockRepositoryDao(t mockConstructorTestingTNewMockRepositoryDao) *MockRepositoryDao { +}) *MockRepositoryDao { mock := &MockRepositoryDao{} mock.Mock.Test(t) diff --git a/pkg/dao/repository_configs.go b/pkg/dao/repository_configs.go index 8eac30d17..8ef769f9e 100644 --- a/pkg/dao/repository_configs.go +++ b/pkg/dao/repository_configs.go @@ -94,6 +94,11 @@ func (r *repositoryConfigDaoImpl) InitializePulpClient(ctx context.Context, orgI func (r repositoryConfigDaoImpl) Create(newRepoReq api.RepositoryRequest) (api.RepositoryResponse, error) { var newRepo models.Repository var newRepoConfig models.RepositoryConfiguration + + if *newRepoReq.OrgID == config.RedHatOrg { + return api.RepositoryResponse{}, errors.New("Creating of Red Hat repositories is not permitted") + } + ApiFieldsToModel(newRepoReq, &newRepoConfig, &newRepo) cleanedUrl := models.CleanupURL(newRepo.URL) @@ -114,7 +119,7 @@ func (r repositoryConfigDaoImpl) Create(newRepoReq api.RepositoryRequest) (api.R } // reload the repoConfig to fetch repository info too - newRepoConfig, err := r.fetchRepoConfig(newRepoConfig.OrgID, newRepoConfig.UUID) + newRepoConfig, err := r.fetchRepoConfig(newRepoConfig.OrgID, newRepoConfig.UUID, false) if err != nil { return api.RepositoryResponse{}, DBErrorToApi(err) } @@ -162,23 +167,31 @@ func (r repositoryConfigDaoImpl) bulkCreate(tx *gorm.DB, newRepositories []api.R newRepoConfigs := make([]models.RepositoryConfiguration, size) newRepos := make([]models.Repository, size) responses := make([]api.RepositoryResponse, size) - errors := make([]error, size) + errorList := make([]error, size) tx.SavePoint("beforecreate") for i := 0; i < size; i++ { + if newRepoConfigs[i].OrgID == config.RedHatOrg { + errorList[i] = errors.New("Creating of Red Hat repositories is not permitted") + tx.RollbackTo("beforecreate") + continue + } + if newRepositories[i].OrgID != nil { newRepoConfigs[i].OrgID = *(newRepositories[i].OrgID) } + if newRepositories[i].AccountID != nil { newRepoConfigs[i].AccountID = *(newRepositories[i].AccountID) } + ApiFieldsToModel(newRepositories[i], &newRepoConfigs[i], &newRepos[i]) newRepos[i].Status = "Pending" cleanedUrl := models.CleanupURL(newRepos[i].URL) create := tx.Where("url = ?", cleanedUrl).FirstOrCreate(&newRepos[i]) if err := create.Error; err != nil { dbErr = DBErrorToApi(err) - errors[i] = dbErr + errorList[i] = dbErr tx.RollbackTo("beforecreate") continue } @@ -186,7 +199,7 @@ func (r repositoryConfigDaoImpl) bulkCreate(tx *gorm.DB, newRepositories []api.R newRepoConfigs[i].RepositoryUUID = newRepos[i].UUID if err := tx.Create(&newRepoConfigs[i]).Error; err != nil { dbErr = DBErrorToApi(err) - errors[i] = dbErr + errorList[i] = dbErr tx.RollbackTo("beforecreate") continue } @@ -203,9 +216,8 @@ func (r repositoryConfigDaoImpl) bulkCreate(tx *gorm.DB, newRepositories []api.R // If there is at least 1 error, return empty response slice. if dbErr == nil { return responses, []error{} - } else { - return []api.RepositoryResponse{}, errors } + return []api.RepositoryResponse{}, errorList } func (p repositoryConfigDaoImpl) InternalOnly_ListReposToSnapshot() ([]models.RepositoryConfiguration, error) { @@ -366,7 +378,7 @@ func (r repositoryConfigDaoImpl) InternalOnly_FetchRepoConfigsForRepoUUID(uuid s func (r repositoryConfigDaoImpl) Fetch(orgID string, uuid string) (api.RepositoryResponse, error) { var repo api.RepositoryResponse - repoConfig, err := r.fetchRepoConfig(orgID, uuid) + repoConfig, err := r.fetchRepoConfig(orgID, uuid, true) if err != nil { return api.RepositoryResponse{}, err } @@ -387,19 +399,26 @@ func (r repositoryConfigDaoImpl) Fetch(orgID string, uuid string) (api.Repositor return repo, nil } -func (r repositoryConfigDaoImpl) fetchRepoConfig(orgID string, uuid string) (models.RepositoryConfiguration, error) { +// fetchRepConfig: "includeRedHatRepos" allows the fetching of red_hat repositories +func (r repositoryConfigDaoImpl) fetchRepoConfig(orgID string, uuid string, includeRedHatRepos bool) (models.RepositoryConfiguration, error) { found := models.RepositoryConfiguration{} + + orgIdsToCheck := []string{orgID} + + if includeRedHatRepos { + orgIdsToCheck = append(orgIdsToCheck, config.RedHatOrg) + } + result := r.db. Preload("Repository").Preload("LastSnapshot"). - Where("UUID = ? AND ORG_ID = ?", UuidifyString(uuid), orgID). + Where("UUID = ? AND ORG_ID IN ?", UuidifyString(uuid), orgIdsToCheck). First(&found) if result.Error != nil { if result.Error == gorm.ErrRecordNotFound { return found, &ce.DaoError{NotFound: true, Message: "Could not find repository with UUID " + uuid} - } else { - return found, DBErrorToApi(result.Error) } + return found, DBErrorToApi(result.Error) } return found, nil } @@ -417,9 +436,8 @@ func (r repositoryConfigDaoImpl) FetchByRepoUuid(orgID string, repoUuid string) if result.Error != nil { if result.Error == gorm.ErrRecordNotFound { return repo, &ce.DaoError{NotFound: true, Message: "Could not find repository with UUID " + repoUuid} - } else { - return repo, DBErrorToApi(result.Error) } + return repo, DBErrorToApi(result.Error) } ModelToApiFields(repoConfig, &repo) @@ -435,7 +453,8 @@ func (r repositoryConfigDaoImpl) Update(orgID, uuid string, repoParams api.Repos // We are updating the repo config & snapshots, so bundle in a transaction err = r.db.Transaction(func(tx *gorm.DB) error { - if repoConfig, err = r.fetchRepoConfig(orgID, uuid); err != nil { + // Setting "includeRedHatRepos" to false here to prevent updating red_hat repositories + if repoConfig, err = r.fetchRepoConfig(orgID, uuid, false); err != nil { return err } ApiFieldsToModel(repoParams, &repoConfig, &repo) @@ -526,7 +545,7 @@ func (r repositoryConfigDaoImpl) SoftDelete(orgID string, uuid string) error { var repoConfig models.RepositoryConfiguration var err error - if repoConfig, err = r.fetchRepoConfig(orgID, uuid); err != nil { + if repoConfig, err = r.fetchRepoConfig(orgID, uuid, false); err != nil { return err } @@ -587,7 +606,7 @@ func (r repositoryConfigDaoImpl) bulkDelete(tx *gorm.DB, orgID string, uuids []s var err error var repoConfig models.RepositoryConfiguration - if repoConfig, err = r.fetchRepoConfig(orgID, uuids[i]); err != nil { + if repoConfig, err = r.fetchRepoConfig(orgID, uuids[i], false); err != nil { dbErr = DBErrorToApi(err) errors[i] = dbErr tx.RollbackTo(save) diff --git a/pkg/dao/repository_configs_mock.go b/pkg/dao/repository_configs_mock.go index 62393be53..01d6d7516 100644 --- a/pkg/dao/repository_configs_mock.go +++ b/pkg/dao/repository_configs_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package dao diff --git a/pkg/dao/repository_configs_test.go b/pkg/dao/repository_configs_test.go index 0912cdd19..f260128a4 100644 --- a/pkg/dao/repository_configs_test.go +++ b/pkg/dao/repository_configs_test.go @@ -109,6 +109,22 @@ func (suite *RepositoryConfigSuite) TestCreateTwiceWithNoSlash() { assert.ErrorContains(suite.T(), err, "Name cannot be blank") } +func (suite *RepositoryConfigSuite) TestCreateRedHatRepository() { + toCreate := api.RepositoryRequest{ + Name: pointy.String(""), + URL: pointy.String("something-no-slash"), + OrgID: pointy.String(config.RedHatOrg), + AccountID: pointy.String("123"), + DistributionArch: pointy.String(""), + DistributionVersions: &[]string{ + config.El9, + }, + } + dao := GetRepositoryConfigDao(suite.tx) + _, err := dao.Create(toCreate) + assert.ErrorContains(suite.T(), err, "Creating of Red Hat repositories is not permitted") +} + func (suite *RepositoryConfigSuite) TestRepositoryCreateAlreadyExists() { t := suite.T() tx := suite.tx diff --git a/pkg/dao/rpms.go b/pkg/dao/rpms.go index 2aa5d23c2..2d09dbd4d 100644 --- a/pkg/dao/rpms.go +++ b/pkg/dao/rpms.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/content-services/content-sources-backend/pkg/api" + "github.com/content-services/content-sources-backend/pkg/config" ce "github.com/content-services/content-sources-backend/pkg/errors" "github.com/content-services/content-sources-backend/pkg/models" "github.com/content-services/yummy/pkg/yum" @@ -28,7 +29,7 @@ func (r rpmDaoImpl) isOwnedRepository(orgID string, repositoryConfigUUID string) var repoConfigs []models.RepositoryConfiguration var count int64 if err := r.db. - Where("org_id = ? and uuid = ?", orgID, UuidifyString(repositoryConfigUUID)). + Where("org_id IN (?, ?) AND uuid = ?", orgID, config.RedHatOrg, UuidifyString(repositoryConfigUUID)). Find(&repoConfigs). Count(&count). Error; err != nil { @@ -40,7 +41,13 @@ func (r rpmDaoImpl) isOwnedRepository(orgID string, repositoryConfigUUID string) return true, nil } -func (r rpmDaoImpl) List(orgID string, repositoryConfigUUID string, limit int, offset int, search string, sortBy string) (api.RepositoryRpmCollectionResponse, int64, error) { +func (r rpmDaoImpl) List( + orgID string, + repositoryConfigUUID string, + limit int, offset int, + search string, + sortBy string, +) (api.RepositoryRpmCollectionResponse, int64, error) { // Check arguments if orgID == "" { return api.RepositoryRpmCollectionResponse{}, 0, fmt.Errorf("orgID can not be an empty string") diff --git a/pkg/dao/rpms_mock.go b/pkg/dao/rpms_mock.go index 155968833..e64417858 100644 --- a/pkg/dao/rpms_mock.go +++ b/pkg/dao/rpms_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.20.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package dao @@ -109,13 +109,12 @@ func (_m *MockRpmDao) Search(orgID string, request api.SearchRpmRequest) ([]api. return r0, r1 } -type mockConstructorTestingTNewMockRpmDao interface { +// NewMockRpmDao creates a new instance of MockRpmDao. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockRpmDao(t interface { mock.TestingT Cleanup(func()) -} - -// NewMockRpmDao creates a new instance of MockRpmDao. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewMockRpmDao(t mockConstructorTestingTNewMockRpmDao) *MockRpmDao { +}) *MockRpmDao { mock := &MockRpmDao{} mock.Mock.Test(t) diff --git a/pkg/dao/rpms_test.go b/pkg/dao/rpms_test.go index 5815a1a94..9707a8f3e 100644 --- a/pkg/dao/rpms_test.go +++ b/pkg/dao/rpms_test.go @@ -109,6 +109,67 @@ func (s *RpmSuite) TestRpmList() { assert.Equal(t, count, int64(0)) } +func (s *RpmSuite) TestRpmListRedHatRepositories() { + var err error + t := s.Suite.T() + + redHatRepo := repoPublicTest.DeepCopy() + redHatRepo.URL = "https://www.public.redhat.com" + if err := s.tx.Create(redHatRepo).Error; err != nil { + s.FailNow("Preparing Repository record: %w", err) + } + + redhatRepoConfig := repoConfigTest1.DeepCopy() + redhatRepoConfig.OrgID = config.RedHatOrg + redhatRepoConfig.Name = "Demo Redhat Repository Config" + redhatRepoConfig.RepositoryUUID = redHatRepo.Base.UUID + if err := s.tx.Create(redhatRepoConfig).Error; err != nil { + s.FailNow("Preparing RepositoryConfiguration record: %w", err) + } + + // Prepare RepositoryRpm records + rpm1 := repoRpmTest1.DeepCopy() + rpm2 := repoRpmTest2.DeepCopy() + dao := GetRpmDao(s.tx) + + err = s.tx.Create(&rpm1).Error + assert.NoError(t, err) + err = s.tx.Create(&rpm2).Error + assert.NoError(t, err) + + // Add one red hat repo + err = s.tx.Create(&models.RepositoryRpm{ + RepositoryUUID: redHatRepo.Base.UUID, + RpmUUID: rpm1.Base.UUID, + }).Error + assert.NoError(t, err) + + //Add one regular repository + err = s.tx.Create(&models.RepositoryRpm{ + RepositoryUUID: s.repo.Base.UUID, + RpmUUID: rpm2.Base.UUID, + }).Error + + assert.NoError(t, err) + + var repoRpmList api.RepositoryRpmCollectionResponse + var count int64 + + // Check red hat repo package (matched "-1" orgID) + repoRpmList, count, err = dao.List("ThisOrgIdWontMatter", redhatRepoConfig.Base.UUID, 10, 0, "", "") + assert.NoError(t, err) + assert.Equal(t, int64(1), count) + assert.Equal(t, repoRpmList.Meta.Count, count) + assert.Equal(t, repoRpmTest1.Name, repoRpmList.Data[0].Name) // Asserts name:asc by default + + // Check custom repo package (checks orgId) + repoRpmList, count, err = dao.List(orgIDTest, s.repoConfig.Base.UUID, 10, 0, "", "") + assert.NoError(t, err) + assert.Equal(t, int64(1), count) + assert.Equal(t, repoRpmList.Meta.Count, count) + assert.Equal(t, repoRpmTest2.Name, repoRpmList.Data[0].Name) // Asserts name:asc by default +} + func (s *RpmSuite) TestRpmListRepoNotFound() { t := s.Suite.T() dao := GetRpmDao(s.tx) diff --git a/pkg/dao/snapshots.go b/pkg/dao/snapshots.go index a0dfd463d..eaa3d2827 100644 --- a/pkg/dao/snapshots.go +++ b/pkg/dao/snapshots.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/content-services/content-sources-backend/pkg/api" + "github.com/content-services/content-sources-backend/pkg/config" ce "github.com/content-services/content-sources-backend/pkg/errors" "github.com/content-services/content-sources-backend/pkg/models" "github.com/content-services/content-sources-backend/pkg/pulp_client" @@ -48,17 +49,28 @@ func (sDao *snapshotDaoImpl) Create(s *models.Snapshot) error { } // List the snapshots for a given repository config -func (sDao *snapshotDaoImpl) List(repoConfigUuid string, paginationData api.PaginationData, _ api.FilterData) (api.SnapshotCollectionResponse, int64, error) { +func (sDao *snapshotDaoImpl) List( + orgID string, + repoConfigUUID string, + paginationData api.PaginationData, + _ api.FilterData, +) (api.SnapshotCollectionResponse, int64, error) { var snaps []models.Snapshot var totalSnaps int64 var repoConfig models.RepositoryConfiguration // First check if repo config exists - result := sDao.db.Where("uuid = ?", UuidifyString(repoConfigUuid)).First(&repoConfig) + result := sDao.db.Where( + "repository_configurations.org_id IN (?,?) AND uuid = ?", + orgID, + config.RedHatOrg, + UuidifyString(repoConfigUUID)). + First(&repoConfig) + if result.Error != nil { if result.Error == gorm.ErrRecordNotFound { return api.SnapshotCollectionResponse{}, totalSnaps, &ce.DaoError{ - Message: "Could not find repository with UUID " + repoConfigUuid, + Message: "Could not find repository with UUID " + repoConfigUUID, NotFound: true, } } @@ -71,12 +83,13 @@ func (sDao *snapshotDaoImpl) List(repoConfigUuid string, paginationData api.Pagi order := convertSortByToSQL(paginationData.SortBy, sortMap, "created_at asc") filteredDB := sDao.db. - Where("snapshots.repository_configuration_uuid = ?", UuidifyString(repoConfigUuid)) + Model(&models.Snapshot{}). + Joins("JOIN repository_configurations ON repository_configuration_uuid = repository_configurations.uuid"). + Where("repository_configuration_uuid = ?", UuidifyString(repoConfigUUID)). + Where("repository_configurations.org_id IN (?,?)", orgID, config.RedHatOrg) // Get count - filteredDB. - Model(&snaps). - Count(&totalSnaps) + filteredDB.Count(&totalSnaps) if filteredDB.Error != nil { return api.SnapshotCollectionResponse{}, 0, filteredDB.Error @@ -123,7 +136,7 @@ func (sDao *snapshotDaoImpl) Fetch(uuid string) (models.Snapshot, error) { func (sDao *snapshotDaoImpl) GetRepositoryConfigurationFile(orgID, snapshotUUID, repoConfigUUID string) (string, error) { rcDao := repositoryConfigDaoImpl{db: sDao.db} - repoConfig, err := rcDao.fetchRepoConfig(orgID, repoConfigUUID) + repoConfig, err := rcDao.fetchRepoConfig(orgID, repoConfigUUID, true) if err != nil { return "", err } diff --git a/pkg/dao/snapshots_mock.go b/pkg/dao/snapshots_mock.go index 476066758..c370f1c83 100644 --- a/pkg/dao/snapshots_mock.go +++ b/pkg/dao/snapshots_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package dao @@ -133,30 +133,30 @@ func (_m *MockSnapshotDao) InitializePulpClient(ctx context.Context, orgID strin return r0 } -// List provides a mock function with given fields: repoConfigUuid, paginationData, _a2 -func (_m *MockSnapshotDao) List(repoConfigUuid string, paginationData api.PaginationData, _a2 api.FilterData) (api.SnapshotCollectionResponse, int64, error) { - ret := _m.Called(repoConfigUuid, paginationData, _a2) +// List provides a mock function with given fields: orgID, repoConfigUuid, paginationData, filterData +func (_m *MockSnapshotDao) List(orgID string, repoConfigUuid string, paginationData api.PaginationData, filterData api.FilterData) (api.SnapshotCollectionResponse, int64, error) { + ret := _m.Called(orgID, repoConfigUuid, paginationData, filterData) var r0 api.SnapshotCollectionResponse var r1 int64 var r2 error - if rf, ok := ret.Get(0).(func(string, api.PaginationData, api.FilterData) (api.SnapshotCollectionResponse, int64, error)); ok { - return rf(repoConfigUuid, paginationData, _a2) + if rf, ok := ret.Get(0).(func(string, string, api.PaginationData, api.FilterData) (api.SnapshotCollectionResponse, int64, error)); ok { + return rf(orgID, repoConfigUuid, paginationData, filterData) } - if rf, ok := ret.Get(0).(func(string, api.PaginationData, api.FilterData) api.SnapshotCollectionResponse); ok { - r0 = rf(repoConfigUuid, paginationData, _a2) + if rf, ok := ret.Get(0).(func(string, string, api.PaginationData, api.FilterData) api.SnapshotCollectionResponse); ok { + r0 = rf(orgID, repoConfigUuid, paginationData, filterData) } else { r0 = ret.Get(0).(api.SnapshotCollectionResponse) } - if rf, ok := ret.Get(1).(func(string, api.PaginationData, api.FilterData) int64); ok { - r1 = rf(repoConfigUuid, paginationData, _a2) + if rf, ok := ret.Get(1).(func(string, string, api.PaginationData, api.FilterData) int64); ok { + r1 = rf(orgID, repoConfigUuid, paginationData, filterData) } else { r1 = ret.Get(1).(int64) } - if rf, ok := ret.Get(2).(func(string, api.PaginationData, api.FilterData) error); ok { - r2 = rf(repoConfigUuid, paginationData, _a2) + if rf, ok := ret.Get(2).(func(string, string, api.PaginationData, api.FilterData) error); ok { + r2 = rf(orgID, repoConfigUuid, paginationData, filterData) } else { r2 = ret.Error(2) } diff --git a/pkg/dao/snapshots_test.go b/pkg/dao/snapshots_test.go index 33303075c..7f69d201f 100644 --- a/pkg/dao/snapshots_test.go +++ b/pkg/dao/snapshots_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/content-services/content-sources-backend/pkg/api" + "github.com/content-services/content-sources-backend/pkg/config" ce "github.com/content-services/content-sources-backend/pkg/errors" "github.com/content-services/content-sources-backend/pkg/models" "github.com/content-services/content-sources-backend/pkg/pulp_client" @@ -59,6 +60,29 @@ func (s *SnapshotsSuite) createRepository() models.RepositoryConfiguration { return rConfig } +func (s *SnapshotsSuite) createRedhatRepository() models.RepositoryConfiguration { + t := s.T() + tx := s.tx + + testRepository := models.Repository{ + URL: "https://example.redhat.com", + LastIntrospectionTime: nil, + LastIntrospectionError: nil, + } + err := tx.Create(&testRepository).Error + assert.NoError(t, err) + + rConfig := models.RepositoryConfiguration{ + Name: "redhatSnapshot", + OrgID: config.RedHatOrg, + RepositoryUUID: testRepository.UUID, + } + + err = tx.Create(&rConfig).Error + assert.NoError(t, err) + return rConfig +} + func (s *SnapshotsSuite) createSnapshot(rConfig models.RepositoryConfiguration) models.Snapshot { t := s.T() tx := s.tx @@ -90,6 +114,7 @@ func (s *SnapshotsSuite) TestCreateAndList() { repoDao := repositoryConfigDaoImpl{db: tx, yumRepo: &mockExt.YumRepositoryMock{}} rConfig := s.createRepository() + pageData := api.PaginationData{ Limit: 100, Offset: 0, @@ -102,9 +127,9 @@ func (s *SnapshotsSuite) TestCreateAndList() { snap := s.createSnapshot(rConfig) - collection, total, err := sDao.List(rConfig.UUID, pageData, filterData) + collection, total, err := sDao.List(rConfig.OrgID, rConfig.UUID, pageData, filterData) - repository, _ := repoDao.fetchRepoConfig(rConfig.OrgID, rConfig.UUID) + repository, _ := repoDao.fetchRepoConfig(rConfig.OrgID, rConfig.UUID, false) repositoryList, repoCount, _ := repoDao.List(rConfig.OrgID, api.PaginationData{Limit: -1}, api.FilterData{}) assert.NoError(t, err) @@ -128,6 +153,55 @@ func (s *SnapshotsSuite) TestCreateAndList() { } } +func (s *SnapshotsSuite) TestCreateAndListRedHatRepo() { + t := s.T() + tx := s.tx + + mockPulpClient := pulp_client.NewMockPulpClient(t) + sDao := snapshotDaoImpl{db: tx, pulpClient: mockPulpClient} + mockPulpClient.On("GetContentPath").Return(testContentPath, nil) + + repoDao := repositoryConfigDaoImpl{db: tx, yumRepo: &mockExt.YumRepositoryMock{}} + + redhatRepositoryConfig := s.createRedhatRepository() + redhatSnap := s.createSnapshot(redhatRepositoryConfig) + + pageData := api.PaginationData{ + Limit: 100, + Offset: 0, + } + filterData := api.FilterData{ + Search: "", + Arch: "", + Version: "", + } + + collection, total, err := sDao.List("ShouldNotMatter", redhatRepositoryConfig.UUID, pageData, filterData) + + repository, _ := repoDao.fetchRepoConfig("ShouldNotMatter", redhatRepositoryConfig.UUID, true) + repositoryList, repoCount, _ := repoDao.List("ShouldNotMatter", api.PaginationData{Limit: -1}, api.FilterData{}) + + assert.NoError(t, err) + assert.Equal(t, int64(1), total) + assert.Equal(t, 1, len(collection.Data)) + if len(collection.Data) > 0 { + assert.Equal(t, redhatSnap.RepositoryPath, collection.Data[0].RepositoryPath) + assert.Equal(t, redhatSnap.ContentCounts, models.ContentCountsType(collection.Data[0].ContentCounts)) + assert.Equal(t, redhatSnap.AddedCounts, models.ContentCountsType(collection.Data[0].AddedCounts)) + assert.Equal(t, redhatSnap.RemovedCounts, models.ContentCountsType(collection.Data[0].RemovedCounts)) + assert.False(t, collection.Data[0].CreatedAt.IsZero()) + // Check that the repositoryConfig has the appropriate values + assert.Equal(t, redhatSnap.UUID, repository.LastSnapshotUUID) + assert.EqualValues(t, redhatSnap.AddedCounts, repository.LastSnapshot.AddedCounts) + assert.EqualValues(t, redhatSnap.RemovedCounts, repository.LastSnapshot.RemovedCounts) + // Check that the list repositoryConfig has the appropriate values + assert.Equal(t, int64(1), repoCount) + assert.Equal(t, redhatSnap.UUID, repositoryList.Data[0].LastSnapshotUUID) + assert.EqualValues(t, redhatSnap.AddedCounts, repositoryList.Data[0].LastSnapshot.AddedCounts) + assert.EqualValues(t, redhatSnap.RemovedCounts, repositoryList.Data[0].LastSnapshot.RemovedCounts) + } +} + func (s *SnapshotsSuite) TestListNoSnapshots() { t := s.T() tx := s.tx @@ -161,7 +235,7 @@ func (s *SnapshotsSuite) TestListNoSnapshots() { err = tx.Create(&rConfig).Error assert.NoError(t, err) - collection, total, err := sDao.List(rConfig.UUID, pageData, filterData) + collection, total, err := sDao.List(rConfig.OrgID, rConfig.UUID, pageData, filterData) assert.NoError(t, err) assert.Equal(t, int64(0), total) assert.Equal(t, 0, len(collection.Data)) @@ -190,7 +264,7 @@ func (s *SnapshotsSuite) TestListPageLimit() { s.createSnapshot(rConfig) } - collection, total, err := sDao.List(rConfig.UUID, pageData, filterData) + collection, total, err := sDao.List(rConfig.OrgID, rConfig.UUID, pageData, filterData) assert.NoError(t, err) assert.Equal(t, int64(11), total) assert.Equal(t, 10, len(collection.Data)) @@ -215,13 +289,58 @@ func (s *SnapshotsSuite) TestListNotFound() { s.createSnapshot(rConfig) - collection, total, err := sDao.List("bad-uuid", pageData, filterData) + collection, total, err := sDao.List(rConfig.OrgID, "bad-uuid", pageData, filterData) + assert.Error(t, err) + daoError, ok := err.(*ce.DaoError) + assert.True(t, ok) + assert.True(t, daoError.NotFound) + assert.Equal(t, int64(0), total) + assert.Equal(t, 0, len(collection.Data)) +} + +func (s *SnapshotsSuite) TestListNotFoundBadOrgId() { + t := s.T() + tx := s.tx + + sDao := snapshotDaoImpl{db: tx} + + testRepository := models.Repository{ + URL: "https://example.com", + LastIntrospectionTime: nil, + LastIntrospectionError: nil, + } + err := tx.Create(&testRepository).Error + assert.NoError(t, err) + + rConfig := models.RepositoryConfiguration{ + Name: "toSnapshot", + OrgID: "not-banana-id", + RepositoryUUID: testRepository.UUID, + } + + err = tx.Create(&rConfig).Error + assert.NoError(t, err) + + pageData := api.PaginationData{ + Limit: 100, + Offset: 0, + } + filterData := api.FilterData{ + Search: "", + Arch: "", + Version: "", + } + + s.createSnapshot(rConfig) + + collection, total, err := sDao.List("bad-banana-id", rConfig.UUID, pageData, filterData) assert.Error(t, err) daoError, ok := err.(*ce.DaoError) assert.True(t, ok) assert.True(t, daoError.NotFound) assert.Equal(t, int64(0), total) assert.Equal(t, 0, len(collection.Data)) + assert.ErrorContains(t, err, "Could not find repository with UUID "+rConfig.UUID) } func (s *SnapshotsSuite) TestFetchForRepoUUID() { diff --git a/pkg/dao/task_info_mock.go b/pkg/dao/task_info_mock.go index fed4d0a88..c75e6ed37 100644 --- a/pkg/dao/task_info_mock.go +++ b/pkg/dao/task_info_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.20.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package dao @@ -105,13 +105,12 @@ func (_m *MockTaskInfoDao) List(OrgID string, pageData api.PaginationData, filte return r0, r1, r2 } -type mockConstructorTestingTNewMockTaskInfoDao interface { +// NewMockTaskInfoDao creates a new instance of MockTaskInfoDao. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockTaskInfoDao(t interface { mock.TestingT Cleanup(func()) -} - -// NewMockTaskInfoDao creates a new instance of MockTaskInfoDao. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewMockTaskInfoDao(t mockConstructorTestingTNewMockTaskInfoDao) *MockTaskInfoDao { +}) *MockTaskInfoDao { mock := &MockTaskInfoDao{} mock.Mock.Test(t) diff --git a/pkg/handler/snapshots.go b/pkg/handler/snapshots.go index 6193d2102..fe4240e9f 100644 --- a/pkg/handler/snapshots.go +++ b/pkg/handler/snapshots.go @@ -51,7 +51,7 @@ func (sh *SnapshotHandler) listSnapshots(c echo.Context) error { return ce.NewErrorResponse(ce.HttpCodeForDaoError(err), "Error initializing pulp client", err.Error()) } - snapshots, totalSnaps, err := sh.DaoRegistry.Snapshot.List(uuid, pageData, filterData) + snapshots, totalSnaps, err := sh.DaoRegistry.Snapshot.List(orgID, uuid, pageData, filterData) if err != nil { return ce.NewErrorResponse(ce.HttpCodeForDaoError(err), "Error listing repository snapshots", err.Error()) } diff --git a/pkg/handler/snapshots_test.go b/pkg/handler/snapshots_test.go index 77f3bc5e1..1f0bcf455 100644 --- a/pkg/handler/snapshots_test.go +++ b/pkg/handler/snapshots_test.go @@ -66,7 +66,7 @@ func (suite *SnapshotSuite) TestSnapshotList() { uuid := "abcadaba" orgID := test_handler.MockOrgId suite.reg.Snapshot.On("InitializePulpClient", mock.AnythingOfType("*context.valueCtx"), orgID).Return(nil).Once() - suite.reg.Snapshot.On("List", uuid, paginationData, api.FilterData{}).Return(collection, int64(1), nil) + suite.reg.Snapshot.On("List", test_handler.MockOrgId, uuid, paginationData, api.FilterData{}).Return(collection, int64(1), nil) path := fmt.Sprintf("%s/repositories/%s/snapshots/?limit=%d", fullRootPath(), uuid, 10) req := httptest.NewRequest(http.MethodGet, path, nil) diff --git a/pkg/middleware/metrics.go b/pkg/middleware/metrics.go index 9117336d8..b440fe70b 100644 --- a/pkg/middleware/metrics.go +++ b/pkg/middleware/metrics.go @@ -56,8 +56,9 @@ func MetricsMiddlewareWithConfig(config *MetricsConfig) echo.MiddlewareFunc { method := ctx.Request().Method path := MatchedRoute(ctx) err := next(ctx) + timeStart := time.Since(start) status := mapStatus(ctx.Response().Status) - defer config.Metrics.HttpStatusHistogram.WithLabelValues(status, method, path).Observe(time.Since(start).Seconds()) + defer config.Metrics.HttpStatusHistogram.WithLabelValues(status, method, path).Observe(timeStart.Seconds()) return err } } diff --git a/pkg/pulp_client/pulp_client_mock.go b/pkg/pulp_client/pulp_client_mock.go index c81f27b1a..ab4d8bef0 100644 --- a/pkg/pulp_client/pulp_client_mock.go +++ b/pkg/pulp_client/pulp_client_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package pulp_client diff --git a/pkg/pulp_client/pulp_global_client_mock.go b/pkg/pulp_client/pulp_global_client_mock.go index 7076d3667..3d12412f5 100644 --- a/pkg/pulp_client/pulp_global_client_mock.go +++ b/pkg/pulp_client/pulp_global_client_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package pulp_client diff --git a/pkg/tasks/client/client_mock.go b/pkg/tasks/client/client_mock.go index 24f083d24..48fc493d5 100644 --- a/pkg/tasks/client/client_mock.go +++ b/pkg/tasks/client/client_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package client diff --git a/pkg/tasks/queue/queue_mock.go b/pkg/tasks/queue/queue_mock.go index 522917c77..d8832cc7c 100644 --- a/pkg/tasks/queue/queue_mock.go +++ b/pkg/tasks/queue/queue_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.33.0. DO NOT EDIT. package queue diff --git a/test/integration/snapshot_test.go b/test/integration/snapshot_test.go index 10f8132cb..e59d885bf 100644 --- a/test/integration/snapshot_test.go +++ b/test/integration/snapshot_test.go @@ -87,7 +87,7 @@ func (s *SnapshotSuite) TestSnapshot() { s.snapshotAndWait(taskClient, repo, repoUuid, accountId) // Verify the snapshot was created - snaps, _, err := s.dao.Snapshot.List(repo.UUID, api.PaginationData{Limit: -1}, api.FilterData{}) + snaps, _, err := s.dao.Snapshot.List(repo.OrgID, repo.UUID, api.PaginationData{Limit: -1}, api.FilterData{}) assert.NoError(s.T(), err) assert.NotEmpty(s.T(), snaps) time.Sleep(5 * time.Second) @@ -138,7 +138,7 @@ func (s *SnapshotSuite) TestSnapshot() { s.WaitOnTask(taskUuid) // Verify the snapshot was deleted - snaps, _, err = s.dao.Snapshot.List(repo.UUID, api.PaginationData{Limit: -1}, api.FilterData{}) + snaps, _, err = s.dao.Snapshot.List(repo.OrgID, repo.UUID, api.PaginationData{Limit: -1}, api.FilterData{}) assert.Error(s.T(), err) assert.Empty(s.T(), snaps.Data) time.Sleep(5 * time.Second) @@ -185,7 +185,7 @@ func (s *SnapshotSuite) snapshotAndWait(taskClient client.TaskClient, repo api.R s.WaitOnTask(taskUuid) // Verify the snapshot was created - snaps, _, err := s.dao.Snapshot.List(repo.UUID, api.PaginationData{Limit: -1}, api.FilterData{}) + snaps, _, err := s.dao.Snapshot.List(repo.OrgID, repo.UUID, api.PaginationData{Limit: -1}, api.FilterData{}) assert.NoError(s.T(), err) assert.NotEmpty(s.T(), snaps) time.Sleep(5 * time.Second) @@ -211,7 +211,7 @@ func (s *SnapshotSuite) cancelAndWait(taskClient client.TaskClient, taskUUID uui s.WaitOnCanceledTask(taskUUID) // Verify the snapshot was not created - snaps, _, err := s.dao.Snapshot.List(repo.UUID, api.PaginationData{}, api.FilterData{}) + snaps, _, err := s.dao.Snapshot.List(repo.OrgID, repo.UUID, api.PaginationData{}, api.FilterData{}) assert.NoError(s.T(), err) assert.Equal(s.T(), api.SnapshotCollectionResponse{Data: []api.SnapshotResponse{}}, snaps) }