Fixes 2702,2887: Add orgID to snapshot list endpoint (#435)

* Fixes 2702,2887: Add orgID to snapshot list endpoint * Add command to snapshot a rhel repo * Add debug to figure out timing issue * Revert "Add debug to figure out timing issue" This reverts commit 9318ada. * adjust snapshot verbage * Update error if is not found error * Add test to ensure the 404 is returned --------- Co-authored-by: Justin Sherrill <[email protected]>
content-services · Nov 13, 2023 · ab41987 · ab41987
1 parent 0316cb8
commit ab41987
Show file tree

Hide file tree

Showing 26 changed files with 494 additions and 155 deletions.
diff --git a/README.md b/README.md
@@ -1,8 +1,8 @@
 # Content Sources
 
 ## What is it?
-Content Sources is an application for storing information about external content (currently YUM repositories) in a central location as well as creating snapshots of those repositories, backed by a Pulp server.
 
+Content Sources is an application for storing information about external content (currently YUM repositories) in a central location as well as creating snapshots of those repositories, backed by a Pulp server.
 
 ## Developing
 
@@ -16,67 +16,68 @@ Content Sources is an application for storing information about external content
 
 Create a config file from the example:
 
-  ```sh
-  $ cp ./configs/config.yaml.example ./configs/config.yaml
-  ```
+```sh
+$ cp ./configs/config.yaml.example ./configs/config.yaml
+```
 
 ### Build needed kafka container
 
-  ```sh
-  $ make compose-build
-  ```
+```sh
+$ make compose-build
+```
 
 ### Start dependency containers
 
-  ```sh
-  $ make compose-up
-  ```
+```sh
+$ make compose-up
+```
 
 ### Run the server!
 
-  ```sh
-  $ make run
-  ```
+```sh
+$ make run
+```
 
 ###
 
 Hit the API:
 
 ```sh
   $ curl -H "$( ./scripts/header.sh 9999 1111 )" http://localhost:8000/api/content-sources/v1.0/repositories/
-  ```
+```
 
 ### Stop dependency containers
+
 When its time to shut down the running containers:
 
-  ```sh
-  $ make compose-down
-  ```
+```sh
+$ make compose-down
+```
 
 And clean the volume that it uses by (this stops the container before doing it if it were running):
 
-  ```sh
-  $ make compose-clean
-  ```
-
-> There are other make rules that could be helpful, run `make help` to list them.  Some are highlighted below
+```sh
+$ make compose-clean
+```
 
+> There are other make rules that could be helpful, run `make help` to list them. Some are highlighted below
 
 ### HOW TO ADD NEW MIGRATION FILES
-You can add new migration files, with the prefixed date attached to the file name, by running the following: 
+
+You can add new migration files, with the prefixed date attached to the file name, by running the following:
 
 ```
 $ go run cmd/dbmigrate/main.go new <name of migration>
 ```
 
 ### Database Commands
+
 Migrate the Database
 
 ```sh
 $ make db-migrate-up
 ```
 
-
 Seed the database
 
 ```sh
@@ -85,9 +86,9 @@ $ make db-migrate-seed
 
 Get an interactive shell:
 
-  ```sh
-  $ make db-shell
-  ```
+```sh
+$ make db-shell
+```
 
 Or open directly a postgres client by running:
 
@@ -125,7 +126,6 @@ Update the `configs/prometheus.yaml` file to set your hostname instead of `local
 $ cat ./configs/prometheus.example.yaml | sed "s/localhost/$(hostname)/g" > ./configs/prometheus.yaml
 ```
 
-
 To start prometheus run:
 
 ```sh
@@ -157,8 +157,8 @@ $ make prometheus-ui
     rbac_timeout: 30
   mocks:
     rbac:
-      user_read_write: ["[email protected]","jdoe"]
-      user_read: ["[email protected]","tdoe"]
+      user_read_write: ["[email protected]", "jdoe"]
+      user_read: ["[email protected]", "tdoe"]
   ```
 
 **Running it**
@@ -200,35 +200,46 @@ $ curl -H "$( ./scripts/header.sh 9999 1111 )" http://localhost:8000/api/content
 $ make openapi
 ```
 
+### Generating/Update mocks:
+
+Ensure [mockery](https://vektra.github.io/mockery/latest/installation/) is installed.
+
+Then:
+
+```sh
+$ go generate ./...
+```
+
 ### Configuration
 
-The default configuration file in ./configs/config.yaml.example shows all available config options.  Any of these can be overridden with an environment variable.  For example  "database.name" can be passed in via an environment variable named "DATABASE_NAME".
+The default configuration file in ./configs/config.yaml.example shows all available config options. Any of these can be overridden with an environment variable. For example "database.name" can be passed in via an environment variable named "DATABASE_NAME".
 
 ### Linting
 
 To use golangci-lint:
+
 1. `make install-golangci-lint`
 2. `make lint`
 
 To use pre-commit linter: `make install-pre-commit`
 
 ### Code Layout
 
-| Path                              | Description                                                                                                                                                                                     |
-|-----------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| [api](./api/)                     | Openapi docs and doc generation code                                                                                                                                                            |
-| [db/migrations](./db/migrations/) | Database Migrations                                                                                                                                                                             |                                                                                                                                                                            |
-| [pkg/api](./pkg/api)              | API Structures that are used for handling data within our API Handlers                                                                                                                          |
-| [pkg/config](./pkg/config)        | Config loading and application bootstrapping code                                                                                                                                               |
-| [pkg/dao](./pkg/dao)              | Database Access Object.  Abstraction layer that provides an interface and implements it for our default database provider (postgresql).  It is separated out for abstraction and easier testing |
-| [pkg/db](./pkg/db)                | Database connection and migration related code                                                                                                                                                  |
-| [pkg/handler](./pkg/handler)      | Methods that directly handle API requests                                                                                                                                                       |
-| [pkg/middleware](./pkg/middleware)| Hold all the middleware components created for the service. |
-| [pkg/event](./pkg/event)        | Event message logic. Mre info [here](./pkg/event/README.md). |
-| [pkg/models](./pkg/models)        | Structs that represent database models (Gorm)                                                                                                                                                   |
-| [pkg/seeds](./pkg/seeds)          | Code to help seed the database for both development and testing                                                                                                                                 |
+| Path                               | Description                                                                                                                                                                                   |
+| ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --- |
+| [api](./api/)                      | Openapi docs and doc generation code                                                                                                                                                          |
+| [db/migrations](./db/migrations/)  | Database Migrations                                                                                                                                                                           |     |
+| [pkg/api](./pkg/api)               | API Structures that are used for handling data within our API Handlers                                                                                                                        |
+| [pkg/config](./pkg/config)         | Config loading and application bootstrapping code                                                                                                                                             |
+| [pkg/dao](./pkg/dao)               | Database Access Object. Abstraction layer that provides an interface and implements it for our default database provider (postgresql). It is separated out for abstraction and easier testing |
+| [pkg/db](./pkg/db)                 | Database connection and migration related code                                                                                                                                                |
+| [pkg/handler](./pkg/handler)       | Methods that directly handle API requests                                                                                                                                                     |
+| [pkg/middleware](./pkg/middleware) | Hold all the middleware components created for the service.                                                                                                                                   |
+| [pkg/event](./pkg/event)           | Event message logic. Mre info [here](./pkg/event/README.md).                                                                                                                                  |
+| [pkg/models](./pkg/models)         | Structs that represent database models (Gorm)                                                                                                                                                 |
+| [pkg/seeds](./pkg/seeds)           | Code to help seed the database for both development and testing                                                                                                                               |
 
 ## More info
 
- * [Architecture](docs/architecture.md)
- * [OpenApi Docs](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/content-services/content-sources-backend/main/api/openapi.json)
+- [Architecture](docs/architecture.md)
+- [OpenApi Docs](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/content-services/content-sources-backend/main/api/openapi.json)
diff --git a/cmd/external-repos/main.go b/cmd/external-repos/main.go
@@ -5,16 +5,19 @@ import (
 	"fmt"
 	"os"
 	"sort"
+	"time"
 
 	"github.com/content-services/content-sources-backend/pkg/config"
 	"github.com/content-services/content-sources-backend/pkg/dao"
 	"github.com/content-services/content-sources-backend/pkg/db"
 	"github.com/content-services/content-sources-backend/pkg/external_repos"
+	"github.com/content-services/content-sources-backend/pkg/pulp_client"
 	"github.com/content-services/content-sources-backend/pkg/tasks/client"
 	"github.com/content-services/content-sources-backend/pkg/tasks/payloads"
 	"github.com/content-services/content-sources-backend/pkg/tasks/queue"
 	_ "github.com/golang-migrate/migrate/v4/source/file"
 	_ "github.com/lib/pq"
+	"github.com/openlyinc/pointy"
 	"github.com/rs/zerolog/log"
 	"gorm.io/gorm"
 )
@@ -33,7 +36,7 @@ func main() {
 	}
 
 	if len(args) < 2 {
-		log.Fatal().Msg("Requires arguments: download, import, introspect, nightly-jobs")
+		log.Fatal().Msg("Requires arguments: download, import, introspect, snapshot, nightly-jobs")
 	}
 	if args[1] == "download" {
 		if len(args) < 3 {
@@ -73,13 +76,31 @@ func main() {
 			log.Panic().Err(errors[i]).Msg("Failed to introspect repository due to fatal errors")
 		}
 		log.Debug().Msgf("Inserted %d packages", count)
+	} else if args[1] == "snapshot" {
+		if len(args) < 3 {
+			log.Error().Msg("Usage:  ./external_repos snapshot URL [URL2]...")
+			os.Exit(1)
+		}
+		var urls []string
+		for i := 2; i < len(args); i++ {
+			urls = append(urls, args[i])
+		}
+		if config.Get().Features.Snapshots.Enabled {
+			waitForPulp()
+			err := enqueueSnapshotRepos(&urls)
+			if err != nil {
+				log.Warn().Msgf("Error enqueuing snapshot tasks: %v", err)
+			}
+		} else {
+			log.Warn().Msg("Snapshotting disabled")
+		}
 	} else if args[1] == "nightly-jobs" {
 		err = enqueueIntrospectAllRepos()
 		if err != nil {
 			log.Error().Err(err).Msg("error queueing introspection tasks")
 		}
 		if config.Get().Features.Snapshots.Enabled {
-			err = enqueueSyncAllRepos()
+			err = enqueueSnapshotRepos(nil)
 			if err != nil {
 				log.Error().Err(err).Msg("error queueing snapshot tasks")
 			}
@@ -110,6 +131,18 @@ func saveToDB(db *gorm.DB) error {
 	return err
 }
 
+func waitForPulp() {
+	for {
+		client := pulp_client.GetPulpClientWithDomain(context.Background(), pulp_client.DefaultDomain)
+		_, err := client.GetRpmRemoteList()
+		if err == nil {
+			return
+		}
+		log.Warn().Err(err).Msg("Pulp isn't up yet, waiting 5s.")
+		time.Sleep(5 * time.Second)
+	}
+}
+
 func scanForExternalRepos(path string) {
 	urls, err := external_repos.IBUrlsFromDir(path)
 	if err != nil {
@@ -161,15 +194,23 @@ func enqueueIntrospectAllRepos() error {
 	return nil
 }
 
-func enqueueSyncAllRepos() error {
+func enqueueSnapshotRepos(urls *[]string) error {
 	q, err := queue.NewPgQueue(db.GetUrl())
 	if err != nil {
 		return fmt.Errorf("error getting new task queue: %w", err)
 	}
 	c := client.NewTaskClient(&q)
 
 	repoConfigDao := dao.GetRepositoryConfigDao(db.DB)
-	repoConfigs, err := repoConfigDao.InternalOnly_ListReposToSnapshot()
+	var filter *dao.ListRepoFilter
+	if urls != nil {
+		filter = &dao.ListRepoFilter{
+			URLs:       urls,
+			RedhatOnly: pointy.Pointer(true),
+		}
+	}
+	repoConfigs, err := repoConfigDao.InternalOnly_ListReposToSnapshot(filter)
+
 	if err != nil {
 		return fmt.Errorf("error getting repository configurations: %w", err)
 	}

diff --git a/deployments/deployment.yaml b/deployments/deployment.yaml
@@ -52,6 +52,12 @@ objects:
                   - introspect
                   - https://cdn.redhat.com/content/dist/layered/rhel8/x86_64/ansible/2/os
                   - https://cdn.redhat.com/content/dist/rhel8/8.8/x86_64/baseos/os
+              - name: snapshot-single-repo
+                inheritEnv: true
+                args:
+                  - /external-repos
+                  - snapshot
+                  - https://cdn.redhat.com/content/dist/layered/rhel8/x86_64/ansible/2/os/
             image: ${IMAGE}:${IMAGE_TAG}
             livenessProbe:
               failureThreshold: 3

diff --git a/pkg/cache/cache_mock.go b/pkg/cache/cache_mock.go
diff --git a/pkg/dao/domain_dao_mock.go b/pkg/dao/domain_dao_mock.go
diff --git a/pkg/dao/interfaces.go b/pkg/dao/interfaces.go
@@ -44,7 +44,7 @@ type RepositoryConfigDao interface {
 	BulkCreate(newRepositories []api.RepositoryRequest) ([]api.RepositoryResponse, []error)
 	Update(orgID, uuid string, repoParams api.RepositoryRequest) (bool, error)
 	Fetch(orgID string, uuid string) (api.RepositoryResponse, error)
-	InternalOnly_ListReposToSnapshot() ([]models.RepositoryConfiguration, error)
+	InternalOnly_ListReposToSnapshot(filter *ListRepoFilter) ([]models.RepositoryConfiguration, error)
 	List(orgID string, paginationData api.PaginationData, filterData api.FilterData) (api.RepositoryCollectionResponse, int64, error)
 	Delete(orgID string, uuid string) error
 	SoftDelete(orgID string, uuid string) error
@@ -79,7 +79,7 @@ type RepositoryDao interface {
 //go:generate mockery --name SnapshotDao --filename snapshots_mock.go --inpackage
 type SnapshotDao interface {
 	Create(snap *models.Snapshot) error
-	List(repoConfigUuid string, paginationData api.PaginationData, _ api.FilterData) (api.SnapshotCollectionResponse, int64, error)
+	List(orgID string, repoConfigUuid string, paginationData api.PaginationData, filterData api.FilterData) (api.SnapshotCollectionResponse, int64, error)
 	FetchForRepoConfigUUID(repoConfigUUID string) ([]models.Snapshot, error)
 	Delete(snapUUID string) error
 	FetchLatestSnapshot(repoConfigUUID string) (api.SnapshotResponse, error)

diff --git a/pkg/dao/metrics_mock.go b/pkg/dao/metrics_mock.go