Fixes 2282: reduce number of introspect tasks

cmd/external-repos/main.go

@@ -22,10 +22,6 @@ import (
 	"gorm.io/gorm"
 )
 
-var (
-	forceIntrospect bool = false
-)
-
 func main() {
 	args := os.Args
 	config.Load()
@@ -60,22 +56,15 @@ func main() {
 			os.Exit(1)
 		}
 		var urls []string
+		forceIntrospect := false
 		for i := 2; i < len(args); i++ {
 			if args[i] != "--force" {
 				urls = append(urls, args[i])
 			} else {
 				forceIntrospect = true
 			}
 		}
-
-		count, introErrors, errors := external_repos.IntrospectAll(context.Background(), &urls, forceIntrospect)
-		for i := 0; i < len(introErrors); i++ {
-			log.Warn().Msgf("Introspection Error: %v", introErrors[i].Error())
-		}
-		for i := 0; i < len(errors); i++ {
-			log.Panic().Err(errors[i]).Msg("Failed to introspect repository due to fatal errors")
-		}
-		log.Debug().Msgf("Inserted %d packages", count)
+		introspectUrls(urls, forceIntrospect)
 	} else if args[1] == "snapshot" {
 		if len(args) < 3 {
 			log.Error().Msg("Usage:  ./external_repos snapshot URL [URL2]...")
@@ -143,6 +132,23 @@ func waitForPulp() {
 	}
 }
 
+func introspectUrls(urls []string, force bool) {
+	repos, err := dao.GetDaoRegistry(db.DB).Repository.ListForIntrospection(&urls, force)
+	if err != nil {
+		log.Fatal().Err(err).Msg("Could not lookup repos to introspect")
+	}
+	for _, repo := range repos {
+		count, introError, error := external_repos.IntrospectUrl(context.Background(), repo.URL)
+		if introError != nil {
+			log.Warn().Msgf("Introspection Error: %v", introError)
+		}
+		if error != nil {
+			log.Panic().Err(error).Msg("Failed to introspect repository due to fatal errors")
+		}
+		log.Debug().Msgf("Inserted %d packages for %v", count, repo.URL)
+	}
+}
+
 func scanForExternalRepos(path string) {
 	urls, err := external_repos.IBUrlsFromDir(path)
 	if err != nil {
@@ -173,7 +179,7 @@ func enqueueIntrospectAllRepos() error {
 		log.Err(err).Msg("error during task cleanup")
 	}
 
-	repos, err := repoDao.List(true)
+	repos, err := repoDao.ListForIntrospection(nil, false)
 	if err != nil {
 		return fmt.Errorf("error getting repositories: %w", err)
 	}

pkg/config/value_constraints.go

@@ -1,5 +1,7 @@
 package config
 
+import "time"
+
 const (
 	StatusValid       = "Valid"       // Repository introspected successfully
 	StatusUnavailable = "Unavailable" // Repository introspected at least once, but now errors
@@ -18,6 +20,8 @@ const (
 
 const RedHatOrg = "-1"
 
+const IntrospectTimeInterval = time.Hour * 23
+
 const ANY_VERSION = "any"
 const El7 = "7"
 const El8 = "8"

pkg/dao/interfaces.go

@@ -76,7 +76,7 @@ type RpmDao interface {
 //go:generate mockery --name RepositoryDao --filename repositories_mock.go --inpackage
 type RepositoryDao interface {
 	FetchForUrl(url string) (Repository, error)
-	List(ignoreFailed bool) ([]Repository, error)
+	ListForIntrospection(urls *[]string, force bool) ([]Repository, error)
 	ListPublic(paginationData api.PaginationData, _ api.FilterData) (api.PublicRepositoryCollectionResponse, int64, error)
 	Update(repo RepositoryUpdate) error
 	FetchRepositoryRPMCount(repoUUID string) (int, error)

pkg/dao/repositories.go

@@ -72,17 +72,27 @@ func (p repositoryDaoImpl) FetchForUrl(url string) (Repository, error) {
 	return internalRepo, nil
 }
 
-func (p repositoryDaoImpl) List(ignoreFailed bool) ([]Repository, error) {
+func (p repositoryDaoImpl) ListForIntrospection(urls *[]string, force bool) ([]Repository, error) {
 	var dbRepos []models.Repository
 	var repos []Repository
 	var repo Repository
-	var result *gorm.DB
 
-	if ignoreFailed {
-		result = p.db.Where("public = true OR failed_introspections_count < ?", config.FailedIntrospectionsLimit+1).Find(&dbRepos)
-	} else {
-		result = p.db.Find(&dbRepos)
+	db := p.db
+	if !force && !config.Get().Options.AlwaysRunCronTasks {
+		introspectThreshold := time.Now().Add(config.IntrospectTimeInterval * -1) // Add a negative duration
+		db = db.Where(
+			db.Where("status != ?", config.StatusValid).
+				Or("last_introspection_time is NULL").                   // It was never introspected
+				Or("last_introspection_time < ?", introspectThreshold)). // It was introspected more than the threshold ago)
+			Where( // It is over the introspection limit and has failed once due to being over the limit, so that last_introspection_error says 'over the limit of failed'
+				db.Where("failed_introspections_count < ?", config.FailedIntrospectionsLimit+1).
+					Or("public = true"))
+	}
+	if urls != nil {
+		db = db.Where("url in ?", *urls)
 	}
+
+	result := db.Find(&dbRepos)
 	if result.Error != nil {
 		return repos, result.Error
 	}