Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixes 4740: Change base query to use inner joins #13

Merged
merged 1 commit into from
Oct 2, 2024
Merged

Fixes 4740: Change base query to use inner joins #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions compose_files/pulp/docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ services:
postgres:
condition: service_healthy
command: pulpcore-manager migrate --noinput
restart: on-failure
volumes:
- "./assets/settings.py:/etc/pulp/settings.py:z"
- "./assets/certs:/etc/pulp/certs:z"
Expand Down
21 changes: 12 additions & 9 deletions pkg/tangy/queries.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,25 +18,28 @@ func contentIdsInVersion(repoId string, versionNum int, namedArgs *pgx.NamedArgs
repoIdName := fmt.Sprintf("%v%v", "repoName", ran)
versionNumName := fmt.Sprintf("%v%v", "versionNum", ran)
query := `
SELECT crc.content_id
FROM core_repositorycontent crc
INNER JOIN core_repositoryversion crv ON (crc.version_added_id = crv.pulp_id)
LEFT OUTER JOIN core_repositoryversion crv2 ON (crc.version_removed_id = crv2.pulp_id)
WHERE crv.repository_id = @%v AND crv.number <= @%v AND NOT (crv2.number <= @%v AND crv2.number IS NOT NULL)
(crv.repository_id = @%v AND crv.number <= @%v AND NOT (crv2.number <= @%v AND crv2.number IS NOT NULL))
`
(*namedArgs)[repoIdName] = repoId
(*namedArgs)[versionNumName] = versionNum
return fmt.Sprintf(query, repoIdName, versionNumName, versionNumName)
}

// Creates a sub query (including parenthesis) to lookup the content IDs of a list of repository versions.
// returns part of a query that joins a table to the needed tables to select content units in a given set of versions
//
// Takes in a pointer to Named args in order to add required named arguments for the query. Multiple queries are created
// and UNION'd together
// The return of this functions should be added to a query such as "select ** from TABLE rp" query,
// Where rp has a column 'content_ptr_id', such as rpm_updaterecord, rpm_package, etc.
// Takes in a pointer to Named args in order to add required named arguments for the query.
func contentIdsInVersions(repoVerMap []ParsedRepoVersion, namedArgs *pgx.NamedArgs) string {
mainQuery := `
INNER JOIN core_repositorycontent crc on rp.content_ptr_id = crc.content_id
INNER JOIN core_repositoryversion crv ON (crc.version_added_id = crv.pulp_id)
LEFT OUTER JOIN core_repositoryversion crv2 ON (crc.version_removed_id = crv2.pulp_id)
WHERE
`
queries := []string{}
for _, parsed := range repoVerMap {
queries = append(queries, contentIdsInVersion(parsed.RepositoryUUID, parsed.Version, namedArgs))
}
return fmt.Sprintf("( %v ) ", strings.Join(queries, " UNION "))
return fmt.Sprintf("%v (%v)", mainQuery, strings.Join(queries, " OR "))
}
40 changes: 20 additions & 20 deletions pkg/tangy/rpm.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ func (t *tangyImpl) RpmRepositoryVersionPackageSearch(ctx context.Context, hrefs
innerUnion := contentIdsInVersions(repoVerMap, &args)

query := `SELECT DISTINCT ON (rp.name) rp.name, rp.summary
FROM rpm_package rp WHERE rp.content_ptr_id IN `
FROM rpm_package rp `

rows, err := conn.Query(context.Background(), query+innerUnion+" AND rp.name ILIKE CONCAT( '%', @nameFilter::text, '%') ORDER BY rp.name LIMIT @limit", args)
if err != nil {
Expand Down Expand Up @@ -140,7 +140,7 @@ func (t *tangyImpl) RpmRepositoryVersionPackageGroupSearch(ctx context.Context,
innerUnion := contentIdsInVersions(repoVerMap, &args)

query := `SELECT DISTINCT ON (rp.name, rp.id, rp.packages) rp.name, rp.id, rp.description, rp.packages
FROM rpm_packagegroup rp WHERE rp.content_ptr_id IN
FROM rpm_packagegroup rp
`

rows, err := conn.Query(ctx, query+innerUnion+"AND rp.name ILIKE CONCAT( '%', @nameFilter::text, '%') ORDER BY rp.name", args)
Expand Down Expand Up @@ -221,7 +221,7 @@ func (t *tangyImpl) RpmRepositoryVersionEnvironmentSearch(ctx context.Context, h
innerUnion := contentIdsInVersions(repoVerMap, &args)

query := `SELECT DISTINCT ON (rp.name, rp.id) rp.name, rp.id, rp.description
FROM rpm_packageenvironment rp WHERE rp.content_ptr_id IN
FROM rpm_packageenvironment rp
`

rows, err := conn.Query(ctx, query+innerUnion+" AND rp.name ILIKE CONCAT( '%', @nameFilter::text, '%') ORDER BY rp.name LIMIT @limit", args)
Expand Down Expand Up @@ -258,7 +258,7 @@ func (t *tangyImpl) RpmRepositoryVersionErrataList(ctx context.Context, hrefs []
return nil, 0, fmt.Errorf("error parsing repository version hrefs: %w", err)
}

countQueryOpen := "select count(*) as total FROM rpm_updaterecord re WHERE re.content_ptr_id IN "
countQueryOpen := "select count(distinct rp.content_ptr_id) as total FROM rpm_updaterecord rp "

args := pgx.NamedArgs{
"searchFilter": filterOpts.Search,
Expand All @@ -270,16 +270,16 @@ func (t *tangyImpl) RpmRepositoryVersionErrataList(ctx context.Context, hrefs []

var concatFilter strings.Builder
if filterOpts.Search != "" {
concatFilter.WriteString(" AND (re.id ILIKE CONCAT( '%', @searchFilter::text, '%') OR re.summary ILIKE CONCAT( '%', @searchFilter::text, '%'))")
concatFilter.WriteString(" AND (rp.id ILIKE CONCAT( '%', @searchFilter::text, '%') OR rp.summary ILIKE CONCAT( '%', @searchFilter::text, '%'))")
}
if filterOpts.Type != nil {
if strings.Contains(filterOpts.Type[0], ",") {
filterOpts.Type = strings.Split(filterOpts.Type[0], ",")
}
args["typeFilter"] = filterOpts.Type
concatFilter.WriteString(" AND (re.type = ANY(@typeFilter)")
concatFilter.WriteString(" AND (rp.type = ANY(@typeFilter)")
if containsString(filterOpts.Type, "other") {
concatFilter.WriteString(" OR NOT (re.type = ANY(@typeList))")
concatFilter.WriteString(" OR NOT (rp.type = ANY(@typeList))")
}
concatFilter.WriteString(")")
}
Expand All @@ -288,9 +288,9 @@ func (t *tangyImpl) RpmRepositoryVersionErrataList(ctx context.Context, hrefs []
filterOpts.Severity = strings.Split(filterOpts.Severity[0], ",")
}
args["severityFilter"] = filterOpts.Severity
concatFilter.WriteString(" AND (re.severity = ANY(@severityFilter)")
concatFilter.WriteString(" AND (rp.severity = ANY(@severityFilter)")
if containsString(filterOpts.Severity, "Unknown") {
concatFilter.WriteString(" OR NOT (re.severity = ANY(@severityList))")
concatFilter.WriteString(" OR NOT (rp.severity = ANY(@severityList))")
}
concatFilter.WriteString(")")
}
Expand All @@ -306,12 +306,12 @@ func (t *tangyImpl) RpmRepositoryVersionErrataList(ctx context.Context, hrefs []
return nil, 0, err
}

queryOpen := `SELECT re.content_ptr_id as id, re.id as ErrataId, re.title, re.summary, re.description, re.issued_date as IssuedDate, re.updated_date as UpdatedDate, re.type, re.severity, re.reboot_suggested as RebootSuggested,
queryOpen := `SELECT distinct rp.content_ptr_id as id, rp.id as ErrataId, rp.title, rp.summary, rp.description, rp.issued_date as IssuedDate, rp.updated_date as UpdatedDate, rp.type, rp.severity, rp.reboot_suggested as RebootSuggested,
(SELECT ARRAY_AGG(ru.ref_id)
FROM rpm_updatereference ru
WHERE ru.update_record_id = re.content_ptr_id
WHERE ru.update_record_id = rp.content_ptr_id
AND ru.ref_type = 'cve') AS CVEs
FROM rpm_updaterecord re WHERE re.content_ptr_id IN `
FROM rpm_updaterecord rp `

args["limit"] = pageOpts.Limit
args["offset"] = pageOpts.Offset
Expand All @@ -321,15 +321,15 @@ func (t *tangyImpl) RpmRepositoryVersionErrataList(ctx context.Context, hrefs []
sortField := strings.Split(pageOpts.SortBy, ":")[0]
switch sortField {
case "issued_date":
orderBy = "re.issued_date"
orderBy = "rp.issued_date"
case "updated_date":
orderBy = "re.updated_date"
orderBy = "rp.updated_date"
case "type":
orderBy = "re.type"
orderBy = "rp.type"
case "severity":
orderBy = "re.severity"
orderBy = "rp.severity"
default:
orderBy = "re.issued_date"
orderBy = "rp.issued_date"
}

if strings.Contains(pageOpts.SortBy, "asc") {
Expand Down Expand Up @@ -374,7 +374,7 @@ func (t *tangyImpl) RpmRepositoryVersionPackageList(ctx context.Context, hrefs [
return nil, 0, fmt.Errorf("error parsing repository version hrefs: %w", err)
}

countQueryOpen := "select count(*) as total FROM rpm_package rp WHERE rp.content_ptr_id IN "
countQueryOpen := "select count(distinct(rp.content_ptr_id)) as total FROM rpm_package rp "
args := pgx.NamedArgs{"nameFilter": "%" + filterOpts.Name + "%"}
innerUnion := contentIdsInVersions(repoVerMap, &args)

Expand All @@ -384,8 +384,8 @@ func (t *tangyImpl) RpmRepositoryVersionPackageList(ctx context.Context, hrefs [
return nil, 0, err
}

queryOpen := `SELECT rp.content_ptr_id as id, rp.name, rp.version, rp.arch, rp.release, rp.epoch, rp.summary
FROM rpm_package rp WHERE rp.content_ptr_id IN `
queryOpen := `SELECT distinct rp.content_ptr_id as id, rp.name, rp.version, rp.arch, rp.release, rp.epoch, rp.summary
FROM rpm_package rp `

args["limit"] = pageOpts.Limit
args["offset"] = pageOpts.Offset
Expand Down
Loading