Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HMS-4934: Add modules to introspected data #922

Merged
merged 6 commits into from
Jan 17, 2025
Merged

HMS-4934: Add modules to introspected data #922

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
da7a2e9
Fixes 4934: module streams for introspected repositories
jlsherrill Dec 3, 2024
b099a0d
return empty array instead of null
jlsherrill Jan 13, 2025
bf577fb
add openapi
jlsherrill Jan 13, 2025
200035c
fix migrate to clear repomd checksum
jlsherrill Jan 14, 2025
3dcbfe0
test fix
jlsherrill Jan 14, 2025
906169d
bump yummy
jlsherrill Jan 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .mockery.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,6 @@ packages:
TemplateDao:
config:
filename: "templates_mock.go"
ModuleStreamsDao:
ModuleStreamDao:
config:
filename: "modules_streams_mock.go"
101 changes: 101 additions & 0 deletions api/docs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,68 @@ const docTemplate = `{
}
}
},
"/module_streams/search": {
"post": {
"description": "List modules and their streams for repositories",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"module_streams"
],
"summary": "List modules and their streams for repositories",
"operationId": "searchRepositoryModuleStreams",
"parameters": [
{
"description": "request body",
"name": "body",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/api.SearchModuleStreamsRequest"
}
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/api.SearchModuleStreams"
}
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/errors.ErrorResponse"
}
},
"401": {
"description": "Unauthorized",
"schema": {
"$ref": "#/definitions/errors.ErrorResponse"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/errors.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/errors.ErrorResponse"
}
}
}
}
},
"/package_groups/names": {
"post": {
"description": "This enables users to search for package groups in a given list of repositories.",
Expand Down Expand Up @@ -4678,6 +4740,45 @@ const docTemplate = `{
}
}
},
"api.SearchModuleStreamsRequest": {
"type": "object",
"required": [
"rpm_names",
"urls",
"uuids"
],
"properties": {
"rpm_names": {
"description": "List of rpm names to search",
"type": "array",
"items": {
"type": "string"
}
},
"search": {
"description": "Search string to search rpm names",
"type": "string"
},
"sort_by": {
"description": "SortBy sets the sort order of the result",
"type": "string"
},
"urls": {
"description": "List of repository URLs to search",
"type": "array",
"items": {
"type": "string"
}
},
"uuids": {
"description": "List of repository UUIDs to search",
"type": "array",
"items": {
"type": "string"
}
}
}
},
"api.SearchPackageGroupResponse": {
"type": "object",
"properties": {
Expand Down
116 changes: 116 additions & 0 deletions api/openapi.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1055,6 +1055,45 @@
},
"type": "object"
},
"api.SearchModuleStreamsRequest": {
"properties": {
"rpm_names": {
"description": "List of rpm names to search",
"items": {
"type": "string"
},
"type": "array"
},
"search": {
"description": "Search string to search rpm names",
"type": "string"
},
"sort_by": {
"description": "SortBy sets the sort order of the result",
"type": "string"
},
"urls": {
"description": "List of repository URLs to search",
"items": {
"type": "string"
},
"type": "array"
},
"uuids": {
"description": "List of repository UUIDs to search",
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"rpm_names",
"urls",
"uuids"
],
"type": "object"
},
"api.SearchPackageGroupResponse": {
"properties": {
"description": {
Expand Down Expand Up @@ -1922,6 +1961,83 @@
]
}
},
"/module_streams/search": {
"post": {
"description": "List modules and their streams for repositories",
"operationId": "searchRepositoryModuleStreams",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/api.SearchModuleStreamsRequest"
}
}
},
"description": "request body",
"required": true,
"x-originalParamName": "body"
},
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"items": {
"$ref": "#/components/schemas/api.SearchModuleStreams"
},
"type": "array"
}
}
},
"description": "OK"
},
"400": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/errors.ErrorResponse"
}
}
},
"description": "Bad Request"
},
"401": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/errors.ErrorResponse"
}
}
},
"description": "Unauthorized"
},
"404": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/errors.ErrorResponse"
}
}
},
"description": "Not Found"
},
"500": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/errors.ErrorResponse"
}
}
},
"description": "Internal Server Error"
}
},
"summary": "List modules and their streams for repositories",
"tags": [
"module_streams"
]
}
},
"/package_groups/names": {
"post": {
"description": "This enables users to search for package groups in a given list of repositories.",
Expand Down
2 changes: 1 addition & 1 deletion db/migrations.latest
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
20250115112007
20250117072504
5 changes: 5 additions & 0 deletions db/migrations/20250117072504_add_module_streams.down.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
BEGIN;

DROP TABLE IF EXISTS module_streams, repositories_module_streams;

COMMIT;
50 changes: 50 additions & 0 deletions db/migrations/20250117072504_add_module_streams.up.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
BEGIN;

CREATE TABLE IF NOT EXISTS module_streams (
uuid UUID UNIQUE NOT NULL PRIMARY KEY,
created_at TIMESTAMP WITH TIME ZONE,
updated_at TIMESTAMP WITH TIME ZONE,
name text NOT NULL,
stream text NOT NULL,
version text NOT NULL,
context text NOT NULL,
arch text NOT NULL,
summary text NOT NULL,
description text NOT NULL,
package_names text[] NOT NULL,
packages text[] NOT NULL,
hash_value text NOT NULL,
profiles jsonb NOT NULL DEFAULT '{}'::jsonb
);

CREATE TABLE IF NOT EXISTS repositories_module_streams (
repository_uuid UUID NOT NULL,
module_stream_uuid UUID NOT NULL
);

CREATE INDEX IF NOT EXISTS module_streams_pkgs_idx ON module_streams USING GIN (package_names);
CREATE INDEX IF NOT EXISTS module_streams_name_idx ON module_streams (uuid, name);

ALTER TABLE ONLY repositories_module_streams
DROP CONSTRAINT IF EXISTS repositories_module_streams_pkey,
ADD CONSTRAINT repositories_module_streams_pkey PRIMARY KEY (repository_uuid, module_stream_uuid);

ALTER TABLE ONLY repositories_module_streams
DROP CONSTRAINT IF EXISTS fk_repositories_module_streams_mstream,
ADD CONSTRAINT fk_repositories_module_streams_mstream
FOREIGN KEY (module_stream_uuid) REFERENCES module_streams(uuid)
ON DELETE CASCADE;

ALTER TABLE ONLY repositories_module_streams
DROP CONSTRAINT IF EXISTS fk_repositories_module_streams_repository,
ADD CONSTRAINT fk_repositories_module_streams_repository
FOREIGN KEY (repository_uuid) REFERENCES repositories(uuid)
ON DELETE CASCADE;

ALTER TABLE ONLY module_streams
DROP CONSTRAINT IF EXISTS fk_module_streams_uniq,
ADD CONSTRAINT fk_module_streams_uniq UNIQUE (hash_value);

UPDATE repositories SET repomd_checksum = '' where public = true;

COMMIT;
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ require (
github.com/ProtonMail/go-crypto v1.1.4
github.com/content-services/lecho/v3 v3.5.2
github.com/content-services/tang v0.0.11
github.com/content-services/yummy v1.0.13
github.com/content-services/yummy v1.0.14
github.com/getkin/kin-openapi v0.127.1-0.20240830113739-c606b5546b12
github.com/go-openapi/spec v0.21.0 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ github.com/content-services/tang v0.0.11 h1:nnXnpfRZjw/qX9xvk9jMcCs9JF9sbc7o65JS
github.com/content-services/tang v0.0.11/go.mod h1:J0A8WTEt8h8xxMHfXN84Ucu4DUItp+uH0tLwrQbtpYE=
github.com/content-services/yummy v1.0.13 h1:6mTnxhWcq4SJN+SiOOqOZuO4dwutvbPgkmxSY1mTnMw=
github.com/content-services/yummy v1.0.13/go.mod h1:MQcxIfFrhA70DnB8lMBmhuk2mRmQwcSBl6FuSwEehf0=
github.com/content-services/yummy v1.0.14 h1:XYoqidBhElo0cMxDyJ/sswmZsN0Cbuf9Yoe8e0iVIMg=
github.com/content-services/yummy v1.0.14/go.mod h1:rrVMHk0HnYXIfp3W3CeWrl3Ev5EojpQo8FWn5ktgCJg=
github.com/content-services/zest/release/v2024 v2024.12.1734541842 h1:vIIWFZ5j76MVg2VuxkXAR56NAvkst7LtAeAugbfJmFU=
github.com/content-services/zest/release/v2024 v2024.12.1734541842/go.mod h1:fUWkjhvNTiS9UaaAWNLYkshl37V9IVmmRaprp9eDmrU=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
Expand Down
8 changes: 8 additions & 0 deletions pkg/api/module_streams.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,14 @@ type SearchSnapshotModuleStreamsRequest struct {
Search string `json:"search"` // Search string to search module names
}

type SearchModuleStreamsRequest struct {
UUIDs []string `json:"uuids" validate:"required"` // List of repository UUIDs to search
URLs []string `json:"urls" validate:"required"` // List of repository URLs to search
RpmNames []string `json:"rpm_names" validate:"required"` // List of rpm names to search
SortBy string `json:"sort_by"` // SortBy sets the sort order of the result
Search string `json:"search"` // Search string to search rpm names
}

type Stream struct {
Name string `json:"name"` // Name of the module
Stream string `json:"stream"` // Module stream version
Expand Down
13 changes: 8 additions & 5 deletions pkg/dao/interfaces.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,10 @@ type DaoRegistry struct {
AdminTask AdminTaskDao
Domain DomainDao
PackageGroup PackageGroupDao
ModuleStream ModuleStreamDao
Environment EnvironmentDao
Template TemplateDao
Uploads UploadDao
ModuleStreams ModuleStreamsDao
}

func GetDaoRegistry(db *gorm.DB) *DaoRegistry {
Expand All @@ -38,9 +38,9 @@ func GetDaoRegistry(db *gorm.DB) *DaoRegistry {
Rpm: &rpmDaoImpl{
db: db,
},
ModuleStreams: &moduleStreamsImpl{db: db},
Repository: repositoryDaoImpl{db: db},
Metrics: metricsDaoImpl{db: db},
ModuleStream: &moduleStreamsImpl{db: db},
Repository: repositoryDaoImpl{db: db},
Metrics: metricsDaoImpl{db: db},
Snapshot: &snapshotDaoImpl{
db: db,
pulpClient: pulp_client.GetPulpClientWithDomain(""),
Expand Down Expand Up @@ -82,8 +82,11 @@ type RepositoryConfigDao interface {
BulkImport(ctx context.Context, reposToImport []api.RepositoryRequest) ([]api.RepositoryImportResponse, []error)
}

type ModuleStreamsDao interface {
type ModuleStreamDao interface {
SearchRepositoryModuleStreams(ctx context.Context, orgID string, request api.SearchModuleStreamsRequest) ([]api.SearchModuleStreams, error)
SearchSnapshotModuleStreams(ctx context.Context, orgID string, request api.SearchSnapshotModuleStreamsRequest) ([]api.SearchModuleStreams, error)
InsertForRepository(ctx context.Context, repoUuid string, pkgGroups []yum.ModuleMD) (int64, error)
OrphanCleanup(ctx context.Context) error
}

type RpmDao interface {
Expand Down
Loading
Loading