Skip to content

Commit

Permalink
HMS-4934: Add modules to introspected data (#922)
Browse files Browse the repository at this point in the history 
* Fixes 4934: module streams for introspected repositories

* return empty array instead of null

* add openapi

* fix migrate to clear repomd checksum

* test fix

* bump yummy
  • Loading branch information
@jlsherrill
jlsherrill authored Jan 17, 2025
1 parent 944fd44 commit d8c1e99
Show file tree
Hide file tree
Showing 23 changed files with 1,079 additions and 39 deletions.
2 changes: 1 addition & 1 deletion .mockery.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,6 @@ packages:
TemplateDao:
config:
filename: "templates_mock.go"
ModuleStreamsDao:
ModuleStreamDao:
config:
filename: "modules_streams_mock.go"
101 changes: 101 additions & 0 deletions api/docs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,68 @@ const docTemplate = `{
}
}
},
"/module_streams/search": {
"post": {
"description": "List modules and their streams for repositories",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"module_streams"
],
"summary": "List modules and their streams for repositories",
"operationId": "searchRepositoryModuleStreams",
"parameters": [
{
"description": "request body",
"name": "body",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/api.SearchModuleStreamsRequest"
}
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/api.SearchModuleStreams"
}
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/errors.ErrorResponse"
}
},
"401": {
"description": "Unauthorized",
"schema": {
"$ref": "#/definitions/errors.ErrorResponse"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/errors.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/errors.ErrorResponse"
}
}
}
}
},
"/package_groups/names": {
"post": {
"description": "This enables users to search for package groups in a given list of repositories.",
Expand Down Expand Up @@ -4678,6 +4740,45 @@ const docTemplate = `{
}
}
},
"api.SearchModuleStreamsRequest": {
"type": "object",
"required": [
"rpm_names",
"urls",
"uuids"
],
"properties": {
"rpm_names": {
"description": "List of rpm names to search",
"type": "array",
"items": {
"type": "string"
}
},
"search": {
"description": "Search string to search rpm names",
"type": "string"
},
"sort_by": {
"description": "SortBy sets the sort order of the result",
"type": "string"
},
"urls": {
"description": "List of repository URLs to search",
"type": "array",
"items": {
"type": "string"
}
},
"uuids": {
"description": "List of repository UUIDs to search",
"type": "array",
"items": {
"type": "string"
}
}
}
},
"api.SearchPackageGroupResponse": {
"type": "object",
"properties": {
Expand Down
116 changes: 116 additions & 0 deletions api/openapi.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1055,6 +1055,45 @@
},
"type": "object"
},
"api.SearchModuleStreamsRequest": {
"properties": {
"rpm_names": {
"description": "List of rpm names to search",
"items": {
"type": "string"
},
"type": "array"
},
"search": {
"description": "Search string to search rpm names",
"type": "string"
},
"sort_by": {
"description": "SortBy sets the sort order of the result",
"type": "string"
},
"urls": {
"description": "List of repository URLs to search",
"items": {
"type": "string"
},
"type": "array"
},
"uuids": {
"description": "List of repository UUIDs to search",
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"rpm_names",
"urls",
"uuids"
],
"type": "object"
},
"api.SearchPackageGroupResponse": {
"properties": {
"description": {
Expand Down Expand Up @@ -1922,6 +1961,83 @@
]
}
},
"/module_streams/search": {
"post": {
"description": "List modules and their streams for repositories",
"operationId": "searchRepositoryModuleStreams",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/api.SearchModuleStreamsRequest"
}
}
},
"description": "request body",
"required": true,
"x-originalParamName": "body"
},
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"items": {
"$ref": "#/components/schemas/api.SearchModuleStreams"
},
"type": "array"
}
}
},
"description": "OK"
},
"400": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/errors.ErrorResponse"
}
}
},
"description": "Bad Request"
},
"401": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/errors.ErrorResponse"
}
}
},
"description": "Unauthorized"
},
"404": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/errors.ErrorResponse"
}
}
},
"description": "Not Found"
},
"500": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/errors.ErrorResponse"
}
}
},
"description": "Internal Server Error"
}
},
"summary": "List modules and their streams for repositories",
"tags": [
"module_streams"
]
}
},
"/package_groups/names": {
"post": {
"description": "This enables users to search for package groups in a given list of repositories.",
Expand Down
2 changes: 1 addition & 1 deletion db/migrations.latest
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
20250115112007
20250117072504
5 changes: 5 additions & 0 deletions db/migrations/20250117072504_add_module_streams.down.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
BEGIN;

DROP TABLE IF EXISTS module_streams, repositories_module_streams;

COMMIT;
50 changes: 50 additions & 0 deletions db/migrations/20250117072504_add_module_streams.up.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
BEGIN;

CREATE TABLE IF NOT EXISTS module_streams (
uuid UUID UNIQUE NOT NULL PRIMARY KEY,
created_at TIMESTAMP WITH TIME ZONE,
updated_at TIMESTAMP WITH TIME ZONE,
name text NOT NULL,
stream text NOT NULL,
version text NOT NULL,
context text NOT NULL,
arch text NOT NULL,
summary text NOT NULL,
description text NOT NULL,
package_names text[] NOT NULL,
packages text[] NOT NULL,
hash_value text NOT NULL,
profiles jsonb NOT NULL DEFAULT '{}'::jsonb
);

CREATE TABLE IF NOT EXISTS repositories_module_streams (
repository_uuid UUID NOT NULL,
module_stream_uuid UUID NOT NULL
);

CREATE INDEX IF NOT EXISTS module_streams_pkgs_idx ON module_streams USING GIN (package_names);
CREATE INDEX IF NOT EXISTS module_streams_name_idx ON module_streams (uuid, name);

ALTER TABLE ONLY repositories_module_streams
DROP CONSTRAINT IF EXISTS repositories_module_streams_pkey,
ADD CONSTRAINT repositories_module_streams_pkey PRIMARY KEY (repository_uuid, module_stream_uuid);

ALTER TABLE ONLY repositories_module_streams
DROP CONSTRAINT IF EXISTS fk_repositories_module_streams_mstream,
ADD CONSTRAINT fk_repositories_module_streams_mstream
FOREIGN KEY (module_stream_uuid) REFERENCES module_streams(uuid)
ON DELETE CASCADE;

ALTER TABLE ONLY repositories_module_streams
DROP CONSTRAINT IF EXISTS fk_repositories_module_streams_repository,
ADD CONSTRAINT fk_repositories_module_streams_repository
FOREIGN KEY (repository_uuid) REFERENCES repositories(uuid)
ON DELETE CASCADE;

ALTER TABLE ONLY module_streams
DROP CONSTRAINT IF EXISTS fk_module_streams_uniq,
ADD CONSTRAINT fk_module_streams_uniq UNIQUE (hash_value);

UPDATE repositories SET repomd_checksum = '' where public = true;

COMMIT;
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ require (
github.com/ProtonMail/go-crypto v1.1.4
github.com/content-services/lecho/v3 v3.5.2
github.com/content-services/tang v0.0.11
github.com/content-services/yummy v1.0.13
github.com/content-services/yummy v1.0.14
github.com/getkin/kin-openapi v0.127.1-0.20240830113739-c606b5546b12
github.com/go-openapi/spec v0.21.0 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ github.com/content-services/tang v0.0.11 h1:nnXnpfRZjw/qX9xvk9jMcCs9JF9sbc7o65JS
github.com/content-services/tang v0.0.11/go.mod h1:J0A8WTEt8h8xxMHfXN84Ucu4DUItp+uH0tLwrQbtpYE=
github.com/content-services/yummy v1.0.13 h1:6mTnxhWcq4SJN+SiOOqOZuO4dwutvbPgkmxSY1mTnMw=
github.com/content-services/yummy v1.0.13/go.mod h1:MQcxIfFrhA70DnB8lMBmhuk2mRmQwcSBl6FuSwEehf0=
github.com/content-services/yummy v1.0.14 h1:XYoqidBhElo0cMxDyJ/sswmZsN0Cbuf9Yoe8e0iVIMg=
github.com/content-services/yummy v1.0.14/go.mod h1:rrVMHk0HnYXIfp3W3CeWrl3Ev5EojpQo8FWn5ktgCJg=
github.com/content-services/zest/release/v2024 v2024.12.1734541842 h1:vIIWFZ5j76MVg2VuxkXAR56NAvkst7LtAeAugbfJmFU=
github.com/content-services/zest/release/v2024 v2024.12.1734541842/go.mod h1:fUWkjhvNTiS9UaaAWNLYkshl37V9IVmmRaprp9eDmrU=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
Expand Down
8 changes: 8 additions & 0 deletions pkg/api/module_streams.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,14 @@ type SearchSnapshotModuleStreamsRequest struct {
Search string `json:"search"` // Search string to search module names
}

type SearchModuleStreamsRequest struct {
UUIDs []string `json:"uuids" validate:"required"` // List of repository UUIDs to search
URLs []string `json:"urls" validate:"required"` // List of repository URLs to search
RpmNames []string `json:"rpm_names" validate:"required"` // List of rpm names to search
SortBy string `json:"sort_by"` // SortBy sets the sort order of the result
Search string `json:"search"` // Search string to search rpm names
}

type Stream struct {
Name string `json:"name"` // Name of the module
Stream string `json:"stream"` // Module stream version
Expand Down
13 changes: 8 additions & 5 deletions pkg/dao/interfaces.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,10 @@ type DaoRegistry struct {
AdminTask AdminTaskDao
Domain DomainDao
PackageGroup PackageGroupDao
ModuleStream ModuleStreamDao
Environment EnvironmentDao
Template TemplateDao
Uploads UploadDao
ModuleStreams ModuleStreamsDao
}

func GetDaoRegistry(db *gorm.DB) *DaoRegistry {
Expand All @@ -38,9 +38,9 @@ func GetDaoRegistry(db *gorm.DB) *DaoRegistry {
Rpm: &rpmDaoImpl{
db: db,
},
ModuleStreams: &moduleStreamsImpl{db: db},
Repository: repositoryDaoImpl{db: db},
Metrics: metricsDaoImpl{db: db},
ModuleStream: &moduleStreamsImpl{db: db},
Repository: repositoryDaoImpl{db: db},
Metrics: metricsDaoImpl{db: db},
Snapshot: &snapshotDaoImpl{
db: db,
pulpClient: pulp_client.GetPulpClientWithDomain(""),
Expand Down Expand Up @@ -82,8 +82,11 @@ type RepositoryConfigDao interface {
BulkImport(ctx context.Context, reposToImport []api.RepositoryRequest) ([]api.RepositoryImportResponse, []error)
}

type ModuleStreamsDao interface {
type ModuleStreamDao interface {
SearchRepositoryModuleStreams(ctx context.Context, orgID string, request api.SearchModuleStreamsRequest) ([]api.SearchModuleStreams, error)
SearchSnapshotModuleStreams(ctx context.Context, orgID string, request api.SearchSnapshotModuleStreamsRequest) ([]api.SearchModuleStreams, error)
InsertForRepository(ctx context.Context, repoUuid string, pkgGroups []yum.ModuleMD) (int64, error)
OrphanCleanup(ctx context.Context) error
}

type RpmDao interface {
Expand Down
Loading

0 comments on commit d8c1e99

Please sign in to comment.