From e191e0e18cf96f6729b9a11b9f4264513481d92d Mon Sep 17 00:00:00 2001 From: rverdile Date: Wed, 3 Jan 2024 09:22:44 -0500 Subject: [PATCH] Add rpm name search and integration testing (#1) * Add rpm name search and integration testing * update config.yaml.example fix query add limit add close method add new test cases add mock * limit whole query --------- Co-authored-by: Justin Sherrill --- .github/workflows/tang-actions.yaml | 54 ++++- .gitignore | 5 +- .golangci.yaml | 14 ++ Makefile | 23 ++ README.md | 62 +++++ compose_files/pulp/assets/bin/nginx.sh | 32 +++ .../certs/database_fields.symmetric.key | 1 + .../pulp/assets/nginx/nginx.conf.template | 89 +++++++ compose_files/pulp/assets/postgres/passwd | 20 ++ compose_files/pulp/assets/settings.py | 18 ++ compose_files/pulp/docker-compose.yml | 137 +++++++++++ configs/config.yaml.example | 15 ++ example.go | 94 ++++++++ go.mod | 43 ++++ go.sum | 99 ++++++++ internal/config/config.go | 57 +++++ internal/test/integration/rpm_test.go | 160 +++++++++++++ internal/zestwrapper/rpm.go | 225 ++++++++++++++++++ mk/compose.mk | 17 ++ mk/help.mk | 13 + mk/includes.mk | 24 ++ mk/test.mk | 3 + mk/variables.mk | 30 +++ pkg/tangy/config.go | 47 ++++ pkg/tangy/interface.go | 62 +++++ pkg/tangy/rpm.go | 100 ++++++++ pkg/tangy/tangy_mock.go | 59 +++++ 27 files changed, 1498 insertions(+), 5 deletions(-) create mode 100644 .golangci.yaml create mode 100644 Makefile create mode 100755 compose_files/pulp/assets/bin/nginx.sh create mode 100644 compose_files/pulp/assets/certs/database_fields.symmetric.key create mode 100644 compose_files/pulp/assets/nginx/nginx.conf.template create mode 100644 compose_files/pulp/assets/postgres/passwd create mode 100644 compose_files/pulp/assets/settings.py create mode 100644 compose_files/pulp/docker-compose.yml create mode 100644 configs/config.yaml.example create mode 100644 example.go create mode 100644 go.sum create mode 100644 internal/config/config.go create mode 100644 internal/test/integration/rpm_test.go create mode 100644 internal/zestwrapper/rpm.go create mode 100644 mk/compose.mk create mode 100644 mk/help.mk create mode 100644 mk/includes.mk create mode 100644 mk/test.mk create mode 100644 mk/variables.mk create mode 100644 pkg/tangy/config.go create mode 100644 pkg/tangy/interface.go create mode 100644 pkg/tangy/rpm.go create mode 100644 pkg/tangy/tangy_mock.go diff --git a/.github/workflows/tang-actions.yaml b/.github/workflows/tang-actions.yaml index 672976d..7c1f6b1 100644 --- a/.github/workflows/tang-actions.yaml +++ b/.github/workflows/tang-actions.yaml @@ -9,13 +9,59 @@ on: paths-ignore: - '**.md' jobs: - govet: - name: Vet + golangci: + name: Lint runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 with: go-version: "1.20" - - run: | - go vet ./... \ No newline at end of file + - name: golangci-lint + uses: golangci/golangci-lint-action@v2 + with: + version: v1.54.2 + skip-go-installation: true + args: --timeout=5m + gotest: + name: Test + runs-on: ubuntu-latest + services: + postgres: + image: postgres:15 + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: "1.20" + - name: start pulp + uses: isbang/compose-action@v1.4.1 + with: + compose-file: ./compose_files/pulp/docker-compose.yml + down-flags: --volumes + - name: Wait for pulp + run: | + docker run --network=host --rm -v ${PWD}:/local curlimages/curl \ + curl --retry-all-errors --fail --retry-delay 10 --retry 32 --retry-max-time 240 http://localhost:8087/pulp/default/api/v3/repositories/rpm/rpm/ -u admin:password + sleep 30 + - name: integration tests + run: | + make test-integration + env: + DATABASE_HOST: localhost + DATABASE_PORT: 5434 + DATABASE_USER: pulp + DATABASE_NAME: pulp + DATABASE_PASSWORD: password + SERVER_URL: http://localhost:8087 + SERVER_USERNAME: admin + SERVER_PASSWORD: password \ No newline at end of file diff --git a/.gitignore b/.gitignore index 447d29d..436757f 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,7 @@ release/ Containerfile # Pulp Oci Images -compose_files/pulp/pulp-oci-images \ No newline at end of file +compose_files/pulp/pulp-oci-images + +#config +configs/config.yaml \ No newline at end of file diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 0000000..710ea5f --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,14 @@ +# Configuration for golangci-lint. See https://golangci-lint.run/usage/configuration/. +linters: + disable-all: false # use default linters + enable: + - gofmt + - whitespace + - govet + - misspell + - forcetypeassert + - gci + - bodyclose +issues: + exclude: + - composite diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..07e51c5 --- /dev/null +++ b/Makefile @@ -0,0 +1,23 @@ +## +# Entrypoint for the Makefile +# +# It is composed at mk/includes.mk by including +# small make files which provides all the necessary +# rules. +# +# Some considerations: +# +# - Variables customization can be +# stored at 'config.env', 'mk/private.mk' files. +# - By default the 'help' rule is executed. +# - No parallel jobs are executed from the main Makefile, +# so that multiple rules from the command line will be +# executed in serial. +## + +include mk/includes.mk + +.NOT_PARALLEL: + +# Set the default rule +.DEFAULT_GOAL := help diff --git a/README.md b/README.md index 0d27c73..90f156f 100644 --- a/README.md +++ b/README.md @@ -1 +1,63 @@ # tang + +The tangy package provides methods to read from a [pulp](https://pulpproject.org/) database. + +## Installation +`go get github.com/content-services/tang` + +## Usage +The tangy package is meant to be imported into an existing project that is using pulp. It can be used like this: +```go +// Pulp database configuration information +dbConfig := tangy.Database{ + Name: "pulp", + Host: "localhost", + Port: 5434, + User: "pulp", + Password: "password", + CACertPath: "", + PoolLimit: 20, +} + +// Create new Tangy instance using database config +t, err := tangy.New(dbConfig, tangy.Logger{Enabled: false}) +if err != nil { + return err +} +defer t.Close() + +// Use Tangy to search for RPMs, by name, that are associated to a specific repository version, returning up to the first 100 results +versionHref := "/pulp/e1c6bee3/api/v3/repositories/rpm/rpm/018c1c95-4281-76eb-b277-842cbad524f4/versions/1/" +rows, err := t.RpmRepositoryVersionPackageSearch(context.Background(), []string{versionHref}, "ninja", 100) +if err != nil { +return err +} +``` +See example.go for a complete example. + +## Developing +To develop for tangy, there are a few more things to know. + +### Create your configuration +`$ cp ./configs/config.yaml.example ./configs/config.yaml` + +### Connecting to pulp + +#### Connect to an existing pulp server +To connect to an existing pulp server, put the corresponding connection information in `configs/config.yaml`. + +#### Create a new pulp server +To create a new pulp server, you can use the provided make commands. You will need to have podman & podman-compose (or docker) installed. +The default values provided in config.yaml.example will work with this server. + +##### Start containers +`make compose-up` + +#### Stop containers +`make compose-down` + +#### Clean container volumes +`make compose-clean` + +### Mocking +Tangy also exports a mock interface you can regenerate using the [mockery](https://github.com/vektra/mockery) tool. \ No newline at end of file diff --git a/compose_files/pulp/assets/bin/nginx.sh b/compose_files/pulp/assets/bin/nginx.sh new file mode 100755 index 0000000..4e34f91 --- /dev/null +++ b/compose_files/pulp/assets/bin/nginx.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# This logic enables us to have multiple servers, and check to see +# if they are scaled every 10 seconds. +# https://serverfault.com/a/821625/189494 +# https://www.nginx.com/blog/dns-service-discovery-nginx-plus#domain-name-variable + +set -e + +if [ "$container" != "podman" ]; then + # the nameserver list under podman is unreliable. + # It will look like "10.89.1.1 192.168.1.1 192.168.1.1", but only the 1st IP works. + # This doesn't mess up `nslookup`, but it messes up `getent hosts` and nginx. + export NAMESERVER=`cat /etc/resolv.conf | grep "nameserver" | awk '{print $2}' | head -n1` +else + export NAMESERVER=`cat /etc/resolv.conf | grep "nameserver" | awk '{print $2}' | tr '\n' ' '` +fi + +echo "Nameserver is: $NAMESERVER" + +echo "Generating nginx config" +envsubst '$NAMESERVER' < /etc/opt/rh/rh-nginx116/nginx/nginx.conf.template > /etc/opt/rh/rh-nginx116/nginx/nginx.conf + +# We cannot use upstream server groups with a DNS resolver without nginx plus +# So we modifying the files to use the variables rather than the upstream server groups +for file in /opt/app-root/etc/nginx.default.d/*.conf ; do + echo "Modifying $file" + sed -i 's/pulp-api/$pulp_api:24817/' $file + sed -i 's/pulp-content/$pulp_content:24816/' $file +done + +echo "Starting nginx" +exec nginx -g "daemon off;" diff --git a/compose_files/pulp/assets/certs/database_fields.symmetric.key b/compose_files/pulp/assets/certs/database_fields.symmetric.key new file mode 100644 index 0000000..14697a0 --- /dev/null +++ b/compose_files/pulp/assets/certs/database_fields.symmetric.key @@ -0,0 +1 @@ +DNmNdwgyZugTax9S64J0FITTr9IHPxbuoF1F1CGPr68= diff --git a/compose_files/pulp/assets/nginx/nginx.conf.template b/compose_files/pulp/assets/nginx/nginx.conf.template new file mode 100644 index 0000000..cf7ebb5 --- /dev/null +++ b/compose_files/pulp/assets/nginx/nginx.conf.template @@ -0,0 +1,89 @@ +error_log /dev/stdout info; +worker_processes 1; +events { + worker_connections 1024; # increase if you have lots of clients + accept_mutex off; # set to 'on' if nginx worker_processes > 1 +} + +http { + access_log /dev/stdout; + include mime.types; + # fallback in case we can't determine a type + default_type application/octet-stream; + sendfile on; + + # If left at the default of 1024, nginx emits a warning about being unable + # to build optimal hash types. + types_hash_max_size 4096; + + server { + # This logic enables us to have multiple servers, and check to see + # if they are scaled every 10 seconds. + # https://www.nginx.com/blog/dns-service-discovery-nginx-plus#domain-name-variable + # https://serverfault.com/a/821625/189494 + resolver $NAMESERVER valid=10s; + set $pulp_api pulp_api; + set $pulp_content pulp_content; + + # Gunicorn docs suggest the use of the "deferred" directive on Linux. + listen 8080 default_server deferred; + listen [::]:8080 default_server deferred; + + # If you have a domain name, this is where to add it + server_name $hostname; + + # The default client_max_body_size is 1m. Clients uploading + # files larger than this will need to chunk said files. + client_max_body_size 10m; + + # Gunicorn docs suggest this value. + keepalive_timeout 5; + + # static files that can change dynamically, or are needed for TLS + # purposes are served through the webserver. + root /opt/app-root/src; + + location /pulp/content/ { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://$pulp_content:24816; + } + + location /pulp/api/v3/ { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://$pulp_api:24817; + } + + location /auth/login/ { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://$pulp_api:24817; + } + + include /opt/app-root/etc/nginx.default.d/*.conf; + + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://$pulp_api:24817; + # static files are served through whitenoise - http://whitenoise.evans.io/en/stable/ + } + } +} diff --git a/compose_files/pulp/assets/postgres/passwd b/compose_files/pulp/assets/postgres/passwd new file mode 100644 index 0000000..0f9afc2 --- /dev/null +++ b/compose_files/pulp/assets/postgres/passwd @@ -0,0 +1,20 @@ +root:x:0:0:root:/root:/bin/bash +daemon:x:1:1:daemon:/usr/sbin:/usr/sbin/nologin +bin:x:2:2:bin:/bin:/usr/sbin/nologin +sys:x:3:3:sys:/dev:/usr/sbin/nologin +sync:x:4:65534:sync:/bin:/bin/sync +games:x:5:60:games:/usr/games:/usr/sbin/nologin +man:x:6:12:man:/var/cache/man:/usr/sbin/nologin +lp:x:7:7:lp:/var/spool/lpd:/usr/sbin/nologin +mail:x:8:8:mail:/var/mail:/usr/sbin/nologin +news:x:9:9:news:/var/spool/news:/usr/sbin/nologin +uucp:x:10:10:uucp:/var/spool/uucp:/usr/sbin/nologin +proxy:x:13:13:proxy:/bin:/usr/sbin/nologin +www-data:x:33:33:www-data:/var/www:/usr/sbin/nologin +backup:x:34:34:backup:/var/backups:/usr/sbin/nologin +list:x:38:38:Mailing List Manager:/var/list:/usr/sbin/nologin +irc:x:39:39:ircd:/run/ircd:/usr/sbin/nologin +gnats:x:41:41:Gnats Bug-Reporting System (admin):/var/lib/gnats:/usr/sbin/nologin +nobody:x:65534:65534:nobody:/nonexistent:/usr/sbin/nologin +_apt:x:100:65534::/nonexistent:/usr/sbin/nologin +postgres:x:26:26::/var/lib/postgresql:/bin/bash diff --git a/compose_files/pulp/assets/settings.py b/compose_files/pulp/assets/settings.py new file mode 100644 index 0000000..54fcfbf --- /dev/null +++ b/compose_files/pulp/assets/settings.py @@ -0,0 +1,18 @@ +SECRET_KEY = "aabbcc" +CONTENT_ORIGIN = "http://pulp_content:24816" +DATABASES = {"default": {"HOST": "postgres", "ENGINE": "django.db.backends.postgresql", "NAME": "pulp", "USER": "pulp", "PASSWORD": "password", "PORT": "5432", "CONN_MAX_AGE": 0, "OPTIONS": {"sslmode": "prefer"}}} +CACHE_ENABLED = True +REDIS_HOST = "redis" +REDIS_PORT = 6379 +REDIS_PASSWORD = "" +ANSIBLE_API_HOSTNAME = "http://pulp_api:24817" +ANSIBLE_CONTENT_HOSTNAME = "http://pulp_content:24816/pulp/content" +ALLOWED_IMPORT_PATHS = ["/tmp"] +ALLOWED_EXPORT_PATHS = ["/tmp"] +TOKEN_SERVER = "http://pulp_api:24817/token/" +TOKEN_AUTH_DISABLED = False +TOKEN_SIGNATURE_ALGORITHM = "ES256" +PUBLIC_KEY_PATH = "/etc/pulp/keys/container_auth_public_key.pem" +PRIVATE_KEY_PATH = "/etc/pulp/keys/container_auth_private_key.pem" +TELEMETRY = False +STATIC_ROOT = "/var/lib/operator/static/" diff --git a/compose_files/pulp/docker-compose.yml b/compose_files/pulp/docker-compose.yml new file mode 100644 index 0000000..31ea482 --- /dev/null +++ b/compose_files/pulp/docker-compose.yml @@ -0,0 +1,137 @@ +version: '3' +services: + postgres: + image: "docker.io/library/postgres:13" + ports: + - "5434:5432" + environment: + POSTGRES_USER: pulp + POSTGRES_PASSWORD: password + POSTGRES_DB: pulp + POSTGRES_INITDB_ARGS: '--auth-host=scram-sha-256' + POSTGRES_HOST_AUTH_METHOD: 'scram-sha-256' + volumes: + - "pg_data:/var/lib/postgresql" + - "./assets/postgres/passwd:/etc/passwd:Z" + restart: always + healthcheck: + test: pg_isready + interval: 5s + retries: 10 + timeout: 3s + + migration_service: + image: "quay.io/cloudservices/pulp-rpm-ubi:latest" + depends_on: + postgres: + condition: service_healthy + command: pulpcore-manager migrate --noinput + volumes: + - "./assets/settings.py:/etc/pulp/settings.py:z" + - "./assets/certs:/etc/pulp/certs:z" + - "pulp:/var/lib/pulp" + + set_init_password_service: + image: "quay.io/cloudservices/pulp-rpm-ubi:latest" + command: set_init_password.sh + depends_on: + migration_service: + condition: service_completed_successfully + environment: + PULP_DEFAULT_ADMIN_PASSWORD: password + volumes: + - "./assets/settings.py:/etc/pulp/settings.py:z" + - "./assets/certs:/etc/pulp/certs:z" + - "pulp:/var/lib/pulp" + + redis: + image: "docker.io/library/redis:latest" + volumes: + - "redis_data:/data" + restart: always + healthcheck: + test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + + pulp_api: + image: "quay.io/cloudservices/pulp-rpm-ubi:latest" + deploy: + replicas: 1 + command: [ 'pulp-api' ] + depends_on: + migration_service: + condition: service_completed_successfully + hostname: pulp-api + user: pulp + volumes: + - "./assets/settings.py:/etc/pulp/settings.py:z" + - "./assets/certs:/etc/pulp/certs:z" + - "pulp:/var/lib/pulp" + environment: + POSTGRES_SERVICE_PORT: 5432 + POSTGRES_SERVICE_HOST: postgres + PULP_ADMIN_PASSWORD: password + PULP_DOMAIN_ENABLED: "true" + restart: always + + pulp_content: + image: "quay.io/cloudservices/pulp-rpm-ubi:latest" + deploy: + replicas: 1 + command: [ 'pulp-content' ] + depends_on: + migration_service: + condition: service_completed_successfully + hostname: pulp-content + user: pulp + volumes: + - "./assets/settings.py:/etc/pulp/settings.py:z" + - "./assets/certs:/etc/pulp/certs:z" + - "pulp:/var/lib/pulp" + environment: + POSTGRES_SERVICE_PORT: 5432 + POSTGRES_SERVICE_HOST: postgres + PULP_DOMAIN_ENABLED: "true" + restart: always + + pulp_web: + image: "pulp/pulp-web:latest" + command: [ '/usr/bin/nginx.sh' ] + depends_on: + migration_service: + condition: service_completed_successfully + ports: + - "8087:8080" + hostname: pulp + user: root + volumes: + - "./assets/bin/nginx.sh:/usr/bin/nginx.sh:Z" + - "./assets/nginx/nginx.conf.template:/etc/opt/rh/rh-nginx116/nginx/nginx.conf.template:Z" + restart: always + + pulp_worker: + image: "quay.io/cloudservices/pulp-rpm-ubi:latest" + deploy: + replicas: 1 + command: [ 'pulp-worker' ] + depends_on: + migration_service: + condition: service_completed_successfully + redis: + condition: service_healthy + user: pulp + volumes: + - "./assets/settings.py:/etc/pulp/settings.py:z" + - "./assets/certs:/etc/pulp/certs:z" + - "pulp:/var/lib/pulp" + environment: + POSTGRES_SERVICE_PORT: 5432 + POSTGRES_SERVICE_HOST: postgres + PULP_DOMAIN_ENABLED: "true" + restart: always +volumes: + pulp: + name: pulp${DEV_VOLUME_SUFFIX:-dev} + pg_data: + name: pg_data${DEV_VOLUME_SUFFIX:-dev} + redis_data: + name: redis_data${DEV_VOLUME_SUFFIX:-dev} diff --git a/configs/config.yaml.example b/configs/config.yaml.example new file mode 100644 index 0000000..11f8c4a --- /dev/null +++ b/configs/config.yaml.example @@ -0,0 +1,15 @@ +# Configuration options for the pulp database +database: + name: "pulp" + host: "localhost" + port: "5434" + user: "pulp" + password: "password" + +# Configuration options for the pulp server +server: + url: "http://localhost:8087" + username: "admin" + password: "password" + storage_type: "local" + download_policy: "on_demand" \ No newline at end of file diff --git a/example.go b/example.go new file mode 100644 index 0000000..cd8ba27 --- /dev/null +++ b/example.go @@ -0,0 +1,94 @@ +package main + +import ( + "context" + "fmt" + + "github.com/content-services/tang/internal/config" + "github.com/content-services/tang/internal/zestwrapper" + "github.com/content-services/tang/pkg/tangy" +) + +func main() { + // Pulp database configuration information + dbConfig := tangy.Database{ + Name: "pulp", + Host: "localhost", + Port: 5434, + User: "pulp", + Password: "password", + CACertPath: "", + PoolLimit: 20, + } + + // Create new Tangy instance using database config + t, err := tangy.New(dbConfig, tangy.Logger{Enabled: false}) + if err != nil { + fmt.Println(err) + return + } + defer t.Close() + + // Call helper function that creates and syncs a repository + versionHref, err := CreateRepositoryVersion() + if err != nil { + fmt.Println(err) + return + } + + // Use Tangy to search for RPMs, by name, that are associated to a specific repository version, returning up to the first 100 results + rows, err := t.RpmRepositoryVersionPackageSearch(context.Background(), []string{versionHref}, "ninja", 100) + if err != nil { + fmt.Println(err) + return + } + + for _, row := range rows { + fmt.Printf("\nName: %v \nSummary: %v", row.Name, row.Summary) + } + fmt.Printf("\n") +} + +func CreateRepositoryVersion() (versionHref string, err error) { + // Create new Pulp API wrapper instance, so we can use it for testing + rpmZest := zestwrapper.NewRpmZest(context.Background(), config.Server{ + Url: "http://localhost:8087", + Username: "admin", + Password: "password", + StorageType: "local", + DownloadPolicy: "on_demand", + }) + + domainName := "example-domain" + + // Create domain and repository, then sync repository, to create a new repository version with rpm packages + _, err = rpmZest.LookupOrCreateDomain(domainName) + if err != nil { + return "", err + } + + repoHref, remoteHref, err := rpmZest.CreateRepository(domainName, "rpm modular", "https://fixtures.pulpproject.org/rpm-modular/") + if err != nil { + return "", err + } + + syncTask, err := rpmZest.SyncRpmRepository(repoHref, remoteHref) + if err != nil { + return "", err + } + + _, err = rpmZest.PollTask(syncTask) + if err != nil { + return "", err + } + + resp, err := rpmZest.GetRpmRepositoryByName(domainName, "rpm modular") + if err != nil { + return "", err + } + if resp.LatestVersionHref == nil { + return "", fmt.Errorf("latest version href is nil") + } + + return *resp.LatestVersionHref, nil +} diff --git a/go.mod b/go.mod index 16aa02a..91b96c8 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,46 @@ module github.com/content-services/tang go 1.20 + +require ( + github.com/content-services/zest/release/v2023 v2023.11.1701177874 + github.com/google/uuid v1.4.0 + github.com/jackc/pgx-zerolog v0.0.0-20230315001418-f978528409eb + github.com/jackc/pgx/v5 v5.5.1 + github.com/rs/zerolog v1.31.0 + github.com/spf13/viper v1.18.1 + github.com/stretchr/testify v1.8.4 + golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb +) + +require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/objx v0.5.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.9.0 // indirect + golang.org/x/crypto v0.16.0 // indirect + golang.org/x/sync v0.5.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..ec1e90e --- /dev/null +++ b/go.sum @@ -0,0 +1,99 @@ +github.com/content-services/zest/release/v2023 v2023.11.1701177874 h1:yC5uRkU78UtVe3oTn0pRwT9KBd9b4SBAfQ9QTargCqg= +github.com/content-services/zest/release/v2023 v2023.11.1701177874/go.mod h1:9pesd98rUBOqg1z2UL65NA1+zZQRcFJY3VIdimAJxL8= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx-zerolog v0.0.0-20230315001418-f978528409eb h1:pSv+zRVeAYjbXRFjyytFIMRBSKWVowCi7KbXSMR/+ug= +github.com/jackc/pgx-zerolog v0.0.0-20230315001418-f978528409eb/go.mod h1:CRUuPsmIajLt3dZIlJ5+O8IDSib6y8yrst8DkCthTa4= +github.com/jackc/pgx/v5 v5.5.1 h1:5I9etrGkLrN+2XPCsi6XLlV5DITbSL/xBZdmAxFcXPI= +github.com/jackc/pgx/v5 v5.5.1/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= +github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.18.1 h1:rmuU42rScKWlhhJDyXZRKJQHXFX02chSVW1IvkPGiVM= +github.com/spf13/viper v1.18.1/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb h1:c0vyKkb6yr3KR7jEfJaOSv4lG7xPkbN6r52aJz1d8a8= +golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..c529115 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,57 @@ +package config + +import ( + "strings" + + "github.com/rs/zerolog/log" + "github.com/spf13/viper" +) + +type Config struct { + Server Server + Database Database +} + +func Get() Config { + var c Config + v := viper.New() + + v.SetConfigName("config.yaml") + v.SetConfigType("yaml") + v.AddConfigPath("./configs/") + v.AddConfigPath("../../../configs/") + v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + v.AutomaticEnv() + + err := v.ReadInConfig() + if err != nil { + log.Err(err).Msg("config.yaml file not loaded") + } + + err = v.Unmarshal(&c) + if err != nil { + panic(err) + } + + return c +} + +// Server configuration options for connecting to a pulp server +type Server struct { + Url string `mapstructure:"url"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` + StorageType string `mapstructure:"storage_type"` + DownloadPolicy string `mapstructure:"download_policy"` +} + +// Database configuration options for connection to a pulp database. Duplicated of tangy.Database. +type Database struct { + Name string `mapstructure:"name"` + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + User string `mapstructure:"user"` + Password string `mapstructure:"password"` + CACertPath string `mapstructure:"ca_cert_path"` + PoolLimit int `mapstructure:"pool_limit"` +} diff --git a/internal/test/integration/rpm_test.go b/internal/test/integration/rpm_test.go new file mode 100644 index 0000000..6e62a53 --- /dev/null +++ b/internal/test/integration/rpm_test.go @@ -0,0 +1,160 @@ +package integration + +import ( + "context" + "math/rand" + "testing" + + "github.com/content-services/tang/internal/config" + "github.com/content-services/tang/internal/zestwrapper" + "github.com/content-services/tang/pkg/tangy" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type RpmSuite struct { + suite.Suite + client *zestwrapper.RpmZest + tangy tangy.Tangy + domainName string + remoteHref string + repoHref string +} + +const testRepoName = "rpm modular" +const testRepoURL = "https://jlsherrill.fedorapeople.org/fake-repos/revision/one/" +const testRepoURLTwo = "https://jlsherrill.fedorapeople.org/fake-repos/revision/two/" + +func (r *RpmSuite) CreateTestRepository(t *testing.T) { + domainName := RandStringBytes(10) + r.domainName = domainName + + _, err := r.client.LookupOrCreateDomain(domainName) + require.NoError(t, err) + + repoHref, remoteHref, err := r.client.CreateRepository(domainName, testRepoName, testRepoURL) + require.NoError(t, err) + + r.repoHref = repoHref + r.remoteHref = remoteHref + + syncTask, err := r.client.SyncRpmRepository(repoHref, remoteHref) + require.NoError(t, err) + + _, err = r.client.PollTask(syncTask) + require.NoError(t, err) +} + +func (r *RpmSuite) UpdateTestRepository(t *testing.T, url string) { + err := r.client.UpdateRemote(r.remoteHref, url) + require.NoError(t, err) + + syncTask, err := r.client.SyncRpmRepository(r.repoHref, r.remoteHref) + require.NoError(t, err) + + _, err = r.client.PollTask(syncTask) + require.NoError(t, err) +} + +func TestRpmSuite(t *testing.T) { + s := config.Get().Server + rpmZest := zestwrapper.NewRpmZest(context.Background(), s) + + dbConfig := config.Get().Database + ta, err := tangy.New(tangy.Database{ + Name: dbConfig.Name, + Host: dbConfig.Host, + Port: dbConfig.Port, + User: dbConfig.User, + Password: dbConfig.Password, + }, tangy.Logger{Enabled: true, Logger: &log.Logger, LogLevel: zerolog.LevelDebugValue}) + require.NoError(t, err) + + r := RpmSuite{} + r.client = &rpmZest + r.tangy = ta + r.CreateTestRepository(t) + suite.Run(t, &r) +} + +func (r *RpmSuite) TestRpmRepositoryVersionPackageSearch() { + resp, err := r.client.GetRpmRepositoryByName(r.domainName, testRepoName) + require.NoError(r.T(), err) + firstVersionHref := resp.LatestVersionHref + require.NotNil(r.T(), firstVersionHref) + + // Search first repository version + search, err := r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*firstVersionHref}, "bea", 100) + assert.NoError(r.T(), err) + assert.Equal(r.T(), search[0].Name, "bear") + search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*firstVersionHref}, "cam", 100) + assert.NoError(r.T(), err) + assert.Empty(r.T(), search) + + // Create second repository version + r.UpdateTestRepository(r.T(), testRepoURLTwo) + resp, err = r.client.GetRpmRepositoryByName(r.domainName, testRepoName) + require.NoError(r.T(), err) + secondVersionHref := resp.LatestVersionHref + require.NotNil(r.T(), secondVersionHref) + + // Search second repository version, should have new package + search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*secondVersionHref}, "bea", 100) + assert.NoError(r.T(), err) + assert.Equal(r.T(), search[0].Name, "bear") + search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*secondVersionHref}, "cam", 100) + assert.NoError(r.T(), err) + assert.Equal(r.T(), search[0].Name, "camel") + + // Re-search the first version, should be the same + search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*firstVersionHref}, "bea", 100) + assert.NoError(r.T(), err) + assert.Equal(r.T(), search[0].Name, "bear") + search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*firstVersionHref}, "cam", 100) + assert.NoError(r.T(), err) + assert.Empty(r.T(), search) + + // Search both versions + search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*firstVersionHref, *secondVersionHref}, "a", 100) + assert.NoError(r.T(), err) + assert.Len(r.T(), search, 2) + assert.Equal(r.T(), search[0].Name, "bear") + assert.Equal(r.T(), search[1].Name, "camel") + + // Create third repository version to remove new package + r.UpdateTestRepository(r.T(), testRepoURL) + resp, err = r.client.GetRpmRepositoryByName(r.domainName, testRepoName) + require.NoError(r.T(), err) + thirdVersionHref := resp.LatestVersionHref + require.NotNil(r.T(), thirdVersionHref) + + // Search third repository version, should not have new package + search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*thirdVersionHref}, "bea", 100) + assert.NoError(r.T(), err) + assert.Equal(r.T(), search[0].Name, "bear") + search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*thirdVersionHref}, "cam", 100) + assert.NoError(r.T(), err) + assert.Empty(r.T(), search) + + // Test search limit + search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*secondVersionHref}, "a", 1) + assert.NoError(r.T(), err) + assert.Len(r.T(), search, 1) + + // Test search empty list + search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{}, "a", 1) + assert.NoError(r.T(), err) + assert.Len(r.T(), search, 0) +} + +func RandStringBytes(n int) string { + const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + b := make([]byte, n) + for i := range b { + b[i] = letterBytes[rand.Intn(len(letterBytes))] + } + return string(b) +} diff --git a/internal/zestwrapper/rpm.go b/internal/zestwrapper/rpm.go new file mode 100644 index 0000000..8d9cef6 --- /dev/null +++ b/internal/zestwrapper/rpm.go @@ -0,0 +1,225 @@ +package zestwrapper + +import ( + "context" + "errors" + "fmt" + "net/http" + "time" + + "github.com/content-services/tang/internal/config" + zest "github.com/content-services/zest/release/v2023" + "golang.org/x/exp/slices" +) + +const DefaultDomain = "default" + +const ( + COMPLETED string = "completed" + WAITING string = "waiting" + RUNNING string = "running" + SKIPPED string = "skipped" + CANCELED string = "canceled" + CANCELING string = "canceling" + FAILED string = "failed" +) + +func NewRpmZest(ctx context.Context, server config.Server) RpmZest { + ctx2 := context.WithValue(ctx, zest.ContextServerIndex, 0) + timeout := 60 * time.Second + transport := &http.Transport{ResponseHeaderTimeout: timeout} + httpClient := http.Client{Transport: transport, Timeout: timeout} + + pulpConfig := zest.NewConfiguration() + pulpConfig.HTTPClient = &httpClient + pulpConfig.Servers = zest.ServerConfigurations{zest.ServerConfiguration{ + URL: server.Url, + }} + ctx2 = context.WithValue(ctx2, zest.ContextBasicAuth, zest.BasicAuth{ + UserName: server.Username, + Password: server.Password, + }) + + return RpmZest{ + client: zest.NewAPIClient(pulpConfig), + ctx: ctx2, + } +} + +type RpmZest struct { + client *zest.APIClient + ctx context.Context +} + +func (r *RpmZest) LookupOrCreateDomain(name string) (string, error) { + d, err := r.LookupDomain(name) + if err != nil { + return "", err + } + if d != "" { + return d, nil + } + + localStorage := zest.STORAGECLASSENUM_PULPCORE_APP_MODELS_STORAGE_FILE_SYSTEM + var domain zest.Domain + emptyConfig := make(map[string]interface{}) + emptyConfig["location"] = fmt.Sprintf("/var/lib/pulp/%v/", name) + domain = *zest.NewDomain(name, localStorage, emptyConfig) + domainResp, resp, err := r.client.DomainsAPI.DomainsCreate(r.ctx, DefaultDomain).Domain(domain).Execute() + if resp != nil && resp.Body != nil { + defer resp.Body.Close() + } + if err != nil { + return "", err + } + return *domainResp.PulpHref, nil +} + +func (r *RpmZest) LookupDomain(name string) (string, error) { + list, resp, err := r.client.DomainsAPI.DomainsList(r.ctx, "default").Name(name).Execute() + if err != nil { + return "", err + } + defer resp.Body.Close() + + if len(list.Results) == 0 { + return "", nil + } else if list.Results[0].PulpHref == nil { + return "", fmt.Errorf("Unexpectedly got a nil href for domain %v", name) + } else { + return *list.Results[0].PulpHref, nil + } +} + +func (r *RpmZest) CreateRepository(domain, name, url string) (repoHref string, remoteHref string, err error) { + rpmRpmRemote := *zest.NewRpmRpmRemote(name, url) + + remoteResponse, httpResp, err := r.client.RemotesRpmAPI.RemotesRpmRpmCreate(r.ctx, domain). + RpmRpmRemote(rpmRpmRemote).Execute() + if err != nil { + return "", "", err + } + defer httpResp.Body.Close() + + rpmRpmRepository := *zest.NewRpmRpmRepository(name) + if remoteResponse.PulpHref != nil { + rpmRpmRepository.SetRemote(*remoteResponse.PulpHref) + } + resp, httpResp, err := r.client.RepositoriesRpmAPI.RepositoriesRpmRpmCreate(r.ctx, domain). + RpmRpmRepository(rpmRpmRepository).Execute() + + if err != nil { + return "", "", err + } + defer httpResp.Body.Close() + + return *resp.PulpHref, *remoteResponse.PulpHref, nil +} + +func (r *RpmZest) UpdateRemote(remoteHref string, url string) error { + _, httpResp, err := r.client.RemotesRpmAPI.RemotesRpmRpmPartialUpdate(r.ctx, remoteHref).PatchedrpmRpmRemote(zest.PatchedrpmRpmRemote{Url: &url}).Execute() + if httpResp != nil { + defer httpResp.Body.Close() + } + if err != nil { + return err + } + return nil +} + +func (r *RpmZest) SyncRpmRepository(rpmRpmRepositoryHref string, remoteHref string) (string, error) { + rpmRepositoryHref := *zest.NewRpmRepositorySyncURL() + rpmRepositoryHref.SetRemote(remoteHref) + rpmRepositoryHref.SetSyncPolicy(zest.SYNCPOLICYENUM_MIRROR_CONTENT_ONLY) + + resp, httpResp, err := r.client.RepositoriesRpmAPI.RepositoriesRpmRpmSync(r.ctx, rpmRpmRepositoryHref). + RpmRepositorySyncURL(rpmRepositoryHref).Execute() + defer httpResp.Body.Close() + if err != nil { + return "", err + } + return resp.Task, nil +} + +// GetTask Fetch a pulp task +func (r *RpmZest) GetTask(taskHref string) (zest.TaskResponse, error) { + task, httpResp, err := r.client.TasksAPI.TasksRead(r.ctx, taskHref).Execute() + + if err != nil { + return zest.TaskResponse{}, err + } + defer httpResp.Body.Close() + + return *task, nil +} + +// PollTask Poll a task and return the final task object +func (r *RpmZest) PollTask(taskHref string) (*zest.TaskResponse, error) { + var task zest.TaskResponse + var err error + inProgress := true + pollCount := 1 + for inProgress { + task, err = r.GetTask(taskHref) + if err != nil { + return nil, err + } + taskState := *task.State + switch { + case slices.Contains([]string{COMPLETED, SKIPPED, CANCELED}, taskState): + inProgress = false + case slices.Contains([]string{WAITING, RUNNING, CANCELING}, taskState): + case taskState == FAILED: + errorStr := TaskErrorString(task) + return &task, errors.New(errorStr) + default: + inProgress = false + } + + if inProgress { + SleepWithBackoff(pollCount) + pollCount += 1 + } + } + return &task, nil +} + +func (r *RpmZest) GetRpmRepositoryByName(domain, name string) (*zest.RpmRpmRepositoryResponse, error) { + resp, httpResp, err := r.client.RepositoriesRpmAPI.RepositoriesRpmRpmList(r.ctx, domain).Name(name).Execute() + + if err != nil { + return nil, err + } + defer httpResp.Body.Close() + + results := resp.GetResults() + if len(results) > 0 { + return &results[0], nil + } else { + return nil, nil + } +} + +func TaskErrorString(task zest.TaskResponse) string { + str := "" + if task.Error != nil { + for key, element := range *task.Error { + str = str + fmt.Sprintf("%v: %v. ", key, element) + } + } + return str +} + +func SleepWithBackoff(iteration int) { + var secs int + if iteration <= 5 { + secs = 1 + } else if iteration > 5 && iteration <= 10 { + secs = 5 + } else if iteration > 10 && iteration <= 20 { + secs = 10 + } else { + secs = 30 + } + time.Sleep(time.Duration(secs) * time.Second) +} diff --git a/mk/compose.mk b/mk/compose.mk new file mode 100644 index 0000000..a9e7d94 --- /dev/null +++ b/mk/compose.mk @@ -0,0 +1,17 @@ +## +# Set of rules to manage podman-compose +# +# Requires 'mk/variables.mk' +## + +.PHONY: compose-up +compose-up: ## Start up service dependencies using podman(docker)-compose + $(PULP_COMPOSE_COMMAND) + +.PHONY: compose-down +compose-down: ## Shut down service dependencies using podman(docker)-compose + $(PULP_COMPOSE_DOWN_COMMAND) + +.PHONY: compose-clean ## Clear out data (dbs, files) for service dependencies +compose-clean: compose-down + $(DOCKER) volume prune --force \ No newline at end of file diff --git a/mk/help.mk b/mk/help.mk new file mode 100644 index 0000000..80ebc51 --- /dev/null +++ b/mk/help.mk @@ -0,0 +1,13 @@ +## +# This file only contains the rule that generate the +# help content from the comments in the different files. +# +# Use '##@ My group text' at the beginning of a line to +# print out a group text. +# +# Use '## My help text' at the end of a rule to print out +# content related with a rule. Try to short the description. +## +.PHONY: help +help: ## Print out the help content + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) diff --git a/mk/includes.mk b/mk/includes.mk new file mode 100644 index 0000000..afe2d9c --- /dev/null +++ b/mk/includes.mk @@ -0,0 +1,24 @@ +## +# The target for this file is just to enumerate which partial +# makefile we want to use to compose our final Makefile. +# +# Unless you are not using conditional assignment within +# the different variable files, this would be the priority: +# - The values indicated at 'configs/config.yaml' file. +# - The values indicated at 'mk/variables.mk' file. This +# file is included into the repository and define the +# default values for the variables, if not assigned yet. +# - The 'mk/meta-*.mk' files just contain the comment to +# print out the group text for the help content. They +# are into independent files, because the order they +# appear into this include file matters, and provide +# the flexibility to print out the group text exactly +# where we want kust changing the order into this file. +# +# This file set the 'help' rule as the default one when +# no arguments are indicated. +## +include mk/variables.mk +include mk/compose.mk +include mk/help.mk +include mk/test.mk diff --git a/mk/test.mk b/mk/test.mk new file mode 100644 index 0000000..c21f736 --- /dev/null +++ b/mk/test.mk @@ -0,0 +1,3 @@ +.PHONY: test-integration +test-integration: ## Run tests for ci + CONFIG_PATH="$(PROJECT_DIR)/configs/" go test $(MOD_VENDOR) ./internal/test/integration/... \ No newline at end of file diff --git a/mk/variables.mk b/mk/variables.mk new file mode 100644 index 0000000..63ceb4c --- /dev/null +++ b/mk/variables.mk @@ -0,0 +1,30 @@ +## Project +COMPOSE_PROJECT_NAME ?= tang +export COMPOSE_PROJECT_NAME + +PROJECT_DIR := $(shell dirname $(abspath $(firstword $(MAKEFILE_LIST)))) + +## Docker/Podman command +ifneq (,$(shell command podman -v 2>/dev/null)) +DOCKER ?= podman +else +ifneq (,$(shell command docker -v 2>/dev/null)) +DOCKER ?= docker +else +DOCKER ?= false +endif +endif + +## Docker Compose +PULP_COMPOSE_FILES ?= "compose_files/pulp/docker-compose.yml" +PULP_COMPOSE_OPTIONS=PULP_POSTGRES_PATH="pulp_db" PULP_STORAGE_PATH="pulp_storage" + +PULP_COMPOSE_COMMAND=$(PULP_COMPOSE_OPTIONS) $(DOCKER)-compose --project-name=$(COMPOSE_PROJECT_NAME) -f $(PULP_COMPOSE_FILES) up --detach +PULP_COMPOSE_DOWN_COMMAND=$(PULP_COMPOSE_OPTIONS) $(DOCKER)-compose --project-name=$(COMPOSE_PROJECT_NAME) -f $(PULP_COMPOSE_FILES) down + +# Tests +ifeq (,$(shell ls -1d vendor 2>/dev/null)) +MOD_VENDOR := +else +MOD_VENDOR ?= -mod vendor +endif diff --git a/pkg/tangy/config.go b/pkg/tangy/config.go new file mode 100644 index 0000000..243456c --- /dev/null +++ b/pkg/tangy/config.go @@ -0,0 +1,47 @@ +package tangy + +import ( + "fmt" + + "github.com/rs/zerolog" +) + +const DefaultMaxPoolLimit = 20 + +// Logger configuration options for logger +type Logger struct { + Logger *zerolog.Logger + LogLevel string + Enabled bool +} + +// Database configuration options for connection to a pulp database +type Database struct { + Name string + Host string + Port int + User string + Password string + CACertPath string `mapstructure:"ca_cert_path"` + PoolLimit int `mapstructure:"pool_limit"` +} + +// Url return url of database +func (d Database) Url() string { + connectStr := fmt.Sprintf( + "user=%s password=%s dbname=%s host=%s port=%d", + d.User, + d.Password, + d.Name, + d.Host, + d.Port, + ) + + var sslStr string + if d.CACertPath == "" { + sslStr = " sslmode=disable" + } else { + sslStr = fmt.Sprintf(" sslmode=verify-full sslrootcert=%s", d.CACertPath) + } + return connectStr + sslStr +} diff --git a/pkg/tangy/interface.go b/pkg/tangy/interface.go new file mode 100644 index 0000000..f57f0f6 --- /dev/null +++ b/pkg/tangy/interface.go @@ -0,0 +1,62 @@ +package tangy + +import ( + "context" + "fmt" + + zerologadapter "github.com/jackc/pgx-zerolog" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/jackc/pgx/v5/tracelog" + "github.com/rs/zerolog/log" +) + +func New(dbConfig Database, logConfig Logger) (Tangy, error) { + pxConfig, err := pgxpool.ParseConfig(dbConfig.Url()) + if err != nil { + return nil, err + } + + if dbConfig.PoolLimit == 0 { + dbConfig.PoolLimit = DefaultMaxPoolLimit + } + pxConfig.MaxConns = int32(dbConfig.PoolLimit) + + if logConfig.Logger != nil && logConfig.Enabled { + zlog := zerologadapter.NewLogger(*logConfig.Logger) + level, err := tracelog.LogLevelFromString(logConfig.LogLevel) + if err != nil { + log.Error().Err(err).Msg("Error setting Pgx log level") + } + pxConfig.ConnConfig.Tracer = &tracelog.TraceLog{ + Logger: zlog, + LogLevel: level, + } + } + + pool, err := pgxpool.NewWithConfig(context.Background(), pxConfig) + if err != nil { + return nil, fmt.Errorf("error establishing connection: %w", err) + } + + t := tangyImpl{ + pool: pool, + logger: logConfig, + } + return &t, nil +} + +type tangyImpl struct { + pool *pgxpool.Pool + logger Logger +} + +//go:generate mockery --name Tangy --filename tangy_mock.go --inpackage +type Tangy interface { + RpmRepositoryVersionPackageSearch(ctx context.Context, hrefs []string, search string, limit int) ([]RpmPackageSearch, error) + Close() +} + +// Close closes the DB connection pool +func (t *tangyImpl) Close() { + t.pool.Close() +} diff --git a/pkg/tangy/rpm.go b/pkg/tangy/rpm.go new file mode 100644 index 0000000..9bd5f95 --- /dev/null +++ b/pkg/tangy/rpm.go @@ -0,0 +1,100 @@ +package tangy + +import ( + "context" + "fmt" + "strconv" + "strings" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +const DefaultLimit = 500 + +type RpmPackageSearch struct { + Name string + Summary string +} + +// RpmRepositoryVersionPackageSearch search for RPMs, by name, associated to repository hrefs, returning an amount up to limit +func (t *tangyImpl) RpmRepositoryVersionPackageSearch(ctx context.Context, hrefs []string, search string, limit int) ([]RpmPackageSearch, error) { + if len(hrefs) == 0 { + return []RpmPackageSearch{}, nil + } + + conn, err := t.pool.Acquire(ctx) + if err != nil { + return nil, err + } + defer conn.Release() + + if limit == 0 { + limit = DefaultLimit + } + + repositoryIDs, versions, err := parseRepositoryVersionHrefs(hrefs) + if err != nil { + return nil, fmt.Errorf("error parsing repository version hrefs: %w", err) + } + + query := `SELECT DISTINCT ON (rp.name) rp.name, rp.summary + FROM rpm_package rp WHERE rp.content_ptr_id IN (` + for i := 0; i < len(repositoryIDs); i++ { + id := repositoryIDs[i] + ver := versions[i] + + query += fmt.Sprintf(` + ( + SELECT crc.content_id + FROM core_repositorycontent crc + INNER JOIN core_repositoryversion crv ON (crc.version_added_id = crv.pulp_id) + LEFT OUTER JOIN core_repositoryversion crv2 ON (crc.version_removed_id = crv2.pulp_id) + WHERE crv.repository_id = '%v' AND crv.number <= %v AND NOT (crv2.number <= %v AND crv2.number IS NOT NULL) + AND rp.name ILIKE CONCAT( '%%', '%v'::text, '%%') + ) + `, id, ver, ver, search) + + if i == len(repositoryIDs)-1 { + query += fmt.Sprintf(") ORDER BY rp.name ASC LIMIT %v;", limit) + break + } + + query += "UNION" + } + rows, err := conn.Query(context.Background(), query) + if err != nil { + return nil, err + } + rpms, err := pgx.CollectRows(rows, pgx.RowToStructByName[RpmPackageSearch]) + if err != nil { + return nil, err + } + return rpms, nil +} + +func parseRepositoryVersionHrefs(hrefs []string) (repositoryIDs []string, versions []int, err error) { + // /pulp/e1c6bee3/api/v3/repositories/rpm/rpm/018c1c95-4281-76eb-b277-842cbad524f4/versions/1/ + for _, href := range hrefs { + splitHref := strings.Split(href, "/") + if len(splitHref) < 10 { + return nil, nil, fmt.Errorf("%v is not a valid href", splitHref) + } + id := splitHref[8] + num := splitHref[10] + + _, err = uuid.Parse(id) + if err != nil { + return nil, nil, fmt.Errorf("%v is not a valid uuid", id) + } + + ver, err := strconv.Atoi(num) + if err != nil { + return nil, nil, fmt.Errorf("%v is not a valid integer", num) + } + + repositoryIDs = append(repositoryIDs, id) + versions = append(versions, ver) + } + return +} diff --git a/pkg/tangy/tangy_mock.go b/pkg/tangy/tangy_mock.go new file mode 100644 index 0000000..c00bbeb --- /dev/null +++ b/pkg/tangy/tangy_mock.go @@ -0,0 +1,59 @@ +// Code generated by mockery v2.32.0. DO NOT EDIT. + +package tangy + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" +) + +// MockTangy is an autogenerated mock type for the Tangy type +type MockTangy struct { + mock.Mock +} + +// Close provides a mock function with given fields: +func (_m *MockTangy) Close() { + _m.Called() +} + +// RpmRepositoryVersionPackageSearch provides a mock function with given fields: ctx, hrefs, search, limit +func (_m *MockTangy) RpmRepositoryVersionPackageSearch(ctx context.Context, hrefs []string, search string, limit int) ([]RpmPackageSearch, error) { + ret := _m.Called(ctx, hrefs, search, limit) + + var r0 []RpmPackageSearch + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, []string, string, int) ([]RpmPackageSearch, error)); ok { + return rf(ctx, hrefs, search, limit) + } + if rf, ok := ret.Get(0).(func(context.Context, []string, string, int) []RpmPackageSearch); ok { + r0 = rf(ctx, hrefs, search, limit) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]RpmPackageSearch) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, []string, string, int) error); ok { + r1 = rf(ctx, hrefs, search, limit) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewMockTangy creates a new instance of MockTangy. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockTangy(t interface { + mock.TestingT + Cleanup(func()) +}) *MockTangy { + mock := &MockTangy{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +}