From 62b28cac5c4b185c9bd2014031cfee11a06c15b5 Mon Sep 17 00:00:00 2001 From: Alex Gustafsson Date: Thu, 12 Dec 2024 17:39:04 +0100 Subject: [PATCH] Add basic support for GitLab --- internal/registry/gitlab/client.go | 297 ++++++++++++++++++ internal/registry/gitlab/client_test.go | 59 ++++ .../imageworkflow/getgitlablatestversion.go | 38 +++ .../imageworkflow/setupregistryclient.go | 8 + internal/workflow/imageworkflow/workflow.go | 31 +- 5 files changed, 432 insertions(+), 1 deletion(-) create mode 100644 internal/registry/gitlab/client.go create mode 100644 internal/registry/gitlab/client_test.go create mode 100644 internal/workflow/imageworkflow/getgitlablatestversion.go diff --git a/internal/registry/gitlab/client.go b/internal/registry/gitlab/client.go new file mode 100644 index 0000000..313828e --- /dev/null +++ b/internal/registry/gitlab/client.go @@ -0,0 +1,297 @@ +package gitlab + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/AlexGustafsson/cupdate/internal/httputil" + "github.com/AlexGustafsson/cupdate/internal/registry" + "github.com/AlexGustafsson/cupdate/internal/registry/oci" +) + +type Client struct { + Client *httputil.Client +} + +func (c *Client) GetRegistryToken(ctx context.Context, image oci.Reference) (string, error) { + // TODO: Registries expose the realm and scheme via Www-Authenticate if 403 + // is given + u, err := url.Parse("https://gitlab.com/jwt/auth?service=container_registry") + if err != nil { + return "", err + } + + query := u.Query() + query.Set("scope", fmt.Sprintf("repository:%s:pull", image.Path)) + u.RawQuery = query.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return "", err + } + + // TODO: The cache doesn't understand graphql, so we can't cache this request + res, err := c.Client.Do(req) + if err != nil { + return "", err + } + + if res.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status code: %s", res.Status) + } + + var result struct { + Token string `json:"token"` + ExpiresIn int `json:"expires_in"` + IssuedAt time.Time `json:"issued_at"` + } + if err := json.NewDecoder(res.Body).Decode(&result); err != nil { + return "", err + } + + return result.Token, nil +} + +func (c *Client) Authorize(ctx context.Context, image oci.Reference, req *http.Request) error { + token, err := c.GetRegistryToken(ctx, image) + if err != nil { + return err + } + + return oci.AuthorizerToken(token).Authorize(ctx, image, req) +} + +func (c *Client) GetProjectContainerRepositories(ctx context.Context, fullPath string) ([]ContainerRepository, error) { + payload, err := json.Marshal(map[string]any{ + "operationName": "getProjectContainerRepositories", + "variables": map[string]any{ + "sort": "UPDATED_DESC", + "fullPath": fullPath, + "first": 20, + }, + "query": `query getProjectContainerRepositories($fullPath: ID!, $name: String, $first: Int, $last: Int, $after: String, $before: String, $sort: ContainerRepositorySort) { + project(fullPath: $fullPath) { + id + containerRepositories( + name: $name + after: $after + before: $before + first: $first + last: $last + sort: $sort + ) { + nodes { + id + location + } + } + } +}`, + }) + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://gitlab.com/api/graphql", bytes.NewReader(payload)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + // TODO: The cache doesn't understand graphql, so we can't cache this request + res, err := c.Client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %s", res.Status) + } + + var result struct { + Errors []any `json:"error"` + Data struct { + Project struct { + ContainerRepositories struct { + Nodes []ContainerRepository `json:"nodes"` + } `json:"containerRepositories"` + } `json:"project"` + } `json:"data"` + } + if err := json.NewDecoder(res.Body).Decode(&result); err != nil { + return nil, err + } else if len(result.Errors) > 0 { + return nil, fmt.Errorf("got unexpected graphql error: %v", result.Errors) + } + + return result.Data.Project.ContainerRepositories.Nodes, nil +} +func (c *Client) GetProjectContainerRepositoryTags(ctx context.Context, id string) ([]ContainerRepositoryTag, error) { + payload, err := json.Marshal(map[string]any{ + "operationName": "getContainerRepositoryTags", + "variables": map[string]any{ + "referrers": true, + "id": id, + "first": 20, + "sort": "PUBLISHED_AT_DESC", + }, + "query": `query getContainerRepositoryTags($id: ContainerRepositoryID!, $first: Int, $last: Int, $after: String, $before: String, $name: String, $sort: ContainerRepositoryTagSort, $referrers: Boolean = false) { + containerRepository(id: $id) { + tags( + after: $after + before: $before + first: $first + last: $last + name: $name + sort: $sort + referrers: $referrers + ) { + nodes { + digest + location + path + name + createdAt + publishedAt + } + } + } +}`}) + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://gitlab.com/api/graphql", bytes.NewReader(payload)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + res, err := c.Client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %s", res.Status) + } + + var result struct { + Errors []any `json:"error"` + Data struct { + ContainerRepository struct { + Tags struct { + Nodes []ContainerRepositoryTag `json:"nodes"` + } `json:"tags"` + } `json:"containerRepository"` + } `json:"data"` + } + if err := json.NewDecoder(res.Body).Decode(&result); err != nil { + return nil, err + } else if len(result.Errors) > 0 { + return nil, fmt.Errorf("got unexpected graphql error: %v", result.Errors) + } + + return result.Data.ContainerRepository.Tags.Nodes, nil +} + +func (c *Client) GetLatestVersion(ctx context.Context, image oci.Reference) (*registry.Image, error) { + if !image.HasTag { + return nil, nil + } + + // There's not going to be any latest version + if image.Tag == "latest" { + return nil, nil + } + + currentVersion, err := oci.ParseVersion(image.Tag) + if err != nil { + return nil, fmt.Errorf("unsupported version: %w", err) + } else if currentVersion == nil { + return nil, fmt.Errorf("unsupported version") + } + + // The repository path is // + parts := strings.Split(image.Path, "/") + if len(parts) < 3 { + return nil, nil + } + + fullPath := strings.Join(parts[0:3], "/") + repositories, err := c.GetProjectContainerRepositories(ctx, fullPath) + if err != nil { + return nil, err + } + + var repository *ContainerRepository + for i := range repositories { + if repositories[i].Location == image.Name() { + r := repositories[i] + repository = &r + break + } + } + + if repository == nil { + return nil, nil + } + + tags, err := c.GetProjectContainerRepositoryTags(ctx, repository.ID) + if err != nil { + return nil, err + } + + // TODO: + // As we've sorted versions in released time, let's assume the first version + // that is higher than ours, is the latest version. Might not be true if the + // current version is 1.0.0, there have been a lot of nightlies or other types + // of tags, so that the page contains only fix 1.0.1, but in reality 2.0.0 was + // released a while ago and would be on the next page, would we be greedy. + // Look at any large image with LTS, such as postgres, node. + for _, tag := range tags { + if tag.Name == "" { + continue + } + + newVersion, err := oci.ParseVersion(tag.Name) + if err != nil || newVersion == nil { + continue + } + + if newVersion.IsCompatible(currentVersion) && newVersion.Compare(currentVersion) >= 0 { + image.Tag = tag.Name + + return ®istry.Image{ + Name: image, + Published: tag.PublishedAt, + Digest: tag.Digest, + }, nil + } + } + + return nil, nil +} + +type ContainerRepository struct { + ID string `json:"id"` + Location string `json:"location"` +} + +type ContainerRepositoryTag struct { + Digest string `json:"digest"` + Location string `json:"location"` + Name string `json:"name"` + CreatedAt time.Time `json:"createdAt"` + PublishedAt time.Time `json:"publishedAt"` + + // ... unused fields +} diff --git a/internal/registry/gitlab/client_test.go b/internal/registry/gitlab/client_test.go new file mode 100644 index 0000000..facd6e8 --- /dev/null +++ b/internal/registry/gitlab/client_test.go @@ -0,0 +1,59 @@ +package gitlab + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/AlexGustafsson/cupdate/internal/cachetest" + "github.com/AlexGustafsson/cupdate/internal/httputil" + "github.com/AlexGustafsson/cupdate/internal/registry/oci" + "github.com/stretchr/testify/require" +) + +func TestGetProjectContainerRepositories(t *testing.T) { + if testing.Short() { + t.Skip() + } + + client := &Client{ + Client: httputil.NewClient(cachetest.NewCache(t), 24*time.Hour), + } + + res, err := client.GetProjectContainerRepositories(context.TODO(), "arm-research/smarter/smarter-device-manager") + require.NoError(t, err) + + fmt.Printf("%+v\n", res) +} + +func TestGetProjectContainerRepositoryTags(t *testing.T) { + if testing.Short() { + t.Skip() + } + + client := &Client{ + Client: httputil.NewClient(cachetest.NewCache(t), 24*time.Hour), + } + + res, err := client.GetProjectContainerRepositoryTags(context.TODO(), "gid://gitlab/ContainerRepository/1080664") + require.NoError(t, err) + + fmt.Printf("%+v\n", res) +} + +func TestClientGetLatestVersion(t *testing.T) { + if testing.Short() { + t.Skip() + } + + client := &Client{ + Client: httputil.NewClient(cachetest.NewCache(t), 24*time.Hour), + } + ref, err := oci.ParseReference("registry.gitlab.com/arm-research/smarter/smarter-device-manager:v1.20.10") + require.NoError(t, err) + actual, err := client.GetLatestVersion(context.TODO(), ref) + require.NoError(t, err) + + fmt.Println(actual) +} diff --git a/internal/workflow/imageworkflow/getgitlablatestversion.go b/internal/workflow/imageworkflow/getgitlablatestversion.go new file mode 100644 index 0000000..1e91ace --- /dev/null +++ b/internal/workflow/imageworkflow/getgitlablatestversion.go @@ -0,0 +1,38 @@ +package imageworkflow + +import ( + "github.com/AlexGustafsson/cupdate/internal/httputil" + "github.com/AlexGustafsson/cupdate/internal/registry/gitlab" + "github.com/AlexGustafsson/cupdate/internal/registry/oci" + "github.com/AlexGustafsson/cupdate/internal/workflow" +) + +func GetGitLabLatestVersion() workflow.Step { + return workflow.Step{ + Name: "Get latest version from GitLab", + Main: func(ctx workflow.Context) (workflow.Command, error) { + httpClient, err := workflow.GetInput[*httputil.Client](ctx, "httpClient", true) + if err != nil { + return nil, err + } + + reference, err := workflow.GetInput[oci.Reference](ctx, "reference", true) + if err != nil { + return nil, err + } + + client := &gitlab.Client{Client: httpClient} + + image, err := client.GetLatestVersion(ctx, reference) + if err != nil { + return nil, err + } + + if image == nil { + return workflow.SetOutput("reference", (*oci.Reference)(nil)), nil + } + + return workflow.SetOutput("reference", &image.Name), nil + }, + } +} diff --git a/internal/workflow/imageworkflow/setupregistryclient.go b/internal/workflow/imageworkflow/setupregistryclient.go index 06f9d3e..ce3a161 100644 --- a/internal/workflow/imageworkflow/setupregistryclient.go +++ b/internal/workflow/imageworkflow/setupregistryclient.go @@ -6,6 +6,7 @@ import ( "github.com/AlexGustafsson/cupdate/internal/httputil" "github.com/AlexGustafsson/cupdate/internal/registry/docker" "github.com/AlexGustafsson/cupdate/internal/registry/ghcr" + "github.com/AlexGustafsson/cupdate/internal/registry/gitlab" "github.com/AlexGustafsson/cupdate/internal/registry/oci" "github.com/AlexGustafsson/cupdate/internal/workflow" ) @@ -41,6 +42,13 @@ func SetupRegistryClient() workflow.Step { Client: httpClient, }, } + case "registry.gitlab.com": + client = &oci.Client{ + Client: httpClient, + Authorizer: &gitlab.Client{ + Client: httpClient, + }, + } case "k8s.gcr.io", "quay.io", "registry.k8s.io": client = &oci.Client{ Client: httpClient, diff --git a/internal/workflow/imageworkflow/workflow.go b/internal/workflow/imageworkflow/workflow.go index 0ff5a7a..046fd9b 100644 --- a/internal/workflow/imageworkflow/workflow.go +++ b/internal/workflow/imageworkflow/workflow.go @@ -248,11 +248,40 @@ func New(httpClient *httputil.Client, data *Data) workflow.Workflow { }), }, }, + { + ID: "gitlab", + Name: "Get GitLab information", + DependsOn: []string{"oci"}, + // Only run for quay images + If: func(ctx workflow.Context) (bool, error) { + domain, err := workflow.GetValue[string](ctx, "job.oci.step.registry.domain") + if err != nil { + return false, err + } + + return domain == "registry.gitlab.com", nil + }, + Steps: []workflow.Step{ + GetGitLabLatestVersion(). + WithID("latest"). + With("reference", data.ImageReference). + With("httpClient", httpClient), + workflow.Run(func(ctx workflow.Context) (workflow.Command, error) { + reference, err := workflow.GetValue[*oci.Reference](ctx, "step.latest.reference") + if err != nil { + return nil, err + } + + data.LatestReference = reference + return nil, nil + }), + }, + }, { ID: "github", Name: "Get GitHub information", // Depend on whatever provides us with the latest image version - DependsOn: []string{"oci", "docker", "ghcr", "quay"}, + DependsOn: []string{"oci", "docker", "ghcr", "quay", "gitlab"}, // Only run for images with a reference to GitHub If: func(ctx workflow.Context) (bool, error) { if data.ImageReference.Domain == "ghcr.io" {