Add support for custom registries, registry auth

cmd/cupdate/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"log/slog"
 	"net/http"
@@ -86,6 +87,10 @@ type Config struct {
 		Target   string `env:"TARGET"`
 		Insecure bool   `env:"INSECURE"`
 	} `envPrefix:"OTEL_"`
+
+	Registry struct {
+		Secrets string `env:"SECRETS"`
+	} `envPrefix:"REGISTRY_"`
 }
 
 func main() {
@@ -118,6 +123,22 @@ func main() {
 
 	slog.Debug("Parsed config", slog.Any("config", config))
 
+	var dockerConfig *oci.DockerConfig
+	if config.Registry.Secrets != "" {
+		file, err := os.Open(config.Registry.Secrets)
+		if err != nil {
+			slog.Error("Failed to read registry secrets", slog.Any("error", err))
+			os.Exit(1)
+		}
+
+		err = json.NewDecoder(file).Decode(&dockerConfig)
+		file.Close()
+		if err != nil {
+			slog.Error("Failed to parse registry secrets", slog.Any("error", err))
+			os.Exit(1)
+		}
+	}
+
 	ctx, cancel := context.WithCancel(context.Background())
 
 	if config.OTEL.Target != "" {
@@ -249,7 +270,7 @@ func main() {
 		httpClient.UserAgent = config.HTTP.UserAgent
 		prometheus.DefaultRegisterer.MustRegister(httpClient)
 
-		worker := worker.New(httpClient, writeStore)
+		worker := worker.New(httpClient, writeStore, dockerConfig)
 		prometheus.DefaultRegisterer.MustRegister(worker)
 
 		gauge := prometheus.NewGauge(prometheus.GaugeOpts{

docs/config.md

@@ -29,3 +29,4 @@ done using environment variables.
 | `CUPDATE_DOCKER_INCLUDE_ALL_CONTAINERS`   | Whether or not to include containers in any state, not just running containers.                                                                         | `false`                         |
 | `CUPDATE_OTEL_TARGET`                     | Target URL to an Open Telemetry GRPC ingest endpoint.                                                                                                   | Required to use Open Telemetry. |
 | `CUPDATE_OTEL_INSECURE`                   | Disable client transport security for the Open Telemetry GRPC connection.                                                                               | `false`                         |
+| `CUPDATE_REGISTRY_SECRETS`                | Path to a JSON file containing registry secrets. See Docker's dockfig.json and Kubernete's `imagePullSecrets`.                                          | None                            |

internal/oci/auth.go

@@ -0,0 +1,141 @@
+package oci
+
+import (
+	"context"
+	"encoding/base64"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+type Authorizer interface {
+	AuthorizeOCIRequest(context.Context, Reference, *http.Request) error
+}
+
+type AuthorizeFunc func(context.Context, Reference, *http.Request) error
+
+func (f AuthorizeFunc) AuthorizeOCIRequest(ctx context.Context, image Reference, req *http.Request) error {
+	return f(ctx, image, req)
+}
+
+type AuthorizerToken string
+
+func (s AuthorizerToken) AuthorizeOCIRequest(ctx context.Context, image Reference, req *http.Request) error {
+	req.Header.Set("Authorization", "Bearer "+string(s))
+	return nil
+}
+
+// See: https://github.com/docker/docker/pull/12009.
+// See: https://kubernetes.io/docs/concepts/containers/images/#config-json.
+// See: https://github.com/kubernetes/kubernetes/blob/1a9feed0cd89f3299ddb6f5eaa5663496c59342c/pkg/credentialprovider/config.go.
+// See: https://github.com/kubernetes/kubernetes/blob/1a9feed0cd89f3299ddb6f5eaa5663496c59342c/pkg/credentialprovider/keyring_test.go#L26.
+type DockerConfig struct {
+	Auths       map[string]DockerConfigEntry `json:"auths"`
+	HttpHeaders map[string]string            `json:"HttpHeaders,omitempty"`
+}
+
+type DockerConfigEntry struct {
+	Username string `json:"username"`
+	Password string `json:"password"`
+	Email    string `json:"email"`
+}
+
+type DockerConfigAuthorizer struct {
+	Config   *DockerConfig
+	Fallback Authorizer
+}
+
+func (a *DockerConfigAuthorizer) AuthorizeOCIRequest(ctx context.Context, image Reference, req *http.Request) error {
+	if a.Config != nil {
+		for k, v := range a.Config.HttpHeaders {
+			req.Header.Set(k, v)
+		}
+
+		matchedPattern := a.Match(req.URL)
+		if matchedPattern != "" {
+			config := a.Config.Auths[matchedPattern]
+
+			credentials := base64.StdEncoding.EncodeToString([]byte(config.Username + ":" + config.Password))
+			req.Header.Set("Authorization", "Basic "+credentials)
+			return nil
+		}
+	}
+
+	if a.Fallback != nil {
+		return a.Fallback.AuthorizeOCIRequest(ctx, image, req)
+	}
+
+	return nil
+}
+
+func (a *DockerConfigAuthorizer) Match(url *url.URL) string {
+	// TODO: This code is not especially nice or efficient. For example, it parses
+	// the URLs every iteration
+	for pattern := range a.Config.Auths {
+		// Compare scheme, if set in pattern - otherwise allow any scheme
+		if !strings.HasPrefix(pattern, "https://") && !strings.HasPrefix(pattern, "http://") {
+			pattern = url.Scheme + "://" + pattern
+		}
+
+		u, err := url.Parse(pattern)
+		if err != nil {
+			continue
+		}
+
+		if url.Scheme != u.Scheme {
+			continue
+		}
+
+		// The Docker client matches either the image or hostname, where the API can
+		// have a /v2/ or /v1/ prefix
+		if strings.HasPrefix(u.Path, "/v2/") || strings.HasPrefix(u.Path, "/v1/") {
+			u.Path = u.Path[3:]
+		}
+
+		// If the pattern has a path specified, match it
+		if u.Path != "" && u.Path != "/" {
+			p := url.Path
+			if !strings.HasSuffix(p, "/") {
+				p += "/"
+			}
+
+			// Make sure paths in patterns match full segments
+			if !strings.HasSuffix(u.Path, "/") {
+				u.Path += "/"
+			}
+
+			if !strings.HasPrefix(p, u.Path) {
+				continue
+			}
+		}
+
+		urlParts := strings.Split(url.Host, ".")
+		patternParts := strings.Split(u.Host, ".")
+
+		// The pattern has more parts of its hostname than the URL
+		if len(urlParts) < len(patternParts) {
+			continue
+		}
+
+		matched := true
+		for i := 0; i < len(patternParts); i++ {
+			if strings.HasPrefix(patternParts[i], "*") {
+				if !strings.HasSuffix(urlParts[i], patternParts[i][1:]) {
+					matched = false
+					break
+				}
+			} else if strings.HasSuffix(patternParts[i], "*") {
+				// TODO:
+			} else if urlParts[i] != patternParts[i] {
+				matched = false
+				break
+			}
+		}
+
+		if matched {
+			return pattern
+		}
+	}
+
+	return ""
+}

internal/oci/auth_test.go

@@ -0,0 +1,160 @@
+package oci
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDockerConfigAuthorizer_MatchPattern(t *testing.T) {
+	testCases := []struct {
+		Pattern  string
+		URL      string
+		Expected bool
+	}{
+		// From: https://kubernetes.io/docs/concepts/containers/images/#config-json
+		{
+			Pattern:  "*.kubernetes.io",
+			URL:      "abc.kubernetes.io",
+			Expected: true,
+		},
+		{
+			Pattern:  "*.kubernetes.io",
+			URL:      "kubernetes.io",
+			Expected: false,
+		},
+		{
+			Pattern:  "*.*.kubernetes.io",
+			URL:      "abc.kubernetes.io",
+			Expected: false,
+		},
+		{
+			Pattern:  "*.*.kubernetes.io",
+			URL:      "abc.def.kubernetes.io",
+			Expected: true,
+		},
+		{
+			Pattern:  "prefix.*.io",
+			URL:      "prefix.kubernetes.io",
+			Expected: true,
+		},
+		{
+			Pattern:  "*-good.kubernetes.io",
+			URL:      "prefix-good.kubernetes.io",
+			Expected: true,
+		},
+		{
+			Pattern:  "my-registry.io/images",
+			URL:      "my-registry.io/images",
+			Expected: true,
+		},
+		{
+			Pattern:  "my-registry.io/images",
+			URL:      "my-registry.io/images/my-image",
+			Expected: true,
+		},
+		{
+			Pattern:  "my-registry.io/images",
+			URL:      "my-registry.io/images/another-image",
+			Expected: true,
+		},
+		{
+			Pattern:  "*.my-registry.io/images",
+			URL:      "sub.my-registry.io/images/my-image",
+			Expected: true,
+		},
+		{
+			Pattern:  "*.my-registry.io/images",
+			URL:      "a.sub.my-registry.io/images/my-image",
+			Expected: false,
+		},
+		{
+			Pattern:  "*.my-registry.io/images",
+			URL:      "a.b.sub.my-registry.io/images/my-image",
+			Expected: false,
+		},
+		{
+			Pattern:  "my-registry.io/images",
+			URL:      "a.sub.my-registry.io/images/my-image",
+			Expected: false,
+		},
+		{
+			Pattern:  "my-registry.io/images",
+			URL:      "a.b.sub.my-registry.io/images/my-image",
+			Expected: false,
+		},
+		// HTTP / HTTPS
+		{
+			Pattern:  "https://example.com",
+			URL:      "https://example.com/images",
+			Expected: true,
+		},
+		{
+			Pattern:  "https://example.com",
+			URL:      "http://example.com/images",
+			Expected: false,
+		},
+		{
+			Pattern:  "example.com",
+			URL:      "https://example.com/images",
+			Expected: true,
+		},
+		{
+			Pattern:  "example.com",
+			URL:      "http://example.com/images",
+			Expected: true,
+		},
+		// IP / port
+		{
+			Pattern:  "example.com:8080",
+			URL:      "example.com:8080/alpine",
+			Expected: true,
+		},
+		{
+			Pattern:  "192.168.1.100:8080",
+			URL:      "192.168.1.100:8080/alpine",
+			Expected: true,
+		},
+		{
+			Pattern:  "192.168.1.100",
+			URL:      "192.168.1.100/alpine",
+			Expected: true,
+		},
+		{
+			Pattern:  "example.com",
+			URL:      "example.com:8080/alpine",
+			Expected: false,
+		},
+		{
+			Pattern:  "192.168.1.100",
+			URL:      "192.168.1.100:8080/alpine",
+			Expected: false,
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(fmt.Sprintf("%s->%s", testCase.Pattern, testCase.URL), func(t *testing.T) {
+			a := &DockerConfigAuthorizer{
+				Config: &DockerConfig{
+					Auths: map[string]DockerConfigEntry{
+						testCase.Pattern: {},
+					},
+				},
+			}
+
+			if !strings.HasPrefix(testCase.URL, "https://") && !strings.HasPrefix(testCase.URL, "http://") {
+				testCase.URL = "https://" + testCase.URL
+			}
+
+			u, err := url.Parse(testCase.URL)
+			require.NoError(t, err)
+
+			actual := a.Match(u)
+			assert.Equal(t, testCase.Expected, actual != "")
+		})
+	}
+}

internal/oci/client.go

@@ -14,23 +14,6 @@ import (
 	"github.com/AlexGustafsson/cupdate/internal/httputil"
 )
 
-type Authorizer interface {
-	AuthorizeOCIRequest(context.Context, Reference, *http.Request) error
-}
-
-type AuthorizeFunc func(context.Context, Reference, *http.Request) error
-
-func (f AuthorizeFunc) AuthorizeOCIRequest(ctx context.Context, image Reference, req *http.Request) error {
-	return f(ctx, image, req)
-}
-
-type AuthorizerToken string
-
-func (s AuthorizerToken) AuthorizeOCIRequest(ctx context.Context, image Reference, req *http.Request) error {
-	req.Header.Set("Authorization", "Bearer "+string(s))
-	return nil
-}
-
 type Client struct {
 	Client     *httputil.Client
 	Authorizer Authorizer
@@ -50,6 +33,9 @@ func (c *Client) GetManifests(ctx context.Context, image Reference) ([]Manifest,
 		return nil, fmt.Errorf("unsupported reference type: must be tagged or digested")
 	}
 
+	// TODO: Change the URL AFTER running through authorizers as the docker fix is
+	// not apparent
+
 	// NOTE: It's rather unclear why we need to do this dance manually and why
 	// docker.io simply doesn't just redirect us
 	domain := strings.Replace(image.Domain, "docker.io", "registry-1.docker.io", 1)