From 64c31e874d2601bd584f5615dda56a5fb5528bfb Mon Sep 17 00:00:00 2001 From: Roald Brunell <87445607+OneFlyingBanana@users.noreply.github.com> Date: Tue, 7 May 2024 15:04:44 +0200 Subject: [PATCH] Fixed remote and file fetchers http-fetcher : Now using correct Harbor v2 API Local image list is based on tag list retrieved via API file-fetcher : Created JSON struct with name, digest and repository URL This data, with optional tag name, can be used to make docker pull commands using only url + digest --- .env | 2 + .gitignore | 1 + go.mod | 2 + go.sum | 4 + image-list/images.json | 24 ++-- internal/store/file-fetch.go | 144 +++++++++++++++++--- internal/store/http-fetch.go | 121 +++++++++++------ internal/store/in-memory-store.go | 212 +++++++++++++++++++++--------- main.go | 7 +- 9 files changed, 381 insertions(+), 136 deletions(-) create mode 100644 .env create mode 100644 .gitignore diff --git a/.env b/.env new file mode 100644 index 0000000..116dbab --- /dev/null +++ b/.env @@ -0,0 +1,2 @@ +HARBOR_USERNAME=admin +HARBOR_PASSWORD=Harbor12345 \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2eea525 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.env \ No newline at end of file diff --git a/go.mod b/go.mod index ee978f5..6c880ef 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,8 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.48.0 // indirect diff --git a/go.sum b/go.sum index ff3edeb..5b175a2 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/image-list/images.json b/image-list/images.json index 5d05159..a653a2a 100644 --- a/image-list/images.json +++ b/image-list/images.json @@ -1,19 +1,19 @@ { - "results": [ + "images": [ { - "name": "alpine:3.1" + "name": "album-server:album2.1.0", + "digest": "sha256:74b3ab50ecaf1765d8066221fbcbbbb2df5f2fe898f7d25c480db4fb6ac2effd", + "repositoryUrl": "https://demo.goharbor.io/v2/myproject" }, { - "name": "alpine:3.2" + "name": "album-server:latest", + "digest": "sha256:0d420809e7f6edca49a308d0233534991aebce5f54ebb3f97ee87dff8b62b701", + "repositoryUrl": "https://demo.goharbor.io/v2/myproject" }, { - "name": "alpine:3.3" - }, - { - "name": "ubuntu:latest" - }, - { - "name": "nginx:1.19" + "name": "album-server:zot-linux-arm64", + "digest": "sha256:671b2e3cdcf32a36335949845fa22883daaeced02082595e71df71ee64eae30f", + "repositoryUrl": "https://demo.goharbor.io/v2/myproject" } - ] -} \ No newline at end of file + ] +} diff --git a/internal/store/file-fetch.go b/internal/store/file-fetch.go index d707ade..9dfb12e 100644 --- a/internal/store/file-fetch.go +++ b/internal/store/file-fetch.go @@ -2,18 +2,29 @@ package store import ( "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" "os" "path/filepath" + "strings" ) type FileImageList struct { Path string } +type ImageData struct { + Images []struct { + Name string `json:"name"` + Digest string `json:"digest"` + RepositoryUrl string `json:"repositoryUrl"` + } `json:"images"` +} + +func (f *FileImageList) Type() string { + return "File" +} + func FileImageListFetcher(relativePath string) *FileImageList { // Get the current working directory dir, err := os.Getwd() @@ -31,22 +42,13 @@ func FileImageListFetcher(relativePath string) *FileImageList { } func (client *FileImageList) List(ctx context.Context) ([]Image, error) { - - fmt.Println("Reading from:", client.Path) - // Read the file data, err := os.ReadFile(client.Path) if err != nil { return nil, err } - // Define a struct to match the JSON structure - var imageData struct { - Results []struct { - Name string `json:"name"` - } `json:"results"` - } - + var imageData ImageData // Parse the JSON data err = json.Unmarshal(data, &imageData) if err != nil { @@ -54,27 +56,129 @@ func (client *FileImageList) List(ctx context.Context) ([]Image, error) { } // Convert the parsed data into a slice of Image structs - images := make([]Image, len(imageData.Results)) - for i, result := range imageData.Results { + images := make([]Image, len(imageData.Images)) + for i, image := range imageData.Images { images[i] = Image{ - Reference: result.Name, + Reference: image.Name, + Digest: image.Digest, } } fmt.Println("Fetched", len(images), "images :", images) + // Print the pull commands to test if stored data is correct and sufficient + fmt.Println("Pull commands for tests :") + client.GetPullCommands(ctx) return images, nil } -func (client *FileImageList) GetHash(ctx context.Context) (string, error) { +func (client *FileImageList) GetDigest(ctx context.Context, tag string) (string, error) { + // Read the file + data, err := os.ReadFile(client.Path) + if err != nil { + return "", err + } + + var imageData ImageData + // Parse the JSON data + err = json.Unmarshal(data, &imageData) + if err != nil { + return "", err + } + + if tag == "" { + return "", fmt.Errorf("tag cannot be empty") + } + + // Iterate over the images to find the one with the matching tag + for _, image := range imageData.Images { + if strings.Contains(image.Name, tag) { + return image.Digest, nil + } + } + // If no image with the matching tag is found, return an error + return "", fmt.Errorf("image with tag %s not found", tag) +} + +func (client *FileImageList) GetTag(ctx context.Context, digest string) (string, error) { // Read the file data, err := os.ReadFile(client.Path) if err != nil { return "", err } - // Hash and return the body - hash := sha256.Sum256(data) - hashString := hex.EncodeToString(hash[:]) + var imageData ImageData + // Parse the JSON data + err = json.Unmarshal(data, &imageData) + if err != nil { + return "", err + } + + if digest == "" { + return "", fmt.Errorf("digest cannot be empty") + } - return hashString, nil + // Iterate over the images to find the one with the matching digest + for _, image := range imageData.Images { + if image.Digest == digest { + return image.Name, nil + } + } + // If no image with the matching digest is found, return an error + return "", fmt.Errorf("image with digest %s not found", digest) + +} + +func (client *FileImageList) GetPullCommands(ctx context.Context) { + // Read the file + data, err := os.ReadFile(client.Path) + if err != nil { + fmt.Println("Error reading file:", err) + return + } + + var imageData ImageData + // Parse the JSON data + err = json.Unmarshal(data, &imageData) + if err != nil { + fmt.Println("Error parsing JSON:", err) + return + } + + // Iterate over the images to construct and print the pull command for each + for _, image := range imageData.Images { + harborUrl := strings.TrimPrefix(image.RepositoryUrl, "https://") + harborUrl = strings.Replace(harborUrl, ".io/v2", ".io", -1) + + // Extract the first part of the image name + parts := strings.Split(image.Name, ":") + if len(parts) > 0 { + // Append the first part of the split result to harborUrl + harborUrl += "/" + parts[0] + } + + pullCommand := fmt.Sprintf("docker pull %s@%s", harborUrl, image.Digest) + fmt.Println(pullCommand) + } +} + +func (client *FileImageList) ListDigests(ctx context.Context) ([]string, error) { + // Read the file + data, err := os.ReadFile(client.Path) + if err != nil { + return nil, err + } + + var imageData ImageData + // Parse the JSON data + err = json.Unmarshal(data, &imageData) + if err != nil { + return nil, err + } + + // Prepare a slice to store the digests + digests := make([]string, len(imageData.Images)) + for i, image := range imageData.Images { + digests[i] = image.Digest + } + return digests, nil } diff --git a/internal/store/http-fetch.go b/internal/store/http-fetch.go index df850d4..58aa1a8 100644 --- a/internal/store/http-fetch.go +++ b/internal/store/http-fetch.go @@ -2,85 +2,132 @@ package store import ( "context" - "crypto/sha256" - "encoding/hex" + "encoding/base64" "encoding/json" "fmt" "io" "net/http" - "path" + "os" + + v1 "github.com/opencontainers/image-spec/specs-go/v1" ) type RemoteImageList struct { BaseURL string } +type TagListResponse struct { + Name string `json:"name"` + Tags []string `json:"tags"` +} + func RemoteImageListFetcher(url string) *RemoteImageList { return &RemoteImageList{ BaseURL: url, } } +func (r *RemoteImageList) Type() string { + return "Remote" +} + func (client *RemoteImageList) List(ctx context.Context) ([]Image, error) { - // Extract the last segment of the BaseURL to use as the image name - lastSegment := path.Base(client.BaseURL) - fmt.Println("Last segment:", lastSegment) - resp, err := http.Get(client.BaseURL) + // Construct the URL for fetching tags + url := client.BaseURL + "/tags/list" + + // Encode credentials for Basic Authentication + username := os.Getenv("HARBOR_USERNAME") + password := os.Getenv("HARBOR_PASSWORD") + auth := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + + // Create a new HTTP request + req, err := http.NewRequest("GET", url, nil) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create request: %w", err) } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to fetch images: %s", resp.Status) + // Set the Authorization header + req.Header.Set("Authorization", "Basic "+auth) + + // Send the request + httpClient := &http.Client{} + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch tags: %w", err) } + defer resp.Body.Close() + // Read the response body body, err := io.ReadAll(resp.Body) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to read response body: %w", err) } - var data struct { - Results []struct { - Name string `json:"name"` - } `json:"results"` + // Unmarshal the JSON response + var tagListResponse TagListResponse + if err := json.Unmarshal(body, &tagListResponse); err != nil { + return nil, fmt.Errorf("failed to unmarshal JSON response: %w", err) } - err = json.Unmarshal(body, &data) - if err != nil { - return nil, err - } + // Prepare a slice to store the images + var images []Image - images := make([]Image, len(data.Results)) - for i, result := range data.Results { - images[i] = Image{ - Reference: fmt.Sprintf("%s:%s", lastSegment, result.Name), - } + // Iterate over the tags and construct the image references + for _, tag := range tagListResponse.Tags { + images = append(images, Image{ + Reference: fmt.Sprintf("%s:%s", tagListResponse.Name, tag), + }) } fmt.Println("Fetched", len(images), "images :", images) - return images, nil } -func (client *RemoteImageList) GetHash(ctx context.Context) (string, error) { - resp, err := http.Get(client.BaseURL) +func (client *RemoteImageList) GetDigest(ctx context.Context, tag string) (string, error) { + // Construct the URL for fetching the manifest + url := client.BaseURL + "/manifests/" + tag + + // Encode credentials for Basic Authentication + username := os.Getenv("HARBOR_USERNAME") + password := os.Getenv("HARBOR_PASSWORD") + auth := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + + // Create a new HTTP request + req, err := http.NewRequest("GET", url, nil) if err != nil { - return "", err + return "", fmt.Errorf("failed to create request: %w", err) } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("failed to fetch images: %s", resp.Status) + // Set the Authorization header + req.Header.Set("Authorization", "Basic "+auth) + + // Send the request + httpClient := &http.Client{} + resp, err := httpClient.Do(req) + if err != nil { + return "", fmt.Errorf("failed to fetch manifest: %w", err) } + defer resp.Body.Close() + // Read the response body body, err := io.ReadAll(resp.Body) if err != nil { - return "", err + return "", fmt.Errorf("failed to read response body: %w", err) + } + + // Unmarshal the JSON response + var manifestResponse v1.Manifest + if err := json.Unmarshal(body, &manifestResponse); err != nil { + return "", fmt.Errorf("failed to unmarshal JSON response: %w", err) } - // Hash and return the body - hash := sha256.Sum256(body) - hashString := hex.EncodeToString(hash[:]) + // Return the digest from the config section of the response + return string(manifestResponse.Config.Digest), nil +} + +func (client *RemoteImageList) GetTag(ctx context.Context, digest string) (string, error) { + return "", fmt.Errorf("not implemented yet") +} - return hashString, nil +func (client *RemoteImageList) ListDigests(ctx context.Context) ([]string, error) { + return nil, fmt.Errorf("not implemented yet") } diff --git a/internal/store/in-memory-store.go b/internal/store/in-memory-store.go index e25622a..2c0ed4c 100644 --- a/internal/store/in-memory-store.go +++ b/internal/store/in-memory-store.go @@ -2,121 +2,205 @@ package store import ( "context" - "errors" "fmt" + "strings" ) type Image struct { + Digest string Reference string } type inMemoryStore struct { - images map[string][]Image + images map[string]string fetcher ImageFetcher } type Storer interface { List(ctx context.Context) ([]Image, error) - Add(ctx context.Context, hash string, imageList []Image) error - Remove(ctx context.Context, hash string) error + Add(ctx context.Context, digest string, image string) error + Remove(ctx context.Context, digest string, image string) error } type ImageFetcher interface { List(ctx context.Context) ([]Image, error) - GetHash(ctx context.Context) (string, error) + ListDigests(ctx context.Context) ([]string, error) + GetDigest(ctx context.Context, tag string) (string, error) + GetTag(ctx context.Context, digest string) (string, error) + Type() string } func NewInMemoryStore(fetcher ImageFetcher) Storer { return &inMemoryStore{ - images: make(map[string][]Image), + images: make(map[string]string), fetcher: fetcher, } } func (s *inMemoryStore) List(ctx context.Context) ([]Image, error) { - // Check if the local store is empty - if len(s.images) == 0 { - fmt.Println("Local store is empty. Fetching images from the remote source...") - // Fetch images from the remote source - imageList, err := s.fetcher.List(ctx) - if err != nil { - return nil, err - } + var imageList []Image + + // Fetch images from the file/remote source + imageList, err := s.fetcher.List(ctx) + if err != nil { + return nil, err + } - // Fetch the remote hash - remoteHash, err := s.fetcher.GetHash(ctx) - if err != nil { - return nil, err + // Trim the imageList elements to remove the project name from the image reference + for i, img := range imageList { + parts := strings.Split(img.Reference, "/") + if len(parts) > 1 { + // Take the second part as the new Reference + imageList[i].Reference = parts[1] } + } - // Add the fetched images and hash to the local store - fmt.Println("Adding fetched images and hash to the local store...") - s.Add(ctx, remoteHash, imageList) - } else { - fmt.Println("Checking for changes in remote source...") + // Since remote fetcher is based on tags retrieved via API and file fetcher must be able to handle a images without tags, they need to be handled separately + // File fetcher will work based on digests instead of tags + switch s.fetcher.Type() { + case "File": - // Fetch the remote hash - remoteHash, err := s.fetcher.GetHash(ctx) - if err != nil { - return nil, err + // Iterate over imageList and call GetTag for each digest + for _, img := range imageList { + // Get the tag for the digest + tag, err := s.fetcher.GetTag(ctx, img.Digest) + if err != nil { + return nil, err + } + // Check if the digest exists and matches the image reference + // If the digest exists and does not match the image, update the store + // If the digest does not exist, add it to the store + s.checkDigestAndImage(img.Digest, tag) } - // Fetch the local hash - localHash, err := s.GetLocalHash(ctx) - if err != nil { - return nil, err + // Create a map to keep track of the digests found in imageList + digestMap := make(map[string]bool) + + // Iterate over imageList and add each digest to the map + for _, img := range imageList { + digestMap[img.Digest] = true } - // If the local and remote hashes are not equal, clear the store and add incoming images - if !areImagesEqual(localHash, remoteHash) { - fmt.Println("WARNING : Local and remote hashes are not equal. Updating the local store with new images...") - fmt.Println("Old Store :", s.images) + // Iterate over the stored list and delete any digest and its image that aren't found in imageList + for digest := range s.images { + if _, exists := digestMap[digest]; !exists { + // The digest does not exist in imageList, so remove it from the store + s.Remove(ctx, digest, s.images[digest]) + } + } - imageList, err := s.fetcher.List(ctx) + case "Remote": + // iterate over imageList and call GetDigest for each tag + for _, img := range imageList { + // Split the image reference to get the tag + tagParts := strings.Split(img.Reference, ":") + // Check if there is a tag part, min length is 1 char + if len(tagParts) < 2 { + fmt.Println("No tag part found in the image reference") + } + // Use the last part as the tag + tag := tagParts[len(tagParts)-1] + // Get the digest for the tag + digest, err := s.fetcher.GetDigest(ctx, tag) + fmt.Printf("Digest for tag \"%s\" is %s\n", tag, digest) if err != nil { return nil, err } - s.Remove(ctx, "") - s.Add(ctx, remoteHash, imageList) - fmt.Println("New Store :", s.images) - } else { - fmt.Println("Local and remote hashes are equal. No update needed.") + + // Check if the image exists and matches the digest + // If the image exists and does not match the digest, update the store + // If the image does not exist, add it to the store + s.checkImageAndDigest(digest, img.Reference) + + } + + // Create imageMap filled with all images from imageList + imageMap := make(map[string]bool) + for _, img := range imageList { + imageMap[img.Reference] = true } + + // Iterate over in memory store and remove any image that is not found in imageMap + for digest, image := range s.images { + if _, exists := imageMap[image]; !exists { + // The image does not exist in imageList, so remove it from the store + s.Remove(ctx, digest, image) + } + } + } - var allImages []Image - for _, images := range s.images { - allImages = append(allImages, images...) + + // Print out the entire store + fmt.Println("Current store:") + for digest, imageRef := range s.images { + fmt.Printf("Digest: %s, Image: %s\n", digest, imageRef) } - return allImages, nil + + return imageList, nil + } -func (s *inMemoryStore) Add(ctx context.Context, hash string, imageList []Image) error { - s.images[hash] = imageList +func (s *inMemoryStore) Add(ctx context.Context, digest string, image string) error { + // Add the image and its digest to the store + s.images[digest] = image + fmt.Printf("Added image: %s, digest: %s\n", image, digest) return nil } -// Remove images from the store based on the provided hash -// If no hash is provided, it clears the entire store -func (s *inMemoryStore) Remove(ctx context.Context, hash string) error { - if hash == "" { - s.images = make(map[string][]Image) - fmt.Println("Store cleared.") - } else { - fmt.Println("Removing images with hash:", hash) - delete(s.images, hash) - } +func (s *inMemoryStore) Remove(ctx context.Context, digest string, image string) error { + // Remove the image and its digest from the store + delete(s.images, digest) + fmt.Printf("Removed image: %s, digest: %s\n", image, digest) return nil } -func areImagesEqual(localHash string, remoteHash string) bool { - return localHash == remoteHash +// checkImageAndDigest checks if the image exists in the store and if the digest matches the image reference +func (s *inMemoryStore) checkImageAndDigest(digest string, image string) bool { + // Check if the received image exists in the store + for _, existingImage := range s.images { + if existingImage == image { + // Image exists, now check if the digest matches + existingDigest, exists := s.images[digest] + if exists && existingDigest == image { + // Digest exists and matches the current image reference + fmt.Printf("Digest for image %s exists in the store and matches\n", image) + return true + } else { + // Digest exists but does not match the current image reference + fmt.Printf("Digest for image %s exists in the store but does not match the current image reference\n", image) + s.Remove(context.Background(), existingDigest, existingImage) + s.Add(context.Background(), digest, image) + return false + } + } + } + + // If the image doesn't exist in the store, add it + fmt.Printf("Image \"%s\" does not exist in the store\n", image) + s.Add(context.Background(), digest, image) + return false } -func (s *inMemoryStore) GetLocalHash(ctx context.Context) (string, error) { - for hash, images := range s.images { - if len(images) > 0 { - return hash, nil +// checkDigestAndImage checks if the digest exists in the store and if the corresponding image matches its digest +func (s *inMemoryStore) checkDigestAndImage(digest string, image string) bool { + // Check if the received digest exists in the store + for existingDigest, existingImage := range s.images { + if existingDigest == digest { + // Digest exists, now check if the corresponding image matches + if existingImage == image { + // Image exists and the corresponding digest matches + return true + } else { + // Image exists but the corresponding digest does not match + fmt.Printf("Digest %s exists in the store but does not match the image %s\n", digest, image) + s.Remove(context.Background(), existingDigest, existingImage) + s.Add(context.Background(), digest, image) + return false + } } } - return "", errors.New("no hash found in the local store") + + // If the digest doesn't exist in the store, add it + s.Add(context.Background(), digest, image) + return false } diff --git a/main.go b/main.go index 446afdf..d985f14 100644 --- a/main.go +++ b/main.go @@ -67,11 +67,12 @@ func run() error { var fetcher store.ImageFetcher for { - fmt.Print("Enter the source (URL or relative file path): ") + fmt.Print("Enter the source (Repository URL or relative file path): ") // For testing purposes : - // https://registry.hub.docker.com/v2/repositories/alpine - // /image-list/images.json + // https://demo.goharbor.io/v2// + // https://demo.goharbor.io/v2/myproject/album-server + // Local file path : /image-list/images.json reader := bufio.NewReader(os.Stdin) input, err := reader.ReadString('\n')