From 96142e03b64eae9fc786ef9ddc0c2a341a146e03 Mon Sep 17 00:00:00 2001 From: John Reese Date: Thu, 4 Jun 2020 16:41:08 -0400 Subject: [PATCH] Add mirror flag to specify mirror location --- README.md | 38 ++++++++-- go.mod | 4 +- go.sum | 39 ++++++---- internal/commands/check.go | 42 +++++++---- internal/commands/list.go | 148 ++++++++++++++++++++++++------------- internal/commands/sync.go | 80 +++++++------------- 6 files changed, 210 insertions(+), 141 deletions(-) diff --git a/README.md b/README.md index e5ed992..fd6f08c 100644 --- a/README.md +++ b/README.md @@ -15,26 +15,54 @@ Feedback, feature requests, and contributions are welcome! ## Usage +The `--mirror` flag tells `imagesync` the host, and optionally a repository path, of the mirror. + +For example, given an `images.txt` of the following: + +```text +foourl.com/bar/nginxdemos/hello:0.2 +foourl.com/bar/alpine:3.11 +foourl.com/bar/coreos/prometheus-operator:v0.39.0 +``` + +Running the command: + +```console +$ imagesync sync images.txt --mirror foourl.com/bar +``` + +Would remove `foourl.com/bar` from the images listed above and pull from `docker.io` behind the scenes. + +**NOTE:** Given that images are not always sourced from docker.io, some assumptions are made. Most notably, the `coreos/` repository will pull from `quay.io` + +This tool assumes that your images use the exact same repository path after the prefix. i.e. Assuming a `--mirror` value of `foourl.com/bar`: + +- `foourl.com/bar/nginxdemos/hello:0.2` will be sourced from `docker.io/nginxdemos/hello:0.2` + +- `foourl.com/bar/coreos/prometheus-operator:v0.39.0` will be sourced from `quay.io/coreos/prometheus-operator:v0.39.0` + +If no `--mirror` flag is used, the images will be read as is. + ### Listing images Print all image references found in a given folder or file. The `--output` flag writes the list to a file. ```console -$ imagesync list manifests --output images.txt +$ imagesync list manifestsPath --output images.txt ``` ### Checking images -Find all image references and checks if there are any newly published tags for the images. +Checks if there are any newly published tags for the images. ```console -$ imagesync check manifests +$ imagesync check images.txt --mirror foourl.com/bar/repo ``` ### Syncing images -Sync all images from source resources to destination resources. The source resources are needed to know which container registry the images are hosted at. +Sync all images in the image list to the mirror repository. ```console -$ imagesync sync manifests kustomizedmanifests +$ imagesync sync images.txt --mirror foourl.com/bar/repo ``` \ No newline at end of file diff --git a/go.mod b/go.mod index 14da618..c43d104 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,9 @@ go 1.14 require ( cuelang.org/go v0.1.2 - github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect github.com/Microsoft/hcsshim v0.8.9 // indirect - github.com/containerd/containerd v1.3.2 github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb // indirect - github.com/coreos/prometheus-operator v0.38.1 + github.com/coreos/prometheus-operator v0.39.0 github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492 github.com/docker/distribution v0.0.0-20191008191456-ae2e973db936 // indirect github.com/docker/docker v0.7.3-0.20190826074503-38ab9da00309 diff --git a/go.sum b/go.sum index 11c1a59..41e3406 100644 --- a/go.sum +++ b/go.sum @@ -98,7 +98,8 @@ github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= -github.com/brancz/gojsontoyaml v0.0.0-20190425155809-e8bd32d46b3d/go.mod h1:IyUJYN1gvWjtLF5ZuygmxbnsAyP3aJS6cHzIuZY50B0= +github.com/brancz/gojsontoyaml v0.0.0-20191212081931-bf2969bbd742/go.mod h1:IyUJYN1gvWjtLF5ZuygmxbnsAyP3aJS6cHzIuZY50B0= +github.com/brancz/kube-rbac-proxy v0.5.0/go.mod h1:cL2VjiIFGS90Cjh5ZZ8+It6tMcBt8rwvuw2J6Mamnl0= github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd h1:rFt+Y/IK1aEZkEHchZRSq9OQbsSzIT/OrI8YFFmRIng= github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= @@ -144,8 +145,8 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/prometheus-operator v0.38.1 h1:43h170Ej5EDGjoaRIuJY9OmjXy/MuGOakKfqrq5h0iU= -github.com/coreos/prometheus-operator v0.38.1/go.mod h1:xZC7/TgeC0/mBaJk+1H9dbHaiEvLYHgX6Mi1h40UPh8= +github.com/coreos/prometheus-operator v0.39.0 h1:sXbjqxHhTdfhdbG/Fpls3nArXZZaC66JTpzebZK2ZOc= +github.com/coreos/prometheus-operator v0.39.0/go.mod h1:erio69w1R/aC14D5nfvAXSlE8FT8jt2Hnavc50Dp33A= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= @@ -199,6 +200,7 @@ github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/proto v1.6.15/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -216,7 +218,6 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/go-bindata/go-bindata v3.1.2+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo= github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= @@ -456,7 +457,7 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jsonnet-bundler/jsonnet-bundler v0.2.0/go.mod h1:/by7P/OoohkI3q4CgSFqcoFsVY+IaNbzOVDknEsKDeU= +github.com/jsonnet-bundler/jsonnet-bundler v0.3.1/go.mod h1:/by7P/OoohkI3q4CgSFqcoFsVY+IaNbzOVDknEsKDeU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -627,6 +628,7 @@ github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1: github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= @@ -800,6 +802,7 @@ golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -921,6 +924,7 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -962,6 +966,7 @@ golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191111182352-50fa39b762bc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200113154838-30cae5f2fb06/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1009,16 +1014,18 @@ google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= @@ -1059,30 +1066,34 @@ k8s.io/api v0.0.0-20190620084959-7cf5895f2711/go.mod h1:TBhBqb1AWbBQbW3XRusr7n7E k8s.io/api v0.0.0-20190813020757-36bff7324fb7/go.mod h1:3Iy+myeAORNCLgjd/Xu9ebwN7Vh59Bw0vh9jhoX+V58= k8s.io/api v0.0.0-20190918155943-95b840bb6a1f/go.mod h1:uWuOHnjmNrtQomJrvEBg0c0HRNyQ+8KTEERVsK0PW48= k8s.io/api v0.0.0-20191115095533-47f6de673b26/go.mod h1:iA/8arsvelvo4IDqIhX4IbjTEKBGgvsf2OraTuRtLFU= -k8s.io/api v0.17.3/go.mod h1:YZ0OTkuw7ipbe305fMpIdf3GLXZKRigjtZaV5gzC2J0= +k8s.io/api v0.0.0-20191122220107-b5267f2975e0/go.mod h1:vYpRfxYkMrmPPSesoHEkGNHxNKTk96REAwqm/inQbs0= k8s.io/api v0.18.2 h1:wG5g5ZmSVgm5B+eHMIbI9EGATS2L8Z72rda19RIEgY8= k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78= k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783/go.mod h1:xvae1SZB3E17UpV59AWc271W/Ph25N+bjPyR63X6tPY= -k8s.io/apiextensions-apiserver v0.17.3/go.mod h1:CJbCyMfkKftAd/X/V6OTHYhVn7zXnDdnkUjS1h0GTeY= +k8s.io/apiextensions-apiserver v0.18.2/go.mod h1:q3faSnRGmYimiocj6cHQ1I3WpLqmDgJFlKL37fC4ZvY= k8s.io/apimachinery v0.0.0-20190612205821-1799e75a0719/go.mod h1:I4A+glKBHiTgiEjQiCCQfCAIcIMFGt291SmsvcrFzJA= k8s.io/apimachinery v0.0.0-20190809020650-423f5d784010/go.mod h1:Waf/xTS2FGRrgXCkO5FP3XxTOWh0qLf2QhL1qFZZ/R8= k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655/go.mod h1:nL6pwRT8NgfF8TT68DBI8uEePRt89cSvoXUVqbkWHq4= k8s.io/apimachinery v0.0.0-20191115015347-3c7067801da2/go.mod h1:dXFS2zaQR8fyzuvRdJDHw2Aerij/yVGJSre0bZQSVJA= -k8s.io/apimachinery v0.17.3/go.mod h1:gxLnyZcGNdZTCLnq3fgzyg2A5BVCHTNDFrw8AmuJ+0g= +k8s.io/apimachinery v0.0.0-20191121175448-79c2a76c473a/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= k8s.io/apimachinery v0.18.2 h1:44CmtbmkzVDAhCpRVSiP2R5PPrC2RtlIv/MoB8xpdRA= k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad/go.mod h1:XPCXEwhjaFN29a8NldXA901ElnKeKLrLtREO9ZhFyhg= -k8s.io/apiserver v0.17.3/go.mod h1:iJtsPpu1ZpEnHaNawpSV0nYTGBhhX2dUlnn7/QS7QiY= +k8s.io/apiserver v0.0.0-20191122221311-9d521947b1e1/go.mod h1:RbsZY5zzBIWnz4KbctZsTVjwIuOpTp4Z8oCgFHN4kZQ= +k8s.io/apiserver v0.18.2/go.mod h1:Xbh066NqrZO8cbsoenCwyDJ1OSi8Ag8I2lezeHxzwzw= k8s.io/client-go v0.0.0-20190620085101-78d2af792bab/go.mod h1:E95RaSlHr79aHaX0aGSwcPNfygDiPKOVXdmivCIZT0k= k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90/go.mod h1:J69/JveO6XESwVgG53q3Uz5OSfgsv4uxpScmmyYOOlk= -k8s.io/client-go v0.17.3/go.mod h1:cLXlTMtWHkuK4tD360KpWz2gG2KtdWEr/OT02i3emRQ= +k8s.io/client-go v0.0.0-20191122220542-ed16ecbdf3a0/go.mod h1:tyxNgOmR/Xi39HrlQ/9LQgiHJgBvmY7gp95o5GpBA4o= +k8s.io/client-go v0.18.2/go.mod h1:Xcm5wVGXX9HAA2JJ2sSBUn3tCJ+4SVlCbl2MNNv+CIU= k8s.io/client-go v12.0.0+incompatible/go.mod h1:E95RaSlHr79aHaX0aGSwcPNfygDiPKOVXdmivCIZT0k= k8s.io/code-generator v0.0.0-20190912054826-cd179ad6a269/go.mod h1:V5BD6M4CyaN5m+VthcclXWsVcT1Hu+glwa1bi3MIsyE= -k8s.io/code-generator v0.17.3/go.mod h1:l8BLVwASXQZTo2xamW5mQNFCe1XPiAesVq7Y1t7PiQQ= +k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= k8s.io/component-base v0.0.0-20190918160511-547f6c5d7090/go.mod h1:933PBGtQFJky3TEwYx4aEPZ4IxqhWh3R6DCmzqIn1hA= -k8s.io/component-base v0.17.3/go.mod h1:GeQf4BrgelWm64PXkIXiPh/XS0hnO42d9gx9BtbZRp8= +k8s.io/component-base v0.0.0-20191122220729-2684fb322cb9/go.mod h1:NFuUusy/X4Tk21m21tcNUihnmp4OI7lXU7/xA+rYXkc= +k8s.io/component-base v0.18.2/go.mod h1:kqLlMuhJNHQ9lz8Z7V5bxUUtjFZnrypArGl58gmDfUM= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= @@ -1098,6 +1109,7 @@ k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3 k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20191114200735-6ca3b61696b6/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= @@ -1107,6 +1119,7 @@ mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIa mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= sigs.k8s.io/controller-tools v0.2.4/go.mod h1:m/ztfQNocGYBgTTCmFdnK94uVvgxeZeE3LtJvd/jIzA= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= diff --git a/internal/commands/check.go b/internal/commands/check.go index 08f4c48..2720eb8 100644 --- a/internal/commands/check.go +++ b/internal/commands/check.go @@ -10,6 +10,7 @@ import ( "github.com/hashicorp/go-version" "github.com/heroku/docker-registry-client/registry" "github.com/spf13/cobra" + "github.com/spf13/viper" ) // NewCheckCommand creates a new list command @@ -17,9 +18,12 @@ func NewCheckCommand() *cobra.Command { cmd := cobra.Command{ Use: "check", Short: "Check for newer images in the remote registry", - Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + if err := viper.BindPFlag("mirror", cmd.Flags().Lookup("mirror")); err != nil { + return fmt.Errorf("bind flag: %w", err) + } + if err := runCheckCommand(args); err != nil { return fmt.Errorf("check: %w", err) } @@ -28,6 +32,8 @@ func NewCheckCommand() *cobra.Command { }, } + cmd.Flags().StringP("mirror", "m", "", "mirror prefix") + return &cmd } @@ -37,11 +43,17 @@ func runCheckCommand(args []string) error { return fmt.Errorf("get working dir: %w", err) } - sourcePath := filepath.Join(workingDir, args[0]) + imageListPath := filepath.Join(workingDir, args[0]) - sourceImages, err := GetImagesInPath(sourcePath) + mirrorImages, err := GetImagesFromFile(imageListPath) if err != nil { - return fmt.Errorf("get before list: %w", err) + return fmt.Errorf("get images from file: %w", err) + } + + var originalImages []DockerImage + for _, mirrorImage := range mirrorImages { + originalImage := getOriginalImage(mirrorImage, viper.GetString("mirror")) + originalImages = append(originalImages, originalImage) } api, err := registry.New("https://registry-1.docker.io/", "", "") @@ -50,25 +62,25 @@ func runCheckCommand(args []string) error { } api.Logf = registry.Quiet - for _, sourceImage := range sourceImages { + for _, originalImage := range originalImages { var newerVersions []string - if sourceImage.Host == "quay.io" { - fmt.Printf("Image %s has quay.io address, skipping...\n", sourceImage) + if originalImage.Host == "quay.io" { + fmt.Printf("Image %s has quay.io address, skipping...\n", originalImage) continue } - sourceTag, err := version.NewVersion(sourceImage.Version) + sourceTag, err := version.NewVersion(originalImage.Version) if err != nil { - fmt.Printf("skipping %v: %v\n", sourceImage, err) + fmt.Printf("skipping %v: %v\n", originalImage, err) continue } var searchRepo string - if !strings.Contains(sourceImage.Repository, "/") { - searchRepo = "library/" + sourceImage.Repository + if !strings.Contains(originalImage.Repository, "/") { + searchRepo = "library/" + originalImage.Repository } else { - searchRepo = sourceImage.Repository + searchRepo = originalImage.Repository } allTags, err := api.Tags(searchRepo) @@ -80,7 +92,7 @@ func runCheckCommand(args []string) error { for _, tag := range allTags { upstreamTag, err := version.NewVersion(tag) if err != nil { - fmt.Printf("skipping %v: %v\n", sourceImage, err) + fmt.Printf("skipping %v: %v\n", originalImage, err) continue } @@ -90,9 +102,9 @@ func runCheckCommand(args []string) error { } if len(newerVersions) > 0 { - fmt.Printf("New versions for %v found: %v\n", sourceImage, newerVersions) + fmt.Printf("New versions for %v found: %v\n", originalImage, newerVersions) } else { - fmt.Printf("%v is up to date!\n", sourceImage) + fmt.Printf("%v is up to date!\n", originalImage) } } diff --git a/internal/commands/list.go b/internal/commands/list.go index a48e9c2..d89fa3a 100644 --- a/internal/commands/list.go +++ b/internal/commands/list.go @@ -1,6 +1,7 @@ package commands import ( + "bufio" "bytes" "fmt" "io/ioutil" @@ -55,36 +56,38 @@ func NewListCommand() *cobra.Command { }, } - cmd.Flags().StringP("output", "o", "", fmt.Sprintf("output path for the image list")) + cmd.Flags().StringP("output", "o", "", "output path for the image list") return &cmd } -func runListCommand(args []string) error { - workingDir, err := os.Getwd() +// GetImagesFromFile reads in a text file of images and returns +// them as DockerImage types +func GetImagesFromFile(filePath string) ([]DockerImage, error) { + file, err := os.Open(filePath) if err != nil { - return fmt.Errorf("get working dir: %w", err) + return nil, fmt.Errorf("opening file: %w", err) } + defer file.Close() - listPath := filepath.Join(workingDir, args[0]) - images, err := GetImagesInPath(listPath) - if err != nil { - return fmt.Errorf("get images from path: %w", err) + scanner := bufio.NewScanner(file) + var images []string + for scanner.Scan() { + images = append(images, scanner.Text()) } - if viper.GetString("output") != "" { - outputFile := filepath.Join(workingDir, viper.GetString("output")) - writeListToFile(images, outputFile) - } else { - for _, image := range images { - fmt.Println(image) - } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("scanning file: %w", err) } - return nil + marshaledImages := marshalImages(images) + + return marshaledImages, nil } -func GetImagesInPath(path string) ([]DockerImage, error) { +// GetImagesFromYaml finds all yaml files in a given path and returns +// all of the images found in the manifests +func GetImagesFromYaml(path string) ([]DockerImage, error) { files, err := getYamlFiles(path) if err != nil { return nil, fmt.Errorf("get yaml files: %w", err) @@ -95,18 +98,8 @@ func GetImagesInPath(path string) ([]DockerImage, error) { return nil, fmt.Errorf("split yaml files: %w", err) } - type BaseSpec struct { - Template corev1.PodTemplateSpec `json:"template" protobuf:"bytes,3,opt,name=template"` - } - - type BaseType struct { - Spec BaseSpec `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"` - } - var imageList []string for _, yamlFile := range yamlFiles { - var contents BaseType - var typeMeta metav1.TypeMeta if err := yaml.Unmarshal(yamlFile, &typeMeta); err != nil { continue @@ -134,26 +127,84 @@ func GetImagesInPath(path string) ([]DockerImage, error) { continue } - if err := yaml.Unmarshal(yamlFile, &contents); err != nil { - continue + type BaseSpec struct { + Template corev1.PodTemplateSpec `json:"template" protobuf:"bytes,3,opt,name=template"` } - if len(contents.Spec.Template.Spec.InitContainers) > 0 { - imageList = append(imageList, getImagesFromContainers(contents.Spec.Template.Spec.InitContainers)...) + type BaseType struct { + Spec BaseSpec `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"` } - if len(contents.Spec.Template.Spec.Containers) > 0 { - imageList = append(imageList, getImagesFromContainers(contents.Spec.Template.Spec.Containers)...) + var contents BaseType + if err := yaml.Unmarshal(yamlFile, &contents); err != nil { + continue } + + imageList = append(imageList, getImagesFromContainers(contents.Spec.Template.Spec.InitContainers)...) + imageList = append(imageList, getImagesFromContainers(contents.Spec.Template.Spec.Containers)...) } dedupedImageList := dedupeImages(imageList) - marshaledImages := marshalImages(dedupedImageList) return marshaledImages, nil } +func runListCommand(args []string) error { + workingDir, err := os.Getwd() + if err != nil { + return fmt.Errorf("get working dir: %w", err) + } + + listPath := filepath.Join(workingDir, args[0]) + images, err := GetImagesFromYaml(listPath) + if err != nil { + return fmt.Errorf("get images from path: %w", err) + } + + if viper.GetString("output") != "" { + outputFile := filepath.Join(workingDir, viper.GetString("output")) + if err := writeListToFile(images, outputFile); err != nil { + return fmt.Errorf("writing list to file: %w", err) + } + } else { + for _, image := range images { + fmt.Println(image) + } + } + + return nil +} + +func getOriginalImage(dockerImage DockerImage, mirrorPrefix string) DockerImage { + quayMappings := []string{ + "kubernetes-ingress-controller", + "coreos", + } + + originalHost := "docker.io" + for _, quayMapping := range quayMappings { + if strings.Contains(dockerImage.Repository, quayMapping) { + originalHost = "quay.io" + } + } + + var originalRepository string + if strings.Contains(mirrorPrefix, "/") { + mirrorRepository := strings.SplitN(mirrorPrefix, "/", 2)[1] + originalRepository = strings.Replace(dockerImage.Repository, mirrorRepository+"/", "", 1) + } + + originalImage := DockerImage{ + Host: originalHost, + Repository: originalRepository, + Name: dockerImage.Name, + Version: dockerImage.Version, + } + + return originalImage +} + func marshalImages(images []string) []DockerImage { var marshaledImages []DockerImage for _, image := range images { @@ -196,7 +247,9 @@ func writeListToFile(images []DockerImage, outputFile string) error { defer f.Close() for _, value := range images { - fmt.Fprintln(f, value) + if _, err := fmt.Fprintln(f, value); err != nil { + return fmt.Errorf("writing image to file: %w", err) + } } return nil @@ -207,23 +260,14 @@ func getImagesFromContainers(containers []corev1.Container) []string { for _, container := range containers { images = append(images, container.Image) - argImages := getImagesFromContainerArgs(container.Args) - - images = append(images, argImages...) - } - - return images -} + for _, arg := range container.Args { + if !strings.Contains(arg, ":") || strings.Contains(arg, "=:") { + continue + } -func getImagesFromContainerArgs(args []string) []string { - var images []string - for _, arg := range args { - if !strings.Contains(arg, ":") || strings.Contains(arg, "=:") { - continue + argTokens := strings.Split(arg, "=") + images = append(images, argTokens[1]) } - - argTokens := strings.Split(arg, "=") - images = append(images, argTokens[1]) } return images @@ -269,9 +313,7 @@ func splitYamlFiles(files []string) ([][]byte, error) { individualYamlFiles := doSplit(fileContent) - for _, yamlFile := range individualYamlFiles { - yamlFiles = append(yamlFiles, yamlFile) - } + yamlFiles = append(yamlFiles, individualYamlFiles...) } return yamlFiles, nil diff --git a/internal/commands/sync.go b/internal/commands/sync.go index 8619ac3..dd94b9d 100644 --- a/internal/commands/sync.go +++ b/internal/commands/sync.go @@ -19,8 +19,8 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/client" "github.com/docker/docker/errdefs" - "github.com/hashicorp/go-version" "github.com/spf13/cobra" + "github.com/spf13/viper" "k8s.io/apimachinery/pkg/util/wait" ) @@ -37,9 +37,13 @@ func NewSyncCommand(logger *log.Logger) *cobra.Command { cmd := cobra.Command{ Use: "sync", Short: "Sync the images found in the repository to another registry", - Args: cobra.ExactArgs(2), + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + if err := viper.BindPFlag("mirror", cmd.Flags().Lookup("mirror")); err != nil { + return fmt.Errorf("bind flag: %w", err) + } + if err := runSyncCommand(logger, args); err != nil { return fmt.Errorf("sync: %w", err) } @@ -50,6 +54,8 @@ func NewSyncCommand(logger *log.Logger) *cobra.Command { }, } + cmd.Flags().StringP("mirror", "m", "", "mirror prefix") + return &cmd } @@ -66,57 +72,49 @@ func runSyncCommand(logger *log.Logger, args []string) error { return fmt.Errorf("get working dir: %w", err) } - sourcePath := filepath.Join(workingDir, args[0]) - destinationPath := filepath.Join(workingDir, args[1]) - - sourceList, err := GetImagesInPath(sourcePath) - if err != nil { - return fmt.Errorf("get before list: %w", err) - } + path := filepath.Join(workingDir, args[0]) - destinationList, err := GetImagesInPath(destinationPath) + mirrorImages, err := GetImagesFromFile(path) if err != nil { return fmt.Errorf("get before list: %w", err) } - imageMap := getImageMap(sourceList, destinationList) - olderExists, err := olderSourceImagesExist(logger, imageMap) - if err != nil { - return fmt.Errorf("checking older versions: %w", err) + var originalImages []DockerImage + for _, mirrorImage := range mirrorImages { + originalImage := getOriginalImage(mirrorImage, viper.GetString("mirror")) + originalImages = append(originalImages, originalImage) } - if olderExists { - return nil - } + imageMap := getImageMap(originalImages, mirrorImages) - if err := pullSourceImages(ctx, cli, logger, sourceList); err != nil { + if err := pullSourceImages(ctx, cli, logger, originalImages); err != nil { return fmt.Errorf("pull source image: %w", err) } logger.Printf("Tagging images...") - for sourceImage, destinationImage := range imageMap { - if err := cli.ImageTag(ctx, sourceImage.String(), destinationImage.String()); err != nil { + for originalImage, mirrorImage := range imageMap { + if err := cli.ImageTag(ctx, originalImage.String(), mirrorImage.String()); err != nil { return fmt.Errorf("tagging image: %w", err) } } - for _, image := range destinationList { - auth, err := getAuthForHost(image.Host) + for _, mirrorImage := range mirrorImages { + auth, err := getAuthForHost(mirrorImage.Host) if err != nil { return fmt.Errorf("getting auth: %w", err) } - logger.Printf("Checking if exists at remote registry: %v\n", image.String()) - imageExists, err := checkImageExistsAtRemote(ctx, cli, image, auth) + imageExists, err := checkImageExistsAtRemote(ctx, cli, mirrorImage, auth) if err != nil { return fmt.Errorf("checking image exists: %w", err) } if imageExists { + logger.Printf("Image %s exists at remote registry. Skipping...", mirrorImage.String()) continue } - if err := pushImageAndWait(ctx, logger, cli, image, auth); err != nil { + if err := pushImageAndWait(ctx, logger, cli, mirrorImage, auth); err != nil { return fmt.Errorf("pushing image to remote: %w", err) } } @@ -230,41 +228,15 @@ func getAuthForHost(host string) (string, error) { return base64.URLEncoding.EncodeToString(jsonAuth), nil } -func olderSourceImagesExist(logger *log.Logger, imageMap map[DockerImage]DockerImage) (bool, error) { - var olderSourceExists bool - for sourceImage, destinationImage := range imageMap { - sourceVersion, err := version.NewVersion(sourceImage.Version) - if err != nil { - return false, fmt.Errorf("new source version: %w", err) - } - - destinationVersion, err := version.NewVersion(destinationImage.Version) - if err != nil { - return false, fmt.Errorf("new destination version: %w", err) - } - - if sourceVersion.LessThan(destinationVersion) { - logger.Printf("Source image %v is older than %v\n", sourceImage, destinationImage) - olderSourceExists = true - } - } - - if olderSourceExists { - logger.Printf("One or more source images are older than the destination. Update the manifests before syncing.\n") - } - - return olderSourceExists, nil -} - func pullSourceImages(ctx context.Context, cli *client.Client, logger *log.Logger, sourceImages []DockerImage) error { for _, image := range sourceImages { - logger.Printf("Checking if exists locally: %s", image) exists, err := imageExistsLocally(ctx, cli, image) if err != nil { return fmt.Errorf("checking local image: %w", err) } if exists { + logger.Printf("Image %s exists locally. Skipping...", image.String()) continue } @@ -308,6 +280,10 @@ func imageExistsLocally(ctx context.Context, cli *client.Client, image DockerIma return false, fmt.Errorf("getting image list: %w", err) } + if image.Host == "docker.io" { + image.Host = "" + } + for _, imageSummary := range imageList { for _, repoTag := range imageSummary.RepoTags { if repoTag == image.String() {