Skip to content

Commit

Permalink
This PR does the following things:
Browse files Browse the repository at this point in the history
1. add new registry api under resources.kubesphere.io/v1alpha3
2. deprecate registry api v1alpha2

Registry API v1alpha2 uses docker client to authenticate image registry
secret, which depends on docker.sock. We used to mount host
`/var/run/docker.sock` to deployment. It will prevent us imgrating to
containerd since no `docker.sock` exists. Registry API v1alpha3 comes to
rescure, it wraps library go-containerregistry and compatible with
docker registry, Harbor etc.
  • Loading branch information
Jeff committed Aug 24, 2021
1 parent c740fef commit 3d2fd1b
Show file tree
Hide file tree
Showing 88 changed files with 10,002 additions and 16 deletions.
433 changes: 430 additions & 3 deletions api/ks-openapi-spec/swagger.json

Large diffs are not rendered by default.

10 changes: 6 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ require (
code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6
github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e // indirect
github.com/Masterminds/semver/v3 v3.1.1
github.com/Microsoft/go-winio v0.4.12 // indirect
github.com/PuerkitoBio/goquery v1.5.0
github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d // indirect
github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535
Expand All @@ -24,7 +23,7 @@ require (
github.com/davecgh/go-spew v1.1.1
github.com/deckarep/golang-set v1.7.1 // indirect
github.com/docker/distribution v2.7.1+incompatible
github.com/docker/docker v1.4.2-0.20200203170920-46ec8731fbce
github.com/docker/docker v20.10.7+incompatible
github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82 // indirect
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
github.com/elastic/go-elasticsearch/v5 v5.6.1
Expand All @@ -50,6 +49,7 @@ require (
github.com/gofrs/uuid v3.2.0+incompatible // indirect
github.com/golang/example v0.0.0-20170904185048-46695d81d1fa
github.com/google/go-cmp v0.5.6
github.com/google/go-containerregistry v0.6.0
github.com/google/uuid v1.1.2
github.com/gorilla/handlers v1.4.0 // indirect
github.com/gorilla/websocket v1.4.2
Expand Down Expand Up @@ -88,8 +88,7 @@ require (
github.com/yvasiyarov/gorelic v0.0.6 // indirect
github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f // indirect
golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e
golang.org/x/net v0.0.0-20210525063256-abc453219eb5 // indirect
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c
google.golang.org/grpc v1.30.0
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
gopkg.in/cas.v2 v2.2.0
Expand Down Expand Up @@ -233,6 +232,7 @@ replace (
github.com/codahale/hdrhistogram => github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd
github.com/containerd/containerd => github.com/containerd/containerd v1.3.0
github.com/containerd/continuity => github.com/containerd/continuity v0.0.0-20181203112020-004b46473808
github.com/containerd/stargz-snapshotter/estargz => github.com/containerd/stargz-snapshotter/estargz v0.7.0
github.com/containernetworking/cni => github.com/containernetworking/cni v0.8.0
github.com/coreos/bbolt => github.com/coreos/bbolt v1.3.3
github.com/coreos/etcd => github.com/coreos/etcd v3.3.17+incompatible
Expand Down Expand Up @@ -379,6 +379,7 @@ replace (
github.com/google/btree => github.com/google/btree v1.0.0
github.com/google/flatbuffers => github.com/google/flatbuffers v1.11.0
github.com/google/go-cmp => github.com/google/go-cmp v0.4.0
github.com/google/go-containerregistry => github.com/google/go-containerregistry v0.6.0
github.com/google/go-github => github.com/google/go-github v17.0.0+incompatible
github.com/google/go-querystring => github.com/google/go-querystring v1.0.0
github.com/google/gofuzz => github.com/google/gofuzz v1.1.0
Expand Down Expand Up @@ -599,6 +600,7 @@ replace (
github.com/rs/cors => github.com/rs/cors v1.6.0
github.com/rubenv/sql-migrate => github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351
github.com/russross/blackfriday => github.com/russross/blackfriday v1.5.2
github.com/russross/blackfriday/v2 => github.com/russross/blackfriday/v2 v2.1.0
github.com/ryanuber/columnize => github.com/ryanuber/columnize v2.1.0+incompatible
github.com/samuel/go-zookeeper => github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da
github.com/santhosh-tekuri/jsonschema => github.com/santhosh-tekuri/jsonschema v1.2.4
Expand Down
6 changes: 6 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,8 @@ github.com/containerd/containerd v1.3.0 h1:xjvXQWABwS2uiv3TWgQt5Uth60Gu86LTGZXMJ
github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 h1:4BX8f882bXEDKfWIf0wa8HRvpnBoPszJJXL+TVbBw4M=
github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/stargz-snapshotter/estargz v0.7.0 h1:1d/rydzTywc76lnjJb6qbPCiTiCwts49AzKps/Ecblw=
github.com/containerd/stargz-snapshotter/estargz v0.7.0/go.mod h1:83VWDqHnurTKliEB0YvWMiCfLDwv4Cjj1X9Vk98GJZw=
github.com/containernetworking/cni v0.8.0 h1:BT9lpgGoH4jw3lFC7Odz2prU5ruiYKcgAjMCbgybcKI=
github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
Expand Down Expand Up @@ -347,6 +349,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-containerregistry v0.6.0 h1:niQ+8XD//kKgArIFwDVBXsWVWbde16LPdHMyNwSC8h4=
github.com/google/go-containerregistry v0.6.0/go.mod h1:euCCtNbZ6tKqi1E72vwDj2xZcN5ttKpZLfa/wSo5iLw=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
Expand Down Expand Up @@ -456,6 +460,7 @@ github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e h1:RgQk53JHp
github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.9.5 h1:U+CaK85mrNNb4k8BNOfgJtJ/gr6kswUCFj6miSzVC6M=
github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg=
Expand Down Expand Up @@ -655,6 +660,7 @@ github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351/go.mod h1:DCgfY80j8GYL7MLEfvcpSFvjD0L5yZq/aZUJmhZklyg=
github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4=
Expand Down
2 changes: 2 additions & 0 deletions pkg/kapis/resources/v1alpha2/register.go
Original file line number Diff line number Diff line change
Expand Up @@ -131,12 +131,14 @@ func AddToContainer(c *restful.Container, k8sClient kubernetes.Interface, factor
To(handler.handleGetNamespaceQuotas))

webservice.Route(webservice.POST("registry/verify").
Deprecate().
To(handler.handleVerifyRegistryCredential).
Metadata(restfulspec.KeyOpenAPITags, []string{constants.RegistryTag}).
Doc("verify if a user has access to the docker registry").
Reads(api.RegistryCredential{}).
Returns(http.StatusOK, api.StatusOK, errors.Error{}))
webservice.Route(webservice.GET("/registry/blob").
Deprecate().
To(handler.handleGetRegistryEntry).
Param(webservice.QueryParameter("image", "query image, condition for filtering.").
Required(true).
Expand Down
90 changes: 90 additions & 0 deletions pkg/kapis/resources/v1alpha3/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,19 @@ limitations under the License.
package v1alpha3

import (
"fmt"
"net/http"
"strings"

"github.com/emicklei/go-restful"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/klog"

"kubesphere.io/kubesphere/pkg/api"
"kubesphere.io/kubesphere/pkg/apiserver/query"
"kubesphere.io/kubesphere/pkg/models/components"
v2 "kubesphere.io/kubesphere/pkg/models/registries/v2"
"kubesphere.io/kubesphere/pkg/models/resources/v1alpha2"
resourcev1alpha2 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha2/resource"
resourcev1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource"
Expand All @@ -35,13 +40,15 @@ type Handler struct {
resourceGetterV1alpha3 *resourcev1alpha3.ResourceGetter
resourcesGetterV1alpha2 *resourcev1alpha2.ResourceGetter
componentsGetter components.ComponentsGetter
registryHelper v2.RegistryHelper
}

func New(resourceGetterV1alpha3 *resourcev1alpha3.ResourceGetter, resourcesGetterV1alpha2 *resourcev1alpha2.ResourceGetter, componentsGetter components.ComponentsGetter) *Handler {
return &Handler{
resourceGetterV1alpha3: resourceGetterV1alpha3,
resourcesGetterV1alpha2: resourcesGetterV1alpha2,
componentsGetter: componentsGetter,
registryHelper: v2.NewRegistryHelper(),
}
}

Expand Down Expand Up @@ -203,3 +210,86 @@ func (h *Handler) handleGetComponents(request *restful.Request, response *restfu

response.WriteEntity(result)
}

// handleVerifyImageRepositorySecret verifies image secret against registry, it takes k8s.io/api/core/v1/types.Secret
// as input, and authenticate registry with credential specified. Returns http.StatusOK if authenticate successfully,
// returns http.StatusUnauthorized if failed.
func (h *Handler) handleVerifyImageRepositorySecret(request *restful.Request, response *restful.Response) {
secret := &v1.Secret{}
err := request.ReadEntity(secret)
if err != nil {
api.HandleBadRequest(response, request, err)
}

ok, err := h.registryHelper.Auth(secret)
if !ok {
klog.Error(err)
api.HandleUnauthorized(response, request, err)
} else {
response.WriteHeaderAndJson(http.StatusOK, secret, restful.MIME_JSON)
}
}

// handleGetImageConfig fetches container image spec described in https://github.com/opencontainers/image-spec/blob/main/manifest.md
func (h *Handler) handleGetImageConfig(request *restful.Request, response *restful.Response) {
secretName := request.QueryParameter("secret")
namespace := request.PathParameter("namespace")
image := request.QueryParameter("image")
var secret *v1.Secret

// empty secret means anoymous fetching
if len(secretName) != 0 {
object, err := h.resourceGetterV1alpha3.Get("secrets", namespace, secretName)
if errors.IsNotFound(err) {
api.HandleNotFound(response, request, err)
}
secret = object.(*v1.Secret)
}

config, err := h.registryHelper.Config(secret, image)
if err != nil {
canonicalizeRegistryError(request, response, err)
return
}

response.WriteHeaderAndJson(http.StatusOK, config, restful.MIME_JSON)
}

// handleGetRepositoryTags fetchs all tags of given repository, no paging.
func (h *Handler) handleGetRepositoryTags(request *restful.Request, response *restful.Response) {
secretName := request.QueryParameter("secret")
namespace := request.PathParameter("namespace")
repository := request.QueryParameter("repository")
var secret *v1.Secret

if len(repository) == 0 {
api.HandleBadRequest(response, request, fmt.Errorf("empty repository name"))
return
}

if len(secretName) != 0 {
object, err := h.resourceGetterV1alpha3.Get("secrets", namespace, secretName)
if errors.IsNotFound(err) {
api.HandleNotFound(response, request, err)
}
secret = object.(*v1.Secret)
}

tags, err := h.registryHelper.ListRepositoryTags(secret, repository)
if err != nil {
canonicalizeRegistryError(request, response, err)
return
}

response.WriteHeaderAndJson(http.StatusOK, tags, restful.MIME_JSON)
}

func canonicalizeRegistryError(request *restful.Request, response *restful.Response, err error) {
if strings.Contains(err.Error(), "Unauthorized") {
api.HandleUnauthorized(response, request, err)
} else if strings.Contains(err.Error(), "MANIFEST_UNKNOWN") {
api.HandleNotFound(response, request, err)
} else {
api.HandleBadRequest(response, request, err)
}
}
30 changes: 30 additions & 0 deletions pkg/kapis/resources/v1alpha3/register.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package v1alpha3
import (
"github.com/emicklei/go-restful"
restfulspec "github.com/emicklei/go-restful-openapi"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/cache"

Expand All @@ -28,6 +29,7 @@ import (
"kubesphere.io/kubesphere/pkg/apiserver/runtime"
"kubesphere.io/kubesphere/pkg/informers"
"kubesphere.io/kubesphere/pkg/models/components"
v2 "kubesphere.io/kubesphere/pkg/models/registries/v2"
resourcev1alpha2 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha2/resource"
resourcev1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource"

Expand Down Expand Up @@ -114,6 +116,34 @@ func AddToContainer(c *restful.Container, informerFactory informers.InformerFact
Doc("Get the health status of system components.").
Returns(http.StatusOK, ok, v1alpha2.HealthStatus{}))

webservice.Route(webservice.POST("/namespaces/{namespace}/registrysecrets/{secret}").
To(handler.handleVerifyImageRepositorySecret).
Param(webservice.PathParameter("namespace", "Namespace of the image repository secret to create.").Required(true)).
Param(webservice.PathParameter("secret", "Secret name of the image repository credential to create").Required(true)).
Param(webservice.BodyParameter("secretSpec", "Secret specification, definition in k8s.io/api/core/v1/types.Secret")).
Reads(v1.Secret{}).
Metadata(restfulspec.KeyOpenAPITags, []string{tagNamespacedResource}).
Doc("Verify image repostiry secret.").
Returns(http.StatusOK, ok, v1.Secret{}))

webservice.Route(webservice.GET("/namespaces/{namespace}/imageconfig").
To(handler.handleGetImageConfig).
Param(webservice.PathParameter("namespace", "Namespace of the image repository secret.").Required(true)).
Param(webservice.QueryParameter("secret", "Secret name of the image repository credential, left empty means anonymous fetch.").Required(false)).
Param(webservice.QueryParameter("image", "Image name to query, e.g. kubesphere/ks-apiserver:v3.1.1").Required(true)).
Metadata(restfulspec.KeyOpenAPITags, []string{tagNamespacedResource}).
Doc("Get image config.").
Returns(http.StatusOK, ok, v2.ImageConfig{}))

webservice.Route(webservice.GET("/namespaces/{namespace}/repositorytags").
To(handler.handleGetRepositoryTags).
Param(webservice.PathParameter("namespace", "Namespace of the image repository secret.").Required(true)).
Param(webservice.QueryParameter("repository", "Repository to query, e.g. calico/cni.").Required(true)).
Param(webservice.QueryParameter("secret", "Secret name of the image repository credential, left empty means anonymous fetch.").Required(false)).
Metadata(restfulspec.KeyOpenAPITags, []string{tagNamespacedResource}).
Doc("List repository tags, this is an experimental API, use it by your own caution.").
Returns(http.StatusOK, ok, v2.RepositoryTags{}))

c.Add(webservice)

return nil
Expand Down
83 changes: 83 additions & 0 deletions pkg/models/registries/v2/options.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package v2

import (
"context"
"net/http"

"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/remote"
)

const (
// DefaultRegistry is the registry name that will be used if no registry
// provided and the default is not overridden.
DefaultRegistry = "index.docker.io"
defaultRegistryAlias = "docker.io"

// DefaultTag is the tag name that will be used if no tag provided and the
// default is not overridden.
DefaultTag = "latest"
)

type options struct {
name []name.Option
remote []remote.Option
platform *v1.Platform
}

func makeOptions(opts ...Option) options {
opt := options{
remote: []remote.Option{
remote.WithAuth(authn.Anonymous),
},
}
for _, o := range opts {
o(&opt)
}
return opt
}

// Option is a functional option
type Option func(*options)

// WithTransport is a functional option for overriding the default transport
// for remote operations.
func WithTransport(t http.RoundTripper) Option {
return func(o *options) {
o.remote = append(o.remote, remote.WithTransport(t))
}
}

// Insecure is an Option that allows image references to be fetched without TLS.
func Insecure(o *options) {
o.name = append(o.name, name.Insecure)
}

// WithAuth is a functional option for overriding the default authenticator
// for remote operations.
//
func WithAuth(auth authn.Authenticator) Option {
return func(o *options) {
// Replace the default keychain at position 0.
o.remote[0] = remote.WithAuth(auth)
}
}

// WithContext is a functional option for setting the context.
func WithContext(ctx context.Context) Option {
return func(o *options) {
o.remote = append(o.remote, remote.WithContext(ctx))
}
}

// WithPlatform is an Option to specify the platform.
func WithPlatform(platform *v1.Platform) Option {
return func(o *options) {
if platform != nil {
o.remote = append(o.remote, remote.WithPlatform(*platform))
}
o.platform = platform
}
}
Loading

0 comments on commit 3d2fd1b

Please sign in to comment.