diff --git a/Dockerfile.sidecar b/Dockerfile.sidecar index 4eac67be..06118fb6 100644 --- a/Dockerfile.sidecar +++ b/Dockerfile.sidecar @@ -43,13 +43,19 @@ RUN set -ex; \ ARG XTRABACKUP_PKG=percona-xtrabackup-24 RUN set -ex; \ apt-get update; \ - apt-get install -y --no-install-recommends gnupg2 wget lsb-release curl bc; \ + apt-get install -y --no-install-recommends gnupg2 wget lsb-release curl bc fuse jq openssh-server; \ wget -P /tmp --no-check-certificate https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb; \ dpkg -i /tmp/percona-release_latest.$(lsb_release -sc)_all.deb; \ apt-get update; \ apt-get install -y --no-install-recommends ${XTRABACKUP_PKG}; \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - +#ADD http://mirrors.woqutech.com/download/qfusion/files/bin/juicefs-1.0.0-rc1-linux-amd64 /usr/local/bin/juicefs +COPY juicefs/juicefs /usr/local/bin/juicefs +RUN chmod +x /usr/local/bin/juicefs && mkdir -p /run/sshd; \ + mkdir -p /root/.ssh; \ + chmod 700 /root/.ssh WORKDIR / COPY --from=builder /workspace/bin/sidecar /usr/local/bin/sidecar -ENTRYPOINT ["sidecar"] +COPY script/sshd.sh /sshd.sh +COPY script/backup.sh /backup.sh +CMD [ "sidecar" ] diff --git a/controllers/mysqlcluster_controller.go b/controllers/mysqlcluster_controller.go index b6405c49..80c33c4a 100644 --- a/controllers/mysqlcluster_controller.go +++ b/controllers/mysqlcluster_controller.go @@ -24,7 +24,9 @@ import ( "github.com/presslabs/controller-util/pkg/syncer" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - policyv1beta1 "k8s.io/api/policy/v1beta1" + + // policyv1beta1 "k8s.io/api/policy/v1beta1" + policyv1beta1 "k8s.io/api/policy/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" @@ -125,6 +127,11 @@ func (r *MysqlClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, err } + secretSShSyncer := clustersyncer.NewSShKeySyncer(r.Client, instance) + if err = syncer.Sync(ctx, secretSShSyncer, r.Recorder); err != nil { + return ctrl.Result{}, err + } + // Todo: modify mysql cm will trigger rolling update but it will not be applied. cmRev := mysqlCMSyncer.Object().(*corev1.ConfigMap).ResourceVersion sctRev := secretSyncer.Object().(*corev1.Secret).ResourceVersion diff --git a/mysqlcluster/container/backup.go b/mysqlcluster/container/backup.go index 8b8e0519..a3e14ad8 100644 --- a/mysqlcluster/container/backup.go +++ b/mysqlcluster/container/backup.go @@ -38,7 +38,7 @@ func (c *backupSidecar) getImage() string { } func (c *backupSidecar) getCommand() []string { - return []string{"sidecar", "http"} + return []string{"sh", "-c", "/sshd.sh ; sidecar http"} } func (c *backupSidecar) getEnvVars() []corev1.EnvVar { @@ -147,5 +147,13 @@ func (c *backupSidecar) getVolumeMounts() []corev1.VolumeMount { Name: utils.SysLocalTimeZone, MountPath: utils.SysLocalTimeZoneMountPath, }, + { + Name: utils.SysFuseVolume, + MountPath: utils.SysFuseVolumnMountPath, + }, + { + Name: utils.SShVolumnName, + MountPath: utils.SshVolumnPath, + }, } } diff --git a/mysqlcluster/container/container.go b/mysqlcluster/container/container.go index 5f347390..a61bd6d8 100644 --- a/mysqlcluster/container/container.go +++ b/mysqlcluster/container/container.go @@ -63,6 +63,7 @@ func getStartupProbe(name string) *corev1.Probe { // EnsureContainer ensure a container by the giving name. func EnsureContainer(name string, c *mysqlcluster.MysqlCluster) corev1.Container { var ctr container + var security *corev1.SecurityContext = nil switch name { case utils.ContainerInitSidecarName: ctr = &initSidecar{c, name} @@ -80,6 +81,14 @@ func EnsureContainer(name string, c *mysqlcluster.MysqlCluster) corev1.Container ctr = &auditLog{c, name} case utils.ContainerBackupName: ctr = &backupSidecar{c, name} + needAdmin := true + security = &corev1.SecurityContext{ + Privileged: &needAdmin, + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"CAP_SYS_ADMIN", + "DAC_READ_SEARCH", + }, + }} } return corev1.Container{ @@ -95,5 +104,6 @@ func EnsureContainer(name string, c *mysqlcluster.MysqlCluster) corev1.Container ReadinessProbe: ctr.getReadinessProbe(), StartupProbe: getStartupProbe(name), VolumeMounts: ctr.getVolumeMounts(), + SecurityContext: security, } } diff --git a/mysqlcluster/genkey.sh b/mysqlcluster/genkey.sh new file mode 100644 index 00000000..c1a87472 --- /dev/null +++ b/mysqlcluster/genkey.sh @@ -0,0 +1,20 @@ + +# ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no sample-mysql-0.sample-mysql.default +kubectl create secret generic sample-ssh-key --from-file=id_ecdsa=/root/.ssh/id_ecdsa --from-file=authorized_keys=/root/.ssh/id_ecdsa.pub + +/usr/sbin/sshd -D -e -f /etc/ssh/sshd_config +mkdir -p /root/.ssh +chmod 700 /root/.ssh +cp /etc/secret-ssh/* /root/.ssh +chmod 600 /root/.ssh/authorized_keys + + +cat <>sshd_config +PermitRootLogin yes +PasswordAuthentication no +PermitEmptyPasswords no +ChallengeResponseAuthentication no +GSSAPIAuthentication no +UseDNS no +UsePAM yes +EOF \ No newline at end of file diff --git a/mysqlcluster/mysqlcluster.go b/mysqlcluster/mysqlcluster.go index 88f00b30..a9f4867a 100644 --- a/mysqlcluster/mysqlcluster.go +++ b/mysqlcluster/mysqlcluster.go @@ -180,7 +180,7 @@ func (c *MysqlCluster) EnsureVolumes() []corev1.Volume { }, ) } - + var defMode int32 = 0600 volumes = append(volumes, corev1.Volume{ Name: utils.MysqlConfVolumeName, @@ -246,6 +246,23 @@ func (c *MysqlCluster) EnsureVolumes() []corev1.Volume { }, }, }, + corev1.Volume{ + Name: utils.SysFuseVolume, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/dev/fuse", + }, + }, + }, + corev1.Volume{ + Name: utils.SShVolumnName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: c.GetNameForResource(utils.SShKey), + DefaultMode: &defMode, + }, + }, + }, ) // add the nfs volumn mount if len(c.Spec.NFSServerAddress) != 0 { @@ -328,6 +345,8 @@ func (c *MysqlCluster) GetNameForResource(name utils.ResourceName) string { return fmt.Sprintf("%s-metrics", c.Name) case utils.Secret: return fmt.Sprintf("%s-secret", c.Name) + case utils.SShKey: + return fmt.Sprintf("%s-ssh-key", c.Name) case utils.XenonMetaData: return fmt.Sprintf("%s-xenon", c.Name) case utils.ConfigMap: diff --git a/mysqlcluster/syncer/headless_service.go b/mysqlcluster/syncer/headless_service.go index 4de862c8..36501629 100644 --- a/mysqlcluster/syncer/headless_service.go +++ b/mysqlcluster/syncer/headless_service.go @@ -57,8 +57,8 @@ func NewHeadlessSVCSyncer(cli client.Client, c *mysqlcluster.MysqlCluster) synce // Use `publishNotReadyAddresses` to be able to access pods even if the pod is not ready. service.Spec.PublishNotReadyAddresses = true - if len(service.Spec.Ports) != 2 { - service.Spec.Ports = make([]corev1.ServicePort, 2) + if len(service.Spec.Ports) != 3 { + service.Spec.Ports = make([]corev1.ServicePort, 3) } service.Spec.Ports[0].Name = utils.MysqlPortName @@ -68,6 +68,10 @@ func NewHeadlessSVCSyncer(cli client.Client, c *mysqlcluster.MysqlCluster) synce service.Spec.Ports[1].Name = utils.XBackupPortName service.Spec.Ports[1].Port = utils.XBackupPort service.Spec.Ports[1].TargetPort = intstr.FromInt(utils.XBackupPort) + // ssh port + service.Spec.Ports[2].Name = utils.SshPortName + service.Spec.Ports[2].Port = utils.SshPort + service.Spec.Ports[2].TargetPort = intstr.FromInt(utils.SshPort) return nil }) } diff --git a/mysqlcluster/syncer/pdb.go b/mysqlcluster/syncer/pdb.go index 1c0a2afc..0e1ab8ea 100644 --- a/mysqlcluster/syncer/pdb.go +++ b/mysqlcluster/syncer/pdb.go @@ -18,7 +18,7 @@ package syncer import ( "github.com/presslabs/controller-util/pkg/syncer" - policyv1beta1 "k8s.io/api/policy/v1beta1" + policyv1beta1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/mysqlcluster/syncer/sshSecret.go b/mysqlcluster/syncer/sshSecret.go new file mode 100644 index 00000000..0744ba64 --- /dev/null +++ b/mysqlcluster/syncer/sshSecret.go @@ -0,0 +1,59 @@ +/* +Copyright 2021 RadonDB. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package syncer + +import ( + "github.com/presslabs/controller-util/pkg/syncer" + "github.com/radondb/radondb-mysql-kubernetes/mysqlcluster" + "github.com/radondb/radondb-mysql-kubernetes/utils" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// NewSecretSyncer returns secret syncer. +func NewSShKeySyncer(cli client.Client, c *mysqlcluster.MysqlCluster) syncer.Interface { + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: c.GetNameForResource(utils.SShKey), + Namespace: c.Namespace, + }, + } + + return syncer.NewObjectSyncer("Secret", c.Unwrap(), secret, cli, func() error { + + if secret.Data == nil { + secret.Data = make(map[string][]byte) + } + + if len(secret.Data["id_ecdsa"]) == 0 { + pub, priv, err := GenSSHKey() + if err != nil { + return err + } + secret.Data["id_ecdsa"] = priv + secret.Data["authorized_keys"] = pub + + } + + return nil + }) +} diff --git a/mysqlcluster/syncer/sshkey.go b/mysqlcluster/syncer/sshkey.go new file mode 100644 index 00000000..80d1afdc --- /dev/null +++ b/mysqlcluster/syncer/sshkey.go @@ -0,0 +1,101 @@ +/* +Copyright 2021 RadonDB. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package syncer + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "encoding/pem" + "log" + + "golang.org/x/crypto/ssh" +) + +// ssh -o UserKnownHostsFile=/dev/null +func GenSSHKey() (pubkey, privekey []byte, err error) { + + bitSize := 4096 + + privateKey, err := generatePrivateKey(bitSize) + if err != nil { + return nil, nil, err + } + + publicKeyBytes, err := generatePublicKey(&privateKey.PublicKey) + if err != nil { + return nil, nil, err + } + + privateKeyBytes := encodePrivateKeyToPEM(privateKey) + + return publicKeyBytes, privateKeyBytes, err + +} + +// generatePrivateKey creates a RSA Private Key of specified byte size +func generatePrivateKey(bitSize int) (*ecdsa.PrivateKey, error) { + // Private Key generation + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, err + } + + // Validate Private Key + // err = privateKey.Validate() + // if err != nil { + // return nil, err + // } + + log.Println("Private Key generated") + return privateKey, nil +} + +// encodePrivateKeyToPEM encodes Private Key from RSA to PEM format +func encodePrivateKeyToPEM(privateKey *ecdsa.PrivateKey) []byte { + // Get ASN.1 DER format + privDER, err := x509.MarshalECPrivateKey(privateKey) + if err != nil { + panic(err) + } + // pem.Block + privBlock := pem.Block{ + Type: "EC PRIVATE KEY", + Headers: nil, + Bytes: privDER, + } + + // Private key in PEM format + privatePEM := pem.EncodeToMemory(&privBlock) + + return privatePEM +} + +// generatePublicKey take a rsa.PublicKey and return bytes suitable for writing to .pub file +// returns in the format "ssh-rsa ..." +func generatePublicKey(privatekey *ecdsa.PublicKey) ([]byte, error) { + publicKey, err := ssh.NewPublicKey(privatekey) + if err != nil { + return nil, err + } + + pubKeyBytes := ssh.MarshalAuthorizedKey(publicKey) + + log.Println("Public key generated") + return pubKeyBytes, nil +} diff --git a/mysqlcluster/syncer/statefulset_test.go b/mysqlcluster/syncer/statefulset_test.go index abb8f4db..797c3850 100644 --- a/mysqlcluster/syncer/statefulset_test.go +++ b/mysqlcluster/syncer/statefulset_test.go @@ -17,6 +17,7 @@ limitations under the License. package syncer import ( + "fmt" "testing" appsv1 "k8s.io/api/apps/v1" @@ -145,3 +146,9 @@ func TestStatefulSetSyncer_sfsUpdated(t *testing.T) { }) } } + +func TestSecretKey(t *testing.T) { + pub, priv := GenSSHKey() + fmt.Println(string(pub)) + fmt.Println(string(priv)) +} diff --git a/utils/constants.go b/utils/constants.go index 8cab16bf..fa203a6c 100644 --- a/utils/constants.go +++ b/utils/constants.go @@ -88,6 +88,12 @@ const ( LogsVolumeName = "logs" DataVolumeName = "data" SysVolumeName = "host-sys" + + // just for juicefs + SysFuseVolume = "host-fuse" + SshPortName = "ssh" + SshPort = 22 + ScriptsVolumeName = "scripts" XenonConfVolumeName = "xenon-conf" InitFileVolumeName = "init-mysql" @@ -100,6 +106,8 @@ const ( LogsVolumeMountPath = "/var/log/mysql" DataVolumeMountPath = "/var/lib/mysql" SysVolumeMountPath = "/host-sys" + + SysFuseVolumnMountPath = "/dev/fuse" ScriptsVolumeMountPath = "/scripts" XenonConfVolumeMountPath = "/etc/xenon" InitFileVolumeMountPath = "/docker-entrypoint-initdb.d" @@ -129,6 +137,10 @@ const ( TlsVolumeName = "tls" // TlsMountPath is the volume mount path for tls TlsMountPath = "/etc/mysql-ssl" + + // ssh path + SShVolumnName = "ssh-key" + SshVolumnPath = "/etc/secret-ssh" ) // ResourceName is the type for aliasing resources that will be created. @@ -165,6 +177,8 @@ const ( JobAnonationDate = "backupDate" // Job Annonations type JobAnonationType = "backupType" + // SSh key + SShKey = "ssh" ) // JobType