Skip to content

Commit

Permalink
*: support juicefs radondb#745
Browse files Browse the repository at this point in the history
  • Loading branch information
acekingke committed Nov 29, 2022
1 parent 41b6f3c commit 52b3ee8
Show file tree
Hide file tree
Showing 12 changed files with 264 additions and 9 deletions.
12 changes: 9 additions & 3 deletions Dockerfile.sidecar
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,19 @@ RUN set -ex; \
ARG XTRABACKUP_PKG=percona-xtrabackup-24
RUN set -ex; \
apt-get update; \
apt-get install -y --no-install-recommends gnupg2 wget lsb-release curl bc; \
apt-get install -y --no-install-recommends gnupg2 wget lsb-release curl bc fuse jq openssh-server; \
wget -P /tmp --no-check-certificate https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb; \
dpkg -i /tmp/percona-release_latest.$(lsb_release -sc)_all.deb; \
apt-get update; \
apt-get install -y --no-install-recommends ${XTRABACKUP_PKG}; \
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

#ADD http://mirrors.woqutech.com/download/qfusion/files/bin/juicefs-1.0.0-rc1-linux-amd64 /usr/local/bin/juicefs
COPY juicefs/juicefs /usr/local/bin/juicefs
RUN chmod +x /usr/local/bin/juicefs && mkdir -p /run/sshd; \
mkdir -p /root/.ssh; \
chmod 700 /root/.ssh
WORKDIR /
COPY --from=builder /workspace/bin/sidecar /usr/local/bin/sidecar
ENTRYPOINT ["sidecar"]
COPY script/sshd.sh /sshd.sh
COPY script/backup.sh /backup.sh
CMD [ "sidecar" ]
9 changes: 8 additions & 1 deletion controllers/mysqlcluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@ import (
"github.com/presslabs/controller-util/pkg/syncer"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
policyv1beta1 "k8s.io/api/policy/v1beta1"

// policyv1beta1 "k8s.io/api/policy/v1beta1"
policyv1beta1 "k8s.io/api/policy/v1"
rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/labels"
Expand Down Expand Up @@ -125,6 +127,11 @@ func (r *MysqlClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request
return ctrl.Result{}, err
}

secretSShSyncer := clustersyncer.NewSShKeySyncer(r.Client, instance)
if err = syncer.Sync(ctx, secretSShSyncer, r.Recorder); err != nil {
return ctrl.Result{}, err
}

// Todo: modify mysql cm will trigger rolling update but it will not be applied.
cmRev := mysqlCMSyncer.Object().(*corev1.ConfigMap).ResourceVersion
sctRev := secretSyncer.Object().(*corev1.Secret).ResourceVersion
Expand Down
10 changes: 9 additions & 1 deletion mysqlcluster/container/backup.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ func (c *backupSidecar) getImage() string {
}

func (c *backupSidecar) getCommand() []string {
return []string{"sidecar", "http"}
return []string{"sh", "-c", "/sshd.sh ; sidecar http"}
}

func (c *backupSidecar) getEnvVars() []corev1.EnvVar {
Expand Down Expand Up @@ -147,5 +147,13 @@ func (c *backupSidecar) getVolumeMounts() []corev1.VolumeMount {
Name: utils.SysLocalTimeZone,
MountPath: utils.SysLocalTimeZoneMountPath,
},
{
Name: utils.SysFuseVolume,
MountPath: utils.SysFuseVolumnMountPath,
},
{
Name: utils.SShVolumnName,
MountPath: utils.SshVolumnPath,
},
}
}
10 changes: 10 additions & 0 deletions mysqlcluster/container/container.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ func getStartupProbe(name string) *corev1.Probe {
// EnsureContainer ensure a container by the giving name.
func EnsureContainer(name string, c *mysqlcluster.MysqlCluster) corev1.Container {
var ctr container
var security *corev1.SecurityContext = nil
switch name {
case utils.ContainerInitSidecarName:
ctr = &initSidecar{c, name}
Expand All @@ -80,6 +81,14 @@ func EnsureContainer(name string, c *mysqlcluster.MysqlCluster) corev1.Container
ctr = &auditLog{c, name}
case utils.ContainerBackupName:
ctr = &backupSidecar{c, name}
needAdmin := true
security = &corev1.SecurityContext{
Privileged: &needAdmin,
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{"CAP_SYS_ADMIN",
"DAC_READ_SEARCH",
},
}}
}

return corev1.Container{
Expand All @@ -95,5 +104,6 @@ func EnsureContainer(name string, c *mysqlcluster.MysqlCluster) corev1.Container
ReadinessProbe: ctr.getReadinessProbe(),
StartupProbe: getStartupProbe(name),
VolumeMounts: ctr.getVolumeMounts(),
SecurityContext: security,
}
}
20 changes: 20 additions & 0 deletions mysqlcluster/genkey.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@

# ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no sample-mysql-0.sample-mysql.default
kubectl create secret generic sample-ssh-key --from-file=id_ecdsa=/root/.ssh/id_ecdsa --from-file=authorized_keys=/root/.ssh/id_ecdsa.pub

/usr/sbin/sshd -D -e -f /etc/ssh/sshd_config
mkdir -p /root/.ssh
chmod 700 /root/.ssh
cp /etc/secret-ssh/* /root/.ssh
chmod 600 /root/.ssh/authorized_keys


cat <<EOF >>sshd_config
PermitRootLogin yes
PasswordAuthentication no
PermitEmptyPasswords no
ChallengeResponseAuthentication no
GSSAPIAuthentication no
UseDNS no
UsePAM yes
EOF
21 changes: 20 additions & 1 deletion mysqlcluster/mysqlcluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ func (c *MysqlCluster) EnsureVolumes() []corev1.Volume {
},
)
}

var defMode int32 = 0600
volumes = append(volumes,
corev1.Volume{
Name: utils.MysqlConfVolumeName,
Expand Down Expand Up @@ -246,6 +246,23 @@ func (c *MysqlCluster) EnsureVolumes() []corev1.Volume {
},
},
},
corev1.Volume{
Name: utils.SysFuseVolume,
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/dev/fuse",
},
},
},
corev1.Volume{
Name: utils.SShVolumnName,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: c.GetNameForResource(utils.SShKey),
DefaultMode: &defMode,
},
},
},
)
// add the nfs volumn mount
if len(c.Spec.NFSServerAddress) != 0 {
Expand Down Expand Up @@ -328,6 +345,8 @@ func (c *MysqlCluster) GetNameForResource(name utils.ResourceName) string {
return fmt.Sprintf("%s-metrics", c.Name)
case utils.Secret:
return fmt.Sprintf("%s-secret", c.Name)
case utils.SShKey:
return fmt.Sprintf("%s-ssh-key", c.Name)
case utils.XenonMetaData:
return fmt.Sprintf("%s-xenon", c.Name)
case utils.ConfigMap:
Expand Down
8 changes: 6 additions & 2 deletions mysqlcluster/syncer/headless_service.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,8 @@ func NewHeadlessSVCSyncer(cli client.Client, c *mysqlcluster.MysqlCluster) synce
// Use `publishNotReadyAddresses` to be able to access pods even if the pod is not ready.
service.Spec.PublishNotReadyAddresses = true

if len(service.Spec.Ports) != 2 {
service.Spec.Ports = make([]corev1.ServicePort, 2)
if len(service.Spec.Ports) != 3 {
service.Spec.Ports = make([]corev1.ServicePort, 3)
}

service.Spec.Ports[0].Name = utils.MysqlPortName
Expand All @@ -68,6 +68,10 @@ func NewHeadlessSVCSyncer(cli client.Client, c *mysqlcluster.MysqlCluster) synce
service.Spec.Ports[1].Name = utils.XBackupPortName
service.Spec.Ports[1].Port = utils.XBackupPort
service.Spec.Ports[1].TargetPort = intstr.FromInt(utils.XBackupPort)
// ssh port
service.Spec.Ports[2].Name = utils.SshPortName
service.Spec.Ports[2].Port = utils.SshPort
service.Spec.Ports[2].TargetPort = intstr.FromInt(utils.SshPort)
return nil
})
}
2 changes: 1 addition & 1 deletion mysqlcluster/syncer/pdb.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ package syncer

import (
"github.com/presslabs/controller-util/pkg/syncer"
policyv1beta1 "k8s.io/api/policy/v1beta1"
policyv1beta1 "k8s.io/api/policy/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"sigs.k8s.io/controller-runtime/pkg/client"
Expand Down
59 changes: 59 additions & 0 deletions mysqlcluster/syncer/sshSecret.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
/*
Copyright 2021 RadonDB.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package syncer

import (
"github.com/presslabs/controller-util/pkg/syncer"
"github.com/radondb/radondb-mysql-kubernetes/mysqlcluster"
"github.com/radondb/radondb-mysql-kubernetes/utils"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
)

// NewSecretSyncer returns secret syncer.
func NewSShKeySyncer(cli client.Client, c *mysqlcluster.MysqlCluster) syncer.Interface {
secret := &corev1.Secret{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "Secret",
},
ObjectMeta: metav1.ObjectMeta{
Name: c.GetNameForResource(utils.SShKey),
Namespace: c.Namespace,
},
}

return syncer.NewObjectSyncer("Secret", c.Unwrap(), secret, cli, func() error {

if secret.Data == nil {
secret.Data = make(map[string][]byte)
}

if len(secret.Data["id_ecdsa"]) == 0 {
pub, priv, err := GenSSHKey()
if err != nil {
return err
}
secret.Data["id_ecdsa"] = priv
secret.Data["authorized_keys"] = pub

}

return nil
})
}
101 changes: 101 additions & 0 deletions mysqlcluster/syncer/sshkey.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
/*
Copyright 2021 RadonDB.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package syncer

import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"encoding/pem"
"log"

"golang.org/x/crypto/ssh"
)

// ssh -o UserKnownHostsFile=/dev/null
func GenSSHKey() (pubkey, privekey []byte, err error) {

bitSize := 4096

privateKey, err := generatePrivateKey(bitSize)
if err != nil {
return nil, nil, err
}

publicKeyBytes, err := generatePublicKey(&privateKey.PublicKey)
if err != nil {
return nil, nil, err
}

privateKeyBytes := encodePrivateKeyToPEM(privateKey)

return publicKeyBytes, privateKeyBytes, err

}

// generatePrivateKey creates a RSA Private Key of specified byte size
func generatePrivateKey(bitSize int) (*ecdsa.PrivateKey, error) {
// Private Key generation
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return nil, err
}

// Validate Private Key
// err = privateKey.Validate()
// if err != nil {
// return nil, err
// }

log.Println("Private Key generated")
return privateKey, nil
}

// encodePrivateKeyToPEM encodes Private Key from RSA to PEM format
func encodePrivateKeyToPEM(privateKey *ecdsa.PrivateKey) []byte {
// Get ASN.1 DER format
privDER, err := x509.MarshalECPrivateKey(privateKey)
if err != nil {
panic(err)
}
// pem.Block
privBlock := pem.Block{
Type: "EC PRIVATE KEY",
Headers: nil,
Bytes: privDER,
}

// Private key in PEM format
privatePEM := pem.EncodeToMemory(&privBlock)

return privatePEM
}

// generatePublicKey take a rsa.PublicKey and return bytes suitable for writing to .pub file
// returns in the format "ssh-rsa ..."
func generatePublicKey(privatekey *ecdsa.PublicKey) ([]byte, error) {
publicKey, err := ssh.NewPublicKey(privatekey)
if err != nil {
return nil, err
}

pubKeyBytes := ssh.MarshalAuthorizedKey(publicKey)

log.Println("Public key generated")
return pubKeyBytes, nil
}
7 changes: 7 additions & 0 deletions mysqlcluster/syncer/statefulset_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package syncer

import (
"fmt"
"testing"

appsv1 "k8s.io/api/apps/v1"
Expand Down Expand Up @@ -145,3 +146,9 @@ func TestStatefulSetSyncer_sfsUpdated(t *testing.T) {
})
}
}

func TestSecretKey(t *testing.T) {
pub, priv := GenSSHKey()
fmt.Println(string(pub))
fmt.Println(string(priv))
}
Loading

0 comments on commit 52b3ee8

Please sign in to comment.