Skip to content

Commit

Permalink
Merge pull request #1970 from percona/K8SPXC-1366
Browse files Browse the repository at this point in the history 
K8SPXC-1366: Add suspendedDeadlineSeconds
  • Loading branch information
@hors
hors authored Feb 11, 2025
2 parents 9a5bdc1 + 3b6bb73 commit 5872a54
Show file tree
Hide file tree
Showing 14 changed files with 563 additions and 64 deletions.
3 changes: 3 additions & 0 deletions config/crd/bases/pxc.percona.com_perconaxtradbclusterbackups.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,9 @@ spec:
type: integer
storageName:
type: string
suspendedDeadlineSeconds:
format: int64
type: integer
type: object
status:
properties:
Expand Down
3 changes: 3 additions & 0 deletions config/crd/bases/pxc.percona.com_perconaxtradbclusters.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1083,6 +1083,9 @@ spec:
type: object
type: object
type: object
suspendedDeadlineSeconds:
format: int64
type: integer
type: object
crVersion:
type: string
Expand Down
1 change: 1 addition & 0 deletions deploy/backup/backup.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ spec:
storageName: fs-pvc
# activeDeadlineSeconds: 3600
# startingDeadlineSeconds: 300
# suspendedDeadlineSeconds: 1200
# containerOptions:
# env:
# - name: VERIFY_TLS
Expand Down
6 changes: 6 additions & 0 deletions deploy/bundle.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,9 @@ spec:
type: integer
storageName:
type: string
suspendedDeadlineSeconds:
format: int64
type: integer
type: object
status:
properties:
Expand Down Expand Up @@ -1997,6 +2000,9 @@ spec:
type: object
type: object
type: object
suspendedDeadlineSeconds:
format: int64
type: integer
type: object
crVersion:
type: string
Expand Down
1 change: 1 addition & 0 deletions deploy/cr.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -606,6 +606,7 @@ spec:
# backoffLimit: 6
# activeDeadlineSeconds: 3600
# startingDeadlineSeconds: 300
# suspendedDeadlineSeconds: 1200
# serviceAccountName: percona-xtradb-cluster-operator
# imagePullSecrets:
# - name: private-registry-credentials
Expand Down
6 changes: 6 additions & 0 deletions deploy/crd.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,9 @@ spec:
type: integer
storageName:
type: string
suspendedDeadlineSeconds:
format: int64
type: integer
type: object
status:
properties:
Expand Down Expand Up @@ -1997,6 +2000,9 @@ spec:
type: object
type: object
type: object
suspendedDeadlineSeconds:
format: int64
type: integer
type: object
crVersion:
type: string
Expand Down
6 changes: 6 additions & 0 deletions deploy/cw-bundle.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,9 @@ spec:
type: integer
storageName:
type: string
suspendedDeadlineSeconds:
format: int64
type: integer
type: object
status:
properties:
Expand Down Expand Up @@ -1997,6 +2000,9 @@ spec:
type: object
type: object
type: object
suspendedDeadlineSeconds:
format: int64
type: integer
type: object
crVersion:
type: string
Expand Down
11 changes: 6 additions & 5 deletions pkg/apis/pxc/v1/pxc_backup_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,11 +47,12 @@ type PerconaXtraDBClusterBackup struct {
}

type PXCBackupSpec struct {
PXCCluster string `json:"pxcCluster"`
StorageName string `json:"storageName,omitempty"`
ContainerOptions *BackupContainerOptions `json:"containerOptions,omitempty"`
StartingDeadlineSeconds *int64 `json:"startingDeadlineSeconds,omitempty"`
ActiveDeadlineSeconds *int64 `json:"activeDeadlineSeconds,omitempty"`
PXCCluster string `json:"pxcCluster"`
StorageName string `json:"storageName,omitempty"`
ContainerOptions *BackupContainerOptions `json:"containerOptions,omitempty"`
ActiveDeadlineSeconds *int64 `json:"activeDeadlineSeconds,omitempty"`
StartingDeadlineSeconds *int64 `json:"startingDeadlineSeconds,omitempty"`
SuspendedDeadlineSeconds *int64 `json:"suspendedDeadlineSeconds,omitempty"`
}

type PXCBackupStatus struct {
Expand Down
25 changes: 13 additions & 12 deletions pkg/apis/pxc/v1/pxc_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,18 +160,19 @@ const (
)

type PXCScheduledBackup struct {
AllowParallel *bool `json:"allowParallel,omitempty"`
Image string `json:"image,omitempty"`
ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"`
Schedule []PXCScheduledBackupSchedule `json:"schedule,omitempty"`
Storages map[string]*BackupStorageSpec `json:"storages,omitempty"`
ServiceAccountName string `json:"serviceAccountName,omitempty"`
Annotations map[string]string `json:"annotations,omitempty"`
PITR PITRSpec `json:"pitr,omitempty"`
BackoffLimit *int32 `json:"backoffLimit,omitempty"`
ActiveDeadlineSeconds *int64 `json:"activeDeadlineSeconds,omitempty"`
StartingDeadlineSeconds *int64 `json:"startingDeadlineSeconds,omitempty"`
AllowParallel *bool `json:"allowParallel,omitempty"`
Image string `json:"image,omitempty"`
ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"`
Schedule []PXCScheduledBackupSchedule `json:"schedule,omitempty"`
Storages map[string]*BackupStorageSpec `json:"storages,omitempty"`
ServiceAccountName string `json:"serviceAccountName,omitempty"`
Annotations map[string]string `json:"annotations,omitempty"`
PITR PITRSpec `json:"pitr,omitempty"`
BackoffLimit *int32 `json:"backoffLimit,omitempty"`
ActiveDeadlineSeconds *int64 `json:"activeDeadlineSeconds,omitempty"`
StartingDeadlineSeconds *int64 `json:"startingDeadlineSeconds,omitempty"`
SuspendedDeadlineSeconds *int64 `json:"suspendedDeadlineSeconds,omitempty"`
}

func (b *PXCScheduledBackup) GetAllowParallel() bool {
Expand Down
14 changes: 12 additions & 2 deletions pkg/apis/pxc/v1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

99 changes: 54 additions & 45 deletions pkg/controller/pxcbackup/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,14 +151,6 @@ func (r *ReconcilePerconaXtraDBClusterBackup) Reconcile(ctx context.Context, req
return rr, nil
}

if err := r.checkStartingDeadline(ctx, cr); err != nil {
if err := r.setFailedStatus(ctx, cr, err); err != nil {
return rr, errors.Wrap(err, "update status")
}

return reconcile.Result{}, nil
}

cluster, err := r.getCluster(ctx, cr)
if err != nil {
return reconcile.Result{}, errors.Wrap(err, "get cluster")
Expand Down Expand Up @@ -187,6 +179,21 @@ func (r *ReconcilePerconaXtraDBClusterBackup) Reconcile(ctx context.Context, req
return reconcile.Result{}, err
}

if err := r.checkDeadlines(ctx, cluster, cr); err != nil {
if err := r.setFailedStatus(ctx, cr, err); err != nil {
return rr, errors.Wrap(err, "update status")
}

if errors.Is(err, errSuspendedDeadlineExceeded) {
log.Info("cleaning up suspended backup job")
if err := r.cleanUpSuspendedJob(ctx, cluster, cr); err != nil {
return reconcile.Result{}, errors.Wrap(err, "clean up suspended job")
}
}

return reconcile.Result{}, nil
}

if err := r.reconcileBackupJob(ctx, cr, cluster); err != nil {
return rr, errors.Wrap(err, "reconcile backup job")
}
Expand Down Expand Up @@ -661,29 +668,6 @@ func (r *ReconcilePerconaXtraDBClusterBackup) updateJobStatus(
return nil
}

func (r *ReconcilePerconaXtraDBClusterBackup) checkStartingDeadline(ctx context.Context, cr *api.PerconaXtraDBClusterBackup) error {
log := logf.FromContext(ctx)

since := time.Since(cr.CreationTimestamp.Time).Seconds()

if cr.Spec.StartingDeadlineSeconds == nil {
return nil
}

if since < float64(*cr.Spec.StartingDeadlineSeconds) {
return nil
}

if cr.Status.State == api.BackupNew {
log.Info("Backup didn't start in startingDeadlineSeconds, failing the backup",
"startingDeadlineSeconds", *cr.Spec.StartingDeadlineSeconds,
"passedSeconds", since)
return errors.New("starting deadline seconds exceeded")
}

return nil
}

func (r *ReconcilePerconaXtraDBClusterBackup) updateStatus(ctx context.Context, cr *api.PerconaXtraDBClusterBackup) error {
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
localCr := new(api.PerconaXtraDBClusterBackup)
Expand Down Expand Up @@ -726,13 +710,8 @@ func (r *ReconcilePerconaXtraDBClusterBackup) suspendJobIfNeeded(

log := logf.FromContext(ctx)

labelKeyBackupType := naming.GetLabelBackupType(cluster)
jobName := naming.BackupJobName(cr.Name, cr.Labels[labelKeyBackupType] == "cron")

err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
job := new(batchv1.Job)

err := r.client.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: jobName}, job)
job, err := r.getBackupJob(ctx, cluster, cr)
if err != nil {
if k8sErrors.IsNotFound(err) {
return nil
Expand All @@ -752,7 +731,7 @@ func (r *ReconcilePerconaXtraDBClusterBackup) suspendJobIfNeeded(
}

log.Info("Suspending backup job",
"job", jobName,
"job", job.Name,
"clusterStatus", cluster.Status.Status,
"readyPXC", cluster.Status.PXC.Ready)

Expand Down Expand Up @@ -785,13 +764,8 @@ func (r *ReconcilePerconaXtraDBClusterBackup) resumeJobIfNeeded(

log := logf.FromContext(ctx)

labelKeyBackupType := naming.GetLabelBackupType(cluster)
jobName := naming.BackupJobName(cr.Name, cr.Labels[labelKeyBackupType] == "cron")

err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
job := new(batchv1.Job)

err := r.client.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: jobName}, job)
job, err := r.getBackupJob(ctx, cluster, cr)
if err != nil {
if k8sErrors.IsNotFound(err) {
return nil
Expand All @@ -811,7 +785,7 @@ func (r *ReconcilePerconaXtraDBClusterBackup) resumeJobIfNeeded(
}

log.Info("Resuming backup job",
"job", jobName,
"job", job.Name,
"clusterStatus", cluster.Status.Status,
"readyPXC", cluster.Status.PXC.Ready)

Expand All @@ -838,3 +812,38 @@ func (r *ReconcilePerconaXtraDBClusterBackup) reconcileBackupJob(

return nil
}

func (r *ReconcilePerconaXtraDBClusterBackup) getBackupJob(
ctx context.Context,
cluster *api.PerconaXtraDBCluster,
cr *api.PerconaXtraDBClusterBackup,
) (*batchv1.Job, error) {
labelKeyBackupType := naming.GetLabelBackupType(cluster)
jobName := naming.BackupJobName(cr.Name, cr.Labels[labelKeyBackupType] == "cron")

job := new(batchv1.Job)

err := r.client.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: jobName}, job)
if err != nil {
return nil, err
}

return job, nil
}

func (r *ReconcilePerconaXtraDBClusterBackup) cleanUpSuspendedJob(
ctx context.Context,
cluster *api.PerconaXtraDBCluster,
cr *api.PerconaXtraDBClusterBackup,
) error {
job, err := r.getBackupJob(ctx, cluster, cr)
if err != nil {
return errors.Wrap(err, "get job")
}

if err := r.client.Delete(ctx, job); err != nil {
return errors.Wrap(err, "delete job")
}

return nil
}
Loading

0 comments on commit 5872a54

Please sign in to comment.