From c80fc49073c2206bab0b4ab81d7d76316bb67f3d Mon Sep 17 00:00:00 2001 From: Patrik Egyed Date: Tue, 10 Aug 2021 15:27:22 +0200 Subject: [PATCH 01/11] feat(API,EKS,NP): refactored volumes --- apis/pipeline/pipeline.yaml | 143 +++++++++++++++++++++++++++++++++++- 1 file changed, 140 insertions(+), 3 deletions(-) diff --git a/apis/pipeline/pipeline.yaml b/apis/pipeline/pipeline.yaml index 7c2f6c4617..d5f67e185f 100644 --- a/apis/pipeline/pipeline.yaml +++ b/apis/pipeline/pipeline.yaml @@ -4593,12 +4593,15 @@ components: autoscaling: $ref: '#/components/schemas/NodePoolAutoScaling' volumeEncryption: + deprecated: true $ref: '#/components/schemas/EKSNodePoolVolumeEncryption' volumeSize: + deprecated: true description: Size of the EBS volume in GBs of the nodes in the pool. type: integer example: 50 volumeType: + deprecated: true description: Type of the EBS volume of the nodes in the pool (default gp3). type: string example: gp3 @@ -4624,15 +4627,149 @@ components: type: string example: ["sg-00000xxxx0000xxx1", "sg-00000xxxx0000xxx2"] useInstanceStore: + deprecated: true description: Setup available instance stores (NVMe disks) to use for Kubelet root if available. As a result emptyDir volumes will be provisioned on local instance storage disks. You can check out available instance storages here https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html#instance-store-volumes. type: boolean + volumes: + $ref: '#/components/schemas/EKSNodePoolVolumes' + + EKSNodePoolVolume: + discriminator: + propertyName: storage + mapping: + ebs: '#/components/schemas/EKSNodePoolVolumeEBS' + instance-store: '#/components/schemas/EKSNodePoolVolumeInstanceStore' + nullable: true + oneOf: + - $ref: '#/components/schemas/EKSNodePoolVolumeEBS' + - $ref: '#/components/schemas/EKSNodePoolVolumeInstanceStore' + + EKSNodePoolVolumes: + description: An associative collection of EKS node pool node + instance volume configuration objects keyed by their semantical + volume names (example instanceRoot, kubeletRoot). + example: | + { + "instanceRoot": { + "encryption": { + "enabled": true, + "encryptionKeyARN": "arn:aws:kms:aws-region:012345678901:key/01234567-0123-0123-0123-012345678901" + } + "size": 50 + "storage": "ebs" + "type": "gp3" + }, + "kubeletRoot": { + "storage": "instance-store" + } + } + nullable: true + properties: + instanceRoot: + description: (Optional) The volume configuration of + the EKS node instance's instance root device + (default ebs gp3 50GiB with control plane + configuration default or AWS account default + encryption). + + The storage of the instance root device + volume is constrained by the AMI used to launch + the EKS node pool node instance. + + For more details see + https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/RootDeviceStorage.html. + $ref: '#/components/schemas/EKSNodePoolVolume' + kubeletRoot: + description: (Optional) The volume configuration of + the Kubernetes kubelet root device (default uses + the instance root device for kubelet root as + well). + + The configuration and availability of the + instance store storage type for kubelet root + device volume depends on the EKS node pool node + instance type. + + For more details see + https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html#instance-store-volumes. + $ref: '#/components/schemas/EKSNodePoolVolume' + type: object + + EKSNodePoolVolumeEBS: + description: Configuration of EBS volumes mounted onto an EKS node + pool node instance. + + EBS volumes are the default to be used, if no configuration is + specified. + example: | + { + "encryption": { + "enabled": true, + "encryptionKeyARN": "arn:aws:kms:aws-region:012345678901:key/01234567-0123-0123-0123-012345678901" + } + "size": 50 + "storage": "ebs" + "type": "gp3" + } + properties: + encryption: + $ref: '#/components/schemas/EKSNodePoolVolumeEncryption' + size: + description: Size of the EBS volume in GiBs of the nodes in + the pool (default 50 GiB). + type: integer + example: 50 + storage: + description: Type of the mounted volume's storage on the + node instances of the node pool. + type: string + enum: + - ebs + type: + description: Type of the EBS volume to mount on the EKS node + pool node instances (default gp3). + type: string + example: gp3 + required: + - storage + type: object + + EKSNodePoolVolumeInstanceStore: + description: Configuration of instance store volumes mounted onto + an EKS node pool node instance. + + Instance store volumes are encrypted at rest by default using an + XTS-AES-256 block cipher implemented in a hardware module on the + instance. + + The size and type of an instance store volume is fixed based on + the instance type the volume is attached to. + + For more details see + https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html#instance-store-volumes. + example: | + { + "storage": "instance-store" + } + properties: + storage: + description: Type of the mounted volume's storage on the + node instances of the node pool. + type: string + enum: + - instance-store + required: + - storage + type: object EKSNodePoolVolumeEncryption: - description: Encryption details of the node volumes in an EKS node pool. + description: Encryption details of the instance volumes in an EKS + node pool (default null -> control plane configuration -> AWS + account default). example: | { "enabled": true, - "encryptionKeyARN": "arn:aws:kms::000000000000:key/00000000-0000-0000-0000-000000000000" + "encryptionKeyARN": "arn:aws:kms:aws-region:012345678901:key/01234567-0123-0123-0123-01234567890a" } nullable: true properties: @@ -4642,7 +4779,7 @@ components: type: boolean encryptionKeyARN: description: KMS key ARN to use for node volume encryption. - example: arn:aws:kms::000000000000:key/00000000-0000-0000-0000-000000000000 + example: arn:aws:kms:aws-region:012345678901:key/01234567-0123-0123-0123-01234567890a type: string required: - enabled # Note: to avoid `omitempty` in generation. From e0d63d6bb6efc2160de2656c9212bdc3ff1d1b1b Mon Sep 17 00:00:00 2001 From: Patrik Egyed Date: Tue, 10 Aug 2021 15:27:49 +0200 Subject: [PATCH 02/11] chore(Pipeline,OpenAPI): regenerated files --- .gen/pipeline/api/openapi.yaml | 104 +++++++++++++++++- .../model_create_node_pool_request.go | 2 + .../pipeline/model_eks_node_pool_all_of.go | 2 + .../pipeline/model_eks_node_pool_volume.go | 25 +++++ .../model_eks_node_pool_volume_ebs.go | 26 +++++ .../model_eks_node_pool_volume_encryption.go | 2 +- ...del_eks_node_pool_volume_instance_store.go | 18 +++ .../pipeline/model_eks_node_pool_volumes.go | 19 ++++ .gen/pipeline/pipeline/model_node_pool.go | 2 + .../pipeline/model_node_pool_summary.go | 2 + 10 files changed, 198 insertions(+), 4 deletions(-) create mode 100644 .gen/pipeline/pipeline/model_eks_node_pool_volume.go create mode 100644 .gen/pipeline/pipeline/model_eks_node_pool_volume_ebs.go create mode 100644 .gen/pipeline/pipeline/model_eks_node_pool_volume_instance_store.go create mode 100644 .gen/pipeline/pipeline/model_eks_node_pool_volumes.go diff --git a/.gen/pipeline/api/openapi.yaml b/.gen/pipeline/api/openapi.yaml index d2f02eb6ec..164979c5ee 100644 --- a/.gen/pipeline/api/openapi.yaml +++ b/.gen/pipeline/api/openapi.yaml @@ -18881,12 +18881,105 @@ components: - $ref: '#/components/schemas/EksNodePool_allOf' description: Node pool object for an EKS cluster. type: object + EKSNodePoolVolume: + discriminator: + mapping: + ebs: '#/components/schemas/EKSNodePoolVolumeEBS' + instance-store: '#/components/schemas/EKSNodePoolVolumeInstanceStore' + propertyName: storage + nullable: true + oneOf: + - $ref: '#/components/schemas/EKSNodePoolVolumeEBS' + - $ref: '#/components/schemas/EKSNodePoolVolumeInstanceStore' + EKSNodePoolVolumes: + description: An associative collection of EKS node pool node instance volume + configuration objects keyed by their semantical volume names (example instanceRoot, + kubeletRoot). + example: | + { + "instanceRoot": { + "encryption": { + "enabled": true, + "encryptionKeyARN": "arn:aws:kms:aws-region:012345678901:key/01234567-0123-0123-0123-012345678901" + } + "size": 50 + "storage": "ebs" + "type": "gp3" + }, + "kubeletRoot": { + "storage": "instance-store" + } + } + nullable: true + properties: + instanceRoot: + $ref: '#/components/schemas/EKSNodePoolVolume' + kubeletRoot: + $ref: '#/components/schemas/EKSNodePoolVolume' + type: object + EKSNodePoolVolumeEBS: + description: |- + Configuration of EBS volumes mounted onto an EKS node pool node instance. + EBS volumes are the default to be used, if no configuration is specified. + example: | + { + "encryption": { + "enabled": true, + "encryptionKeyARN": "arn:aws:kms:aws-region:012345678901:key/01234567-0123-0123-0123-012345678901" + } + "size": 50 + "storage": "ebs" + "type": "gp3" + } + properties: + encryption: + $ref: '#/components/schemas/EKSNodePoolVolumeEncryption' + size: + description: Size of the EBS volume in GiBs of the nodes in the pool (default + 50 GiB). + example: 50 + type: integer + storage: + description: Type of the mounted volume's storage on the node instances + of the node pool. + enum: + - ebs + type: string + type: + description: Type of the EBS volume to mount on the EKS node pool node instances + (default gp3). + example: gp3 + type: string + required: + - storage + type: object + EKSNodePoolVolumeInstanceStore: + description: |- + Configuration of instance store volumes mounted onto an EKS node pool node instance. + Instance store volumes are encrypted at rest by default using an XTS-AES-256 block cipher implemented in a hardware module on the instance. + The size and type of an instance store volume is fixed based on the instance type the volume is attached to. + For more details see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html#instance-store-volumes. + example: | + { + "storage": "instance-store" + } + properties: + storage: + description: Type of the mounted volume's storage on the node instances + of the node pool. + enum: + - instance-store + type: string + required: + - storage + type: object EKSNodePoolVolumeEncryption: - description: Encryption details of the node volumes in an EKS node pool. + description: Encryption details of the instance volumes in an EKS node pool + (default null -> control plane configuration -> AWS account default). example: | { "enabled": true, - "encryptionKeyARN": "arn:aws:kms::000000000000:key/00000000-0000-0000-0000-000000000000" + "encryptionKeyARN": "arn:aws:kms:aws-region:012345678901:key/01234567-0123-0123-0123-01234567890a" } nullable: true properties: @@ -18896,7 +18989,7 @@ components: type: boolean encryptionKeyARN: description: KMS key ARN to use for node volume encryption. - example: arn:aws:kms::000000000000:key/00000000-0000-0000-0000-000000000000 + example: arn:aws:kms:aws-region:012345678901:key/01234567-0123-0123-0123-01234567890a type: string required: - enabled @@ -23005,10 +23098,12 @@ components: volumeEncryption: $ref: '#/components/schemas/EKSNodePoolVolumeEncryption' volumeSize: + deprecated: true description: Size of the EBS volume in GBs of the nodes in the pool. example: 50 type: integer volumeType: + deprecated: true description: Type of the EBS volume of the nodes in the pool (default gp3). example: gp3 type: string @@ -23039,11 +23134,14 @@ components: type: string type: array useInstanceStore: + deprecated: true description: Setup available instance stores (NVMe disks) to use for Kubelet root if available. As a result emptyDir volumes will be provisioned on local instance storage disks. You can check out available instance storages here https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html#instance-store-volumes. type: boolean + volumes: + $ref: '#/components/schemas/EKSNodePoolVolumes' required: - instanceType EksUpdateNodePoolRequest_allOf: diff --git a/.gen/pipeline/pipeline/model_create_node_pool_request.go b/.gen/pipeline/pipeline/model_create_node_pool_request.go index 43c3decc63..fecc305053 100644 --- a/.gen/pipeline/pipeline/model_create_node_pool_request.go +++ b/.gen/pipeline/pipeline/model_create_node_pool_request.go @@ -48,5 +48,7 @@ type CreateNodePoolRequest struct { // Setup available instance stores (NVMe disks) to use for Kubelet root if available. As a result emptyDir volumes will be provisioned on local instance storage disks. You can check out available instance storages here https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html#instance-store-volumes. UseInstanceStore bool `json:"useInstanceStore,omitempty"` + Volumes *EksNodePoolVolumes `json:"volumes,omitempty"` + NodePools map[string]NodePool `json:"nodePools,omitempty"` } diff --git a/.gen/pipeline/pipeline/model_eks_node_pool_all_of.go b/.gen/pipeline/pipeline/model_eks_node_pool_all_of.go index 735134f412..0997debfdb 100644 --- a/.gen/pipeline/pipeline/model_eks_node_pool_all_of.go +++ b/.gen/pipeline/pipeline/model_eks_node_pool_all_of.go @@ -38,4 +38,6 @@ type EksNodePoolAllOf struct { // Setup available instance stores (NVMe disks) to use for Kubelet root if available. As a result emptyDir volumes will be provisioned on local instance storage disks. You can check out available instance storages here https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html#instance-store-volumes. UseInstanceStore bool `json:"useInstanceStore,omitempty"` + + Volumes *EksNodePoolVolumes `json:"volumes,omitempty"` } diff --git a/.gen/pipeline/pipeline/model_eks_node_pool_volume.go b/.gen/pipeline/pipeline/model_eks_node_pool_volume.go new file mode 100644 index 0000000000..15f0eb1c11 --- /dev/null +++ b/.gen/pipeline/pipeline/model_eks_node_pool_volume.go @@ -0,0 +1,25 @@ +/* + * Pipeline API + * + * Pipeline is a feature rich application platform, built for containers on top of Kubernetes to automate the DevOps experience, continuous application development and the lifecycle of deployments. + * + * API version: latest + * Contact: info@banzaicloud.com + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package pipeline + +type EksNodePoolVolume struct { + + Encryption *EksNodePoolVolumeEncryption `json:"encryption,omitempty"` + + // Size of the EBS volume in GiBs of the nodes in the pool (default 50 GiB). + Size int32 `json:"size,omitempty"` + + // Type of the mounted volume's storage on the node instances of the node pool. + Storage string `json:"storage"` + + // Type of the EBS volume to mount on the EKS node pool node instances (default gp3). + Type string `json:"type,omitempty"` +} diff --git a/.gen/pipeline/pipeline/model_eks_node_pool_volume_ebs.go b/.gen/pipeline/pipeline/model_eks_node_pool_volume_ebs.go new file mode 100644 index 0000000000..f7f3424e92 --- /dev/null +++ b/.gen/pipeline/pipeline/model_eks_node_pool_volume_ebs.go @@ -0,0 +1,26 @@ +/* + * Pipeline API + * + * Pipeline is a feature rich application platform, built for containers on top of Kubernetes to automate the DevOps experience, continuous application development and the lifecycle of deployments. + * + * API version: latest + * Contact: info@banzaicloud.com + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package pipeline + +// EksNodePoolVolumeEbs - Configuration of EBS volumes mounted onto an EKS node pool node instance. EBS volumes are the default to be used, if no configuration is specified. +type EksNodePoolVolumeEbs struct { + + Encryption *EksNodePoolVolumeEncryption `json:"encryption,omitempty"` + + // Size of the EBS volume in GiBs of the nodes in the pool (default 50 GiB). + Size int32 `json:"size,omitempty"` + + // Type of the mounted volume's storage on the node instances of the node pool. + Storage string `json:"storage"` + + // Type of the EBS volume to mount on the EKS node pool node instances (default gp3). + Type string `json:"type,omitempty"` +} diff --git a/.gen/pipeline/pipeline/model_eks_node_pool_volume_encryption.go b/.gen/pipeline/pipeline/model_eks_node_pool_volume_encryption.go index ff59dd4837..be7e5c142e 100644 --- a/.gen/pipeline/pipeline/model_eks_node_pool_volume_encryption.go +++ b/.gen/pipeline/pipeline/model_eks_node_pool_volume_encryption.go @@ -10,7 +10,7 @@ package pipeline -// EksNodePoolVolumeEncryption - Encryption details of the node volumes in an EKS node pool. +// EksNodePoolVolumeEncryption - Encryption details of the instance volumes in an EKS node pool (default null -> control plane configuration -> AWS account default). type EksNodePoolVolumeEncryption struct { // Indicator of encrypted node pool node volumes. diff --git a/.gen/pipeline/pipeline/model_eks_node_pool_volume_instance_store.go b/.gen/pipeline/pipeline/model_eks_node_pool_volume_instance_store.go new file mode 100644 index 0000000000..59cc5b917e --- /dev/null +++ b/.gen/pipeline/pipeline/model_eks_node_pool_volume_instance_store.go @@ -0,0 +1,18 @@ +/* + * Pipeline API + * + * Pipeline is a feature rich application platform, built for containers on top of Kubernetes to automate the DevOps experience, continuous application development and the lifecycle of deployments. + * + * API version: latest + * Contact: info@banzaicloud.com + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package pipeline + +// EksNodePoolVolumeInstanceStore - Configuration of instance store volumes mounted onto an EKS node pool node instance. Instance store volumes are encrypted at rest by default using an XTS-AES-256 block cipher implemented in a hardware module on the instance. The size and type of an instance store volume is fixed based on the instance type the volume is attached to. For more details see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html#instance-store-volumes. +type EksNodePoolVolumeInstanceStore struct { + + // Type of the mounted volume's storage on the node instances of the node pool. + Storage string `json:"storage"` +} diff --git a/.gen/pipeline/pipeline/model_eks_node_pool_volumes.go b/.gen/pipeline/pipeline/model_eks_node_pool_volumes.go new file mode 100644 index 0000000000..0fd020b66d --- /dev/null +++ b/.gen/pipeline/pipeline/model_eks_node_pool_volumes.go @@ -0,0 +1,19 @@ +/* + * Pipeline API + * + * Pipeline is a feature rich application platform, built for containers on top of Kubernetes to automate the DevOps experience, continuous application development and the lifecycle of deployments. + * + * API version: latest + * Contact: info@banzaicloud.com + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package pipeline + +// EksNodePoolVolumes - An associative collection of EKS node pool node instance volume configuration objects keyed by their semantical volume names (example instanceRoot, kubeletRoot). +type EksNodePoolVolumes struct { + + InstanceRoot *EksNodePoolVolume `json:"instanceRoot,omitempty"` + + KubeletRoot *EksNodePoolVolume `json:"kubeletRoot,omitempty"` +} diff --git a/.gen/pipeline/pipeline/model_node_pool.go b/.gen/pipeline/pipeline/model_node_pool.go index a0089291ee..bbd8e711d3 100644 --- a/.gen/pipeline/pipeline/model_node_pool.go +++ b/.gen/pipeline/pipeline/model_node_pool.go @@ -47,4 +47,6 @@ type NodePool struct { // Setup available instance stores (NVMe disks) to use for Kubelet root if available. As a result emptyDir volumes will be provisioned on local instance storage disks. You can check out available instance storages here https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html#instance-store-volumes. UseInstanceStore bool `json:"useInstanceStore,omitempty"` + + Volumes *EksNodePoolVolumes `json:"volumes,omitempty"` } diff --git a/.gen/pipeline/pipeline/model_node_pool_summary.go b/.gen/pipeline/pipeline/model_node_pool_summary.go index 36eed00a3d..746d9dbdab 100644 --- a/.gen/pipeline/pipeline/model_node_pool_summary.go +++ b/.gen/pipeline/pipeline/model_node_pool_summary.go @@ -49,6 +49,8 @@ type NodePoolSummary struct { // Setup available instance stores (NVMe disks) to use for Kubelet root if available. As a result emptyDir volumes will be provisioned on local instance storage disks. You can check out available instance storages here https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html#instance-store-volumes. UseInstanceStore bool `json:"useInstanceStore,omitempty"` + Volumes *EksNodePoolVolumes `json:"volumes,omitempty"` + // Current status of the node pool. Status string `json:"status,omitempty"` From 208d017f97e7ef3ad7b851989f9f28746df99227 Mon Sep 17 00:00:00 2001 From: Magyari Sandor Szilard Date: Wed, 18 Aug 2021 18:31:02 +0200 Subject: [PATCH 03/11] create nodepool(s): generic volume handling --- .idea/pipeline.iml | 2 +- .../eks/eksadapter/node_pool_processor.go | 31 ++++ .../distribution/eks/ekscluster/eks.go | 97 ++++++++++-- .../eks/eksprovider/driver/cluster_creator.go | 52 +++++-- .../eks/eksprovider/driver/cluster_updater.go | 28 ++-- .../activity_calculate_node_pool_version.go | 53 ++++--- ...tivity_calculate_node_pool_version_test.go | 15 +- .../workflow/create_asg_activity.go | 142 +++++++++++------- .../workflow/create_asg_activity_test.go | 2 - .../workflow/create_node_pool_workflow.go | 21 +-- .../eksworkflow/workflow_update_node_pool.go | 9 +- .../cluster/distribution/eks/node_pool.go | 32 +++- internal/cluster/distribution/eks/service.go | 15 ++ pkg/errors/errors.go | 6 +- src/cluster/eks_update_cluster.go | 7 +- templates/eks/amazon-eks-nodepool-cf.yaml | 135 ++++++++++++++--- 16 files changed, 467 insertions(+), 180 deletions(-) diff --git a/.idea/pipeline.iml b/.idea/pipeline.iml index 6d296e3ccf..78c8edcc55 100644 --- a/.idea/pipeline.iml +++ b/.idea/pipeline.iml @@ -1,6 +1,6 @@ - +