Skip to content

Commit

Permalink
test updates
Browse files Browse the repository at this point in the history
  • Loading branch information
jmdeal committed Oct 16, 2023
1 parent d9d6b11 commit 16a1b5e
Show file tree
Hide file tree
Showing 12 changed files with 70 additions and 44 deletions.
18 changes: 10 additions & 8 deletions pkg/operator/options/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,19 +40,21 @@ type Options struct {
IsolatedVPC bool
VMMemoryOverheadPercent float64
InterruptionQueueName string
ReservedENIs int

setFlags map[string]bool
}

func (o *Options) AddFlags(fs *flag.FlagSet) {
fs.StringVar(&o.AssumeRoleARN, "aws-assume-role-arn", env.WithDefaultString("AWS_ASSUME_ROLE_ARN", ""), "Role to assume for calling AWS services.")
fs.DurationVar(&o.AssumeRoleDuration, "aws-assume-role-duration", env.WithDefaultDuration("AWS_ASSUME_ROLE_DURATION", 15*time.Minute), "Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless aws.assumeRole set.")
fs.StringVar(&o.ClusterCABundle, "aws-cluster-ca-bundle", env.WithDefaultString("AWS_CLUSTER_CA_BUNDLE", ""), "Cluster CA bundle for nodes to use for TLS connections with the API server. If not set, this is taken from the controller's TLS configuration.")
fs.StringVar(&o.ClusterName, "aws-cluster-name", env.WithDefaultString("AWS_CLUSTER_NAME", ""), "[REQUIRED] The kubernetes cluster name for resource discovery.")
fs.StringVar(&o.ClusterEndpoint, "aws-cluster-endpoint", env.WithDefaultString("AWS_CLUSTER_ENDPOINT", ""), "The external kubernetes cluster endpoint for new nodes to connect with. If not specified, will discover the cluster endpoint using DescribeCluster API.")
fs.BoolVar(&o.IsolatedVPC, "aws-isolated-vpc", env.WithDefaultBool("AWS_ISOLATED_VPC", false), "If true, then assume we can't reach AWS services which don't have a VPC endpoint. This also has the effect of disabling look-ups to the AWS pricing endpoint.")
fs.Float64Var(&o.VMMemoryOverheadPercent, "aws-vm-memory-overhead-percent", env.WithDefaultFloat64("AWS_VM_MEMORY_OVERHEAD_PERCENT", 0.075), "The VM memory overhead as a percent that will be subtracted from the total memory for all instance types.")
fs.StringVar(&o.InterruptionQueueName, "aws-interruption-queue-name", env.WithDefaultString("AWS_INTERRUPTION_QUEUE_NAME", ""), "aws.interruptionQueueName is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.")
fs.StringVar(&o.AssumeRoleARN, "assume-role-arn", env.WithDefaultString("ASSUME_ROLE_ARN", ""), "Role to assume for calling AWS services.")
fs.DurationVar(&o.AssumeRoleDuration, "assume-role-duration", env.WithDefaultDuration("ASSUME_ROLE_DURATION", 15*time.Minute), "Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless aws.assumeRole set.")
fs.StringVar(&o.ClusterCABundle, "cluster-ca-bundle", env.WithDefaultString("CLUSTER_CA_BUNDLE", ""), "Cluster CA bundle for nodes to use for TLS connections with the API server. If not set, this is taken from the controller's TLS configuration.")
fs.StringVar(&o.ClusterName, "cluster-name", env.WithDefaultString("CLUSTER_NAME", ""), "[REQUIRED] The kubernetes cluster name for resource discovery.")
fs.StringVar(&o.ClusterEndpoint, "cluster-endpoint", env.WithDefaultString("CLUSTER_ENDPOINT", ""), "The external kubernetes cluster endpoint for new nodes to connect with. If not specified, will discover the cluster endpoint using DescribeCluster API.")
fs.BoolVar(&o.IsolatedVPC, "isolated-vpc", env.WithDefaultBool("ISOLATED_VPC", false), "If true, then assume we can't reach AWS services which don't have a VPC endpoint. This also has the effect of disabling look-ups to the AWS pricing endpoint.")
fs.Float64Var(&o.VMMemoryOverheadPercent, "vm-memory-overhead-percent", env.WithDefaultFloat64("VM_MEMORY_OVERHEAD_PERCENT", 0.075), "The VM memory overhead as a percent that will be subtracted from the total memory for all instance types.")
fs.StringVar(&o.InterruptionQueueName, "interruption-queue-name", env.WithDefaultString("INTERRUPTION_QUEUE_NAME", ""), "Interruption queue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.")
fs.IntVar(&o.ReservedENIs, "reserved-enis", env.WithDefaultInt("RESERVED_ENIS", 0), "Reserved ENIs are not included in the calculations for max-pods or kube-reserved. This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html.")
}

func (o *Options) Parse(fs *flag.FlagSet, args ...string) error {
Expand Down
5 changes: 3 additions & 2 deletions pkg/providers/instance/nodeclass_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package instance_test

import (
"fmt"
"os"
"time"

"github.com/aws/aws-sdk-go/aws"
Expand Down Expand Up @@ -100,7 +101,7 @@ var _ = Describe("NodeClass/InstanceProvider", func() {
},
Tags: []*ec2.Tag{
{
Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)),
Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", lo.Must(os.LookupEnv("CLUSTER_NAME")))),
Value: aws.String("owned"),
},
{
Expand All @@ -109,7 +110,7 @@ var _ = Describe("NodeClass/InstanceProvider", func() {
},
{
Key: aws.String(corev1beta1.ManagedByAnnotationKey),
Value: aws.String(settings.FromContext(ctx).ClusterName),
Value: aws.String(lo.Must(os.LookupEnv("CLUSTER_NAME"))),
},
},
PrivateDnsName: aws.String(fake.PrivateDNSName()),
Expand Down
5 changes: 3 additions & 2 deletions pkg/providers/instance/nodetemplate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package instance_test

import (
"fmt"
"os"
"time"

"github.com/aws/aws-sdk-go/aws"
Expand Down Expand Up @@ -114,7 +115,7 @@ var _ = Describe("NodeTemplate/InstanceProvider", func() {
},
Tags: []*ec2.Tag{
{
Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)),
Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", lo.Must(os.LookupEnv("CLUSTER_NAME")))),
Value: aws.String("owned"),
},
{
Expand All @@ -123,7 +124,7 @@ var _ = Describe("NodeTemplate/InstanceProvider", func() {
},
{
Key: aws.String(v1alpha5.MachineManagedByAnnotationKey),
Value: aws.String(settings.FromContext(ctx).ClusterName),
Value: aws.String(lo.Must(os.LookupEnv("CLUSTER_NAME"))),
},
},
PrivateDnsName: aws.String(fake.PrivateDNSName()),
Expand Down
17 changes: 9 additions & 8 deletions pkg/providers/instance/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package instance_test
import (
"context"
"fmt"
"os"
"testing"
"time"

Expand Down Expand Up @@ -92,7 +93,7 @@ var _ = Describe("Combined/InstanceProvider", func() {
},
Tags: []*ec2.Tag{
{
Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)),
Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", lo.Must(os.LookupEnv("CLUSTER_NAME")))),
Value: aws.String("owned"),
},
{
Expand All @@ -101,7 +102,7 @@ var _ = Describe("Combined/InstanceProvider", func() {
},
{
Key: aws.String(v1alpha5.MachineManagedByAnnotationKey),
Value: aws.String(settings.FromContext(ctx).ClusterName),
Value: aws.String(lo.Must(os.LookupEnv("CLUSTER_NAME"))),
},
},
PrivateDnsName: aws.String(fake.PrivateDNSName()),
Expand All @@ -127,7 +128,7 @@ var _ = Describe("Combined/InstanceProvider", func() {
},
Tags: []*ec2.Tag{
{
Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)),
Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", lo.Must(os.LookupEnv("CLUSTER_NAME")))),
Value: aws.String("owned"),
},
{
Expand All @@ -136,7 +137,7 @@ var _ = Describe("Combined/InstanceProvider", func() {
},
{
Key: aws.String(corev1beta1.ManagedByAnnotationKey),
Value: aws.String(settings.FromContext(ctx).ClusterName),
Value: aws.String(lo.Must(os.LookupEnv("CLUSTER_NAME"))),
},
},
PrivateDnsName: aws.String(fake.PrivateDNSName()),
Expand Down Expand Up @@ -194,7 +195,7 @@ var _ = Describe("Combined/InstanceProvider", func() {
},
Tags: []*ec2.Tag{
{
Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)),
Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", lo.Must(os.LookupEnv("CLUSTER_NAME")))),
Value: aws.String("owned"),
},
{
Expand All @@ -203,7 +204,7 @@ var _ = Describe("Combined/InstanceProvider", func() {
},
{
Key: aws.String(v1alpha5.MachineManagedByAnnotationKey),
Value: aws.String(settings.FromContext(ctx).ClusterName),
Value: aws.String(lo.Must(os.LookupEnv("CLUSTER_NAME"))),
},
},
PrivateDnsName: aws.String(fake.PrivateDNSName()),
Expand All @@ -229,7 +230,7 @@ var _ = Describe("Combined/InstanceProvider", func() {
},
Tags: []*ec2.Tag{
{
Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)),
Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", lo.Must(os.LookupEnv("CLUSTER_NAME")))),
Value: aws.String("owned"),
},
{
Expand All @@ -238,7 +239,7 @@ var _ = Describe("Combined/InstanceProvider", func() {
},
{
Key: aws.String(corev1beta1.ManagedByAnnotationKey),
Value: aws.String(settings.FromContext(ctx).ClusterName),
Value: aws.String(lo.Must(os.LookupEnv("CLUSTER_NAME"))),
},
},
PrivateDnsName: aws.String(fake.PrivateDNSName()),
Expand Down
19 changes: 6 additions & 13 deletions pkg/providers/instancetype/nodeclass_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (

corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1"
corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider"
"github.com/aws/karpenter/pkg/operator/options"
"github.com/aws/karpenter-core/pkg/scheduling"
coretest "github.com/aws/karpenter-core/pkg/test"
. "github.com/aws/karpenter-core/pkg/test/expectations"
Expand Down Expand Up @@ -645,16 +646,8 @@ var _ = Describe("NodeClass/InstanceTypes", func() {
Context("Overhead", func() {
var info *ec2.InstanceTypeInfo
BeforeEach(func() {
ctx, err := (&settings.Settings{}).Inject(ctx, &v1.ConfigMap{
Data: map[string]string{
"aws.clusterName": "karpenter-cluster",
},
})
Expect(err).To(BeNil())

s := settings.FromContext(ctx)
ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{
VMMemoryOverheadPercent: &s.VMMemoryOverheadPercent,
ctx = options.ToContext(ctx, test.Options(test.OptionsFields{
ClusterName: lo.ToPtr("CLUSTER_NAME"),
}))

var ok bool
Expand Down Expand Up @@ -713,7 +706,7 @@ var _ = Describe("NodeClass/InstanceTypes", func() {
})
Context("Eviction Thresholds", func() {
BeforeEach(func() {
ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{
ctx = options.ToContext(ctx, test.Options(test.OptionsFields{
VMMemoryOverheadPercent: lo.ToPtr[float64](0),
}))
})
Expand Down Expand Up @@ -950,7 +943,7 @@ var _ = Describe("NodeClass/InstanceTypes", func() {
}
})
It("should reserve ENIs when aws.reservedENIs is set and is used in max-pods calculation", func() {
ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{
ctx = options.ToContext(ctx, test.Options(test.OptionsFields{
ReservedENIs: lo.ToPtr(1),
}))

Expand All @@ -970,7 +963,7 @@ var _ = Describe("NodeClass/InstanceTypes", func() {
Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", maxPods))
})
It("should reserve ENIs when aws.reservedENIs is set and not go below 0 ENIs in max-pods calculation", func() {
ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{
ctx = options.ToContext(ctx, test.Options(test.OptionsFields{
ReservedENIs: lo.ToPtr(1_000_000),
}))

Expand Down
4 changes: 2 additions & 2 deletions pkg/providers/instancetype/nodetemplate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -983,7 +983,7 @@ var _ = Describe("NodeTemplate/InstanceTypes", func() {
}
})
It("should reserve ENIs when aws.reservedENIs is set and is used in max-pods calculation", func() {
ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{
ctx = options.ToContext(ctx, test.Options(test.OptionsFields{
ReservedENIs: lo.ToPtr(1),
}))

Expand All @@ -1003,7 +1003,7 @@ var _ = Describe("NodeTemplate/InstanceTypes", func() {
Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", maxPods))
})
It("should reserve ENIs when aws.reservedENIs is set and not go below 0 ENIs in max-pods calculation", func() {
ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{
ctx = options.ToContext(ctx, test.Options(test.OptionsFields{
ReservedENIs: lo.ToPtr(1_000_000),
}))

Expand Down
2 changes: 1 addition & 1 deletion pkg/providers/instancetype/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ func generateSpotPricing(cp *cloudprovider.CloudProvider, nodePool *corev1beta1.

func makeFakeInstances() []*ec2.InstanceTypeInfo {
var instanceTypes []*ec2.InstanceTypeInfo
ctx := settings.ToContext(context.Background(), &settings.Settings{IsolatedVPC: true})
ctx := options.ToContext(context.Background(), &options.Options{IsolatedVPC: true})
// Use keys from the static pricing data so that we guarantee pricing for the data
// Create uniform instance data so all of them schedule for a given pod
for _, it := range pricing.NewProvider(ctx, nil, nil, "us-east-1").InstanceTypes() {
Expand Down
4 changes: 2 additions & 2 deletions pkg/providers/instancetype/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,7 @@ func ephemeralStorage(amiFamily amifamily.AMIFamily, blockDeviceMappings []*v1be
func awsPodENI(ctx context.Context, name string) *resource.Quantity {
// https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html#supported-instance-types
limits, ok := Limits[name]
if settings.FromContext(ctx).EnablePodENI && ok && limits.IsTrunkingCompatible {
if ok && limits.IsTrunkingCompatible {
return resources.Quantity(fmt.Sprint(limits.BranchInterface))
}
return resources.Quantity("0")
Expand Down Expand Up @@ -309,7 +309,7 @@ func ENILimitedPods(ctx context.Context, info *ec2.InstanceTypeInfo) *resource.Q
// VPC CNI only uses the default network interface
// https://github.com/aws/amazon-vpc-cni-k8s/blob/3294231c0dce52cfe473bf6c62f47956a3b333b6/scripts/gen_vpc_ip_limits.go#L162
networkInterfaces := *info.NetworkInfo.NetworkCards[*info.NetworkInfo.DefaultNetworkCardIndex].MaximumNetworkInterfaces
usableNetworkInterfaces := lo.Max([]int64{(networkInterfaces - int64(settings.FromContext(ctx).ReservedENIs)), 0})
usableNetworkInterfaces := lo.Max([]int64{(networkInterfaces - int64(options.FromContext(ctx).ReservedENIs)), 0})
if usableNetworkInterfaces == 0 {
return resource.NewQuantity(0, resource.DecimalSI)
}
Expand Down
21 changes: 17 additions & 4 deletions pkg/providers/launchtemplate/nodeclass_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ import (
"github.com/aws/karpenter/pkg/apis/settings"
"github.com/aws/karpenter/pkg/apis/v1beta1"
"github.com/aws/karpenter/pkg/fake"
"github.com/aws/karpenter/pkg/operator/options"
"github.com/aws/karpenter/pkg/providers/amifamily/bootstrap"
"github.com/aws/karpenter/pkg/providers/instancetype"
"github.com/aws/karpenter/pkg/test"
Expand Down Expand Up @@ -692,7 +693,10 @@ var _ = Describe("EC2NodeClass/LaunchTemplates", func() {
It("should calculate memory overhead based on eni limited pods when ENI limited", func() {
ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{
EnableENILimitedPodDensity: lo.ToPtr(false),
VMMemoryOverheadPercent: lo.ToPtr[float64](0),
}))

ctx = options.ToContext(ctx, test.Options(test.OptionsFields{
VMMemoryOverheadPercent: lo.ToPtr[float64](0),
}))

nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2
Expand All @@ -703,7 +707,10 @@ var _ = Describe("EC2NodeClass/LaunchTemplates", func() {
It("should calculate memory overhead based on eni limited pods when not ENI limited", func() {
ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{
EnableENILimitedPodDensity: lo.ToPtr(false),
VMMemoryOverheadPercent: lo.ToPtr[float64](0),
}))

ctx = options.ToContext(ctx, test.Options(test.OptionsFields{
VMMemoryOverheadPercent: lo.ToPtr[float64](0),
}))

nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2
Expand Down Expand Up @@ -742,7 +749,10 @@ var _ = Describe("EC2NodeClass/LaunchTemplates", func() {
It("should calculate memory overhead based on eni limited pods when ENI limited", func() {
ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{
EnableENILimitedPodDensity: lo.ToPtr(true),
VMMemoryOverheadPercent: lo.ToPtr[float64](0),
}))

ctx = options.ToContext(ctx, test.Options(test.OptionsFields{
VMMemoryOverheadPercent: lo.ToPtr[float64](0),
}))

nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket
Expand All @@ -753,7 +763,10 @@ var _ = Describe("EC2NodeClass/LaunchTemplates", func() {
It("should calculate memory overhead based on max pods when not ENI limited", func() {
ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{
EnableENILimitedPodDensity: lo.ToPtr(false),
VMMemoryOverheadPercent: lo.ToPtr[float64](0),
}))

ctx = options.ToContext(ctx, test.Options(test.OptionsFields{
VMMemoryOverheadPercent: lo.ToPtr[float64](0),
}))

nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket
Expand Down
13 changes: 13 additions & 0 deletions pkg/providers/launchtemplate/nodetemplate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import (
"github.com/aws/karpenter/pkg/apis/settings"
"github.com/aws/karpenter/pkg/apis/v1alpha1"
"github.com/aws/karpenter/pkg/fake"
"github.com/aws/karpenter/pkg/operator/options"
"github.com/aws/karpenter/pkg/providers/amifamily/bootstrap"
"github.com/aws/karpenter/pkg/providers/instancetype"
"github.com/aws/karpenter/pkg/test"
Expand Down Expand Up @@ -707,6 +708,9 @@ var _ = Describe("NodeTemplate/LaunchTemplates", func() {
It("should calculate memory overhead based on eni limited pods when ENI limited", func() {
ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{
EnableENILimitedPodDensity: lo.ToPtr(false),
}))

ctx = options.ToContext(ctx, test.Options(test.OptionsFields{
VMMemoryOverheadPercent: lo.ToPtr[float64](0),
}))

Expand All @@ -718,6 +722,9 @@ var _ = Describe("NodeTemplate/LaunchTemplates", func() {
It("should calculate memory overhead based on eni limited pods when not ENI limited", func() {
ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{
EnableENILimitedPodDensity: lo.ToPtr(false),
}))

ctx = options.ToContext(ctx, test.Options(test.OptionsFields{
VMMemoryOverheadPercent: lo.ToPtr[float64](0),
}))

Expand Down Expand Up @@ -757,6 +764,9 @@ var _ = Describe("NodeTemplate/LaunchTemplates", func() {
It("should calculate memory overhead based on eni limited pods when ENI limited", func() {
ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{
EnableENILimitedPodDensity: lo.ToPtr(true),
}))

ctx = options.ToContext(ctx, test.Options(test.OptionsFields{
VMMemoryOverheadPercent: lo.ToPtr[float64](0),
}))

Expand All @@ -768,6 +778,9 @@ var _ = Describe("NodeTemplate/LaunchTemplates", func() {
It("should calculate memory overhead based on max pods when not ENI limited", func() {
ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{
EnableENILimitedPodDensity: lo.ToPtr(false),
}))

ctx = options.ToContext(ctx, test.Options(test.OptionsFields{
VMMemoryOverheadPercent: lo.ToPtr[float64](0),
}))

Expand Down
2 changes: 2 additions & 0 deletions pkg/test/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ type OptionsFields struct {
IsolatedVPC *bool
VMMemoryOverheadPercent *float64
InterruptionQueueName *string
ReservedENIs *int
}

func Options(overrides ...OptionsFields) *options.Options {
Expand All @@ -51,5 +52,6 @@ func Options(overrides ...OptionsFields) *options.Options {
IsolatedVPC: lo.FromPtrOr(opts.IsolatedVPC, false),
VMMemoryOverheadPercent: lo.FromPtrOr(opts.VMMemoryOverheadPercent, 0.075),
InterruptionQueueName: lo.FromPtrOr(opts.InterruptionQueueName, ""),
ReservedENIs: lo.FromPtrOr(opts.ReservedENIs, 0),
}
}
Loading

0 comments on commit 16a1b5e

Please sign in to comment.