Skip to content

Commit

Permalink
fix: cherry-pick local zone fix to v0.32.x (#5327)
Browse files Browse the repository at this point in the history
  • Loading branch information
jmdeal authored Dec 16, 2023
1 parent c89ac2c commit c3d15e8
Showing 1 changed file with 23 additions and 32 deletions.
55 changes: 23 additions & 32 deletions pkg/providers/instancetype/instancetype.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ import (
const (
InstanceTypesCacheKey = "types"
InstanceTypeOfferingsCacheKey = "offerings"
AvailabilityZonesCacheKey = "zones"
ZonesCacheKey = "zones"
)

type Provider struct {
Expand Down Expand Up @@ -97,12 +97,9 @@ func (p *Provider) List(ctx context.Context, kc *corev1beta1.KubeletConfiguratio
if err != nil {
return nil, err
}
// Get AvailabilityZones from EC2
availabilityZones, err := p.getAvailabilityZones(ctx)
if err != nil {
return nil, err
}
// Constrain AZs from subnets
// Get zones from instancetypeOfferings
zones := p.getZones(ctx, instanceTypeOfferings)
// Constrain zones from subnets
subnets, err := p.subnetProvider.List(ctx, nodeClass)
if err != nil {
return nil, err
Expand All @@ -121,7 +118,7 @@ func (p *Provider) List(ctx context.Context, kc *corev1beta1.KubeletConfiguratio
return item.([]*cloudprovider.InstanceType), nil
}
result := lo.Map(instanceTypes, func(i *ec2.InstanceTypeInfo, _ int) *cloudprovider.InstanceType {
return NewInstanceType(ctx, i, kc, p.region, nodeClass, p.createOfferings(ctx, i, instanceTypeOfferings[aws.StringValue(i.InstanceType)], availabilityZones, subnetZones))
return NewInstanceType(ctx, i, kc, p.region, nodeClass, p.createOfferings(ctx, i, instanceTypeOfferings[aws.StringValue(i.InstanceType)], zones, subnetZones))
})
for _, instanceType := range instanceTypes {
InstanceTypeVCPU.With(prometheus.Labels{
Expand All @@ -142,18 +139,18 @@ func (p *Provider) LivenessProbe(req *http.Request) error {
return p.pricingProvider.LivenessProbe(req)
}

func (p *Provider) createOfferings(ctx context.Context, instanceType *ec2.InstanceTypeInfo, instanceTypeZones, availabilityZones, subnetZones sets.Set[string]) []cloudprovider.Offering {
func (p *Provider) createOfferings(ctx context.Context, instanceType *ec2.InstanceTypeInfo, instanceTypeZones, zones, subnetZones sets.Set[string]) []cloudprovider.Offering {
var offerings []cloudprovider.Offering
for az := range availabilityZones {
for zone := range zones {
// while usage classes should be a distinct set, there's no guarantee of that
for capacityType := range sets.NewString(aws.StringValueSlice(instanceType.SupportedUsageClasses)...) {
// exclude any offerings that have recently seen an insufficient capacity error from EC2
isUnavailable := p.unavailableOfferings.IsUnavailable(*instanceType.InstanceType, az, capacityType)
isUnavailable := p.unavailableOfferings.IsUnavailable(*instanceType.InstanceType, zone, capacityType)
var price float64
var ok bool
switch capacityType {
case ec2.UsageClassTypeSpot:
price, ok = p.pricingProvider.SpotPrice(*instanceType.InstanceType, az)
price, ok = p.pricingProvider.SpotPrice(*instanceType.InstanceType, zone)
case ec2.UsageClassTypeOnDemand:
price, ok = p.pricingProvider.OnDemandPrice(*instanceType.InstanceType)
case "capacity-block":
Expand All @@ -163,9 +160,9 @@ func (p *Provider) createOfferings(ctx context.Context, instanceType *ec2.Instan
logging.FromContext(ctx).Errorf("Received unknown capacity type %s for instance type %s", capacityType, *instanceType.InstanceType)
continue
}
available := !isUnavailable && ok && instanceTypeZones.Has(az) && subnetZones.Has(az)
available := !isUnavailable && ok && instanceTypeZones.Has(zone) && subnetZones.Has(zone)
offerings = append(offerings, cloudprovider.Offering{
Zone: az,
Zone: zone,
CapacityType: capacityType,
Price: price,
Available: available,
Expand All @@ -175,34 +172,28 @@ func (p *Provider) createOfferings(ctx context.Context, instanceType *ec2.Instan
return offerings
}

func (p *Provider) getAvailabilityZones(ctx context.Context) (sets.Set[string], error) {
func (p *Provider) getZones(ctx context.Context, instanceTypeOfferings map[string]sets.Set[string]) sets.Set[string] {
// DO NOT REMOVE THIS LOCK ----------------------------------------------------------------------------
// We lock here so that multiple callers to getAvailabilityZones do not result in cache misses and multiple
// calls to EC2 when we could have just made one call.
// TODO @joinnis: This can be made more efficient by holding a Read lock and only obtaining the Write if not in cache
p.mu.Lock()
defer p.mu.Unlock()
if cached, ok := p.cache.Get(AvailabilityZonesCacheKey); ok {
return cached.(sets.Set[string]), nil
}

// Get zones from EC2
instanceTypeZones := sets.Set[string]{}
output, err := p.ec2api.DescribeAvailabilityZonesWithContext(ctx, &ec2.DescribeAvailabilityZonesInput{})
if err != nil {
return nil, fmt.Errorf("describing availability zones, %w", err)
if cached, ok := p.cache.Get(ZonesCacheKey); ok {
return cached.(sets.Set[string])
}
for i := range output.AvailabilityZones {
zone := output.AvailabilityZones[i]
if aws.StringValue(zone.ZoneType) == "availability-zone" {
instanceTypeZones.Insert(aws.StringValue(zone.ZoneName))
// Get zones from offerings
zones := sets.Set[string]{}
for _, offeringZones := range instanceTypeOfferings {
for zone := range offeringZones {
zones.Insert(zone)
}
}
if p.cm.HasChanged("zones", instanceTypeZones) {
logging.FromContext(ctx).With("zones", instanceTypeZones.UnsortedList()).Debugf("discovered availability zones")
if p.cm.HasChanged("zones", zones) {
logging.FromContext(ctx).With("zones", zones.UnsortedList()).Debugf("discovered zones")
}
p.cache.Set(AvailabilityZonesCacheKey, instanceTypeZones, 24*time.Hour)
return instanceTypeZones, nil
p.cache.Set(ZonesCacheKey, zones, 24*time.Hour)
return zones
}

func (p *Provider) getInstanceTypeOfferings(ctx context.Context) (map[string]sets.Set[string], error) {
Expand Down

0 comments on commit c3d15e8

Please sign in to comment.