Skip to content

Commit

Permalink
fix network bandwidth script and run codegen
Browse files Browse the repository at this point in the history
  • Loading branch information
@jigisha620
jigisha620 committed Apr 25, 2024
1 parent b103ff0 commit c5e0e25
Show file tree
Hide file tree
Showing 6 changed files with 318 additions and 45 deletions.
75 changes: 41 additions & 34 deletions hack/code/bandwidth_gen/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,13 @@ import (
)

var uriSelectors = map[string]string{
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/general-purpose-instances.html": "#general-purpose-network-performance",
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/compute-optimized-instances.html": "#compute-network-performance",
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/memory-optimized-instances.html": "#memory-network-perf",
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/storage-optimized-instances.html": "#storage-network-performance",
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/accelerated-computing-instances.html": "#gpu-network-performance",
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/high-performance-computing-instances.html": "#hpc-network-performance",
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/general-purpose-instances.html": "#gp_network",
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/compute-optimized-instances.html": "#co_network",
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/memory-optimized-instances.html": "#mo_network",
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/storage-optimized-instances.html": "#so_network",
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/accelerated-computing-instances.html": "#ac_network",
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/high-performance-computing-instances.html": "#hpc_network",
"https://docs.aws.amazon.com/ec2/latest/instancetypes/pg.html": "#pg_network",
}

const fileFormat = `
Expand Down Expand Up @@ -70,13 +71,14 @@ func main() {

doc := lo.Must(goquery.NewDocumentFromReader(response.Body))

// grab two tables that contain the network performance values
// first table will contain all the instance type and bandwidth data
// some rows will will have vague describe such as "Very Low", "Low", "Low to Moderate", etc.
// These instance types will can be found on the second table with absolute values in Gbps
// If the instance type is skipped on the first table it will be grabbed on the second table
// grab the table that contains the network performance values
// some rows will have vague description such as "Very Low", "Low", "Low to Moderate", etc.
// These instance types will be ignored
for _, row := range doc.Find(selector).NextAllFiltered(".table-container").Eq(0).Find("tbody").Find("tr").Nodes {
instanceTypeData := row.FirstChild.NextSibling.FirstChild.FirstChild.Data
instanceTypeData := strings.TrimSpace(row.FirstChild.NextSibling.FirstChild.Data)
if !strings.ContainsAny(instanceTypeData, ".") {
continue
}
bandwidthData := row.FirstChild.NextSibling.NextSibling.NextSibling.FirstChild.Data
// exclude all rows that contain any of the following strings
if containsAny(bandwidthData, "Low", "Moderate", "High", "Up to") {
Expand All @@ -92,30 +94,9 @@ func main() {
bandwidth[instanceTypeData] = int64(lo.Must(strconv.ParseFloat(bandwidthSlice[0], 64)) * 1000)
}
}

// Collect instance types bandwidth data from the baseline/bandwidth table underneath the standard table
// The HPC network performance doc is laid out differently than the other docs. There is no table underneath
// the standard table that contains information for network performance with baseline and burst bandwidth.
if selector != "#hpc-network-performance" {
for _, row := range doc.Find(selector).NextAllFiltered(".table-container").Eq(1).Find("tbody").Find("tr").Nodes {
instanceTypeData := row.FirstChild.NextSibling.FirstChild.FirstChild.Data
bandwidthData := row.FirstChild.NextSibling.NextSibling.NextSibling.FirstChild.Data
bandwidth[instanceTypeData] = int64(lo.Must(strconv.ParseFloat(bandwidthData, 64)) * 1000)
}
}
}()
}
if err := os.Setenv("AWS_SDK_LOAD_CONFIG", "true"); err != nil {
log.Fatalf("setting AWS_SDK_LOAD_CONFIG, %s", err)
}
if err := os.Setenv("AWS_REGION", "us-east-1"); err != nil {
log.Fatalf("setting AWS_REGION, %s", err)
}
sess := session.Must(session.NewSession())
ec2api := ec2.New(sess)
instanceTypesOutput := lo.Must(ec2api.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{}))
allInstanceTypes := lo.Map(instanceTypesOutput.InstanceTypes, func(info *ec2.InstanceTypeInfo, _ int) string { return *info.InstanceType })

allInstanceTypes := getAllInstanceTypes()
instanceTypes := lo.Keys(bandwidth)
// 2d sort for readability
sort.Strings(allInstanceTypes)
Expand Down Expand Up @@ -150,3 +131,29 @@ func containsAny(value string, excludedSubstrings ...string) bool {
}
return false
}

func getAllInstanceTypes() []string {
if err := os.Setenv("AWS_SDK_LOAD_CONFIG", "true"); err != nil {
log.Fatalf("setting AWS_SDK_LOAD_CONFIG, %s", err)
}
if err := os.Setenv("AWS_REGION", "us-east-1"); err != nil {
log.Fatalf("setting AWS_REGION, %s", err)
}
sess := session.Must(session.NewSession())
ec2api := ec2.New(sess)
var allInstanceTypes []string

params := &ec2.DescribeInstanceTypesInput{}
// Retrieve the instance types in a loop using NextToken
for {
result := lo.Must(ec2api.DescribeInstanceTypes(params))
allInstanceTypes = append(allInstanceTypes, lo.Map(result.InstanceTypes, func(info *ec2.InstanceTypeInfo, _ int) string { return *info.InstanceType })...)
// Check if they are any instances left
if result.NextToken != nil {
params.NextToken = result.NextToken
} else {
break
}
}
return allInstanceTypes
}
6 changes: 3 additions & 3 deletions pkg/fake/zz_generated.describe_instance_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -366,18 +366,18 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{
EfaInfo: &ec2.EfaInfo{
MaximumEfaInterfaces: aws.Int64(2),
},
MaximumNetworkInterfaces: aws.Int64(14),
MaximumNetworkInterfaces: aws.Int64(16),
Ipv4AddressesPerInterface: aws.Int64(50),
EncryptionInTransitSupported: aws.Bool(true),
DefaultNetworkCardIndex: aws.Int64(0),
NetworkCards: []*ec2.NetworkCardInfo{
{
NetworkCardIndex: aws.Int64(0),
MaximumNetworkInterfaces: aws.Int64(7),
MaximumNetworkInterfaces: aws.Int64(8),
},
{
NetworkCardIndex: aws.Int64(1),
MaximumNetworkInterfaces: aws.Int64(7),
MaximumNetworkInterfaces: aws.Int64(8),
},
},
},
Expand Down
55 changes: 52 additions & 3 deletions pkg/providers/instancetype/zz_generated.bandwidth.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,55 @@ package instancetype

var (
InstanceTypeBandwidthMegabits = map[string]int64{
// c1.medium is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// c1.xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// c3.2xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// c3.4xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// c3.large is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// c3.xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// c4.2xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// c4.4xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// c4.large is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// c4.xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// d2.2xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// d2.4xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// d2.xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// f1.2xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// f1.4xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// g3.4xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// g3s.xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// i2.2xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// i2.4xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// i2.xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// m1.large is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// m1.medium is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// m1.small is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// m1.xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// m2.2xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// m2.4xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// m2.xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// m3.2xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// m3.large is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// m3.medium is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// m3.xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// m4.2xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// m4.4xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// m4.large is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// m4.xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// p2.xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// p3.2xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// r3.2xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// r3.4xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// r3.large is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// r3.xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// t1.micro is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// t2.2xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// t2.large is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// t2.medium is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// t2.micro is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// t2.nano is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// t2.small is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// t2.xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
"t3.nano": 32,
"t3a.nano": 32,
"t4g.nano": 32,
Expand Down Expand Up @@ -185,9 +226,9 @@ var (
"c5d.2xlarge": 2500,
"c6g.2xlarge": 2500,
"c6gd.2xlarge": 2500,
"f1.2xlarge": 2500,
"g5.xlarge": 2500,
"g5g.2xlarge": 2500,
"g6.xlarge": 2500,
"h1.2xlarge": 2500,
"i3.2xlarge": 2500,
"m5.2xlarge": 2500,
Expand Down Expand Up @@ -261,11 +302,10 @@ var (
"c5n.xlarge": 5000,
"c6g.4xlarge": 5000,
"c6gd.4xlarge": 5000,
"f1.4xlarge": 5000,
"g3.4xlarge": 5000,
"g4dn.xlarge": 5000,
"g5.2xlarge": 5000,
"g5g.4xlarge": 5000,
"g6.2xlarge": 5000,
"h1.4xlarge": 5000,
"i3.4xlarge": 5000,
"inf1.2xlarge": 5000,
Expand Down Expand Up @@ -345,6 +385,8 @@ var (
"g3.8xlarge": 10000,
"g4dn.2xlarge": 10000,
"g5.4xlarge": 10000,
"g6.4xlarge": 10000,
"gr6.4xlarge": 10000,
"h1.8xlarge": 10000,
"i2.8xlarge": 10000,
"i3.8xlarge": 10000,
Expand Down Expand Up @@ -508,6 +550,9 @@ var (
"g5.8xlarge": 25000,
"g5g.16xlarge": 25000,
"g5g.metal": 25000,
"g6.16xlarge": 25000,
"g6.8xlarge": 25000,
"gr6.8xlarge": 25000,
"h1.16xlarge": 25000,
"i3.16xlarge": 25000,
"i3.metal": 25000,
Expand Down Expand Up @@ -602,6 +647,7 @@ var (
"r7i.metal-24xl": 37500,
"d3en.6xlarge": 40000,
"g5.12xlarge": 40000,
"g6.12xlarge": 40000,
"c5n.9xlarge": 50000,
"c6a.32xlarge": 50000,
"c6a.48xlarge": 50000,
Expand All @@ -623,6 +669,7 @@ var (
"g4dn.16xlarge": 50000,
"g4dn.8xlarge": 50000,
"g5.24xlarge": 50000,
"g6.24xlarge": 50000,
"i3en.12xlarge": 50000,
"im4gn.8xlarge": 50000,
"inf2.24xlarge": 50000,
Expand Down Expand Up @@ -691,6 +738,7 @@ var (
"dl2q.24xlarge": 100000,
"g4dn.metal": 100000,
"g5.48xlarge": 100000,
"g6.48xlarge": 100000,
"hpc6a.48xlarge": 100000,
"i3en.24xlarge": 100000,
"i3en.metal": 100000,
Expand Down Expand Up @@ -738,6 +786,7 @@ var (
"c6in.32xlarge": 200000,
"c6in.metal": 200000,
"c7gn.16xlarge": 200000,
"c7gn.metal": 200000,
"hpc6id.32xlarge": 200000,
"hpc7g.16xlarge": 200000,
"hpc7g.4xlarge": 200000,
Expand Down
Loading

0 comments on commit c5e0e25

Please sign in to comment.