Skip to content

Commit

Permalink
fix network bandwidth script and run codegen
Browse files Browse the repository at this point in the history
  • Loading branch information
@jigisha620
jigisha620 committed May 5, 2024
1 parent b17e0eb commit bbdc07c
Show file tree
Hide file tree
Showing 6 changed files with 9,500 additions and 50 deletions.
9,171 changes: 9,171 additions & 0 deletions hack/code/bandwidth_gen/example/gp.html
View file

Large diffs are not rendered by default.

81 changes: 47 additions & 34 deletions hack/code/bandwidth_gen/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,13 @@ import (
)

var uriSelectors = map[string]string{
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/general-purpose-instances.html": "#general-purpose-network-performance",
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/compute-optimized-instances.html": "#compute-network-performance",
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/memory-optimized-instances.html": "#memory-network-perf",
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/storage-optimized-instances.html": "#storage-network-performance",
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/accelerated-computing-instances.html": "#gpu-network-performance",
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/high-performance-computing-instances.html": "#hpc-network-performance",
"https://docs.aws.amazon.com/ec2/latest/instancetypes/gp.html": "#gp_network",
"https://docs.aws.amazon.com/ec2/latest/instancetypes/co.html": "#co_network",
"https://docs.aws.amazon.com/ec2/latest/instancetypes/mo.html": "#mo_network",
"https://docs.aws.amazon.com/ec2/latest/instancetypes/so.html": "#so_network",
"https://docs.aws.amazon.com/ec2/latest/instancetypes/ac.html": "#ac_network",
"https://docs.aws.amazon.com/ec2/latest/instancetypes/hpc.html": "#hpc_network",
"https://docs.aws.amazon.com/ec2/latest/instancetypes/pg.html": "#pg_network",
}

const fileFormat = `
Expand All @@ -62,6 +63,7 @@ func main() {
}

bandwidth := map[string]int64{}
vagueBandwidth := map[string]string{}

for uri, selector := range uriSelectors {
func() {
Expand All @@ -70,16 +72,18 @@ func main() {

doc := lo.Must(goquery.NewDocumentFromReader(response.Body))

// grab two tables that contain the network performance values
// first table will contain all the instance type and bandwidth data
// some rows will will have vague describe such as "Very Low", "Low", "Low to Moderate", etc.
// These instance types will can be found on the second table with absolute values in Gbps
// If the instance type is skipped on the first table it will be grabbed on the second table
// grab the table that contains the network performance values. Some instance types will have vague
// description for bandwidth such as "Very Low", "Low", "Low to Moderate", etc. These instance types
// will be ignored since we don't know the exact bandwidth for these instance types
for _, row := range doc.Find(selector).NextAllFiltered(".table-container").Eq(0).Find("tbody").Find("tr").Nodes {
instanceTypeData := row.FirstChild.NextSibling.FirstChild.FirstChild.Data
instanceTypeData := strings.TrimSpace(row.FirstChild.NextSibling.FirstChild.Data)
if !strings.ContainsAny(instanceTypeData, ".") {
continue
}
bandwidthData := row.FirstChild.NextSibling.NextSibling.NextSibling.FirstChild.Data
// exclude all rows that contain any of the following strings
if containsAny(bandwidthData, "Low", "Moderate", "High", "Up to") {
vagueBandwidth[instanceTypeData] = bandwidthData
continue
}
bandwidthSlice := strings.Split(bandwidthData, " ")
Expand All @@ -92,30 +96,9 @@ func main() {
bandwidth[instanceTypeData] = int64(lo.Must(strconv.ParseFloat(bandwidthSlice[0], 64)) * 1000)
}
}

// Collect instance types bandwidth data from the baseline/bandwidth table underneath the standard table
// The HPC network performance doc is laid out differently than the other docs. There is no table underneath
// the standard table that contains information for network performance with baseline and burst bandwidth.
if selector != "#hpc-network-performance" {
for _, row := range doc.Find(selector).NextAllFiltered(".table-container").Eq(1).Find("tbody").Find("tr").Nodes {
instanceTypeData := row.FirstChild.NextSibling.FirstChild.FirstChild.Data
bandwidthData := row.FirstChild.NextSibling.NextSibling.NextSibling.FirstChild.Data
bandwidth[instanceTypeData] = int64(lo.Must(strconv.ParseFloat(bandwidthData, 64)) * 1000)
}
}
}()
}
if err := os.Setenv("AWS_SDK_LOAD_CONFIG", "true"); err != nil {
log.Fatalf("setting AWS_SDK_LOAD_CONFIG, %s", err)
}
if err := os.Setenv("AWS_REGION", "us-east-1"); err != nil {
log.Fatalf("setting AWS_REGION, %s", err)
}
sess := session.Must(session.NewSession())
ec2api := ec2.New(sess)
instanceTypesOutput := lo.Must(ec2api.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{}))
allInstanceTypes := lo.Map(instanceTypesOutput.InstanceTypes, func(info *ec2.InstanceTypeInfo, _ int) string { return *info.InstanceType })

allInstanceTypes := getAllInstanceTypes()
instanceTypes := lo.Keys(bandwidth)
// 2d sort for readability
sort.Strings(allInstanceTypes)
Expand All @@ -127,6 +110,10 @@ func main() {
// Generate body
var body string
for _, instanceType := range lo.Without(allInstanceTypes, instanceTypes...) {
if lo.Contains(lo.Keys(vagueBandwidth), instanceType) {
body += fmt.Sprintf("// %s has vague bandwidth information, bandwidth is %s\n", instanceType, vagueBandwidth[instanceType])
continue
}
body += fmt.Sprintf("// %s is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html\n", instanceType)
}
for _, instanceType := range instanceTypes {
Expand All @@ -150,3 +137,29 @@ func containsAny(value string, excludedSubstrings ...string) bool {
}
return false
}

func getAllInstanceTypes() []string {
if err := os.Setenv("AWS_SDK_LOAD_CONFIG", "true"); err != nil {
log.Fatalf("setting AWS_SDK_LOAD_CONFIG, %s", err)
}
if err := os.Setenv("AWS_REGION", "us-east-1"); err != nil {
log.Fatalf("setting AWS_REGION, %s", err)
}
sess := session.Must(session.NewSession())
ec2api := ec2.New(sess)
var allInstanceTypes []string

params := &ec2.DescribeInstanceTypesInput{}
// Retrieve the instance types in a loop using NextToken
for {
result := lo.Must(ec2api.DescribeInstanceTypes(params))
allInstanceTypes = append(allInstanceTypes, lo.Map(result.InstanceTypes, func(info *ec2.InstanceTypeInfo, _ int) string { return *info.InstanceType })...)
// Check if they are any instances left
if result.NextToken != nil {
params.NextToken = result.NextToken
} else {
break
}
}
return allInstanceTypes
}
71 changes: 60 additions & 11 deletions pkg/providers/instancetype/zz_generated.bandwidth.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,55 @@ package instancetype

var (
InstanceTypeBandwidthMegabits = map[string]int64{
// c3.large is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// c4.4xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// i2.2xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// m2.4xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// m4.4xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// r3.large is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// t1.micro is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// t2.2xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// c1.medium has vague bandwidth information, bandwidth is Moderate
// c1.xlarge has vague bandwidth information, bandwidth is High
// c3.2xlarge has vague bandwidth information, bandwidth is High
// c3.4xlarge has vague bandwidth information, bandwidth is High
// c3.large has vague bandwidth information, bandwidth is Moderate
// c3.xlarge has vague bandwidth information, bandwidth is Moderate
// c4.2xlarge has vague bandwidth information, bandwidth is High
// c4.4xlarge has vague bandwidth information, bandwidth is High
// c4.large has vague bandwidth information, bandwidth is Moderate
// c4.xlarge has vague bandwidth information, bandwidth is High
// d2.2xlarge has vague bandwidth information, bandwidth is High
// d2.4xlarge has vague bandwidth information, bandwidth is High
// d2.xlarge has vague bandwidth information, bandwidth is Moderate
// f1.2xlarge has vague bandwidth information, bandwidth is Up to 10 Gigabit
// f1.4xlarge has vague bandwidth information, bandwidth is Up to 10 Gigabit
// g3.4xlarge has vague bandwidth information, bandwidth is Up to 10 Gigabit
// g3s.xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// i2.2xlarge has vague bandwidth information, bandwidth is High
// i2.4xlarge has vague bandwidth information, bandwidth is High
// i2.xlarge has vague bandwidth information, bandwidth is Moderate
// m1.large has vague bandwidth information, bandwidth is Moderate
// m1.medium has vague bandwidth information, bandwidth is Moderate
// m1.small has vague bandwidth information, bandwidth is Low
// m1.xlarge has vague bandwidth information, bandwidth is High
// m2.2xlarge has vague bandwidth information, bandwidth is Moderate
// m2.4xlarge has vague bandwidth information, bandwidth is High
// m2.xlarge has vague bandwidth information, bandwidth is Moderate
// m3.2xlarge has vague bandwidth information, bandwidth is High
// m3.large has vague bandwidth information, bandwidth is Moderate
// m3.medium has vague bandwidth information, bandwidth is Moderate
// m3.xlarge has vague bandwidth information, bandwidth is High
// m4.2xlarge has vague bandwidth information, bandwidth is High
// m4.4xlarge has vague bandwidth information, bandwidth is High
// m4.large has vague bandwidth information, bandwidth is Moderate
// m4.xlarge has vague bandwidth information, bandwidth is High
// p2.xlarge has vague bandwidth information, bandwidth is High
// p3.2xlarge has vague bandwidth information, bandwidth is Up to 10 Gigabit
// r3.2xlarge has vague bandwidth information, bandwidth is High
// r3.4xlarge has vague bandwidth information, bandwidth is High
// r3.large has vague bandwidth information, bandwidth is Moderate
// r3.xlarge has vague bandwidth information, bandwidth is Moderate
// t1.micro has vague bandwidth information, bandwidth is Very Low
// t2.2xlarge has vague bandwidth information, bandwidth is Moderate
// t2.large has vague bandwidth information, bandwidth is Low to Moderate
// t2.medium has vague bandwidth information, bandwidth is Low to Moderate
// t2.micro has vague bandwidth information, bandwidth is Low to Moderate
// t2.nano has vague bandwidth information, bandwidth is Low to Moderate
// t2.small has vague bandwidth information, bandwidth is Low to Moderate
// t2.xlarge has vague bandwidth information, bandwidth is Moderate
"t3.nano": 32,
"t3a.nano": 32,
"t4g.nano": 32,
Expand Down Expand Up @@ -185,9 +226,9 @@ var (
"c5d.2xlarge": 2500,
"c6g.2xlarge": 2500,
"c6gd.2xlarge": 2500,
"f1.2xlarge": 2500,
"g5.xlarge": 2500,
"g5g.2xlarge": 2500,
"g6.xlarge": 2500,
"h1.2xlarge": 2500,
"i3.2xlarge": 2500,
"m5.2xlarge": 2500,
Expand Down Expand Up @@ -261,11 +302,10 @@ var (
"c5n.xlarge": 5000,
"c6g.4xlarge": 5000,
"c6gd.4xlarge": 5000,
"f1.4xlarge": 5000,
"g3.4xlarge": 5000,
"g4dn.xlarge": 5000,
"g5.2xlarge": 5000,
"g5g.4xlarge": 5000,
"g6.2xlarge": 5000,
"h1.4xlarge": 5000,
"i3.4xlarge": 5000,
"inf1.2xlarge": 5000,
Expand Down Expand Up @@ -345,6 +385,8 @@ var (
"g3.8xlarge": 10000,
"g4dn.2xlarge": 10000,
"g5.4xlarge": 10000,
"g6.4xlarge": 10000,
"gr6.4xlarge": 10000,
"h1.8xlarge": 10000,
"i2.8xlarge": 10000,
"i3.8xlarge": 10000,
Expand Down Expand Up @@ -508,6 +550,9 @@ var (
"g5.8xlarge": 25000,
"g5g.16xlarge": 25000,
"g5g.metal": 25000,
"g6.16xlarge": 25000,
"g6.8xlarge": 25000,
"gr6.8xlarge": 25000,
"h1.16xlarge": 25000,
"i3.16xlarge": 25000,
"i3.metal": 25000,
Expand Down Expand Up @@ -602,6 +647,7 @@ var (
"r7i.metal-24xl": 37500,
"d3en.6xlarge": 40000,
"g5.12xlarge": 40000,
"g6.12xlarge": 40000,
"c5n.9xlarge": 50000,
"c6a.32xlarge": 50000,
"c6a.48xlarge": 50000,
Expand All @@ -623,6 +669,7 @@ var (
"g4dn.16xlarge": 50000,
"g4dn.8xlarge": 50000,
"g5.24xlarge": 50000,
"g6.24xlarge": 50000,
"i3en.12xlarge": 50000,
"im4gn.8xlarge": 50000,
"inf2.24xlarge": 50000,
Expand Down Expand Up @@ -691,6 +738,7 @@ var (
"dl2q.24xlarge": 100000,
"g4dn.metal": 100000,
"g5.48xlarge": 100000,
"g6.48xlarge": 100000,
"hpc6a.48xlarge": 100000,
"i3en.24xlarge": 100000,
"i3en.metal": 100000,
Expand Down Expand Up @@ -738,6 +786,7 @@ var (
"c6in.32xlarge": 200000,
"c6in.metal": 200000,
"c7gn.16xlarge": 200000,
"c7gn.metal": 200000,
"hpc6id.32xlarge": 200000,
"hpc7g.16xlarge": 200000,
"hpc7g.4xlarge": 200000,
Expand Down
Loading

0 comments on commit bbdc07c

Please sign in to comment.