Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix network bandwidth script and run codegen #6099

Merged
merged 1 commit into from
May 5, 2024
Merged

fix network bandwidth script and run codegen #6099

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9,171 changes: 9,171 additions & 0 deletions hack/code/bandwidth_gen/example/gp.html
View file
Open in desktop

Large diffs are not rendered by default.

81 changes: 47 additions & 34 deletions hack/code/bandwidth_gen/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,13 @@ import (
)

var uriSelectors = map[string]string{
jigisha620 marked this conversation as resolved.
Show resolved Hide resolved
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/general-purpose-instances.html": "#general-purpose-network-performance",
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/compute-optimized-instances.html": "#compute-network-performance",
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/memory-optimized-instances.html": "#memory-network-perf",
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/storage-optimized-instances.html": "#storage-network-performance",
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/accelerated-computing-instances.html": "#gpu-network-performance",
"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/high-performance-computing-instances.html": "#hpc-network-performance",
"https://docs.aws.amazon.com/ec2/latest/instancetypes/gp.html": "#gp_network",
"https://docs.aws.amazon.com/ec2/latest/instancetypes/co.html": "#co_network",
"https://docs.aws.amazon.com/ec2/latest/instancetypes/mo.html": "#mo_network",
"https://docs.aws.amazon.com/ec2/latest/instancetypes/so.html": "#so_network",
"https://docs.aws.amazon.com/ec2/latest/instancetypes/ac.html": "#ac_network",
"https://docs.aws.amazon.com/ec2/latest/instancetypes/hpc.html": "#hpc_network",
"https://docs.aws.amazon.com/ec2/latest/instancetypes/pg.html": "#pg_network",
}

const fileFormat = `
Expand All @@ -62,6 +63,7 @@ func main() {
}

bandwidth := map[string]int64{}
vagueBandwidth := map[string]string{}

for uri, selector := range uriSelectors {
func() {
Expand All @@ -70,16 +72,18 @@ func main() {

doc := lo.Must(goquery.NewDocumentFromReader(response.Body))

// grab two tables that contain the network performance values
// first table will contain all the instance type and bandwidth data
// some rows will will have vague describe such as "Very Low", "Low", "Low to Moderate", etc.
// These instance types will can be found on the second table with absolute values in Gbps
// If the instance type is skipped on the first table it will be grabbed on the second table
// grab the table that contains the network performance values. Some instance types will have vague
// description for bandwidth such as "Very Low", "Low", "Low to Moderate", etc. These instance types
// will be ignored since we don't know the exact bandwidth for these instance types
for _, row := range doc.Find(selector).NextAllFiltered(".table-container").Eq(0).Find("tbody").Find("tr").Nodes {
instanceTypeData := row.FirstChild.NextSibling.FirstChild.FirstChild.Data
instanceTypeData := strings.TrimSpace(row.FirstChild.NextSibling.FirstChild.Data)
if !strings.ContainsAny(instanceTypeData, ".") {
continue
}
bandwidthData := row.FirstChild.NextSibling.NextSibling.NextSibling.FirstChild.Data
// exclude all rows that contain any of the following strings
if containsAny(bandwidthData, "Low", "Moderate", "High", "Up to") {
vagueBandwidth[instanceTypeData] = bandwidthData
continue
}
bandwidthSlice := strings.Split(bandwidthData, " ")
Expand All @@ -92,30 +96,9 @@ func main() {
bandwidth[instanceTypeData] = int64(lo.Must(strconv.ParseFloat(bandwidthSlice[0], 64)) * 1000)
}
}

// Collect instance types bandwidth data from the baseline/bandwidth table underneath the standard table
// The HPC network performance doc is laid out differently than the other docs. There is no table underneath
// the standard table that contains information for network performance with baseline and burst bandwidth.
if selector != "#hpc-network-performance" {
for _, row := range doc.Find(selector).NextAllFiltered(".table-container").Eq(1).Find("tbody").Find("tr").Nodes {
instanceTypeData := row.FirstChild.NextSibling.FirstChild.FirstChild.Data
bandwidthData := row.FirstChild.NextSibling.NextSibling.NextSibling.FirstChild.Data
bandwidth[instanceTypeData] = int64(lo.Must(strconv.ParseFloat(bandwidthData, 64)) * 1000)
}
}
}()
}
if err := os.Setenv("AWS_SDK_LOAD_CONFIG", "true"); err != nil {
log.Fatalf("setting AWS_SDK_LOAD_CONFIG, %s", err)
}
if err := os.Setenv("AWS_REGION", "us-east-1"); err != nil {
log.Fatalf("setting AWS_REGION, %s", err)
}
sess := session.Must(session.NewSession())
ec2api := ec2.New(sess)
instanceTypesOutput := lo.Must(ec2api.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{}))
allInstanceTypes := lo.Map(instanceTypesOutput.InstanceTypes, func(info *ec2.InstanceTypeInfo, _ int) string { return *info.InstanceType })

allInstanceTypes := getAllInstanceTypes()
instanceTypes := lo.Keys(bandwidth)
// 2d sort for readability
sort.Strings(allInstanceTypes)
Expand All @@ -127,6 +110,10 @@ func main() {
// Generate body
var body string
for _, instanceType := range lo.Without(allInstanceTypes, instanceTypes...) {
if lo.Contains(lo.Keys(vagueBandwidth), instanceType) {
body += fmt.Sprintf("// %s has vague bandwidth information, bandwidth is %s\n", instanceType, vagueBandwidth[instanceType])
continue
}
body += fmt.Sprintf("// %s is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html\n", instanceType)
}
for _, instanceType := range instanceTypes {
Expand All @@ -150,3 +137,29 @@ func containsAny(value string, excludedSubstrings ...string) bool {
}
return false
}

func getAllInstanceTypes() []string {
if err := os.Setenv("AWS_SDK_LOAD_CONFIG", "true"); err != nil {
log.Fatalf("setting AWS_SDK_LOAD_CONFIG, %s", err)
}
if err := os.Setenv("AWS_REGION", "us-east-1"); err != nil {
log.Fatalf("setting AWS_REGION, %s", err)
}
sess := session.Must(session.NewSession())
ec2api := ec2.New(sess)
var allInstanceTypes []string

params := &ec2.DescribeInstanceTypesInput{}
// Retrieve the instance types in a loop using NextToken
for {
result := lo.Must(ec2api.DescribeInstanceTypes(params))
allInstanceTypes = append(allInstanceTypes, lo.Map(result.InstanceTypes, func(info *ec2.InstanceTypeInfo, _ int) string { return *info.InstanceType })...)
// Check if they are any instances left
if result.NextToken != nil {
params.NextToken = result.NextToken
} else {
break
}
}
return allInstanceTypes
}
71 changes: 60 additions & 11 deletions pkg/providers/instancetype/zz_generated.bandwidth.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,55 @@ package instancetype

var (
InstanceTypeBandwidthMegabits = map[string]int64{
// c3.large is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// c4.4xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// i2.2xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// m2.4xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// m4.4xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// r3.large is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// t1.micro is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// t2.2xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// c1.medium has vague bandwidth information, bandwidth is Moderate
// c1.xlarge has vague bandwidth information, bandwidth is High
// c3.2xlarge has vague bandwidth information, bandwidth is High
// c3.4xlarge has vague bandwidth information, bandwidth is High
// c3.large has vague bandwidth information, bandwidth is Moderate
// c3.xlarge has vague bandwidth information, bandwidth is Moderate
// c4.2xlarge has vague bandwidth information, bandwidth is High
// c4.4xlarge has vague bandwidth information, bandwidth is High
// c4.large has vague bandwidth information, bandwidth is Moderate
// c4.xlarge has vague bandwidth information, bandwidth is High
// d2.2xlarge has vague bandwidth information, bandwidth is High
// d2.4xlarge has vague bandwidth information, bandwidth is High
// d2.xlarge has vague bandwidth information, bandwidth is Moderate
// f1.2xlarge has vague bandwidth information, bandwidth is Up to 10 Gigabit
// f1.4xlarge has vague bandwidth information, bandwidth is Up to 10 Gigabit
// g3.4xlarge has vague bandwidth information, bandwidth is Up to 10 Gigabit
// g3s.xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html
// i2.2xlarge has vague bandwidth information, bandwidth is High
// i2.4xlarge has vague bandwidth information, bandwidth is High
// i2.xlarge has vague bandwidth information, bandwidth is Moderate
// m1.large has vague bandwidth information, bandwidth is Moderate
// m1.medium has vague bandwidth information, bandwidth is Moderate
// m1.small has vague bandwidth information, bandwidth is Low
// m1.xlarge has vague bandwidth information, bandwidth is High
// m2.2xlarge has vague bandwidth information, bandwidth is Moderate
// m2.4xlarge has vague bandwidth information, bandwidth is High
// m2.xlarge has vague bandwidth information, bandwidth is Moderate
// m3.2xlarge has vague bandwidth information, bandwidth is High
// m3.large has vague bandwidth information, bandwidth is Moderate
// m3.medium has vague bandwidth information, bandwidth is Moderate
// m3.xlarge has vague bandwidth information, bandwidth is High
// m4.2xlarge has vague bandwidth information, bandwidth is High
// m4.4xlarge has vague bandwidth information, bandwidth is High
// m4.large has vague bandwidth information, bandwidth is Moderate
// m4.xlarge has vague bandwidth information, bandwidth is High
// p2.xlarge has vague bandwidth information, bandwidth is High
// p3.2xlarge has vague bandwidth information, bandwidth is Up to 10 Gigabit
// r3.2xlarge has vague bandwidth information, bandwidth is High
// r3.4xlarge has vague bandwidth information, bandwidth is High
// r3.large has vague bandwidth information, bandwidth is Moderate
// r3.xlarge has vague bandwidth information, bandwidth is Moderate
// t1.micro has vague bandwidth information, bandwidth is Very Low
// t2.2xlarge has vague bandwidth information, bandwidth is Moderate
// t2.large has vague bandwidth information, bandwidth is Low to Moderate
// t2.medium has vague bandwidth information, bandwidth is Low to Moderate
// t2.micro has vague bandwidth information, bandwidth is Low to Moderate
// t2.nano has vague bandwidth information, bandwidth is Low to Moderate
// t2.small has vague bandwidth information, bandwidth is Low to Moderate
// t2.xlarge has vague bandwidth information, bandwidth is Moderate
"t3.nano": 32,
"t3a.nano": 32,
"t4g.nano": 32,
Expand Down Expand Up @@ -185,9 +226,9 @@ var (
"c5d.2xlarge": 2500,
"c6g.2xlarge": 2500,
"c6gd.2xlarge": 2500,
"f1.2xlarge": 2500,
"g5.xlarge": 2500,
"g5g.2xlarge": 2500,
"g6.xlarge": 2500,
"h1.2xlarge": 2500,
"i3.2xlarge": 2500,
"m5.2xlarge": 2500,
Expand Down Expand Up @@ -261,11 +302,10 @@ var (
"c5n.xlarge": 5000,
"c6g.4xlarge": 5000,
"c6gd.4xlarge": 5000,
"f1.4xlarge": 5000,
"g3.4xlarge": 5000,
"g4dn.xlarge": 5000,
"g5.2xlarge": 5000,
"g5g.4xlarge": 5000,
"g6.2xlarge": 5000,
"h1.4xlarge": 5000,
"i3.4xlarge": 5000,
"inf1.2xlarge": 5000,
Expand Down Expand Up @@ -345,6 +385,8 @@ var (
"g3.8xlarge": 10000,
"g4dn.2xlarge": 10000,
"g5.4xlarge": 10000,
"g6.4xlarge": 10000,
"gr6.4xlarge": 10000,
"h1.8xlarge": 10000,
"i2.8xlarge": 10000,
"i3.8xlarge": 10000,
Expand Down Expand Up @@ -508,6 +550,9 @@ var (
"g5.8xlarge": 25000,
"g5g.16xlarge": 25000,
"g5g.metal": 25000,
"g6.16xlarge": 25000,
"g6.8xlarge": 25000,
"gr6.8xlarge": 25000,
"h1.16xlarge": 25000,
"i3.16xlarge": 25000,
"i3.metal": 25000,
Expand Down Expand Up @@ -602,6 +647,7 @@ var (
"r7i.metal-24xl": 37500,
"d3en.6xlarge": 40000,
"g5.12xlarge": 40000,
"g6.12xlarge": 40000,
"c5n.9xlarge": 50000,
"c6a.32xlarge": 50000,
"c6a.48xlarge": 50000,
Expand All @@ -623,6 +669,7 @@ var (
"g4dn.16xlarge": 50000,
"g4dn.8xlarge": 50000,
"g5.24xlarge": 50000,
"g6.24xlarge": 50000,
"i3en.12xlarge": 50000,
"im4gn.8xlarge": 50000,
"inf2.24xlarge": 50000,
Expand Down Expand Up @@ -691,6 +738,7 @@ var (
"dl2q.24xlarge": 100000,
"g4dn.metal": 100000,
"g5.48xlarge": 100000,
"g6.48xlarge": 100000,
"hpc6a.48xlarge": 100000,
"i3en.24xlarge": 100000,
"i3en.metal": 100000,
Expand Down Expand Up @@ -738,6 +786,7 @@ var (
"c6in.32xlarge": 200000,
"c6in.metal": 200000,
"c7gn.16xlarge": 200000,
"c7gn.metal": 200000,
"hpc6id.32xlarge": 200000,
"hpc7g.16xlarge": 200000,
"hpc7g.4xlarge": 200000,
Expand Down
Loading
Loading