From 10033c8896ef6a7c7e3e9728f05163775b675de0 Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Thu, 8 Feb 2018 10:56:47 -0800 Subject: [PATCH 1/3] blah modified: autospotting.go modified: core/autoscaling.go modified: core/config.go modified: core/region.go --- autospotting.go | 9 +++++++++ core/autoscaling.go | 4 ++++ core/config.go | 1 + core/region.go | 2 +- 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/autospotting.go b/autospotting.go index 24bf5e15..1e5ec5fa 100644 --- a/autospotting.go +++ b/autospotting.go @@ -49,6 +49,7 @@ func run() { "disallowed_instance_types=%v "+ "on_demand_price_multiplier=%.2f "+ "spot_price_buffer_percentage=%.3f "+ + "spot_product_description=%v "+ "bidding_policy=%s", conf.Regions, conf.MinOnDemandNumber, @@ -57,6 +58,7 @@ func run() { conf.DisallowedInstanceTypes, conf.OnDemandPriceMultiplier, conf.SpotPriceBufferPercentage, + conf.SpotProductDescription, conf.BiddingPolicy) autospotting.Run(conf.Config) @@ -147,6 +149,13 @@ func (c *cfgData) parseCommandLineFlags() { "\tenforced using the tag: "+autospotting.SpotPriceBufferPercentageTag+". If the bid exceeds\n"+ "\tthe on-demand price, we place a bid at on-demand price itself.") + //flag.StringVar(&c.SpotProductDescription, "spot_product_description", autospotting.DefaultSpotProductDescription, + lag.StringVar(&c.SpotProductDescription, "spot_product_description", "Linux/UNIX", + "They type of Spot product, typically operating system, (default: 'Linux/UNIX')\n"+ + "\t Valid choices: \n"+ + "\t 'Linux/UNIX', 'SUSE Linux', 'Windows', 'Linux/UNIX (Amazon VPC)',\n"+ + "\t 'SUSE Linux (Amazon VPC)', 'Windows (Amazon VPC)'") + flag.StringVar(&c.BiddingPolicy, "bidding_policy", "normal", "Policy choice for spot bid. If set to 'normal', we bid at the on-demand price. If set to 'aggressive',\n"+ "\twe bid at a percentage value above the spot price. ") diff --git a/core/autoscaling.go b/core/autoscaling.go index ea9cfbba..7ee66767 100644 --- a/core/autoscaling.go +++ b/core/autoscaling.go @@ -47,6 +47,10 @@ const ( // Default constant values should be defined below: + // DefaultSpotProductDescription stores the default operating system + // to use when looking up spot price history in the market. + DefaultSpotProductDescription = "Linux/UNIX" + // DefaultMinOnDemandValue stores the default on-demand capacity to be kept // running in a group managed by autospotting. DefaultMinOnDemandValue = 0 diff --git a/core/config.go b/core/config.go index 64cbfe56..9274df25 100644 --- a/core/config.go +++ b/core/config.go @@ -28,6 +28,7 @@ type Config struct { DisallowedInstanceTypes string OnDemandPriceMultiplier float64 SpotPriceBufferPercentage float64 + SpotProductDescription string BiddingPolicy string // This is only here for tests, where we want to be able to somehow mock diff --git a/core/region.go b/core/region.go index 522aa6c4..4ed2cf97 100644 --- a/core/region.go +++ b/core/region.go @@ -206,7 +206,7 @@ func (r *region) requestSpotPrices() error { // Retrieve all current spot prices from the current region. // TODO: add support for other OSes - err := s.fetch("Linux/UNIX", 0, nil, nil) + err := s.fetch(r.conf.SpotProductDescription, 0, nil, nil) if err != nil { return errors.New("Couldn't fetch spot prices in " + r.name) From e323bdca72a53ae109a7ef39717f436634f899cf Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Thu, 8 Feb 2018 13:46:00 -0800 Subject: [PATCH 2/3] Adjusting help flag text and derive flag defaults from constants. modified: autospotting.go modified: core/autoscaling.go --- autospotting.go | 55 +++++++++++++++++++++------------------------ core/autoscaling.go | 2 +- 2 files changed, 27 insertions(+), 30 deletions(-) diff --git a/autospotting.go b/autospotting.go index 1e58106b..52c07057 100644 --- a/autospotting.go +++ b/autospotting.go @@ -102,58 +102,55 @@ func (c *cfgData) initialize() { func (c *cfgData) parseCommandLineFlags() { flag.StringVar(&c.Regions, "regions", "", - "Regions where it should be activated (comma or whitespace separated list, "+ + "\n\tRegions where it should be activated (comma or whitespace separated list, "+ "also supports globs), by default it runs on all regions.\n\t"+ - "Example: ./autospotting -regions 'eu-*,us-east-1'") + "Example: ./autospotting -regions 'eu-*,us-east-1'\n") - flag.Int64Var(&c.MinOnDemandNumber, "min_on_demand_number", 0, - "On-demand capacity (as absolute number) ensured to be running in each of your groups.\n\t"+ + flag.Int64Var(&c.MinOnDemandNumber, "min_on_demand_number", autospotting.DefaultMinOnDemandValue, + "\n\tOn-demand capacity (as absolute number) ensured to be running in each of your groups.\n\t"+ "Can be overridden on a per-group basis using the tag "+ - autospotting.OnDemandNumberLong) + autospotting.OnDemandNumberLong+".\n") flag.Float64Var(&c.MinOnDemandPercentage, "min_on_demand_percentage", 0.0, - "On-demand capacity (percentage of the total number of instances in the group) "+ + "\n\tOn-demand capacity (percentage of the total number of instances in the group) "+ "ensured to be running in each of your groups.\n\t"+ "Can be overridden on a per-group basis using the tag "+ autospotting.OnDemandPercentageTag+ - "\n\tIt is ignored if min_on_demand_number is also set.") + "\n\tIt is ignored if min_on_demand_number is also set.\n") flag.StringVar(&c.AllowedInstanceTypes, "allowed_instance_types", "", - "If specified, the spot instances will have a specific instance type:\n"+ - "\tcurrent: the same as initial on-demand instances\n"+ - "\t: the actual instance type to use") + "\n\tIf specified, the spot instances will be of these types.\n"+ + "\tIf missing, the type is autodetected frome each ASG based on it's Launch Configuration.\n"+ + "\tAccepts a list of comma or whitespace seperated instance types (supports globs).\n"+ + "\tExample: ./autospotting -allowed_instance_types 'c5.*,c4.xlarge'\n") flag.StringVar(&c.DisallowedInstanceTypes, "disallowed_instance_types", "", - "If specified, the spot instances will _never_ be of this type. "+ - "This should be a list of instance types (comma or whitespace separated, "+ - "also supports globs).\n\t"+ - "Example: ./autospotting -disallowed_instance_types 't2.*,c4.xlarge'") + "\n\tIf specified, the spot instances will _never_ be of these types.\n"+ + "\tAccepts a list of comma or whitespace seperated instance types (supports globs).\n"+ + "\tExample: ./autospotting -disallowed_instance_types 't2.*,c4.xlarge'\n") flag.Float64Var(&c.OnDemandPriceMultiplier, "on_demand_price_multiplier", 1.0, - "Multiplier for the on-demand price. This is useful for volume discounts or if you want to\n"+ + "\n\tMultiplier for the on-demand price. This is useful for volume discounts or if you want to\n"+ "\tset your bid price to be higher than the on demand price to reduce the chances that your\n"+ - "\tspot instances will be terminated.") + "\tspot instances will be terminated.\n") - flag.Float64Var(&c.SpotPriceBufferPercentage, "spot_price_buffer_percentage", 10, - "Percentage Value of the bid above the current spot price. A spot bid would be placed at a value :\n"+ + flag.Float64Var(&c.SpotPriceBufferPercentage, "spot_price_buffer_percentage", autospotting.DefaultSpotPriceBufferPercentage, + "\n\tPercentage Value of the bid above the current spot price. A spot bid would be placed at a value :\n"+ "\tcurrent_spot_price * [1 + (spot_price_buffer_percentage/100.0)]. The main benefit is that\n"+ "\tit protects the group from running spot instances that got significantly more expensive than\n"+ "\twhen they were initially launched, but still somewhat less than the on-demand price. Can be\n"+ "\tenforced using the tag: "+autospotting.SpotPriceBufferPercentageTag+". If the bid exceeds\n"+ - "\tthe on-demand price, we place a bid at on-demand price itself.") + "\tthe on-demand price, we place a bid at on-demand price itself.\n") - //flag.StringVar(&c.SpotProductDescription, "spot_product_description", autospotting.DefaultSpotProductDescription, - lag.StringVar(&c.SpotProductDescription, "spot_product_description", "Linux/UNIX", - "They type of Spot product, typically operating system, (default: 'Linux/UNIX')\n"+ - "\t Valid choices: \n"+ - "\t 'Linux/UNIX', 'SUSE Linux', 'Windows', 'Linux/UNIX (Amazon VPC)',\n"+ - "\t 'SUSE Linux (Amazon VPC)', 'Windows (Amazon VPC)'") + flag.StringVar(&c.SpotProductDescription, "spot_product_description", autospotting.DefaultSpotProductDescription, + "\n\tThe Spot Product or operating system to use when looking up spot price history in the market.\n"+ + "\tValid choices: Linux/UNIX | SUSE Linux | Windows | Linux/UNIX (Amazon VPC) | SUSE Linux (Amazon VPC) | Windows (Amazon VPC)\n") - flag.StringVar(&c.BiddingPolicy, "bidding_policy", "normal", - "Policy choice for spot bid. If set to 'normal', we bid at the on-demand price. If set to 'aggressive',\n"+ - "\twe bid at a percentage value above the spot price. ") + flag.StringVar(&c.BiddingPolicy, "bidding_policy", autospotting.DefaultBiddingPolicy, + "\n\tPolicy choice for spot bid. If set to 'normal', we bid at the on-demand price.\n"+ + "\tIf set to 'aggressive', we bid at a percentage value above the spot price configurable using the spot_price_buffer_percentage.\n") - v := flag.Bool("version", false, "Print version number and exit.") + v := flag.Bool("version", false, "Print version number and exit.\n") flag.Parse() diff --git a/core/autoscaling.go b/core/autoscaling.go index 7ee66767..6943d64d 100644 --- a/core/autoscaling.go +++ b/core/autoscaling.go @@ -49,7 +49,7 @@ const ( // DefaultSpotProductDescription stores the default operating system // to use when looking up spot price history in the market. - DefaultSpotProductDescription = "Linux/UNIX" + DefaultSpotProductDescription = "Linux/UNIX (Amazon VPC)" // DefaultMinOnDemandValue stores the default on-demand capacity to be kept // running in a group managed by autospotting. From d4fbe54f246a8c529a8c27dc3d059611b828f334 Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Thu, 8 Feb 2018 14:28:08 -0800 Subject: [PATCH 3/3] Allow access to new config option in CFT and TF modified: autospotting.go modified: cloudformation/stacks/AutoSpotting/template.json modified: terraform/autospotting.tf modified: terraform/autospotting/lambda.tf modified: terraform/variables.tf --- autospotting.go | 2 +- cloudformation/stacks/AutoSpotting/template.json | 6 ++++++ terraform/autospotting.tf | 1 + terraform/autospotting/lambda.tf | 1 + terraform/variables.tf | 5 +++++ 5 files changed, 14 insertions(+), 1 deletion(-) diff --git a/autospotting.go b/autospotting.go index 52c07057..812dd366 100644 --- a/autospotting.go +++ b/autospotting.go @@ -148,7 +148,7 @@ func (c *cfgData) parseCommandLineFlags() { flag.StringVar(&c.BiddingPolicy, "bidding_policy", autospotting.DefaultBiddingPolicy, "\n\tPolicy choice for spot bid. If set to 'normal', we bid at the on-demand price.\n"+ - "\tIf set to 'aggressive', we bid at a percentage value above the spot price configurable using the spot_price_buffer_percentage.\n") + "\tIf set to 'aggressive', we bid at a percentage value above the spot price configurable using the spot_price_buffer_percentage.\n") v := flag.Bool("version", false, "Print version number and exit.\n") diff --git a/cloudformation/stacks/AutoSpotting/template.json b/cloudformation/stacks/AutoSpotting/template.json index 5408b8eb..7e153f56 100644 --- a/cloudformation/stacks/AutoSpotting/template.json +++ b/cloudformation/stacks/AutoSpotting/template.json @@ -52,6 +52,11 @@ "Description": "Percentage Value of the bid above the current spot price. A spot bid would be placed at a value = current_spot_price * [1 + (spot_price_buffer_percentage/100.0)]. The main benefit is that it protects the group from running spot instances that got significantly more expensive than when they were initially launched, but still somewhat less than the on-demand price.", "Type": "Number" }, + "SpotProductDescription": { + "Default": "Linux/Unix (Amazon VPC)", + "Description": "The Spot Product or operating system to use when looking up spot price history in the market. Valid choices: Linux/UNIX | SUSE Linux | Windows | Linux/UNIX (Amazon VPC) | SUSE Linux (Amazon VPC) | Windows (Amazon VPC)", + "Type": "String" + }, "BiddingPolicy": { "Default": "normal", "Description": "Policy choice for spot bid. If set to 'normal', we bid at the on-demand price. If set to 'aggressive', we bid at a multiple of the spot price.", @@ -104,6 +109,7 @@ "MIN_ON_DEMAND_PERCENTAGE": { "Ref": "MinOnDemandPercentage" }, "ON_DEMAND_PRICE_MULTIPLIER": { "Ref": "OnDemandPriceMultiplier" }, "SPOT_PRICE_BUFFER_PERCENTAGE": { "Ref": "SpotPricePercentageBuffer"}, + "SPOT_PRODUCT_DESCRIPTION": { "Ref": "SpotProductDescription"}, "BIDDING_POLICY": { "Ref": "BiddingPolicy"}, "REGIONS": { "Ref": "Regions" }, "ALLOWED_INSTANCE_TYPES": { "Ref": "AllowedInstanceTypes" }, diff --git a/terraform/autospotting.tf b/terraform/autospotting.tf index e8b8affc..4058bbe1 100644 --- a/terraform/autospotting.tf +++ b/terraform/autospotting.tf @@ -6,6 +6,7 @@ module "autospotting" { autospotting_min_on_demand_percentage = "${var.asg_min_on_demand_percentage}" autospotting_on_demand_price_multiplier = "${var.asg_on_demand_price_multiplier}" autospotting_spot_price_buffer_percentage = "${var.asg_spot_price_buffer_percentage}" + autospotting_spot_product_description = "${var.asg_spot_product_description}" autospotting_bidding_policy = "${var.asg_bidding_policy}" autospotting_regions_enabled = "${var.asg_regions_enabled}" diff --git a/terraform/autospotting/lambda.tf b/terraform/autospotting/lambda.tf index 0fa038a6..94a8e862 100644 --- a/terraform/autospotting/lambda.tf +++ b/terraform/autospotting/lambda.tf @@ -15,6 +15,7 @@ resource "aws_lambda_function" "autospotting" { MIN_ON_DEMAND_PERCENTAGE = "${var.autospotting_min_on_demand_percentage}" ON_DEMAND_PRICE_MULTIPLIER = "${var.autospotting_on_demand_price_multiplier}" SPOT_PRICE_BUFFER_PERCENTAGE = "${var.autospotting_spot_price_buffer_percentage}" + SPOT_PRODUCT_DESCRIPTION = "${var.autospotting_spot_product_description}" BIDDING_POLICY = "${var.autospotting_bidding_policy}" REGIONS = "${var.autospotting_regions_enabled}" } diff --git a/terraform/variables.tf b/terraform/variables.tf index 6f485c74..38d780bc 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -24,6 +24,11 @@ variable "asg_on_demand_price_multiplier" { default = "1.0" } +variable "asg_spot_product_description" { + description = "The Spot Product or operating system to use when looking up spot price history in the market. Valid choices: Linux/UNIX | SUSE Linux | Windows | Linux/UNIX (Amazon VPC) | SUSE Linux (Amazon VPC) | Windows (Amazon VPC)" + default = "Linux/UNIX (Amazon VPC)" +} + variable "asg_spot_price_buffer_percentage" { description = "Percentage above the current spot price to place the bid" default = "10.0"