From 9d2f68f76d8f0b952fecfc4fb17cb97302c5183d Mon Sep 17 00:00:00 2001 From: Michal Kubenka Date: Sun, 18 Jun 2017 02:19:05 +0200 Subject: [PATCH 1/2] Add spot instances support. --- README.md | 5 +++ lib/vagrant-aws/action/run_instance.rb | 62 +++++++++++++++++++++++++- lib/vagrant-aws/config.rb | 32 ++++++++++++- locales/en.yml | 6 ++- spec/vagrant-aws/config_spec.rb | 3 ++ 5 files changed, 104 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 31bbf8cd..0fad80df 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ EC2 and VPC. * Define region-specific configurations so Vagrant can manage machines in multiple regions. * Package running instances into new vagrant-aws friendly boxes +* Spot Instance Support ## Usage @@ -170,6 +171,10 @@ This provider exposes quite a few provider-specific configuration options: when you initiate shutdown from the instance. * `endpoint` - The endpoint URL for connecting to AWS (or an AWS-like service). Only required for non AWS clouds, such as [eucalyptus](https://github.com/eucalyptus/eucalyptus/wiki). +* `spot_instance` - Boolean value; indicates whether the config is for a spot instance, or on-demand. For more information about spot instances, see the [AWS Documentation](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/how-spot-instances-work.html) +* `spot_max_price` - Decimal value; state the maximum bid for your spot instance. Ignored if `spot_instance` is not true. +* `spot_valid_until` - Timestamp; when this spot instance request should expire, destroying any related instances. Ignored if `spot_instance` is not true. + These can be set like typical provider-specific configuration: ```ruby diff --git a/lib/vagrant-aws/action/run_instance.rb b/lib/vagrant-aws/action/run_instance.rb index b37e4c64..6786f499 100644 --- a/lib/vagrant-aws/action/run_instance.rb +++ b/lib/vagrant-aws/action/run_instance.rb @@ -107,7 +107,11 @@ def call(env) end begin - server = env[:aws_compute].servers.create(options) + server = if region_config.spot_instance + server_from_spot_request(env, region_config, options) + else + env[:aws_compute].servers.create(options) + end rescue Fog::Compute::AWS::NotFound => e # Invalid subnet doesn't have its own error so we catch and # check the error message here. @@ -212,6 +216,62 @@ def call(env) @app.call(env) end + # returns a fog server or nil + def server_from_spot_request(env, config, options) + options.merge!({ + :price => config.spot_max_price, + :valid_until => config.spot_valid_until + }) + + env[:ui].info(I18n.t("vagrant_aws.launching_spot_instance")) + env[:ui].info(" -- Price: #{config.spot_max_price}") + env[:ui].info(" -- Valid until: #{config.spot_valid_until}") if config.spot_valid_until + + # create the spot instance + spot_req = env[:aws_compute].spot_requests.create(options) + + @logger.info("Spot request ID: #{spot_req.id}") + + # initialize state + status_code = "" + while true + sleep 5 + + spot_req.reload() + + # display something whenever the status code changes + if status_code != spot_req.state + env[:ui].info(spot_req.fault) + status_code = spot_req.state + end + spot_state = spot_req.state.to_sym + case spot_state + when :not_created, :open + @logger.debug("Spot request #{spot_state} #{status_code}, waiting") + when :active + break; # :) + when :closed, :cancelled, :failed + msg = "Spot request #{spot_state} #{status_code}, aborting" + @logger.error(msg) + raise Errors::FogError, :message => msg + else + @logger.debug("Unknown spot state #{spot_state} #{status_code}, waiting") + end + end + # cancel the spot request but let the server go thru + spot_req.destroy() + + server = env[:aws_compute].servers.get(spot_req.instance_id) + + # Spot Instances don't support tagging arguments on creation + # Retrospectively tag the server to handle this + if !config.tags.empty? + env[:aws_compute].create_tags(server.identity, config.tags) + end + + server + end + def recover(env) return if env["vagrant.error"].is_a?(Vagrant::Errors::VagrantError) diff --git a/lib/vagrant-aws/config.rb b/lib/vagrant-aws/config.rb index 30a19cb4..f4fd597b 100644 --- a/lib/vagrant-aws/config.rb +++ b/lib/vagrant-aws/config.rb @@ -196,6 +196,21 @@ class Config < Vagrant.plugin("2", :config) # @return [String] attr_accessor :aws_profile + # Launch as spot instance + # + # @return [Boolean] + attr_accessor :spot_instance + + # Spot request max price + # + # @return [String] + attr_accessor :spot_max_price + + # Spot request validity + # + # @return [Time] + attr_accessor :spot_valid_until + def initialize(region_specific=false) @access_key_id = UNSET_VALUE @ami = UNSET_VALUE @@ -233,6 +248,9 @@ def initialize(region_specific=false) @tenancy = UNSET_VALUE @aws_dir = UNSET_VALUE @aws_profile = UNSET_VALUE + @spot_instance = UNSET_VALUE + @spot_max_price = UNSET_VALUE + @spot_valid_until = UNSET_VALUE # Internal state (prefix with __ so they aren't automatically # merged) @@ -409,6 +427,15 @@ def finalize! # default to nil @kernel_id = nil if @kernel_id == UNSET_VALUE + # By default don't use spot requests + @spot_instance = false if @spot_instance == UNSET_VALUE + + # Required, no default + @spot_max_price = nil if @spot_max_price == UNSET_VALUE + + # Default: Request is effective indefinitely. + @spot_valid_until = nil if @spot_valid_until == UNSET_VALUE + # Compile our region specific configurations only within # NON-REGION-SPECIFIC configurations. if !@__region_specific @@ -460,6 +487,7 @@ def validate(machine) end errors << I18n.t("vagrant_aws.config.ami_required", :region => @region) if config.ami.nil? + errors << I18n.t("vagrant_aws.config.spot_price_required") if config.spot_instance && config.spot_max_price.nil? end { "AWS Provider" => errors } @@ -479,14 +507,14 @@ def get_region_config(name) class Credentials < Vagrant.plugin("2", :config) - # This module reads AWS config and credentials. + # This module reads AWS config and credentials. # Behaviour aims to mimic what is described in AWS documentation: # http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html # http://docs.aws.amazon.com/cli/latest/topic/config-vars.html # Which is the following (stopping at the first successful case): # 1) read config and credentials from environment variables # 2) read config and credentials from files at location defined by environment variables - # 3) read config and credentials from files at default location + # 3) read config and credentials from files at default location # # The mandatory fields for a successful "get credentials" are the id and the secret keys. # Region is not required since Config#finalize falls back to sensible defaults. diff --git a/locales/en.yml b/locales/en.yml index ddc3d4d6..10b219f2 100644 --- a/locales/en.yml +++ b/locales/en.yml @@ -18,6 +18,8 @@ en: launching_instance: |- Launching an instance with the following settings... + launching_spot_instance: |- + Launching a spot request instance with the following settings... launch_no_keypair: |- Warning! You didn't specify a keypair to launch your instance with. This can sometimes result in not being able to access your instance. @@ -68,6 +70,8 @@ en: An access key ID must be specified via "access_key_id" ami_required: |- An AMI must be configured via "ami" (region: #{region}) + spot_price_required: |- + Spot request is missing "spot_max_price" private_key_missing: |- The specified private key for AWS could not be found region_required: |- @@ -104,7 +108,7 @@ en: Error: %{err} instance_package_timeout: |- The AMI failed to become "ready" in AWS. The timeout currently - set waiting for the instance to become ready is %{timeout} seconds. For + set waiting for the instance to become ready is %{timeout} seconds. For larger instances AMI burning may take long periods of time. Please ensure the timeout is set high enough, it can be changed by adjusting the `instance_package_timeout` configuration on the AWS provider. diff --git a/spec/vagrant-aws/config_spec.rb b/spec/vagrant-aws/config_spec.rb index eb309743..48eda53f 100644 --- a/spec/vagrant-aws/config_spec.rb +++ b/spec/vagrant-aws/config_spec.rb @@ -58,6 +58,9 @@ its("associate_public_ip") { should == false } its("unregister_elb_from_az") { should == true } its("tenancy") { should == "default" } + its("spot_instance") { should == false } + its("spot_max_price") { should be_nil } + its("spot_valid_until") { should be_nil } end describe "overriding defaults" do From e6cbf02af8f1de12459dd414f230668b7d7451f7 Mon Sep 17 00:00:00 2001 From: Michal Kubenka Date: Sun, 18 Jun 2017 02:55:09 +0200 Subject: [PATCH 2/2] If spot price is not defined compute bid automatically from price history. --- README.md | 3 ++- lib/vagrant-aws/action/run_instance.rb | 29 ++++++++++++++++++++++++-- lib/vagrant-aws/config.rb | 11 ++++++++-- locales/en.yml | 2 -- spec/vagrant-aws/config_spec.rb | 1 + 5 files changed, 39 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 0fad80df..2b76e82c 100644 --- a/README.md +++ b/README.md @@ -172,7 +172,8 @@ This provider exposes quite a few provider-specific configuration options: * `endpoint` - The endpoint URL for connecting to AWS (or an AWS-like service). Only required for non AWS clouds, such as [eucalyptus](https://github.com/eucalyptus/eucalyptus/wiki). * `spot_instance` - Boolean value; indicates whether the config is for a spot instance, or on-demand. For more information about spot instances, see the [AWS Documentation](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/how-spot-instances-work.html) -* `spot_max_price` - Decimal value; state the maximum bid for your spot instance. Ignored if `spot_instance` is not true. +* `spot_max_price` - Decimal value; state the maximum bid for your spot instance. If nil, it will compute average price in `region` for selected `instance_type`. +* `spot_price_product_description` - The product description for the spot price history used to compute average price. Defaults to 'Linux/UNIX'. * `spot_valid_until` - Timestamp; when this spot instance request should expire, destroying any related instances. Ignored if `spot_instance` is not true. These can be set like typical provider-specific configuration: diff --git a/lib/vagrant-aws/action/run_instance.rb b/lib/vagrant-aws/action/run_instance.rb index 6786f499..169666f4 100644 --- a/lib/vagrant-aws/action/run_instance.rb +++ b/lib/vagrant-aws/action/run_instance.rb @@ -218,13 +218,38 @@ def call(env) # returns a fog server or nil def server_from_spot_request(env, config, options) + if config.spot_max_price.nil? + spot_price_current = env[:aws_compute].describe_spot_price_history({ + 'StartTime' => Time.now.iso8601, + 'EndTime' => Time.now.iso8601, + 'InstanceType' => [config.instance_type], + 'ProductDescription' => [config.spot_price_product_description.nil? ? 'Linux/UNIX' : config.spot_price_product_description] + }) + + spot_price_current.body['spotPriceHistorySet'].each do |set| + (@price_set ||= []) << set['spotPrice'].to_f + end + + if @price_set.nil? + raise Errors::FogError, + :message => "Could not find any history spot prices." + end + + avg_price = @price_set.inject(0.0) { |sum, el| sum + el } / @price_set.size + + # make the bid 10% higher than the average + price = (avg_price * 1.1).round(4) + else + price = config.spot_max_price + end + options.merge!({ - :price => config.spot_max_price, + :price => price, :valid_until => config.spot_valid_until }) env[:ui].info(I18n.t("vagrant_aws.launching_spot_instance")) - env[:ui].info(" -- Price: #{config.spot_max_price}") + env[:ui].info(" -- Price: #{price}") env[:ui].info(" -- Valid until: #{config.spot_valid_until}") if config.spot_valid_until # create the spot instance diff --git a/lib/vagrant-aws/config.rb b/lib/vagrant-aws/config.rb index f4fd597b..c7b460c6 100644 --- a/lib/vagrant-aws/config.rb +++ b/lib/vagrant-aws/config.rb @@ -211,6 +211,11 @@ class Config < Vagrant.plugin("2", :config) # @return [Time] attr_accessor :spot_valid_until + # The product description for the spot price history + # + # @return [String] + attr_accessor :spot_price_product_description + def initialize(region_specific=false) @access_key_id = UNSET_VALUE @ami = UNSET_VALUE @@ -430,12 +435,15 @@ def finalize! # By default don't use spot requests @spot_instance = false if @spot_instance == UNSET_VALUE - # Required, no default + # default to nil @spot_max_price = nil if @spot_max_price == UNSET_VALUE # Default: Request is effective indefinitely. @spot_valid_until = nil if @spot_valid_until == UNSET_VALUE + # default to nil + @spot_price_product_description = nil if @spot_price_product_description == UNSET_VALUE + # Compile our region specific configurations only within # NON-REGION-SPECIFIC configurations. if !@__region_specific @@ -487,7 +495,6 @@ def validate(machine) end errors << I18n.t("vagrant_aws.config.ami_required", :region => @region) if config.ami.nil? - errors << I18n.t("vagrant_aws.config.spot_price_required") if config.spot_instance && config.spot_max_price.nil? end { "AWS Provider" => errors } diff --git a/locales/en.yml b/locales/en.yml index 10b219f2..2a11409b 100644 --- a/locales/en.yml +++ b/locales/en.yml @@ -70,8 +70,6 @@ en: An access key ID must be specified via "access_key_id" ami_required: |- An AMI must be configured via "ami" (region: #{region}) - spot_price_required: |- - Spot request is missing "spot_max_price" private_key_missing: |- The specified private key for AWS could not be found region_required: |- diff --git a/spec/vagrant-aws/config_spec.rb b/spec/vagrant-aws/config_spec.rb index 48eda53f..656a7396 100644 --- a/spec/vagrant-aws/config_spec.rb +++ b/spec/vagrant-aws/config_spec.rb @@ -60,6 +60,7 @@ its("tenancy") { should == "default" } its("spot_instance") { should == false } its("spot_max_price") { should be_nil } + its("spot_price_product_description") { should be_nil } its("spot_valid_until") { should be_nil } end