From 5371044b6fc1dee4130717f139e267278311eb0d Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Mon, 7 Nov 2016 09:53:44 -0500 Subject: [PATCH] Spot fleet (#760) * initial spot fleet. * Add cloudformation spot fleet support. * If no spot fleet ids, return all. --- moto/cloudformation/parsing.py | 1 + moto/ec2/models.py | 175 +++++++++++++++++- moto/ec2/responses/__init__.py | 2 + moto/ec2/responses/spot_fleets.py | 122 ++++++++++++ moto/ec2/utils.py | 5 + .../test_cloudformation_stack_integration.py | 75 ++++++++ tests/test_ec2/test_spot_fleet.py | 143 ++++++++++++++ 7 files changed, 518 insertions(+), 5 deletions(-) create mode 100644 moto/ec2/responses/spot_fleets.py create mode 100644 tests/test_ec2/test_spot_fleet.py diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index 690b95631..06b460495 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -35,6 +35,7 @@ MODEL_MAP = { "AWS::EC2::RouteTable": ec2_models.RouteTable, "AWS::EC2::SecurityGroup": ec2_models.SecurityGroup, "AWS::EC2::SecurityGroupIngress": ec2_models.SecurityGroupIngress, + "AWS::EC2::SpotFleet": ec2_models.SpotFleetRequest, "AWS::EC2::Subnet": ec2_models.Subnet, "AWS::EC2::SubnetRouteTableAssociation": ec2_models.SubnetRouteTableAssociation, "AWS::EC2::Volume": ec2_models.Volume, diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 902ebdac4..3c4da13b4 100755 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -15,7 +15,7 @@ from boto.ec2.launchspecification import LaunchSpecification from moto.core import BaseBackend from moto.core.models import Model -from moto.core.utils import iso_8601_datetime_with_milliseconds +from moto.core.utils import iso_8601_datetime_with_milliseconds, camelcase_to_underscores from .exceptions import ( EC2ClientError, DependencyViolationError, @@ -81,6 +81,7 @@ from .utils import ( split_route_id, random_security_group_id, random_snapshot_id, + random_spot_fleet_request_id, random_spot_request_id, random_subnet_id, random_subnet_association_id, @@ -2565,6 +2566,170 @@ class SpotRequestBackend(object): return requests +class SpotFleetLaunchSpec(object): + def __init__(self, ebs_optimized, group_set, iam_instance_profile, image_id, + instance_type, key_name, monitoring, spot_price, subnet_id, user_data, + weighted_capacity): + self.ebs_optimized = ebs_optimized + self.group_set = group_set + self.iam_instance_profile = iam_instance_profile + self.image_id = image_id + self.instance_type = instance_type + self.key_name = key_name + self.monitoring = monitoring + self.spot_price = spot_price + self.subnet_id = subnet_id + self.user_data = user_data + self.weighted_capacity = float(weighted_capacity) + + +class SpotFleetRequest(TaggedEC2Resource): + + def __init__(self, ec2_backend, spot_fleet_request_id, spot_price, + target_capacity, iam_fleet_role, allocation_strategy, launch_specs): + + self.ec2_backend = ec2_backend + self.id = spot_fleet_request_id + self.spot_price = spot_price + self.target_capacity = int(target_capacity) + self.iam_fleet_role = iam_fleet_role + self.allocation_strategy = allocation_strategy + self.state = "active" + self.fulfilled_capacity = self.target_capacity + + self.launch_specs = [] + for spec in launch_specs: + self.launch_specs.append(SpotFleetLaunchSpec( + ebs_optimized=spec['ebs_optimized'], + group_set=[val for key, val in spec.items() if key.startswith("group_set")], + iam_instance_profile=spec.get('iam_instance_profile._arn'), + image_id=spec['image_id'], + instance_type=spec['instance_type'], + key_name=spec.get('key_name'), + monitoring=spec.get('monitoring._enabled'), + spot_price=spec.get('spot_price', self.spot_price), + subnet_id=spec['subnet_id'], + user_data=spec.get('user_data'), + weighted_capacity=spec['weighted_capacity'], + ) + ) + + self.spot_requests = [] + self.create_spot_requests() + + @property + def physical_resource_id(self): + return self.id + + @classmethod + def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): + properties = cloudformation_json['Properties']['SpotFleetRequestConfigData'] + ec2_backend = ec2_backends[region_name] + + spot_price = properties['SpotPrice'] + target_capacity = properties['TargetCapacity'] + iam_fleet_role = properties['IamFleetRole'] + allocation_strategy = properties['AllocationStrategy'] + launch_specs = properties["LaunchSpecifications"] + launch_specs = [ + dict([(camelcase_to_underscores(key), val) for key, val in launch_spec.items()]) + for launch_spec + in launch_specs + ] + + spot_fleet_request = ec2_backend.request_spot_fleet(spot_price, + target_capacity, iam_fleet_role, allocation_strategy, launch_specs) + + return spot_fleet_request + + + def get_launch_spec_counts(self): + weight_map = defaultdict(int) + + if self.allocation_strategy == 'diversified': + weight_so_far = 0 + launch_spec_index = 0 + while True: + launch_spec = self.launch_specs[launch_spec_index % len(self.launch_specs)] + weight_map[launch_spec] += 1 + weight_so_far += launch_spec.weighted_capacity + if weight_so_far >= self.target_capacity: + break + launch_spec_index += 1 + else: # lowestPrice + cheapest_spec = sorted(self.launch_specs, key=lambda spec: float(spec.spot_price))[0] + extra = 1 if self.target_capacity % cheapest_spec.weighted_capacity else 0 + weight_map[cheapest_spec] = int(self.target_capacity // cheapest_spec.weighted_capacity) + extra + + return weight_map.items() + + def create_spot_requests(self): + for launch_spec, count in self.get_launch_spec_counts(): + requests = self.ec2_backend.request_spot_instances( + price=launch_spec.spot_price, + image_id=launch_spec.image_id, + count=count, + type="persistent", + valid_from=None, + valid_until=None, + launch_group=None, + availability_zone_group=None, + key_name=launch_spec.key_name, + security_groups=launch_spec.group_set, + user_data=launch_spec.user_data, + instance_type=launch_spec.instance_type, + placement=None, + kernel_id=None, + ramdisk_id=None, + monitoring_enabled=launch_spec.monitoring, + subnet_id=launch_spec.subnet_id, + ) + self.spot_requests.extend(requests) + return self.spot_requests + + def terminate_instances(self): + pass + + +class SpotFleetBackend(object): + def __init__(self): + self.spot_fleet_requests = {} + super(SpotFleetBackend, self).__init__() + + def request_spot_fleet(self, spot_price, target_capacity, iam_fleet_role, + allocation_strategy, launch_specs): + + spot_fleet_request_id = random_spot_fleet_request_id() + request = SpotFleetRequest(self, spot_fleet_request_id, spot_price, + target_capacity, iam_fleet_role, allocation_strategy, launch_specs) + self.spot_fleet_requests[spot_fleet_request_id] = request + return request + + def get_spot_fleet_request(self, spot_fleet_request_id): + return self.spot_fleet_requests[spot_fleet_request_id] + + def describe_spot_fleet_instances(self, spot_fleet_request_id): + spot_fleet = self.get_spot_fleet_request(spot_fleet_request_id) + return spot_fleet.spot_requests + + def describe_spot_fleet_requests(self, spot_fleet_request_ids): + requests = self.spot_fleet_requests.values() + + if spot_fleet_request_ids: + requests = [request for request in requests if request.id in spot_fleet_request_ids] + + return requests + + def cancel_spot_fleet_requests(self, spot_fleet_request_ids, terminate_instances): + spot_requests = [] + for spot_fleet_request_id in spot_fleet_request_ids: + spot_fleet = self.spot_fleet_requests.pop(spot_fleet_request_id) + if terminate_instances: + spot_fleet.terminate_instances() + spot_requests.append(spot_fleet) + return spot_requests + + class ElasticAddress(object): def __init__(self, domain): self.public_ip = random_ip() @@ -3189,10 +3354,10 @@ class EC2Backend(BaseBackend, InstanceBackend, TagBackend, AmiBackend, NetworkInterfaceBackend, VPNConnectionBackend, VPCPeeringConnectionBackend, RouteTableBackend, RouteBackend, InternetGatewayBackend, - VPCGatewayAttachmentBackend, SpotRequestBackend, - ElasticAddressBackend, KeyPairBackend, DHCPOptionsSetBackend, - NetworkAclBackend, VpnGatewayBackend, CustomerGatewayBackend, - NatGatewayBackend): + VPCGatewayAttachmentBackend, SpotFleetBackend, + SpotRequestBackend,ElasticAddressBackend, KeyPairBackend, + DHCPOptionsSetBackend, NetworkAclBackend, VpnGatewayBackend, + CustomerGatewayBackend, NatGatewayBackend): def __init__(self, region_name): super(EC2Backend, self).__init__() diff --git a/moto/ec2/responses/__init__.py b/moto/ec2/responses/__init__.py index d939178fb..2049998ad 100644 --- a/moto/ec2/responses/__init__.py +++ b/moto/ec2/responses/__init__.py @@ -19,6 +19,7 @@ from .placement_groups import PlacementGroups from .reserved_instances import ReservedInstances from .route_tables import RouteTables from .security_groups import SecurityGroups +from .spot_fleets import SpotFleets from .spot_instances import SpotInstances from .subnets import Subnets from .tags import TagResponse @@ -52,6 +53,7 @@ class EC2Response( ReservedInstances, RouteTables, SecurityGroups, + SpotFleets, SpotInstances, Subnets, TagResponse, diff --git a/moto/ec2/responses/spot_fleets.py b/moto/ec2/responses/spot_fleets.py new file mode 100644 index 000000000..f49b60e68 --- /dev/null +++ b/moto/ec2/responses/spot_fleets.py @@ -0,0 +1,122 @@ +from __future__ import unicode_literals +from moto.core.responses import BaseResponse + + +class SpotFleets(BaseResponse): + + def cancel_spot_fleet_requests(self): + spot_fleet_request_ids = self._get_multi_param("SpotFleetRequestId.") + terminate_instances = self._get_param("TerminateInstances") + spot_fleets = self.ec2_backend.cancel_spot_fleet_requests(spot_fleet_request_ids, terminate_instances) + template = self.response_template(CANCEL_SPOT_FLEETS_TEMPLATE) + return template.render(spot_fleets=spot_fleets) + + def describe_spot_fleet_instances(self): + spot_fleet_request_id = self._get_param("SpotFleetRequestId") + + spot_requests = self.ec2_backend.describe_spot_fleet_instances(spot_fleet_request_id) + template = self.response_template(DESCRIBE_SPOT_FLEET_INSTANCES_TEMPLATE) + return template.render(spot_request_id=spot_fleet_request_id, spot_requests=spot_requests) + + def describe_spot_fleet_requests(self): + spot_fleet_request_ids = self._get_multi_param("SpotFleetRequestId.") + + requests = self.ec2_backend.describe_spot_fleet_requests(spot_fleet_request_ids) + template = self.response_template(DESCRIBE_SPOT_FLEET_TEMPLATE) + return template.render(requests=requests) + + def request_spot_fleet(self): + spot_config = self._get_dict_param("SpotFleetRequestConfig.") + spot_price = spot_config['spot_price'] + target_capacity = spot_config['target_capacity'] + iam_fleet_role = spot_config['iam_fleet_role'] + allocation_strategy = spot_config['allocation_strategy'] + + launch_specs = self._get_list_prefix("SpotFleetRequestConfig.LaunchSpecifications") + + request = self.ec2_backend.request_spot_fleet( + spot_price=spot_price, + target_capacity=target_capacity, + iam_fleet_role=iam_fleet_role, + allocation_strategy=allocation_strategy, + launch_specs=launch_specs, + ) + + template = self.response_template(REQUEST_SPOT_FLEET_TEMPLATE) + return template.render(request=request) + +REQUEST_SPOT_FLEET_TEMPLATE = """ + 60262cc5-2bd4-4c8d-98ed-example + {{ request.id }} +""" + +DESCRIBE_SPOT_FLEET_TEMPLATE = """ + 4d68a6cc-8f2e-4be1-b425-example + + {% for request in requests %} + + {{ request.id }} + {{ request.state }} + + {{ request.spot_price }} + {{ request.target_capacity }} + {{ request.iam_fleet_role }} + {{ request.allocation_strategy }} + {{ request.fulfilled_capacity }} + + {% for launch_spec in request.launch_specs %} + + {{ launch_spec.subnet_id }} + {{ launch_spec.ebs_optimized }} + {{ launch_spec.image_id }} + {{ launch_spec.instance_type }} + {{ launch_spec.iam_instance_profile }} + {{ launch_spec.key_name }} + {{ launch_spec.monitoring }} + {{ launch_spec.spot_price }} + {{ launch_spec.user_data }} + {{ launch_spec.weighted_capacity }} + + {% for group in launch_spec.group_set %} + + {{ group }} + + {% endfor %} + + + {% endfor %} + + + + {% endfor %} + +""" + +DESCRIBE_SPOT_FLEET_INSTANCES_TEMPLATE = """ + cfb09950-45e2-472d-a6a9-example + {{ spot_request_id }} + + {% for spot_request in spot_requests %} + + {{ spot_request.instance_id }} + {{ spot_request.id }} + {{ spot_request.instance_type }} + + {% endfor %} + + +""" + +CANCEL_SPOT_FLEETS_TEMPLATE = """ + e12d2fe5-6503-4b4b-911c-example + + + {% for spot_fleet in spot_fleets %} + + {{ spot_fleet.id }} + cancelled_terminating + active + + {% endfor %} + +""" diff --git a/moto/ec2/utils.py b/moto/ec2/utils.py index d9ebb4258..4d0f75254 100644 --- a/moto/ec2/utils.py +++ b/moto/ec2/utils.py @@ -20,6 +20,7 @@ EC2_RESOURCE_TO_PREFIX = { 'security-group': 'sg', 'snapshot': 'snap', 'spot-instance-request': 'sir', + 'spot-fleet-request': 'sfr', 'subnet': 'subnet', 'reservation': 'r', 'volume': 'vol', @@ -65,6 +66,10 @@ def random_spot_request_id(): return random_id(prefix=EC2_RESOURCE_TO_PREFIX['spot-instance-request']) +def random_spot_fleet_request_id(): + return random_id(prefix=EC2_RESOURCE_TO_PREFIX['spot-fleet-request']) + + def random_subnet_id(): return random_id(prefix=EC2_RESOURCE_TO_PREFIX['subnet']) diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index d3928dc48..59a179618 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -1876,3 +1876,78 @@ def test_stack_kms(): result['KeyMetadata']['Enabled'].should.equal(True) result['KeyMetadata']['KeyUsage'].should.equal('ENCRYPT_DECRYPT') + + +@mock_cloudformation() +@mock_ec2() +def test_stack_spot_fleet(): + spot_fleet_template = { + 'Resources': { + "SpotFleet": { + "Type": "AWS::EC2::SpotFleet", + "Properties": { + "SpotFleetRequestConfigData": { + "IamFleetRole": "arn:aws:iam::123456789012:role/fleet", + "SpotPrice": "0.12", + "TargetCapacity": 6, + "AllocationStrategy": "diversified", + "LaunchSpecifications": [ + { + "EbsOptimized": "false", + "InstanceType": 't2.small', + "ImageId": "ami-1234", + "SubnetId": "subnet-123", + "WeightedCapacity": "2", + "SpotPrice": "0.13", + }, + { + "EbsOptimized": "true", + "InstanceType": 't2.large', + "ImageId": "ami-1234", + "Monitoring": { "Enabled": "true" }, + "SecurityGroups": [{"GroupId": "sg-123"}], + "SubnetId": "subnet-123", + "IamInstanceProfile": {"Arn": "arn:aws:iam::123456789012:role/fleet"}, + "WeightedCapacity": "4", + "SpotPrice": "10.00", + } + ] + } + } + } + } + } + spot_fleet_template_json = json.dumps(spot_fleet_template) + + cf_conn = boto3.client('cloudformation', 'us-east-1') + stack_id = cf_conn.create_stack( + StackName='test_stack', + TemplateBody=spot_fleet_template_json, + )['StackId'] + + stack_resources = cf_conn.list_stack_resources(StackName=stack_id) + stack_resources['StackResourceSummaries'].should.have.length_of(1) + spot_fleet_id = stack_resources['StackResourceSummaries'][0]['PhysicalResourceId'] + + conn = boto3.client('ec2', 'us-east-1') + spot_fleet_requests = conn.describe_spot_fleet_requests(SpotFleetRequestIds=[spot_fleet_id])['SpotFleetRequestConfigs'] + len(spot_fleet_requests).should.equal(1) + spot_fleet_request = spot_fleet_requests[0] + spot_fleet_request['SpotFleetRequestState'].should.equal("active") + spot_fleet_config = spot_fleet_request['SpotFleetRequestConfig'] + + spot_fleet_config['SpotPrice'].should.equal('0.12') + spot_fleet_config['TargetCapacity'].should.equal(6) + spot_fleet_config['IamFleetRole'].should.equal('arn:aws:iam::123456789012:role/fleet') + spot_fleet_config['AllocationStrategy'].should.equal('diversified') + spot_fleet_config['FulfilledCapacity'].should.equal(6.0) + + len(spot_fleet_config['LaunchSpecifications']).should.equal(2) + launch_spec = spot_fleet_config['LaunchSpecifications'][0] + + launch_spec['EbsOptimized'].should.equal(False) + launch_spec['ImageId'].should.equal("ami-1234") + launch_spec['InstanceType'].should.equal("t2.small") + launch_spec['SubnetId'].should.equal("subnet-123") + launch_spec['SpotPrice'].should.equal("0.13") + launch_spec['WeightedCapacity'].should.equal(2.0) diff --git a/tests/test_ec2/test_spot_fleet.py b/tests/test_ec2/test_spot_fleet.py new file mode 100644 index 000000000..9a52ec80b --- /dev/null +++ b/tests/test_ec2/test_spot_fleet.py @@ -0,0 +1,143 @@ +from __future__ import unicode_literals + +import boto3 +import sure # noqa + +from moto import mock_ec2 + +SPOT_REQUEST_CONFIG = { + 'ClientToken': 'string', + 'SpotPrice': '0.12', + 'TargetCapacity': 6, + 'IamFleetRole': 'arn:aws:iam::123456789012:role/fleet', + 'LaunchSpecifications': [{ + 'ImageId': 'ami-123', + 'KeyName': 'my-key', + 'SecurityGroups': [ + { + 'GroupId': 'sg-123' + }, + ], + 'UserData': 'some user data', + 'InstanceType': 't2.small', + 'BlockDeviceMappings': [ + { + 'VirtualName': 'string', + 'DeviceName': 'string', + 'Ebs': { + 'SnapshotId': 'string', + 'VolumeSize': 123, + 'DeleteOnTermination': True|False, + 'VolumeType': 'standard', + 'Iops': 123, + 'Encrypted': True|False + }, + 'NoDevice': 'string' + }, + ], + 'Monitoring': { + 'Enabled': True + }, + 'SubnetId': 'subnet-1234', + 'IamInstanceProfile': { + 'Arn': 'arn:aws:iam::123456789012:role/fleet' + }, + 'EbsOptimized': False, + 'WeightedCapacity': 2.0, + 'SpotPrice': '0.13' + }, { + 'ImageId': 'ami-123', + 'KeyName': 'my-key', + 'SecurityGroups': [ + { + 'GroupId': 'sg-123' + }, + ], + 'UserData': 'some user data', + 'InstanceType': 't2.large', + 'Monitoring': { + 'Enabled': True + }, + 'SubnetId': 'subnet-1234', + 'IamInstanceProfile': { + 'Arn': 'arn:aws:iam::123456789012:role/fleet' + }, + 'EbsOptimized': False, + 'WeightedCapacity': 4.0, + 'SpotPrice': '10.00', + }], + 'AllocationStrategy': 'lowestPrice', + 'FulfilledCapacity': 6, +} + + +@mock_ec2 +def test_create_spot_fleet_with_lowest_price(): + conn = boto3.client("ec2", region_name='us-west-2') + + spot_fleet_res = conn.request_spot_fleet( + SpotFleetRequestConfig=SPOT_REQUEST_CONFIG + ) + spot_fleet_id = spot_fleet_res['SpotFleetRequestId'] + + spot_fleet_requests = conn.describe_spot_fleet_requests(SpotFleetRequestIds=[spot_fleet_id])['SpotFleetRequestConfigs'] + len(spot_fleet_requests).should.equal(1) + spot_fleet_request = spot_fleet_requests[0] + spot_fleet_request['SpotFleetRequestState'].should.equal("active") + spot_fleet_config = spot_fleet_request['SpotFleetRequestConfig'] + + spot_fleet_config['SpotPrice'].should.equal('0.12') + spot_fleet_config['TargetCapacity'].should.equal(6) + spot_fleet_config['IamFleetRole'].should.equal('arn:aws:iam::123456789012:role/fleet') + spot_fleet_config['AllocationStrategy'].should.equal('lowestPrice') + spot_fleet_config['FulfilledCapacity'].should.equal(6.0) + + len(spot_fleet_config['LaunchSpecifications']).should.equal(2) + launch_spec = spot_fleet_config['LaunchSpecifications'][0] + + launch_spec['EbsOptimized'].should.equal(False) + launch_spec['SecurityGroups'].should.equal([{"GroupId": "sg-123"}]) + launch_spec['IamInstanceProfile'].should.equal({"Arn": "arn:aws:iam::123456789012:role/fleet"}) + launch_spec['ImageId'].should.equal("ami-123") + launch_spec['InstanceType'].should.equal("t2.small") + launch_spec['KeyName'].should.equal("my-key") + launch_spec['Monitoring'].should.equal({"Enabled": True}) + launch_spec['SpotPrice'].should.equal("0.13") + launch_spec['SubnetId'].should.equal("subnet-1234") + launch_spec['UserData'].should.equal("some user data") + launch_spec['WeightedCapacity'].should.equal(2.0) + + instance_res = conn.describe_spot_fleet_instances(SpotFleetRequestId=spot_fleet_id) + instances = instance_res['ActiveInstances'] + len(instances).should.equal(3) + + +@mock_ec2 +def test_create_diversified_spot_fleet(): + conn = boto3.client("ec2", region_name='us-west-2') + diversified_config = SPOT_REQUEST_CONFIG.copy() + diversified_config['AllocationStrategy'] = 'diversified' + + spot_fleet_res = conn.request_spot_fleet( + SpotFleetRequestConfig=diversified_config + ) + spot_fleet_id = spot_fleet_res['SpotFleetRequestId'] + + instance_res = conn.describe_spot_fleet_instances(SpotFleetRequestId=spot_fleet_id) + instances = instance_res['ActiveInstances'] + len(instances).should.equal(2) + + +@mock_ec2 +def test_cancel_spot_fleet_request(): + conn = boto3.client("ec2", region_name='us-west-2') + + spot_fleet_res = conn.request_spot_fleet( + SpotFleetRequestConfig=SPOT_REQUEST_CONFIG, + ) + spot_fleet_id = spot_fleet_res['SpotFleetRequestId'] + + conn.cancel_spot_fleet_requests(SpotFleetRequestIds=[spot_fleet_id], TerminateInstances=True) + + spot_fleet_requests = conn.describe_spot_fleet_requests(SpotFleetRequestIds=[spot_fleet_id])['SpotFleetRequestConfigs'] + len(spot_fleet_requests).should.equal(0)