Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Spot fleet (#760)

Browse Source 
* initial spot fleet.

* Add cloudformation spot fleet support.

* If no spot fleet ids, return all.
This commit is contained in:
Steve Pulec 2016-11-07 09:53:44 -05:00 committed by GitHub
parent 078156c642
commit 5371044b6f
7 changed files with 518 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
moto/cloudformation/parsing.py
View File

@ -35,6 +35,7 @@ MODEL_MAP = {
"AWS::EC2::RouteTable": ec2_models.RouteTable,
"AWS::EC2::SecurityGroup": ec2_models.SecurityGroup,
"AWS::EC2::SecurityGroupIngress": ec2_models.SecurityGroupIngress,
"AWS::EC2::SpotFleet": ec2_models.SpotFleetRequest,
"AWS::EC2::Subnet": ec2_models.Subnet,
"AWS::EC2::SubnetRouteTableAssociation": ec2_models.SubnetRouteTableAssociation,
"AWS::EC2::Volume": ec2_models.Volume,

175
moto/ec2/models.py
View File

@ -15,7 +15,7 @@ from boto.ec2.launchspecification import LaunchSpecification
from moto.core import BaseBackend
from moto.core.models import Model
from moto.core.utils import iso_8601_datetime_with_milliseconds
from moto.core.utils import iso_8601_datetime_with_milliseconds, camelcase_to_underscores
from .exceptions import (
EC2ClientError,
DependencyViolationError,
@ -81,6 +81,7 @@ from .utils import (
split_route_id,
random_security_group_id,
random_snapshot_id,
random_spot_fleet_request_id,
random_spot_request_id,
random_subnet_id,
random_subnet_association_id,
@ -2565,6 +2566,170 @@ class SpotRequestBackend(object):
return requests
class SpotFleetLaunchSpec(object):
def __init__(self, ebs_optimized, group_set, iam_instance_profile, image_id,
instance_type, key_name, monitoring, spot_price, subnet_id, user_data,
weighted_capacity):
self.ebs_optimized = ebs_optimized
self.group_set = group_set
self.iam_instance_profile = iam_instance_profile
self.image_id = image_id
self.instance_type = instance_type
self.key_name = key_name
self.monitoring = monitoring
self.spot_price = spot_price
self.subnet_id = subnet_id
self.user_data = user_data
self.weighted_capacity = float(weighted_capacity)
class SpotFleetRequest(TaggedEC2Resource):
def __init__(self, ec2_backend, spot_fleet_request_id, spot_price,
target_capacity, iam_fleet_role, allocation_strategy, launch_specs):
self.ec2_backend = ec2_backend
self.id = spot_fleet_request_id
self.spot_price = spot_price
self.target_capacity = int(target_capacity)
self.iam_fleet_role = iam_fleet_role
self.allocation_strategy = allocation_strategy
self.state = "active"
self.fulfilled_capacity = self.target_capacity
self.launch_specs = []
for spec in launch_specs:
self.launch_specs.append(SpotFleetLaunchSpec(
ebs_optimized=spec['ebs_optimized'],
group_set=[val for key, val in spec.items() if key.startswith("group_set")],
iam_instance_profile=spec.get('iam_instance_profile._arn'),
image_id=spec['image_id'],
instance_type=spec['instance_type'],
key_name=spec.get('key_name'),
monitoring=spec.get('monitoring._enabled'),
spot_price=spec.get('spot_price', self.spot_price),
subnet_id=spec['subnet_id'],
user_data=spec.get('user_data'),
weighted_capacity=spec['weighted_capacity'],
)
)
self.spot_requests = []
self.create_spot_requests()
@property
def physical_resource_id(self):
return self.id
@classmethod
def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name):
properties = cloudformation_json['Properties']['SpotFleetRequestConfigData']
ec2_backend = ec2_backends[region_name]
spot_price = properties['SpotPrice']
target_capacity = properties['TargetCapacity']
iam_fleet_role = properties['IamFleetRole']
allocation_strategy = properties['AllocationStrategy']
launch_specs = properties["LaunchSpecifications"]
launch_specs = [
dict([(camelcase_to_underscores(key), val) for key, val in launch_spec.items()])
for launch_spec
in launch_specs
]
spot_fleet_request = ec2_backend.request_spot_fleet(spot_price,
target_capacity, iam_fleet_role, allocation_strategy, launch_specs)
return spot_fleet_request
def get_launch_spec_counts(self):
weight_map = defaultdict(int)
if self.allocation_strategy == 'diversified':
weight_so_far = 0
launch_spec_index = 0
while True:
launch_spec = self.launch_specs[launch_spec_index % len(self.launch_specs)]
weight_map[launch_spec] += 1
weight_so_far += launch_spec.weighted_capacity
if weight_so_far >= self.target_capacity:
break
launch_spec_index += 1
else: # lowestPrice
cheapest_spec = sorted(self.launch_specs, key=lambda spec: float(spec.spot_price))[0]
extra = 1 if self.target_capacity % cheapest_spec.weighted_capacity else 0
weight_map[cheapest_spec] = int(self.target_capacity // cheapest_spec.weighted_capacity) + extra
return weight_map.items()
def create_spot_requests(self):
for launch_spec, count in self.get_launch_spec_counts():
requests = self.ec2_backend.request_spot_instances(
price=launch_spec.spot_price,
image_id=launch_spec.image_id,
count=count,
type="persistent",
valid_from=None,
valid_until=None,
launch_group=None,
availability_zone_group=None,
key_name=launch_spec.key_name,
security_groups=launch_spec.group_set,
user_data=launch_spec.user_data,
instance_type=launch_spec.instance_type,
placement=None,
kernel_id=None,
ramdisk_id=None,
monitoring_enabled=launch_spec.monitoring,
subnet_id=launch_spec.subnet_id,
)
self.spot_requests.extend(requests)
return self.spot_requests
def terminate_instances(self):
pass
class SpotFleetBackend(object):
def __init__(self):
self.spot_fleet_requests = {}
super(SpotFleetBackend, self).__init__()
def request_spot_fleet(self, spot_price, target_capacity, iam_fleet_role,
allocation_strategy, launch_specs):
spot_fleet_request_id = random_spot_fleet_request_id()
request = SpotFleetRequest(self, spot_fleet_request_id, spot_price,
target_capacity, iam_fleet_role, allocation_strategy, launch_specs)
self.spot_fleet_requests[spot_fleet_request_id] = request
return request
def get_spot_fleet_request(self, spot_fleet_request_id):
return self.spot_fleet_requests[spot_fleet_request_id]
def describe_spot_fleet_instances(self, spot_fleet_request_id):
spot_fleet = self.get_spot_fleet_request(spot_fleet_request_id)
return spot_fleet.spot_requests
def describe_spot_fleet_requests(self, spot_fleet_request_ids):
requests = self.spot_fleet_requests.values()
if spot_fleet_request_ids:
requests = [request for request in requests if request.id in spot_fleet_request_ids]
return requests
def cancel_spot_fleet_requests(self, spot_fleet_request_ids, terminate_instances):
spot_requests = []
for spot_fleet_request_id in spot_fleet_request_ids:
spot_fleet = self.spot_fleet_requests.pop(spot_fleet_request_id)
if terminate_instances:
spot_fleet.terminate_instances()
spot_requests.append(spot_fleet)
return spot_requests
class ElasticAddress(object):
def __init__(self, domain):
self.public_ip = random_ip()
@ -3189,10 +3354,10 @@ class EC2Backend(BaseBackend, InstanceBackend, TagBackend, AmiBackend,
NetworkInterfaceBackend, VPNConnectionBackend,
VPCPeeringConnectionBackend,
RouteTableBackend, RouteBackend, InternetGatewayBackend,
VPCGatewayAttachmentBackend, SpotRequestBackend,
ElasticAddressBackend, KeyPairBackend, DHCPOptionsSetBackend,
NetworkAclBackend, VpnGatewayBackend, CustomerGatewayBackend,
NatGatewayBackend):
VPCGatewayAttachmentBackend, SpotFleetBackend,
SpotRequestBackend,ElasticAddressBackend, KeyPairBackend,
DHCPOptionsSetBackend, NetworkAclBackend, VpnGatewayBackend,
CustomerGatewayBackend, NatGatewayBackend):
def __init__(self, region_name):
super(EC2Backend, self).__init__()

2
moto/ec2/responses/__init__.py
View File

@ -19,6 +19,7 @@ from .placement_groups import PlacementGroups
from .reserved_instances import ReservedInstances
from .route_tables import RouteTables
from .security_groups import SecurityGroups
from .spot_fleets import SpotFleets
from .spot_instances import SpotInstances
from .subnets import Subnets
from .tags import TagResponse
@ -52,6 +53,7 @@ class EC2Response(
ReservedInstances,
RouteTables,
SecurityGroups,
SpotFleets,
SpotInstances,
Subnets,
TagResponse,

122
moto/ec2/responses/spot_fleets.py Normal file
View File

@ -0,0 +1,122 @@
from __future__ import unicode_literals
from moto.core.responses import BaseResponse
class SpotFleets(BaseResponse):
def cancel_spot_fleet_requests(self):
spot_fleet_request_ids = self._get_multi_param("SpotFleetRequestId.")
terminate_instances = self._get_param("TerminateInstances")
spot_fleets = self.ec2_backend.cancel_spot_fleet_requests(spot_fleet_request_ids, terminate_instances)
template = self.response_template(CANCEL_SPOT_FLEETS_TEMPLATE)
return template.render(spot_fleets=spot_fleets)
def describe_spot_fleet_instances(self):
spot_fleet_request_id = self._get_param("SpotFleetRequestId")
spot_requests = self.ec2_backend.describe_spot_fleet_instances(spot_fleet_request_id)
template = self.response_template(DESCRIBE_SPOT_FLEET_INSTANCES_TEMPLATE)
return template.render(spot_request_id=spot_fleet_request_id, spot_requests=spot_requests)
def describe_spot_fleet_requests(self):
spot_fleet_request_ids = self._get_multi_param("SpotFleetRequestId.")
requests = self.ec2_backend.describe_spot_fleet_requests(spot_fleet_request_ids)
template = self.response_template(DESCRIBE_SPOT_FLEET_TEMPLATE)
return template.render(requests=requests)
def request_spot_fleet(self):
spot_config = self._get_dict_param("SpotFleetRequestConfig.")
spot_price = spot_config['spot_price']
target_capacity = spot_config['target_capacity']
iam_fleet_role = spot_config['iam_fleet_role']
allocation_strategy = spot_config['allocation_strategy']
launch_specs = self._get_list_prefix("SpotFleetRequestConfig.LaunchSpecifications")
request = self.ec2_backend.request_spot_fleet(
spot_price=spot_price,
target_capacity=target_capacity,
iam_fleet_role=iam_fleet_role,
allocation_strategy=allocation_strategy,
launch_specs=launch_specs,
)
template = self.response_template(REQUEST_SPOT_FLEET_TEMPLATE)
return template.render(request=request)
REQUEST_SPOT_FLEET_TEMPLATE = """<RequestSpotFleetResponse xmlns="http://ec2.amazonaws.com/doc/2016-09-15/">
<requestId>60262cc5-2bd4-4c8d-98ed-example</requestId>
<spotFleetRequestId>{{ request.id }}</spotFleetRequestId>
</RequestSpotFleetResponse>"""
DESCRIBE_SPOT_FLEET_TEMPLATE = """<DescribeSpotFleetRequestsResponse xmlns="http://ec2.amazonaws.com/doc/2016-09-15/">
<requestId>4d68a6cc-8f2e-4be1-b425-example</requestId>
<spotFleetRequestConfigSet>
{% for request in requests %}
<item>
<spotFleetRequestId>{{ request.id }}</spotFleetRequestId>
<spotFleetRequestState>{{ request.state }}</spotFleetRequestState>
<spotFleetRequestConfig>
<spotPrice>{{ request.spot_price }}</spotPrice>
<targetCapacity>{{ request.target_capacity }}</targetCapacity>
<iamFleetRole>{{ request.iam_fleet_role }}</iamFleetRole>
<allocationStrategy>{{ request.allocation_strategy }}</allocationStrategy>
<fulfilledCapacity>{{ request.fulfilled_capacity }}</fulfilledCapacity>
<launchSpecifications>
{% for launch_spec in request.launch_specs %}
<item>
<subnetId>{{ launch_spec.subnet_id }}</subnetId>
<ebsOptimized>{{ launch_spec.ebs_optimized }}</ebsOptimized>
<imageId>{{ launch_spec.image_id }}</imageId>
<instanceType>{{ launch_spec.instance_type }}</instanceType>
<iamInstanceProfile><arn>{{ launch_spec.iam_instance_profile }}</arn></iamInstanceProfile>
<keyName>{{ launch_spec.key_name }}</keyName>
<monitoring><enabled>{{ launch_spec.monitoring }}</enabled></monitoring>
<spotPrice>{{ launch_spec.spot_price }}</spotPrice>
<userData>{{ launch_spec.user_data }}</userData>
<weightedCapacity>{{ launch_spec.weighted_capacity }}</weightedCapacity>
<groupSet>
{% for group in launch_spec.group_set %}
<item>
<groupId>{{ group }}</groupId>
</item>
{% endfor %}
</groupSet>
</item>
{% endfor %}
</launchSpecifications>
</spotFleetRequestConfig>
</item>
{% endfor %}
</spotFleetRequestConfigSet>
</DescribeSpotFleetRequestsResponse>"""
DESCRIBE_SPOT_FLEET_INSTANCES_TEMPLATE = """<DescribeSpotFleetInstancesResponse xmlns="http://ec2.amazonaws.com/doc/2016-09-15/">
<requestId>cfb09950-45e2-472d-a6a9-example</requestId>
<spotFleetRequestId>{{ spot_request_id }}</spotFleetRequestId>
<activeInstanceSet>
{% for spot_request in spot_requests %}
<item>
<instanceId>{{ spot_request.instance_id }}</instanceId>
<spotInstanceRequestId>{{ spot_request.id }}</spotInstanceRequestId>
<instanceType>{{ spot_request.instance_type }}</instanceType>
</item>
{% endfor %}
</activeInstanceSet>
</DescribeSpotFleetInstancesResponse>
"""
CANCEL_SPOT_FLEETS_TEMPLATE = """<CancelSpotFleetRequestsResponse xmlns="http://ec2.amazonaws.com/doc/2016-09-15/">
<requestId>e12d2fe5-6503-4b4b-911c-example</requestId>
<unsuccessfulFleetRequestSet/>
<successfulFleetRequestSet>
{% for spot_fleet in spot_fleets %}
<item>
<spotFleetRequestId>{{ spot_fleet.id }}</spotFleetRequestId>
<currentSpotFleetRequestState>cancelled_terminating</currentSpotFleetRequestState>
<previousSpotFleetRequestState>active</previousSpotFleetRequestState>
</item>
{% endfor %}
</successfulFleetRequestSet>
</CancelSpotFleetRequestsResponse>"""

5
moto/ec2/utils.py
View File

@ -20,6 +20,7 @@ EC2_RESOURCE_TO_PREFIX = {
'security-group': 'sg',
'snapshot': 'snap',
'spot-instance-request': 'sir',
'spot-fleet-request': 'sfr',
'subnet': 'subnet',
'reservation': 'r',
'volume': 'vol',
@ -65,6 +66,10 @@ def random_spot_request_id():
return random_id(prefix=EC2_RESOURCE_TO_PREFIX['spot-instance-request'])
def random_spot_fleet_request_id():
return random_id(prefix=EC2_RESOURCE_TO_PREFIX['spot-fleet-request'])
def random_subnet_id():
return random_id(prefix=EC2_RESOURCE_TO_PREFIX['subnet'])

75
tests/test_cloudformation/test_cloudformation_stack_integration.py
View File

@ -1876,3 +1876,78 @@ def test_stack_kms():
result['KeyMetadata']['Enabled'].should.equal(True)
result['KeyMetadata']['KeyUsage'].should.equal('ENCRYPT_DECRYPT')
@mock_cloudformation()
@mock_ec2()
def test_stack_spot_fleet():
spot_fleet_template = {
'Resources': {
"SpotFleet": {
"Type": "AWS::EC2::SpotFleet",
"Properties": {
"SpotFleetRequestConfigData": {
"IamFleetRole": "arn:aws:iam::123456789012:role/fleet",
"SpotPrice": "0.12",
"TargetCapacity": 6,
"AllocationStrategy": "diversified",
"LaunchSpecifications": [
{
"EbsOptimized": "false",
"InstanceType": 't2.small',
"ImageId": "ami-1234",
"SubnetId": "subnet-123",
"WeightedCapacity": "2",
"SpotPrice": "0.13",
},
{
"EbsOptimized": "true",
"InstanceType": 't2.large',
"ImageId": "ami-1234",
"Monitoring": { "Enabled": "true" },
"SecurityGroups": [{"GroupId": "sg-123"}],
"SubnetId": "subnet-123",
"IamInstanceProfile": {"Arn": "arn:aws:iam::123456789012:role/fleet"},
"WeightedCapacity": "4",
"SpotPrice": "10.00",
}
]
}
}
}
}
}
spot_fleet_template_json = json.dumps(spot_fleet_template)
cf_conn = boto3.client('cloudformation', 'us-east-1')
stack_id = cf_conn.create_stack(
StackName='test_stack',
TemplateBody=spot_fleet_template_json,
)['StackId']
stack_resources = cf_conn.list_stack_resources(StackName=stack_id)
stack_resources['StackResourceSummaries'].should.have.length_of(1)
spot_fleet_id = stack_resources['StackResourceSummaries'][0]['PhysicalResourceId']
conn = boto3.client('ec2', 'us-east-1')
spot_fleet_requests = conn.describe_spot_fleet_requests(SpotFleetRequestIds=[spot_fleet_id])['SpotFleetRequestConfigs']
len(spot_fleet_requests).should.equal(1)
spot_fleet_request = spot_fleet_requests[0]
spot_fleet_request['SpotFleetRequestState'].should.equal("active")
spot_fleet_config = spot_fleet_request['SpotFleetRequestConfig']
spot_fleet_config['SpotPrice'].should.equal('0.12')
spot_fleet_config['TargetCapacity'].should.equal(6)
spot_fleet_config['IamFleetRole'].should.equal('arn:aws:iam::123456789012:role/fleet')
spot_fleet_config['AllocationStrategy'].should.equal('diversified')
spot_fleet_config['FulfilledCapacity'].should.equal(6.0)
len(spot_fleet_config['LaunchSpecifications']).should.equal(2)
launch_spec = spot_fleet_config['LaunchSpecifications'][0]
launch_spec['EbsOptimized'].should.equal(False)
launch_spec['ImageId'].should.equal("ami-1234")
launch_spec['InstanceType'].should.equal("t2.small")
launch_spec['SubnetId'].should.equal("subnet-123")
launch_spec['SpotPrice'].should.equal("0.13")
launch_spec['WeightedCapacity'].should.equal(2.0)

143
tests/test_ec2/test_spot_fleet.py Normal file
View File

@ -0,0 +1,143 @@
from __future__ import unicode_literals
import boto3
import sure # noqa
from moto import mock_ec2
SPOT_REQUEST_CONFIG = {
'ClientToken': 'string',
'SpotPrice': '0.12',
'TargetCapacity': 6,
'IamFleetRole': 'arn:aws:iam::123456789012:role/fleet',
'LaunchSpecifications': [{
'ImageId': 'ami-123',
'KeyName': 'my-key',
'SecurityGroups': [
{
'GroupId': 'sg-123'
},
],
'UserData': 'some user data',
'InstanceType': 't2.small',
'BlockDeviceMappings': [
{
'VirtualName': 'string',
'DeviceName': 'string',
'Ebs': {
'SnapshotId': 'string',
'VolumeSize': 123,
'DeleteOnTermination': True|False,
'VolumeType': 'standard',
'Iops': 123,
'Encrypted': True|False
},
'NoDevice': 'string'
},
],
'Monitoring': {
'Enabled': True
},
'SubnetId': 'subnet-1234',
'IamInstanceProfile': {
'Arn': 'arn:aws:iam::123456789012:role/fleet'
},
'EbsOptimized': False,
'WeightedCapacity': 2.0,
'SpotPrice': '0.13'
}, {
'ImageId': 'ami-123',
'KeyName': 'my-key',
'SecurityGroups': [
{
'GroupId': 'sg-123'
},
],
'UserData': 'some user data',
'InstanceType': 't2.large',
'Monitoring': {
'Enabled': True
},
'SubnetId': 'subnet-1234',
'IamInstanceProfile': {
'Arn': 'arn:aws:iam::123456789012:role/fleet'
},
'EbsOptimized': False,
'WeightedCapacity': 4.0,
'SpotPrice': '10.00',
}],
'AllocationStrategy': 'lowestPrice',
'FulfilledCapacity': 6,
}
@mock_ec2
def test_create_spot_fleet_with_lowest_price():
conn = boto3.client("ec2", region_name='us-west-2')
spot_fleet_res = conn.request_spot_fleet(
SpotFleetRequestConfig=SPOT_REQUEST_CONFIG
)
spot_fleet_id = spot_fleet_res['SpotFleetRequestId']
spot_fleet_requests = conn.describe_spot_fleet_requests(SpotFleetRequestIds=[spot_fleet_id])['SpotFleetRequestConfigs']
len(spot_fleet_requests).should.equal(1)
spot_fleet_request = spot_fleet_requests[0]
spot_fleet_request['SpotFleetRequestState'].should.equal("active")
spot_fleet_config = spot_fleet_request['SpotFleetRequestConfig']
spot_fleet_config['SpotPrice'].should.equal('0.12')
spot_fleet_config['TargetCapacity'].should.equal(6)
spot_fleet_config['IamFleetRole'].should.equal('arn:aws:iam::123456789012:role/fleet')
spot_fleet_config['AllocationStrategy'].should.equal('lowestPrice')
spot_fleet_config['FulfilledCapacity'].should.equal(6.0)
len(spot_fleet_config['LaunchSpecifications']).should.equal(2)
launch_spec = spot_fleet_config['LaunchSpecifications'][0]
launch_spec['EbsOptimized'].should.equal(False)
launch_spec['SecurityGroups'].should.equal([{"GroupId": "sg-123"}])
launch_spec['IamInstanceProfile'].should.equal({"Arn": "arn:aws:iam::123456789012:role/fleet"})
launch_spec['ImageId'].should.equal("ami-123")
launch_spec['InstanceType'].should.equal("t2.small")
launch_spec['KeyName'].should.equal("my-key")
launch_spec['Monitoring'].should.equal({"Enabled": True})
launch_spec['SpotPrice'].should.equal("0.13")
launch_spec['SubnetId'].should.equal("subnet-1234")
launch_spec['UserData'].should.equal("some user data")
launch_spec['WeightedCapacity'].should.equal(2.0)
instance_res = conn.describe_spot_fleet_instances(SpotFleetRequestId=spot_fleet_id)
instances = instance_res['ActiveInstances']
len(instances).should.equal(3)
@mock_ec2
def test_create_diversified_spot_fleet():
conn = boto3.client("ec2", region_name='us-west-2')
diversified_config = SPOT_REQUEST_CONFIG.copy()
diversified_config['AllocationStrategy'] = 'diversified'
spot_fleet_res = conn.request_spot_fleet(
SpotFleetRequestConfig=diversified_config
)
spot_fleet_id = spot_fleet_res['SpotFleetRequestId']
instance_res = conn.describe_spot_fleet_instances(SpotFleetRequestId=spot_fleet_id)
instances = instance_res['ActiveInstances']
len(instances).should.equal(2)
@mock_ec2
def test_cancel_spot_fleet_request():
conn = boto3.client("ec2", region_name='us-west-2')
spot_fleet_res = conn.request_spot_fleet(
SpotFleetRequestConfig=SPOT_REQUEST_CONFIG,
)
spot_fleet_id = spot_fleet_res['SpotFleetRequestId']
conn.cancel_spot_fleet_requests(SpotFleetRequestIds=[spot_fleet_id], TerminateInstances=True)
spot_fleet_requests = conn.describe_spot_fleet_requests(SpotFleetRequestIds=[spot_fleet_id])['SpotFleetRequestConfigs']
len(spot_fleet_requests).should.equal(0)