Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

EC2 SpotFleetRequests - support LaunchTemplates (#5084)

Browse Source
This commit is contained in:
Bert Blommers 2022-05-01 13:52:38 +00:00 committed by GitHub
parent 12421068bd
commit 6b70cd1b6b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 232 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
moto/core/responses.py
View File

@ -725,25 +725,19 @@ class BaseResponse(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
return results
def _parse_tag_specification(self, param_prefix):
tags = self._get_list_prefix(param_prefix)
def _parse_tag_specification(self):
# [{"ResourceType": _type, "Tag": [{"Key": k, "Value": v}, ..]}]
tag_spec = self._get_multi_param("TagSpecification")
# {_type: {k: v, ..}}
tags = {}
for spec in tag_spec:
if spec["ResourceType"] not in tags:
tags[spec["ResourceType"]] = {}
tags[spec["ResourceType"]].update(
{tag["Key"]: tag["Value"] for tag in spec["Tag"]}
)
results = defaultdict(dict)
for tag in tags:
resource_type = tag.pop("resource_type")
param_index = 1
while True:
key_name = "tag.{0}._key".format(param_index)
value_name = "tag.{0}._value".format(param_index)
try:
results[resource_type][tag[key_name]] = tag[value_name]
except KeyError:
break
param_index += 1
return results
return tags
def _get_object_map(self, prefix, name="Name", value="Value"):
"""

116
moto/ec2/_models/spot_requests.py
View File

@ -1,7 +1,6 @@
from collections import defaultdict
from moto.core.models import Model, CloudFormationModel
from moto.core.utils import camelcase_to_underscores
from moto.packages.boto.ec2.launchspecification import LaunchSpecification
from moto.packages.boto.ec2.spotinstancerequest import (
SpotInstanceRequest as BotoSpotRequest,
@ -215,6 +214,7 @@ class SpotFleetRequest(TaggedEC2Resource, CloudFormationModel):
iam_fleet_role,
allocation_strategy,
launch_specs,
launch_template_config,
):
self.ec2_backend = ec2_backend
@ -227,29 +227,62 @@ class SpotFleetRequest(TaggedEC2Resource, CloudFormationModel):
self.fulfilled_capacity = 0.0
self.launch_specs = []
for spec in launch_specs:
launch_specs_from_config = []
for config in launch_template_config or []:
spec = config["LaunchTemplateSpecification"]
if "LaunchTemplateId" in spec:
launch_template = self.ec2_backend.get_launch_template(
template_id=spec["LaunchTemplateId"]
)
elif "LaunchTemplateName" in spec:
launch_template = self.ec2_backend.get_launch_template_by_name(
name=spec["LaunchTemplateName"]
)
else:
continue
launch_template_data = launch_template.latest_version().data
new_launch_template = launch_template_data.copy()
if config.get("Overrides"):
overrides = list(config["Overrides"].values())[0]
new_launch_template.update(overrides)
launch_specs_from_config.append(new_launch_template)
for spec in (launch_specs or []) + launch_specs_from_config:
tags = self._extract_tags(spec)
self.launch_specs.append(
SpotFleetLaunchSpec(
ebs_optimized=spec["ebs_optimized"],
group_set=[
val for key, val in spec.items() if key.startswith("group_set")
],
iam_instance_profile=spec.get("iam_instance_profile._arn"),
image_id=spec["image_id"],
instance_type=spec["instance_type"],
key_name=spec.get("key_name"),
monitoring=spec.get("monitoring._enabled"),
spot_price=spec.get("spot_price", self.spot_price),
subnet_id=spec["subnet_id"],
tag_specifications=self._parse_tag_specifications(spec),
user_data=spec.get("user_data"),
weighted_capacity=spec["weighted_capacity"],
ebs_optimized=spec.get("EbsOptimized"),
group_set=spec.get("GroupSet", []),
iam_instance_profile=spec.get("IamInstanceProfile"),
image_id=spec["ImageId"],
instance_type=spec["InstanceType"],
key_name=spec.get("KeyName"),
monitoring=spec.get("Monitoring"),
spot_price=spec.get("SpotPrice", self.spot_price),
subnet_id=spec.get("SubnetId"),
tag_specifications=tags,
user_data=spec.get("UserData"),
weighted_capacity=spec.get("WeightedCapacity", 1),
)
)
self.spot_requests = []
self.create_spot_requests(self.target_capacity)
def _extract_tags(self, spec):
# IN: [{"ResourceType": _type, "Tag": [{"Key": k, "Value": v}, ..]}]
# OUT: {_type: {k: v, ..}}
tag_spec_set = spec.get("TagSpecificationSet", [])
tags = {}
for tag_spec in tag_spec_set:
if tag_spec["ResourceType"] not in tags:
tags[tag_spec["ResourceType"]] = {}
tags[tag_spec["ResourceType"]].update(
{tag["Key"]: tag["Value"] for tag in tag_spec["Tag"]}
)
return tags
@property
def physical_resource_id(self):
return self.id
@ -277,15 +310,6 @@ class SpotFleetRequest(TaggedEC2Resource, CloudFormationModel):
iam_fleet_role = properties["IamFleetRole"]
allocation_strategy = properties["AllocationStrategy"]
launch_specs = properties["LaunchSpecifications"]
launch_specs = [
dict(
[
(camelcase_to_underscores(key), val)
for key, val in launch_spec.items()
]
)
for launch_spec in launch_specs
]
spot_fleet_request = ec2_backend.request_spot_fleet(
spot_price,
@ -377,46 +401,6 @@ class SpotFleetRequest(TaggedEC2Resource, CloudFormationModel):
]
self.ec2_backend.terminate_instances(instance_ids)
def _parse_tag_specifications(self, spec):
try:
tag_spec_num = max(
[
int(key.split(".")[1])
for key in spec
if key.startswith("tag_specification_set")
]
)
except ValueError: # no tag specifications
return {}
tag_specifications = {}
for si in range(1, tag_spec_num + 1):
resource_type = spec[
"tag_specification_set.{si}._resource_type".format(si=si)
]
tags = [
key
for key in spec
if key.startswith("tag_specification_set.{si}._tag".format(si=si))
]
tag_num = max([int(key.split(".")[3]) for key in tags])
tag_specifications[resource_type] = dict(
(
spec[
"tag_specification_set.{si}._tag.{ti}._key".format(si=si, ti=ti)
],
spec[
"tag_specification_set.{si}._tag.{ti}._value".format(
si=si, ti=ti
)
],
)
for ti in range(1, tag_num + 1)
)
return tag_specifications
class SpotFleetBackend(object):
def __init__(self):
@ -430,6 +414,7 @@ class SpotFleetBackend(object):
iam_fleet_role,
allocation_strategy,
launch_specs,
launch_template_config=None,
):
spot_fleet_request_id = random_spot_fleet_request_id()
@ -441,6 +426,7 @@ class SpotFleetBackend(object):
iam_fleet_role,
allocation_strategy,
launch_specs,
launch_template_config,
)
self.spot_fleet_requests[spot_fleet_request_id] = request
return request

8
moto/ec2/responses/elastic_block_store.py
View File

@ -17,7 +17,7 @@ class ElasticBlockStore(EC2BaseResponse):
source_snapshot_id = self._get_param("SourceSnapshotId")
source_region = self._get_param("SourceRegion")
description = self._get_param("Description")
tags = self._parse_tag_specification("TagSpecification")
tags = self._parse_tag_specification()
snapshot_tags = tags.get("snapshot", {})
if self.is_not_dryrun("CopySnapshot"):
snapshot = self.ec2_backend.copy_snapshot(
@ -30,7 +30,7 @@ class ElasticBlockStore(EC2BaseResponse):
def create_snapshot(self):
volume_id = self._get_param("VolumeId")
description = self._get_param("Description")
tags = self._parse_tag_specification("TagSpecification")
tags = self._parse_tag_specification()
snapshot_tags = tags.get("snapshot", {})
if self.is_not_dryrun("CreateSnapshot"):
snapshot = self.ec2_backend.create_snapshot(volume_id, description)
@ -42,7 +42,7 @@ class ElasticBlockStore(EC2BaseResponse):
params = self._get_params()
instance_spec = params.get("InstanceSpecification")
description = params.get("Description", "")
tags = self._parse_tag_specification("TagSpecification")
tags = self._parse_tag_specification()
snapshot_tags = tags.get("snapshot", {})
if self.is_not_dryrun("CreateSnapshots"):
@ -57,7 +57,7 @@ class ElasticBlockStore(EC2BaseResponse):
zone = self._get_param("AvailabilityZone")
snapshot_id = self._get_param("SnapshotId")
volume_type = self._get_param("VolumeType")
tags = self._parse_tag_specification("TagSpecification")
tags = self._parse_tag_specification()
volume_tags = tags.get("volume", {})
encrypted = self._get_bool_param("Encrypted", if_none=False)
kms_key_id = self._get_param("KmsKeyId")

2
moto/ec2/responses/flow_logs.py
View File

@ -15,7 +15,7 @@ class FlowLogs(EC2BaseResponse):
max_aggregation_interval = self._get_param("MaxAggregationInterval")
validate_resource_ids(resource_ids)
tags = self._parse_tag_specification("TagSpecification")
tags = self._parse_tag_specification()
tags = tags.get("vpc-flow-log", {})
if self.is_not_dryrun("CreateFlowLogs"):
flow_logs, errors = self.ec2_backend.create_flow_logs(

2
moto/ec2/responses/instances.py
View File

@ -58,7 +58,7 @@ class InstanceResponse(EC2BaseResponse):
"nics": self._get_multi_param("NetworkInterface."),
"private_ip": self._get_param("PrivateIpAddress"),
"associate_public_ip": self._get_param("AssociatePublicIpAddress"),
"tags": self._parse_tag_specification("TagSpecification"),
"tags": self._parse_tag_specification(),
"ebs_optimized": self._get_param("EbsOptimized") or False,
"instance_market_options": self._get_param(
"InstanceMarketOptions.MarketType"

2
moto/ec2/responses/launch_templates.py
View File

@ -94,7 +94,7 @@ class LaunchTemplates(EC2BaseResponse):
def create_launch_template(self):
name = self._get_param("LaunchTemplateName")
version_description = self._get_param("VersionDescription")
tag_spec = self._parse_tag_specification("TagSpecification")
tag_spec = self._parse_tag_specification()
raw_template_data = self._get_dict_param("LaunchTemplateData.")
parsed_template_data = parse_object(raw_template_data)

19
moto/ec2/responses/spot_fleets.py
View File

@ -42,14 +42,18 @@ class SpotFleets(BaseResponse):
return template.render(successful=successful)
def request_spot_fleet(self):
spot_config = self._get_dict_param("SpotFleetRequestConfig.")
spot_price = spot_config.get("spot_price")
target_capacity = spot_config["target_capacity"]
iam_fleet_role = spot_config["iam_fleet_role"]
allocation_strategy = spot_config["allocation_strategy"]
spot_config = self._get_multi_param_dict("SpotFleetRequestConfig")
spot_price = spot_config.get("SpotPrice")
target_capacity = spot_config["TargetCapacity"]
iam_fleet_role = spot_config["IamFleetRole"]
allocation_strategy = spot_config["AllocationStrategy"]
launch_specs = self._get_list_prefix(
"SpotFleetRequestConfig.LaunchSpecifications"
launch_specs = spot_config.get("LaunchSpecifications")
launch_template_config = list(
self._get_params()
.get("SpotFleetRequestConfig", {})
.get("LaunchTemplateConfigs", {})
.values()
)
request = self.ec2_backend.request_spot_fleet(
@ -58,6 +62,7 @@ class SpotFleets(BaseResponse):
iam_fleet_role=iam_fleet_role,
allocation_strategy=allocation_strategy,
launch_specs=launch_specs,
launch_template_config=launch_template_config,
)
template = self.response_template(REQUEST_SPOT_FLEET_TEMPLATE)

150
tests/test_ec2/test_spot_fleet.py
View File

@ -1,9 +1,11 @@
import boto3
import sure # noqa # pylint: disable=unused-import
import pytest
from moto import mock_ec2
from moto.core import ACCOUNT_ID
from tests import EXAMPLE_AMI_ID
from uuid import uuid4
def get_subnet_id(conn):
@ -138,6 +140,154 @@ def test_create_diversified_spot_fleet():
instances[0]["InstanceId"].should.contain("i-")
@mock_ec2
@pytest.mark.parametrize("allocation_strategy", ["diversified", "lowestCost"])
def test_request_spot_fleet_using_launch_template_config__name(allocation_strategy):
conn = boto3.client("ec2", region_name="us-east-2")
template_data = {
"ImageId": "ami-04d4e25790238c5f4",
"InstanceType": "t2.medium",
"DisableApiTermination": False,
"TagSpecifications": [
{"ResourceType": "instance", "Tags": [{"Key": "test", "Value": "value"}]}
],
"SecurityGroupIds": ["sg-abcd1234"],
}
template_name = str(uuid4())
conn.create_launch_template(
LaunchTemplateName=template_name, LaunchTemplateData=template_data
)
template_config = {
"ClientToken": "string",
"SpotPrice": "0.01",
"TargetCapacity": 1,
"IamFleetRole": "arn:aws:iam::486285699788:role/aws-ec2-spot-fleet-tagging-role",
"LaunchTemplateConfigs": [
{
"LaunchTemplateSpecification": {
"LaunchTemplateName": template_name,
"Version": "$Latest",
}
}
],
"AllocationStrategy": allocation_strategy,
}
spot_fleet_res = conn.request_spot_fleet(SpotFleetRequestConfig=template_config)
spot_fleet_id = spot_fleet_res["SpotFleetRequestId"]
instance_res = conn.describe_spot_fleet_instances(SpotFleetRequestId=spot_fleet_id)
instances = instance_res["ActiveInstances"]
len(instances).should.equal(1)
instance_types = set([instance["InstanceType"] for instance in instances])
instance_types.should.equal(set(["t2.medium"]))
instances[0]["InstanceId"].should.contain("i-")
@mock_ec2
def test_request_spot_fleet_using_launch_template_config__id():
conn = boto3.client("ec2", region_name="us-east-2")
template_data = {
"ImageId": "ami-04d4e25790238c5f4",
"InstanceType": "t2.medium",
"DisableApiTermination": False,
"TagSpecifications": [
{"ResourceType": "instance", "Tags": [{"Key": "test", "Value": "value"}]}
],
"SecurityGroupIds": ["sg-abcd1234"],
}
template_name = str(uuid4())
template = conn.create_launch_template(
LaunchTemplateName=template_name, LaunchTemplateData=template_data
)["LaunchTemplate"]
template_id = template["LaunchTemplateId"]
template_config = {
"ClientToken": "string",
"SpotPrice": "0.01",
"TargetCapacity": 1,
"IamFleetRole": "arn:aws:iam::486285699788:role/aws-ec2-spot-fleet-tagging-role",
"LaunchTemplateConfigs": [
{"LaunchTemplateSpecification": {"LaunchTemplateId": template_id}}
],
"AllocationStrategy": "lowestCost",
}
spot_fleet_res = conn.request_spot_fleet(SpotFleetRequestConfig=template_config)
spot_fleet_id = spot_fleet_res["SpotFleetRequestId"]
instance_res = conn.describe_spot_fleet_instances(SpotFleetRequestId=spot_fleet_id)
instances = instance_res["ActiveInstances"]
len(instances).should.equal(1)
instance_types = set([instance["InstanceType"] for instance in instances])
instance_types.should.equal(set(["t2.medium"]))
instances[0]["InstanceId"].should.contain("i-")
@mock_ec2
def test_request_spot_fleet_using_launch_template_config__overrides():
conn = boto3.client("ec2", region_name="us-east-2")
subnet_id = get_subnet_id(conn)
template_data = {
"ImageId": "ami-04d4e25790238c5f4",
"InstanceType": "t2.medium",
"DisableApiTermination": False,
"TagSpecifications": [
{"ResourceType": "instance", "Tags": [{"Key": "test", "Value": "value"}]}
],
"SecurityGroupIds": ["sg-abcd1234"],
}
template_name = str(uuid4())
template = conn.create_launch_template(
LaunchTemplateName=template_name, LaunchTemplateData=template_data
)["LaunchTemplate"]
template_id = template["LaunchTemplateId"]
template_config = {
"ClientToken": "string",
"SpotPrice": "0.01",
"TargetCapacity": 1,
"IamFleetRole": "arn:aws:iam::486285699788:role/aws-ec2-spot-fleet-tagging-role",
"LaunchTemplateConfigs": [
{
"LaunchTemplateSpecification": {"LaunchTemplateId": template_id},
"Overrides": [
{
"InstanceType": "t2.nano",
"SubnetId": subnet_id,
"AvailabilityZone": "us-west-1",
"WeightedCapacity": 2,
}
],
}
],
"AllocationStrategy": "lowestCost",
}
spot_fleet_res = conn.request_spot_fleet(SpotFleetRequestConfig=template_config)
spot_fleet_id = spot_fleet_res["SpotFleetRequestId"]
instance_res = conn.describe_spot_fleet_instances(SpotFleetRequestId=spot_fleet_id)
instances = instance_res["ActiveInstances"]
instances.should.have.length_of(1)
instances[0].should.have.key("InstanceType").equals("t2.nano")
instance = conn.describe_instances(
InstanceIds=[i["InstanceId"] for i in instances]
)["Reservations"][0]["Instances"][0]
instance.should.have.key("SubnetId").equals(subnet_id)
@mock_ec2
def test_create_spot_fleet_request_with_tag_spec():
conn = boto3.client("ec2", region_name="us-west-2")