From 6b70cd1b6b1cf493b66b6fcaaea9d1041331e836 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Sun, 1 May 2022 13:52:38 +0000 Subject: [PATCH] EC2 SpotFleetRequests - support LaunchTemplates (#5084) --- moto/core/responses.py | 30 ++--- moto/ec2/_models/spot_requests.py | 116 ++++++++--------- moto/ec2/responses/elastic_block_store.py | 8 +- moto/ec2/responses/flow_logs.py | 2 +- moto/ec2/responses/instances.py | 2 +- moto/ec2/responses/launch_templates.py | 2 +- moto/ec2/responses/spot_fleets.py | 19 ++- tests/test_ec2/test_spot_fleet.py | 150 ++++++++++++++++++++++ 8 files changed, 232 insertions(+), 97 deletions(-) diff --git a/moto/core/responses.py b/moto/core/responses.py index 69c4ab8f3..2555ba2cd 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -725,25 +725,19 @@ class BaseResponse(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): return results - def _parse_tag_specification(self, param_prefix): - tags = self._get_list_prefix(param_prefix) + def _parse_tag_specification(self): + # [{"ResourceType": _type, "Tag": [{"Key": k, "Value": v}, ..]}] + tag_spec = self._get_multi_param("TagSpecification") + # {_type: {k: v, ..}} + tags = {} + for spec in tag_spec: + if spec["ResourceType"] not in tags: + tags[spec["ResourceType"]] = {} + tags[spec["ResourceType"]].update( + {tag["Key"]: tag["Value"] for tag in spec["Tag"]} + ) - results = defaultdict(dict) - for tag in tags: - resource_type = tag.pop("resource_type") - - param_index = 1 - while True: - key_name = "tag.{0}._key".format(param_index) - value_name = "tag.{0}._value".format(param_index) - - try: - results[resource_type][tag[key_name]] = tag[value_name] - except KeyError: - break - param_index += 1 - - return results + return tags def _get_object_map(self, prefix, name="Name", value="Value"): """ diff --git a/moto/ec2/_models/spot_requests.py b/moto/ec2/_models/spot_requests.py index 853fb74c8..5688aeca7 100644 --- a/moto/ec2/_models/spot_requests.py +++ b/moto/ec2/_models/spot_requests.py @@ -1,7 +1,6 @@ from collections import defaultdict from moto.core.models import Model, CloudFormationModel -from moto.core.utils import camelcase_to_underscores from moto.packages.boto.ec2.launchspecification import LaunchSpecification from moto.packages.boto.ec2.spotinstancerequest import ( SpotInstanceRequest as BotoSpotRequest, @@ -215,6 +214,7 @@ class SpotFleetRequest(TaggedEC2Resource, CloudFormationModel): iam_fleet_role, allocation_strategy, launch_specs, + launch_template_config, ): self.ec2_backend = ec2_backend @@ -227,29 +227,62 @@ class SpotFleetRequest(TaggedEC2Resource, CloudFormationModel): self.fulfilled_capacity = 0.0 self.launch_specs = [] - for spec in launch_specs: + + launch_specs_from_config = [] + for config in launch_template_config or []: + spec = config["LaunchTemplateSpecification"] + if "LaunchTemplateId" in spec: + launch_template = self.ec2_backend.get_launch_template( + template_id=spec["LaunchTemplateId"] + ) + elif "LaunchTemplateName" in spec: + launch_template = self.ec2_backend.get_launch_template_by_name( + name=spec["LaunchTemplateName"] + ) + else: + continue + launch_template_data = launch_template.latest_version().data + new_launch_template = launch_template_data.copy() + if config.get("Overrides"): + overrides = list(config["Overrides"].values())[0] + new_launch_template.update(overrides) + launch_specs_from_config.append(new_launch_template) + + for spec in (launch_specs or []) + launch_specs_from_config: + tags = self._extract_tags(spec) self.launch_specs.append( SpotFleetLaunchSpec( - ebs_optimized=spec["ebs_optimized"], - group_set=[ - val for key, val in spec.items() if key.startswith("group_set") - ], - iam_instance_profile=spec.get("iam_instance_profile._arn"), - image_id=spec["image_id"], - instance_type=spec["instance_type"], - key_name=spec.get("key_name"), - monitoring=spec.get("monitoring._enabled"), - spot_price=spec.get("spot_price", self.spot_price), - subnet_id=spec["subnet_id"], - tag_specifications=self._parse_tag_specifications(spec), - user_data=spec.get("user_data"), - weighted_capacity=spec["weighted_capacity"], + ebs_optimized=spec.get("EbsOptimized"), + group_set=spec.get("GroupSet", []), + iam_instance_profile=spec.get("IamInstanceProfile"), + image_id=spec["ImageId"], + instance_type=spec["InstanceType"], + key_name=spec.get("KeyName"), + monitoring=spec.get("Monitoring"), + spot_price=spec.get("SpotPrice", self.spot_price), + subnet_id=spec.get("SubnetId"), + tag_specifications=tags, + user_data=spec.get("UserData"), + weighted_capacity=spec.get("WeightedCapacity", 1), ) ) self.spot_requests = [] self.create_spot_requests(self.target_capacity) + def _extract_tags(self, spec): + # IN: [{"ResourceType": _type, "Tag": [{"Key": k, "Value": v}, ..]}] + # OUT: {_type: {k: v, ..}} + tag_spec_set = spec.get("TagSpecificationSet", []) + tags = {} + for tag_spec in tag_spec_set: + if tag_spec["ResourceType"] not in tags: + tags[tag_spec["ResourceType"]] = {} + tags[tag_spec["ResourceType"]].update( + {tag["Key"]: tag["Value"] for tag in tag_spec["Tag"]} + ) + return tags + @property def physical_resource_id(self): return self.id @@ -277,15 +310,6 @@ class SpotFleetRequest(TaggedEC2Resource, CloudFormationModel): iam_fleet_role = properties["IamFleetRole"] allocation_strategy = properties["AllocationStrategy"] launch_specs = properties["LaunchSpecifications"] - launch_specs = [ - dict( - [ - (camelcase_to_underscores(key), val) - for key, val in launch_spec.items() - ] - ) - for launch_spec in launch_specs - ] spot_fleet_request = ec2_backend.request_spot_fleet( spot_price, @@ -377,46 +401,6 @@ class SpotFleetRequest(TaggedEC2Resource, CloudFormationModel): ] self.ec2_backend.terminate_instances(instance_ids) - def _parse_tag_specifications(self, spec): - try: - tag_spec_num = max( - [ - int(key.split(".")[1]) - for key in spec - if key.startswith("tag_specification_set") - ] - ) - except ValueError: # no tag specifications - return {} - - tag_specifications = {} - for si in range(1, tag_spec_num + 1): - resource_type = spec[ - "tag_specification_set.{si}._resource_type".format(si=si) - ] - - tags = [ - key - for key in spec - if key.startswith("tag_specification_set.{si}._tag".format(si=si)) - ] - tag_num = max([int(key.split(".")[3]) for key in tags]) - tag_specifications[resource_type] = dict( - ( - spec[ - "tag_specification_set.{si}._tag.{ti}._key".format(si=si, ti=ti) - ], - spec[ - "tag_specification_set.{si}._tag.{ti}._value".format( - si=si, ti=ti - ) - ], - ) - for ti in range(1, tag_num + 1) - ) - - return tag_specifications - class SpotFleetBackend(object): def __init__(self): @@ -430,6 +414,7 @@ class SpotFleetBackend(object): iam_fleet_role, allocation_strategy, launch_specs, + launch_template_config=None, ): spot_fleet_request_id = random_spot_fleet_request_id() @@ -441,6 +426,7 @@ class SpotFleetBackend(object): iam_fleet_role, allocation_strategy, launch_specs, + launch_template_config, ) self.spot_fleet_requests[spot_fleet_request_id] = request return request diff --git a/moto/ec2/responses/elastic_block_store.py b/moto/ec2/responses/elastic_block_store.py index e30202708..5a88a2a85 100644 --- a/moto/ec2/responses/elastic_block_store.py +++ b/moto/ec2/responses/elastic_block_store.py @@ -17,7 +17,7 @@ class ElasticBlockStore(EC2BaseResponse): source_snapshot_id = self._get_param("SourceSnapshotId") source_region = self._get_param("SourceRegion") description = self._get_param("Description") - tags = self._parse_tag_specification("TagSpecification") + tags = self._parse_tag_specification() snapshot_tags = tags.get("snapshot", {}) if self.is_not_dryrun("CopySnapshot"): snapshot = self.ec2_backend.copy_snapshot( @@ -30,7 +30,7 @@ class ElasticBlockStore(EC2BaseResponse): def create_snapshot(self): volume_id = self._get_param("VolumeId") description = self._get_param("Description") - tags = self._parse_tag_specification("TagSpecification") + tags = self._parse_tag_specification() snapshot_tags = tags.get("snapshot", {}) if self.is_not_dryrun("CreateSnapshot"): snapshot = self.ec2_backend.create_snapshot(volume_id, description) @@ -42,7 +42,7 @@ class ElasticBlockStore(EC2BaseResponse): params = self._get_params() instance_spec = params.get("InstanceSpecification") description = params.get("Description", "") - tags = self._parse_tag_specification("TagSpecification") + tags = self._parse_tag_specification() snapshot_tags = tags.get("snapshot", {}) if self.is_not_dryrun("CreateSnapshots"): @@ -57,7 +57,7 @@ class ElasticBlockStore(EC2BaseResponse): zone = self._get_param("AvailabilityZone") snapshot_id = self._get_param("SnapshotId") volume_type = self._get_param("VolumeType") - tags = self._parse_tag_specification("TagSpecification") + tags = self._parse_tag_specification() volume_tags = tags.get("volume", {}) encrypted = self._get_bool_param("Encrypted", if_none=False) kms_key_id = self._get_param("KmsKeyId") diff --git a/moto/ec2/responses/flow_logs.py b/moto/ec2/responses/flow_logs.py index 45aeed789..58445ac76 100644 --- a/moto/ec2/responses/flow_logs.py +++ b/moto/ec2/responses/flow_logs.py @@ -15,7 +15,7 @@ class FlowLogs(EC2BaseResponse): max_aggregation_interval = self._get_param("MaxAggregationInterval") validate_resource_ids(resource_ids) - tags = self._parse_tag_specification("TagSpecification") + tags = self._parse_tag_specification() tags = tags.get("vpc-flow-log", {}) if self.is_not_dryrun("CreateFlowLogs"): flow_logs, errors = self.ec2_backend.create_flow_logs( diff --git a/moto/ec2/responses/instances.py b/moto/ec2/responses/instances.py index d62e03aa2..e63754d9c 100644 --- a/moto/ec2/responses/instances.py +++ b/moto/ec2/responses/instances.py @@ -58,7 +58,7 @@ class InstanceResponse(EC2BaseResponse): "nics": self._get_multi_param("NetworkInterface."), "private_ip": self._get_param("PrivateIpAddress"), "associate_public_ip": self._get_param("AssociatePublicIpAddress"), - "tags": self._parse_tag_specification("TagSpecification"), + "tags": self._parse_tag_specification(), "ebs_optimized": self._get_param("EbsOptimized") or False, "instance_market_options": self._get_param( "InstanceMarketOptions.MarketType" diff --git a/moto/ec2/responses/launch_templates.py b/moto/ec2/responses/launch_templates.py index ce4afa5c8..0029c0d62 100644 --- a/moto/ec2/responses/launch_templates.py +++ b/moto/ec2/responses/launch_templates.py @@ -94,7 +94,7 @@ class LaunchTemplates(EC2BaseResponse): def create_launch_template(self): name = self._get_param("LaunchTemplateName") version_description = self._get_param("VersionDescription") - tag_spec = self._parse_tag_specification("TagSpecification") + tag_spec = self._parse_tag_specification() raw_template_data = self._get_dict_param("LaunchTemplateData.") parsed_template_data = parse_object(raw_template_data) diff --git a/moto/ec2/responses/spot_fleets.py b/moto/ec2/responses/spot_fleets.py index 8b0b3e20f..c38c08dee 100644 --- a/moto/ec2/responses/spot_fleets.py +++ b/moto/ec2/responses/spot_fleets.py @@ -42,14 +42,18 @@ class SpotFleets(BaseResponse): return template.render(successful=successful) def request_spot_fleet(self): - spot_config = self._get_dict_param("SpotFleetRequestConfig.") - spot_price = spot_config.get("spot_price") - target_capacity = spot_config["target_capacity"] - iam_fleet_role = spot_config["iam_fleet_role"] - allocation_strategy = spot_config["allocation_strategy"] + spot_config = self._get_multi_param_dict("SpotFleetRequestConfig") + spot_price = spot_config.get("SpotPrice") + target_capacity = spot_config["TargetCapacity"] + iam_fleet_role = spot_config["IamFleetRole"] + allocation_strategy = spot_config["AllocationStrategy"] - launch_specs = self._get_list_prefix( - "SpotFleetRequestConfig.LaunchSpecifications" + launch_specs = spot_config.get("LaunchSpecifications") + launch_template_config = list( + self._get_params() + .get("SpotFleetRequestConfig", {}) + .get("LaunchTemplateConfigs", {}) + .values() ) request = self.ec2_backend.request_spot_fleet( @@ -58,6 +62,7 @@ class SpotFleets(BaseResponse): iam_fleet_role=iam_fleet_role, allocation_strategy=allocation_strategy, launch_specs=launch_specs, + launch_template_config=launch_template_config, ) template = self.response_template(REQUEST_SPOT_FLEET_TEMPLATE) diff --git a/tests/test_ec2/test_spot_fleet.py b/tests/test_ec2/test_spot_fleet.py index 4ba9c91bb..379198b19 100644 --- a/tests/test_ec2/test_spot_fleet.py +++ b/tests/test_ec2/test_spot_fleet.py @@ -1,9 +1,11 @@ import boto3 import sure # noqa # pylint: disable=unused-import +import pytest from moto import mock_ec2 from moto.core import ACCOUNT_ID from tests import EXAMPLE_AMI_ID +from uuid import uuid4 def get_subnet_id(conn): @@ -138,6 +140,154 @@ def test_create_diversified_spot_fleet(): instances[0]["InstanceId"].should.contain("i-") +@mock_ec2 +@pytest.mark.parametrize("allocation_strategy", ["diversified", "lowestCost"]) +def test_request_spot_fleet_using_launch_template_config__name(allocation_strategy): + + conn = boto3.client("ec2", region_name="us-east-2") + + template_data = { + "ImageId": "ami-04d4e25790238c5f4", + "InstanceType": "t2.medium", + "DisableApiTermination": False, + "TagSpecifications": [ + {"ResourceType": "instance", "Tags": [{"Key": "test", "Value": "value"}]} + ], + "SecurityGroupIds": ["sg-abcd1234"], + } + + template_name = str(uuid4()) + conn.create_launch_template( + LaunchTemplateName=template_name, LaunchTemplateData=template_data + ) + + template_config = { + "ClientToken": "string", + "SpotPrice": "0.01", + "TargetCapacity": 1, + "IamFleetRole": "arn:aws:iam::486285699788:role/aws-ec2-spot-fleet-tagging-role", + "LaunchTemplateConfigs": [ + { + "LaunchTemplateSpecification": { + "LaunchTemplateName": template_name, + "Version": "$Latest", + } + } + ], + "AllocationStrategy": allocation_strategy, + } + + spot_fleet_res = conn.request_spot_fleet(SpotFleetRequestConfig=template_config) + spot_fleet_id = spot_fleet_res["SpotFleetRequestId"] + + instance_res = conn.describe_spot_fleet_instances(SpotFleetRequestId=spot_fleet_id) + instances = instance_res["ActiveInstances"] + len(instances).should.equal(1) + instance_types = set([instance["InstanceType"] for instance in instances]) + instance_types.should.equal(set(["t2.medium"])) + instances[0]["InstanceId"].should.contain("i-") + + +@mock_ec2 +def test_request_spot_fleet_using_launch_template_config__id(): + + conn = boto3.client("ec2", region_name="us-east-2") + + template_data = { + "ImageId": "ami-04d4e25790238c5f4", + "InstanceType": "t2.medium", + "DisableApiTermination": False, + "TagSpecifications": [ + {"ResourceType": "instance", "Tags": [{"Key": "test", "Value": "value"}]} + ], + "SecurityGroupIds": ["sg-abcd1234"], + } + + template_name = str(uuid4()) + template = conn.create_launch_template( + LaunchTemplateName=template_name, LaunchTemplateData=template_data + )["LaunchTemplate"] + template_id = template["LaunchTemplateId"] + + template_config = { + "ClientToken": "string", + "SpotPrice": "0.01", + "TargetCapacity": 1, + "IamFleetRole": "arn:aws:iam::486285699788:role/aws-ec2-spot-fleet-tagging-role", + "LaunchTemplateConfigs": [ + {"LaunchTemplateSpecification": {"LaunchTemplateId": template_id}} + ], + "AllocationStrategy": "lowestCost", + } + + spot_fleet_res = conn.request_spot_fleet(SpotFleetRequestConfig=template_config) + spot_fleet_id = spot_fleet_res["SpotFleetRequestId"] + + instance_res = conn.describe_spot_fleet_instances(SpotFleetRequestId=spot_fleet_id) + instances = instance_res["ActiveInstances"] + len(instances).should.equal(1) + instance_types = set([instance["InstanceType"] for instance in instances]) + instance_types.should.equal(set(["t2.medium"])) + instances[0]["InstanceId"].should.contain("i-") + + +@mock_ec2 +def test_request_spot_fleet_using_launch_template_config__overrides(): + + conn = boto3.client("ec2", region_name="us-east-2") + subnet_id = get_subnet_id(conn) + + template_data = { + "ImageId": "ami-04d4e25790238c5f4", + "InstanceType": "t2.medium", + "DisableApiTermination": False, + "TagSpecifications": [ + {"ResourceType": "instance", "Tags": [{"Key": "test", "Value": "value"}]} + ], + "SecurityGroupIds": ["sg-abcd1234"], + } + + template_name = str(uuid4()) + template = conn.create_launch_template( + LaunchTemplateName=template_name, LaunchTemplateData=template_data + )["LaunchTemplate"] + template_id = template["LaunchTemplateId"] + + template_config = { + "ClientToken": "string", + "SpotPrice": "0.01", + "TargetCapacity": 1, + "IamFleetRole": "arn:aws:iam::486285699788:role/aws-ec2-spot-fleet-tagging-role", + "LaunchTemplateConfigs": [ + { + "LaunchTemplateSpecification": {"LaunchTemplateId": template_id}, + "Overrides": [ + { + "InstanceType": "t2.nano", + "SubnetId": subnet_id, + "AvailabilityZone": "us-west-1", + "WeightedCapacity": 2, + } + ], + } + ], + "AllocationStrategy": "lowestCost", + } + + spot_fleet_res = conn.request_spot_fleet(SpotFleetRequestConfig=template_config) + spot_fleet_id = spot_fleet_res["SpotFleetRequestId"] + + instance_res = conn.describe_spot_fleet_instances(SpotFleetRequestId=spot_fleet_id) + instances = instance_res["ActiveInstances"] + instances.should.have.length_of(1) + instances[0].should.have.key("InstanceType").equals("t2.nano") + + instance = conn.describe_instances( + InstanceIds=[i["InstanceId"] for i in instances] + )["Reservations"][0]["Instances"][0] + instance.should.have.key("SubnetId").equals(subnet_id) + + @mock_ec2 def test_create_spot_fleet_request_with_tag_spec(): conn = boto3.client("ec2", region_name="us-west-2")