EC2 SpotFleetRequests - support LaunchTemplates (#5084)

moto/core/responses.py

@@ -725,25 +725,19 @@ class BaseResponse(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
 
         return results
 
-    def _parse_tag_specification(self, param_prefix):
+    def _parse_tag_specification(self):
-        tags = self._get_list_prefix(param_prefix)
+        # [{"ResourceType": _type, "Tag": [{"Key": k, "Value": v}, ..]}]
+        tag_spec = self._get_multi_param("TagSpecification")
+        # {_type: {k: v, ..}}
+        tags = {}
+        for spec in tag_spec:
+            if spec["ResourceType"] not in tags:
+                tags[spec["ResourceType"]] = {}
+            tags[spec["ResourceType"]].update(
+                {tag["Key"]: tag["Value"] for tag in spec["Tag"]}
+            )
 
-        results = defaultdict(dict)
+        return tags
-        for tag in tags:
-            resource_type = tag.pop("resource_type")
-
-            param_index = 1
-            while True:
-                key_name = "tag.{0}._key".format(param_index)
-                value_name = "tag.{0}._value".format(param_index)
-
-                try:
-                    results[resource_type][tag[key_name]] = tag[value_name]
-                except KeyError:
-                    break
-                param_index += 1
-
-        return results
 
     def _get_object_map(self, prefix, name="Name", value="Value"):
         """

moto/ec2/_models/spot_requests.py

@@ -1,7 +1,6 @@
 from collections import defaultdict
 
 from moto.core.models import Model, CloudFormationModel
-from moto.core.utils import camelcase_to_underscores
 from moto.packages.boto.ec2.launchspecification import LaunchSpecification
 from moto.packages.boto.ec2.spotinstancerequest import (
     SpotInstanceRequest as BotoSpotRequest,
@@ -215,6 +214,7 @@ class SpotFleetRequest(TaggedEC2Resource, CloudFormationModel):
         iam_fleet_role,
         allocation_strategy,
         launch_specs,
+        launch_template_config,
     ):
 
         self.ec2_backend = ec2_backend
@@ -227,29 +227,62 @@ class SpotFleetRequest(TaggedEC2Resource, CloudFormationModel):
         self.fulfilled_capacity = 0.0
 
         self.launch_specs = []
-        for spec in launch_specs:
+
+        launch_specs_from_config = []
+        for config in launch_template_config or []:
+            spec = config["LaunchTemplateSpecification"]
+            if "LaunchTemplateId" in spec:
+                launch_template = self.ec2_backend.get_launch_template(
+                    template_id=spec["LaunchTemplateId"]
+                )
+            elif "LaunchTemplateName" in spec:
+                launch_template = self.ec2_backend.get_launch_template_by_name(
+                    name=spec["LaunchTemplateName"]
+                )
+            else:
+                continue
+            launch_template_data = launch_template.latest_version().data
+            new_launch_template = launch_template_data.copy()
+            if config.get("Overrides"):
+                overrides = list(config["Overrides"].values())[0]
+                new_launch_template.update(overrides)
+            launch_specs_from_config.append(new_launch_template)
+
+        for spec in (launch_specs or []) + launch_specs_from_config:
+            tags = self._extract_tags(spec)
             self.launch_specs.append(
                 SpotFleetLaunchSpec(
-                    ebs_optimized=spec["ebs_optimized"],
+                    ebs_optimized=spec.get("EbsOptimized"),
-                    group_set=[
+                    group_set=spec.get("GroupSet", []),
-                        val for key, val in spec.items() if key.startswith("group_set")
+                    iam_instance_profile=spec.get("IamInstanceProfile"),
-                    ],
+                    image_id=spec["ImageId"],
-                    iam_instance_profile=spec.get("iam_instance_profile._arn"),
+                    instance_type=spec["InstanceType"],
-                    image_id=spec["image_id"],
+                    key_name=spec.get("KeyName"),
-                    instance_type=spec["instance_type"],
+                    monitoring=spec.get("Monitoring"),
-                    key_name=spec.get("key_name"),
+                    spot_price=spec.get("SpotPrice", self.spot_price),
-                    monitoring=spec.get("monitoring._enabled"),
+                    subnet_id=spec.get("SubnetId"),
-                    spot_price=spec.get("spot_price", self.spot_price),
+                    tag_specifications=tags,
-                    subnet_id=spec["subnet_id"],
+                    user_data=spec.get("UserData"),
-                    tag_specifications=self._parse_tag_specifications(spec),
+                    weighted_capacity=spec.get("WeightedCapacity", 1),
-                    user_data=spec.get("user_data"),
-                    weighted_capacity=spec["weighted_capacity"],
                 )
             )
 
         self.spot_requests = []
         self.create_spot_requests(self.target_capacity)
 
+    def _extract_tags(self, spec):
+        # IN:  [{"ResourceType": _type, "Tag": [{"Key": k, "Value": v}, ..]}]
+        # OUT: {_type: {k: v, ..}}
+        tag_spec_set = spec.get("TagSpecificationSet", [])
+        tags = {}
+        for tag_spec in tag_spec_set:
+            if tag_spec["ResourceType"] not in tags:
+                tags[tag_spec["ResourceType"]] = {}
+            tags[tag_spec["ResourceType"]].update(
+                {tag["Key"]: tag["Value"] for tag in tag_spec["Tag"]}
+            )
+        return tags
+
     @property
     def physical_resource_id(self):
         return self.id
@@ -277,15 +310,6 @@ class SpotFleetRequest(TaggedEC2Resource, CloudFormationModel):
         iam_fleet_role = properties["IamFleetRole"]
         allocation_strategy = properties["AllocationStrategy"]
         launch_specs = properties["LaunchSpecifications"]
-        launch_specs = [
-            dict(
-                [
-                    (camelcase_to_underscores(key), val)
-                    for key, val in launch_spec.items()
-                ]
-            )
-            for launch_spec in launch_specs
-        ]
 
         spot_fleet_request = ec2_backend.request_spot_fleet(
             spot_price,
@@ -377,46 +401,6 @@ class SpotFleetRequest(TaggedEC2Resource, CloudFormationModel):
         ]
         self.ec2_backend.terminate_instances(instance_ids)
 
-    def _parse_tag_specifications(self, spec):
-        try:
-            tag_spec_num = max(
-                [
-                    int(key.split(".")[1])
-                    for key in spec
-                    if key.startswith("tag_specification_set")
-                ]
-            )
-        except ValueError:  # no tag specifications
-            return {}
-
-        tag_specifications = {}
-        for si in range(1, tag_spec_num + 1):
-            resource_type = spec[
-                "tag_specification_set.{si}._resource_type".format(si=si)
-            ]
-
-            tags = [
-                key
-                for key in spec
-                if key.startswith("tag_specification_set.{si}._tag".format(si=si))
-            ]
-            tag_num = max([int(key.split(".")[3]) for key in tags])
-            tag_specifications[resource_type] = dict(
-                (
-                    spec[
-                        "tag_specification_set.{si}._tag.{ti}._key".format(si=si, ti=ti)
-                    ],
-                    spec[
-                        "tag_specification_set.{si}._tag.{ti}._value".format(
-                            si=si, ti=ti
-                        )
-                    ],
-                )
-                for ti in range(1, tag_num + 1)
-            )
-
-        return tag_specifications
-
 
 class SpotFleetBackend(object):
     def __init__(self):
@@ -430,6 +414,7 @@ class SpotFleetBackend(object):
         iam_fleet_role,
         allocation_strategy,
         launch_specs,
+        launch_template_config=None,
     ):
 
         spot_fleet_request_id = random_spot_fleet_request_id()
@@ -441,6 +426,7 @@ class SpotFleetBackend(object):
             iam_fleet_role,
             allocation_strategy,
             launch_specs,
+            launch_template_config,
         )
         self.spot_fleet_requests[spot_fleet_request_id] = request
         return request

moto/ec2/responses/elastic_block_store.py

@@ -17,7 +17,7 @@ class ElasticBlockStore(EC2BaseResponse):
         source_snapshot_id = self._get_param("SourceSnapshotId")
         source_region = self._get_param("SourceRegion")
         description = self._get_param("Description")
-        tags = self._parse_tag_specification("TagSpecification")
+        tags = self._parse_tag_specification()
         snapshot_tags = tags.get("snapshot", {})
         if self.is_not_dryrun("CopySnapshot"):
             snapshot = self.ec2_backend.copy_snapshot(
@@ -30,7 +30,7 @@ class ElasticBlockStore(EC2BaseResponse):
     def create_snapshot(self):
         volume_id = self._get_param("VolumeId")
         description = self._get_param("Description")
-        tags = self._parse_tag_specification("TagSpecification")
+        tags = self._parse_tag_specification()
         snapshot_tags = tags.get("snapshot", {})
         if self.is_not_dryrun("CreateSnapshot"):
             snapshot = self.ec2_backend.create_snapshot(volume_id, description)
@@ -42,7 +42,7 @@ class ElasticBlockStore(EC2BaseResponse):
         params = self._get_params()
         instance_spec = params.get("InstanceSpecification")
         description = params.get("Description", "")
-        tags = self._parse_tag_specification("TagSpecification")
+        tags = self._parse_tag_specification()
         snapshot_tags = tags.get("snapshot", {})
 
         if self.is_not_dryrun("CreateSnapshots"):
@@ -57,7 +57,7 @@ class ElasticBlockStore(EC2BaseResponse):
         zone = self._get_param("AvailabilityZone")
         snapshot_id = self._get_param("SnapshotId")
         volume_type = self._get_param("VolumeType")
-        tags = self._parse_tag_specification("TagSpecification")
+        tags = self._parse_tag_specification()
         volume_tags = tags.get("volume", {})
         encrypted = self._get_bool_param("Encrypted", if_none=False)
         kms_key_id = self._get_param("KmsKeyId")

moto/ec2/responses/flow_logs.py

@@ -15,7 +15,7 @@ class FlowLogs(EC2BaseResponse):
         max_aggregation_interval = self._get_param("MaxAggregationInterval")
         validate_resource_ids(resource_ids)
 
-        tags = self._parse_tag_specification("TagSpecification")
+        tags = self._parse_tag_specification()
         tags = tags.get("vpc-flow-log", {})
         if self.is_not_dryrun("CreateFlowLogs"):
             flow_logs, errors = self.ec2_backend.create_flow_logs(

moto/ec2/responses/instances.py

@@ -58,7 +58,7 @@ class InstanceResponse(EC2BaseResponse):
             "nics": self._get_multi_param("NetworkInterface."),
             "private_ip": self._get_param("PrivateIpAddress"),
             "associate_public_ip": self._get_param("AssociatePublicIpAddress"),
-            "tags": self._parse_tag_specification("TagSpecification"),
+            "tags": self._parse_tag_specification(),
             "ebs_optimized": self._get_param("EbsOptimized") or False,
             "instance_market_options": self._get_param(
                 "InstanceMarketOptions.MarketType"

moto/ec2/responses/launch_templates.py

@@ -94,7 +94,7 @@ class LaunchTemplates(EC2BaseResponse):
     def create_launch_template(self):
         name = self._get_param("LaunchTemplateName")
         version_description = self._get_param("VersionDescription")
-        tag_spec = self._parse_tag_specification("TagSpecification")
+        tag_spec = self._parse_tag_specification()
 
         raw_template_data = self._get_dict_param("LaunchTemplateData.")
         parsed_template_data = parse_object(raw_template_data)

moto/ec2/responses/spot_fleets.py

@@ -42,14 +42,18 @@ class SpotFleets(BaseResponse):
         return template.render(successful=successful)
 
     def request_spot_fleet(self):
-        spot_config = self._get_dict_param("SpotFleetRequestConfig.")
+        spot_config = self._get_multi_param_dict("SpotFleetRequestConfig")
-        spot_price = spot_config.get("spot_price")
+        spot_price = spot_config.get("SpotPrice")
-        target_capacity = spot_config["target_capacity"]
+        target_capacity = spot_config["TargetCapacity"]
-        iam_fleet_role = spot_config["iam_fleet_role"]
+        iam_fleet_role = spot_config["IamFleetRole"]
-        allocation_strategy = spot_config["allocation_strategy"]
+        allocation_strategy = spot_config["AllocationStrategy"]
 
-        launch_specs = self._get_list_prefix(
+        launch_specs = spot_config.get("LaunchSpecifications")
-            "SpotFleetRequestConfig.LaunchSpecifications"
+        launch_template_config = list(
+            self._get_params()
+            .get("SpotFleetRequestConfig", {})
+            .get("LaunchTemplateConfigs", {})
+            .values()
         )
 
         request = self.ec2_backend.request_spot_fleet(
@@ -58,6 +62,7 @@ class SpotFleets(BaseResponse):
             iam_fleet_role=iam_fleet_role,
             allocation_strategy=allocation_strategy,
             launch_specs=launch_specs,
+            launch_template_config=launch_template_config,
         )
 
         template = self.response_template(REQUEST_SPOT_FLEET_TEMPLATE)

tests/test_ec2/test_spot_fleet.py

@@ -1,9 +1,11 @@
 import boto3
 import sure  # noqa # pylint: disable=unused-import
+import pytest
 
 from moto import mock_ec2
 from moto.core import ACCOUNT_ID
 from tests import EXAMPLE_AMI_ID
+from uuid import uuid4
 
 
 def get_subnet_id(conn):
@@ -138,6 +140,154 @@ def test_create_diversified_spot_fleet():
     instances[0]["InstanceId"].should.contain("i-")
 
 
+@mock_ec2
+@pytest.mark.parametrize("allocation_strategy", ["diversified", "lowestCost"])
+def test_request_spot_fleet_using_launch_template_config__name(allocation_strategy):
+
+    conn = boto3.client("ec2", region_name="us-east-2")
+
+    template_data = {
+        "ImageId": "ami-04d4e25790238c5f4",
+        "InstanceType": "t2.medium",
+        "DisableApiTermination": False,
+        "TagSpecifications": [
+            {"ResourceType": "instance", "Tags": [{"Key": "test", "Value": "value"}]}
+        ],
+        "SecurityGroupIds": ["sg-abcd1234"],
+    }
+
+    template_name = str(uuid4())
+    conn.create_launch_template(
+        LaunchTemplateName=template_name, LaunchTemplateData=template_data
+    )
+
+    template_config = {
+        "ClientToken": "string",
+        "SpotPrice": "0.01",
+        "TargetCapacity": 1,
+        "IamFleetRole": "arn:aws:iam::486285699788:role/aws-ec2-spot-fleet-tagging-role",
+        "LaunchTemplateConfigs": [
+            {
+                "LaunchTemplateSpecification": {
+                    "LaunchTemplateName": template_name,
+                    "Version": "$Latest",
+                }
+            }
+        ],
+        "AllocationStrategy": allocation_strategy,
+    }
+
+    spot_fleet_res = conn.request_spot_fleet(SpotFleetRequestConfig=template_config)
+    spot_fleet_id = spot_fleet_res["SpotFleetRequestId"]
+
+    instance_res = conn.describe_spot_fleet_instances(SpotFleetRequestId=spot_fleet_id)
+    instances = instance_res["ActiveInstances"]
+    len(instances).should.equal(1)
+    instance_types = set([instance["InstanceType"] for instance in instances])
+    instance_types.should.equal(set(["t2.medium"]))
+    instances[0]["InstanceId"].should.contain("i-")
+
+
+@mock_ec2
+def test_request_spot_fleet_using_launch_template_config__id():
+
+    conn = boto3.client("ec2", region_name="us-east-2")
+
+    template_data = {
+        "ImageId": "ami-04d4e25790238c5f4",
+        "InstanceType": "t2.medium",
+        "DisableApiTermination": False,
+        "TagSpecifications": [
+            {"ResourceType": "instance", "Tags": [{"Key": "test", "Value": "value"}]}
+        ],
+        "SecurityGroupIds": ["sg-abcd1234"],
+    }
+
+    template_name = str(uuid4())
+    template = conn.create_launch_template(
+        LaunchTemplateName=template_name, LaunchTemplateData=template_data
+    )["LaunchTemplate"]
+    template_id = template["LaunchTemplateId"]
+
+    template_config = {
+        "ClientToken": "string",
+        "SpotPrice": "0.01",
+        "TargetCapacity": 1,
+        "IamFleetRole": "arn:aws:iam::486285699788:role/aws-ec2-spot-fleet-tagging-role",
+        "LaunchTemplateConfigs": [
+            {"LaunchTemplateSpecification": {"LaunchTemplateId": template_id}}
+        ],
+        "AllocationStrategy": "lowestCost",
+    }
+
+    spot_fleet_res = conn.request_spot_fleet(SpotFleetRequestConfig=template_config)
+    spot_fleet_id = spot_fleet_res["SpotFleetRequestId"]
+
+    instance_res = conn.describe_spot_fleet_instances(SpotFleetRequestId=spot_fleet_id)
+    instances = instance_res["ActiveInstances"]
+    len(instances).should.equal(1)
+    instance_types = set([instance["InstanceType"] for instance in instances])
+    instance_types.should.equal(set(["t2.medium"]))
+    instances[0]["InstanceId"].should.contain("i-")
+
+
+@mock_ec2
+def test_request_spot_fleet_using_launch_template_config__overrides():
+
+    conn = boto3.client("ec2", region_name="us-east-2")
+    subnet_id = get_subnet_id(conn)
+
+    template_data = {
+        "ImageId": "ami-04d4e25790238c5f4",
+        "InstanceType": "t2.medium",
+        "DisableApiTermination": False,
+        "TagSpecifications": [
+            {"ResourceType": "instance", "Tags": [{"Key": "test", "Value": "value"}]}
+        ],
+        "SecurityGroupIds": ["sg-abcd1234"],
+    }
+
+    template_name = str(uuid4())
+    template = conn.create_launch_template(
+        LaunchTemplateName=template_name, LaunchTemplateData=template_data
+    )["LaunchTemplate"]
+    template_id = template["LaunchTemplateId"]
+
+    template_config = {
+        "ClientToken": "string",
+        "SpotPrice": "0.01",
+        "TargetCapacity": 1,
+        "IamFleetRole": "arn:aws:iam::486285699788:role/aws-ec2-spot-fleet-tagging-role",
+        "LaunchTemplateConfigs": [
+            {
+                "LaunchTemplateSpecification": {"LaunchTemplateId": template_id},
+                "Overrides": [
+                    {
+                        "InstanceType": "t2.nano",
+                        "SubnetId": subnet_id,
+                        "AvailabilityZone": "us-west-1",
+                        "WeightedCapacity": 2,
+                    }
+                ],
+            }
+        ],
+        "AllocationStrategy": "lowestCost",
+    }
+
+    spot_fleet_res = conn.request_spot_fleet(SpotFleetRequestConfig=template_config)
+    spot_fleet_id = spot_fleet_res["SpotFleetRequestId"]
+
+    instance_res = conn.describe_spot_fleet_instances(SpotFleetRequestId=spot_fleet_id)
+    instances = instance_res["ActiveInstances"]
+    instances.should.have.length_of(1)
+    instances[0].should.have.key("InstanceType").equals("t2.nano")
+
+    instance = conn.describe_instances(
+        InstanceIds=[i["InstanceId"] for i in instances]
+    )["Reservations"][0]["Instances"][0]
+    instance.should.have.key("SubnetId").equals(subnet_id)
+
+
 @mock_ec2
 def test_create_spot_fleet_request_with_tag_spec():
     conn = boto3.client("ec2", region_name="us-west-2")