Autoscaling - create custom BlockDevice in addition to default image block device (#5050)

moto/autoscaling/models.py

@@ -131,6 +131,10 @@ class FakeLaunchConfiguration(CloudFormationModel):
         ebs_optimized,
         associate_public_ip_address,
         block_device_mapping_dict,
+        region_name,
+        metadata_options,
+        classic_link_vpc_id,
+        classic_link_vpc_security_groups,
     ):
         self.name = name
         self.image_id = image_id
@@ -146,6 +150,10 @@ class FakeLaunchConfiguration(CloudFormationModel):
         self.ebs_optimized = ebs_optimized
         self.associate_public_ip_address = associate_public_ip_address
         self.block_device_mapping_dict = block_device_mapping_dict
+        self.metadata_options = metadata_options
+        self.classic_link_vpc_id = classic_link_vpc_id
+        self.classic_link_vpc_security_groups = classic_link_vpc_security_groups
+        self.arn = f"arn:aws:autoscaling:{region_name}:{ACCOUNT_ID}:launchConfiguration:9dbbbf87-6141-428a-a409-0752edbe6cad:launchConfigurationName/{self.name}"
 
     @classmethod
     def create_from_instance(cls, name, instance, backend):
@@ -253,6 +261,8 @@ class FakeLaunchConfiguration(CloudFormationModel):
             mount_point = mapping.get("DeviceName")
             if mapping.get("VirtualName") and "ephemeral" in mapping.get("VirtualName"):
                 block_type.ephemeral_name = mapping.get("VirtualName")
+            elif mapping.get("NoDevice", "false") == "true":
+                block_type.no_device = "true"
             else:
                 ebs = mapping.get("Ebs", {})
                 block_type.volume_type = ebs.get("VolumeType")
@@ -260,6 +270,8 @@ class FakeLaunchConfiguration(CloudFormationModel):
                 block_type.delete_on_termination = ebs.get("DeleteOnTermination")
                 block_type.size = ebs.get("VolumeSize")
                 block_type.iops = ebs.get("Iops")
+                block_type.throughput = ebs.get("Throughput")
+                block_type.encrypted = ebs.get("Encrypted")
             block_device_map[mount_point] = block_type
         return block_device_map
 
@@ -678,6 +690,9 @@ class AutoScalingBackend(BaseBackend):
         associate_public_ip_address,
         block_device_mappings,
         instance_id=None,
+        metadata_options=None,
+        classic_link_vpc_id=None,
+        classic_link_vpc_security_groups=None,
     ):
         valid_requests = [
             instance_id is not None,
@@ -705,6 +720,10 @@ class AutoScalingBackend(BaseBackend):
             ebs_optimized=ebs_optimized,
             associate_public_ip_address=associate_public_ip_address,
             block_device_mapping_dict=block_device_mappings,
+            region_name=self.region,
+            metadata_options=metadata_options,
+            classic_link_vpc_id=classic_link_vpc_id,
+            classic_link_vpc_security_groups=classic_link_vpc_security_groups,
         )
         self.launch_configurations[name] = launch_configuration
         return launch_configuration

moto/autoscaling/responses.py

@@ -37,6 +37,9 @@ class AutoScalingResponse(BaseResponse):
             associate_public_ip_address=params.get("AssociatePublicIpAddress"),
             block_device_mappings=params.get("BlockDeviceMappings"),
             instance_id=params.get("InstanceId"),
+            metadata_options=params.get("MetadataOptions"),
+            classic_link_vpc_id=params.get("ClassicLinkVPCId"),
+            classic_link_vpc_security_groups=params.get("ClassicLinkVPCSecurityGroups"),
         )
         template = self.response_template(CREATE_LAUNCH_CONFIGURATION_TEMPLATE)
         return template.render()
@@ -449,13 +452,27 @@ DESCRIBE_LAUNCH_CONFIGURATIONS_TEMPLATE = """<DescribeLaunchConfigurationsRespon
       {% for launch_configuration in launch_configurations %}
         <member>
           <AssociatePublicIpAddress>{{ 'true' if launch_configuration.associate_public_ip_address else 'false' }}</AssociatePublicIpAddress>
+          {% if launch_configuration.classic_link_vpc_id %}
+          <ClassicLinkVPCId>{{ launch_configuration.classic_link_vpc_id }}</ClassicLinkVPCId>
+          {% endif %}
+          {% if launch_configuration.classic_link_vpc_security_groups %}
+          <ClassicLinkVPCSecurityGroups>
+            {% for sg in launch_configuration.classic_link_vpc_security_groups %}
+            <member>{{ sg }}</member>
+            {% endfor %}
+          </ClassicLinkVPCSecurityGroups>
+          {% endif %}
           <SecurityGroups>
             {% for security_group in launch_configuration.security_groups %}
               <member>{{ security_group }}</member>
             {% endfor %}
           </SecurityGroups>
           <CreatedTime>2013-01-21T23:04:42.200Z</CreatedTime>
+          {% if launch_configuration.kernel_id %}
           <KernelId>{{ launch_configuration.kernel_id }}</KernelId>
+          {% else %}
+          <KernelId/>
+          {% endif %}
           {% if launch_configuration.instance_profile_name %}
             <IamInstanceProfile>{{ launch_configuration.instance_profile_name }}</IamInstanceProfile>
           {% endif %}
@@ -466,7 +483,7 @@ DESCRIBE_LAUNCH_CONFIGURATIONS_TEMPLATE = """<DescribeLaunchConfigurationsRespon
             <UserData/>
           {% endif %}
           <InstanceType>{{ launch_configuration.instance_type }}</InstanceType>
-          <LaunchConfigurationARN>arn:aws:autoscaling:us-east-1:803981987763:launchConfiguration:9dbbbf87-6141-428a-a409-0752edbe6cad:launchConfigurationName/{{ launch_configuration.name }}</LaunchConfigurationARN>
+          <LaunchConfigurationARN>{{ launch_configuration.arn }}</LaunchConfigurationARN>
           {% if launch_configuration.block_device_mappings %}
             <BlockDeviceMappings>
             {% for mount_point, mapping in launch_configuration.block_device_mappings.items() %}
@@ -474,6 +491,8 @@ DESCRIBE_LAUNCH_CONFIGURATIONS_TEMPLATE = """<DescribeLaunchConfigurationsRespon
                 <DeviceName>{{ mount_point }}</DeviceName>
                 {% if mapping.ephemeral_name %}
                 <VirtualName>{{ mapping.ephemeral_name }}</VirtualName>
+                {% elif mapping.no_device %}
+                <NoDevice>true</NoDevice>
                 {% else %}
                 <Ebs>
                 {% if mapping.snapshot_id %}
@@ -485,8 +504,18 @@ DESCRIBE_LAUNCH_CONFIGURATIONS_TEMPLATE = """<DescribeLaunchConfigurationsRespon
                 {% if mapping.iops %}
                   <Iops>{{ mapping.iops }}</Iops>
                 {% endif %}
+                {% if mapping.throughput %}
+                  <Throughput>{{ mapping.throughput }}</Throughput>
+                {% endif %}
+                {% if mapping.delete_on_termination is not none %}
                   <DeleteOnTermination>{{ mapping.delete_on_termination }}</DeleteOnTermination>
+                {% endif %}
+                {% if mapping.volume_type %}
                   <VolumeType>{{ mapping.volume_type }}</VolumeType>
+                {% endif %}
+                  {% if mapping.encrypted %}
+                  <Encrypted>{{ mapping.encrypted }}</Encrypted>
+                  {% endif %}
                 </Ebs>
                 {% endif %}
               </member>
@@ -501,7 +530,11 @@ DESCRIBE_LAUNCH_CONFIGURATIONS_TEMPLATE = """<DescribeLaunchConfigurationsRespon
           {% else %}
             <KeyName/>
           {% endif %}
+          {% if launch_configuration.ramdisk_id %}
           <RamdiskId>{{ launch_configuration.ramdisk_id }}</RamdiskId>
+          {% else %}
+          <RamdiskId/>
+          {% endif %}
           <EbsOptimized>{{ launch_configuration.ebs_optimized }}</EbsOptimized>
           <InstanceMonitoring>
             <Enabled>{{ launch_configuration.instance_monitoring_enabled }}</Enabled>
@@ -509,6 +542,13 @@ DESCRIBE_LAUNCH_CONFIGURATIONS_TEMPLATE = """<DescribeLaunchConfigurationsRespon
           {% if launch_configuration.spot_price %}
             <SpotPrice>{{ launch_configuration.spot_price }}</SpotPrice>
           {% endif %}
+          {% if launch_configuration.metadata_options %}
+          <MetadataOptions>
+            <HttpTokens>{{ launch_configuration.metadata_options.get("HttpTokens") }}</HttpTokens>
+            <HttpPutResponseHopLimit>{{ launch_configuration.metadata_options.get("HttpPutResponseHopLimit") }}</HttpPutResponseHopLimit>
+            <HttpEndpoint>{{ launch_configuration.metadata_options.get("HttpEndpoint") }}</HttpEndpoint>
+          </MetadataOptions>
+          {% endif %}
         </member>
       {% endfor %}
     </LaunchConfigurations>

moto/ec2/_models/amis.py

@@ -8,6 +8,7 @@ from ..exceptions import (
     InvalidAMIAttributeItemValueError,
     MalformedAMIIdError,
     InvalidTaggableResourceType,
+    UnvailableAMIIdError,
 )
 from .core import TaggedEC2Resource
 from ..utils import (
@@ -146,6 +147,7 @@ class AmiBackend(object):
 
     def __init__(self):
         self.amis = {}
+        self.deleted_amis = list()
         self._load_amis()
         super().__init__()
 
@@ -216,6 +218,7 @@ class AmiBackend(object):
         if len(ami_ids):
             # boto3 seems to default to just searching based on ami ids if that parameter is passed
             # and if no images are found, it raises an errors
+            # Note that we can search for images that have been previously deleted, without raising any errors
             malformed_ami_ids = [
                 ami_id for ami_id in ami_ids if not ami_id.startswith("ami-")
             ]
@@ -223,7 +226,10 @@ class AmiBackend(object):
                 raise MalformedAMIIdError(malformed_ami_ids)
 
             images = [ami for ami in images if ami.id in ami_ids]
-            if len(images) == 0:
+            deleted_images = [
+                ami_id for ami_id in ami_ids if ami_id in self.deleted_amis
+            ]
+            if len(images) + len(deleted_images) == 0:
                 raise InvalidAMIIdError(ami_ids)
         else:
             # Limit images by launch permissions
@@ -257,7 +263,10 @@ class AmiBackend(object):
     def deregister_image(self, ami_id):
         if ami_id in self.amis:
             self.amis.pop(ami_id)
+            self.deleted_amis.append(ami_id)
             return True
+        elif ami_id in self.deleted_amis:
+            raise UnvailableAMIIdError(ami_id)
         raise InvalidAMIIdError(ami_id)
 
     def get_launch_permission_groups(self, ami_id):

moto/ec2/_models/instances.py

@@ -557,6 +557,8 @@ class InstanceBackend(object):
             new_reservation.instances.append(new_instance)
             new_instance.add_tags(instance_tags)
             block_device_mappings = None
+            if "block_device_mappings" not in kwargs:
+                new_instance.setup_defaults()
             if "block_device_mappings" in kwargs:
                 block_device_mappings = kwargs["block_device_mappings"]
             elif kwargs.get("launch_template"):
@@ -590,8 +592,6 @@ class InstanceBackend(object):
                             kms_key_id,
                             volume_type=volume_type,
                         )
-            else:
-                new_instance.setup_defaults()
             if kwargs.get("instance_market_options"):
                 new_instance.lifecycle = "spot"
             # Tag all created volumes.

moto/ec2/exceptions.py

@@ -258,6 +258,14 @@ class InvalidAMIIdError(EC2ClientError):
         )
 
 
+class UnvailableAMIIdError(EC2ClientError):
+    def __init__(self, ami_id):
+        super().__init__(
+            "InvalidAMIID.Unavailable",
+            "The image id '[{0}]' is no longer available".format(ami_id),
+        )
+
+
 class InvalidAMIAttributeItemValueError(EC2ClientError):
     def __init__(self, attribute, value):
         super().__init__(

moto/ec2/resources/amis.json

@@ -661,5 +661,22 @@
     "root_device_type": "ebs",
     "sriov": "simple",
     "virtualization_type": "hvm"
+  },
+  {
+    "architecture": "x86_64",
+    "ami_id": "ami-04681a1dbd79675b6",
+    "image_location": "amazon/amzn2-ami-minimal-pv-2.0.20180810-x86_64-gp2",
+    "image_type": "machine",
+    "public": true,
+    "owner_id": "137112412989",
+    "platform": "Linux/UNIX",
+    "state": "available",
+    "description": "Example InstanceStore AMI used by AutoScaling tests",
+    "hypervisor": "xen",
+    "name": "amzn-ami-minimal-pv-2.0.20180810-x86_64-gp2",
+    "root_device_name": "/dev/xvda",
+    "root_device_type": "instance-store",
+    "sriov": "simple",
+    "virtualization_type": "hvm"
   }
 ]

tests/terraformtests/terraform-tests.success.txt

@@ -8,8 +8,11 @@ apigatewayv2:
   - TestAccAPIGatewayV2RouteResponse
   - TestAccAPIGatewayV2VPCLink
 autoscaling:
-  - TestAccAutoScalingGroupDataSource
   - TestAccAutoScalingAttachment
+  - TestAccAutoScalingGroupDataSource
+  - TestAccAutoScalingGroupTag
+  - TestAccAutoScalingLaunchConfigurationDataSource
+  - TestAccAutoScalingLaunchConfiguration_
 batch:
   - TestAccBatchJobDefinition
 cloudtrail:

tests/test_autoscaling/test_autoscaling.py

@@ -2810,5 +2810,9 @@ def test_create_template_with_block_device():
     )
     ec2_client = boto3.client("ec2", region_name="ap-southeast-2")
     volumes = ec2_client.describe_volumes()["Volumes"]
-    volumes[0]["VolumeType"].should.equal("gp3")
+    # The standard root volume
-    volumes[0]["Size"].should.equal(20)
+    volumes[0]["VolumeType"].should.equal("gp2")
+    volumes[0]["Size"].should.equal(8)
+    # Our Ebs-volume
+    volumes[1]["VolumeType"].should.equal("gp3")
+    volumes[1]["Size"].should.equal(20)

tests/test_autoscaling/test_launch_configurations.py

@@ -99,7 +99,6 @@ def test_create_launch_configuration_with_block_device_mappings():
     xvdp.should.have.key("Ebs")
     xvdp["Ebs"]["SnapshotId"].should.equal("snap-1234abcd")
     xvdp["Ebs"]["VolumeType"].should.equal("standard")
-    xvdp["Ebs"]["DeleteOnTermination"].should.equal(False)
 
     xvdb["VirtualName"].should.equal("ephemeral0")
     xvdb.shouldnt.have.key("Ebs")
@@ -109,16 +108,32 @@ def test_create_launch_configuration_with_block_device_mappings():
 def test_create_launch_configuration_additional_parameters():
     client = boto3.client("autoscaling", region_name="us-east-1")
     client.create_launch_configuration(
+        ClassicLinkVPCId="vpc_id",
+        ClassicLinkVPCSecurityGroups=["classic_sg1"],
         LaunchConfigurationName="tester",
         ImageId=EXAMPLE_AMI_ID,
         InstanceType="t1.micro",
         EbsOptimized=True,
         AssociatePublicIpAddress=True,
+        MetadataOptions={
+            "HttpTokens": "optional",
+            "HttpPutResponseHopLimit": 123,
+            "HttpEndpoint": "disabled",
+        },
     )
 
     launch_config = client.describe_launch_configurations()["LaunchConfigurations"][0]
+    launch_config["ClassicLinkVPCId"].should.equal("vpc_id")
+    launch_config["ClassicLinkVPCSecurityGroups"].should.equal(["classic_sg1"])
     launch_config["EbsOptimized"].should.equal(True)
     launch_config["AssociatePublicIpAddress"].should.equal(True)
+    launch_config["MetadataOptions"].should.equal(
+        {
+            "HttpTokens": "optional",
+            "HttpPutResponseHopLimit": 123,
+            "HttpEndpoint": "disabled",
+        }
+    )
 
 
 @mock_autoscaling
@@ -287,7 +302,10 @@ def test_launch_config_with_block_device_mappings__volumes_are_created():
     volumes = ec2_client.describe_volumes(
         Filters=[{"Name": "attachment.instance-id", "Values": [instance_id]}]
     )["Volumes"]
-    volumes.should.have.length_of(1)
+    volumes.should.have.length_of(2)
-    volumes[0].should.have.key("Size").equals(10)
+    volumes[0].should.have.key("Size").equals(8)
     volumes[0].should.have.key("Encrypted").equals(False)
-    volumes[0].should.have.key("VolumeType").equals("standard")
+    volumes[0].should.have.key("VolumeType").equals("gp2")
+    volumes[1].should.have.key("Size").equals(10)
+    volumes[1].should.have.key("Encrypted").equals(False)
+    volumes[1].should.have.key("VolumeType").equals("standard")

tests/test_ec2/test_amis.py

@@ -117,10 +117,41 @@ def test_ami_create_and_delete():
         ec2.deregister_image(ImageId=image_id)
     ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
     err = ex.value.response["Error"]
+    err["Code"].should.equal("InvalidAMIID.Unavailable")
+    ex.value.response["ResponseMetadata"]["RequestId"].should_not.equal(None)
+
+
+@mock_ec2
+def test_deregister_image__unknown():
+    ec2 = boto3.client("ec2", region_name="us-east-1")
+
+    with pytest.raises(ClientError) as ex:
+        ec2.deregister_image(ImageId="ami-unknown-ami")
+    ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
+    err = ex.value.response["Error"]
     err["Code"].should.equal("InvalidAMIID.NotFound")
     ex.value.response["ResponseMetadata"]["RequestId"].should_not.equal(None)
 
 
+@mock_ec2
+def test_deregister_image__and_describe():
+    ec2 = boto3.client("ec2", region_name="us-east-1")
+
+    reservation = ec2.run_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1)
+    instance = reservation["Instances"][0]
+    instance_id = instance["InstanceId"]
+
+    image_id = ec2.create_image(
+        InstanceId=instance_id, Name="test-ami", Description="this is a test ami"
+    )["ImageId"]
+
+    ec2.deregister_image(ImageId=image_id)
+
+    # Searching for a deleted image ID should not throw an error
+    # It should simply not return this image
+    ec2.describe_images(ImageIds=[image_id])["Images"].should.have.length_of(0)
+
+
 @mock_ec2
 def test_ami_copy_dryrun():
     ec2 = boto3.client("ec2", region_name="us-west-1")