Merge branch 'master' of github.com:ecumene/moto

docs/docs/services/ec2.rst

@@ -12,6 +12,8 @@
 ec2
 ===
 
+.. autoclass:: moto.ec2.models.EC2Backend
+
 |start-h3| Example usage |end-h3|
 
 .. sourcecode:: python

moto/ec2/models.py

@@ -8642,6 +8642,20 @@ class EC2Backend(
     IamInstanceProfileAssociationBackend,
     CarrierGatewayBackend,
 ):
+    """
+    Implementation of the AWS EC2 endpoint.
+
+    moto includes a limited set of AMIs in `moto/ec2/resources/amis.json`.  If you require specific
+    AMIs to be available during your tests, you can provide your own AMI definitions by setting the
+    environment variable `MOTO_AMIS_PATH` to point to a JSON file containing definitions of the
+    required AMIs.
+
+    To create such a file, refer to `scripts/get_amis.py`
+
+    .. note:: You must set `MOTO_AMIS_PATH` before importing moto.
+
+    """
+
     def __init__(self, region_name):
         self.region_name = region_name
         super().__init__()

moto/ec2/utils.py

@@ -786,3 +786,55 @@ def describe_tag_filter(filters, instances):
                     if need_delete:
                         result.remove(instance)
     return result
+
+
+def gen_moto_amis(described_images, drop_images_missing_keys=True):
+    """Convert `boto3.EC2.Client.describe_images` output to form acceptable to `MOTO_AMIS_PATH`
+
+    Parameters
+    ==========
+    described_images : list of dicts
+        as returned by :ref:`boto3:EC2.Client.describe_images` in "Images" key
+    drop_images_missing_keys : bool, default=True
+        When `True` any entry in `images` that is missing a required key will silently
+        be excluded from the returned list
+
+    Throws
+    ======
+    `KeyError` when `drop_images_missing_keys` is `False` and a required key is missing
+    from an element of `images`
+
+    Returns
+    =======
+    list of dicts suitable to be serialized into JSON as a target for `MOTO_AMIS_PATH` environment
+    variable.
+
+    See Also
+    ========
+    * :ref:`moto.ec2.models.EC2Backend`
+    """
+    result = []
+    for image in described_images:
+        try:
+            tmp = {
+                "ami_id": image["ImageId"],
+                "name": image["Name"],
+                "description": image["Description"],
+                "owner_id": image["OwnerId"],
+                "public": image["Public"],
+                "virtualization_type": image["VirtualizationType"],
+                "architecture": image["Architecture"],
+                "state": image["State"],
+                "platform": image.get("Platform"),
+                "image_type": image["ImageType"],
+                "hypervisor": image["Hypervisor"],
+                "root_device_name": image["RootDeviceName"],
+                "root_device_type": image["RootDeviceType"],
+                "sriov": image.get("SriovNetSupport", "simple"),
+            }
+            result.append(tmp)
+        except Exception as err:
+            if not drop_images_missing_keys:
+                raise err
+
+    return result

moto/iam/models.py

@@ -2339,7 +2339,8 @@ class IAMBackend(BaseBackend):
         user.delete_policy(policy_name)
 
     def delete_policy(self, policy_arn):
-        del self.managed_policies[policy_arn]
+        policy = self.get_policy(policy_arn)
+        del self.managed_policies[policy.arn]
 
     def create_access_key(self, user_name=None, status="Active"):
         user = self.get_user(user_name)

moto/sqs/models.py

@@ -37,7 +37,7 @@ from .exceptions import (
     InvalidAttributeValue,
 )
 
-from moto.core import ACCOUNT_ID as DEFAULT_ACCOUNT_ID
+from moto.core import ACCOUNT_ID
 
 DEFAULT_SENDER_ID = "AIDAIT2UOQQY3AUEKVGXU"
 
@@ -262,7 +262,7 @@ class Queue(CloudFormationModel):
         now = unix_time()
         self.created_timestamp = now
         self.queue_arn = "arn:aws:sqs:{0}:{1}:{2}".format(
-            self.region, DEFAULT_ACCOUNT_ID, self.name
+            self.region, ACCOUNT_ID, self.name
         )
         self.dead_letter_queue = None
 
@@ -454,8 +454,11 @@ class Queue(CloudFormationModel):
     def delete_from_cloudformation_json(
         cls, resource_name, cloudformation_json, region_name
     ):
+        # ResourceName will be the full queue URL - we only need the name
+        # https://sqs.us-west-1.amazonaws.com/123456789012/queue_name
+        queue_name = resource_name.split("/")[-1]
         sqs_backend = sqs_backends[region_name]
-        sqs_backend.delete_queue(resource_name)
+        sqs_backend.delete_queue(queue_name)
 
     @property
     def approximate_number_of_messages_delayed(self):
@@ -471,7 +474,7 @@ class Queue(CloudFormationModel):
 
     @property
     def physical_resource_id(self):
-        return self.name
+        return f"https://sqs.{self.region}.amazonaws.com/{ACCOUNT_ID}/{self.name}"
 
     @property
     def attributes(self):
@@ -505,7 +508,7 @@ class Queue(CloudFormationModel):
 
     def url(self, request_url):
         return "{0}://{1}/{2}/{3}".format(
-            request_url.scheme, request_url.netloc, DEFAULT_ACCOUNT_ID, self.name
+            request_url.scheme, request_url.netloc, ACCOUNT_ID, self.name
         )
 
     @property

scripts/get_amis.py

@@ -1,6 +1,8 @@
 import boto3
 import json
 
+from moto.ec2.utils import gen_moto_amis
+
 # Taken from free tier list when creating an instance
 instances = [
     "ami-760aaa0f",
@@ -43,27 +45,6 @@ client = boto3.client("ec2", region_name="eu-west-1")
 
 test = client.describe_images(ImageIds=instances)
 
-result = []
-for image in test["Images"]:
-    try:
-        tmp = {
-            "ami_id": image["ImageId"],
-            "name": image["Name"],
-            "description": image["Description"],
-            "owner_id": image["OwnerId"],
-            "public": image["Public"],
-            "virtualization_type": image["VirtualizationType"],
-            "architecture": image["Architecture"],
-            "state": image["State"],
-            "platform": image.get("Platform"),
-            "image_type": image["ImageType"],
-            "hypervisor": image["Hypervisor"],
-            "root_device_name": image["RootDeviceName"],
-            "root_device_type": image["RootDeviceType"],
-            "sriov": image.get("SriovNetSupport", "simple"),
-        }
-        result.append(tmp)
-    except Exception as err:
-        pass
+result = gen_moto_amis(test["Images"])
 
 print(json.dumps(result, indent=2))

tests/test_ec2/test_utils.py

@@ -1,5 +1,8 @@
+from copy import deepcopy
 import ipaddress
+import sure  # noqa # pylint: disable=unused-import
 from unittest.mock import patch
+from pytest import raises
 
 from moto.ec2 import utils
 
@@ -23,3 +26,36 @@ def test_random_ipv6_cidr():
         cidr_address = utils.random_ipv6_cidr()
         # this will throw value error if host bits are set
         ipaddress.ip_network(cidr_address)
+
+
+def test_gen_moto_amis():
+    image_with_all_reqd_keys = {
+        "ImageId": "ami-03cf127a",
+        "State": "available",
+        "Public": True,
+        "OwnerId": "801119661308",
+        "RootDeviceType": "ebs",
+        "RootDeviceName": "/dev/sda1",
+        "Description": "Microsoft Windows Server 2016 Nano Locale English AMI provided by Amazon",
+        "ImageType": "machine",
+        "Architecture": "x86_64",
+        "Name": "Windows_Server-2016-English-Nano-Base-2017.10.13",
+        "VirtualizationType": "hvm",
+        "Hypervisor": "xen",
+    }
+
+    images = []
+    images.append(deepcopy(image_with_all_reqd_keys))
+    images.append(deepcopy(image_with_all_reqd_keys))
+
+    # make one of the copies of the image miss a key
+    images[1].pop("Public")
+
+    # with drop=True, it shouldn't throw but will give us only one AMI in the result
+    images.should.have.length_of(2)
+    amis = utils.gen_moto_amis(images, drop_images_missing_keys=True)
+    amis.should.have.length_of(1)
+
+    # with drop=False, it should raise KeyError because of the missing key
+    with raises(KeyError, match="'Public'"):
+        utils.gen_moto_amis(images, drop_images_missing_keys=False)

tests/test_sqs/test_sqs_cloudformation.py

@@ -62,7 +62,7 @@ def test_describe_stack_subresources():
     for s in stack.resource_summaries.all():
         s.resource_type.should.equal("AWS::SQS::Queue")
         s.logical_id.should.equal("QueueGroup")
-        s.physical_resource_id.should.equal(q_name)
+        s.physical_resource_id.should.contain(f"/{q_name}")
 
 
 @mock_sqs
@@ -83,7 +83,8 @@ def test_list_stack_resources():
 
     queue.should.have.key("ResourceType").equal("AWS::SQS::Queue")
     queue.should.have.key("LogicalResourceId").should.equal("QueueGroup")
-    queue.should.have.key("PhysicalResourceId").should.equal(q_name)
+    expected_url = f"https://sqs.us-east-1.amazonaws.com/{ACCOUNT_ID}/{q_name}"
+    queue.should.have.key("PhysicalResourceId").should.equal(expected_url)
 
 
 @mock_sqs
@@ -96,7 +97,7 @@ def test_create_from_cloudformation_json_with_tags():
     cf.create_stack(StackName=stack_name, TemplateBody=sqs_template_with_tags)
 
     response = cf.describe_stack_resources(StackName=stack_name)
-    q_name = response["StackResources"][0]["PhysicalResourceId"]
+    q_name = response["StackResources"][0]["PhysicalResourceId"].split("/")[-1]
 
     all_urls = client.list_queues(QueueNamePrefix=q_name)["QueueUrls"]
     queue_url = [url for url in all_urls if url.endswith(q_name)][0]