refactor scripts/get_amis.py into moto.ec2.utils.gen_moto_amis() (#4867)

docs/docs/services/ec2.rst

@@ -12,6 +12,8 @@
 ec2
 ===
 
+.. autoclass:: moto.ec2.models.EC2Backend
+
 |start-h3| Example usage |end-h3|
 
 .. sourcecode:: python

moto/ec2/models.py

@@ -8642,6 +8642,20 @@ class EC2Backend(
     IamInstanceProfileAssociationBackend,
     CarrierGatewayBackend,
 ):
+    """
+    Implementation of the AWS EC2 endpoint.
+
+    moto includes a limited set of AMIs in `moto/ec2/resources/amis.json`.  If you require specific
+    AMIs to be available during your tests, you can provide your own AMI definitions by setting the
+    environment variable `MOTO_AMIS_PATH` to point to a JSON file containing definitions of the
+    required AMIs.
+
+    To create such a file, refer to `scripts/get_amis.py`
+
+    .. note:: You must set `MOTO_AMIS_PATH` before importing moto.
+
+    """
+
     def __init__(self, region_name):
         self.region_name = region_name
         super().__init__()

moto/ec2/utils.py

@@ -786,3 +786,55 @@ def describe_tag_filter(filters, instances):
                     if need_delete:
                         result.remove(instance)
     return result
+
+
+def gen_moto_amis(described_images, drop_images_missing_keys=True):
+    """Convert `boto3.EC2.Client.describe_images` output to form acceptable to `MOTO_AMIS_PATH`
+
+    Parameters
+    ==========
+    described_images : list of dicts
+        as returned by :ref:`boto3:EC2.Client.describe_images` in "Images" key
+    drop_images_missing_keys : bool, default=True
+        When `True` any entry in `images` that is missing a required key will silently
+        be excluded from the returned list
+
+    Throws
+    ======
+    `KeyError` when `drop_images_missing_keys` is `False` and a required key is missing
+    from an element of `images`
+
+    Returns
+    =======
+    list of dicts suitable to be serialized into JSON as a target for `MOTO_AMIS_PATH` environment
+    variable.
+
+    See Also
+    ========
+    * :ref:`moto.ec2.models.EC2Backend`
+    """
+    result = []
+    for image in described_images:
+        try:
+            tmp = {
+                "ami_id": image["ImageId"],
+                "name": image["Name"],
+                "description": image["Description"],
+                "owner_id": image["OwnerId"],
+                "public": image["Public"],
+                "virtualization_type": image["VirtualizationType"],
+                "architecture": image["Architecture"],
+                "state": image["State"],
+                "platform": image.get("Platform"),
+                "image_type": image["ImageType"],
+                "hypervisor": image["Hypervisor"],
+                "root_device_name": image["RootDeviceName"],
+                "root_device_type": image["RootDeviceType"],
+                "sriov": image.get("SriovNetSupport", "simple"),
+            }
+            result.append(tmp)
+        except Exception as err:
+            if not drop_images_missing_keys:
+                raise err
+
+    return result

scripts/get_amis.py

@@ -1,6 +1,8 @@
 import boto3
 import json
 
+from moto.ec2.utils import gen_moto_amis
+
 # Taken from free tier list when creating an instance
 instances = [
     "ami-760aaa0f",
@@ -43,27 +45,6 @@ client = boto3.client("ec2", region_name="eu-west-1")
 
 test = client.describe_images(ImageIds=instances)
 
-result = []
+result = gen_moto_amis(test["Images"])
-for image in test["Images"]:
-    try:
-        tmp = {
-            "ami_id": image["ImageId"],
-            "name": image["Name"],
-            "description": image["Description"],
-            "owner_id": image["OwnerId"],
-            "public": image["Public"],
-            "virtualization_type": image["VirtualizationType"],
-            "architecture": image["Architecture"],
-            "state": image["State"],
-            "platform": image.get("Platform"),
-            "image_type": image["ImageType"],
-            "hypervisor": image["Hypervisor"],
-            "root_device_name": image["RootDeviceName"],
-            "root_device_type": image["RootDeviceType"],
-            "sriov": image.get("SriovNetSupport", "simple"),
-        }
-        result.append(tmp)
-    except Exception as err:
-        pass
 
 print(json.dumps(result, indent=2))

tests/test_ec2/test_utils.py

@@ -1,5 +1,8 @@
+from copy import deepcopy
 import ipaddress
+import sure  # noqa # pylint: disable=unused-import
 from unittest.mock import patch
+from pytest import raises
 
 from moto.ec2 import utils
 
@@ -23,3 +26,36 @@ def test_random_ipv6_cidr():
         cidr_address = utils.random_ipv6_cidr()
         # this will throw value error if host bits are set
         ipaddress.ip_network(cidr_address)
+
+
+def test_gen_moto_amis():
+    image_with_all_reqd_keys = {
+        "ImageId": "ami-03cf127a",
+        "State": "available",
+        "Public": True,
+        "OwnerId": "801119661308",
+        "RootDeviceType": "ebs",
+        "RootDeviceName": "/dev/sda1",
+        "Description": "Microsoft Windows Server 2016 Nano Locale English AMI provided by Amazon",
+        "ImageType": "machine",
+        "Architecture": "x86_64",
+        "Name": "Windows_Server-2016-English-Nano-Base-2017.10.13",
+        "VirtualizationType": "hvm",
+        "Hypervisor": "xen",
+    }
+
+    images = []
+    images.append(deepcopy(image_with_all_reqd_keys))
+    images.append(deepcopy(image_with_all_reqd_keys))
+
+    # make one of the copies of the image miss a key
+    images[1].pop("Public")
+
+    # with drop=True, it shouldn't throw but will give us only one AMI in the result
+    images.should.have.length_of(2)
+    amis = utils.gen_moto_amis(images, drop_images_missing_keys=True)
+    amis.should.have.length_of(1)
+
+    # with drop=False, it should raise KeyError because of the missing key
+    with raises(KeyError, match="'Public'"):
+        utils.gen_moto_amis(images, drop_images_missing_keys=False)