Merge pull request #3508 from bblommers/feature/ec2-describe-instance-type-offerings

EC2: describe instance type offerings

MANIFEST.in

@@ -1,6 +1,7 @@
 include README.md LICENSE AUTHORS.md
 include requirements.txt requirements-dev.txt tox.ini
 include moto/ec2/resources/instance_types.json
+include moto/ec2/resources/instance_type_offerings/*/*.json
 include moto/ec2/resources/amis.json
 include moto/cognitoidp/resources/*.json
 include moto/dynamodb2/parsing/reserved_keywords.txt

moto/ec2/exceptions.py

@@ -198,6 +198,14 @@ class InvalidInstanceIdError(EC2ClientError):
         )
 
 
+class InvalidInstanceTypeError(EC2ClientError):
+    def __init__(self, instance_type):
+        super(InvalidInstanceTypeError, self).__init__(
+            "InvalidInstanceType.NotFound",
+            "The instance type '{0}' does not exist".format(instance_type),
+        )
+
+
 class InvalidAMIIdError(EC2ClientError):
     def __init__(self, ami_id):
         super(InvalidAMIIdError, self).__init__(

moto/ec2/models.py

@@ -34,6 +34,7 @@ from moto.core.utils import (
 )
 from moto.core import ACCOUNT_ID
 from moto.kms import kms_backends
+from os import listdir
 
 from .exceptions import (
     CidrLimitExceeded,
@@ -56,6 +57,7 @@ from .exceptions import (
     InvalidDomainError,
     InvalidID,
     InvalidInstanceIdError,
+    InvalidInstanceTypeError,
     InvalidInternetGatewayIdError,
     InvalidKeyPairDuplicateError,
     InvalidKeyPairFormatError,
@@ -174,6 +176,21 @@ INSTANCE_TYPES = _load_resource(
     resource_filename(__name__, "resources/instance_types.json")
 )
 
+offerings_path = "resources/instance_type_offerings"
+INSTANCE_TYPE_OFFERINGS = {}
+for location_type in listdir(resource_filename(__name__, offerings_path)):
+    INSTANCE_TYPE_OFFERINGS[location_type] = {}
+    for region in listdir(
+        resource_filename(__name__, offerings_path + "/" + location_type)
+    ):
+        full_path = resource_filename(
+            __name__, offerings_path + "/" + location_type + "/" + region
+        )
+        INSTANCE_TYPE_OFFERINGS[location_type][
+            region.replace(".json", "")
+        ] = _load_resource(full_path)
+
+
 AMIS = _load_resource(
     os.environ.get("MOTO_AMIS_PATH")
     or resource_filename(__name__, "resources/amis.json"),
@@ -1127,6 +1144,51 @@ class InstanceBackend(object):
         return reservations
 
 
+class InstanceTypeBackend(object):
+    def __init__(self):
+        super(InstanceTypeBackend, self).__init__()
+
+    def describe_instance_types(self, instance_types=None):
+        matches = INSTANCE_TYPES.values()
+        if instance_types:
+            matches = [t for t in matches if t.get("apiname") in instance_types]
+            if len(instance_types) > len(matches):
+                unknown_ids = set(instance_types) - set(matches)
+                raise InvalidInstanceTypeError(unknown_ids)
+        return matches
+
+
+class InstanceTypeOfferingBackend(object):
+    def __init__(self):
+        super(InstanceTypeOfferingBackend, self).__init__()
+
+    def describe_instance_type_offerings(self, location_type=None, filters=None):
+        location_type = location_type or "region"
+        matches = INSTANCE_TYPE_OFFERINGS[location_type]
+        matches = matches[self.region_name]
+
+        def matches_filters(offering, filters):
+            def matches_filter(key, values):
+                if key == "location":
+                    if location_type in ("availability-zone", "availability-zone-id"):
+                        return offering.get("Location") in values
+                    elif location_type == "region":
+                        return any(
+                            v for v in values if offering.get("Location").startswith(v)
+                        )
+                    else:
+                        return False
+                elif key == "instance-type":
+                    return offering.get("InstanceType") in values
+                else:
+                    return False
+
+            return all([matches_filter(key, values) for key, values in filters.items()])
+
+        matches = [o for o in matches if matches_filters(o, filters)]
+        return matches
+
+
 class KeyPair(object):
     def __init__(self, name, fingerprint, material):
         self.name = name
@@ -6015,6 +6077,8 @@ class IamInstanceProfileAssociationBackend(object):
 class EC2Backend(
     BaseBackend,
     InstanceBackend,
+    InstanceTypeBackend,
+    InstanceTypeOfferingBackend,
     TagBackend,
     EBSBackend,
     RegionsAndZonesBackend,

moto/ec2/responses/instances.py

@@ -1,5 +1,4 @@
 from __future__ import unicode_literals
-from moto.packages.boto.ec2.instancetype import InstanceType
 
 from moto.autoscaling import autoscaling_backends
 from moto.core.responses import BaseResponse
@@ -144,12 +143,20 @@ class InstanceResponse(BaseResponse):
         return template.render(instances=instances)
 
     def describe_instance_types(self):
-        instance_types = [
-            InstanceType(name="t1.micro", cores=1, memory=644874240, disk=0)
-        ]
+        instance_type_filters = self._get_multi_param("InstanceType")
+        instance_types = self.ec2_backend.describe_instance_types(instance_type_filters)
         template = self.response_template(EC2_DESCRIBE_INSTANCE_TYPES)
         return template.render(instance_types=instance_types)
 
+    def describe_instance_type_offerings(self):
+        location_type_filters = self._get_param("LocationType")
+        filter_dict = filters_from_querystring(self.querystring)
+        offerings = self.ec2_backend.describe_instance_type_offerings(
+            location_type_filters, filter_dict
+        )
+        template = self.response_template(EC2_DESCRIBE_INSTANCE_TYPE_OFFERINGS)
+        return template.render(instance_type_offerings=offerings)
+
     def describe_instance_attribute(self):
         # TODO this and modify below should raise IncorrectInstanceState if
         # instance not in stopped state
@@ -818,17 +825,17 @@ EC2_DESCRIBE_INSTANCE_TYPES = """<?xml version="1.0" encoding="UTF-8"?>
     <instanceTypeSet>
     {% for instance_type in instance_types %}
         <item>
-            <instanceType>{{ instance_type.name }}</instanceType>
+            <instanceType>{{ instance_type.apiname }}</instanceType>
             <vCpuInfo>
-                <defaultVCpus>{{ instance_type.cores }}</defaultVCpus>
-                <defaultCores>{{ instance_type.cores }}</defaultCores>
+                <defaultVCpus>{{ instance_type.vcpus|int }}</defaultVCpus>
+                <defaultCores>{{ instance_type.vcpus|int }}</defaultCores>
                 <defaultThreadsPerCore>1</defaultThreadsPerCore>
             </vCpuInfo>
             <memoryInfo>
-                <sizeInMiB>{{ instance_type.memory }}</sizeInMiB>
+                <sizeInMiB>{{ instance_type.memory|int }}</sizeInMiB>
             </memoryInfo>
             <instanceStorageInfo>
-                <totalSizeInGB>{{ instance_type.disk }}</totalSizeInGB>
+                <totalSizeInGB>{{ instance_type.storage|int }}</totalSizeInGB>
             </instanceStorageInfo>
             <processorInfo>
                 <supportedArchitectures>
@@ -841,3 +848,18 @@ EC2_DESCRIBE_INSTANCE_TYPES = """<?xml version="1.0" encoding="UTF-8"?>
     {% endfor %}
     </instanceTypeSet>
 </DescribeInstanceTypesResponse>"""
+
+
+EC2_DESCRIBE_INSTANCE_TYPE_OFFERINGS = """<?xml version="1.0" encoding="UTF-8"?>
+<DescribeInstanceTypeOfferingsResponse xmlns="http://api.outscale.com/wsdl/fcuext/2014-04-15/">
+    <requestId>f8b86168-d034-4e65-b48d-3b84c78e64af</requestId>
+    <instanceTypeOfferingSet>
+    {% for offering in instance_type_offerings %}
+        <item>
+            <instanceType>{{ offering.InstanceType }}</instanceType>
+            <location>{{ offering.Location }}</location>
+            <locationType>{{ offering.LocationType }}</locationType>
+        </item>
+    {% endfor %}
+    </instanceTypeOfferingSet>
+</DescribeInstanceTypeOfferingsResponse>"""

scripts/ec2_get_instance_type_offerings.py

@@ -0,0 +1,63 @@
+"""
+Get InstanceTypeOfferings from AWS
+Stores result in moto/ec2/resources/instance_type_offerings/{location_type}/{region}.json
+Where {location_type} is one of region/availability-zone/availability-zone-id
+
+Note that you will get the following error if a region is not available to you:
+  An error occurred (AuthFailure) when calling the DescribeInstanceTypeOfferings operation:
+  AWS was not able to validate the provided access credentials
+"""
+import boto3
+import json
+import os
+import subprocess
+from boto3 import Session
+from time import sleep
+
+PATH = "moto/ec2/resources/instance_type_offerings"
+TYPES = ["region", "availability-zone", "availability-zone-id"]
+
+
+def main():
+    print("Getting InstanceTypeOfferings from all regions")
+    regions = []
+    regions.extend(Session().get_available_regions("ec2"))
+    regions.extend(Session().get_available_regions("ec2", partition_name="aws-us-gov"))
+    regions.extend(Session().get_available_regions("ec2", partition_name="aws-cn"))
+    print("Found " + str(len(regions)) + " regions")
+
+    root_dir = (
+        subprocess.check_output(["git", "rev-parse", "--show-toplevel"])
+        .decode()
+        .strip()
+    )
+    for region in regions:
+        for location_type in TYPES:
+            ec2 = boto3.client("ec2", region_name=region)
+            dest = os.path.join(root_dir, "{0}/{1}/{2}.json".format(PATH, location_type, region))
+            try:
+                instances = []
+                offerings = ec2.describe_instance_type_offerings(
+                    LocationType=location_type
+                )
+                instances.extend(offerings["InstanceTypeOfferings"])
+                next_token = offerings.get("NextToken", "")
+                while next_token:
+                    offerings = ec2.describe_instance_type_offerings(
+                        LocationType=location_type,
+                        NextToken=next_token
+                    )
+                    instances.extend(offerings["InstanceTypeOfferings"])
+                    next_token = offerings.get("NextToken", None)
+                print("Writing data to {0}".format(dest))
+                with open(dest, "w+") as open_file:
+                    json.dump(instances, open_file, sort_keys=True)
+            except Exception as e:
+                print("Unable to write data to {0}".format(dest))
+                print(e)
+        # We don't want it to look like we're DDOS'ing AWS
+        sleep(1)
+
+
+if __name__ == "__main__":
+    main()

tests/test_ec2/test_instance_type_offerings.py

@@ -0,0 +1,85 @@
+from __future__ import unicode_literals
+
+import boto3
+import sure  # noqa
+
+from moto import mock_ec2
+
+
+@mock_ec2
+def test_describe_instance_type_offerings():
+    client = boto3.client("ec2", "us-east-1")
+    offerings = client.describe_instance_type_offerings()
+
+    offerings.should.have.key("InstanceTypeOfferings")
+    offerings["InstanceTypeOfferings"].should_not.be.empty
+    offerings["InstanceTypeOfferings"][0].should.have.key("InstanceType")
+    offerings["InstanceTypeOfferings"][0].should.have.key("Location")
+    offerings["InstanceTypeOfferings"][0].should.have.key("LocationType")
+
+
+@mock_ec2
+def test_describe_instance_type_offering_filter_by_type():
+    client = boto3.client("ec2", "us-east-1")
+
+    # Verify offerings of a specific instance type
+    offerings = client.describe_instance_type_offerings(
+        Filters=[{"Name": "instance-type", "Values": ["t2.nano"]}]
+    )
+
+    offerings.should.have.key("InstanceTypeOfferings")
+    offerings["InstanceTypeOfferings"].should_not.be.empty
+    offerings = offerings["InstanceTypeOfferings"]
+    offerings.should.have.length_of(1)
+    offerings[0]["InstanceType"].should.equal("t2.nano")
+    offerings[0]["Location"].should.equal("us-east-1")
+
+    # Verify offerings of that instance type per availibility zone
+    offerings = client.describe_instance_type_offerings(
+        LocationType="availability-zone",
+        Filters=[{"Name": "instance-type", "Values": ["t2.nano"]}],
+    )
+    offerings.should.have.key("InstanceTypeOfferings")
+    offerings = offerings["InstanceTypeOfferings"]
+    offerings.should.have.length_of(6)
+    for offering in offerings:
+        offering["InstanceType"].should.equal("t2.nano")
+        offering["LocationType"].should.equal("availability-zone")
+        offering["Location"].should.match("us-east-1[a-f]")
+
+
+@mock_ec2
+def test_describe_instance_type_offering_filter_by_zone():
+    client = boto3.client("ec2", "us-east-1")
+    offerings = client.describe_instance_type_offerings(
+        LocationType="availability-zone",
+        Filters=[{"Name": "location", "Values": ["us-east-1c"]}],
+    )
+
+    offerings.should.have.key("InstanceTypeOfferings")
+    offerings = offerings["InstanceTypeOfferings"]
+    offerings.should_not.be.empty
+    offerings.should.have.length_of(353)
+    assert all([o["LocationType"] == "availability-zone" for o in offerings])
+    assert all([o["Location"] == "us-east-1c" for o in offerings])
+    assert any([o["InstanceType"] == "a1.2xlarge" for o in offerings])
+
+
+@mock_ec2
+def test_describe_instance_type_offering_filter_by_zone_id():
+    client = boto3.client("ec2", "ca-central-1")
+    offerings = client.describe_instance_type_offerings(
+        LocationType="availability-zone-id",
+        Filters=[
+            {"Name": "location", "Values": ["cac1-az1"]},
+            {"Name": "instance-type", "Values": ["c5.9xlarge"]},
+        ],
+    )
+
+    offerings.should.have.key("InstanceTypeOfferings")
+    offerings = offerings["InstanceTypeOfferings"]
+    offerings.should_not.be.empty
+    offerings.should.have.length_of(1)
+    offerings[0]["LocationType"].should.equal("availability-zone-id")
+    offerings[0]["InstanceType"].should.equal("c5.9xlarge")
+    offerings[0]["Location"].should.equal("cac1-az1")

tests/test_ec2/test_instance_types.py

@@ -2,6 +2,9 @@ from __future__ import unicode_literals
 
 import boto3
 import sure  # noqa
+import pytest
+
+from botocore.exceptions import ClientError
 
 from moto import mock_ec2
 
@@ -16,3 +19,34 @@ def test_describe_instance_types():
     instance_types["InstanceTypes"][0].should.have.key("InstanceType")
     instance_types["InstanceTypes"][0].should.have.key("MemoryInfo")
     instance_types["InstanceTypes"][0]["MemoryInfo"].should.have.key("SizeInMiB")
+
+
+@mock_ec2
+def test_describe_instance_types_filter_by_type():
+    client = boto3.client("ec2", "us-east-1")
+    instance_types = client.describe_instance_types(
+        InstanceTypes=["t1.micro", "t2.nano"]
+    )
+
+    instance_types.should.have.key("InstanceTypes")
+    instance_types["InstanceTypes"].should_not.be.empty
+    instance_types["InstanceTypes"].should.have.length_of(2)
+    instance_types["InstanceTypes"][0]["InstanceType"].should.be.within(
+        ["t1.micro", "t2.nano"]
+    )
+    instance_types["InstanceTypes"][1]["InstanceType"].should.be.within(
+        ["t1.micro", "t2.nano"]
+    )
+
+
+@mock_ec2
+def test_describe_instance_types_unknown_type():
+    client = boto3.client("ec2", "us-east-1")
+
+    with pytest.raises(ClientError) as err:
+        client.describe_instance_types(InstanceTypes=["t1.non_existent"])
+        err.response["Error"]["Code"].should.equal("ValidationException")
+        err.response["Error"]["Message"].split(":")[0].should.look_like(
+            "The instance type '{'t1.non_existent'}' does not exist"
+        )
+        err.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)