Fix unknown instance type (#3710)

* Use apiname when comparing unknown instance ids

* Use get_instance_types API to populate instance_types.json

* Fix scope of instances array when retrieving instance types

moto/batch/models.py

@@ -915,7 +915,10 @@ class BatchBackend(BaseBackend):
                 instance_type = "m4.4xlarge"
 
             instance_vcpus.append(
-                (EC2_INSTANCE_TYPES[instance_type]["vcpus"], instance_type)
+                (
+                    EC2_INSTANCE_TYPES[instance_type]["VCpuInfo"]["DefaultVCpus"],
+                    instance_type,
+                )
             )
 
         instance_vcpus = sorted(instance_vcpus, key=lambda item: item[0], reverse=True)

moto/ec2/models.py

@@ -1156,9 +1156,11 @@ class InstanceTypeBackend(object):
     def describe_instance_types(self, instance_types=None):
         matches = INSTANCE_TYPES.values()
         if instance_types:
-            matches = [t for t in matches if t.get("apiname") in instance_types]
+            matches = [t for t in matches if t.get("InstanceType") in instance_types]
             if len(instance_types) > len(matches):
-                unknown_ids = set(instance_types) - set(matches)
+                unknown_ids = set(instance_types) - set(
+                    t.get("InstanceType") for t in matches
+                )
                 raise InvalidInstanceTypeError(unknown_ids)
         return matches
 

moto/ec2/responses/instances.py

@@ -825,23 +825,25 @@ EC2_DESCRIBE_INSTANCE_TYPES = """<?xml version="1.0" encoding="UTF-8"?>
     <instanceTypeSet>
     {% for instance_type in instance_types %}
         <item>
-            <instanceType>{{ instance_type.apiname }}</instanceType>
+            <instanceType>{{ instance_type.InstanceType }}</instanceType>
             <vCpuInfo>
-                <defaultVCpus>{{ instance_type.vcpus|int }}</defaultVCpus>
-                <defaultCores>{{ instance_type.vcpus|int }}</defaultCores>
-                <defaultThreadsPerCore>1</defaultThreadsPerCore>
+                <defaultVCpus>{{ instance_type.get('VCpuInfo', {}).get('DefaultVCpus', 0)|int }}</defaultVCpus>
+                <defaultCores>{{ instance_type.get('VCpuInfo', {}).get('DefaultCores', 0)|int }}</defaultCores>
+                <defaultThreadsPerCore>{{ instance_type.get('VCpuInfo').get('DefaultThreadsPerCore', 0)|int }}</defaultThreadsPerCore>
             </vCpuInfo>
             <memoryInfo>
-                <sizeInMiB>{{ instance_type.memory|int }}</sizeInMiB>
+                <sizeInMiB>{{ instance_type.get('MemoryInfo', {}).get('SizeInMiB', 0)|int }}</sizeInMiB>
             </memoryInfo>
             <instanceStorageInfo>
-                <totalSizeInGB>{{ instance_type.storage|int }}</totalSizeInGB>
+                <totalSizeInGB>{{ instance_type.get('InstanceStorageInfo', {}).get('TotalSizeInGB', 0)|int }}</totalSizeInGB>
             </instanceStorageInfo>
             <processorInfo>
                 <supportedArchitectures>
+                    {% for arch in instance_type.get('ProcessorInfo', {}).get('SupportedArchitectures', []) %}
                     <item>
-                        x86_64
+                        {{ arch }}
                     </item>
+                    {% endfor %}
                 </supportedArchitectures>
             </processorInfo>
         </item>

requirements-dev.txt

@@ -10,5 +10,4 @@ prompt-toolkit==2.0.10 # 3.x is not available with python2
 click==6.7
 inflection==0.3.1
 lxml==4.2.3
-beautifulsoup4==4.6.0
 

scripts/get_instance_info.py

@@ -3,157 +3,43 @@
 import json
 import os
 import subprocess
-import requests
-from bs4 import BeautifulSoup
 
+from time import sleep
 
-class Instance(object):
-    def __init__(self, instance):
-        self.instance = instance
-
-    def _get_td(self, td):
-        return self.instance.find("td", attrs={"class": td})
-
-    def _get_sort(self, td):
-        return float(self.instance.find("td", attrs={"class": td}).find("span")["sort"])
-
-    @property
-    def name(self):
-        return self._get_td("name").text.strip()
-
-    @property
-    def apiname(self):
-        return self._get_td("apiname").text.strip()
-
-    @property
-    def memory(self):
-        return self._get_sort("memory")
-
-    @property
-    def computeunits(self):
-        return self._get_sort("computeunits")
-
-    @property
-    def vcpus(self):
-        return self._get_sort("vcpus")
-
-    @property
-    def gpus(self):
-        return int(self._get_td("gpus").text.strip())
-
-    @property
-    def fpga(self):
-        return int(self._get_td("fpga").text.strip())
-
-    @property
-    def ecu_per_vcpu(self):
-        return self._get_sort("ecu-per-vcpu")
-
-    @property
-    def physical_processor(self):
-        return self._get_td("physical_processor").text.strip()
-
-    @property
-    def clock_speed_ghz(self):
-        return self._get_td("clock_speed_ghz").text.strip()
-
-    @property
-    def intel_avx(self):
-        return self._get_td("intel_avx").text.strip()
-
-    @property
-    def intel_avx2(self):
-        return self._get_td("intel_avx2").text.strip()
-
-    @property
-    def intel_turbo(self):
-        return self._get_td("intel_turbo").text.strip()
-
-    @property
-    def storage(self):
-        return self._get_sort("storage")
-
-    @property
-    def architecture(self):
-        return self._get_td("architecture").text.strip()
-
-    @property
-    def network_perf(self):  # 2 == low
-        return self._get_sort("networkperf")
-
-    @property
-    def ebs_max_bandwidth(self):
-        return self._get_sort("ebs-max-bandwidth")
-
-    @property
-    def ebs_throughput(self):
-        return self._get_sort("ebs-throughput")
-
-    @property
-    def ebs_iops(self):
-        return self._get_sort("ebs-iops")
-
-    @property
-    def max_ips(self):
-        return int(self._get_td("maxips").text.strip())
-
-    @property
-    def enhanced_networking(self):
-        return self._get_td("enhanced-networking").text.strip() != "No"
-
-    @property
-    def vpc_only(self):
-        return self._get_td("vpc-only").text.strip() != "No"
-
-    @property
-    def ipv6_support(self):
-        return self._get_td("ipv6-support").text.strip() != "No"
-
-    @property
-    def placement_group_support(self):
-        return self._get_td("placement-group-support").text.strip() != "No"
-
-    @property
-    def linux_virtualization(self):
-        return self._get_td("linux-virtualization").text.strip()
-
-    def to_dict(self):
-        result = {}
-
-        for attr in [
-            x
-            for x in self.__class__.__dict__.keys()
-            if not x.startswith("_") and x != "to_dict"
-        ]:
-            try:
-                result[attr] = getattr(self, attr)
-            except ValueError as ex:
-                if "'N/A'" in str(ex):
-                    print(
-                        "Skipping attribute '{0}' for instance type '{1}' (not found)".format(
-                            attr, self.name
-                        )
-                    )
-                else:
-                    raise
-
-        return self.apiname, result
+import boto3
+from boto3 import Session
 
 
 def main():
-    print("Getting HTML from http://www.ec2instances.info")
-    page_request = requests.get("http://www.ec2instances.info")
-    soup = BeautifulSoup(page_request.text, "html.parser")
-    data_table = soup.find(id="data")
+    print("Getting InstanceTypes from all regions")
+    regions = []
+    regions.extend(Session().get_available_regions("ec2"))
+    regions.extend(Session().get_available_regions("ec2", partition_name="aws-us-gov"))
+    regions.extend(Session().get_available_regions("ec2", partition_name="aws-cn"))
+    print("Found " + str(len(regions)) + " regions")
 
-    print("Finding data in table")
-    instances = data_table.find("tbody").find_all("tr")
+    instances = []
+    for region in regions:
+        try:
+            ec2 = boto3.client("ec2", region_name=region)
+            offerings = ec2.describe_instance_types()
+            instances.extend(offerings["InstanceTypes"])
+            next_token = offerings.get("NextToken", "")
+            while next_token:
+                offerings = ec2.describe_instance_types(
+                    NextToken=next_token
+                )
+                instances.extend(offerings["InstanceTypes"])
+                next_token = offerings.get("NextToken", None)
+        except Exception:
+            print("Could not fetch instance types from region:", region)
+        # We don't want it to look like we're DDOS'ing AWS
+        sleep(1)
 
     print("Parsing data")
     result = {}
     for instance in instances:
-        instance_id, instance_data = Instance(instance).to_dict()
-        result[instance_id] = instance_data
+        result[instance.get('InstanceType')] = instance
 
     root_dir = (
         subprocess.check_output(["git", "rev-parse", "--show-toplevel"])