From 4a402b01cfdd8ffe8947e384471bd7a31faa6d86 Mon Sep 17 00:00:00 2001 From: mickeypash Date: Mon, 23 Oct 2017 19:12:49 +0100 Subject: [PATCH 01/83] Correct response when trying to delete a volume that is attached to an EC2 instance. Created a VolumeInUse error and did a simple check on the delete_volume method. --- moto/ec2/exceptions.py | 9 +++++++++ moto/ec2/models.py | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/moto/ec2/exceptions.py b/moto/ec2/exceptions.py index e5432baf7..ae279d5b2 100644 --- a/moto/ec2/exceptions.py +++ b/moto/ec2/exceptions.py @@ -244,6 +244,15 @@ class InvalidVolumeAttachmentError(EC2ClientError): .format(volume_id, instance_id)) +class VolumeInUseError(EC2ClientError): + + def __init__(self, volume_id, instance_id): + super(VolumeInUseError, self).__init__( + "VolumeInUse", + "Volume {0} is currently attached to {1}" + .format(volume_id, instance_id)) + + class InvalidDomainError(EC2ClientError): def __init__(self, domain): diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 7fa7e1009..011258520 100755 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -45,6 +45,7 @@ from .exceptions import ( InvalidAMIAttributeItemValueError, InvalidSnapshotIdError, InvalidVolumeIdError, + VolumeInUseError, InvalidVolumeAttachmentError, InvalidDomainError, InvalidAddressError, @@ -1813,6 +1814,10 @@ class EBSBackend(object): def delete_volume(self, volume_id): if volume_id in self.volumes: + volume = self.volumes[volume_id] + instance_id = volume.attachment.instance.id + if volume.attachment is not None: + raise VolumeInUseError(volume_id, instance_id) return self.volumes.pop(volume_id) raise InvalidVolumeIdError(volume_id) From d5b841fb6c4e8fdb4c94bd9becc74441a96ec6da Mon Sep 17 00:00:00 2001 From: mickeypash Date: Mon, 13 Nov 2017 19:58:21 +0000 Subject: [PATCH 02/83] Fixing volume.attachment is None --- moto/ec2/models.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index bad32d653..b9cbe0407 100755 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -1812,9 +1812,8 @@ class EBSBackend(object): def delete_volume(self, volume_id): if volume_id in self.volumes: volume = self.volumes[volume_id] - instance_id = volume.attachment.instance.id if volume.attachment is not None: - raise VolumeInUseError(volume_id, instance_id) + raise VolumeInUseError(volume_id, volume.attachment.instance.id) return self.volumes.pop(volume_id) raise InvalidVolumeIdError(volume_id) From 41b1482b595cef56ff9b0b49380a86555cb7e6cc Mon Sep 17 00:00:00 2001 From: mickeypash Date: Sat, 20 Jul 2019 21:36:21 +0100 Subject: [PATCH 03/83] Simplify conditional --- moto/ec2/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 79838147e..f91835581 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -2064,7 +2064,7 @@ class EBSBackend(object): def delete_volume(self, volume_id): if volume_id in self.volumes: volume = self.volumes[volume_id] - if volume.attachment is not None: + if volume.attachment: raise VolumeInUseError(volume_id, volume.attachment.instance.id) return self.volumes.pop(volume_id) raise InvalidVolumeIdError(volume_id) From b1da99aedaee0d7db7c8d68dfd477440705648e7 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Fri, 20 Mar 2020 12:29:04 +0000 Subject: [PATCH 04/83] #2797 - DynamoDB - Allow case insensitive AND in KeyConditionExpression --- moto/dynamodb2/responses.py | 6 ++++-- tests/test_dynamodb2/test_dynamodb.py | 9 ++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index c72ded2c3..c13078a72 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -459,8 +459,10 @@ class DynamoHandler(BaseResponse): for k, v in six.iteritems(self.body.get("ExpressionAttributeNames", {})) ) - if " AND " in key_condition_expression: - expressions = key_condition_expression.split(" AND ", 1) + if " and " in key_condition_expression.lower(): + expressions = re.split( + " AND ", key_condition_expression, maxsplit=1, flags=re.IGNORECASE + ) index_hash_key = [key for key in index if key["KeyType"] == "HASH"][0] hash_key_var = reverse_attribute_lookup.get( diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 062208863..191f19c36 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1408,6 +1408,13 @@ def test_filter_expression(): filter_expr.expr(row1).should.be(True) filter_expr.expr(row2).should.be(False) + # lowercase AND test + filter_expr = moto.dynamodb2.comparisons.get_filter_expression( + "Id > :v0 and Subs < :v1", {}, {":v0": {"N": "5"}, ":v1": {"N": "7"}} + ) + filter_expr.expr(row1).should.be(True) + filter_expr.expr(row2).should.be(False) + # OR test filter_expr = moto.dynamodb2.comparisons.get_filter_expression( "Id = :v0 OR Id=:v1", {}, {":v0": {"N": "5"}, ":v1": {"N": "8"}} @@ -2719,7 +2726,7 @@ def test_query_gsi_with_range_key(): res = dynamodb.query( TableName="test", IndexName="test_gsi", - KeyConditionExpression="gsi_hash_key = :gsi_hash_key AND gsi_range_key = :gsi_range_key", + KeyConditionExpression="gsi_hash_key = :gsi_hash_key and gsi_range_key = :gsi_range_key", ExpressionAttributeValues={ ":gsi_hash_key": {"S": "key1"}, ":gsi_range_key": {"S": "range1"}, From 5b596c8a78ffc0c5a6ee1b9b28629c3f04bd5396 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Fri, 20 Mar 2020 15:17:55 +0000 Subject: [PATCH 05/83] #2699 - EC2 - Add Volumes using CloudFormation --- moto/ec2/models.py | 14 ++++++++++- tests/test_ec2/test_instances.py | 40 +++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index be39bab28..1b363a193 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -556,6 +556,10 @@ class Instance(TaggedEC2Resource, BotoInstance): # worst case we'll get IP address exaustion... rarely pass + def add_block_device(self, size, device_path): + volume = self.ec2_backend.create_volume(size, self.region_name) + self.ec2_backend.attach_volume(volume.id, self.id, device_path) + def setup_defaults(self): # Default have an instance with root volume should you not wish to # override with attach volume cmd. @@ -620,6 +624,7 @@ class Instance(TaggedEC2Resource, BotoInstance): subnet_id=properties.get("SubnetId"), key_name=properties.get("KeyName"), private_ip=properties.get("PrivateIpAddress"), + block_device_mappings=properties.get("BlockDeviceMappings", {}), ) instance = reservation.instances[0] for tag in properties.get("Tags", []): @@ -872,7 +877,14 @@ class InstanceBackend(object): ) new_reservation.instances.append(new_instance) new_instance.add_tags(instance_tags) - new_instance.setup_defaults() + if "block_device_mappings" in kwargs: + for block_device in kwargs["block_device_mappings"]: + new_instance.add_block_device( + block_device["Ebs"]["VolumeSize"], block_device["DeviceName"] + ) + else: + new_instance.setup_defaults() + return new_reservation def start_instances(self, instance_ids): diff --git a/tests/test_ec2/test_instances.py b/tests/test_ec2/test_instances.py index 85ba0fe01..4d1cbb28d 100644 --- a/tests/test_ec2/test_instances.py +++ b/tests/test_ec2/test_instances.py @@ -9,6 +9,7 @@ from nose.tools import assert_raises import base64 import datetime import ipaddress +import json import six import boto @@ -18,7 +19,7 @@ from boto.exception import EC2ResponseError, EC2ResponseError from freezegun import freeze_time import sure # noqa -from moto import mock_ec2_deprecated, mock_ec2 +from moto import mock_ec2_deprecated, mock_ec2, mock_cloudformation from tests.helpers import requires_boto_gte @@ -1399,3 +1400,40 @@ def test_describe_instance_attribute(): invalid_instance_attribute=invalid_instance_attribute ) ex.exception.response["Error"]["Message"].should.equal(message) + + +@mock_ec2 +@mock_cloudformation +def test_volume_size_through_cloudformation(): + ec2 = boto3.client("ec2", region_name="us-east-1") + cf = boto3.client("cloudformation", region_name="us-east-1") + + volume_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "testInstance": { + "Type": "AWS::EC2::Instance", + "Properties": { + "ImageId": "ami-d3adb33f", + "KeyName": "dummy", + "InstanceType": "t2.micro", + "BlockDeviceMappings": [ + {"DeviceName": "/dev/sda2", "Ebs": {"VolumeSize": "50"}} + ], + "Tags": [ + {"Key": "foo", "Value": "bar"}, + {"Key": "blah", "Value": "baz"}, + ], + }, + } + }, + } + template_json = json.dumps(volume_template) + cf.create_stack(StackName="test_stack", TemplateBody=template_json) + instances = ec2.describe_instances() + volume = instances["Reservations"][0]["Instances"][0]["BlockDeviceMappings"][0][ + "Ebs" + ] + + volumes = ec2.describe_volumes(VolumeIds=[volume["VolumeId"]]) + volumes["Volumes"][0]["Size"].should.equal(50) From da1a2118bb12ca3279952d88154ed221f9f0fd1e Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Fri, 20 Mar 2020 16:17:21 +0000 Subject: [PATCH 06/83] EC2 - Verify default block exists before tearing down --- moto/ec2/models.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 1b363a193..c391c88f3 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -567,9 +567,10 @@ class Instance(TaggedEC2Resource, BotoInstance): self.ec2_backend.attach_volume(volume.id, self.id, "/dev/sda1") def teardown_defaults(self): - volume_id = self.block_device_mapping["/dev/sda1"].volume_id - self.ec2_backend.detach_volume(volume_id, self.id, "/dev/sda1") - self.ec2_backend.delete_volume(volume_id) + if "/dev/sda1" in self.block_device_mapping: + volume_id = self.block_device_mapping["/dev/sda1"].volume_id + self.ec2_backend.detach_volume(volume_id, self.id, "/dev/sda1") + self.ec2_backend.delete_volume(volume_id) @property def get_block_device_mapping(self): From e82e1e3f397cd610d3ed0316c37325cdfe55926b Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Sat, 21 Mar 2020 12:20:09 +0000 Subject: [PATCH 07/83] DynamoDB - Add 1MB item size check --- moto/dynamodb2/models.py | 11 ++++++++ tests/test_dynamodb2/test_dynamodb.py | 38 +++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 54dccd56d..a35eded61 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -285,6 +285,9 @@ class Item(BaseModel): def __repr__(self): return "Item: {0}".format(self.to_json()) + def size(self): + return sum([bytesize(key) + value.size() for key, value in self.attrs.items()]) + def to_json(self): attributes = {} for attribute_key, attribute in self.attrs.items(): @@ -1123,6 +1126,14 @@ class Table(BaseModel): break last_evaluated_key = None + size_limit = 1000000 # DynamoDB has a 1MB size limit + item_size = sum([res.size() for res in results]) + if item_size > size_limit: + item_size = idx = 0 + while item_size + results[idx].size() < size_limit: + item_size += results[idx].size() + idx += 1 + limit = min(limit, idx) if limit else idx if limit and len(results) > limit: results = results[:limit] last_evaluated_key = {self.hash_key_attr: results[-1].hash_key} diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 062208863..daae79232 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -4132,3 +4132,41 @@ def test_gsi_verify_negative_number_order(): [float(item["gsiK1SortKey"]) for item in resp["Items"]].should.equal( [-0.7, -0.6, 0.7] ) + + +@mock_dynamodb2 +def test_dynamodb_max_1mb_limit(): + ddb = boto3.resource("dynamodb", region_name="eu-west-1") + + table_name = "populated-mock-table" + table = ddb.create_table( + TableName=table_name, + KeySchema=[ + {"AttributeName": "partition_key", "KeyType": "HASH"}, + {"AttributeName": "sort_key", "KeyType": "SORT"}, + ], + AttributeDefinitions=[ + {"AttributeName": "partition_key", "AttributeType": "S"}, + {"AttributeName": "sort_key", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + # Populate the table + items = [ + { + "partition_key": "partition_key_val", # size=30 + "sort_key": "sort_key_value____" + str(i), # size=30 + } + for i in range(10000, 29999) + ] + with table.batch_writer() as batch: + for item in items: + batch.put_item(Item=item) + + response = table.query( + KeyConditionExpression=Key("partition_key").eq("partition_key_val") + ) + # We shouldn't get everything back - the total result set is well over 1MB + assert response["Count"] < len(items) + response["LastEvaluatedKey"].shouldnt.be(None) From c3865532f9ca6237261591277bde1afbe910099e Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Mon, 23 Mar 2020 15:53:39 +0000 Subject: [PATCH 08/83] #2711 - Register default S3 metrics in CloudWatch --- moto/cloudwatch/models.py | 50 ++++++++++++++++--- moto/cloudwatch/responses.py | 5 +- moto/s3/models.py | 33 ++++++++++++ moto/s3/utils.py | 6 +++ tests/test_cloudwatch/test_cloudwatch.py | 40 +++++++++++++-- .../test_cloudwatch/test_cloudwatch_boto3.py | 16 ++++-- 6 files changed, 133 insertions(+), 17 deletions(-) diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py index a8a1b1d19..523eb10f3 100644 --- a/moto/cloudwatch/models.py +++ b/moto/cloudwatch/models.py @@ -22,6 +22,14 @@ class Dimension(object): self.name = name self.value = value + def __eq__(self, item): + if isinstance(item, Dimension): + return self.name == item.name and self.value == item.value + return False + + def __ne__(self, item): # Only needed on Py2; Py3 defines it implicitly + return self != item + def daterange(start, stop, step=timedelta(days=1), inclusive=False): """ @@ -124,6 +132,17 @@ class MetricDatum(BaseModel): Dimension(dimension["Name"], dimension["Value"]) for dimension in dimensions ] + def filter(self, namespace, name, dimensions): + if namespace and namespace != self.namespace: + return False + if name and name != self.name: + return False + if dimensions and any( + Dimension(d["Name"], d["Value"]) not in self.dimensions for d in dimensions + ): + return False + return True + class Dashboard(BaseModel): def __init__(self, name, body): @@ -202,6 +221,15 @@ class CloudWatchBackend(BaseBackend): self.metric_data = [] self.paged_metric_data = {} + @property + # Retrieve a list of all OOTB metrics that are provided by metrics providers + # Computed on the fly + def aws_metric_data(self): + md = [] + for name, service in metric_providers.items(): + md.extend(service.get_cloudwatch_metrics()) + return md + def put_metric_alarm( self, name, @@ -334,7 +362,7 @@ class CloudWatchBackend(BaseBackend): return data def get_all_metrics(self): - return self.metric_data + return self.metric_data + self.aws_metric_data def put_dashboard(self, name, body): self.dashboards[name] = Dashboard(name, body) @@ -386,7 +414,7 @@ class CloudWatchBackend(BaseBackend): self.alarms[alarm_name].update_state(reason, reason_data, state_value) - def list_metrics(self, next_token, namespace, metric_name): + def list_metrics(self, next_token, namespace, metric_name, dimensions): if next_token: if next_token not in self.paged_metric_data: raise RESTError( @@ -397,15 +425,16 @@ class CloudWatchBackend(BaseBackend): del self.paged_metric_data[next_token] # Cant reuse same token twice return self._get_paginated(metrics) else: - metrics = self.get_filtered_metrics(metric_name, namespace) + metrics = self.get_filtered_metrics(metric_name, namespace, dimensions) return self._get_paginated(metrics) - def get_filtered_metrics(self, metric_name, namespace): + def get_filtered_metrics(self, metric_name, namespace, dimensions): metrics = self.get_all_metrics() - if namespace: - metrics = [md for md in metrics if md.namespace == namespace] - if metric_name: - metrics = [md for md in metrics if md.name == metric_name] + metrics = [ + md + for md in metrics + if md.filter(namespace=namespace, name=metric_name, dimensions=dimensions) + ] return metrics def _get_paginated(self, metrics): @@ -443,3 +472,8 @@ for region in Session().get_available_regions( cloudwatch_backends[region] = CloudWatchBackend() for region in Session().get_available_regions("cloudwatch", partition_name="aws-cn"): cloudwatch_backends[region] = CloudWatchBackend() + +# List of services that provide OOTB CW metrics +# See the S3Backend constructor for an example +# TODO: We might have to separate this out per region for non-global services +metric_providers = {} diff --git a/moto/cloudwatch/responses.py b/moto/cloudwatch/responses.py index 7993c9f06..dccc30216 100644 --- a/moto/cloudwatch/responses.py +++ b/moto/cloudwatch/responses.py @@ -124,9 +124,10 @@ class CloudWatchResponse(BaseResponse): def list_metrics(self): namespace = self._get_param("Namespace") metric_name = self._get_param("MetricName") + dimensions = self._get_multi_param("Dimensions.member") next_token = self._get_param("NextToken") next_token, metrics = self.cloudwatch_backend.list_metrics( - next_token, namespace, metric_name + next_token, namespace, metric_name, dimensions ) template = self.response_template(LIST_METRICS_TEMPLATE) return template.render(metrics=metrics, next_token=next_token) @@ -342,7 +343,7 @@ LIST_METRICS_TEMPLATE = """ Date: Sat, 28 Mar 2020 13:41:17 +0000 Subject: [PATCH 09/83] #2239 - Initial implementation of CW.get_metric_data --- moto/cloudwatch/models.py | 37 +++ moto/cloudwatch/responses.py | 41 +++ .../test_cloudwatch/test_cloudwatch_boto3.py | 259 ++++++++++++++++++ 3 files changed, 337 insertions(+) diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py index a8a1b1d19..bddb94a12 100644 --- a/moto/cloudwatch/models.py +++ b/moto/cloudwatch/models.py @@ -295,6 +295,43 @@ class CloudWatchBackend(BaseBackend): ) ) + def get_metric_data(self, queries, start_time, end_time): + period_data = [ + md for md in self.metric_data if start_time <= md.timestamp <= end_time + ] + results = [] + for query in queries: + query_ns = query["metric_stat._metric._namespace"] + query_name = query["metric_stat._metric._metric_name"] + query_data = [ + md + for md in period_data + if md.namespace == query_ns and md.name == query_name + ] + metric_values = [m.value for m in query_data] + result_vals = [] + stat = query["metric_stat._stat"] + if len(metric_values) > 0: + if stat == "Average": + result_vals.append(sum(metric_values) / len(metric_values)) + elif stat == "Minimum": + result_vals.append(min(metric_values)) + elif stat == "Maximum": + result_vals.append(max(metric_values)) + elif stat == "Sum": + result_vals.append(sum(metric_values)) + + label = query["metric_stat._metric._metric_name"] + " " + stat + results.append( + { + "id": query["id"], + "label": label, + "vals": result_vals, + "timestamps": [datetime.now() for _ in result_vals], + } + ) + return results + def get_metric_statistics( self, namespace, metric_name, start_time, end_time, period, stats ): diff --git a/moto/cloudwatch/responses.py b/moto/cloudwatch/responses.py index 7993c9f06..7e75a38f0 100644 --- a/moto/cloudwatch/responses.py +++ b/moto/cloudwatch/responses.py @@ -92,6 +92,18 @@ class CloudWatchResponse(BaseResponse): template = self.response_template(PUT_METRIC_DATA_TEMPLATE) return template.render() + @amzn_request_id + def get_metric_data(self): + start = dtparse(self._get_param("StartTime")) + end = dtparse(self._get_param("EndTime")) + queries = self._get_list_prefix("MetricDataQueries.member") + results = self.cloudwatch_backend.get_metric_data( + start_time=start, end_time=end, queries=queries + ) + + template = self.response_template(GET_METRIC_DATA_TEMPLATE) + return template.render(results=results) + @amzn_request_id def get_metric_statistics(self): namespace = self._get_param("Namespace") @@ -285,6 +297,35 @@ PUT_METRIC_DATA_TEMPLATE = """ + + + {{ request_id }} + + + + + {% for result in results %} + + {{ result.id }} + + Complete + + {% for val in result.timestamps %} + {{ val }} + {% endfor %} + + + {% for val in result.vals %} + {{ val }} + {% endfor %} + + + {% endfor %} + + +""" + GET_METRIC_STATISTICS_TEMPLATE = """ diff --git a/tests/test_cloudwatch/test_cloudwatch_boto3.py b/tests/test_cloudwatch/test_cloudwatch_boto3.py index 7fe144052..2b1caff02 100644 --- a/tests/test_cloudwatch/test_cloudwatch_boto3.py +++ b/tests/test_cloudwatch/test_cloudwatch_boto3.py @@ -3,6 +3,7 @@ import boto3 from botocore.exceptions import ClientError from datetime import datetime, timedelta +from freezegun import freeze_time from nose.tools import assert_raises from uuid import uuid4 import pytz @@ -211,6 +212,35 @@ def test_get_metric_statistics(): datapoint["Sum"].should.equal(1.5) +@mock_cloudwatch +@freeze_time("2020-02-10 18:44:05") +def test_custom_timestamp(): + utc_now = datetime.now(tz=pytz.utc) + time = "2020-02-10T18:44:09Z" + cw = boto3.client("cloudwatch", "eu-west-1") + + cw.put_metric_data( + Namespace="tester", + MetricData=[dict(MetricName="metric1", Value=1.5, Timestamp=time)], + ) + + cw.put_metric_data( + Namespace="tester", + MetricData=[ + dict(MetricName="metric2", Value=1.5, Timestamp=datetime(2020, 2, 10)) + ], + ) + + stats = cw.get_metric_statistics( + Namespace="tester", + MetricName="metric", + StartTime=utc_now - timedelta(seconds=60), + EndTime=utc_now + timedelta(seconds=60), + Period=60, + Statistics=["SampleCount", "Sum"], + ) + + @mock_cloudwatch def test_list_metrics(): cloudwatch = boto3.client("cloudwatch", "eu-west-1") @@ -292,3 +322,232 @@ def create_metrics(cloudwatch, namespace, metrics=5, data_points=5): Namespace=namespace, MetricData=[{"MetricName": metric_name, "Value": j, "Unit": "Seconds"}], ) + + +@mock_cloudwatch +def test_get_metric_data_within_timeframe(): + utc_now = datetime.now(tz=pytz.utc) + cloudwatch = boto3.client("cloudwatch", "eu-west-1") + namespace1 = "my_namespace/" + # put metric data + values = [0, 2, 4, 3.5, 7, 100] + cloudwatch.put_metric_data( + Namespace=namespace1, + MetricData=[ + {"MetricName": "metric1", "Value": val, "Unit": "Seconds"} for val in values + ], + ) + # get_metric_data + stats = ["Average", "Sum", "Minimum", "Maximum"] + response = cloudwatch.get_metric_data( + MetricDataQueries=[ + { + "Id": "result_" + stat, + "MetricStat": { + "Metric": {"Namespace": namespace1, "MetricName": "metric1"}, + "Period": 60, + "Stat": stat, + }, + } + for stat in stats + ], + StartTime=utc_now - timedelta(seconds=60), + EndTime=utc_now + timedelta(seconds=60), + ) + # + # Assert Average/Min/Max/Sum is returned as expected + avg = [ + res for res in response["MetricDataResults"] if res["Id"] == "result_Average" + ][0] + avg["Label"].should.equal("metric1 Average") + avg["StatusCode"].should.equal("Complete") + [int(val) for val in avg["Values"]].should.equal([19]) + + sum_ = [res for res in response["MetricDataResults"] if res["Id"] == "result_Sum"][ + 0 + ] + sum_["Label"].should.equal("metric1 Sum") + sum_["StatusCode"].should.equal("Complete") + [val for val in sum_["Values"]].should.equal([sum(values)]) + + min_ = [ + res for res in response["MetricDataResults"] if res["Id"] == "result_Minimum" + ][0] + min_["Label"].should.equal("metric1 Minimum") + min_["StatusCode"].should.equal("Complete") + [int(val) for val in min_["Values"]].should.equal([0]) + + max_ = [ + res for res in response["MetricDataResults"] if res["Id"] == "result_Maximum" + ][0] + max_["Label"].should.equal("metric1 Maximum") + max_["StatusCode"].should.equal("Complete") + [int(val) for val in max_["Values"]].should.equal([100]) + + +@mock_cloudwatch +def test_get_metric_data_partially_within_timeframe(): + utc_now = datetime.now(tz=pytz.utc) + yesterday = utc_now - timedelta(days=1) + last_week = utc_now - timedelta(days=7) + cloudwatch = boto3.client("cloudwatch", "eu-west-1") + namespace1 = "my_namespace/" + # put metric data + values = [0, 2, 4, 3.5, 7, 100] + cloudwatch.put_metric_data( + Namespace=namespace1, + MetricData=[ + { + "MetricName": "metric1", + "Value": 10, + "Unit": "Seconds", + "Timestamp": utc_now, + } + ], + ) + cloudwatch.put_metric_data( + Namespace=namespace1, + MetricData=[ + { + "MetricName": "metric1", + "Value": 20, + "Unit": "Seconds", + "Timestamp": yesterday, + } + ], + ) + cloudwatch.put_metric_data( + Namespace=namespace1, + MetricData=[ + { + "MetricName": "metric1", + "Value": 50, + "Unit": "Seconds", + "Timestamp": last_week, + } + ], + ) + # get_metric_data + response = cloudwatch.get_metric_data( + MetricDataQueries=[ + { + "Id": "result", + "MetricStat": { + "Metric": {"Namespace": namespace1, "MetricName": "metric1"}, + "Period": 60, + "Stat": "Sum", + }, + } + ], + StartTime=yesterday - timedelta(seconds=60), + EndTime=utc_now + timedelta(seconds=60), + ) + # + # Assert Last week's data is not returned + len(response["MetricDataResults"]).should.equal(1) + sum_ = response["MetricDataResults"][0] + sum_["Label"].should.equal("metric1 Sum") + sum_["StatusCode"].should.equal("Complete") + sum_["Values"].should.equal([30.0]) + + +@mock_cloudwatch +def test_get_metric_data_outside_timeframe(): + utc_now = datetime.now(tz=pytz.utc) + last_week = utc_now - timedelta(days=7) + cloudwatch = boto3.client("cloudwatch", "eu-west-1") + namespace1 = "my_namespace/" + # put metric data + cloudwatch.put_metric_data( + Namespace=namespace1, + MetricData=[ + { + "MetricName": "metric1", + "Value": 50, + "Unit": "Seconds", + "Timestamp": last_week, + } + ], + ) + # get_metric_data + response = cloudwatch.get_metric_data( + MetricDataQueries=[ + { + "Id": "result", + "MetricStat": { + "Metric": {"Namespace": namespace1, "MetricName": "metric1"}, + "Period": 60, + "Stat": "Sum", + }, + } + ], + StartTime=utc_now - timedelta(seconds=60), + EndTime=utc_now + timedelta(seconds=60), + ) + # + # Assert Last week's data is not returned + len(response["MetricDataResults"]).should.equal(1) + response["MetricDataResults"][0]["Id"].should.equal("result") + response["MetricDataResults"][0]["StatusCode"].should.equal("Complete") + response["MetricDataResults"][0]["Values"].should.equal([]) + + +@mock_cloudwatch +def test_get_metric_data_for_multiple_metrics(): + utc_now = datetime.now(tz=pytz.utc) + cloudwatch = boto3.client("cloudwatch", "eu-west-1") + namespace = "my_namespace/" + # put metric data + cloudwatch.put_metric_data( + Namespace=namespace, + MetricData=[ + { + "MetricName": "metric1", + "Value": 50, + "Unit": "Seconds", + "Timestamp": utc_now, + } + ], + ) + cloudwatch.put_metric_data( + Namespace=namespace, + MetricData=[ + { + "MetricName": "metric2", + "Value": 25, + "Unit": "Seconds", + "Timestamp": utc_now, + } + ], + ) + # get_metric_data + response = cloudwatch.get_metric_data( + MetricDataQueries=[ + { + "Id": "result1", + "MetricStat": { + "Metric": {"Namespace": namespace, "MetricName": "metric1"}, + "Period": 60, + "Stat": "Sum", + }, + }, + { + "Id": "result2", + "MetricStat": { + "Metric": {"Namespace": namespace, "MetricName": "metric2"}, + "Period": 60, + "Stat": "Sum", + }, + }, + ], + StartTime=utc_now - timedelta(seconds=60), + EndTime=utc_now + timedelta(seconds=60), + ) + # + len(response["MetricDataResults"]).should.equal(2) + + res1 = [res for res in response["MetricDataResults"] if res["Id"] == "result1"][0] + res1["Values"].should.equal([50.0]) + + res2 = [res for res in response["MetricDataResults"] if res["Id"] == "result2"][0] + res2["Values"].should.equal([25.0]) From af08d71310862727669cb1a5041df64472857191 Mon Sep 17 00:00:00 2001 From: Guilherme Martins Crocetti Date: Wed, 1 Apr 2020 21:57:46 -0300 Subject: [PATCH 10/83] add support for RetentionInDays for LogGroup --- moto/cloudwatch/models.py | 2 +- moto/logs/models.py | 28 +++++++++---------- .../test_cloudformation_stack_integration.py | 1 + tests/test_logs/test_logs.py | 13 ++++----- 4 files changed, 20 insertions(+), 24 deletions(-) diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py index a8a1b1d19..4cd4df156 100644 --- a/moto/cloudwatch/models.py +++ b/moto/cloudwatch/models.py @@ -431,7 +431,7 @@ class LogGroup(BaseModel): properties = cloudformation_json["Properties"] log_group_name = properties["LogGroupName"] tags = properties.get("Tags", {}) - return logs_backends[region_name].create_log_group(log_group_name, tags) + return logs_backends[region_name].create_log_group(log_group_name, tags, **properties) cloudwatch_backends = {} diff --git a/moto/logs/models.py b/moto/logs/models.py index 5e21d8793..755605734 100644 --- a/moto/logs/models.py +++ b/moto/logs/models.py @@ -134,7 +134,7 @@ class LogStream: return None, 0 events = sorted( - filter(filter_func, self.events), key=lambda event: event.timestamp, + filter(filter_func, self.events), key=lambda event: event.timestamp ) direction, index = get_index_and_direction_from_token(next_token) @@ -169,11 +169,7 @@ class LogStream: if end_index > final_index: end_index = final_index elif end_index < 0: - return ( - [], - "b/{:056d}".format(0), - "f/{:056d}".format(0), - ) + return ([], "b/{:056d}".format(0), "f/{:056d}".format(0)) events_page = [ event.to_response_dict() for event in events[start_index : end_index + 1] @@ -219,7 +215,7 @@ class LogStream: class LogGroup: - def __init__(self, region, name, tags): + def __init__(self, region, name, tags, **kwargs): self.name = name self.region = region self.arn = "arn:aws:logs:{region}:1:log-group:{log_group}".format( @@ -228,9 +224,9 @@ class LogGroup: self.creationTime = int(unix_time_millis()) self.tags = tags self.streams = dict() # {name: LogStream} - self.retentionInDays = ( - None # AWS defaults to Never Expire for log group retention - ) + self.retention_in_days = kwargs.get( + "RetentionInDays" + ) # AWS defaults to Never Expire for log group retention def create_log_stream(self, log_stream_name): if log_stream_name in self.streams: @@ -368,12 +364,12 @@ class LogGroup: "storedBytes": sum(s.storedBytes for s in self.streams.values()), } # AWS only returns retentionInDays if a value is set for the log group (ie. not Never Expire) - if self.retentionInDays: - log_group["retentionInDays"] = self.retentionInDays + if self.retention_in_days: + log_group["retentionInDays"] = self.retention_in_days return log_group def set_retention_policy(self, retention_in_days): - self.retentionInDays = retention_in_days + self.retention_in_days = retention_in_days def list_tags(self): return self.tags if self.tags else {} @@ -401,10 +397,12 @@ class LogsBackend(BaseBackend): self.__dict__ = {} self.__init__(region_name) - def create_log_group(self, log_group_name, tags): + def create_log_group(self, log_group_name, tags, **kwargs): if log_group_name in self.groups: raise ResourceAlreadyExistsException() - self.groups[log_group_name] = LogGroup(self.region_name, log_group_name, tags) + self.groups[log_group_name] = LogGroup( + self.region_name, log_group_name, tags, **kwargs + ) return self.groups[log_group_name] def ensure_log_group(self, log_group_name, tags): diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index e50179660..b7fe580da 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -2380,6 +2380,7 @@ def test_create_log_group_using_fntransform(): logs_conn = boto3.client("logs", region_name="us-west-2") log_group = logs_conn.describe_log_groups()["logGroups"][0] log_group["logGroupName"].should.equal("some-log-group") + log_group["retentionInDays"].should.be.equal(90) @mock_cloudformation diff --git a/tests/test_logs/test_logs.py b/tests/test_logs/test_logs.py index e8f60ff03..2429d7e93 100644 --- a/tests/test_logs/test_logs.py +++ b/tests/test_logs/test_logs.py @@ -12,17 +12,14 @@ _logs_region = "us-east-1" if settings.TEST_SERVER_MODE else "us-west-2" @mock_logs -def test_log_group_create(): +def test_create_log_group(): conn = boto3.client("logs", "us-west-2") - log_group_name = "dummy" - response = conn.create_log_group(logGroupName=log_group_name) - response = conn.describe_log_groups(logGroupNamePrefix=log_group_name) - assert len(response["logGroups"]) == 1 - # AWS defaults to Never Expire for log group retention - assert response["logGroups"][0].get("retentionInDays") == None + response = conn.create_log_group(logGroupName="dummy") + response = conn.describe_log_groups() - response = conn.delete_log_group(logGroupName=log_group_name) + response["logGroups"].should.have.length_of(1) + response["logGroups"][0].should_not.have.key("retentionInDays") @mock_logs From c15ca133b85a228060489758ae76c75583bd4c65 Mon Sep 17 00:00:00 2001 From: Guilherme Martins Crocetti Date: Wed, 1 Apr 2020 22:00:20 -0300 Subject: [PATCH 11/83] add support for Fn::GetAtt in event's cloudformation --- moto/events/models.py | 8 ++++ .../test_cloudformation_stack_integration.py | 37 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/moto/events/models.py b/moto/events/models.py index f68b63e38..3a6f1bbc7 100644 --- a/moto/events/models.py +++ b/moto/events/models.py @@ -59,6 +59,14 @@ class Rule(BaseModel): if index is not None: self.targets.pop(index) + def get_cfn_attribute(self, attribute_name): + from moto.cloudformation.exceptions import UnformattedGetAttTemplateException + + if attribute_name == "Arn": + return self.arn + + raise UnformattedGetAttTemplateException() + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index b7fe580da..94367f1dc 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -2463,3 +2463,40 @@ def test_stack_events_create_rule_without_name_integration(): rules = boto3.client("events", "us-west-2").list_rules() rules["Rules"][0]["Name"].should.contain("test_stack-Event-") + + +@mock_cloudformation +@mock_events +@mock_logs +def test_stack_events_create_rule_as_target(): + events_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "SecurityGroup": { + "Type": "AWS::Logs::LogGroup", + "Properties": { + "LogGroupName": {"Fn::GetAtt": ["Event", "Arn"]}, + "RetentionInDays": 3, + } + }, + "Event": { + "Type": "AWS::Events::Rule", + "Properties": { + "State": "ENABLED", + "ScheduleExpression": "rate(5 minutes)", + }, + } + }, + } + cf_conn = boto3.client("cloudformation", "us-west-2") + cf_conn.create_stack( + StackName="test_stack", TemplateBody=json.dumps(events_template), + ) + + rules = boto3.client("events", "us-west-2").list_rules() + log_groups = boto3.client("logs", "us-west-2").describe_log_groups() + + rules["Rules"][0]["Name"].should.contain("test_stack-Event-") + + log_groups["logGroups"][0]["logGroupName"].should.equal(rules["Rules"][0]["Arn"]) + log_groups["logGroups"][0]["retentionInDays"].should.equal(3) From c25f6a72da03e2ee9fd939bded406ae7f56f2339 Mon Sep 17 00:00:00 2001 From: Guilherme Martins Crocetti Date: Wed, 1 Apr 2020 22:11:50 -0300 Subject: [PATCH 12/83] refactor put_rule test --- tests/test_events/test_events.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/test_events/test_events.py b/tests/test_events/test_events.py index 27006ff1b..5b4e958d6 100644 --- a/tests/test_events/test_events.py +++ b/tests/test_events/test_events.py @@ -79,13 +79,23 @@ def generate_environment(): @mock_events def test_put_rule(): client = boto3.client("events", "us-west-2") - client.list_rules()["Rules"].should.have.length_of(0) - rule_data = get_random_rule() + rule_data = { + "Name": "my-event", + "ScheduleExpression": "rate(5 minutes)", + "EventPattern": '{"source": ["test-source"]}', + } + client.put_rule(**rule_data) - client.list_rules()["Rules"].should.have.length_of(1) + rules = client.list_rules()["Rules"] + + rules.should.have.length_of(1) + rules[0]["Name"].should.equal(rule_data["Name"]) + rules[0]["ScheduleExpression"].should.equal(rule_data["ScheduleExpression"]) + rules[0]["EventPattern"].should.equal(rule_data["EventPattern"]) + rules[0]["State"].should.equal("ENABLED") @mock_events From 759107445394efe2e169935dfc6ae161f898aec4 Mon Sep 17 00:00:00 2001 From: Guilherme Martins Crocetti Date: Wed, 1 Apr 2020 22:12:17 -0300 Subject: [PATCH 13/83] add physical_resource_id support for Rule --- moto/events/models.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/moto/events/models.py b/moto/events/models.py index 3a6f1bbc7..e1224242e 100644 --- a/moto/events/models.py +++ b/moto/events/models.py @@ -26,6 +26,10 @@ class Rule(BaseModel): self.role_arn = kwargs.get("RoleArn") self.targets = [] + @property + def physical_resource_id(self): + return self.name + # This song and dance for targets is because we need order for Limits and NextTokens, but can't use OrderedDicts # with Python 2.6, so tracking it with an array it is. def _check_target_exists(self, target_id): From 503eeb51aea4391012ef022f51347a9276d010be Mon Sep 17 00:00:00 2001 From: Guilherme Martins Crocetti Date: Wed, 1 Apr 2020 22:48:40 -0300 Subject: [PATCH 14/83] style with black --- moto/cloudwatch/models.py | 4 +++- .../test_cloudformation_stack_integration.py | 16 +++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py index 4cd4df156..bc941809b 100644 --- a/moto/cloudwatch/models.py +++ b/moto/cloudwatch/models.py @@ -431,7 +431,9 @@ class LogGroup(BaseModel): properties = cloudformation_json["Properties"] log_group_name = properties["LogGroupName"] tags = properties.get("Tags", {}) - return logs_backends[region_name].create_log_group(log_group_name, tags, **properties) + return logs_backends[region_name].create_log_group( + log_group_name, tags, **properties + ) cloudwatch_backends = {} diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index 94367f1dc..c99bf16f4 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -2373,9 +2373,7 @@ def test_create_log_group_using_fntransform(): } cf_conn = boto3.client("cloudformation", "us-west-2") - cf_conn.create_stack( - StackName="test_stack", TemplateBody=json.dumps(template), - ) + cf_conn.create_stack(StackName="test_stack", TemplateBody=json.dumps(template)) logs_conn = boto3.client("logs", region_name="us-west-2") log_group = logs_conn.describe_log_groups()["logGroups"][0] @@ -2401,7 +2399,7 @@ def test_stack_events_create_rule_integration(): } cf_conn = boto3.client("cloudformation", "us-west-2") cf_conn.create_stack( - StackName="test_stack", TemplateBody=json.dumps(events_template), + StackName="test_stack", TemplateBody=json.dumps(events_template) ) rules = boto3.client("events", "us-west-2").list_rules() @@ -2429,7 +2427,7 @@ def test_stack_events_delete_rule_integration(): } cf_conn = boto3.client("cloudformation", "us-west-2") cf_conn.create_stack( - StackName="test_stack", TemplateBody=json.dumps(events_template), + StackName="test_stack", TemplateBody=json.dumps(events_template) ) rules = boto3.client("events", "us-west-2").list_rules() @@ -2458,7 +2456,7 @@ def test_stack_events_create_rule_without_name_integration(): } cf_conn = boto3.client("cloudformation", "us-west-2") cf_conn.create_stack( - StackName="test_stack", TemplateBody=json.dumps(events_template), + StackName="test_stack", TemplateBody=json.dumps(events_template) ) rules = boto3.client("events", "us-west-2").list_rules() @@ -2477,7 +2475,7 @@ def test_stack_events_create_rule_as_target(): "Properties": { "LogGroupName": {"Fn::GetAtt": ["Event", "Arn"]}, "RetentionInDays": 3, - } + }, }, "Event": { "Type": "AWS::Events::Rule", @@ -2485,12 +2483,12 @@ def test_stack_events_create_rule_as_target(): "State": "ENABLED", "ScheduleExpression": "rate(5 minutes)", }, - } + }, }, } cf_conn = boto3.client("cloudformation", "us-west-2") cf_conn.create_stack( - StackName="test_stack", TemplateBody=json.dumps(events_template), + StackName="test_stack", TemplateBody=json.dumps(events_template) ) rules = boto3.client("events", "us-west-2").list_rules() From 231b1000571c1720655989c82760116402643935 Mon Sep 17 00:00:00 2001 From: mickeypash Date: Fri, 3 Apr 2020 01:50:17 +0100 Subject: [PATCH 15/83] Add test scaffold. Currently broken --- tests/test_ec2/test_elastic_block_store.py | 38 ++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/test_ec2/test_elastic_block_store.py b/tests/test_ec2/test_elastic_block_store.py index 3c7e17ec8..0e39d2069 100644 --- a/tests/test_ec2/test_elastic_block_store.py +++ b/tests/test_ec2/test_elastic_block_store.py @@ -13,6 +13,7 @@ from freezegun import freeze_time import sure # noqa from moto import mock_ec2_deprecated, mock_ec2 +from moto.ec2.exceptions import VolumeInUseError from moto.ec2.models import OWNER_ID @@ -53,6 +54,43 @@ def test_create_and_delete_volume(): cm.exception.request_id.should_not.be.none +@mock_ec2_deprecated +def test_delete_attached_volume(): + conn = boto.ec2.connect_to_region("us-east-1") + reservation = conn.run_instances("ami-1234abcd") + # create an instance + instance = reservation.instances[0] + # create a volume + volume = conn.create_volume(80, "us-east-1a") + # attach volume to instance + volume.attach(instance.id, "/dev/sdh") + + volume.update() + volume.volume_state().should.equal("in-use") + volume.attachment_state().should.equal("attached") + + volume.attach_data.instance_id.should.equal(instance.id) + + # attempt to delete volume + # assert raises VolumeInUseError + with assert_raises(VolumeInUseError) as ex: + volume.delete() + ex.exception.error_code.should.equal("VolumeInUse") + ex.exception.status.should.equal(400) + ex.exception.message.should.equal(f"Volume {volume.id} is currently attached to {instance_id}") + + volume.detach() + + volume.update() + volume.volume_state().should.equal("available") + + volume.delete() + + all_volumes = conn.get_all_volumes() + my_volume = [item for item in all_volumes if item.id == volume.id] + my_volume.should.have.length_of(0) + + @mock_ec2_deprecated def test_create_encrypted_volume_dryrun(): conn = boto.ec2.connect_to_region("us-east-1") From 76b9cbe16d76decef8becad643c1426dba2c927d Mon Sep 17 00:00:00 2001 From: mickeypash Date: Fri, 3 Apr 2020 02:14:14 +0100 Subject: [PATCH 16/83] Fix test --- tests/test_ec2/test_elastic_block_store.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_ec2/test_elastic_block_store.py b/tests/test_ec2/test_elastic_block_store.py index 0e39d2069..1182610e8 100644 --- a/tests/test_ec2/test_elastic_block_store.py +++ b/tests/test_ec2/test_elastic_block_store.py @@ -13,7 +13,6 @@ from freezegun import freeze_time import sure # noqa from moto import mock_ec2_deprecated, mock_ec2 -from moto.ec2.exceptions import VolumeInUseError from moto.ec2.models import OWNER_ID @@ -73,11 +72,11 @@ def test_delete_attached_volume(): # attempt to delete volume # assert raises VolumeInUseError - with assert_raises(VolumeInUseError) as ex: + with assert_raises(EC2ResponseError) as ex: volume.delete() ex.exception.error_code.should.equal("VolumeInUse") ex.exception.status.should.equal(400) - ex.exception.message.should.equal(f"Volume {volume.id} is currently attached to {instance_id}") + ex.exception.message.should.equal(f"Volume {volume.id} is currently attached to {instance.id}") volume.detach() From d3367b8a90b25fa2fab323889f72717054e63d54 Mon Sep 17 00:00:00 2001 From: mickeypash Date: Fri, 3 Apr 2020 02:27:46 +0100 Subject: [PATCH 17/83] Black formatting --- moto/ec2/exceptions.py | 5 ++--- tests/test_ec2/test_elastic_block_store.py | 4 +++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/moto/ec2/exceptions.py b/moto/ec2/exceptions.py index 4df507a0d..5af4690ae 100644 --- a/moto/ec2/exceptions.py +++ b/moto/ec2/exceptions.py @@ -232,12 +232,11 @@ class InvalidVolumeAttachmentError(EC2ClientError): class VolumeInUseError(EC2ClientError): - def __init__(self, volume_id, instance_id): super(VolumeInUseError, self).__init__( "VolumeInUse", - "Volume {0} is currently attached to {1}" - .format(volume_id, instance_id)) + "Volume {0} is currently attached to {1}".format(volume_id, instance_id), + ) class InvalidDomainError(EC2ClientError): diff --git a/tests/test_ec2/test_elastic_block_store.py b/tests/test_ec2/test_elastic_block_store.py index 1182610e8..ac9c7e3d9 100644 --- a/tests/test_ec2/test_elastic_block_store.py +++ b/tests/test_ec2/test_elastic_block_store.py @@ -76,7 +76,9 @@ def test_delete_attached_volume(): volume.delete() ex.exception.error_code.should.equal("VolumeInUse") ex.exception.status.should.equal(400) - ex.exception.message.should.equal(f"Volume {volume.id} is currently attached to {instance.id}") + ex.exception.message.should.equal( + f"Volume {volume.id} is currently attached to {instance.id}" + ) volume.detach() From 9ab02e17d528d461b5a05b7aafa623addae65966 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Fri, 3 Apr 2020 10:30:05 +0100 Subject: [PATCH 18/83] #883 - Lambda - Add test to verify remove_permission functinonality --- moto/awslambda/models.py | 4 ++-- moto/awslambda/responses.py | 6 ++--- tests/test_awslambda/test_lambda.py | 36 +++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index 9cdf2397c..589a790ae 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -1006,11 +1006,11 @@ class LambdaBackend(BaseBackend): return True return False - def add_policy_statement(self, function_name, raw): + def add_permission(self, function_name, raw): fn = self.get_function(function_name) fn.policy.add_statement(raw) - def del_policy_statement(self, function_name, sid, revision=""): + def remove_permission(self, function_name, sid, revision=""): fn = self.get_function(function_name) fn.policy.del_statement(sid, revision) diff --git a/moto/awslambda/responses.py b/moto/awslambda/responses.py index ce6c93f16..4213840f6 100644 --- a/moto/awslambda/responses.py +++ b/moto/awslambda/responses.py @@ -146,7 +146,7 @@ class LambdaResponse(BaseResponse): function_name = path.split("/")[-2] if self.lambda_backend.get_function(function_name): statement = self.body - self.lambda_backend.add_policy_statement(function_name, statement) + self.lambda_backend.add_permission(function_name, statement) return 200, {}, json.dumps({"Statement": statement}) else: return 404, {}, "{}" @@ -166,9 +166,7 @@ class LambdaResponse(BaseResponse): statement_id = path.split("/")[-1].split("?")[0] revision = querystring.get("RevisionId", "") if self.lambda_backend.get_function(function_name): - self.lambda_backend.del_policy_statement( - function_name, statement_id, revision - ) + self.lambda_backend.remove_permission(function_name, statement_id, revision) return 204, {}, "{}" else: return 404, {}, "{}" diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index eb8453e43..e67576518 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -1677,6 +1677,42 @@ def test_create_function_with_unknown_arn(): ) +@mock_lambda +def test_remove_function_permission(): + conn = boto3.client("lambda", _lambda_region) + zip_content = get_test_zip_file1() + conn.create_function( + FunctionName="testFunction", + Runtime="python2.7", + Role=(get_role_name()), + Handler="lambda_function.handler", + Code={"ZipFile": zip_content}, + Description="test lambda function", + Timeout=3, + MemorySize=128, + Publish=True, + ) + + conn.add_permission( + FunctionName="testFunction", + StatementId="1", + Action="lambda:InvokeFunction", + Principal="432143214321", + SourceArn="arn:aws:lambda:us-west-2:account-id:function:helloworld", + SourceAccount="123412341234", + EventSourceToken="blah", + Qualifier="2", + ) + + remove = conn.remove_permission( + FunctionName="testFunction", StatementId="1", Qualifier="2", + ) + remove["ResponseMetadata"]["HTTPStatusCode"].should.equal(204) + policy = conn.get_policy(FunctionName="testFunction", Qualifier="2")["Policy"] + policy = json.loads(policy) + policy["Statement"].should.equal([]) + + def create_invalid_lambda(role): conn = boto3.client("lambda", _lambda_region) zip_content = get_test_zip_file1() From a6864f483db1c1b292098a67381c6f20437a8dd2 Mon Sep 17 00:00:00 2001 From: mickeypash Date: Fri, 3 Apr 2020 14:17:55 +0100 Subject: [PATCH 19/83] Use Python 2 format --- tests/test_ec2/test_elastic_block_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ec2/test_elastic_block_store.py b/tests/test_ec2/test_elastic_block_store.py index ac9c7e3d9..4bd2a8dfa 100644 --- a/tests/test_ec2/test_elastic_block_store.py +++ b/tests/test_ec2/test_elastic_block_store.py @@ -77,7 +77,7 @@ def test_delete_attached_volume(): ex.exception.error_code.should.equal("VolumeInUse") ex.exception.status.should.equal(400) ex.exception.message.should.equal( - f"Volume {volume.id} is currently attached to {instance.id}" + "Volume {0} is currently attached to {1}".format(volume.id, instance.id) ) volume.detach() From 280db9df6c43f606721d07b51e104eda8e065313 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Sat, 4 Apr 2020 14:09:38 +0100 Subject: [PATCH 20/83] #2800 - CognitoIdentity - Fix format of Identity ID --- moto/cognitoidentity/utils.py | 4 ++-- tests/test_cognitoidentity/test_cognitoidentity.py | 8 +++++--- tests/test_cognitoidentity/test_server.py | 1 - 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/moto/cognitoidentity/utils.py b/moto/cognitoidentity/utils.py index 6143d5121..54016ad17 100644 --- a/moto/cognitoidentity/utils.py +++ b/moto/cognitoidentity/utils.py @@ -1,5 +1,5 @@ -from moto.core.utils import get_random_hex +from uuid import uuid4 def get_random_identity_id(region): - return "{0}:{1}".format(region, get_random_hex(length=19)) + return "{0}:{1}".format(region, uuid4()) diff --git a/tests/test_cognitoidentity/test_cognitoidentity.py b/tests/test_cognitoidentity/test_cognitoidentity.py index 8eae183c6..0ec7acfb0 100644 --- a/tests/test_cognitoidentity/test_cognitoidentity.py +++ b/tests/test_cognitoidentity/test_cognitoidentity.py @@ -7,6 +7,7 @@ from nose.tools import assert_raises from moto import mock_cognitoidentity from moto.cognitoidentity.utils import get_random_identity_id from moto.core import ACCOUNT_ID +from uuid import UUID @mock_cognitoidentity @@ -83,8 +84,10 @@ def test_describe_identity_pool_with_invalid_id_raises_error(): # testing a helper function def test_get_random_identity_id(): - assert len(get_random_identity_id("us-west-2")) > 0 - assert len(get_random_identity_id("us-west-2").split(":")[1]) == 19 + identity_id = get_random_identity_id("us-west-2") + region, id = identity_id.split(":") + region.should.equal("us-west-2") + UUID(id, version=4) # Will throw an error if it's not a valid UUID @mock_cognitoidentity @@ -96,7 +99,6 @@ def test_get_id(): IdentityPoolId="us-west-2:12345", Logins={"someurl": "12345"}, ) - print(result) assert ( result.get("IdentityId", "").startswith("us-west-2") or result.get("ResponseMetadata").get("HTTPStatusCode") == 200 diff --git a/tests/test_cognitoidentity/test_server.py b/tests/test_cognitoidentity/test_server.py index 903dae290..8c4229f06 100644 --- a/tests/test_cognitoidentity/test_server.py +++ b/tests/test_cognitoidentity/test_server.py @@ -48,6 +48,5 @@ def test_get_id(): }, ) - print(res.data) json_data = json.loads(res.data.decode("utf-8")) assert ":" in json_data["IdentityId"] From 16db824d8ae293ae99877b8c4ea5a73785c5e1a5 Mon Sep 17 00:00:00 2001 From: David Holroyd Date: Mon, 6 Apr 2020 00:19:19 +0100 Subject: [PATCH 21/83] Fix response XML structure --- moto/s3/responses.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 197cd9080..06e7353d3 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -1868,18 +1868,16 @@ S3_DELETE_KEYS_RESPONSE = """ {% endfor %} """ -S3_DELETE_OBJECT_SUCCESS = """ - - 200 - OK - +S3_DELETE_OBJECT_SUCCESS = """ + + 200 + OK """ -S3_OBJECT_RESPONSE = """ - - {{ key.etag }} - {{ key.last_modified_ISO8601 }} - +S3_OBJECT_RESPONSE = """ + + {{ key.etag }} + {{ key.last_modified_ISO8601 }} """ S3_OBJECT_ACL_RESPONSE = """ From 09de93412e4ec3b197205f8b8e9ea9c8a57ab961 Mon Sep 17 00:00:00 2001 From: jess Date: Mon, 6 Apr 2020 17:21:26 +1000 Subject: [PATCH 22/83] Prevent JSON dumps error when dealing with complex types --- moto/dynamodbstreams/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/dynamodbstreams/models.py b/moto/dynamodbstreams/models.py index dc6f0e0d3..f62c49877 100644 --- a/moto/dynamodbstreams/models.py +++ b/moto/dynamodbstreams/models.py @@ -7,7 +7,7 @@ import base64 from boto3 import Session from moto.core import BaseBackend, BaseModel -from moto.dynamodb2.models import dynamodb_backends +from moto.dynamodb2.models import dynamodb_backends, DynamoJsonEncoder class ShardIterator(BaseModel): @@ -137,7 +137,7 @@ class DynamoDBStreamsBackend(BaseBackend): def get_records(self, iterator_arn, limit): shard_iterator = self.shard_iterators[iterator_arn] - return json.dumps(shard_iterator.get(limit)) + return json.dumps(shard_iterator.get(limit), cls=DynamoJsonEncoder) dynamodbstreams_backends = {} From b6e73776d56ed47d4080c5a000bd3754492e1219 Mon Sep 17 00:00:00 2001 From: jess Date: Mon, 6 Apr 2020 18:41:46 +1000 Subject: [PATCH 23/83] alter testcase to trigger issue #2868 --- tests/test_dynamodbstreams/test_dynamodbstreams.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_dynamodbstreams/test_dynamodbstreams.py b/tests/test_dynamodbstreams/test_dynamodbstreams.py index 8fad0ff23..d4d619a56 100644 --- a/tests/test_dynamodbstreams/test_dynamodbstreams.py +++ b/tests/test_dynamodbstreams/test_dynamodbstreams.py @@ -121,12 +121,14 @@ class TestCore: assert "Records" in resp assert len(resp["Records"]) == 0 + def test_get_records_seq(self): conn = boto3.client("dynamodb", region_name="us-east-1") conn.put_item( TableName="test-streams", - Item={"id": {"S": "entry1"}, "first_col": {"S": "foo"}}, + Item={"id": {"S": "entry1"}, "first_col": {"S": "foo"}} + ) conn.put_item( TableName="test-streams", @@ -134,6 +136,7 @@ class TestCore: "id": {"S": "entry1"}, "first_col": {"S": "bar"}, "second_col": {"S": "baz"}, + "a": {"L": [{"M": {"b": {"S": "bar1"}}}]} }, ) conn.delete_item(TableName="test-streams", Key={"id": {"S": "entry1"}}) From 4c2460ddfdf4dd7263bf2899df95ea3c903e2b84 Mon Sep 17 00:00:00 2001 From: jess Date: Mon, 6 Apr 2020 18:45:23 +1000 Subject: [PATCH 24/83] fix whitespace changes --- tests/test_dynamodbstreams/test_dynamodbstreams.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_dynamodbstreams/test_dynamodbstreams.py b/tests/test_dynamodbstreams/test_dynamodbstreams.py index d4d619a56..c75d66e7f 100644 --- a/tests/test_dynamodbstreams/test_dynamodbstreams.py +++ b/tests/test_dynamodbstreams/test_dynamodbstreams.py @@ -121,14 +121,12 @@ class TestCore: assert "Records" in resp assert len(resp["Records"]) == 0 - def test_get_records_seq(self): conn = boto3.client("dynamodb", region_name="us-east-1") conn.put_item( TableName="test-streams", Item={"id": {"S": "entry1"}, "first_col": {"S": "foo"}} - ) conn.put_item( TableName="test-streams", From 49fd7988ab936d5d81ed00dd25c3b4e408566992 Mon Sep 17 00:00:00 2001 From: jess Date: Mon, 6 Apr 2020 19:55:54 +1000 Subject: [PATCH 25/83] make black happy --- tests/test_dynamodbstreams/test_dynamodbstreams.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_dynamodbstreams/test_dynamodbstreams.py b/tests/test_dynamodbstreams/test_dynamodbstreams.py index c75d66e7f..065d7280e 100644 --- a/tests/test_dynamodbstreams/test_dynamodbstreams.py +++ b/tests/test_dynamodbstreams/test_dynamodbstreams.py @@ -126,7 +126,7 @@ class TestCore: conn.put_item( TableName="test-streams", - Item={"id": {"S": "entry1"}, "first_col": {"S": "foo"}} + Item={"id": {"S": "entry1"}, "first_col": {"S": "foo"}}, ) conn.put_item( TableName="test-streams", @@ -134,7 +134,7 @@ class TestCore: "id": {"S": "entry1"}, "first_col": {"S": "bar"}, "second_col": {"S": "baz"}, - "a": {"L": [{"M": {"b": {"S": "bar1"}}}]} + "a": {"L": [{"M": {"b": {"S": "bar1"}}}]}, }, ) conn.delete_item(TableName="test-streams", Key={"id": {"S": "entry1"}}) From 81ca5c3ab0b62f1f2a7396a6e3e89e973d6774a6 Mon Sep 17 00:00:00 2001 From: usmankb Date: Mon, 6 Apr 2020 21:25:59 +0530 Subject: [PATCH 26/83] Implemented describe_instance_credit_specifications function in ec2 #2150 localstack --- moto/ec2/models.py | 6 ++++++ moto/ec2/responses/instances.py | 18 ++++++++++++++++++ tests/test_ec2/test_instances.py | 8 ++++++++ 3 files changed, 32 insertions(+) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index bf4936d09..a4c15c56d 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -937,6 +937,12 @@ class InstanceBackend(object): value = getattr(instance, key) return instance, value + def describe_instance_credit_specifications(self,instance_ids): + queried_instances = [] + for instance in self.get_multi_instances_by_id(instance_ids): + queried_instances.append(instance) + return queried_instances + def all_instances(self, filters=None): instances = [] for reservation in self.all_reservations(): diff --git a/moto/ec2/responses/instances.py b/moto/ec2/responses/instances.py index 29c346f82..9e68eed7a 100644 --- a/moto/ec2/responses/instances.py +++ b/moto/ec2/responses/instances.py @@ -168,6 +168,12 @@ class InstanceResponse(BaseResponse): return template.render(instance=instance, attribute=attribute, value=value) + def describe_instance_credit_specifications(self): + instance_ids = self._get_multi_param("InstanceId") + instance = self.ec2_backend.describe_instance_credit_specifications(instance_ids) + template = self.response_template(EC2_DESCRIBE_INSTANCE_CREDIT_SPECIFICATIONS) + return template.render(instances=instance) + def modify_instance_attribute(self): handlers = [ self._dot_value_instance_attribute_handler, @@ -671,6 +677,18 @@ EC2_DESCRIBE_INSTANCE_ATTRIBUTE = """ + 1b234b5c-d6ef-7gh8-90i1-j2345678901 + + {% for instance in instances %} + + {{ instance.id }} + standard + + {% endfor %} + +""" + EC2_DESCRIBE_INSTANCE_GROUPSET_ATTRIBUTE = """ 59dbff89-35bd-4eac-99ed-be587EXAMPLE {{ instance.id }} diff --git a/tests/test_ec2/test_instances.py b/tests/test_ec2/test_instances.py index 85ba0fe01..f4fcbb186 100644 --- a/tests/test_ec2/test_instances.py +++ b/tests/test_ec2/test_instances.py @@ -1165,6 +1165,14 @@ def test_describe_instance_status_with_instance_filter_deprecated(): cm.exception.status.should.equal(400) cm.exception.request_id.should_not.be.none +@mock_ec2 +def test_describe_instance_credit_specifications(): + conn = boto3.client("ec2", region_name="us-west-1") + + # We want to filter based on this one + reservation = conn.run_instances(ImageId="ami-1234abcd", MinCount=1, MaxCount=1) + result = conn.describe_instance_credit_specifications(InstanceIds=[reservation["Instances"][0]["InstanceId"]]) + assert result['InstanceCreditSpecifications'][0]['InstanceId'] == reservation["Instances"][0]["InstanceId"] @mock_ec2 def test_describe_instance_status_with_instance_filter(): From a845de114209ead7606c0a2a1690fd03b10dde04 Mon Sep 17 00:00:00 2001 From: David Holroyd Date: Mon, 6 Apr 2020 21:01:43 +0100 Subject: [PATCH 27/83] PutObject and DeleteObject should produce no XML S3 itself produces an empty body, with any response metadata in HTTP headers only. --- moto/s3/responses.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 06e7353d3..22cd45c08 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -1232,9 +1232,8 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): ) new_key.set_tagging(tagging) - template = self.response_template(S3_OBJECT_RESPONSE) response_headers.update(new_key.response_dict) - return 200, response_headers, template.render(key=new_key) + return 200, response_headers, "" def _key_response_head(self, bucket_name, query, key_name, headers): response_headers = {} @@ -1552,8 +1551,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): return 204, {}, "" version_id = query.get("versionId", [None])[0] self.backend.delete_key(bucket_name, key_name, version_id=version_id) - template = self.response_template(S3_DELETE_OBJECT_SUCCESS) - return 204, {}, template.render() + return 204, {}, "" def _complete_multipart_body(self, body): ps = minidom.parseString(body).getElementsByTagName("Part") @@ -1868,18 +1866,6 @@ S3_DELETE_KEYS_RESPONSE = """ {% endfor %} """ -S3_DELETE_OBJECT_SUCCESS = """ - - 200 - OK -""" - -S3_OBJECT_RESPONSE = """ - - {{ key.etag }} - {{ key.last_modified_ISO8601 }} - """ - S3_OBJECT_ACL_RESPONSE = """ From aae49493c466eb17d7c42cf6b41d739914271873 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 7 Apr 2020 08:49:19 +0100 Subject: [PATCH 28/83] Linting --- moto/ec2/models.py | 2 +- moto/ec2/responses/instances.py | 4 +++- tests/test_ec2/test_instances.py | 11 +++++++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index a4c15c56d..83e12eea7 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -937,7 +937,7 @@ class InstanceBackend(object): value = getattr(instance, key) return instance, value - def describe_instance_credit_specifications(self,instance_ids): + def describe_instance_credit_specifications(self, instance_ids): queried_instances = [] for instance in self.get_multi_instances_by_id(instance_ids): queried_instances.append(instance) diff --git a/moto/ec2/responses/instances.py b/moto/ec2/responses/instances.py index 9e68eed7a..490ffb642 100644 --- a/moto/ec2/responses/instances.py +++ b/moto/ec2/responses/instances.py @@ -170,7 +170,9 @@ class InstanceResponse(BaseResponse): def describe_instance_credit_specifications(self): instance_ids = self._get_multi_param("InstanceId") - instance = self.ec2_backend.describe_instance_credit_specifications(instance_ids) + instance = self.ec2_backend.describe_instance_credit_specifications( + instance_ids + ) template = self.response_template(EC2_DESCRIBE_INSTANCE_CREDIT_SPECIFICATIONS) return template.render(instances=instance) diff --git a/tests/test_ec2/test_instances.py b/tests/test_ec2/test_instances.py index f4fcbb186..595faa5ba 100644 --- a/tests/test_ec2/test_instances.py +++ b/tests/test_ec2/test_instances.py @@ -1165,14 +1165,21 @@ def test_describe_instance_status_with_instance_filter_deprecated(): cm.exception.status.should.equal(400) cm.exception.request_id.should_not.be.none + @mock_ec2 def test_describe_instance_credit_specifications(): conn = boto3.client("ec2", region_name="us-west-1") # We want to filter based on this one reservation = conn.run_instances(ImageId="ami-1234abcd", MinCount=1, MaxCount=1) - result = conn.describe_instance_credit_specifications(InstanceIds=[reservation["Instances"][0]["InstanceId"]]) - assert result['InstanceCreditSpecifications'][0]['InstanceId'] == reservation["Instances"][0]["InstanceId"] + result = conn.describe_instance_credit_specifications( + InstanceIds=[reservation["Instances"][0]["InstanceId"]] + ) + assert ( + result["InstanceCreditSpecifications"][0]["InstanceId"] + == reservation["Instances"][0]["InstanceId"] + ) + @mock_ec2 def test_describe_instance_status_with_instance_filter(): From 856c07de63cd545e0836ec2083ee3836c5c56f71 Mon Sep 17 00:00:00 2001 From: usmankb Date: Wed, 8 Apr 2020 03:18:42 +0530 Subject: [PATCH 29/83] aws apigateway create,get domain names --- moto/apigateway/exceptions.py | 8 +++ moto/apigateway/models.py | 84 +++++++++++++++++++++++- moto/apigateway/responses.py | 59 +++++++++++++++++ moto/apigateway/urls.py | 2 + tests/test_apigateway/test_apigateway.py | 17 +++++ 5 files changed, 169 insertions(+), 1 deletion(-) diff --git a/moto/apigateway/exceptions.py b/moto/apigateway/exceptions.py index ccb870f52..24f06f3f1 100644 --- a/moto/apigateway/exceptions.py +++ b/moto/apigateway/exceptions.py @@ -119,3 +119,11 @@ class ApiKeyAlreadyExists(RESTError): super(ApiKeyAlreadyExists, self).__init__( "ConflictException", "API Key already exists" ) + +class DomainNameNotFound(RESTError): + code = 404 + + def __init__(self): + super(DomainNameNotFound, self).__init__( + "NotFoundException", "Invalid Domain Name specified" + ) diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index 5b02e6204..7ca7e6315 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -34,6 +34,7 @@ from .exceptions import ( NoIntegrationDefined, NoMethodDefined, ApiKeyAlreadyExists, + DomainNameNotFound ) STAGE_URL = "https://{api_id}.execute-api.{region_name}.amazonaws.com/{stage_name}" @@ -463,7 +464,6 @@ class RestAPI(BaseModel): self.deployments = {} self.authorizers = {} self.stages = {} - self.resources = {} self.add_child("/") # Add default child @@ -609,6 +609,51 @@ class RestAPI(BaseModel): return self.deployments.pop(deployment_id) +class DomainName(BaseModel,dict): + def __init__(self, domain_name, **kwargs): + super(DomainName, self).__init__() + self["domainName"] = domain_name + self["regionalDomainName"] = domain_name, + self["distributionDomainName"] = domain_name, + self["domainNameStatus"] = "AVAILABLE" + self["domainNameStatusMessage"] = "Domain Name Available" + self["regionalHostedZoneId"] = "Z2FDTNDATAQYW2" + self["distributionHostedZoneId"] = "Z2FDTNDATAQYW2" + self["certificateUploadDate"] = int(time.time()) + if kwargs.get("certificate_name"): + self["certificateName"] = kwargs.get("certificate_name") + if kwargs.get("certificate_arn"): + self["certificateArn"] = kwargs.get("certificate_arn") + if kwargs.get("certificate_body"): + self["certificateBody"] = kwargs.get("certificate_body") + if kwargs.get("tags"): + self["tags"] = kwargs.get("tags" ) + if kwargs.get("security_policy"): + self["securityPolicy"] = kwargs.get("security_policy") + if kwargs.get("certificate_chain"): + self["certificateChain"] = kwargs.get("certificate_chain") + if kwargs.get("regional_certificate_name"): + self["regionalCertificateName"] = kwargs.get( + "regional_certificate_name" + ) + if kwargs.get("certificate_private_key"): + self["certificatePrivateKey"] = kwargs.get( + "certificate_private_key" + ) + if kwargs.get("regional_certificate_arn"): + self["regionalCertificateArn"] = kwargs.get( + "regional_certificate_arn" + ) + if kwargs.get("endpoint_configuration"): + self["endpointConfiguration"] = kwargs.get( + "endpoint_configuration" + ) + if kwargs.get("generate_cli_skeleton"): + self["generateCliSkeleton"] = kwargs.get( + "generate_cli_skeleton" + ) + + class APIGatewayBackend(BaseBackend): def __init__(self, region_name): super(APIGatewayBackend, self).__init__() @@ -616,6 +661,7 @@ class APIGatewayBackend(BaseBackend): self.keys = {} self.usage_plans = {} self.usage_plan_keys = {} + self.domain_names = {} self.region_name = region_name def reset(self): @@ -1001,6 +1047,42 @@ class APIGatewayBackend(BaseBackend): except Exception: return False + def create_domain_name(self, domain_name, + certificate_name=None, tags=None, + certificate_arn=None, certificate_body=None, + certificate_private_key=None, + certificate_chain=None, + regional_certificate_name=None, + regional_certificate_arn=None, + endpoint_configuration=None, + security_policy=None, + generate_cli_skeleton=None): + if not domain_name: + raise DomainNameNotFound() + + new_domain_name = DomainName( + domain_name=domain_name, + certificate_name=certificate_name, + certificate_arn=certificate_arn, + certificate_body=certificate_body, + certificate_private_key=certificate_private_key, + certificate_chain=certificate_chain, + regional_certificate_name=regional_certificate_name, + regional_certificate_arn=regional_certificate_arn, + endpoint_configuration=endpoint_configuration, + tags=tags, security_policy=security_policy, + generate_cli_skeleton=generate_cli_skeleton, + ) + + self.domain_names[domain_name] = new_domain_name + return new_domain_name + + def get_domain_names(self): + return list(self.domain_names.values()) + + def get_domain_name(self, domain_name): + return self.domain_names[domain_name] + apigateway_backends = {} for region_name in Session().get_available_regions("apigateway"): diff --git a/moto/apigateway/responses.py b/moto/apigateway/responses.py index f0ed6adc9..ce3bcbb8e 100644 --- a/moto/apigateway/responses.py +++ b/moto/apigateway/responses.py @@ -527,3 +527,62 @@ class APIGatewayResponse(BaseResponse): usage_plan_id, key_id ) return 200, {}, json.dumps(usage_plan_response) + + def domain_names(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + + try: + if self.method == "GET": + domain_names = self.backend.get_domain_names() + return 200, {}, json.dumps({"item": domain_names}) + + elif self.method == "POST": + domain_name = self._get_param("domainName") + certificate_name = self._get_param("certificateName") + tags = self._get_param("tags") + certificate_arn = self._get_param("certificateArn") + certificate_body = self._get_param("certificateBody") + certificate_private_key = self._get_param( + "certificatePrivateKey" + ) + + certificate_chain = self._get_param("certificateChain") + regional_certificate_name = self._get_param( + "regionalCertificateName" + ) + regional_certificate_arn = self._get_param( + "regionalCertificateArn" + ) + endpoint_configuration = self._get_param( + "endpointConfiguration" + ) + security_policy = self._get_param("securityPolicy") + generate_cli_skeleton = self._get_param( + "generateCliSkeleton" + ) + domain_name_resp = self.backend.create_domain_name( + domain_name, certificate_name, tags, certificate_arn, + certificate_body, certificate_private_key, + certificate_chain, regional_certificate_name, + regional_certificate_arn, endpoint_configuration, + security_policy, generate_cli_skeleton + ) + + return 200, {}, json.dumps(domain_name_resp) + except BadRequestException as e: + return self.error( + "com.amazonaws.dynamodb.v20111205#BadRequestException", e.message + ) + + def domain_name_induvidual(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + + url_path_parts = self.path.split("/") + domain_name = url_path_parts[2] + domain_names={} + + if self.method == "GET": + if domain_name is not None: + domain_names = self.backend.get_domain_name(domain_name) + + return 200, {}, json.dumps(domain_names) diff --git a/moto/apigateway/urls.py b/moto/apigateway/urls.py index 4ef6ae72b..6c3b7f6bb 100644 --- a/moto/apigateway/urls.py +++ b/moto/apigateway/urls.py @@ -21,6 +21,8 @@ url_paths = { "{0}/apikeys$": APIGatewayResponse().apikeys, "{0}/apikeys/(?P[^/]+)": APIGatewayResponse().apikey_individual, "{0}/usageplans$": APIGatewayResponse().usage_plans, + "{0}/domainnames$": APIGatewayResponse().domain_names, + "{0}/domainnames/(?P[^/]+)/?$": APIGatewayResponse().domain_name_induvidual, "{0}/usageplans/(?P[^/]+)/?$": APIGatewayResponse().usage_plan_individual, "{0}/usageplans/(?P[^/]+)/keys$": APIGatewayResponse().usage_plan_keys, "{0}/usageplans/(?P[^/]+)/keys/(?P[^/]+)/?$": APIGatewayResponse().usage_plan_key_individual, diff --git a/tests/test_apigateway/test_apigateway.py b/tests/test_apigateway/test_apigateway.py index 0952f2674..22e062cc5 100644 --- a/tests/test_apigateway/test_apigateway.py +++ b/tests/test_apigateway/test_apigateway.py @@ -1483,6 +1483,23 @@ def test_deployment(): stage["description"].should.equal("_new_description_") +@mock_apigateway +def test_create_domain_names(): + client = boto3.client("apigateway", region_name="us-west-2") + domain_name = "testDomain" + test_certificate_name = "test.certificate" + test_certificate_private_key = "testPrivateKey" + response = client.create_domain_name(domainName=domain_name, certificateName=test_certificate_name, + certificatePrivateKey=test_certificate_private_key) + + response["domainName"].should.equal(domain_name) + response["certificateName"].should.equal(test_certificate_name) + result = client.get_domain_names() + result["items"][0]["domainName"].should.equal(domain_name) + result = client.get_domain_name(domainName=domain_name) + result["domainName"].should.equal(domain_name) + + @mock_apigateway def test_http_proxying_integration(): responses.add( From 0163eb6a9dfc7809159183c84a27bb7be67fd47d Mon Sep 17 00:00:00 2001 From: Theodore Wong Date: Tue, 7 Apr 2020 15:32:44 -0700 Subject: [PATCH 30/83] Changed mock_ecs to support ecs.run_task calls with a default cluster --- moto/ecs/models.py | 5 ++- tests/test_ecs/test_ecs_boto3.py | 65 ++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/moto/ecs/models.py b/moto/ecs/models.py index 30e4687c4..33d4dcf72 100644 --- a/moto/ecs/models.py +++ b/moto/ecs/models.py @@ -604,7 +604,10 @@ class EC2ContainerServiceBackend(BaseBackend): raise Exception("{0} is not a task_definition".format(task_definition_name)) def run_task(self, cluster_str, task_definition_str, count, overrides, started_by): - cluster_name = cluster_str.split("/")[-1] + if cluster_str: + cluster_name = cluster_str.split("/")[-1] + else: + cluster_name = "default" if cluster_name in self.clusters: cluster = self.clusters[cluster_name] else: diff --git a/tests/test_ecs/test_ecs_boto3.py b/tests/test_ecs/test_ecs_boto3.py index 69c920192..7fd90b412 100644 --- a/tests/test_ecs/test_ecs_boto3.py +++ b/tests/test_ecs/test_ecs_boto3.py @@ -1122,6 +1122,71 @@ def test_run_task(): response["tasks"][0]["stoppedReason"].should.equal("") +@mock_ec2 +@mock_ecs +def test_run_task_default_cluster(): + client = boto3.client("ecs", region_name="us-east-1") + ec2 = boto3.resource("ec2", region_name="us-east-1") + + test_cluster_name = "default" + + _ = client.create_cluster(clusterName=test_cluster_name) + + test_instance = ec2.create_instances( + ImageId="ami-1234abcd", MinCount=1, MaxCount=1 + )[0] + + instance_id_document = json.dumps( + ec2_utils.generate_instance_identity_document(test_instance) + ) + + response = client.register_container_instance( + cluster=test_cluster_name, instanceIdentityDocument=instance_id_document + ) + + _ = client.register_task_definition( + family="test_ecs_task", + containerDefinitions=[ + { + "name": "hello_world", + "image": "docker/hello-world:latest", + "cpu": 1024, + "memory": 400, + "essential": True, + "environment": [ + {"name": "AWS_ACCESS_KEY_ID", "value": "SOME_ACCESS_KEY"} + ], + "logConfiguration": {"logDriver": "json-file"}, + } + ], + ) + response = client.run_task( + launchType="FARGATE", + overrides={}, + taskDefinition="test_ecs_task", + count=2, + startedBy="moto", + ) + len(response["tasks"]).should.equal(2) + response["tasks"][0]["taskArn"].should.contain( + "arn:aws:ecs:us-east-1:012345678910:task/" + ) + response["tasks"][0]["clusterArn"].should.equal( + "arn:aws:ecs:us-east-1:012345678910:cluster/default" + ) + response["tasks"][0]["taskDefinitionArn"].should.equal( + "arn:aws:ecs:us-east-1:012345678910:task-definition/test_ecs_task:1" + ) + response["tasks"][0]["containerInstanceArn"].should.contain( + "arn:aws:ecs:us-east-1:012345678910:container-instance/" + ) + response["tasks"][0]["overrides"].should.equal({}) + response["tasks"][0]["lastStatus"].should.equal("RUNNING") + response["tasks"][0]["desiredStatus"].should.equal("RUNNING") + response["tasks"][0]["startedBy"].should.equal("moto") + response["tasks"][0]["stoppedReason"].should.equal("") + + @mock_ec2 @mock_ecs def test_start_task(): From 8237fdaff0562246d12abf9dd6588643db7f7105 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 8 Apr 2020 11:06:30 +0100 Subject: [PATCH 31/83] Linting --- moto/apigateway/exceptions.py | 1 + moto/apigateway/models.py | 58 +++++++++++------------- moto/apigateway/responses.py | 39 ++++++++-------- tests/test_apigateway/test_apigateway.py | 7 ++- 4 files changed, 51 insertions(+), 54 deletions(-) diff --git a/moto/apigateway/exceptions.py b/moto/apigateway/exceptions.py index 24f06f3f1..4a808945c 100644 --- a/moto/apigateway/exceptions.py +++ b/moto/apigateway/exceptions.py @@ -120,6 +120,7 @@ class ApiKeyAlreadyExists(RESTError): "ConflictException", "API Key already exists" ) + class DomainNameNotFound(RESTError): code = 404 diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index 7ca7e6315..6bef6f019 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -34,7 +34,7 @@ from .exceptions import ( NoIntegrationDefined, NoMethodDefined, ApiKeyAlreadyExists, - DomainNameNotFound + DomainNameNotFound, ) STAGE_URL = "https://{api_id}.execute-api.{region_name}.amazonaws.com/{stage_name}" @@ -609,12 +609,12 @@ class RestAPI(BaseModel): return self.deployments.pop(deployment_id) -class DomainName(BaseModel,dict): +class DomainName(BaseModel, dict): def __init__(self, domain_name, **kwargs): super(DomainName, self).__init__() self["domainName"] = domain_name - self["regionalDomainName"] = domain_name, - self["distributionDomainName"] = domain_name, + self["regionalDomainName"] = domain_name + self["distributionDomainName"] = domain_name self["domainNameStatus"] = "AVAILABLE" self["domainNameStatusMessage"] = "Domain Name Available" self["regionalHostedZoneId"] = "Z2FDTNDATAQYW2" @@ -627,31 +627,21 @@ class DomainName(BaseModel,dict): if kwargs.get("certificate_body"): self["certificateBody"] = kwargs.get("certificate_body") if kwargs.get("tags"): - self["tags"] = kwargs.get("tags" ) + self["tags"] = kwargs.get("tags") if kwargs.get("security_policy"): self["securityPolicy"] = kwargs.get("security_policy") if kwargs.get("certificate_chain"): self["certificateChain"] = kwargs.get("certificate_chain") if kwargs.get("regional_certificate_name"): - self["regionalCertificateName"] = kwargs.get( - "regional_certificate_name" - ) + self["regionalCertificateName"] = kwargs.get("regional_certificate_name") if kwargs.get("certificate_private_key"): - self["certificatePrivateKey"] = kwargs.get( - "certificate_private_key" - ) + self["certificatePrivateKey"] = kwargs.get("certificate_private_key") if kwargs.get("regional_certificate_arn"): - self["regionalCertificateArn"] = kwargs.get( - "regional_certificate_arn" - ) + self["regionalCertificateArn"] = kwargs.get("regional_certificate_arn") if kwargs.get("endpoint_configuration"): - self["endpointConfiguration"] = kwargs.get( - "endpoint_configuration" - ) + self["endpointConfiguration"] = kwargs.get("endpoint_configuration") if kwargs.get("generate_cli_skeleton"): - self["generateCliSkeleton"] = kwargs.get( - "generate_cli_skeleton" - ) + self["generateCliSkeleton"] = kwargs.get("generate_cli_skeleton") class APIGatewayBackend(BaseBackend): @@ -1047,16 +1037,21 @@ class APIGatewayBackend(BaseBackend): except Exception: return False - def create_domain_name(self, domain_name, - certificate_name=None, tags=None, - certificate_arn=None, certificate_body=None, - certificate_private_key=None, - certificate_chain=None, - regional_certificate_name=None, - regional_certificate_arn=None, - endpoint_configuration=None, - security_policy=None, - generate_cli_skeleton=None): + def create_domain_name( + self, + domain_name, + certificate_name=None, + tags=None, + certificate_arn=None, + certificate_body=None, + certificate_private_key=None, + certificate_chain=None, + regional_certificate_name=None, + regional_certificate_arn=None, + endpoint_configuration=None, + security_policy=None, + generate_cli_skeleton=None, + ): if not domain_name: raise DomainNameNotFound() @@ -1070,7 +1065,8 @@ class APIGatewayBackend(BaseBackend): regional_certificate_name=regional_certificate_name, regional_certificate_arn=regional_certificate_arn, endpoint_configuration=endpoint_configuration, - tags=tags, security_policy=security_policy, + tags=tags, + security_policy=security_policy, generate_cli_skeleton=generate_cli_skeleton, ) diff --git a/moto/apigateway/responses.py b/moto/apigateway/responses.py index ce3bcbb8e..a3587e97a 100644 --- a/moto/apigateway/responses.py +++ b/moto/apigateway/responses.py @@ -542,30 +542,27 @@ class APIGatewayResponse(BaseResponse): tags = self._get_param("tags") certificate_arn = self._get_param("certificateArn") certificate_body = self._get_param("certificateBody") - certificate_private_key = self._get_param( - "certificatePrivateKey" - ) + certificate_private_key = self._get_param("certificatePrivateKey") certificate_chain = self._get_param("certificateChain") - regional_certificate_name = self._get_param( - "regionalCertificateName" - ) - regional_certificate_arn = self._get_param( - "regionalCertificateArn" - ) - endpoint_configuration = self._get_param( - "endpointConfiguration" - ) + regional_certificate_name = self._get_param("regionalCertificateName") + regional_certificate_arn = self._get_param("regionalCertificateArn") + endpoint_configuration = self._get_param("endpointConfiguration") security_policy = self._get_param("securityPolicy") - generate_cli_skeleton = self._get_param( - "generateCliSkeleton" - ) + generate_cli_skeleton = self._get_param("generateCliSkeleton") domain_name_resp = self.backend.create_domain_name( - domain_name, certificate_name, tags, certificate_arn, - certificate_body, certificate_private_key, - certificate_chain, regional_certificate_name, - regional_certificate_arn, endpoint_configuration, - security_policy, generate_cli_skeleton + domain_name, + certificate_name, + tags, + certificate_arn, + certificate_body, + certificate_private_key, + certificate_chain, + regional_certificate_name, + regional_certificate_arn, + endpoint_configuration, + security_policy, + generate_cli_skeleton, ) return 200, {}, json.dumps(domain_name_resp) @@ -579,7 +576,7 @@ class APIGatewayResponse(BaseResponse): url_path_parts = self.path.split("/") domain_name = url_path_parts[2] - domain_names={} + domain_names = {} if self.method == "GET": if domain_name is not None: diff --git a/tests/test_apigateway/test_apigateway.py b/tests/test_apigateway/test_apigateway.py index 22e062cc5..29b07b7ab 100644 --- a/tests/test_apigateway/test_apigateway.py +++ b/tests/test_apigateway/test_apigateway.py @@ -1489,8 +1489,11 @@ def test_create_domain_names(): domain_name = "testDomain" test_certificate_name = "test.certificate" test_certificate_private_key = "testPrivateKey" - response = client.create_domain_name(domainName=domain_name, certificateName=test_certificate_name, - certificatePrivateKey=test_certificate_private_key) + response = client.create_domain_name( + domainName=domain_name, + certificateName=test_certificate_name, + certificatePrivateKey=test_certificate_private_key, + ) response["domainName"].should.equal(domain_name) response["certificateName"].should.equal(test_certificate_name) From 5c7e0b56afaeeafda053f9f15022d4520f9b2e2e Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 8 Apr 2020 13:53:53 +0100 Subject: [PATCH 32/83] #2877 - Ensure NetworkInterfaces are assigned to the default Subnet --- moto/ec2/models.py | 9 +++++- tests/test_ec2/test_instances.py | 2 +- tests/test_ec2/test_subnets.py | 47 ++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index be39bab28..2611c2f1a 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -775,7 +775,14 @@ class Instance(TaggedEC2Resource, BotoInstance): if "SubnetId" in nic: subnet = self.ec2_backend.get_subnet(nic["SubnetId"]) else: - subnet = None + # Get default Subnet + subnet = [ + subnet + for subnet in self.ec2_backend.get_all_subnets( + filters={"availabilityZone": self._placement.zone} + ) + if subnet.default_for_az + ][0] group_id = nic.get("SecurityGroupId") group_ids = [group_id] if group_id else [] diff --git a/tests/test_ec2/test_instances.py b/tests/test_ec2/test_instances.py index 85ba0fe01..fe1631223 100644 --- a/tests/test_ec2/test_instances.py +++ b/tests/test_ec2/test_instances.py @@ -71,7 +71,7 @@ def test_instance_launch_and_terminate(): instance.id.should.equal(instance.id) instance.state.should.equal("running") instance.launch_time.should.equal("2014-01-01T05:00:00.000Z") - instance.vpc_id.should.equal(None) + instance.vpc_id.shouldnt.equal(None) instance.placement.should.equal("us-east-1a") root_device_name = instance.root_device_name diff --git a/tests/test_ec2/test_subnets.py b/tests/test_ec2/test_subnets.py index 7bb57aab4..a16693d5f 100644 --- a/tests/test_ec2/test_subnets.py +++ b/tests/test_ec2/test_subnets.py @@ -599,3 +599,50 @@ def validate_subnet_details_after_creating_eni( for eni in enis_created: client.delete_network_interface(NetworkInterfaceId=eni["NetworkInterfaceId"]) client.delete_subnet(SubnetId=subnet["SubnetId"]) + + +@mock_ec2 +def test_run_instances_should_attach_to_default_subnet(): + ec2 = boto3.resource("ec2", region_name="us-west-1") + client = boto3.client("ec2", region_name="us-west-1") + ec2.create_security_group(GroupName="sg01", Description="Test security group sg01") + # run_instances + instances = client.run_instances( + MinCount=1, + MaxCount=1, + SecurityGroups=["sg01"], + TagSpecifications=[ + { + "ResourceType": "instance", + "Tags": [{"Key": "Name", "Value": "test-01"},], + } + ], + ) + default_subnet_id = client.describe_subnets()["Subnets"][0]["SubnetId"] + instances["Instances"][0]["NetworkInterfaces"][0]["SubnetId"].should.equal( + default_subnet_id + ) + + +@mock_ec2 +def test_describe_subnets_where_network_interface_has_no_subnets_attached(): + # https://github.com/spulec/moto/issues/2877 + # create security groups + ec2 = boto3.resource("ec2", region_name="us-west-1") + client = boto3.client("ec2", region_name="us-west-1") + ec2.create_security_group(GroupName="sg01", Description="Test security group sg01") + # run_instances + client.run_instances( + MinCount=1, + MaxCount=1, + SecurityGroups=["sg01"], + TagSpecifications=[ + { + "ResourceType": "instance", + "Tags": [{"Key": "Name", "Value": "test-01"},], + } + ], + ) + # describe_subnets + subnets = client.describe_subnets()["Subnets"] + subnets[0]["AvailableIpAddressCount"].should.equal(4090) From 8475804a8b37ea11fcdd5acde02dec1f6ca31b9b Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 8 Apr 2020 14:02:35 +0100 Subject: [PATCH 33/83] Simplify tests --- tests/test_ec2/test_subnets.py | 40 +++++----------------------------- 1 file changed, 5 insertions(+), 35 deletions(-) diff --git a/tests/test_ec2/test_subnets.py b/tests/test_ec2/test_subnets.py index a16693d5f..eae0bc468 100644 --- a/tests/test_ec2/test_subnets.py +++ b/tests/test_ec2/test_subnets.py @@ -603,46 +603,16 @@ def validate_subnet_details_after_creating_eni( @mock_ec2 def test_run_instances_should_attach_to_default_subnet(): + # https://github.com/spulec/moto/issues/2877 ec2 = boto3.resource("ec2", region_name="us-west-1") client = boto3.client("ec2", region_name="us-west-1") ec2.create_security_group(GroupName="sg01", Description="Test security group sg01") # run_instances - instances = client.run_instances( - MinCount=1, - MaxCount=1, - SecurityGroups=["sg01"], - TagSpecifications=[ - { - "ResourceType": "instance", - "Tags": [{"Key": "Name", "Value": "test-01"},], - } - ], - ) - default_subnet_id = client.describe_subnets()["Subnets"][0]["SubnetId"] + instances = client.run_instances(MinCount=1, MaxCount=1, SecurityGroups=["sg01"],) + # Assert subnet is created appropriately + subnets = client.describe_subnets()["Subnets"] + default_subnet_id = subnets[0]["SubnetId"] instances["Instances"][0]["NetworkInterfaces"][0]["SubnetId"].should.equal( default_subnet_id ) - - -@mock_ec2 -def test_describe_subnets_where_network_interface_has_no_subnets_attached(): - # https://github.com/spulec/moto/issues/2877 - # create security groups - ec2 = boto3.resource("ec2", region_name="us-west-1") - client = boto3.client("ec2", region_name="us-west-1") - ec2.create_security_group(GroupName="sg01", Description="Test security group sg01") - # run_instances - client.run_instances( - MinCount=1, - MaxCount=1, - SecurityGroups=["sg01"], - TagSpecifications=[ - { - "ResourceType": "instance", - "Tags": [{"Key": "Name", "Value": "test-01"},], - } - ], - ) - # describe_subnets - subnets = client.describe_subnets()["Subnets"] subnets[0]["AvailableIpAddressCount"].should.equal(4090) From 414fcf7bbd0ac261b83928f5eec9166ef4748aa3 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 8 Apr 2020 15:14:39 +0100 Subject: [PATCH 34/83] Fix AvailibilityZones in CF tests --- .../test_cloudformation_stack_integration.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index e50179660..67ef0af9b 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -495,7 +495,7 @@ def test_autoscaling_group_with_elb(): "my-as-group": { "Type": "AWS::AutoScaling::AutoScalingGroup", "Properties": { - "AvailabilityZones": ["us-east1"], + "AvailabilityZones": ["us-east-1a"], "LaunchConfigurationName": {"Ref": "my-launch-config"}, "MinSize": "2", "MaxSize": "2", @@ -522,7 +522,7 @@ def test_autoscaling_group_with_elb(): "my-elb": { "Type": "AWS::ElasticLoadBalancing::LoadBalancer", "Properties": { - "AvailabilityZones": ["us-east1"], + "AvailabilityZones": ["us-east-1a"], "Listeners": [ { "LoadBalancerPort": "80", @@ -545,10 +545,10 @@ def test_autoscaling_group_with_elb(): web_setup_template_json = json.dumps(web_setup_template) - conn = boto.cloudformation.connect_to_region("us-west-1") + conn = boto.cloudformation.connect_to_region("us-east-1") conn.create_stack("web_stack", template_body=web_setup_template_json) - autoscale_conn = boto.ec2.autoscale.connect_to_region("us-west-1") + autoscale_conn = boto.ec2.autoscale.connect_to_region("us-east-1") autoscale_group = autoscale_conn.get_all_groups()[0] autoscale_group.launch_config_name.should.contain("my-launch-config") autoscale_group.load_balancers[0].should.equal("my-elb") @@ -557,7 +557,7 @@ def test_autoscaling_group_with_elb(): autoscale_conn.get_all_launch_configurations().should.have.length_of(1) # Confirm the ELB was actually created - elb_conn = boto.ec2.elb.connect_to_region("us-west-1") + elb_conn = boto.ec2.elb.connect_to_region("us-east-1") elb_conn.get_all_load_balancers().should.have.length_of(1) stack = conn.describe_stacks()[0] @@ -584,7 +584,7 @@ def test_autoscaling_group_with_elb(): elb_resource.physical_resource_id.should.contain("my-elb") # confirm the instances were created with the right tags - ec2_conn = boto.ec2.connect_to_region("us-west-1") + ec2_conn = boto.ec2.connect_to_region("us-east-1") reservations = ec2_conn.get_all_reservations() len(reservations).should.equal(1) reservation = reservations[0] @@ -604,7 +604,7 @@ def test_autoscaling_group_update(): "my-as-group": { "Type": "AWS::AutoScaling::AutoScalingGroup", "Properties": { - "AvailabilityZones": ["us-west-1"], + "AvailabilityZones": ["us-west-1a"], "LaunchConfigurationName": {"Ref": "my-launch-config"}, "MinSize": "2", "MaxSize": "2", From af57cfc7ec122668d492964dbe29f49afb60f26f Mon Sep 17 00:00:00 2001 From: usmankb Date: Wed, 8 Apr 2020 21:54:26 +0530 Subject: [PATCH 35/83] Added more tests and coverage --- moto/apigateway/exceptions.py | 10 +++++ moto/apigateway/models.py | 22 ++++++---- moto/apigateway/responses.py | 43 ++++++++++++++------ tests/test_apigateway/test_apigateway.py | 51 +++++++++++++++++++++++- 4 files changed, 103 insertions(+), 23 deletions(-) diff --git a/moto/apigateway/exceptions.py b/moto/apigateway/exceptions.py index 24f06f3f1..c9c90cea5 100644 --- a/moto/apigateway/exceptions.py +++ b/moto/apigateway/exceptions.py @@ -120,6 +120,16 @@ class ApiKeyAlreadyExists(RESTError): "ConflictException", "API Key already exists" ) + +class InvalidDomainName(BadRequestException): + code = 404 + + def __init__(self): + super(InvalidDomainName, self).__init__( + "BadRequestException", "No Domain Name specified" + ) + + class DomainNameNotFound(RESTError): code = 404 diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index 7ca7e6315..7707bd9d5 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -34,7 +34,8 @@ from .exceptions import ( NoIntegrationDefined, NoMethodDefined, ApiKeyAlreadyExists, - DomainNameNotFound + DomainNameNotFound, + InvalidDomainName ) STAGE_URL = "https://{api_id}.execute-api.{region_name}.amazonaws.com/{stage_name}" @@ -1047,25 +1048,26 @@ class APIGatewayBackend(BaseBackend): except Exception: return False - def create_domain_name(self, domain_name, - certificate_name=None, tags=None, - certificate_arn=None, certificate_body=None, - certificate_private_key=None, + def create_domain_name(self,domain_name, + certificate_name=None,certificate_private_key=None, + tags=None, certificate_arn=None, + certificate_body=None, certificate_chain=None, regional_certificate_name=None, regional_certificate_arn=None, endpoint_configuration=None, security_policy=None, generate_cli_skeleton=None): + if not domain_name: - raise DomainNameNotFound() + raise InvalidDomainName() new_domain_name = DomainName( domain_name=domain_name, certificate_name=certificate_name, + certificate_private_key=certificate_private_key, certificate_arn=certificate_arn, certificate_body=certificate_body, - certificate_private_key=certificate_private_key, certificate_chain=certificate_chain, regional_certificate_name=regional_certificate_name, regional_certificate_arn=regional_certificate_arn, @@ -1081,7 +1083,11 @@ class APIGatewayBackend(BaseBackend): return list(self.domain_names.values()) def get_domain_name(self, domain_name): - return self.domain_names[domain_name] + domain_info = self.domain_names.get(domain_name) + if domain_info is None: + raise DomainNameNotFound + else: + return self.domain_names[domain_name] apigateway_backends = {} diff --git a/moto/apigateway/responses.py b/moto/apigateway/responses.py index ce3bcbb8e..ec05c605e 100644 --- a/moto/apigateway/responses.py +++ b/moto/apigateway/responses.py @@ -11,6 +11,8 @@ from .exceptions import ( AuthorizerNotFoundException, StageNotFoundException, ApiKeyAlreadyExists, + DomainNameNotFound, + InvalidDomainName ) API_KEY_SOURCES = ["AUTHORIZER", "HEADER"] @@ -561,17 +563,22 @@ class APIGatewayResponse(BaseResponse): "generateCliSkeleton" ) domain_name_resp = self.backend.create_domain_name( - domain_name, certificate_name, tags, certificate_arn, - certificate_body, certificate_private_key, - certificate_chain, regional_certificate_name, - regional_certificate_arn, endpoint_configuration, - security_policy, generate_cli_skeleton + domain_name, certificate_name, + certificate_private_key,tags, certificate_arn, + certificate_body, certificate_chain, + regional_certificate_name, regional_certificate_arn, + endpoint_configuration, security_policy, + generate_cli_skeleton ) - return 200, {}, json.dumps(domain_name_resp) - except BadRequestException as e: - return self.error( - "com.amazonaws.dynamodb.v20111205#BadRequestException", e.message + + except InvalidDomainName as error: + return ( + error.code, + {}, + '{{"message":"{0}","code":"{1}"}}'.format( + error.message, error.error_type + ), ) def domain_name_induvidual(self, request, full_url, headers): @@ -580,9 +587,19 @@ class APIGatewayResponse(BaseResponse): url_path_parts = self.path.split("/") domain_name = url_path_parts[2] domain_names={} + try: + if self.method == "GET": + if domain_name is not None: + domain_names = self.backend.get_domain_name(domain_name) + return 200, {}, json.dumps(domain_names) + + except DomainNameNotFound as error: + return ( + error.code, + {}, + '{{"message":"{0}","code":"{1}"}}'.format( + error.message, error.error_type + ), + ) - if self.method == "GET": - if domain_name is not None: - domain_names = self.backend.get_domain_name(domain_name) - return 200, {}, json.dumps(domain_names) diff --git a/tests/test_apigateway/test_apigateway.py b/tests/test_apigateway/test_apigateway.py index 22e062cc5..accd0fad2 100644 --- a/tests/test_apigateway/test_apigateway.py +++ b/tests/test_apigateway/test_apigateway.py @@ -1489,15 +1489,62 @@ def test_create_domain_names(): domain_name = "testDomain" test_certificate_name = "test.certificate" test_certificate_private_key = "testPrivateKey" - response = client.create_domain_name(domainName=domain_name, certificateName=test_certificate_name, - certificatePrivateKey=test_certificate_private_key) + # success case with valid params + response = client.create_domain_name(domainName=domain_name, + certificateName=test_certificate_name, + certificatePrivateKey=test_certificate_private_key) response["domainName"].should.equal(domain_name) response["certificateName"].should.equal(test_certificate_name) + # without domain name it should throw BadRequestException + with assert_raises(ClientError) as ex: + client.create_domain_name(domainName="") + + ex.exception.response["Error"]["Message"].should.equal( + "No Domain Name specified") + ex.exception.response["Error"]["Code"].should.equal( + "BadRequestException") + + +@mock_apigateway +def test_get_domain_names(): + client = boto3.client("apigateway", region_name="us-west-2") + # without any domain names already present + result = client.get_domain_names() + result["items"].should.equal([]) + domain_name = "testDomain" + test_certificate_name = "test.certificate" + response = client.create_domain_name(domainName=domain_name, + certificateName=test_certificate_name) + + response["domainName"].should.equal(domain_name) + response["certificateName"].should.equal(test_certificate_name) + response["domainNameStatus"].should.equal("AVAILABLE") + # after adding a new domain name result = client.get_domain_names() result["items"][0]["domainName"].should.equal(domain_name) + result["items"][0]["certificateName"].should.equal(test_certificate_name) + result["items"][0]["domainNameStatus"].should.equal("AVAILABLE") + + +@mock_apigateway +def test_get_domain_name(): + client = boto3.client("apigateway", region_name="us-west-2") + domain_name = "testDomain" + # quering an invalid domain name which is not present + with assert_raises(ClientError) as ex: + client.get_domain_name(domainName=domain_name) + + ex.exception.response["Error"]["Message"].should.equal( + "Invalid Domain Name specified") + ex.exception.response["Error"]["Code"].should.equal( + "NotFoundException") + # adding a domain name + client.create_domain_name(domainName=domain_name) + # retrieving the data of added domain name. result = client.get_domain_name(domainName=domain_name) result["domainName"].should.equal(domain_name) + result["domainNameStatus"].should.equal("AVAILABLE") @mock_apigateway From 82311087f442dea0ea9f177efcce1851e66c2f18 Mon Sep 17 00:00:00 2001 From: usmankb Date: Wed, 8 Apr 2020 22:04:48 +0530 Subject: [PATCH 36/83] linting --- moto/apigateway/responses.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/moto/apigateway/responses.py b/moto/apigateway/responses.py index e27982b2a..8bef1f13d 100644 --- a/moto/apigateway/responses.py +++ b/moto/apigateway/responses.py @@ -591,7 +591,7 @@ class APIGatewayResponse(BaseResponse): url_path_parts = self.path.split("/") domain_name = url_path_parts[2] - domain_names={} + domain_names = {} try: if self.method == "GET": if domain_name is not None: @@ -605,6 +605,3 @@ class APIGatewayResponse(BaseResponse): error.message, error.error_type ), ) - - - From 1654280e43fe227faa3df79836182e5089557fc2 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Thu, 9 Apr 2020 08:12:44 +0100 Subject: [PATCH 37/83] Linting --- moto/apigateway/models.py | 2 +- moto/apigateway/responses.py | 22 ++++++---------------- tests/test_apigateway/test_apigateway.py | 17 ++++++++--------- 3 files changed, 15 insertions(+), 26 deletions(-) diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index d376db5ce..16462e278 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -35,7 +35,7 @@ from .exceptions import ( NoMethodDefined, ApiKeyAlreadyExists, DomainNameNotFound, - InvalidDomainName + InvalidDomainName, ) STAGE_URL = "https://{api_id}.execute-api.{region_name}.amazonaws.com/{stage_name}" diff --git a/moto/apigateway/responses.py b/moto/apigateway/responses.py index 8bef1f13d..e4723f0d4 100644 --- a/moto/apigateway/responses.py +++ b/moto/apigateway/responses.py @@ -12,7 +12,7 @@ from .exceptions import ( StageNotFoundException, ApiKeyAlreadyExists, DomainNameNotFound, - InvalidDomainName + InvalidDomainName, ) API_KEY_SOURCES = ["AUTHORIZER", "HEADER"] @@ -544,23 +544,13 @@ class APIGatewayResponse(BaseResponse): tags = self._get_param("tags") certificate_arn = self._get_param("certificateArn") certificate_body = self._get_param("certificateBody") - certificate_private_key = self._get_param( - "certificatePrivateKey" - ) + certificate_private_key = self._get_param("certificatePrivateKey") certificate_chain = self._get_param("certificateChain") - regional_certificate_name = self._get_param( - "regionalCertificateName" - ) - regional_certificate_arn = self._get_param( - "regionalCertificateArn" - ) - endpoint_configuration = self._get_param( - "endpointConfiguration" - ) + regional_certificate_name = self._get_param("regionalCertificateName") + regional_certificate_arn = self._get_param("regionalCertificateArn") + endpoint_configuration = self._get_param("endpointConfiguration") security_policy = self._get_param("securityPolicy") - generate_cli_skeleton = self._get_param( - "generateCliSkeleton" - ) + generate_cli_skeleton = self._get_param("generateCliSkeleton") domain_name_resp = self.backend.create_domain_name( domain_name, certificate_name, diff --git a/tests/test_apigateway/test_apigateway.py b/tests/test_apigateway/test_apigateway.py index ec1049ac5..a1a380974 100644 --- a/tests/test_apigateway/test_apigateway.py +++ b/tests/test_apigateway/test_apigateway.py @@ -1501,10 +1501,8 @@ def test_create_domain_names(): with assert_raises(ClientError) as ex: client.create_domain_name(domainName="") - ex.exception.response["Error"]["Message"].should.equal( - "No Domain Name specified") - ex.exception.response["Error"]["Code"].should.equal( - "BadRequestException") + ex.exception.response["Error"]["Message"].should.equal("No Domain Name specified") + ex.exception.response["Error"]["Code"].should.equal("BadRequestException") @mock_apigateway @@ -1515,8 +1513,9 @@ def test_get_domain_names(): result["items"].should.equal([]) domain_name = "testDomain" test_certificate_name = "test.certificate" - response = client.create_domain_name(domainName=domain_name, - certificateName=test_certificate_name) + response = client.create_domain_name( + domainName=domain_name, certificateName=test_certificate_name + ) response["domainName"].should.equal(domain_name) response["certificateName"].should.equal(test_certificate_name) @@ -1537,9 +1536,9 @@ def test_get_domain_name(): client.get_domain_name(domainName=domain_name) ex.exception.response["Error"]["Message"].should.equal( - "Invalid Domain Name specified") - ex.exception.response["Error"]["Code"].should.equal( - "NotFoundException") + "Invalid Domain Name specified" + ) + ex.exception.response["Error"]["Code"].should.equal("NotFoundException") # adding a domain name client.create_domain_name(domainName=domain_name) # retrieving the data of added domain name. From 7a9cdd4fd24978ba127e0c41a558d2ef740e0975 Mon Sep 17 00:00:00 2001 From: usmankb Date: Sat, 11 Apr 2020 08:37:00 +0530 Subject: [PATCH 38/83] Adding missing Param zoneId in the describe-availability-zone --- moto/ec2/responses/availability_zones_and_regions.py | 1 + .../test_ec2/test_availability_zones_and_regions.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/moto/ec2/responses/availability_zones_and_regions.py b/moto/ec2/responses/availability_zones_and_regions.py index d63e2f4ad..28cc3a495 100644 --- a/moto/ec2/responses/availability_zones_and_regions.py +++ b/moto/ec2/responses/availability_zones_and_regions.py @@ -35,6 +35,7 @@ DESCRIBE_ZONES_RESPONSE = """ Date: Sun, 12 Apr 2020 13:44:16 +1000 Subject: [PATCH 39/83] Add instance-id filter to describe_auto_scaling_instances --- moto/autoscaling/models.py | 6 +++-- moto/autoscaling/responses.py | 4 +++- tests/test_autoscaling/test_autoscaling.py | 28 +++++++++++++++++++++- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/moto/autoscaling/models.py b/moto/autoscaling/models.py index 45ee7d192..84ae9c76b 100644 --- a/moto/autoscaling/models.py +++ b/moto/autoscaling/models.py @@ -655,10 +655,12 @@ class AutoScalingBackend(BaseBackend): self.set_desired_capacity(group_name, 0) self.autoscaling_groups.pop(group_name, None) - def describe_auto_scaling_instances(self): + def describe_auto_scaling_instances(self, instance_ids): instance_states = [] for group in self.autoscaling_groups.values(): - instance_states.extend(group.instance_states) + instance_states.extend( + [x for x in group.instance_states if not instance_ids or x.instance.id in instance_ids] + ) return instance_states def attach_instances(self, group_name, instance_ids): diff --git a/moto/autoscaling/responses.py b/moto/autoscaling/responses.py index 83e2f7d5a..41c79edb4 100644 --- a/moto/autoscaling/responses.py +++ b/moto/autoscaling/responses.py @@ -226,7 +226,9 @@ class AutoScalingResponse(BaseResponse): return template.render() def describe_auto_scaling_instances(self): - instance_states = self.autoscaling_backend.describe_auto_scaling_instances() + instance_states = self.autoscaling_backend.describe_auto_scaling_instances( + instance_ids=self._get_multi_param("InstanceIds.member") + ) template = self.response_template(DESCRIBE_AUTOSCALING_INSTANCES_TEMPLATE) return template.render(instance_states=instance_states) diff --git a/tests/test_autoscaling/test_autoscaling.py b/tests/test_autoscaling/test_autoscaling.py index 2e7255381..094708ec9 100644 --- a/tests/test_autoscaling/test_autoscaling.py +++ b/tests/test_autoscaling/test_autoscaling.py @@ -843,13 +843,39 @@ def test_describe_autoscaling_instances_boto3(): NewInstancesProtectedFromScaleIn=True, ) + response = client.describe_auto_scaling_instances() + len(response["AutoScalingInstances"]).should.equal(5) + for instance in response["AutoScalingInstances"]: + instance["AutoScalingGroupName"].should.equal("test_asg") + instance["AvailabilityZone"].should.equal("us-east-1a") + instance["ProtectedFromScaleIn"].should.equal(True) + + +@mock_autoscaling +def test_describe_autoscaling_instances_instanceid_filter(): + mocked_networking = setup_networking() + client = boto3.client("autoscaling", region_name="us-east-1") + _ = client.create_launch_configuration( + LaunchConfigurationName="test_launch_configuration" + ) + _ = client.create_auto_scaling_group( + AutoScalingGroupName="test_asg", + LaunchConfigurationName="test_launch_configuration", + MinSize=0, + MaxSize=20, + DesiredCapacity=5, + VPCZoneIdentifier=mocked_networking["subnet1"], + NewInstancesProtectedFromScaleIn=True, + ) + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) instance_ids = [ instance["InstanceId"] for instance in response["AutoScalingGroups"][0]["Instances"] ] - response = client.describe_auto_scaling_instances(InstanceIds=instance_ids) + response = client.describe_auto_scaling_instances(InstanceIds=instance_ids[0:2]) # Filter by first 2 of 5 + len(response["AutoScalingInstances"]).should.equal(2) for instance in response["AutoScalingInstances"]: instance["AutoScalingGroupName"].should.equal("test_asg") instance["AvailabilityZone"].should.equal("us-east-1a") From 965046aa39bce3c57d3648bdadc21c8b97599a3b Mon Sep 17 00:00:00 2001 From: DenverJ Date: Sun, 12 Apr 2020 17:08:40 +1000 Subject: [PATCH 40/83] Fix formatting --- moto/autoscaling/models.py | 6 +++++- tests/test_autoscaling/test_autoscaling.py | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/moto/autoscaling/models.py b/moto/autoscaling/models.py index 84ae9c76b..88577433e 100644 --- a/moto/autoscaling/models.py +++ b/moto/autoscaling/models.py @@ -659,7 +659,11 @@ class AutoScalingBackend(BaseBackend): instance_states = [] for group in self.autoscaling_groups.values(): instance_states.extend( - [x for x in group.instance_states if not instance_ids or x.instance.id in instance_ids] + [ + x + for x in group.instance_states + if not instance_ids or x.instance.id in instance_ids + ] ) return instance_states diff --git a/tests/test_autoscaling/test_autoscaling.py b/tests/test_autoscaling/test_autoscaling.py index 094708ec9..5cf3dc6ff 100644 --- a/tests/test_autoscaling/test_autoscaling.py +++ b/tests/test_autoscaling/test_autoscaling.py @@ -874,7 +874,9 @@ def test_describe_autoscaling_instances_instanceid_filter(): for instance in response["AutoScalingGroups"][0]["Instances"] ] - response = client.describe_auto_scaling_instances(InstanceIds=instance_ids[0:2]) # Filter by first 2 of 5 + response = client.describe_auto_scaling_instances( + InstanceIds=instance_ids[0:2] + ) # Filter by first 2 of 5 len(response["AutoScalingInstances"]).should.equal(2) for instance in response["AutoScalingInstances"]: instance["AutoScalingGroupName"].should.equal("test_asg") From 79e63e3bcff9f8ba4f0246ed1c4310191bed6d83 Mon Sep 17 00:00:00 2001 From: usmankb Date: Sun, 12 Apr 2020 17:49:22 +0530 Subject: [PATCH 41/83] Added implementation for create-model,get-models,get-model in api gateway --- moto/apigateway/exceptions.py | 36 ++++++ moto/apigateway/models.py | 90 +++++++++++++- moto/apigateway/responses.py | 68 +++++++++++ moto/apigateway/urls.py | 2 + tests/test_apigateway/test_apigateway.py | 142 +++++++++++++++++++++++ 5 files changed, 337 insertions(+), 1 deletion(-) diff --git a/moto/apigateway/exceptions.py b/moto/apigateway/exceptions.py index c9c90cea5..8f6d21aa0 100644 --- a/moto/apigateway/exceptions.py +++ b/moto/apigateway/exceptions.py @@ -137,3 +137,39 @@ class DomainNameNotFound(RESTError): super(DomainNameNotFound, self).__init__( "NotFoundException", "Invalid Domain Name specified" ) + + +class InvalidRestApiId(BadRequestException): + code = 404 + + def __init__(self): + super(InvalidRestApiId, self).__init__( + "BadRequestException", "No Rest API Id specified" + ) + + +class InvalidModelName(BadRequestException): + code = 404 + + def __init__(self): + super(InvalidModelName, self).__init__( + "BadRequestException", "No Model Name specified" + ) + + +class RestAPINotFound(RESTError): + code = 404 + + def __init__(self): + super(RestAPINotFound, self).__init__( + "NotFoundException", "Invalid Rest API Id specified" + ) + + +class ModelNotFound(RESTError): + code = 404 + + def __init__(self): + super(ModelNotFound, self).__init__( + "NotFoundException", "Invalid Model Name specified" + ) diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index 16462e278..5ce95742e 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -36,6 +36,10 @@ from .exceptions import ( ApiKeyAlreadyExists, DomainNameNotFound, InvalidDomainName, + InvalidRestApiId, + InvalidModelName, + RestAPINotFound, + ModelNotFound ) STAGE_URL = "https://{api_id}.execute-api.{region_name}.amazonaws.com/{stage_name}" @@ -466,6 +470,7 @@ class RestAPI(BaseModel): self.authorizers = {} self.stages = {} self.resources = {} + self.models = {} self.add_child("/") # Add default child def __repr__(self): @@ -494,6 +499,27 @@ class RestAPI(BaseModel): self.resources[child_id] = child return child + def add_model(self, + name, + description=None, + schema=None, + content_type=None, + cli_input_json=None, + generate_cli_skeleton=None): + model_id = create_id() + new_model = Model( + id=model_id, + name=name, + description=description, + schema=schema, + content_type=content_type, + cli_input_json=cli_input_json, + generate_cli_skeleton=generate_cli_skeleton) + + self.models[name] = new_model + return new_model + + def get_resource_for_path(self, path_after_stage_name): for resource in self.resources.values(): if resource.get_path() == path_after_stage_name: @@ -645,6 +671,24 @@ class DomainName(BaseModel, dict): self["generateCliSkeleton"] = kwargs.get("generate_cli_skeleton") +class Model(BaseModel,dict): + def __init__(self, id, name, **kwargs): + super(Model, self).__init__() + self["id"] = id + self["name"] = name + if kwargs.get("description"): + self["description"] = kwargs.get("description") + if kwargs.get("schema"): + self["schema"] = kwargs.get("schema") + if kwargs.get("content_type"): + self["contentType"] = kwargs.get("content_type") + if kwargs.get("cli_input_json"): + self["cliInputJson"] = kwargs.get("cli_input_json") + if kwargs.get("generate_cli_skeleton"): + self["generateCliSkeleton"] = kwargs.get("generate_cli_skeleton") + + + class APIGatewayBackend(BaseBackend): def __init__(self, region_name): super(APIGatewayBackend, self).__init__() @@ -653,6 +697,7 @@ class APIGatewayBackend(BaseBackend): self.usage_plans = {} self.usage_plan_keys = {} self.domain_names = {} + self.models = {} self.region_name = region_name def reset(self): @@ -682,7 +727,9 @@ class APIGatewayBackend(BaseBackend): return rest_api def get_rest_api(self, function_id): - rest_api = self.apis[function_id] + rest_api = self.apis.get(function_id) + if rest_api is None: + raise RestAPINotFound() return rest_api def list_apis(self): @@ -1085,6 +1132,47 @@ class APIGatewayBackend(BaseBackend): else: return self.domain_names[domain_name] + def create_model(self, + rest_api_id, + name, + content_type, + description=None, + schema=None, + cli_input_json=None, + generate_cli_skeleton=None): + + if not rest_api_id: + raise InvalidRestApiId + if not name: + raise InvalidModelName + + api = self.get_rest_api(rest_api_id) + new_model = api.add_model( + name=name, + description=description, + schema=schema, + content_type=content_type, + cli_input_json=cli_input_json, + generate_cli_skeleton=generate_cli_skeleton) + + return new_model + + def get_models(self, rest_api_id): + if not rest_api_id: + raise InvalidRestApiId + api = self.get_rest_api(rest_api_id) + models = api.models.values() + return list(models) + + def get_model(self, rest_api_id, model_name): + if not rest_api_id: + raise InvalidRestApiId + api = self.get_rest_api(rest_api_id) + model = api.models.get(model_name) + if model is None: + raise ModelNotFound + return model + apigateway_backends = {} for region_name in Session().get_available_regions("apigateway"): diff --git a/moto/apigateway/responses.py b/moto/apigateway/responses.py index e4723f0d4..c18b7f6c4 100644 --- a/moto/apigateway/responses.py +++ b/moto/apigateway/responses.py @@ -13,6 +13,10 @@ from .exceptions import ( ApiKeyAlreadyExists, DomainNameNotFound, InvalidDomainName, + InvalidRestApiId, + InvalidModelName, + RestAPINotFound, + ModelNotFound ) API_KEY_SOURCES = ["AUTHORIZER", "HEADER"] @@ -595,3 +599,67 @@ class APIGatewayResponse(BaseResponse): error.message, error.error_type ), ) + + def models(self,request, full_url, headers): + self.setup_class(request, full_url, headers) + rest_api_id = self.path.replace("/restapis/", "", 1).split("/")[0] + + try: + if self.method == "GET": + models = self.backend.get_models( + rest_api_id + ) + return 200, {}, json.dumps({"item": models}) + + elif self.method == "POST": + name = self._get_param("name") + description = self._get_param("description") + schema = self._get_param("schema") + content_type = self._get_param("contentType") + cli_input_json = self._get_param("cliInputJson") + generate_cli_skeleton = self._get_param( + "generateCliSkeleton" + ) + model = self.backend.create_model( + rest_api_id, + name, + content_type, + description, + schema, + cli_input_json, + generate_cli_skeleton + ) + + return 200, {}, json.dumps(model) + + except (InvalidRestApiId, InvalidModelName,RestAPINotFound) as error: + return ( + error.code, + {}, + '{{"message":"{0}","code":"{1}"}}'.format( + error.message, error.error_type + ), + ) + + def model_induvidual(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + url_path_parts = self.path.split("/") + rest_api_id = url_path_parts[2] + model_name = url_path_parts[4] + model_info = {} + try: + if self.method == "GET": + model_info = self.backend.get_model( + rest_api_id, + model_name + ) + return 200, {}, json.dumps(model_info) + except (ModelNotFound, RestAPINotFound, InvalidRestApiId, + InvalidModelName) as error: + return ( + error.code, + {}, + '{{"message":"{0}","code":"{1}"}}'.format( + error.message, error.error_type + ), + ) \ No newline at end of file diff --git a/moto/apigateway/urls.py b/moto/apigateway/urls.py index 6c3b7f6bb..751d8ae65 100644 --- a/moto/apigateway/urls.py +++ b/moto/apigateway/urls.py @@ -22,6 +22,8 @@ url_paths = { "{0}/apikeys/(?P[^/]+)": APIGatewayResponse().apikey_individual, "{0}/usageplans$": APIGatewayResponse().usage_plans, "{0}/domainnames$": APIGatewayResponse().domain_names, + "{0}/restapis/(?P[^/]+)/models": APIGatewayResponse().models, + "{0}/restapis/(?P[^/]+)/models/(?P[^/]+)/?$": APIGatewayResponse().model_induvidual, "{0}/domainnames/(?P[^/]+)/?$": APIGatewayResponse().domain_name_induvidual, "{0}/usageplans/(?P[^/]+)/?$": APIGatewayResponse().usage_plan_individual, "{0}/usageplans/(?P[^/]+)/keys$": APIGatewayResponse().usage_plan_keys, diff --git a/tests/test_apigateway/test_apigateway.py b/tests/test_apigateway/test_apigateway.py index a1a380974..3a6b75104 100644 --- a/tests/test_apigateway/test_apigateway.py +++ b/tests/test_apigateway/test_apigateway.py @@ -1547,6 +1547,148 @@ def test_get_domain_name(): result["domainNameStatus"].should.equal("AVAILABLE") +@mock_apigateway +def test_create_model(): + client = boto3.client("apigateway", region_name="us-west-2") + response = client.create_rest_api(name="my_api", + description="this is my api" + ) + rest_api_id = response["id"] + dummy_rest_api_id = 'a12b3c4d' + model_name = "testModel" + description = "test model" + content_type = 'application/json' + # success case with valid params + response = client.create_model( + restApiId=rest_api_id, + name=model_name, + description=description, + contentType=content_type + ) + response["name"].should.equal(model_name) + response["description"].should.equal(description) + + # with an invalid rest_api_id it should throw NotFoundException + with assert_raises(ClientError) as ex: + client.create_model( + restApiId=dummy_rest_api_id, + name=model_name, + description=description, + contentType=content_type + ) + ex.exception.response["Error"]["Message"].should.equal( + "Invalid Rest API Id specified" + ) + ex.exception.response["Error"]["Code"].should.equal( + "NotFoundException" + ) + + with assert_raises(ClientError) as ex: + client.create_model( + restApiId=rest_api_id, + name="", + description=description, + contentType=content_type + ) + + ex.exception.response["Error"]["Message"].should.equal( + "No Model Name specified" + ) + ex.exception.response["Error"]["Code"].should.equal( + "BadRequestException" + ) + + +@mock_apigateway +def test_get_api_models(): + client = boto3.client("apigateway", region_name="us-west-2") + response = client.create_rest_api( + name="my_api", + description="this is my api" + ) + rest_api_id = response["id"] + model_name = "testModel" + description = "test model" + content_type = 'application/json' + # when no models are present + result = client.get_models( + restApiId=rest_api_id + ) + result["items"].should.equal([]) + # add a model + client.create_model( + restApiId=rest_api_id, + name=model_name, + description=description, + contentType=content_type + ) + # get models after adding + result = client.get_models( + restApiId=rest_api_id + ) + result["items"][0]["name"] = model_name + result["items"][0]["description"] = description + + +@mock_apigateway +def test_get_model_by_name(): + client = boto3.client("apigateway", region_name="us-west-2") + response = client.create_rest_api( + name="my_api", + description="this is my api" + ) + rest_api_id = response["id"] + dummy_rest_api_id = 'a12b3c4d' + model_name = "testModel" + description = "test model" + content_type = 'application/json' + # add a model + client.create_model( + restApiId=rest_api_id, + name=model_name, + description=description, + contentType=content_type + ) + # get models after adding + result = client.get_model( + restApiId=rest_api_id, modelName=model_name + ) + result["name"] = model_name + result["description"] = description + + with assert_raises(ClientError) as ex: + client.get_model( + restApiId=dummy_rest_api_id, modelName=model_name + ) + ex.exception.response["Error"]["Message"].should.equal( + "Invalid Rest API Id specified" + ) + ex.exception.response["Error"]["Code"].should.equal( + "NotFoundException" + ) + + +@mock_apigateway +def test_get_model_with_invalid_name(): + client = boto3.client("apigateway", region_name="us-west-2") + response = client.create_rest_api( + name="my_api", + description="this is my api" + ) + rest_api_id = response["id"] + # test with an invalid model name + with assert_raises(ClientError) as ex: + client.get_model( + restApiId=rest_api_id, modelName="fake" + ) + ex.exception.response["Error"]["Message"].should.equal( + "Invalid Model Name specified" + ) + ex.exception.response["Error"]["Code"].should.equal( + "NotFoundException" + ) + + @mock_apigateway def test_http_proxying_integration(): responses.add( From 1c96a05314ac3e4555dc07d1c5c1acf4cd9e7da8 Mon Sep 17 00:00:00 2001 From: usmankb Date: Sun, 12 Apr 2020 18:10:23 +0530 Subject: [PATCH 42/83] linting --- moto/apigateway/models.py | 17 ++++++++--------- moto/apigateway/responses.py | 4 ++-- moto/apigateway/urls.py | 2 +- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index 5ce95742e..b6a14b163 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -500,12 +500,12 @@ class RestAPI(BaseModel): return child def add_model(self, - name, - description=None, - schema=None, - content_type=None, - cli_input_json=None, - generate_cli_skeleton=None): + name, + description=None, + schema=None, + content_type=None, + cli_input_json=None, + generate_cli_skeleton=None): model_id = create_id() new_model = Model( id=model_id, @@ -519,7 +519,6 @@ class RestAPI(BaseModel): self.models[name] = new_model return new_model - def get_resource_for_path(self, path_after_stage_name): for resource in self.resources.values(): if resource.get_path() == path_after_stage_name: @@ -688,7 +687,6 @@ class Model(BaseModel,dict): self["generateCliSkeleton"] = kwargs.get("generate_cli_skeleton") - class APIGatewayBackend(BaseBackend): def __init__(self, region_name): super(APIGatewayBackend, self).__init__() @@ -1171,7 +1169,8 @@ class APIGatewayBackend(BaseBackend): model = api.models.get(model_name) if model is None: raise ModelNotFound - return model + else: + return model apigateway_backends = {} diff --git a/moto/apigateway/responses.py b/moto/apigateway/responses.py index c18b7f6c4..02ff536f3 100644 --- a/moto/apigateway/responses.py +++ b/moto/apigateway/responses.py @@ -639,7 +639,7 @@ class APIGatewayResponse(BaseResponse): '{{"message":"{0}","code":"{1}"}}'.format( error.message, error.error_type ), - ) + ) def model_induvidual(self, request, full_url, headers): self.setup_class(request, full_url, headers) @@ -662,4 +662,4 @@ class APIGatewayResponse(BaseResponse): '{{"message":"{0}","code":"{1}"}}'.format( error.message, error.error_type ), - ) \ No newline at end of file + ) diff --git a/moto/apigateway/urls.py b/moto/apigateway/urls.py index 751d8ae65..cb48e225f 100644 --- a/moto/apigateway/urls.py +++ b/moto/apigateway/urls.py @@ -22,7 +22,7 @@ url_paths = { "{0}/apikeys/(?P[^/]+)": APIGatewayResponse().apikey_individual, "{0}/usageplans$": APIGatewayResponse().usage_plans, "{0}/domainnames$": APIGatewayResponse().domain_names, - "{0}/restapis/(?P[^/]+)/models": APIGatewayResponse().models, + "{0}/restapis/(?P[^/]+)/models$": APIGatewayResponse().models, "{0}/restapis/(?P[^/]+)/models/(?P[^/]+)/?$": APIGatewayResponse().model_induvidual, "{0}/domainnames/(?P[^/]+)/?$": APIGatewayResponse().domain_name_induvidual, "{0}/usageplans/(?P[^/]+)/?$": APIGatewayResponse().usage_plan_individual, From 4be97916bfa80b04764ca9aa0024ba91a57232ae Mon Sep 17 00:00:00 2001 From: pvbouwel Date: Sun, 12 Apr 2020 20:05:35 +0100 Subject: [PATCH 43/83] Allow reuse of components packed in models.py By having models.py as one big file it causes to easily create circular dependencies. With the current setup it is not possible to re-use DynamoType. This refactor moves it out to its own file while trying to keep the structure as much as it is. --- moto/dynamodb2/__init__.py | 2 +- .../{models.py => models/__init__.py} | 227 +----------------- moto/dynamodb2/models/dynamo_type.py | 206 ++++++++++++++++ moto/dynamodb2/models/utilities.py | 17 ++ moto/dynamodb2/responses.py | 2 +- 5 files changed, 230 insertions(+), 224 deletions(-) rename moto/dynamodb2/{models.py => models/__init__.py} (86%) create mode 100644 moto/dynamodb2/models/dynamo_type.py create mode 100644 moto/dynamodb2/models/utilities.py diff --git a/moto/dynamodb2/__init__.py b/moto/dynamodb2/__init__.py index 3d6e8ec1f..d141511c8 100644 --- a/moto/dynamodb2/__init__.py +++ b/moto/dynamodb2/__init__.py @@ -1,5 +1,5 @@ from __future__ import unicode_literals -from .models import dynamodb_backends as dynamodb_backends2 +from moto.dynamodb2.models import dynamodb_backends as dynamodb_backends2 from ..core.models import base_decorator, deprecated_base_decorator dynamodb_backend2 = dynamodb_backends2["us-east-1"] diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models/__init__.py similarity index 86% rename from moto/dynamodb2/models.py rename to moto/dynamodb2/models/__init__.py index 152e719c4..29713d211 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models/__init__.py @@ -6,7 +6,6 @@ import decimal import json import re import uuid -import six from boto3 import Session from botocore.exceptions import ParamValidationError @@ -14,10 +13,11 @@ from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel from moto.core.utils import unix_time from moto.core.exceptions import JsonRESTError -from .comparisons import get_comparison_func -from .comparisons import get_filter_expression -from .comparisons import get_expected -from .exceptions import InvalidIndexNameError, InvalidUpdateExpression, ItemSizeTooLarge +from moto.dynamodb2.comparisons import get_filter_expression +from moto.dynamodb2.comparisons import get_expected +from moto.dynamodb2.exceptions import InvalidIndexNameError, ItemSizeTooLarge +from moto.dynamodb2.models.utilities import bytesize, attribute_is_list +from moto.dynamodb2.models.dynamo_type import DynamoType class DynamoJsonEncoder(json.JSONEncoder): @@ -30,223 +30,6 @@ def dynamo_json_dump(dynamo_object): return json.dumps(dynamo_object, cls=DynamoJsonEncoder) -def bytesize(val): - return len(str(val).encode("utf-8")) - - -def attribute_is_list(attr): - """ - Checks if attribute denotes a list, and returns the name of the list and the given list index if so - :param attr: attr or attr[index] - :return: attr, index or None - """ - list_index_update = re.match("(.+)\\[([0-9]+)\\]", attr) - if list_index_update: - attr = list_index_update.group(1) - return attr, list_index_update.group(2) if list_index_update else None - - -class DynamoType(object): - """ - http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DataModel.html#DataModelDataTypes - """ - - def __init__(self, type_as_dict): - if type(type_as_dict) == DynamoType: - self.type = type_as_dict.type - self.value = type_as_dict.value - else: - self.type = list(type_as_dict)[0] - self.value = list(type_as_dict.values())[0] - if self.is_list(): - self.value = [DynamoType(val) for val in self.value] - elif self.is_map(): - self.value = dict((k, DynamoType(v)) for k, v in self.value.items()) - - def get(self, key): - if not key: - return self - else: - key_head = key.split(".")[0] - key_tail = ".".join(key.split(".")[1:]) - if key_head not in self.value: - self.value[key_head] = DynamoType({"NONE": None}) - return self.value[key_head].get(key_tail) - - def set(self, key, new_value, index=None): - if index: - index = int(index) - if type(self.value) is not list: - raise InvalidUpdateExpression - if index >= len(self.value): - self.value.append(new_value) - # {'L': [DynamoType, ..]} ==> DynamoType.set() - self.value[min(index, len(self.value) - 1)].set(key, new_value) - else: - attr = (key or "").split(".").pop(0) - attr, list_index = attribute_is_list(attr) - if not key: - # {'S': value} ==> {'S': new_value} - self.type = new_value.type - self.value = new_value.value - else: - if attr not in self.value: # nonexistingattribute - type_of_new_attr = "M" if "." in key else new_value.type - self.value[attr] = DynamoType({type_of_new_attr: {}}) - # {'M': {'foo': DynamoType}} ==> DynamoType.set(new_value) - self.value[attr].set( - ".".join(key.split(".")[1:]), new_value, list_index - ) - - def delete(self, key, index=None): - if index: - if not key: - if int(index) < len(self.value): - del self.value[int(index)] - elif "." in key: - self.value[int(index)].delete(".".join(key.split(".")[1:])) - else: - self.value[int(index)].delete(key) - else: - attr = key.split(".")[0] - attr, list_index = attribute_is_list(attr) - - if list_index: - self.value[attr].delete(".".join(key.split(".")[1:]), list_index) - elif "." in key: - self.value[attr].delete(".".join(key.split(".")[1:])) - else: - self.value.pop(key) - - def filter(self, projection_expressions): - nested_projections = [ - expr[0 : expr.index(".")] for expr in projection_expressions if "." in expr - ] - if self.is_map(): - expressions_to_delete = [] - for attr in self.value: - if ( - attr not in projection_expressions - and attr not in nested_projections - ): - expressions_to_delete.append(attr) - elif attr in nested_projections: - relevant_expressions = [ - expr[len(attr + ".") :] - for expr in projection_expressions - if expr.startswith(attr + ".") - ] - self.value[attr].filter(relevant_expressions) - for expr in expressions_to_delete: - self.value.pop(expr) - - def __hash__(self): - return hash((self.type, self.value)) - - def __eq__(self, other): - return self.type == other.type and self.value == other.value - - def __ne__(self, other): - return self.type != other.type or self.value != other.value - - def __lt__(self, other): - return self.cast_value < other.cast_value - - def __le__(self, other): - return self.cast_value <= other.cast_value - - def __gt__(self, other): - return self.cast_value > other.cast_value - - def __ge__(self, other): - return self.cast_value >= other.cast_value - - def __repr__(self): - return "DynamoType: {0}".format(self.to_json()) - - @property - def cast_value(self): - if self.is_number(): - try: - return int(self.value) - except ValueError: - return float(self.value) - elif self.is_set(): - sub_type = self.type[0] - return set([DynamoType({sub_type: v}).cast_value for v in self.value]) - elif self.is_list(): - return [DynamoType(v).cast_value for v in self.value] - elif self.is_map(): - return dict([(k, DynamoType(v).cast_value) for k, v in self.value.items()]) - else: - return self.value - - def child_attr(self, key): - """ - Get Map or List children by key. str for Map, int for List. - - Returns DynamoType or None. - """ - if isinstance(key, six.string_types) and self.is_map(): - if "." in key and key.split(".")[0] in self.value: - return self.value[key.split(".")[0]].child_attr( - ".".join(key.split(".")[1:]) - ) - elif "." not in key and key in self.value: - return DynamoType(self.value[key]) - - if isinstance(key, int) and self.is_list(): - idx = key - if 0 <= idx < len(self.value): - return DynamoType(self.value[idx]) - - return None - - def size(self): - if self.is_number(): - value_size = len(str(self.value)) - elif self.is_set(): - sub_type = self.type[0] - value_size = sum([DynamoType({sub_type: v}).size() for v in self.value]) - elif self.is_list(): - value_size = sum([v.size() for v in self.value]) - elif self.is_map(): - value_size = sum( - [bytesize(k) + DynamoType(v).size() for k, v in self.value.items()] - ) - elif type(self.value) == bool: - value_size = 1 - else: - value_size = bytesize(self.value) - return value_size - - def to_json(self): - return {self.type: self.value} - - def compare(self, range_comparison, range_objs): - """ - Compares this type against comparison filters - """ - range_values = [obj.cast_value for obj in range_objs] - comparison_func = get_comparison_func(range_comparison) - return comparison_func(self.cast_value, *range_values) - - def is_number(self): - return self.type == "N" - - def is_set(self): - return self.type == "SS" or self.type == "NS" or self.type == "BS" - - def is_list(self): - return self.type == "L" - - def is_map(self): - return self.type == "M" - - def same_type(self, other): - return self.type == other.type - - # https://github.com/spulec/moto/issues/1874 # Ensure that the total size of an item does not exceed 400kb class LimitedSizeDict(dict): diff --git a/moto/dynamodb2/models/dynamo_type.py b/moto/dynamodb2/models/dynamo_type.py new file mode 100644 index 000000000..300804c1e --- /dev/null +++ b/moto/dynamodb2/models/dynamo_type.py @@ -0,0 +1,206 @@ +import six + +from moto.dynamodb2.comparisons import get_comparison_func +from moto.dynamodb2.exceptions import InvalidUpdateExpression +from moto.dynamodb2.models.utilities import attribute_is_list, bytesize + + +class DynamoType(object): + """ + http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DataModel.html#DataModelDataTypes + """ + + def __init__(self, type_as_dict): + if type(type_as_dict) == DynamoType: + self.type = type_as_dict.type + self.value = type_as_dict.value + else: + self.type = list(type_as_dict)[0] + self.value = list(type_as_dict.values())[0] + if self.is_list(): + self.value = [DynamoType(val) for val in self.value] + elif self.is_map(): + self.value = dict((k, DynamoType(v)) for k, v in self.value.items()) + + def get(self, key): + if not key: + return self + else: + key_head = key.split(".")[0] + key_tail = ".".join(key.split(".")[1:]) + if key_head not in self.value: + self.value[key_head] = DynamoType({"NONE": None}) + return self.value[key_head].get(key_tail) + + def set(self, key, new_value, index=None): + if index: + index = int(index) + if type(self.value) is not list: + raise InvalidUpdateExpression + if index >= len(self.value): + self.value.append(new_value) + # {'L': [DynamoType, ..]} ==> DynamoType.set() + self.value[min(index, len(self.value) - 1)].set(key, new_value) + else: + attr = (key or "").split(".").pop(0) + attr, list_index = attribute_is_list(attr) + if not key: + # {'S': value} ==> {'S': new_value} + self.type = new_value.type + self.value = new_value.value + else: + if attr not in self.value: # nonexistingattribute + type_of_new_attr = "M" if "." in key else new_value.type + self.value[attr] = DynamoType({type_of_new_attr: {}}) + # {'M': {'foo': DynamoType}} ==> DynamoType.set(new_value) + self.value[attr].set( + ".".join(key.split(".")[1:]), new_value, list_index + ) + + def delete(self, key, index=None): + if index: + if not key: + if int(index) < len(self.value): + del self.value[int(index)] + elif "." in key: + self.value[int(index)].delete(".".join(key.split(".")[1:])) + else: + self.value[int(index)].delete(key) + else: + attr = key.split(".")[0] + attr, list_index = attribute_is_list(attr) + + if list_index: + self.value[attr].delete(".".join(key.split(".")[1:]), list_index) + elif "." in key: + self.value[attr].delete(".".join(key.split(".")[1:])) + else: + self.value.pop(key) + + def filter(self, projection_expressions): + nested_projections = [ + expr[0 : expr.index(".")] for expr in projection_expressions if "." in expr + ] + if self.is_map(): + expressions_to_delete = [] + for attr in self.value: + if ( + attr not in projection_expressions + and attr not in nested_projections + ): + expressions_to_delete.append(attr) + elif attr in nested_projections: + relevant_expressions = [ + expr[len(attr + ".") :] + for expr in projection_expressions + if expr.startswith(attr + ".") + ] + self.value[attr].filter(relevant_expressions) + for expr in expressions_to_delete: + self.value.pop(expr) + + def __hash__(self): + return hash((self.type, self.value)) + + def __eq__(self, other): + return self.type == other.type and self.value == other.value + + def __ne__(self, other): + return self.type != other.type or self.value != other.value + + def __lt__(self, other): + return self.cast_value < other.cast_value + + def __le__(self, other): + return self.cast_value <= other.cast_value + + def __gt__(self, other): + return self.cast_value > other.cast_value + + def __ge__(self, other): + return self.cast_value >= other.cast_value + + def __repr__(self): + return "DynamoType: {0}".format(self.to_json()) + + @property + def cast_value(self): + if self.is_number(): + try: + return int(self.value) + except ValueError: + return float(self.value) + elif self.is_set(): + sub_type = self.type[0] + return set([DynamoType({sub_type: v}).cast_value for v in self.value]) + elif self.is_list(): + return [DynamoType(v).cast_value for v in self.value] + elif self.is_map(): + return dict([(k, DynamoType(v).cast_value) for k, v in self.value.items()]) + else: + return self.value + + def child_attr(self, key): + """ + Get Map or List children by key. str for Map, int for List. + + Returns DynamoType or None. + """ + if isinstance(key, six.string_types) and self.is_map(): + if "." in key and key.split(".")[0] in self.value: + return self.value[key.split(".")[0]].child_attr( + ".".join(key.split(".")[1:]) + ) + elif "." not in key and key in self.value: + return DynamoType(self.value[key]) + + if isinstance(key, int) and self.is_list(): + idx = key + if 0 <= idx < len(self.value): + return DynamoType(self.value[idx]) + + return None + + def size(self): + if self.is_number(): + value_size = len(str(self.value)) + elif self.is_set(): + sub_type = self.type[0] + value_size = sum([DynamoType({sub_type: v}).size() for v in self.value]) + elif self.is_list(): + value_size = sum([v.size() for v in self.value]) + elif self.is_map(): + value_size = sum( + [bytesize(k) + DynamoType(v).size() for k, v in self.value.items()] + ) + elif type(self.value) == bool: + value_size = 1 + else: + value_size = bytesize(self.value) + return value_size + + def to_json(self): + return {self.type: self.value} + + def compare(self, range_comparison, range_objs): + """ + Compares this type against comparison filters + """ + range_values = [obj.cast_value for obj in range_objs] + comparison_func = get_comparison_func(range_comparison) + return comparison_func(self.cast_value, *range_values) + + def is_number(self): + return self.type == "N" + + def is_set(self): + return self.type == "SS" or self.type == "NS" or self.type == "BS" + + def is_list(self): + return self.type == "L" + + def is_map(self): + return self.type == "M" + + def same_type(self, other): + return self.type == other.type diff --git a/moto/dynamodb2/models/utilities.py b/moto/dynamodb2/models/utilities.py new file mode 100644 index 000000000..9dd6f1e9f --- /dev/null +++ b/moto/dynamodb2/models/utilities.py @@ -0,0 +1,17 @@ +import re + + +def bytesize(val): + return len(str(val).encode("utf-8")) + + +def attribute_is_list(attr): + """ + Checks if attribute denotes a list, and returns the name of the list and the given list index if so + :param attr: attr or attr[index] + :return: attr, index or None + """ + list_index_update = re.match("(.+)\\[([0-9]+)\\]", attr) + if list_index_update: + attr = list_index_update.group(1) + return attr, list_index_update.group(2) if list_index_update else None diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 78126f7f1..65484aa08 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -10,7 +10,7 @@ import six from moto.core.responses import BaseResponse from moto.core.utils import camelcase_to_underscores, amzn_request_id from .exceptions import InvalidIndexNameError, InvalidUpdateExpression, ItemSizeTooLarge -from .models import dynamodb_backends, dynamo_json_dump +from moto.dynamodb2.models import dynamodb_backends, dynamo_json_dump TRANSACTION_MAX_ITEMS = 25 From d745dfd3d2eff02b628bd021374810a86ea73d4d Mon Sep 17 00:00:00 2001 From: DenverJ Date: Mon, 13 Apr 2020 10:50:01 +1000 Subject: [PATCH 44/83] Implement enter_standby, exit_standby and terminate_instance_in_auto_scaling_group --- moto/autoscaling/models.py | 67 ++- moto/autoscaling/responses.py | 144 ++++- tests/test_autoscaling/test_autoscaling.py | 584 ++++++++++++++++++++- 3 files changed, 777 insertions(+), 18 deletions(-) diff --git a/moto/autoscaling/models.py b/moto/autoscaling/models.py index 88577433e..b757672d0 100644 --- a/moto/autoscaling/models.py +++ b/moto/autoscaling/models.py @@ -267,6 +267,9 @@ class FakeAutoScalingGroup(BaseModel): self.tags = tags if tags else [] self.set_desired_capacity(desired_capacity) + def active_instances(self): + return [x for x in self.instance_states if x.lifecycle_state == "InService"] + def _set_azs_and_vpcs(self, availability_zones, vpc_zone_identifier, update=False): # for updates, if only AZs are provided, they must not clash with # the AZs of existing VPCs @@ -413,9 +416,11 @@ class FakeAutoScalingGroup(BaseModel): else: self.desired_capacity = new_capacity - curr_instance_count = len(self.instance_states) + curr_instance_count = len(self.active_instances()) if self.desired_capacity == curr_instance_count: + self.autoscaling_backend.update_attached_elbs(self.name) + self.autoscaling_backend.update_attached_target_groups(self.name) return if self.desired_capacity > curr_instance_count: @@ -442,6 +447,8 @@ class FakeAutoScalingGroup(BaseModel): self.instance_states = list( set(self.instance_states) - set(instances_to_remove) ) + self.autoscaling_backend.update_attached_elbs(self.name) + self.autoscaling_backend.update_attached_target_groups(self.name) def get_propagated_tags(self): propagated_tags = {} @@ -703,7 +710,7 @@ class AutoScalingBackend(BaseBackend): def detach_instances(self, group_name, instance_ids, should_decrement): group = self.autoscaling_groups[group_name] - original_size = len(group.instance_states) + original_size = group.desired_capacity detached_instances = [ x for x in group.instance_states if x.instance.id in instance_ids @@ -720,13 +727,8 @@ class AutoScalingBackend(BaseBackend): if should_decrement: group.desired_capacity = original_size - len(instance_ids) - else: - count_needed = len(instance_ids) - group.replace_autoscaling_group_instances( - count_needed, group.get_propagated_tags() - ) - self.update_attached_elbs(group_name) + group.set_desired_capacity(group.desired_capacity) return detached_instances def set_desired_capacity(self, group_name, desired_capacity): @@ -791,7 +793,9 @@ class AutoScalingBackend(BaseBackend): def update_attached_elbs(self, group_name): group = self.autoscaling_groups[group_name] - group_instance_ids = set(state.instance.id for state in group.instance_states) + group_instance_ids = set( + state.instance.id for state in group.active_instances() + ) # skip this if group.load_balancers is empty # otherwise elb_backend.describe_load_balancers returns all available load balancers @@ -908,15 +912,15 @@ class AutoScalingBackend(BaseBackend): autoscaling_group_name, autoscaling_group, ) in self.autoscaling_groups.items(): - original_instance_count = len(autoscaling_group.instance_states) + original_active_instance_count = len(autoscaling_group.active_instances()) autoscaling_group.instance_states = list( filter( lambda i_state: i_state.instance.id not in instance_ids, autoscaling_group.instance_states, ) ) - difference = original_instance_count - len( - autoscaling_group.instance_states + difference = original_active_instance_count - len( + autoscaling_group.active_instances() ) if difference > 0: autoscaling_group.replace_autoscaling_group_instances( @@ -924,6 +928,45 @@ class AutoScalingBackend(BaseBackend): ) self.update_attached_elbs(autoscaling_group_name) + def enter_standby_instances(self, group_name, instance_ids, should_decrement): + group = self.autoscaling_groups[group_name] + original_size = group.desired_capacity + standby_instances = [] + for instance_state in group.instance_states: + if instance_state.instance.id in instance_ids: + instance_state.lifecycle_state = "Standby" + standby_instances.append(instance_state) + if should_decrement: + group.desired_capacity = group.desired_capacity - len(instance_ids) + else: + group.set_desired_capacity(group.desired_capacity) + return standby_instances, original_size, group.desired_capacity + + def exit_standby_instances(self, group_name, instance_ids): + group = self.autoscaling_groups[group_name] + original_size = group.desired_capacity + standby_instances = [] + for instance_state in group.instance_states: + if instance_state.instance.id in instance_ids: + instance_state.lifecycle_state = "InService" + standby_instances.append(instance_state) + group.desired_capacity = group.desired_capacity + len(instance_ids) + return standby_instances, original_size, group.desired_capacity + + def terminate_instance(self, instance_id, should_decrement): + instance = self.ec2_backend.get_instance(instance_id) + instance_state = next( + instance_state + for group in self.autoscaling_groups.values() + for instance_state in group.instance_states + if instance_state.instance.id == instance.id + ) + group = instance.autoscaling_group + original_size = group.desired_capacity + self.detach_instances(group.name, [instance.id], should_decrement) + self.ec2_backend.terminate_instances([instance.id]) + return instance_state, original_size, group.desired_capacity + autoscaling_backends = {} for region, ec2_backend in ec2_backends.items(): diff --git a/moto/autoscaling/responses.py b/moto/autoscaling/responses.py index 41c79edb4..06b68aa4b 100644 --- a/moto/autoscaling/responses.py +++ b/moto/autoscaling/responses.py @@ -1,7 +1,12 @@ from __future__ import unicode_literals +import datetime from moto.core.responses import BaseResponse -from moto.core.utils import amz_crc32, amzn_request_id +from moto.core.utils import ( + amz_crc32, + amzn_request_id, + iso_8601_datetime_with_milliseconds, +) from .models import autoscaling_backends @@ -291,6 +296,50 @@ class AutoScalingResponse(BaseResponse): template = self.response_template(DETACH_LOAD_BALANCERS_TEMPLATE) return template.render() + @amz_crc32 + @amzn_request_id + def enter_standby(self): + group_name = self._get_param("AutoScalingGroupName") + instance_ids = self._get_multi_param("InstanceIds.member") + should_decrement_string = self._get_param("ShouldDecrementDesiredCapacity") + if should_decrement_string == "true": + should_decrement = True + else: + should_decrement = False + ( + standby_instances, + original_size, + desired_capacity, + ) = self.autoscaling_backend.enter_standby_instances( + group_name, instance_ids, should_decrement + ) + template = self.response_template(ENTER_STANDBY_TEMPLATE) + return template.render( + standby_instances=standby_instances, + should_decrement=should_decrement, + original_size=original_size, + desired_capacity=desired_capacity, + timestamp=iso_8601_datetime_with_milliseconds(datetime.datetime.utcnow()), + ) + + @amz_crc32 + @amzn_request_id + def exit_standby(self): + group_name = self._get_param("AutoScalingGroupName") + instance_ids = self._get_multi_param("InstanceIds.member") + ( + standby_instances, + original_size, + desired_capacity, + ) = self.autoscaling_backend.exit_standby_instances(group_name, instance_ids) + template = self.response_template(EXIT_STANDBY_TEMPLATE) + return template.render( + standby_instances=standby_instances, + original_size=original_size, + desired_capacity=desired_capacity, + timestamp=iso_8601_datetime_with_milliseconds(datetime.datetime.utcnow()), + ) + def suspend_processes(self): autoscaling_group_name = self._get_param("AutoScalingGroupName") scaling_processes = self._get_multi_param("ScalingProcesses.member") @@ -310,6 +359,29 @@ class AutoScalingResponse(BaseResponse): template = self.response_template(SET_INSTANCE_PROTECTION_TEMPLATE) return template.render() + @amz_crc32 + @amzn_request_id + def terminate_instance_in_auto_scaling_group(self): + instance_id = self._get_param("InstanceId") + should_decrement_string = self._get_param("ShouldDecrementDesiredCapacity") + if should_decrement_string == "true": + should_decrement = True + else: + should_decrement = False + ( + instance, + original_size, + desired_capacity, + ) = self.autoscaling_backend.terminate_instance(instance_id, should_decrement) + template = self.response_template(TERMINATE_INSTANCES_TEMPLATE) + return template.render( + instance=instance, + should_decrement=should_decrement, + original_size=original_size, + desired_capacity=desired_capacity, + timestamp=iso_8601_datetime_with_milliseconds(datetime.datetime.utcnow()), + ) + CREATE_LAUNCH_CONFIGURATION_TEMPLATE = """ @@ -707,3 +779,73 @@ SET_INSTANCE_PROTECTION_TEMPLATE = """ + + + {% for instance in standby_instances %} + + 12345678-1234-1234-1234-123456789012 + {{ group_name }} + {% if should_decrement %} + At {{ timestamp }} instance {{ instance.instance.id }} was moved to standby in response to a user request, shrinking the capacity from {{ original_size }} to {{ desired_capacity }}. + {% else %} + At {{ timestamp }} instance {{ instance.instance.id }} was moved to standby in response to a user request. + {% endif %} + Moving EC2 instance to Standby: {{ instance.instance.id }} + 50 + {{ timestamp }} +
{"Subnet ID":"??","Availability Zone":"{{ instance.instance.placement }}"}
+ InProgress + + {% endfor %} + + + + 7c6e177f-f082-11e1-ac58-3714bEXAMPLE + +""" + +EXIT_STANDBY_TEMPLATE = """ + + + {% for instance in standby_instances %} + + 12345678-1234-1234-1234-123456789012 + {{ group_name }} + Moving EC2 instance out of Standby: {{ instance.instance.id }} + 30 + At {{ timestamp }} instance {{ instance.instance.id }} was moved out of standby in response to a user request, increasing the capacity from {{ original_size }} to {{ desired_capacity }}. + {{ timestamp }} +
{"Subnet ID":"??","Availability Zone":"{{ instance.instance.placement }}"}
+ PreInService + + {% endfor %} + + + + 7c6e177f-f082-11e1-ac58-3714bEXAMPLE + +""" + +TERMINATE_INSTANCES_TEMPLATE = """ + + + 35b5c464-0b63-2fc7-1611-467d4a7f2497EXAMPLE + {{ group_name }} + {% if should_decrement %} + At {{ timestamp }} instance {{ instance.instance.id }} was taken out of service in response to a user request, shrinking the capacity from {{ original_size }} to {{ desired_capacity }}. + {% else %} + At {{ timestamp }} instance {{ instance.instance.id }} was taken out of service in response to a user request. + {% endif %} + Terminating EC2 instance: {{ instance.instance.id }} + 0 + {{ timestamp }} +
{"Subnet ID":"??","Availability Zone":"{{ instance.instance.placement }}"}
+ InProgress + + + + a1ba8fb9-31d6-4d9a-ace1-a7f76749df11EXAMPLE + +""" diff --git a/tests/test_autoscaling/test_autoscaling.py b/tests/test_autoscaling/test_autoscaling.py index 5cf3dc6ff..3a10f20ff 100644 --- a/tests/test_autoscaling/test_autoscaling.py +++ b/tests/test_autoscaling/test_autoscaling.py @@ -1102,8 +1102,6 @@ def test_detach_one_instance_decrement(): ec2_client = boto3.client("ec2", region_name="us-east-1") - response = ec2_client.describe_instances(InstanceIds=[instance_to_detach]) - response = client.detach_instances( AutoScalingGroupName="test_asg", InstanceIds=[instance_to_detach], @@ -1156,8 +1154,6 @@ def test_detach_one_instance(): ec2_client = boto3.client("ec2", region_name="us-east-1") - response = ec2_client.describe_instances(InstanceIds=[instance_to_detach]) - response = client.detach_instances( AutoScalingGroupName="test_asg", InstanceIds=[instance_to_detach], @@ -1178,6 +1174,516 @@ def test_detach_one_instance(): tags.should.have.length_of(2) +@mock_autoscaling +@mock_ec2 +def test_standby_one_instance_decrement(): + mocked_networking = setup_networking() + client = boto3.client("autoscaling", region_name="us-east-1") + _ = client.create_launch_configuration( + LaunchConfigurationName="test_launch_configuration" + ) + client.create_auto_scaling_group( + AutoScalingGroupName="test_asg", + LaunchConfigurationName="test_launch_configuration", + MinSize=0, + MaxSize=2, + DesiredCapacity=2, + Tags=[ + { + "ResourceId": "test_asg", + "ResourceType": "auto-scaling-group", + "Key": "propogated-tag-key", + "Value": "propagate-tag-value", + "PropagateAtLaunch": True, + } + ], + VPCZoneIdentifier=mocked_networking["subnet1"], + ) + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + instance_to_standby = response["AutoScalingGroups"][0]["Instances"][0]["InstanceId"] + instance_to_keep = response["AutoScalingGroups"][0]["Instances"][1]["InstanceId"] + + ec2_client = boto3.client("ec2", region_name="us-east-1") + + response = client.enter_standby( + AutoScalingGroupName="test_asg", + InstanceIds=[instance_to_standby], + ShouldDecrementDesiredCapacity=True, + ) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + response["AutoScalingGroups"][0]["Instances"].should.have.length_of(2) + response["AutoScalingGroups"][0]["DesiredCapacity"].should.equal(1) + + response = client.describe_auto_scaling_instances(InstanceIds=[instance_to_standby]) + response["AutoScalingInstances"][0]["LifecycleState"].should.equal("Standby") + + # test to ensure tag has been retained (standby instance is still part of the ASG) + response = ec2_client.describe_instances() + for reservation in response["Reservations"]: + for instance in reservation["Instances"]: + tags = instance["Tags"] + tags.should.have.length_of(2) + + +@mock_autoscaling +@mock_ec2 +def test_standby_one_instance(): + mocked_networking = setup_networking() + client = boto3.client("autoscaling", region_name="us-east-1") + _ = client.create_launch_configuration( + LaunchConfigurationName="test_launch_configuration" + ) + client.create_auto_scaling_group( + AutoScalingGroupName="test_asg", + LaunchConfigurationName="test_launch_configuration", + MinSize=0, + MaxSize=2, + DesiredCapacity=2, + Tags=[ + { + "ResourceId": "test_asg", + "ResourceType": "auto-scaling-group", + "Key": "propogated-tag-key", + "Value": "propagate-tag-value", + "PropagateAtLaunch": True, + } + ], + VPCZoneIdentifier=mocked_networking["subnet1"], + ) + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + instance_to_standby = response["AutoScalingGroups"][0]["Instances"][0]["InstanceId"] + instance_to_keep = response["AutoScalingGroups"][0]["Instances"][1]["InstanceId"] + + ec2_client = boto3.client("ec2", region_name="us-east-1") + + response = client.enter_standby( + AutoScalingGroupName="test_asg", + InstanceIds=[instance_to_standby], + ShouldDecrementDesiredCapacity=False, + ) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + response["AutoScalingGroups"][0]["Instances"].should.have.length_of(3) + response["AutoScalingGroups"][0]["DesiredCapacity"].should.equal(2) + + response = client.describe_auto_scaling_instances(InstanceIds=[instance_to_standby]) + response["AutoScalingInstances"][0]["LifecycleState"].should.equal("Standby") + + # test to ensure tag has been retained (standby instance is still part of the ASG) + response = ec2_client.describe_instances() + for reservation in response["Reservations"]: + for instance in reservation["Instances"]: + tags = instance["Tags"] + tags.should.have.length_of(2) + + +@mock_elb +@mock_autoscaling +@mock_ec2 +def test_standby_elb_update(): + mocked_networking = setup_networking() + client = boto3.client("autoscaling", region_name="us-east-1") + _ = client.create_launch_configuration( + LaunchConfigurationName="test_launch_configuration" + ) + client.create_auto_scaling_group( + AutoScalingGroupName="test_asg", + LaunchConfigurationName="test_launch_configuration", + MinSize=0, + MaxSize=2, + DesiredCapacity=2, + Tags=[ + { + "ResourceId": "test_asg", + "ResourceType": "auto-scaling-group", + "Key": "propogated-tag-key", + "Value": "propagate-tag-value", + "PropagateAtLaunch": True, + } + ], + VPCZoneIdentifier=mocked_networking["subnet1"], + ) + + elb_client = boto3.client("elb", region_name="us-east-1") + elb_client.create_load_balancer( + LoadBalancerName="my-lb", + Listeners=[{"Protocol": "tcp", "LoadBalancerPort": 80, "InstancePort": 8080}], + AvailabilityZones=["us-east-1a", "us-east-1b"], + ) + + response = client.attach_load_balancers( + AutoScalingGroupName="test_asg", LoadBalancerNames=["my-lb"] + ) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + instance_to_standby = response["AutoScalingGroups"][0]["Instances"][0]["InstanceId"] + + response = client.enter_standby( + AutoScalingGroupName="test_asg", + InstanceIds=[instance_to_standby], + ShouldDecrementDesiredCapacity=False, + ) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + response["AutoScalingGroups"][0]["Instances"].should.have.length_of(3) + response["AutoScalingGroups"][0]["DesiredCapacity"].should.equal(2) + + response = client.describe_auto_scaling_instances(InstanceIds=[instance_to_standby]) + response["AutoScalingInstances"][0]["LifecycleState"].should.equal("Standby") + + response = elb_client.describe_load_balancers(LoadBalancerNames=["my-lb"]) + list(response["LoadBalancerDescriptions"][0]["Instances"]).should.have.length_of(2) + + +@mock_autoscaling +@mock_ec2 +def test_standby_terminate_instance_decrement(): + mocked_networking = setup_networking() + client = boto3.client("autoscaling", region_name="us-east-1") + _ = client.create_launch_configuration( + LaunchConfigurationName="test_launch_configuration" + ) + client.create_auto_scaling_group( + AutoScalingGroupName="test_asg", + LaunchConfigurationName="test_launch_configuration", + MinSize=0, + MaxSize=3, + DesiredCapacity=2, + Tags=[ + { + "ResourceId": "test_asg", + "ResourceType": "auto-scaling-group", + "Key": "propogated-tag-key", + "Value": "propagate-tag-value", + "PropagateAtLaunch": True, + } + ], + VPCZoneIdentifier=mocked_networking["subnet1"], + ) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + instance_to_standby_terminate = response["AutoScalingGroups"][0]["Instances"][0][ + "InstanceId" + ] + + ec2_client = boto3.client("ec2", region_name="us-east-1") + + response = client.enter_standby( + AutoScalingGroupName="test_asg", + InstanceIds=[instance_to_standby_terminate], + ShouldDecrementDesiredCapacity=False, + ) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + response["AutoScalingGroups"][0]["Instances"].should.have.length_of(3) + response["AutoScalingGroups"][0]["DesiredCapacity"].should.equal(2) + + response = client.describe_auto_scaling_instances( + InstanceIds=[instance_to_standby_terminate] + ) + response["AutoScalingInstances"][0]["LifecycleState"].should.equal("Standby") + + response = client.terminate_instance_in_auto_scaling_group( + InstanceId=instance_to_standby_terminate, ShouldDecrementDesiredCapacity=True + ) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + # AWS still decrements desired capacity ASG if requested, even if the terminated instance is in standby + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + response["AutoScalingGroups"][0]["Instances"].should.have.length_of(1) + response["AutoScalingGroups"][0]["Instances"][0]["InstanceId"].should_not.equal( + instance_to_standby_terminate + ) + response["AutoScalingGroups"][0]["DesiredCapacity"].should.equal(1) + + response = ec2_client.describe_instances( + InstanceIds=[instance_to_standby_terminate] + ) + response["Reservations"][0]["Instances"][0]["State"]["Name"].should.equal( + "terminated" + ) + + +@mock_autoscaling +@mock_ec2 +def test_standby_terminate_instance_no_decrement(): + mocked_networking = setup_networking() + client = boto3.client("autoscaling", region_name="us-east-1") + _ = client.create_launch_configuration( + LaunchConfigurationName="test_launch_configuration" + ) + client.create_auto_scaling_group( + AutoScalingGroupName="test_asg", + LaunchConfigurationName="test_launch_configuration", + MinSize=0, + MaxSize=3, + DesiredCapacity=2, + Tags=[ + { + "ResourceId": "test_asg", + "ResourceType": "auto-scaling-group", + "Key": "propogated-tag-key", + "Value": "propagate-tag-value", + "PropagateAtLaunch": True, + } + ], + VPCZoneIdentifier=mocked_networking["subnet1"], + ) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + instance_to_standby_terminate = response["AutoScalingGroups"][0]["Instances"][0][ + "InstanceId" + ] + + ec2_client = boto3.client("ec2", region_name="us-east-1") + + response = client.enter_standby( + AutoScalingGroupName="test_asg", + InstanceIds=[instance_to_standby_terminate], + ShouldDecrementDesiredCapacity=False, + ) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + response["AutoScalingGroups"][0]["Instances"].should.have.length_of(3) + response["AutoScalingGroups"][0]["DesiredCapacity"].should.equal(2) + + response = client.describe_auto_scaling_instances( + InstanceIds=[instance_to_standby_terminate] + ) + response["AutoScalingInstances"][0]["LifecycleState"].should.equal("Standby") + + response = client.terminate_instance_in_auto_scaling_group( + InstanceId=instance_to_standby_terminate, ShouldDecrementDesiredCapacity=False + ) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + group = response["AutoScalingGroups"][0] + group["Instances"].should.have.length_of(2) + instance_to_standby_terminate.shouldnt.be.within( + [x["InstanceId"] for x in group["Instances"]] + ) + group["DesiredCapacity"].should.equal(2) + + response = ec2_client.describe_instances( + InstanceIds=[instance_to_standby_terminate] + ) + response["Reservations"][0]["Instances"][0]["State"]["Name"].should.equal( + "terminated" + ) + + +@mock_autoscaling +@mock_ec2 +def test_standby_detach_instance_decrement(): + mocked_networking = setup_networking() + client = boto3.client("autoscaling", region_name="us-east-1") + _ = client.create_launch_configuration( + LaunchConfigurationName="test_launch_configuration" + ) + client.create_auto_scaling_group( + AutoScalingGroupName="test_asg", + LaunchConfigurationName="test_launch_configuration", + MinSize=0, + MaxSize=3, + DesiredCapacity=2, + Tags=[ + { + "ResourceId": "test_asg", + "ResourceType": "auto-scaling-group", + "Key": "propogated-tag-key", + "Value": "propagate-tag-value", + "PropagateAtLaunch": True, + } + ], + VPCZoneIdentifier=mocked_networking["subnet1"], + ) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + instance_to_standby_detach = response["AutoScalingGroups"][0]["Instances"][0][ + "InstanceId" + ] + + ec2_client = boto3.client("ec2", region_name="us-east-1") + + response = client.enter_standby( + AutoScalingGroupName="test_asg", + InstanceIds=[instance_to_standby_detach], + ShouldDecrementDesiredCapacity=False, + ) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + response["AutoScalingGroups"][0]["Instances"].should.have.length_of(3) + response["AutoScalingGroups"][0]["DesiredCapacity"].should.equal(2) + + response = client.describe_auto_scaling_instances( + InstanceIds=[instance_to_standby_detach] + ) + response["AutoScalingInstances"][0]["LifecycleState"].should.equal("Standby") + + response = client.detach_instances( + AutoScalingGroupName="test_asg", + InstanceIds=[instance_to_standby_detach], + ShouldDecrementDesiredCapacity=True, + ) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + # AWS still decrements desired capacity ASG if requested, even if the detached instance was in standby + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + response["AutoScalingGroups"][0]["Instances"].should.have.length_of(1) + response["AutoScalingGroups"][0]["Instances"][0]["InstanceId"].should_not.equal( + instance_to_standby_detach + ) + response["AutoScalingGroups"][0]["DesiredCapacity"].should.equal(1) + + response = ec2_client.describe_instances(InstanceIds=[instance_to_standby_detach]) + response["Reservations"][0]["Instances"][0]["State"]["Name"].should.equal("running") + + +@mock_autoscaling +@mock_ec2 +def test_standby_detach_instance_no_decrement(): + mocked_networking = setup_networking() + client = boto3.client("autoscaling", region_name="us-east-1") + _ = client.create_launch_configuration( + LaunchConfigurationName="test_launch_configuration" + ) + client.create_auto_scaling_group( + AutoScalingGroupName="test_asg", + LaunchConfigurationName="test_launch_configuration", + MinSize=0, + MaxSize=3, + DesiredCapacity=2, + Tags=[ + { + "ResourceId": "test_asg", + "ResourceType": "auto-scaling-group", + "Key": "propogated-tag-key", + "Value": "propagate-tag-value", + "PropagateAtLaunch": True, + } + ], + VPCZoneIdentifier=mocked_networking["subnet1"], + ) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + instance_to_standby_detach = response["AutoScalingGroups"][0]["Instances"][0][ + "InstanceId" + ] + + ec2_client = boto3.client("ec2", region_name="us-east-1") + + response = client.enter_standby( + AutoScalingGroupName="test_asg", + InstanceIds=[instance_to_standby_detach], + ShouldDecrementDesiredCapacity=False, + ) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + response["AutoScalingGroups"][0]["Instances"].should.have.length_of(3) + response["AutoScalingGroups"][0]["DesiredCapacity"].should.equal(2) + + response = client.describe_auto_scaling_instances( + InstanceIds=[instance_to_standby_detach] + ) + response["AutoScalingInstances"][0]["LifecycleState"].should.equal("Standby") + + response = client.detach_instances( + AutoScalingGroupName="test_asg", + InstanceIds=[instance_to_standby_detach], + ShouldDecrementDesiredCapacity=False, + ) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + group = response["AutoScalingGroups"][0] + group["Instances"].should.have.length_of(2) + instance_to_standby_detach.shouldnt.be.within( + [x["InstanceId"] for x in group["Instances"]] + ) + group["DesiredCapacity"].should.equal(2) + + response = ec2_client.describe_instances(InstanceIds=[instance_to_standby_detach]) + response["Reservations"][0]["Instances"][0]["State"]["Name"].should.equal("running") + + +@mock_autoscaling +@mock_ec2 +def test_standby_exit_standby(): + mocked_networking = setup_networking() + client = boto3.client("autoscaling", region_name="us-east-1") + _ = client.create_launch_configuration( + LaunchConfigurationName="test_launch_configuration" + ) + client.create_auto_scaling_group( + AutoScalingGroupName="test_asg", + LaunchConfigurationName="test_launch_configuration", + MinSize=0, + MaxSize=3, + DesiredCapacity=2, + Tags=[ + { + "ResourceId": "test_asg", + "ResourceType": "auto-scaling-group", + "Key": "propogated-tag-key", + "Value": "propagate-tag-value", + "PropagateAtLaunch": True, + } + ], + VPCZoneIdentifier=mocked_networking["subnet1"], + ) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + instance_to_standby_exit_standby = response["AutoScalingGroups"][0]["Instances"][0][ + "InstanceId" + ] + + ec2_client = boto3.client("ec2", region_name="us-east-1") + + response = client.enter_standby( + AutoScalingGroupName="test_asg", + InstanceIds=[instance_to_standby_exit_standby], + ShouldDecrementDesiredCapacity=False, + ) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + response["AutoScalingGroups"][0]["Instances"].should.have.length_of(3) + response["AutoScalingGroups"][0]["DesiredCapacity"].should.equal(2) + + response = client.describe_auto_scaling_instances( + InstanceIds=[instance_to_standby_exit_standby] + ) + response["AutoScalingInstances"][0]["LifecycleState"].should.equal("Standby") + + response = client.exit_standby( + AutoScalingGroupName="test_asg", InstanceIds=[instance_to_standby_exit_standby], + ) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + group = response["AutoScalingGroups"][0] + group["Instances"].should.have.length_of(3) + instance_to_standby_exit_standby.should.be.within( + [x["InstanceId"] for x in group["Instances"]] + ) + group["DesiredCapacity"].should.equal(3) + + response = ec2_client.describe_instances( + InstanceIds=[instance_to_standby_exit_standby] + ) + response["Reservations"][0]["Instances"][0]["State"]["Name"].should.equal("running") + + @mock_autoscaling @mock_ec2 def test_attach_one_instance(): @@ -1411,7 +1917,7 @@ def test_set_desired_capacity_down_boto3(): @mock_autoscaling @mock_ec2 -def test_terminate_instance_in_autoscaling_group(): +def test_terminate_instance_via_ec2_in_autoscaling_group(): mocked_networking = setup_networking() client = boto3.client("autoscaling", region_name="us-east-1") _ = client.create_launch_configuration( @@ -1440,3 +1946,71 @@ def test_terminate_instance_in_autoscaling_group(): for instance in response["AutoScalingGroups"][0]["Instances"] ) replaced_instance_id.should_not.equal(original_instance_id) + + +@mock_autoscaling +@mock_ec2 +def test_terminate_instance_in_auto_scaling_group_decrement(): + mocked_networking = setup_networking() + client = boto3.client("autoscaling", region_name="us-east-1") + _ = client.create_launch_configuration( + LaunchConfigurationName="test_launch_configuration" + ) + _ = client.create_auto_scaling_group( + AutoScalingGroupName="test_asg", + LaunchConfigurationName="test_launch_configuration", + MinSize=0, + DesiredCapacity=1, + MaxSize=2, + VPCZoneIdentifier=mocked_networking["subnet1"], + NewInstancesProtectedFromScaleIn=False, + ) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + original_instance_id = next( + instance["InstanceId"] + for instance in response["AutoScalingGroups"][0]["Instances"] + ) + client.terminate_instance_in_auto_scaling_group( + InstanceId=original_instance_id, ShouldDecrementDesiredCapacity=True + ) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + response["AutoScalingGroups"][0]["Instances"].should.equal([]) + response["AutoScalingGroups"][0]["DesiredCapacity"].should.equal(0) + + +@mock_autoscaling +@mock_ec2 +def test_terminate_instance_in_auto_scaling_group_no_decrement(): + mocked_networking = setup_networking() + client = boto3.client("autoscaling", region_name="us-east-1") + _ = client.create_launch_configuration( + LaunchConfigurationName="test_launch_configuration" + ) + _ = client.create_auto_scaling_group( + AutoScalingGroupName="test_asg", + LaunchConfigurationName="test_launch_configuration", + MinSize=0, + DesiredCapacity=1, + MaxSize=2, + VPCZoneIdentifier=mocked_networking["subnet1"], + NewInstancesProtectedFromScaleIn=False, + ) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + original_instance_id = next( + instance["InstanceId"] + for instance in response["AutoScalingGroups"][0]["Instances"] + ) + client.terminate_instance_in_auto_scaling_group( + InstanceId=original_instance_id, ShouldDecrementDesiredCapacity=False + ) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=["test_asg"]) + replaced_instance_id = next( + instance["InstanceId"] + for instance in response["AutoScalingGroups"][0]["Instances"] + ) + replaced_instance_id.should_not.equal(original_instance_id) + response["AutoScalingGroups"][0]["DesiredCapacity"].should.equal(1) From 1e1fe3ee4bffb5470795d8aab16fa3de1145f5a6 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 14 Apr 2020 07:48:13 +0100 Subject: [PATCH 45/83] Update moto/dynamodb2/models.py Co-Authored-By: pvbouwel --- moto/dynamodb2/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index a35eded61..de2a06fd4 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -286,7 +286,7 @@ class Item(BaseModel): return "Item: {0}".format(self.to_json()) def size(self): - return sum([bytesize(key) + value.size() for key, value in self.attrs.items()]) + return sum(bytesize(key) + value.size() for key, value in self.attrs.items()) def to_json(self): attributes = {} From 8122a40be064f28dd0aa2ea8567cc2fb2ce4dea8 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 14 Apr 2020 07:48:20 +0100 Subject: [PATCH 46/83] Update moto/dynamodb2/models.py Co-Authored-By: pvbouwel --- moto/dynamodb2/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index de2a06fd4..62c60efb0 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -1127,7 +1127,7 @@ class Table(BaseModel): last_evaluated_key = None size_limit = 1000000 # DynamoDB has a 1MB size limit - item_size = sum([res.size() for res in results]) + item_size = sum(res.size() for res in results) if item_size > size_limit: item_size = idx = 0 while item_size + results[idx].size() < size_limit: From c2b4c397f272f80684c395150159028ed265fd20 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 14 Apr 2020 07:53:15 +0100 Subject: [PATCH 47/83] DDB test - Fix KeySchema, and set BillingMode for easier online testing --- tests/test_dynamodb2/test_dynamodb.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index daae79232..e00a45e1d 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -4143,13 +4143,13 @@ def test_dynamodb_max_1mb_limit(): TableName=table_name, KeySchema=[ {"AttributeName": "partition_key", "KeyType": "HASH"}, - {"AttributeName": "sort_key", "KeyType": "SORT"}, + {"AttributeName": "sort_key", "KeyType": "RANGE"}, ], AttributeDefinitions=[ {"AttributeName": "partition_key", "AttributeType": "S"}, {"AttributeName": "sort_key", "AttributeType": "S"}, ], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + BillingMode="PAY_PER_REQUEST", ) # Populate the table @@ -4170,3 +4170,4 @@ def test_dynamodb_max_1mb_limit(): # We shouldn't get everything back - the total result set is well over 1MB assert response["Count"] < len(items) response["LastEvaluatedKey"].shouldnt.be(None) + From 69f963a3c28acf3a0d0d1f2dac955e99cbb6a9c4 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 14 Apr 2020 08:06:00 +0100 Subject: [PATCH 48/83] Linting --- moto/apigateway/models.py | 44 +++++++------ moto/apigateway/responses.py | 29 ++++----- tests/test_apigateway/test_apigateway.py | 83 ++++++++---------------- 3 files changed, 64 insertions(+), 92 deletions(-) diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index b6a14b163..e5e5e3bfd 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -39,7 +39,7 @@ from .exceptions import ( InvalidRestApiId, InvalidModelName, RestAPINotFound, - ModelNotFound + ModelNotFound, ) STAGE_URL = "https://{api_id}.execute-api.{region_name}.amazonaws.com/{stage_name}" @@ -499,13 +499,15 @@ class RestAPI(BaseModel): self.resources[child_id] = child return child - def add_model(self, - name, - description=None, - schema=None, - content_type=None, - cli_input_json=None, - generate_cli_skeleton=None): + def add_model( + self, + name, + description=None, + schema=None, + content_type=None, + cli_input_json=None, + generate_cli_skeleton=None, + ): model_id = create_id() new_model = Model( id=model_id, @@ -514,7 +516,8 @@ class RestAPI(BaseModel): schema=schema, content_type=content_type, cli_input_json=cli_input_json, - generate_cli_skeleton=generate_cli_skeleton) + generate_cli_skeleton=generate_cli_skeleton, + ) self.models[name] = new_model return new_model @@ -670,7 +673,7 @@ class DomainName(BaseModel, dict): self["generateCliSkeleton"] = kwargs.get("generate_cli_skeleton") -class Model(BaseModel,dict): +class Model(BaseModel, dict): def __init__(self, id, name, **kwargs): super(Model, self).__init__() self["id"] = id @@ -1130,14 +1133,16 @@ class APIGatewayBackend(BaseBackend): else: return self.domain_names[domain_name] - def create_model(self, - rest_api_id, - name, - content_type, - description=None, - schema=None, - cli_input_json=None, - generate_cli_skeleton=None): + def create_model( + self, + rest_api_id, + name, + content_type, + description=None, + schema=None, + cli_input_json=None, + generate_cli_skeleton=None, + ): if not rest_api_id: raise InvalidRestApiId @@ -1151,7 +1156,8 @@ class APIGatewayBackend(BaseBackend): schema=schema, content_type=content_type, cli_input_json=cli_input_json, - generate_cli_skeleton=generate_cli_skeleton) + generate_cli_skeleton=generate_cli_skeleton, + ) return new_model diff --git a/moto/apigateway/responses.py b/moto/apigateway/responses.py index 02ff536f3..822d4c0ce 100644 --- a/moto/apigateway/responses.py +++ b/moto/apigateway/responses.py @@ -16,7 +16,7 @@ from .exceptions import ( InvalidRestApiId, InvalidModelName, RestAPINotFound, - ModelNotFound + ModelNotFound, ) API_KEY_SOURCES = ["AUTHORIZER", "HEADER"] @@ -600,15 +600,13 @@ class APIGatewayResponse(BaseResponse): ), ) - def models(self,request, full_url, headers): + def models(self, request, full_url, headers): self.setup_class(request, full_url, headers) rest_api_id = self.path.replace("/restapis/", "", 1).split("/")[0] try: if self.method == "GET": - models = self.backend.get_models( - rest_api_id - ) + models = self.backend.get_models(rest_api_id) return 200, {}, json.dumps({"item": models}) elif self.method == "POST": @@ -617,9 +615,7 @@ class APIGatewayResponse(BaseResponse): schema = self._get_param("schema") content_type = self._get_param("contentType") cli_input_json = self._get_param("cliInputJson") - generate_cli_skeleton = self._get_param( - "generateCliSkeleton" - ) + generate_cli_skeleton = self._get_param("generateCliSkeleton") model = self.backend.create_model( rest_api_id, name, @@ -627,12 +623,12 @@ class APIGatewayResponse(BaseResponse): description, schema, cli_input_json, - generate_cli_skeleton + generate_cli_skeleton, ) return 200, {}, json.dumps(model) - except (InvalidRestApiId, InvalidModelName,RestAPINotFound) as error: + except (InvalidRestApiId, InvalidModelName, RestAPINotFound) as error: return ( error.code, {}, @@ -649,13 +645,14 @@ class APIGatewayResponse(BaseResponse): model_info = {} try: if self.method == "GET": - model_info = self.backend.get_model( - rest_api_id, - model_name - ) + model_info = self.backend.get_model(rest_api_id, model_name) return 200, {}, json.dumps(model_info) - except (ModelNotFound, RestAPINotFound, InvalidRestApiId, - InvalidModelName) as error: + except ( + ModelNotFound, + RestAPINotFound, + InvalidRestApiId, + InvalidModelName, + ) as error: return ( error.code, {}, diff --git a/tests/test_apigateway/test_apigateway.py b/tests/test_apigateway/test_apigateway.py index 3a6b75104..596ed2dd4 100644 --- a/tests/test_apigateway/test_apigateway.py +++ b/tests/test_apigateway/test_apigateway.py @@ -1550,82 +1550,67 @@ def test_get_domain_name(): @mock_apigateway def test_create_model(): client = boto3.client("apigateway", region_name="us-west-2") - response = client.create_rest_api(name="my_api", - description="this is my api" - ) + response = client.create_rest_api(name="my_api", description="this is my api") rest_api_id = response["id"] - dummy_rest_api_id = 'a12b3c4d' + dummy_rest_api_id = "a12b3c4d" model_name = "testModel" description = "test model" - content_type = 'application/json' + content_type = "application/json" # success case with valid params response = client.create_model( restApiId=rest_api_id, name=model_name, description=description, - contentType=content_type + contentType=content_type, ) response["name"].should.equal(model_name) response["description"].should.equal(description) # with an invalid rest_api_id it should throw NotFoundException with assert_raises(ClientError) as ex: - client.create_model( + client.create_model( restApiId=dummy_rest_api_id, name=model_name, description=description, - contentType=content_type + contentType=content_type, ) ex.exception.response["Error"]["Message"].should.equal( "Invalid Rest API Id specified" ) - ex.exception.response["Error"]["Code"].should.equal( - "NotFoundException" - ) + ex.exception.response["Error"]["Code"].should.equal("NotFoundException") with assert_raises(ClientError) as ex: - client.create_model( + client.create_model( restApiId=rest_api_id, name="", description=description, - contentType=content_type + contentType=content_type, ) - ex.exception.response["Error"]["Message"].should.equal( - "No Model Name specified" - ) - ex.exception.response["Error"]["Code"].should.equal( - "BadRequestException" - ) + ex.exception.response["Error"]["Message"].should.equal("No Model Name specified") + ex.exception.response["Error"]["Code"].should.equal("BadRequestException") @mock_apigateway def test_get_api_models(): client = boto3.client("apigateway", region_name="us-west-2") - response = client.create_rest_api( - name="my_api", - description="this is my api" - ) + response = client.create_rest_api(name="my_api", description="this is my api") rest_api_id = response["id"] model_name = "testModel" description = "test model" - content_type = 'application/json' + content_type = "application/json" # when no models are present - result = client.get_models( - restApiId=rest_api_id - ) + result = client.get_models(restApiId=rest_api_id) result["items"].should.equal([]) # add a model client.create_model( restApiId=rest_api_id, name=model_name, description=description, - contentType=content_type + contentType=content_type, ) # get models after adding - result = client.get_models( - restApiId=rest_api_id - ) + result = client.get_models(restApiId=rest_api_id) result["items"][0]["name"] = model_name result["items"][0]["description"] = description @@ -1633,60 +1618,44 @@ def test_get_api_models(): @mock_apigateway def test_get_model_by_name(): client = boto3.client("apigateway", region_name="us-west-2") - response = client.create_rest_api( - name="my_api", - description="this is my api" - ) + response = client.create_rest_api(name="my_api", description="this is my api") rest_api_id = response["id"] - dummy_rest_api_id = 'a12b3c4d' + dummy_rest_api_id = "a12b3c4d" model_name = "testModel" description = "test model" - content_type = 'application/json' + content_type = "application/json" # add a model client.create_model( restApiId=rest_api_id, name=model_name, description=description, - contentType=content_type + contentType=content_type, ) # get models after adding - result = client.get_model( - restApiId=rest_api_id, modelName=model_name - ) + result = client.get_model(restApiId=rest_api_id, modelName=model_name) result["name"] = model_name result["description"] = description with assert_raises(ClientError) as ex: - client.get_model( - restApiId=dummy_rest_api_id, modelName=model_name - ) + client.get_model(restApiId=dummy_rest_api_id, modelName=model_name) ex.exception.response["Error"]["Message"].should.equal( "Invalid Rest API Id specified" ) - ex.exception.response["Error"]["Code"].should.equal( - "NotFoundException" - ) + ex.exception.response["Error"]["Code"].should.equal("NotFoundException") @mock_apigateway def test_get_model_with_invalid_name(): client = boto3.client("apigateway", region_name="us-west-2") - response = client.create_rest_api( - name="my_api", - description="this is my api" - ) + response = client.create_rest_api(name="my_api", description="this is my api") rest_api_id = response["id"] # test with an invalid model name with assert_raises(ClientError) as ex: - client.get_model( - restApiId=rest_api_id, modelName="fake" - ) + client.get_model(restApiId=rest_api_id, modelName="fake") ex.exception.response["Error"]["Message"].should.equal( "Invalid Model Name specified" ) - ex.exception.response["Error"]["Code"].should.equal( - "NotFoundException" - ) + ex.exception.response["Error"]["Code"].should.equal("NotFoundException") @mock_apigateway From a6902e87137da229d63d138b898a63ffd12fe326 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 15 Apr 2020 07:26:09 +0100 Subject: [PATCH 49/83] Update tests/test_dynamodb2/test_dynamodb.py Co-Authored-By: Guilherme Martins Crocetti --- tests/test_dynamodb2/test_dynamodb.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index e00a45e1d..2b4c0969c 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -4168,6 +4168,5 @@ def test_dynamodb_max_1mb_limit(): KeyConditionExpression=Key("partition_key").eq("partition_key_val") ) # We shouldn't get everything back - the total result set is well over 1MB - assert response["Count"] < len(items) + len(items).should.be.greater_than(response["Count"]) response["LastEvaluatedKey"].shouldnt.be(None) - From f04d64d9816e219eeb5b3b310d3870664a174050 Mon Sep 17 00:00:00 2001 From: Jacob-House Date: Wed, 15 Apr 2020 18:48:33 -0230 Subject: [PATCH 50/83] Update EC2 instance type list --- moto/ec2/resources/instance_types.json | 2 +- scripts/get_instance_info.py | 96 ++++++++++++++++---------- 2 files changed, 59 insertions(+), 39 deletions(-) diff --git a/moto/ec2/resources/instance_types.json b/moto/ec2/resources/instance_types.json index 2fa2e4e93..a1b55ba21 100644 --- a/moto/ec2/resources/instance_types.json +++ b/moto/ec2/resources/instance_types.json @@ -1 +1 @@ -{"m1.xlarge": {"ecu_per_vcpu": 2.0, "network_perf": 9.0, "intel_avx": "", "name": "M1 General Purpose Extra Large", "architecture": "64-bit", "linux_virtualization": "PV", "storage": 1680.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "", "apiname": "m1.xlarge", "computeunits": 8.0, "ebs_throughput": 125.0, "vpc_only": false, "max_ips": 60, "physical_processor": "", "fpga": 0, "intel_turbo": "", "enhanced_networking": false, "ebs_iops": 8000.0, "vcpus": 4.0, "memory": 15.0, "ebs_max_bandwidth": 1000.0, "gpus": 0, "ipv6_support": false}, "i3.4xlarge": {"ecu_per_vcpu": 3.3125, "network_perf": 11.0, "intel_avx": "Yes", "name": "I3 High I/O Quadruple Extra Large", "architecture": "64-bit", "linux_virtualization": "Unknown", "storage": 3800.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "i3.4xlarge", "computeunits": 53.0, "ebs_throughput": 400.0, "vpc_only": true, "max_ips": 240, "physical_processor": "Intel Xeon E5-2686 v4", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 16000.0, "vcpus": 16.0, "memory": 122.0, "ebs_max_bandwidth": 3500.0, "gpus": 0, "ipv6_support": true}, "i2.xlarge": {"ecu_per_vcpu": 3.5, "network_perf": 7.0, "intel_avx": "", "name": "I2 Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 800.0, "placement_group_support": true, "intel_avx2": "", "clock_speed_ghz": "", "apiname": "i2.xlarge", "computeunits": 14.0, "ebs_throughput": 62.5, "vpc_only": false, "max_ips": 60, "physical_processor": "", "fpga": 0, "intel_turbo": "", "enhanced_networking": false, "ebs_iops": 4000.0, "vcpus": 4.0, "memory": 30.5, "ebs_max_bandwidth": 500.0, "gpus": 0, "ipv6_support": true}, "hs1.8xlarge": {"ecu_per_vcpu": 2.1875, "network_perf": 12.0, "intel_avx": "", "name": "High Storage Eight Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM, PV", "storage": 48000.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "", "apiname": "hs1.8xlarge", "computeunits": 35.0, "ebs_throughput": 0.0, "vpc_only": false, "max_ips": 240, "physical_processor": "", "fpga": 0, "intel_turbo": "", "enhanced_networking": false, "ebs_iops": 0.0, "vcpus": 16.0, "memory": 117.0, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": false}, "t2.micro": {"ecu_per_vcpu": 0.0, "network_perf": 4.0, "intel_avx": "Yes", "name": "T2 Micro", "architecture": "32/64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "Yes", "apiname": "t2.micro", "computeunits": 0.1, "ebs_throughput": 0.0, "vpc_only": true, "max_ips": 4, "physical_processor": "Intel Xeon family", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": false, "ebs_iops": 0.0, "vcpus": 1.0, "memory": 1.0, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": true}, "d2.4xlarge": {"ecu_per_vcpu": 3.5, "network_perf": 9.0, "intel_avx": "Yes", "name": "D2 Quadruple Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 24000.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "d2.4xlarge", "computeunits": 56.0, "ebs_throughput": 250.0, "vpc_only": false, "max_ips": 240, "physical_processor": "Intel Xeon E5-2676 v3", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 16000.0, "vcpus": 16.0, "memory": 122.0, "ebs_max_bandwidth": 2000.0, "gpus": 0, "ipv6_support": true}, "m2.xlarge": {"ecu_per_vcpu": 3.25, "network_perf": 6.0, "intel_avx": "", "name": "M2 High Memory Extra Large", "architecture": "64-bit", "linux_virtualization": "PV", "storage": 420.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "", "apiname": "m2.xlarge", "computeunits": 6.5, "ebs_throughput": 0.0, "vpc_only": false, "max_ips": 60, "physical_processor": "", "fpga": 0, "intel_turbo": "", "enhanced_networking": false, "ebs_iops": 0.0, "vcpus": 2.0, "memory": 17.1, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": false}, "p2.xlarge": {"ecu_per_vcpu": 3.0, "network_perf": 9.0, "intel_avx": "Yes", "name": "General Purpose GPU Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "p2.xlarge", "computeunits": 12.0, "ebs_throughput": 93.75, "vpc_only": true, "max_ips": 60, "physical_processor": "Intel Xeon E5-2686 v4", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 6000.0, "vcpus": 4.0, "memory": 61.0, "ebs_max_bandwidth": 750.0, "gpus": 1, "ipv6_support": true}, "i2.4xlarge": {"ecu_per_vcpu": 3.3125, "network_perf": 9.0, "intel_avx": "", "name": "I2 Quadruple Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 3200.0, "placement_group_support": true, "intel_avx2": "", "clock_speed_ghz": "", "apiname": "i2.4xlarge", "computeunits": 53.0, "ebs_throughput": 250.0, "vpc_only": false, "max_ips": 240, "physical_processor": "", "fpga": 0, "intel_turbo": "", "enhanced_networking": false, "ebs_iops": 16000.0, "vcpus": 16.0, "memory": 122.0, "ebs_max_bandwidth": 2000.0, "gpus": 0, "ipv6_support": true}, "t1.micro": {"ecu_per_vcpu": 0.0, "network_perf": 0.0, "intel_avx": "", "name": "T1 Micro", "architecture": "32/64-bit", "linux_virtualization": "PV", "storage": 0.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "", "apiname": "t1.micro", "computeunits": 0.0, "ebs_throughput": 0.0, "vpc_only": false, "max_ips": 4, "physical_processor": "", "fpga": 0, "intel_turbo": "", "enhanced_networking": false, "ebs_iops": 0.0, "vcpus": 1.0, "memory": 0.613, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": false}, "d2.xlarge": {"ecu_per_vcpu": 3.5, "network_perf": 7.0, "intel_avx": "Yes", "name": "D2 Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 6000.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "d2.xlarge", "computeunits": 14.0, "ebs_throughput": 93.75, "vpc_only": false, "max_ips": 60, "physical_processor": "Intel Xeon E5-2676 v3", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 6000.0, "vcpus": 4.0, "memory": 30.5, "ebs_max_bandwidth": 750.0, "gpus": 0, "ipv6_support": true}, "r3.2xlarge": {"ecu_per_vcpu": 3.25, "network_perf": 9.0, "intel_avx": "Yes", "name": "R3 High-Memory Double Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 160.0, "placement_group_support": true, "intel_avx2": "", "clock_speed_ghz": "Yes", "apiname": "r3.2xlarge", "computeunits": 26.0, "ebs_throughput": 125.0, "vpc_only": false, "max_ips": 60, "physical_processor": "Intel Xeon E5-2670 v2", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 8000.0, "vcpus": 8.0, "memory": 61.0, "ebs_max_bandwidth": 1000.0, "gpus": 0, "ipv6_support": true}, "i3.8xlarge": {"ecu_per_vcpu": 3.09375, "network_perf": 13.0, "intel_avx": "Yes", "name": "I3 High I/O Eight Extra Large", "architecture": "64-bit", "linux_virtualization": "Unknown", "storage": 7600.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "i3.8xlarge", "computeunits": 99.0, "ebs_throughput": 850.0, "vpc_only": true, "max_ips": 240, "physical_processor": "Intel Xeon E5-2686 v4", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 32500.0, "vcpus": 32.0, "memory": 244.0, "ebs_max_bandwidth": 7000.0, "gpus": 0, "ipv6_support": true}, "c3.2xlarge": {"ecu_per_vcpu": 3.5, "network_perf": 9.0, "intel_avx": "Yes", "name": "C3 High-CPU Double Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM, PV", "storage": 160.0, "placement_group_support": true, "intel_avx2": "", "clock_speed_ghz": "Yes", "apiname": "c3.2xlarge", "computeunits": 28.0, "ebs_throughput": 125.0, "vpc_only": false, "max_ips": 60, "physical_processor": "Intel Xeon E5-2680 v2", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 8000.0, "vcpus": 8.0, "memory": 15.0, "ebs_max_bandwidth": 1000.0, "gpus": 0, "ipv6_support": true}, "g2.8xlarge": {"ecu_per_vcpu": 3.25, "network_perf": 16.0, "intel_avx": "", "name": "G2 Eight Extra Large", "architecture": "64-bit", "linux_virtualization": "Unknown", "storage": 240.0, "placement_group_support": true, "intel_avx2": "", "clock_speed_ghz": "", "apiname": "g2.8xlarge", "computeunits": 104.0, "ebs_throughput": 0.0, "vpc_only": false, "max_ips": 240, "physical_processor": "", "fpga": 0, "intel_turbo": "", "enhanced_networking": false, "ebs_iops": 0.0, "vcpus": 32.0, "memory": 60.0, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": false}, "t2.medium": {"ecu_per_vcpu": 0.0, "network_perf": 4.0, "intel_avx": "Yes", "name": "T2 Medium", "architecture": "32/64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "Yes", "apiname": "t2.medium", "computeunits": 0.4, "ebs_throughput": 0.0, "vpc_only": true, "max_ips": 18, "physical_processor": "Intel Xeon family", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": false, "ebs_iops": 0.0, "vcpus": 2.0, "memory": 4.0, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": true}, "m4.xlarge": {"ecu_per_vcpu": 3.25, "network_perf": 9.0, "intel_avx": "Yes", "name": "M4 Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "m4.xlarge", "computeunits": 13.0, "ebs_throughput": 93.75, "vpc_only": true, "max_ips": 60, "physical_processor": "Intel Xeon E5-2676 v3", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 6000.0, "vcpus": 4.0, "memory": 16.0, "ebs_max_bandwidth": 750.0, "gpus": 0, "ipv6_support": true}, "x1.16xlarge": {"ecu_per_vcpu": 2.7265625, "network_perf": 13.0, "intel_avx": "Yes", "name": "X1 Extra High-Memory 16xlarge", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 1920.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "x1.16xlarge", "computeunits": 174.5, "ebs_throughput": 875.0, "vpc_only": true, "max_ips": 240, "physical_processor": "Intel Xeon E7-8880 v3", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 40000.0, "vcpus": 64.0, "memory": 976.0, "ebs_max_bandwidth": 7000.0, "gpus": 0, "ipv6_support": true}, "p2.8xlarge": {"ecu_per_vcpu": 2.9375, "network_perf": 13.0, "intel_avx": "Yes", "name": "General Purpose GPU Eight Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "p2.8xlarge", "computeunits": 94.0, "ebs_throughput": 625.0, "vpc_only": true, "max_ips": 240, "physical_processor": "Intel Xeon E5-2686 v4", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 32500.0, "vcpus": 32.0, "memory": 488.0, "ebs_max_bandwidth": 5000.0, "gpus": 8, "ipv6_support": true}, "f1.16xlarge": {"ecu_per_vcpu": 2.9375, "network_perf": 17.0, "intel_avx": "Yes", "name": "F1 16xlarge", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 3760.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "f1.16xlarge", "computeunits": 188.0, "ebs_throughput": 1750.0, "vpc_only": true, "max_ips": 400, "physical_processor": "Intel Xeon E5-2686 v4", "fpga": 8, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 75000.0, "vcpus": 64.0, "memory": 976.0, "ebs_max_bandwidth": 14000.0, "gpus": 0, "ipv6_support": true}, "r4.8xlarge": {"ecu_per_vcpu": 3.09375, "network_perf": 13.0, "intel_avx": "Yes", "name": "R4 High-Memory Eight Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "r4.8xlarge", "computeunits": 99.0, "ebs_throughput": 875.0, "vpc_only": true, "max_ips": 240, "physical_processor": "Intel Xeon E5-2686 v4", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 37500.0, "vcpus": 32.0, "memory": 244.0, "ebs_max_bandwidth": 7000.0, "gpus": 0, "ipv6_support": true}, "g3.4xlarge": {"ecu_per_vcpu": 2.9375, "network_perf": 11.0, "intel_avx": "Yes", "name": "G3 Quadruple Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "g3.4xlarge", "computeunits": 47.0, "ebs_throughput": 437.0, "vpc_only": true, "max_ips": 240, "physical_processor": "Intel Xeon E5-2686 v4", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 20000.0, "vcpus": 16.0, "memory": 122.0, "ebs_max_bandwidth": 3500.0, "gpus": 1, "ipv6_support": true}, "cg1.4xlarge": {"ecu_per_vcpu": 2.09375, "network_perf": 12.0, "intel_avx": "", "name": "Cluster GPU Quadruple Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 1680.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "", "apiname": "cg1.4xlarge", "computeunits": 33.5, "ebs_throughput": 0.0, "vpc_only": false, "max_ips": 240, "physical_processor": "", "fpga": 0, "intel_turbo": "", "enhanced_networking": false, "ebs_iops": 0.0, "vcpus": 16.0, "memory": 22.5, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": false}, "c4.large": {"ecu_per_vcpu": 4.0, "network_perf": 7.0, "intel_avx": "Yes", "name": "C4 High-CPU Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "c4.large", "computeunits": 8.0, "ebs_throughput": 62.5, "vpc_only": true, "max_ips": 30, "physical_processor": "Intel Xeon E5-2666 v3", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 4000.0, "vcpus": 2.0, "memory": 3.75, "ebs_max_bandwidth": 500.0, "gpus": 0, "ipv6_support": true}, "m4.16xlarge": {"ecu_per_vcpu": 2.9375, "network_perf": 17.0, "intel_avx": "Yes", "name": "M4 16xlarge", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "m4.16xlarge", "computeunits": 188.0, "ebs_throughput": 1250.0, "vpc_only": true, "max_ips": 240, "physical_processor": "Intel Xeon E5-2686 v4", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 65000.0, "vcpus": 64.0, "memory": 256.0, "ebs_max_bandwidth": 10000.0, "gpus": 0, "ipv6_support": true}, "r4.4xlarge": {"ecu_per_vcpu": 3.3125, "network_perf": 11.0, "intel_avx": "Yes", "name": "R4 High-Memory Quadruple Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "r4.4xlarge", "computeunits": 53.0, "ebs_throughput": 437.0, "vpc_only": true, "max_ips": 240, "physical_processor": "Intel Xeon E5-2686 v4", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 18750.0, "vcpus": 16.0, "memory": 122.0, "ebs_max_bandwidth": 3500.0, "gpus": 0, "ipv6_support": true}, "r4.2xlarge": {"ecu_per_vcpu": 3.375, "network_perf": 11.0, "intel_avx": "Yes", "name": "R4 High-Memory Double Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "r4.2xlarge", "computeunits": 27.0, "ebs_throughput": 218.0, "vpc_only": true, "max_ips": 60, "physical_processor": "Intel Xeon E5-2686 v4", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 12000.0, "vcpus": 8.0, "memory": 61.0, "ebs_max_bandwidth": 1750.0, "gpus": 0, "ipv6_support": true}, "c3.xlarge": {"ecu_per_vcpu": 3.5, "network_perf": 7.0, "intel_avx": "Yes", "name": "C3 High-CPU Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM, PV", "storage": 80.0, "placement_group_support": true, "intel_avx2": "", "clock_speed_ghz": "Yes", "apiname": "c3.xlarge", "computeunits": 14.0, "ebs_throughput": 62.5, "vpc_only": false, "max_ips": 60, "physical_processor": "Intel Xeon E5-2680 v2", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 4000.0, "vcpus": 4.0, "memory": 7.5, "ebs_max_bandwidth": 500.0, "gpus": 0, "ipv6_support": true}, "i3.large": {"ecu_per_vcpu": 3.5, "network_perf": 11.0, "intel_avx": "Yes", "name": "I3 High I/O Large", "architecture": "64-bit", "linux_virtualization": "Unknown", "storage": 475.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "i3.large", "computeunits": 7.0, "ebs_throughput": 50.0, "vpc_only": true, "max_ips": 30, "physical_processor": "Intel Xeon E5-2686 v4", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 3000.0, "vcpus": 2.0, "memory": 15.25, "ebs_max_bandwidth": 425.0, "gpus": 0, "ipv6_support": true}, "r4.xlarge": {"ecu_per_vcpu": 3.375, "network_perf": 11.0, "intel_avx": "Yes", "name": "R4 High-Memory Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "r4.xlarge", "computeunits": 13.5, "ebs_throughput": 109.0, "vpc_only": true, "max_ips": 60, "physical_processor": "Intel Xeon E5-2686 v4", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 6000.0, "vcpus": 4.0, "memory": 30.5, "ebs_max_bandwidth": 875.0, "gpus": 0, "ipv6_support": true}, "m2.2xlarge": {"ecu_per_vcpu": 3.25, "network_perf": 7.0, "intel_avx": "", "name": "M2 High Memory Double Extra Large", "architecture": "64-bit", "linux_virtualization": "PV", "storage": 850.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "", "apiname": "m2.2xlarge", "computeunits": 13.0, "ebs_throughput": 62.5, "vpc_only": false, "max_ips": 120, "physical_processor": "", "fpga": 0, "intel_turbo": "", "enhanced_networking": false, "ebs_iops": 4000.0, "vcpus": 4.0, "memory": 34.2, "ebs_max_bandwidth": 500.0, "gpus": 0, "ipv6_support": false}, "m3.medium": {"ecu_per_vcpu": 3.0, "network_perf": 6.0, "intel_avx": "Yes", "name": "M3 General Purpose Medium", "architecture": "64-bit", "linux_virtualization": "HVM, PV", "storage": 4.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "Yes", "apiname": "m3.medium", "computeunits": 3.0, "ebs_throughput": 0.0, "vpc_only": false, "max_ips": 12, "physical_processor": "Intel Xeon E5-2670 v2", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": false, "ebs_iops": 0.0, "vcpus": 1.0, "memory": 3.75, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": false}, "r3.4xlarge": {"ecu_per_vcpu": 3.25, "network_perf": 9.0, "intel_avx": "Yes", "name": "R3 High-Memory Quadruple Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 320.0, "placement_group_support": true, "intel_avx2": "", "clock_speed_ghz": "Yes", "apiname": "r3.4xlarge", "computeunits": 52.0, "ebs_throughput": 250.0, "vpc_only": false, "max_ips": 240, "physical_processor": "Intel Xeon E5-2670 v2", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 16000.0, "vcpus": 16.0, "memory": 122.0, "ebs_max_bandwidth": 2000.0, "gpus": 0, "ipv6_support": true}, "t2.small": {"ecu_per_vcpu": 0.0, "network_perf": 4.0, "intel_avx": "Yes", "name": "T2 Small", "architecture": "32/64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "Yes", "apiname": "t2.small", "computeunits": 0.2, "ebs_throughput": 0.0, "vpc_only": true, "max_ips": 8, "physical_processor": "Intel Xeon family", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": false, "ebs_iops": 0.0, "vcpus": 1.0, "memory": 2.0, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": true}, "r3.large": {"ecu_per_vcpu": 3.25, "network_perf": 6.0, "intel_avx": "Yes", "name": "R3 High-Memory Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 32.0, "placement_group_support": true, "intel_avx2": "", "clock_speed_ghz": "Yes", "apiname": "r3.large", "computeunits": 6.5, "ebs_throughput": 0.0, "vpc_only": false, "max_ips": 30, "physical_processor": "Intel Xeon E5-2670 v2", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 0.0, "vcpus": 2.0, "memory": 15.25, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": true}, "i3.16xlarge": {"ecu_per_vcpu": 3.125, "network_perf": 17.0, "intel_avx": "Yes", "name": "I3 High I/O 16xlarge", "architecture": "64-bit", "linux_virtualization": "Unknown", "storage": 15200.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "i3.16xlarge", "computeunits": 200.0, "ebs_throughput": 1750.0, "vpc_only": true, "max_ips": 750, "physical_processor": "Intel Xeon E5-2686 v4", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 65000.0, "vcpus": 64.0, "memory": 488.0, "ebs_max_bandwidth": 14000.0, "gpus": 0, "ipv6_support": true}, "c3.large": {"ecu_per_vcpu": 3.5, "network_perf": 6.0, "intel_avx": "Yes", "name": "C3 High-CPU Large", "architecture": "32/64-bit", "linux_virtualization": "HVM, PV", "storage": 32.0, "placement_group_support": true, "intel_avx2": "", "clock_speed_ghz": "Yes", "apiname": "c3.large", "computeunits": 7.0, "ebs_throughput": 0.0, "vpc_only": false, "max_ips": 30, "physical_processor": "Intel Xeon E5-2680 v2", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 0.0, "vcpus": 2.0, "memory": 3.75, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": true}, "i2.2xlarge": {"ecu_per_vcpu": 3.375, "network_perf": 7.0, "intel_avx": "", "name": "I2 Double Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 1600.0, "placement_group_support": true, "intel_avx2": "", "clock_speed_ghz": "", "apiname": "i2.2xlarge", "computeunits": 27.0, "ebs_throughput": 125.0, "vpc_only": false, "max_ips": 60, "physical_processor": "", "fpga": 0, "intel_turbo": "", "enhanced_networking": false, "ebs_iops": 8000.0, "vcpus": 8.0, "memory": 61.0, "ebs_max_bandwidth": 1000.0, "gpus": 0, "ipv6_support": true}, "i3.xlarge": {"ecu_per_vcpu": 3.25, "network_perf": 11.0, "intel_avx": "Yes", "name": "I3 High I/O Extra Large", "architecture": "64-bit", "linux_virtualization": "Unknown", "storage": 950.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "i3.xlarge", "computeunits": 13.0, "ebs_throughput": 100.0, "vpc_only": true, "max_ips": 60, "physical_processor": "Intel Xeon E5-2686 v4", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 6000.0, "vcpus": 4.0, "memory": 30.5, "ebs_max_bandwidth": 850.0, "gpus": 0, "ipv6_support": true}, "i2.8xlarge": {"ecu_per_vcpu": 3.25, "network_perf": 13.0, "intel_avx": "", "name": "I2 Eight Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 6400.0, "placement_group_support": true, "intel_avx2": "", "clock_speed_ghz": "", "apiname": "i2.8xlarge", "computeunits": 104.0, "ebs_throughput": 0.0, "vpc_only": false, "max_ips": 240, "physical_processor": "", "fpga": 0, "intel_turbo": "", "enhanced_networking": false, "ebs_iops": 0.0, "vcpus": 32.0, "memory": 244.0, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": true}, "r4.16xlarge": {"ecu_per_vcpu": 3.046875, "network_perf": 17.0, "intel_avx": "Yes", "name": "R4 High-Memory 16xlarge", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "r4.16xlarge", "computeunits": 195.0, "ebs_throughput": 1750.0, "vpc_only": true, "max_ips": 750, "physical_processor": "Intel Xeon E5-2686 v4", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 75000.0, "vcpus": 64.0, "memory": 488.0, "ebs_max_bandwidth": 14000.0, "gpus": 0, "ipv6_support": true}, "g3.8xlarge": {"ecu_per_vcpu": 2.9375, "network_perf": 13.0, "intel_avx": "Yes", "name": "G3 Eight Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "g3.8xlarge", "computeunits": 94.0, "ebs_throughput": 875.0, "vpc_only": true, "max_ips": 240, "physical_processor": "Intel Xeon E5-2686 v4", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 40000.0, "vcpus": 32.0, "memory": 244.0, "ebs_max_bandwidth": 7000.0, "gpus": 2, "ipv6_support": true}, "c3.4xlarge": {"ecu_per_vcpu": 3.4375, "network_perf": 9.0, "intel_avx": "Yes", "name": "C3 High-CPU Quadruple Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM, PV", "storage": 320.0, "placement_group_support": true, "intel_avx2": "", "clock_speed_ghz": "Yes", "apiname": "c3.4xlarge", "computeunits": 55.0, "ebs_throughput": 250.0, "vpc_only": false, "max_ips": 240, "physical_processor": "Intel Xeon E5-2680 v2", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 16000.0, "vcpus": 16.0, "memory": 30.0, "ebs_max_bandwidth": 2000.0, "gpus": 0, "ipv6_support": true}, "r4.large": {"ecu_per_vcpu": 3.5, "network_perf": 11.0, "intel_avx": "Yes", "name": "R4 High-Memory Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "r4.large", "computeunits": 7.0, "ebs_throughput": 54.0, "vpc_only": true, "max_ips": 30, "physical_processor": "Intel Xeon E5-2686 v4", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 3000.0, "vcpus": 2.0, "memory": 15.25, "ebs_max_bandwidth": 437.0, "gpus": 0, "ipv6_support": true}, "f1.2xlarge": {"ecu_per_vcpu": 3.25, "network_perf": 11.0, "intel_avx": "Yes", "name": "F1 Double Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 470.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "f1.2xlarge", "computeunits": 26.0, "ebs_throughput": 200.0, "vpc_only": true, "max_ips": 60, "physical_processor": "Intel Xeon E5-2686 v4", "fpga": 1, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 12000.0, "vcpus": 8.0, "memory": 122.0, "ebs_max_bandwidth": 1700.0, "gpus": 0, "ipv6_support": true}, "m4.2xlarge": {"ecu_per_vcpu": 3.25, "network_perf": 9.0, "intel_avx": "Yes", "name": "M4 Double Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "m4.2xlarge", "computeunits": 26.0, "ebs_throughput": 125.0, "vpc_only": true, "max_ips": 60, "physical_processor": "Intel Xeon E5-2676 v3", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 8000.0, "vcpus": 8.0, "memory": 32.0, "ebs_max_bandwidth": 1000.0, "gpus": 0, "ipv6_support": true}, "m3.2xlarge": {"ecu_per_vcpu": 3.25, "network_perf": 9.0, "intel_avx": "Yes", "name": "M3 General Purpose Double Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM, PV", "storage": 160.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "Yes", "apiname": "m3.2xlarge", "computeunits": 26.0, "ebs_throughput": 125.0, "vpc_only": false, "max_ips": 120, "physical_processor": "Intel Xeon E5-2670 v2", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": false, "ebs_iops": 8000.0, "vcpus": 8.0, "memory": 30.0, "ebs_max_bandwidth": 1000.0, "gpus": 0, "ipv6_support": false}, "c3.8xlarge": {"ecu_per_vcpu": 3.375, "network_perf": 12.0, "intel_avx": "Yes", "name": "C3 High-CPU Eight Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM, PV", "storage": 640.0, "placement_group_support": true, "intel_avx2": "", "clock_speed_ghz": "Yes", "apiname": "c3.8xlarge", "computeunits": 108.0, "ebs_throughput": 0.0, "vpc_only": false, "max_ips": 240, "physical_processor": "Intel Xeon E5-2680 v2", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 0.0, "vcpus": 32.0, "memory": 60.0, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": true}, "cr1.8xlarge": {"ecu_per_vcpu": 2.75, "network_perf": 12.0, "intel_avx": "", "name": "High Memory Cluster Eight Extra Large", "architecture": "64-bit", "linux_virtualization": "Unknown", "storage": 240.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "", "apiname": "cr1.8xlarge", "computeunits": 88.0, "ebs_throughput": 0.0, "vpc_only": false, "max_ips": 240, "physical_processor": "", "fpga": 0, "intel_turbo": "", "enhanced_networking": false, "ebs_iops": 0.0, "vcpus": 32.0, "memory": 244.0, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": false}, "cc2.8xlarge": {"ecu_per_vcpu": 2.75, "network_perf": 12.0, "intel_avx": "", "name": "Cluster Compute Eight Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 3360.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "", "apiname": "cc2.8xlarge", "computeunits": 88.0, "ebs_throughput": 0.0, "vpc_only": false, "max_ips": 240, "physical_processor": "", "fpga": 0, "intel_turbo": "", "enhanced_networking": false, "ebs_iops": 0.0, "vcpus": 32.0, "memory": 60.5, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": false}, "m1.large": {"ecu_per_vcpu": 2.0, "network_perf": 7.0, "intel_avx": "", "name": "M1 General Purpose Large", "architecture": "64-bit", "linux_virtualization": "PV", "storage": 840.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "", "apiname": "m1.large", "computeunits": 4.0, "ebs_throughput": 62.5, "vpc_only": false, "max_ips": 30, "physical_processor": "", "fpga": 0, "intel_turbo": "", "enhanced_networking": false, "ebs_iops": 4000.0, "vcpus": 2.0, "memory": 7.5, "ebs_max_bandwidth": 500.0, "gpus": 0, "ipv6_support": false}, "r3.xlarge": {"ecu_per_vcpu": 3.25, "network_perf": 7.0, "intel_avx": "Yes", "name": "R3 High-Memory Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 80.0, "placement_group_support": true, "intel_avx2": "", "clock_speed_ghz": "Yes", "apiname": "r3.xlarge", "computeunits": 13.0, "ebs_throughput": 62.5, "vpc_only": false, "max_ips": 60, "physical_processor": "Intel Xeon E5-2670 v2", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 4000.0, "vcpus": 4.0, "memory": 30.5, "ebs_max_bandwidth": 500.0, "gpus": 0, "ipv6_support": true}, "g3.16xlarge": {"ecu_per_vcpu": 2.9375, "network_perf": 17.0, "intel_avx": "Yes", "name": "G3 16xlarge", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "g3.16xlarge", "computeunits": 188.0, "ebs_throughput": 1750.0, "vpc_only": true, "max_ips": 750, "physical_processor": "Intel Xeon E5-2686 v4", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 80000.0, "vcpus": 64.0, "memory": 488.0, "ebs_max_bandwidth": 14000.0, "gpus": 4, "ipv6_support": true}, "m1.medium": {"ecu_per_vcpu": 2.0, "network_perf": 6.0, "intel_avx": "", "name": "M1 General Purpose Medium", "architecture": "32/64-bit", "linux_virtualization": "PV", "storage": 410.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "", "apiname": "m1.medium", "computeunits": 2.0, "ebs_throughput": 0.0, "vpc_only": false, "max_ips": 12, "physical_processor": "", "fpga": 0, "intel_turbo": "", "enhanced_networking": false, "ebs_iops": 0.0, "vcpus": 1.0, "memory": 3.75, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": false}, "i3.2xlarge": {"ecu_per_vcpu": 3.375, "network_perf": 11.0, "intel_avx": "Yes", "name": "I3 High I/O Double Extra Large", "architecture": "64-bit", "linux_virtualization": "Unknown", "storage": 1900.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "i3.2xlarge", "computeunits": 27.0, "ebs_throughput": 200.0, "vpc_only": true, "max_ips": 60, "physical_processor": "Intel Xeon E5-2686 v4", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 12000.0, "vcpus": 8.0, "memory": 61.0, "ebs_max_bandwidth": 1700.0, "gpus": 0, "ipv6_support": true}, "t2.xlarge": {"ecu_per_vcpu": 0.0, "network_perf": 6.0, "intel_avx": "Yes", "name": "T2 Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "Yes", "apiname": "t2.xlarge", "computeunits": 0.9, "ebs_throughput": 0.0, "vpc_only": true, "max_ips": 45, "physical_processor": "Intel Xeon family", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": false, "ebs_iops": 0.0, "vcpus": 4.0, "memory": 16.0, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": true}, "g2.2xlarge": {"ecu_per_vcpu": 3.25, "network_perf": 16.0, "intel_avx": "", "name": "G2 Double Extra Large", "architecture": "64-bit", "linux_virtualization": "Unknown", "storage": 60.0, "placement_group_support": true, "intel_avx2": "", "clock_speed_ghz": "", "apiname": "g2.2xlarge", "computeunits": 26.0, "ebs_throughput": 125.0, "vpc_only": false, "max_ips": 60, "physical_processor": "", "fpga": 0, "intel_turbo": "", "enhanced_networking": false, "ebs_iops": 8000.0, "vcpus": 8.0, "memory": 15.0, "ebs_max_bandwidth": 1000.0, "gpus": 0, "ipv6_support": false}, "c1.medium": {"ecu_per_vcpu": 2.5, "network_perf": 6.0, "intel_avx": "", "name": "C1 High-CPU Medium", "architecture": "32/64-bit", "linux_virtualization": "PV", "storage": 350.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "", "apiname": "c1.medium", "computeunits": 5.0, "ebs_throughput": 0.0, "vpc_only": false, "max_ips": 12, "physical_processor": "", "fpga": 0, "intel_turbo": "", "enhanced_networking": false, "ebs_iops": 0.0, "vcpus": 2.0, "memory": 1.7, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": false}, "t2.large": {"ecu_per_vcpu": 0.0, "network_perf": 4.0, "intel_avx": "Yes", "name": "T2 Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "Yes", "apiname": "t2.large", "computeunits": 0.6, "ebs_throughput": 0.0, "vpc_only": true, "max_ips": 36, "physical_processor": "Intel Xeon family", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": false, "ebs_iops": 0.0, "vcpus": 2.0, "memory": 8.0, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": true}, "d2.2xlarge": {"ecu_per_vcpu": 3.5, "network_perf": 9.0, "intel_avx": "Yes", "name": "D2 Double Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 12000.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "d2.2xlarge", "computeunits": 28.0, "ebs_throughput": 125.0, "vpc_only": false, "max_ips": 60, "physical_processor": "Intel Xeon E5-2676 v3", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 8000.0, "vcpus": 8.0, "memory": 61.0, "ebs_max_bandwidth": 1000.0, "gpus": 0, "ipv6_support": true}, "c4.8xlarge": {"ecu_per_vcpu": 3.66666666667, "network_perf": 13.0, "intel_avx": "Yes", "name": "C4 High-CPU Eight Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "c4.8xlarge", "computeunits": 132.0, "ebs_throughput": 500.0, "vpc_only": true, "max_ips": 240, "physical_processor": "Intel Xeon E5-2666 v3", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 32000.0, "vcpus": 36.0, "memory": 60.0, "ebs_max_bandwidth": 4000.0, "gpus": 0, "ipv6_support": true}, "c4.2xlarge": {"ecu_per_vcpu": 3.875, "network_perf": 9.0, "intel_avx": "Yes", "name": "C4 High-CPU Double Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "c4.2xlarge", "computeunits": 31.0, "ebs_throughput": 125.0, "vpc_only": true, "max_ips": 60, "physical_processor": "Intel Xeon E5-2666 v3", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 8000.0, "vcpus": 8.0, "memory": 15.0, "ebs_max_bandwidth": 1000.0, "gpus": 0, "ipv6_support": true}, "x1e.32xlarge": {"ecu_per_vcpu": 2.65625, "network_perf": 17.0, "intel_avx": "Yes", "name": "X1E 32xlarge", "architecture": "64-bit", "linux_virtualization": "Unknown", "storage": 3840.0, "placement_group_support": false, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "x1e.32xlarge", "computeunits": 340.0, "ebs_throughput": 1750.0, "vpc_only": true, "max_ips": 240, "physical_processor": "Intel Xeon E7-8880 v3", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 80000.0, "vcpus": 128.0, "memory": 3904.0, "ebs_max_bandwidth": 14000.0, "gpus": 0, "ipv6_support": false}, "m4.10xlarge": {"ecu_per_vcpu": 3.1125, "network_perf": 13.0, "intel_avx": "Yes", "name": "M4 Deca Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "m4.10xlarge", "computeunits": 124.5, "ebs_throughput": 500.0, "vpc_only": true, "max_ips": 240, "physical_processor": "Intel Xeon E5-2676 v3", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 32000.0, "vcpus": 40.0, "memory": 160.0, "ebs_max_bandwidth": 4000.0, "gpus": 0, "ipv6_support": true}, "t2.2xlarge": {"ecu_per_vcpu": 0.0, "network_perf": 6.0, "intel_avx": "Yes", "name": "T2 Double Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "Yes", "apiname": "t2.2xlarge", "computeunits": 1.35, "ebs_throughput": 0.0, "vpc_only": true, "max_ips": 45, "physical_processor": "Intel Xeon family", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": false, "ebs_iops": 0.0, "vcpus": 8.0, "memory": 32.0, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": true}, "m4.4xlarge": {"ecu_per_vcpu": 3.34375, "network_perf": 9.0, "intel_avx": "Yes", "name": "M4 Quadruple Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "m4.4xlarge", "computeunits": 53.5, "ebs_throughput": 250.0, "vpc_only": true, "max_ips": 240, "physical_processor": "Intel Xeon E5-2676 v3", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 16000.0, "vcpus": 16.0, "memory": 64.0, "ebs_max_bandwidth": 2000.0, "gpus": 0, "ipv6_support": true}, "t2.nano": {"ecu_per_vcpu": 0.0, "network_perf": 2.0, "intel_avx": "Yes", "name": "T2 Nano", "architecture": "32/64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "Yes", "apiname": "t2.nano", "computeunits": 0.05, "ebs_throughput": 0.0, "vpc_only": true, "max_ips": 4, "physical_processor": "Intel Xeon family", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": false, "ebs_iops": 0.0, "vcpus": 1.0, "memory": 0.5, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": true}, "d2.8xlarge": {"ecu_per_vcpu": 3.22222222222, "network_perf": 13.0, "intel_avx": "Yes", "name": "D2 Eight Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 48000.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "d2.8xlarge", "computeunits": 116.0, "ebs_throughput": 500.0, "vpc_only": false, "max_ips": 240, "physical_processor": "Intel Xeon E5-2676 v3", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 32000.0, "vcpus": 36.0, "memory": 244.0, "ebs_max_bandwidth": 4000.0, "gpus": 0, "ipv6_support": true}, "m3.large": {"ecu_per_vcpu": 3.25, "network_perf": 6.0, "intel_avx": "Yes", "name": "M3 General Purpose Large", "architecture": "64-bit", "linux_virtualization": "HVM, PV", "storage": 32.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "Yes", "apiname": "m3.large", "computeunits": 6.5, "ebs_throughput": 0.0, "vpc_only": false, "max_ips": 30, "physical_processor": "Intel Xeon E5-2670 v2", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": false, "ebs_iops": 0.0, "vcpus": 2.0, "memory": 7.5, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": false}, "m2.4xlarge": {"ecu_per_vcpu": 3.25, "network_perf": 9.0, "intel_avx": "", "name": "M2 High Memory Quadruple Extra Large", "architecture": "64-bit", "linux_virtualization": "PV", "storage": 1680.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "", "apiname": "m2.4xlarge", "computeunits": 26.0, "ebs_throughput": 125.0, "vpc_only": false, "max_ips": 240, "physical_processor": "", "fpga": 0, "intel_turbo": "", "enhanced_networking": false, "ebs_iops": 8000.0, "vcpus": 8.0, "memory": 68.4, "ebs_max_bandwidth": 1000.0, "gpus": 0, "ipv6_support": false}, "m1.small": {"ecu_per_vcpu": 1.0, "network_perf": 2.0, "intel_avx": "", "name": "M1 General Purpose Small", "architecture": "32/64-bit", "linux_virtualization": "PV", "storage": 160.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "", "apiname": "m1.small", "computeunits": 1.0, "ebs_throughput": 0.0, "vpc_only": false, "max_ips": 8, "physical_processor": "", "fpga": 0, "intel_turbo": "", "enhanced_networking": false, "ebs_iops": 0.0, "vcpus": 1.0, "memory": 1.7, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": false}, "c1.xlarge": {"ecu_per_vcpu": 2.5, "network_perf": 9.0, "intel_avx": "", "name": "C1 High-CPU Extra Large", "architecture": "64-bit", "linux_virtualization": "PV", "storage": 1680.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "", "apiname": "c1.xlarge", "computeunits": 20.0, "ebs_throughput": 125.0, "vpc_only": false, "max_ips": 60, "physical_processor": "", "fpga": 0, "intel_turbo": "", "enhanced_networking": false, "ebs_iops": 8000.0, "vcpus": 8.0, "memory": 7.0, "ebs_max_bandwidth": 1000.0, "gpus": 0, "ipv6_support": false}, "x1.32xlarge": {"ecu_per_vcpu": 2.7265625, "network_perf": 17.0, "intel_avx": "Yes", "name": "X1 Extra High-Memory 32xlarge", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 3840.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "x1.32xlarge", "computeunits": 349.0, "ebs_throughput": 1750.0, "vpc_only": true, "max_ips": 240, "physical_processor": "Intel Xeon E7-8880 v3", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 80000.0, "vcpus": 128.0, "memory": 1952.0, "ebs_max_bandwidth": 14000.0, "gpus": 0, "ipv6_support": true}, "r3.8xlarge": {"ecu_per_vcpu": 3.25, "network_perf": 12.0, "intel_avx": "Yes", "name": "R3 High-Memory Eight Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 640.0, "placement_group_support": true, "intel_avx2": "", "clock_speed_ghz": "Yes", "apiname": "r3.8xlarge", "computeunits": 104.0, "ebs_throughput": 0.0, "vpc_only": false, "max_ips": 240, "physical_processor": "Intel Xeon E5-2670 v2", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 0.0, "vcpus": 32.0, "memory": 244.0, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": true}, "m4.large": {"ecu_per_vcpu": 3.25, "network_perf": 7.0, "intel_avx": "Yes", "name": "M4 Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "m4.large", "computeunits": 6.5, "ebs_throughput": 56.25, "vpc_only": true, "max_ips": 20, "physical_processor": "Intel Xeon E5-2676 v3", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 3600.0, "vcpus": 2.0, "memory": 8.0, "ebs_max_bandwidth": 450.0, "gpus": 0, "ipv6_support": true}, "p2.16xlarge": {"ecu_per_vcpu": 2.9375, "network_perf": 17.0, "intel_avx": "Yes", "name": "General Purpose GPU 16xlarge", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "p2.16xlarge", "computeunits": 188.0, "ebs_throughput": 1250.0, "vpc_only": true, "max_ips": 240, "physical_processor": "Intel Xeon E5-2686 v4", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 65000.0, "vcpus": 64.0, "memory": 732.0, "ebs_max_bandwidth": 10000.0, "gpus": 16, "ipv6_support": true}, "hi1.4xlarge": {"ecu_per_vcpu": 2.1875, "network_perf": 12.0, "intel_avx": "", "name": "HI1. High I/O Quadruple Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM, PV", "storage": 2048.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "", "apiname": "hi1.4xlarge", "computeunits": 35.0, "ebs_throughput": 0.0, "vpc_only": false, "max_ips": 240, "physical_processor": "", "fpga": 0, "intel_turbo": "", "enhanced_networking": false, "ebs_iops": 0.0, "vcpus": 16.0, "memory": 60.5, "ebs_max_bandwidth": 0.0, "gpus": 0, "ipv6_support": false}, "c4.4xlarge": {"ecu_per_vcpu": 3.875, "network_perf": 9.0, "intel_avx": "Yes", "name": "C4 High-CPU Quadruple Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "c4.4xlarge", "computeunits": 62.0, "ebs_throughput": 250.0, "vpc_only": true, "max_ips": 240, "physical_processor": "Intel Xeon E5-2666 v3", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 16000.0, "vcpus": 16.0, "memory": 30.0, "ebs_max_bandwidth": 2000.0, "gpus": 0, "ipv6_support": true}, "c4.xlarge": {"ecu_per_vcpu": 4.0, "network_perf": 9.0, "intel_avx": "Yes", "name": "C4 High-CPU Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM", "storage": 0.0, "placement_group_support": true, "intel_avx2": "Yes", "clock_speed_ghz": "Yes", "apiname": "c4.xlarge", "computeunits": 16.0, "ebs_throughput": 93.75, "vpc_only": true, "max_ips": 60, "physical_processor": "Intel Xeon E5-2666 v3", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": true, "ebs_iops": 6000.0, "vcpus": 4.0, "memory": 7.5, "ebs_max_bandwidth": 750.0, "gpus": 0, "ipv6_support": true}, "m3.xlarge": {"ecu_per_vcpu": 3.25, "network_perf": 9.0, "intel_avx": "Yes", "name": "M3 General Purpose Extra Large", "architecture": "64-bit", "linux_virtualization": "HVM, PV", "storage": 80.0, "placement_group_support": false, "intel_avx2": "", "clock_speed_ghz": "Yes", "apiname": "m3.xlarge", "computeunits": 13.0, "ebs_throughput": 62.5, "vpc_only": false, "max_ips": 60, "physical_processor": "Intel Xeon E5-2670 v2", "fpga": 0, "intel_turbo": "Yes", "enhanced_networking": false, "ebs_iops": 4000.0, "vcpus": 4.0, "memory": 15.0, "ebs_max_bandwidth": 500.0, "gpus": 0, "ipv6_support": false}} \ No newline at end of file +{"a1.2xlarge": {"apiname": "a1.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 0.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 16.0, "name": "A1 Double Extra Large", "network_perf": 10.0, "physical_processor": "AWS Graviton Processor", "placement_group_support": false, "storage": 0.0, "vcpus": 8.0, "vpc_only": true}, "a1.4xlarge": {"apiname": "a1.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 0.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 32.0, "name": "A1 Quadruple Extra Large", "network_perf": 10.0, "physical_processor": "AWS Graviton Processor", "placement_group_support": false, "storage": 0.0, "vcpus": 16.0, "vpc_only": true}, "a1.large": {"apiname": "a1.large", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 0.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 30, "memory": 4.0, "name": "A1 Large", "network_perf": 10.0, "physical_processor": "AWS Graviton Processor", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "a1.medium": {"apiname": "a1.medium", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 0.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 8, "memory": 2.0, "name": "A1 Medium", "network_perf": 10.0, "physical_processor": "AWS Graviton Processor", "placement_group_support": false, "storage": 0.0, "vcpus": 1.0, "vpc_only": true}, "a1.metal": {"apiname": "a1.metal", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 32.0, "name": "A1 Metal", "network_perf": 10.0, "physical_processor": "AWS Graviton Processor", "placement_group_support": false, "storage": 0.0, "vcpus": 16.0, "vpc_only": true}, "a1.xlarge": {"apiname": "a1.xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 0.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 8.0, "name": "A1 Extra Large", "network_perf": 10.0, "physical_processor": "AWS Graviton Processor", "placement_group_support": false, "storage": 0.0, "vcpus": 4.0, "vpc_only": true}, "c1.medium": {"apiname": "c1.medium", "architecture": "32/64-bit", "clock_speed_ghz": "unknown", "computeunits": 5.0, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 2.5, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": false, "linux_virtualization": "PV", "max_ips": 12, "memory": 1.7, "name": "C1 High-CPU Medium", "network_perf": 6.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 350.0, "vcpus": 2.0, "vpc_only": false}, "c1.xlarge": {"apiname": "c1.xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 20.0, "ebs_iops": 8000.0, "ebs_max_bandwidth": 1000.0, "ebs_throughput": 125.0, "ecu_per_vcpu": 2.5, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": false, "linux_virtualization": "PV", "max_ips": 60, "memory": 7.0, "name": "C1 High-CPU Extra Large", "network_perf": 8.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 1680.0, "vcpus": 8.0, "vpc_only": false}, "c3.2xlarge": {"apiname": "c3.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.8 GHz", "computeunits": 28.0, "ebs_iops": 8000.0, "ebs_max_bandwidth": 1000.0, "ebs_throughput": 125.0, "ecu_per_vcpu": 3.5, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM, PV", "max_ips": 60, "memory": 15.0, "name": "C3 High-CPU Double Extra Large", "network_perf": 8.0, "physical_processor": "Intel Xeon E5-2680 v2 (Ivy Bridge)", "placement_group_support": false, "storage": 160.0, "vcpus": 8.0, "vpc_only": false}, "c3.4xlarge": {"apiname": "c3.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.8 GHz", "computeunits": 55.0, "ebs_iops": 16000.0, "ebs_max_bandwidth": 2000.0, "ebs_throughput": 250.0, "ecu_per_vcpu": 3.4375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM, PV", "max_ips": 240, "memory": 30.0, "name": "C3 High-CPU Quadruple Extra Large", "network_perf": 8.0, "physical_processor": "Intel Xeon E5-2680 v2 (Ivy Bridge)", "placement_group_support": false, "storage": 320.0, "vcpus": 16.0, "vpc_only": false}, "c3.8xlarge": {"apiname": "c3.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.8 GHz", "computeunits": 108.0, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 3.375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM, PV", "max_ips": 240, "memory": 60.0, "name": "C3 High-CPU Eight Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon E5-2680 v2 (Ivy Bridge)", "placement_group_support": false, "storage": 640.0, "vcpus": 32.0, "vpc_only": false}, "c3.large": {"apiname": "c3.large", "architecture": "32/64-bit", "clock_speed_ghz": "2.8 GHz", "computeunits": 7.0, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 3.5, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM, PV", "max_ips": 30, "memory": 3.75, "name": "C3 High-CPU Large", "network_perf": 6.0, "physical_processor": "Intel Xeon E5-2680 v2 (Ivy Bridge)", "placement_group_support": false, "storage": 32.0, "vcpus": 2.0, "vpc_only": false}, "c3.xlarge": {"apiname": "c3.xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.8 GHz", "computeunits": 14.0, "ebs_iops": 4000.0, "ebs_max_bandwidth": 500.0, "ebs_throughput": 62.5, "ecu_per_vcpu": 3.5, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM, PV", "max_ips": 60, "memory": 7.5, "name": "C3 High-CPU Extra Large", "network_perf": 6.0, "physical_processor": "Intel Xeon E5-2680 v2 (Ivy Bridge)", "placement_group_support": false, "storage": 80.0, "vcpus": 4.0, "vpc_only": false}, "c4.2xlarge": {"apiname": "c4.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.9 GHz", "computeunits": 31.0, "ebs_iops": 8000.0, "ebs_max_bandwidth": 1000.0, "ebs_throughput": 125.0, "ecu_per_vcpu": 3.875, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 15.0, "name": "C4 High-CPU Double Extra Large", "network_perf": 8.0, "physical_processor": "Intel Xeon E5-2666 v3 (Haswell)", "placement_group_support": false, "storage": 0.0, "vcpus": 8.0, "vpc_only": true}, "c4.4xlarge": {"apiname": "c4.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.9 GHz", "computeunits": 62.0, "ebs_iops": 16000.0, "ebs_max_bandwidth": 2000.0, "ebs_throughput": 250.0, "ecu_per_vcpu": 3.875, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 30.0, "name": "C4 High-CPU Quadruple Extra Large", "network_perf": 8.0, "physical_processor": "Intel Xeon E5-2666 v3 (Haswell)", "placement_group_support": false, "storage": 0.0, "vcpus": 16.0, "vpc_only": true}, "c4.8xlarge": {"apiname": "c4.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.9 GHz", "computeunits": 132.0, "ebs_iops": 32000.0, "ebs_max_bandwidth": 4000.0, "ebs_throughput": 500.0, "ecu_per_vcpu": 3.6666666666666665, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 60.0, "name": "C4 High-CPU Eight Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon E5-2666 v3 (Haswell)", "placement_group_support": false, "storage": 0.0, "vcpus": 36.0, "vpc_only": true}, "c4.large": {"apiname": "c4.large", "architecture": "64-bit", "clock_speed_ghz": "2.9 GHz", "computeunits": 8.0, "ebs_iops": 4000.0, "ebs_max_bandwidth": 500.0, "ebs_throughput": 62.5, "ecu_per_vcpu": 4.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 30, "memory": 3.75, "name": "C4 High-CPU Large", "network_perf": 6.0, "physical_processor": "Intel Xeon E5-2666 v3 (Haswell)", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "c4.xlarge": {"apiname": "c4.xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.9 GHz", "computeunits": 16.0, "ebs_iops": 6000.0, "ebs_max_bandwidth": 750.0, "ebs_throughput": 93.75, "ecu_per_vcpu": 4.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 7.5, "name": "C4 High-CPU Extra Large", "network_perf": 8.0, "physical_processor": "Intel Xeon E5-2666 v3 (Haswell)", "placement_group_support": false, "storage": 0.0, "vcpus": 4.0, "vpc_only": true}, "c5.12xlarge": {"apiname": "c5.12xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 188.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 3.9166666666666665, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 96.0, "name": "C5 High-CPU 12xlarge", "network_perf": 14.0, "physical_processor": "Intel Xeon Platinum 8275L", "placement_group_support": false, "storage": 0.0, "vcpus": 48.0, "vpc_only": true}, "c5.18xlarge": {"apiname": "c5.18xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 281.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 3.9027777777777777, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 144.0, "name": "C5 High-CPU 18xlarge", "network_perf": 20.0, "physical_processor": "Intel Xeon Platinum 8124M", "placement_group_support": false, "storage": 0.0, "vcpus": 72.0, "vpc_only": true}, "c5.24xlarge": {"apiname": "c5.24xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 375.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 3.90625, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 192.0, "name": "C5 High-CPU 24xlarge", "network_perf": 20.0, "physical_processor": "Intel Xeon Platinum 8275L", "placement_group_support": false, "storage": 0.0, "vcpus": 96.0, "vpc_only": true}, "c5.2xlarge": {"apiname": "c5.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 34.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 4.25, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 16.0, "name": "C5 High-CPU Double Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon Platinum 8124M", "placement_group_support": false, "storage": 0.0, "vcpus": 8.0, "vpc_only": true}, "c5.4xlarge": {"apiname": "c5.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 68.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 4.25, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 32.0, "name": "C5 High-CPU Quadruple Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon Platinum 8124M", "placement_group_support": false, "storage": 0.0, "vcpus": 16.0, "vpc_only": true}, "c5.9xlarge": {"apiname": "c5.9xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 141.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 3.9166666666666665, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 72.0, "name": "C5 High-CPU 9xlarge", "network_perf": 12.0, "physical_processor": "Intel Xeon Platinum 8124M", "placement_group_support": false, "storage": 0.0, "vcpus": 36.0, "vpc_only": true}, "c5.large": {"apiname": "c5.large", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 9.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 4.5, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 30, "memory": 4.0, "name": "C5 High-CPU Large", "network_perf": 10.0, "physical_processor": "Intel Xeon Platinum 8124M", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "c5.metal": {"apiname": "c5.metal", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 375.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 3.90625, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 192.0, "name": "C5 High-CPU Metal", "network_perf": 20.0, "physical_processor": "Intel Xeon Platinum 8275L", "placement_group_support": false, "storage": 0.0, "vcpus": 96.0, "vpc_only": true}, "c5.xlarge": {"apiname": "c5.xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 17.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 4.25, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 8.0, "name": "C5 High-CPU Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon Platinum 8124M", "placement_group_support": false, "storage": 0.0, "vcpus": 4.0, "vpc_only": true}, "c5d.12xlarge": {"apiname": "c5d.12xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 188.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 3.9166666666666665, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 96.0, "name": "C5 High-CPU 12xlarge", "network_perf": 14.0, "physical_processor": "Intel Xeon Platinum 8275L", "placement_group_support": false, "storage": 1800.0, "vcpus": 48.0, "vpc_only": true}, "c5d.18xlarge": {"apiname": "c5d.18xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 281.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 3.9027777777777777, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 144.0, "name": "C5 High-CPU 18xlarge", "network_perf": 20.0, "physical_processor": "Intel Xeon Platinum 8124M", "placement_group_support": false, "storage": 1800.0, "vcpus": 72.0, "vpc_only": true}, "c5d.24xlarge": {"apiname": "c5d.24xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 375.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 3.90625, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 192.0, "name": "C5 High-CPU 24xlarge", "network_perf": 20.0, "physical_processor": "Intel Xeon Platinum 8275L", "placement_group_support": false, "storage": 3600.0, "vcpus": 96.0, "vpc_only": true}, "c5d.2xlarge": {"apiname": "c5d.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 34.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 4.25, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 16.0, "name": "C5 High-CPU Double Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon Platinum 8124M", "placement_group_support": false, "storage": 200.0, "vcpus": 8.0, "vpc_only": true}, "c5d.4xlarge": {"apiname": "c5d.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 68.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 4.25, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 32.0, "name": "C5 High-CPU Quadruple Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon Platinum 8124M", "placement_group_support": false, "storage": 400.0, "vcpus": 16.0, "vpc_only": true}, "c5d.9xlarge": {"apiname": "c5d.9xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 141.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 3.9166666666666665, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 72.0, "name": "C5 High-CPU 9xlarge", "network_perf": 12.0, "physical_processor": "Intel Xeon Platinum 8124M", "placement_group_support": false, "storage": 900.0, "vcpus": 36.0, "vpc_only": true}, "c5d.large": {"apiname": "c5d.large", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 9.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 4.5, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 30, "memory": 4.0, "name": "C5 High-CPU Large", "network_perf": 10.0, "physical_processor": "Intel Xeon Platinum 8124M", "placement_group_support": false, "storage": 50.0, "vcpus": 2.0, "vpc_only": true}, "c5d.metal": {"apiname": "c5d.metal", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 375.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 3.90625, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 192.0, "name": "C5 High-CPU Metal", "network_perf": 20.0, "physical_processor": "Intel Xeon Platinum 8275L", "placement_group_support": false, "storage": 3600.0, "vcpus": 96.0, "vpc_only": true}, "c5d.xlarge": {"apiname": "c5d.xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 17.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 4.25, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 8.0, "name": "C5 High-CPU Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon Platinum 8124M", "placement_group_support": false, "storage": 100.0, "vcpus": 4.0, "vpc_only": true}, "c5n.18xlarge": {"apiname": "c5n.18xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 0.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 750, "memory": 192.0, "name": "C5N 18xlarge", "network_perf": 26.0, "physical_processor": "Intel Xeon Platinum 8124M", "placement_group_support": false, "storage": 0.0, "vcpus": 72.0, "vpc_only": true}, "c5n.2xlarge": {"apiname": "c5n.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 0.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 21.0, "name": "C5N Double Extra Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8124M", "placement_group_support": false, "storage": 0.0, "vcpus": 8.0, "vpc_only": true}, "c5n.4xlarge": {"apiname": "c5n.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 0.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 42.0, "name": "C5N Quadruple Extra Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8124M", "placement_group_support": false, "storage": 0.0, "vcpus": 16.0, "vpc_only": true}, "c5n.9xlarge": {"apiname": "c5n.9xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 0.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 96.0, "name": "C5N 9xlarge", "network_perf": 22.0, "physical_processor": "Intel Xeon Platinum 8124M", "placement_group_support": false, "storage": 0.0, "vcpus": 36.0, "vpc_only": true}, "c5n.large": {"apiname": "c5n.large", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 0.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 30, "memory": 5.25, "name": "C5N Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8124M", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "c5n.metal": {"apiname": "c5n.metal", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 750, "memory": 192.0, "name": "C5N Metal", "network_perf": 26.0, "physical_processor": "Intel Xeon Platinum 8124M", "placement_group_support": false, "storage": 0.0, "vcpus": 72.0, "vpc_only": true}, "c5n.xlarge": {"apiname": "c5n.xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.0 Ghz", "computeunits": 0.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 10.5, "name": "C5N Extra Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8124M", "placement_group_support": false, "storage": 0.0, "vcpus": 4.0, "vpc_only": true}, "cc2.8xlarge": {"apiname": "cc2.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.6 GHz", "computeunits": 88.0, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 2.75, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": false, "linux_virtualization": "HVM", "max_ips": 240, "memory": 60.5, "name": "Cluster Compute Eight Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon E5-2670", "placement_group_support": false, "storage": 3360.0, "vcpus": 32.0, "vpc_only": false}, "cr1.8xlarge": {"apiname": "cr1.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 88.0, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 2.75, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": false, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 244.0, "name": "High Memory Cluster Eight Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon E5-2670", "placement_group_support": false, "storage": 240.0, "vcpus": 32.0, "vpc_only": false}, "d2.2xlarge": {"apiname": "d2.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.4 GHz", "computeunits": 28.0, "ebs_iops": 8000.0, "ebs_max_bandwidth": 1000.0, "ebs_throughput": 125.0, "ecu_per_vcpu": 3.5, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 61.0, "name": "D2 Double Extra Large", "network_perf": 8.0, "physical_processor": "Intel Xeon E5-2676v3 (Haswell)", "placement_group_support": false, "storage": 12000.0, "vcpus": 8.0, "vpc_only": false}, "d2.4xlarge": {"apiname": "d2.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.4 GHz", "computeunits": 56.0, "ebs_iops": 16000.0, "ebs_max_bandwidth": 2000.0, "ebs_throughput": 250.0, "ecu_per_vcpu": 3.5, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 122.0, "name": "D2 Quadruple Extra Large", "network_perf": 8.0, "physical_processor": "Intel Xeon E5-2676v3 (Haswell)", "placement_group_support": false, "storage": 24000.0, "vcpus": 16.0, "vpc_only": false}, "d2.8xlarge": {"apiname": "d2.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.4 GHz", "computeunits": 116.0, "ebs_iops": 32000.0, "ebs_max_bandwidth": 4000.0, "ebs_throughput": 500.0, "ecu_per_vcpu": 3.2222222222222223, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 244.0, "name": "D2 Eight Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon E5-2676v3 (Haswell)", "placement_group_support": false, "storage": 48000.0, "vcpus": 36.0, "vpc_only": false}, "d2.xlarge": {"apiname": "d2.xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.4 GHz", "computeunits": 14.0, "ebs_iops": 6000.0, "ebs_max_bandwidth": 750.0, "ebs_throughput": 93.75, "ecu_per_vcpu": 3.5, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 30.5, "name": "D2 Extra Large", "network_perf": 6.0, "physical_processor": "Intel Xeon E5-2676v3 (Haswell)", "placement_group_support": false, "storage": 6000.0, "vcpus": 4.0, "vpc_only": false}, "f1.16xlarge": {"apiname": "f1.16xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 188.0, "ebs_iops": 75000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 2.9375, "enhanced_networking": false, "fpga": 8, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 400, "memory": 976.0, "name": "F1 16xlarge", "network_perf": 16.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 3760.0, "vcpus": 64.0, "vpc_only": true}, "f1.2xlarge": {"apiname": "f1.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 26.0, "ebs_iops": 12000.0, "ebs_max_bandwidth": 1700.0, "ebs_throughput": 212.5, "ecu_per_vcpu": 3.25, "enhanced_networking": false, "fpga": 1, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 122.0, "name": "F1 Double Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 470.0, "vcpus": 8.0, "vpc_only": true}, "f1.4xlarge": {"apiname": "f1.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 52.0, "ebs_iops": 44000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 400.0, "ecu_per_vcpu": 3.25, "enhanced_networking": false, "fpga": 2, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 244.0, "name": "F1 Quadruple Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 940.0, "vcpus": 16.0, "vpc_only": true}, "g2.2xlarge": {"apiname": "g2.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.6 GHz", "computeunits": 26.0, "ebs_iops": 8000.0, "ebs_max_bandwidth": 1000.0, "ebs_throughput": 125.0, "ecu_per_vcpu": 3.25, "enhanced_networking": false, "fpga": 0, "gpus": 1, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": false, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 15.0, "name": "G2 Double Extra Large", "network_perf": 8.0, "physical_processor": "Intel Xeon E5-2670 (Sandy Bridge)", "placement_group_support": false, "storage": 60.0, "vcpus": 8.0, "vpc_only": false}, "g2.8xlarge": {"apiname": "g2.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.6 GHz", "computeunits": 104.0, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 3.25, "enhanced_networking": false, "fpga": 0, "gpus": 4, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": false, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 60.0, "name": "G2 Eight Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon E5-2670 (Sandy Bridge)", "placement_group_support": false, "storage": 240.0, "vcpus": 32.0, "vpc_only": false}, "g3.16xlarge": {"apiname": "g3.16xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 188.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 2.9375, "enhanced_networking": true, "fpga": 0, "gpus": 4, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 488.0, "name": "G3 16xlarge", "network_perf": 16.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 0.0, "vcpus": 64.0, "vpc_only": true}, "g3.4xlarge": {"apiname": "g3.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 47.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 2.9375, "enhanced_networking": true, "fpga": 0, "gpus": 1, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 122.0, "name": "G3 Quadruple Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 0.0, "vcpus": 16.0, "vpc_only": true}, "g3.8xlarge": {"apiname": "g3.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 94.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 2.9375, "enhanced_networking": true, "fpga": 0, "gpus": 2, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 244.0, "name": "G3 Eight Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 0.0, "vcpus": 32.0, "vpc_only": true}, "g3s.xlarge": {"apiname": "g3s.xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 13.0, "ebs_iops": 5000.0, "ebs_max_bandwidth": 850.0, "ebs_throughput": 100.0, "ecu_per_vcpu": 3.25, "enhanced_networking": false, "fpga": 0, "gpus": 1, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 30.5, "name": "G3S Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 0.0, "vcpus": 4.0, "vpc_only": true}, "g4dn.12xlarge": {"apiname": "g4dn.12xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 4, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 192.0, "name": "G4DN 12xlarge", "network_perf": 22.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 900.0, "vcpus": 48.0, "vpc_only": true}, "g4dn.16xlarge": {"apiname": "g4dn.16xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 1, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 750, "memory": 256.0, "name": "G4DN 16xlarge", "network_perf": 22.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 900.0, "vcpus": 64.0, "vpc_only": true}, "g4dn.2xlarge": {"apiname": "g4dn.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 1, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 30, "memory": 32.0, "name": "G4DN Double Extra Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 225.0, "vcpus": 8.0, "vpc_only": true}, "g4dn.4xlarge": {"apiname": "g4dn.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 1, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 30, "memory": 64.0, "name": "G4DN Quadruple Extra Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 225.0, "vcpus": 16.0, "vpc_only": true}, "g4dn.8xlarge": {"apiname": "g4dn.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 1, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 128.0, "name": "G4DN Eight Extra Large", "network_perf": 22.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 900.0, "vcpus": 32.0, "vpc_only": true}, "g4dn.metal": {"apiname": "g4dn.metal", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 8, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "memory": 384.0, "name": "G4DN Metal", "network_perf": 26.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 0.0, "vcpus": 96.0, "vpc_only": true}, "g4dn.xlarge": {"apiname": "g4dn.xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 10000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 1, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 30, "memory": 16.0, "name": "G4DN Extra Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 125.0, "vcpus": 4.0, "vpc_only": true}, "h1.16xlarge": {"apiname": "h1.16xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 188.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 2.9375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 256.0, "name": "H1 16xlarge", "network_perf": 20.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 16000.0, "vcpus": 64.0, "vpc_only": true}, "h1.2xlarge": {"apiname": "h1.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 26.0, "ebs_iops": 12000.0, "ebs_max_bandwidth": 1750.0, "ebs_throughput": 218.75, "ecu_per_vcpu": 3.25, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 32.0, "name": "H1 Double Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 2000.0, "vcpus": 8.0, "vpc_only": true}, "h1.4xlarge": {"apiname": "h1.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 53.5, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 3.34375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 64.0, "name": "H1 Quadruple Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 4000.0, "vcpus": 16.0, "vpc_only": true}, "h1.8xlarge": {"apiname": "h1.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 99.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 3.09375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 128.0, "name": "H1 Eight Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 8000.0, "vcpus": 32.0, "vpc_only": true}, "hs1.8xlarge": {"apiname": "hs1.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "2 GHz", "computeunits": 35.0, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 2.0588235294117645, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": false, "linux_virtualization": "HVM, PV", "max_ips": 240, "memory": 117.0, "name": "High Storage Eight Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon E5-2650", "placement_group_support": false, "storage": 48000.0, "vcpus": 17.0, "vpc_only": false}, "i2.2xlarge": {"apiname": "i2.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 27.0, "ebs_iops": 8000.0, "ebs_max_bandwidth": 1000.0, "ebs_throughput": 125.0, "ecu_per_vcpu": 3.375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 61.0, "name": "I2 Double Extra Large", "network_perf": 8.0, "physical_processor": "Intel Xeon E5-2670 v2 (Ivy Bridge)", "placement_group_support": false, "storage": 1600.0, "vcpus": 8.0, "vpc_only": false}, "i2.4xlarge": {"apiname": "i2.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 53.0, "ebs_iops": 16000.0, "ebs_max_bandwidth": 2000.0, "ebs_throughput": 250.0, "ecu_per_vcpu": 3.3125, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 122.0, "name": "I2 Quadruple Extra Large", "network_perf": 8.0, "physical_processor": "Intel Xeon E5-2670 v2 (Ivy Bridge)", "placement_group_support": false, "storage": 3200.0, "vcpus": 16.0, "vpc_only": false}, "i2.8xlarge": {"apiname": "i2.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 104.0, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 3.25, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 244.0, "name": "I2 Eight Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon E5-2670 v2 (Ivy Bridge)", "placement_group_support": false, "storage": 6400.0, "vcpus": 32.0, "vpc_only": false}, "i2.xlarge": {"apiname": "i2.xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 14.0, "ebs_iops": 4000.0, "ebs_max_bandwidth": 500.0, "ebs_throughput": 62.5, "ecu_per_vcpu": 3.5, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 30.5, "name": "I2 Extra Large", "network_perf": 6.0, "physical_processor": "Intel Xeon E5-2670 v2 (Ivy Bridge)", "placement_group_support": false, "storage": 800.0, "vcpus": 4.0, "vpc_only": false}, "i3.16xlarge": {"apiname": "i3.16xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 200.0, "ebs_iops": 65000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 3.125, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 488.0, "name": "I3 High I/O 16xlarge", "network_perf": 16.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 15200.0, "vcpus": 64.0, "vpc_only": true}, "i3.2xlarge": {"apiname": "i3.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 27.0, "ebs_iops": 12000.0, "ebs_max_bandwidth": 1700.0, "ebs_throughput": 212.5, "ecu_per_vcpu": 3.375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 61.0, "name": "I3 High I/O Double Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 1900.0, "vcpus": 8.0, "vpc_only": true}, "i3.4xlarge": {"apiname": "i3.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 53.0, "ebs_iops": 16000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 3.3125, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 122.0, "name": "I3 High I/O Quadruple Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 3800.0, "vcpus": 16.0, "vpc_only": true}, "i3.8xlarge": {"apiname": "i3.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 99.0, "ebs_iops": 32500.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 3.09375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 244.0, "name": "I3 High I/O Eight Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 7600.0, "vcpus": 32.0, "vpc_only": true}, "i3.large": {"apiname": "i3.large", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 7.0, "ebs_iops": 3000.0, "ebs_max_bandwidth": 425.0, "ebs_throughput": 53.13, "ecu_per_vcpu": 3.5, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 30, "memory": 15.25, "name": "I3 High I/O Large", "network_perf": 10.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 475.0, "vcpus": 2.0, "vpc_only": true}, "i3.metal": {"apiname": "i3.metal", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 208.0, "ebs_iops": 65000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 2.888888888888889, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 512.0, "name": "I3 High I/O Metal", "network_perf": 20.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 15200.0, "vcpus": 72.0, "vpc_only": true}, "i3.xlarge": {"apiname": "i3.xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 13.0, "ebs_iops": 6000.0, "ebs_max_bandwidth": 850.0, "ebs_throughput": 106.25, "ecu_per_vcpu": 3.25, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 30.5, "name": "I3 High I/O Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 950.0, "vcpus": 4.0, "vpc_only": true}, "i3en.12xlarge": {"apiname": "i3en.12xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.1 GHz", "computeunits": 0.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 384.0, "name": "I3EN 12xlarge", "network_perf": 22.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 30000.0, "vcpus": 48.0, "vpc_only": true}, "i3en.24xlarge": {"apiname": "i3en.24xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.1 GHz", "computeunits": 0.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 750, "memory": 768.0, "name": "I3EN 24xlarge", "network_perf": 26.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 60000.0, "vcpus": 96.0, "vpc_only": true}, "i3en.2xlarge": {"apiname": "i3en.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.1 GHz", "computeunits": 0.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 64.0, "name": "I3EN Double Extra Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 5000.0, "vcpus": 8.0, "vpc_only": true}, "i3en.3xlarge": {"apiname": "i3en.3xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.1 GHz", "computeunits": 0.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 96.0, "name": "I3EN 3xlarge", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 7500.0, "vcpus": 12.0, "vpc_only": true}, "i3en.6xlarge": {"apiname": "i3en.6xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.1 GHz", "computeunits": 0.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 192.0, "name": "I3EN 6xlarge", "network_perf": 20.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 15000.0, "vcpus": 24.0, "vpc_only": true}, "i3en.large": {"apiname": "i3en.large", "architecture": "64-bit", "clock_speed_ghz": "3.1 GHz", "computeunits": 0.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 30, "memory": 16.0, "name": "I3EN Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 1250.0, "vcpus": 2.0, "vpc_only": true}, "i3en.metal": {"apiname": "i3en.metal", "architecture": "64-bit", "clock_speed_ghz": "3.1 GHz", "computeunits": 0.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 750, "memory": 768.0, "name": "I3EN Metal", "network_perf": 26.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 60000.0, "vcpus": 96.0, "vpc_only": true}, "i3en.xlarge": {"apiname": "i3en.xlarge", "architecture": "64-bit", "clock_speed_ghz": "3.1 GHz", "computeunits": 0.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 32.0, "name": "I3EN Extra Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 2500.0, "vcpus": 4.0, "vpc_only": true}, "m1.large": {"apiname": "m1.large", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 4.0, "ebs_iops": 4000.0, "ebs_max_bandwidth": 500.0, "ebs_throughput": 62.5, "ecu_per_vcpu": 2.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": false, "linux_virtualization": "PV", "max_ips": 30, "memory": 7.5, "name": "M1 General Purpose Large", "network_perf": 6.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 840.0, "vcpus": 2.0, "vpc_only": false}, "m1.medium": {"apiname": "m1.medium", "architecture": "32/64-bit", "clock_speed_ghz": "unknown", "computeunits": 2.0, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 2.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": false, "linux_virtualization": "PV", "max_ips": 12, "memory": 3.75, "name": "M1 General Purpose Medium", "network_perf": 6.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 410.0, "vcpus": 1.0, "vpc_only": false}, "m1.small": {"apiname": "m1.small", "architecture": "32/64-bit", "clock_speed_ghz": "unknown", "computeunits": 1.0, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 1.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": false, "linux_virtualization": "PV", "max_ips": 8, "memory": 1.7, "name": "M1 General Purpose Small", "network_perf": 2.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 160.0, "vcpus": 1.0, "vpc_only": false}, "m1.xlarge": {"apiname": "m1.xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 8.0, "ebs_iops": 8000.0, "ebs_max_bandwidth": 1000.0, "ebs_throughput": 125.0, "ecu_per_vcpu": 2.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": false, "linux_virtualization": "PV", "max_ips": 60, "memory": 15.0, "name": "M1 General Purpose Extra Large", "network_perf": 8.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 1680.0, "vcpus": 4.0, "vpc_only": false}, "m2.2xlarge": {"apiname": "m2.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 13.0, "ebs_iops": 4000.0, "ebs_max_bandwidth": 500.0, "ebs_throughput": 62.5, "ecu_per_vcpu": 3.25, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": false, "linux_virtualization": "PV", "max_ips": 120, "memory": 34.2, "name": "M2 High Memory Double Extra Large", "network_perf": 6.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 850.0, "vcpus": 4.0, "vpc_only": false}, "m2.4xlarge": {"apiname": "m2.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 26.0, "ebs_iops": 8000.0, "ebs_max_bandwidth": 1000.0, "ebs_throughput": 125.0, "ecu_per_vcpu": 3.25, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": false, "linux_virtualization": "PV", "max_ips": 240, "memory": 68.4, "name": "M2 High Memory Quadruple Extra Large", "network_perf": 8.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 1680.0, "vcpus": 8.0, "vpc_only": false}, "m2.xlarge": {"apiname": "m2.xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 6.5, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 3.25, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": false, "linux_virtualization": "PV", "max_ips": 60, "memory": 17.1, "name": "M2 High Memory Extra Large", "network_perf": 6.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 420.0, "vcpus": 2.0, "vpc_only": false}, "m3.2xlarge": {"apiname": "m3.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 26.0, "ebs_iops": 8000.0, "ebs_max_bandwidth": 1000.0, "ebs_throughput": 125.0, "ecu_per_vcpu": 3.25, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": false, "linux_virtualization": "HVM, PV", "max_ips": 120, "memory": 30.0, "name": "M3 General Purpose Double Extra Large", "network_perf": 8.0, "physical_processor": "Intel Xeon E5-2670 v2 (Ivy Bridge/Sandy Bridge)", "placement_group_support": false, "storage": 160.0, "vcpus": 8.0, "vpc_only": false}, "m3.large": {"apiname": "m3.large", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 6.5, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 3.25, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": false, "linux_virtualization": "HVM, PV", "max_ips": 30, "memory": 7.5, "name": "M3 General Purpose Large", "network_perf": 6.0, "physical_processor": "Intel Xeon E5-2670 v2 (Ivy Bridge/Sandy Bridge)", "placement_group_support": false, "storage": 32.0, "vcpus": 2.0, "vpc_only": false}, "m3.medium": {"apiname": "m3.medium", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 3.0, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 3.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": false, "linux_virtualization": "HVM, PV", "max_ips": 12, "memory": 3.75, "name": "M3 General Purpose Medium", "network_perf": 6.0, "physical_processor": "Intel Xeon E5-2670 v2 (Ivy Bridge/Sandy Bridge)", "placement_group_support": false, "storage": 4.0, "vcpus": 1.0, "vpc_only": false}, "m3.xlarge": {"apiname": "m3.xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 13.0, "ebs_iops": 4000.0, "ebs_max_bandwidth": 500.0, "ebs_throughput": 62.5, "ecu_per_vcpu": 3.25, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": false, "linux_virtualization": "HVM, PV", "max_ips": 60, "memory": 15.0, "name": "M3 General Purpose Extra Large", "network_perf": 8.0, "physical_processor": "Intel Xeon E5-2670 v2 (Ivy Bridge/Sandy Bridge)", "placement_group_support": false, "storage": 80.0, "vcpus": 4.0, "vpc_only": false}, "m4.10xlarge": {"apiname": "m4.10xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.4 GHz", "computeunits": 124.5, "ebs_iops": 32000.0, "ebs_max_bandwidth": 4000.0, "ebs_throughput": 500.0, "ecu_per_vcpu": 3.1125, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 160.0, "name": "M4 General Purpose Deca Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon E5-2676 v3 (Haswell)", "placement_group_support": false, "storage": 0.0, "vcpus": 40.0, "vpc_only": true}, "m4.16xlarge": {"apiname": "m4.16xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 188.0, "ebs_iops": 65000.0, "ebs_max_bandwidth": 10000.0, "ebs_throughput": 1250.0, "ecu_per_vcpu": 2.9375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 256.0, "name": "M4 General Purpose 16xlarge", "network_perf": 16.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 0.0, "vcpus": 64.0, "vpc_only": true}, "m4.2xlarge": {"apiname": "m4.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.4 GHz", "computeunits": 26.0, "ebs_iops": 8000.0, "ebs_max_bandwidth": 1000.0, "ebs_throughput": 125.0, "ecu_per_vcpu": 3.25, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 32.0, "name": "M4 General Purpose Double Extra Large", "network_perf": 8.0, "physical_processor": "Intel Xeon E5-2676 v3 (Haswell)", "placement_group_support": false, "storage": 0.0, "vcpus": 8.0, "vpc_only": true}, "m4.4xlarge": {"apiname": "m4.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.4 GHz", "computeunits": 53.5, "ebs_iops": 16000.0, "ebs_max_bandwidth": 2000.0, "ebs_throughput": 250.0, "ecu_per_vcpu": 3.34375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 64.0, "name": "M4 General Purpose Quadruple Extra Large", "network_perf": 8.0, "physical_processor": "Intel Xeon E5-2676 v3 (Haswell)", "placement_group_support": false, "storage": 0.0, "vcpus": 16.0, "vpc_only": true}, "m4.large": {"apiname": "m4.large", "architecture": "64-bit", "clock_speed_ghz": "2.4 GHz", "computeunits": 6.5, "ebs_iops": 3600.0, "ebs_max_bandwidth": 450.0, "ebs_throughput": 56.25, "ecu_per_vcpu": 3.25, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 20, "memory": 8.0, "name": "M4 General Purpose Large", "network_perf": 6.0, "physical_processor": "Intel Xeon E5-2676 v3 (Haswell)", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "m4.xlarge": {"apiname": "m4.xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.4 GHz", "computeunits": 13.0, "ebs_iops": 6000.0, "ebs_max_bandwidth": 750.0, "ebs_throughput": 93.75, "ecu_per_vcpu": 3.25, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 16.0, "name": "M4 General Purpose Extra Large", "network_perf": 8.0, "physical_processor": "Intel Xeon E5-2676 v3 (Haswell)", "placement_group_support": false, "storage": 0.0, "vcpus": 4.0, "vpc_only": true}, "m5.12xlarge": {"apiname": "m5.12xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 173.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 3.6041666666666665, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 192.0, "name": "M5 General Purpose 12xlarge", "network_perf": 12.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 0.0, "vcpus": 48.0, "vpc_only": true}, "m5.16xlarge": {"apiname": "m5.16xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 262.0, "ebs_iops": 60000.0, "ebs_max_bandwidth": 10000.0, "ebs_throughput": 1250.0, "ecu_per_vcpu": 4.09375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 256.0, "name": "M5 General Purpose 16xlarge", "network_perf": 16.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 0.0, "vcpus": 64.0, "vpc_only": true}, "m5.24xlarge": {"apiname": "m5.24xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 345.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 3.59375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 384.0, "name": "M5 General Purpose 24xlarge", "network_perf": 20.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 0.0, "vcpus": 96.0, "vpc_only": true}, "m5.2xlarge": {"apiname": "m5.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 31.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 3.875, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 32.0, "name": "M5 General Purpose Double Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 0.0, "vcpus": 8.0, "vpc_only": true}, "m5.4xlarge": {"apiname": "m5.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 60.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 3.75, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 64.0, "name": "M5 General Purpose Quadruple Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 0.0, "vcpus": 16.0, "vpc_only": true}, "m5.8xlarge": {"apiname": "m5.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 131.0, "ebs_iops": 30000.0, "ebs_max_bandwidth": 5000.0, "ebs_throughput": 625.0, "ecu_per_vcpu": 4.09375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 128.0, "name": "M5 General Purpose Eight Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 0.0, "vcpus": 32.0, "vpc_only": true}, "m5.large": {"apiname": "m5.large", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 8.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 4.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 30, "memory": 8.0, "name": "M5 General Purpose Large", "network_perf": 10.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "m5.metal": {"apiname": "m5.metal", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 345.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 3.59375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 384.0, "name": "M5 General Purpose Metal", "network_perf": 20.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 0.0, "vcpus": 96.0, "vpc_only": true}, "m5.xlarge": {"apiname": "m5.xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 16.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 4.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 16.0, "name": "M5 General Purpose Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 0.0, "vcpus": 4.0, "vpc_only": true}, "m5a.12xlarge": {"apiname": "m5a.12xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 30000.0, "ebs_max_bandwidth": 5000.0, "ebs_throughput": 625.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 192.0, "name": "M5A 12xlarge", "network_perf": 12.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 48.0, "vpc_only": true}, "m5a.16xlarge": {"apiname": "m5a.16xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 750, "memory": 256.0, "name": "M5A 16xlarge", "network_perf": 14.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 64.0, "vpc_only": true}, "m5a.24xlarge": {"apiname": "m5a.24xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 60000.0, "ebs_max_bandwidth": 10000.0, "ebs_throughput": 1250.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 750, "memory": 384.0, "name": "M5A 24xlarge", "network_perf": 16.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 96.0, "vpc_only": true}, "m5a.2xlarge": {"apiname": "m5a.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 16000.0, "ebs_max_bandwidth": 2120.0, "ebs_throughput": 265.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 32.0, "name": "M5A Double Extra Large", "network_perf": 10.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 8.0, "vpc_only": true}, "m5a.4xlarge": {"apiname": "m5a.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 16000.0, "ebs_max_bandwidth": 2120.0, "ebs_throughput": 265.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 64.0, "name": "M5A Quadruple Extra Large", "network_perf": 10.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 16.0, "vpc_only": true}, "m5a.8xlarge": {"apiname": "m5a.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 128.0, "name": "M5A Eight Extra Large", "network_perf": 10.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 32.0, "vpc_only": true}, "m5a.large": {"apiname": "m5a.large", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 16000.0, "ebs_max_bandwidth": 2120.0, "ebs_throughput": 265.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 30, "memory": 8.0, "name": "M5A Large", "network_perf": 10.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "m5a.xlarge": {"apiname": "m5a.xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 16000.0, "ebs_max_bandwidth": 2120.0, "ebs_throughput": 265.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 16.0, "name": "M5A Extra Large", "network_perf": 10.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 4.0, "vpc_only": true}, "m5ad.12xlarge": {"apiname": "m5ad.12xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 30000.0, "ebs_max_bandwidth": 5000.0, "ebs_throughput": 675.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 192.0, "name": "M5AD 12xlarge", "network_perf": 12.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 1800.0, "vcpus": 48.0, "vpc_only": true}, "m5ad.24xlarge": {"apiname": "m5ad.24xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 60000.0, "ebs_max_bandwidth": 10000.0, "ebs_throughput": 1250.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 750, "memory": 384.0, "name": "M5AD 24xlarge", "network_perf": 16.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 3600.0, "vcpus": 96.0, "vpc_only": true}, "m5ad.2xlarge": {"apiname": "m5ad.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 16000.0, "ebs_max_bandwidth": 2120.0, "ebs_throughput": 265.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 32.0, "name": "M5AD Double Extra Large", "network_perf": 10.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 300.0, "vcpus": 8.0, "vpc_only": true}, "m5ad.4xlarge": {"apiname": "m5ad.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 16000.0, "ebs_max_bandwidth": 2120.0, "ebs_throughput": 265.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 64.0, "name": "M5AD Quadruple Extra Large", "network_perf": 10.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 600.0, "vcpus": 16.0, "vpc_only": true}, "m5ad.large": {"apiname": "m5ad.large", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 16000.0, "ebs_max_bandwidth": 2120.0, "ebs_throughput": 265.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 30, "memory": 8.0, "name": "M5AD Large", "network_perf": 10.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 75.0, "vcpus": 2.0, "vpc_only": true}, "m5ad.xlarge": {"apiname": "m5ad.xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 16000.0, "ebs_max_bandwidth": 2120.0, "ebs_throughput": 265.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 16.0, "name": "M5AD Extra Large", "network_perf": 10.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 150.0, "vcpus": 4.0, "vpc_only": true}, "m5d.12xlarge": {"apiname": "m5d.12xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 173.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 3.6041666666666665, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 192.0, "name": "M5 General Purpose 12xlarge", "network_perf": 12.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 1800.0, "vcpus": 48.0, "vpc_only": true}, "m5d.16xlarge": {"apiname": "m5d.16xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 262.0, "ebs_iops": 60000.0, "ebs_max_bandwidth": 10000.0, "ebs_throughput": 1250.0, "ecu_per_vcpu": 4.09375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 256.0, "name": "M5 General Purpose 16xlarge", "network_perf": 16.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 2400.0, "vcpus": 64.0, "vpc_only": true}, "m5d.24xlarge": {"apiname": "m5d.24xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 345.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 3.59375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 384.0, "name": "M5 General Purpose 24xlarge", "network_perf": 20.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 3600.0, "vcpus": 96.0, "vpc_only": true}, "m5d.2xlarge": {"apiname": "m5d.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 31.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 3.875, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 32.0, "name": "M5 General Purpose Double Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 300.0, "vcpus": 8.0, "vpc_only": true}, "m5d.4xlarge": {"apiname": "m5d.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 60.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 3.75, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 64.0, "name": "M5 General Purpose Quadruple Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 600.0, "vcpus": 16.0, "vpc_only": true}, "m5d.8xlarge": {"apiname": "m5d.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 131.0, "ebs_iops": 30000.0, "ebs_max_bandwidth": 5000.0, "ebs_throughput": 625.0, "ecu_per_vcpu": 4.09375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 128.0, "name": "M5 General Purpose Eight Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 1200.0, "vcpus": 32.0, "vpc_only": true}, "m5d.large": {"apiname": "m5d.large", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 8.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 4.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 30, "memory": 8.0, "name": "M5 General Purpose Large", "network_perf": 10.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 75.0, "vcpus": 2.0, "vpc_only": true}, "m5d.metal": {"apiname": "m5d.metal", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 345.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 3.59375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 384.0, "name": "M5 General Purpose Metal", "network_perf": 20.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 3600.0, "vcpus": 96.0, "vpc_only": true}, "m5d.xlarge": {"apiname": "m5d.xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 16.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 4.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 16.0, "name": "M5 General Purpose Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 150.0, "vcpus": 4.0, "vpc_only": true}, "m5dn.12xlarge": {"apiname": "m5dn.12xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 192.0, "name": "M5DN 12xlarge", "network_perf": 22.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 1800.0, "vcpus": 48.0, "vpc_only": true}, "m5dn.16xlarge": {"apiname": "m5dn.16xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 60000.0, "ebs_max_bandwidth": 10000.0, "ebs_throughput": 1250.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 750, "memory": 256.0, "name": "M5DN 16xlarge", "network_perf": 24.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 2400.0, "vcpus": 64.0, "vpc_only": true}, "m5dn.24xlarge": {"apiname": "m5dn.24xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 750, "memory": 384.0, "name": "M5DN 24xlarge", "network_perf": 26.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 3600.0, "vcpus": 96.0, "vpc_only": true}, "m5dn.2xlarge": {"apiname": "m5dn.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 32.0, "name": "M5DN Double Extra Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 300.0, "vcpus": 8.0, "vpc_only": true}, "m5dn.4xlarge": {"apiname": "m5dn.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 64.0, "name": "M5DN Quadruple Extra Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 600.0, "vcpus": 16.0, "vpc_only": true}, "m5dn.8xlarge": {"apiname": "m5dn.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 30000.0, "ebs_max_bandwidth": 5000.0, "ebs_throughput": 625.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 128.0, "name": "M5DN Eight Extra Large", "network_perf": 20.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 1200.0, "vcpus": 32.0, "vpc_only": true}, "m5dn.large": {"apiname": "m5dn.large", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 30, "memory": 8.0, "name": "M5DN Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 75.0, "vcpus": 2.0, "vpc_only": true}, "m5dn.metal": {"apiname": "m5dn.metal", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "memory": 384.0, "name": "M5DN Metal", "network_perf": 26.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 0.0, "vcpus": 96.0, "vpc_only": true}, "m5dn.xlarge": {"apiname": "m5dn.xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 16.0, "name": "M5DN Extra Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 150.0, "vcpus": 4.0, "vpc_only": true}, "m5n.12xlarge": {"apiname": "m5n.12xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 192.0, "name": "M5N 12xlarge", "network_perf": 22.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 0.0, "vcpus": 48.0, "vpc_only": true}, "m5n.16xlarge": {"apiname": "m5n.16xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 60000.0, "ebs_max_bandwidth": 10000.0, "ebs_throughput": 1250.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 750, "memory": 256.0, "name": "M5N 16xlarge", "network_perf": 24.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 0.0, "vcpus": 64.0, "vpc_only": true}, "m5n.24xlarge": {"apiname": "m5n.24xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 750, "memory": 384.0, "name": "M5N 24xlarge", "network_perf": 26.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 0.0, "vcpus": 96.0, "vpc_only": true}, "m5n.2xlarge": {"apiname": "m5n.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 32.0, "name": "M5N Double Extra Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 0.0, "vcpus": 8.0, "vpc_only": true}, "m5n.4xlarge": {"apiname": "m5n.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 64.0, "name": "M5N Quadruple Extra Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 0.0, "vcpus": 16.0, "vpc_only": true}, "m5n.8xlarge": {"apiname": "m5n.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 30000.0, "ebs_max_bandwidth": 5000.0, "ebs_throughput": 625.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 128.0, "name": "M5N Eight Extra Large", "network_perf": 20.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 0.0, "vcpus": 32.0, "vpc_only": true}, "m5n.large": {"apiname": "m5n.large", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 30, "memory": 8.0, "name": "M5N Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "m5n.metal": {"apiname": "m5n.metal", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "memory": 384.0, "name": "M5N Metal", "network_perf": 26.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 0.0, "vcpus": 96.0, "vpc_only": true}, "m5n.xlarge": {"apiname": "m5n.xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 16.0, "name": "M5N Extra Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 0.0, "vcpus": 4.0, "vpc_only": true}, "p2.16xlarge": {"apiname": "p2.16xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 188.0, "ebs_iops": 65000.0, "ebs_max_bandwidth": 10000.0, "ebs_throughput": 1250.0, "ecu_per_vcpu": 2.9375, "enhanced_networking": true, "fpga": 0, "gpus": 16, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 768.0, "name": "General Purpose GPU 16xlarge", "network_perf": 16.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 0.0, "vcpus": 64.0, "vpc_only": true}, "p2.8xlarge": {"apiname": "p2.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 94.0, "ebs_iops": 32500.0, "ebs_max_bandwidth": 5000.0, "ebs_throughput": 625.0, "ecu_per_vcpu": 2.9375, "enhanced_networking": true, "fpga": 0, "gpus": 8, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 488.0, "name": "General Purpose GPU Eight Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 0.0, "vcpus": 32.0, "vpc_only": true}, "p2.xlarge": {"apiname": "p2.xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 12.0, "ebs_iops": 6000.0, "ebs_max_bandwidth": 750.0, "ebs_throughput": 93.75, "ecu_per_vcpu": 3.0, "enhanced_networking": true, "fpga": 0, "gpus": 1, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 61.0, "name": "General Purpose GPU Extra Large", "network_perf": 8.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 0.0, "vcpus": 4.0, "vpc_only": true}, "p3.16xlarge": {"apiname": "p3.16xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 188.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 2.9375, "enhanced_networking": true, "fpga": 0, "gpus": 8, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 488.0, "name": "P3 16xlarge", "network_perf": 20.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 0.0, "vcpus": 64.0, "vpc_only": true}, "p3.2xlarge": {"apiname": "p3.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 26.0, "ebs_iops": 10000.0, "ebs_max_bandwidth": 1750.0, "ebs_throughput": 218.0, "ecu_per_vcpu": 3.25, "enhanced_networking": true, "fpga": 0, "gpus": 1, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 61.0, "name": "P3 Double Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 0.0, "vcpus": 8.0, "vpc_only": true}, "p3.8xlarge": {"apiname": "p3.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 94.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 2.9375, "enhanced_networking": true, "fpga": 0, "gpus": 4, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 244.0, "name": "P3 Eight Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 0.0, "vcpus": 32.0, "vpc_only": true}, "p3dn.24xlarge": {"apiname": "p3dn.24xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 345.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 3.59375, "enhanced_networking": true, "fpga": 0, "gpus": 8, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 750, "memory": 768.0, "name": "P3DN 24xlarge", "network_perf": 26.0, "physical_processor": "Intel Xeon Platinum 8175 (Skylake)", "placement_group_support": false, "storage": 1800.0, "vcpus": 96.0, "vpc_only": true}, "r3.2xlarge": {"apiname": "r3.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 26.0, "ebs_iops": 8000.0, "ebs_max_bandwidth": 1000.0, "ebs_throughput": 125.0, "ecu_per_vcpu": 3.25, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 61.0, "name": "R3 High-Memory Double Extra Large", "network_perf": 8.0, "physical_processor": "Intel Xeon E5-2670 v2 (Ivy Bridge)", "placement_group_support": false, "storage": 160.0, "vcpus": 8.0, "vpc_only": false}, "r3.4xlarge": {"apiname": "r3.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 52.0, "ebs_iops": 16000.0, "ebs_max_bandwidth": 2000.0, "ebs_throughput": 250.0, "ecu_per_vcpu": 3.25, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 122.0, "name": "R3 High-Memory Quadruple Extra Large", "network_perf": 8.0, "physical_processor": "Intel Xeon E5-2670 v2 (Ivy Bridge)", "placement_group_support": false, "storage": 320.0, "vcpus": 16.0, "vpc_only": false}, "r3.8xlarge": {"apiname": "r3.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 104.0, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 3.25, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 244.0, "name": "R3 High-Memory Eight Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon E5-2670 v2 (Ivy Bridge)", "placement_group_support": false, "storage": 640.0, "vcpus": 32.0, "vpc_only": false}, "r3.large": {"apiname": "r3.large", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 6.5, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 3.25, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 30, "memory": 15.25, "name": "R3 High-Memory Large", "network_perf": 6.0, "physical_processor": "Intel Xeon E5-2670 v2 (Ivy Bridge)", "placement_group_support": false, "storage": 32.0, "vcpus": 2.0, "vpc_only": false}, "r3.xlarge": {"apiname": "r3.xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 13.0, "ebs_iops": 4000.0, "ebs_max_bandwidth": 500.0, "ebs_throughput": 62.5, "ecu_per_vcpu": 3.25, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 30.5, "name": "R3 High-Memory Extra Large", "network_perf": 6.0, "physical_processor": "Intel Xeon E5-2670 v2 (Ivy Bridge)", "placement_group_support": false, "storage": 80.0, "vcpus": 4.0, "vpc_only": false}, "r4.16xlarge": {"apiname": "r4.16xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 195.0, "ebs_iops": 75000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 3.046875, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 488.0, "name": "R4 High-Memory 16xlarge", "network_perf": 16.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 0.0, "vcpus": 64.0, "vpc_only": true}, "r4.2xlarge": {"apiname": "r4.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 27.0, "ebs_iops": 12000.0, "ebs_max_bandwidth": 1700.0, "ebs_throughput": 212.5, "ecu_per_vcpu": 3.375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 61.0, "name": "R4 High-Memory Double Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 0.0, "vcpus": 8.0, "vpc_only": true}, "r4.4xlarge": {"apiname": "r4.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 53.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 3.3125, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 122.0, "name": "R4 High-Memory Quadruple Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 0.0, "vcpus": 16.0, "vpc_only": true}, "r4.8xlarge": {"apiname": "r4.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 99.0, "ebs_iops": 37500.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 3.09375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 244.0, "name": "R4 High-Memory Eight Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 0.0, "vcpus": 32.0, "vpc_only": true}, "r4.large": {"apiname": "r4.large", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 7.0, "ebs_iops": 3000.0, "ebs_max_bandwidth": 425.0, "ebs_throughput": 53.13, "ecu_per_vcpu": 3.5, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 30, "memory": 15.25, "name": "R4 High-Memory Large", "network_perf": 10.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "r4.xlarge": {"apiname": "r4.xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 13.5, "ebs_iops": 6000.0, "ebs_max_bandwidth": 850.0, "ebs_throughput": 106.25, "ecu_per_vcpu": 3.375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 30.5, "name": "R4 High-Memory Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon E5-2686 v4 (Broadwell)", "placement_group_support": false, "storage": 0.0, "vcpus": 4.0, "vpc_only": true}, "r5.12xlarge": {"apiname": "r5.12xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 173.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 3.6041666666666665, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 384.0, "name": "R5 12xlarge", "network_perf": 12.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 0.0, "vcpus": 48.0, "vpc_only": true}, "r5.16xlarge": {"apiname": "r5.16xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 262.0, "ebs_iops": 60000.0, "ebs_max_bandwidth": 10000.0, "ebs_throughput": 1250.0, "ecu_per_vcpu": 4.09375, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 512.0, "name": "R5 16xlarge", "network_perf": 16.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 0.0, "vcpus": 64.0, "vpc_only": true}, "r5.24xlarge": {"apiname": "r5.24xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 347.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 3.6145833333333335, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 768.0, "name": "R5 24xlarge", "network_perf": 20.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 0.0, "vcpus": 96.0, "vpc_only": true}, "r5.2xlarge": {"apiname": "r5.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 38.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 4.75, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 64.0, "name": "R5 Double Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 0.0, "vcpus": 8.0, "vpc_only": true}, "r5.4xlarge": {"apiname": "r5.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 71.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 4.4375, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 128.0, "name": "R5 Quadruple Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 0.0, "vcpus": 16.0, "vpc_only": true}, "r5.8xlarge": {"apiname": "r5.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 131.0, "ebs_iops": 30000.0, "ebs_max_bandwidth": 5000.0, "ebs_throughput": 625.0, "ecu_per_vcpu": 4.09375, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 256.0, "name": "R5 Eight Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 0.0, "vcpus": 32.0, "vpc_only": true}, "r5.large": {"apiname": "r5.large", "architecture": "64-bit", "clock_speed_ghz": "3.1 GHz", "computeunits": 9.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 4.5, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 30, "memory": 16.0, "name": "R5 Large", "network_perf": 12.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "r5.metal": {"apiname": "r5.metal", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 347.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 3.6145833333333335, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 768.0, "name": "R5 Metal", "network_perf": 20.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 0.0, "vcpus": 96.0, "vpc_only": true}, "r5.xlarge": {"apiname": "r5.xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 19.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 4.75, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 32.0, "name": "R5 Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 0.0, "vcpus": 4.0, "vpc_only": true}, "r5a.12xlarge": {"apiname": "r5a.12xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 30000.0, "ebs_max_bandwidth": 5000.0, "ebs_throughput": 625.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 384.0, "name": "R5A 12xlarge", "network_perf": 12.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 48.0, "vpc_only": true}, "r5a.16xlarge": {"apiname": "r5a.16xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 750, "memory": 512.0, "name": "R5A 16xlarge", "network_perf": 14.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 64.0, "vpc_only": true}, "r5a.24xlarge": {"apiname": "r5a.24xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 60000.0, "ebs_max_bandwidth": 10000.0, "ebs_throughput": 1250.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 750, "memory": 768.0, "name": "R5A 24xlarge", "network_perf": 16.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 96.0, "vpc_only": true}, "r5a.2xlarge": {"apiname": "r5a.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 16000.0, "ebs_max_bandwidth": 2120.0, "ebs_throughput": 265.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 64.0, "name": "R5A Double Extra Large", "network_perf": 12.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 8.0, "vpc_only": true}, "r5a.4xlarge": {"apiname": "r5a.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 16000.0, "ebs_max_bandwidth": 2120.0, "ebs_throughput": 265.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 128.0, "name": "R5A Quadruple Extra Large", "network_perf": 12.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 16.0, "vpc_only": true}, "r5a.8xlarge": {"apiname": "r5a.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 32000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 256.0, "name": "R5A Eight Extra Large", "network_perf": 10.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 32.0, "vpc_only": true}, "r5a.large": {"apiname": "r5a.large", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 16000.0, "ebs_max_bandwidth": 2120.0, "ebs_throughput": 265.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 30, "memory": 16.0, "name": "R5A Large", "network_perf": 12.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "r5a.xlarge": {"apiname": "r5a.xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 16000.0, "ebs_max_bandwidth": 2120.0, "ebs_throughput": 265.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 32.0, "name": "R5A Extra Large", "network_perf": 12.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 4.0, "vpc_only": true}, "r5ad.12xlarge": {"apiname": "r5ad.12xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 30000.0, "ebs_max_bandwidth": 5000.0, "ebs_throughput": 625.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 384.0, "name": "R5AD 12xlarge", "network_perf": 12.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 1800.0, "vcpus": 48.0, "vpc_only": true}, "r5ad.24xlarge": {"apiname": "r5ad.24xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 60000.0, "ebs_max_bandwidth": 10000.0, "ebs_throughput": 1250.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 750, "memory": 768.0, "name": "R5AD 24xlarge", "network_perf": 16.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 3600.0, "vcpus": 96.0, "vpc_only": true}, "r5ad.2xlarge": {"apiname": "r5ad.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 16000.0, "ebs_max_bandwidth": 2210.0, "ebs_throughput": 265.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 64.0, "name": "R5AD Double Extra Large", "network_perf": 12.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 300.0, "vcpus": 8.0, "vpc_only": true}, "r5ad.4xlarge": {"apiname": "r5ad.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 16000.0, "ebs_max_bandwidth": 2210.0, "ebs_throughput": 265.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 128.0, "name": "R5AD Quadruple Extra Large", "network_perf": 12.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 600.0, "vcpus": 16.0, "vpc_only": true}, "r5ad.large": {"apiname": "r5ad.large", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 16000.0, "ebs_max_bandwidth": 2210.0, "ebs_throughput": 265.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 30, "memory": 16.0, "name": "R5AD Large", "network_perf": 12.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 75.0, "vcpus": 2.0, "vpc_only": true}, "r5ad.xlarge": {"apiname": "r5ad.xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.0, "ebs_iops": 16000.0, "ebs_max_bandwidth": 2210.0, "ebs_throughput": 265.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 32.0, "name": "R5AD Extra Large", "network_perf": 12.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 150.0, "vcpus": 4.0, "vpc_only": true}, "r5d.12xlarge": {"apiname": "r5d.12xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 173.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 3.6041666666666665, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 384.0, "name": "R5D 12xlarge", "network_perf": 12.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 1800.0, "vcpus": 48.0, "vpc_only": true}, "r5d.16xlarge": {"apiname": "r5d.16xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 262.0, "ebs_iops": 60000.0, "ebs_max_bandwidth": 10000.0, "ebs_throughput": 1250.0, "ecu_per_vcpu": 4.09375, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 512.0, "name": "R5D 16xlarge", "network_perf": 16.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 2400.0, "vcpus": 64.0, "vpc_only": true}, "r5d.24xlarge": {"apiname": "r5d.24xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 347.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 3.6145833333333335, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 768.0, "name": "R5D 24xlarge", "network_perf": 20.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 3600.0, "vcpus": 96.0, "vpc_only": true}, "r5d.2xlarge": {"apiname": "r5d.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 38.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 4.75, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 64.0, "name": "R5D Double Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 300.0, "vcpus": 8.0, "vpc_only": true}, "r5d.4xlarge": {"apiname": "r5d.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 71.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 4.4375, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 128.0, "name": "R5D Quadruple Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 600.0, "vcpus": 16.0, "vpc_only": true}, "r5d.8xlarge": {"apiname": "r5d.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 131.0, "ebs_iops": 30000.0, "ebs_max_bandwidth": 5000.0, "ebs_throughput": 625.0, "ecu_per_vcpu": 4.09375, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 256.0, "name": "R5D Eight Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 1200.0, "vcpus": 32.0, "vpc_only": true}, "r5d.large": {"apiname": "r5d.large", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 10.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 5.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 30, "memory": 16.0, "name": "R5D Large", "network_perf": 12.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 75.0, "vcpus": 2.0, "vpc_only": true}, "r5d.metal": {"apiname": "r5d.metal", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 347.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 3.6145833333333335, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 768.0, "name": "R5D Metal", "network_perf": 20.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 3600.0, "vcpus": 96.0, "vpc_only": true}, "r5d.xlarge": {"apiname": "r5d.xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 19.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 4.75, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 32.0, "name": "R5D Extra Large", "network_perf": 12.0, "physical_processor": "Intel Xeon Platinum 8175", "placement_group_support": false, "storage": 150.0, "vcpus": 4.0, "vpc_only": true}, "r5dn.12xlarge": {"apiname": "r5dn.12xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 384.0, "name": "R5DN 12xlarge", "network_perf": 22.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 1800.0, "vcpus": 48.0, "vpc_only": true}, "r5dn.16xlarge": {"apiname": "r5dn.16xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 60000.0, "ebs_max_bandwidth": 10000.0, "ebs_throughput": 1250.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 750, "memory": 512.0, "name": "R5DN 16xlarge", "network_perf": 24.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 2400.0, "vcpus": 64.0, "vpc_only": true}, "r5dn.24xlarge": {"apiname": "r5dn.24xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 750, "memory": 768.0, "name": "R5DN 24xlarge", "network_perf": 26.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 3600.0, "vcpus": 96.0, "vpc_only": true}, "r5dn.2xlarge": {"apiname": "r5dn.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 64.0, "name": "R5DN Double Extra Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 300.0, "vcpus": 8.0, "vpc_only": true}, "r5dn.4xlarge": {"apiname": "r5dn.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 128.0, "name": "R5DN Quadruple Extra Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 600.0, "vcpus": 16.0, "vpc_only": true}, "r5dn.8xlarge": {"apiname": "r5dn.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 30000.0, "ebs_max_bandwidth": 5000.0, "ebs_throughput": 625.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 256.0, "name": "R5DN Eight Extra Large", "network_perf": 20.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 1200.0, "vcpus": 32.0, "vpc_only": true}, "r5dn.large": {"apiname": "r5dn.large", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 30, "memory": 16.0, "name": "R5DN Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 75.0, "vcpus": 2.0, "vpc_only": true}, "r5dn.metal": {"apiname": "r5dn.metal", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "memory": 768.0, "name": "R5DN Metal", "network_perf": 26.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 0.0, "vcpus": 96.0, "vpc_only": true}, "r5dn.xlarge": {"apiname": "r5dn.xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 32.0, "name": "R5DN Extra Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 150.0, "vcpus": 4.0, "vpc_only": true}, "r5n.12xlarge": {"apiname": "r5n.12xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 384.0, "name": "R5N 12xlarge", "network_perf": 22.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 0.0, "vcpus": 48.0, "vpc_only": true}, "r5n.16xlarge": {"apiname": "r5n.16xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 60000.0, "ebs_max_bandwidth": 10000.0, "ebs_throughput": 1250.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 750, "memory": 512.0, "name": "R5N 16xlarge", "network_perf": 24.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 0.0, "vcpus": 64.0, "vpc_only": true}, "r5n.24xlarge": {"apiname": "r5n.24xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 750, "memory": 768.0, "name": "R5N 24xlarge", "network_perf": 26.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 0.0, "vcpus": 96.0, "vpc_only": true}, "r5n.2xlarge": {"apiname": "r5n.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 64.0, "name": "R5N Double Extra Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 0.0, "vcpus": 8.0, "vpc_only": true}, "r5n.4xlarge": {"apiname": "r5n.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 128.0, "name": "R5N Quadruple Extra Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 0.0, "vcpus": 16.0, "vpc_only": true}, "r5n.8xlarge": {"apiname": "r5n.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 30000.0, "ebs_max_bandwidth": 5000.0, "ebs_throughput": 625.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 240, "memory": 256.0, "name": "R5N Eight Extra Large", "network_perf": 20.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 0.0, "vcpus": 32.0, "vpc_only": true}, "r5n.large": {"apiname": "r5n.large", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 30, "memory": 16.0, "name": "R5N Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "r5n.metal": {"apiname": "r5n.metal", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "memory": 768.0, "name": "R5N Metal", "network_perf": 26.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 0.0, "vcpus": 96.0, "vpc_only": true}, "r5n.xlarge": {"apiname": "r5n.xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 18750.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 32.0, "name": "R5N Extra Large", "network_perf": 18.0, "physical_processor": "Intel Xeon Platinum 8259 (Cascade Lake)", "placement_group_support": false, "storage": 0.0, "vcpus": 4.0, "vpc_only": true}, "t1.micro": {"apiname": "t1.micro", "architecture": "32/64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": false, "linux_virtualization": "PV", "max_ips": 4, "memory": 0.613, "name": "T1 Micro", "network_perf": 0.0, "physical_processor": "Variable", "placement_group_support": false, "storage": 0.0, "vcpus": 1.0, "vpc_only": false}, "t2.2xlarge": {"apiname": "t2.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "Up to 3.0 GHz", "computeunits": 1.3599999999999999, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 45, "memory": 32.0, "name": "T2 Double Extra Large", "network_perf": 6.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 0.0, "vcpus": 8.0, "vpc_only": true}, "t2.large": {"apiname": "t2.large", "architecture": "64-bit", "clock_speed_ghz": "Up to 3.0 GHz", "computeunits": 0.6, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 36, "memory": 8.0, "name": "T2 Large", "network_perf": 4.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "t2.medium": {"apiname": "t2.medium", "architecture": "32/64-bit", "clock_speed_ghz": "Up to 3.3 GHz", "computeunits": 0.4, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 18, "memory": 4.0, "name": "T2 Medium", "network_perf": 4.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "t2.micro": {"apiname": "t2.micro", "architecture": "32/64-bit", "clock_speed_ghz": "Up to 3.3 GHz", "computeunits": 0.1, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 4, "memory": 1.0, "name": "T2 Micro", "network_perf": 4.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 0.0, "vcpus": 1.0, "vpc_only": true}, "t2.nano": {"apiname": "t2.nano", "architecture": "32/64-bit", "clock_speed_ghz": "Up to 3.3 GHz", "computeunits": 0.05, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 4, "memory": 0.5, "name": "T2 Nano", "network_perf": 2.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 0.0, "vcpus": 1.0, "vpc_only": true}, "t2.small": {"apiname": "t2.small", "architecture": "32/64-bit", "clock_speed_ghz": "Up to 3.3 GHz", "computeunits": 0.2, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 12, "memory": 2.0, "name": "T2 Small", "network_perf": 4.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 0.0, "vcpus": 1.0, "vpc_only": true}, "t2.xlarge": {"apiname": "t2.xlarge", "architecture": "64-bit", "clock_speed_ghz": "Up to 3.0 GHz", "computeunits": 0.9, "ebs_iops": 0.0, "ebs_max_bandwidth": 0.0, "ebs_throughput": 0.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "unknown", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 45, "memory": 16.0, "name": "T2 Extra Large", "network_perf": 6.0, "physical_processor": "Intel Xeon Family", "placement_group_support": false, "storage": 0.0, "vcpus": 4.0, "vpc_only": true}, "t3.2xlarge": {"apiname": "t3.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 3.2, "ebs_iops": 15700.0, "ebs_max_bandwidth": 2048.0, "ebs_throughput": 256.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 32.0, "name": "T3 Double Extra Large", "network_perf": 6.0, "physical_processor": "Intel Skylake E5 2686 v5 (2.5 GHz)", "placement_group_support": false, "storage": 0.0, "vcpus": 8.0, "vpc_only": true}, "t3.large": {"apiname": "t3.large", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.6, "ebs_iops": 15700.0, "ebs_max_bandwidth": 2048.0, "ebs_throughput": 256.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 36, "memory": 8.0, "name": "T3 Large", "network_perf": 4.0, "physical_processor": "Intel Skylake E5 2686 v5 (2.5 GHz)", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "t3.medium": {"apiname": "t3.medium", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.4, "ebs_iops": 11800.0, "ebs_max_bandwidth": 1536.0, "ebs_throughput": 192.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 18, "memory": 4.0, "name": "T3 Medium", "network_perf": 4.0, "physical_processor": "Intel Skylake E5 2686 v5 (2.5 GHz)", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "t3.micro": {"apiname": "t3.micro", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.2, "ebs_iops": 11800.0, "ebs_max_bandwidth": 1536.0, "ebs_throughput": 192.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 4, "memory": 1.0, "name": "T3 Micro", "network_perf": 4.0, "physical_processor": "Intel Skylake E5 2686 v5 (2.5 GHz)", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "t3.nano": {"apiname": "t3.nano", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.1, "ebs_iops": 11800.0, "ebs_max_bandwidth": 1536.0, "ebs_throughput": 192.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 4, "memory": 0.5, "name": "T3 Nano", "network_perf": 2.0, "physical_processor": "Intel Skylake E5 2686 v5 (2.5 GHz)", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "t3.small": {"apiname": "t3.small", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.4, "ebs_iops": 11800.0, "ebs_max_bandwidth": 1536.0, "ebs_throughput": 192.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 12, "memory": 2.0, "name": "T3 Small", "network_perf": 4.0, "physical_processor": "Intel Skylake E5 2686 v5 (2.5 GHz)", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "t3.xlarge": {"apiname": "t3.xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 1.6, "ebs_iops": 15700.0, "ebs_max_bandwidth": 2048.0, "ebs_throughput": 256.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 16.0, "name": "T3 Extra Large", "network_perf": 6.0, "physical_processor": "Intel Skylake E5 2686 v5 (2.5 GHz)", "placement_group_support": false, "storage": 0.0, "vcpus": 4.0, "vpc_only": true}, "t3a.2xlarge": {"apiname": "t3a.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 3.2, "ebs_iops": 15700.0, "ebs_max_bandwidth": 2048.0, "ebs_throughput": 256.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 32.0, "name": "T3A Double Extra Large", "network_perf": 6.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 8.0, "vpc_only": true}, "t3a.large": {"apiname": "t3a.large", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.6, "ebs_iops": 15700.0, "ebs_max_bandwidth": 2048.0, "ebs_throughput": 256.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 36, "memory": 8.0, "name": "T3A Large", "network_perf": 4.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "t3a.medium": {"apiname": "t3a.medium", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.4, "ebs_iops": 11800.0, "ebs_max_bandwidth": 1536.0, "ebs_throughput": 192.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 18, "memory": 4.0, "name": "T3A Medium", "network_perf": 4.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "t3a.micro": {"apiname": "t3a.micro", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.2, "ebs_iops": 11800.0, "ebs_max_bandwidth": 1536.0, "ebs_throughput": 192.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 4, "memory": 1.0, "name": "T3A Micro", "network_perf": 4.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "t3a.nano": {"apiname": "t3a.nano", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.1, "ebs_iops": 11800.0, "ebs_max_bandwidth": 1536.0, "ebs_throughput": 192.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 4, "memory": 0.5, "name": "T3A Nano", "network_perf": 2.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "t3a.small": {"apiname": "t3a.small", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 0.4, "ebs_iops": 11800.0, "ebs_max_bandwidth": 1536.0, "ebs_throughput": 192.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 8, "memory": 2.0, "name": "T3A Small", "network_perf": 4.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 2.0, "vpc_only": true}, "t3a.xlarge": {"apiname": "t3a.xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.5 GHz", "computeunits": 1.6, "ebs_iops": 15700.0, "ebs_max_bandwidth": 2048.0, "ebs_throughput": 256.0, "ecu_per_vcpu": 0.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 60, "memory": 16.0, "name": "T3A Extra Large", "network_perf": 6.0, "physical_processor": "AMD EPYC 7571", "placement_group_support": false, "storage": 0.0, "vcpus": 4.0, "vpc_only": true}, "u-12tb1.metal": {"apiname": "u-12tb1.metal", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 150, "memory": 12288.0, "name": "U-12TB1 Metal", "network_perf": 20.0, "physical_processor": "Intel Xeon Scalable (Skylake) processors", "placement_group_support": false, "storage": 0.0, "vcpus": 448.0, "vpc_only": true}, "u-18tb1.metal": {"apiname": "u-18tb1.metal", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 160000.0, "ebs_max_bandwidth": 28000.0, "ebs_throughput": 3500.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 750, "memory": 18432.0, "name": "U-18TB1 Metal", "network_perf": 26.0, "physical_processor": "Intel Xeon Platinum 8280L (Cascade Lake)", "placement_group_support": false, "storage": 0.0, "vcpus": 448.0, "vpc_only": true}, "u-24tb1.metal": {"apiname": "u-24tb1.metal", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 160000.0, "ebs_max_bandwidth": 28000.0, "ebs_throughput": 3500.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 750, "memory": 24576.0, "name": "U-24TB1 Metal", "network_perf": 26.0, "physical_processor": "Intel Xeon Platinum 8280L (Cascade Lake)", "placement_group_support": false, "storage": 0.0, "vcpus": 448.0, "vpc_only": true}, "u-6tb1.metal": {"apiname": "u-6tb1.metal", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 150, "memory": 6144.0, "name": "U-6TB1 Metal", "network_perf": 20.0, "physical_processor": "Intel Xeon Scalable (Skylake) processors", "placement_group_support": false, "storage": 0.0, "vcpus": 448.0, "vpc_only": true}, "u-9tb1.metal": {"apiname": "u-9tb1.metal", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 0.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 0.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "Unknown", "max_ips": 150, "memory": 9216.0, "name": "U-9TB1 Metal", "network_perf": 20.0, "physical_processor": "Intel Xeon Scalable (Skylake) processors", "placement_group_support": false, "storage": 0.0, "vcpus": 448.0, "vpc_only": true}, "x1.16xlarge": {"apiname": "x1.16xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 174.5, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 2.7265625, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 976.0, "name": "X1 Extra High-Memory 16xlarge", "network_perf": 8.0, "physical_processor": "High Frequency Intel Xeon E7-8880 v3 (Haswell)", "placement_group_support": false, "storage": 1920.0, "vcpus": 64.0, "vpc_only": true}, "x1.32xlarge": {"apiname": "x1.32xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 349.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 2.7265625, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 1952.0, "name": "X1 Extra High-Memory 32xlarge", "network_perf": 8.0, "physical_processor": "High Frequency Intel Xeon E7-8880 v3 (Haswell)", "placement_group_support": false, "storage": 3840.0, "vcpus": 128.0, "vpc_only": true}, "x1e.16xlarge": {"apiname": "x1e.16xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 179.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 2.796875, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 1952.0, "name": "X1E 16xlarge", "network_perf": 12.0, "physical_processor": "High Frequency Intel Xeon E7-8880 v3 (Haswell)", "placement_group_support": false, "storage": 1920.0, "vcpus": 64.0, "vpc_only": true}, "x1e.2xlarge": {"apiname": "x1e.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 23.0, "ebs_iops": 7400.0, "ebs_max_bandwidth": 1000.0, "ebs_throughput": 125.0, "ecu_per_vcpu": 2.875, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 244.0, "name": "X1E Double Extra Large", "network_perf": 10.0, "physical_processor": "High Frequency Intel Xeon E7-8880 v3 (Haswell)", "placement_group_support": false, "storage": 240.0, "vcpus": 8.0, "vpc_only": true}, "x1e.32xlarge": {"apiname": "x1e.32xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 340.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 2.65625, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 3904.0, "name": "X1E 32xlarge", "network_perf": 20.0, "physical_processor": "High Frequency Intel Xeon E7-8880 v3 (Haswell)", "placement_group_support": false, "storage": 3840.0, "vcpus": 128.0, "vpc_only": true}, "x1e.4xlarge": {"apiname": "x1e.4xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 47.0, "ebs_iops": 10000.0, "ebs_max_bandwidth": 1750.0, "ebs_throughput": 218.75, "ecu_per_vcpu": 2.9375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 488.0, "name": "X1E Quadruple Extra Large", "network_perf": 10.0, "physical_processor": "High Frequency Intel Xeon E7-8880 v3 (Haswell)", "placement_group_support": false, "storage": 480.0, "vcpus": 16.0, "vpc_only": true}, "x1e.8xlarge": {"apiname": "x1e.8xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 91.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 437.5, "ecu_per_vcpu": 2.84375, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "Yes", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 976.0, "name": "X1E Eight Extra Large", "network_perf": 10.0, "physical_processor": "High Frequency Intel Xeon E7-8880 v3 (Haswell)", "placement_group_support": false, "storage": 960.0, "vcpus": 32.0, "vpc_only": true}, "x1e.xlarge": {"apiname": "x1e.xlarge", "architecture": "64-bit", "clock_speed_ghz": "2.3 GHz", "computeunits": 12.0, "ebs_iops": 3700.0, "ebs_max_bandwidth": 500.0, "ebs_throughput": 62.5, "ecu_per_vcpu": 3.0, "enhanced_networking": true, "fpga": 0, "gpus": 0, "intel_avx": "Yes", "intel_avx2": "Yes", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 30, "memory": 122.0, "name": "X1E Extra Large", "network_perf": 10.0, "physical_processor": "High Frequency Intel Xeon E7-8880 v3 (Haswell)", "placement_group_support": false, "storage": 120.0, "vcpus": 4.0, "vpc_only": true}, "z1d.12xlarge": {"apiname": "z1d.12xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 271.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 5.645833333333333, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 384.0, "name": "Z1D 12xlarge", "network_perf": 20.0, "physical_processor": "Intel Xeon Platinum 8151", "placement_group_support": false, "storage": 1800.0, "vcpus": 48.0, "vpc_only": true}, "z1d.2xlarge": {"apiname": "z1d.2xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 53.0, "ebs_iops": 13333.0, "ebs_max_bandwidth": 2333.0, "ebs_throughput": 292.0, "ecu_per_vcpu": 6.625, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 64.0, "name": "Z1D Double Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon Platinum 8151", "placement_group_support": false, "storage": 300.0, "vcpus": 8.0, "vpc_only": true}, "z1d.3xlarge": {"apiname": "z1d.3xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 75.0, "ebs_iops": 20000.0, "ebs_max_bandwidth": 3500.0, "ebs_throughput": 438.0, "ecu_per_vcpu": 6.25, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 96.0, "name": "Z1D 3xlarge", "network_perf": 10.0, "physical_processor": "Intel Xeon Platinum 8151", "placement_group_support": false, "storage": 450.0, "vcpus": 12.0, "vpc_only": true}, "z1d.6xlarge": {"apiname": "z1d.6xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 134.0, "ebs_iops": 40000.0, "ebs_max_bandwidth": 7000.0, "ebs_throughput": 875.0, "ecu_per_vcpu": 5.583333333333333, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 240, "memory": 192.0, "name": "Z1D 6xlarge", "network_perf": 12.0, "physical_processor": "Intel Xeon Platinum 8151", "placement_group_support": false, "storage": 900.0, "vcpus": 24.0, "vpc_only": true}, "z1d.large": {"apiname": "z1d.large", "architecture": "64-bit", "clock_speed_ghz": "4.0 GHz", "computeunits": 15.0, "ebs_iops": 13333.0, "ebs_max_bandwidth": 2333.0, "ebs_throughput": 291.0, "ecu_per_vcpu": 7.5, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 30, "memory": 16.0, "name": "Z1D Large", "network_perf": 10.0, "physical_processor": "Intel Xeon Platinum 8151", "placement_group_support": false, "storage": 75.0, "vcpus": 2.0, "vpc_only": true}, "z1d.metal": {"apiname": "z1d.metal", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 271.0, "ebs_iops": 80000.0, "ebs_max_bandwidth": 14000.0, "ebs_throughput": 1750.0, "ecu_per_vcpu": 5.645833333333333, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 750, "memory": 384.0, "name": "Z1D Metal", "network_perf": 20.0, "physical_processor": "Intel Xeon Platinum 8151", "placement_group_support": false, "storage": 1800.0, "vcpus": 48.0, "vpc_only": true}, "z1d.xlarge": {"apiname": "z1d.xlarge", "architecture": "64-bit", "clock_speed_ghz": "unknown", "computeunits": 28.0, "ebs_iops": 13333.0, "ebs_max_bandwidth": 2333.0, "ebs_throughput": 291.0, "ecu_per_vcpu": 7.0, "enhanced_networking": false, "fpga": 0, "gpus": 0, "intel_avx": "unknown", "intel_avx2": "unknown", "intel_turbo": "unknown", "ipv6_support": true, "linux_virtualization": "HVM", "max_ips": 60, "memory": 32.0, "name": "Z1D Extra Large", "network_perf": 10.0, "physical_processor": "Intel Xeon Platinum 8151", "placement_group_support": false, "storage": 150.0, "vcpus": 4.0, "vpc_only": true}} \ No newline at end of file diff --git a/scripts/get_instance_info.py b/scripts/get_instance_info.py index f883c0cae..7aea257f8 100755 --- a/scripts/get_instance_info.py +++ b/scripts/get_instance_info.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + import json import os import subprocess @@ -11,128 +12,142 @@ class Instance(object): self.instance = instance def _get_td(self, td): - return self.instance.find('td', attrs={'class': td}) + return self.instance.find("td", attrs={"class": td}) def _get_sort(self, td): - return float(self.instance.find('td', attrs={'class': td}).find('span')['sort']) + return float(self.instance.find("td", attrs={"class": td}).find("span")["sort"]) @property def name(self): - return self._get_td('name').text.strip() + return self._get_td("name").text.strip() @property def apiname(self): - return self._get_td('apiname').text.strip() + return self._get_td("apiname").text.strip() @property def memory(self): - return self._get_sort('memory') + return self._get_sort("memory") @property def computeunits(self): - return self._get_sort('computeunits') + return self._get_sort("computeunits") @property def vcpus(self): - return self._get_sort('vcpus') + return self._get_sort("vcpus") @property def gpus(self): - return int(self._get_td('gpus').text.strip()) + return int(self._get_td("gpus").text.strip()) @property def fpga(self): - return int(self._get_td('fpga').text.strip()) + return int(self._get_td("fpga").text.strip()) @property def ecu_per_vcpu(self): - return self._get_sort('ecu-per-vcpu') + return self._get_sort("ecu-per-vcpu") @property def physical_processor(self): - return self._get_td('physical_processor').text.strip() + return self._get_td("physical_processor").text.strip() @property def clock_speed_ghz(self): - return self._get_td('clock_speed_ghz').text.strip() + return self._get_td("clock_speed_ghz").text.strip() @property def intel_avx(self): - return self._get_td('intel_avx').text.strip() + return self._get_td("intel_avx").text.strip() @property def intel_avx2(self): - return self._get_td('intel_avx2').text.strip() + return self._get_td("intel_avx2").text.strip() @property def intel_turbo(self): - return self._get_td('intel_turbo').text.strip() + return self._get_td("intel_turbo").text.strip() @property def storage(self): - return self._get_sort('storage') + return self._get_sort("storage") @property def architecture(self): - return self._get_td('architecture').text.strip() + return self._get_td("architecture").text.strip() @property def network_perf(self): # 2 == low - return self._get_sort('networkperf') + return self._get_sort("networkperf") @property def ebs_max_bandwidth(self): - return self._get_sort('ebs-max-bandwidth') + return self._get_sort("ebs-max-bandwidth") @property def ebs_throughput(self): - return self._get_sort('ebs-throughput') + return self._get_sort("ebs-throughput") @property def ebs_iops(self): - return self._get_sort('ebs-iops') + return self._get_sort("ebs-iops") @property def max_ips(self): - return int(self._get_td('maxips').text.strip()) + return int(self._get_td("maxips").text.strip()) @property def enhanced_networking(self): - return self._get_td('enhanced-networking').text.strip() != 'No' + return self._get_td("enhanced-networking").text.strip() != "No" @property def vpc_only(self): - return self._get_td('vpc-only').text.strip() != 'No' + return self._get_td("vpc-only").text.strip() != "No" @property def ipv6_support(self): - return self._get_td('ipv6-support').text.strip() != 'No' + return self._get_td("ipv6-support").text.strip() != "No" @property def placement_group_support(self): - return self._get_td('placement-group-support').text.strip() != 'No' + return self._get_td("placement-group-support").text.strip() != "No" @property def linux_virtualization(self): - return self._get_td('linux-virtualization').text.strip() + return self._get_td("linux-virtualization").text.strip() def to_dict(self): result = {} - for attr in [x for x in self.__class__.__dict__.keys() if not x.startswith('_') and x != 'to_dict']: - result[attr] = getattr(self, attr) + for attr in [ + x + for x in self.__class__.__dict__.keys() + if not x.startswith("_") and x != "to_dict" + ]: + try: + result[attr] = getattr(self, attr) + except ValueError as ex: + if "'N/A'" in str(ex): + print( + "Skipping attribute '{0}' for instance type '{1}' (not found)".format( + attr, self.name + ) + ) + else: + raise return self.apiname, result def main(): print("Getting HTML from http://www.ec2instances.info") - page_request = requests.get('http://www.ec2instances.info') - soup = BeautifulSoup(page_request.text, 'html.parser') - data_table = soup.find(id='data') + page_request = requests.get("http://www.ec2instances.info") + soup = BeautifulSoup(page_request.text, "html.parser") + data_table = soup.find(id="data") print("Finding data in table") - instances = data_table.find('tbody').find_all('tr') + instances = data_table.find("tbody").find_all("tr") print("Parsing data") result = {} @@ -140,11 +155,16 @@ def main(): instance_id, instance_data = Instance(instance).to_dict() result[instance_id] = instance_data - root_dir = subprocess.check_output(['git', 'rev-parse', '--show-toplevel']).decode().strip() - dest = os.path.join(root_dir, 'moto/ec2/resources/instance_types.json') + root_dir = ( + subprocess.check_output(["git", "rev-parse", "--show-toplevel"]) + .decode() + .strip() + ) + dest = os.path.join(root_dir, "moto/ec2/resources/instance_types.json") print("Writing data to {0}".format(dest)) - with open(dest, 'w') as open_file: - json.dump(result, open_file) + with open(dest, "w") as open_file: + json.dump(result, open_file, sort_keys=True) -if __name__ == '__main__': + +if __name__ == "__main__": main() From 870b34ba7693e88df38d2f2765b972cfda955cee Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Thu, 16 Apr 2020 07:09:50 +0100 Subject: [PATCH 51/83] Spacing --- tests/test_dynamodb2/test_dynamodb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 8a8c69a8c..a0a5e6406 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -4213,5 +4213,5 @@ def test_dynamodb_max_1mb_limit(): KeyConditionExpression=Key("partition_key").eq("partition_key_val") ) # We shouldn't get everything back - the total result set is well over 1MB - len(items).should.be.greater_than(response["Count"]) + len(items).should.be.greater_than(response["Count"]) response["LastEvaluatedKey"].shouldnt.be(None) From 92bbc3fbacbbe65fc6d9e134d15e78c97d3e256c Mon Sep 17 00:00:00 2001 From: Tim Date: Thu, 16 Apr 2020 08:20:43 -0700 Subject: [PATCH 52/83] Adds initial support for secretsmanager update_secret The support in this patch is preliminary and may or may not be feature complete. It provides the basic support for update_secret so that future work can build on it as needed. --- moto/secretsmanager/models.py | 28 +++++++ moto/secretsmanager/responses.py | 10 +++ .../test_secretsmanager.py | 76 +++++++++++++++++++ 3 files changed, 114 insertions(+) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 294a6401e..11a024be6 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -107,6 +107,34 @@ class SecretsManagerBackend(BaseBackend): return response + def update_secret( + self, secret_id, secret_string=None, secret_binary=None, **kwargs + ): + + # error if secret does not exist + if secret_id not in self.secrets.keys(): + raise SecretNotFoundException() + + if "deleted_date" in self.secrets[secret_id]: + raise InvalidRequestException( + "An error occurred (InvalidRequestException) when calling the UpdateSecret operation: " + "You can't perform this operation on the secret because it was marked for deletion." + ) + + version_id = self._add_secret( + secret_id, secret_string=secret_string, secret_binary=secret_binary + ) + + response = json.dumps( + { + "ARN": secret_arn(self.region, secret_id), + "Name": secret_id, + "VersionId": version_id, + } + ) + + return response + def create_secret( self, name, secret_string=None, secret_binary=None, tags=[], **kwargs ): diff --git a/moto/secretsmanager/responses.py b/moto/secretsmanager/responses.py index 28af7b91d..757b888a3 100644 --- a/moto/secretsmanager/responses.py +++ b/moto/secretsmanager/responses.py @@ -29,6 +29,16 @@ class SecretsManagerResponse(BaseResponse): tags=tags, ) + def update_secret(self): + secret_id = self._get_param("SecretId") + secret_string = self._get_param("SecretString") + secret_binary = self._get_param("SecretBinary") + return secretsmanager_backends[self.region].update_secret( + secret_id=secret_id, + secret_string=secret_string, + secret_binary=secret_binary, + ) + def get_random_password(self): password_length = self._get_param("PasswordLength", if_none=32) exclude_characters = self._get_param("ExcludeCharacters", if_none="") diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index 3b8c74e81..49d1dc925 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -711,3 +711,79 @@ def test_can_list_secret_version_ids(): returned_version_ids = [v["VersionId"] for v in versions_list["Versions"]] assert [first_version_id, second_version_id].sort() == returned_version_ids.sort() + + +@mock_secretsmanager +def test_update_secret(): + conn = boto3.client("secretsmanager", region_name="us-west-2") + + created_secret = conn.create_secret(Name="test-secret", SecretString="foosecret") + + assert created_secret["ARN"] + assert created_secret["Name"] == "test-secret" + assert created_secret["VersionId"] != "" + + secret = conn.get_secret_value(SecretId="test-secret") + assert secret["SecretString"] == "foosecret" + + updated_secret = conn.update_secret( + SecretId="test-secret", SecretString="barsecret" + ) + + assert updated_secret["ARN"] + assert updated_secret["Name"] == "test-secret" + assert updated_secret["VersionId"] != "" + + secret = conn.get_secret_value(SecretId="test-secret") + assert secret["SecretString"] == "barsecret" + assert created_secret["VersionId"] != updated_secret["VersionId"] + + +@mock_secretsmanager +def test_update_secret_which_does_not_exit(): + conn = boto3.client("secretsmanager", region_name="us-west-2") + + with assert_raises(ClientError) as cm: + updated_secret = conn.update_secret( + SecretId="test-secret", SecretString="barsecret" + ) + + assert_equal( + "Secrets Manager can't find the specified secret.", + cm.exception.response["Error"]["Message"], + ) + + +@mock_secretsmanager +def test_update_secret_marked_as_deleted(): + conn = boto3.client("secretsmanager", region_name="us-west-2") + + created_secret = conn.create_secret(Name="test-secret", SecretString="foosecret") + deleted_secret = conn.delete_secret(SecretId="test-secret") + + with assert_raises(ClientError) as cm: + updated_secret = conn.update_secret( + SecretId="test-secret", SecretString="barsecret" + ) + + assert ( + "because it was marked for deletion." + in cm.exception.response["Error"]["Message"] + ) + + +@mock_secretsmanager +def test_update_secret_marked_as_deleted_after_restoring(): + conn = boto3.client("secretsmanager", region_name="us-west-2") + + created_secret = conn.create_secret(Name="test-secret", SecretString="foosecret") + deleted_secret = conn.delete_secret(SecretId="test-secret") + restored_secret = conn.restore_secret(SecretId="test-secret") + + updated_secret = conn.update_secret( + SecretId="test-secret", SecretString="barsecret" + ) + + assert updated_secret["ARN"] + assert updated_secret["Name"] == "test-secret" + assert updated_secret["VersionId"] != "" From 4dc46a697d21eed1c22edcb2f8ffafbaa9e5445a Mon Sep 17 00:00:00 2001 From: Hugo Lopes Tavares Date: Thu, 16 Apr 2020 15:14:37 -0400 Subject: [PATCH 53/83] Bugfix: Allow stop_db_instance for compatible engines From the RDS documentation: You can stop and start a DB instance whether it is configured for a single Availability Zone or for Multi-AZ, for database engines that support Multi-AZ deployments. You can't stop an Amazon RDS for SQL Server DB instance in a Multi-AZ configuration. https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_StopInstance.html#USER_StopInstance.Limitations --- moto/rds2/models.py | 5 ++++- tests/test_rds2/test_rds2.py | 31 +++++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/moto/rds2/models.py b/moto/rds2/models.py index 963af1c63..722d7d4fd 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -865,7 +865,10 @@ class RDS2Backend(BaseBackend): def stop_database(self, db_instance_identifier, db_snapshot_identifier=None): database = self.describe_databases(db_instance_identifier)[0] # todo: certain rds types not allowed to be stopped at this time. - if database.is_replica or database.multi_az: + # https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_StopInstance.html#USER_StopInstance.Limitations + if database.is_replica or ( + database.multi_az and database.engine.lower().startswith("sqlserver") + ): # todo: more db types not supported by stop/start instance api raise InvalidDBClusterStateFaultError(db_instance_identifier) if database.status != "available": diff --git a/tests/test_rds2/test_rds2.py b/tests/test_rds2/test_rds2.py index e93ff43e9..13e35549a 100644 --- a/tests/test_rds2/test_rds2.py +++ b/tests/test_rds2/test_rds2.py @@ -183,12 +183,12 @@ def test_start_database(): @mock_rds2 -def test_fail_to_stop_multi_az(): +def test_fail_to_stop_multi_az_and_sqlserver(): conn = boto3.client("rds", region_name="us-west-2") database = conn.create_db_instance( DBInstanceIdentifier="db-master-1", AllocatedStorage=10, - Engine="postgres", + Engine="sqlserver-ee", DBName="staging-postgres", DBInstanceClass="db.m1.small", LicenseModel="license-included", @@ -213,6 +213,33 @@ def test_fail_to_stop_multi_az(): ).should.throw(ClientError) +@mock_rds2 +def test_stop_multi_az_postgres(): + conn = boto3.client("rds", region_name="us-west-2") + database = conn.create_db_instance( + DBInstanceIdentifier="db-master-1", + AllocatedStorage=10, + Engine="postgres", + DBName="staging-postgres", + DBInstanceClass="db.m1.small", + LicenseModel="license-included", + MasterUsername="root", + MasterUserPassword="hunter2", + Port=1234, + DBSecurityGroups=["my_sg"], + MultiAZ=True, + ) + + mydb = conn.describe_db_instances( + DBInstanceIdentifier=database["DBInstance"]["DBInstanceIdentifier"] + )["DBInstances"][0] + mydb["DBInstanceStatus"].should.equal("available") + + response = conn.stop_db_instance(DBInstanceIdentifier=mydb["DBInstanceIdentifier"]) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + response["DBInstance"]["DBInstanceStatus"].should.equal("stopped") + + @mock_rds2 def test_fail_to_stop_readreplica(): conn = boto3.client("rds", region_name="us-west-2") From 76a249c0ecbc616588b5ccf26224ad9efa9a05c9 Mon Sep 17 00:00:00 2001 From: Andrey Kislyuk Date: Thu, 16 Apr 2020 21:28:27 -0700 Subject: [PATCH 54/83] awslambda: Do not assume X-Amz-Invocation-Type is set --- moto/awslambda/responses.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/awslambda/responses.py b/moto/awslambda/responses.py index ce6c93f16..28b0e74fd 100644 --- a/moto/awslambda/responses.py +++ b/moto/awslambda/responses.py @@ -184,9 +184,9 @@ class LambdaResponse(BaseResponse): function_name, qualifier, self.body, self.headers, response_headers ) if payload: - if request.headers["X-Amz-Invocation-Type"] == "Event": + if request.headers.get("X-Amz-Invocation-Type") == "Event": status_code = 202 - elif request.headers["X-Amz-Invocation-Type"] == "DryRun": + elif request.headers.get("X-Amz-Invocation-Type") == "DryRun": status_code = 204 else: status_code = 200 From 7ea419dd54f475660f4e927ad39d62d226a43513 Mon Sep 17 00:00:00 2001 From: pvbouwel Date: Sat, 11 Apr 2020 11:07:22 +0100 Subject: [PATCH 55/83] Better DDB expressions support1: TokenizationDDB Currently the mock for DynamoDB has adhoc code to implement its updateExpression functionality. This series will transform the logic such that Update Expressions are processed as follows: 1) Expression gets parsed into a tokenlist (tokenized) -> This commit 2) Tokenlist get transformed to expression tree (AST) 3) The AST gets validated (full semantic correctness) 4) AST gets processed to perform the update This alows for a more realistic mocking. It will throw exceptions much more aggressively avoiding situations where a test passes against the mock but fails with an exception when running against AWS. Introduction of step 3 also allows to have the update expression as an atomic unit of work. So updates at the start of the expression cannot be performed if there is an error further down the expression. This specific commit will tokenize expressions but the tokenlist is not yet used. It is purely to keep clear boundaries. It does do a minor refactoring of the exceptions to allow more re-use and to ease testing. This series of changes is to aid providing a long-term solution for https://github.com/spulec/moto/issues/2806. --- moto/dynamodb2/exceptions.py | 58 +++- moto/dynamodb2/parsing/__init__.py | 0 moto/dynamodb2/parsing/tokens.py | 210 ++++++++++++++ moto/dynamodb2/responses.py | 14 +- .../test_dynamodb_expression_tokenizer.py | 259 ++++++++++++++++++ 5 files changed, 527 insertions(+), 14 deletions(-) create mode 100644 moto/dynamodb2/parsing/__init__.py create mode 100644 moto/dynamodb2/parsing/tokens.py create mode 100644 tests/test_dynamodb2/test_dynamodb_expression_tokenizer.py diff --git a/moto/dynamodb2/exceptions.py b/moto/dynamodb2/exceptions.py index 1f3b5f974..4c5dfd447 100644 --- a/moto/dynamodb2/exceptions.py +++ b/moto/dynamodb2/exceptions.py @@ -2,9 +2,59 @@ class InvalidIndexNameError(ValueError): pass -class InvalidUpdateExpression(ValueError): - pass +class MockValidationException(ValueError): + def __init__(self, message): + self.exception_msg = message -class ItemSizeTooLarge(Exception): - message = "Item size has exceeded the maximum allowed size" +class InvalidUpdateExpression(MockValidationException): + invalid_update_expression_msg = ( + "The document path provided in the update expression is invalid for update" + ) + + def __init__(self): + super(InvalidUpdateExpression, self).__init__( + self.invalid_update_expression_msg + ) + + +class UpdateExprSyntaxError(MockValidationException): + update_expr_syntax_error_msg = ( + "Invalid UpdateExpression: Syntax error; {error_detail}" + ) + + def __init__(self, error_detail): + self.error_detail = error_detail + super(UpdateExprSyntaxError, self).__init__( + self.update_expr_syntax_error_msg.format(error_detail=error_detail) + ) + + +class InvalidTokenException(UpdateExprSyntaxError): + token_detail_msg = 'token: "{token}", near: "{near}"' + + def __init__(self, token, near): + self.token = token + self.near = near + super(InvalidTokenException, self).__init__( + self.token_detail_msg.format(token=token, near=near) + ) + + +class InvalidExpressionAttributeNameKey(MockValidationException): + invalid_expr_attr_name_msg = ( + 'ExpressionAttributeNames contains invalid key: Syntax error; key: "{key}"' + ) + + def __init__(self, key): + self.key = key + super(InvalidExpressionAttributeNameKey, self).__init__( + self.invalid_expr_attr_name_msg.format(key=key) + ) + + +class ItemSizeTooLarge(MockValidationException): + item_size_too_large_msg = "Item size has exceeded the maximum allowed size" + + def __init__(self): + super(ItemSizeTooLarge, self).__init__(self.item_size_too_large_msg) diff --git a/moto/dynamodb2/parsing/__init__.py b/moto/dynamodb2/parsing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/moto/dynamodb2/parsing/tokens.py b/moto/dynamodb2/parsing/tokens.py new file mode 100644 index 000000000..07d65ae64 --- /dev/null +++ b/moto/dynamodb2/parsing/tokens.py @@ -0,0 +1,210 @@ +import re + +from moto.dynamodb2.exceptions import ( + InvalidTokenException, + InvalidExpressionAttributeNameKey, +) + + +class Token(object): + _TOKEN_INSTANCE = None + MINUS_SIGN = "-" + PLUS_SIGN = "+" + SPACE_SIGN = " " + EQUAL_SIGN = "=" + OPEN_ROUND_BRACKET = "(" + CLOSE_ROUND_BRACKET = ")" + COMMA = "," + SPACE = " " + DOT = "." + OPEN_SQUARE_BRACKET = "[" + CLOSE_SQUARE_BRACKET = "]" + + SPECIAL_CHARACTERS = [ + MINUS_SIGN, + PLUS_SIGN, + SPACE_SIGN, + EQUAL_SIGN, + OPEN_ROUND_BRACKET, + CLOSE_ROUND_BRACKET, + COMMA, + SPACE, + DOT, + OPEN_SQUARE_BRACKET, + CLOSE_SQUARE_BRACKET, + ] + + # Attribute: an identifier that is an attribute + ATTRIBUTE = 0 + # Place holder for attribute name + ATTRIBUTE_NAME = 1 + # Placeholder for attribute value starts with : + ATTRIBUTE_VALUE = 2 + # WhiteSpace shall be grouped together + WHITESPACE = 3 + # Placeholder for a number + NUMBER = 4 + + PLACEHOLDER_NAMES = { + ATTRIBUTE: "Attribute", + ATTRIBUTE_NAME: "AttributeName", + ATTRIBUTE_VALUE: "AttributeValue", + WHITESPACE: "Whitespace", + NUMBER: "Number", + } + + def __init__(self, token_type, value): + assert ( + token_type in self.SPECIAL_CHARACTERS + or token_type in self.PLACEHOLDER_NAMES + ) + self.type = token_type + self.value = value + + def __repr__(self): + if isinstance(self.type, int): + return 'Token("{tt}", "{tv}")'.format( + tt=self.PLACEHOLDER_NAMES[self.type], tv=self.value + ) + else: + return 'Token("{tt}", "{tv}")'.format(tt=self.type, tv=self.value) + + def __eq__(self, other): + return self.type == other.type and self.value == other.value + + +class ExpressionTokenizer(object): + """ + Takes a string and returns a list of tokens. While attribute names in DynamoDB must be between 1 and 255 characters + long there are no other restrictions for attribute names. For expressions however there are additional rules. If an + attribute name does not adhere then it must be passed via an ExpressionAttributeName. This tokenizer is aware of the + rules of Expression attributes. + + We consider a Token as a tuple which has the tokenType + + From https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.ExpressionAttributeNames.html + 1) If an attribute name begins with a number or contains a space, a special character, or a reserved word, you + must use an expression attribute name to replace that attribute's name in the expression. + => So spaces,+,- or other special characters do identify tokens in update expressions + + 2) When using a dot (.) in an attribute name you must use expression-attribute-names. A dot in an expression + will be interpreted as a separator in a document path + + 3) For a nested structure if you want to use expression_attribute_names you must specify one per part of the + path. Since for members of expression_attribute_names the . is part of the name + + """ + + @classmethod + def is_simple_token_character(cls, character): + return character.isalnum() or character in ("_", ":", "#") + + @classmethod + def is_possible_token_boundary(cls, character): + return ( + character in Token.SPECIAL_CHARACTERS + or not cls.is_simple_token_character(character) + ) + + @classmethod + def is_expression_attribute(cls, input_string): + return re.compile("^[a-zA-Z][a-zA-Z0-9_]*$").match(input_string) is not None + + @classmethod + def is_expression_attribute_name(cls, input_string): + """ + https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.ExpressionAttributeNames.html + An expression attribute name must begin with a pound sign (#), and be followed by one or more alphanumeric + characters. + """ + return input_string.startswith("#") and cls.is_expression_attribute( + input_string[1:] + ) + + @classmethod + def is_expression_attribute_value(cls, input_string): + return re.compile("^:[a-zA-Z0-9_]*$").match(input_string) is not None + + def raise_unexpected_token(self): + """If during parsing an unexpected token is encountered""" + if len(self.token_list) == 0: + near = "" + else: + if len(self.token_list) == 1: + near = self.token_list[-1].value + else: + if self.token_list[-1].type == Token.WHITESPACE: + # Last token was whitespace take 2nd last token value as well to help User orientate + near = self.token_list[-2].value + self.token_list[-1].value + else: + near = self.token_list[-1].value + + problematic_token = self.staged_characters[0] + raise InvalidTokenException(problematic_token, near + self.staged_characters) + + def __init__(self, input_expression_str): + self.input_expression_str = input_expression_str + self.token_list = [] + self.staged_characters = "" + + @classmethod + def make_list(cls, input_expression_str): + assert isinstance(input_expression_str, str) + return ExpressionTokenizer(input_expression_str)._make_list() + + def add_token(self, token_type, token_value): + self.token_list.append(Token(token_type, token_value)) + + def add_token_from_stage(self, token_type): + self.add_token(token_type, self.staged_characters) + self.staged_characters = "" + + def process_staged_characters(self): + if len(self.staged_characters) == 0: + return + if self.staged_characters.startswith("#"): + if self.is_expression_attribute_name(self.staged_characters): + self.add_token_from_stage(Token.ATTRIBUTE_NAME) + else: + raise InvalidExpressionAttributeNameKey(self.staged_characters) + elif self.staged_characters.isnumeric(): + self.add_token_from_stage(Token.NUMBER) + elif self.is_expression_attribute(self.staged_characters): + self.add_token_from_stage(Token.ATTRIBUTE) + elif self.is_expression_attribute_value(self.staged_characters): + self.add_token_from_stage(Token.ATTRIBUTE_VALUE) + else: + self.raise_unexpected_token() + + def _make_list(self): + """ + Just go through characters if a character is not a token boundary stage it for adding it as a grouped token + later if it is a tokenboundary process staged characters and then process the token boundary as well. + """ + for character in self.input_expression_str: + if not self.is_possible_token_boundary(character): + self.staged_characters += character + else: + self.process_staged_characters() + + if character == Token.SPACE: + if ( + len(self.token_list) > 0 + and self.token_list[-1].type == Token.WHITESPACE + ): + self.token_list[-1].value = ( + self.token_list[-1].value + character + ) + else: + self.add_token(Token.WHITESPACE, character) + elif character in Token.SPECIAL_CHARACTERS: + self.add_token(character, character) + elif not self.is_simple_token_character(character): + self.staged_characters += character + self.raise_unexpected_token() + else: + raise NotImplementedError( + "Encountered character which was not implemented : " + character + ) + self.process_staged_characters() + return self.token_list diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 65484aa08..d21d1d756 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -9,7 +9,7 @@ import six from moto.core.responses import BaseResponse from moto.core.utils import camelcase_to_underscores, amzn_request_id -from .exceptions import InvalidIndexNameError, InvalidUpdateExpression, ItemSizeTooLarge +from .exceptions import InvalidIndexNameError, InvalidUpdateExpression, ItemSizeTooLarge, MockValidationException from moto.dynamodb2.models import dynamodb_backends, dynamo_json_dump @@ -298,7 +298,7 @@ class DynamoHandler(BaseResponse): ) except ItemSizeTooLarge: er = "com.amazonaws.dynamodb.v20111205#ValidationException" - return self.error(er, ItemSizeTooLarge.message) + return self.error(er, ItemSizeTooLarge.item_size_too_large_msg) except KeyError as ke: er = "com.amazonaws.dynamodb.v20111205#ValidationException" return self.error(er, ke.args[0]) @@ -764,15 +764,9 @@ class DynamoHandler(BaseResponse): expected, condition_expression, ) - except InvalidUpdateExpression: + except MockValidationException as mve: er = "com.amazonaws.dynamodb.v20111205#ValidationException" - return self.error( - er, - "The document path provided in the update expression is invalid for update", - ) - except ItemSizeTooLarge: - er = "com.amazonaws.dynamodb.v20111205#ValidationException" - return self.error(er, ItemSizeTooLarge.message) + return self.error(er, mve.exception_msg) except ValueError: er = "com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException" return self.error( diff --git a/tests/test_dynamodb2/test_dynamodb_expression_tokenizer.py b/tests/test_dynamodb2/test_dynamodb_expression_tokenizer.py new file mode 100644 index 000000000..3330d431e --- /dev/null +++ b/tests/test_dynamodb2/test_dynamodb_expression_tokenizer.py @@ -0,0 +1,259 @@ +from moto.dynamodb2.exceptions import ( + InvalidTokenException, + InvalidExpressionAttributeNameKey, +) +from moto.dynamodb2.parsing.tokens import ExpressionTokenizer, Token + + +def test_expression_tokenizer_single_set_action(): + set_action = "SET attrName = :attrValue" + token_list = ExpressionTokenizer.make_list(set_action) + assert token_list == [ + Token(Token.ATTRIBUTE, "SET"), + Token(Token.WHITESPACE, " "), + Token(Token.ATTRIBUTE, "attrName"), + Token(Token.WHITESPACE, " "), + Token(Token.EQUAL_SIGN, "="), + Token(Token.WHITESPACE, " "), + Token(Token.ATTRIBUTE_VALUE, ":attrValue"), + ] + + +def test_expression_tokenizer_single_set_action_leading_space(): + set_action = "Set attrName = :attrValue" + token_list = ExpressionTokenizer.make_list(set_action) + assert token_list == [ + Token(Token.ATTRIBUTE, "Set"), + Token(Token.WHITESPACE, " "), + Token(Token.ATTRIBUTE, "attrName"), + Token(Token.WHITESPACE, " "), + Token(Token.EQUAL_SIGN, "="), + Token(Token.WHITESPACE, " "), + Token(Token.ATTRIBUTE_VALUE, ":attrValue"), + ] + + +def test_expression_tokenizer_single_set_action_attribute_name_leading_space(): + set_action = "SET #a = :attrValue" + token_list = ExpressionTokenizer.make_list(set_action) + assert token_list == [ + Token(Token.ATTRIBUTE, "SET"), + Token(Token.WHITESPACE, " "), + Token(Token.ATTRIBUTE_NAME, "#a"), + Token(Token.WHITESPACE, " "), + Token(Token.EQUAL_SIGN, "="), + Token(Token.WHITESPACE, " "), + Token(Token.ATTRIBUTE_VALUE, ":attrValue"), + ] + + +def test_expression_tokenizer_single_set_action_trailing_space(): + set_action = "SET attrName = :attrValue " + token_list = ExpressionTokenizer.make_list(set_action) + assert token_list == [ + Token(Token.ATTRIBUTE, "SET"), + Token(Token.WHITESPACE, " "), + Token(Token.ATTRIBUTE, "attrName"), + Token(Token.WHITESPACE, " "), + Token(Token.EQUAL_SIGN, "="), + Token(Token.WHITESPACE, " "), + Token(Token.ATTRIBUTE_VALUE, ":attrValue"), + Token(Token.WHITESPACE, " "), + ] + + +def test_expression_tokenizer_single_set_action_multi_spaces(): + set_action = "SET attrName = :attrValue " + token_list = ExpressionTokenizer.make_list(set_action) + assert token_list == [ + Token(Token.ATTRIBUTE, "SET"), + Token(Token.WHITESPACE, " "), + Token(Token.ATTRIBUTE, "attrName"), + Token(Token.WHITESPACE, " "), + Token(Token.EQUAL_SIGN, "="), + Token(Token.WHITESPACE, " "), + Token(Token.ATTRIBUTE_VALUE, ":attrValue"), + Token(Token.WHITESPACE, " "), + ] + + +def test_expression_tokenizer_single_set_action_with_numbers_in_identifiers(): + set_action = "SET attrName3 = :attr3Value" + token_list = ExpressionTokenizer.make_list(set_action) + assert token_list == [ + Token(Token.ATTRIBUTE, "SET"), + Token(Token.WHITESPACE, " "), + Token(Token.ATTRIBUTE, "attrName3"), + Token(Token.WHITESPACE, " "), + Token(Token.EQUAL_SIGN, "="), + Token(Token.WHITESPACE, " "), + Token(Token.ATTRIBUTE_VALUE, ":attr3Value"), + ] + + +def test_expression_tokenizer_single_set_action_with_underscore_in_identifier(): + set_action = "SET attr_Name = :attr_Value" + token_list = ExpressionTokenizer.make_list(set_action) + assert token_list == [ + Token(Token.ATTRIBUTE, "SET"), + Token(Token.WHITESPACE, " "), + Token(Token.ATTRIBUTE, "attr_Name"), + Token(Token.WHITESPACE, " "), + Token(Token.EQUAL_SIGN, "="), + Token(Token.WHITESPACE, " "), + Token(Token.ATTRIBUTE_VALUE, ":attr_Value"), + ] + + +def test_expression_tokenizer_leading_underscore_in_attribute_name_expression(): + """Leading underscore is not allowed for an attribute name""" + set_action = "SET attrName = _idid" + try: + ExpressionTokenizer.make_list(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "_" + assert te.near == "= _idid" + + +def test_expression_tokenizer_leading_underscore_in_attribute_value_expression(): + """Leading underscore is allowed in an attribute value""" + set_action = "SET attrName = :_attrValue" + token_list = ExpressionTokenizer.make_list(set_action) + assert token_list == [ + Token(Token.ATTRIBUTE, "SET"), + Token(Token.WHITESPACE, " "), + Token(Token.ATTRIBUTE, "attrName"), + Token(Token.WHITESPACE, " "), + Token(Token.EQUAL_SIGN, "="), + Token(Token.WHITESPACE, " "), + Token(Token.ATTRIBUTE_VALUE, ":_attrValue"), + ] + + +def test_expression_tokenizer_single_set_action_nested_attribute(): + set_action = "SET attrName.elem = :attrValue" + token_list = ExpressionTokenizer.make_list(set_action) + assert token_list == [ + Token(Token.ATTRIBUTE, "SET"), + Token(Token.WHITESPACE, " "), + Token(Token.ATTRIBUTE, "attrName"), + Token(Token.DOT, "."), + Token(Token.ATTRIBUTE, "elem"), + Token(Token.WHITESPACE, " "), + Token(Token.EQUAL_SIGN, "="), + Token(Token.WHITESPACE, " "), + Token(Token.ATTRIBUTE_VALUE, ":attrValue"), + ] + + +def test_expression_tokenizer_list_index_with_sub_attribute(): + set_action = "SET itemmap.itemlist[1].foos=:Item" + token_list = ExpressionTokenizer.make_list(set_action) + assert token_list == [ + Token(Token.ATTRIBUTE, "SET"), + Token(Token.WHITESPACE, " "), + Token(Token.ATTRIBUTE, "itemmap"), + Token(Token.DOT, "."), + Token(Token.ATTRIBUTE, "itemlist"), + Token(Token.OPEN_SQUARE_BRACKET, "["), + Token(Token.NUMBER, "1"), + Token(Token.CLOSE_SQUARE_BRACKET, "]"), + Token(Token.DOT, "."), + Token(Token.ATTRIBUTE, "foos"), + Token(Token.EQUAL_SIGN, "="), + Token(Token.ATTRIBUTE_VALUE, ":Item"), + ] + + +def test_expression_tokenizer_list_index_surrounded_with_whitespace(): + set_action = "SET itemlist[ 1 ]=:Item" + token_list = ExpressionTokenizer.make_list(set_action) + assert token_list == [ + Token(Token.ATTRIBUTE, "SET"), + Token(Token.WHITESPACE, " "), + Token(Token.ATTRIBUTE, "itemlist"), + Token(Token.OPEN_SQUARE_BRACKET, "["), + Token(Token.WHITESPACE, " "), + Token(Token.NUMBER, "1"), + Token(Token.WHITESPACE, " "), + Token(Token.CLOSE_SQUARE_BRACKET, "]"), + Token(Token.EQUAL_SIGN, "="), + Token(Token.ATTRIBUTE_VALUE, ":Item"), + ] + + +def test_expression_tokenizer_single_set_action_attribute_name_invalid_key(): + """ + ExpressionAttributeNames contains invalid key: Syntax error; key: "#va#l2" + """ + set_action = "SET #va#l2 = 3" + try: + ExpressionTokenizer.make_list(set_action) + assert False, "Exception not raised correctly" + except InvalidExpressionAttributeNameKey as e: + assert e.key == "#va#l2" + + +def test_expression_tokenizer_single_set_action_attribute_name_invalid_key_double_hash(): + """ + ExpressionAttributeNames contains invalid key: Syntax error; key: "#va#l" + """ + set_action = "SET #va#l = 3" + try: + ExpressionTokenizer.make_list(set_action) + assert False, "Exception not raised correctly" + except InvalidExpressionAttributeNameKey as e: + assert e.key == "#va#l" + + +def test_expression_tokenizer_single_set_action_attribute_name_valid_key(): + set_action = "SET attr=#val2" + token_list = ExpressionTokenizer.make_list(set_action) + assert token_list == [ + Token(Token.ATTRIBUTE, "SET"), + Token(Token.WHITESPACE, " "), + Token(Token.ATTRIBUTE, "attr"), + Token(Token.EQUAL_SIGN, "="), + Token(Token.ATTRIBUTE_NAME, "#val2"), + ] + + +def test_expression_tokenizer_just_a_pipe(): + set_action = "|" + try: + ExpressionTokenizer.make_list(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "|" + assert te.near == "|" + + +def test_expression_tokenizer_just_a_pipe_with_leading_white_spaces(): + set_action = " |" + try: + ExpressionTokenizer.make_list(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "|" + assert te.near == " |" + + +def test_expression_tokenizer_just_a_pipe_for_set_expression(): + set_action = "SET|" + try: + ExpressionTokenizer.make_list(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "|" + assert te.near == "SET|" + + +def test_expression_tokenizer_just_an_attribute_and_a_pipe_for_set_expression(): + set_action = "SET a|" + try: + ExpressionTokenizer.make_list(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "|" + assert te.near == "a|" From 9ed613e197e1d6e85f9631f7a15a2b8ce2f19b2e Mon Sep 17 00:00:00 2001 From: pvbouwel Date: Sat, 11 Apr 2020 21:17:16 +0100 Subject: [PATCH 56/83] Better DDB expressions support2: ExpressionTree Part of structured approach for UpdateExpressions: 1) Expression gets parsed into a tokenlist (tokenized) 2) Tokenlist get transformed to expression tree (AST) -> This commit 3) The AST gets validated (full semantic correctness) 4) AST gets processed to perform the update This commit uses the tokenlist to build an expression tree. This tree is not yet used. Still it allows to raise additional Validation Exceptions which previously were missed silently therefore it allows tests to catch these type of ValidationException. For that reason DDB UpdateExpressions will be parsed already. It also makes sure we won't break existing tests. One of the existing tests had to be changed in order to still pass: - test_dynamodb_table_with_range_key.test_update_item_with_expression This test passed in a numeric literal which is not supported by DynamoDB and with the current tokenization it would get the same error as in AWS DynamoDB. --- moto/dynamodb2/models/__init__.py | 12 +- moto/dynamodb2/parsing/ast_nodes.py | 205 ++++ moto/dynamodb2/parsing/expressions.py | 1010 +++++++++++++++++ moto/dynamodb2/parsing/reserved_keywords.py | 29 + moto/dynamodb2/parsing/reserved_keywords.txt | 573 ++++++++++ moto/dynamodb2/parsing/tokens.py | 17 +- moto/dynamodb2/responses.py | 5 - tests/test_dynamodb2/test_dynamodb.py | 68 ++ .../test_dynamodb_expressions.py | 395 +++++++ .../test_dynamodb_table_with_range_key.py | 16 +- 10 files changed, 2317 insertions(+), 13 deletions(-) create mode 100644 moto/dynamodb2/parsing/ast_nodes.py create mode 100644 moto/dynamodb2/parsing/expressions.py create mode 100644 moto/dynamodb2/parsing/reserved_keywords.py create mode 100644 moto/dynamodb2/parsing/reserved_keywords.txt create mode 100644 tests/test_dynamodb2/test_dynamodb_expressions.py diff --git a/moto/dynamodb2/models/__init__.py b/moto/dynamodb2/models/__init__.py index 29713d211..1f448f288 100644 --- a/moto/dynamodb2/models/__init__.py +++ b/moto/dynamodb2/models/__init__.py @@ -14,10 +14,11 @@ from moto.core import BaseBackend, BaseModel from moto.core.utils import unix_time from moto.core.exceptions import JsonRESTError from moto.dynamodb2.comparisons import get_filter_expression -from moto.dynamodb2.comparisons import get_expected -from moto.dynamodb2.exceptions import InvalidIndexNameError, ItemSizeTooLarge +from moto.dynamodb2.comparisons import get_expected, get_comparison_func +from moto.dynamodb2.exceptions import InvalidIndexNameError, ItemSizeTooLarge, InvalidUpdateExpression from moto.dynamodb2.models.utilities import bytesize, attribute_is_list from moto.dynamodb2.models.dynamo_type import DynamoType +from moto.dynamodb2.parsing.expressions import UpdateExpressionParser class DynamoJsonEncoder(json.JSONEncoder): @@ -1197,6 +1198,13 @@ class DynamoDBBackend(BaseBackend): ): table = self.get_table(table_name) + # Support spaces between operators in an update expression + # E.g. `a = b + c` -> `a=b+c` + if update_expression: + # Parse expression to get validation errors + UpdateExpressionParser.make(update_expression) + update_expression = re.sub(r"\s*([=\+-])\s*", "\\1", update_expression) + if all([table.hash_key_attr in key, table.range_key_attr in key]): # Covers cases where table has hash and range keys, ``key`` param # will be a dict diff --git a/moto/dynamodb2/parsing/ast_nodes.py b/moto/dynamodb2/parsing/ast_nodes.py new file mode 100644 index 000000000..78c7b6b2b --- /dev/null +++ b/moto/dynamodb2/parsing/ast_nodes.py @@ -0,0 +1,205 @@ +import abc +import six + + +@six.add_metaclass(abc.ABCMeta) +class Node: + def __init__(self, children=None): + self.type = self.__class__.__name__ + assert children is None or isinstance(children, list) + self.children = children + self.parent = None + + if isinstance(children, list): + for child in children: + if isinstance(child, Node): + child.set_parent(self) + + def set_parent(self, parent_node): + self.parent = parent_node + + +class LeafNode(Node): + """A LeafNode is a Node where none of the children are Nodes themselves.""" + + def __init__(self, children=None): + super(LeafNode, self).__init__(children) + + +@six.add_metaclass(abc.ABCMeta) +class Expression(Node): + """ + Abstract Syntax Tree representing the expression + + For the Grammar start here and jump down into the classes at the righ-hand side to look further. Nodes marked with + a star are abstract and won't appear in the final AST. + + Expression* => UpdateExpression + Expression* => ConditionExpression + """ + + +class UpdateExpression(Expression): + """ + UpdateExpression => UpdateExpressionClause* + UpdateExpression => UpdateExpressionClause* UpdateExpression + """ + + +@six.add_metaclass(abc.ABCMeta) +class UpdateExpressionClause(UpdateExpression): + """ + UpdateExpressionClause* => UpdateExpressionSetClause + UpdateExpressionClause* => UpdateExpressionRemoveClause + UpdateExpressionClause* => UpdateExpressionAddClause + UpdateExpressionClause* => UpdateExpressionDeleteClause + """ + + +class UpdateExpressionSetClause(UpdateExpressionClause): + """ + UpdateExpressionSetClause => SET SetActions + """ + + +class UpdateExpressionSetActions(UpdateExpressionClause): + """ + UpdateExpressionSetClause => SET SetActions + + SetActions => SetAction + SetActions => SetAction , SetActions + + """ + + +class UpdateExpressionSetAction(UpdateExpressionClause): + """ + SetAction => Path = Value + """ + + +class UpdateExpressionRemoveActions(UpdateExpressionClause): + """ + UpdateExpressionSetClause => REMOVE RemoveActions + + RemoveActions => RemoveAction + RemoveActions => RemoveAction , RemoveActions + """ + + +class UpdateExpressionRemoveAction(UpdateExpressionClause): + """ + RemoveAction => Path + """ + + +class UpdateExpressionAddActions(UpdateExpressionClause): + """ + UpdateExpressionAddClause => ADD RemoveActions + + AddActions => AddAction + AddActions => AddAction , AddActions + """ + + +class UpdateExpressionAddAction(UpdateExpressionClause): + """ + AddAction => Path Value + """ + + +class UpdateExpressionDeleteActions(UpdateExpressionClause): + """ + UpdateExpressionDeleteClause => DELETE RemoveActions + + DeleteActions => DeleteAction + DeleteActions => DeleteAction , DeleteActions + """ + + +class UpdateExpressionDeleteAction(UpdateExpressionClause): + """ + DeleteAction => Path Value + """ + + +class UpdateExpressionPath(UpdateExpressionClause): + pass + + +class UpdateExpressionValue(UpdateExpressionClause): + """ + Value => Operand + Value => Operand + Value + Value => Operand - Value + """ + + +class UpdateExpressionGroupedValue(UpdateExpressionClause): + """ + GroupedValue => ( Value ) + """ + + +class UpdateExpressionRemoveClause(UpdateExpressionClause): + """ + UpdateExpressionRemoveClause => REMOVE RemoveActions + """ + + +class UpdateExpressionAddClause(UpdateExpressionClause): + """ + UpdateExpressionAddClause => ADD AddActions + """ + + +class UpdateExpressionDeleteClause(UpdateExpressionClause): + """ + UpdateExpressionDeleteClause => DELETE DeleteActions + """ + + +class ExpressionPathDescender(Node): + """Node identifying descender into nested structure (.) in expression""" + + +class ExpressionSelector(LeafNode): + """Node identifying selector [selection_index] in expresion""" + + def __init__(self, selection_index): + super(ExpressionSelector, self).__init__(children=[selection_index]) + + +class ExpressionAttribute(LeafNode): + """An attribute identifier as used in the DDB item""" + + def __init__(self, attribute): + super(ExpressionAttribute, self).__init__(children=[attribute]) + + +class ExpressionAttributeName(LeafNode): + """An ExpressionAttributeName is an alias for an attribute identifier""" + + def __init__(self, attribute_name): + super(ExpressionAttributeName, self).__init__(children=[attribute_name]) + + +class ExpressionAttributeValue(LeafNode): + """An ExpressionAttributeValue is an alias for an value""" + + def __init__(self, value): + super(ExpressionAttributeValue, self).__init__(children=[value]) + + +class ExpressionValueOperator(LeafNode): + """An ExpressionValueOperator is an operation that works on 2 values""" + + def __init__(self, value): + super(ExpressionValueOperator, self).__init__(children=[value]) + + +class UpdateExpressionFunction(Node): + """ + A Node representing a function of an Update Expression. The first child is the function name the others are the + arguments. + """ diff --git a/moto/dynamodb2/parsing/expressions.py b/moto/dynamodb2/parsing/expressions.py new file mode 100644 index 000000000..e418bb47e --- /dev/null +++ b/moto/dynamodb2/parsing/expressions.py @@ -0,0 +1,1010 @@ +import logging +from abc import abstractmethod +import abc +import six +from collections import deque + +from moto.dynamodb2.parsing.ast_nodes import ( + UpdateExpression, + UpdateExpressionSetClause, + UpdateExpressionSetActions, + UpdateExpressionSetAction, + UpdateExpressionRemoveActions, + UpdateExpressionRemoveAction, + UpdateExpressionPath, + UpdateExpressionValue, + UpdateExpressionGroupedValue, + UpdateExpressionRemoveClause, + ExpressionPathDescender, + ExpressionSelector, + ExpressionAttribute, + ExpressionAttributeName, + ExpressionAttributeValue, + ExpressionValueOperator, + UpdateExpressionFunction, + UpdateExpressionAddClause, + UpdateExpressionAddActions, + UpdateExpressionAddAction, + UpdateExpressionDeleteAction, + UpdateExpressionDeleteActions, + UpdateExpressionDeleteClause, +) +from moto.dynamodb2.exceptions import InvalidTokenException +from moto.dynamodb2.parsing.tokens import Token, ExpressionTokenizer + + +class NestableExpressionParserMixin(object): + """ + For nodes that can be nested in themselves (recursive). Take for example UpdateExpression's grammar: + + UpdateExpression => UpdateExpressionClause* + UpdateExpression => UpdateExpressionClause* UpdateExpression + + If we consider it of structure + NestableExpression => TargetClause* + NestableExpression => TargetClause* NestableExpression + + This pattern comes back multiple times. This Mixin adds re-usability for that type of pattern. + + This approach is taken since it allows to remain the ordering of the Nodes as how the corresponding tokens where + in the originating expression. + """ + + def __init__(self, *args, **kwargs): + self.target_clauses = deque() + + def _parse_target_clause(self, factory_class): + """ + + Args: + factory_class: The factory for the target clause e.g. UpdateExpressionSetClauseParser + + Returns: + + """ + logging.debug( + "Move token pos {pos} to continue parsing with specific factory class {fc}".format( + pos=self.token_pos, fc=factory_class.__class__.__name__ + ) + ) + # noinspection PyProtectedMember + ast, token_pos = factory_class(**self._initializer_args())._parse_with_pos() + self.target_clauses.append(ast) + logging.debug( + "Continue where previous parsing ended {token_pos}".format( + token_pos=token_pos + ) + ) + self.token_pos = token_pos + + @abstractmethod + def _initializer_args(self): + """ + Get the arguments of the initializer. This is implemented by the calling class. See ExpressionParser for an + example. + + Returns: + dict: A dictionary of the initializer arguments + """ + + @classmethod + @abstractmethod + def _nestable_class(cls): + """ + Get the class of the Node that will be created that would be nested. For the example in the docstring this would + be UpdateExpression + + Returns: + class: The class of the Nodes that will be created. + """ + + def _create_node(self): + """ + target_clauses has the nodes in order of encountering. Go through them backwards and build the tree bottom up. + + This way left-deep-descending traversal will process nodes in order. + + Continuing the example of an UpdateExpression: + For example SET a=3 REMOVE b + UpdateExpression + / \ + SET a=3 UpdateExpression + | + REMOVE b + + self.target_clauses looks like: ( SET a=3 >> REMOVE b ) + Returns: + moto.dynamodb2.ast_nodes.Node: Node of an AST representing the Expression as produced by the factory. + """ + assert len(self.target_clauses) > 0, "No nodes for {cn}".format( + cn=self.__class__.__name__ + ) + target_node = self._nestable_class()(children=[self.target_clauses.pop()]) + while len(self.target_clauses) > 0: + target_node = self._nestable_class()( + children=[self.target_clauses.pop(), target_node] + ) + return target_node + + +@six.add_metaclass(abc.ABCMeta) +class ExpressionParser: + """Abstract class""" + + def __init__(self, expression_token_list, token_pos=0): + """ + + Args: + expression_token_list: + token_pos(int): Location where parsing is + """ + self.token_list = expression_token_list + self.token_pos = token_pos + + def _initializer_args(self): + return {"expression_token_list": self.token_list, "token_pos": self.token_pos} + + @abstractmethod + def _parse(self): + """ + Start parsing the token_list from token_pos for the factory type. + + Returns: + moto.dynamodb2.ast_nodes.Node: AST which is root node of resulting abstract syntax tree + """ + + @classmethod + def is_possible_start(cls, token): + return token is not None and cls._is_possible_start(token) + + @classmethod + @abstractmethod + def _is_possible_start(cls, token): + """ + + Args: + token(moto.dynamodb2.tokens.Token): + + Returns: + bool: True if token is a possible start for entries processed by `cls` + """ + + def _parse_with_pos(self): + """ + Start parsing the token_list from token_pos for the factory type and also return the resulting token_pos. + + Returns: + (ast, token_pos): tuple of AST which is root node of resulting abstract syntax tree and token_pos is the + position in the tokenlist. + """ + return self._parse(), self.token_pos + + def parse(self): + return self._parse() + + def get_next_token_type(self): + """ + Get the type of the next token to be processed + + Returns: + str: Token type or None if no more next token + """ + try: + return self.get_next_token().type + except AttributeError: + return None + + def get_next_token(self): + """ + Get the next token to be processed + + Returns: + moto.dynamodb2.tokens.Token: or None if no more next token + """ + try: + return self.token_list[self.token_pos] + except IndexError: + return None + + def get_next_token_value(self): + """ + Get the value of the next token to be processed + + Returns: + str: value or None if no more next token + """ + try: + return self.get_next_token().value + except AttributeError: + return None + + def is_at_end(self): + """Return boolean indicating whether we are at end of the parsing""" + return self.token_pos == len(self.token_list) + + def is_at_start(self): + """Return boolean indicating whether we are at start of the parsing""" + return self.token_pos == 0 + + def get_last_token_value(self): + """Get the last token that was correctly parsed or return empty string""" + if self.token_pos > 0: + return self.token_list[self.token_pos - 1].value + else: + return "" + + def get_last_token_type(self): + """Get the last token type that was correctly parsed or return None""" + if self.token_pos > 0: + return self.token_list[self.token_pos - 1].type + else: + return None + + def get_2nd_last_token_value_if_last_was_whitespace(self): + """Get the 2nd last token that was correctly parsed if last one was whitespace or return empty string""" + if self.token_pos > 1 and self.get_last_token_type() == Token.WHITESPACE: + return self.token_list[self.token_pos - 2].value + else: + return "" + + def get_following_token_value(self): + """Get the token value after the one that is being parsed or empty string if non existent.""" + try: + return self.token_list[self.token_pos + 1].value + except IndexError: + return "" + + def get_following_token_type(self): + """Get the token type after the one that is being parsed or None if non existent.""" + try: + return self.token_list[self.token_pos + 1].type + except IndexError: + return None + + def get_2nd_following_token_value_if_following_was_whitespace(self): + """Get the 2nd following token that was correctly parsed if 1st one was whitespace or return empty string""" + if self.get_following_token_type() == Token.WHITESPACE: + try: + return self.token_list[self.token_pos + 2].value + except IndexError: + return "" + else: + return "" + + def skip_white_space(self): + try: + while self.get_next_token_type() == Token.WHITESPACE: + self.token_pos += 1 + except IndexError: + assert self.token_pos > 0, "We should always have positive indexes" + logging.debug("We are out of range so end is reached") + + def process_token_of_type(self, token_type): + """ + Maker sure the next token is of type `token_type` if not raise unexpected token + Args: + token_type: A token type + + Returns: + str: The value if the token is of type `token_type` + """ + if self.get_next_token_type() == token_type: + token_value = self.get_next_token_value() + self.goto_next_significant_token() + return token_value + else: + self.raise_unexpected_token() + + def goto_next_significant_token(self): + """Continue past current token and skip all whitespaces""" + self.token_pos += 1 + self.skip_white_space() + + def raise_unexpected_token(self): + if self.is_at_end(): + problematic_token = "" + problematic_token_in_near = "" + else: + problematic_token_in_near = problematic_token = self.get_next_token_value() + + near = "".join( + [ + self.get_2nd_last_token_value_if_last_was_whitespace(), + self.get_last_token_value(), + problematic_token_in_near, + self.get_following_token_value(), + self.get_2nd_following_token_value_if_following_was_whitespace(), + ] + ) + + raise InvalidTokenException(problematic_token, near) + + +class NestableBinExpressionParser(ExpressionParser): + """ + For nodes that can be nested in themselves (recursive) but with an operation. Take for example + UpdateExpressionValue's grammar: + + Value => Operand* + Value => Operand* + Value + Value => Operand* - Value + + If we consider it of structure + NestableBinExpression => TargetClause* + NestableBinExpression => TargetClause* BinOp NestableBinExpression + + This pattern comes back multiple times. This Mixin adds re-usability for that type of pattern. + + This approach is taken since it allows to remain the ordering of the Nodes as how the corresponding tokens where + in the originating expression. + """ + + def __init__(self, *args, **kwargs): + super(NestableBinExpressionParser, self).__init__(*args, **kwargs) + self.target_nodes = deque() + + def _parse_target_clause(self, factory_class): + """ + + Args: + factory_class: The factory for the target clause e.g. UpdateExpressionSetClauseParser + + Returns: + + """ + # noinspection PyProtectedMember + ast, self.token_pos = factory_class( + **self._initializer_args() + )._parse_with_pos() + self.target_nodes.append(ast) + logging.debug( + "Continue where previous parsing ended {token_pos}".format( + token_pos=self.token_pos + ) + ) + + def _parse(self): + self._parse_target_clause(self._operand_factory_class()) + while self._binop_factory_class().is_possible_start(self.get_next_token()): + self._parse_target_clause(self._binop_factory_class()) + if self._operand_factory_class().is_possible_start(self.get_next_token()): + self._parse_target_clause(self._operand_factory_class()) + else: + self.raise_unexpected_token() + + @abstractmethod + def _operand_factory_class(self): + """ + Get the Parser class of the Operands for the Binary operations/actions. + + Returns: + class: + """ + + @abstractmethod + def _binop_factory_class(self): + """ + Get a factory that gets the possible binary operation. + + Returns: + class: A class extending ExpressionParser + """ + + def _create_node(self): + """ + target_clauses has the nodes in order of encountering. Go through them forward and build the tree bottom up. + For simplicity docstring will use Operand Node rather than the specific node + + This way left-deep-descending traversal will process nodes in order. + + Continuing the example of an UpdateExpressionValue: + For example value => a + :val - :val2 + UpdateExpressionValue + / | \ + UpdateExpressionValue BinOp Operand + / | | | | + UpdateExpressionValue BinOp Operand - :val2 + / | | + Operand + :val + | + a + + self.target_nodes looks like: ( a >> + >> :val >> - >> :val2 ) + Returns: + moto.dynamodb2.ast_nodes.Node: Node of an AST representing the Expression as produced by the factory. + """ + if len(self.target_nodes) == 1: + return UpdateExpressionValue(children=[self.target_nodes.popleft()]) + else: + target_node = UpdateExpressionValue( + children=[ + self.target_nodes.popleft(), + self.target_nodes.popleft(), + self.target_nodes.popleft(), + ] + ) + while len(self.target_nodes) >= 2: + target_node = UpdateExpressionValue( + children=[ + target_node, + self.target_nodes.popleft(), + self.target_nodes.popleft(), + ] + ) + assert len(self.target_nodes) == 0 + return target_node + + +class UpdateExpressionParser(ExpressionParser, NestableExpressionParserMixin): + """ + Parser to create update expressions + """ + + @classmethod + def _sub_factories(cls): + return [ + UpdateExpressionSetClauseParser, + UpdateExpressionAddClauseParser, + UpdateExpressionDeleteClauseParser, + UpdateExpressionRemoveClauseParser, + ] + + @classmethod + def _is_possible_start(cls, token): + pass + + def __init__(self, *args, **kwargs): + super(UpdateExpressionParser, self).__init__(*args, **kwargs) + NestableExpressionParserMixin.__init__(self) + + @classmethod + def _nestable_class(cls): + return UpdateExpression + + def _parse_expression_clause(self, factory_class): + return self._parse_target_clause(factory_class) + + def _parse_by_a_subfactory(self): + for sub_factory in self._sub_factories(): + if sub_factory.is_possible_start(self.get_next_token()): + self._parse_expression_clause(sub_factory) + return True + return False + + def _parse(self): + """ + Update Expression is the top-most node therefore it is expected to end up at the end of the expression. + """ + while True: + self.skip_white_space() + if self.is_at_end(): + logging.debug("End reached") + break + elif self._parse_by_a_subfactory(): + continue + else: + self.raise_unexpected_token() + + return self._create_node(), self.token_pos + + @classmethod + def make(cls, expression_str): + token_list = ExpressionTokenizer.make_list(expression_str) + return cls(token_list).parse() + + +class UpdateExpressionSetClauseParser(ExpressionParser): + """ + UpdateExpressionSetClause => SET SetActions + """ + + @classmethod + def _is_possible_start(cls, token): + return token.type == Token.ATTRIBUTE and token.value.upper() == "SET" + + def _parse(self): + assert self.is_possible_start(self.get_next_token()) + self.goto_next_significant_token() + ast, self.token_pos = UpdateExpressionSetActionsParser( + **self._initializer_args() + )._parse_with_pos() + # noinspection PyProtectedMember + return UpdateExpressionSetClause(children=[ast]) + + +class UpdateExpressionActionsParser(ExpressionParser, NestableExpressionParserMixin): + """ + UpdateExpressionSetActions + """ + + def __init__(self, *args, **kwargs): + super(UpdateExpressionActionsParser, self).__init__(*args, **kwargs) + NestableExpressionParserMixin.__init__(self) + + @classmethod + def _is_possible_start(cls, token): + raise RuntimeError( + "{class_name} cannot be identified by the next token.".format( + class_name=cls._nestable_class().__name__ + ) + ) + + @classmethod + @abstractmethod + def _nestable_class(cls): + return UpdateExpressionSetActions + + @classmethod + @abstractmethod + def _nested_expression_parser_class(cls): + """Returns the parser for the query part that creates the nested nodes""" + + def _parse(self): + """ + UpdateExpressionSetActions is inside the expression so it can be followed by others. Process SetActions one by + one until no more SetAction. + """ + self.skip_white_space() + + while self._nested_expression_parser_class().is_possible_start( + self.get_next_token() + ): + self._parse_target_clause(self._nested_expression_parser_class()) + self.skip_white_space() + if self.get_next_token_type() == Token.COMMA: + self.goto_next_significant_token() + else: + break + + if len(self.target_clauses) == 0: + logging.debug( + "Didn't encounter a single {nc} in {nepc}.".format( + nc=self._nestable_class().__name__, + nepc=self._nested_expression_parser_class().__name__, + ) + ) + self.raise_unexpected_token() + + return self._create_node() + + +class UpdateExpressionSetActionsParser(UpdateExpressionActionsParser): + """ + UpdateExpressionSetActions + """ + + @classmethod + def _nested_expression_parser_class(cls): + return UpdateExpressionSetActionParser + + @classmethod + def _nestable_class(cls): + return UpdateExpressionSetActions + + +class UpdateExpressionSetActionParser(ExpressionParser): + """ + SetAction => Path = Value + + So we create an UpdateExpressionSetAction Node that has 2 children. Left child Path and right child Value. + """ + + @classmethod + def _is_possible_start(cls, token): + return UpdateExpressionPathParser.is_possible_start(token) + + def _parse(self): + """ + UpdateExpressionSetActionParser only gets called when expecting a SetAction. So we should be aggressive on + raising invalid Tokens. We can thus do the following: + 1) Process path + 2) skip whitespace if there are any + 3) Process equal-sign token + 4) skip whitespace if there are any + 3) Process value + + """ + path, self.token_pos = UpdateExpressionPathParser( + **self._initializer_args() + )._parse_with_pos() + self.skip_white_space() + self.process_token_of_type(Token.EQUAL_SIGN) + self.skip_white_space() + value, self.token_pos = UpdateExpressionValueParser( + **self._initializer_args() + )._parse_with_pos() + return UpdateExpressionSetAction(children=[path, value]) + + +class UpdateExpressionPathParser(ExpressionParser): + """ + Paths are selectors within items to specify a part within an Item. DynamoDB does not impose much restrictions on the + data it stores but it does store more strict restrictions on how they are represented in UpdateExpression's. + + """ + + def __init__(self, *args, **kwargs): + super(UpdateExpressionPathParser, self).__init__(*args, **kwargs) + self.path_nodes = [] + + @classmethod + def _is_possible_start(cls, token): + """ + Args: + token(Token): the token to be checked + + Returns: + bool: Whether the token could be the start of an UpdateExpressionPath + """ + if token.type == Token.ATTRIBUTE_NAME: + return True + elif token.type == Token.ATTRIBUTE and token.value.upper() != "REMOVE": + """We have to make sure remove is not passed""" + return True + return False + + def _parse(self): + return self.process_path() + + def process_path(self): + self.parse_path() + return UpdateExpressionPath(children=self.path_nodes) + + def parse_path(self): + """ + A path is comprised of: + - Attribute: the name of an attribute as how it is stored which has no special characters + - ATTRIBUTE_NAME: A placeholder that has no special characters except leading # to refer to attributes that + have a name that is not allowed in an UpdateExpression) + - DOT's: These are used to decent in a nested structure. When a DOT is in a path expression it is never part + of an attribute name but always means to descent into a MAP. We will call each descend a patch + chain + - SELECTORs: E.g.: [1] These are used to select an element in ordered datatypes like a list. + + Whitespaces can be between all these elements that build a path. For SELECTORs it is also allowed to have + whitespaces between brackets and numbers but the number cannot be split up with spaces + + Attributes and attribute_names must be separated with DOT's. + Returns: + UpdateExpressionPath: + """ + self.parse_path_chain() + while self.is_next_token_start_of_patch_chain(): + self.process_dot() + self.parse_path_chain() + + def is_next_token_start_of_patch_chain(self): + return self.get_next_token_type() == Token.DOT + + def process_dot(self): + self.path_nodes.append(ExpressionPathDescender()) + self.goto_next_significant_token() + + def parse_path_chain(self): + self.process_attribute_identifying_token() + self.skip_white_space() + while self.is_next_token_start_of_selector(): + self.process_selector() + self.skip_white_space() + + def process_attribute_identifying_token(self): + if self.get_next_token_type() == Token.ATTRIBUTE: + self.path_nodes.append(ExpressionAttribute(self.get_next_token_value())) + elif self.get_next_token_type() == Token.ATTRIBUTE_NAME: + self.path_nodes.append(ExpressionAttributeName(self.get_next_token_value())) + else: + self.raise_unexpected_token() + + self.goto_next_significant_token() + + def is_next_token_start_of_selector(self): + return self.get_next_token_type() == Token.OPEN_SQUARE_BRACKET + + def process_selector(self): + """ + Process the selector is only called when a selector must be processed. So do the following actions: + - skip opening bracket + - skip optional spaces + - read numeric literal + - skip optional spaces + - pass closing bracket + """ + self.process_token_of_type(Token.OPEN_SQUARE_BRACKET) + selector_value = self.process_token_of_type(Token.NUMBER) + self.process_token_of_type(Token.CLOSE_SQUARE_BRACKET) + self.path_nodes.append(ExpressionSelector(selector_value)) + + +class UpdateExpressionValueParser(NestableBinExpressionParser): + @classmethod + def _is_possible_start(cls, token): + return UpdateExpressionOperandParser.is_possible_start(token) + + def _operand_factory_class(self): + return UpdateExpressionOperandParser + + def _binop_factory_class(self): + return UpdateExpressionValueOperatorParser + + +class UpdateExpressionGroupedValueParser(ExpressionParser): + """ + A grouped value is an Update Expression value clause that is surrounded by round brackets. Each Operand can be + a grouped value by itself. + """ + + def _parse(self): + self.process_token_of_type(Token.OPEN_ROUND_BRACKET) + value, self.token_pos = UpdateExpressionValueParser( + **self._initializer_args() + )._parse_with_pos() + self.process_token_of_type(Token.CLOSE_ROUND_BRACKET) + return UpdateExpressionGroupedValue(children=value) + + @classmethod + def _is_possible_start(cls, token): + return token.type == Token.OPEN_ROUND_BRACKET + + +class UpdateExpressionValueOperatorParser(ExpressionParser): + OPERATION_TOKENS = [Token.PLUS_SIGN, Token.MINUS_SIGN] + + @classmethod + def _is_possible_start(cls, token): + return token.type in cls.OPERATION_TOKENS + + def _parse(self): + operation_value = self.get_next_token_value() + assert operation_value in self.OPERATION_TOKENS + self.goto_next_significant_token() + return ExpressionValueOperator(operation_value) + + +class UpdateExpressionOperandParser(ExpressionParser): + """ + Grammar + Operand* => AttributeValue + Operand* => UpdateExpressionFunction + Operand* => Path + Operand* => GroupedValue + """ + + @classmethod + def _sub_factories(cls): + return [ + UpdateExpressionAttributeValueParser, + UpdateExpressionFunctionParser, + UpdateExpressionPathParser, + UpdateExpressionGroupedValueParser, + ] + + @classmethod + def _is_possible_start(cls, token): + return any(parser.is_possible_start(token) for parser in cls._sub_factories()) + + def _parse(self): + for factory in self._sub_factories(): + if factory.is_possible_start(self.get_next_token()): + node, self.token_pos = factory( + **self._initializer_args() + )._parse_with_pos() + return node + self.raise_unexpected_token() + + +class UpdateExpressionAttributeValueParser(ExpressionParser): + def _parse(self): + attr_value = ExpressionAttributeValue( + self.process_token_of_type(Token.ATTRIBUTE_VALUE) + ) + return attr_value + + @classmethod + def _is_possible_start(cls, token): + return token.type == Token.ATTRIBUTE_VALUE + + +class UpdateExpressionFunctionParser(ExpressionParser): + """ + A helper to process a function of an Update Expression + """ + + # TODO(pbbouwel): Function names are supposedly case sensitive according to doc add tests + # Map function to the factories for its elements + FUNCTIONS = { + "if_not_exists": [UpdateExpressionPathParser, UpdateExpressionValueParser], + "list_append": [UpdateExpressionOperandParser, UpdateExpressionOperandParser], + } + + @classmethod + def _is_possible_start(cls, token): + """ + Check whether a token is supposed to be a function + Args: + token(Token): the token to check + + Returns: + bool: True if token is the start of a function. + """ + if token.type == Token.ATTRIBUTE: + return token.value in cls.FUNCTIONS.keys() + else: + return False + + def _parse(self): + function_name = self.get_next_token_value() + self.goto_next_significant_token() + self.process_token_of_type(Token.OPEN_ROUND_BRACKET) + function_elements = [function_name] + function_arguments = self.FUNCTIONS[function_name] + for i, func_elem_factory in enumerate(function_arguments): + func_elem, self.token_pos = func_elem_factory( + **self._initializer_args() + )._parse_with_pos() + function_elements.append(func_elem) + if i + 1 < len(function_arguments): + self.skip_white_space() + self.process_token_of_type(Token.COMMA) + self.process_token_of_type(Token.CLOSE_ROUND_BRACKET) + return UpdateExpressionFunction(children=function_elements) + + +class UpdateExpressionRemoveClauseParser(ExpressionParser): + """ + UpdateExpressionRemoveClause => REMOVE RemoveActions + """ + + def _parse(self): + assert self.is_possible_start(self.get_next_token()) + self.goto_next_significant_token() + ast, self.token_pos = UpdateExpressionRemoveActionsParser( + **self._initializer_args() + )._parse_with_pos() + # noinspection PyProtectedMember + return UpdateExpressionRemoveClause(children=[ast]) + + @classmethod + def _is_possible_start(cls, token): + """REMOVE is not a keyword""" + return token.type == Token.ATTRIBUTE and token.value.upper() == "REMOVE" + + +class UpdateExpressionRemoveActionsParser(UpdateExpressionActionsParser): + """ + UpdateExpressionSetActions + """ + + @classmethod + def _nested_expression_parser_class(cls): + return UpdateExpressionRemoveActionParser + + @classmethod + def _nestable_class(cls): + return UpdateExpressionRemoveActions + + +class UpdateExpressionRemoveActionParser(ExpressionParser): + """ + RemoveAction => Path = Value + + So we create an UpdateExpressionSetAction Node that has 2 children. Left child Path and right child Value. + """ + + @classmethod + def _is_possible_start(cls, token): + return UpdateExpressionPathParser.is_possible_start(token) + + def _parse(self): + """ + UpdateExpressionRemoveActionParser only gets called when expecting a RemoveAction. So we should be aggressive on + raising invalid Tokens. We can thus do the following: + 1) Process path + 2) skip whitespace if there are any + + """ + path, self.token_pos = UpdateExpressionPathParser( + **self._initializer_args() + )._parse_with_pos() + self.skip_white_space() + return UpdateExpressionRemoveAction(children=[path]) + + +class UpdateExpressionAddClauseParser(ExpressionParser): + def _parse(self): + assert self.is_possible_start(self.get_next_token()) + self.goto_next_significant_token() + ast, self.token_pos = UpdateExpressionAddActionsParser( + **self._initializer_args() + )._parse_with_pos() + # noinspection PyProtectedMember + return UpdateExpressionAddClause(children=[ast]) + + @classmethod + def _is_possible_start(cls, token): + return token.type == Token.ATTRIBUTE and token.value.upper() == "ADD" + + +class UpdateExpressionAddActionsParser(UpdateExpressionActionsParser): + """ + UpdateExpressionSetActions + """ + + @classmethod + def _nested_expression_parser_class(cls): + return UpdateExpressionAddActionParser + + @classmethod + def _nestable_class(cls): + return UpdateExpressionAddActions + + +@six.add_metaclass(abc.ABCMeta) +class UpdateExpressionPathValueParser(ExpressionParser): + def _parse_path_and_value(self): + """ + UpdateExpressionAddActionParser only gets called when expecting an AddAction. So we should be aggressive on + raising invalid Tokens. We can thus do the following: + 1) Process path + 2) skip whitespace if there are any + 3) Process a value + 4) skip whitespace if there are any + + Returns: + [path, value]: A list containing the Path node and the AttributeValue nodes + """ + path, self.token_pos = UpdateExpressionPathParser( + **self._initializer_args() + )._parse_with_pos() + self.skip_white_space() + value, self.token_pos = UpdateExpressionAttributeValueParser( + **self._initializer_args() + )._parse_with_pos() + self.skip_white_space() + return [path, value] + + +class UpdateExpressionAddActionParser(UpdateExpressionPathValueParser): + @classmethod + def _is_possible_start(cls, token): + return UpdateExpressionPathParser.is_possible_start(token) + + def _parse(self): + return UpdateExpressionAddAction(children=self._parse_path_and_value()) + + +class UpdateExpressionDeleteClauseParser(ExpressionParser): + def _parse(self): + assert self.is_possible_start(self.get_next_token()) + self.goto_next_significant_token() + ast, self.token_pos = UpdateExpressionDeleteActionsParser( + **self._initializer_args() + )._parse_with_pos() + # noinspection PyProtectedMember + return UpdateExpressionDeleteClause(children=[ast]) + + @classmethod + def _is_possible_start(cls, token): + return token.type == Token.ATTRIBUTE and token.value.upper() == "DELETE" + + +class UpdateExpressionDeleteActionsParser(UpdateExpressionActionsParser): + """ + UpdateExpressionSetActions + """ + + @classmethod + def _nested_expression_parser_class(cls): + return UpdateExpressionDeleteActionParser + + @classmethod + def _nestable_class(cls): + return UpdateExpressionDeleteActions + + +class UpdateExpressionDeleteActionParser(UpdateExpressionPathValueParser): + @classmethod + def _is_possible_start(cls, token): + return UpdateExpressionPathParser.is_possible_start(token) + + def _parse(self): + return UpdateExpressionDeleteAction(children=self._parse_path_and_value()) diff --git a/moto/dynamodb2/parsing/reserved_keywords.py b/moto/dynamodb2/parsing/reserved_keywords.py new file mode 100644 index 000000000..d82b16e98 --- /dev/null +++ b/moto/dynamodb2/parsing/reserved_keywords.py @@ -0,0 +1,29 @@ +class ReservedKeywords(list): + """ + DynamoDB has an extensive list of keywords. Keywords are considered when validating the expression Tree. + Not earlier since an update expression like "SET path = VALUE 1" fails with: + 'Invalid UpdateExpression: Syntax error; token: "1", near: "VALUE 1"' + """ + + KEYWORDS = None + + @classmethod + def get_reserved_keywords(cls): + if cls.KEYWORDS is None: + cls.KEYWORDS = cls._get_reserved_keywords() + return cls.KEYWORDS + + @classmethod + def _get_reserved_keywords(cls): + """ + Get a list of reserved keywords of DynamoDB + """ + try: + import importlib.resources as pkg_resources + except ImportError: + import importlib_resources as pkg_resources + + reserved_keywords = pkg_resources.read_text( + "moto.dynamodb2.parsing", "reserved_keywords.txt" + ) + return reserved_keywords.split() diff --git a/moto/dynamodb2/parsing/reserved_keywords.txt b/moto/dynamodb2/parsing/reserved_keywords.txt new file mode 100644 index 000000000..7c0106127 --- /dev/null +++ b/moto/dynamodb2/parsing/reserved_keywords.txt @@ -0,0 +1,573 @@ +ABORT +ABSOLUTE +ACTION +ADD +AFTER +AGENT +AGGREGATE +ALL +ALLOCATE +ALTER +ANALYZE +AND +ANY +ARCHIVE +ARE +ARRAY +AS +ASC +ASCII +ASENSITIVE +ASSERTION +ASYMMETRIC +AT +ATOMIC +ATTACH +ATTRIBUTE +AUTH +AUTHORIZATION +AUTHORIZE +AUTO +AVG +BACK +BACKUP +BASE +BATCH +BEFORE +BEGIN +BETWEEN +BIGINT +BINARY +BIT +BLOB +BLOCK +BOOLEAN +BOTH +BREADTH +BUCKET +BULK +BY +BYTE +CALL +CALLED +CALLING +CAPACITY +CASCADE +CASCADED +CASE +CAST +CATALOG +CHAR +CHARACTER +CHECK +CLASS +CLOB +CLOSE +CLUSTER +CLUSTERED +CLUSTERING +CLUSTERS +COALESCE +COLLATE +COLLATION +COLLECTION +COLUMN +COLUMNS +COMBINE +COMMENT +COMMIT +COMPACT +COMPILE +COMPRESS +CONDITION +CONFLICT +CONNECT +CONNECTION +CONSISTENCY +CONSISTENT +CONSTRAINT +CONSTRAINTS +CONSTRUCTOR +CONSUMED +CONTINUE +CONVERT +COPY +CORRESPONDING +COUNT +COUNTER +CREATE +CROSS +CUBE +CURRENT +CURSOR +CYCLE +DATA +DATABASE +DATE +DATETIME +DAY +DEALLOCATE +DEC +DECIMAL +DECLARE +DEFAULT +DEFERRABLE +DEFERRED +DEFINE +DEFINED +DEFINITION +DELETE +DELIMITED +DEPTH +DEREF +DESC +DESCRIBE +DESCRIPTOR +DETACH +DETERMINISTIC +DIAGNOSTICS +DIRECTORIES +DISABLE +DISCONNECT +DISTINCT +DISTRIBUTE +DO +DOMAIN +DOUBLE +DROP +DUMP +DURATION +DYNAMIC +EACH +ELEMENT +ELSE +ELSEIF +EMPTY +ENABLE +END +EQUAL +EQUALS +ERROR +ESCAPE +ESCAPED +EVAL +EVALUATE +EXCEEDED +EXCEPT +EXCEPTION +EXCEPTIONS +EXCLUSIVE +EXEC +EXECUTE +EXISTS +EXIT +EXPLAIN +EXPLODE +EXPORT +EXPRESSION +EXTENDED +EXTERNAL +EXTRACT +FAIL +FALSE +FAMILY +FETCH +FIELDS +FILE +FILTER +FILTERING +FINAL +FINISH +FIRST +FIXED +FLATTERN +FLOAT +FOR +FORCE +FOREIGN +FORMAT +FORWARD +FOUND +FREE +FROM +FULL +FUNCTION +FUNCTIONS +GENERAL +GENERATE +GET +GLOB +GLOBAL +GO +GOTO +GRANT +GREATER +GROUP +GROUPING +HANDLER +HASH +HAVE +HAVING +HEAP +HIDDEN +HOLD +HOUR +IDENTIFIED +IDENTITY +IF +IGNORE +IMMEDIATE +IMPORT +IN +INCLUDING +INCLUSIVE +INCREMENT +INCREMENTAL +INDEX +INDEXED +INDEXES +INDICATOR +INFINITE +INITIALLY +INLINE +INNER +INNTER +INOUT +INPUT +INSENSITIVE +INSERT +INSTEAD +INT +INTEGER +INTERSECT +INTERVAL +INTO +INVALIDATE +IS +ISOLATION +ITEM +ITEMS +ITERATE +JOIN +KEY +KEYS +LAG +LANGUAGE +LARGE +LAST +LATERAL +LEAD +LEADING +LEAVE +LEFT +LENGTH +LESS +LEVEL +LIKE +LIMIT +LIMITED +LINES +LIST +LOAD +LOCAL +LOCALTIME +LOCALTIMESTAMP +LOCATION +LOCATOR +LOCK +LOCKS +LOG +LOGED +LONG +LOOP +LOWER +MAP +MATCH +MATERIALIZED +MAX +MAXLEN +MEMBER +MERGE +METHOD +METRICS +MIN +MINUS +MINUTE +MISSING +MOD +MODE +MODIFIES +MODIFY +MODULE +MONTH +MULTI +MULTISET +NAME +NAMES +NATIONAL +NATURAL +NCHAR +NCLOB +NEW +NEXT +NO +NONE +NOT +NULL +NULLIF +NUMBER +NUMERIC +OBJECT +OF +OFFLINE +OFFSET +OLD +ON +ONLINE +ONLY +OPAQUE +OPEN +OPERATOR +OPTION +OR +ORDER +ORDINALITY +OTHER +OTHERS +OUT +OUTER +OUTPUT +OVER +OVERLAPS +OVERRIDE +OWNER +PAD +PARALLEL +PARAMETER +PARAMETERS +PARTIAL +PARTITION +PARTITIONED +PARTITIONS +PATH +PERCENT +PERCENTILE +PERMISSION +PERMISSIONS +PIPE +PIPELINED +PLAN +POOL +POSITION +PRECISION +PREPARE +PRESERVE +PRIMARY +PRIOR +PRIVATE +PRIVILEGES +PROCEDURE +PROCESSED +PROJECT +PROJECTION +PROPERTY +PROVISIONING +PUBLIC +PUT +QUERY +QUIT +QUORUM +RAISE +RANDOM +RANGE +RANK +RAW +READ +READS +REAL +REBUILD +RECORD +RECURSIVE +REDUCE +REF +REFERENCE +REFERENCES +REFERENCING +REGEXP +REGION +REINDEX +RELATIVE +RELEASE +REMAINDER +RENAME +REPEAT +REPLACE +REQUEST +RESET +RESIGNAL +RESOURCE +RESPONSE +RESTORE +RESTRICT +RESULT +RETURN +RETURNING +RETURNS +REVERSE +REVOKE +RIGHT +ROLE +ROLES +ROLLBACK +ROLLUP +ROUTINE +ROW +ROWS +RULE +RULES +SAMPLE +SATISFIES +SAVE +SAVEPOINT +SCAN +SCHEMA +SCOPE +SCROLL +SEARCH +SECOND +SECTION +SEGMENT +SEGMENTS +SELECT +SELF +SEMI +SENSITIVE +SEPARATE +SEQUENCE +SERIALIZABLE +SESSION +SET +SETS +SHARD +SHARE +SHARED +SHORT +SHOW +SIGNAL +SIMILAR +SIZE +SKEWED +SMALLINT +SNAPSHOT +SOME +SOURCE +SPACE +SPACES +SPARSE +SPECIFIC +SPECIFICTYPE +SPLIT +SQL +SQLCODE +SQLERROR +SQLEXCEPTION +SQLSTATE +SQLWARNING +START +STATE +STATIC +STATUS +STORAGE +STORE +STORED +STREAM +STRING +STRUCT +STYLE +SUB +SUBMULTISET +SUBPARTITION +SUBSTRING +SUBTYPE +SUM +SUPER +SYMMETRIC +SYNONYM +SYSTEM +TABLE +TABLESAMPLE +TEMP +TEMPORARY +TERMINATED +TEXT +THAN +THEN +THROUGHPUT +TIME +TIMESTAMP +TIMEZONE +TINYINT +TO +TOKEN +TOTAL +TOUCH +TRAILING +TRANSACTION +TRANSFORM +TRANSLATE +TRANSLATION +TREAT +TRIGGER +TRIM +TRUE +TRUNCATE +TTL +TUPLE +TYPE +UNDER +UNDO +UNION +UNIQUE +UNIT +UNKNOWN +UNLOGGED +UNNEST +UNPROCESSED +UNSIGNED +UNTIL +UPDATE +UPPER +URL +USAGE +USE +USER +USERS +USING +UUID +VACUUM +VALUE +VALUED +VALUES +VARCHAR +VARIABLE +VARIANCE +VARINT +VARYING +VIEW +VIEWS +VIRTUAL +VOID +WAIT +WHEN +WHENEVER +WHERE +WHILE +WINDOW +WITH +WITHIN +WITHOUT +WORK +WRAPPED +WRITE +YEAR +ZONE diff --git a/moto/dynamodb2/parsing/tokens.py b/moto/dynamodb2/parsing/tokens.py index 07d65ae64..4fbb7883a 100644 --- a/moto/dynamodb2/parsing/tokens.py +++ b/moto/dynamodb2/parsing/tokens.py @@ -1,4 +1,5 @@ import re +import sys from moto.dynamodb2.exceptions import ( InvalidTokenException, @@ -147,9 +148,17 @@ class ExpressionTokenizer(object): self.token_list = [] self.staged_characters = "" + @classmethod + def is_py2(cls): + return sys.version_info[0] == 2 + @classmethod def make_list(cls, input_expression_str): - assert isinstance(input_expression_str, str) + if cls.is_py2(): + pass + else: + assert isinstance(input_expression_str, str) + return ExpressionTokenizer(input_expression_str)._make_list() def add_token(self, token_type, token_value): @@ -159,6 +168,10 @@ class ExpressionTokenizer(object): self.add_token(token_type, self.staged_characters) self.staged_characters = "" + @classmethod + def is_numeric(cls, input_str): + return re.compile("[0-9]+").match(input_str) is not None + def process_staged_characters(self): if len(self.staged_characters) == 0: return @@ -167,7 +180,7 @@ class ExpressionTokenizer(object): self.add_token_from_stage(Token.ATTRIBUTE_NAME) else: raise InvalidExpressionAttributeNameKey(self.staged_characters) - elif self.staged_characters.isnumeric(): + elif self.is_numeric(self.staged_characters): self.add_token_from_stage(Token.NUMBER) elif self.is_expression_attribute(self.staged_characters): self.add_token_from_stage(Token.ATTRIBUTE) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index d21d1d756..a5aeeac70 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -748,11 +748,6 @@ class DynamoHandler(BaseResponse): expression_attribute_names = self.body.get("ExpressionAttributeNames", {}) expression_attribute_values = self.body.get("ExpressionAttributeValues", {}) - # Support spaces between operators in an update expression - # E.g. `a = b + c` -> `a=b+c` - if update_expression: - update_expression = re.sub(r"\s*([=\+-])\s*", "\\1", update_expression) - try: item = self.dynamodb_backend.update_item( name, diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index bec24c966..09401d562 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals, print_function +import re from decimal import Decimal import six @@ -4177,3 +4178,70 @@ def test_gsi_verify_negative_number_order(): [float(item["gsiK1SortKey"]) for item in resp["Items"]].should.equal( [-0.7, -0.6, 0.7] ) + + +def assert_raise_syntax_error(client_error, token, near): + """ + Assert whether a client_error is as expected Syntax error. Syntax error looks like: `syntax_error_template` + + Args: + client_error(ClientError): The ClientError exception that was raised + token(str): The token that ws unexpected + near(str): The part in the expression that shows where the error occurs it generally has the preceding token the + optional separation and the problematic token. + """ + syntax_error_template = ( + 'Invalid UpdateExpression: Syntax error; token: "{token}", near: "{near}"' + ) + expected_syntax_error = syntax_error_template.format(token=token, near=near) + assert client_error.response["Error"]["Code"] == "ValidationException" + assert expected_syntax_error == client_error.response["Error"]["Message"] + + +@mock_dynamodb2 +def test_update_expression_with_numeric_literal_instead_of_value(): + """ + DynamoDB requires literals to be passed in as values. If they are put literally in the expression a token error will + be raised + """ + dynamodb = boto3.client("dynamodb", region_name="eu-west-1") + + dynamodb.create_table( + TableName="moto-test", + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ) + + try: + dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "1"}}, + UpdateExpression="SET MyStr = myNum + 1", + ) + assert False, "Validation exception not thrown" + except dynamodb.exceptions.ClientError as e: + assert_raise_syntax_error(e, "1", "+ 1") + + +@mock_dynamodb2 +def test_update_expression_with_multiple_set_clauses_must_be_comma_separated(): + """ + An UpdateExpression can have multiple set clauses but if they are passed in without the separating comma. + """ + dynamodb = boto3.client("dynamodb", region_name="eu-west-1") + + dynamodb.create_table( + TableName="moto-test", + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ) + + try: + dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "1"}}, + UpdateExpression="SET MyStr = myNum Mystr2 myNum2", + ) + assert False, "Validation exception not thrown" + except dynamodb.exceptions.ClientError as e: + assert_raise_syntax_error(e, "Mystr2", "myNum Mystr2 myNum2") diff --git a/tests/test_dynamodb2/test_dynamodb_expressions.py b/tests/test_dynamodb2/test_dynamodb_expressions.py new file mode 100644 index 000000000..1066231af --- /dev/null +++ b/tests/test_dynamodb2/test_dynamodb_expressions.py @@ -0,0 +1,395 @@ +from moto.dynamodb2.exceptions import InvalidTokenException +from moto.dynamodb2.parsing.expressions import UpdateExpressionParser +from moto.dynamodb2.parsing.reserved_keywords import ReservedKeywords + + +def test_get_reserved_keywords(): + reserved_keywords = ReservedKeywords.get_reserved_keywords() + assert "SET" in reserved_keywords + assert "DELETE" in reserved_keywords + assert "ADD" in reserved_keywords + # REMOVE is not part of the list of reserved keywords. + assert "REMOVE" not in reserved_keywords + + +def test_update_expression_numeric_literal_in_expression(): + set_action = "SET attrName = 3" + try: + UpdateExpressionParser.make(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "3" + assert te.near == "= 3" + + +def test_expression_tokenizer_multi_number_numeric_literal_in_expression(): + set_action = "SET attrName = 34" + try: + UpdateExpressionParser.make(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "34" + assert te.near == "= 34" + + +def test_expression_tokenizer_numeric_literal_unclosed_square_bracket(): + set_action = "SET MyStr[ 3" + try: + UpdateExpressionParser.make(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "" + assert te.near == "3" + + +def test_expression_tokenizer_wrong_closing_bracket_with_space(): + set_action = "SET MyStr[3 )" + try: + UpdateExpressionParser.make(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == ")" + assert te.near == "3 )" + + +def test_expression_tokenizer_wrong_closing_bracket(): + set_action = "SET MyStr[3)" + try: + UpdateExpressionParser.make(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == ")" + assert te.near == "3)" + + +def test_expression_tokenizer_only_numeric_literal_for_set(): + set_action = "SET 2" + try: + UpdateExpressionParser.make(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "2" + assert te.near == "SET 2" + + +def test_expression_tokenizer_only_numeric_literal(): + set_action = "2" + try: + UpdateExpressionParser.make(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "2" + assert te.near == "2" + + +def test_expression_tokenizer_set_closing_round_bracket(): + set_action = "SET )" + try: + UpdateExpressionParser.make(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == ")" + assert te.near == "SET )" + + +def test_expression_tokenizer_set_closing_followed_by_numeric_literal(): + set_action = "SET ) 3" + try: + UpdateExpressionParser.make(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == ")" + assert te.near == "SET ) 3" + + +def test_expression_tokenizer_numeric_literal_unclosed_square_bracket_trailing_space(): + set_action = "SET MyStr[ 3 " + try: + UpdateExpressionParser.make(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "" + assert te.near == "3 " + + +def test_expression_tokenizer_unbalanced_round_brackets_only_opening(): + set_action = "SET MyStr = (:_val" + try: + UpdateExpressionParser.make(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "" + assert te.near == ":_val" + + +def test_expression_tokenizer_unbalanced_round_brackets_only_opening_trailing_space(): + set_action = "SET MyStr = (:_val " + try: + UpdateExpressionParser.make(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "" + assert te.near == ":_val " + + +def test_expression_tokenizer_unbalanced_square_brackets_only_opening(): + set_action = "SET MyStr = [:_val" + try: + UpdateExpressionParser.make(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "[" + assert te.near == "= [:_val" + + +def test_expression_tokenizer_unbalanced_square_brackets_only_opening_trailing_spaces(): + set_action = "SET MyStr = [:_val " + try: + UpdateExpressionParser.make(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "[" + assert te.near == "= [:_val" + + +def test_expression_tokenizer_unbalanced_round_brackets_multiple_opening(): + set_action = "SET MyStr = (:_val + (:val2" + try: + UpdateExpressionParser.make(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "" + assert te.near == ":val2" + + +def test_expression_tokenizer_unbalanced_round_brackets_only_closing(): + set_action = "SET MyStr = ):_val" + try: + UpdateExpressionParser.make(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == ")" + assert te.near == "= ):_val" + + +def test_expression_tokenizer_unbalanced_square_brackets_only_closing(): + set_action = "SET MyStr = ]:_val" + try: + UpdateExpressionParser.make(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "]" + assert te.near == "= ]:_val" + + +def test_expression_tokenizer_unbalanced_round_brackets_only_closing_followed_by_other_parts(): + set_action = "SET MyStr = ):_val + :val2" + try: + UpdateExpressionParser.make(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == ")" + assert te.near == "= ):_val" + + +def test_update_expression_starts_with_keyword_reset_followed_by_identifier(): + update_expression = "RESET NonExistent" + try: + UpdateExpressionParser.make(update_expression) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "RESET" + assert te.near == "RESET NonExistent" + + +def test_update_expression_starts_with_keyword_reset_followed_by_identifier_and_value(): + update_expression = "RESET NonExistent value" + try: + UpdateExpressionParser.make(update_expression) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "RESET" + assert te.near == "RESET NonExistent" + + +def test_update_expression_starts_with_leading_spaces_and_keyword_reset_followed_by_identifier_and_value(): + update_expression = " RESET NonExistent value" + try: + UpdateExpressionParser.make(update_expression) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "RESET" + assert te.near == " RESET NonExistent" + + +def test_update_expression_with_only_keyword_reset(): + update_expression = "RESET" + try: + UpdateExpressionParser.make(update_expression) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "RESET" + assert te.near == "RESET" + + +def test_update_nested_expression_with_selector_just_should_fail_parsing_at_numeric_literal_value(): + update_expression = "SET a[0].b = 5" + try: + UpdateExpressionParser.make(update_expression) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "5" + assert te.near == "= 5" + + +def test_update_nested_expression_with_selector_and_spaces_should_only_fail_parsing_at_numeric_literal_value(): + update_expression = "SET a [ 2 ]. b = 5" + try: + UpdateExpressionParser.make(update_expression) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "5" + assert te.near == "= 5" + + +def test_update_nested_expression_with_double_selector_and_spaces_should_only_fail_parsing_at_numeric_literal_value(): + update_expression = "SET a [2][ 3 ]. b = 5" + try: + UpdateExpressionParser.make(update_expression) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "5" + assert te.near == "= 5" + + +def test_update_nested_expression_should_only_fail_parsing_at_numeric_literal_value(): + update_expression = "SET a . b = 5" + try: + UpdateExpressionParser.make(update_expression) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "5" + assert te.near == "= 5" + + +def test_nested_selectors_in_update_expression_should_fail_at_nesting(): + update_expression = "SET a [ [2] ]. b = 5" + try: + UpdateExpressionParser.make(update_expression) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "[" + assert te.near == "[ [2" + + +def test_update_expression_number_in_selector_cannot_be_splite(): + update_expression = "SET a [2 1]. b = 5" + try: + UpdateExpressionParser.make(update_expression) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "1" + assert te.near == "2 1]" + + +def test_update_expression_cannot_have_successive_attributes(): + update_expression = "SET #a a = 5" + try: + UpdateExpressionParser.make(update_expression) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "a" + assert te.near == "#a a =" + + +def test_update_expression_path_with_both_attribute_and_attribute_name_should_only_fail_at_numeric_value(): + update_expression = "SET #a.a = 5" + try: + UpdateExpressionParser.make(update_expression) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "5" + assert te.near == "= 5" + + +def test_expression_tokenizer_2_same_operators_back_to_back(): + set_action = "SET MyStr = NoExist + + :_val " + try: + UpdateExpressionParser.make(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "+" + assert te.near == "+ + :_val" + + +def test_expression_tokenizer_2_different_operators_back_to_back(): + set_action = "SET MyStr = NoExist + - :_val " + try: + UpdateExpressionParser.make(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "-" + assert te.near == "+ - :_val" + + +def test_update_expression_remove_does_not_allow_operations(): + remove_action = "REMOVE NoExist + " + try: + UpdateExpressionParser.make(remove_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "+" + assert te.near == "NoExist + " + + +def test_update_expression_add_does_not_allow_attribute_after_path(): + """value here is not really a value since a value starts with a colon (:)""" + add_expr = "ADD attr val foobar" + try: + UpdateExpressionParser.make(add_expr) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "val" + assert te.near == "attr val foobar" + + +def test_update_expression_add_does_not_allow_attribute_foobar_after_value(): + add_expr = "ADD attr :val foobar" + try: + UpdateExpressionParser.make(add_expr) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "foobar" + assert te.near == ":val foobar" + + +def test_update_expression_delete_does_not_allow_attribute_after_path(): + """value here is not really a value since a value starts with a colon (:)""" + delete_expr = "DELETE attr val" + try: + UpdateExpressionParser.make(delete_expr) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "val" + assert te.near == "attr val" + + +def test_update_expression_delete_does_not_allow_attribute_foobar_after_value(): + delete_expr = "DELETE attr :val foobar" + try: + UpdateExpressionParser.make(delete_expr) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "foobar" + assert te.near == ":val foobar" + + +def test_update_expression_parsing_is_not_keyword_aware(): + """path and VALUE are keywords. Yet a token error will be thrown for the numeric literal 1.""" + delete_expr = "SET path = VALUE 1" + try: + UpdateExpressionParser.make(delete_expr) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "1" + assert te.near == "VALUE 1" diff --git a/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py b/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py index c433a3a31..1aa2175c1 100644 --- a/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py +++ b/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py @@ -1254,14 +1254,22 @@ def test_update_item_with_expression(): item_key = {"forum_name": "the-key", "subject": "123"} - table.update_item(Key=item_key, UpdateExpression="SET field=2") + table.update_item( + Key=item_key, + UpdateExpression="SET field = :field_value", + ExpressionAttributeValues={":field_value": 2}, + ) dict(table.get_item(Key=item_key)["Item"]).should.equal( - {"field": "2", "forum_name": "the-key", "subject": "123"} + {"field": Decimal("2"), "forum_name": "the-key", "subject": "123"} ) - table.update_item(Key=item_key, UpdateExpression="SET field = 3") + table.update_item( + Key=item_key, + UpdateExpression="SET field = :field_value", + ExpressionAttributeValues={":field_value": 3}, + ) dict(table.get_item(Key=item_key)["Item"]).should.equal( - {"field": "3", "forum_name": "the-key", "subject": "123"} + {"field": Decimal("3"), "forum_name": "the-key", "subject": "123"} ) From 891801d5697f80ab44afe8a20e5896e8807237b6 Mon Sep 17 00:00:00 2001 From: Bob Wombat Hogg Date: Sat, 18 Apr 2020 07:46:28 -0400 Subject: [PATCH 57/83] Use ISO 8601 format for ELB DescribeLoadBalancers --- moto/elb/models.py | 5 ++++- moto/elb/responses.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/moto/elb/models.py b/moto/elb/models.py index f77811623..4991b0754 100644 --- a/moto/elb/models.py +++ b/moto/elb/models.py @@ -1,6 +1,9 @@ from __future__ import unicode_literals import datetime + +import pytz + from boto.ec2.elb.attributes import ( LbAttributes, ConnectionSettingAttribute, @@ -83,7 +86,7 @@ class FakeLoadBalancer(BaseModel): self.zones = zones self.listeners = [] self.backends = [] - self.created_time = datetime.datetime.now() + self.created_time = datetime.datetime.now(pytz.utc) self.scheme = scheme self.attributes = FakeLoadBalancer.get_default_attributes() self.policies = Policies() diff --git a/moto/elb/responses.py b/moto/elb/responses.py index de21f23e7..79db5a788 100644 --- a/moto/elb/responses.py +++ b/moto/elb/responses.py @@ -442,7 +442,7 @@ DESCRIBE_LOAD_BALANCERS_TEMPLATE = """= 1 + return self.children[n] + + +class DDBTypedValue(Node): + """ + A node representing a DDBTyped value. This can be any structure as supported by DyanmoDB. The node only has 1 child + which is the value of type `DynamoType`. + """ + + def __init__(self, value): + assert isinstance(value, DynamoType), "DDBTypedValue must be of DynamoType" + super(DDBTypedValue, self).__init__(children=[value]) + + def get_value(self): + return self.children[0] + + +class NoneExistingPath(LeafNode): + """A placeholder for Paths that did not exist in the Item.""" + + def __init__(self, creatable=False): + super(NoneExistingPath, self).__init__(children=[creatable]) + + def is_creatable(self): + """Can this path be created if need be. For example path creating element in a dictionary or creating a new + attribute under root level of an item.""" + return self.children[0] + + +class DepthFirstTraverser(object): + """ + Helper class that allows depth first traversal and to implement custom processing for certain AST nodes. The + processor of a node must return the new resulting node. This node will be placed in the tree. Processing of a + node using this traverser should therefore only transform child nodes. The returned node will get the same parent + as the node before processing had. + """ + + @abstractmethod + def _processing_map(self): + """ + A map providing a processing function per node class type to a function that takes in a Node object and + processes it. A Node can only be processed by a single function and they are considered in order. Therefore if + multiple classes from a single class hierarchy strain are used the more specific classes have to be put before + the less specific ones. That requires overriding `nodes_to_be_processed`. If no multiple classes form a single + class hierarchy strain are used the default implementation of `nodes_to_be_processed` should be OK. + Returns: + dict: Mapping a Node Class to a processing function. + """ + pass + + def nodes_to_be_processed(self): + """Cached accessor for getting Node types that need to be processed.""" + return tuple(k for k in self._processing_map().keys()) + + def process(self, node): + """Process a Node""" + for class_key, processor in self._processing_map().items(): + if isinstance(node, class_key): + return processor(node) + + def pre_processing_of_child(self, parent_node, child_id): + """Hook that is called pre-processing of the child at position `child_id`""" + pass + + def traverse_node_recursively(self, node, child_id=-1): + """ + Traverse nodes depth first processing nodes bottom up (if root node is considered the top). + + Args: + node(Node): The node which is the last node to be processed but which allows to identify all the + work (which is in the children) + child_id(int): The index in the list of children from the parent that this node corresponds to + + Returns: + Node: The node of the new processed AST + """ + if isinstance(node, Node): + parent_node = node.parent + if node.children is not None: + for i, child_node in enumerate(node.children): + self.pre_processing_of_child(node, i) + self.traverse_node_recursively(child_node, i) + # noinspection PyTypeChecker + if isinstance(node, self.nodes_to_be_processed()): + node = self.process(node) + node.parent = parent_node + parent_node.children[child_id] = node + return node + + def traverse(self, node): + return self.traverse_node_recursively(node) + + +class NodeDepthLeftTypeFetcher(object): + """Helper class to fetch a node of a specific type. Depth left-first traversal""" + + def __init__(self, node_type, root_node): + assert issubclass(node_type, Node) + self.node_type = node_type + self.root_node = root_node + self.queue = deque() + self.add_nodes_left_to_right_depth_first(self.root_node) + + def add_nodes_left_to_right_depth_first(self, node): + if isinstance(node, Node) and node.children is not None: + for child_node in node.children: + self.add_nodes_left_to_right_depth_first(child_node) + self.queue.append(child_node) + self.queue.append(node) + + def __iter__(self): + return self + + def next(self): + return self.__next__() + + def __next__(self): + while len(self.queue) > 0: + candidate = self.queue.popleft() + if isinstance(candidate, self.node_type): + return candidate + else: + raise StopIteration diff --git a/moto/dynamodb2/parsing/expressions.py b/moto/dynamodb2/parsing/expressions.py index e418bb47e..4c1d42a55 100644 --- a/moto/dynamodb2/parsing/expressions.py +++ b/moto/dynamodb2/parsing/expressions.py @@ -29,7 +29,7 @@ from moto.dynamodb2.parsing.ast_nodes import ( UpdateExpressionDeleteActions, UpdateExpressionDeleteClause, ) -from moto.dynamodb2.exceptions import InvalidTokenException +from moto.dynamodb2.exceptions import InvalidTokenException, InvalidUpdateExpression from moto.dynamodb2.parsing.tokens import Token, ExpressionTokenizer @@ -371,6 +371,7 @@ class NestableBinExpressionParser(ExpressionParser): self._parse_target_clause(self._operand_factory_class()) else: self.raise_unexpected_token() + return self._create_node() @abstractmethod def _operand_factory_class(self): @@ -485,7 +486,7 @@ class UpdateExpressionParser(ExpressionParser, NestableExpressionParserMixin): else: self.raise_unexpected_token() - return self._create_node(), self.token_pos + return self._create_node() @classmethod def make(cls, expression_str): @@ -804,15 +805,41 @@ class UpdateExpressionAttributeValueParser(ExpressionParser): return token.type == Token.ATTRIBUTE_VALUE +class UpdateExpressionAttributeValueOrPathParser(ExpressionParser): + def _parse(self): + if UpdateExpressionAttributeValueParser.is_possible_start( + self.get_next_token() + ): + token, self.token_pos = UpdateExpressionAttributeValueParser( + **self._initializer_args() + )._parse_with_pos() + else: + token, self.token_pos = UpdateExpressionPathParser( + **self._initializer_args() + )._parse_with_pos() + return token + + @classmethod + def _is_possible_start(cls, token): + return any( + [ + UpdateExpressionAttributeValueParser.is_possible_start(token), + UpdateExpressionPathParser.is_possible_start(token), + ] + ) + + class UpdateExpressionFunctionParser(ExpressionParser): """ A helper to process a function of an Update Expression """ - # TODO(pbbouwel): Function names are supposedly case sensitive according to doc add tests # Map function to the factories for its elements FUNCTIONS = { - "if_not_exists": [UpdateExpressionPathParser, UpdateExpressionValueParser], + "if_not_exists": [ + UpdateExpressionPathParser, + UpdateExpressionAttributeValueOrPathParser, + ], "list_append": [UpdateExpressionOperandParser, UpdateExpressionOperandParser], } @@ -833,6 +860,9 @@ class UpdateExpressionFunctionParser(ExpressionParser): def _parse(self): function_name = self.get_next_token_value() + if function_name not in self.FUNCTIONS.keys(): + # Function names are case sensitive + raise InvalidUpdateExpression(function_name) self.goto_next_significant_token() self.process_token_of_type(Token.OPEN_ROUND_BRACKET) function_elements = [function_name] diff --git a/moto/dynamodb2/parsing/validators.py b/moto/dynamodb2/parsing/validators.py new file mode 100644 index 000000000..180c7a874 --- /dev/null +++ b/moto/dynamodb2/parsing/validators.py @@ -0,0 +1,341 @@ +""" +See docstring class Validator below for more details on validation +""" +from abc import abstractmethod +from copy import deepcopy + +from moto.dynamodb2.exceptions import ( + AttributeIsReservedKeyword, + ExpressionAttributeValueNotDefined, + AttributeDoesNotExist, + ExpressionAttributeNameNotDefined, + IncorrectOperandType, + InvalidUpdateExpressionInvalidDocumentPath, +) +from moto.dynamodb2.models import DynamoType +from moto.dynamodb2.parsing.ast_nodes import ( + ExpressionAttribute, + UpdateExpressionPath, + UpdateExpressionSetAction, + UpdateExpressionAddAction, + UpdateExpressionDeleteAction, + UpdateExpressionRemoveAction, + DDBTypedValue, + ExpressionAttributeValue, + ExpressionAttributeName, + DepthFirstTraverser, + NoneExistingPath, + UpdateExpressionFunction, + ExpressionPathDescender, + UpdateExpressionValue, + ExpressionValueOperator, + ExpressionSelector, +) +from moto.dynamodb2.parsing.reserved_keywords import ReservedKeywords + + +class ExpressionAttributeValueProcessor(DepthFirstTraverser): + def __init__(self, expression_attribute_values): + self.expression_attribute_values = expression_attribute_values + + def _processing_map(self): + return { + ExpressionAttributeValue: self.replace_expression_attribute_value_with_value + } + + def replace_expression_attribute_value_with_value(self, node): + """A node representing an Expression Attribute Value. Resolve and replace value""" + assert isinstance(node, ExpressionAttributeValue) + attribute_value_name = node.get_value_name() + try: + target = self.expression_attribute_values[attribute_value_name] + except KeyError: + raise ExpressionAttributeValueNotDefined( + attribute_value=attribute_value_name + ) + return DDBTypedValue(DynamoType(target)) + + +class ExpressionAttributeResolvingProcessor(DepthFirstTraverser): + def _processing_map(self): + return { + UpdateExpressionSetAction: self.disable_resolving, + UpdateExpressionPath: self.process_expression_path_node, + } + + def __init__(self, expression_attribute_names, item): + self.expression_attribute_names = expression_attribute_names + self.item = item + self.resolving = False + + def pre_processing_of_child(self, parent_node, child_id): + """ + We have to enable resolving if we are processing a child of UpdateExpressionSetAction that is not first. + Because first argument is path to be set, 2nd argument would be the value. + """ + if isinstance( + parent_node, + ( + UpdateExpressionSetAction, + UpdateExpressionRemoveAction, + UpdateExpressionDeleteAction, + UpdateExpressionAddAction, + ), + ): + if child_id == 0: + self.resolving = False + else: + self.resolving = True + + def disable_resolving(self, node=None): + self.resolving = False + return node + + def process_expression_path_node(self, node): + """Resolve ExpressionAttribute if not part of a path and resolving is enabled.""" + if self.resolving: + return self.resolve_expression_path(node) + else: + # Still resolve but return original note to make sure path is correct Just make sure nodes are creatable. + result_node = self.resolve_expression_path(node) + if ( + isinstance(result_node, NoneExistingPath) + and not result_node.is_creatable() + ): + raise InvalidUpdateExpressionInvalidDocumentPath() + + return node + + def resolve_expression_path(self, node): + assert isinstance(node, UpdateExpressionPath) + + target = deepcopy(self.item.attrs) + for child in node.children: + # First replace placeholder with attribute_name + attr_name = None + if isinstance(child, ExpressionAttributeName): + attr_placeholder = child.get_attribute_name_placeholder() + try: + attr_name = self.expression_attribute_names[attr_placeholder] + except KeyError: + raise ExpressionAttributeNameNotDefined(attr_placeholder) + elif isinstance(child, ExpressionAttribute): + attr_name = child.get_attribute_name() + self.raise_exception_if_keyword(attr_name) + if attr_name is not None: + # Resolv attribute_name + try: + target = target[attr_name] + except (KeyError, TypeError): + if child == node.children[-1]: + return NoneExistingPath(creatable=True) + return NoneExistingPath() + else: + if isinstance(child, ExpressionPathDescender): + continue + elif isinstance(child, ExpressionSelector): + index = child.get_index() + if target.is_list(): + try: + target = target[index] + except IndexError: + # When a list goes out of bounds when assigning that is no problem when at the assignment + # side. It will just append to the list. + if child == node.children[-1]: + return NoneExistingPath(creatable=True) + return NoneExistingPath() + else: + raise InvalidUpdateExpressionInvalidDocumentPath + else: + raise NotImplementedError( + "Path resolution for {t}".format(t=type(child)) + ) + return DDBTypedValue(DynamoType(target)) + + @classmethod + def raise_exception_if_keyword(cls, attribute): + if attribute.upper() in ReservedKeywords.get_reserved_keywords(): + raise AttributeIsReservedKeyword(attribute) + + +class UpdateExpressionFunctionEvaluator(DepthFirstTraverser): + """ + At time of writing there are only 2 functions for DDB UpdateExpressions. They both are specific to the SET + expression as per the official AWS docs: + https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ + Expressions.UpdateExpressions.html#Expressions.UpdateExpressions.SET + """ + + def _processing_map(self): + return {UpdateExpressionFunction: self.process_function} + + def process_function(self, node): + assert isinstance(node, UpdateExpressionFunction) + function_name = node.get_function_name() + first_arg = node.get_nth_argument(1) + second_arg = node.get_nth_argument(2) + + if function_name == "if_not_exists": + if isinstance(first_arg, NoneExistingPath): + result = second_arg + else: + result = first_arg + assert isinstance(result, (DDBTypedValue, NoneExistingPath)) + return result + elif function_name == "list_append": + first_arg = self.get_list_from_ddb_typed_value(first_arg, function_name) + second_arg = self.get_list_from_ddb_typed_value(second_arg, function_name) + for list_element in second_arg.value: + first_arg.value.append(list_element) + return DDBTypedValue(first_arg) + else: + raise NotImplementedError( + "Unsupported function for moto {name}".format(name=function_name) + ) + + @classmethod + def get_list_from_ddb_typed_value(cls, node, function_name): + assert isinstance(node, DDBTypedValue) + dynamo_value = node.get_value() + assert isinstance(dynamo_value, DynamoType) + if not dynamo_value.is_list(): + raise IncorrectOperandType(function_name, dynamo_value.type) + return dynamo_value + + +class NoneExistingPathChecker(DepthFirstTraverser): + """ + Pass through the AST and make sure there are no none-existing paths. + """ + + def _processing_map(self): + return {NoneExistingPath: self.raise_none_existing_path} + + def raise_none_existing_path(self, node): + raise AttributeDoesNotExist + + +class ExecuteOperations(DepthFirstTraverser): + def _processing_map(self): + return {UpdateExpressionValue: self.process_update_expression_value} + + def process_update_expression_value(self, node): + """ + If an UpdateExpressionValue only has a single child the node will be replaced with the childe. + Otherwise it has 3 children and the middle one is an ExpressionValueOperator which details how to combine them + Args: + node(Node): + + Returns: + Node: The resulting node of the operation if present or the child. + """ + assert isinstance(node, UpdateExpressionValue) + if len(node.children) == 1: + return node.children[0] + elif len(node.children) == 3: + operator_node = node.children[1] + assert isinstance(operator_node, ExpressionValueOperator) + operator = operator_node.get_operator() + left_operand = self.get_dynamo_value_from_ddb_typed_value(node.children[0]) + right_operand = self.get_dynamo_value_from_ddb_typed_value(node.children[2]) + if operator == "+": + return self.get_sum(left_operand, right_operand) + elif operator == "-": + return self.get_subtraction(left_operand, right_operand) + else: + raise NotImplementedError( + "Moto does not support operator {operator}".format( + operator=operator + ) + ) + else: + raise NotImplementedError( + "UpdateExpressionValue only has implementations for 1 or 3 children." + ) + + @classmethod + def get_dynamo_value_from_ddb_typed_value(cls, node): + assert isinstance(node, DDBTypedValue) + dynamo_value = node.get_value() + assert isinstance(dynamo_value, DynamoType) + return dynamo_value + + @classmethod + def get_sum(cls, left_operand, right_operand): + """ + Args: + left_operand(DynamoType): + right_operand(DynamoType): + + Returns: + DDBTypedValue: + """ + try: + return DDBTypedValue(left_operand + right_operand) + except TypeError: + raise IncorrectOperandType("+", left_operand.type) + + @classmethod + def get_subtraction(cls, left_operand, right_operand): + """ + Args: + left_operand(DynamoType): + right_operand(DynamoType): + + Returns: + DDBTypedValue: + """ + try: + return DDBTypedValue(left_operand - right_operand) + except TypeError: + raise IncorrectOperandType("-", left_operand.type) + + +class Validator(object): + """ + A validator is used to validate expressions which are passed in as an AST. + """ + + def __init__( + self, expression, expression_attribute_names, expression_attribute_values, item + ): + """ + Besides validation the Validator should also replace referenced parts of an item which is cheapest upon + validation. + + Args: + expression(Node): The root node of the AST representing the expression to be validated + expression_attribute_names(ExpressionAttributeNames): + expression_attribute_values(ExpressionAttributeValues): + item(Item): The item which will be updated (pointed to by Key of update_item) + """ + self.expression_attribute_names = expression_attribute_names + self.expression_attribute_values = expression_attribute_values + self.item = item + self.processors = self.get_ast_processors() + self.node_to_validate = deepcopy(expression) + + @abstractmethod + def get_ast_processors(self): + """Get the different processors that go through the AST tree and processes the nodes.""" + + def validate(self): + n = self.node_to_validate + for processor in self.processors: + n = processor.traverse(n) + return n + + +class UpdateExpressionValidator(Validator): + def get_ast_processors(self): + """Get the different processors that go through the AST tree and processes the nodes.""" + processors = [ + ExpressionAttributeValueProcessor(self.expression_attribute_values), + ExpressionAttributeResolvingProcessor( + self.expression_attribute_names, self.item + ), + UpdateExpressionFunctionEvaluator(), + NoneExistingPathChecker(), + ExecuteOperations(), + ] + return processors diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index a5aeeac70..d14a54873 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -9,7 +9,7 @@ import six from moto.core.responses import BaseResponse from moto.core.utils import camelcase_to_underscores, amzn_request_id -from .exceptions import InvalidIndexNameError, InvalidUpdateExpression, ItemSizeTooLarge, MockValidationException +from .exceptions import InvalidIndexNameError, ItemSizeTooLarge, MockValidationException from moto.dynamodb2.models import dynamodb_backends, dynamo_json_dump diff --git a/tests/test_dynamodb2/test_dynamodb_expressions.py b/tests/test_dynamodb2/test_dynamodb_expressions.py index 1066231af..2c82d8bc4 100644 --- a/tests/test_dynamodb2/test_dynamodb_expressions.py +++ b/tests/test_dynamodb2/test_dynamodb_expressions.py @@ -393,3 +393,13 @@ def test_update_expression_parsing_is_not_keyword_aware(): except InvalidTokenException as te: assert te.token == "1" assert te.near == "VALUE 1" + + +def test_expression_if_not_exists_is_not_valid_in_remove_statement(): + set_action = "REMOVE if_not_exists(a,b)" + try: + UpdateExpressionParser.make(set_action) + assert False, "Exception not raised correctly" + except InvalidTokenException as te: + assert te.token == "(" + assert te.near == "if_not_exists(a" diff --git a/tests/test_dynamodb2/test_dynamodb_validation.py b/tests/test_dynamodb2/test_dynamodb_validation.py new file mode 100644 index 000000000..d60dd48f6 --- /dev/null +++ b/tests/test_dynamodb2/test_dynamodb_validation.py @@ -0,0 +1,464 @@ +from moto.dynamodb2.exceptions import ( + AttributeIsReservedKeyword, + ExpressionAttributeValueNotDefined, + AttributeDoesNotExist, + ExpressionAttributeNameNotDefined, + IncorrectOperandType, + InvalidUpdateExpressionInvalidDocumentPath, +) +from moto.dynamodb2.models import Item, DynamoType +from moto.dynamodb2.parsing.ast_nodes import ( + NodeDepthLeftTypeFetcher, + UpdateExpressionSetAction, + UpdateExpressionValue, + DDBTypedValue, +) +from moto.dynamodb2.parsing.expressions import UpdateExpressionParser +from moto.dynamodb2.parsing.validators import UpdateExpressionValidator +from parameterized import parameterized + + +def test_validation_of_update_expression_with_keyword(): + try: + update_expression = "SET myNum = path + :val" + update_expression_values = {":val": {"N": "3"}} + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "path": {"N": "3"}}, + ) + UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values=update_expression_values, + item=item, + ).validate() + assert False, "No exception raised" + except AttributeIsReservedKeyword as e: + assert e.keyword == "path" + + +@parameterized( + ["SET a = #b + :val2", "SET a = :val2 + #b",] +) +def test_validation_of_a_set_statement_with_incorrect_passed_value(update_expression): + """ + By running permutations it shows that values are replaced prior to resolving attributes. + + An error occurred (ValidationException) when calling the UpdateItem operation: Invalid UpdateExpression: + An expression attribute value used in expression is not defined; attribute value: :val2 + """ + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "b": {"N": "3"}}, + ) + try: + UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names={"#b": "ok"}, + expression_attribute_values={":val": {"N": "3"}}, + item=item, + ).validate() + except ExpressionAttributeValueNotDefined as e: + assert e.attribute_value == ":val2" + + +def test_validation_of_update_expression_with_attribute_that_does_not_exist_in_item(): + """ + When an update expression tries to get an attribute that does not exist it must throw the appropriate exception. + + An error occurred (ValidationException) when calling the UpdateItem operation: + The provided expression refers to an attribute that does not exist in the item + """ + try: + update_expression = "SET a = nonexistent" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "path": {"N": "3"}}, + ) + UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values=None, + item=item, + ).validate() + assert False, "No exception raised" + except AttributeDoesNotExist: + assert True + + +@parameterized( + ["SET a = #c", "SET a = #c + #d",] +) +def test_validation_of_update_expression_with_attribute_name_that_is_not_defined( + update_expression, +): + """ + When an update expression tries to get an attribute name that is not provided it must throw an exception. + + An error occurred (ValidationException) when calling the UpdateItem operation: Invalid UpdateExpression: + An expression attribute name used in the document path is not defined; attribute name: #c + """ + try: + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "path": {"N": "3"}}, + ) + UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names={"#b": "ok"}, + expression_attribute_values=None, + item=item, + ).validate() + assert False, "No exception raised" + except ExpressionAttributeNameNotDefined as e: + assert e.not_defined_attribute_name == "#c" + + +def test_validation_of_if_not_exists_not_existing_invalid_replace_value(): + try: + update_expression = "SET a = if_not_exists(b, a.c)" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "a": {"S": "A"}}, + ) + UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values=None, + item=item, + ).validate() + assert False, "No exception raised" + except AttributeDoesNotExist: + assert True + + +def get_first_node_of_type(ast, node_type): + return next(NodeDepthLeftTypeFetcher(node_type, ast)) + + +def get_set_action_value(ast): + """ + Helper that takes an AST and gets the first UpdateExpressionSetAction and retrieves the value of that action. + This should only be called on validated expressions. + Args: + ast(Node): + + Returns: + DynamoType: The DynamoType object representing the Dynamo value. + """ + set_action = get_first_node_of_type(ast, UpdateExpressionSetAction) + typed_value = set_action.children[1] + assert isinstance(typed_value, DDBTypedValue) + dynamo_value = typed_value.children[0] + assert isinstance(dynamo_value, DynamoType) + return dynamo_value + + +def test_validation_of_if_not_exists_not_existing_value(): + update_expression = "SET a = if_not_exists(b, a)" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "a": {"S": "A"}}, + ) + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values=None, + item=item, + ).validate() + dynamo_value = get_set_action_value(validated_ast) + assert dynamo_value == DynamoType({"S": "A"}) + + +def test_validation_of_if_not_exists_with_existing_attribute_should_return_attribute(): + update_expression = "SET a = if_not_exists(b, a)" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "a": {"S": "A"}, "b": {"S": "B"}}, + ) + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values=None, + item=item, + ).validate() + dynamo_value = get_set_action_value(validated_ast) + assert dynamo_value == DynamoType({"S": "B"}) + + +def test_validation_of_if_not_exists_with_existing_attribute_should_return_value(): + update_expression = "SET a = if_not_exists(b, :val)" + update_expression_values = {":val": {"N": "4"}} + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "b": {"N": "3"}}, + ) + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values=update_expression_values, + item=item, + ).validate() + dynamo_value = get_set_action_value(validated_ast) + assert dynamo_value == DynamoType({"N": "3"}) + + +def test_validation_of_if_not_exists_with_non_existing_attribute_should_return_value(): + update_expression = "SET a = if_not_exists(b, :val)" + update_expression_values = {":val": {"N": "4"}} + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}}, + ) + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values=update_expression_values, + item=item, + ).validate() + dynamo_value = get_set_action_value(validated_ast) + assert dynamo_value == DynamoType({"N": "4"}) + + +def test_validation_of_sum_operation(): + update_expression = "SET a = a + b" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "a": {"N": "3"}, "b": {"N": "4"}}, + ) + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values=None, + item=item, + ).validate() + dynamo_value = get_set_action_value(validated_ast) + assert dynamo_value == DynamoType({"N": "7"}) + + +def test_validation_homogeneous_list_append_function(): + update_expression = "SET ri = list_append(ri, :vals)" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "ri": {"L": [{"S": "i1"}, {"S": "i2"}]}}, + ) + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values={":vals": {"L": [{"S": "i3"}, {"S": "i4"}]}}, + item=item, + ).validate() + dynamo_value = get_set_action_value(validated_ast) + assert dynamo_value == DynamoType( + {"L": [{"S": "i1"}, {"S": "i2"}, {"S": "i3"}, {"S": "i4"}]} + ) + + +def test_validation_hetereogenous_list_append_function(): + update_expression = "SET ri = list_append(ri, :vals)" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "ri": {"L": [{"S": "i1"}, {"S": "i2"}]}}, + ) + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values={":vals": {"L": [{"N": "3"}]}}, + item=item, + ).validate() + dynamo_value = get_set_action_value(validated_ast) + assert dynamo_value == DynamoType({"L": [{"S": "i1"}, {"S": "i2"}, {"N": "3"}]}) + + +def test_validation_list_append_function_with_non_list_arg(): + """ + Must error out: + Invalid UpdateExpression: Incorrect operand type for operator or function; + operator or function: list_append, operand type: S' + Returns: + + """ + try: + update_expression = "SET ri = list_append(ri, :vals)" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "ri": {"L": [{"S": "i1"}, {"S": "i2"}]}}, + ) + UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values={":vals": {"S": "N"}}, + item=item, + ).validate() + except IncorrectOperandType as e: + assert e.operand_type == "S" + assert e.operator_or_function == "list_append" + + +def test_sum_with_incompatible_types(): + """ + Must error out: + Invalid UpdateExpression: Incorrect operand type for operator or function; operator or function: +, operand type: S' + Returns: + + """ + try: + update_expression = "SET ri = :val + :val2" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "ri": {"L": [{"S": "i1"}, {"S": "i2"}]}}, + ) + UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values={":val": {"S": "N"}, ":val2": {"N": "3"}}, + item=item, + ).validate() + except IncorrectOperandType as e: + assert e.operand_type == "S" + assert e.operator_or_function == "+" + + +def test_validation_of_subraction_operation(): + update_expression = "SET ri = :val - :val2" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "a": {"N": "3"}, "b": {"N": "4"}}, + ) + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values={":val": {"N": "1"}, ":val2": {"N": "3"}}, + item=item, + ).validate() + dynamo_value = get_set_action_value(validated_ast) + assert dynamo_value == DynamoType({"N": "-2"}) + + +def test_cannot_index_into_a_string(): + """ + Must error out: + The document path provided in the update expression is invalid for update' + """ + try: + update_expression = "set itemstr[1]=:Item" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "foo2"}, "itemstr": {"S": "somestring"}}, + ) + UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values={":Item": {"S": "string_update"}}, + item=item, + ).validate() + assert False, "Must raise exception" + except InvalidUpdateExpressionInvalidDocumentPath: + assert True + + +def test_validation_set_path_does_not_need_to_be_resolvable_when_setting_a_new_attribute(): + """If this step just passes we are happy enough""" + update_expression = "set d=a" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "foo2"}, "a": {"N": "3"}}, + ) + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values=None, + item=item, + ).validate() + dynamo_value = get_set_action_value(validated_ast) + assert dynamo_value == DynamoType({"N": "3"}) + + +def test_validation_set_path_does_not_need_to_be_resolvable_but_must_be_creatable_when_setting_a_new_attribute(): + try: + update_expression = "set d.e=a" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "foo2"}, "a": {"N": "3"}}, + ) + UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values=None, + item=item, + ).validate() + assert False, "Must raise exception" + except InvalidUpdateExpressionInvalidDocumentPath: + assert True From e6b51a28ee884697cba89a68e5f9948880c25199 Mon Sep 17 00:00:00 2001 From: pvbouwel Date: Sun, 19 Apr 2020 16:50:53 +0100 Subject: [PATCH 59/83] Enable AST Validation This commit puts AST validation on the execution path. This means updates get validated prior to being executed. There were quite a few tests that were not working against Amazon DDB. These tests I considered broken and as such this commit adapts them such that they pass against Amazon DDB. test_update_item_on_map() => One of the SET actions would try to set a nested element by specifying the nesting on the path rather than by putting a map as a value for a non-existent key. This got changed. test_item_size_is_under_400KB => Used the keyword "item" which DDB doesn't like. Change to cont in order to keep the same sizings. => Secondly the size error messages differs a bit depending whether it is part of the update or part of a put_item. For an update it should be: Item size to update has exceeded the maximum allowed size otherwise it is Item size has exceeded the maximum allowed size' test_remove_top_level_attribute => Used a keyword item. Use ExpressionAttributeNames test_update_item_double_nested_remove => Used keywords name & first. Migrated to non-deprecated API and use ExpressionAttributeNames test_update_item_set & test_boto3_update_item_conditions_pass & test_boto3_update_item_conditions_pass_because_expect_not_exists & test_boto3_update_item_conditions_pass_because_expect_not_exists_by_compare_to_null & test_boto3_update_item_conditions_pass_because_expect_exists_by_compare_to_not_null & test_boto3_update_item_conditions_fail & test_boto3_update_item_conditions_fail_because_expect_not_exists & test_boto3_update_item_conditions_fail_because_expect_not_exists_by_compare_to_null => Were broken tests which had string literal instead of value placeholder --- moto/dynamodb2/exceptions.py | 11 ++++ moto/dynamodb2/models/__init__.py | 22 +++++-- tests/test_dynamodb2/test_dynamodb.py | 54 ++++++++++++---- .../test_dynamodb_table_without_range_key.py | 63 +++++++++++++------ 4 files changed, 117 insertions(+), 33 deletions(-) diff --git a/moto/dynamodb2/exceptions.py b/moto/dynamodb2/exceptions.py index a6acae071..5dd87ef6b 100644 --- a/moto/dynamodb2/exceptions.py +++ b/moto/dynamodb2/exceptions.py @@ -111,6 +111,17 @@ class ItemSizeTooLarge(MockValidationException): super(ItemSizeTooLarge, self).__init__(self.item_size_too_large_msg) +class ItemSizeToUpdateTooLarge(MockValidationException): + item_size_to_update_too_large_msg = ( + "Item size to update has exceeded the maximum allowed size" + ) + + def __init__(self): + super(ItemSizeToUpdateTooLarge, self).__init__( + self.item_size_to_update_too_large_msg + ) + + class IncorrectOperandType(InvalidUpdateExpression): inv_operand_msg = "Incorrect operand type for operator or function; operator or function: {f}, operand type: {t}" diff --git a/moto/dynamodb2/models/__init__.py b/moto/dynamodb2/models/__init__.py index 1f448f288..00825e06a 100644 --- a/moto/dynamodb2/models/__init__.py +++ b/moto/dynamodb2/models/__init__.py @@ -14,11 +14,16 @@ from moto.core import BaseBackend, BaseModel from moto.core.utils import unix_time from moto.core.exceptions import JsonRESTError from moto.dynamodb2.comparisons import get_filter_expression -from moto.dynamodb2.comparisons import get_expected, get_comparison_func -from moto.dynamodb2.exceptions import InvalidIndexNameError, ItemSizeTooLarge, InvalidUpdateExpression +from moto.dynamodb2.comparisons import get_expected +from moto.dynamodb2.exceptions import ( + InvalidIndexNameError, + ItemSizeTooLarge, + ItemSizeToUpdateTooLarge, +) from moto.dynamodb2.models.utilities import bytesize, attribute_is_list from moto.dynamodb2.models.dynamo_type import DynamoType from moto.dynamodb2.parsing.expressions import UpdateExpressionParser +from moto.dynamodb2.parsing.validators import UpdateExpressionValidator class DynamoJsonEncoder(json.JSONEncoder): @@ -151,7 +156,10 @@ class Item(BaseModel): if "." in key and attr not in self.attrs: raise ValueError # Setting nested attr not allowed if first attr does not exist yet elif attr not in self.attrs: - self.attrs[attr] = dyn_value # set new top-level attribute + try: + self.attrs[attr] = dyn_value # set new top-level attribute + except ItemSizeTooLarge: + raise ItemSizeToUpdateTooLarge() else: self.attrs[attr].set( ".".join(key.split(".")[1:]), dyn_value, list_index @@ -1202,7 +1210,7 @@ class DynamoDBBackend(BaseBackend): # E.g. `a = b + c` -> `a=b+c` if update_expression: # Parse expression to get validation errors - UpdateExpressionParser.make(update_expression) + update_expression_ast = UpdateExpressionParser.make(update_expression) update_expression = re.sub(r"\s*([=\+-])\s*", "\\1", update_expression) if all([table.hash_key_attr in key, table.range_key_attr in key]): @@ -1247,6 +1255,12 @@ class DynamoDBBackend(BaseBackend): item = table.get_item(hash_value, range_value) if update_expression: + UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=expression_attribute_names, + expression_attribute_values=expression_attribute_values, + item=item, + ).validate() item.update( update_expression, expression_attribute_names, diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 09401d562..0004001bc 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -2147,13 +2147,33 @@ def test_update_item_on_map(): # Nonexistent nested attributes are supported for existing top-level attributes. table.update_item( Key={"forum_name": "the-key", "subject": "123"}, - UpdateExpression="SET body.#nested.#data = :tb, body.nested.#nonexistentnested.#data = :tb2", + UpdateExpression="SET body.#nested.#data = :tb", + ExpressionAttributeNames={"#nested": "nested", "#data": "data",}, + ExpressionAttributeValues={":tb": "new_value"}, + ) + # Running this against AWS DDB gives an exception so make sure it also fails.: + with assert_raises(client.exceptions.ClientError): + # botocore.exceptions.ClientError: An error occurred (ValidationException) when calling the UpdateItem + # operation: The document path provided in the update expression is invalid for update + table.update_item( + Key={"forum_name": "the-key", "subject": "123"}, + UpdateExpression="SET body.#nested.#nonexistentnested.#data = :tb2", + ExpressionAttributeNames={ + "#nested": "nested", + "#nonexistentnested": "nonexistentnested", + "#data": "data", + }, + ExpressionAttributeValues={":tb2": "other_value"}, + ) + + table.update_item( + Key={"forum_name": "the-key", "subject": "123"}, + UpdateExpression="SET body.#nested.#nonexistentnested = :tb2", ExpressionAttributeNames={ "#nested": "nested", "#nonexistentnested": "nonexistentnested", - "#data": "data", }, - ExpressionAttributeValues={":tb": "new_value", ":tb2": "other_value"}, + ExpressionAttributeValues={":tb2": {"data": "other_value"}}, ) resp = table.scan() @@ -2161,8 +2181,8 @@ def test_update_item_on_map(): {"nested": {"data": "new_value", "nonexistentnested": {"data": "other_value"}}} ) - # Test nested value for a nonexistent attribute. - with assert_raises(client.exceptions.ConditionalCheckFailedException): + # Test nested value for a nonexistent attribute throws a ClientError. + with assert_raises(client.exceptions.ClientError): table.update_item( Key={"forum_name": "the-key", "subject": "123"}, UpdateExpression="SET nonexistent.#nested = :tb", @@ -3184,7 +3204,10 @@ def test_remove_top_level_attribute(): TableName=table_name, Item={"id": {"S": "foo"}, "item": {"S": "bar"}} ) client.update_item( - TableName=table_name, Key={"id": {"S": "foo"}}, UpdateExpression="REMOVE item" + TableName=table_name, + Key={"id": {"S": "foo"}}, + UpdateExpression="REMOVE #i", + ExpressionAttributeNames={"#i": "item"}, ) # result = client.get_item(TableName=table_name, Key={"id": {"S": "foo"}})["Item"] @@ -3359,21 +3382,21 @@ def test_item_size_is_under_400KB(): assert_failure_due_to_item_size( func=client.put_item, TableName="moto-test", - Item={"id": {"S": "foo"}, "item": {"S": large_item}}, + Item={"id": {"S": "foo"}, "cont": {"S": large_item}}, ) assert_failure_due_to_item_size( - func=table.put_item, Item={"id": "bar", "item": large_item} + func=table.put_item, Item={"id": "bar", "cont": large_item} ) - assert_failure_due_to_item_size( + assert_failure_due_to_item_size_to_update( func=client.update_item, TableName="moto-test", Key={"id": {"S": "foo2"}}, - UpdateExpression="set item=:Item", + UpdateExpression="set cont=:Item", ExpressionAttributeValues={":Item": {"S": large_item}}, ) # Assert op fails when updating a nested item assert_failure_due_to_item_size( - func=table.put_item, Item={"id": "bar", "itemlist": [{"item": large_item}]} + func=table.put_item, Item={"id": "bar", "itemlist": [{"cont": large_item}]} ) assert_failure_due_to_item_size( func=client.put_item, @@ -3394,6 +3417,15 @@ def assert_failure_due_to_item_size(func, **kwargs): ) +def assert_failure_due_to_item_size_to_update(func, **kwargs): + with assert_raises(ClientError) as ex: + func(**kwargs) + ex.exception.response["Error"]["Code"].should.equal("ValidationException") + ex.exception.response["Error"]["Message"].should.equal( + "Item size to update has exceeded the maximum allowed size" + ) + + @mock_dynamodb2 # https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Query.html#DDB-Query-request-KeyConditionExpression def test_hash_key_cannot_use_begins_with_operations(): diff --git a/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py b/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py index 08d7724f8..b5cc01c84 100644 --- a/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py +++ b/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py @@ -443,23 +443,40 @@ def test_update_item_nested_remove(): dict(returned_item).should.equal({"username": "steve", "Meta": {}}) -@mock_dynamodb2_deprecated +@mock_dynamodb2 def test_update_item_double_nested_remove(): - conn = boto.dynamodb2.connect_to_region("us-east-1") - table = Table.create("messages", schema=[HashKey("username")]) + conn = boto3.client("dynamodb", region_name="us-east-1") + conn.create_table( + TableName="messages", + KeySchema=[{"AttributeName": "username", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "username", "AttributeType": "S"}], + BillingMode="PAY_PER_REQUEST", + ) - data = {"username": "steve", "Meta": {"Name": {"First": "Steve", "Last": "Urkel"}}} - table.put_item(data=data) + item = { + "username": {"S": "steve"}, + "Meta": { + "M": {"Name": {"M": {"First": {"S": "Steve"}, "Last": {"S": "Urkel"}}}} + }, + } + conn.put_item(TableName="messages", Item=item) key_map = {"username": {"S": "steve"}} # Then remove the Meta.FullName field - conn.update_item("messages", key_map, update_expression="REMOVE Meta.Name.First") - - returned_item = table.get_item(username="steve") - dict(returned_item).should.equal( - {"username": "steve", "Meta": {"Name": {"Last": "Urkel"}}} + conn.update_item( + TableName="messages", + Key=key_map, + UpdateExpression="REMOVE Meta.#N.#F", + ExpressionAttributeNames={"#N": "Name", "#F": "First"}, ) + returned_item = conn.get_item(TableName="messages", Key=key_map) + expected_item = { + "username": {"S": "steve"}, + "Meta": {"M": {"Name": {"M": {"Last": {"S": "Urkel"}}}}}, + } + dict(returned_item["Item"]).should.equal(expected_item) + @mock_dynamodb2_deprecated def test_update_item_set(): @@ -471,7 +488,10 @@ def test_update_item_set(): key_map = {"username": {"S": "steve"}} conn.update_item( - "messages", key_map, update_expression="SET foo=bar, blah=baz REMOVE SentBy" + "messages", + key_map, + update_expression="SET foo=:bar, blah=:baz REMOVE SentBy", + expression_attribute_values={":bar": {"S": "bar"}, ":baz": {"S": "baz"}}, ) returned_item = table.get_item(username="steve") @@ -616,8 +636,9 @@ def test_boto3_update_item_conditions_fail(): table.put_item(Item={"username": "johndoe", "foo": "baz"}) table.update_item.when.called_with( Key={"username": "johndoe"}, - UpdateExpression="SET foo=bar", + UpdateExpression="SET foo=:bar", Expected={"foo": {"Value": "bar"}}, + ExpressionAttributeValues={":bar": "bar"}, ).should.throw(botocore.client.ClientError) @@ -627,8 +648,9 @@ def test_boto3_update_item_conditions_fail_because_expect_not_exists(): table.put_item(Item={"username": "johndoe", "foo": "baz"}) table.update_item.when.called_with( Key={"username": "johndoe"}, - UpdateExpression="SET foo=bar", + UpdateExpression="SET foo=:bar", Expected={"foo": {"Exists": False}}, + ExpressionAttributeValues={":bar": "bar"}, ).should.throw(botocore.client.ClientError) @@ -638,8 +660,9 @@ def test_boto3_update_item_conditions_fail_because_expect_not_exists_by_compare_ table.put_item(Item={"username": "johndoe", "foo": "baz"}) table.update_item.when.called_with( Key={"username": "johndoe"}, - UpdateExpression="SET foo=bar", + UpdateExpression="SET foo=:bar", Expected={"foo": {"ComparisonOperator": "NULL"}}, + ExpressionAttributeValues={":bar": "bar"}, ).should.throw(botocore.client.ClientError) @@ -649,8 +672,9 @@ def test_boto3_update_item_conditions_pass(): table.put_item(Item={"username": "johndoe", "foo": "bar"}) table.update_item( Key={"username": "johndoe"}, - UpdateExpression="SET foo=baz", + UpdateExpression="SET foo=:baz", Expected={"foo": {"Value": "bar"}}, + ExpressionAttributeValues={":baz": "baz"}, ) returned_item = table.get_item(Key={"username": "johndoe"}) assert dict(returned_item)["Item"]["foo"].should.equal("baz") @@ -662,8 +686,9 @@ def test_boto3_update_item_conditions_pass_because_expect_not_exists(): table.put_item(Item={"username": "johndoe", "foo": "bar"}) table.update_item( Key={"username": "johndoe"}, - UpdateExpression="SET foo=baz", + UpdateExpression="SET foo=:baz", Expected={"whatever": {"Exists": False}}, + ExpressionAttributeValues={":baz": "baz"}, ) returned_item = table.get_item(Key={"username": "johndoe"}) assert dict(returned_item)["Item"]["foo"].should.equal("baz") @@ -675,8 +700,9 @@ def test_boto3_update_item_conditions_pass_because_expect_not_exists_by_compare_ table.put_item(Item={"username": "johndoe", "foo": "bar"}) table.update_item( Key={"username": "johndoe"}, - UpdateExpression="SET foo=baz", + UpdateExpression="SET foo=:baz", Expected={"whatever": {"ComparisonOperator": "NULL"}}, + ExpressionAttributeValues={":baz": "baz"}, ) returned_item = table.get_item(Key={"username": "johndoe"}) assert dict(returned_item)["Item"]["foo"].should.equal("baz") @@ -688,8 +714,9 @@ def test_boto3_update_item_conditions_pass_because_expect_exists_by_compare_to_n table.put_item(Item={"username": "johndoe", "foo": "bar"}) table.update_item( Key={"username": "johndoe"}, - UpdateExpression="SET foo=baz", + UpdateExpression="SET foo=:baz", Expected={"foo": {"ComparisonOperator": "NOT_NULL"}}, + ExpressionAttributeValues={":baz": "baz"}, ) returned_item = table.get_item(Key={"username": "johndoe"}) assert dict(returned_item)["Item"]["foo"].should.equal("baz") From 3a774ed0e0b22a42cf206533a9c4e6952089937f Mon Sep 17 00:00:00 2001 From: pvbouwel Date: Sun, 19 Apr 2020 17:55:00 +0100 Subject: [PATCH 60/83] Make sure reserved_keywords.txt is packaged with the library. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 79b9875ee..b142f3203 100755 --- a/setup.py +++ b/setup.py @@ -100,4 +100,5 @@ setup( project_urls={ "Documentation": "http://docs.getmoto.org/en/latest/", }, + data_files=[('', ['moto/dynamodb2/parsing/reserved_keywords.txt'])], ) From ed5e0b586c50955515fb2ed72a7c028ece91a9d3 Mon Sep 17 00:00:00 2001 From: Dmitry Ryzhikov Date: Mon, 20 Apr 2020 00:15:00 +0300 Subject: [PATCH 61/83] Handle ValueError raised on missing table name --- moto/dynamodb2/responses.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 65484aa08..32f10abc0 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -92,16 +92,24 @@ class DynamoHandler(BaseResponse): def list_tables(self): body = self.body limit = body.get("Limit", 100) - if body.get("ExclusiveStartTableName"): - last = body.get("ExclusiveStartTableName") - start = list(self.dynamodb_backend.tables.keys()).index(last) + 1 + all_tables = list(self.dynamodb_backend.tables.keys()) + + exclusive_start_table_name = body.get("ExclusiveStartTableName") + if exclusive_start_table_name: + try: + last_table_index = all_tables.index(exclusive_start_table_name) + except ValueError: + start = len(all_tables) + else: + start = last_table_index + 1 else: start = 0 - all_tables = list(self.dynamodb_backend.tables.keys()) + if limit: tables = all_tables[start : start + limit] else: tables = all_tables[start:] + response = {"TableNames": tables} if limit and len(all_tables) > start + limit: response["LastEvaluatedTableName"] = tables[-1] From 0d04306861f82cb7489b1d6d261dbaf3d6c745dd Mon Sep 17 00:00:00 2001 From: Asher Foa <1268088+asherf@users.noreply.github.com> Date: Sun, 19 Apr 2020 19:12:40 -0700 Subject: [PATCH 62/83] Fix deprecation warning. --- moto/ec2/urls.py | 2 +- moto/ssm/models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/ec2/urls.py b/moto/ec2/urls.py index 4d85b2f56..78f234320 100644 --- a/moto/ec2/urls.py +++ b/moto/ec2/urls.py @@ -2,6 +2,6 @@ from __future__ import unicode_literals from .responses import EC2Response -url_bases = ["https?://ec2\.(.+)\.amazonaws\.com(|\.cn)"] +url_bases = [r"https?://ec2\.(.+)\.amazonaws\.com(|\.cn)"] url_paths = {"{0}/": EC2Response.dispatch} diff --git a/moto/ssm/models.py b/moto/ssm/models.py index 201f43c5a..3ce3b3a22 100644 --- a/moto/ssm/models.py +++ b/moto/ssm/models.py @@ -651,7 +651,7 @@ class SimpleSystemManagerBackend(BaseBackend): label.startswith("aws") or label.startswith("ssm") or label[:1].isdigit() - or not re.match("^[a-zA-z0-9_\.\-]*$", label) + or not re.match(r"^[a-zA-z0-9_\.\-]*$", label) ): invalid_labels.append(label) continue From ad0805de0ec3546767f5c13141e3d072b8c6f496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20K=C3=A4ufl?= Date: Mon, 20 Apr 2020 09:19:24 +0200 Subject: [PATCH 63/83] Add Python 3.8 to trove classifiers --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 79b9875ee..684c0dcea 100755 --- a/setup.py +++ b/setup.py @@ -94,6 +94,7 @@ setup( "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "License :: OSI Approved :: Apache Software License", "Topic :: Software Development :: Testing", ], From 1a3a7d6a92619ec4142f5d4263f4b308b15e4209 Mon Sep 17 00:00:00 2001 From: Dmitry Ryzhikov Date: Mon, 20 Apr 2020 20:23:37 +0300 Subject: [PATCH 64/83] Add test for missing table name --- tests/test_dynamodb2/test_dynamodb.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index bec24c966..cb9230a4a 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -4177,3 +4177,12 @@ def test_gsi_verify_negative_number_order(): [float(item["gsiK1SortKey"]) for item in resp["Items"]].should.equal( [-0.7, -0.6, 0.7] ) + + +@mock_dynamodb2 +def test_list_tables_exclusive_start_table_name_empty(): + client = boto3.client("dynamodb", region_name="us-east-1") + + resp = client.list_tables(Limit=1, ExclusiveStartTableName="whatever") + + len(resp["TableNames"]).should.equal(0) From b6789a2cc7d7ab9c2bd3fdd5b95d00e6fa20758d Mon Sep 17 00:00:00 2001 From: Tomoya Iwata Date: Tue, 21 Apr 2020 14:11:53 +0900 Subject: [PATCH 65/83] Added existence check of target thing to IoT ListThingPrincipals fix #2910 --- moto/iot/exceptions.py | 4 ++-- moto/iot/models.py | 8 ++++++++ tests/test_iot/test_iot.py | 7 +++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/moto/iot/exceptions.py b/moto/iot/exceptions.py index d114a12ad..7a578c221 100644 --- a/moto/iot/exceptions.py +++ b/moto/iot/exceptions.py @@ -7,10 +7,10 @@ class IoTClientError(JsonRESTError): class ResourceNotFoundException(IoTClientError): - def __init__(self): + def __init__(self, msg=None): self.code = 404 super(ResourceNotFoundException, self).__init__( - "ResourceNotFoundException", "The specified resource does not exist" + "ResourceNotFoundException", msg or "The specified resource does not exist" ) diff --git a/moto/iot/models.py b/moto/iot/models.py index de4383b96..51a23b6c6 100644 --- a/moto/iot/models.py +++ b/moto/iot/models.py @@ -805,6 +805,14 @@ class IoTBackend(BaseBackend): return thing_names def list_thing_principals(self, thing_name): + + things = [_ for _ in self.things.values() if _.thing_name == thing_name] + if len(things) == 0: + raise ResourceNotFoundException( + "Failed to list principals for thing %s because the thing does not exist in your account" + % thing_name + ) + principals = [ k[0] for k, v in self.principal_things.items() if k[1] == thing_name ] diff --git a/tests/test_iot/test_iot.py b/tests/test_iot/test_iot.py index f8c4f579c..f3c151714 100644 --- a/tests/test_iot/test_iot.py +++ b/tests/test_iot/test_iot.py @@ -728,6 +728,13 @@ def test_principal_thing(): res = client.list_thing_principals(thingName=thing_name) res.should.have.key("principals").which.should.have.length_of(0) + with assert_raises(ClientError) as e: + client.list_thing_principals(thingName='xxx') + + e.exception.response["Error"]["Code"].should.equal("ResourceNotFoundException") + e.exception.response["Error"]["Message"].should.equal( + "Failed to list principals for thing xxx because the thing does not exist in your account" + ) @mock_iot def test_delete_principal_thing(): From d9b782be0a6944426347378345b2289732a2c7d9 Mon Sep 17 00:00:00 2001 From: Tomoya Iwata Date: Tue, 21 Apr 2020 14:43:04 +0900 Subject: [PATCH 66/83] fix lint --- tests/test_iot/test_iot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_iot/test_iot.py b/tests/test_iot/test_iot.py index f3c151714..2f43de5b9 100644 --- a/tests/test_iot/test_iot.py +++ b/tests/test_iot/test_iot.py @@ -729,13 +729,14 @@ def test_principal_thing(): res.should.have.key("principals").which.should.have.length_of(0) with assert_raises(ClientError) as e: - client.list_thing_principals(thingName='xxx') + client.list_thing_principals(thingName="xxx") e.exception.response["Error"]["Code"].should.equal("ResourceNotFoundException") e.exception.response["Error"]["Message"].should.equal( "Failed to list principals for thing xxx because the thing does not exist in your account" ) + @mock_iot def test_delete_principal_thing(): client = boto3.client("iot", region_name="ap-northeast-1") From 12669400b715ba9a3eb7759407fbf61f1283874c Mon Sep 17 00:00:00 2001 From: thatguysimon Date: Tue, 21 Apr 2020 16:53:22 +0300 Subject: [PATCH 67/83] Mark sts>get_caller_identity as implemented Seems like it's implemented but not marked --- IMPLEMENTATION_COVERAGE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 705618524..36caec175 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -7210,13 +7210,13 @@ - [ ] update_vtl_device_type ## sts -50% implemented +62% implemented - [X] assume_role - [ ] assume_role_with_saml - [X] assume_role_with_web_identity - [ ] decode_authorization_message - [ ] get_access_key_info -- [ ] get_caller_identity +- [X] get_caller_identity - [X] get_federation_token - [X] get_session_token From 753a39ed0d195c3f3092d2f22fa361f93711f57b Mon Sep 17 00:00:00 2001 From: MarcosBernal Date: Tue, 21 Apr 2020 20:10:39 +0200 Subject: [PATCH 68/83] Add get_databases method to glue moto client. Update IMPLEMENTATION_COVERAGE.md with methods that were covered previously --- IMPLEMENTATION_COVERAGE.md | 18 +++++++++--------- moto/glue/models.py | 3 +++ moto/glue/responses.py | 4 ++++ tests/test_glue/test_datacatalog.py | 21 +++++++++++++++++++++ 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 705618524..82ee2f046 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3351,11 +3351,11 @@ - [ ] update_listener ## glue -4% implemented -- [ ] batch_create_partition +14/123 = 11% implemented +- [X] batch_create_partition - [ ] batch_delete_connection -- [ ] batch_delete_partition -- [ ] batch_delete_table +- [X] batch_delete_partition +- [X] batch_delete_table - [ ] batch_delete_table_version - [ ] batch_get_crawlers - [ ] batch_get_dev_endpoints @@ -3372,7 +3372,7 @@ - [ ] create_dev_endpoint - [ ] create_job - [ ] create_ml_transform -- [ ] create_partition +- [X] create_partition - [ ] create_script - [ ] create_security_configuration - [X] create_table @@ -3404,7 +3404,7 @@ - [ ] get_crawlers - [ ] get_data_catalog_encryption_settings - [X] get_database -- [ ] get_databases +- [X] get_databases - [ ] get_dataflow_graph - [ ] get_dev_endpoint - [ ] get_dev_endpoints @@ -3418,7 +3418,7 @@ - [ ] get_ml_task_runs - [ ] get_ml_transform - [ ] get_ml_transforms -- [ ] get_partition +- [X] get_partition - [ ] get_partitions - [ ] get_plan - [ ] get_resource_policy @@ -3470,8 +3470,8 @@ - [ ] update_dev_endpoint - [ ] update_job - [ ] update_ml_transform -- [ ] update_partition -- [ ] update_table +- [X] update_partition +- [X] update_table - [ ] update_trigger - [ ] update_user_defined_function - [ ] update_workflow diff --git a/moto/glue/models.py b/moto/glue/models.py index 8f3396d9a..cf930cfb2 100644 --- a/moto/glue/models.py +++ b/moto/glue/models.py @@ -34,6 +34,9 @@ class GlueBackend(BaseBackend): except KeyError: raise DatabaseNotFoundException(database_name) + def get_databases(self): + return [self.databases[key] for key in self.databases] if self.databases else [] + def create_table(self, database_name, table_name, table_input): database = self.get_database(database_name) diff --git a/moto/glue/responses.py b/moto/glue/responses.py index bf7b5776b..4fb144bba 100644 --- a/moto/glue/responses.py +++ b/moto/glue/responses.py @@ -30,6 +30,10 @@ class GlueResponse(BaseResponse): database = self.glue_backend.get_database(database_name) return json.dumps({"Database": {"Name": database.name}}) + def get_databases(self): + database_list = self.glue_backend.get_databases() + return json.dumps({"DatabaseList": [{"Name": database.name} for database in database_list]}) + def create_table(self): database_name = self.parameters.get("DatabaseName") table_input = self.parameters.get("TableInput") diff --git a/tests/test_glue/test_datacatalog.py b/tests/test_glue/test_datacatalog.py index 31731e598..54fb17451 100644 --- a/tests/test_glue/test_datacatalog.py +++ b/tests/test_glue/test_datacatalog.py @@ -52,6 +52,27 @@ def test_get_database_not_exits(): ) +@mock_glue +def test_get_databases_empty(): + client = boto3.client("glue", region_name="us-east-1") + response = client.get_databases() + response["DatabaseList"].should.have.length_of(0) + + +@mock_glue +def test_get_databases_several_items(): + client = boto3.client("glue", region_name="us-east-1") + database_name_1, database_name_2 = "firstdatabase", "seconddatabase" + + helpers.create_database(client, database_name_1) + helpers.create_database(client, database_name_2) + + database_list = sorted(client.get_databases()["DatabaseList"], key=lambda x: x["Name"]) + database_list.should.have.length_of(2) + database_list[0].should.equal({"Name": database_name_1}) + database_list[1].should.equal({"Name": database_name_2}) + + @mock_glue def test_create_table(): client = boto3.client("glue", region_name="us-east-1") From 9381c670ab5d9ab169b071c4cf7580d04dfb4636 Mon Sep 17 00:00:00 2001 From: MarcosBernal Date: Tue, 21 Apr 2020 22:33:55 +0200 Subject: [PATCH 69/83] change code style to pass black --check --- moto/glue/responses.py | 4 +++- tests/test_glue/test_datacatalog.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/moto/glue/responses.py b/moto/glue/responses.py index 4fb144bba..66185e099 100644 --- a/moto/glue/responses.py +++ b/moto/glue/responses.py @@ -32,7 +32,9 @@ class GlueResponse(BaseResponse): def get_databases(self): database_list = self.glue_backend.get_databases() - return json.dumps({"DatabaseList": [{"Name": database.name} for database in database_list]}) + return json.dumps( + {"DatabaseList": [{"Name": database.name} for database in database_list]} + ) def create_table(self): database_name = self.parameters.get("DatabaseName") diff --git a/tests/test_glue/test_datacatalog.py b/tests/test_glue/test_datacatalog.py index 54fb17451..bc68b48f6 100644 --- a/tests/test_glue/test_datacatalog.py +++ b/tests/test_glue/test_datacatalog.py @@ -67,7 +67,9 @@ def test_get_databases_several_items(): helpers.create_database(client, database_name_1) helpers.create_database(client, database_name_2) - database_list = sorted(client.get_databases()["DatabaseList"], key=lambda x: x["Name"]) + database_list = sorted( + client.get_databases()["DatabaseList"], key=lambda x: x["Name"] + ) database_list.should.have.length_of(2) database_list[0].should.equal({"Name": database_name_1}) database_list[1].should.equal({"Name": database_name_2}) From 156ba56fdc94414e45ff83763d04fca91b111513 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 15 Apr 2019 19:57:42 -0500 Subject: [PATCH 70/83] set default status for s3 posts and add support for success_action_redirect. --- moto/s3/responses.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 22cd45c08..2f52e0d4a 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -776,8 +776,9 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): template = self.response_template(S3_DELETE_BUCKET_WITH_ITEMS_ERROR) return 409, {}, template.render(bucket=removed_bucket) - def _bucket_response_post(self, request, body, bucket_name): - if not request.headers.get("Content-Length"): + def _bucket_response_post(self, request, body, bucket_name, headers): + response_headers = {} + if not request.headers.get('Content-Length'): return 411, {}, "Content-Length required" path = self._get_path(request) @@ -810,13 +811,21 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): else: f = request.files["file"].stream.read() + if 'success_action_redirect' in form: + response_headers['Location'] = form['success_action_redirect'] + + if 'success_action_status' in form: + status_code = form['success_action_status'] + else: + status_code = 204 + new_key = self.backend.set_key(bucket_name, key, f) # Metadata metadata = metadata_from_headers(form) new_key.set_metadata(metadata) - return 200, {}, "" + return status_code, response_headers, "" @staticmethod def _get_path(request): From b3f6e5ab2fed73cfc9f66de92b16cfa52e3602bc Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 29 May 2019 15:22:29 -0500 Subject: [PATCH 71/83] add test --- moto/s3/responses.py | 2 ++ tests/test_s3/test_s3.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 2f52e0d4a..5526646a3 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -816,6 +816,8 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): if 'success_action_status' in form: status_code = form['success_action_status'] + elif 'success_action_redirect' in form: + status_code = 303 else: status_code = 204 diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 303ed523d..f7040e006 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -14,6 +14,7 @@ from io import BytesIO import mimetypes import zlib import pickle +import uuid import json import boto @@ -4428,3 +4429,34 @@ def test_s3_config_dict(): assert not logging_bucket["supplementaryConfiguration"].get( "BucketTaggingConfiguration" ) + + +@mock_s3 +def test_creating_presigned_post(): + bucket = 'presigned-test' + s3 = boto3.client('s3', region_name='us-east-1') + s3.create_bucket(Bucket=bucket) + success_url = 'http://localhost/completed' + fdata = b'test data\n' + file_uid = uuid.uuid4() + conditions = [ + {"Content-Type": 'text/plain'}, + {"x-amz-server-side-encryption": "AES256"}, + {'success_action_redirect': success_url}, + ] + conditions.append(["content-length-range", 1, 30]) + data = s3.generate_presigned_post( + Bucket=bucket, + Key='{file_uid}.txt'.format(file_uid=file_uid), + Fields={ + 'content-type': 'text/plain', + 'success_action_redirect': success_url, + 'x-amz-server-side-encryption': 'AES256' + }, + Conditions=conditions, + ExpiresIn=1000, + ) + resp = requests.post(data['url'], data=data['fields'], files={'file': fdata}, allow_redirects=False) + assert resp.headers['Location'] == url + assert resp.status_code == 303 + assert s3.get_object(Bucket=bucket, Key='{file_uuid}.txt'.format(file_uid=file_uid))['Body'].read() == fdata From 49b056563a2396727e17253d09e6924ce24ef09e Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 21 Apr 2020 19:51:48 -0500 Subject: [PATCH 72/83] process multipart form --- moto/s3/responses.py | 49 +++++++++++++++++++++++++++++----------- tests/test_s3/test_s3.py | 4 ++-- 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 5526646a3..92a82e4ff 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -7,7 +7,7 @@ import six from botocore.awsrequest import AWSPreparedRequest from moto.core.utils import str_to_rfc_1123_datetime, py2_strip_unicode_keys -from six.moves.urllib.parse import parse_qs, urlparse, unquote +from six.moves.urllib.parse import parse_qs, urlparse, unquote, parse_qsl import xmltodict @@ -143,6 +143,31 @@ def is_delete_keys(request, path, bucket_name): ) +def _process_multipart_formdata(request): + """ + When not using the live server, the request does not pass through flask, so it is not processed. + This will only be used in places where we end up with a requests PreparedRequest. + """ + form = {} + boundkey = request.headers['Content-Type'][len('multipart/form-data; boundary='):] + boundary = f'--{boundkey}' + data = request.body.decode().split(boundary) + fields = [field.split('\r\n\r\n') for field in data][1:-1] + for key, value in fields: + key, value = key.replace('\r\n', ''), value.replace('\r\n', '') + key = key.split('; ') + if len(key) == 2: + disposition, name = key + filename = None + else: + disposition, name, filename = key + name = name[len('name='):].strip('"') + if disposition.endswith('form-data'): + form[name] = value + import code; code.interact(local=locals()) + return form + + class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): def __init__(self, backend): super(ResponseObject, self).__init__() @@ -776,9 +801,9 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): template = self.response_template(S3_DELETE_BUCKET_WITH_ITEMS_ERROR) return 409, {}, template.render(bucket=removed_bucket) - def _bucket_response_post(self, request, body, bucket_name, headers): + def _bucket_response_post(self, request, body, bucket_name): response_headers = {} - if not request.headers.get('Content-Length'): + if not request.headers.get("Content-Length"): return 411, {}, "Content-Length required" path = self._get_path(request) @@ -796,14 +821,12 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): if hasattr(request, "form"): # Not HTTPretty form = request.form + elif request.headers.get('Content-Type').startswith('multipart/form-data'): + form = _process_multipart_formdata(request) else: # HTTPretty, build new form object body = body.decode() - - form = {} - for kv in body.split("&"): - k, v = kv.split("=") - form[k] = v + form = dict(parse_qsl(body)) key = form["key"] if "file" in form: @@ -811,12 +834,12 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): else: f = request.files["file"].stream.read() - if 'success_action_redirect' in form: - response_headers['Location'] = form['success_action_redirect'] + if "success_action_redirect" in form: + response_headers["Location"] = form["success_action_redirect"] - if 'success_action_status' in form: - status_code = form['success_action_status'] - elif 'success_action_redirect' in form: + if "success_action_status" in form: + status_code = form["success_action_status"] + elif "success_action_redirect" in form: status_code = 303 else: status_code = 204 diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index f7040e006..c226a7b3b 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -4457,6 +4457,6 @@ def test_creating_presigned_post(): ExpiresIn=1000, ) resp = requests.post(data['url'], data=data['fields'], files={'file': fdata}, allow_redirects=False) - assert resp.headers['Location'] == url + assert resp.headers['Location'] == success_url assert resp.status_code == 303 - assert s3.get_object(Bucket=bucket, Key='{file_uuid}.txt'.format(file_uid=file_uid))['Body'].read() == fdata + assert s3.get_object(Bucket=bucket, Key='{file_uid}.txt'.format(file_uid=file_uid))['Body'].read() == fdata From 4b0ba7320433b4b66488fc851c828f2ec1b56836 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 21 Apr 2020 20:13:53 -0500 Subject: [PATCH 73/83] use werkzeug hooray, thanks pallets discord! --- moto/s3/responses.py | 34 ++++++++-------------------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 92a82e4ff..965d15f57 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -5,6 +5,7 @@ import sys import six from botocore.awsrequest import AWSPreparedRequest +from werkzeug.wrappers import Request from moto.core.utils import str_to_rfc_1123_datetime, py2_strip_unicode_keys from six.moves.urllib.parse import parse_qs, urlparse, unquote, parse_qsl @@ -143,31 +144,6 @@ def is_delete_keys(request, path, bucket_name): ) -def _process_multipart_formdata(request): - """ - When not using the live server, the request does not pass through flask, so it is not processed. - This will only be used in places where we end up with a requests PreparedRequest. - """ - form = {} - boundkey = request.headers['Content-Type'][len('multipart/form-data; boundary='):] - boundary = f'--{boundkey}' - data = request.body.decode().split(boundary) - fields = [field.split('\r\n\r\n') for field in data][1:-1] - for key, value in fields: - key, value = key.replace('\r\n', ''), value.replace('\r\n', '') - key = key.split('; ') - if len(key) == 2: - disposition, name = key - filename = None - else: - disposition, name, filename = key - name = name[len('name='):].strip('"') - if disposition.endswith('form-data'): - form[name] = value - import code; code.interact(local=locals()) - return form - - class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): def __init__(self, backend): super(ResponseObject, self).__init__() @@ -822,7 +798,13 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): # Not HTTPretty form = request.form elif request.headers.get('Content-Type').startswith('multipart/form-data'): - form = _process_multipart_formdata(request) + request = Request.from_values( + input_stream=six.BytesIO(request.body), + content_length=request.headers['Content-Length'], + content_type=request.headers['Content-Type'], + method='POST', + ) + form = request.form else: # HTTPretty, build new form object body = body.decode() From 80b27a6b93d0c52d8f9f5349ec87efd036a66247 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 21 Apr 2020 21:43:32 -0500 Subject: [PATCH 74/83] blacken --- moto/s3/responses.py | 8 ++++---- tests/test_s3/test_s3.py | 33 ++++++++++++++++++++------------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 965d15f57..6ac139a14 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -797,12 +797,12 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): if hasattr(request, "form"): # Not HTTPretty form = request.form - elif request.headers.get('Content-Type').startswith('multipart/form-data'): + elif request.headers.get("Content-Type").startswith("multipart/form-data"): request = Request.from_values( input_stream=six.BytesIO(request.body), - content_length=request.headers['Content-Length'], - content_type=request.headers['Content-Type'], - method='POST', + content_length=request.headers["Content-Length"], + content_type=request.headers["Content-Type"], + method="POST", ) form = request.form else: diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index c226a7b3b..ffbd73966 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -4433,30 +4433,37 @@ def test_s3_config_dict(): @mock_s3 def test_creating_presigned_post(): - bucket = 'presigned-test' - s3 = boto3.client('s3', region_name='us-east-1') + bucket = "presigned-test" + s3 = boto3.client("s3", region_name="us-east-1") s3.create_bucket(Bucket=bucket) - success_url = 'http://localhost/completed' - fdata = b'test data\n' + success_url = "http://localhost/completed" + fdata = b"test data\n" file_uid = uuid.uuid4() conditions = [ - {"Content-Type": 'text/plain'}, + {"Content-Type": "text/plain"}, {"x-amz-server-side-encryption": "AES256"}, - {'success_action_redirect': success_url}, + {"success_action_redirect": success_url}, ] conditions.append(["content-length-range", 1, 30]) data = s3.generate_presigned_post( Bucket=bucket, - Key='{file_uid}.txt'.format(file_uid=file_uid), + Key="{file_uid}.txt".format(file_uid=file_uid), Fields={ - 'content-type': 'text/plain', - 'success_action_redirect': success_url, - 'x-amz-server-side-encryption': 'AES256' + "content-type": "text/plain", + "success_action_redirect": success_url, + "x-amz-server-side-encryption": "AES256", }, Conditions=conditions, ExpiresIn=1000, ) - resp = requests.post(data['url'], data=data['fields'], files={'file': fdata}, allow_redirects=False) - assert resp.headers['Location'] == success_url + resp = requests.post( + data["url"], data=data["fields"], files={"file": fdata}, allow_redirects=False + ) + assert resp.headers["Location"] == success_url assert resp.status_code == 303 - assert s3.get_object(Bucket=bucket, Key='{file_uid}.txt'.format(file_uid=file_uid))['Body'].read() == fdata + assert ( + s3.get_object(Bucket=bucket, Key="{file_uid}.txt".format(file_uid=file_uid))[ + "Body" + ].read() + == fdata + ) From 50a147592debbbb5e887d40d34ae146dbb266cdd Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 22 Apr 2020 09:08:30 -0500 Subject: [PATCH 75/83] Make all CallbackResponse requests into a Werkzeug Request The "request" object in CallbackResponse is the PreparedRequest send by whatever client is used to contact the mocked moto service. This can end up with unparsed bodies, as we added for processing presigned post requests in #2155. This will make sure that all of the requests comming in from mocked functions also get processed by werkzeug as if it was running a live server. --- moto/core/models.py | 20 ++++++++++++++++++++ moto/s3/responses.py | 9 --------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/moto/core/models.py b/moto/core/models.py index 73942c669..460823bd6 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -12,6 +12,8 @@ from io import BytesIO from collections import defaultdict from botocore.handlers import BUILTIN_HANDLERS from botocore.awsrequest import AWSResponse +from six.moves.urllib.parse import urlparse +from werkzeug.wrappers import Request import mock from moto import settings @@ -175,6 +177,24 @@ class CallbackResponse(responses.CallbackResponse): """ Need to override this so we can pass decode_content=False """ + if not isinstance(request, Request): + url = urlparse(request.url) + if request.body is None: + body = None + elif isinstance(request.body, six.text_type): + body = six.BytesIO(six.b(request.body)) + else: + body = six.BytesIO(request.body) + req = Request.from_values( + path='?'.join([url.path, url.query]), + input_stream=body, + content_length=request.headers.get("Content-Length"), + content_type=request.headers.get("Content-Type"), + method=request.method, + base_url='{scheme}://{netloc}'.format(scheme=url.scheme, netloc=url.netloc), + headers=[(k, v) for k, v in six.iteritems(request.headers)] + ) + request = req headers = self.get_headers() result = self.callback(request) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 6ac139a14..442489a8a 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -5,7 +5,6 @@ import sys import six from botocore.awsrequest import AWSPreparedRequest -from werkzeug.wrappers import Request from moto.core.utils import str_to_rfc_1123_datetime, py2_strip_unicode_keys from six.moves.urllib.parse import parse_qs, urlparse, unquote, parse_qsl @@ -797,14 +796,6 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): if hasattr(request, "form"): # Not HTTPretty form = request.form - elif request.headers.get("Content-Type").startswith("multipart/form-data"): - request = Request.from_values( - input_stream=six.BytesIO(request.body), - content_length=request.headers["Content-Length"], - content_type=request.headers["Content-Type"], - method="POST", - ) - form = request.form else: # HTTPretty, build new form object body = body.decode() From 4cd2b201b5cb07165816b1e3a0c453e7a5410d1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcos=20Bernal=20Espa=C3=B1a?= Date: Wed, 22 Apr 2020 16:44:25 +0200 Subject: [PATCH 76/83] Update IMPLEMENTATION_COVERAGE.md Co-Authored-By: Bert Blommers --- IMPLEMENTATION_COVERAGE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 82ee2f046..78c7ba0e4 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3351,7 +3351,7 @@ - [ ] update_listener ## glue -14/123 = 11% implemented +11% implemented - [X] batch_create_partition - [ ] batch_delete_connection - [X] batch_delete_partition From d9e2aeed5856ef762779d8920572398a1ed6c4c1 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 22 Apr 2020 10:02:25 -0500 Subject: [PATCH 77/83] blacken --- moto/core/models.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/moto/core/models.py b/moto/core/models.py index 460823bd6..1ee11607a 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -186,13 +186,15 @@ class CallbackResponse(responses.CallbackResponse): else: body = six.BytesIO(request.body) req = Request.from_values( - path='?'.join([url.path, url.query]), + path="?".join([url.path, url.query]), input_stream=body, content_length=request.headers.get("Content-Length"), content_type=request.headers.get("Content-Type"), method=request.method, - base_url='{scheme}://{netloc}'.format(scheme=url.scheme, netloc=url.netloc), - headers=[(k, v) for k, v in six.iteritems(request.headers)] + base_url="{scheme}://{netloc}".format( + scheme=url.scheme, netloc=url.netloc + ), + headers=[(k, v) for k, v in six.iteritems(request.headers)], ) request = req headers = self.get_headers() From 343b20a5fbfebac4cebf3f1dbd6e794084fb65fe Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 22 Apr 2020 16:36:41 +0100 Subject: [PATCH 78/83] Update CONTRIBUTING to add Linting info --- CONTRIBUTING.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 40da55ccf..941fc0624 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,10 @@ Moto has a [Code of Conduct](https://github.com/spulec/moto/blob/master/CODE_OF_ ## Running the tests locally -Moto has a Makefile which has some helpful commands for getting setup. You should be able to run `make init` to install the dependencies and then `make test` to run the tests. +Moto has a Makefile which has some helpful commands for getting setup. You should be able to run `make init` to install the dependencies and then `make test` to run the tests. + +## Linting +Run `make lint` or `black --check moto tests` to verify whether your code confirms to the guidelines. ## Is there a missing feature? From 194de2b6eaf5d884b6bbce8856d6f0e21eb45149 Mon Sep 17 00:00:00 2001 From: Asher Foa <1268088+asherf@users.noreply.github.com> Date: Wed, 22 Apr 2020 13:32:12 -0700 Subject: [PATCH 79/83] Add af-south-1 The new version of botcore adds this region: https://github.com/boto/botocore/commit/f7dc4730ad34c6c3322da7d43ba64452bb3ae0d8#diff-9dfab05d4ba739e097a193e8b5fa61caR13 Which in turn, breaks moto: ``` /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/__init__.py:3: in from .acm import mock_acm # noqa /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/acm/__init__.py:2: in from .models import acm_backends /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/acm/models.py:7: in from moto.ec2 import ec2_backends /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/__init__.py:2: in from .models import ec2_backends /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:5169: in ec2_backends = { /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:5170: in region.name: EC2Backend(region.name) for region in RegionsAndZonesBackend.regions /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:5083: in __init__ super(EC2Backend, self).__init__() /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:852: in __init__ super(InstanceBackend, self).__init__() /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:1127: in __init__ super(TagBackend, self).__init__() /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:2368: in __init__ super(EBSBackend, self).__init__() /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:1904: in __init__ super(SecurityGroupBackend, self).__init__() /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:1355: in __init__ self._load_amis() /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:1362: in _load_amis self.amis[ami_id] = Ami(self, **ami) /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:1313: in __init__ volume = self.ec2_backend.create_volume(15, region_name) /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:2372: in create_volume zone = self.get_zone_by_name(zone_name) /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:1713: in get_zone_by_name for zone in self.zones[self.region_name]: E KeyError: 'af-south-1' ``` --- moto/ec2/models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index ee9b0fcc4..dc8e617e0 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -1512,6 +1512,9 @@ class RegionsAndZonesBackend(object): regions.append(Region(region, "ec2.{}.amazonaws.com.cn".format(region))) zones = { + "af-south-1": [ + Zone(region_name="af-south-1", name="af-south-1a", zone_id="afs1-az1"), + ], "ap-south-1": [ Zone(region_name="ap-south-1", name="ap-south-1a", zone_id="aps1-az1"), Zone(region_name="ap-south-1", name="ap-south-1b", zone_id="aps1-az3"), From 1d31ea6397ef2349ea89ba481b1c2af23bde9d05 Mon Sep 17 00:00:00 2001 From: Asher Foa <1268088+asherf@users.noreply.github.com> Date: Thu, 23 Apr 2020 08:25:14 -0700 Subject: [PATCH 80/83] add two more zones. --- moto/ec2/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index dc8e617e0..332c8f030 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -1514,6 +1514,8 @@ class RegionsAndZonesBackend(object): zones = { "af-south-1": [ Zone(region_name="af-south-1", name="af-south-1a", zone_id="afs1-az1"), + Zone(region_name="af-south-1", name="af-south-1b", zone_id="afs1-az2"), + Zone(region_name="af-south-1", name="af-south-1c", zone_id="afs1-az3"), ], "ap-south-1": [ Zone(region_name="ap-south-1", name="ap-south-1a", zone_id="aps1-az1"), From a658900d69ca4ae36a4b265161809a529aabb211 Mon Sep 17 00:00:00 2001 From: JohnWC Date: Sat, 25 Apr 2020 03:13:36 -0500 Subject: [PATCH 81/83] Add policy to apigateway --- moto/apigateway/models.py | 4 ++++ moto/apigateway/responses.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index e5e5e3bfd..e011af601 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -461,6 +461,7 @@ class RestAPI(BaseModel): self.description = description self.create_date = int(time.time()) self.api_key_source = kwargs.get("api_key_source") or "HEADER" + self.policy = kwargs.get("policy") or None self.endpoint_configuration = kwargs.get("endpoint_configuration") or { "types": ["EDGE"] } @@ -485,6 +486,7 @@ class RestAPI(BaseModel): "apiKeySource": self.api_key_source, "endpointConfiguration": self.endpoint_configuration, "tags": self.tags, + "policy": self.policy, } def add_child(self, path, parent_id=None): @@ -713,6 +715,7 @@ class APIGatewayBackend(BaseBackend): api_key_source=None, endpoint_configuration=None, tags=None, + policy=None, ): api_id = create_id() rest_api = RestAPI( @@ -723,6 +726,7 @@ class APIGatewayBackend(BaseBackend): api_key_source=api_key_source, endpoint_configuration=endpoint_configuration, tags=tags, + policy=policy, ) self.apis[api_id] = rest_api return rest_api diff --git a/moto/apigateway/responses.py b/moto/apigateway/responses.py index 822d4c0ce..a3c41a6d4 100644 --- a/moto/apigateway/responses.py +++ b/moto/apigateway/responses.py @@ -59,6 +59,7 @@ class APIGatewayResponse(BaseResponse): api_key_source = self._get_param("apiKeySource") endpoint_configuration = self._get_param("endpointConfiguration") tags = self._get_param("tags") + policy = self._get_param("policy") # Param validation if api_key_source and api_key_source not in API_KEY_SOURCES: @@ -94,6 +95,7 @@ class APIGatewayResponse(BaseResponse): api_key_source=api_key_source, endpoint_configuration=endpoint_configuration, tags=tags, + policy=policy, ) return 200, {}, json.dumps(rest_api.to_dict()) From 0828c5af9dfff7430537cfb26cc62a8523d9cef3 Mon Sep 17 00:00:00 2001 From: JohnWC Date: Sat, 25 Apr 2020 03:27:59 -0500 Subject: [PATCH 82/83] Add unit test for add apigateway with policy --- tests/test_apigateway/test_apigateway.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_apigateway/test_apigateway.py b/tests/test_apigateway/test_apigateway.py index 596ed2dd4..107dc5d05 100644 --- a/tests/test_apigateway/test_apigateway.py +++ b/tests/test_apigateway/test_apigateway.py @@ -69,6 +69,24 @@ def test_create_rest_api_with_tags(): response["tags"].should.equal({"MY_TAG1": "MY_VALUE1"}) +@mock_apigateway +def test_create_rest_api_with_policy(): + client = boto3.client("apigateway", region_name="us-west-2") + + policy = "{\"Version\": \"2012-10-17\",\"Statement\": []}" + response = client.create_rest_api( + name="my_api", + description="this is my api", + policy=policy + ) + api_id = response["id"] + + response = client.get_rest_api(restApiId=api_id) + + assert "policy" in response + response["policy"].should.equal(policy) + + @mock_apigateway def test_create_rest_api_invalid_apikeysource(): client = boto3.client("apigateway", region_name="us-west-2") From 4a800d8f2c8677b098ea0a2c41deface8236c267 Mon Sep 17 00:00:00 2001 From: JohnWC Date: Sat, 25 Apr 2020 11:24:54 -0500 Subject: [PATCH 83/83] Updated for black --- tests/test_apigateway/test_apigateway.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_apigateway/test_apigateway.py b/tests/test_apigateway/test_apigateway.py index 107dc5d05..b04328a03 100644 --- a/tests/test_apigateway/test_apigateway.py +++ b/tests/test_apigateway/test_apigateway.py @@ -73,11 +73,9 @@ def test_create_rest_api_with_tags(): def test_create_rest_api_with_policy(): client = boto3.client("apigateway", region_name="us-west-2") - policy = "{\"Version\": \"2012-10-17\",\"Statement\": []}" + policy = '{"Version": "2012-10-17","Statement": []}' response = client.create_rest_api( - name="my_api", - description="this is my api", - policy=policy + name="my_api", description="this is my api", policy=policy ) api_id = response["id"]