From 79f0cc9e9e845461c337d1d297727483cfd0b651 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 4 Aug 2021 17:24:26 +0100 Subject: [PATCH] Tech debt - remove dead DynamoDB code and add RDS tests (#4136) --- moto/autoscaling/models.py | 2 +- moto/cloudformation/models.py | 2 +- moto/cloudformation/parsing.py | 2 +- moto/cognitoidentity/models.py | 2 +- moto/cognitoidp/models.py | 2 +- moto/compat.py | 16 --- moto/core/models.py | 2 +- moto/core/responses.py | 2 +- moto/datapipeline/models.py | 2 +- moto/datapipeline/utils.py | 2 +- moto/datasync/models.py | 2 +- moto/dynamodb/models.py | 2 +- moto/dynamodb2/models/__init__.py | 12 +- moto/dynamodb2/models/dynamo_type.py | 77 +----------- moto/dynamodb2/models/utilities.py | 15 --- moto/ec2/models.py | 2 +- moto/elb/models.py | 2 +- moto/elbv2/models.py | 2 +- moto/glue/models.py | 2 +- moto/kinesis/models.py | 2 +- moto/rds2/models.py | 2 +- moto/rds2/utils.py | 2 +- moto/redshift/models.py | 2 +- moto/sns/models.py | 2 +- tests/compat.py | 5 - .../test_cloudformation/test_stack_parsing.py | 2 +- tests/test_core/test_responses.py | 2 +- tests/test_core/test_server.py | 2 +- tests/test_rds2/test_rds2_cloudformation.py | 114 ++++++++++++++++++ tests/test_s3/test_s3_utils.py | 2 +- tests/test_s3/test_server.py | 2 +- 31 files changed, 143 insertions(+), 146 deletions(-) delete mode 100644 moto/compat.py delete mode 100644 tests/compat.py create mode 100644 tests/test_rds2/test_rds2_cloudformation.py diff --git a/moto/autoscaling/models.py b/moto/autoscaling/models.py index f90c5de5e..1fd09d928 100644 --- a/moto/autoscaling/models.py +++ b/moto/autoscaling/models.py @@ -8,7 +8,7 @@ from moto.packages.boto.ec2.blockdevicemapping import ( ) from moto.ec2.exceptions import InvalidInstanceIdError -from moto.compat import OrderedDict +from collections import OrderedDict from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.core.utils import camelcase_to_underscores from moto.ec2 import ec2_backends diff --git a/moto/cloudformation/models.py b/moto/cloudformation/models.py index a21f420ab..e631a4d6a 100644 --- a/moto/cloudformation/models.py +++ b/moto/cloudformation/models.py @@ -6,7 +6,7 @@ import uuid from boto3 import Session -from moto.compat import OrderedDict +from collections import OrderedDict from moto.core import BaseBackend, BaseModel from moto.core.utils import iso_8601_datetime_without_milliseconds diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index bcc8b556a..46c6e154b 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -6,7 +6,7 @@ import copy import warnings import re -from moto.compat import collections_abc +import collections.abc as collections_abc # This ugly section of imports is necessary because we # build the list of CloudFormationModel subclasses using diff --git a/moto/cognitoidentity/models.py b/moto/cognitoidentity/models.py index 282dd0a37..2558e4ba0 100644 --- a/moto/cognitoidentity/models.py +++ b/moto/cognitoidentity/models.py @@ -5,7 +5,7 @@ import json from boto3 import Session -from moto.compat import OrderedDict +from collections import OrderedDict from moto.core import BaseBackend, BaseModel from moto.core.utils import iso_8601_datetime_with_milliseconds from .exceptions import ResourceNotFoundError diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 860da9563..32d1afc6d 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -10,7 +10,7 @@ import time import uuid from boto3 import Session from jose import jws -from moto.compat import OrderedDict +from collections import OrderedDict from moto.core import BaseBackend, BaseModel from moto.core import ACCOUNT_ID as DEFAULT_ACCOUNT_ID from .exceptions import ( diff --git a/moto/compat.py b/moto/compat.py deleted file mode 100644 index 311b50273..000000000 --- a/moto/compat.py +++ /dev/null @@ -1,16 +0,0 @@ -try: - from collections import OrderedDict # noqa -except ImportError: - # python 2.6 or earlier, use backport - from ordereddict import OrderedDict # noqa - -try: - import collections.abc as collections_abc # noqa -except ImportError: - import collections as collections_abc # noqa - -try: - from unittest.mock import patch # noqa -except ImportError: - # for python 2.7 - from mock import patch # noqa diff --git a/moto/core/models.py b/moto/core/models.py index 3c7e8ad0a..06ec14d51 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -22,7 +22,7 @@ from werkzeug.wrappers import Request from moto import settings import responses from moto.packages.httpretty import HTTPretty -from moto.compat import patch +from unittest.mock import patch from .utils import ( convert_httpretty_response, convert_regex_to_flask_path, diff --git a/moto/core/responses.py b/moto/core/responses.py index a47d013ea..9765b04b4 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -21,7 +21,7 @@ import xmltodict from werkzeug.exceptions import HTTPException import boto3 -from moto.compat import OrderedDict +from collections import OrderedDict from moto.core.utils import camelcase_to_underscores, method_names_from_class from moto import settings diff --git a/moto/datapipeline/models.py b/moto/datapipeline/models.py index e517b8f3e..ed1996329 100644 --- a/moto/datapipeline/models.py +++ b/moto/datapipeline/models.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import datetime from boto3 import Session -from moto.compat import OrderedDict +from collections import OrderedDict from moto.core import BaseBackend, BaseModel, CloudFormationModel from .utils import get_random_pipeline_id, remove_capitalization_of_dict_keys diff --git a/moto/datapipeline/utils.py b/moto/datapipeline/utils.py index c3c6820a2..cb9c5e46c 100644 --- a/moto/datapipeline/utils.py +++ b/moto/datapipeline/utils.py @@ -1,4 +1,4 @@ -from moto.compat import collections_abc +import collections.abc as collections_abc from moto.core.utils import get_random_hex diff --git a/moto/datasync/models.py b/moto/datasync/models.py index 702cace5b..ed60a5946 100644 --- a/moto/datasync/models.py +++ b/moto/datasync/models.py @@ -1,6 +1,6 @@ from boto3 import Session -from moto.compat import OrderedDict +from collections import OrderedDict from moto.core import BaseBackend, BaseModel from .exceptions import InvalidRequestException diff --git a/moto/dynamodb/models.py b/moto/dynamodb/models.py index 1a3b4afce..7349a744d 100644 --- a/moto/dynamodb/models.py +++ b/moto/dynamodb/models.py @@ -3,7 +3,7 @@ from collections import defaultdict import datetime import json -from moto.compat import OrderedDict +from collections import OrderedDict from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.core.utils import unix_time from moto.core import ACCOUNT_ID diff --git a/moto/dynamodb2/models/__init__.py b/moto/dynamodb2/models/__init__.py index 373f932d3..e9968e5c1 100644 --- a/moto/dynamodb2/models/__init__.py +++ b/moto/dynamodb2/models/__init__.py @@ -8,7 +8,7 @@ import re import uuid from boto3 import Session -from moto.compat import OrderedDict +from collections import OrderedDict from moto.core import ACCOUNT_ID from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.core.utils import unix_time, unix_time_millis @@ -958,16 +958,6 @@ class Table(CloudFormationModel): return results, last_evaluated_key - def lookup(self, *args, **kwargs): - if not self.schema: - self.describe() - for x, arg in enumerate(args): - kwargs[self.schema[x].name] = arg - ret = self.get_item(**kwargs) - if not ret.keys(): - return None - return ret - def delete(self, region_name): dynamodb_backends[region_name].delete_table(self.name) diff --git a/moto/dynamodb2/models/dynamo_type.py b/moto/dynamodb2/models/dynamo_type.py index 5a8fa93eb..f5da843d2 100644 --- a/moto/dynamodb2/models/dynamo_type.py +++ b/moto/dynamodb2/models/dynamo_type.py @@ -1,6 +1,6 @@ from moto.dynamodb2.comparisons import get_comparison_func -from moto.dynamodb2.exceptions import InvalidUpdateExpression, IncorrectDataType -from moto.dynamodb2.models.utilities import attribute_is_list, bytesize +from moto.dynamodb2.exceptions import IncorrectDataType +from moto.dynamodb2.models.utilities import bytesize class DDBType(object): @@ -36,14 +36,7 @@ class DDBTypeConversion(object): Returns: str: The human readable form of the DDBType. """ - try: - human_type_str = cls._human_type_mapping[abbreviated_type] - except KeyError: - raise ValueError( - "Invalid abbreviated_type {at}".format(at=abbreviated_type) - ) - - return human_type_str + return cls._human_type_mapping.get(abbreviated_type, abbreviated_type) class DynamoType(object): @@ -63,70 +56,6 @@ class DynamoType(object): elif self.is_map(): self.value = dict((k, DynamoType(v)) for k, v in self.value.items()) - def get(self, key): - if not key: - return self - else: - key_head = key.split(".")[0] - key_tail = ".".join(key.split(".")[1:]) - if key_head not in self.value: - self.value[key_head] = DynamoType({"NONE": None}) - return self.value[key_head].get(key_tail) - - def set(self, key, new_value, index=None): - if index: - index = int(index) - if type(self.value) is not list: - raise InvalidUpdateExpression - if index >= len(self.value): - self.value.append(new_value) - # {'L': [DynamoType, ..]} ==> DynamoType.set() - self.value[min(index, len(self.value) - 1)].set(key, new_value) - else: - attr = (key or "").split(".").pop(0) - attr, list_index = attribute_is_list(attr) - if not key: - # {'S': value} ==> {'S': new_value} - self.type = new_value.type - self.value = new_value.value - else: - if attr not in self.value: # nonexistingattribute - type_of_new_attr = DDBType.MAP if "." in key else new_value.type - self.value[attr] = DynamoType({type_of_new_attr: {}}) - # {'M': {'foo': DynamoType}} ==> DynamoType.set(new_value) - self.value[attr].set( - ".".join(key.split(".")[1:]), new_value, list_index - ) - - def __contains__(self, item): - if self.type == DDBType.STRING: - return False - try: - self.__getitem__(item) - return True - except KeyError: - return False - - def delete(self, key, index=None): - if index: - if not key: - if int(index) < len(self.value): - del self.value[int(index)] - elif "." in key: - self.value[int(index)].delete(".".join(key.split(".")[1:])) - else: - self.value[int(index)].delete(key) - else: - attr = key.split(".")[0] - attr, list_index = attribute_is_list(attr) - - if list_index: - self.value[attr].delete(".".join(key.split(".")[1:]), list_index) - elif "." in key: - self.value[attr].delete(".".join(key.split(".")[1:])) - else: - self.value.pop(key) - def filter(self, projection_expressions): nested_projections = [ expr[0 : expr.index(".")] for expr in projection_expressions if "." in expr diff --git a/moto/dynamodb2/models/utilities.py b/moto/dynamodb2/models/utilities.py index 839087dce..28c6676f5 100644 --- a/moto/dynamodb2/models/utilities.py +++ b/moto/dynamodb2/models/utilities.py @@ -1,17 +1,2 @@ -import re - - def bytesize(val): return len(val.encode("utf-8")) - - -def attribute_is_list(attr): - """ - Checks if attribute denotes a list, and returns the name of the list and the given list index if so - :param attr: attr or attr[index] - :return: attr, index or None - """ - list_index_update = re.match("(.+)\\[([0-9]+)\\]", attr) - if list_index_update: - attr = list_index_update.group(1) - return attr, list_index_update.group(2) if list_index_update else None diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 920825bc1..0546f790a 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -24,7 +24,7 @@ from moto.packages.boto.ec2.spotinstancerequest import ( ) from moto.packages.boto.ec2.launchspecification import LaunchSpecification -from moto.compat import OrderedDict +from collections import OrderedDict from moto.core import BaseBackend from moto.core.models import Model, BaseModel, CloudFormationModel from moto.core.utils import ( diff --git a/moto/elb/models.py b/moto/elb/models.py index bb245a0fb..4336d4961 100644 --- a/moto/elb/models.py +++ b/moto/elb/models.py @@ -12,7 +12,7 @@ from moto.packages.boto.ec2.elb.attributes import ( CrossZoneLoadBalancingAttribute, ) from moto.packages.boto.ec2.elb.policies import Policies, OtherPolicy -from moto.compat import OrderedDict +from collections import OrderedDict from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.ec2.models import ec2_backends from .exceptions import ( diff --git a/moto/elbv2/models.py b/moto/elbv2/models.py index 8b7d79bfa..a3c60d40f 100644 --- a/moto/elbv2/models.py +++ b/moto/elbv2/models.py @@ -4,7 +4,7 @@ import datetime import re from jinja2 import Template from botocore.exceptions import ParamValidationError -from moto.compat import OrderedDict +from collections import OrderedDict from moto.core.exceptions import RESTError from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.core.utils import ( diff --git a/moto/glue/models.py b/moto/glue/models.py index a434628b7..d3e730a5d 100644 --- a/moto/glue/models.py +++ b/moto/glue/models.py @@ -4,7 +4,7 @@ import time from datetime import datetime from moto.core import BaseBackend, BaseModel -from moto.compat import OrderedDict +from collections import OrderedDict from .exceptions import ( JsonRESTError, DatabaseAlreadyExistsException, diff --git a/moto/kinesis/models.py b/moto/kinesis/models.py index a385a0312..3e42a1487 100644 --- a/moto/kinesis/models.py +++ b/moto/kinesis/models.py @@ -10,7 +10,7 @@ from hashlib import md5 from boto3 import Session -from moto.compat import OrderedDict +from collections import OrderedDict from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.core.utils import unix_time from moto.core import ACCOUNT_ID diff --git a/moto/rds2/models.py b/moto/rds2/models.py index 7c1e14b1b..04f9f72eb 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -8,7 +8,7 @@ from collections import defaultdict from boto3 import Session from jinja2 import Template from re import compile as re_compile -from moto.compat import OrderedDict +from collections import OrderedDict from moto.core import BaseBackend, BaseModel, CloudFormationModel, ACCOUNT_ID from moto.core.utils import iso_8601_datetime_with_milliseconds diff --git a/moto/rds2/utils.py b/moto/rds2/utils.py index 594a2daab..90937a44c 100644 --- a/moto/rds2/utils.py +++ b/moto/rds2/utils.py @@ -5,7 +5,7 @@ from collections import namedtuple from botocore.utils import merge_dicts -from moto.compat import OrderedDict +from collections import OrderedDict FilterDef = namedtuple( "FilterDef", diff --git a/moto/redshift/models.py b/moto/redshift/models.py index 4ff3aeb15..df8fcab44 100644 --- a/moto/redshift/models.py +++ b/moto/redshift/models.py @@ -5,7 +5,7 @@ import datetime from boto3 import Session -from moto.compat import OrderedDict +from collections import OrderedDict from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.core.utils import iso_8601_datetime_with_milliseconds from moto.utilities.utils import random_string diff --git a/moto/sns/models.py b/moto/sns/models.py index d46ce86de..f460431f9 100644 --- a/moto/sns/models.py +++ b/moto/sns/models.py @@ -9,7 +9,7 @@ import re from boto3 import Session -from moto.compat import OrderedDict +from collections import OrderedDict from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.core.utils import ( iso_8601_datetime_with_milliseconds, diff --git a/tests/compat.py b/tests/compat.py deleted file mode 100644 index 4af2156f7..000000000 --- a/tests/compat.py +++ /dev/null @@ -1,5 +0,0 @@ -try: - from unittest.mock import patch -except ImportError: - # for python 2.7 - from mock import patch diff --git a/tests/test_cloudformation/test_stack_parsing.py b/tests/test_cloudformation/test_stack_parsing.py index 843b1b22c..81a46dae6 100644 --- a/tests/test_cloudformation/test_stack_parsing.py +++ b/tests/test_cloudformation/test_stack_parsing.py @@ -4,7 +4,7 @@ import json import yaml import sure # noqa -from tests.compat import patch +from unittest.mock import patch from moto.cloudformation.exceptions import ValidationError from moto.cloudformation.models import FakeStack diff --git a/tests/test_core/test_responses.py b/tests/test_core/test_responses.py index 73440dacc..2e4b8e300 100644 --- a/tests/test_core/test_responses.py +++ b/tests/test_core/test_responses.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals import sure # noqa -from moto.compat import OrderedDict +from collections import OrderedDict from botocore.awsrequest import AWSPreparedRequest diff --git a/tests/test_core/test_server.py b/tests/test_core/test_server.py index 905b662d5..d25270700 100644 --- a/tests/test_core/test_server.py +++ b/tests/test_core/test_server.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals import sure # noqa -from tests.compat import patch +from unittest.mock import patch from moto.server import main, create_backend_app, DomainDispatcherApplication diff --git a/tests/test_rds2/test_rds2_cloudformation.py b/tests/test_rds2/test_rds2_cloudformation.py new file mode 100644 index 000000000..a3554aa43 --- /dev/null +++ b/tests/test_rds2/test_rds2_cloudformation.py @@ -0,0 +1,114 @@ +import boto3 +import json +import sure # noqa +from moto import mock_cloudformation, mock_ec2, mock_rds2 + + +@mock_ec2 +@mock_rds2 +@mock_cloudformation +def test_create_subnetgroup_via_cf(): + vpc_conn = boto3.client("ec2", "us-west-2") + vpc = vpc_conn.create_vpc(CidrBlock="10.0.0.0/16")["Vpc"] + subnet = vpc_conn.create_subnet(VpcId=vpc["VpcId"], CidrBlock="10.0.1.0/24")[ + "Subnet" + ] + + rds = boto3.client("rds", region_name="us-west-2") + cf = boto3.client("cloudformation", region_name="us-west-2") + + template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "subnet": { + "Type": "AWS::RDS::DBSubnetGroup", + "Properties": { + "DBSubnetGroupName": "subnetgroupname", + "DBSubnetGroupDescription": "subnetgroupdesc", + "SubnetIds": [subnet["SubnetId"]], + }, + } + }, + } + template_json = json.dumps(template) + cf.create_stack(StackName="test_stack", TemplateBody=template_json) + + response = rds.describe_db_subnet_groups()["DBSubnetGroups"] + response.should.have.length_of(1) + + created_subnet = response[0] + created_subnet.should.have.key("DBSubnetGroupName").equal("subnetgroupname") + created_subnet.should.have.key("DBSubnetGroupDescription").equal("subnetgroupdesc") + created_subnet.should.have.key("VpcId").equal(vpc["VpcId"]) + + +@mock_ec2 +@mock_rds2 +@mock_cloudformation +def test_create_dbinstance_via_cf(): + vpc_conn = boto3.client("ec2", "us-west-2") + vpc = vpc_conn.create_vpc(CidrBlock="10.0.0.0/16")["Vpc"] + vpc_conn.create_subnet(VpcId=vpc["VpcId"], CidrBlock="10.0.1.0/24") + + rds = boto3.client("rds", region_name="us-west-2") + cf = boto3.client("cloudformation", region_name="us-west-2") + + template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "db": { + "Type": "AWS::RDS::DBInstance", + "Properties": { + "Port": 3307, + "Engine": "mysql", + # Required - throws exception when describing an instance without tags + "Tags": [], + }, + } + }, + } + template_json = json.dumps(template) + cf.create_stack(StackName="test_stack", TemplateBody=template_json) + + summaries = cf.list_stack_resources(StackName="test_stack")[ + "StackResourceSummaries" + ] + + db_instance_identifier = summaries[0]["PhysicalResourceId"] + resp = rds.describe_db_instances()["DBInstances"] + resp.should.have.length_of(1) + + created = resp[0] + created["DBInstanceIdentifier"].should.equal(db_instance_identifier) + created["Engine"].should.equal("mysql") + created["DBInstanceStatus"].should.equal("available") + + +@mock_ec2 +@mock_rds2 +@mock_cloudformation +def test_create_dbsecuritygroup_via_cf(): + vpc_conn = boto3.client("ec2", "us-west-2") + vpc = vpc_conn.create_vpc(CidrBlock="10.0.0.0/16")["Vpc"] + vpc_conn.create_subnet(VpcId=vpc["VpcId"], CidrBlock="10.0.1.0/24") + + rds = boto3.client("rds", region_name="us-west-2") + cf = boto3.client("cloudformation", region_name="us-west-2") + + template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "db": { + "Type": "AWS::RDS::DBSecurityGroup", + "Properties": {"GroupDescription": "my sec group"}, + } + }, + } + template_json = json.dumps(template) + cf.create_stack(StackName="test_stack", TemplateBody=template_json) + + result = rds.describe_db_security_groups()["DBSecurityGroups"] + result.should.have.length_of(1) + + created = result[0] + created["DBSecurityGroupDescription"].should.equal("my sec group") diff --git a/tests/test_s3/test_s3_utils.py b/tests/test_s3/test_s3_utils.py index 37fed92ff..0c14d8860 100644 --- a/tests/test_s3/test_s3_utils.py +++ b/tests/test_s3/test_s3_utils.py @@ -8,7 +8,7 @@ from moto.s3.utils import ( clean_key_name, undo_clean_key_name, ) -from tests.compat import patch +from unittest.mock import patch def test_base_url(): diff --git a/tests/test_s3/test_server.py b/tests/test_s3/test_server.py index dfdfdc2f8..4d85ea5bf 100644 --- a/tests/test_s3/test_server.py +++ b/tests/test_s3/test_server.py @@ -7,7 +7,7 @@ import sure # noqa from flask.testing import FlaskClient import moto.server as server -from tests.compat import patch +from unittest.mock import patch """ Test the different server responses