Tech debt - remove dead DynamoDB code and add RDS tests (#4136)

moto/autoscaling/models.py

@@ -8,7 +8,7 @@ from moto.packages.boto.ec2.blockdevicemapping import (
 )
 from moto.ec2.exceptions import InvalidInstanceIdError
 
-from moto.compat import OrderedDict
+from collections import OrderedDict
 from moto.core import BaseBackend, BaseModel, CloudFormationModel
 from moto.core.utils import camelcase_to_underscores
 from moto.ec2 import ec2_backends

moto/cloudformation/models.py

@@ -6,7 +6,7 @@ import uuid
 
 from boto3 import Session
 
-from moto.compat import OrderedDict
+from collections import OrderedDict
 from moto.core import BaseBackend, BaseModel
 from moto.core.utils import iso_8601_datetime_without_milliseconds
 

moto/cloudformation/parsing.py

@@ -6,7 +6,7 @@ import copy
 import warnings
 import re
 
-from moto.compat import collections_abc
+import collections.abc as collections_abc
 
 # This ugly section of imports is necessary because we
 # build the list of CloudFormationModel subclasses using

moto/cognitoidentity/models.py

@@ -5,7 +5,7 @@ import json
 
 from boto3 import Session
 
-from moto.compat import OrderedDict
+from collections import OrderedDict
 from moto.core import BaseBackend, BaseModel
 from moto.core.utils import iso_8601_datetime_with_milliseconds
 from .exceptions import ResourceNotFoundError

moto/cognitoidp/models.py

@@ -10,7 +10,7 @@ import time
 import uuid
 from boto3 import Session
 from jose import jws
-from moto.compat import OrderedDict
+from collections import OrderedDict
 from moto.core import BaseBackend, BaseModel
 from moto.core import ACCOUNT_ID as DEFAULT_ACCOUNT_ID
 from .exceptions import (

moto/compat.py

@@ -1,16 +0,0 @@
-try:
-    from collections import OrderedDict  # noqa
-except ImportError:
-    # python 2.6 or earlier, use backport
-    from ordereddict import OrderedDict  # noqa
-
-try:
-    import collections.abc as collections_abc  # noqa
-except ImportError:
-    import collections as collections_abc  # noqa
-
-try:
-    from unittest.mock import patch  # noqa
-except ImportError:
-    # for python 2.7
-    from mock import patch  # noqa

moto/core/models.py

@@ -22,7 +22,7 @@ from werkzeug.wrappers import Request
 from moto import settings
 import responses
 from moto.packages.httpretty import HTTPretty
-from moto.compat import patch
+from unittest.mock import patch
 from .utils import (
     convert_httpretty_response,
     convert_regex_to_flask_path,

moto/core/responses.py

@@ -21,7 +21,7 @@ import xmltodict
 from werkzeug.exceptions import HTTPException
 
 import boto3
-from moto.compat import OrderedDict
+from collections import OrderedDict
 from moto.core.utils import camelcase_to_underscores, method_names_from_class
 from moto import settings
 

moto/datapipeline/models.py

@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import datetime
 from boto3 import Session
 
-from moto.compat import OrderedDict
+from collections import OrderedDict
 from moto.core import BaseBackend, BaseModel, CloudFormationModel
 from .utils import get_random_pipeline_id, remove_capitalization_of_dict_keys
 

moto/datapipeline/utils.py

@@ -1,4 +1,4 @@
-from moto.compat import collections_abc
+import collections.abc as collections_abc
 from moto.core.utils import get_random_hex
 
 

moto/datasync/models.py

@@ -1,6 +1,6 @@
 from boto3 import Session
 
-from moto.compat import OrderedDict
+from collections import OrderedDict
 from moto.core import BaseBackend, BaseModel
 
 from .exceptions import InvalidRequestException

moto/dynamodb/models.py

@@ -3,7 +3,7 @@ from collections import defaultdict
 import datetime
 import json
 
-from moto.compat import OrderedDict
+from collections import OrderedDict
 from moto.core import BaseBackend, BaseModel, CloudFormationModel
 from moto.core.utils import unix_time
 from moto.core import ACCOUNT_ID

moto/dynamodb2/models/__init__.py

@@ -8,7 +8,7 @@ import re
 import uuid
 
 from boto3 import Session
-from moto.compat import OrderedDict
+from collections import OrderedDict
 from moto.core import ACCOUNT_ID
 from moto.core import BaseBackend, BaseModel, CloudFormationModel
 from moto.core.utils import unix_time, unix_time_millis
@@ -958,16 +958,6 @@ class Table(CloudFormationModel):
 
         return results, last_evaluated_key
 
-    def lookup(self, *args, **kwargs):
-        if not self.schema:
-            self.describe()
-        for x, arg in enumerate(args):
-            kwargs[self.schema[x].name] = arg
-        ret = self.get_item(**kwargs)
-        if not ret.keys():
-            return None
-        return ret
-
     def delete(self, region_name):
         dynamodb_backends[region_name].delete_table(self.name)
 

moto/dynamodb2/models/dynamo_type.py

@@ -1,6 +1,6 @@
 from moto.dynamodb2.comparisons import get_comparison_func
-from moto.dynamodb2.exceptions import InvalidUpdateExpression, IncorrectDataType
-from moto.dynamodb2.models.utilities import attribute_is_list, bytesize
+from moto.dynamodb2.exceptions import IncorrectDataType
+from moto.dynamodb2.models.utilities import bytesize
 
 
 class DDBType(object):
@@ -36,14 +36,7 @@ class DDBTypeConversion(object):
         Returns:
             str: The human readable form of the DDBType.
         """
-        try:
-            human_type_str = cls._human_type_mapping[abbreviated_type]
-        except KeyError:
-            raise ValueError(
-                "Invalid abbreviated_type {at}".format(at=abbreviated_type)
-            )
-
-        return human_type_str
+        return cls._human_type_mapping.get(abbreviated_type, abbreviated_type)
 
 
 class DynamoType(object):
@@ -63,70 +56,6 @@ class DynamoType(object):
         elif self.is_map():
             self.value = dict((k, DynamoType(v)) for k, v in self.value.items())
 
-    def get(self, key):
-        if not key:
-            return self
-        else:
-            key_head = key.split(".")[0]
-            key_tail = ".".join(key.split(".")[1:])
-            if key_head not in self.value:
-                self.value[key_head] = DynamoType({"NONE": None})
-            return self.value[key_head].get(key_tail)
-
-    def set(self, key, new_value, index=None):
-        if index:
-            index = int(index)
-            if type(self.value) is not list:
-                raise InvalidUpdateExpression
-            if index >= len(self.value):
-                self.value.append(new_value)
-            # {'L': [DynamoType, ..]} ==> DynamoType.set()
-            self.value[min(index, len(self.value) - 1)].set(key, new_value)
-        else:
-            attr = (key or "").split(".").pop(0)
-            attr, list_index = attribute_is_list(attr)
-            if not key:
-                # {'S': value} ==> {'S': new_value}
-                self.type = new_value.type
-                self.value = new_value.value
-            else:
-                if attr not in self.value:  # nonexistingattribute
-                    type_of_new_attr = DDBType.MAP if "." in key else new_value.type
-                    self.value[attr] = DynamoType({type_of_new_attr: {}})
-                # {'M': {'foo': DynamoType}} ==> DynamoType.set(new_value)
-                self.value[attr].set(
-                    ".".join(key.split(".")[1:]), new_value, list_index
-                )
-
-    def __contains__(self, item):
-        if self.type == DDBType.STRING:
-            return False
-        try:
-            self.__getitem__(item)
-            return True
-        except KeyError:
-            return False
-
-    def delete(self, key, index=None):
-        if index:
-            if not key:
-                if int(index) < len(self.value):
-                    del self.value[int(index)]
-            elif "." in key:
-                self.value[int(index)].delete(".".join(key.split(".")[1:]))
-            else:
-                self.value[int(index)].delete(key)
-        else:
-            attr = key.split(".")[0]
-            attr, list_index = attribute_is_list(attr)
-
-            if list_index:
-                self.value[attr].delete(".".join(key.split(".")[1:]), list_index)
-            elif "." in key:
-                self.value[attr].delete(".".join(key.split(".")[1:]))
-            else:
-                self.value.pop(key)
-
     def filter(self, projection_expressions):
         nested_projections = [
             expr[0 : expr.index(".")] for expr in projection_expressions if "." in expr

moto/dynamodb2/models/utilities.py

@@ -1,17 +1,2 @@
-import re
-
-
 def bytesize(val):
     return len(val.encode("utf-8"))
-
-
-def attribute_is_list(attr):
-    """
-    Checks if attribute denotes a list, and returns the name of the list and the given list index if so
-    :param attr: attr or attr[index]
-    :return: attr, index or None
-    """
-    list_index_update = re.match("(.+)\\[([0-9]+)\\]", attr)
-    if list_index_update:
-        attr = list_index_update.group(1)
-    return attr, list_index_update.group(2) if list_index_update else None

moto/ec2/models.py

@@ -24,7 +24,7 @@ from moto.packages.boto.ec2.spotinstancerequest import (
 )
 from moto.packages.boto.ec2.launchspecification import LaunchSpecification
 
-from moto.compat import OrderedDict
+from collections import OrderedDict
 from moto.core import BaseBackend
 from moto.core.models import Model, BaseModel, CloudFormationModel
 from moto.core.utils import (

moto/elb/models.py

@@ -12,7 +12,7 @@ from moto.packages.boto.ec2.elb.attributes import (
     CrossZoneLoadBalancingAttribute,
 )
 from moto.packages.boto.ec2.elb.policies import Policies, OtherPolicy
-from moto.compat import OrderedDict
+from collections import OrderedDict
 from moto.core import BaseBackend, BaseModel, CloudFormationModel
 from moto.ec2.models import ec2_backends
 from .exceptions import (

moto/elbv2/models.py

@@ -4,7 +4,7 @@ import datetime
 import re
 from jinja2 import Template
 from botocore.exceptions import ParamValidationError
-from moto.compat import OrderedDict
+from collections import OrderedDict
 from moto.core.exceptions import RESTError
 from moto.core import BaseBackend, BaseModel, CloudFormationModel
 from moto.core.utils import (

moto/glue/models.py

@@ -4,7 +4,7 @@ import time
 from datetime import datetime
 
 from moto.core import BaseBackend, BaseModel
-from moto.compat import OrderedDict
+from collections import OrderedDict
 from .exceptions import (
     JsonRESTError,
     DatabaseAlreadyExistsException,

moto/kinesis/models.py

@@ -10,7 +10,7 @@ from hashlib import md5
 
 from boto3 import Session
 
-from moto.compat import OrderedDict
+from collections import OrderedDict
 from moto.core import BaseBackend, BaseModel, CloudFormationModel
 from moto.core.utils import unix_time
 from moto.core import ACCOUNT_ID

moto/rds2/models.py

@@ -8,7 +8,7 @@ from collections import defaultdict
 from boto3 import Session
 from jinja2 import Template
 from re import compile as re_compile
-from moto.compat import OrderedDict
+from collections import OrderedDict
 from moto.core import BaseBackend, BaseModel, CloudFormationModel, ACCOUNT_ID
 
 from moto.core.utils import iso_8601_datetime_with_milliseconds

moto/rds2/utils.py

@@ -5,7 +5,7 @@ from collections import namedtuple
 
 from botocore.utils import merge_dicts
 
-from moto.compat import OrderedDict
+from collections import OrderedDict
 
 FilterDef = namedtuple(
     "FilterDef",

moto/redshift/models.py

@@ -5,7 +5,7 @@ import datetime
 
 from boto3 import Session
 
-from moto.compat import OrderedDict
+from collections import OrderedDict
 from moto.core import BaseBackend, BaseModel, CloudFormationModel
 from moto.core.utils import iso_8601_datetime_with_milliseconds
 from moto.utilities.utils import random_string

moto/sns/models.py

@@ -9,7 +9,7 @@ import re
 
 from boto3 import Session
 
-from moto.compat import OrderedDict
+from collections import OrderedDict
 from moto.core import BaseBackend, BaseModel, CloudFormationModel
 from moto.core.utils import (
     iso_8601_datetime_with_milliseconds,

tests/compat.py

@@ -1,5 +0,0 @@
-try:
-    from unittest.mock import patch
-except ImportError:
-    # for python 2.7
-    from mock import patch

tests/test_cloudformation/test_stack_parsing.py

@@ -4,7 +4,7 @@ import json
 import yaml
 
 import sure  # noqa
-from tests.compat import patch
+from unittest.mock import patch
 
 from moto.cloudformation.exceptions import ValidationError
 from moto.cloudformation.models import FakeStack

tests/test_core/test_responses.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 import sure  # noqa
 
-from moto.compat import OrderedDict
+from collections import OrderedDict
 
 from botocore.awsrequest import AWSPreparedRequest
 

tests/test_core/test_server.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 import sure  # noqa
 
-from tests.compat import patch
+from unittest.mock import patch
 from moto.server import main, create_backend_app, DomainDispatcherApplication
 
 

tests/test_rds2/test_rds2_cloudformation.py

@@ -0,0 +1,114 @@
+import boto3
+import json
+import sure  # noqa
+from moto import mock_cloudformation, mock_ec2, mock_rds2
+
+
+@mock_ec2
+@mock_rds2
+@mock_cloudformation
+def test_create_subnetgroup_via_cf():
+    vpc_conn = boto3.client("ec2", "us-west-2")
+    vpc = vpc_conn.create_vpc(CidrBlock="10.0.0.0/16")["Vpc"]
+    subnet = vpc_conn.create_subnet(VpcId=vpc["VpcId"], CidrBlock="10.0.1.0/24")[
+        "Subnet"
+    ]
+
+    rds = boto3.client("rds", region_name="us-west-2")
+    cf = boto3.client("cloudformation", region_name="us-west-2")
+
+    template = {
+        "AWSTemplateFormatVersion": "2010-09-09",
+        "Resources": {
+            "subnet": {
+                "Type": "AWS::RDS::DBSubnetGroup",
+                "Properties": {
+                    "DBSubnetGroupName": "subnetgroupname",
+                    "DBSubnetGroupDescription": "subnetgroupdesc",
+                    "SubnetIds": [subnet["SubnetId"]],
+                },
+            }
+        },
+    }
+    template_json = json.dumps(template)
+    cf.create_stack(StackName="test_stack", TemplateBody=template_json)
+
+    response = rds.describe_db_subnet_groups()["DBSubnetGroups"]
+    response.should.have.length_of(1)
+
+    created_subnet = response[0]
+    created_subnet.should.have.key("DBSubnetGroupName").equal("subnetgroupname")
+    created_subnet.should.have.key("DBSubnetGroupDescription").equal("subnetgroupdesc")
+    created_subnet.should.have.key("VpcId").equal(vpc["VpcId"])
+
+
+@mock_ec2
+@mock_rds2
+@mock_cloudformation
+def test_create_dbinstance_via_cf():
+    vpc_conn = boto3.client("ec2", "us-west-2")
+    vpc = vpc_conn.create_vpc(CidrBlock="10.0.0.0/16")["Vpc"]
+    vpc_conn.create_subnet(VpcId=vpc["VpcId"], CidrBlock="10.0.1.0/24")
+
+    rds = boto3.client("rds", region_name="us-west-2")
+    cf = boto3.client("cloudformation", region_name="us-west-2")
+
+    template = {
+        "AWSTemplateFormatVersion": "2010-09-09",
+        "Resources": {
+            "db": {
+                "Type": "AWS::RDS::DBInstance",
+                "Properties": {
+                    "Port": 3307,
+                    "Engine": "mysql",
+                    # Required - throws exception when describing an instance without tags
+                    "Tags": [],
+                },
+            }
+        },
+    }
+    template_json = json.dumps(template)
+    cf.create_stack(StackName="test_stack", TemplateBody=template_json)
+
+    summaries = cf.list_stack_resources(StackName="test_stack")[
+        "StackResourceSummaries"
+    ]
+
+    db_instance_identifier = summaries[0]["PhysicalResourceId"]
+    resp = rds.describe_db_instances()["DBInstances"]
+    resp.should.have.length_of(1)
+
+    created = resp[0]
+    created["DBInstanceIdentifier"].should.equal(db_instance_identifier)
+    created["Engine"].should.equal("mysql")
+    created["DBInstanceStatus"].should.equal("available")
+
+
+@mock_ec2
+@mock_rds2
+@mock_cloudformation
+def test_create_dbsecuritygroup_via_cf():
+    vpc_conn = boto3.client("ec2", "us-west-2")
+    vpc = vpc_conn.create_vpc(CidrBlock="10.0.0.0/16")["Vpc"]
+    vpc_conn.create_subnet(VpcId=vpc["VpcId"], CidrBlock="10.0.1.0/24")
+
+    rds = boto3.client("rds", region_name="us-west-2")
+    cf = boto3.client("cloudformation", region_name="us-west-2")
+
+    template = {
+        "AWSTemplateFormatVersion": "2010-09-09",
+        "Resources": {
+            "db": {
+                "Type": "AWS::RDS::DBSecurityGroup",
+                "Properties": {"GroupDescription": "my sec group"},
+            }
+        },
+    }
+    template_json = json.dumps(template)
+    cf.create_stack(StackName="test_stack", TemplateBody=template_json)
+
+    result = rds.describe_db_security_groups()["DBSecurityGroups"]
+    result.should.have.length_of(1)
+
+    created = result[0]
+    created["DBSecurityGroupDescription"].should.equal("my sec group")

tests/test_s3/test_s3_utils.py

@@ -8,7 +8,7 @@ from moto.s3.utils import (
     clean_key_name,
     undo_clean_key_name,
 )
-from tests.compat import patch
+from unittest.mock import patch
 
 
 def test_base_url():

tests/test_s3/test_server.py

@@ -7,7 +7,7 @@ import sure  # noqa
 
 from flask.testing import FlaskClient
 import moto.server as server
-from tests.compat import patch
+from unittest.mock import patch
 
 """
 Test the different server responses