Merge branch 'master' into add_instance_type_offerings

.coveragerc

@@ -3,6 +3,7 @@
 exclude_lines =
     if __name__ == .__main__.:
     raise NotImplemented.
+    return NotImplemented
     def __repr__
 
 [run]

.travis.yml

@@ -1,4 +1,4 @@
-dist: bionic
+dist: focal
 language: python
 services:
 - docker
@@ -27,7 +27,7 @@ install:
     docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:${PYTHON_DOCKER_TAG} /moto/travis_moto_server.sh &
   fi
   travis_retry pip install -r requirements-dev.txt
-  travis_retry pip install "docker>=2.5.1,<=4.2.2" # Limit version due to old Docker Engine in Travis https://github.com/docker/docker-py/issues/2639
+  travis_retry pip install docker>=2.5.1
   travis_retry pip install boto==2.45.0
   travis_retry pip install boto3
   travis_retry pip install dist/moto*.gz

IMPLEMENTATION_COVERAGE.md

@@ -8223,7 +8223,7 @@
 - [X] assume_role_with_web_identity
 - [ ] decode_authorization_message
 - [ ] get_access_key_info
-- [ ] get_caller_identity
+- [x] get_caller_identity
 - [X] get_federation_token
 - [X] get_session_token
 </details>

Makefile

@@ -39,7 +39,7 @@ upload_pypi_artifact:
 	twine upload dist/*
 
 push_dockerhub_image:
-	docker build -t motoserver/moto .
+	docker build -t motoserver/moto . --tag moto:`python setup.py --version`
 	docker push motoserver/moto
 
 tag_github_release:

moto/autoscaling/models.py

@@ -2,7 +2,10 @@ from __future__ import unicode_literals
 
 import random
 
-from boto.ec2.blockdevicemapping import BlockDeviceType, BlockDeviceMapping
+from moto.packages.boto.ec2.blockdevicemapping import (
+    BlockDeviceType,
+    BlockDeviceMapping,
+)
 from moto.ec2.exceptions import InvalidInstanceIdError
 
 from moto.compat import OrderedDict

moto/cloudformation/parsing.py

@@ -50,7 +50,7 @@ from .exceptions import (
     UnformattedGetAttTemplateException,
     ValidationError,
 )
-from boto.cloudformation.stack import Output
+from moto.packages.boto.cloudformation.stack import Output
 
 # List of supported CloudFormation models
 MODEL_LIST = CloudFormationModel.__subclasses__()

moto/cognitoidp/models.py

@@ -1066,5 +1066,7 @@ def find_region_by_value(key, value):
 
             if key == "access_token" and value in user_pool.access_tokens:
                 return region
-
-    return cognitoidp_backends.keys()[0]
+    # If we can't find the `client_id` or `access_token`, we just pass
+    # back a default backend region, which will raise the appropriate
+    # error message (e.g. NotAuthorized or NotFound).
+    return list(cognitoidp_backends)[0]

moto/dynamodb2/models/__init__.py

@@ -292,11 +292,19 @@ class SecondaryIndex(BaseModel):
         :return:
         """
         if self.projection:
-            if self.projection.get("ProjectionType", None) == "KEYS_ONLY":
-                allowed_attributes = ",".join(
-                    self.table_key_attrs + [key["AttributeName"] for key in self.schema]
+            projection_type = self.projection.get("ProjectionType", None)
+            key_attributes = self.table_key_attrs + [
+                key["AttributeName"] for key in self.schema
+            ]
+
+            if projection_type == "KEYS_ONLY":
+                item.filter(",".join(key_attributes))
+            elif projection_type == "INCLUDE":
+                allowed_attributes = key_attributes + self.projection.get(
+                    "NonKeyAttributes", []
                 )
-                item.filter(allowed_attributes)
+                item.filter(",".join(allowed_attributes))
+            # ALL is handled implicitly by not filtering
         return item
 
 

moto/ec2/models.py

@@ -15,10 +15,15 @@ from pkg_resources import resource_filename
 from collections import defaultdict
 import weakref
 from datetime import datetime
-from boto.ec2.instance import Instance as BotoInstance, Reservation
-from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType
-from boto.ec2.spotinstancerequest import SpotInstanceRequest as BotoSpotRequest
-from boto.ec2.launchspecification import LaunchSpecification
+from moto.packages.boto.ec2.instance import Instance as BotoInstance, Reservation
+from moto.packages.boto.ec2.blockdevicemapping import (
+    BlockDeviceMapping,
+    BlockDeviceType,
+)
+from moto.packages.boto.ec2.spotinstancerequest import (
+    SpotInstanceRequest as BotoSpotRequest,
+)
+from moto.packages.boto.ec2.launchspecification import LaunchSpecification
 
 from moto.compat import OrderedDict
 from moto.core import BaseBackend

moto/ec2instanceconnect/models.py

@@ -1,4 +1,4 @@
-import boto.ec2
+from boto3 import Session
 import json
 from moto.core import BaseBackend
 
@@ -11,5 +11,9 @@ class Ec2InstanceConnectBackend(BaseBackend):
 
 
 ec2instanceconnect_backends = {}
-for region in boto.ec2.regions():
-    ec2instanceconnect_backends[region.name] = Ec2InstanceConnectBackend()
+for region in Session().get_available_regions("ec2"):
+    ec2instanceconnect_backends[region] = Ec2InstanceConnectBackend()
+for region in Session().get_available_regions("ec2", partition_name="aws-us-gov"):
+    ec2instanceconnect_backends[region] = Ec2InstanceConnectBackend()
+for region in Session().get_available_regions("ec2", partition_name="aws-cn"):
+    ec2instanceconnect_backends[region] = Ec2InstanceConnectBackend()

moto/elb/models.py

@@ -4,14 +4,14 @@ import datetime
 
 import pytz
 
-from boto.ec2.elb.attributes import (
+from moto.packages.boto.ec2.elb.attributes import (
     LbAttributes,
     ConnectionSettingAttribute,
     ConnectionDrainingAttribute,
     AccessLogAttribute,
     CrossZoneLoadBalancingAttribute,
 )
-from boto.ec2.elb.policies import Policies, OtherPolicy
+from moto.packages.boto.ec2.elb.policies import Policies, OtherPolicy
 from moto.compat import OrderedDict
 from moto.core import BaseBackend, BaseModel, CloudFormationModel
 from moto.ec2.models import ec2_backends

moto/elb/responses.py

@@ -1,11 +1,11 @@
 from __future__ import unicode_literals
-from boto.ec2.elb.attributes import (
+from moto.packages.boto.ec2.elb.attributes import (
     ConnectionSettingAttribute,
     ConnectionDrainingAttribute,
     AccessLogAttribute,
     CrossZoneLoadBalancingAttribute,
 )
-from boto.ec2.elb.policies import AppCookieStickinessPolicy, OtherPolicy
+from moto.packages.boto.ec2.elb.policies import AppCookieStickinessPolicy, OtherPolicy
 
 from moto.core.responses import BaseResponse
 from .models import elb_backends

moto/emr/responses.py

@@ -13,7 +13,7 @@ from moto.core.responses import xml_to_json_response
 from moto.core.utils import tags_from_query_string
 from .exceptions import EmrError
 from .models import emr_backends
-from .utils import steps_from_query_string, Unflattener
+from .utils import steps_from_query_string, Unflattener, ReleaseLabel
 
 
 def generate_boto3_response(operation):
@@ -323,7 +323,9 @@ class ElasticMapReduceResponse(BaseResponse):
         custom_ami_id = self._get_param("CustomAmiId")
         if custom_ami_id:
             kwargs["custom_ami_id"] = custom_ami_id
-            if release_label and release_label < "emr-5.7.0":
+            if release_label and (
+                ReleaseLabel(release_label) < ReleaseLabel("emr-5.7.0")
+            ):
                 message = "Custom AMI is not allowed"
                 raise EmrError(
                     error_type="ValidationException",

moto/emr/utils.py

@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 import random
+import re
 import string
 from moto.core.utils import camelcase_to_underscores
 
@@ -144,3 +145,76 @@ class CamelToUnderscoresWalker:
     @staticmethod
     def parse_scalar(x):
         return x
+
+
+class ReleaseLabel(object):
+
+    version_re = re.compile(r"^emr-(\d+)\.(\d+)\.(\d+)$")
+
+    def __init__(self, release_label):
+        major, minor, patch = self.parse(release_label)
+
+        self.major = major
+        self.minor = minor
+        self.patch = patch
+
+    @classmethod
+    def parse(cls, release_label):
+        if not release_label:
+            raise ValueError("Invalid empty ReleaseLabel: %r" % release_label)
+
+        match = cls.version_re.match(release_label)
+        if not match:
+            raise ValueError("Invalid ReleaseLabel: %r" % release_label)
+
+        major, minor, patch = match.groups()
+
+        major = int(major)
+        minor = int(minor)
+        patch = int(patch)
+
+        return major, minor, patch
+
+    def __str__(self):
+        version = "emr-%d.%d.%d" % (self.major, self.minor, self.patch)
+        return version
+
+    def __repr__(self):
+        return "%s(%r)" % (self.__class__.__name__, str(self))
+
+    def __iter__(self):
+        return iter((self.major, self.minor, self.patch))
+
+    def __eq__(self, other):
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        return (
+            self.major == other.major
+            and self.minor == other.minor
+            and self.patch == other.patch
+        )
+
+    def __ne__(self, other):
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        return tuple(self) != tuple(other)
+
+    def __lt__(self, other):
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        return tuple(self) < tuple(other)
+
+    def __le__(self, other):
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        return tuple(self) <= tuple(other)
+
+    def __gt__(self, other):
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        return tuple(self) > tuple(other)
+
+    def __ge__(self, other):
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        return tuple(self) >= tuple(other)

moto/packages/boto/README.md

@@ -0,0 +1,18 @@
+## Removing the `boto` Dependency
+
+In order to rid `moto` of a direct dependency on the long-deprecated `boto` 
+package, a subset of the `boto` code has been vendored here.
+
+This directory contains only the `boto` files required for `moto` to run, 
+which is a very small subset of the original package's contents.  Furthermore, 
+the `boto` models collected here have been stripped of all superfluous 
+methods/attributes not used by `moto`.  (Any copyright headers on the 
+original files have been left intact.)
+
+## Next Steps
+
+Currently, a small number of `moto` models inherit from these `boto` classes.
+With some additional work, the inheritance can be dropped in favor of simply
+adding the required methods/properties from these `boto` models to their 
+respective `moto` subclasses, which would allow for these files/directories 
+to be removed entirely.

moto/packages/boto/cloudformation/stack.py

@@ -0,0 +1,9 @@
+class Output(object):
+    def __init__(self, connection=None):
+        self.connection = connection
+        self.description = None
+        self.key = None
+        self.value = None
+
+    def __repr__(self):
+        return 'Output:"%s"="%s"' % (self.key, self.value)

moto/packages/boto/ec2/blockdevicemapping.py

@@ -0,0 +1,83 @@
+# Copyright (c) 2009-2012 Mitch Garnaat http://garnaat.org/
+# Copyright (c) 2012 Amazon.com, Inc. or its affiliates.  All Rights Reserved
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+
+
+class BlockDeviceType(object):
+    """
+    Represents parameters for a block device.
+    """
+
+    def __init__(
+        self,
+        connection=None,
+        ephemeral_name=None,
+        no_device=False,
+        volume_id=None,
+        snapshot_id=None,
+        status=None,
+        attach_time=None,
+        delete_on_termination=False,
+        size=None,
+        volume_type=None,
+        iops=None,
+        encrypted=None,
+    ):
+        self.connection = connection
+        self.ephemeral_name = ephemeral_name
+        self.no_device = no_device
+        self.volume_id = volume_id
+        self.snapshot_id = snapshot_id
+        self.status = status
+        self.attach_time = attach_time
+        self.delete_on_termination = delete_on_termination
+        self.size = size
+        self.volume_type = volume_type
+        self.iops = iops
+        self.encrypted = encrypted
+
+
+# for backwards compatibility
+EBSBlockDeviceType = BlockDeviceType
+
+
+class BlockDeviceMapping(dict):
+    """
+    Represents a collection of BlockDeviceTypes when creating ec2 instances.
+
+    Example:
+    dev_sda1 = BlockDeviceType()
+    dev_sda1.size = 100   # change root volume to 100GB instead of default
+    bdm = BlockDeviceMapping()
+    bdm['/dev/sda1'] = dev_sda1
+    reservation = image.run(..., block_device_map=bdm, ...)
+    """
+
+    def __init__(self, connection=None):
+        """
+        :type connection: :class:`boto.ec2.EC2Connection`
+        :param connection: Optional connection.
+        """
+        dict.__init__(self)
+        self.connection = connection
+        self.current_name = None
+        self.current_value = None

moto/packages/boto/ec2/ec2object.py

@@ -0,0 +1,48 @@
+# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/
+# Copyright (c) 2010, Eucalyptus Systems, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+"""
+Represents an EC2 Object
+"""
+from moto.packages.boto.ec2.tag import TagSet
+
+
+class EC2Object(object):
+    def __init__(self, connection=None):
+        self.connection = connection
+        self.region = None
+
+
+class TaggedEC2Object(EC2Object):
+    """
+    Any EC2 resource that can be tagged should be represented
+    by a Python object that subclasses this class.  This class
+    has the mechanism in place to handle the tagSet element in
+    the Describe* responses.  If tags are found, it will create
+    a TagSet object and allow it to parse and collect the tags
+    into a dict that is stored in the "tags" attribute of the
+    object.
+    """
+
+    def __init__(self, connection=None):
+        super(TaggedEC2Object, self).__init__(connection)
+        self.tags = TagSet()

moto/packages/boto/ec2/elb/attributes.py

@@ -0,0 +1,100 @@
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+# Created by Chris Huegle for TellApart, Inc.
+
+
+class ConnectionSettingAttribute(object):
+    """
+    Represents the ConnectionSetting segment of ELB Attributes.
+    """
+
+    def __init__(self, connection=None):
+        self.idle_timeout = None
+
+    def __repr__(self):
+        return "ConnectionSettingAttribute(%s)" % (self.idle_timeout)
+
+
+class CrossZoneLoadBalancingAttribute(object):
+    """
+    Represents the CrossZoneLoadBalancing segement of ELB Attributes.
+    """
+
+    def __init__(self, connection=None):
+        self.enabled = None
+
+    def __repr__(self):
+        return "CrossZoneLoadBalancingAttribute(%s)" % (self.enabled)
+
+
+class AccessLogAttribute(object):
+    """
+    Represents the AccessLog segment of ELB attributes.
+    """
+
+    def __init__(self, connection=None):
+        self.enabled = None
+        self.s3_bucket_name = None
+        self.s3_bucket_prefix = None
+        self.emit_interval = None
+
+    def __repr__(self):
+        return "AccessLog(%s, %s, %s, %s)" % (
+            self.enabled,
+            self.s3_bucket_name,
+            self.s3_bucket_prefix,
+            self.emit_interval,
+        )
+
+
+class ConnectionDrainingAttribute(object):
+    """
+    Represents the ConnectionDraining segment of ELB attributes.
+    """
+
+    def __init__(self, connection=None):
+        self.enabled = None
+        self.timeout = None
+
+    def __repr__(self):
+        return "ConnectionDraining(%s, %s)" % (self.enabled, self.timeout)
+
+
+class LbAttributes(object):
+    """
+    Represents the Attributes of an Elastic Load Balancer.
+    """
+
+    def __init__(self, connection=None):
+        self.connection = connection
+        self.cross_zone_load_balancing = CrossZoneLoadBalancingAttribute(
+            self.connection
+        )
+        self.access_log = AccessLogAttribute(self.connection)
+        self.connection_draining = ConnectionDrainingAttribute(self.connection)
+        self.connecting_settings = ConnectionSettingAttribute(self.connection)
+
+    def __repr__(self):
+        return "LbAttributes(%s, %s, %s, %s)" % (
+            repr(self.cross_zone_load_balancing),
+            repr(self.access_log),
+            repr(self.connection_draining),
+            repr(self.connecting_settings),
+        )

moto/packages/boto/ec2/elb/policies.py

@@ -0,0 +1,55 @@
+# Copyright (c) 2010 Reza Lotun http://reza.lotun.name
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+
+class AppCookieStickinessPolicy(object):
+    def __init__(self, connection=None):
+        self.cookie_name = None
+        self.policy_name = None
+
+    def __repr__(self):
+        return "AppCookieStickiness(%s, %s)" % (self.policy_name, self.cookie_name)
+
+
+class OtherPolicy(object):
+    def __init__(self, connection=None):
+        self.policy_name = None
+
+    def __repr__(self):
+        return "OtherPolicy(%s)" % (self.policy_name)
+
+
+class Policies(object):
+    """
+    ELB Policies
+    """
+
+    def __init__(self, connection=None):
+        self.connection = connection
+        self.app_cookie_stickiness_policies = None
+        self.lb_cookie_stickiness_policies = None
+        self.other_policies = None
+
+    def __repr__(self):
+        app = "AppCookieStickiness%s" % self.app_cookie_stickiness_policies
+        lb = "LBCookieStickiness%s" % self.lb_cookie_stickiness_policies
+        other = "Other%s" % self.other_policies
+        return "Policies(%s,%s,%s)" % (app, lb, other)

moto/packages/boto/ec2/image.py

@@ -0,0 +1,25 @@
+# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/
+# Copyright (c) 2010, Eucalyptus Systems, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+
+class ProductCodes(list):
+    pass

moto/packages/boto/ec2/instance.py

@@ -0,0 +1,217 @@
+# Copyright (c) 2006-2012 Mitch Garnaat http://garnaat.org/
+# Copyright (c) 2010, Eucalyptus Systems, Inc.
+# Copyright (c) 2012 Amazon.com, Inc. or its affiliates.  All Rights Reserved
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+"""
+Represents an EC2 Instance
+"""
+from moto.packages.boto.ec2.ec2object import EC2Object, TaggedEC2Object
+from moto.packages.boto.ec2.image import ProductCodes
+
+
+class InstanceState(object):
+    """
+    The state of the instance.
+
+    :ivar code: The low byte represents the state. The high byte is an
+        opaque internal value and should be ignored.  Valid values:
+
+        * 0 (pending)
+        * 16 (running)
+        * 32 (shutting-down)
+        * 48 (terminated)
+        * 64 (stopping)
+        * 80 (stopped)
+
+    :ivar name: The name of the state of the instance.  Valid values:
+
+        * "pending"
+        * "running"
+        * "shutting-down"
+        * "terminated"
+        * "stopping"
+        * "stopped"
+    """
+
+    def __init__(self, code=0, name=None):
+        self.code = code
+        self.name = name
+
+    def __repr__(self):
+        return "%s(%d)" % (self.name, self.code)
+
+
+class InstancePlacement(object):
+    """
+    The location where the instance launched.
+
+    :ivar zone: The Availability Zone of the instance.
+    :ivar group_name: The name of the placement group the instance is
+        in (for cluster compute instances).
+    :ivar tenancy: The tenancy of the instance (if the instance is
+        running within a VPC). An instance with a tenancy of dedicated
+        runs on single-tenant hardware.
+    """
+
+    def __init__(self, zone=None, group_name=None, tenancy=None):
+        self.zone = zone
+        self.group_name = group_name
+        self.tenancy = tenancy
+
+    def __repr__(self):
+        return self.zone
+
+
+class Reservation(EC2Object):
+    """
+    Represents a Reservation response object.
+
+    :ivar id: The unique ID of the Reservation.
+    :ivar owner_id: The unique ID of the owner of the Reservation.
+    :ivar groups: A list of Group objects representing the security
+                  groups associated with launched instances.
+    :ivar instances: A list of Instance objects launched in this
+                     Reservation.
+    """
+
+    def __init__(self, connection=None):
+        super(Reservation, self).__init__(connection)
+        self.id = None
+        self.owner_id = None
+        self.groups = []
+        self.instances = []
+
+    def __repr__(self):
+        return "Reservation:%s" % self.id
+
+
+class Instance(TaggedEC2Object):
+    """
+    Represents an instance.
+
+    :ivar id: The unique ID of the Instance.
+    :ivar groups: A list of Group objects representing the security
+                  groups associated with the instance.
+    :ivar public_dns_name: The public dns name of the instance.
+    :ivar private_dns_name: The private dns name of the instance.
+    :ivar state: The string representation of the instance's current state.
+    :ivar state_code: An integer representation of the instance's
+        current state.
+    :ivar previous_state: The string representation of the instance's
+        previous state.
+    :ivar previous_state_code: An integer representation of the
+        instance's current state.
+    :ivar key_name: The name of the SSH key associated with the instance.
+    :ivar instance_type: The type of instance (e.g. m1.small).
+    :ivar launch_time: The time the instance was launched.
+    :ivar image_id: The ID of the AMI used to launch this instance.
+    :ivar placement: The availability zone in which the instance is running.
+    :ivar placement_group: The name of the placement group the instance
+        is in (for cluster compute instances).
+    :ivar placement_tenancy: The tenancy of the instance, if the instance
+        is running within a VPC.  An instance with a tenancy of dedicated
+        runs on a single-tenant hardware.
+    :ivar kernel: The kernel associated with the instance.
+    :ivar ramdisk: The ramdisk associated with the instance.
+    :ivar architecture: The architecture of the image (i386|x86_64).
+    :ivar hypervisor: The hypervisor used.
+    :ivar virtualization_type: The type of virtualization used.
+    :ivar product_codes: A list of product codes associated with this instance.
+    :ivar ami_launch_index: This instances position within it's launch group.
+    :ivar monitored: A boolean indicating whether monitoring is enabled or not.
+    :ivar monitoring_state: A string value that contains the actual value
+        of the monitoring element returned by EC2.
+    :ivar spot_instance_request_id: The ID of the spot instance request
+        if this is a spot instance.
+    :ivar subnet_id: The VPC Subnet ID, if running in VPC.
+    :ivar vpc_id: The VPC ID, if running in VPC.
+    :ivar private_ip_address: The private IP address of the instance.
+    :ivar ip_address: The public IP address of the instance.
+    :ivar platform: Platform of the instance (e.g. Windows)
+    :ivar root_device_name: The name of the root device.
+    :ivar root_device_type: The root device type (ebs|instance-store).
+    :ivar block_device_mapping: The Block Device Mapping for the instance.
+    :ivar state_reason: The reason for the most recent state transition.
+    :ivar interfaces: List of Elastic Network Interfaces associated with
+        this instance.
+    :ivar ebs_optimized: Whether instance is using optimized EBS volumes
+        or not.
+    :ivar instance_profile: A Python dict containing the instance
+        profile id and arn associated with this instance.
+    """
+
+    def __init__(self, connection=None):
+        super(Instance, self).__init__(connection)
+        self.id = None
+        self.dns_name = None
+        self.public_dns_name = None
+        self.private_dns_name = None
+        self.key_name = None
+        self.instance_type = None
+        self.launch_time = None
+        self.image_id = None
+        self.kernel = None
+        self.ramdisk = None
+        self.product_codes = ProductCodes()
+        self.ami_launch_index = None
+        self.monitored = False
+        self.monitoring_state = None
+        self.spot_instance_request_id = None
+        self.subnet_id = None
+        self.vpc_id = None
+        self.private_ip_address = None
+        self.ip_address = None
+        self.requester_id = None
+        self._in_monitoring_element = False
+        self.persistent = False
+        self.root_device_name = None
+        self.root_device_type = None
+        self.block_device_mapping = None
+        self.state_reason = None
+        self.group_name = None
+        self.client_token = None
+        self.eventsSet = None
+        self.groups = []
+        self.platform = None
+        self.interfaces = []
+        self.hypervisor = None
+        self.virtualization_type = None
+        self.architecture = None
+        self.instance_profile = None
+        self._previous_state = None
+        self._state = InstanceState()
+        self._placement = InstancePlacement()
+
+    def __repr__(self):
+        return "Instance:%s" % self.id
+
+    @property
+    def state(self):
+        return self._state.name
+
+    @property
+    def state_code(self):
+        return self._state.code
+
+    @property
+    def placement(self):
+        return self._placement.zone

moto/packages/boto/ec2/instancetype.py

@@ -0,0 +1,50 @@
+# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+
+from moto.packages.boto.ec2.ec2object import EC2Object
+
+
+class InstanceType(EC2Object):
+    """
+    Represents an EC2 VM Type
+
+    :ivar name: The name of the vm type
+    :ivar cores: The number of cpu cores for this vm type
+    :ivar memory: The amount of memory in megabytes for this vm type
+    :ivar disk: The amount of disk space in gigabytes for this vm type
+    """
+
+    def __init__(self, connection=None, name=None, cores=None, memory=None, disk=None):
+        super(InstanceType, self).__init__(connection)
+        self.connection = connection
+        self.name = name
+        self.cores = cores
+        self.memory = memory
+        self.disk = disk
+
+    def __repr__(self):
+        return "InstanceType:%s-%s,%s,%s" % (
+            self.name,
+            self.cores,
+            self.memory,
+            self.disk,
+        )

moto/packages/boto/ec2/launchspecification.py

@@ -0,0 +1,48 @@
+# Copyright (c) 2006-2012 Mitch Garnaat http://garnaat.org/
+# Copyright (c) 2012 Amazon.com, Inc. or its affiliates.  All Rights Reserved
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+"""
+Represents a launch specification for Spot instances.
+"""
+
+from moto.packages.boto.ec2.ec2object import EC2Object
+
+
+class LaunchSpecification(EC2Object):
+    def __init__(self, connection=None):
+        super(LaunchSpecification, self).__init__(connection)
+        self.key_name = None
+        self.instance_type = None
+        self.image_id = None
+        self.groups = []
+        self.placement = None
+        self.kernel = None
+        self.ramdisk = None
+        self.monitored = False
+        self.subnet_id = None
+        self._in_monitoring_element = False
+        self.block_device_mapping = None
+        self.instance_profile = None
+        self.ebs_optimized = False
+
+    def __repr__(self):
+        return "LaunchSpecification(%s)" % self.image_id

moto/packages/boto/ec2/spotinstancerequest.py

@@ -0,0 +1,85 @@
+# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/
+# Copyright (c) 2010, Eucalyptus Systems, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+"""
+Represents an EC2 Spot Instance Request
+"""
+
+from moto.packages.boto.ec2.ec2object import TaggedEC2Object
+
+
+class SpotInstanceRequest(TaggedEC2Object):
+    """
+
+    :ivar id: The ID of the Spot Instance Request.
+    :ivar price: The maximum hourly price for any Spot Instance launched to
+        fulfill the request.
+    :ivar type: The Spot Instance request type.
+    :ivar state: The state of the Spot Instance request.
+    :ivar fault: The fault codes for the Spot Instance request, if any.
+    :ivar valid_from: The start date of the request. If this is a one-time
+        request, the request becomes active at this date and time and remains
+        active until all instances launch, the request expires, or the request is
+        canceled. If the request is persistent, the request becomes active at this
+        date and time and remains active until it expires or is canceled.
+    :ivar valid_until: The end date of the request. If this is a one-time
+        request, the request remains active until all instances launch, the request
+        is canceled, or this date is reached. If the request is persistent, it
+        remains active until it is canceled or this date is reached.
+    :ivar launch_group: The instance launch group. Launch groups are Spot
+        Instances that launch together and terminate together.
+    :ivar launched_availability_zone: foo
+    :ivar product_description: The Availability Zone in which the bid is
+        launched.
+    :ivar availability_zone_group: The Availability Zone group. If you specify
+        the same Availability Zone group for all Spot Instance requests, all Spot
+        Instances are launched in the same Availability Zone.
+    :ivar create_time: The time stamp when the Spot Instance request was
+        created.
+    :ivar launch_specification: Additional information for launching instances.
+    :ivar instance_id: The instance ID, if an instance has been launched to
+        fulfill the Spot Instance request.
+    :ivar status: The status code and status message describing the Spot
+        Instance request.
+
+    """
+
+    def __init__(self, connection=None):
+        super(SpotInstanceRequest, self).__init__(connection)
+        self.id = None
+        self.price = None
+        self.type = None
+        self.state = None
+        self.fault = None
+        self.valid_from = None
+        self.valid_until = None
+        self.launch_group = None
+        self.launched_availability_zone = None
+        self.product_description = None
+        self.availability_zone_group = None
+        self.create_time = None
+        self.launch_specification = None
+        self.instance_id = None
+        self.status = None
+
+    def __repr__(self):
+        return "SpotInstanceRequest:%s" % self.id

moto/packages/boto/ec2/tag.py

@@ -0,0 +1,35 @@
+# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/
+# Copyright (c) 2010, Eucalyptus Systems, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+
+class TagSet(dict):
+    """
+    A TagSet is used to collect the tags associated with a particular
+    EC2 resource.  Not all resources can be tagged but for those that
+    can, this dict object will be used to collect those values.  See
+    :class:`boto.ec2.ec2object.TaggedEC2Object` for more details.
+    """
+
+    def __init__(self, connection=None):
+        self.connection = connection
+        self._current_key = None
+        self._current_value = None

moto/rds/models.py

@@ -1,6 +1,6 @@
 from __future__ import unicode_literals
 
-import boto.rds
+from boto3 import Session
 from jinja2 import Template
 
 from moto.core import BaseBackend, CloudFormationModel
@@ -335,6 +335,10 @@ class RDSBackend(BaseBackend):
         return rds2_backends[self.region]
 
 
-rds_backends = dict(
-    (region.name, RDSBackend(region.name)) for region in boto.rds.regions()
-)
+rds_backends = {}
+for region in Session().get_available_regions("rds"):
+    rds_backends[region] = RDSBackend(region)
+for region in Session().get_available_regions("rds", partition_name="aws-us-gov"):
+    rds_backends[region] = RDSBackend(region)
+for region in Session().get_available_regions("rds", partition_name="aws-cn"):
+    rds_backends[region] = RDSBackend(region)

moto/rds2/models.py

@@ -9,7 +9,8 @@ from boto3 import Session
 from jinja2 import Template
 from re import compile as re_compile
 from moto.compat import OrderedDict
-from moto.core import BaseBackend, BaseModel, CloudFormationModel
+from moto.core import BaseBackend, BaseModel, CloudFormationModel, ACCOUNT_ID
+
 from moto.core.utils import iso_8601_datetime_with_milliseconds
 from moto.ec2.models import ec2_backends
 from .exceptions import (
@@ -157,6 +158,7 @@ class Database(CloudFormationModel):
                     family=db_family,
                     description=description,
                     tags={},
+                    region=self.region,
                 )
             ]
         else:
@@ -1172,7 +1174,7 @@ class RDS2Backend(BaseBackend):
                 "InvalidParameterValue",
                 "The parameter DBParameterGroupName must be provided and must not be blank.",
             )
-
+        db_parameter_group_kwargs["region"] = self.region
         db_parameter_group = DBParameterGroup(**db_parameter_group_kwargs)
         self.db_parameter_groups[db_parameter_group_id] = db_parameter_group
         return db_parameter_group
@@ -1471,13 +1473,18 @@ class OptionGroupOptionSetting(object):
         return template.render(option_group_option_setting=self)
 
 
+def make_rds_arn(region, name):
+    return "arn:aws:rds:{0}:{1}:pg:{2}".format(region, ACCOUNT_ID, name)
+
+
 class DBParameterGroup(CloudFormationModel):
-    def __init__(self, name, description, family, tags):
+    def __init__(self, name, description, family, tags, region):
         self.name = name
         self.description = description
         self.family = family
         self.tags = tags
         self.parameters = defaultdict(dict)
+        self.arn = make_rds_arn(region, name)
 
     def to_xml(self):
         template = Template(
@@ -1485,6 +1492,7 @@ class DBParameterGroup(CloudFormationModel):
           <DBParameterGroupName>{{ param_group.name }}</DBParameterGroupName>
           <DBParameterGroupFamily>{{ param_group.family }}</DBParameterGroupFamily>
           <Description>{{ param_group.description }}</Description>
+          <DBParameterGroupArn>{{ param_group.arn }}</DBParameterGroupArn>
         </DBParameterGroup>"""
         )
         return template.render(param_group=self)

moto/redshift/exceptions.py

@@ -143,3 +143,17 @@ class ClusterAlreadyExistsFaultError(RedshiftClientError):
         super(ClusterAlreadyExistsFaultError, self).__init__(
             "ClusterAlreadyExists", "Cluster already exists"
         )
+
+
+class InvalidParameterCombinationError(RedshiftClientError):
+    def __init__(self, message):
+        super(InvalidParameterCombinationError, self).__init__(
+            "InvalidParameterCombination", message
+        )
+
+
+class UnknownSnapshotCopyRegionFaultError(RedshiftClientError):
+    def __init__(self, message):
+        super(UnknownSnapshotCopyRegionFaultError, self).__init__(
+            "UnknownSnapshotCopyRegionFault", message
+        )

moto/redshift/models.py

@@ -4,7 +4,7 @@ import copy
 import datetime
 
 from boto3 import Session
-from botocore.exceptions import ClientError
+
 from moto.compat import OrderedDict
 from moto.core import BaseBackend, BaseModel, CloudFormationModel
 from moto.core.utils import iso_8601_datetime_with_milliseconds
@@ -17,6 +17,7 @@ from .exceptions import (
     ClusterSnapshotAlreadyExistsError,
     ClusterSnapshotNotFoundError,
     ClusterSubnetGroupNotFoundError,
+    InvalidParameterCombinationError,
     InvalidParameterValueError,
     InvalidSubnetError,
     ResourceNotFoundFaultError,
@@ -25,6 +26,7 @@ from .exceptions import (
     SnapshotCopyDisabledFaultError,
     SnapshotCopyGrantAlreadyExistsFaultError,
     SnapshotCopyGrantNotFoundFaultError,
+    UnknownSnapshotCopyRegionFaultError,
 )
 
 
@@ -576,6 +578,10 @@ class RedshiftBackend(BaseBackend):
                 raise InvalidParameterValueError(
                     "SnapshotCopyGrantName is required for Snapshot Copy on KMS encrypted clusters."
                 )
+            if kwargs["destination_region"] == self.region:
+                raise UnknownSnapshotCopyRegionFaultError(
+                    "Invalid region {}".format(self.region)
+                )
             status = {
                 "DestinationRegion": kwargs["destination_region"],
                 "RetentionPeriod": kwargs["retention_period"],
@@ -655,10 +661,8 @@ class RedshiftBackend(BaseBackend):
                 cluster_skip_final_snapshot is False
                 and cluster_snapshot_identifer is None
             ):
-                raise ClientError(
-                    "InvalidParameterValue",
-                    "FinalSnapshotIdentifier is required for Snapshot copy "
-                    "when SkipFinalSnapshot is False",
+                raise InvalidParameterCombinationError(
+                    "FinalClusterSnapshotIdentifier is required unless SkipFinalClusterSnapshot is specified."
                 )
             elif (
                 cluster_skip_final_snapshot is False
@@ -777,7 +781,6 @@ class RedshiftBackend(BaseBackend):
                     cluster_snapshots.append(snapshot)
             if cluster_snapshots:
                 return cluster_snapshots
-            raise ClusterNotFoundError(cluster_identifier)
 
         if snapshot_identifier:
             if snapshot_identifier in self.snapshots:

moto/server.py

@@ -93,6 +93,11 @@ class DomainDispatcherApplication(object):
                 # S3 is the last resort when the target is also unknown
                 service, region = DEFAULT_SERVICE_REGION
 
+        if service == "EventBridge":
+            # Go SDK uses 'EventBridge' in the SigV4 request instead of 'events'
+            # see https://github.com/spulec/moto/issues/3494
+            service = "events"
+
         if service == "dynamodb":
             if environ["HTTP_X_AMZ_TARGET"].startswith("DynamoDBStreams"):
                 host = "dynamodbstreams"

setup.py

@@ -32,7 +32,6 @@ def get_version():
 
 
 install_requires = [
-    "boto>=2.36.0",
     "boto3>=1.9.201",
     "botocore>=1.12.201",
     "cryptography>=2.3.0",

tests/test_cloudformation/test_stack_parsing.py

@@ -15,7 +15,7 @@ from moto.cloudformation.parsing import (
 from moto.sqs.models import Queue
 from moto.s3.models import FakeBucket
 from moto.cloudformation.utils import yaml_tag_constructor
-from boto.cloudformation.stack import Output
+from moto.packages.boto.cloudformation.stack import Output
 
 
 dummy_template = {

tests/test_cognitoidp/test_cognitoidp.py

@@ -1840,6 +1840,31 @@ def test_admin_set_user_password():
     result["UserStatus"].should.equal("CONFIRMED")
 
 
+@mock_cognitoidp
+def test_change_password_with_invalid_token_raises_error():
+    client = boto3.client("cognito-idp", "us-west-2")
+    with pytest.raises(ClientError) as ex:
+        client.change_password(
+            AccessToken=str(uuid.uuid4()),
+            PreviousPassword="previous_password",
+            ProposedPassword="newer_password",
+        )
+    ex.value.response["Error"]["Code"].should.equal("NotAuthorizedException")
+
+
+@mock_cognitoidp
+def test_confirm_forgot_password_with_non_existent_client_id_raises_error():
+    client = boto3.client("cognito-idp", "us-west-2")
+    with pytest.raises(ClientError) as ex:
+        client.confirm_forgot_password(
+            ClientId="non-existent-client-id",
+            Username="not-existent-username",
+            ConfirmationCode=str(uuid.uuid4()),
+            Password=str(uuid.uuid4()),
+        )
+    ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException")
+
+
 # Test will retrieve public key from cognito.amazonaws.com/.well-known/jwks.json,
 # which isnt mocked in ServerMode
 if not settings.TEST_SERVER_MODE:

tests/test_dynamodb2/test_dynamodb.py

@@ -5523,6 +5523,61 @@ def test_gsi_projection_type_keys_only():
     )
 
 
+@mock_dynamodb2
+def test_gsi_projection_type_include():
+    table_schema = {
+        "KeySchema": [{"AttributeName": "partitionKey", "KeyType": "HASH"}],
+        "GlobalSecondaryIndexes": [
+            {
+                "IndexName": "GSI-INC",
+                "KeySchema": [
+                    {"AttributeName": "gsiK1PartitionKey", "KeyType": "HASH"},
+                    {"AttributeName": "gsiK1SortKey", "KeyType": "RANGE"},
+                ],
+                "Projection": {
+                    "ProjectionType": "INCLUDE",
+                    "NonKeyAttributes": ["projectedAttribute"],
+                },
+            }
+        ],
+        "AttributeDefinitions": [
+            {"AttributeName": "partitionKey", "AttributeType": "S"},
+            {"AttributeName": "gsiK1PartitionKey", "AttributeType": "S"},
+            {"AttributeName": "gsiK1SortKey", "AttributeType": "S"},
+        ],
+    }
+
+    item = {
+        "partitionKey": "pk-1",
+        "gsiK1PartitionKey": "gsi-pk",
+        "gsiK1SortKey": "gsi-sk",
+        "projectedAttribute": "lore ipsum",
+        "nonProjectedAttribute": "dolor sit amet",
+    }
+
+    dynamodb = boto3.resource("dynamodb", region_name="us-east-1")
+    dynamodb.create_table(
+        TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema
+    )
+    table = dynamodb.Table("test-table")
+    table.put_item(Item=item)
+
+    items = table.query(
+        KeyConditionExpression=Key("gsiK1PartitionKey").eq("gsi-pk"),
+        IndexName="GSI-INC",
+    )["Items"]
+    items.should.have.length_of(1)
+    # Item should only include keys and additionally projected attributes only
+    items[0].should.equal(
+        {
+            "gsiK1PartitionKey": "gsi-pk",
+            "gsiK1SortKey": "gsi-sk",
+            "partitionKey": "pk-1",
+            "projectedAttribute": "lore ipsum",
+        }
+    )
+
+
 @mock_dynamodb2
 def test_lsi_projection_type_keys_only():
     table_schema = {

tests/test_ec2/test_ec2_cloudformation.py

@@ -2,6 +2,9 @@ from moto import mock_cloudformation_deprecated, mock_ec2_deprecated
 from moto import mock_cloudformation, mock_ec2
 from tests.test_cloudformation.fixtures import vpc_eni
 import boto
+import boto.ec2
+import boto.cloudformation
+import boto.vpc
 import boto3
 import json
 import sure  # noqa

tests/test_emr/test_emr_boto3.py

@@ -636,7 +636,7 @@ def test_run_job_flow_with_custom_ami():
 
     args = deepcopy(run_job_flow_args)
     args["CustomAmiId"] = "MyEmrCustomAmi"
-    args["ReleaseLabel"] = "emr-5.7.0"
+    args["ReleaseLabel"] = "emr-5.31.0"
     cluster_id = client.run_job_flow(**args)["JobFlowId"]
     resp = client.describe_cluster(ClusterId=cluster_id)
     resp["Cluster"]["CustomAmiId"].should.equal("MyEmrCustomAmi")

tests/test_emr/test_utils.py

@@ -0,0 +1,49 @@
+import pytest
+
+from moto.emr.utils import ReleaseLabel
+
+
+def test_invalid_release_labels_raise_exception():
+    invalid_releases = [
+        "",
+        "0",
+        "1.0",
+        "emr-2.0",
+    ]
+    for invalid_release in invalid_releases:
+        with pytest.raises(ValueError):
+            ReleaseLabel(invalid_release)
+
+
+def test_release_label_comparisons():
+    assert str(ReleaseLabel("emr-5.1.2")) == "emr-5.1.2"
+
+    assert ReleaseLabel("emr-5.0.0") != ReleaseLabel("emr-5.0.1")
+    assert ReleaseLabel("emr-5.0.0") == ReleaseLabel("emr-5.0.0")
+
+    assert ReleaseLabel("emr-5.31.0") > ReleaseLabel("emr-5.7.0")
+    assert ReleaseLabel("emr-6.0.0") > ReleaseLabel("emr-5.7.0")
+
+    assert ReleaseLabel("emr-5.7.0") < ReleaseLabel("emr-5.10.0")
+    assert ReleaseLabel("emr-5.10.0") < ReleaseLabel("emr-5.10.1")
+
+    assert ReleaseLabel("emr-5.60.0") >= ReleaseLabel("emr-5.7.0")
+    assert ReleaseLabel("emr-6.0.0") >= ReleaseLabel("emr-6.0.0")
+
+    assert ReleaseLabel("emr-5.7.0") <= ReleaseLabel("emr-5.17.0")
+    assert ReleaseLabel("emr-5.7.0") <= ReleaseLabel("emr-5.7.0")
+
+    releases_unsorted = [
+        ReleaseLabel("emr-5.60.2"),
+        ReleaseLabel("emr-4.0.1"),
+        ReleaseLabel("emr-4.0.0"),
+        ReleaseLabel("emr-5.7.3"),
+    ]
+    releases_sorted = [str(label) for label in sorted(releases_unsorted)]
+    expected = [
+        "emr-4.0.0",
+        "emr-4.0.1",
+        "emr-5.7.3",
+        "emr-5.60.2",
+    ]
+    assert releases_sorted == expected

tests/test_iam/test_iam_policies.py

@@ -3,6 +3,7 @@ import json
 import boto3
 from botocore.exceptions import ClientError
 import pytest
+import sure  # noqa
 
 from moto import mock_iam
 
@@ -1611,31 +1612,25 @@ valid_policy_documents = [
 ]
 
 
-def test_create_policy_with_invalid_policy_documents():
-    for test_case in invalid_policy_document_test_cases:
-        yield check_create_policy_with_invalid_policy_document, test_case
-
-
-def test_create_policy_with_valid_policy_documents():
-    for valid_policy_document in valid_policy_documents:
-        yield check_create_policy_with_valid_policy_document, valid_policy_document
-
-
+@pytest.mark.parametrize("invalid_policy_document", invalid_policy_document_test_cases)
 @mock_iam
-def check_create_policy_with_invalid_policy_document(test_case):
+def test_create_policy_with_invalid_policy_document(invalid_policy_document):
     conn = boto3.client("iam", region_name="us-east-1")
     with pytest.raises(ClientError) as ex:
         conn.create_policy(
             PolicyName="TestCreatePolicy",
-            PolicyDocument=json.dumps(test_case["document"]),
+            PolicyDocument=json.dumps(invalid_policy_document["document"]),
         )
     ex.value.response["Error"]["Code"].should.equal("MalformedPolicyDocument")
     ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
-    ex.value.response["Error"]["Message"].should.equal(test_case["error_message"])
+    ex.value.response["Error"]["Message"].should.equal(
+        invalid_policy_document["error_message"]
+    )
 
 
+@pytest.mark.parametrize("valid_policy_document", valid_policy_documents)
 @mock_iam
-def check_create_policy_with_valid_policy_document(valid_policy_document):
+def test_create_policy_with_valid_policy_document(valid_policy_document):
     conn = boto3.client("iam", region_name="us-east-1")
     conn.create_policy(
         PolicyName="TestCreatePolicy", PolicyDocument=json.dumps(valid_policy_document)

tests/test_rds2/test_rds2.py

@@ -4,6 +4,7 @@ from botocore.exceptions import ClientError, ParamValidationError
 import boto3
 import sure  # noqa
 from moto import mock_ec2, mock_kms, mock_rds2
+from moto.core import ACCOUNT_ID
 
 
 @mock_rds2
@@ -1504,7 +1505,9 @@ def test_create_database_with_encrypted_storage():
 
 @mock_rds2
 def test_create_db_parameter_group():
-    conn = boto3.client("rds", region_name="us-west-2")
+    region = "us-west-2"
+    pg_name = "test"
+    conn = boto3.client("rds", region_name=region)
     db_parameter_group = conn.create_db_parameter_group(
         DBParameterGroupName="test",
         DBParameterGroupFamily="mysql5.6",
@@ -1518,6 +1521,9 @@ def test_create_db_parameter_group():
     db_parameter_group["DBParameterGroup"]["Description"].should.equal(
         "test parameter group"
     )
+    db_parameter_group["DBParameterGroup"]["DBParameterGroupArn"].should.equal(
+        "arn:aws:rds:{0}:{1}:pg:{2}".format(region, ACCOUNT_ID, pg_name)
+    )
 
 
 @mock_rds2
@@ -1629,9 +1635,11 @@ def test_create_db_parameter_group_duplicate():
 
 @mock_rds2
 def test_describe_db_parameter_group():
-    conn = boto3.client("rds", region_name="us-west-2")
+    region = "us-west-2"
+    pg_name = "test"
+    conn = boto3.client("rds", region_name=region)
     conn.create_db_parameter_group(
-        DBParameterGroupName="test",
+        DBParameterGroupName=pg_name,
         DBParameterGroupFamily="mysql5.6",
         Description="test parameter group",
     )
@@ -1639,6 +1647,9 @@ def test_describe_db_parameter_group():
     db_parameter_groups["DBParameterGroups"][0]["DBParameterGroupName"].should.equal(
         "test"
     )
+    db_parameter_groups["DBParameterGroups"][0]["DBParameterGroupArn"].should.equal(
+        "arn:aws:rds:{0}:{1}:pg:{2}".format(region, ACCOUNT_ID, pg_name)
+    )
 
 
 @mock_rds2

tests/test_redshift/test_redshift.py

@@ -43,7 +43,7 @@ def test_create_cluster_boto3():
 
 
 @mock_redshift
-def test_create_cluster_boto3():
+def test_create_cluster_with_enhanced_vpc_routing_enabled():
     client = boto3.client("redshift", region_name="us-east-1")
     response = client.create_cluster(
         DBName="test",
@@ -76,7 +76,7 @@ def test_create_snapshot_copy_grant():
 
     client.describe_snapshot_copy_grants.when.called_with(
         SnapshotCopyGrantName="test-us-east-1"
-    ).should.throw(Exception)
+    ).should.throw(ClientError)
 
 
 @mock_redshift
@@ -424,7 +424,7 @@ def test_delete_cluster():
     )
 
     conn.delete_cluster.when.called_with(cluster_identifier, False).should.throw(
-        AttributeError
+        boto.exception.JSONResponseError
     )
 
     clusters = conn.describe_clusters()["DescribeClustersResponse"][
@@ -826,12 +826,11 @@ def test_describe_cluster_snapshots():
 @mock_redshift
 def test_describe_cluster_snapshots_not_found_error():
     client = boto3.client("redshift", region_name="us-east-1")
-    cluster_identifier = "my_cluster"
-    snapshot_identifier = "my_snapshot"
+    cluster_identifier = "non-existent-cluster-id"
+    snapshot_identifier = "non-existent-snapshot-id"
 
-    client.describe_cluster_snapshots.when.called_with(
-        ClusterIdentifier=cluster_identifier
-    ).should.throw(ClientError, "Cluster {} not found.".format(cluster_identifier))
+    resp = client.describe_cluster_snapshots(ClusterIdentifier=cluster_identifier)
+    resp["Snapshots"].should.have.length_of(0)
 
     client.describe_cluster_snapshots.when.called_with(
         SnapshotIdentifier=snapshot_identifier
@@ -867,8 +866,8 @@ def test_delete_cluster_snapshot():
 
     # Delete invalid id
     client.delete_cluster_snapshot.when.called_with(
-        SnapshotIdentifier="not-a-snapshot"
-    ).should.throw(ClientError)
+        SnapshotIdentifier="non-existent"
+    ).should.throw(ClientError, "Snapshot non-existent not found.")
 
 
 @mock_redshift
@@ -892,7 +891,7 @@ def test_cluster_snapshot_already_exists():
 
     client.create_cluster_snapshot.when.called_with(
         SnapshotIdentifier=snapshot_identifier, ClusterIdentifier=cluster_identifier
-    ).should.throw(ClientError)
+    ).should.throw(ClientError, "{} already exists".format(snapshot_identifier))
 
 
 @mock_redshift
@@ -1269,6 +1268,15 @@ def test_enable_snapshot_copy():
     ex.value.response["Error"]["Message"].should.contain(
         "SnapshotCopyGrantName is required for Snapshot Copy on KMS encrypted clusters."
     )
+    with pytest.raises(ClientError) as ex:
+        client.enable_snapshot_copy(
+            ClusterIdentifier="test",
+            DestinationRegion="us-east-1",
+            RetentionPeriod=3,
+            SnapshotCopyGrantName="invalid-us-east-1-to-us-east-1",
+        )
+    ex.value.response["Error"]["Code"].should.equal("UnknownSnapshotCopyRegionFault")
+    ex.value.response["Error"]["Message"].should.contain("Invalid region us-east-1")
     client.enable_snapshot_copy(
         ClusterIdentifier="test",
         DestinationRegion="us-west-2",
@@ -1364,3 +1372,74 @@ def test_create_duplicate_cluster_fails():
     client.create_cluster.when.called_with(**kwargs).should.throw(
         ClientError, "ClusterAlreadyExists"
     )
+
+
+@mock_redshift
+def test_delete_cluster_with_final_snapshot():
+    client = boto3.client("redshift", region_name="us-east-1")
+
+    with pytest.raises(ClientError) as ex:
+        client.delete_cluster(ClusterIdentifier="non-existent")
+    ex.value.response["Error"]["Code"].should.equal("ClusterNotFound")
+    ex.value.response["Error"]["Message"].should.match(r"Cluster .+ not found.")
+
+    cluster_identifier = "my_cluster"
+    client.create_cluster(
+        ClusterIdentifier=cluster_identifier,
+        ClusterType="single-node",
+        DBName="test",
+        MasterUsername="user",
+        MasterUserPassword="password",
+        NodeType="ds2.xlarge",
+    )
+
+    with pytest.raises(ClientError) as ex:
+        client.delete_cluster(
+            ClusterIdentifier=cluster_identifier, SkipFinalClusterSnapshot=False
+        )
+    ex.value.response["Error"]["Code"].should.equal("InvalidParameterCombination")
+    ex.value.response["Error"]["Message"].should.contain(
+        "FinalClusterSnapshotIdentifier is required unless SkipFinalClusterSnapshot is specified."
+    )
+
+    snapshot_identifier = "my_snapshot"
+    client.delete_cluster(
+        ClusterIdentifier=cluster_identifier,
+        SkipFinalClusterSnapshot=False,
+        FinalClusterSnapshotIdentifier=snapshot_identifier,
+    )
+
+    resp = client.describe_cluster_snapshots(ClusterIdentifier=cluster_identifier)
+    resp["Snapshots"].should.have.length_of(1)
+    resp["Snapshots"][0]["SnapshotIdentifier"].should.equal(snapshot_identifier)
+    resp["Snapshots"][0]["SnapshotType"].should.equal("manual")
+
+    with pytest.raises(ClientError) as ex:
+        client.describe_clusters(ClusterIdentifier=cluster_identifier)
+    ex.value.response["Error"]["Code"].should.equal("ClusterNotFound")
+    ex.value.response["Error"]["Message"].should.match(r"Cluster .+ not found.")
+
+
+@mock_redshift
+def test_delete_cluster_without_final_snapshot():
+    client = boto3.client("redshift", region_name="us-east-1")
+    cluster_identifier = "my_cluster"
+    client.create_cluster(
+        ClusterIdentifier=cluster_identifier,
+        ClusterType="single-node",
+        DBName="test",
+        MasterUsername="user",
+        MasterUserPassword="password",
+        NodeType="ds2.xlarge",
+    )
+    client.delete_cluster(
+        ClusterIdentifier=cluster_identifier, SkipFinalClusterSnapshot=True
+    )
+
+    resp = client.describe_cluster_snapshots(ClusterIdentifier=cluster_identifier)
+    resp["Snapshots"].should.have.length_of(0)
+
+    with pytest.raises(ClientError) as ex:
+        client.describe_clusters(ClusterIdentifier=cluster_identifier)
+    ex.value.response["Error"]["Code"].should.equal("ClusterNotFound")
+    ex.value.response["Error"]["Message"].should.match(r"Cluster .+ not found.")

tests/test_redshift/test_server.py

@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import json
 import sure  # noqa
 
 import moto.server as server
@@ -20,3 +19,14 @@ def test_describe_clusters():
 
     result = res.data.decode("utf-8")
     result.should.contain("<Clusters></Clusters>")
+
+
+@mock_redshift
+def test_describe_clusters_with_json_content_type():
+    backend = server.create_backend_app("redshift")
+    test_client = backend.test_client()
+
+    res = test_client.get("/?Action=DescribeClusters&ContentType=JSON")
+
+    result = res.data.decode("utf-8")
+    result.should.contain('{"Clusters": []}')

travis_moto_server.sh

@@ -1,8 +1,4 @@
 #!/usr/bin/env bash
 set -e
-# TravisCI on bionic dist uses old version of Docker Engine
-# which is incompatibile with newer docker-py
-# See https://github.com/docker/docker-py/issues/2639
-pip install "docker>=2.5.1,<=4.2.2"
 pip install $(ls /moto/dist/moto*.gz)[server,all]
 moto_server -H 0.0.0.0 -p 5000