From d8b124fbf4a641508cb54ecc77c53c0b2cf8e622 Mon Sep 17 00:00:00 2001 From: captainkerk Date: Sun, 28 Jan 2018 03:06:57 +0000 Subject: [PATCH 01/14] added: enable/disable/modify redshift snapshot copy methods --- moto/redshift/exceptions.py | 19 ++++++++ moto/redshift/models.py | 47 +++++++++++++++++- moto/redshift/responses.py | 55 +++++++++++++++++++++ tests/test_redshift/test_redshift.py | 72 ++++++++++++++++++++++++++++ 4 files changed, 191 insertions(+), 2 deletions(-) diff --git a/moto/redshift/exceptions.py b/moto/redshift/exceptions.py index a89ed5a04..138afd442 100644 --- a/moto/redshift/exceptions.py +++ b/moto/redshift/exceptions.py @@ -93,3 +93,22 @@ class ResourceNotFoundFaultError(RedshiftClientError): msg = message super(ResourceNotFoundFaultError, self).__init__( 'ResourceNotFoundFault', msg) + + +class SnapshotCopyDisabledFaultError(RedshiftClientError): + def __init__(self, cluster_identifier): + super(SnapshotCopyDisabledFaultError, self).__init__( + 'SnapshotCopyDisabledFault', + "Cannot modify retention period because snapshot copy is disabled on Cluster {0}.".format(cluster_identifier)) + +class SnapshotCopyAlreadyDisabledFaultError(RedshiftClientError): + def __init__(self, cluster_identifier): + super(SnapshotCopyAlreadyDisabledFaultError, self).__init__( + 'SnapshotCopyAlreadyDisabledFault', + "Snapshot Copy is already disabled on Cluster {0}.".format(cluster_identifier)) + +class SnapshotCopyAlreadyEnabledFaultError(RedshiftClientError): + def __init__(self, cluster_identifier): + super(SnapshotCopyAlreadyEnabledFaultError, self).__init__( + 'SnapshotCopyAlreadyEnabledFault', + "Snapshot Copy is already enabled on Cluster {0}.".format(cluster_identifier)) diff --git a/moto/redshift/models.py b/moto/redshift/models.py index fa642ef01..2bab77f66 100644 --- a/moto/redshift/models.py +++ b/moto/redshift/models.py @@ -17,7 +17,10 @@ from .exceptions import ( ClusterSubnetGroupNotFoundError, InvalidParameterValueError, InvalidSubnetError, - ResourceNotFoundFaultError + ResourceNotFoundFaultError, + SnapshotCopyDisabledFaultError, + SnapshotCopyAlreadyDisabledFaultError, + SnapshotCopyAlreadyEnabledFaultError, ) @@ -80,6 +83,7 @@ class Cluster(TaggableResourceMixin, BaseModel): self.cluster_subnet_group_name = cluster_subnet_group_name self.publicly_accessible = publicly_accessible self.encrypted = encrypted + self.cluster_snapshot_copy_status = {} self.allow_version_upgrade = allow_version_upgrade if allow_version_upgrade is not None else True self.cluster_version = cluster_version if cluster_version else "1.0" @@ -194,7 +198,7 @@ class Cluster(TaggableResourceMixin, BaseModel): return self.cluster_identifier def to_json(self): - return { + json_response = { "MasterUsername": self.master_username, "MasterUserPassword": "****", "ClusterVersion": self.cluster_version, @@ -223,6 +227,7 @@ class Cluster(TaggableResourceMixin, BaseModel): "NodeType": self.node_type, "ClusterIdentifier": self.cluster_identifier, "AllowVersionUpgrade": self.allow_version_upgrade, + "Endpoint": { "Address": self.endpoint, "Port": self.port @@ -231,6 +236,10 @@ class Cluster(TaggableResourceMixin, BaseModel): "Tags": self.tags } + if self.cluster_snapshot_copy_status: + json_response['ClusterSnapshotCopyStatus'] = self.cluster_snapshot_copy_status + return json_response + class SubnetGroup(TaggableResourceMixin, BaseModel): @@ -417,6 +426,40 @@ class RedshiftBackend(BaseBackend): self.__dict__ = {} self.__init__(ec2_backend, region_name) + def enable_snapshot_copy(self, **kwargs): + cluster_identifier = kwargs['cluster_identifier'] + cluster = self.clusters[cluster_identifier] + if not cluster.cluster_snapshot_copy_status: + status = { + 'DestinationRegion': kwargs['destination_region'], + 'RetentionPeriod': kwargs['retention_period'], + 'SnapshotCopyGrantName': kwargs['snapshot_copy_grant_name'], + } + cluster.cluster_snapshot_copy_status = status + return cluster + + else: + raise SnapshotCopyAlreadyEnabledFaultError(cluster_identifier) + + + def disable_snapshot_copy(self, **kwargs): + cluster_identifier = kwargs['cluster_identifier'] + cluster = self.clusters[cluster_identifier] + if cluster.cluster_snapshot_copy_status: + cluster.cluster_snapshot_copy_status = {} + else: + raise SnapshotCopyAlreadyDisabledFaultError(cluster_identifier) + return cluster + + + def modify_snapshot_copy_retention_period(self, cluster_identifier, retention_period): + cluster = self.clusters[cluster_identifier] + if cluster.cluster_snapshot_copy_status: + cluster.cluster_snapshot_copy_status['RetentionPeriod'] = retention_period + else: + raise SnapshotCopyDisabledFaultError(cluster_identifier) + return cluster + def create_cluster(self, **cluster_kwargs): cluster_identifier = cluster_kwargs['cluster_identifier'] cluster = Cluster(self, **cluster_kwargs) diff --git a/moto/redshift/responses.py b/moto/redshift/responses.py index a320f9cae..bd7223c8c 100644 --- a/moto/redshift/responses.py +++ b/moto/redshift/responses.py @@ -501,3 +501,58 @@ class RedshiftResponse(BaseResponse): } } }) + + def enable_snapshot_copy(self): + snapshot_copy_kwargs = { + 'cluster_identifier': self._get_param('ClusterIdentifier'), + 'destination_region': self._get_param('DestinationRegion'), + 'retention_period': self._get_param('RetentionPeriod'), + 'snapshot_copy_grant_name': self._get_param('SnapshotCopyGrantName'), + } + cluster = self.redshift_backend.enable_snapshot_copy(**snapshot_copy_kwargs) + + return self.get_response({ + "EnableSnapshotCopyResponse": { + "EnableSnapshotCopyResult": { + "Cluster": cluster.to_json() + }, + "ResponseMetadata": { + "RequestId": "384ac68d-3775-11df-8963-01868b7c937a", + } + } + }) + + def disable_snapshot_copy(self): + snapshot_copy_kwargs = { + 'cluster_identifier': self._get_param('ClusterIdentifier'), + } + cluster = self.redshift_backend.disable_snapshot_copy(**snapshot_copy_kwargs) + + return self.get_response({ + "DisableSnapshotCopyResponse": { + "DisableSnapshotCopyResult": { + "Cluster": cluster.to_json() + }, + "ResponseMetadata": { + "RequestId": "384ac68d-3775-11df-8963-01868b7c937a", + } + } + }) + + def modify_snapshot_copy_retention_period(self): + snapshot_copy_kwargs = { + 'cluster_identifier': self._get_param('ClusterIdentifier'), + 'retention_period': self._get_param('RetentionPeriod'), + } + cluster = self.redshift_backend.modify_snapshot_copy_retention_period(**snapshot_copy_kwargs) + + return self.get_response({ + "ModifySnapshotCopyRetentionPeriodResponse": { + "ModifySnapshotCopyRetentionPeriodResult": { + "Clusters": [cluster.to_json()] + }, + "ResponseMetadata": { + "RequestId": "384ac68d-3775-11df-8963-01868b7c937a", + } + } + }) diff --git a/tests/test_redshift/test_redshift.py b/tests/test_redshift/test_redshift.py index cebaa3ec7..46400d34e 100644 --- a/tests/test_redshift/test_redshift.py +++ b/tests/test_redshift/test_redshift.py @@ -1042,3 +1042,75 @@ def test_tagged_resource_not_found_error(): ResourceName='bad:arn' ).should.throw(ClientError, "Tagging is not supported for this type of resource") + +@mock_redshift +def test_enable_snapshot_copy(): + client = boto3.client('redshift', region_name='us-east-1') + client.create_cluster( + DBName='test', + ClusterIdentifier='test', + ClusterType='single-node', + NodeType='ds2.xlarge', + MasterUsername='user', + MasterUserPassword='password', + ) + client.enable_snapshot_copy( + ClusterIdentifier='test', + DestinationRegion='us-west-2', + RetentionPeriod=3, + SnapshotCopyGrantName='copy-us-east-1-to-us-west-2' + ) + response = client.describe_clusters(ClusterIdentifier='test') + cluster_snapshot_copy_status = response['Clusters'][0]['ClusterSnapshotCopyStatus'] + cluster_snapshot_copy_status['RetentionPeriod'].should.equal(3) + cluster_snapshot_copy_status['DestinationRegion'].should.equal('us-west-2') + cluster_snapshot_copy_status['SnapshotCopyGrantName'].should.equal('copy-us-east-1-to-us-west-2') + + +@mock_redshift +def test_disable_snapshot_copy(): + client = boto3.client('redshift', region_name='us-east-1') + client.create_cluster( + DBName='test', + ClusterIdentifier='test', + ClusterType='single-node', + NodeType='ds2.xlarge', + MasterUsername='user', + MasterUserPassword='password', + ) + client.enable_snapshot_copy( + ClusterIdentifier='test', + DestinationRegion='us-west-2', + RetentionPeriod=3, + SnapshotCopyGrantName='copy-us-east-1-to-us-west-2', + ) + client.disable_snapshot_copy( + ClusterIdentifier='test', + ) + response = client.describe_clusters(ClusterIdentifier='test') + response['Clusters'][0].shouldnt.contain('ClusterSnapshotCopyStatus') + +@mock_redshift +def test_modify_snapshot_copy_retention_period(): + client = boto3.client('redshift', region_name='us-east-1') + client.create_cluster( + DBName='test', + ClusterIdentifier='test', + ClusterType='single-node', + NodeType='ds2.xlarge', + MasterUsername='user', + MasterUserPassword='password', + ) + client.enable_snapshot_copy( + ClusterIdentifier='test', + DestinationRegion='us-west-2', + RetentionPeriod=3, + SnapshotCopyGrantName='copy-us-east-1-to-us-west-2', + ) + client.modify_snapshot_copy_retention_period( + ClusterIdentifier='test', + RetentionPeriod=5, + ) + response = client.describe_clusters(ClusterIdentifier='test') + cluster_snapshot_copy_status = response['Clusters'][0]['ClusterSnapshotCopyStatus'] + cluster_snapshot_copy_status['RetentionPeriod'].should.equal(5) From 92798b9a9fef52f5a79d5cd0b3ea48d2310d2ee7 Mon Sep 17 00:00:00 2001 From: captainkerk Date: Sun, 28 Jan 2018 03:27:06 +0000 Subject: [PATCH 02/14] improve error handling --- moto/redshift/models.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/moto/redshift/models.py b/moto/redshift/models.py index 2bab77f66..4975868d0 100644 --- a/moto/redshift/models.py +++ b/moto/redshift/models.py @@ -83,7 +83,6 @@ class Cluster(TaggableResourceMixin, BaseModel): self.cluster_subnet_group_name = cluster_subnet_group_name self.publicly_accessible = publicly_accessible self.encrypted = encrypted - self.cluster_snapshot_copy_status = {} self.allow_version_upgrade = allow_version_upgrade if allow_version_upgrade is not None else True self.cluster_version = cluster_version if cluster_version else "1.0" @@ -236,8 +235,10 @@ class Cluster(TaggableResourceMixin, BaseModel): "Tags": self.tags } - if self.cluster_snapshot_copy_status: + try: json_response['ClusterSnapshotCopyStatus'] = self.cluster_snapshot_copy_status + except AttributeError: + pass return json_response @@ -429,7 +430,7 @@ class RedshiftBackend(BaseBackend): def enable_snapshot_copy(self, **kwargs): cluster_identifier = kwargs['cluster_identifier'] cluster = self.clusters[cluster_identifier] - if not cluster.cluster_snapshot_copy_status: + if not hasattr(cluster, 'cluster_snapshot_copy_status'): status = { 'DestinationRegion': kwargs['destination_region'], 'RetentionPeriod': kwargs['retention_period'], @@ -437,28 +438,25 @@ class RedshiftBackend(BaseBackend): } cluster.cluster_snapshot_copy_status = status return cluster - else: raise SnapshotCopyAlreadyEnabledFaultError(cluster_identifier) - def disable_snapshot_copy(self, **kwargs): cluster_identifier = kwargs['cluster_identifier'] cluster = self.clusters[cluster_identifier] - if cluster.cluster_snapshot_copy_status: - cluster.cluster_snapshot_copy_status = {} + if hasattr(cluster, 'cluster_snapshot_copy_status'): + del cluster.cluster_snapshot_copy_status + return cluster else: raise SnapshotCopyAlreadyDisabledFaultError(cluster_identifier) - return cluster - def modify_snapshot_copy_retention_period(self, cluster_identifier, retention_period): cluster = self.clusters[cluster_identifier] - if cluster.cluster_snapshot_copy_status: + if hasattr(cluster, 'cluster_snapshot_copy_status'): cluster.cluster_snapshot_copy_status['RetentionPeriod'] = retention_period + return cluster else: raise SnapshotCopyDisabledFaultError(cluster_identifier) - return cluster def create_cluster(self, **cluster_kwargs): cluster_identifier = cluster_kwargs['cluster_identifier'] From ed066582714d071e2db78ac96f3bd014a8b28a4a Mon Sep 17 00:00:00 2001 From: captainkerk Date: Sun, 28 Jan 2018 03:28:49 +0000 Subject: [PATCH 03/14] address spacing issues --- moto/redshift/exceptions.py | 2 ++ moto/redshift/models.py | 1 - moto/redshift/responses.py | 2 +- tests/test_redshift/test_redshift.py | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/moto/redshift/exceptions.py b/moto/redshift/exceptions.py index 138afd442..865aaeab2 100644 --- a/moto/redshift/exceptions.py +++ b/moto/redshift/exceptions.py @@ -101,12 +101,14 @@ class SnapshotCopyDisabledFaultError(RedshiftClientError): 'SnapshotCopyDisabledFault', "Cannot modify retention period because snapshot copy is disabled on Cluster {0}.".format(cluster_identifier)) + class SnapshotCopyAlreadyDisabledFaultError(RedshiftClientError): def __init__(self, cluster_identifier): super(SnapshotCopyAlreadyDisabledFaultError, self).__init__( 'SnapshotCopyAlreadyDisabledFault', "Snapshot Copy is already disabled on Cluster {0}.".format(cluster_identifier)) + class SnapshotCopyAlreadyEnabledFaultError(RedshiftClientError): def __init__(self, cluster_identifier): super(SnapshotCopyAlreadyEnabledFaultError, self).__init__( diff --git a/moto/redshift/models.py b/moto/redshift/models.py index 4975868d0..44e944c3b 100644 --- a/moto/redshift/models.py +++ b/moto/redshift/models.py @@ -226,7 +226,6 @@ class Cluster(TaggableResourceMixin, BaseModel): "NodeType": self.node_type, "ClusterIdentifier": self.cluster_identifier, "AllowVersionUpgrade": self.allow_version_upgrade, - "Endpoint": { "Address": self.endpoint, "Port": self.port diff --git a/moto/redshift/responses.py b/moto/redshift/responses.py index bd7223c8c..724a61b68 100644 --- a/moto/redshift/responses.py +++ b/moto/redshift/responses.py @@ -555,4 +555,4 @@ class RedshiftResponse(BaseResponse): "RequestId": "384ac68d-3775-11df-8963-01868b7c937a", } } - }) + }) diff --git a/tests/test_redshift/test_redshift.py b/tests/test_redshift/test_redshift.py index 46400d34e..79da9f193 100644 --- a/tests/test_redshift/test_redshift.py +++ b/tests/test_redshift/test_redshift.py @@ -1090,6 +1090,7 @@ def test_disable_snapshot_copy(): response = client.describe_clusters(ClusterIdentifier='test') response['Clusters'][0].shouldnt.contain('ClusterSnapshotCopyStatus') + @mock_redshift def test_modify_snapshot_copy_retention_period(): client = boto3.client('redshift', region_name='us-east-1') From 7130dd5239891a5ae2a9219e690c8c6f1c4e5906 Mon Sep 17 00:00:00 2001 From: captainkerk Date: Sun, 28 Jan 2018 03:53:32 +0000 Subject: [PATCH 04/14] rework to follow spec for encrypted/unencrypted clusters --- moto/redshift/models.py | 7 +++++++ moto/redshift/responses.py | 2 +- tests/test_redshift/test_redshift.py | 26 ++++++++++++++++++++++++-- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/moto/redshift/models.py b/moto/redshift/models.py index 44e944c3b..f0ea9f5f9 100644 --- a/moto/redshift/models.py +++ b/moto/redshift/models.py @@ -4,6 +4,7 @@ import copy import datetime import boto.redshift +from botocore.exceptions import ClientError from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel from moto.core.utils import iso_8601_datetime_with_milliseconds @@ -430,6 +431,12 @@ class RedshiftBackend(BaseBackend): cluster_identifier = kwargs['cluster_identifier'] cluster = self.clusters[cluster_identifier] if not hasattr(cluster, 'cluster_snapshot_copy_status'): + if cluster.encrypted == 'true' and kwargs['snapshot_copy_grant_name'] is None: + raise ClientError( + 'InvalidParameterValue', + 'SnapshotCopyGrantName is required for Snapshot Copy ' + 'on KMS encrypted clusters.' + ) status = { 'DestinationRegion': kwargs['destination_region'], 'RetentionPeriod': kwargs['retention_period'], diff --git a/moto/redshift/responses.py b/moto/redshift/responses.py index 724a61b68..63945c00b 100644 --- a/moto/redshift/responses.py +++ b/moto/redshift/responses.py @@ -506,7 +506,7 @@ class RedshiftResponse(BaseResponse): snapshot_copy_kwargs = { 'cluster_identifier': self._get_param('ClusterIdentifier'), 'destination_region': self._get_param('DestinationRegion'), - 'retention_period': self._get_param('RetentionPeriod'), + 'retention_period': self._get_param('RetentionPeriod', 7), 'snapshot_copy_grant_name': self._get_param('SnapshotCopyGrantName'), } cluster = self.redshift_backend.enable_snapshot_copy(**snapshot_copy_kwargs) diff --git a/tests/test_redshift/test_redshift.py b/tests/test_redshift/test_redshift.py index 79da9f193..32deb74bc 100644 --- a/tests/test_redshift/test_redshift.py +++ b/tests/test_redshift/test_redshift.py @@ -1047,12 +1047,13 @@ def test_tagged_resource_not_found_error(): def test_enable_snapshot_copy(): client = boto3.client('redshift', region_name='us-east-1') client.create_cluster( - DBName='test', ClusterIdentifier='test', ClusterType='single-node', - NodeType='ds2.xlarge', + DBName='test', + Encrypted=True, MasterUsername='user', MasterUserPassword='password', + NodeType='ds2.xlarge', ) client.enable_snapshot_copy( ClusterIdentifier='test', @@ -1067,6 +1068,27 @@ def test_enable_snapshot_copy(): cluster_snapshot_copy_status['SnapshotCopyGrantName'].should.equal('copy-us-east-1-to-us-west-2') +@mock_redshift +def test_enable_snapshot_copy_unencrypted(): + client = boto3.client('redshift', region_name='us-east-1') + client.create_cluster( + ClusterIdentifier='test', + ClusterType='single-node', + DBName='test', + MasterUsername='user', + MasterUserPassword='password', + NodeType='ds2.xlarge', + ) + client.enable_snapshot_copy( + ClusterIdentifier='test', + DestinationRegion='us-west-2', + ) + response = client.describe_clusters(ClusterIdentifier='test') + cluster_snapshot_copy_status = response['Clusters'][0]['ClusterSnapshotCopyStatus'] + cluster_snapshot_copy_status['RetentionPeriod'].should.equal(7) + cluster_snapshot_copy_status['DestinationRegion'].should.equal('us-west-2') + + @mock_redshift def test_disable_snapshot_copy(): client = boto3.client('redshift', region_name='us-east-1') From f3debf8f6ff57c023231dedd2ee97e7e34774b85 Mon Sep 17 00:00:00 2001 From: grahamlyons Date: Mon, 29 Jan 2018 13:53:44 +0000 Subject: [PATCH 05/14] Test and fix bug for snapshot searching The logic which contructed a list of values for parameters with multiple values was flawed in that e.g. `Subnet.1` and `Subnet.10` would be have their values counted against `Subnet.1` because they share a prefix. This now checks for a starting `.` before counting that name as having the requested prefix. --- moto/core/responses.py | 4 ++++ tests/test_ec2/test_elastic_block_store.py | 25 +++++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/moto/core/responses.py b/moto/core/responses.py index d254d1f85..278a24dc4 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -345,6 +345,10 @@ class BaseResponse(_TemplateEnvironmentMixin): if is_tracked(name) or not name.startswith(param_prefix): continue + if len(name) > len(param_prefix) and \ + not name[len(param_prefix):].startswith('.'): + continue + match = self.param_list_regex.search(name[len(param_prefix):]) if len(name) > len(param_prefix) else None if match: prefix = param_prefix + match.group(1) diff --git a/tests/test_ec2/test_elastic_block_store.py b/tests/test_ec2/test_elastic_block_store.py index 9c07f38d6..fc0677cfe 100644 --- a/tests/test_ec2/test_elastic_block_store.py +++ b/tests/test_ec2/test_elastic_block_store.py @@ -5,10 +5,11 @@ from nose.tools import assert_raises from moto.ec2 import ec2_backends import boto +import boto3 from boto.exception import EC2ResponseError import sure # noqa -from moto import mock_ec2_deprecated +from moto import mock_ec2_deprecated, mock_ec2 @mock_ec2_deprecated @@ -579,3 +580,25 @@ def test_volume_tag_escaping(): snaps = [snap for snap in conn.get_all_snapshots() if snap.id == snapshot.id] dict(snaps[0].tags).should.equal({'key': ''}) + + +@mock_ec2 +def test_search_for_many_snapshots(): + ec2_client = boto3.client('ec2', region_name='eu-west-1') + + volume_response = ec2_client.create_volume( + AvailabilityZone='eu-west-1a', Size=10 + ) + + snapshot_ids = [] + for i in range(1, 20): + create_snapshot_response = ec2_client.create_snapshot( + VolumeId=volume_response['VolumeId'] + ) + snapshot_ids.append(create_snapshot_response['SnapshotId']) + + snapshots_response = ec2_client.describe_snapshots( + SnapshotIds=snapshot_ids + ) + + assert len(snapshots_response['Snapshots']) == len(snapshot_ids) From 363f734e2b80f976aaba0f80ac10849a6d0bde63 Mon Sep 17 00:00:00 2001 From: rhard7 Date: Mon, 29 Jan 2018 12:37:23 -0800 Subject: [PATCH 06/14] fixes apigateway timestamp to match aws --- moto/apigateway/models.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index e7ff98119..cc8696104 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -1,12 +1,11 @@ from __future__ import absolute_import from __future__ import unicode_literals -import datetime import requests +import time from moto.packages.responses import responses from moto.core import BaseBackend, BaseModel -from moto.core.utils import iso_8601_datetime_with_milliseconds from .utils import create_id from .exceptions import StageNotFoundException @@ -20,8 +19,7 @@ class Deployment(BaseModel, dict): self['id'] = deployment_id self['stageName'] = name self['description'] = description - self['createdDate'] = iso_8601_datetime_with_milliseconds( - datetime.datetime.now()) + self['createdDate'] = int(time.time()) class IntegrationResponse(BaseModel, dict): @@ -300,7 +298,7 @@ class RestAPI(BaseModel): self.region_name = region_name self.name = name self.description = description - self.create_date = datetime.datetime.utcnow() + self.create_date = int(time.time()) self.deployments = {} self.stages = {} @@ -313,7 +311,7 @@ class RestAPI(BaseModel): "id": self.id, "name": self.name, "description": self.description, - "createdDate": iso_8601_datetime_with_milliseconds(self.create_date), + "createdDate": int(time.time()), } def add_child(self, path, parent_id=None): From c7bcbadc6ec1e32ff80cc805834ecb03fabeef42 Mon Sep 17 00:00:00 2001 From: Taro Sato Date: Tue, 30 Jan 2018 13:48:04 -0800 Subject: [PATCH 07/14] Fix the S3 HEAD response body --- moto/s3/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 8d2caf098..1b32698e4 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -172,7 +172,7 @@ class ResponseObject(_TemplateEnvironmentMixin): # HEAD (which the real API responds with), and instead # raises NoSuchBucket, leading to inconsistency in # error response between real and mocked responses. - return 404, {}, "Not Found" + return 404, {}, "" return 200, {}, "" def _bucket_response_get(self, bucket_name, querystring, headers): From 5e70d0ce4c24d8f365b0998fb12535dceb0cf416 Mon Sep 17 00:00:00 2001 From: Taro Sato Date: Tue, 30 Jan 2018 16:10:43 -0800 Subject: [PATCH 08/14] Support both virtual-hosted-style and path-style URLs for region name parsing --- moto/s3/responses.py | 9 +++------ moto/s3/utils.py | 19 +++++++++++++++++++ tests/test_s3/test_s3_utils.py | 20 +++++++++++++++++++- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 8d2caf098..57c435b30 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -18,10 +18,10 @@ from .exceptions import BucketAlreadyExists, S3ClientError, MissingBucket, Missi MalformedACLError from .models import s3_backend, get_canned_acl, FakeGrantee, FakeGrant, FakeAcl, FakeKey, FakeTagging, FakeTagSet, \ FakeTag -from .utils import bucket_name_from_url, metadata_from_headers +from .utils import bucket_name_from_url, metadata_from_headers, parse_region_from_url from xml.dom import minidom -REGION_URL_REGEX = r'\.s3-(.+?)\.amazonaws\.com' + DEFAULT_REGION_NAME = 'us-east-1' @@ -128,10 +128,7 @@ class ResponseObject(_TemplateEnvironmentMixin): parsed_url = urlparse(full_url) querystring = parse_qs(parsed_url.query, keep_blank_values=True) method = request.method - region_name = DEFAULT_REGION_NAME - region_match = re.search(REGION_URL_REGEX, full_url) - if region_match: - region_name = region_match.groups()[0] + region_name = parse_region_from_url(full_url) bucket_name = self.parse_bucket_name_from_url(request, full_url) if not bucket_name: diff --git a/moto/s3/utils.py b/moto/s3/utils.py index a121eae3a..8968d2ad2 100644 --- a/moto/s3/utils.py +++ b/moto/s3/utils.py @@ -1,4 +1,5 @@ from __future__ import unicode_literals +import logging from boto.s3.key import Key import re @@ -6,6 +7,10 @@ import six from six.moves.urllib.parse import urlparse, unquote import sys + +log = logging.getLogger(__name__) + + bucket_name_regex = re.compile("(.+).s3(.*).amazonaws.com") @@ -27,6 +32,20 @@ def bucket_name_from_url(url): return None +REGION_URL_REGEX = re.compile( + r'^https?://(s3[-\.](?P.+)\.amazonaws\.com/(.+)|' + r'(.+)\.s3-(?P.+)\.amazonaws\.com)/?') + + +def parse_region_from_url(url): + match = REGION_URL_REGEX.search(url) + if match: + region = match.group('region1') or match.group('region2') + else: + region = 'us-east-1' + return region + + def metadata_from_headers(headers): metadata = {} meta_regex = re.compile( diff --git a/tests/test_s3/test_s3_utils.py b/tests/test_s3/test_s3_utils.py index b4f56d89a..f1dfc04d1 100644 --- a/tests/test_s3/test_s3_utils.py +++ b/tests/test_s3/test_s3_utils.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals from sure import expect -from moto.s3.utils import bucket_name_from_url, _VersionedKeyStore +from moto.s3.utils import bucket_name_from_url, _VersionedKeyStore, parse_region_from_url def test_base_url(): @@ -53,3 +53,21 @@ def test_versioned_key_store(): d.setlist('key', [[1], [2]]) d['key'].should.have.length_of(1) d.getlist('key').should.be.equal([[1], [2]]) + + +def test_parse_region_from_url(): + expected = 'us-west-2' + for url in ['http://s3-us-west-2.amazonaws.com/bucket', + 'http://s3.us-west-2.amazonaws.com/bucket', + 'http://bucket.s3-us-west-2.amazonaws.com', + 'https://s3-us-west-2.amazonaws.com/bucket', + 'https://s3.us-west-2.amazonaws.com/bucket', + 'https://bucket.s3-us-west-2.amazonaws.com']: + parse_region_from_url(url).should.equal(expected) + + expected = 'us-east-1' + for url in ['http://s3.amazonaws.com/bucket', + 'http://bucket.s3.amazonaws.com', + 'https://s3.amazonaws.com/bucket', + 'https://bucket.s3.amazonaws.com']: + parse_region_from_url(url).should.equal(expected) From d090a8188c8842467acda141a3d25841c9df2dee Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Thu, 1 Feb 2018 11:05:19 +0900 Subject: [PATCH 09/14] fixing version number Fixes #1481 --- moto/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/__init__.py b/moto/__init__.py index 9d292a3e1..c38212b42 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging # logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '1.2.0', +__version__ = '1.2.0' from .acm import mock_acm # flake8: noqa from .apigateway import mock_apigateway, mock_apigateway_deprecated # flake8: noqa From 5f61950096e2d33cfb42021d5f661a71ae192357 Mon Sep 17 00:00:00 2001 From: Hugo Lopes Tavares Date: Thu, 1 Feb 2018 17:09:10 -0500 Subject: [PATCH 10/14] Make SpotPrice optional when requesting a spot fleet When price is omitted, AWS will default to the on-demand price --- moto/ec2/models.py | 5 +- moto/ec2/responses/spot_fleets.py | 6 +- requirements-dev.txt | 2 +- .../test_cloudformation_stack_integration.py | 72 +++++++++++++++++++ tests/test_ec2/test_spot_fleet.py | 27 +++++++ 5 files changed, 108 insertions(+), 4 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index f877d3772..bfc672ed7 100755 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -2943,7 +2943,7 @@ class SpotFleetRequest(TaggedEC2Resource): 'Properties']['SpotFleetRequestConfigData'] ec2_backend = ec2_backends[region_name] - spot_price = properties['SpotPrice'] + spot_price = properties.get('SpotPrice') target_capacity = properties['TargetCapacity'] iam_fleet_role = properties['IamFleetRole'] allocation_strategy = properties['AllocationStrategy'] @@ -2977,7 +2977,8 @@ class SpotFleetRequest(TaggedEC2Resource): launch_spec_index += 1 else: # lowestPrice cheapest_spec = sorted( - self.launch_specs, key=lambda spec: float(spec.spot_price))[0] + # FIXME: change `+inf` to the on demand price scaled to weighted capacity when it's not present + self.launch_specs, key=lambda spec: float(spec.spot_price or '+inf'))[0] weight_so_far = weight_to_add + (weight_to_add % cheapest_spec.weighted_capacity) weight_map[cheapest_spec] = int( weight_so_far // cheapest_spec.weighted_capacity) diff --git a/moto/ec2/responses/spot_fleets.py b/moto/ec2/responses/spot_fleets.py index 81d1e0146..0366af9d6 100644 --- a/moto/ec2/responses/spot_fleets.py +++ b/moto/ec2/responses/spot_fleets.py @@ -40,7 +40,7 @@ class SpotFleets(BaseResponse): def request_spot_fleet(self): spot_config = self._get_dict_param("SpotFleetRequestConfig.") - spot_price = spot_config['spot_price'] + spot_price = spot_config.get('spot_price') target_capacity = spot_config['target_capacity'] iam_fleet_role = spot_config['iam_fleet_role'] allocation_strategy = spot_config['allocation_strategy'] @@ -78,7 +78,9 @@ DESCRIBE_SPOT_FLEET_TEMPLATE = """ Date: Mon, 19 Feb 2018 17:28:35 +0200 Subject: [PATCH 11/14] Change lambda backend to support docker changes --- moto/awslambda/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index 947691bcf..3c3d3ea66 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -104,7 +104,7 @@ class _DockerDataVolumeContext: # It doesn't exist so we need to create it self._vol_ref.volume = self._lambda_func.docker_client.volumes.create(self._lambda_func.code_sha_256) - container = self._lambda_func.docker_client.containers.run('alpine', 'sleep 100', volumes={self.name: '/tmp/data'}, detach=True) + container = self._lambda_func.docker_client.containers.run('alpine', 'sleep 100', volumes={self.name: {'bind': '/tmp/data', 'mode': 'rw'}}, detach=True) try: tar_bytes = zip2tar(self._lambda_func.code_bytes) container.put_archive('/tmp/data', tar_bytes) @@ -309,7 +309,7 @@ class LambdaFunction(BaseModel): finally: if container: try: - exit_code = container.wait(timeout=300) + exit_code = container.wait(timeout=300)['StatusCode'] except requests.exceptions.ReadTimeout: exit_code = -1 container.stop() From 63be8a6c3869675a0c8f20ef6e3763bc5c037890 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Tue, 6 Mar 2018 08:06:12 -0500 Subject: [PATCH 12/14] Fix test coverage report. --- setup.cfg | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup.cfg b/setup.cfg index 3c6e79cf3..fb04c16a8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,8 @@ +[nosetests] +verbosity=1 +detailed-errors=1 +with-coverage=1 +cover-package=moto + [bdist_wheel] universal=1 From 9a8b36debc633b29b68c2f5c313632cc4ecb6b57 Mon Sep 17 00:00:00 2001 From: Dave Golombek Date: Tue, 6 Mar 2018 16:56:15 -0500 Subject: [PATCH 13/14] ELBv2.create_listener links TargetGroup to LB In order to search target_groups by LB, we need this link in place. Resolves #1500 --- moto/elbv2/models.py | 4 ++++ tests/test_elbv2/test_elbv2.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/moto/elbv2/models.py b/moto/elbv2/models.py index 726b1a164..8921581d3 100644 --- a/moto/elbv2/models.py +++ b/moto/elbv2/models.py @@ -486,6 +486,10 @@ class ELBv2Backend(BaseBackend): arn = load_balancer_arn.replace(':loadbalancer/', ':listener/') + "/%s%s" % (port, id(self)) listener = FakeListener(load_balancer_arn, arn, protocol, port, ssl_policy, certificate, default_actions) balancer.listeners[listener.arn] = listener + for action in default_actions: + if action['target_group_arn'] in self.target_groups.keys(): + target_group = self.target_groups[action['target_group_arn']] + target_group.load_balancer_arns.append(load_balancer_arn) return listener def describe_load_balancers(self, arns, names): diff --git a/tests/test_elbv2/test_elbv2.py b/tests/test_elbv2/test_elbv2.py index 4fb527525..ce092976a 100644 --- a/tests/test_elbv2/test_elbv2.py +++ b/tests/test_elbv2/test_elbv2.py @@ -340,6 +340,10 @@ def test_create_target_group_and_listeners(): 'Type': 'forward'}]) http_listener_arn = listener.get('ListenerArn') + response = conn.describe_target_groups(LoadBalancerArn=load_balancer_arn, + Names=['a-target']) + response.get('TargetGroups').should.have.length_of(1) + # And another with SSL response = conn.create_listener( LoadBalancerArn=load_balancer_arn, From 31eac49e1555c5345021a252cb0c95043197ea16 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Tue, 6 Mar 2018 17:48:23 -0500 Subject: [PATCH 14/14] Lock down version of aws-xray-sdk See https://travis-ci.org/spulec/moto/jobs/350056229 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 27c635944..57140401b 100755 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ install_requires = [ "mock", "docker>=2.5.1", "jsondiff==1.1.1", - "aws-xray-sdk>=0.93", + "aws-xray-sdk<0.96,>=0.93", ] extras_require = {