From 4a275ccf952bcbe45e12358ec50204ca4dca3664 Mon Sep 17 00:00:00 2001 From: Darien Hager Date: Thu, 19 Apr 2018 23:10:46 -0700 Subject: [PATCH 01/19] Add failing unit-test (errors treating dict as json string) --- .../test_cloudformation_stack_crud_boto3.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py index 1dbf80fb5..9bfae6174 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py @@ -148,10 +148,41 @@ dummy_import_template = { } } +dummy_redrive_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "MainQueue": { + "Type": "AWS::SQS::Queue", + "Properties": { + "QueueName": "mainqueue.fifo", + "FifoQueue": True, + "ContentBasedDeduplication": False, + "RedrivePolicy": { + "deadLetterTargetArn": { + "Fn::GetAtt": [ + "DeadLetterQueue", + "Arn" + ] + }, + "maxReceiveCount": 5 + } + } + }, + "DeadLetterQueue": { + "Type": "AWS::SQS::Queue", + "Properties": { + "FifoQueue": True + } + }, + } +} + dummy_template_json = json.dumps(dummy_template) dummy_update_template_json = json.dumps(dummy_update_template) dummy_output_template_json = json.dumps(dummy_output_template) dummy_import_template_json = json.dumps(dummy_import_template) +dummy_redrive_template_json = json.dumps(dummy_redrive_template) + @mock_cloudformation @@ -746,3 +777,19 @@ def test_stack_with_imports(): output = output_stack.outputs[0]['OutputValue'] queue = ec2_resource.get_queue_by_name(QueueName=output) queue.should_not.be.none + + +@mock_sqs +@mock_cloudformation +def test_non_json_redrive_policy(): + cf = boto3.resource('cloudformation', region_name='us-east-1') + + stack = cf.create_stack( + StackName="test_stack1", + TemplateBody=dummy_redrive_template_json + ) + + stack.Resource('MainQueue').resource_status\ + .should.equal("CREATE_COMPLETE") + stack.Resource('DeadLetterQueue').resource_status\ + .should.equal("CREATE_COMPLETE") From 5cd4d5e02f41dcfe9fa20e7b7faa943f94d5fccf Mon Sep 17 00:00:00 2001 From: Darien Hager Date: Thu, 19 Apr 2018 23:25:10 -0700 Subject: [PATCH 02/19] Change SQS model to support non-JSON redrive policies. Does not affect other limitations in SQS APIs. --- moto/sqs/models.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/moto/sqs/models.py b/moto/sqs/models.py index 044759e4f..48e7409ef 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -232,11 +232,18 @@ class Queue(BaseModel): self.last_modified_timestamp = now - def _setup_dlq(self, policy_json): - try: - self.redrive_policy = json.loads(policy_json) - except ValueError: - raise RESTError('InvalidParameterValue', 'Redrive policy does not contain valid json') + def _setup_dlq(self, policy): + + if isinstance(policy, six.text_type): + try: + self.redrive_policy = json.loads(policy) + except ValueError: + raise RESTError('InvalidParameterValue', 'Redrive policy is not a dict or valid json') + elif isinstance(policy, dict): + self.redrive_policy = policy + else: + raise RESTError('InvalidParameterValue', 'Redrive policy is not a dict or valid json') + if 'deadLetterTargetArn' not in self.redrive_policy: raise RESTError('InvalidParameterValue', 'Redrive policy does not contain deadLetterTargetArn') From 3c9d8bca467af10550e92e482ad129cca4e7ebc9 Mon Sep 17 00:00:00 2001 From: Darien Hager Date: Fri, 20 Apr 2018 11:46:12 -0700 Subject: [PATCH 03/19] Remove whitespace to satisfy flake8 formatting --- moto/sqs/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/moto/sqs/models.py b/moto/sqs/models.py index 48e7409ef..71ba9c7f3 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -244,7 +244,6 @@ class Queue(BaseModel): else: raise RESTError('InvalidParameterValue', 'Redrive policy is not a dict or valid json') - if 'deadLetterTargetArn' not in self.redrive_policy: raise RESTError('InvalidParameterValue', 'Redrive policy does not contain deadLetterTargetArn') if 'maxReceiveCount' not in self.redrive_policy: From ac016a7bb361d84b03201640dce690609c394fef Mon Sep 17 00:00:00 2001 From: Fujimoto Seiji Date: Tue, 24 Apr 2018 11:12:17 +0900 Subject: [PATCH 04/19] Implement describe_log_groups() method for CloudWatchLogs This patch teaches `LogsResponse` class how to handle the DescribeLogGroups request, so that we can mock out the `boto.describe_log_groups()` call. With this change in place, we can write as below: @mock_logs def test_log_group(): conn = boto3.client('logs', 'us-west-2') some_method_to_init_log_groups() resp = conn.describe_log_groups(logGroupNamePrefix='myapp') assert ... This should be fairly useful for a number of programs which handles CloudWatchLogs. Signed-off-by: Fujimoto Seiji --- moto/logs/models.py | 28 ++++++++++++++++++++++++++++ moto/logs/responses.py | 12 ++++++++++++ tests/test_logs/test_logs.py | 4 ++++ 3 files changed, 44 insertions(+) diff --git a/moto/logs/models.py b/moto/logs/models.py index 3ae697a27..3e1c7b955 100644 --- a/moto/logs/models.py +++ b/moto/logs/models.py @@ -132,6 +132,9 @@ class LogGroup: def __init__(self, region, name, tags): self.name = name self.region = region + self.arn = "arn:aws:logs:{region}:1:log-group:{log_group}".format( + region=region, log_group=name) + self.creationTime = unix_time_millis() self.tags = tags self.streams = dict() # {name: LogStream} @@ -197,6 +200,16 @@ class LogGroup: searched_streams = [{"logStreamName": stream.logStreamName, "searchedCompletely": True} for stream in streams] return events_page, next_token, searched_streams + def to_describe_dict(self): + return { + "arn": self.arn, + "creationTime": self.creationTime, + "logGroupName": self.name, + "metricFilterCount": 0, + "retentionInDays": 30, + "storedBytes": sum(s.storedBytes for s in self.streams.values()), + } + class LogsBackend(BaseBackend): def __init__(self, region_name): @@ -223,6 +236,21 @@ class LogsBackend(BaseBackend): raise ResourceNotFoundException() del self.groups[log_group_name] + def describe_log_groups(self, limit, log_group_name_prefix, next_token): + if log_group_name_prefix is None: + log_group_name_prefix = '' + if next_token is None: + next_token = 0 + + groups = sorted(group.to_describe_dict() for name, group in self.groups.items() if name.startswith(log_group_name_prefix)) + groups_page = groups[next_token:next_token + limit] + + next_token += limit + if next_token >= len(groups): + next_token = None + + return groups_page, next_token + def create_log_stream(self, log_group_name, log_stream_name): if log_group_name not in self.groups: raise ResourceNotFoundException() diff --git a/moto/logs/responses.py b/moto/logs/responses.py index 7bf481908..4bec86cb2 100644 --- a/moto/logs/responses.py +++ b/moto/logs/responses.py @@ -33,6 +33,18 @@ class LogsResponse(BaseResponse): self.logs_backend.delete_log_group(log_group_name) return '' + def describe_log_groups(self): + log_group_name_prefix = self._get_param('logGroupNamePrefix') + next_token = self._get_param('nextToken') + limit = self._get_param('limit', 50) + assert limit <= 50 + groups, next_token = self.logs_backend.describe_log_groups( + limit, log_group_name_prefix, next_token) + return json.dumps({ + "logGroups": groups, + "nextToken": next_token + }) + def create_log_stream(self): log_group_name = self._get_param('logGroupName') log_stream_name = self._get_param('logStreamName') diff --git a/tests/test_logs/test_logs.py b/tests/test_logs/test_logs.py index a9a7f5260..3f924cc55 100644 --- a/tests/test_logs/test_logs.py +++ b/tests/test_logs/test_logs.py @@ -13,6 +13,10 @@ def test_log_group_create(): conn = boto3.client('logs', 'us-west-2') log_group_name = 'dummy' response = conn.create_log_group(logGroupName=log_group_name) + + response = conn.describe_log_groups(logGroupNamePrefix=log_group_name) + assert len(response['logGroups']) == 1 + response = conn.delete_log_group(logGroupName=log_group_name) From e1d9c2878f2a5bc2b628e0631b9d9e590c7fec3b Mon Sep 17 00:00:00 2001 From: Brian Pandola Date: Tue, 24 Apr 2018 17:30:17 -0700 Subject: [PATCH 05/19] Add support for Redshift.Waiter.ClusterRestored * Add `restored_from_snapshot` boolean to Cluster metadata. * Return `RestoreStatus` from describe_db_clusters if cluster was restored from a snapshot. Fixes #1506 --- moto/redshift/models.py | 17 +++++++++-- tests/test_redshift/test_redshift.py | 42 ++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/moto/redshift/models.py b/moto/redshift/models.py index 4eb9d6b5c..4eafcfc79 100644 --- a/moto/redshift/models.py +++ b/moto/redshift/models.py @@ -73,7 +73,8 @@ class Cluster(TaggableResourceMixin, BaseModel): preferred_maintenance_window, cluster_parameter_group_name, automated_snapshot_retention_period, port, cluster_version, allow_version_upgrade, number_of_nodes, publicly_accessible, - encrypted, region_name, tags=None, iam_roles_arn=None): + encrypted, region_name, tags=None, iam_roles_arn=None, + restored_from_snapshot=False): super(Cluster, self).__init__(region_name, tags) self.redshift_backend = redshift_backend self.cluster_identifier = cluster_identifier @@ -119,6 +120,7 @@ class Cluster(TaggableResourceMixin, BaseModel): self.number_of_nodes = 1 self.iam_roles_arn = iam_roles_arn or [] + self.restored_from_snapshot = restored_from_snapshot @classmethod def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): @@ -242,7 +244,15 @@ class Cluster(TaggableResourceMixin, BaseModel): "IamRoleArn": iam_role_arn } for iam_role_arn in self.iam_roles_arn] } - + if self.restored_from_snapshot: + json_response['RestoreStatus'] = { + 'Status': 'completed', + 'CurrentRestoreRateInMegaBytesPerSecond': 123.0, + 'SnapshotSizeInMegaBytes': 123, + 'ProgressInMegaBytes': 123, + 'ElapsedTimeInSeconds': 123, + 'EstimatedTimeToCompletionInSeconds': 123 + } try: json_response['ClusterSnapshotCopyStatus'] = self.cluster_snapshot_copy_status except AttributeError: @@ -639,7 +649,8 @@ class RedshiftBackend(BaseBackend): "cluster_version": snapshot.cluster.cluster_version, "number_of_nodes": snapshot.cluster.number_of_nodes, "encrypted": snapshot.cluster.encrypted, - "tags": snapshot.cluster.tags + "tags": snapshot.cluster.tags, + "restored_from_snapshot": True } create_kwargs.update(kwargs) return self.create_cluster(**create_kwargs) diff --git a/tests/test_redshift/test_redshift.py b/tests/test_redshift/test_redshift.py index 96e3ee5b3..6e027b86c 100644 --- a/tests/test_redshift/test_redshift.py +++ b/tests/test_redshift/test_redshift.py @@ -818,6 +818,48 @@ def test_create_cluster_from_snapshot(): new_cluster['Endpoint']['Port'].should.equal(1234) +@mock_redshift +def test_create_cluster_from_snapshot_with_waiter(): + client = boto3.client('redshift', region_name='us-east-1') + original_cluster_identifier = 'original-cluster' + original_snapshot_identifier = 'original-snapshot' + new_cluster_identifier = 'new-cluster' + + client.create_cluster( + ClusterIdentifier=original_cluster_identifier, + ClusterType='single-node', + NodeType='ds2.xlarge', + MasterUsername='username', + MasterUserPassword='password', + ) + client.create_cluster_snapshot( + SnapshotIdentifier=original_snapshot_identifier, + ClusterIdentifier=original_cluster_identifier + ) + response = client.restore_from_cluster_snapshot( + ClusterIdentifier=new_cluster_identifier, + SnapshotIdentifier=original_snapshot_identifier, + Port=1234 + ) + response['Cluster']['ClusterStatus'].should.equal('creating') + + client.get_waiter('cluster_restored').wait( + ClusterIdentifier=new_cluster_identifier, + WaiterConfig={ + 'Delay': 1, + 'MaxAttempts': 2, + } + ) + + response = client.describe_clusters( + ClusterIdentifier=new_cluster_identifier + ) + new_cluster = response['Clusters'][0] + new_cluster['NodeType'].should.equal('ds2.xlarge') + new_cluster['MasterUsername'].should.equal('username') + new_cluster['Endpoint']['Port'].should.equal(1234) + + @mock_redshift def test_create_cluster_from_non_existent_snapshot(): client = boto3.client('redshift', region_name='us-east-1') From 94fa94c2dfe130fd0714a7fda2fffbb86329774b Mon Sep 17 00:00:00 2001 From: Tom Grace Date: Wed, 2 May 2018 13:31:35 +0100 Subject: [PATCH 06/19] 1606 Add additional fields to Batch job status endpoint --- moto/batch/models.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/moto/batch/models.py b/moto/batch/models.py index 8b3b81ccb..c47ca6e97 100644 --- a/moto/batch/models.py +++ b/moto/batch/models.py @@ -295,6 +295,14 @@ class Job(threading.Thread, BaseModel): } if self.job_stopped: result['stoppedAt'] = datetime2int(self.job_stopped_at) + result['container'] = {} + result['container']['command'] = ['/bin/sh -c "for a in `seq 1 10`; do echo Hello World; sleep 1; done"'] + result['container']['privileged'] = False + result['container']['readonlyRootFilesystem'] = False + result['container']['ulimits'] = {} + result['container']['vcpus'] = 1 + result['container']['volumes'] = '' + result['container']['logStreamName'] = self.log_stream_name if self.job_stopped_reason is not None: result['statusReason'] = self.job_stopped_reason return result @@ -378,6 +386,7 @@ class Job(threading.Thread, BaseModel): # Send to cloudwatch log_group = '/aws/batch/job' stream_name = '{0}/default/{1}'.format(self.job_definition.name, self.job_id) + self.log_stream_name = stream_name self._log_backend.ensure_log_group(log_group, None) self._log_backend.create_log_stream(log_group, stream_name) self._log_backend.put_log_events(log_group, stream_name, logs, None) From 7a57dc203422396a4b8ae23cd5bb0955222163a9 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Thu, 3 May 2018 01:40:49 -0700 Subject: [PATCH 07/19] fix errors --- moto/s3/models.py | 3 +++ moto/s3/responses.py | 8 +++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 3b4623d61..9e58fdb47 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -859,6 +859,9 @@ class S3Backend(BaseBackend): if str(key.version_id) != str(version_id) ] ) + + if not bucket.keys.getlist(key_name): + bucket.keys.pop(key_name) return True except KeyError: return False diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 02a9ac40e..f8b3077d3 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -24,7 +24,6 @@ from xml.dom import minidom DEFAULT_REGION_NAME = 'us-east-1' - def parse_key_name(pth): return pth.lstrip("/") @@ -706,8 +705,11 @@ class ResponseObject(_TemplateEnvironmentMixin): if 'x-amz-copy-source' in request.headers: # Copy key - src_key_parsed = urlparse(unquote(request.headers.get("x-amz-copy-source"))) - src_bucket, src_key = src_key_parsed.path.lstrip("/").split("/", 1) + # you can have a quoted ?version=abc with a version Id, so work on + # we need to parse the unquoted string first + src_key_parsed = urlparse(request.headers.get("x-amz-copy-source")) + src_bucket, src_key = unquote(src_key_parsed.path).\ + lstrip("/").split("/", 1) src_version_id = parse_qs(src_key_parsed.query).get( 'versionId', [None])[0] self.backend.copy_key(src_bucket, src_key, bucket_name, key_name, From 93a404ec3726c86fb7453e398a3102afbcb3049a Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Thu, 3 May 2018 02:10:17 -0700 Subject: [PATCH 08/19] pep --- moto/s3/responses.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index f8b3077d3..5e7cf0fe5 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -24,6 +24,7 @@ from xml.dom import minidom DEFAULT_REGION_NAME = 'us-east-1' + def parse_key_name(pth): return pth.lstrip("/") From 07540a35fe23d74cebf0a3e136745dd777a1fb77 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Thu, 3 May 2018 02:30:29 -0700 Subject: [PATCH 09/19] add unittest --- tests/test_s3/test_s3.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 369426758..9f37791cb 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -1405,6 +1405,19 @@ def test_boto3_deleted_versionings_list(): assert len(listed['Contents']) == 1 +@mock_s3 +def test_boto3_delete_versioned_bucket(): + client = boto3.client('s3', region_name='us-east-1') + + client.create_bucket(Bucket='blah') + client.put_bucket_versioning(Bucket='blah', VersioningConfiguration={'Status': 'Enabled'}) + + resp = client.put_object(Bucket='blah', Key='test1', Body=b'test1') + client.delete_object(Bucket='blah', Key='test1', VersionId=resp["VersionId"]) + + client.delete_bucket(Bucket='blah') + + @mock_s3 def test_boto3_head_object_if_modified_since(): s3 = boto3.client('s3', region_name='us-east-1') From 5d6655a7eed858caf90cbff6bcea2e18e13d38d7 Mon Sep 17 00:00:00 2001 From: djkiourtsis Date: Thu, 26 Apr 2018 15:30:33 -0400 Subject: [PATCH 10/19] Add gocloud backend to lambda backends Boto does does not include the govcloud backends when displaying lambda regions. In order to test lambda with a govcloud region, the region must be explicitly added. --- moto/awslambda/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index 80b4ffba3..d49df81c7 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -675,3 +675,4 @@ lambda_backends = {_region.name: LambdaBackend(_region.name) for _region in boto.awslambda.regions()} lambda_backends['ap-southeast-2'] = LambdaBackend('ap-southeast-2') +lambda_backends['us-gov-west-1'] = LambdaBackend('us-gov-west-1') From 2ac8954b13da428bf1fdff63893e2d9714f72c18 Mon Sep 17 00:00:00 2001 From: jbergknoff-10e Date: Thu, 3 May 2018 14:09:56 -0500 Subject: [PATCH 11/19] Accept paths to user-controlled SSL cert --- moto/server.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/moto/server.py b/moto/server.py index e9f4c0904..7862dfe9a 100644 --- a/moto/server.py +++ b/moto/server.py @@ -186,9 +186,18 @@ def main(argv=sys.argv[1:]): parser.add_argument( '-s', '--ssl', action='store_true', - help='Enable SSL encrypted connection (use https://... URL)', + help='Enable SSL encrypted connection with auto-generated certificate (use https://... URL)', default=False ) + parser.add_argument( + '-c', '--ssl-cert', type=str, + help='Path to SSL certificate', + default=None) + parser.add_argument( + '-k', '--ssl-key', type=str, + help='Path to SSL private key', + default=None) + args = parser.parse_args(argv) @@ -197,9 +206,15 @@ def main(argv=sys.argv[1:]): create_backend_app, service=args.service) main_app.debug = True + ssl_context = None + if args.ssl_key and args.ssl_cert: + ssl_context = (args.ssl_cert, args.ssl_key) + elif args.ssl: + ssl_context = 'adhoc' + run_simple(args.host, args.port, main_app, threaded=True, use_reloader=args.reload, - ssl_context='adhoc' if args.ssl else None) + ssl_context=ssl_context) if __name__ == '__main__': From 86fed8ba27a489d09a6c590dcdd76053c2357dd6 Mon Sep 17 00:00:00 2001 From: jbergknoff-10e Date: Fri, 4 May 2018 16:42:16 -0500 Subject: [PATCH 12/19] lint --- moto/server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/moto/server.py b/moto/server.py index 7862dfe9a..d76b5601c 100644 --- a/moto/server.py +++ b/moto/server.py @@ -198,7 +198,6 @@ def main(argv=sys.argv[1:]): help='Path to SSL private key', default=None) - args = parser.parse_args(argv) # Wrap the main application From b08fc8beded178f3de6580e471faf6be380c42bf Mon Sep 17 00:00:00 2001 From: Ben Jolitz Date: Fri, 4 May 2018 16:30:47 -0700 Subject: [PATCH 13/19] allow topic names to start/end with `_`, `-` --- moto/sns/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/sns/models.py b/moto/sns/models.py index 1c1be6680..6ee6c79ed 100644 --- a/moto/sns/models.py +++ b/moto/sns/models.py @@ -240,7 +240,7 @@ class SNSBackend(BaseBackend): self.sms_attributes.update(attrs) def create_topic(self, name): - fails_constraints = not re.match(r'^[a-zA-Z0-9](?:[A-Za-z0-9_-]{0,253}[a-zA-Z0-9])?$', name) + fails_constraints = not re.match(r'^[a-zA-Z0-9\_\-]{0,256}$', name) if fails_constraints: raise InvalidParameterValue("Topic names must be made up of only uppercase and lowercase ASCII letters, numbers, underscores, and hyphens, and must be between 1 and 256 characters long.") candidate_topic = Topic(name, self) From 0ed18c8b8a23193d6d5989e3869abbf3cb699e6b Mon Sep 17 00:00:00 2001 From: Ben Jolitz Date: Fri, 4 May 2018 16:33:43 -0700 Subject: [PATCH 14/19] remove extraneous backslash --- moto/sns/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/sns/models.py b/moto/sns/models.py index 6ee6c79ed..65dcc6cff 100644 --- a/moto/sns/models.py +++ b/moto/sns/models.py @@ -240,7 +240,7 @@ class SNSBackend(BaseBackend): self.sms_attributes.update(attrs) def create_topic(self, name): - fails_constraints = not re.match(r'^[a-zA-Z0-9\_\-]{0,256}$', name) + fails_constraints = not re.match(r'^[a-zA-Z0-9_-]{0,256}$', name) if fails_constraints: raise InvalidParameterValue("Topic names must be made up of only uppercase and lowercase ASCII letters, numbers, underscores, and hyphens, and must be between 1 and 256 characters long.") candidate_topic = Topic(name, self) From d21c387eb643e3c789469b846ec1c639408f09e0 Mon Sep 17 00:00:00 2001 From: Ben Jolitz Date: Fri, 4 May 2018 18:22:47 -0700 Subject: [PATCH 15/19] Support optional Source, parse from header The Email ``from`` header is either formatted as ``name
`` or ``address``. This commit will use `parseaddr` to extract a ``(name, address)`` tuple, which we will use the ``address`` to check if it's verified. Also support the case where ``Source`` is omitted (which AWS requires the ``from`` header to be set). --- moto/ses/models.py | 23 ++++++++++--- moto/ses/responses.py | 5 ++- tests/test_ses/test_ses_boto3.py | 56 ++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 5 deletions(-) diff --git a/moto/ses/models.py b/moto/ses/models.py index 179f4d8e0..b1135a406 100644 --- a/moto/ses/models.py +++ b/moto/ses/models.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import email +from email.utils import parseaddr from moto.core import BaseBackend, BaseModel from .exceptions import MessageRejectedError @@ -84,13 +85,27 @@ class SESBackend(BaseBackend): return message def send_raw_email(self, source, destinations, raw_data): - if source not in self.addresses: - raise MessageRejectedError( - "Did not have authority to send from email %s" % source - ) + if source is not None: + _, source_email_address = parseaddr(source) + if source_email_address not in self.addresses: + raise MessageRejectedError( + "Did not have authority to send from email %s" % source_email_address + ) recipient_count = len(destinations) message = email.message_from_string(raw_data) + if source is None: + if message['from'] is None: + raise MessageRejectedError( + "Source not specified" + ) + + _, source_email_address = parseaddr(message['from']) + if source_email_address not in self.addresses: + raise MessageRejectedError( + "Did not have authority to send from email %s" % source_email_address + ) + for header in 'TO', 'CC', 'BCC': recipient_count += sum( d.strip() and 1 or 0 diff --git a/moto/ses/responses.py b/moto/ses/responses.py index 6cd018aa6..bdf873836 100644 --- a/moto/ses/responses.py +++ b/moto/ses/responses.py @@ -75,7 +75,10 @@ class EmailResponse(BaseResponse): return template.render(message=message) def send_raw_email(self): - source = self.querystring.get('Source')[0] + source = self.querystring.get('Source') + if source is not None: + source, = source + raw_data = self.querystring.get('RawMessage.Data')[0] raw_data = base64.b64decode(raw_data) if six.PY3: diff --git a/tests/test_ses/test_ses_boto3.py b/tests/test_ses/test_ses_boto3.py index 5d39f61d4..e800b8035 100644 --- a/tests/test_ses/test_ses_boto3.py +++ b/tests/test_ses/test_ses_boto3.py @@ -136,3 +136,59 @@ def test_send_raw_email(): send_quota = conn.get_send_quota() sent_count = int(send_quota['SentLast24Hours']) sent_count.should.equal(2) + + +@mock_ses +def test_send_raw_email_without_source(): + conn = boto3.client('ses', region_name='us-east-1') + + message = MIMEMultipart() + message['Subject'] = 'Test' + message['From'] = 'test@example.com' + message['To'] = 'to@example.com, foo@example.com' + + # Message body + part = MIMEText('test file attached') + message.attach(part) + + # Attachment + part = MIMEText('contents of test file here') + part.add_header('Content-Disposition', 'attachment; filename=test.txt') + message.attach(part) + + kwargs = dict( + RawMessage={'Data': message.as_string()}, + ) + + conn.send_raw_email.when.called_with(**kwargs).should.throw(ClientError) + + conn.verify_email_identity(EmailAddress="test@example.com") + conn.send_raw_email(**kwargs) + + send_quota = conn.get_send_quota() + sent_count = int(send_quota['SentLast24Hours']) + sent_count.should.equal(2) + + +@mock_ses +def test_send_raw_email_without_source_or_from(): + conn = boto3.client('ses', region_name='us-east-1') + + message = MIMEMultipart() + message['Subject'] = 'Test' + message['To'] = 'to@example.com, foo@example.com' + + # Message body + part = MIMEText('test file attached') + message.attach(part) + # Attachment + part = MIMEText('contents of test file here') + part.add_header('Content-Disposition', 'attachment; filename=test.txt') + message.attach(part) + + kwargs = dict( + RawMessage={'Data': message.as_string()}, + ) + + conn.send_raw_email.when.called_with(**kwargs).should.throw(ClientError) + From 289d3614633f54d1a4a34bdfb85baf6e8c0bb1fd Mon Sep 17 00:00:00 2001 From: Ben Jolitz Date: Fri, 4 May 2018 19:16:12 -0700 Subject: [PATCH 16/19] add check for at least 1 character as the minumum length REF: http://boto3.readthedocs.io/en/latest/reference/services/sns.html#SNS.Client.create_topic --- moto/sns/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/sns/models.py b/moto/sns/models.py index 65dcc6cff..d6105c1d0 100644 --- a/moto/sns/models.py +++ b/moto/sns/models.py @@ -240,7 +240,7 @@ class SNSBackend(BaseBackend): self.sms_attributes.update(attrs) def create_topic(self, name): - fails_constraints = not re.match(r'^[a-zA-Z0-9_-]{0,256}$', name) + fails_constraints = not re.match(r'^[a-zA-Z0-9_-]{1,256}$', name) if fails_constraints: raise InvalidParameterValue("Topic names must be made up of only uppercase and lowercase ASCII letters, numbers, underscores, and hyphens, and must be between 1 and 256 characters long.") candidate_topic = Topic(name, self) From 45b529fef4c15e26e7850fc0777c7b8debc78f2d Mon Sep 17 00:00:00 2001 From: Ben Jolitz Date: Fri, 4 May 2018 19:17:56 -0700 Subject: [PATCH 17/19] parameterize topic name create/delete --- tests/test_sns/test_topics_boto3.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/tests/test_sns/test_topics_boto3.py b/tests/test_sns/test_topics_boto3.py index 95dd41f89..7d9a27b18 100644 --- a/tests/test_sns/test_topics_boto3.py +++ b/tests/test_sns/test_topics_boto3.py @@ -13,23 +13,24 @@ from moto.sns.models import DEFAULT_TOPIC_POLICY, DEFAULT_EFFECTIVE_DELIVERY_POL @mock_sns def test_create_and_delete_topic(): conn = boto3.client("sns", region_name="us-east-1") - conn.create_topic(Name="some-topic") + for topic_name in ('some-topic', '-some-topic-', '_some-topic_', 'a' * 256): + conn.create_topic(Name=topic_name) - topics_json = conn.list_topics() - topics = topics_json["Topics"] - topics.should.have.length_of(1) - topics[0]['TopicArn'].should.equal( - "arn:aws:sns:{0}:123456789012:some-topic" - .format(conn._client_config.region_name) - ) + topics_json = conn.list_topics() + topics = topics_json["Topics"] + topics.should.have.length_of(1) + topics[0]['TopicArn'].should.equal( + "arn:aws:sns:{0}:123456789012:{1}" + .format(conn._client_config.region_name, topic_name) + ) - # Delete the topic - conn.delete_topic(TopicArn=topics[0]['TopicArn']) + # Delete the topic + conn.delete_topic(TopicArn=topics[0]['TopicArn']) - # And there should now be 0 topics - topics_json = conn.list_topics() - topics = topics_json["Topics"] - topics.should.have.length_of(0) + # And there should now be 0 topics + topics_json = conn.list_topics() + topics = topics_json["Topics"] + topics.should.have.length_of(0) @mock_sns def test_create_topic_should_be_indempodent(): From 1a0a951b06c1da77bf61e498d501b488b328b506 Mon Sep 17 00:00:00 2001 From: bclodius Date: Sat, 5 May 2018 15:22:29 -0400 Subject: [PATCH 18/19] Fixes #1608 --- moto/cloudwatch/models.py | 7 ++++++- moto/cloudwatch/responses.py | 2 +- tests/test_cloudwatch/test_cloudwatch_boto3.py | 3 ++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py index ba6569981..441c74176 100644 --- a/moto/cloudwatch/models.py +++ b/moto/cloudwatch/models.py @@ -229,8 +229,13 @@ class CloudWatchBackend(BaseBackend): def put_metric_data(self, namespace, metric_data): for metric_member in metric_data: + # Preserve "datetime" for get_metric_statistics comparisons + timestamp = metric_member.get('Timestamp') + if timestamp is not None and type(timestamp) != datetime: + timestamp = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%fZ') + timestamp = timestamp.replace(tzinfo=tzutc()) self.metric_data.append(MetricDatum( - namespace, metric_member['MetricName'], float(metric_member['Value']), metric_member.get('Dimensions.member', _EMPTY_LIST), metric_member.get('Timestamp'))) + namespace, metric_member['MetricName'], float(metric_member['Value']), metric_member.get('Dimensions.member', _EMPTY_LIST), timestamp)) def get_metric_statistics(self, namespace, metric_name, start_time, end_time, period, stats): period_delta = timedelta(seconds=period) diff --git a/moto/cloudwatch/responses.py b/moto/cloudwatch/responses.py index c080d4620..8118f35ba 100644 --- a/moto/cloudwatch/responses.py +++ b/moto/cloudwatch/responses.py @@ -272,7 +272,7 @@ GET_METRIC_STATISTICS_TEMPLATE = """=2.5.1", "jsondiff==1.1.1", "aws-xray-sdk<0.96,>=0.93", - "responses", + "responses>=0.9.0", ] extras_require = {