From 37d63886406cb12fff7792ed87b3cb6f2364818c Mon Sep 17 00:00:00 2001 From: Brian Pandola Date: Thu, 19 Apr 2018 20:21:27 -0700 Subject: [PATCH 01/48] Fix rds.describe_db_snapshots bugs * Correctly return all snapshots for a given DBInstanceIdentifier. * If an invalid DBInstanceIdentifier is passed in, return an empty array instead of raising a ClientError (which is what AWS actually does). Fixes #1569 --- moto/rds2/models.py | 5 +++-- tests/test_rds2/test_rds2.py | 7 +++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/moto/rds2/models.py b/moto/rds2/models.py index 268ae5af2..eb0c0387a 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -722,10 +722,11 @@ class RDS2Backend(BaseBackend): def describe_snapshots(self, db_instance_identifier, db_snapshot_identifier): if db_instance_identifier: + db_instance_snapshots = [] for snapshot in self.snapshots.values(): if snapshot.database.db_instance_identifier == db_instance_identifier: - return [snapshot] - raise DBSnapshotNotFoundError() + db_instance_snapshots.append(snapshot) + return db_instance_snapshots if db_snapshot_identifier: if db_snapshot_identifier in self.snapshots: diff --git a/tests/test_rds2/test_rds2.py b/tests/test_rds2/test_rds2.py index ea0ab378f..78ba17b53 100644 --- a/tests/test_rds2/test_rds2.py +++ b/tests/test_rds2/test_rds2.py @@ -350,8 +350,6 @@ def test_describe_db_snapshots(): MasterUserPassword='hunter2', Port=1234, DBSecurityGroups=["my_sg"]) - conn.describe_db_snapshots.when.called_with( - DBInstanceIdentifier="db-primary-1").should.throw(ClientError) created = conn.create_db_snapshot(DBInstanceIdentifier='db-primary-1', DBSnapshotIdentifier='snapshot-1').get('DBSnapshot') @@ -366,6 +364,11 @@ def test_describe_db_snapshots(): snapshot.should.equal(created) snapshot.get('Engine').should.equal('postgres') + conn.create_db_snapshot(DBInstanceIdentifier='db-primary-1', + DBSnapshotIdentifier='snapshot-2') + snapshots = conn.describe_db_snapshots(DBInstanceIdentifier='db-primary-1').get('DBSnapshots') + snapshots.should.have.length_of(2) + @mock_rds2 def test_delete_db_snapshot(): From 4a275ccf952bcbe45e12358ec50204ca4dca3664 Mon Sep 17 00:00:00 2001 From: Darien Hager Date: Thu, 19 Apr 2018 23:10:46 -0700 Subject: [PATCH 02/48] Add failing unit-test (errors treating dict as json string) --- .../test_cloudformation_stack_crud_boto3.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py index 1dbf80fb5..9bfae6174 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py @@ -148,10 +148,41 @@ dummy_import_template = { } } +dummy_redrive_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "MainQueue": { + "Type": "AWS::SQS::Queue", + "Properties": { + "QueueName": "mainqueue.fifo", + "FifoQueue": True, + "ContentBasedDeduplication": False, + "RedrivePolicy": { + "deadLetterTargetArn": { + "Fn::GetAtt": [ + "DeadLetterQueue", + "Arn" + ] + }, + "maxReceiveCount": 5 + } + } + }, + "DeadLetterQueue": { + "Type": "AWS::SQS::Queue", + "Properties": { + "FifoQueue": True + } + }, + } +} + dummy_template_json = json.dumps(dummy_template) dummy_update_template_json = json.dumps(dummy_update_template) dummy_output_template_json = json.dumps(dummy_output_template) dummy_import_template_json = json.dumps(dummy_import_template) +dummy_redrive_template_json = json.dumps(dummy_redrive_template) + @mock_cloudformation @@ -746,3 +777,19 @@ def test_stack_with_imports(): output = output_stack.outputs[0]['OutputValue'] queue = ec2_resource.get_queue_by_name(QueueName=output) queue.should_not.be.none + + +@mock_sqs +@mock_cloudformation +def test_non_json_redrive_policy(): + cf = boto3.resource('cloudformation', region_name='us-east-1') + + stack = cf.create_stack( + StackName="test_stack1", + TemplateBody=dummy_redrive_template_json + ) + + stack.Resource('MainQueue').resource_status\ + .should.equal("CREATE_COMPLETE") + stack.Resource('DeadLetterQueue').resource_status\ + .should.equal("CREATE_COMPLETE") From 5cd4d5e02f41dcfe9fa20e7b7faa943f94d5fccf Mon Sep 17 00:00:00 2001 From: Darien Hager Date: Thu, 19 Apr 2018 23:25:10 -0700 Subject: [PATCH 03/48] Change SQS model to support non-JSON redrive policies. Does not affect other limitations in SQS APIs. --- moto/sqs/models.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/moto/sqs/models.py b/moto/sqs/models.py index 044759e4f..48e7409ef 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -232,11 +232,18 @@ class Queue(BaseModel): self.last_modified_timestamp = now - def _setup_dlq(self, policy_json): - try: - self.redrive_policy = json.loads(policy_json) - except ValueError: - raise RESTError('InvalidParameterValue', 'Redrive policy does not contain valid json') + def _setup_dlq(self, policy): + + if isinstance(policy, six.text_type): + try: + self.redrive_policy = json.loads(policy) + except ValueError: + raise RESTError('InvalidParameterValue', 'Redrive policy is not a dict or valid json') + elif isinstance(policy, dict): + self.redrive_policy = policy + else: + raise RESTError('InvalidParameterValue', 'Redrive policy is not a dict or valid json') + if 'deadLetterTargetArn' not in self.redrive_policy: raise RESTError('InvalidParameterValue', 'Redrive policy does not contain deadLetterTargetArn') From 3c9d8bca467af10550e92e482ad129cca4e7ebc9 Mon Sep 17 00:00:00 2001 From: Darien Hager Date: Fri, 20 Apr 2018 11:46:12 -0700 Subject: [PATCH 04/48] Remove whitespace to satisfy flake8 formatting --- moto/sqs/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/moto/sqs/models.py b/moto/sqs/models.py index 48e7409ef..71ba9c7f3 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -244,7 +244,6 @@ class Queue(BaseModel): else: raise RESTError('InvalidParameterValue', 'Redrive policy is not a dict or valid json') - if 'deadLetterTargetArn' not in self.redrive_policy: raise RESTError('InvalidParameterValue', 'Redrive policy does not contain deadLetterTargetArn') if 'maxReceiveCount' not in self.redrive_policy: From b25e80188aabacb88f4c1bdc4b7c9c549c5982d9 Mon Sep 17 00:00:00 2001 From: Fujimoto Seiji Date: Tue, 24 Apr 2018 15:02:17 +0900 Subject: [PATCH 05/48] AWSServiceSpec: Fix `TypeError` exceptions within json.load() The load() method provided by the built-in JSON module does not accept a byte-type value in Python 3.5 (or versions before), and will raise an exception if one is passed. For details, please see: https://bugs.python.org/issue17909 Thus, for better compatibility, we'd better decode the content of the JSON file before passing it to the parser, instead of letting the module to guess the encoding. --- moto/core/responses.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moto/core/responses.py b/moto/core/responses.py index ed4792083..0f133e72c 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -5,6 +5,7 @@ import datetime import json import logging import re +import io import pytz from moto.core.exceptions import DryRunClientError @@ -622,7 +623,7 @@ class AWSServiceSpec(object): def __init__(self, path): self.path = resource_filename('botocore', path) - with open(self.path, "rb") as f: + with io.open(self.path, 'r', encoding='utf-8') as f: spec = json.load(f) self.metadata = spec['metadata'] self.operations = spec['operations'] From ac016a7bb361d84b03201640dce690609c394fef Mon Sep 17 00:00:00 2001 From: Fujimoto Seiji Date: Tue, 24 Apr 2018 11:12:17 +0900 Subject: [PATCH 06/48] Implement describe_log_groups() method for CloudWatchLogs This patch teaches `LogsResponse` class how to handle the DescribeLogGroups request, so that we can mock out the `boto.describe_log_groups()` call. With this change in place, we can write as below: @mock_logs def test_log_group(): conn = boto3.client('logs', 'us-west-2') some_method_to_init_log_groups() resp = conn.describe_log_groups(logGroupNamePrefix='myapp') assert ... This should be fairly useful for a number of programs which handles CloudWatchLogs. Signed-off-by: Fujimoto Seiji --- moto/logs/models.py | 28 ++++++++++++++++++++++++++++ moto/logs/responses.py | 12 ++++++++++++ tests/test_logs/test_logs.py | 4 ++++ 3 files changed, 44 insertions(+) diff --git a/moto/logs/models.py b/moto/logs/models.py index 3ae697a27..3e1c7b955 100644 --- a/moto/logs/models.py +++ b/moto/logs/models.py @@ -132,6 +132,9 @@ class LogGroup: def __init__(self, region, name, tags): self.name = name self.region = region + self.arn = "arn:aws:logs:{region}:1:log-group:{log_group}".format( + region=region, log_group=name) + self.creationTime = unix_time_millis() self.tags = tags self.streams = dict() # {name: LogStream} @@ -197,6 +200,16 @@ class LogGroup: searched_streams = [{"logStreamName": stream.logStreamName, "searchedCompletely": True} for stream in streams] return events_page, next_token, searched_streams + def to_describe_dict(self): + return { + "arn": self.arn, + "creationTime": self.creationTime, + "logGroupName": self.name, + "metricFilterCount": 0, + "retentionInDays": 30, + "storedBytes": sum(s.storedBytes for s in self.streams.values()), + } + class LogsBackend(BaseBackend): def __init__(self, region_name): @@ -223,6 +236,21 @@ class LogsBackend(BaseBackend): raise ResourceNotFoundException() del self.groups[log_group_name] + def describe_log_groups(self, limit, log_group_name_prefix, next_token): + if log_group_name_prefix is None: + log_group_name_prefix = '' + if next_token is None: + next_token = 0 + + groups = sorted(group.to_describe_dict() for name, group in self.groups.items() if name.startswith(log_group_name_prefix)) + groups_page = groups[next_token:next_token + limit] + + next_token += limit + if next_token >= len(groups): + next_token = None + + return groups_page, next_token + def create_log_stream(self, log_group_name, log_stream_name): if log_group_name not in self.groups: raise ResourceNotFoundException() diff --git a/moto/logs/responses.py b/moto/logs/responses.py index 7bf481908..4bec86cb2 100644 --- a/moto/logs/responses.py +++ b/moto/logs/responses.py @@ -33,6 +33,18 @@ class LogsResponse(BaseResponse): self.logs_backend.delete_log_group(log_group_name) return '' + def describe_log_groups(self): + log_group_name_prefix = self._get_param('logGroupNamePrefix') + next_token = self._get_param('nextToken') + limit = self._get_param('limit', 50) + assert limit <= 50 + groups, next_token = self.logs_backend.describe_log_groups( + limit, log_group_name_prefix, next_token) + return json.dumps({ + "logGroups": groups, + "nextToken": next_token + }) + def create_log_stream(self): log_group_name = self._get_param('logGroupName') log_stream_name = self._get_param('logStreamName') diff --git a/tests/test_logs/test_logs.py b/tests/test_logs/test_logs.py index a9a7f5260..3f924cc55 100644 --- a/tests/test_logs/test_logs.py +++ b/tests/test_logs/test_logs.py @@ -13,6 +13,10 @@ def test_log_group_create(): conn = boto3.client('logs', 'us-west-2') log_group_name = 'dummy' response = conn.create_log_group(logGroupName=log_group_name) + + response = conn.describe_log_groups(logGroupNamePrefix=log_group_name) + assert len(response['logGroups']) == 1 + response = conn.delete_log_group(logGroupName=log_group_name) From e1d9c2878f2a5bc2b628e0631b9d9e590c7fec3b Mon Sep 17 00:00:00 2001 From: Brian Pandola Date: Tue, 24 Apr 2018 17:30:17 -0700 Subject: [PATCH 07/48] Add support for Redshift.Waiter.ClusterRestored * Add `restored_from_snapshot` boolean to Cluster metadata. * Return `RestoreStatus` from describe_db_clusters if cluster was restored from a snapshot. Fixes #1506 --- moto/redshift/models.py | 17 +++++++++-- tests/test_redshift/test_redshift.py | 42 ++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/moto/redshift/models.py b/moto/redshift/models.py index 4eb9d6b5c..4eafcfc79 100644 --- a/moto/redshift/models.py +++ b/moto/redshift/models.py @@ -73,7 +73,8 @@ class Cluster(TaggableResourceMixin, BaseModel): preferred_maintenance_window, cluster_parameter_group_name, automated_snapshot_retention_period, port, cluster_version, allow_version_upgrade, number_of_nodes, publicly_accessible, - encrypted, region_name, tags=None, iam_roles_arn=None): + encrypted, region_name, tags=None, iam_roles_arn=None, + restored_from_snapshot=False): super(Cluster, self).__init__(region_name, tags) self.redshift_backend = redshift_backend self.cluster_identifier = cluster_identifier @@ -119,6 +120,7 @@ class Cluster(TaggableResourceMixin, BaseModel): self.number_of_nodes = 1 self.iam_roles_arn = iam_roles_arn or [] + self.restored_from_snapshot = restored_from_snapshot @classmethod def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): @@ -242,7 +244,15 @@ class Cluster(TaggableResourceMixin, BaseModel): "IamRoleArn": iam_role_arn } for iam_role_arn in self.iam_roles_arn] } - + if self.restored_from_snapshot: + json_response['RestoreStatus'] = { + 'Status': 'completed', + 'CurrentRestoreRateInMegaBytesPerSecond': 123.0, + 'SnapshotSizeInMegaBytes': 123, + 'ProgressInMegaBytes': 123, + 'ElapsedTimeInSeconds': 123, + 'EstimatedTimeToCompletionInSeconds': 123 + } try: json_response['ClusterSnapshotCopyStatus'] = self.cluster_snapshot_copy_status except AttributeError: @@ -639,7 +649,8 @@ class RedshiftBackend(BaseBackend): "cluster_version": snapshot.cluster.cluster_version, "number_of_nodes": snapshot.cluster.number_of_nodes, "encrypted": snapshot.cluster.encrypted, - "tags": snapshot.cluster.tags + "tags": snapshot.cluster.tags, + "restored_from_snapshot": True } create_kwargs.update(kwargs) return self.create_cluster(**create_kwargs) diff --git a/tests/test_redshift/test_redshift.py b/tests/test_redshift/test_redshift.py index 96e3ee5b3..6e027b86c 100644 --- a/tests/test_redshift/test_redshift.py +++ b/tests/test_redshift/test_redshift.py @@ -818,6 +818,48 @@ def test_create_cluster_from_snapshot(): new_cluster['Endpoint']['Port'].should.equal(1234) +@mock_redshift +def test_create_cluster_from_snapshot_with_waiter(): + client = boto3.client('redshift', region_name='us-east-1') + original_cluster_identifier = 'original-cluster' + original_snapshot_identifier = 'original-snapshot' + new_cluster_identifier = 'new-cluster' + + client.create_cluster( + ClusterIdentifier=original_cluster_identifier, + ClusterType='single-node', + NodeType='ds2.xlarge', + MasterUsername='username', + MasterUserPassword='password', + ) + client.create_cluster_snapshot( + SnapshotIdentifier=original_snapshot_identifier, + ClusterIdentifier=original_cluster_identifier + ) + response = client.restore_from_cluster_snapshot( + ClusterIdentifier=new_cluster_identifier, + SnapshotIdentifier=original_snapshot_identifier, + Port=1234 + ) + response['Cluster']['ClusterStatus'].should.equal('creating') + + client.get_waiter('cluster_restored').wait( + ClusterIdentifier=new_cluster_identifier, + WaiterConfig={ + 'Delay': 1, + 'MaxAttempts': 2, + } + ) + + response = client.describe_clusters( + ClusterIdentifier=new_cluster_identifier + ) + new_cluster = response['Clusters'][0] + new_cluster['NodeType'].should.equal('ds2.xlarge') + new_cluster['MasterUsername'].should.equal('username') + new_cluster['Endpoint']['Port'].should.equal(1234) + + @mock_redshift def test_create_cluster_from_non_existent_snapshot(): client = boto3.client('redshift', region_name='us-east-1') From 94fa94c2dfe130fd0714a7fda2fffbb86329774b Mon Sep 17 00:00:00 2001 From: Tom Grace Date: Wed, 2 May 2018 13:31:35 +0100 Subject: [PATCH 08/48] 1606 Add additional fields to Batch job status endpoint --- moto/batch/models.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/moto/batch/models.py b/moto/batch/models.py index 8b3b81ccb..c47ca6e97 100644 --- a/moto/batch/models.py +++ b/moto/batch/models.py @@ -295,6 +295,14 @@ class Job(threading.Thread, BaseModel): } if self.job_stopped: result['stoppedAt'] = datetime2int(self.job_stopped_at) + result['container'] = {} + result['container']['command'] = ['/bin/sh -c "for a in `seq 1 10`; do echo Hello World; sleep 1; done"'] + result['container']['privileged'] = False + result['container']['readonlyRootFilesystem'] = False + result['container']['ulimits'] = {} + result['container']['vcpus'] = 1 + result['container']['volumes'] = '' + result['container']['logStreamName'] = self.log_stream_name if self.job_stopped_reason is not None: result['statusReason'] = self.job_stopped_reason return result @@ -378,6 +386,7 @@ class Job(threading.Thread, BaseModel): # Send to cloudwatch log_group = '/aws/batch/job' stream_name = '{0}/default/{1}'.format(self.job_definition.name, self.job_id) + self.log_stream_name = stream_name self._log_backend.ensure_log_group(log_group, None) self._log_backend.create_log_stream(log_group, stream_name) self._log_backend.put_log_events(log_group, stream_name, logs, None) From 7a57dc203422396a4b8ae23cd5bb0955222163a9 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Thu, 3 May 2018 01:40:49 -0700 Subject: [PATCH 09/48] fix errors --- moto/s3/models.py | 3 +++ moto/s3/responses.py | 8 +++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 3b4623d61..9e58fdb47 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -859,6 +859,9 @@ class S3Backend(BaseBackend): if str(key.version_id) != str(version_id) ] ) + + if not bucket.keys.getlist(key_name): + bucket.keys.pop(key_name) return True except KeyError: return False diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 02a9ac40e..f8b3077d3 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -24,7 +24,6 @@ from xml.dom import minidom DEFAULT_REGION_NAME = 'us-east-1' - def parse_key_name(pth): return pth.lstrip("/") @@ -706,8 +705,11 @@ class ResponseObject(_TemplateEnvironmentMixin): if 'x-amz-copy-source' in request.headers: # Copy key - src_key_parsed = urlparse(unquote(request.headers.get("x-amz-copy-source"))) - src_bucket, src_key = src_key_parsed.path.lstrip("/").split("/", 1) + # you can have a quoted ?version=abc with a version Id, so work on + # we need to parse the unquoted string first + src_key_parsed = urlparse(request.headers.get("x-amz-copy-source")) + src_bucket, src_key = unquote(src_key_parsed.path).\ + lstrip("/").split("/", 1) src_version_id = parse_qs(src_key_parsed.query).get( 'versionId', [None])[0] self.backend.copy_key(src_bucket, src_key, bucket_name, key_name, From 93a404ec3726c86fb7453e398a3102afbcb3049a Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Thu, 3 May 2018 02:10:17 -0700 Subject: [PATCH 10/48] pep --- moto/s3/responses.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index f8b3077d3..5e7cf0fe5 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -24,6 +24,7 @@ from xml.dom import minidom DEFAULT_REGION_NAME = 'us-east-1' + def parse_key_name(pth): return pth.lstrip("/") From 07540a35fe23d74cebf0a3e136745dd777a1fb77 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Thu, 3 May 2018 02:30:29 -0700 Subject: [PATCH 11/48] add unittest --- tests/test_s3/test_s3.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 369426758..9f37791cb 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -1405,6 +1405,19 @@ def test_boto3_deleted_versionings_list(): assert len(listed['Contents']) == 1 +@mock_s3 +def test_boto3_delete_versioned_bucket(): + client = boto3.client('s3', region_name='us-east-1') + + client.create_bucket(Bucket='blah') + client.put_bucket_versioning(Bucket='blah', VersioningConfiguration={'Status': 'Enabled'}) + + resp = client.put_object(Bucket='blah', Key='test1', Body=b'test1') + client.delete_object(Bucket='blah', Key='test1', VersionId=resp["VersionId"]) + + client.delete_bucket(Bucket='blah') + + @mock_s3 def test_boto3_head_object_if_modified_since(): s3 = boto3.client('s3', region_name='us-east-1') From 5d6655a7eed858caf90cbff6bcea2e18e13d38d7 Mon Sep 17 00:00:00 2001 From: djkiourtsis Date: Thu, 26 Apr 2018 15:30:33 -0400 Subject: [PATCH 12/48] Add gocloud backend to lambda backends Boto does does not include the govcloud backends when displaying lambda regions. In order to test lambda with a govcloud region, the region must be explicitly added. --- moto/awslambda/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index 80b4ffba3..d49df81c7 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -675,3 +675,4 @@ lambda_backends = {_region.name: LambdaBackend(_region.name) for _region in boto.awslambda.regions()} lambda_backends['ap-southeast-2'] = LambdaBackend('ap-southeast-2') +lambda_backends['us-gov-west-1'] = LambdaBackend('us-gov-west-1') From 2ac8954b13da428bf1fdff63893e2d9714f72c18 Mon Sep 17 00:00:00 2001 From: jbergknoff-10e Date: Thu, 3 May 2018 14:09:56 -0500 Subject: [PATCH 13/48] Accept paths to user-controlled SSL cert --- moto/server.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/moto/server.py b/moto/server.py index e9f4c0904..7862dfe9a 100644 --- a/moto/server.py +++ b/moto/server.py @@ -186,9 +186,18 @@ def main(argv=sys.argv[1:]): parser.add_argument( '-s', '--ssl', action='store_true', - help='Enable SSL encrypted connection (use https://... URL)', + help='Enable SSL encrypted connection with auto-generated certificate (use https://... URL)', default=False ) + parser.add_argument( + '-c', '--ssl-cert', type=str, + help='Path to SSL certificate', + default=None) + parser.add_argument( + '-k', '--ssl-key', type=str, + help='Path to SSL private key', + default=None) + args = parser.parse_args(argv) @@ -197,9 +206,15 @@ def main(argv=sys.argv[1:]): create_backend_app, service=args.service) main_app.debug = True + ssl_context = None + if args.ssl_key and args.ssl_cert: + ssl_context = (args.ssl_cert, args.ssl_key) + elif args.ssl: + ssl_context = 'adhoc' + run_simple(args.host, args.port, main_app, threaded=True, use_reloader=args.reload, - ssl_context='adhoc' if args.ssl else None) + ssl_context=ssl_context) if __name__ == '__main__': From 9e7b86faef70029e1290ad61dada646c2795a222 Mon Sep 17 00:00:00 2001 From: Barry O'Neill Date: Thu, 3 May 2018 19:47:36 -0400 Subject: [PATCH 14/48] Issue 1615 - missing Value should not kill put_metric_data --- moto/cloudwatch/models.py | 2 +- .../test_cloudwatch/test_cloudwatch_boto3.py | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) mode change 100644 => 100755 tests/test_cloudwatch/test_cloudwatch_boto3.py diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py index ba6569981..54fc4fe34 100644 --- a/moto/cloudwatch/models.py +++ b/moto/cloudwatch/models.py @@ -230,7 +230,7 @@ class CloudWatchBackend(BaseBackend): def put_metric_data(self, namespace, metric_data): for metric_member in metric_data: self.metric_data.append(MetricDatum( - namespace, metric_member['MetricName'], float(metric_member['Value']), metric_member.get('Dimensions.member', _EMPTY_LIST), metric_member.get('Timestamp'))) + namespace, metric_member['MetricName'], float(metric_member.get('Value', 0)), metric_member.get('Dimensions.member', _EMPTY_LIST), metric_member.get('Timestamp'))) def get_metric_statistics(self, namespace, metric_name, start_time, end_time, period, stats): period_delta = timedelta(seconds=period) diff --git a/tests/test_cloudwatch/test_cloudwatch_boto3.py b/tests/test_cloudwatch/test_cloudwatch_boto3.py old mode 100644 new mode 100755 index 5fbf75749..41170b37c --- a/tests/test_cloudwatch/test_cloudwatch_boto3.py +++ b/tests/test_cloudwatch/test_cloudwatch_boto3.py @@ -162,6 +162,37 @@ def test_put_metric_data_no_dimensions(): metric['MetricName'].should.equal('metric') + +@mock_cloudwatch +def test_put_metric_data_with_statistics(): + conn = boto3.client('cloudwatch', region_name='us-east-1') + + conn.put_metric_data( + Namespace='tester', + MetricData=[ + dict( + MetricName='statmetric', + Timestamp=datetime(2015, 1, 1), + # no Value to test https://github.com/spulec/moto/issues/1615 + StatisticValues=dict( + SampleCount=123.0, + Sum=123.0, + Minimum=123.0, + Maximum=123.0 + ), + Unit='Milliseconds', + StorageResolution=123 + ) + ] + ) + + metrics = conn.list_metrics()['Metrics'] + metrics.should.have.length_of(1) + metric = metrics[0] + metric['Namespace'].should.equal('tester') + metric['MetricName'].should.equal('statmetric') + # TODO: test statistics - https://github.com/spulec/moto/issues/1615 + @mock_cloudwatch def test_get_metric_statistics(): conn = boto3.client('cloudwatch', region_name='us-east-1') From 86fed8ba27a489d09a6c590dcdd76053c2357dd6 Mon Sep 17 00:00:00 2001 From: jbergknoff-10e Date: Fri, 4 May 2018 16:42:16 -0500 Subject: [PATCH 15/48] lint --- moto/server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/moto/server.py b/moto/server.py index 7862dfe9a..d76b5601c 100644 --- a/moto/server.py +++ b/moto/server.py @@ -198,7 +198,6 @@ def main(argv=sys.argv[1:]): help='Path to SSL private key', default=None) - args = parser.parse_args(argv) # Wrap the main application From b08fc8beded178f3de6580e471faf6be380c42bf Mon Sep 17 00:00:00 2001 From: Ben Jolitz Date: Fri, 4 May 2018 16:30:47 -0700 Subject: [PATCH 16/48] allow topic names to start/end with `_`, `-` --- moto/sns/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/sns/models.py b/moto/sns/models.py index 1c1be6680..6ee6c79ed 100644 --- a/moto/sns/models.py +++ b/moto/sns/models.py @@ -240,7 +240,7 @@ class SNSBackend(BaseBackend): self.sms_attributes.update(attrs) def create_topic(self, name): - fails_constraints = not re.match(r'^[a-zA-Z0-9](?:[A-Za-z0-9_-]{0,253}[a-zA-Z0-9])?$', name) + fails_constraints = not re.match(r'^[a-zA-Z0-9\_\-]{0,256}$', name) if fails_constraints: raise InvalidParameterValue("Topic names must be made up of only uppercase and lowercase ASCII letters, numbers, underscores, and hyphens, and must be between 1 and 256 characters long.") candidate_topic = Topic(name, self) From 0ed18c8b8a23193d6d5989e3869abbf3cb699e6b Mon Sep 17 00:00:00 2001 From: Ben Jolitz Date: Fri, 4 May 2018 16:33:43 -0700 Subject: [PATCH 17/48] remove extraneous backslash --- moto/sns/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/sns/models.py b/moto/sns/models.py index 6ee6c79ed..65dcc6cff 100644 --- a/moto/sns/models.py +++ b/moto/sns/models.py @@ -240,7 +240,7 @@ class SNSBackend(BaseBackend): self.sms_attributes.update(attrs) def create_topic(self, name): - fails_constraints = not re.match(r'^[a-zA-Z0-9\_\-]{0,256}$', name) + fails_constraints = not re.match(r'^[a-zA-Z0-9_-]{0,256}$', name) if fails_constraints: raise InvalidParameterValue("Topic names must be made up of only uppercase and lowercase ASCII letters, numbers, underscores, and hyphens, and must be between 1 and 256 characters long.") candidate_topic = Topic(name, self) From d21c387eb643e3c789469b846ec1c639408f09e0 Mon Sep 17 00:00:00 2001 From: Ben Jolitz Date: Fri, 4 May 2018 18:22:47 -0700 Subject: [PATCH 18/48] Support optional Source, parse from header The Email ``from`` header is either formatted as ``name
`` or ``address``. This commit will use `parseaddr` to extract a ``(name, address)`` tuple, which we will use the ``address`` to check if it's verified. Also support the case where ``Source`` is omitted (which AWS requires the ``from`` header to be set). --- moto/ses/models.py | 23 ++++++++++--- moto/ses/responses.py | 5 ++- tests/test_ses/test_ses_boto3.py | 56 ++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 5 deletions(-) diff --git a/moto/ses/models.py b/moto/ses/models.py index 179f4d8e0..b1135a406 100644 --- a/moto/ses/models.py +++ b/moto/ses/models.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import email +from email.utils import parseaddr from moto.core import BaseBackend, BaseModel from .exceptions import MessageRejectedError @@ -84,13 +85,27 @@ class SESBackend(BaseBackend): return message def send_raw_email(self, source, destinations, raw_data): - if source not in self.addresses: - raise MessageRejectedError( - "Did not have authority to send from email %s" % source - ) + if source is not None: + _, source_email_address = parseaddr(source) + if source_email_address not in self.addresses: + raise MessageRejectedError( + "Did not have authority to send from email %s" % source_email_address + ) recipient_count = len(destinations) message = email.message_from_string(raw_data) + if source is None: + if message['from'] is None: + raise MessageRejectedError( + "Source not specified" + ) + + _, source_email_address = parseaddr(message['from']) + if source_email_address not in self.addresses: + raise MessageRejectedError( + "Did not have authority to send from email %s" % source_email_address + ) + for header in 'TO', 'CC', 'BCC': recipient_count += sum( d.strip() and 1 or 0 diff --git a/moto/ses/responses.py b/moto/ses/responses.py index 6cd018aa6..bdf873836 100644 --- a/moto/ses/responses.py +++ b/moto/ses/responses.py @@ -75,7 +75,10 @@ class EmailResponse(BaseResponse): return template.render(message=message) def send_raw_email(self): - source = self.querystring.get('Source')[0] + source = self.querystring.get('Source') + if source is not None: + source, = source + raw_data = self.querystring.get('RawMessage.Data')[0] raw_data = base64.b64decode(raw_data) if six.PY3: diff --git a/tests/test_ses/test_ses_boto3.py b/tests/test_ses/test_ses_boto3.py index 5d39f61d4..e800b8035 100644 --- a/tests/test_ses/test_ses_boto3.py +++ b/tests/test_ses/test_ses_boto3.py @@ -136,3 +136,59 @@ def test_send_raw_email(): send_quota = conn.get_send_quota() sent_count = int(send_quota['SentLast24Hours']) sent_count.should.equal(2) + + +@mock_ses +def test_send_raw_email_without_source(): + conn = boto3.client('ses', region_name='us-east-1') + + message = MIMEMultipart() + message['Subject'] = 'Test' + message['From'] = 'test@example.com' + message['To'] = 'to@example.com, foo@example.com' + + # Message body + part = MIMEText('test file attached') + message.attach(part) + + # Attachment + part = MIMEText('contents of test file here') + part.add_header('Content-Disposition', 'attachment; filename=test.txt') + message.attach(part) + + kwargs = dict( + RawMessage={'Data': message.as_string()}, + ) + + conn.send_raw_email.when.called_with(**kwargs).should.throw(ClientError) + + conn.verify_email_identity(EmailAddress="test@example.com") + conn.send_raw_email(**kwargs) + + send_quota = conn.get_send_quota() + sent_count = int(send_quota['SentLast24Hours']) + sent_count.should.equal(2) + + +@mock_ses +def test_send_raw_email_without_source_or_from(): + conn = boto3.client('ses', region_name='us-east-1') + + message = MIMEMultipart() + message['Subject'] = 'Test' + message['To'] = 'to@example.com, foo@example.com' + + # Message body + part = MIMEText('test file attached') + message.attach(part) + # Attachment + part = MIMEText('contents of test file here') + part.add_header('Content-Disposition', 'attachment; filename=test.txt') + message.attach(part) + + kwargs = dict( + RawMessage={'Data': message.as_string()}, + ) + + conn.send_raw_email.when.called_with(**kwargs).should.throw(ClientError) + From 289d3614633f54d1a4a34bdfb85baf6e8c0bb1fd Mon Sep 17 00:00:00 2001 From: Ben Jolitz Date: Fri, 4 May 2018 19:16:12 -0700 Subject: [PATCH 19/48] add check for at least 1 character as the minumum length REF: http://boto3.readthedocs.io/en/latest/reference/services/sns.html#SNS.Client.create_topic --- moto/sns/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/sns/models.py b/moto/sns/models.py index 65dcc6cff..d6105c1d0 100644 --- a/moto/sns/models.py +++ b/moto/sns/models.py @@ -240,7 +240,7 @@ class SNSBackend(BaseBackend): self.sms_attributes.update(attrs) def create_topic(self, name): - fails_constraints = not re.match(r'^[a-zA-Z0-9_-]{0,256}$', name) + fails_constraints = not re.match(r'^[a-zA-Z0-9_-]{1,256}$', name) if fails_constraints: raise InvalidParameterValue("Topic names must be made up of only uppercase and lowercase ASCII letters, numbers, underscores, and hyphens, and must be between 1 and 256 characters long.") candidate_topic = Topic(name, self) From 45b529fef4c15e26e7850fc0777c7b8debc78f2d Mon Sep 17 00:00:00 2001 From: Ben Jolitz Date: Fri, 4 May 2018 19:17:56 -0700 Subject: [PATCH 20/48] parameterize topic name create/delete --- tests/test_sns/test_topics_boto3.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/tests/test_sns/test_topics_boto3.py b/tests/test_sns/test_topics_boto3.py index 95dd41f89..7d9a27b18 100644 --- a/tests/test_sns/test_topics_boto3.py +++ b/tests/test_sns/test_topics_boto3.py @@ -13,23 +13,24 @@ from moto.sns.models import DEFAULT_TOPIC_POLICY, DEFAULT_EFFECTIVE_DELIVERY_POL @mock_sns def test_create_and_delete_topic(): conn = boto3.client("sns", region_name="us-east-1") - conn.create_topic(Name="some-topic") + for topic_name in ('some-topic', '-some-topic-', '_some-topic_', 'a' * 256): + conn.create_topic(Name=topic_name) - topics_json = conn.list_topics() - topics = topics_json["Topics"] - topics.should.have.length_of(1) - topics[0]['TopicArn'].should.equal( - "arn:aws:sns:{0}:123456789012:some-topic" - .format(conn._client_config.region_name) - ) + topics_json = conn.list_topics() + topics = topics_json["Topics"] + topics.should.have.length_of(1) + topics[0]['TopicArn'].should.equal( + "arn:aws:sns:{0}:123456789012:{1}" + .format(conn._client_config.region_name, topic_name) + ) - # Delete the topic - conn.delete_topic(TopicArn=topics[0]['TopicArn']) + # Delete the topic + conn.delete_topic(TopicArn=topics[0]['TopicArn']) - # And there should now be 0 topics - topics_json = conn.list_topics() - topics = topics_json["Topics"] - topics.should.have.length_of(0) + # And there should now be 0 topics + topics_json = conn.list_topics() + topics = topics_json["Topics"] + topics.should.have.length_of(0) @mock_sns def test_create_topic_should_be_indempodent(): From 1a0a951b06c1da77bf61e498d501b488b328b506 Mon Sep 17 00:00:00 2001 From: bclodius Date: Sat, 5 May 2018 15:22:29 -0400 Subject: [PATCH 21/48] Fixes #1608 --- moto/cloudwatch/models.py | 7 ++++++- moto/cloudwatch/responses.py | 2 +- tests/test_cloudwatch/test_cloudwatch_boto3.py | 3 ++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py index ba6569981..441c74176 100644 --- a/moto/cloudwatch/models.py +++ b/moto/cloudwatch/models.py @@ -229,8 +229,13 @@ class CloudWatchBackend(BaseBackend): def put_metric_data(self, namespace, metric_data): for metric_member in metric_data: + # Preserve "datetime" for get_metric_statistics comparisons + timestamp = metric_member.get('Timestamp') + if timestamp is not None and type(timestamp) != datetime: + timestamp = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%fZ') + timestamp = timestamp.replace(tzinfo=tzutc()) self.metric_data.append(MetricDatum( - namespace, metric_member['MetricName'], float(metric_member['Value']), metric_member.get('Dimensions.member', _EMPTY_LIST), metric_member.get('Timestamp'))) + namespace, metric_member['MetricName'], float(metric_member['Value']), metric_member.get('Dimensions.member', _EMPTY_LIST), timestamp)) def get_metric_statistics(self, namespace, metric_name, start_time, end_time, period, stats): period_delta = timedelta(seconds=period) diff --git a/moto/cloudwatch/responses.py b/moto/cloudwatch/responses.py index c080d4620..8118f35ba 100644 --- a/moto/cloudwatch/responses.py +++ b/moto/cloudwatch/responses.py @@ -272,7 +272,7 @@ GET_METRIC_STATISTICS_TEMPLATE = """/.well-known/jwks.json$': CognitoIdpJsonWebKeyResponse().serve_json_web_key, +} diff --git a/moto/server.py b/moto/server.py index e9f4c0904..d6b0ee083 100644 --- a/moto/server.py +++ b/moto/server.py @@ -69,8 +69,13 @@ class DomainDispatcherApplication(object): _, _, region, service, _ = environ['HTTP_AUTHORIZATION'].split(",")[0].split()[ 1].split("/") except (KeyError, ValueError): + # Some cognito-idp endpoints (e.g. change password) do not receive an auth header. + if environ.get('HTTP_X_AMZ_TARGET', '').startswith('AWSCognitoIdentityProviderService'): + service = 'cognito-idp' + else: + service = 's3' + region = 'us-east-1' - service = 's3' if service == 'dynamodb': dynamo_api_version = environ['HTTP_X_AMZ_TARGET'].split("_")[1].split(".")[0] # If Newer API version, use dynamodb2 diff --git a/setup.py b/setup.py index ebbf6f0cd..8582e6b85 100755 --- a/setup.py +++ b/setup.py @@ -19,6 +19,7 @@ install_requires = [ "pyaml", "pytz", "python-dateutil<3.0.0,>=2.1", + "python-jose<3.0.0", "mock", "docker>=2.5.1", "jsondiff==1.1.1", diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py new file mode 100644 index 000000000..b2bd469ce --- /dev/null +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -0,0 +1,539 @@ +from __future__ import unicode_literals + +import boto3 +import json +import os +import uuid + +from jose import jws +from moto import mock_cognitoidp +import sure # noqa + + +@mock_cognitoidp +def test_create_user_pool(): + conn = boto3.client("cognito-idp", "us-west-2") + + name = str(uuid.uuid4()) + value = str(uuid.uuid4()) + result = conn.create_user_pool( + PoolName=name, + LambdaConfig={ + "PreSignUp": value + } + ) + + result["UserPool"]["Id"].should_not.be.none + result["UserPool"]["Name"].should.equal(name) + result["UserPool"]["LambdaConfig"]["PreSignUp"].should.equal(value) + + +@mock_cognitoidp +def test_list_user_pools(): + conn = boto3.client("cognito-idp", "us-west-2") + + name = str(uuid.uuid4()) + conn.create_user_pool(PoolName=name) + result = conn.list_user_pools(MaxResults=10) + result["UserPools"].should.have.length_of(1) + result["UserPools"][0]["Name"].should.equal(name) + + +@mock_cognitoidp +def test_describe_user_pool(): + conn = boto3.client("cognito-idp", "us-west-2") + + name = str(uuid.uuid4()) + value = str(uuid.uuid4()) + user_pool_details = conn.create_user_pool( + PoolName=name, + LambdaConfig={ + "PreSignUp": value + } + ) + + result = conn.describe_user_pool(UserPoolId=user_pool_details["UserPool"]["Id"]) + result["UserPool"]["Name"].should.equal(name) + result["UserPool"]["LambdaConfig"]["PreSignUp"].should.equal(value) + + +@mock_cognitoidp +def test_delete_user_pool(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + conn.list_user_pools(MaxResults=10)["UserPools"].should.have.length_of(1) + conn.delete_user_pool(UserPoolId=user_pool_id) + conn.list_user_pools(MaxResults=10)["UserPools"].should.have.length_of(0) + + +@mock_cognitoidp +def test_create_user_pool_domain(): + conn = boto3.client("cognito-idp", "us-west-2") + + domain = str(uuid.uuid4()) + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + result = conn.create_user_pool_domain(UserPoolId=user_pool_id, Domain=domain) + result["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + +@mock_cognitoidp +def test_describe_user_pool_domain(): + conn = boto3.client("cognito-idp", "us-west-2") + + domain = str(uuid.uuid4()) + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + conn.create_user_pool_domain(UserPoolId=user_pool_id, Domain=domain) + result = conn.describe_user_pool_domain(Domain=domain) + result["DomainDescription"]["Domain"].should.equal(domain) + result["DomainDescription"]["UserPoolId"].should.equal(user_pool_id) + result["DomainDescription"]["AWSAccountId"].should_not.be.none + + +@mock_cognitoidp +def test_delete_user_pool_domain(): + conn = boto3.client("cognito-idp", "us-west-2") + + domain = str(uuid.uuid4()) + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + conn.create_user_pool_domain(UserPoolId=user_pool_id, Domain=domain) + result = conn.delete_user_pool_domain(UserPoolId=user_pool_id, Domain=domain) + result["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + result = conn.describe_user_pool_domain(Domain=domain) + # This is a surprising behavior of the real service: describing a missing domain comes + # back with status 200 and a DomainDescription of {} + result["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + result["DomainDescription"].keys().should.have.length_of(0) + + +@mock_cognitoidp +def test_create_user_pool_client(): + conn = boto3.client("cognito-idp", "us-west-2") + + client_name = str(uuid.uuid4()) + value = str(uuid.uuid4()) + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + result = conn.create_user_pool_client( + UserPoolId=user_pool_id, + ClientName=client_name, + CallbackURLs=[value], + ) + + result["UserPoolClient"]["UserPoolId"].should.equal(user_pool_id) + result["UserPoolClient"]["ClientId"].should_not.be.none + result["UserPoolClient"]["ClientName"].should.equal(client_name) + result["UserPoolClient"]["CallbackURLs"].should.have.length_of(1) + result["UserPoolClient"]["CallbackURLs"][0].should.equal(value) + + +@mock_cognitoidp +def test_list_user_pool_clients(): + conn = boto3.client("cognito-idp", "us-west-2") + + client_name = str(uuid.uuid4()) + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + conn.create_user_pool_client(UserPoolId=user_pool_id, ClientName=client_name) + result = conn.list_user_pool_clients(UserPoolId=user_pool_id, MaxResults=10) + result["UserPoolClients"].should.have.length_of(1) + result["UserPoolClients"][0]["ClientName"].should.equal(client_name) + + +@mock_cognitoidp +def test_describe_user_pool_client(): + conn = boto3.client("cognito-idp", "us-west-2") + + client_name = str(uuid.uuid4()) + value = str(uuid.uuid4()) + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + client_details = conn.create_user_pool_client( + UserPoolId=user_pool_id, + ClientName=client_name, + CallbackURLs=[value], + ) + + result = conn.describe_user_pool_client( + UserPoolId=user_pool_id, + ClientId=client_details["UserPoolClient"]["ClientId"], + ) + + result["UserPoolClient"]["ClientName"].should.equal(client_name) + result["UserPoolClient"]["CallbackURLs"].should.have.length_of(1) + result["UserPoolClient"]["CallbackURLs"][0].should.equal(value) + + +@mock_cognitoidp +def test_update_user_pool_client(): + conn = boto3.client("cognito-idp", "us-west-2") + + old_client_name = str(uuid.uuid4()) + new_client_name = str(uuid.uuid4()) + old_value = str(uuid.uuid4()) + new_value = str(uuid.uuid4()) + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + client_details = conn.create_user_pool_client( + UserPoolId=user_pool_id, + ClientName=old_client_name, + CallbackURLs=[old_value], + ) + + result = conn.update_user_pool_client( + UserPoolId=user_pool_id, + ClientId=client_details["UserPoolClient"]["ClientId"], + ClientName=new_client_name, + CallbackURLs=[new_value], + ) + + result["UserPoolClient"]["ClientName"].should.equal(new_client_name) + result["UserPoolClient"]["CallbackURLs"].should.have.length_of(1) + result["UserPoolClient"]["CallbackURLs"][0].should.equal(new_value) + + +@mock_cognitoidp +def test_delete_user_pool_client(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + client_details = conn.create_user_pool_client( + UserPoolId=user_pool_id, + ClientName=str(uuid.uuid4()), + ) + + conn.delete_user_pool_client( + UserPoolId=user_pool_id, + ClientId=client_details["UserPoolClient"]["ClientId"], + ) + + caught = False + try: + conn.describe_user_pool_client( + UserPoolId=user_pool_id, + ClientId=client_details["UserPoolClient"]["ClientId"], + ) + except conn.exceptions.ResourceNotFoundException: + caught = True + + caught.should.be.true + + +@mock_cognitoidp +def test_create_identity_provider(): + conn = boto3.client("cognito-idp", "us-west-2") + + provider_name = str(uuid.uuid4()) + provider_type = "Facebook" + value = str(uuid.uuid4()) + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + result = conn.create_identity_provider( + UserPoolId=user_pool_id, + ProviderName=provider_name, + ProviderType=provider_type, + ProviderDetails={ + "thing": value + }, + ) + + result["IdentityProvider"]["UserPoolId"].should.equal(user_pool_id) + result["IdentityProvider"]["ProviderName"].should.equal(provider_name) + result["IdentityProvider"]["ProviderType"].should.equal(provider_type) + result["IdentityProvider"]["ProviderDetails"]["thing"].should.equal(value) + + +@mock_cognitoidp +def test_list_identity_providers(): + conn = boto3.client("cognito-idp", "us-west-2") + + provider_name = str(uuid.uuid4()) + provider_type = "Facebook" + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + conn.create_identity_provider( + UserPoolId=user_pool_id, + ProviderName=provider_name, + ProviderType=provider_type, + ProviderDetails={}, + ) + + result = conn.list_identity_providers( + UserPoolId=user_pool_id, + MaxResults=10, + ) + + result["Providers"].should.have.length_of(1) + result["Providers"][0]["ProviderName"].should.equal(provider_name) + result["Providers"][0]["ProviderType"].should.equal(provider_type) + + +@mock_cognitoidp +def test_describe_identity_providers(): + conn = boto3.client("cognito-idp", "us-west-2") + + provider_name = str(uuid.uuid4()) + provider_type = "Facebook" + value = str(uuid.uuid4()) + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + conn.create_identity_provider( + UserPoolId=user_pool_id, + ProviderName=provider_name, + ProviderType=provider_type, + ProviderDetails={ + "thing": value + }, + ) + + result = conn.describe_identity_provider( + UserPoolId=user_pool_id, + ProviderName=provider_name, + ) + + result["IdentityProvider"]["UserPoolId"].should.equal(user_pool_id) + result["IdentityProvider"]["ProviderName"].should.equal(provider_name) + result["IdentityProvider"]["ProviderType"].should.equal(provider_type) + result["IdentityProvider"]["ProviderDetails"]["thing"].should.equal(value) + + +@mock_cognitoidp +def test_delete_identity_providers(): + conn = boto3.client("cognito-idp", "us-west-2") + + provider_name = str(uuid.uuid4()) + provider_type = "Facebook" + value = str(uuid.uuid4()) + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + conn.create_identity_provider( + UserPoolId=user_pool_id, + ProviderName=provider_name, + ProviderType=provider_type, + ProviderDetails={ + "thing": value + }, + ) + + conn.delete_identity_provider(UserPoolId=user_pool_id, ProviderName=provider_name) + + caught = False + try: + conn.describe_identity_provider( + UserPoolId=user_pool_id, + ProviderName=provider_name, + ) + except conn.exceptions.ResourceNotFoundException: + caught = True + + caught.should.be.true + + +@mock_cognitoidp +def test_admin_create_user(): + conn = boto3.client("cognito-idp", "us-west-2") + + username = str(uuid.uuid4()) + value = str(uuid.uuid4()) + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + result = conn.admin_create_user( + UserPoolId=user_pool_id, + Username=username, + UserAttributes=[ + {"Name": "thing", "Value": value} + ], + ) + + result["User"]["Username"].should.equal(username) + result["User"]["UserStatus"].should.equal("FORCE_CHANGE_PASSWORD") + result["User"]["Attributes"].should.have.length_of(1) + result["User"]["Attributes"][0]["Name"].should.equal("thing") + result["User"]["Attributes"][0]["Value"].should.equal(value) + + +@mock_cognitoidp +def test_admin_get_user(): + conn = boto3.client("cognito-idp", "us-west-2") + + username = str(uuid.uuid4()) + value = str(uuid.uuid4()) + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + conn.admin_create_user( + UserPoolId=user_pool_id, + Username=username, + UserAttributes=[ + {"Name": "thing", "Value": value} + ], + ) + + result = conn.admin_get_user(UserPoolId=user_pool_id, Username=username) + result["Username"].should.equal(username) + result["UserAttributes"].should.have.length_of(1) + result["UserAttributes"][0]["Name"].should.equal("thing") + result["UserAttributes"][0]["Value"].should.equal(value) + + +@mock_cognitoidp +def test_list_users(): + conn = boto3.client("cognito-idp", "us-west-2") + + username = str(uuid.uuid4()) + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + conn.admin_create_user(UserPoolId=user_pool_id, Username=username) + result = conn.list_users(UserPoolId=user_pool_id) + result["Users"].should.have.length_of(1) + result["Users"][0]["Username"].should.equal(username) + + +@mock_cognitoidp +def test_admin_delete_user(): + conn = boto3.client("cognito-idp", "us-west-2") + + username = str(uuid.uuid4()) + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + conn.admin_create_user(UserPoolId=user_pool_id, Username=username) + conn.admin_delete_user(UserPoolId=user_pool_id, Username=username) + + caught = False + try: + conn.admin_get_user(UserPoolId=user_pool_id, Username=username) + except conn.exceptions.ResourceNotFoundException: + caught = True + + caught.should.be.true + + +def authentication_flow(conn): + username = str(uuid.uuid4()) + temporary_password = str(uuid.uuid4()) + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + client_id = conn.create_user_pool_client( + UserPoolId=user_pool_id, + ClientName=str(uuid.uuid4()), + )["UserPoolClient"]["ClientId"] + + conn.admin_create_user( + UserPoolId=user_pool_id, + Username=username, + TemporaryPassword=temporary_password, + ) + + result = conn.admin_initiate_auth( + UserPoolId=user_pool_id, + ClientId=client_id, + AuthFlow="ADMIN_NO_SRP_AUTH", + AuthParameters={ + "USERNAME": username, + "PASSWORD": temporary_password + }, + ) + + # A newly created user is forced to set a new password + result["ChallengeName"].should.equal("NEW_PASSWORD_REQUIRED") + result["Session"].should_not.be.none + + # This sets a new password and logs the user in (creates tokens) + new_password = str(uuid.uuid4()) + result = conn.respond_to_auth_challenge( + Session=result["Session"], + ClientId=client_id, + ChallengeName="NEW_PASSWORD_REQUIRED", + ChallengeResponses={ + "USERNAME": username, + "NEW_PASSWORD": new_password + } + ) + + result["AuthenticationResult"]["IdToken"].should_not.be.none + result["AuthenticationResult"]["AccessToken"].should_not.be.none + + return { + "user_pool_id": user_pool_id, + "client_id": client_id, + "id_token": result["AuthenticationResult"]["IdToken"], + "access_token": result["AuthenticationResult"]["AccessToken"], + "username": username, + "password": new_password, + } + + +@mock_cognitoidp +def test_authentication_flow(): + conn = boto3.client("cognito-idp", "us-west-2") + + authentication_flow(conn) + + +@mock_cognitoidp +def test_token_legitimacy(): + conn = boto3.client("cognito-idp", "us-west-2") + + path = "../../moto/cognitoidp/resources/jwks-public.json" + with open(os.path.join(os.path.dirname(__file__), path)) as f: + json_web_key = json.loads(f.read())["keys"][0] + + outputs = authentication_flow(conn) + id_token = outputs["id_token"] + access_token = outputs["access_token"] + client_id = outputs["client_id"] + issuer = "https://cognito-idp.us-west-2.amazonaws.com/{}".format(outputs["user_pool_id"]) + id_claims = json.loads(jws.verify(id_token, json_web_key, "RS256")) + id_claims["iss"].should.equal(issuer) + id_claims["aud"].should.equal(client_id) + access_claims = json.loads(jws.verify(access_token, json_web_key, "RS256")) + access_claims["iss"].should.equal(issuer) + access_claims["aud"].should.equal(client_id) + + +@mock_cognitoidp +def test_change_password(): + conn = boto3.client("cognito-idp", "us-west-2") + + outputs = authentication_flow(conn) + + # Take this opportunity to test change_password, which requires an access token. + newer_password = str(uuid.uuid4()) + conn.change_password( + AccessToken=outputs["access_token"], + PreviousPassword=outputs["password"], + ProposedPassword=newer_password, + ) + + # Log in again, which should succeed without a challenge because the user is no + # longer in the force-new-password state. + result = conn.admin_initiate_auth( + UserPoolId=outputs["user_pool_id"], + ClientId=outputs["client_id"], + AuthFlow="ADMIN_NO_SRP_AUTH", + AuthParameters={ + "USERNAME": outputs["username"], + "PASSWORD": newer_password, + }, + ) + + result["AuthenticationResult"].should_not.be.none + + +@mock_cognitoidp +def test_forgot_password(): + conn = boto3.client("cognito-idp", "us-west-2") + + result = conn.forgot_password(ClientId=str(uuid.uuid4()), Username=str(uuid.uuid4())) + result["CodeDeliveryDetails"].should_not.be.none + + +@mock_cognitoidp +def test_confirm_forgot_password(): + conn = boto3.client("cognito-idp", "us-west-2") + + username = str(uuid.uuid4()) + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + client_id = conn.create_user_pool_client( + UserPoolId=user_pool_id, + ClientName=str(uuid.uuid4()), + )["UserPoolClient"]["ClientId"] + + conn.admin_create_user( + UserPoolId=user_pool_id, + Username=username, + TemporaryPassword=str(uuid.uuid4()), + ) + + conn.confirm_forgot_password( + ClientId=client_id, + Username=username, + ConfirmationCode=str(uuid.uuid4()), + Password=str(uuid.uuid4()), + ) From 12188733b73fe5511692f05f0a1ed586a2fa7777 Mon Sep 17 00:00:00 2001 From: zane Date: Thu, 10 May 2018 23:39:19 -0700 Subject: [PATCH 23/48] adding Address reallocate capability for EIP --- moto/ec2/models.py | 15 ++++++++++----- moto/ec2/responses/elastic_ip_addresses.py | 7 ++++++- tests/test_ec2/test_elastic_ip_addresses.py | 11 +++++++++++ 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 31bfb4839..674e0bddb 100755 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -3159,8 +3159,11 @@ class SpotFleetBackend(object): class ElasticAddress(object): - def __init__(self, domain): - self.public_ip = random_ip() + def __init__(self, domain, address=None): + if address: + self.public_ip = address + else: + self.public_ip = random_ip() self.allocation_id = random_eip_allocation_id() if domain == "vpc" else None self.domain = domain self.instance = None @@ -3222,11 +3225,13 @@ class ElasticAddressBackend(object): self.addresses = [] super(ElasticAddressBackend, self).__init__() - def allocate_address(self, domain): + def allocate_address(self, domain, address=None): if domain not in ['standard', 'vpc']: raise InvalidDomainError(domain) - - address = ElasticAddress(domain) + if address: + address = ElasticAddress(domain, address) + else: + address = ElasticAddress(domain) self.addresses.append(address) return address diff --git a/moto/ec2/responses/elastic_ip_addresses.py b/moto/ec2/responses/elastic_ip_addresses.py index 11c1d9c1f..6e1c9fe38 100644 --- a/moto/ec2/responses/elastic_ip_addresses.py +++ b/moto/ec2/responses/elastic_ip_addresses.py @@ -7,8 +7,13 @@ class ElasticIPAddresses(BaseResponse): def allocate_address(self): domain = self._get_param('Domain', if_none='standard') + reallocate_address = self._get_param('Address', if_none=None) if self.is_not_dryrun('AllocateAddress'): - address = self.ec2_backend.allocate_address(domain) + if reallocate_address: + address = self.ec2_backend.allocate_address( + domain, address=reallocate_address) + else: + address = self.ec2_backend.allocate_address(domain) template = self.response_template(ALLOCATE_ADDRESS_RESPONSE) return template.render(address=address) diff --git a/tests/test_ec2/test_elastic_ip_addresses.py b/tests/test_ec2/test_elastic_ip_addresses.py index 709bdc33b..ca6637b18 100644 --- a/tests/test_ec2/test_elastic_ip_addresses.py +++ b/tests/test_ec2/test_elastic_ip_addresses.py @@ -62,6 +62,17 @@ def test_eip_allocate_vpc(): logging.debug("vpc alloc_id:".format(vpc.allocation_id)) vpc.release() +@mock_ec2 +def test_specific_eip_allocate_vpc(): + """Allocate VPC EIP with specific address""" + service = boto3.resource('ec2', region_name='us-west-1') + client = boto3.client('ec2', region_name='us-west-1') + + vpc = client.allocate_address(Domain="vpc", Address="127.38.43.222") + vpc['Domain'].should.be.equal("vpc") + vpc['PublicIp'].should.be.equal("127.38.43.222") + logging.debug("vpc alloc_id:".format(vpc['AllocationId'])) + @mock_ec2_deprecated def test_eip_allocate_invalid_domain(): From 2e75d0219c9b9f719d8aa517627eedd2036cb301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mau=20Zs=C3=B3fia=20=C3=81brah=C3=A1m?= Date: Fri, 4 May 2018 20:33:56 +0200 Subject: [PATCH 24/48] Support dynamodb2 nested map creation to mirror actual db --- moto/dynamodb2/models.py | 9 ++++---- tests/test_dynamodb2/test_dynamodb.py | 31 ++++++++++++++++++++++++--- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 73b09d73c..c4aa10237 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -176,16 +176,17 @@ class Item(BaseModel): key_parts = key.split('.') attr = key_parts.pop(0) if attr not in self.attrs: - raise ValueError() + raise ValueError last_val = self.attrs[attr].value for key_part in key_parts: # Hack but it'll do, traverses into a dict - if list(last_val.keys())[0] == 'M': - last_val = last_val['M'] + last_val_type = list(last_val.keys()) + if last_val_type and last_val_type[0] == 'M': + last_val = last_val['M'] if key_part not in last_val: - raise ValueError() + last_val[key_part] = {'M': {}} last_val = last_val[key_part] diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 20ff80167..2c76d339f 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -6,6 +6,7 @@ import boto3 from boto3.dynamodb.conditions import Attr import sure # noqa import requests +from pytest import raises from moto import mock_dynamodb2, mock_dynamodb2_deprecated from moto.dynamodb2 import dynamodb_backend2 from boto.exception import JSONResponseError @@ -1052,6 +1053,7 @@ def test_query_missing_expr_names(): @mock_dynamodb2 def test_update_item_on_map(): dynamodb = boto3.resource('dynamodb', region_name='us-east-1') + client = boto3.client('dynamodb') # Create the DynamoDB table. dynamodb.create_table( @@ -1092,21 +1094,44 @@ def test_update_item_on_map(): resp = table.scan() resp['Items'][0]['body'].should.equal({'nested': {'data': 'test'}}) + # Nonexistent nested attributes are supported for existing top-level attributes. table.update_item(Key={ 'forum_name': 'the-key', 'subject': '123' }, - UpdateExpression='SET body.#nested.#data = :tb', + UpdateExpression='SET body.#nested.#data = :tb, body.nested.#nonexistentnested.#data = :tb2', ExpressionAttributeNames={ '#nested': 'nested', + '#nonexistentnested': 'nonexistentnested', '#data': 'data' }, ExpressionAttributeValues={ - ':tb': 'new_value' + ':tb': 'new_value', + ':tb2': 'other_value' }) resp = table.scan() - resp['Items'][0]['body'].should.equal({'nested': {'data': 'new_value'}}) + resp['Items'][0]['body'].should.equal({ + 'nested': { + 'data': 'new_value', + 'nonexistentnested': {'data': 'other_value'} + } + }) + + # Test nested value for a nonexistent attribute. + with raises(client.exceptions.ConditionalCheckFailedException): + table.update_item(Key={ + 'forum_name': 'the-key', + 'subject': '123' + }, + UpdateExpression='SET nonexistent.#nested = :tb', + ExpressionAttributeNames={ + '#nested': 'nested' + }, + ExpressionAttributeValues={ + ':tb': 'new_value' + }) + # https://github.com/spulec/moto/issues/1358 From 64fc0d3556c3efcf60b1d4282dbaa69b956ab0b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mau=20Zs=C3=B3fia=20=C3=81brah=C3=A1m?= Date: Mon, 14 May 2018 14:16:39 +0200 Subject: [PATCH 25/48] add region for test --- tests/test_dynamodb2/test_dynamodb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 2c76d339f..93188001f 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1053,7 +1053,7 @@ def test_query_missing_expr_names(): @mock_dynamodb2 def test_update_item_on_map(): dynamodb = boto3.resource('dynamodb', region_name='us-east-1') - client = boto3.client('dynamodb') + client = boto3.client('dynamodb', region_name='us-east-1') # Create the DynamoDB table. dynamodb.create_table( From 1e01356f99141c354b16c1a816c9776df1aabe8b Mon Sep 17 00:00:00 2001 From: cpitchford Date: Tue, 15 May 2018 16:45:49 +0100 Subject: [PATCH 26/48] Bugfix: describe_event_bus()['Policy'] should be JSON string, not object boto3.client('events')['Policy'] must be a string as per: http://boto3.readthedocs.io/en/latest/reference/services/events.html#CloudWatchEvents.Client.describe_event_bus Adding json.dumps() around the policy value ensures we do not return an unexpected dict This change corrects an error when attempting to decode the policy: json.load(boto3.client('events').describe_event_bus()['Policy']) --- moto/events/models.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/moto/events/models.py b/moto/events/models.py index 5c1d507ca..0885d8d4b 100644 --- a/moto/events/models.py +++ b/moto/events/models.py @@ -1,5 +1,6 @@ import os import re +import json from moto.core.exceptions import JsonRESTError from moto.core import BaseBackend, BaseModel @@ -238,8 +239,10 @@ class EventsBackend(BaseBackend): 'Action': 'events:{0}'.format(data['action']), 'Resource': arn }) + policy = {'Version': '2012-10-17', 'Statement': statements} + policy_json = json.dumps(policy) return { - 'Policy': {'Version': '2012-10-17', 'Statement': statements}, + 'Policy': policy_json, 'Name': 'default', 'Arn': arn } From e85106c708d593a8048b6a30014ef0744b454942 Mon Sep 17 00:00:00 2001 From: cpitchford Date: Tue, 15 May 2018 17:04:59 +0100 Subject: [PATCH 27/48] describe_event_bus returns json, not dict Correct the assumption that describe_event_bus()['Policy'] is a dict As per http://boto3.readthedocs.io/en/latest/reference/services/events.html#CloudWatchEvents.Client.describe_event_bus It should be a JSON encoded string Here we decode the JSON before we look inside the policy --- tests/test_events/test_events.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_events/test_events.py b/tests/test_events/test_events.py index e839bde5b..835b3b283 100644 --- a/tests/test_events/test_events.py +++ b/tests/test_events/test_events.py @@ -1,6 +1,7 @@ import random import boto3 +import json from moto.events import mock_events from botocore.exceptions import ClientError @@ -181,13 +182,15 @@ def test_permissions(): client.put_permission(Action='PutEvents', Principal='222222222222', StatementId='Account2') resp = client.describe_event_bus() - assert len(resp['Policy']['Statement']) == 2 + resp_policy = json.loads(resp['Policy']) + assert len(resp_policy['Statement']) == 2 client.remove_permission(StatementId='Account2') resp = client.describe_event_bus() - assert len(resp['Policy']['Statement']) == 1 - assert resp['Policy']['Statement'][0]['Sid'] == 'Account1' + resp_policy = json.loads(resp['Policy']) + assert len(resp_policy['Statement']) == 1 + assert resp_policy['Statement'][0]['Sid'] == 'Account1' @mock_events From b61989cb35949f724518160b1b10c21765466a4b Mon Sep 17 00:00:00 2001 From: cpitchford Date: Tue, 15 May 2018 18:28:35 +0100 Subject: [PATCH 28/48] Bugfix: put_permission action parameter Boto3/AWS requires that the Action parameter of put_permissions is fully qualified as "events:PutEvents" not "PutEvents" --- moto/events/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/events/models.py b/moto/events/models.py index 5c1d507ca..038299996 100644 --- a/moto/events/models.py +++ b/moto/events/models.py @@ -210,7 +210,7 @@ class EventsBackend(BaseBackend): raise NotImplementedError() def put_permission(self, action, principal, statement_id): - if action is None or action != 'PutEvents': + if action is None or action != 'events:PutEvents': raise JsonRESTError('InvalidParameterValue', 'Action must be PutEvents') if principal is None or self.ACCOUNT_ID.match(principal) is None: From b6e795ce613cc336a541d948bfd91c406e843751 Mon Sep 17 00:00:00 2001 From: cpitchford Date: Tue, 15 May 2018 18:30:30 +0100 Subject: [PATCH 29/48] Testing using fully qualified Action events:PutEvents The Action parameter to put_permission must be fully qualified as events:PutEvents as per: http://boto3.readthedocs.io/en/latest/reference/services/events.html#CloudWatchEvents.Client.put_permission --- tests/test_events/test_events.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_events/test_events.py b/tests/test_events/test_events.py index e839bde5b..c421ba521 100644 --- a/tests/test_events/test_events.py +++ b/tests/test_events/test_events.py @@ -177,8 +177,8 @@ def test_remove_targets(): def test_permissions(): client = boto3.client('events', 'eu-central-1') - client.put_permission(Action='PutEvents', Principal='111111111111', StatementId='Account1') - client.put_permission(Action='PutEvents', Principal='222222222222', StatementId='Account2') + client.put_permission(Action='events:PutEvents', Principal='111111111111', StatementId='Account1') + client.put_permission(Action='events:PutEvents', Principal='222222222222', StatementId='Account2') resp = client.describe_event_bus() assert len(resp['Policy']['Statement']) == 2 From 176f19136c22f9fb32a6dc0aaed813ddad15915b Mon Sep 17 00:00:00 2001 From: cpitchford Date: Tue, 15 May 2018 18:58:16 +0100 Subject: [PATCH 30/48] Action is now recorded as event:PutEvents (fully qualified) We no longer need to amend how the Action is reported in the describe_event_bus policy --- moto/events/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/events/models.py b/moto/events/models.py index 038299996..07f19cbab 100644 --- a/moto/events/models.py +++ b/moto/events/models.py @@ -235,7 +235,7 @@ class EventsBackend(BaseBackend): 'Sid': statement_id, 'Effect': 'Allow', 'Principal': {'AWS': 'arn:aws:iam::{0}:root'.format(data['principal'])}, - 'Action': 'events:{0}'.format(data['action']), + 'Action': data['action'], 'Resource': arn }) return { From d4cc7f1399a309a6bb81c628ab0603e46430a043 Mon Sep 17 00:00:00 2001 From: Alan Justino da Silva Date: Tue, 15 May 2018 15:13:01 -0300 Subject: [PATCH 31/48] LambdaVersion providing ARN on __repr__ Just a dirty patch --- moto/awslambda/models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index 80b4ffba3..1ebfc2823 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -445,6 +445,9 @@ class LambdaVersion(BaseModel): def __init__(self, spec): self.version = spec['Version'] + def __repr__(self): + return str(self.logical_resource_id) + @classmethod def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): From fc32a79139b9a33bba725b1f1d9c37908a131f92 Mon Sep 17 00:00:00 2001 From: cpitchford Date: Wed, 16 May 2018 18:13:44 +0100 Subject: [PATCH 32/48] New line in LaunchConfigurationARN statement The statement that defines LaunchConfigurationARN had a newline and whitespace prefix The ARN reported in describe_launch_configurations has this newline and white space. Arns should not have whitespace or newlines! --- moto/autoscaling/responses.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/moto/autoscaling/responses.py b/moto/autoscaling/responses.py index c7170e17e..be7f2627a 100644 --- a/moto/autoscaling/responses.py +++ b/moto/autoscaling/responses.py @@ -320,8 +320,7 @@ DESCRIBE_LAUNCH_CONFIGURATIONS_TEMPLATE = """ {% endif %} {{ launch_configuration.instance_type }} - arn:aws:autoscaling:us-east-1:803981987763:launchConfiguration: - 9dbbbf87-6141-428a-a409-0752edbe6cad:launchConfigurationName/{{ launch_configuration.name }} + arn:aws:autoscaling:us-east-1:803981987763:launchConfiguration:9dbbbf87-6141-428a-a409-0752edbe6cad:launchConfigurationName/{{ launch_configuration.name }} {% if launch_configuration.block_device_mappings %} {% for mount_point, mapping in launch_configuration.block_device_mappings.items() %} From ffc2f4ca059eeaeebeb7c82c613bbc741a6ba385 Mon Sep 17 00:00:00 2001 From: cpitchford Date: Wed, 16 May 2018 18:46:50 +0100 Subject: [PATCH 33/48] Spaces also exist in AutoScalingGroupARN AutoScalingGroupARN has whitespace and newline that leaks into describe_auto_scaling_groups --- moto/autoscaling/responses.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/moto/autoscaling/responses.py b/moto/autoscaling/responses.py index be7f2627a..5586c51dd 100644 --- a/moto/autoscaling/responses.py +++ b/moto/autoscaling/responses.py @@ -516,8 +516,7 @@ DESCRIBE_AUTOSCALING_GROUPS_TEMPLATE = """{{ group.health_check_period }} {{ group.default_cooldown }} - arn:aws:autoscaling:us-east-1:803981987763:autoScalingGroup:ca861182-c8f9-4ca7-b1eb-cd35505f5ebb - :autoScalingGroupName/{{ group.name }} + arn:aws:autoscaling:us-east-1:803981987763:autoScalingGroup:ca861182-c8f9-4ca7-b1eb-cd35505f5ebb:autoScalingGroupName/{{ group.name }} {% if group.termination_policies %} {% for policy in group.termination_policies %} From b5bdf6693c1ed615571b1099766c411600d3db10 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Tue, 29 May 2018 21:57:54 -0400 Subject: [PATCH 34/48] Require version of responses that supports callbacks. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ebbf6f0cd..ba7db78b4 100755 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ install_requires = [ "docker>=2.5.1", "jsondiff==1.1.1", "aws-xray-sdk<0.96,>=0.93", - "responses", + "responses>=0.9.0", ] extras_require = { From 222cb1535c1027a2eec0bb09c01a18b38cc540b4 Mon Sep 17 00:00:00 2001 From: Daniel Birnstiel Date: Tue, 29 May 2018 16:06:25 +0200 Subject: [PATCH 35/48] Add RawMessageDelivery for SNS subscriptions (fixes #1571) --- moto/sns/models.py | 5 ++++- tests/test_sns/test_publishing_boto3.py | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/moto/sns/models.py b/moto/sns/models.py index 1c1be6680..c538e63f0 100644 --- a/moto/sns/models.py +++ b/moto/sns/models.py @@ -94,7 +94,10 @@ class Subscription(BaseModel): if self.protocol == 'sqs': queue_name = self.endpoint.split(":")[-1] region = self.endpoint.split(":")[3] - enveloped_message = json.dumps(self.get_post_data(message, message_id, subject, message_attributes=message_attributes), sort_keys=True, indent=2, separators=(',', ': ')) + if self.attributes.get('RawMessageDelivery') != 'true': + enveloped_message = json.dumps(self.get_post_data(message, message_id, subject, message_attributes=message_attributes), sort_keys=True, indent=2, separators=(',', ': ')) + else: + enveloped_message = message sqs_backends[region].send_message(queue_name, enveloped_message) elif self.protocol in ['http', 'https']: post_data = self.get_post_data(message, message_id, subject) diff --git a/tests/test_sns/test_publishing_boto3.py b/tests/test_sns/test_publishing_boto3.py index 7db072287..7fc9374e1 100644 --- a/tests/test_sns/test_publishing_boto3.py +++ b/tests/test_sns/test_publishing_boto3.py @@ -42,6 +42,29 @@ def test_publish_to_sqs(): acquired_message.should.equal(expected) +@mock_sqs +@mock_sns +def test_publish_to_sqs_raw(): + sns = boto3.resource('sns', region_name='us-east-1') + topic = sns.create_topic(Name='some-topic') + + sqs = boto3.resource('sqs', region_name='us-east-1') + queue = sqs.create_queue(QueueName='test-queue') + + subscription = topic.subscribe( + Protocol='sqs', Endpoint=queue.attributes['QueueArn']) + + subscription.set_attributes( + AttributeName='RawMessageDelivery', AttributeValue='true') + + message = 'my message' + with freeze_time("2015-01-01 12:00:00"): + topic.publish(Message=message) + + messages = queue.receive_messages(MaxNumberOfMessages=1) + messages[0].body.should.equal(message) + + @mock_sqs @mock_sns def test_publish_to_sqs_bad(): From 9b8e62e1f1f258d661a4bb4f71b8d44b9b1c17cd Mon Sep 17 00:00:00 2001 From: Daniel Birnstiel Date: Wed, 30 May 2018 09:22:46 +0200 Subject: [PATCH 36/48] Add MessageGroupId support to SQS queues (fixes #1655) --- moto/sqs/models.py | 53 +++++++++++++++--- tests/test_sqs/test_sqs.py | 109 +++++++++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+), 9 deletions(-) diff --git a/moto/sqs/models.py b/moto/sqs/models.py index 9c8858bc0..19def38c4 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -180,6 +180,7 @@ class Queue(BaseModel): self.permissions = {} self._messages = [] + self._pending_messages = set() now = unix_time() self.created_timestamp = now @@ -209,6 +210,16 @@ class Queue(BaseModel): if self.fifo_queue and not self.name.endswith('.fifo'): raise MessageAttributesInvalid('Queue name must end in .fifo for FIFO queues') + @property + def pending_messages(self): + return self._pending_messages + + @property + def pending_message_groups(self): + return set(message.group_id + for message in self._pending_messages + if message.group_id is not None) + def _set_attributes(self, attributes, now=None): if not now: now = unix_time() @@ -448,6 +459,7 @@ class SQSBackend(BaseBackend): """ queue = self.get_queue(queue_name) result = [] + previous_result_count = len(result) polling_end = unix_time() + wait_seconds_timeout @@ -457,19 +469,25 @@ class SQSBackend(BaseBackend): if result or (wait_seconds_timeout and unix_time() > polling_end): break - if len(queue.messages) == 0: - # we want to break here, otherwise it will be an infinite loop - if wait_seconds_timeout == 0: - break - - import time - time.sleep(0.001) - continue - messages_to_dlq = [] + for message in queue.messages: if not message.visible: continue + + if message in queue.pending_messages: + # The message is pending but is visible again, so the + # consumer must have timed out. + queue.pending_messages.remove(message) + + if message.group_id and queue.fifo_queue: + if message.group_id in queue.pending_message_groups: + # There is already one active message with the same + # group, so we cannot deliver this one. + continue + + queue.pending_messages.add(message) + if queue.dead_letter_queue is not None and message.approximate_receive_count >= queue.redrive_policy['maxReceiveCount']: messages_to_dlq.append(message) continue @@ -485,6 +503,18 @@ class SQSBackend(BaseBackend): queue._messages.remove(message) queue.dead_letter_queue.add_message(message) + if previous_result_count == len(result): + if wait_seconds_timeout == 0: + # There is timeout and we have added no additional results, + # so break to avoid an infinite loop. + break + + import time + time.sleep(0.001) + continue + + previous_result_count = len(result) + return result def delete_message(self, queue_name, receipt_handle): @@ -494,6 +524,7 @@ class SQSBackend(BaseBackend): # Only delete message if it is not visible and the reciept_handle # matches. if message.receipt_handle == receipt_handle: + queue.pending_messages.remove(message) continue new_messages.append(message) queue._messages = new_messages @@ -505,6 +536,10 @@ class SQSBackend(BaseBackend): if message.visible: raise MessageNotInflight message.change_visibility(visibility_timeout) + if message.visible: + # If the message is visible again, remove it from pending + # messages. + queue.pending_messages.remove(message) return raise ReceiptHandleIsInvalid diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index 1280fed80..cfe481bea 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -1063,3 +1063,112 @@ def test_redrive_policy_set_attributes(): assert 'RedrivePolicy' in copy.attributes copy_policy = json.loads(copy.attributes['RedrivePolicy']) assert copy_policy == redrive_policy + + +@mock_sqs +def test_receive_messages_with_message_group_id(): + sqs = boto3.resource('sqs', region_name='us-east-1') + queue = sqs.create_queue(QueueName="test-queue.fifo", + Attributes={ + 'FifoQueue': 'true', + }) + queue.set_attributes(Attributes={"VisibilityTimeout": "3600"}) + queue.send_message( + MessageBody="message-1", + MessageGroupId="group" + ) + queue.send_message( + MessageBody="message-2", + MessageGroupId="group" + ) + + messages = queue.receive_messages() + messages.should.have.length_of(1) + message = messages[0] + + # received message is not deleted! + + messages = queue.receive_messages(WaitTimeSeconds=0) + messages.should.have.length_of(0) + + # message is now processed, next one should be available + message.delete() + messages = queue.receive_messages() + messages.should.have.length_of(1) + + +@mock_sqs +def test_receive_messages_with_message_group_id_on_requeue(): + sqs = boto3.resource('sqs', region_name='us-east-1') + queue = sqs.create_queue(QueueName="test-queue.fifo", + Attributes={ + 'FifoQueue': 'true', + }) + queue.set_attributes(Attributes={"VisibilityTimeout": "3600"}) + queue.send_message( + MessageBody="message-1", + MessageGroupId="group" + ) + queue.send_message( + MessageBody="message-2", + MessageGroupId="group" + ) + + messages = queue.receive_messages() + messages.should.have.length_of(1) + message = messages[0] + + # received message is not deleted! + + messages = queue.receive_messages(WaitTimeSeconds=0) + messages.should.have.length_of(0) + + # message is now available again, next one should be available + message.change_visibility(VisibilityTimeout=0) + messages = queue.receive_messages() + messages.should.have.length_of(1) + messages[0].message_id.should.equal(message.message_id) + + +@mock_sqs +def test_receive_messages_with_message_group_id_on_visibility_timeout(): + if os.environ.get('TEST_SERVER_MODE', 'false').lower() == 'true': + raise SkipTest('Cant manipulate time in server mode') + + with freeze_time("2015-01-01 12:00:00"): + sqs = boto3.resource('sqs', region_name='us-east-1') + queue = sqs.create_queue(QueueName="test-queue.fifo", + Attributes={ + 'FifoQueue': 'true', + }) + queue.set_attributes(Attributes={"VisibilityTimeout": "3600"}) + queue.send_message( + MessageBody="message-1", + MessageGroupId="group" + ) + queue.send_message( + MessageBody="message-2", + MessageGroupId="group" + ) + + messages = queue.receive_messages() + messages.should.have.length_of(1) + message = messages[0] + + # received message is not deleted! + + messages = queue.receive_messages(WaitTimeSeconds=0) + messages.should.have.length_of(0) + + message.change_visibility(VisibilityTimeout=10) + + with freeze_time("2015-01-01 12:00:05"): + # no timeout yet + messages = queue.receive_messages(WaitTimeSeconds=0) + messages.should.have.length_of(0) + + with freeze_time("2015-01-01 12:00:15"): + # message is now available again, next one should be available + messages = queue.receive_messages() + messages.should.have.length_of(1) + messages[0].message_id.should.equal(message.message_id) From da8bd545bfa3e77e63da6c1dd29cfba57c6abc79 Mon Sep 17 00:00:00 2001 From: Theodore Wong Date: Wed, 30 May 2018 11:59:25 -0700 Subject: [PATCH 37/48] Fixed CF creation to trap imports of non-existent values --- moto/cloudformation/exceptions.py | 12 +++ moto/cloudformation/models.py | 2 +- moto/cloudformation/parsing.py | 4 +- .../test_cloudformation/test_import_value.py | 85 +++++++++++++++++++ 4 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 tests/test_cloudformation/test_import_value.py diff --git a/moto/cloudformation/exceptions.py b/moto/cloudformation/exceptions.py index 56a95382a..6ea15c5ca 100644 --- a/moto/cloudformation/exceptions.py +++ b/moto/cloudformation/exceptions.py @@ -33,6 +33,18 @@ class MissingParameterError(BadRequest): ) +class ExportNotFound(BadRequest): + """Exception to raise if a template tries to import a non-existent export""" + + def __init__(self, export_name): + template = Template(ERROR_RESPONSE) + super(ExportNotFound, self).__init__() + self.description = template.render( + code='ExportNotFound', + message="No export named {0} found.".format(export_name) + ) + + ERROR_RESPONSE = """ Sender diff --git a/moto/cloudformation/models.py b/moto/cloudformation/models.py index 57f42df56..e5ab7255d 100644 --- a/moto/cloudformation/models.py +++ b/moto/cloudformation/models.py @@ -38,7 +38,7 @@ class FakeStack(BaseModel): resource_status_reason="User Initiated") self.description = self.template_dict.get('Description') - self.cross_stack_resources = cross_stack_resources or [] + self.cross_stack_resources = cross_stack_resources or {} self.resource_map = self._create_resource_map() self.output_map = self._create_output_map() self._add_stack_event("CREATE_COMPLETE") diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index 849d8c917..173937345 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -28,7 +28,7 @@ from moto.s3 import models as s3_models from moto.sns import models as sns_models from moto.sqs import models as sqs_models from .utils import random_suffix -from .exceptions import MissingParameterError, UnformattedGetAttTemplateException, ValidationError +from .exceptions import ExportNotFound, MissingParameterError, UnformattedGetAttTemplateException, ValidationError from boto.cloudformation.stack import Output MODEL_MAP = { @@ -206,6 +206,8 @@ def clean_json(resource_json, resources_map): values = [x.value for x in resources_map.cross_stack_resources.values() if x.name == cleaned_val] if any(values): return values[0] + else: + raise ExportNotFound(cleaned_val) if 'Fn::GetAZs' in resource_json: region = resource_json.get('Fn::GetAZs') or DEFAULT_REGION diff --git a/tests/test_cloudformation/test_import_value.py b/tests/test_cloudformation/test_import_value.py new file mode 100644 index 000000000..297c1a7be --- /dev/null +++ b/tests/test_cloudformation/test_import_value.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, division, print_function, unicode_literals + +# Standard library modules +import unittest + +# Third-party modules +import boto3 +from botocore.exceptions import ClientError + +# Package modules +from moto import mock_cloudformation + +SG_STACK_NAME = 'simple-sg-stack' +SG_TEMPLATE = """ +AWSTemplateFormatVersion: 2010-09-09 +Description: Simple test CF template for moto_cloudformation + + +Resources: + SimpleSecurityGroup: + Type: AWS::EC2::SecurityGroup + Description: "A simple security group" + Properties: + GroupName: simple-security-group + GroupDescription: "A simple security group" + SecurityGroupEgress: + - + Description: "Egress to remote HTTPS servers" + CidrIp: 0.0.0.0/0 + IpProtocol: tcp + FromPort: 443 + ToPort: 443 + +Outputs: + SimpleSecurityGroupName: + Value: !GetAtt SimpleSecurityGroup.GroupId + Export: + Name: "SimpleSecurityGroup" + +""" + +EC2_STACK_NAME = 'simple-ec2-stack' +EC2_TEMPLATE = """ +--- +# The latest template format version is "2010-09-09" and as of 2018-04-09 +# is currently the only valid value. +AWSTemplateFormatVersion: 2010-09-09 +Description: Simple test CF template for moto_cloudformation + + +Resources: + SimpleInstance: + Type: AWS::EC2::Instance + Properties: + ImageId: ami-03cf127a + InstanceType: t2.micro + SecurityGroups: !Split [',', !ImportValue SimpleSecurityGroup] +""" + + +class TestSimpleInstance(unittest.TestCase): + def test_simple_instance(self): + """Test that we can create a simple CloudFormation stack that imports values from an existing CloudFormation stack""" + with mock_cloudformation(): + client = boto3.client('cloudformation') + client.create_stack(StackName=SG_STACK_NAME, TemplateBody=SG_TEMPLATE) + response = client.create_stack(StackName=EC2_STACK_NAME, TemplateBody=EC2_TEMPLATE) + self.assertIn('StackId', response) + response = client.describe_stacks(StackName=response['StackId']) + self.assertIn('Stacks', response) + stack_info = response['Stacks'] + self.assertEqual(1, len(stack_info)) + self.assertIn('StackName', stack_info[0]) + self.assertEqual(EC2_STACK_NAME, stack_info[0]['StackName']) + + def test_simple_instance_missing_export(self): + """Test that we get an exception if a CloudFormation stack tries to imports a non-existent export value""" + with mock_cloudformation(): + client = boto3.client('cloudformation') + with self.assertRaises(ClientError) as e: + client.create_stack(StackName=EC2_STACK_NAME, TemplateBody=EC2_TEMPLATE) + self.assertIn('Error', e.exception.response) + self.assertIn('Code', e.exception.response['Error']) + self.assertEqual('ExportNotFound', e.exception.response['Error']['Code']) From 96aba002427e7724d40a33d7b7a2631240952ab4 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Wed, 30 May 2018 20:39:32 -0400 Subject: [PATCH 38/48] Add freeze_time for ebs test. --- tests/test_ec2/test_elastic_block_store.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_ec2/test_elastic_block_store.py b/tests/test_ec2/test_elastic_block_store.py index 32ce1be22..8930838c6 100644 --- a/tests/test_ec2/test_elastic_block_store.py +++ b/tests/test_ec2/test_elastic_block_store.py @@ -8,6 +8,7 @@ import boto import boto3 from botocore.exceptions import ClientError from boto.exception import EC2ResponseError +from freezegun import freeze_time import sure # noqa from moto import mock_ec2_deprecated, mock_ec2 @@ -588,6 +589,7 @@ def test_volume_tag_escaping(): dict(snaps[0].tags).should.equal({'key': ''}) +@freeze_time @mock_ec2 def test_copy_snapshot(): ec2_client = boto3.client('ec2', region_name='eu-west-1') From 6008d6b5b18cc3aaaac5d17cfd717a98bdcc3dbd Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Wed, 30 May 2018 21:04:18 -0400 Subject: [PATCH 39/48] Add Cloudformation AWS::URLSuffix. Closes 1649. --- moto/cloudformation/parsing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index 849d8c917..bad9ee620 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -369,6 +369,7 @@ class ResourceMap(collections.Mapping): "AWS::Region": self._region_name, "AWS::StackId": stack_id, "AWS::StackName": stack_name, + "AWS::URLSuffix": "amazonaws.com", "AWS::NoValue": None, } From b0d5eaf0c634353d9827c5091ff6d49cf927ae83 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Wed, 30 May 2018 23:19:56 -0400 Subject: [PATCH 40/48] Add implementation coverage to readme. Closes #1570 --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index ff69fc171..3fbee44f8 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,8 @@ It gets even better! Moto isn't just for Python code and it isn't just for S3. L |------------------------------------------------------------------------------| ``` +For a full list of endpoint [implementation coverage](https://github.com/spulec/moto/blob/master/IMPLEMENTATION_COVERAGE.md) + ### Another Example Imagine you have a function that you use to launch new ec2 instances: From 324d17fd25f17e6bac24fdb80ca994a7e234f0f7 Mon Sep 17 00:00:00 2001 From: postmart Date: Thu, 31 May 2018 19:53:56 +0800 Subject: [PATCH 41/48] fixes wrong IAM get_user_policy() response --- moto/iam/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 9931cb8d0..786afab08 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -407,7 +407,7 @@ class IamResponse(BaseResponse): return template.render( user_name=user_name, policy_name=policy_name, - policy_document=policy_document + policy_document=policy_document.get('policy_document') ) def list_user_policies(self): From 29e6b2b070d3af0e0502216ffdc1da3355ad7b3f Mon Sep 17 00:00:00 2001 From: Gary Donovan Date: Fri, 1 Jun 2018 08:38:35 +1000 Subject: [PATCH 42/48] Fix subtle typo --- moto/dynamodb/responses.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/moto/dynamodb/responses.py b/moto/dynamodb/responses.py index d4f832be2..990069a46 100644 --- a/moto/dynamodb/responses.py +++ b/moto/dynamodb/responses.py @@ -62,13 +62,13 @@ class DynamoHandler(BaseResponse): name = body['TableName'] key_schema = body['KeySchema'] - hash_hey = key_schema['HashKeyElement'] - hash_key_attr = hash_hey['AttributeName'] - hash_key_type = hash_hey['AttributeType'] + hash_key = key_schema['HashKeyElement'] + hash_key_attr = hash_key['AttributeName'] + hash_key_type = hash_key['AttributeType'] - range_hey = key_schema.get('RangeKeyElement', {}) - range_key_attr = range_hey.get('AttributeName') - range_key_type = range_hey.get('AttributeType') + range_key = key_schema.get('RangeKeyElement', {}) + range_key_attr = range_key.get('AttributeName') + range_key_type = range_key.get('AttributeType') throughput = body["ProvisionedThroughput"] read_units = throughput["ReadCapacityUnits"] From 76c69c0dc5b38e83c030f509da2cceed3a8e30db Mon Sep 17 00:00:00 2001 From: Theodore Wong Date: Thu, 31 May 2018 16:31:04 -0700 Subject: [PATCH 43/48] Added region parameter to boto3 calls --- tests/test_cloudformation/test_import_value.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_cloudformation/test_import_value.py b/tests/test_cloudformation/test_import_value.py index 297c1a7be..04c2b5801 100644 --- a/tests/test_cloudformation/test_import_value.py +++ b/tests/test_cloudformation/test_import_value.py @@ -11,6 +11,8 @@ from botocore.exceptions import ClientError # Package modules from moto import mock_cloudformation +AWS_REGION = 'us-west-1' + SG_STACK_NAME = 'simple-sg-stack' SG_TEMPLATE = """ AWSTemplateFormatVersion: 2010-09-09 @@ -63,7 +65,7 @@ class TestSimpleInstance(unittest.TestCase): def test_simple_instance(self): """Test that we can create a simple CloudFormation stack that imports values from an existing CloudFormation stack""" with mock_cloudformation(): - client = boto3.client('cloudformation') + client = boto3.client('cloudformation', region_name=AWS_REGION) client.create_stack(StackName=SG_STACK_NAME, TemplateBody=SG_TEMPLATE) response = client.create_stack(StackName=EC2_STACK_NAME, TemplateBody=EC2_TEMPLATE) self.assertIn('StackId', response) @@ -77,7 +79,7 @@ class TestSimpleInstance(unittest.TestCase): def test_simple_instance_missing_export(self): """Test that we get an exception if a CloudFormation stack tries to imports a non-existent export value""" with mock_cloudformation(): - client = boto3.client('cloudformation') + client = boto3.client('cloudformation', region_name=AWS_REGION) with self.assertRaises(ClientError) as e: client.create_stack(StackName=EC2_STACK_NAME, TemplateBody=EC2_TEMPLATE) self.assertIn('Error', e.exception.response) From 620530c4ee1aa0ae9b16ec2943549963a49c0306 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Thu, 31 May 2018 22:07:13 -0400 Subject: [PATCH 44/48] Fix security group egress rule outout. Closes #1612 --- moto/ec2/models.py | 2 +- moto/ec2/responses/security_groups.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 674e0bddb..4e26f0f65 100755 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -1324,7 +1324,7 @@ class SecurityGroup(TaggedEC2Resource): self.name = name self.description = description self.ingress_rules = [] - self.egress_rules = [SecurityRule(-1, -1, -1, ['0.0.0.0/0'], [])] + self.egress_rules = [SecurityRule(-1, None, None, ['0.0.0.0/0'], [])] self.enis = {} self.vpc_id = vpc_id self.owner_id = "123456789012" diff --git a/moto/ec2/responses/security_groups.py b/moto/ec2/responses/security_groups.py index 9118c01b3..0009ff131 100644 --- a/moto/ec2/responses/security_groups.py +++ b/moto/ec2/responses/security_groups.py @@ -179,8 +179,12 @@ DESCRIBE_SECURITY_GROUPS_RESPONSE = """ Date: Thu, 31 May 2018 23:05:50 -0400 Subject: [PATCH 46/48] Fix creating SQS queue with same attributes. Closes #1663. --- moto/awslambda/models.py | 1 + moto/sqs/exceptions.py | 9 +++++++++ moto/sqs/models.py | 8 +++++++- tests/test_sqs/test_sqs.py | 23 +++++++++++++++++++++++ 4 files changed, 40 insertions(+), 1 deletion(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index 1404e0936..b11bde042 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -47,6 +47,7 @@ _stderr_regex = re.compile(r'START|END|REPORT RequestId: .*') _orig_adapter_send = requests.adapters.HTTPAdapter.send docker_3 = docker.__version__.startswith("3") + def zip2tar(zip_bytes): with TemporaryDirectory() as td: tarname = os.path.join(td, 'data.tar') diff --git a/moto/sqs/exceptions.py b/moto/sqs/exceptions.py index baf721b53..5f1cc46b2 100644 --- a/moto/sqs/exceptions.py +++ b/moto/sqs/exceptions.py @@ -1,4 +1,5 @@ from __future__ import unicode_literals +from moto.core.exceptions import RESTError class MessageNotInflight(Exception): @@ -21,3 +22,11 @@ class MessageAttributesInvalid(Exception): class QueueDoesNotExist(Exception): status_code = 404 description = "The specified queue does not exist for this wsdl version." + + +class QueueAlreadyExists(RESTError): + code = 400 + + def __init__(self, message): + super(QueueAlreadyExists, self).__init__( + "QueueAlreadyExists", message) diff --git a/moto/sqs/models.py b/moto/sqs/models.py index 797568781..b8db356e9 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -18,6 +18,7 @@ from .exceptions import ( MessageAttributesInvalid, MessageNotInflight, QueueDoesNotExist, + QueueAlreadyExists, ReceiptHandleIsInvalid, ) @@ -383,7 +384,12 @@ class SQSBackend(BaseBackend): def create_queue(self, name, **kwargs): queue = self.queues.get(name) - if queue is None: + if queue: + # Queue already exist. If attributes don't match, throw error + for key, value in kwargs.items(): + if getattr(queue, camelcase_to_underscores(key)) != value: + raise QueueAlreadyExists("The specified queue already exists.") + else: try: kwargs.pop('region') except KeyError: diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index cfe481bea..d3e4ca917 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -40,6 +40,29 @@ def test_create_fifo_queue_fail(): raise RuntimeError('Should of raised InvalidParameterValue Exception') +@mock_sqs +def test_create_queue_with_different_attributes_fail(): + sqs = boto3.client('sqs', region_name='us-east-1') + + sqs.create_queue( + QueueName='test-queue', + Attributes={ + 'VisibilityTimeout': '10', + } + ) + try: + sqs.create_queue( + QueueName='test-queue', + Attributes={ + 'VisibilityTimeout': '60', + } + ) + except botocore.exceptions.ClientError as err: + err.response['Error']['Code'].should.equal('QueueAlreadyExists') + else: + raise RuntimeError('Should of raised QueueAlreadyExists Exception') + + @mock_sqs def test_create_fifo_queue(): sqs = boto3.client('sqs', region_name='us-east-1') From d9a4501d24465a606c9f0637f59f823df339602a Mon Sep 17 00:00:00 2001 From: hsuhans Date: Sat, 2 Jun 2018 06:18:27 +0800 Subject: [PATCH 47/48] Change SNS http header to actual setting --- moto/sns/models.py | 2 +- tests/test_sns/test_publishing_boto3.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/sns/models.py b/moto/sns/models.py index db6760604..ebdf5cd16 100644 --- a/moto/sns/models.py +++ b/moto/sns/models.py @@ -101,7 +101,7 @@ class Subscription(BaseModel): sqs_backends[region].send_message(queue_name, enveloped_message) elif self.protocol in ['http', 'https']: post_data = self.get_post_data(message, message_id, subject) - requests.post(self.endpoint, json=post_data) + requests.post(self.endpoint, json=post_data, headers={'Content-Type': 'text/plain; charset=UTF-8'}) elif self.protocol == 'lambda': # TODO: support bad function name # http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html diff --git a/tests/test_sns/test_publishing_boto3.py b/tests/test_sns/test_publishing_boto3.py index 7fc9374e1..65d2f25cc 100644 --- a/tests/test_sns/test_publishing_boto3.py +++ b/tests/test_sns/test_publishing_boto3.py @@ -253,7 +253,7 @@ def test_publish_to_sqs_in_different_region(): @mock_sns def test_publish_to_http(): def callback(request): - request.headers["Content-Type"].should.equal("application/json") + request.headers["Content-Type"].should.equal("text/plain; charset=UTF-8") json.loads.when.called_with( request.body.decode() ).should_not.throw(Exception) From 80929292584ee78affc07643d16fae6bb31b4014 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 3 Jun 2018 20:29:15 -0400 Subject: [PATCH 48/48] Add repr for apigateway RestAPI. --- moto/apigateway/models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index 160b443b0..868262ccc 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -328,6 +328,9 @@ class RestAPI(BaseModel): self.resources = {} self.add_child('/') # Add default child + def __repr__(self): + return str(self.id) + def to_dict(self): return { "id": self.id,