From a974a3dfe4204e0f0eb690726754a5434486f053 Mon Sep 17 00:00:00 2001 From: Mike Liu Date: Wed, 25 Apr 2018 16:22:10 -0400 Subject: [PATCH 01/42] Create a Command class for the ssm backend. This class will make it easier to keep track of commands in a list for the SSM backend later on when we implement the ListCommands API call. --- moto/ssm/models.py | 113 +++++++++++++++++++++++++++++++++------------ 1 file changed, 84 insertions(+), 29 deletions(-) diff --git a/moto/ssm/models.py b/moto/ssm/models.py index fc74e1524..4f1bca213 100644 --- a/moto/ssm/models.py +++ b/moto/ssm/models.py @@ -58,11 +58,74 @@ class Parameter(BaseModel): return r +MAX_TIMEOUT_SECONDS = 3600 + + +class Command(BaseModel): + def __init__(self, comment='', document_name='', timeout_seconds=MAX_TIMEOUT_SECONDS, + instance_ids=[], max_concurrency='', max_errors='', + notification_config={}, output_s3_bucket_name='', + output_s3_key_prefix='', output_s3_region='', parameters={}, + service_role_arn='', targets=[]): + + self.error_count = 0 + self.completed_count = len(instance_ids) + self.target_count = len(instance_ids) + self.command_id = str(uuid.uuid4()) + self.status = 'Success' + self.status_details = 'Details placeholder' + + now = datetime.datetime.now() + self.requested_date_time = now.isoformat() + expires_after = now + datetime.timedelta(0, timeout_seconds) + self.expires_after = expires_after.isoformat() + + self.comment = comment + self.document_name = document_name + self.instance_ids = instance_ids + self.max_concurrency = max_concurrency + self.max_errors = max_errors + self.notification_config = notification_config + self.output_s3_bucket_name = output_s3_bucket_name + self.output_s3_key_prefix = output_s3_key_prefix + self.output_s3_region = output_s3_region + self.parameters = parameters + self.service_role_arn = service_role_arn + self.targets = targets + + def response_object(self): + r = { + 'CommandId': self.command_id, + 'Comment': self.comment, + 'CompletedCount': self.completed_count, + 'DocumentName': self.document_name, + 'ErrorCount': self.error_count, + 'ExpiresAfter': self.expires_after, + 'InstanceIds': self.instance_ids, + 'MaxConcurrency': self.max_concurrency, + 'MaxErrors': self.max_errors, + 'NotificationConfig': self.notification_config, + 'OutputS3Region': self.output_s3_region, + 'OutputS3BucketName': self.output_s3_bucket_name, + 'OutputS3KeyPrefix': self.output_s3_key_prefix, + 'Parameters': self.parameters, + 'RequestedDateTime': self.requested_date_time, + 'ServiceRole': self.service_role_arn, + 'Status': self.status, + 'StatusDetails': self.status_details, + 'TargetCount': self.target_count, + 'Targets': self.targets, + } + + return r + + class SimpleSystemManagerBackend(BaseBackend): def __init__(self): self._parameters = {} self._resource_tags = defaultdict(lambda: defaultdict(dict)) + self._commands = [] def delete_parameter(self, name): try: @@ -142,36 +205,28 @@ class SimpleSystemManagerBackend(BaseBackend): return self._resource_tags[resource_type][resource_id] def send_command(self, **kwargs): - instances = kwargs.get('InstanceIds', []) - now = datetime.datetime.now() - expires_after = now + datetime.timedelta(0, int(kwargs.get('TimeoutSeconds', 3600))) + command = Command( + comment=kwargs.get('Comment', ''), + document_name=kwargs.get('DocumentName'), + timeout_seconds=kwargs.get('TimeoutSeconds', 3600), + instance_ids=kwargs.get('InstanceIds', []), + max_concurrency=kwargs.get('MaxConcurrency', '50'), + max_errors=kwargs.get('MaxErrors', '0'), + notification_config=kwargs.get('NotificationConfig', { + 'NotificationArn': 'string', + 'NotificationEvents': ['Success'], + 'NotificationType': 'Command' + }), + output_s3_bucket_name=kwargs.get('OutputS3BucketName', ''), + output_s3_key_prefix=kwargs.get('OutputS3KeyPrefix', ''), + output_s3_region=kwargs.get('OutputS3Region', ''), + parameters=kwargs.get('Parameters', {}), + service_role_arn=kwargs.get('ServiceRoleArn', ''), + targets=kwargs.get('Targets', [])) + + self._commands.append(command) return { - 'Command': { - 'CommandId': str(uuid.uuid4()), - 'DocumentName': kwargs['DocumentName'], - 'Comment': kwargs.get('Comment'), - 'ExpiresAfter': expires_after.isoformat(), - 'Parameters': kwargs['Parameters'], - 'InstanceIds': kwargs['InstanceIds'], - 'Targets': kwargs.get('targets'), - 'RequestedDateTime': now.isoformat(), - 'Status': 'Success', - 'StatusDetails': 'string', - 'OutputS3Region': kwargs.get('OutputS3Region'), - 'OutputS3BucketName': kwargs.get('OutputS3BucketName'), - 'OutputS3KeyPrefix': kwargs.get('OutputS3KeyPrefix'), - 'MaxConcurrency': 'string', - 'MaxErrors': 'string', - 'TargetCount': len(instances), - 'CompletedCount': len(instances), - 'ErrorCount': 0, - 'ServiceRole': kwargs.get('ServiceRoleArn'), - 'NotificationConfig': { - 'NotificationArn': 'string', - 'NotificationEvents': ['Success'], - 'NotificationType': 'Command' - } - } + 'Command': command.response_object() } From 1016487c78b32fa0a67d093c3a43f39cd68aa231 Mon Sep 17 00:00:00 2001 From: Mike Liu Date: Wed, 25 Apr 2018 16:25:44 -0400 Subject: [PATCH 02/42] Implement the ListCommands API endpoint for the SSM client. Currently only supports getting commands by CommandId and InstanceIds. --- moto/ssm/models.py | 32 ++++++++++++++++++++++++++++++++ moto/ssm/responses.py | 5 +++++ 2 files changed, 37 insertions(+) diff --git a/moto/ssm/models.py b/moto/ssm/models.py index 4f1bca213..688cffcd5 100644 --- a/moto/ssm/models.py +++ b/moto/ssm/models.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from collections import defaultdict from moto.core import BaseBackend, BaseModel +from moto.core.exceptions import RESTError from moto.ec2 import ec2_backends import datetime @@ -229,6 +230,37 @@ class SimpleSystemManagerBackend(BaseBackend): 'Command': command.response_object() } + def list_commands(self, **kwargs): + """ + https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_ListCommands.html + """ + commands = self._commands + + command_id = kwargs.get('CommandId', None) + if command_id: + commands = [self.get_command_by_id(command_id)] + instance_id = kwargs.get('InstanceId', None) + if instance_id: + commands = self.get_commands_by_instance_id(instance_id) + + return { + 'Commands': [command.response_object() for command in commands] + } + + def get_command_by_id(self, id): + command = next( + (command for command in self._commands if command.command_id == id), None) + + if command is None: + raise RESTError('InvalidCommandId', 'Invalid command id.') + + return command + + def get_commands_by_instance_id(self, instance_id): + return [ + command for command in self._commands + if instance_id in command.instance_ids] + ssm_backends = {} for region, ec2_backend in ec2_backends.items(): diff --git a/moto/ssm/responses.py b/moto/ssm/responses.py index d9906a82e..e86fe05e3 100644 --- a/moto/ssm/responses.py +++ b/moto/ssm/responses.py @@ -204,3 +204,8 @@ class SimpleSystemManagerResponse(BaseResponse): return json.dumps( self.ssm_backend.send_command(**self.request_params) ) + + def list_commands(self): + return json.dumps( + self.ssm_backend.list_commands(**self.request_params) + ) From a0882316eca4aeaf9b833b97aa4838b509016373 Mon Sep 17 00:00:00 2001 From: Mike Liu Date: Wed, 25 Apr 2018 16:27:07 -0400 Subject: [PATCH 03/42] Tests for ListCommands SSM API endpoint. Test that ListCommands returns commands sent by SendCommand as well as filters by CommandId and InstanceId. In addition update the SendCommand test for optional parameters. --- tests/test_ssm/test_ssm_boto3.py | 60 ++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/test_ssm/test_ssm_boto3.py b/tests/test_ssm/test_ssm_boto3.py index 0531d1780..9270a7222 100644 --- a/tests/test_ssm/test_ssm_boto3.py +++ b/tests/test_ssm/test_ssm_boto3.py @@ -4,6 +4,10 @@ import boto3 import botocore.exceptions import sure # noqa import datetime +import uuid + +from botocore.exceptions import ClientError +from nose.tools import assert_raises from moto import mock_ssm @@ -497,3 +501,59 @@ def test_send_command(): cmd['OutputS3KeyPrefix'].should.equal('pref') cmd['ExpiresAfter'].should.be.greater_than(before) + + # test sending a command without any optional parameters + response = client.send_command( + DocumentName=ssm_document) + + cmd = response['Command'] + + cmd['CommandId'].should_not.be(None) + cmd['DocumentName'].should.equal(ssm_document) + + +@mock_ssm +def test_list_commands(): + client = boto3.client('ssm', region_name='us-east-1') + + ssm_document = 'AWS-RunShellScript' + params = {'commands': ['#!/bin/bash\necho \'hello world\'']} + + response = client.send_command( + InstanceIds=['i-123456'], + DocumentName=ssm_document, + Parameters=params, + OutputS3Region='us-east-2', + OutputS3BucketName='the-bucket', + OutputS3KeyPrefix='pref') + + cmd = response['Command'] + cmd_id = cmd['CommandId'] + + # get the command by id + response = client.list_commands( + CommandId=cmd_id) + + cmds = response['Commands'] + len(cmds).should.equal(1) + cmds[0]['CommandId'].should.equal(cmd_id) + + # add another command with the same instance id to test listing by + # instance id + client.send_command( + InstanceIds=['i-123456'], + DocumentName=ssm_document) + + response = client.list_commands( + InstanceId='i-123456') + + cmds = response['Commands'] + len(cmds).should.equal(2) + + for cmd in cmds: + cmd['InstanceIds'].should.contain('i-123456') + + # test the error case for an invalid command id + with assert_raises(ClientError): + response = client.list_commands( + CommandId=str(uuid.uuid4())) From bbf70bf21c4a477a4575d33c337cfe453485f45e Mon Sep 17 00:00:00 2001 From: Mike Liu Date: Mon, 11 Jun 2018 12:28:11 -0400 Subject: [PATCH 04/42] Fix using mutable default arguments. According to http://docs.python-guide.org/en/latest/writing/gotchas/#mutable-default-arguments using mutable default arguments is not a good practice since it doesn't perform intuitively. For example lists and dictionaries as default arguments are initialized ONCE instead of on each invocation of the function. --- moto/ssm/models.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/moto/ssm/models.py b/moto/ssm/models.py index 688cffcd5..f839e2e14 100644 --- a/moto/ssm/models.py +++ b/moto/ssm/models.py @@ -64,10 +64,22 @@ MAX_TIMEOUT_SECONDS = 3600 class Command(BaseModel): def __init__(self, comment='', document_name='', timeout_seconds=MAX_TIMEOUT_SECONDS, - instance_ids=[], max_concurrency='', max_errors='', - notification_config={}, output_s3_bucket_name='', - output_s3_key_prefix='', output_s3_region='', parameters={}, - service_role_arn='', targets=[]): + instance_ids=None, max_concurrency='', max_errors='', + notification_config=None, output_s3_bucket_name='', + output_s3_key_prefix='', output_s3_region='', parameters=None, + service_role_arn='', targets=None): + + if instance_ids is None: + instance_ids = [] + + if notification_config is None: + notification_config = {} + + if parameters is None: + parameters = {} + + if targets is None: + targets = [] self.error_count = 0 self.completed_count = len(instance_ids) From 29da006f787a30bba1359c4b91a0210921e4e73a Mon Sep 17 00:00:00 2001 From: Sanjeev Suresh Date: Thu, 21 Jun 2018 15:26:27 -0700 Subject: [PATCH 05/42] changed the getList default to an empty list instead of None, because otherwise an exception is raised when trying to iterate over an empty list --- moto/s3/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 431c9c988..c38eea5c7 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -718,7 +718,7 @@ class S3Backend(BaseBackend): if key_name in bucket.keys: key = bucket.keys[key_name] else: - for key_version in bucket.keys.getlist(key_name): + for key_version in bucket.keys.getlist(key_name, default=[]): if str(key_version.version_id) == str(version_id): key = key_version break From 2ebd5f735973463d67f423dc7ddcaf3ee832a13a Mon Sep 17 00:00:00 2001 From: Sanjeev Suresh Date: Fri, 22 Jun 2018 11:59:01 -0700 Subject: [PATCH 06/42] tests for prefixes that return empty result sets --- tests/test_s3/test_s3.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 9f37791cb..c5edc1173 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -2362,6 +2362,35 @@ def test_boto3_list_object_versions(): response['Body'].read().should.equal(items[-1]) +@mock_s3 +def test_boto3_bad_prefix_list_object_versions(): + s3 = boto3.client('s3', region_name='us-east-1') + bucket_name = 'mybucket' + key = 'key-with-versions' + bad_prefix = 'key-that-does-not-exist' + s3.create_bucket(Bucket=bucket_name) + s3.put_bucket_versioning( + Bucket=bucket_name, + VersioningConfiguration={ + 'Status': 'Enabled' + } + ) + items = (six.b('v1'), six.b('v2')) + for body in items: + s3.put_object( + Bucket=bucket_name, + Key=key, + Body=body + ) + response = s3.list_object_versions( + Bucket=bucket_name, + Prefix=bad_prefix, + ) + response['ResponseMetadata']['HTTPStatusCode'].should.equal(200) + response.should_not.contain('Versions') + response.should_not.contain('DeleteMarkers') + + @mock_s3 def test_boto3_delete_markers(): s3 = boto3.client('s3', region_name='us-east-1') From 7c1fd0a2f1b834dcc7cd6e9c852cff765b066cc7 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 24 Jun 2018 20:13:39 -0400 Subject: [PATCH 07/42] Fix ECS update_service and describing tasks. --- moto/ecs/exceptions.py | 11 +++++++++++ moto/ecs/models.py | 10 ++++++---- tests/test_ecs/test_ecs_boto3.py | 23 +++++++++++++++++++++++ 3 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 moto/ecs/exceptions.py diff --git a/moto/ecs/exceptions.py b/moto/ecs/exceptions.py new file mode 100644 index 000000000..c23d6fd1d --- /dev/null +++ b/moto/ecs/exceptions.py @@ -0,0 +1,11 @@ +from __future__ import unicode_literals +from moto.core.exceptions import RESTError + + +class ServiceNotFoundException(RESTError): + code = 400 + + def __init__(self, service_name): + super(ServiceNotFoundException, self).__init__( + error_type="ServiceNotFoundException", + message="The service {0} does not exist".format(service_name)) diff --git a/moto/ecs/models.py b/moto/ecs/models.py index 3c51cd03f..b8dc1b738 100644 --- a/moto/ecs/models.py +++ b/moto/ecs/models.py @@ -10,6 +10,8 @@ from moto.core import BaseBackend, BaseModel from moto.ec2 import ec2_backends from copy import copy +from .exceptions import ServiceNotFoundException + class BaseObject(BaseModel): @@ -601,8 +603,9 @@ class EC2ContainerServiceBackend(BaseBackend): raise Exception("tasks cannot be empty") response = [] for cluster, cluster_tasks in self.tasks.items(): - for task_id, task in cluster_tasks.items(): - if task_id in tasks or task.task_arn in tasks: + for task_arn, task in cluster_tasks.items(): + task_id = task_arn.split("/")[-1] + if task_arn in tasks or task.task_arn in tasks or any(task_id in task for task in tasks): response.append(task) return response @@ -698,8 +701,7 @@ class EC2ContainerServiceBackend(BaseBackend): cluster_service_pair].desired_count = desired_count return self.services[cluster_service_pair] else: - raise Exception("cluster {0} or service {1} does not exist".format( - cluster_name, service_name)) + raise ServiceNotFoundException(service_name) def delete_service(self, cluster_name, service_name): cluster_service_pair = '{0}:{1}'.format(cluster_name, service_name) diff --git a/tests/test_ecs/test_ecs_boto3.py b/tests/test_ecs/test_ecs_boto3.py index d2cfd3724..b2d06d035 100644 --- a/tests/test_ecs/test_ecs_boto3.py +++ b/tests/test_ecs/test_ecs_boto3.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from copy import deepcopy +from botocore.exceptions import ClientError import boto3 import sure # noqa import json @@ -450,6 +451,21 @@ def test_update_service(): response['service']['desiredCount'].should.equal(0) +@mock_ecs +def test_update_missing_service(): + client = boto3.client('ecs', region_name='us-east-1') + _ = client.create_cluster( + clusterName='test_ecs_cluster' + ) + + client.update_service.when.called_with( + cluster='test_ecs_cluster', + service='test_ecs_service', + taskDefinition='test_ecs_task', + desiredCount=0 + ).should.throw(ClientError) + + @mock_ecs def test_delete_service(): client = boto3.client('ecs', region_name='us-east-1') @@ -1054,6 +1070,13 @@ def test_describe_tasks(): set([response['tasks'][0]['taskArn'], response['tasks'] [1]['taskArn']]).should.equal(set(tasks_arns)) + # Test we can pass task ids instead of ARNs + response = client.describe_tasks( + cluster='test_ecs_cluster', + tasks=[tasks_arns[0].split("/")[-1]] + ) + len(response['tasks']).should.equal(1) + @mock_ecs def describe_task_definition(): From e32a6408618b47997ffac15f86a50d5d07f82787 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Huyghebaert?= Date: Mon, 25 Jun 2018 12:34:10 -0400 Subject: [PATCH 08/42] Fix for spulec/moto#1698 - ECR list_images missing RepositoryNotFoundException --- moto/ecr/models.py | 23 ++++++++++++++--------- tests/test_ecr/test_ecr_boto3.py | 28 +++++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/moto/ecr/models.py b/moto/ecr/models.py index e20c550c9..79ef9cf52 100644 --- a/moto/ecr/models.py +++ b/moto/ecr/models.py @@ -200,17 +200,22 @@ class ECRBackend(BaseBackend): """ maxResults and filtering not implemented """ - images = [] - for repository in self.repositories.values(): - if repository_name: - if repository.name != repository_name: - continue + repository = None + found = False + if repository_name in self.repositories: + repository = self.repositories[repository_name] if registry_id: - if repository.registry_id != registry_id: - continue + if repository.registry_id == registry_id: + found = True + else: + found = True - for image in repository.images: - images.append(image) + if not found: + raise RepositoryNotFoundException(repository_name, registry_id or DEFAULT_REGISTRY_ID) + + images = [] + for image in repository.images: + images.append(image) return images def describe_images(self, repository_name, registry_id=None, image_ids=None): diff --git a/tests/test_ecr/test_ecr_boto3.py b/tests/test_ecr/test_ecr_boto3.py index 7651dc832..d689e4e18 100644 --- a/tests/test_ecr/test_ecr_boto3.py +++ b/tests/test_ecr/test_ecr_boto3.py @@ -247,9 +247,31 @@ def test_list_images(): len(response['imageIds']).should.be(1) response['imageIds'][0]['imageTag'].should.equal('oldest') - response = client.list_images(repositoryName='test_repository_2', registryId='109876543210') - type(response['imageIds']).should.be(list) - len(response['imageIds']).should.be(0) + +@mock_ecr +def test_list_images_from_repository_that_doesnt_exist(): + client = boto3.client('ecr', region_name='us-east-1') + _ = client.create_repository( + repositoryName='test_repository_1' + ) + + # non existing repo + error_msg = re.compile( + r".*The repository with name 'repo-that-doesnt-exist' does not exist in the registry with id '123'.*", + re.MULTILINE) + client.list_images.when.called_with( + repositoryName='repo-that-doesnt-exist', + registryId='123', + ).should.throw(Exception, error_msg) + + # repo does not exist in specified registry + error_msg = re.compile( + r".*The repository with name 'test_repository_1' does not exist in the registry with id '222'.*", + re.MULTILINE) + client.list_images.when.called_with( + repositoryName='test_repository_1', + registryId='222', + ).should.throw(Exception, error_msg) @mock_ecr From f8fdd439ad17e9b10ff0e7ca9fa2d78423504ea7 Mon Sep 17 00:00:00 2001 From: Waleed Hamied Date: Tue, 3 Jul 2018 15:36:41 -0400 Subject: [PATCH 09/42] Added support for multipart upload confirmation with unquoted etags --- moto/s3/models.py | 7 +++++-- tests/test_s3/test_s3.py | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 431c9c988..30ecf1595 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -175,11 +175,14 @@ class FakeMultipart(BaseModel): count = 0 for pn, etag in body: part = self.parts.get(pn) - if part is None or part.etag != etag: + part_etag = None + if part is not None: + part_etag = part.etag.replace('"', '') + etag = etag.replace('"', '') + if part is None or part_etag != etag: raise InvalidPart() if last is not None and len(last.value) < UPLOAD_PART_MIN_SIZE: raise EntityTooSmall() - part_etag = part.etag.replace('"', '') md5s.extend(decode_hex(part_etag)[0]) total.extend(part.value) last = part diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 9f37791cb..6de79f874 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -225,6 +225,29 @@ def test_multipart_invalid_order(): bucket.complete_multipart_upload.when.called_with( multipart.key_name, multipart.id, xml).should.throw(S3ResponseError) +@mock_s3_deprecated +@reduced_min_part_size +def test_multipart_etag_quotes_stripped(): + # Create Bucket so that test can run + conn = boto.connect_s3('the_key', 'the_secret') + bucket = conn.create_bucket('mybucket') + + multipart = bucket.initiate_multipart_upload("the-key") + part1 = b'0' * REDUCED_PART_SIZE + etag1 = multipart.upload_part_from_file(BytesIO(part1), 1).etag + # last part, can be less than 5 MB + part2 = b'1' + etag2 = multipart.upload_part_from_file(BytesIO(part2), 2).etag + # Strip quotes from etags + etag1 = etag1.replace('"','') + etag2 = etag2.replace('"','') + xml = "{0}{1}" + xml = xml.format(1, etag1) + xml.format(2, etag2) + xml = "{0}".format(xml) + bucket.complete_multipart_upload.when.called_with( + multipart.key_name, multipart.id, xml).should_not.throw(S3ResponseError) + # we should get both parts as the key contents + bucket.get_key("the-key").etag.should.equal(EXPECTED_ETAG) @mock_s3_deprecated @reduced_min_part_size From 46dd351965160f262f916f7749b872f89d62fe5f Mon Sep 17 00:00:00 2001 From: Henadzi Tsaryk Date: Fri, 13 Jul 2018 12:06:28 +0300 Subject: [PATCH 10/42] Add ApproximateArrivalTimestamp and MillisBehindLatest to Kinesis get_records response (#1715) * Add ApproximateArrivalTimestamp to Kinesis response * Add MillisBehindLatest to Kinesis get_records response --- moto/kinesis/models.py | 24 +++++++---- moto/kinesis/responses.py | 5 ++- tests/test_kinesis/test_kinesis.py | 67 +++++++++++++++++++++++++++++- 3 files changed, 84 insertions(+), 12 deletions(-) diff --git a/moto/kinesis/models.py b/moto/kinesis/models.py index d9ea3b897..d9a47ea87 100644 --- a/moto/kinesis/models.py +++ b/moto/kinesis/models.py @@ -19,19 +19,20 @@ from .utils import compose_shard_iterator, compose_new_shard_iterator, decompose class Record(BaseModel): - def __init__(self, partition_key, data, sequence_number, explicit_hash_key): self.partition_key = partition_key self.data = data self.sequence_number = sequence_number self.explicit_hash_key = explicit_hash_key - self.create_at = unix_time() + self.created_at_datetime = datetime.datetime.utcnow() + self.created_at = unix_time(self.created_at_datetime) def to_json(self): return { "Data": self.data, "PartitionKey": self.partition_key, "SequenceNumber": str(self.sequence_number), + "ApproximateArrivalTimestamp": self.created_at_datetime.isoformat() } @@ -50,16 +51,21 @@ class Shard(BaseModel): def get_records(self, last_sequence_id, limit): last_sequence_id = int(last_sequence_id) results = [] + secs_behind_latest = 0 for sequence_number, record in self.records.items(): if sequence_number > last_sequence_id: results.append(record) last_sequence_id = sequence_number + very_last_record = self.records[next(reversed(self.records))] + secs_behind_latest = very_last_record.created_at - record.created_at + if len(results) == limit: break - return results, last_sequence_id + millis_behind_latest = int(secs_behind_latest * 1000) + return results, last_sequence_id, millis_behind_latest def put_record(self, partition_key, data, explicit_hash_key): # Note: this function is not safe for concurrency @@ -83,12 +89,12 @@ class Shard(BaseModel): return 0 def get_sequence_number_at(self, at_timestamp): - if not self.records or at_timestamp < list(self.records.values())[0].create_at: + if not self.records or at_timestamp < list(self.records.values())[0].created_at: return 0 else: # find the last item in the list that was created before # at_timestamp - r = next((r for r in reversed(self.records.values()) if r.create_at < at_timestamp), None) + r = next((r for r in reversed(self.records.values()) if r.created_at < at_timestamp), None) return r.sequence_number def to_json(self): @@ -226,7 +232,7 @@ class DeliveryStream(BaseModel): self.records = [] self.status = 'ACTIVE' - self.create_at = datetime.datetime.utcnow() + self.created_at = datetime.datetime.utcnow() self.last_updated = datetime.datetime.utcnow() @property @@ -267,7 +273,7 @@ class DeliveryStream(BaseModel): def to_dict(self): return { "DeliveryStreamDescription": { - "CreateTimestamp": time.mktime(self.create_at.timetuple()), + "CreateTimestamp": time.mktime(self.created_at.timetuple()), "DeliveryStreamARN": self.arn, "DeliveryStreamName": self.name, "DeliveryStreamStatus": self.status, @@ -329,12 +335,12 @@ class KinesisBackend(BaseBackend): stream = self.describe_stream(stream_name) shard = stream.get_shard(shard_id) - records, last_sequence_id = shard.get_records(last_sequence_id, limit) + records, last_sequence_id, millis_behind_latest = shard.get_records(last_sequence_id, limit) next_shard_iterator = compose_shard_iterator( stream_name, shard, last_sequence_id) - return next_shard_iterator, records + return next_shard_iterator, records, millis_behind_latest def put_record(self, stream_name, partition_key, explicit_hash_key, sequence_number_for_ordering, data): stream = self.describe_stream(stream_name) diff --git a/moto/kinesis/responses.py b/moto/kinesis/responses.py index b9b4883ef..72b2af4ce 100644 --- a/moto/kinesis/responses.py +++ b/moto/kinesis/responses.py @@ -80,12 +80,13 @@ class KinesisResponse(BaseResponse): shard_iterator = self.parameters.get("ShardIterator") limit = self.parameters.get("Limit") - next_shard_iterator, records = self.kinesis_backend.get_records( + next_shard_iterator, records, millis_behind_latest = self.kinesis_backend.get_records( shard_iterator, limit) return json.dumps({ "NextShardIterator": next_shard_iterator, - "Records": [record.to_json() for record in records] + "Records": [record.to_json() for record in records], + 'MillisBehindLatest': millis_behind_latest }) def put_record(self): diff --git a/tests/test_kinesis/test_kinesis.py b/tests/test_kinesis/test_kinesis.py index e3d350023..c70236978 100644 --- a/tests/test_kinesis/test_kinesis.py +++ b/tests/test_kinesis/test_kinesis.py @@ -89,6 +89,7 @@ def test_basic_shard_iterator(): response = conn.get_records(shard_iterator) shard_iterator = response['NextShardIterator'] response['Records'].should.equal([]) + response['MillisBehindLatest'].should.equal(0) @mock_kinesis_deprecated @@ -225,6 +226,7 @@ def test_get_records_after_sequence_number(): response = conn.get_records(shard_iterator) # And the first result returned should be the third item response['Records'][0]['Data'].should.equal('3') + response['MillisBehindLatest'].should.equal(0) @mock_kinesis_deprecated @@ -262,6 +264,7 @@ def test_get_records_latest(): response['Records'].should.have.length_of(1) response['Records'][0]['PartitionKey'].should.equal('last_record') response['Records'][0]['Data'].should.equal('last_record') + response['MillisBehindLatest'].should.equal(0) @mock_kinesis @@ -305,6 +308,7 @@ def test_get_records_at_timestamp(): response['Records'].should.have.length_of(len(keys)) partition_keys = [r['PartitionKey'] for r in response['Records']] partition_keys.should.equal(keys) + response['MillisBehindLatest'].should.equal(0) @mock_kinesis @@ -330,10 +334,69 @@ def test_get_records_at_very_old_timestamp(): shard_iterator = response['ShardIterator'] response = conn.get_records(ShardIterator=shard_iterator) - response['Records'].should.have.length_of(len(keys)) partition_keys = [r['PartitionKey'] for r in response['Records']] partition_keys.should.equal(keys) + response['MillisBehindLatest'].should.equal(0) + + +@mock_kinesis +def test_get_records_timestamp_filtering(): + conn = boto3.client('kinesis', region_name="us-west-2") + stream_name = "my_stream" + conn.create_stream(StreamName=stream_name, ShardCount=1) + + conn.put_record(StreamName=stream_name, + Data='0', + PartitionKey='0') + + time.sleep(1.0) + timestamp = datetime.datetime.utcnow() + + conn.put_record(StreamName=stream_name, + Data='1', + PartitionKey='1') + + response = conn.describe_stream(StreamName=stream_name) + shard_id = response['StreamDescription']['Shards'][0]['ShardId'] + response = conn.get_shard_iterator(StreamName=stream_name, + ShardId=shard_id, + ShardIteratorType='AT_TIMESTAMP', + Timestamp=timestamp) + shard_iterator = response['ShardIterator'] + + response = conn.get_records(ShardIterator=shard_iterator) + response['Records'].should.have.length_of(1) + response['Records'][0]['PartitionKey'].should.equal('1') + response['Records'][0]['ApproximateArrivalTimestamp'].should.be.\ + greater_than(timestamp) + response['MillisBehindLatest'].should.equal(0) + + +@mock_kinesis +def test_get_records_millis_behind_latest(): + conn = boto3.client('kinesis', region_name="us-west-2") + stream_name = "my_stream" + conn.create_stream(StreamName=stream_name, ShardCount=1) + + conn.put_record(StreamName=stream_name, + Data='0', + PartitionKey='0') + time.sleep(1.0) + conn.put_record(StreamName=stream_name, + Data='1', + PartitionKey='1') + + response = conn.describe_stream(StreamName=stream_name) + shard_id = response['StreamDescription']['Shards'][0]['ShardId'] + response = conn.get_shard_iterator(StreamName=stream_name, + ShardId=shard_id, + ShardIteratorType='TRIM_HORIZON') + shard_iterator = response['ShardIterator'] + + response = conn.get_records(ShardIterator=shard_iterator, Limit=1) + response['Records'].should.have.length_of(1) + response['MillisBehindLatest'].should.be.greater_than(0) @mock_kinesis @@ -363,6 +426,7 @@ def test_get_records_at_very_new_timestamp(): response = conn.get_records(ShardIterator=shard_iterator) response['Records'].should.have.length_of(0) + response['MillisBehindLatest'].should.equal(0) @mock_kinesis @@ -385,6 +449,7 @@ def test_get_records_from_empty_stream_at_timestamp(): response = conn.get_records(ShardIterator=shard_iterator) response['Records'].should.have.length_of(0) + response['MillisBehindLatest'].should.equal(0) @mock_kinesis_deprecated From 7d201c48b5696cb525975477f119e7de6fe7c78c Mon Sep 17 00:00:00 2001 From: Andrew Basson Date: Fri, 13 Jul 2018 11:07:09 +0200 Subject: [PATCH 11/42] Fix tier typo in get_amis.py (#1714) --- scripts/get_amis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/get_amis.py b/scripts/get_amis.py index 81f69c5dd..687dab2d4 100644 --- a/scripts/get_amis.py +++ b/scripts/get_amis.py @@ -1,7 +1,7 @@ import boto3 import json -# Taken from free tear list when creating an instance +# Taken from free tier list when creating an instance instances = [ 'ami-760aaa0f', 'ami-bb9a6bc2', 'ami-35e92e4c', 'ami-785db401', 'ami-b7e93bce', 'ami-dca37ea5', 'ami-999844e0', 'ami-9b32e8e2', 'ami-f8e54081', 'ami-bceb39c5', 'ami-03cf127a', 'ami-1ecc1e67', 'ami-c2ff2dbb', 'ami-12c6146b', From 802402bdba82a60c2c837046c9f3aa53623613d1 Mon Sep 17 00:00:00 2001 From: Gary Donovan Date: Fri, 13 Jul 2018 19:11:10 +1000 Subject: [PATCH 12/42] Tweak comparison to treat NULL/NOT_NULL correctly. (#1709) The AWS documentation says that a ComparisonOperator of NULL means the attribute should not exist, whereas NOT_NULL means that the attribute should exist. It explicitly says that an attribute with a value of NULL is considered to exist, which contradicts our previous implementation. This affects both put_item and get_item in dynamodb2. https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Condition.html --- .gitignore | 1 + moto/dynamodb2/comparisons.py | 6 +- moto/dynamodb2/models.py | 18 ++-- .../test_dynamodb_table_without_range_key.py | 94 ++++++++++++++++++- 4 files changed, 107 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index c4b8c5034..7f57e98e9 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ build/ python_env .ropeproject/ .pytest_cache/ +venv/ diff --git a/moto/dynamodb2/comparisons.py b/moto/dynamodb2/comparisons.py index 51d62fb83..53226c557 100644 --- a/moto/dynamodb2/comparisons.py +++ b/moto/dynamodb2/comparisons.py @@ -29,8 +29,10 @@ COMPARISON_FUNCS = { 'GT': GT_FUNCTION, '>': GT_FUNCTION, - 'NULL': lambda item_value: item_value is None, - 'NOT_NULL': lambda item_value: item_value is not None, + # NULL means the value should not exist at all + 'NULL': lambda item_value: False, + # NOT_NULL means the value merely has to exist, and values of None are valid + 'NOT_NULL': lambda item_value: True, 'CONTAINS': lambda item_value, test_value: test_value in item_value, 'NOT_CONTAINS': lambda item_value, test_value: test_value not in item_value, 'BEGINS_WITH': lambda item_value, test_value: item_value.startswith(test_value), diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index db6bf04a3..b327c7a4b 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -409,7 +409,8 @@ class Table(BaseModel): current_attr = current for key, val in expected.items(): - if 'Exists' in val and val['Exists'] is False: + if 'Exists' in val and val['Exists'] is False \ + or 'ComparisonOperator' in val and val['ComparisonOperator'] == 'NULL': if key in current_attr: raise ValueError("The conditional request failed") elif key not in current_attr: @@ -419,8 +420,10 @@ class Table(BaseModel): elif 'ComparisonOperator' in val: comparison_func = get_comparison_func( val['ComparisonOperator']) - dynamo_types = [DynamoType(ele) for ele in val[ - "AttributeValueList"]] + dynamo_types = [ + DynamoType(ele) for ele in + val.get("AttributeValueList", []) + ] for t in dynamo_types: if not comparison_func(current_attr[key].value, t.value): raise ValueError('The conditional request failed') @@ -827,7 +830,8 @@ class DynamoDBBackend(BaseBackend): expected = {} for key, val in expected.items(): - if 'Exists' in val and val['Exists'] is False: + if 'Exists' in val and val['Exists'] is False \ + or 'ComparisonOperator' in val and val['ComparisonOperator'] == 'NULL': if key in item_attr: raise ValueError("The conditional request failed") elif key not in item_attr: @@ -837,8 +841,10 @@ class DynamoDBBackend(BaseBackend): elif 'ComparisonOperator' in val: comparison_func = get_comparison_func( val['ComparisonOperator']) - dynamo_types = [DynamoType(ele) for ele in val[ - "AttributeValueList"]] + dynamo_types = [ + DynamoType(ele) for ele in + val.get("AttributeValueList", []) + ] for t in dynamo_types: if not comparison_func(item_attr[key].value, t.value): raise ValueError('The conditional request failed') diff --git a/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py b/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py index 5e635d5ef..15e5284b7 100644 --- a/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py +++ b/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py @@ -596,7 +596,50 @@ def test_boto3_conditions(): @mock_dynamodb2 -def test_boto3_put_item_conditions_fails(): +def test_boto3_put_item_conditions_pass(): + table = _create_user_table() + table.put_item(Item={'username': 'johndoe', 'foo': 'bar'}) + table.put_item( + Item={'username': 'johndoe', 'foo': 'baz'}, + Expected={ + 'foo': { + 'ComparisonOperator': 'EQ', + 'AttributeValueList': ['bar'] + } + }) + final_item = table.get_item(Key={'username': 'johndoe'}) + assert dict(final_item)['Item']['foo'].should.equal("baz") + +@mock_dynamodb2 +def test_boto3_put_item_conditions_pass_because_expect_not_exists_by_compare_to_null(): + table = _create_user_table() + table.put_item(Item={'username': 'johndoe', 'foo': 'bar'}) + table.put_item( + Item={'username': 'johndoe', 'foo': 'baz'}, + Expected={ + 'whatever': { + 'ComparisonOperator': 'NULL', + } + }) + final_item = table.get_item(Key={'username': 'johndoe'}) + assert dict(final_item)['Item']['foo'].should.equal("baz") + +@mock_dynamodb2 +def test_boto3_put_item_conditions_pass_because_expect_exists_by_compare_to_not_null(): + table = _create_user_table() + table.put_item(Item={'username': 'johndoe', 'foo': 'bar'}) + table.put_item( + Item={'username': 'johndoe', 'foo': 'baz'}, + Expected={ + 'foo': { + 'ComparisonOperator': 'NOT_NULL', + } + }) + final_item = table.get_item(Key={'username': 'johndoe'}) + assert dict(final_item)['Item']['foo'].should.equal("baz") + +@mock_dynamodb2 +def test_boto3_put_item_conditions_fail(): table = _create_user_table() table.put_item(Item={'username': 'johndoe', 'foo': 'bar'}) table.put_item.when.called_with( @@ -609,7 +652,7 @@ def test_boto3_put_item_conditions_fails(): }).should.throw(botocore.client.ClientError) @mock_dynamodb2 -def test_boto3_update_item_conditions_fails(): +def test_boto3_update_item_conditions_fail(): table = _create_user_table() table.put_item(Item={'username': 'johndoe', 'foo': 'baz'}) table.update_item.when.called_with( @@ -622,7 +665,7 @@ def test_boto3_update_item_conditions_fails(): }).should.throw(botocore.client.ClientError) @mock_dynamodb2 -def test_boto3_update_item_conditions_fails_because_expect_not_exists(): +def test_boto3_update_item_conditions_fail_because_expect_not_exists(): table = _create_user_table() table.put_item(Item={'username': 'johndoe', 'foo': 'baz'}) table.update_item.when.called_with( @@ -634,6 +677,19 @@ def test_boto3_update_item_conditions_fails_because_expect_not_exists(): } }).should.throw(botocore.client.ClientError) +@mock_dynamodb2 +def test_boto3_update_item_conditions_fail_because_expect_not_exists_by_compare_to_null(): + table = _create_user_table() + table.put_item(Item={'username': 'johndoe', 'foo': 'baz'}) + table.update_item.when.called_with( + Key={'username': 'johndoe'}, + UpdateExpression='SET foo=bar', + Expected={ + 'foo': { + 'ComparisonOperator': 'NULL', + } + }).should.throw(botocore.client.ClientError) + @mock_dynamodb2 def test_boto3_update_item_conditions_pass(): table = _create_user_table() @@ -650,7 +706,7 @@ def test_boto3_update_item_conditions_pass(): assert dict(returned_item)['Item']['foo'].should.equal("baz") @mock_dynamodb2 -def test_boto3_update_item_conditions_pass_because_expext_not_exists(): +def test_boto3_update_item_conditions_pass_because_expect_not_exists(): table = _create_user_table() table.put_item(Item={'username': 'johndoe', 'foo': 'bar'}) table.update_item( @@ -664,6 +720,36 @@ def test_boto3_update_item_conditions_pass_because_expext_not_exists(): returned_item = table.get_item(Key={'username': 'johndoe'}) assert dict(returned_item)['Item']['foo'].should.equal("baz") +@mock_dynamodb2 +def test_boto3_update_item_conditions_pass_because_expect_not_exists_by_compare_to_null(): + table = _create_user_table() + table.put_item(Item={'username': 'johndoe', 'foo': 'bar'}) + table.update_item( + Key={'username': 'johndoe'}, + UpdateExpression='SET foo=baz', + Expected={ + 'whatever': { + 'ComparisonOperator': 'NULL', + } + }) + returned_item = table.get_item(Key={'username': 'johndoe'}) + assert dict(returned_item)['Item']['foo'].should.equal("baz") + +@mock_dynamodb2 +def test_boto3_update_item_conditions_pass_because_expect_exists_by_compare_to_not_null(): + table = _create_user_table() + table.put_item(Item={'username': 'johndoe', 'foo': 'bar'}) + table.update_item( + Key={'username': 'johndoe'}, + UpdateExpression='SET foo=baz', + Expected={ + 'foo': { + 'ComparisonOperator': 'NOT_NULL', + } + }) + returned_item = table.get_item(Key={'username': 'johndoe'}) + assert dict(returned_item)['Item']['foo'].should.equal("baz") + @mock_dynamodb2 def test_boto3_put_item_conditions_pass(): table = _create_user_table() From 51db19067c72bb7feb5a2a44fb89f33374dd34f8 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Fri, 13 Jul 2018 19:21:33 +1000 Subject: [PATCH 13/42] Allow attributes to be set with subscribe command (#1705) --- moto/sns/responses.py | 5 ++ tests/test_sns/test_subscriptions_boto3.py | 66 ++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/moto/sns/responses.py b/moto/sns/responses.py index 035d56584..8c1bb885e 100644 --- a/moto/sns/responses.py +++ b/moto/sns/responses.py @@ -181,6 +181,7 @@ class SNSResponse(BaseResponse): topic_arn = self._get_param('TopicArn') endpoint = self._get_param('Endpoint') protocol = self._get_param('Protocol') + attributes = self._get_attributes() if protocol == 'sms' and not is_e164(endpoint): return self._error( @@ -190,6 +191,10 @@ class SNSResponse(BaseResponse): subscription = self.backend.subscribe(topic_arn, endpoint, protocol) + if attributes is not None: + for attr_name, attr_value in attributes.items(): + self.backend.set_subscription_attributes(subscription.arn, attr_name, attr_value) + if self.request_json: return json.dumps({ "SubscribeResponse": { diff --git a/tests/test_sns/test_subscriptions_boto3.py b/tests/test_sns/test_subscriptions_boto3.py index 98075e617..2a56c8213 100644 --- a/tests/test_sns/test_subscriptions_boto3.py +++ b/tests/test_sns/test_subscriptions_boto3.py @@ -182,6 +182,72 @@ def test_subscription_paging(): topic1_subscriptions.shouldnt.have("NextToken") +@mock_sns +def test_creating_subscription_with_attributes(): + conn = boto3.client('sns', region_name='us-east-1') + conn.create_topic(Name="some-topic") + response = conn.list_topics() + topic_arn = response["Topics"][0]['TopicArn'] + + delivery_policy = json.dumps({ + 'healthyRetryPolicy': { + "numRetries": 10, + "minDelayTarget": 1, + "maxDelayTarget":2 + } + }) + + filter_policy = json.dumps({ + "store": ["example_corp"], + "event": ["order_cancelled"], + "encrypted": [False], + "customer_interests": ["basketball", "baseball"] + }) + + conn.subscribe(TopicArn=topic_arn, + Protocol="http", + Endpoint="http://example.com/", + Attributes={ + 'RawMessageDelivery': 'true', + 'DeliveryPolicy': delivery_policy, + 'FilterPolicy': filter_policy + }) + + subscriptions = conn.list_subscriptions()["Subscriptions"] + subscriptions.should.have.length_of(1) + subscription = subscriptions[0] + subscription["TopicArn"].should.equal(topic_arn) + subscription["Protocol"].should.equal("http") + subscription["SubscriptionArn"].should.contain(topic_arn) + subscription["Endpoint"].should.equal("http://example.com/") + + # Test the subscription attributes have been set + subscription_arn = subscription["SubscriptionArn"] + attrs = conn.get_subscription_attributes( + SubscriptionArn=subscription_arn + ) + + attrs['Attributes']['RawMessageDelivery'].should.equal('true') + attrs['Attributes']['DeliveryPolicy'].should.equal(delivery_policy) + attrs['Attributes']['FilterPolicy'].should.equal(filter_policy) + + # Now unsubscribe the subscription + conn.unsubscribe(SubscriptionArn=subscription["SubscriptionArn"]) + + # And there should be zero subscriptions left + subscriptions = conn.list_subscriptions()["Subscriptions"] + subscriptions.should.have.length_of(0) + + # invalid attr name + with assert_raises(ClientError): + conn.subscribe(TopicArn=topic_arn, + Protocol="http", + Endpoint="http://example.com/", + Attributes={ + 'InvalidName': 'true' + }) + + @mock_sns def test_set_subscription_attributes(): conn = boto3.client('sns', region_name='us-east-1') From dcdaca898437dc88bc00ddb53db69ed579460f3f Mon Sep 17 00:00:00 2001 From: Nate Peterson Date: Fri, 13 Jul 2018 03:24:11 -0600 Subject: [PATCH 14/42] parameters return from root path (#1701) --- moto/ssm/models.py | 2 +- tests/test_ssm/test_ssm_boto3.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/moto/ssm/models.py b/moto/ssm/models.py index aaeccc887..bdc98e61b 100644 --- a/moto/ssm/models.py +++ b/moto/ssm/models.py @@ -100,7 +100,7 @@ class SimpleSystemManagerBackend(BaseBackend): # difference here. path = path.rstrip('/') + '/' for param in self._parameters: - if not param.startswith(path): + if path != '/' and not param.startswith(path): continue if '/' in param[len(path) + 1:] and not recursive: continue diff --git a/tests/test_ssm/test_ssm_boto3.py b/tests/test_ssm/test_ssm_boto3.py index ad48fd7ed..e58879bc7 100644 --- a/tests/test_ssm/test_ssm_boto3.py +++ b/tests/test_ssm/test_ssm_boto3.py @@ -95,6 +95,27 @@ def test_get_parameters_by_path(): Type='SecureString', KeyId='alias/aws/ssm') + client.put_parameter( + Name='foo', + Description='A test parameter', + Value='bar', + Type='String') + + client.put_parameter( + Name='baz', + Description='A test parameter', + Value='qux', + Type='String') + + response = client.get_parameters_by_path(Path='/', Recursive=False) + len(response['Parameters']).should.equal(2) + {p['Value'] for p in response['Parameters']}.should.equal( + set(['bar', 'qux']) + ) + + response = client.get_parameters_by_path(Path='/', Recursive=True) + len(response['Parameters']).should.equal(9) + response = client.get_parameters_by_path(Path='/foo') len(response['Parameters']).should.equal(2) {p['Value'] for p in response['Parameters']}.should.equal( @@ -417,6 +438,7 @@ def test_describe_parameters_filter_keyid(): response['Parameters'][0]['Type'].should.equal('SecureString') ''.should.equal(response.get('NextToken', '')) + @mock_ssm def test_describe_parameters_attributes(): client = boto3.client('ssm', region_name='us-east-1') @@ -445,6 +467,7 @@ def test_describe_parameters_attributes(): response['Parameters'][1].get('Description').should.be.none response['Parameters'][1]['Version'].should.equal(1) + @mock_ssm def test_get_parameter_invalid(): client = client = boto3.client('ssm', region_name='us-east-1') From c3b690114c11703c110cd310ca24cceb948fffab Mon Sep 17 00:00:00 2001 From: temyers Date: Fri, 13 Jul 2018 18:40:54 +0800 Subject: [PATCH 15/42] Add support for CloudFormation Fn::GetAtt to KMS Key (#1681) --- moto/kms/models.py | 6 +++ tests/test_cloudformation/fixtures/kms_key.py | 39 +++++++++++++++++++ .../test_cloudformation/test_stack_parsing.py | 15 +++++++ 3 files changed, 60 insertions(+) create mode 100644 tests/test_cloudformation/fixtures/kms_key.py diff --git a/moto/kms/models.py b/moto/kms/models.py index ca27f030a..89ebf0082 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -58,6 +58,12 @@ class Key(BaseModel): return key + def get_cfn_attribute(self, attribute_name): + from moto.cloudformation.exceptions import UnformattedGetAttTemplateException + if attribute_name == 'Arn': + return self.arn + raise UnformattedGetAttTemplateException() + class KmsBackend(BaseBackend): diff --git a/tests/test_cloudformation/fixtures/kms_key.py b/tests/test_cloudformation/fixtures/kms_key.py new file mode 100644 index 000000000..366dbfcf5 --- /dev/null +++ b/tests/test_cloudformation/fixtures/kms_key.py @@ -0,0 +1,39 @@ +from __future__ import unicode_literals + +template = { + "AWSTemplateFormatVersion": "2010-09-09", + + "Description": "AWS CloudFormation Sample Template to create a KMS Key. The Fn::GetAtt is used to retrieve the ARN", + + "Resources" : { + "myKey" : { + "Type" : "AWS::KMS::Key", + "Properties" : { + "Description": "Sample KmsKey", + "EnableKeyRotation": False, + "Enabled": True, + "KeyPolicy" : { + "Version": "2012-10-17", + "Id": "key-default-1", + "Statement": [ + { + "Sid": "Enable IAM User Permissions", + "Effect": "Allow", + "Principal": { + "AWS": { "Fn::Join" : ["" , ["arn:aws:iam::", {"Ref" : "AWS::AccountId"} ,":root" ]] } + }, + "Action": "kms:*", + "Resource": "*" + } + ] + } + } + } + }, + "Outputs" : { + "KeyArn" : { + "Description": "Generated Key Arn", + "Value" : { "Fn::GetAtt" : [ "myKey", "Arn" ] } + } + } +} \ No newline at end of file diff --git a/tests/test_cloudformation/test_stack_parsing.py b/tests/test_cloudformation/test_stack_parsing.py index af7e608db..d25c69cf1 100644 --- a/tests/test_cloudformation/test_stack_parsing.py +++ b/tests/test_cloudformation/test_stack_parsing.py @@ -254,6 +254,21 @@ def test_parse_stack_with_get_attribute_outputs(): output.should.be.a(Output) output.value.should.equal("my-queue") +def test_parse_stack_with_get_attribute_kms(): + from .fixtures.kms_key import template + + template_json = json.dumps(template) + stack = FakeStack( + stack_id="test_id", + name="test_stack", + template=template_json, + parameters={}, + region_name='us-west-1') + + stack.output_map.should.have.length_of(1) + list(stack.output_map.keys())[0].should.equal('KeyArn') + output = list(stack.output_map.values())[0] + output.should.be.a(Output) def test_parse_stack_with_get_availability_zones(): stack = FakeStack( From c20e8568e0b0a168569496ed602fec3aa912d18c Mon Sep 17 00:00:00 2001 From: Aidan Fewster Date: Fri, 13 Jul 2018 12:53:00 +0100 Subject: [PATCH 16/42] APIGateway - Generate API key value when no value provided (#1713) --- moto/apigateway/models.py | 6 +----- tests/test_apigateway/test_apigateway.py | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index 868262ccc..5fdaed1e8 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -300,11 +300,7 @@ class ApiKey(BaseModel, dict): generateDistinctId=False, value=None, stageKeys=None, customerId=None): super(ApiKey, self).__init__() self['id'] = create_id() - if generateDistinctId: - # Best guess of what AWS does internally - self['value'] = ''.join(random.sample(string.ascii_letters + string.digits, 40)) - else: - self['value'] = value + self['value'] = value if value else ''.join(random.sample(string.ascii_letters + string.digits, 40)) self['name'] = name self['customerId'] = customerId self['description'] = description diff --git a/tests/test_apigateway/test_apigateway.py b/tests/test_apigateway/test_apigateway.py index 3f75b3ebd..99fef0481 100644 --- a/tests/test_apigateway/test_apigateway.py +++ b/tests/test_apigateway/test_apigateway.py @@ -981,7 +981,7 @@ def test_api_keys(): apikey['value'].should.equal(apikey_value) apikey_name = 'TESTKEY2' - payload = {'name': apikey_name, 'generateDistinctId': True} + payload = {'name': apikey_name } response = client.create_api_key(**payload) apikey_id = response['id'] apikey = client.get_api_key(apiKey=apikey_id) From 43e430560c82699ef1f34b87714348114db8a8c5 Mon Sep 17 00:00:00 2001 From: Aidan Fewster Date: Tue, 10 Jul 2018 14:58:02 +0100 Subject: [PATCH 17/42] APIGateway: Added API for usage plans --- moto/apigateway/models.py | 29 +++++++++++++++ moto/apigateway/responses.py | 22 ++++++++++++ moto/apigateway/urls.py | 2 ++ tests/test_apigateway/test_apigateway.py | 37 +++++++++++++++++++ tests/test_apigateway/test_server.py | 45 +++++++++++++++++++++++- 5 files changed, 134 insertions(+), 1 deletion(-) diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index 5fdaed1e8..d29a7669f 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -309,6 +309,19 @@ class ApiKey(BaseModel, dict): self['stageKeys'] = stageKeys +class UsagePlan(BaseModel, dict): + + def __init__(self, name=None, description=None, apiStages=[], + throttle=None, quota=None): + super(UsagePlan, self).__init__() + self['id'] = create_id() + self['name'] = name + self['description'] = description + self['apiStages'] = apiStages + self['throttle'] = throttle + self['quota'] = quota + + class RestAPI(BaseModel): def __init__(self, id, region_name, name, description): @@ -408,6 +421,7 @@ class APIGatewayBackend(BaseBackend): super(APIGatewayBackend, self).__init__() self.apis = {} self.keys = {} + self.usage_plans = {} self.region_name = region_name def reset(self): @@ -576,6 +590,21 @@ class APIGatewayBackend(BaseBackend): self.keys.pop(api_key_id) return {} + def create_usage_plan(self, payload): + plan = UsagePlan(**payload) + self.usage_plans[plan['id']] = plan + return plan + + def get_usage_plans(self): + return list(self.usage_plans.values()) + + def get_usage_plan(self, usage_plan_id): + return self.usage_plans[usage_plan_id] + + def delete_usage_plan(self, usage_plan_id): + self.usage_plans.pop(usage_plan_id) + return {} + apigateway_backends = {} for region_name in Session().get_available_regions('apigateway'): diff --git a/moto/apigateway/responses.py b/moto/apigateway/responses.py index ff6ef1f33..1d119baf7 100644 --- a/moto/apigateway/responses.py +++ b/moto/apigateway/responses.py @@ -248,3 +248,25 @@ class APIGatewayResponse(BaseResponse): elif self.method == 'DELETE': apikey_response = self.backend.delete_apikey(apikey) return 200, {}, json.dumps(apikey_response) + + def usage_plans(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + + if self.method == 'POST': + usage_plan_response = self.backend.create_usage_plan(json.loads(self.body)) + elif self.method == 'GET': + usage_plans_response = self.backend.get_usage_plans() + return 200, {}, json.dumps({"item": usage_plans_response}) + return 200, {}, json.dumps(usage_plan_response) + + def usage_plan_individual(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + + url_path_parts = self.path.split("/") + usage_plan = url_path_parts[2] + + if self.method == 'GET': + usage_plan_response = self.backend.get_usage_plan(usage_plan) + elif self.method == 'DELETE': + usage_plan_response = self.backend.delete_usage_plan(usage_plan) + return 200, {}, json.dumps(usage_plan_response) diff --git a/moto/apigateway/urls.py b/moto/apigateway/urls.py index ca1f445a7..a2f4ace9a 100644 --- a/moto/apigateway/urls.py +++ b/moto/apigateway/urls.py @@ -20,4 +20,6 @@ url_paths = { '{0}/restapis/(?P[^/]+)/resources/(?P[^/]+)/methods/(?P[^/]+)/integration/responses/(?P\d+)/?$': APIGatewayResponse().integration_responses, '{0}/apikeys$': APIGatewayResponse().apikeys, '{0}/apikeys/(?P[^/]+)': APIGatewayResponse().apikey_individual, + '{0}/usageplans$': APIGatewayResponse().usage_plans, + '{0}/usageplans/(?P[^/]+)': APIGatewayResponse().usage_plan_individual, } diff --git a/tests/test_apigateway/test_apigateway.py b/tests/test_apigateway/test_apigateway.py index 99fef0481..ea57c43f4 100644 --- a/tests/test_apigateway/test_apigateway.py +++ b/tests/test_apigateway/test_apigateway.py @@ -995,3 +995,40 @@ def test_api_keys(): response = client.get_api_keys() len(response['items']).should.equal(1) + +@mock_apigateway +def test_usage_plans(): + region_name = 'us-west-2' + client = boto3.client('apigateway', region_name=region_name) + response = client.get_usage_plans() + len(response['items']).should.equal(0) + + usage_plan_name = 'TEST-PLAN' + payload = {'name': usage_plan_name} + response = client.create_usage_plan(**payload) + usage_plan = client.get_usage_plan(usagePlanId=response['id']) + usage_plan['name'].should.equal(usage_plan_name) + usage_plan['apiStages'].should.equal([]) + + usage_plan_name = 'TEST-PLAN-2' + usage_plan_description = 'Description' + usage_plan_quota = {'limit': 10, 'period': 'DAY', 'offset': 0} + usage_plan_throttle = {'rateLimit': 2, 'burstLimit': 1} + usage_plan_api_stages = [{'apiId': 'foo', 'stage': 'bar'}] + payload = {'name': usage_plan_name, 'description': usage_plan_description, 'quota': usage_plan_quota, 'throttle': usage_plan_throttle, 'apiStages': usage_plan_api_stages} + response = client.create_usage_plan(**payload) + usage_plan_id = response['id'] + usage_plan = client.get_usage_plan(usagePlanId=usage_plan_id) + usage_plan['name'].should.equal(usage_plan_name) + usage_plan['description'].should.equal(usage_plan_description) + usage_plan['apiStages'].should.equal(usage_plan_api_stages) + usage_plan['throttle'].should.equal(usage_plan_throttle) + usage_plan['quota'].should.equal(usage_plan_quota) + + response = client.get_usage_plans() + len(response['items']).should.equal(2) + + client.delete_usage_plan(usagePlanId=usage_plan_id) + + response = client.get_usage_plans() + len(response['items']).should.equal(1) diff --git a/tests/test_apigateway/test_server.py b/tests/test_apigateway/test_server.py index f2a29e253..b76a39e53 100644 --- a/tests/test_apigateway/test_server.py +++ b/tests/test_apigateway/test_server.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals import sure # noqa +import json import moto.server as server @@ -9,8 +10,50 @@ Test the different server responses def test_list_apis(): - backend = server.create_backend_app("apigateway") + backend = server.create_backend_app('apigateway') test_client = backend.test_client() res = test_client.get('/restapis') res.data.should.equal(b'{"item": []}') + +def test_usage_plans_apis(): + backend = server.create_backend_app('apigateway') + test_client = backend.test_client() + + ''' + List usage plans (expect empty) + ''' + res = test_client.get('/usageplans') + json.loads(res.data)["item"].should.have.length_of(0) + + ''' + Create usage plan + ''' + res = test_client.post('/usageplans', data=json.dumps({'name': 'test'})) + created_plan = json.loads(res.data) + created_plan['name'].should.equal('test') + + ''' + List usage plans (expect 1 plan) + ''' + res = test_client.get('/usageplans') + json.loads(res.data)["item"].should.have.length_of(1) + + ''' + Get single usage plan + ''' + res = test_client.get('/usageplans/{0}'.format(created_plan["id"])) + fetched_plan = json.loads(res.data) + fetched_plan.should.equal(created_plan) + + ''' + Delete usage plan + ''' + res = test_client.delete('/usageplans/{0}'.format(created_plan["id"])) + res.data.should.equal(b'{}') + + ''' + List usage plans (expect empty again) + ''' + res = test_client.get('/usageplans') + json.loads(res.data)["item"].should.have.length_of(0) From 9bd6f0a725b5fea685e8b470155495193e643fc8 Mon Sep 17 00:00:00 2001 From: Aidan Fewster Date: Wed, 11 Jul 2018 17:17:58 +0100 Subject: [PATCH 18/42] APIGateway: Added usage plan keys API --- moto/apigateway/exceptions.py | 8 +++ moto/apigateway/models.py | 40 +++++++++++++- moto/apigateway/responses.py | 33 +++++++++++- moto/apigateway/urls.py | 4 +- tests/test_apigateway/test_apigateway.py | 52 ++++++++++++++++++ tests/test_apigateway/test_server.py | 68 +++++++++++++++++------- 6 files changed, 184 insertions(+), 21 deletions(-) diff --git a/moto/apigateway/exceptions.py b/moto/apigateway/exceptions.py index d4cf8d1c7..62fa24392 100644 --- a/moto/apigateway/exceptions.py +++ b/moto/apigateway/exceptions.py @@ -8,3 +8,11 @@ class StageNotFoundException(RESTError): def __init__(self): super(StageNotFoundException, self).__init__( "NotFoundException", "Invalid stage identifier specified") + + +class ApiKeyNotFoundException(RESTError): + code = 404 + + def __init__(self): + super(ApiKeyNotFoundException, self).__init__( + "NotFoundException", "Invalid API Key identifier specified") diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index d29a7669f..4094c7a69 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -10,7 +10,7 @@ from boto3.session import Session import responses from moto.core import BaseBackend, BaseModel from .utils import create_id -from .exceptions import StageNotFoundException +from .exceptions import StageNotFoundException, ApiKeyNotFoundException STAGE_URL = "https://{api_id}.execute-api.{region_name}.amazonaws.com/{stage_name}" @@ -322,6 +322,16 @@ class UsagePlan(BaseModel, dict): self['quota'] = quota +class UsagePlanKey(BaseModel, dict): + + def __init__(self, id, type, name, value): + super(UsagePlanKey, self).__init__() + self['id'] = id + self['name'] = name + self['type'] = type + self['value'] = value + + class RestAPI(BaseModel): def __init__(self, id, region_name, name, description): @@ -422,6 +432,7 @@ class APIGatewayBackend(BaseBackend): self.apis = {} self.keys = {} self.usage_plans = {} + self.usage_plan_keys = {} self.region_name = region_name def reset(self): @@ -605,6 +616,33 @@ class APIGatewayBackend(BaseBackend): self.usage_plans.pop(usage_plan_id) return {} + def create_usage_plan_key(self, usage_plan_id, payload): + if usage_plan_id not in self.usage_plan_keys: + self.usage_plan_keys[usage_plan_id] = {} + + key_id = payload["keyId"] + if key_id not in self.keys: + raise ApiKeyNotFoundException() + + api_key = self.keys[key_id] + + usage_plan_key = UsagePlanKey(id=key_id, type=payload["keyType"], name=api_key["name"], value=api_key["value"]) + self.usage_plan_keys[usage_plan_id][usage_plan_key['id']] = usage_plan_key + return usage_plan_key + + def get_usage_plan_keys(self, usage_plan_id): + if usage_plan_id not in self.usage_plan_keys: + return [] + + return list(self.usage_plan_keys[usage_plan_id].values()) + + def get_usage_plan_key(self, usage_plan_id, key_id): + return self.usage_plan_keys[usage_plan_id][key_id] + + def delete_usage_plan_key(self, usage_plan_id, key_id): + self.usage_plan_keys[usage_plan_id].pop(key_id) + return {} + apigateway_backends = {} for region_name in Session().get_available_regions('apigateway'): diff --git a/moto/apigateway/responses.py b/moto/apigateway/responses.py index 1d119baf7..7364ae2cb 100644 --- a/moto/apigateway/responses.py +++ b/moto/apigateway/responses.py @@ -4,7 +4,7 @@ import json from moto.core.responses import BaseResponse from .models import apigateway_backends -from .exceptions import StageNotFoundException +from .exceptions import StageNotFoundException, ApiKeyNotFoundException class APIGatewayResponse(BaseResponse): @@ -270,3 +270,34 @@ class APIGatewayResponse(BaseResponse): elif self.method == 'DELETE': usage_plan_response = self.backend.delete_usage_plan(usage_plan) return 200, {}, json.dumps(usage_plan_response) + + def usage_plan_keys(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + + url_path_parts = self.path.split("/") + usage_plan_id = url_path_parts[2] + + if self.method == 'POST': + try: + usage_plan_response = self.backend.create_usage_plan_key(usage_plan_id, json.loads(self.body)) + except ApiKeyNotFoundException as error: + return error.code, {}, '{{"message":"{0}","code":"{1}"}}'.format(error.message, error.error_type) + + elif self.method == 'GET': + usage_plans_response = self.backend.get_usage_plan_keys(usage_plan_id) + return 200, {}, json.dumps({"item": usage_plans_response}) + + return 200, {}, json.dumps(usage_plan_response) + + def usage_plan_key_individual(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + + url_path_parts = self.path.split("/") + usage_plan_id = url_path_parts[2] + key_id = url_path_parts[4] + + if self.method == 'GET': + usage_plan_response = self.backend.get_usage_plan_key(usage_plan_id, key_id) + elif self.method == 'DELETE': + usage_plan_response = self.backend.delete_usage_plan_key(usage_plan_id, key_id) + return 200, {}, json.dumps(usage_plan_response) diff --git a/moto/apigateway/urls.py b/moto/apigateway/urls.py index a2f4ace9a..5c6d372fa 100644 --- a/moto/apigateway/urls.py +++ b/moto/apigateway/urls.py @@ -21,5 +21,7 @@ url_paths = { '{0}/apikeys$': APIGatewayResponse().apikeys, '{0}/apikeys/(?P[^/]+)': APIGatewayResponse().apikey_individual, '{0}/usageplans$': APIGatewayResponse().usage_plans, - '{0}/usageplans/(?P[^/]+)': APIGatewayResponse().usage_plan_individual, + '{0}/usageplans/(?P[^/]+)/?$': APIGatewayResponse().usage_plan_individual, + '{0}/usageplans/(?P[^/]+)/keys$': APIGatewayResponse().usage_plan_keys, + '{0}/usageplans/(?P[^/]+)/keys/(?P[^/]+)/?$': APIGatewayResponse().usage_plan_key_individual, } diff --git a/tests/test_apigateway/test_apigateway.py b/tests/test_apigateway/test_apigateway.py index ea57c43f4..8a2c4370d 100644 --- a/tests/test_apigateway/test_apigateway.py +++ b/tests/test_apigateway/test_apigateway.py @@ -1032,3 +1032,55 @@ def test_usage_plans(): response = client.get_usage_plans() len(response['items']).should.equal(1) + +@mock_apigateway +def test_usage_plan_keys(): + region_name = 'us-west-2' + usage_plan_id = 'test_usage_plan_id' + client = boto3.client('apigateway', region_name=region_name) + usage_plan_id = "test" + + # Create an API key so we can use it + key_name = 'test-api-key' + response = client.create_api_key(name=key_name) + key_id = response["id"] + key_value = response["value"] + + # Get current plan keys (expect none) + response = client.get_usage_plan_keys(usagePlanId=usage_plan_id) + len(response['items']).should.equal(0) + + # Create usage plan key + key_type = 'API_KEY' + payload = {'usagePlanId': usage_plan_id, 'keyId': key_id, 'keyType': key_type } + response = client.create_usage_plan_key(**payload) + usage_plan_key_id = response["id"] + + # Get current plan keys (expect 1) + response = client.get_usage_plan_keys(usagePlanId=usage_plan_id) + len(response['items']).should.equal(1) + + # Get a single usage plan key and check it matches the created one + usage_plan_key = client.get_usage_plan_key(usagePlanId=usage_plan_id, keyId=usage_plan_key_id) + usage_plan_key['name'].should.equal(key_name) + usage_plan_key['id'].should.equal(key_id) + usage_plan_key['type'].should.equal(key_type) + usage_plan_key['value'].should.equal(key_value) + + # Delete usage plan key + client.delete_usage_plan_key(usagePlanId=usage_plan_id, keyId=key_id) + + # Get current plan keys (expect none) + response = client.get_usage_plan_keys(usagePlanId=usage_plan_id) + len(response['items']).should.equal(0) + +@mock_apigateway +def test_create_usage_plan_key_non_existent_api_key(): + region_name = 'us-west-2' + usage_plan_id = 'test_usage_plan_id' + client = boto3.client('apigateway', region_name=region_name) + usage_plan_id = "test" + + # Attempt to create a usage plan key for a API key that doesn't exists + payload = {'usagePlanId': usage_plan_id, 'keyId': 'non-existent', 'keyType': 'API_KEY' } + client.create_usage_plan_key.when.called_with(**payload).should.throw(ClientError) diff --git a/tests/test_apigateway/test_server.py b/tests/test_apigateway/test_server.py index b76a39e53..953d942cc 100644 --- a/tests/test_apigateway/test_server.py +++ b/tests/test_apigateway/test_server.py @@ -20,40 +20,72 @@ def test_usage_plans_apis(): backend = server.create_backend_app('apigateway') test_client = backend.test_client() - ''' - List usage plans (expect empty) - ''' + # List usage plans (expect empty) res = test_client.get('/usageplans') json.loads(res.data)["item"].should.have.length_of(0) - ''' - Create usage plan - ''' + # Create usage plan res = test_client.post('/usageplans', data=json.dumps({'name': 'test'})) created_plan = json.loads(res.data) created_plan['name'].should.equal('test') - ''' - List usage plans (expect 1 plan) - ''' + # List usage plans (expect 1 plan) res = test_client.get('/usageplans') json.loads(res.data)["item"].should.have.length_of(1) - ''' - Get single usage plan - ''' + # Get single usage plan res = test_client.get('/usageplans/{0}'.format(created_plan["id"])) fetched_plan = json.loads(res.data) fetched_plan.should.equal(created_plan) - ''' - Delete usage plan - ''' + # Delete usage plan res = test_client.delete('/usageplans/{0}'.format(created_plan["id"])) res.data.should.equal(b'{}') - ''' - List usage plans (expect empty again) - ''' + # List usage plans (expect empty again) res = test_client.get('/usageplans') json.loads(res.data)["item"].should.have.length_of(0) + +def test_usage_plans_keys(): + backend = server.create_backend_app('apigateway') + test_client = backend.test_client() + usage_plan_id = 'test_plan_id' + + # Create API key to be used in tests + res = test_client.post('/apikeys', data=json.dumps({'name': 'test'})) + created_api_key = json.loads(res.data) + + # List usage plans keys (expect empty) + res = test_client.get('/usageplans/{0}/keys'.format(usage_plan_id)) + json.loads(res.data)["item"].should.have.length_of(0) + + # Create usage plan key + res = test_client.post('/usageplans/{0}/keys'.format(usage_plan_id), data=json.dumps({'keyId': created_api_key["id"], 'keyType': 'API_KEY'})) + created_usage_plan_key = json.loads(res.data) + + # List usage plans keys (expect 1 key) + res = test_client.get('/usageplans/{0}/keys'.format(usage_plan_id)) + json.loads(res.data)["item"].should.have.length_of(1) + + # Get single usage plan key + res = test_client.get('/usageplans/{0}/keys/{1}'.format(usage_plan_id, created_api_key["id"])) + fetched_plan_key = json.loads(res.data) + fetched_plan_key.should.equal(created_usage_plan_key) + + # Delete usage plan key + res = test_client.delete('/usageplans/{0}/keys/{1}'.format(usage_plan_id, created_api_key["id"])) + res.data.should.equal(b'{}') + + # List usage plans keys (expect to be empty again) + res = test_client.get('/usageplans/{0}/keys'.format(usage_plan_id)) + json.loads(res.data)["item"].should.have.length_of(0) + +def test_create_usage_plans_key_non_existent_api_key(): + backend = server.create_backend_app('apigateway') + test_client = backend.test_client() + usage_plan_id = 'test_plan_id' + + # Create usage plan key with non-existent api key + res = test_client.post('/usageplans/{0}/keys'.format(usage_plan_id), data=json.dumps({'keyId': 'non-existent', 'keyType': 'API_KEY'})) + res.status_code.should.equal(404) + From ba1ceee95f289334b68330555598e80e581622ef Mon Sep 17 00:00:00 2001 From: Zane Williamson Date: Sat, 14 Jul 2018 00:39:19 -0700 Subject: [PATCH 19/42] Adding create_secret, exception handle, fix (#1680) --- moto/secretsmanager/exceptions.py | 15 ++++++ moto/secretsmanager/models.py | 42 +++++++++++++--- moto/secretsmanager/responses.py | 8 +++ .../test_secretsmanager.py | 24 ++++++++- tests/test_secretsmanager/test_server.py | 49 +++++++++++++++++-- 5 files changed, 126 insertions(+), 12 deletions(-) create mode 100644 moto/secretsmanager/exceptions.py diff --git a/moto/secretsmanager/exceptions.py b/moto/secretsmanager/exceptions.py new file mode 100644 index 000000000..99d74f281 --- /dev/null +++ b/moto/secretsmanager/exceptions.py @@ -0,0 +1,15 @@ +from __future__ import unicode_literals +from moto.core.exceptions import JsonRESTError + + +class SecretsManagerClientError(JsonRESTError): + code = 400 + + +class ResourceNotFoundException(SecretsManagerClientError): + def __init__(self): + self.code = 404 + super(ResourceNotFoundException, self).__init__( + "ResourceNotFoundException", + "Secrets Manager can't find the specified secret" + ) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index fb09d20e4..a553953d4 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -6,14 +6,17 @@ import json import boto3 from moto.core import BaseBackend, BaseModel +from .exceptions import ResourceNotFoundException class SecretsManager(BaseModel): def __init__(self, region_name, **kwargs): + self.region = region_name self.secret_id = kwargs.get('secret_id', '') self.version_id = kwargs.get('version_id', '') self.version_stage = kwargs.get('version_stage', '') + self.secret_string = '' class SecretsManagerBackend(BaseBackend): @@ -22,15 +25,25 @@ class SecretsManagerBackend(BaseBackend): super(SecretsManagerBackend, self).__init__() self.region = region_name self.secret_id = kwargs.get('secret_id', '') + self.name = kwargs.get('name', '') self.createdate = int(time.time()) + self.secret_string = '' + + def reset(self): + region_name = self.region + self.__dict__ = {} + self.__init__(region_name) def get_secret_value(self, secret_id, version_id, version_stage): + if self.secret_id == '': + raise ResourceNotFoundException() + response = json.dumps({ - "ARN": self.secret_arn(), + "ARN": self.secret_arn(self.region, self.secret_id), "Name": self.secret_id, "VersionId": "A435958A-D821-4193-B719-B7769357AER4", - "SecretString": "mysecretstring", + "SecretString": self.secret_string, "VersionStages": [ "AWSCURRENT", ], @@ -39,11 +52,26 @@ class SecretsManagerBackend(BaseBackend): return response - def secret_arn(self): + def create_secret(self, name, secret_string, **kwargs): + + self.secret_string = secret_string + self.secret_id = name + + response = json.dumps({ + "ARN": self.secret_arn(self.region, name), + "Name": self.secret_id, + "VersionId": "A435958A-D821-4193-B719-B7769357AER4", + }) + + return response + + def secret_arn(self, region, secret_id): return "arn:aws:secretsmanager:{0}:1234567890:secret:{1}-rIjad".format( - self.region, self.secret_id) + region, secret_id) -available_regions = boto3.session.Session().get_available_regions("secretsmanager") -print(available_regions) -secretsmanager_backends = {region: SecretsManagerBackend(region_name=region) for region in available_regions} +available_regions = ( + boto3.session.Session().get_available_regions("secretsmanager") +) +secretsmanager_backends = {region: SecretsManagerBackend(region_name=region) + for region in available_regions} diff --git a/moto/secretsmanager/responses.py b/moto/secretsmanager/responses.py index 144a254ec..52a838732 100644 --- a/moto/secretsmanager/responses.py +++ b/moto/secretsmanager/responses.py @@ -15,3 +15,11 @@ class SecretsManagerResponse(BaseResponse): secret_id=secret_id, version_id=version_id, version_stage=version_stage) + + def create_secret(self): + name = self._get_param('Name') + secret_string = self._get_param('SecretString') + return secretsmanager_backends[self.region].create_secret( + name=name, + secret_string=secret_string + ) diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index df4f0f69e..d5abd6abd 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -3,11 +3,33 @@ from __future__ import unicode_literals import boto3 from moto import mock_secretsmanager +from botocore.exceptions import ClientError import sure # noqa +from nose.tools import assert_raises @mock_secretsmanager def test_get_secret_value(): conn = boto3.client('secretsmanager', region_name='us-west-2') + create_secret = conn.create_secret(Name='java-util-test-password', + SecretString="foosecret") result = conn.get_secret_value(SecretId='java-util-test-password') - assert result['SecretString'] == 'mysecretstring' + assert result['SecretString'] == 'foosecret' + +@mock_secretsmanager +def test_get_secret_that_does_not_exist(): + conn = boto3.client('secretsmanager', region_name='us-west-2') + + with assert_raises(ClientError): + result = conn.get_secret_value(SecretId='i-dont-exist') + +@mock_secretsmanager +def test_create_secret(): + conn = boto3.client('secretsmanager', region_name='us-east-1') + + result = conn.create_secret(Name='test-secret', SecretString="foosecret") + assert result['ARN'] == ( + 'arn:aws:secretsmanager:us-east-1:1234567890:secret:test-secret-rIjad') + assert result['Name'] == 'test-secret' + secret = conn.get_secret_value(SecretId='test-secret') + assert secret['SecretString'] == 'foosecret' diff --git a/tests/test_secretsmanager/test_server.py b/tests/test_secretsmanager/test_server.py index 142e9fe7d..2f73ece07 100644 --- a/tests/test_secretsmanager/test_server.py +++ b/tests/test_secretsmanager/test_server.py @@ -7,7 +7,7 @@ import moto.server as server from moto import mock_secretsmanager ''' -Test the different server responses +Test the different server responses for secretsmanager ''' @@ -17,11 +17,52 @@ def test_get_secret_value(): backend = server.create_backend_app("secretsmanager") test_client = backend.test_client() - res = test_client.post('/', - data={"SecretId": "test", "VersionStage": "AWSCURRENT"}, + create_secret = test_client.post('/', + data={"Name": "test-secret", + "SecretString": "foo-secret"}, + headers={ + "X-Amz-Target": "secretsmanager.CreateSecret"}, + ) + get_secret = test_client.post('/', + data={"SecretId": "test-secret", + "VersionStage": "AWSCURRENT"}, headers={ "X-Amz-Target": "secretsmanager.GetSecretValue"}, ) + json_data = json.loads(get_secret.data.decode("utf-8")) + assert json_data['SecretString'] == 'foo-secret' + +@mock_secretsmanager +def test_get_secret_that_does_not_exist(): + + backend = server.create_backend_app("secretsmanager") + test_client = backend.test_client() + + get_secret = test_client.post('/', + data={"SecretId": "i-dont-exist", + "VersionStage": "AWSCURRENT"}, + headers={ + "X-Amz-Target": "secretsmanager.GetSecretValue"}, + ) + json_data = json.loads(get_secret.data.decode("utf-8")) + assert json_data['message'] == "Secrets Manager can't find the specified secret" + assert json_data['__type'] == 'ResourceNotFoundException' + +@mock_secretsmanager +def test_create_secret(): + + backend = server.create_backend_app("secretsmanager") + test_client = backend.test_client() + + res = test_client.post('/', + data={"Name": "test-secret", + "SecretString": "foo-secret"}, + headers={ + "X-Amz-Target": "secretsmanager.CreateSecret"}, + ) + json_data = json.loads(res.data.decode("utf-8")) - assert json_data['SecretString'] == "mysecretstring" + assert json_data['ARN'] == ( + 'arn:aws:secretsmanager:us-east-1:1234567890:secret:test-secret-rIjad') + assert json_data['Name'] == 'test-secret' From ff80ecb56ddf4b0dfc080005d1759f8493144a97 Mon Sep 17 00:00:00 2001 From: Nathan Mische Date: Fri, 13 Jul 2018 10:41:22 -0400 Subject: [PATCH 20/42] Adding account id to ManagedPolicy ARN --- moto/iam/models.py | 12 ++++++++---- tests/test_iam/test_iam.py | 35 ++++++++++++++++++++++------------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/moto/iam/models.py b/moto/iam/models.py index 32ca144c3..8b632e555 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -50,10 +50,6 @@ class Policy(BaseModel): self.create_datetime = datetime.now(pytz.utc) self.update_datetime = datetime.now(pytz.utc) - @property - def arn(self): - return 'arn:aws:iam::aws:policy{0}{1}'.format(self.path, self.name) - class PolicyVersion(object): @@ -82,6 +78,10 @@ class ManagedPolicy(Policy): self.attachment_count -= 1 del obj.managed_policies[self.name] + @property + def arn(self): + return "arn:aws:iam::{0}:policy{1}{2}".format(ACCOUNT_ID, self.path, self.name) + class AWSManagedPolicy(ManagedPolicy): """AWS-managed policy.""" @@ -93,6 +93,10 @@ class AWSManagedPolicy(ManagedPolicy): path=data.get('Path'), document=data.get('Document')) + @property + def arn(self): + return 'arn:aws:iam::aws:policy{0}{1}'.format(self.path, self.name) + # AWS defines some of its own managed policies and we periodically # import them via `make aws_managed_policies` diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index b4dfe532d..182a60661 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -262,18 +262,27 @@ def test_update_assume_role_policy(): role.assume_role_policy_document.should.equal("my-policy") +@mock_iam +def test_create_policy(): + conn = boto3.client('iam', region_name='us-east-1') + response = conn.create_policy( + PolicyName="TestCreatePolicy", + PolicyDocument='{"some":"policy"}') + response['Policy']['Arn'].should.equal("arn:aws:iam::123456789012:policy/TestCreatePolicy") + + @mock_iam def test_create_policy_versions(): conn = boto3.client('iam', region_name='us-east-1') with assert_raises(ClientError): conn.create_policy_version( - PolicyArn="arn:aws:iam::aws:policy/TestCreatePolicyVersion", + PolicyArn="arn:aws:iam::123456789012:policy/TestCreatePolicyVersion", PolicyDocument='{"some":"policy"}') conn.create_policy( PolicyName="TestCreatePolicyVersion", PolicyDocument='{"some":"policy"}') version = conn.create_policy_version( - PolicyArn="arn:aws:iam::aws:policy/TestCreatePolicyVersion", + PolicyArn="arn:aws:iam::123456789012:policy/TestCreatePolicyVersion", PolicyDocument='{"some":"policy"}') version.get('PolicyVersion').get('Document').should.equal({'some': 'policy'}) @@ -285,14 +294,14 @@ def test_get_policy_version(): PolicyName="TestGetPolicyVersion", PolicyDocument='{"some":"policy"}') version = conn.create_policy_version( - PolicyArn="arn:aws:iam::aws:policy/TestGetPolicyVersion", + PolicyArn="arn:aws:iam::123456789012:policy/TestGetPolicyVersion", PolicyDocument='{"some":"policy"}') with assert_raises(ClientError): conn.get_policy_version( - PolicyArn="arn:aws:iam::aws:policy/TestGetPolicyVersion", + PolicyArn="arn:aws:iam::123456789012:policy/TestGetPolicyVersion", VersionId='v2-does-not-exist') retrieved = conn.get_policy_version( - PolicyArn="arn:aws:iam::aws:policy/TestGetPolicyVersion", + PolicyArn="arn:aws:iam::123456789012:policy/TestGetPolicyVersion", VersionId=version.get('PolicyVersion').get('VersionId')) retrieved.get('PolicyVersion').get('Document').should.equal({'some': 'policy'}) @@ -302,18 +311,18 @@ def test_list_policy_versions(): conn = boto3.client('iam', region_name='us-east-1') with assert_raises(ClientError): versions = conn.list_policy_versions( - PolicyArn="arn:aws:iam::aws:policy/TestListPolicyVersions") + PolicyArn="arn:aws:iam::123456789012:policy/TestListPolicyVersions") conn.create_policy( PolicyName="TestListPolicyVersions", PolicyDocument='{"some":"policy"}') conn.create_policy_version( - PolicyArn="arn:aws:iam::aws:policy/TestListPolicyVersions", + PolicyArn="arn:aws:iam::123456789012:policy/TestListPolicyVersions", PolicyDocument='{"first":"policy"}') conn.create_policy_version( - PolicyArn="arn:aws:iam::aws:policy/TestListPolicyVersions", + PolicyArn="arn:aws:iam::123456789012:policy/TestListPolicyVersions", PolicyDocument='{"second":"policy"}') versions = conn.list_policy_versions( - PolicyArn="arn:aws:iam::aws:policy/TestListPolicyVersions") + PolicyArn="arn:aws:iam::123456789012:policy/TestListPolicyVersions") versions.get('Versions')[0].get('Document').should.equal({'first': 'policy'}) versions.get('Versions')[1].get('Document').should.equal({'second': 'policy'}) @@ -325,17 +334,17 @@ def test_delete_policy_version(): PolicyName="TestDeletePolicyVersion", PolicyDocument='{"some":"policy"}') conn.create_policy_version( - PolicyArn="arn:aws:iam::aws:policy/TestDeletePolicyVersion", + PolicyArn="arn:aws:iam::123456789012:policy/TestDeletePolicyVersion", PolicyDocument='{"first":"policy"}') with assert_raises(ClientError): conn.delete_policy_version( - PolicyArn="arn:aws:iam::aws:policy/TestDeletePolicyVersion", + PolicyArn="arn:aws:iam::123456789012:policy/TestDeletePolicyVersion", VersionId='v2-nope-this-does-not-exist') conn.delete_policy_version( - PolicyArn="arn:aws:iam::aws:policy/TestDeletePolicyVersion", + PolicyArn="arn:aws:iam::123456789012:policy/TestDeletePolicyVersion", VersionId='v1') versions = conn.list_policy_versions( - PolicyArn="arn:aws:iam::aws:policy/TestDeletePolicyVersion") + PolicyArn="arn:aws:iam::123456789012:policy/TestDeletePolicyVersion") len(versions.get('Versions')).should.equal(0) From f50c6c2fb060024e4e6e3ffd91a316a659480963 Mon Sep 17 00:00:00 2001 From: Robert C Jensen Date: Tue, 17 Jul 2018 20:12:05 -0400 Subject: [PATCH 21/42] feature: add parameters back to Message models --- moto/ses/models.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/moto/ses/models.py b/moto/ses/models.py index b1135a406..3dced60f2 100644 --- a/moto/ses/models.py +++ b/moto/ses/models.py @@ -13,14 +13,21 @@ RECIPIENT_LIMIT = 50 class Message(BaseModel): - def __init__(self, message_id): + def __init__(self, message_id, source, subject, body, destinations): self.id = message_id + self.source = source + self.subject = subject + self.body = body + self.destinations = destinations class RawMessage(BaseModel): - def __init__(self, message_id): + def __init__(self, message_id, source, destinations, raw_data): self.id = message_id + self.source = source + self.destinations = destinations + self.raw_data = raw_data class SESQuota(BaseModel): @@ -79,7 +86,7 @@ class SESBackend(BaseBackend): ) message_id = get_random_message_id() - message = Message(message_id) + message = Message(message_id, source, subject, body, destinations) self.sent_messages.append(message) self.sent_message_count += recipient_count return message @@ -116,7 +123,7 @@ class SESBackend(BaseBackend): self.sent_message_count += recipient_count message_id = get_random_message_id() - message = RawMessage(message_id) + message = RawMessage(message_id, source, destinations, raw_data) self.sent_messages.append(message) return message From 6c7a22c7d704e4002c891fb1c87bce3631bd9d51 Mon Sep 17 00:00:00 2001 From: zane Date: Mon, 16 Jul 2018 12:39:59 -0700 Subject: [PATCH 22/42] Added get_random_password mock with tests --- moto/secretsmanager/exceptions.py | 14 +++ moto/secretsmanager/models.py | 42 ++++++- moto/secretsmanager/responses.py | 21 ++++ moto/secretsmanager/utils.py | 72 ++++++++++++ .../test_secretsmanager.py | 110 ++++++++++++++++++ 5 files changed, 253 insertions(+), 6 deletions(-) create mode 100644 moto/secretsmanager/utils.py diff --git a/moto/secretsmanager/exceptions.py b/moto/secretsmanager/exceptions.py index 99d74f281..a72a32645 100644 --- a/moto/secretsmanager/exceptions.py +++ b/moto/secretsmanager/exceptions.py @@ -13,3 +13,17 @@ class ResourceNotFoundException(SecretsManagerClientError): "ResourceNotFoundException", "Secrets Manager can't find the specified secret" ) + + +class ClientError(SecretsManagerClientError): + def __init__(self, message): + super(ClientError, self).__init__( + 'InvalidParameterValue', + message) + + +class InvalidParameterException(SecretsManagerClientError): + def __init__(self, message): + super(InvalidParameterException, self).__init__( + 'InvalidParameterException', + message) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index a553953d4..3923f90b0 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -6,7 +6,12 @@ import json import boto3 from moto.core import BaseBackend, BaseModel -from .exceptions import ResourceNotFoundException +from .exceptions import ( + ResourceNotFoundException, + InvalidParameterException, + ClientError +) +from .utils import random_password, secret_arn class SecretsManager(BaseModel): @@ -40,7 +45,7 @@ class SecretsManagerBackend(BaseBackend): raise ResourceNotFoundException() response = json.dumps({ - "ARN": self.secret_arn(self.region, self.secret_id), + "ARN": secret_arn(self.region, self.secret_id), "Name": self.secret_id, "VersionId": "A435958A-D821-4193-B719-B7769357AER4", "SecretString": self.secret_string, @@ -58,16 +63,41 @@ class SecretsManagerBackend(BaseBackend): self.secret_id = name response = json.dumps({ - "ARN": self.secret_arn(self.region, name), + "ARN": secret_arn(self.region, name), "Name": self.secret_id, "VersionId": "A435958A-D821-4193-B719-B7769357AER4", }) return response - def secret_arn(self, region, secret_id): - return "arn:aws:secretsmanager:{0}:1234567890:secret:{1}-rIjad".format( - region, secret_id) + def get_random_password(self, password_length, + exclude_characters, exclude_numbers, + exclude_punctuation, exclude_uppercase, + exclude_lowercase, include_space, + require_each_included_type): + # password size must have value less than or equal to 4096 + if password_length > 4096: + raise ClientError( + "ClientError: An error occurred (ValidationException) \ + when calling the GetRandomPassword operation: 1 validation error detected: Value '{}' at 'passwordLength' \ + failed to satisfy constraint: Member must have value less than or equal to 4096".format(password_length)) + if password_length < 4: + raise InvalidParameterException( + "InvalidParameterException: An error occurred (InvalidParameterException) \ + when calling the GetRandomPassword operation: Password length is too short based on the required types.") + + response = json.dumps({ + "RandomPassword": random_password(password_length, + exclude_characters, + exclude_numbers, + exclude_punctuation, + exclude_uppercase, + exclude_lowercase, + include_space, + require_each_included_type) + }) + + return response available_regions = ( diff --git a/moto/secretsmanager/responses.py b/moto/secretsmanager/responses.py index 52a838732..06387560a 100644 --- a/moto/secretsmanager/responses.py +++ b/moto/secretsmanager/responses.py @@ -23,3 +23,24 @@ class SecretsManagerResponse(BaseResponse): name=name, secret_string=secret_string ) + + def get_random_password(self): + password_length = self._get_param('PasswordLength', if_none=32) + exclude_characters = self._get_param('ExcludeCharacters', if_none='') + exclude_numbers = self._get_param('ExcludeNumbers', if_none=False) + exclude_punctuation = self._get_param('ExcludePunctuation', if_none=False) + exclude_uppercase = self._get_param('ExcludeUppercase', if_none=False) + exclude_lowercase = self._get_param('ExcludeLowercase', if_none=False) + include_space = self._get_param('IncludeSpace', if_none=False) + require_each_included_type = self._get_param( + 'RequireEachIncludedType', if_none=True) + return secretsmanager_backends[self.region].get_random_password( + password_length=password_length, + exclude_characters=exclude_characters, + exclude_numbers=exclude_numbers, + exclude_punctuation=exclude_punctuation, + exclude_uppercase=exclude_uppercase, + exclude_lowercase=exclude_lowercase, + include_space=include_space, + require_each_included_type=require_each_included_type + ) diff --git a/moto/secretsmanager/utils.py b/moto/secretsmanager/utils.py new file mode 100644 index 000000000..2cb92020a --- /dev/null +++ b/moto/secretsmanager/utils.py @@ -0,0 +1,72 @@ +from __future__ import unicode_literals + +import random +import string +import six +import re + + +def random_password(password_length, exclude_characters, exclude_numbers, + exclude_punctuation, exclude_uppercase, exclude_lowercase, + include_space, require_each_included_type): + + password = '' + required_characters = '' + + if not exclude_lowercase and not exclude_uppercase: + password += string.ascii_letters + required_characters += random.choice(_exclude_characters( + string.ascii_lowercase, exclude_characters)) + required_characters += random.choice(_exclude_characters( + string.ascii_uppercase, exclude_characters)) + elif not exclude_lowercase: + password += string.ascii_lowercase + required_characters += random.choice(_exclude_characters( + string.ascii_lowercase, exclude_characters)) + elif not exclude_uppercase: + password += string.ascii_uppercase + required_characters += random.choice(_exclude_characters( + string.ascii_uppercase, exclude_characters)) + if not exclude_numbers: + password += string.digits + required_characters += random.choice(_exclude_characters( + string.digits, exclude_characters)) + if not exclude_punctuation: + password += string.punctuation + required_characters += random.choice(_exclude_characters( + string.punctuation, exclude_characters)) + if include_space: + password += " " + required_characters += " " + + password = ''.join( + six.text_type(random.choice(password)) + for x in range(password_length)) + + if require_each_included_type: + password = _add_password_require_each_included_type( + password, required_characters) + + password = _exclude_characters(password, exclude_characters) + return password + + +def secret_arn(region, secret_id): + return "arn:aws:secretsmanager:{0}:1234567890:secret:{1}-rIjad".format( + region, secret_id) + + +def _exclude_characters(password, exclude_characters): + for c in exclude_characters: + if c in string.punctuation: + # Escape punctuation regex usage + c = "\{0}".format(c) + password = re.sub(c, '', str(password)) + return password + + +def _add_password_require_each_included_type(password, required_characters): + password_with_required_char = password[:-len(required_characters)] + password_with_required_char += required_characters + + return password_with_required_char diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index d5abd6abd..6fefeb56f 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -5,6 +5,8 @@ import boto3 from moto import mock_secretsmanager from botocore.exceptions import ClientError import sure # noqa +import string +import unittest from nose.tools import assert_raises @mock_secretsmanager @@ -33,3 +35,111 @@ def test_create_secret(): assert result['Name'] == 'test-secret' secret = conn.get_secret_value(SecretId='test-secret') assert secret['SecretString'] == 'foosecret' + +@mock_secretsmanager +def test_get_random_password_default_length(): + conn = boto3.client('secretsmanager', region_name='us-west-2') + + random_password = conn.get_random_password() + assert len(random_password['RandomPassword']) == 32 + +@mock_secretsmanager +def test_get_random_password_default_requirements(): + # When require_each_included_type, default true + conn = boto3.client('secretsmanager', region_name='us-west-2') + + random_password = conn.get_random_password() + # Should contain lowercase, upppercase, digit, special character + assert any(c.islower() for c in random_password['RandomPassword']) + assert any(c.isupper() for c in random_password['RandomPassword']) + assert any(c.isdigit() for c in random_password['RandomPassword']) + assert any(c in string.punctuation + for c in random_password['RandomPassword']) + +@mock_secretsmanager +def test_get_random_password_custom_length(): + conn = boto3.client('secretsmanager', region_name='us-west-2') + + random_password = conn.get_random_password(PasswordLength=50) + assert len(random_password['RandomPassword']) == 50 + +@mock_secretsmanager +def test_get_random_exclude_lowercase(): + conn = boto3.client('secretsmanager', region_name='us-west-2') + + random_password = conn.get_random_password(PasswordLength=55, + ExcludeLowercase=True) + assert any(c.islower() for c in random_password['RandomPassword']) == False + +@mock_secretsmanager +def test_get_random_exclude_uppercase(): + conn = boto3.client('secretsmanager', region_name='us-west-2') + + random_password = conn.get_random_password(PasswordLength=55, + ExcludeUppercase=True) + assert any(c.isupper() for c in random_password['RandomPassword']) == False + +@mock_secretsmanager +def test_get_random_exclude_characters_and_symbols(): + conn = boto3.client('secretsmanager', region_name='us-west-2') + + random_password = conn.get_random_password(PasswordLength=20, + ExcludeCharacters='xyzDje@?!.') + assert any(c in 'xyzDje@?!.' for c in random_password['RandomPassword']) == False + +@mock_secretsmanager +def test_get_random_exclude_numbers(): + conn = boto3.client('secretsmanager', region_name='us-west-2') + + random_password = conn.get_random_password(PasswordLength=100, + ExcludeNumbers=True) + assert any(c.isdigit() for c in random_password['RandomPassword']) == False + +@mock_secretsmanager +def test_get_random_exclude_punctuation(): + conn = boto3.client('secretsmanager', region_name='us-west-2') + + random_password = conn.get_random_password(PasswordLength=100, + ExcludePunctuation=True) + assert any(c in string.punctuation + for c in random_password['RandomPassword']) == False + +@mock_secretsmanager +def test_get_random_include_space_false(): + conn = boto3.client('secretsmanager', region_name='us-west-2') + + random_password = conn.get_random_password(PasswordLength=300) + assert any(c.isspace() for c in random_password['RandomPassword']) == False + +@mock_secretsmanager +def test_get_random_include_space_true(): + conn = boto3.client('secretsmanager', region_name='us-west-2') + + random_password = conn.get_random_password(PasswordLength=4, + IncludeSpace=True) + assert any(c.isspace() for c in random_password['RandomPassword']) == True + +@mock_secretsmanager +def test_get_random_require_each_included_type(): + conn = boto3.client('secretsmanager', region_name='us-west-2') + + random_password = conn.get_random_password(PasswordLength=4, + RequireEachIncludedType=True) + assert any(c in string.punctuation for c in random_password['RandomPassword']) == True + assert any(c in string.ascii_lowercase for c in random_password['RandomPassword']) == True + assert any(c in string.ascii_uppercase for c in random_password['RandomPassword']) == True + assert any(c in string.digits for c in random_password['RandomPassword']) == True + +@mock_secretsmanager +def test_get_random_too_short_password(): + conn = boto3.client('secretsmanager', region_name='us-west-2') + + with assert_raises(ClientError): + random_password = conn.get_random_password(PasswordLength=3) + +@mock_secretsmanager +def test_get_random_too_long_password(): + conn = boto3.client('secretsmanager', region_name='us-west-2') + + with assert_raises(Exception): + random_password = conn.get_random_password(PasswordLength=5555) From 2e5e7e7f5ea7347da632eb7dcab3c588fec25a9e Mon Sep 17 00:00:00 2001 From: Gary Donovan Date: Wed, 25 Jul 2018 08:11:04 +1000 Subject: [PATCH 23/42] Fix typo in test name (#1729) --- tests/test_ec2/test_internet_gateways.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ec2/test_internet_gateways.py b/tests/test_ec2/test_internet_gateways.py index 5842621cd..3a1d0fda9 100644 --- a/tests/test_ec2/test_internet_gateways.py +++ b/tests/test_ec2/test_internet_gateways.py @@ -199,7 +199,7 @@ def test_igw_desribe(): @mock_ec2_deprecated -def test_igw_desribe_bad_id(): +def test_igw_describe_bad_id(): """ internet gateway fail to fetch by bad id """ conn = boto.connect_vpc('the_key', 'the_secret') with assert_raises(EC2ResponseError) as cm: From 77f0a61c9f89202ae09bb7c5ed1e325139ffc516 Mon Sep 17 00:00:00 2001 From: TheDooner64 Date: Tue, 10 Jul 2018 13:50:47 -0400 Subject: [PATCH 24/42] Add scaffolding for Glue service, including create_database and get_database for the Glue Data Catalog --- moto/__init__.py | 1 + moto/glue/__init__.py | 5 +++++ moto/glue/exceptions.py | 9 ++++++++ moto/glue/models.py | 27 ++++++++++++++++++++++++ moto/glue/responses.py | 27 ++++++++++++++++++++++++ moto/glue/urls.py | 11 ++++++++++ moto/glue/utils.py | 1 + tests/test_glue/test_datacatalog.py | 30 +++++++++++++++++++++++++++ tests/test_s3/test_s3_storageclass.py | 3 --- tests/test_s3/test_s3_utils.py | 1 - 10 files changed, 111 insertions(+), 4 deletions(-) create mode 100644 moto/glue/__init__.py create mode 100644 moto/glue/exceptions.py create mode 100644 moto/glue/models.py create mode 100644 moto/glue/responses.py create mode 100644 moto/glue/urls.py create mode 100644 moto/glue/utils.py create mode 100644 tests/test_glue/test_datacatalog.py diff --git a/moto/__init__.py b/moto/__init__.py index 0ce5e54d1..e5881cfca 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -24,6 +24,7 @@ from .elbv2 import mock_elbv2 # flake8: noqa from .emr import mock_emr, mock_emr_deprecated # flake8: noqa from .events import mock_events # flake8: noqa from .glacier import mock_glacier, mock_glacier_deprecated # flake8: noqa +from .glue import mock_glue # flake8: noqa from .iam import mock_iam, mock_iam_deprecated # flake8: noqa from .kinesis import mock_kinesis, mock_kinesis_deprecated # flake8: noqa from .kms import mock_kms, mock_kms_deprecated # flake8: noqa diff --git a/moto/glue/__init__.py b/moto/glue/__init__.py new file mode 100644 index 000000000..6b1f13326 --- /dev/null +++ b/moto/glue/__init__.py @@ -0,0 +1,5 @@ +from __future__ import unicode_literals +from .models import glue_backend + +glue_backends = {"global": glue_backend} +mock_glue = glue_backend.decorator diff --git a/moto/glue/exceptions.py b/moto/glue/exceptions.py new file mode 100644 index 000000000..0c8760f18 --- /dev/null +++ b/moto/glue/exceptions.py @@ -0,0 +1,9 @@ +from __future__ import unicode_literals +from moto.core.exceptions import RESTError + + +class GlueClientError(RESTError): + + def __init__(self, *args, **kwargs): + kwargs.setdefault('template', 'single_error') + super(GlueClientError, self).__init__(*args, **kwargs) diff --git a/moto/glue/models.py b/moto/glue/models.py new file mode 100644 index 000000000..55cd46bcb --- /dev/null +++ b/moto/glue/models.py @@ -0,0 +1,27 @@ +from __future__ import unicode_literals + +from moto.core import BaseBackend, BaseModel +from moto.compat import OrderedDict + + +class GlueBackend(BaseBackend): + + def __init__(self): + self.databases = OrderedDict() + + def create_database(self, database_name): + database = FakeDatabase(database_name) + self.databases[database_name] = database + return database + + def get_database(self, database_name): + return self.databases[database_name] + + +class FakeDatabase(BaseModel): + + def __init__(self, database_name): + self.name = database_name + + +glue_backend = GlueBackend() diff --git a/moto/glue/responses.py b/moto/glue/responses.py new file mode 100644 index 000000000..f3ef6eb4d --- /dev/null +++ b/moto/glue/responses.py @@ -0,0 +1,27 @@ +from __future__ import unicode_literals + +import json + +from moto.core.responses import BaseResponse +from .models import glue_backend + + +class GlueResponse(BaseResponse): + + @property + def glue_backend(self): + return glue_backend + + @property + def parameters(self): + return json.loads(self.body) + + def create_database(self): + database_name = self.parameters['DatabaseInput']['Name'] + self.glue_backend.create_database(database_name) + return "" + + def get_database(self): + database_name = self.parameters.get('Name') + database = self.glue_backend.get_database(database_name) + return json.dumps({'Database': {'Name': database.name}}) diff --git a/moto/glue/urls.py b/moto/glue/urls.py new file mode 100644 index 000000000..f3eaa9cad --- /dev/null +++ b/moto/glue/urls.py @@ -0,0 +1,11 @@ +from __future__ import unicode_literals + +from .responses import GlueResponse + +url_bases = [ + "https?://glue(.*).amazonaws.com" +] + +url_paths = { + '{0}/$': GlueResponse.dispatch +} diff --git a/moto/glue/utils.py b/moto/glue/utils.py new file mode 100644 index 000000000..baffc4882 --- /dev/null +++ b/moto/glue/utils.py @@ -0,0 +1 @@ +from __future__ import unicode_literals diff --git a/tests/test_glue/test_datacatalog.py b/tests/test_glue/test_datacatalog.py new file mode 100644 index 000000000..77ad1c013 --- /dev/null +++ b/tests/test_glue/test_datacatalog.py @@ -0,0 +1,30 @@ +from __future__ import unicode_literals + +import sure # noqa +import boto3 + +from moto import mock_glue + + +def create_database(client, database_name): + return client.create_database( + DatabaseInput={ + 'Name': database_name + } + ) + + +def get_database(client, database_name): + return client.get_database(Name=database_name) + + +@mock_glue +def test_create_database(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + create_database(client, database_name) + + response = get_database(client, database_name) + database = response['Database'] + + database.should.equal({'Name': database_name}) diff --git a/tests/test_s3/test_s3_storageclass.py b/tests/test_s3/test_s3_storageclass.py index c4c83a285..2ed966022 100644 --- a/tests/test_s3/test_s3_storageclass.py +++ b/tests/test_s3/test_s3_storageclass.py @@ -101,6 +101,3 @@ def test_s3_default_storage_class(): # tests that the default storage class is still STANDARD list_of_objects["Contents"][0]["StorageClass"].should.equal("STANDARD") - - - diff --git a/tests/test_s3/test_s3_utils.py b/tests/test_s3/test_s3_utils.py index 9cda1f157..d874a0f1e 100644 --- a/tests/test_s3/test_s3_utils.py +++ b/tests/test_s3/test_s3_utils.py @@ -21,7 +21,6 @@ def test_force_ignore_subdomain_for_bucketnames(): os.environ['S3_IGNORE_SUBDOMAIN_BUCKETNAME'] = '1' expect(bucket_name_from_url('https://subdomain.localhost:5000/abc/resource')).should.equal(None) del(os.environ['S3_IGNORE_SUBDOMAIN_BUCKETNAME']) - def test_versioned_key_store(): From e67a8c6f1bc9079d6aa61da5bca54cf6533b7e4d Mon Sep 17 00:00:00 2001 From: TheDooner64 Date: Tue, 10 Jul 2018 13:52:53 -0400 Subject: [PATCH 25/42] Revert minor changes to s3 tests --- tests/test_s3/test_s3_storageclass.py | 3 +++ tests/test_s3/test_s3_utils.py | 1 + 2 files changed, 4 insertions(+) diff --git a/tests/test_s3/test_s3_storageclass.py b/tests/test_s3/test_s3_storageclass.py index 2ed966022..99908c501 100644 --- a/tests/test_s3/test_s3_storageclass.py +++ b/tests/test_s3/test_s3_storageclass.py @@ -101,3 +101,6 @@ def test_s3_default_storage_class(): # tests that the default storage class is still STANDARD list_of_objects["Contents"][0]["StorageClass"].should.equal("STANDARD") + + + diff --git a/tests/test_s3/test_s3_utils.py b/tests/test_s3/test_s3_utils.py index d874a0f1e..ce9f54c75 100644 --- a/tests/test_s3/test_s3_utils.py +++ b/tests/test_s3/test_s3_utils.py @@ -23,6 +23,7 @@ def test_force_ignore_subdomain_for_bucketnames(): del(os.environ['S3_IGNORE_SUBDOMAIN_BUCKETNAME']) + def test_versioned_key_store(): d = _VersionedKeyStore() From c5c57efbb5d08828dea08822c90b292d0d183cdb Mon Sep 17 00:00:00 2001 From: TheDooner64 Date: Wed, 11 Jul 2018 11:39:40 -0400 Subject: [PATCH 26/42] Creating a database that already exists in the glue data catalog raises an exception --- moto/glue/exceptions.py | 16 +++++++++++----- moto/glue/models.py | 4 ++++ tests/test_glue/test_datacatalog.py | 14 ++++++++++++++ 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/moto/glue/exceptions.py b/moto/glue/exceptions.py index 0c8760f18..2f9d16d26 100644 --- a/moto/glue/exceptions.py +++ b/moto/glue/exceptions.py @@ -1,9 +1,15 @@ from __future__ import unicode_literals -from moto.core.exceptions import RESTError +from moto.core.exceptions import JsonRESTError -class GlueClientError(RESTError): +class GlueClientError(JsonRESTError): + code = 400 - def __init__(self, *args, **kwargs): - kwargs.setdefault('template', 'single_error') - super(GlueClientError, self).__init__(*args, **kwargs) + +class DatabaseAlreadyExistsException(GlueClientError): + def __init__(self): + self.code = 400 + super(DatabaseAlreadyExistsException, self).__init__( + 'DatabaseAlreadyExistsException', + 'Database already exists.' + ) diff --git a/moto/glue/models.py b/moto/glue/models.py index 55cd46bcb..357a2a52d 100644 --- a/moto/glue/models.py +++ b/moto/glue/models.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from moto.core import BaseBackend, BaseModel from moto.compat import OrderedDict +from.exceptions import DatabaseAlreadyExistsException class GlueBackend(BaseBackend): @@ -10,6 +11,9 @@ class GlueBackend(BaseBackend): self.databases = OrderedDict() def create_database(self, database_name): + if database_name in self.databases: + raise DatabaseAlreadyExistsException() + database = FakeDatabase(database_name) self.databases[database_name] = database return database diff --git a/tests/test_glue/test_datacatalog.py b/tests/test_glue/test_datacatalog.py index 77ad1c013..c7cdb1a7c 100644 --- a/tests/test_glue/test_datacatalog.py +++ b/tests/test_glue/test_datacatalog.py @@ -1,7 +1,9 @@ from __future__ import unicode_literals import sure # noqa +from nose.tools import assert_raises import boto3 +from botocore.client import ClientError from moto import mock_glue @@ -28,3 +30,15 @@ def test_create_database(): database = response['Database'] database.should.equal({'Name': database_name}) + + +@mock_glue +def test_create_database_already_exists(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'anewdatabase' + create_database(client, database_name) + + with assert_raises(ClientError) as exc: + create_database(client, database_name) + + exc.exception.response['Error']['Code'].should.equal('DatabaseAlreadyExistsException') From d988ee15fe9d587c8cf0b62e1eaf2c277b7a58d3 Mon Sep 17 00:00:00 2001 From: TheDooner64 Date: Thu, 26 Jul 2018 17:05:09 -0400 Subject: [PATCH 27/42] Add create_table, get_table, and get_tables for the Glue Data Catalog --- moto/glue/exceptions.py | 9 +++ moto/glue/models.py | 31 +++++++- moto/glue/responses.py | 36 +++++++++ tests/test_glue/__init__.py | 1 + tests/test_glue/fixtures/__init__.py | 1 + tests/test_glue/fixtures/datacatalog.py | 31 ++++++++ tests/test_glue/helpers.py | 46 ++++++++++++ tests/test_glue/test_datacatalog.py | 98 ++++++++++++++++++++----- 8 files changed, 235 insertions(+), 18 deletions(-) create mode 100644 tests/test_glue/__init__.py create mode 100644 tests/test_glue/fixtures/__init__.py create mode 100644 tests/test_glue/fixtures/datacatalog.py create mode 100644 tests/test_glue/helpers.py diff --git a/moto/glue/exceptions.py b/moto/glue/exceptions.py index 2f9d16d26..62ea1525c 100644 --- a/moto/glue/exceptions.py +++ b/moto/glue/exceptions.py @@ -13,3 +13,12 @@ class DatabaseAlreadyExistsException(GlueClientError): 'DatabaseAlreadyExistsException', 'Database already exists.' ) + + +class TableAlreadyExistsException(GlueClientError): + def __init__(self): + self.code = 400 + super(TableAlreadyExistsException, self).__init__( + 'TableAlreadyExistsException', + 'Table already exists.' + ) diff --git a/moto/glue/models.py b/moto/glue/models.py index 357a2a52d..9f7e7657d 100644 --- a/moto/glue/models.py +++ b/moto/glue/models.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from moto.core import BaseBackend, BaseModel from moto.compat import OrderedDict -from.exceptions import DatabaseAlreadyExistsException +from.exceptions import DatabaseAlreadyExistsException, TableAlreadyExistsException class GlueBackend(BaseBackend): @@ -21,11 +21,40 @@ class GlueBackend(BaseBackend): def get_database(self, database_name): return self.databases[database_name] + def create_table(self, database_name, table_name, table_input): + database = self.get_database(database_name) + + if table_name in database.tables: + raise TableAlreadyExistsException() + + table = FakeTable(database_name, table_name, table_input) + database.tables[table_name] = table + return table + + def get_table(self, database_name, table_name): + database = self.get_database(database_name) + return database.tables[table_name] + + def get_tables(self, database_name): + database = self.get_database(database_name) + return [table for table_name, table in database.tables.iteritems()] + class FakeDatabase(BaseModel): def __init__(self, database_name): self.name = database_name + self.tables = OrderedDict() + + +class FakeTable(BaseModel): + + def __init__(self, database_name, table_name, table_input): + self.database_name = database_name + self.name = table_name + self.table_input = table_input + self.storage_descriptor = self.table_input.get('StorageDescriptor', {}) + self.partition_keys = self.table_input.get('PartitionKeys', []) glue_backend = GlueBackend() diff --git a/moto/glue/responses.py b/moto/glue/responses.py index f3ef6eb4d..bb64c40d4 100644 --- a/moto/glue/responses.py +++ b/moto/glue/responses.py @@ -25,3 +25,39 @@ class GlueResponse(BaseResponse): database_name = self.parameters.get('Name') database = self.glue_backend.get_database(database_name) return json.dumps({'Database': {'Name': database.name}}) + + def create_table(self): + database_name = self.parameters.get('DatabaseName') + table_input = self.parameters.get('TableInput') + table_name = table_input.get('Name') + self.glue_backend.create_table(database_name, table_name, table_input) + return "" + + def get_table(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('Name') + table = self.glue_backend.get_table(database_name, table_name) + return json.dumps({ + 'Table': { + 'DatabaseName': table.database_name, + 'Name': table.name, + 'PartitionKeys': table.partition_keys, + 'StorageDescriptor': table.storage_descriptor + } + }) + + def get_tables(self): + database_name = self.parameters.get('DatabaseName') + tables = self.glue_backend.get_tables(database_name) + return json.dumps( + { + 'TableList': [ + { + 'DatabaseName': table.database_name, + 'Name': table.name, + 'PartitionKeys': table.partition_keys, + 'StorageDescriptor': table.storage_descriptor + } for table in tables + ] + } + ) diff --git a/tests/test_glue/__init__.py b/tests/test_glue/__init__.py new file mode 100644 index 000000000..baffc4882 --- /dev/null +++ b/tests/test_glue/__init__.py @@ -0,0 +1 @@ +from __future__ import unicode_literals diff --git a/tests/test_glue/fixtures/__init__.py b/tests/test_glue/fixtures/__init__.py new file mode 100644 index 000000000..baffc4882 --- /dev/null +++ b/tests/test_glue/fixtures/__init__.py @@ -0,0 +1 @@ +from __future__ import unicode_literals diff --git a/tests/test_glue/fixtures/datacatalog.py b/tests/test_glue/fixtures/datacatalog.py new file mode 100644 index 000000000..b2efe4154 --- /dev/null +++ b/tests/test_glue/fixtures/datacatalog.py @@ -0,0 +1,31 @@ +from __future__ import unicode_literals + +TABLE_INPUT = { + 'Owner': 'a_fake_owner', + 'Parameters': { + 'EXTERNAL': 'TRUE', + }, + 'Retention': 0, + 'StorageDescriptor': { + 'BucketColumns': [], + 'Compressed': False, + 'InputFormat': 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat', + 'NumberOfBuckets': -1, + 'OutputFormat': 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat', + 'Parameters': {}, + 'SerdeInfo': { + 'Parameters': { + 'serialization.format': '1' + }, + 'SerializationLibrary': 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' + }, + 'SkewedInfo': { + 'SkewedColumnNames': [], + 'SkewedColumnValueLocationMaps': {}, + 'SkewedColumnValues': [] + }, + 'SortColumns': [], + 'StoredAsSubDirectories': False + }, + 'TableType': 'EXTERNAL_TABLE', +} diff --git a/tests/test_glue/helpers.py b/tests/test_glue/helpers.py new file mode 100644 index 000000000..4a51f9117 --- /dev/null +++ b/tests/test_glue/helpers.py @@ -0,0 +1,46 @@ +from __future__ import unicode_literals + +import copy + +from .fixtures.datacatalog import TABLE_INPUT + + +def create_database(client, database_name): + return client.create_database( + DatabaseInput={ + 'Name': database_name + } + ) + + +def get_database(client, database_name): + return client.get_database(Name=database_name) + + +def create_table_input(table_name, s3_location, columns=[], partition_keys=[]): + table_input = copy.deepcopy(TABLE_INPUT) + table_input['Name'] = table_name + table_input['PartitionKeys'] = partition_keys + table_input['StorageDescriptor']['Columns'] = columns + table_input['StorageDescriptor']['Location'] = s3_location + return table_input + + +def create_table(client, database_name, table_name, table_input): + return client.create_table( + DatabaseName=database_name, + TableInput=table_input + ) + + +def get_table(client, database_name, table_name): + return client.get_table( + DatabaseName=database_name, + Name=table_name + ) + + +def get_tables(client, database_name): + return client.get_tables( + DatabaseName=database_name + ) diff --git a/tests/test_glue/test_datacatalog.py b/tests/test_glue/test_datacatalog.py index c7cdb1a7c..7dabeb1f3 100644 --- a/tests/test_glue/test_datacatalog.py +++ b/tests/test_glue/test_datacatalog.py @@ -6,27 +6,16 @@ import boto3 from botocore.client import ClientError from moto import mock_glue - - -def create_database(client, database_name): - return client.create_database( - DatabaseInput={ - 'Name': database_name - } - ) - - -def get_database(client, database_name): - return client.get_database(Name=database_name) +from . import helpers @mock_glue def test_create_database(): client = boto3.client('glue', region_name='us-east-1') database_name = 'myspecialdatabase' - create_database(client, database_name) + helpers.create_database(client, database_name) - response = get_database(client, database_name) + response = helpers.get_database(client, database_name) database = response['Database'] database.should.equal({'Name': database_name}) @@ -35,10 +24,85 @@ def test_create_database(): @mock_glue def test_create_database_already_exists(): client = boto3.client('glue', region_name='us-east-1') - database_name = 'anewdatabase' - create_database(client, database_name) + database_name = 'cantcreatethisdatabasetwice' + helpers.create_database(client, database_name) with assert_raises(ClientError) as exc: - create_database(client, database_name) + helpers.create_database(client, database_name) exc.exception.response['Error']['Code'].should.equal('DatabaseAlreadyExistsException') + + +@mock_glue +def test_create_table(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + helpers.create_database(client, database_name) + + table_name = 'myspecialtable' + s3_location = 's3://my-bucket/{database_name}/{table_name}'.format( + database_name=database_name, + table_name=table_name + ) + + table_input = helpers.create_table_input(table_name, s3_location) + helpers.create_table(client, database_name, table_name, table_input) + + response = helpers.get_table(client, database_name, table_name) + table = response['Table'] + + table['Name'].should.equal(table_input['Name']) + table['StorageDescriptor'].should.equal(table_input['StorageDescriptor']) + table['PartitionKeys'].should.equal(table_input['PartitionKeys']) + + +@mock_glue +def test_create_table_already_exists(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + helpers.create_database(client, database_name) + + table_name = 'cantcreatethistabletwice' + s3_location = 's3://my-bucket/{database_name}/{table_name}'.format( + database_name=database_name, + table_name=table_name + ) + + table_input = helpers.create_table_input(table_name, s3_location) + helpers.create_table(client, database_name, table_name, table_input) + + with assert_raises(ClientError) as exc: + helpers.create_table(client, database_name, table_name, table_input) + + exc.exception.response['Error']['Code'].should.equal('TableAlreadyExistsException') + + +@mock_glue +def test_get_tables(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + helpers.create_database(client, database_name) + + table_names = ['myfirsttable', 'mysecondtable', 'mythirdtable'] + table_inputs = {} + + for table_name in table_names: + s3_location = 's3://my-bucket/{database_name}/{table_name}'.format( + database_name=database_name, + table_name=table_name + ) + table_input = helpers.create_table_input(table_name, s3_location) + table_inputs[table_name] = table_input + helpers.create_table(client, database_name, table_name, table_input) + + response = helpers.get_tables(client, database_name) + + tables = response['TableList'] + + assert len(tables) == 3 + + for table in tables: + table_name = table['Name'] + table_name.should.equal(table_inputs[table_name]['Name']) + table['StorageDescriptor'].should.equal(table_inputs[table_name]['StorageDescriptor']) + table['PartitionKeys'].should.equal(table_inputs[table_name]['PartitionKeys']) From 9339a476d2b2c7d23d254c11fa474f6e69426611 Mon Sep 17 00:00:00 2001 From: TheDooner64 Date: Sun, 5 Aug 2018 19:46:40 -0400 Subject: [PATCH 28/42] Adjust glue get_tables method to use items instead of iteritems --- moto/glue/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/glue/models.py b/moto/glue/models.py index 9f7e7657d..09b7d60ed 100644 --- a/moto/glue/models.py +++ b/moto/glue/models.py @@ -37,7 +37,7 @@ class GlueBackend(BaseBackend): def get_tables(self, database_name): database = self.get_database(database_name) - return [table for table_name, table in database.tables.iteritems()] + return [table for table_name, table in database.tables.items()] class FakeDatabase(BaseModel): From cce3a678aa382ef29d06eefc10d4ddd01ec25f18 Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Mon, 6 Aug 2018 14:40:33 -0700 Subject: [PATCH 29/42] Implement secretsmanager.DescribeSecret and tests. --- moto/secretsmanager/models.py | 31 +++++++++++++ moto/secretsmanager/responses.py | 6 +++ .../test_secretsmanager.py | 18 ++++++++ tests/test_secretsmanager/test_server.py | 43 +++++++++++++++++++ 4 files changed, 98 insertions(+) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 3923f90b0..da3f3e6fe 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -33,6 +33,9 @@ class SecretsManagerBackend(BaseBackend): self.name = kwargs.get('name', '') self.createdate = int(time.time()) self.secret_string = '' + self.rotation_enabled = False + self.rotation_lambda_arn = '' + self.auto_rotate_after_days = 1 def reset(self): region_name = self.region @@ -70,6 +73,34 @@ class SecretsManagerBackend(BaseBackend): return response + def describe_secret(self, secret_id): + if self.secret_id == '': + raise ResourceNotFoundException + + response = json.dumps({ + "ARN": secret_arn(self.region, self.secret_id), + "Name": self.secret_id, + "Description": "", + "KmsKeyId": "", + "RotationEnabled": self.rotation_enabled, + "RotationLambdaARN": self.rotation_lambda_arn, + "RotationRules": { + "AutomaticallyAfterDays": self.auto_rotate_after_days + }, + "LastRotatedDate": None, + "LastChangedDate": None, + "LastAccessedDate": None, + "DeletedDate": None, + "Tags": [ + { + "Key": "", + "Value": "" + }, + ] + }) + + return response + def get_random_password(self, password_length, exclude_characters, exclude_numbers, exclude_punctuation, exclude_uppercase, diff --git a/moto/secretsmanager/responses.py b/moto/secretsmanager/responses.py index 06387560a..c50c6a6e1 100644 --- a/moto/secretsmanager/responses.py +++ b/moto/secretsmanager/responses.py @@ -44,3 +44,9 @@ class SecretsManagerResponse(BaseResponse): include_space=include_space, require_each_included_type=require_each_included_type ) + + def describe_secret(self): + secret_id = self._get_param('SecretId') + return secretsmanager_backends[self.region].describe_secret( + secret_id=secret_id + ) diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index 6fefeb56f..0ef54b45b 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -143,3 +143,21 @@ def test_get_random_too_long_password(): with assert_raises(Exception): random_password = conn.get_random_password(PasswordLength=5555) + +@mock_secretsmanager +def test_describe_secret(): + conn = boto3.client('secretsmanager', region_name='us-west-2') + conn.create_secret(Name='test-secret', + SecretString='foosecret') + + secret_description = conn.describe_secret(SecretId='test-secret') + assert secret_description # Returned dict is not empty + assert secret_description['ARN'] == ( + 'arn:aws:secretsmanager:us-west-2:1234567890:secret:test-secret-rIjad') + +@mock_secretsmanager +def test_describe_secret_that_does_not_exist(): + conn = boto3.client('secretsmanager', region_name='us-west-2') + + with assert_raises(ClientError): + result = conn.get_secret_value(SecretId='i-dont-exist') diff --git a/tests/test_secretsmanager/test_server.py b/tests/test_secretsmanager/test_server.py index 2f73ece07..370a483a8 100644 --- a/tests/test_secretsmanager/test_server.py +++ b/tests/test_secretsmanager/test_server.py @@ -66,3 +66,46 @@ def test_create_secret(): assert json_data['ARN'] == ( 'arn:aws:secretsmanager:us-east-1:1234567890:secret:test-secret-rIjad') assert json_data['Name'] == 'test-secret' + +@mock_secretsmanager +def test_describe_secret(): + + backend = server.create_backend_app('secretsmanager') + test_client = backend.test_client() + + create_secret = test_client.post('/', + data={"Name": "test-secret", + "SecretString": "foosecret"}, + headers={ + "X-Amz-Target": "secretsmanager.CreateSecret" + }, + ) + describe_secret = test_client.post('/', + data={"SecretId": "test-secret"}, + headers={ + "X-Amz-Target": "secretsmanager.DescribeSecret" + }, + ) + + json_data = json.loads(describe_secret.data.decode("utf-8")) + assert json_data # Returned dict is not empty + assert json_data['ARN'] == ( + 'arn:aws:secretsmanager:us-east-1:1234567890:secret:test-secret-rIjad' + ) + +@mock_secretsmanager +def test_describe_secret_that_does_not_exist(): + + backend = server.create_backend_app('secretsmanager') + test_client = backend.test_client() + + describe_secret = test_client.post('/', + data={"SecretId": "i-dont-exist"}, + headers={ + "X-Amz-Target": "secretsmanager.DescribeSecret" + }, + ) + + json_data = json.loads(describe_secret.data.decode("utf-8")) + assert json_data['message'] == "Secrets Manager can't find the specified secret" + assert json_data['__type'] == 'ResourceNotFoundException' From 65ef61ca1d61294acb5f41173bc3914467d8ce98 Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Mon, 6 Aug 2018 15:54:37 -0700 Subject: [PATCH 30/42] Fix linter warning. --- moto/secretsmanager/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index da3f3e6fe..26eb64c57 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -76,7 +76,7 @@ class SecretsManagerBackend(BaseBackend): def describe_secret(self, secret_id): if self.secret_id == '': raise ResourceNotFoundException - + response = json.dumps({ "ARN": secret_arn(self.region, self.secret_id), "Name": self.secret_id, From a42006462164347efaa45712590f9c26158103d7 Mon Sep 17 00:00:00 2001 From: Will Bengtson Date: Tue, 7 Aug 2018 10:31:36 -0700 Subject: [PATCH 31/42] IAM get account authorization details (#1736) * start of get_account_authorization_details for iam * add get_account_authorization_details dynamic template * remove old commented out template * Fix flake8 problems and add unit test --- moto/iam/models.py | 27 +++++++ moto/iam/responses.py | 153 +++++++++++++++++++++++++++++++++++++ tests/test_iam/test_iam.py | 65 ++++++++++++++++ 3 files changed, 245 insertions(+) diff --git a/moto/iam/models.py b/moto/iam/models.py index 8b632e555..697be7988 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -905,5 +905,32 @@ class IAMBackend(BaseBackend): def delete_account_alias(self, alias): self.account_aliases = [] + def get_account_authorization_details(self, filter): + policies = self.managed_policies.values() + local_policies = set(policies) - set(aws_managed_policies) + returned_policies = [] + + if len(filter) == 0: + return { + 'instance_profiles': self.instance_profiles.values(), + 'roles': self.roles.values(), + 'groups': self.groups.values(), + 'users': self.users.values(), + 'managed_policies': self.managed_policies.values() + } + + if 'AWSManagedPolicy' in filter: + returned_policies = aws_managed_policies + if 'LocalManagedPolicy' in filter: + returned_policies = returned_policies + list(local_policies) + + return { + 'instance_profiles': self.instance_profiles.values(), + 'roles': self.roles.values() if 'Role' in filter else [], + 'groups': self.groups.values() if 'Group' in filter else [], + 'users': self.users.values() if 'User' in filter else [], + 'managed_policies': returned_policies + } + iam_backend = IAMBackend() diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 786afab08..9c1241c36 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -534,6 +534,18 @@ class IamResponse(BaseResponse): template = self.response_template(DELETE_ACCOUNT_ALIAS_TEMPLATE) return template.render() + def get_account_authorization_details(self): + filter_param = self._get_multi_param('Filter.member') + account_details = iam_backend.get_account_authorization_details(filter_param) + template = self.response_template(GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE) + return template.render( + instance_profiles=account_details['instance_profiles'], + policies=account_details['managed_policies'], + users=account_details['users'], + groups=account_details['groups'], + roles=account_details['roles'] + ) + ATTACH_ROLE_POLICY_TEMPLATE = """ @@ -1309,3 +1321,144 @@ DELETE_ACCOUNT_ALIAS_TEMPLATE = """ + + + {% for group in groups %} + + {{ group.path }} + {{ group.name }} + {{ group.id }} + {{ group.arn }} + + {% endfor %} + + false + + + 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE + +""" + + +GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """ + + false + + {% for user in users %} + + + + {{ user.id }} + {{ user.path }} + {{ user.name }} + {{ user.arn }} + 2012-05-09T15:45:35Z + + {% endfor %} + + + EXAMPLEkakv9BCuUNFDtxWSyfzetYwEx2ADc8dnzfvERF5S6YMvXKx41t6gCl/eeaCX3Jo94/ + bKqezEAg8TEVS99EKFLxm3jtbpl25FDWEXAMPLE + + + {% for group in groups %} + + {{ group.id }} + + {% for policy in group.managed_policies %} + + {{ policy.name }} + {{ policy.arn }} + + {% endfor %} + + {{ group.name }} + {{ group.path }} + {{ group.arn }} + 2012-05-09T16:27:11Z + + + {% endfor %} + + + {% for role in roles %} + + + + {% for policy in role.managed_policies %} + + {{ policy.name }} + {{ policy.arn }} + + {% endfor %} + + + {% for profile in instance_profiles %} + + {{ profile.id }} + + {% for role in profile.roles %} + + {{ role.path }} + {{ role.arn }} + {{ role.name }} + {{ role.assume_role_policy_document }} + 2012-05-09T15:45:35Z + {{ role.id }} + + {% endfor %} + + {{ profile.name }} + {{ profile.path }} + {{ profile.arn }} + 2012-05-09T16:27:11Z + + {% endfor %} + + {{ role.path }} + {{ role.arn }} + {{ role.name }} + {{ role.assume_role_policy_document }} + 2014-07-30T17:09:20Z + {{ role.id }} + + {% endfor %} + + + {% for policy in policies %} + + {{ policy.name }} + {{ policy.default_version_id }} + {{ policy.id }} + {{ policy.path }} + + + + {"Version":"2012-10-17","Statement":{"Effect":"Allow", + "Action":["iam:CreatePolicy","iam:CreatePolicyVersion", + "iam:DeletePolicy","iam:DeletePolicyVersion","iam:GetPolicy", + "iam:GetPolicyVersion","iam:ListPolicies", + "iam:ListPolicyVersions","iam:SetDefaultPolicyVersion"], + "Resource":"*"}} + + true + v1 + 2012-05-09T16:27:11Z + + + {{ policy.arn }} + 1 + 2012-05-09T16:27:11Z + true + 2012-05-09T16:27:11Z + + {% endfor %} + + + + 92e79ae7-7399-11e4-8c85-4b53eEXAMPLE + +""" diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 182a60661..2225f0644 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -678,3 +678,68 @@ def test_update_access_key(): Status='Inactive') resp = client.list_access_keys(UserName=username) resp['AccessKeyMetadata'][0]['Status'].should.equal('Inactive') + + +@mock_iam +def test_get_account_authorization_details(): + import json + conn = boto3.client('iam', region_name='us-east-1') + conn.create_role(RoleName="my-role", AssumeRolePolicyDocument="some policy", Path="/my-path/") + conn.create_user(Path='/', UserName='testCloudAuxUser') + conn.create_group(Path='/', GroupName='testCloudAuxGroup') + conn.create_policy( + PolicyName='testCloudAuxPolicy', + Path='/', + PolicyDocument=json.dumps({ + "Version": "2012-10-17", + "Statement": [ + { + "Action": "s3:ListBucket", + "Resource": "*", + "Effect": "Allow", + } + ] + }), + Description='Test CloudAux Policy' + ) + + result = conn.get_account_authorization_details(Filter=['Role']) + len(result['RoleDetailList']) == 1 + len(result['UserDetailList']) == 0 + len(result['GroupDetailList']) == 0 + len(result['Policies']) == 0 + + result = conn.get_account_authorization_details(Filter=['User']) + len(result['RoleDetailList']) == 0 + len(result['UserDetailList']) == 1 + len(result['GroupDetailList']) == 0 + len(result['Policies']) == 0 + + result = conn.get_account_authorization_details(Filter=['Group']) + len(result['RoleDetailList']) == 0 + len(result['UserDetailList']) == 0 + len(result['GroupDetailList']) == 1 + len(result['Policies']) == 0 + + result = conn.get_account_authorization_details(Filter=['LocalManagedPolicy']) + len(result['RoleDetailList']) == 0 + len(result['UserDetailList']) == 0 + len(result['GroupDetailList']) == 0 + len(result['Policies']) == 1 + + # Check for greater than 1 since this should always be greater than one but might change. + # See iam/aws_managed_policies.py + result = conn.get_account_authorization_details(Filter=['AWSManagedPolicy']) + len(result['RoleDetailList']) == 0 + len(result['UserDetailList']) == 0 + len(result['GroupDetailList']) == 0 + len(result['Policies']) > 1 + + result = conn.get_account_authorization_details() + len(result['RoleDetailList']) == 1 + len(result['UserDetailList']) == 1 + len(result['GroupDetailList']) == 1 + len(result['Policies']) > 1 + + + From ba9e795394ea0c6490e1a7aa4bfef69d8c1b42bc Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Tue, 7 Aug 2018 10:53:21 -0700 Subject: [PATCH 32/42] Version 1.3.4 (#1757) * bumping to version 1.3.4 * updating changelog * fixing generation of implementation coverage --- .bumpversion.cfg | 2 +- CHANGELOG.md | 8 ++ IMPLEMENTATION_COVERAGE.md | 113 ++++++----------------------- moto/__init__.py | 2 +- scripts/implementation_coverage.py | 29 +++++--- setup.py | 2 +- 6 files changed, 49 insertions(+), 107 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 3e15854ef..479af4af8 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.3.3 +current_version = 1.3.4 [bumpversion:file:setup.py] diff --git a/CHANGELOG.md b/CHANGELOG.md index fb3a5d8d5..0bf5d2535 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ Moto Changelog =================== +1.3.4 +------ + + * IAM get account authorization details + * adding account id to ManagedPolicy ARN + * APIGateway usage plans and usage plan keys + * ECR list images + 1.3.3 ------ diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 411f55a8b..98a216c79 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -58,7 +58,6 @@ - [ ] get_room - [ ] get_room_skill_parameter - [ ] get_skill_group -- [ ] list_device_events - [ ] list_skills - [ ] list_tags - [ ] put_room_skill_parameter @@ -82,7 +81,7 @@ - [ ] update_room - [ ] update_skill_group -## apigateway - 17% implemented +## apigateway - 24% implemented - [ ] create_api_key - [ ] create_authorizer - [ ] create_base_path_mapping @@ -95,8 +94,8 @@ - [X] create_resource - [X] create_rest_api - [X] create_stage -- [ ] create_usage_plan -- [ ] create_usage_plan_key +- [X] create_usage_plan +- [X] create_usage_plan_key - [ ] create_vpc_link - [ ] delete_api_key - [ ] delete_authorizer @@ -116,8 +115,8 @@ - [X] delete_resource - [X] delete_rest_api - [ ] delete_stage -- [ ] delete_usage_plan -- [ ] delete_usage_plan_key +- [X] delete_usage_plan +- [X] delete_usage_plan_key - [ ] delete_vpc_link - [ ] flush_stage_authorizers_cache - [ ] flush_stage_cache @@ -162,10 +161,10 @@ - [X] get_stages - [ ] get_tags - [ ] get_usage -- [ ] get_usage_plan -- [ ] get_usage_plan_key -- [ ] get_usage_plan_keys -- [ ] get_usage_plans +- [X] get_usage_plan +- [X] get_usage_plan_key +- [X] get_usage_plan_keys +- [X] get_usage_plans - [ ] get_vpc_link - [ ] get_vpc_links - [ ] import_api_keys @@ -352,7 +351,6 @@ - [ ] delete_scaling_plan - [ ] describe_scaling_plan_resources - [ ] describe_scaling_plans -- [ ] update_scaling_plan ## batch - 93% implemented - [ ] cancel_job @@ -767,8 +765,6 @@ - [ ] create_pipeline - [ ] delete_custom_action_type - [ ] delete_pipeline -- [ ] delete_webhook -- [ ] deregister_webhook_with_third_party - [ ] disable_stage_transition - [ ] enable_stage_transition - [ ] get_job_details @@ -779,7 +775,6 @@ - [ ] list_action_types - [ ] list_pipeline_executions - [ ] list_pipelines -- [ ] list_webhooks - [ ] poll_for_jobs - [ ] poll_for_third_party_jobs - [ ] put_action_revision @@ -788,8 +783,6 @@ - [ ] put_job_success_result - [ ] put_third_party_job_failure_result - [ ] put_third_party_job_success_result -- [ ] put_webhook -- [ ] register_webhook_with_third_party - [ ] retry_stage_execution - [ ] start_pipeline_execution - [ ] update_pipeline @@ -1065,7 +1058,6 @@ - [ ] create_project - [ ] create_remote_access_session - [ ] create_upload -- [ ] create_vpce_configuration - [ ] delete_device_pool - [ ] delete_instance_profile - [ ] delete_network_profile @@ -1073,7 +1065,6 @@ - [ ] delete_remote_access_session - [ ] delete_run - [ ] delete_upload -- [ ] delete_vpce_configuration - [ ] get_account_settings - [ ] get_device - [ ] get_device_instance @@ -1089,7 +1080,6 @@ - [ ] get_suite - [ ] get_test - [ ] get_upload -- [ ] get_vpce_configuration - [ ] install_to_remote_access_session - [ ] list_artifacts - [ ] list_device_instances @@ -1109,7 +1099,6 @@ - [ ] list_tests - [ ] list_unique_problems - [ ] list_uploads -- [ ] list_vpce_configurations - [ ] purchase_offering - [ ] renew_offering - [ ] schedule_run @@ -1120,7 +1109,6 @@ - [ ] update_instance_profile - [ ] update_network_profile - [ ] update_project -- [ ] update_vpce_configuration ## directconnect - 0% implemented - [ ] allocate_connection_on_interconnect @@ -1277,7 +1265,7 @@ - [ ] update_radius - [ ] verify_trust -## dynamodb - 21% implemented +## dynamodb - 22% implemented - [ ] batch_get_item - [ ] batch_write_item - [ ] create_backup @@ -1289,7 +1277,6 @@ - [ ] describe_backup - [ ] describe_continuous_backups - [ ] describe_global_table -- [ ] describe_global_table_settings - [ ] describe_limits - [ ] describe_table - [ ] describe_time_to_live @@ -1307,7 +1294,6 @@ - [ ] untag_resource - [ ] update_continuous_backups - [ ] update_global_table -- [ ] update_global_table_settings - [ ] update_item - [ ] update_table - [ ] update_time_to_live @@ -1318,7 +1304,7 @@ - [ ] get_shard_iterator - [ ] list_streams -## ec2 - 36% implemented +## ec2 - 37% implemented - [ ] accept_reserved_instances_exchange_quote - [ ] accept_vpc_endpoint_connections - [X] accept_vpc_peering_connection @@ -1356,7 +1342,6 @@ - [ ] create_default_vpc - [X] create_dhcp_options - [ ] create_egress_only_internet_gateway -- [ ] create_fleet - [ ] create_flow_logs - [ ] create_fpga_image - [X] create_image @@ -1391,7 +1376,6 @@ - [X] delete_customer_gateway - [ ] delete_dhcp_options - [ ] delete_egress_only_internet_gateway -- [ ] delete_fleets - [ ] delete_flow_logs - [ ] delete_fpga_image - [X] delete_internet_gateway @@ -1433,9 +1417,6 @@ - [ ] describe_egress_only_internet_gateways - [ ] describe_elastic_gpus - [ ] describe_export_tasks -- [ ] describe_fleet_history -- [ ] describe_fleet_instances -- [ ] describe_fleets - [ ] describe_flow_logs - [ ] describe_fpga_image_attribute - [ ] describe_fpga_images @@ -1532,7 +1513,6 @@ - [X] import_key_pair - [ ] import_snapshot - [ ] import_volume -- [ ] modify_fleet - [ ] modify_fpga_image_attribute - [ ] modify_hosts - [ ] modify_id_format @@ -1905,11 +1885,8 @@ - [ ] delete_delivery_stream - [ ] describe_delivery_stream - [ ] list_delivery_streams -- [ ] list_tags_for_delivery_stream - [ ] put_record - [ ] put_record_batch -- [ ] tag_delivery_stream -- [ ] untag_delivery_stream - [ ] update_destination ## fms - 0% implemented @@ -2231,7 +2208,7 @@ - [ ] describe_event_types - [ ] describe_events -## iam - 47% implemented +## iam - 48% implemented - [ ] add_client_id_to_open_id_connect_provider - [X] add_role_to_instance_profile - [X] add_user_to_group @@ -2281,7 +2258,7 @@ - [X] enable_mfa_device - [ ] generate_credential_report - [ ] get_access_key_last_used -- [ ] get_account_authorization_details +- [X] get_account_authorization_details - [ ] get_account_password_policy - [ ] get_account_summary - [ ] get_context_keys_for_custom_policy @@ -2536,38 +2513,6 @@ - [ ] start_next_pending_job_execution - [ ] update_job_execution -## iotanalytics - 0% implemented -- [ ] batch_put_message -- [ ] cancel_pipeline_reprocessing -- [ ] create_channel -- [ ] create_dataset -- [ ] create_dataset_content -- [ ] create_datastore -- [ ] create_pipeline -- [ ] delete_channel -- [ ] delete_dataset -- [ ] delete_dataset_content -- [ ] delete_datastore -- [ ] delete_pipeline -- [ ] describe_channel -- [ ] describe_dataset -- [ ] describe_datastore -- [ ] describe_logging_options -- [ ] describe_pipeline -- [ ] get_dataset_content -- [ ] list_channels -- [ ] list_datasets -- [ ] list_datastores -- [ ] list_pipelines -- [ ] put_logging_options -- [ ] run_pipeline_activity -- [ ] sample_channel_data -- [ ] start_pipeline_reprocessing -- [ ] update_channel -- [ ] update_dataset -- [ ] update_datastore -- [ ] update_pipeline - ## kinesis - 56% implemented - [X] add_tags_to_stream - [X] create_stream @@ -2815,7 +2760,7 @@ - [ ] update_domain_entry - [ ] update_load_balancer_attribute -## logs - 24% implemented +## logs - 27% implemented - [ ] associate_kms_key - [ ] cancel_export_task - [ ] create_export_task @@ -2830,7 +2775,7 @@ - [ ] delete_subscription_filter - [ ] describe_destinations - [ ] describe_export_tasks -- [ ] describe_log_groups +- [X] describe_log_groups - [X] describe_log_streams - [ ] describe_metric_filters - [ ] describe_resource_policies @@ -3569,9 +3514,6 @@ - [ ] update_tags_for_domain - [ ] view_billing -## runtime.sagemaker - 0% implemented -- [ ] invoke_endpoint - ## s3 - 15% implemented - [ ] abort_multipart_upload - [ ] complete_multipart_upload @@ -3703,13 +3645,13 @@ - [ ] put_attributes - [ ] select -## secretsmanager - 0% implemented +## secretsmanager - 20% implemented - [ ] cancel_rotate_secret -- [ ] create_secret +- [X] create_secret - [ ] delete_secret - [ ] describe_secret -- [ ] get_random_password -- [ ] get_secret_value +- [X] get_random_password +- [X] get_secret_value - [ ] list_secret_version_ids - [ ] list_secrets - [ ] put_secret_value @@ -3984,7 +3926,7 @@ - [X] tag_queue - [X] untag_queue -## ssm - 10% implemented +## ssm - 11% implemented - [X] add_tags_to_resource - [ ] cancel_command - [ ] create_activation @@ -3997,7 +3939,6 @@ - [ ] delete_activation - [ ] delete_association - [ ] delete_document -- [ ] delete_inventory - [ ] delete_maintenance_window - [X] delete_parameter - [X] delete_parameters @@ -4021,7 +3962,6 @@ - [ ] describe_instance_patch_states - [ ] describe_instance_patch_states_for_patch_group - [ ] describe_instance_patches -- [ ] describe_inventory_deletions - [ ] describe_maintenance_window_execution_task_invocations - [ ] describe_maintenance_window_execution_tasks - [ ] describe_maintenance_window_executions @@ -4053,7 +3993,7 @@ - [ ] list_association_versions - [ ] list_associations - [ ] list_command_invocations -- [ ] list_commands +- [X] list_commands - [ ] list_compliance_items - [ ] list_compliance_summaries - [ ] list_document_versions @@ -4464,36 +4404,25 @@ - [ ] update_resource ## workspaces - 0% implemented -- [ ] associate_ip_groups -- [ ] authorize_ip_rules -- [ ] create_ip_group - [ ] create_tags - [ ] create_workspaces -- [ ] delete_ip_group - [ ] delete_tags -- [ ] describe_ip_groups - [ ] describe_tags - [ ] describe_workspace_bundles - [ ] describe_workspace_directories - [ ] describe_workspaces - [ ] describe_workspaces_connection_status -- [ ] disassociate_ip_groups - [ ] modify_workspace_properties -- [ ] modify_workspace_state - [ ] reboot_workspaces - [ ] rebuild_workspaces -- [ ] revoke_ip_rules - [ ] start_workspaces - [ ] stop_workspaces - [ ] terminate_workspaces -- [ ] update_rules_of_ip_group ## xray - 0% implemented - [ ] batch_get_traces -- [ ] get_encryption_config - [ ] get_service_graph - [ ] get_trace_graph - [ ] get_trace_summaries -- [ ] put_encryption_config - [ ] put_telemetry_records - [ ] put_trace_segments diff --git a/moto/__init__.py b/moto/__init__.py index 0ce5e54d1..80c782a87 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging # logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '1.3.3' +__version__ = '1.3.4' from .acm import mock_acm # flake8: noqa from .apigateway import mock_apigateway, mock_apigateway_deprecated # flake8: noqa diff --git a/scripts/implementation_coverage.py b/scripts/implementation_coverage.py index 74ce9590d..f15f3fb1c 100755 --- a/scripts/implementation_coverage.py +++ b/scripts/implementation_coverage.py @@ -6,6 +6,9 @@ from botocore.session import Session import boto3 +script_dir = os.path.dirname(os.path.abspath(__file__)) + + def get_moto_implementation(service_name): if not hasattr(moto, service_name): return None @@ -72,20 +75,22 @@ def write_implementation_coverage_to_file(coverage): except OSError: pass - for service_name in sorted(coverage): - implemented = coverage.get(service_name)['implemented'] - not_implemented = coverage.get(service_name)['not_implemented'] - operations = sorted(implemented + not_implemented) + implementation_coverage_file = "{}/../IMPLEMENTATION_COVERAGE.md".format(script_dir) + # rewrite the implementation coverage file with updated values + print("Writing to {}".format(implementation_coverage_file)) + with open(implementation_coverage_file, "a+") as file: + for service_name in sorted(coverage): + implemented = coverage.get(service_name)['implemented'] + not_implemented = coverage.get(service_name)['not_implemented'] + operations = sorted(implemented + not_implemented) - if implemented and not_implemented: - percentage_implemented = int(100.0 * len(implemented) / (len(implemented) + len(not_implemented))) - elif implemented: - percentage_implemented = 100 - else: - percentage_implemented = 0 + if implemented and not_implemented: + percentage_implemented = int(100.0 * len(implemented) / (len(implemented) + len(not_implemented))) + elif implemented: + percentage_implemented = 100 + else: + percentage_implemented = 0 - # rewrite the implementation coverage file with updated values - with open("../IMPLEMENTATION_COVERAGE.md", "a+") as file: file.write("\n") file.write("## {} - {}% implemented\n".format(service_name, percentage_implemented)) for op in operations: diff --git a/setup.py b/setup.py index 62f9026d7..304ce5f38 100755 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ else: setup( name='moto', - version='1.3.3', + version='1.3.4', description='A library that allows your python tests to easily' ' mock out the boto library', author='Steve Pulec', From 3830757ec64152e92aac4bc96b837e2108c0563f Mon Sep 17 00:00:00 2001 From: TheDooner64 Date: Tue, 7 Aug 2018 16:57:20 -0400 Subject: [PATCH 33/42] Add glue to backends to support server mode --- moto/backends.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/moto/backends.py b/moto/backends.py index cd8fe174f..8d707373f 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -20,6 +20,7 @@ from moto.elbv2 import elbv2_backends from moto.emr import emr_backends from moto.events import events_backends from moto.glacier import glacier_backends +from moto.glue import glue_backends from moto.iam import iam_backends from moto.instance_metadata import instance_metadata_backends from moto.kinesis import kinesis_backends @@ -65,6 +66,7 @@ BACKENDS = { 'events': events_backends, 'emr': emr_backends, 'glacier': glacier_backends, + 'glue': glue_backends, 'iam': iam_backends, 'moto_api': moto_api_backends, 'instance_metadata': instance_metadata_backends, From b47fc7465067f8cdf429c61b9e217d7bcf68e9ee Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Thu, 9 Aug 2018 18:19:33 -0700 Subject: [PATCH 34/42] Set correct default auto rotation period. --- moto/secretsmanager/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 26eb64c57..b6f3b3d86 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -35,7 +35,7 @@ class SecretsManagerBackend(BaseBackend): self.secret_string = '' self.rotation_enabled = False self.rotation_lambda_arn = '' - self.auto_rotate_after_days = 1 + self.auto_rotate_after_days = 0 def reset(self): region_name = self.region From 1f3256ed40576806628a99dcac4d2d91515bac59 Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Fri, 10 Aug 2018 16:40:31 -0700 Subject: [PATCH 35/42] Issue 1770: Deal with the friendly name properly - Save friendly name in create_secret. - Reference the saved friendly name in responses that have "Name" field. - Verify the received secret_id matches the current value. Don't just test for an empty string. - Add test for mismatched secret_id. --- moto/secretsmanager/models.py | 7 ++++--- tests/test_secretsmanager/test_secretsmanager.py | 9 +++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 3923f90b0..31dadf73f 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -41,12 +41,12 @@ class SecretsManagerBackend(BaseBackend): def get_secret_value(self, secret_id, version_id, version_stage): - if self.secret_id == '': + if secret_id not in (self.secret_id, self.name): raise ResourceNotFoundException() response = json.dumps({ "ARN": secret_arn(self.region, self.secret_id), - "Name": self.secret_id, + "Name": self.name, "VersionId": "A435958A-D821-4193-B719-B7769357AER4", "SecretString": self.secret_string, "VersionStages": [ @@ -61,10 +61,11 @@ class SecretsManagerBackend(BaseBackend): self.secret_string = secret_string self.secret_id = name + self.name = name response = json.dumps({ "ARN": secret_arn(self.region, name), - "Name": self.secret_id, + "Name": self.name, "VersionId": "A435958A-D821-4193-B719-B7769357AER4", }) diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index 6fefeb56f..a1544f72c 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -25,6 +25,15 @@ def test_get_secret_that_does_not_exist(): with assert_raises(ClientError): result = conn.get_secret_value(SecretId='i-dont-exist') +@mock_secretsmanager +def test_get_secret_with_mismatched_id(): + conn = boto3.client('secretsmanager', region_name='us-west-2') + create_secret = conn.create_secret(Name='java-util-test-password', + SecretString="foosecret") + + with assert_raises(ClientError): + result = conn.get_secret_value(SecretId='i-dont-exist') + @mock_secretsmanager def test_create_secret(): conn = boto3.client('secretsmanager', region_name='us-east-1') From 92bc3ff910cc4323825c80f5687856131e569804 Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Mon, 13 Aug 2018 12:41:43 -0700 Subject: [PATCH 36/42] Issue 1753: Add support for DescribeSecret - Add helper method to validate the secret identifier from the client. - Update describe_secret to use new helper method. - Insert friendly name into "Name" field of returned description (was SecretId). ***Assumes acceptance of PR 1772. --- moto/secretsmanager/models.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index b6f3b3d86..bba16ac53 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -42,6 +42,9 @@ class SecretsManagerBackend(BaseBackend): self.__dict__ = {} self.__init__(region_name) + def _is_valid_identifier(self, identifier): + return identifier in (self.name, self.secret_id) + def get_secret_value(self, secret_id, version_id, version_stage): if self.secret_id == '': @@ -74,12 +77,12 @@ class SecretsManagerBackend(BaseBackend): return response def describe_secret(self, secret_id): - if self.secret_id == '': + if not self._is_valid_identifier(secret_id): raise ResourceNotFoundException response = json.dumps({ "ARN": secret_arn(self.region, self.secret_id), - "Name": self.secret_id, + "Name": self.name, "Description": "", "KmsKeyId": "", "RotationEnabled": self.rotation_enabled, From 85d3250b893b4d4505486013168fcf69da79da78 Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Mon, 13 Aug 2018 12:49:35 -0700 Subject: [PATCH 37/42] Issue 1753: add test for mismatched secret --- tests/test_secretsmanager/test_secretsmanager.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index 0ef54b45b..2691e2783 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -161,3 +161,12 @@ def test_describe_secret_that_does_not_exist(): with assert_raises(ClientError): result = conn.get_secret_value(SecretId='i-dont-exist') + +@mock_secretsmanager +def test_describe_secret_that_does_not_match(): + conn = boto3.client('secretsmanager', region_name='us-west-2') + conn.create_secret(Name='test-secret', + SecretString='foosecret') + + with assert_raises(ClientError): + result = conn.get_secret_value(SecretId='i-dont-match') From b2c672a074c1346e2ac05b5fe7353c987a83f7a8 Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Mon, 13 Aug 2018 12:53:22 -0700 Subject: [PATCH 38/42] Issue 1753: add server test for mismatched secret --- tests/test_secretsmanager/test_server.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_secretsmanager/test_server.py b/tests/test_secretsmanager/test_server.py index 370a483a8..8c6f7b970 100644 --- a/tests/test_secretsmanager/test_server.py +++ b/tests/test_secretsmanager/test_server.py @@ -109,3 +109,27 @@ def test_describe_secret_that_does_not_exist(): json_data = json.loads(describe_secret.data.decode("utf-8")) assert json_data['message'] == "Secrets Manager can't find the specified secret" assert json_data['__type'] == 'ResourceNotFoundException' + +@mock_secretsmanager +def test_describe_secret_that_does_not_match(): + + backend = server.create_backend_app('secretsmanager') + test_client = backend.test_client() + + create_secret = test_client.post('/', + data={"Name": "test-secret", + "SecretString": "foosecret"}, + headers={ + "X-Amz-Target": "secretsmanager.CreateSecret" + }, + ) + describe_secret = test_client.post('/', + data={"SecretId": "i-dont-match"}, + headers={ + "X-Amz-Target": "secretsmanager.DescribeSecret" + }, + ) + + json_data = json.loads(describe_secret.data.decode("utf-8")) + assert json_data['message'] == "Secrets Manager can't find the specified secret" + assert json_data['__type'] == 'ResourceNotFoundException' From 48a71ae329765188cf241dc3662d028e78cc345d Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Tue, 14 Aug 2018 12:04:39 -0700 Subject: [PATCH 39/42] Issue 1753: Add support for DescribeSecret - Merge changes from upstream master. - Update get_secret_value to use helper method for validating secret identifier. - Update implementation coverage checklist. --- IMPLEMENTATION_COVERAGE.md | 4 ++-- moto/secretsmanager/models.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 98a216c79..938cc3549 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3645,11 +3645,11 @@ - [ ] put_attributes - [ ] select -## secretsmanager - 20% implemented +## secretsmanager - 27% implemented - [ ] cancel_rotate_secret - [X] create_secret - [ ] delete_secret -- [ ] describe_secret +- [X] describe_secret - [X] get_random_password - [X] get_secret_value - [ ] list_secret_version_ids diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 07b563633..c60feb530 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -47,7 +47,7 @@ class SecretsManagerBackend(BaseBackend): def get_secret_value(self, secret_id, version_id, version_stage): - if secret_id not in (self.secret_id, self.name): + if not self._is_valid_identifier(secret_id): raise ResourceNotFoundException() response = json.dumps({ From abc15f0ae8838106ad2eee510bc7141f0c2363ea Mon Sep 17 00:00:00 2001 From: Gary Donovan Date: Mon, 27 Aug 2018 17:29:55 +1000 Subject: [PATCH 40/42] Mitigate #1793 by restricting maximum version for botocore. botocore v1.11.0 changed it's internal implementation so that it now uses a different library for HTTP requests. This means that moto's mocking will not work, and test code will inadvertently call the live AWS service. As an interim solution to reduce the impact of this breakage, we restrict the "required" (ie. recommended) version of botocore so that users will be less likely to use an incompatible version, and will receive a pip warning when they do. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 62f9026d7..6f817f042 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ install_requires = [ "Jinja2>=2.7.3", "boto>=2.36.0", "boto3>=1.6.16", - "botocore>=1.9.16", + "botocore>=1.9.16,<1.11", "cookies", "cryptography>=2.0.0", "requests>=2.5", From 1371b742cfe74140ee1d5a41f7b89b1f3611ad1e Mon Sep 17 00:00:00 2001 From: Subhankar Sett Date: Tue, 28 Aug 2018 11:02:36 -0400 Subject: [PATCH 41/42] Update README.md Fixes a very minor typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a6926a58f..8618b4042 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ def test_add_servers(): ``` #### Using moto 1.0.X with boto2 -moto 1.0.X mock docorators are defined for boto3 and do not work with boto2. Use the @mock_AWSSVC_deprecated to work with boto2. +moto 1.0.X mock decorators are defined for boto3 and do not work with boto2. Use the @mock_AWSSVC_deprecated to work with boto2. Using moto with boto2 ```python From c4b630e20fb9004ef4b1eab345ceab5ef9e4c2ff Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Wed, 29 Aug 2018 08:44:03 -0400 Subject: [PATCH 42/42] Version 1.3.5. --- CHANGELOG.md | 6 ++++++ moto/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bf5d2535..202da6ce6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,12 @@ Moto Changelog =================== +1.3.5 +----- + + * Pin down botocore issue as temporary fix for #1793. + * More features on secrets manager + 1.3.4 ------ diff --git a/moto/__init__.py b/moto/__init__.py index 680bc9d38..b7b653200 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging # logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '1.3.4' +__version__ = '1.3.5' from .acm import mock_acm # flake8: noqa from .apigateway import mock_apigateway, mock_apigateway_deprecated # flake8: noqa diff --git a/setup.py b/setup.py index 4e6bb3f13..16aaf1452 100755 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ else: setup( name='moto', - version='1.3.4', + version='1.3.5', description='A library that allows your python tests to easily' ' mock out the boto library', author='Steve Pulec',