From 1267e201c53039146955a3df0085d520f5f1d063 Mon Sep 17 00:00:00 2001 From: Robert Rose Date: Thu, 28 Jun 2018 12:31:08 -0700 Subject: [PATCH 001/106] Updated index.rst to fix overflow The EC2 endpoint status was overflowing and creating a scroll bar on my screen. It was bugging me so I fixed it via the GitHub web interface. Will test to ensure it builds correctly when I get home from work. --- docs/index.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 321342401..66e12e4bd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -34,11 +34,11 @@ Currently implemented Services: | - DynamoDB2 | - @mock_dynamodb2 | - core endpoints + partial indexes| +-----------------------+---------------------+-----------------------------------+ | EC2 | @mock_ec2 | core endpoints done | -| - AMI | | core endpoints done | -| - EBS | | core endpoints done | -| - Instances | | all endpoints done | -| - Security Groups | | core endpoints done | -| - Tags | | all endpoints done | +| - AMI | | - core endpoints done | +| - EBS | | - core endpoints done | +| - Instances | | - all endpoints done | +| - Security Groups | | - core endpoints done | +| - Tags | | - all endpoints done | +-----------------------+---------------------+-----------------------------------+ | ECS | @mock_ecs | basic endpoints done | +-----------------------+---------------------+-----------------------------------+ From 12807bb6f0ad1840881b077175409c0c8dacbfb5 Mon Sep 17 00:00:00 2001 From: Mike Liu Date: Tue, 24 Jul 2018 16:15:53 -0400 Subject: [PATCH 002/106] Add get_command_invocation endpoint for AWS SSM. Users can make send_command requests and then retrieve the invocations of those commands with get_command_invocation. Getting a command invocation by instance and command id is supported but only the 'aws:runShellScript' plugin name is supported and only one plugin in a document is supported. --- moto/ssm/models.py | 71 ++++++++++++++++++++++++++++++-- moto/ssm/responses.py | 5 +++ tests/test_ssm/test_ssm_boto3.py | 40 ++++++++++++++++++ 3 files changed, 112 insertions(+), 4 deletions(-) diff --git a/moto/ssm/models.py b/moto/ssm/models.py index 656a14839..b3e34eae4 100644 --- a/moto/ssm/models.py +++ b/moto/ssm/models.py @@ -88,9 +88,9 @@ class Command(BaseModel): self.status = 'Success' self.status_details = 'Details placeholder' - now = datetime.datetime.now() - self.requested_date_time = now.isoformat() - expires_after = now + datetime.timedelta(0, timeout_seconds) + self.requested_date_time = datetime.datetime.now() + self.requested_date_time_iso = self.requested_date_time.isoformat() + expires_after = self.requested_date_time + datetime.timedelta(0, timeout_seconds) self.expires_after = expires_after.isoformat() self.comment = comment @@ -106,6 +106,12 @@ class Command(BaseModel): self.service_role_arn = service_role_arn self.targets = targets + # Create invocations with a single run command plugin. + self.invocations = [] + for instance_id in self.instance_ids: + self.invocations.append( + self.invocation_response(instance_id, "aws:runShellScript")) + def response_object(self): r = { 'CommandId': self.command_id, @@ -122,7 +128,7 @@ class Command(BaseModel): 'OutputS3BucketName': self.output_s3_bucket_name, 'OutputS3KeyPrefix': self.output_s3_key_prefix, 'Parameters': self.parameters, - 'RequestedDateTime': self.requested_date_time, + 'RequestedDateTime': self.requested_date_time_iso, 'ServiceRole': self.service_role_arn, 'Status': self.status, 'StatusDetails': self.status_details, @@ -132,6 +138,51 @@ class Command(BaseModel): return r + def invocation_response(self, instance_id, plugin_name): + # Calculate elapsed time from requested time and now. Use a hardcoded + # elapsed time since there is no easy way to convert a timedelta to + # an ISO 8601 duration string. + elapsed_time_iso = "PT5M" + elapsed_time_delta = datetime.timedelta(minutes=5) + end_time = self.requested_date_time + elapsed_time_delta + + r = { + 'CommandId': self.command_id, + 'InstanceId': instance_id, + 'Comment': self.comment, + 'DocumentName': self.document_name, + 'PluginName': plugin_name, + 'ResponseCode': 0, + 'ExecutionStartDateTime': self.requested_date_time_iso, + 'ExecutionElapsedTime': elapsed_time_iso, + 'ExecutionEndDateTime': end_time.isoformat(), + 'Status': 'Success', + 'StatusDetails': 'Success', + 'StandardOutputContent': '', + 'StandardOutputUrl': '', + 'StandardErrorContent': '', + } + + return r + + def get_invocation(self, instance_id, plugin_name): + invocation = next( + (invocation for invocation in self.invocations + if invocation['InstanceId'] == instance_id), None) + + if invocation is None: + raise RESTError( + 'InvocationDoesNotExist', + 'An error occurred (InvocationDoesNotExist) when calling the GetCommandInvocation operation') + + if plugin_name is not None and invocation['PluginName'] != plugin_name: + raise RESTError( + 'InvocationDoesNotExist', + 'An error occurred (InvocationDoesNotExist) when calling the GetCommandInvocation operation') + + + return invocation + class SimpleSystemManagerBackend(BaseBackend): @@ -298,6 +349,18 @@ class SimpleSystemManagerBackend(BaseBackend): command for command in self._commands if instance_id in command.instance_ids] + def get_command_invocation(self, **kwargs): + """ + https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_GetCommandInvocation.html + """ + + command_id = kwargs.get('CommandId') + instance_id = kwargs.get('InstanceId') + plugin_name = kwargs.get('PluginName', None) + + command = self.get_command_by_id(command_id) + return command.get_invocation(instance_id, plugin_name) + ssm_backends = {} for region, ec2_backend in ec2_backends.items(): diff --git a/moto/ssm/responses.py b/moto/ssm/responses.py index fd0d8b630..eb05e51b6 100644 --- a/moto/ssm/responses.py +++ b/moto/ssm/responses.py @@ -210,3 +210,8 @@ class SimpleSystemManagerResponse(BaseResponse): return json.dumps( self.ssm_backend.list_commands(**self.request_params) ) + + def get_command_invocation(self): + return json.dumps( + self.ssm_backend.get_command_invocation(**self.request_params) + ) diff --git a/tests/test_ssm/test_ssm_boto3.py b/tests/test_ssm/test_ssm_boto3.py index 7a0685d56..8df565cf6 100644 --- a/tests/test_ssm/test_ssm_boto3.py +++ b/tests/test_ssm/test_ssm_boto3.py @@ -668,3 +668,43 @@ def test_list_commands(): with assert_raises(ClientError): response = client.list_commands( CommandId=str(uuid.uuid4())) + +@mock_ssm +def test_get_command_invocation(): + client = boto3.client('ssm', region_name='us-east-1') + + ssm_document = 'AWS-RunShellScript' + params = {'commands': ['#!/bin/bash\necho \'hello world\'']} + + response = client.send_command( + InstanceIds=['i-123456', 'i-234567', 'i-345678'], + DocumentName=ssm_document, + Parameters=params, + OutputS3Region='us-east-2', + OutputS3BucketName='the-bucket', + OutputS3KeyPrefix='pref') + + cmd = response['Command'] + cmd_id = cmd['CommandId'] + + instance_id = 'i-345678' + invocation_response = client.get_command_invocation( + CommandId=cmd_id, + InstanceId=instance_id, + PluginName='aws:runShellScript') + + invocation_response['CommandId'].should.equal(cmd_id) + invocation_response['InstanceId'].should.equal(instance_id) + + # test the error case for an invalid instance id + with assert_raises(ClientError): + invocation_response = client.get_command_invocation( + CommandId=cmd_id, + InstanceId='i-FAKE') + + # test the error case for an invalid plugin name + with assert_raises(ClientError): + invocation_response = client.get_command_invocation( + CommandId=cmd_id, + InstanceId=instance_id, + PluginName='FAKE') From 7d78a08f9539b6d3dbeb7b6a9766eb042911943c Mon Sep 17 00:00:00 2001 From: Robert C Jensen Date: Thu, 26 Jul 2018 21:08:31 -0400 Subject: [PATCH 003/106] bugfix: support name-addr mailbox form --- moto/ses/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moto/ses/models.py b/moto/ses/models.py index 3dced60f2..71fe9d9a1 100644 --- a/moto/ses/models.py +++ b/moto/ses/models.py @@ -49,7 +49,8 @@ class SESBackend(BaseBackend): self.sent_messages = [] self.sent_message_count = 0 - def _is_verified_address(self, address): + def _is_verified_address(self, source): + _, address = parseaddr(source) if address in self.addresses: return True user, host = address.split('@', 1) From 2fad7c72024ad20a95a1fbe4cd0d3daca4698dd9 Mon Sep 17 00:00:00 2001 From: Tomoya Iwata Date: Thu, 2 Aug 2018 12:38:50 +0900 Subject: [PATCH 004/106] update dynamodb2 update_item add empty string handle, fix(#1744) --- moto/dynamodb2/responses.py | 24 ++++++++++----- tests/test_dynamodb2/test_dynamodb.py | 42 +++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 3c7e7ffc2..493e17833 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -20,6 +20,17 @@ def has_empty_keys_or_values(_dict): ) +def get_empty_str_error(): + er = 'com.amazonaws.dynamodb.v20111205#ValidationException' + return (400, + {'server': 'amazon.com'}, + dynamo_json_dump({'__type': er, + 'message': ('One or more parameter values were ' + 'invalid: An AttributeValue may not ' + 'contain an empty string')} + )) + + class DynamoHandler(BaseResponse): def get_endpoint_name(self, headers): @@ -174,14 +185,7 @@ class DynamoHandler(BaseResponse): item = self.body['Item'] if has_empty_keys_or_values(item): - er = 'com.amazonaws.dynamodb.v20111205#ValidationException' - return (400, - {'server': 'amazon.com'}, - dynamo_json_dump({'__type': er, - 'message': ('One or more parameter values were ' - 'invalid: An AttributeValue may not ' - 'contain an empty string')} - )) + return get_empty_str_error() overwrite = 'Expected' not in self.body if not overwrite: @@ -523,6 +527,7 @@ class DynamoHandler(BaseResponse): return dynamo_json_dump(item_dict) def update_item(self): + name = self.body['TableName'] key = self.body['Key'] update_expression = self.body.get('UpdateExpression') @@ -533,6 +538,9 @@ class DynamoHandler(BaseResponse): 'ExpressionAttributeValues', {}) existing_item = self.dynamodb_backend.get_item(name, key) + if has_empty_keys_or_values(expression_attribute_values): + return get_empty_str_error() + if 'Expected' in self.body: expected = self.body['Expected'] else: diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index ab8f25856..243de2701 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -201,6 +201,48 @@ def test_item_add_empty_string_exception(): ) +@requires_boto_gte("2.9") +@mock_dynamodb2 +def test_update_item_with_empty_string_exception(): + name = 'TestTable' + conn = boto3.client('dynamodb', + region_name='us-west-2', + aws_access_key_id="ak", + aws_secret_access_key="sk") + conn.create_table(TableName=name, + KeySchema=[{'AttributeName':'forum_name','KeyType':'HASH'}], + AttributeDefinitions=[{'AttributeName':'forum_name','AttributeType':'S'}], + ProvisionedThroughput={'ReadCapacityUnits':5,'WriteCapacityUnits':5}) + + conn.put_item( + TableName=name, + Item={ + 'forum_name': { 'S': 'LOLCat Forum' }, + 'subject': { 'S': 'Check this out!' }, + 'Body': { 'S': 'http://url_to_lolcat.gif'}, + 'SentBy': { 'S': "test" }, + 'ReceivedTime': { 'S': '12/9/2011 11:36:03 PM'}, + } + ) + + with assert_raises(ClientError) as ex: + conn.update_item( + TableName=name, + Key={ + 'forum_name': { 'S': 'LOLCat Forum'}, + }, + UpdateExpression='set Body=:Body', + ExpressionAttributeValues={ + ':Body': {'S': ''} + }) + + ex.exception.response['Error']['Code'].should.equal('ValidationException') + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(400) + ex.exception.response['Error']['Message'].should.equal( + 'One or more parameter values were invalid: An AttributeValue may not contain an empty string' + ) + + @requires_boto_gte("2.9") @mock_dynamodb2 def test_query_invalid_table(): From ac4197ca47d6f1a54a21f554154e2a1cc7b87522 Mon Sep 17 00:00:00 2001 From: brett55 Date: Thu, 2 Aug 2018 16:09:10 -0600 Subject: [PATCH 005/106] - Updated ResourceAlreadyExistsException(LogsClientError) to latest Boto3 error msg when using CWL Logs --- moto/logs/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/logs/exceptions.py b/moto/logs/exceptions.py index cc83452ea..bb02eced3 100644 --- a/moto/logs/exceptions.py +++ b/moto/logs/exceptions.py @@ -29,5 +29,5 @@ class ResourceAlreadyExistsException(LogsClientError): self.code = 400 super(ResourceAlreadyExistsException, self).__init__( 'ResourceAlreadyExistsException', - 'The specified resource already exists.' + 'The specified log group already exists' ) From ae2e6fef632c5233639702dc304aa4cf793e4e95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Faucon?= Date: Mon, 6 Aug 2018 10:47:00 +0200 Subject: [PATCH 006/106] Add support for unregistered instances in ELB DescribeInstancesHealth API. --- moto/elb/responses.py | 26 ++++++++++++++++++-------- tests/test_elb/test_elb.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/moto/elb/responses.py b/moto/elb/responses.py index 40d6ec2f9..b512f56e9 100644 --- a/moto/elb/responses.py +++ b/moto/elb/responses.py @@ -259,12 +259,22 @@ class ELBResponse(BaseResponse): def describe_instance_health(self): load_balancer_name = self._get_param('LoadBalancerName') - instance_ids = [list(param.values())[0] for param in self._get_list_prefix('Instances.member')] - if len(instance_ids) == 0: - instance_ids = self.elb_backend.get_load_balancer( - load_balancer_name).instance_ids + provided_instance_ids = [ + list(param.values())[0] + for param in self._get_list_prefix('Instances.member') + ] + registered_instances_id = self.elb_backend.get_load_balancer( + load_balancer_name).instance_ids + if len(provided_instance_ids) == 0: + provided_instance_ids = registered_instances_id template = self.response_template(DESCRIBE_INSTANCE_HEALTH_TEMPLATE) - return template.render(instance_ids=instance_ids) + instances = [] + for instance_id in provided_instance_ids: + state = "InService" \ + if instance_id in registered_instances_id\ + else "Unknown" + instances.append({"InstanceId": instance_id, "State": state}) + return template.render(instances=instances) def add_tags(self): @@ -689,11 +699,11 @@ SET_LOAD_BALANCER_POLICIES_FOR_BACKEND_SERVER_TEMPLATE = """ - {% for instance_id in instance_ids %} + {% for instance in instances %} N/A - {{ instance_id }} - InService + {{ instance['InstanceId'] }} + {{ instance['State'] }} N/A {% endfor %} diff --git a/tests/test_elb/test_elb.py b/tests/test_elb/test_elb.py index 5827e70c7..6ffcf84da 100644 --- a/tests/test_elb/test_elb.py +++ b/tests/test_elb/test_elb.py @@ -723,6 +723,40 @@ def test_describe_instance_health(): instances_health[0].state.should.equal('InService') +@mock_ec2 +@mock_elb +def test_describe_instance_health_boto3(): + elb = boto3.client('elb') + ec2 = boto3.client('ec2') + instances = ec2.run_instances(MinCount=2, MaxCount=2)['Instances'] + lb_name = "my_load_balancer" + elb.create_load_balancer( + Listeners=[{ + 'InstancePort': 80, + 'LoadBalancerPort': 8080, + 'Protocol': 'HTTP' + }], + LoadBalancerName=lb_name, + ) + elb.register_instances_with_load_balancer( + LoadBalancerName=lb_name, + Instances=[{'InstanceId': instances[0]['InstanceId']}] + ) + instances_health = elb.describe_instance_health( + LoadBalancerName=lb_name, + Instances=[{'InstanceId': instance['InstanceId']} for instance in instances] + ) + instances_health['InstanceStates'].should.have.length_of(2) + instances_health['InstanceStates'][0]['InstanceId'].\ + should.equal(instances[0]['InstanceId']) + instances_health['InstanceStates'][0]['State'].\ + should.equal('InService') + instances_health['InstanceStates'][1]['InstanceId'].\ + should.equal(instances[1]['InstanceId']) + instances_health['InstanceStates'][1]['State'].\ + should.equal('Unknown') + + @mock_elb def test_add_remove_tags(): client = boto3.client('elb', region_name='us-east-1') From 5d7a432af2dd2b19b671bc33fec42ed0ef8ef5f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Faucon?= Date: Mon, 6 Aug 2018 11:01:33 +0200 Subject: [PATCH 007/106] Tests on Travis: specify region name. --- tests/test_elb/test_elb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_elb/test_elb.py b/tests/test_elb/test_elb.py index 6ffcf84da..a67508430 100644 --- a/tests/test_elb/test_elb.py +++ b/tests/test_elb/test_elb.py @@ -726,8 +726,8 @@ def test_describe_instance_health(): @mock_ec2 @mock_elb def test_describe_instance_health_boto3(): - elb = boto3.client('elb') - ec2 = boto3.client('ec2') + elb = boto3.client('elb', region_name="us-east-1") + ec2 = boto3.client('ec2', region_name="us-east-1") instances = ec2.run_instances(MinCount=2, MaxCount=2)['Instances'] lb_name = "my_load_balancer" elb.create_load_balancer( From f23288d9b92d8519239a2beeeb0dc27b97676f17 Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Tue, 7 Aug 2018 13:55:13 +0200 Subject: [PATCH 008/106] Changed the 'create_access_token' function in order to add the extra data into 'create_jwt' function --- moto/cognitoidp/models.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 52a73f89f..1119edcbe 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -84,7 +84,11 @@ class CognitoIdpUserPool(BaseModel): return refresh_token def create_access_token(self, client_id, username): - access_token, expires_in = self.create_jwt(client_id, username) + extra_data = self.get_user_extra_data_by_client_id( + client_id, username + ) + access_token, expires_in = self.create_jwt(client_id, username, + extra_data=extra_data) self.access_tokens[access_token] = (client_id, username) return access_token, expires_in @@ -97,6 +101,21 @@ class CognitoIdpUserPool(BaseModel): id_token, _ = self.create_id_token(client_id, username) return access_token, id_token, expires_in + def get_user_extra_data_by_client_id(self, client_id, username): + extra_data = {} + current_client = self.clients.get(client_id, None) + if current_client: + for readable_field in current_client.get_readable_fields(): + attribute = list(filter( + lambda f: f['Name'] == readable_field, + self.users.get(username).attributes + )) + if len(attribute) > 0: + extra_data.update({ + attribute[0]['Name']: attribute[0]['Value'] + }) + return extra_data + class CognitoIdpUserPoolDomain(BaseModel): @@ -138,6 +157,9 @@ class CognitoIdpUserPoolClient(BaseModel): return user_pool_client_json + def get_readable_fields(self): + return self.extended_config.get('ReadAttributes', []) + class CognitoIdpIdentityProvider(BaseModel): From 4776228b6824e31a43060082ef4ef15b490de3af Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Tue, 7 Aug 2018 13:55:32 +0200 Subject: [PATCH 009/106] Testing new feature --- tests/test_cognitoidp/test_cognitoidp.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index b2bd469ce..fda24146d 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -325,6 +325,7 @@ def test_delete_identity_providers(): def test_admin_create_user(): conn = boto3.client("cognito-idp", "us-west-2") + username = str(uuid.uuid4()) username = str(uuid.uuid4()) value = str(uuid.uuid4()) user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] @@ -399,15 +400,22 @@ def authentication_flow(conn): username = str(uuid.uuid4()) temporary_password = str(uuid.uuid4()) user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + user_attribute_name = str(uuid.uuid4()) + user_attribute_value = str(uuid.uuid4()) client_id = conn.create_user_pool_client( UserPoolId=user_pool_id, ClientName=str(uuid.uuid4()), + ReadAttributes=[user_attribute_name] )["UserPoolClient"]["ClientId"] conn.admin_create_user( UserPoolId=user_pool_id, Username=username, TemporaryPassword=temporary_password, + UserAttributes=[{ + 'Name': user_attribute_name, + 'Value': user_attribute_value + }] ) result = conn.admin_initiate_auth( @@ -446,6 +454,9 @@ def authentication_flow(conn): "access_token": result["AuthenticationResult"]["AccessToken"], "username": username, "password": new_password, + "additional_fields": { + user_attribute_name: user_attribute_value + } } @@ -475,6 +486,8 @@ def test_token_legitimacy(): access_claims = json.loads(jws.verify(access_token, json_web_key, "RS256")) access_claims["iss"].should.equal(issuer) access_claims["aud"].should.equal(client_id) + for k, v in outputs["additional_fields"].items(): + access_claims[k].should.equal(v) @mock_cognitoidp From 9e667d6b2582d0919f8f98dbd241a520124540ef Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Tue, 7 Aug 2018 16:59:15 -0400 Subject: [PATCH 010/106] Add get_policy IAM action --- moto/iam/models.py | 28 +++++++++++----------------- moto/iam/responses.py | 25 +++++++++++++++++++++++++ tests/test_iam/test_iam.py | 9 +++++++++ 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/moto/iam/models.py b/moto/iam/models.py index 697be7988..c8f95f3c9 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -72,11 +72,11 @@ class ManagedPolicy(Policy): def attach_to(self, obj): self.attachment_count += 1 - obj.managed_policies[self.name] = self + obj.managed_policies[self.arn] = self def detach_from(self, obj): self.attachment_count -= 1 - del obj.managed_policies[self.name] + del obj.managed_policies[self.arn] @property def arn(self): @@ -477,11 +477,13 @@ class IAMBackend(BaseBackend): document=policy_document, path=path, ) - self.managed_policies[policy.name] = policy + self.managed_policies[policy.arn] = policy return policy - def get_policy(self, policy_name): - return self.managed_policies.get(policy_name) + def get_policy(self, policy_arn): + if policy_arn not in self.managed_policies: + raise IAMNotFoundException("Policy {0} not found".format(policy_arn)) + return self.managed_policies.get(policy_arn) def list_attached_role_policies(self, role_name, marker=None, max_items=100, path_prefix='/'): policies = self.get_role(role_name).managed_policies.values() @@ -575,9 +577,7 @@ class IAMBackend(BaseBackend): return role.policies.keys() def create_policy_version(self, policy_arn, policy_document, set_as_default): - policy_name = policy_arn.split(':')[-1] - policy_name = policy_name.split('/')[1] - policy = self.get_policy(policy_name) + policy = self.get_policy(policy_arn) if not policy: raise IAMNotFoundException("Policy not found") version = PolicyVersion(policy_arn, policy_document, set_as_default) @@ -587,9 +587,7 @@ class IAMBackend(BaseBackend): return version def get_policy_version(self, policy_arn, version_id): - policy_name = policy_arn.split(':')[-1] - policy_name = policy_name.split('/')[1] - policy = self.get_policy(policy_name) + policy = self.get_policy(policy_arn) if not policy: raise IAMNotFoundException("Policy not found") for version in policy.versions: @@ -598,17 +596,13 @@ class IAMBackend(BaseBackend): raise IAMNotFoundException("Policy version not found") def list_policy_versions(self, policy_arn): - policy_name = policy_arn.split(':')[-1] - policy_name = policy_name.split('/')[1] - policy = self.get_policy(policy_name) + policy = self.get_policy(policy_arn) if not policy: raise IAMNotFoundException("Policy not found") return policy.versions def delete_policy_version(self, policy_arn, version_id): - policy_name = policy_arn.split(':')[-1] - policy_name = policy_name.split('/')[1] - policy = self.get_policy(policy_name) + policy = self.get_policy(policy_arn) if not policy: raise IAMNotFoundException("Policy not found") for i, v in enumerate(policy.versions): diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 9c1241c36..e77cec358 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -58,6 +58,12 @@ class IamResponse(BaseResponse): template = self.response_template(CREATE_POLICY_TEMPLATE) return template.render(policy=policy) + def get_policy(self): + policy_arn = self._get_param('PolicyArn') + policy = iam_backend.get_policy(policy_arn) + template = self.response_template(GET_POLICY_TEMPLATE) + return template.render(policy=policy) + def list_attached_role_policies(self): marker = self._get_param('Marker') max_items = self._get_int_param('MaxItems', 100) @@ -601,6 +607,25 @@ CREATE_POLICY_TEMPLATE = """ """ +GET_POLICY_TEMPLATE = """ + + + {{ policy.name }} + {{ policy.description }} + {{ policy.default_version_id }} + {{ policy.id }} + {{ policy.path }} + {{ policy.arn }} + {{ policy.attachment_count }} + {{ policy.create_datetime.isoformat() }} + {{ policy.update_datetime.isoformat() }} + + + + 684f0917-3d22-11e4-a4a0-cffb9EXAMPLE + +""" + LIST_ATTACHED_ROLE_POLICIES_TEMPLATE = """ {% if marker is none %} diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 2225f0644..61cd073fe 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -286,6 +286,15 @@ def test_create_policy_versions(): PolicyDocument='{"some":"policy"}') version.get('PolicyVersion').get('Document').should.equal({'some': 'policy'}) +@mock_iam +def test_get_policy(): + conn = boto3.client('iam', region_name='us-east-1') + response = conn.create_policy( + PolicyName="TestGetPolicy", + PolicyDocument='{"some":"policy"}') + policy = conn.get_policy( + PolicyArn="arn:aws:iam::123456789012:policy/TestGetPolicy") + response['Policy']['Arn'].should.equal("arn:aws:iam::123456789012:policy/TestGetPolicy") @mock_iam def test_get_policy_version(): From 5ae5ae0efae3312eed7da1cff1ea8785b7161b36 Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Tue, 7 Aug 2018 17:24:15 -0400 Subject: [PATCH 011/106] Correct deviation in behavior of policy versions from standard API --- moto/iam/models.py | 8 ++++++-- tests/test_iam/test_iam.py | 22 ++++++++++++++-------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/moto/iam/models.py b/moto/iam/models.py index c8f95f3c9..7b7c14b6c 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -37,7 +37,7 @@ class Policy(BaseModel): description=None, document=None, path=None): - self.document = document or {} + #self.document = document or {} self.name = name self.attachment_count = 0 @@ -45,7 +45,7 @@ class Policy(BaseModel): self.id = random_policy_id() self.path = path or '/' self.default_version_id = default_version_id or 'v1' - self.versions = [] + self.versions = [PolicyVersion(self.arn, document, True)] self.create_datetime = datetime.now(pytz.utc) self.update_datetime = datetime.now(pytz.utc) @@ -582,6 +582,7 @@ class IAMBackend(BaseBackend): raise IAMNotFoundException("Policy not found") version = PolicyVersion(policy_arn, policy_document, set_as_default) policy.versions.append(version) + version.version_id = 'v{0}'.format(len(policy.versions)) if set_as_default: policy.default_version_id = version.version_id return version @@ -605,6 +606,9 @@ class IAMBackend(BaseBackend): policy = self.get_policy(policy_arn) if not policy: raise IAMNotFoundException("Policy not found") + if version_id == policy.default_version_id: + raise IAMConflictException( + "Cannot delete the default version of a policy") for i, v in enumerate(policy.versions): if v.version_id == version_id: del policy.versions[i] diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 61cd073fe..bc23ff712 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -296,6 +296,7 @@ def test_get_policy(): PolicyArn="arn:aws:iam::123456789012:policy/TestGetPolicy") response['Policy']['Arn'].should.equal("arn:aws:iam::123456789012:policy/TestGetPolicy") + @mock_iam def test_get_policy_version(): conn = boto3.client('iam', region_name='us-east-1') @@ -323,17 +324,22 @@ def test_list_policy_versions(): PolicyArn="arn:aws:iam::123456789012:policy/TestListPolicyVersions") conn.create_policy( PolicyName="TestListPolicyVersions", - PolicyDocument='{"some":"policy"}') - conn.create_policy_version( - PolicyArn="arn:aws:iam::123456789012:policy/TestListPolicyVersions", PolicyDocument='{"first":"policy"}') + versions = conn.list_policy_versions( + PolicyArn="arn:aws:iam::123456789012:policy/TestListPolicyVersions") + versions.get('Versions')[0].get('VersionId').should.equal('v1') + conn.create_policy_version( PolicyArn="arn:aws:iam::123456789012:policy/TestListPolicyVersions", PolicyDocument='{"second":"policy"}') + conn.create_policy_version( + PolicyArn="arn:aws:iam::123456789012:policy/TestListPolicyVersions", + PolicyDocument='{"third":"policy"}') versions = conn.list_policy_versions( PolicyArn="arn:aws:iam::123456789012:policy/TestListPolicyVersions") - versions.get('Versions')[0].get('Document').should.equal({'first': 'policy'}) + print(versions.get('Versions')) versions.get('Versions')[1].get('Document').should.equal({'second': 'policy'}) + versions.get('Versions')[2].get('Document').should.equal({'third': 'policy'}) @mock_iam @@ -341,20 +347,20 @@ def test_delete_policy_version(): conn = boto3.client('iam', region_name='us-east-1') conn.create_policy( PolicyName="TestDeletePolicyVersion", - PolicyDocument='{"some":"policy"}') + PolicyDocument='{"first":"policy"}') conn.create_policy_version( PolicyArn="arn:aws:iam::123456789012:policy/TestDeletePolicyVersion", - PolicyDocument='{"first":"policy"}') + PolicyDocument='{"second":"policy"}') with assert_raises(ClientError): conn.delete_policy_version( PolicyArn="arn:aws:iam::123456789012:policy/TestDeletePolicyVersion", VersionId='v2-nope-this-does-not-exist') conn.delete_policy_version( PolicyArn="arn:aws:iam::123456789012:policy/TestDeletePolicyVersion", - VersionId='v1') + VersionId='v2') versions = conn.list_policy_versions( PolicyArn="arn:aws:iam::123456789012:policy/TestDeletePolicyVersion") - len(versions.get('Versions')).should.equal(0) + len(versions.get('Versions')).should.equal(1) @mock_iam_deprecated() From de4d5d9d6250669b0e0f84ccdbccb879556aad76 Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Wed, 8 Aug 2018 11:13:25 -0400 Subject: [PATCH 012/106] Correct flake8 failures --- moto/iam/models.py | 1 - moto/iam/responses.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/moto/iam/models.py b/moto/iam/models.py index 7b7c14b6c..4d884fa2f 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -37,7 +37,6 @@ class Policy(BaseModel): description=None, document=None, path=None): - #self.document = document or {} self.name = name self.attachment_count = 0 diff --git a/moto/iam/responses.py b/moto/iam/responses.py index e77cec358..9e8d21396 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -62,7 +62,7 @@ class IamResponse(BaseResponse): policy_arn = self._get_param('PolicyArn') policy = iam_backend.get_policy(policy_arn) template = self.response_template(GET_POLICY_TEMPLATE) - return template.render(policy=policy) + return template.render(policy=policy) def list_attached_role_policies(self): marker = self._get_param('Marker') @@ -611,7 +611,7 @@ GET_POLICY_TEMPLATE = """ {{ policy.name }} - {{ policy.description }} + {{ policy.description }} {{ policy.default_version_id }} {{ policy.id }} {{ policy.path }} From 9d1c66531070774be508f17f8d8dd12429fed2e9 Mon Sep 17 00:00:00 2001 From: Nathan Sutton Date: Wed, 8 Aug 2018 19:14:56 -0500 Subject: [PATCH 013/106] Force receive_message_wait_time_seconds to be int When a queue is created with the ReceiveMessageWaitTimeSeconds attribute the value is never converted to an integer. When the ReceiveMessage action is called it tries to compare the string ReceiveMessageWaitTimeSeconds with the min and max wait times which raises a TypeError. The solution is to convert this value to an integer before comparing. --- moto/sqs/responses.py | 2 +- tests/test_sqs/test_sqs.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/moto/sqs/responses.py b/moto/sqs/responses.py index c489d7118..b4f64b14e 100644 --- a/moto/sqs/responses.py +++ b/moto/sqs/responses.py @@ -336,7 +336,7 @@ class SQSResponse(BaseResponse): try: wait_time = int(self.querystring.get("WaitTimeSeconds")[0]) except TypeError: - wait_time = queue.receive_message_wait_time_seconds + wait_time = int(queue.receive_message_wait_time_seconds) if wait_time < 0 or wait_time > 20: return self._error( diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index d3e4ca917..bd6bbf0fb 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -1195,3 +1195,16 @@ def test_receive_messages_with_message_group_id_on_visibility_timeout(): messages = queue.receive_messages() messages.should.have.length_of(1) messages[0].message_id.should.equal(message.message_id) + +@mock_sqs +def test_receive_message_for_queue_with_receive_message_wait_time_seconds_set(): + sqs = boto3.resource('sqs', region_name='us-east-1') + + queue = sqs.create_queue( + QueueName='test-queue', + Attributes={ + 'ReceiveMessageWaitTimeSeconds': '2', + } + ) + + queue.receive_messages() From 8393c7f20be1395fcd1f8d3ccc19e946d679ee0b Mon Sep 17 00:00:00 2001 From: Nathan Sutton Date: Wed, 8 Aug 2018 21:10:13 -0500 Subject: [PATCH 014/106] Don't error on double create queue with same attrs Creating a queue a second time with the same attributes should not raise an error. This change makes it work correctly. --- moto/sqs/models.py | 20 ++++++++++++++++---- tests/test_sqs/test_sqs.py | 27 +++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/moto/sqs/models.py b/moto/sqs/models.py index b8db356e9..f3262a988 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -385,10 +385,22 @@ class SQSBackend(BaseBackend): def create_queue(self, name, **kwargs): queue = self.queues.get(name) if queue: - # Queue already exist. If attributes don't match, throw error - for key, value in kwargs.items(): - if getattr(queue, camelcase_to_underscores(key)) != value: - raise QueueAlreadyExists("The specified queue already exists.") + try: + kwargs.pop('region') + except KeyError: + pass + + new_queue = Queue(name, region=self.region_name, **kwargs) + + queue_attributes = queue.attributes + new_queue_attributes = new_queue.attributes + + for key in ['CreatedTimestamp', 'LastModifiedTimestamp']: + queue_attributes.pop(key) + new_queue_attributes.pop(key) + + if queue_attributes != new_queue_attributes: + raise QueueAlreadyExists("The specified queue already exists.") else: try: kwargs.pop('region') diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index d3e4ca917..5d90d38b2 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -40,6 +40,33 @@ def test_create_fifo_queue_fail(): raise RuntimeError('Should of raised InvalidParameterValue Exception') +@mock_sqs +def test_create_queue_with_same_attributes(): + sqs = boto3.client('sqs', region_name='us-east-1') + + dlq_url = sqs.create_queue(QueueName='test-queue-dlq')['QueueUrl'] + dlq_arn = sqs.get_queue_attributes(QueueUrl=dlq_url)['Attributes']['QueueArn'] + + attributes = { + 'DelaySeconds': '900', + 'MaximumMessageSize': '262144', + 'MessageRetentionPeriod': '1209600', + 'ReceiveMessageWaitTimeSeconds': '20', + 'RedrivePolicy': '{"deadLetterTargetArn": "%s", "maxReceiveCount": 100}' % (dlq_arn), + 'VisibilityTimeout': '43200' + } + + sqs.create_queue( + QueueName='test-queue', + Attributes=attributes + ) + + sqs.create_queue( + QueueName='test-queue', + Attributes=attributes + ) + + @mock_sqs def test_create_queue_with_different_attributes_fail(): sqs = boto3.client('sqs', region_name='us-east-1') From 1b1fc4c030c446e23165cf28ad503de340ba645a Mon Sep 17 00:00:00 2001 From: Mike Liu Date: Wed, 25 Jul 2018 14:24:01 -0400 Subject: [PATCH 015/106] Support getting cloudformation targets for the SSM backend. SendCommand allows filtering for instances by tags. This adds support for filtering by a cloud formation stack resource when creating command invocations. --- moto/ssm/models.py | 32 ++++++++++++- tests/test_ssm/test_ssm_boto3.py | 78 +++++++++++++++++++++++++++++++- 2 files changed, 107 insertions(+), 3 deletions(-) diff --git a/moto/ssm/models.py b/moto/ssm/models.py index b3e34eae4..65d2f7b87 100644 --- a/moto/ssm/models.py +++ b/moto/ssm/models.py @@ -5,10 +5,12 @@ from collections import defaultdict from moto.core import BaseBackend, BaseModel from moto.core.exceptions import RESTError from moto.ec2 import ec2_backends +from moto.cloudformation import cloudformation_backends import datetime import time import uuid +import itertools class Parameter(BaseModel): @@ -67,7 +69,7 @@ class Command(BaseModel): instance_ids=None, max_concurrency='', max_errors='', notification_config=None, output_s3_bucket_name='', output_s3_key_prefix='', output_s3_region='', parameters=None, - service_role_arn='', targets=None): + service_role_arn='', targets=None, backend_region='us-east-1'): if instance_ids is None: instance_ids = [] @@ -105,6 +107,14 @@ class Command(BaseModel): self.parameters = parameters self.service_role_arn = service_role_arn self.targets = targets + self.backend_region = backend_region + + # Get instance ids from a cloud formation stack target. + stack_instance_ids = [self.get_instance_ids_by_stack_ids(target['Values']) for + target in self.targets if + target['Key'] == 'tag:aws:cloudformation:stack-name'] + + self.instance_ids += list(itertools.chain.from_iterable(stack_instance_ids)) # Create invocations with a single run command plugin. self.invocations = [] @@ -112,6 +122,18 @@ class Command(BaseModel): self.invocations.append( self.invocation_response(instance_id, "aws:runShellScript")) + def get_instance_ids_by_stack_ids(self, stack_ids): + instance_ids = [] + cloudformation_backend = cloudformation_backends[self.backend_region] + for stack_id in stack_ids: + stack_resources = cloudformation_backend.list_stack_resources(stack_id) + instance_resources = [ + instance.id for instance in stack_resources + if instance.type == "AWS::EC2::Instance"] + instance_ids.extend(instance_resources) + + return instance_ids + def response_object(self): r = { 'CommandId': self.command_id, @@ -191,6 +213,11 @@ class SimpleSystemManagerBackend(BaseBackend): self._resource_tags = defaultdict(lambda: defaultdict(dict)) self._commands = [] + # figure out what region we're in + for region, backend in ssm_backends.items(): + if backend == self: + self._region = region + def delete_parameter(self, name): try: del self._parameters[name] @@ -311,7 +338,8 @@ class SimpleSystemManagerBackend(BaseBackend): output_s3_region=kwargs.get('OutputS3Region', ''), parameters=kwargs.get('Parameters', {}), service_role_arn=kwargs.get('ServiceRoleArn', ''), - targets=kwargs.get('Targets', [])) + targets=kwargs.get('Targets', []), + backend_region=self._region) self._commands.append(command) return { diff --git a/tests/test_ssm/test_ssm_boto3.py b/tests/test_ssm/test_ssm_boto3.py index 8df565cf6..f8ef3a237 100644 --- a/tests/test_ssm/test_ssm_boto3.py +++ b/tests/test_ssm/test_ssm_boto3.py @@ -5,11 +5,12 @@ import botocore.exceptions import sure # noqa import datetime import uuid +import json from botocore.exceptions import ClientError from nose.tools import assert_raises -from moto import mock_ssm +from moto import mock_ssm, mock_cloudformation @mock_ssm @@ -708,3 +709,78 @@ def test_get_command_invocation(): CommandId=cmd_id, InstanceId=instance_id, PluginName='FAKE') + +@mock_ssm +@mock_cloudformation +def test_get_command_invocations_from_stack(): + stack_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Description": "Test Stack", + "Resources": { + "EC2Instance1": { + "Type": "AWS::EC2::Instance", + "Properties": { + "ImageId": "ami-test-image-id", + "KeyName": "test", + "InstanceType": "t2.micro", + "Tags": [ + { + "Key": "Test Description", + "Value": "Test tag" + }, + { + "Key": "Test Name", + "Value": "Name tag for tests" + } + ] + } + } + }, + "Outputs": { + "test": { + "Description": "Test Output", + "Value": "Test output value", + "Export": { + "Name": "Test value to export" + } + }, + "PublicIP": { + "Value": "Test public ip" + } + } + } + + cloudformation_client = boto3.client( + 'cloudformation', + region_name='us-east-1') + + stack_template_str = json.dumps(stack_template) + + response = cloudformation_client.create_stack( + StackName='test_stack', + TemplateBody=stack_template_str, + Capabilities=('CAPABILITY_IAM', )) + + client = boto3.client('ssm', region_name='us-east-1') + + ssm_document = 'AWS-RunShellScript' + params = {'commands': ['#!/bin/bash\necho \'hello world\'']} + + response = client.send_command( + Targets=[{ + 'Key': 'tag:aws:cloudformation:stack-name', + 'Values': ('test_stack', )}], + DocumentName=ssm_document, + Parameters=params, + OutputS3Region='us-east-2', + OutputS3BucketName='the-bucket', + OutputS3KeyPrefix='pref') + + cmd = response['Command'] + cmd_id = cmd['CommandId'] + instance_ids = cmd['InstanceIds'] + + invocation_response = client.get_command_invocation( + CommandId=cmd_id, + InstanceId=instance_ids[0], + PluginName='aws:runShellScript') From de532b93b78a01763ac622bd538ce12c41b86893 Mon Sep 17 00:00:00 2001 From: Mike Liu Date: Thu, 9 Aug 2018 10:53:32 -0400 Subject: [PATCH 016/106] Fix flake8. Remove extra whitespace. --- moto/ssm/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/moto/ssm/models.py b/moto/ssm/models.py index 65d2f7b87..f16a7d981 100644 --- a/moto/ssm/models.py +++ b/moto/ssm/models.py @@ -202,7 +202,6 @@ class Command(BaseModel): 'InvocationDoesNotExist', 'An error occurred (InvocationDoesNotExist) when calling the GetCommandInvocation operation') - return invocation From df28ec03d2cc8c44387fa0ac41f57a3431dd6a45 Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Fri, 10 Aug 2018 16:07:27 +0200 Subject: [PATCH 017/106] Added extra test --- tests/test_cognitoidp/test_cognitoidp.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index fda24146d..d1d57f307 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -4,8 +4,10 @@ import boto3 import json import os import uuid +import jwt from jose import jws + from moto import mock_cognitoidp import sure # noqa @@ -446,6 +448,9 @@ def authentication_flow(conn): result["AuthenticationResult"]["IdToken"].should_not.be.none result["AuthenticationResult"]["AccessToken"].should_not.be.none + jwt.decode( + result["AuthenticationResult"]["AccessToken"], verify=False + ).get(user_attribute_name, '').should.be.equal(user_attribute_value) return { "user_pool_id": user_pool_id, From aae3ab6b900b879841e75474a9fcc716f238009f Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Fri, 10 Aug 2018 16:26:09 +0200 Subject: [PATCH 018/106] Added package dependency --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 655be0616..11f59e069 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,3 +15,4 @@ click==6.7 inflection==0.3.1 lxml==4.0.0 beautifulsoup4==4.6.0 +PyJWT==1.6.4 From f9762a5ecb98599109172d0af5295b6f13d1c57f Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Fri, 10 Aug 2018 16:52:48 +0200 Subject: [PATCH 019/106] Removing failing test in order to check coverage and Travis test --- requirements-dev.txt | 1 - tests/test_cognitoidp/test_cognitoidp.py | 4 ---- 2 files changed, 5 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 11f59e069..655be0616 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,4 +15,3 @@ click==6.7 inflection==0.3.1 lxml==4.0.0 beautifulsoup4==4.6.0 -PyJWT==1.6.4 diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index d1d57f307..98153feef 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -4,7 +4,6 @@ import boto3 import json import os import uuid -import jwt from jose import jws @@ -448,9 +447,6 @@ def authentication_flow(conn): result["AuthenticationResult"]["IdToken"].should_not.be.none result["AuthenticationResult"]["AccessToken"].should_not.be.none - jwt.decode( - result["AuthenticationResult"]["AccessToken"], verify=False - ).get(user_attribute_name, '').should.be.equal(user_attribute_value) return { "user_pool_id": user_pool_id, From b75e78a3cdca3e5158702623364700ca29400c45 Mon Sep 17 00:00:00 2001 From: Manuel Adarve Date: Mon, 13 Aug 2018 17:09:39 +0200 Subject: [PATCH 020/106] ECS: Added schedulingStrategy support Co-authored-by: Jonas Gratz --- moto/ecs/models.py | 8 ++- moto/ecs/responses.py | 3 +- tests/test_ecs/test_ecs_boto3.py | 115 +++++++++++++++++++++++++++++++ 3 files changed, 122 insertions(+), 4 deletions(-) diff --git a/moto/ecs/models.py b/moto/ecs/models.py index 55fb4d4d9..d00853843 100644 --- a/moto/ecs/models.py +++ b/moto/ecs/models.py @@ -179,7 +179,7 @@ class Task(BaseObject): class Service(BaseObject): - def __init__(self, cluster, service_name, task_definition, desired_count, load_balancers=None): + def __init__(self, cluster, service_name, task_definition, desired_count, load_balancers=None, scheduling_strategy=None): self.cluster_arn = cluster.arn self.arn = 'arn:aws:ecs:us-east-1:012345678910:service/{0}'.format( service_name) @@ -202,6 +202,7 @@ class Service(BaseObject): } ] self.load_balancers = load_balancers if load_balancers is not None else [] + self.scheduling_strategy = scheduling_strategy if scheduling_strategy is not None else 'REPLICA' self.pending_count = 0 @property @@ -214,6 +215,7 @@ class Service(BaseObject): del response_object['name'], response_object['arn'] response_object['serviceName'] = self.name response_object['serviceArn'] = self.arn + response_object['schedulingStrategy'] = self.scheduling_strategy for deployment in response_object['deployments']: if isinstance(deployment['createdAt'], datetime): @@ -655,7 +657,7 @@ class EC2ContainerServiceBackend(BaseBackend): raise Exception("Could not find task {} on cluster {}".format( task_str, cluster_name)) - def create_service(self, cluster_str, service_name, task_definition_str, desired_count, load_balancers=None): + def create_service(self, cluster_str, service_name, task_definition_str, desired_count, load_balancers=None, scheduling_strategy=None): cluster_name = cluster_str.split('/')[-1] if cluster_name in self.clusters: cluster = self.clusters[cluster_name] @@ -665,7 +667,7 @@ class EC2ContainerServiceBackend(BaseBackend): desired_count = desired_count if desired_count is not None else 0 service = Service(cluster, service_name, - task_definition, desired_count, load_balancers) + task_definition, desired_count, load_balancers, scheduling_strategy) cluster_service_pair = '{0}:{1}'.format(cluster_name, service_name) self.services[cluster_service_pair] = service diff --git a/moto/ecs/responses.py b/moto/ecs/responses.py index 9455d7a28..e0bfefc02 100644 --- a/moto/ecs/responses.py +++ b/moto/ecs/responses.py @@ -154,8 +154,9 @@ class EC2ContainerServiceResponse(BaseResponse): task_definition_str = self._get_param('taskDefinition') desired_count = self._get_int_param('desiredCount') load_balancers = self._get_param('loadBalancers') + scheduling_strategy = self._get_param('schedulingStrategy') service = self.ecs_backend.create_service( - cluster_str, service_name, task_definition_str, desired_count, load_balancers) + cluster_str, service_name, task_definition_str, desired_count, load_balancers, scheduling_strategy) return json.dumps({ 'service': service.response_object }) diff --git a/tests/test_ecs/test_ecs_boto3.py b/tests/test_ecs/test_ecs_boto3.py index bf72dc230..70c1463ee 100644 --- a/tests/test_ecs/test_ecs_boto3.py +++ b/tests/test_ecs/test_ecs_boto3.py @@ -304,6 +304,52 @@ def test_create_service(): response['service']['status'].should.equal('ACTIVE') response['service']['taskDefinition'].should.equal( 'arn:aws:ecs:us-east-1:012345678910:task-definition/test_ecs_task:1') + response['service']['schedulingStrategy'].should.equal('REPLICA') + +@mock_ecs +def test_create_service_scheduling_strategy(): + client = boto3.client('ecs', region_name='us-east-1') + _ = client.create_cluster( + clusterName='test_ecs_cluster' + ) + _ = client.register_task_definition( + family='test_ecs_task', + containerDefinitions=[ + { + 'name': 'hello_world', + 'image': 'docker/hello-world:latest', + 'cpu': 1024, + 'memory': 400, + 'essential': True, + 'environment': [{ + 'name': 'AWS_ACCESS_KEY_ID', + 'value': 'SOME_ACCESS_KEY' + }], + 'logConfiguration': {'logDriver': 'json-file'} + } + ] + ) + response = client.create_service( + cluster='test_ecs_cluster', + serviceName='test_ecs_service', + taskDefinition='test_ecs_task', + desiredCount=2, + schedulingStrategy='DAEMON', + ) + response['service']['clusterArn'].should.equal( + 'arn:aws:ecs:us-east-1:012345678910:cluster/test_ecs_cluster') + response['service']['desiredCount'].should.equal(2) + len(response['service']['events']).should.equal(0) + len(response['service']['loadBalancers']).should.equal(0) + response['service']['pendingCount'].should.equal(0) + response['service']['runningCount'].should.equal(0) + response['service']['serviceArn'].should.equal( + 'arn:aws:ecs:us-east-1:012345678910:service/test_ecs_service') + response['service']['serviceName'].should.equal('test_ecs_service') + response['service']['status'].should.equal('ACTIVE') + response['service']['taskDefinition'].should.equal( + 'arn:aws:ecs:us-east-1:012345678910:task-definition/test_ecs_task:1') + response['service']['schedulingStrategy'].should.equal('DAEMON') @mock_ecs @@ -411,6 +457,72 @@ def test_describe_services(): response['services'][0]['deployments'][0]['status'].should.equal('PRIMARY') +@mock_ecs +def test_describe_services_scheduling_strategy(): + client = boto3.client('ecs', region_name='us-east-1') + _ = client.create_cluster( + clusterName='test_ecs_cluster' + ) + _ = client.register_task_definition( + family='test_ecs_task', + containerDefinitions=[ + { + 'name': 'hello_world', + 'image': 'docker/hello-world:latest', + 'cpu': 1024, + 'memory': 400, + 'essential': True, + 'environment': [{ + 'name': 'AWS_ACCESS_KEY_ID', + 'value': 'SOME_ACCESS_KEY' + }], + 'logConfiguration': {'logDriver': 'json-file'} + } + ] + ) + _ = client.create_service( + cluster='test_ecs_cluster', + serviceName='test_ecs_service1', + taskDefinition='test_ecs_task', + desiredCount=2 + ) + _ = client.create_service( + cluster='test_ecs_cluster', + serviceName='test_ecs_service2', + taskDefinition='test_ecs_task', + desiredCount=2, + schedulingStrategy='DAEMON' + ) + _ = client.create_service( + cluster='test_ecs_cluster', + serviceName='test_ecs_service3', + taskDefinition='test_ecs_task', + desiredCount=2 + ) + response = client.describe_services( + cluster='test_ecs_cluster', + services=['test_ecs_service1', + 'arn:aws:ecs:us-east-1:012345678910:service/test_ecs_service2', + 'test_ecs_service3'] + ) + len(response['services']).should.equal(3) + response['services'][0]['serviceArn'].should.equal( + 'arn:aws:ecs:us-east-1:012345678910:service/test_ecs_service1') + response['services'][0]['serviceName'].should.equal('test_ecs_service1') + response['services'][1]['serviceArn'].should.equal( + 'arn:aws:ecs:us-east-1:012345678910:service/test_ecs_service2') + response['services'][1]['serviceName'].should.equal('test_ecs_service2') + + response['services'][0]['deployments'][0]['desiredCount'].should.equal(2) + response['services'][0]['deployments'][0]['pendingCount'].should.equal(2) + response['services'][0]['deployments'][0]['runningCount'].should.equal(0) + response['services'][0]['deployments'][0]['status'].should.equal('PRIMARY') + + response['services'][0]['schedulingStrategy'].should.equal('REPLICA') + response['services'][1]['schedulingStrategy'].should.equal('DAEMON') + response['services'][2]['schedulingStrategy'].should.equal('REPLICA') + + @mock_ecs def test_update_service(): client = boto3.client('ecs', region_name='us-east-1') @@ -449,6 +561,7 @@ def test_update_service(): desiredCount=0 ) response['service']['desiredCount'].should.equal(0) + response['service']['schedulingStrategy'].should.equal('REPLICA') @mock_ecs @@ -515,8 +628,10 @@ def test_delete_service(): 'arn:aws:ecs:us-east-1:012345678910:service/test_ecs_service') response['service']['serviceName'].should.equal('test_ecs_service') response['service']['status'].should.equal('ACTIVE') + response['service']['schedulingStrategy'].should.equal('REPLICA') response['service']['taskDefinition'].should.equal( 'arn:aws:ecs:us-east-1:012345678910:task-definition/test_ecs_task:1') + @mock_ec2 From 354d48fb8da739ee7134df41ff55128b2842e821 Mon Sep 17 00:00:00 2001 From: Chih-Hsuan Yen Date: Tue, 24 Jul 2018 21:11:35 +0800 Subject: [PATCH 021/106] Fix HTTPretty on Python 3.7 This is a revised backport of https://github.com/gabrielfalcao/HTTPretty/commit/5776d97da3992b9071db5e21faf175f6e8729060 and the following fixup https://github.com/gabrielfalcao/HTTPretty/pull/341 --- moto/packages/httpretty/core.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/moto/packages/httpretty/core.py b/moto/packages/httpretty/core.py index e0f3a7e69..8ad9168a5 100644 --- a/moto/packages/httpretty/core.py +++ b/moto/packages/httpretty/core.py @@ -85,6 +85,7 @@ old_socksocket = None old_ssl_wrap_socket = None old_sslwrap_simple = None old_sslsocket = None +old_sslcontext_wrap_socket = None if PY3: # pragma: no cover basestring = (bytes, str) @@ -100,6 +101,10 @@ try: # pragma: no cover if not PY3: old_sslwrap_simple = ssl.sslwrap_simple old_sslsocket = ssl.SSLSocket + try: + old_sslcontext_wrap_socket = ssl.SSLContext.wrap_socket + except AttributeError: + pass except ImportError: # pragma: no cover ssl = None @@ -281,7 +286,7 @@ class fakesock(object): return { 'notAfter': shift.strftime('%b %d %H:%M:%S GMT'), 'subjectAltName': ( - ('DNS', '*%s' % self._host), + ('DNS', '*.%s' % self._host), ('DNS', self._host), ('DNS', '*'), ), @@ -772,7 +777,7 @@ class URIMatcher(object): def __init__(self, uri, entries, match_querystring=False): self._match_querystring = match_querystring - if type(uri).__name__ == 'SRE_Pattern': + if type(uri).__name__ in ('SRE_Pattern', 'Pattern'): self.regex = uri result = urlsplit(uri.pattern) if result.scheme == 'https': @@ -1012,6 +1017,10 @@ class httpretty(HttpBaseClass): if ssl: ssl.wrap_socket = old_ssl_wrap_socket ssl.SSLSocket = old_sslsocket + try: + ssl.SSLContext.wrap_socket = old_sslcontext_wrap_socket + except AttributeError: + pass ssl.__dict__['wrap_socket'] = old_ssl_wrap_socket ssl.__dict__['SSLSocket'] = old_sslsocket @@ -1058,6 +1067,14 @@ class httpretty(HttpBaseClass): ssl.wrap_socket = fake_wrap_socket ssl.SSLSocket = FakeSSLSocket + try: + def fake_sslcontext_wrap_socket(cls, *args, **kwargs): + return fake_wrap_socket(*args, **kwargs) + + ssl.SSLContext.wrap_socket = fake_sslcontext_wrap_socket + except AttributeError: + pass + ssl.__dict__['wrap_socket'] = fake_wrap_socket ssl.__dict__['SSLSocket'] = FakeSSLSocket From d00300bc1b79d4f059a60683a56b559a96c6049f Mon Sep 17 00:00:00 2001 From: Chih-Hsuan Yen Date: Tue, 24 Jul 2018 21:14:52 +0800 Subject: [PATCH 022/106] Test Python 3.7 on Travis CI --- .travis.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.travis.yml b/.travis.yml index f1b7ac40d..de22818b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,19 @@ python: env: - TEST_SERVER_MODE=false - TEST_SERVER_MODE=true +# Due to incomplete Python 3.7 support on Travis CI ( +# https://github.com/travis-ci/travis-ci/issues/9815), +# using a matrix is necessary +matrix: + include: + - python: 3.7 + env: TEST_SERVER_MODE=false + dist: xenial + sudo: true + - python: 3.7 + env: TEST_SERVER_MODE=true + dist: xenial + sudo: true before_install: - export BOTO_CONFIG=/dev/null install: From 52248589bf162d4f74aaad517ebac47e26c0b1a3 Mon Sep 17 00:00:00 2001 From: Chih-Hsuan Yen Date: Tue, 24 Jul 2018 21:32:43 +0800 Subject: [PATCH 023/106] Update lxml and sure for Python 3.7 compatibility --- requirements-dev.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 655be0616..111cd5f3f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ -r requirements.txt mock nose -sure==1.2.24 +sure==1.4.11 coverage flake8==3.5.0 freezegun @@ -13,5 +13,5 @@ six>=1.9 prompt-toolkit==1.0.14 click==6.7 inflection==0.3.1 -lxml==4.0.0 +lxml==4.2.3 beautifulsoup4==4.6.0 From 32c32ee5d764064327ba9881bc7cb310f2fa1296 Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Wed, 15 Aug 2018 10:29:58 -0700 Subject: [PATCH 024/106] Moto-1781: Implement rotate_secret response. --- moto/secretsmanager/responses.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/moto/secretsmanager/responses.py b/moto/secretsmanager/responses.py index c50c6a6e1..b8b6872a8 100644 --- a/moto/secretsmanager/responses.py +++ b/moto/secretsmanager/responses.py @@ -50,3 +50,15 @@ class SecretsManagerResponse(BaseResponse): return secretsmanager_backends[self.region].describe_secret( secret_id=secret_id ) + + def rotate_secret(self): + client_request_token = self._get_param('ClientRequestToken') + rotation_lambda_arn = self._get_param('RotationLambdaARN') + rotation_rules = self._get_param('RotationRules') + secret_id = self._get_param('SecretId') + return secretsmanager_backends[self.region].rotate_secret( + secret_id=secret_id, + client_request_token=client_request_token, + rotation_lambda_arn=rotation_lambda_arn, + rotation_rules=rotation_rules + ) From 0b0d16e8ec2b7b1b78d6a44a843d52e5db3bbdbc Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Wed, 15 Aug 2018 10:33:38 -0700 Subject: [PATCH 025/106] Opportunistic update to unit test for consistency. --- tests/test_secretsmanager/test_secretsmanager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index c631fabb0..d3fc85467 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -26,13 +26,13 @@ def test_get_secret_that_does_not_exist(): result = conn.get_secret_value(SecretId='i-dont-exist') @mock_secretsmanager -def test_get_secret_with_mismatched_id(): +def test_get_secret_that_does_not_match(): conn = boto3.client('secretsmanager', region_name='us-west-2') create_secret = conn.create_secret(Name='java-util-test-password', SecretString="foosecret") with assert_raises(ClientError): - result = conn.get_secret_value(SecretId='i-dont-exist') + result = conn.get_secret_value(SecretId='i-dont-match') @mock_secretsmanager def test_create_secret(): From 69a78ba7c98dba3886a63d7923fa18bea94eaea1 Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Wed, 15 Aug 2018 10:52:30 -0700 Subject: [PATCH 026/106] Moto-1781: Create initial stub of rotate_secret and tests. --- moto/secretsmanager/models.py | 15 +++++++++++++++ tests/test_secretsmanager/test_secretsmanager.py | 16 ++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index c60feb530..4d9c4ecbe 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -36,6 +36,7 @@ class SecretsManagerBackend(BaseBackend): self.rotation_enabled = False self.rotation_lambda_arn = '' self.auto_rotate_after_days = 0 + self.version_id = '' def reset(self): region_name = self.region @@ -105,6 +106,20 @@ class SecretsManagerBackend(BaseBackend): return response + def rotate_secret(self, secret_id, client_request_token=None, + rotation_lambda_arn=None, rotation_rules=None): + + if not self._is_valid_identifier(secret_id): + raise ResourceNotFoundException + + response = json.dumps({ + "ARN": secret_arn(self.region, self.secret_id), + "Name": self.name, + "VersionId": self.version_id + }) + + return response + def get_random_password(self, password_length, exclude_characters, exclude_numbers, exclude_punctuation, exclude_uppercase, diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index d3fc85467..bbc87e5c1 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -179,3 +179,19 @@ def test_describe_secret_that_does_not_match(): with assert_raises(ClientError): result = conn.get_secret_value(SecretId='i-dont-match') + +@mock_secretsmanager +def test_rotate_secret_that_does_not_exist(): + conn = boto3.client('secretsmanager', 'us-west-2') + + with assert_raises(ClientError): + result = conn.rotate_secret(SecretId='i-dont-exist') + +@mock_secretsmanager +def test_rotate_secret_that_does_not_match(): + conn = boto3.client('secretsmanager', region_name='us-west-2') + conn.create_secret(Name='test-secret', + SecretString='foosecret') + + with assert_raises(ClientError): + result = conn.rotate_secret(SecretId='i-dont-match') From 43277a59b9df04f6017265c7a57fd84e7d697f10 Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Wed, 15 Aug 2018 11:04:44 -0700 Subject: [PATCH 027/106] Moto-1781: finish implementation of rotate_secret and add happy test. - Implement RotateSecret to support initial setup of secret rotation. Moto's implementation of secrets is currently flat and needs to gain some dimension before full rotation can be simulated. - Add the happy path unit test. --- moto/secretsmanager/models.py | 36 +++++++++++++++++++ .../test_secretsmanager.py | 16 +++++++++ 2 files changed, 52 insertions(+) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 4d9c4ecbe..1404a0ec8 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -109,9 +109,45 @@ class SecretsManagerBackend(BaseBackend): def rotate_secret(self, secret_id, client_request_token=None, rotation_lambda_arn=None, rotation_rules=None): + rotation_days = 'AutomaticallyAfterDays' + if not self._is_valid_identifier(secret_id): raise ResourceNotFoundException + if client_request_token: + token_length = len(client_request_token) + if token_length < 32 or token_length > 64: + msg = ( + 'ClientRequestToken ' + 'must be 32-64 characters long.' + ) + raise InvalidParameterException(msg) + + if rotation_lambda_arn: + if len(rotation_lambda_arn) > 2048: + msg = ( + 'RotationLambdaARN ' + 'must <= 2048 characters long.' + ) + raise InvalidParameterException(msg) + + if rotation_rules: + if rotation_days in rotation_rules: + rotation_period = rotation_rules[rotation_days] + if rotation_period < 1 or rotation_period > 1000: + msg = ( + 'RotationRules.AutomaticallyAfterDays ' + 'must be within 1-1000.' + ) + raise InvalidParameterException(msg) + + self.version_id = client_request_token or '' + self.rotation_lambda_arn = rotation_lambda_arn or '' + if rotation_rules: + self.auto_rotate_after_days = rotation_rules.get(rotation_days, 0) + if self.auto_rotate_after_days > 0: + self.rotation_enabled = True + response = json.dumps({ "ARN": secret_arn(self.region, self.secret_id), "Name": self.name, diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index bbc87e5c1..4b15a3c01 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -180,6 +180,22 @@ def test_describe_secret_that_does_not_match(): with assert_raises(ClientError): result = conn.get_secret_value(SecretId='i-dont-match') +@mock_secretsmanager +def test_rotate_secret(): + secret_name = 'test-secret' + conn = boto3.client('secretsmanager', region_name='us-west-2') + conn.create_secret(Name=secret_name, + SecretString='foosecret') + + rotated_secret = conn.rotate_secret(SecretId=secret_name) + + assert rotated_secret + assert rotated_secret['ARN'] == ( + 'arn:aws:secretsmanager:us-west-2:1234567890:secret:test-secret-rIjad' + ) + assert rotated_secret['Name'] == secret_name + assert rotated_secret['VersionId'] != '' + @mock_secretsmanager def test_rotate_secret_that_does_not_exist(): conn = boto3.client('secretsmanager', 'us-west-2') From 6e716a5926f12e9292125db516b3ab8614ed224f Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Wed, 15 Aug 2018 11:06:26 -0700 Subject: [PATCH 028/106] Moto-1781: Update implementation coverage. --- IMPLEMENTATION_COVERAGE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 938cc3549..b6524b451 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3645,7 +3645,7 @@ - [ ] put_attributes - [ ] select -## secretsmanager - 27% implemented +## secretsmanager - 33% implemented - [ ] cancel_rotate_secret - [X] create_secret - [ ] delete_secret @@ -3656,7 +3656,7 @@ - [ ] list_secrets - [ ] put_secret_value - [ ] restore_secret -- [ ] rotate_secret +- [X] rotate_secret - [ ] tag_resource - [ ] untag_resource - [ ] update_secret From b2997304b4dd76040f8bca005683427d4ce5fdea Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Wed, 15 Aug 2018 11:20:29 -0700 Subject: [PATCH 029/106] Moto-1781: Implement standalone unit tests to cover bad parameters. --- .../test_secretsmanager.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index 4b15a3c01..3e7b31c72 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -211,3 +211,56 @@ def test_rotate_secret_that_does_not_match(): with assert_raises(ClientError): result = conn.rotate_secret(SecretId='i-dont-match') + +@mock_secretsmanager +def test_rotate_secret_client_request_token_too_short(): + # Test is intentionally empty. Boto3 catches too short ClientRequestToken + # and raises ParamValidationError before Moto can see it. + # test_server actually handles this error. + assert True + +@mock_secretsmanager +def test_rotate_secret_client_request_token_too_long(): + secret_name = 'test-secret' + conn = boto3.client('secretsmanager', region_name='us-west-2') + conn.create_secret(Name=secret_name, + SecretString='foosecret') + + client_request_token = ( + 'ED9F8B6C-85B7-446A-B7E4-38F2A3BEB13C-' + 'ED9F8B6C-85B7-446A-B7E4-38F2A3BEB13C' + ) + with assert_raises(ClientError): + result = conn.rotate_secret(SecretId=secret_name, + ClientRequestToken=client_request_token) + +@mock_secretsmanager +def test_rotate_secret_rotation_lambda_arn_too_long(): + secret_name = 'test-secret' + conn = boto3.client('secretsmanager', region_name='us-west-2') + conn.create_secret(Name=secret_name, + SecretString='foosecret') + + rotation_lambda_arn = '85B7-446A-B7E4' * 147 # == 2058 characters + with assert_raises(ClientError): + result = conn.rotate_secret(SecretId=secret_name, + RotationLambdaARN=rotation_lambda_arn) + +@mock_secretsmanager +def test_rotate_secret_rotation_period_zero(): + # Test is intentionally empty. Boto3 catches zero day rotation period + # and raises ParamValidationError before Moto can see it. + # test_server actually handles this error. + assert True + +@mock_secretsmanager +def test_rotate_secret_rotation_period_too_long(): + secret_name = 'test-secret' + conn = boto3.client('secretsmanager', region_name='us-west-2') + conn.create_secret(Name=secret_name, + SecretString='foosecret') + + rotation_rules = {'AutomaticallyAfterDays': 1001} + with assert_raises(ClientError): + result = conn.rotate_secret(SecretId=secret_name, + RotationRules=rotation_rules) From 4ced0ce0dbae6a3d488681bb3d6546b39f04db85 Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Wed, 15 Aug 2018 11:22:07 -0700 Subject: [PATCH 030/106] Opportunistic update to unit test for consistency. --- tests/test_secretsmanager/test_server.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_secretsmanager/test_server.py b/tests/test_secretsmanager/test_server.py index 8c6f7b970..a45263304 100644 --- a/tests/test_secretsmanager/test_server.py +++ b/tests/test_secretsmanager/test_server.py @@ -49,6 +49,27 @@ def test_get_secret_that_does_not_exist(): assert json_data['message'] == "Secrets Manager can't find the specified secret" assert json_data['__type'] == 'ResourceNotFoundException' +@mock_secretsmanager +def test_get_secret_that_does_not_match(): + backend = server.create_backend_app("secretsmanager") + test_client = backend.test_client() + + create_secret = test_client.post('/', + data={"Name": "test-secret", + "SecretString": "foo-secret"}, + headers={ + "X-Amz-Target": "secretsmanager.CreateSecret"}, + ) + get_secret = test_client.post('/', + data={"SecretId": "i-dont-match", + "VersionStage": "AWSCURRENT"}, + headers={ + "X-Amz-Target": "secretsmanager.GetSecretValue"}, + ) + json_data = json.loads(get_secret.data.decode("utf-8")) + assert json_data['message'] == "Secrets Manager can't find the specified secret" + assert json_data['__type'] == 'ResourceNotFoundException' + @mock_secretsmanager def test_create_secret(): From 71ed78141ab03f634c49c52d109740d7f545ce1b Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Wed, 15 Aug 2018 11:32:54 -0700 Subject: [PATCH 031/106] Moto-1781: Implement server unit tests. *** Includes 2 commented out tests that generate errors pointing deeper in the code. *** --- tests/test_secretsmanager/test_server.py | 208 +++++++++++++++++++++++ 1 file changed, 208 insertions(+) diff --git a/tests/test_secretsmanager/test_server.py b/tests/test_secretsmanager/test_server.py index a45263304..b5494ef96 100644 --- a/tests/test_secretsmanager/test_server.py +++ b/tests/test_secretsmanager/test_server.py @@ -154,3 +154,211 @@ def test_describe_secret_that_does_not_match(): json_data = json.loads(describe_secret.data.decode("utf-8")) assert json_data['message'] == "Secrets Manager can't find the specified secret" assert json_data['__type'] == 'ResourceNotFoundException' + +@mock_secretsmanager +def test_rotate_secret(): + backend = server.create_backend_app('secretsmanager') + test_client = backend.test_client() + + create_secret = test_client.post('/', + data={"Name": "test-secret", + "SecretString": "foosecret"}, + headers={ + "X-Amz-Target": "secretsmanager.CreateSecret" + }, + ) + + client_request_token = "EXAMPLE2-90ab-cdef-fedc-ba987SECRET2" + rotate_secret = test_client.post('/', + data={"SecretId": "test-secret", + "ClientRequestToken": client_request_token}, + headers={ + "X-Amz-Target": "secretsmanager.RotateSecret" + }, + ) + + json_data = json.loads(rotate_secret.data.decode("utf-8")) + assert json_data # Returned dict is not empty + assert json_data['ARN'] == ( + 'arn:aws:secretsmanager:us-east-1:1234567890:secret:test-secret-rIjad' + ) + assert json_data['Name'] == 'test-secret' + assert json_data['VersionId'] == client_request_token + +@mock_secretsmanager +def test_rotate_secret_that_does_not_exist(): + backend = server.create_backend_app('secretsmanager') + test_client = backend.test_client() + + rotate_secret = test_client.post('/', + data={"SecretId": "i-dont-exist"}, + headers={ + "X-Amz-Target": "secretsmanager.RotateSecret" + }, + ) + + json_data = json.loads(rotate_secret.data.decode("utf-8")) + assert json_data['message'] == "Secrets Manager can't find the specified secret" + assert json_data['__type'] == 'ResourceNotFoundException' + +@mock_secretsmanager +def test_rotate_secret_that_does_not_match(): + backend = server.create_backend_app('secretsmanager') + test_client = backend.test_client() + + create_secret = test_client.post('/', + data={"Name": "test-secret", + "SecretString": "foosecret"}, + headers={ + "X-Amz-Target": "secretsmanager.CreateSecret" + }, + ) + + rotate_secret = test_client.post('/', + data={"SecretId": "i-dont-match"}, + headers={ + "X-Amz-Target": "secretsmanager.RotateSecret" + }, + ) + + json_data = json.loads(rotate_secret.data.decode("utf-8")) + assert json_data['message'] == "Secrets Manager can't find the specified secret" + assert json_data['__type'] == 'ResourceNotFoundException' + +@mock_secretsmanager +def test_rotate_secret_client_request_token_too_short(): + backend = server.create_backend_app('secretsmanager') + test_client = backend.test_client() + + create_secret = test_client.post('/', + data={"Name": "test-secret", + "SecretString": "foosecret"}, + headers={ + "X-Amz-Target": "secretsmanager.CreateSecret" + }, + ) + + client_request_token = "ED9F8B6C-85B7-B7E4-38F2A3BEB13C" + rotate_secret = test_client.post('/', + data={"SecretId": "test-secret", + "ClientRequestToken": client_request_token}, + headers={ + "X-Amz-Target": "secretsmanager.RotateSecret" + }, + ) + + json_data = json.loads(rotate_secret.data.decode("utf-8")) + assert json_data['message'] == "ClientRequestToken must be 32-64 characters long." + assert json_data['__type'] == 'InvalidParameterException' + +@mock_secretsmanager +def test_rotate_secret_client_request_token_too_long(): + backend = server.create_backend_app('secretsmanager') + test_client = backend.test_client() + + create_secret = test_client.post('/', + data={"Name": "test-secret", + "SecretString": "foosecret"}, + headers={ + "X-Amz-Target": "secretsmanager.CreateSecret" + }, + ) + + client_request_token = ( + 'ED9F8B6C-85B7-446A-B7E4-38F2A3BEB13C-' + 'ED9F8B6C-85B7-446A-B7E4-38F2A3BEB13C' + ) + rotate_secret = test_client.post('/', + data={"SecretId": "test-secret", + "ClientRequestToken": client_request_token}, + headers={ + "X-Amz-Target": "secretsmanager.RotateSecret" + }, + ) + + json_data = json.loads(rotate_secret.data.decode("utf-8")) + assert json_data['message'] == "ClientRequestToken must be 32-64 characters long." + assert json_data['__type'] == 'InvalidParameterException' + +@mock_secretsmanager +def test_rotate_secret_rotation_lambda_arn_too_long(): + backend = server.create_backend_app('secretsmanager') + test_client = backend.test_client() + + create_secret = test_client.post('/', + data={"Name": "test-secret", + "SecretString": "foosecret"}, + headers={ + "X-Amz-Target": "secretsmanager.CreateSecret" + }, + ) + + rotation_lambda_arn = '85B7-446A-B7E4' * 147 # == 2058 characters + rotate_secret = test_client.post('/', + data={"SecretId": "test-secret", + "RotationLambdaARN": rotation_lambda_arn}, + headers={ + "X-Amz-Target": "secretsmanager.RotateSecret" + }, + ) + + json_data = json.loads(rotate_secret.data.decode("utf-8")) + assert json_data['message'] == "RotationLambdaARN must <= 2048 characters long." + assert json_data['__type'] == 'InvalidParameterException' + + +# +# The following tests should work, but fail on the embedded dict in +# RotationRules. The error message suggests a problem deeper in the code, which +# needs further investigation. +# + +# @mock_secretsmanager +# def test_rotate_secret_rotation_period_zero(): +# backend = server.create_backend_app('secretsmanager') +# test_client = backend.test_client() + +# create_secret = test_client.post('/', +# data={"Name": "test-secret", +# "SecretString": "foosecret"}, +# headers={ +# "X-Amz-Target": "secretsmanager.CreateSecret" +# }, +# ) + +# rotate_secret = test_client.post('/', +# data={"SecretId": "test-secret", +# "RotationRules": {"AutomaticallyAfterDays": 0}}, +# headers={ +# "X-Amz-Target": "secretsmanager.RotateSecret" +# }, +# ) + +# json_data = json.loads(rotate_secret.data.decode("utf-8")) +# assert json_data['message'] == "RotationLambdaARN must <= 2048 characters long." +# assert json_data['__type'] == 'InvalidParameterException' + +# @mock_secretsmanager +# def test_rotate_secret_rotation_period_too_long(): +# backend = server.create_backend_app('secretsmanager') +# test_client = backend.test_client() + +# create_secret = test_client.post('/', +# data={"Name": "test-secret", +# "SecretString": "foosecret"}, +# headers={ +# "X-Amz-Target": "secretsmanager.CreateSecret" +# }, +# ) + +# rotate_secret = test_client.post('/', +# data={"SecretId": "test-secret", +# "RotationRules": {"AutomaticallyAfterDays": 1001}}, +# headers={ +# "X-Amz-Target": "secretsmanager.RotateSecret" +# }, +# ) + +# json_data = json.loads(rotate_secret.data.decode("utf-8")) +# assert json_data['message'] == "RotationLambdaARN must <= 2048 characters long." +# assert json_data['__type'] == 'InvalidParameterException' From 6985f27167ad7323dad4f9896b42a32a9b8e6836 Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Wed, 15 Aug 2018 17:11:58 -0700 Subject: [PATCH 032/106] Moto-1781: Add unit tests to verify that rotation is enabled. - Add standalone unit test to verify that rotation is enabled and the rotation interval is correct. - Add server test to verify that rotation is enabled and the rotation interval is correct. Commented out until nested dict error is sorted. - Fix incorrectly asserted message strings. --- .../test_secretsmanager.py | 20 ++++++ tests/test_secretsmanager/test_server.py | 61 ++++++++++++++++++- 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index 3e7b31c72..ec384a660 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -196,6 +196,26 @@ def test_rotate_secret(): assert rotated_secret['Name'] == secret_name assert rotated_secret['VersionId'] != '' +@mock_secretsmanager +def test_rotate_secret_enable_rotation(): + secret_name = 'test-secret' + conn = boto3.client('secretsmanager', region_name='us-west-2') + conn.create_secret(Name=secret_name, + SecretString='foosecret') + + initial_description = conn.describe_secret(SecretId=secret_name) + assert initial_description + assert initial_description['RotationEnabled'] is False + assert initial_description['RotationRules']['AutomaticallyAfterDays'] == 0 + + conn.rotate_secret(SecretId=secret_name, + RotationRules={'AutomaticallyAfterDays': 42}) + + rotated_description = conn.describe_secret(SecretId=secret_name) + assert rotated_description + assert rotated_description['RotationEnabled'] is True + assert rotated_description['RotationRules']['AutomaticallyAfterDays'] == 42 + @mock_secretsmanager def test_rotate_secret_that_does_not_exist(): conn = boto3.client('secretsmanager', 'us-west-2') diff --git a/tests/test_secretsmanager/test_server.py b/tests/test_secretsmanager/test_server.py index b5494ef96..e573f9b67 100644 --- a/tests/test_secretsmanager/test_server.py +++ b/tests/test_secretsmanager/test_server.py @@ -185,6 +185,63 @@ def test_rotate_secret(): assert json_data['Name'] == 'test-secret' assert json_data['VersionId'] == client_request_token +# @mock_secretsmanager +# def test_rotate_secret_enable_rotation(): +# backend = server.create_backend_app('secretsmanager') +# test_client = backend.test_client() + +# create_secret = test_client.post( +# '/', +# data={ +# "Name": "test-secret", +# "SecretString": "foosecret" +# }, +# headers={ +# "X-Amz-Target": "secretsmanager.CreateSecret" +# }, +# ) + +# initial_description = test_client.post( +# '/', +# data={ +# "SecretId": "test-secret" +# }, +# headers={ +# "X-Amz-Target": "secretsmanager.DescribeSecret" +# }, +# ) + +# json_data = json.loads(initial_description.data.decode("utf-8")) +# assert json_data # Returned dict is not empty +# assert json_data['RotationEnabled'] is False +# assert json_data['RotationRules']['AutomaticallyAfterDays'] == 0 + +# rotate_secret = test_client.post( +# '/', +# data={ +# "SecretId": "test-secret", +# "RotationRules": {"AutomaticallyAfterDays": 42} +# }, +# headers={ +# "X-Amz-Target": "secretsmanager.RotateSecret" +# }, +# ) + +# rotated_description = test_client.post( +# '/', +# data={ +# "SecretId": "test-secret" +# }, +# headers={ +# "X-Amz-Target": "secretsmanager.DescribeSecret" +# }, +# ) + +# json_data = json.loads(rotated_description.data.decode("utf-8")) +# assert json_data # Returned dict is not empty +# assert json_data['RotationEnabled'] is True +# assert json_data['RotationRules']['AutomaticallyAfterDays'] == 42 + @mock_secretsmanager def test_rotate_secret_that_does_not_exist(): backend = server.create_backend_app('secretsmanager') @@ -335,7 +392,7 @@ def test_rotate_secret_rotation_lambda_arn_too_long(): # ) # json_data = json.loads(rotate_secret.data.decode("utf-8")) -# assert json_data['message'] == "RotationLambdaARN must <= 2048 characters long." +# assert json_data['message'] == "RotationRules.AutomaticallyAfterDays must be within 1-1000." # assert json_data['__type'] == 'InvalidParameterException' # @mock_secretsmanager @@ -360,5 +417,5 @@ def test_rotate_secret_rotation_lambda_arn_too_long(): # ) # json_data = json.loads(rotate_secret.data.decode("utf-8")) -# assert json_data['message'] == "RotationLambdaARN must <= 2048 characters long." +# assert json_data['message'] == "RotationRules.AutomaticallyAfterDays must be within 1-1000." # assert json_data['__type'] == 'InvalidParameterException' From 001a7d0278d8827019622541609e036e3cecc6f5 Mon Sep 17 00:00:00 2001 From: chrisLeeTW Date: Sat, 11 Aug 2018 22:14:04 +0800 Subject: [PATCH 033/106] type of json field - nextSequenceToken that return by cloudwatch logs putLogEvents should be string, not int. --- moto/logs/models.py | 2 +- tests/test_logs/test_logs.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/moto/logs/models.py b/moto/logs/models.py index 3e1c7b955..a4ff9db46 100644 --- a/moto/logs/models.py +++ b/moto/logs/models.py @@ -86,7 +86,7 @@ class LogStream: self.events += [LogEvent(self.lastIngestionTime, log_event) for log_event in log_events] self.uploadSequenceToken += 1 - return self.uploadSequenceToken + return '{:056d}'.format(self.uploadSequenceToken) def get_log_events(self, log_group_name, log_stream_name, start_time, end_time, limit, next_token, start_from_head): def filter_func(event): diff --git a/tests/test_logs/test_logs.py b/tests/test_logs/test_logs.py index 3f924cc55..05bd3c823 100644 --- a/tests/test_logs/test_logs.py +++ b/tests/test_logs/test_logs.py @@ -1,5 +1,6 @@ import boto3 import sure # noqa +import six from botocore.exceptions import ClientError from moto import mock_logs, settings @@ -47,7 +48,7 @@ def test_exceptions(): logEvents=[ { 'timestamp': 0, - 'message': 'line' + 'message': 'line' }, ], ) @@ -79,7 +80,7 @@ def test_put_logs(): {'timestamp': 0, 'message': 'hello'}, {'timestamp': 0, 'message': 'world'} ] - conn.put_log_events( + putRes = conn.put_log_events( logGroupName=log_group_name, logStreamName=log_stream_name, logEvents=messages @@ -89,6 +90,9 @@ def test_put_logs(): logStreamName=log_stream_name ) events = res['events'] + nextSequenceToken = putRes['nextSequenceToken'] + assert isinstance(nextSequenceToken, six.string_types) == True + assert len(nextSequenceToken) == 56 events.should.have.length_of(2) From c6f5e816ccda6386976a5e3c9677a6b9839ec108 Mon Sep 17 00:00:00 2001 From: Brian Pandola Date: Mon, 20 Aug 2018 18:48:13 -0700 Subject: [PATCH 034/106] Add `ClusterCreateTime` to Redshift response Fixes #1778 --- moto/redshift/models.py | 2 ++ tests/test_redshift/test_redshift.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/moto/redshift/models.py b/moto/redshift/models.py index 4eafcfc79..70cbb95cb 100644 --- a/moto/redshift/models.py +++ b/moto/redshift/models.py @@ -78,6 +78,7 @@ class Cluster(TaggableResourceMixin, BaseModel): super(Cluster, self).__init__(region_name, tags) self.redshift_backend = redshift_backend self.cluster_identifier = cluster_identifier + self.create_time = iso_8601_datetime_with_milliseconds(datetime.datetime.now()) self.status = 'available' self.node_type = node_type self.master_username = master_username @@ -237,6 +238,7 @@ class Cluster(TaggableResourceMixin, BaseModel): "Address": self.endpoint, "Port": self.port }, + 'ClusterCreateTime': self.create_time, "PendingModifiedValues": [], "Tags": self.tags, "IamRoles": [{ diff --git a/tests/test_redshift/test_redshift.py b/tests/test_redshift/test_redshift.py index 6e027b86c..9208c92dd 100644 --- a/tests/test_redshift/test_redshift.py +++ b/tests/test_redshift/test_redshift.py @@ -1,5 +1,7 @@ from __future__ import unicode_literals +import datetime + import boto import boto3 from boto.redshift.exceptions import ( @@ -32,6 +34,8 @@ def test_create_cluster_boto3(): MasterUserPassword='password', ) response['Cluster']['NodeType'].should.equal('ds2.xlarge') + create_time = response['Cluster']['ClusterCreateTime'] + create_time.should.be.lower_than(datetime.datetime.now(create_time.tzinfo)) @mock_redshift From 50f8d5e13c70686da3b865832ae3e0ec4fa4aad2 Mon Sep 17 00:00:00 2001 From: Matthew Neal Date: Wed, 29 Aug 2018 12:42:31 -0400 Subject: [PATCH 035/106] Pin boto3 version to 1.7.84 Pinning boto3 to this version would remove the mismatch between boto3 and the botocore dependency in setup.py. Closes #1800 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 16aaf1452..bcb48a967 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ import sys install_requires = [ "Jinja2>=2.7.3", "boto>=2.36.0", - "boto3>=1.6.16", + "boto3>=1.6.16,<1.8", "botocore>=1.9.16,<1.11", "cookies", "cryptography>=2.0.0", From 54cbc98506d261be9ee2228b9ccf584e8b723234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Luis=20Diaz?= Date: Mon, 3 Sep 2018 16:34:19 -0300 Subject: [PATCH 036/106] Return given host for 'moto-api' --- moto/server.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/moto/server.py b/moto/server.py index aad47757a..ba2470478 100644 --- a/moto/server.py +++ b/moto/server.py @@ -34,6 +34,9 @@ class DomainDispatcherApplication(object): self.service = service def get_backend_for_host(self, host): + if host == 'moto_api': + return host + if self.service: return self.service From 28aa5d34b0bf2417089bf49fd1573a620720997a Mon Sep 17 00:00:00 2001 From: Ollie Ford Date: Wed, 5 Sep 2018 10:43:39 +0100 Subject: [PATCH 037/106] Add failing test for #1809 --- tests/test_core/test_decorator_calls.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_core/test_decorator_calls.py b/tests/test_core/test_decorator_calls.py index 9e3638cc2..5d2f6a4ef 100644 --- a/tests/test_core/test_decorator_calls.py +++ b/tests/test_core/test_decorator_calls.py @@ -85,3 +85,14 @@ class TesterWithSetup(unittest.TestCase): def test_still_the_same(self): bucket = self.conn.get_bucket('mybucket') bucket.name.should.equal("mybucket") + + +@mock_s3_deprecated +class TesterWithStaticmethod(object): + + @staticmethod + def static(*args): + assert not args or not isinstance(args[0], TesterWithStaticmethod) + + def test_no_instance_sent_to_staticmethod(self): + self.static() From 0ac989cfd4a0e684df2f0170821d1598526484bf Mon Sep 17 00:00:00 2001 From: Ollie Ford Date: Wed, 5 Sep 2018 10:39:09 +0100 Subject: [PATCH 038/106] Fix #1809: skip patching staticmethods --- moto/core/models.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/moto/core/models.py b/moto/core/models.py index 92dc2a980..adc06a9c0 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -89,6 +89,17 @@ class BaseMockAWS(object): if inspect.ismethod(attr_value) and attr_value.__self__ is klass: continue + # Check if this is a staticmethod. If so, skip patching + for cls in inspect.getmro(klass): + if attr_value.__name__ not in cls.__dict__: + continue + bound_attr_value = cls.__dict__[attr_value.__name__] + if not isinstance(bound_attr_value, staticmethod): + break + else: + # It is a staticmethod, skip patching + continue + try: setattr(klass, attr, self(attr_value, reset=False)) except TypeError: From d60d562c6262e5e2b58a2d8e5ee9f5f12eb09fc3 Mon Sep 17 00:00:00 2001 From: Justin McCormick Date: Sun, 9 Sep 2018 02:14:28 -0400 Subject: [PATCH 039/106] Add AWS::Partition as a variable available to CloudFormation templates --- moto/cloudformation/parsing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index c4059a06b..35b05d101 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -387,6 +387,7 @@ class ResourceMap(collections.Mapping): "AWS::StackName": stack_name, "AWS::URLSuffix": "amazonaws.com", "AWS::NoValue": None, + "AWS::Partition": "aws", } def __getitem__(self, key): From 756b5d6671c04e8a13d86e82224ea35c32a053e4 Mon Sep 17 00:00:00 2001 From: Jordan Upiter Date: Fri, 5 Jan 2018 15:12:45 -0500 Subject: [PATCH 040/106] Add support for multiple delete markers on an s3 object --- moto/s3/models.py | 11 ++++--- moto/s3/responses.py | 6 ++-- tests/test_s3/test_s3.py | 66 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 7 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index cf5628141..f3994b5d8 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -27,8 +27,14 @@ class FakeDeleteMarker(BaseModel): def __init__(self, key): self.key = key + self.name = key.name + self.last_modified = datetime.datetime.utcnow() self._version_id = key.version_id + 1 + @property + def last_modified_ISO8601(self): + return iso_8601_datetime_with_milliseconds(self.last_modified) + @property def version_id(self): return self._version_id @@ -630,10 +636,7 @@ class S3Backend(BaseBackend): latest_versions = {} for version in versions: - if isinstance(version, FakeDeleteMarker): - name = version.key.name - else: - name = version.name + name = version.name version_id = version.version_id maximum_version_per_key[name] = max( version_id, diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 5e7cf0fe5..f8dc7e42b 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -1273,10 +1273,10 @@ S3_BUCKET_GET_VERSIONS = """ {% endfor %} {% for marker in delete_marker_list %} - {{ marker.key.name }} + {{ marker.name }} {{ marker.version_id }} - {% if latest_versions[marker.key.name] == marker.version_id %}true{% else %}false{% endif %} - {{ marker.key.last_modified_ISO8601 }} + {% if latest_versions[marker.name] == marker.version_id %}true{% else %}false{% endif %} + {{ marker.last_modified_ISO8601 }} 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a webfile diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 9a68d1bbb..6e339abb6 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -2471,6 +2471,72 @@ def test_boto3_delete_markers(): oldest['Key'].should.equal('key-with-versions-and-unicode-ó') +@mock_s3 +def test_boto3_multiple_delete_markers(): + s3 = boto3.client('s3', region_name='us-east-1') + bucket_name = 'mybucket' + key = u'key-with-versions-and-unicode-ó' + s3.create_bucket(Bucket=bucket_name) + s3.put_bucket_versioning( + Bucket=bucket_name, + VersioningConfiguration={ + 'Status': 'Enabled' + } + ) + items = (six.b('v1'), six.b('v2')) + for body in items: + s3.put_object( + Bucket=bucket_name, + Key=key, + Body=body + ) + + # Delete the object twice to add multiple delete markers + s3.delete_object(Bucket=bucket_name, Key=key) + s3.delete_object(Bucket=bucket_name, Key=key) + + response = s3.list_object_versions(Bucket=bucket_name) + response['DeleteMarkers'].should.have.length_of(2) + + with assert_raises(ClientError) as e: + s3.get_object( + Bucket=bucket_name, + Key=key + ) + e.response['Error']['Code'].should.equal('404') + + # Remove both delete markers to restore the object + s3.delete_object( + Bucket=bucket_name, + Key=key, + VersionId='2' + ) + s3.delete_object( + Bucket=bucket_name, + Key=key, + VersionId='3' + ) + + response = s3.get_object( + Bucket=bucket_name, + Key=key + ) + response['Body'].read().should.equal(items[-1]) + response = s3.list_object_versions(Bucket=bucket_name) + response['Versions'].should.have.length_of(2) + + # We've asserted there is only 2 records so one is newest, one is oldest + latest = list(filter(lambda item: item['IsLatest'], response['Versions']))[0] + oldest = list(filter(lambda item: not item['IsLatest'], response['Versions']))[0] + + # Double check ordering of version ID's + latest['VersionId'].should.equal('1') + oldest['VersionId'].should.equal('0') + + # Double check the name is still unicode + latest['Key'].should.equal('key-with-versions-and-unicode-ó') + oldest['Key'].should.equal('key-with-versions-and-unicode-ó') + @mock_s3 def test_get_stream_gzipped(): payload = b"this is some stuff here" From 4ffff9161e1a859f5beef431f576e1151e247d95 Mon Sep 17 00:00:00 2001 From: Daniel Huang Date: Thu, 20 Sep 2018 11:23:20 -0700 Subject: [PATCH 041/106] Remove unused cookies dependency --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 16aaf1452..df00255ed 100755 --- a/setup.py +++ b/setup.py @@ -10,7 +10,6 @@ install_requires = [ "boto>=2.36.0", "boto3>=1.6.16", "botocore>=1.9.16,<1.11", - "cookies", "cryptography>=2.0.0", "requests>=2.5", "xmltodict", From 881afc8f4a35564caef6098d383f7213fb14693e Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 21 Sep 2018 08:31:31 -0500 Subject: [PATCH 042/106] update RDS models to include CopyTagsToSnapshot --- AUTHORS.md | 1 + moto/rds/models.py | 6 +++++ moto/rds2/models.py | 8 +++++++ tests/test_rds2/test_rds2.py | 44 ++++++++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+) diff --git a/AUTHORS.md b/AUTHORS.md index 6b7c96291..0a152505a 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -53,3 +53,4 @@ Moto is written by Steve Pulec with contributions from: * [Jim Shields](https://github.com/jimjshields) * [William Richard](https://github.com/william-richard) * [Alex Casalboni](https://github.com/alexcasalboni) +* [Jon Beilke](https://github.com/jrbeilke) diff --git a/moto/rds/models.py b/moto/rds/models.py index 77deff09d..feecefe0c 100644 --- a/moto/rds/models.py +++ b/moto/rds/models.py @@ -48,6 +48,10 @@ class Database(BaseModel): if self.publicly_accessible is None: self.publicly_accessible = True + self.copy_tags_to_snapshot = kwargs.get("copy_tags_to_snapshot") + if self.copy_tags_to_snapshot is None: + self.copy_tags_to_snapshot = False + self.backup_retention_period = kwargs.get("backup_retention_period") if self.backup_retention_period is None: self.backup_retention_period = 1 @@ -137,6 +141,7 @@ class Database(BaseModel): "multi_az": properties.get("MultiAZ"), "port": properties.get('Port', 3306), "publicly_accessible": properties.get("PubliclyAccessible"), + "copy_tags_to_snapshot": properties.get("CopyTagsToSnapshot"), "region": region_name, "security_groups": security_groups, "storage_encrypted": properties.get("StorageEncrypted"), @@ -217,6 +222,7 @@ class Database(BaseModel): {% endif %} {{ database.publicly_accessible }} + {{ database.copy_tags_to_snapshot }} {{ database.auto_minor_version_upgrade }} {{ database.allocated_storage }} {{ database.storage_encrypted }} diff --git a/moto/rds2/models.py b/moto/rds2/models.py index 3fc4b6d65..c656f5ec3 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -73,6 +73,9 @@ class Database(BaseModel): self.publicly_accessible = kwargs.get("publicly_accessible") if self.publicly_accessible is None: self.publicly_accessible = True + self.copy_tags_to_snapshot = kwargs.get("copy_tags_to_snapshot") + if self.copy_tags_to_snapshot is None: + self.copy_tags_to_snapshot = False self.backup_retention_period = kwargs.get("backup_retention_period") if self.backup_retention_period is None: self.backup_retention_period = 1 @@ -208,6 +211,7 @@ class Database(BaseModel): {% endif %} {{ database.publicly_accessible }} + {{ database.copy_tags_to_snapshot }} {{ database.auto_minor_version_upgrade }} {{ database.allocated_storage }} {{ database.storage_encrypted }} @@ -304,6 +308,7 @@ class Database(BaseModel): "db_parameter_group_name": properties.get('DBParameterGroupName'), "port": properties.get('Port', 3306), "publicly_accessible": properties.get("PubliclyAccessible"), + "copy_tags_to_snapshot": properties.get("CopyTagsToSnapshot"), "region": region_name, "security_groups": security_groups, "storage_encrypted": properties.get("StorageEncrypted"), @@ -362,6 +367,7 @@ class Database(BaseModel): "PreferredBackupWindow": "{{ database.preferred_backup_window }}", "PreferredMaintenanceWindow": "{{ database.preferred_maintenance_window }}", "PubliclyAccessible": "{{ database.publicly_accessible }}", + "CopyTagsToSnapshot": "{{ database.copy_tags_to_snapshot }}", "AllocatedStorage": "{{ database.allocated_storage }}", "Endpoint": { "Address": "{{ database.address }}", @@ -691,6 +697,8 @@ class RDS2Backend(BaseBackend): raise DBSnapshotAlreadyExistsError(db_snapshot_identifier) if len(self.snapshots) >= int(os.environ.get('MOTO_RDS_SNAPSHOT_LIMIT', '100')): raise SnapshotQuotaExceededError() + if not database.copy_tags_to_snapshot: + tags = None snapshot = Snapshot(database, db_snapshot_identifier, tags) self.snapshots[db_snapshot_identifier] = snapshot return snapshot diff --git a/tests/test_rds2/test_rds2.py b/tests/test_rds2/test_rds2.py index 80dcd4f53..7fecfeca9 100644 --- a/tests/test_rds2/test_rds2.py +++ b/tests/test_rds2/test_rds2.py @@ -33,6 +33,7 @@ def test_create_database(): db_instance['DBInstanceIdentifier'].should.equal("db-master-1") db_instance['IAMDatabaseAuthenticationEnabled'].should.equal(False) db_instance['DbiResourceId'].should.contain("db-") + db_instance['CopyTagsToSnapshot'].should.equal(False) @mock_rds2 @@ -339,6 +340,49 @@ def test_create_db_snapshots(): snapshot.get('Engine').should.equal('postgres') snapshot.get('DBInstanceIdentifier').should.equal('db-primary-1') snapshot.get('DBSnapshotIdentifier').should.equal('g-1') + result = conn.list_tags_for_resource(ResourceName=snapshot['DBSnapshot']['DBSnapshotArn']) + result['TagList'].should.equal([]) + + +@mock_rds2 +def test_create_db_snapshots_copy_tags(): + conn = boto3.client('rds', region_name='us-west-2') + conn.create_db_snapshot.when.called_with( + DBInstanceIdentifier='db-primary-1', + DBSnapshotIdentifier='snapshot-1').should.throw(ClientError) + + conn.create_db_instance(DBInstanceIdentifier='db-primary-1', + AllocatedStorage=10, + Engine='postgres', + DBName='staging-postgres', + DBInstanceClass='db.m1.small', + MasterUsername='root', + MasterUserPassword='hunter2', + Port=1234, + DBSecurityGroups=["my_sg"], + CopyTagsToSnapshot=True, + Tags=[ + { + 'Key': 'foo', + 'Value': 'bar', + }, + { + 'Key': 'foo1', + 'Value': 'bar1', + }, + ]) + + snapshot = conn.create_db_snapshot(DBInstanceIdentifier='db-primary-1', + DBSnapshotIdentifier='g-1').get('DBSnapshot') + + snapshot.get('Engine').should.equal('postgres') + snapshot.get('DBInstanceIdentifier').should.equal('db-primary-1') + snapshot.get('DBSnapshotIdentifier').should.equal('g-1') + result = conn.list_tags_for_resource(ResourceName=snapshot['DBSnapshot']['DBSnapshotArn']) + result['TagList'].should.equal([{'Value': 'bar', + 'Key': 'foo'}, + {'Value': 'bar1', + 'Key': 'foo1'}]) @mock_rds2 From bf9b37142e7d95102dfe11c506d5ae1a3f2779ed Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 21 Sep 2018 08:49:45 -0500 Subject: [PATCH 043/106] no need for [DBSnapshot] with list_tags_for_resource as the retuned snapshot already handles it --- tests/test_rds2/test_rds2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_rds2/test_rds2.py b/tests/test_rds2/test_rds2.py index 7fecfeca9..0f802bb5c 100644 --- a/tests/test_rds2/test_rds2.py +++ b/tests/test_rds2/test_rds2.py @@ -340,7 +340,7 @@ def test_create_db_snapshots(): snapshot.get('Engine').should.equal('postgres') snapshot.get('DBInstanceIdentifier').should.equal('db-primary-1') snapshot.get('DBSnapshotIdentifier').should.equal('g-1') - result = conn.list_tags_for_resource(ResourceName=snapshot['DBSnapshot']['DBSnapshotArn']) + result = conn.list_tags_for_resource(ResourceName=snapshot['DBSnapshotArn']) result['TagList'].should.equal([]) @@ -378,7 +378,7 @@ def test_create_db_snapshots_copy_tags(): snapshot.get('Engine').should.equal('postgres') snapshot.get('DBInstanceIdentifier').should.equal('db-primary-1') snapshot.get('DBSnapshotIdentifier').should.equal('g-1') - result = conn.list_tags_for_resource(ResourceName=snapshot['DBSnapshot']['DBSnapshotArn']) + result = conn.list_tags_for_resource(ResourceName=snapshot['DBSnapshotArn']) result['TagList'].should.equal([{'Value': 'bar', 'Key': 'foo'}, {'Value': 'bar1', From cedb89dc3b2a7e4701b68f06a010d17f3a06f98f Mon Sep 17 00:00:00 2001 From: hans Date: Fri, 21 Sep 2018 23:29:04 +0800 Subject: [PATCH 044/106] Fix #1830 Add support for cross-region VPC peering Add a class level store in models/VPCBackend of ec2 for saving vpcs of all regions info. Any instance can correctly find vpc in another region when connecting vpc of cross-region or vpc of same region. Modify vpc_peering_connections in ec2/responses to handle vpc peering of same region or cross region. Update vpc_peering_connections response template content to latest (2016-11-15) . Add vpc cross region peering successful test case. Add vpc cross region peering fail test case. Related: https://github.com/spulec/moto/issues/1830 Reference CreateVpcPeeringConnection Sample Response https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateVpcPeeringConnection.html --- moto/ec2/models.py | 18 +++++++++ moto/ec2/responses/vpc_peering_connections.py | 31 +++++++++------ tests/test_ec2/test_vpc_peering.py | 39 ++++++++++++++++++- 3 files changed, 76 insertions(+), 12 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 4e26f0f65..b94cac479 100755 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -13,6 +13,7 @@ from pkg_resources import resource_filename import boto.ec2 from collections import defaultdict +import weakref from datetime import datetime from boto.ec2.instance import Instance as BotoInstance, Reservation from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType @@ -2115,10 +2116,20 @@ class VPC(TaggedEC2Resource): class VPCBackend(object): + __refs__ = defaultdict(list) + def __init__(self): self.vpcs = {} + self.__refs__[self.__class__].append(weakref.ref(self)) super(VPCBackend, self).__init__() + @classmethod + def get_instances(cls): + for inst_ref in cls.__refs__[cls]: + inst = inst_ref() + if inst is not None: + yield inst + def create_vpc(self, cidr_block, instance_tenancy='default', amazon_provided_ipv6_cidr_block=False): vpc_id = random_vpc_id() vpc = VPC(self, vpc_id, cidr_block, len(self.vpcs) == 0, instance_tenancy, amazon_provided_ipv6_cidr_block) @@ -2142,6 +2153,13 @@ class VPCBackend(object): raise InvalidVPCIdError(vpc_id) return self.vpcs.get(vpc_id) + # get vpc by vpc id and aws region + def get_cross_vpc(self, vpc_id, peer_region): + for vpcs in self.get_instances(): + if vpcs.region_name == peer_region: + match_vpc = vpcs.get_vpc(vpc_id) + return match_vpc + def get_all_vpcs(self, vpc_ids=None, filters=None): matches = self.vpcs.values() if vpc_ids: diff --git a/moto/ec2/responses/vpc_peering_connections.py b/moto/ec2/responses/vpc_peering_connections.py index 1bccce4f6..49d752893 100644 --- a/moto/ec2/responses/vpc_peering_connections.py +++ b/moto/ec2/responses/vpc_peering_connections.py @@ -5,8 +5,12 @@ from moto.core.responses import BaseResponse class VPCPeeringConnections(BaseResponse): def create_vpc_peering_connection(self): + peer_region = self._get_param('PeerRegion') + if peer_region == self.region or peer_region is None: + peer_vpc = self.ec2_backend.get_vpc(self._get_param('PeerVpcId')) + else: + peer_vpc = self.ec2_backend.get_cross_vpc(self._get_param('PeerVpcId'), peer_region) vpc = self.ec2_backend.get_vpc(self._get_param('VpcId')) - peer_vpc = self.ec2_backend.get_vpc(self._get_param('PeerVpcId')) vpc_pcx = self.ec2_backend.create_vpc_peering_connection(vpc, peer_vpc) template = self.response_template( CREATE_VPC_PEERING_CONNECTION_RESPONSE) @@ -41,26 +45,31 @@ class VPCPeeringConnections(BaseResponse): CREATE_VPC_PEERING_CONNECTION_RESPONSE = """ - - 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE - - {{ vpc_pcx.id }} + + 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE + + {{ vpc_pcx.id }} - 777788889999 - {{ vpc_pcx.vpc.id }} - {{ vpc_pcx.vpc.cidr_block }} + 777788889999 + {{ vpc_pcx.vpc.id }} + {{ vpc_pcx.vpc.cidr_block }} + + false + false + false + 123456789012 {{ vpc_pcx.peer_vpc.id }} - initiating-request - Initiating request to {accepter ID}. + initiating-request + Initiating Request to {accepter ID} 2014-02-18T14:37:25.000Z - + """ diff --git a/tests/test_ec2/test_vpc_peering.py b/tests/test_ec2/test_vpc_peering.py index 6722eed60..1f98791b3 100644 --- a/tests/test_ec2/test_vpc_peering.py +++ b/tests/test_ec2/test_vpc_peering.py @@ -2,12 +2,15 @@ from __future__ import unicode_literals # Ensure 'assert_raises' context manager support for Python 2.6 import tests.backport_assert_raises from nose.tools import assert_raises +from moto.ec2.exceptions import EC2ClientError +from botocore.exceptions import ClientError +import boto3 import boto from boto.exception import EC2ResponseError import sure # noqa -from moto import mock_ec2_deprecated +from moto import mock_ec2, mock_ec2_deprecated from tests.helpers import requires_boto_gte @@ -93,3 +96,37 @@ def test_vpc_peering_connections_delete(): cm.exception.code.should.equal('InvalidVpcPeeringConnectionId.NotFound') cm.exception.status.should.equal(400) cm.exception.request_id.should_not.be.none + + +@mock_ec2 +def test_vpc_peering_connections_cross_region(): + # create vpc in us-west-1 and ap-northeast-1 + ec2_usw1 = boto3.resource('ec2', region_name='us-west-1') + vpc_usw1 = ec2_usw1.create_vpc(CidrBlock='10.90.0.0/16') + ec2_apn1 = boto3.resource('ec2', region_name='ap-northeast-1') + vpc_apn1 = ec2_apn1.create_vpc(CidrBlock='10.20.0.0/16') + # create peering + vpc_pcx = ec2_usw1.create_vpc_peering_connection( + VpcId=vpc_usw1.id, + PeerVpcId=vpc_apn1.id, + PeerRegion='ap-northeast-1', + ) + vpc_pcx.status['Code'].should.equal('initiating-request') + vpc_pcx.requester_vpc.id.should.equal(vpc_usw1.id) + vpc_pcx.accepter_vpc.id.should.equal(vpc_apn1.id) + + +@mock_ec2 +def test_vpc_peering_connections_cross_region_fail(): + # create vpc in us-west-1 and ap-northeast-1 + ec2_usw1 = boto3.resource('ec2', region_name='us-west-1') + vpc_usw1 = ec2_usw1.create_vpc(CidrBlock='10.90.0.0/16') + ec2_apn1 = boto3.resource('ec2', region_name='ap-northeast-1') + vpc_apn1 = ec2_apn1.create_vpc(CidrBlock='10.20.0.0/16') + # create peering wrong region with no vpc + with assert_raises(ClientError) as cm: + ec2_usw1.create_vpc_peering_connection( + VpcId=vpc_usw1.id, + PeerVpcId=vpc_apn1.id, + PeerRegion='ap-northeast-2') + cm.exception.response['Error']['Code'].should.equal('InvalidVpcID.NotFound') From 276da0616851183bd5a2cd6dd8f80882816aea4a Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 21 Sep 2018 10:39:42 -0500 Subject: [PATCH 045/106] added new merge_taglists() to moto.core.utils for merging lists of tags with precedence (ie. during rds2.create_snapshot) --- moto/core/utils.py | 9 +++++++++ moto/rds2/models.py | 5 +++-- tests/test_core/test_utils.py | 27 ++++++++++++++++++++++++++- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/moto/core/utils.py b/moto/core/utils.py index 86e7632b0..2cdcd07f1 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -286,3 +286,12 @@ def amzn_request_id(f): return status, headers, body return _wrapper + + +def merge_taglists(taglist_a, taglist_b): + ''' Merges two tag lists into a single tag list with Keys in the second list taking precedence''' + tags_a = {t['Key']:t for t in taglist_a} + tags_b = {t['Key']:t for t in taglist_b} + merged_tags = tags_a.copy() + merged_tags.update(tags_b) + return merged_tags.values() diff --git a/moto/rds2/models.py b/moto/rds2/models.py index c656f5ec3..d564e93b1 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -13,6 +13,7 @@ from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel from moto.core.utils import get_random_hex from moto.core.utils import iso_8601_datetime_with_milliseconds +from moto.core.utils import merge_taglists from moto.ec2.models import ec2_backends from .exceptions import (RDSClientError, DBInstanceNotFoundError, @@ -697,8 +698,8 @@ class RDS2Backend(BaseBackend): raise DBSnapshotAlreadyExistsError(db_snapshot_identifier) if len(self.snapshots) >= int(os.environ.get('MOTO_RDS_SNAPSHOT_LIMIT', '100')): raise SnapshotQuotaExceededError() - if not database.copy_tags_to_snapshot: - tags = None + if database.copy_tags_to_snapshot: + tags = merge_taglists(database.tags, tags) snapshot = Snapshot(database, db_snapshot_identifier, tags) self.snapshots[db_snapshot_identifier] = snapshot return snapshot diff --git a/tests/test_core/test_utils.py b/tests/test_core/test_utils.py index 8dbf21716..0fa4f3c59 100644 --- a/tests/test_core/test_utils.py +++ b/tests/test_core/test_utils.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import sure # noqa from freezegun import freeze_time -from moto.core.utils import camelcase_to_underscores, underscores_to_camelcase, unix_time +from moto.core.utils import camelcase_to_underscores, underscores_to_camelcase, unix_time, merge_taglists def test_camelcase_to_underscores(): @@ -28,3 +28,28 @@ def test_underscores_to_camelcase(): @freeze_time("2015-01-01 12:00:00") def test_unix_time(): unix_time().should.equal(1420113600.0) + + +def test_merge_taglists(): + taglist_a = [ + { + 'Key': 'foo', + 'Value': 'bar', + }, + { + 'Key': 'foo1', + 'Value': 'bar1', + }, + ] + taglist_b = [ + { + 'Key': 'foo1', + 'Value': 'bar1b', + }, + ] + taglist_merged = merge_taglists(taglist_a, taglist_b) + len(taglist_merged).should.equal(2) + tag_foo = [t for t in taglist_merged if t['Key']=='foo'] + tag_foo1 = [t for t in taglist_merged if t['Key']=='foo1'] + tag_foo[0].should.equal({'Key': 'foo','Value': 'bar',}) + tag_foo1[0].should.equal({'Key': 'foo1','Value': 'bar1b',}) From 1729681106203f4b692d12283ec01ffadaca20c4 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 21 Sep 2018 10:45:22 -0500 Subject: [PATCH 046/106] formatting fix for E231 missing whitespace after : --- moto/core/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/core/utils.py b/moto/core/utils.py index 2cdcd07f1..21ac8c2e8 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -290,8 +290,8 @@ def amzn_request_id(f): def merge_taglists(taglist_a, taglist_b): ''' Merges two tag lists into a single tag list with Keys in the second list taking precedence''' - tags_a = {t['Key']:t for t in taglist_a} - tags_b = {t['Key']:t for t in taglist_b} + tags_a = {t['Key']: t for t in taglist_a} + tags_b = {t['Key']: t for t in taglist_b} merged_tags = tags_a.copy() merged_tags.update(tags_b) return merged_tags.values() From 1b8b32a663dc52a3a138331bc79fce4ffa38d316 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 21 Sep 2018 11:13:33 -0500 Subject: [PATCH 047/106] add CopyTagsToSnapshot to db_kwargs --- moto/rds2/responses.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/rds2/responses.py b/moto/rds2/responses.py index eddb0042b..6b1da103b 100644 --- a/moto/rds2/responses.py +++ b/moto/rds2/responses.py @@ -19,6 +19,7 @@ class RDS2Response(BaseResponse): "allocated_storage": self._get_int_param('AllocatedStorage'), "availability_zone": self._get_param("AvailabilityZone"), "backup_retention_period": self._get_param("BackupRetentionPeriod"), + "copy_tags_to_snapshot": self._get_param("CopyTagsToSnapshot"), "db_instance_class": self._get_param('DBInstanceClass'), "db_instance_identifier": self._get_param('DBInstanceIdentifier'), "db_name": self._get_param("DBName"), From 6eb490ac78d217a2c570db997c926ba08b5c9503 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 21 Sep 2018 12:03:13 -0500 Subject: [PATCH 048/106] add support for tags to rds snapshots --- moto/rds2/models.py | 18 +++++- tests/test_rds2/test_rds2.py | 111 +++++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 2 deletions(-) diff --git a/moto/rds2/models.py b/moto/rds2/models.py index d564e93b1..b85fe8b05 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -463,6 +463,20 @@ class Snapshot(BaseModel): """) return template.render(snapshot=self, database=self.database) + def get_tags(self): + return self.tags + + def add_tags(self, tags): + new_keys = [tag_set['Key'] for tag_set in tags] + self.tags = [tag_set for tag_set in self.tags if tag_set[ + 'Key'] not in new_keys] + self.tags.extend(tags) + return self.tags + + def remove_tags(self, tag_keys): + self.tags = [tag_set for tag_set in self.tags if tag_set[ + 'Key'] not in tag_keys] + class SecurityGroup(BaseModel): @@ -1037,8 +1051,8 @@ class RDS2Backend(BaseBackend): if resource_name in self.security_groups: return self.security_groups[resource_name].get_tags() elif resource_type == 'snapshot': # DB Snapshot - # TODO: Complete call to tags on resource type DB Snapshot - return [] + if resource_name in self.snapshots: + return self.snapshots[resource_name].get_tags() elif resource_type == 'subgrp': # DB subnet group if resource_name in self.subnet_groups: return self.subnet_groups[resource_name].get_tags() diff --git a/tests/test_rds2/test_rds2.py b/tests/test_rds2/test_rds2.py index 0f802bb5c..cf9805444 100644 --- a/tests/test_rds2/test_rds2.py +++ b/tests/test_rds2/test_rds2.py @@ -700,6 +700,117 @@ def test_remove_tags_db(): len(result['TagList']).should.equal(1) +@mock_rds2 +def test_list_tags_snapshot(): + conn = boto3.client('rds', region_name='us-west-2') + result = conn.list_tags_for_resource( + ResourceName='arn:aws:rds:us-west-2:1234567890:snapshot:foo') + result['TagList'].should.equal([]) + conn.create_db_instance(DBInstanceIdentifier='db-primary-1', + AllocatedStorage=10, + Engine='postgres', + DBName='staging-postgres', + DBInstanceClass='db.m1.small', + MasterUsername='root', + MasterUserPassword='hunter2', + Port=1234, + DBSecurityGroups=["my_sg"]) + snapshot = conn.create_db_snapshot(DBInstanceIdentifier='db-primary-1', + DBSnapshotIdentifier='snapshot-with-tags', + Tags=[ + { + 'Key': 'foo', + 'Value': 'bar', + }, + { + 'Key': 'foo1', + 'Value': 'bar1', + }, + ]) + result = conn.list_tags_for_resource(ResourceName=snapshot['DBSnapshot']['DBSnapshotArn']) + result['TagList'].should.equal([{'Value': 'bar', + 'Key': 'foo'}, + {'Value': 'bar1', + 'Key': 'foo1'}]) + + +@mock_rds2 +def test_add_tags_snapshot(): + conn = boto3.client('rds', region_name='us-west-2') + conn.create_db_instance(DBInstanceIdentifier='db-primary-1', + AllocatedStorage=10, + Engine='postgres', + DBName='staging-postgres', + DBInstanceClass='db.m1.small', + MasterUsername='root', + MasterUserPassword='hunter2', + Port=1234, + DBSecurityGroups=["my_sg"]) + snapshot = conn.create_db_snapshot(DBInstanceIdentifier='db-primary-1', + DBSnapshotIdentifier='snapshot-without-tags', + Tags=[ + { + 'Key': 'foo', + 'Value': 'bar', + }, + { + 'Key': 'foo1', + 'Value': 'bar1', + }, + ]) + result = conn.list_tags_for_resource( + ResourceName='arn:aws:rds:us-west-2:1234567890:snapshot:snapshot-without-tags') + list(result['TagList']).should.have.length_of(2) + conn.add_tags_to_resource(ResourceName='arn:aws:rds:us-west-2:1234567890:snapshot:snapshot-without-tags', + Tags=[ + { + 'Key': 'foo', + 'Value': 'fish', + }, + { + 'Key': 'foo2', + 'Value': 'bar2', + }, + ]) + result = conn.list_tags_for_resource( + ResourceName='arn:aws:rds:us-west-2:1234567890:snapshot:snapshot-without-tags') + list(result['TagList']).should.have.length_of(3) + + +@mock_rds2 +def test_remove_tags_snapshot(): + conn = boto3.client('rds', region_name='us-west-2') + conn.create_db_instance(DBInstanceIdentifier='db-primary-1', + AllocatedStorage=10, + Engine='postgres', + DBName='staging-postgres', + DBInstanceClass='db.m1.small', + MasterUsername='root', + MasterUserPassword='hunter2', + Port=1234, + DBSecurityGroups=["my_sg"]) + snapshot = conn.create_db_snapshot(DBInstanceIdentifier='db-primary-1', + DBSnapshotIdentifier='snapshot-with-tags', + Tags=[ + { + 'Key': 'foo', + 'Value': 'bar', + }, + { + 'Key': 'foo1', + 'Value': 'bar1', + }, + ]) + result = conn.list_tags_for_resource( + ResourceName='arn:aws:rds:us-west-2:1234567890:snapshot:snapshot-with-tags') + list(result['TagList']).should.have.length_of(2) + conn.remove_tags_from_resource( + ResourceName='arn:aws:rds:us-west-2:1234567890:snapshot:snapshot-with-tags', TagKeys=['foo']) + result = conn.list_tags_for_resource( + ResourceName='arn:aws:rds:us-west-2:1234567890:snapshot:snapshot-with-tags') + len(result['TagList']).should.equal(1) + + @mock_rds2 def test_add_tags_option_group(): conn = boto3.client('rds', region_name='us-west-2') From 7daee905a5a62b534d3e098558ce29d40278751f Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 21 Sep 2018 13:28:13 -0500 Subject: [PATCH 049/106] remove merge_taglists as AWS will only take submitted tags or tags from db but not both when creating snapshot --- moto/core/utils.py | 9 --------- moto/rds2/models.py | 15 ++++++++------- moto/rds2/responses.py | 2 +- tests/test_core/test_utils.py | 27 +-------------------------- 4 files changed, 10 insertions(+), 43 deletions(-) diff --git a/moto/core/utils.py b/moto/core/utils.py index 21ac8c2e8..86e7632b0 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -286,12 +286,3 @@ def amzn_request_id(f): return status, headers, body return _wrapper - - -def merge_taglists(taglist_a, taglist_b): - ''' Merges two tag lists into a single tag list with Keys in the second list taking precedence''' - tags_a = {t['Key']: t for t in taglist_a} - tags_b = {t['Key']: t for t in taglist_b} - merged_tags = tags_a.copy() - merged_tags.update(tags_b) - return merged_tags.values() diff --git a/moto/rds2/models.py b/moto/rds2/models.py index b85fe8b05..b37306b12 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -13,7 +13,6 @@ from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel from moto.core.utils import get_random_hex from moto.core.utils import iso_8601_datetime_with_milliseconds -from moto.core.utils import merge_taglists from moto.ec2.models import ec2_backends from .exceptions import (RDSClientError, DBInstanceNotFoundError, @@ -418,10 +417,10 @@ class Database(BaseModel): class Snapshot(BaseModel): - def __init__(self, database, snapshot_id, tags=None): + def __init__(self, database, snapshot_id, tags): self.database = database self.snapshot_id = snapshot_id - self.tags = tags or [] + self.tags = tags self.created_at = iso_8601_datetime_with_milliseconds(datetime.datetime.now()) @property @@ -712,8 +711,10 @@ class RDS2Backend(BaseBackend): raise DBSnapshotAlreadyExistsError(db_snapshot_identifier) if len(self.snapshots) >= int(os.environ.get('MOTO_RDS_SNAPSHOT_LIMIT', '100')): raise SnapshotQuotaExceededError() - if database.copy_tags_to_snapshot: - tags = merge_taglists(database.tags, tags) + if tags is None: + tags = list() + if database.copy_tags_to_snapshot and not tags: + tags = database.get_tags() snapshot = Snapshot(database, db_snapshot_identifier, tags) self.snapshots[db_snapshot_identifier] = snapshot return snapshot @@ -810,13 +811,13 @@ class RDS2Backend(BaseBackend): def delete_database(self, db_instance_identifier, db_snapshot_name=None): if db_instance_identifier in self.databases: + if db_snapshot_name: + self.create_snapshot(db_instance_identifier, db_snapshot_name) database = self.databases.pop(db_instance_identifier) if database.is_replica: primary = self.find_db_from_id(database.source_db_identifier) primary.remove_replica(database) database.status = 'deleting' - if db_snapshot_name: - self.snapshots[db_snapshot_name] = Snapshot(database, db_snapshot_name) return database else: raise DBInstanceNotFoundError(db_instance_identifier) diff --git a/moto/rds2/responses.py b/moto/rds2/responses.py index 6b1da103b..66d4e0c52 100644 --- a/moto/rds2/responses.py +++ b/moto/rds2/responses.py @@ -160,7 +160,7 @@ class RDS2Response(BaseResponse): def create_db_snapshot(self): db_instance_identifier = self._get_param('DBInstanceIdentifier') db_snapshot_identifier = self._get_param('DBSnapshotIdentifier') - tags = self._get_param('Tags', []) + tags = self.unpack_complex_list_params('Tags.Tag', ('Key', 'Value')) snapshot = self.backend.create_snapshot(db_instance_identifier, db_snapshot_identifier, tags) template = self.response_template(CREATE_SNAPSHOT_TEMPLATE) return template.render(snapshot=snapshot) diff --git a/tests/test_core/test_utils.py b/tests/test_core/test_utils.py index 0fa4f3c59..8dbf21716 100644 --- a/tests/test_core/test_utils.py +++ b/tests/test_core/test_utils.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import sure # noqa from freezegun import freeze_time -from moto.core.utils import camelcase_to_underscores, underscores_to_camelcase, unix_time, merge_taglists +from moto.core.utils import camelcase_to_underscores, underscores_to_camelcase, unix_time def test_camelcase_to_underscores(): @@ -28,28 +28,3 @@ def test_underscores_to_camelcase(): @freeze_time("2015-01-01 12:00:00") def test_unix_time(): unix_time().should.equal(1420113600.0) - - -def test_merge_taglists(): - taglist_a = [ - { - 'Key': 'foo', - 'Value': 'bar', - }, - { - 'Key': 'foo1', - 'Value': 'bar1', - }, - ] - taglist_b = [ - { - 'Key': 'foo1', - 'Value': 'bar1b', - }, - ] - taglist_merged = merge_taglists(taglist_a, taglist_b) - len(taglist_merged).should.equal(2) - tag_foo = [t for t in taglist_merged if t['Key']=='foo'] - tag_foo1 = [t for t in taglist_merged if t['Key']=='foo1'] - tag_foo[0].should.equal({'Key': 'foo','Value': 'bar',}) - tag_foo1[0].should.equal({'Key': 'foo1','Value': 'bar1b',}) From 245e3a5f719bcf74846e789d3c3f1b4f9b56ca53 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 21 Sep 2018 13:33:10 -0500 Subject: [PATCH 050/106] formatting fix for E111 indentation is not a multiple of four --- moto/rds2/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/rds2/models.py b/moto/rds2/models.py index b37306b12..159a56ebd 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -712,7 +712,7 @@ class RDS2Backend(BaseBackend): if len(self.snapshots) >= int(os.environ.get('MOTO_RDS_SNAPSHOT_LIMIT', '100')): raise SnapshotQuotaExceededError() if tags is None: - tags = list() + tags = list() if database.copy_tags_to_snapshot and not tags: tags = database.get_tags() snapshot = Snapshot(database, db_snapshot_identifier, tags) From 67a0e06059101e90a0370bc1f7b10bbe1414821c Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 21 Sep 2018 13:54:07 -0500 Subject: [PATCH 051/106] allow for adding and removing tags on rds snapshots --- moto/rds2/models.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/moto/rds2/models.py b/moto/rds2/models.py index 159a56ebd..fee004f76 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -1083,7 +1083,8 @@ class RDS2Backend(BaseBackend): if resource_name in self.security_groups: return self.security_groups[resource_name].remove_tags(tag_keys) elif resource_type == 'snapshot': # DB Snapshot - return None + if resource_name in self.snapshots: + return self.snapshots[resource_name].remove_tags(tag_keys) elif resource_type == 'subgrp': # DB subnet group if resource_name in self.subnet_groups: return self.subnet_groups[resource_name].remove_tags(tag_keys) @@ -1112,7 +1113,8 @@ class RDS2Backend(BaseBackend): if resource_name in self.security_groups: return self.security_groups[resource_name].add_tags(tags) elif resource_type == 'snapshot': # DB Snapshot - return [] + if resource_name in self.snapshots: + return self.snapshots[resource_name].add_tags(tags) elif resource_type == 'subgrp': # DB subnet group if resource_name in self.subnet_groups: return self.subnet_groups[resource_name].add_tags(tags) From 57f9691a52245ffd9d7c0317e90953d054070c65 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sat, 22 Sep 2018 15:36:25 -0400 Subject: [PATCH 052/106] Version 1.3.6 --- CHANGELOG.md | 5 +++++ moto/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 202da6ce6..7f7ee4448 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ Moto Changelog =================== +1.3.6 +----- + + * Fix boto3 pinning. + 1.3.5 ----- diff --git a/moto/__init__.py b/moto/__init__.py index b7b653200..37628e6e4 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging # logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '1.3.5' +__version__ = '1.3.6' from .acm import mock_acm # flake8: noqa from .apigateway import mock_apigateway, mock_apigateway_deprecated # flake8: noqa diff --git a/setup.py b/setup.py index 919b9849b..dad9ab9bb 100755 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ else: setup( name='moto', - version='1.3.5', + version='1.3.6', description='A library that allows your python tests to easily' ' mock out the boto library', author='Steve Pulec', From 8bdcc6244d118ab58d8eb50ceb0a15dfbeeecdf6 Mon Sep 17 00:00:00 2001 From: Justin Eyster Date: Mon, 24 Sep 2018 15:58:35 -0400 Subject: [PATCH 053/106] Addresses security vulnerability in cryptography<2.3 Discovered using pipenv's security check feature that there's a vulnerability in the cryptography package versions<2.3. > Checking installed package safety... 36351: cryptography >=1.9.0,<2.3 resolved (2.2.2 installed)! python-cryptography versions >=1.9.0 and <2.3 did not enforce a minimum tag length for finalize_with_tag API. If a user did not validate the input length prior to passing it to finalize_with_tag an attacker could craft an invalid payload with a shortened tag (e.g. 1 byte) such that they would have a 1 in 256 chance of passing the MAC check. GCM tag forgeries can cause key leakage. More details here: http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-10903 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index dad9ab9bb..98780dd5a 100755 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ install_requires = [ "boto>=2.36.0", "boto3>=1.6.16,<1.8", "botocore>=1.9.16,<1.11", - "cryptography>=2.0.0", + "cryptography>=2.3.0", "requests>=2.5", "xmltodict", "six>1.9", From 2253bbf36123cf542f95f33b0037984686d199d6 Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Tue, 7 Aug 2018 13:55:13 +0200 Subject: [PATCH 054/106] Changed the 'create_access_token' function in order to add the extra data into 'create_jwt' function --- moto/cognitoidp/models.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 52a73f89f..1119edcbe 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -84,7 +84,11 @@ class CognitoIdpUserPool(BaseModel): return refresh_token def create_access_token(self, client_id, username): - access_token, expires_in = self.create_jwt(client_id, username) + extra_data = self.get_user_extra_data_by_client_id( + client_id, username + ) + access_token, expires_in = self.create_jwt(client_id, username, + extra_data=extra_data) self.access_tokens[access_token] = (client_id, username) return access_token, expires_in @@ -97,6 +101,21 @@ class CognitoIdpUserPool(BaseModel): id_token, _ = self.create_id_token(client_id, username) return access_token, id_token, expires_in + def get_user_extra_data_by_client_id(self, client_id, username): + extra_data = {} + current_client = self.clients.get(client_id, None) + if current_client: + for readable_field in current_client.get_readable_fields(): + attribute = list(filter( + lambda f: f['Name'] == readable_field, + self.users.get(username).attributes + )) + if len(attribute) > 0: + extra_data.update({ + attribute[0]['Name']: attribute[0]['Value'] + }) + return extra_data + class CognitoIdpUserPoolDomain(BaseModel): @@ -138,6 +157,9 @@ class CognitoIdpUserPoolClient(BaseModel): return user_pool_client_json + def get_readable_fields(self): + return self.extended_config.get('ReadAttributes', []) + class CognitoIdpIdentityProvider(BaseModel): From ab8f4dd159fd66ba8f9dedb9e8445a3ca33fb308 Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Tue, 7 Aug 2018 13:55:32 +0200 Subject: [PATCH 055/106] Testing new feature --- tests/test_cognitoidp/test_cognitoidp.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index b2bd469ce..fda24146d 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -325,6 +325,7 @@ def test_delete_identity_providers(): def test_admin_create_user(): conn = boto3.client("cognito-idp", "us-west-2") + username = str(uuid.uuid4()) username = str(uuid.uuid4()) value = str(uuid.uuid4()) user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] @@ -399,15 +400,22 @@ def authentication_flow(conn): username = str(uuid.uuid4()) temporary_password = str(uuid.uuid4()) user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + user_attribute_name = str(uuid.uuid4()) + user_attribute_value = str(uuid.uuid4()) client_id = conn.create_user_pool_client( UserPoolId=user_pool_id, ClientName=str(uuid.uuid4()), + ReadAttributes=[user_attribute_name] )["UserPoolClient"]["ClientId"] conn.admin_create_user( UserPoolId=user_pool_id, Username=username, TemporaryPassword=temporary_password, + UserAttributes=[{ + 'Name': user_attribute_name, + 'Value': user_attribute_value + }] ) result = conn.admin_initiate_auth( @@ -446,6 +454,9 @@ def authentication_flow(conn): "access_token": result["AuthenticationResult"]["AccessToken"], "username": username, "password": new_password, + "additional_fields": { + user_attribute_name: user_attribute_value + } } @@ -475,6 +486,8 @@ def test_token_legitimacy(): access_claims = json.loads(jws.verify(access_token, json_web_key, "RS256")) access_claims["iss"].should.equal(issuer) access_claims["aud"].should.equal(client_id) + for k, v in outputs["additional_fields"].items(): + access_claims[k].should.equal(v) @mock_cognitoidp From 9adb7c818df7bbaf842c49b82cba8579fb7bd8a9 Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Fri, 10 Aug 2018 16:07:27 +0200 Subject: [PATCH 056/106] Added extra test --- tests/test_cognitoidp/test_cognitoidp.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index fda24146d..d1d57f307 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -4,8 +4,10 @@ import boto3 import json import os import uuid +import jwt from jose import jws + from moto import mock_cognitoidp import sure # noqa @@ -446,6 +448,9 @@ def authentication_flow(conn): result["AuthenticationResult"]["IdToken"].should_not.be.none result["AuthenticationResult"]["AccessToken"].should_not.be.none + jwt.decode( + result["AuthenticationResult"]["AccessToken"], verify=False + ).get(user_attribute_name, '').should.be.equal(user_attribute_value) return { "user_pool_id": user_pool_id, From c2b8a7bbb0d1b53b20e7b019a1c8b65b9359abab Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Fri, 10 Aug 2018 16:26:09 +0200 Subject: [PATCH 057/106] Added package dependency --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 111cd5f3f..54802ad89 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,3 +15,4 @@ click==6.7 inflection==0.3.1 lxml==4.2.3 beautifulsoup4==4.6.0 +PyJWT==1.6.4 From e69d2834e8905f42eb02ae0a451acfe891e64a50 Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Fri, 10 Aug 2018 16:52:48 +0200 Subject: [PATCH 058/106] Removing failing test in order to check coverage and Travis test --- requirements-dev.txt | 1 - tests/test_cognitoidp/test_cognitoidp.py | 4 ---- 2 files changed, 5 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 54802ad89..111cd5f3f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,4 +15,3 @@ click==6.7 inflection==0.3.1 lxml==4.2.3 beautifulsoup4==4.6.0 -PyJWT==1.6.4 diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index d1d57f307..98153feef 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -4,7 +4,6 @@ import boto3 import json import os import uuid -import jwt from jose import jws @@ -448,9 +447,6 @@ def authentication_flow(conn): result["AuthenticationResult"]["IdToken"].should_not.be.none result["AuthenticationResult"]["AccessToken"].should_not.be.none - jwt.decode( - result["AuthenticationResult"]["AccessToken"], verify=False - ).get(user_attribute_name, '').should.be.equal(user_attribute_value) return { "user_pool_id": user_pool_id, From da76ea633bfc6fef1ac1619dd311c379b253b833 Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Fri, 28 Sep 2018 16:31:57 +0200 Subject: [PATCH 059/106] Removed redundand line --- tests/test_cognitoidp/test_cognitoidp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index 98153feef..b879cc42a 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -326,7 +326,6 @@ def test_delete_identity_providers(): def test_admin_create_user(): conn = boto3.client("cognito-idp", "us-west-2") - username = str(uuid.uuid4()) username = str(uuid.uuid4()) value = str(uuid.uuid4()) user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] From edbc57e00d74bdfc39a604e692c5607ca57ac705 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sat, 14 Jul 2018 11:35:37 -0700 Subject: [PATCH 060/106] add support for AWS Organizations endpoints covers so far: - create_organization - describe_organization - create_account - describe_account - list_accounts all tests passing. could use some advise from maintaners. --- moto/__init__.py | 1 + moto/backends.py | 2 + moto/organizations/__init__.py | 6 + moto/organizations/models.py | 131 ++++++++++++ moto/organizations/responses.py | 48 +++++ moto/organizations/urls.py | 10 + moto/organizations/utils.py | 34 +++ tests/test_organizations/__init__.py | 0 tests/test_organizations/object_syntax.py | 24 +++ .../test_organizations_boto3.py | 193 ++++++++++++++++++ .../test_organizations_utils.py | 26 +++ 11 files changed, 475 insertions(+) create mode 100644 moto/organizations/__init__.py create mode 100644 moto/organizations/models.py create mode 100644 moto/organizations/responses.py create mode 100644 moto/organizations/urls.py create mode 100644 moto/organizations/utils.py create mode 100644 tests/test_organizations/__init__.py create mode 100644 tests/test_organizations/object_syntax.py create mode 100644 tests/test_organizations/test_organizations_boto3.py create mode 100644 tests/test_organizations/test_organizations_utils.py diff --git a/moto/__init__.py b/moto/__init__.py index 0ce5e54d1..2301b7ca1 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -27,6 +27,7 @@ from .glacier import mock_glacier, mock_glacier_deprecated # flake8: noqa from .iam import mock_iam, mock_iam_deprecated # flake8: noqa from .kinesis import mock_kinesis, mock_kinesis_deprecated # flake8: noqa from .kms import mock_kms, mock_kms_deprecated # flake8: noqa +from .organizations import mock_organizations # flake8: noqa from .opsworks import mock_opsworks, mock_opsworks_deprecated # flake8: noqa from .polly import mock_polly # flake8: noqa from .rds import mock_rds, mock_rds_deprecated # flake8: noqa diff --git a/moto/backends.py b/moto/backends.py index cd8fe174f..25fcec09a 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -26,6 +26,7 @@ from moto.kinesis import kinesis_backends from moto.kms import kms_backends from moto.logs import logs_backends from moto.opsworks import opsworks_backends +from moto.organizations import organizations_backends from moto.polly import polly_backends from moto.rds2 import rds2_backends from moto.redshift import redshift_backends @@ -72,6 +73,7 @@ BACKENDS = { 'kinesis': kinesis_backends, 'kms': kms_backends, 'opsworks': opsworks_backends, + 'organizations': organizations_backends, 'polly': polly_backends, 'redshift': redshift_backends, 'rds': rds2_backends, diff --git a/moto/organizations/__init__.py b/moto/organizations/__init__.py new file mode 100644 index 000000000..372782dd3 --- /dev/null +++ b/moto/organizations/__init__.py @@ -0,0 +1,6 @@ +from __future__ import unicode_literals +from .models import organizations_backend +from ..core.models import base_decorator + +organizations_backends = {"global": organizations_backend} +mock_organizations = base_decorator(organizations_backends) diff --git a/moto/organizations/models.py b/moto/organizations/models.py new file mode 100644 index 000000000..50212af29 --- /dev/null +++ b/moto/organizations/models.py @@ -0,0 +1,131 @@ +from __future__ import unicode_literals + +import datetime +import time + +from moto.core import BaseBackend, BaseModel +from moto.core.utils import unix_time +from moto.organizations import utils + +MASTER_ACCOUNT_ID = '123456789012' +MASTER_ACCOUNT_EMAIL = 'fakeorg@moto-example.com' +ORGANIZATION_ARN_FORMAT = 'arn:aws:organizations::{0}:organization/{1}' +MASTER_ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{0}' +ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{2}' + + +class FakeOrganization(BaseModel): + + def __init__(self, feature_set): + self.id = utils.make_random_org_id() + self.feature_set = feature_set + self.master_account_id = MASTER_ACCOUNT_ID + self.master_account_email = MASTER_ACCOUNT_EMAIL + self.available_policy_types = [{ + 'Type': 'SERVICE_CONTROL_POLICY', + 'Status': 'ENABLED' + }] + + @property + def arn(self): + return ORGANIZATION_ARN_FORMAT.format(self.master_account_id, self.id) + + @property + def master_account_arn(self): + return MASTER_ACCOUNT_ARN_FORMAT.format(self.master_account_id, self.id) + + def _describe(self): + return { + 'Organization': { + 'Id': self.id, + 'Arn': self.arn, + 'FeatureSet': self.feature_set, + 'MasterAccountArn': self.master_account_arn, + 'MasterAccountId': self.master_account_id, + 'MasterAccountEmail': self.master_account_email, + 'AvailablePolicyTypes': self.available_policy_types, + } + } + + +class FakeAccount(BaseModel): + + def __init__(self, organization, **kwargs): + self.organization_id = organization.id + self.master_account_id = organization.master_account_id + self.create_account_status_id = utils.make_random_create_account_status_id() + self.account_id = utils.make_random_account_id() + self.account_name = kwargs['AccountName'] + self.email = kwargs['Email'] + self.create_time = datetime.datetime.utcnow() + self.status = 'ACTIVE' + self.joined_method = 'CREATED' + + @property + def arn(self): + return ACCOUNT_ARN_FORMAT.format( + self.master_account_id, + self.organization_id, + self.account_id + ) + + @property + def create_account_status(self): + return { + 'CreateAccountStatus': { + 'Id': self.create_account_status_id, + 'AccountName': self.account_name, + 'State': 'SUCCEEDED', + 'RequestedTimestamp': unix_time(self.create_time), + 'CompletedTimestamp': unix_time(self.create_time), + 'AccountId': self.account_id, + } + } + + def describe(self): + return { + 'Account': { + 'Id': self.account_id, + 'Arn': self.arn, + 'Email': self.email, + 'Name': self.account_name, + 'Status': self.status, + 'JoinedMethod': self.joined_method, + 'JoinedTimestamp': unix_time(self.create_time), + } + } + + +class OrganizationsBackend(BaseBackend): + + def __init__(self): + self.org = None + self.accounts = [] + + def create_organization(self, **kwargs): + self.org = FakeOrganization(kwargs['FeatureSet']) + return self.org._describe() + + def describe_organization(self): + return self.org._describe() + + def create_account(self, **kwargs): + new_account = FakeAccount(self.org, **kwargs) + self.accounts.append(new_account) + return new_account.create_account_status + + def describe_account(self, **kwargs): + account = [account for account in self.accounts + if account.account_id == kwargs['AccountId']][0] + return account.describe() + + def list_accounts(self): + return dict( + Accounts=[account.describe()['Account'] for account in self.accounts] + ) + + +organizations_backend = OrganizationsBackend() + + + diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py new file mode 100644 index 000000000..10288dedb --- /dev/null +++ b/moto/organizations/responses.py @@ -0,0 +1,48 @@ +from __future__ import unicode_literals +import json + +from moto.core.responses import BaseResponse +from .models import organizations_backend + + +class OrganizationsResponse(BaseResponse): + + @property + def organizations_backend(self): + return organizations_backend + + @property + def request_params(self): + try: + return json.loads(self.body) + except ValueError: + return {} + + def _get_param(self, param, default=None): + return self.request_params.get(param, default) + + def create_organization(self): + return json.dumps( + self.organizations_backend.create_organization(**self.request_params) + ) + + def describe_organization(self): + return json.dumps( + self.organizations_backend.describe_organization() + ) + + def create_account(self): + return json.dumps( + self.organizations_backend.create_account(**self.request_params) + ) + + def describe_account(self): + return json.dumps( + self.organizations_backend.describe_account(**self.request_params) + ) + + def list_accounts(self): + return json.dumps( + self.organizations_backend.list_accounts() + ) + diff --git a/moto/organizations/urls.py b/moto/organizations/urls.py new file mode 100644 index 000000000..7911f5b53 --- /dev/null +++ b/moto/organizations/urls.py @@ -0,0 +1,10 @@ +from __future__ import unicode_literals +from .responses import OrganizationsResponse + +url_bases = [ + "https?://organizations.(.+).amazonaws.com", +] + +url_paths = { + '{0}/$': OrganizationsResponse.dispatch, +} diff --git a/moto/organizations/utils.py b/moto/organizations/utils.py new file mode 100644 index 000000000..5916dc5ea --- /dev/null +++ b/moto/organizations/utils.py @@ -0,0 +1,34 @@ +from __future__ import unicode_literals + +import random +import string + +CHARSET=string.ascii_lowercase + string.digits +ORG_ID_SIZE = 10 +ROOT_ID_SIZE = 4 +ACCOUNT_ID_SIZE = 12 +CREATE_ACCOUNT_STATUS_ID_SIZE = 8 + + +def make_random_org_id(): + # The regex pattern for an organization ID string requires "o-" + # followed by from 10 to 32 lower-case letters or digits. + # e.g. 'o-vipjnq5z86' + return 'o-' + ''.join(random.choice(CHARSET) for x in range(ORG_ID_SIZE)) + +def make_random_root_id(): + # The regex pattern for a root ID string requires "r-" followed by + # from 4 to 32 lower-case letters or digits. + # e.g. 'r-3zwx' + return 'r-' + ''.join(random.choice(CHARSET) for x in range(ROOT_ID_SIZE)) + +def make_random_account_id(): + # The regex pattern for an account ID string requires exactly 12 digits. + # e.g. '488633172133' + return ''.join([random.choice(string.digits) for n in range(ACCOUNT_ID_SIZE)]) + +def make_random_create_account_status_id(): + # The regex pattern for an create account request ID string requires + # "car-" followed by from 8 to 32 lower-case letters or digits. + # e.g. 'car-35gxzwrp' + return 'car-' + ''.join(random.choice(CHARSET) for x in range(CREATE_ACCOUNT_STATUS_ID_SIZE)) diff --git a/tests/test_organizations/__init__.py b/tests/test_organizations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_organizations/object_syntax.py b/tests/test_organizations/object_syntax.py new file mode 100644 index 000000000..35437be62 --- /dev/null +++ b/tests/test_organizations/object_syntax.py @@ -0,0 +1,24 @@ +""" +Temporary functions for checking object structures while specing out +models. This module will go away. +""" + +import yaml +import moto +from moto import organizations as orgs + + +# utils +print(orgs.utils.make_random_org_id()) +print(orgs.utils.make_random_root_id()) +print(orgs.utils.make_random_account_id()) +print(orgs.utils.make_random_create_account_id()) + +# models +my_org = orgs.models.FakeOrganization(feature_set = 'ALL') +print(yaml.dump(my_org._describe())) +#assert False + +my_account = orgs.models.FakeAccount(my_org, AccountName='blee01', Email='blee01@moto-example.org') +print(yaml.dump(my_account)) +#assert False diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py new file mode 100644 index 000000000..fa70f3306 --- /dev/null +++ b/tests/test_organizations/test_organizations_boto3.py @@ -0,0 +1,193 @@ +from __future__ import unicode_literals + +import boto3 +import botocore.exceptions +import sure # noqa +import yaml +import re +import datetime + +import moto +from moto import mock_organizations +from moto.organizations.models import ( + MASTER_ACCOUNT_ID, + MASTER_ACCOUNT_EMAIL, + ORGANIZATION_ARN_FORMAT, + MASTER_ACCOUNT_ARN_FORMAT, + ACCOUNT_ARN_FORMAT, +) +from .test_organizations_utils import ( + ORG_ID_REGEX, + ROOT_ID_REGEX, + ACCOUNT_ID_REGEX, + CREATE_ACCOUNT_STATUS_ID_REGEX, +) + +EMAIL_REGEX = "^.+@[a-zA-Z0-9-.]+.[a-zA-Z]{2,3}|[0-9]{1,3}$" + + +def validate_organization(response): + org = response['Organization'] + sorted(org.keys()).should.equal([ + 'Arn', + 'AvailablePolicyTypes', + 'FeatureSet', + 'Id', + 'MasterAccountArn', + 'MasterAccountEmail', + 'MasterAccountId', + ]) + org['Id'].should.match(ORG_ID_REGEX) + org['MasterAccountId'].should.equal(MASTER_ACCOUNT_ID) + org['MasterAccountArn'].should.equal(MASTER_ACCOUNT_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + )) + org['Arn'].should.equal(ORGANIZATION_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + )) + org['MasterAccountEmail'].should.equal(MASTER_ACCOUNT_EMAIL) + org['FeatureSet'].should.be.within(['ALL', 'CONSOLIDATED_BILLING']) + org['AvailablePolicyTypes'].should.equal([{ + 'Type': 'SERVICE_CONTROL_POLICY', + 'Status': 'ENABLED' + }]) + # + #'Organization': { + # 'Id': 'string', + # 'Arn': 'string', + # 'FeatureSet': 'ALL'|'CONSOLIDATED_BILLING', + # 'MasterAccountArn': 'string', + # 'MasterAccountId': 'string', + # 'MasterAccountEmail': 'string', + # 'AvailablePolicyTypes': [ + # { + # 'Type': 'SERVICE_CONTROL_POLICY', + # 'Status': 'ENABLED'|'PENDING_ENABLE'|'PENDING_DISABLE' + # }, + # ] + #} + +def validate_account(org, account): + sorted(account.keys()).should.equal([ + 'Arn', + 'Email', + 'Id', + 'JoinedMethod', + 'JoinedTimestamp', + 'Name', + 'Status', + ]) + account['Id'].should.match(ACCOUNT_ID_REGEX) + account['Arn'].should.equal(ACCOUNT_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + account['Id'], + )) + account['Email'].should.match(EMAIL_REGEX) + account['JoinedMethod'].should.be.within(['INVITED', 'CREATED']) + account['Status'].should.be.within(['ACTIVE', 'SUSPENDED']) + account['Name'].should.be.a(str) + account['JoinedTimestamp'].should.be.a(datetime.datetime) + #'Account': { + # 'Id': 'string', + # 'Arn': 'string', + # 'Email': 'string', + # 'Name': 'string', + # 'Status': 'ACTIVE'|'SUSPENDED', + # 'JoinedMethod': 'INVITED'|'CREATED', + # 'JoinedTimestamp': datetime(2015, 1, 1) + #} + +def validate_create_account_status(create_status): + sorted(create_status.keys()).should.equal([ + 'AccountId', + 'AccountName', + 'CompletedTimestamp', + 'Id', + 'RequestedTimestamp', + 'State', + ]) + create_status['Id'].should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) + create_status['AccountId'].should.match(ACCOUNT_ID_REGEX) + create_status['AccountName'].should.be.a(str) + create_status['State'].should.equal('SUCCEEDED') + create_status['RequestedTimestamp'].should.be.a(datetime.datetime) + create_status['CompletedTimestamp'].should.be.a(datetime.datetime) + #'CreateAccountStatus': { + # 'Id': 'string', + # 'AccountName': 'string', + # 'State': 'IN_PROGRESS'|'SUCCEEDED'|'FAILED', + # 'RequestedTimestamp': datetime(2015, 1, 1), + # 'CompletedTimestamp': datetime(2015, 1, 1), + # 'AccountId': 'string', + # 'FailureReason': 'ACCOUNT_LIMIT_EXCEEDED'|'EMAIL_ALREADY_EXISTS'|'INVALID_ADDRESS'|'INVALID_EMAIL'|'CONCURRENT_ACCOUNT_MODIFICATION'|'INTERNAL_FAILURE' + #} + +@mock_organizations +def test_create_organization(): + client = boto3.client('organizations', region_name='us-east-1') + response = client.create_organization(FeatureSet='ALL') + #print(yaml.dump(response)) + validate_organization(response) + response['Organization']['FeatureSet'].should.equal('ALL') + #assert False + +@mock_organizations +def test_describe_organization(): + client = boto3.client('organizations', region_name='us-east-1') + client.create_organization(FeatureSet='ALL') + response = client.describe_organization() + #print(yaml.dump(response)) + validate_organization(response) + #assert False + + +mockname = 'mock-account' +mockdomain = 'moto-example.org' +mockemail = '@'.join([mockname, mockdomain]) + +@mock_organizations +def test_create_account(): + client = boto3.client('organizations', region_name='us-east-1') + client.create_organization(FeatureSet='ALL') + create_status = client.create_account( + AccountName=mockname, Email=mockemail)['CreateAccountStatus'] + #print(yaml.dump(create_status, default_flow_style=False)) + validate_create_account_status(create_status) + create_status['AccountName'].should.equal(mockname) + #assert False + +@mock_organizations +def test_describe_account(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + account_id = client.create_account( + AccountName=mockname, Email=mockemail)['CreateAccountStatus']['AccountId'] + response = client.describe_account(AccountId=account_id) + #print(yaml.dump(response, default_flow_style=False)) + validate_account(org, response['Account']) + response['Account']['Name'].should.equal(mockname) + response['Account']['Email'].should.equal(mockemail) + #assert False + +@mock_organizations +def test_list_accounts(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + for i in range(5): + name = mockname + str(i) + email = name + '@' + mockdomain + client.create_account(AccountName=name, Email=email) + response = client.list_accounts() + #print(yaml.dump(response, default_flow_style=False)) + response.should.have.key('Accounts') + accounts = response['Accounts'] + len(accounts).should.equal(5) + for account in accounts: + validate_account(org, account) + accounts[3]['Name'].should.equal(mockname + '3') + accounts[2]['Email'].should.equal(mockname + '2' + '@' + mockdomain) + #assert False + diff --git a/tests/test_organizations/test_organizations_utils.py b/tests/test_organizations/test_organizations_utils.py new file mode 100644 index 000000000..8144c327f --- /dev/null +++ b/tests/test_organizations/test_organizations_utils.py @@ -0,0 +1,26 @@ +from __future__ import unicode_literals + +import sure # noqa +import moto +from moto.organizations import utils + +ORG_ID_REGEX = r'o-[a-z0-9]{%s}' % utils.ORG_ID_SIZE +ROOT_ID_REGEX = r'r-[a-z0-9]{%s}' % utils.ROOT_ID_SIZE +ACCOUNT_ID_REGEX = r'[0-9]{%s}' % utils.ACCOUNT_ID_SIZE +CREATE_ACCOUNT_STATUS_ID_REGEX = r'car-[a-z0-9]{%s}' % utils.CREATE_ACCOUNT_STATUS_ID_SIZE + +def test_make_random_org_id(): + org_id = utils.make_random_org_id() + org_id.should.match(ORG_ID_REGEX) + +def test_make_random_root_id(): + org_id = utils.make_random_root_id() + org_id.should.match(ROOT_ID_REGEX) + +def test_make_random_account_id(): + account_id = utils.make_random_account_id() + account_id.should.match(ACCOUNT_ID_REGEX) + +def test_make_random_create_account_status_id(): + create_account_status_id = utils.make_random_create_account_status_id() + create_account_status_id.should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) From f20898da0ef6c9a8c46866fb596e713054afc349 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sat, 14 Jul 2018 12:28:19 -0700 Subject: [PATCH 061/106] add info on organizations support to docs --- IMPLEMENTATION_COVERAGE.md | 12 ++++++------ README.md | 2 ++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 411f55a8b..837850ac6 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3147,22 +3147,22 @@ - [ ] update_server - [ ] update_server_engine_attributes -## organizations - 0% implemented +## organizations - 8% implemented - [ ] accept_handshake - [ ] attach_policy - [ ] cancel_handshake -- [ ] create_account -- [ ] create_organization +- [X] create_account +- [X] create_organization - [ ] create_organizational_unit - [ ] create_policy - [ ] decline_handshake - [ ] delete_organization - [ ] delete_organizational_unit - [ ] delete_policy -- [ ] describe_account +- [X] describe_account - [ ] describe_create_account_status - [ ] describe_handshake -- [ ] describe_organization +- [X] describe_organization - [ ] describe_organizational_unit - [ ] describe_policy - [ ] detach_policy @@ -3173,7 +3173,7 @@ - [ ] enable_policy_type - [ ] invite_account_to_organization - [ ] leave_organization -- [ ] list_accounts +- [X] list_accounts - [ ] list_accounts_for_parent - [ ] list_aws_service_access_for_organization - [ ] list_children diff --git a/README.md b/README.md index a6926a58f..e4fcb6508 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,8 @@ It gets even better! Moto isn't just for Python code and it isn't just for S3. L |------------------------------------------------------------------------------| | KMS | @mock_kms | basic endpoints done | |------------------------------------------------------------------------------| +| Organizations | @mock_organizations | some endpoints done | +|------------------------------------------------------------------------------| | Polly | @mock_polly | all endpoints done | |------------------------------------------------------------------------------| | RDS | @mock_rds | core endpoints done | From c40d2be646884e333098bdb5fb1941f8f327cba4 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sat, 14 Jul 2018 13:23:15 -0700 Subject: [PATCH 062/106] organizations: clean up for flake8 --- moto/organizations/models.py | 8 +-- moto/organizations/responses.py | 1 - moto/organizations/utils.py | 11 ++-- tests/test_organizations/object_syntax.py | 3 +- .../test_organizations_boto3.py | 53 +++++-------------- .../test_organizations_utils.py | 5 +- 6 files changed, 26 insertions(+), 55 deletions(-) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 50212af29..e1d59c1da 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import datetime -import time from moto.core import BaseBackend, BaseModel from moto.core.utils import unix_time @@ -25,7 +24,7 @@ class FakeOrganization(BaseModel): 'Type': 'SERVICE_CONTROL_POLICY', 'Status': 'ENABLED' }] - + @property def arn(self): return ORGANIZATION_ARN_FORMAT.format(self.master_account_id, self.id) @@ -115,7 +114,7 @@ class OrganizationsBackend(BaseBackend): return new_account.create_account_status def describe_account(self, **kwargs): - account = [account for account in self.accounts + account = [account for account in self.accounts if account.account_id == kwargs['AccountId']][0] return account.describe() @@ -126,6 +125,3 @@ class OrganizationsBackend(BaseBackend): organizations_backend = OrganizationsBackend() - - - diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py index 10288dedb..7c8d45014 100644 --- a/moto/organizations/responses.py +++ b/moto/organizations/responses.py @@ -45,4 +45,3 @@ class OrganizationsResponse(BaseResponse): return json.dumps( self.organizations_backend.list_accounts() ) - diff --git a/moto/organizations/utils.py b/moto/organizations/utils.py index 5916dc5ea..1b111c1a7 100644 --- a/moto/organizations/utils.py +++ b/moto/organizations/utils.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import random import string -CHARSET=string.ascii_lowercase + string.digits +CHARSET = string.ascii_lowercase + string.digits ORG_ID_SIZE = 10 ROOT_ID_SIZE = 4 ACCOUNT_ID_SIZE = 12 @@ -11,24 +11,27 @@ CREATE_ACCOUNT_STATUS_ID_SIZE = 8 def make_random_org_id(): - # The regex pattern for an organization ID string requires "o-" + # The regex pattern for an organization ID string requires "o-" # followed by from 10 to 32 lower-case letters or digits. # e.g. 'o-vipjnq5z86' return 'o-' + ''.join(random.choice(CHARSET) for x in range(ORG_ID_SIZE)) + def make_random_root_id(): - # The regex pattern for a root ID string requires "r-" followed by + # The regex pattern for a root ID string requires "r-" followed by # from 4 to 32 lower-case letters or digits. # e.g. 'r-3zwx' return 'r-' + ''.join(random.choice(CHARSET) for x in range(ROOT_ID_SIZE)) + def make_random_account_id(): # The regex pattern for an account ID string requires exactly 12 digits. # e.g. '488633172133' return ''.join([random.choice(string.digits) for n in range(ACCOUNT_ID_SIZE)]) + def make_random_create_account_status_id(): - # The regex pattern for an create account request ID string requires + # The regex pattern for an create account request ID string requires # "car-" followed by from 8 to 32 lower-case letters or digits. # e.g. 'car-35gxzwrp' return 'car-' + ''.join(random.choice(CHARSET) for x in range(CREATE_ACCOUNT_STATUS_ID_SIZE)) diff --git a/tests/test_organizations/object_syntax.py b/tests/test_organizations/object_syntax.py index 35437be62..2779d1d07 100644 --- a/tests/test_organizations/object_syntax.py +++ b/tests/test_organizations/object_syntax.py @@ -4,7 +4,6 @@ models. This module will go away. """ import yaml -import moto from moto import organizations as orgs @@ -15,7 +14,7 @@ print(orgs.utils.make_random_account_id()) print(orgs.utils.make_random_create_account_id()) # models -my_org = orgs.models.FakeOrganization(feature_set = 'ALL') +my_org = orgs.models.FakeOrganization(feature_set='ALL') print(yaml.dump(my_org._describe())) #assert False diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index fa70f3306..7ae8d9577 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -1,13 +1,9 @@ from __future__ import unicode_literals import boto3 -import botocore.exceptions import sure # noqa -import yaml -import re import datetime -import moto from moto import mock_organizations from moto.organizations.models import ( MASTER_ACCOUNT_ID, @@ -53,21 +49,7 @@ def validate_organization(response): 'Type': 'SERVICE_CONTROL_POLICY', 'Status': 'ENABLED' }]) - # - #'Organization': { - # 'Id': 'string', - # 'Arn': 'string', - # 'FeatureSet': 'ALL'|'CONSOLIDATED_BILLING', - # 'MasterAccountArn': 'string', - # 'MasterAccountId': 'string', - # 'MasterAccountEmail': 'string', - # 'AvailablePolicyTypes': [ - # { - # 'Type': 'SERVICE_CONTROL_POLICY', - # 'Status': 'ENABLED'|'PENDING_ENABLE'|'PENDING_DISABLE' - # }, - # ] - #} + def validate_account(org, account): sorted(account.keys()).should.equal([ @@ -84,21 +66,13 @@ def validate_account(org, account): org['MasterAccountId'], org['Id'], account['Id'], - )) + )) account['Email'].should.match(EMAIL_REGEX) account['JoinedMethod'].should.be.within(['INVITED', 'CREATED']) account['Status'].should.be.within(['ACTIVE', 'SUSPENDED']) account['Name'].should.be.a(str) account['JoinedTimestamp'].should.be.a(datetime.datetime) - #'Account': { - # 'Id': 'string', - # 'Arn': 'string', - # 'Email': 'string', - # 'Name': 'string', - # 'Status': 'ACTIVE'|'SUSPENDED', - # 'JoinedMethod': 'INVITED'|'CREATED', - # 'JoinedTimestamp': datetime(2015, 1, 1) - #} + def validate_create_account_status(create_status): sorted(create_status.keys()).should.equal([ @@ -115,15 +89,7 @@ def validate_create_account_status(create_status): create_status['State'].should.equal('SUCCEEDED') create_status['RequestedTimestamp'].should.be.a(datetime.datetime) create_status['CompletedTimestamp'].should.be.a(datetime.datetime) - #'CreateAccountStatus': { - # 'Id': 'string', - # 'AccountName': 'string', - # 'State': 'IN_PROGRESS'|'SUCCEEDED'|'FAILED', - # 'RequestedTimestamp': datetime(2015, 1, 1), - # 'CompletedTimestamp': datetime(2015, 1, 1), - # 'AccountId': 'string', - # 'FailureReason': 'ACCOUNT_LIMIT_EXCEEDED'|'EMAIL_ALREADY_EXISTS'|'INVALID_ADDRESS'|'INVALID_EMAIL'|'CONCURRENT_ACCOUNT_MODIFICATION'|'INTERNAL_FAILURE' - #} + @mock_organizations def test_create_organization(): @@ -134,6 +100,7 @@ def test_create_organization(): response['Organization']['FeatureSet'].should.equal('ALL') #assert False + @mock_organizations def test_describe_organization(): client = boto3.client('organizations', region_name='us-east-1') @@ -148,23 +115,27 @@ mockname = 'mock-account' mockdomain = 'moto-example.org' mockemail = '@'.join([mockname, mockdomain]) + @mock_organizations def test_create_account(): client = boto3.client('organizations', region_name='us-east-1') client.create_organization(FeatureSet='ALL') create_status = client.create_account( - AccountName=mockname, Email=mockemail)['CreateAccountStatus'] + AccountName=mockname, Email=mockemail + )['CreateAccountStatus'] #print(yaml.dump(create_status, default_flow_style=False)) validate_create_account_status(create_status) create_status['AccountName'].should.equal(mockname) #assert False + @mock_organizations def test_describe_account(): client = boto3.client('organizations', region_name='us-east-1') org = client.create_organization(FeatureSet='ALL')['Organization'] account_id = client.create_account( - AccountName=mockname, Email=mockemail)['CreateAccountStatus']['AccountId'] + AccountName=mockname, Email=mockemail + )['CreateAccountStatus']['AccountId'] response = client.describe_account(AccountId=account_id) #print(yaml.dump(response, default_flow_style=False)) validate_account(org, response['Account']) @@ -172,6 +143,7 @@ def test_describe_account(): response['Account']['Email'].should.equal(mockemail) #assert False + @mock_organizations def test_list_accounts(): client = boto3.client('organizations', region_name='us-east-1') @@ -190,4 +162,3 @@ def test_list_accounts(): accounts[3]['Name'].should.equal(mockname + '3') accounts[2]['Email'].should.equal(mockname + '2' + '@' + mockdomain) #assert False - diff --git a/tests/test_organizations/test_organizations_utils.py b/tests/test_organizations/test_organizations_utils.py index 8144c327f..3e29d5cb0 100644 --- a/tests/test_organizations/test_organizations_utils.py +++ b/tests/test_organizations/test_organizations_utils.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import sure # noqa -import moto from moto.organizations import utils ORG_ID_REGEX = r'o-[a-z0-9]{%s}' % utils.ORG_ID_SIZE @@ -9,18 +8,22 @@ ROOT_ID_REGEX = r'r-[a-z0-9]{%s}' % utils.ROOT_ID_SIZE ACCOUNT_ID_REGEX = r'[0-9]{%s}' % utils.ACCOUNT_ID_SIZE CREATE_ACCOUNT_STATUS_ID_REGEX = r'car-[a-z0-9]{%s}' % utils.CREATE_ACCOUNT_STATUS_ID_SIZE + def test_make_random_org_id(): org_id = utils.make_random_org_id() org_id.should.match(ORG_ID_REGEX) + def test_make_random_root_id(): org_id = utils.make_random_root_id() org_id.should.match(ROOT_ID_REGEX) + def test_make_random_account_id(): account_id = utils.make_random_account_id() account_id.should.match(ACCOUNT_ID_REGEX) + def test_make_random_create_account_status_id(): create_account_status_id = utils.make_random_create_account_status_id() create_account_status_id.should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) From 6c0c6148f15b7a9133a0c9cce190e6870f787d6e Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sun, 15 Jul 2018 10:31:16 -0700 Subject: [PATCH 063/106] organizations: add endpoint list_roots --- moto/organizations/models.py | 70 +++++++++++++++++-- moto/organizations/responses.py | 5 ++ moto/organizations/utils.py | 14 ++++ tests/test_organizations/object_syntax.py | 6 +- .../test_organizations_boto3.py | 24 +++++++ .../test_organizations_utils.py | 11 ++- 6 files changed, 122 insertions(+), 8 deletions(-) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index e1d59c1da..265654030 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -11,7 +11,7 @@ MASTER_ACCOUNT_EMAIL = 'fakeorg@moto-example.com' ORGANIZATION_ARN_FORMAT = 'arn:aws:organizations::{0}:organization/{1}' MASTER_ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{0}' ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{2}' - +ROOT_ARN_FORMAT = 'arn:aws:organizations::{0}:root/{1}/{2}' class FakeOrganization(BaseModel): @@ -33,7 +33,7 @@ class FakeOrganization(BaseModel): def master_account_arn(self): return MASTER_ACCOUNT_ARN_FORMAT.format(self.master_account_id, self.id) - def _describe(self): + def describe(self): return { 'Organization': { 'Id': self.id, @@ -95,18 +95,80 @@ class FakeAccount(BaseModel): } +class FakeOrganizationalUnit(BaseModel): + + def __init__(self, organization, **kwargs): + self.organization_id = organization.id + self.master_account_id = organization.master_account_id + self.id = utils.make_random_ou_id() + self.name = kwargs['Name'] + + @property + def arn(self): + return OU_ARN_FORMAT.format( + self.master_account_id, + self.organization_id, + self.id + ) + + def describe(self): + return { + 'OrganizationalUnit': { + 'Id': self.id, + 'Arn': self.arn, + 'Name': self.name, + } + } + + +class FakeRoot(BaseModel): + + def __init__(self, organization, **kwargs): + self.organization_id = organization.id + self.master_account_id = organization.master_account_id + self.id = utils.make_random_root_id() + self.name = 'Root' + self.policy_types = [{ + 'Type': 'SERVICE_CONTROL_POLICY', + 'Status': 'ENABLED' + }] + + @property + def arn(self): + return ROOT_ARN_FORMAT.format( + self.master_account_id, + self.organization_id, + self.id + ) + + def describe(self): + return { + 'Id': self.id, + 'Arn': self.arn, + 'Name': self.name, + 'PolicyTypes': self.policy_types + } + + class OrganizationsBackend(BaseBackend): def __init__(self): self.org = None self.accounts = [] + self.roots = [] def create_organization(self, **kwargs): self.org = FakeOrganization(kwargs['FeatureSet']) - return self.org._describe() + self.roots.append(FakeRoot(self.org)) + return self.org.describe() def describe_organization(self): - return self.org._describe() + return self.org.describe() + + def list_roots(self): + return dict( + Roots=[root.describe() for root in self.roots] + ) def create_account(self, **kwargs): new_account = FakeAccount(self.org, **kwargs) diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py index 7c8d45014..1804a3fcf 100644 --- a/moto/organizations/responses.py +++ b/moto/organizations/responses.py @@ -31,6 +31,11 @@ class OrganizationsResponse(BaseResponse): self.organizations_backend.describe_organization() ) + def list_roots(self): + return json.dumps( + self.organizations_backend.list_roots() + ) + def create_account(self): return json.dumps( self.organizations_backend.create_account(**self.request_params) diff --git a/moto/organizations/utils.py b/moto/organizations/utils.py index 1b111c1a7..c7e5c71cd 100644 --- a/moto/organizations/utils.py +++ b/moto/organizations/utils.py @@ -7,6 +7,7 @@ CHARSET = string.ascii_lowercase + string.digits ORG_ID_SIZE = 10 ROOT_ID_SIZE = 4 ACCOUNT_ID_SIZE = 12 +OU_ID_SUFFIX_SIZE = 8 CREATE_ACCOUNT_STATUS_ID_SIZE = 8 @@ -24,6 +25,19 @@ def make_random_root_id(): return 'r-' + ''.join(random.choice(CHARSET) for x in range(ROOT_ID_SIZE)) +def make_random_ou_id(root_id): + # The regex pattern for an organizational unit ID string requires "ou-" + # followed by from 4 to 32 lower-case letters or digits (the ID of the root + # that contains the OU) followed by a second "-" dash and from 8 to 32 + # additional lower-case letters or digits. + # e.g. ou-g8sd-5oe3bjaw + return '-'.join([ + 'ou', + root_id.partition('-')[2], + ''.join(random.choice(CHARSET) for x in range(OU_ID_SUFFIX_SIZE)), + ]) + + def make_random_account_id(): # The regex pattern for an account ID string requires exactly 12 digits. # e.g. '488633172133' diff --git a/tests/test_organizations/object_syntax.py b/tests/test_organizations/object_syntax.py index 2779d1d07..3fb86b9d5 100644 --- a/tests/test_organizations/object_syntax.py +++ b/tests/test_organizations/object_syntax.py @@ -9,9 +9,11 @@ from moto import organizations as orgs # utils print(orgs.utils.make_random_org_id()) -print(orgs.utils.make_random_root_id()) +root_id = orgs.utils.make_random_root_id() +print(root_id) +print(orgs.utils.make_random_ou_id(root_id)) print(orgs.utils.make_random_account_id()) -print(orgs.utils.make_random_create_account_id()) +print(orgs.utils.make_random_create_account_status_id()) # models my_org = orgs.models.FakeOrganization(feature_set='ALL') diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index 7ae8d9577..f4dc3b445 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import boto3 import sure # noqa import datetime +import yaml from moto import mock_organizations from moto.organizations.models import ( @@ -11,6 +12,7 @@ from moto.organizations.models import ( ORGANIZATION_ARN_FORMAT, MASTER_ACCOUNT_ARN_FORMAT, ACCOUNT_ARN_FORMAT, + ROOT_ARN_FORMAT, ) from .test_organizations_utils import ( ORG_ID_REGEX, @@ -111,6 +113,28 @@ def test_describe_organization(): #assert False +@mock_organizations +def test_list_roots(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + response = client.list_roots() + #print(yaml.dump(response, default_flow_style=False)) + response.should.have.key('Roots').should.be.a(list) + response['Roots'].should_not.be.empty + root = response['Roots'][0] + root.should.have.key('Id').should.match(ROOT_ID_REGEX) + root.should.have.key('Arn').should.equal(ROOT_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + root['Id'], + )) + root.should.have.key('Name').should.be.a(str) + root.should.have.key('PolicyTypes').should.be.a(list) + root['PolicyTypes'][0].should.have.key('Type').should.equal('SERVICE_CONTROL_POLICY') + root['PolicyTypes'][0].should.have.key('Status').should.equal('ENABLED') + #assert False + + mockname = 'mock-account' mockdomain = 'moto-example.org' mockemail = '@'.join([mockname, mockdomain]) diff --git a/tests/test_organizations/test_organizations_utils.py b/tests/test_organizations/test_organizations_utils.py index 3e29d5cb0..d27201446 100644 --- a/tests/test_organizations/test_organizations_utils.py +++ b/tests/test_organizations/test_organizations_utils.py @@ -5,6 +5,7 @@ from moto.organizations import utils ORG_ID_REGEX = r'o-[a-z0-9]{%s}' % utils.ORG_ID_SIZE ROOT_ID_REGEX = r'r-[a-z0-9]{%s}' % utils.ROOT_ID_SIZE +OU_ID_REGEX = r'ou-[a-z0-9]{%s}-[a-z0-9]{%s}' % (utils.ROOT_ID_SIZE, utils.OU_ID_SUFFIX_SIZE) ACCOUNT_ID_REGEX = r'[0-9]{%s}' % utils.ACCOUNT_ID_SIZE CREATE_ACCOUNT_STATUS_ID_REGEX = r'car-[a-z0-9]{%s}' % utils.CREATE_ACCOUNT_STATUS_ID_SIZE @@ -15,8 +16,14 @@ def test_make_random_org_id(): def test_make_random_root_id(): - org_id = utils.make_random_root_id() - org_id.should.match(ROOT_ID_REGEX) + root_id = utils.make_random_root_id() + root_id.should.match(ROOT_ID_REGEX) + + +def test_make_random_ou_id(): + root_id = utils.make_random_root_id() + ou_id = utils.make_random_ou_id(root_id) + ou_id.should.match(OU_ID_REGEX) def test_make_random_account_id(): From beebb9abc871f826597e12ba29bcdaa2cd78f4af Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sun, 15 Jul 2018 11:49:26 -0700 Subject: [PATCH 064/106] organizations: add 2 more endpoints create_organizational_unit describe_organizational_unit --- IMPLEMENTATION_COVERAGE.md | 8 +-- moto/organizations/models.py | 72 +++++++++++-------- moto/organizations/responses.py | 10 +++ .../test_organizations_boto3.py | 49 +++++++++++++ 4 files changed, 107 insertions(+), 32 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 837850ac6..7b4bc5e4d 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3147,13 +3147,13 @@ - [ ] update_server - [ ] update_server_engine_attributes -## organizations - 8% implemented +## organizations - 19% implemented - [ ] accept_handshake - [ ] attach_policy - [ ] cancel_handshake - [X] create_account - [X] create_organization -- [ ] create_organizational_unit +- [X] create_organizational_unit - [ ] create_policy - [ ] decline_handshake - [ ] delete_organization @@ -3163,7 +3163,7 @@ - [ ] describe_create_account_status - [ ] describe_handshake - [X] describe_organization -- [ ] describe_organizational_unit +- [X] describe_organizational_unit - [ ] describe_policy - [ ] detach_policy - [ ] disable_aws_service_access @@ -3184,7 +3184,7 @@ - [ ] list_parents - [ ] list_policies - [ ] list_policies_for_target -- [ ] list_roots +- [X] list_roots - [ ] list_targets_for_policy - [ ] move_account - [ ] remove_account_from_organization diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 265654030..461072ce3 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -12,6 +12,7 @@ ORGANIZATION_ARN_FORMAT = 'arn:aws:organizations::{0}:organization/{1}' MASTER_ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{0}' ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{2}' ROOT_ARN_FORMAT = 'arn:aws:organizations::{0}:root/{1}/{2}' +OU_ARN_FORMAT = 'arn:aws:organizations::{0}:ou/{1}/{2}' class FakeOrganization(BaseModel): @@ -95,32 +96,6 @@ class FakeAccount(BaseModel): } -class FakeOrganizationalUnit(BaseModel): - - def __init__(self, organization, **kwargs): - self.organization_id = organization.id - self.master_account_id = organization.master_account_id - self.id = utils.make_random_ou_id() - self.name = kwargs['Name'] - - @property - def arn(self): - return OU_ARN_FORMAT.format( - self.master_account_id, - self.organization_id, - self.id - ) - - def describe(self): - return { - 'OrganizationalUnit': { - 'Id': self.id, - 'Arn': self.arn, - 'Name': self.name, - } - } - - class FakeRoot(BaseModel): def __init__(self, organization, **kwargs): @@ -150,12 +125,40 @@ class FakeRoot(BaseModel): } + +class FakeOrganizationalUnit(BaseModel): + + def __init__(self, organization, root_id, **kwargs): + self.organization_id = organization.id + self.master_account_id = organization.master_account_id + self.id = utils.make_random_ou_id(root_id) + self.name = kwargs['Name'] + self.parent_id = kwargs['ParentId'] + + @property + def arn(self): + return OU_ARN_FORMAT.format( + self.master_account_id, + self.organization_id, + self.id + ) + + def describe(self): + return { + 'OrganizationalUnit': { + 'Id': self.id, + 'Arn': self.arn, + 'Name': self.name, + } + } + class OrganizationsBackend(BaseBackend): def __init__(self): self.org = None self.accounts = [] self.roots = [] + self.ou = [] def create_organization(self, **kwargs): self.org = FakeOrganization(kwargs['FeatureSet']) @@ -170,14 +173,27 @@ class OrganizationsBackend(BaseBackend): Roots=[root.describe() for root in self.roots] ) + def create_organizational_unit(self, **kwargs): + new_ou = FakeOrganizationalUnit(self.org, self.roots[0].id, **kwargs) + self.ou.append(new_ou) + return new_ou.describe() + + def describe_organizational_unit(self, **kwargs): + ou = [ + ou for ou in self.ou if ou.id == kwargs['OrganizationalUnitId'] + ].pop(0) + return ou.describe() + def create_account(self, **kwargs): new_account = FakeAccount(self.org, **kwargs) self.accounts.append(new_account) return new_account.create_account_status def describe_account(self, **kwargs): - account = [account for account in self.accounts - if account.account_id == kwargs['AccountId']][0] + account = [ + account for account in self.accounts + if account.account_id == kwargs['AccountId'] + ].pop(0) return account.describe() def list_accounts(self): diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py index 1804a3fcf..4f0643cf6 100644 --- a/moto/organizations/responses.py +++ b/moto/organizations/responses.py @@ -36,6 +36,16 @@ class OrganizationsResponse(BaseResponse): self.organizations_backend.list_roots() ) + def create_organizational_unit(self): + return json.dumps( + self.organizations_backend.create_organizational_unit(**self.request_params) + ) + + def describe_organizational_unit(self): + return json.dumps( + self.organizations_backend.describe_organizational_unit(**self.request_params) + ) + def create_account(self): return json.dumps( self.organizations_backend.create_account(**self.request_params) diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index f4dc3b445..0e398bd43 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -13,10 +13,12 @@ from moto.organizations.models import ( MASTER_ACCOUNT_ARN_FORMAT, ACCOUNT_ARN_FORMAT, ROOT_ARN_FORMAT, + OU_ARN_FORMAT, ) from .test_organizations_utils import ( ORG_ID_REGEX, ROOT_ID_REGEX, + OU_ID_REGEX, ACCOUNT_ID_REGEX, CREATE_ACCOUNT_STATUS_ID_REGEX, ) @@ -53,6 +55,18 @@ def validate_organization(response): }]) +def validate_organizationa_unit(org, response): + response.should.have.key('OrganizationalUnit').should.be.a(dict) + ou = response['OrganizationalUnit'] + ou.should.have.key('Id').should.match(OU_ID_REGEX) + ou.should.have.key('Arn').should.equal(OU_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + ou['Id'], + )) + ou.should.have.key('Name').should.equal(ou_name) + + def validate_account(org, account): sorted(account.keys()).should.equal([ 'Arn', @@ -113,6 +127,9 @@ def test_describe_organization(): #assert False +# Organizational Units +ou_name = 'ou01' + @mock_organizations def test_list_roots(): client = boto3.client('organizations', region_name='us-east-1') @@ -135,6 +152,38 @@ def test_list_roots(): #assert False +@mock_organizations +def test_create_organizational_unit(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + response = client.create_organizational_unit( + ParentId=root_id, + Name=ou_name, + ) + #print(yaml.dump(response, default_flow_style=False)) + validate_organizationa_unit(org, response) + #assert False + + +@mock_organizations +def test_describe_organizational_unit(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + ou_id = client.create_organizational_unit( + ParentId=root_id, + Name=ou_name, + )['OrganizationalUnit']['Id'] + response = client.describe_organizational_unit( + OrganizationalUnitId=ou_id, + ) + print(yaml.dump(response, default_flow_style=False)) + validate_organizationa_unit(org, response) + #assert False + + +# Accounts mockname = 'mock-account' mockdomain = 'moto-example.org' mockemail = '@'.join([mockname, mockdomain]) From fc2447c6a49139ce9fd06bbc48aa23f3fd1bcae7 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sun, 15 Jul 2018 13:58:27 -0700 Subject: [PATCH 065/106] organiziaions: 2 new endpoints: list_organizational_units_for_parents list_parents --- IMPLEMENTATION_COVERAGE.md | 6 +- moto/organizations/models.py | 32 ++++++++++- moto/organizations/responses.py | 10 ++++ .../test_organizations_boto3.py | 57 ++++++++++++++++--- 4 files changed, 92 insertions(+), 13 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 7b4bc5e4d..84c574bc3 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3147,7 +3147,7 @@ - [ ] update_server - [ ] update_server_engine_attributes -## organizations - 19% implemented +## organizations - 20% implemented - [ ] accept_handshake - [ ] attach_policy - [ ] cancel_handshake @@ -3180,8 +3180,8 @@ - [ ] list_create_account_status - [ ] list_handshakes_for_account - [ ] list_handshakes_for_organization -- [ ] list_organizational_units_for_parent -- [ ] list_parents +- [X] list_organizational_units_for_parent +- [X] list_parents - [ ] list_policies - [ ] list_policies_for_target - [X] list_roots diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 461072ce3..b46c62798 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -14,6 +14,7 @@ ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{2}' ROOT_ARN_FORMAT = 'arn:aws:organizations::{0}:root/{1}/{2}' OU_ARN_FORMAT = 'arn:aws:organizations::{0}:ou/{1}/{2}' + class FakeOrganization(BaseModel): def __init__(self, feature_set): @@ -125,7 +126,6 @@ class FakeRoot(BaseModel): } - class FakeOrganizationalUnit(BaseModel): def __init__(self, organization, root_id, **kwargs): @@ -152,6 +152,7 @@ class FakeOrganizationalUnit(BaseModel): } } + class OrganizationsBackend(BaseBackend): def __init__(self): @@ -184,6 +185,35 @@ class OrganizationsBackend(BaseBackend): ].pop(0) return ou.describe() + def list_organizational_units_for_parent(self, **kwargs): + return dict( + OrganizationalUnits=[ + { + 'Id': ou.id, + 'Arn': ou.arn, + 'Name': ou.name, + } + for ou in self.ou + if ou.parent_id == kwargs['ParentId'] + ] + ) + + def list_parents(self, **kwargs): + parent_id = [ + ou.parent_id for ou in self.ou if ou.id == kwargs['ChildId'] + ].pop(0) + root_parents = [ + dict(Id=root.id, Type='ROOT') + for root in self.roots + if root.id == parent_id + ] + ou_parents = [ + dict(Id=ou.id, Type='ORGANIZATIONAL_UNIT') + for ou in self.ou + if ou.id == parent_id + ] + return dict(Parents=root_parents + ou_parents) + def create_account(self, **kwargs): new_account = FakeAccount(self.org, **kwargs) self.accounts.append(new_account) diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py index 4f0643cf6..9fdb73317 100644 --- a/moto/organizations/responses.py +++ b/moto/organizations/responses.py @@ -46,6 +46,16 @@ class OrganizationsResponse(BaseResponse): self.organizations_backend.describe_organizational_unit(**self.request_params) ) + def list_organizational_units_for_parent(self): + return json.dumps( + self.organizations_backend.list_organizational_units_for_parent(**self.request_params) + ) + + def list_parents(self): + return json.dumps( + self.organizations_backend.list_parents(**self.request_params) + ) + def create_account(self): return json.dumps( self.organizations_backend.create_account(**self.request_params) diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index 0e398bd43..8375ffbe3 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -55,7 +55,7 @@ def validate_organization(response): }]) -def validate_organizationa_unit(org, response): +def validate_organizational_unit(org, response): response.should.have.key('OrganizationalUnit').should.be.a(dict) ou = response['OrganizationalUnit'] ou.should.have.key('Id').should.match(OU_ID_REGEX) @@ -64,7 +64,7 @@ def validate_organizationa_unit(org, response): org['Id'], ou['Id'], )) - ou.should.have.key('Name').should.equal(ou_name) + ou.should.have.key('Name').should.be.a(str) def validate_account(org, account): @@ -128,7 +128,6 @@ def test_describe_organization(): # Organizational Units -ou_name = 'ou01' @mock_organizations def test_list_roots(): @@ -157,12 +156,14 @@ def test_create_organizational_unit(): client = boto3.client('organizations', region_name='us-east-1') org = client.create_organization(FeatureSet='ALL')['Organization'] root_id = client.list_roots()['Roots'][0]['Id'] + ou_name = 'ou01' response = client.create_organizational_unit( ParentId=root_id, Name=ou_name, ) #print(yaml.dump(response, default_flow_style=False)) - validate_organizationa_unit(org, response) + validate_organizational_unit(org, response) + response['OrganizationalUnit']['Name'].should.equal(ou_name) #assert False @@ -173,13 +174,51 @@ def test_describe_organizational_unit(): root_id = client.list_roots()['Roots'][0]['Id'] ou_id = client.create_organizational_unit( ParentId=root_id, - Name=ou_name, + Name='ou01', )['OrganizationalUnit']['Id'] - response = client.describe_organizational_unit( - OrganizationalUnitId=ou_id, - ) + response = client.describe_organizational_unit(OrganizationalUnitId=ou_id) print(yaml.dump(response, default_flow_style=False)) - validate_organizationa_unit(org, response) + validate_organizational_unit(org, response) + #assert False + + +@mock_organizations +def test_list_organizational_units_for_parent(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + client.create_organizational_unit(ParentId=root_id, Name='ou01') + client.create_organizational_unit(ParentId=root_id, Name='ou02') + client.create_organizational_unit(ParentId=root_id, Name='ou03') + response = client.list_organizational_units_for_parent(ParentId=root_id) + print(yaml.dump(response, default_flow_style=False)) + response.should.have.key('OrganizationalUnits').should.be.a(list) + for ou in response['OrganizationalUnits']: + validate_organizational_unit(org, dict(OrganizationalUnit=ou)) + #assert False + + +@mock_organizations +def test_list_parents(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + + ou01 = client.create_organizational_unit(ParentId=root_id, Name='ou01') + ou01_id = ou01['OrganizationalUnit']['Id'] + response01 = client.list_parents(ChildId=ou01_id) + #print(yaml.dump(response01, default_flow_style=False)) + response01.should.have.key('Parents').should.be.a(list) + response01['Parents'][0].should.have.key('Id').should.equal(root_id) + response01['Parents'][0].should.have.key('Type').should.equal('ROOT') + + ou02 = client.create_organizational_unit(ParentId=ou01_id, Name='ou02') + ou02_id = ou02['OrganizationalUnit']['Id'] + response02 = client.list_parents(ChildId=ou02_id) + #print(yaml.dump(response02, default_flow_style=False)) + response02.should.have.key('Parents').should.be.a(list) + response02['Parents'][0].should.have.key('Id').should.equal(ou01_id) + response02['Parents'][0].should.have.key('Type').should.equal('ORGANIZATIONAL_UNIT') #assert False From 009dcdb21a10399f7b4fb6f9465267de0e7324bb Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sun, 15 Jul 2018 15:25:34 -0700 Subject: [PATCH 066/106] organizations: and another 2 endpoints: list_accounts_for_parent move_account --- IMPLEMENTATION_COVERAGE.md | 6 ++-- moto/organizations/models.py | 25 +++++++++++-- moto/organizations/responses.py | 10 ++++++ .../test_organizations_boto3.py | 36 +++++++++++++++++++ 4 files changed, 72 insertions(+), 5 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 84c574bc3..e1943aa9a 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3147,7 +3147,7 @@ - [ ] update_server - [ ] update_server_engine_attributes -## organizations - 20% implemented +## organizations - 28% implemented - [ ] accept_handshake - [ ] attach_policy - [ ] cancel_handshake @@ -3174,7 +3174,7 @@ - [ ] invite_account_to_organization - [ ] leave_organization - [X] list_accounts -- [ ] list_accounts_for_parent +- [X] list_accounts_for_parent - [ ] list_aws_service_access_for_organization - [ ] list_children - [ ] list_create_account_status @@ -3186,7 +3186,7 @@ - [ ] list_policies_for_target - [X] list_roots - [ ] list_targets_for_policy -- [ ] move_account +- [X] move_account - [ ] remove_account_from_organization - [ ] update_organizational_unit - [ ] update_policy diff --git a/moto/organizations/models.py b/moto/organizations/models.py index b46c62798..bbcc1479b 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -51,7 +51,7 @@ class FakeOrganization(BaseModel): class FakeAccount(BaseModel): - def __init__(self, organization, **kwargs): + def __init__(self, organization, root_id, **kwargs): self.organization_id = organization.id self.master_account_id = organization.master_account_id self.create_account_status_id = utils.make_random_create_account_status_id() @@ -61,6 +61,7 @@ class FakeAccount(BaseModel): self.create_time = datetime.datetime.utcnow() self.status = 'ACTIVE' self.joined_method = 'CREATED' + self.parent_id = root_id @property def arn(self): @@ -215,7 +216,7 @@ class OrganizationsBackend(BaseBackend): return dict(Parents=root_parents + ou_parents) def create_account(self, **kwargs): - new_account = FakeAccount(self.org, **kwargs) + new_account = FakeAccount(self.org, self.roots[0].id, **kwargs) self.accounts.append(new_account) return new_account.create_account_status @@ -231,5 +232,25 @@ class OrganizationsBackend(BaseBackend): Accounts=[account.describe()['Account'] for account in self.accounts] ) + def list_accounts_for_parent(self, **kwargs): + return dict( + Accounts=[ + account.describe()['Account'] + for account in self.accounts + if account.parent_id == kwargs['ParentId'] + ] + ) + + def move_account(self, **kwargs): + new_parent_id = kwargs['DestinationParentId'] + all_parent_id = [parent.id for parent in self.roots + self.ou] + account = [ + account for account in self.accounts if account.account_id == kwargs['AccountId'] + ].pop(0) + assert new_parent_id in all_parent_id + assert account.parent_id == kwargs['SourceParentId'] + index = self.accounts.index(account) + self.accounts[index].parent_id = new_parent_id + organizations_backend = OrganizationsBackend() diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py index 9fdb73317..6684ae685 100644 --- a/moto/organizations/responses.py +++ b/moto/organizations/responses.py @@ -70,3 +70,13 @@ class OrganizationsResponse(BaseResponse): return json.dumps( self.organizations_backend.list_accounts() ) + + def list_accounts_for_parent(self): + return json.dumps( + self.organizations_backend.list_accounts_for_parent(**self.request_params) + ) + + def move_account(self): + return json.dumps( + self.organizations_backend.move_account(**self.request_params) + ) diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index 8375ffbe3..5355e2716 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -274,3 +274,39 @@ def test_list_accounts(): accounts[3]['Name'].should.equal(mockname + '3') accounts[2]['Email'].should.equal(mockname + '2' + '@' + mockdomain) #assert False + + +@mock_organizations +def test_list_accounts_for_parent(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + account_id = client.create_account( + AccountName=mockname, + Email=mockemail, + )['CreateAccountStatus']['AccountId'] + response = client.list_accounts_for_parent(ParentId=root_id) + #print(yaml.dump(response, default_flow_style=False)) + account_id.should.be.within([account['Id'] for account in response['Accounts']]) + #assert False + + +@mock_organizations +def test_move_account(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + account_id = client.create_account( + AccountName=mockname, Email=mockemail + )['CreateAccountStatus']['AccountId'] + ou01 = client.create_organizational_unit(ParentId=root_id, Name='ou01') + ou01_id = ou01['OrganizationalUnit']['Id'] + client.move_account( + AccountId=account_id, + SourceParentId=root_id, + DestinationParentId=ou01_id, + ) + response = client.list_accounts_for_parent(ParentId=ou01_id) + #print(yaml.dump(response, default_flow_style=False)) + account_id.should.be.within([account['Id'] for account in response['Accounts']]) + #assert False From 9b5c6c4f0f04392ab40df7404020fd70340e7347 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sun, 15 Jul 2018 18:48:56 -0700 Subject: [PATCH 067/106] organizations.model.FakeAccount: rename attributes: account_id -> id account_name -> name --- moto/organizations/models.py | 71 ++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index bbcc1479b..8a00918d1 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -55,8 +55,8 @@ class FakeAccount(BaseModel): self.organization_id = organization.id self.master_account_id = organization.master_account_id self.create_account_status_id = utils.make_random_create_account_status_id() - self.account_id = utils.make_random_account_id() - self.account_name = kwargs['AccountName'] + self.id = utils.make_random_account_id() + self.name = kwargs['AccountName'] self.email = kwargs['Email'] self.create_time = datetime.datetime.utcnow() self.status = 'ACTIVE' @@ -68,7 +68,7 @@ class FakeAccount(BaseModel): return ACCOUNT_ARN_FORMAT.format( self.master_account_id, self.organization_id, - self.account_id + self.id ) @property @@ -76,21 +76,21 @@ class FakeAccount(BaseModel): return { 'CreateAccountStatus': { 'Id': self.create_account_status_id, - 'AccountName': self.account_name, + 'AccountName': self.name, 'State': 'SUCCEEDED', 'RequestedTimestamp': unix_time(self.create_time), 'CompletedTimestamp': unix_time(self.create_time), - 'AccountId': self.account_id, + 'AccountId': self.id, } } def describe(self): return { 'Account': { - 'Id': self.account_id, + 'Id': self.id, 'Arn': self.arn, 'Email': self.email, - 'Name': self.account_name, + 'Name': self.name, 'Status': self.status, 'JoinedMethod': self.joined_method, 'JoinedTimestamp': unix_time(self.create_time), @@ -98,6 +98,33 @@ class FakeAccount(BaseModel): } +class FakeOrganizationalUnit(BaseModel): + + def __init__(self, organization, root_id, **kwargs): + self.organization_id = organization.id + self.master_account_id = organization.master_account_id + self.id = utils.make_random_ou_id(root_id) + self.name = kwargs['Name'] + self.parent_id = kwargs['ParentId'] + + @property + def arn(self): + return OU_ARN_FORMAT.format( + self.master_account_id, + self.organization_id, + self.id + ) + + def describe(self): + return { + 'OrganizationalUnit': { + 'Id': self.id, + 'Arn': self.arn, + 'Name': self.name, + } + } + + class FakeRoot(BaseModel): def __init__(self, organization, **kwargs): @@ -127,32 +154,6 @@ class FakeRoot(BaseModel): } -class FakeOrganizationalUnit(BaseModel): - - def __init__(self, organization, root_id, **kwargs): - self.organization_id = organization.id - self.master_account_id = organization.master_account_id - self.id = utils.make_random_ou_id(root_id) - self.name = kwargs['Name'] - self.parent_id = kwargs['ParentId'] - - @property - def arn(self): - return OU_ARN_FORMAT.format( - self.master_account_id, - self.organization_id, - self.id - ) - - def describe(self): - return { - 'OrganizationalUnit': { - 'Id': self.id, - 'Arn': self.arn, - 'Name': self.name, - } - } - class OrganizationsBackend(BaseBackend): @@ -223,7 +224,7 @@ class OrganizationsBackend(BaseBackend): def describe_account(self, **kwargs): account = [ account for account in self.accounts - if account.account_id == kwargs['AccountId'] + if account.id == kwargs['AccountId'] ].pop(0) return account.describe() @@ -245,7 +246,7 @@ class OrganizationsBackend(BaseBackend): new_parent_id = kwargs['DestinationParentId'] all_parent_id = [parent.id for parent in self.roots + self.ou] account = [ - account for account in self.accounts if account.account_id == kwargs['AccountId'] + account for account in self.accounts if account.id == kwargs['AccountId'] ].pop(0) assert new_parent_id in all_parent_id assert account.parent_id == kwargs['SourceParentId'] From 30a9aa33e55e35070eb2468c0d377eb79e7a14e3 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sun, 15 Jul 2018 20:39:13 -0700 Subject: [PATCH 068/106] organizations: endpoint list_parents now support account_id param refactered classes: FakeRoot inherits from FakeOrganizationsUnit add root_id attribute to class FakeOrganization dropped 'roots' attribute from class OrganizationaBackend --- moto/organizations/models.py | 84 ++++++++++--------- .../test_organizations_boto3.py | 79 +++++++++++------ 2 files changed, 98 insertions(+), 65 deletions(-) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 8a00918d1..609ce3831 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import datetime +import re from moto.core import BaseBackend, BaseModel from moto.core.utils import unix_time @@ -19,6 +20,7 @@ class FakeOrganization(BaseModel): def __init__(self, feature_set): self.id = utils.make_random_org_id() + self.root_id = utils.make_random_root_id() self.feature_set = feature_set self.master_account_id = MASTER_ACCOUNT_ID self.master_account_email = MASTER_ACCOUNT_EMAIL @@ -51,7 +53,7 @@ class FakeOrganization(BaseModel): class FakeAccount(BaseModel): - def __init__(self, organization, root_id, **kwargs): + def __init__(self, organization, **kwargs): self.organization_id = organization.id self.master_account_id = organization.master_account_id self.create_account_status_id = utils.make_random_create_account_status_id() @@ -61,7 +63,7 @@ class FakeAccount(BaseModel): self.create_time = datetime.datetime.utcnow() self.status = 'ACTIVE' self.joined_method = 'CREATED' - self.parent_id = root_id + self.parent_id = organization.root_id @property def arn(self): @@ -100,16 +102,18 @@ class FakeAccount(BaseModel): class FakeOrganizationalUnit(BaseModel): - def __init__(self, organization, root_id, **kwargs): + def __init__(self, organization, **kwargs): + self.type = 'ORGANIZATIONAL_UNIT' self.organization_id = organization.id self.master_account_id = organization.master_account_id - self.id = utils.make_random_ou_id(root_id) - self.name = kwargs['Name'] - self.parent_id = kwargs['ParentId'] + self.id = utils.make_random_ou_id(organization.root_id) + self.name = kwargs.get('Name') + self.parent_id = kwargs.get('ParentId') + self._arn_format = OU_ARN_FORMAT @property def arn(self): - return OU_ARN_FORMAT.format( + return self._arn_format.format( self.master_account_id, self.organization_id, self.id @@ -125,25 +129,18 @@ class FakeOrganizationalUnit(BaseModel): } -class FakeRoot(BaseModel): +class FakeRoot(FakeOrganizationalUnit): def __init__(self, organization, **kwargs): - self.organization_id = organization.id - self.master_account_id = organization.master_account_id - self.id = utils.make_random_root_id() + super().__init__(organization, **kwargs) + self.type = 'ROOT' + self.id = organization.root_id self.name = 'Root' self.policy_types = [{ 'Type': 'SERVICE_CONTROL_POLICY', 'Status': 'ENABLED' }] - - @property - def arn(self): - return ROOT_ARN_FORMAT.format( - self.master_account_id, - self.organization_id, - self.id - ) + self._arn_format = ROOT_ARN_FORMAT def describe(self): return { @@ -154,18 +151,16 @@ class FakeRoot(BaseModel): } - class OrganizationsBackend(BaseBackend): def __init__(self): self.org = None self.accounts = [] - self.roots = [] self.ou = [] def create_organization(self, **kwargs): self.org = FakeOrganization(kwargs['FeatureSet']) - self.roots.append(FakeRoot(self.org)) + self.ou.append(FakeRoot(self.org)) return self.org.describe() def describe_organization(self): @@ -173,11 +168,11 @@ class OrganizationsBackend(BaseBackend): def list_roots(self): return dict( - Roots=[root.describe() for root in self.roots] + Roots=[ou.describe() for ou in self.ou if isinstance(ou, FakeRoot)] ) def create_organizational_unit(self, **kwargs): - new_ou = FakeOrganizationalUnit(self.org, self.roots[0].id, **kwargs) + new_ou = FakeOrganizationalUnit(self.org, **kwargs) self.ou.append(new_ou) return new_ou.describe() @@ -201,23 +196,29 @@ class OrganizationsBackend(BaseBackend): ) def list_parents(self, **kwargs): - parent_id = [ - ou.parent_id for ou in self.ou if ou.id == kwargs['ChildId'] - ].pop(0) - root_parents = [ - dict(Id=root.id, Type='ROOT') - for root in self.roots - if root.id == parent_id - ] - ou_parents = [ - dict(Id=ou.id, Type='ORGANIZATIONAL_UNIT') - for ou in self.ou - if ou.id == parent_id - ] - return dict(Parents=root_parents + ou_parents) + if re.compile(r'[0-9]{12}').match(kwargs['ChildId']): + parent_id = [ + account.parent_id for account in self.accounts + if account.id == kwargs['ChildId'] + ].pop(0) + else: + parent_id = [ + ou.parent_id for ou in self.ou + if ou.id == kwargs['ChildId'] + ].pop(0) + return dict( + Parents=[ + { + 'Id': ou.id, + 'Type': ou.type, + } + for ou in self.ou + if ou.id == parent_id + ] + ) def create_account(self, **kwargs): - new_account = FakeAccount(self.org, self.roots[0].id, **kwargs) + new_account = FakeAccount(self.org, **kwargs) self.accounts.append(new_account) return new_account.create_account_status @@ -244,9 +245,10 @@ class OrganizationsBackend(BaseBackend): def move_account(self, **kwargs): new_parent_id = kwargs['DestinationParentId'] - all_parent_id = [parent.id for parent in self.roots + self.ou] + all_parent_id = [parent.id for parent in self.ou] account = [ - account for account in self.accounts if account.id == kwargs['AccountId'] + account for account in self.accounts + if account.id == kwargs['AccountId'] ].pop(0) assert new_parent_id in all_parent_id assert account.parent_id == kwargs['SourceParentId'] diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index 5355e2716..2b692489f 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -198,30 +198,6 @@ def test_list_organizational_units_for_parent(): #assert False -@mock_organizations -def test_list_parents(): - client = boto3.client('organizations', region_name='us-east-1') - org = client.create_organization(FeatureSet='ALL')['Organization'] - root_id = client.list_roots()['Roots'][0]['Id'] - - ou01 = client.create_organizational_unit(ParentId=root_id, Name='ou01') - ou01_id = ou01['OrganizationalUnit']['Id'] - response01 = client.list_parents(ChildId=ou01_id) - #print(yaml.dump(response01, default_flow_style=False)) - response01.should.have.key('Parents').should.be.a(list) - response01['Parents'][0].should.have.key('Id').should.equal(root_id) - response01['Parents'][0].should.have.key('Type').should.equal('ROOT') - - ou02 = client.create_organizational_unit(ParentId=ou01_id, Name='ou02') - ou02_id = ou02['OrganizationalUnit']['Id'] - response02 = client.list_parents(ChildId=ou02_id) - #print(yaml.dump(response02, default_flow_style=False)) - response02.should.have.key('Parents').should.be.a(list) - response02['Parents'][0].should.have.key('Id').should.equal(ou01_id) - response02['Parents'][0].should.have.key('Type').should.equal('ORGANIZATIONAL_UNIT') - #assert False - - # Accounts mockname = 'mock-account' mockdomain = 'moto-example.org' @@ -310,3 +286,58 @@ def test_move_account(): #print(yaml.dump(response, default_flow_style=False)) account_id.should.be.within([account['Id'] for account in response['Accounts']]) #assert False + + +@mock_organizations +def test_list_parents_for_ou(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + ou01 = client.create_organizational_unit(ParentId=root_id, Name='ou01') + ou01_id = ou01['OrganizationalUnit']['Id'] + response01 = client.list_parents(ChildId=ou01_id) + #print(yaml.dump(response01, default_flow_style=False)) + response01.should.have.key('Parents').should.be.a(list) + response01['Parents'][0].should.have.key('Id').should.equal(root_id) + response01['Parents'][0].should.have.key('Type').should.equal('ROOT') + ou02 = client.create_organizational_unit(ParentId=ou01_id, Name='ou02') + ou02_id = ou02['OrganizationalUnit']['Id'] + response02 = client.list_parents(ChildId=ou02_id) + #print(yaml.dump(response02, default_flow_style=False)) + response02.should.have.key('Parents').should.be.a(list) + response02['Parents'][0].should.have.key('Id').should.equal(ou01_id) + response02['Parents'][0].should.have.key('Type').should.equal('ORGANIZATIONAL_UNIT') + #assert False + + +@mock_organizations +def test_list_parents_for_accounts(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + ou01 = client.create_organizational_unit(ParentId=root_id, Name='ou01') + ou01_id = ou01['OrganizationalUnit']['Id'] + account01_id = client.create_account( + AccountName='account01', + Email='account01@moto-example.org' + )['CreateAccountStatus']['AccountId'] + account02_id = client.create_account( + AccountName='account02', + Email='account02@moto-example.org' + )['CreateAccountStatus']['AccountId'] + client.move_account( + AccountId=account02_id, + SourceParentId=root_id, + DestinationParentId=ou01_id, + ) + response01 = client.list_parents(ChildId=account01_id) + #print(yaml.dump(response01, default_flow_style=False)) + response01.should.have.key('Parents').should.be.a(list) + response01['Parents'][0].should.have.key('Id').should.equal(root_id) + response01['Parents'][0].should.have.key('Type').should.equal('ROOT') + response02 = client.list_parents(ChildId=account02_id) + #print(yaml.dump(response02, default_flow_style=False)) + response02.should.have.key('Parents').should.be.a(list) + response02['Parents'][0].should.have.key('Id').should.equal(ou01_id) + response02['Parents'][0].should.have.key('Type').should.equal('ORGANIZATIONAL_UNIT') + #assert False From 8f400b7110b7804ab7445c5aab3e909376833e79 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sun, 15 Jul 2018 22:19:42 -0700 Subject: [PATCH 069/106] organizations: add endpoint list_chilren --- moto/organizations/models.py | 60 ++++++++++++------- moto/organizations/responses.py | 5 ++ .../test_organizations_boto3.py | 41 +++++++++++++ 3 files changed, 84 insertions(+), 22 deletions(-) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 609ce3831..2e0d6b40d 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -195,28 +195,6 @@ class OrganizationsBackend(BaseBackend): ] ) - def list_parents(self, **kwargs): - if re.compile(r'[0-9]{12}').match(kwargs['ChildId']): - parent_id = [ - account.parent_id for account in self.accounts - if account.id == kwargs['ChildId'] - ].pop(0) - else: - parent_id = [ - ou.parent_id for ou in self.ou - if ou.id == kwargs['ChildId'] - ].pop(0) - return dict( - Parents=[ - { - 'Id': ou.id, - 'Type': ou.type, - } - for ou in self.ou - if ou.id == parent_id - ] - ) - def create_account(self, **kwargs): new_account = FakeAccount(self.org, **kwargs) self.accounts.append(new_account) @@ -255,5 +233,43 @@ class OrganizationsBackend(BaseBackend): index = self.accounts.index(account) self.accounts[index].parent_id = new_parent_id + def list_parents(self, **kwargs): + if re.compile(r'[0-9]{12}').match(kwargs['ChildId']): + obj_list = self.accounts + else: + obj_list = self.ou + parent_id = [ + obj.parent_id for obj in obj_list + if obj.id == kwargs['ChildId'] + ].pop(0) + return dict( + Parents=[ + { + 'Id': ou.id, + 'Type': ou.type, + } + for ou in self.ou + if ou.id == parent_id + ] + ) + + def list_children(self, **kwargs): + if kwargs['ChildType'] == 'ACCOUNT': + obj_list = self.accounts + elif kwargs['ChildType'] == 'ORGANIZATIONAL_UNIT': + obj_list = self.ou + else: + raise ValueError + return dict( + Children=[ + { + 'Id': obj.id, + 'Type': kwargs['ChildType'], + } + for obj in obj_list + if obj.parent_id == kwargs['ParentId'] + ] + ) + organizations_backend = OrganizationsBackend() diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py index 6684ae685..966c3fbf3 100644 --- a/moto/organizations/responses.py +++ b/moto/organizations/responses.py @@ -80,3 +80,8 @@ class OrganizationsResponse(BaseResponse): return json.dumps( self.organizations_backend.move_account(**self.request_params) ) + + def list_children(self): + return json.dumps( + self.organizations_backend.list_children(**self.request_params) + ) diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index 2b692489f..496bcd74d 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -341,3 +341,44 @@ def test_list_parents_for_accounts(): response02['Parents'][0].should.have.key('Id').should.equal(ou01_id) response02['Parents'][0].should.have.key('Type').should.equal('ORGANIZATIONAL_UNIT') #assert False + + +@mock_organizations +def test_list_chidlren(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + ou01 = client.create_organizational_unit(ParentId=root_id, Name='ou01') + ou01_id = ou01['OrganizationalUnit']['Id'] + ou02 = client.create_organizational_unit(ParentId=ou01_id, Name='ou02') + ou02_id = ou02['OrganizationalUnit']['Id'] + account01_id = client.create_account( + AccountName='account01', + Email='account01@moto-example.org' + )['CreateAccountStatus']['AccountId'] + account02_id = client.create_account( + AccountName='account02', + Email='account02@moto-example.org' + )['CreateAccountStatus']['AccountId'] + client.move_account( + AccountId=account02_id, + SourceParentId=root_id, + DestinationParentId=ou01_id, + ) + response01 = client.list_children(ParentId=root_id, ChildType='ACCOUNT') + response02 = client.list_children(ParentId=root_id, ChildType='ORGANIZATIONAL_UNIT') + response03 = client.list_children(ParentId=ou01_id, ChildType='ACCOUNT') + response04 = client.list_children(ParentId=ou01_id, ChildType='ORGANIZATIONAL_UNIT') + #print(yaml.dump(response01, default_flow_style=False)) + #print(yaml.dump(response02, default_flow_style=False)) + #print(yaml.dump(response03, default_flow_style=False)) + #print(yaml.dump(response04, default_flow_style=False)) + response01['Children'][0]['Id'].should.equal(account01_id) + response01['Children'][0]['Type'].should.equal('ACCOUNT') + response02['Children'][0]['Id'].should.equal(ou01_id) + response02['Children'][0]['Type'].should.equal('ORGANIZATIONAL_UNIT') + response03['Children'][0]['Id'].should.equal(account02_id) + response03['Children'][0]['Type'].should.equal('ACCOUNT') + response04['Children'][0]['Id'].should.equal(ou02_id) + response04['Children'][0]['Type'].should.equal('ORGANIZATIONAL_UNIT') + #assert False From 01912bdca7e2603035c4905c0274f835045d2a2a Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Mon, 16 Jul 2018 07:23:06 -0700 Subject: [PATCH 070/106] organizations: fix python 2.7 test errors --- moto/organizations/models.py | 2 +- tests/test_organizations/test_organizations_boto3.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 2e0d6b40d..a165395f7 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -132,7 +132,7 @@ class FakeOrganizationalUnit(BaseModel): class FakeRoot(FakeOrganizationalUnit): def __init__(self, organization, **kwargs): - super().__init__(organization, **kwargs) + super(FakeRoot, self).__init__(organization, **kwargs) self.type = 'ROOT' self.id = organization.root_id self.name = 'Root' diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index 496bcd74d..0ac2b61bf 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -4,6 +4,7 @@ import boto3 import sure # noqa import datetime import yaml +import six from moto import mock_organizations from moto.organizations.models import ( @@ -64,7 +65,7 @@ def validate_organizational_unit(org, response): org['Id'], ou['Id'], )) - ou.should.have.key('Name').should.be.a(str) + ou.should.have.key('Name').should.be.a(six.string_types) def validate_account(org, account): @@ -86,7 +87,7 @@ def validate_account(org, account): account['Email'].should.match(EMAIL_REGEX) account['JoinedMethod'].should.be.within(['INVITED', 'CREATED']) account['Status'].should.be.within(['ACTIVE', 'SUSPENDED']) - account['Name'].should.be.a(str) + account['Name'].should.be.a(six.string_types) account['JoinedTimestamp'].should.be.a(datetime.datetime) @@ -101,7 +102,7 @@ def validate_create_account_status(create_status): ]) create_status['Id'].should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) create_status['AccountId'].should.match(ACCOUNT_ID_REGEX) - create_status['AccountName'].should.be.a(str) + create_status['AccountName'].should.be.a(six.string_types) create_status['State'].should.equal('SUCCEEDED') create_status['RequestedTimestamp'].should.be.a(datetime.datetime) create_status['CompletedTimestamp'].should.be.a(datetime.datetime) @@ -144,7 +145,7 @@ def test_list_roots(): org['Id'], root['Id'], )) - root.should.have.key('Name').should.be.a(str) + root.should.have.key('Name').should.be.a(six.string_types) root.should.have.key('PolicyTypes').should.be.a(list) root['PolicyTypes'][0].should.have.key('Type').should.equal('SERVICE_CONTROL_POLICY') root['PolicyTypes'][0].should.have.key('Status').should.equal('ENABLED') From 40e422b74d56e545f4a1a903213a56bebff5c53f Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Thu, 19 Jul 2018 11:50:24 -0700 Subject: [PATCH 071/106] [issue #1720] Add support for AWS Organizations ready for pull request did a little cleanup refactoring local tests pass --- moto/organizations/models.py | 22 +-- moto/organizations/utils.py | 8 + tests/test_organizations/object_syntax.py | 25 --- .../organizations_test_utils.py | 136 ++++++++++++++++ .../test_organizations_boto3.py | 153 +----------------- .../test_organizations_utils.py | 36 ----- 6 files changed, 158 insertions(+), 222 deletions(-) delete mode 100644 tests/test_organizations/object_syntax.py create mode 100644 tests/test_organizations/organizations_test_utils.py delete mode 100644 tests/test_organizations/test_organizations_utils.py diff --git a/moto/organizations/models.py b/moto/organizations/models.py index a165395f7..0f3c67400 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -7,14 +7,6 @@ from moto.core import BaseBackend, BaseModel from moto.core.utils import unix_time from moto.organizations import utils -MASTER_ACCOUNT_ID = '123456789012' -MASTER_ACCOUNT_EMAIL = 'fakeorg@moto-example.com' -ORGANIZATION_ARN_FORMAT = 'arn:aws:organizations::{0}:organization/{1}' -MASTER_ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{0}' -ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{2}' -ROOT_ARN_FORMAT = 'arn:aws:organizations::{0}:root/{1}/{2}' -OU_ARN_FORMAT = 'arn:aws:organizations::{0}:ou/{1}/{2}' - class FakeOrganization(BaseModel): @@ -22,8 +14,8 @@ class FakeOrganization(BaseModel): self.id = utils.make_random_org_id() self.root_id = utils.make_random_root_id() self.feature_set = feature_set - self.master_account_id = MASTER_ACCOUNT_ID - self.master_account_email = MASTER_ACCOUNT_EMAIL + self.master_account_id = utils.MASTER_ACCOUNT_ID + self.master_account_email = utils.MASTER_ACCOUNT_EMAIL self.available_policy_types = [{ 'Type': 'SERVICE_CONTROL_POLICY', 'Status': 'ENABLED' @@ -31,11 +23,11 @@ class FakeOrganization(BaseModel): @property def arn(self): - return ORGANIZATION_ARN_FORMAT.format(self.master_account_id, self.id) + return utils.ORGANIZATION_ARN_FORMAT.format(self.master_account_id, self.id) @property def master_account_arn(self): - return MASTER_ACCOUNT_ARN_FORMAT.format(self.master_account_id, self.id) + return utils.MASTER_ACCOUNT_ARN_FORMAT.format(self.master_account_id, self.id) def describe(self): return { @@ -67,7 +59,7 @@ class FakeAccount(BaseModel): @property def arn(self): - return ACCOUNT_ARN_FORMAT.format( + return utils.ACCOUNT_ARN_FORMAT.format( self.master_account_id, self.organization_id, self.id @@ -109,7 +101,7 @@ class FakeOrganizationalUnit(BaseModel): self.id = utils.make_random_ou_id(organization.root_id) self.name = kwargs.get('Name') self.parent_id = kwargs.get('ParentId') - self._arn_format = OU_ARN_FORMAT + self._arn_format = utils.OU_ARN_FORMAT @property def arn(self): @@ -140,7 +132,7 @@ class FakeRoot(FakeOrganizationalUnit): 'Type': 'SERVICE_CONTROL_POLICY', 'Status': 'ENABLED' }] - self._arn_format = ROOT_ARN_FORMAT + self._arn_format = utils.ROOT_ARN_FORMAT def describe(self): return { diff --git a/moto/organizations/utils.py b/moto/organizations/utils.py index c7e5c71cd..007afa6ed 100644 --- a/moto/organizations/utils.py +++ b/moto/organizations/utils.py @@ -3,6 +3,14 @@ from __future__ import unicode_literals import random import string +MASTER_ACCOUNT_ID = '123456789012' +MASTER_ACCOUNT_EMAIL = 'fakeorg@moto-example.com' +ORGANIZATION_ARN_FORMAT = 'arn:aws:organizations::{0}:organization/{1}' +MASTER_ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{0}' +ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{2}' +ROOT_ARN_FORMAT = 'arn:aws:organizations::{0}:root/{1}/{2}' +OU_ARN_FORMAT = 'arn:aws:organizations::{0}:ou/{1}/{2}' + CHARSET = string.ascii_lowercase + string.digits ORG_ID_SIZE = 10 ROOT_ID_SIZE = 4 diff --git a/tests/test_organizations/object_syntax.py b/tests/test_organizations/object_syntax.py deleted file mode 100644 index 3fb86b9d5..000000000 --- a/tests/test_organizations/object_syntax.py +++ /dev/null @@ -1,25 +0,0 @@ -""" -Temporary functions for checking object structures while specing out -models. This module will go away. -""" - -import yaml -from moto import organizations as orgs - - -# utils -print(orgs.utils.make_random_org_id()) -root_id = orgs.utils.make_random_root_id() -print(root_id) -print(orgs.utils.make_random_ou_id(root_id)) -print(orgs.utils.make_random_account_id()) -print(orgs.utils.make_random_create_account_status_id()) - -# models -my_org = orgs.models.FakeOrganization(feature_set='ALL') -print(yaml.dump(my_org._describe())) -#assert False - -my_account = orgs.models.FakeAccount(my_org, AccountName='blee01', Email='blee01@moto-example.org') -print(yaml.dump(my_account)) -#assert False diff --git a/tests/test_organizations/organizations_test_utils.py b/tests/test_organizations/organizations_test_utils.py new file mode 100644 index 000000000..6548b1830 --- /dev/null +++ b/tests/test_organizations/organizations_test_utils.py @@ -0,0 +1,136 @@ +from __future__ import unicode_literals + +import six +import sure # noqa +import datetime +from moto.organizations import utils + +EMAIL_REGEX = "^.+@[a-zA-Z0-9-.]+.[a-zA-Z]{2,3}|[0-9]{1,3}$" +ORG_ID_REGEX = r'o-[a-z0-9]{%s}' % utils.ORG_ID_SIZE +ROOT_ID_REGEX = r'r-[a-z0-9]{%s}' % utils.ROOT_ID_SIZE +OU_ID_REGEX = r'ou-[a-z0-9]{%s}-[a-z0-9]{%s}' % (utils.ROOT_ID_SIZE, utils.OU_ID_SUFFIX_SIZE) +ACCOUNT_ID_REGEX = r'[0-9]{%s}' % utils.ACCOUNT_ID_SIZE +CREATE_ACCOUNT_STATUS_ID_REGEX = r'car-[a-z0-9]{%s}' % utils.CREATE_ACCOUNT_STATUS_ID_SIZE + + +def test_make_random_org_id(): + org_id = utils.make_random_org_id() + org_id.should.match(ORG_ID_REGEX) + + +def test_make_random_root_id(): + root_id = utils.make_random_root_id() + root_id.should.match(ROOT_ID_REGEX) + + +def test_make_random_ou_id(): + root_id = utils.make_random_root_id() + ou_id = utils.make_random_ou_id(root_id) + ou_id.should.match(OU_ID_REGEX) + + +def test_make_random_account_id(): + account_id = utils.make_random_account_id() + account_id.should.match(ACCOUNT_ID_REGEX) + + +def test_make_random_create_account_status_id(): + create_account_status_id = utils.make_random_create_account_status_id() + create_account_status_id.should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) + + +def validate_organization(response): + org = response['Organization'] + sorted(org.keys()).should.equal([ + 'Arn', + 'AvailablePolicyTypes', + 'FeatureSet', + 'Id', + 'MasterAccountArn', + 'MasterAccountEmail', + 'MasterAccountId', + ]) + org['Id'].should.match(ORG_ID_REGEX) + org['MasterAccountId'].should.equal(utils.MASTER_ACCOUNT_ID) + org['MasterAccountArn'].should.equal(utils.MASTER_ACCOUNT_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + )) + org['Arn'].should.equal(utils.ORGANIZATION_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + )) + org['MasterAccountEmail'].should.equal(utils.MASTER_ACCOUNT_EMAIL) + org['FeatureSet'].should.be.within(['ALL', 'CONSOLIDATED_BILLING']) + org['AvailablePolicyTypes'].should.equal([{ + 'Type': 'SERVICE_CONTROL_POLICY', + 'Status': 'ENABLED' + }]) + + +def validate_roots(org, response): + response.should.have.key('Roots').should.be.a(list) + response['Roots'].should_not.be.empty + root = response['Roots'][0] + root.should.have.key('Id').should.match(ROOT_ID_REGEX) + root.should.have.key('Arn').should.equal(utils.ROOT_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + root['Id'], + )) + root.should.have.key('Name').should.be.a(six.string_types) + root.should.have.key('PolicyTypes').should.be.a(list) + root['PolicyTypes'][0].should.have.key('Type').should.equal('SERVICE_CONTROL_POLICY') + root['PolicyTypes'][0].should.have.key('Status').should.equal('ENABLED') + + +def validate_organizational_unit(org, response): + response.should.have.key('OrganizationalUnit').should.be.a(dict) + ou = response['OrganizationalUnit'] + ou.should.have.key('Id').should.match(OU_ID_REGEX) + ou.should.have.key('Arn').should.equal(utils.OU_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + ou['Id'], + )) + ou.should.have.key('Name').should.be.a(six.string_types) + + +def validate_account(org, account): + sorted(account.keys()).should.equal([ + 'Arn', + 'Email', + 'Id', + 'JoinedMethod', + 'JoinedTimestamp', + 'Name', + 'Status', + ]) + account['Id'].should.match(ACCOUNT_ID_REGEX) + account['Arn'].should.equal(utils.ACCOUNT_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + account['Id'], + )) + account['Email'].should.match(EMAIL_REGEX) + account['JoinedMethod'].should.be.within(['INVITED', 'CREATED']) + account['Status'].should.be.within(['ACTIVE', 'SUSPENDED']) + account['Name'].should.be.a(six.string_types) + account['JoinedTimestamp'].should.be.a(datetime.datetime) + + +def validate_create_account_status(create_status): + sorted(create_status.keys()).should.equal([ + 'AccountId', + 'AccountName', + 'CompletedTimestamp', + 'Id', + 'RequestedTimestamp', + 'State', + ]) + create_status['Id'].should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) + create_status['AccountId'].should.match(ACCOUNT_ID_REGEX) + create_status['AccountName'].should.be.a(six.string_types) + create_status['State'].should.equal('SUCCEEDED') + create_status['RequestedTimestamp'].should.be.a(datetime.datetime) + create_status['CompletedTimestamp'].should.be.a(datetime.datetime) diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index 0ac2b61bf..c2f06774a 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -2,120 +2,24 @@ from __future__ import unicode_literals import boto3 import sure # noqa -import datetime import yaml -import six from moto import mock_organizations -from moto.organizations.models import ( - MASTER_ACCOUNT_ID, - MASTER_ACCOUNT_EMAIL, - ORGANIZATION_ARN_FORMAT, - MASTER_ACCOUNT_ARN_FORMAT, - ACCOUNT_ARN_FORMAT, - ROOT_ARN_FORMAT, - OU_ARN_FORMAT, +from .organizations_test_utils import ( + validate_organization, + validate_roots, + validate_organizational_unit, + validate_account, + validate_create_account_status, ) -from .test_organizations_utils import ( - ORG_ID_REGEX, - ROOT_ID_REGEX, - OU_ID_REGEX, - ACCOUNT_ID_REGEX, - CREATE_ACCOUNT_STATUS_ID_REGEX, -) - -EMAIL_REGEX = "^.+@[a-zA-Z0-9-.]+.[a-zA-Z]{2,3}|[0-9]{1,3}$" - - -def validate_organization(response): - org = response['Organization'] - sorted(org.keys()).should.equal([ - 'Arn', - 'AvailablePolicyTypes', - 'FeatureSet', - 'Id', - 'MasterAccountArn', - 'MasterAccountEmail', - 'MasterAccountId', - ]) - org['Id'].should.match(ORG_ID_REGEX) - org['MasterAccountId'].should.equal(MASTER_ACCOUNT_ID) - org['MasterAccountArn'].should.equal(MASTER_ACCOUNT_ARN_FORMAT.format( - org['MasterAccountId'], - org['Id'], - )) - org['Arn'].should.equal(ORGANIZATION_ARN_FORMAT.format( - org['MasterAccountId'], - org['Id'], - )) - org['MasterAccountEmail'].should.equal(MASTER_ACCOUNT_EMAIL) - org['FeatureSet'].should.be.within(['ALL', 'CONSOLIDATED_BILLING']) - org['AvailablePolicyTypes'].should.equal([{ - 'Type': 'SERVICE_CONTROL_POLICY', - 'Status': 'ENABLED' - }]) - - -def validate_organizational_unit(org, response): - response.should.have.key('OrganizationalUnit').should.be.a(dict) - ou = response['OrganizationalUnit'] - ou.should.have.key('Id').should.match(OU_ID_REGEX) - ou.should.have.key('Arn').should.equal(OU_ARN_FORMAT.format( - org['MasterAccountId'], - org['Id'], - ou['Id'], - )) - ou.should.have.key('Name').should.be.a(six.string_types) - - -def validate_account(org, account): - sorted(account.keys()).should.equal([ - 'Arn', - 'Email', - 'Id', - 'JoinedMethod', - 'JoinedTimestamp', - 'Name', - 'Status', - ]) - account['Id'].should.match(ACCOUNT_ID_REGEX) - account['Arn'].should.equal(ACCOUNT_ARN_FORMAT.format( - org['MasterAccountId'], - org['Id'], - account['Id'], - )) - account['Email'].should.match(EMAIL_REGEX) - account['JoinedMethod'].should.be.within(['INVITED', 'CREATED']) - account['Status'].should.be.within(['ACTIVE', 'SUSPENDED']) - account['Name'].should.be.a(six.string_types) - account['JoinedTimestamp'].should.be.a(datetime.datetime) - - -def validate_create_account_status(create_status): - sorted(create_status.keys()).should.equal([ - 'AccountId', - 'AccountName', - 'CompletedTimestamp', - 'Id', - 'RequestedTimestamp', - 'State', - ]) - create_status['Id'].should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) - create_status['AccountId'].should.match(ACCOUNT_ID_REGEX) - create_status['AccountName'].should.be.a(six.string_types) - create_status['State'].should.equal('SUCCEEDED') - create_status['RequestedTimestamp'].should.be.a(datetime.datetime) - create_status['CompletedTimestamp'].should.be.a(datetime.datetime) @mock_organizations def test_create_organization(): client = boto3.client('organizations', region_name='us-east-1') response = client.create_organization(FeatureSet='ALL') - #print(yaml.dump(response)) validate_organization(response) response['Organization']['FeatureSet'].should.equal('ALL') - #assert False @mock_organizations @@ -123,9 +27,7 @@ def test_describe_organization(): client = boto3.client('organizations', region_name='us-east-1') client.create_organization(FeatureSet='ALL') response = client.describe_organization() - #print(yaml.dump(response)) validate_organization(response) - #assert False # Organizational Units @@ -135,21 +37,7 @@ def test_list_roots(): client = boto3.client('organizations', region_name='us-east-1') org = client.create_organization(FeatureSet='ALL')['Organization'] response = client.list_roots() - #print(yaml.dump(response, default_flow_style=False)) - response.should.have.key('Roots').should.be.a(list) - response['Roots'].should_not.be.empty - root = response['Roots'][0] - root.should.have.key('Id').should.match(ROOT_ID_REGEX) - root.should.have.key('Arn').should.equal(ROOT_ARN_FORMAT.format( - org['MasterAccountId'], - org['Id'], - root['Id'], - )) - root.should.have.key('Name').should.be.a(six.string_types) - root.should.have.key('PolicyTypes').should.be.a(list) - root['PolicyTypes'][0].should.have.key('Type').should.equal('SERVICE_CONTROL_POLICY') - root['PolicyTypes'][0].should.have.key('Status').should.equal('ENABLED') - #assert False + validate_roots(org, response) @mock_organizations @@ -162,10 +50,8 @@ def test_create_organizational_unit(): ParentId=root_id, Name=ou_name, ) - #print(yaml.dump(response, default_flow_style=False)) validate_organizational_unit(org, response) response['OrganizationalUnit']['Name'].should.equal(ou_name) - #assert False @mock_organizations @@ -178,9 +64,7 @@ def test_describe_organizational_unit(): Name='ou01', )['OrganizationalUnit']['Id'] response = client.describe_organizational_unit(OrganizationalUnitId=ou_id) - print(yaml.dump(response, default_flow_style=False)) validate_organizational_unit(org, response) - #assert False @mock_organizations @@ -192,11 +76,9 @@ def test_list_organizational_units_for_parent(): client.create_organizational_unit(ParentId=root_id, Name='ou02') client.create_organizational_unit(ParentId=root_id, Name='ou03') response = client.list_organizational_units_for_parent(ParentId=root_id) - print(yaml.dump(response, default_flow_style=False)) response.should.have.key('OrganizationalUnits').should.be.a(list) for ou in response['OrganizationalUnits']: validate_organizational_unit(org, dict(OrganizationalUnit=ou)) - #assert False # Accounts @@ -212,10 +94,8 @@ def test_create_account(): create_status = client.create_account( AccountName=mockname, Email=mockemail )['CreateAccountStatus'] - #print(yaml.dump(create_status, default_flow_style=False)) validate_create_account_status(create_status) create_status['AccountName'].should.equal(mockname) - #assert False @mock_organizations @@ -226,11 +106,9 @@ def test_describe_account(): AccountName=mockname, Email=mockemail )['CreateAccountStatus']['AccountId'] response = client.describe_account(AccountId=account_id) - #print(yaml.dump(response, default_flow_style=False)) validate_account(org, response['Account']) response['Account']['Name'].should.equal(mockname) response['Account']['Email'].should.equal(mockemail) - #assert False @mock_organizations @@ -242,7 +120,6 @@ def test_list_accounts(): email = name + '@' + mockdomain client.create_account(AccountName=name, Email=email) response = client.list_accounts() - #print(yaml.dump(response, default_flow_style=False)) response.should.have.key('Accounts') accounts = response['Accounts'] len(accounts).should.equal(5) @@ -250,7 +127,6 @@ def test_list_accounts(): validate_account(org, account) accounts[3]['Name'].should.equal(mockname + '3') accounts[2]['Email'].should.equal(mockname + '2' + '@' + mockdomain) - #assert False @mock_organizations @@ -263,9 +139,7 @@ def test_list_accounts_for_parent(): Email=mockemail, )['CreateAccountStatus']['AccountId'] response = client.list_accounts_for_parent(ParentId=root_id) - #print(yaml.dump(response, default_flow_style=False)) account_id.should.be.within([account['Id'] for account in response['Accounts']]) - #assert False @mock_organizations @@ -284,9 +158,7 @@ def test_move_account(): DestinationParentId=ou01_id, ) response = client.list_accounts_for_parent(ParentId=ou01_id) - #print(yaml.dump(response, default_flow_style=False)) account_id.should.be.within([account['Id'] for account in response['Accounts']]) - #assert False @mock_organizations @@ -297,18 +169,15 @@ def test_list_parents_for_ou(): ou01 = client.create_organizational_unit(ParentId=root_id, Name='ou01') ou01_id = ou01['OrganizationalUnit']['Id'] response01 = client.list_parents(ChildId=ou01_id) - #print(yaml.dump(response01, default_flow_style=False)) response01.should.have.key('Parents').should.be.a(list) response01['Parents'][0].should.have.key('Id').should.equal(root_id) response01['Parents'][0].should.have.key('Type').should.equal('ROOT') ou02 = client.create_organizational_unit(ParentId=ou01_id, Name='ou02') ou02_id = ou02['OrganizationalUnit']['Id'] response02 = client.list_parents(ChildId=ou02_id) - #print(yaml.dump(response02, default_flow_style=False)) response02.should.have.key('Parents').should.be.a(list) response02['Parents'][0].should.have.key('Id').should.equal(ou01_id) response02['Parents'][0].should.have.key('Type').should.equal('ORGANIZATIONAL_UNIT') - #assert False @mock_organizations @@ -332,16 +201,13 @@ def test_list_parents_for_accounts(): DestinationParentId=ou01_id, ) response01 = client.list_parents(ChildId=account01_id) - #print(yaml.dump(response01, default_flow_style=False)) response01.should.have.key('Parents').should.be.a(list) response01['Parents'][0].should.have.key('Id').should.equal(root_id) response01['Parents'][0].should.have.key('Type').should.equal('ROOT') response02 = client.list_parents(ChildId=account02_id) - #print(yaml.dump(response02, default_flow_style=False)) response02.should.have.key('Parents').should.be.a(list) response02['Parents'][0].should.have.key('Id').should.equal(ou01_id) response02['Parents'][0].should.have.key('Type').should.equal('ORGANIZATIONAL_UNIT') - #assert False @mock_organizations @@ -370,10 +236,6 @@ def test_list_chidlren(): response02 = client.list_children(ParentId=root_id, ChildType='ORGANIZATIONAL_UNIT') response03 = client.list_children(ParentId=ou01_id, ChildType='ACCOUNT') response04 = client.list_children(ParentId=ou01_id, ChildType='ORGANIZATIONAL_UNIT') - #print(yaml.dump(response01, default_flow_style=False)) - #print(yaml.dump(response02, default_flow_style=False)) - #print(yaml.dump(response03, default_flow_style=False)) - #print(yaml.dump(response04, default_flow_style=False)) response01['Children'][0]['Id'].should.equal(account01_id) response01['Children'][0]['Type'].should.equal('ACCOUNT') response02['Children'][0]['Id'].should.equal(ou01_id) @@ -382,4 +244,3 @@ def test_list_chidlren(): response03['Children'][0]['Type'].should.equal('ACCOUNT') response04['Children'][0]['Id'].should.equal(ou02_id) response04['Children'][0]['Type'].should.equal('ORGANIZATIONAL_UNIT') - #assert False diff --git a/tests/test_organizations/test_organizations_utils.py b/tests/test_organizations/test_organizations_utils.py deleted file mode 100644 index d27201446..000000000 --- a/tests/test_organizations/test_organizations_utils.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import unicode_literals - -import sure # noqa -from moto.organizations import utils - -ORG_ID_REGEX = r'o-[a-z0-9]{%s}' % utils.ORG_ID_SIZE -ROOT_ID_REGEX = r'r-[a-z0-9]{%s}' % utils.ROOT_ID_SIZE -OU_ID_REGEX = r'ou-[a-z0-9]{%s}-[a-z0-9]{%s}' % (utils.ROOT_ID_SIZE, utils.OU_ID_SUFFIX_SIZE) -ACCOUNT_ID_REGEX = r'[0-9]{%s}' % utils.ACCOUNT_ID_SIZE -CREATE_ACCOUNT_STATUS_ID_REGEX = r'car-[a-z0-9]{%s}' % utils.CREATE_ACCOUNT_STATUS_ID_SIZE - - -def test_make_random_org_id(): - org_id = utils.make_random_org_id() - org_id.should.match(ORG_ID_REGEX) - - -def test_make_random_root_id(): - root_id = utils.make_random_root_id() - root_id.should.match(ROOT_ID_REGEX) - - -def test_make_random_ou_id(): - root_id = utils.make_random_root_id() - ou_id = utils.make_random_ou_id(root_id) - ou_id.should.match(OU_ID_REGEX) - - -def test_make_random_account_id(): - account_id = utils.make_random_account_id() - account_id.should.match(ACCOUNT_ID_REGEX) - - -def test_make_random_create_account_status_id(): - create_account_status_id = utils.make_random_create_account_status_id() - create_account_status_id.should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) From 05928b1497831e38a642baadc438b8238d65b4bf Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Fri, 20 Jul 2018 13:54:53 -0700 Subject: [PATCH 072/106] [issue #1720] Add support for AWS Organizations added exception handling in class OrganizationsBackend --- IMPLEMENTATION_COVERAGE.md | 4 +- README.md | 2 +- moto/organizations/models.py | 76 ++++++++++++------- .../test_organizations_boto3.py | 69 ++++++++++++++++- 4 files changed, 120 insertions(+), 31 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index e1943aa9a..d19d2473e 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3147,7 +3147,7 @@ - [ ] update_server - [ ] update_server_engine_attributes -## organizations - 28% implemented +## organizations - 30% implemented - [ ] accept_handshake - [ ] attach_policy - [ ] cancel_handshake @@ -3176,7 +3176,7 @@ - [X] list_accounts - [X] list_accounts_for_parent - [ ] list_aws_service_access_for_organization -- [ ] list_children +- [X] list_children - [ ] list_create_account_status - [ ] list_handshakes_for_account - [ ] list_handshakes_for_organization diff --git a/README.md b/README.md index e4fcb6508..189bf2c4f 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ It gets even better! Moto isn't just for Python code and it isn't just for S3. L |------------------------------------------------------------------------------| | KMS | @mock_kms | basic endpoints done | |------------------------------------------------------------------------------| -| Organizations | @mock_organizations | some endpoints done | +| Organizations | @mock_organizations | some core endpoints done | |------------------------------------------------------------------------------| | Polly | @mock_polly | all endpoints done | |------------------------------------------------------------------------------| diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 0f3c67400..91896f537 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -4,6 +4,7 @@ import datetime import re from moto.core import BaseBackend, BaseModel +from moto.core.exceptions import RESTError from moto.core.utils import unix_time from moto.organizations import utils @@ -168,13 +169,31 @@ class OrganizationsBackend(BaseBackend): self.ou.append(new_ou) return new_ou.describe() + def get_organizational_unit_by_id(self, ou_id): + ou = next((ou for ou in self.ou if ou.id == ou_id), None) + if ou is None: + raise RESTError( + 'OrganizationalUnitNotFoundException', + "You specified an organizational unit that doesn't exist." + ) + return ou + + def validate_parent_id(self, parent_id): + try: + self.get_organizational_unit_by_id(parent_id) + except RESTError as e: + raise RESTError( + 'ParentNotFoundException', + "You specified parent that doesn't exist." + ) + return parent_id + def describe_organizational_unit(self, **kwargs): - ou = [ - ou for ou in self.ou if ou.id == kwargs['OrganizationalUnitId'] - ].pop(0) + ou = self.get_organizational_unit_by_id(kwargs['OrganizationalUnitId']) return ou.describe() def list_organizational_units_for_parent(self, **kwargs): + parent_id = self.validate_parent_id(kwargs['ParentId']) return dict( OrganizationalUnits=[ { @@ -183,7 +202,7 @@ class OrganizationsBackend(BaseBackend): 'Name': ou.name, } for ou in self.ou - if ou.parent_id == kwargs['ParentId'] + if ou.parent_id == parent_id ] ) @@ -192,11 +211,20 @@ class OrganizationsBackend(BaseBackend): self.accounts.append(new_account) return new_account.create_account_status - def describe_account(self, **kwargs): - account = [ + def get_account_by_id(self, account_id): + account = next(( account for account in self.accounts - if account.id == kwargs['AccountId'] - ].pop(0) + if account.id == account_id + ), None) + if account is None: + raise RESTError( + 'AccountNotFoundException', + "You specified an account that doesn't exist." + ) + return account + + def describe_account(self, **kwargs): + account = self.get_account_by_id(kwargs['AccountId']) return account.describe() def list_accounts(self): @@ -205,35 +233,27 @@ class OrganizationsBackend(BaseBackend): ) def list_accounts_for_parent(self, **kwargs): + parent_id = self.validate_parent_id(kwargs['ParentId']) return dict( Accounts=[ account.describe()['Account'] for account in self.accounts - if account.parent_id == kwargs['ParentId'] + if account.parent_id == parent_id ] ) def move_account(self, **kwargs): - new_parent_id = kwargs['DestinationParentId'] - all_parent_id = [parent.id for parent in self.ou] - account = [ - account for account in self.accounts - if account.id == kwargs['AccountId'] - ].pop(0) - assert new_parent_id in all_parent_id - assert account.parent_id == kwargs['SourceParentId'] + new_parent_id = self.validate_parent_id(kwargs['DestinationParentId']) + self.validate_parent_id(kwargs['SourceParentId']) + account = self.get_account_by_id(kwargs['AccountId']) index = self.accounts.index(account) self.accounts[index].parent_id = new_parent_id def list_parents(self, **kwargs): if re.compile(r'[0-9]{12}').match(kwargs['ChildId']): - obj_list = self.accounts + child_object = self.get_account_by_id(kwargs['ChildId']) else: - obj_list = self.ou - parent_id = [ - obj.parent_id for obj in obj_list - if obj.id == kwargs['ChildId'] - ].pop(0) + child_object = self.get_organizational_unit_by_id(kwargs['ChildId']) return dict( Parents=[ { @@ -241,17 +261,21 @@ class OrganizationsBackend(BaseBackend): 'Type': ou.type, } for ou in self.ou - if ou.id == parent_id + if ou.id == child_object.parent_id ] ) def list_children(self, **kwargs): + parent_id = self.validate_parent_id(kwargs['ParentId']) if kwargs['ChildType'] == 'ACCOUNT': obj_list = self.accounts elif kwargs['ChildType'] == 'ORGANIZATIONAL_UNIT': obj_list = self.ou else: - raise ValueError + raise RESTError( + 'InvalidInputException', + 'You specified an invalid value.' + ) return dict( Children=[ { @@ -259,7 +283,7 @@ class OrganizationsBackend(BaseBackend): 'Type': kwargs['ChildType'], } for obj in obj_list - if obj.parent_id == kwargs['ParentId'] + if obj.parent_id == parent_id ] ) diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index c2f06774a..ae9bacd82 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -2,9 +2,11 @@ from __future__ import unicode_literals import boto3 import sure # noqa -import yaml +from botocore.exceptions import ClientError +from nose.tools import assert_raises from moto import mock_organizations +from moto.organizations import utils from .organizations_test_utils import ( validate_organization, validate_roots, @@ -67,6 +69,20 @@ def test_describe_organizational_unit(): validate_organizational_unit(org, response) +@mock_organizations +def test_describe_organizational_unit_exception(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + with assert_raises(ClientError) as e: + response = client.describe_organizational_unit( + OrganizationalUnitId=utils.make_random_root_id() + ) + ex = e.exception + ex.operation_name.should.equal('DescribeOrganizationalUnit') + ex.response['Error']['Code'].should.equal('400') + ex.response['Error']['Message'].should.contain('OrganizationalUnitNotFoundException') + + @mock_organizations def test_list_organizational_units_for_parent(): client = boto3.client('organizations', region_name='us-east-1') @@ -81,6 +97,19 @@ def test_list_organizational_units_for_parent(): validate_organizational_unit(org, dict(OrganizationalUnit=ou)) +@mock_organizations +def test_list_organizational_units_for_parent_exception(): + client = boto3.client('organizations', region_name='us-east-1') + with assert_raises(ClientError) as e: + response = client.list_organizational_units_for_parent( + ParentId=utils.make_random_root_id() + ) + ex = e.exception + ex.operation_name.should.equal('ListOrganizationalUnitsForParent') + ex.response['Error']['Code'].should.equal('400') + ex.response['Error']['Message'].should.contain('ParentNotFoundException') + + # Accounts mockname = 'mock-account' mockdomain = 'moto-example.org' @@ -111,6 +140,17 @@ def test_describe_account(): response['Account']['Email'].should.equal(mockemail) +@mock_organizations +def test_describe_account_exception(): + client = boto3.client('organizations', region_name='us-east-1') + with assert_raises(ClientError) as e: + response = client.describe_account(AccountId=utils.make_random_account_id()) + ex = e.exception + ex.operation_name.should.equal('DescribeAccount') + ex.response['Error']['Code'].should.equal('400') + ex.response['Error']['Message'].should.contain('AccountNotFoundException') + + @mock_organizations def test_list_accounts(): client = boto3.client('organizations', region_name='us-east-1') @@ -211,7 +251,7 @@ def test_list_parents_for_accounts(): @mock_organizations -def test_list_chidlren(): +def test_list_children(): client = boto3.client('organizations', region_name='us-east-1') org = client.create_organization(FeatureSet='ALL')['Organization'] root_id = client.list_roots()['Roots'][0]['Id'] @@ -244,3 +284,28 @@ def test_list_chidlren(): response03['Children'][0]['Type'].should.equal('ACCOUNT') response04['Children'][0]['Id'].should.equal(ou02_id) response04['Children'][0]['Type'].should.equal('ORGANIZATIONAL_UNIT') + + +@mock_organizations +def test_list_children_exception(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + with assert_raises(ClientError) as e: + response = client.list_children( + ParentId=utils.make_random_root_id(), + ChildType='ACCOUNT' + ) + ex = e.exception + ex.operation_name.should.equal('ListChildren') + ex.response['Error']['Code'].should.equal('400') + ex.response['Error']['Message'].should.contain('ParentNotFoundException') + with assert_raises(ClientError) as e: + response = client.list_children( + ParentId=root_id, + ChildType='BLEE' + ) + ex = e.exception + ex.operation_name.should.equal('ListChildren') + ex.response['Error']['Code'].should.equal('400') + ex.response['Error']['Message'].should.contain('InvalidInputException') From 4356e951e10795fcb9b73ac6b67540fbfeda2001 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Fri, 20 Jul 2018 14:07:04 -0700 Subject: [PATCH 073/106] [issue #1720] Add support for AWS Organizations fix travis build error --- moto/organizations/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 91896f537..c02b1a492 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -181,7 +181,7 @@ class OrganizationsBackend(BaseBackend): def validate_parent_id(self, parent_id): try: self.get_organizational_unit_by_id(parent_id) - except RESTError as e: + except RESTError: raise RESTError( 'ParentNotFoundException', "You specified parent that doesn't exist." From b8be517be08c5d7b42315ac1af7da23d978d5f9f Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Thu, 26 Jul 2018 12:20:43 -0700 Subject: [PATCH 074/106] organizations support: add exception handling for describe_organizations --- moto/organizations/models.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index c02b1a492..9d5fe3886 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -157,6 +157,11 @@ class OrganizationsBackend(BaseBackend): return self.org.describe() def describe_organization(self): + if not self.org: + raise RESTError( + 'AWSOrganizationsNotInUseException', + "Your account is not a member of an organization." + ) return self.org.describe() def list_roots(self): From 3afb2862c0cdb8e9d229b7c7e4141e1abfa57d00 Mon Sep 17 00:00:00 2001 From: William Richard Date: Mon, 1 Oct 2018 16:30:23 -0400 Subject: [PATCH 075/106] Filter event log ids should be strings Based on the boto docs, eventId should be returned as a string. https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/logs.html#CloudWatchLogs.Client.filter_log_events --- moto/logs/models.py | 2 +- tests/test_logs/test_logs.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/moto/logs/models.py b/moto/logs/models.py index a4ff9db46..ca1fdc4ad 100644 --- a/moto/logs/models.py +++ b/moto/logs/models.py @@ -19,7 +19,7 @@ class LogEvent: def to_filter_dict(self): return { - "eventId": self.eventId, + "eventId": str(self.eventId), "ingestionTime": self.ingestionTime, # "logStreamName": "message": self.message, diff --git a/tests/test_logs/test_logs.py b/tests/test_logs/test_logs.py index 05bd3c823..e3d46fd87 100644 --- a/tests/test_logs/test_logs.py +++ b/tests/test_logs/test_logs.py @@ -121,4 +121,8 @@ def test_filter_logs_interleaved(): interleaved=True, ) events = res['events'] - events.should.have.length_of(2) + for original_message, resulting_event in zip(messages, events): + resulting_event['eventId'].should.equal(str(resulting_event['eventId'])) + resulting_event['timestamp'].should.equal(original_message['timestamp']) + resulting_event['message'].should.equal(original_message['message']) + From ea4fcaa82a969ffdc2482fbd18c41cf117e22cc9 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Wed, 3 Oct 2018 00:40:28 -0500 Subject: [PATCH 076/106] add support for NoncurrentVersionTransition, NoncurrentVersionExpiration, and AbortIncompleteMultipartUpload actions to s3 lifecycle rules --- moto/s3/models.py | 24 ++++-- moto/s3/responses.py | 16 ++++ tests/test_s3/test_s3_lifecycle.py | 129 +++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+), 6 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index f3994b5d8..4b1a343d1 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -341,8 +341,9 @@ class LifecycleAndFilter(BaseModel): class LifecycleRule(BaseModel): def __init__(self, id=None, prefix=None, lc_filter=None, status=None, expiration_days=None, - expiration_date=None, transition_days=None, expired_object_delete_marker=None, - transition_date=None, storage_class=None): + expiration_date=None, transition_days=None, transition_date=None, storage_class=None, + expired_object_delete_marker=None, nve_noncurrent_days=None, nvt_noncurrent_days=None, + nvt_storage_class=None, aimu_days=None): self.id = id self.prefix = prefix self.filter = lc_filter @@ -351,8 +352,12 @@ class LifecycleRule(BaseModel): self.expiration_date = expiration_date self.transition_days = transition_days self.transition_date = transition_date - self.expired_object_delete_marker = expired_object_delete_marker self.storage_class = storage_class + self.expired_object_delete_marker = expired_object_delete_marker + self.nve_noncurrent_days = nve_noncurrent_days + self.nvt_noncurrent_days = nvt_noncurrent_days + self.nvt_storage_class = nvt_storage_class + self.aimu_days = aimu_days class CorsRule(BaseModel): @@ -414,8 +419,12 @@ class FakeBucket(BaseModel): def set_lifecycle(self, rules): self.rules = [] for rule in rules: + # Extract actions from Lifecycle rule expiration = rule.get('Expiration') transition = rule.get('Transition') + nve = rule.get('NoncurrentVersionExpiration') + nvt = rule.get('NoncurrentVersionTransition') + aimu = rule.get('AbortIncompleteMultipartUpload') eodm = None if expiration and expiration.get("ExpiredObjectDeleteMarker") is not None: @@ -459,11 +468,14 @@ class FakeBucket(BaseModel): status=rule['Status'], expiration_days=expiration.get('Days') if expiration else None, expiration_date=expiration.get('Date') if expiration else None, - expired_object_delete_marker=eodm, transition_days=transition.get('Days') if transition else None, transition_date=transition.get('Date') if transition else None, - storage_class=transition[ - 'StorageClass'] if transition else None, + storage_class=transition.get('StorageClass') if transition else None, + expired_object_delete_marker=eodm, + nve_noncurrent_days = nve.get('NoncurrentDays') if nve else None, + nvt_noncurrent_days = nvt.get('NoncurrentDays') if nvt else None, + nvt_storage_class = nvt.get('StorageClass') if nvt else None, + aimu_days=aimu.get('DaysAfterInitiation') if aimu else None, )) def delete_lifecycle(self): diff --git a/moto/s3/responses.py b/moto/s3/responses.py index f8dc7e42b..de101a193 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -1228,6 +1228,22 @@ S3_BUCKET_LIFECYCLE_CONFIGURATION = """ {% endif %} {% endif %} + {% if rule.nvt_noncurrent_days and rule.nvt_storage_class %} + + {{ rule.nvt_noncurrent_days }} + {{ rule.nvt_storage_class }} + + {% endif %} + {% if rule.nve_noncurrent_days %} + + {{ rule.nve_noncurrent_days }} + + {% endif %} + {% if rule.aimu_days %} + + {{ rule.aimu_days }} + + {% endif %} {% endfor %} diff --git a/tests/test_s3/test_s3_lifecycle.py b/tests/test_s3/test_s3_lifecycle.py index d176e95c6..9c4bd49bf 100644 --- a/tests/test_s3/test_s3_lifecycle.py +++ b/tests/test_s3/test_s3_lifecycle.py @@ -191,6 +191,135 @@ def test_lifecycle_with_eodm(): assert err.exception.response["Error"]["Code"] == "MalformedXML" +@mock_s3 +def test_lifecycle_with_nve(): + client = boto3.client("s3") + client.create_bucket(Bucket="bucket") + + lfc = { + "Rules": [ + { + "NoncurrentVersionExpiration": { + "NoncurrentDays": 30 + }, + "ID": "wholebucket", + "Filter": { + "Prefix": "" + }, + "Status": "Enabled" + } + ] + } + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["NoncurrentVersionExpiration"]["NoncurrentDays"] == 30 + + # Change NoncurrentDays: + lfc["Rules"][0]["NoncurrentVersionExpiration"]["NoncurrentDays"] = 10 + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["NoncurrentVersionExpiration"]["NoncurrentDays"] == 10 + + # With failures for missing children: + del lfc["Rules"][0]["NoncurrentVersionExpiration"]["NoncurrentDays"] + with assert_raises(ClientError) as err: + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + assert err.exception.response["Error"]["Code"] == "MalformedXML" + + +@mock_s3 +def test_lifecycle_with_nvt(): + client = boto3.client("s3") + client.create_bucket(Bucket="bucket") + + lfc = { + "Rules": [ + { + "NoncurrentVersionTransition": { + "NoncurrentDays": 30, + "StorageClass": "ONEZONE_IA" + }, + "ID": "wholebucket", + "Filter": { + "Prefix": "" + }, + "Status": "Enabled" + } + ] + } + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] == 30 + assert result["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] == "ONEZONE_IA" + + # Change NoncurrentDays: + lfc["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] = 10 + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] == 10 + + # Change StorageClass: + lfc["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] = "GLACIER" + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] == "GLACIER" + + # With failures for missing children: + del lfc["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] + with assert_raises(ClientError) as err: + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + assert err.exception.response["Error"]["Code"] == "MalformedXML" + lfc["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] = 30 + + del lfc["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] + with assert_raises(ClientError) as err: + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + assert err.exception.response["Error"]["Code"] == "MalformedXML" + + +@mock_s3 +def test_lifecycle_with_aimu(): + client = boto3.client("s3") + client.create_bucket(Bucket="bucket") + + lfc = { + "Rules": [ + { + "AbortIncompleteMultipartUpload": { + "DaysAfterInitiation": 7 + }, + "ID": "wholebucket", + "Filter": { + "Prefix": "" + }, + "Status": "Enabled" + } + ] + } + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] == 7 + + # Change DaysAfterInitiation: + lfc["Rules"][0]["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] = 30 + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] == 30 + + # With failures for missing children: + del lfc["Rules"][0]["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] + with assert_raises(ClientError) as err: + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + assert err.exception.response["Error"]["Code"] == "MalformedXML" + + @mock_s3_deprecated def test_lifecycle_with_glacier_transition(): conn = boto.s3.connect_to_region("us-west-1") From 691a8722a898edaa824d941a79a11a7e6f6627dd Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Wed, 3 Oct 2018 00:45:47 -0500 Subject: [PATCH 077/106] formatting fix for flake8 due to extra spaces --- moto/s3/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 4b1a343d1..d889b8cab 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -472,9 +472,9 @@ class FakeBucket(BaseModel): transition_date=transition.get('Date') if transition else None, storage_class=transition.get('StorageClass') if transition else None, expired_object_delete_marker=eodm, - nve_noncurrent_days = nve.get('NoncurrentDays') if nve else None, - nvt_noncurrent_days = nvt.get('NoncurrentDays') if nvt else None, - nvt_storage_class = nvt.get('StorageClass') if nvt else None, + nve_noncurrent_days=nve.get('NoncurrentDays') if nve else None, + nvt_noncurrent_days=nvt.get('NoncurrentDays') if nvt else None, + nvt_storage_class=nvt.get('StorageClass') if nvt else None, aimu_days=aimu.get('DaysAfterInitiation') if aimu else None, )) From 9b5f983cb5b4f02a9d7d0842f4f73e01bcab671b Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Wed, 3 Oct 2018 01:11:11 -0500 Subject: [PATCH 078/106] add action validation to set_lifecycle() --- moto/s3/models.py | 35 +++++++++++++++++++++++------- tests/test_s3/test_s3_lifecycle.py | 22 +++++++++---------- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index d889b8cab..fc977bb99 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -419,12 +419,31 @@ class FakeBucket(BaseModel): def set_lifecycle(self, rules): self.rules = [] for rule in rules: - # Extract actions from Lifecycle rule + # Extract and validate actions from Lifecycle rule expiration = rule.get('Expiration') transition = rule.get('Transition') - nve = rule.get('NoncurrentVersionExpiration') - nvt = rule.get('NoncurrentVersionTransition') - aimu = rule.get('AbortIncompleteMultipartUpload') + + nve_noncurrent_days = None + if rule.get('NoncurrentVersionExpiration'): + if not rule["NoncurrentVersionExpiration"].get('NoncurrentDays'): + raise MalformedXML() + nve_noncurrent_days = rule["NoncurrentVersionExpiration"]["NoncurrentDays"] + + nvt_noncurrent_days = None + nvt_storage_class = None + if rule.get('NoncurrentVersionTransition'): + if not rule["NoncurrentVersionTransition"].get('NoncurrentDays'): + raise MalformedXML() + if not rule["NoncurrentVersionTransition"].get('StorageClass'): + raise MalformedXML() + nvt_noncurrent_days = rule["NoncurrentVersionTransition"]["NoncurrentDays"] + nvt_storage_class = rule["NoncurrentVersionTransition"]["StorageClass"] + + aimu_days = None + if rule.get('AbortIncompleteMultipartUpload'): + if not rule["AbortIncompleteMultipartUpload"].get('DaysAfterInitiation'): + raise MalformedXML() + aimu_days = rule["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] eodm = None if expiration and expiration.get("ExpiredObjectDeleteMarker") is not None: @@ -472,10 +491,10 @@ class FakeBucket(BaseModel): transition_date=transition.get('Date') if transition else None, storage_class=transition.get('StorageClass') if transition else None, expired_object_delete_marker=eodm, - nve_noncurrent_days=nve.get('NoncurrentDays') if nve else None, - nvt_noncurrent_days=nvt.get('NoncurrentDays') if nvt else None, - nvt_storage_class=nvt.get('StorageClass') if nvt else None, - aimu_days=aimu.get('DaysAfterInitiation') if aimu else None, + nve_noncurrent_days=nve_noncurrent_days, + nvt_noncurrent_days=nvt_noncurrent_days, + nvt_storage_class=nvt_storage_class, + aimu_days=aimu_days, )) def delete_lifecycle(self): diff --git a/tests/test_s3/test_s3_lifecycle.py b/tests/test_s3/test_s3_lifecycle.py index 9c4bd49bf..ff89dc113 100644 --- a/tests/test_s3/test_s3_lifecycle.py +++ b/tests/test_s3/test_s3_lifecycle.py @@ -237,10 +237,10 @@ def test_lifecycle_with_nvt(): lfc = { "Rules": [ { - "NoncurrentVersionTransition": { + "NoncurrentVersionTransitions": [{ "NoncurrentDays": 30, "StorageClass": "ONEZONE_IA" - }, + }], "ID": "wholebucket", "Filter": { "Prefix": "" @@ -252,31 +252,31 @@ def test_lifecycle_with_nvt(): client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) result = client.get_bucket_lifecycle_configuration(Bucket="bucket") assert len(result["Rules"]) == 1 - assert result["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] == 30 - assert result["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] == "ONEZONE_IA" + assert result["Rules"][0]["NoncurrentVersionTransitions"][0]["NoncurrentDays"] == 30 + assert result["Rules"][0]["NoncurrentVersionTransitions"][0]["StorageClass"] == "ONEZONE_IA" # Change NoncurrentDays: - lfc["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] = 10 + lfc["Rules"][0]["NoncurrentVersionTransitions"][0]["NoncurrentDays"] = 10 client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) result = client.get_bucket_lifecycle_configuration(Bucket="bucket") assert len(result["Rules"]) == 1 - assert result["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] == 10 + assert result["Rules"][0]["NoncurrentVersionTransitions"][0]["NoncurrentDays"] == 10 # Change StorageClass: - lfc["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] = "GLACIER" + lfc["Rules"][0]["NoncurrentVersionTransitions"][0]["StorageClass"] = "GLACIER" client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) result = client.get_bucket_lifecycle_configuration(Bucket="bucket") assert len(result["Rules"]) == 1 - assert result["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] == "GLACIER" + assert result["Rules"][0]["NoncurrentVersionTransitions"][0]["StorageClass"] == "GLACIER" # With failures for missing children: - del lfc["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] + del lfc["Rules"][0]["NoncurrentVersionTransitions"][0]["NoncurrentDays"] with assert_raises(ClientError) as err: client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) assert err.exception.response["Error"]["Code"] == "MalformedXML" - lfc["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] = 30 + lfc["Rules"][0]["NoncurrentVersionTransitions"][0]["NoncurrentDays"] = 30 - del lfc["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] + del lfc["Rules"][0]["NoncurrentVersionTransitions"][0]["StorageClass"] with assert_raises(ClientError) as err: client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) assert err.exception.response["Error"]["Code"] == "MalformedXML" From a1a8ac7286ab30c2888b879bc16ecd2bd91dd1d7 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Wed, 3 Oct 2018 01:26:09 -0500 Subject: [PATCH 079/106] check for None in lifecycle actions --- moto/s3/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index fc977bb99..39f36982d 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -425,23 +425,23 @@ class FakeBucket(BaseModel): nve_noncurrent_days = None if rule.get('NoncurrentVersionExpiration'): - if not rule["NoncurrentVersionExpiration"].get('NoncurrentDays'): + if rule["NoncurrentVersionExpiration"].get('NoncurrentDays') is None: raise MalformedXML() nve_noncurrent_days = rule["NoncurrentVersionExpiration"]["NoncurrentDays"] nvt_noncurrent_days = None nvt_storage_class = None if rule.get('NoncurrentVersionTransition'): - if not rule["NoncurrentVersionTransition"].get('NoncurrentDays'): + if rule["NoncurrentVersionTransition"].get('NoncurrentDays') is None: raise MalformedXML() - if not rule["NoncurrentVersionTransition"].get('StorageClass'): + if rule["NoncurrentVersionTransition"].get('StorageClass') is None: raise MalformedXML() nvt_noncurrent_days = rule["NoncurrentVersionTransition"]["NoncurrentDays"] nvt_storage_class = rule["NoncurrentVersionTransition"]["StorageClass"] aimu_days = None if rule.get('AbortIncompleteMultipartUpload'): - if not rule["AbortIncompleteMultipartUpload"].get('DaysAfterInitiation'): + if rule["AbortIncompleteMultipartUpload"].get('DaysAfterInitiation') is None: raise MalformedXML() aimu_days = rule["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] From 5783d662064e248084675c9358bc7e50c4666c51 Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor Date: Tue, 2 Oct 2018 17:25:14 +0100 Subject: [PATCH 080/106] Mock more of the Glue Data Catalog APIs This adds some of the missing Get/Update/Create APIs relating to the Glue data catalog -- but not yet all of them, and none of the Batch* API calls. --- moto/glue/exceptions.py | 59 +++- moto/glue/models.py | 104 ++++++- moto/glue/responses.py | 103 +++++-- tests/test_glue/fixtures/datacatalog.py | 25 ++ tests/test_glue/helpers.py | 81 +++++- tests/test_glue/test_datacatalog.py | 362 ++++++++++++++++++++++-- 6 files changed, 673 insertions(+), 61 deletions(-) diff --git a/moto/glue/exceptions.py b/moto/glue/exceptions.py index 62ea1525c..8972adb35 100644 --- a/moto/glue/exceptions.py +++ b/moto/glue/exceptions.py @@ -6,19 +6,56 @@ class GlueClientError(JsonRESTError): code = 400 -class DatabaseAlreadyExistsException(GlueClientError): - def __init__(self): - self.code = 400 - super(DatabaseAlreadyExistsException, self).__init__( - 'DatabaseAlreadyExistsException', - 'Database already exists.' +class AlreadyExistsException(GlueClientError): + def __init__(self, typ): + super(GlueClientError, self).__init__( + 'AlreadyExistsException', + '%s already exists.' % (typ), ) -class TableAlreadyExistsException(GlueClientError): +class DatabaseAlreadyExistsException(AlreadyExistsException): def __init__(self): - self.code = 400 - super(TableAlreadyExistsException, self).__init__( - 'TableAlreadyExistsException', - 'Table already exists.' + super(DatabaseAlreadyExistsException, self).__init__('Database') + + +class TableAlreadyExistsException(AlreadyExistsException): + def __init__(self): + super(TableAlreadyExistsException, self).__init__('Table') + + +class PartitionAlreadyExistsException(AlreadyExistsException): + def __init__(self): + super(PartitionAlreadyExistsException, self).__init__('Partition') + + +class EntityNotFoundException(GlueClientError): + def __init__(self, msg): + super(GlueClientError, self).__init__( + 'EntityNotFoundException', + msg, ) + + +class DatabaseNotFoundException(EntityNotFoundException): + def __init__(self, db): + super(DatabaseNotFoundException, self).__init__( + 'Database %s not found.' % db, + ) + + +class TableNotFoundException(EntityNotFoundException): + def __init__(self, tbl): + super(TableNotFoundException, self).__init__( + 'Table %s not found.' % tbl, + ) + + +class PartitionNotFoundException(EntityNotFoundException): + def __init__(self): + super(PartitionNotFoundException, self).__init__("Cannot find partition.") + + +class VersionNotFoundException(EntityNotFoundException): + def __init__(self): + super(VersionNotFoundException, self).__init__("Version not found.") diff --git a/moto/glue/models.py b/moto/glue/models.py index 09b7d60ed..bcf2ec4bf 100644 --- a/moto/glue/models.py +++ b/moto/glue/models.py @@ -1,8 +1,19 @@ from __future__ import unicode_literals +import time + from moto.core import BaseBackend, BaseModel from moto.compat import OrderedDict -from.exceptions import DatabaseAlreadyExistsException, TableAlreadyExistsException +from.exceptions import ( + JsonRESTError, + DatabaseAlreadyExistsException, + DatabaseNotFoundException, + TableAlreadyExistsException, + TableNotFoundException, + PartitionAlreadyExistsException, + PartitionNotFoundException, + VersionNotFoundException, +) class GlueBackend(BaseBackend): @@ -19,7 +30,10 @@ class GlueBackend(BaseBackend): return database def get_database(self, database_name): - return self.databases[database_name] + try: + return self.databases[database_name] + except KeyError: + raise DatabaseNotFoundException(database_name) def create_table(self, database_name, table_name, table_input): database = self.get_database(database_name) @@ -33,7 +47,10 @@ class GlueBackend(BaseBackend): def get_table(self, database_name, table_name): database = self.get_database(database_name) - return database.tables[table_name] + try: + return database.tables[table_name] + except KeyError: + raise TableNotFoundException(table_name) def get_tables(self, database_name): database = self.get_database(database_name) @@ -52,9 +69,84 @@ class FakeTable(BaseModel): def __init__(self, database_name, table_name, table_input): self.database_name = database_name self.name = table_name - self.table_input = table_input - self.storage_descriptor = self.table_input.get('StorageDescriptor', {}) - self.partition_keys = self.table_input.get('PartitionKeys', []) + self.partitions = OrderedDict() + self.versions = [] + self.update(table_input) + + def update(self, table_input): + self.versions.append(table_input) + + def get_version(self, ver): + try: + if not isinstance(ver, int): + # "1" goes to [0] + ver = int(ver) - 1 + except ValueError as e: + raise JsonRESTError("InvalidInputException", str(e)) + + try: + return self.versions[ver] + except IndexError: + raise VersionNotFoundException() + + def as_dict(self, version=-1): + obj = { + 'DatabaseName': self.database_name, + 'Name': self.name, + } + obj.update(self.get_version(version)) + return obj + + def create_partition(self, partiton_input): + partition = FakePartition(self.database_name, self.name, partiton_input) + key = str(partition.values) + if key in self.partitions: + raise PartitionAlreadyExistsException() + self.partitions[str(partition.values)] = partition + + def get_partitions(self): + return [p for str_part_values, p in self.partitions.items()] + + def get_partition(self, values): + try: + return self.partitions[str(values)] + except KeyError: + raise PartitionNotFoundException() + + def update_partition(self, old_values, partiton_input): + partition = FakePartition(self.database_name, self.name, partiton_input) + key = str(partition.values) + if old_values == partiton_input['Values']: + # Altering a partition in place. Don't remove it so the order of + # returned partitions doesn't change + if key not in self.partitions: + raise PartitionNotFoundException() + else: + removed = self.partitions.pop(str(old_values), None) + if removed is None: + raise PartitionNotFoundException() + if key in self.partitions: + # Trying to update to overwrite a partition that exists + raise PartitionAlreadyExistsException() + self.partitions[key] = partition + + +class FakePartition(BaseModel): + def __init__(self, database_name, table_name, partiton_input): + self.creation_time = time.time() + self.database_name = database_name + self.table_name = table_name + self.partition_input = partiton_input + self.values = self.partition_input.get('Values', []) + + def as_dict(self): + obj = { + 'DatabaseName': self.database_name, + 'TableName': self.table_name, + 'CreationTime': self.creation_time, + } + obj.update(self.partition_input) + return obj glue_backend = GlueBackend() diff --git a/moto/glue/responses.py b/moto/glue/responses.py index bb64c40d4..84cc6f901 100644 --- a/moto/glue/responses.py +++ b/moto/glue/responses.py @@ -37,27 +37,94 @@ class GlueResponse(BaseResponse): database_name = self.parameters.get('DatabaseName') table_name = self.parameters.get('Name') table = self.glue_backend.get_table(database_name, table_name) + + return json.dumps({'Table': table.as_dict()}) + + def update_table(self): + database_name = self.parameters.get('DatabaseName') + table_input = self.parameters.get('TableInput') + table_name = table_input.get('Name') + table = self.glue_backend.get_table(database_name, table_name) + table.update(table_input) + return "" + + def get_table_versions(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + table = self.glue_backend.get_table(database_name, table_name) + return json.dumps({ - 'Table': { - 'DatabaseName': table.database_name, - 'Name': table.name, - 'PartitionKeys': table.partition_keys, - 'StorageDescriptor': table.storage_descriptor - } + "TableVersions": [ + { + "Table": table.as_dict(version=n), + "VersionId": str(n + 1), + } for n in range(len(table.versions)) + ], + }) + + def get_table_version(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + table = self.glue_backend.get_table(database_name, table_name) + ver_id = self.parameters.get('VersionId') + + return json.dumps({ + "TableVersion": { + "Table": table.as_dict(version=ver_id), + "VersionId": ver_id, + }, }) def get_tables(self): database_name = self.parameters.get('DatabaseName') tables = self.glue_backend.get_tables(database_name) - return json.dumps( - { - 'TableList': [ - { - 'DatabaseName': table.database_name, - 'Name': table.name, - 'PartitionKeys': table.partition_keys, - 'StorageDescriptor': table.storage_descriptor - } for table in tables - ] - } - ) + return json.dumps({ + 'TableList': [ + table.as_dict() for table in tables + ] + }) + + def get_partitions(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + if 'Expression' in self.parameters: + raise NotImplementedError("Expression filtering in get_partitions is not implemented in moto") + table = self.glue_backend.get_table(database_name, table_name) + + return json.dumps({ + 'Partitions': [ + p.as_dict() for p in table.get_partitions() + ] + }) + + def get_partition(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + values = self.parameters.get('PartitionValues') + + table = self.glue_backend.get_table(database_name, table_name) + + p = table.get_partition(values) + + return json.dumps({'Partition': p.as_dict()}) + + def create_partition(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + part_input = self.parameters.get('PartitionInput') + + table = self.glue_backend.get_table(database_name, table_name) + table.create_partition(part_input) + + return "" + + def update_partition(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + part_input = self.parameters.get('PartitionInput') + part_to_update = self.parameters.get('PartitionValueList') + + table = self.glue_backend.get_table(database_name, table_name) + table.update_partition(part_to_update, part_input) + + return "" diff --git a/tests/test_glue/fixtures/datacatalog.py b/tests/test_glue/fixtures/datacatalog.py index b2efe4154..edad2f0f4 100644 --- a/tests/test_glue/fixtures/datacatalog.py +++ b/tests/test_glue/fixtures/datacatalog.py @@ -29,3 +29,28 @@ TABLE_INPUT = { }, 'TableType': 'EXTERNAL_TABLE', } + + +PARTITION_INPUT = { + # 'DatabaseName': 'dbname', + 'StorageDescriptor': { + 'BucketColumns': [], + 'Columns': [], + 'Compressed': False, + 'InputFormat': 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat', + 'Location': 's3://.../partition=value', + 'NumberOfBuckets': -1, + 'OutputFormat': 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat', + 'Parameters': {}, + 'SerdeInfo': { + 'Parameters': {'path': 's3://...', 'serialization.format': '1'}, + 'SerializationLibrary': 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'}, + 'SkewedInfo': {'SkewedColumnNames': [], + 'SkewedColumnValueLocationMaps': {}, + 'SkewedColumnValues': []}, + 'SortColumns': [], + 'StoredAsSubDirectories': False, + }, + # 'TableName': 'source_table', + # 'Values': ['2018-06-26'], +} diff --git a/tests/test_glue/helpers.py b/tests/test_glue/helpers.py index 4a51f9117..331b99867 100644 --- a/tests/test_glue/helpers.py +++ b/tests/test_glue/helpers.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals import copy -from .fixtures.datacatalog import TABLE_INPUT +from .fixtures.datacatalog import TABLE_INPUT, PARTITION_INPUT def create_database(client, database_name): @@ -17,22 +17,38 @@ def get_database(client, database_name): return client.get_database(Name=database_name) -def create_table_input(table_name, s3_location, columns=[], partition_keys=[]): +def create_table_input(database_name, table_name, columns=[], partition_keys=[]): table_input = copy.deepcopy(TABLE_INPUT) table_input['Name'] = table_name table_input['PartitionKeys'] = partition_keys table_input['StorageDescriptor']['Columns'] = columns - table_input['StorageDescriptor']['Location'] = s3_location + table_input['StorageDescriptor']['Location'] = 's3://my-bucket/{database_name}/{table_name}'.format( + database_name=database_name, + table_name=table_name + ) return table_input -def create_table(client, database_name, table_name, table_input): +def create_table(client, database_name, table_name, table_input=None, **kwargs): + if table_input is None: + table_input = create_table_input(database_name, table_name, **kwargs) + return client.create_table( DatabaseName=database_name, TableInput=table_input ) +def update_table(client, database_name, table_name, table_input=None, **kwargs): + if table_input is None: + table_input = create_table_input(database_name, table_name, **kwargs) + + return client.update_table( + DatabaseName=database_name, + TableInput=table_input, + ) + + def get_table(client, database_name, table_name): return client.get_table( DatabaseName=database_name, @@ -44,3 +60,60 @@ def get_tables(client, database_name): return client.get_tables( DatabaseName=database_name ) + + +def get_table_versions(client, database_name, table_name): + return client.get_table_versions( + DatabaseName=database_name, + TableName=table_name + ) + + +def get_table_version(client, database_name, table_name, version_id): + return client.get_table_version( + DatabaseName=database_name, + TableName=table_name, + VersionId=version_id, + ) + + +def create_partition_input(database_name, table_name, values=[], columns=[]): + root_path = 's3://my-bucket/{database_name}/{table_name}'.format( + database_name=database_name, + table_name=table_name + ) + + part_input = copy.deepcopy(PARTITION_INPUT) + part_input['Values'] = values + part_input['StorageDescriptor']['Columns'] = columns + part_input['StorageDescriptor']['SerdeInfo']['Parameters']['path'] = root_path + return part_input + + +def create_partition(client, database_name, table_name, partiton_input=None, **kwargs): + if partiton_input is None: + partiton_input = create_partition_input(database_name, table_name, **kwargs) + return client.create_partition( + DatabaseName=database_name, + TableName=table_name, + PartitionInput=partiton_input + ) + + +def update_partition(client, database_name, table_name, old_values=[], partiton_input=None, **kwargs): + if partiton_input is None: + partiton_input = create_partition_input(database_name, table_name, **kwargs) + return client.update_partition( + DatabaseName=database_name, + TableName=table_name, + PartitionInput=partiton_input, + PartitionValueList=old_values, + ) + + +def get_partition(client, database_name, table_name, values): + return client.get_partition( + DatabaseName=database_name, + TableName=table_name, + PartitionValues=values, + ) diff --git a/tests/test_glue/test_datacatalog.py b/tests/test_glue/test_datacatalog.py index 7dabeb1f3..a457d5127 100644 --- a/tests/test_glue/test_datacatalog.py +++ b/tests/test_glue/test_datacatalog.py @@ -1,10 +1,15 @@ from __future__ import unicode_literals import sure # noqa +import re from nose.tools import assert_raises import boto3 from botocore.client import ClientError + +from datetime import datetime +import pytz + from moto import mock_glue from . import helpers @@ -30,7 +35,19 @@ def test_create_database_already_exists(): with assert_raises(ClientError) as exc: helpers.create_database(client, database_name) - exc.exception.response['Error']['Code'].should.equal('DatabaseAlreadyExistsException') + exc.exception.response['Error']['Code'].should.equal('AlreadyExistsException') + + +@mock_glue +def test_get_database_not_exits(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'nosuchdatabase' + + with assert_raises(ClientError) as exc: + helpers.get_database(client, database_name) + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + exc.exception.response['Error']['Message'].should.match('Database nosuchdatabase not found') @mock_glue @@ -40,12 +57,7 @@ def test_create_table(): helpers.create_database(client, database_name) table_name = 'myspecialtable' - s3_location = 's3://my-bucket/{database_name}/{table_name}'.format( - database_name=database_name, - table_name=table_name - ) - - table_input = helpers.create_table_input(table_name, s3_location) + table_input = helpers.create_table_input(database_name, table_name) helpers.create_table(client, database_name, table_name, table_input) response = helpers.get_table(client, database_name, table_name) @@ -63,18 +75,12 @@ def test_create_table_already_exists(): helpers.create_database(client, database_name) table_name = 'cantcreatethistabletwice' - s3_location = 's3://my-bucket/{database_name}/{table_name}'.format( - database_name=database_name, - table_name=table_name - ) - - table_input = helpers.create_table_input(table_name, s3_location) - helpers.create_table(client, database_name, table_name, table_input) + helpers.create_table(client, database_name, table_name) with assert_raises(ClientError) as exc: - helpers.create_table(client, database_name, table_name, table_input) + helpers.create_table(client, database_name, table_name) - exc.exception.response['Error']['Code'].should.equal('TableAlreadyExistsException') + exc.exception.response['Error']['Code'].should.equal('AlreadyExistsException') @mock_glue @@ -87,11 +93,7 @@ def test_get_tables(): table_inputs = {} for table_name in table_names: - s3_location = 's3://my-bucket/{database_name}/{table_name}'.format( - database_name=database_name, - table_name=table_name - ) - table_input = helpers.create_table_input(table_name, s3_location) + table_input = helpers.create_table_input(database_name, table_name) table_inputs[table_name] = table_input helpers.create_table(client, database_name, table_name, table_input) @@ -99,10 +101,326 @@ def test_get_tables(): tables = response['TableList'] - assert len(tables) == 3 + tables.should.have.length_of(3) for table in tables: table_name = table['Name'] table_name.should.equal(table_inputs[table_name]['Name']) table['StorageDescriptor'].should.equal(table_inputs[table_name]['StorageDescriptor']) table['PartitionKeys'].should.equal(table_inputs[table_name]['PartitionKeys']) + + +@mock_glue +def test_get_table_versions(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + helpers.create_database(client, database_name) + + table_name = 'myfirsttable' + version_inputs = {} + + table_input = helpers.create_table_input(database_name, table_name) + helpers.create_table(client, database_name, table_name, table_input) + version_inputs["1"] = table_input + + columns = [{'Name': 'country', 'Type': 'string'}] + table_input = helpers.create_table_input(database_name, table_name, columns=columns) + helpers.update_table(client, database_name, table_name, table_input) + version_inputs["2"] = table_input + + # Updateing with an indentical input should still create a new version + helpers.update_table(client, database_name, table_name, table_input) + version_inputs["3"] = table_input + + response = helpers.get_table_versions(client, database_name, table_name) + + vers = response['TableVersions'] + + vers.should.have.length_of(3) + vers[0]['Table']['StorageDescriptor']['Columns'].should.equal([]) + vers[-1]['Table']['StorageDescriptor']['Columns'].should.equal(columns) + + for n, ver in enumerate(vers): + n = str(n + 1) + ver['VersionId'].should.equal(n) + ver['Table']['Name'].should.equal(table_name) + ver['Table']['StorageDescriptor'].should.equal(version_inputs[n]['StorageDescriptor']) + ver['Table']['PartitionKeys'].should.equal(version_inputs[n]['PartitionKeys']) + + response = helpers.get_table_version(client, database_name, table_name, "3") + ver = response['TableVersion'] + + ver['VersionId'].should.equal("3") + ver['Table']['Name'].should.equal(table_name) + ver['Table']['StorageDescriptor']['Columns'].should.equal(columns) + + +@mock_glue +def test_get_table_version_not_found(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + + with assert_raises(ClientError) as exc: + helpers.get_table_version(client, database_name, 'myfirsttable', "20") + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + exc.exception.response['Error']['Message'].should.match('version', re.I) + + +@mock_glue +def test_get_table_version_invalid_input(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + + with assert_raises(ClientError) as exc: + helpers.get_table_version(client, database_name, 'myfirsttable', "10not-an-int") + + exc.exception.response['Error']['Code'].should.equal('InvalidInputException') + + +@mock_glue +def test_get_table_not_exits(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + helpers.create_database(client, database_name) + + with assert_raises(ClientError) as exc: + helpers.get_table(client, database_name, 'myfirsttable') + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + exc.exception.response['Error']['Message'].should.match('Table myfirsttable not found') + + +@mock_glue +def test_get_table_when_database_not_exits(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'nosuchdatabase' + + with assert_raises(ClientError) as exc: + helpers.get_table(client, database_name, 'myfirsttable') + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + exc.exception.response['Error']['Message'].should.match('Database nosuchdatabase not found') + + +@mock_glue +def test_get_partitions_empty(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + helpers.create_database(client, database_name) + + helpers.create_table(client, database_name, table_name) + + response = client.get_partitions(DatabaseName=database_name, TableName=table_name) + + response['Partitions'].should.have.length_of(0) + + +@mock_glue +def test_create_partition(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + values = ['2018-10-01'] + helpers.create_database(client, database_name) + + helpers.create_table(client, database_name, table_name) + + before = datetime.now(pytz.utc) + + part_input = helpers.create_partition_input(database_name, table_name, values=values) + helpers.create_partition(client, database_name, table_name, part_input) + + after = datetime.now(pytz.utc) + + response = client.get_partitions(DatabaseName=database_name, TableName=table_name) + + partitions = response['Partitions'] + + partitions.should.have.length_of(1) + + partition = partitions[0] + + partition['TableName'].should.equal(table_name) + partition['StorageDescriptor'].should.equal(part_input['StorageDescriptor']) + partition['Values'].should.equal(values) + partition['CreationTime'].should.be.greater_than(before) + partition['CreationTime'].should.be.lower_than(after) + + +@mock_glue +def test_create_partition_already_exist(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + values = ['2018-10-01'] + helpers.create_database(client, database_name) + + helpers.create_table(client, database_name, table_name) + + helpers.create_partition(client, database_name, table_name, values=values) + + with assert_raises(ClientError) as exc: + helpers.create_partition(client, database_name, table_name, values=values) + + exc.exception.response['Error']['Code'].should.equal('AlreadyExistsException') + + +@mock_glue +def test_get_partition_not_found(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + values = ['2018-10-01'] + helpers.create_database(client, database_name) + + helpers.create_table(client, database_name, table_name) + + with assert_raises(ClientError) as exc: + helpers.get_partition(client, database_name, table_name, values) + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + exc.exception.response['Error']['Message'].should.match('partition') + + +@mock_glue +def test_get_partition(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + helpers.create_database(client, database_name) + + helpers.create_table(client, database_name, table_name) + + values = [['2018-10-01'], ['2018-09-01']] + + helpers.create_partition(client, database_name, table_name, values=values[0]) + helpers.create_partition(client, database_name, table_name, values=values[1]) + + response = client.get_partition(DatabaseName=database_name, TableName=table_name, PartitionValues=values[1]) + + partition = response['Partition'] + + partition['TableName'].should.equal(table_name) + partition['Values'].should.equal(values[1]) + + +@mock_glue +def test_update_partition_not_found_moving(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + + with assert_raises(ClientError) as exc: + helpers.update_partition(client, database_name, table_name, old_values=['0000-00-00'], values=['2018-10-02']) + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + exc.exception.response['Error']['Message'].should.match('partition') + + +@mock_glue +def test_update_partition_not_found_change_in_place(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + values = ['2018-10-01'] + + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + + with assert_raises(ClientError) as exc: + helpers.update_partition(client, database_name, table_name, old_values=values, values=values) + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + exc.exception.response['Error']['Message'].should.match('partition') + + +@mock_glue +def test_update_partition_cannot_overwrite(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + helpers.create_database(client, database_name) + + helpers.create_table(client, database_name, table_name) + + values = [['2018-10-01'], ['2018-09-01']] + + helpers.create_partition(client, database_name, table_name, values=values[0]) + helpers.create_partition(client, database_name, table_name, values=values[1]) + + with assert_raises(ClientError) as exc: + helpers.update_partition(client, database_name, table_name, old_values=values[0], values=values[1]) + + exc.exception.response['Error']['Code'].should.equal('AlreadyExistsException') + + +@mock_glue +def test_update_partition(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + values = ['2018-10-01'] + + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + helpers.create_partition(client, database_name, table_name, values=values) + + response = helpers.update_partition( + client, + database_name, + table_name, + old_values=values, + values=values, + columns=[{'Name': 'country', 'Type': 'string'}], + ) + + response = client.get_partition(DatabaseName=database_name, TableName=table_name, PartitionValues=values) + partition = response['Partition'] + + partition['TableName'].should.equal(table_name) + partition['StorageDescriptor']['Columns'].should.equal([{'Name': 'country', 'Type': 'string'}]) + + +@mock_glue +def test_update_partition_move(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + values = ['2018-10-01'] + new_values = ['2018-09-01'] + + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + helpers.create_partition(client, database_name, table_name, values=values) + + response = helpers.update_partition( + client, + database_name, + table_name, + old_values=values, + values=new_values, + columns=[{'Name': 'country', 'Type': 'string'}], + ) + + with assert_raises(ClientError) as exc: + helpers.get_partition(client, database_name, table_name, values) + + # Old partition shouldn't exist anymore + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + + response = client.get_partition(DatabaseName=database_name, TableName=table_name, PartitionValues=new_values) + partition = response['Partition'] + + partition['TableName'].should.equal(table_name) + partition['StorageDescriptor']['Columns'].should.equal([{'Name': 'country', 'Type': 'string'}]) From 5b3b52752de5b4edca796c33d9dfd46b99c24a82 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Thu, 4 Oct 2018 10:25:16 -0500 Subject: [PATCH 081/106] explicitly check that lifecycle actions are not None when setting lifecycle --- moto/s3/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 39f36982d..bb4d7848c 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -424,14 +424,14 @@ class FakeBucket(BaseModel): transition = rule.get('Transition') nve_noncurrent_days = None - if rule.get('NoncurrentVersionExpiration'): + if rule.get('NoncurrentVersionExpiration') is not None: if rule["NoncurrentVersionExpiration"].get('NoncurrentDays') is None: raise MalformedXML() nve_noncurrent_days = rule["NoncurrentVersionExpiration"]["NoncurrentDays"] nvt_noncurrent_days = None nvt_storage_class = None - if rule.get('NoncurrentVersionTransition'): + if rule.get('NoncurrentVersionTransition') is not None: if rule["NoncurrentVersionTransition"].get('NoncurrentDays') is None: raise MalformedXML() if rule["NoncurrentVersionTransition"].get('StorageClass') is None: @@ -440,7 +440,7 @@ class FakeBucket(BaseModel): nvt_storage_class = rule["NoncurrentVersionTransition"]["StorageClass"] aimu_days = None - if rule.get('AbortIncompleteMultipartUpload'): + if rule.get('AbortIncompleteMultipartUpload') is not None: if rule["AbortIncompleteMultipartUpload"].get('DaysAfterInitiation') is None: raise MalformedXML() aimu_days = rule["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] From 12e0a38b56794888ce9eb572ab9c3dadaef96dba Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 11:04:55 -0500 Subject: [PATCH 082/106] add TODO for testing exceptions with aimu and nve --- tests/test_s3/test_s3_lifecycle.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/test_s3/test_s3_lifecycle.py b/tests/test_s3/test_s3_lifecycle.py index ff89dc113..3d533a641 100644 --- a/tests/test_s3/test_s3_lifecycle.py +++ b/tests/test_s3/test_s3_lifecycle.py @@ -222,11 +222,7 @@ def test_lifecycle_with_nve(): assert len(result["Rules"]) == 1 assert result["Rules"][0]["NoncurrentVersionExpiration"]["NoncurrentDays"] == 10 - # With failures for missing children: - del lfc["Rules"][0]["NoncurrentVersionExpiration"]["NoncurrentDays"] - with assert_raises(ClientError) as err: - client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) - assert err.exception.response["Error"]["Code"] == "MalformedXML" + # TODO: Add test for failures due to missing children @mock_s3 @@ -313,11 +309,7 @@ def test_lifecycle_with_aimu(): assert len(result["Rules"]) == 1 assert result["Rules"][0]["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] == 30 - # With failures for missing children: - del lfc["Rules"][0]["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] - with assert_raises(ClientError) as err: - client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) - assert err.exception.response["Error"]["Code"] == "MalformedXML" + # TODO: Add test for failures due to missing children @mock_s3_deprecated From 60ec840eef44c395e83342469ab81c0ca5a46daf Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 15:55:47 -0500 Subject: [PATCH 083/106] add disable_key, enable_key, cancel_key_deletion, and schedule_key_deletion actions to KMS endpoint --- moto/kms/models.py | 30 +++++++++++++- moto/kms/responses.py | 46 ++++++++++++++++++++++ tests/test_kms/test_kms.py | 80 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 155 insertions(+), 1 deletion(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index 89ebf0082..30a01d366 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -12,11 +12,13 @@ class Key(BaseModel): self.id = generate_key_id() self.policy = policy self.key_usage = key_usage + self.key_state = "Enabled" self.description = description self.enabled = True self.region = region self.account_id = "0123456789012" self.key_rotation_status = False + self.deletion_date = None @property def physical_resource_id(self): @@ -27,7 +29,7 @@ class Key(BaseModel): return "arn:aws:kms:{0}:{1}:key/{2}".format(self.region, self.account_id, self.id) def to_dict(self): - return { + key_dict = { "KeyMetadata": { "AWSAccountId": self.account_id, "Arn": self.arn, @@ -36,8 +38,11 @@ class Key(BaseModel): "Enabled": self.enabled, "KeyId": self.id, "KeyUsage": self.key_usage, + "KeyState": self.key_state, } } + key_dict['KeyMetadata']['DeletionDate'] = self.deletion_date if self.key_state == 'PendingDeletion' + return key_dict def delete(self, region_name): kms_backends[region_name].delete_key(self.id) @@ -138,6 +143,29 @@ class KmsBackend(BaseBackend): def get_key_policy(self, key_id): return self.keys[self.get_key_id(key_id)].policy + def disable_key(self, key_id): + if key_id in self.keys: + self.keys[key_id].enabled = False + self.keys[key_id].key_state = 'Disabled' + + def enable_key(self, key_id): + if key_id in self.keys: + self.keys[key_id].enabled = True + self.keys[key_id].key_state = 'Enabled' + + def cancel_key_deletion(self, key_id): + if key_id in self.keys: + self.keys[key_id].key_state = 'Disabled' + self.keys[key_id].deletion_date = None + + def schedule_key_deletion(self, key_id, pending_window_in_days=30): + if key_id in self.keys: + if 7 <= pending_window_in_days <= 30: + self.keys[key_id].enabled = False + self.keys[key_id].key_state = 'PendingDeletion' + self.keys[key_id].deletion_date = datetime.now() + timedelta(days=pending_window_in_days) + return self.keys[key_id].deletion_date + kms_backends = {} for region in boto.kms.regions(): diff --git a/moto/kms/responses.py b/moto/kms/responses.py index 0f544e954..e782d862e 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -233,6 +233,52 @@ class KmsResponse(BaseResponse): value = self.parameters.get("CiphertextBlob") return json.dumps({"Plaintext": base64.b64decode(value).decode("utf-8")}) + def disable_key(self): + key_id = self.parameters.get('KeyId') + _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) + try: + self.kms_backend.disable_key(key_id) + except KeyError: + raise JSONResponseError(404, 'Not Found', body={ + 'message': "Key 'arn:aws:kms:{region}:012345678912:key/{key_id}' does not exist".format(region=self.region, key_id=key_id), + '__type': 'NotFoundException'}) + return json.dumps(None) + + def enable_key(self): + key_id = self.parameters.get('KeyId') + _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) + try: + self.kms_backend.enable_key(key_id) + except KeyError: + raise JSONResponseError(404, 'Not Found', body={ + 'message': "Key 'arn:aws:kms:{region}:012345678912:key/{key_id}' does not exist".format(region=self.region, key_id=key_id), + '__type': 'NotFoundException'}) + return json.dumps(None) + + def cancel_key_deletion(self): + key_id = self.parameters.get('KeyId') + _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) + try: + self.kms_backend.cancel_key_deletion(key_id) + except KeyError: + raise JSONResponseError(404, 'Not Found', body={ + 'message': "Key 'arn:aws:kms:{region}:012345678912:key/{key_id}' does not exist".format(region=self.region, key_id=key_id), + '__type': 'NotFoundException'}) + return json.dumps({'KeyId': key_id}) + + def schedule_key_deletion(self): + key_id = self.parameters.get('KeyId') + _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) + try: + return json.dumps({ + 'KeyId': key_id, + 'DeletionDate': self.kms_backend.schedule_key_deletion(key_id) + }) + except KeyError: + raise JSONResponseError(404, 'Not Found', body={ + 'message': "Key 'arn:aws:kms:{region}:012345678912:key/{key_id}' does not exist".format(region=self.region, key_id=key_id), + '__type': 'NotFoundException'}) + def _assert_valid_key_id(key_id): if not re.match(r'^[A-F0-9]{8}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{12}$', key_id, re.IGNORECASE): diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 96715de71..9779f02a6 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -617,3 +617,83 @@ def test_kms_encrypt_boto3(): response = client.decrypt(CiphertextBlob=response['CiphertextBlob']) response['Plaintext'].should.equal(b'bar') + + +@mock_kms +def test_disable_key(): + client = boto3.client('kms', region_name='us-east-1') + key = client.create_key(description='disable-key') + client.disable_key( + KeyId=key['KeyMetadata']['KeyId'] + ) + + result = client.describe_key(KeyId='disable-key') + assert result["KeyMetadata"]["Enabled"] == False + assert result["KeyMetadata"]["KeyState"] == 'Disabled' + + +@mock_kms +def test_enable_key(): + client = boto3.client('kms', region_name='us-east-1') + key = client.create_key(description='enable-key') + client.disable_key( + KeyId=key['KeyMetadata']['KeyId'] + ) + client.enable_key( + KeyId=key['KeyMetadata']['KeyId'] + ) + + result = client.describe_key(KeyId='enable-key') + assert result["KeyMetadata"]["Enabled"] == True + assert result["KeyMetadata"]["KeyState"] == 'Enabled' + + +@mock_kms +def test_schedule_key_deletion(): + client = boto3.client('kms', region_name='us-east-1') + key = client.create_key(description='schedule-key-deletion') + response = client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'] + ) + assert response['KeyId'] == 'schedule-key-deletion' + assert response['DeletionDate'] == datetime.now() + timedelta(days=30) + + result = client.describe_key(KeyId='schedule-key-deletion') + assert result["KeyMetadata"]["Enabled"] == False + assert result["KeyMetadata"]["KeyState"] == 'PendingDeletion' + assert 'DeletionDate' in result["KeyMetadata"] + + +@mock_kms +def test_schedule_key_deletion_custom(): + client = boto3.client('kms', region_name='us-east-1') + key = client.create_key(description='schedule-key-deletion') + response = client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'], + PendingWindowInDays=7 + ) + assert response['KeyId'] == 'schedule-key-deletion' + assert response['DeletionDate'] == datetime.now() + timedelta(days=7) + + result = client.describe_key(KeyId='schedule-key-deletion') + assert result["KeyMetadata"]["Enabled"] == False + assert result["KeyMetadata"]["KeyState"] == 'PendingDeletion' + assert 'DeletionDate' in result["KeyMetadata"] + + +@mock_kms +def test_cancel_key_deletion(): + client = boto3.client('kms', region_name='us-east-1') + key = client.create_key(description='cancel-key-deletion') + client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'] + ) + response = client.cancel_key_deletion( + KeyId=key['KeyMetadata']['KeyId'] + ) + assert response['KeyId'] == 'cancel-key-deletion' + + result = client.describe_key(KeyId='cancel-key-deletion') + assert result["KeyMetadata"]["Enabled"] == False + assert result["KeyMetadata"]["KeyState"] == 'Disabled' + assert 'DeletionDate' not in result["KeyMetadata"] From 15c24e49f0633b7cbe09378dfe22dc94e9190e1f Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 16:00:20 -0500 Subject: [PATCH 084/106] fix formatting for including DeletionDate in response --- moto/kms/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index 30a01d366..4ac6ee845 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -41,7 +41,8 @@ class Key(BaseModel): "KeyState": self.key_state, } } - key_dict['KeyMetadata']['DeletionDate'] = self.deletion_date if self.key_state == 'PendingDeletion' + if self.key_state == 'PendingDeletion': + key_dict['KeyMetadata']['DeletionDate'] = self.deletion_date return key_dict def delete(self, region_name): From 7e96203020ea4cd873e0a585884b156896f49265 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 16:21:16 -0500 Subject: [PATCH 085/106] add freezegun and test DeletionDate for chedule_key_deletion --- moto/kms/models.py | 1 + tests/test_kms/test_kms.py | 26 +++++++++++++++----------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index 4ac6ee845..113bd1733 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -4,6 +4,7 @@ import boto.kms from moto.core import BaseBackend, BaseModel from .utils import generate_key_id from collections import defaultdict +from datetime import datetime, timedelta class Key(BaseModel): diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 9779f02a6..287ef9158 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -8,6 +8,8 @@ from boto.kms.exceptions import AlreadyExistsException, NotFoundException import sure # noqa from moto import mock_kms, mock_kms_deprecated from nose.tools import assert_raises +from freezegun import freeze_time +from datetime import datetime, timedelta @mock_kms_deprecated @@ -652,11 +654,12 @@ def test_enable_key(): def test_schedule_key_deletion(): client = boto3.client('kms', region_name='us-east-1') key = client.create_key(description='schedule-key-deletion') - response = client.schedule_key_deletion( - KeyId=key['KeyMetadata']['KeyId'] - ) - assert response['KeyId'] == 'schedule-key-deletion' - assert response['DeletionDate'] == datetime.now() + timedelta(days=30) + with freeze_time("2015-01-01 12:00:00"): + response = client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'] + ) + assert response['KeyId'] == 'schedule-key-deletion' + assert response['DeletionDate'] == datetime.now() + timedelta(days=30) result = client.describe_key(KeyId='schedule-key-deletion') assert result["KeyMetadata"]["Enabled"] == False @@ -668,12 +671,13 @@ def test_schedule_key_deletion(): def test_schedule_key_deletion_custom(): client = boto3.client('kms', region_name='us-east-1') key = client.create_key(description='schedule-key-deletion') - response = client.schedule_key_deletion( - KeyId=key['KeyMetadata']['KeyId'], - PendingWindowInDays=7 - ) - assert response['KeyId'] == 'schedule-key-deletion' - assert response['DeletionDate'] == datetime.now() + timedelta(days=7) + with freeze_time("2015-01-01 12:00:00"): + response = client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'], + PendingWindowInDays=7 + ) + assert response['KeyId'] == 'schedule-key-deletion' + assert response['DeletionDate'] == datetime.now() + timedelta(days=7) result = client.describe_key(KeyId='schedule-key-deletion') assert result["KeyMetadata"]["Enabled"] == False From 695b4349ba62865cd3e9987af65c39a761f2c35b Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 21:43:12 -0500 Subject: [PATCH 086/106] indentation fix --- moto/kms/responses.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/kms/responses.py b/moto/kms/responses.py index e782d862e..dc37c8b59 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -271,8 +271,8 @@ class KmsResponse(BaseResponse): _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) try: return json.dumps({ - 'KeyId': key_id, - 'DeletionDate': self.kms_backend.schedule_key_deletion(key_id) + 'KeyId': key_id, + 'DeletionDate': self.kms_backend.schedule_key_deletion(key_id) }) except KeyError: raise JSONResponseError(404, 'Not Found', body={ From a29daf411baf056f3fc924abae7e7bcbe224f5cc Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 21:56:32 -0500 Subject: [PATCH 087/106] fix invalid variables used in kms testing --- tests/test_kms/test_kms.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 287ef9158..ab7513b4d 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -623,8 +623,8 @@ def test_kms_encrypt_boto3(): @mock_kms def test_disable_key(): - client = boto3.client('kms', region_name='us-east-1') - key = client.create_key(description='disable-key') + client = boto3.client('kms') + key = client.create_key(Description='disable-key') client.disable_key( KeyId=key['KeyMetadata']['KeyId'] ) @@ -636,8 +636,8 @@ def test_disable_key(): @mock_kms def test_enable_key(): - client = boto3.client('kms', region_name='us-east-1') - key = client.create_key(description='enable-key') + client = boto3.client('kms') + key = client.create_key(Description='enable-key') client.disable_key( KeyId=key['KeyMetadata']['KeyId'] ) @@ -652,8 +652,8 @@ def test_enable_key(): @mock_kms def test_schedule_key_deletion(): - client = boto3.client('kms', region_name='us-east-1') - key = client.create_key(description='schedule-key-deletion') + client = boto3.client('kms') + key = client.create_key(Description='schedule-key-deletion') with freeze_time("2015-01-01 12:00:00"): response = client.schedule_key_deletion( KeyId=key['KeyMetadata']['KeyId'] @@ -669,8 +669,8 @@ def test_schedule_key_deletion(): @mock_kms def test_schedule_key_deletion_custom(): - client = boto3.client('kms', region_name='us-east-1') - key = client.create_key(description='schedule-key-deletion') + client = boto3.client('kms') + key = client.create_key(Description='schedule-key-deletion') with freeze_time("2015-01-01 12:00:00"): response = client.schedule_key_deletion( KeyId=key['KeyMetadata']['KeyId'], @@ -687,8 +687,8 @@ def test_schedule_key_deletion_custom(): @mock_kms def test_cancel_key_deletion(): - client = boto3.client('kms', region_name='us-east-1') - key = client.create_key(description='cancel-key-deletion') + client = boto3.client('kms') + key = client.create_key(Description='cancel-key-deletion') client.schedule_key_deletion( KeyId=key['KeyMetadata']['KeyId'] ) From 786b9ca519f830424bdddbcb69e0322f40c0d9b0 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 22:17:48 -0500 Subject: [PATCH 088/106] need region for kms client --- tests/test_kms/test_kms.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index ab7513b4d..0dc93b252 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -623,7 +623,7 @@ def test_kms_encrypt_boto3(): @mock_kms def test_disable_key(): - client = boto3.client('kms') + client = boto3.client('kms', region_name='us-east-1') key = client.create_key(Description='disable-key') client.disable_key( KeyId=key['KeyMetadata']['KeyId'] @@ -636,7 +636,7 @@ def test_disable_key(): @mock_kms def test_enable_key(): - client = boto3.client('kms') + client = boto3.client('kms', region_name='us-east-1') key = client.create_key(Description='enable-key') client.disable_key( KeyId=key['KeyMetadata']['KeyId'] @@ -652,7 +652,7 @@ def test_enable_key(): @mock_kms def test_schedule_key_deletion(): - client = boto3.client('kms') + client = boto3.client('kms', region_name='us-east-1') key = client.create_key(Description='schedule-key-deletion') with freeze_time("2015-01-01 12:00:00"): response = client.schedule_key_deletion( @@ -669,7 +669,7 @@ def test_schedule_key_deletion(): @mock_kms def test_schedule_key_deletion_custom(): - client = boto3.client('kms') + client = boto3.client('kms', region_name='us-east-1') key = client.create_key(Description='schedule-key-deletion') with freeze_time("2015-01-01 12:00:00"): response = client.schedule_key_deletion( @@ -687,7 +687,7 @@ def test_schedule_key_deletion_custom(): @mock_kms def test_cancel_key_deletion(): - client = boto3.client('kms') + client = boto3.client('kms', region_name='us-east-1') key = client.create_key(Description='cancel-key-deletion') client.schedule_key_deletion( KeyId=key['KeyMetadata']['KeyId'] From 372f749831344f979d349238e15585f68dabc5ca Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 22:46:19 -0500 Subject: [PATCH 089/106] format DeletionDate properly for JSON serialization --- moto/kms/responses.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moto/kms/responses.py b/moto/kms/responses.py index dc37c8b59..fe7d28523 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -9,6 +9,7 @@ from boto.exception import JSONResponseError from boto.kms.exceptions import AlreadyExistsException, NotFoundException from moto.core.responses import BaseResponse +from moto.core.utils import iso_8601_datetime_without_milliseconds from .models import kms_backends reserved_aliases = [ @@ -272,7 +273,7 @@ class KmsResponse(BaseResponse): try: return json.dumps({ 'KeyId': key_id, - 'DeletionDate': self.kms_backend.schedule_key_deletion(key_id) + 'DeletionDate': iso_8601_datetime_without_milliseconds(self.kms_backend.schedule_key_deletion(key_id)) }) except KeyError: raise JSONResponseError(404, 'Not Found', body={ From f596069dabd1852f67f3a8a63e0e01029089cb43 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 23:35:34 -0500 Subject: [PATCH 090/106] use initial KeyMetadata for identifying keys in KMS tests --- tests/test_kms/test_kms.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 0dc93b252..69cd508ba 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -658,10 +658,10 @@ def test_schedule_key_deletion(): response = client.schedule_key_deletion( KeyId=key['KeyMetadata']['KeyId'] ) - assert response['KeyId'] == 'schedule-key-deletion' + assert response['KeyId'] == key['KeyMetadata']['KeyId'] assert response['DeletionDate'] == datetime.now() + timedelta(days=30) - result = client.describe_key(KeyId='schedule-key-deletion') + result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False assert result["KeyMetadata"]["KeyState"] == 'PendingDeletion' assert 'DeletionDate' in result["KeyMetadata"] @@ -676,10 +676,10 @@ def test_schedule_key_deletion_custom(): KeyId=key['KeyMetadata']['KeyId'], PendingWindowInDays=7 ) - assert response['KeyId'] == 'schedule-key-deletion' + assert response['KeyId'] == key['KeyMetadata']['KeyId'] assert response['DeletionDate'] == datetime.now() + timedelta(days=7) - result = client.describe_key(KeyId='schedule-key-deletion') + result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False assert result["KeyMetadata"]["KeyState"] == 'PendingDeletion' assert 'DeletionDate' in result["KeyMetadata"] @@ -695,9 +695,9 @@ def test_cancel_key_deletion(): response = client.cancel_key_deletion( KeyId=key['KeyMetadata']['KeyId'] ) - assert response['KeyId'] == 'cancel-key-deletion' + assert response['KeyId'] == key['KeyMetadata']['KeyId'] - result = client.describe_key(KeyId='cancel-key-deletion') + result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False assert result["KeyMetadata"]["KeyState"] == 'Disabled' assert 'DeletionDate' not in result["KeyMetadata"] From 6277983e3f01daa19247e0f217941fd2d3ad4b1d Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 23:48:19 -0500 Subject: [PATCH 091/106] missed some KeyMetadata and need to transform datetime for testing --- tests/test_kms/test_kms.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 69cd508ba..a06a07324 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -7,6 +7,7 @@ from boto.exception import JSONResponseError from boto.kms.exceptions import AlreadyExistsException, NotFoundException import sure # noqa from moto import mock_kms, mock_kms_deprecated +from moto.core.utils import iso_8601_datetime_without_milliseconds from nose.tools import assert_raises from freezegun import freeze_time from datetime import datetime, timedelta @@ -629,7 +630,7 @@ def test_disable_key(): KeyId=key['KeyMetadata']['KeyId'] ) - result = client.describe_key(KeyId='disable-key') + result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False assert result["KeyMetadata"]["KeyState"] == 'Disabled' @@ -645,7 +646,7 @@ def test_enable_key(): KeyId=key['KeyMetadata']['KeyId'] ) - result = client.describe_key(KeyId='enable-key') + result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == True assert result["KeyMetadata"]["KeyState"] == 'Enabled' @@ -659,7 +660,7 @@ def test_schedule_key_deletion(): KeyId=key['KeyMetadata']['KeyId'] ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == datetime.now() + timedelta(days=30) + assert response['DeletionDate'] == iso_8601_datetime_without_milliseconds(datetime.now() + timedelta(days=30)) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False @@ -677,7 +678,7 @@ def test_schedule_key_deletion_custom(): PendingWindowInDays=7 ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == datetime.now() + timedelta(days=7) + assert response['DeletionDate'] == iso_8601_datetime_without_milliseconds(datetime.now() + timedelta(days=7)) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False From 21c8914efe65465ef90351dfa13441bf74e67427 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Sat, 6 Oct 2018 00:13:47 -0500 Subject: [PATCH 092/106] include pending days input for schedule key deletion and update tests since boto client returns DeletionDate as datetime --- moto/kms/responses.py | 3 ++- tests/test_kms/test_kms.py | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/moto/kms/responses.py b/moto/kms/responses.py index fe7d28523..831045e3c 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -269,11 +269,12 @@ class KmsResponse(BaseResponse): def schedule_key_deletion(self): key_id = self.parameters.get('KeyId') + pending_window_in_days = self.parameters.get('PendingWindowInDays') _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) try: return json.dumps({ 'KeyId': key_id, - 'DeletionDate': iso_8601_datetime_without_milliseconds(self.kms_backend.schedule_key_deletion(key_id)) + 'DeletionDate': iso_8601_datetime_without_milliseconds(self.kms_backend.schedule_key_deletion(key_id, pending_window_in_days)) }) except KeyError: raise JSONResponseError(404, 'Not Found', body={ diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index a06a07324..218df1d8f 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -7,7 +7,6 @@ from boto.exception import JSONResponseError from boto.kms.exceptions import AlreadyExistsException, NotFoundException import sure # noqa from moto import mock_kms, mock_kms_deprecated -from moto.core.utils import iso_8601_datetime_without_milliseconds from nose.tools import assert_raises from freezegun import freeze_time from datetime import datetime, timedelta @@ -660,7 +659,7 @@ def test_schedule_key_deletion(): KeyId=key['KeyMetadata']['KeyId'] ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == iso_8601_datetime_without_milliseconds(datetime.now() + timedelta(days=30)) + assert response['DeletionDate'] == datetime.now() + timedelta(days=30) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False @@ -678,7 +677,7 @@ def test_schedule_key_deletion_custom(): PendingWindowInDays=7 ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == iso_8601_datetime_without_milliseconds(datetime.now() + timedelta(days=7)) + assert response['DeletionDate'] == datetime.now() + timedelta(days=7) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False From 59c233f43158d453fd9f2392a8db4d451e34a46d Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Sat, 6 Oct 2018 00:33:23 -0500 Subject: [PATCH 093/106] avoid needing to import datetime and dealing with timezone vs naive datetimes in tests --- tests/test_kms/test_kms.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 218df1d8f..b09459e1d 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -9,7 +9,6 @@ import sure # noqa from moto import mock_kms, mock_kms_deprecated from nose.tools import assert_raises from freezegun import freeze_time -from datetime import datetime, timedelta @mock_kms_deprecated @@ -659,7 +658,7 @@ def test_schedule_key_deletion(): KeyId=key['KeyMetadata']['KeyId'] ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == datetime.now() + timedelta(days=30) + assert response['DeletionDate'] == '2015-01-31T12:00:00.000Z' result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False @@ -677,7 +676,7 @@ def test_schedule_key_deletion_custom(): PendingWindowInDays=7 ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == datetime.now() + timedelta(days=7) + assert response['DeletionDate'] == '2015-01-08T12:00:00.000Z' result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False From 9b25d56a3585fcc5a90a8e606b63cacf170d4351 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Sat, 6 Oct 2018 01:18:26 -0500 Subject: [PATCH 094/106] need datetime for tests since thats what boto3 returns and add default for PendingWindowInDays --- moto/kms/models.py | 2 +- moto/kms/responses.py | 5 ++++- tests/test_kms/test_kms.py | 5 +++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index 113bd1733..01b3d9711 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -160,7 +160,7 @@ class KmsBackend(BaseBackend): self.keys[key_id].key_state = 'Disabled' self.keys[key_id].deletion_date = None - def schedule_key_deletion(self, key_id, pending_window_in_days=30): + def schedule_key_deletion(self, key_id, pending_window_in_days): if key_id in self.keys: if 7 <= pending_window_in_days <= 30: self.keys[key_id].enabled = False diff --git a/moto/kms/responses.py b/moto/kms/responses.py index 831045e3c..7caeafcaa 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -269,7 +269,10 @@ class KmsResponse(BaseResponse): def schedule_key_deletion(self): key_id = self.parameters.get('KeyId') - pending_window_in_days = self.parameters.get('PendingWindowInDays') + if self.parameters.get('PendingWindowInDays') is None: + pending_window_in_days = 30 + else: + pending_window_in_days = self.parameters.get('PendingWindowInDays') _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) try: return json.dumps({ diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index b09459e1d..e16bd2cea 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -9,6 +9,7 @@ import sure # noqa from moto import mock_kms, mock_kms_deprecated from nose.tools import assert_raises from freezegun import freeze_time +from datetime import datetime, timedelta @mock_kms_deprecated @@ -658,7 +659,7 @@ def test_schedule_key_deletion(): KeyId=key['KeyMetadata']['KeyId'] ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == '2015-01-31T12:00:00.000Z' + assert response['DeletionDate'] == datetime(2015, 1, 31, 12, 0, tzinfo=tzlocal()) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False @@ -676,7 +677,7 @@ def test_schedule_key_deletion_custom(): PendingWindowInDays=7 ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == '2015-01-08T12:00:00.000Z' + assert response['DeletionDate'] == datetime(2015, 1, 8, 12, 0, tzinfo=tzlocal()) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False From 76baab74ad6a1c4ebe8d3037bf202ca473187875 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Sat, 6 Oct 2018 01:33:02 -0500 Subject: [PATCH 095/106] missing tzlocal --- tests/test_kms/test_kms.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index e16bd2cea..159f45af4 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -10,6 +10,7 @@ from moto import mock_kms, mock_kms_deprecated from nose.tools import assert_raises from freezegun import freeze_time from datetime import datetime, timedelta +from dateutil.tz import tzlocal @mock_kms_deprecated From 398dcd8230d8a2e235527bb72a0766b02d3cd6fb Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Sat, 6 Oct 2018 01:47:22 -0500 Subject: [PATCH 096/106] transform DeletionDate in model instead to accomodate Key.to_dict --- moto/kms/models.py | 5 +++-- moto/kms/responses.py | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index 01b3d9711..bb39d1b24 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import boto.kms from moto.core import BaseBackend, BaseModel +from moto.core.utils import iso_8601_datetime_without_milliseconds from .utils import generate_key_id from collections import defaultdict from datetime import datetime, timedelta @@ -43,7 +44,7 @@ class Key(BaseModel): } } if self.key_state == 'PendingDeletion': - key_dict['KeyMetadata']['DeletionDate'] = self.deletion_date + key_dict['KeyMetadata']['DeletionDate'] = iso_8601_datetime_without_milliseconds(self.deletion_date) return key_dict def delete(self, region_name): @@ -166,7 +167,7 @@ class KmsBackend(BaseBackend): self.keys[key_id].enabled = False self.keys[key_id].key_state = 'PendingDeletion' self.keys[key_id].deletion_date = datetime.now() + timedelta(days=pending_window_in_days) - return self.keys[key_id].deletion_date + return iso_8601_datetime_without_milliseconds(self.keys[key_id].deletion_date) kms_backends = {} diff --git a/moto/kms/responses.py b/moto/kms/responses.py index 7caeafcaa..5883f51ec 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -9,7 +9,6 @@ from boto.exception import JSONResponseError from boto.kms.exceptions import AlreadyExistsException, NotFoundException from moto.core.responses import BaseResponse -from moto.core.utils import iso_8601_datetime_without_milliseconds from .models import kms_backends reserved_aliases = [ @@ -277,7 +276,7 @@ class KmsResponse(BaseResponse): try: return json.dumps({ 'KeyId': key_id, - 'DeletionDate': iso_8601_datetime_without_milliseconds(self.kms_backend.schedule_key_deletion(key_id, pending_window_in_days)) + 'DeletionDate': self.kms_backend.schedule_key_deletion(key_id, pending_window_in_days) }) except KeyError: raise JSONResponseError(404, 'Not Found', body={ From c2595b2eef9c147ccc03d2ce7ce2b59b6cec4d35 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Mon, 8 Oct 2018 08:29:21 -0500 Subject: [PATCH 097/106] cant manipulate time in server mode tests --- tests/test_kms/test_kms.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 159f45af4..4f8503ceb 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -655,12 +655,19 @@ def test_enable_key(): def test_schedule_key_deletion(): client = boto3.client('kms', region_name='us-east-1') key = client.create_key(Description='schedule-key-deletion') - with freeze_time("2015-01-01 12:00:00"): + if os.environ.get('TEST_SERVER_MODE', 'false').lower() == 'false': + with freeze_time("2015-01-01 12:00:00"): + response = client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'] + ) + assert response['KeyId'] == key['KeyMetadata']['KeyId'] + assert response['DeletionDate'] == datetime(2015, 1, 31, 12, 0, tzinfo=tzlocal()) + else: + # Can't manipulate time in server mode response = client.schedule_key_deletion( KeyId=key['KeyMetadata']['KeyId'] ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == datetime(2015, 1, 31, 12, 0, tzinfo=tzlocal()) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False @@ -672,13 +679,21 @@ def test_schedule_key_deletion(): def test_schedule_key_deletion_custom(): client = boto3.client('kms', region_name='us-east-1') key = client.create_key(Description='schedule-key-deletion') - with freeze_time("2015-01-01 12:00:00"): + if os.environ.get('TEST_SERVER_MODE', 'false').lower() == 'false': + with freeze_time("2015-01-01 12:00:00"): + response = client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'], + PendingWindowInDays=7 + ) + assert response['KeyId'] == key['KeyMetadata']['KeyId'] + assert response['DeletionDate'] == datetime(2015, 1, 8, 12, 0, tzinfo=tzlocal()) + else: + # Can't manipulate time in server mode response = client.schedule_key_deletion( KeyId=key['KeyMetadata']['KeyId'], PendingWindowInDays=7 ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == datetime(2015, 1, 8, 12, 0, tzinfo=tzlocal()) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False From 181e9690b84db61f180d31f30ea3e92776cb9a49 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Mon, 8 Oct 2018 08:38:49 -0500 Subject: [PATCH 098/106] need os for checking server mode env variable --- tests/test_kms/test_kms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 4f8503ceb..8bccae27a 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -1,5 +1,5 @@ from __future__ import unicode_literals -import re +import os, re import boto3 import boto.kms From c1ebec1b352cf29b9ba666019954b6667757d738 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Mon, 8 Oct 2018 10:17:51 -0500 Subject: [PATCH 099/106] remove start_time from attrib comparison in test_copy_snapshot() --- tests/test_ec2/test_elastic_block_store.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_ec2/test_elastic_block_store.py b/tests/test_ec2/test_elastic_block_store.py index 8930838c6..442e41dde 100644 --- a/tests/test_ec2/test_elastic_block_store.py +++ b/tests/test_ec2/test_elastic_block_store.py @@ -615,8 +615,8 @@ def test_copy_snapshot(): dest = dest_ec2.Snapshot(copy_snapshot_response['SnapshotId']) attribs = ['data_encryption_key_id', 'encrypted', - 'kms_key_id', 'owner_alias', 'owner_id', 'progress', - 'start_time', 'state', 'state_message', + 'kms_key_id', 'owner_alias', 'owner_id', + 'progress', 'state', 'state_message', 'tags', 'volume_id', 'volume_size'] for attrib in attribs: From d9577f9d3d8c7e4aa32bdb346a7d8a5c4a55d2f4 Mon Sep 17 00:00:00 2001 From: George Alton Date: Mon, 8 Oct 2018 19:04:47 +0100 Subject: [PATCH 100/106] Ensures a UserPool Id starts like {region}_ --- moto/cognitoidp/models.py | 2 +- tests/test_cognitoidp/test_cognitoidp.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 52a73f89f..db54ded7c 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -24,7 +24,7 @@ class CognitoIdpUserPool(BaseModel): def __init__(self, region, name, extended_config): self.region = region - self.id = str(uuid.uuid4()) + self.id = "{}_{}".format(self.region, str(uuid.uuid4().hex)) self.name = name self.status = None self.extended_config = extended_config or {} diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index b2bd469ce..76e86a691 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -24,6 +24,7 @@ def test_create_user_pool(): ) result["UserPool"]["Id"].should_not.be.none + result["UserPool"]["Id"].should.match(r'[\w-]+_[0-9a-zA-Z]+') result["UserPool"]["Name"].should.equal(name) result["UserPool"]["LambdaConfig"]["PreSignUp"].should.equal(value) From 7a88e634eb0e7780a07645faad5545ce2689531d Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Mon, 8 Oct 2018 15:27:19 -0700 Subject: [PATCH 101/106] organizations: add exception test for describe_organization endpoint --- tests/test_organizations/test_organizations_boto3.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index ae9bacd82..dfac5feeb 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -32,6 +32,17 @@ def test_describe_organization(): validate_organization(response) +@mock_organizations +def test_describe_organization_exception(): + client = boto3.client('organizations', region_name='us-east-1') + with assert_raises(ClientError) as e: + response = client.describe_organization() + ex = e.exception + ex.operation_name.should.equal('DescribeOrganization') + ex.response['Error']['Code'].should.equal('400') + ex.response['Error']['Message'].should.contain('AWSOrganizationsNotInUseException') + + # Organizational Units @mock_organizations From 9081a160d3df9d36c27af5efc87d81457bd74e4f Mon Sep 17 00:00:00 2001 From: Danny Cosson Date: Tue, 9 Oct 2018 10:28:15 -0700 Subject: [PATCH 102/106] fixes for cognito identity library --- moto/cognitoidentity/responses.py | 5 ++++- moto/cognitoidentity/utils.py | 2 +- tests/test_cognitoidentity/test_cognitoidentity.py | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/moto/cognitoidentity/responses.py b/moto/cognitoidentity/responses.py index ea54b2cff..e7b428329 100644 --- a/moto/cognitoidentity/responses.py +++ b/moto/cognitoidentity/responses.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from moto.core.responses import BaseResponse from .models import cognitoidentity_backends +from .utils import get_random_identity_id class CognitoIdentityResponse(BaseResponse): @@ -31,4 +32,6 @@ class CognitoIdentityResponse(BaseResponse): return cognitoidentity_backends[self.region].get_credentials_for_identity(self._get_param('IdentityId')) def get_open_id_token_for_developer_identity(self): - return cognitoidentity_backends[self.region].get_open_id_token_for_developer_identity(self._get_param('IdentityId')) + return cognitoidentity_backends[self.region].get_open_id_token_for_developer_identity( + self._get_param('IdentityId') or get_random_identity_id(self.region) + ) diff --git a/moto/cognitoidentity/utils.py b/moto/cognitoidentity/utils.py index 359631763..6143d5121 100644 --- a/moto/cognitoidentity/utils.py +++ b/moto/cognitoidentity/utils.py @@ -2,4 +2,4 @@ from moto.core.utils import get_random_hex def get_random_identity_id(region): - return "{0}:{0}".format(region, get_random_hex(length=19)) + return "{0}:{1}".format(region, get_random_hex(length=19)) diff --git a/tests/test_cognitoidentity/test_cognitoidentity.py b/tests/test_cognitoidentity/test_cognitoidentity.py index a38107b99..ac79fa223 100644 --- a/tests/test_cognitoidentity/test_cognitoidentity.py +++ b/tests/test_cognitoidentity/test_cognitoidentity.py @@ -31,6 +31,7 @@ def test_create_identity_pool(): # testing a helper function def test_get_random_identity_id(): assert len(get_random_identity_id('us-west-2')) > 0 + assert len(get_random_identity_id('us-west-2').split(':')[1]) == 19 @mock_cognitoidentity @@ -69,3 +70,16 @@ def test_get_open_id_token_for_developer_identity(): ) assert len(result['Token']) assert result['IdentityId'] == '12345' + +@mock_cognitoidentity +def test_get_open_id_token_for_developer_identity_when_no_explicit_identity_id(): + conn = boto3.client('cognito-identity', 'us-west-2') + result = conn.get_open_id_token_for_developer_identity( + IdentityPoolId='us-west-2:12345', + Logins={ + 'someurl': '12345' + }, + TokenDuration=123 + ) + assert len(result['Token']) > 0 + assert len(result['IdentityId']) > 0 From 2c15d71c2c872d2d452e16b61d5faf613073338b Mon Sep 17 00:00:00 2001 From: Kosei Kitahara Date: Thu, 11 Oct 2018 18:21:53 +0900 Subject: [PATCH 103/106] Allow spaces to if_not_exists --- moto/dynamodb2/models.py | 2 +- tests/test_dynamodb2/test_dynamodb.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index b327c7a4b..0fa96f3b6 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -154,7 +154,7 @@ class Item(BaseModel): # If not exists, changes value to a default if needed, else its the same as it was if value.startswith('if_not_exists'): # Function signature - match = re.match(r'.*if_not_exists\((?P.+),\s*(?P.+)\).*', value) + match = re.match(r'.*if_not_exists\s*\((?P.+),\s*(?P.+)\).*', value) if not match: raise TypeError diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 243de2701..70feaf7f6 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1220,7 +1220,8 @@ def test_update_if_not_exists(): 'forum_name': 'the-key', 'subject': '123' }, - UpdateExpression='SET created_at = if_not_exists(created_at, :created_at)', + # if_not_exists without space + UpdateExpression='SET created_at=if_not_exists(created_at,:created_at)', ExpressionAttributeValues={ ':created_at': 123 } @@ -1233,7 +1234,8 @@ def test_update_if_not_exists(): 'forum_name': 'the-key', 'subject': '123' }, - UpdateExpression='SET created_at = if_not_exists(created_at, :created_at)', + # if_not_exists with space + UpdateExpression='SET created_at = if_not_exists (created_at, :created_at)', ExpressionAttributeValues={ ':created_at': 456 } From cf157287e707117447ccb637d8ffbf642eaad240 Mon Sep 17 00:00:00 2001 From: Kosei Kitahara Date: Fri, 12 Oct 2018 16:08:05 +0900 Subject: [PATCH 104/106] Fix wrong type if exists --- moto/dynamodb2/models.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index b327c7a4b..02b2933f2 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -162,12 +162,13 @@ class Item(BaseModel): # If it already exists, get its value so we dont overwrite it if path in self.attrs: - value = self.attrs[path].cast_value + value = self.attrs[path] - if value in expression_attribute_values: - value = DynamoType(expression_attribute_values[value]) - else: - value = DynamoType({"S": value}) + if type(value) != DynamoType: + if value in expression_attribute_values: + value = DynamoType(expression_attribute_values[value]) + else: + value = DynamoType({"S": value}) if '.' not in key: self.attrs[key] = value From 13c2e699328eebca5ae8d92797a9fc031b461752 Mon Sep 17 00:00:00 2001 From: Kosei Kitahara Date: Fri, 12 Oct 2018 16:59:52 +0900 Subject: [PATCH 105/106] Allow extra spaces to attribute_exists and attribute_not_exists too --- moto/dynamodb2/responses.py | 8 ++++---- tests/test_dynamodb2/test_dynamodb.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 493e17833..e2f1ef1cc 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -204,9 +204,9 @@ class DynamoHandler(BaseResponse): if cond_items: expected = {} overwrite = False - exists_re = re.compile('^attribute_exists\((.*)\)$') + exists_re = re.compile('^attribute_exists\s*\((.*)\)$') not_exists_re = re.compile( - '^attribute_not_exists\((.*)\)$') + '^attribute_not_exists\s*\((.*)\)$') for cond in cond_items: exists_m = exists_re.match(cond) @@ -556,9 +556,9 @@ class DynamoHandler(BaseResponse): if cond_items: expected = {} - exists_re = re.compile('^attribute_exists\((.*)\)$') + exists_re = re.compile('^attribute_exists\s*\((.*)\)$') not_exists_re = re.compile( - '^attribute_not_exists\((.*)\)$') + '^attribute_not_exists\s*\((.*)\)$') for cond in cond_items: exists_m = exists_re.match(cond) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 70feaf7f6..afc919dd7 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -700,8 +700,8 @@ def test_filter_expression(): filter_expr = moto.dynamodb2.comparisons.get_filter_expression('Id IN :v0', {}, {':v0': {'NS': [7, 8, 9]}}) filter_expr.expr(row1).should.be(True) - # attribute function tests - filter_expr = moto.dynamodb2.comparisons.get_filter_expression('attribute_exists(Id) AND attribute_not_exists(User)', {}, {}) + # attribute function tests (with extra spaces) + filter_expr = moto.dynamodb2.comparisons.get_filter_expression('attribute_exists(Id) AND attribute_not_exists (User)', {}, {}) filter_expr.expr(row1).should.be(True) filter_expr = moto.dynamodb2.comparisons.get_filter_expression('attribute_type(Id, N)', {}, {}) From 27ca96519b5517fa51b954972ee95c46f77d8782 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Mon, 15 Oct 2018 01:37:25 -0400 Subject: [PATCH 106/106] Fix extra whitespace in s3. Closes #1844. --- moto/s3/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index de101a193..962025cb1 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -1449,7 +1449,7 @@ S3_MULTIPART_LIST_RESPONSE = """ STANDARD 1 - {{ count }} + {{ count }} {{ count }} false {% for part in parts %}