From 0b15bb13b6c4173bde85e8592689839f6f1c1154 Mon Sep 17 00:00:00 2001 From: Gary Donovan Date: Thu, 10 Jan 2019 21:39:12 +1100 Subject: [PATCH 01/52] Make EQ conditions work reliably in DynamoDB. The AWS API represents a set object as a list of values. Internally moto also represents a set as a list. This means that when we do value comparisons, the order of the values can cause a set equality test to fail. --- moto/dynamodb2/models.py | 16 +++----- .../test_dynamodb_table_without_range_key.py | 41 +++++++++++++++++++ 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 8187ceaf9..677bbfb07 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -66,6 +66,8 @@ class DynamoType(object): return int(self.value) except ValueError: return float(self.value) + elif self.is_set(): + return set(self.value) else: return self.value @@ -509,15 +511,12 @@ class Table(BaseModel): elif 'Value' in val and DynamoType(val['Value']).value != current_attr[key].value: raise ValueError("The conditional request failed") elif 'ComparisonOperator' in val: - comparison_func = get_comparison_func( - val['ComparisonOperator']) dynamo_types = [ DynamoType(ele) for ele in val.get("AttributeValueList", []) ] - for t in dynamo_types: - if not comparison_func(current_attr[key].value, t.value): - raise ValueError('The conditional request failed') + if not current_attr[key].compare(val['ComparisonOperator'], dynamo_types): + raise ValueError('The conditional request failed') if range_value: self.items[hash_value][range_value] = item else: @@ -946,15 +945,12 @@ class DynamoDBBackend(BaseBackend): elif 'Value' in val and DynamoType(val['Value']).value != item_attr[key].value: raise ValueError("The conditional request failed") elif 'ComparisonOperator' in val: - comparison_func = get_comparison_func( - val['ComparisonOperator']) dynamo_types = [ DynamoType(ele) for ele in val.get("AttributeValueList", []) ] - for t in dynamo_types: - if not comparison_func(item_attr[key].value, t.value): - raise ValueError('The conditional request failed') + if not item_attr[key].compare(val['ComparisonOperator'], dynamo_types): + raise ValueError('The conditional request failed') # Update does not fail on new items, so create one if item is None: diff --git a/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py b/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py index 15e5284b7..874804db0 100644 --- a/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py +++ b/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py @@ -750,6 +750,47 @@ def test_boto3_update_item_conditions_pass_because_expect_exists_by_compare_to_n returned_item = table.get_item(Key={'username': 'johndoe'}) assert dict(returned_item)['Item']['foo'].should.equal("baz") + +@mock_dynamodb2 +def test_boto3_update_settype_item_with_conditions(): + class OrderedSet(set): + """A set with predictable iteration order""" + def __init__(self, values): + super(OrderedSet, self).__init__(values) + self.__ordered_values = values + + def __iter__(self): + return iter(self.__ordered_values) + + table = _create_user_table() + table.put_item(Item={'username': 'johndoe'}) + table.update_item( + Key={'username': 'johndoe'}, + UpdateExpression='SET foo=:new_value', + ExpressionAttributeValues={ + ':new_value': OrderedSet(['hello', 'world']), + }, + ) + + table.update_item( + Key={'username': 'johndoe'}, + UpdateExpression='SET foo=:new_value', + ExpressionAttributeValues={ + ':new_value': set(['baz']), + }, + Expected={ + 'foo': { + 'ComparisonOperator': 'EQ', + 'AttributeValueList': [ + OrderedSet(['world', 'hello']), # Opposite order to original + ], + } + }, + ) + returned_item = table.get_item(Key={'username': 'johndoe'}) + assert dict(returned_item)['Item']['foo'].should.equal(set(['baz'])) + + @mock_dynamodb2 def test_boto3_put_item_conditions_pass(): table = _create_user_table() From 6bd7e5941f7c13c01959539067e9e565770da7f4 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Tue, 15 Jan 2019 17:34:22 +0100 Subject: [PATCH 02/52] Added test case for running multiple instances in the same command. --- tests/test_ec2/test_instances.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_ec2/test_instances.py b/tests/test_ec2/test_instances.py index 84b4fbd7d..c0f0eea4d 100644 --- a/tests/test_ec2/test_instances.py +++ b/tests/test_ec2/test_instances.py @@ -1254,3 +1254,18 @@ def test_create_instance_ebs_optimized(): ) instance.load() instance.ebs_optimized.should.be(False) + +@mock_ec2 +def test_run_multiple_instances_in_same_command(): + instance_count = 4 + client = boto3.client('ec2', region_name='us-east-1') + client.run_instances(ImageId='ami-1234abcd', + MinCount=instance_count, + MaxCount=instance_count) + reservations = client.describe_instances()['Reservations'] + + reservations[0]['Instances'].should.have.length_of(instance_count) + + instances = reservations[0]['Instances'] + for i in range(0, instance_count): + instances[i]['AmiLaunchIndex'].should.be(i) From 22288ef83b32e32d221c129fe1206e0aae38fe43 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Tue, 15 Jan 2019 17:36:10 +0100 Subject: [PATCH 03/52] Implemented initializing ami_launch_index property. --- moto/ec2/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index a450943b7..cc333e790 100755 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -388,6 +388,7 @@ class Instance(TaggedEC2Resource, BotoInstance): self.ebs_optimized = kwargs.get("ebs_optimized", False) self.source_dest_check = "true" self.launch_time = utc_date_and_time() + self.ami_launch_index = kwargs.get("ami_launch_index", 0) self.disable_api_termination = kwargs.get("disable_api_termination", False) self._spot_fleet_id = kwargs.get("spot_fleet_id", None) associate_public_ip = kwargs.get("associate_public_ip", False) From ca5a8033e54ac049448c4c36394d80c588640668 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Tue, 15 Jan 2019 17:37:22 +0100 Subject: [PATCH 04/52] Implemented providing ami_launch_index to the new Instance's constructor. --- moto/ec2/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index cc333e790..11ad8b101 100755 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -720,6 +720,7 @@ class InstanceBackend(object): instance_tags = tags.get('instance', {}) for index in range(count): + kwargs["ami_launch_index"] = index new_instance = Instance( self, image_id, From e4768662818c42fe920379ce7cd7b093380da12c Mon Sep 17 00:00:00 2001 From: acsbendi Date: Tue, 15 Jan 2019 17:38:44 +0100 Subject: [PATCH 05/52] Implemented showing ami_launch_index property in responses. --- moto/ec2/responses/instances.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/ec2/responses/instances.py b/moto/ec2/responses/instances.py index 3e4705f92..49f1face7 100644 --- a/moto/ec2/responses/instances.py +++ b/moto/ec2/responses/instances.py @@ -244,7 +244,7 @@ EC2_RUN_INSTANCES = """ Date: Mon, 21 Jan 2019 15:36:49 +0100 Subject: [PATCH 08/52] Implemented adding default entries to default network ACLs. --- moto/ec2/models.py | 14 ++++++++++++++ tests/test_ec2/test_network_acls.py | 24 ++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index a00cb7c3b..d008b759c 100755 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -3560,8 +3560,22 @@ class NetworkAclBackend(object): self.get_vpc(vpc_id) network_acl = NetworkAcl(self, network_acl_id, vpc_id, default) self.network_acls[network_acl_id] = network_acl + if default: + self.add_default_entries(network_acl_id) return network_acl + def add_default_entries(self, network_acl_id): + default_acl_entries = [ + {'rule_number': 100, 'rule_action': 'allow', 'egress': 'true'}, + {'rule_number': 32767, 'rule_action': 'deny', 'egress': 'true'}, + {'rule_number': 100, 'rule_action': 'allow', 'egress': 'false'}, + {'rule_number': 32767, 'rule_action': 'deny', 'egress': 'false'} + ] + for entry in default_acl_entries: + self.create_network_acl_entry(network_acl_id=network_acl_id, rule_number=entry['rule_number'], protocol='-1', + rule_action=entry['rule_action'], egress=entry['egress'], cidr_block='0.0.0.0/0', + icmp_code=None, icmp_type=None, port_range_from=None, port_range_to=None) + def get_all_network_acls(self, network_acl_ids=None, filters=None): network_acls = self.network_acls.values() diff --git a/tests/test_ec2/test_network_acls.py b/tests/test_ec2/test_network_acls.py index a4eb193e2..9c92c949e 100644 --- a/tests/test_ec2/test_network_acls.py +++ b/tests/test_ec2/test_network_acls.py @@ -190,3 +190,27 @@ def test_new_subnet_in_new_vpc_associates_with_default_network_acl(): new_vpcs_default_network_acl.vpc_id.should.equal(new_vpc.id) new_vpcs_default_network_acl.associations.should.have.length_of(1) new_vpcs_default_network_acl.associations[0]['SubnetId'].should.equal(subnet.id) + + +@mock_ec2 +def test_default_network_acl_default_entries(): + ec2 = boto3.resource('ec2', region_name='us-west-1') + default_network_acl = next(iter(ec2.network_acls.all()), None) + default_network_acl.is_default.should.be.ok + + default_network_acl.entries.should.have.length_of(4) + unique_entries = [] + for entry in default_network_acl.entries: + entry['CidrBlock'].should.equal('0.0.0.0/0') + entry['Protocol'].should.equal('-1') + entry['RuleNumber'].should.be.within([100, 32767]) + entry['RuleAction'].should.be.within(['allow', 'deny']) + assert type(entry['Egress']) is bool + if entry['RuleAction'] == 'allow': + entry['RuleNumber'].should.be.equal(100) + else: + entry['RuleNumber'].should.be.equal(32767) + if entry not in unique_entries: + unique_entries.append(entry) + + unique_entries.should.have.length_of(4) From 337601b5fb9cbff9fad5dce719fc4c30dc2d5c96 Mon Sep 17 00:00:00 2001 From: Mark Challoner Date: Tue, 30 Oct 2018 11:09:35 +0000 Subject: [PATCH 09/52] Add CloudFormation methods delete_change_set, describe_change_set and list_change_sets. --- IMPLEMENTATION_COVERAGE.md | 6 +- moto/cloudformation/models.py | 102 ++++++++++++++-- moto/cloudformation/parsing.py | 54 +++++++-- moto/cloudformation/responses.py | 111 ++++++++++++++++++ .../test_cloudformation_stack_crud_boto3.py | 58 ++++++++- 5 files changed, 308 insertions(+), 23 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 735af6002..42367db9b 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -477,12 +477,12 @@ - [X] create_stack - [ ] create_stack_instances - [ ] create_stack_set -- [ ] delete_change_set +- [X] delete_change_set - [X] delete_stack - [ ] delete_stack_instances - [ ] delete_stack_set - [ ] describe_account_limits -- [ ] describe_change_set +- [X] describe_change_set - [ ] describe_stack_events - [ ] describe_stack_instance - [ ] describe_stack_resource @@ -495,7 +495,7 @@ - [ ] get_stack_policy - [ ] get_template - [ ] get_template_summary -- [ ] list_change_sets +- [X] list_change_sets - [X] list_exports - [ ] list_imports - [ ] list_stack_instances diff --git a/moto/cloudformation/models.py b/moto/cloudformation/models.py index c45c5d5fe..864e98a92 100644 --- a/moto/cloudformation/models.py +++ b/moto/cloudformation/models.py @@ -127,6 +127,49 @@ class FakeStack(BaseModel): self.status = "DELETE_COMPLETE" +class FakeChange(BaseModel): + + def __init__(self, action, logical_resource_id, resource_type): + self.action = action + self.logical_resource_id = logical_resource_id + self.resource_type = resource_type + + +class FakeChangeSet(FakeStack): + + def __init__(self, stack_id, stack_name, stack_template, change_set_id, change_set_name, template, parameters, region_name, notification_arns=None, tags=None, role_arn=None, cross_stack_resources=None): + super(FakeChangeSet, self).__init__( + stack_id, + stack_name, + stack_template, + parameters, + region_name, + notification_arns=notification_arns, + tags=tags, + role_arn=role_arn, + cross_stack_resources=cross_stack_resources, + create_change_set=True, + ) + self.stack_name = stack_name + self.change_set_id = change_set_id + self.change_set_name = change_set_name + self.changes = self.diff(template=template, parameters=parameters) + + def diff(self, template, parameters=None): + self.template = template + self._parse_template() + changes = [] + resources_by_action = self.resource_map.diff(self.template_dict, parameters) + for action, resources in resources_by_action.items(): + for resource_name, resource in resources.items(): + changes.append(FakeChange( + action=action, + logical_resource_id=resource_name, + resource_type=resource['ResourceType'], + )) + return changes + + class FakeEvent(BaseModel): def __init__(self, stack_id, stack_name, logical_resource_id, physical_resource_id, resource_type, resource_status, resource_status_reason=None, resource_properties=None): @@ -171,24 +214,62 @@ class CloudFormationBackend(BaseBackend): return new_stack def create_change_set(self, stack_name, change_set_name, template, parameters, region_name, change_set_type, notification_arns=None, tags=None, role_arn=None): + stack_id = None + stack_template = None if change_set_type == 'UPDATE': stacks = self.stacks.values() stack = None for s in stacks: if s.name == stack_name: stack = s + stack_id = stack.stack_id + stack_template = stack.template if stack is None: raise ValidationError(stack_name) - else: - stack = self.create_stack(stack_name, template, parameters, - region_name, notification_arns, tags, - role_arn, create_change_set=True) + stack_id = generate_stack_id(stack_name) + stack_template = template + change_set_id = generate_changeset_id(change_set_name, region_name) - self.stacks[change_set_name] = {'Id': change_set_id, - 'StackId': stack.stack_id} - self.change_sets[change_set_id] = stack - return change_set_id, stack.stack_id + new_change_set = FakeChangeSet( + stack_id=stack_id, + stack_name=stack_name, + stack_template=stack_template, + change_set_id=change_set_id, + change_set_name=change_set_name, + template=template, + parameters=parameters, + region_name=region_name, + notification_arns=notification_arns, + tags=tags, + role_arn=role_arn, + cross_stack_resources=self.exports + ) + self.change_sets[change_set_id] = new_change_set + self.stacks[stack_id] = new_change_set + return change_set_id, stack_id + + def delete_change_set(self, change_set_name, stack_name=None): + if change_set_name in self.change_sets: + # This means arn was passed in + del self.change_sets[change_set_name] + else: + for cs in self.change_sets: + if self.change_sets[cs].change_set_name == change_set_name: + del self.change_sets[cs] + + def describe_change_set(self, change_set_name, stack_name=None): + change_set = None + if change_set_name in self.change_sets: + # This means arn was passed in + change_set = self.change_sets[change_set_name] + else: + for cs in self.change_sets: + if self.change_sets[cs].change_set_name == change_set_name: + change_set = self.change_sets[cs] + if change_set is None: + raise ValidationError(change_set_name) + return change_set def execute_change_set(self, change_set_name, stack_name=None): stack = None @@ -197,7 +278,7 @@ class CloudFormationBackend(BaseBackend): stack = self.change_sets[change_set_name] else: for cs in self.change_sets: - if self.change_sets[cs].name == change_set_name: + if self.change_sets[cs].change_set_name == change_set_name: stack = self.change_sets[cs] if stack is None: raise ValidationError(stack_name) @@ -223,6 +304,9 @@ class CloudFormationBackend(BaseBackend): else: return list(stacks) + def list_change_sets(self): + return self.change_sets.values() + def list_stacks(self): return [ v for v in self.stacks.values() diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index 35b05d101..0be68944b 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -465,36 +465,70 @@ class ResourceMap(collections.Mapping): ec2_models.ec2_backends[self._region_name].create_tags( [self[resource].physical_resource_id], self.tags) - def update(self, template, parameters=None): + def diff(self, template, parameters=None): if parameters: self.input_parameters = parameters self.load_mapping() self.load_parameters() self.load_conditions() + old_template = self._resource_json_map + new_template = template['Resources'] + + resource_names_by_action = { + 'Add': set(new_template) - set(old_template), + 'Modify': set(name for name in new_template if name in old_template and new_template[ + name] != old_template[name]), + 'Remove': set(old_template) - set(new_template) + } + resources_by_action = { + 'Add': {}, + 'Modify': {}, + 'Remove': {}, + } + + for resource_name in resource_names_by_action['Add']: + resources_by_action['Add'][resource_name] = { + 'LogicalResourceId': resource_name, + 'ResourceType': new_template[resource_name]['Type'] + } + + for resource_name in resource_names_by_action['Modify']: + resources_by_action['Modify'][resource_name] = { + 'LogicalResourceId': resource_name, + 'ResourceType': new_template[resource_name]['Type'] + } + + for resource_name in resource_names_by_action['Remove']: + resources_by_action['Remove'][resource_name] = { + 'LogicalResourceId': resource_name, + 'ResourceType': old_template[resource_name]['Type'] + } + + return resources_by_action + + def update(self, template, parameters=None): + resources_by_action = self.diff(template, parameters) + old_template = self._resource_json_map new_template = template['Resources'] self._resource_json_map = new_template - new_resource_names = set(new_template) - set(old_template) - for resource_name in new_resource_names: + for resource_name, resource in resources_by_action['Add'].items(): resource_json = new_template[resource_name] new_resource = parse_and_create_resource( resource_name, resource_json, self, self._region_name) self._parsed_resources[resource_name] = new_resource - removed_resource_names = set(old_template) - set(new_template) - for resource_name in removed_resource_names: + for resource_name, resource in resources_by_action['Remove'].items(): resource_json = old_template[resource_name] parse_and_delete_resource( resource_name, resource_json, self, self._region_name) self._parsed_resources.pop(resource_name) - resources_to_update = set(name for name in new_template if name in old_template and new_template[ - name] != old_template[name]) tries = 1 - while resources_to_update and tries < 5: - for resource_name in resources_to_update.copy(): + while resources_by_action['Modify'] and tries < 5: + for resource_name, resource in resources_by_action['Modify'].copy().items(): resource_json = new_template[resource_name] try: changed_resource = parse_and_update_resource( @@ -505,7 +539,7 @@ class ResourceMap(collections.Mapping): last_exception = e else: self._parsed_resources[resource_name] = changed_resource - resources_to_update.remove(resource_name) + del resources_by_action['Modify'][resource_name] tries += 1 if tries == 5: raise last_exception diff --git a/moto/cloudformation/responses.py b/moto/cloudformation/responses.py index 9e67e931a..84805efaf 100644 --- a/moto/cloudformation/responses.py +++ b/moto/cloudformation/responses.py @@ -120,6 +120,31 @@ class CloudFormationResponse(BaseResponse): template = self.response_template(CREATE_CHANGE_SET_RESPONSE_TEMPLATE) return template.render(stack_id=stack_id, change_set_id=change_set_id) + def delete_change_set(self): + stack_name = self._get_param('StackName') + change_set_name = self._get_param('ChangeSetName') + + self.cloudformation_backend.delete_change_set(change_set_name=change_set_name, stack_name=stack_name) + if self.request_json: + return json.dumps({ + 'DeleteChangeSetResponse': { + 'DeleteChangeSetResult': {}, + } + }) + else: + template = self.response_template(DELETE_CHANGE_SET_RESPONSE_TEMPLATE) + return template.render() + + def describe_change_set(self): + stack_name = self._get_param('StackName') + change_set_name = self._get_param('ChangeSetName') + change_set = self.cloudformation_backend.describe_change_set( + change_set_name=change_set_name, + stack_name=stack_name, + ) + template = self.response_template(DESCRIBE_CHANGE_SET_RESPONSE_TEMPLATE) + return template.render(change_set=change_set) + @amzn_request_id def execute_change_set(self): stack_name = self._get_param('StackName') @@ -187,6 +212,11 @@ class CloudFormationResponse(BaseResponse): template = self.response_template(DESCRIBE_STACK_EVENTS_RESPONSE) return template.render(stack=stack) + def list_change_sets(self): + change_sets = self.cloudformation_backend.list_change_sets() + template = self.response_template(LIST_CHANGE_SETS_RESPONSE) + return template.render(change_sets=change_sets) + def list_stacks(self): stacks = self.cloudformation_backend.list_stacks() template = self.response_template(LIST_STACKS_RESPONSE) @@ -354,6 +384,66 @@ CREATE_CHANGE_SET_RESPONSE_TEMPLATE = """ """ +DELETE_CHANGE_SET_RESPONSE_TEMPLATE = """ + + + + 3d3200a1-810e-3023-6cc3-example + + +""" + +DESCRIBE_CHANGE_SET_RESPONSE_TEMPLATE = """ + + {{ change_set.change_set_id }} + {{ change_set.change_set_name }} + {{ change_set.stack_id }} + {{ change_set.stack_name }} + {{ change_set.description }} + + {% for param_name, param_value in change_set.stack_parameters.items() %} + + {{ param_name }} + {{ param_value }} + + {% endfor %} + + 2011-05-23T15:47:44Z + {{ change_set.execution_status }} + {{ change_set.status }} + {{ change_set.status_reason }} + {% if change_set.notification_arns %} + + {% for notification_arn in change_set.notification_arns %} + {{ notification_arn }} + {% endfor %} + + {% else %} + + {% endif %} + {% if change_set.role_arn %} + {{ change_set.role_arn }} + {% endif %} + {% if change_set.changes %} + + {% for change in change_set.changes %} + + Resource + + {{ change.action }} + {{ change.logical_resource_id }} + {{ change.resource_type }} + + + {% endfor %} + + {% endif %} + {% if next_token %} + {{ next_token }} + {% endif %} + +""" + EXECUTE_CHANGE_SET_RESPONSE_TEMPLATE = """ @@ -479,6 +569,27 @@ DESCRIBE_STACK_EVENTS_RESPONSE = """ + + + {% for change_set in change_sets %} + + {{ change_set.stack_id }} + {{ change_set.stack_name }} + {{ change_set.change_set_id }} + {{ change_set.change_set_name }} + {{ change_set.execution_status }} + {{ change_set.status }} + {{ change_set.status_reason }} + 2011-05-23T15:47:44Z + {{ change_set.description }} + + {% endfor %} + + +""" + + LIST_STACKS_RESPONSE = """ diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py index 4585da056..2511de6da 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py @@ -399,6 +399,32 @@ def test_create_change_set_from_s3_url(): assert 'arn:aws:cloudformation:us-east-1:123456789:stack/NewStack' in response['StackId'] +@mock_cloudformation +def test_describe_change_set(): + cf_conn = boto3.client('cloudformation', region_name='us-east-1') + cf_conn.create_change_set( + StackName='NewStack', + TemplateBody=dummy_template_json, + ChangeSetName='NewChangeSet', + ChangeSetType='CREATE', + ) + + stack = cf_conn.describe_change_set(ChangeSetName="NewChangeSet") + stack['ChangeSetName'].should.equal('NewChangeSet') + stack['StackName'].should.equal('NewStack') + + cf_conn.create_change_set( + StackName='NewStack', + TemplateBody=dummy_update_template_json, + ChangeSetName='NewChangeSet2', + ChangeSetType='UPDATE', + ) + stack = cf_conn.describe_change_set(ChangeSetName="NewChangeSet2") + stack['ChangeSetName'].should.equal('NewChangeSet2') + stack['StackName'].should.equal('NewStack') + stack['Changes'].should.have.length_of(2) + + @mock_cloudformation def test_execute_change_set_w_arn(): cf_conn = boto3.client('cloudformation', region_name='us-east-1') @@ -420,7 +446,7 @@ def test_execute_change_set_w_name(): ChangeSetName='NewChangeSet', ChangeSetType='CREATE', ) - cf_conn.execute_change_set(ChangeSetName='NewStack', StackName='NewStack') + cf_conn.execute_change_set(ChangeSetName='NewChangeSet', StackName='NewStack') @mock_cloudformation @@ -489,6 +515,20 @@ def test_describe_stack_by_stack_id(): stack_by_id['StackName'].should.equal("test_stack") +@mock_cloudformation +def test_list_change_sets(): + cf_conn = boto3.client('cloudformation', region_name='us-east-1') + cf_conn.create_change_set( + StackName='NewStack2', + TemplateBody=dummy_template_json, + ChangeSetName='NewChangeSet2', + ChangeSetType='CREATE', + ) + change_set = cf_conn.list_change_sets(StackName='NewStack2')['Summaries'][0] + change_set['StackName'].should.equal('NewStack2') + change_set['ChangeSetName'].should.equal('NewChangeSet2') + + @mock_cloudformation def test_list_stacks(): cf = boto3.resource('cloudformation', region_name='us-east-1') @@ -521,6 +561,22 @@ def test_delete_stack_from_resource(): list(cf.stacks.all()).should.have.length_of(0) +@mock_cloudformation +@mock_ec2 +def test_delete_change_set(): + cf_conn = boto3.client('cloudformation', region_name='us-east-1') + cf_conn.create_change_set( + StackName='NewStack', + TemplateBody=dummy_template_json, + ChangeSetName='NewChangeSet', + ChangeSetType='CREATE', + ) + + cf_conn.list_change_sets(StackName='NewStack')['Summaries'].should.have.length_of(1) + cf_conn.delete_change_set(ChangeSetName='NewChangeSet', StackName='NewStack') + cf_conn.list_change_sets(StackName='NewStack')['Summaries'].should.have.length_of(0) + + @mock_cloudformation @mock_ec2 def test_delete_stack_by_name(): From b4b63202d9bb4879db151c38a778692f23ab8f05 Mon Sep 17 00:00:00 2001 From: David Morrison Date: Wed, 23 Jan 2019 15:59:31 -0800 Subject: [PATCH 10/52] tag specifications for spot fleet requests --- moto/ec2/models.py | 35 ++++++++++++++++++++---- moto/ec2/responses/spot_fleets.py | 15 +++++++++++ tests/test_ec2/test_spot_fleet.py | 44 ++++++++++++++++++++++++++++++- 3 files changed, 88 insertions(+), 6 deletions(-) mode change 100755 => 100644 moto/ec2/models.py diff --git a/moto/ec2/models.py b/moto/ec2/models.py old mode 100755 new mode 100644 index a450943b7..cd20e7e0e --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -2879,7 +2879,7 @@ class SpotInstanceRequest(BotoSpotRequest, TaggedEC2Resource): def __init__(self, ec2_backend, spot_request_id, price, image_id, type, valid_from, valid_until, launch_group, availability_zone_group, key_name, security_groups, user_data, instance_type, placement, - kernel_id, ramdisk_id, monitoring_enabled, subnet_id, spot_fleet_id, + kernel_id, ramdisk_id, monitoring_enabled, subnet_id, tags, spot_fleet_id, **kwargs): super(SpotInstanceRequest, self).__init__(**kwargs) ls = LaunchSpecification() @@ -2903,6 +2903,7 @@ class SpotInstanceRequest(BotoSpotRequest, TaggedEC2Resource): ls.monitored = monitoring_enabled ls.subnet_id = subnet_id self.spot_fleet_id = spot_fleet_id + self.tags = tags if security_groups: for group_name in security_groups: @@ -2936,6 +2937,7 @@ class SpotInstanceRequest(BotoSpotRequest, TaggedEC2Resource): security_group_names=[], security_group_ids=self.launch_specification.groups, spot_fleet_id=self.spot_fleet_id, + tags=self.tags, ) instance = reservation.instances[0] return instance @@ -2951,15 +2953,16 @@ class SpotRequestBackend(object): valid_until, launch_group, availability_zone_group, key_name, security_groups, user_data, instance_type, placement, kernel_id, ramdisk_id, - monitoring_enabled, subnet_id, spot_fleet_id=None): + monitoring_enabled, subnet_id, tags=None, spot_fleet_id=None): requests = [] + tags = tags or {} for _ in range(count): spot_request_id = random_spot_request_id() request = SpotInstanceRequest(self, spot_request_id, price, image_id, type, valid_from, valid_until, launch_group, availability_zone_group, key_name, security_groups, user_data, instance_type, placement, kernel_id, ramdisk_id, - monitoring_enabled, subnet_id, spot_fleet_id) + monitoring_enabled, subnet_id, tags, spot_fleet_id) self.spot_instance_requests[spot_request_id] = request requests.append(request) return requests @@ -2979,8 +2982,8 @@ class SpotRequestBackend(object): class SpotFleetLaunchSpec(object): def __init__(self, ebs_optimized, group_set, iam_instance_profile, image_id, - instance_type, key_name, monitoring, spot_price, subnet_id, user_data, - weighted_capacity): + instance_type, key_name, monitoring, spot_price, subnet_id, tag_specifications, + user_data, weighted_capacity): self.ebs_optimized = ebs_optimized self.group_set = group_set self.iam_instance_profile = iam_instance_profile @@ -2990,6 +2993,7 @@ class SpotFleetLaunchSpec(object): self.monitoring = monitoring self.spot_price = spot_price self.subnet_id = subnet_id + self.tag_specifications = tag_specifications self.user_data = user_data self.weighted_capacity = float(weighted_capacity) @@ -3020,6 +3024,7 @@ class SpotFleetRequest(TaggedEC2Resource): monitoring=spec.get('monitoring._enabled'), spot_price=spec.get('spot_price', self.spot_price), subnet_id=spec['subnet_id'], + tag_specifications=self._parse_tag_specifications(spec), user_data=spec.get('user_data'), weighted_capacity=spec['weighted_capacity'], ) @@ -3102,6 +3107,7 @@ class SpotFleetRequest(TaggedEC2Resource): monitoring_enabled=launch_spec.monitoring, subnet_id=launch_spec.subnet_id, spot_fleet_id=self.id, + tags=launch_spec.tag_specifications, ) self.spot_requests.extend(requests) self.fulfilled_capacity += added_weight @@ -3124,6 +3130,25 @@ class SpotFleetRequest(TaggedEC2Resource): self.spot_requests = [req for req in self.spot_requests if req.instance.id not in instance_ids] self.ec2_backend.terminate_instances(instance_ids) + def _parse_tag_specifications(self, spec): + try: + tag_spec_num = max([int(key.split('.')[1]) for key in spec if key.startswith("tag_specification_set")]) + except ValueError: # no tag specifications + return {} + + tag_specifications = {} + for si in range(1, tag_spec_num + 1): + resource_type = spec["tag_specification_set.{si}._resource_type".format(si=si)] + + tags = [key for key in spec if key.startswith("tag_specification_set.{si}._tag".format(si=si))] + tag_num = max([int(key.split('.')[3]) for key in tags]) + tag_specifications[resource_type] = dict(( + spec["tag_specification_set.{si}._tag.{ti}._key".format(si=si, ti=ti)], + spec["tag_specification_set.{si}._tag.{ti}._value".format(si=si, ti=ti)], + ) for ti in range(1, tag_num + 1)) + + return tag_specifications + class SpotFleetBackend(object): def __init__(self): diff --git a/moto/ec2/responses/spot_fleets.py b/moto/ec2/responses/spot_fleets.py index 0366af9d6..bb9aeb4ca 100644 --- a/moto/ec2/responses/spot_fleets.py +++ b/moto/ec2/responses/spot_fleets.py @@ -107,6 +107,21 @@ DESCRIBE_SPOT_FLEET_TEMPLATE = """ Date: Fri, 25 Jan 2019 14:47:23 -0500 Subject: [PATCH 11/52] ECR now uses api.ecr as its endpoint. This changed in botocore 1.12.85. See https://github.com/boto/botocore/commit/b5fa8a51393e24fbc7337a357a2d51a0f750c627 This fixes https://github.com/spulec/moto/issues/2035 --- moto/ecr/urls.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/ecr/urls.py b/moto/ecr/urls.py index 86b8a8dbc..5b12cd843 100644 --- a/moto/ecr/urls.py +++ b/moto/ecr/urls.py @@ -3,6 +3,7 @@ from .responses import ECRResponse url_bases = [ "https?://ecr.(.+).amazonaws.com", + "https?://api.ecr.(.+).amazonaws.com", ] url_paths = { From 1a36c0c377738809c7df6a9087970ef1b79cced5 Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Tue, 29 Jan 2019 18:09:31 -0800 Subject: [PATCH 12/52] IAM Role Tagging support --- IMPLEMENTATION_COVERAGE.md | 30 +++--- moto/iam/exceptions.py | 45 +++++++++ moto/iam/models.py | 89 ++++++++++++++++- moto/iam/responses.py | 81 ++++++++++++++++ setup.py | 4 +- tests/test_iam/test_iam.py | 193 +++++++++++++++++++++++++++++++++++++ 6 files changed, 426 insertions(+), 16 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 735af6002..a9002ea7d 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -2208,7 +2208,7 @@ - [ ] describe_event_types - [ ] describe_events -## iam - 48% implemented +## iam - 62% implemented - [ ] add_client_id_to_open_id_connect_provider - [X] add_role_to_instance_profile - [X] add_user_to_group @@ -2247,7 +2247,7 @@ - [X] delete_server_certificate - [ ] delete_service_linked_role - [ ] delete_service_specific_credential -- [ ] delete_signing_certificate +- [X] delete_signing_certificate - [ ] delete_ssh_public_key - [X] delete_user - [X] delete_user_policy @@ -2279,7 +2279,7 @@ - [ ] get_ssh_public_key - [X] get_user - [X] get_user_policy -- [ ] list_access_keys +- [X] list_access_keys - [X] list_account_aliases - [X] list_attached_group_policies - [X] list_attached_role_policies @@ -2287,19 +2287,21 @@ - [ ] list_entities_for_policy - [X] list_group_policies - [X] list_groups -- [ ] list_groups_for_user -- [ ] list_instance_profiles -- [ ] list_instance_profiles_for_role +- [X] list_groups_for_user +- [X] list_instance_profiles +- [X] list_instance_profiles_for_role - [X] list_mfa_devices - [ ] list_open_id_connect_providers - [X] list_policies - [X] list_policy_versions - [X] list_role_policies -- [ ] list_roles +- [X] list_roles +- [X] list_role_tags +- [ ] list_user_tags - [X] list_saml_providers -- [ ] list_server_certificates +- [X] list_server_certificates - [ ] list_service_specific_credentials -- [ ] list_signing_certificates +- [X] list_signing_certificates - [ ] list_ssh_public_keys - [X] list_user_policies - [X] list_users @@ -2315,6 +2317,10 @@ - [ ] set_default_policy_version - [ ] simulate_custom_policy - [ ] simulate_principal_policy +- [X] tag_role +- [ ] tag_user +- [X] untag_role +- [ ] untag_user - [X] update_access_key - [ ] update_account_password_policy - [ ] update_assume_role_policy @@ -2326,11 +2332,11 @@ - [X] update_saml_provider - [ ] update_server_certificate - [ ] update_service_specific_credential -- [ ] update_signing_certificate +- [X] update_signing_certificate - [ ] update_ssh_public_key - [ ] update_user -- [ ] upload_server_certificate -- [ ] upload_signing_certificate +- [X] upload_server_certificate +- [X] upload_signing_certificate - [ ] upload_ssh_public_key ## importexport - 0% implemented diff --git a/moto/iam/exceptions.py b/moto/iam/exceptions.py index 61922ea18..5b13277da 100644 --- a/moto/iam/exceptions.py +++ b/moto/iam/exceptions.py @@ -32,3 +32,48 @@ class MalformedCertificate(RESTError): def __init__(self, cert): super(MalformedCertificate, self).__init__( 'MalformedCertificate', 'Certificate {cert} is malformed'.format(cert=cert)) + + +class DuplicateTags(RESTError): + code = 400 + + def __init__(self): + super(DuplicateTags, self).__init__( + 'InvalidInput', 'Duplicate tag keys found. Please note that Tag keys are case insensitive.') + + +class TagKeyTooBig(RESTError): + code = 400 + + def __init__(self, tag, param='tags.X.member.key'): + super(TagKeyTooBig, self).__init__( + 'ValidationError', "1 validation error detected: Value '{}' at '{}' failed to satisfy " + "constraint: Member must have length less than or equal to 128.".format(tag, param)) + + +class TagValueTooBig(RESTError): + code = 400 + + def __init__(self, tag): + super(TagValueTooBig, self).__init__( + 'ValidationError', "1 validation error detected: Value '{}' at 'tags.X.member.value' failed to satisfy " + "constraint: Member must have length less than or equal to 256.".format(tag)) + + +class InvalidTagCharacters(RESTError): + code = 400 + + def __init__(self, tag, param='tags.X.member.key'): + message = "1 validation error detected: Value '{}' at '{}' failed to satisfy ".format(tag, param) + message += "constraint: Member must satisfy regular expression pattern: [\\p{L}\\p{Z}\\p{N}_.:/=+\\-@]+" + + super(InvalidTagCharacters, self).__init__('ValidationError', message) + + +class TooManyTags(RESTError): + code = 400 + + def __init__(self, tags, param='tags'): + super(TooManyTags, self).__init__( + 'ValidationError', "1 validation error detected: Value '{}' at '{}' failed to satisfy " + "constraint: Member must have length less than or equal to 50.".format(tags, param)) diff --git a/moto/iam/models.py b/moto/iam/models.py index 5e4bb8263..86a5d7a32 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -3,6 +3,7 @@ import base64 import sys from datetime import datetime import json +import re from cryptography import x509 from cryptography.hazmat.backends import default_backend @@ -12,7 +13,8 @@ from moto.core import BaseBackend, BaseModel from moto.core.utils import iso_8601_datetime_without_milliseconds from .aws_managed_policies import aws_managed_policies_data -from .exceptions import IAMNotFoundException, IAMConflictException, IAMReportNotPresentException, MalformedCertificate +from .exceptions import IAMNotFoundException, IAMConflictException, IAMReportNotPresentException, MalformedCertificate, \ + DuplicateTags, TagKeyTooBig, InvalidTagCharacters, TooManyTags, TagValueTooBig from .utils import random_access_key, random_alphanumeric, random_resource_id, random_policy_id ACCOUNT_ID = 123456789012 @@ -32,7 +34,6 @@ class MFADevice(object): class Policy(BaseModel): - is_attachable = False def __init__(self, @@ -132,6 +133,7 @@ class Role(BaseModel): self.policies = {} self.managed_policies = {} self.create_date = datetime.now(pytz.utc) + self.tags = {} @classmethod def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): @@ -175,6 +177,9 @@ class Role(BaseModel): raise NotImplementedError('"Fn::GetAtt" : [ "{0}" , "Arn" ]"') raise UnformattedGetAttTemplateException() + def get_tags(self): + return [self.tags[tag] for tag in self.tags] + class InstanceProfile(BaseModel): @@ -614,6 +619,86 @@ class IAMBackend(BaseBackend): role = self.get_role(role_name) return role.policies.keys() + def _validate_tag_key(self, tag_key, exception_param='tags.X.member.key'): + """Validates the tag key. + + :param all_tags: Dict to check if there is a duplicate tag. + :param tag_key: The tag key to check against. + :param exception_param: The exception parameter to send over to help format the message. This is to reflect + the difference between the tag and untag APIs. + :return: + """ + # Validate that the key length is correct: + if len(tag_key) > 128: + raise TagKeyTooBig(tag_key, param=exception_param) + + # Validate that the tag key fits the proper Regex: + # [\w\s_.:/=+\-@]+ SHOULD be the same as the Java regex on the AWS documentation: [\p{L}\p{Z}\p{N}_.:/=+\-@]+ + match = re.findall(r'[\w\s_.:/=+\-@]+', tag_key) + # Kudos if you can come up with a better way of doing a global search :) + if not len(match) or len(match[0]) < len(tag_key): + raise InvalidTagCharacters(tag_key, param=exception_param) + + def _check_tag_duplicate(self, all_tags, tag_key): + """Validates that a tag key is not a duplicate + + :param all_tags: Dict to check if there is a duplicate tag. + :param tag_key: The tag key to check against. + :return: + """ + if tag_key in all_tags: + raise DuplicateTags() + + def list_role_tags(self, role_name, marker, max_items=100): + role = self.get_role(role_name) + + max_items = int(max_items) + tag_index = sorted(role.tags) + start_idx = int(marker) if marker else 0 + + tag_index = tag_index[start_idx:start_idx + max_items] + + if len(role.tags) <= (start_idx + max_items): + marker = None + else: + marker = str(start_idx + max_items) + + # Make the tag list of dict's: + tags = [role.tags[tag] for tag in tag_index] + + return tags, marker + + def tag_role(self, role_name, tags): + if len(tags) > 50: + raise TooManyTags(tags) + + role = self.get_role(role_name) + + tag_keys = {} + for tag in tags: + # Need to index by the lowercase tag key since the keys are case insensitive, but their case is retained. + ref_key = tag['Key'].lower() + self._check_tag_duplicate(tag_keys, ref_key) + self._validate_tag_key(tag['Key']) + if len(tag['Value']) > 256: + raise TagValueTooBig(tag['Value']) + + tag_keys[ref_key] = tag + + role.tags.update(tag_keys) + + def untag_role(self, role_name, tag_keys): + if len(tag_keys) > 50: + raise TooManyTags(tag_keys, param='tagKeys') + + role = self.get_role(role_name) + + for key in tag_keys: + ref_key = key.lower() + self._validate_tag_key(key, exception_param='tagKeys') + + role.tags.pop(ref_key, None) + def create_policy_version(self, policy_arn, policy_document, set_as_default): policy = self.get_policy(policy_arn) if not policy: diff --git a/moto/iam/responses.py b/moto/iam/responses.py index d0e749d57..aeeb54936 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -625,6 +625,34 @@ class IamResponse(BaseResponse): template = self.response_template(LIST_SIGNING_CERTIFICATES_TEMPLATE) return template.render(user_name=user_name, certificates=certs) + def list_role_tags(self): + role_name = self._get_param('RoleName') + marker = self._get_param('Marker') + max_items = self._get_param('MaxItems', 100) + + tags, marker = iam_backend.list_role_tags(role_name, marker, max_items) + + template = self.response_template(LIST_ROLE_TAG_TEMPLATE) + return template.render(tags=tags, marker=marker) + + def tag_role(self): + role_name = self._get_param('RoleName') + tags = self._get_multi_param('Tags.member') + + iam_backend.tag_role(role_name, tags) + + template = self.response_template(TAG_ROLE_TEMPLATE) + return template.render() + + def untag_role(self): + role_name = self._get_param('RoleName') + tag_keys = self._get_multi_param('TagKeys.member') + + iam_backend.untag_role(role_name, tag_keys) + + template = self.response_template(UNTAG_ROLE_TEMPLATE) + return template.render() + ATTACH_ROLE_POLICY_TEMPLATE = """ @@ -878,6 +906,16 @@ GET_ROLE_TEMPLATE = """ {% endfor %} + + {% for tag in role.get_tags() %} + + {{ tag['Key'] }} + {{ tag['Value'] }} + + {% endfor %} + {% for profile in instance_profiles %} @@ -1671,3 +1717,38 @@ LIST_SIGNING_CERTIFICATES_TEMPLATE = """ 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE """ + + +TAG_ROLE_TEMPLATE = """ + + EXAMPLE8-90ab-cdef-fedc-ba987EXAMPLE + +""" + + +LIST_ROLE_TAG_TEMPLATE = """ + + {{ 'true' if marker else 'false' }} + {% if marker %} + {{ marker }} + {% endif %} + + {% for tag in tags %} + + {{ tag['Key'] }} + {{ tag['Value'] }} + + {% endfor %} + + + + EXAMPLE8-90ab-cdef-fedc-ba987EXAMPLE + +""" + + +UNTAG_ROLE_TEMPLATE = """ + + EXAMPLE8-90ab-cdef-fedc-ba987EXAMPLE + +""" diff --git a/setup.py b/setup.py index 74683836e..8ac06c4dd 100755 --- a/setup.py +++ b/setup.py @@ -21,8 +21,8 @@ def read(*parts): install_requires = [ "Jinja2>=2.7.3", "boto>=2.36.0", - "boto3>=1.6.16", - "botocore>=1.12.13", + "boto3>=1.9.86", + "botocore>=1.12.86", "cryptography>=2.3.0", "requests>=2.5", "xmltodict", diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 2b5f16d77..15364928a 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -306,6 +306,7 @@ def test_create_policy_versions(): PolicyDocument='{"some":"policy"}') version.get('PolicyVersion').get('Document').should.equal({'some': 'policy'}) + @mock_iam def test_get_policy(): conn = boto3.client('iam', region_name='us-east-1') @@ -579,6 +580,7 @@ def test_get_credential_report(): 'get_credential_report_result']['content'].encode('ascii')).decode('ascii') report.should.match(r'.*my-user.*') + @mock_iam def test_boto3_get_credential_report(): conn = boto3.client('iam', region_name='us-east-1') @@ -780,12 +782,24 @@ def test_get_account_authorization_details(): conn.create_instance_profile(InstanceProfileName='ipn') conn.add_role_to_instance_profile(InstanceProfileName='ipn', RoleName='my-role') + conn.tag_role(RoleName='my-role', Tags=[ + { + 'Key': 'somekey', + 'Value': 'somevalue' + }, + { + 'Key': 'someotherkey', + 'Value': 'someothervalue' + } + ]) + result = conn.get_account_authorization_details(Filter=['Role']) assert len(result['RoleDetailList']) == 1 assert len(result['UserDetailList']) == 0 assert len(result['GroupDetailList']) == 0 assert len(result['Policies']) == 0 assert len(result['RoleDetailList'][0]['InstanceProfileList']) == 1 + assert len(result['RoleDetailList'][0]['Tags']) == 2 result = conn.get_account_authorization_details(Filter=['User']) assert len(result['RoleDetailList']) == 0 @@ -872,6 +886,7 @@ def test_signing_certs(): with assert_raises(ClientError): client.delete_signing_certificate(UserName='notauser', CertificateId=cert_id) + @mock_iam() def test_create_saml_provider(): conn = boto3.client('iam', region_name='us-east-1') @@ -881,6 +896,7 @@ def test_create_saml_provider(): ) response['SAMLProviderArn'].should.equal("arn:aws:iam::123456789012:saml-provider/TestSAMLProvider") + @mock_iam() def test_get_saml_provider(): conn = boto3.client('iam', region_name='us-east-1') @@ -893,6 +909,7 @@ def test_get_saml_provider(): ) response['SAMLMetadataDocument'].should.equal('a' * 1024) + @mock_iam() def test_list_saml_providers(): conn = boto3.client('iam', region_name='us-east-1') @@ -903,6 +920,7 @@ def test_list_saml_providers(): response = conn.list_saml_providers() response['SAMLProviderList'][0]['Arn'].should.equal("arn:aws:iam::123456789012:saml-provider/TestSAMLProvider") + @mock_iam() def test_delete_saml_provider(): conn = boto3.client('iam', region_name='us-east-1') @@ -929,3 +947,178 @@ def test_delete_saml_provider(): # Verify that it's not in the list: resp = conn.list_signing_certificates(UserName='testing') assert not resp['Certificates'] + + +@mock_iam() +def test_tag_role(): + """Tests both the tag_role and get_role_tags capability""" + conn = boto3.client('iam', region_name='us-east-1') + conn.create_role(RoleName="my-role", AssumeRolePolicyDocument="{}") + + # Get without tags: + role = conn.get_role(RoleName='my-role')['Role'] + assert not role.get('Tags') + + # With proper tag values: + conn.tag_role(RoleName='my-role', Tags=[ + { + 'Key': 'somekey', + 'Value': 'somevalue' + }, + { + 'Key': 'someotherkey', + 'Value': 'someothervalue' + } + ]) + + # Get role: + role = conn.get_role(RoleName='my-role')['Role'] + assert len(role['Tags']) == 2 + assert role['Tags'][0]['Key'] == 'somekey' + assert role['Tags'][0]['Value'] == 'somevalue' + assert role['Tags'][1]['Key'] == 'someotherkey' + assert role['Tags'][1]['Value'] == 'someothervalue' + + # Same -- but for list_role_tags: + tags = conn.list_role_tags(RoleName='my-role') + assert len(tags['Tags']) == 2 + assert role['Tags'][0]['Key'] == 'somekey' + assert role['Tags'][0]['Value'] == 'somevalue' + assert role['Tags'][1]['Key'] == 'someotherkey' + assert role['Tags'][1]['Value'] == 'someothervalue' + assert not tags['IsTruncated'] + assert not tags.get('Marker') + + # Test pagination: + tags = conn.list_role_tags(RoleName='my-role', MaxItems=1) + assert len(tags['Tags']) == 1 + assert tags['IsTruncated'] + assert tags['Tags'][0]['Key'] == 'somekey' + assert tags['Tags'][0]['Value'] == 'somevalue' + assert tags['Marker'] == '1' + + tags = conn.list_role_tags(RoleName='my-role', Marker=tags['Marker']) + assert len(tags['Tags']) == 1 + assert tags['Tags'][0]['Key'] == 'someotherkey' + assert tags['Tags'][0]['Value'] == 'someothervalue' + assert not tags['IsTruncated'] + assert not tags.get('Marker') + + # Test updating an existing tag: + conn.tag_role(RoleName='my-role', Tags=[ + { + 'Key': 'somekey', + 'Value': 'somenewvalue' + } + ]) + tags = conn.list_role_tags(RoleName='my-role') + assert len(tags['Tags']) == 2 + assert tags['Tags'][0]['Key'] == 'somekey' + assert tags['Tags'][0]['Value'] == 'somenewvalue' + + # Empty is good: + conn.tag_role(RoleName='my-role', Tags=[ + { + 'Key': 'somekey', + 'Value': '' + } + ]) + tags = conn.list_role_tags(RoleName='my-role') + assert len(tags['Tags']) == 2 + assert tags['Tags'][0]['Key'] == 'somekey' + assert tags['Tags'][0]['Value'] == '' + + # Test creating tags with invalid values: + # With more than 50 tags: + with assert_raises(ClientError) as ce: + too_many_tags = list(map(lambda x: {'Key': str(x), 'Value': str(x)}, range(0, 51))) + conn.tag_role(RoleName='my-role', Tags=too_many_tags) + assert 'failed to satisfy constraint: Member must have length less than or equal to 50.' \ + in ce.exception.response['Error']['Message'] + + # With a duplicate tag: + with assert_raises(ClientError) as ce: + conn.tag_role(RoleName='my-role', Tags=[{'Key': '0', 'Value': ''}, {'Key': '0', 'Value': ''}]) + assert 'Duplicate tag keys found. Please note that Tag keys are case insensitive.' \ + in ce.exception.response['Error']['Message'] + + # Duplicate tag with different casing: + with assert_raises(ClientError) as ce: + conn.tag_role(RoleName='my-role', Tags=[{'Key': 'a', 'Value': ''}, {'Key': 'A', 'Value': ''}]) + assert 'Duplicate tag keys found. Please note that Tag keys are case insensitive.' \ + in ce.exception.response['Error']['Message'] + + # With a really big key: + with assert_raises(ClientError) as ce: + conn.tag_role(RoleName='my-role', Tags=[{'Key': '0' * 129, 'Value': ''}]) + assert 'Member must have length less than or equal to 128.' in ce.exception.response['Error']['Message'] + + # With a really big value: + with assert_raises(ClientError) as ce: + conn.tag_role(RoleName='my-role', Tags=[{'Key': '0', 'Value': '0' * 257}]) + assert 'Member must have length less than or equal to 256.' in ce.exception.response['Error']['Message'] + + # With an invalid character: + with assert_raises(ClientError) as ce: + conn.tag_role(RoleName='my-role', Tags=[{'Key': 'NOWAY!', 'Value': ''}]) + assert 'Member must satisfy regular expression pattern: [\\p{L}\\p{Z}\\p{N}_.:/=+\\-@]+' \ + in ce.exception.response['Error']['Message'] + + # With a role that doesn't exist: + with assert_raises(ClientError): + conn.tag_role(RoleName='notarole', Tags=[{'Key': 'some', 'Value': 'value'}]) + + +@mock_iam +def test_untag_role(): + conn = boto3.client('iam', region_name='us-east-1') + conn.create_role(RoleName="my-role", AssumeRolePolicyDocument="{}") + + # With proper tag values: + conn.tag_role(RoleName='my-role', Tags=[ + { + 'Key': 'somekey', + 'Value': 'somevalue' + }, + { + 'Key': 'someotherkey', + 'Value': 'someothervalue' + } + ]) + + # Remove them: + conn.untag_role(RoleName='my-role', TagKeys=['somekey']) + tags = conn.list_role_tags(RoleName='my-role') + assert len(tags['Tags']) == 1 + assert tags['Tags'][0]['Key'] == 'someotherkey' + assert tags['Tags'][0]['Value'] == 'someothervalue' + + # And again: + conn.untag_role(RoleName='my-role', TagKeys=['someotherkey']) + tags = conn.list_role_tags(RoleName='my-role') + assert not tags['Tags'] + + # Test removing tags with invalid values: + # With more than 50 tags: + with assert_raises(ClientError) as ce: + conn.untag_role(RoleName='my-role', TagKeys=[str(x) for x in range(0, 51)]) + assert 'failed to satisfy constraint: Member must have length less than or equal to 50.' \ + in ce.exception.response['Error']['Message'] + assert 'tagKeys' in ce.exception.response['Error']['Message'] + + # With a really big key: + with assert_raises(ClientError) as ce: + conn.untag_role(RoleName='my-role', TagKeys=['0' * 129]) + assert 'Member must have length less than or equal to 128.' in ce.exception.response['Error']['Message'] + assert 'tagKeys' in ce.exception.response['Error']['Message'] + + # With an invalid character: + with assert_raises(ClientError) as ce: + conn.untag_role(RoleName='my-role', TagKeys=['NOWAY!']) + assert 'Member must satisfy regular expression pattern: [\\p{L}\\p{Z}\\p{N}_.:/=+\\-@]+' \ + in ce.exception.response['Error']['Message'] + assert 'tagKeys' in ce.exception.response['Error']['Message'] + + # With a role that doesn't exist: + with assert_raises(ClientError): + conn.untag_role(RoleName='notarole', TagKeys=['somevalue']) From dfceab9bf0f54a3cc8738322f3d4d00a6dfdf444 Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Mon, 4 Feb 2019 13:44:01 -0800 Subject: [PATCH 13/52] Some fixes to get_account_authorization_details --- moto/iam/responses.py | 63 +++++++++++++++++++++++++------------- tests/test_iam/test_iam.py | 40 +++++++++++++++++------- 2 files changed, 71 insertions(+), 32 deletions(-) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index aeeb54936..818c6de36 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -554,7 +554,8 @@ class IamResponse(BaseResponse): policies=account_details['managed_policies'], users=account_details['users'], groups=account_details['groups'], - roles=account_details['roles'] + roles=account_details['roles'], + get_groups_for_user=iam_backend.get_groups_for_user ) def create_saml_provider(self): @@ -1499,8 +1500,19 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """ {% for user in users %} - - + + {% for group in get_groups_for_user(user.name) %} + {{ group.name }} + {% endfor %} + + + {% for policy in user.managed_policies %} + + {{ user.managed_policies[policy].name }} + {{ policy }} + + {% endfor %} + {{ user.id }} {{ user.path }} {{ user.name }} @@ -1514,25 +1526,39 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """ {{ group.id }} - {% for policy in group.managed_policies %} - - {{ policy.name }} - {{ policy.arn }} - + {% for policy_arn in group.managed_policies %} + + {{ group.managed_policies[policy_arn].name }} + {{ policy_arn }} + {% endfor %} {{ group.name }} {{ group.path }} {{ group.arn }} {{ group.create_date }} - + + {% for policy in group.policies %} + + {{ policy }} + {{ group.get_policy(policy) }} + + {% endfor %} + {% endfor %} {% for role in roles %} - + + {% for inline_policy in role.policies %} + + {{ inline_policy }} + {{ role.policies[inline_policy] }} + + {% endfor %} + {% for policy in role.managed_policies %} @@ -1589,19 +1615,14 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """{{ policy.id }} {{ policy.path }} + {% for policy_version in policy.versions %} - - {"Version":"2012-10-17","Statement":{"Effect":"Allow", - "Action":["iam:CreatePolicy","iam:CreatePolicyVersion", - "iam:DeletePolicy","iam:DeletePolicyVersion","iam:GetPolicy", - "iam:GetPolicyVersion","iam:ListPolicies", - "iam:ListPolicyVersions","iam:SetDefaultPolicyVersion"], - "Resource":"*"}} - - true - v1 - 2012-05-09T16:27:11Z + {{ policy_version.document }} + {{ policy_version.is_default }} + {{ policy_version.version_id }} + {{ policy_version.create_datetime }} + {% endfor %} {{ policy.arn }} 1 diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 15364928a..ac538649c 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -759,6 +759,17 @@ def test_get_access_key_last_used(): @mock_iam def test_get_account_authorization_details(): import json + test_policy = json.dumps({ + "Version": "2012-10-17", + "Statement": [ + { + "Action": "s3:ListBucket", + "Resource": "*", + "Effect": "Allow", + } + ] + }) + conn = boto3.client('iam', region_name='us-east-1') conn.create_role(RoleName="my-role", AssumeRolePolicyDocument="some policy", Path="/my-path/") conn.create_user(Path='/', UserName='testUser') @@ -766,22 +777,22 @@ def test_get_account_authorization_details(): conn.create_policy( PolicyName='testPolicy', Path='/', - PolicyDocument=json.dumps({ - "Version": "2012-10-17", - "Statement": [ - { - "Action": "s3:ListBucket", - "Resource": "*", - "Effect": "Allow", - } - ] - }), + PolicyDocument=test_policy, Description='Test Policy' ) + # Attach things to the user and group: + conn.put_user_policy(UserName='testUser', PolicyName='testPolicy', PolicyDocument=test_policy) + conn.put_group_policy(GroupName='testGroup', PolicyName='testPolicy', PolicyDocument=test_policy) + + conn.attach_user_policy(UserName='testUser', PolicyArn='arn:aws:iam::123456789012:policy/testPolicy') + conn.attach_group_policy(GroupName='testGroup', PolicyArn='arn:aws:iam::123456789012:policy/testPolicy') + + conn.add_user_to_group(UserName='testUser', GroupName='testGroup') + + # Add things to the role: conn.create_instance_profile(InstanceProfileName='ipn') conn.add_role_to_instance_profile(InstanceProfileName='ipn', RoleName='my-role') - conn.tag_role(RoleName='my-role', Tags=[ { 'Key': 'somekey', @@ -792,6 +803,7 @@ def test_get_account_authorization_details(): 'Value': 'someothervalue' } ]) + conn.put_role_policy(RoleName='my-role', PolicyName='test-policy', PolicyDocument=test_policy) result = conn.get_account_authorization_details(Filter=['Role']) assert len(result['RoleDetailList']) == 1 @@ -800,10 +812,13 @@ def test_get_account_authorization_details(): assert len(result['Policies']) == 0 assert len(result['RoleDetailList'][0]['InstanceProfileList']) == 1 assert len(result['RoleDetailList'][0]['Tags']) == 2 + assert len(result['RoleDetailList'][0]['RolePolicyList']) == 1 result = conn.get_account_authorization_details(Filter=['User']) assert len(result['RoleDetailList']) == 0 assert len(result['UserDetailList']) == 1 + assert len(result['UserDetailList'][0]['GroupList']) == 1 + assert len(result['UserDetailList'][0]['AttachedManagedPolicies']) == 1 assert len(result['GroupDetailList']) == 0 assert len(result['Policies']) == 0 @@ -811,6 +826,8 @@ def test_get_account_authorization_details(): assert len(result['RoleDetailList']) == 0 assert len(result['UserDetailList']) == 0 assert len(result['GroupDetailList']) == 1 + assert len(result['GroupDetailList'][0]['GroupPolicyList']) == 1 + assert len(result['GroupDetailList'][0]['AttachedManagedPolicies']) == 1 assert len(result['Policies']) == 0 result = conn.get_account_authorization_details(Filter=['LocalManagedPolicy']) @@ -818,6 +835,7 @@ def test_get_account_authorization_details(): assert len(result['UserDetailList']) == 0 assert len(result['GroupDetailList']) == 0 assert len(result['Policies']) == 1 + assert len(result['Policies'][0]['PolicyVersionList']) == 1 # Check for greater than 1 since this should always be greater than one but might change. # See iam/aws_managed_policies.py From 94672799cf735aa0079fd27e448e2d2bf6596e7e Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Thu, 7 Feb 2019 17:32:31 -0800 Subject: [PATCH 14/52] Fixed a bug in get_account_authorization_details for attached managed policies. Fixed a bug in the XML template for `get_account_authorization_details` where attached managed policies were empty strings. --- moto/iam/responses.py | 6 +++--- tests/test_iam/test_iam.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 818c6de36..e3cc4b90b 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -1560,10 +1560,10 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """ - {% for policy in role.managed_policies %} + {% for policy_arn in role.managed_policies %} - {{ policy.name }} - {{ policy.arn }} + {{ role.managed_policies[policy_arn].name }} + {{ policy_arn }} {% endfor %} diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index ac538649c..d5f1bb4f9 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -804,6 +804,7 @@ def test_get_account_authorization_details(): } ]) conn.put_role_policy(RoleName='my-role', PolicyName='test-policy', PolicyDocument=test_policy) + conn.attach_role_policy(RoleName='my-role', PolicyArn='arn:aws:iam::123456789012:policy/testPolicy') result = conn.get_account_authorization_details(Filter=['Role']) assert len(result['RoleDetailList']) == 1 @@ -813,6 +814,10 @@ def test_get_account_authorization_details(): assert len(result['RoleDetailList'][0]['InstanceProfileList']) == 1 assert len(result['RoleDetailList'][0]['Tags']) == 2 assert len(result['RoleDetailList'][0]['RolePolicyList']) == 1 + assert len(result['RoleDetailList'][0]['AttachedManagedPolicies']) == 1 + assert result['RoleDetailList'][0]['AttachedManagedPolicies'][0]['PolicyName'] == 'testPolicy' + assert result['RoleDetailList'][0]['AttachedManagedPolicies'][0]['PolicyArn'] == \ + 'arn:aws:iam::123456789012:policy/testPolicy' result = conn.get_account_authorization_details(Filter=['User']) assert len(result['RoleDetailList']) == 0 @@ -821,6 +826,9 @@ def test_get_account_authorization_details(): assert len(result['UserDetailList'][0]['AttachedManagedPolicies']) == 1 assert len(result['GroupDetailList']) == 0 assert len(result['Policies']) == 0 + assert result['UserDetailList'][0]['AttachedManagedPolicies'][0]['PolicyName'] == 'testPolicy' + assert result['UserDetailList'][0]['AttachedManagedPolicies'][0]['PolicyArn'] == \ + 'arn:aws:iam::123456789012:policy/testPolicy' result = conn.get_account_authorization_details(Filter=['Group']) assert len(result['RoleDetailList']) == 0 @@ -829,6 +837,9 @@ def test_get_account_authorization_details(): assert len(result['GroupDetailList'][0]['GroupPolicyList']) == 1 assert len(result['GroupDetailList'][0]['AttachedManagedPolicies']) == 1 assert len(result['Policies']) == 0 + assert result['GroupDetailList'][0]['AttachedManagedPolicies'][0]['PolicyName'] == 'testPolicy' + assert result['GroupDetailList'][0]['AttachedManagedPolicies'][0]['PolicyArn'] == \ + 'arn:aws:iam::123456789012:policy/testPolicy' result = conn.get_account_authorization_details(Filter=['LocalManagedPolicy']) assert len(result['RoleDetailList']) == 0 From f035b9613d750131b71f3cb4468e0260b6d7351d Mon Sep 17 00:00:00 2001 From: Greg Sterin Date: Thu, 14 Feb 2019 18:10:01 -0800 Subject: [PATCH 15/52] support a bit more of the dynamoDB ConditionExpression syntax --- moto/dynamodb2/responses.py | 109 +++++++++++++++++--------- tests/test_dynamodb2/test_dynamodb.py | 107 +++++++++++++++++++++++++ 2 files changed, 181 insertions(+), 35 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 60886e171..bd09b6890 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -31,6 +31,68 @@ def get_empty_str_error(): )) +def condition_expression_to_expected(condition_expression, expression_attribute_names, expression_attribute_values): + """ + Limited condition expression syntax parsing. + Supports Global Negation ex: NOT(inner expressions). + Supports simple AND conditions ex: cond_a AND cond_b and cond_c. + Atomic expressions supported are attribute_exists(key), attribute_not_exists(key) and #key = :value. + """ + expected = {} + if condition_expression and 'OR' not in condition_expression: + reverse_re = re.compile('^NOT\s*\((.*)\)$') + reverse_m = reverse_re.match(condition_expression.strip()) + + reverse = False + if reverse_m: + reverse = True + condition_expression = reverse_m.group(1) + + cond_items = [c.strip() for c in condition_expression.split('AND')] + if cond_items: + overwrite = False + exists_re = re.compile('^attribute_exists\s*\((.*)\)$') + not_exists_re = re.compile( + '^attribute_not_exists\s*\((.*)\)$') + equals_re= re.compile('^(#?\w+)\s*=\s*(\:?\w+)') + + for cond in cond_items: + exists_m = exists_re.match(cond) + not_exists_m = not_exists_re.match(cond) + equals_m = equals_re.match(cond) + + if exists_m: + attribute_name = expression_attribute_names_lookup(exists_m.group(1), expression_attribute_names) + expected[attribute_name] = {'Exists': True if not reverse else False} + elif not_exists_m: + attribute_name = expression_attribute_names_lookup(not_exists_m.group(1), expression_attribute_names) + expected[attribute_name] = {'Exists': False if not reverse else True} + elif equals_m: + attribute_name = expression_attribute_names_lookup(equals_m.group(1), expression_attribute_names) + attribute_value = expression_attribute_values_lookup(equals_m.group(2), expression_attribute_values) + expected[attribute_name] = { + 'AttributeValueList': [attribute_value], + 'ComparisonOperator': 'EQ' if not reverse else 'NEQ'} + + return expected + + +def expression_attribute_names_lookup(attribute_name, expression_attribute_names): + if attribute_name.startswith('#') and attribute_name in expression_attribute_names: + return expression_attribute_names[attribute_name] + else: + return attribute_name + + +def expression_attribute_values_lookup(attribute_value, expression_attribute_values): + if isinstance(attribute_value, six.string_types) and \ + attribute_value.startswith(':') and\ + attribute_value in expression_attribute_values: + return expression_attribute_values[attribute_value] + else: + return attribute_value + + class DynamoHandler(BaseResponse): def get_endpoint_name(self, headers): @@ -220,24 +282,13 @@ class DynamoHandler(BaseResponse): # expression if not expected: condition_expression = self.body.get('ConditionExpression') - if condition_expression and 'OR' not in condition_expression: - cond_items = [c.strip() - for c in condition_expression.split('AND')] - - if cond_items: - expected = {} - overwrite = False - exists_re = re.compile('^attribute_exists\s*\((.*)\)$') - not_exists_re = re.compile( - '^attribute_not_exists\s*\((.*)\)$') - - for cond in cond_items: - exists_m = exists_re.match(cond) - not_exists_m = not_exists_re.match(cond) - if exists_m: - expected[exists_m.group(1)] = {'Exists': True} - elif not_exists_m: - expected[not_exists_m.group(1)] = {'Exists': False} + expression_attribute_names = self.body.get('ExpressionAttributeNames', {}) + expression_attribute_values = self.body.get('ExpressionAttributeValues', {}) + expected = condition_expression_to_expected(condition_expression, + expression_attribute_names, + expression_attribute_values) + if expected: + overwrite = False try: result = self.dynamodb_backend.put_item(name, item, expected, overwrite) @@ -590,23 +641,11 @@ class DynamoHandler(BaseResponse): # expression if not expected: condition_expression = self.body.get('ConditionExpression') - if condition_expression and 'OR' not in condition_expression: - cond_items = [c.strip() - for c in condition_expression.split('AND')] - - if cond_items: - expected = {} - exists_re = re.compile('^attribute_exists\s*\((.*)\)$') - not_exists_re = re.compile( - '^attribute_not_exists\s*\((.*)\)$') - - for cond in cond_items: - exists_m = exists_re.match(cond) - not_exists_m = not_exists_re.match(cond) - if exists_m: - expected[exists_m.group(1)] = {'Exists': True} - elif not_exists_m: - expected[not_exists_m.group(1)] = {'Exists': False} + expression_attribute_names = self.body.get('ExpressionAttributeNames', {}) + expression_attribute_values = self.body.get('ExpressionAttributeValues', {}) + expected = condition_expression_to_expected(condition_expression, + expression_attribute_names, + expression_attribute_values) # Support spaces between operators in an update expression # E.g. `a = b + c` -> `a=b+c` diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index ea9be19c1..3d6f1de65 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1505,3 +1505,110 @@ def test_dynamodb_streams_2(): assert 'LatestStreamLabel' in resp['TableDescription'] assert 'LatestStreamArn' in resp['TableDescription'] +@mock_dynamodb2 +def test_condition_expressions(): + client = boto3.client('dynamodb', region_name='us-east-1') + dynamodb = boto3.resource('dynamodb', region_name='us-east-1') + + # Create the DynamoDB table. + client.create_table( + TableName='test1', + AttributeDefinitions=[{'AttributeName': 'client', 'AttributeType': 'S'}, {'AttributeName': 'app', 'AttributeType': 'S'}], + KeySchema=[{'AttributeName': 'client', 'KeyType': 'HASH'}, {'AttributeName': 'app', 'KeyType': 'RANGE'}], + ProvisionedThroughput={'ReadCapacityUnits': 123, 'WriteCapacityUnits': 123} + ) + client.put_item( + TableName='test1', + Item={ + 'client': {'S': 'client1'}, + 'app': {'S': 'app1'}, + 'match': {'S': 'match'}, + 'existing': {'S': 'existing'}, + } + ) + + client.put_item( + TableName='test1', + Item={ + 'client': {'S': 'client1'}, + 'app': {'S': 'app1'}, + 'match': {'S': 'match'}, + 'existing': {'S': 'existing'}, + }, + ConditionExpression='attribute_exists(#existing) AND attribute_not_exists(#nonexistent) AND #match = :match', + ExpressionAttributeNames={ + '#existing': 'existing', + '#nonexistent': 'nope', + '#match': 'match', + }, + ExpressionAttributeValues={ + ':match': {'S': 'match'} + } + ) + + client.put_item( + TableName='test1', + Item={ + 'client': {'S': 'client1'}, + 'app': {'S': 'app1'}, + 'match': {'S': 'match'}, + 'existing': {'S': 'existing'}, + }, + ConditionExpression='NOT(attribute_exists(#nonexistent1) AND attribute_exists(#nonexistent2))', + ExpressionAttributeNames={ + '#nonexistent1': 'nope', + '#nonexistent2': 'nope2' + } + ) + + with assert_raises(client.exceptions.ConditionalCheckFailedException): + client.put_item( + TableName='test1', + Item={ + 'client': {'S': 'client1'}, + 'app': {'S': 'app1'}, + 'match': {'S': 'match'}, + 'existing': {'S': 'existing'}, + }, + ConditionExpression='attribute_exists(#nonexistent1) AND attribute_exists(#nonexistent2)', + ExpressionAttributeNames={ + '#nonexistent1': 'nope', + '#nonexistent2': 'nope2' + } + ) + + with assert_raises(client.exceptions.ConditionalCheckFailedException): + client.put_item( + TableName='test1', + Item={ + 'client': {'S': 'client1'}, + 'app': {'S': 'app1'}, + 'match': {'S': 'match'}, + 'existing': {'S': 'existing'}, + }, + ConditionExpression='NOT(attribute_not_exists(#nonexistent1) AND attribute_not_exists(#nonexistent2))', + ExpressionAttributeNames={ + '#nonexistent1': 'nope', + '#nonexistent2': 'nope2' + } + ) + + with assert_raises(client.exceptions.ConditionalCheckFailedException): + client.put_item( + TableName='test1', + Item={ + 'client': {'S': 'client1'}, + 'app': {'S': 'app1'}, + 'match': {'S': 'match'}, + 'existing': {'S': 'existing'}, + }, + ConditionExpression='attribute_exists(#existing) AND attribute_not_exists(#nonexistent) AND #match = :match', + ExpressionAttributeNames={ + '#existing': 'existing', + '#nonexistent': 'nope', + '#match': 'match', + }, + ExpressionAttributeValues={ + ':match': {'S': 'match2'} + } + ) From c1232a7a23b4022c008f273ea71e74b590056773 Mon Sep 17 00:00:00 2001 From: Greg Sterin Date: Fri, 15 Feb 2019 13:49:42 -0800 Subject: [PATCH 16/52] linting --- moto/dynamodb2/responses.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index bd09b6890..49095f09c 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -50,11 +50,10 @@ def condition_expression_to_expected(condition_expression, expression_attribute_ cond_items = [c.strip() for c in condition_expression.split('AND')] if cond_items: - overwrite = False exists_re = re.compile('^attribute_exists\s*\((.*)\)$') not_exists_re = re.compile( '^attribute_not_exists\s*\((.*)\)$') - equals_re= re.compile('^(#?\w+)\s*=\s*(\:?\w+)') + equals_re = re.compile('^(#?\w+)\s*=\s*(\:?\w+)') for cond in cond_items: exists_m = exists_re.match(cond) From 4c6f08d463afca69113a8d6b4beb6fd110fa42d0 Mon Sep 17 00:00:00 2001 From: William Rubel Date: Sat, 16 Feb 2019 09:27:23 -0600 Subject: [PATCH 17/52] Update kms and lambda to work with terraform --- moto/awslambda/models.py | 8 ++++ moto/awslambda/responses.py | 17 ++++++++- moto/kms/models.py | 16 +++++++- moto/kms/responses.py | 22 +++++++++++ tests/test_awslambda/test_lambda.py | 34 +++++++++++++++++ tests/test_kms/test_kms.py | 57 +++++++++++++++++++++++++++++ 6 files changed, 152 insertions(+), 2 deletions(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index a37a15e27..9fc41c11e 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -500,6 +500,11 @@ class LambdaStorage(object): except ValueError: return self._functions[name]['latest'] + def list_versions_by_function(self, name): + if name not in self._functions: + return None + return [self._functions[name]['latest']] + def get_arn(self, arn): return self._arns.get(arn, None) @@ -607,6 +612,9 @@ class LambdaBackend(BaseBackend): def get_function(self, function_name, qualifier=None): return self._lambdas.get_function(function_name, qualifier) + def list_versions_by_function(self, function_name): + return self._lambdas.list_versions_by_function(function_name) + def get_function_by_arn(self, function_arn): return self._lambdas.get_arn(function_arn) diff --git a/moto/awslambda/responses.py b/moto/awslambda/responses.py index 1a9a4df83..bcd2da903 100644 --- a/moto/awslambda/responses.py +++ b/moto/awslambda/responses.py @@ -52,7 +52,11 @@ class LambdaResponse(BaseResponse): self.setup_class(request, full_url, headers) if request.method == 'GET': # This is ListVersionByFunction - raise ValueError("Cannot handle request") + + path = request.path if hasattr(request, 'path') else path_url(request.url) + function_name = path.split('/')[-2] + return self._list_versions_by_function(function_name) + elif request.method == 'POST': return self._publish_function(request, full_url, headers) else: @@ -151,6 +155,17 @@ class LambdaResponse(BaseResponse): return 200, {}, json.dumps(result) + def _list_versions_by_function(self, function_name): + result = { + 'Versions': [] + } + + for fn in self.lambda_backend.list_versions_by_function(function_name): + json_data = fn.get_configuration() + result['Versions'].append(json_data) + + return 200, {}, json.dumps(result) + def _create_function(self, request, full_url, headers): try: fn = self.lambda_backend.create_function(self.json_body) diff --git a/moto/kms/models.py b/moto/kms/models.py index bb39d1b24..fb4040617 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -21,6 +21,7 @@ class Key(BaseModel): self.account_id = "0123456789012" self.key_rotation_status = False self.deletion_date = None + self.tags = {} @property def physical_resource_id(self): @@ -35,7 +36,7 @@ class Key(BaseModel): "KeyMetadata": { "AWSAccountId": self.account_id, "Arn": self.arn, - "CreationDate": "2015-01-01 00:00:00", + "CreationDate": datetime.strftime(datetime.utcnow(), "%Y-%m-%dT%H:%M:%SZ"), "Description": self.description, "Enabled": self.enabled, "KeyId": self.id, @@ -84,6 +85,19 @@ class KmsBackend(BaseBackend): self.keys[key.id] = key return key + def update_key_description(self, key_id, description): + key = self.keys[self.get_key_id(key_id)] + key.description = description + + def tag_resource(self, key_id, tags): + key = self.keys[self.get_key_id(key_id)] + key.tags = tags + + + def list_resource_tags(self, key_id): + key = self.keys[self.get_key_id(key_id)] + return key.tags + def delete_key(self, key_id): if key_id in self.keys: if key_id in self.key_to_aliases: diff --git a/moto/kms/responses.py b/moto/kms/responses.py index 5883f51ec..b66ca267c 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -38,6 +38,28 @@ class KmsResponse(BaseResponse): policy, key_usage, description, self.region) return json.dumps(key.to_dict()) + def update_key_description(self): + key_id = self.parameters.get('KeyId') + description = self.parameters.get('Description') + + self.kms_backend.update_key_description(key_id,description) + return json.dumps(None) + + def tag_resource(self): + key_id = self.parameters.get('KeyId') + tags = self.parameters.get('Tags') + self.kms_backend.tag_resource(key_id,tags) + return json.dumps({}) + + def list_resource_tags(self): + key_id = self.parameters.get('KeyId') + tags = self.kms_backend.list_resource_tags(key_id) + return json.dumps({ + "Tags": tags, + "NextMarker": None, + "Truncated": False, + }) + def describe_key(self): key_id = self.parameters.get('KeyId') try: diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index 8ea9cc6fd..9c5120f51 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -819,3 +819,37 @@ def get_function_policy(): assert isinstance(response['Policy'], str) res = json.loads(response['Policy']) assert res['Statement'][0]['Action'] == 'lambda:InvokeFunction' + + + +@mock_lambda +@mock_s3 +def test_list_versions_by_function(): + s3_conn = boto3.client('s3', 'us-west-2') + s3_conn.create_bucket(Bucket='test-bucket') + + zip_content = get_test_zip_file2() + s3_conn.put_object(Bucket='test-bucket', Key='test.zip', Body=zip_content) + conn = boto3.client('lambda', 'us-west-2') + + conn.create_function( + FunctionName='testFunction', + Runtime='python2.7', + Role='test-iam-role', + Handler='lambda_function.lambda_handler', + Code={ + 'S3Bucket': 'test-bucket', + 'S3Key': 'test.zip', + }, + Description='test lambda function', + Timeout=3, + MemorySize=128, + Publish=True, + ) + + conn.publish_version(FunctionName='testFunction') + + versions = conn.list_versions_by_function(FunctionName='testFunction') + + assert versions['Versions'][0]['FunctionArn'] == 'arn:aws:lambda:us-west-2:123456789012:function:testFunction:$LATEST' + diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 8bccae27a..520c7262c 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -717,3 +717,60 @@ def test_cancel_key_deletion(): assert result["KeyMetadata"]["Enabled"] == False assert result["KeyMetadata"]["KeyState"] == 'Disabled' assert 'DeletionDate' not in result["KeyMetadata"] + + +@mock_kms +def test_update_key_description(): + client = boto3.client('kms', region_name='us-east-1') + key = client.create_key(Description='old_description') + key_id = key['KeyMetadata']['KeyId'] + + result = client.update_key_description(KeyId=key_id, Description='new_description') + assert 'ResponseMetadata' in result + + +@mock_kms +def test_tag_resource(): + client = boto3.client('kms', region_name='us-east-1') + key = client.create_key(Description='cancel-key-deletion') + response = client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'] + ) + + keyid = response['KeyId'] + response = client.tag_resource( + KeyId=keyid, + Tags=[ + { + 'TagKey': 'string', + 'TagValue': 'string' + }, + ] + ) + + # Shouldn't have any data, just header + assert len(response.keys()) == 1 + + +@mock_kms +def test_list_resource_tags(): + client = boto3.client('kms', region_name='us-east-1') + key = client.create_key(Description='cancel-key-deletion') + response = client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'] + ) + + keyid = response['KeyId'] + response = client.tag_resource( + KeyId=keyid, + Tags=[ + { + 'TagKey': 'string', + 'TagValue': 'string' + }, + ] + ) + + response = client.list_resource_tags(KeyId=keyid) + assert response['Tags'][0]['TagKey'] == 'string' + assert response['Tags'][0]['TagValue'] == 'string' From 0a3ff94e66da2bacb27e8a788034ae2c8d0e29be Mon Sep 17 00:00:00 2001 From: William Rubel Date: Sat, 16 Feb 2019 09:37:27 -0600 Subject: [PATCH 18/52] Update kms and lambda to work with terraform --- moto/kms/models.py | 2 -- moto/kms/responses.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index fb4040617..9d13589c1 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -64,7 +64,6 @@ class Key(BaseModel): ) key.key_rotation_status = properties['EnableKeyRotation'] key.enabled = properties['Enabled'] - return key def get_cfn_attribute(self, attribute_name): @@ -93,7 +92,6 @@ class KmsBackend(BaseBackend): key = self.keys[self.get_key_id(key_id)] key.tags = tags - def list_resource_tags(self, key_id): key = self.keys[self.get_key_id(key_id)] return key.tags diff --git a/moto/kms/responses.py b/moto/kms/responses.py index b66ca267c..2674f765c 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -42,13 +42,13 @@ class KmsResponse(BaseResponse): key_id = self.parameters.get('KeyId') description = self.parameters.get('Description') - self.kms_backend.update_key_description(key_id,description) + self.kms_backend.update_key_description(key_id, description) return json.dumps(None) def tag_resource(self): key_id = self.parameters.get('KeyId') tags = self.parameters.get('Tags') - self.kms_backend.tag_resource(key_id,tags) + self.kms_backend.tag_resource(key_id, tags) return json.dumps({}) def list_resource_tags(self): From 6897a118ed59819c911642e946a5378bad53a3c7 Mon Sep 17 00:00:00 2001 From: Domenico Testa Date: Sat, 16 Feb 2019 16:42:00 +0100 Subject: [PATCH 19/52] Fixing S3 copy_key error when using unicode. --- moto/s3/responses.py | 5 +++-- tests/test_s3/test_s3.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 43f690bc8..e4f97cd43 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -19,7 +19,8 @@ from .exceptions import BucketAlreadyExists, S3ClientError, MissingBucket, Missi MalformedACLError, InvalidNotificationARN, InvalidNotificationEvent from .models import s3_backend, get_canned_acl, FakeGrantee, FakeGrant, FakeAcl, FakeKey, FakeTagging, FakeTagSet, \ FakeTag -from .utils import bucket_name_from_url, metadata_from_headers, parse_region_from_url +from .utils import bucket_name_from_url, clean_key_name, metadata_from_headers, \ + parse_region_from_url from xml.dom import minidom @@ -733,7 +734,7 @@ class ResponseObject(_TemplateEnvironmentMixin): # Copy key # you can have a quoted ?version=abc with a version Id, so work on # we need to parse the unquoted string first - src_key = request.headers.get("x-amz-copy-source") + src_key = clean_key_name(request.headers.get("x-amz-copy-source")) if isinstance(src_key, six.binary_type): src_key = src_key.decode('utf-8') src_key_parsed = urlparse(src_key) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 7a53804ff..eea720240 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -418,6 +418,22 @@ def test_copy_key(): "new-key").get_contents_as_string().should.equal(b"some value") +@mock_s3_deprecated +def test_copy_key_with_unicode(): + conn = boto.connect_s3('the_key', 'the_secret') + bucket = conn.create_bucket("foobar") + key = Key(bucket) + key.key = "the-unicode-💩-key" + key.set_contents_from_string("some value") + + bucket.copy_key('new-key', 'foobar', 'the-unicode-💩-key') + + bucket.get_key( + "the-unicode-💩-key").get_contents_as_string().should.equal(b"some value") + bucket.get_key( + "new-key").get_contents_as_string().should.equal(b"some value") + + @mock_s3_deprecated def test_copy_key_with_version(): conn = boto.connect_s3('the_key', 'the_secret') From 5372e6840fabdd0394135e647dc07f8c13178a31 Mon Sep 17 00:00:00 2001 From: William Rubel Date: Sat, 16 Feb 2019 12:37:46 -0600 Subject: [PATCH 20/52] Increasing code coverage --- moto/awslambda/responses.py | 8 +++-- tests/test_awslambda/test_lambda.py | 53 +++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/moto/awslambda/responses.py b/moto/awslambda/responses.py index bcd2da903..d4eb73bc3 100644 --- a/moto/awslambda/responses.py +++ b/moto/awslambda/responses.py @@ -160,9 +160,11 @@ class LambdaResponse(BaseResponse): 'Versions': [] } - for fn in self.lambda_backend.list_versions_by_function(function_name): - json_data = fn.get_configuration() - result['Versions'].append(json_data) + functions = self.lambda_backend.list_versions_by_function(function_name) + if functions: + for fn in functions: + json_data = fn.get_configuration() + result['Versions'].append(json_data) return 200, {}, json.dumps(result) diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index 9c5120f51..c05f9f0ac 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -853,3 +853,56 @@ def test_list_versions_by_function(): assert versions['Versions'][0]['FunctionArn'] == 'arn:aws:lambda:us-west-2:123456789012:function:testFunction:$LATEST' + + +@mock_lambda +@mock_s3 +def test_create_function_with_already_exists(): + s3_conn = boto3.client('s3', 'us-west-2') + s3_conn.create_bucket(Bucket='test-bucket') + + zip_content = get_test_zip_file2() + s3_conn.put_object(Bucket='test-bucket', Key='test.zip', Body=zip_content) + conn = boto3.client('lambda', 'us-west-2') + + conn.create_function( + FunctionName='testFunction', + Runtime='python2.7', + Role='test-iam-role', + Handler='lambda_function.lambda_handler', + Code={ + 'S3Bucket': 'test-bucket', + 'S3Key': 'test.zip', + }, + Description='test lambda function', + Timeout=3, + MemorySize=128, + Publish=True, + ) + + response = conn.create_function( + FunctionName='testFunction', + Runtime='python2.7', + Role='test-iam-role', + Handler='lambda_function.lambda_handler', + Code={ + 'S3Bucket': 'test-bucket', + 'S3Key': 'test.zip', + }, + Description='test lambda function', + Timeout=3, + MemorySize=128, + Publish=True, + ) + + assert response['FunctionName'] == 'testFunction' + + +@mock_lambda +@mock_s3 +def test_list_versions_by_function_for_nonexistent_function(): + conn = boto3.client('lambda', 'us-west-2') + + versions = conn.list_versions_by_function(FunctionName='testFunction') + + assert len(versions['Versions']) == 0 \ No newline at end of file From 8ad28f8400b6deae46cb666c56758278035a2478 Mon Sep 17 00:00:00 2001 From: William Rubel Date: Sat, 16 Feb 2019 20:53:27 -0600 Subject: [PATCH 21/52] Adding additional tests to increase coverage --- file.tmp | 9 ++++ moto/packages/httpretty/core.py | 60 +++++++++++++++++++++++++++ moto/packages/httpretty/http.py | 3 +- tests/test_packages/__init__.py | 8 ++++ tests/test_packages/test_httpretty.py | 37 +++++++++++++++++ 5 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 file.tmp create mode 100644 tests/test_packages/__init__.py create mode 100644 tests/test_packages/test_httpretty.py diff --git a/file.tmp b/file.tmp new file mode 100644 index 000000000..80053c647 --- /dev/null +++ b/file.tmp @@ -0,0 +1,9 @@ + + AWSTemplateFormatVersion: '2010-09-09' + Description: Simple CloudFormation Test Template + Resources: + S3Bucket: + Type: AWS::S3::Bucket + Properties: + AccessControl: PublicRead + BucketName: cf-test-bucket-1 diff --git a/moto/packages/httpretty/core.py b/moto/packages/httpretty/core.py index 8ad9168a5..168f18431 100644 --- a/moto/packages/httpretty/core.py +++ b/moto/packages/httpretty/core.py @@ -72,6 +72,19 @@ from datetime import datetime from datetime import timedelta from errno import EAGAIN +import logging +from inspect import currentframe +import inspect + +logging.basicConfig(filename='/tmp/models.log',level=logging.DEBUG) + +DEBUG=0 + +def get_linenumber(): + cf = currentframe() + return " - "+str(cf.f_back.f_lineno) + + # Some versions of python internally shadowed the # SocketType variable incorrectly https://bugs.python.org/issue20386 BAD_SOCKET_SHADOW = socket.socket != socket.SocketType @@ -155,15 +168,34 @@ class HTTPrettyRequest(BaseHTTPRequestHandler, BaseClass): """ def __init__(self, headers, body=''): + + if DEBUG: + logging.debug('__init__ - ' + ' -caller: ' + str( + inspect.stack()[1][3]) + "-" + get_linenumber()) + logging.debug('headers: '+str(headers)) + # first of all, lets make sure that if headers or body are # unicode strings, it must be converted into a utf-8 encoded # byte string self.raw_headers = utf8(headers.strip()) + + if DEBUG: + logging.debug('raw_headers: '+str(self.raw_headers)) + self.body = utf8(body) + if DEBUG: + logging.debug('body: '+str(self.body)) + # Now let's concatenate the headers with the body, and create # `rfile` based on it self.rfile = StringIO(b'\r\n\r\n'.join([self.raw_headers, self.body])) + + if DEBUG: + logging.debug('rfile: '+str(self.rfile)) + + self.wfile = StringIO() # Creating `wfile` as an empty # StringIO, just to avoid any real # I/O calls @@ -171,6 +203,10 @@ class HTTPrettyRequest(BaseHTTPRequestHandler, BaseClass): # parsing the request line preemptively self.raw_requestline = self.rfile.readline() + if DEBUG: + logging.debug('raw_requestline: '+str(self.raw_requestline)) + + # initiating the error attributes with None self.error_code = None self.error_message = None @@ -182,6 +218,9 @@ class HTTPrettyRequest(BaseHTTPRequestHandler, BaseClass): # making the HTTP method string available as the command self.method = self.command + if DEBUG: + logging.debug('method: '+str(self.method)) + # Now 2 convenient attributes for the HTTPretty API: # `querystring` holds a dictionary with the parsed query string @@ -207,8 +246,23 @@ class HTTPrettyRequest(BaseHTTPRequestHandler, BaseClass): ) def parse_querystring(self, qs): + + if DEBUG: + logging.debug('parse_querystring - ' + ' -caller: ' + str( + inspect.stack()[1][3]) + "-" + get_linenumber()) + logging.debug('qs: '+str(qs)) + expanded = unquote_utf8(qs) + + if DEBUG: + logging.debug('expanded: '+str(expanded)) + parsed = parse_qs(expanded) + + if DEBUG: + logging.debug('parsed: '+str(parsed)) + result = {} for k in parsed: result[k] = list(map(decode_utf8, parsed[k])) @@ -218,6 +272,12 @@ class HTTPrettyRequest(BaseHTTPRequestHandler, BaseClass): def parse_request_body(self, body): """ Attempt to parse the post based on the content-type passed. Return the regular body if not """ + if DEBUG: + logging.debug('parse_request_body - ' + ' -caller: ' + str( + inspect.stack()[1][3]) + "-" + get_linenumber()) + logging.debug('body: '+str(body)) + PARSING_FUNCTIONS = { 'application/json': json.loads, 'text/json': json.loads, diff --git a/moto/packages/httpretty/http.py b/moto/packages/httpretty/http.py index 7e9a56885..17d580f5f 100644 --- a/moto/packages/httpretty/http.py +++ b/moto/packages/httpretty/http.py @@ -29,7 +29,6 @@ import re from .compat import BaseClass from .utils import decode_utf8 - STATUSES = { 100: "Continue", 101: "Switching Protocols", @@ -134,6 +133,7 @@ def parse_requestline(s): ... ValueError: Not a Request-Line """ + methods = '|'.join(HttpBaseClass.METHODS) m = re.match(r'(' + methods + ')\s+(.*)\s+HTTP/(1.[0|1])', s, re.I) if m: @@ -146,6 +146,7 @@ def last_requestline(sent_data): """ Find the last line in sent_data that can be parsed with parse_requestline """ + for line in reversed(sent_data): try: parse_requestline(decode_utf8(line)) diff --git a/tests/test_packages/__init__.py b/tests/test_packages/__init__.py new file mode 100644 index 000000000..bf582e0b3 --- /dev/null +++ b/tests/test_packages/__init__.py @@ -0,0 +1,8 @@ +from __future__ import unicode_literals + +import logging +# Disable extra logging for tests +logging.getLogger('boto').setLevel(logging.CRITICAL) +logging.getLogger('boto3').setLevel(logging.CRITICAL) +logging.getLogger('botocore').setLevel(logging.CRITICAL) +logging.getLogger('nose').setLevel(logging.CRITICAL) diff --git a/tests/test_packages/test_httpretty.py b/tests/test_packages/test_httpretty.py new file mode 100644 index 000000000..48277a2de --- /dev/null +++ b/tests/test_packages/test_httpretty.py @@ -0,0 +1,37 @@ +# #!/usr/bin/env python +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +import mock + +from moto.packages.httpretty.core import HTTPrettyRequest, fake_gethostname, fake_gethostbyname + + +def test_parse_querystring(): + + core = HTTPrettyRequest(headers='test test HTTP/1.1') + + qs = 'test test' + response = core.parse_querystring(qs) + + assert response == {} + +def test_parse_request_body(): + core = HTTPrettyRequest(headers='test test HTTP/1.1') + + qs = 'test' + response = core.parse_request_body(qs) + + assert response == 'test' + +def test_fake_gethostname(): + + response = fake_gethostname() + + assert response == 'localhost' + +def test_fake_gethostbyname(): + + host = 'test' + response = fake_gethostbyname(host=host) + + assert response == '127.0.0.1' \ No newline at end of file From b29fd4a997a105f6c3a3b19a87852a56e75db65b Mon Sep 17 00:00:00 2001 From: William Rubel Date: Sat, 16 Feb 2019 20:53:54 -0600 Subject: [PATCH 22/52] Adding additional tests to increase coverage --- file.tmp | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 file.tmp diff --git a/file.tmp b/file.tmp deleted file mode 100644 index 80053c647..000000000 --- a/file.tmp +++ /dev/null @@ -1,9 +0,0 @@ - - AWSTemplateFormatVersion: '2010-09-09' - Description: Simple CloudFormation Test Template - Resources: - S3Bucket: - Type: AWS::S3::Bucket - Properties: - AccessControl: PublicRead - BucketName: cf-test-bucket-1 From e35d99ff09023e09b2e252db4ac1e1d58863dfb5 Mon Sep 17 00:00:00 2001 From: William Rubel Date: Sun, 17 Feb 2019 09:25:35 -0600 Subject: [PATCH 23/52] Trying to improve coverage --- file.tmp | 9 ++++++ tests/test_batch/test_batch.py | 59 ++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 file.tmp diff --git a/file.tmp b/file.tmp new file mode 100644 index 000000000..80053c647 --- /dev/null +++ b/file.tmp @@ -0,0 +1,9 @@ + + AWSTemplateFormatVersion: '2010-09-09' + Description: Simple CloudFormation Test Template + Resources: + S3Bucket: + Type: AWS::S3::Bucket + Properties: + AccessControl: PublicRead + BucketName: cf-test-bucket-1 diff --git a/tests/test_batch/test_batch.py b/tests/test_batch/test_batch.py index ec24cd911..310ac0b48 100644 --- a/tests/test_batch/test_batch.py +++ b/tests/test_batch/test_batch.py @@ -323,6 +323,54 @@ def test_create_job_queue(): resp.should.contain('jobQueues') len(resp['jobQueues']).should.equal(0) + # Create job queue which already exists + try: + resp = batch_client.create_job_queue( + jobQueueName='test_job_queue', + state='ENABLED', + priority=123, + computeEnvironmentOrder=[ + { + 'order': 123, + 'computeEnvironment': arn + }, + ] + ) + + except ClientError as err: + err.response['Error']['Code'].should.equal('ClientException') + + + # Create job queue with incorrect state + try: + resp = batch_client.create_job_queue( + jobQueueName='test_job_queue2', + state='JUNK', + priority=123, + computeEnvironmentOrder=[ + { + 'order': 123, + 'computeEnvironment': arn + }, + ] + ) + + except ClientError as err: + err.response['Error']['Code'].should.equal('ClientException') + + # Create job queue with no compute env + try: + resp = batch_client.create_job_queue( + jobQueueName='test_job_queue3', + state='JUNK', + priority=123, + computeEnvironmentOrder=[ + + ] + ) + + except ClientError as err: + err.response['Error']['Code'].should.equal('ClientException') @mock_ec2 @mock_ecs @@ -397,6 +445,17 @@ def test_update_job_queue(): len(resp['jobQueues']).should.equal(1) resp['jobQueues'][0]['priority'].should.equal(5) + batch_client.update_job_queue( + jobQueue='test_job_queue', + priority=5 + ) + + resp = batch_client.describe_job_queues() + resp.should.contain('jobQueues') + len(resp['jobQueues']).should.equal(1) + resp['jobQueues'][0]['priority'].should.equal(5) + + @mock_ec2 @mock_ecs From 921a993330a96130b12331f29944d93578a8bd2c Mon Sep 17 00:00:00 2001 From: William Rubel Date: Sun, 17 Feb 2019 14:30:43 -0600 Subject: [PATCH 24/52] cleaning up files --- file.tmp | 9 ---- moto/iam/models.py | 13 +++-- moto/iam/responses.py | 79 +++++++++++++++++++++++++++-- moto/packages/httpretty/core.py | 62 +--------------------- moto/packages/httpretty/http.py | 2 - tests/test_awslambda/test_lambda.py | 2 - tests/test_kms/test_kms.py | 1 - 7 files changed, 85 insertions(+), 83 deletions(-) delete mode 100644 file.tmp diff --git a/file.tmp b/file.tmp deleted file mode 100644 index 80053c647..000000000 --- a/file.tmp +++ /dev/null @@ -1,9 +0,0 @@ - - AWSTemplateFormatVersion: '2010-09-09' - Description: Simple CloudFormation Test Template - Resources: - S3Bucket: - Type: AWS::S3::Bucket - Properties: - AccessControl: PublicRead - BucketName: cf-test-bucket-1 diff --git a/moto/iam/models.py b/moto/iam/models.py index 86a5d7a32..a5f40b996 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -51,8 +51,8 @@ class Policy(BaseModel): self.default_version_id = default_version_id or 'v1' self.versions = [PolicyVersion(self.arn, document, True)] - self.create_datetime = datetime.now(pytz.utc) - self.update_datetime = datetime.now(pytz.utc) + self.create_datetime = datetime.strftime(datetime.utcnow(), "%Y-%m-%dT%H:%M:%SZ") + self.update_datetime = datetime.strftime(datetime.utcnow(), "%Y-%m-%dT%H:%M:%SZ") class SAMLProvider(BaseModel): @@ -76,7 +76,7 @@ class PolicyVersion(object): self.is_default = is_default self.version_id = 'v1' - self.create_datetime = datetime.now(pytz.utc) + self.create_datetime = datetime.strftime(datetime.utcnow(), "%Y-%m-%dT%H:%M:%SZ") class ManagedPolicy(Policy): @@ -132,8 +132,9 @@ class Role(BaseModel): self.path = path or '/' self.policies = {} self.managed_policies = {} - self.create_date = datetime.now(pytz.utc) + self.create_date = datetime.strftime(datetime.utcnow(), "%Y-%m-%dT%H:%M:%SZ") self.tags = {} + self.description = "" @classmethod def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): @@ -473,6 +474,10 @@ class IAMBackend(BaseBackend): policy = arns[policy_arn] policy.attach_to(self.get_role(role_name)) + def update_role_description(self, role_name, role_description): + role = self.get_role(role_name) + role.description = role_description + def detach_role_policy(self, policy_arn, role_name): arns = dict((p.arn, p) for p in self.managed_policies.values()) try: diff --git a/moto/iam/responses.py b/moto/iam/responses.py index e3cc4b90b..e624c18c3 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -107,6 +107,10 @@ class IamResponse(BaseResponse): template = self.response_template(LIST_POLICIES_TEMPLATE) return template.render(policies=policies, marker=marker) + def list_entities_for_policy(self): + template = self.response_template(LIST_ENTITIES_FOR_POLICY_TEMPLATE) + return template.render() + def create_role(self): role_name = self._get_param('RoleName') path = self._get_param('Path') @@ -169,6 +173,20 @@ class IamResponse(BaseResponse): template = self.response_template(GENERIC_EMPTY_TEMPLATE) return template.render(name="UpdateAssumeRolePolicyResponse") + def update_role_description(self): + role_name = self._get_param('RoleName') + description = self._get_param('Description') + role = iam_backend.update_role_description(role_name,description) + template = self.response_template(UPDATE_ROLE_DESCRIPTION_TEMPLATE) + return template.render(role=role) + + def update_role(self): + role_name = self._get_param('RoleName') + description = self._get_param('Description') + role = iam_backend.update_role_description(role_name,description) + template = self.response_template(UPDATE_ROLE_DESCRIPTION_TEMPLATE) + return template.render(role=role) + def create_policy_version(self): policy_arn = self._get_param('PolicyArn') policy_document = self._get_param('PolicyDocument') @@ -654,6 +672,33 @@ class IamResponse(BaseResponse): template = self.response_template(UNTAG_ROLE_TEMPLATE) return template.render() +LIST_ENTITIES_FOR_POLICY_TEMPLATE = """ + + + + DevRole + + + + + Dev + + + false + + + Alice + + + Bob + + + + + eb358e22-9d1f-11e4-93eb-190ecEXAMPLE + +""" + ATTACH_ROLE_POLICY_TEMPLATE = """ @@ -696,12 +741,12 @@ CREATE_POLICY_TEMPLATE = """ {{ policy.arn }} {{ policy.attachment_count }} - {{ policy.create_datetime.isoformat() }} + {{ policy.create_datetime }} {{ policy.default_version_id }} {{ policy.path }} {{ policy.id }} {{ policy.name }} - {{ policy.update_datetime.isoformat() }} + {{ policy.update_datetime }} @@ -719,8 +764,8 @@ GET_POLICY_TEMPLATE = """ {{ policy.path }} {{ policy.arn }} {{ policy.attachment_count }} - {{ policy.create_datetime.isoformat() }} - {{ policy.update_datetime.isoformat() }} + {{ policy.create_datetime }} + {{ policy.update_datetime }} @@ -898,6 +943,32 @@ GET_ROLE_POLICY_TEMPLATE = """ + + + {{ role.path }} + {{ role.arn }} + {{ role.name }} + {{ role.assume_role_policy_document }} + {{ role.create_date }} + {{ role.id }} + {% if role.tags %} + + {% for tag in role.get_tags() %} + + {{ tag['Key'] }} + {{ tag['Value'] }} + + {% endfor %} + + {% endif %} + + + + df37e965-9967-11e1-a4c3-270EXAMPLE04 + +""" + GET_ROLE_TEMPLATE = """ diff --git a/moto/packages/httpretty/core.py b/moto/packages/httpretty/core.py index 168f18431..4eb92108f 100644 --- a/moto/packages/httpretty/core.py +++ b/moto/packages/httpretty/core.py @@ -72,19 +72,6 @@ from datetime import datetime from datetime import timedelta from errno import EAGAIN -import logging -from inspect import currentframe -import inspect - -logging.basicConfig(filename='/tmp/models.log',level=logging.DEBUG) - -DEBUG=0 - -def get_linenumber(): - cf = currentframe() - return " - "+str(cf.f_back.f_lineno) - - # Some versions of python internally shadowed the # SocketType variable incorrectly https://bugs.python.org/issue20386 BAD_SOCKET_SHADOW = socket.socket != socket.SocketType @@ -168,34 +155,15 @@ class HTTPrettyRequest(BaseHTTPRequestHandler, BaseClass): """ def __init__(self, headers, body=''): - - if DEBUG: - logging.debug('__init__ - ' - ' -caller: ' + str( - inspect.stack()[1][3]) + "-" + get_linenumber()) - logging.debug('headers: '+str(headers)) - # first of all, lets make sure that if headers or body are # unicode strings, it must be converted into a utf-8 encoded # byte string self.raw_headers = utf8(headers.strip()) - - if DEBUG: - logging.debug('raw_headers: '+str(self.raw_headers)) - self.body = utf8(body) - if DEBUG: - logging.debug('body: '+str(self.body)) - # Now let's concatenate the headers with the body, and create # `rfile` based on it self.rfile = StringIO(b'\r\n\r\n'.join([self.raw_headers, self.body])) - - if DEBUG: - logging.debug('rfile: '+str(self.rfile)) - - self.wfile = StringIO() # Creating `wfile` as an empty # StringIO, just to avoid any real # I/O calls @@ -203,10 +171,6 @@ class HTTPrettyRequest(BaseHTTPRequestHandler, BaseClass): # parsing the request line preemptively self.raw_requestline = self.rfile.readline() - if DEBUG: - logging.debug('raw_requestline: '+str(self.raw_requestline)) - - # initiating the error attributes with None self.error_code = None self.error_message = None @@ -218,9 +182,6 @@ class HTTPrettyRequest(BaseHTTPRequestHandler, BaseClass): # making the HTTP method string available as the command self.method = self.command - if DEBUG: - logging.debug('method: '+str(self.method)) - # Now 2 convenient attributes for the HTTPretty API: # `querystring` holds a dictionary with the parsed query string @@ -246,23 +207,8 @@ class HTTPrettyRequest(BaseHTTPRequestHandler, BaseClass): ) def parse_querystring(self, qs): - - if DEBUG: - logging.debug('parse_querystring - ' - ' -caller: ' + str( - inspect.stack()[1][3]) + "-" + get_linenumber()) - logging.debug('qs: '+str(qs)) - expanded = unquote_utf8(qs) - - if DEBUG: - logging.debug('expanded: '+str(expanded)) - parsed = parse_qs(expanded) - - if DEBUG: - logging.debug('parsed: '+str(parsed)) - result = {} for k in parsed: result[k] = list(map(decode_utf8, parsed[k])) @@ -272,12 +218,6 @@ class HTTPrettyRequest(BaseHTTPRequestHandler, BaseClass): def parse_request_body(self, body): """ Attempt to parse the post based on the content-type passed. Return the regular body if not """ - if DEBUG: - logging.debug('parse_request_body - ' - ' -caller: ' + str( - inspect.stack()[1][3]) + "-" + get_linenumber()) - logging.debug('body: '+str(body)) - PARSING_FUNCTIONS = { 'application/json': json.loads, 'text/json': json.loads, @@ -1173,4 +1113,4 @@ def httprettified(test): if isinstance(test, ClassTypes): return decorate_class(test) - return decorate_callable(test) + return decorate_callable(test) \ No newline at end of file diff --git a/moto/packages/httpretty/http.py b/moto/packages/httpretty/http.py index 17d580f5f..ee1625905 100644 --- a/moto/packages/httpretty/http.py +++ b/moto/packages/httpretty/http.py @@ -133,7 +133,6 @@ def parse_requestline(s): ... ValueError: Not a Request-Line """ - methods = '|'.join(HttpBaseClass.METHODS) m = re.match(r'(' + methods + ')\s+(.*)\s+HTTP/(1.[0|1])', s, re.I) if m: @@ -146,7 +145,6 @@ def last_requestline(sent_data): """ Find the last line in sent_data that can be parsed with parse_requestline """ - for line in reversed(sent_data): try: parse_requestline(decode_utf8(line)) diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index c05f9f0ac..71f6746a9 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -821,7 +821,6 @@ def get_function_policy(): assert res['Statement'][0]['Action'] == 'lambda:InvokeFunction' - @mock_lambda @mock_s3 def test_list_versions_by_function(): @@ -854,7 +853,6 @@ def test_list_versions_by_function(): assert versions['Versions'][0]['FunctionArn'] == 'arn:aws:lambda:us-west-2:123456789012:function:testFunction:$LATEST' - @mock_lambda @mock_s3 def test_create_function_with_already_exists(): diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 520c7262c..0f7bab4cd 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -1,6 +1,5 @@ from __future__ import unicode_literals import os, re - import boto3 import boto.kms from boto.exception import JSONResponseError From 37cb5ab4e69f1ef977256aba07f38f546d40e918 Mon Sep 17 00:00:00 2001 From: William Rubel Date: Sun, 17 Feb 2019 14:36:53 -0600 Subject: [PATCH 25/52] Add test for roles --- moto/iam/responses.py | 5 +++-- tests/test_iam/test_iam.py | 27 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index e624c18c3..c981f9b35 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -176,14 +176,14 @@ class IamResponse(BaseResponse): def update_role_description(self): role_name = self._get_param('RoleName') description = self._get_param('Description') - role = iam_backend.update_role_description(role_name,description) + role = iam_backend.update_role_description(role_name, description) template = self.response_template(UPDATE_ROLE_DESCRIPTION_TEMPLATE) return template.render(role=role) def update_role(self): role_name = self._get_param('RoleName') description = self._get_param('Description') - role = iam_backend.update_role_description(role_name,description) + role = iam_backend.update_role_description(role_name, description) template = self.response_template(UPDATE_ROLE_DESCRIPTION_TEMPLATE) return template.render(role=role) @@ -672,6 +672,7 @@ class IamResponse(BaseResponse): template = self.response_template(UNTAG_ROLE_TEMPLATE) return template.render() + LIST_ENTITIES_FOR_POLICY_TEMPLATE = """ diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index d5f1bb4f9..80a7fe99e 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -1151,3 +1151,30 @@ def test_untag_role(): # With a role that doesn't exist: with assert_raises(ClientError): conn.untag_role(RoleName='notarole', TagKeys=['somevalue']) + + +@mock_iam() +def test_update_role_description(): + conn = boto3.client('iam', region_name='us-east-1') + + with assert_raises(ClientError): + conn.delete_role(RoleName="my-role") + + conn.create_role(RoleName="my-role", AssumeRolePolicyDocument="some policy", Path="/my-path/") + role = conn.get_role(RoleName="my-role") + response = conn.update_role_description(RoleName="my-role", Description="test") + + assert response['Role']['RoleName'] == 'my-role' + +@mock_iam() +def test_update_role(): + conn = boto3.client('iam', region_name='us-east-1') + + with assert_raises(ClientError): + conn.delete_role(RoleName="my-role") + + conn.create_role(RoleName="my-role", AssumeRolePolicyDocument="some policy", Path="/my-path/") + role = conn.get_role(RoleName="my-role") + response = conn.update_role_description(RoleName="my-role", Description="test") + + assert response['Role']['RoleName'] == 'my-role' \ No newline at end of file From 31258e9e9e35f254259ad20759a39a88423266fe Mon Sep 17 00:00:00 2001 From: William Rubel Date: Sun, 17 Feb 2019 15:23:59 -0600 Subject: [PATCH 26/52] Add test for roles --- moto/iam/models.py | 8 ++++---- moto/iam/responses.py | 12 ++++++------ tests/test_iam/test_iam.py | 4 +--- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/moto/iam/models.py b/moto/iam/models.py index a5f40b996..e0f725299 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -51,8 +51,8 @@ class Policy(BaseModel): self.default_version_id = default_version_id or 'v1' self.versions = [PolicyVersion(self.arn, document, True)] - self.create_datetime = datetime.strftime(datetime.utcnow(), "%Y-%m-%dT%H:%M:%SZ") - self.update_datetime = datetime.strftime(datetime.utcnow(), "%Y-%m-%dT%H:%M:%SZ") + self.create_datetime = datetime.now(pytz.utc) + self.update_datetime = datetime.now(pytz.utc) class SAMLProvider(BaseModel): @@ -76,7 +76,7 @@ class PolicyVersion(object): self.is_default = is_default self.version_id = 'v1' - self.create_datetime = datetime.strftime(datetime.utcnow(), "%Y-%m-%dT%H:%M:%SZ") + self.create_datetime = datetime.now(pytz.utc) class ManagedPolicy(Policy): @@ -132,7 +132,7 @@ class Role(BaseModel): self.path = path or '/' self.policies = {} self.managed_policies = {} - self.create_date = datetime.strftime(datetime.utcnow(), "%Y-%m-%dT%H:%M:%SZ") + self.create_date = datetime.now(pytz.utc) self.tags = {} self.description = "" diff --git a/moto/iam/responses.py b/moto/iam/responses.py index c981f9b35..5be07430f 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -742,12 +742,12 @@ CREATE_POLICY_TEMPLATE = """ {{ policy.arn }} {{ policy.attachment_count }} - {{ policy.create_datetime }} + {{ policy.create_datetime.isoformat() }} {{ policy.default_version_id }} {{ policy.path }} {{ policy.id }} {{ policy.name }} - {{ policy.update_datetime }} + {{ policy.update_datetime.isoformat() }} @@ -765,8 +765,8 @@ GET_POLICY_TEMPLATE = """ {{ policy.path }} {{ policy.arn }} {{ policy.attachment_count }} - {{ policy.create_datetime }} - {{ policy.update_datetime }} + {{ policy.create_datetime.isoformat() }} + {{ policy.update_datetime.isoformat() }} @@ -858,7 +858,7 @@ LIST_POLICIES_TEMPLATE = """ {{ policy.path }} {{ policy.id }} {{ policy.name }} - {{ policy.update_datetime.isoformat() }} + {{ policy.update_datetime }} {% endfor %} @@ -951,7 +951,7 @@ UPDATE_ROLE_DESCRIPTION_TEMPLATE = """ Date: Sun, 17 Feb 2019 15:35:49 -0600 Subject: [PATCH 27/52] Add test for roles --- moto/iam/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/iam/models.py b/moto/iam/models.py index e0f725299..f71ad5352 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -477,6 +477,7 @@ class IAMBackend(BaseBackend): def update_role_description(self, role_name, role_description): role = self.get_role(role_name) role.description = role_description + return role def detach_role_policy(self, policy_arn, role_name): arns = dict((p.arn, p) for p in self.managed_policies.values()) From 6e7bd088b30452bff0f1601cde089cdfd9a8ee4d Mon Sep 17 00:00:00 2001 From: William Rubel Date: Sun, 17 Feb 2019 16:04:28 -0600 Subject: [PATCH 28/52] Add test for roles --- tests/test_iam/test_iam.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 85c51012f..f728f0dca 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -1176,3 +1176,21 @@ def test_update_role(): response = conn.update_role_description(RoleName="my-role", Description="test") assert response['Role']['RoleName'] == 'my-role' + +@mock_iam() +def test_list_entities_for_policy(): + conn = boto3.client('iam', region_name='us-east-1') + + with assert_raises(ClientError): + conn.delete_role(RoleName="my-role") + + conn.create_role(RoleName="my-role", AssumeRolePolicyDocument="some policy", Path="/my-path/") + + role = conn.get_role(RoleName="my-role") + arn = role.get('Role').get('Arn') + + response = conn.list_entities_for_policy( + PolicyArn=arn + ) + + assert response['PolicyGroups'][0]['GroupName'] == 'Dev' From 0e73cddf2fb8039e056b4b67a6948f8c4eefcae9 Mon Sep 17 00:00:00 2001 From: William Rubel Date: Sun, 17 Feb 2019 16:25:49 -0600 Subject: [PATCH 29/52] Add test for roles --- tests/test_iam/test_iam.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index f728f0dca..be25c2e96 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -1174,7 +1174,6 @@ def test_update_role(): conn.create_role(RoleName="my-role", AssumeRolePolicyDocument="some policy", Path="/my-path/") response = conn.update_role_description(RoleName="my-role", Description="test") - assert response['Role']['RoleName'] == 'my-role' @mock_iam() @@ -1192,5 +1191,4 @@ def test_list_entities_for_policy(): response = conn.list_entities_for_policy( PolicyArn=arn ) - assert response['PolicyGroups'][0]['GroupName'] == 'Dev' From 63b692356d07a65468ae0f507d5390397e07663c Mon Sep 17 00:00:00 2001 From: William Rubel Date: Sun, 17 Feb 2019 16:49:54 -0600 Subject: [PATCH 30/52] Fix policy date --- moto/iam/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 5be07430f..7ee4b6345 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -858,7 +858,7 @@ LIST_POLICIES_TEMPLATE = """ {{ policy.path }} {{ policy.id }} {{ policy.name }} - {{ policy.update_datetime }} + {{ policy.update_datetime.isoformat() }} {% endfor %} From 37a765db8d13faa30d64240ff2dc7b098c0237d7 Mon Sep 17 00:00:00 2001 From: William Rubel Date: Sun, 17 Feb 2019 17:12:27 -0600 Subject: [PATCH 31/52] Fix policy date --- tests/test_iam/test_iam.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index be25c2e96..1114f72de 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -1176,6 +1176,17 @@ def test_update_role(): response = conn.update_role_description(RoleName="my-role", Description="test") assert response['Role']['RoleName'] == 'my-role' +@mock_iam() +def test_update_role(): + conn = boto3.client('iam', region_name='us-east-1') + + with assert_raises(ClientError): + conn.delete_role(RoleName="my-role") + + conn.create_role(RoleName="my-role", AssumeRolePolicyDocument="some policy", Path="/my-path/") + response = conn.update_role(RoleName="my-role", Description="test") + assert response['Role']['RoleName'] == 'my-role' + @mock_iam() def test_list_entities_for_policy(): conn = boto3.client('iam', region_name='us-east-1') From ce7b5ebf54021f64cf855f17f60a77f4b09262ce Mon Sep 17 00:00:00 2001 From: William Rubel Date: Sun, 17 Feb 2019 21:37:33 -0600 Subject: [PATCH 32/52] Fix policy date --- file.tmp | 9 +++++++++ moto/iam/models.py | 5 +++++ moto/iam/responses.py | 12 ++++++++++-- tests/test_iam/test_iam.py | 2 +- 4 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 file.tmp diff --git a/file.tmp b/file.tmp new file mode 100644 index 000000000..80053c647 --- /dev/null +++ b/file.tmp @@ -0,0 +1,9 @@ + + AWSTemplateFormatVersion: '2010-09-09' + Description: Simple CloudFormation Test Template + Resources: + S3Bucket: + Type: AWS::S3::Bucket + Properties: + AccessControl: PublicRead + BucketName: cf-test-bucket-1 diff --git a/moto/iam/models.py b/moto/iam/models.py index f71ad5352..80c7da29a 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -479,6 +479,11 @@ class IAMBackend(BaseBackend): role.description = role_description return role + def update_role(self, role_name, role_description): + role = self.get_role(role_name) + role.description = role_description + return role + def detach_role_policy(self, policy_arn, role_name): arns = dict((p.arn, p) for p in self.managed_policies.values()) try: diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 7ee4b6345..26781ab6f 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -183,8 +183,8 @@ class IamResponse(BaseResponse): def update_role(self): role_name = self._get_param('RoleName') description = self._get_param('Description') - role = iam_backend.update_role_description(role_name, description) - template = self.response_template(UPDATE_ROLE_DESCRIPTION_TEMPLATE) + role = iam_backend.update_role(role_name, description) + template = self.response_template(UPDATE_ROLE_TEMPLATE) return template.render(role=role) def create_policy_version(self): @@ -944,6 +944,14 @@ GET_ROLE_POLICY_TEMPLATE = """ + + + + df37e965-9967-11e1-a4c3-270EXAMPLE04 + +""" + UPDATE_ROLE_DESCRIPTION_TEMPLATE = """ diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 1114f72de..77ba17a5a 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -1185,7 +1185,7 @@ def test_update_role(): conn.create_role(RoleName="my-role", AssumeRolePolicyDocument="some policy", Path="/my-path/") response = conn.update_role(RoleName="my-role", Description="test") - assert response['Role']['RoleName'] == 'my-role' + assert len(response.keys()) == 1 @mock_iam() def test_list_entities_for_policy(): From 8048e39dc0053d3772fc9d586b05f7fe7fa338be Mon Sep 17 00:00:00 2001 From: William Rubel Date: Sun, 17 Feb 2019 22:32:39 -0600 Subject: [PATCH 33/52] Fix policy date --- tests/test_awslambda/test_lambda.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index 71f6746a9..171f6360a 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -12,6 +12,8 @@ import sure # noqa from freezegun import freeze_time from moto import mock_lambda, mock_s3, mock_ec2, mock_sns, mock_logs, settings +from nose.tools import assert_raises +from botocore.exceptions import ClientError _lambda_region = 'us-west-2' @@ -397,6 +399,11 @@ def test_get_function(): result = conn.get_function(FunctionName='testFunction', Qualifier='$LATEST') result['Configuration']['Version'].should.equal('$LATEST') + # Test get function when can't find function name + with assert_raises(ClientError): + conn.get_function(FunctionName='junk', Qualifier='$LATEST') + + @mock_lambda @mock_s3 From e229d248ad4d207a00c84ac441ab3517157b3444 Mon Sep 17 00:00:00 2001 From: William Rubel Date: Mon, 18 Feb 2019 08:52:37 -0600 Subject: [PATCH 34/52] Trying to improve coverage --- file.tmp | 9 --------- tests/test_awslambda/test_lambda.py | 2 +- tests/test_events/test_events.py | 8 +++++++- 3 files changed, 8 insertions(+), 11 deletions(-) delete mode 100644 file.tmp diff --git a/file.tmp b/file.tmp deleted file mode 100644 index 80053c647..000000000 --- a/file.tmp +++ /dev/null @@ -1,9 +0,0 @@ - - AWSTemplateFormatVersion: '2010-09-09' - Description: Simple CloudFormation Test Template - Resources: - S3Bucket: - Type: AWS::S3::Bucket - Properties: - AccessControl: PublicRead - BucketName: cf-test-bucket-1 diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index 171f6360a..435d394e3 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -910,4 +910,4 @@ def test_list_versions_by_function_for_nonexistent_function(): versions = conn.list_versions_by_function(FunctionName='testFunction') - assert len(versions['Versions']) == 0 \ No newline at end of file + assert len(versions['Versions']) == 0 diff --git a/tests/test_events/test_events.py b/tests/test_events/test_events.py index 80630c5b8..810e0bbe5 100644 --- a/tests/test_events/test_events.py +++ b/tests/test_events/test_events.py @@ -7,7 +7,6 @@ from moto.events import mock_events from botocore.exceptions import ClientError from nose.tools import assert_raises - RULES = [ {'Name': 'test1', 'ScheduleExpression': 'rate(5 minutes)'}, {'Name': 'test2', 'ScheduleExpression': 'rate(1 minute)'}, @@ -109,6 +108,13 @@ def test_enable_disable_rule(): rule = client.describe_rule(Name=rule_name) assert(rule['State'] == 'ENABLED') + # Test invalid name + try: + client.enable_rule(Name='junk') + + except ClientError as ce: + assert ce.response['Error']['Code'] == 'ResourceNotFoundException' + @mock_events def test_list_rule_names_by_target(): From c46bc9ae83a180d8c451d907cfa430c97680ddab Mon Sep 17 00:00:00 2001 From: William Rubel Date: Mon, 18 Feb 2019 09:15:07 -0600 Subject: [PATCH 35/52] Trying to improve coverage --- tests/test_events/test_events.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_events/test_events.py b/tests/test_events/test_events.py index 810e0bbe5..a9d90ec32 100644 --- a/tests/test_events/test_events.py +++ b/tests/test_events/test_events.py @@ -1,5 +1,4 @@ import random - import boto3 import json From 37845e41a687441815606e7bfe2412288488b32a Mon Sep 17 00:00:00 2001 From: William Rubel Date: Mon, 18 Feb 2019 09:44:48 -0600 Subject: [PATCH 36/52] Trying to improve coverage --- tests/test_awslambda/test_lambda.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index 435d394e3..7f3b44b79 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -907,7 +907,6 @@ def test_create_function_with_already_exists(): @mock_s3 def test_list_versions_by_function_for_nonexistent_function(): conn = boto3.client('lambda', 'us-west-2') - versions = conn.list_versions_by_function(FunctionName='testFunction') assert len(versions['Versions']) == 0 From e9d8021c86e15da9dc1e5f634ecf877d5fb147ff Mon Sep 17 00:00:00 2001 From: William Rubel Date: Mon, 18 Feb 2019 21:20:29 -0600 Subject: [PATCH 37/52] Fixing list entities for policy --- moto/iam/models.py | 10 +++++ moto/iam/responses.py | 90 ++++++++++++++++++++++++++++++++------ tests/test_iam/test_iam.py | 79 +++++++++++++++++++++++++++++---- 3 files changed, 158 insertions(+), 21 deletions(-) diff --git a/moto/iam/models.py b/moto/iam/models.py index 80c7da29a..92ac19da7 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -892,6 +892,16 @@ class IAMBackend(BaseBackend): return users + def list_roles(self, path_prefix, marker, max_items): + roles = None + try: + roles = self.roles.values() + except KeyError: + raise IAMNotFoundException( + "Users {0}, {1}, {2} not found".format(path_prefix, marker, max_items)) + + return roles + def upload_signing_certificate(self, user_name, body): user = self.get_user(user_name) cert_id = random_resource_id(size=32) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 26781ab6f..278f13f2d 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -108,8 +108,69 @@ class IamResponse(BaseResponse): return template.render(policies=policies, marker=marker) def list_entities_for_policy(self): + policy_arn = self._get_param('PolicyArn') + + # Options 'User'|'Role'|'Group'|'LocalManagedPolicy'|'AWSManagedPolicy + entity = self._get_param('EntityFilter') + path_prefix = self._get_param('PathPrefix') + policy_usage_filter = self._get_param('PolicyUsageFilter') + marker = self._get_param('Marker') + max_items = self._get_param('MaxItems') + + entity_roles = [] + entity_groups = [] + entity_users = [] + + if entity == 'User': + users = iam_backend.list_users(path_prefix, marker, max_items) + if users: + for user in users: + for p in user.managed_policies: + if p == policy_arn: + entity_users.append(user.name) + + elif entity == 'Role': + roles = iam_backend.list_roles(path_prefix, marker, max_items) + if roles: + for role in roles: + for p in role.managed_policies: + if p == policy_arn: + entity_roles.append(role.name) + + elif entity == 'Group': + groups = iam_backend.list_groups() + if groups: + for group in groups: + for p in group.managed_policies: + if p == policy_arn: + entity_groups.append(group.name) + + elif entity == 'LocalManagedPolicy' or entity == 'AWSManagedPolicy': + users = iam_backend.list_users(path_prefix, marker, max_items) + if users: + for user in users: + for p in user.managed_policies: + if p == policy_arn: + entity_users.append(user.name) + + roles = iam_backend.list_roles(path_prefix, marker, max_items) + if roles: + for role in roles: + for p in role.managed_policies: + if p == policy_arn: + entity_roles.append(role.name) + + groups = iam_backend.list_groups() + if groups: + for group in groups: + for p in group.managed_policies: + if p == policy_arn: + entity_groups.append(group.name) + + template = self.response_template(LIST_ENTITIES_FOR_POLICY_TEMPLATE) - return template.render() + return template.render(roles=entity_roles, users=entity_users, groups=entity_groups) + def create_role(self): role_name = self._get_param('RoleName') @@ -676,23 +737,26 @@ class IamResponse(BaseResponse): LIST_ENTITIES_FOR_POLICY_TEMPLATE = """ - - DevRole - + {% for role in roles %} + + {{ role }} + + {% endfor %} - - Dev - + {% for group in groups %} + + {{ group }} + + {% endfor %} false - - Alice - - - Bob - + {% for user in users %} + + {{ user }} + + {% endfor %} diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 77ba17a5a..ceec5e06a 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -1187,19 +1187,82 @@ def test_update_role(): response = conn.update_role(RoleName="my-role", Description="test") assert len(response.keys()) == 1 + @mock_iam() def test_list_entities_for_policy(): + import json + test_policy = json.dumps({ + "Version": "2012-10-17", + "Statement": [ + { + "Action": "s3:ListBucket", + "Resource": "*", + "Effect": "Allow", + } + ] + }) + conn = boto3.client('iam', region_name='us-east-1') - - with assert_raises(ClientError): - conn.delete_role(RoleName="my-role") - conn.create_role(RoleName="my-role", AssumeRolePolicyDocument="some policy", Path="/my-path/") + conn.create_user(Path='/', UserName='testUser') + conn.create_group(Path='/', GroupName='testGroup') + conn.create_policy( + PolicyName='testPolicy', + Path='/', + PolicyDocument=test_policy, + Description='Test Policy' + ) - role = conn.get_role(RoleName="my-role") - arn = role.get('Role').get('Arn') + # Attach things to the user and group: + conn.put_user_policy(UserName='testUser', PolicyName='testPolicy', PolicyDocument=test_policy) + conn.put_group_policy(GroupName='testGroup', PolicyName='testPolicy', PolicyDocument=test_policy) + + conn.attach_user_policy(UserName='testUser', PolicyArn='arn:aws:iam::123456789012:policy/testPolicy') + conn.attach_group_policy(GroupName='testGroup', PolicyArn='arn:aws:iam::123456789012:policy/testPolicy') + + conn.add_user_to_group(UserName='testUser', GroupName='testGroup') + + # Add things to the role: + conn.create_instance_profile(InstanceProfileName='ipn') + conn.add_role_to_instance_profile(InstanceProfileName='ipn', RoleName='my-role') + conn.tag_role(RoleName='my-role', Tags=[ + { + 'Key': 'somekey', + 'Value': 'somevalue' + }, + { + 'Key': 'someotherkey', + 'Value': 'someothervalue' + } + ]) + conn.put_role_policy(RoleName='my-role', PolicyName='test-policy', PolicyDocument=test_policy) + conn.attach_role_policy(RoleName='my-role', PolicyArn='arn:aws:iam::123456789012:policy/testPolicy') response = conn.list_entities_for_policy( - PolicyArn=arn + PolicyArn='arn:aws:iam::123456789012:policy/testPolicy', + EntityFilter='Role' ) - assert response['PolicyGroups'][0]['GroupName'] == 'Dev' + assert response['PolicyRoles'] == [{'RoleName': 'my-role'}] + + response = conn.list_entities_for_policy( + PolicyArn='arn:aws:iam::123456789012:policy/testPolicy', + EntityFilter='User', + ) + assert response['PolicyUsers'] == [{'UserName': 'testUser'}] + + response = conn.list_entities_for_policy( + PolicyArn='arn:aws:iam::123456789012:policy/testPolicy', + EntityFilter='Group', + ) + assert response['PolicyGroups'] == [{'GroupName': 'testGroup'}] + + response = conn.list_entities_for_policy( + PolicyArn='arn:aws:iam::123456789012:policy/testPolicy', + EntityFilter='LocalManagedPolicy', + ) + assert response['PolicyGroups'] == [{'GroupName': 'testGroup'}] + assert response['PolicyUsers'] == [{'UserName': 'testUser'}] + assert response['PolicyRoles'] == [{'RoleName': 'my-role'}] + + + From 59deb4d6c0d3199f8d21a881f87e33a18cc95b0b Mon Sep 17 00:00:00 2001 From: William Rubel Date: Mon, 18 Feb 2019 21:25:29 -0600 Subject: [PATCH 38/52] Fixing list entities for policy --- moto/iam/responses.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 278f13f2d..72b2e464c 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -113,7 +113,7 @@ class IamResponse(BaseResponse): # Options 'User'|'Role'|'Group'|'LocalManagedPolicy'|'AWSManagedPolicy entity = self._get_param('EntityFilter') path_prefix = self._get_param('PathPrefix') - policy_usage_filter = self._get_param('PolicyUsageFilter') + #policy_usage_filter = self._get_param('PolicyUsageFilter') marker = self._get_param('Marker') max_items = self._get_param('MaxItems') @@ -167,11 +167,9 @@ class IamResponse(BaseResponse): if p == policy_arn: entity_groups.append(group.name) - template = self.response_template(LIST_ENTITIES_FOR_POLICY_TEMPLATE) return template.render(roles=entity_roles, users=entity_users, groups=entity_groups) - def create_role(self): role_name = self._get_param('RoleName') path = self._get_param('Path') From a5208222b4d017ed8d6f02fe0aed0e2da2dbaa3c Mon Sep 17 00:00:00 2001 From: William Rubel Date: Mon, 18 Feb 2019 21:29:09 -0600 Subject: [PATCH 39/52] Fixing list entities for policy --- moto/iam/responses.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 72b2e464c..5b19c9cdc 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -113,7 +113,7 @@ class IamResponse(BaseResponse): # Options 'User'|'Role'|'Group'|'LocalManagedPolicy'|'AWSManagedPolicy entity = self._get_param('EntityFilter') path_prefix = self._get_param('PathPrefix') - #policy_usage_filter = self._get_param('PolicyUsageFilter') + # policy_usage_filter = self._get_param('PolicyUsageFilter') marker = self._get_param('Marker') max_items = self._get_param('MaxItems') @@ -739,7 +739,7 @@ LIST_ENTITIES_FOR_POLICY_TEMPLATE = """ {{ role }} - {% endfor %} + {% endfor %} {% for group in groups %} From e6e7f235acc165a52cdc4a3e0d9180142d17d735 Mon Sep 17 00:00:00 2001 From: woohooyayaya Date: Mon, 25 Feb 2019 16:14:24 -0800 Subject: [PATCH 40/52] fix creation date in create key response to unix timestamp type --- moto/kms/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index 9d13589c1..9fbb2b587 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -36,7 +36,7 @@ class Key(BaseModel): "KeyMetadata": { "AWSAccountId": self.account_id, "Arn": self.arn, - "CreationDate": datetime.strftime(datetime.utcnow(), "%Y-%m-%dT%H:%M:%SZ"), + "CreationDate": datetime.strftime(datetime.utcnow(), "%s"), "Description": self.description, "Enabled": self.enabled, "KeyId": self.id, From c43f311a5e57008a5a875321d02cba2cf2cd9db9 Mon Sep 17 00:00:00 2001 From: Julian Mehnle Date: Mon, 25 Feb 2019 21:10:36 -0800 Subject: [PATCH 41/52] IMPLEMENTATION_COVERAGE.md: SQS get_queue_url is implemented SQS get_queue_url was implemented in dbf2368aa. --- IMPLEMENTATION_COVERAGE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index fe12c069a..a5650f572 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3919,7 +3919,7 @@ - [ ] delete_message_batch - [X] delete_queue - [ ] get_queue_attributes -- [ ] get_queue_url +- [X] get_queue_url - [X] list_dead_letter_source_queues - [ ] list_queue_tags - [X] list_queues From 7b7cf5bd1149dd385a047173347a2d5c984aac01 Mon Sep 17 00:00:00 2001 From: Chih-Hsuan Yen Date: Fri, 1 Mar 2019 17:14:18 +0800 Subject: [PATCH 42/52] Bump idna to 2.8 requests 2.21.0 allows idna 2.8 [1] [1] https://github.com/kennethreitz/requests/commit/8761e9736f7d5508a5547cdf3adecbe0b7306278 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8ac06c4dd..99be632db 100755 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ install_requires = [ "jsondiff==1.1.2", "aws-xray-sdk!=0.96,>=0.93", "responses>=0.9.0", - "idna<2.8,>=2.5", + "idna<2.9,>=2.5", "cfn-lint", ] From 9992e23e68fb7f9fb8e07a163d89069aaf662096 Mon Sep 17 00:00:00 2001 From: Chih-Hsuan Yen Date: Fri, 1 Mar 2019 19:22:26 +0800 Subject: [PATCH 43/52] Fix compatibility with cryptography 2.6 [1] https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst#26---2019-02-27 --- moto/acm/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/acm/models.py b/moto/acm/models.py index 39be8945d..15a1bd44d 100644 --- a/moto/acm/models.py +++ b/moto/acm/models.py @@ -243,7 +243,7 @@ class CertBundle(BaseModel): 'KeyAlgorithm': key_algo, 'NotAfter': datetime_to_epoch(self._cert.not_valid_after), 'NotBefore': datetime_to_epoch(self._cert.not_valid_before), - 'Serial': self._cert.serial, + 'Serial': self._cert.serial_number, 'SignatureAlgorithm': self._cert.signature_algorithm_oid._name.upper().replace('ENCRYPTION', ''), 'Status': self.status, # One of PENDING_VALIDATION, ISSUED, INACTIVE, EXPIRED, VALIDATION_TIMED_OUT, REVOKED, FAILED. 'Subject': 'CN={0}'.format(self.common_name), From d61ce0584bf53640409b1011e9e46a589a21dab0 Mon Sep 17 00:00:00 2001 From: Mark Challoner Date: Wed, 27 Feb 2019 10:54:55 +0000 Subject: [PATCH 44/52] Check record type when upserting. Previously this was not checked so an existing record (e.g. with type A) would be overwritten on upsert by a record with the same name but different type (e.g. TXT). This commit also: * publicizes the type variable appending the underscore affix (required to maintain compatibility with CloudFormation which sets type as the CF type), * fixes a wrong assumption in tests that UPSERT applies a change to Type (it creates a distinct record instead), * Updates ACM model to use serial_number instead of deprecated and remove serial causing Travis failures. --- moto/acm/models.py | 2 +- moto/route53/models.py | 12 ++++++------ tests/test_route53/test_route53.py | 20 +++++++++++++++----- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/moto/acm/models.py b/moto/acm/models.py index 39be8945d..15a1bd44d 100644 --- a/moto/acm/models.py +++ b/moto/acm/models.py @@ -243,7 +243,7 @@ class CertBundle(BaseModel): 'KeyAlgorithm': key_algo, 'NotAfter': datetime_to_epoch(self._cert.not_valid_after), 'NotBefore': datetime_to_epoch(self._cert.not_valid_before), - 'Serial': self._cert.serial, + 'Serial': self._cert.serial_number, 'SignatureAlgorithm': self._cert.signature_algorithm_oid._name.upper().replace('ENCRYPTION', ''), 'Status': self.status, # One of PENDING_VALIDATION, ISSUED, INACTIVE, EXPIRED, VALIDATION_TIMED_OUT, REVOKED, FAILED. 'Subject': 'CN={0}'.format(self.common_name), diff --git a/moto/route53/models.py b/moto/route53/models.py index d483d22e2..3760d3817 100644 --- a/moto/route53/models.py +++ b/moto/route53/models.py @@ -24,7 +24,7 @@ class HealthCheck(BaseModel): self.id = health_check_id self.ip_address = health_check_args.get("ip_address") self.port = health_check_args.get("port", 80) - self._type = health_check_args.get("type") + self.type_ = health_check_args.get("type") self.resource_path = health_check_args.get("resource_path") self.fqdn = health_check_args.get("fqdn") self.search_string = health_check_args.get("search_string") @@ -58,7 +58,7 @@ class HealthCheck(BaseModel): {{ health_check.ip_address }} {{ health_check.port }} - {{ health_check._type }} + {{ health_check.type_ }} {{ health_check.resource_path }} {{ health_check.fqdn }} {{ health_check.request_interval }} @@ -76,7 +76,7 @@ class RecordSet(BaseModel): def __init__(self, kwargs): self.name = kwargs.get('Name') - self._type = kwargs.get('Type') + self.type_ = kwargs.get('Type') self.ttl = kwargs.get('TTL') self.records = kwargs.get('ResourceRecords', []) self.set_identifier = kwargs.get('SetIdentifier') @@ -130,7 +130,7 @@ class RecordSet(BaseModel): def to_xml(self): template = Template(""" {{ record_set.name }} - {{ record_set._type }} + {{ record_set.type_ }} {% if record_set.set_identifier %} {{ record_set.set_identifier }} {% endif %} @@ -183,7 +183,7 @@ class FakeZone(BaseModel): def upsert_rrset(self, record_set): new_rrset = RecordSet(record_set) for i, rrset in enumerate(self.rrsets): - if rrset.name == new_rrset.name: + if rrset.name == new_rrset.name and rrset.type_ == new_rrset.type_: self.rrsets[i] = new_rrset break else: @@ -202,7 +202,7 @@ class FakeZone(BaseModel): record_sets = list(self.rrsets) # Copy the list if start_type: record_sets = [ - record_set for record_set in record_sets if record_set._type >= start_type] + record_set for record_set in record_sets if record_set.type_ >= start_type] if start_name: record_sets = [ record_set for record_set in record_sets if record_set.name >= start_name] diff --git a/tests/test_route53/test_route53.py b/tests/test_route53/test_route53.py index 1ced9d937..d730f8dcf 100644 --- a/tests/test_route53/test_route53.py +++ b/tests/test_route53/test_route53.py @@ -98,6 +98,16 @@ def test_rrset(): rrsets.should.have.length_of(1) rrsets[0].resource_records[0].should.equal('5.6.7.8') + changes = ResourceRecordSets(conn, zoneid) + change = changes.add_change("UPSERT", "foo.bar.testdns.aws.com", "TXT") + change.add_value("foo") + changes.commit() + + rrsets = conn.get_all_rrsets(zoneid) + rrsets.should.have.length_of(2) + rrsets[0].resource_records[0].should.equal('5.6.7.8') + rrsets[1].resource_records[0].should.equal('foo') + changes = ResourceRecordSets(conn, zoneid) changes.add_change("DELETE", "foo.bar.testdns.aws.com", "A") changes.commit() @@ -520,7 +530,7 @@ def test_change_resource_record_sets_crud_valid(): # Create A Record. a_record_endpoint_payload = { - 'Comment': 'create A record prod.redis.db', + 'Comment': 'Create A record prod.redis.db', 'Changes': [ { 'Action': 'CREATE', @@ -545,15 +555,15 @@ def test_change_resource_record_sets_crud_valid(): a_record_detail['TTL'].should.equal(10) a_record_detail['ResourceRecords'].should.equal([{'Value': '127.0.0.1'}]) - # Update type to CNAME + # Update A Record. cname_record_endpoint_payload = { - 'Comment': 'Update to CNAME prod.redis.db', + 'Comment': 'Update A record prod.redis.db', 'Changes': [ { 'Action': 'UPSERT', 'ResourceRecordSet': { 'Name': 'prod.redis.db.', - 'Type': 'CNAME', + 'Type': 'A', 'TTL': 60, 'ResourceRecords': [{ 'Value': '192.168.1.1' @@ -568,7 +578,7 @@ def test_change_resource_record_sets_crud_valid(): len(response['ResourceRecordSets']).should.equal(1) cname_record_detail = response['ResourceRecordSets'][0] cname_record_detail['Name'].should.equal('prod.redis.db.') - cname_record_detail['Type'].should.equal('CNAME') + cname_record_detail['Type'].should.equal('A') cname_record_detail['TTL'].should.equal(60) cname_record_detail['ResourceRecords'].should.equal([{'Value': '192.168.1.1'}]) From 840045c688969de17923e92308685ebc90fbbda4 Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Sat, 2 Mar 2019 18:56:49 -0500 Subject: [PATCH 45/52] fix sorting of log groups --- moto/logs/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moto/logs/models.py b/moto/logs/models.py index ca1fdc4ad..e105d4d14 100644 --- a/moto/logs/models.py +++ b/moto/logs/models.py @@ -242,7 +242,8 @@ class LogsBackend(BaseBackend): if next_token is None: next_token = 0 - groups = sorted(group.to_describe_dict() for name, group in self.groups.items() if name.startswith(log_group_name_prefix)) + groups = [group.to_describe_dict() for name, group in self.groups.items() if name.startswith(log_group_name_prefix)] + groups = sorted(groups, key=lambda x: x['creationTime'], reverse=True) groups_page = groups[next_token:next_token + limit] next_token += limit From bc116ab750f8c83623a3a759825a29e4ff916720 Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Mon, 25 Feb 2019 16:27:25 -0800 Subject: [PATCH 46/52] Basic AWS Config service support. --- .gitignore | 3 +- README.md | 4 +- moto/backends.py | 3 +- moto/config/__init__.py | 4 + moto/config/exceptions.py | 149 ++++++++++ moto/config/models.py | 333 +++++++++++++++++++++ moto/config/responses.py | 53 ++++ moto/config/urls.py | 10 + tests/test_config/test_config.py | 487 +++++++++++++++++++++++++++++++ 9 files changed, 1043 insertions(+), 3 deletions(-) create mode 100644 moto/config/__init__.py create mode 100644 moto/config/exceptions.py create mode 100644 moto/config/models.py create mode 100644 moto/config/responses.py create mode 100644 moto/config/urls.py create mode 100644 tests/test_config/test_config.py diff --git a/.gitignore b/.gitignore index 47e5efbe0..efb489651 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,5 @@ python_env .pytest_cache/ venv/ .python-version -.vscode/ \ No newline at end of file +.vscode/ +tests/file.tmp diff --git a/README.md b/README.md index d6e9f30a1..aeff847ed 100644 --- a/README.md +++ b/README.md @@ -70,10 +70,12 @@ It gets even better! Moto isn't just for Python code and it isn't just for S3. L |------------------------------------------------------------------------------| | CloudwatchEvents | @mock_events | all endpoints done | |------------------------------------------------------------------------------| -| Cognito Identity | @mock_cognitoidentity| basic endpoints done | +| Cognito Identity | @mock_cognitoidentity| basic endpoints done | |------------------------------------------------------------------------------| | Cognito Identity Provider | @mock_cognitoidp| basic endpoints done | |------------------------------------------------------------------------------| +| Config | @mock_config | basic endpoints done | +|------------------------------------------------------------------------------| | Data Pipeline | @mock_datapipeline| basic endpoints done | |------------------------------------------------------------------------------| | DynamoDB | @mock_dynamodb | core endpoints done | diff --git a/moto/backends.py b/moto/backends.py index 1a333415e..90cc803a7 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -46,7 +46,7 @@ from moto.iot import iot_backends from moto.iotdata import iotdata_backends from moto.batch import batch_backends from moto.resourcegroupstaggingapi import resourcegroupstaggingapi_backends - +from moto.config import config_backends BACKENDS = { 'acm': acm_backends, @@ -57,6 +57,7 @@ BACKENDS = { 'cloudwatch': cloudwatch_backends, 'cognito-identity': cognitoidentity_backends, 'cognito-idp': cognitoidp_backends, + 'config': config_backends, 'datapipeline': datapipeline_backends, 'dynamodb': dynamodb_backends, 'dynamodb2': dynamodb_backends2, diff --git a/moto/config/__init__.py b/moto/config/__init__.py new file mode 100644 index 000000000..9ca6a5917 --- /dev/null +++ b/moto/config/__init__.py @@ -0,0 +1,4 @@ +from .models import config_backends +from ..core.models import base_decorator + +mock_config = base_decorator(config_backends) diff --git a/moto/config/exceptions.py b/moto/config/exceptions.py new file mode 100644 index 000000000..b2b01d6a0 --- /dev/null +++ b/moto/config/exceptions.py @@ -0,0 +1,149 @@ +from __future__ import unicode_literals +from moto.core.exceptions import JsonRESTError + + +class NameTooLongException(JsonRESTError): + code = 400 + + def __init__(self, name, location): + message = '1 validation error detected: Value \'{name}\' at \'{location}\' failed to satisfy' \ + ' constraint: Member must have length less than or equal to 256'.format(name=name, location=location) + super(NameTooLongException, self).__init__("ValidationException", message) + + +class InvalidConfigurationRecorderNameException(JsonRESTError): + code = 400 + + def __init__(self, name): + message = 'The configuration recorder name \'{name}\' is not valid, blank string.'.format(name=name) + super(InvalidConfigurationRecorderNameException, self).__init__("InvalidConfigurationRecorderNameException", + message) + + +class MaxNumberOfConfigurationRecordersExceededException(JsonRESTError): + code = 400 + + def __init__(self, name): + message = 'Failed to put configuration recorder \'{name}\' because the maximum number of ' \ + 'configuration recorders: 1 is reached.'.format(name=name) + super(MaxNumberOfConfigurationRecordersExceededException, self).__init__( + "MaxNumberOfConfigurationRecordersExceededException", message) + + +class InvalidRecordingGroupException(JsonRESTError): + code = 400 + + def __init__(self): + message = 'The recording group provided is not valid' + super(InvalidRecordingGroupException, self).__init__("InvalidRecordingGroupException", message) + + +class InvalidResourceTypeException(JsonRESTError): + code = 400 + + def __init__(self, bad_list, good_list): + message = '{num} validation error detected: Value \'{bad_list}\' at ' \ + '\'configurationRecorder.recordingGroup.resourceTypes\' failed to satisfy constraint: ' \ + 'Member must satisfy constraint: [Member must satisfy enum value set: {good_list}]'.format( + num=len(bad_list), bad_list=bad_list, good_list=good_list) + # For PY2: + message = str(message) + + super(InvalidResourceTypeException, self).__init__("ValidationException", message) + + +class NoSuchConfigurationRecorderException(JsonRESTError): + code = 400 + + def __init__(self, name): + message = 'Cannot find configuration recorder with the specified name \'{name}\'.'.format(name=name) + super(NoSuchConfigurationRecorderException, self).__init__("NoSuchConfigurationRecorderException", message) + + +class InvalidDeliveryChannelNameException(JsonRESTError): + code = 400 + + def __init__(self, name): + message = 'The delivery channel name \'{name}\' is not valid, blank string.'.format(name=name) + super(InvalidDeliveryChannelNameException, self).__init__("InvalidDeliveryChannelNameException", + message) + + +class NoSuchBucketException(JsonRESTError): + """We are *only* validating that there is value that is not '' here.""" + code = 400 + + def __init__(self): + message = 'Cannot find a S3 bucket with an empty bucket name.' + super(NoSuchBucketException, self).__init__("NoSuchBucketException", message) + + +class InvalidS3KeyPrefixException(JsonRESTError): + code = 400 + + def __init__(self): + message = 'The s3 key prefix \'\' is not valid, empty s3 key prefix.' + super(InvalidS3KeyPrefixException, self).__init__("InvalidS3KeyPrefixException", message) + + +class InvalidSNSTopicARNException(JsonRESTError): + """We are *only* validating that there is value that is not '' here.""" + code = 400 + + def __init__(self): + message = 'The sns topic arn \'\' is not valid.' + super(InvalidSNSTopicARNException, self).__init__("InvalidSNSTopicARNException", message) + + +class InvalidDeliveryFrequency(JsonRESTError): + code = 400 + + def __init__(self, value, good_list): + message = '1 validation error detected: Value \'{value}\' at ' \ + '\'deliveryChannel.configSnapshotDeliveryProperties.deliveryFrequency\' failed to satisfy ' \ + 'constraint: Member must satisfy enum value set: {good_list}'.format(value=value, good_list=good_list) + super(InvalidDeliveryFrequency, self).__init__("InvalidDeliveryFrequency", message) + + +class MaxNumberOfDeliveryChannelsExceededException(JsonRESTError): + code = 400 + + def __init__(self, name): + message = 'Failed to put delivery channel \'{name}\' because the maximum number of ' \ + 'delivery channels: 1 is reached.'.format(name=name) + super(MaxNumberOfDeliveryChannelsExceededException, self).__init__( + "MaxNumberOfDeliveryChannelsExceededException", message) + + +class NoSuchDeliveryChannelException(JsonRESTError): + code = 400 + + def __init__(self, name): + message = 'Cannot find delivery channel with specified name \'{name}\'.'.format(name=name) + super(NoSuchDeliveryChannelException, self).__init__("NoSuchDeliveryChannelException", message) + + +class NoAvailableConfigurationRecorderException(JsonRESTError): + code = 400 + + def __init__(self): + message = 'Configuration recorder is not available to put delivery channel.' + super(NoAvailableConfigurationRecorderException, self).__init__("NoAvailableConfigurationRecorderException", + message) + + +class NoAvailableDeliveryChannelException(JsonRESTError): + code = 400 + + def __init__(self): + message = 'Delivery channel is not available to start configuration recorder.' + super(NoAvailableDeliveryChannelException, self).__init__("NoAvailableDeliveryChannelException", message) + + +class LastDeliveryChannelDeleteFailedException(JsonRESTError): + code = 400 + + def __init__(self, name): + message = 'Failed to delete last specified delivery channel with name \'{name}\', because there, ' \ + 'because there is a running configuration recorder.'.format(name=name) + super(LastDeliveryChannelDeleteFailedException, self).__init__("LastDeliveryChannelDeleteFailedException", message) diff --git a/moto/config/models.py b/moto/config/models.py new file mode 100644 index 000000000..5e43c6b12 --- /dev/null +++ b/moto/config/models.py @@ -0,0 +1,333 @@ +import json +import time +import pkg_resources + +from datetime import datetime + +from boto3 import Session + +from moto.config.exceptions import InvalidResourceTypeException, InvalidDeliveryFrequency, \ + InvalidConfigurationRecorderNameException, NameTooLongException, \ + MaxNumberOfConfigurationRecordersExceededException, InvalidRecordingGroupException, \ + NoSuchConfigurationRecorderException, NoAvailableConfigurationRecorderException, \ + InvalidDeliveryChannelNameException, NoSuchBucketException, InvalidS3KeyPrefixException, \ + InvalidSNSTopicARNException, MaxNumberOfDeliveryChannelsExceededException, NoAvailableDeliveryChannelException, \ + NoSuchDeliveryChannelException, LastDeliveryChannelDeleteFailedException + +from moto.core import BaseBackend, BaseModel + +DEFAULT_ACCOUNT_ID = 123456789012 + + +def datetime2int(date): + return int(time.mktime(date.timetuple())) + + +def snake_to_camels(original): + parts = original.split('_') + + camel_cased = parts[0].lower() + ''.join(p.title() for p in parts[1:]) + camel_cased = camel_cased.replace('Arn', 'ARN') # Config uses 'ARN' instead of 'Arn' + + return camel_cased + + +class ConfigEmptyDictable(BaseModel): + """Base class to make serialization easy. This assumes that the sub-class will NOT return 'None's in the JSON.""" + + def to_dict(self): + data = {} + for item, value in self.__dict__.items(): + if value is not None: + if isinstance(value, ConfigEmptyDictable): + data[snake_to_camels(item)] = value.to_dict() + else: + data[snake_to_camels(item)] = value + + return data + + +class ConfigRecorderStatus(ConfigEmptyDictable): + + def __init__(self, name): + self.name = name + + self.recording = False + self.last_start_time = None + self.last_stop_time = None + self.last_status = None + self.last_error_code = None + self.last_error_message = None + self.last_status_change_time = None + + def start(self): + self.recording = True + self.last_status = 'PENDING' + self.last_start_time = datetime2int(datetime.utcnow()) + self.last_status_change_time = datetime2int(datetime.utcnow()) + + def stop(self): + self.recording = False + self.last_stop_time = datetime2int(datetime.utcnow()) + self.last_status_change_time = datetime2int(datetime.utcnow()) + + +class ConfigDeliverySnapshotProperties(ConfigEmptyDictable): + + def __init__(self, delivery_frequency): + self.delivery_frequency = delivery_frequency + + +class ConfigDeliveryChannel(ConfigEmptyDictable): + + def __init__(self, name, s3_bucket_name, prefix=None, sns_arn=None, snapshot_properties=None): + self.name = name + self.s3_bucket_name = s3_bucket_name + self.s3_key_prefix = prefix + self.sns_topic_arn = sns_arn + self.config_snapshot_delivery_properties = snapshot_properties + + +class RecordingGroup(ConfigEmptyDictable): + + def __init__(self, all_supported=True, include_global_resource_types=False, resource_types=None): + self.all_supported = all_supported + self.include_global_resource_types = include_global_resource_types + self.resource_types = resource_types + + +class ConfigRecorder(ConfigEmptyDictable): + + def __init__(self, role_arn, recording_group, name='default', status=None): + self.name = name + self.role_arn = role_arn + self.recording_group = recording_group + + if not status: + self.status = ConfigRecorderStatus(name) + else: + self.status = status + + +class ConfigBackend(BaseBackend): + + def __init__(self): + self.recorders = {} + self.delivery_channels = {} + + @staticmethod + def _validate_resource_types(resource_list): + # Load the service file: + resource_package = 'botocore' + resource_path = '/'.join(('data', 'config', '2014-11-12', 'service-2.json')) + conifg_schema = json.loads(pkg_resources.resource_string(resource_package, resource_path)) + + # Verify that each entry exists in the supported list: + bad_list = [] + for resource in resource_list: + # For PY2: + r_str = str(resource) + + if r_str not in conifg_schema['shapes']['ResourceType']['enum']: + bad_list.append(r_str) + + if bad_list: + raise InvalidResourceTypeException(bad_list, conifg_schema['shapes']['ResourceType']['enum']) + + @staticmethod + def _validate_delivery_snapshot_properties(properties): + # Load the service file: + resource_package = 'botocore' + resource_path = '/'.join(('data', 'config', '2014-11-12', 'service-2.json')) + conifg_schema = json.loads(pkg_resources.resource_string(resource_package, resource_path)) + + # Verify that the deliveryFrequency is set to an acceptable value: + if properties.get('deliveryFrequency', None) not in \ + conifg_schema['shapes']['MaximumExecutionFrequency']['enum']: + raise InvalidDeliveryFrequency(properties.get('deliveryFrequency', None), + conifg_schema['shapes']['MaximumExecutionFrequency']['enum']) + + def put_configuration_recorder(self, config_recorder): + # Validate the name: + if not config_recorder.get('name'): + raise InvalidConfigurationRecorderNameException(config_recorder.get('name')) + if len(config_recorder.get('name')) > 256: + raise NameTooLongException(config_recorder.get('name'), 'configurationRecorder.name') + + # We're going to assume that the passed in Role ARN is correct. + + # Config currently only allows 1 configuration recorder for an account: + if len(self.recorders) == 1 and not self.recorders.get(config_recorder['name']): + raise MaxNumberOfConfigurationRecordersExceededException(config_recorder['name']) + + # Is this updating an existing one? + recorder_status = None + if self.recorders.get(config_recorder['name']): + recorder_status = self.recorders[config_recorder['name']].status + + # Validate the Recording Group: + if not config_recorder.get('recordingGroup'): + recording_group = RecordingGroup() + else: + rg = config_recorder['recordingGroup'] + + # Can't have both the resource types specified and the other flags as True. + if rg.get('resourceTypes') and ( + rg.get('allSupported', True) or + rg.get('includeGlobalResourceTypes', False)): + raise InvalidRecordingGroupException() + + # If an empty dict is provided, then bad: + if not rg.get('resourceTypes', False) \ + and not rg.get('resourceTypes') \ + and not rg.get('includeGlobalResourceTypes', False): + raise InvalidRecordingGroupException() + + # Validate that the list provided is correct: + self._validate_resource_types(rg.get('resourceTypes', [])) + + recording_group = RecordingGroup( + all_supported=rg.get('allSupported', True), + include_global_resource_types=rg.get('includeGlobalResourceTypes', False), + resource_types=rg.get('resourceTypes', []) + ) + + self.recorders[config_recorder['name']] = \ + ConfigRecorder(config_recorder['roleARN'], recording_group, name=config_recorder['name'], + status=recorder_status) + + def describe_configuration_recorders(self, recorder_names): + recorders = [] + + if recorder_names: + for rn in recorder_names: + if not self.recorders.get(rn): + raise NoSuchConfigurationRecorderException(rn) + + # Format the recorder: + recorders.append(self.recorders[rn].to_dict()) + + else: + for recorder in self.recorders.values(): + recorders.append(recorder.to_dict()) + + return recorders + + def describe_configuration_recorder_status(self, recorder_names): + recorders = [] + + if recorder_names: + for rn in recorder_names: + if not self.recorders.get(rn): + raise NoSuchConfigurationRecorderException(rn) + + # Format the recorder: + recorders.append(self.recorders[rn].status.to_dict()) + + else: + for recorder in self.recorders.values(): + recorders.append(recorder.status.to_dict()) + + return recorders + + def put_delivery_channel(self, delivery_channel): + # Must have a configuration recorder: + if not self.recorders: + raise NoAvailableConfigurationRecorderException() + + # Validate the name: + if not delivery_channel.get('name'): + raise InvalidDeliveryChannelNameException(delivery_channel.get('name')) + if len(delivery_channel.get('name')) > 256: + raise NameTooLongException(delivery_channel.get('name'), 'deliveryChannel.name') + + # We are going to assume that the bucket exists -- but will verify if the bucket provided is blank: + if not delivery_channel.get('s3BucketName'): + raise NoSuchBucketException() + + # We are going to assume that the bucket has the correct policy attached to it. We are only going to verify + # if the prefix provided is not an empty string: + if delivery_channel.get('s3KeyPrefix', None) == '': + raise InvalidS3KeyPrefixException() + + # Ditto for SNS -- Only going to assume that the ARN provided is not an empty string: + if delivery_channel.get('snsTopicARN', None) == '': + raise InvalidSNSTopicARNException() + + # Config currently only allows 1 delivery channel for an account: + if len(self.delivery_channels) == 1 and not self.delivery_channels.get(delivery_channel['name']): + raise MaxNumberOfDeliveryChannelsExceededException(delivery_channel['name']) + + if not delivery_channel.get('configSnapshotDeliveryProperties'): + dp = None + + else: + # Validate the config snapshot delivery properties: + self._validate_delivery_snapshot_properties(delivery_channel['configSnapshotDeliveryProperties']) + + dp = ConfigDeliverySnapshotProperties( + delivery_channel['configSnapshotDeliveryProperties']['deliveryFrequency']) + + self.delivery_channels[delivery_channel['name']] = \ + ConfigDeliveryChannel(delivery_channel['name'], delivery_channel['s3BucketName'], + prefix=delivery_channel.get('s3KeyPrefix', None), + sns_arn=delivery_channel.get('snsTopicARN', None), + snapshot_properties=dp) + + def describe_delivery_channels(self, channel_names): + channels = [] + + if channel_names: + for cn in channel_names: + if not self.delivery_channels.get(cn): + raise NoSuchDeliveryChannelException(cn) + + # Format the delivery channel: + channels.append(self.delivery_channels[cn].to_dict()) + + else: + for channel in self.delivery_channels.values(): + channels.append(channel.to_dict()) + + return channels + + def start_configuration_recorder(self, recorder_name): + if not self.recorders.get(recorder_name): + raise NoSuchConfigurationRecorderException(recorder_name) + + # Must have a delivery channel available as well: + if not self.delivery_channels: + raise NoAvailableDeliveryChannelException() + + # Start recording: + self.recorders[recorder_name].status.start() + + def stop_configuration_recorder(self, recorder_name): + if not self.recorders.get(recorder_name): + raise NoSuchConfigurationRecorderException(recorder_name) + + # Stop recording: + self.recorders[recorder_name].status.stop() + + def delete_configuration_recorder(self, recorder_name): + if not self.recorders.get(recorder_name): + raise NoSuchConfigurationRecorderException(recorder_name) + + del self.recorders[recorder_name] + + def delete_delivery_channel(self, channel_name): + if not self.delivery_channels.get(channel_name): + raise NoSuchDeliveryChannelException(channel_name) + + # Check if a channel is recording -- if so, bad -- (there can only be 1 recorder): + for recorder in self.recorders.values(): + if recorder.status.recording: + raise LastDeliveryChannelDeleteFailedException(channel_name) + + del self.delivery_channels[channel_name] + + +config_backends = {} +boto3_session = Session() +for region in boto3_session.get_available_regions('config'): + config_backends[region] = ConfigBackend() diff --git a/moto/config/responses.py b/moto/config/responses.py new file mode 100644 index 000000000..286b2349f --- /dev/null +++ b/moto/config/responses.py @@ -0,0 +1,53 @@ +import json +from moto.core.responses import BaseResponse +from .models import config_backends + + +class ConfigResponse(BaseResponse): + + @property + def config_backend(self): + return config_backends[self.region] + + def put_configuration_recorder(self): + self.config_backend.put_configuration_recorder(self._get_param('ConfigurationRecorder')) + return "" + + def describe_configuration_recorders(self): + recorders = self.config_backend.describe_configuration_recorders(self._get_param('ConfigurationRecorderNames')) + schema = {'ConfigurationRecorders': recorders} + return json.dumps(schema) + + def describe_configuration_recorder_status(self): + recorder_statuses = self.config_backend.describe_configuration_recorder_status( + self._get_param('ConfigurationRecorderNames')) + schema = {'ConfigurationRecordersStatus': recorder_statuses} + return json.dumps(schema) + + def put_delivery_channel(self): + self.config_backend.put_delivery_channel(self._get_param('DeliveryChannel')) + return "" + + def describe_delivery_channels(self): + delivery_channels = self.config_backend.describe_delivery_channels(self._get_param('DeliveryChannelNames')) + schema = {'DeliveryChannels': delivery_channels} + return json.dumps(schema) + + def describe_delivery_channel_status(self): + raise NotImplementedError() + + def delete_delivery_channel(self): + self.config_backend.delete_delivery_channel(self._get_param('DeliveryChannelName')) + return "" + + def delete_configuration_recorder(self): + self.config_backend.delete_configuration_recorder(self._get_param('ConfigurationRecorderName')) + return "" + + def start_configuration_recorder(self): + self.config_backend.start_configuration_recorder(self._get_param('ConfigurationRecorderName')) + return "" + + def stop_configuration_recorder(self): + self.config_backend.stop_configuration_recorder(self._get_param('ConfigurationRecorderName')) + return "" diff --git a/moto/config/urls.py b/moto/config/urls.py new file mode 100644 index 000000000..fd7b6969f --- /dev/null +++ b/moto/config/urls.py @@ -0,0 +1,10 @@ +from __future__ import unicode_literals +from .responses import ConfigResponse + +url_bases = [ + "https?://config.(.+).amazonaws.com", +] + +url_paths = { + '{0}/$': ConfigResponse.dispatch, +} diff --git a/tests/test_config/test_config.py b/tests/test_config/test_config.py new file mode 100644 index 000000000..a61d2a29c --- /dev/null +++ b/tests/test_config/test_config.py @@ -0,0 +1,487 @@ +from datetime import datetime, timedelta + +import boto3 +from botocore.exceptions import ClientError +from nose.tools import assert_raises + +from moto.config import mock_config + + +@mock_config +def test_put_configuration_recorder(): + client = boto3.client('config', region_name='us-west-2') + + # Try without a name supplied: + with assert_raises(ClientError) as ce: + client.put_configuration_recorder(ConfigurationRecorder={'roleARN': 'somearn'}) + assert ce.exception.response['Error']['Code'] == 'InvalidConfigurationRecorderNameException' + assert 'is not valid, blank string.' in ce.exception.response['Error']['Message'] + + # Try with a really long name: + with assert_raises(ClientError) as ce: + client.put_configuration_recorder(ConfigurationRecorder={'name': 'a' * 257, 'roleARN': 'somearn'}) + assert ce.exception.response['Error']['Code'] == 'ValidationException' + assert 'Member must have length less than or equal to 256' in ce.exception.response['Error']['Message'] + + # With resource types and flags set to True: + bad_groups = [ + {'allSupported': True, 'includeGlobalResourceTypes': True, 'resourceTypes': ['item']}, + {'allSupported': False, 'includeGlobalResourceTypes': True, 'resourceTypes': ['item']}, + {'allSupported': True, 'includeGlobalResourceTypes': False, 'resourceTypes': ['item']}, + {'allSupported': False, 'includeGlobalResourceTypes': False, 'resourceTypes': []} + ] + + for bg in bad_groups: + with assert_raises(ClientError) as ce: + client.put_configuration_recorder(ConfigurationRecorder={ + 'name': 'default', + 'roleARN': 'somearn', + 'recordingGroup': bg + }) + assert ce.exception.response['Error']['Code'] == 'InvalidRecordingGroupException' + assert ce.exception.response['Error']['Message'] == 'The recording group provided is not valid' + + # With an invalid Resource Type: + with assert_raises(ClientError) as ce: + client.put_configuration_recorder(ConfigurationRecorder={ + 'name': 'default', + 'roleARN': 'somearn', + 'recordingGroup': { + 'allSupported': False, + 'includeGlobalResourceTypes': False, + # 2 good, and 2 bad: + 'resourceTypes': ['AWS::EC2::Volume', 'LOLNO', 'AWS::EC2::VPC', 'LOLSTILLNO'] + } + }) + assert ce.exception.response['Error']['Code'] == 'ValidationException' + assert "2 validation error detected: Value '['LOLNO', 'LOLSTILLNO']" in str(ce.exception.response['Error']['Message']) + assert 'AWS::EC2::Instance' in ce.exception.response['Error']['Message'] + + # Create a proper one: + client.put_configuration_recorder(ConfigurationRecorder={ + 'name': 'testrecorder', + 'roleARN': 'somearn', + 'recordingGroup': { + 'allSupported': False, + 'includeGlobalResourceTypes': False, + 'resourceTypes': ['AWS::EC2::Volume', 'AWS::EC2::VPC'] + } + }) + + result = client.describe_configuration_recorders()['ConfigurationRecorders'] + assert len(result) == 1 + assert result[0]['name'] == 'testrecorder' + assert result[0]['roleARN'] == 'somearn' + assert not result[0]['recordingGroup']['allSupported'] + assert not result[0]['recordingGroup']['includeGlobalResourceTypes'] + assert len(result[0]['recordingGroup']['resourceTypes']) == 2 + assert 'AWS::EC2::Volume' in result[0]['recordingGroup']['resourceTypes'] \ + and 'AWS::EC2::VPC' in result[0]['recordingGroup']['resourceTypes'] + + # Now update the configuration recorder: + client.put_configuration_recorder(ConfigurationRecorder={ + 'name': 'testrecorder', + 'roleARN': 'somearn', + 'recordingGroup': { + 'allSupported': True, + 'includeGlobalResourceTypes': True + } + }) + result = client.describe_configuration_recorders()['ConfigurationRecorders'] + assert len(result) == 1 + assert result[0]['name'] == 'testrecorder' + assert result[0]['roleARN'] == 'somearn' + assert result[0]['recordingGroup']['allSupported'] + assert result[0]['recordingGroup']['includeGlobalResourceTypes'] + assert len(result[0]['recordingGroup']['resourceTypes']) == 0 + + # With a default recording group (i.e. lacking one) + client.put_configuration_recorder(ConfigurationRecorder={'name': 'testrecorder', 'roleARN': 'somearn'}) + result = client.describe_configuration_recorders()['ConfigurationRecorders'] + assert len(result) == 1 + assert result[0]['name'] == 'testrecorder' + assert result[0]['roleARN'] == 'somearn' + assert result[0]['recordingGroup']['allSupported'] + assert not result[0]['recordingGroup']['includeGlobalResourceTypes'] + assert not result[0]['recordingGroup'].get('resourceTypes') + + # Can currently only have exactly 1 Config Recorder in an account/region: + with assert_raises(ClientError) as ce: + client.put_configuration_recorder(ConfigurationRecorder={ + 'name': 'someotherrecorder', + 'roleARN': 'somearn', + 'recordingGroup': { + 'allSupported': False, + 'includeGlobalResourceTypes': False, + } + }) + assert ce.exception.response['Error']['Code'] == 'MaxNumberOfConfigurationRecordersExceededException' + assert "maximum number of configuration recorders: 1 is reached." in ce.exception.response['Error']['Message'] + + +@mock_config +def test_describe_configurations(): + client = boto3.client('config', region_name='us-west-2') + + # Without any configurations: + result = client.describe_configuration_recorders() + assert not result['ConfigurationRecorders'] + + client.put_configuration_recorder(ConfigurationRecorder={ + 'name': 'testrecorder', + 'roleARN': 'somearn', + 'recordingGroup': { + 'allSupported': False, + 'includeGlobalResourceTypes': False, + 'resourceTypes': ['AWS::EC2::Volume', 'AWS::EC2::VPC'] + } + }) + + result = client.describe_configuration_recorders()['ConfigurationRecorders'] + assert len(result) == 1 + assert result[0]['name'] == 'testrecorder' + assert result[0]['roleARN'] == 'somearn' + assert not result[0]['recordingGroup']['allSupported'] + assert not result[0]['recordingGroup']['includeGlobalResourceTypes'] + assert len(result[0]['recordingGroup']['resourceTypes']) == 2 + assert 'AWS::EC2::Volume' in result[0]['recordingGroup']['resourceTypes'] \ + and 'AWS::EC2::VPC' in result[0]['recordingGroup']['resourceTypes'] + + # Specify an incorrect name: + with assert_raises(ClientError) as ce: + client.describe_configuration_recorders(ConfigurationRecorderNames=['wrong']) + assert ce.exception.response['Error']['Code'] == 'NoSuchConfigurationRecorderException' + assert 'wrong' in ce.exception.response['Error']['Message'] + + # And with both a good and wrong name: + with assert_raises(ClientError) as ce: + client.describe_configuration_recorders(ConfigurationRecorderNames=['testrecorder', 'wrong']) + assert ce.exception.response['Error']['Code'] == 'NoSuchConfigurationRecorderException' + assert 'wrong' in ce.exception.response['Error']['Message'] + + +@mock_config +def test_delivery_channels(): + client = boto3.client('config', region_name='us-west-2') + + # Try without a config recorder: + with assert_raises(ClientError) as ce: + client.put_delivery_channel(DeliveryChannel={}) + assert ce.exception.response['Error']['Code'] == 'NoAvailableConfigurationRecorderException' + assert ce.exception.response['Error']['Message'] == 'Configuration recorder is not available to ' \ + 'put delivery channel.' + + # Create a config recorder to continue testing: + client.put_configuration_recorder(ConfigurationRecorder={ + 'name': 'testrecorder', + 'roleARN': 'somearn', + 'recordingGroup': { + 'allSupported': False, + 'includeGlobalResourceTypes': False, + 'resourceTypes': ['AWS::EC2::Volume', 'AWS::EC2::VPC'] + } + }) + + # Try without a name supplied: + with assert_raises(ClientError) as ce: + client.put_delivery_channel(DeliveryChannel={}) + assert ce.exception.response['Error']['Code'] == 'InvalidDeliveryChannelNameException' + assert 'is not valid, blank string.' in ce.exception.response['Error']['Message'] + + # Try with a really long name: + with assert_raises(ClientError) as ce: + client.put_delivery_channel(DeliveryChannel={'name': 'a' * 257}) + assert ce.exception.response['Error']['Code'] == 'ValidationException' + assert 'Member must have length less than or equal to 256' in ce.exception.response['Error']['Message'] + + # Without specifying a bucket name: + with assert_raises(ClientError) as ce: + client.put_delivery_channel(DeliveryChannel={'name': 'testchannel'}) + assert ce.exception.response['Error']['Code'] == 'NoSuchBucketException' + assert ce.exception.response['Error']['Message'] == 'Cannot find a S3 bucket with an empty bucket name.' + + with assert_raises(ClientError) as ce: + client.put_delivery_channel(DeliveryChannel={'name': 'testchannel', 's3BucketName': ''}) + assert ce.exception.response['Error']['Code'] == 'NoSuchBucketException' + assert ce.exception.response['Error']['Message'] == 'Cannot find a S3 bucket with an empty bucket name.' + + # With an empty string for the S3 key prefix: + with assert_raises(ClientError) as ce: + client.put_delivery_channel(DeliveryChannel={ + 'name': 'testchannel', 's3BucketName': 'somebucket', 's3KeyPrefix': ''}) + assert ce.exception.response['Error']['Code'] == 'InvalidS3KeyPrefixException' + assert 'empty s3 key prefix.' in ce.exception.response['Error']['Message'] + + # With an empty string for the SNS ARN: + with assert_raises(ClientError) as ce: + client.put_delivery_channel(DeliveryChannel={ + 'name': 'testchannel', 's3BucketName': 'somebucket', 'snsTopicARN': ''}) + assert ce.exception.response['Error']['Code'] == 'InvalidSNSTopicARNException' + assert 'The sns topic arn' in ce.exception.response['Error']['Message'] + + # With an invalid delivery frequency: + with assert_raises(ClientError) as ce: + client.put_delivery_channel(DeliveryChannel={ + 'name': 'testchannel', + 's3BucketName': 'somebucket', + 'configSnapshotDeliveryProperties': {'deliveryFrequency': 'WRONG'} + }) + assert ce.exception.response['Error']['Code'] == 'InvalidDeliveryFrequency' + assert 'WRONG' in ce.exception.response['Error']['Message'] + assert 'TwentyFour_Hours' in ce.exception.response['Error']['Message'] + + # Create a proper one: + client.put_delivery_channel(DeliveryChannel={'name': 'testchannel', 's3BucketName': 'somebucket'}) + result = client.describe_delivery_channels()['DeliveryChannels'] + assert len(result) == 1 + assert len(result[0].keys()) == 2 + assert result[0]['name'] == 'testchannel' + assert result[0]['s3BucketName'] == 'somebucket' + + # Overwrite it with another proper configuration: + client.put_delivery_channel(DeliveryChannel={ + 'name': 'testchannel', + 's3BucketName': 'somebucket', + 'snsTopicARN': 'sometopicarn', + 'configSnapshotDeliveryProperties': {'deliveryFrequency': 'TwentyFour_Hours'} + }) + result = client.describe_delivery_channels()['DeliveryChannels'] + assert len(result) == 1 + assert len(result[0].keys()) == 4 + assert result[0]['name'] == 'testchannel' + assert result[0]['s3BucketName'] == 'somebucket' + assert result[0]['snsTopicARN'] == 'sometopicarn' + assert result[0]['configSnapshotDeliveryProperties']['deliveryFrequency'] == 'TwentyFour_Hours' + + # Can only have 1: + with assert_raises(ClientError) as ce: + client.put_delivery_channel(DeliveryChannel={'name': 'testchannel2', 's3BucketName': 'somebucket'}) + assert ce.exception.response['Error']['Code'] == 'MaxNumberOfDeliveryChannelsExceededException' + assert 'because the maximum number of delivery channels: 1 is reached.' in ce.exception.response['Error']['Message'] + + +@mock_config +def test_describe_delivery_channels(): + client = boto3.client('config', region_name='us-west-2') + client.put_configuration_recorder(ConfigurationRecorder={ + 'name': 'testrecorder', + 'roleARN': 'somearn', + 'recordingGroup': { + 'allSupported': False, + 'includeGlobalResourceTypes': False, + 'resourceTypes': ['AWS::EC2::Volume', 'AWS::EC2::VPC'] + } + }) + + # Without any channels: + result = client.describe_delivery_channels() + assert not result['DeliveryChannels'] + + client.put_delivery_channel(DeliveryChannel={'name': 'testchannel', 's3BucketName': 'somebucket'}) + result = client.describe_delivery_channels()['DeliveryChannels'] + assert len(result) == 1 + assert len(result[0].keys()) == 2 + assert result[0]['name'] == 'testchannel' + assert result[0]['s3BucketName'] == 'somebucket' + + # Overwrite it with another proper configuration: + client.put_delivery_channel(DeliveryChannel={ + 'name': 'testchannel', + 's3BucketName': 'somebucket', + 'snsTopicARN': 'sometopicarn', + 'configSnapshotDeliveryProperties': {'deliveryFrequency': 'TwentyFour_Hours'} + }) + result = client.describe_delivery_channels()['DeliveryChannels'] + assert len(result) == 1 + assert len(result[0].keys()) == 4 + assert result[0]['name'] == 'testchannel' + assert result[0]['s3BucketName'] == 'somebucket' + assert result[0]['snsTopicARN'] == 'sometopicarn' + assert result[0]['configSnapshotDeliveryProperties']['deliveryFrequency'] == 'TwentyFour_Hours' + + # Specify an incorrect name: + with assert_raises(ClientError) as ce: + client.describe_delivery_channels(DeliveryChannelNames=['wrong']) + assert ce.exception.response['Error']['Code'] == 'NoSuchDeliveryChannelException' + assert 'wrong' in ce.exception.response['Error']['Message'] + + # And with both a good and wrong name: + with assert_raises(ClientError) as ce: + client.describe_delivery_channels(DeliveryChannelNames=['testchannel', 'wrong']) + assert ce.exception.response['Error']['Code'] == 'NoSuchDeliveryChannelException' + assert 'wrong' in ce.exception.response['Error']['Message'] + + +@mock_config +def test_start_configuration_recorder(): + client = boto3.client('config', region_name='us-west-2') + + # Without a config recorder: + with assert_raises(ClientError) as ce: + client.start_configuration_recorder(ConfigurationRecorderName='testrecorder') + assert ce.exception.response['Error']['Code'] == 'NoSuchConfigurationRecorderException' + + # Make the config recorder; + client.put_configuration_recorder(ConfigurationRecorder={ + 'name': 'testrecorder', + 'roleARN': 'somearn', + 'recordingGroup': { + 'allSupported': False, + 'includeGlobalResourceTypes': False, + 'resourceTypes': ['AWS::EC2::Volume', 'AWS::EC2::VPC'] + } + }) + + # Without a delivery channel: + with assert_raises(ClientError) as ce: + client.start_configuration_recorder(ConfigurationRecorderName='testrecorder') + assert ce.exception.response['Error']['Code'] == 'NoAvailableDeliveryChannelException' + + # Make the delivery channel: + client.put_delivery_channel(DeliveryChannel={'name': 'testchannel', 's3BucketName': 'somebucket'}) + + # Start it: + client.start_configuration_recorder(ConfigurationRecorderName='testrecorder') + + # Verify it's enabled: + result = client.describe_configuration_recorder_status()['ConfigurationRecordersStatus'] + lower_bound = (datetime.utcnow() - timedelta(minutes=5)) + assert result[0]['recording'] + assert result[0]['lastStatus'] == 'PENDING' + assert lower_bound < result[0]['lastStartTime'].replace(tzinfo=None) <= datetime.utcnow() + assert lower_bound < result[0]['lastStatusChangeTime'].replace(tzinfo=None) <= datetime.utcnow() + + +@mock_config +def test_stop_configuration_recorder(): + client = boto3.client('config', region_name='us-west-2') + + # Without a config recorder: + with assert_raises(ClientError) as ce: + client.stop_configuration_recorder(ConfigurationRecorderName='testrecorder') + assert ce.exception.response['Error']['Code'] == 'NoSuchConfigurationRecorderException' + + # Make the config recorder; + client.put_configuration_recorder(ConfigurationRecorder={ + 'name': 'testrecorder', + 'roleARN': 'somearn', + 'recordingGroup': { + 'allSupported': False, + 'includeGlobalResourceTypes': False, + 'resourceTypes': ['AWS::EC2::Volume', 'AWS::EC2::VPC'] + } + }) + + # Make the delivery channel for creation: + client.put_delivery_channel(DeliveryChannel={'name': 'testchannel', 's3BucketName': 'somebucket'}) + + # Start it: + client.start_configuration_recorder(ConfigurationRecorderName='testrecorder') + client.stop_configuration_recorder(ConfigurationRecorderName='testrecorder') + + # Verify it's disabled: + result = client.describe_configuration_recorder_status()['ConfigurationRecordersStatus'] + lower_bound = (datetime.utcnow() - timedelta(minutes=5)) + assert not result[0]['recording'] + assert result[0]['lastStatus'] == 'PENDING' + assert lower_bound < result[0]['lastStartTime'].replace(tzinfo=None) <= datetime.utcnow() + assert lower_bound < result[0]['lastStopTime'].replace(tzinfo=None) <= datetime.utcnow() + assert lower_bound < result[0]['lastStatusChangeTime'].replace(tzinfo=None) <= datetime.utcnow() + + +@mock_config +def test_describe_configuration_recorder_status(): + client = boto3.client('config', region_name='us-west-2') + + # Without any: + result = client.describe_configuration_recorder_status() + assert not result['ConfigurationRecordersStatus'] + + # Make the config recorder; + client.put_configuration_recorder(ConfigurationRecorder={ + 'name': 'testrecorder', + 'roleARN': 'somearn', + 'recordingGroup': { + 'allSupported': False, + 'includeGlobalResourceTypes': False, + 'resourceTypes': ['AWS::EC2::Volume', 'AWS::EC2::VPC'] + } + }) + + # Without specifying a config recorder: + result = client.describe_configuration_recorder_status()['ConfigurationRecordersStatus'] + assert len(result) == 1 + assert result[0]['name'] == 'testrecorder' + assert not result[0]['recording'] + + # With a proper name: + result = client.describe_configuration_recorder_status( + ConfigurationRecorderNames=['testrecorder'])['ConfigurationRecordersStatus'] + assert len(result) == 1 + assert result[0]['name'] == 'testrecorder' + assert not result[0]['recording'] + + # Invalid name: + with assert_raises(ClientError) as ce: + client.describe_configuration_recorder_status(ConfigurationRecorderNames=['testrecorder', 'wrong']) + assert ce.exception.response['Error']['Code'] == 'NoSuchConfigurationRecorderException' + assert 'wrong' in ce.exception.response['Error']['Message'] + + +@mock_config +def test_delete_configuration_recorder(): + client = boto3.client('config', region_name='us-west-2') + + # Make the config recorder; + client.put_configuration_recorder(ConfigurationRecorder={ + 'name': 'testrecorder', + 'roleARN': 'somearn', + 'recordingGroup': { + 'allSupported': False, + 'includeGlobalResourceTypes': False, + 'resourceTypes': ['AWS::EC2::Volume', 'AWS::EC2::VPC'] + } + }) + + # Delete it: + client.delete_configuration_recorder(ConfigurationRecorderName='testrecorder') + + # Try again -- it should be deleted: + with assert_raises(ClientError) as ce: + client.delete_configuration_recorder(ConfigurationRecorderName='testrecorder') + assert ce.exception.response['Error']['Code'] == 'NoSuchConfigurationRecorderException' + + +@mock_config +def test_delete_delivery_channel(): + client = boto3.client('config', region_name='us-west-2') + + # Need a recorder to test the constraint on recording being enabled: + client.put_configuration_recorder(ConfigurationRecorder={ + 'name': 'testrecorder', + 'roleARN': 'somearn', + 'recordingGroup': { + 'allSupported': False, + 'includeGlobalResourceTypes': False, + 'resourceTypes': ['AWS::EC2::Volume', 'AWS::EC2::VPC'] + } + }) + client.put_delivery_channel(DeliveryChannel={'name': 'testchannel', 's3BucketName': 'somebucket'}) + client.start_configuration_recorder(ConfigurationRecorderName='testrecorder') + + # With the recorder enabled: + with assert_raises(ClientError) as ce: + client.delete_delivery_channel(DeliveryChannelName='testchannel') + assert ce.exception.response['Error']['Code'] == 'LastDeliveryChannelDeleteFailedException' + assert 'because there is a running configuration recorder.' in ce.exception.response['Error']['Message'] + + # Stop recording: + client.stop_configuration_recorder(ConfigurationRecorderName='testrecorder') + + # Try again: + client.delete_delivery_channel(DeliveryChannelName='testchannel') + + # Verify: + with assert_raises(ClientError) as ce: + client.delete_delivery_channel(DeliveryChannelName='testchannel') + assert ce.exception.response['Error']['Code'] == 'NoSuchDeliveryChannelException' From 74ff2ccc959ef3950318a71a884b70877841bb3d Mon Sep 17 00:00:00 2001 From: Domenico Testa Date: Mon, 4 Mar 2019 09:16:43 +0100 Subject: [PATCH 47/52] Reformatting to get better coverage results --- moto/s3/responses.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index e4f97cd43..6240c6b53 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -19,8 +19,7 @@ from .exceptions import BucketAlreadyExists, S3ClientError, MissingBucket, Missi MalformedACLError, InvalidNotificationARN, InvalidNotificationEvent from .models import s3_backend, get_canned_acl, FakeGrantee, FakeGrant, FakeAcl, FakeKey, FakeTagging, FakeTagSet, \ FakeTag -from .utils import bucket_name_from_url, clean_key_name, metadata_from_headers, \ - parse_region_from_url +from .utils import bucket_name_from_url, clean_key_name, metadata_from_headers, parse_region_from_url from xml.dom import minidom From d6022417f5c45b1f5c245f2125ffcfeef40a2ce9 Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Thu, 7 Mar 2019 12:53:01 -0800 Subject: [PATCH 48/52] Forgot to add Config to the `__init__.py`. --- moto/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/__init__.py b/moto/__init__.py index e86c499a7..8e9b91bce 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -13,6 +13,7 @@ from .cloudformation import mock_cloudformation, mock_cloudformation_deprecated from .cloudwatch import mock_cloudwatch, mock_cloudwatch_deprecated # flake8: noqa from .cognitoidentity import mock_cognitoidentity, mock_cognitoidentity_deprecated # flake8: noqa from .cognitoidp import mock_cognitoidp, mock_cognitoidp_deprecated # flake8: noqa +from .config import mock_config # flake8: noqa from .datapipeline import mock_datapipeline, mock_datapipeline_deprecated # flake8: noqa from .dynamodb import mock_dynamodb, mock_dynamodb_deprecated # flake8: noqa from .dynamodb2 import mock_dynamodb2, mock_dynamodb2_deprecated # flake8: noqa From 15b3ede3cc3b1eaadbd2fedff8404c213883e017 Mon Sep 17 00:00:00 2001 From: Tay Frost Date: Thu, 7 Mar 2019 16:59:03 -0500 Subject: [PATCH 49/52] Add test for case where ebs volume has no tags. This commit adds a test for a case where an EBS volume has no tags. When an EBS volume has no tags, calls to the aws ec2 endpoints `create_volume` and `describe_volumes` do not include the `Tags` key in the `response.Volumes[]` object. However, moto does include the `Tags` key in this case. This discrepancy in behaviour can result in code passing a moto test but failing in production. Sample snippets that trigger this condition: ``` def create_volume_and_then_get_tags_from_response(): client = boto3.client('ec2', region_name='us-east-1') volume_response = client.create_volume( Size=10, AvailabilityZone='us-east-1a' ) keys = volume_response['Keys'] ``` ``` def create_volume_and_then_get_tags_from_describe_volumes(): client = boto3.client('ec2', region_name='us-east-1') volume_response = client.create_volume( Size=10, AvailabilityZone='us-east-1a' ) volume_describe_response = client.describe_volumes() keys = volume_describe_response['Volumes'][0]['Keys'] ``` Both sample snippets will succeed in a moto test, but fail with a `KeyError` when using the aws api. --- tests/test_ec2/test_elastic_block_store.py | 24 ++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/test_ec2/test_elastic_block_store.py b/tests/test_ec2/test_elastic_block_store.py index 442e41dde..8f4a00b13 100644 --- a/tests/test_ec2/test_elastic_block_store.py +++ b/tests/test_ec2/test_elastic_block_store.py @@ -589,6 +589,18 @@ def test_volume_tag_escaping(): dict(snaps[0].tags).should.equal({'key': ''}) +@mock_ec2 +def test_volume_property_hidden_when_no_tags_exist(): + ec2_client = boto3.client('ec2', region_name='us-east-1') + + volume_response = ec2_client.create_volume( + Size=10, + AvailabilityZone='us-east-1a' + ) + + volume_response.get('Tags').should.equal(None) + + @freeze_time @mock_ec2 def test_copy_snapshot(): @@ -602,26 +614,26 @@ def test_copy_snapshot(): create_snapshot_response = ec2_client.create_snapshot( VolumeId=volume_response['VolumeId'] ) - + copy_snapshot_response = dest_ec2_client.copy_snapshot( SourceSnapshotId=create_snapshot_response['SnapshotId'], SourceRegion="eu-west-1" ) - + ec2 = boto3.resource('ec2', region_name='eu-west-1') dest_ec2 = boto3.resource('ec2', region_name='eu-west-2') - + source = ec2.Snapshot(create_snapshot_response['SnapshotId']) dest = dest_ec2.Snapshot(copy_snapshot_response['SnapshotId']) - + attribs = ['data_encryption_key_id', 'encrypted', 'kms_key_id', 'owner_alias', 'owner_id', 'progress', 'state', 'state_message', 'tags', 'volume_id', 'volume_size'] - + for attrib in attribs: getattr(source, attrib).should.equal(getattr(dest, attrib)) - + # Copy from non-existent source ID. with assert_raises(ClientError) as cm: create_snapshot_error = ec2_client.create_snapshot( From 7b236c4dedac874486ed9947c94847375404212a Mon Sep 17 00:00:00 2001 From: Tay Frost Date: Thu, 7 Mar 2019 17:07:15 -0500 Subject: [PATCH 50/52] bugfix ebs volume tag behaviour This commit modifies the response format of the ec2 calls `create_volume` and `describe_volumes`. Previously, these calls would always include a `Tags` key in the response, even when a volume has no tags. Now, the `Tags` key will not be included in the response if the volume has no tags. When an EBS volume has no tags, calls to the aws ec2 endpoints `create_volume` and `describe_volumes` do not include the `Tags` key in the `response.Volumes[]` object. However, moto does include the `Tags` key in this case. This discrepancy in behaviour can result in code passing a moto test but failing in production. Sample snippets that trigger this condition: ``` def create_volume_and_then_get_tags_from_response(): client = boto3.client('ec2', region_name='us-east-1') volume_response = client.create_volume( Size=10, AvailabilityZone='us-east-1a' ) keys = volume_response['Keys'] ``` ``` def create_volume_and_then_get_tags_from_describe_volumes(): client = boto3.client('ec2', region_name='us-east-1') volume_response = client.create_volume( Size=10, AvailabilityZone='us-east-1a' ) volume_describe_response = client.describe_volumes() keys = volume_describe_response['Volumes'][0]['Keys'] ``` Both sample snippets will succeed in a moto test, but fail with a `KeyError` when using the aws api. --- moto/ec2/responses/elastic_block_store.py | 44 ++++++++++++----------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/moto/ec2/responses/elastic_block_store.py b/moto/ec2/responses/elastic_block_store.py index aa0d7f73b..acd37b283 100644 --- a/moto/ec2/responses/elastic_block_store.py +++ b/moto/ec2/responses/elastic_block_store.py @@ -150,16 +150,18 @@ CREATE_VOLUME_RESPONSE = """ {% for key in key_list %} {{ key.name }} - {{ key.version_id }} + {% if key.version_id is none %}null{% else %}{{ key.version_id }}{% endif %} {% if latest_versions[key.name] == key.version_id %}true{% else %}false{% endif %} {{ key.last_modified_ISO8601 }} {{ key.etag }} diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index eea720240..cf45822b5 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -444,7 +444,12 @@ def test_copy_key_with_version(): key.set_contents_from_string("some value") key.set_contents_from_string("another value") - bucket.copy_key('new-key', 'foobar', 'the-key', src_version_id='0') + key = [ + key.version_id + for key in bucket.get_all_versions() + if not key.is_latest + ][0] + bucket.copy_key('new-key', 'foobar', 'the-key', src_version_id=key) bucket.get_key( "the-key").get_contents_as_string().should.equal(b"another value") @@ -818,16 +823,19 @@ def test_key_version(): bucket = conn.create_bucket('foobar') bucket.configure_versioning(versioning=True) + versions = [] + key = Key(bucket) key.key = 'the-key' key.version_id.should.be.none key.set_contents_from_string('some string') - key.version_id.should.equal('0') + versions.append(key.version_id) key.set_contents_from_string('some string') - key.version_id.should.equal('1') + versions.append(key.version_id) + set(versions).should.have.length_of(2) key = bucket.get_key('the-key') - key.version_id.should.equal('1') + key.version_id.should.equal(versions[-1]) @mock_s3_deprecated @@ -836,23 +844,25 @@ def test_list_versions(): bucket = conn.create_bucket('foobar') bucket.configure_versioning(versioning=True) + key_versions = [] + key = Key(bucket, 'the-key') key.version_id.should.be.none key.set_contents_from_string("Version 1") - key.version_id.should.equal('0') + key_versions.append(key.version_id) key.set_contents_from_string("Version 2") - key.version_id.should.equal('1') + key_versions.append(key.version_id) + key_versions.should.have.length_of(2) versions = list(bucket.list_versions()) - versions.should.have.length_of(2) versions[0].name.should.equal('the-key') - versions[0].version_id.should.equal('0') + versions[0].version_id.should.equal(key_versions[0]) versions[0].get_contents_as_string().should.equal(b"Version 1") versions[1].name.should.equal('the-key') - versions[1].version_id.should.equal('1') + versions[1].version_id.should.equal(key_versions[1]) versions[1].get_contents_as_string().should.equal(b"Version 2") key = Key(bucket, 'the2-key') @@ -1483,16 +1493,22 @@ def test_boto3_head_object_with_versioning(): s3.Object('blah', 'hello.txt').put(Body=old_content) s3.Object('blah', 'hello.txt').put(Body=new_content) + versions = list(s3.Bucket('blah').object_versions.all()) + latest = list(filter(lambda item: item.is_latest, versions))[0] + oldest = list(filter(lambda item: not item.is_latest, versions))[0] + head_object = s3.Object('blah', 'hello.txt').meta.client.head_object( Bucket='blah', Key='hello.txt') - head_object['VersionId'].should.equal('1') + head_object['VersionId'].should.equal(latest.id) head_object['ContentLength'].should.equal(len(new_content)) old_head_object = s3.Object('blah', 'hello.txt').meta.client.head_object( - Bucket='blah', Key='hello.txt', VersionId='0') - old_head_object['VersionId'].should.equal('0') + Bucket='blah', Key='hello.txt', VersionId=oldest.id) + old_head_object['VersionId'].should.equal(oldest.id) old_head_object['ContentLength'].should.equal(len(old_content)) + old_head_object['VersionId'].should_not.equal(head_object['VersionId']) + @mock_s3 def test_boto3_copy_object_with_versioning(): @@ -1507,9 +1523,6 @@ def test_boto3_copy_object_with_versioning(): obj1_version = client.get_object(Bucket='blah', Key='test1')['VersionId'] obj2_version = client.get_object(Bucket='blah', Key='test2')['VersionId'] - # Versions should be the same - obj1_version.should.equal(obj2_version) - client.copy_object(CopySource={'Bucket': 'blah', 'Key': 'test1'}, Bucket='blah', Key='test2') obj2_version_new = client.get_object(Bucket='blah', Key='test2')['VersionId'] @@ -2507,6 +2520,75 @@ def test_boto3_list_object_versions(): response['Body'].read().should.equal(items[-1]) +@mock_s3 +def test_boto3_list_object_versions_with_versioning_disabled(): + s3 = boto3.client('s3', region_name='us-east-1') + bucket_name = 'mybucket' + key = 'key-with-versions' + s3.create_bucket(Bucket=bucket_name) + items = (six.b('v1'), six.b('v2')) + for body in items: + s3.put_object( + Bucket=bucket_name, + Key=key, + Body=body + ) + response = s3.list_object_versions( + Bucket=bucket_name + ) + + # One object version should be returned + len(response['Versions']).should.equal(1) + response['Versions'][0]['Key'].should.equal(key) + + # The version id should be the string null + response['Versions'][0]['VersionId'].should.equal('null') + + # Test latest object version is returned + response = s3.get_object(Bucket=bucket_name, Key=key) + response['Body'].read().should.equal(items[-1]) + + +@mock_s3 +def test_boto3_list_object_versions_with_versioning_enabled_late(): + s3 = boto3.client('s3', region_name='us-east-1') + bucket_name = 'mybucket' + key = 'key-with-versions' + s3.create_bucket(Bucket=bucket_name) + items = (six.b('v1'), six.b('v2')) + s3.put_object( + Bucket=bucket_name, + Key=key, + Body=six.b('v1') + ) + s3.put_bucket_versioning( + Bucket=bucket_name, + VersioningConfiguration={ + 'Status': 'Enabled' + } + ) + s3.put_object( + Bucket=bucket_name, + Key=key, + Body=six.b('v2') + ) + response = s3.list_object_versions( + Bucket=bucket_name + ) + + # Two object versions should be returned + len(response['Versions']).should.equal(2) + keys = set([item['Key'] for item in response['Versions']]) + keys.should.equal({key}) + + # There should still be a null version id. + versionsId = set([item['VersionId'] for item in response['Versions']]) + versionsId.should.contain('null') + + # Test latest object version is returned + response = s3.get_object(Bucket=bucket_name, Key=key) + response['Body'].read().should.equal(items[-1]) + @mock_s3 def test_boto3_bad_prefix_list_object_versions(): s3 = boto3.client('s3', region_name='us-east-1') @@ -2563,18 +2645,25 @@ def test_boto3_delete_markers(): Bucket=bucket_name, Key=key ) - e.response['Error']['Code'].should.equal('404') + e.exception.response['Error']['Code'].should.equal('NoSuchKey') + + response = s3.list_object_versions( + Bucket=bucket_name + ) + response['Versions'].should.have.length_of(2) + response['DeleteMarkers'].should.have.length_of(1) s3.delete_object( Bucket=bucket_name, Key=key, - VersionId='2' + VersionId=response['DeleteMarkers'][0]['VersionId'] ) response = s3.get_object( Bucket=bucket_name, Key=key ) response['Body'].read().should.equal(items[-1]) + response = s3.list_object_versions( Bucket=bucket_name ) @@ -2583,10 +2672,8 @@ def test_boto3_delete_markers(): # We've asserted there is only 2 records so one is newest, one is oldest latest = list(filter(lambda item: item['IsLatest'], response['Versions']))[0] oldest = list(filter(lambda item: not item['IsLatest'], response['Versions']))[0] - # Double check ordering of version ID's - latest['VersionId'].should.equal('1') - oldest['VersionId'].should.equal('0') + latest['VersionId'].should_not.equal(oldest['VersionId']) # Double check the name is still unicode latest['Key'].should.equal('key-with-versions-and-unicode-ó') @@ -2631,12 +2718,12 @@ def test_boto3_multiple_delete_markers(): s3.delete_object( Bucket=bucket_name, Key=key, - VersionId='2' + VersionId=response['DeleteMarkers'][0]['VersionId'] ) s3.delete_object( Bucket=bucket_name, Key=key, - VersionId='3' + VersionId=response['DeleteMarkers'][1]['VersionId'] ) response = s3.get_object( @@ -2652,8 +2739,7 @@ def test_boto3_multiple_delete_markers(): oldest = list(filter(lambda item: not item['IsLatest'], response['Versions']))[0] # Double check ordering of version ID's - latest['VersionId'].should.equal('1') - oldest['VersionId'].should.equal('0') + latest['VersionId'].should_not.equal(oldest['VersionId']) # Double check the name is still unicode latest['Key'].should.equal('key-with-versions-and-unicode-ó') From 6d67418c415f65870adbd6d3ecfa4e48163f46e1 Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Mon, 11 Mar 2019 13:25:36 -0700 Subject: [PATCH 52/52] Fixed validation bugs in put_configuration_recorder --- moto/config/models.py | 14 ++++++++------ tests/test_config/test_config.py | 6 +++++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/moto/config/models.py b/moto/config/models.py index 5e43c6b12..cd6e07afa 100644 --- a/moto/config/models.py +++ b/moto/config/models.py @@ -166,21 +166,23 @@ class ConfigBackend(BaseBackend): recorder_status = self.recorders[config_recorder['name']].status # Validate the Recording Group: - if not config_recorder.get('recordingGroup'): + if config_recorder.get('recordingGroup') is None: recording_group = RecordingGroup() else: rg = config_recorder['recordingGroup'] + # If an empty dict is passed in, then bad: + if not rg: + raise InvalidRecordingGroupException() + # Can't have both the resource types specified and the other flags as True. if rg.get('resourceTypes') and ( - rg.get('allSupported', True) or + rg.get('allSupported', False) or rg.get('includeGlobalResourceTypes', False)): raise InvalidRecordingGroupException() - # If an empty dict is provided, then bad: - if not rg.get('resourceTypes', False) \ - and not rg.get('resourceTypes') \ - and not rg.get('includeGlobalResourceTypes', False): + # Must supply resourceTypes if 'allSupported' is not supplied: + if not rg.get('allSupported') and not rg.get('resourceTypes'): raise InvalidRecordingGroupException() # Validate that the list provided is correct: diff --git a/tests/test_config/test_config.py b/tests/test_config/test_config.py index a61d2a29c..96c62455c 100644 --- a/tests/test_config/test_config.py +++ b/tests/test_config/test_config.py @@ -28,7 +28,11 @@ def test_put_configuration_recorder(): {'allSupported': True, 'includeGlobalResourceTypes': True, 'resourceTypes': ['item']}, {'allSupported': False, 'includeGlobalResourceTypes': True, 'resourceTypes': ['item']}, {'allSupported': True, 'includeGlobalResourceTypes': False, 'resourceTypes': ['item']}, - {'allSupported': False, 'includeGlobalResourceTypes': False, 'resourceTypes': []} + {'allSupported': False, 'includeGlobalResourceTypes': False, 'resourceTypes': []}, + {'includeGlobalResourceTypes': False, 'resourceTypes': []}, + {'includeGlobalResourceTypes': True}, + {'resourceTypes': []}, + {} ] for bg in bad_groups: