From bc1674cb197af5866a43ef11d3d60ea5f3fe02ab Mon Sep 17 00:00:00 2001 From: usmangani1 Date: Mon, 27 Jul 2020 18:38:01 +0530 Subject: [PATCH 01/11] CF : Added support for get template summary (#3179) * CF : Added support for get template summary * Linting Co-authored-by: Bert Blommers --- moto/cloudformation/responses.py | 64 ++++++++++++++++--- .../test_cloudformation_stack_crud_boto3.py | 50 +++++++++++++++ 2 files changed, 105 insertions(+), 9 deletions(-) diff --git a/moto/cloudformation/responses.py b/moto/cloudformation/responses.py index 8672c706d..c7ced0186 100644 --- a/moto/cloudformation/responses.py +++ b/moto/cloudformation/responses.py @@ -10,6 +10,31 @@ from moto.s3 import s3_backend from moto.core import ACCOUNT_ID from .models import cloudformation_backends from .exceptions import ValidationError +from .utils import yaml_tag_constructor + + +def get_template_summary_response_from_template(template_body): + def get_resource_types(template_dict): + resources = {} + for key, value in template_dict.items(): + if key == "Resources": + resources = value + + resource_types = [] + for key, value in resources.items(): + resource_types.append(value["Type"]) + return resource_types + + yaml.add_multi_constructor("", yaml_tag_constructor) + + try: + template_dict = yaml.load(template_body, Loader=yaml.Loader) + except (yaml.parser.ParserError, yaml.scanner.ScannerError): + template_dict = json.loads(template_body) + + resources_types = get_resource_types(template_dict) + template_dict["resourceTypes"] = resources_types + return template_dict class CloudFormationResponse(BaseResponse): @@ -269,6 +294,20 @@ class CloudFormationResponse(BaseResponse): template = self.response_template(GET_TEMPLATE_RESPONSE_TEMPLATE) return template.render(stack=stack) + def get_template_summary(self): + stack_name = self._get_param("StackName") + template_url = self._get_param("TemplateURL") + stack_body = self._get_param("TemplateBody") + + if stack_name: + stack_body = self.cloudformation_backend.get_stack(stack_name).template + elif template_url: + stack_body = self._get_stack_from_s3_url(template_url) + + template_summary = get_template_summary_response_from_template(stack_body) + template = self.response_template(GET_TEMPLATE_SUMMARY_TEMPLATE) + return template.render(template_summary=template_summary) + def update_stack(self): stack_name = self._get_param("StackName") role_arn = self._get_param("RoleARN") @@ -743,7 +782,6 @@ DESCRIBE_STACKS_TEMPLATE = """ """ - DESCRIBE_STACK_RESOURCE_RESPONSE_TEMPLATE = """ @@ -758,7 +796,6 @@ DESCRIBE_STACK_RESOURCE_RESPONSE_TEMPLATE = """ """ - DESCRIBE_STACK_RESOURCES_RESPONSE = """ @@ -777,7 +814,6 @@ DESCRIBE_STACK_RESOURCES_RESPONSE = """ """ - DESCRIBE_STACK_EVENTS_RESPONSE = """ @@ -802,7 +838,6 @@ DESCRIBE_STACK_EVENTS_RESPONSE = """ @@ -823,7 +858,6 @@ LIST_CHANGE_SETS_RESPONSE = """ """ - LIST_STACKS_RESPONSE = """ @@ -840,7 +874,6 @@ LIST_STACKS_RESPONSE = """ """ - LIST_STACKS_RESOURCES_RESPONSE = """ @@ -860,7 +893,6 @@ LIST_STACKS_RESOURCES_RESPONSE = """ """ - GET_TEMPLATE_RESPONSE_TEMPLATE = """ {{ stack.template }} @@ -870,7 +902,6 @@ GET_TEMPLATE_RESPONSE_TEMPLATE = """ """ - DELETE_STACK_RESPONSE_TEMPLATE = """ 5ccc7dcd-744c-11e5-be70-example @@ -878,7 +909,6 @@ DELETE_STACK_RESPONSE_TEMPLATE = """ """ - LIST_EXPORTS_RESPONSE = """ @@ -1139,3 +1169,19 @@ LIST_STACK_SET_OPERATION_RESULTS_RESPONSE_TEMPLATE = ( """ ) + +GET_TEMPLATE_SUMMARY_TEMPLATE = """ + + {{ template_summary.Description }} + {% for resource in template_summary.resourceTypes %} + + {{ resource }} + + {% endfor %} + {{ template_summary.AWSTemplateFormatVersion }} + + + b9b4b068-3a41-11e5-94eb-example + + +""" diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py index 1ebce46d7..0bfaf9f09 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py @@ -35,6 +35,14 @@ dummy_template = { }, } +dummy_template3 = { + "AWSTemplateFormatVersion": "2010-09-09", + "Description": "Stack 3", + "Resources": { + "VPC": {"Properties": {"CidrBlock": "192.168.0.0/16"}, "Type": "AWS::EC2::VPC"} + }, +} + dummy_template_yaml = """--- AWSTemplateFormatVersion: 2010-09-09 Description: Stack1 with yaml template @@ -668,6 +676,48 @@ def test_boto3_create_stack_with_short_form_func_yaml(): ) +@mock_s3 +@mock_cloudformation +def test_get_template_summary(): + s3 = boto3.client("s3") + s3_conn = boto3.resource("s3", region_name="us-east-1") + + conn = boto3.client("cloudformation", region_name="us-east-1") + result = conn.get_template_summary(TemplateBody=json.dumps(dummy_template3)) + + result["ResourceTypes"].should.equal(["AWS::EC2::VPC"]) + result["Version"].should.equal("2010-09-09") + result["Description"].should.equal("Stack 3") + + conn.create_stack(StackName="test_stack", TemplateBody=json.dumps(dummy_template3)) + + result = conn.get_template_summary(StackName="test_stack") + + result["ResourceTypes"].should.equal(["AWS::EC2::VPC"]) + result["Version"].should.equal("2010-09-09") + result["Description"].should.equal("Stack 3") + + s3_conn.create_bucket(Bucket="foobar") + s3_conn.Object("foobar", "template-key").put(Body=json.dumps(dummy_template3)) + + key_url = s3.generate_presigned_url( + ClientMethod="get_object", Params={"Bucket": "foobar", "Key": "template-key"} + ) + + conn.create_stack(StackName="stack_from_url", TemplateURL=key_url) + result = conn.get_template_summary(TemplateURL=key_url) + result["ResourceTypes"].should.equal(["AWS::EC2::VPC"]) + result["Version"].should.equal("2010-09-09") + result["Description"].should.equal("Stack 3") + + conn = boto3.client("cloudformation", region_name="us-east-1") + result = conn.get_template_summary(TemplateBody=dummy_template_yaml) + + result["ResourceTypes"].should.equal(["AWS::EC2::Instance"]) + result["Version"].should.equal("2010-09-09") + result["Description"].should.equal("Stack1 with yaml template") + + @mock_cloudformation def test_boto3_create_stack_with_ref_yaml(): cf_conn = boto3.client("cloudformation", region_name="us-east-1") From 1db42fb865aabffdd9b6c60e286231a64e87f77f Mon Sep 17 00:00:00 2001 From: usmangani1 Date: Mon, 27 Jul 2020 20:02:41 +0530 Subject: [PATCH 02/11] FIX : IAM - Added support for pathPrefix in list_users_function (#3180) * FIX:IAM-Added support for pathPrefix in list_users_function * removed changes for roles * Added test for non decorator * changed filter function Co-authored-by: usmankb --- moto/iam/models.py | 8 ++++++++ moto/iam/responses.py | 1 - tests/test_iam/test_iam.py | 6 ++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/moto/iam/models.py b/moto/iam/models.py index 82dc84be5..49755e57a 100755 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -946,6 +946,10 @@ class AccountSummary(BaseModel): return len(self._iam_backend.users) +def filter_items_with_path_prefix(path_prefix, items): + return [role for role in items if role.path.startswith(path_prefix)] + + class IAMBackend(BaseBackend): def __init__(self): self.instance_profiles = {} @@ -1490,7 +1494,11 @@ class IAMBackend(BaseBackend): def list_users(self, path_prefix, marker, max_items): users = None try: + users = self.users.values() + if path_prefix: + users = filter_items_with_path_prefix(path_prefix, users) + except KeyError: raise IAMNotFoundException( "Users {0}, {1}, {2} not found".format(path_prefix, marker, max_items) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 8eb1730ea..6f785f8ac 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -337,7 +337,6 @@ class IamResponse(BaseResponse): def list_roles(self): roles = iam_backend.get_roles() - template = self.response_template(LIST_ROLES_TEMPLATE) return template.render(roles=roles) diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 4ae5ad49e..610333303 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -762,6 +762,12 @@ def test_list_users(): user["Path"].should.equal("/") user["Arn"].should.equal("arn:aws:iam::{}:user/my-user".format(ACCOUNT_ID)) + conn.create_user(UserName="my-user-1", Path="myUser") + response = conn.list_users(PathPrefix="my") + user = response["Users"][0] + user["UserName"].should.equal("my-user-1") + user["Path"].should.equal("myUser") + @mock_iam() def test_user_policies(): From 6adee0cbaf7a1971fff8809cc07dc4b23a08cc2e Mon Sep 17 00:00:00 2001 From: Macwan Nevil Date: Mon, 27 Jul 2020 23:23:15 +0530 Subject: [PATCH 03/11] Bugfix: RedrivePolicy Issue SNS (#3186) * Bugfix: S3 time precision issue fixed * Bugfix: S3 time precision issue fixed * s3 timeformat fix * Quickfix S3 timefix * Bugfix: Redrive Policy Allow * Linting Fixed --- moto/sns/models.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/moto/sns/models.py b/moto/sns/models.py index 85196cd8f..76376e58f 100644 --- a/moto/sns/models.py +++ b/moto/sns/models.py @@ -580,7 +580,12 @@ class SNSBackend(BaseBackend): return subscription.attributes def set_subscription_attributes(self, arn, name, value): - if name not in ["RawMessageDelivery", "DeliveryPolicy", "FilterPolicy"]: + if name not in [ + "RawMessageDelivery", + "DeliveryPolicy", + "FilterPolicy", + "RedrivePolicy", + ]: raise SNSInvalidParameter("AttributeName") # TODO: should do validation From 126f5a5155af519673faf8aef4dcc3232c969447 Mon Sep 17 00:00:00 2001 From: Jordan Reiter Date: Tue, 28 Jul 2020 05:17:35 -0400 Subject: [PATCH 04/11] Implement Filter: Contains functionality for describe_params (#3189) * Implement Filter: Contains functionality for describe_params This commit adds the Contains functionality. Tests were created to mimic behavior in AWS/boto3, including that filters with values in the form of `/name` will match parameters named `/name/match` but not parameters named `match/with/other-name`. In the test example, a Contains filter with the value `/tan` would match: `/tangent-3` and `tangram-4` but not `standby-5`. * Enforce parameter filter restrictions on get_parameters_by_path According to the boto3 documentation [1], `Name`, `Path`, and `Tier` are not allowed values for `Key` in a parameter filter for `get_parameters_by_path`. This commit enforces this by calling `_validate_parameter_filters` from the `get_parameters_by_path` method, and adding a check to `_validate_parameter_filters`. I added 3 test cases to `test_get_parameters_by_path` which check for the correct exception when calling with a parameter filter using any of these keys. * Code formatted to match style * Refactored logic --- moto/ssm/models.py | 20 ++++++++++++-- tests/test_ssm/test_ssm_boto3.py | 46 ++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/moto/ssm/models.py b/moto/ssm/models.py index 28175bb06..3c29097e8 100644 --- a/moto/ssm/models.py +++ b/moto/ssm/models.py @@ -965,6 +965,13 @@ class SimpleSystemManagerBackend(BaseBackend): "The following filter key is not valid: Label. Valid filter keys include: [Path, Name, Type, KeyId, Tier]." ) + if by_path and key in ["Name", "Path", "Tier"]: + raise InvalidFilterKey( + "The following filter key is not valid: {key}. Valid filter keys include: [Type, KeyId].".format( + key=key + ) + ) + if not values: raise InvalidFilterValue( "The following filter values are missing : null for filter key Name." @@ -1024,7 +1031,10 @@ class SimpleSystemManagerBackend(BaseBackend): ) ) - if key != "Path" and option not in ["Equals", "BeginsWith"]: + allowed_options = ["Equals", "BeginsWith"] + if key == "Name": + allowed_options += ["Contains"] + if key != "Path" and option not in allowed_options: raise InvalidFilterOption( "The following filter option is not valid: {option}. Valid options include: [BeginsWith, Equals].".format( option=option @@ -1084,6 +1094,9 @@ class SimpleSystemManagerBackend(BaseBackend): max_results=10, ): """Implement the get-parameters-by-path-API in the backend.""" + + self._validate_parameter_filters(filters, by_path=True) + result = [] # path could be with or without a trailing /. we handle this # difference here. @@ -1134,7 +1147,8 @@ class SimpleSystemManagerBackend(BaseBackend): what = parameter.keyid elif key == "Name": what = "/" + parameter.name.lstrip("/") - values = ["/" + value.lstrip("/") for value in values] + if option != "Contains": + values = ["/" + value.lstrip("/") for value in values] elif key == "Path": what = "/" + parameter.name.lstrip("/") values = ["/" + value.strip("/") for value in values] @@ -1147,6 +1161,8 @@ class SimpleSystemManagerBackend(BaseBackend): what.startswith(value) for value in values ): return False + elif option == "Contains" and not any(value in what for value in values): + return False elif option == "Equals" and not any(what == value for value in values): return False elif option == "OneLevel": diff --git a/tests/test_ssm/test_ssm_boto3.py b/tests/test_ssm/test_ssm_boto3.py index 837f81bf5..e899613e0 100644 --- a/tests/test_ssm/test_ssm_boto3.py +++ b/tests/test_ssm/test_ssm_boto3.py @@ -198,6 +198,33 @@ def test_get_parameters_by_path(): len(response["Parameters"]).should.equal(1) response.should_not.have.key("NextToken") + filters = [{"Key": "Name", "Values": ["error"]}] + client.get_parameters_by_path.when.called_with( + Path="/baz", ParameterFilters=filters + ).should.throw( + ClientError, + "The following filter key is not valid: Name. " + "Valid filter keys include: [Type, KeyId].", + ) + + filters = [{"Key": "Path", "Values": ["/error"]}] + client.get_parameters_by_path.when.called_with( + Path="/baz", ParameterFilters=filters + ).should.throw( + ClientError, + "The following filter key is not valid: Path. " + "Valid filter keys include: [Type, KeyId].", + ) + + filters = [{"Key": "Tier", "Values": ["Standard"]}] + client.get_parameters_by_path.when.called_with( + Path="/baz", ParameterFilters=filters + ).should.throw( + ClientError, + "The following filter key is not valid: Tier. " + "Valid filter keys include: [Type, KeyId].", + ) + @mock_ssm def test_put_parameter(): @@ -504,6 +531,9 @@ def test_describe_parameters_with_parameter_filters_name(): client = boto3.client("ssm", region_name="us-east-1") client.put_parameter(Name="param", Value="value", Type="String") client.put_parameter(Name="/param-2", Value="value-2", Type="String") + client.put_parameter(Name="/tangent-3", Value="value-3", Type="String") + client.put_parameter(Name="tangram-4", Value="value-4", Type="String") + client.put_parameter(Name="standby-5", Value="value-5", Type="String") response = client.describe_parameters( ParameterFilters=[{"Key": "Name", "Values": ["param"]}] @@ -543,6 +573,22 @@ def test_describe_parameters_with_parameter_filters_name(): parameters.should.have.length_of(2) response.should_not.have.key("NextToken") + response = client.describe_parameters( + ParameterFilters=[{"Key": "Name", "Option": "Contains", "Values": ["ram"]}] + ) + + parameters = response["Parameters"] + parameters.should.have.length_of(3) + response.should_not.have.key("NextToken") + + response = client.describe_parameters( + ParameterFilters=[{"Key": "Name", "Option": "Contains", "Values": ["/tan"]}] + ) + + parameters = response["Parameters"] + parameters.should.have.length_of(2) + response.should_not.have.key("NextToken") + @mock_ssm def test_describe_parameters_with_parameter_filters_path(): From 97139d4253db409b28265c87dcbadc529a6549de Mon Sep 17 00:00:00 2001 From: Ninh Khong Date: Tue, 28 Jul 2020 20:34:26 +0700 Subject: [PATCH 05/11] Fix : SQS - Added support for attribute labels for send_message function (#3181) * Fix : SQS - Added support for attribute labels for send_message function * Add integration test on receive message function * Add send message invalid datetype integration test and fix SQS MessageAttributesInvalid exceptions --- moto/sqs/exceptions.py | 8 ++-- moto/sqs/models.py | 18 ++++++-- moto/sqs/responses.py | 13 +----- moto/sqs/utils.py | 2 +- tests/test_sqs/test_sqs.py | 92 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 114 insertions(+), 19 deletions(-) diff --git a/moto/sqs/exceptions.py b/moto/sqs/exceptions.py index 77d7b9fb2..46d2af400 100644 --- a/moto/sqs/exceptions.py +++ b/moto/sqs/exceptions.py @@ -16,11 +16,13 @@ class ReceiptHandleIsInvalid(RESTError): ) -class MessageAttributesInvalid(Exception): - status_code = 400 +class MessageAttributesInvalid(RESTError): + code = 400 def __init__(self, description): - self.description = description + super(MessageAttributesInvalid, self).__init__( + "MessageAttributesInvalid", description + ) class QueueDoesNotExist(RESTError): diff --git a/moto/sqs/models.py b/moto/sqs/models.py index 5b6e6410a..085416457 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -87,7 +87,19 @@ class Message(BaseModel): struct_format = "!I".encode("ascii") # ensure it's a bytestring for name in sorted(self.message_attributes.keys()): attr = self.message_attributes[name] - data_type = attr["data_type"] + data_type_parts = attr["data_type"].split(".") + data_type = data_type_parts[0] + + if data_type not in [ + "String", + "Binary", + "Number", + ]: + raise MessageAttributesInvalid( + "The message attribute '{0}' has an invalid message attribute type, the set of supported type prefixes is Binary, Number, and String.".format( + name[0] + ) + ) encoded = utf8("") # Each part of each attribute is encoded right after it's @@ -243,9 +255,7 @@ class Queue(BaseModel): # Check some conditions if self.fifo_queue and not self.name.endswith(".fifo"): - raise MessageAttributesInvalid( - "Queue name must end in .fifo for FIFO queues" - ) + raise InvalidParameterValue("Queue name must end in .fifo for FIFO queues") @property def pending_messages(self): diff --git a/moto/sqs/responses.py b/moto/sqs/responses.py index 54a8bc267..29804256c 100644 --- a/moto/sqs/responses.py +++ b/moto/sqs/responses.py @@ -9,7 +9,6 @@ from six.moves.urllib.parse import urlparse from .exceptions import ( EmptyBatchRequest, InvalidAttributeName, - MessageAttributesInvalid, MessageNotInflight, ReceiptHandleIsInvalid, ) @@ -82,12 +81,7 @@ class SQSResponse(BaseResponse): request_url = urlparse(self.uri) queue_name = self._get_param("QueueName") - try: - queue = self.sqs_backend.create_queue( - queue_name, self.tags, **self.attribute - ) - except MessageAttributesInvalid as e: - return self._error("InvalidParameterValue", e.description) + queue = self.sqs_backend.create_queue(queue_name, self.tags, **self.attribute) template = self.response_template(CREATE_QUEUE_RESPONSE) return template.render(queue_url=queue.url(request_url)) @@ -225,10 +219,7 @@ class SQSResponse(BaseResponse): if len(message) > MAXIMUM_MESSAGE_LENGTH: return ERROR_TOO_LONG_RESPONSE, dict(status=400) - try: - message_attributes = parse_message_attributes(self.querystring) - except MessageAttributesInvalid as e: - return e.description, dict(status=e.status_code) + message_attributes = parse_message_attributes(self.querystring) queue_name = self._get_queue_name() diff --git a/moto/sqs/utils.py b/moto/sqs/utils.py index f3b8bbfe8..315fce56b 100644 --- a/moto/sqs/utils.py +++ b/moto/sqs/utils.py @@ -34,7 +34,7 @@ def parse_message_attributes(querystring, base="", value_namespace="Value."): ) data_type_parts = data_type[0].split(".") - if len(data_type_parts) > 2 or data_type_parts[0] not in [ + if data_type_parts[0] not in [ "String", "Binary", "Number", diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index 2ed757f6a..9e3896154 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -248,6 +248,50 @@ def test_message_with_complex_attributes(): messages.should.have.length_of(1) +@mock_sqs +def test_message_with_attributes_have_labels(): + sqs = boto3.resource("sqs", region_name="us-east-1") + queue = sqs.create_queue(QueueName="blah") + msg = queue.send_message( + MessageBody="derp", + MessageAttributes={ + "timestamp": { + "DataType": "Number.java.lang.Long", + "StringValue": "1493147359900", + } + }, + ) + msg.get("MD5OfMessageBody").should.equal("58fd9edd83341c29f1aebba81c31e257") + msg.get("MD5OfMessageAttributes").should.equal("235c5c510d26fb653d073faed50ae77c") + msg.get("MessageId").should_not.contain(" \n") + + messages = queue.receive_messages() + messages.should.have.length_of(1) + + +@mock_sqs +def test_message_with_attributes_invalid_datatype(): + sqs = boto3.resource("sqs", region_name="us-east-1") + queue = sqs.create_queue(QueueName="blah") + + with assert_raises(ClientError) as e: + queue.send_message( + MessageBody="derp", + MessageAttributes={ + "timestamp": { + "DataType": "InvalidNumber", + "StringValue": "149314735990a", + } + }, + ) + ex = e.exception + ex.response["Error"]["Code"].should.equal("MessageAttributesInvalid") + ex.response["Error"]["Message"].should.equal( + "The message attribute 'timestamp' has an invalid message attribute type, the set of supported type " + "prefixes is Binary, Number, and String." + ) + + @mock_sqs def test_send_message_with_message_group_id(): sqs = boto3.resource("sqs", region_name="us-east-1") @@ -532,6 +576,54 @@ def test_send_receive_message_with_attributes(): ) +@mock_sqs +def test_send_receive_message_with_attributes_with_labels(): + sqs = boto3.resource("sqs", region_name="us-east-1") + conn = boto3.client("sqs", region_name="us-east-1") + conn.create_queue(QueueName="test-queue") + queue = sqs.Queue("test-queue") + + body_one = "this is a test message" + body_two = "this is another test message" + + queue.send_message( + MessageBody=body_one, + MessageAttributes={ + "timestamp": { + "StringValue": "1493147359900", + "DataType": "Number.java.lang.Long", + } + }, + ) + + queue.send_message( + MessageBody=body_two, + MessageAttributes={ + "timestamp": { + "StringValue": "1493147359901", + "DataType": "Number.java.lang.Long", + } + }, + ) + + messages = conn.receive_message(QueueUrl=queue.url, MaxNumberOfMessages=2)[ + "Messages" + ] + + message1 = messages[0] + message2 = messages[1] + + message1.get("Body").should.equal(body_one) + message2.get("Body").should.equal(body_two) + + message1.get("MD5OfMessageAttributes").should.equal( + "235c5c510d26fb653d073faed50ae77c" + ) + message2.get("MD5OfMessageAttributes").should.equal( + "994258b45346a2cc3f9cbb611aa7af30" + ) + + @mock_sqs def test_send_receive_message_timestamps(): sqs = boto3.resource("sqs", region_name="us-east-1") From 28d1d762af57393e34e222496b574c253597f149 Mon Sep 17 00:00:00 2001 From: Jordan Reiter Date: Tue, 28 Jul 2020 10:26:59 -0400 Subject: [PATCH 06/11] Enforce parameter naming (#3190) * Enforce parameter naming Parameters are not allowed to start with `ssm` or `aws`. This commit adds error messages which correspond exactly to the error messages returned by boto3. * Fix for Python 2 compatibility f-strings not supported in Python 2.7 --- moto/ssm/exceptions.py | 7 ++++ moto/ssm/models.py | 18 +++++++++ tests/test_ssm/test_ssm_boto3.py | 67 ++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+) diff --git a/moto/ssm/exceptions.py b/moto/ssm/exceptions.py index 2e715f16a..f68e47029 100644 --- a/moto/ssm/exceptions.py +++ b/moto/ssm/exceptions.py @@ -78,6 +78,13 @@ class InvalidDocumentOperation(JsonRESTError): ) +class AccessDeniedException(JsonRESTError): + code = 400 + + def __init__(self, message): + super(AccessDeniedException, self).__init__("AccessDeniedException", message) + + class InvalidDocumentContent(JsonRESTError): code = 400 diff --git a/moto/ssm/models.py b/moto/ssm/models.py index 3c29097e8..37d56c2dd 100644 --- a/moto/ssm/models.py +++ b/moto/ssm/models.py @@ -27,6 +27,7 @@ from .exceptions import ( ParameterNotFound, DocumentAlreadyExists, InvalidDocumentOperation, + AccessDeniedException, InvalidDocument, InvalidDocumentContent, InvalidDocumentVersion, @@ -1254,6 +1255,23 @@ class SimpleSystemManagerBackend(BaseBackend): def put_parameter( self, name, description, value, type, allowed_pattern, keyid, overwrite ): + if name.lower().lstrip("/").startswith("aws") or name.lower().lstrip( + "/" + ).startswith("ssm"): + is_path = name.count("/") > 1 + if name.lower().startswith("/aws") and is_path: + raise AccessDeniedException( + "No access to reserved parameter name: {name}.".format(name=name) + ) + if not is_path: + invalid_prefix_error = 'Parameter name: can\'t be prefixed with "aws" or "ssm" (case-insensitive).' + else: + invalid_prefix_error = ( + 'Parameter name: can\'t be prefixed with "ssm" (case-insensitive). ' + "If formed as a path, it can consist of sub-paths divided by slash symbol; each sub-path can be " + "formed as a mix of letters, numbers and the following 3 symbols .-_" + ) + raise ValidationException(invalid_prefix_error) previous_parameter_versions = self._parameters[name] if len(previous_parameter_versions) == 0: previous_parameter = None diff --git a/tests/test_ssm/test_ssm_boto3.py b/tests/test_ssm/test_ssm_boto3.py index e899613e0..9715866e9 100644 --- a/tests/test_ssm/test_ssm_boto3.py +++ b/tests/test_ssm/test_ssm_boto3.py @@ -299,6 +299,73 @@ def test_put_parameter(): ) +@mock_ssm +def test_put_parameter_invalid_names(): + client = boto3.client("ssm", region_name="us-east-1") + + invalid_prefix_err = ( + 'Parameter name: can\'t be prefixed with "aws" or "ssm" (case-insensitive).' + ) + + client.put_parameter.when.called_with( + Name="ssm_test", Value="value", Type="String" + ).should.throw( + ClientError, invalid_prefix_err, + ) + + client.put_parameter.when.called_with( + Name="SSM_TEST", Value="value", Type="String" + ).should.throw( + ClientError, invalid_prefix_err, + ) + + client.put_parameter.when.called_with( + Name="aws_test", Value="value", Type="String" + ).should.throw( + ClientError, invalid_prefix_err, + ) + + client.put_parameter.when.called_with( + Name="AWS_TEST", Value="value", Type="String" + ).should.throw( + ClientError, invalid_prefix_err, + ) + + ssm_path = "/ssm_test/path/to/var" + client.put_parameter.when.called_with( + Name=ssm_path, Value="value", Type="String" + ).should.throw( + ClientError, + 'Parameter name: can\'t be prefixed with "ssm" (case-insensitive). If formed as a path, it can consist of ' + "sub-paths divided by slash symbol; each sub-path can be formed as a mix of letters, numbers and the following " + "3 symbols .-_", + ) + + ssm_path = "/SSM/PATH/TO/VAR" + client.put_parameter.when.called_with( + Name=ssm_path, Value="value", Type="String" + ).should.throw( + ClientError, + 'Parameter name: can\'t be prefixed with "ssm" (case-insensitive). If formed as a path, it can consist of ' + "sub-paths divided by slash symbol; each sub-path can be formed as a mix of letters, numbers and the following " + "3 symbols .-_", + ) + + aws_path = "/aws_test/path/to/var" + client.put_parameter.when.called_with( + Name=aws_path, Value="value", Type="String" + ).should.throw( + ClientError, "No access to reserved parameter name: {}.".format(aws_path), + ) + + aws_path = "/AWS/PATH/TO/VAR" + client.put_parameter.when.called_with( + Name=aws_path, Value="value", Type="String" + ).should.throw( + ClientError, "No access to reserved parameter name: {}.".format(aws_path), + ) + + @mock_ssm def test_put_parameter_china(): client = boto3.client("ssm", region_name="cn-north-1") From ff1f5651429d69fbb390caf6ffc7b430dca8ded9 Mon Sep 17 00:00:00 2001 From: Ninh Khong Date: Tue, 28 Jul 2020 22:59:22 +0700 Subject: [PATCH 07/11] Enhance function get_parameter by parameter name, version or labels (#3191) Co-authored-by: Ninh Khong --- moto/ssm/models.py | 29 ++++++++++++++-- tests/test_ssm/test_ssm_boto3.py | 58 ++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/moto/ssm/models.py b/moto/ssm/models.py index 37d56c2dd..07812c316 100644 --- a/moto/ssm/models.py +++ b/moto/ssm/models.py @@ -1189,8 +1189,33 @@ class SimpleSystemManagerBackend(BaseBackend): return True def get_parameter(self, name, with_decryption): - if name in self._parameters: - return self._parameters[name][-1] + name_parts = name.split(":") + name_prefix = name_parts[0] + + if len(name_parts) > 2: + return None + + if name_prefix in self._parameters: + if len(name_parts) == 1: + return self._parameters[name][-1] + + if len(name_parts) == 2: + version_or_label = name_parts[1] + parameters = self._parameters[name_prefix] + + if version_or_label.isdigit(): + result = list( + filter(lambda x: str(x.version) == version_or_label, parameters) + ) + if len(result) > 0: + return result[-1] + + result = list( + filter(lambda x: version_or_label in x.labels, parameters) + ) + if len(result) > 0: + return result[-1] + return None def label_parameter_version(self, name, version, labels): diff --git a/tests/test_ssm/test_ssm_boto3.py b/tests/test_ssm/test_ssm_boto3.py index 9715866e9..cc79ce93d 100644 --- a/tests/test_ssm/test_ssm_boto3.py +++ b/tests/test_ssm/test_ssm_boto3.py @@ -396,6 +396,64 @@ def test_get_parameter(): ) +@mock_ssm +def test_get_parameter_with_version_and_labels(): + client = boto3.client("ssm", region_name="us-east-1") + + client.put_parameter( + Name="test-1", Description="A test parameter", Value="value", Type="String" + ) + client.put_parameter( + Name="test-2", Description="A test parameter", Value="value", Type="String" + ) + + client.label_parameter_version( + Name="test-2", ParameterVersion=1, Labels=["test-label"] + ) + + response = client.get_parameter(Name="test-1:1", WithDecryption=False) + + response["Parameter"]["Name"].should.equal("test-1") + response["Parameter"]["Value"].should.equal("value") + response["Parameter"]["Type"].should.equal("String") + response["Parameter"]["LastModifiedDate"].should.be.a(datetime.datetime) + response["Parameter"]["ARN"].should.equal( + "arn:aws:ssm:us-east-1:1234567890:parameter/test-1" + ) + + response = client.get_parameter(Name="test-2:1", WithDecryption=False) + response["Parameter"]["Name"].should.equal("test-2") + response["Parameter"]["Value"].should.equal("value") + response["Parameter"]["Type"].should.equal("String") + response["Parameter"]["LastModifiedDate"].should.be.a(datetime.datetime) + response["Parameter"]["ARN"].should.equal( + "arn:aws:ssm:us-east-1:1234567890:parameter/test-2" + ) + + response = client.get_parameter(Name="test-2:test-label", WithDecryption=False) + response["Parameter"]["Name"].should.equal("test-2") + response["Parameter"]["Value"].should.equal("value") + response["Parameter"]["Type"].should.equal("String") + response["Parameter"]["LastModifiedDate"].should.be.a(datetime.datetime) + response["Parameter"]["ARN"].should.equal( + "arn:aws:ssm:us-east-1:1234567890:parameter/test-2" + ) + + with assert_raises(ClientError) as ex: + client.get_parameter(Name="test-2:2:3", WithDecryption=False) + ex.exception.response["Error"]["Code"].should.equal("ParameterNotFound") + ex.exception.response["Error"]["Message"].should.equal( + "Parameter test-2:2:3 not found." + ) + + with assert_raises(ClientError) as ex: + client.get_parameter(Name="test-2:2", WithDecryption=False) + ex.exception.response["Error"]["Code"].should.equal("ParameterNotFound") + ex.exception.response["Error"]["Message"].should.equal( + "Parameter test-2:2 not found." + ) + + @mock_ssm def test_get_parameters_errors(): client = boto3.client("ssm", region_name="us-east-1") From 736c8b77ce8620ecb45d5babaa37715976543bd3 Mon Sep 17 00:00:00 2001 From: jweite Date: Wed, 29 Jul 2020 02:47:18 -0400 Subject: [PATCH 08/11] Fixed Failures in CloudFormation Provisioning of S3 Buckets When Stack has Long Name... (#3169) * Fixed defect with CloudFormation provisioning of S3 buckets occuring when stack has a long name, resulting in the default S3 bucket name's length exceeding its 63 char limit. * PR 3169 July 23, 2020 2:57a ET comment: added additional asserts to assure provisioned bucket's name complies. Fixed bug in my earlier change that could produce default bucket names with illegal upper-case characters in it. Co-authored-by: Joseph Weitekamp --- moto/cloudformation/parsing.py | 6 +++++ .../test_cloudformation_stack_crud_boto3.py | 25 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index 0a3e0a0c2..58409901d 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -317,6 +317,12 @@ def generate_resource_name(resource_type, stack_name, logical_id): if truncated_name_prefix.endswith("-"): truncated_name_prefix = truncated_name_prefix[:-1] return "{0}-{1}".format(truncated_name_prefix, my_random_suffix) + elif resource_type == "AWS::S3::Bucket": + right_hand_part_of_name = "-{0}-{1}".format(logical_id, random_suffix()) + max_stack_name_portion_len = 63 - len(right_hand_part_of_name) + return "{0}{1}".format( + stack_name[:max_stack_name_portion_len], right_hand_part_of_name + ).lower() else: return "{0}-{1}-{2}".format(stack_name, logical_id, random_suffix()) diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py index 0bfaf9f09..41d3fad3e 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py @@ -654,6 +654,31 @@ def test_boto3_create_stack(): ) +@mock_cloudformation +def test_boto3_create_stack_s3_long_name(): + cf_conn = boto3.client("cloudformation", region_name="us-east-1") + + stack_name = "MyLongStackName01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012" + + template = '{"Resources":{"HelloBucket":{"Type":"AWS::S3::Bucket"}}}' + + cf_conn.create_stack(StackName=stack_name, TemplateBody=template) + + cf_conn.get_template(StackName=stack_name)["TemplateBody"].should.equal( + json.loads(template, object_pairs_hook=OrderedDict) + ) + provisioned_resource = cf_conn.list_stack_resources(StackName=stack_name)[ + "StackResourceSummaries" + ][0] + provisioned_bucket_name = provisioned_resource["PhysicalResourceId"] + len(provisioned_bucket_name).should.be.lower_than(64) + logical_name_lower_case = provisioned_resource["LogicalResourceId"].lower() + bucket_name_stack_name_prefix = provisioned_bucket_name[ + : provisioned_bucket_name.index("-" + logical_name_lower_case) + ] + stack_name.lower().should.contain(bucket_name_stack_name_prefix) + + @mock_cloudformation def test_boto3_create_stack_with_yaml(): cf_conn = boto3.client("cloudformation", region_name="us-east-1") From 08a08b6af8572f71870344b24c9de46e05315cf3 Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Wed, 29 Jul 2020 12:44:02 +0200 Subject: [PATCH 09/11] Fix SQS tag list from CloudFormation resource creation (#3197) * fix sqs tag list from cloudformation resource creation the method `create_from_cloudformation_json` of the Sqs resource does not handle the difference of format of the Tags field in the resource template and the format expected in Sqs resource class. In cfn resource template Tags is specified as a list of dicts. But the Sqs resource expects that the tags field be a single dict. This behaviour causes a crash when a queue is created with tags from `create_from_cloudformation_json` and later the list_queue_tags is called because it tries to call `items` from `queue.tags` but tags is actually a list of dicts. * fix comment * fix linter * minor Co-authored-by: Hudo Assenco --- moto/core/utils.py | 11 +++++++++++ moto/sqs/models.py | 11 +++++++++-- tests/test_sqs/test_sqs.py | 38 +++++++++++++++++++++++++++++++++++++- 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/moto/core/utils.py b/moto/core/utils.py index c9bf93473..235b895ec 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -356,3 +356,14 @@ def tags_from_query_string( else: response_values[tag_key] = None return response_values + + +def tags_from_cloudformation_tags_list(tags_list): + """Return tags in dict form from cloudformation resource tags form (list of dicts)""" + tags = {} + for entry in tags_list: + key = entry["Key"] + value = entry["Value"] + tags[key] = value + + return tags diff --git a/moto/sqs/models.py b/moto/sqs/models.py index 085416457..4befbb50a 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -18,6 +18,7 @@ from moto.core.utils import ( get_random_message_id, unix_time, unix_time_millis, + tags_from_cloudformation_tags_list, ) from .utils import generate_receipt_handle from .exceptions import ( @@ -357,11 +358,17 @@ class Queue(BaseModel): def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name ): - properties = cloudformation_json["Properties"] + properties = deepcopy(cloudformation_json["Properties"]) + # remove Tags from properties and convert tags list to dict + tags = properties.pop("Tags", []) + tags_dict = tags_from_cloudformation_tags_list(tags) sqs_backend = sqs_backends[region_name] return sqs_backend.create_queue( - name=properties["QueueName"], region=region_name, **properties + name=properties["QueueName"], + tags=tags_dict, + region=region_name, + **properties ) @classmethod diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index 9e3896154..61edcaa9b 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -17,12 +17,34 @@ from boto.exception import SQSError from boto.sqs.message import Message, RawMessage from botocore.exceptions import ClientError from freezegun import freeze_time -from moto import mock_sqs, mock_sqs_deprecated, settings +from moto import mock_sqs, mock_sqs_deprecated, mock_cloudformation, settings from nose import SkipTest from nose.tools import assert_raises from tests.helpers import requires_boto_gte from moto.core import ACCOUNT_ID +sqs_template_with_tags = """ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "SQSQueue": { + "Type": "AWS::SQS::Queue", + "Properties": { + "Tags" : [ + { + "Key" : "keyname1", + "Value" : "value1" + }, + { + "Key" : "keyname2", + "Value" : "value2" + } + ] + } + } + } +}""" + @mock_sqs def test_create_fifo_queue_fail(): @@ -1933,3 +1955,17 @@ def test_send_messages_to_fifo_without_message_group_id(): ex.response["Error"]["Message"].should.equal( "The request must contain the parameter MessageGroupId." ) + + +@mock_sqs +@mock_cloudformation +def test_create_from_cloudformation_json_with_tags(): + cf = boto3.client("cloudformation", region_name="us-east-1") + client = boto3.client("sqs", region_name="us-east-1") + + cf.create_stack(StackName="test-sqs", TemplateBody=sqs_template_with_tags) + + queue_url = client.list_queues()["QueueUrls"][0] + + queue_tags = client.list_queue_tags(QueueUrl=queue_url)["Tags"] + queue_tags.should.equal({"keyname1": "value1", "keyname2": "value2"}) From 50d71eccbee12ce5cf2c1ecf61bfcbd60317976d Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Wed, 29 Jul 2020 18:36:37 +0100 Subject: [PATCH 10/11] Fix XML schema for ec2.describe_instance_types (#3194) * Add test for describe_instance_types It currently fails due to an invalid XML schema * Add more detail to test * Fix the XML schema for describe_instance_types --- moto/ec2/responses/instances.py | 26 +++++++++++++++++++------- tests/test_ec2/test_instance_types.py | 18 ++++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) create mode 100644 tests/test_ec2/test_instance_types.py diff --git a/moto/ec2/responses/instances.py b/moto/ec2/responses/instances.py index 9090847be..e9843399f 100644 --- a/moto/ec2/responses/instances.py +++ b/moto/ec2/responses/instances.py @@ -818,13 +818,25 @@ EC2_DESCRIBE_INSTANCE_TYPES = """ {% for instance_type in instance_types %} - {{ instance_type.name }} - {{ instance_type.cores }} - {{ instance_type.memory }} - {{ instance_type.disk }} - {{ instance_type.storageCount }} - {{ instance_type.maxIpAddresses }} - {{ instance_type.ebsOptimizedAvailable }} + {{ instance_type.name }} + + {{ instance_type.cores }} + {{ instance_type.cores }} + 1 + + + {{ instance_type.memory }} + + + {{ instance_type.disk }} + + + + + x86_64 + + + {% endfor %} diff --git a/tests/test_ec2/test_instance_types.py b/tests/test_ec2/test_instance_types.py new file mode 100644 index 000000000..1385d6113 --- /dev/null +++ b/tests/test_ec2/test_instance_types.py @@ -0,0 +1,18 @@ +from __future__ import unicode_literals + +import boto3 +import sure # noqa + +from moto import mock_ec2 + + +@mock_ec2 +def test_describe_instance_types(): + client = boto3.client("ec2", "us-east-1") + instance_types = client.describe_instance_types() + + instance_types.should.have.key("InstanceTypes") + instance_types["InstanceTypes"].should_not.be.empty + instance_types["InstanceTypes"][0].should.have.key("InstanceType") + instance_types["InstanceTypes"][0].should.have.key("MemoryInfo") + instance_types["InstanceTypes"][0]["MemoryInfo"].should.have.key("SizeInMiB") From a9ac09952b31ddd9fb2ab5bf92bf603ca72e10d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anton=20Gr=C3=BCbel?= Date: Fri, 31 Jul 2020 08:18:52 +0200 Subject: [PATCH 11/11] Fix resource groups tests (#3204) --- moto/resourcegroups/urls.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/moto/resourcegroups/urls.py b/moto/resourcegroups/urls.py index b40179145..3e5f7b7f5 100644 --- a/moto/resourcegroups/urls.py +++ b/moto/resourcegroups/urls.py @@ -4,9 +4,14 @@ from .responses import ResourceGroupsResponse url_bases = ["https?://resource-groups(-fips)?.(.+).amazonaws.com"] url_paths = { + "{0}/delete-group$": ResourceGroupsResponse.dispatch, + "{0}/get-group$": ResourceGroupsResponse.dispatch, + "{0}/get-group-query$": ResourceGroupsResponse.dispatch, "{0}/groups$": ResourceGroupsResponse.dispatch, "{0}/groups/(?P[^/]+)$": ResourceGroupsResponse.dispatch, "{0}/groups/(?P[^/]+)/query$": ResourceGroupsResponse.dispatch, "{0}/groups-list$": ResourceGroupsResponse.dispatch, "{0}/resources/(?P[^/]+)/tags$": ResourceGroupsResponse.dispatch, + "{0}/update-group$": ResourceGroupsResponse.dispatch, + "{0}/update-group-query$": ResourceGroupsResponse.dispatch, }