From 23dfecc845b774493c814745e712f8feb7296402 Mon Sep 17 00:00:00 2001 From: gruebel Date: Sat, 21 Mar 2020 19:25:25 +0100 Subject: [PATCH 01/11] Fix missing MessageAttributes when using RawMessageDelivery --- moto/sns/models.py | 40 +++++++++++++----- tests/test_sns/test_publishing_boto3.py | 54 +++++++++++++++++-------- 2 files changed, 66 insertions(+), 28 deletions(-) diff --git a/moto/sns/models.py b/moto/sns/models.py index d6791eecf..85196cd8f 100644 --- a/moto/sns/models.py +++ b/moto/sns/models.py @@ -146,20 +146,38 @@ class Subscription(BaseModel): queue_name = self.endpoint.split(":")[-1] region = self.endpoint.split(":")[3] if self.attributes.get("RawMessageDelivery") != "true": - enveloped_message = json.dumps( - self.get_post_data( - message, - message_id, - subject, - message_attributes=message_attributes, + sqs_backends[region].send_message( + queue_name, + json.dumps( + self.get_post_data( + message, + message_id, + subject, + message_attributes=message_attributes, + ), + sort_keys=True, + indent=2, + separators=(",", ": "), ), - sort_keys=True, - indent=2, - separators=(",", ": "), ) else: - enveloped_message = message - sqs_backends[region].send_message(queue_name, enveloped_message) + raw_message_attributes = {} + for key, value in message_attributes.items(): + type = "string_value" + type_value = value["Value"] + if value["Type"].startswith("Binary"): + type = "binary_value" + elif value["Type"].startswith("Number"): + type_value = "{0:g}".format(value["Value"]) + + raw_message_attributes[key] = { + "data_type": value["Type"], + type: type_value, + } + + sqs_backends[region].send_message( + queue_name, message, message_attributes=raw_message_attributes + ) elif self.protocol in ["http", "https"]: post_data = self.get_post_data(message, message_id, subject) requests.post( diff --git a/tests/test_sns/test_publishing_boto3.py b/tests/test_sns/test_publishing_boto3.py index 51e0a9f57..fddd9125c 100644 --- a/tests/test_sns/test_publishing_boto3.py +++ b/tests/test_sns/test_publishing_boto3.py @@ -148,34 +148,42 @@ def test_publish_to_sqs_msg_attr_byte_value(): conn.create_topic(Name="some-topic") response = conn.list_topics() topic_arn = response["Topics"][0]["TopicArn"] - - sqs_conn = boto3.resource("sqs", region_name="us-east-1") - queue = sqs_conn.create_queue(QueueName="test-queue") - + sqs = boto3.resource("sqs", region_name="us-east-1") + queue = sqs.create_queue(QueueName="test-queue") + conn.subscribe( + TopicArn=topic_arn, Protocol="sqs", Endpoint=queue.attributes["QueueArn"], + ) + queue_raw = sqs.create_queue(QueueName="test-queue-raw") conn.subscribe( TopicArn=topic_arn, Protocol="sqs", - Endpoint="arn:aws:sqs:us-east-1:{}:test-queue".format(ACCOUNT_ID), + Endpoint=queue_raw.attributes["QueueArn"], + Attributes={"RawMessageDelivery": "true"}, ) - message = "my message" + conn.publish( TopicArn=topic_arn, - Message=message, + Message="my message", MessageAttributes={ "store": {"DataType": "Binary", "BinaryValue": b"\x02\x03\x04"} }, ) - messages = queue.receive_messages(MaxNumberOfMessages=5) - message_attributes = [json.loads(m.body)["MessageAttributes"] for m in messages] - message_attributes.should.equal( - [ - { - "store": { - "Type": "Binary", - "Value": base64.b64encode(b"\x02\x03\x04").decode(), - } + + message = json.loads(queue.receive_messages()[0].body) + message["Message"].should.equal("my message") + message["MessageAttributes"].should.equal( + { + "store": { + "Type": "Binary", + "Value": base64.b64encode(b"\x02\x03\x04").decode(), } - ] + } + ) + + message = queue_raw.receive_messages()[0] + message.body.should.equal("my message") + message.message_attributes.should.equal( + {"store": {"DataType": "Binary", "BinaryValue": b"\x02\x03\x04"}} ) @@ -187,6 +195,12 @@ def test_publish_to_sqs_msg_attr_number_type(): sqs = boto3.resource("sqs", region_name="us-east-1") queue = sqs.create_queue(QueueName="test-queue") topic.subscribe(Protocol="sqs", Endpoint=queue.attributes["QueueArn"]) + queue_raw = sqs.create_queue(QueueName="test-queue-raw") + topic.subscribe( + Protocol="sqs", + Endpoint=queue_raw.attributes["QueueArn"], + Attributes={"RawMessageDelivery": "true"}, + ) topic.publish( Message="test message", @@ -199,6 +213,12 @@ def test_publish_to_sqs_msg_attr_number_type(): {"retries": {"Type": "Number", "Value": 0}} ) + message = queue_raw.receive_messages()[0] + message.body.should.equal("test message") + message.message_attributes.should.equal( + {"retries": {"DataType": "Number", "StringValue": "0"}} + ) + @mock_sns def test_publish_sms(): From 7318523b50c48e3aed3b52ef745dde34f3f2a9ed Mon Sep 17 00:00:00 2001 From: Guilherme Martins Crocetti Date: Sun, 22 Mar 2020 16:30:16 -0300 Subject: [PATCH 02/11] Add cloudformation support for EventBridge --- moto/cloudformation/parsing.py | 3 +++ moto/events/models.py | 18 ++++++++++++++ .../test_cloudformation_stack_crud.py | 24 +++++++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index 79276c8fc..60eee63aa 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -1,3 +1,4 @@ + from __future__ import unicode_literals import functools import json @@ -18,6 +19,7 @@ from moto.ec2 import models as ec2_models from moto.ecs import models as ecs_models from moto.elb import models as elb_models from moto.elbv2 import models as elbv2_models +from moto.events import models as events_models from moto.iam import models as iam_models from moto.kinesis import models as kinesis_models from moto.kms import models as kms_models @@ -94,6 +96,7 @@ MODEL_MAP = { "AWS::SNS::Topic": sns_models.Topic, "AWS::S3::Bucket": s3_models.FakeBucket, "AWS::SQS::Queue": sqs_models.Queue, + "AWS::Events::Rule": events_models.Rule, } # http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-name.html diff --git a/moto/events/models.py b/moto/events/models.py index a80b86daa..2f6f3b869 100644 --- a/moto/events/models.py +++ b/moto/events/models.py @@ -55,6 +55,24 @@ class Rule(BaseModel): if index is not None: self.targets.pop(index) + @classmethod + def create_from_cloudformation_json( + cls, resource_name, cloudformation_json, region_name + ): + properties = cloudformation_json["Properties"] + event_backend = events_backends[region_name] + event_name = properties.get("Name") or resource_name + return event_backend.put_rule(name=event_name, **properties) + + @classmethod + def delete_from_cloudformation_json( + cls, resource_name, cloudformation_json, region_name + ): + properties = cloudformation_json["Properties"] + event_backend = events_backends[region_name] + event_name = properties.get("Name") or resource_name + event_backend.delete_rule(name=event_name) + class EventBus(BaseModel): def __init__(self, region_name, name): diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud.py b/tests/test_cloudformation/test_cloudformation_stack_crud.py index 3d1b2ab8c..f6d359ec0 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud.py @@ -596,6 +596,30 @@ def test_create_stack_kinesis(): assert len(resources) == 1 +@mock_cloudformation_deprecated +def test_create_stack_events(): + conn = boto.connect_cloudformation() + dummy_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Description": "Stack Kinesis Test 1", + "Parameters": {}, + "Resources": { + "event": { + "Type": "AWS::Events::Rule", + "Properties": { + "State": "ENABLED", + "ScheduleExpression": "rate(5 minutes)", + }, + } + }, + } + conn.create_stack("test_stack_events_1", template_body=json.dumps(dummy_template)) + stack = conn.describe_stacks()[0] + + resources = stack.list_resources() + resources.should.have.length_of(1) + + def get_role_name(): with mock_iam_deprecated(): iam = boto.connect_iam() From a1f664d2bbbb4788d567afe2f2ae9f42ba924240 Mon Sep 17 00:00:00 2001 From: Guilherme Martins Crocetti Date: Sun, 22 Mar 2020 17:32:37 -0300 Subject: [PATCH 03/11] Change put_rule (and it's response) and fix tests_events/ --- moto/events/models.py | 8 ++++---- moto/events/responses.py | 4 ++-- tests/test_events/test_events.py | 19 ++++++++++++++++--- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/moto/events/models.py b/moto/events/models.py index 2f6f3b869..5c7662ba8 100644 --- a/moto/events/models.py +++ b/moto/events/models.py @@ -250,10 +250,10 @@ class EventsBackend(BaseBackend): return return_obj def put_rule(self, name, **kwargs): - rule = Rule(name, self.region_name, **kwargs) - self.rules[rule.name] = rule - self.rules_order.append(rule.name) - return rule.arn + new_rule = Rule(name, self.region_name, **kwargs) + self.rules[new_rule.name] = new_rule + self.rules_order.append(new_rule.name) + return new_rule def put_targets(self, name, targets): rule = self.rules.get(name) diff --git a/moto/events/responses.py b/moto/events/responses.py index c9931aabc..55a664b24 100644 --- a/moto/events/responses.py +++ b/moto/events/responses.py @@ -191,7 +191,7 @@ class EventsHandler(BaseResponse): "ValidationException", "Parameter ScheduleExpression is not valid." ) - rule_arn = self.events_backend.put_rule( + rule = self.events_backend.put_rule( name, ScheduleExpression=sched_exp, EventPattern=event_pattern, @@ -200,7 +200,7 @@ class EventsHandler(BaseResponse): RoleArn=role_arn, ) - return json.dumps({"RuleArn": rule_arn}), self.response_headers + return json.dumps({"RuleArn": rule.arn}), self.response_headers def put_targets(self): rule_name = self._get_param("Rule") diff --git a/tests/test_events/test_events.py b/tests/test_events/test_events.py index 80fadb449..27006ff1b 100644 --- a/tests/test_events/test_events.py +++ b/tests/test_events/test_events.py @@ -1,15 +1,16 @@ -from moto.events.models import EventsBackend -from moto.events import mock_events import json import random import unittest import boto3 +import sure # noqa from botocore.exceptions import ClientError -from moto.core.exceptions import JsonRESTError from nose.tools import assert_raises from moto.core import ACCOUNT_ID +from moto.core.exceptions import JsonRESTError +from moto.events import mock_events +from moto.events.models import EventsBackend RULES = [ {"Name": "test1", "ScheduleExpression": "rate(5 minutes)"}, @@ -75,6 +76,18 @@ def generate_environment(): return client +@mock_events +def test_put_rule(): + client = boto3.client("events", "us-west-2") + + client.list_rules()["Rules"].should.have.length_of(0) + + rule_data = get_random_rule() + client.put_rule(**rule_data) + + client.list_rules()["Rules"].should.have.length_of(1) + + @mock_events def test_list_rules(): client = generate_environment() From 98a17dfc464c3b7a73a73c62e7f2868b9018d7a1 Mon Sep 17 00:00:00 2001 From: Guilherme Martins Crocetti Date: Sun, 22 Mar 2020 18:03:42 -0300 Subject: [PATCH 04/11] Add test for boto3 integration --- .../test_cloudformation_stack_crud.py | 8 ++--- .../test_cloudformation_stack_integration.py | 29 +++++++++++++++++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud.py b/tests/test_cloudformation/test_cloudformation_stack_crud.py index f6d359ec0..d3a03d2bb 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud.py @@ -597,12 +597,10 @@ def test_create_stack_kinesis(): @mock_cloudformation_deprecated -def test_create_stack_events(): +def test_create_stack_events_rule(): conn = boto.connect_cloudformation() - dummy_template = { + events_template = { "AWSTemplateFormatVersion": "2010-09-09", - "Description": "Stack Kinesis Test 1", - "Parameters": {}, "Resources": { "event": { "Type": "AWS::Events::Rule", @@ -613,7 +611,7 @@ def test_create_stack_events(): } }, } - conn.create_stack("test_stack_events_1", template_body=json.dumps(dummy_template)) + conn.create_stack("test_stack_events_1", template_body=json.dumps(events_template)) stack = conn.describe_stacks()[0] resources = stack.list_resources() diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index a612156c4..2e84180b3 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -29,6 +29,7 @@ from moto import ( mock_ec2_deprecated, mock_elb, mock_elb_deprecated, + mock_events, mock_iam_deprecated, mock_kms, mock_lambda, @@ -2379,3 +2380,31 @@ def test_create_log_group_using_fntransform(): logs_conn = boto3.client("logs", region_name="us-west-2") log_group = logs_conn.describe_log_groups()["logGroups"][0] log_group["logGroupName"].should.equal("some-log-group") + + +@mock_cloudformation +@mock_events +def test_stack_events_rule_integration(): + events_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "event": { + "Type": "AWS::Events::Rule", + "Properties": { + "Name": "quick-fox", + "State": "ENABLED", + "ScheduleExpression": "rate(5 minutes)", + }, + } + }, + } + cf_conn = boto3.client("cloudformation", "us-west-2") + cf_conn.create_stack( + StackName="test_stack", TemplateBody=json.dumps(events_template), + ) + + result = boto3.client("events", "us-west-2").list_rules() + result["Rules"].should.have.length_of(1) + result["Rules"][0]["Name"].should.equal("quick-fox") + result["Rules"][0]["State"].should.equal("ENABLED") + result["Rules"][0]["ScheduleExpression"].should.equal("rate(5 minutes)") From 6180cf7a45d294d31d63c514aa22ed78b0cf77e0 Mon Sep 17 00:00:00 2001 From: Guilherme Martins Crocetti Date: Sun, 22 Mar 2020 18:08:12 -0300 Subject: [PATCH 05/11] Fix blank space --- moto/cloudformation/parsing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index 60eee63aa..cc4daf9ce 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -1,4 +1,3 @@ - from __future__ import unicode_literals import functools import json From c96efe531e3de2ba88028cf44e753cdcbe902b7a Mon Sep 17 00:00:00 2001 From: Guilherme Martins Crocetti Date: Mon, 23 Mar 2020 22:14:34 -0300 Subject: [PATCH 06/11] Add delete method for cloudformation's deletion --- moto/events/models.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/moto/events/models.py b/moto/events/models.py index 5c7662ba8..5f5909907 100644 --- a/moto/events/models.py +++ b/moto/events/models.py @@ -26,12 +26,6 @@ class Rule(BaseModel): self.role_arn = kwargs.get("RoleArn") self.targets = [] - def enable(self): - self.state = "ENABLED" - - def disable(self): - self.state = "DISABLED" - # This song and dance for targets is because we need order for Limits and NextTokens, but can't use OrderedDicts # with Python 2.6, so tracking it with an array it is. def _check_target_exists(self, target_id): @@ -40,6 +34,16 @@ class Rule(BaseModel): return i return None + def enable(self): + self.state = "ENABLED" + + def disable(self): + self.state = "DISABLED" + + def delete(self, region_name): + event_backend = events_backends[region_name] + event_backend.delete_rule(name=self.name) + def put_targets(self, targets): # Not testing for valid ARNs. for target in targets: From 788b8fb6e152a3ae52827009679e059bd3874c75 Mon Sep 17 00:00:00 2001 From: Guilherme Martins Crocetti Date: Mon, 23 Mar 2020 22:17:02 -0300 Subject: [PATCH 07/11] Add tests for Events::Rule integration with cf --- .../test_cloudformation_stack_crud.py | 22 ------ .../test_cloudformation_stack_integration.py | 68 +++++++++++++++++-- 2 files changed, 61 insertions(+), 29 deletions(-) diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud.py b/tests/test_cloudformation/test_cloudformation_stack_crud.py index d3a03d2bb..3d1b2ab8c 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud.py @@ -596,28 +596,6 @@ def test_create_stack_kinesis(): assert len(resources) == 1 -@mock_cloudformation_deprecated -def test_create_stack_events_rule(): - conn = boto.connect_cloudformation() - events_template = { - "AWSTemplateFormatVersion": "2010-09-09", - "Resources": { - "event": { - "Type": "AWS::Events::Rule", - "Properties": { - "State": "ENABLED", - "ScheduleExpression": "rate(5 minutes)", - }, - } - }, - } - conn.create_stack("test_stack_events_1", template_body=json.dumps(events_template)) - stack = conn.describe_stacks()[0] - - resources = stack.list_resources() - resources.should.have.length_of(1) - - def get_role_name(): with mock_iam_deprecated(): iam = boto.connect_iam() diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index 2e84180b3..e50179660 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -2384,11 +2384,11 @@ def test_create_log_group_using_fntransform(): @mock_cloudformation @mock_events -def test_stack_events_rule_integration(): +def test_stack_events_create_rule_integration(): events_template = { "AWSTemplateFormatVersion": "2010-09-09", "Resources": { - "event": { + "Event": { "Type": "AWS::Events::Rule", "Properties": { "Name": "quick-fox", @@ -2403,8 +2403,62 @@ def test_stack_events_rule_integration(): StackName="test_stack", TemplateBody=json.dumps(events_template), ) - result = boto3.client("events", "us-west-2").list_rules() - result["Rules"].should.have.length_of(1) - result["Rules"][0]["Name"].should.equal("quick-fox") - result["Rules"][0]["State"].should.equal("ENABLED") - result["Rules"][0]["ScheduleExpression"].should.equal("rate(5 minutes)") + rules = boto3.client("events", "us-west-2").list_rules() + rules["Rules"].should.have.length_of(1) + rules["Rules"][0]["Name"].should.equal("quick-fox") + rules["Rules"][0]["State"].should.equal("ENABLED") + rules["Rules"][0]["ScheduleExpression"].should.equal("rate(5 minutes)") + + +@mock_cloudformation +@mock_events +def test_stack_events_delete_rule_integration(): + events_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "Event": { + "Type": "AWS::Events::Rule", + "Properties": { + "Name": "quick-fox", + "State": "ENABLED", + "ScheduleExpression": "rate(5 minutes)", + }, + } + }, + } + cf_conn = boto3.client("cloudformation", "us-west-2") + cf_conn.create_stack( + StackName="test_stack", TemplateBody=json.dumps(events_template), + ) + + rules = boto3.client("events", "us-west-2").list_rules() + rules["Rules"].should.have.length_of(1) + + cf_conn.delete_stack(StackName="test_stack") + + rules = boto3.client("events", "us-west-2").list_rules() + rules["Rules"].should.have.length_of(0) + + +@mock_cloudformation +@mock_events +def test_stack_events_create_rule_without_name_integration(): + events_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "Event": { + "Type": "AWS::Events::Rule", + "Properties": { + "State": "ENABLED", + "ScheduleExpression": "rate(5 minutes)", + }, + } + }, + } + cf_conn = boto3.client("cloudformation", "us-west-2") + cf_conn.create_stack( + StackName="test_stack", TemplateBody=json.dumps(events_template), + ) + + rules = boto3.client("events", "us-west-2").list_rules() + rules["Rules"][0]["Name"].should.contain("test_stack-Event-") From 2e20ad14df2069094cd658bb7eeeabda1ec6b226 Mon Sep 17 00:00:00 2001 From: Asher Foa <1268088+asherf@users.noreply.github.com> Date: Wed, 25 Mar 2020 11:07:59 -0700 Subject: [PATCH 08/11] Fix some 'DeprecationWarning: invalid escape sequence' warnings and use str.format for string interpolation. Similar to https://github.com/spulec/moto/pull/2811 --- moto/elbv2/models.py | 23 ++++++++++++++--------- moto/events/models.py | 4 ++-- moto/packages/httpretty/http.py | 2 +- moto/secretsmanager/utils.py | 2 +- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/moto/elbv2/models.py b/moto/elbv2/models.py index fdce9a8c2..a6da0d01c 100644 --- a/moto/elbv2/models.py +++ b/moto/elbv2/models.py @@ -582,11 +582,13 @@ class ELBv2Backend(BaseBackend): report='Missing required parameter in Actions[%s].FixedResponseConfig: "StatusCode"' % i ) - if not re.match(r"^(2|4|5)\d\d$", status_code): + expression = r"^(2|4|5)\d\d$" + if not re.match(expression, status_code): raise InvalidStatusCodeActionTypeError( - "1 validation error detected: Value '%s' at 'actions.%s.member.fixedResponseConfig.statusCode' failed to satisfy constraint: \ -Member must satisfy regular expression pattern: ^(2|4|5)\d\d$" - % (status_code, index) + "1 validation error detected: Value '{}' at 'actions.{}.member.fixedResponseConfig.statusCode' failed to satisfy constraint: \ +Member must satisfy regular expression pattern: {}".format( + status_code, index, expression + ) ) content_type = action.data["fixed_response_config._content_type"] if content_type and content_type not in [ @@ -603,16 +605,19 @@ Member must satisfy regular expression pattern: ^(2|4|5)\d\d$" def create_target_group(self, name, **kwargs): if len(name) > 32: raise InvalidTargetGroupNameError( - "Target group name '%s' cannot be longer than '32' characters" % name + "Target group name '{}' cannot be longer than '32' characters".format( + name + ) ) - if not re.match("^[a-zA-Z0-9\-]+$", name): + if not re.match(r"^[a-zA-Z0-9\-]+$", name): raise InvalidTargetGroupNameError( - "Target group name '%s' can only contain characters that are alphanumeric characters or hyphens(-)" - % name + "Target group name '{}' can only contain characters that are alphanumeric characters or hyphens(-)".format( + name + ) ) # undocumented validation - if not re.match("(?!.*--)(?!^-)(?!.*-$)^[A-Za-z0-9-]+$", name): + if not re.match(r"(?!.*--)(?!^-)(?!.*-$)^[A-Za-z0-9-]+$", name): raise InvalidTargetGroupNameError( "1 validation error detected: Value '%s' at 'targetGroup.targetGroupArn.targetGroupName' failed to satisfy constraint: Member must satisfy regular expression pattern: (?!.*--)(?!^-)(?!.*-$)^[A-Za-z0-9-]+$" % name diff --git a/moto/events/models.py b/moto/events/models.py index 5f5909907..f68b63e38 100644 --- a/moto/events/models.py +++ b/moto/events/models.py @@ -305,12 +305,12 @@ class EventsBackend(BaseBackend): if principal is None or self.ACCOUNT_ID.match(principal) is None: raise JsonRESTError( - "InvalidParameterValue", "Principal must match ^(\d{1,12}|\*)$" + "InvalidParameterValue", r"Principal must match ^(\d{1,12}|\*)$" ) if statement_id is None or self.STATEMENT_ID.match(statement_id) is None: raise JsonRESTError( - "InvalidParameterValue", "StatementId must match ^[a-zA-Z0-9-_]{1,64}$" + "InvalidParameterValue", r"StatementId must match ^[a-zA-Z0-9-_]{1,64}$" ) event_bus._permissions[statement_id] = { diff --git a/moto/packages/httpretty/http.py b/moto/packages/httpretty/http.py index 20c00707e..1b4379f5b 100644 --- a/moto/packages/httpretty/http.py +++ b/moto/packages/httpretty/http.py @@ -134,7 +134,7 @@ def parse_requestline(s): ValueError: Not a Request-Line """ methods = "|".join(HttpBaseClass.METHODS) - m = re.match(r"(" + methods + ")\s+(.*)\s+HTTP/(1.[0|1])", s, re.I) + m = re.match(r"({})\s+(.*)\s+HTTP/(1.[0|1])".format(methods), s, re.I) if m: return m.group(1).upper(), m.group(2), m.group(3) else: diff --git a/moto/secretsmanager/utils.py b/moto/secretsmanager/utils.py index 73275ee05..6033db613 100644 --- a/moto/secretsmanager/utils.py +++ b/moto/secretsmanager/utils.py @@ -89,7 +89,7 @@ def _exclude_characters(password, exclude_characters): for c in exclude_characters: if c in string.punctuation: # Escape punctuation regex usage - c = "\{0}".format(c) + c = r"\{0}".format(c) password = re.sub(c, "", str(password)) return password From bb8d4180540602088e24f5cd6ce90eeb52f6e4fb Mon Sep 17 00:00:00 2001 From: Constantino Schillebeeckx Date: Fri, 27 Mar 2020 15:35:50 -0500 Subject: [PATCH 09/11] fix: stepfunction stop_execution Fixes #2846 Calling stop_execution on a stepfunction should set the status to `ABORTED` not `SUCCEEDED`. --- moto/stepfunctions/models.py | 2 +- tests/test_stepfunctions/test_stepfunctions.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/stepfunctions/models.py b/moto/stepfunctions/models.py index de530b863..e36598f23 100644 --- a/moto/stepfunctions/models.py +++ b/moto/stepfunctions/models.py @@ -46,7 +46,7 @@ class Execution: self.stop_date = None def stop(self): - self.status = "SUCCEEDED" + self.status = "ABORTED" self.stop_date = iso_8601_datetime_without_milliseconds(datetime.now()) diff --git a/tests/test_stepfunctions/test_stepfunctions.py b/tests/test_stepfunctions/test_stepfunctions.py index 3e0a8115d..eb2ace53d 100644 --- a/tests/test_stepfunctions/test_stepfunctions.py +++ b/tests/test_stepfunctions/test_stepfunctions.py @@ -516,7 +516,7 @@ def test_state_machine_describe_execution_after_stoppage(): description = client.describe_execution(executionArn=execution["executionArn"]) # description["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) - description["status"].should.equal("SUCCEEDED") + description["status"].should.equal("ABORTED") description["stopDate"].should.be.a(datetime) From 349b381390ae34c809e50cd04c3886053cff4776 Mon Sep 17 00:00:00 2001 From: Andrea Amorosi Date: Sat, 28 Mar 2020 17:59:42 +0000 Subject: [PATCH 10/11] Fixed dynamodb2 put_item ValidationException --- moto/dynamodb2/models.py | 8 ++++- moto/dynamodb2/responses.py | 3 ++ tests/test_dynamodb2/test_dynamodb.py | 45 +++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 54dccd56d..152e719c4 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -800,13 +800,19 @@ class Table(BaseModel): overwrite=False, ): if self.hash_key_attr not in item_attrs.keys(): - raise ValueError( + raise KeyError( "One or more parameter values were invalid: Missing the key " + self.hash_key_attr + " in the item" ) hash_value = DynamoType(item_attrs.get(self.hash_key_attr)) if self.has_range_key: + if self.range_key_attr not in item_attrs.keys(): + raise KeyError( + "One or more parameter values were invalid: Missing the key " + + self.range_key_attr + + " in the item" + ) range_value = DynamoType(item_attrs.get(self.range_key_attr)) else: range_value = None diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index c72ded2c3..78126f7f1 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -299,6 +299,9 @@ class DynamoHandler(BaseResponse): except ItemSizeTooLarge: er = "com.amazonaws.dynamodb.v20111205#ValidationException" return self.error(er, ItemSizeTooLarge.message) + except KeyError as ke: + er = "com.amazonaws.dynamodb.v20111205#ValidationException" + return self.error(er, ke.args[0]) except ValueError as ve: er = "com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException" return self.error(er, str(ve)) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 062208863..bec24c966 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1345,6 +1345,25 @@ def test_get_item_returns_consumed_capacity(): assert "TableName" in response["ConsumedCapacity"] +@mock_dynamodb2 +def test_put_empty_item(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + dynamodb.create_table( + AttributeDefinitions=[{"AttributeName": "structure_id", "AttributeType": "S"},], + TableName="test", + KeySchema=[{"AttributeName": "structure_id", "KeyType": "HASH"},], + ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, + ) + table = dynamodb.Table("test") + + with assert_raises(ClientError) as ex: + table.put_item(Item={}) + ex.exception.response["Error"]["Message"].should.equal( + "One or more parameter values were invalid: Missing the key structure_id in the item" + ) + ex.exception.response["Error"]["Code"].should.equal("ValidationException") + + @mock_dynamodb2 def test_put_item_nonexisting_hash_key(): dynamodb = boto3.resource("dynamodb", region_name="us-east-1") @@ -1361,6 +1380,32 @@ def test_put_item_nonexisting_hash_key(): ex.exception.response["Error"]["Message"].should.equal( "One or more parameter values were invalid: Missing the key structure_id in the item" ) + ex.exception.response["Error"]["Code"].should.equal("ValidationException") + + +@mock_dynamodb2 +def test_put_item_nonexisting_range_key(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + dynamodb.create_table( + AttributeDefinitions=[ + {"AttributeName": "structure_id", "AttributeType": "S"}, + {"AttributeName": "added_at", "AttributeType": "N"}, + ], + TableName="test", + KeySchema=[ + {"AttributeName": "structure_id", "KeyType": "HASH"}, + {"AttributeName": "added_at", "KeyType": "RANGE"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, + ) + table = dynamodb.Table("test") + + with assert_raises(ClientError) as ex: + table.put_item(Item={"structure_id": "abcdef"}) + ex.exception.response["Error"]["Message"].should.equal( + "One or more parameter values were invalid: Missing the key added_at in the item" + ) + ex.exception.response["Error"]["Code"].should.equal("ValidationException") def test_filter_expression(): From 0c191ac33b3f38a05bd41ed8ee1e082c926de3d4 Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Mon, 30 Mar 2020 17:23:33 -0700 Subject: [PATCH 11/11] Raise errors on tagging buckets with aws:* Cannot tag S3 buckets with reserved tag key space `aws:` --- moto/s3/exceptions.py | 9 +++++++++ moto/s3/responses.py | 6 ++++++ tests/test_s3/test_s3.py | 18 ++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/moto/s3/exceptions.py b/moto/s3/exceptions.py index e26f384d5..c38a4f467 100644 --- a/moto/s3/exceptions.py +++ b/moto/s3/exceptions.py @@ -368,3 +368,12 @@ class WrongPublicAccessBlockAccountIdError(S3ClientError): super(WrongPublicAccessBlockAccountIdError, self).__init__( "AccessDenied", "Access Denied" ) + + +class NoSystemTags(S3ClientError): + code = 400 + + def __init__(self): + super(NoSystemTags, self).__init__( + "InvalidTag", "System tags cannot be added/updated by requester" + ) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index b74be9a63..197cd9080 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -34,6 +34,7 @@ from .exceptions import ( InvalidNotificationARN, InvalidNotificationEvent, ObjectNotInActiveTierError, + NoSystemTags, ) from .models import ( s3_backend, @@ -1399,6 +1400,11 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): for tag in parsed_xml["Tagging"]["TagSet"]["Tag"]: tags.append(FakeTag(tag["Key"], tag["Value"])) + # Verify that "aws:" is not in the tags. If so, then this is a problem: + for tag in tags: + if tag.key.startswith("aws:"): + raise NoSystemTags() + tag_set = FakeTagSet(tags) tagging = FakeTagging(tag_set) return tagging diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 800daaef8..303ed523d 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -2413,6 +2413,24 @@ def test_boto3_put_bucket_tagging(): "Cannot provide multiple Tags with the same key" ) + # Cannot put tags that are "system" tags - i.e. tags that start with "aws:" + with assert_raises(ClientError) as ce: + s3.put_bucket_tagging( + Bucket=bucket_name, + Tagging={"TagSet": [{"Key": "aws:sometag", "Value": "nope"}]}, + ) + e = ce.exception + e.response["Error"]["Code"].should.equal("InvalidTag") + e.response["Error"]["Message"].should.equal( + "System tags cannot be added/updated by requester" + ) + + # This is OK though: + s3.put_bucket_tagging( + Bucket=bucket_name, + Tagging={"TagSet": [{"Key": "something:aws:stuff", "Value": "this is fine"}]}, + ) + @mock_s3 def test_boto3_get_bucket_tagging():