Merge pull request #37 from spulec/master

Merge upstream

moto/cloudformation/parsing.py

@@ -18,6 +18,7 @@ from moto.ec2 import models as ec2_models
 from moto.ecs import models as ecs_models
 from moto.elb import models as elb_models
 from moto.elbv2 import models as elbv2_models
+from moto.events import models as events_models
 from moto.iam import models as iam_models
 from moto.kinesis import models as kinesis_models
 from moto.kms import models as kms_models
@@ -94,6 +95,7 @@ MODEL_MAP = {
     "AWS::SNS::Topic": sns_models.Topic,
     "AWS::S3::Bucket": s3_models.FakeBucket,
     "AWS::SQS::Queue": sqs_models.Queue,
+    "AWS::Events::Rule": events_models.Rule,
 }
 
 # http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-name.html

moto/dynamodb2/models.py

@@ -800,13 +800,19 @@ class Table(BaseModel):
         overwrite=False,
     ):
         if self.hash_key_attr not in item_attrs.keys():
-            raise ValueError(
+            raise KeyError(
                 "One or more parameter values were invalid: Missing the key "
                 + self.hash_key_attr
                 + " in the item"
             )
         hash_value = DynamoType(item_attrs.get(self.hash_key_attr))
         if self.has_range_key:
+            if self.range_key_attr not in item_attrs.keys():
+                raise KeyError(
+                    "One or more parameter values were invalid: Missing the key "
+                    + self.range_key_attr
+                    + " in the item"
+                )
             range_value = DynamoType(item_attrs.get(self.range_key_attr))
         else:
             range_value = None

moto/dynamodb2/responses.py

@@ -299,6 +299,9 @@ class DynamoHandler(BaseResponse):
         except ItemSizeTooLarge:
             er = "com.amazonaws.dynamodb.v20111205#ValidationException"
             return self.error(er, ItemSizeTooLarge.message)
+        except KeyError as ke:
+            er = "com.amazonaws.dynamodb.v20111205#ValidationException"
+            return self.error(er, ke.args[0])
         except ValueError as ve:
             er = "com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException"
             return self.error(er, str(ve))

moto/elbv2/models.py

@@ -582,11 +582,13 @@ class ELBv2Backend(BaseBackend):
                 report='Missing required parameter in Actions[%s].FixedResponseConfig: "StatusCode"'
                 % i
             )
-        if not re.match(r"^(2|4|5)\d\d$", status_code):
+        expression = r"^(2|4|5)\d\d$"
+        if not re.match(expression, status_code):
             raise InvalidStatusCodeActionTypeError(
-                "1 validation error detected: Value '%s' at 'actions.%s.member.fixedResponseConfig.statusCode' failed to satisfy constraint: \
+                "1 validation error detected: Value '{}' at 'actions.{}.member.fixedResponseConfig.statusCode' failed to satisfy constraint: \
-Member must satisfy regular expression pattern: ^(2|4|5)\d\d$"
+Member must satisfy regular expression pattern: {}".format(
-                % (status_code, index)
+                    status_code, index, expression
+                )
             )
         content_type = action.data["fixed_response_config._content_type"]
         if content_type and content_type not in [
@@ -603,16 +605,19 @@ Member must satisfy regular expression pattern: ^(2|4|5)\d\d$"
     def create_target_group(self, name, **kwargs):
         if len(name) > 32:
             raise InvalidTargetGroupNameError(
-                "Target group name '%s' cannot be longer than '32' characters" % name
+                "Target group name '{}' cannot be longer than '32' characters".format(
+                    name
                 )
-        if not re.match("^[a-zA-Z0-9\-]+$", name):
+            )
+        if not re.match(r"^[a-zA-Z0-9\-]+$", name):
             raise InvalidTargetGroupNameError(
-                "Target group name '%s' can only contain characters that are alphanumeric characters or hyphens(-)"
+                "Target group name '{}' can only contain characters that are alphanumeric characters or hyphens(-)".format(
-                % name
+                    name
+                )
             )
 
         # undocumented validation
-        if not re.match("(?!.*--)(?!^-)(?!.*-$)^[A-Za-z0-9-]+$", name):
+        if not re.match(r"(?!.*--)(?!^-)(?!.*-$)^[A-Za-z0-9-]+$", name):
             raise InvalidTargetGroupNameError(
                 "1 validation error detected: Value '%s' at 'targetGroup.targetGroupArn.targetGroupName' failed to satisfy constraint: Member must satisfy regular expression pattern: (?!.*--)(?!^-)(?!.*-$)^[A-Za-z0-9-]+$"
                 % name

moto/events/models.py

@@ -26,12 +26,6 @@ class Rule(BaseModel):
         self.role_arn = kwargs.get("RoleArn")
         self.targets = []
 
-    def enable(self):
-        self.state = "ENABLED"
-
-    def disable(self):
-        self.state = "DISABLED"
-
     # This song and dance for targets is because we need order for Limits and NextTokens, but can't use OrderedDicts
     # with Python 2.6, so tracking it with an array it is.
     def _check_target_exists(self, target_id):
@@ -40,6 +34,16 @@ class Rule(BaseModel):
                 return i
         return None
 
+    def enable(self):
+        self.state = "ENABLED"
+
+    def disable(self):
+        self.state = "DISABLED"
+
+    def delete(self, region_name):
+        event_backend = events_backends[region_name]
+        event_backend.delete_rule(name=self.name)
+
     def put_targets(self, targets):
         # Not testing for valid ARNs.
         for target in targets:
@@ -55,6 +59,24 @@ class Rule(BaseModel):
             if index is not None:
                 self.targets.pop(index)
 
+    @classmethod
+    def create_from_cloudformation_json(
+        cls, resource_name, cloudformation_json, region_name
+    ):
+        properties = cloudformation_json["Properties"]
+        event_backend = events_backends[region_name]
+        event_name = properties.get("Name") or resource_name
+        return event_backend.put_rule(name=event_name, **properties)
+
+    @classmethod
+    def delete_from_cloudformation_json(
+        cls, resource_name, cloudformation_json, region_name
+    ):
+        properties = cloudformation_json["Properties"]
+        event_backend = events_backends[region_name]
+        event_name = properties.get("Name") or resource_name
+        event_backend.delete_rule(name=event_name)
+
 
 class EventBus(BaseModel):
     def __init__(self, region_name, name):
@@ -232,10 +254,10 @@ class EventsBackend(BaseBackend):
         return return_obj
 
     def put_rule(self, name, **kwargs):
-        rule = Rule(name, self.region_name, **kwargs)
+        new_rule = Rule(name, self.region_name, **kwargs)
-        self.rules[rule.name] = rule
+        self.rules[new_rule.name] = new_rule
-        self.rules_order.append(rule.name)
+        self.rules_order.append(new_rule.name)
-        return rule.arn
+        return new_rule
 
     def put_targets(self, name, targets):
         rule = self.rules.get(name)
@@ -283,12 +305,12 @@ class EventsBackend(BaseBackend):
 
         if principal is None or self.ACCOUNT_ID.match(principal) is None:
             raise JsonRESTError(
-                "InvalidParameterValue", "Principal must match ^(\d{1,12}|\*)$"
+                "InvalidParameterValue", r"Principal must match ^(\d{1,12}|\*)$"
             )
 
         if statement_id is None or self.STATEMENT_ID.match(statement_id) is None:
             raise JsonRESTError(
-                "InvalidParameterValue", "StatementId must match ^[a-zA-Z0-9-_]{1,64}$"
+                "InvalidParameterValue", r"StatementId must match ^[a-zA-Z0-9-_]{1,64}$"
             )
 
         event_bus._permissions[statement_id] = {

moto/events/responses.py

@@ -191,7 +191,7 @@ class EventsHandler(BaseResponse):
                     "ValidationException", "Parameter ScheduleExpression is not valid."
                 )
 
-        rule_arn = self.events_backend.put_rule(
+        rule = self.events_backend.put_rule(
             name,
             ScheduleExpression=sched_exp,
             EventPattern=event_pattern,
@@ -200,7 +200,7 @@ class EventsHandler(BaseResponse):
             RoleArn=role_arn,
         )
 
-        return json.dumps({"RuleArn": rule_arn}), self.response_headers
+        return json.dumps({"RuleArn": rule.arn}), self.response_headers
 
     def put_targets(self):
         rule_name = self._get_param("Rule")

moto/packages/httpretty/http.py

@@ -134,7 +134,7 @@ def parse_requestline(s):
     ValueError: Not a Request-Line
     """
     methods = "|".join(HttpBaseClass.METHODS)
-    m = re.match(r"(" + methods + ")\s+(.*)\s+HTTP/(1.[0|1])", s, re.I)
+    m = re.match(r"({})\s+(.*)\s+HTTP/(1.[0|1])".format(methods), s, re.I)
     if m:
         return m.group(1).upper(), m.group(2), m.group(3)
     else:

moto/s3/exceptions.py

@@ -368,3 +368,12 @@ class WrongPublicAccessBlockAccountIdError(S3ClientError):
         super(WrongPublicAccessBlockAccountIdError, self).__init__(
             "AccessDenied", "Access Denied"
         )
+
+
+class NoSystemTags(S3ClientError):
+    code = 400
+
+    def __init__(self):
+        super(NoSystemTags, self).__init__(
+            "InvalidTag", "System tags cannot be added/updated by requester"
+        )

moto/s3/responses.py

@@ -34,6 +34,7 @@ from .exceptions import (
     InvalidNotificationARN,
     InvalidNotificationEvent,
     ObjectNotInActiveTierError,
+    NoSystemTags,
 )
 from .models import (
     s3_backend,
@@ -1399,6 +1400,11 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
                 for tag in parsed_xml["Tagging"]["TagSet"]["Tag"]:
                     tags.append(FakeTag(tag["Key"], tag["Value"]))
 
+        # Verify that "aws:" is not in the tags. If so, then this is a problem:
+        for tag in tags:
+            if tag.key.startswith("aws:"):
+                raise NoSystemTags()
+
         tag_set = FakeTagSet(tags)
         tagging = FakeTagging(tag_set)
         return tagging

moto/secretsmanager/utils.py

@@ -89,7 +89,7 @@ def _exclude_characters(password, exclude_characters):
     for c in exclude_characters:
         if c in string.punctuation:
             # Escape punctuation regex usage
-            c = "\{0}".format(c)
+            c = r"\{0}".format(c)
         password = re.sub(c, "", str(password))
     return password
 

moto/sns/models.py

@@ -146,7 +146,9 @@ class Subscription(BaseModel):
             queue_name = self.endpoint.split(":")[-1]
             region = self.endpoint.split(":")[3]
             if self.attributes.get("RawMessageDelivery") != "true":
-                enveloped_message = json.dumps(
+                sqs_backends[region].send_message(
+                    queue_name,
+                    json.dumps(
                         self.get_post_data(
                             message,
                             message_id,
@@ -156,10 +158,26 @@ class Subscription(BaseModel):
                         sort_keys=True,
                         indent=2,
                         separators=(",", ": "),
+                    ),
                 )
             else:
-                enveloped_message = message
+                raw_message_attributes = {}
-            sqs_backends[region].send_message(queue_name, enveloped_message)
+                for key, value in message_attributes.items():
+                    type = "string_value"
+                    type_value = value["Value"]
+                    if value["Type"].startswith("Binary"):
+                        type = "binary_value"
+                    elif value["Type"].startswith("Number"):
+                        type_value = "{0:g}".format(value["Value"])
+
+                    raw_message_attributes[key] = {
+                        "data_type": value["Type"],
+                        type: type_value,
+                    }
+
+                sqs_backends[region].send_message(
+                    queue_name, message, message_attributes=raw_message_attributes
+                )
         elif self.protocol in ["http", "https"]:
             post_data = self.get_post_data(message, message_id, subject)
             requests.post(

moto/stepfunctions/models.py

@@ -46,7 +46,7 @@ class Execution:
         self.stop_date = None
 
     def stop(self):
-        self.status = "SUCCEEDED"
+        self.status = "ABORTED"
         self.stop_date = iso_8601_datetime_without_milliseconds(datetime.now())
 
 

tests/test_cloudformation/test_cloudformation_stack_integration.py

@@ -29,6 +29,7 @@ from moto import (
     mock_ec2_deprecated,
     mock_elb,
     mock_elb_deprecated,
+    mock_events,
     mock_iam_deprecated,
     mock_kms,
     mock_lambda,
@@ -2379,3 +2380,85 @@ def test_create_log_group_using_fntransform():
     logs_conn = boto3.client("logs", region_name="us-west-2")
     log_group = logs_conn.describe_log_groups()["logGroups"][0]
     log_group["logGroupName"].should.equal("some-log-group")
+
+
+@mock_cloudformation
+@mock_events
+def test_stack_events_create_rule_integration():
+    events_template = {
+        "AWSTemplateFormatVersion": "2010-09-09",
+        "Resources": {
+            "Event": {
+                "Type": "AWS::Events::Rule",
+                "Properties": {
+                    "Name": "quick-fox",
+                    "State": "ENABLED",
+                    "ScheduleExpression": "rate(5 minutes)",
+                },
+            }
+        },
+    }
+    cf_conn = boto3.client("cloudformation", "us-west-2")
+    cf_conn.create_stack(
+        StackName="test_stack", TemplateBody=json.dumps(events_template),
+    )
+
+    rules = boto3.client("events", "us-west-2").list_rules()
+    rules["Rules"].should.have.length_of(1)
+    rules["Rules"][0]["Name"].should.equal("quick-fox")
+    rules["Rules"][0]["State"].should.equal("ENABLED")
+    rules["Rules"][0]["ScheduleExpression"].should.equal("rate(5 minutes)")
+
+
+@mock_cloudformation
+@mock_events
+def test_stack_events_delete_rule_integration():
+    events_template = {
+        "AWSTemplateFormatVersion": "2010-09-09",
+        "Resources": {
+            "Event": {
+                "Type": "AWS::Events::Rule",
+                "Properties": {
+                    "Name": "quick-fox",
+                    "State": "ENABLED",
+                    "ScheduleExpression": "rate(5 minutes)",
+                },
+            }
+        },
+    }
+    cf_conn = boto3.client("cloudformation", "us-west-2")
+    cf_conn.create_stack(
+        StackName="test_stack", TemplateBody=json.dumps(events_template),
+    )
+
+    rules = boto3.client("events", "us-west-2").list_rules()
+    rules["Rules"].should.have.length_of(1)
+
+    cf_conn.delete_stack(StackName="test_stack")
+
+    rules = boto3.client("events", "us-west-2").list_rules()
+    rules["Rules"].should.have.length_of(0)
+
+
+@mock_cloudformation
+@mock_events
+def test_stack_events_create_rule_without_name_integration():
+    events_template = {
+        "AWSTemplateFormatVersion": "2010-09-09",
+        "Resources": {
+            "Event": {
+                "Type": "AWS::Events::Rule",
+                "Properties": {
+                    "State": "ENABLED",
+                    "ScheduleExpression": "rate(5 minutes)",
+                },
+            }
+        },
+    }
+    cf_conn = boto3.client("cloudformation", "us-west-2")
+    cf_conn.create_stack(
+        StackName="test_stack", TemplateBody=json.dumps(events_template),
+    )
+
+    rules = boto3.client("events", "us-west-2").list_rules()
+    rules["Rules"][0]["Name"].should.contain("test_stack-Event-")

tests/test_dynamodb2/test_dynamodb.py

@@ -1345,6 +1345,25 @@ def test_get_item_returns_consumed_capacity():
     assert "TableName" in response["ConsumedCapacity"]
 
 
+@mock_dynamodb2
+def test_put_empty_item():
+    dynamodb = boto3.resource("dynamodb", region_name="us-east-1")
+    dynamodb.create_table(
+        AttributeDefinitions=[{"AttributeName": "structure_id", "AttributeType": "S"},],
+        TableName="test",
+        KeySchema=[{"AttributeName": "structure_id", "KeyType": "HASH"},],
+        ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123},
+    )
+    table = dynamodb.Table("test")
+
+    with assert_raises(ClientError) as ex:
+        table.put_item(Item={})
+    ex.exception.response["Error"]["Message"].should.equal(
+        "One or more parameter values were invalid: Missing the key structure_id in the item"
+    )
+    ex.exception.response["Error"]["Code"].should.equal("ValidationException")
+
+
 @mock_dynamodb2
 def test_put_item_nonexisting_hash_key():
     dynamodb = boto3.resource("dynamodb", region_name="us-east-1")
@@ -1361,6 +1380,32 @@ def test_put_item_nonexisting_hash_key():
     ex.exception.response["Error"]["Message"].should.equal(
         "One or more parameter values were invalid: Missing the key structure_id in the item"
     )
+    ex.exception.response["Error"]["Code"].should.equal("ValidationException")
+
+
+@mock_dynamodb2
+def test_put_item_nonexisting_range_key():
+    dynamodb = boto3.resource("dynamodb", region_name="us-east-1")
+    dynamodb.create_table(
+        AttributeDefinitions=[
+            {"AttributeName": "structure_id", "AttributeType": "S"},
+            {"AttributeName": "added_at", "AttributeType": "N"},
+        ],
+        TableName="test",
+        KeySchema=[
+            {"AttributeName": "structure_id", "KeyType": "HASH"},
+            {"AttributeName": "added_at", "KeyType": "RANGE"},
+        ],
+        ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123},
+    )
+    table = dynamodb.Table("test")
+
+    with assert_raises(ClientError) as ex:
+        table.put_item(Item={"structure_id": "abcdef"})
+    ex.exception.response["Error"]["Message"].should.equal(
+        "One or more parameter values were invalid: Missing the key added_at in the item"
+    )
+    ex.exception.response["Error"]["Code"].should.equal("ValidationException")
 
 
 def test_filter_expression():

tests/test_events/test_events.py

@@ -1,15 +1,16 @@
-from moto.events.models import EventsBackend
-from moto.events import mock_events
 import json
 import random
 import unittest
 
 import boto3
+import sure  # noqa
 from botocore.exceptions import ClientError
-from moto.core.exceptions import JsonRESTError
 from nose.tools import assert_raises
 
 from moto.core import ACCOUNT_ID
+from moto.core.exceptions import JsonRESTError
+from moto.events import mock_events
+from moto.events.models import EventsBackend
 
 RULES = [
     {"Name": "test1", "ScheduleExpression": "rate(5 minutes)"},
@@ -75,6 +76,18 @@ def generate_environment():
     return client
 
 
+@mock_events
+def test_put_rule():
+    client = boto3.client("events", "us-west-2")
+
+    client.list_rules()["Rules"].should.have.length_of(0)
+
+    rule_data = get_random_rule()
+    client.put_rule(**rule_data)
+
+    client.list_rules()["Rules"].should.have.length_of(1)
+
+
 @mock_events
 def test_list_rules():
     client = generate_environment()

tests/test_s3/test_s3.py

@@ -2413,6 +2413,24 @@ def test_boto3_put_bucket_tagging():
         "Cannot provide multiple Tags with the same key"
     )
 
+    # Cannot put tags that are "system" tags - i.e. tags that start with "aws:"
+    with assert_raises(ClientError) as ce:
+        s3.put_bucket_tagging(
+            Bucket=bucket_name,
+            Tagging={"TagSet": [{"Key": "aws:sometag", "Value": "nope"}]},
+        )
+    e = ce.exception
+    e.response["Error"]["Code"].should.equal("InvalidTag")
+    e.response["Error"]["Message"].should.equal(
+        "System tags cannot be added/updated by requester"
+    )
+
+    # This is OK though:
+    s3.put_bucket_tagging(
+        Bucket=bucket_name,
+        Tagging={"TagSet": [{"Key": "something:aws:stuff", "Value": "this is fine"}]},
+    )
+
 
 @mock_s3
 def test_boto3_get_bucket_tagging():

tests/test_sns/test_publishing_boto3.py

@@ -148,34 +148,42 @@ def test_publish_to_sqs_msg_attr_byte_value():
     conn.create_topic(Name="some-topic")
     response = conn.list_topics()
     topic_arn = response["Topics"][0]["TopicArn"]
-
+    sqs = boto3.resource("sqs", region_name="us-east-1")
-    sqs_conn = boto3.resource("sqs", region_name="us-east-1")
+    queue = sqs.create_queue(QueueName="test-queue")
-    queue = sqs_conn.create_queue(QueueName="test-queue")
+    conn.subscribe(
-
+        TopicArn=topic_arn, Protocol="sqs", Endpoint=queue.attributes["QueueArn"],
+    )
+    queue_raw = sqs.create_queue(QueueName="test-queue-raw")
     conn.subscribe(
         TopicArn=topic_arn,
         Protocol="sqs",
-        Endpoint="arn:aws:sqs:us-east-1:{}:test-queue".format(ACCOUNT_ID),
+        Endpoint=queue_raw.attributes["QueueArn"],
+        Attributes={"RawMessageDelivery": "true"},
     )
-    message = "my message"
+
     conn.publish(
         TopicArn=topic_arn,
-        Message=message,
+        Message="my message",
         MessageAttributes={
             "store": {"DataType": "Binary", "BinaryValue": b"\x02\x03\x04"}
         },
     )
-    messages = queue.receive_messages(MaxNumberOfMessages=5)
+
-    message_attributes = [json.loads(m.body)["MessageAttributes"] for m in messages]
+    message = json.loads(queue.receive_messages()[0].body)
-    message_attributes.should.equal(
+    message["Message"].should.equal("my message")
-        [
+    message["MessageAttributes"].should.equal(
         {
             "store": {
                 "Type": "Binary",
                 "Value": base64.b64encode(b"\x02\x03\x04").decode(),
             }
         }
-        ]
+    )
+
+    message = queue_raw.receive_messages()[0]
+    message.body.should.equal("my message")
+    message.message_attributes.should.equal(
+        {"store": {"DataType": "Binary", "BinaryValue": b"\x02\x03\x04"}}
     )
 
 
@@ -187,6 +195,12 @@ def test_publish_to_sqs_msg_attr_number_type():
     sqs = boto3.resource("sqs", region_name="us-east-1")
     queue = sqs.create_queue(QueueName="test-queue")
     topic.subscribe(Protocol="sqs", Endpoint=queue.attributes["QueueArn"])
+    queue_raw = sqs.create_queue(QueueName="test-queue-raw")
+    topic.subscribe(
+        Protocol="sqs",
+        Endpoint=queue_raw.attributes["QueueArn"],
+        Attributes={"RawMessageDelivery": "true"},
+    )
 
     topic.publish(
         Message="test message",
@@ -199,6 +213,12 @@ def test_publish_to_sqs_msg_attr_number_type():
         {"retries": {"Type": "Number", "Value": 0}}
     )
 
+    message = queue_raw.receive_messages()[0]
+    message.body.should.equal("test message")
+    message.message_attributes.should.equal(
+        {"retries": {"DataType": "Number", "StringValue": "0"}}
+    )
+
 
 @mock_sns
 def test_publish_sms():

tests/test_stepfunctions/test_stepfunctions.py

@@ -516,7 +516,7 @@ def test_state_machine_describe_execution_after_stoppage():
     description = client.describe_execution(executionArn=execution["executionArn"])
     #
     description["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
-    description["status"].should.equal("SUCCEEDED")
+    description["status"].should.equal("ABORTED")
     description["stopDate"].should.be.a(datetime)