From 7b5613b3312f35ba5d93f17ccfe2d6891b162615 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Mon, 9 Mar 2020 12:47:44 +0000 Subject: [PATCH 01/32] #2774 - Re-add test, and update requirements to working botocore --- requirements-dev.txt | 2 +- tests/test_core/test_auth.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 2aaca300b..4e5f4e8a0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,7 +10,7 @@ freezegun flask boto>=2.45.0 boto3>=1.4.4 -botocore>=1.12.13 +botocore>=1.15.13 six>=1.9 parameterized>=0.7.0 prompt-toolkit==1.0.14 diff --git a/tests/test_core/test_auth.py b/tests/test_core/test_auth.py index 29273cea7..b391d82c8 100644 --- a/tests/test_core/test_auth.py +++ b/tests/test_core/test_auth.py @@ -298,6 +298,40 @@ def test_access_denied_with_not_allowing_policy(): ) +@set_initial_no_auth_action_count(3) +@mock_ec2 +def test_access_denied_for_run_instances(): + # https://github.com/spulec/moto/issues/2774 + # The run-instances method was broken between botocore versions 1.15.8 and 1.15.12 + # This was due to the inclusion of '"idempotencyToken":true' in the response, somehow altering the signature and breaking the authentication + # Keeping this test in place in case botocore decides to break again + user_name = "test-user" + inline_policy_document = { + "Version": "2012-10-17", + "Statement": [ + {"Effect": "Allow", "Action": ["ec2:Describe*"], "Resource": "*"} + ], + } + access_key = create_user_with_access_key_and_inline_policy( + user_name, inline_policy_document + ) + client = boto3.client( + "ec2", + region_name="us-east-1", + aws_access_key_id=access_key["AccessKeyId"], + aws_secret_access_key=access_key["SecretAccessKey"], + ) + with assert_raises(ClientError) as ex: + client.run_instances(MaxCount=1, MinCount=1) + ex.exception.response["Error"]["Code"].should.equal("AccessDenied") + ex.exception.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(403) + ex.exception.response["Error"]["Message"].should.equal( + "User: arn:aws:iam::{account_id}:user/{user_name} is not authorized to perform: {operation}".format( + account_id=ACCOUNT_ID, user_name=user_name, operation="ec2:RunInstances", + ) + ) + + @set_initial_no_auth_action_count(3) @mock_ec2 def test_access_denied_with_denying_policy(): From dc98fca8532ad6a031948e263841f7cd1944dab1 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Mon, 9 Mar 2020 15:14:51 +0000 Subject: [PATCH 02/32] #718 - Allow filtering by multiple tags --- moto/ec2/utils.py | 7 +++++++ tests/test_ec2/test_tags.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/moto/ec2/utils.py b/moto/ec2/utils.py index 2301248c1..18a038b10 100644 --- a/moto/ec2/utils.py +++ b/moto/ec2/utils.py @@ -252,6 +252,7 @@ def dhcp_configuration_from_querystring(querystring, option="DhcpConfiguration") def filters_from_querystring(querystring_dict): response_values = {} + last_tag_key = None for key, value in querystring_dict.items(): match = re.search(r"Filter.(\d).Name", key) if match: @@ -262,6 +263,10 @@ def filters_from_querystring(querystring_dict): for filter_key, filter_value in querystring_dict.items() if filter_key.startswith(value_prefix) ] + if value[0] == "tag-key": + last_tag_key = "tag:" + filter_values[0] + elif last_tag_key and value[0] == "tag-value": + response_values[last_tag_key] = filter_values response_values[value[0]] = filter_values return response_values @@ -329,6 +334,8 @@ def tag_filter_matches(obj, filter_name, filter_values): tag_values = get_obj_tag_names(obj) elif filter_name == "tag-value": tag_values = get_obj_tag_values(obj) + elif filter_name.startswith("tag:"): + tag_values = get_obj_tag_values(obj) else: tag_values = [get_obj_tag(obj, filter_name) or ""] diff --git a/tests/test_ec2/test_tags.py b/tests/test_ec2/test_tags.py index 29d2cb1e3..789f7b976 100644 --- a/tests/test_ec2/test_tags.py +++ b/tests/test_ec2/test_tags.py @@ -468,3 +468,36 @@ def test_delete_tag_empty_resource(): ex.exception.response["Error"]["Message"].should.equal( "The request must contain the parameter resourceIdSet" ) + + +@mock_ec2 +def test_retrieve_resource_with_multiple_tags(): + ec2 = boto3.resource("ec2") + blue, green = ec2.create_instances(ImageId="ANY_ID", MinCount=2, MaxCount=2) + ec2.create_tags( + Resources=[blue.instance_id], + Tags=[ + {"Key": "environment", "Value": "blue"}, + {"Key": "application", "Value": "api"}, + ], + ) + ec2.create_tags( + Resources=[green.instance_id], + Tags=[ + {"Key": "environment", "Value": "green"}, + {"Key": "application", "Value": "api"}, + ], + ) + green_instances = list(ec2.instances.filter(Filters=(get_filter("green")))) + green_instances.should.equal([green]) + blue_instances = list(ec2.instances.filter(Filters=(get_filter("blue")))) + blue_instances.should.equal([blue]) + + +def get_filter(color): + return [ + {"Name": "tag-key", "Values": ["application"]}, + {"Name": "tag-value", "Values": ["api"]}, + {"Name": "tag-key", "Values": ["environment"]}, + {"Name": "tag-value", "Values": [color]}, + ] From 0e489a8a287c36af36a9728fe62699b84575ebd4 Mon Sep 17 00:00:00 2001 From: Steven Davidovitz Date: Tue, 10 Mar 2020 01:05:46 -0700 Subject: [PATCH 03/32] support mock versions < 3.0.5 --- requirements-dev.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4e5f4e8a0..2b43bcf9d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ -r requirements.txt -mock==3.0.5 # Last version compatible with Python 2.7 +mock<=3.0.5 # Last version compatible with Python 2.7 nose black; python_version >= '3.6' regex==2019.11.1; python_version >= '3.6' # Needed for black diff --git a/setup.py b/setup.py index b806f7bae..9bb7cf522 100755 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ install_requires = [ "pytz", "python-dateutil<3.0.0,>=2.1", "python-jose<4.0.0", - "mock==3.0.5", + "mock<=3.0.5", "docker>=2.5.1", "jsondiff>=1.1.2", "aws-xray-sdk!=0.96,>=0.93", From 9eeb375911420416d5b6ad78ca3062b231a85ceb Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 10 Mar 2020 09:26:44 +0000 Subject: [PATCH 04/32] Add region to test case --- tests/test_ec2/test_tags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ec2/test_tags.py b/tests/test_ec2/test_tags.py index 789f7b976..92ed18dd4 100644 --- a/tests/test_ec2/test_tags.py +++ b/tests/test_ec2/test_tags.py @@ -472,7 +472,7 @@ def test_delete_tag_empty_resource(): @mock_ec2 def test_retrieve_resource_with_multiple_tags(): - ec2 = boto3.resource("ec2") + ec2 = boto3.resource("ec2", region_name="us-west-1") blue, green = ec2.create_instances(ImageId="ANY_ID", MinCount=2, MaxCount=2) ec2.create_tags( Resources=[blue.instance_id], From 994ab9aadf23e4d4f4f64bb582a73848a9113416 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 10 Mar 2020 12:42:18 +0000 Subject: [PATCH 05/32] #718 - EC2 - Guarantee order when filtering tags from querystring --- moto/ec2/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/ec2/utils.py b/moto/ec2/utils.py index 18a038b10..74fe3d27b 100644 --- a/moto/ec2/utils.py +++ b/moto/ec2/utils.py @@ -253,7 +253,7 @@ def dhcp_configuration_from_querystring(querystring, option="DhcpConfiguration") def filters_from_querystring(querystring_dict): response_values = {} last_tag_key = None - for key, value in querystring_dict.items(): + for key, value in sorted(querystring_dict.items()): match = re.search(r"Filter.(\d).Name", key) if match: filter_index = match.groups()[0] From f17d5f8e4d14d36fb662eb360f10158a78df8a0c Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 10 Mar 2020 12:56:33 +0000 Subject: [PATCH 06/32] #657 - S3 - Verify content type is set/returned as appropriate --- tests/test_s3/test_s3.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 48655ee17..48939be26 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -11,6 +11,7 @@ from six.moves.urllib.error import HTTPError from functools import wraps from gzip import GzipFile from io import BytesIO +import mimetypes import zlib import pickle @@ -2024,6 +2025,24 @@ def test_boto3_get_object(): e.exception.response["Error"]["Code"].should.equal("NoSuchKey") +@mock_s3 +def test_boto3_s3_content_type(): + s3 = boto3.resource("s3", region_name=DEFAULT_REGION_NAME) + my_bucket = s3.Bucket("my-cool-bucket") + my_bucket.create() + local_path = "test_s3.py" + s3_path = local_path + s3 = boto3.resource("s3", verify=False) + + with open(local_path, "rb") as _file: + content_type = mimetypes.guess_type(local_path) + s3.Object(my_bucket.name, s3_path).put( + ContentType=content_type[0], Body=_file, ACL="public-read" + ) + + s3.Object(my_bucket.name, s3_path).content_type.should.equal(content_type[0]) + + @mock_s3 def test_boto3_get_missing_object_with_part_number(): s3 = boto3.resource("s3", region_name=DEFAULT_REGION_NAME) From 6ba00d9ad19854fa78532de907a114ed0d49ee42 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 10 Mar 2020 13:25:40 +0000 Subject: [PATCH 07/32] #1054 - DynamoDB - Improve error handling for put_item without keys --- moto/dynamodb2/models.py | 7 ++++++- moto/dynamodb2/responses.py | 6 ++---- tests/test_dynamodb2/test_dynamodb.py | 18 ++++++++++++++++++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 8e5a61755..1527821ed 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -792,6 +792,12 @@ class Table(BaseModel): expression_attribute_values=None, overwrite=False, ): + if self.hash_key_attr not in item_attrs.keys(): + raise ValueError( + "One or more parameter values were invalid: Missing the key " + + self.hash_key_attr + + " in the item" + ) hash_value = DynamoType(item_attrs.get(self.hash_key_attr)) if self.has_range_key: range_value = DynamoType(item_attrs.get(self.range_key_attr)) @@ -808,7 +814,6 @@ class Table(BaseModel): else: lookup_range_value = DynamoType(expected_range_value) current = self.get_item(hash_value, lookup_range_value) - item = Item( hash_value, self.hash_key_type, range_value, self.range_key_type, item_attrs ) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index d3767c3fd..9bcb0d541 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -293,11 +293,9 @@ class DynamoHandler(BaseResponse): except ItemSizeTooLarge: er = "com.amazonaws.dynamodb.v20111205#ValidationException" return self.error(er, ItemSizeTooLarge.message) - except ValueError: + except ValueError as ve: er = "com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException" - return self.error( - er, "A condition specified in the operation could not be evaluated." - ) + return self.error(er, str(ve)) if result: item_dict = result.to_json() diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 428b58f81..0f2be6a2e 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1344,6 +1344,24 @@ def test_get_item_returns_consumed_capacity(): assert "TableName" in response["ConsumedCapacity"] +@mock_dynamodb2 +def test_put_item_nonexisting_hash_key(): + dynamodb = boto3.resource("dynamodb") + dynamodb.create_table( + AttributeDefinitions=[{"AttributeName": "structure_id", "AttributeType": "S"},], + TableName="test", + KeySchema=[{"AttributeName": "structure_id", "KeyType": "HASH"},], + ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, + ) + table = dynamodb.Table("test") + + with assert_raises(ClientError) as ex: + table.put_item(Item={"a_terribly_misguided_id_attribute": "abcdef"}) + ex.exception.response["Error"]["Message"].should.equal( + "One or more parameter values were invalid: Missing the key structure_id in the item" + ) + + def test_filter_expression(): row1 = moto.dynamodb2.models.Item( None, From e9930b0cb2dd66e0057511f742ae5021ad377407 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 10 Mar 2020 13:30:38 +0000 Subject: [PATCH 08/32] S3 - test fix - Use plain text as content, instead of file --- tests/test_s3/test_s3.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 48939be26..800daaef8 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -2030,17 +2030,15 @@ def test_boto3_s3_content_type(): s3 = boto3.resource("s3", region_name=DEFAULT_REGION_NAME) my_bucket = s3.Bucket("my-cool-bucket") my_bucket.create() - local_path = "test_s3.py" - s3_path = local_path + s3_path = "test_s3.py" s3 = boto3.resource("s3", verify=False) - with open(local_path, "rb") as _file: - content_type = mimetypes.guess_type(local_path) - s3.Object(my_bucket.name, s3_path).put( - ContentType=content_type[0], Body=_file, ACL="public-read" - ) + content_type = "text/python-x" + s3.Object(my_bucket.name, s3_path).put( + ContentType=content_type, Body=b"some python code", ACL="public-read" + ) - s3.Object(my_bucket.name, s3_path).content_type.should.equal(content_type[0]) + s3.Object(my_bucket.name, s3_path).content_type.should.equal(content_type) @mock_s3 From 315ac32f0906c36ebf0525c7a9f83ceb8d8cf7ac Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 10 Mar 2020 14:28:12 +0000 Subject: [PATCH 09/32] Add region to test case --- tests/test_dynamodb2/test_dynamodb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 0f2be6a2e..24e0626c9 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1346,7 +1346,7 @@ def test_get_item_returns_consumed_capacity(): @mock_dynamodb2 def test_put_item_nonexisting_hash_key(): - dynamodb = boto3.resource("dynamodb") + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") dynamodb.create_table( AttributeDefinitions=[{"AttributeName": "structure_id", "AttributeType": "S"},], TableName="test", From ab68d14649a2378b77a749e5ddaed30cd406e1e9 Mon Sep 17 00:00:00 2001 From: Huang syunwei Date: Mon, 9 Apr 2018 14:13:41 +1000 Subject: [PATCH 10/32] Fix bug of put metric data with timestamp, timestamp should be a date time object instead of a string --- moto/cloudwatch/models.py | 8 +- tests/test_cloudwatch/test_cloudwatch.py | 265 +++++++++++++---------- 2 files changed, 152 insertions(+), 121 deletions(-) diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py index 13b31ddfe..716a29633 100644 --- a/moto/cloudwatch/models.py +++ b/moto/cloudwatch/models.py @@ -2,13 +2,14 @@ import json from boto3 import Session -from moto.core.utils import iso_8601_datetime_with_milliseconds +from moto.core.utils import iso_8601_datetime_without_milliseconds from moto.core import BaseBackend, BaseModel from moto.core.exceptions import RESTError from datetime import datetime, timedelta from dateutil.tz import tzutc from uuid import uuid4 from .utils import make_arn_for_dashboard +from dateutil import parser from moto.core import ACCOUNT_ID as DEFAULT_ACCOUNT_ID @@ -146,7 +147,7 @@ class Dashboard(BaseModel): class Statistics: def __init__(self, stats, dt): - self.timestamp = iso_8601_datetime_with_milliseconds(dt) + self.timestamp = iso_8601_datetime_without_milliseconds(dt) self.values = [] self.stats = stats @@ -278,8 +279,7 @@ class CloudWatchBackend(BaseBackend): # Preserve "datetime" for get_metric_statistics comparisons timestamp = metric_member.get("Timestamp") if timestamp is not None and type(timestamp) != datetime: - timestamp = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%fZ") - timestamp = timestamp.replace(tzinfo=tzutc()) + timestamp = parser.parse(timestamp) self.metric_data.append( MetricDatum( namespace, diff --git a/tests/test_cloudwatch/test_cloudwatch.py b/tests/test_cloudwatch/test_cloudwatch.py index cc624e852..49d9b63d2 100644 --- a/tests/test_cloudwatch/test_cloudwatch.py +++ b/tests/test_cloudwatch/test_cloudwatch.py @@ -1,117 +1,148 @@ -import boto -from boto.ec2.cloudwatch.alarm import MetricAlarm -import sure # noqa - -from moto import mock_cloudwatch_deprecated - - -def alarm_fixture(name="tester", action=None): - action = action or ["arn:alarm"] - return MetricAlarm( - name=name, - namespace="{0}_namespace".format(name), - metric="{0}_metric".format(name), - comparison=">=", - threshold=2.0, - period=60, - evaluation_periods=5, - statistic="Average", - description="A test", - dimensions={"InstanceId": ["i-0123456,i-0123457"]}, - alarm_actions=action, - ok_actions=["arn:ok"], - insufficient_data_actions=["arn:insufficient"], - unit="Seconds", - ) - - -@mock_cloudwatch_deprecated -def test_create_alarm(): - conn = boto.connect_cloudwatch() - - alarm = alarm_fixture() - conn.create_alarm(alarm) - - alarms = conn.describe_alarms() - alarms.should.have.length_of(1) - alarm = alarms[0] - alarm.name.should.equal("tester") - alarm.namespace.should.equal("tester_namespace") - alarm.metric.should.equal("tester_metric") - alarm.comparison.should.equal(">=") - alarm.threshold.should.equal(2.0) - alarm.period.should.equal(60) - alarm.evaluation_periods.should.equal(5) - alarm.statistic.should.equal("Average") - alarm.description.should.equal("A test") - dict(alarm.dimensions).should.equal({"InstanceId": ["i-0123456,i-0123457"]}) - list(alarm.alarm_actions).should.equal(["arn:alarm"]) - list(alarm.ok_actions).should.equal(["arn:ok"]) - list(alarm.insufficient_data_actions).should.equal(["arn:insufficient"]) - alarm.unit.should.equal("Seconds") - - -@mock_cloudwatch_deprecated -def test_delete_alarm(): - conn = boto.connect_cloudwatch() - - alarms = conn.describe_alarms() - alarms.should.have.length_of(0) - - alarm = alarm_fixture() - conn.create_alarm(alarm) - - alarms = conn.describe_alarms() - alarms.should.have.length_of(1) - - alarms[0].delete() - - alarms = conn.describe_alarms() - alarms.should.have.length_of(0) - - -@mock_cloudwatch_deprecated -def test_put_metric_data(): - conn = boto.connect_cloudwatch() - - conn.put_metric_data( - namespace="tester", - name="metric", - value=1.5, - dimensions={"InstanceId": ["i-0123456,i-0123457"]}, - ) - - metrics = conn.list_metrics() - metrics.should.have.length_of(1) - metric = metrics[0] - metric.namespace.should.equal("tester") - metric.name.should.equal("metric") - dict(metric.dimensions).should.equal({"InstanceId": ["i-0123456,i-0123457"]}) - - -@mock_cloudwatch_deprecated -def test_describe_alarms(): - conn = boto.connect_cloudwatch() - - alarms = conn.describe_alarms() - alarms.should.have.length_of(0) - - conn.create_alarm(alarm_fixture(name="nfoobar", action="afoobar")) - conn.create_alarm(alarm_fixture(name="nfoobaz", action="afoobaz")) - conn.create_alarm(alarm_fixture(name="nbarfoo", action="abarfoo")) - conn.create_alarm(alarm_fixture(name="nbazfoo", action="abazfoo")) - - alarms = conn.describe_alarms() - alarms.should.have.length_of(4) - alarms = conn.describe_alarms(alarm_name_prefix="nfoo") - alarms.should.have.length_of(2) - alarms = conn.describe_alarms(alarm_names=["nfoobar", "nbarfoo", "nbazfoo"]) - alarms.should.have.length_of(3) - alarms = conn.describe_alarms(action_prefix="afoo") - alarms.should.have.length_of(2) - - for alarm in conn.describe_alarms(): - alarm.delete() - - alarms = conn.describe_alarms() - alarms.should.have.length_of(0) +import boto +from boto.ec2.cloudwatch.alarm import MetricAlarm +from datetime import datetime +import sure # noqa + +from moto import mock_cloudwatch_deprecated + + +def alarm_fixture(name="tester", action=None): + action = action or ["arn:alarm"] + return MetricAlarm( + name=name, + namespace="{0}_namespace".format(name), + metric="{0}_metric".format(name), + comparison=">=", + threshold=2.0, + period=60, + evaluation_periods=5, + statistic="Average", + description="A test", + dimensions={"InstanceId": ["i-0123456,i-0123457"]}, + alarm_actions=action, + ok_actions=["arn:ok"], + insufficient_data_actions=["arn:insufficient"], + unit="Seconds", + ) + + +@mock_cloudwatch_deprecated +def test_create_alarm(): + conn = boto.connect_cloudwatch() + + alarm = alarm_fixture() + conn.create_alarm(alarm) + + alarms = conn.describe_alarms() + alarms.should.have.length_of(1) + alarm = alarms[0] + alarm.name.should.equal("tester") + alarm.namespace.should.equal("tester_namespace") + alarm.metric.should.equal("tester_metric") + alarm.comparison.should.equal(">=") + alarm.threshold.should.equal(2.0) + alarm.period.should.equal(60) + alarm.evaluation_periods.should.equal(5) + alarm.statistic.should.equal("Average") + alarm.description.should.equal("A test") + dict(alarm.dimensions).should.equal({"InstanceId": ["i-0123456,i-0123457"]}) + list(alarm.alarm_actions).should.equal(["arn:alarm"]) + list(alarm.ok_actions).should.equal(["arn:ok"]) + list(alarm.insufficient_data_actions).should.equal(["arn:insufficient"]) + alarm.unit.should.equal("Seconds") + + +@mock_cloudwatch_deprecated +def test_delete_alarm(): + conn = boto.connect_cloudwatch() + + alarms = conn.describe_alarms() + alarms.should.have.length_of(0) + + alarm = alarm_fixture() + conn.create_alarm(alarm) + + alarms = conn.describe_alarms() + alarms.should.have.length_of(1) + + alarms[0].delete() + + alarms = conn.describe_alarms() + alarms.should.have.length_of(0) + + +@mock_cloudwatch_deprecated +def test_put_metric_data(): + conn = boto.connect_cloudwatch() + + conn.put_metric_data( + namespace="tester", + name="metric", + value=1.5, + dimensions={"InstanceId": ["i-0123456,i-0123457"]}, + ) + + metrics = conn.list_metrics() + metrics.should.have.length_of(1) + metric = metrics[0] + metric.namespace.should.equal("tester") + metric.name.should.equal("metric") + dict(metric.dimensions).should.equal({"InstanceId": ["i-0123456,i-0123457"]}) + + +@mock_cloudwatch_deprecated +def test_describe_alarms(): + conn = boto.connect_cloudwatch() + + alarms = conn.describe_alarms() + alarms.should.have.length_of(0) + + conn.create_alarm(alarm_fixture(name="nfoobar", action="afoobar")) + conn.create_alarm(alarm_fixture(name="nfoobaz", action="afoobaz")) + conn.create_alarm(alarm_fixture(name="nbarfoo", action="abarfoo")) + conn.create_alarm(alarm_fixture(name="nbazfoo", action="abazfoo")) + + alarms = conn.describe_alarms() + alarms.should.have.length_of(4) + alarms = conn.describe_alarms(alarm_name_prefix="nfoo") + alarms.should.have.length_of(2) + alarms = conn.describe_alarms(alarm_names=["nfoobar", "nbarfoo", "nbazfoo"]) + alarms.should.have.length_of(3) + alarms = conn.describe_alarms(action_prefix="afoo") + alarms.should.have.length_of(2) + + for alarm in conn.describe_alarms(): + alarm.delete() + + alarms = conn.describe_alarms() + alarms.should.have.length_of(0) + + +@mock_cloudwatch_deprecated +def test_get_metric_statistics(): + conn = boto.connect_cloudwatch() + + metric_timestamp = datetime(2018, 4, 9, 13, 0, 0, 0) + + conn.put_metric_data( + namespace='tester', + name='metric', + value=1.5, + dimensions={'InstanceId': ['i-0123456,i-0123457']}, + timestamp=metric_timestamp + ) + + metric_kwargs = dict( + namespace='tester', + metric_name='metric', + start_time=metric_timestamp, + end_time=datetime.now(), + period=3600, + statistics=['Minimum'] + ) + + datapoints = conn.get_metric_statistics(**metric_kwargs) + datapoints.should.have.length_of(1) + datapoint = datapoints[0] + datapoint.should.have.key('Minimum').which.should.equal(1.5) + datapoint.should.have.key('Timestamp').which.should.equal(metric_timestamp) From 0e433691556243f34d995d5eb544bb2ed40c5e4c Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 11 Mar 2020 12:47:40 +0000 Subject: [PATCH 11/32] Linting --- tests/test_cloudwatch/test_cloudwatch.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_cloudwatch/test_cloudwatch.py b/tests/test_cloudwatch/test_cloudwatch.py index 49d9b63d2..dee8aa605 100644 --- a/tests/test_cloudwatch/test_cloudwatch.py +++ b/tests/test_cloudwatch/test_cloudwatch.py @@ -125,24 +125,24 @@ def test_get_metric_statistics(): metric_timestamp = datetime(2018, 4, 9, 13, 0, 0, 0) conn.put_metric_data( - namespace='tester', - name='metric', + namespace="tester", + name="metric", value=1.5, - dimensions={'InstanceId': ['i-0123456,i-0123457']}, - timestamp=metric_timestamp + dimensions={"InstanceId": ["i-0123456,i-0123457"]}, + timestamp=metric_timestamp, ) metric_kwargs = dict( - namespace='tester', - metric_name='metric', + namespace="tester", + metric_name="metric", start_time=metric_timestamp, end_time=datetime.now(), period=3600, - statistics=['Minimum'] + statistics=["Minimum"], ) datapoints = conn.get_metric_statistics(**metric_kwargs) datapoints.should.have.length_of(1) datapoint = datapoints[0] - datapoint.should.have.key('Minimum').which.should.equal(1.5) - datapoint.should.have.key('Timestamp').which.should.equal(metric_timestamp) + datapoint.should.have.key("Minimum").which.should.equal(1.5) + datapoint.should.have.key("Timestamp").which.should.equal(metric_timestamp) From 20364b177a5afb1d0452c835daa60086e6eb289e Mon Sep 17 00:00:00 2001 From: Luis Pollo Date: Wed, 7 Nov 2018 15:58:26 -0600 Subject: [PATCH 12/32] Fix IAM role name when parsed from CloudFormation JSON. --- moto/iam/models.py | 4 +++- .../test_cloudformation_stack_integration.py | 7 ++++--- 2 files changed, 7 insertions(+), 4 deletions(-) mode change 100644 => 100755 moto/iam/models.py diff --git a/moto/iam/models.py b/moto/iam/models.py old mode 100644 new mode 100755 index 18b3a7a6f..7ac3a4f9e --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -12,6 +12,7 @@ import re from cryptography import x509 from cryptography.hazmat.backends import default_backend from six.moves.urllib.parse import urlparse +from uuid import uuid4 from moto.core.exceptions import RESTError from moto.core import BaseBackend, BaseModel, ACCOUNT_ID @@ -330,9 +331,10 @@ class Role(BaseModel): cls, resource_name, cloudformation_json, region_name ): properties = cloudformation_json["Properties"] + role_name = properties['RoleName'] if 'RoleName' in properties else str(uuid4())[0:5] role = iam_backend.create_role( - role_name=resource_name, + role_name=role_name, assume_role_policy_document=properties["AssumeRolePolicyDocument"], path=properties.get("Path", "/"), permissions_boundary=properties.get("PermissionsBoundary", ""), diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index e296ef2ed..45a2045b3 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -909,6 +909,7 @@ def test_iam_roles(): }, "my-role-no-path": { "Properties": { + "RoleName": "my-role-no-path-name", "AssumeRolePolicyDocument": { "Statement": [ { @@ -936,13 +937,13 @@ def test_iam_roles(): role_name_to_id = {} for role_result in role_results: role = iam_conn.get_role(role_result.role_name) - role.role_name.should.contain("my-role") - if "with-path" in role.role_name: + if "my-role" not in role.role_name: role_name_to_id["with-path"] = role.role_id role.path.should.equal("my-path") + len(role.role_name).should.equal(5) # Role name is not specified, so randomly generated - can't check exact name else: role_name_to_id["no-path"] = role.role_id - role.role_name.should.contain("no-path") + role.role_name.should.equal("my-role-no-path-name") role.path.should.equal("/") instance_profile_responses = iam_conn.list_instance_profiles()[ From 9163f042927eb688a2a93194b2a0f5d4a47a8aca Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 11 Mar 2020 13:19:40 +0000 Subject: [PATCH 13/32] Linting --- moto/iam/models.py | 4 +++- .../test_cloudformation_stack_integration.py | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/moto/iam/models.py b/moto/iam/models.py index 7ac3a4f9e..e34ca7cf8 100755 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -331,7 +331,9 @@ class Role(BaseModel): cls, resource_name, cloudformation_json, region_name ): properties = cloudformation_json["Properties"] - role_name = properties['RoleName'] if 'RoleName' in properties else str(uuid4())[0:5] + role_name = ( + properties["RoleName"] if "RoleName" in properties else str(uuid4())[0:5] + ) role = iam_backend.create_role( role_name=role_name, diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index 45a2045b3..5a3181449 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -918,7 +918,7 @@ def test_iam_roles(): "Principal": {"Service": ["ec2.amazonaws.com"]}, } ] - } + }, }, "Type": "AWS::IAM::Role", }, @@ -940,7 +940,9 @@ def test_iam_roles(): if "my-role" not in role.role_name: role_name_to_id["with-path"] = role.role_id role.path.should.equal("my-path") - len(role.role_name).should.equal(5) # Role name is not specified, so randomly generated - can't check exact name + len(role.role_name).should.equal( + 5 + ) # Role name is not specified, so randomly generated - can't check exact name else: role_name_to_id["no-path"] = role.role_id role.role_name.should.equal("my-role-no-path-name") From 57056954954ddea47e82d7443f3f401c41c2476f Mon Sep 17 00:00:00 2001 From: Brent Driskill Date: Sun, 8 Mar 2020 20:32:01 -0400 Subject: [PATCH 14/32] SSM: Added support for label_parameter_version and getting labels on get_parameter_history --- moto/ssm/exceptions.py | 17 ++ moto/ssm/models.py | 66 ++++++- moto/ssm/responses.py | 13 +- tests/test_ssm/test_ssm_boto3.py | 303 +++++++++++++++++++++++++++++++ 4 files changed, 396 insertions(+), 3 deletions(-) diff --git a/moto/ssm/exceptions.py b/moto/ssm/exceptions.py index 3458fe7d3..1c7c26ed9 100644 --- a/moto/ssm/exceptions.py +++ b/moto/ssm/exceptions.py @@ -22,6 +22,23 @@ class InvalidFilterValue(JsonRESTError): def __init__(self, message): super(InvalidFilterValue, self).__init__("InvalidFilterValue", message) +class ParameterNotFound(JsonRESTError): + code = 400 + + def __init__(self, message): + super(ParameterNotFound, self).__init__("ParameterNotFound", message) + +class ParameterVersionNotFound(JsonRESTError): + code = 400 + + def __init__(self, message): + super(ParameterVersionNotFound, self).__init__("ParameterVersionNotFound", message) + +class ParameterVersionLabelLimitExceeded(JsonRESTError): + code = 400 + + def __init__(self, message): + super(ParameterVersionLabelLimitExceeded, self).__init__("ParameterVersionLabelLimitExceeded", message) class ValidationException(JsonRESTError): code = 400 diff --git a/moto/ssm/models.py b/moto/ssm/models.py index a7518d405..2806a0fe0 100644 --- a/moto/ssm/models.py +++ b/moto/ssm/models.py @@ -19,6 +19,9 @@ from .exceptions import ( InvalidFilterValue, InvalidFilterOption, InvalidFilterKey, + ParameterVersionLabelLimitExceeded, + ParameterVersionNotFound, + ParameterNotFound ) @@ -32,7 +35,7 @@ class Parameter(BaseModel): allowed_pattern, keyid, last_modified_date, - version, + version ): self.name = name self.type = type @@ -41,6 +44,7 @@ class Parameter(BaseModel): self.keyid = keyid self.last_modified_date = last_modified_date self.version = version + self.labels = [] if self.type == "SecureString": if not self.keyid: @@ -75,7 +79,7 @@ class Parameter(BaseModel): return r - def describe_response_object(self, decrypt=False): + def describe_response_object(self, decrypt=False, include_labels=False): r = self.response_object(decrypt) r["LastModifiedDate"] = round(self.last_modified_date, 3) r["LastModifiedUser"] = "N/A" @@ -89,6 +93,9 @@ class Parameter(BaseModel): if self.allowed_pattern: r["AllowedPattern"] = self.allowed_pattern + if include_labels: + r["Labels"] = self.labels + return r @@ -614,6 +621,61 @@ class SimpleSystemManagerBackend(BaseBackend): return self._parameters[name][-1] return None + def label_parameter_version(self, name, version, labels): + previous_parameter_versions = self._parameters[name] + if not previous_parameter_versions: + raise ParameterNotFound( + "Parameter %s not found." % name + ) + found_parameter = None + labels_needing_removal = [] + if not version: + version = 1 + for parameter in previous_parameter_versions: + if parameter.version >= version: + version = parameter.version + for parameter in previous_parameter_versions: + if parameter.version == version: + found_parameter = parameter + else: + for label in labels: + if label in parameter.labels: + labels_needing_removal.append(label) + if not found_parameter: + raise ParameterVersionNotFound( + "Systems Manager could not find version %s of %s. " + "Verify the version and try again." % (version, name) + ) + labels_to_append = [] + invalid_labels = [] + for label in labels: + if label.startswith("aws") or label.startswith("ssm") or label[:1].isdigit() or not re.match("^[a-zA-z0-9_\.\-]*$", label): + invalid_labels.append(label) + continue + if len(label) > 100: + raise ValidationException( + "1 validation error detected: " + "Value '[%s]' at 'labels' failed to satisfy constraint: " + "Member must satisfy constraint: " + "[Member must have length less than or equal to 100, Member must have length greater than or equal to 1]" % label + ) + continue + if label not in found_parameter.labels: + labels_to_append.append(label) + if (len(found_parameter.labels) + len(labels_to_append)) > 10: + raise ParameterVersionLabelLimitExceeded( + "An error occurred (ParameterVersionLabelLimitExceeded) when calling the LabelParameterVersion operation: " + "A parameter version can have maximum 10 labels." + "Move one or more labels to another version and try again." + ) + found_parameter.labels = found_parameter.labels + labels_to_append + for parameter in previous_parameter_versions: + if parameter.version != version: + for label in parameter.labels[:]: + if label in labels_needing_removal: + parameter.labels.remove(label) + return [invalid_labels, version] + def put_parameter( self, name, description, value, type, allowed_pattern, keyid, overwrite ): diff --git a/moto/ssm/responses.py b/moto/ssm/responses.py index 831737848..f453518ab 100644 --- a/moto/ssm/responses.py +++ b/moto/ssm/responses.py @@ -168,12 +168,23 @@ class SimpleSystemManagerResponse(BaseResponse): response = {"Parameters": []} for parameter_version in result: param_data = parameter_version.describe_response_object( - decrypt=with_decryption + decrypt=with_decryption, + include_labels=True ) response["Parameters"].append(param_data) return json.dumps(response) + def label_parameter_version(self): + name = self._get_param("Name") + version = self._get_param("ParameterVersion") + labels = self._get_param("Labels") + + invalid_labels, version = self.ssm_backend.label_parameter_version(name, version, labels) + + response = {"InvalidLabels": invalid_labels, "ParameterVersion": version} + return json.dumps(response) + def add_tags_to_resource(self): resource_id = self._get_param("ResourceId") resource_type = self._get_param("ResourceType") diff --git a/tests/test_ssm/test_ssm_boto3.py b/tests/test_ssm/test_ssm_boto3.py index bb674fb65..c2813772d 100644 --- a/tests/test_ssm/test_ssm_boto3.py +++ b/tests/test_ssm/test_ssm_boto3.py @@ -897,6 +897,7 @@ def test_get_parameter_history(): param["Value"].should.equal("value-%d" % index) param["Version"].should.equal(index + 1) param["Description"].should.equal("A test parameter version %d" % index) + param["Labels"].should.equal([]) len(parameters_response).should.equal(3) @@ -937,6 +938,308 @@ def test_get_parameter_history_with_secure_string(): len(parameters_response).should.equal(3) +@mock_ssm +def test_label_parameter_version(): + client = boto3.client("ssm", region_name="us-east-1") + + test_parameter_name = "test" + client.put_parameter(Name=test_parameter_name, Description="A test parameter", Value="value", Type="String") + + response = client.label_parameter_version(Name=test_parameter_name, Labels=["test-label"]) + response["InvalidLabels"].should.equal([]) + response["ParameterVersion"].should.equal(1) + +@mock_ssm +def test_label_parameter_version_with_specific_version(): + client = boto3.client("ssm", region_name="us-east-1") + + test_parameter_name = "test" + client.put_parameter(Name=test_parameter_name, Description="A test parameter", Value="value", Type="String") + + response = client.label_parameter_version(Name=test_parameter_name, ParameterVersion=1, Labels=["test-label"]) + response["InvalidLabels"].should.equal([]) + response["ParameterVersion"].should.equal(1) + +@mock_ssm +def test_label_parameter_version_twice(): + client = boto3.client("ssm", region_name="us-east-1") + + test_parameter_name = "test" + test_labels = ["test-label"] + client.put_parameter(Name=test_parameter_name, Description="A test parameter", Value="value", Type="String") + + response = client.label_parameter_version(Name=test_parameter_name, ParameterVersion=1, Labels=test_labels) + response["InvalidLabels"].should.equal([]) + response["ParameterVersion"].should.equal(1) + response = client.label_parameter_version(Name=test_parameter_name, ParameterVersion=1, Labels=test_labels) + response["InvalidLabels"].should.equal([]) + response["ParameterVersion"].should.equal(1) + + response = client.get_parameter_history(Name=test_parameter_name) + len(response["Parameters"]).should.equal(1) + response["Parameters"][0]["Labels"].should.equal(test_labels) + +@mock_ssm +def test_label_parameter_moving_versions(): + client = boto3.client("ssm", region_name="us-east-1") + + test_parameter_name = "test" + test_labels = ["test-label"] + + for i in range(3): + client.put_parameter( + Name=test_parameter_name, + Description="A test parameter version %d" % i, + Value="value-%d" % i, + Type="String", + Overwrite=True + ) + + response = client.label_parameter_version(Name=test_parameter_name, ParameterVersion=1, Labels=test_labels) + response["InvalidLabels"].should.equal([]) + response["ParameterVersion"].should.equal(1) + response = client.label_parameter_version(Name=test_parameter_name, ParameterVersion=2, Labels=test_labels) + response["InvalidLabels"].should.equal([]) + response["ParameterVersion"].should.equal(2) + + response = client.get_parameter_history(Name=test_parameter_name) + parameters_response = response["Parameters"] + + for index, param in enumerate(parameters_response): + param["Name"].should.equal(test_parameter_name) + param["Type"].should.equal("String") + param["Value"].should.equal("value-%d" % index) + param["Version"].should.equal(index + 1) + param["Description"].should.equal("A test parameter version %d" % index) + labels = test_labels if param["Version"] == 2 else [] + param["Labels"].should.equal(labels) + + len(parameters_response).should.equal(3) + +@mock_ssm +def test_label_parameter_moving_versions_complex(): + client = boto3.client("ssm", region_name="us-east-1") + + test_parameter_name = "test" + + for i in range(3): + client.put_parameter( + Name=test_parameter_name, + Description="A test parameter version %d" % i, + Value="value-%d" % i, + Type="String", + Overwrite=True + ) + + response = client.label_parameter_version(Name=test_parameter_name, ParameterVersion=1, Labels=["test-label1", "test-label2", "test-label3"]) + response["InvalidLabels"].should.equal([]) + response["ParameterVersion"].should.equal(1) + response = client.label_parameter_version(Name=test_parameter_name, ParameterVersion=2, Labels=["test-label2", "test-label3"]) + response["InvalidLabels"].should.equal([]) + response["ParameterVersion"].should.equal(2) + + response = client.get_parameter_history(Name=test_parameter_name) + parameters_response = response["Parameters"] + + for index, param in enumerate(parameters_response): + param["Name"].should.equal(test_parameter_name) + param["Type"].should.equal("String") + param["Value"].should.equal("value-%d" % index) + param["Version"].should.equal(index + 1) + param["Description"].should.equal("A test parameter version %d" % index) + labels = ["test-label2", "test-label3"] if param["Version"] == 2 else (["test-label1"] if param["Version"] == 1 else []) + param["Labels"].should.equal(labels) + + len(parameters_response).should.equal(3) + +@mock_ssm +def test_label_parameter_version_exception_ten_labels_at_once(): + client = boto3.client("ssm", region_name="us-east-1") + + test_parameter_name = "test" + test_labels = ["test-label1", "test-label2", "test-label3", "test-label4", "test-label5", "test-label6", "test-label7", "test-label8", "test-label9", "test-label10", "test-label11"] + + client.put_parameter(Name=test_parameter_name, Description="A test parameter", Value="value", Type="String") + client.label_parameter_version.when.called_with( + Name="test", ParameterVersion=1, Labels=test_labels + ).should.throw( + ClientError, + "An error occurred (ParameterVersionLabelLimitExceeded) when calling the LabelParameterVersion operation: " + "A parameter version can have maximum 10 labels." + "Move one or more labels to another version and try again." + ) + +@mock_ssm +def test_label_parameter_version_exception_ten_labels_over_multiple_calls(): + client = boto3.client("ssm", region_name="us-east-1") + + test_parameter_name = "test" + + client.put_parameter(Name=test_parameter_name, Description="A test parameter", Value="value", Type="String") + client.label_parameter_version(Name=test_parameter_name, ParameterVersion=1, Labels=["test-label1", "test-label2", "test-label3", "test-label4", "test-label5"]) + client.label_parameter_version.when.called_with( + Name="test", ParameterVersion=1, Labels=["test-label6", "test-label7", "test-label8", "test-label9", "test-label10", "test-label11"] + ).should.throw( + ClientError, + "An error occurred (ParameterVersionLabelLimitExceeded) when calling the LabelParameterVersion operation: " + "A parameter version can have maximum 10 labels." + "Move one or more labels to another version and try again." + ) + +@mock_ssm +def test_label_parameter_version_invalid_name(): + client = boto3.client("ssm", region_name="us-east-1") + + test_parameter_name = "test" + + response = client.label_parameter_version.when.called_with( + Name=test_parameter_name, Labels=["test-label"] + ).should.throw( + ClientError, + "An error occurred (ParameterNotFound) when calling the LabelParameterVersion operation: " + "Parameter test not found." + ) + +@mock_ssm +def test_label_parameter_version_invalid_parameter_version(): + client = boto3.client("ssm", region_name="us-east-1") + + test_parameter_name = "test" + client.put_parameter(Name=test_parameter_name, Description="A test parameter", Value="value", Type="String") + + response = client.label_parameter_version.when.called_with( + Name=test_parameter_name, Labels=["test-label"], ParameterVersion=5 + ).should.throw( + ClientError, + "An error occurred (ParameterVersionNotFound) when calling the LabelParameterVersion operation: " + "Systems Manager could not find version 5 of test. " + "Verify the version and try again." + ) + +@mock_ssm +def test_label_parameter_version_invalid_label(): + client = boto3.client("ssm", region_name="us-east-1") + + test_parameter_name = "test" + client.put_parameter(Name=test_parameter_name, Description="A test parameter", Value="value", Type="String") + response = client.label_parameter_version(Name=test_parameter_name, ParameterVersion=1, Labels=["awsabc"]) + response["InvalidLabels"].should.equal(["awsabc"]) + + response = client.label_parameter_version(Name=test_parameter_name, ParameterVersion=1, Labels=["ssmabc"]) + response["InvalidLabels"].should.equal(["ssmabc"]) + + response = client.label_parameter_version(Name=test_parameter_name, ParameterVersion=1, Labels=["9abc"]) + response["InvalidLabels"].should.equal(["9abc"]) + + response = client.label_parameter_version(Name=test_parameter_name, ParameterVersion=1, Labels=["abc/123"]) + response["InvalidLabels"].should.equal(["abc/123"]) + + client.label_parameter_version.when.called_with( + Name=test_parameter_name, ParameterVersion=1, Labels=["a"*101] + ).should.throw( + ClientError, + "1 validation error detected: " + "Value '[%s]' at 'labels' failed to satisfy constraint: " + "Member must satisfy constraint: " + "[Member must have length less than or equal to 100, Member must have length greater than or equal to 1]" % ("a"*101) + ) + + +@mock_ssm +def test_get_parameter_history_with_label(): + client = boto3.client("ssm", region_name="us-east-1") + + test_parameter_name = "test" + test_labels = ["test-label"] + + for i in range(3): + client.put_parameter( + Name=test_parameter_name, + Description="A test parameter version %d" % i, + Value="value-%d" % i, + Type="String", + Overwrite=True, + ) + + client.label_parameter_version(Name=test_parameter_name, ParameterVersion=1, Labels=test_labels) + + response = client.get_parameter_history(Name=test_parameter_name) + parameters_response = response["Parameters"] + + for index, param in enumerate(parameters_response): + param["Name"].should.equal(test_parameter_name) + param["Type"].should.equal("String") + param["Value"].should.equal("value-%d" % index) + param["Version"].should.equal(index + 1) + param["Description"].should.equal("A test parameter version %d" % index) + labels = test_labels if param["Version"] == 1 else [] + param["Labels"].should.equal(labels) + + len(parameters_response).should.equal(3) + +@mock_ssm +def test_get_parameter_history_with_label_non_latest(): + client = boto3.client("ssm", region_name="us-east-1") + + test_parameter_name = "test" + test_labels = ["test-label"] + + for i in range(3): + client.put_parameter( + Name=test_parameter_name, + Description="A test parameter version %d" % i, + Value="value-%d" % i, + Type="String", + Overwrite=True, + ) + + client.label_parameter_version(Name=test_parameter_name, ParameterVersion=2, Labels=test_labels) + + response = client.get_parameter_history(Name=test_parameter_name) + parameters_response = response["Parameters"] + + for index, param in enumerate(parameters_response): + param["Name"].should.equal(test_parameter_name) + param["Type"].should.equal("String") + param["Value"].should.equal("value-%d" % index) + param["Version"].should.equal(index + 1) + param["Description"].should.equal("A test parameter version %d" % index) + labels = test_labels if param["Version"] == 2 else [] + param["Labels"].should.equal(labels) + + len(parameters_response).should.equal(3) + +@mock_ssm +def test_get_parameter_history_with_label_latest_assumed(): + client = boto3.client("ssm", region_name="us-east-1") + + test_parameter_name = "test" + test_labels = ["test-label"] + + for i in range(3): + client.put_parameter( + Name=test_parameter_name, + Description="A test parameter version %d" % i, + Value="value-%d" % i, + Type="String", + Overwrite=True, + ) + + client.label_parameter_version(Name=test_parameter_name, Labels=test_labels) + + response = client.get_parameter_history(Name=test_parameter_name) + parameters_response = response["Parameters"] + + for index, param in enumerate(parameters_response): + param["Name"].should.equal(test_parameter_name) + param["Type"].should.equal("String") + param["Value"].should.equal("value-%d" % index) + param["Version"].should.equal(index + 1) + param["Description"].should.equal("A test parameter version %d" % index) + labels = test_labels if param["Version"] == 3 else [] + param["Labels"].should.equal(labels) + + len(parameters_response).should.equal(3) @mock_ssm def test_get_parameter_history_missing_parameter(): From e3e4b741d8aae3acc7408bdb48f733c66fa59d39 Mon Sep 17 00:00:00 2001 From: Brent Driskill Date: Wed, 11 Mar 2020 11:57:04 -0400 Subject: [PATCH 15/32] SSM: Fix the formatting associated with label_parameter_version/get_parameter_history updates --- moto/ssm/exceptions.py | 12 +- moto/ssm/models.py | 20 ++-- moto/ssm/responses.py | 7 +- tests/test_ssm/test_ssm_boto3.py | 184 +++++++++++++++++++++++++------ 4 files changed, 176 insertions(+), 47 deletions(-) diff --git a/moto/ssm/exceptions.py b/moto/ssm/exceptions.py index 1c7c26ed9..83ae26b6c 100644 --- a/moto/ssm/exceptions.py +++ b/moto/ssm/exceptions.py @@ -22,23 +22,31 @@ class InvalidFilterValue(JsonRESTError): def __init__(self, message): super(InvalidFilterValue, self).__init__("InvalidFilterValue", message) + class ParameterNotFound(JsonRESTError): code = 400 def __init__(self, message): super(ParameterNotFound, self).__init__("ParameterNotFound", message) + class ParameterVersionNotFound(JsonRESTError): code = 400 def __init__(self, message): - super(ParameterVersionNotFound, self).__init__("ParameterVersionNotFound", message) + super(ParameterVersionNotFound, self).__init__( + "ParameterVersionNotFound", message + ) + class ParameterVersionLabelLimitExceeded(JsonRESTError): code = 400 def __init__(self, message): - super(ParameterVersionLabelLimitExceeded, self).__init__("ParameterVersionLabelLimitExceeded", message) + super(ParameterVersionLabelLimitExceeded, self).__init__( + "ParameterVersionLabelLimitExceeded", message + ) + class ValidationException(JsonRESTError): code = 400 diff --git a/moto/ssm/models.py b/moto/ssm/models.py index 2806a0fe0..201f43c5a 100644 --- a/moto/ssm/models.py +++ b/moto/ssm/models.py @@ -21,7 +21,7 @@ from .exceptions import ( InvalidFilterKey, ParameterVersionLabelLimitExceeded, ParameterVersionNotFound, - ParameterNotFound + ParameterNotFound, ) @@ -35,7 +35,7 @@ class Parameter(BaseModel): allowed_pattern, keyid, last_modified_date, - version + version, ): self.name = name self.type = type @@ -624,9 +624,7 @@ class SimpleSystemManagerBackend(BaseBackend): def label_parameter_version(self, name, version, labels): previous_parameter_versions = self._parameters[name] if not previous_parameter_versions: - raise ParameterNotFound( - "Parameter %s not found." % name - ) + raise ParameterNotFound("Parameter %s not found." % name) found_parameter = None labels_needing_removal = [] if not version: @@ -645,11 +643,16 @@ class SimpleSystemManagerBackend(BaseBackend): raise ParameterVersionNotFound( "Systems Manager could not find version %s of %s. " "Verify the version and try again." % (version, name) - ) + ) labels_to_append = [] invalid_labels = [] for label in labels: - if label.startswith("aws") or label.startswith("ssm") or label[:1].isdigit() or not re.match("^[a-zA-z0-9_\.\-]*$", label): + if ( + label.startswith("aws") + or label.startswith("ssm") + or label[:1].isdigit() + or not re.match("^[a-zA-z0-9_\.\-]*$", label) + ): invalid_labels.append(label) continue if len(label) > 100: @@ -657,7 +660,8 @@ class SimpleSystemManagerBackend(BaseBackend): "1 validation error detected: " "Value '[%s]' at 'labels' failed to satisfy constraint: " "Member must satisfy constraint: " - "[Member must have length less than or equal to 100, Member must have length greater than or equal to 1]" % label + "[Member must have length less than or equal to 100, Member must have length greater than or equal to 1]" + % label ) continue if label not in found_parameter.labels: diff --git a/moto/ssm/responses.py b/moto/ssm/responses.py index f453518ab..45d2dec0a 100644 --- a/moto/ssm/responses.py +++ b/moto/ssm/responses.py @@ -168,8 +168,7 @@ class SimpleSystemManagerResponse(BaseResponse): response = {"Parameters": []} for parameter_version in result: param_data = parameter_version.describe_response_object( - decrypt=with_decryption, - include_labels=True + decrypt=with_decryption, include_labels=True ) response["Parameters"].append(param_data) @@ -180,7 +179,9 @@ class SimpleSystemManagerResponse(BaseResponse): version = self._get_param("ParameterVersion") labels = self._get_param("Labels") - invalid_labels, version = self.ssm_backend.label_parameter_version(name, version, labels) + invalid_labels, version = self.ssm_backend.label_parameter_version( + name, version, labels + ) response = {"InvalidLabels": invalid_labels, "ParameterVersion": version} return json.dumps(response) diff --git a/tests/test_ssm/test_ssm_boto3.py b/tests/test_ssm/test_ssm_boto3.py index c2813772d..170cd8a3e 100644 --- a/tests/test_ssm/test_ssm_boto3.py +++ b/tests/test_ssm/test_ssm_boto3.py @@ -938,40 +938,66 @@ def test_get_parameter_history_with_secure_string(): len(parameters_response).should.equal(3) + @mock_ssm def test_label_parameter_version(): client = boto3.client("ssm", region_name="us-east-1") test_parameter_name = "test" - client.put_parameter(Name=test_parameter_name, Description="A test parameter", Value="value", Type="String") + client.put_parameter( + Name=test_parameter_name, + Description="A test parameter", + Value="value", + Type="String", + ) - response = client.label_parameter_version(Name=test_parameter_name, Labels=["test-label"]) + response = client.label_parameter_version( + Name=test_parameter_name, Labels=["test-label"] + ) response["InvalidLabels"].should.equal([]) response["ParameterVersion"].should.equal(1) + @mock_ssm def test_label_parameter_version_with_specific_version(): client = boto3.client("ssm", region_name="us-east-1") test_parameter_name = "test" - client.put_parameter(Name=test_parameter_name, Description="A test parameter", Value="value", Type="String") + client.put_parameter( + Name=test_parameter_name, + Description="A test parameter", + Value="value", + Type="String", + ) - response = client.label_parameter_version(Name=test_parameter_name, ParameterVersion=1, Labels=["test-label"]) + response = client.label_parameter_version( + Name=test_parameter_name, ParameterVersion=1, Labels=["test-label"] + ) response["InvalidLabels"].should.equal([]) response["ParameterVersion"].should.equal(1) + @mock_ssm def test_label_parameter_version_twice(): client = boto3.client("ssm", region_name="us-east-1") test_parameter_name = "test" test_labels = ["test-label"] - client.put_parameter(Name=test_parameter_name, Description="A test parameter", Value="value", Type="String") + client.put_parameter( + Name=test_parameter_name, + Description="A test parameter", + Value="value", + Type="String", + ) - response = client.label_parameter_version(Name=test_parameter_name, ParameterVersion=1, Labels=test_labels) + response = client.label_parameter_version( + Name=test_parameter_name, ParameterVersion=1, Labels=test_labels + ) response["InvalidLabels"].should.equal([]) response["ParameterVersion"].should.equal(1) - response = client.label_parameter_version(Name=test_parameter_name, ParameterVersion=1, Labels=test_labels) + response = client.label_parameter_version( + Name=test_parameter_name, ParameterVersion=1, Labels=test_labels + ) response["InvalidLabels"].should.equal([]) response["ParameterVersion"].should.equal(1) @@ -979,6 +1005,7 @@ def test_label_parameter_version_twice(): len(response["Parameters"]).should.equal(1) response["Parameters"][0]["Labels"].should.equal(test_labels) + @mock_ssm def test_label_parameter_moving_versions(): client = boto3.client("ssm", region_name="us-east-1") @@ -992,13 +1019,17 @@ def test_label_parameter_moving_versions(): Description="A test parameter version %d" % i, Value="value-%d" % i, Type="String", - Overwrite=True + Overwrite=True, ) - response = client.label_parameter_version(Name=test_parameter_name, ParameterVersion=1, Labels=test_labels) + response = client.label_parameter_version( + Name=test_parameter_name, ParameterVersion=1, Labels=test_labels + ) response["InvalidLabels"].should.equal([]) response["ParameterVersion"].should.equal(1) - response = client.label_parameter_version(Name=test_parameter_name, ParameterVersion=2, Labels=test_labels) + response = client.label_parameter_version( + Name=test_parameter_name, ParameterVersion=2, Labels=test_labels + ) response["InvalidLabels"].should.equal([]) response["ParameterVersion"].should.equal(2) @@ -1016,6 +1047,7 @@ def test_label_parameter_moving_versions(): len(parameters_response).should.equal(3) + @mock_ssm def test_label_parameter_moving_versions_complex(): client = boto3.client("ssm", region_name="us-east-1") @@ -1028,13 +1060,21 @@ def test_label_parameter_moving_versions_complex(): Description="A test parameter version %d" % i, Value="value-%d" % i, Type="String", - Overwrite=True + Overwrite=True, ) - response = client.label_parameter_version(Name=test_parameter_name, ParameterVersion=1, Labels=["test-label1", "test-label2", "test-label3"]) + response = client.label_parameter_version( + Name=test_parameter_name, + ParameterVersion=1, + Labels=["test-label1", "test-label2", "test-label3"], + ) response["InvalidLabels"].should.equal([]) response["ParameterVersion"].should.equal(1) - response = client.label_parameter_version(Name=test_parameter_name, ParameterVersion=2, Labels=["test-label2", "test-label3"]) + response = client.label_parameter_version( + Name=test_parameter_name, + ParameterVersion=2, + Labels=["test-label2", "test-label3"], + ) response["InvalidLabels"].should.equal([]) response["ParameterVersion"].should.equal(2) @@ -1047,45 +1087,93 @@ def test_label_parameter_moving_versions_complex(): param["Value"].should.equal("value-%d" % index) param["Version"].should.equal(index + 1) param["Description"].should.equal("A test parameter version %d" % index) - labels = ["test-label2", "test-label3"] if param["Version"] == 2 else (["test-label1"] if param["Version"] == 1 else []) + labels = ( + ["test-label2", "test-label3"] + if param["Version"] == 2 + else (["test-label1"] if param["Version"] == 1 else []) + ) param["Labels"].should.equal(labels) len(parameters_response).should.equal(3) + @mock_ssm def test_label_parameter_version_exception_ten_labels_at_once(): client = boto3.client("ssm", region_name="us-east-1") test_parameter_name = "test" - test_labels = ["test-label1", "test-label2", "test-label3", "test-label4", "test-label5", "test-label6", "test-label7", "test-label8", "test-label9", "test-label10", "test-label11"] + test_labels = [ + "test-label1", + "test-label2", + "test-label3", + "test-label4", + "test-label5", + "test-label6", + "test-label7", + "test-label8", + "test-label9", + "test-label10", + "test-label11", + ] - client.put_parameter(Name=test_parameter_name, Description="A test parameter", Value="value", Type="String") + client.put_parameter( + Name=test_parameter_name, + Description="A test parameter", + Value="value", + Type="String", + ) client.label_parameter_version.when.called_with( Name="test", ParameterVersion=1, Labels=test_labels ).should.throw( ClientError, "An error occurred (ParameterVersionLabelLimitExceeded) when calling the LabelParameterVersion operation: " "A parameter version can have maximum 10 labels." - "Move one or more labels to another version and try again." + "Move one or more labels to another version and try again.", ) + @mock_ssm def test_label_parameter_version_exception_ten_labels_over_multiple_calls(): client = boto3.client("ssm", region_name="us-east-1") test_parameter_name = "test" - client.put_parameter(Name=test_parameter_name, Description="A test parameter", Value="value", Type="String") - client.label_parameter_version(Name=test_parameter_name, ParameterVersion=1, Labels=["test-label1", "test-label2", "test-label3", "test-label4", "test-label5"]) + client.put_parameter( + Name=test_parameter_name, + Description="A test parameter", + Value="value", + Type="String", + ) + client.label_parameter_version( + Name=test_parameter_name, + ParameterVersion=1, + Labels=[ + "test-label1", + "test-label2", + "test-label3", + "test-label4", + "test-label5", + ], + ) client.label_parameter_version.when.called_with( - Name="test", ParameterVersion=1, Labels=["test-label6", "test-label7", "test-label8", "test-label9", "test-label10", "test-label11"] + Name="test", + ParameterVersion=1, + Labels=[ + "test-label6", + "test-label7", + "test-label8", + "test-label9", + "test-label10", + "test-label11", + ], ).should.throw( ClientError, "An error occurred (ParameterVersionLabelLimitExceeded) when calling the LabelParameterVersion operation: " "A parameter version can have maximum 10 labels." - "Move one or more labels to another version and try again." + "Move one or more labels to another version and try again.", ) + @mock_ssm def test_label_parameter_version_invalid_name(): client = boto3.client("ssm", region_name="us-east-1") @@ -1097,15 +1185,21 @@ def test_label_parameter_version_invalid_name(): ).should.throw( ClientError, "An error occurred (ParameterNotFound) when calling the LabelParameterVersion operation: " - "Parameter test not found." + "Parameter test not found.", ) + @mock_ssm def test_label_parameter_version_invalid_parameter_version(): client = boto3.client("ssm", region_name="us-east-1") test_parameter_name = "test" - client.put_parameter(Name=test_parameter_name, Description="A test parameter", Value="value", Type="String") + client.put_parameter( + Name=test_parameter_name, + Description="A test parameter", + Value="value", + Type="String", + ) response = client.label_parameter_version.when.called_with( Name=test_parameter_name, Labels=["test-label"], ParameterVersion=5 @@ -1113,37 +1207,52 @@ def test_label_parameter_version_invalid_parameter_version(): ClientError, "An error occurred (ParameterVersionNotFound) when calling the LabelParameterVersion operation: " "Systems Manager could not find version 5 of test. " - "Verify the version and try again." + "Verify the version and try again.", ) + @mock_ssm def test_label_parameter_version_invalid_label(): client = boto3.client("ssm", region_name="us-east-1") test_parameter_name = "test" - client.put_parameter(Name=test_parameter_name, Description="A test parameter", Value="value", Type="String") - response = client.label_parameter_version(Name=test_parameter_name, ParameterVersion=1, Labels=["awsabc"]) + client.put_parameter( + Name=test_parameter_name, + Description="A test parameter", + Value="value", + Type="String", + ) + response = client.label_parameter_version( + Name=test_parameter_name, ParameterVersion=1, Labels=["awsabc"] + ) response["InvalidLabels"].should.equal(["awsabc"]) - response = client.label_parameter_version(Name=test_parameter_name, ParameterVersion=1, Labels=["ssmabc"]) + response = client.label_parameter_version( + Name=test_parameter_name, ParameterVersion=1, Labels=["ssmabc"] + ) response["InvalidLabels"].should.equal(["ssmabc"]) - response = client.label_parameter_version(Name=test_parameter_name, ParameterVersion=1, Labels=["9abc"]) + response = client.label_parameter_version( + Name=test_parameter_name, ParameterVersion=1, Labels=["9abc"] + ) response["InvalidLabels"].should.equal(["9abc"]) - response = client.label_parameter_version(Name=test_parameter_name, ParameterVersion=1, Labels=["abc/123"]) + response = client.label_parameter_version( + Name=test_parameter_name, ParameterVersion=1, Labels=["abc/123"] + ) response["InvalidLabels"].should.equal(["abc/123"]) client.label_parameter_version.when.called_with( - Name=test_parameter_name, ParameterVersion=1, Labels=["a"*101] + Name=test_parameter_name, ParameterVersion=1, Labels=["a" * 101] ).should.throw( ClientError, "1 validation error detected: " "Value '[%s]' at 'labels' failed to satisfy constraint: " "Member must satisfy constraint: " - "[Member must have length less than or equal to 100, Member must have length greater than or equal to 1]" % ("a"*101) + "[Member must have length less than or equal to 100, Member must have length greater than or equal to 1]" + % ("a" * 101), ) - + @mock_ssm def test_get_parameter_history_with_label(): @@ -1161,7 +1270,9 @@ def test_get_parameter_history_with_label(): Overwrite=True, ) - client.label_parameter_version(Name=test_parameter_name, ParameterVersion=1, Labels=test_labels) + client.label_parameter_version( + Name=test_parameter_name, ParameterVersion=1, Labels=test_labels + ) response = client.get_parameter_history(Name=test_parameter_name) parameters_response = response["Parameters"] @@ -1177,6 +1288,7 @@ def test_get_parameter_history_with_label(): len(parameters_response).should.equal(3) + @mock_ssm def test_get_parameter_history_with_label_non_latest(): client = boto3.client("ssm", region_name="us-east-1") @@ -1193,7 +1305,9 @@ def test_get_parameter_history_with_label_non_latest(): Overwrite=True, ) - client.label_parameter_version(Name=test_parameter_name, ParameterVersion=2, Labels=test_labels) + client.label_parameter_version( + Name=test_parameter_name, ParameterVersion=2, Labels=test_labels + ) response = client.get_parameter_history(Name=test_parameter_name) parameters_response = response["Parameters"] @@ -1209,6 +1323,7 @@ def test_get_parameter_history_with_label_non_latest(): len(parameters_response).should.equal(3) + @mock_ssm def test_get_parameter_history_with_label_latest_assumed(): client = boto3.client("ssm", region_name="us-east-1") @@ -1241,6 +1356,7 @@ def test_get_parameter_history_with_label_latest_assumed(): len(parameters_response).should.equal(3) + @mock_ssm def test_get_parameter_history_missing_parameter(): client = boto3.client("ssm", region_name="us-east-1") From 2e0bc1aff302608680bea62c424b4b2cc8e9af84 Mon Sep 17 00:00:00 2001 From: Asher Foa <1268088+asherf@users.noreply.github.com> Date: Wed, 11 Mar 2020 12:45:01 -0700 Subject: [PATCH 16/32] Loosen jinja2 requirement. This allows repos consuming moto to use the latest jinaj2 2.11.x patched version (currently 2.11.1) w/o breaking moto's python 2 support. See https://github.com/spulec/moto/pull/2776 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9bb7cf522..193761191 100755 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def get_version(): install_requires = [ "setuptools==44.0.0", - "Jinja2==2.11.0", + "Jinja2<3.0.0,>=2.10.1", "boto>=2.36.0", "boto3>=1.9.201", "botocore>=1.12.201", From 649b497f71cce95a6474a3ff6f3c9c3339efb68f Mon Sep 17 00:00:00 2001 From: Laurence de Bruxelles Date: Thu, 12 Mar 2020 09:38:02 +0000 Subject: [PATCH 17/32] Loosen idna requirement requests 2.23.0 allows idna<3 [1] [1] psf/requests@c46f55b --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 193761191..79b9875ee 100755 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ install_requires = [ "jsondiff>=1.1.2", "aws-xray-sdk!=0.96,>=0.93", "responses>=0.9.0", - "idna<2.9,>=2.5", + "idna<3,>=2.5", "cfn-lint>=0.4.0", "sshpubkeys>=3.1.0,<4.0", "zipp==0.6.0", From ad5314ad0686e76353eed6415267ae355bd4d795 Mon Sep 17 00:00:00 2001 From: mzgierski Date: Wed, 15 May 2019 17:04:31 +0200 Subject: [PATCH 18/32] Enable the test that AWS-Batch describe_jobs fails at. --- tests/test_batch/test_batch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_batch/test_batch.py b/tests/test_batch/test_batch.py index 141d6b343..de4b349e0 100644 --- a/tests/test_batch/test_batch.py +++ b/tests/test_batch/test_batch.py @@ -692,7 +692,8 @@ def test_submit_job_by_name(): # SLOW TESTS -@expected_failure + +# @expected_failure @mock_logs @mock_ec2 @mock_ecs From bfeaf73109c8f5c1712badb8f785022e357bf95b Mon Sep 17 00:00:00 2001 From: mzgierski Date: Wed, 15 May 2019 17:05:45 +0200 Subject: [PATCH 19/32] Fix the AWS-Batch describe_jobs problem with not-yet-started jobs. --- moto/batch/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moto/batch/models.py b/moto/batch/models.py index fc35f2997..a5986b7a4 100644 --- a/moto/batch/models.py +++ b/moto/batch/models.py @@ -338,10 +338,11 @@ class Job(threading.Thread, BaseModel): "jobId": self.job_id, "jobName": self.job_name, "jobQueue": self.job_queue.arn, - "startedAt": datetime2int(self.job_started_at), "status": self.job_state, "dependsOn": [], } + if result['status'] not in ['SUBMITTED', 'PENDING', 'RUNNABLE', 'STARTING']: + result['startedAt'] = datetime2int(self.job_started_at) if self.job_stopped: result["stoppedAt"] = datetime2int(self.job_stopped_at) result["container"] = {} From bb5a54ca4b42a77f98c276009552087a3926a031 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Thu, 12 Mar 2020 13:37:46 +0000 Subject: [PATCH 20/32] Batch - Fix tests --- moto/batch/models.py | 6 ++++-- tests/test_batch/test_batch.py | 37 ++++++++-------------------------- 2 files changed, 12 insertions(+), 31 deletions(-) diff --git a/moto/batch/models.py b/moto/batch/models.py index a5986b7a4..08f4cbdb2 100644 --- a/moto/batch/models.py +++ b/moto/batch/models.py @@ -301,7 +301,7 @@ class Job(threading.Thread, BaseModel): self.job_name = name self.job_id = str(uuid.uuid4()) self.job_definition = job_def - self.container_overrides = container_overrides + self.container_overrides = container_overrides or {} self.job_queue = job_queue self.job_state = "SUBMITTED" # One of SUBMITTED | PENDING | RUNNABLE | STARTING | RUNNING | SUCCEEDED | FAILED self.job_queue.jobs.append(self) @@ -317,6 +317,7 @@ class Job(threading.Thread, BaseModel): self.docker_client = docker.from_env() self._log_backend = log_backend + self.log_stream_name = None # Unfortunately mocking replaces this method w/o fallback enabled, so we # need to replace it if we detect it's been mocked @@ -504,7 +505,8 @@ class Job(threading.Thread, BaseModel): for line in logs_stdout + logs_stderr: date, line = line.split(" ", 1) date = dateutil.parser.parse(date) - date = int(date.timestamp()) + # TODO: Replace with int(date.timestamp()) once we yeet Python2 out of the window + date = int((time.mktime(date.timetuple()) + date.microsecond / 1000000.0)) logs.append({"timestamp": date, "message": line.strip()}) # Send to cloudwatch diff --git a/tests/test_batch/test_batch.py b/tests/test_batch/test_batch.py index de4b349e0..6eedf452c 100644 --- a/tests/test_batch/test_batch.py +++ b/tests/test_batch/test_batch.py @@ -10,17 +10,6 @@ import functools import nose -def expected_failure(test): - @functools.wraps(test) - def inner(*args, **kwargs): - try: - test(*args, **kwargs) - except Exception as err: - raise nose.SkipTest - - return inner - - DEFAULT_REGION = "eu-central-1" @@ -693,7 +682,6 @@ def test_submit_job_by_name(): # SLOW TESTS -# @expected_failure @mock_logs @mock_ec2 @mock_ecs @@ -721,13 +709,13 @@ def test_submit_job(): queue_arn = resp["jobQueueArn"] resp = batch_client.register_job_definition( - jobDefinitionName="sleep10", + jobDefinitionName="sayhellotomylittlefriend", type="container", containerProperties={ - "image": "busybox", + "image": "busybox:latest", "vcpus": 1, "memory": 128, - "command": ["sleep", "10"], + "command": ["echo", "hello"], }, ) job_def_arn = resp["jobDefinitionArn"] @@ -741,13 +729,6 @@ def test_submit_job(): while datetime.datetime.now() < future: resp = batch_client.describe_jobs(jobs=[job_id]) - print( - "{0}:{1} {2}".format( - resp["jobs"][0]["jobName"], - resp["jobs"][0]["jobId"], - resp["jobs"][0]["status"], - ) - ) if resp["jobs"][0]["status"] == "FAILED": raise RuntimeError("Batch job failed") @@ -764,10 +745,9 @@ def test_submit_job(): resp = logs_client.get_log_events( logGroupName="/aws/batch/job", logStreamName=ls_name ) - len(resp["events"]).should.be.greater_than(5) + [event['message'] for event in resp["events"]].should.equal(['hello']) -@expected_failure @mock_logs @mock_ec2 @mock_ecs @@ -795,13 +775,13 @@ def test_list_jobs(): queue_arn = resp["jobQueueArn"] resp = batch_client.register_job_definition( - jobDefinitionName="sleep10", + jobDefinitionName="sleep5", type="container", containerProperties={ - "image": "busybox", + "image": "busybox:latest", "vcpus": 1, "memory": 128, - "command": ["sleep", "10"], + "command": ["sleep", "5"], }, ) job_def_arn = resp["jobDefinitionArn"] @@ -844,7 +824,6 @@ def test_list_jobs(): len(resp_finished_jobs2["jobSummaryList"]).should.equal(2) -@expected_failure @mock_logs @mock_ec2 @mock_ecs @@ -875,7 +854,7 @@ def test_terminate_job(): jobDefinitionName="sleep10", type="container", containerProperties={ - "image": "busybox", + "image": "busybox:latest", "vcpus": 1, "memory": 128, "command": ["sleep", "10"], From 1b031aeeb0a2816bc153d64d73d47251ec642465 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Thu, 12 Mar 2020 14:07:34 +0000 Subject: [PATCH 21/32] Linting --- moto/batch/models.py | 8 +++++--- tests/test_batch/test_batch.py | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/moto/batch/models.py b/moto/batch/models.py index 08f4cbdb2..95ad64789 100644 --- a/moto/batch/models.py +++ b/moto/batch/models.py @@ -342,8 +342,8 @@ class Job(threading.Thread, BaseModel): "status": self.job_state, "dependsOn": [], } - if result['status'] not in ['SUBMITTED', 'PENDING', 'RUNNABLE', 'STARTING']: - result['startedAt'] = datetime2int(self.job_started_at) + if result["status"] not in ["SUBMITTED", "PENDING", "RUNNABLE", "STARTING"]: + result["startedAt"] = datetime2int(self.job_started_at) if self.job_stopped: result["stoppedAt"] = datetime2int(self.job_stopped_at) result["container"] = {} @@ -506,7 +506,9 @@ class Job(threading.Thread, BaseModel): date, line = line.split(" ", 1) date = dateutil.parser.parse(date) # TODO: Replace with int(date.timestamp()) once we yeet Python2 out of the window - date = int((time.mktime(date.timetuple()) + date.microsecond / 1000000.0)) + date = int( + (time.mktime(date.timetuple()) + date.microsecond / 1000000.0) + ) logs.append({"timestamp": date, "message": line.strip()}) # Send to cloudwatch diff --git a/tests/test_batch/test_batch.py b/tests/test_batch/test_batch.py index 6eedf452c..4b75fb857 100644 --- a/tests/test_batch/test_batch.py +++ b/tests/test_batch/test_batch.py @@ -682,6 +682,7 @@ def test_submit_job_by_name(): # SLOW TESTS + @mock_logs @mock_ec2 @mock_ecs @@ -745,7 +746,7 @@ def test_submit_job(): resp = logs_client.get_log_events( logGroupName="/aws/batch/job", logStreamName=ls_name ) - [event['message'] for event in resp["events"]].should.equal(['hello']) + [event["message"] for event in resp["events"]].should.equal(["hello"]) @mock_logs From b74625db0cf6676bd57cd09e610a202fe176117d Mon Sep 17 00:00:00 2001 From: Tomoya Iwata Date: Sun, 13 Jan 2019 17:38:38 +0900 Subject: [PATCH 22/32] add support for dynamodb transact_get_items --- IMPLEMENTATION_COVERAGE.md | 4 +- moto/dynamodb2/responses.py | 69 ++++++ tests/test_dynamodb2/test_dynamodb.py | 308 +++++++++++++++++++++++++- 3 files changed, 378 insertions(+), 3 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index a22cc3bfb..705618524 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -2237,7 +2237,7 @@ - [ ] verify_trust ## dynamodb -17% implemented +24% implemented - [ ] batch_get_item - [ ] batch_write_item - [ ] create_backup @@ -2268,7 +2268,7 @@ - [ ] restore_table_to_point_in_time - [X] scan - [ ] tag_resource -- [ ] transact_get_items +- [X] transact_get_items - [ ] transact_write_items - [ ] untag_resource - [ ] update_continuous_backups diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index d3767c3fd..c9b526121 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -10,6 +10,9 @@ from .exceptions import InvalidIndexNameError, InvalidUpdateExpression, ItemSize from .models import dynamodb_backends, dynamo_json_dump +TRANSACTION_MAX_ITEMS = 10 + + def has_empty_keys_or_values(_dict): if _dict == "": return True @@ -828,3 +831,69 @@ class DynamoHandler(BaseResponse): ttl_spec = self.dynamodb_backend.describe_ttl(name) return json.dumps({"TimeToLiveDescription": ttl_spec}) + + def transact_get_items(self): + transact_items = self.body['TransactItems'] + responses = list() + + if len(transact_items) > TRANSACTION_MAX_ITEMS: + msg = "1 validation error detected: Value '[" + err_list = list() + request_id = 268435456 + for _ in transact_items: + request_id += 1 + hex_request_id = format(request_id, 'x') + err_list.append('com.amazonaws.dynamodb.v20120810.TransactGetItem@%s' % hex_request_id) + msg += ', '.join(err_list) + msg += "'] at 'transactItems' failed to satisfy constraint: " \ + "Member must have length less than or equal to %s" % TRANSACTION_MAX_ITEMS + + return self.error('ValidationException', msg) + + dedup_list = [i for n, i in enumerate(transact_items) if i not in transact_items[n + 1:]] + if len(transact_items) != len(dedup_list): + er = 'com.amazon.coral.validate#ValidationException' + return self.error(er, 'Transaction request cannot include multiple operations on one item') + + ret_consumed_capacity = self.body.get('ReturnConsumedCapacity', 'NONE') + consumed_capacity = dict() + + for transact_item in transact_items: + + table_name = transact_item['Get']['TableName'] + key = transact_item['Get']['Key'] + try: + item = self.dynamodb_backend.get_item(table_name, key) + except ValueError as e: + er = 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException' + return self.error(er, 'Requested resource not found') + + if not item: + continue + + item_describe = item.describe_attrs(False) + responses.append(item_describe) + + table_capacity = consumed_capacity.get(table_name, {}) + table_capacity['TableName'] = table_name + capacity_units = table_capacity.get('CapacityUnits', 0) + 2.0 + table_capacity['CapacityUnits'] = capacity_units + read_capacity_units = table_capacity.get('ReadCapacityUnits', 0) + 2.0 + table_capacity['ReadCapacityUnits'] = read_capacity_units + consumed_capacity[table_name] = table_capacity + + if ret_consumed_capacity == 'INDEXES': + table_capacity['Table'] = { + 'CapacityUnits': capacity_units, + 'ReadCapacityUnits': read_capacity_units + } + + result = dict() + result.update({ + 'Responses': responses}) + if ret_consumed_capacity != 'NONE': + result.update({ + 'ConsumedCapacity': [v for v in consumed_capacity.values()] + }) + + return dynamo_json_dump(result) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 428b58f81..e439eeeb9 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -6,8 +6,9 @@ import six import boto import boto3 from boto3.dynamodb.conditions import Attr, Key -import sure # noqa +import re import requests +import sure # noqa from moto import mock_dynamodb2, mock_dynamodb2_deprecated from moto.dynamodb2 import dynamodb_backend2, dynamodb_backends2 from boto.exception import JSONResponseError @@ -3792,3 +3793,308 @@ def test_query_catches_when_no_filters(): ex.exception.response["Error"]["Message"].should.equal( "Either KeyConditions or QueryFilter should be present" ) + + +@mock_dynamodb2 +def test_invalid_transact_get_items(): + + dynamodb = boto3.resource('dynamodb', region_name='us-east-1') + dynamodb.create_table( + TableName='test1', + KeySchema=[{'AttributeName': 'id', 'KeyType': 'HASH'}], + AttributeDefinitions=[{'AttributeName': 'id', 'AttributeType': 'S'}], + ProvisionedThroughput={'ReadCapacityUnits': 5, 'WriteCapacityUnits': 5} + ) + table = dynamodb.Table('test1') + table.put_item(Item={ + 'id': '1', + 'val': '1', + }) + + table.put_item(Item={ + 'id': '1', + 'val': '2', + }) + + client = boto3.client('dynamodb', region_name='us-east-1') + + with assert_raises(ClientError) as ex: + client.transact_get_items(TransactItems=[ + { + 'Get': { + 'Key': { + 'id': {'S': '1'}, + }, + 'TableName': 'test1' + } + }, + { + 'Get': { + 'Key': { + 'id': {'S': '1'}, + }, + 'TableName': 'test1' + } + } + ]) + + ex.exception.response['Error']['Code'].should.equal('ValidationException') + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(400) + ex.exception.response['Error']['Message'].should.equal( + 'Transaction request cannot include multiple operations on one item' + ) + + with assert_raises(ClientError) as ex: + client.transact_get_items(TransactItems=[ + {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}}, + {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}}, + {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}}, + {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}}, + {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}}, + {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}}, + {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}}, + {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}}, + {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}}, + {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}}, + {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}}, + ]) + + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(400) + ex.exception.response['Error']['Message'].should.match( + r'failed to satisfy constraint: Member must have length less than or equal to 10', re.I + ) + + with assert_raises(ClientError) as ex: + client.transact_get_items(TransactItems=[ + { + 'Get': { + 'Key': { + 'id': {'S': '1'}, + }, + 'TableName': 'test1' + } + }, + { + 'Get': { + 'Key': { + 'id': {'S': '1'}, + }, + 'TableName': 'non_exists_table' + } + } + ]) + + ex.exception.response['Error']['Code'].should.equal('ResourceNotFoundException') + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(400) + ex.exception.response['Error']['Message'].should.equal( + 'Requested resource not found' + ) + + +@mock_dynamodb2 +def test_valid_transact_get_items(): + dynamodb = boto3.resource('dynamodb', region_name='us-east-1') + dynamodb.create_table( + TableName='test1', + KeySchema=[ + {'AttributeName': 'id', 'KeyType': 'HASH'}, + {'AttributeName': 'sort_key', 'KeyType': 'RANGE'}, + ], + AttributeDefinitions=[ + {'AttributeName': 'id', 'AttributeType': 'S'}, + {'AttributeName': 'sort_key', 'AttributeType': 'S'}, + ], + ProvisionedThroughput={'ReadCapacityUnits': 5, 'WriteCapacityUnits': 5} + ) + table1 = dynamodb.Table('test1') + table1.put_item(Item={ + 'id': '1', + 'sort_key': '1', + }) + + table1.put_item(Item={ + 'id': '1', + 'sort_key': '2', + }) + + dynamodb.create_table( + TableName='test2', + KeySchema=[ + {'AttributeName': 'id', 'KeyType': 'HASH'}, + {'AttributeName': 'sort_key', 'KeyType': 'RANGE'}, + ], + AttributeDefinitions=[ + {'AttributeName': 'id', 'AttributeType': 'S'}, + {'AttributeName': 'sort_key', 'AttributeType': 'S'}, + ], + ProvisionedThroughput={'ReadCapacityUnits': 5, 'WriteCapacityUnits': 5} + ) + table2 = dynamodb.Table('test2') + table2.put_item(Item={ + 'id': '1', + 'sort_key': '1', + }) + + client = boto3.client('dynamodb', region_name='us-east-1') + res = client.transact_get_items(TransactItems=[ + { + 'Get': { + 'Key': { + 'id': {'S': '1'}, + 'sort_key': {'S': '1'} + }, + 'TableName': 'test1' + } + }, + { + 'Get': { + 'Key': { + 'id': {'S': 'non_exists_key'}, + 'sort_key': {'S': '2'} + }, + 'TableName': 'test1' + } + } + ]) + res['Responses'][0]['Item'].should.equal({ + 'id': {'S': '1'}, + 'sort_key': {'S': '1'} + }) + len(res['Responses']).should.equal(1) + + res = client.transact_get_items(TransactItems=[ + { + 'Get': { + 'Key': { + 'id': {'S': '1'}, + 'sort_key': {'S': '1'} + }, + 'TableName': 'test1' + } + }, + { + 'Get': { + 'Key': { + 'id': {'S': '1'}, + 'sort_key': {'S': '2'} + }, + 'TableName': 'test1' + } + }, + { + 'Get': { + 'Key': { + 'id': {'S': '1'}, + 'sort_key': {'S': '1'} + }, + 'TableName': 'test2' + } + }, + ]) + + res['Responses'][0]['Item'].should.equal({ + 'id': {'S': '1'}, + 'sort_key': {'S': '1'} + }) + + res['Responses'][1]['Item'].should.equal({ + 'id': {'S': '1'}, + 'sort_key': {'S': '2'} + }) + + res['Responses'][2]['Item'].should.equal({ + 'id': {'S': '1'}, + 'sort_key': {'S': '1'} + }) + + res = client.transact_get_items(TransactItems=[ + { + 'Get': { + 'Key': { + 'id': {'S': '1'}, + 'sort_key': {'S': '1'} + }, + 'TableName': 'test1' + } + }, + { + 'Get': { + 'Key': { + 'id': {'S': '1'}, + 'sort_key': {'S': '2'} + }, + 'TableName': 'test1' + } + }, + { + 'Get': { + 'Key': { + 'id': {'S': '1'}, + 'sort_key': {'S': '1'} + }, + 'TableName': 'test2' + } + }, + ], ReturnConsumedCapacity='TOTAL') + + res['ConsumedCapacity'][0].should.equal({ + 'TableName': 'test1', + 'CapacityUnits': 4.0, + 'ReadCapacityUnits': 4.0 + }) + + res['ConsumedCapacity'][1].should.equal({ + 'TableName': 'test2', + 'CapacityUnits': 2.0, + 'ReadCapacityUnits': 2.0 + }) + + res = client.transact_get_items(TransactItems=[ + { + 'Get': { + 'Key': { + 'id': {'S': '1'}, + 'sort_key': {'S': '1'} + }, + 'TableName': 'test1' + } + }, + { + 'Get': { + 'Key': { + 'id': {'S': '1'}, + 'sort_key': {'S': '2'} + }, + 'TableName': 'test1' + } + }, + { + 'Get': { + 'Key': { + 'id': {'S': '1'}, + 'sort_key': {'S': '1'} + }, + 'TableName': 'test2' + } + }, + ], ReturnConsumedCapacity='INDEXES') + + res['ConsumedCapacity'][0].should.equal({ + 'TableName': 'test1', + 'CapacityUnits': 4.0, + 'ReadCapacityUnits': 4.0, + 'Table': { + 'CapacityUnits': 4.0, + 'ReadCapacityUnits': 4.0, + } + }) + + res['ConsumedCapacity'][1].should.equal({ + 'TableName': 'test2', + 'CapacityUnits': 2.0, + 'ReadCapacityUnits': 2.0, + 'Table': { + 'CapacityUnits': 2.0, + 'ReadCapacityUnits': 2.0, + } + }) From 5a7da61833222c603113c5be03d3e117494efb8b Mon Sep 17 00:00:00 2001 From: Tomoya Iwata Date: Sun, 13 Jan 2019 18:32:27 +0900 Subject: [PATCH 23/32] remove unused local variable --- moto/dynamodb2/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index c9b526121..90cbcedda 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -864,7 +864,7 @@ class DynamoHandler(BaseResponse): key = transact_item['Get']['Key'] try: item = self.dynamodb_backend.get_item(table_name, key) - except ValueError as e: + except ValueError: er = 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException' return self.error(er, 'Requested resource not found') From caebe222d7846b87139a9eec08a1376db9c4d0d6 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Thu, 12 Mar 2020 14:24:53 +0000 Subject: [PATCH 24/32] DynamoDB - Transact_get_items - Remove error condition --- moto/dynamodb2/responses.py | 7 +---- tests/test_dynamodb2/test_dynamodb.py | 40 ++------------------------- 2 files changed, 3 insertions(+), 44 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 90cbcedda..a5e465a1a 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -10,7 +10,7 @@ from .exceptions import InvalidIndexNameError, InvalidUpdateExpression, ItemSize from .models import dynamodb_backends, dynamo_json_dump -TRANSACTION_MAX_ITEMS = 10 +TRANSACTION_MAX_ITEMS = 25 def has_empty_keys_or_values(_dict): @@ -850,11 +850,6 @@ class DynamoHandler(BaseResponse): return self.error('ValidationException', msg) - dedup_list = [i for n, i in enumerate(transact_items) if i not in transact_items[n + 1:]] - if len(transact_items) != len(dedup_list): - er = 'com.amazon.coral.validate#ValidationException' - return self.error(er, 'Transaction request cannot include multiple operations on one item') - ret_consumed_capacity = self.body.get('ReturnConsumedCapacity', 'NONE') consumed_capacity = dict() diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index e439eeeb9..cfe071f44 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -3820,48 +3820,12 @@ def test_invalid_transact_get_items(): with assert_raises(ClientError) as ex: client.transact_get_items(TransactItems=[ - { - 'Get': { - 'Key': { - 'id': {'S': '1'}, - }, - 'TableName': 'test1' - } - }, - { - 'Get': { - 'Key': { - 'id': {'S': '1'}, - }, - 'TableName': 'test1' - } - } - ]) - - ex.exception.response['Error']['Code'].should.equal('ValidationException') - ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(400) - ex.exception.response['Error']['Message'].should.equal( - 'Transaction request cannot include multiple operations on one item' - ) - - with assert_raises(ClientError) as ex: - client.transact_get_items(TransactItems=[ - {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}}, - {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}}, - {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}}, - {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}}, - {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}}, - {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}}, - {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}}, - {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}}, - {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}}, - {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}}, - {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}}, + {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}} for i in range(26) ]) ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(400) ex.exception.response['Error']['Message'].should.match( - r'failed to satisfy constraint: Member must have length less than or equal to 10', re.I + r'failed to satisfy constraint: Member must have length less than or equal to 25', re.I ) with assert_raises(ClientError) as ex: From 71d3941daf09a9276dd29f3692585200bd6ea7fa Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Thu, 12 Mar 2020 14:26:23 +0000 Subject: [PATCH 25/32] Linting --- moto/dynamodb2/responses.py | 57 ++-- tests/test_dynamodb2/test_dynamodb.py | 402 +++++++++++--------------- 2 files changed, 204 insertions(+), 255 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index a5e465a1a..3d25c7e49 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -833,7 +833,7 @@ class DynamoHandler(BaseResponse): return json.dumps({"TimeToLiveDescription": ttl_spec}) def transact_get_items(self): - transact_items = self.body['TransactItems'] + transact_items = self.body["TransactItems"] responses = list() if len(transact_items) > TRANSACTION_MAX_ITEMS: @@ -842,26 +842,32 @@ class DynamoHandler(BaseResponse): request_id = 268435456 for _ in transact_items: request_id += 1 - hex_request_id = format(request_id, 'x') - err_list.append('com.amazonaws.dynamodb.v20120810.TransactGetItem@%s' % hex_request_id) - msg += ', '.join(err_list) - msg += "'] at 'transactItems' failed to satisfy constraint: " \ - "Member must have length less than or equal to %s" % TRANSACTION_MAX_ITEMS + hex_request_id = format(request_id, "x") + err_list.append( + "com.amazonaws.dynamodb.v20120810.TransactGetItem@%s" + % hex_request_id + ) + msg += ", ".join(err_list) + msg += ( + "'] at 'transactItems' failed to satisfy constraint: " + "Member must have length less than or equal to %s" + % TRANSACTION_MAX_ITEMS + ) - return self.error('ValidationException', msg) + return self.error("ValidationException", msg) - ret_consumed_capacity = self.body.get('ReturnConsumedCapacity', 'NONE') + ret_consumed_capacity = self.body.get("ReturnConsumedCapacity", "NONE") consumed_capacity = dict() for transact_item in transact_items: - table_name = transact_item['Get']['TableName'] - key = transact_item['Get']['Key'] + table_name = transact_item["Get"]["TableName"] + key = transact_item["Get"]["Key"] try: item = self.dynamodb_backend.get_item(table_name, key) except ValueError: - er = 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException' - return self.error(er, 'Requested resource not found') + er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" + return self.error(er, "Requested resource not found") if not item: continue @@ -870,25 +876,22 @@ class DynamoHandler(BaseResponse): responses.append(item_describe) table_capacity = consumed_capacity.get(table_name, {}) - table_capacity['TableName'] = table_name - capacity_units = table_capacity.get('CapacityUnits', 0) + 2.0 - table_capacity['CapacityUnits'] = capacity_units - read_capacity_units = table_capacity.get('ReadCapacityUnits', 0) + 2.0 - table_capacity['ReadCapacityUnits'] = read_capacity_units + table_capacity["TableName"] = table_name + capacity_units = table_capacity.get("CapacityUnits", 0) + 2.0 + table_capacity["CapacityUnits"] = capacity_units + read_capacity_units = table_capacity.get("ReadCapacityUnits", 0) + 2.0 + table_capacity["ReadCapacityUnits"] = read_capacity_units consumed_capacity[table_name] = table_capacity - if ret_consumed_capacity == 'INDEXES': - table_capacity['Table'] = { - 'CapacityUnits': capacity_units, - 'ReadCapacityUnits': read_capacity_units + if ret_consumed_capacity == "INDEXES": + table_capacity["Table"] = { + "CapacityUnits": capacity_units, + "ReadCapacityUnits": read_capacity_units, } result = dict() - result.update({ - 'Responses': responses}) - if ret_consumed_capacity != 'NONE': - result.update({ - 'ConsumedCapacity': [v for v in consumed_capacity.values()] - }) + result.update({"Responses": responses}) + if ret_consumed_capacity != "NONE": + result.update({"ConsumedCapacity": [v for v in consumed_capacity.values()]}) return dynamo_json_dump(result) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index cfe071f44..f67711689 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -3798,267 +3798,213 @@ def test_query_catches_when_no_filters(): @mock_dynamodb2 def test_invalid_transact_get_items(): - dynamodb = boto3.resource('dynamodb', region_name='us-east-1') + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") dynamodb.create_table( - TableName='test1', - KeySchema=[{'AttributeName': 'id', 'KeyType': 'HASH'}], - AttributeDefinitions=[{'AttributeName': 'id', 'AttributeType': 'S'}], - ProvisionedThroughput={'ReadCapacityUnits': 5, 'WriteCapacityUnits': 5} + TableName="test1", + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table = dynamodb.Table("test1") + table.put_item( + Item={"id": "1", "val": "1",} ) - table = dynamodb.Table('test1') - table.put_item(Item={ - 'id': '1', - 'val': '1', - }) - table.put_item(Item={ - 'id': '1', - 'val': '2', - }) + table.put_item( + Item={"id": "1", "val": "2",} + ) - client = boto3.client('dynamodb', region_name='us-east-1') + client = boto3.client("dynamodb", region_name="us-east-1") with assert_raises(ClientError) as ex: - client.transact_get_items(TransactItems=[ - {'Get': {'Key': {'id': {'S': '1'}}, 'TableName': 'test1'}} for i in range(26) - ]) + client.transact_get_items( + TransactItems=[ + {"Get": {"Key": {"id": {"S": "1"}}, "TableName": "test1"}} + for i in range(26) + ] + ) - ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(400) - ex.exception.response['Error']['Message'].should.match( - r'failed to satisfy constraint: Member must have length less than or equal to 25', re.I + ex.exception.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.exception.response["Error"]["Message"].should.match( + r"failed to satisfy constraint: Member must have length less than or equal to 25", + re.I, ) with assert_raises(ClientError) as ex: - client.transact_get_items(TransactItems=[ - { - 'Get': { - 'Key': { - 'id': {'S': '1'}, - }, - 'TableName': 'test1' - } - }, - { - 'Get': { - 'Key': { - 'id': {'S': '1'}, - }, - 'TableName': 'non_exists_table' - } - } - ]) + client.transact_get_items( + TransactItems=[ + {"Get": {"Key": {"id": {"S": "1"},}, "TableName": "test1"}}, + {"Get": {"Key": {"id": {"S": "1"},}, "TableName": "non_exists_table"}}, + ] + ) - ex.exception.response['Error']['Code'].should.equal('ResourceNotFoundException') - ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(400) - ex.exception.response['Error']['Message'].should.equal( - 'Requested resource not found' + ex.exception.response["Error"]["Code"].should.equal("ResourceNotFoundException") + ex.exception.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.exception.response["Error"]["Message"].should.equal( + "Requested resource not found" ) @mock_dynamodb2 def test_valid_transact_get_items(): - dynamodb = boto3.resource('dynamodb', region_name='us-east-1') + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") dynamodb.create_table( - TableName='test1', + TableName="test1", KeySchema=[ - {'AttributeName': 'id', 'KeyType': 'HASH'}, - {'AttributeName': 'sort_key', 'KeyType': 'RANGE'}, + {"AttributeName": "id", "KeyType": "HASH"}, + {"AttributeName": "sort_key", "KeyType": "RANGE"}, ], AttributeDefinitions=[ - {'AttributeName': 'id', 'AttributeType': 'S'}, - {'AttributeName': 'sort_key', 'AttributeType': 'S'}, + {"AttributeName": "id", "AttributeType": "S"}, + {"AttributeName": "sort_key", "AttributeType": "S"}, ], - ProvisionedThroughput={'ReadCapacityUnits': 5, 'WriteCapacityUnits': 5} + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table1 = dynamodb.Table("test1") + table1.put_item( + Item={"id": "1", "sort_key": "1",} ) - table1 = dynamodb.Table('test1') - table1.put_item(Item={ - 'id': '1', - 'sort_key': '1', - }) - table1.put_item(Item={ - 'id': '1', - 'sort_key': '2', - }) + table1.put_item( + Item={"id": "1", "sort_key": "2",} + ) dynamodb.create_table( - TableName='test2', + TableName="test2", KeySchema=[ - {'AttributeName': 'id', 'KeyType': 'HASH'}, - {'AttributeName': 'sort_key', 'KeyType': 'RANGE'}, + {"AttributeName": "id", "KeyType": "HASH"}, + {"AttributeName": "sort_key", "KeyType": "RANGE"}, ], AttributeDefinitions=[ - {'AttributeName': 'id', 'AttributeType': 'S'}, - {'AttributeName': 'sort_key', 'AttributeType': 'S'}, + {"AttributeName": "id", "AttributeType": "S"}, + {"AttributeName": "sort_key", "AttributeType": "S"}, ], - ProvisionedThroughput={'ReadCapacityUnits': 5, 'WriteCapacityUnits': 5} + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table2 = dynamodb.Table("test2") + table2.put_item( + Item={"id": "1", "sort_key": "1",} ) - table2 = dynamodb.Table('test2') - table2.put_item(Item={ - 'id': '1', - 'sort_key': '1', - }) - client = boto3.client('dynamodb', region_name='us-east-1') - res = client.transact_get_items(TransactItems=[ + client = boto3.client("dynamodb", region_name="us-east-1") + res = client.transact_get_items( + TransactItems=[ + { + "Get": { + "Key": {"id": {"S": "1"}, "sort_key": {"S": "1"}}, + "TableName": "test1", + } + }, + { + "Get": { + "Key": {"id": {"S": "non_exists_key"}, "sort_key": {"S": "2"}}, + "TableName": "test1", + } + }, + ] + ) + res["Responses"][0]["Item"].should.equal({"id": {"S": "1"}, "sort_key": {"S": "1"}}) + len(res["Responses"]).should.equal(1) + + res = client.transact_get_items( + TransactItems=[ + { + "Get": { + "Key": {"id": {"S": "1"}, "sort_key": {"S": "1"}}, + "TableName": "test1", + } + }, + { + "Get": { + "Key": {"id": {"S": "1"}, "sort_key": {"S": "2"}}, + "TableName": "test1", + } + }, + { + "Get": { + "Key": {"id": {"S": "1"}, "sort_key": {"S": "1"}}, + "TableName": "test2", + } + }, + ] + ) + + res["Responses"][0]["Item"].should.equal({"id": {"S": "1"}, "sort_key": {"S": "1"}}) + + res["Responses"][1]["Item"].should.equal({"id": {"S": "1"}, "sort_key": {"S": "2"}}) + + res["Responses"][2]["Item"].should.equal({"id": {"S": "1"}, "sort_key": {"S": "1"}}) + + res = client.transact_get_items( + TransactItems=[ + { + "Get": { + "Key": {"id": {"S": "1"}, "sort_key": {"S": "1"}}, + "TableName": "test1", + } + }, + { + "Get": { + "Key": {"id": {"S": "1"}, "sort_key": {"S": "2"}}, + "TableName": "test1", + } + }, + { + "Get": { + "Key": {"id": {"S": "1"}, "sort_key": {"S": "1"}}, + "TableName": "test2", + } + }, + ], + ReturnConsumedCapacity="TOTAL", + ) + + res["ConsumedCapacity"][0].should.equal( + {"TableName": "test1", "CapacityUnits": 4.0, "ReadCapacityUnits": 4.0} + ) + + res["ConsumedCapacity"][1].should.equal( + {"TableName": "test2", "CapacityUnits": 2.0, "ReadCapacityUnits": 2.0} + ) + + res = client.transact_get_items( + TransactItems=[ + { + "Get": { + "Key": {"id": {"S": "1"}, "sort_key": {"S": "1"}}, + "TableName": "test1", + } + }, + { + "Get": { + "Key": {"id": {"S": "1"}, "sort_key": {"S": "2"}}, + "TableName": "test1", + } + }, + { + "Get": { + "Key": {"id": {"S": "1"}, "sort_key": {"S": "1"}}, + "TableName": "test2", + } + }, + ], + ReturnConsumedCapacity="INDEXES", + ) + + res["ConsumedCapacity"][0].should.equal( { - 'Get': { - 'Key': { - 'id': {'S': '1'}, - 'sort_key': {'S': '1'} - }, - 'TableName': 'test1' - } - }, - { - 'Get': { - 'Key': { - 'id': {'S': 'non_exists_key'}, - 'sort_key': {'S': '2'} - }, - 'TableName': 'test1' - } + "TableName": "test1", + "CapacityUnits": 4.0, + "ReadCapacityUnits": 4.0, + "Table": {"CapacityUnits": 4.0, "ReadCapacityUnits": 4.0,}, } - ]) - res['Responses'][0]['Item'].should.equal({ - 'id': {'S': '1'}, - 'sort_key': {'S': '1'} - }) - len(res['Responses']).should.equal(1) + ) - res = client.transact_get_items(TransactItems=[ + res["ConsumedCapacity"][1].should.equal( { - 'Get': { - 'Key': { - 'id': {'S': '1'}, - 'sort_key': {'S': '1'} - }, - 'TableName': 'test1' - } - }, - { - 'Get': { - 'Key': { - 'id': {'S': '1'}, - 'sort_key': {'S': '2'} - }, - 'TableName': 'test1' - } - }, - { - 'Get': { - 'Key': { - 'id': {'S': '1'}, - 'sort_key': {'S': '1'} - }, - 'TableName': 'test2' - } - }, - ]) - - res['Responses'][0]['Item'].should.equal({ - 'id': {'S': '1'}, - 'sort_key': {'S': '1'} - }) - - res['Responses'][1]['Item'].should.equal({ - 'id': {'S': '1'}, - 'sort_key': {'S': '2'} - }) - - res['Responses'][2]['Item'].should.equal({ - 'id': {'S': '1'}, - 'sort_key': {'S': '1'} - }) - - res = client.transact_get_items(TransactItems=[ - { - 'Get': { - 'Key': { - 'id': {'S': '1'}, - 'sort_key': {'S': '1'} - }, - 'TableName': 'test1' - } - }, - { - 'Get': { - 'Key': { - 'id': {'S': '1'}, - 'sort_key': {'S': '2'} - }, - 'TableName': 'test1' - } - }, - { - 'Get': { - 'Key': { - 'id': {'S': '1'}, - 'sort_key': {'S': '1'} - }, - 'TableName': 'test2' - } - }, - ], ReturnConsumedCapacity='TOTAL') - - res['ConsumedCapacity'][0].should.equal({ - 'TableName': 'test1', - 'CapacityUnits': 4.0, - 'ReadCapacityUnits': 4.0 - }) - - res['ConsumedCapacity'][1].should.equal({ - 'TableName': 'test2', - 'CapacityUnits': 2.0, - 'ReadCapacityUnits': 2.0 - }) - - res = client.transact_get_items(TransactItems=[ - { - 'Get': { - 'Key': { - 'id': {'S': '1'}, - 'sort_key': {'S': '1'} - }, - 'TableName': 'test1' - } - }, - { - 'Get': { - 'Key': { - 'id': {'S': '1'}, - 'sort_key': {'S': '2'} - }, - 'TableName': 'test1' - } - }, - { - 'Get': { - 'Key': { - 'id': {'S': '1'}, - 'sort_key': {'S': '1'} - }, - 'TableName': 'test2' - } - }, - ], ReturnConsumedCapacity='INDEXES') - - res['ConsumedCapacity'][0].should.equal({ - 'TableName': 'test1', - 'CapacityUnits': 4.0, - 'ReadCapacityUnits': 4.0, - 'Table': { - 'CapacityUnits': 4.0, - 'ReadCapacityUnits': 4.0, + "TableName": "test2", + "CapacityUnits": 2.0, + "ReadCapacityUnits": 2.0, + "Table": {"CapacityUnits": 2.0, "ReadCapacityUnits": 2.0,}, } - }) - - res['ConsumedCapacity'][1].should.equal({ - 'TableName': 'test2', - 'CapacityUnits': 2.0, - 'ReadCapacityUnits': 2.0, - 'Table': { - 'CapacityUnits': 2.0, - 'ReadCapacityUnits': 2.0, - } - }) + ) From 1409618b954a53ae4832a1212f18f957c1a7775c Mon Sep 17 00:00:00 2001 From: Justin Hipple Date: Wed, 11 Mar 2020 16:30:42 -0500 Subject: [PATCH 26/32] Fix a misleading error message AWSEvents.DescribeRule throws an error that references a rule named "test" rather than the specified rule name when a rule with the specified name does not exist. It has been fixed to reference the specified rule name. --- moto/events/responses.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/moto/events/responses.py b/moto/events/responses.py index 68c2114a6..c9931aabc 100644 --- a/moto/events/responses.py +++ b/moto/events/responses.py @@ -62,7 +62,9 @@ class EventsHandler(BaseResponse): rule = self.events_backend.describe_rule(name) if not rule: - return self.error("ResourceNotFoundException", "Rule test does not exist.") + return self.error( + "ResourceNotFoundException", "Rule " + name + " does not exist." + ) rule_dict = self._generate_rule_dict(rule) return json.dumps(rule_dict), self.response_headers From 374b623e1d50d0ef9632a9e9bac6efa4fadc81ec Mon Sep 17 00:00:00 2001 From: Asher Foa <1268088+asherf@users.noreply.github.com> Date: Thu, 12 Mar 2020 09:34:25 -0700 Subject: [PATCH 27/32] Fix some 'DeprecationWarning: invalid escape sequence' warnings and use str.format for string interpolation. I am seeing a lot of deperecation warnings when I use moto for my tests (running under pytest), so I figured I'll clean up some of them. --- moto/cloudformation/parsing.py | 8 ++++---- moto/core/models.py | 4 ++-- moto/core/utils.py | 2 +- moto/dynamodb2/comparisons.py | 20 +++++++++++--------- moto/ecr/models.py | 2 +- moto/s3/utils.py | 2 +- 6 files changed, 20 insertions(+), 18 deletions(-) diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index 34d96acc6..d7e15c7b4 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -196,13 +196,13 @@ def clean_json(resource_json, resources_map): ) else: fn_sub_value = clean_json(resource_json["Fn::Sub"], resources_map) - to_sub = re.findall('(?=\${)[^!^"]*?}', fn_sub_value) - literals = re.findall('(?=\${!)[^"]*?}', fn_sub_value) + to_sub = re.findall(r'(?=\${)[^!^"]*?}', fn_sub_value) + literals = re.findall(r'(?=\${!)[^"]*?}', fn_sub_value) for sub in to_sub: if "." in sub: cleaned_ref = clean_json( { - "Fn::GetAtt": re.findall('(?<=\${)[^"]*?(?=})', sub)[ + "Fn::GetAtt": re.findall(r'(?<=\${)[^"]*?(?=})', sub)[ 0 ].split(".") }, @@ -210,7 +210,7 @@ def clean_json(resource_json, resources_map): ) else: cleaned_ref = clean_json( - {"Ref": re.findall('(?<=\${)[^"]*?(?=})', sub)[0]}, + {"Ref": re.findall(r'(?<=\${)[^"]*?(?=})', sub)[0]}, resources_map, ) fn_sub_value = fn_sub_value.replace(sub, cleaned_ref) diff --git a/moto/core/models.py b/moto/core/models.py index 8ca74d5b5..73942c669 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -347,7 +347,7 @@ class BotocoreEventMockAWS(BaseMockAWS): responses_mock.add( CallbackResponse( method=method, - url=re.compile("https?://.+.amazonaws.com/.*"), + url=re.compile(r"https?://.+.amazonaws.com/.*"), callback=not_implemented_callback, stream=True, match_querystring=False, @@ -356,7 +356,7 @@ class BotocoreEventMockAWS(BaseMockAWS): botocore_mock.add( CallbackResponse( method=method, - url=re.compile("https?://.+.amazonaws.com/.*"), + url=re.compile(r"https?://.+.amazonaws.com/.*"), callback=not_implemented_callback, stream=True, match_querystring=False, diff --git a/moto/core/utils.py b/moto/core/utils.py index efad5679c..f61b040e0 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -95,7 +95,7 @@ def convert_regex_to_flask_path(url_path): match_name, match_pattern = reg.groups() return ''.format(match_pattern, match_name) - url_path = re.sub("\(\?P<(.*?)>(.*?)\)", caller, url_path) + url_path = re.sub(r"\(\?P<(.*?)>(.*?)\)", caller, url_path) if url_path.endswith("/?"): # Flask does own handling of trailing slashes diff --git a/moto/dynamodb2/comparisons.py b/moto/dynamodb2/comparisons.py index 29951d92d..d17ae6875 100644 --- a/moto/dynamodb2/comparisons.py +++ b/moto/dynamodb2/comparisons.py @@ -251,9 +251,9 @@ class ConditionExpressionParser: def _lex_one_node(self, remaining_expression): # TODO: Handle indexing like [1] - attribute_regex = "(:|#)?[A-z0-9\-_]+" + attribute_regex = r"(:|#)?[A-z0-9\-_]+" patterns = [ - (self.Nonterminal.WHITESPACE, re.compile("^ +")), + (self.Nonterminal.WHITESPACE, re.compile(r"^ +")), ( self.Nonterminal.COMPARATOR, re.compile( @@ -270,12 +270,14 @@ class ConditionExpressionParser: ( self.Nonterminal.OPERAND, re.compile( - "^" + attribute_regex + "(\." + attribute_regex + "|\[[0-9]\])*" + r"^{attribute_regex}(\.{attribute_regex}|\[[0-9]\])*".format( + attribute_regex=attribute_regex + ) ), ), - (self.Nonterminal.COMMA, re.compile("^,")), - (self.Nonterminal.LEFT_PAREN, re.compile("^\(")), - (self.Nonterminal.RIGHT_PAREN, re.compile("^\)")), + (self.Nonterminal.COMMA, re.compile(r"^,")), + (self.Nonterminal.LEFT_PAREN, re.compile(r"^\(")), + (self.Nonterminal.RIGHT_PAREN, re.compile(r"^\)")), ] for nonterminal, pattern in patterns: @@ -285,7 +287,7 @@ class ConditionExpressionParser: break else: # pragma: no cover raise ValueError( - "Cannot parse condition starting at: " + remaining_expression + "Cannot parse condition starting at:{}".format(remaining_expression) ) node = self.Node( @@ -318,7 +320,7 @@ class ConditionExpressionParser: for child in children: self._assert( child.nonterminal == self.Nonterminal.IDENTIFIER, - "Cannot use %s in path" % child.text, + "Cannot use {} in path".format(child.text), [node], ) output.append( @@ -392,7 +394,7 @@ class ConditionExpressionParser: elif name.startswith("["): # e.g. [123] if not name.endswith("]"): # pragma: no cover - raise ValueError("Bad path element %s" % name) + raise ValueError("Bad path element {}".format(name)) return self.Node( nonterminal=self.Nonterminal.IDENTIFIER, kind=self.Kind.LITERAL, diff --git a/moto/ecr/models.py b/moto/ecr/models.py index f84df79aa..88b058e1e 100644 --- a/moto/ecr/models.py +++ b/moto/ecr/models.py @@ -403,7 +403,7 @@ class ECRBackend(BaseBackend): # If we have a digest, is it valid? if "imageDigest" in image_id: - pattern = re.compile("^[0-9a-zA-Z_+\.-]+:[0-9a-fA-F]{64}") + pattern = re.compile(r"^[0-9a-zA-Z_+\.-]+:[0-9a-fA-F]{64}") if not pattern.match(image_id.get("imageDigest")): response["failures"].append( { diff --git a/moto/s3/utils.py b/moto/s3/utils.py index e22b6b860..6855c9b25 100644 --- a/moto/s3/utils.py +++ b/moto/s3/utils.py @@ -52,7 +52,7 @@ def parse_region_from_url(url): def metadata_from_headers(headers): metadata = {} - meta_regex = re.compile("^x-amz-meta-([a-zA-Z0-9\-_]+)$", flags=re.IGNORECASE) + meta_regex = re.compile(r"^x-amz-meta-([a-zA-Z0-9\-_]+)$", flags=re.IGNORECASE) for header, value in headers.items(): if isinstance(header, six.string_types): result = meta_regex.match(header) From 8bffff4620b7a7325a828e2c383e6f7b892b412c Mon Sep 17 00:00:00 2001 From: Tim Gatzemeier Date: Mon, 16 Mar 2020 18:48:29 +0100 Subject: [PATCH 28/32] set actions enabled in template on describe images this is to avoid errors with terraform relates to https://github.com/localstack/localstack/issues/2161 --- moto/cloudwatch/models.py | 4 ++++ moto/cloudwatch/responses.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py index 716a29633..bdba09930 100644 --- a/moto/cloudwatch/models.py +++ b/moto/cloudwatch/models.py @@ -67,6 +67,7 @@ class FakeAlarm(BaseModel): ok_actions, insufficient_data_actions, unit, + actions_enabled, ): self.name = name self.namespace = namespace @@ -80,6 +81,7 @@ class FakeAlarm(BaseModel): self.dimensions = [ Dimension(dimension["name"], dimension["value"]) for dimension in dimensions ] + self.actions_enabled = actions_enabled self.alarm_actions = alarm_actions self.ok_actions = ok_actions self.insufficient_data_actions = insufficient_data_actions @@ -215,6 +217,7 @@ class CloudWatchBackend(BaseBackend): ok_actions, insufficient_data_actions, unit, + actions_enabled, ): alarm = FakeAlarm( name, @@ -231,6 +234,7 @@ class CloudWatchBackend(BaseBackend): ok_actions, insufficient_data_actions, unit, + actions_enabled, ) self.alarms[name] = alarm return alarm diff --git a/moto/cloudwatch/responses.py b/moto/cloudwatch/responses.py index 7872e71fd..dbc9d8c5a 100644 --- a/moto/cloudwatch/responses.py +++ b/moto/cloudwatch/responses.py @@ -28,6 +28,7 @@ class CloudWatchResponse(BaseResponse): dimensions = self._get_list_prefix("Dimensions.member") alarm_actions = self._get_multi_param("AlarmActions.member") ok_actions = self._get_multi_param("OKActions.member") + actions_enabled = self._get_multi_param("ActionsEnabled") insufficient_data_actions = self._get_multi_param( "InsufficientDataActions.member" ) @@ -47,6 +48,7 @@ class CloudWatchResponse(BaseResponse): ok_actions, insufficient_data_actions, unit, + actions_enabled, ) template = self.response_template(PUT_METRIC_ALARM_TEMPLATE) return template.render(alarm=alarm) From 9d3ee116d3ec53c6ee20d2df0823ac09694e0f37 Mon Sep 17 00:00:00 2001 From: Tim Gatzemeier Date: Mon, 16 Mar 2020 20:14:41 +0100 Subject: [PATCH 29/32] add test case for actions_enabled field --- tests/test_cloudwatch/test_cloudwatch_boto3.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_cloudwatch/test_cloudwatch_boto3.py b/tests/test_cloudwatch/test_cloudwatch_boto3.py index 5bd9ed13d..1935a4181 100644 --- a/tests/test_cloudwatch/test_cloudwatch_boto3.py +++ b/tests/test_cloudwatch/test_cloudwatch_boto3.py @@ -128,11 +128,13 @@ def test_alarm_state(): len(resp["MetricAlarms"]).should.equal(1) resp["MetricAlarms"][0]["AlarmName"].should.equal("testalarm1") resp["MetricAlarms"][0]["StateValue"].should.equal("ALARM") + resp["MetricAlarms"][0]["ActionsEnabled"].should.equal("True") resp = client.describe_alarms(StateValue="OK") len(resp["MetricAlarms"]).should.equal(1) resp["MetricAlarms"][0]["AlarmName"].should.equal("testalarm2") resp["MetricAlarms"][0]["StateValue"].should.equal("OK") + resp["MetricAlarms"][0]["ActionsEnabled"].should.equal("True") # Just for sanity resp = client.describe_alarms() From 1fdb0e987dc882ba380d2665bd52f40dfd800e7e Mon Sep 17 00:00:00 2001 From: Tim Gatzemeier Date: Mon, 16 Mar 2020 21:45:18 +0100 Subject: [PATCH 30/32] get single param for actions enabled --- moto/cloudwatch/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/cloudwatch/responses.py b/moto/cloudwatch/responses.py index dbc9d8c5a..7993c9f06 100644 --- a/moto/cloudwatch/responses.py +++ b/moto/cloudwatch/responses.py @@ -28,7 +28,7 @@ class CloudWatchResponse(BaseResponse): dimensions = self._get_list_prefix("Dimensions.member") alarm_actions = self._get_multi_param("AlarmActions.member") ok_actions = self._get_multi_param("OKActions.member") - actions_enabled = self._get_multi_param("ActionsEnabled") + actions_enabled = self._get_param("ActionsEnabled") insufficient_data_actions = self._get_multi_param( "InsufficientDataActions.member" ) From 50974aa9b2d1c71b4be580e4236e04fac31c7e95 Mon Sep 17 00:00:00 2001 From: Tim Gatzemeier Date: Mon, 16 Mar 2020 21:45:29 +0100 Subject: [PATCH 31/32] add test cases to ensure actions enabled is correctly returned --- tests/test_cloudwatch/test_cloudwatch.py | 12 +++++++++--- tests/test_cloudwatch/test_cloudwatch_boto3.py | 6 +++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/test_cloudwatch/test_cloudwatch.py b/tests/test_cloudwatch/test_cloudwatch.py index dee8aa605..f86b57d54 100644 --- a/tests/test_cloudwatch/test_cloudwatch.py +++ b/tests/test_cloudwatch/test_cloudwatch.py @@ -101,15 +101,22 @@ def test_describe_alarms(): conn.create_alarm(alarm_fixture(name="nfoobaz", action="afoobaz")) conn.create_alarm(alarm_fixture(name="nbarfoo", action="abarfoo")) conn.create_alarm(alarm_fixture(name="nbazfoo", action="abazfoo")) - + + enabled = alarm_fixture(name="enabled1", action=["abarfoo"]) + enabled.add_alarm_action("arn:alarm") + conn.create_alarm(enabled) + alarms = conn.describe_alarms() - alarms.should.have.length_of(4) + alarms.should.have.length_of(5) alarms = conn.describe_alarms(alarm_name_prefix="nfoo") alarms.should.have.length_of(2) alarms = conn.describe_alarms(alarm_names=["nfoobar", "nbarfoo", "nbazfoo"]) alarms.should.have.length_of(3) alarms = conn.describe_alarms(action_prefix="afoo") alarms.should.have.length_of(2) + alarms = conn.describe_alarms(alarm_name_prefix="enabled") + alarms.should.have.length_of(1) + alarms[0].actions_enabled.should.equal("true") for alarm in conn.describe_alarms(): alarm.delete() @@ -117,7 +124,6 @@ def test_describe_alarms(): alarms = conn.describe_alarms() alarms.should.have.length_of(0) - @mock_cloudwatch_deprecated def test_get_metric_statistics(): conn = boto.connect_cloudwatch() diff --git a/tests/test_cloudwatch/test_cloudwatch_boto3.py b/tests/test_cloudwatch/test_cloudwatch_boto3.py index 1935a4181..6bef2b3f2 100644 --- a/tests/test_cloudwatch/test_cloudwatch_boto3.py +++ b/tests/test_cloudwatch/test_cloudwatch_boto3.py @@ -104,6 +104,7 @@ def test_alarm_state(): Statistic="Average", Threshold=2, ComparisonOperator="GreaterThanThreshold", + ActionsEnabled=True, ) client.put_metric_alarm( AlarmName="testalarm2", @@ -128,19 +129,18 @@ def test_alarm_state(): len(resp["MetricAlarms"]).should.equal(1) resp["MetricAlarms"][0]["AlarmName"].should.equal("testalarm1") resp["MetricAlarms"][0]["StateValue"].should.equal("ALARM") - resp["MetricAlarms"][0]["ActionsEnabled"].should.equal("True") + resp["MetricAlarms"][0]["ActionsEnabled"].should.equal(True) resp = client.describe_alarms(StateValue="OK") len(resp["MetricAlarms"]).should.equal(1) resp["MetricAlarms"][0]["AlarmName"].should.equal("testalarm2") resp["MetricAlarms"][0]["StateValue"].should.equal("OK") - resp["MetricAlarms"][0]["ActionsEnabled"].should.equal("True") + resp["MetricAlarms"][0]["ActionsEnabled"].should.equal(False) # Just for sanity resp = client.describe_alarms() len(resp["MetricAlarms"]).should.equal(2) - @mock_cloudwatch def test_put_metric_data_no_dimensions(): conn = boto3.client("cloudwatch", region_name="us-east-1") From 6e490a91909b6be7e371db6241a09291eb0d81da Mon Sep 17 00:00:00 2001 From: Tim Gatzemeier Date: Mon, 16 Mar 2020 21:58:50 +0100 Subject: [PATCH 32/32] make linter happy --- tests/test_cloudwatch/test_cloudwatch.py | 5 +++-- tests/test_cloudwatch/test_cloudwatch_boto3.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_cloudwatch/test_cloudwatch.py b/tests/test_cloudwatch/test_cloudwatch.py index f86b57d54..5a05a55e1 100644 --- a/tests/test_cloudwatch/test_cloudwatch.py +++ b/tests/test_cloudwatch/test_cloudwatch.py @@ -101,11 +101,11 @@ def test_describe_alarms(): conn.create_alarm(alarm_fixture(name="nfoobaz", action="afoobaz")) conn.create_alarm(alarm_fixture(name="nbarfoo", action="abarfoo")) conn.create_alarm(alarm_fixture(name="nbazfoo", action="abazfoo")) - + enabled = alarm_fixture(name="enabled1", action=["abarfoo"]) enabled.add_alarm_action("arn:alarm") conn.create_alarm(enabled) - + alarms = conn.describe_alarms() alarms.should.have.length_of(5) alarms = conn.describe_alarms(alarm_name_prefix="nfoo") @@ -124,6 +124,7 @@ def test_describe_alarms(): alarms = conn.describe_alarms() alarms.should.have.length_of(0) + @mock_cloudwatch_deprecated def test_get_metric_statistics(): conn = boto.connect_cloudwatch() diff --git a/tests/test_cloudwatch/test_cloudwatch_boto3.py b/tests/test_cloudwatch/test_cloudwatch_boto3.py index 6bef2b3f2..7fe144052 100644 --- a/tests/test_cloudwatch/test_cloudwatch_boto3.py +++ b/tests/test_cloudwatch/test_cloudwatch_boto3.py @@ -141,6 +141,7 @@ def test_alarm_state(): resp = client.describe_alarms() len(resp["MetricAlarms"]).should.equal(2) + @mock_cloudwatch def test_put_metric_data_no_dimensions(): conn = boto3.client("cloudwatch", region_name="us-east-1")