From 736f8b5a8f5e68c9b6225b99316ac930c6e5cdca Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Mon, 24 Feb 2020 08:24:14 +0000 Subject: [PATCH 01/17] Refactor - reuse logic that expects CW log message --- tests/test_awslambda/test_lambda.py | 65 ++++++++++------------------- 1 file changed, 22 insertions(+), 43 deletions(-) diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index 2bd8f4bb3..48539c0e6 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -1111,7 +1111,6 @@ def test_create_event_source_mapping(): @mock_lambda @mock_sqs def test_invoke_function_from_sqs(): - logs_conn = boto3.client("logs", region_name="us-east-1") sqs = boto3.resource("sqs", region_name="us-east-1") queue = sqs.create_queue(QueueName="test-sqs-queue1") @@ -1137,32 +1136,18 @@ def test_invoke_function_from_sqs(): sqs_client = boto3.client("sqs", region_name="us-east-1") sqs_client.send_message(QueueUrl=queue.url, MessageBody="test") - start = time.time() - while (time.time() - start) < 30: - result = logs_conn.describe_log_streams(logGroupName="/aws/lambda/testFunction") - log_streams = result.get("logStreams") - if not log_streams: - time.sleep(1) - continue - assert len(log_streams) == 1 - result = logs_conn.get_log_events( - logGroupName="/aws/lambda/testFunction", - logStreamName=log_streams[0]["logStreamName"], - ) - for event in result.get("events"): - if event["message"] == "get_test_zip_file3 success": - return - time.sleep(1) + expected_msg = "get_test_zip_file3 success" + log_group = "/aws/lambda/testFunction" + msg_showed_up = wait_for_log_msg(expected_msg, log_group) - assert False, "Test Failed" + assert msg_showed_up, "Message was not found in log_group, so sending an SQS message did not result in a successful Lambda execution" @mock_logs @mock_lambda @mock_dynamodb2 def test_invoke_function_from_dynamodb_put(): - logs_conn = boto3.client("logs", region_name="us-east-1") dynamodb = boto3.client("dynamodb", region_name="us-east-1") table_name = "table_with_stream" table = dynamodb.create_table( @@ -1197,32 +1182,18 @@ def test_invoke_function_from_dynamodb_put(): assert response["State"] == "Enabled" dynamodb.put_item(TableName=table_name, Item={"id": {"S": "item 1"}}) - start = time.time() - while (time.time() - start) < 30: - result = logs_conn.describe_log_streams(logGroupName="/aws/lambda/testFunction") - log_streams = result.get("logStreams") - if not log_streams: - time.sleep(1) - continue - assert len(log_streams) == 1 - result = logs_conn.get_log_events( - logGroupName="/aws/lambda/testFunction", - logStreamName=log_streams[0]["logStreamName"], - ) - for event in result.get("events"): - if event["message"] == "get_test_zip_file3 success": - return - time.sleep(1) + expected_msg = "get_test_zip_file3 success" + log_group = "/aws/lambda/testFunction" + msg_showed_up = wait_for_log_msg(expected_msg, log_group) - assert False, "Test Failed" + assert msg_showed_up, "Message was not found in log_group, so inserting DynamoDB did not result in a successful Lambda execution" @mock_logs @mock_lambda @mock_dynamodb2 def test_invoke_function_from_dynamodb_update(): - logs_conn = boto3.client("logs", region_name="us-east-1") dynamodb = boto3.client("dynamodb", region_name="us-east-1") table_name = "table_with_stream" table = dynamodb.create_table( @@ -1263,9 +1234,18 @@ def test_invoke_function_from_dynamodb_update(): ExpressionAttributeNames={"#attr": "new_attr"}, ExpressionAttributeValues={":val": {"S": "new_val"}}, ) + expected_msg = "get_test_zip_file3 success" + log_group = "/aws/lambda/testFunction" + msg_showed_up = wait_for_log_msg(expected_msg, log_group) + + assert msg_showed_up, "Message was not found in log_group, so updating DynamoDB did not result in a successful Lambda execution" + + +def wait_for_log_msg(expected_msg, log_group): + logs_conn = boto3.client("logs", region_name="us-east-1") start = time.time() while (time.time() - start) < 30: - result = logs_conn.describe_log_streams(logGroupName="/aws/lambda/testFunction") + result = logs_conn.describe_log_streams(logGroupName=log_group) log_streams = result.get("logStreams") if not log_streams: time.sleep(1) @@ -1273,15 +1253,14 @@ def test_invoke_function_from_dynamodb_update(): assert len(log_streams) == 1 result = logs_conn.get_log_events( - logGroupName="/aws/lambda/testFunction", + logGroupName=log_group, logStreamName=log_streams[0]["logStreamName"], ) for event in result.get("events"): - if event["message"] == "get_test_zip_file3 success": - return + if event["message"] == expected_msg: + return True time.sleep(1) - - assert False, "Test Failed" + return False @mock_logs From 038ff620b2e1e67fde707b38ee518e4a53088249 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Mon, 24 Feb 2020 09:28:52 +0000 Subject: [PATCH 02/17] DDB Streams - Bugfix where processed items are resend every time --- moto/awslambda/models.py | 2 +- tests/test_awslambda/test_lambda.py | 60 ++++++++++++++++++----------- 2 files changed, 39 insertions(+), 23 deletions(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index 939952d5e..9cdf2397c 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -981,7 +981,7 @@ class LambdaBackend(BaseBackend): ] } func = self._lambdas.get_arn(function_arn) - func.invoke(json.dumps(event), {}, {}) + return func.invoke(json.dumps(event), {}, {}) def list_tags(self, resource): return self.get_function_by_arn(resource).tags diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index 48539c0e6..eb8453e43 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -70,6 +70,7 @@ def lambda_handler(event, context): def get_test_zip_file3(): pfunc = """ def lambda_handler(event, context): + print("Nr_of_records("+str(len(event['Records']))+")") print("get_test_zip_file3 success") return event """ @@ -1139,9 +1140,13 @@ def test_invoke_function_from_sqs(): expected_msg = "get_test_zip_file3 success" log_group = "/aws/lambda/testFunction" - msg_showed_up = wait_for_log_msg(expected_msg, log_group) + msg_showed_up, all_logs = wait_for_log_msg(expected_msg, log_group) - assert msg_showed_up, "Message was not found in log_group, so sending an SQS message did not result in a successful Lambda execution" + assert msg_showed_up, ( + expected_msg + + " was not found after sending an SQS message. All logs: " + + all_logs + ) @mock_logs @@ -1185,9 +1190,11 @@ def test_invoke_function_from_dynamodb_put(): expected_msg = "get_test_zip_file3 success" log_group = "/aws/lambda/testFunction" - msg_showed_up = wait_for_log_msg(expected_msg, log_group) + msg_showed_up, all_logs = wait_for_log_msg(expected_msg, log_group) - assert msg_showed_up, "Message was not found in log_group, so inserting DynamoDB did not result in a successful Lambda execution" + assert msg_showed_up, ( + expected_msg + " was not found after a DDB insert. All logs: " + all_logs + ) @mock_logs @@ -1205,7 +1212,6 @@ def test_invoke_function_from_dynamodb_update(): "StreamViewType": "NEW_AND_OLD_IMAGES", }, ) - dynamodb.put_item(TableName=table_name, Item={"id": {"S": "item 1"}}) conn = boto3.client("lambda", region_name="us-east-1") func = conn.create_function( @@ -1220,13 +1226,17 @@ def test_invoke_function_from_dynamodb_update(): Publish=True, ) - response = conn.create_event_source_mapping( + conn.create_event_source_mapping( EventSourceArn=table["TableDescription"]["LatestStreamArn"], FunctionName=func["FunctionArn"], ) - assert response["EventSourceArn"] == table["TableDescription"]["LatestStreamArn"] - assert response["State"] == "Enabled" + dynamodb.put_item(TableName=table_name, Item={"id": {"S": "item 1"}}) + log_group = "/aws/lambda/testFunction" + expected_msg = "get_test_zip_file3 success" + msg_showed_up, all_logs = wait_for_log_msg(expected_msg, log_group) + assert "Nr_of_records(1)" in all_logs, "Only one item should be inserted" + dynamodb.update_item( TableName=table_name, Key={"id": {"S": "item 1"}}, @@ -1234,33 +1244,39 @@ def test_invoke_function_from_dynamodb_update(): ExpressionAttributeNames={"#attr": "new_attr"}, ExpressionAttributeValues={":val": {"S": "new_val"}}, ) - expected_msg = "get_test_zip_file3 success" - log_group = "/aws/lambda/testFunction" - msg_showed_up = wait_for_log_msg(expected_msg, log_group) + msg_showed_up, all_logs = wait_for_log_msg(expected_msg, log_group) - assert msg_showed_up, "Message was not found in log_group, so updating DynamoDB did not result in a successful Lambda execution" + assert msg_showed_up, ( + expected_msg + " was not found after updating DDB. All logs: " + str(all_logs) + ) + assert "Nr_of_records(1)" in all_logs, "Only one item should be updated" + assert ( + "Nr_of_records(2)" not in all_logs + ), "The inserted item should not show up again" def wait_for_log_msg(expected_msg, log_group): logs_conn = boto3.client("logs", region_name="us-east-1") + received_messages = [] start = time.time() - while (time.time() - start) < 30: + while (time.time() - start) < 10: result = logs_conn.describe_log_streams(logGroupName=log_group) log_streams = result.get("logStreams") if not log_streams: time.sleep(1) continue - assert len(log_streams) == 1 - result = logs_conn.get_log_events( - logGroupName=log_group, - logStreamName=log_streams[0]["logStreamName"], - ) - for event in result.get("events"): - if event["message"] == expected_msg: - return True + for log_stream in log_streams: + result = logs_conn.get_log_events( + logGroupName=log_group, logStreamName=log_stream["logStreamName"], + ) + received_messages.extend( + [event["message"] for event in result.get("events")] + ) + if expected_msg in received_messages: + return True, received_messages time.sleep(1) - return False + return False, received_messages @mock_logs From 939bd1cd86ad62b4e33b937a8f21253664f085d0 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Mon, 24 Feb 2020 13:43:19 +0000 Subject: [PATCH 03/17] EC2 - Add some filters for describe_instance_status --- moto/ec2/models.py | 39 +++++++++++++++------ moto/ec2/responses/instances.py | 19 ++++++++-- tests/test_ec2/test_instances.py | 59 +++++++++++++++++++++++++++++++- 3 files changed, 102 insertions(+), 15 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 166d8e646..ef506e443 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -822,6 +822,21 @@ class Instance(TaggedEC2Resource, BotoInstance): return self.public_ip raise UnformattedGetAttTemplateException() + def applies(self, filters): + if filters: + applicable = False + for f in filters: + acceptable_values = f['values'] + if f['name'] == "instance-state-name": + if self._state.name in acceptable_values: + applicable = True + if f['name'] == "instance-state-code": + if str(self._state.code) in acceptable_values: + applicable = True + return applicable + # If there are no filters, all instances are valid + return True + class InstanceBackend(object): def __init__(self): @@ -921,22 +936,23 @@ class InstanceBackend(object): value = getattr(instance, key) return instance, value - def all_instances(self): + def all_instances(self, filters=None): instances = [] for reservation in self.all_reservations(): for instance in reservation.instances: - instances.append(instance) - return instances - - def all_running_instances(self): - instances = [] - for reservation in self.all_reservations(): - for instance in reservation.instances: - if instance.state_code == 16: + if instance.applies(filters): instances.append(instance) return instances - def get_multi_instances_by_id(self, instance_ids): + def all_running_instances(self, filters=None): + instances = [] + for reservation in self.all_reservations(): + for instance in reservation.instances: + if instance.state_code == 16 and instance.applies(filters): + instances.append(instance) + return instances + + def get_multi_instances_by_id(self, instance_ids, filters=None): """ :param instance_ids: A string list with instance ids :return: A list with instance objects @@ -946,7 +962,8 @@ class InstanceBackend(object): for reservation in self.all_reservations(): for instance in reservation.instances: if instance.id in instance_ids: - result.append(instance) + if instance.applies(filters): + result.append(instance) # TODO: Trim error message down to specific invalid id. if instance_ids and len(instance_ids) > len(result): diff --git a/moto/ec2/responses/instances.py b/moto/ec2/responses/instances.py index b9e572d29..9b1105291 100644 --- a/moto/ec2/responses/instances.py +++ b/moto/ec2/responses/instances.py @@ -113,16 +113,29 @@ class InstanceResponse(BaseResponse): template = self.response_template(EC2_START_INSTANCES) return template.render(instances=instances) + def _get_list_of_dict_params(self, param_prefix, _dct): + """ + Simplified version of _get_dict_param + Allows you to pass in a custom dict instead of using self.querystring by default + """ + params = [] + for key, value in _dct.items(): + if key.startswith(param_prefix): + params.append(value) + return params + def describe_instance_status(self): instance_ids = self._get_multi_param("InstanceId") include_all_instances = self._get_param("IncludeAllInstances") == "true" + filters = self._get_list_prefix("Filter") + filters = [{'name': f['name'], 'values': self._get_list_of_dict_params("value.", f)} for f in filters] if instance_ids: - instances = self.ec2_backend.get_multi_instances_by_id(instance_ids) + instances = self.ec2_backend.get_multi_instances_by_id(instance_ids, filters) elif include_all_instances: - instances = self.ec2_backend.all_instances() + instances = self.ec2_backend.all_instances(filters) else: - instances = self.ec2_backend.all_running_instances() + instances = self.ec2_backend.all_running_instances(filters) template = self.response_template(EC2_INSTANCE_STATUS) return template.render(instances=instances) diff --git a/tests/test_ec2/test_instances.py b/tests/test_ec2/test_instances.py index 041bc8c85..ac6a4f4ec 100644 --- a/tests/test_ec2/test_instances.py +++ b/tests/test_ec2/test_instances.py @@ -1144,7 +1144,7 @@ def test_describe_instance_status_with_instances(): @mock_ec2_deprecated -def test_describe_instance_status_with_instance_filter(): +def test_describe_instance_status_with_instance_filter_deprecated(): conn = boto.connect_ec2("the_key", "the_secret") # We want to filter based on this one @@ -1166,6 +1166,63 @@ def test_describe_instance_status_with_instance_filter(): cm.exception.request_id.should_not.be.none +@mock_ec2 +def test_describe_instance_status_with_instance_filter(): + conn = boto3.client("ec2", region_name="us-west-1") + + # We want to filter based on this one + reservation = conn.run_instances(ImageId="ami-1234abcd", MinCount=3, MaxCount=3) + instance1 = reservation['Instances'][0] + instance2 = reservation['Instances'][1] + instance3 = reservation['Instances'][2] + conn.stop_instances(InstanceIds=[instance1['InstanceId']]) + stopped_instance_ids = [instance1['InstanceId']] + running_instance_ids = sorted([instance2['InstanceId'], instance3['InstanceId']]) + all_instance_ids = sorted(stopped_instance_ids + running_instance_ids) + + # Filter instance using the state name + state_name_filter = { + "running_and_stopped": [ + {"Name": "instance-state-name", "Values": ["running", "stopped"]} + ], + "running": [{"Name": "instance-state-name", "Values": ["running"]}], + "stopped": [{"Name": "instance-state-name", "Values": ["stopped"]}], + } + + found_statuses = conn.describe_instance_status(IncludeAllInstances=True, Filters=state_name_filter["running_and_stopped"])['InstanceStatuses'] + found_instance_ids = [status['InstanceId'] for status in found_statuses] + sorted(found_instance_ids).should.equal(all_instance_ids) + + found_statuses = conn.describe_instance_status(IncludeAllInstances=True, Filters=state_name_filter["running"])['InstanceStatuses'] + found_instance_ids = [status['InstanceId'] for status in found_statuses] + sorted(found_instance_ids).should.equal(running_instance_ids) + + found_statuses = conn.describe_instance_status(IncludeAllInstances=True, Filters=state_name_filter["stopped"])['InstanceStatuses'] + found_instance_ids = [status['InstanceId'] for status in found_statuses] + sorted(found_instance_ids).should.equal(stopped_instance_ids) + + # Filter instance using the state code + state_code_filter = { + "running_and_stopped": [ + {"Name": "instance-state-code", "Values": ["16", "80"]} + ], + "running": [{"Name": "instance-state-code", "Values": ["16"]}], + "stopped": [{"Name": "instance-state-code", "Values": ["80"]}], + } + + found_statuses = conn.describe_instance_status(IncludeAllInstances=True, Filters=state_code_filter["running_and_stopped"])['InstanceStatuses'] + found_instance_ids = [status['InstanceId'] for status in found_statuses] + sorted(found_instance_ids).should.equal(all_instance_ids) + + found_statuses = conn.describe_instance_status(IncludeAllInstances=True, Filters=state_code_filter["running"])['InstanceStatuses'] + found_instance_ids = [status['InstanceId'] for status in found_statuses] + sorted(found_instance_ids).should.equal(running_instance_ids) + + found_statuses = conn.describe_instance_status(IncludeAllInstances=True, Filters=state_code_filter["stopped"])['InstanceStatuses'] + found_instance_ids = [status['InstanceId'] for status in found_statuses] + sorted(found_instance_ids).should.equal(stopped_instance_ids) + + @requires_boto_gte("2.32.0") @mock_ec2_deprecated def test_describe_instance_status_with_non_running_instances(): From 3aeb5f504319fd901482bc7f3428b3e27774c217 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Mon, 24 Feb 2020 13:43:58 +0000 Subject: [PATCH 04/17] Linting --- moto/ec2/models.py | 6 ++-- moto/ec2/responses/instances.py | 9 ++++-- tests/test_ec2/test_instances.py | 48 ++++++++++++++++++++------------ 3 files changed, 40 insertions(+), 23 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index ef506e443..9c720cda8 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -826,11 +826,11 @@ class Instance(TaggedEC2Resource, BotoInstance): if filters: applicable = False for f in filters: - acceptable_values = f['values'] - if f['name'] == "instance-state-name": + acceptable_values = f["values"] + if f["name"] == "instance-state-name": if self._state.name in acceptable_values: applicable = True - if f['name'] == "instance-state-code": + if f["name"] == "instance-state-code": if str(self._state.code) in acceptable_values: applicable = True return applicable diff --git a/moto/ec2/responses/instances.py b/moto/ec2/responses/instances.py index 9b1105291..29c346f82 100644 --- a/moto/ec2/responses/instances.py +++ b/moto/ec2/responses/instances.py @@ -128,10 +128,15 @@ class InstanceResponse(BaseResponse): instance_ids = self._get_multi_param("InstanceId") include_all_instances = self._get_param("IncludeAllInstances") == "true" filters = self._get_list_prefix("Filter") - filters = [{'name': f['name'], 'values': self._get_list_of_dict_params("value.", f)} for f in filters] + filters = [ + {"name": f["name"], "values": self._get_list_of_dict_params("value.", f)} + for f in filters + ] if instance_ids: - instances = self.ec2_backend.get_multi_instances_by_id(instance_ids, filters) + instances = self.ec2_backend.get_multi_instances_by_id( + instance_ids, filters + ) elif include_all_instances: instances = self.ec2_backend.all_instances(filters) else: diff --git a/tests/test_ec2/test_instances.py b/tests/test_ec2/test_instances.py index ac6a4f4ec..85ba0fe01 100644 --- a/tests/test_ec2/test_instances.py +++ b/tests/test_ec2/test_instances.py @@ -1172,12 +1172,12 @@ def test_describe_instance_status_with_instance_filter(): # We want to filter based on this one reservation = conn.run_instances(ImageId="ami-1234abcd", MinCount=3, MaxCount=3) - instance1 = reservation['Instances'][0] - instance2 = reservation['Instances'][1] - instance3 = reservation['Instances'][2] - conn.stop_instances(InstanceIds=[instance1['InstanceId']]) - stopped_instance_ids = [instance1['InstanceId']] - running_instance_ids = sorted([instance2['InstanceId'], instance3['InstanceId']]) + instance1 = reservation["Instances"][0] + instance2 = reservation["Instances"][1] + instance3 = reservation["Instances"][2] + conn.stop_instances(InstanceIds=[instance1["InstanceId"]]) + stopped_instance_ids = [instance1["InstanceId"]] + running_instance_ids = sorted([instance2["InstanceId"], instance3["InstanceId"]]) all_instance_ids = sorted(stopped_instance_ids + running_instance_ids) # Filter instance using the state name @@ -1189,16 +1189,22 @@ def test_describe_instance_status_with_instance_filter(): "stopped": [{"Name": "instance-state-name", "Values": ["stopped"]}], } - found_statuses = conn.describe_instance_status(IncludeAllInstances=True, Filters=state_name_filter["running_and_stopped"])['InstanceStatuses'] - found_instance_ids = [status['InstanceId'] for status in found_statuses] + found_statuses = conn.describe_instance_status( + IncludeAllInstances=True, Filters=state_name_filter["running_and_stopped"] + )["InstanceStatuses"] + found_instance_ids = [status["InstanceId"] for status in found_statuses] sorted(found_instance_ids).should.equal(all_instance_ids) - found_statuses = conn.describe_instance_status(IncludeAllInstances=True, Filters=state_name_filter["running"])['InstanceStatuses'] - found_instance_ids = [status['InstanceId'] for status in found_statuses] + found_statuses = conn.describe_instance_status( + IncludeAllInstances=True, Filters=state_name_filter["running"] + )["InstanceStatuses"] + found_instance_ids = [status["InstanceId"] for status in found_statuses] sorted(found_instance_ids).should.equal(running_instance_ids) - found_statuses = conn.describe_instance_status(IncludeAllInstances=True, Filters=state_name_filter["stopped"])['InstanceStatuses'] - found_instance_ids = [status['InstanceId'] for status in found_statuses] + found_statuses = conn.describe_instance_status( + IncludeAllInstances=True, Filters=state_name_filter["stopped"] + )["InstanceStatuses"] + found_instance_ids = [status["InstanceId"] for status in found_statuses] sorted(found_instance_ids).should.equal(stopped_instance_ids) # Filter instance using the state code @@ -1210,16 +1216,22 @@ def test_describe_instance_status_with_instance_filter(): "stopped": [{"Name": "instance-state-code", "Values": ["80"]}], } - found_statuses = conn.describe_instance_status(IncludeAllInstances=True, Filters=state_code_filter["running_and_stopped"])['InstanceStatuses'] - found_instance_ids = [status['InstanceId'] for status in found_statuses] + found_statuses = conn.describe_instance_status( + IncludeAllInstances=True, Filters=state_code_filter["running_and_stopped"] + )["InstanceStatuses"] + found_instance_ids = [status["InstanceId"] for status in found_statuses] sorted(found_instance_ids).should.equal(all_instance_ids) - found_statuses = conn.describe_instance_status(IncludeAllInstances=True, Filters=state_code_filter["running"])['InstanceStatuses'] - found_instance_ids = [status['InstanceId'] for status in found_statuses] + found_statuses = conn.describe_instance_status( + IncludeAllInstances=True, Filters=state_code_filter["running"] + )["InstanceStatuses"] + found_instance_ids = [status["InstanceId"] for status in found_statuses] sorted(found_instance_ids).should.equal(running_instance_ids) - found_statuses = conn.describe_instance_status(IncludeAllInstances=True, Filters=state_code_filter["stopped"])['InstanceStatuses'] - found_instance_ids = [status['InstanceId'] for status in found_statuses] + found_statuses = conn.describe_instance_status( + IncludeAllInstances=True, Filters=state_code_filter["stopped"] + )["InstanceStatuses"] + found_instance_ids = [status["InstanceId"] for status in found_statuses] sorted(found_instance_ids).should.equal(stopped_instance_ids) From 28b4305759d49bafd2d49943956aaed4def10429 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Mon, 24 Feb 2020 11:53:27 -0600 Subject: [PATCH 05/17] add rudimentary support for Config PutEvaluations with TestMode for now --- moto/config/exceptions.py | 10 +++++ moto/config/models.py | 21 +++++++++++ moto/config/responses.py | 8 ++++ tests/test_config/test_config.py | 65 ++++++++++++++++++++++++++++++++ 4 files changed, 104 insertions(+) diff --git a/moto/config/exceptions.py b/moto/config/exceptions.py index 4a0dc0d73..6b6498d34 100644 --- a/moto/config/exceptions.py +++ b/moto/config/exceptions.py @@ -366,3 +366,13 @@ class TooManyResourceKeys(JsonRESTError): message = str(message) super(TooManyResourceKeys, self).__init__("ValidationException", message) + + +class InvalidResultTokenException(JsonRESTError): + code = 400 + + def __init__(self): + message = "The resultToken provided is invalid" + super(InvalidResultTokenException, self).__init__( + "InvalidResultTokenException", message + ) diff --git a/moto/config/models.py b/moto/config/models.py index a66576979..242a219e4 100644 --- a/moto/config/models.py +++ b/moto/config/models.py @@ -40,6 +40,7 @@ from moto.config.exceptions import ( TooManyResourceIds, ResourceNotDiscoveredException, TooManyResourceKeys, + InvalidResultTokenException, ) from moto.core import BaseBackend, BaseModel @@ -1089,6 +1090,26 @@ class ConfigBackend(BaseBackend): "UnprocessedResourceIdentifiers": not_found, } + def put_evaluations(self, evaluations=None, result_token=None, test_mode=False): + if not evaluations: + raise InvalidParameterValueException( + "The Evaluations object in your request cannot be null." + "Add the required parameters and try again." + ) + + if not result_token: + raise InvalidResultTokenException() + + # Moto only supports PutEvaluations with test mode currently (missing rule and token support) + if not test_mode: + raise NotImplementedError( + "PutEvaluations without TestMode is not yet implemented" + ) + + return { + "FailedEvaluations": [], + } # At this time, moto is not adding failed evaluations. + config_backends = {} for region in Session().get_available_regions("config"): diff --git a/moto/config/responses.py b/moto/config/responses.py index e977945c9..3b647b5bf 100644 --- a/moto/config/responses.py +++ b/moto/config/responses.py @@ -151,3 +151,11 @@ class ConfigResponse(BaseResponse): self._get_param("ResourceIdentifiers"), ) return json.dumps(schema) + + def put_evaluations(self): + evaluations = self.config_backend.put_evaluations( + self._get_param("Evaluations"), + self._get_param("ResultToken"), + self._get_param("TestMode"), + ) + return json.dumps(evaluations) diff --git a/tests/test_config/test_config.py b/tests/test_config/test_config.py index d5ec8f0bc..09fe8ed91 100644 --- a/tests/test_config/test_config.py +++ b/tests/test_config/test_config.py @@ -1802,3 +1802,68 @@ def test_batch_get_aggregate_resource_config(): len(result["UnprocessedResourceIdentifiers"]) == 1 and result["UnprocessedResourceIdentifiers"][0]["SourceRegion"] == "eu-west-1" ) + + +@mock_config +def test_put_evaluations(): + client = boto3.client("config", region_name="us-west-2") + + # Try without Evaluations supplied: + with assert_raises(ClientError) as ce: + client.put_evaluations(Evaluations=[], ResultToken="test", TestMode=True) + assert ce.exception.response["Error"]["Code"] == "InvalidParameterValueException" + assert ( + "The Evaluations object in your request cannot be null" + in ce.exception.response["Error"]["Message"] + ) + + # Try without a ResultToken supplied: + with assert_raises(ClientError) as ce: + client.put_evaluations( + Evaluations=[ + { + "ComplianceResourceType": "AWS::ApiGateway::RestApi", + "ComplianceResourceId": "test-api", + "ComplianceType": "INSUFFICIENT_DATA", + "OrderingTimestamp": datetime(2015, 1, 1), + } + ], + ResultToken="", + TestMode=True, + ) + assert ce.exception.response["Error"]["Code"] == "InvalidResultTokenException" + + # Try without TestMode supplied: + with assert_raises(NotImplementedError) as ce: + client.put_evaluations( + Evaluations=[ + { + "ComplianceResourceType": "AWS::ApiGateway::RestApi", + "ComplianceResourceId": "test-api", + "ComplianceType": "INSUFFICIENT_DATA", + "OrderingTimestamp": datetime(2015, 1, 1), + } + ], + ResultToken="test", + ) + + # Now with proper params: + response = client.put_evaluations( + Evaluations=[ + { + "ComplianceResourceType": "AWS::ApiGateway::RestApi", + "ComplianceResourceId": "test-api", + "ComplianceType": "INSUFFICIENT_DATA", + "OrderingTimestamp": datetime(2015, 1, 1), + } + ], + TestMode=True, + ResultToken="test", + ) + + # this is hard to match against, so remove it + response["ResponseMetadata"].pop("HTTPHeaders", None) + response["ResponseMetadata"].pop("RetryAttempts", None) + response.should.equal( + {"FailedEvaluations": [], "ResponseMetadata": {"HTTPStatusCode": 200,},} + ) From 4c43ca362f8e6d247407368dbe73d0a74f66ea2c Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Mon, 24 Feb 2020 13:01:38 -0600 Subject: [PATCH 06/17] add workaround for NotImplementedError failing server mode tests --- tests/test_config/test_config.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/tests/test_config/test_config.py b/tests/test_config/test_config.py index 09fe8ed91..8e6c3ec4c 100644 --- a/tests/test_config/test_config.py +++ b/tests/test_config/test_config.py @@ -3,6 +3,7 @@ from datetime import datetime, timedelta import boto3 from botocore.exceptions import ClientError +from nose import SkipTest from nose.tools import assert_raises from moto import mock_s3 @@ -1833,19 +1834,22 @@ def test_put_evaluations(): ) assert ce.exception.response["Error"]["Code"] == "InvalidResultTokenException" - # Try without TestMode supplied: - with assert_raises(NotImplementedError) as ce: - client.put_evaluations( - Evaluations=[ - { - "ComplianceResourceType": "AWS::ApiGateway::RestApi", - "ComplianceResourceId": "test-api", - "ComplianceType": "INSUFFICIENT_DATA", - "OrderingTimestamp": datetime(2015, 1, 1), - } - ], - ResultToken="test", - ) + if os.environ.get("TEST_SERVER_MODE", "false").lower() == "true": + raise SkipTest("Does not work in server mode due to error in Workzeug") + else: + # Try without TestMode supplied: + with assert_raises(NotImplementedError): + client.put_evaluations( + Evaluations=[ + { + "ComplianceResourceType": "AWS::ApiGateway::RestApi", + "ComplianceResourceId": "test-api", + "ComplianceType": "INSUFFICIENT_DATA", + "OrderingTimestamp": datetime(2015, 1, 1), + } + ], + ResultToken="test", + ) # Now with proper params: response = client.put_evaluations( From c3581dbd0b1287d8a0e7fe5c5dba8a64223fd159 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Mon, 24 Feb 2020 13:25:36 -0600 Subject: [PATCH 07/17] add missing os import for config tests --- tests/test_config/test_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_config/test_config.py b/tests/test_config/test_config.py index 8e6c3ec4c..1ffd52a2c 100644 --- a/tests/test_config/test_config.py +++ b/tests/test_config/test_config.py @@ -1,4 +1,5 @@ import json +import os from datetime import datetime, timedelta import boto3 From 47349b30df93d383d938effdcd6462af53fc1812 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Thu, 27 Feb 2020 08:54:57 +0000 Subject: [PATCH 08/17] #2567 - When mocking URLs, always return the first match --- moto/core/models.py | 18 ++++++++++++++++++ tests/test_s3/test_s3.py | 16 ++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/moto/core/models.py b/moto/core/models.py index ffb2ffd9f..8ca74d5b5 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -7,6 +7,7 @@ import inspect import os import re import six +import types from io import BytesIO from collections import defaultdict from botocore.handlers import BUILTIN_HANDLERS @@ -217,12 +218,29 @@ botocore_mock = responses.RequestsMock( assert_all_requests_are_fired=False, target="botocore.vendored.requests.adapters.HTTPAdapter.send", ) + responses_mock = responses._default_mock # Add passthrough to allow any other requests to work # Since this uses .startswith, it applies to http and https requests. responses_mock.add_passthru("http") +def _find_first_match(self, request): + for i, match in enumerate(self._matches): + if match.matches(request): + return match + + return None + + +# Modify behaviour of the matcher to only/always return the first match +# Default behaviour is to return subsequent matches for subsequent requests, which leads to https://github.com/spulec/moto/issues/2567 +# - First request matches on the appropriate S3 URL +# - Same request, executed again, will be matched on the subsequent match, which happens to be the catch-all, not-yet-implemented, callback +# Fix: Always return the first match +responses_mock._find_match = types.MethodType(_find_first_match, responses_mock) + + BOTOCORE_HTTP_METHODS = ["GET", "DELETE", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"] diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 2193f8b27..48655ee17 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -39,6 +39,7 @@ import moto.s3.models as s3model from moto.core.exceptions import InvalidNextTokenException from moto.core.utils import py2_strip_unicode_keys + if settings.TEST_SERVER_MODE: REDUCED_PART_SIZE = s3model.UPLOAD_PART_MIN_SIZE EXPECTED_ETAG = '"140f92a6df9f9e415f74a1463bcee9bb-2"' @@ -1018,12 +1019,23 @@ def test_s3_object_in_public_bucket(): s3_anonymous.Object(key="file.txt", bucket_name="test-bucket").get() exc.exception.response["Error"]["Code"].should.equal("403") + +@mock_s3 +def test_s3_object_in_public_bucket_using_multiple_presigned_urls(): + s3 = boto3.resource("s3") + bucket = s3.Bucket("test-bucket") + bucket.create( + ACL="public-read", CreateBucketConfiguration={"LocationConstraint": "us-west-1"} + ) + bucket.put_object(Body=b"ABCD", Key="file.txt") + params = {"Bucket": "test-bucket", "Key": "file.txt"} presigned_url = boto3.client("s3").generate_presigned_url( "get_object", params, ExpiresIn=900 ) - response = requests.get(presigned_url) - assert response.status_code == 200 + for i in range(1, 10): + response = requests.get(presigned_url) + assert response.status_code == 200, "Failed on req number {}".format(i) @mock_s3 From 5b9b9656476f852ff00c8095ea5a28d330c15d87 Mon Sep 17 00:00:00 2001 From: aimannajjar Date: Wed, 18 Dec 2019 17:29:13 -0500 Subject: [PATCH 09/17] [ec2-sg] added logic to create a second default egress rule for ipv6 --- moto/ec2/models.py | 6 ++++++ tests/test_ec2/test_security_groups.py | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 166d8e646..8afa30aa4 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -1722,6 +1722,12 @@ class SecurityGroup(TaggedEC2Resource): self.vpc_id = vpc_id self.owner_id = OWNER_ID + # Append default IPv6 egress rule for VPCs with IPv6 support + if vpc_id: + vpc = self.ec2_backend.vpcs.get(vpc_id) + if vpc and len(vpc.get_cidr_block_association_set(ipv6=True)) > 0: + self.egress_rules.append(SecurityRule("-1", None, None, [], [])) + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/tests/test_ec2/test_security_groups.py b/tests/test_ec2/test_security_groups.py index bb9c8f52a..ac9f39b57 100644 --- a/tests/test_ec2/test_security_groups.py +++ b/tests/test_ec2/test_security_groups.py @@ -123,6 +123,18 @@ def test_create_two_security_groups_with_same_name_in_different_vpc(): set(group_names).should.equal(set(["default", "test security group"])) +@mock_ec2 +def test_create_two_security_groups_in_vpc_with_ipv6_enabled(): + ec2 = boto3.resource("ec2", region_name="us-west-1") + vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16", AmazonProvidedIpv6CidrBlock=True) + + security_group = ec2.create_security_group( + GroupName="sg01", Description="Test security group sg01", VpcId=vpc.id + ) + + security_group.ip_permissions_egress.should.have.length_of(2) + + @mock_ec2_deprecated def test_deleting_security_groups(): conn = boto.connect_ec2("the_key", "the_secret") From 639c1abcb486cd083d801122ae7f37dcaa65292f Mon Sep 17 00:00:00 2001 From: aimannajjar Date: Sun, 1 Mar 2020 08:23:31 -0500 Subject: [PATCH 10/17] clarifying comment in test case --- tests/test_ec2/test_security_groups.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_ec2/test_security_groups.py b/tests/test_ec2/test_security_groups.py index ac9f39b57..7e936b7a5 100644 --- a/tests/test_ec2/test_security_groups.py +++ b/tests/test_ec2/test_security_groups.py @@ -132,6 +132,7 @@ def test_create_two_security_groups_in_vpc_with_ipv6_enabled(): GroupName="sg01", Description="Test security group sg01", VpcId=vpc.id ) + # The security group must have two defaul egress rules (one for ipv4 and aonther for ipv6) security_group.ip_permissions_egress.should.have.length_of(2) From a9b06776671c081733ae82c21ba191a4f1965f3d Mon Sep 17 00:00:00 2001 From: addomafi Date: Thu, 5 Mar 2020 18:11:49 -0300 Subject: [PATCH 11/17] #2784 Adding missing support for EbsConfiguration on EMR run_job_flow --- moto/emr/models.py | 2 ++ tests/test_emr/test_emr_boto3.py | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/moto/emr/models.py b/moto/emr/models.py index d9ec2fd69..72c588166 100644 --- a/moto/emr/models.py +++ b/moto/emr/models.py @@ -35,6 +35,7 @@ class FakeInstanceGroup(BaseModel): name=None, id=None, bid_price=None, + ebs_configuration=None, ): self.id = id or random_instance_group_id() @@ -51,6 +52,7 @@ class FakeInstanceGroup(BaseModel): self.num_instances = instance_count self.role = instance_role self.type = instance_type + self.ebs_configuration = ebs_configuration self.creation_datetime = datetime.now(pytz.utc) self.start_datetime = datetime.now(pytz.utc) diff --git a/tests/test_emr/test_emr_boto3.py b/tests/test_emr/test_emr_boto3.py index d849247bd..fc7170ba9 100644 --- a/tests/test_emr/test_emr_boto3.py +++ b/tests/test_emr/test_emr_boto3.py @@ -61,6 +61,23 @@ input_instance_groups = [ "Name": "task-2", "BidPrice": "0.05", }, + { + "InstanceCount": 10, + "InstanceRole": "TASK", + "InstanceType": "c1.xlarge", + "Market": "SPOT", + "Name": "task-3", + "BidPrice": "0.05", + "EbsConfiguration": { + "EbsBlockDeviceConfigs": [ + { + "VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 800}, + "VolumesPerInstance": 6, + }, + ], + "EbsOptimized": True, + }, + }, ] @@ -447,6 +464,8 @@ def test_run_job_flow_with_instance_groups(): x["Market"].should.equal(y["Market"]) if "BidPrice" in y: x["BidPrice"].should.equal(y["BidPrice"]) + if "EbsConfiguration" in y: + x["EbsConfiguration"].should.equal(y["EbsConfiguration"]) @mock_emr @@ -604,6 +623,8 @@ def test_instance_groups(): y = input_groups[x["Name"]] if hasattr(y, "BidPrice"): x["BidPrice"].should.equal("BidPrice") + if "EbsConfiguration" in y: + x["EbsConfiguration"].should.equal(y["EbsConfiguration"]) x["CreationDateTime"].should.be.a("datetime.datetime") # x['EndDateTime'].should.be.a('datetime.datetime') x.should.have.key("InstanceGroupId") @@ -623,6 +644,8 @@ def test_instance_groups(): y = input_groups[x["Name"]] if hasattr(y, "BidPrice"): x["BidPrice"].should.equal("BidPrice") + if "EbsConfiguration" in y: + x["EbsConfiguration"].should.equal(y["EbsConfiguration"]) # Configurations # EbsBlockDevices # EbsOptimized From c8dfbe95753fcaa01578eda2798d47c62c86102f Mon Sep 17 00:00:00 2001 From: addomafi Date: Fri, 6 Mar 2020 15:12:44 -0300 Subject: [PATCH 12/17] #2784 Adding missing support for EbsConfiguration on EMR instance groups --- moto/emr/responses.py | 98 +++++++++++++++++++++++++++++++- tests/test_emr/test_emr_boto3.py | 23 ++++---- 2 files changed, 109 insertions(+), 12 deletions(-) diff --git a/moto/emr/responses.py b/moto/emr/responses.py index 38b9774e1..3bb595bbb 100644 --- a/moto/emr/responses.py +++ b/moto/emr/responses.py @@ -73,6 +73,8 @@ class ElasticMapReduceResponse(BaseResponse): instance_groups = self._get_list_prefix("InstanceGroups.member") for item in instance_groups: item["instance_count"] = int(item["instance_count"]) + # Adding support to EbsConfiguration + self._parse_ebs_configuration(item) instance_groups = self.backend.add_instance_groups(jobflow_id, instance_groups) template = self.response_template(ADD_INSTANCE_GROUPS_TEMPLATE) return template.render(instance_groups=instance_groups) @@ -324,6 +326,8 @@ class ElasticMapReduceResponse(BaseResponse): if instance_groups: for ig in instance_groups: ig["instance_count"] = int(ig["instance_count"]) + # Adding support to EbsConfiguration + self._parse_ebs_configuration(ig) self.backend.add_instance_groups(cluster.id, instance_groups) tags = self._get_list_prefix("Tags.member") @@ -335,6 +339,83 @@ class ElasticMapReduceResponse(BaseResponse): template = self.response_template(RUN_JOB_FLOW_TEMPLATE) return template.render(cluster=cluster) + def _has_key_prefix(self, key_prefix, value): + for key in value: # iter on both keys and values + if key.startswith(key_prefix): + return True + return False + + def _parse_ebs_configuration(self, instance_group): + key_ebs_config = "ebs_configuration" + ebs_configuration = dict() + # Filter only EBS config keys + for key in instance_group: + if key.startswith(key_ebs_config): + ebs_configuration[key] = instance_group[key] + + if len(ebs_configuration) > 0: + # Key that should be extracted + ebs_optimized = "ebs_optimized" + ebs_block_device_configs = "ebs_block_device_configs" + volume_specification = "volume_specification" + size_in_gb = "size_in_gb" + volume_type = "volume_type" + iops = "iops" + volumes_per_instance = "volumes_per_instance" + + key_ebs_optimized = f"{key_ebs_config}._{ebs_optimized}" + # EbsOptimized config + if key_ebs_optimized in ebs_configuration: + instance_group.pop(key_ebs_optimized) + ebs_configuration[ebs_optimized] = ebs_configuration.pop( + key_ebs_optimized + ) + + # Ebs Blocks + ebs_blocks = [] + idx = 1 + keyfmt = f"{key_ebs_config}._{ebs_block_device_configs}.member.{{}}" + key = keyfmt.format(idx) + while self._has_key_prefix(key, ebs_configuration): + vlespc_keyfmt = f"{key}._{volume_specification}._{{}}" + vol_size = vlespc_keyfmt.format(size_in_gb) + vol_iops = vlespc_keyfmt.format(iops) + vol_type = vlespc_keyfmt.format(volume_type) + + ebs_block = dict() + ebs_block[volume_specification] = dict() + if vol_size in ebs_configuration: + instance_group.pop(vol_size) + ebs_block[volume_specification][size_in_gb] = int( + ebs_configuration.pop(vol_size) + ) + if vol_iops in ebs_configuration: + instance_group.pop(vol_iops) + ebs_block[volume_specification][iops] = ebs_configuration.pop( + vol_iops + ) + if vol_type in ebs_configuration: + instance_group.pop(vol_type) + ebs_block[volume_specification][ + volume_type + ] = ebs_configuration.pop(vol_type) + + per_instance = f"{key}._{volumes_per_instance}" + if per_instance in ebs_configuration: + instance_group.pop(per_instance) + ebs_block[volumes_per_instance] = int( + ebs_configuration.pop(per_instance) + ) + + if len(ebs_block) > 0: + ebs_blocks.append(ebs_block) + idx += 1 + key = keyfmt.format(idx) + + if len(ebs_blocks) > 0: + ebs_configuration[ebs_block_device_configs] = ebs_blocks + instance_group[key_ebs_config] = ebs_configuration + @generate_boto3_response("SetTerminationProtection") def set_termination_protection(self): termination_protection = self._get_param("TerminationProtected") @@ -754,7 +835,22 @@ LIST_INSTANCE_GROUPS_TEMPLATE = """ Date: Fri, 6 Mar 2020 18:10:39 -0300 Subject: [PATCH 13/17] #2784 Implementing assertions for testcase with instance groups --- tests/test_emr/test_emr_boto3.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/tests/test_emr/test_emr_boto3.py b/tests/test_emr/test_emr_boto3.py index a00de164b..524cdcd55 100644 --- a/tests/test_emr/test_emr_boto3.py +++ b/tests/test_emr/test_emr_boto3.py @@ -438,6 +438,19 @@ def test_run_job_flow_with_visible_to_all_users(): resp = client.describe_cluster(ClusterId=cluster_id) resp["Cluster"]["VisibleToAllUsers"].should.equal(expected) +def _do_assertion_ebs_configuration(x, y): + total_volumes = 0 + total_size = 0 + for ebs_block in y["EbsConfiguration"]["EbsBlockDeviceConfigs"]: + total_volumes += ebs_block["VolumesPerInstance"] + total_size += ebs_block["VolumeSpecification"]["SizeInGB"] + # Multiply by total volumes + total_size = total_size * total_volumes + comp_total_size = 0 + for ebs_block in x["EbsBlockDevices"]: + comp_total_size += ebs_block["VolumeSpecification"]["SizeInGB"] + len(x["EbsBlockDevices"]).should.equal(total_volumes) + comp_total_size.should.equal(comp_total_size) @mock_emr def test_run_job_flow_with_instance_groups(): @@ -456,8 +469,9 @@ def test_run_job_flow_with_instance_groups(): x["Market"].should.equal(y["Market"]) if "BidPrice" in y: x["BidPrice"].should.equal(y["BidPrice"]) + if "EbsConfiguration" in y: - x["EbsConfiguration"].should.equal(y["EbsConfiguration"]) + _do_assertion_ebs_configuration(x, y) @mock_emr @@ -635,18 +649,7 @@ def test_instance_groups(): if hasattr(y, "BidPrice"): x["BidPrice"].should.equal("BidPrice") if "EbsConfiguration" in y: - total_volumes = 0 - total_size = 0 - for ebs_block in y["EbsConfiguration"]["EbsBlockDeviceConfigs"]: - total_volumes += ebs_block["VolumesPerInstance"] - total_size += ebs_block["VolumeSpecification"]["SizeInGB"] - # Multiply by total volumes - total_size = total_size * total_volumes - comp_total_size = 0 - for ebs_block in x["EbsBlockDevices"]: - comp_total_size += ebs_block["VolumeSpecification"]["SizeInGB"] - len(x["EbsBlockDevices"]).should.equal(total_volumes) - comp_total_size.should.equal(comp_total_size) + _do_assertion_ebs_configuration(x, y) # Configurations # EbsBlockDevices # EbsOptimized From c6eca1843435413615b1650e2161a0f4890819d7 Mon Sep 17 00:00:00 2001 From: addomafi Date: Fri, 6 Mar 2020 18:11:07 -0300 Subject: [PATCH 14/17] Reformat --- tests/test_emr/test_emr_boto3.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_emr/test_emr_boto3.py b/tests/test_emr/test_emr_boto3.py index 524cdcd55..adfc3fa9c 100644 --- a/tests/test_emr/test_emr_boto3.py +++ b/tests/test_emr/test_emr_boto3.py @@ -438,6 +438,7 @@ def test_run_job_flow_with_visible_to_all_users(): resp = client.describe_cluster(ClusterId=cluster_id) resp["Cluster"]["VisibleToAllUsers"].should.equal(expected) + def _do_assertion_ebs_configuration(x, y): total_volumes = 0 total_size = 0 @@ -452,6 +453,7 @@ def _do_assertion_ebs_configuration(x, y): len(x["EbsBlockDevices"]).should.equal(total_volumes) comp_total_size.should.equal(comp_total_size) + @mock_emr def test_run_job_flow_with_instance_groups(): input_groups = dict((g["Name"], g) for g in input_instance_groups) From 155cf82791cef359ee7d6404da14090a18396d1b Mon Sep 17 00:00:00 2001 From: addomafi Date: Sat, 7 Mar 2020 07:43:59 -0300 Subject: [PATCH 15/17] Keeping support to python 2 --- moto/emr/responses.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/moto/emr/responses.py b/moto/emr/responses.py index 3bb595bbb..a9a4aae93 100644 --- a/moto/emr/responses.py +++ b/moto/emr/responses.py @@ -363,7 +363,7 @@ class ElasticMapReduceResponse(BaseResponse): iops = "iops" volumes_per_instance = "volumes_per_instance" - key_ebs_optimized = f"{key_ebs_config}._{ebs_optimized}" + key_ebs_optimized = "{0}._{1}".format(key_ebs_config, ebs_optimized) # EbsOptimized config if key_ebs_optimized in ebs_configuration: instance_group.pop(key_ebs_optimized) @@ -374,10 +374,10 @@ class ElasticMapReduceResponse(BaseResponse): # Ebs Blocks ebs_blocks = [] idx = 1 - keyfmt = f"{key_ebs_config}._{ebs_block_device_configs}.member.{{}}" + keyfmt = "{0}._{1}.member.{{}}".format(key_ebs_config, ebs_block_device_configs) key = keyfmt.format(idx) while self._has_key_prefix(key, ebs_configuration): - vlespc_keyfmt = f"{key}._{volume_specification}._{{}}" + vlespc_keyfmt = "{0}._{1}._{{}}".format(key, volume_specification) vol_size = vlespc_keyfmt.format(size_in_gb) vol_iops = vlespc_keyfmt.format(iops) vol_type = vlespc_keyfmt.format(volume_type) @@ -400,7 +400,7 @@ class ElasticMapReduceResponse(BaseResponse): volume_type ] = ebs_configuration.pop(vol_type) - per_instance = f"{key}._{volumes_per_instance}" + per_instance = "{0}._{1}".format(key, volumes_per_instance) if per_instance in ebs_configuration: instance_group.pop(per_instance) ebs_block[volumes_per_instance] = int( From a6c1d474128d4097a157b300a4b24948e32656ea Mon Sep 17 00:00:00 2001 From: addomafi Date: Sat, 7 Mar 2020 08:21:27 -0300 Subject: [PATCH 16/17] Reformat --- moto/emr/responses.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/moto/emr/responses.py b/moto/emr/responses.py index a9a4aae93..3708db0ed 100644 --- a/moto/emr/responses.py +++ b/moto/emr/responses.py @@ -374,7 +374,9 @@ class ElasticMapReduceResponse(BaseResponse): # Ebs Blocks ebs_blocks = [] idx = 1 - keyfmt = "{0}._{1}.member.{{}}".format(key_ebs_config, ebs_block_device_configs) + keyfmt = "{0}._{1}.member.{{}}".format( + key_ebs_config, ebs_block_device_configs + ) key = keyfmt.format(idx) while self._has_key_prefix(key, ebs_configuration): vlespc_keyfmt = "{0}._{1}._{{}}".format(key, volume_specification) From 28af7412f80d973a2bf34130fc2926519861c06a Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Sun, 8 Mar 2020 20:56:21 +0100 Subject: [PATCH 17/17] Change RESTError to JsonRESTError for ImageNotFoundException, update test to expect ImageNotFoundException --- moto/ecr/exceptions.py | 4 ++-- tests/test_ecr/test_ecr_boto3.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/moto/ecr/exceptions.py b/moto/ecr/exceptions.py index 9b55f0589..6d1713a6a 100644 --- a/moto/ecr/exceptions.py +++ b/moto/ecr/exceptions.py @@ -1,5 +1,5 @@ from __future__ import unicode_literals -from moto.core.exceptions import RESTError +from moto.core.exceptions import RESTError, JsonRESTError class RepositoryNotFoundException(RESTError): @@ -13,7 +13,7 @@ class RepositoryNotFoundException(RESTError): ) -class ImageNotFoundException(RESTError): +class ImageNotFoundException(JsonRESTError): code = 400 def __init__(self, image_id, repository_name, registry_id): diff --git a/tests/test_ecr/test_ecr_boto3.py b/tests/test_ecr/test_ecr_boto3.py index 82a2c7521..6c6840a7e 100644 --- a/tests/test_ecr/test_ecr_boto3.py +++ b/tests/test_ecr/test_ecr_boto3.py @@ -538,7 +538,7 @@ def test_describe_image_that_doesnt_exist(): repositoryName="test_repository", imageIds=[{"imageTag": "testtag"}], registryId="123", - ).should.throw(ClientError, error_msg1) + ).should.throw(client.exceptions.ImageNotFoundException, error_msg1) error_msg2 = re.compile( r".*The repository with name 'repo-that-doesnt-exist' does not exist in the registry with id '123'.*",