From 002683fd13c74028912f934408c0376b410bd487 Mon Sep 17 00:00:00 2001 From: Laurie O Date: Wed, 26 Feb 2020 00:19:39 +1000 Subject: [PATCH 01/16] Return empty task-token on no-task To match the SWF documentation, an empty task is one where the task-token is the empty string, rather than being a nonexistant key Signed-off-by: Laurie O --- moto/swf/responses.py | 6 ++++-- tests/test_swf/responses/test_activity_tasks.py | 4 ++-- tests/test_swf/responses/test_decision_tasks.py | 8 ++++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/moto/swf/responses.py b/moto/swf/responses.py index 2b7794ffd..c8c601fa7 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -423,7 +423,9 @@ class SWFResponse(BaseResponse): if decision: return json.dumps(decision.to_full_dict(reverse_order=reverse_order)) else: - return json.dumps({"previousStartedEventId": 0, "startedEventId": 0}) + return json.dumps( + {"previousStartedEventId": 0, "startedEventId": 0, "taskToken": ""} + ) def count_pending_decision_tasks(self): domain_name = self._params["domain"] @@ -457,7 +459,7 @@ class SWFResponse(BaseResponse): if activity_task: return json.dumps(activity_task.to_full_dict()) else: - return json.dumps({"startedEventId": 0}) + return json.dumps({"startedEventId": 0, "taskToken": ""}) def count_pending_activity_tasks(self): domain_name = self._params["domain"] diff --git a/tests/test_swf/responses/test_activity_tasks.py b/tests/test_swf/responses/test_activity_tasks.py index 0b72b7ca7..4fa965b11 100644 --- a/tests/test_swf/responses/test_activity_tasks.py +++ b/tests/test_swf/responses/test_activity_tasks.py @@ -35,14 +35,14 @@ def test_poll_for_activity_task_when_one(): def test_poll_for_activity_task_when_none(): conn = setup_workflow() resp = conn.poll_for_activity_task("test-domain", "activity-task-list") - resp.should.equal({"startedEventId": 0}) + resp.should.equal({"startedEventId": 0, "taskToken": ""}) @mock_swf_deprecated def test_poll_for_activity_task_on_non_existent_queue(): conn = setup_workflow() resp = conn.poll_for_activity_task("test-domain", "non-existent-queue") - resp.should.equal({"startedEventId": 0}) + resp.should.equal({"startedEventId": 0, "taskToken": ""}) # CountPendingActivityTasks endpoint diff --git a/tests/test_swf/responses/test_decision_tasks.py b/tests/test_swf/responses/test_decision_tasks.py index 6389536e6..bdf690ce0 100644 --- a/tests/test_swf/responses/test_decision_tasks.py +++ b/tests/test_swf/responses/test_decision_tasks.py @@ -38,14 +38,18 @@ def test_poll_for_decision_task_when_none(): resp = conn.poll_for_decision_task("test-domain", "queue") # this is the DecisionTask representation you get from the real SWF # after waiting 60s when there's no decision to be taken - resp.should.equal({"previousStartedEventId": 0, "startedEventId": 0}) + resp.should.equal( + {"previousStartedEventId": 0, "startedEventId": 0, "taskToken": ""} + ) @mock_swf_deprecated def test_poll_for_decision_task_on_non_existent_queue(): conn = setup_workflow() resp = conn.poll_for_decision_task("test-domain", "non-existent-queue") - resp.should.equal({"previousStartedEventId": 0, "startedEventId": 0}) + resp.should.equal( + {"previousStartedEventId": 0, "startedEventId": 0, "taskToken": ""} + ) @mock_swf_deprecated From 369285b7ca1de06aaa296ea5d51482dad2492d83 Mon Sep 17 00:00:00 2001 From: Laurie O Date: Wed, 26 Feb 2020 01:06:58 +1000 Subject: [PATCH 02/16] Don't 0-default previous started event ID Signed-off-by: Laurie O --- moto/swf/models/decision_task.py | 5 +++-- tests/test_swf/models/test_decision_task.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/moto/swf/models/decision_task.py b/moto/swf/models/decision_task.py index c8c9824a2..d7236a0ad 100644 --- a/moto/swf/models/decision_task.py +++ b/moto/swf/models/decision_task.py @@ -15,7 +15,7 @@ class DecisionTask(BaseModel): self.workflow_type = workflow_execution.workflow_type self.task_token = str(uuid.uuid4()) self.scheduled_event_id = scheduled_event_id - self.previous_started_event_id = 0 + self.previous_started_event_id = None self.started_event_id = None self.started_timestamp = None self.start_to_close_timeout = ( @@ -40,10 +40,11 @@ class DecisionTask(BaseModel): hsh = { "events": [evt.to_dict() for evt in events], "taskToken": self.task_token, - "previousStartedEventId": self.previous_started_event_id, "workflowExecution": self.workflow_execution.to_short_dict(), "workflowType": self.workflow_type.to_short_dict(), } + if self.previous_started_event_id is not None: + hsh["previousStartedEventId"] = self.previous_started_event_id if self.started_event_id: hsh["startedEventId"] = self.started_event_id return hsh diff --git a/tests/test_swf/models/test_decision_task.py b/tests/test_swf/models/test_decision_task.py index 0661adffb..8296f0472 100644 --- a/tests/test_swf/models/test_decision_task.py +++ b/tests/test_swf/models/test_decision_task.py @@ -24,7 +24,7 @@ def test_decision_task_full_dict_representation(): fd = dt.to_full_dict() fd["events"].should.be.a("list") - fd["previousStartedEventId"].should.equal(0) + fd.should_not.contain("previousStartedEventId") fd.should_not.contain("startedEventId") fd.should.contain("taskToken") fd["workflowExecution"].should.equal(wfe.to_short_dict()) From 209c9997061b88d11ba62e700d5df05b706345ef Mon Sep 17 00:00:00 2001 From: Laurie O Date: Wed, 26 Feb 2020 01:08:03 +1000 Subject: [PATCH 03/16] Keep track of previous started event ID Closes #2107 Signed-off-by: Laurie O --- moto/swf/models/decision_task.py | 3 ++- moto/swf/models/workflow_execution.py | 4 +++- tests/test_swf/models/test_decision_task.py | 3 ++- .../test_swf/responses/test_decision_tasks.py | 24 +++++++++++++++++++ 4 files changed, 31 insertions(+), 3 deletions(-) diff --git a/moto/swf/models/decision_task.py b/moto/swf/models/decision_task.py index d7236a0ad..aaf810f08 100644 --- a/moto/swf/models/decision_task.py +++ b/moto/swf/models/decision_task.py @@ -49,10 +49,11 @@ class DecisionTask(BaseModel): hsh["startedEventId"] = self.started_event_id return hsh - def start(self, started_event_id): + def start(self, started_event_id, previous_started_event_id=None): self.state = "STARTED" self.started_timestamp = unix_time() self.started_event_id = started_event_id + self.previous_started_event_id = previous_started_event_id def complete(self): self._check_workflow_execution_open() diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 17ce819fb..035a47558 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -82,6 +82,7 @@ class WorkflowExecution(BaseModel): self._events = [] # child workflows self.child_workflow_executions = [] + self._previous_started_event_id = None def __repr__(self): return "WorkflowExecution(run_id: {0})".format(self.run_id) @@ -295,7 +296,8 @@ class WorkflowExecution(BaseModel): scheduled_event_id=dt.scheduled_event_id, identity=identity, ) - dt.start(evt.event_id) + dt.start(evt.event_id, self._previous_started_event_id) + self._previous_started_event_id = evt.event_id def complete_decision_task( self, task_token, decisions=None, execution_context=None diff --git a/tests/test_swf/models/test_decision_task.py b/tests/test_swf/models/test_decision_task.py index 8296f0472..8ddb230e2 100644 --- a/tests/test_swf/models/test_decision_task.py +++ b/tests/test_swf/models/test_decision_task.py @@ -30,9 +30,10 @@ def test_decision_task_full_dict_representation(): fd["workflowExecution"].should.equal(wfe.to_short_dict()) fd["workflowType"].should.equal(wft.to_short_dict()) - dt.start(1234) + dt.start(1234, 1230) fd = dt.to_full_dict() fd["startedEventId"].should.equal(1234) + fd["previousStartedEventId"].should.equal(1230) def test_decision_task_first_timeout(): diff --git a/tests/test_swf/responses/test_decision_tasks.py b/tests/test_swf/responses/test_decision_tasks.py index 6389536e6..6493302f9 100644 --- a/tests/test_swf/responses/test_decision_tasks.py +++ b/tests/test_swf/responses/test_decision_tasks.py @@ -30,6 +30,30 @@ def test_poll_for_decision_task_when_one(): ) +@mock_swf_deprecated +def test_poll_for_decision_task_previous_started_event_id(): + conn = setup_workflow() + + resp = conn.poll_for_decision_task("test-domain", "queue") + assert resp["workflowExecution"]["runId"] == conn.run_id + assert "previousStartedEventId" not in resp + + # Require a failing decision, in this case a non-existant activity type + attrs = { + "activityId": "spam", + "activityType": {"name": "test-activity", "version": "v1.42"}, + "taskList": "eggs", + } + decision = { + "decisionType": "ScheduleActivityTask", + "scheduleActivityTaskDecisionAttributes": attrs, + } + conn.respond_decision_task_completed(resp["taskToken"], decisions=[decision]) + resp = conn.poll_for_decision_task("test-domain", "queue") + assert resp["workflowExecution"]["runId"] == conn.run_id + assert resp["previousStartedEventId"] == 3 + + @mock_swf_deprecated def test_poll_for_decision_task_when_none(): conn = setup_workflow() From 607e0a845241b7033e14bae57117ee275c44b7bb Mon Sep 17 00:00:00 2001 From: Olabode Anise Date: Tue, 25 Feb 2020 20:25:44 -0500 Subject: [PATCH 04/16] ENH: changes the behavior of delete_parameter to respond with a 400 error when the parameter does not exist. Currently, the delete_parameter function for the ssm client will respond with a dict containing a key of Invalid Parameter which has a value of a list containing the parameter name that was requested to be deleted when a parameter with said name doesn't exist which doesn't match the behavior of boto3. --- moto/ssm/models.py | 5 +---- moto/ssm/responses.py | 8 +++++++- tests/test_ssm/test_ssm_boto3.py | 14 ++++++++++++++ 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/moto/ssm/models.py b/moto/ssm/models.py index 60c47f021..a7518d405 100644 --- a/moto/ssm/models.py +++ b/moto/ssm/models.py @@ -278,10 +278,7 @@ class SimpleSystemManagerBackend(BaseBackend): self._region = region def delete_parameter(self, name): - try: - del self._parameters[name] - except KeyError: - pass + return self._parameters.pop(name, None) def delete_parameters(self, names): result = [] diff --git a/moto/ssm/responses.py b/moto/ssm/responses.py index 1b13780a8..831737848 100644 --- a/moto/ssm/responses.py +++ b/moto/ssm/responses.py @@ -22,7 +22,13 @@ class SimpleSystemManagerResponse(BaseResponse): def delete_parameter(self): name = self._get_param("Name") - self.ssm_backend.delete_parameter(name) + result = self.ssm_backend.delete_parameter(name) + if result is None: + error = { + "__type": "ParameterNotFound", + "message": "Parameter {0} not found.".format(name), + } + return json.dumps(error), dict(status=400) return json.dumps({}) def delete_parameters(self): diff --git a/tests/test_ssm/test_ssm_boto3.py b/tests/test_ssm/test_ssm_boto3.py index 5b978520d..26db26dcb 100644 --- a/tests/test_ssm/test_ssm_boto3.py +++ b/tests/test_ssm/test_ssm_boto3.py @@ -30,6 +30,20 @@ def test_delete_parameter(): len(response["Parameters"]).should.equal(0) +@mock_ssm +def test_delete_nonexistent_parameter(): + client = boto3.client("ssm", region_name="us-east-1") + + try: + client.delete_parameter(Name="test_noexist") + raise RuntimeError("Should of failed") + except botocore.exceptions.ClientError as err: + err.operation_name.should.equal("DeleteParameter") + err.response["Error"]["Message"].should.equal( + "Parameter test_noexist not found." + ) + + @mock_ssm def test_delete_parameters(): client = boto3.client("ssm", region_name="us-east-1") From 9227845121788a4804da5e026d9f90baf9541bd7 Mon Sep 17 00:00:00 2001 From: Olabode Anise Date: Thu, 27 Feb 2020 07:29:13 -0500 Subject: [PATCH 05/16] using assert_raises --- tests/test_ssm/test_ssm_boto3.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/test_ssm/test_ssm_boto3.py b/tests/test_ssm/test_ssm_boto3.py index 26db26dcb..bb674fb65 100644 --- a/tests/test_ssm/test_ssm_boto3.py +++ b/tests/test_ssm/test_ssm_boto3.py @@ -34,14 +34,12 @@ def test_delete_parameter(): def test_delete_nonexistent_parameter(): client = boto3.client("ssm", region_name="us-east-1") - try: + with assert_raises(ClientError) as ex: client.delete_parameter(Name="test_noexist") - raise RuntimeError("Should of failed") - except botocore.exceptions.ClientError as err: - err.operation_name.should.equal("DeleteParameter") - err.response["Error"]["Message"].should.equal( - "Parameter test_noexist not found." - ) + ex.exception.response["Error"]["Code"].should.equal("ParameterNotFound") + ex.exception.response["Error"]["Message"].should.equal( + "Parameter test_noexist not found." + ) @mock_ssm From 00134d2df37bb4dcd5f447ef951d383bfec0903c Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Sat, 29 Feb 2020 09:41:06 +0000 Subject: [PATCH 06/16] Fix dependencies to versions that support Python 2 --- requirements-dev.txt | 2 +- setup.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index c5f055a26..2aaca300b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ -r requirements.txt -mock +mock==3.0.5 # Last version compatible with Python 2.7 nose black; python_version >= '3.6' regex==2019.11.1; python_version >= '3.6' # Needed for black diff --git a/setup.py b/setup.py index 1dde71ac7..b806f7bae 100755 --- a/setup.py +++ b/setup.py @@ -1,20 +1,20 @@ #!/usr/bin/env python from __future__ import unicode_literals import codecs +from io import open import os import re import setuptools from setuptools import setup, find_packages import sys - # Borrowed from pip at https://github.com/pypa/pip/blob/62c27dee45625e1b63d1e023b0656310f276e050/setup.py#L11-L15 here = os.path.abspath(os.path.dirname(__file__)) def read(*parts): # intentionally *not* adding an encoding option to open, See: # https://github.com/pypa/virtualenv/issues/201#issuecomment-3145690 - with codecs.open(os.path.join(here, *parts), 'r') as fp: + with open(os.path.join(here, *parts), 'r') as fp: return fp.read() @@ -28,7 +28,8 @@ def get_version(): install_requires = [ - "Jinja2>=2.10.1", + "setuptools==44.0.0", + "Jinja2==2.11.0", "boto>=2.36.0", "boto3>=1.9.201", "botocore>=1.12.201", @@ -41,14 +42,16 @@ install_requires = [ "pytz", "python-dateutil<3.0.0,>=2.1", "python-jose<4.0.0", - "mock", + "mock==3.0.5", "docker>=2.5.1", "jsondiff>=1.1.2", "aws-xray-sdk!=0.96,>=0.93", "responses>=0.9.0", "idna<2.9,>=2.5", "cfn-lint>=0.4.0", - "sshpubkeys>=3.1.0,<4.0" + "sshpubkeys>=3.1.0,<4.0", + "zipp==0.6.0", + "more-itertools==5.0.0" ] extras_require = { From 9fdeaca589956d7911e8b05491e3e15f7a9bcfad Mon Sep 17 00:00:00 2001 From: Jon Nangle Date: Mon, 2 Mar 2020 12:46:15 +0000 Subject: [PATCH 07/16] Support GeoLocation and Failover on Route 53 --- moto/route53/models.py | 12 +++++ tests/test_route53/test_route53.py | 82 ++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/moto/route53/models.py b/moto/route53/models.py index 2ae03e54d..0bdefd25b 100644 --- a/moto/route53/models.py +++ b/moto/route53/models.py @@ -88,6 +88,8 @@ class RecordSet(BaseModel): self.hosted_zone_name = kwargs.get("HostedZoneName") self.hosted_zone_id = kwargs.get("HostedZoneId") self.alias_target = kwargs.get("AliasTarget") + self.failover = kwargs.get("Failover") + self.geo_location = kwargs.get("GeoLocation") @classmethod def create_from_cloudformation_json( @@ -154,6 +156,16 @@ class RecordSet(BaseModel): {% if record_set.ttl %} {{ record_set.ttl }} {% endif %} + {% if record_set.failover %} + {{ record_set.failover }} + {% endif %} + {% if record_set.geo_location %} + + {% for geo_key in ['ContinentCode','CountryCode','SubdivisionCode'] %} + {% if record_set.geo_location[geo_key] %}<{{ geo_key }}>{{ record_set.geo_location[geo_key] }}{% endif %} + {% endfor %} + + {% endif %} {% if record_set.alias_target %} {{ record_set.alias_target['HostedZoneId'] }} diff --git a/tests/test_route53/test_route53.py b/tests/test_route53/test_route53.py index 746c78719..8cf148c14 100644 --- a/tests/test_route53/test_route53.py +++ b/tests/test_route53/test_route53.py @@ -753,6 +753,88 @@ def test_change_weighted_resource_record_sets(): record["Weight"].should.equal(10) +@mock_route53 +def test_failover_record_sets(): + conn = boto3.client("route53", region_name="us-east-2") + conn.create_hosted_zone( + Name="test.zone.", CallerReference=str(hash("test")) + ) + zones = conn.list_hosted_zones_by_name(DNSName="test.zone.") + hosted_zone_id = zones["HostedZones"][0]["Id"] + + # Create geolocation record + conn.change_resource_record_sets( + HostedZoneId=hosted_zone_id, + ChangeBatch={ + "Changes": [ + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "failover.test.zone.", + "Type": "A", + "TTL": 10, + "ResourceRecords": [{"Value": "127.0.0.1"}], + "Failover": "PRIMARY" + } + } + ] + } + ) + + response = conn.list_resource_record_sets(HostedZoneId=hosted_zone_id) + record = response["ResourceRecordSets"][0] + record["Failover"].should.equal("PRIMARY") + + +@mock_route53 +def test_geolocation_record_sets(): + conn = boto3.client("route53", region_name="us-east-2") + conn.create_hosted_zone( + Name="test.zone.", CallerReference=str(hash("test")) + ) + zones = conn.list_hosted_zones_by_name(DNSName="test.zone.") + hosted_zone_id = zones["HostedZones"][0]["Id"] + + # Create geolocation record + conn.change_resource_record_sets( + HostedZoneId=hosted_zone_id, + ChangeBatch={ + "Changes": [ + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "georecord1.test.zone.", + "Type": "A", + "TTL": 10, + "ResourceRecords": [{"Value": "127.0.0.1"}], + "GeoLocation": { + "ContinentCode": "EU" + } + } + }, + { + "Action": "CREATE", + "ResourceRecordSet": { + "Name": "georecord2.test.zone.", + "Type": "A", + "TTL": 10, + "ResourceRecords": [{"Value": "127.0.0.2"}], + "GeoLocation": { + "CountryCode": "US", + "SubdivisionCode": "NY" + } + } + } + ] + } + ) + + response = conn.list_resource_record_sets(HostedZoneId=hosted_zone_id) + rrs = response["ResourceRecordSets"] + rrs[0]["GeoLocation"].should.equal({"ContinentCode": "EU"}) + rrs[1]["GeoLocation"].should.equal({"CountryCode": "US", "SubdivisionCode": "NY"}) + + @mock_route53 def test_change_resource_record_invalid(): conn = boto3.client("route53", region_name="us-east-1") From bebcf52851e5dd094dd8b670f1ff4b65d6599f9d Mon Sep 17 00:00:00 2001 From: Jon Nangle Date: Mon, 2 Mar 2020 13:07:34 +0000 Subject: [PATCH 08/16] Formatting --- tests/test_route53/test_route53.py | 31 +++++++++++------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/tests/test_route53/test_route53.py b/tests/test_route53/test_route53.py index 8cf148c14..8c036441c 100644 --- a/tests/test_route53/test_route53.py +++ b/tests/test_route53/test_route53.py @@ -756,9 +756,7 @@ def test_change_weighted_resource_record_sets(): @mock_route53 def test_failover_record_sets(): conn = boto3.client("route53", region_name="us-east-2") - conn.create_hosted_zone( - Name="test.zone.", CallerReference=str(hash("test")) - ) + conn.create_hosted_zone(Name="test.zone.", CallerReference=str(hash("test"))) zones = conn.list_hosted_zones_by_name(DNSName="test.zone.") hosted_zone_id = zones["HostedZones"][0]["Id"] @@ -774,11 +772,11 @@ def test_failover_record_sets(): "Type": "A", "TTL": 10, "ResourceRecords": [{"Value": "127.0.0.1"}], - "Failover": "PRIMARY" - } + "Failover": "PRIMARY", + }, } ] - } + }, ) response = conn.list_resource_record_sets(HostedZoneId=hosted_zone_id) @@ -789,9 +787,7 @@ def test_failover_record_sets(): @mock_route53 def test_geolocation_record_sets(): conn = boto3.client("route53", region_name="us-east-2") - conn.create_hosted_zone( - Name="test.zone.", CallerReference=str(hash("test")) - ) + conn.create_hosted_zone(Name="test.zone.", CallerReference=str(hash("test"))) zones = conn.list_hosted_zones_by_name(DNSName="test.zone.") hosted_zone_id = zones["HostedZones"][0]["Id"] @@ -807,10 +803,8 @@ def test_geolocation_record_sets(): "Type": "A", "TTL": 10, "ResourceRecords": [{"Value": "127.0.0.1"}], - "GeoLocation": { - "ContinentCode": "EU" - } - } + "GeoLocation": {"ContinentCode": "EU"}, + }, }, { "Action": "CREATE", @@ -819,14 +813,11 @@ def test_geolocation_record_sets(): "Type": "A", "TTL": 10, "ResourceRecords": [{"Value": "127.0.0.2"}], - "GeoLocation": { - "CountryCode": "US", - "SubdivisionCode": "NY" - } - } - } + "GeoLocation": {"CountryCode": "US", "SubdivisionCode": "NY"}, + }, + }, ] - } + }, ) response = conn.list_resource_record_sets(HostedZoneId=hosted_zone_id) From bc1c9a27f1fd2eaf3b41e0e6e5a11da0deab88b0 Mon Sep 17 00:00:00 2001 From: Heyder Date: Wed, 4 Mar 2020 23:17:03 -0300 Subject: [PATCH 09/16] fix use of _get_default result on _get_appended_list --- moto/dynamodb2/models.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 88f750775..747fa93a7 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -450,9 +450,7 @@ class Item(BaseModel): old_list_key = list_append_re.group(1) # old_key could be a function itself (if_not_exists) if old_list_key.startswith("if_not_exists"): - old_list = DynamoType( - expression_attribute_values[self._get_default(old_list_key)] - ) + old_list = DynamoType(self._get_default(old_list_key)) else: old_list = self.attrs[old_list_key.split(".")[0]] if "." in old_list_key: From f8dd5a13c65c8d30f235e99b334c6d00f00a0dac Mon Sep 17 00:00:00 2001 From: Heyder Date: Wed, 4 Mar 2020 23:56:30 -0300 Subject: [PATCH 10/16] fix case if don't have attrs --- moto/dynamodb2/models.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 747fa93a7..7d995486d 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -450,7 +450,11 @@ class Item(BaseModel): old_list_key = list_append_re.group(1) # old_key could be a function itself (if_not_exists) if old_list_key.startswith("if_not_exists"): - old_list = DynamoType(self._get_default(old_list_key)) + old_list = self._get_default(old_list_key) + if not isinstance(old_list, DynamoType): + old_list = DynamoType( + expression_attribute_values[old_list] + ) else: old_list = self.attrs[old_list_key.split(".")[0]] if "." in old_list_key: From 916add9ac54845a676b29d7d4b47bea624c217ab Mon Sep 17 00:00:00 2001 From: Laurie O Date: Thu, 5 Mar 2020 23:37:17 +1000 Subject: [PATCH 11/16] Add SWF domain and type undeprecation Signed-off-by: Laurie O --- moto/swf/models/__init__.py | 13 ++++ moto/swf/responses.py | 23 ++++++ .../test_swf/responses/test_activity_types.py | 74 +++++++++++++++++++ tests/test_swf/responses/test_domains.py | 53 +++++++++++++ .../test_swf/responses/test_workflow_types.py | 72 ++++++++++++++++++ 5 files changed, 235 insertions(+) diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index e5b285f5b..010c8c734 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -121,6 +121,12 @@ class SWFBackend(BaseBackend): raise SWFDomainDeprecatedFault(name) domain.status = "DEPRECATED" + def undeprecate_domain(self, name): + domain = self._get_domain(name) + if domain.status == "REGISTERED": + raise SWFDomainAlreadyExistsFault(name) + domain.status = "REGISTERED" + def describe_domain(self, name): return self._get_domain(name) @@ -148,6 +154,13 @@ class SWFBackend(BaseBackend): raise SWFTypeDeprecatedFault(_type) _type.status = "DEPRECATED" + def undeprecate_type(self, kind, domain_name, name, version): + domain = self._get_domain(domain_name) + _type = domain.get_type(kind, name, version) + if _type.status == "REGISTERED": + raise SWFTypeAlreadyExistsFault(_type) + _type.status = "REGISTERED" + def describe_type(self, kind, domain_name, name, version): domain = self._get_domain(domain_name) return domain.get_type(kind, name, version) diff --git a/moto/swf/responses.py b/moto/swf/responses.py index c8c601fa7..17ec7281a 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -92,6 +92,17 @@ class SWFResponse(BaseResponse): self.swf_backend.deprecate_type(kind, domain, name, version) return "" + def _undeprecate_type(self, kind): + domain = self._params["domain"] + _type_args = self._params["{0}Type".format(kind)] + name = _type_args["name"] + version = _type_args["version"] + self._check_string(domain) + self._check_string(name) + self._check_string(version) + self.swf_backend.undeprecate_type(kind, domain, name, version) + return "" + # TODO: implement pagination def list_domains(self): status = self._params["registrationStatus"] @@ -219,6 +230,12 @@ class SWFResponse(BaseResponse): self.swf_backend.deprecate_domain(name) return "" + def undeprecate_domain(self): + name = self._params["name"] + self._check_string(name) + self.swf_backend.undeprecate_domain(name) + return "" + def describe_domain(self): name = self._params["name"] self._check_string(name) @@ -278,6 +295,9 @@ class SWFResponse(BaseResponse): def deprecate_activity_type(self): return self._deprecate_type("activity") + def undeprecate_activity_type(self): + return self._undeprecate_type("activity") + def describe_activity_type(self): return self._describe_type("activity") @@ -333,6 +353,9 @@ class SWFResponse(BaseResponse): def deprecate_workflow_type(self): return self._deprecate_type("workflow") + def undeprecate_workflow_type(self): + return self._undeprecate_type("workflow") + def describe_workflow_type(self): return self._describe_type("workflow") diff --git a/tests/test_swf/responses/test_activity_types.py b/tests/test_swf/responses/test_activity_types.py index 3fa9ad6b1..d49e5d4cb 100644 --- a/tests/test_swf/responses/test_activity_types.py +++ b/tests/test_swf/responses/test_activity_types.py @@ -1,8 +1,11 @@ import boto from boto.swf.exceptions import SWFResponseError +import boto3 +from botocore.exceptions import ClientError import sure # noqa from moto import mock_swf_deprecated +from moto import mock_swf # RegisterActivityType endpoint @@ -110,6 +113,77 @@ def test_deprecate_non_existent_activity_type(): ).should.throw(SWFResponseError) +# DeprecateActivityType endpoint +@mock_swf +def test_undeprecate_activity_type(): + client = boto3.client("swf", region_name="us-east-1") + client.register_domain( + name="test-domain", workflowExecutionRetentionPeriodInDays="60" + ) + client.register_activity_type( + domain="test-domain", name="test-activity", version="v1.0" + ) + client.deprecate_activity_type( + domain="test-domain", activityType={"name": "test-activity", "version": "v1.0"} + ) + client.undeprecate_activity_type( + domain="test-domain", activityType={"name": "test-activity", "version": "v1.0"} + ) + + resp = client.describe_activity_type( + domain="test-domain", activityType={"name": "test-activity", "version": "v1.0"} + ) + resp["typeInfo"]["status"].should.equal("REGISTERED") + + +@mock_swf +def test_undeprecate_already_undeprecated_activity_type(): + client = boto3.client("swf", region_name="us-east-1") + client.register_domain( + name="test-domain", workflowExecutionRetentionPeriodInDays="60" + ) + client.register_activity_type( + domain="test-domain", name="test-activity", version="v1.0" + ) + client.deprecate_activity_type( + domain="test-domain", activityType={"name": "test-activity", "version": "v1.0"} + ) + client.undeprecate_activity_type( + domain="test-domain", activityType={"name": "test-activity", "version": "v1.0"} + ) + + client.undeprecate_activity_type.when.called_with( + domain="test-domain", activityType={"name": "test-activity", "version": "v1.0"} + ).should.throw(ClientError) + + +@mock_swf +def test_undeprecate_never_deprecated_activity_type(): + client = boto3.client("swf", region_name="us-east-1") + client.register_domain( + name="test-domain", workflowExecutionRetentionPeriodInDays="60" + ) + client.register_activity_type( + domain="test-domain", name="test-activity", version="v1.0" + ) + + client.undeprecate_activity_type.when.called_with( + domain="test-domain", activityType={"name": "test-activity", "version": "v1.0"} + ).should.throw(ClientError) + + +@mock_swf +def test_undeprecate_non_existent_activity_type(): + client = boto3.client("swf", region_name="us-east-1") + client.register_domain( + name="test-domain", workflowExecutionRetentionPeriodInDays="60" + ) + + client.undeprecate_activity_type.when.called_with( + domain="test-domain", activityType={"name": "test-activity", "version": "v1.0"} + ).should.throw(ClientError) + + # DescribeActivityType endpoint @mock_swf_deprecated def test_describe_activity_type(): diff --git a/tests/test_swf/responses/test_domains.py b/tests/test_swf/responses/test_domains.py index 199219d27..59ba551a6 100644 --- a/tests/test_swf/responses/test_domains.py +++ b/tests/test_swf/responses/test_domains.py @@ -1,8 +1,11 @@ import boto from boto.swf.exceptions import SWFResponseError +import boto3 +from botocore.exceptions import ClientError import sure # noqa from moto import mock_swf_deprecated +from moto import mock_swf # RegisterDomain endpoint @@ -94,6 +97,56 @@ def test_deprecate_non_existent_domain(): ) +# UndeprecateDomain endpoint +@mock_swf +def test_undeprecate_domain(): + client = boto3.client("swf", region_name="us-east-1") + client.register_domain( + name="test-domain", workflowExecutionRetentionPeriodInDays="60" + ) + client.deprecate_domain(name="test-domain") + client.undeprecate_domain(name="test-domain") + + resp = client.describe_domain(name="test-domain") + + resp["domainInfo"]["status"].should.equal("REGISTERED") + + +@mock_swf +def test_undeprecate_already_undeprecated_domain(): + client = boto3.client("swf", region_name="us-east-1") + client.register_domain( + name="test-domain", workflowExecutionRetentionPeriodInDays="60" + ) + client.deprecate_domain(name="test-domain") + client.undeprecate_domain(name="test-domain") + + client.undeprecate_domain.when.called_with(name="test-domain").should.throw( + ClientError + ) + + +@mock_swf +def test_undeprecate_never_deprecated_domain(): + client = boto3.client("swf", region_name="us-east-1") + client.register_domain( + name="test-domain", workflowExecutionRetentionPeriodInDays="60" + ) + + client.undeprecate_domain.when.called_with(name="test-domain").should.throw( + ClientError + ) + + +@mock_swf +def test_undeprecate_non_existent_domain(): + client = boto3.client("swf", region_name="us-east-1") + + client.undeprecate_domain.when.called_with(name="non-existent").should.throw( + ClientError + ) + + # DescribeDomain endpoint @mock_swf_deprecated def test_describe_domain(): diff --git a/tests/test_swf/responses/test_workflow_types.py b/tests/test_swf/responses/test_workflow_types.py index 72aa814d2..e1990596b 100644 --- a/tests/test_swf/responses/test_workflow_types.py +++ b/tests/test_swf/responses/test_workflow_types.py @@ -5,6 +5,7 @@ import boto3 from moto import mock_swf_deprecated from moto import mock_swf from boto.swf.exceptions import SWFResponseError +from botocore.exceptions import ClientError # RegisterWorkflowType endpoint @@ -112,6 +113,77 @@ def test_deprecate_non_existent_workflow_type(): ).should.throw(SWFResponseError) +# UndeprecateWorkflowType endpoint +@mock_swf +def test_undeprecate_workflow_type(): + client = boto3.client("swf", region_name="us-east-1") + client.register_domain( + name="test-domain", workflowExecutionRetentionPeriodInDays="60" + ) + client.register_workflow_type( + domain="test-domain", name="test-workflow", version="v1.0" + ) + client.deprecate_workflow_type( + domain="test-domain", workflowType={"name": "test-workflow", "version": "v1.0"} + ) + client.undeprecate_workflow_type( + domain="test-domain", workflowType={"name": "test-workflow", "version": "v1.0"} + ) + + resp = client.describe_workflow_type( + domain="test-domain", workflowType={"name": "test-workflow", "version": "v1.0"} + ) + resp["typeInfo"]["status"].should.equal("REGISTERED") + + +@mock_swf +def test_undeprecate_already_undeprecated_workflow_type(): + client = boto3.client("swf", region_name="us-east-1") + client.register_domain( + name="test-domain", workflowExecutionRetentionPeriodInDays="60" + ) + client.register_workflow_type( + domain="test-domain", name="test-workflow", version="v1.0" + ) + client.deprecate_workflow_type( + domain="test-domain", workflowType={"name": "test-workflow", "version": "v1.0"} + ) + client.undeprecate_workflow_type( + domain="test-domain", workflowType={"name": "test-workflow", "version": "v1.0"} + ) + + client.undeprecate_workflow_type.when.called_with( + domain="test-domain", workflowType={"name": "test-workflow", "version": "v1.0"} + ).should.throw(ClientError) + + +@mock_swf +def test_undeprecate_never_deprecated_workflow_type(): + client = boto3.client("swf", region_name="us-east-1") + client.register_domain( + name="test-domain", workflowExecutionRetentionPeriodInDays="60" + ) + client.register_workflow_type( + domain="test-domain", name="test-workflow", version="v1.0" + ) + + client.undeprecate_workflow_type.when.called_with( + domain="test-domain", workflowType={"name": "test-workflow", "version": "v1.0"} + ).should.throw(ClientError) + + +@mock_swf +def test_undeprecate_non_existent_workflow_type(): + client = boto3.client("swf", region_name="us-east-1") + client.register_domain( + name="test-domain", workflowExecutionRetentionPeriodInDays="60" + ) + + client.undeprecate_workflow_type.when.called_with( + domain="test-domain", workflowType={"name": "test-workflow", "version": "v1.0"} + ).should.throw(ClientError) + + # DescribeWorkflowType endpoint @mock_swf_deprecated def test_describe_workflow_type(): From 27e7336debcfc2d13e2b62738cb9ee7931fa165b Mon Sep 17 00:00:00 2001 From: Laurie O Date: Thu, 5 Mar 2020 23:39:26 +1000 Subject: [PATCH 12/16] Mark new SWF end-points as implemented Signed-off-by: Laurie O --- IMPLEMENTATION_COVERAGE.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index a863d483d..a22cc3bfb 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -7272,9 +7272,9 @@ - [X] start_workflow_execution - [ ] tag_resource - [X] terminate_workflow_execution -- [ ] undeprecate_activity_type -- [ ] undeprecate_domain -- [ ] undeprecate_workflow_type +- [X] undeprecate_activity_type +- [X] undeprecate_domain +- [X] undeprecate_workflow_type - [ ] untag_resource ## textract From b19cf8a08538425f5210d5e5cd0447065ef318ca Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Thu, 5 Mar 2020 15:49:30 +0000 Subject: [PATCH 13/16] #2774 - Run Policy test on different method --- tests/test_core/test_auth.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_core/test_auth.py b/tests/test_core/test_auth.py index a8fde5d8c..767f743dc 100644 --- a/tests/test_core/test_auth.py +++ b/tests/test_core/test_auth.py @@ -275,7 +275,7 @@ def test_access_denied_with_not_allowing_policy(): inline_policy_document = { "Version": "2012-10-17", "Statement": [ - {"Effect": "Allow", "Action": ["ec2:Describe*"], "Resource": "*"} + {"Effect": "Allow", "Action": ["ec2:Run*"], "Resource": "*"} ], } access_key = create_user_with_access_key_and_inline_policy( @@ -288,12 +288,12 @@ def test_access_denied_with_not_allowing_policy(): aws_secret_access_key=access_key["SecretAccessKey"], ) with assert_raises(ClientError) as ex: - client.run_instances(MaxCount=1, MinCount=1) + client.describe_instances() ex.exception.response["Error"]["Code"].should.equal("AccessDenied") ex.exception.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(403) ex.exception.response["Error"]["Message"].should.equal( "User: arn:aws:iam::{account_id}:user/{user_name} is not authorized to perform: {operation}".format( - account_id=ACCOUNT_ID, user_name=user_name, operation="ec2:RunInstances" + account_id=ACCOUNT_ID, user_name=user_name, operation="ec2:DescribeInstances" ) ) From c6b66cb001c3a2b00d5244966fe86fd37dff099a Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Thu, 5 Mar 2020 17:22:54 +0000 Subject: [PATCH 14/16] Linting --- tests/test_core/test_auth.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_core/test_auth.py b/tests/test_core/test_auth.py index 767f743dc..29273cea7 100644 --- a/tests/test_core/test_auth.py +++ b/tests/test_core/test_auth.py @@ -274,9 +274,7 @@ def test_access_denied_with_not_allowing_policy(): user_name = "test-user" inline_policy_document = { "Version": "2012-10-17", - "Statement": [ - {"Effect": "Allow", "Action": ["ec2:Run*"], "Resource": "*"} - ], + "Statement": [{"Effect": "Allow", "Action": ["ec2:Run*"], "Resource": "*"}], } access_key = create_user_with_access_key_and_inline_policy( user_name, inline_policy_document @@ -293,7 +291,9 @@ def test_access_denied_with_not_allowing_policy(): ex.exception.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(403) ex.exception.response["Error"]["Message"].should.equal( "User: arn:aws:iam::{account_id}:user/{user_name} is not authorized to perform: {operation}".format( - account_id=ACCOUNT_ID, user_name=user_name, operation="ec2:DescribeInstances" + account_id=ACCOUNT_ID, + user_name=user_name, + operation="ec2:DescribeInstances", ) ) From fa7f83bc2f10c9157e8b9484e5ef1219e54b2af1 Mon Sep 17 00:00:00 2001 From: "heyder.dias" Date: Thu, 5 Mar 2020 17:05:00 -0300 Subject: [PATCH 15/16] add test to nested if_not_exists and property already exists --- tests/test_dynamodb2/test_dynamodb.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 180f460c0..d36fdc7fa 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -3634,6 +3634,31 @@ def test_update_supports_list_append_with_nested_if_not_exists_operation(): ) +@mock_dynamodb2 +def test_update_supports_list_append_with_nested_if_not_exists_operation_and_property_already_exists(): + dynamo = boto3.resource("dynamodb", region_name="us-west-1") + table_name = "test" + + dynamo.create_table( + TableName=table_name, + AttributeDefinitions=[{"AttributeName": "Id", "AttributeType": "S"}], + KeySchema=[{"AttributeName": "Id", "KeyType": "HASH"}], + ProvisionedThroughput={"ReadCapacityUnits": 20, "WriteCapacityUnits": 20}, + ) + + table = dynamo.Table(table_name) + + table.put_item(Item={"Id": "item-id", "event_history":["other_value"]}) + table.update_item( + Key={"Id": "item-id"}, + UpdateExpression="SET event_history = list_append(if_not_exists(event_history, :empty_list), :new_value)", + ExpressionAttributeValues={":empty_list": [], ":new_value": ["some_value"]}, + ) + table.get_item(Key={"Id": "item-id"})["Item"].should.equal( + {"Id": "item-id", "event_history": ["other_value", "some_value"]} + ) + + @mock_dynamodb2 def test_update_catches_invalid_list_append_operation(): client = boto3.client("dynamodb", region_name="us-east-1") From f5080e539dd140e556c0a280d9770b552d9aeaba Mon Sep 17 00:00:00 2001 From: "heyder.dias" Date: Thu, 5 Mar 2020 18:39:20 -0300 Subject: [PATCH 16/16] fix lint check --- moto/dynamodb2/models.py | 4 +--- tests/test_dynamodb2/test_dynamodb.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 7d995486d..8e5a61755 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -452,9 +452,7 @@ class Item(BaseModel): if old_list_key.startswith("if_not_exists"): old_list = self._get_default(old_list_key) if not isinstance(old_list, DynamoType): - old_list = DynamoType( - expression_attribute_values[old_list] - ) + old_list = DynamoType(expression_attribute_values[old_list]) else: old_list = self.attrs[old_list_key.split(".")[0]] if "." in old_list_key: diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index d36fdc7fa..428b58f81 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -3648,7 +3648,7 @@ def test_update_supports_list_append_with_nested_if_not_exists_operation_and_pro table = dynamo.Table(table_name) - table.put_item(Item={"Id": "item-id", "event_history":["other_value"]}) + table.put_item(Item={"Id": "item-id", "event_history": ["other_value"]}) table.update_item( Key={"Id": "item-id"}, UpdateExpression="SET event_history = list_append(if_not_exists(event_history, :empty_list), :new_value)",