From 8e3fd6c7dea1e2bb78dc13e6325e385a9f22a708 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Wed, 30 Sep 2015 09:08:24 +0200 Subject: [PATCH 01/94] Add SWF endpoints: RegisterDomain, DeprecateDomain, ListDomains, DescribeDomain --- moto/__init__.py | 1 + moto/swf/__init__.py | 12 ++++ moto/swf/exceptions.py | 32 +++++++++++ moto/swf/models.py | 67 ++++++++++++++++++++++ moto/swf/responses.py | 103 +++++++++++++++++++++++++++++++++ moto/swf/urls.py | 9 +++ tests/test_swf/test_swf.py | 113 +++++++++++++++++++++++++++++++++++++ 7 files changed, 337 insertions(+) create mode 100644 moto/swf/__init__.py create mode 100644 moto/swf/exceptions.py create mode 100644 moto/swf/models.py create mode 100644 moto/swf/responses.py create mode 100644 moto/swf/urls.py create mode 100644 tests/test_swf/test_swf.py diff --git a/moto/__init__.py b/moto/__init__.py index be7cfdda3..70ba9ca20 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -28,3 +28,4 @@ from .sns import mock_sns # flake8: noqa from .sqs import mock_sqs # flake8: noqa from .sts import mock_sts # flake8: noqa from .route53 import mock_route53 # flake8: noqa +from .swf import mock_swf # flake8: noqa diff --git a/moto/swf/__init__.py b/moto/swf/__init__.py new file mode 100644 index 000000000..7e43ca392 --- /dev/null +++ b/moto/swf/__init__.py @@ -0,0 +1,12 @@ +from __future__ import unicode_literals +from .models import swf_backends +from ..core.models import MockAWS + +swf_backend = swf_backends['us-east-1'] + + +def mock_swf(func=None): + if func: + return MockAWS(swf_backends)(func) + else: + return MockAWS(swf_backends) diff --git a/moto/swf/exceptions.py b/moto/swf/exceptions.py new file mode 100644 index 000000000..bfbb96020 --- /dev/null +++ b/moto/swf/exceptions.py @@ -0,0 +1,32 @@ +from __future__ import unicode_literals + +from boto.exception import JSONResponseError + + +class SWFClientError(JSONResponseError): + def __init__(self, message, __type): + super(SWFClientError, self).__init__( + 400, "Bad Request", + body={"message": message, "__type": __type} + ) + + +class SWFUnknownResourceFault(SWFClientError): + def __init__(self, resource_type, resource_name): + super(SWFUnknownResourceFault, self).__init__( + "Unknown {}: {}".format(resource_type, resource_name), + "com.amazonaws.swf.base.model#UnknownResourceFault") + + +class SWFDomainAlreadyExistsFault(SWFClientError): + def __init__(self, domain_name): + super(SWFDomainAlreadyExistsFault, self).__init__( + domain_name, + "com.amazonaws.swf.base.model#DomainAlreadyExistsFault") + + +class SWFDomainDeprecatedFault(SWFClientError): + def __init__(self, domain_name): + super(SWFDomainDeprecatedFault, self).__init__( + domain_name, + "com.amazonaws.swf.base.model#DomainDeprecatedFault") diff --git a/moto/swf/models.py b/moto/swf/models.py new file mode 100644 index 000000000..61ed508aa --- /dev/null +++ b/moto/swf/models.py @@ -0,0 +1,67 @@ +from __future__ import unicode_literals + +import boto.swf + +from moto.core import BaseBackend +from .exceptions import ( + SWFUnknownResourceFault, + SWFDomainAlreadyExistsFault, + SWFDomainDeprecatedFault, +) + + +class Domain(object): + def __init__(self, name, retention, description=None): + self.name = name + self.retention = retention + self.description = description + self.status = "REGISTERED" + + def __repr__(self): + return "Domain(name: %s, retention: %s, description: %s)" % (self.name, self.retention, self.description) + + +class SWFBackend(BaseBackend): + def __init__(self, region_name): + self.region_name = region_name + self.domains = [] + super(SWFBackend, self).__init__() + + def reset(self): + region_name = self.region_name + self.__dict__ = {} + self.__init__(region_name) + + def _get_domain(self, name, ignore_empty=False): + matching = [domain for domain in self.domains if domain.name == name] + if not matching and not ignore_empty: + raise SWFUnknownResourceFault("domain", name) + if matching: + return matching[0] + return None + + def list_domains(self, status): + return [domain for domain in self.domains + if domain.status == status] + + def register_domain(self, name, workflow_execution_retention_period_in_days, + description=None): + if self._get_domain(name, ignore_empty=True): + raise SWFDomainAlreadyExistsFault(name) + domain = Domain(name, workflow_execution_retention_period_in_days, + description) + self.domains.append(domain) + + def deprecate_domain(self, name): + domain = self._get_domain(name) + if domain.status == "DEPRECATED": + raise SWFDomainDeprecatedFault(name) + domain.status = "DEPRECATED" + + def describe_domain(self, name): + return self._get_domain(name) + + +swf_backends = {} +for region in boto.swf.regions(): + swf_backends[region.name] = SWFBackend(region.name) diff --git a/moto/swf/responses.py b/moto/swf/responses.py new file mode 100644 index 000000000..ec23a9536 --- /dev/null +++ b/moto/swf/responses.py @@ -0,0 +1,103 @@ +import json +import logging +import six + +from moto.core.responses import BaseResponse +from werkzeug.exceptions import HTTPException +from moto.core.utils import camelcase_to_underscores, method_names_from_class + +from .models import swf_backends + + +class SWFResponse(BaseResponse): + + @property + def swf_backend(self): + return swf_backends[self.region] + + # SWF actions are not dispatched via URLs but via a specific header called + # "x-amz-target", in the form of com.amazonaws.swf.service.model.SimpleWorkflowService. + # This is not supported directly in BaseResponse sor for now we override + # the call_action() method + # See: http://docs.aws.amazon.com/amazonswf/latest/developerguide/UsingJSON-swf.html + def call_action(self): + headers = self.response_headers + # Headers are case-insensitive. Probably a better way to do this. + match = self.headers.get('x-amz-target') or self.headers.get('X-Amz-Target') + if match: + # TODO: see if we can call "[-1]" in BaseResponse, which would + # allow to remove that + action = match.split(".")[-1] + + action = camelcase_to_underscores(action) + method_names = method_names_from_class(self.__class__) + if action in method_names: + method = getattr(self, action) + try: + response = method() + except HTTPException as http_error: + response = http_error.description, dict(status=http_error.code) + if isinstance(response, six.string_types): + return 200, headers, response + else: + body, new_headers = response + status = new_headers.get('status', 200) + headers.update(new_headers) + return status, headers, body + raise NotImplementedError("The {0} action has not been implemented".format(action)) + + # SWF parameters are passed through a JSON body, so let's ease retrieval + @property + def _params(self): + return json.loads(self.body) + + def list_domains(self): + status = self._params.get("registrationStatus") + domains = self.swf_backend.list_domains(status) + template = self.response_template(LIST_DOMAINS_TEMPLATE) + return template.render(domains=domains) + + def register_domain(self): + name = self._params.get("name") + description = self._params.get("description") + retention = self._params.get("workflowExecutionRetentionPeriodInDays") + domain = self.swf_backend.register_domain(name, retention, + description=description) + template = self.response_template("") + return template.render() + + def deprecate_domain(self): + name = self._params.get("name") + domain = self.swf_backend.deprecate_domain(name) + template = self.response_template("") + return template.render() + + def describe_domain(self): + name = self._params.get("name") + domain = self.swf_backend.describe_domain(name) + template = self.response_template(DESCRIBE_DOMAIN_TEMPLATE) + return template.render(domain=domain) + + +LIST_DOMAINS_TEMPLATE = """{ + "domainInfos": [ + {% for domain in domains %} + { + "description": "{{ domain.description }}", + "name": "{{ domain.name }}", + "status": "{{ domain.status }}" + } + {% endfor %} + ] +}""" + +DESCRIBE_DOMAIN_TEMPLATE = """{ + "configuration": { + "workflowExecutionRetentionPeriodInDays": "{{ domain.retention }}" + }, + "domainInfo": { + "description": "{{ domain.description }}", + "name": "{{ domain.name }}", + "status": "{{ domain.status }}" + } +}""" diff --git a/moto/swf/urls.py b/moto/swf/urls.py new file mode 100644 index 000000000..582c874fc --- /dev/null +++ b/moto/swf/urls.py @@ -0,0 +1,9 @@ +from .responses import SWFResponse + +url_bases = [ + "https?://swf.(.+).amazonaws.com", +] + +url_paths = { + '{0}/$': SWFResponse.dispatch, +} diff --git a/tests/test_swf/test_swf.py b/tests/test_swf/test_swf.py new file mode 100644 index 000000000..4f0617fc1 --- /dev/null +++ b/tests/test_swf/test_swf.py @@ -0,0 +1,113 @@ +import boto +from nose.tools import assert_raises +from sure import expect + +from moto import mock_swf +from moto.swf.exceptions import ( + SWFUnknownResourceFault, + SWFDomainAlreadyExistsFault, + SWFDomainDeprecatedFault, +) + + +# RegisterDomain endpoint +# ListDomain endpoint +@mock_swf +def test_register_domain(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", 60, description="A test domain") + + all_domains = conn.list_domains("REGISTERED") + domain = all_domains["domainInfos"][0] + + domain["name"].should.equal("test-domain") + domain["status"].should.equal("REGISTERED") + domain["description"].should.equal("A test domain") + +@mock_swf +def test_register_already_existing_domain(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", 60, description="A test domain") + + with assert_raises(SWFDomainAlreadyExistsFault) as err: + conn.register_domain("test-domain", 60, description="A test domain") + + ex = err.exception + ex.status.should.equal(400) + ex.error_code.should.equal("DomainAlreadyExistsFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#DomainAlreadyExistsFault", + "message": "test-domain" + }) + + +# DeprecateDomain endpoint +@mock_swf +def test_deprecate_domain(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", 60, description="A test domain") + conn.deprecate_domain("test-domain") + + all_domains = conn.list_domains("DEPRECATED") + domain = all_domains["domainInfos"][0] + + domain["name"].should.equal("test-domain") + +@mock_swf +def test_deprecate_already_deprecated_domain(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", 60, description="A test domain") + conn.deprecate_domain("test-domain") + + with assert_raises(SWFDomainDeprecatedFault) as err: + conn.deprecate_domain("test-domain") + + ex = err.exception + ex.status.should.equal(400) + ex.error_code.should.equal("DomainDeprecatedFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#DomainDeprecatedFault", + "message": "test-domain" + }) + +@mock_swf +def test_deprecate_non_existent_domain(): + conn = boto.connect_swf("the_key", "the_secret") + + with assert_raises(SWFUnknownResourceFault) as err: + conn.deprecate_domain("non-existent") + + ex = err.exception + ex.status.should.equal(400) + ex.error_code.should.equal("UnknownResourceFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#UnknownResourceFault", + "message": "Unknown domain: non-existent" + }) + +# DescribeDomain endpoint +@mock_swf +def test_describe_domain(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", 60, description="A test domain") + + domain = conn.describe_domain("test-domain") + domain["configuration"]["workflowExecutionRetentionPeriodInDays"].should.equal("60") + domain["domainInfo"]["description"].should.equal("A test domain") + domain["domainInfo"]["name"].should.equal("test-domain") + domain["domainInfo"]["status"].should.equal("REGISTERED") + +@mock_swf +def test_describe_non_existent_domain(): + conn = boto.connect_swf("the_key", "the_secret") + + with assert_raises(SWFUnknownResourceFault) as err: + conn.describe_domain("non-existent") + + ex = err.exception + ex.status.should.equal(400) + ex.error_code.should.equal("UnknownResourceFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#UnknownResourceFault", + "message": "Unknown domain: non-existent" + }) From 5392978eaf98d468feee69ed33cedbfb0616f6a6 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Wed, 30 Sep 2015 11:16:02 +0200 Subject: [PATCH 02/94] Check parameters are strings on SWF endpoints SWF endpoints raise a 400 Bad Request for non-string types, and boto doesn't enforce it as of today, so better have some safety nets in moto to avoid this common mistake. Example exception raised by Boto: SWFResponseError: SWFResponseError: 400 Bad Request {u'Message': u'class java.lang.Short can not be converted to an String', u'__type': u'com.amazon.coral.service#SerializationException'} --- moto/swf/exceptions.py | 10 ++++++++++ moto/swf/models.py | 12 ++++++++++++ tests/test_swf/test_swf.py | 28 ++++++++++++++++++++++------ 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/moto/swf/exceptions.py b/moto/swf/exceptions.py index bfbb96020..f58c0360a 100644 --- a/moto/swf/exceptions.py +++ b/moto/swf/exceptions.py @@ -30,3 +30,13 @@ class SWFDomainDeprecatedFault(SWFClientError): super(SWFDomainDeprecatedFault, self).__init__( domain_name, "com.amazonaws.swf.base.model#DomainDeprecatedFault") + + +class SWFSerializationException(JSONResponseError): + def __init__(self): + message = "class java.lang.Foo can not be converted to an String (not a real SWF exception)" + __type = "com.amazonaws.swf.base.model#SerializationException" + super(SWFSerializationException, self).__init__( + 400, "Bad Request", + body={"Message": message, "__type": __type} + ) diff --git a/moto/swf/models.py b/moto/swf/models.py index 61ed508aa..880e00b3d 100644 --- a/moto/swf/models.py +++ b/moto/swf/models.py @@ -7,6 +7,7 @@ from .exceptions import ( SWFUnknownResourceFault, SWFDomainAlreadyExistsFault, SWFDomainDeprecatedFault, + SWFSerializationException, ) @@ -40,12 +41,21 @@ class SWFBackend(BaseBackend): return matching[0] return None + def _check_string(self, parameter): + if not isinstance(parameter, basestring): + raise SWFSerializationException() + def list_domains(self, status): + self._check_string(status) return [domain for domain in self.domains if domain.status == status] def register_domain(self, name, workflow_execution_retention_period_in_days, description=None): + self._check_string(name) + self._check_string(workflow_execution_retention_period_in_days) + if description: + self._check_string(description) if self._get_domain(name, ignore_empty=True): raise SWFDomainAlreadyExistsFault(name) domain = Domain(name, workflow_execution_retention_period_in_days, @@ -53,12 +63,14 @@ class SWFBackend(BaseBackend): self.domains.append(domain) def deprecate_domain(self, name): + self._check_string(name) domain = self._get_domain(name) if domain.status == "DEPRECATED": raise SWFDomainDeprecatedFault(name) domain.status = "DEPRECATED" def describe_domain(self, name): + self._check_string(name) return self._get_domain(name) diff --git a/tests/test_swf/test_swf.py b/tests/test_swf/test_swf.py index 4f0617fc1..1a43e4cd0 100644 --- a/tests/test_swf/test_swf.py +++ b/tests/test_swf/test_swf.py @@ -7,6 +7,7 @@ from moto.swf.exceptions import ( SWFUnknownResourceFault, SWFDomainAlreadyExistsFault, SWFDomainDeprecatedFault, + SWFSerializationException, ) @@ -15,7 +16,7 @@ from moto.swf.exceptions import ( @mock_swf def test_register_domain(): conn = boto.connect_swf("the_key", "the_secret") - conn.register_domain("test-domain", 60, description="A test domain") + conn.register_domain("test-domain", "60", description="A test domain") all_domains = conn.list_domains("REGISTERED") domain = all_domains["domainInfos"][0] @@ -27,10 +28,10 @@ def test_register_domain(): @mock_swf def test_register_already_existing_domain(): conn = boto.connect_swf("the_key", "the_secret") - conn.register_domain("test-domain", 60, description="A test domain") + conn.register_domain("test-domain", "60", description="A test domain") with assert_raises(SWFDomainAlreadyExistsFault) as err: - conn.register_domain("test-domain", 60, description="A test domain") + conn.register_domain("test-domain", "60", description="A test domain") ex = err.exception ex.status.should.equal(400) @@ -40,12 +41,27 @@ def test_register_already_existing_domain(): "message": "test-domain" }) +@mock_swf +def test_register_with_wrong_parameter_type(): + conn = boto.connect_swf("the_key", "the_secret") + + with assert_raises(SWFSerializationException) as err: + conn.register_domain("test-domain", 60, description="A test domain") + + ex = err.exception + ex.status.should.equal(400) + ex.error_code.should.equal("SerializationException") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#SerializationException", + "Message": "class java.lang.Foo can not be converted to an String (not a real SWF exception)" + }) + # DeprecateDomain endpoint @mock_swf def test_deprecate_domain(): conn = boto.connect_swf("the_key", "the_secret") - conn.register_domain("test-domain", 60, description="A test domain") + conn.register_domain("test-domain", "60", description="A test domain") conn.deprecate_domain("test-domain") all_domains = conn.list_domains("DEPRECATED") @@ -56,7 +72,7 @@ def test_deprecate_domain(): @mock_swf def test_deprecate_already_deprecated_domain(): conn = boto.connect_swf("the_key", "the_secret") - conn.register_domain("test-domain", 60, description="A test domain") + conn.register_domain("test-domain", "60", description="A test domain") conn.deprecate_domain("test-domain") with assert_raises(SWFDomainDeprecatedFault) as err: @@ -89,7 +105,7 @@ def test_deprecate_non_existent_domain(): @mock_swf def test_describe_domain(): conn = boto.connect_swf("the_key", "the_secret") - conn.register_domain("test-domain", 60, description="A test domain") + conn.register_domain("test-domain", "60", description="A test domain") domain = conn.describe_domain("test-domain") domain["configuration"]["workflowExecutionRetentionPeriodInDays"].should.equal("60") From 9440531d0c1d66ebaf740ebd36502c6e66b513f4 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Wed, 30 Sep 2015 11:26:25 +0200 Subject: [PATCH 03/94] Move SWF domain related tests in their own file It will simplify other objects integration --- tests/test_swf/{test_swf.py => test_domains.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/test_swf/{test_swf.py => test_domains.py} (100%) diff --git a/tests/test_swf/test_swf.py b/tests/test_swf/test_domains.py similarity index 100% rename from tests/test_swf/test_swf.py rename to tests/test_swf/test_domains.py From 49bbd7399edc69a28fbcdc20f8d3f9c01b1c8204 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Wed, 30 Sep 2015 11:28:40 +0200 Subject: [PATCH 04/94] Add some TODO comments in SWF mocks --- moto/swf/responses.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/moto/swf/responses.py b/moto/swf/responses.py index ec23a9536..6bfce1ffb 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -51,6 +51,8 @@ class SWFResponse(BaseResponse): def _params(self): return json.loads(self.body) + # TODO: implement "reverseOrder" option + # TODO: implement pagination def list_domains(self): status = self._params.get("registrationStatus") domains = self.swf_backend.list_domains(status) From 2c3b286b6b94a4329123a4b695ab30fa7fa48e95 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Wed, 30 Sep 2015 11:40:33 +0200 Subject: [PATCH 05/94] Improve SWF Domain representation --- moto/swf/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/swf/models.py b/moto/swf/models.py index 880e00b3d..c1aaff1d2 100644 --- a/moto/swf/models.py +++ b/moto/swf/models.py @@ -19,7 +19,7 @@ class Domain(object): self.status = "REGISTERED" def __repr__(self): - return "Domain(name: %s, retention: %s, description: %s)" % (self.name, self.retention, self.description) + return "Domain(name: %(name)s, status: %(status)s)" % self.__dict__ class SWFBackend(BaseBackend): From 3e2c7dec83c34cce6b541924e5ea25c5067441cb Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Wed, 30 Sep 2015 12:50:17 +0200 Subject: [PATCH 06/94] Fix json template for listing SWF domains --- moto/swf/responses.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/moto/swf/responses.py b/moto/swf/responses.py index 6bfce1ffb..57cebd7b6 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -83,13 +83,13 @@ class SWFResponse(BaseResponse): LIST_DOMAINS_TEMPLATE = """{ "domainInfos": [ - {% for domain in domains %} + {%- for domain in domains %} { "description": "{{ domain.description }}", "name": "{{ domain.name }}", "status": "{{ domain.status }}" - } - {% endfor %} + }{% if not loop.last %},{% endif %} + {%- endfor %} ] }""" From cb46eac5131febc01a620a2ca4241169277edf49 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Wed, 30 Sep 2015 12:54:54 +0200 Subject: [PATCH 07/94] Implement naive reverseOrder option for SWF's ListDomains endpoint --- moto/swf/models.py | 10 +++++++--- moto/swf/responses.py | 4 ++-- tests/test_swf/test_domains.py | 25 ++++++++++++++++++++++++- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/moto/swf/models.py b/moto/swf/models.py index c1aaff1d2..db1ca94fb 100644 --- a/moto/swf/models.py +++ b/moto/swf/models.py @@ -45,10 +45,14 @@ class SWFBackend(BaseBackend): if not isinstance(parameter, basestring): raise SWFSerializationException() - def list_domains(self, status): + def list_domains(self, status, reverse_order=None): self._check_string(status) - return [domain for domain in self.domains - if domain.status == status] + domains = [domain for domain in self.domains + if domain.status == status] + domains = sorted(domains, key=lambda domain: domain.name) + if reverse_order: + domains = reversed(domains) + return domains def register_domain(self, name, workflow_execution_retention_period_in_days, description=None): diff --git a/moto/swf/responses.py b/moto/swf/responses.py index 57cebd7b6..8e2dae712 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -51,11 +51,11 @@ class SWFResponse(BaseResponse): def _params(self): return json.loads(self.body) - # TODO: implement "reverseOrder" option # TODO: implement pagination def list_domains(self): status = self._params.get("registrationStatus") - domains = self.swf_backend.list_domains(status) + reverse_order = self._params.get("reverseOrder", None) + domains = self.swf_backend.list_domains(status, reverse_order=reverse_order) template = self.response_template(LIST_DOMAINS_TEMPLATE) return template.render(domains=domains) diff --git a/tests/test_swf/test_domains.py b/tests/test_swf/test_domains.py index 1a43e4cd0..7ab3dddc3 100644 --- a/tests/test_swf/test_domains.py +++ b/tests/test_swf/test_domains.py @@ -12,7 +12,6 @@ from moto.swf.exceptions import ( # RegisterDomain endpoint -# ListDomain endpoint @mock_swf def test_register_domain(): conn = boto.connect_swf("the_key", "the_secret") @@ -57,6 +56,30 @@ def test_register_with_wrong_parameter_type(): }) +# ListDomain endpoint +@mock_swf +def test_list_domains_order(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("b-test-domain", "60") + conn.register_domain("a-test-domain", "60") + conn.register_domain("c-test-domain", "60") + + all_domains = conn.list_domains("REGISTERED") + names = [domain["name"] for domain in all_domains["domainInfos"]] + names.should.equal(["a-test-domain", "b-test-domain", "c-test-domain"]) + +@mock_swf +def test_list_domains_reverse_order(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("b-test-domain", "60") + conn.register_domain("a-test-domain", "60") + conn.register_domain("c-test-domain", "60") + + all_domains = conn.list_domains("REGISTERED", reverse_order=True) + names = [domain["name"] for domain in all_domains["domainInfos"]] + names.should.equal(["c-test-domain", "b-test-domain", "a-test-domain"]) + + # DeprecateDomain endpoint @mock_swf def test_deprecate_domain(): From b680b2ec3ce023fec170e299a82b940974ae691b Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Wed, 30 Sep 2015 15:24:49 +0200 Subject: [PATCH 08/94] Add SWF endpoints: RegisterActivityType, DeprecateActivityType, ListActivityType, DescribeActivityType --- moto/swf/exceptions.py | 20 +++- moto/swf/models.py | 81 ++++++++++++- moto/swf/responses.py | 93 +++++++++++++++ tests/test_swf/test_activity_types.py | 161 ++++++++++++++++++++++++++ tests/test_swf/test_domains.py | 5 +- 5 files changed, 353 insertions(+), 7 deletions(-) create mode 100644 tests/test_swf/test_activity_types.py diff --git a/moto/swf/exceptions.py b/moto/swf/exceptions.py index f58c0360a..61066d294 100644 --- a/moto/swf/exceptions.py +++ b/moto/swf/exceptions.py @@ -33,10 +33,26 @@ class SWFDomainDeprecatedFault(SWFClientError): class SWFSerializationException(JSONResponseError): - def __init__(self): - message = "class java.lang.Foo can not be converted to an String (not a real SWF exception)" + def __init__(self, value): + message = "class java.lang.Foo can not be converted to an String " + message += " (not a real SWF exception ; happened on: {})".format(value) __type = "com.amazonaws.swf.base.model#SerializationException" super(SWFSerializationException, self).__init__( 400, "Bad Request", body={"Message": message, "__type": __type} ) + + +class SWFTypeAlreadyExistsFault(SWFClientError): + def __init__(self, name, version): + super(SWFTypeAlreadyExistsFault, self).__init__( + "ActivityType=[name={}, version={}]".format(name, version), + "com.amazonaws.swf.base.model#TypeAlreadyExistsFault") + + +class SWFTypeDeprecatedFault(SWFClientError): + def __init__(self, name, version): + super(SWFTypeDeprecatedFault, self).__init__( + "ActivityType=[name={}, version={}]".format(name, version), + "com.amazonaws.swf.base.model#TypeDeprecatedFault") + diff --git a/moto/swf/models.py b/moto/swf/models.py index db1ca94fb..2a2980177 100644 --- a/moto/swf/models.py +++ b/moto/swf/models.py @@ -1,4 +1,5 @@ from __future__ import unicode_literals +from collections import defaultdict import boto.swf @@ -8,6 +9,8 @@ from .exceptions import ( SWFDomainAlreadyExistsFault, SWFDomainDeprecatedFault, SWFSerializationException, + SWFTypeAlreadyExistsFault, + SWFTypeDeprecatedFault, ) @@ -17,10 +20,44 @@ class Domain(object): self.retention = retention self.description = description self.status = "REGISTERED" + self.activity_types = defaultdict(dict) def __repr__(self): return "Domain(name: %(name)s, status: %(status)s)" % self.__dict__ + def get_activity_type(self, name, version, ignore_empty=False): + try: + return self.activity_types[name][version] + except KeyError: + if not ignore_empty: + raise SWFUnknownResourceFault( + "type", + "ActivityType=[name={}, version={}]".format(name, version) + ) + + def add_activity_type(self, actype): + self.activity_types[actype.name][actype.version] = actype + + def find_activity_types(self, status): + _all = [] + for _, family in self.activity_types.iteritems(): + for _, actype in family.iteritems(): + if actype.status == status: + _all.append(actype) + return _all + + +class ActivityType(object): + def __init__(self, name, version, **kwargs): + self.name = name + self.version = version + self.status = "REGISTERED" + for key, value in kwargs.iteritems(): + self.__setattr__(key, value) + + def __repr__(self): + return "ActivityType(name: %(name)s, version: %(version)s)" % self.__dict__ + class SWFBackend(BaseBackend): def __init__(self, region_name): @@ -43,7 +80,7 @@ class SWFBackend(BaseBackend): def _check_string(self, parameter): if not isinstance(parameter, basestring): - raise SWFSerializationException() + raise SWFSerializationException(parameter) def list_domains(self, status, reverse_order=None): self._check_string(status) @@ -77,6 +114,48 @@ class SWFBackend(BaseBackend): self._check_string(name) return self._get_domain(name) + def list_activity_types(self, domain_name, status, reverse_order=None): + self._check_string(domain_name) + self._check_string(status) + domain = self._get_domain(domain_name) + actypes = domain.find_activity_types(status) + actypes = sorted(actypes, key=lambda domain: domain.name) + if reverse_order: + actypes = reversed(actypes) + return actypes + + def register_activity_type(self, domain_name, name, version, **kwargs): + self._check_string(domain_name) + self._check_string(name) + self._check_string(version) + for _, value in kwargs.iteritems(): + if value == (None,): + print _ + if value is not None: + self._check_string(value) + domain = self._get_domain(domain_name) + if domain.get_activity_type(name, version, ignore_empty=True): + raise SWFTypeAlreadyExistsFault(name, version) + activity_type = ActivityType(name, version, **kwargs) + domain.add_activity_type(activity_type) + + def deprecate_activity_type(self, domain_name, name, version): + self._check_string(domain_name) + self._check_string(name) + self._check_string(version) + domain = self._get_domain(domain_name) + actype = domain.get_activity_type(name, version) + if actype.status == "DEPRECATED": + raise SWFTypeDeprecatedFault(name, version) + actype.status = "DEPRECATED" + + def describe_activity_type(self, domain_name, name, version): + self._check_string(domain_name) + self._check_string(name) + self._check_string(version) + domain = self._get_domain(domain_name) + return domain.get_activity_type(name, version) + swf_backends = {} for region in boto.swf.regions(): diff --git a/moto/swf/responses.py b/moto/swf/responses.py index 8e2dae712..f27d34211 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -80,6 +80,60 @@ class SWFResponse(BaseResponse): template = self.response_template(DESCRIBE_DOMAIN_TEMPLATE) return template.render(domain=domain) + # TODO: implement pagination + def list_activity_types(self): + domain_name = self._params.get("domain") + status = self._params.get("registrationStatus") + reverse_order = self._params.get("reverseOrder", None) + actypes = self.swf_backend.list_activity_types(domain_name, status, reverse_order=reverse_order) + template = self.response_template(LIST_ACTIVITY_TYPES_TEMPLATE) + return template.render(actypes=actypes) + + def register_activity_type(self): + domain = self._params.get("domain") + name = self._params.get("name") + version = self._params.get("version") + default_task_list = self._params.get("defaultTaskList") + if default_task_list: + task_list = default_task_list.get("name") + else: + task_list = None + default_task_heartbeat_timeout = self._params.get("defaultTaskHeartbeatTimeout") + default_task_schedule_to_close_timeout = self._params.get("defaultTaskScheduleToCloseTimeout") + default_task_schedule_to_start_timeout = self._params.get("defaultTaskScheduleToStartTimeout") + default_task_start_to_close_timeout = self._params.get("defaultTaskStartToCloseTimeout") + description = self._params.get("description") + # TODO: add defaultTaskPriority when boto gets to support it + activity_type = self.swf_backend.register_activity_type( + domain, name, version, task_list=task_list, + default_task_heartbeat_timeout=default_task_heartbeat_timeout, + default_task_schedule_to_close_timeout=default_task_schedule_to_close_timeout, + default_task_schedule_to_start_timeout=default_task_schedule_to_start_timeout, + default_task_start_to_close_timeout=default_task_start_to_close_timeout, + description=description, + ) + template = self.response_template("") + return template.render() + + def deprecate_activity_type(self): + domain = self._params.get("domain") + actype = self._params.get("activityType") + name = actype["name"] + version = actype["version"] + domain = self.swf_backend.deprecate_activity_type(domain, name, version) + template = self.response_template("") + return template.render() + + def describe_activity_type(self): + domain = self._params.get("domain") + actype = self._params.get("activityType") + + name = actype["name"] + version = actype["version"] + actype = self.swf_backend.describe_activity_type(domain, name, version) + template = self.response_template(DESCRIBE_ACTIVITY_TYPE_TEMPLATE) + return template.render(actype=actype) + LIST_DOMAINS_TEMPLATE = """{ "domainInfos": [ @@ -103,3 +157,42 @@ DESCRIBE_DOMAIN_TEMPLATE = """{ "status": "{{ domain.status }}" } }""" + +LIST_ACTIVITY_TYPES_TEMPLATE = """{ + "typeInfos": [ + {%- for actype in actypes %} + { + "activityType": { + "name": "{{ actype.name }}", + "version": "{{ actype.version }}" + }, + "creationDate": 1420066800, + {% if actype.status == "DEPRECATED" %}"deprecationDate": 1422745200,{% endif %} + {% if actype.description %}"description": "{{ actype.description }}",{% endif %} + "status": "{{ actype.status }}" + }{% if not loop.last %},{% endif %} + {%- endfor %} + ] +}""" + +DESCRIBE_ACTIVITY_TYPE_TEMPLATE = """{ + "configuration": { + {% if actype.default_task_heartbeat_timeout %}"defaultTaskHeartbeatTimeout": "{{ actype.default_task_heartbeat_timeout }}",{% endif %} + {% if actype.task_list %}"defaultTaskList": { "name": "{{ actype.task_list }}" },{% endif %} + {% if actype.default_task_schedule_to_close_timeout %}"defaultTaskScheduleToCloseTimeout": "{{ actype.default_task_schedule_to_close_timeout }}",{% endif %} + {% if actype.default_task_schedule_to_start_timeout %}"defaultTaskScheduleToStartTimeout": "{{ actype.default_task_schedule_to_start_timeout }}",{% endif %} + {% if actype.default_task_start_to_close_timeout %}"defaultTaskStartToCloseTimeout": "{{ actype.default_task_start_to_close_timeout }}",{% endif %} + "__moto_placeholder": "(avoid dealing with coma in json)" + }, + "typeInfo": { + "activityType": { + "name": "{{ actype.name }}", + "version": "{{ actype.version }}" + }, + "creationDate": 1420066800, + {% if actype.status == "DEPRECATED" %}"deprecationDate": 1422745200,{% endif %} + {% if actype.description %}"description": "{{ actype.description }}",{% endif %} + "status": "{{ actype.status }}" + } +}""" + diff --git a/tests/test_swf/test_activity_types.py b/tests/test_swf/test_activity_types.py new file mode 100644 index 000000000..19f586a0b --- /dev/null +++ b/tests/test_swf/test_activity_types.py @@ -0,0 +1,161 @@ +import boto +from nose.tools import assert_raises +from sure import expect + +from moto import mock_swf +from moto.swf.exceptions import ( + SWFUnknownResourceFault, + SWFTypeAlreadyExistsFault, + SWFTypeDeprecatedFault, + SWFSerializationException, +) + + +# RegisterActivityType endpoint +@mock_swf +def test_register_activity_type(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60") + conn.register_activity_type("test-domain", "test-activity", "v1.0") + + types = conn.list_activity_types("test-domain", "REGISTERED") + actype = types["typeInfos"][0] + actype["activityType"]["name"].should.equal("test-activity") + actype["activityType"]["version"].should.equal("v1.0") + +@mock_swf +def test_register_already_existing_activity_type(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60") + conn.register_activity_type("test-domain", "test-activity", "v1.0") + + with assert_raises(SWFTypeAlreadyExistsFault) as err: + conn.register_activity_type("test-domain", "test-activity", "v1.0") + + ex = err.exception + ex.status.should.equal(400) + ex.error_code.should.equal("TypeAlreadyExistsFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#TypeAlreadyExistsFault", + "message": "ActivityType=[name=test-activity, version=v1.0]" + }) + +@mock_swf +def test_register_with_wrong_parameter_type(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60") + + with assert_raises(SWFSerializationException) as err: + conn.register_activity_type("test-domain", "test-activity", 12) + + ex = err.exception + ex.status.should.equal(400) + ex.error_code.should.equal("SerializationException") + ex.body["__type"].should.equal("com.amazonaws.swf.base.model#SerializationException") + + +# ListActivityTypes endpoint +@mock_swf +def test_list_activity_types(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60") + conn.register_activity_type("test-domain", "b-test-activity", "v1.0") + conn.register_activity_type("test-domain", "a-test-activity", "v1.0") + conn.register_activity_type("test-domain", "c-test-activity", "v1.0") + + all_activity_types = conn.list_activity_types("test-domain", "REGISTERED") + names = [activity_type["activityType"]["name"] for activity_type in all_activity_types["typeInfos"]] + names.should.equal(["a-test-activity", "b-test-activity", "c-test-activity"]) + +@mock_swf +def test_list_activity_types_reverse_order(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60") + conn.register_activity_type("test-domain", "b-test-activity", "v1.0") + conn.register_activity_type("test-domain", "a-test-activity", "v1.0") + conn.register_activity_type("test-domain", "c-test-activity", "v1.0") + + all_activity_types = conn.list_activity_types("test-domain", "REGISTERED", + reverse_order=True) + names = [activity_type["activityType"]["name"] for activity_type in all_activity_types["typeInfos"]] + names.should.equal(["c-test-activity", "b-test-activity", "a-test-activity"]) + + +# DeprecateActivityType endpoint +@mock_swf +def test_deprecate_activity_type(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60") + conn.register_activity_type("test-domain", "test-activity", "v1.0") + conn.deprecate_activity_type("test-domain", "test-activity", "v1.0") + + actypes = conn.list_activity_types("test-domain", "DEPRECATED") + actype = actypes["typeInfos"][0] + actype["activityType"]["name"].should.equal("test-activity") + actype["activityType"]["version"].should.equal("v1.0") + +@mock_swf +def test_deprecate_already_deprecated_activity_type(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60") + conn.register_activity_type("test-domain", "test-activity", "v1.0") + conn.deprecate_activity_type("test-domain", "test-activity", "v1.0") + + with assert_raises(SWFTypeDeprecatedFault) as err: + conn.deprecate_activity_type("test-domain", "test-activity", "v1.0") + + ex = err.exception + ex.status.should.equal(400) + ex.error_code.should.equal("TypeDeprecatedFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#TypeDeprecatedFault", + "message": "ActivityType=[name=test-activity, version=v1.0]" + }) + +@mock_swf +def test_deprecate_non_existent_activity_type(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60") + + with assert_raises(SWFUnknownResourceFault) as err: + conn.deprecate_activity_type("test-domain", "non-existent", "v1.0") + + ex = err.exception + ex.status.should.equal(400) + ex.error_code.should.equal("UnknownResourceFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#UnknownResourceFault", + "message": "Unknown type: ActivityType=[name=non-existent, version=v1.0]" + }) + +# DescribeActivityType endpoint +@mock_swf +def test_describe_activity_type(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60") + conn.register_activity_type("test-domain", "test-activity", "v1.0", + task_list="foo", default_task_heartbeat_timeout="32") + + actype = conn.describe_activity_type("test-domain", "test-activity", "v1.0") + actype["configuration"]["defaultTaskList"]["name"].should.equal("foo") + actype["configuration"].keys().should_not.contain("defaultTaskScheduleToClose") + infos = actype["typeInfo"] + infos["activityType"]["name"].should.equal("test-activity") + infos["activityType"]["version"].should.equal("v1.0") + infos["status"].should.equal("REGISTERED") + +@mock_swf +def test_describe_non_existent_activity_type(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60") + + with assert_raises(SWFUnknownResourceFault) as err: + conn.describe_activity_type("test-domain", "non-existent", "v1.0") + + ex = err.exception + ex.status.should.equal(400) + ex.error_code.should.equal("UnknownResourceFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#UnknownResourceFault", + "message": "Unknown type: ActivityType=[name=non-existent, version=v1.0]" + }) diff --git a/tests/test_swf/test_domains.py b/tests/test_swf/test_domains.py index 7ab3dddc3..8f3a75569 100644 --- a/tests/test_swf/test_domains.py +++ b/tests/test_swf/test_domains.py @@ -50,10 +50,7 @@ def test_register_with_wrong_parameter_type(): ex = err.exception ex.status.should.equal(400) ex.error_code.should.equal("SerializationException") - ex.body.should.equal({ - "__type": "com.amazonaws.swf.base.model#SerializationException", - "Message": "class java.lang.Foo can not be converted to an String (not a real SWF exception)" - }) + ex.body["__type"].should.equal("com.amazonaws.swf.base.model#SerializationException") # ListDomain endpoint From c4e903706ccfc018267c156dc1618d7e53079a8a Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Wed, 30 Sep 2015 19:26:42 +0200 Subject: [PATCH 09/94] Add SWF endpoints: RegisterWorkflowType, DeprecateWorkflowType, ListWorkflowTypes, DescribeWorkflowType --- moto/swf/exceptions.py | 8 +- moto/swf/models.py | 110 ++++++++++++++--- moto/swf/responses.py | 129 +++++++++++++++----- tests/test_swf/test_workflow_types.py | 162 ++++++++++++++++++++++++++ 4 files changed, 364 insertions(+), 45 deletions(-) create mode 100644 tests/test_swf/test_workflow_types.py diff --git a/moto/swf/exceptions.py b/moto/swf/exceptions.py index 61066d294..5510f1a88 100644 --- a/moto/swf/exceptions.py +++ b/moto/swf/exceptions.py @@ -44,15 +44,15 @@ class SWFSerializationException(JSONResponseError): class SWFTypeAlreadyExistsFault(SWFClientError): - def __init__(self, name, version): + def __init__(self, _type): super(SWFTypeAlreadyExistsFault, self).__init__( - "ActivityType=[name={}, version={}]".format(name, version), + "{}=[name={}, version={}]".format(_type.__class__.__name__, _type.name, _type.version), "com.amazonaws.swf.base.model#TypeAlreadyExistsFault") class SWFTypeDeprecatedFault(SWFClientError): - def __init__(self, name, version): + def __init__(self, _type): super(SWFTypeDeprecatedFault, self).__init__( - "ActivityType=[name={}, version={}]".format(name, version), + "{}=[name={}, version={}]".format(_type.__class__.__name__, _type.name, _type.version), "com.amazonaws.swf.base.model#TypeDeprecatedFault") diff --git a/moto/swf/models.py b/moto/swf/models.py index 2a2980177..95ee5e82a 100644 --- a/moto/swf/models.py +++ b/moto/swf/models.py @@ -21,6 +21,7 @@ class Domain(object): self.description = description self.status = "REGISTERED" self.activity_types = defaultdict(dict) + self.workflow_types = defaultdict(dict) def __repr__(self): return "Domain(name: %(name)s, status: %(status)s)" % self.__dict__ @@ -35,15 +36,39 @@ class Domain(object): "ActivityType=[name={}, version={}]".format(name, version) ) - def add_activity_type(self, actype): - self.activity_types[actype.name][actype.version] = actype + def add_activity_type(self, _type): + self.activity_types[_type.name][_type.version] = _type def find_activity_types(self, status): _all = [] for _, family in self.activity_types.iteritems(): - for _, actype in family.iteritems(): - if actype.status == status: - _all.append(actype) + for _, _type in family.iteritems(): + if _type.status == status: + _all.append(_type) + return _all + + # TODO: refactor it with get_activity_type() + def get_workflow_type(self, name, version, ignore_empty=False): + try: + return self.workflow_types[name][version] + except KeyError: + if not ignore_empty: + raise SWFUnknownResourceFault( + "type", + "WorkflowType=[name={}, version={}]".format(name, version) + ) + + # TODO: refactor it with add_activity_type() + def add_workflow_type(self, _type): + self.workflow_types[_type.name][_type.version] = _type + + # TODO: refactor it with find_activity_types() + def find_workflow_types(self, status): + _all = [] + for _, family in self.workflow_types.iteritems(): + for _, _type in family.iteritems(): + if _type.status == status: + _all.append(_type) return _all @@ -59,6 +84,18 @@ class ActivityType(object): return "ActivityType(name: %(name)s, version: %(version)s)" % self.__dict__ +class WorkflowType(object): + def __init__(self, name, version, **kwargs): + self.name = name + self.version = version + self.status = "REGISTERED" + for key, value in kwargs.iteritems(): + self.__setattr__(key, value) + + def __repr__(self): + return "WorkflowType(name: %(name)s, version: %(version)s)" % self.__dict__ + + class SWFBackend(BaseBackend): def __init__(self, region_name): self.region_name = region_name @@ -118,11 +155,11 @@ class SWFBackend(BaseBackend): self._check_string(domain_name) self._check_string(status) domain = self._get_domain(domain_name) - actypes = domain.find_activity_types(status) - actypes = sorted(actypes, key=lambda domain: domain.name) + _types = domain.find_activity_types(status) + _types = sorted(_types, key=lambda domain: domain.name) if reverse_order: - actypes = reversed(actypes) - return actypes + _types = reversed(_types) + return _types def register_activity_type(self, domain_name, name, version, **kwargs): self._check_string(domain_name) @@ -134,8 +171,9 @@ class SWFBackend(BaseBackend): if value is not None: self._check_string(value) domain = self._get_domain(domain_name) - if domain.get_activity_type(name, version, ignore_empty=True): - raise SWFTypeAlreadyExistsFault(name, version) + _type = domain.get_activity_type(name, version, ignore_empty=True) + if _type: + raise SWFTypeAlreadyExistsFault(_type) activity_type = ActivityType(name, version, **kwargs) domain.add_activity_type(activity_type) @@ -144,10 +182,10 @@ class SWFBackend(BaseBackend): self._check_string(name) self._check_string(version) domain = self._get_domain(domain_name) - actype = domain.get_activity_type(name, version) - if actype.status == "DEPRECATED": - raise SWFTypeDeprecatedFault(name, version) - actype.status = "DEPRECATED" + _type = domain.get_activity_type(name, version) + if _type.status == "DEPRECATED": + raise SWFTypeDeprecatedFault(_type) + _type.status = "DEPRECATED" def describe_activity_type(self, domain_name, name, version): self._check_string(domain_name) @@ -156,6 +194,48 @@ class SWFBackend(BaseBackend): domain = self._get_domain(domain_name) return domain.get_activity_type(name, version) + def list_workflow_types(self, domain_name, status, reverse_order=None): + self._check_string(domain_name) + self._check_string(status) + domain = self._get_domain(domain_name) + _types = domain.find_workflow_types(status) + _types = sorted(_types, key=lambda domain: domain.name) + if reverse_order: + _types = reversed(_types) + return _types + + def register_workflow_type(self, domain_name, name, version, **kwargs): + self._check_string(domain_name) + self._check_string(name) + self._check_string(version) + for _, value in kwargs.iteritems(): + if value == (None,): + print _ + if value is not None: + self._check_string(value) + domain = self._get_domain(domain_name) + _type = domain.get_workflow_type(name, version, ignore_empty=True) + if _type: + raise SWFTypeAlreadyExistsFault(_type) + workflow_type = WorkflowType(name, version, **kwargs) + domain.add_workflow_type(workflow_type) + + def deprecate_workflow_type(self, domain_name, name, version): + self._check_string(domain_name) + self._check_string(name) + self._check_string(version) + domain = self._get_domain(domain_name) + _type = domain.get_workflow_type(name, version) + if _type.status == "DEPRECATED": + raise SWFTypeDeprecatedFault(_type) + _type.status = "DEPRECATED" + + def describe_workflow_type(self, domain_name, name, version): + self._check_string(domain_name) + self._check_string(name) + self._check_string(version) + domain = self._get_domain(domain_name) + return domain.get_workflow_type(name, version) swf_backends = {} for region in boto.swf.regions(): diff --git a/moto/swf/responses.py b/moto/swf/responses.py index f27d34211..1f00d5646 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -85,9 +85,9 @@ class SWFResponse(BaseResponse): domain_name = self._params.get("domain") status = self._params.get("registrationStatus") reverse_order = self._params.get("reverseOrder", None) - actypes = self.swf_backend.list_activity_types(domain_name, status, reverse_order=reverse_order) + types = self.swf_backend.list_activity_types(domain_name, status, reverse_order=reverse_order) template = self.response_template(LIST_ACTIVITY_TYPES_TEMPLATE) - return template.render(actypes=actypes) + return template.render(types=types) def register_activity_type(self): domain = self._params.get("domain") @@ -117,22 +117,78 @@ class SWFResponse(BaseResponse): def deprecate_activity_type(self): domain = self._params.get("domain") - actype = self._params.get("activityType") - name = actype["name"] - version = actype["version"] + _type = self._params.get("activityType") + name = _type["name"] + version = _type["version"] domain = self.swf_backend.deprecate_activity_type(domain, name, version) template = self.response_template("") return template.render() def describe_activity_type(self): domain = self._params.get("domain") - actype = self._params.get("activityType") + _type = self._params.get("activityType") - name = actype["name"] - version = actype["version"] - actype = self.swf_backend.describe_activity_type(domain, name, version) + name = _type["name"] + version = _type["version"] + _type = self.swf_backend.describe_activity_type(domain, name, version) template = self.response_template(DESCRIBE_ACTIVITY_TYPE_TEMPLATE) - return template.render(actype=actype) + return template.render(_type=_type) + + # TODO: implement pagination + # TODO: refactor with list_activity_types() + def list_workflow_types(self): + domain_name = self._params.get("domain") + status = self._params.get("registrationStatus") + reverse_order = self._params.get("reverseOrder", None) + types = self.swf_backend.list_workflow_types(domain_name, status, reverse_order=reverse_order) + template = self.response_template(LIST_WORKFLOW_TYPES_TEMPLATE) + return template.render(types=types) + + def register_workflow_type(self): + domain = self._params.get("domain") + name = self._params.get("name") + version = self._params.get("version") + default_task_list = self._params.get("defaultTaskList") + if default_task_list: + task_list = default_task_list.get("name") + else: + task_list = None + default_child_policy = self._params.get("defaultChildPolicy") + default_task_start_to_close_timeout = self._params.get("defaultTaskStartToCloseTimeout") + default_execution_start_to_close_timeout = self._params.get("defaultTaskExecutionStartToCloseTimeout") + description = self._params.get("description") + # TODO: add defaultTaskPriority when boto gets to support it + # TODO: add defaultLambdaRole when boto gets to support it + workflow_type = self.swf_backend.register_workflow_type( + domain, name, version, task_list=task_list, + default_child_policy=default_child_policy, + default_task_start_to_close_timeout=default_task_start_to_close_timeout, + default_execution_start_to_close_timeout=default_execution_start_to_close_timeout, + description=description, + ) + template = self.response_template("") + return template.render() + + # TODO: refactor with deprecate_activity_type() + def deprecate_workflow_type(self): + domain = self._params.get("domain") + _type = self._params.get("workflowType") + name = _type["name"] + version = _type["version"] + domain = self.swf_backend.deprecate_workflow_type(domain, name, version) + template = self.response_template("") + return template.render() + + # TODO: refactor with describe_activity_type() + def describe_workflow_type(self): + domain = self._params.get("domain") + _type = self._params.get("workflowType") + + name = _type["name"] + version = _type["version"] + _type = self.swf_backend.describe_workflow_type(domain, name, version) + template = self.response_template(DESCRIBE_WORKFLOW_TYPE_TEMPLATE) + return template.render(_type=_type) LIST_DOMAINS_TEMPLATE = """{ @@ -160,16 +216,16 @@ DESCRIBE_DOMAIN_TEMPLATE = """{ LIST_ACTIVITY_TYPES_TEMPLATE = """{ "typeInfos": [ - {%- for actype in actypes %} + {%- for _type in types %} { "activityType": { - "name": "{{ actype.name }}", - "version": "{{ actype.version }}" + "name": "{{ _type.name }}", + "version": "{{ _type.version }}" }, "creationDate": 1420066800, - {% if actype.status == "DEPRECATED" %}"deprecationDate": 1422745200,{% endif %} - {% if actype.description %}"description": "{{ actype.description }}",{% endif %} - "status": "{{ actype.status }}" + {% if _type.status == "DEPRECATED" %}"deprecationDate": 1422745200,{% endif %} + {% if _type.description %}"description": "{{ _type.description }}",{% endif %} + "status": "{{ _type.status }}" }{% if not loop.last %},{% endif %} {%- endfor %} ] @@ -177,22 +233,43 @@ LIST_ACTIVITY_TYPES_TEMPLATE = """{ DESCRIBE_ACTIVITY_TYPE_TEMPLATE = """{ "configuration": { - {% if actype.default_task_heartbeat_timeout %}"defaultTaskHeartbeatTimeout": "{{ actype.default_task_heartbeat_timeout }}",{% endif %} - {% if actype.task_list %}"defaultTaskList": { "name": "{{ actype.task_list }}" },{% endif %} - {% if actype.default_task_schedule_to_close_timeout %}"defaultTaskScheduleToCloseTimeout": "{{ actype.default_task_schedule_to_close_timeout }}",{% endif %} - {% if actype.default_task_schedule_to_start_timeout %}"defaultTaskScheduleToStartTimeout": "{{ actype.default_task_schedule_to_start_timeout }}",{% endif %} - {% if actype.default_task_start_to_close_timeout %}"defaultTaskStartToCloseTimeout": "{{ actype.default_task_start_to_close_timeout }}",{% endif %} + {% if _type.default_task_heartbeat_timeout %}"defaultTaskHeartbeatTimeout": "{{ _type.default_task_heartbeat_timeout }}",{% endif %} + {% if _type.task_list %}"defaultTaskList": { "name": "{{ _type.task_list }}" },{% endif %} + {% if _type.default_task_schedule_to_close_timeout %}"defaultTaskScheduleToCloseTimeout": "{{ _type.default_task_schedule_to_close_timeout }}",{% endif %} + {% if _type.default_task_schedule_to_start_timeout %}"defaultTaskScheduleToStartTimeout": "{{ _type.default_task_schedule_to_start_timeout }}",{% endif %} + {% if _type.default_task_start_to_close_timeout %}"defaultTaskStartToCloseTimeout": "{{ _type.default_task_start_to_close_timeout }}",{% endif %} "__moto_placeholder": "(avoid dealing with coma in json)" }, "typeInfo": { "activityType": { - "name": "{{ actype.name }}", - "version": "{{ actype.version }}" + "name": "{{ _type.name }}", + "version": "{{ _type.version }}" }, "creationDate": 1420066800, - {% if actype.status == "DEPRECATED" %}"deprecationDate": 1422745200,{% endif %} - {% if actype.description %}"description": "{{ actype.description }}",{% endif %} - "status": "{{ actype.status }}" + {% if _type.status == "DEPRECATED" %}"deprecationDate": 1422745200,{% endif %} + {% if _type.description %}"description": "{{ _type.description }}",{% endif %} + "status": "{{ _type.status }}" } }""" +LIST_WORKFLOW_TYPES_TEMPLATE = LIST_ACTIVITY_TYPES_TEMPLATE.replace("activityType", "workflowType") + +DESCRIBE_WORKFLOW_TYPE_TEMPLATE = """{ + "configuration": { + {% if _type.default_child_policy %}"defaultChildPolicy": "{{ _type.default_child_policy }}",{% endif %} + {% if _type.default_execution_start_to_close_timeout %}"defaultExecutionStartToCloseTimeout": "{{ _type.default_execution_start_to_close_timeout }}",{% endif %} + {% if _type.task_list %}"defaultTaskList": { "name": "{{ _type.task_list }}" },{% endif %} + {% if _type.default_task_start_to_close_timeout %}"defaultTaskStartToCloseTimeout": "{{ _type.default_task_start_to_close_timeout }}",{% endif %} + "__moto_placeholder": "(avoid dealing with coma in json)" + }, + "typeInfo": { + "workflowType": { + "name": "{{ _type.name }}", + "version": "{{ _type.version }}" + }, + "creationDate": 1420066800, + {% if _type.status == "DEPRECATED" %}"deprecationDate": 1422745200,{% endif %} + {% if _type.description %}"description": "{{ _type.description }}",{% endif %} + "status": "{{ _type.status }}" + } +}""" diff --git a/tests/test_swf/test_workflow_types.py b/tests/test_swf/test_workflow_types.py new file mode 100644 index 000000000..67424710f --- /dev/null +++ b/tests/test_swf/test_workflow_types.py @@ -0,0 +1,162 @@ +import boto +from nose.tools import assert_raises +from sure import expect + +from moto import mock_swf +from moto.swf.exceptions import ( + SWFUnknownResourceFault, + SWFTypeAlreadyExistsFault, + SWFTypeDeprecatedFault, + SWFSerializationException, +) + + +# RegisterWorkflowType endpoint +@mock_swf +def test_register_workflow_type(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60") + conn.register_workflow_type("test-domain", "test-workflow", "v1.0") + + types = conn.list_workflow_types("test-domain", "REGISTERED") + actype = types["typeInfos"][0] + actype["workflowType"]["name"].should.equal("test-workflow") + actype["workflowType"]["version"].should.equal("v1.0") + +@mock_swf +def test_register_already_existing_workflow_type(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60") + conn.register_workflow_type("test-domain", "test-workflow", "v1.0") + + with assert_raises(SWFTypeAlreadyExistsFault) as err: + conn.register_workflow_type("test-domain", "test-workflow", "v1.0") + + ex = err.exception + ex.status.should.equal(400) + ex.error_code.should.equal("TypeAlreadyExistsFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#TypeAlreadyExistsFault", + "message": "WorkflowType=[name=test-workflow, version=v1.0]" + }) + +@mock_swf +def test_register_with_wrong_parameter_type(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60") + + with assert_raises(SWFSerializationException) as err: + conn.register_workflow_type("test-domain", "test-workflow", 12) + + ex = err.exception + ex.status.should.equal(400) + ex.error_code.should.equal("SerializationException") + ex.body["__type"].should.equal("com.amazonaws.swf.base.model#SerializationException") + + +# ListWorkflowTypes endpoint +@mock_swf +def test_list_workflow_types(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60") + conn.register_workflow_type("test-domain", "b-test-workflow", "v1.0") + conn.register_workflow_type("test-domain", "a-test-workflow", "v1.0") + conn.register_workflow_type("test-domain", "c-test-workflow", "v1.0") + + all_workflow_types = conn.list_workflow_types("test-domain", "REGISTERED") + names = [activity_type["workflowType"]["name"] for activity_type in all_workflow_types["typeInfos"]] + names.should.equal(["a-test-workflow", "b-test-workflow", "c-test-workflow"]) + +@mock_swf +def test_list_workflow_types_reverse_order(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60") + conn.register_workflow_type("test-domain", "b-test-workflow", "v1.0") + conn.register_workflow_type("test-domain", "a-test-workflow", "v1.0") + conn.register_workflow_type("test-domain", "c-test-workflow", "v1.0") + + all_workflow_types = conn.list_workflow_types("test-domain", "REGISTERED", + reverse_order=True) + names = [activity_type["workflowType"]["name"] for activity_type in all_workflow_types["typeInfos"]] + names.should.equal(["c-test-workflow", "b-test-workflow", "a-test-workflow"]) + + +# DeprecateWorkflowType endpoint +@mock_swf +def test_deprecate_workflow_type(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60") + conn.register_workflow_type("test-domain", "test-workflow", "v1.0") + conn.deprecate_workflow_type("test-domain", "test-workflow", "v1.0") + + actypes = conn.list_workflow_types("test-domain", "DEPRECATED") + actype = actypes["typeInfos"][0] + actype["workflowType"]["name"].should.equal("test-workflow") + actype["workflowType"]["version"].should.equal("v1.0") + +@mock_swf +def test_deprecate_already_deprecated_workflow_type(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60") + conn.register_workflow_type("test-domain", "test-workflow", "v1.0") + conn.deprecate_workflow_type("test-domain", "test-workflow", "v1.0") + + with assert_raises(SWFTypeDeprecatedFault) as err: + conn.deprecate_workflow_type("test-domain", "test-workflow", "v1.0") + + ex = err.exception + ex.status.should.equal(400) + ex.error_code.should.equal("TypeDeprecatedFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#TypeDeprecatedFault", + "message": "WorkflowType=[name=test-workflow, version=v1.0]" + }) + +@mock_swf +def test_deprecate_non_existent_workflow_type(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60") + + with assert_raises(SWFUnknownResourceFault) as err: + conn.deprecate_workflow_type("test-domain", "non-existent", "v1.0") + + ex = err.exception + ex.status.should.equal(400) + ex.error_code.should.equal("UnknownResourceFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#UnknownResourceFault", + "message": "Unknown type: WorkflowType=[name=non-existent, version=v1.0]" + }) + +# DescribeWorkflowType endpoint +@mock_swf +def test_describe_workflow_type(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60") + conn.register_workflow_type("test-domain", "test-workflow", "v1.0", + task_list="foo", default_child_policy="TERMINATE") + + actype = conn.describe_workflow_type("test-domain", "test-workflow", "v1.0") + actype["configuration"]["defaultTaskList"]["name"].should.equal("foo") + actype["configuration"]["defaultChildPolicy"].should.equal("TERMINATE") + actype["configuration"].keys().should_not.contain("defaultTaskStartToCloseTimeout") + infos = actype["typeInfo"] + infos["workflowType"]["name"].should.equal("test-workflow") + infos["workflowType"]["version"].should.equal("v1.0") + infos["status"].should.equal("REGISTERED") + +@mock_swf +def test_describe_non_existent_workflow_type(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60") + + with assert_raises(SWFUnknownResourceFault) as err: + conn.describe_workflow_type("test-domain", "non-existent", "v1.0") + + ex = err.exception + ex.status.should.equal(400) + ex.error_code.should.equal("UnknownResourceFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#UnknownResourceFault", + "message": "Unknown type: WorkflowType=[name=non-existent, version=v1.0]" + }) From 6e6b325225e8b6bc6eaaa12d56ecadd0ef89d7e2 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Wed, 30 Sep 2015 21:01:05 +0200 Subject: [PATCH 10/94] Deduplicate logic between ActivityType's and WorkflowType's --- moto/swf/models.py | 110 +++++++++++------------------------------- moto/swf/responses.py | 86 ++++++++++++++------------------- 2 files changed, 64 insertions(+), 132 deletions(-) diff --git a/moto/swf/models.py b/moto/swf/models.py index 95ee5e82a..8d5d8f0e4 100644 --- a/moto/swf/models.py +++ b/moto/swf/models.py @@ -26,46 +26,31 @@ class Domain(object): def __repr__(self): return "Domain(name: %(name)s, status: %(status)s)" % self.__dict__ - def get_activity_type(self, name, version, ignore_empty=False): + def get_type(self, kind, name, version, ignore_empty=False): try: - return self.activity_types[name][version] + _types = getattr(self, "{}_types".format(kind)) + return _types[name][version] except KeyError: if not ignore_empty: raise SWFUnknownResourceFault( "type", - "ActivityType=[name={}, version={}]".format(name, version) + "{}Type=[name={}, version={}]".format( + kind.capitalize(), name, version + ) ) - def add_activity_type(self, _type): - self.activity_types[_type.name][_type.version] = _type + def add_type(self, _type): + if isinstance(_type, ActivityType): + self.activity_types[_type.name][_type.version] = _type + elif isinstance(_type, WorkflowType): + self.workflow_types[_type.name][_type.version] = _type + else: + raise ValueError("Unknown SWF type: {}".format(_type)) - def find_activity_types(self, status): + def find_types(self, kind, status): _all = [] - for _, family in self.activity_types.iteritems(): - for _, _type in family.iteritems(): - if _type.status == status: - _all.append(_type) - return _all - - # TODO: refactor it with get_activity_type() - def get_workflow_type(self, name, version, ignore_empty=False): - try: - return self.workflow_types[name][version] - except KeyError: - if not ignore_empty: - raise SWFUnknownResourceFault( - "type", - "WorkflowType=[name={}, version={}]".format(name, version) - ) - - # TODO: refactor it with add_activity_type() - def add_workflow_type(self, _type): - self.workflow_types[_type.name][_type.version] = _type - - # TODO: refactor it with find_activity_types() - def find_workflow_types(self, status): - _all = [] - for _, family in self.workflow_types.iteritems(): + _types = getattr(self, "{}_types".format(kind)) + for _, family in _types.iteritems(): for _, _type in family.iteritems(): if _type.status == status: _all.append(_type) @@ -151,17 +136,17 @@ class SWFBackend(BaseBackend): self._check_string(name) return self._get_domain(name) - def list_activity_types(self, domain_name, status, reverse_order=None): + def list_types(self, kind, domain_name, status, reverse_order=None): self._check_string(domain_name) self._check_string(status) domain = self._get_domain(domain_name) - _types = domain.find_activity_types(status) + _types = domain.find_types(kind, status) _types = sorted(_types, key=lambda domain: domain.name) if reverse_order: _types = reversed(_types) return _types - def register_activity_type(self, domain_name, name, version, **kwargs): + def register_type(self, kind, domain_name, name, version, **kwargs): self._check_string(domain_name) self._check_string(name) self._check_string(version) @@ -171,71 +156,30 @@ class SWFBackend(BaseBackend): if value is not None: self._check_string(value) domain = self._get_domain(domain_name) - _type = domain.get_activity_type(name, version, ignore_empty=True) + _type = domain.get_type(kind, name, version, ignore_empty=True) if _type: raise SWFTypeAlreadyExistsFault(_type) - activity_type = ActivityType(name, version, **kwargs) - domain.add_activity_type(activity_type) + _class = globals()["{}Type".format(kind.capitalize())] + _type = _class(name, version, **kwargs) + domain.add_type(_type) - def deprecate_activity_type(self, domain_name, name, version): + def deprecate_type(self, kind, domain_name, name, version): self._check_string(domain_name) self._check_string(name) self._check_string(version) domain = self._get_domain(domain_name) - _type = domain.get_activity_type(name, version) + _type = domain.get_type(kind, name, version) if _type.status == "DEPRECATED": raise SWFTypeDeprecatedFault(_type) _type.status = "DEPRECATED" - def describe_activity_type(self, domain_name, name, version): + def describe_type(self, kind, domain_name, name, version): self._check_string(domain_name) self._check_string(name) self._check_string(version) domain = self._get_domain(domain_name) - return domain.get_activity_type(name, version) + return domain.get_type(kind, name, version) - def list_workflow_types(self, domain_name, status, reverse_order=None): - self._check_string(domain_name) - self._check_string(status) - domain = self._get_domain(domain_name) - _types = domain.find_workflow_types(status) - _types = sorted(_types, key=lambda domain: domain.name) - if reverse_order: - _types = reversed(_types) - return _types - - def register_workflow_type(self, domain_name, name, version, **kwargs): - self._check_string(domain_name) - self._check_string(name) - self._check_string(version) - for _, value in kwargs.iteritems(): - if value == (None,): - print _ - if value is not None: - self._check_string(value) - domain = self._get_domain(domain_name) - _type = domain.get_workflow_type(name, version, ignore_empty=True) - if _type: - raise SWFTypeAlreadyExistsFault(_type) - workflow_type = WorkflowType(name, version, **kwargs) - domain.add_workflow_type(workflow_type) - - def deprecate_workflow_type(self, domain_name, name, version): - self._check_string(domain_name) - self._check_string(name) - self._check_string(version) - domain = self._get_domain(domain_name) - _type = domain.get_workflow_type(name, version) - if _type.status == "DEPRECATED": - raise SWFTypeDeprecatedFault(_type) - _type.status = "DEPRECATED" - - def describe_workflow_type(self, domain_name, name, version): - self._check_string(domain_name) - self._check_string(name) - self._check_string(version) - domain = self._get_domain(domain_name) - return domain.get_workflow_type(name, version) swf_backends = {} for region in boto.swf.regions(): diff --git a/moto/swf/responses.py b/moto/swf/responses.py index 1f00d5646..a551aab2e 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -51,6 +51,33 @@ class SWFResponse(BaseResponse): def _params(self): return json.loads(self.body) + def _list_types(self, kind, template_str): + domain_name = self._params.get("domain") + status = self._params.get("registrationStatus") + reverse_order = self._params.get("reverseOrder", None) + types = self.swf_backend.list_types(kind, domain_name, status, reverse_order=reverse_order) + template = self.response_template(template_str) + return template.render(types=types) + + def _describe_type(self, kind, template_str): + domain = self._params.get("domain") + _type = self._params.get("{}Type".format(kind)) + + name = _type["name"] + version = _type["version"] + _type = self.swf_backend.describe_type(kind, domain, name, version) + template = self.response_template(template_str) + return template.render(_type=_type) + + def _deprecate_type(self, kind): + domain = self._params.get("domain") + _type = self._params.get("{}Type".format(kind)) + name = _type["name"] + version = _type["version"] + self.swf_backend.deprecate_type(kind, domain, name, version) + template = self.response_template("") + return template.render() + # TODO: implement pagination def list_domains(self): status = self._params.get("registrationStatus") @@ -82,12 +109,7 @@ class SWFResponse(BaseResponse): # TODO: implement pagination def list_activity_types(self): - domain_name = self._params.get("domain") - status = self._params.get("registrationStatus") - reverse_order = self._params.get("reverseOrder", None) - types = self.swf_backend.list_activity_types(domain_name, status, reverse_order=reverse_order) - template = self.response_template(LIST_ACTIVITY_TYPES_TEMPLATE) - return template.render(types=types) + return self._list_types("activity", LIST_ACTIVITY_TYPES_TEMPLATE) def register_activity_type(self): domain = self._params.get("domain") @@ -104,8 +126,8 @@ class SWFResponse(BaseResponse): default_task_start_to_close_timeout = self._params.get("defaultTaskStartToCloseTimeout") description = self._params.get("description") # TODO: add defaultTaskPriority when boto gets to support it - activity_type = self.swf_backend.register_activity_type( - domain, name, version, task_list=task_list, + activity_type = self.swf_backend.register_type( + "activity", domain, name, version, task_list=task_list, default_task_heartbeat_timeout=default_task_heartbeat_timeout, default_task_schedule_to_close_timeout=default_task_schedule_to_close_timeout, default_task_schedule_to_start_timeout=default_task_schedule_to_start_timeout, @@ -116,33 +138,14 @@ class SWFResponse(BaseResponse): return template.render() def deprecate_activity_type(self): - domain = self._params.get("domain") - _type = self._params.get("activityType") - name = _type["name"] - version = _type["version"] - domain = self.swf_backend.deprecate_activity_type(domain, name, version) - template = self.response_template("") - return template.render() + return self._deprecate_type("activity") def describe_activity_type(self): - domain = self._params.get("domain") - _type = self._params.get("activityType") + return self._describe_type("activity", DESCRIBE_ACTIVITY_TYPE_TEMPLATE) - name = _type["name"] - version = _type["version"] - _type = self.swf_backend.describe_activity_type(domain, name, version) - template = self.response_template(DESCRIBE_ACTIVITY_TYPE_TEMPLATE) - return template.render(_type=_type) - - # TODO: implement pagination # TODO: refactor with list_activity_types() def list_workflow_types(self): - domain_name = self._params.get("domain") - status = self._params.get("registrationStatus") - reverse_order = self._params.get("reverseOrder", None) - types = self.swf_backend.list_workflow_types(domain_name, status, reverse_order=reverse_order) - template = self.response_template(LIST_WORKFLOW_TYPES_TEMPLATE) - return template.render(types=types) + return self._list_types("workflow", LIST_WORKFLOW_TYPES_TEMPLATE) def register_workflow_type(self): domain = self._params.get("domain") @@ -159,8 +162,8 @@ class SWFResponse(BaseResponse): description = self._params.get("description") # TODO: add defaultTaskPriority when boto gets to support it # TODO: add defaultLambdaRole when boto gets to support it - workflow_type = self.swf_backend.register_workflow_type( - domain, name, version, task_list=task_list, + workflow_type = self.swf_backend.register_type( + "workflow", domain, name, version, task_list=task_list, default_child_policy=default_child_policy, default_task_start_to_close_timeout=default_task_start_to_close_timeout, default_execution_start_to_close_timeout=default_execution_start_to_close_timeout, @@ -169,26 +172,11 @@ class SWFResponse(BaseResponse): template = self.response_template("") return template.render() - # TODO: refactor with deprecate_activity_type() def deprecate_workflow_type(self): - domain = self._params.get("domain") - _type = self._params.get("workflowType") - name = _type["name"] - version = _type["version"] - domain = self.swf_backend.deprecate_workflow_type(domain, name, version) - template = self.response_template("") - return template.render() + return self._deprecate_type("workflow") - # TODO: refactor with describe_activity_type() def describe_workflow_type(self): - domain = self._params.get("domain") - _type = self._params.get("workflowType") - - name = _type["name"] - version = _type["version"] - _type = self.swf_backend.describe_workflow_type(domain, name, version) - template = self.response_template(DESCRIBE_WORKFLOW_TYPE_TEMPLATE) - return template.render(_type=_type) + return self._describe_type("workflow", DESCRIBE_WORKFLOW_TYPE_TEMPLATE) LIST_DOMAINS_TEMPLATE = """{ From 948335558487e1435ae1a252ab1e87bcdd2eccca Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Thu, 1 Oct 2015 21:09:39 +0200 Subject: [PATCH 11/94] Prepare SWF objects representations directly via json.dumps() ... instead of jinja2 templates that are absolutely not suited for this purpose, and hard to test. --- moto/swf/models.py | 98 ++++++++++++++++++++ moto/swf/responses.py | 128 +++++--------------------- tests/test_swf/test_activity_types.py | 33 ++++++- tests/test_swf/test_domains.py | 12 ++- 4 files changed, 164 insertions(+), 107 deletions(-) diff --git a/moto/swf/models.py b/moto/swf/models.py index 8d5d8f0e4..421e911c0 100644 --- a/moto/swf/models.py +++ b/moto/swf/models.py @@ -4,6 +4,8 @@ from collections import defaultdict import boto.swf from moto.core import BaseBackend +from moto.core.utils import camelcase_to_underscores + from .exceptions import ( SWFUnknownResourceFault, SWFDomainAlreadyExistsFault, @@ -26,6 +28,15 @@ class Domain(object): def __repr__(self): return "Domain(name: %(name)s, status: %(status)s)" % self.__dict__ + def to_dict(self): + hsh = { + "name": self.name, + "status": self.status, + } + if self.description: + hsh["description"] = self.description + return hsh + def get_type(self, kind, name, version, ignore_empty=False): try: _types = getattr(self, "{}_types".format(kind)) @@ -62,12 +73,57 @@ class ActivityType(object): self.name = name self.version = version self.status = "REGISTERED" + if "description" in kwargs: + self.description = kwargs.pop("description") for key, value in kwargs.iteritems(): self.__setattr__(key, value) def __repr__(self): return "ActivityType(name: %(name)s, version: %(version)s)" % self.__dict__ + @property + def _configuration_keys(self): + return [ + "defaultTaskHeartbeatTimeout", + "defaultTaskScheduleToCloseTimeout", + "defaultTaskScheduleToStartTimeout", + "defaultTaskStartToCloseTimeout", + ] + + def to_short_dict(self): + return { + "name": self.name, + "version": self.version, + } + + def to_medium_dict(self): + hsh = { + "activityType": self.to_short_dict(), + "creationDate": 1420066800, + "status": self.status, + } + if self.status == "DEPRECATED": + hsh["deprecationDate"] = 1422745200 + if hasattr(self, "description"): + hsh["description"] = self.description + return hsh + + def to_full_dict(self): + hsh = { + "typeInfo": self.to_medium_dict(), + "configuration": {} + } + if hasattr(self, "task_list"): + hsh["configuration"]["defaultTaskList"] = {"name": self.task_list} + for key in self._configuration_keys: + attr = camelcase_to_underscores(key) + if not hasattr(self, attr): + continue + if not getattr(self, attr): + continue + hsh["configuration"][key] = getattr(self, attr) + return hsh + class WorkflowType(object): def __init__(self, name, version, **kwargs): @@ -80,6 +136,48 @@ class WorkflowType(object): def __repr__(self): return "WorkflowType(name: %(name)s, version: %(version)s)" % self.__dict__ + @property + def _configuration_keys(self): + return [ + "defaultChildPolicy", + "defaultExecutionStartToCloseTimeout", + "defaultTaskStartToCloseTimeout", + ] + + def to_short_dict(self): + return { + "name": self.name, + "version": self.version, + } + + def to_medium_dict(self): + hsh = { + "workflowType": self.to_short_dict(), + "creationDate": 1420066800, + "status": self.status, + } + if self.status == "DEPRECATED": + hsh["deprecationDate"] = 1422745200 + if hasattr(self, "description"): + hsh["description"] = self.description + return hsh + + def to_full_dict(self): + hsh = { + "typeInfo": self.to_medium_dict(), + "configuration": {} + } + if hasattr(self, "task_list"): + hsh["configuration"]["defaultTaskList"] = {"name": self.task_list} + for key in self._configuration_keys: + attr = camelcase_to_underscores(key) + if not hasattr(self, attr): + continue + if getattr(self, attr) is None: + continue + hsh["configuration"][key] = getattr(self, attr) + return hsh + class SWFBackend(BaseBackend): def __init__(self, region_name): diff --git a/moto/swf/responses.py b/moto/swf/responses.py index a551aab2e..82c505653 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -51,23 +51,24 @@ class SWFResponse(BaseResponse): def _params(self): return json.loads(self.body) - def _list_types(self, kind, template_str): + def _list_types(self, kind): domain_name = self._params.get("domain") status = self._params.get("registrationStatus") reverse_order = self._params.get("reverseOrder", None) types = self.swf_backend.list_types(kind, domain_name, status, reverse_order=reverse_order) - template = self.response_template(template_str) - return template.render(types=types) + return json.dumps({ + "typeInfos": [_type.to_medium_dict() for _type in types] + }) - def _describe_type(self, kind, template_str): + def _describe_type(self, kind): domain = self._params.get("domain") - _type = self._params.get("{}Type".format(kind)) + _type_args = self._params.get("{}Type".format(kind)) - name = _type["name"] - version = _type["version"] + name = _type_args["name"] + version = _type_args["version"] _type = self.swf_backend.describe_type(kind, domain, name, version) - template = self.response_template(template_str) - return template.render(_type=_type) + + return json.dumps(_type.to_full_dict()) def _deprecate_type(self, kind): domain = self._params.get("domain") @@ -83,8 +84,9 @@ class SWFResponse(BaseResponse): status = self._params.get("registrationStatus") reverse_order = self._params.get("reverseOrder", None) domains = self.swf_backend.list_domains(status, reverse_order=reverse_order) - template = self.response_template(LIST_DOMAINS_TEMPLATE) - return template.render(domains=domains) + return json.dumps({ + "domainInfos": [domain.to_dict() for domain in domains] + }) def register_domain(self): name = self._params.get("name") @@ -92,24 +94,24 @@ class SWFResponse(BaseResponse): retention = self._params.get("workflowExecutionRetentionPeriodInDays") domain = self.swf_backend.register_domain(name, retention, description=description) - template = self.response_template("") - return template.render() + return "" def deprecate_domain(self): name = self._params.get("name") domain = self.swf_backend.deprecate_domain(name) - template = self.response_template("") - return template.render() + return "" def describe_domain(self): name = self._params.get("name") domain = self.swf_backend.describe_domain(name) - template = self.response_template(DESCRIBE_DOMAIN_TEMPLATE) - return template.render(domain=domain) + return json.dumps({ + "configuration": { "workflowExecutionRetentionPeriodInDays": domain.retention }, + "domainInfo": domain.to_dict() + }) # TODO: implement pagination def list_activity_types(self): - return self._list_types("activity", LIST_ACTIVITY_TYPES_TEMPLATE) + return self._list_types("activity") def register_activity_type(self): domain = self._params.get("domain") @@ -141,11 +143,11 @@ class SWFResponse(BaseResponse): return self._deprecate_type("activity") def describe_activity_type(self): - return self._describe_type("activity", DESCRIBE_ACTIVITY_TYPE_TEMPLATE) + return self._describe_type("activity") # TODO: refactor with list_activity_types() def list_workflow_types(self): - return self._list_types("workflow", LIST_WORKFLOW_TYPES_TEMPLATE) + return self._list_types("workflow") def register_workflow_type(self): domain = self._params.get("domain") @@ -176,88 +178,4 @@ class SWFResponse(BaseResponse): return self._deprecate_type("workflow") def describe_workflow_type(self): - return self._describe_type("workflow", DESCRIBE_WORKFLOW_TYPE_TEMPLATE) - - -LIST_DOMAINS_TEMPLATE = """{ - "domainInfos": [ - {%- for domain in domains %} - { - "description": "{{ domain.description }}", - "name": "{{ domain.name }}", - "status": "{{ domain.status }}" - }{% if not loop.last %},{% endif %} - {%- endfor %} - ] -}""" - -DESCRIBE_DOMAIN_TEMPLATE = """{ - "configuration": { - "workflowExecutionRetentionPeriodInDays": "{{ domain.retention }}" - }, - "domainInfo": { - "description": "{{ domain.description }}", - "name": "{{ domain.name }}", - "status": "{{ domain.status }}" - } -}""" - -LIST_ACTIVITY_TYPES_TEMPLATE = """{ - "typeInfos": [ - {%- for _type in types %} - { - "activityType": { - "name": "{{ _type.name }}", - "version": "{{ _type.version }}" - }, - "creationDate": 1420066800, - {% if _type.status == "DEPRECATED" %}"deprecationDate": 1422745200,{% endif %} - {% if _type.description %}"description": "{{ _type.description }}",{% endif %} - "status": "{{ _type.status }}" - }{% if not loop.last %},{% endif %} - {%- endfor %} - ] -}""" - -DESCRIBE_ACTIVITY_TYPE_TEMPLATE = """{ - "configuration": { - {% if _type.default_task_heartbeat_timeout %}"defaultTaskHeartbeatTimeout": "{{ _type.default_task_heartbeat_timeout }}",{% endif %} - {% if _type.task_list %}"defaultTaskList": { "name": "{{ _type.task_list }}" },{% endif %} - {% if _type.default_task_schedule_to_close_timeout %}"defaultTaskScheduleToCloseTimeout": "{{ _type.default_task_schedule_to_close_timeout }}",{% endif %} - {% if _type.default_task_schedule_to_start_timeout %}"defaultTaskScheduleToStartTimeout": "{{ _type.default_task_schedule_to_start_timeout }}",{% endif %} - {% if _type.default_task_start_to_close_timeout %}"defaultTaskStartToCloseTimeout": "{{ _type.default_task_start_to_close_timeout }}",{% endif %} - "__moto_placeholder": "(avoid dealing with coma in json)" - }, - "typeInfo": { - "activityType": { - "name": "{{ _type.name }}", - "version": "{{ _type.version }}" - }, - "creationDate": 1420066800, - {% if _type.status == "DEPRECATED" %}"deprecationDate": 1422745200,{% endif %} - {% if _type.description %}"description": "{{ _type.description }}",{% endif %} - "status": "{{ _type.status }}" - } -}""" - -LIST_WORKFLOW_TYPES_TEMPLATE = LIST_ACTIVITY_TYPES_TEMPLATE.replace("activityType", "workflowType") - -DESCRIBE_WORKFLOW_TYPE_TEMPLATE = """{ - "configuration": { - {% if _type.default_child_policy %}"defaultChildPolicy": "{{ _type.default_child_policy }}",{% endif %} - {% if _type.default_execution_start_to_close_timeout %}"defaultExecutionStartToCloseTimeout": "{{ _type.default_execution_start_to_close_timeout }}",{% endif %} - {% if _type.task_list %}"defaultTaskList": { "name": "{{ _type.task_list }}" },{% endif %} - {% if _type.default_task_start_to_close_timeout %}"defaultTaskStartToCloseTimeout": "{{ _type.default_task_start_to_close_timeout }}",{% endif %} - "__moto_placeholder": "(avoid dealing with coma in json)" - }, - "typeInfo": { - "workflowType": { - "name": "{{ _type.name }}", - "version": "{{ _type.version }}" - }, - "creationDate": 1420066800, - {% if _type.status == "DEPRECATED" %}"deprecationDate": 1422745200,{% endif %} - {% if _type.description %}"description": "{{ _type.description }}",{% endif %} - "status": "{{ _type.status }}" - } -}""" + return self._describe_type("workflow") diff --git a/tests/test_swf/test_activity_types.py b/tests/test_swf/test_activity_types.py index 19f586a0b..567b0fa0c 100644 --- a/tests/test_swf/test_activity_types.py +++ b/tests/test_swf/test_activity_types.py @@ -3,6 +3,7 @@ from nose.tools import assert_raises from sure import expect from moto import mock_swf +from moto.swf.models import ActivityType from moto.swf.exceptions import ( SWFUnknownResourceFault, SWFTypeAlreadyExistsFault, @@ -11,6 +12,37 @@ from moto.swf.exceptions import ( ) +# Models +def test_short_dict_representation(): + _type = ActivityType("test-activity", "v1.0") + _type.to_short_dict().should.equal({"name": "test-activity", "version": "v1.0"}) + +def test_medium_dict_representation(): + _type = ActivityType("test-activity", "v1.0") + _type.to_medium_dict()["activityType"].should.equal(_type.to_short_dict()) + _type.to_medium_dict()["status"].should.equal("REGISTERED") + _type.to_medium_dict().should.contain("creationDate") + _type.to_medium_dict().should_not.contain("deprecationDate") + _type.to_medium_dict().should_not.contain("description") + + _type.description = "foo bar" + _type.to_medium_dict()["description"].should.equal("foo bar") + + _type.status = "DEPRECATED" + _type.to_medium_dict().should.contain("deprecationDate") + +def test_full_dict_representation(): + _type = ActivityType("test-activity", "v1.0") + _type.to_full_dict()["typeInfo"].should.equal(_type.to_medium_dict()) + _type.to_full_dict()["configuration"].should.equal({}) + + _type.task_list = "foo" + _type.to_full_dict()["configuration"]["defaultTaskList"].should.equal({"name":"foo"}) + + _type.default_task_heartbeat_timeout = "60" + _type.to_full_dict()["configuration"]["defaultTaskHeartbeatTimeout"].should.equal("60") + + # RegisterActivityType endpoint @mock_swf def test_register_activity_type(): @@ -138,7 +170,6 @@ def test_describe_activity_type(): actype = conn.describe_activity_type("test-domain", "test-activity", "v1.0") actype["configuration"]["defaultTaskList"]["name"].should.equal("foo") - actype["configuration"].keys().should_not.contain("defaultTaskScheduleToClose") infos = actype["typeInfo"] infos["activityType"]["name"].should.equal("test-activity") infos["activityType"]["version"].should.equal("v1.0") diff --git a/tests/test_swf/test_domains.py b/tests/test_swf/test_domains.py index 8f3a75569..2e25c1ac1 100644 --- a/tests/test_swf/test_domains.py +++ b/tests/test_swf/test_domains.py @@ -3,6 +3,7 @@ from nose.tools import assert_raises from sure import expect from moto import mock_swf +from moto.swf.models import Domain from moto.swf.exceptions import ( SWFUnknownResourceFault, SWFDomainAlreadyExistsFault, @@ -11,6 +12,15 @@ from moto.swf.exceptions import ( ) +# Models +def test_dict_representation(): + domain = Domain("foo", "52") + domain.to_dict().should.equal({"name":"foo", "status":"REGISTERED"}) + + domain.description = "foo bar" + domain.to_dict()["description"].should.equal("foo bar") + + # RegisterDomain endpoint @mock_swf def test_register_domain(): @@ -53,7 +63,7 @@ def test_register_with_wrong_parameter_type(): ex.body["__type"].should.equal("com.amazonaws.swf.base.model#SerializationException") -# ListDomain endpoint +# ListDomains endpoint @mock_swf def test_list_domains_order(): conn = boto.connect_swf("the_key", "the_secret") From 5c02fcd94b3c1b0daaff27825d570f95d210fdf1 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Thu, 1 Oct 2015 21:25:25 +0200 Subject: [PATCH 12/94] Abstract away SWF *Type models logic into a GenericType class --- moto/swf/models.py | 76 +++++++++------------------ tests/test_swf/test_activity_types.py | 31 ----------- tests/test_swf/test_models.py | 46 ++++++++++++++++ 3 files changed, 72 insertions(+), 81 deletions(-) create mode 100644 tests/test_swf/test_models.py diff --git a/moto/swf/models.py b/moto/swf/models.py index 421e911c0..9a35eb110 100644 --- a/moto/swf/models.py +++ b/moto/swf/models.py @@ -68,7 +68,7 @@ class Domain(object): return _all -class ActivityType(object): +class GenericType(object): def __init__(self, name, version, **kwargs): self.name = name self.version = version @@ -78,17 +78,13 @@ class ActivityType(object): for key, value in kwargs.iteritems(): self.__setattr__(key, value) - def __repr__(self): - return "ActivityType(name: %(name)s, version: %(version)s)" % self.__dict__ + @property + def kind(self): + raise NotImplementedError() @property def _configuration_keys(self): - return [ - "defaultTaskHeartbeatTimeout", - "defaultTaskScheduleToCloseTimeout", - "defaultTaskScheduleToStartTimeout", - "defaultTaskStartToCloseTimeout", - ] + raise NotImplementedError() def to_short_dict(self): return { @@ -98,7 +94,7 @@ class ActivityType(object): def to_medium_dict(self): hsh = { - "activityType": self.to_short_dict(), + "{}Type".format(self.kind): self.to_short_dict(), "creationDate": 1420066800, "status": self.status, } @@ -124,15 +120,25 @@ class ActivityType(object): hsh["configuration"][key] = getattr(self, attr) return hsh +class ActivityType(GenericType): + def __repr__(self): + return "ActivityType(name: %(name)s, version: %(version)s)" % self.__dict__ -class WorkflowType(object): - def __init__(self, name, version, **kwargs): - self.name = name - self.version = version - self.status = "REGISTERED" - for key, value in kwargs.iteritems(): - self.__setattr__(key, value) + @property + def _configuration_keys(self): + return [ + "defaultTaskHeartbeatTimeout", + "defaultTaskScheduleToCloseTimeout", + "defaultTaskScheduleToStartTimeout", + "defaultTaskStartToCloseTimeout", + ] + @property + def kind(self): + return "activity" + + +class WorkflowType(GenericType): def __repr__(self): return "WorkflowType(name: %(name)s, version: %(version)s)" % self.__dict__ @@ -144,39 +150,9 @@ class WorkflowType(object): "defaultTaskStartToCloseTimeout", ] - def to_short_dict(self): - return { - "name": self.name, - "version": self.version, - } - - def to_medium_dict(self): - hsh = { - "workflowType": self.to_short_dict(), - "creationDate": 1420066800, - "status": self.status, - } - if self.status == "DEPRECATED": - hsh["deprecationDate"] = 1422745200 - if hasattr(self, "description"): - hsh["description"] = self.description - return hsh - - def to_full_dict(self): - hsh = { - "typeInfo": self.to_medium_dict(), - "configuration": {} - } - if hasattr(self, "task_list"): - hsh["configuration"]["defaultTaskList"] = {"name": self.task_list} - for key in self._configuration_keys: - attr = camelcase_to_underscores(key) - if not hasattr(self, attr): - continue - if getattr(self, attr) is None: - continue - hsh["configuration"][key] = getattr(self, attr) - return hsh + @property + def kind(self): + return "workflow" class SWFBackend(BaseBackend): diff --git a/tests/test_swf/test_activity_types.py b/tests/test_swf/test_activity_types.py index 567b0fa0c..91ea12576 100644 --- a/tests/test_swf/test_activity_types.py +++ b/tests/test_swf/test_activity_types.py @@ -12,37 +12,6 @@ from moto.swf.exceptions import ( ) -# Models -def test_short_dict_representation(): - _type = ActivityType("test-activity", "v1.0") - _type.to_short_dict().should.equal({"name": "test-activity", "version": "v1.0"}) - -def test_medium_dict_representation(): - _type = ActivityType("test-activity", "v1.0") - _type.to_medium_dict()["activityType"].should.equal(_type.to_short_dict()) - _type.to_medium_dict()["status"].should.equal("REGISTERED") - _type.to_medium_dict().should.contain("creationDate") - _type.to_medium_dict().should_not.contain("deprecationDate") - _type.to_medium_dict().should_not.contain("description") - - _type.description = "foo bar" - _type.to_medium_dict()["description"].should.equal("foo bar") - - _type.status = "DEPRECATED" - _type.to_medium_dict().should.contain("deprecationDate") - -def test_full_dict_representation(): - _type = ActivityType("test-activity", "v1.0") - _type.to_full_dict()["typeInfo"].should.equal(_type.to_medium_dict()) - _type.to_full_dict()["configuration"].should.equal({}) - - _type.task_list = "foo" - _type.to_full_dict()["configuration"]["defaultTaskList"].should.equal({"name":"foo"}) - - _type.default_task_heartbeat_timeout = "60" - _type.to_full_dict()["configuration"]["defaultTaskHeartbeatTimeout"].should.equal("60") - - # RegisterActivityType endpoint @mock_swf def test_register_activity_type(): diff --git a/tests/test_swf/test_models.py b/tests/test_swf/test_models.py new file mode 100644 index 000000000..2211e2264 --- /dev/null +++ b/tests/test_swf/test_models.py @@ -0,0 +1,46 @@ +from sure import expect + +from moto.swf.models import GenericType + + +class FooType(GenericType): + @property + def kind(self): + return "foo" + + @property + def _configuration_keys(self): + return ["justAnExampleTimeout"] + + +def test_short_dict_representation(): + _type = FooType("test-foo", "v1.0") + _type.to_short_dict().should.equal({"name": "test-foo", "version": "v1.0"}) + +def test_medium_dict_representation(): + _type = FooType("test-foo", "v1.0") + _type.to_medium_dict()["fooType"].should.equal(_type.to_short_dict()) + _type.to_medium_dict()["status"].should.equal("REGISTERED") + _type.to_medium_dict().should.contain("creationDate") + _type.to_medium_dict().should_not.contain("deprecationDate") + _type.to_medium_dict().should_not.contain("description") + + _type.description = "foo bar" + _type.to_medium_dict()["description"].should.equal("foo bar") + + _type.status = "DEPRECATED" + _type.to_medium_dict().should.contain("deprecationDate") + +def test_full_dict_representation(): + _type = FooType("test-foo", "v1.0") + _type.to_full_dict()["typeInfo"].should.equal(_type.to_medium_dict()) + _type.to_full_dict()["configuration"].should.equal({}) + + _type.task_list = "foo" + _type.to_full_dict()["configuration"]["defaultTaskList"].should.equal({"name":"foo"}) + + _type.just_an_example_timeout = "60" + _type.to_full_dict()["configuration"]["justAnExampleTimeout"].should.equal("60") + + _type.non_whitelisted_property = "34" + _type.to_full_dict()["configuration"].keys().should.equal(["defaultTaskList", "justAnExampleTimeout"]) From 080b79338d8039d68eadacaa98695bf7ee2e5c82 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Thu, 1 Oct 2015 21:33:47 +0200 Subject: [PATCH 13/94] Simplify how we store SWF types inside a SWF domain --- moto/swf/models.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/moto/swf/models.py b/moto/swf/models.py index 9a35eb110..27295b710 100644 --- a/moto/swf/models.py +++ b/moto/swf/models.py @@ -22,8 +22,10 @@ class Domain(object): self.retention = retention self.description = description self.status = "REGISTERED" - self.activity_types = defaultdict(dict) - self.workflow_types = defaultdict(dict) + self.types = { + "activity": defaultdict(dict), + "workflow": defaultdict(dict), + } def __repr__(self): return "Domain(name: %(name)s, status: %(status)s)" % self.__dict__ @@ -39,8 +41,7 @@ class Domain(object): def get_type(self, kind, name, version, ignore_empty=False): try: - _types = getattr(self, "{}_types".format(kind)) - return _types[name][version] + return self.types[kind][name][version] except KeyError: if not ignore_empty: raise SWFUnknownResourceFault( @@ -51,17 +52,11 @@ class Domain(object): ) def add_type(self, _type): - if isinstance(_type, ActivityType): - self.activity_types[_type.name][_type.version] = _type - elif isinstance(_type, WorkflowType): - self.workflow_types[_type.name][_type.version] = _type - else: - raise ValueError("Unknown SWF type: {}".format(_type)) + self.types[_type.kind][_type.name][_type.version] = _type def find_types(self, kind, status): _all = [] - _types = getattr(self, "{}_types".format(kind)) - for _, family in _types.iteritems(): + for _, family in self.types[kind].iteritems(): for _, _type in family.iteritems(): if _type.status == status: _all.append(_type) From 8b02c0b85e8125edb6757c43c78f8a1e73f38464 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Fri, 2 Oct 2015 03:37:14 +0200 Subject: [PATCH 14/94] Move *Type __repr__ to GenericType --- moto/swf/models.py | 11 +++++------ tests/test_swf/test_models.py | 4 ++++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/moto/swf/models.py b/moto/swf/models.py index 27295b710..4cdfb3536 100644 --- a/moto/swf/models.py +++ b/moto/swf/models.py @@ -73,6 +73,11 @@ class GenericType(object): for key, value in kwargs.iteritems(): self.__setattr__(key, value) + def __repr__(self): + cls = self.__class__.__name__ + attrs = "name: %(name)s, version: %(version)s" % self.__dict__ + return "{}({})".format(cls, attrs) + @property def kind(self): raise NotImplementedError() @@ -116,9 +121,6 @@ class GenericType(object): return hsh class ActivityType(GenericType): - def __repr__(self): - return "ActivityType(name: %(name)s, version: %(version)s)" % self.__dict__ - @property def _configuration_keys(self): return [ @@ -134,9 +136,6 @@ class ActivityType(GenericType): class WorkflowType(GenericType): - def __repr__(self): - return "WorkflowType(name: %(name)s, version: %(version)s)" % self.__dict__ - @property def _configuration_keys(self): return [ diff --git a/tests/test_swf/test_models.py b/tests/test_swf/test_models.py index 2211e2264..2843e559a 100644 --- a/tests/test_swf/test_models.py +++ b/tests/test_swf/test_models.py @@ -44,3 +44,7 @@ def test_full_dict_representation(): _type.non_whitelisted_property = "34" _type.to_full_dict()["configuration"].keys().should.equal(["defaultTaskList", "justAnExampleTimeout"]) + +def test_string_representation(): + _type = FooType("test-foo", "v1.0") + str(_type).should.equal("FooType(name: test-foo, version: v1.0)") From 036ab194ba62d7189470d0eddf1e6eb0b453417e Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Fri, 2 Oct 2015 03:42:40 +0200 Subject: [PATCH 15/94] Add 'status' to SWF *Type string representation --- moto/swf/models.py | 2 +- tests/test_swf/test_models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/swf/models.py b/moto/swf/models.py index 4cdfb3536..3b0eebfe6 100644 --- a/moto/swf/models.py +++ b/moto/swf/models.py @@ -75,7 +75,7 @@ class GenericType(object): def __repr__(self): cls = self.__class__.__name__ - attrs = "name: %(name)s, version: %(version)s" % self.__dict__ + attrs = "name: %(name)s, version: %(version)s, status: %(status)s" % self.__dict__ return "{}({})".format(cls, attrs) @property diff --git a/tests/test_swf/test_models.py b/tests/test_swf/test_models.py index 2843e559a..014cbed4f 100644 --- a/tests/test_swf/test_models.py +++ b/tests/test_swf/test_models.py @@ -47,4 +47,4 @@ def test_full_dict_representation(): def test_string_representation(): _type = FooType("test-foo", "v1.0") - str(_type).should.equal("FooType(name: test-foo, version: v1.0)") + str(_type).should.equal("FooType(name: test-foo, version: v1.0, status: REGISTERED)") From 33c478bc62e22f1375c35efec6928b8e16676e32 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Fri, 2 Oct 2015 03:46:23 +0200 Subject: [PATCH 16/94] Move SWF Domain related tests with other models tests --- tests/test_swf/test_domains.py | 10 ---------- tests/test_swf/test_models.py | 27 ++++++++++++++++++++++----- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/tests/test_swf/test_domains.py b/tests/test_swf/test_domains.py index 2e25c1ac1..013eb1b63 100644 --- a/tests/test_swf/test_domains.py +++ b/tests/test_swf/test_domains.py @@ -3,7 +3,6 @@ from nose.tools import assert_raises from sure import expect from moto import mock_swf -from moto.swf.models import Domain from moto.swf.exceptions import ( SWFUnknownResourceFault, SWFDomainAlreadyExistsFault, @@ -12,15 +11,6 @@ from moto.swf.exceptions import ( ) -# Models -def test_dict_representation(): - domain = Domain("foo", "52") - domain.to_dict().should.equal({"name":"foo", "status":"REGISTERED"}) - - domain.description = "foo bar" - domain.to_dict()["description"].should.equal("foo bar") - - # RegisterDomain endpoint @mock_swf def test_register_domain(): diff --git a/tests/test_swf/test_models.py b/tests/test_swf/test_models.py index 014cbed4f..c3773ba61 100644 --- a/tests/test_swf/test_models.py +++ b/tests/test_swf/test_models.py @@ -1,8 +1,25 @@ from sure import expect -from moto.swf.models import GenericType +from moto.swf.models import ( + Domain, + GenericType, +) +# Domain +def test_domain_dict_representation(): + domain = Domain("foo", "52") + domain.to_dict().should.equal({"name":"foo", "status":"REGISTERED"}) + + domain.description = "foo bar" + domain.to_dict()["description"].should.equal("foo bar") + +def test_domain_string_representation(): + domain = Domain("my-domain", "60") + str(domain).should.equal("Domain(name: my-domain, status: REGISTERED)") + + +# GenericType (ActivityType, WorkflowType) class FooType(GenericType): @property def kind(self): @@ -13,11 +30,11 @@ class FooType(GenericType): return ["justAnExampleTimeout"] -def test_short_dict_representation(): +def test_type_short_dict_representation(): _type = FooType("test-foo", "v1.0") _type.to_short_dict().should.equal({"name": "test-foo", "version": "v1.0"}) -def test_medium_dict_representation(): +def test_type_medium_dict_representation(): _type = FooType("test-foo", "v1.0") _type.to_medium_dict()["fooType"].should.equal(_type.to_short_dict()) _type.to_medium_dict()["status"].should.equal("REGISTERED") @@ -31,7 +48,7 @@ def test_medium_dict_representation(): _type.status = "DEPRECATED" _type.to_medium_dict().should.contain("deprecationDate") -def test_full_dict_representation(): +def test_type_full_dict_representation(): _type = FooType("test-foo", "v1.0") _type.to_full_dict()["typeInfo"].should.equal(_type.to_medium_dict()) _type.to_full_dict()["configuration"].should.equal({}) @@ -45,6 +62,6 @@ def test_full_dict_representation(): _type.non_whitelisted_property = "34" _type.to_full_dict()["configuration"].keys().should.equal(["defaultTaskList", "justAnExampleTimeout"]) -def test_string_representation(): +def test_type_string_representation(): _type = FooType("test-foo", "v1.0") str(_type).should.equal("FooType(name: test-foo, version: v1.0, status: REGISTERED)") From 168f61c6a89a7405c22c5b39c7fb3508ae2ca7b9 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Fri, 2 Oct 2015 03:58:26 +0200 Subject: [PATCH 17/94] Remove useless usage of templating in SWF responses implementation --- moto/swf/responses.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/moto/swf/responses.py b/moto/swf/responses.py index 82c505653..aa627a71e 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -76,8 +76,7 @@ class SWFResponse(BaseResponse): name = _type["name"] version = _type["version"] self.swf_backend.deprecate_type(kind, domain, name, version) - template = self.response_template("") - return template.render() + return "" # TODO: implement pagination def list_domains(self): @@ -136,8 +135,7 @@ class SWFResponse(BaseResponse): default_task_start_to_close_timeout=default_task_start_to_close_timeout, description=description, ) - template = self.response_template("") - return template.render() + return "" def deprecate_activity_type(self): return self._deprecate_type("activity") @@ -171,8 +169,7 @@ class SWFResponse(BaseResponse): default_execution_start_to_close_timeout=default_execution_start_to_close_timeout, description=description, ) - template = self.response_template("") - return template.render() + return "" def deprecate_workflow_type(self): return self._deprecate_type("workflow") From fbcdd5f2bd556dab5c8a36ca1215c8a52c2eef27 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Fri, 2 Oct 2015 04:17:52 +0200 Subject: [PATCH 18/94] Use dict[] to document required keys in SWF responses --- moto/swf/responses.py | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/moto/swf/responses.py b/moto/swf/responses.py index aa627a71e..9da554ede 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -52,8 +52,8 @@ class SWFResponse(BaseResponse): return json.loads(self.body) def _list_types(self, kind): - domain_name = self._params.get("domain") - status = self._params.get("registrationStatus") + domain_name = self._params["domain"] + status = self._params["registrationStatus"] reverse_order = self._params.get("reverseOrder", None) types = self.swf_backend.list_types(kind, domain_name, status, reverse_order=reverse_order) return json.dumps({ @@ -61,9 +61,8 @@ class SWFResponse(BaseResponse): }) def _describe_type(self, kind): - domain = self._params.get("domain") - _type_args = self._params.get("{}Type".format(kind)) - + domain = self._params["domain"] + _type_args = self._params["{}Type".format(kind)] name = _type_args["name"] version = _type_args["version"] _type = self.swf_backend.describe_type(kind, domain, name, version) @@ -71,16 +70,16 @@ class SWFResponse(BaseResponse): return json.dumps(_type.to_full_dict()) def _deprecate_type(self, kind): - domain = self._params.get("domain") - _type = self._params.get("{}Type".format(kind)) - name = _type["name"] - version = _type["version"] + domain = self._params["domain"] + _type_args = self._params["{}Type".format(kind)] + name = _type_args["name"] + version = _type_args["version"] self.swf_backend.deprecate_type(kind, domain, name, version) return "" # TODO: implement pagination def list_domains(self): - status = self._params.get("registrationStatus") + status = self._params["registrationStatus"] reverse_order = self._params.get("reverseOrder", None) domains = self.swf_backend.list_domains(status, reverse_order=reverse_order) return json.dumps({ @@ -88,20 +87,20 @@ class SWFResponse(BaseResponse): }) def register_domain(self): - name = self._params.get("name") + name = self._params["name"] + retention = self._params["workflowExecutionRetentionPeriodInDays"] description = self._params.get("description") - retention = self._params.get("workflowExecutionRetentionPeriodInDays") domain = self.swf_backend.register_domain(name, retention, description=description) return "" def deprecate_domain(self): - name = self._params.get("name") + name = self._params["name"] domain = self.swf_backend.deprecate_domain(name) return "" def describe_domain(self): - name = self._params.get("name") + name = self._params["name"] domain = self.swf_backend.describe_domain(name) return json.dumps({ "configuration": { "workflowExecutionRetentionPeriodInDays": domain.retention }, @@ -113,9 +112,9 @@ class SWFResponse(BaseResponse): return self._list_types("activity") def register_activity_type(self): - domain = self._params.get("domain") - name = self._params.get("name") - version = self._params.get("version") + domain = self._params["domain"] + name = self._params["name"] + version = self._params["version"] default_task_list = self._params.get("defaultTaskList") if default_task_list: task_list = default_task_list.get("name") @@ -148,9 +147,9 @@ class SWFResponse(BaseResponse): return self._list_types("workflow") def register_workflow_type(self): - domain = self._params.get("domain") - name = self._params.get("name") - version = self._params.get("version") + domain = self._params["domain"] + name = self._params["name"] + version = self._params["version"] default_task_list = self._params.get("defaultTaskList") if default_task_list: task_list = default_task_list.get("name") From 92cf64c2adafebb35b2dbc1e7f5b685e6bf34a1e Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Fri, 2 Oct 2015 05:03:10 +0200 Subject: [PATCH 19/94] Add SWF endpoint: StartWorkflowExecution --- moto/swf/exceptions.py | 7 +++ moto/swf/models.py | 71 ++++++++++++++++++++-- moto/swf/responses.py | 29 +++++++++ tests/test_swf/test_models.py | 17 ++++++ tests/test_swf/test_workflow_executions.py | 59 ++++++++++++++++++ 5 files changed, 177 insertions(+), 6 deletions(-) create mode 100644 tests/test_swf/test_workflow_executions.py diff --git a/moto/swf/exceptions.py b/moto/swf/exceptions.py index 5510f1a88..bd04002f4 100644 --- a/moto/swf/exceptions.py +++ b/moto/swf/exceptions.py @@ -56,3 +56,10 @@ class SWFTypeDeprecatedFault(SWFClientError): "{}=[name={}, version={}]".format(_type.__class__.__name__, _type.name, _type.version), "com.amazonaws.swf.base.model#TypeDeprecatedFault") + +class SWFWorkflowExecutionAlreadyStartedFault(JSONResponseError): + def __init__(self): + super(SWFWorkflowExecutionAlreadyStartedFault, self).__init__( + 400, "Bad Request", + body={"__type": "com.amazonaws.swf.base.model#WorkflowExecutionAlreadyStartedFault"} + ) diff --git a/moto/swf/models.py b/moto/swf/models.py index 3b0eebfe6..921b0d97c 100644 --- a/moto/swf/models.py +++ b/moto/swf/models.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals from collections import defaultdict +import uuid import boto.swf @@ -13,6 +14,7 @@ from .exceptions import ( SWFSerializationException, SWFTypeAlreadyExistsFault, SWFTypeDeprecatedFault, + SWFWorkflowExecutionAlreadyStartedFault, ) @@ -26,6 +28,12 @@ class Domain(object): "activity": defaultdict(dict), "workflow": defaultdict(dict), } + # Workflow executions have an id, which unicity is guaranteed + # at domain level (not super clear in the docs, but I checked + # that against SWF API) ; hence the storage method as a dict + # of "workflow_id (client determined)" => WorkflowExecution() + # here. + self.workflow_executions = {} def __repr__(self): return "Domain(name: %(name)s, status: %(status)s)" % self.__dict__ @@ -62,6 +70,11 @@ class Domain(object): _all.append(_type) return _all + def add_workflow_execution(self, workflow_execution_id, workflow_execution): + if self.workflow_executions.get(workflow_execution_id): + raise SWFWorkflowExecutionAlreadyStartedFault() + self.workflow_executions[workflow_execution_id] = workflow_execution + class GenericType(object): def __init__(self, name, version, **kwargs): @@ -120,6 +133,7 @@ class GenericType(object): hsh["configuration"][key] = getattr(self, attr) return hsh + class ActivityType(GenericType): @property def _configuration_keys(self): @@ -149,6 +163,17 @@ class WorkflowType(GenericType): return "workflow" +class WorkflowExecution(object): + def __init__(self, workflow_type, **kwargs): + self.workflow_type = workflow_type + self.run_id = uuid.uuid4().hex + for key, value in kwargs.iteritems(): + self.__setattr__(key, value) + + def __repr__(self): + return "WorkflowExecution(run_id: {})".format(self.run_id) + + class SWFBackend(BaseBackend): def __init__(self, region_name): self.region_name = region_name @@ -168,10 +193,25 @@ class SWFBackend(BaseBackend): return matching[0] return None + def _check_none_or_string(self, parameter): + if parameter is not None: + self._check_string(parameter) + def _check_string(self, parameter): if not isinstance(parameter, basestring): raise SWFSerializationException(parameter) + def _check_none_or_list_of_strings(self, parameter): + if parameter is not None: + self._check_list_of_strings(parameter) + + def _check_list_of_strings(self, parameter): + if not isinstance(parameter, list): + raise SWFSerializationException(parameter) + for i in parameter: + if not isinstance(i, basestring): + raise SWFSerializationException(parameter) + def list_domains(self, status, reverse_order=None): self._check_string(status) domains = [domain for domain in self.domains @@ -185,8 +225,7 @@ class SWFBackend(BaseBackend): description=None): self._check_string(name) self._check_string(workflow_execution_retention_period_in_days) - if description: - self._check_string(description) + self._check_none_or_string(description) if self._get_domain(name, ignore_empty=True): raise SWFDomainAlreadyExistsFault(name) domain = Domain(name, workflow_execution_retention_period_in_days, @@ -219,10 +258,7 @@ class SWFBackend(BaseBackend): self._check_string(name) self._check_string(version) for _, value in kwargs.iteritems(): - if value == (None,): - print _ - if value is not None: - self._check_string(value) + self._check_none_or_string(value) domain = self._get_domain(domain_name) _type = domain.get_type(kind, name, version, ignore_empty=True) if _type: @@ -248,6 +284,29 @@ class SWFBackend(BaseBackend): domain = self._get_domain(domain_name) return domain.get_type(kind, name, version) + # TODO: find what triggers a "DefaultUndefinedFault" and implement it + # (didn't found in boto source code, nor in the docs, nor on a Google search) + # (will try to reach support) + def start_workflow_execution(self, domain_name, workflow_execution_id, + workflow_name, workflow_version, + tag_list=None, **kwargs): + self._check_string(domain_name) + self._check_string(workflow_execution_id) + self._check_string(workflow_name) + self._check_string(workflow_version) + self._check_none_or_list_of_strings(tag_list) + for _, value in kwargs.iteritems(): + self._check_none_or_string(value) + + domain = self._get_domain(domain_name) + wf_type = domain.get_type("workflow", workflow_name, workflow_version) + if wf_type.status == "DEPRECATED": + raise SWFTypeDeprecatedFault(wf_type) + wfe = WorkflowExecution(wf_type, tag_list=tag_list, **kwargs) + domain.add_workflow_execution(workflow_execution_id, wfe) + + return wfe + swf_backends = {} for region in boto.swf.regions(): diff --git a/moto/swf/responses.py b/moto/swf/responses.py index 9da554ede..8e37cbaea 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -175,3 +175,32 @@ class SWFResponse(BaseResponse): def describe_workflow_type(self): return self._describe_type("workflow") + + def start_workflow_execution(self): + domain = self._params["domain"] + workflow_id = self._params["workflowId"] + _workflow_type = self._params["workflowType"] + workflow_name = _workflow_type["name"] + workflow_version = _workflow_type["version"] + _default_task_list = self._params.get("defaultTaskList") + if _default_task_list: + task_list = _default_task_list.get("name") + else: + task_list = None + child_policy = self._params.get("childPolicy") + execution_start_to_close_timeout = self._params.get("executionStartToCloseTimeout") + input_ = self._params.get("input") + tag_list = self._params.get("tagList") + task_start_to_close_timeout = self._params.get("taskStartToCloseTimeout") + + wfe = self.swf_backend.start_workflow_execution( + domain, workflow_id, workflow_name, workflow_version, + task_list=task_list, child_policy=child_policy, + execution_start_to_close_timeout=execution_start_to_close_timeout, + input=input_, tag_list=tag_list, + task_start_to_close_timeout=task_start_to_close_timeout + ) + + return json.dumps({ + "runId": wfe.run_id + }) diff --git a/tests/test_swf/test_models.py b/tests/test_swf/test_models.py index c3773ba61..67b6f6dca 100644 --- a/tests/test_swf/test_models.py +++ b/tests/test_swf/test_models.py @@ -3,6 +3,7 @@ from sure import expect from moto.swf.models import ( Domain, GenericType, + WorkflowExecution, ) @@ -65,3 +66,19 @@ def test_type_full_dict_representation(): def test_type_string_representation(): _type = FooType("test-foo", "v1.0") str(_type).should.equal("FooType(name: test-foo, version: v1.0, status: REGISTERED)") + + +# WorkflowExecution +def test_workflow_execution_creation(): + wfe = WorkflowExecution("workflow_type_whatever", child_policy="TERMINATE") + wfe.workflow_type.should.equal("workflow_type_whatever") + wfe.child_policy.should.equal("TERMINATE") + +def test_workflow_execution_string_representation(): + wfe = WorkflowExecution("workflow_type_whatever", child_policy="TERMINATE") + str(wfe).should.match(r"^WorkflowExecution\(run_id: .*\)") + +def test_workflow_execution_generates_a_random_run_id(): + wfe1 = WorkflowExecution("workflow_type_whatever") + wfe2 = WorkflowExecution("workflow_type_whatever") + wfe1.run_id.should_not.equal(wfe2.run_id) diff --git a/tests/test_swf/test_workflow_executions.py b/tests/test_swf/test_workflow_executions.py new file mode 100644 index 000000000..930fba538 --- /dev/null +++ b/tests/test_swf/test_workflow_executions.py @@ -0,0 +1,59 @@ +import boto +from nose.tools import assert_raises +from sure import expect + +from moto import mock_swf +from moto.swf.exceptions import ( + SWFWorkflowExecutionAlreadyStartedFault, + SWFTypeDeprecatedFault, +) + + +# Utils +@mock_swf +def setup_swf_environment(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60", description="A test domain") + conn.register_workflow_type("test-domain", "test-workflow", "v1.0") + conn.register_activity_type("test-domain", "test-activity", "v1.1") + return conn + + +# StartWorkflowExecution endpoint +@mock_swf +def test_start_workflow_execution(): + conn = setup_swf_environment() + + wf = conn.start_workflow_execution("test-domain", "uid-abcd1234", "test-workflow", "v1.0") + wf.should.contain("runId") + +@mock_swf +def test_start_already_started_workflow_execution(): + conn = setup_swf_environment() + conn.start_workflow_execution("test-domain", "uid-abcd1234", "test-workflow", "v1.0") + + with assert_raises(SWFWorkflowExecutionAlreadyStartedFault) as err: + conn.start_workflow_execution("test-domain", "uid-abcd1234", "test-workflow", "v1.0") + + ex = err.exception + ex.status.should.equal(400) + ex.error_code.should.equal("WorkflowExecutionAlreadyStartedFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#WorkflowExecutionAlreadyStartedFault", + }) + +@mock_swf +def test_start_workflow_execution_on_deprecated_type(): + conn = setup_swf_environment() + conn.deprecate_workflow_type("test-domain", "test-workflow", "v1.0") + + with assert_raises(SWFTypeDeprecatedFault) as err: + conn.start_workflow_execution("test-domain", "uid-abcd1234", "test-workflow", "v1.0") + + ex = err.exception + ex.status.should.equal(400) + ex.error_code.should.equal("TypeDeprecatedFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#TypeDeprecatedFault", + "message": "WorkflowType=[name=test-workflow, version=v1.0]" + }) From c08c20d1972099330a2a4c3b0c91f355a0b47081 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Fri, 2 Oct 2015 09:33:35 +0200 Subject: [PATCH 20/94] Move SWF Domain full dict representation to model --- moto/swf/models.py | 10 +++++++++- moto/swf/responses.py | 7 ++----- tests/test_swf/test_models.py | 13 ++++++++++--- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/moto/swf/models.py b/moto/swf/models.py index 921b0d97c..6fe724038 100644 --- a/moto/swf/models.py +++ b/moto/swf/models.py @@ -38,7 +38,7 @@ class Domain(object): def __repr__(self): return "Domain(name: %(name)s, status: %(status)s)" % self.__dict__ - def to_dict(self): + def to_short_dict(self): hsh = { "name": self.name, "status": self.status, @@ -47,6 +47,14 @@ class Domain(object): hsh["description"] = self.description return hsh + def to_full_dict(self): + return { + "domainInfo": self.to_short_dict(), + "configuration": { + "workflowExecutionRetentionPeriodInDays": self.retention, + } + } + def get_type(self, kind, name, version, ignore_empty=False): try: return self.types[kind][name][version] diff --git a/moto/swf/responses.py b/moto/swf/responses.py index 8e37cbaea..105db4c06 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -83,7 +83,7 @@ class SWFResponse(BaseResponse): reverse_order = self._params.get("reverseOrder", None) domains = self.swf_backend.list_domains(status, reverse_order=reverse_order) return json.dumps({ - "domainInfos": [domain.to_dict() for domain in domains] + "domainInfos": [domain.to_short_dict() for domain in domains] }) def register_domain(self): @@ -102,10 +102,7 @@ class SWFResponse(BaseResponse): def describe_domain(self): name = self._params["name"] domain = self.swf_backend.describe_domain(name) - return json.dumps({ - "configuration": { "workflowExecutionRetentionPeriodInDays": domain.retention }, - "domainInfo": domain.to_dict() - }) + return json.dumps(domain.to_full_dict()) # TODO: implement pagination def list_activity_types(self): diff --git a/tests/test_swf/test_models.py b/tests/test_swf/test_models.py index 67b6f6dca..7f636673c 100644 --- a/tests/test_swf/test_models.py +++ b/tests/test_swf/test_models.py @@ -8,12 +8,19 @@ from moto.swf.models import ( # Domain -def test_domain_dict_representation(): +def test_domain_short_dict_representation(): domain = Domain("foo", "52") - domain.to_dict().should.equal({"name":"foo", "status":"REGISTERED"}) + domain.to_short_dict().should.equal({"name":"foo", "status":"REGISTERED"}) domain.description = "foo bar" - domain.to_dict()["description"].should.equal("foo bar") + domain.to_short_dict()["description"].should.equal("foo bar") + +def test_domain_full_dict_representation(): + domain = Domain("foo", "52") + + domain.to_full_dict()["domainInfo"].should.equal(domain.to_short_dict()) + _config = domain.to_full_dict()["configuration"] + _config["workflowExecutionRetentionPeriodInDays"].should.equal("52") def test_domain_string_representation(): domain = Domain("my-domain", "60") From a589dc08b50b360b2c4b27280663b9ac6b257273 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Fri, 2 Oct 2015 09:41:29 +0200 Subject: [PATCH 21/94] Make workflow_id a required property of WorkflowExecution Given the response of DescribeWorkflowExecution endpoint, the WorkflowExecution has to know about its own workflowId. --- moto/swf/models.py | 19 +++++++++++-------- tests/test_swf/test_models.py | 8 ++++---- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/moto/swf/models.py b/moto/swf/models.py index 6fe724038..dc29dc664 100644 --- a/moto/swf/models.py +++ b/moto/swf/models.py @@ -78,10 +78,11 @@ class Domain(object): _all.append(_type) return _all - def add_workflow_execution(self, workflow_execution_id, workflow_execution): - if self.workflow_executions.get(workflow_execution_id): + def add_workflow_execution(self, workflow_execution): + _id = workflow_execution.workflow_id + if self.workflow_executions.get(_id): raise SWFWorkflowExecutionAlreadyStartedFault() - self.workflow_executions[workflow_execution_id] = workflow_execution + self.workflow_executions[_id] = workflow_execution class GenericType(object): @@ -172,8 +173,9 @@ class WorkflowType(GenericType): class WorkflowExecution(object): - def __init__(self, workflow_type, **kwargs): + def __init__(self, workflow_type, workflow_id, **kwargs): self.workflow_type = workflow_type + self.workflow_id = workflow_id self.run_id = uuid.uuid4().hex for key, value in kwargs.iteritems(): self.__setattr__(key, value) @@ -295,11 +297,11 @@ class SWFBackend(BaseBackend): # TODO: find what triggers a "DefaultUndefinedFault" and implement it # (didn't found in boto source code, nor in the docs, nor on a Google search) # (will try to reach support) - def start_workflow_execution(self, domain_name, workflow_execution_id, + def start_workflow_execution(self, domain_name, workflow_id, workflow_name, workflow_version, tag_list=None, **kwargs): self._check_string(domain_name) - self._check_string(workflow_execution_id) + self._check_string(workflow_id) self._check_string(workflow_name) self._check_string(workflow_version) self._check_none_or_list_of_strings(tag_list) @@ -310,8 +312,9 @@ class SWFBackend(BaseBackend): wf_type = domain.get_type("workflow", workflow_name, workflow_version) if wf_type.status == "DEPRECATED": raise SWFTypeDeprecatedFault(wf_type) - wfe = WorkflowExecution(wf_type, tag_list=tag_list, **kwargs) - domain.add_workflow_execution(workflow_execution_id, wfe) + wfe = WorkflowExecution(wf_type, workflow_id, + tag_list=tag_list, **kwargs) + domain.add_workflow_execution(wfe) return wfe diff --git a/tests/test_swf/test_models.py b/tests/test_swf/test_models.py index 7f636673c..33006cbf7 100644 --- a/tests/test_swf/test_models.py +++ b/tests/test_swf/test_models.py @@ -77,15 +77,15 @@ def test_type_string_representation(): # WorkflowExecution def test_workflow_execution_creation(): - wfe = WorkflowExecution("workflow_type_whatever", child_policy="TERMINATE") + wfe = WorkflowExecution("workflow_type_whatever", "ab1234", child_policy="TERMINATE") wfe.workflow_type.should.equal("workflow_type_whatever") wfe.child_policy.should.equal("TERMINATE") def test_workflow_execution_string_representation(): - wfe = WorkflowExecution("workflow_type_whatever", child_policy="TERMINATE") + wfe = WorkflowExecution("workflow_type_whatever", "ab1234", child_policy="TERMINATE") str(wfe).should.match(r"^WorkflowExecution\(run_id: .*\)") def test_workflow_execution_generates_a_random_run_id(): - wfe1 = WorkflowExecution("workflow_type_whatever") - wfe2 = WorkflowExecution("workflow_type_whatever") + wfe1 = WorkflowExecution("workflow_type_whatever", "ab1234") + wfe2 = WorkflowExecution("workflow_type_whatever", "ab1235") wfe1.run_id.should_not.equal(wfe2.run_id) From 2878252816d85692ead417b7fd1820a72ea5eccb Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Fri, 2 Oct 2015 17:42:28 +0200 Subject: [PATCH 22/94] Add SWF endpoint: DescribeWorkflowExecution --- moto/swf/models.py | 74 ++++++++++++++++++++++ moto/swf/responses.py | 10 ++- tests/test_swf/test_models.py | 42 ++++++++++++ tests/test_swf/test_workflow_executions.py | 28 ++++++++ 4 files changed, 153 insertions(+), 1 deletion(-) diff --git a/moto/swf/models.py b/moto/swf/models.py index dc29dc664..253336be0 100644 --- a/moto/swf/models.py +++ b/moto/swf/models.py @@ -84,6 +84,17 @@ class Domain(object): raise SWFWorkflowExecutionAlreadyStartedFault() self.workflow_executions[_id] = workflow_execution + def get_workflow_execution(self, run_id, workflow_id): + wfe = self.workflow_executions.get(workflow_id) + if not wfe or wfe.run_id != run_id: + raise SWFUnknownResourceFault( + "execution", + "WorkflowExecution=[workflowId={}, runId={}]".format( + workflow_id, run_id + ) + ) + return wfe + class GenericType(object): def __init__(self, name, version, **kwargs): @@ -177,12 +188,68 @@ class WorkflowExecution(object): self.workflow_type = workflow_type self.workflow_id = workflow_id self.run_id = uuid.uuid4().hex + self.execution_status = "OPEN" + self.cancel_requested = False + #config for key, value in kwargs.iteritems(): self.__setattr__(key, value) + #counters + self.open_counts = { + "openTimers": 0, + "openDecisionTasks": 0, + "openActivityTasks": 0, + "openChildWorkflowExecutions": 0, + } def __repr__(self): return "WorkflowExecution(run_id: {})".format(self.run_id) + @property + def _configuration_keys(self): + return [ + "executionStartToCloseTimeout", + "childPolicy", + "taskPriority", + "taskStartToCloseTimeout", + ] + + def to_short_dict(self): + return { + "workflowId": self.workflow_id, + "runId": self.run_id + } + + def to_medium_dict(self): + hsh = { + "execution": self.to_short_dict(), + "workflowType": self.workflow_type.to_short_dict(), + "startTimestamp": 1420066800.123, + "executionStatus": self.execution_status, + "cancelRequested": self.cancel_requested, + } + if hasattr(self, "tag_list"): + hsh["tagList"] = self.tag_list + return hsh + + def to_full_dict(self): + hsh = { + "executionInfo": self.to_medium_dict(), + "executionConfiguration": {} + } + #configuration + if hasattr(self, "task_list"): + hsh["executionConfiguration"]["taskList"] = {"name": self.task_list} + for key in self._configuration_keys: + attr = camelcase_to_underscores(key) + if not hasattr(self, attr): + continue + if not getattr(self, attr): + continue + hsh["executionConfiguration"][key] = getattr(self, attr) + #counters + hsh["openCounts"] = self.open_counts + return hsh + class SWFBackend(BaseBackend): def __init__(self, region_name): @@ -318,6 +385,13 @@ class SWFBackend(BaseBackend): return wfe + def describe_workflow_execution(self, domain_name, run_id, workflow_id): + self._check_string(domain_name) + self._check_string(run_id) + self._check_string(workflow_id) + domain = self._get_domain(domain_name) + return domain.get_workflow_execution(run_id, workflow_id) + swf_backends = {} for region in boto.swf.regions(): diff --git a/moto/swf/responses.py b/moto/swf/responses.py index 105db4c06..b4e349931 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -139,7 +139,6 @@ class SWFResponse(BaseResponse): def describe_activity_type(self): return self._describe_type("activity") - # TODO: refactor with list_activity_types() def list_workflow_types(self): return self._list_types("workflow") @@ -201,3 +200,12 @@ class SWFResponse(BaseResponse): return json.dumps({ "runId": wfe.run_id }) + + def describe_workflow_execution(self): + domain_name = self._params["domain"] + _workflow_execution = self._params["execution"] + run_id = _workflow_execution["runId"] + workflow_id = _workflow_execution["workflowId"] + + wfe = self.swf_backend.describe_workflow_execution(domain_name, run_id, workflow_id) + return json.dumps(wfe.to_full_dict()) diff --git a/tests/test_swf/test_models.py b/tests/test_swf/test_models.py index 33006cbf7..bf6437f5b 100644 --- a/tests/test_swf/test_models.py +++ b/tests/test_swf/test_models.py @@ -3,6 +3,7 @@ from sure import expect from moto.swf.models import ( Domain, GenericType, + WorkflowType, WorkflowExecution, ) @@ -89,3 +90,44 @@ def test_workflow_execution_generates_a_random_run_id(): wfe1 = WorkflowExecution("workflow_type_whatever", "ab1234") wfe2 = WorkflowExecution("workflow_type_whatever", "ab1235") wfe1.run_id.should_not.equal(wfe2.run_id) + +def test_workflow_execution_short_dict_representation(): + wf_type = WorkflowType("test-workflow", "v1.0") + wfe = WorkflowExecution(wf_type, "ab1234") + + sd = wfe.to_short_dict() + sd["workflowId"].should.equal("ab1234") + sd.should.contain("runId") + +def test_workflow_execution_medium_dict_representation(): + wf_type = WorkflowType("test-workflow", "v1.0") + wfe = WorkflowExecution(wf_type, "ab1234") + + md = wfe.to_medium_dict() + md["execution"].should.equal(wfe.to_short_dict()) + md["workflowType"].should.equal(wf_type.to_short_dict()) + md["startTimestamp"].should.be.a('float') + md["executionStatus"].should.equal("OPEN") + md["cancelRequested"].should.equal(False) + md.should_not.contain("tagList") + + wfe.tag_list = ["foo", "bar", "baz"] + md = wfe.to_medium_dict() + md["tagList"].should.equal(["foo", "bar", "baz"]) + +def test_workflow_execution_full_dict_representation(): + wf_type = WorkflowType("test-workflow", "v1.0") + wfe = WorkflowExecution(wf_type, "ab1234") + + fd = wfe.to_full_dict() + fd["executionInfo"].should.equal(wfe.to_medium_dict()) + fd["openCounts"]["openTimers"].should.equal(0) + fd["openCounts"]["openDecisionTasks"].should.equal(0) + fd["openCounts"]["openActivityTasks"].should.equal(0) + fd["executionConfiguration"].should.equal({}) + + wfe.task_list = "special" + wfe.task_start_to_close_timeout = "45" + fd = wfe.to_full_dict() + fd["executionConfiguration"]["taskList"]["name"].should.equal("special") + fd["executionConfiguration"]["taskStartToCloseTimeout"].should.equal("45") diff --git a/tests/test_swf/test_workflow_executions.py b/tests/test_swf/test_workflow_executions.py index 930fba538..4082ab540 100644 --- a/tests/test_swf/test_workflow_executions.py +++ b/tests/test_swf/test_workflow_executions.py @@ -6,6 +6,7 @@ from moto import mock_swf from moto.swf.exceptions import ( SWFWorkflowExecutionAlreadyStartedFault, SWFTypeDeprecatedFault, + SWFUnknownResourceFault, ) @@ -57,3 +58,30 @@ def test_start_workflow_execution_on_deprecated_type(): "__type": "com.amazonaws.swf.base.model#TypeDeprecatedFault", "message": "WorkflowType=[name=test-workflow, version=v1.0]" }) + + +# DescribeWorkflowExecution endpoint +@mock_swf +def test_describe_workflow_execution(): + conn = setup_swf_environment() + hsh = conn.start_workflow_execution("test-domain", "uid-abcd1234", "test-workflow", "v1.0") + run_id = hsh["runId"] + + wfe = conn.describe_workflow_execution("test-domain", run_id, "uid-abcd1234") + wfe["executionInfo"]["execution"]["workflowId"].should.equal("uid-abcd1234") + wfe["executionInfo"]["executionStatus"].should.equal("OPEN") + +@mock_swf +def test_describe_non_existent_workflow_execution(): + conn = setup_swf_environment() + + with assert_raises(SWFUnknownResourceFault) as err: + conn.describe_workflow_execution("test-domain", "wrong-run-id", "wrong-workflow-id") + + ex = err.exception + ex.status.should.equal(400) + ex.error_code.should.equal("UnknownResourceFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#UnknownResourceFault", + "message": "Unknown execution: WorkflowExecution=[workflowId=wrong-workflow-id, runId=wrong-run-id]" + }) From 1026fb819fd759370eb749e852f66e3416c537da Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Sat, 3 Oct 2015 11:24:03 +0200 Subject: [PATCH 23/94] Split SWF models into their own file Given the docs[1] we will implement a hundred models or so if we want to have a full implementation of the SWF API, so better not have a 3k lines long models.py file, too hard to manipulate. [1] http://docs.aws.amazon.com/amazonswf/latest/apireference/API_DecisionTask.html --- moto/swf/models.py | 398 -------------------------- moto/swf/models/__init__.py | 166 +++++++++++ moto/swf/models/activity_type.py | 16 ++ moto/swf/models/domain.py | 85 ++++++ moto/swf/models/generic_type.py | 61 ++++ moto/swf/models/workflow_execution.py | 72 +++++ moto/swf/models/workflow_type.py | 15 + 7 files changed, 415 insertions(+), 398 deletions(-) delete mode 100644 moto/swf/models.py create mode 100644 moto/swf/models/__init__.py create mode 100644 moto/swf/models/activity_type.py create mode 100644 moto/swf/models/domain.py create mode 100644 moto/swf/models/generic_type.py create mode 100644 moto/swf/models/workflow_execution.py create mode 100644 moto/swf/models/workflow_type.py diff --git a/moto/swf/models.py b/moto/swf/models.py deleted file mode 100644 index 253336be0..000000000 --- a/moto/swf/models.py +++ /dev/null @@ -1,398 +0,0 @@ -from __future__ import unicode_literals -from collections import defaultdict -import uuid - -import boto.swf - -from moto.core import BaseBackend -from moto.core.utils import camelcase_to_underscores - -from .exceptions import ( - SWFUnknownResourceFault, - SWFDomainAlreadyExistsFault, - SWFDomainDeprecatedFault, - SWFSerializationException, - SWFTypeAlreadyExistsFault, - SWFTypeDeprecatedFault, - SWFWorkflowExecutionAlreadyStartedFault, -) - - -class Domain(object): - def __init__(self, name, retention, description=None): - self.name = name - self.retention = retention - self.description = description - self.status = "REGISTERED" - self.types = { - "activity": defaultdict(dict), - "workflow": defaultdict(dict), - } - # Workflow executions have an id, which unicity is guaranteed - # at domain level (not super clear in the docs, but I checked - # that against SWF API) ; hence the storage method as a dict - # of "workflow_id (client determined)" => WorkflowExecution() - # here. - self.workflow_executions = {} - - def __repr__(self): - return "Domain(name: %(name)s, status: %(status)s)" % self.__dict__ - - def to_short_dict(self): - hsh = { - "name": self.name, - "status": self.status, - } - if self.description: - hsh["description"] = self.description - return hsh - - def to_full_dict(self): - return { - "domainInfo": self.to_short_dict(), - "configuration": { - "workflowExecutionRetentionPeriodInDays": self.retention, - } - } - - def get_type(self, kind, name, version, ignore_empty=False): - try: - return self.types[kind][name][version] - except KeyError: - if not ignore_empty: - raise SWFUnknownResourceFault( - "type", - "{}Type=[name={}, version={}]".format( - kind.capitalize(), name, version - ) - ) - - def add_type(self, _type): - self.types[_type.kind][_type.name][_type.version] = _type - - def find_types(self, kind, status): - _all = [] - for _, family in self.types[kind].iteritems(): - for _, _type in family.iteritems(): - if _type.status == status: - _all.append(_type) - return _all - - def add_workflow_execution(self, workflow_execution): - _id = workflow_execution.workflow_id - if self.workflow_executions.get(_id): - raise SWFWorkflowExecutionAlreadyStartedFault() - self.workflow_executions[_id] = workflow_execution - - def get_workflow_execution(self, run_id, workflow_id): - wfe = self.workflow_executions.get(workflow_id) - if not wfe or wfe.run_id != run_id: - raise SWFUnknownResourceFault( - "execution", - "WorkflowExecution=[workflowId={}, runId={}]".format( - workflow_id, run_id - ) - ) - return wfe - - -class GenericType(object): - def __init__(self, name, version, **kwargs): - self.name = name - self.version = version - self.status = "REGISTERED" - if "description" in kwargs: - self.description = kwargs.pop("description") - for key, value in kwargs.iteritems(): - self.__setattr__(key, value) - - def __repr__(self): - cls = self.__class__.__name__ - attrs = "name: %(name)s, version: %(version)s, status: %(status)s" % self.__dict__ - return "{}({})".format(cls, attrs) - - @property - def kind(self): - raise NotImplementedError() - - @property - def _configuration_keys(self): - raise NotImplementedError() - - def to_short_dict(self): - return { - "name": self.name, - "version": self.version, - } - - def to_medium_dict(self): - hsh = { - "{}Type".format(self.kind): self.to_short_dict(), - "creationDate": 1420066800, - "status": self.status, - } - if self.status == "DEPRECATED": - hsh["deprecationDate"] = 1422745200 - if hasattr(self, "description"): - hsh["description"] = self.description - return hsh - - def to_full_dict(self): - hsh = { - "typeInfo": self.to_medium_dict(), - "configuration": {} - } - if hasattr(self, "task_list"): - hsh["configuration"]["defaultTaskList"] = {"name": self.task_list} - for key in self._configuration_keys: - attr = camelcase_to_underscores(key) - if not hasattr(self, attr): - continue - if not getattr(self, attr): - continue - hsh["configuration"][key] = getattr(self, attr) - return hsh - - -class ActivityType(GenericType): - @property - def _configuration_keys(self): - return [ - "defaultTaskHeartbeatTimeout", - "defaultTaskScheduleToCloseTimeout", - "defaultTaskScheduleToStartTimeout", - "defaultTaskStartToCloseTimeout", - ] - - @property - def kind(self): - return "activity" - - -class WorkflowType(GenericType): - @property - def _configuration_keys(self): - return [ - "defaultChildPolicy", - "defaultExecutionStartToCloseTimeout", - "defaultTaskStartToCloseTimeout", - ] - - @property - def kind(self): - return "workflow" - - -class WorkflowExecution(object): - def __init__(self, workflow_type, workflow_id, **kwargs): - self.workflow_type = workflow_type - self.workflow_id = workflow_id - self.run_id = uuid.uuid4().hex - self.execution_status = "OPEN" - self.cancel_requested = False - #config - for key, value in kwargs.iteritems(): - self.__setattr__(key, value) - #counters - self.open_counts = { - "openTimers": 0, - "openDecisionTasks": 0, - "openActivityTasks": 0, - "openChildWorkflowExecutions": 0, - } - - def __repr__(self): - return "WorkflowExecution(run_id: {})".format(self.run_id) - - @property - def _configuration_keys(self): - return [ - "executionStartToCloseTimeout", - "childPolicy", - "taskPriority", - "taskStartToCloseTimeout", - ] - - def to_short_dict(self): - return { - "workflowId": self.workflow_id, - "runId": self.run_id - } - - def to_medium_dict(self): - hsh = { - "execution": self.to_short_dict(), - "workflowType": self.workflow_type.to_short_dict(), - "startTimestamp": 1420066800.123, - "executionStatus": self.execution_status, - "cancelRequested": self.cancel_requested, - } - if hasattr(self, "tag_list"): - hsh["tagList"] = self.tag_list - return hsh - - def to_full_dict(self): - hsh = { - "executionInfo": self.to_medium_dict(), - "executionConfiguration": {} - } - #configuration - if hasattr(self, "task_list"): - hsh["executionConfiguration"]["taskList"] = {"name": self.task_list} - for key in self._configuration_keys: - attr = camelcase_to_underscores(key) - if not hasattr(self, attr): - continue - if not getattr(self, attr): - continue - hsh["executionConfiguration"][key] = getattr(self, attr) - #counters - hsh["openCounts"] = self.open_counts - return hsh - - -class SWFBackend(BaseBackend): - def __init__(self, region_name): - self.region_name = region_name - self.domains = [] - super(SWFBackend, self).__init__() - - def reset(self): - region_name = self.region_name - self.__dict__ = {} - self.__init__(region_name) - - def _get_domain(self, name, ignore_empty=False): - matching = [domain for domain in self.domains if domain.name == name] - if not matching and not ignore_empty: - raise SWFUnknownResourceFault("domain", name) - if matching: - return matching[0] - return None - - def _check_none_or_string(self, parameter): - if parameter is not None: - self._check_string(parameter) - - def _check_string(self, parameter): - if not isinstance(parameter, basestring): - raise SWFSerializationException(parameter) - - def _check_none_or_list_of_strings(self, parameter): - if parameter is not None: - self._check_list_of_strings(parameter) - - def _check_list_of_strings(self, parameter): - if not isinstance(parameter, list): - raise SWFSerializationException(parameter) - for i in parameter: - if not isinstance(i, basestring): - raise SWFSerializationException(parameter) - - def list_domains(self, status, reverse_order=None): - self._check_string(status) - domains = [domain for domain in self.domains - if domain.status == status] - domains = sorted(domains, key=lambda domain: domain.name) - if reverse_order: - domains = reversed(domains) - return domains - - def register_domain(self, name, workflow_execution_retention_period_in_days, - description=None): - self._check_string(name) - self._check_string(workflow_execution_retention_period_in_days) - self._check_none_or_string(description) - if self._get_domain(name, ignore_empty=True): - raise SWFDomainAlreadyExistsFault(name) - domain = Domain(name, workflow_execution_retention_period_in_days, - description) - self.domains.append(domain) - - def deprecate_domain(self, name): - self._check_string(name) - domain = self._get_domain(name) - if domain.status == "DEPRECATED": - raise SWFDomainDeprecatedFault(name) - domain.status = "DEPRECATED" - - def describe_domain(self, name): - self._check_string(name) - return self._get_domain(name) - - def list_types(self, kind, domain_name, status, reverse_order=None): - self._check_string(domain_name) - self._check_string(status) - domain = self._get_domain(domain_name) - _types = domain.find_types(kind, status) - _types = sorted(_types, key=lambda domain: domain.name) - if reverse_order: - _types = reversed(_types) - return _types - - def register_type(self, kind, domain_name, name, version, **kwargs): - self._check_string(domain_name) - self._check_string(name) - self._check_string(version) - for _, value in kwargs.iteritems(): - self._check_none_or_string(value) - domain = self._get_domain(domain_name) - _type = domain.get_type(kind, name, version, ignore_empty=True) - if _type: - raise SWFTypeAlreadyExistsFault(_type) - _class = globals()["{}Type".format(kind.capitalize())] - _type = _class(name, version, **kwargs) - domain.add_type(_type) - - def deprecate_type(self, kind, domain_name, name, version): - self._check_string(domain_name) - self._check_string(name) - self._check_string(version) - domain = self._get_domain(domain_name) - _type = domain.get_type(kind, name, version) - if _type.status == "DEPRECATED": - raise SWFTypeDeprecatedFault(_type) - _type.status = "DEPRECATED" - - def describe_type(self, kind, domain_name, name, version): - self._check_string(domain_name) - self._check_string(name) - self._check_string(version) - domain = self._get_domain(domain_name) - return domain.get_type(kind, name, version) - - # TODO: find what triggers a "DefaultUndefinedFault" and implement it - # (didn't found in boto source code, nor in the docs, nor on a Google search) - # (will try to reach support) - def start_workflow_execution(self, domain_name, workflow_id, - workflow_name, workflow_version, - tag_list=None, **kwargs): - self._check_string(domain_name) - self._check_string(workflow_id) - self._check_string(workflow_name) - self._check_string(workflow_version) - self._check_none_or_list_of_strings(tag_list) - for _, value in kwargs.iteritems(): - self._check_none_or_string(value) - - domain = self._get_domain(domain_name) - wf_type = domain.get_type("workflow", workflow_name, workflow_version) - if wf_type.status == "DEPRECATED": - raise SWFTypeDeprecatedFault(wf_type) - wfe = WorkflowExecution(wf_type, workflow_id, - tag_list=tag_list, **kwargs) - domain.add_workflow_execution(wfe) - - return wfe - - def describe_workflow_execution(self, domain_name, run_id, workflow_id): - self._check_string(domain_name) - self._check_string(run_id) - self._check_string(workflow_id) - domain = self._get_domain(domain_name) - return domain.get_workflow_execution(run_id, workflow_id) - - -swf_backends = {} -for region in boto.swf.regions(): - swf_backends[region.name] = SWFBackend(region.name) diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py new file mode 100644 index 000000000..a7be6f64e --- /dev/null +++ b/moto/swf/models/__init__.py @@ -0,0 +1,166 @@ +from __future__ import unicode_literals + +import boto.swf + +from moto.core import BaseBackend + +from ..exceptions import ( + SWFUnknownResourceFault, + SWFDomainAlreadyExistsFault, + SWFDomainDeprecatedFault, + SWFSerializationException, + SWFTypeAlreadyExistsFault, + SWFTypeDeprecatedFault, +) +from .activity_type import ActivityType +from .domain import Domain +from .generic_type import GenericType +from .workflow_type import WorkflowType +from .workflow_execution import WorkflowExecution + + +class SWFBackend(BaseBackend): + def __init__(self, region_name): + self.region_name = region_name + self.domains = [] + super(SWFBackend, self).__init__() + + def reset(self): + region_name = self.region_name + self.__dict__ = {} + self.__init__(region_name) + + def _get_domain(self, name, ignore_empty=False): + matching = [domain for domain in self.domains if domain.name == name] + if not matching and not ignore_empty: + raise SWFUnknownResourceFault("domain", name) + if matching: + return matching[0] + return None + + def _check_none_or_string(self, parameter): + if parameter is not None: + self._check_string(parameter) + + def _check_string(self, parameter): + if not isinstance(parameter, basestring): + raise SWFSerializationException(parameter) + + def _check_none_or_list_of_strings(self, parameter): + if parameter is not None: + self._check_list_of_strings(parameter) + + def _check_list_of_strings(self, parameter): + if not isinstance(parameter, list): + raise SWFSerializationException(parameter) + for i in parameter: + if not isinstance(i, basestring): + raise SWFSerializationException(parameter) + + def list_domains(self, status, reverse_order=None): + self._check_string(status) + domains = [domain for domain in self.domains + if domain.status == status] + domains = sorted(domains, key=lambda domain: domain.name) + if reverse_order: + domains = reversed(domains) + return domains + + def register_domain(self, name, workflow_execution_retention_period_in_days, + description=None): + self._check_string(name) + self._check_string(workflow_execution_retention_period_in_days) + self._check_none_or_string(description) + if self._get_domain(name, ignore_empty=True): + raise SWFDomainAlreadyExistsFault(name) + domain = Domain(name, workflow_execution_retention_period_in_days, + description) + self.domains.append(domain) + + def deprecate_domain(self, name): + self._check_string(name) + domain = self._get_domain(name) + if domain.status == "DEPRECATED": + raise SWFDomainDeprecatedFault(name) + domain.status = "DEPRECATED" + + def describe_domain(self, name): + self._check_string(name) + return self._get_domain(name) + + def list_types(self, kind, domain_name, status, reverse_order=None): + self._check_string(domain_name) + self._check_string(status) + domain = self._get_domain(domain_name) + _types = domain.find_types(kind, status) + _types = sorted(_types, key=lambda domain: domain.name) + if reverse_order: + _types = reversed(_types) + return _types + + def register_type(self, kind, domain_name, name, version, **kwargs): + self._check_string(domain_name) + self._check_string(name) + self._check_string(version) + for _, value in kwargs.iteritems(): + self._check_none_or_string(value) + domain = self._get_domain(domain_name) + _type = domain.get_type(kind, name, version, ignore_empty=True) + if _type: + raise SWFTypeAlreadyExistsFault(_type) + _class = globals()["{}Type".format(kind.capitalize())] + _type = _class(name, version, **kwargs) + domain.add_type(_type) + + def deprecate_type(self, kind, domain_name, name, version): + self._check_string(domain_name) + self._check_string(name) + self._check_string(version) + domain = self._get_domain(domain_name) + _type = domain.get_type(kind, name, version) + if _type.status == "DEPRECATED": + raise SWFTypeDeprecatedFault(_type) + _type.status = "DEPRECATED" + + def describe_type(self, kind, domain_name, name, version): + self._check_string(domain_name) + self._check_string(name) + self._check_string(version) + domain = self._get_domain(domain_name) + return domain.get_type(kind, name, version) + + # TODO: find what triggers a "DefaultUndefinedFault" and implement it + # (didn't found in boto source code, nor in the docs, nor on a Google search) + # (will try to reach support) + def start_workflow_execution(self, domain_name, workflow_id, + workflow_name, workflow_version, + tag_list=None, **kwargs): + self._check_string(domain_name) + self._check_string(workflow_id) + self._check_string(workflow_name) + self._check_string(workflow_version) + self._check_none_or_list_of_strings(tag_list) + for _, value in kwargs.iteritems(): + self._check_none_or_string(value) + + domain = self._get_domain(domain_name) + wf_type = domain.get_type("workflow", workflow_name, workflow_version) + if wf_type.status == "DEPRECATED": + raise SWFTypeDeprecatedFault(wf_type) + wfe = WorkflowExecution(wf_type, workflow_id, + tag_list=tag_list, **kwargs) + domain.add_workflow_execution(wfe) + + return wfe + + def describe_workflow_execution(self, domain_name, run_id, workflow_id): + self._check_string(domain_name) + self._check_string(run_id) + self._check_string(workflow_id) + domain = self._get_domain(domain_name) + return domain.get_workflow_execution(run_id, workflow_id) + + +swf_backends = {} +for region in boto.swf.regions(): + swf_backends[region.name] = SWFBackend(region.name) diff --git a/moto/swf/models/activity_type.py b/moto/swf/models/activity_type.py new file mode 100644 index 000000000..95a83ca7a --- /dev/null +++ b/moto/swf/models/activity_type.py @@ -0,0 +1,16 @@ +from .generic_type import GenericType + + +class ActivityType(GenericType): + @property + def _configuration_keys(self): + return [ + "defaultTaskHeartbeatTimeout", + "defaultTaskScheduleToCloseTimeout", + "defaultTaskScheduleToStartTimeout", + "defaultTaskStartToCloseTimeout", + ] + + @property + def kind(self): + return "activity" diff --git a/moto/swf/models/domain.py b/moto/swf/models/domain.py new file mode 100644 index 000000000..4b30d3932 --- /dev/null +++ b/moto/swf/models/domain.py @@ -0,0 +1,85 @@ +from __future__ import unicode_literals +from collections import defaultdict + +from ..exceptions import ( + SWFUnknownResourceFault, + SWFWorkflowExecutionAlreadyStartedFault, +) + + +class Domain(object): + def __init__(self, name, retention, description=None): + self.name = name + self.retention = retention + self.description = description + self.status = "REGISTERED" + self.types = { + "activity": defaultdict(dict), + "workflow": defaultdict(dict), + } + # Workflow executions have an id, which unicity is guaranteed + # at domain level (not super clear in the docs, but I checked + # that against SWF API) ; hence the storage method as a dict + # of "workflow_id (client determined)" => WorkflowExecution() + # here. + self.workflow_executions = {} + + def __repr__(self): + return "Domain(name: %(name)s, status: %(status)s)" % self.__dict__ + + def to_short_dict(self): + hsh = { + "name": self.name, + "status": self.status, + } + if self.description: + hsh["description"] = self.description + return hsh + + def to_full_dict(self): + return { + "domainInfo": self.to_short_dict(), + "configuration": { + "workflowExecutionRetentionPeriodInDays": self.retention, + } + } + + def get_type(self, kind, name, version, ignore_empty=False): + try: + return self.types[kind][name][version] + except KeyError: + if not ignore_empty: + raise SWFUnknownResourceFault( + "type", + "{}Type=[name={}, version={}]".format( + kind.capitalize(), name, version + ) + ) + + def add_type(self, _type): + self.types[_type.kind][_type.name][_type.version] = _type + + def find_types(self, kind, status): + _all = [] + for _, family in self.types[kind].iteritems(): + for _, _type in family.iteritems(): + if _type.status == status: + _all.append(_type) + return _all + + def add_workflow_execution(self, workflow_execution): + _id = workflow_execution.workflow_id + if self.workflow_executions.get(_id): + raise SWFWorkflowExecutionAlreadyStartedFault() + self.workflow_executions[_id] = workflow_execution + + def get_workflow_execution(self, run_id, workflow_id): + wfe = self.workflow_executions.get(workflow_id) + if not wfe or wfe.run_id != run_id: + raise SWFUnknownResourceFault( + "execution", + "WorkflowExecution=[workflowId={}, runId={}]".format( + workflow_id, run_id + ) + ) + return wfe diff --git a/moto/swf/models/generic_type.py b/moto/swf/models/generic_type.py new file mode 100644 index 000000000..334382ecd --- /dev/null +++ b/moto/swf/models/generic_type.py @@ -0,0 +1,61 @@ +from __future__ import unicode_literals + +from moto.core.utils import camelcase_to_underscores + + +class GenericType(object): + def __init__(self, name, version, **kwargs): + self.name = name + self.version = version + self.status = "REGISTERED" + if "description" in kwargs: + self.description = kwargs.pop("description") + for key, value in kwargs.iteritems(): + self.__setattr__(key, value) + + def __repr__(self): + cls = self.__class__.__name__ + attrs = "name: %(name)s, version: %(version)s, status: %(status)s" % self.__dict__ + return "{}({})".format(cls, attrs) + + @property + def kind(self): + raise NotImplementedError() + + @property + def _configuration_keys(self): + raise NotImplementedError() + + def to_short_dict(self): + return { + "name": self.name, + "version": self.version, + } + + def to_medium_dict(self): + hsh = { + "{}Type".format(self.kind): self.to_short_dict(), + "creationDate": 1420066800, + "status": self.status, + } + if self.status == "DEPRECATED": + hsh["deprecationDate"] = 1422745200 + if hasattr(self, "description"): + hsh["description"] = self.description + return hsh + + def to_full_dict(self): + hsh = { + "typeInfo": self.to_medium_dict(), + "configuration": {} + } + if hasattr(self, "task_list"): + hsh["configuration"]["defaultTaskList"] = {"name": self.task_list} + for key in self._configuration_keys: + attr = camelcase_to_underscores(key) + if not hasattr(self, attr): + continue + if not getattr(self, attr): + continue + hsh["configuration"][key] = getattr(self, attr) + return hsh diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py new file mode 100644 index 000000000..88b3070ac --- /dev/null +++ b/moto/swf/models/workflow_execution.py @@ -0,0 +1,72 @@ +from __future__ import unicode_literals +import uuid + +from moto.core.utils import camelcase_to_underscores + + +class WorkflowExecution(object): + def __init__(self, workflow_type, workflow_id, **kwargs): + self.workflow_type = workflow_type + self.workflow_id = workflow_id + self.run_id = uuid.uuid4().hex + self.execution_status = "OPEN" + self.cancel_requested = False + #config + for key, value in kwargs.iteritems(): + self.__setattr__(key, value) + #counters + self.open_counts = { + "openTimers": 0, + "openDecisionTasks": 0, + "openActivityTasks": 0, + "openChildWorkflowExecutions": 0, + } + + def __repr__(self): + return "WorkflowExecution(run_id: {})".format(self.run_id) + + @property + def _configuration_keys(self): + return [ + "executionStartToCloseTimeout", + "childPolicy", + "taskPriority", + "taskStartToCloseTimeout", + ] + + def to_short_dict(self): + return { + "workflowId": self.workflow_id, + "runId": self.run_id + } + + def to_medium_dict(self): + hsh = { + "execution": self.to_short_dict(), + "workflowType": self.workflow_type.to_short_dict(), + "startTimestamp": 1420066800.123, + "executionStatus": self.execution_status, + "cancelRequested": self.cancel_requested, + } + if hasattr(self, "tag_list"): + hsh["tagList"] = self.tag_list + return hsh + + def to_full_dict(self): + hsh = { + "executionInfo": self.to_medium_dict(), + "executionConfiguration": {} + } + #configuration + if hasattr(self, "task_list"): + hsh["executionConfiguration"]["taskList"] = {"name": self.task_list} + for key in self._configuration_keys: + attr = camelcase_to_underscores(key) + if not hasattr(self, attr): + continue + if not getattr(self, attr): + continue + hsh["executionConfiguration"][key] = getattr(self, attr) + #counters + hsh["openCounts"] = self.open_counts + return hsh diff --git a/moto/swf/models/workflow_type.py b/moto/swf/models/workflow_type.py new file mode 100644 index 000000000..ddb2475b2 --- /dev/null +++ b/moto/swf/models/workflow_type.py @@ -0,0 +1,15 @@ +from .generic_type import GenericType + + +class WorkflowType(GenericType): + @property + def _configuration_keys(self): + return [ + "defaultChildPolicy", + "defaultExecutionStartToCloseTimeout", + "defaultTaskStartToCloseTimeout", + ] + + @property + def kind(self): + return "workflow" From 6a8636ad21fa8737e484dda7fb2896124d8e1c27 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Sun, 4 Oct 2015 08:48:42 +0200 Subject: [PATCH 24/94] Remove unused import in SWF test --- tests/test_swf/test_activity_types.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_swf/test_activity_types.py b/tests/test_swf/test_activity_types.py index 91ea12576..c166b725f 100644 --- a/tests/test_swf/test_activity_types.py +++ b/tests/test_swf/test_activity_types.py @@ -3,7 +3,6 @@ from nose.tools import assert_raises from sure import expect from moto import mock_swf -from moto.swf.models import ActivityType from moto.swf.exceptions import ( SWFUnknownResourceFault, SWFTypeAlreadyExistsFault, From 3ce5b293569ed63809d3e9e1b12505f2ee27f9b3 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Sun, 4 Oct 2015 11:09:18 +0200 Subject: [PATCH 25/94] Handle WorkflowExecution/WorkflowType options inheritance ... and potential resulting DefaultUndefinedFault errors. --- moto/swf/exceptions.py | 12 +++ moto/swf/models/workflow_execution.py | 26 +++++- moto/swf/responses.py | 2 +- tests/test_swf/test_models.py | 93 ++++++++++++++++++---- tests/test_swf/test_workflow_executions.py | 7 +- 5 files changed, 119 insertions(+), 21 deletions(-) diff --git a/moto/swf/exceptions.py b/moto/swf/exceptions.py index bd04002f4..cbab4e200 100644 --- a/moto/swf/exceptions.py +++ b/moto/swf/exceptions.py @@ -63,3 +63,15 @@ class SWFWorkflowExecutionAlreadyStartedFault(JSONResponseError): 400, "Bad Request", body={"__type": "com.amazonaws.swf.base.model#WorkflowExecutionAlreadyStartedFault"} ) + + +class SWFDefaultUndefinedFault(SWFClientError): + def __init__(self, key): + # TODO: move that into moto.core.utils maybe? + words = key.split("_") + key_camel_case = words.pop(0) + for word in words: + key_camel_case += word.capitalize() + super(SWFDefaultUndefinedFault, self).__init__( + key_camel_case, "com.amazonaws.swf.base.model#DefaultUndefinedFault" + ) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 88b3070ac..345809748 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -3,6 +3,8 @@ import uuid from moto.core.utils import camelcase_to_underscores +from ..exceptions import SWFDefaultUndefinedFault + class WorkflowExecution(object): def __init__(self, workflow_type, workflow_id, **kwargs): @@ -11,10 +13,16 @@ class WorkflowExecution(object): self.run_id = uuid.uuid4().hex self.execution_status = "OPEN" self.cancel_requested = False - #config - for key, value in kwargs.iteritems(): - self.__setattr__(key, value) - #counters + # args processing + # NB: the order follows boto/SWF order of exceptions appearance (if no + # param is set, # SWF will raise DefaultUndefinedFault errors in the + # same order as the few lines that follow) + self._set_from_kwargs_or_workflow_type(kwargs, "execution_start_to_close_timeout") + self._set_from_kwargs_or_workflow_type(kwargs, "task_list", "task_list") + self._set_from_kwargs_or_workflow_type(kwargs, "task_start_to_close_timeout") + self._set_from_kwargs_or_workflow_type(kwargs, "child_policy") + self.input = kwargs.get("input") + # counters self.open_counts = { "openTimers": 0, "openDecisionTasks": 0, @@ -25,6 +33,16 @@ class WorkflowExecution(object): def __repr__(self): return "WorkflowExecution(run_id: {})".format(self.run_id) + def _set_from_kwargs_or_workflow_type(self, kwargs, local_key, workflow_type_key=None): + if workflow_type_key is None: + workflow_type_key = "default_"+local_key + value = kwargs.get(local_key) + if not value and hasattr(self.workflow_type, workflow_type_key): + value = getattr(self.workflow_type, workflow_type_key) + if not value: + raise SWFDefaultUndefinedFault(local_key) + setattr(self, local_key, value) + @property def _configuration_keys(self): return [ diff --git a/moto/swf/responses.py b/moto/swf/responses.py index b4e349931..210a5be15 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -153,7 +153,7 @@ class SWFResponse(BaseResponse): task_list = None default_child_policy = self._params.get("defaultChildPolicy") default_task_start_to_close_timeout = self._params.get("defaultTaskStartToCloseTimeout") - default_execution_start_to_close_timeout = self._params.get("defaultTaskExecutionStartToCloseTimeout") + default_execution_start_to_close_timeout = self._params.get("defaultExecutionStartToCloseTimeout") description = self._params.get("description") # TODO: add defaultTaskPriority when boto gets to support it # TODO: add defaultLambdaRole when boto gets to support it diff --git a/tests/test_swf/test_models.py b/tests/test_swf/test_models.py index bf6437f5b..0d54cb67f 100644 --- a/tests/test_swf/test_models.py +++ b/tests/test_swf/test_models.py @@ -1,4 +1,5 @@ from sure import expect +from nose.tools import assert_raises from moto.swf.models import ( Domain, @@ -6,8 +7,20 @@ from moto.swf.models import ( WorkflowType, WorkflowExecution, ) +from moto.swf.exceptions import ( + SWFDefaultUndefinedFault, +) +# utils +def test_workflow_type(): + return WorkflowType( + "test-workflow", "v1.0", + task_list="queue", default_child_policy="ABANDON", + default_execution_start_to_close_timeout="300", + default_task_start_to_close_timeout="300", + ) + # Domain def test_domain_short_dict_representation(): domain = Domain("foo", "52") @@ -78,21 +91,62 @@ def test_type_string_representation(): # WorkflowExecution def test_workflow_execution_creation(): - wfe = WorkflowExecution("workflow_type_whatever", "ab1234", child_policy="TERMINATE") - wfe.workflow_type.should.equal("workflow_type_whatever") + wft = test_workflow_type() + wfe = WorkflowExecution(wft, "ab1234", child_policy="TERMINATE") + wfe.workflow_type.should.equal(wft) wfe.child_policy.should.equal("TERMINATE") +def test_workflow_execution_creation_child_policy_logic(): + WorkflowExecution( + WorkflowType( + "test-workflow", "v1.0", + task_list="queue", default_child_policy="ABANDON", + default_execution_start_to_close_timeout="300", + default_task_start_to_close_timeout="300", + ), + "ab1234" + ).child_policy.should.equal("ABANDON") + + WorkflowExecution( + WorkflowType( + "test-workflow", "v1.0", task_list="queue", + default_execution_start_to_close_timeout="300", + default_task_start_to_close_timeout="300", + ), + "ab1234", + child_policy="REQUEST_CANCEL" + ).child_policy.should.equal("REQUEST_CANCEL") + + with assert_raises(SWFDefaultUndefinedFault) as err: + WorkflowExecution(WorkflowType("test-workflow", "v1.0"), "ab1234") + + ex = err.exception + ex.status.should.equal(400) + ex.error_code.should.equal("DefaultUndefinedFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#DefaultUndefinedFault", + "message": "executionStartToCloseTimeout" + }) + + def test_workflow_execution_string_representation(): - wfe = WorkflowExecution("workflow_type_whatever", "ab1234", child_policy="TERMINATE") + wft = test_workflow_type() + wfe = WorkflowExecution(wft, "ab1234", child_policy="TERMINATE") str(wfe).should.match(r"^WorkflowExecution\(run_id: .*\)") def test_workflow_execution_generates_a_random_run_id(): - wfe1 = WorkflowExecution("workflow_type_whatever", "ab1234") - wfe2 = WorkflowExecution("workflow_type_whatever", "ab1235") + wft = test_workflow_type() + wfe1 = WorkflowExecution(wft, "ab1234", child_policy="TERMINATE") + wfe2 = WorkflowExecution(wft, "ab1235", child_policy="TERMINATE") wfe1.run_id.should_not.equal(wfe2.run_id) def test_workflow_execution_short_dict_representation(): - wf_type = WorkflowType("test-workflow", "v1.0") + wf_type = WorkflowType( + "test-workflow", "v1.0", + task_list="queue", default_child_policy="ABANDON", + default_execution_start_to_close_timeout="300", + default_task_start_to_close_timeout="300", + ) wfe = WorkflowExecution(wf_type, "ab1234") sd = wfe.to_short_dict() @@ -100,7 +154,12 @@ def test_workflow_execution_short_dict_representation(): sd.should.contain("runId") def test_workflow_execution_medium_dict_representation(): - wf_type = WorkflowType("test-workflow", "v1.0") + wf_type = WorkflowType( + "test-workflow", "v1.0", + task_list="queue", default_child_policy="ABANDON", + default_execution_start_to_close_timeout="300", + default_task_start_to_close_timeout="300", + ) wfe = WorkflowExecution(wf_type, "ab1234") md = wfe.to_medium_dict() @@ -116,7 +175,12 @@ def test_workflow_execution_medium_dict_representation(): md["tagList"].should.equal(["foo", "bar", "baz"]) def test_workflow_execution_full_dict_representation(): - wf_type = WorkflowType("test-workflow", "v1.0") + wf_type = WorkflowType( + "test-workflow", "v1.0", + task_list="queue", default_child_policy="ABANDON", + default_execution_start_to_close_timeout="300", + default_task_start_to_close_timeout="300", + ) wfe = WorkflowExecution(wf_type, "ab1234") fd = wfe.to_full_dict() @@ -124,10 +188,9 @@ def test_workflow_execution_full_dict_representation(): fd["openCounts"]["openTimers"].should.equal(0) fd["openCounts"]["openDecisionTasks"].should.equal(0) fd["openCounts"]["openActivityTasks"].should.equal(0) - fd["executionConfiguration"].should.equal({}) - - wfe.task_list = "special" - wfe.task_start_to_close_timeout = "45" - fd = wfe.to_full_dict() - fd["executionConfiguration"]["taskList"]["name"].should.equal("special") - fd["executionConfiguration"]["taskStartToCloseTimeout"].should.equal("45") + fd["executionConfiguration"].should.equal({ + "childPolicy": "ABANDON", + "executionStartToCloseTimeout": "300", + "taskList": {"name": "queue"}, + "taskStartToCloseTimeout": "300", + }) diff --git a/tests/test_swf/test_workflow_executions.py b/tests/test_swf/test_workflow_executions.py index 4082ab540..e73d37f23 100644 --- a/tests/test_swf/test_workflow_executions.py +++ b/tests/test_swf/test_workflow_executions.py @@ -15,7 +15,12 @@ from moto.swf.exceptions import ( def setup_swf_environment(): conn = boto.connect_swf("the_key", "the_secret") conn.register_domain("test-domain", "60", description="A test domain") - conn.register_workflow_type("test-domain", "test-workflow", "v1.0") + conn.register_workflow_type( + "test-domain", "test-workflow", "v1.0", + task_list="queue", default_child_policy="TERMINATE", + default_execution_start_to_close_timeout="300", + default_task_start_to_close_timeout="300", + ) conn.register_activity_type("test-domain", "test-activity", "v1.1") return conn From 464aef293ca84eb12749cd34da661e14ad774d18 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Sun, 4 Oct 2015 23:37:50 +0200 Subject: [PATCH 26/94] Add SWF endpoint GetWorkflowExecutionHistory and associated HistoryEvent model --- moto/swf/models/__init__.py | 2 + moto/swf/models/history_event.py | 59 ++++++++++++++++++++++ moto/swf/models/workflow_execution.py | 21 ++++++++ moto/swf/responses.py | 12 +++++ tests/test_swf/__init__.py | 0 tests/test_swf/test_models.py | 43 ++++++++++++---- tests/test_swf/test_workflow_executions.py | 23 +++++++++ tests/test_swf/utils.py | 24 +++++++++ 8 files changed, 173 insertions(+), 11 deletions(-) create mode 100644 moto/swf/models/history_event.py create mode 100644 tests/test_swf/__init__.py create mode 100644 tests/test_swf/utils.py diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index a7be6f64e..c47d704a2 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -15,6 +15,7 @@ from ..exceptions import ( from .activity_type import ActivityType from .domain import Domain from .generic_type import GenericType +from .history_event import HistoryEvent from .workflow_type import WorkflowType from .workflow_execution import WorkflowExecution @@ -150,6 +151,7 @@ class SWFBackend(BaseBackend): wfe = WorkflowExecution(wf_type, workflow_id, tag_list=tag_list, **kwargs) domain.add_workflow_execution(wfe) + wfe.start() return wfe diff --git a/moto/swf/models/history_event.py b/moto/swf/models/history_event.py new file mode 100644 index 000000000..c88f9fbe9 --- /dev/null +++ b/moto/swf/models/history_event.py @@ -0,0 +1,59 @@ +from __future__ import unicode_literals +from datetime import datetime +from time import mktime + + +class HistoryEvent(object): + def __init__(self, event_id, event_type, **kwargs): + self.event_id = event_id + self.event_type = event_type + self.event_timestamp = float(mktime(datetime.now().timetuple())) + for key, value in kwargs.iteritems(): + self.__setattr__(key, value) + # break soon if attributes are not valid + self.event_attributes() + + def to_dict(self): + return { + "eventId": self.event_id, + "eventType": self.event_type, + "eventTimestamp": self.event_timestamp, + self._attributes_key(): self.event_attributes() + } + + def _attributes_key(self): + key = "{}EventAttributes".format(self.event_type) + key = key[0].lower() + key[1:] + return key + + def event_attributes(self): + if self.event_type == "WorkflowExecutionStarted": + wfe = self.workflow_execution + hsh = { + "childPolicy": wfe.child_policy, + "executionStartToCloseTimeout": wfe.execution_start_to_close_timeout, + "parentInitiatedEventId": 0, + "taskList": { + "name": wfe.task_list + }, + "taskStartToCloseTimeout": wfe.task_start_to_close_timeout, + "workflowType": { + "name": wfe.workflow_type.name, + "version": wfe.workflow_type.version + } + } + return hsh + elif self.event_type == "DecisionTaskScheduled": + wfe = self.workflow_execution + return { + "startToCloseTimeout": wfe.task_start_to_close_timeout, + "taskList": {"name": wfe.task_list} + } + elif self.event_type == "DecisionTaskStarted": + return { + "scheduledEventId": self.scheduled_event_id + } + else: + raise NotImplementedError( + "HistoryEvent does not implement attributes for type '{}'".format(self.event_type) + ) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 345809748..fa6d28dd0 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -4,6 +4,7 @@ import uuid from moto.core.utils import camelcase_to_underscores from ..exceptions import SWFDefaultUndefinedFault +from .history_event import HistoryEvent class WorkflowExecution(object): @@ -29,6 +30,8 @@ class WorkflowExecution(object): "openActivityTasks": 0, "openChildWorkflowExecutions": 0, } + # events + self.events = [] def __repr__(self): return "WorkflowExecution(run_id: {})".format(self.run_id) @@ -88,3 +91,21 @@ class WorkflowExecution(object): #counters hsh["openCounts"] = self.open_counts return hsh + + def next_event_id(self): + event_ids = [evt.event_id for evt in self.events] + return max(event_ids or [0]) + + def _add_event(self, *args, **kwargs): + evt = HistoryEvent(self.next_event_id(), *args, **kwargs) + self.events.append(evt) + + def start(self): + self._add_event( + "WorkflowExecutionStarted", + workflow_execution=self, + ) + self._add_event( + "DecisionTaskScheduled", + workflow_execution=self, + ) diff --git a/moto/swf/responses.py b/moto/swf/responses.py index 210a5be15..8f7aa0344 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -209,3 +209,15 @@ class SWFResponse(BaseResponse): wfe = self.swf_backend.describe_workflow_execution(domain_name, run_id, workflow_id) return json.dumps(wfe.to_full_dict()) + + def get_workflow_execution_history(self): + domain_name = self._params["domain"] + _workflow_execution = self._params["execution"] + run_id = _workflow_execution["runId"] + workflow_id = _workflow_execution["workflowId"] + # TODO: implement reverseOrder + + wfe = self.swf_backend.describe_workflow_execution(domain_name, run_id, workflow_id) + return json.dumps({ + "events": [evt.to_dict() for evt in wfe.events] + }) diff --git a/tests/test_swf/__init__.py b/tests/test_swf/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_swf/test_models.py b/tests/test_swf/test_models.py index 0d54cb67f..a8b76330f 100644 --- a/tests/test_swf/test_models.py +++ b/tests/test_swf/test_models.py @@ -1,9 +1,11 @@ from sure import expect from nose.tools import assert_raises +from freezegun import freeze_time from moto.swf.models import ( Domain, GenericType, + HistoryEvent, WorkflowType, WorkflowExecution, ) @@ -11,15 +13,8 @@ from moto.swf.exceptions import ( SWFDefaultUndefinedFault, ) +from .utils import get_basic_workflow_type -# utils -def test_workflow_type(): - return WorkflowType( - "test-workflow", "v1.0", - task_list="queue", default_child_policy="ABANDON", - default_execution_start_to_close_timeout="300", - default_task_start_to_close_timeout="300", - ) # Domain def test_domain_short_dict_representation(): @@ -91,7 +86,7 @@ def test_type_string_representation(): # WorkflowExecution def test_workflow_execution_creation(): - wft = test_workflow_type() + wft = get_basic_workflow_type() wfe = WorkflowExecution(wft, "ab1234", child_policy="TERMINATE") wfe.workflow_type.should.equal(wft) wfe.child_policy.should.equal("TERMINATE") @@ -130,12 +125,12 @@ def test_workflow_execution_creation_child_policy_logic(): def test_workflow_execution_string_representation(): - wft = test_workflow_type() + wft = get_basic_workflow_type() wfe = WorkflowExecution(wft, "ab1234", child_policy="TERMINATE") str(wfe).should.match(r"^WorkflowExecution\(run_id: .*\)") def test_workflow_execution_generates_a_random_run_id(): - wft = test_workflow_type() + wft = get_basic_workflow_type() wfe1 = WorkflowExecution(wft, "ab1234", child_policy="TERMINATE") wfe2 = WorkflowExecution(wft, "ab1235", child_policy="TERMINATE") wfe1.run_id.should_not.equal(wfe2.run_id) @@ -194,3 +189,29 @@ def test_workflow_execution_full_dict_representation(): "taskList": {"name": "queue"}, "taskStartToCloseTimeout": "300", }) + + +# HistoryEvent +@freeze_time("2015-01-01 12:00:00") +def test_history_event_creation(): + he = HistoryEvent(123, "DecisionTaskStarted", scheduled_event_id=2) + he.event_id.should.equal(123) + he.event_type.should.equal("DecisionTaskStarted") + he.event_timestamp.should.equal(1420110000.0) + +@freeze_time("2015-01-01 12:00:00") +def test_history_event_to_dict_representation(): + he = HistoryEvent(123, "DecisionTaskStarted", scheduled_event_id=2) + he.to_dict().should.equal({ + "eventId": 123, + "eventType": "DecisionTaskStarted", + "eventTimestamp": 1420110000.0, + "decisionTaskStartedEventAttributes": { + "scheduledEventId": 2 + } + }) + +def test_history_event_breaks_on_initialization_if_not_implemented(): + HistoryEvent.when.called_with( + 123, "UnknownHistoryEvent" + ).should.throw(NotImplementedError) diff --git a/tests/test_swf/test_workflow_executions.py b/tests/test_swf/test_workflow_executions.py index e73d37f23..27feb8b4a 100644 --- a/tests/test_swf/test_workflow_executions.py +++ b/tests/test_swf/test_workflow_executions.py @@ -90,3 +90,26 @@ def test_describe_non_existent_workflow_execution(): "__type": "com.amazonaws.swf.base.model#UnknownResourceFault", "message": "Unknown execution: WorkflowExecution=[workflowId=wrong-workflow-id, runId=wrong-run-id]" }) + + +# GetWorkflowExecutionHistory endpoint +@mock_swf +def test_get_workflow_execution_history(): + conn = setup_swf_environment() + hsh = conn.start_workflow_execution("test-domain", "uid-abcd1234", "test-workflow", "v1.0") + run_id = hsh["runId"] + + resp = conn.get_workflow_execution_history("test-domain", run_id, "uid-abcd1234") + resp["events"].should.be.a("list") + evt = resp["events"][0] + evt["eventType"].should.equal("WorkflowExecutionStarted") + + +@mock_swf +def test_get_workflow_execution_history_on_non_existent_workflow_execution(): + conn = setup_swf_environment() + + with assert_raises(SWFUnknownResourceFault) as err: + conn.get_workflow_execution_history("test-domain", "wrong-run-id", "wrong-workflow-id") + + # (the rest is already tested above) diff --git a/tests/test_swf/utils.py b/tests/test_swf/utils.py new file mode 100644 index 000000000..f106e2e02 --- /dev/null +++ b/tests/test_swf/utils.py @@ -0,0 +1,24 @@ +from moto.swf.models import ( + WorkflowType, +) + + +# A generic test WorkflowType +def _generic_workflow_type_attributes(): + return [ + "test-workflow", "v1.0" + ], { + "task_list": "queue", + "default_child_policy": "ABANDON", + "default_execution_start_to_close_timeout": "300", + "default_task_start_to_close_timeout": "300", + } + +def get_basic_workflow_type(): + args, kwargs = _generic_workflow_type_attributes() + return WorkflowType(*args, **kwargs) + +def mock_basic_workflow_type(domain_name, conn): + args, kwargs = _generic_workflow_type_attributes() + conn.register_workflow_type(domain_name, *args, **kwargs) + return conn From 8d435d8afe4a8f7b994421d027de29be8b77de08 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 5 Oct 2015 10:05:35 +0200 Subject: [PATCH 27/94] Refactor SWF exceptions testing so responses tests get simpler --- tests/test_swf/test_activity_types.py | 64 +++---------- tests/test_swf/test_domains.py | 62 +++--------- tests/test_swf/test_exceptions.py | 105 +++++++++++++++++++++ tests/test_swf/test_models.py | 14 +-- tests/test_swf/test_workflow_executions.py | 46 +++------ tests/test_swf/test_workflow_types.py | 62 +++--------- 6 files changed, 165 insertions(+), 188 deletions(-) create mode 100644 tests/test_swf/test_exceptions.py diff --git a/tests/test_swf/test_activity_types.py b/tests/test_swf/test_activity_types.py index c166b725f..e8612ef0f 100644 --- a/tests/test_swf/test_activity_types.py +++ b/tests/test_swf/test_activity_types.py @@ -1,5 +1,4 @@ import boto -from nose.tools import assert_raises from sure import expect from moto import mock_swf @@ -29,30 +28,18 @@ def test_register_already_existing_activity_type(): conn.register_domain("test-domain", "60") conn.register_activity_type("test-domain", "test-activity", "v1.0") - with assert_raises(SWFTypeAlreadyExistsFault) as err: - conn.register_activity_type("test-domain", "test-activity", "v1.0") - - ex = err.exception - ex.status.should.equal(400) - ex.error_code.should.equal("TypeAlreadyExistsFault") - ex.body.should.equal({ - "__type": "com.amazonaws.swf.base.model#TypeAlreadyExistsFault", - "message": "ActivityType=[name=test-activity, version=v1.0]" - }) + conn.register_activity_type.when.called_with( + "test-domain", "test-activity", "v1.0" + ).should.throw(SWFTypeAlreadyExistsFault) @mock_swf def test_register_with_wrong_parameter_type(): conn = boto.connect_swf("the_key", "the_secret") conn.register_domain("test-domain", "60") - with assert_raises(SWFSerializationException) as err: - conn.register_activity_type("test-domain", "test-activity", 12) - - ex = err.exception - ex.status.should.equal(400) - ex.error_code.should.equal("SerializationException") - ex.body["__type"].should.equal("com.amazonaws.swf.base.model#SerializationException") - + conn.register_activity_type.when.called_with( + "test-domain", "test-activity", 12 + ).should.throw(SWFSerializationException) # ListActivityTypes endpoint @mock_swf @@ -101,32 +88,18 @@ def test_deprecate_already_deprecated_activity_type(): conn.register_activity_type("test-domain", "test-activity", "v1.0") conn.deprecate_activity_type("test-domain", "test-activity", "v1.0") - with assert_raises(SWFTypeDeprecatedFault) as err: - conn.deprecate_activity_type("test-domain", "test-activity", "v1.0") - - ex = err.exception - ex.status.should.equal(400) - ex.error_code.should.equal("TypeDeprecatedFault") - ex.body.should.equal({ - "__type": "com.amazonaws.swf.base.model#TypeDeprecatedFault", - "message": "ActivityType=[name=test-activity, version=v1.0]" - }) + conn.deprecate_activity_type.when.called_with( + "test-domain", "test-activity", "v1.0" + ).should.throw(SWFTypeDeprecatedFault) @mock_swf def test_deprecate_non_existent_activity_type(): conn = boto.connect_swf("the_key", "the_secret") conn.register_domain("test-domain", "60") - with assert_raises(SWFUnknownResourceFault) as err: - conn.deprecate_activity_type("test-domain", "non-existent", "v1.0") - - ex = err.exception - ex.status.should.equal(400) - ex.error_code.should.equal("UnknownResourceFault") - ex.body.should.equal({ - "__type": "com.amazonaws.swf.base.model#UnknownResourceFault", - "message": "Unknown type: ActivityType=[name=non-existent, version=v1.0]" - }) + conn.deprecate_activity_type.when.called_with( + "test-domain", "non-existent", "v1.0" + ).should.throw(SWFUnknownResourceFault) # DescribeActivityType endpoint @mock_swf @@ -148,13 +121,6 @@ def test_describe_non_existent_activity_type(): conn = boto.connect_swf("the_key", "the_secret") conn.register_domain("test-domain", "60") - with assert_raises(SWFUnknownResourceFault) as err: - conn.describe_activity_type("test-domain", "non-existent", "v1.0") - - ex = err.exception - ex.status.should.equal(400) - ex.error_code.should.equal("UnknownResourceFault") - ex.body.should.equal({ - "__type": "com.amazonaws.swf.base.model#UnknownResourceFault", - "message": "Unknown type: ActivityType=[name=non-existent, version=v1.0]" - }) + conn.describe_activity_type.when.called_with( + "test-domain", "non-existent", "v1.0" + ).should.throw(SWFUnknownResourceFault) diff --git a/tests/test_swf/test_domains.py b/tests/test_swf/test_domains.py index 013eb1b63..f43200aaf 100644 --- a/tests/test_swf/test_domains.py +++ b/tests/test_swf/test_domains.py @@ -1,5 +1,4 @@ import boto -from nose.tools import assert_raises from sure import expect from moto import mock_swf @@ -29,28 +28,17 @@ def test_register_already_existing_domain(): conn = boto.connect_swf("the_key", "the_secret") conn.register_domain("test-domain", "60", description="A test domain") - with assert_raises(SWFDomainAlreadyExistsFault) as err: - conn.register_domain("test-domain", "60", description="A test domain") - - ex = err.exception - ex.status.should.equal(400) - ex.error_code.should.equal("DomainAlreadyExistsFault") - ex.body.should.equal({ - "__type": "com.amazonaws.swf.base.model#DomainAlreadyExistsFault", - "message": "test-domain" - }) + conn.register_domain.when.called_with( + "test-domain", "60", description="A test domain" + ).should.throw(SWFDomainAlreadyExistsFault) @mock_swf def test_register_with_wrong_parameter_type(): conn = boto.connect_swf("the_key", "the_secret") - with assert_raises(SWFSerializationException) as err: - conn.register_domain("test-domain", 60, description="A test domain") - - ex = err.exception - ex.status.should.equal(400) - ex.error_code.should.equal("SerializationException") - ex.body["__type"].should.equal("com.amazonaws.swf.base.model#SerializationException") + conn.register_domain.when.called_with( + "test-domain", 60, description="A test domain" + ).should.throw(SWFSerializationException) # ListDomains endpoint @@ -95,31 +83,18 @@ def test_deprecate_already_deprecated_domain(): conn.register_domain("test-domain", "60", description="A test domain") conn.deprecate_domain("test-domain") - with assert_raises(SWFDomainDeprecatedFault) as err: - conn.deprecate_domain("test-domain") - - ex = err.exception - ex.status.should.equal(400) - ex.error_code.should.equal("DomainDeprecatedFault") - ex.body.should.equal({ - "__type": "com.amazonaws.swf.base.model#DomainDeprecatedFault", - "message": "test-domain" - }) + conn.deprecate_domain.when.called_with( + "test-domain" + ).should.throw(SWFDomainDeprecatedFault) @mock_swf def test_deprecate_non_existent_domain(): conn = boto.connect_swf("the_key", "the_secret") - with assert_raises(SWFUnknownResourceFault) as err: - conn.deprecate_domain("non-existent") + conn.deprecate_domain.when.called_with( + "non-existent" + ).should.throw(SWFUnknownResourceFault) - ex = err.exception - ex.status.should.equal(400) - ex.error_code.should.equal("UnknownResourceFault") - ex.body.should.equal({ - "__type": "com.amazonaws.swf.base.model#UnknownResourceFault", - "message": "Unknown domain: non-existent" - }) # DescribeDomain endpoint @mock_swf @@ -137,13 +112,6 @@ def test_describe_domain(): def test_describe_non_existent_domain(): conn = boto.connect_swf("the_key", "the_secret") - with assert_raises(SWFUnknownResourceFault) as err: - conn.describe_domain("non-existent") - - ex = err.exception - ex.status.should.equal(400) - ex.error_code.should.equal("UnknownResourceFault") - ex.body.should.equal({ - "__type": "com.amazonaws.swf.base.model#UnknownResourceFault", - "message": "Unknown domain: non-existent" - }) + conn.describe_domain.when.called_with( + "non-existent" + ).should.throw(SWFUnknownResourceFault) diff --git a/tests/test_swf/test_exceptions.py b/tests/test_swf/test_exceptions.py new file mode 100644 index 000000000..27d48c261 --- /dev/null +++ b/tests/test_swf/test_exceptions.py @@ -0,0 +1,105 @@ +from __future__ import unicode_literals + +from moto.swf.exceptions import ( + SWFClientError, + SWFUnknownResourceFault, + SWFDomainAlreadyExistsFault, + SWFDomainDeprecatedFault, + SWFSerializationException, + SWFTypeAlreadyExistsFault, + SWFTypeDeprecatedFault, + SWFWorkflowExecutionAlreadyStartedFault, + SWFDefaultUndefinedFault, +) +from moto.swf.models import ( + WorkflowType, +) + +def test_swf_client_error(): + ex = SWFClientError("error message", "ASpecificType") + + ex.status.should.equal(400) + ex.error_code.should.equal("ASpecificType") + ex.body.should.equal({ + "__type": "ASpecificType", + "message": "error message" + }) + +def test_swf_unknown_resource_fault(): + ex = SWFUnknownResourceFault("type", "detail") + + ex.status.should.equal(400) + ex.error_code.should.equal("UnknownResourceFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#UnknownResourceFault", + "message": "Unknown type: detail" + }) + +def test_swf_domain_already_exists_fault(): + ex = SWFDomainAlreadyExistsFault("domain-name") + + ex.status.should.equal(400) + ex.error_code.should.equal("DomainAlreadyExistsFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#DomainAlreadyExistsFault", + "message": "domain-name" + }) + +def test_swf_domain_deprecated_fault(): + ex = SWFDomainDeprecatedFault("domain-name") + + ex.status.should.equal(400) + ex.error_code.should.equal("DomainDeprecatedFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#DomainDeprecatedFault", + "message": "domain-name" + }) + +def test_swf_serialization_exception(): + ex = SWFSerializationException("value") + + ex.status.should.equal(400) + ex.error_code.should.equal("SerializationException") + ex.body["__type"].should.equal("com.amazonaws.swf.base.model#SerializationException") + ex.body["Message"].should.match(r"class java.lang.Foo can not be converted to an String") + +def test_swf_type_already_exists_fault(): + wft = WorkflowType("wf-name", "wf-version") + ex = SWFTypeAlreadyExistsFault(wft) + + ex.status.should.equal(400) + ex.error_code.should.equal("TypeAlreadyExistsFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#TypeAlreadyExistsFault", + "message": "WorkflowType=[name=wf-name, version=wf-version]" + }) + +def test_swf_type_deprecated_fault(): + wft = WorkflowType("wf-name", "wf-version") + ex = SWFTypeDeprecatedFault(wft) + + ex.status.should.equal(400) + ex.error_code.should.equal("TypeDeprecatedFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#TypeDeprecatedFault", + "message": "WorkflowType=[name=wf-name, version=wf-version]" + }) + +def test_swf_workflow_execution_already_started_fault(): + ex = SWFWorkflowExecutionAlreadyStartedFault() + + ex.status.should.equal(400) + ex.error_code.should.equal("WorkflowExecutionAlreadyStartedFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#WorkflowExecutionAlreadyStartedFault", + }) + +def test_swf_default_undefined_fault(): + ex = SWFDefaultUndefinedFault("execution_start_to_close_timeout") + + ex.status.should.equal(400) + ex.error_code.should.equal("DefaultUndefinedFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#DefaultUndefinedFault", + "message": "executionStartToCloseTimeout", + }) diff --git a/tests/test_swf/test_models.py b/tests/test_swf/test_models.py index a8b76330f..813afc3ca 100644 --- a/tests/test_swf/test_models.py +++ b/tests/test_swf/test_models.py @@ -1,5 +1,4 @@ from sure import expect -from nose.tools import assert_raises from freezegun import freeze_time from moto.swf.models import ( @@ -112,16 +111,9 @@ def test_workflow_execution_creation_child_policy_logic(): child_policy="REQUEST_CANCEL" ).child_policy.should.equal("REQUEST_CANCEL") - with assert_raises(SWFDefaultUndefinedFault) as err: - WorkflowExecution(WorkflowType("test-workflow", "v1.0"), "ab1234") - - ex = err.exception - ex.status.should.equal(400) - ex.error_code.should.equal("DefaultUndefinedFault") - ex.body.should.equal({ - "__type": "com.amazonaws.swf.base.model#DefaultUndefinedFault", - "message": "executionStartToCloseTimeout" - }) + WorkflowExecution.when.called_with( + WorkflowType("test-workflow", "v1.0"), "ab1234" + ).should.throw(SWFDefaultUndefinedFault) def test_workflow_execution_string_representation(): diff --git a/tests/test_swf/test_workflow_executions.py b/tests/test_swf/test_workflow_executions.py index 27feb8b4a..07aea90fb 100644 --- a/tests/test_swf/test_workflow_executions.py +++ b/tests/test_swf/test_workflow_executions.py @@ -1,5 +1,4 @@ import boto -from nose.tools import assert_raises from sure import expect from moto import mock_swf @@ -38,31 +37,18 @@ def test_start_already_started_workflow_execution(): conn = setup_swf_environment() conn.start_workflow_execution("test-domain", "uid-abcd1234", "test-workflow", "v1.0") - with assert_raises(SWFWorkflowExecutionAlreadyStartedFault) as err: - conn.start_workflow_execution("test-domain", "uid-abcd1234", "test-workflow", "v1.0") - - ex = err.exception - ex.status.should.equal(400) - ex.error_code.should.equal("WorkflowExecutionAlreadyStartedFault") - ex.body.should.equal({ - "__type": "com.amazonaws.swf.base.model#WorkflowExecutionAlreadyStartedFault", - }) + conn.start_workflow_execution.when.called_with( + "test-domain", "uid-abcd1234", "test-workflow", "v1.0" + ).should.throw(SWFWorkflowExecutionAlreadyStartedFault) @mock_swf def test_start_workflow_execution_on_deprecated_type(): conn = setup_swf_environment() conn.deprecate_workflow_type("test-domain", "test-workflow", "v1.0") - with assert_raises(SWFTypeDeprecatedFault) as err: - conn.start_workflow_execution("test-domain", "uid-abcd1234", "test-workflow", "v1.0") - - ex = err.exception - ex.status.should.equal(400) - ex.error_code.should.equal("TypeDeprecatedFault") - ex.body.should.equal({ - "__type": "com.amazonaws.swf.base.model#TypeDeprecatedFault", - "message": "WorkflowType=[name=test-workflow, version=v1.0]" - }) + conn.start_workflow_execution.when.called_with( + "test-domain", "uid-abcd1234", "test-workflow", "v1.0" + ).should.throw(SWFTypeDeprecatedFault) # DescribeWorkflowExecution endpoint @@ -80,16 +66,9 @@ def test_describe_workflow_execution(): def test_describe_non_existent_workflow_execution(): conn = setup_swf_environment() - with assert_raises(SWFUnknownResourceFault) as err: - conn.describe_workflow_execution("test-domain", "wrong-run-id", "wrong-workflow-id") - - ex = err.exception - ex.status.should.equal(400) - ex.error_code.should.equal("UnknownResourceFault") - ex.body.should.equal({ - "__type": "com.amazonaws.swf.base.model#UnknownResourceFault", - "message": "Unknown execution: WorkflowExecution=[workflowId=wrong-workflow-id, runId=wrong-run-id]" - }) + conn.describe_workflow_execution.when.called_with( + "test-domain", "wrong-run-id", "wrong-workflow-id" + ).should.throw(SWFUnknownResourceFault) # GetWorkflowExecutionHistory endpoint @@ -109,7 +88,6 @@ def test_get_workflow_execution_history(): def test_get_workflow_execution_history_on_non_existent_workflow_execution(): conn = setup_swf_environment() - with assert_raises(SWFUnknownResourceFault) as err: - conn.get_workflow_execution_history("test-domain", "wrong-run-id", "wrong-workflow-id") - - # (the rest is already tested above) + conn.get_workflow_execution_history.when.called_with( + "test-domain", "wrong-run-id", "wrong-workflow-id" + ).should.throw(SWFUnknownResourceFault) diff --git a/tests/test_swf/test_workflow_types.py b/tests/test_swf/test_workflow_types.py index 67424710f..adcd81cc6 100644 --- a/tests/test_swf/test_workflow_types.py +++ b/tests/test_swf/test_workflow_types.py @@ -1,5 +1,4 @@ import boto -from nose.tools import assert_raises from sure import expect from moto import mock_swf @@ -29,29 +28,18 @@ def test_register_already_existing_workflow_type(): conn.register_domain("test-domain", "60") conn.register_workflow_type("test-domain", "test-workflow", "v1.0") - with assert_raises(SWFTypeAlreadyExistsFault) as err: - conn.register_workflow_type("test-domain", "test-workflow", "v1.0") - - ex = err.exception - ex.status.should.equal(400) - ex.error_code.should.equal("TypeAlreadyExistsFault") - ex.body.should.equal({ - "__type": "com.amazonaws.swf.base.model#TypeAlreadyExistsFault", - "message": "WorkflowType=[name=test-workflow, version=v1.0]" - }) + conn.register_workflow_type.when.called_with( + "test-domain", "test-workflow", "v1.0" + ).should.throw(SWFTypeAlreadyExistsFault) @mock_swf def test_register_with_wrong_parameter_type(): conn = boto.connect_swf("the_key", "the_secret") conn.register_domain("test-domain", "60") - with assert_raises(SWFSerializationException) as err: - conn.register_workflow_type("test-domain", "test-workflow", 12) - - ex = err.exception - ex.status.should.equal(400) - ex.error_code.should.equal("SerializationException") - ex.body["__type"].should.equal("com.amazonaws.swf.base.model#SerializationException") + conn.register_workflow_type.when.called_with( + "test-domain", "test-workflow", 12 + ).should.throw(SWFSerializationException) # ListWorkflowTypes endpoint @@ -101,32 +89,19 @@ def test_deprecate_already_deprecated_workflow_type(): conn.register_workflow_type("test-domain", "test-workflow", "v1.0") conn.deprecate_workflow_type("test-domain", "test-workflow", "v1.0") - with assert_raises(SWFTypeDeprecatedFault) as err: - conn.deprecate_workflow_type("test-domain", "test-workflow", "v1.0") - - ex = err.exception - ex.status.should.equal(400) - ex.error_code.should.equal("TypeDeprecatedFault") - ex.body.should.equal({ - "__type": "com.amazonaws.swf.base.model#TypeDeprecatedFault", - "message": "WorkflowType=[name=test-workflow, version=v1.0]" - }) + conn.deprecate_workflow_type.when.called_with( + "test-domain", "test-workflow", "v1.0" + ).should.throw(SWFTypeDeprecatedFault) @mock_swf def test_deprecate_non_existent_workflow_type(): conn = boto.connect_swf("the_key", "the_secret") conn.register_domain("test-domain", "60") - with assert_raises(SWFUnknownResourceFault) as err: - conn.deprecate_workflow_type("test-domain", "non-existent", "v1.0") + conn.deprecate_workflow_type.when.called_with( + "test-domain", "non-existent", "v1.0" + ).should.throw(SWFUnknownResourceFault) - ex = err.exception - ex.status.should.equal(400) - ex.error_code.should.equal("UnknownResourceFault") - ex.body.should.equal({ - "__type": "com.amazonaws.swf.base.model#UnknownResourceFault", - "message": "Unknown type: WorkflowType=[name=non-existent, version=v1.0]" - }) # DescribeWorkflowType endpoint @mock_swf @@ -150,13 +125,6 @@ def test_describe_non_existent_workflow_type(): conn = boto.connect_swf("the_key", "the_secret") conn.register_domain("test-domain", "60") - with assert_raises(SWFUnknownResourceFault) as err: - conn.describe_workflow_type("test-domain", "non-existent", "v1.0") - - ex = err.exception - ex.status.should.equal(400) - ex.error_code.should.equal("UnknownResourceFault") - ex.body.should.equal({ - "__type": "com.amazonaws.swf.base.model#UnknownResourceFault", - "message": "Unknown type: WorkflowType=[name=non-existent, version=v1.0]" - }) + conn.describe_workflow_type.when.called_with( + "test-domain", "non-existent", "v1.0" + ).should.throw(SWFUnknownResourceFault) From c16da9da2de1037f4ccbcaf0ba47b4504be8cdef Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Sun, 11 Oct 2015 14:02:55 +0200 Subject: [PATCH 28/94] Add SWF endpoint PollForDecisionTask and associated DecisionTask model --- moto/swf/models/__init__.py | 31 +++++++++++++++++ moto/swf/models/decision_task.py | 35 +++++++++++++++++++ moto/swf/models/history_event.py | 5 ++- moto/swf/models/workflow_execution.py | 38 ++++++++++++++++++++- moto/swf/responses.py | 13 ++++++++ tests/test_swf/test_decision_tasks.py | 46 +++++++++++++++++++++++++ tests/test_swf/test_models.py | 48 ++++++++++++++++++++++++++- 7 files changed, 213 insertions(+), 3 deletions(-) create mode 100644 moto/swf/models/decision_task.py create mode 100644 tests/test_swf/test_decision_tasks.py diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index c47d704a2..b1c00738d 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -13,6 +13,7 @@ from ..exceptions import ( SWFTypeDeprecatedFault, ) from .activity_type import ActivityType +from .decision_task import DecisionTask from .domain import Domain from .generic_type import GenericType from .history_event import HistoryEvent @@ -162,6 +163,36 @@ class SWFBackend(BaseBackend): domain = self._get_domain(domain_name) return domain.get_workflow_execution(run_id, workflow_id) + def poll_for_decision_task(self, domain_name, task_list, identity=None): + self._check_string(domain_name) + self._check_string(task_list) + domain = self._get_domain(domain_name) + # Real SWF cases: + # - case 1: there's a decision task to return, return it + # - case 2: there's no decision task to return, so wait for timeout + # and if a new decision is schedule, start and return it + # - case 3: timeout reached, no decision, return an empty decision + # (e.g. a decision with an empty "taskToken") + # + # For the sake of simplicity, we forget case 2 for now, so either + # there's a DecisionTask to return, either we return a blank one. + # + # SWF client libraries should cope with that easily as long as tests + # aren't distributed. + # + # TODO: handle long polling (case 2) for decision tasks + decision_candidates = [] + for wf_id, wf_execution in domain.workflow_executions.iteritems(): + decision_candidates += wf_execution.scheduled_decision_tasks + if any(decision_candidates): + # TODO: handle task priorities (but not supported by boto for now) + decision = min(decision_candidates, key=lambda d: d.scheduled_at) + wfe = decision.workflow_execution + wfe.start_decision_task(decision.task_token, identity=identity) + return decision + else: + return None + swf_backends = {} for region in boto.swf.regions(): diff --git a/moto/swf/models/decision_task.py b/moto/swf/models/decision_task.py new file mode 100644 index 000000000..60d0f7131 --- /dev/null +++ b/moto/swf/models/decision_task.py @@ -0,0 +1,35 @@ +from __future__ import unicode_literals +from datetime import datetime +import uuid + + +class DecisionTask(object): + def __init__(self, workflow_execution, scheduled_event_id): + self.workflow_execution = workflow_execution + self.workflow_type = workflow_execution.workflow_type + self.task_token = str(uuid.uuid4()) + self.scheduled_event_id = scheduled_event_id + self.previous_started_event_id = 0 + self.started_event_id = None + self.state = "SCHEDULED" + # this is *not* necessarily coherent with workflow execution history, + # but that shouldn't be a problem for tests + self.scheduled_at = datetime.now() + + def to_full_dict(self): + hsh = { + "events": [ + evt.to_dict() for evt in self.workflow_execution.events + ], + "taskToken": self.task_token, + "previousStartedEventId": self.previous_started_event_id, + "workflowExecution": self.workflow_execution.to_short_dict(), + "workflowType": self.workflow_type.to_short_dict(), + } + if self.started_event_id: + hsh["startedEventId"] = self.started_event_id + return hsh + + def start(self, started_event_id): + self.state = "STARTED" + self.started_event_id = started_event_id diff --git a/moto/swf/models/history_event.py b/moto/swf/models/history_event.py index c88f9fbe9..93d0cfe41 100644 --- a/moto/swf/models/history_event.py +++ b/moto/swf/models/history_event.py @@ -50,9 +50,12 @@ class HistoryEvent(object): "taskList": {"name": wfe.task_list} } elif self.event_type == "DecisionTaskStarted": - return { + hsh = { "scheduledEventId": self.scheduled_event_id } + if hasattr(self, "identity") and self.identity: + hsh["identity"] = self.identity + return hsh else: raise NotImplementedError( "HistoryEvent does not implement attributes for type '{}'".format(self.event_type) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index fa6d28dd0..110001641 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -4,6 +4,7 @@ import uuid from moto.core.utils import camelcase_to_underscores from ..exceptions import SWFDefaultUndefinedFault +from .decision_task import DecisionTask from .history_event import HistoryEvent @@ -32,6 +33,10 @@ class WorkflowExecution(object): } # events self.events = [] + # tasks + self.decision_tasks = [] + self.activity_tasks = [] + self.child_workflow_executions = [] def __repr__(self): return "WorkflowExecution(run_id: {})".format(self.run_id) @@ -99,13 +104,44 @@ class WorkflowExecution(object): def _add_event(self, *args, **kwargs): evt = HistoryEvent(self.next_event_id(), *args, **kwargs) self.events.append(evt) + return evt def start(self): self._add_event( "WorkflowExecutionStarted", workflow_execution=self, ) - self._add_event( + self.schedule_decision_task() + + def schedule_decision_task(self): + self.open_counts["openDecisionTasks"] += 1 + evt = self._add_event( "DecisionTaskScheduled", workflow_execution=self, ) + self.decision_tasks.append(DecisionTask(self, evt.event_id)) + + @property + def scheduled_decision_tasks(self): + return filter( + lambda t: t.state == "SCHEDULED", + self.decision_tasks + ) + + def _find_decision_task(self, task_token): + for dt in self.decision_tasks: + if dt.task_token == task_token: + return dt + raise ValueError( + "No decision task with token: {}".format(task_token) + ) + + def start_decision_task(self, task_token, identity=None): + dt = self._find_decision_task(task_token) + evt = self._add_event( + "DecisionTaskStarted", + workflow_execution=self, + scheduled_event_id=dt.scheduled_event_id, + identity=identity + ) + dt.start(evt.event_id) diff --git a/moto/swf/responses.py b/moto/swf/responses.py index 8f7aa0344..cd2b99d6c 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -221,3 +221,16 @@ class SWFResponse(BaseResponse): return json.dumps({ "events": [evt.to_dict() for evt in wfe.events] }) + + def poll_for_decision_task(self): + domain_name = self._params["domain"] + task_list = self._params["taskList"]["name"] + identity = self._params.get("identity") + # TODO: implement reverseOrder + decision = self.swf_backend.poll_for_decision_task( + domain_name, task_list, identity=identity + ) + if decision: + return json.dumps(decision.to_full_dict()) + else: + return json.dumps({"previousStartedEventId": 0, "startedEventId": 0}) diff --git a/tests/test_swf/test_decision_tasks.py b/tests/test_swf/test_decision_tasks.py new file mode 100644 index 000000000..133b830ab --- /dev/null +++ b/tests/test_swf/test_decision_tasks.py @@ -0,0 +1,46 @@ +import boto +from sure import expect + +from moto import mock_swf +from moto.swf.exceptions import ( + SWFUnknownResourceFault, +) + +from .utils import mock_basic_workflow_type + + +@mock_swf +def setup_workflow(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60", description="A test domain") + conn = mock_basic_workflow_type("test-domain", conn) + conn.register_activity_type("test-domain", "test-activity", "v1.1") + wfe = conn.start_workflow_execution("test-domain", "uid-abcd1234", "test-workflow", "v1.0") + conn.run_id = wfe["runId"] + return conn + + +# PollForDecisionTask endpoint +@mock_swf +def test_poll_for_decision_task_when_one(): + conn = setup_workflow() + + resp = conn.get_workflow_execution_history("test-domain", conn.run_id, "uid-abcd1234") + types = [evt["eventType"] for evt in resp["events"]] + types.should.equal(["WorkflowExecutionStarted", "DecisionTaskScheduled"]) + + resp = conn.poll_for_decision_task("test-domain", "queue", identity="srv01") + types = [evt["eventType"] for evt in resp["events"]] + types.should.equal(["WorkflowExecutionStarted", "DecisionTaskScheduled", "DecisionTaskStarted"]) + + resp["events"][-1]["decisionTaskStartedEventAttributes"]["identity"].should.equal("srv01") + +@mock_swf +def test_poll_for_decision_task_when_none(): + conn = setup_workflow() + conn.poll_for_decision_task("test-domain", "queue") + + resp = conn.poll_for_decision_task("test-domain", "queue") + # this is the DecisionTask representation you get from the real SWF + # after waiting 60s when there's no decision to be taken + resp.should.equal({"previousStartedEventId": 0, "startedEventId": 0}) diff --git a/tests/test_swf/test_models.py b/tests/test_swf/test_models.py index 813afc3ca..e7153898f 100644 --- a/tests/test_swf/test_models.py +++ b/tests/test_swf/test_models.py @@ -2,6 +2,7 @@ from sure import expect from freezegun import freeze_time from moto.swf.models import ( + DecisionTask, Domain, GenericType, HistoryEvent, @@ -115,7 +116,6 @@ def test_workflow_execution_creation_child_policy_logic(): WorkflowType("test-workflow", "v1.0"), "ab1234" ).should.throw(SWFDefaultUndefinedFault) - def test_workflow_execution_string_representation(): wft = get_basic_workflow_type() wfe = WorkflowExecution(wft, "ab1234", child_policy="TERMINATE") @@ -182,6 +182,24 @@ def test_workflow_execution_full_dict_representation(): "taskStartToCloseTimeout": "300", }) +def test_workflow_execution_schedule_decision_task(): + wft = get_basic_workflow_type() + wfe = WorkflowExecution(wft, "ab1234") + wfe.open_counts["openDecisionTasks"].should.equal(0) + wfe.schedule_decision_task() + wfe.open_counts["openDecisionTasks"].should.equal(1) + +def test_workflow_execution_start_decision_task(): + wft = get_basic_workflow_type() + wfe = WorkflowExecution(wft, "ab1234") + wfe.schedule_decision_task() + dt = wfe.decision_tasks[0] + wfe.start_decision_task(dt.task_token, identity="srv01") + dt = wfe.decision_tasks[0] + dt.state.should.equal("STARTED") + wfe.events[-1].event_type.should.equal("DecisionTaskStarted") + wfe.events[-1].identity.should.equal("srv01") + # HistoryEvent @freeze_time("2015-01-01 12:00:00") @@ -207,3 +225,31 @@ def test_history_event_breaks_on_initialization_if_not_implemented(): HistoryEvent.when.called_with( 123, "UnknownHistoryEvent" ).should.throw(NotImplementedError) + + +# DecisionTask +def test_decision_task_creation(): + wft = get_basic_workflow_type() + wfe = WorkflowExecution(wft, "ab1234") + dt = DecisionTask(wfe, 123) + dt.workflow_execution.should.equal(wfe) + dt.state.should.equal("SCHEDULED") + dt.task_token.should_not.be.empty + dt.started_event_id.should.be.none + +def test_decision_task_full_dict_representation(): + wft = get_basic_workflow_type() + wfe = WorkflowExecution(wft, "ab1234") + dt = DecisionTask(wfe, 123) + + fd = dt.to_full_dict() + fd["events"].should.be.a("list") + fd["previousStartedEventId"].should.equal(0) + fd.should_not.contain("startedEventId") + fd.should.contain("taskToken") + fd["workflowExecution"].should.equal(wfe.to_short_dict()) + fd["workflowType"].should.equal(wft.to_short_dict()) + + dt.start(1234) + fd = dt.to_full_dict() + fd["startedEventId"].should.equal(1234) From aa4adbb76ee56efb6825a441df3a90b52025ac51 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Sun, 11 Oct 2015 19:14:31 +0200 Subject: [PATCH 29/94] Implement reverseOrder option for GetWorkflowExecutionHistory and PollForDecisionTask --- moto/swf/models/decision_task.py | 5 +++-- moto/swf/models/workflow_execution.py | 12 +++++++++--- moto/swf/responses.py | 12 +++++++----- tests/test_swf/test_decision_tasks.py | 7 +++++++ tests/test_swf/test_models.py | 4 ++-- tests/test_swf/test_workflow_executions.py | 15 ++++++++++++--- 6 files changed, 40 insertions(+), 15 deletions(-) diff --git a/moto/swf/models/decision_task.py b/moto/swf/models/decision_task.py index 60d0f7131..e9c3c00f6 100644 --- a/moto/swf/models/decision_task.py +++ b/moto/swf/models/decision_task.py @@ -16,10 +16,11 @@ class DecisionTask(object): # but that shouldn't be a problem for tests self.scheduled_at = datetime.now() - def to_full_dict(self): + def to_full_dict(self, reverse_order=False): + events = self.workflow_execution.events(reverse_order=reverse_order) hsh = { "events": [ - evt.to_dict() for evt in self.workflow_execution.events + evt.to_dict() for evt in events ], "taskToken": self.task_token, "previousStartedEventId": self.previous_started_event_id, diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 110001641..8a719f354 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -32,7 +32,7 @@ class WorkflowExecution(object): "openChildWorkflowExecutions": 0, } # events - self.events = [] + self._events = [] # tasks self.decision_tasks = [] self.activity_tasks = [] @@ -97,13 +97,19 @@ class WorkflowExecution(object): hsh["openCounts"] = self.open_counts return hsh + def events(self, reverse_order=False): + if reverse_order: + return reversed(self._events) + else: + return self._events + def next_event_id(self): - event_ids = [evt.event_id for evt in self.events] + event_ids = [evt.event_id for evt in self._events] return max(event_ids or [0]) def _add_event(self, *args, **kwargs): evt = HistoryEvent(self.next_event_id(), *args, **kwargs) - self.events.append(evt) + self._events.append(evt) return evt def start(self): diff --git a/moto/swf/responses.py b/moto/swf/responses.py index cd2b99d6c..8f8e86747 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -215,22 +215,24 @@ class SWFResponse(BaseResponse): _workflow_execution = self._params["execution"] run_id = _workflow_execution["runId"] workflow_id = _workflow_execution["workflowId"] - # TODO: implement reverseOrder - + reverse_order = self._params.get("reverseOrder", None) wfe = self.swf_backend.describe_workflow_execution(domain_name, run_id, workflow_id) + events = wfe.events(reverse_order=reverse_order) return json.dumps({ - "events": [evt.to_dict() for evt in wfe.events] + "events": [evt.to_dict() for evt in events] }) def poll_for_decision_task(self): domain_name = self._params["domain"] task_list = self._params["taskList"]["name"] identity = self._params.get("identity") - # TODO: implement reverseOrder + reverse_order = self._params.get("reverseOrder", None) decision = self.swf_backend.poll_for_decision_task( domain_name, task_list, identity=identity ) if decision: - return json.dumps(decision.to_full_dict()) + return json.dumps( + decision.to_full_dict(reverse_order=reverse_order) + ) else: return json.dumps({"previousStartedEventId": 0, "startedEventId": 0}) diff --git a/tests/test_swf/test_decision_tasks.py b/tests/test_swf/test_decision_tasks.py index 133b830ab..2ff1f7625 100644 --- a/tests/test_swf/test_decision_tasks.py +++ b/tests/test_swf/test_decision_tasks.py @@ -44,3 +44,10 @@ def test_poll_for_decision_task_when_none(): # this is the DecisionTask representation you get from the real SWF # after waiting 60s when there's no decision to be taken resp.should.equal({"previousStartedEventId": 0, "startedEventId": 0}) + +@mock_swf +def test_poll_for_decision_task_with_reverse_order(): + conn = setup_workflow() + resp = conn.poll_for_decision_task("test-domain", "queue", reverse_order=True) + types = [evt["eventType"] for evt in resp["events"]] + types.should.equal(["DecisionTaskStarted", "DecisionTaskScheduled", "WorkflowExecutionStarted"]) diff --git a/tests/test_swf/test_models.py b/tests/test_swf/test_models.py index e7153898f..92be04935 100644 --- a/tests/test_swf/test_models.py +++ b/tests/test_swf/test_models.py @@ -197,8 +197,8 @@ def test_workflow_execution_start_decision_task(): wfe.start_decision_task(dt.task_token, identity="srv01") dt = wfe.decision_tasks[0] dt.state.should.equal("STARTED") - wfe.events[-1].event_type.should.equal("DecisionTaskStarted") - wfe.events[-1].identity.should.equal("srv01") + wfe.events()[-1].event_type.should.equal("DecisionTaskStarted") + wfe.events()[-1].identity.should.equal("srv01") # HistoryEvent diff --git a/tests/test_swf/test_workflow_executions.py b/tests/test_swf/test_workflow_executions.py index 07aea90fb..1b8d599f9 100644 --- a/tests/test_swf/test_workflow_executions.py +++ b/tests/test_swf/test_workflow_executions.py @@ -79,10 +79,19 @@ def test_get_workflow_execution_history(): run_id = hsh["runId"] resp = conn.get_workflow_execution_history("test-domain", run_id, "uid-abcd1234") - resp["events"].should.be.a("list") - evt = resp["events"][0] - evt["eventType"].should.equal("WorkflowExecutionStarted") + types = [evt["eventType"] for evt in resp["events"]] + types.should.equal(["WorkflowExecutionStarted", "DecisionTaskScheduled"]) +@mock_swf +def test_get_workflow_execution_history_with_reverse_order(): + conn = setup_swf_environment() + hsh = conn.start_workflow_execution("test-domain", "uid-abcd1234", "test-workflow", "v1.0") + run_id = hsh["runId"] + + resp = conn.get_workflow_execution_history("test-domain", run_id, "uid-abcd1234", + reverse_order=True) + types = [evt["eventType"] for evt in resp["events"]] + types.should.equal(["DecisionTaskScheduled", "WorkflowExecutionStarted"]) @mock_swf def test_get_workflow_execution_history_on_non_existent_workflow_execution(): From c310a04c7479224974cfbbdacfdf135084b5767e Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Sun, 11 Oct 2015 19:19:48 +0200 Subject: [PATCH 30/94] Remove obsolete command about DefaultUndefinedFault (already implemented a few commits ago) --- moto/swf/models/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index b1c00738d..16f6c8915 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -131,9 +131,6 @@ class SWFBackend(BaseBackend): domain = self._get_domain(domain_name) return domain.get_type(kind, name, version) - # TODO: find what triggers a "DefaultUndefinedFault" and implement it - # (didn't found in boto source code, nor in the docs, nor on a Google search) - # (will try to reach support) def start_workflow_execution(self, domain_name, workflow_id, workflow_name, workflow_version, tag_list=None, **kwargs): From 1ccadb169feb869ec021576cfb7f7e9633b0d96f Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Sun, 11 Oct 2015 22:06:27 +0200 Subject: [PATCH 31/94] Simplify WorkflowExecution model since it always has a task list --- moto/swf/models/workflow_execution.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 8a719f354..33e032e2f 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -81,11 +81,11 @@ class WorkflowExecution(object): def to_full_dict(self): hsh = { "executionInfo": self.to_medium_dict(), - "executionConfiguration": {} + "executionConfiguration": { + "taskList": {"name": self.task_list} + } } #configuration - if hasattr(self, "task_list"): - hsh["executionConfiguration"]["taskList"] = {"name": self.task_list} for key in self._configuration_keys: attr = camelcase_to_underscores(key) if not hasattr(self, attr): From 4e223d23181dcd9da61b09f93b1ac31c842b8e3a Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Sun, 11 Oct 2015 22:11:07 +0200 Subject: [PATCH 32/94] Fix PollForDecisionTask not respecting requested task list --- moto/swf/models/__init__.py | 5 +++-- tests/test_swf/test_decision_tasks.py | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index 16f6c8915..622f38a02 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -179,8 +179,9 @@ class SWFBackend(BaseBackend): # # TODO: handle long polling (case 2) for decision tasks decision_candidates = [] - for wf_id, wf_execution in domain.workflow_executions.iteritems(): - decision_candidates += wf_execution.scheduled_decision_tasks + for _, wfe in domain.workflow_executions.iteritems(): + if wfe.task_list == task_list: + decision_candidates += wfe.scheduled_decision_tasks if any(decision_candidates): # TODO: handle task priorities (but not supported by boto for now) decision = min(decision_candidates, key=lambda d: d.scheduled_at) diff --git a/tests/test_swf/test_decision_tasks.py b/tests/test_swf/test_decision_tasks.py index 2ff1f7625..313736d7d 100644 --- a/tests/test_swf/test_decision_tasks.py +++ b/tests/test_swf/test_decision_tasks.py @@ -45,6 +45,12 @@ def test_poll_for_decision_task_when_none(): # after waiting 60s when there's no decision to be taken resp.should.equal({"previousStartedEventId": 0, "startedEventId": 0}) +@mock_swf +def test_poll_for_decision_task_on_non_existent_queue(): + conn = setup_workflow() + resp = conn.poll_for_decision_task("test-domain", "non-existent-queue") + resp.should.equal({"previousStartedEventId": 0, "startedEventId": 0}) + @mock_swf def test_poll_for_decision_task_with_reverse_order(): conn = setup_workflow() From a137e5c5c9122c031e20e9a7f793d895f8acc606 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Sun, 11 Oct 2015 22:14:16 +0200 Subject: [PATCH 33/94] Add SWF endpoint CountPendingDecisionTasks --- moto/swf/models/__init__.py | 10 ++++++++++ moto/swf/responses.py | 6 ++++++ tests/test_swf/test_decision_tasks.py | 15 +++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index 622f38a02..5b4b96c87 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -191,6 +191,16 @@ class SWFBackend(BaseBackend): else: return None + def count_pending_decision_tasks(self, domain_name, task_list): + self._check_string(domain_name) + self._check_string(task_list) + domain = self._get_domain(domain_name) + count = 0 + for _, wfe in domain.workflow_executions.iteritems(): + if wfe.task_list == task_list: + count += wfe.open_counts["openDecisionTasks"] + return count + swf_backends = {} for region in boto.swf.regions(): diff --git a/moto/swf/responses.py b/moto/swf/responses.py index 8f8e86747..7832e3ebf 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -236,3 +236,9 @@ class SWFResponse(BaseResponse): ) else: return json.dumps({"previousStartedEventId": 0, "startedEventId": 0}) + + def count_pending_decision_tasks(self): + domain_name = self._params["domain"] + task_list = self._params["taskList"]["name"] + count = self.swf_backend.count_pending_decision_tasks(domain_name, task_list) + return json.dumps({"count": count, "truncated": False}) diff --git a/tests/test_swf/test_decision_tasks.py b/tests/test_swf/test_decision_tasks.py index 313736d7d..06cdd4556 100644 --- a/tests/test_swf/test_decision_tasks.py +++ b/tests/test_swf/test_decision_tasks.py @@ -57,3 +57,18 @@ def test_poll_for_decision_task_with_reverse_order(): resp = conn.poll_for_decision_task("test-domain", "queue", reverse_order=True) types = [evt["eventType"] for evt in resp["events"]] types.should.equal(["DecisionTaskStarted", "DecisionTaskScheduled", "WorkflowExecutionStarted"]) + + +# CountPendingDecisionTasks endpoint +@mock_swf +def test_count_pending_decision_tasks(): + conn = setup_workflow() + conn.poll_for_decision_task("test-domain", "queue") + resp = conn.count_pending_decision_tasks("test-domain", "queue") + resp.should.equal({"count": 1, "truncated": False}) + +@mock_swf +def test_count_pending_decision_tasks_on_non_existent_task_list(): + conn = setup_workflow() + resp = conn.count_pending_decision_tasks("test-domain", "non-existent") + resp.should.equal({"count": 0, "truncated": False}) From c72c198208e71d743b902214c78c3517e57093a9 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 12 Oct 2015 08:38:14 +0200 Subject: [PATCH 34/94] Fix WorkflowExecution event ids not increasing --- moto/swf/models/workflow_execution.py | 2 +- tests/test_swf/test_models.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 33e032e2f..0bd632787 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -105,7 +105,7 @@ class WorkflowExecution(object): def next_event_id(self): event_ids = [evt.event_id for evt in self._events] - return max(event_ids or [0]) + return max(event_ids or [0]) + 1 def _add_event(self, *args, **kwargs): evt = HistoryEvent(self.next_event_id(), *args, **kwargs) diff --git a/tests/test_swf/test_models.py b/tests/test_swf/test_models.py index 92be04935..59c6eac59 100644 --- a/tests/test_swf/test_models.py +++ b/tests/test_swf/test_models.py @@ -200,6 +200,15 @@ def test_workflow_execution_start_decision_task(): wfe.events()[-1].event_type.should.equal("DecisionTaskStarted") wfe.events()[-1].identity.should.equal("srv01") +def test_workflow_execution_history_events_ids(): + wft = get_basic_workflow_type() + wfe = WorkflowExecution(wft, "ab1234") + wfe._add_event("WorkflowExecutionStarted", workflow_execution=wfe) + wfe._add_event("DecisionTaskScheduled", workflow_execution=wfe) + wfe._add_event("DecisionTaskStarted", workflow_execution=wfe, scheduled_event_id=2) + ids = [evt.event_id for evt in wfe.events()] + ids.should.equal([1, 2, 3]) + # HistoryEvent @freeze_time("2015-01-01 12:00:00") From d97c770849a21615551c4a9d8e5f104db540c339 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 12 Oct 2015 11:08:52 +0200 Subject: [PATCH 35/94] Add first version of SWF endpoint RespondDecisionTaskCompleted There's just the structure for now, for now the workflow execution doesn't know how to handle any decision type. --- moto/swf/exceptions.py | 16 ++++++- moto/swf/models/__init__.py | 54 +++++++++++++++++++++++ moto/swf/models/decision_task.py | 3 ++ moto/swf/models/history_event.py | 8 ++++ moto/swf/models/workflow_execution.py | 39 +++++++++++++++++ moto/swf/responses.py | 10 +++++ tests/test_swf/test_decision_tasks.py | 62 +++++++++++++++++++++++++++ tests/test_swf/test_exceptions.py | 21 +++++++++ 8 files changed, 211 insertions(+), 2 deletions(-) diff --git a/moto/swf/exceptions.py b/moto/swf/exceptions.py index cbab4e200..5f3daf108 100644 --- a/moto/swf/exceptions.py +++ b/moto/swf/exceptions.py @@ -12,9 +12,13 @@ class SWFClientError(JSONResponseError): class SWFUnknownResourceFault(SWFClientError): - def __init__(self, resource_type, resource_name): + def __init__(self, resource_type, resource_name=None): + if resource_name: + message = "Unknown {}: {}".format(resource_type, resource_name) + else: + message = "Unknown {}".format(resource_type) super(SWFUnknownResourceFault, self).__init__( - "Unknown {}: {}".format(resource_type, resource_name), + message, "com.amazonaws.swf.base.model#UnknownResourceFault") @@ -75,3 +79,11 @@ class SWFDefaultUndefinedFault(SWFClientError): super(SWFDefaultUndefinedFault, self).__init__( key_camel_case, "com.amazonaws.swf.base.model#DefaultUndefinedFault" ) + + +class SWFValidationException(SWFClientError): + def __init__(self, message): + super(SWFValidationException, self).__init__( + message, + "com.amazon.coral.validate#ValidationException" + ) diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index 5b4b96c87..c91da492e 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -11,6 +11,7 @@ from ..exceptions import ( SWFSerializationException, SWFTypeAlreadyExistsFault, SWFTypeDeprecatedFault, + SWFValidationException, ) from .activity_type import ActivityType from .decision_task import DecisionTask @@ -201,6 +202,59 @@ class SWFBackend(BaseBackend): count += wfe.open_counts["openDecisionTasks"] return count + def respond_decision_task_completed(self, task_token, + decisions=None, + execution_context=None): + self._check_string(task_token) + self._check_none_or_string(execution_context) + # let's find decision task + decision_task = None + for domain in self.domains: + for _, wfe in domain.workflow_executions.iteritems(): + for dt in wfe.decision_tasks: + if dt.task_token == task_token: + decision_task = dt + # no decision task found + if not decision_task: + # In the real world, SWF distinguishes an obviously invalid token and a + # token that has no corresponding decision task. For the latter it seems + # to wait until a task with that token comes up (which looks like a smart + # choice in an eventually-consistent system). The call doesn't seem to + # timeout shortly, it takes 3 or 4 minutes to result in: + # BotoServerError: 500 Internal Server Error + # {"__type":"com.amazon.coral.service#InternalFailure"} + # This behavior is not documented clearly in SWF docs and we'll ignore it + # in moto, as there is no obvious reason to rely on it in tests. + raise SWFValidationException("Invalid token") + # decision task found, but WorflowExecution is CLOSED + wfe = decision_task.workflow_execution + if wfe.execution_status != "OPEN": + raise SWFUnknownResourceFault( + "execution", + "WorkflowExecution=[workflowId={}, runId={}]".format( + wfe.workflow_id, wfe.run_id + ) + ) + # decision task found, but already completed + if decision_task.state != "STARTED": + if decision_task.state == "COMPLETED": + raise SWFUnknownResourceFault( + "decision task, scheduledEventId = {}".format(decision_task.scheduled_event_id) + ) + else: + raise ValueError( + "This shouldn't happen: you have to PollForDecisionTask to get a token, " + "which changes DecisionTask status to 'STARTED' ; then it can only change " + "to 'COMPLETED'. If you didn't hack moto/swf internals, this is probably " + "a bug in moto, please report it, thanks!" + ) + # everything's good + if decision_task: + wfe = decision_task.workflow_execution + wfe.complete_decision_task(decision_task.task_token, + decisions=decisions, + execution_context=execution_context) + swf_backends = {} for region in boto.swf.regions(): diff --git a/moto/swf/models/decision_task.py b/moto/swf/models/decision_task.py index e9c3c00f6..967c94fa5 100644 --- a/moto/swf/models/decision_task.py +++ b/moto/swf/models/decision_task.py @@ -34,3 +34,6 @@ class DecisionTask(object): def start(self, started_event_id): self.state = "STARTED" self.started_event_id = started_event_id + + def complete(self): + self.state = "COMPLETED" diff --git a/moto/swf/models/history_event.py b/moto/swf/models/history_event.py index 93d0cfe41..10e97ecb3 100644 --- a/moto/swf/models/history_event.py +++ b/moto/swf/models/history_event.py @@ -56,6 +56,14 @@ class HistoryEvent(object): if hasattr(self, "identity") and self.identity: hsh["identity"] = self.identity return hsh + elif self.event_type == "DecisionTaskCompleted": + hsh = { + "scheduledEventId": self.scheduled_event_id, + "startedEventId": self.started_event_id, + } + if hasattr(self, "execution_context") and self.execution_context: + hsh["executionContext"] = self.execution_context + return hsh else: raise NotImplementedError( "HistoryEvent does not implement attributes for type '{}'".format(self.event_type) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 0bd632787..bee4305c0 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -151,3 +151,42 @@ class WorkflowExecution(object): identity=identity ) dt.start(evt.event_id) + + def complete_decision_task(self, task_token, decisions=None, execution_context=None): + # TODO: check if decision can really complete in case of malformed "decisions" + dt = self._find_decision_task(task_token) + evt = self._add_event( + "DecisionTaskCompleted", + scheduled_event_id=dt.scheduled_event_id, + started_event_id=dt.started_event_id, + execution_context=execution_context, + ) + dt.complete() + self.handle_decisions(decisions) + + def handle_decisions(self, decisions): + """ + Handles a Decision according to SWF docs. + See: http://docs.aws.amazon.com/amazonswf/latest/apireference/API_Decision.html + """ + # 'decisions' can be None per boto.swf defaults, so better exiting + # directly for falsy values + if not decisions: + return + # handle each decision separately, in order + for decision in decisions: + decision_type = decision["decisionType"] + # TODO: implement Decision type: CancelTimer + # TODO: implement Decision type: CancelWorkflowExecution + # TODO: implement Decision type: CompleteWorkflowExecution + # TODO: implement Decision type: ContinueAsNewWorkflowExecution + # TODO: implement Decision type: FailWorkflowExecution + # TODO: implement Decision type: RecordMarker + # TODO: implement Decision type: RequestCancelActivityTask + # TODO: implement Decision type: RequestCancelExternalWorkflowExecution + # TODO: implement Decision type: ScheduleActivityTask + # TODO: implement Decision type: ScheduleLambdaFunction + # TODO: implement Decision type: SignalExternalWorkflowExecution + # TODO: implement Decision type: StartChildWorkflowExecution + # TODO: implement Decision type: StartTimer + raise NotImplementedError("Cannot handle decision: {}".format(decision_type)) diff --git a/moto/swf/responses.py b/moto/swf/responses.py index 7832e3ebf..9000d03f0 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -242,3 +242,13 @@ class SWFResponse(BaseResponse): task_list = self._params["taskList"]["name"] count = self.swf_backend.count_pending_decision_tasks(domain_name, task_list) return json.dumps({"count": count, "truncated": False}) + + + def respond_decision_task_completed(self): + task_token = self._params["taskToken"] + execution_context = self._params.get("executionContext") + decisions = self._params.get("decisions") + self.swf_backend.respond_decision_task_completed( + task_token, decisions=decisions, execution_context=execution_context + ) + return "" diff --git a/tests/test_swf/test_decision_tasks.py b/tests/test_swf/test_decision_tasks.py index 06cdd4556..fe84f2ebd 100644 --- a/tests/test_swf/test_decision_tasks.py +++ b/tests/test_swf/test_decision_tasks.py @@ -2,8 +2,10 @@ import boto from sure import expect from moto import mock_swf +from moto.swf import swf_backend from moto.swf.exceptions import ( SWFUnknownResourceFault, + SWFValidationException, ) from .utils import mock_basic_workflow_type @@ -72,3 +74,63 @@ def test_count_pending_decision_tasks_on_non_existent_task_list(): conn = setup_workflow() resp = conn.count_pending_decision_tasks("test-domain", "non-existent") resp.should.equal({"count": 0, "truncated": False}) + + +# RespondDecisionTaskCompleted endpoint +@mock_swf +def test_respond_decision_task_completed_with_no_decision(): + conn = setup_workflow() + + resp = conn.poll_for_decision_task("test-domain", "queue") + task_token = resp["taskToken"] + + resp = conn.respond_decision_task_completed(task_token) + resp.should.be.none + + resp = conn.get_workflow_execution_history("test-domain", conn.run_id, "uid-abcd1234") + types = [evt["eventType"] for evt in resp["events"]] + types.should.equal([ + "WorkflowExecutionStarted", + "DecisionTaskScheduled", + "DecisionTaskStarted", + "DecisionTaskCompleted", + ]) + evt = resp["events"][-1] + evt["decisionTaskCompletedEventAttributes"].should.equal({ + "scheduledEventId": 2, + "startedEventId": 3, + }) + +@mock_swf +def test_respond_decision_task_completed_with_wrong_token(): + conn = setup_workflow() + resp = conn.poll_for_decision_task("test-domain", "queue") + conn.respond_decision_task_completed.when.called_with( + "not-a-correct-token" + ).should.throw(SWFValidationException) + +@mock_swf +def test_respond_decision_task_completed_on_close_workflow_execution(): + conn = setup_workflow() + resp = conn.poll_for_decision_task("test-domain", "queue") + task_token = resp["taskToken"] + + # bad: we're closing workflow execution manually, but endpoints are not coded for now.. + wfe = swf_backend.domains[0].workflow_executions.values()[0] + wfe.execution_status = "CLOSED" + # /bad + + conn.respond_decision_task_completed.when.called_with( + task_token + ).should.throw(SWFUnknownResourceFault) + +@mock_swf +def test_respond_decision_task_completed_with_task_already_completed(): + conn = setup_workflow() + resp = conn.poll_for_decision_task("test-domain", "queue") + task_token = resp["taskToken"] + conn.respond_decision_task_completed(task_token) + + conn.respond_decision_task_completed.when.called_with( + task_token + ).should.throw(SWFUnknownResourceFault) diff --git a/tests/test_swf/test_exceptions.py b/tests/test_swf/test_exceptions.py index 27d48c261..a98e16feb 100644 --- a/tests/test_swf/test_exceptions.py +++ b/tests/test_swf/test_exceptions.py @@ -10,6 +10,7 @@ from moto.swf.exceptions import ( SWFTypeDeprecatedFault, SWFWorkflowExecutionAlreadyStartedFault, SWFDefaultUndefinedFault, + SWFValidationException, ) from moto.swf.models import ( WorkflowType, @@ -35,6 +36,16 @@ def test_swf_unknown_resource_fault(): "message": "Unknown type: detail" }) +def test_swf_unknown_resource_fault_with_only_one_parameter(): + ex = SWFUnknownResourceFault("foo bar baz") + + ex.status.should.equal(400) + ex.error_code.should.equal("UnknownResourceFault") + ex.body.should.equal({ + "__type": "com.amazonaws.swf.base.model#UnknownResourceFault", + "message": "Unknown foo bar baz" + }) + def test_swf_domain_already_exists_fault(): ex = SWFDomainAlreadyExistsFault("domain-name") @@ -103,3 +114,13 @@ def test_swf_default_undefined_fault(): "__type": "com.amazonaws.swf.base.model#DefaultUndefinedFault", "message": "executionStartToCloseTimeout", }) + +def test_swf_validation_exception(): + ex = SWFValidationException("Invalid token") + + ex.status.should.equal(400) + ex.error_code.should.equal("ValidationException") + ex.body.should.equal({ + "__type": "com.amazon.coral.validate#ValidationException", + "message": "Invalid token", + }) From 381eb5eb0f675a44d9d8b534192019421dee27d8 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 12 Oct 2015 23:32:11 +0200 Subject: [PATCH 36/94] Implement CompleteWorkflowExecution decision --- moto/swf/models/history_event.py | 7 ++++ moto/swf/models/workflow_execution.py | 46 +++++++++++++++++---------- tests/test_swf/test_decision_tasks.py | 23 ++++++++++++++ tests/test_swf/test_models.py | 10 ++++++ 4 files changed, 70 insertions(+), 16 deletions(-) diff --git a/moto/swf/models/history_event.py b/moto/swf/models/history_event.py index 10e97ecb3..4c7168b06 100644 --- a/moto/swf/models/history_event.py +++ b/moto/swf/models/history_event.py @@ -64,6 +64,13 @@ class HistoryEvent(object): if hasattr(self, "execution_context") and self.execution_context: hsh["executionContext"] = self.execution_context return hsh + elif self.event_type == "WorkflowExecutionCompleted": + hsh = { + "decisionTaskCompletedEventId": self.decision_task_completed_event_id, + } + if hasattr(self, "result") and self.result: + hsh["result"] = self.result + return hsh else: raise NotImplementedError( "HistoryEvent does not implement attributes for type '{}'".format(self.event_type) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index bee4305c0..fe35b2fdb 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -162,9 +162,9 @@ class WorkflowExecution(object): execution_context=execution_context, ) dt.complete() - self.handle_decisions(decisions) + self.handle_decisions(evt.event_id, decisions) - def handle_decisions(self, decisions): + def handle_decisions(self, event_id, decisions): """ Handles a Decision according to SWF docs. See: http://docs.aws.amazon.com/amazonswf/latest/apireference/API_Decision.html @@ -176,17 +176,31 @@ class WorkflowExecution(object): # handle each decision separately, in order for decision in decisions: decision_type = decision["decisionType"] - # TODO: implement Decision type: CancelTimer - # TODO: implement Decision type: CancelWorkflowExecution - # TODO: implement Decision type: CompleteWorkflowExecution - # TODO: implement Decision type: ContinueAsNewWorkflowExecution - # TODO: implement Decision type: FailWorkflowExecution - # TODO: implement Decision type: RecordMarker - # TODO: implement Decision type: RequestCancelActivityTask - # TODO: implement Decision type: RequestCancelExternalWorkflowExecution - # TODO: implement Decision type: ScheduleActivityTask - # TODO: implement Decision type: ScheduleLambdaFunction - # TODO: implement Decision type: SignalExternalWorkflowExecution - # TODO: implement Decision type: StartChildWorkflowExecution - # TODO: implement Decision type: StartTimer - raise NotImplementedError("Cannot handle decision: {}".format(decision_type)) + attributes_key = "{}{}EventAttributes".format( + decision_type[0].lower(), decision_type[1:] + ) + attributes = decision.get(attributes_key, {}) + if decision_type == "CompleteWorkflowExecution": + self.complete(event_id, attributes.get("result")) + else: + # TODO: implement Decision type: CancelTimer + # TODO: implement Decision type: CancelWorkflowExecution + # TODO: implement Decision type: ContinueAsNewWorkflowExecution + # TODO: implement Decision type: FailWorkflowExecution + # TODO: implement Decision type: RecordMarker + # TODO: implement Decision type: RequestCancelActivityTask + # TODO: implement Decision type: RequestCancelExternalWorkflowExecution + # TODO: implement Decision type: ScheduleActivityTask + # TODO: implement Decision type: ScheduleLambdaFunction + # TODO: implement Decision type: SignalExternalWorkflowExecution + # TODO: implement Decision type: StartChildWorkflowExecution + # TODO: implement Decision type: StartTimer + raise NotImplementedError("Cannot handle decision: {}".format(decision_type)) + + def complete(self, event_id, result=None): + self.execution_status = "CLOSED" + evt = self._add_event( + "WorkflowExecutionCompleted", + decision_task_completed_event_id=event_id, + result=result, + ) diff --git a/tests/test_swf/test_decision_tasks.py b/tests/test_swf/test_decision_tasks.py index fe84f2ebd..9db1a2649 100644 --- a/tests/test_swf/test_decision_tasks.py +++ b/tests/test_swf/test_decision_tasks.py @@ -134,3 +134,26 @@ def test_respond_decision_task_completed_with_task_already_completed(): conn.respond_decision_task_completed.when.called_with( task_token ).should.throw(SWFUnknownResourceFault) + +@mock_swf +def test_respond_decision_task_completed_with_complete_workflow_execution(): + conn = setup_workflow() + resp = conn.poll_for_decision_task("test-domain", "queue") + task_token = resp["taskToken"] + + decisions = [{ + "decisionType": "CompleteWorkflowExecution", + "completeWorkflowExecutionEventAttributes": {} + }] + resp = conn.respond_decision_task_completed(task_token, decisions=decisions) + resp.should.be.none + + resp = conn.get_workflow_execution_history("test-domain", conn.run_id, "uid-abcd1234") + types = [evt["eventType"] for evt in resp["events"]] + types.should.equal([ + "WorkflowExecutionStarted", + "DecisionTaskScheduled", + "DecisionTaskStarted", + "DecisionTaskCompleted", + "WorkflowExecutionCompleted", + ]) diff --git a/tests/test_swf/test_models.py b/tests/test_swf/test_models.py index 59c6eac59..31dc3a9ad 100644 --- a/tests/test_swf/test_models.py +++ b/tests/test_swf/test_models.py @@ -209,6 +209,16 @@ def test_workflow_execution_history_events_ids(): ids = [evt.event_id for evt in wfe.events()] ids.should.equal([1, 2, 3]) +def test_workflow_execution_complete(): + wft = get_basic_workflow_type() + wfe = WorkflowExecution(wft, "ab1234") + wfe.complete(123, result="foo") + + wfe.execution_status.should.equal("CLOSED") + wfe.events()[-1].event_type.should.equal("WorkflowExecutionCompleted") + wfe.events()[-1].decision_task_completed_event_id.should.equal(123) + wfe.events()[-1].result.should.equal("foo") + # HistoryEvent @freeze_time("2015-01-01 12:00:00") From 0749b30fb4f5ad44a7d9e4b91e3136efdbfbd006 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 19 Oct 2015 00:09:51 +0200 Subject: [PATCH 37/94] Add some basic checks on SWF decisions, more to come later --- moto/swf/exceptions.py | 32 +++++++++++++ moto/swf/models/workflow_execution.py | 69 ++++++++++++++++++++++++++- tests/test_swf/test_decision_tasks.py | 34 +++++++++++++ tests/test_swf/test_exceptions.py | 26 ++++++++++ 4 files changed, 160 insertions(+), 1 deletion(-) diff --git a/moto/swf/exceptions.py b/moto/swf/exceptions.py index 5f3daf108..4d0a9dede 100644 --- a/moto/swf/exceptions.py +++ b/moto/swf/exceptions.py @@ -87,3 +87,35 @@ class SWFValidationException(SWFClientError): message, "com.amazon.coral.validate#ValidationException" ) + + +class SWFDecisionValidationException(SWFClientError): + def __init__(self, problems): + # messages + messages = [] + for pb in problems: + if pb["type"] == "null_value": + messages.append( + "Value null at '%(where)s' failed to satisfy constraint: "\ + "Member must not be null" % pb + ) + elif pb["type"] == "bad_decision_type": + messages.append( + "Value '%(value)s' at '%(where)s' failed to satisfy constraint: " \ + "Member must satisfy enum value set: " \ + "[%(possible_values)s]" % pb + ) + else: + raise ValueError( + "Unhandled decision constraint type: {}".format(pb["type"]) + ) + # prefix + count = len(problems) + if count < 2: + prefix = "{} validation error detected:" + else: + prefix = "{} validation errors detected:" + super(SWFDecisionValidationException, self).__init__( + prefix.format(count) + "; ".join(messages), + "com.amazon.coral.validate#ValidationException" + ) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index fe35b2fdb..a9530ef70 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -3,12 +3,36 @@ import uuid from moto.core.utils import camelcase_to_underscores -from ..exceptions import SWFDefaultUndefinedFault +from ..exceptions import ( + SWFDefaultUndefinedFault, + SWFValidationException, + SWFDecisionValidationException, +) from .decision_task import DecisionTask from .history_event import HistoryEvent +# TODO: extract decision related logic into a Decision class class WorkflowExecution(object): + + # NB: the list is ordered exactly as in SWF validation exceptions so we can + # mimic error messages closely ; don't reorder it without checking SWF. + KNOWN_DECISION_TYPES = [ + "CompleteWorkflowExecution", + "StartTimer", + "RequestCancelExternalWorkflowExecution", + "SignalExternalWorkflowExecution", + "CancelTimer", + "RecordMarker", + "ScheduleActivityTask", + "ContinueAsNewWorkflowExecution", + "ScheduleLambdaFunction", + "FailWorkflowExecution", + "RequestCancelActivityTask", + "StartChildWorkflowExecution", + "CancelWorkflowExecution" + ] + def __init__(self, workflow_type, workflow_id, **kwargs): self.workflow_type = workflow_type self.workflow_id = workflow_id @@ -154,6 +178,7 @@ class WorkflowExecution(object): def complete_decision_task(self, task_token, decisions=None, execution_context=None): # TODO: check if decision can really complete in case of malformed "decisions" + self.validate_decisions(decisions) dt = self._find_decision_task(task_token) evt = self._add_event( "DecisionTaskCompleted", @@ -164,6 +189,48 @@ class WorkflowExecution(object): dt.complete() self.handle_decisions(evt.event_id, decisions) + def validate_decisions(self, decisions): + """ + Performs some basic validations on decisions. The real SWF service + seems to break early and *not* process any decision if there's a + validation problem, such as a malformed decision for instance. I didn't + find an explicit documentation for that though, so criticisms welcome. + """ + if not decisions: + return + + problems = [] + + # check close decision is last + # TODO: see what happens on real SWF service if we ask for 2 close decisions + for dcs in decisions[:-1]: + close_decision_types = [ + "CompleteWorkflowExecution", + "FailWorkflowExecution", + "CancelWorkflowExecution", + ] + if dcs["decisionType"] in close_decision_types: + raise SWFValidationException( + "Close must be last decision in list" + ) + + decision_number = 0 + for dcs in decisions: + decision_number += 1 + # TODO: check decision types mandatory attributes + # check decision type is correct + if dcs["decisionType"] not in self.KNOWN_DECISION_TYPES: + problems.append({ + "type": "bad_decision_type", + "value": dcs["decisionType"], + "where": "decisions.{}.member.decisionType".format(decision_number), + "possible_values": ", ".join(self.KNOWN_DECISION_TYPES), + }) + + # raise if any problem + if any(problems): + raise SWFDecisionValidationException(problems) + def handle_decisions(self, event_id, decisions): """ Handles a Decision according to SWF docs. diff --git a/tests/test_swf/test_decision_tasks.py b/tests/test_swf/test_decision_tasks.py index 9db1a2649..cf83ec404 100644 --- a/tests/test_swf/test_decision_tasks.py +++ b/tests/test_swf/test_decision_tasks.py @@ -6,6 +6,7 @@ from moto.swf import swf_backend from moto.swf.exceptions import ( SWFUnknownResourceFault, SWFValidationException, + SWFDecisionValidationException, ) from .utils import mock_basic_workflow_type @@ -157,3 +158,36 @@ def test_respond_decision_task_completed_with_complete_workflow_execution(): "DecisionTaskCompleted", "WorkflowExecutionCompleted", ]) + +@mock_swf +def test_respond_decision_task_completed_with_close_decision_not_last(): + conn = setup_workflow() + resp = conn.poll_for_decision_task("test-domain", "queue") + task_token = resp["taskToken"] + + decisions = [ + { "decisionType": "CompleteWorkflowExecution" }, + { "decisionType": "WeDontCare" }, + ] + + conn.respond_decision_task_completed.when.called_with( + task_token, decisions=decisions + ).should.throw(SWFValidationException, r"Close must be last decision in list") + +@mock_swf +def test_respond_decision_task_completed_with_invalid_decision_type(): + conn = setup_workflow() + resp = conn.poll_for_decision_task("test-domain", "queue") + task_token = resp["taskToken"] + + decisions = [ + { "decisionType": "BadDecisionType" }, + { "decisionType": "CompleteWorkflowExecution" }, + ] + + conn.respond_decision_task_completed.when.called_with( + task_token, decisions=decisions + ).should.throw( + SWFDecisionValidationException, + r"Value 'BadDecisionType' at 'decisions.1.member.decisionType'" + ) diff --git a/tests/test_swf/test_exceptions.py b/tests/test_swf/test_exceptions.py index a98e16feb..394493dbc 100644 --- a/tests/test_swf/test_exceptions.py +++ b/tests/test_swf/test_exceptions.py @@ -11,6 +11,7 @@ from moto.swf.exceptions import ( SWFWorkflowExecutionAlreadyStartedFault, SWFDefaultUndefinedFault, SWFValidationException, + SWFDecisionValidationException, ) from moto.swf.models import ( WorkflowType, @@ -124,3 +125,28 @@ def test_swf_validation_exception(): "__type": "com.amazon.coral.validate#ValidationException", "message": "Invalid token", }) + +def test_swf_decision_validation_error(): + ex = SWFDecisionValidationException([ + { "type": "null_value", + "where": "decisions.1.member.startTimerDecisionAttributes.startToFireTimeout" }, + { "type": "bad_decision_type", + "value": "FooBar", + "where": "decisions.1.member.decisionType", + "possible_values": "Foo, Bar, Baz"}, + ]) + + ex.status.should.equal(400) + ex.error_code.should.equal("ValidationException") + ex.body["__type"].should.equal("com.amazon.coral.validate#ValidationException") + + msg = ex.body["message"] + msg.should.match(r"^2 validation errors detected:") + msg.should.match( + r"Value null at 'decisions.1.member.startTimerDecisionAttributes.startToFireTimeout' "\ + r"failed to satisfy constraint: Member must not be null;" + ) + msg.should.match( + r"Value 'FooBar' at 'decisions.1.member.decisionType' failed to satisfy constraint: " \ + r"Member must satisfy enum value set: \[Foo, Bar, Baz\]" + ) From 507351612ed635c756e27b9528f2f21c605288ee Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 19 Oct 2015 05:43:58 +0200 Subject: [PATCH 38/94] Fix missing space in decision validation error --- moto/swf/exceptions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/swf/exceptions.py b/moto/swf/exceptions.py index 4d0a9dede..d79281100 100644 --- a/moto/swf/exceptions.py +++ b/moto/swf/exceptions.py @@ -112,9 +112,9 @@ class SWFDecisionValidationException(SWFClientError): # prefix count = len(problems) if count < 2: - prefix = "{} validation error detected:" + prefix = "{} validation error detected: " else: - prefix = "{} validation errors detected:" + prefix = "{} validation errors detected: " super(SWFDecisionValidationException, self).__init__( prefix.format(count) + "; ".join(messages), "com.amazon.coral.validate#ValidationException" From 558b84fb6a1059862b9dd78c2ea4494895de403f Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 19 Oct 2015 09:25:54 +0200 Subject: [PATCH 39/94] Add checks for *DecisionAttributes within RespondDecisionTaskCompleted --- moto/swf/constants.py | 85 +++++++++++++++++++++++++++ moto/swf/models/history_event.py | 5 +- moto/swf/models/workflow_execution.py | 34 +++++++++-- moto/swf/utils.py | 2 + tests/test_swf/test_decision_tasks.py | 39 ++++++++++++ tests/test_swf/test_utils.py | 11 ++++ 6 files changed, 169 insertions(+), 7 deletions(-) create mode 100644 moto/swf/constants.py create mode 100644 moto/swf/utils.py create mode 100644 tests/test_swf/test_utils.py diff --git a/moto/swf/constants.py b/moto/swf/constants.py new file mode 100644 index 000000000..75a3de5a9 --- /dev/null +++ b/moto/swf/constants.py @@ -0,0 +1,85 @@ +# List decision fields and if they're required or not +# +# See http://docs.aws.amazon.com/amazonswf/latest/apireference/API_RespondDecisionTaskCompleted.html +# and subsequent docs for each decision type. +DECISIONS_FIELDS = { + "cancelTimerDecisionAttributes": { + "timerId": { "type": "string", "required": True } + }, + "cancelWorkflowExecutionDecisionAttributes": { + "details": { "type": "string", "required": False } + }, + "completeWorkflowExecutionDecisionAttributes": { + "result": { "type": "string", "required": False } + }, + "continueAsNewWorkflowExecutionDecisionAttributes": { + "childPolicy": { "type": "string", "required": False }, + "executionStartToCloseTimeout": { "type": "string", "required": False }, + "input": { "type": "string", "required": False }, + "lambdaRole": { "type": "string", "required": False }, + "tagList": { "type": "string", "array": True, "required": False }, + "taskList": { "type": "TaskList", "required": False }, + "taskPriority": { "type": "string", "required": False }, + "taskStartToCloseTimeout": { "type": "string", "required": False }, + "workflowTypeVersion": { "type": "string", "required": False } + }, + "failWorkflowExecutionDecisionAttributes": { + "details": { "type": "string", "required": False }, + "reason": { "type": "string", "required": False } + }, + "recordMarkerDecisionAttributes": { + "details": { "type": "string", "required": False }, + "markerName": { "type": "string", "required": True } + }, + "requestCancelActivityTaskDecisionAttributes": { + "activityId": { "type": "string", "required": True } + }, + "requestCancelExternalWorkflowExecutionDecisionAttributes": { + "control": { "type": "string", "required": False }, + "runId": { "type": "string", "required": False }, + "workflowId": { "type": "string", "required": True } + }, + "scheduleActivityTaskDecisionAttributes": { + "activityId": { "type": "string", "required": True }, + "activityType": { "type": "ActivityType", "required": True }, + "control": { "type": "string", "required": False }, + "heartbeatTimeout": { "type": "string", "required": False }, + "input": { "type": "string", "required": False }, + "scheduleToCloseTimeout": { "type": "string", "required": False }, + "scheduleToStartTimeout": { "type": "string", "required": False }, + "startToCloseTimeout": { "type": "string", "required": False }, + "taskList": { "type": "TaskList", "required": False }, + "taskPriority": { "type": "string", "required": False } + }, + "scheduleLambdaFunctionDecisionAttributes": { + "id": { "type": "string", "required": True }, + "input": { "type": "string", "required": False }, + "name": { "type": "string", "required": True }, + "startToCloseTimeout": { "type": "string", "required": False } + }, + "signalExternalWorkflowExecutionDecisionAttributes": { + "control": { "type": "string", "required": False }, + "input": { "type": "string", "required": False }, + "runId": { "type": "string", "required": False }, + "signalName": { "type": "string", "required": True }, + "workflowId": { "type": "string", "required": True } + }, + "startChildWorkflowExecutionDecisionAttributes": { + "childPolicy": { "type": "string", "required": False }, + "control": { "type": "string", "required": False }, + "executionStartToCloseTimeout": { "type": "string", "required": False }, + "input": { "type": "string", "required": False }, + "lambdaRole": { "type": "string", "required": False }, + "tagList": { "type": "string", "array": True, "required": False }, + "taskList": { "type": "TaskList", "required": False }, + "taskPriority": { "type": "string", "required": False }, + "taskStartToCloseTimeout": { "type": "string", "required": False }, + "workflowId": { "type": "string", "required": True }, + "workflowType": { "type": "WorkflowType", "required": True } + }, + "startTimerDecisionAttributes": { + "control": { "type": "string", "required": False }, + "startToFireTimeout": { "type": "string", "required": True }, + "timerId": { "type": "string", "required": True } + } +} diff --git a/moto/swf/models/history_event.py b/moto/swf/models/history_event.py index 4c7168b06..87e8f1e35 100644 --- a/moto/swf/models/history_event.py +++ b/moto/swf/models/history_event.py @@ -2,6 +2,8 @@ from __future__ import unicode_literals from datetime import datetime from time import mktime +from ..utils import decapitalize + class HistoryEvent(object): def __init__(self, event_id, event_type, **kwargs): @@ -23,8 +25,7 @@ class HistoryEvent(object): def _attributes_key(self): key = "{}EventAttributes".format(self.event_type) - key = key[0].lower() + key[1:] - return key + return decapitalize(key) def event_attributes(self): if self.event_type == "WorkflowExecutionStarted": diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index a9530ef70..8d297975a 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -3,11 +3,15 @@ import uuid from moto.core.utils import camelcase_to_underscores +from ..constants import ( + DECISIONS_FIELDS, +) from ..exceptions import ( SWFDefaultUndefinedFault, SWFValidationException, SWFDecisionValidationException, ) +from ..utils import decapitalize from .decision_task import DecisionTask from .history_event import HistoryEvent @@ -189,6 +193,19 @@ class WorkflowExecution(object): dt.complete() self.handle_decisions(evt.event_id, decisions) + def _check_decision_attributes(self, kind, value, decision_id): + problems = [] + constraints = DECISIONS_FIELDS.get(kind, {}) + for key, constraint in constraints.iteritems(): + if constraint["required"] and not value.get(key): + problems.append({ + "type": "null_value", + "where": "decisions.{}.member.{}.{}".format( + decision_id, kind, key + ) + }) + return problems + def validate_decisions(self, decisions): """ Performs some basic validations on decisions. The real SWF service @@ -202,7 +219,7 @@ class WorkflowExecution(object): problems = [] # check close decision is last - # TODO: see what happens on real SWF service if we ask for 2 close decisions + # (the real SWF service also works that way if you provide 2 close decision tasks) for dcs in decisions[:-1]: close_decision_types = [ "CompleteWorkflowExecution", @@ -217,7 +234,16 @@ class WorkflowExecution(object): decision_number = 0 for dcs in decisions: decision_number += 1 - # TODO: check decision types mandatory attributes + # check decision types mandatory attributes + # NB: the real SWF service seems to check attributes even for attributes list + # that are not in line with the decisionType, so we do the same + attrs_to_check = filter(lambda x: x.endswith("DecisionAttributes"), dcs.keys()) + if dcs["decisionType"] in self.KNOWN_DECISION_TYPES: + decision_type = dcs["decisionType"] + decision_attr = "{}DecisionAttributes".format(decapitalize(decision_type)) + attrs_to_check.append(decision_attr) + for attr in attrs_to_check: + problems += self._check_decision_attributes(attr, dcs.get(attr, {}), decision_number) # check decision type is correct if dcs["decisionType"] not in self.KNOWN_DECISION_TYPES: problems.append({ @@ -243,9 +269,7 @@ class WorkflowExecution(object): # handle each decision separately, in order for decision in decisions: decision_type = decision["decisionType"] - attributes_key = "{}{}EventAttributes".format( - decision_type[0].lower(), decision_type[1:] - ) + attributes_key = "{}EventAttributes".format(decapitalize(decision_type)) attributes = decision.get(attributes_key, {}) if decision_type == "CompleteWorkflowExecution": self.complete(event_id, attributes.get("result")) diff --git a/moto/swf/utils.py b/moto/swf/utils.py new file mode 100644 index 000000000..1b85f4ca9 --- /dev/null +++ b/moto/swf/utils.py @@ -0,0 +1,2 @@ +def decapitalize(key): + return key[0].lower() + key[1:] diff --git a/tests/test_swf/test_decision_tasks.py b/tests/test_swf/test_decision_tasks.py index cf83ec404..b67575b3f 100644 --- a/tests/test_swf/test_decision_tasks.py +++ b/tests/test_swf/test_decision_tasks.py @@ -191,3 +191,42 @@ def test_respond_decision_task_completed_with_invalid_decision_type(): SWFDecisionValidationException, r"Value 'BadDecisionType' at 'decisions.1.member.decisionType'" ) + +@mock_swf +def test_respond_decision_task_completed_with_missing_attributes(): + conn = setup_workflow() + resp = conn.poll_for_decision_task("test-domain", "queue") + task_token = resp["taskToken"] + + decisions = [ + { + "decisionType": "should trigger even with incorrect decision type", + "startTimerDecisionAttributes": {} + }, + ] + + conn.respond_decision_task_completed.when.called_with( + task_token, decisions=decisions + ).should.throw( + SWFDecisionValidationException, + r"Value null at 'decisions.1.member.startTimerDecisionAttributes.timerId' " \ + r"failed to satisfy constraint: Member must not be null" + ) + +@mock_swf +def test_respond_decision_task_completed_with_missing_attributes_totally(): + conn = setup_workflow() + resp = conn.poll_for_decision_task("test-domain", "queue") + task_token = resp["taskToken"] + + decisions = [ + { "decisionType": "StartTimer" }, + ] + + conn.respond_decision_task_completed.when.called_with( + task_token, decisions=decisions + ).should.throw( + SWFDecisionValidationException, + r"Value null at 'decisions.1.member.startTimerDecisionAttributes.timerId' " \ + r"failed to satisfy constraint: Member must not be null" + ) diff --git a/tests/test_swf/test_utils.py b/tests/test_swf/test_utils.py new file mode 100644 index 000000000..6d11ba5fc --- /dev/null +++ b/tests/test_swf/test_utils.py @@ -0,0 +1,11 @@ +from sure import expect +from moto.swf.utils import decapitalize + +def test_decapitalize(): + cases = { + "fooBar": "fooBar", + "FooBar": "fooBar", + "FOO BAR": "fOO BAR", + } + for before, after in cases.iteritems(): + decapitalize(before).should.equal(after) From 6810973b768318ab1017d83b884a9dc574cf1856 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Fri, 23 Oct 2015 09:05:52 +0200 Subject: [PATCH 40/94] Update obsolete comment about SWF decisions completion --- moto/swf/models/workflow_execution.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 8d297975a..164217b9a 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -181,7 +181,8 @@ class WorkflowExecution(object): dt.start(evt.event_id) def complete_decision_task(self, task_token, decisions=None, execution_context=None): - # TODO: check if decision can really complete in case of malformed "decisions" + # In case of a malformed or invalid decision task, SWF will raise an error and + # it won't perform any of the decisions in the decision set. self.validate_decisions(decisions) dt = self._find_decision_task(task_token) evt = self._add_event( From 417f732b530f51397fd7c6c3579cad6d6a2461a0 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Sat, 24 Oct 2015 04:35:21 +0200 Subject: [PATCH 41/94] Implement FailWorkflowExecution decision --- moto/swf/models/history_event.py | 9 +++++++ moto/swf/models/workflow_execution.py | 39 ++++++++++++++++++++++++--- tests/test_swf/test_decision_tasks.py | 29 +++++++++++++++++++- tests/test_swf/test_models.py | 29 ++++++++++++++++++++ 4 files changed, 101 insertions(+), 5 deletions(-) diff --git a/moto/swf/models/history_event.py b/moto/swf/models/history_event.py index 87e8f1e35..0cb8e83eb 100644 --- a/moto/swf/models/history_event.py +++ b/moto/swf/models/history_event.py @@ -72,6 +72,15 @@ class HistoryEvent(object): if hasattr(self, "result") and self.result: hsh["result"] = self.result return hsh + elif self.event_type == "WorkflowExecutionFailed": + hsh = { + "decisionTaskCompletedEventId": self.decision_task_completed_event_id, + } + if hasattr(self, "details") and self.details: + hsh["details"] = self.details + if hasattr(self, "reason") and self.reason: + hsh["reason"] = self.reason + return hsh else: raise NotImplementedError( "HistoryEvent does not implement attributes for type '{}'".format(self.event_type) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 164217b9a..f53c10280 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -1,4 +1,6 @@ from __future__ import unicode_literals +from datetime import datetime +from time import mktime import uuid from moto.core.utils import camelcase_to_underscores @@ -38,11 +40,20 @@ class WorkflowExecution(object): ] def __init__(self, workflow_type, workflow_id, **kwargs): - self.workflow_type = workflow_type self.workflow_id = workflow_id self.run_id = uuid.uuid4().hex - self.execution_status = "OPEN" + # WorkflowExecutionInfo self.cancel_requested = False + # TODO: check valid values among: + # COMPLETED | FAILED | CANCELED | TERMINATED | CONTINUED_AS_NEW | TIMED_OUT + # TODO: implement them all + self.close_status = None + self.close_timestamp = None + self.execution_status = "OPEN" + self.parent = None + self.start_timestamp = None + self.tag_list = [] # TODO + self.workflow_type = workflow_type # args processing # NB: the order follows boto/SWF order of exceptions appearance (if no # param is set, # SWF will raise DefaultUndefinedFault errors in the @@ -102,7 +113,7 @@ class WorkflowExecution(object): "executionStatus": self.execution_status, "cancelRequested": self.cancel_requested, } - if hasattr(self, "tag_list"): + if hasattr(self, "tag_list") and self.tag_list: hsh["tagList"] = self.tag_list return hsh @@ -140,7 +151,12 @@ class WorkflowExecution(object): self._events.append(evt) return evt + # TODO: move it in utils + def _now_timestamp(self): + return float(mktime(datetime.now().timetuple())) + def start(self): + self.start_timestamp = self._now_timestamp() self._add_event( "WorkflowExecutionStarted", workflow_execution=self, @@ -274,11 +290,12 @@ class WorkflowExecution(object): attributes = decision.get(attributes_key, {}) if decision_type == "CompleteWorkflowExecution": self.complete(event_id, attributes.get("result")) + elif decision_type == "FailWorkflowExecution": + self.fail(event_id, attributes.get("details"), attributes.get("reason")) else: # TODO: implement Decision type: CancelTimer # TODO: implement Decision type: CancelWorkflowExecution # TODO: implement Decision type: ContinueAsNewWorkflowExecution - # TODO: implement Decision type: FailWorkflowExecution # TODO: implement Decision type: RecordMarker # TODO: implement Decision type: RequestCancelActivityTask # TODO: implement Decision type: RequestCancelExternalWorkflowExecution @@ -291,8 +308,22 @@ class WorkflowExecution(object): def complete(self, event_id, result=None): self.execution_status = "CLOSED" + self.close_status = "COMPLETED" + self.close_timestamp = self._now_timestamp() evt = self._add_event( "WorkflowExecutionCompleted", decision_task_completed_event_id=event_id, result=result, ) + + def fail(self, event_id, details=None, reason=None): + # TODO: implement lenght constraints on details/reason + self.execution_status = "CLOSED" + self.close_status = "FAILED" + self.close_timestamp = self._now_timestamp() + evt = self._add_event( + "WorkflowExecutionFailed", + decision_task_completed_event_id=event_id, + details=details, + reason=reason, + ) diff --git a/tests/test_swf/test_decision_tasks.py b/tests/test_swf/test_decision_tasks.py index b67575b3f..6ccd0780b 100644 --- a/tests/test_swf/test_decision_tasks.py +++ b/tests/test_swf/test_decision_tasks.py @@ -144,7 +144,7 @@ def test_respond_decision_task_completed_with_complete_workflow_execution(): decisions = [{ "decisionType": "CompleteWorkflowExecution", - "completeWorkflowExecutionEventAttributes": {} + "completeWorkflowExecutionEventAttributes": {"result": "foo bar"} }] resp = conn.respond_decision_task_completed(task_token, decisions=decisions) resp.should.be.none @@ -158,6 +158,7 @@ def test_respond_decision_task_completed_with_complete_workflow_execution(): "DecisionTaskCompleted", "WorkflowExecutionCompleted", ]) + resp["events"][-1]["workflowExecutionCompletedEventAttributes"]["result"].should.equal("foo bar") @mock_swf def test_respond_decision_task_completed_with_close_decision_not_last(): @@ -230,3 +231,29 @@ def test_respond_decision_task_completed_with_missing_attributes_totally(): r"Value null at 'decisions.1.member.startTimerDecisionAttributes.timerId' " \ r"failed to satisfy constraint: Member must not be null" ) + +@mock_swf +def test_respond_decision_task_completed_with_fail_workflow_execution(): + conn = setup_workflow() + resp = conn.poll_for_decision_task("test-domain", "queue") + task_token = resp["taskToken"] + + decisions = [{ + "decisionType": "FailWorkflowExecution", + "failWorkflowExecutionEventAttributes": {"reason": "my rules", "details": "foo"} + }] + resp = conn.respond_decision_task_completed(task_token, decisions=decisions) + resp.should.be.none + + resp = conn.get_workflow_execution_history("test-domain", conn.run_id, "uid-abcd1234") + types = [evt["eventType"] for evt in resp["events"]] + types.should.equal([ + "WorkflowExecutionStarted", + "DecisionTaskScheduled", + "DecisionTaskStarted", + "DecisionTaskCompleted", + "WorkflowExecutionFailed", + ]) + attrs = resp["events"][-1]["workflowExecutionFailedEventAttributes"] + attrs["reason"].should.equal("my rules") + attrs["details"].should.equal("foo") diff --git a/tests/test_swf/test_models.py b/tests/test_swf/test_models.py index 31dc3a9ad..5af593821 100644 --- a/tests/test_swf/test_models.py +++ b/tests/test_swf/test_models.py @@ -209,16 +209,45 @@ def test_workflow_execution_history_events_ids(): ids = [evt.event_id for evt in wfe.events()] ids.should.equal([1, 2, 3]) +@freeze_time("2015-01-01 12:00:00") +def test_workflow_execution_start(): + wft = get_basic_workflow_type() + wfe = WorkflowExecution(wft, "ab1234") + wfe.events().should.equal([]) + + wfe.start() + wfe.start_timestamp.should.equal(1420110000.0) + wfe.events().should.have.length_of(2) + wfe.events()[0].event_type.should.equal("WorkflowExecutionStarted") + wfe.events()[1].event_type.should.equal("DecisionTaskScheduled") + +@freeze_time("2015-01-02 12:00:00") def test_workflow_execution_complete(): wft = get_basic_workflow_type() wfe = WorkflowExecution(wft, "ab1234") wfe.complete(123, result="foo") wfe.execution_status.should.equal("CLOSED") + wfe.close_status.should.equal("COMPLETED") + wfe.close_timestamp.should.equal(1420196400.0) wfe.events()[-1].event_type.should.equal("WorkflowExecutionCompleted") wfe.events()[-1].decision_task_completed_event_id.should.equal(123) wfe.events()[-1].result.should.equal("foo") +@freeze_time("2015-01-02 12:00:00") +def test_workflow_execution_fail(): + wft = get_basic_workflow_type() + wfe = WorkflowExecution(wft, "ab1234") + wfe.fail(123, details="some details", reason="my rules") + + wfe.execution_status.should.equal("CLOSED") + wfe.close_status.should.equal("FAILED") + wfe.close_timestamp.should.equal(1420196400.0) + wfe.events()[-1].event_type.should.equal("WorkflowExecutionFailed") + wfe.events()[-1].decision_task_completed_event_id.should.equal(123) + wfe.events()[-1].details.should.equal("some details") + wfe.events()[-1].reason.should.equal("my rules") + # HistoryEvent @freeze_time("2015-01-01 12:00:00") From 918cf8a4e31e3ce497c687359485c5f9fc85ccce Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Sat, 24 Oct 2015 05:10:01 +0200 Subject: [PATCH 42/94] Fix decision parameters: attributes are in foo*Decision*Attributes --- moto/swf/models/workflow_execution.py | 2 +- tests/test_swf/test_decision_tasks.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index f53c10280..c22d3e021 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -286,7 +286,7 @@ class WorkflowExecution(object): # handle each decision separately, in order for decision in decisions: decision_type = decision["decisionType"] - attributes_key = "{}EventAttributes".format(decapitalize(decision_type)) + attributes_key = "{}DecisionAttributes".format(decapitalize(decision_type)) attributes = decision.get(attributes_key, {}) if decision_type == "CompleteWorkflowExecution": self.complete(event_id, attributes.get("result")) diff --git a/tests/test_swf/test_decision_tasks.py b/tests/test_swf/test_decision_tasks.py index 6ccd0780b..a07eecbfb 100644 --- a/tests/test_swf/test_decision_tasks.py +++ b/tests/test_swf/test_decision_tasks.py @@ -144,7 +144,7 @@ def test_respond_decision_task_completed_with_complete_workflow_execution(): decisions = [{ "decisionType": "CompleteWorkflowExecution", - "completeWorkflowExecutionEventAttributes": {"result": "foo bar"} + "completeWorkflowExecutionDecisionAttributes": {"result": "foo bar"} }] resp = conn.respond_decision_task_completed(task_token, decisions=decisions) resp.should.be.none @@ -240,7 +240,7 @@ def test_respond_decision_task_completed_with_fail_workflow_execution(): decisions = [{ "decisionType": "FailWorkflowExecution", - "failWorkflowExecutionEventAttributes": {"reason": "my rules", "details": "foo"} + "failWorkflowExecutionDecisionAttributes": {"reason": "my rules", "details": "foo"} }] resp = conn.respond_decision_task_completed(task_token, decisions=decisions) resp.should.be.none From 49e44c8ee653a7e1260dd5c407c9733cf25b00e9 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Sat, 24 Oct 2015 12:05:42 +0200 Subject: [PATCH 43/94] Fix openDecisionTasks counter not updated when we complete a DecisionTask --- moto/swf/models/workflow_execution.py | 9 ++++++--- tests/test_swf/test_decision_tasks.py | 9 +++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index c22d3e021..3d2031c7e 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -279,10 +279,10 @@ class WorkflowExecution(object): Handles a Decision according to SWF docs. See: http://docs.aws.amazon.com/amazonswf/latest/apireference/API_Decision.html """ - # 'decisions' can be None per boto.swf defaults, so better exiting - # directly for falsy values + # 'decisions' can be None per boto.swf defaults, so replace it with something iterable if not decisions: - return + decisions = [] + # handle each decision separately, in order for decision in decisions: decision_type = decision["decisionType"] @@ -306,6 +306,9 @@ class WorkflowExecution(object): # TODO: implement Decision type: StartTimer raise NotImplementedError("Cannot handle decision: {}".format(decision_type)) + # finally decrement counter if and only if everything went well + self.open_counts["openDecisionTasks"] -= 1 + def complete(self, event_id, result=None): self.execution_status = "CLOSED" self.close_status = "COMPLETED" diff --git a/tests/test_swf/test_decision_tasks.py b/tests/test_swf/test_decision_tasks.py index a07eecbfb..568d68c2d 100644 --- a/tests/test_swf/test_decision_tasks.py +++ b/tests/test_swf/test_decision_tasks.py @@ -76,6 +76,15 @@ def test_count_pending_decision_tasks_on_non_existent_task_list(): resp = conn.count_pending_decision_tasks("test-domain", "non-existent") resp.should.equal({"count": 0, "truncated": False}) +@mock_swf +def test_count_pending_decision_tasks_after_decision_completes(): + conn = setup_workflow() + resp = conn.poll_for_decision_task("test-domain", "queue") + conn.respond_decision_task_completed(resp["taskToken"]) + + resp = conn.count_pending_decision_tasks("test-domain", "queue") + resp.should.equal({"count": 0, "truncated": False}) + # RespondDecisionTaskCompleted endpoint @mock_swf From fa4608be987979402275d48b7c19cea97bed8033 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Sat, 24 Oct 2015 12:57:06 +0200 Subject: [PATCH 44/94] Add basic ActivityTask model --- moto/swf/models/__init__.py | 1 + moto/swf/models/activity_task.py | 20 ++++++++++++++++++++ tests/test_swf/test_models.py | 24 ++++++++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 moto/swf/models/activity_task.py diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index c91da492e..3265d73da 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -13,6 +13,7 @@ from ..exceptions import ( SWFTypeDeprecatedFault, SWFValidationException, ) +from .activity_task import ActivityTask from .activity_type import ActivityType from .decision_task import DecisionTask from .domain import Domain diff --git a/moto/swf/models/activity_task.py b/moto/swf/models/activity_task.py new file mode 100644 index 000000000..298984a21 --- /dev/null +++ b/moto/swf/models/activity_task.py @@ -0,0 +1,20 @@ +from __future__ import unicode_literals +import uuid + + +class ActivityTask(object): + def __init__(self, activity_id, activity_type, workflow_execution, input=None): + self.activity_id = activity_id + self.activity_type = activity_type + self.input = input + self.started_event_id = None + self.state = "SCHEDULED" + self.task_token = str(uuid.uuid4()) + self.workflow_execution = workflow_execution + + def start(self, started_event_id): + self.state = "STARTED" + self.started_event_id = started_event_id + + def complete(self): + self.state = "COMPLETED" diff --git a/tests/test_swf/test_models.py b/tests/test_swf/test_models.py index 5af593821..4707140e0 100644 --- a/tests/test_swf/test_models.py +++ b/tests/test_swf/test_models.py @@ -2,6 +2,7 @@ from sure import expect from freezegun import freeze_time from moto.swf.models import ( + ActivityTask, DecisionTask, Domain, GenericType, @@ -301,3 +302,26 @@ def test_decision_task_full_dict_representation(): dt.start(1234) fd = dt.to_full_dict() fd["startedEventId"].should.equal(1234) + + +# ActivityTask +def test_activity_task_creation(): + wft = get_basic_workflow_type() + wfe = WorkflowExecution(wft, "ab1234") + task = ActivityTask( + activity_id="my-activity-123", + activity_type="foo", + input="optional", + workflow_execution=wfe, + ) + task.workflow_execution.should.equal(wfe) + task.state.should.equal("SCHEDULED") + task.task_token.should_not.be.empty + task.started_event_id.should.be.none + + task.start(123) + task.state.should.equal("STARTED") + task.started_event_id.should.equal(123) + + task.complete() + task.state.should.equal("COMPLETED") From 53630dc061c4d710dfd407560e68219d36a39318 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Sun, 25 Oct 2015 11:30:11 +0100 Subject: [PATCH 45/94] Add a Domain to WorkflowExecution objects This will be needed later for finding an activity type for instance. --- moto/swf/models/__init__.py | 2 +- moto/swf/models/workflow_execution.py | 3 +- tests/test_swf/test_models.py | 68 ++++++++++++++++----------- tests/test_swf/utils.py | 6 +++ 4 files changed, 50 insertions(+), 29 deletions(-) diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index 3265d73da..2c040f8db 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -148,7 +148,7 @@ class SWFBackend(BaseBackend): wf_type = domain.get_type("workflow", workflow_name, workflow_version) if wf_type.status == "DEPRECATED": raise SWFTypeDeprecatedFault(wf_type) - wfe = WorkflowExecution(wf_type, workflow_id, + wfe = WorkflowExecution(domain, wf_type, workflow_id, tag_list=tag_list, **kwargs) domain.add_workflow_execution(wfe) wfe.start() diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 3d2031c7e..5445f9a71 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -39,7 +39,8 @@ class WorkflowExecution(object): "CancelWorkflowExecution" ] - def __init__(self, workflow_type, workflow_id, **kwargs): + def __init__(self, domain, workflow_type, workflow_id, **kwargs): + self.domain = domain self.workflow_id = workflow_id self.run_id = uuid.uuid4().hex # WorkflowExecutionInfo diff --git a/tests/test_swf/test_models.py b/tests/test_swf/test_models.py index 4707140e0..45e541e75 100644 --- a/tests/test_swf/test_models.py +++ b/tests/test_swf/test_models.py @@ -14,7 +14,18 @@ from moto.swf.exceptions import ( SWFDefaultUndefinedFault, ) -from .utils import get_basic_workflow_type +from .utils import ( + get_basic_domain, + get_basic_workflow_type, +) + + +# Some utility methods +# TODO: move them in utils +def make_workflow_execution(**kwargs): + domain = get_basic_domain() + wft = get_basic_workflow_type() + return WorkflowExecution(domain, wft, "ab1234", **kwargs) # Domain @@ -87,13 +98,19 @@ def test_type_string_representation(): # WorkflowExecution def test_workflow_execution_creation(): + domain = get_basic_domain() wft = get_basic_workflow_type() - wfe = WorkflowExecution(wft, "ab1234", child_policy="TERMINATE") + wfe = WorkflowExecution(domain, wft, "ab1234", child_policy="TERMINATE") + + wfe.domain.should.equal(domain) wfe.workflow_type.should.equal(wft) wfe.child_policy.should.equal("TERMINATE") def test_workflow_execution_creation_child_policy_logic(): + domain = get_basic_domain() + WorkflowExecution( + domain, WorkflowType( "test-workflow", "v1.0", task_list="queue", default_child_policy="ABANDON", @@ -104,6 +121,7 @@ def test_workflow_execution_creation_child_policy_logic(): ).child_policy.should.equal("ABANDON") WorkflowExecution( + domain, WorkflowType( "test-workflow", "v1.0", task_list="queue", default_execution_start_to_close_timeout="300", @@ -114,41 +132,44 @@ def test_workflow_execution_creation_child_policy_logic(): ).child_policy.should.equal("REQUEST_CANCEL") WorkflowExecution.when.called_with( + domain, WorkflowType("test-workflow", "v1.0"), "ab1234" ).should.throw(SWFDefaultUndefinedFault) def test_workflow_execution_string_representation(): - wft = get_basic_workflow_type() - wfe = WorkflowExecution(wft, "ab1234", child_policy="TERMINATE") + wfe = make_workflow_execution(child_policy="TERMINATE") str(wfe).should.match(r"^WorkflowExecution\(run_id: .*\)") def test_workflow_execution_generates_a_random_run_id(): + domain = get_basic_domain() wft = get_basic_workflow_type() - wfe1 = WorkflowExecution(wft, "ab1234", child_policy="TERMINATE") - wfe2 = WorkflowExecution(wft, "ab1235", child_policy="TERMINATE") + wfe1 = WorkflowExecution(domain, wft, "ab1234", child_policy="TERMINATE") + wfe2 = WorkflowExecution(domain, wft, "ab1235", child_policy="TERMINATE") wfe1.run_id.should_not.equal(wfe2.run_id) def test_workflow_execution_short_dict_representation(): + domain = get_basic_domain() wf_type = WorkflowType( "test-workflow", "v1.0", task_list="queue", default_child_policy="ABANDON", default_execution_start_to_close_timeout="300", default_task_start_to_close_timeout="300", ) - wfe = WorkflowExecution(wf_type, "ab1234") + wfe = WorkflowExecution(domain, wf_type, "ab1234") sd = wfe.to_short_dict() sd["workflowId"].should.equal("ab1234") sd.should.contain("runId") def test_workflow_execution_medium_dict_representation(): + domain = get_basic_domain() wf_type = WorkflowType( "test-workflow", "v1.0", task_list="queue", default_child_policy="ABANDON", default_execution_start_to_close_timeout="300", default_task_start_to_close_timeout="300", ) - wfe = WorkflowExecution(wf_type, "ab1234") + wfe = WorkflowExecution(domain, wf_type, "ab1234") md = wfe.to_medium_dict() md["execution"].should.equal(wfe.to_short_dict()) @@ -163,13 +184,14 @@ def test_workflow_execution_medium_dict_representation(): md["tagList"].should.equal(["foo", "bar", "baz"]) def test_workflow_execution_full_dict_representation(): + domain = get_basic_domain() wf_type = WorkflowType( "test-workflow", "v1.0", task_list="queue", default_child_policy="ABANDON", default_execution_start_to_close_timeout="300", default_task_start_to_close_timeout="300", ) - wfe = WorkflowExecution(wf_type, "ab1234") + wfe = WorkflowExecution(domain, wf_type, "ab1234") fd = wfe.to_full_dict() fd["executionInfo"].should.equal(wfe.to_medium_dict()) @@ -184,15 +206,13 @@ def test_workflow_execution_full_dict_representation(): }) def test_workflow_execution_schedule_decision_task(): - wft = get_basic_workflow_type() - wfe = WorkflowExecution(wft, "ab1234") + wfe = make_workflow_execution() wfe.open_counts["openDecisionTasks"].should.equal(0) wfe.schedule_decision_task() wfe.open_counts["openDecisionTasks"].should.equal(1) def test_workflow_execution_start_decision_task(): - wft = get_basic_workflow_type() - wfe = WorkflowExecution(wft, "ab1234") + wfe = make_workflow_execution() wfe.schedule_decision_task() dt = wfe.decision_tasks[0] wfe.start_decision_task(dt.task_token, identity="srv01") @@ -202,8 +222,7 @@ def test_workflow_execution_start_decision_task(): wfe.events()[-1].identity.should.equal("srv01") def test_workflow_execution_history_events_ids(): - wft = get_basic_workflow_type() - wfe = WorkflowExecution(wft, "ab1234") + wfe = make_workflow_execution() wfe._add_event("WorkflowExecutionStarted", workflow_execution=wfe) wfe._add_event("DecisionTaskScheduled", workflow_execution=wfe) wfe._add_event("DecisionTaskStarted", workflow_execution=wfe, scheduled_event_id=2) @@ -212,8 +231,7 @@ def test_workflow_execution_history_events_ids(): @freeze_time("2015-01-01 12:00:00") def test_workflow_execution_start(): - wft = get_basic_workflow_type() - wfe = WorkflowExecution(wft, "ab1234") + wfe = make_workflow_execution() wfe.events().should.equal([]) wfe.start() @@ -224,8 +242,7 @@ def test_workflow_execution_start(): @freeze_time("2015-01-02 12:00:00") def test_workflow_execution_complete(): - wft = get_basic_workflow_type() - wfe = WorkflowExecution(wft, "ab1234") + wfe = make_workflow_execution() wfe.complete(123, result="foo") wfe.execution_status.should.equal("CLOSED") @@ -237,8 +254,7 @@ def test_workflow_execution_complete(): @freeze_time("2015-01-02 12:00:00") def test_workflow_execution_fail(): - wft = get_basic_workflow_type() - wfe = WorkflowExecution(wft, "ab1234") + wfe = make_workflow_execution() wfe.fail(123, details="some details", reason="my rules") wfe.execution_status.should.equal("CLOSED") @@ -278,8 +294,7 @@ def test_history_event_breaks_on_initialization_if_not_implemented(): # DecisionTask def test_decision_task_creation(): - wft = get_basic_workflow_type() - wfe = WorkflowExecution(wft, "ab1234") + wfe = make_workflow_execution() dt = DecisionTask(wfe, 123) dt.workflow_execution.should.equal(wfe) dt.state.should.equal("SCHEDULED") @@ -287,8 +302,8 @@ def test_decision_task_creation(): dt.started_event_id.should.be.none def test_decision_task_full_dict_representation(): - wft = get_basic_workflow_type() - wfe = WorkflowExecution(wft, "ab1234") + wfe = make_workflow_execution() + wft = wfe.workflow_type dt = DecisionTask(wfe, 123) fd = dt.to_full_dict() @@ -306,8 +321,7 @@ def test_decision_task_full_dict_representation(): # ActivityTask def test_activity_task_creation(): - wft = get_basic_workflow_type() - wfe = WorkflowExecution(wft, "ab1234") + wfe = make_workflow_execution() task = ActivityTask( activity_id="my-activity-123", activity_type="foo", diff --git a/tests/test_swf/utils.py b/tests/test_swf/utils.py index f106e2e02..5c93fe79b 100644 --- a/tests/test_swf/utils.py +++ b/tests/test_swf/utils.py @@ -1,8 +1,14 @@ from moto.swf.models import ( + Domain, WorkflowType, ) +# A test Domain +def get_basic_domain(): + return Domain("test-domain", "90") + + # A generic test WorkflowType def _generic_workflow_type_attributes(): return [ From 5e086223c28996e9753f8b5f77a7daf9b5b9a4c4 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 26 Oct 2015 00:43:35 +0100 Subject: [PATCH 46/94] Implement ScheduleActivityTask decision --- moto/swf/models/domain.py | 4 + moto/swf/models/generic_type.py | 11 +- moto/swf/models/history_event.py | 24 ++++ moto/swf/models/workflow_execution.py | 101 +++++++++++++- tests/test_swf/test_decision_tasks.py | 55 +++++++- tests/test_swf/test_models.py | 181 ++++++++++++++++++++++++++ 6 files changed, 371 insertions(+), 5 deletions(-) diff --git a/moto/swf/models/domain.py b/moto/swf/models/domain.py index 4b30d3932..4ed914528 100644 --- a/moto/swf/models/domain.py +++ b/moto/swf/models/domain.py @@ -23,6 +23,7 @@ class Domain(object): # of "workflow_id (client determined)" => WorkflowExecution() # here. self.workflow_executions = {} + self.task_lists = defaultdict(list) def __repr__(self): return "Domain(name: %(name)s, status: %(status)s)" % self.__dict__ @@ -83,3 +84,6 @@ class Domain(object): ) ) return wfe + + def add_to_task_list(self, task_list, obj): + self.task_lists[task_list].append(obj) diff --git a/moto/swf/models/generic_type.py b/moto/swf/models/generic_type.py index 334382ecd..904296ab3 100644 --- a/moto/swf/models/generic_type.py +++ b/moto/swf/models/generic_type.py @@ -12,6 +12,13 @@ class GenericType(object): self.description = kwargs.pop("description") for key, value in kwargs.iteritems(): self.__setattr__(key, value) + # default values set to none + for key in self._configuration_keys: + attr = camelcase_to_underscores(key) + if not hasattr(self, attr): + self.__setattr__(attr, None) + if not hasattr(self, "task_list"): + self.task_list = None def __repr__(self): cls = self.__class__.__name__ @@ -49,12 +56,10 @@ class GenericType(object): "typeInfo": self.to_medium_dict(), "configuration": {} } - if hasattr(self, "task_list"): + if self.task_list: hsh["configuration"]["defaultTaskList"] = {"name": self.task_list} for key in self._configuration_keys: attr = camelcase_to_underscores(key) - if not hasattr(self, attr): - continue if not getattr(self, attr): continue hsh["configuration"][key] = getattr(self, attr) diff --git a/moto/swf/models/history_event.py b/moto/swf/models/history_event.py index 0cb8e83eb..eca00fb0b 100644 --- a/moto/swf/models/history_event.py +++ b/moto/swf/models/history_event.py @@ -81,6 +81,30 @@ class HistoryEvent(object): if hasattr(self, "reason") and self.reason: hsh["reason"] = self.reason return hsh + elif self.event_type == "ActivityTaskScheduled": + hsh = { + "activityId": self.attributes["activityId"], + "activityType": self.activity_type.to_short_dict(), + "decisionTaskCompletedEventId": self.decision_task_completed_event_id, + "taskList": { + "name": self.task_list, + }, + } + for attr in ["control", "heartbeatTimeout", "input", "scheduleToCloseTimeout", + "scheduleToStartTimeout", "startToCloseTimeout", "taskPriority"]: + if self.attributes.get(attr): + hsh[attr] = self.attributes[attr] + return hsh + elif self.event_type == "ScheduleActivityTaskFailed": + # TODO: implement other possible failure mode: OPEN_ACTIVITIES_LIMIT_EXCEEDED + # NB: some failure modes are not implemented and probably won't be implemented in the + # future, such as ACTIVITY_CREATION_RATE_EXCEEDED or OPERATION_NOT_PERMITTED + return { + "activityId": self.activity_id, + "activityType": self.activity_type.to_short_dict(), + "cause": self.cause, + "decisionTaskCompletedEventId": self.decision_task_completed_event_id, + } else: raise NotImplementedError( "HistoryEvent does not implement attributes for type '{}'".format(self.event_type) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 5445f9a71..df75a99b8 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -14,6 +14,8 @@ from ..exceptions import ( SWFDecisionValidationException, ) from ..utils import decapitalize +from .activity_task import ActivityTask +from .activity_type import ActivityType from .decision_task import DecisionTask from .history_event import HistoryEvent @@ -209,7 +211,10 @@ class WorkflowExecution(object): execution_context=execution_context, ) dt.complete() + self.should_schedule_decision_next = False self.handle_decisions(evt.event_id, decisions) + if self.should_schedule_decision_next: + self.schedule_decision_task() def _check_decision_attributes(self, kind, value, decision_id): problems = [] @@ -293,6 +298,8 @@ class WorkflowExecution(object): self.complete(event_id, attributes.get("result")) elif decision_type == "FailWorkflowExecution": self.fail(event_id, attributes.get("details"), attributes.get("reason")) + elif decision_type == "ScheduleActivityTask": + self.schedule_activity_task(event_id, attributes) else: # TODO: implement Decision type: CancelTimer # TODO: implement Decision type: CancelWorkflowExecution @@ -300,7 +307,6 @@ class WorkflowExecution(object): # TODO: implement Decision type: RecordMarker # TODO: implement Decision type: RequestCancelActivityTask # TODO: implement Decision type: RequestCancelExternalWorkflowExecution - # TODO: implement Decision type: ScheduleActivityTask # TODO: implement Decision type: ScheduleLambdaFunction # TODO: implement Decision type: SignalExternalWorkflowExecution # TODO: implement Decision type: StartChildWorkflowExecution @@ -331,3 +337,96 @@ class WorkflowExecution(object): details=details, reason=reason, ) + + def schedule_activity_task(self, event_id, attributes): + activity_type = self.domain.get_type( + "activity", + attributes["activityType"]["name"], + attributes["activityType"]["version"], + ignore_empty=True, + ) + if not activity_type: + fake_type = ActivityType(attributes["activityType"]["name"], + attributes["activityType"]["version"]) + self._add_event( + "ScheduleActivityTaskFailed", + activity_id=attributes["activityId"], + activity_type=fake_type, + cause="ACTIVITY_TYPE_DOES_NOT_EXIST", + decision_task_completed_event_id=event_id, + ) + self.should_schedule_decision_next = True + return + if activity_type.status == "DEPRECATED": + self._add_event( + "ScheduleActivityTaskFailed", + activity_id=attributes["activityId"], + activity_type=activity_type, + cause="ACTIVITY_TYPE_DEPRECATED", + decision_task_completed_event_id=event_id, + ) + self.should_schedule_decision_next = True + return + if any(at for at in self.activity_tasks + if at.activity_id == attributes["activityId"]): + self._add_event( + "ScheduleActivityTaskFailed", + activity_id=attributes["activityId"], + activity_type=activity_type, + cause="ACTIVITY_ID_ALREADY_IN_USE", + decision_task_completed_event_id=event_id, + ) + self.should_schedule_decision_next = True + return + + # find task list or default task list, else fail + task_list = attributes.get("taskList", {}).get("name") + if not task_list and activity_type.task_list: + task_list = activity_type.task_list + if not task_list: + self._add_event( + "ScheduleActivityTaskFailed", + activity_id=attributes["activityId"], + activity_type=activity_type, + cause="DEFAULT_TASK_LIST_UNDEFINED", + decision_task_completed_event_id=event_id, + ) + self.should_schedule_decision_next = True + return + + # find timeouts or default timeout, else fail + timeouts = {} + for _type in ["scheduleToStartTimeout", "scheduleToCloseTimeout", "startToCloseTimeout", "heartbeatTimeout"]: + default_key = "default_task_"+camelcase_to_underscores(_type) + default_value = getattr(activity_type, default_key) + timeouts[_type] = attributes.get(_type, default_value) + if not timeouts[_type]: + error_key = default_key.replace("default_task_", "default_") + self._add_event( + "ScheduleActivityTaskFailed", + activity_id=attributes["activityId"], + activity_type=activity_type, + cause="{}_UNDEFINED".format(error_key.upper()), + decision_task_completed_event_id=event_id, + ) + self.should_schedule_decision_next = True + return + + task = ActivityTask( + activity_id=attributes["activityId"], + activity_type=activity_type, + input=attributes.get("input"), + workflow_execution=self, + ) + # Only add event and increment counters if nothing went wrong + # TODO: don't store activity tasks in 2 places... + self.activity_tasks.append(task) + self.domain.add_to_task_list(task_list, task) + self._add_event( + "ActivityTaskScheduled", + decision_task_completed_event_id=event_id, + activity_type=activity_type, + attributes=attributes, + task_list=task_list, + ) + self.open_counts["openActivityTasks"] += 1 diff --git a/tests/test_swf/test_decision_tasks.py b/tests/test_swf/test_decision_tasks.py index 568d68c2d..ecf59223f 100644 --- a/tests/test_swf/test_decision_tasks.py +++ b/tests/test_swf/test_decision_tasks.py @@ -17,7 +17,13 @@ def setup_workflow(): conn = boto.connect_swf("the_key", "the_secret") conn.register_domain("test-domain", "60", description="A test domain") conn = mock_basic_workflow_type("test-domain", conn) - conn.register_activity_type("test-domain", "test-activity", "v1.1") + conn.register_activity_type( + "test-domain", "test-activity", "v1.1", + default_task_heartbeat_timeout="600", + default_task_schedule_to_close_timeout="600", + default_task_schedule_to_start_timeout="600", + default_task_start_to_close_timeout="600", + ) wfe = conn.start_workflow_execution("test-domain", "uid-abcd1234", "test-workflow", "v1.0") conn.run_id = wfe["runId"] return conn @@ -266,3 +272,50 @@ def test_respond_decision_task_completed_with_fail_workflow_execution(): attrs = resp["events"][-1]["workflowExecutionFailedEventAttributes"] attrs["reason"].should.equal("my rules") attrs["details"].should.equal("foo") + +@mock_swf +def test_respond_decision_task_completed_with_schedule_activity_task(): + conn = setup_workflow() + resp = conn.poll_for_decision_task("test-domain", "queue") + task_token = resp["taskToken"] + + decisions = [{ + "decisionType": "ScheduleActivityTask", + "scheduleActivityTaskDecisionAttributes": { + "activityId": "my-activity-001", + "activityType": { + "name": "test-activity", + "version": "v1.1" + }, + "heartbeatTimeout": "60", + "input": "123", + "taskList": { + "name": "my-task-list" + }, + } + }] + resp = conn.respond_decision_task_completed(task_token, decisions=decisions) + resp.should.be.none + + resp = conn.get_workflow_execution_history("test-domain", conn.run_id, "uid-abcd1234") + types = [evt["eventType"] for evt in resp["events"]] + types.should.equal([ + "WorkflowExecutionStarted", + "DecisionTaskScheduled", + "DecisionTaskStarted", + "DecisionTaskCompleted", + "ActivityTaskScheduled", + ]) + resp["events"][-1]["activityTaskScheduledEventAttributes"].should.equal({ + "decisionTaskCompletedEventId": 4, + "activityId": "my-activity-001", + "activityType": { + "name": "test-activity", + "version": "v1.1", + }, + "heartbeatTimeout": "60", + "input": "123", + "taskList": { + "name": "my-task-list" + }, + }) diff --git a/tests/test_swf/test_models.py b/tests/test_swf/test_models.py index 45e541e75..a298139e8 100644 --- a/tests/test_swf/test_models.py +++ b/tests/test_swf/test_models.py @@ -3,6 +3,7 @@ from freezegun import freeze_time from moto.swf.models import ( ActivityTask, + ActivityType, DecisionTask, Domain, GenericType, @@ -24,6 +25,7 @@ from .utils import ( # TODO: move them in utils def make_workflow_execution(**kwargs): domain = get_basic_domain() + domain.add_type(ActivityType("test-activity", "v1.1")) wft = get_basic_workflow_type() return WorkflowExecution(domain, wft, "ab1234", **kwargs) @@ -47,6 +49,13 @@ def test_domain_string_representation(): domain = Domain("my-domain", "60") str(domain).should.equal("Domain(name: my-domain, status: REGISTERED)") +def test_domain_add_to_task_list(): + domain = Domain("my-domain", "60") + domain.add_to_task_list("foo", "bar") + dict(domain.task_lists).should.equal({ + "foo": ["bar"] + }) + # GenericType (ActivityType, WorkflowType) class FooType(GenericType): @@ -265,6 +274,178 @@ def test_workflow_execution_fail(): wfe.events()[-1].details.should.equal("some details") wfe.events()[-1].reason.should.equal("my rules") +def test_workflow_execution_schedule_activity_task(): + wfe = make_workflow_execution() + wfe.schedule_activity_task(123, { + "activityId": "my-activity-001", + "activityType": { "name": "test-activity", "version": "v1.1" }, + "taskList": { "name": "task-list-name" }, + "scheduleToStartTimeout": "600", + "scheduleToCloseTimeout": "600", + "startToCloseTimeout": "600", + "heartbeatTimeout": "300", + }) + + wfe.open_counts["openActivityTasks"].should.equal(1) + last_event = wfe.events()[-1] + last_event.event_type.should.equal("ActivityTaskScheduled") + last_event.decision_task_completed_event_id.should.equal(123) + last_event.task_list.should.equal("task-list-name") + + wfe.activity_tasks.should.have.length_of(1) + task = wfe.activity_tasks[0] + task.activity_id.should.equal("my-activity-001") + task.activity_type.name.should.equal("test-activity") + wfe.domain.task_lists["task-list-name"].should.contain(task) + +def test_workflow_execution_schedule_activity_task_without_task_list_should_take_default(): + wfe = make_workflow_execution() + wfe.domain.add_type( + ActivityType("test-activity", "v1.2", task_list="foobar") + ) + wfe.schedule_activity_task(123, { + "activityId": "my-activity-001", + "activityType": { "name": "test-activity", "version": "v1.2" }, + "scheduleToStartTimeout": "600", + "scheduleToCloseTimeout": "600", + "startToCloseTimeout": "600", + "heartbeatTimeout": "300", + }) + + wfe.open_counts["openActivityTasks"].should.equal(1) + last_event = wfe.events()[-1] + last_event.event_type.should.equal("ActivityTaskScheduled") + last_event.task_list.should.equal("foobar") + + task = wfe.activity_tasks[0] + wfe.domain.task_lists["foobar"].should.contain(task) + +def test_workflow_execution_schedule_activity_task_should_fail_if_wrong_attributes(): + wfe = make_workflow_execution() + at = ActivityType("test-activity", "v1.1") + at.status = "DEPRECATED" + wfe.domain.add_type(at) + wfe.domain.add_type(ActivityType("test-activity", "v1.2")) + + hsh = { + "activityId": "my-activity-001", + "activityType": { "name": "test-activity-does-not-exists", "version": "v1.1" }, + } + + wfe.schedule_activity_task(123, hsh) + last_event = wfe.events()[-1] + last_event.event_type.should.equal("ScheduleActivityTaskFailed") + last_event.cause.should.equal("ACTIVITY_TYPE_DOES_NOT_EXIST") + + hsh["activityType"]["name"] = "test-activity" + wfe.schedule_activity_task(123, hsh) + last_event = wfe.events()[-1] + last_event.event_type.should.equal("ScheduleActivityTaskFailed") + last_event.cause.should.equal("ACTIVITY_TYPE_DEPRECATED") + + hsh["activityType"]["version"] = "v1.2" + wfe.schedule_activity_task(123, hsh) + last_event = wfe.events()[-1] + last_event.event_type.should.equal("ScheduleActivityTaskFailed") + last_event.cause.should.equal("DEFAULT_TASK_LIST_UNDEFINED") + + hsh["taskList"] = { "name": "foobar" } + wfe.schedule_activity_task(123, hsh) + last_event = wfe.events()[-1] + last_event.event_type.should.equal("ScheduleActivityTaskFailed") + last_event.cause.should.equal("DEFAULT_SCHEDULE_TO_START_TIMEOUT_UNDEFINED") + + hsh["scheduleToStartTimeout"] = "600" + wfe.schedule_activity_task(123, hsh) + last_event = wfe.events()[-1] + last_event.event_type.should.equal("ScheduleActivityTaskFailed") + last_event.cause.should.equal("DEFAULT_SCHEDULE_TO_CLOSE_TIMEOUT_UNDEFINED") + + hsh["scheduleToCloseTimeout"] = "600" + wfe.schedule_activity_task(123, hsh) + last_event = wfe.events()[-1] + last_event.event_type.should.equal("ScheduleActivityTaskFailed") + last_event.cause.should.equal("DEFAULT_START_TO_CLOSE_TIMEOUT_UNDEFINED") + + hsh["startToCloseTimeout"] = "600" + wfe.schedule_activity_task(123, hsh) + last_event = wfe.events()[-1] + last_event.event_type.should.equal("ScheduleActivityTaskFailed") + last_event.cause.should.equal("DEFAULT_HEARTBEAT_TIMEOUT_UNDEFINED") + + wfe.open_counts["openActivityTasks"].should.equal(0) + wfe.activity_tasks.should.have.length_of(0) + wfe.domain.task_lists.should.have.length_of(0) + + hsh["heartbeatTimeout"] = "300" + wfe.schedule_activity_task(123, hsh) + last_event = wfe.events()[-1] + last_event.event_type.should.equal("ActivityTaskScheduled") + + task = wfe.activity_tasks[0] + wfe.domain.task_lists["foobar"].should.contain(task) + wfe.open_counts["openDecisionTasks"].should.equal(0) + wfe.open_counts["openActivityTasks"].should.equal(1) + +def test_workflow_execution_schedule_activity_task_failure_triggers_new_decision(): + wfe = make_workflow_execution() + wfe.start() + task_token = wfe.decision_tasks[-1].task_token + wfe.start_decision_task(task_token) + wfe.complete_decision_task(task_token, decisions=[ + { + "decisionType": "ScheduleActivityTask", + "scheduleActivityTaskDecisionAttributes": { + "activityId": "my-activity-001", + "activityType": { "name": "test-activity-does-not-exist", "version": "v1.2" }, + } + }, + { + "decisionType": "ScheduleActivityTask", + "scheduleActivityTaskDecisionAttributes": { + "activityId": "my-activity-001", + "activityType": { "name": "test-activity-does-not-exist", "version": "v1.2" }, + } + }, + ]) + + wfe.open_counts["openActivityTasks"].should.equal(0) + wfe.open_counts["openDecisionTasks"].should.equal(1) + last_events = wfe.events()[-3:] + last_events[0].event_type.should.equal("ScheduleActivityTaskFailed") + last_events[1].event_type.should.equal("ScheduleActivityTaskFailed") + last_events[2].event_type.should.equal("DecisionTaskScheduled") + +def test_workflow_execution_schedule_activity_task_with_same_activity_id(): + wfe = make_workflow_execution() + + wfe.schedule_activity_task(123, { + "activityId": "my-activity-001", + "activityType": { "name": "test-activity", "version": "v1.1" }, + "taskList": { "name": "task-list-name" }, + "scheduleToStartTimeout": "600", + "scheduleToCloseTimeout": "600", + "startToCloseTimeout": "600", + "heartbeatTimeout": "300", + }) + wfe.open_counts["openActivityTasks"].should.equal(1) + last_event = wfe.events()[-1] + last_event.event_type.should.equal("ActivityTaskScheduled") + + wfe.schedule_activity_task(123, { + "activityId": "my-activity-001", + "activityType": { "name": "test-activity", "version": "v1.1" }, + "taskList": { "name": "task-list-name" }, + "scheduleToStartTimeout": "600", + "scheduleToCloseTimeout": "600", + "startToCloseTimeout": "600", + "heartbeatTimeout": "300", + }) + wfe.open_counts["openActivityTasks"].should.equal(1) + last_event = wfe.events()[-1] + last_event.event_type.should.equal("ScheduleActivityTaskFailed") + last_event.cause.should.equal("ACTIVITY_ID_ALREADY_IN_USE") + # HistoryEvent @freeze_time("2015-01-01 12:00:00") From a71300588288764e9321230b639494a5293a3eda Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 26 Oct 2015 05:50:15 +0100 Subject: [PATCH 47/94] Simplify implementation of ScheduleActivityTask decision --- moto/swf/models/workflow_execution.py | 74 +++++++++------------------ 1 file changed, 25 insertions(+), 49 deletions(-) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index df75a99b8..b0e6427cd 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -200,6 +200,9 @@ class WorkflowExecution(object): dt.start(evt.event_id) def complete_decision_task(self, task_token, decisions=None, execution_context=None): + # 'decisions' can be None per boto.swf defaults, so replace it with something iterable + if not decisions: + decisions = [] # In case of a malformed or invalid decision task, SWF will raise an error and # it won't perform any of the decisions in the decision set. self.validate_decisions(decisions) @@ -236,9 +239,6 @@ class WorkflowExecution(object): validation problem, such as a malformed decision for instance. I didn't find an explicit documentation for that though, so criticisms welcome. """ - if not decisions: - return - problems = [] # check close decision is last @@ -285,10 +285,6 @@ class WorkflowExecution(object): Handles a Decision according to SWF docs. See: http://docs.aws.amazon.com/amazonswf/latest/apireference/API_Decision.html """ - # 'decisions' can be None per boto.swf defaults, so replace it with something iterable - if not decisions: - decisions = [] - # handle each decision separately, in order for decision in decisions: decision_type = decision["decisionType"] @@ -339,6 +335,17 @@ class WorkflowExecution(object): ) def schedule_activity_task(self, event_id, attributes): + # Helper function to avoid repeating ourselves in the next sections + def fail_schedule_activity_task(_type, _cause): + self._add_event( + "ScheduleActivityTaskFailed", + activity_id=attributes["activityId"], + activity_type=_type, + cause=_cause, + decision_task_completed_event_id=event_id, + ) + self.should_schedule_decision_next = True + activity_type = self.domain.get_type( "activity", attributes["activityType"]["name"], @@ -348,35 +355,16 @@ class WorkflowExecution(object): if not activity_type: fake_type = ActivityType(attributes["activityType"]["name"], attributes["activityType"]["version"]) - self._add_event( - "ScheduleActivityTaskFailed", - activity_id=attributes["activityId"], - activity_type=fake_type, - cause="ACTIVITY_TYPE_DOES_NOT_EXIST", - decision_task_completed_event_id=event_id, - ) - self.should_schedule_decision_next = True + fail_schedule_activity_task(fake_type, + "ACTIVITY_TYPE_DOES_NOT_EXIST") return if activity_type.status == "DEPRECATED": - self._add_event( - "ScheduleActivityTaskFailed", - activity_id=attributes["activityId"], - activity_type=activity_type, - cause="ACTIVITY_TYPE_DEPRECATED", - decision_task_completed_event_id=event_id, - ) - self.should_schedule_decision_next = True + fail_schedule_activity_task(activity_type, + "ACTIVITY_TYPE_DEPRECATED") return - if any(at for at in self.activity_tasks - if at.activity_id == attributes["activityId"]): - self._add_event( - "ScheduleActivityTaskFailed", - activity_id=attributes["activityId"], - activity_type=activity_type, - cause="ACTIVITY_ID_ALREADY_IN_USE", - decision_task_completed_event_id=event_id, - ) - self.should_schedule_decision_next = True + if any(at for at in self.activity_tasks if at.activity_id == attributes["activityId"]): + fail_schedule_activity_task(activity_type, + "ACTIVITY_ID_ALREADY_IN_USE") return # find task list or default task list, else fail @@ -384,14 +372,8 @@ class WorkflowExecution(object): if not task_list and activity_type.task_list: task_list = activity_type.task_list if not task_list: - self._add_event( - "ScheduleActivityTaskFailed", - activity_id=attributes["activityId"], - activity_type=activity_type, - cause="DEFAULT_TASK_LIST_UNDEFINED", - decision_task_completed_event_id=event_id, - ) - self.should_schedule_decision_next = True + fail_schedule_activity_task(activity_type, + "DEFAULT_TASK_LIST_UNDEFINED") return # find timeouts or default timeout, else fail @@ -402,14 +384,8 @@ class WorkflowExecution(object): timeouts[_type] = attributes.get(_type, default_value) if not timeouts[_type]: error_key = default_key.replace("default_task_", "default_") - self._add_event( - "ScheduleActivityTaskFailed", - activity_id=attributes["activityId"], - activity_type=activity_type, - cause="{}_UNDEFINED".format(error_key.upper()), - decision_task_completed_event_id=event_id, - ) - self.should_schedule_decision_next = True + fail_schedule_activity_task(activity_type, + "{}_UNDEFINED".format(error_key.upper())) return task = ActivityTask( From eadc07bf61caf9daeb22eccc5c95441a1e8235ec Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 26 Oct 2015 06:31:00 +0100 Subject: [PATCH 48/94] Reorganize SWF tests so they're shorter and easier to use --- tests/test_swf/models/__init__.py | 0 tests/test_swf/models/test_activity_task.py | 26 +++ tests/test_swf/models/test_decision_task.py | 31 ++++ tests/test_swf/models/test_domain.py | 29 +++ tests/test_swf/models/test_generic_type.py | 51 ++++++ tests/test_swf/models/test_history_event.py | 29 +++ .../test_workflow_execution.py} | 168 +----------------- tests/test_swf/responses/__init__.py | 0 .../{ => responses}/test_activity_types.py | 0 .../{ => responses}/test_decision_tasks.py | 2 +- .../test_swf/{ => responses}/test_domains.py | 0 .../test_workflow_executions.py | 0 .../{ => responses}/test_workflow_types.py | 0 tests/test_swf/utils.py | 12 +- 14 files changed, 180 insertions(+), 168 deletions(-) create mode 100644 tests/test_swf/models/__init__.py create mode 100644 tests/test_swf/models/test_activity_task.py create mode 100644 tests/test_swf/models/test_decision_task.py create mode 100644 tests/test_swf/models/test_domain.py create mode 100644 tests/test_swf/models/test_generic_type.py create mode 100644 tests/test_swf/models/test_history_event.py rename tests/test_swf/{test_models.py => models/test_workflow_execution.py} (71%) create mode 100644 tests/test_swf/responses/__init__.py rename tests/test_swf/{ => responses}/test_activity_types.py (100%) rename tests/test_swf/{ => responses}/test_decision_tasks.py (99%) rename tests/test_swf/{ => responses}/test_domains.py (100%) rename tests/test_swf/{ => responses}/test_workflow_executions.py (100%) rename tests/test_swf/{ => responses}/test_workflow_types.py (100%) diff --git a/tests/test_swf/models/__init__.py b/tests/test_swf/models/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_swf/models/test_activity_task.py b/tests/test_swf/models/test_activity_task.py new file mode 100644 index 000000000..2e2bba2f6 --- /dev/null +++ b/tests/test_swf/models/test_activity_task.py @@ -0,0 +1,26 @@ +from sure import expect + +from moto.swf.models import ActivityTask + +from ..utils import make_workflow_execution + + +def test_activity_task_creation(): + wfe = make_workflow_execution() + task = ActivityTask( + activity_id="my-activity-123", + activity_type="foo", + input="optional", + workflow_execution=wfe, + ) + task.workflow_execution.should.equal(wfe) + task.state.should.equal("SCHEDULED") + task.task_token.should_not.be.empty + task.started_event_id.should.be.none + + task.start(123) + task.state.should.equal("STARTED") + task.started_event_id.should.equal(123) + + task.complete() + task.state.should.equal("COMPLETED") diff --git a/tests/test_swf/models/test_decision_task.py b/tests/test_swf/models/test_decision_task.py new file mode 100644 index 000000000..f1156a19e --- /dev/null +++ b/tests/test_swf/models/test_decision_task.py @@ -0,0 +1,31 @@ +from sure import expect + +from moto.swf.models import DecisionTask + +from ..utils import make_workflow_execution + + +def test_decision_task_creation(): + wfe = make_workflow_execution() + dt = DecisionTask(wfe, 123) + dt.workflow_execution.should.equal(wfe) + dt.state.should.equal("SCHEDULED") + dt.task_token.should_not.be.empty + dt.started_event_id.should.be.none + +def test_decision_task_full_dict_representation(): + wfe = make_workflow_execution() + wft = wfe.workflow_type + dt = DecisionTask(wfe, 123) + + fd = dt.to_full_dict() + fd["events"].should.be.a("list") + fd["previousStartedEventId"].should.equal(0) + fd.should_not.contain("startedEventId") + fd.should.contain("taskToken") + fd["workflowExecution"].should.equal(wfe.to_short_dict()) + fd["workflowType"].should.equal(wft.to_short_dict()) + + dt.start(1234) + fd = dt.to_full_dict() + fd["startedEventId"].should.equal(1234) diff --git a/tests/test_swf/models/test_domain.py b/tests/test_swf/models/test_domain.py new file mode 100644 index 000000000..68e3f8903 --- /dev/null +++ b/tests/test_swf/models/test_domain.py @@ -0,0 +1,29 @@ +from sure import expect + +from moto.swf.models import Domain + + +def test_domain_short_dict_representation(): + domain = Domain("foo", "52") + domain.to_short_dict().should.equal({"name":"foo", "status":"REGISTERED"}) + + domain.description = "foo bar" + domain.to_short_dict()["description"].should.equal("foo bar") + +def test_domain_full_dict_representation(): + domain = Domain("foo", "52") + + domain.to_full_dict()["domainInfo"].should.equal(domain.to_short_dict()) + _config = domain.to_full_dict()["configuration"] + _config["workflowExecutionRetentionPeriodInDays"].should.equal("52") + +def test_domain_string_representation(): + domain = Domain("my-domain", "60") + str(domain).should.equal("Domain(name: my-domain, status: REGISTERED)") + +def test_domain_add_to_task_list(): + domain = Domain("my-domain", "60") + domain.add_to_task_list("foo", "bar") + dict(domain.task_lists).should.equal({ + "foo": ["bar"] + }) diff --git a/tests/test_swf/models/test_generic_type.py b/tests/test_swf/models/test_generic_type.py new file mode 100644 index 000000000..8937836e5 --- /dev/null +++ b/tests/test_swf/models/test_generic_type.py @@ -0,0 +1,51 @@ +from sure import expect + +from moto.swf.models import GenericType + + +# Tests for GenericType (ActivityType, WorkflowType) +class FooType(GenericType): + @property + def kind(self): + return "foo" + + @property + def _configuration_keys(self): + return ["justAnExampleTimeout"] + + +def test_type_short_dict_representation(): + _type = FooType("test-foo", "v1.0") + _type.to_short_dict().should.equal({"name": "test-foo", "version": "v1.0"}) + +def test_type_medium_dict_representation(): + _type = FooType("test-foo", "v1.0") + _type.to_medium_dict()["fooType"].should.equal(_type.to_short_dict()) + _type.to_medium_dict()["status"].should.equal("REGISTERED") + _type.to_medium_dict().should.contain("creationDate") + _type.to_medium_dict().should_not.contain("deprecationDate") + _type.to_medium_dict().should_not.contain("description") + + _type.description = "foo bar" + _type.to_medium_dict()["description"].should.equal("foo bar") + + _type.status = "DEPRECATED" + _type.to_medium_dict().should.contain("deprecationDate") + +def test_type_full_dict_representation(): + _type = FooType("test-foo", "v1.0") + _type.to_full_dict()["typeInfo"].should.equal(_type.to_medium_dict()) + _type.to_full_dict()["configuration"].should.equal({}) + + _type.task_list = "foo" + _type.to_full_dict()["configuration"]["defaultTaskList"].should.equal({"name":"foo"}) + + _type.just_an_example_timeout = "60" + _type.to_full_dict()["configuration"]["justAnExampleTimeout"].should.equal("60") + + _type.non_whitelisted_property = "34" + _type.to_full_dict()["configuration"].keys().should.equal(["defaultTaskList", "justAnExampleTimeout"]) + +def test_type_string_representation(): + _type = FooType("test-foo", "v1.0") + str(_type).should.equal("FooType(name: test-foo, version: v1.0, status: REGISTERED)") diff --git a/tests/test_swf/models/test_history_event.py b/tests/test_swf/models/test_history_event.py new file mode 100644 index 000000000..89d73210c --- /dev/null +++ b/tests/test_swf/models/test_history_event.py @@ -0,0 +1,29 @@ +from sure import expect +from freezegun import freeze_time + +from moto.swf.models import HistoryEvent + + +@freeze_time("2015-01-01 12:00:00") +def test_history_event_creation(): + he = HistoryEvent(123, "DecisionTaskStarted", scheduled_event_id=2) + he.event_id.should.equal(123) + he.event_type.should.equal("DecisionTaskStarted") + he.event_timestamp.should.equal(1420110000.0) + +@freeze_time("2015-01-01 12:00:00") +def test_history_event_to_dict_representation(): + he = HistoryEvent(123, "DecisionTaskStarted", scheduled_event_id=2) + he.to_dict().should.equal({ + "eventId": 123, + "eventType": "DecisionTaskStarted", + "eventTimestamp": 1420110000.0, + "decisionTaskStartedEventAttributes": { + "scheduledEventId": 2 + } + }) + +def test_history_event_breaks_on_initialization_if_not_implemented(): + HistoryEvent.when.called_with( + 123, "UnknownHistoryEvent" + ).should.throw(NotImplementedError) diff --git a/tests/test_swf/test_models.py b/tests/test_swf/models/test_workflow_execution.py similarity index 71% rename from tests/test_swf/test_models.py rename to tests/test_swf/models/test_workflow_execution.py index a298139e8..b793d8d2c 100644 --- a/tests/test_swf/test_models.py +++ b/tests/test_swf/models/test_workflow_execution.py @@ -2,12 +2,7 @@ from sure import expect from freezegun import freeze_time from moto.swf.models import ( - ActivityTask, ActivityType, - DecisionTask, - Domain, - GenericType, - HistoryEvent, WorkflowType, WorkflowExecution, ) @@ -15,97 +10,13 @@ from moto.swf.exceptions import ( SWFDefaultUndefinedFault, ) -from .utils import ( +from ..utils import ( get_basic_domain, get_basic_workflow_type, + make_workflow_execution, ) -# Some utility methods -# TODO: move them in utils -def make_workflow_execution(**kwargs): - domain = get_basic_domain() - domain.add_type(ActivityType("test-activity", "v1.1")) - wft = get_basic_workflow_type() - return WorkflowExecution(domain, wft, "ab1234", **kwargs) - - -# Domain -def test_domain_short_dict_representation(): - domain = Domain("foo", "52") - domain.to_short_dict().should.equal({"name":"foo", "status":"REGISTERED"}) - - domain.description = "foo bar" - domain.to_short_dict()["description"].should.equal("foo bar") - -def test_domain_full_dict_representation(): - domain = Domain("foo", "52") - - domain.to_full_dict()["domainInfo"].should.equal(domain.to_short_dict()) - _config = domain.to_full_dict()["configuration"] - _config["workflowExecutionRetentionPeriodInDays"].should.equal("52") - -def test_domain_string_representation(): - domain = Domain("my-domain", "60") - str(domain).should.equal("Domain(name: my-domain, status: REGISTERED)") - -def test_domain_add_to_task_list(): - domain = Domain("my-domain", "60") - domain.add_to_task_list("foo", "bar") - dict(domain.task_lists).should.equal({ - "foo": ["bar"] - }) - - -# GenericType (ActivityType, WorkflowType) -class FooType(GenericType): - @property - def kind(self): - return "foo" - - @property - def _configuration_keys(self): - return ["justAnExampleTimeout"] - - -def test_type_short_dict_representation(): - _type = FooType("test-foo", "v1.0") - _type.to_short_dict().should.equal({"name": "test-foo", "version": "v1.0"}) - -def test_type_medium_dict_representation(): - _type = FooType("test-foo", "v1.0") - _type.to_medium_dict()["fooType"].should.equal(_type.to_short_dict()) - _type.to_medium_dict()["status"].should.equal("REGISTERED") - _type.to_medium_dict().should.contain("creationDate") - _type.to_medium_dict().should_not.contain("deprecationDate") - _type.to_medium_dict().should_not.contain("description") - - _type.description = "foo bar" - _type.to_medium_dict()["description"].should.equal("foo bar") - - _type.status = "DEPRECATED" - _type.to_medium_dict().should.contain("deprecationDate") - -def test_type_full_dict_representation(): - _type = FooType("test-foo", "v1.0") - _type.to_full_dict()["typeInfo"].should.equal(_type.to_medium_dict()) - _type.to_full_dict()["configuration"].should.equal({}) - - _type.task_list = "foo" - _type.to_full_dict()["configuration"]["defaultTaskList"].should.equal({"name":"foo"}) - - _type.just_an_example_timeout = "60" - _type.to_full_dict()["configuration"]["justAnExampleTimeout"].should.equal("60") - - _type.non_whitelisted_property = "34" - _type.to_full_dict()["configuration"].keys().should.equal(["defaultTaskList", "justAnExampleTimeout"]) - -def test_type_string_representation(): - _type = FooType("test-foo", "v1.0") - str(_type).should.equal("FooType(name: test-foo, version: v1.0, status: REGISTERED)") - - -# WorkflowExecution def test_workflow_execution_creation(): domain = get_basic_domain() wft = get_basic_workflow_type() @@ -445,78 +356,3 @@ def test_workflow_execution_schedule_activity_task_with_same_activity_id(): last_event = wfe.events()[-1] last_event.event_type.should.equal("ScheduleActivityTaskFailed") last_event.cause.should.equal("ACTIVITY_ID_ALREADY_IN_USE") - - -# HistoryEvent -@freeze_time("2015-01-01 12:00:00") -def test_history_event_creation(): - he = HistoryEvent(123, "DecisionTaskStarted", scheduled_event_id=2) - he.event_id.should.equal(123) - he.event_type.should.equal("DecisionTaskStarted") - he.event_timestamp.should.equal(1420110000.0) - -@freeze_time("2015-01-01 12:00:00") -def test_history_event_to_dict_representation(): - he = HistoryEvent(123, "DecisionTaskStarted", scheduled_event_id=2) - he.to_dict().should.equal({ - "eventId": 123, - "eventType": "DecisionTaskStarted", - "eventTimestamp": 1420110000.0, - "decisionTaskStartedEventAttributes": { - "scheduledEventId": 2 - } - }) - -def test_history_event_breaks_on_initialization_if_not_implemented(): - HistoryEvent.when.called_with( - 123, "UnknownHistoryEvent" - ).should.throw(NotImplementedError) - - -# DecisionTask -def test_decision_task_creation(): - wfe = make_workflow_execution() - dt = DecisionTask(wfe, 123) - dt.workflow_execution.should.equal(wfe) - dt.state.should.equal("SCHEDULED") - dt.task_token.should_not.be.empty - dt.started_event_id.should.be.none - -def test_decision_task_full_dict_representation(): - wfe = make_workflow_execution() - wft = wfe.workflow_type - dt = DecisionTask(wfe, 123) - - fd = dt.to_full_dict() - fd["events"].should.be.a("list") - fd["previousStartedEventId"].should.equal(0) - fd.should_not.contain("startedEventId") - fd.should.contain("taskToken") - fd["workflowExecution"].should.equal(wfe.to_short_dict()) - fd["workflowType"].should.equal(wft.to_short_dict()) - - dt.start(1234) - fd = dt.to_full_dict() - fd["startedEventId"].should.equal(1234) - - -# ActivityTask -def test_activity_task_creation(): - wfe = make_workflow_execution() - task = ActivityTask( - activity_id="my-activity-123", - activity_type="foo", - input="optional", - workflow_execution=wfe, - ) - task.workflow_execution.should.equal(wfe) - task.state.should.equal("SCHEDULED") - task.task_token.should_not.be.empty - task.started_event_id.should.be.none - - task.start(123) - task.state.should.equal("STARTED") - task.started_event_id.should.equal(123) - - task.complete() - task.state.should.equal("COMPLETED") diff --git a/tests/test_swf/responses/__init__.py b/tests/test_swf/responses/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_swf/test_activity_types.py b/tests/test_swf/responses/test_activity_types.py similarity index 100% rename from tests/test_swf/test_activity_types.py rename to tests/test_swf/responses/test_activity_types.py diff --git a/tests/test_swf/test_decision_tasks.py b/tests/test_swf/responses/test_decision_tasks.py similarity index 99% rename from tests/test_swf/test_decision_tasks.py rename to tests/test_swf/responses/test_decision_tasks.py index ecf59223f..f20626ee1 100644 --- a/tests/test_swf/test_decision_tasks.py +++ b/tests/test_swf/responses/test_decision_tasks.py @@ -9,7 +9,7 @@ from moto.swf.exceptions import ( SWFDecisionValidationException, ) -from .utils import mock_basic_workflow_type +from ..utils import mock_basic_workflow_type @mock_swf diff --git a/tests/test_swf/test_domains.py b/tests/test_swf/responses/test_domains.py similarity index 100% rename from tests/test_swf/test_domains.py rename to tests/test_swf/responses/test_domains.py diff --git a/tests/test_swf/test_workflow_executions.py b/tests/test_swf/responses/test_workflow_executions.py similarity index 100% rename from tests/test_swf/test_workflow_executions.py rename to tests/test_swf/responses/test_workflow_executions.py diff --git a/tests/test_swf/test_workflow_types.py b/tests/test_swf/responses/test_workflow_types.py similarity index 100% rename from tests/test_swf/test_workflow_types.py rename to tests/test_swf/responses/test_workflow_types.py diff --git a/tests/test_swf/utils.py b/tests/test_swf/utils.py index 5c93fe79b..55121b319 100644 --- a/tests/test_swf/utils.py +++ b/tests/test_swf/utils.py @@ -1,6 +1,8 @@ from moto.swf.models import ( + ActivityType, Domain, WorkflowType, + WorkflowExecution, ) @@ -9,7 +11,7 @@ def get_basic_domain(): return Domain("test-domain", "90") -# A generic test WorkflowType +# A test WorkflowType def _generic_workflow_type_attributes(): return [ "test-workflow", "v1.0" @@ -28,3 +30,11 @@ def mock_basic_workflow_type(domain_name, conn): args, kwargs = _generic_workflow_type_attributes() conn.register_workflow_type(domain_name, *args, **kwargs) return conn + + +# A test WorkflowExecution +def make_workflow_execution(**kwargs): + domain = get_basic_domain() + domain.add_type(ActivityType("test-activity", "v1.1")) + wft = get_basic_workflow_type() + return WorkflowExecution(domain, wft, "ab1234", **kwargs) From 83c08b76556576dcea397f45f88ec4b50af08a85 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 26 Oct 2015 06:31:58 +0100 Subject: [PATCH 49/94] Remove unused import --- moto/swf/responses.py | 1 - 1 file changed, 1 deletion(-) diff --git a/moto/swf/responses.py b/moto/swf/responses.py index 9000d03f0..334bac217 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -1,5 +1,4 @@ import json -import logging import six from moto.core.responses import BaseResponse From be71909a8c0aa18f1f597c8dc08a423c0fe01336 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 26 Oct 2015 10:55:55 +0100 Subject: [PATCH 50/94] Rework task lists for activity/decision tasks --- moto/swf/models/domain.py | 28 ++++++++++++++++-- moto/swf/models/workflow_execution.py | 29 ++++++++++++++----- tests/test_swf/models/test_domain.py | 25 ++++++++++++++-- .../models/test_workflow_execution.py | 8 ++--- 4 files changed, 72 insertions(+), 18 deletions(-) diff --git a/moto/swf/models/domain.py b/moto/swf/models/domain.py index 4ed914528..d04a41841 100644 --- a/moto/swf/models/domain.py +++ b/moto/swf/models/domain.py @@ -23,7 +23,8 @@ class Domain(object): # of "workflow_id (client determined)" => WorkflowExecution() # here. self.workflow_executions = {} - self.task_lists = defaultdict(list) + self.activity_task_lists = {} + self.decision_task_lists = {} def __repr__(self): return "Domain(name: %(name)s, status: %(status)s)" % self.__dict__ @@ -85,5 +86,26 @@ class Domain(object): ) return wfe - def add_to_task_list(self, task_list, obj): - self.task_lists[task_list].append(obj) + def add_to_activity_task_list(self, task_list, obj): + if not task_list in self.activity_task_lists: + self.activity_task_lists[task_list] = [] + self.activity_task_lists[task_list].append(obj) + + @property + def activity_tasks(self): + _all = [] + for _, tasks in self.activity_task_lists.iteritems(): + _all += tasks + return _all + + def add_to_decision_task_list(self, task_list, obj): + if not task_list in self.decision_task_lists: + self.decision_task_lists[task_list] = [] + self.decision_task_lists[task_list].append(obj) + + @property + def decision_tasks(self): + _all = [] + for _, tasks in self.decision_task_lists.iteritems(): + _all += tasks + return _all diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index b0e6427cd..a9a277254 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -75,9 +75,7 @@ class WorkflowExecution(object): } # events self._events = [] - # tasks - self.decision_tasks = [] - self.activity_tasks = [] + # child workflows self.child_workflow_executions = [] def __repr__(self): @@ -167,12 +165,22 @@ class WorkflowExecution(object): self.schedule_decision_task() def schedule_decision_task(self): - self.open_counts["openDecisionTasks"] += 1 evt = self._add_event( "DecisionTaskScheduled", workflow_execution=self, ) - self.decision_tasks.append(DecisionTask(self, evt.event_id)) + self.domain.add_to_decision_task_list( + self.task_list, + DecisionTask(self, evt.event_id), + ) + self.open_counts["openDecisionTasks"] += 1 + + @property + def decision_tasks(self): + return filter( + lambda t: t.workflow_execution == self, + self.domain.decision_tasks + ) @property def scheduled_decision_tasks(self): @@ -181,6 +189,13 @@ class WorkflowExecution(object): self.decision_tasks ) + @property + def activity_tasks(self): + return filter( + lambda t: t.workflow_execution == self, + self.domain.activity_tasks + ) + def _find_decision_task(self, task_token): for dt in self.decision_tasks: if dt.task_token == task_token: @@ -395,9 +410,7 @@ class WorkflowExecution(object): workflow_execution=self, ) # Only add event and increment counters if nothing went wrong - # TODO: don't store activity tasks in 2 places... - self.activity_tasks.append(task) - self.domain.add_to_task_list(task_list, task) + self.domain.add_to_activity_task_list(task_list, task) self._add_event( "ActivityTaskScheduled", decision_task_completed_event_id=event_id, diff --git a/tests/test_swf/models/test_domain.py b/tests/test_swf/models/test_domain.py index 68e3f8903..5d2982e10 100644 --- a/tests/test_swf/models/test_domain.py +++ b/tests/test_swf/models/test_domain.py @@ -21,9 +21,28 @@ def test_domain_string_representation(): domain = Domain("my-domain", "60") str(domain).should.equal("Domain(name: my-domain, status: REGISTERED)") -def test_domain_add_to_task_list(): +def test_domain_add_to_activity_task_list(): domain = Domain("my-domain", "60") - domain.add_to_task_list("foo", "bar") - dict(domain.task_lists).should.equal({ + domain.add_to_activity_task_list("foo", "bar") + domain.activity_task_lists.should.equal({ "foo": ["bar"] }) + +def test_domain_activity_tasks(): + domain = Domain("my-domain", "60") + domain.add_to_activity_task_list("foo", "bar") + domain.add_to_activity_task_list("other", "baz") + domain.activity_tasks.should.equal(["bar", "baz"]) + +def test_domain_add_to_decision_task_list(): + domain = Domain("my-domain", "60") + domain.add_to_decision_task_list("foo", "bar") + domain.decision_task_lists.should.equal({ + "foo": ["bar"] + }) + +def test_domain_decision_tasks(): + domain = Domain("my-domain", "60") + domain.add_to_decision_task_list("foo", "bar") + domain.add_to_decision_task_list("other", "baz") + domain.decision_tasks.should.equal(["bar", "baz"]) diff --git a/tests/test_swf/models/test_workflow_execution.py b/tests/test_swf/models/test_workflow_execution.py index b793d8d2c..c6660ab91 100644 --- a/tests/test_swf/models/test_workflow_execution.py +++ b/tests/test_swf/models/test_workflow_execution.py @@ -207,7 +207,7 @@ def test_workflow_execution_schedule_activity_task(): task = wfe.activity_tasks[0] task.activity_id.should.equal("my-activity-001") task.activity_type.name.should.equal("test-activity") - wfe.domain.task_lists["task-list-name"].should.contain(task) + wfe.domain.activity_task_lists["task-list-name"].should.contain(task) def test_workflow_execution_schedule_activity_task_without_task_list_should_take_default(): wfe = make_workflow_execution() @@ -229,7 +229,7 @@ def test_workflow_execution_schedule_activity_task_without_task_list_should_take last_event.task_list.should.equal("foobar") task = wfe.activity_tasks[0] - wfe.domain.task_lists["foobar"].should.contain(task) + wfe.domain.activity_task_lists["foobar"].should.contain(task) def test_workflow_execution_schedule_activity_task_should_fail_if_wrong_attributes(): wfe = make_workflow_execution() @@ -286,7 +286,7 @@ def test_workflow_execution_schedule_activity_task_should_fail_if_wrong_attribut wfe.open_counts["openActivityTasks"].should.equal(0) wfe.activity_tasks.should.have.length_of(0) - wfe.domain.task_lists.should.have.length_of(0) + wfe.domain.activity_task_lists.should.have.length_of(0) hsh["heartbeatTimeout"] = "300" wfe.schedule_activity_task(123, hsh) @@ -294,7 +294,7 @@ def test_workflow_execution_schedule_activity_task_should_fail_if_wrong_attribut last_event.event_type.should.equal("ActivityTaskScheduled") task = wfe.activity_tasks[0] - wfe.domain.task_lists["foobar"].should.contain(task) + wfe.domain.activity_task_lists["foobar"].should.contain(task) wfe.open_counts["openDecisionTasks"].should.equal(0) wfe.open_counts["openActivityTasks"].should.equal(1) From a0e484fa6d49df9666a3b02b9595e290e458e77e Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 26 Oct 2015 18:05:45 +0100 Subject: [PATCH 51/94] Move setup_workflow() test function in test utils --- .../test_swf/responses/test_decision_tasks.py | 19 +---------------- tests/test_swf/utils.py | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/tests/test_swf/responses/test_decision_tasks.py b/tests/test_swf/responses/test_decision_tasks.py index f20626ee1..0dfe369ec 100644 --- a/tests/test_swf/responses/test_decision_tasks.py +++ b/tests/test_swf/responses/test_decision_tasks.py @@ -9,24 +9,7 @@ from moto.swf.exceptions import ( SWFDecisionValidationException, ) -from ..utils import mock_basic_workflow_type - - -@mock_swf -def setup_workflow(): - conn = boto.connect_swf("the_key", "the_secret") - conn.register_domain("test-domain", "60", description="A test domain") - conn = mock_basic_workflow_type("test-domain", conn) - conn.register_activity_type( - "test-domain", "test-activity", "v1.1", - default_task_heartbeat_timeout="600", - default_task_schedule_to_close_timeout="600", - default_task_schedule_to_start_timeout="600", - default_task_start_to_close_timeout="600", - ) - wfe = conn.start_workflow_execution("test-domain", "uid-abcd1234", "test-workflow", "v1.0") - conn.run_id = wfe["runId"] - return conn +from ..utils import setup_workflow # PollForDecisionTask endpoint diff --git a/tests/test_swf/utils.py b/tests/test_swf/utils.py index 55121b319..6fcec9d46 100644 --- a/tests/test_swf/utils.py +++ b/tests/test_swf/utils.py @@ -1,3 +1,6 @@ +import boto + +from moto import mock_swf from moto.swf.models import ( ActivityType, Domain, @@ -38,3 +41,21 @@ def make_workflow_execution(**kwargs): domain.add_type(ActivityType("test-activity", "v1.1")) wft = get_basic_workflow_type() return WorkflowExecution(domain, wft, "ab1234", **kwargs) + + +# Setup a complete example workflow and return the connection object +@mock_swf +def setup_workflow(): + conn = boto.connect_swf("the_key", "the_secret") + conn.register_domain("test-domain", "60", description="A test domain") + conn = mock_basic_workflow_type("test-domain", conn) + conn.register_activity_type( + "test-domain", "test-activity", "v1.1", + default_task_heartbeat_timeout="600", + default_task_schedule_to_close_timeout="600", + default_task_schedule_to_start_timeout="600", + default_task_start_to_close_timeout="600", + ) + wfe = conn.start_workflow_execution("test-domain", "uid-abcd1234", "test-workflow", "v1.0") + conn.run_id = wfe["runId"] + return conn From d650f71d9cd4fb5597193b0a15101029b70d9bca Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 26 Oct 2015 18:06:04 +0100 Subject: [PATCH 52/94] Simplify decision task handling in SWF backend --- moto/swf/models/__init__.py | 18 +++++++++--------- moto/swf/models/workflow_execution.py | 7 ------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index 2c040f8db..9fff96ebb 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -180,16 +180,16 @@ class SWFBackend(BaseBackend): # aren't distributed. # # TODO: handle long polling (case 2) for decision tasks - decision_candidates = [] - for _, wfe in domain.workflow_executions.iteritems(): - if wfe.task_list == task_list: - decision_candidates += wfe.scheduled_decision_tasks - if any(decision_candidates): + candidates = [] + for _task_list, tasks in domain.decision_task_lists.iteritems(): + if _task_list == task_list: + candidates += filter(lambda t: t.state == "SCHEDULED", tasks) + if any(candidates): # TODO: handle task priorities (but not supported by boto for now) - decision = min(decision_candidates, key=lambda d: d.scheduled_at) - wfe = decision.workflow_execution - wfe.start_decision_task(decision.task_token, identity=identity) - return decision + task = min(candidates, key=lambda d: d.scheduled_at) + wfe = task.workflow_execution + wfe.start_decision_task(task.task_token, identity=identity) + return task else: return None diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index a9a277254..0b1984af2 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -182,13 +182,6 @@ class WorkflowExecution(object): self.domain.decision_tasks ) - @property - def scheduled_decision_tasks(self): - return filter( - lambda t: t.state == "SCHEDULED", - self.decision_tasks - ) - @property def activity_tasks(self): return filter( From 761ab816f96b4709701b31a34ff08a736440b453 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 26 Oct 2015 23:16:59 +0100 Subject: [PATCH 53/94] Add SWF endpoint PollForActivityTask --- moto/swf/models/__init__.py | 31 ++++++++++++ moto/swf/models/activity_task.py | 20 +++++++- moto/swf/models/history_event.py | 8 +++ moto/swf/models/workflow_execution.py | 36 +++++++++---- moto/swf/responses.py | 14 ++++++ tests/test_swf/models/test_activity_task.py | 30 ++++++++++- .../models/test_workflow_execution.py | 50 +++++++++---------- .../test_swf/responses/test_activity_tasks.py | 46 +++++++++++++++++ 8 files changed, 197 insertions(+), 38 deletions(-) create mode 100644 tests/test_swf/responses/test_activity_tasks.py diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index 9fff96ebb..c8cb7916d 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -256,6 +256,37 @@ class SWFBackend(BaseBackend): decisions=decisions, execution_context=execution_context) + def poll_for_activity_task(self, domain_name, task_list, identity=None): + self._check_string(domain_name) + self._check_string(task_list) + domain = self._get_domain(domain_name) + # Real SWF cases: + # - case 1: there's an activity task to return, return it + # - case 2: there's no activity task to return, so wait for timeout + # and if a new activity is scheduled, return it + # - case 3: timeout reached, no activity task, return an empty response + # (e.g. a response with an empty "taskToken") + # + # For the sake of simplicity, we forget case 2 for now, so either + # there's an ActivityTask to return, either we return a blank one. + # + # SWF client libraries should cope with that easily as long as tests + # aren't distributed. + # + # TODO: handle long polling (case 2) for activity tasks + candidates = [] + for _task_list, tasks in domain.activity_task_lists.iteritems(): + if _task_list == task_list: + candidates += filter(lambda t: t.state == "SCHEDULED", tasks) + if any(candidates): + # TODO: handle task priorities (but not supported by boto for now) + task = min(candidates, key=lambda d: d.scheduled_at) + wfe = task.workflow_execution + wfe.start_activity_task(task.task_token, identity=identity) + return task + else: + return None + swf_backends = {} for region in boto.swf.regions(): diff --git a/moto/swf/models/activity_task.py b/moto/swf/models/activity_task.py index 298984a21..c7b68d9cc 100644 --- a/moto/swf/models/activity_task.py +++ b/moto/swf/models/activity_task.py @@ -1,16 +1,34 @@ from __future__ import unicode_literals +from datetime import datetime import uuid class ActivityTask(object): - def __init__(self, activity_id, activity_type, workflow_execution, input=None): + def __init__(self, activity_id, activity_type, scheduled_event_id, + workflow_execution, input=None): self.activity_id = activity_id self.activity_type = activity_type self.input = input + self.scheduled_event_id = scheduled_event_id self.started_event_id = None self.state = "SCHEDULED" self.task_token = str(uuid.uuid4()) self.workflow_execution = workflow_execution + # this is *not* necessarily coherent with workflow execution history, + # but that shouldn't be a problem for tests + self.scheduled_at = datetime.now() + + def to_full_dict(self): + hsh = { + "activityId": self.activity_id, + "activityType": self.activity_type.to_short_dict(), + "taskToken": self.task_token, + "startedEventId": self.started_event_id, + "workflowExecution": self.workflow_execution.to_short_dict(), + } + if self.input: + hsh["input"] = self.input + return hsh def start(self, started_event_id): self.state = "STARTED" diff --git a/moto/swf/models/history_event.py b/moto/swf/models/history_event.py index eca00fb0b..45a839038 100644 --- a/moto/swf/models/history_event.py +++ b/moto/swf/models/history_event.py @@ -105,6 +105,14 @@ class HistoryEvent(object): "cause": self.cause, "decisionTaskCompletedEventId": self.decision_task_completed_event_id, } + elif self.event_type == "ActivityTaskStarted": + # TODO: merge it with DecisionTaskStarted + hsh = { + "scheduledEventId": self.scheduled_event_id + } + if hasattr(self, "identity") and self.identity: + hsh["identity"] = self.identity + return hsh else: raise NotImplementedError( "HistoryEvent does not implement attributes for type '{}'".format(self.event_type) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 0b1984af2..dfa7380f9 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -396,19 +396,37 @@ class WorkflowExecution(object): "{}_UNDEFINED".format(error_key.upper())) return - task = ActivityTask( - activity_id=attributes["activityId"], - activity_type=activity_type, - input=attributes.get("input"), - workflow_execution=self, - ) - # Only add event and increment counters if nothing went wrong - self.domain.add_to_activity_task_list(task_list, task) - self._add_event( + # Only add event and increment counters now that nothing went wrong + evt = self._add_event( "ActivityTaskScheduled", decision_task_completed_event_id=event_id, activity_type=activity_type, attributes=attributes, task_list=task_list, ) + task = ActivityTask( + activity_id=attributes["activityId"], + activity_type=activity_type, + input=attributes.get("input"), + scheduled_event_id=evt.event_id, + workflow_execution=self, + ) + self.domain.add_to_activity_task_list(task_list, task) self.open_counts["openActivityTasks"] += 1 + + def _find_activity_task(self, task_token): + for task in self.activity_tasks: + if task.task_token == task_token: + return task + raise ValueError( + "No activity task with token: {}".format(task_token) + ) + + def start_activity_task(self, task_token, identity=None): + task = self._find_activity_task(task_token) + evt = self._add_event( + "ActivityTaskStarted", + scheduled_event_id=task.scheduled_event_id, + identity=identity + ) + task.start(evt.event_id) diff --git a/moto/swf/responses.py b/moto/swf/responses.py index 334bac217..66493aef6 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -251,3 +251,17 @@ class SWFResponse(BaseResponse): task_token, decisions=decisions, execution_context=execution_context ) return "" + + def poll_for_activity_task(self): + domain_name = self._params["domain"] + task_list = self._params["taskList"]["name"] + identity = self._params.get("identity") + activity_task = self.swf_backend.poll_for_activity_task( + domain_name, task_list, identity=identity + ) + if activity_task: + return json.dumps( + activity_task.to_full_dict() + ) + else: + return json.dumps({"startedEventId": 0}) diff --git a/tests/test_swf/models/test_activity_task.py b/tests/test_swf/models/test_activity_task.py index 2e2bba2f6..d691cc054 100644 --- a/tests/test_swf/models/test_activity_task.py +++ b/tests/test_swf/models/test_activity_task.py @@ -1,6 +1,9 @@ from sure import expect -from moto.swf.models import ActivityTask +from moto.swf.models import ( + ActivityTask, + ActivityType, +) from ..utils import make_workflow_execution @@ -11,6 +14,7 @@ def test_activity_task_creation(): activity_id="my-activity-123", activity_type="foo", input="optional", + scheduled_event_id=117, workflow_execution=wfe, ) task.workflow_execution.should.equal(wfe) @@ -24,3 +28,27 @@ def test_activity_task_creation(): task.complete() task.state.should.equal("COMPLETED") + +def test_activity_task_full_dict_representation(): + wfe = make_workflow_execution() + wft = wfe.workflow_type + at = ActivityTask( + activity_id="my-activity-123", + activity_type=ActivityType("foo", "v1.0"), + input="optional", + scheduled_event_id=117, + workflow_execution=wfe, + ) + at.start(1234) + + fd = at.to_full_dict() + fd["activityId"].should.equal("my-activity-123") + fd["activityType"]["version"].should.equal("v1.0") + fd["input"].should.equal("optional") + fd["startedEventId"].should.equal(1234) + fd.should.contain("taskToken") + fd["workflowExecution"].should.equal(wfe.to_short_dict()) + + at.start(1234) + fd = at.to_full_dict() + fd["startedEventId"].should.equal(1234) diff --git a/tests/test_swf/models/test_workflow_execution.py b/tests/test_swf/models/test_workflow_execution.py index c6660ab91..edacade87 100644 --- a/tests/test_swf/models/test_workflow_execution.py +++ b/tests/test_swf/models/test_workflow_execution.py @@ -17,6 +17,16 @@ from ..utils import ( ) +VALID_ACTIVITY_TASK_ATTRIBUTES = { + "activityId": "my-activity-001", + "activityType": { "name": "test-activity", "version": "v1.1" }, + "taskList": { "name": "task-list-name" }, + "scheduleToStartTimeout": "600", + "scheduleToCloseTimeout": "600", + "startToCloseTimeout": "600", + "heartbeatTimeout": "300", +} + def test_workflow_execution_creation(): domain = get_basic_domain() wft = get_basic_workflow_type() @@ -187,15 +197,7 @@ def test_workflow_execution_fail(): def test_workflow_execution_schedule_activity_task(): wfe = make_workflow_execution() - wfe.schedule_activity_task(123, { - "activityId": "my-activity-001", - "activityType": { "name": "test-activity", "version": "v1.1" }, - "taskList": { "name": "task-list-name" }, - "scheduleToStartTimeout": "600", - "scheduleToCloseTimeout": "600", - "startToCloseTimeout": "600", - "heartbeatTimeout": "300", - }) + wfe.schedule_activity_task(123, VALID_ACTIVITY_TASK_ATTRIBUTES) wfe.open_counts["openActivityTasks"].should.equal(1) last_event = wfe.events()[-1] @@ -330,29 +332,23 @@ def test_workflow_execution_schedule_activity_task_failure_triggers_new_decision def test_workflow_execution_schedule_activity_task_with_same_activity_id(): wfe = make_workflow_execution() - wfe.schedule_activity_task(123, { - "activityId": "my-activity-001", - "activityType": { "name": "test-activity", "version": "v1.1" }, - "taskList": { "name": "task-list-name" }, - "scheduleToStartTimeout": "600", - "scheduleToCloseTimeout": "600", - "startToCloseTimeout": "600", - "heartbeatTimeout": "300", - }) + wfe.schedule_activity_task(123, VALID_ACTIVITY_TASK_ATTRIBUTES) wfe.open_counts["openActivityTasks"].should.equal(1) last_event = wfe.events()[-1] last_event.event_type.should.equal("ActivityTaskScheduled") - wfe.schedule_activity_task(123, { - "activityId": "my-activity-001", - "activityType": { "name": "test-activity", "version": "v1.1" }, - "taskList": { "name": "task-list-name" }, - "scheduleToStartTimeout": "600", - "scheduleToCloseTimeout": "600", - "startToCloseTimeout": "600", - "heartbeatTimeout": "300", - }) + wfe.schedule_activity_task(123, VALID_ACTIVITY_TASK_ATTRIBUTES) wfe.open_counts["openActivityTasks"].should.equal(1) last_event = wfe.events()[-1] last_event.event_type.should.equal("ScheduleActivityTaskFailed") last_event.cause.should.equal("ACTIVITY_ID_ALREADY_IN_USE") + +def test_workflow_execution_start_activity_task(): + wfe = make_workflow_execution() + wfe.schedule_activity_task(123, VALID_ACTIVITY_TASK_ATTRIBUTES) + task_token = wfe.activity_tasks[-1].task_token + wfe.start_activity_task(task_token, identity="worker01") + task = wfe.activity_tasks[-1] + task.state.should.equal("STARTED") + wfe.events()[-1].event_type.should.equal("ActivityTaskStarted") + wfe.events()[-1].identity.should.equal("worker01") diff --git a/tests/test_swf/responses/test_activity_tasks.py b/tests/test_swf/responses/test_activity_tasks.py new file mode 100644 index 000000000..ea95c556a --- /dev/null +++ b/tests/test_swf/responses/test_activity_tasks.py @@ -0,0 +1,46 @@ +import boto +from sure import expect + +from moto import mock_swf +from moto.swf.exceptions import SWFUnknownResourceFault + +from ..utils import setup_workflow + + +# PollForActivityTask endpoint +@mock_swf +def test_poll_for_activity_task_when_one(): + conn = setup_workflow() + decision_token = conn.poll_for_decision_task("test-domain", "queue")["taskToken"] + conn.respond_decision_task_completed(decision_token, decisions=[ + { + "decisionType": "ScheduleActivityTask", + "scheduleActivityTaskDecisionAttributes": { + "activityId": "my-activity-001", + "activityType": { "name": "test-activity", "version": "v1.1" }, + "taskList": { "name": "activity-task-list" }, + } + } + ]) + resp = conn.poll_for_activity_task("test-domain", "activity-task-list", identity="surprise") + resp["activityId"].should.equal("my-activity-001") + resp["taskToken"].should_not.be.none + + resp = conn.get_workflow_execution_history("test-domain", conn.run_id, "uid-abcd1234") + resp["events"][-1]["eventType"].should.equal("ActivityTaskStarted") + resp["events"][-1]["activityTaskStartedEventAttributes"].should.equal( + { "identity": "surprise", "scheduledEventId": 5 } + ) + +@mock_swf +def test_poll_for_activity_task_when_none(): + conn = setup_workflow() + resp = conn.poll_for_activity_task("test-domain", "activity-task-list") + resp.should.equal({"startedEventId": 0}) + +@mock_swf +def test_poll_for_activity_task_on_non_existent_queue(): + conn = setup_workflow() + resp = conn.poll_for_activity_task("test-domain", "non-existent-queue") + resp.should.equal({"startedEventId": 0}) + From 08643945dfda5f4e6a5ce7bb3a15a7983d33bf64 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 26 Oct 2015 23:32:36 +0100 Subject: [PATCH 54/94] Add SWF endpoint CountPendingActivityTasks --- moto/swf/models/__init__.py | 11 ++++++++ moto/swf/responses.py | 7 +++++- .../test_swf/responses/test_activity_tasks.py | 25 +++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index c8cb7916d..19ec6eb43 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -287,6 +287,17 @@ class SWFBackend(BaseBackend): else: return None + def count_pending_activity_tasks(self, domain_name, task_list): + self._check_string(domain_name) + self._check_string(task_list) + domain = self._get_domain(domain_name) + count = 0 + for _task_list, tasks in domain.activity_task_lists.iteritems(): + if _task_list == task_list: + pending = [t for t in tasks if t.state in ["SCHEDULED", "STARTED"]] + count += len(pending) + return count + swf_backends = {} for region in boto.swf.regions(): diff --git a/moto/swf/responses.py b/moto/swf/responses.py index 66493aef6..890b23576 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -242,7 +242,6 @@ class SWFResponse(BaseResponse): count = self.swf_backend.count_pending_decision_tasks(domain_name, task_list) return json.dumps({"count": count, "truncated": False}) - def respond_decision_task_completed(self): task_token = self._params["taskToken"] execution_context = self._params.get("executionContext") @@ -265,3 +264,9 @@ class SWFResponse(BaseResponse): ) else: return json.dumps({"startedEventId": 0}) + + def count_pending_activity_tasks(self): + domain_name = self._params["domain"] + task_list = self._params["taskList"]["name"] + count = self.swf_backend.count_pending_activity_tasks(domain_name, task_list) + return json.dumps({"count": count, "truncated": False}) diff --git a/tests/test_swf/responses/test_activity_tasks.py b/tests/test_swf/responses/test_activity_tasks.py index ea95c556a..a4fb86841 100644 --- a/tests/test_swf/responses/test_activity_tasks.py +++ b/tests/test_swf/responses/test_activity_tasks.py @@ -44,3 +44,28 @@ def test_poll_for_activity_task_on_non_existent_queue(): resp = conn.poll_for_activity_task("test-domain", "non-existent-queue") resp.should.equal({"startedEventId": 0}) + +# CountPendingActivityTasks endpoint +@mock_swf +def test_count_pending_activity_tasks(): + conn = setup_workflow() + decision_token = conn.poll_for_decision_task("test-domain", "queue")["taskToken"] + conn.respond_decision_task_completed(decision_token, decisions=[ + { + "decisionType": "ScheduleActivityTask", + "scheduleActivityTaskDecisionAttributes": { + "activityId": "my-activity-001", + "activityType": { "name": "test-activity", "version": "v1.1" }, + "taskList": { "name": "activity-task-list" }, + } + } + ]) + + resp = conn.count_pending_activity_tasks("test-domain", "activity-task-list") + resp.should.equal({"count": 1, "truncated": False}) + +@mock_swf +def test_count_pending_decision_tasks_on_non_existent_task_list(): + conn = setup_workflow() + resp = conn.count_pending_activity_tasks("test-domain", "non-existent") + resp.should.equal({"count": 0, "truncated": False}) From c9e8ad03f85f38332e060458bcbc902b9b5f9ad2 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Tue, 27 Oct 2015 05:17:33 +0100 Subject: [PATCH 55/94] Add SWF endpoint RespondActivityTaskCompleted --- moto/swf/models/__init__.py | 43 ++++++++ moto/swf/models/history_event.py | 8 ++ moto/swf/models/workflow_execution.py | 13 +++ moto/swf/responses.py | 8 ++ .../models/test_workflow_execution.py | 19 ++++ .../test_swf/responses/test_activity_tasks.py | 98 +++++++++++++++---- 6 files changed, 172 insertions(+), 17 deletions(-) diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index 19ec6eb43..f488a24a7 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -298,6 +298,49 @@ class SWFBackend(BaseBackend): count += len(pending) return count + def respond_activity_task_completed(self, task_token, result=None): + self._check_string(task_token) + self._check_none_or_string(result) + # let's find the activity task + activity_task = None + for domain in self.domains: + for _, wfe in domain.workflow_executions.iteritems(): + for task in wfe.activity_tasks: + if task.task_token == task_token: + activity_task = task + # no task found + if not activity_task: + # Same as for decision tasks, we raise an invalid token BOTH for clearly + # wrong SWF tokens and OK tokens but not used correctly. This should not + # be a problem in moto. + raise SWFValidationException("Invalid token") + # activity task found, but WorflowExecution is CLOSED + wfe = activity_task.workflow_execution + if wfe.execution_status != "OPEN": + raise SWFUnknownResourceFault( + "execution", + "WorkflowExecution=[workflowId={}, runId={}]".format( + wfe.workflow_id, wfe.run_id + ) + ) + # activity task found, but already completed + if activity_task.state != "STARTED": + if activity_task.state == "COMPLETED": + raise SWFUnknownResourceFault( + "activity, scheduledEventId = {}".format(activity_task.scheduled_event_id) + ) + else: + raise ValueError( + "This shouldn't happen: you have to PollForActivityTask to get a token, " + "which changes ActivityTask status to 'STARTED' ; then it can only change " + "to 'COMPLETED'. If you didn't hack moto/swf internals, this is probably " + "a bug in moto, please report it, thanks!" + ) + # everything's good + if activity_task: + wfe = activity_task.workflow_execution + wfe.complete_activity_task(activity_task.task_token, result=result) + swf_backends = {} for region in boto.swf.regions(): diff --git a/moto/swf/models/history_event.py b/moto/swf/models/history_event.py index 45a839038..a6507c9f9 100644 --- a/moto/swf/models/history_event.py +++ b/moto/swf/models/history_event.py @@ -113,6 +113,14 @@ class HistoryEvent(object): if hasattr(self, "identity") and self.identity: hsh["identity"] = self.identity return hsh + elif self.event_type == "ActivityTaskCompleted": + hsh = { + "scheduledEventId": self.scheduled_event_id, + "startedEventId": self.started_event_id, + } + if hasattr(self, "result") and self.result is not None: + hsh["result"] = self.result + return hsh else: raise NotImplementedError( "HistoryEvent does not implement attributes for type '{}'".format(self.event_type) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index dfa7380f9..e111a8f4b 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -430,3 +430,16 @@ class WorkflowExecution(object): identity=identity ) task.start(evt.event_id) + + def complete_activity_task(self, task_token, result=None): + task = self._find_activity_task(task_token) + evt = self._add_event( + "ActivityTaskCompleted", + scheduled_event_id=task.scheduled_event_id, + started_event_id=task.started_event_id, + result=result, + ) + task.complete() + self.open_counts["openActivityTasks"] -= 1 + # TODO: ensure we don't schedule multiple decisions at the same time! + self.schedule_decision_task() diff --git a/moto/swf/responses.py b/moto/swf/responses.py index 890b23576..3d180afcd 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -270,3 +270,11 @@ class SWFResponse(BaseResponse): task_list = self._params["taskList"]["name"] count = self.swf_backend.count_pending_activity_tasks(domain_name, task_list) return json.dumps({"count": count, "truncated": False}) + + def respond_activity_task_completed(self): + task_token = self._params["taskToken"] + result = self._params.get("result") + self.swf_backend.respond_activity_task_completed( + task_token, result=result + ) + return "" diff --git a/tests/test_swf/models/test_workflow_execution.py b/tests/test_swf/models/test_workflow_execution.py index edacade87..30aafceb2 100644 --- a/tests/test_swf/models/test_workflow_execution.py +++ b/tests/test_swf/models/test_workflow_execution.py @@ -352,3 +352,22 @@ def test_workflow_execution_start_activity_task(): task.state.should.equal("STARTED") wfe.events()[-1].event_type.should.equal("ActivityTaskStarted") wfe.events()[-1].identity.should.equal("worker01") + +def test_complete_activity_task(): + wfe = make_workflow_execution() + wfe.schedule_activity_task(123, VALID_ACTIVITY_TASK_ATTRIBUTES) + task_token = wfe.activity_tasks[-1].task_token + + wfe.open_counts["openActivityTasks"].should.equal(1) + wfe.open_counts["openDecisionTasks"].should.equal(0) + + wfe.start_activity_task(task_token, identity="worker01") + wfe.complete_activity_task(task_token, result="a superb result") + + task = wfe.activity_tasks[-1] + task.state.should.equal("COMPLETED") + wfe.events()[-2].event_type.should.equal("ActivityTaskCompleted") + wfe.events()[-1].event_type.should.equal("DecisionTaskScheduled") + + wfe.open_counts["openActivityTasks"].should.equal(0) + wfe.open_counts["openDecisionTasks"].should.equal(1) diff --git a/tests/test_swf/responses/test_activity_tasks.py b/tests/test_swf/responses/test_activity_tasks.py index a4fb86841..f5b053f6d 100644 --- a/tests/test_swf/responses/test_activity_tasks.py +++ b/tests/test_swf/responses/test_activity_tasks.py @@ -2,25 +2,31 @@ import boto from sure import expect from moto import mock_swf -from moto.swf.exceptions import SWFUnknownResourceFault +from moto.swf import swf_backend +from moto.swf.exceptions import ( + SWFValidationException, + SWFUnknownResourceFault, +) from ..utils import setup_workflow +SCHEDULE_ACTIVITY_TASK_DECISION = { + "decisionType": "ScheduleActivityTask", + "scheduleActivityTaskDecisionAttributes": { + "activityId": "my-activity-001", + "activityType": { "name": "test-activity", "version": "v1.1" }, + "taskList": { "name": "activity-task-list" }, + } +} + # PollForActivityTask endpoint @mock_swf def test_poll_for_activity_task_when_one(): conn = setup_workflow() decision_token = conn.poll_for_decision_task("test-domain", "queue")["taskToken"] conn.respond_decision_task_completed(decision_token, decisions=[ - { - "decisionType": "ScheduleActivityTask", - "scheduleActivityTaskDecisionAttributes": { - "activityId": "my-activity-001", - "activityType": { "name": "test-activity", "version": "v1.1" }, - "taskList": { "name": "activity-task-list" }, - } - } + SCHEDULE_ACTIVITY_TASK_DECISION ]) resp = conn.poll_for_activity_task("test-domain", "activity-task-list", identity="surprise") resp["activityId"].should.equal("my-activity-001") @@ -51,14 +57,7 @@ def test_count_pending_activity_tasks(): conn = setup_workflow() decision_token = conn.poll_for_decision_task("test-domain", "queue")["taskToken"] conn.respond_decision_task_completed(decision_token, decisions=[ - { - "decisionType": "ScheduleActivityTask", - "scheduleActivityTaskDecisionAttributes": { - "activityId": "my-activity-001", - "activityType": { "name": "test-activity", "version": "v1.1" }, - "taskList": { "name": "activity-task-list" }, - } - } + SCHEDULE_ACTIVITY_TASK_DECISION ]) resp = conn.count_pending_activity_tasks("test-domain", "activity-task-list") @@ -69,3 +68,68 @@ def test_count_pending_decision_tasks_on_non_existent_task_list(): conn = setup_workflow() resp = conn.count_pending_activity_tasks("test-domain", "non-existent") resp.should.equal({"count": 0, "truncated": False}) + + +# RespondActivityTaskCompleted endpoint +@mock_swf +def test_poll_for_activity_task_when_one(): + conn = setup_workflow() + decision_token = conn.poll_for_decision_task("test-domain", "queue")["taskToken"] + conn.respond_decision_task_completed(decision_token, decisions=[ + SCHEDULE_ACTIVITY_TASK_DECISION + ]) + activity_token = conn.poll_for_activity_task("test-domain", "activity-task-list")["taskToken"] + + resp = conn.respond_activity_task_completed(activity_token, result="result of the task") + resp.should.be.none + + resp = conn.get_workflow_execution_history("test-domain", conn.run_id, "uid-abcd1234") + resp["events"][-2]["eventType"].should.equal("ActivityTaskCompleted") + resp["events"][-2]["activityTaskCompletedEventAttributes"].should.equal( + { "result": "result of the task", "scheduledEventId": 5, "startedEventId": 6 } + ) + +@mock_swf +def test_respond_activity_task_completed_with_wrong_token(): + conn = setup_workflow() + decision_token = conn.poll_for_decision_task("test-domain", "queue")["taskToken"] + conn.respond_decision_task_completed(decision_token, decisions=[ + SCHEDULE_ACTIVITY_TASK_DECISION + ]) + conn.poll_for_activity_task("test-domain", "activity-task-list") + conn.respond_activity_task_completed.when.called_with( + "not-a-correct-token" + ).should.throw(SWFValidationException, "Invalid token") + +@mock_swf +def test_respond_activity_task_completed_on_closed_workflow_execution(): + conn = setup_workflow() + decision_token = conn.poll_for_decision_task("test-domain", "queue")["taskToken"] + conn.respond_decision_task_completed(decision_token, decisions=[ + SCHEDULE_ACTIVITY_TASK_DECISION + ]) + activity_token = conn.poll_for_activity_task("test-domain", "activity-task-list")["taskToken"] + + # bad: we're closing workflow execution manually, but endpoints are not coded for now.. + wfe = swf_backend.domains[0].workflow_executions.values()[0] + wfe.execution_status = "CLOSED" + # /bad + + conn.respond_activity_task_completed.when.called_with( + activity_token + ).should.throw(SWFUnknownResourceFault, "WorkflowExecution=") + +@mock_swf +def test_respond_activity_task_completed_with_task_already_completed(): + conn = setup_workflow() + decision_token = conn.poll_for_decision_task("test-domain", "queue")["taskToken"] + conn.respond_decision_task_completed(decision_token, decisions=[ + SCHEDULE_ACTIVITY_TASK_DECISION + ]) + activity_token = conn.poll_for_activity_task("test-domain", "activity-task-list")["taskToken"] + + conn.respond_activity_task_completed(activity_token) + + conn.respond_activity_task_completed.when.called_with( + activity_token + ).should.throw(SWFUnknownResourceFault, "Unknown activity, scheduledEventId = 5") From fd12e317f86d9615769e1eb0686b3bafa0be8dff Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Wed, 28 Oct 2015 12:29:57 +0100 Subject: [PATCH 56/94] Add SWF endpoint RespondActivityTaskFailed --- moto/swf/models/__init__.py | 25 ++++++++---- moto/swf/models/activity_task.py | 3 ++ moto/swf/models/history_event.py | 11 +++++ moto/swf/models/workflow_execution.py | 14 +++++++ moto/swf/responses.py | 9 +++++ tests/test_swf/models/test_activity_task.py | 6 +++ .../test_swf/responses/test_activity_tasks.py | 40 ++++++++++++++++++- 7 files changed, 100 insertions(+), 8 deletions(-) diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index f488a24a7..a206e3e78 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -298,10 +298,7 @@ class SWFBackend(BaseBackend): count += len(pending) return count - def respond_activity_task_completed(self, task_token, result=None): - self._check_string(task_token) - self._check_none_or_string(result) - # let's find the activity task + def _find_activity_task_from_token(self, task_token): activity_task = None for domain in self.domains: for _, wfe in domain.workflow_executions.iteritems(): @@ -337,9 +334,23 @@ class SWFBackend(BaseBackend): "a bug in moto, please report it, thanks!" ) # everything's good - if activity_task: - wfe = activity_task.workflow_execution - wfe.complete_activity_task(activity_task.task_token, result=result) + return activity_task + + def respond_activity_task_completed(self, task_token, result=None): + self._check_string(task_token) + self._check_none_or_string(result) + activity_task = self._find_activity_task_from_token(task_token) + wfe = activity_task.workflow_execution + wfe.complete_activity_task(activity_task.task_token, result=result) + + def respond_activity_task_failed(self, task_token, reason=None, details=None): + self._check_string(task_token) + # TODO: implement length limits on reason and details (common pb with client libs) + self._check_none_or_string(reason) + self._check_none_or_string(details) + activity_task = self._find_activity_task_from_token(task_token) + wfe = activity_task.workflow_execution + wfe.fail_activity_task(activity_task.task_token, reason=reason, details=details) swf_backends = {} diff --git a/moto/swf/models/activity_task.py b/moto/swf/models/activity_task.py index c7b68d9cc..6baa01b5d 100644 --- a/moto/swf/models/activity_task.py +++ b/moto/swf/models/activity_task.py @@ -36,3 +36,6 @@ class ActivityTask(object): def complete(self): self.state = "COMPLETED" + + def fail(self): + self.state = "FAILED" diff --git a/moto/swf/models/history_event.py b/moto/swf/models/history_event.py index a6507c9f9..798cc810c 100644 --- a/moto/swf/models/history_event.py +++ b/moto/swf/models/history_event.py @@ -121,6 +121,17 @@ class HistoryEvent(object): if hasattr(self, "result") and self.result is not None: hsh["result"] = self.result return hsh + elif self.event_type == "ActivityTaskFailed": + # TODO: maybe merge it with ActivityTaskCompleted (different optional params tho) + hsh = { + "scheduledEventId": self.scheduled_event_id, + "startedEventId": self.started_event_id, + } + if hasattr(self, "reason") and self.reason is not None: + hsh["reason"] = self.reason + if hasattr(self, "details") and self.details is not None: + hsh["details"] = self.details + return hsh else: raise NotImplementedError( "HistoryEvent does not implement attributes for type '{}'".format(self.event_type) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index e111a8f4b..900025dad 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -443,3 +443,17 @@ class WorkflowExecution(object): self.open_counts["openActivityTasks"] -= 1 # TODO: ensure we don't schedule multiple decisions at the same time! self.schedule_decision_task() + + def fail_activity_task(self, task_token, reason=None, details=None): + task = self._find_activity_task(task_token) + evt = self._add_event( + "ActivityTaskFailed", + scheduled_event_id=task.scheduled_event_id, + started_event_id=task.started_event_id, + reason=reason, + details=details, + ) + task.fail() + self.open_counts["openActivityTasks"] -= 1 + # TODO: ensure we don't schedule multiple decisions at the same time! + self.schedule_decision_task() diff --git a/moto/swf/responses.py b/moto/swf/responses.py index 3d180afcd..c90fd5d8d 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -278,3 +278,12 @@ class SWFResponse(BaseResponse): task_token, result=result ) return "" + + def respond_activity_task_failed(self): + task_token = self._params["taskToken"] + reason = self._params.get("reason") + details = self._params.get("details") + self.swf_backend.respond_activity_task_failed( + task_token, reason=reason, details=details + ) + return "" diff --git a/tests/test_swf/models/test_activity_task.py b/tests/test_swf/models/test_activity_task.py index d691cc054..93c842c8e 100644 --- a/tests/test_swf/models/test_activity_task.py +++ b/tests/test_swf/models/test_activity_task.py @@ -29,6 +29,12 @@ def test_activity_task_creation(): task.complete() task.state.should.equal("COMPLETED") + # NB: this doesn't make any sense for SWF, a task shouldn't go from a + # "COMPLETED" state to a "FAILED" one, but this is an internal state on our + # side and we don't care about invalid state transitions for now. + task.fail() + task.state.should.equal("FAILED") + def test_activity_task_full_dict_representation(): wfe = make_workflow_execution() wft = wfe.workflow_type diff --git a/tests/test_swf/responses/test_activity_tasks.py b/tests/test_swf/responses/test_activity_tasks.py index f5b053f6d..13825f856 100644 --- a/tests/test_swf/responses/test_activity_tasks.py +++ b/tests/test_swf/responses/test_activity_tasks.py @@ -72,7 +72,7 @@ def test_count_pending_decision_tasks_on_non_existent_task_list(): # RespondActivityTaskCompleted endpoint @mock_swf -def test_poll_for_activity_task_when_one(): +def test_respond_activity_task_completed(): conn = setup_workflow() decision_token = conn.poll_for_decision_task("test-domain", "queue")["taskToken"] conn.respond_decision_task_completed(decision_token, decisions=[ @@ -133,3 +133,41 @@ def test_respond_activity_task_completed_with_task_already_completed(): conn.respond_activity_task_completed.when.called_with( activity_token ).should.throw(SWFUnknownResourceFault, "Unknown activity, scheduledEventId = 5") + + +# RespondActivityTaskFailed endpoint +@mock_swf +def test_respond_activity_task_failed(): + conn = setup_workflow() + decision_token = conn.poll_for_decision_task("test-domain", "queue")["taskToken"] + conn.respond_decision_task_completed(decision_token, decisions=[ + SCHEDULE_ACTIVITY_TASK_DECISION + ]) + activity_token = conn.poll_for_activity_task("test-domain", "activity-task-list")["taskToken"] + + resp = conn.respond_activity_task_failed(activity_token, + reason="short reason", + details="long details") + resp.should.be.none + + resp = conn.get_workflow_execution_history("test-domain", conn.run_id, "uid-abcd1234") + resp["events"][-2]["eventType"].should.equal("ActivityTaskFailed") + resp["events"][-2]["activityTaskFailedEventAttributes"].should.equal( + { "reason": "short reason", "details": "long details", + "scheduledEventId": 5, "startedEventId": 6 } + ) + +@mock_swf +def test_respond_activity_task_completed_with_wrong_token(): + # NB: we just test ONE failure case for RespondActivityTaskFailed + # because the safeguards are shared with RespondActivityTaskCompleted, so + # no need to retest everything end-to-end. + conn = setup_workflow() + decision_token = conn.poll_for_decision_task("test-domain", "queue")["taskToken"] + conn.respond_decision_task_completed(decision_token, decisions=[ + SCHEDULE_ACTIVITY_TASK_DECISION + ]) + conn.poll_for_activity_task("test-domain", "activity-task-list") + conn.respond_activity_task_failed.when.called_with( + "not-a-correct-token" + ).should.throw(SWFValidationException, "Invalid token") From 98948a01c88d0a69bb48f4b27a451c75507d19fe Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Sat, 31 Oct 2015 21:13:44 +0100 Subject: [PATCH 57/94] Add missing attributes in DescribeWorkflowExecution responses --- moto/swf/models/workflow_execution.py | 10 ++++++++++ tests/test_swf/models/test_workflow_execution.py | 10 +++++++++- tests/test_swf/responses/test_decision_tasks.py | 14 +++++++++++++- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 900025dad..40efab156 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -53,6 +53,8 @@ class WorkflowExecution(object): self.close_status = None self.close_timestamp = None self.execution_status = "OPEN" + self.latest_activity_task_timestamp = None + self.latest_execution_context = None self.parent = None self.start_timestamp = None self.tag_list = [] # TODO @@ -72,6 +74,7 @@ class WorkflowExecution(object): "openDecisionTasks": 0, "openActivityTasks": 0, "openChildWorkflowExecutions": 0, + "openLambdaFunctions": 0, } # events self._events = [] @@ -135,6 +138,11 @@ class WorkflowExecution(object): hsh["executionConfiguration"][key] = getattr(self, attr) #counters hsh["openCounts"] = self.open_counts + #latest things + if self.latest_execution_context: + hsh["latestExecutionContext"] = self.latest_execution_context + if self.latest_activity_task_timestamp: + hsh["latestActivityTaskTimestamp"] = self.latest_activity_task_timestamp return hsh def events(self, reverse_order=False): @@ -226,6 +234,7 @@ class WorkflowExecution(object): self.handle_decisions(evt.event_id, decisions) if self.should_schedule_decision_next: self.schedule_decision_task() + self.latest_execution_context = execution_context def _check_decision_attributes(self, kind, value, decision_id): problems = [] @@ -413,6 +422,7 @@ class WorkflowExecution(object): ) self.domain.add_to_activity_task_list(task_list, task) self.open_counts["openActivityTasks"] += 1 + self.latest_activity_task_timestamp = self._now_timestamp() def _find_activity_task(self, task_token): for task in self.activity_tasks: diff --git a/tests/test_swf/models/test_workflow_execution.py b/tests/test_swf/models/test_workflow_execution.py index 30aafceb2..21f7c5448 100644 --- a/tests/test_swf/models/test_workflow_execution.py +++ b/tests/test_swf/models/test_workflow_execution.py @@ -195,10 +195,15 @@ def test_workflow_execution_fail(): wfe.events()[-1].details.should.equal("some details") wfe.events()[-1].reason.should.equal("my rules") +@freeze_time("2015-01-01 12:00:00") def test_workflow_execution_schedule_activity_task(): wfe = make_workflow_execution() + wfe.latest_activity_task_timestamp.should.be.none + wfe.schedule_activity_task(123, VALID_ACTIVITY_TASK_ATTRIBUTES) + wfe.latest_activity_task_timestamp.should.equal(1420110000.0) + wfe.open_counts["openActivityTasks"].should.equal(1) last_event = wfe.events()[-1] last_event.event_type.should.equal("ActivityTaskScheduled") @@ -305,7 +310,9 @@ def test_workflow_execution_schedule_activity_task_failure_triggers_new_decision wfe.start() task_token = wfe.decision_tasks[-1].task_token wfe.start_decision_task(task_token) - wfe.complete_decision_task(task_token, decisions=[ + wfe.complete_decision_task(task_token, + execution_context="free-form execution context", + decisions=[ { "decisionType": "ScheduleActivityTask", "scheduleActivityTaskDecisionAttributes": { @@ -322,6 +329,7 @@ def test_workflow_execution_schedule_activity_task_failure_triggers_new_decision }, ]) + wfe.latest_execution_context.should.equal("free-form execution context") wfe.open_counts["openActivityTasks"].should.equal(0) wfe.open_counts["openDecisionTasks"].should.equal(1) last_events = wfe.events()[-3:] diff --git a/tests/test_swf/responses/test_decision_tasks.py b/tests/test_swf/responses/test_decision_tasks.py index 0dfe369ec..64f51d30b 100644 --- a/tests/test_swf/responses/test_decision_tasks.py +++ b/tests/test_swf/responses/test_decision_tasks.py @@ -1,4 +1,5 @@ import boto +from freezegun import freeze_time from sure import expect from moto import mock_swf @@ -83,7 +84,10 @@ def test_respond_decision_task_completed_with_no_decision(): resp = conn.poll_for_decision_task("test-domain", "queue") task_token = resp["taskToken"] - resp = conn.respond_decision_task_completed(task_token) + resp = conn.respond_decision_task_completed( + task_token, + execution_context="free-form context", + ) resp.should.be.none resp = conn.get_workflow_execution_history("test-domain", conn.run_id, "uid-abcd1234") @@ -96,10 +100,14 @@ def test_respond_decision_task_completed_with_no_decision(): ]) evt = resp["events"][-1] evt["decisionTaskCompletedEventAttributes"].should.equal({ + "executionContext": "free-form context", "scheduledEventId": 2, "startedEventId": 3, }) + resp = conn.describe_workflow_execution("test-domain", conn.run_id, "uid-abcd1234") + resp["latestExecutionContext"].should.equal("free-form context") + @mock_swf def test_respond_decision_task_completed_with_wrong_token(): conn = setup_workflow() @@ -257,6 +265,7 @@ def test_respond_decision_task_completed_with_fail_workflow_execution(): attrs["details"].should.equal("foo") @mock_swf +@freeze_time("2015-01-01 12:00:00") def test_respond_decision_task_completed_with_schedule_activity_task(): conn = setup_workflow() resp = conn.poll_for_decision_task("test-domain", "queue") @@ -302,3 +311,6 @@ def test_respond_decision_task_completed_with_schedule_activity_task(): "name": "my-task-list" }, }) + + resp = conn.describe_workflow_execution("test-domain", conn.run_id, "uid-abcd1234") + resp["latestActivityTaskTimestamp"].should.equal(1420110000.0) From 96d6bb056b5cb156980f81ae1bb9c784f3161b25 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Sun, 1 Nov 2015 21:55:07 +0100 Subject: [PATCH 58/94] Add SWF endpoint TerminateWorkflowExecution --- moto/swf/models/__init__.py | 20 ++++-- moto/swf/models/domain.py | 34 ++++++++--- moto/swf/models/history_event.py | 11 ++++ moto/swf/models/workflow_execution.py | 19 ++++++ moto/swf/responses.py | 13 ++++ tests/test_swf/models/test_domain.py | 51 ++++++++++++++++ .../models/test_workflow_execution.py | 15 +++++ .../test_swf/responses/test_activity_tasks.py | 2 +- .../test_swf/responses/test_decision_tasks.py | 2 +- .../responses/test_workflow_executions.py | 61 +++++++++++++++++++ 10 files changed, 212 insertions(+), 16 deletions(-) diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index a206e3e78..27f56a711 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -160,7 +160,7 @@ class SWFBackend(BaseBackend): self._check_string(run_id) self._check_string(workflow_id) domain = self._get_domain(domain_name) - return domain.get_workflow_execution(run_id, workflow_id) + return domain.get_workflow_execution(workflow_id, run_id=run_id) def poll_for_decision_task(self, domain_name, task_list, identity=None): self._check_string(domain_name) @@ -198,7 +198,7 @@ class SWFBackend(BaseBackend): self._check_string(task_list) domain = self._get_domain(domain_name) count = 0 - for _, wfe in domain.workflow_executions.iteritems(): + for wfe in domain.workflow_executions: if wfe.task_list == task_list: count += wfe.open_counts["openDecisionTasks"] return count @@ -211,7 +211,7 @@ class SWFBackend(BaseBackend): # let's find decision task decision_task = None for domain in self.domains: - for _, wfe in domain.workflow_executions.iteritems(): + for wfe in domain.workflow_executions: for dt in wfe.decision_tasks: if dt.task_token == task_token: decision_task = dt @@ -301,7 +301,7 @@ class SWFBackend(BaseBackend): def _find_activity_task_from_token(self, task_token): activity_task = None for domain in self.domains: - for _, wfe in domain.workflow_executions.iteritems(): + for wfe in domain.workflow_executions: for task in wfe.activity_tasks: if task.task_token == task_token: activity_task = task @@ -352,6 +352,18 @@ class SWFBackend(BaseBackend): wfe = activity_task.workflow_execution wfe.fail_activity_task(activity_task.task_token, reason=reason, details=details) + def terminate_workflow_execution(self, domain_name, workflow_id, child_policy=None, + details=None, reason=None, run_id=None): + self._check_string(domain_name) + self._check_string(workflow_id) + self._check_none_or_string(child_policy) + self._check_none_or_string(details) + self._check_none_or_string(reason) + self._check_none_or_string(run_id) + domain = self._get_domain(domain_name) + wfe = domain.get_workflow_execution(workflow_id, run_id=run_id, raise_if_closed=True) + wfe.terminate(child_policy=child_policy, details=details, reason=reason) + swf_backends = {} for region in boto.swf.regions(): diff --git a/moto/swf/models/domain.py b/moto/swf/models/domain.py index d04a41841..9caf307f4 100644 --- a/moto/swf/models/domain.py +++ b/moto/swf/models/domain.py @@ -22,7 +22,7 @@ class Domain(object): # that against SWF API) ; hence the storage method as a dict # of "workflow_id (client determined)" => WorkflowExecution() # here. - self.workflow_executions = {} + self.workflow_executions = [] self.activity_task_lists = {} self.decision_task_lists = {} @@ -71,18 +71,32 @@ class Domain(object): def add_workflow_execution(self, workflow_execution): _id = workflow_execution.workflow_id - if self.workflow_executions.get(_id): + # TODO: handle this better: this should raise ONLY if there's an OPEN wfe with this ID + if any(wfe.workflow_id == _id for wfe in self.workflow_executions): raise SWFWorkflowExecutionAlreadyStartedFault() - self.workflow_executions[_id] = workflow_execution + self.workflow_executions.append(workflow_execution) - def get_workflow_execution(self, run_id, workflow_id): - wfe = self.workflow_executions.get(workflow_id) - if not wfe or wfe.run_id != run_id: - raise SWFUnknownResourceFault( - "execution", - "WorkflowExecution=[workflowId={}, runId={}]".format( - workflow_id, run_id + def get_workflow_execution(self, workflow_id, run_id=None, raise_if_closed=False): + if run_id: + _all = [w for w in self.workflow_executions + if w.workflow_id == workflow_id and w.run_id == run_id] + else: + _all = [w for w in self.workflow_executions + if w.workflow_id == workflow_id and w.execution_status == "OPEN"] + wfe = _all[0] if _all else None + if raise_if_closed and wfe and wfe.execution_status == "CLOSED": + wfe = None + if run_id: + if not wfe or wfe.run_id != run_id: + raise SWFUnknownResourceFault( + "execution", + "WorkflowExecution=[workflowId={}, runId={}]".format( + workflow_id, run_id + ) ) + elif not wfe: + raise SWFUnknownResourceFault( + "execution, workflowId = {}".format(workflow_id) ) return wfe diff --git a/moto/swf/models/history_event.py b/moto/swf/models/history_event.py index 798cc810c..eb3c1f795 100644 --- a/moto/swf/models/history_event.py +++ b/moto/swf/models/history_event.py @@ -132,6 +132,17 @@ class HistoryEvent(object): if hasattr(self, "details") and self.details is not None: hsh["details"] = self.details return hsh + elif self.event_type == "WorkflowExecutionTerminated": + hsh = { + "childPolicy": self.child_policy, + } + if self.cause: + hsh["cause"] = self.cause + if self.details: + hsh["details"] = self.details + if self.reason: + hsh["reason"] = self.reason + return hsh else: raise NotImplementedError( "HistoryEvent does not implement attributes for type '{}'".format(self.event_type) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 40efab156..10d134dca 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -50,6 +50,7 @@ class WorkflowExecution(object): # TODO: check valid values among: # COMPLETED | FAILED | CANCELED | TERMINATED | CONTINUED_AS_NEW | TIMED_OUT # TODO: implement them all + self.close_cause = None self.close_status = None self.close_timestamp = None self.execution_status = "OPEN" @@ -467,3 +468,21 @@ class WorkflowExecution(object): self.open_counts["openActivityTasks"] -= 1 # TODO: ensure we don't schedule multiple decisions at the same time! self.schedule_decision_task() + + def terminate(self, child_policy=None, details=None, reason=None): + # TODO: handle child policy for child workflows here + # TODO: handle cause="CHILD_POLICY_APPLIED" + # Until this, we set cause manually to "OPERATOR_INITIATED" + cause = "OPERATOR_INITIATED" + if not child_policy: + child_policy = self.child_policy + self._add_event( + "WorkflowExecutionTerminated", + cause=cause, + child_policy=child_policy, + details=details, + reason=reason, + ) + self.execution_status = "CLOSED" + self.close_status = "TERMINATED" + self.close_cause = "OPERATOR_INITIATED" diff --git a/moto/swf/responses.py b/moto/swf/responses.py index c90fd5d8d..ffadc73f2 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -287,3 +287,16 @@ class SWFResponse(BaseResponse): task_token, reason=reason, details=details ) return "" + + def terminate_workflow_execution(self): + domain_name = self._params["domain"] + workflow_id = self._params["workflowId"] + child_policy = self._params.get("childPolicy") + details = self._params.get("details") + reason = self._params.get("reason") + run_id = self._params.get("runId") + self.swf_backend.terminate_workflow_execution( + domain_name, workflow_id, child_policy=child_policy, + details=details, reason=reason, run_id=run_id + ) + return "" diff --git a/tests/test_swf/models/test_domain.py b/tests/test_swf/models/test_domain.py index 5d2982e10..215cbace0 100644 --- a/tests/test_swf/models/test_domain.py +++ b/tests/test_swf/models/test_domain.py @@ -1,8 +1,17 @@ +from collections import namedtuple from sure import expect +from moto.swf.exceptions import SWFUnknownResourceFault from moto.swf.models import Domain +# Fake WorkflowExecution for tests purposes +WorkflowExecution = namedtuple( + "WorkflowExecution", + ["workflow_id", "run_id", "execution_status"] +) + + def test_domain_short_dict_representation(): domain = Domain("foo", "52") domain.to_short_dict().should.equal({"name":"foo", "status":"REGISTERED"}) @@ -46,3 +55,45 @@ def test_domain_decision_tasks(): domain.add_to_decision_task_list("foo", "bar") domain.add_to_decision_task_list("other", "baz") domain.decision_tasks.should.equal(["bar", "baz"]) + +def test_domain_get_workflow_execution(): + domain = Domain("my-domain", "60") + + wfe1 = WorkflowExecution(workflow_id="wf-id-1", run_id="run-id-1", execution_status="OPEN") + wfe2 = WorkflowExecution(workflow_id="wf-id-1", run_id="run-id-2", execution_status="CLOSED") + wfe3 = WorkflowExecution(workflow_id="wf-id-2", run_id="run-id-3", execution_status="OPEN") + wfe4 = WorkflowExecution(workflow_id="wf-id-3", run_id="run-id-4", execution_status="CLOSED") + domain.workflow_executions = [wfe1, wfe2, wfe3, wfe4] + + # get workflow execution through workflow_id and run_id + domain.get_workflow_execution("wf-id-1", run_id="run-id-1").should.equal(wfe1) + domain.get_workflow_execution("wf-id-1", run_id="run-id-2").should.equal(wfe2) + domain.get_workflow_execution("wf-id-3", run_id="run-id-4").should.equal(wfe4) + domain.get_workflow_execution.when.called_with( + "wf-id-1", run_id="non-existent" + ).should.throw( + SWFUnknownResourceFault, + "Unknown execution: WorkflowExecution=[workflowId=wf-id-1, runId=non-existent]" + ) + + # get OPEN workflow execution by default if no run_id + domain.get_workflow_execution("wf-id-1").should.equal(wfe1) + domain.get_workflow_execution.when.called_with( + "wf-id-3" + ).should.throw( + SWFUnknownResourceFault, "Unknown execution, workflowId = wf-id-3" + ) + domain.get_workflow_execution.when.called_with( + "wf-id-non-existent" + ).should.throw( + SWFUnknownResourceFault, "Unknown execution, workflowId = wf-id-non-existent" + ) + + # raise_if_closed attribute + domain.get_workflow_execution("wf-id-1", run_id="run-id-1", raise_if_closed=True).should.equal(wfe1) + domain.get_workflow_execution.when.called_with( + "wf-id-3", run_id="run-id-4", raise_if_closed=True + ).should.throw( + SWFUnknownResourceFault, + "Unknown execution: WorkflowExecution=[workflowId=wf-id-3, runId=run-id-4]" + ) diff --git a/tests/test_swf/models/test_workflow_execution.py b/tests/test_swf/models/test_workflow_execution.py index 21f7c5448..ced636969 100644 --- a/tests/test_swf/models/test_workflow_execution.py +++ b/tests/test_swf/models/test_workflow_execution.py @@ -379,3 +379,18 @@ def test_complete_activity_task(): wfe.open_counts["openActivityTasks"].should.equal(0) wfe.open_counts["openDecisionTasks"].should.equal(1) + +def test_terminate(): + wfe = make_workflow_execution() + wfe.schedule_decision_task() + wfe.terminate() + + wfe.execution_status.should.equal("CLOSED") + wfe.close_status.should.equal("TERMINATED") + wfe.close_cause.should.equal("OPERATOR_INITIATED") + wfe.open_counts["openDecisionTasks"].should.equal(1) + + last_event = wfe.events()[-1] + last_event.event_type.should.equal("WorkflowExecutionTerminated") + # take default child_policy if not provided (as here) + last_event.child_policy.should.equal("ABANDON") diff --git a/tests/test_swf/responses/test_activity_tasks.py b/tests/test_swf/responses/test_activity_tasks.py index 13825f856..3c7f82b5e 100644 --- a/tests/test_swf/responses/test_activity_tasks.py +++ b/tests/test_swf/responses/test_activity_tasks.py @@ -111,7 +111,7 @@ def test_respond_activity_task_completed_on_closed_workflow_execution(): activity_token = conn.poll_for_activity_task("test-domain", "activity-task-list")["taskToken"] # bad: we're closing workflow execution manually, but endpoints are not coded for now.. - wfe = swf_backend.domains[0].workflow_executions.values()[0] + wfe = swf_backend.domains[0].workflow_executions[-1] wfe.execution_status = "CLOSED" # /bad diff --git a/tests/test_swf/responses/test_decision_tasks.py b/tests/test_swf/responses/test_decision_tasks.py index 64f51d30b..9510b31fd 100644 --- a/tests/test_swf/responses/test_decision_tasks.py +++ b/tests/test_swf/responses/test_decision_tasks.py @@ -123,7 +123,7 @@ def test_respond_decision_task_completed_on_close_workflow_execution(): task_token = resp["taskToken"] # bad: we're closing workflow execution manually, but endpoints are not coded for now.. - wfe = swf_backend.domains[0].workflow_executions.values()[0] + wfe = swf_backend.domains[0].workflow_executions[-1] wfe.execution_status = "CLOSED" # /bad diff --git a/tests/test_swf/responses/test_workflow_executions.py b/tests/test_swf/responses/test_workflow_executions.py index 1b8d599f9..f4125f77c 100644 --- a/tests/test_swf/responses/test_workflow_executions.py +++ b/tests/test_swf/responses/test_workflow_executions.py @@ -100,3 +100,64 @@ def test_get_workflow_execution_history_on_non_existent_workflow_execution(): conn.get_workflow_execution_history.when.called_with( "test-domain", "wrong-run-id", "wrong-workflow-id" ).should.throw(SWFUnknownResourceFault) + + +# TerminateWorkflowExecution endpoint +@mock_swf +def test_terminate_workflow_execution(): + conn = setup_swf_environment() + run_id = conn.start_workflow_execution( + "test-domain", "uid-abcd1234", "test-workflow", "v1.0" + )["runId"] + + resp = conn.terminate_workflow_execution("test-domain", "uid-abcd1234", + details="some details", + reason="a more complete reason", + run_id=run_id) + resp.should.be.none + + resp = conn.get_workflow_execution_history("test-domain", run_id, "uid-abcd1234") + evt = resp["events"][-1] + evt["eventType"].should.equal("WorkflowExecutionTerminated") + attrs = evt["workflowExecutionTerminatedEventAttributes"] + attrs["details"].should.equal("some details") + attrs["reason"].should.equal("a more complete reason") + attrs["cause"].should.equal("OPERATOR_INITIATED") + +@mock_swf +def test_terminate_workflow_execution_with_wrong_workflow_or_run_id(): + conn = setup_swf_environment() + run_id = conn.start_workflow_execution( + "test-domain", "uid-abcd1234", "test-workflow", "v1.0" + )["runId"] + + # terminate workflow execution + resp = conn.terminate_workflow_execution("test-domain", "uid-abcd1234") + + # already closed, with run_id + conn.terminate_workflow_execution.when.called_with( + "test-domain", "uid-abcd1234", run_id=run_id + ).should.throw( + SWFUnknownResourceFault, "WorkflowExecution=[workflowId=uid-abcd1234, runId=" + ) + + # already closed, without run_id + conn.terminate_workflow_execution.when.called_with( + "test-domain", "uid-abcd1234" + ).should.throw( + SWFUnknownResourceFault, "Unknown execution, workflowId = uid-abcd1234" + ) + + # wrong workflow id + conn.terminate_workflow_execution.when.called_with( + "test-domain", "uid-non-existent" + ).should.throw( + SWFUnknownResourceFault, "Unknown execution, workflowId = uid-non-existent" + ) + + # wrong run_id + conn.terminate_workflow_execution.when.called_with( + "test-domain", "uid-abcd1234", run_id="foo" + ).should.throw( + SWFUnknownResourceFault, "WorkflowExecution=[workflowId=uid-abcd1234, runId=" + ) From 804d2e91b5c669dc6d3888236e0a69b5747b6cde Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Sun, 1 Nov 2015 22:23:15 +0100 Subject: [PATCH 59/94] Improve workflow selection before raising a WorkflowExecutionAlreadyStartedFault --- moto/swf/models/domain.py | 29 ++++++++++++++-------------- tests/test_swf/models/test_domain.py | 3 +++ 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/moto/swf/models/domain.py b/moto/swf/models/domain.py index 9caf307f4..0174661ce 100644 --- a/moto/swf/models/domain.py +++ b/moto/swf/models/domain.py @@ -71,33 +71,32 @@ class Domain(object): def add_workflow_execution(self, workflow_execution): _id = workflow_execution.workflow_id - # TODO: handle this better: this should raise ONLY if there's an OPEN wfe with this ID - if any(wfe.workflow_id == _id for wfe in self.workflow_executions): + if self.get_workflow_execution(_id, raise_if_none=False): raise SWFWorkflowExecutionAlreadyStartedFault() self.workflow_executions.append(workflow_execution) - def get_workflow_execution(self, workflow_id, run_id=None, raise_if_closed=False): + def get_workflow_execution(self, workflow_id, run_id=None, + raise_if_none=True, raise_if_closed=False): + # query if run_id: _all = [w for w in self.workflow_executions if w.workflow_id == workflow_id and w.run_id == run_id] else: _all = [w for w in self.workflow_executions if w.workflow_id == workflow_id and w.execution_status == "OPEN"] + # reduce wfe = _all[0] if _all else None + # raise if closed / none if raise_if_closed and wfe and wfe.execution_status == "CLOSED": wfe = None - if run_id: - if not wfe or wfe.run_id != run_id: - raise SWFUnknownResourceFault( - "execution", - "WorkflowExecution=[workflowId={}, runId={}]".format( - workflow_id, run_id - ) - ) - elif not wfe: - raise SWFUnknownResourceFault( - "execution, workflowId = {}".format(workflow_id) - ) + if not wfe and raise_if_none: + if run_id: + args = ["execution", "WorkflowExecution=[workflowId={}, runId={}]".format( + workflow_id, run_id)] + else: + args = ["execution, workflowId = {}".format(workflow_id)] + raise SWFUnknownResourceFault(*args) + # at last return workflow execution return wfe def add_to_activity_task_list(self, task_list, obj): diff --git a/tests/test_swf/models/test_domain.py b/tests/test_swf/models/test_domain.py index 215cbace0..0efa0029d 100644 --- a/tests/test_swf/models/test_domain.py +++ b/tests/test_swf/models/test_domain.py @@ -97,3 +97,6 @@ def test_domain_get_workflow_execution(): SWFUnknownResourceFault, "Unknown execution: WorkflowExecution=[workflowId=wf-id-3, runId=run-id-4]" ) + + # raise_if_none attribute + domain.get_workflow_execution("foo", raise_if_none=False).should.be.none From f576f3765c03e25dd56516171cbcf9036061c702 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 2 Nov 2015 10:26:40 +0100 Subject: [PATCH 60/94] Add SWF endpoint RecordActivityTaskHeartbeat --- moto/swf/models/__init__.py | 6 ++++ moto/swf/models/activity_task.py | 6 ++++ moto/swf/models/history_event.py | 4 +-- moto/swf/models/workflow_execution.py | 14 ++++------ moto/swf/responses.py | 9 ++++++ moto/swf/utils.py | 7 +++++ tests/test_swf/models/test_activity_task.py | 20 +++++++++++++ .../test_swf/responses/test_activity_tasks.py | 28 +++++++++++++++++++ tests/test_swf/test_utils.py | 12 +++++++- 9 files changed, 94 insertions(+), 12 deletions(-) diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index 27f56a711..37c7827d7 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -364,6 +364,12 @@ class SWFBackend(BaseBackend): wfe = domain.get_workflow_execution(workflow_id, run_id=run_id, raise_if_closed=True) wfe.terminate(child_policy=child_policy, details=details, reason=reason) + def record_activity_task_heartbeat(self, task_token, details=None): + self._check_string(task_token) + self._check_none_or_string(details) + activity_task = self._find_activity_task_from_token(task_token) + activity_task.reset_heartbeat_clock() + swf_backends = {} for region in boto.swf.regions(): diff --git a/moto/swf/models/activity_task.py b/moto/swf/models/activity_task.py index 6baa01b5d..b099bc677 100644 --- a/moto/swf/models/activity_task.py +++ b/moto/swf/models/activity_task.py @@ -2,6 +2,8 @@ from __future__ import unicode_literals from datetime import datetime import uuid +from ..utils import now_timestamp + class ActivityTask(object): def __init__(self, activity_id, activity_type, scheduled_event_id, @@ -9,6 +11,7 @@ class ActivityTask(object): self.activity_id = activity_id self.activity_type = activity_type self.input = input + self.last_heartbeat_timestamp = now_timestamp() self.scheduled_event_id = scheduled_event_id self.started_event_id = None self.state = "SCHEDULED" @@ -39,3 +42,6 @@ class ActivityTask(object): def fail(self): self.state = "FAILED" + + def reset_heartbeat_clock(self): + self.last_heartbeat_timestamp = now_timestamp() diff --git a/moto/swf/models/history_event.py b/moto/swf/models/history_event.py index eb3c1f795..aa5d09498 100644 --- a/moto/swf/models/history_event.py +++ b/moto/swf/models/history_event.py @@ -2,14 +2,14 @@ from __future__ import unicode_literals from datetime import datetime from time import mktime -from ..utils import decapitalize +from ..utils import decapitalize, now_timestamp class HistoryEvent(object): def __init__(self, event_id, event_type, **kwargs): self.event_id = event_id self.event_type = event_type - self.event_timestamp = float(mktime(datetime.now().timetuple())) + self.event_timestamp = now_timestamp() for key, value in kwargs.iteritems(): self.__setattr__(key, value) # break soon if attributes are not valid diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 10d134dca..5674e892a 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -13,7 +13,7 @@ from ..exceptions import ( SWFValidationException, SWFDecisionValidationException, ) -from ..utils import decapitalize +from ..utils import decapitalize, now_timestamp from .activity_task import ActivityTask from .activity_type import ActivityType from .decision_task import DecisionTask @@ -161,12 +161,8 @@ class WorkflowExecution(object): self._events.append(evt) return evt - # TODO: move it in utils - def _now_timestamp(self): - return float(mktime(datetime.now().timetuple())) - def start(self): - self.start_timestamp = self._now_timestamp() + self.start_timestamp = now_timestamp() self._add_event( "WorkflowExecutionStarted", workflow_execution=self, @@ -333,7 +329,7 @@ class WorkflowExecution(object): def complete(self, event_id, result=None): self.execution_status = "CLOSED" self.close_status = "COMPLETED" - self.close_timestamp = self._now_timestamp() + self.close_timestamp = now_timestamp() evt = self._add_event( "WorkflowExecutionCompleted", decision_task_completed_event_id=event_id, @@ -344,7 +340,7 @@ class WorkflowExecution(object): # TODO: implement lenght constraints on details/reason self.execution_status = "CLOSED" self.close_status = "FAILED" - self.close_timestamp = self._now_timestamp() + self.close_timestamp = now_timestamp() evt = self._add_event( "WorkflowExecutionFailed", decision_task_completed_event_id=event_id, @@ -423,7 +419,7 @@ class WorkflowExecution(object): ) self.domain.add_to_activity_task_list(task_list, task) self.open_counts["openActivityTasks"] += 1 - self.latest_activity_task_timestamp = self._now_timestamp() + self.latest_activity_task_timestamp = now_timestamp() def _find_activity_task(self, task_token): for task in self.activity_tasks: diff --git a/moto/swf/responses.py b/moto/swf/responses.py index ffadc73f2..3c4d6ec88 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -300,3 +300,12 @@ class SWFResponse(BaseResponse): details=details, reason=reason, run_id=run_id ) return "" + + def record_activity_task_heartbeat(self): + task_token = self._params["taskToken"] + details = self._params.get("details") + self.swf_backend.record_activity_task_heartbeat( + task_token, details=details + ) + # TODO: make it dynamic when we implement activity tasks cancellation + return json.dumps({"cancelRequested": False}) diff --git a/moto/swf/utils.py b/moto/swf/utils.py index 1b85f4ca9..02603bea9 100644 --- a/moto/swf/utils.py +++ b/moto/swf/utils.py @@ -1,2 +1,9 @@ +from datetime import datetime +from time import mktime + + def decapitalize(key): return key[0].lower() + key[1:] + +def now_timestamp(): + return float(mktime(datetime.now().timetuple())) diff --git a/tests/test_swf/models/test_activity_task.py b/tests/test_swf/models/test_activity_task.py index 93c842c8e..5c47d091e 100644 --- a/tests/test_swf/models/test_activity_task.py +++ b/tests/test_swf/models/test_activity_task.py @@ -1,3 +1,4 @@ +from freezegun import freeze_time from sure import expect from moto.swf.models import ( @@ -58,3 +59,22 @@ def test_activity_task_full_dict_representation(): at.start(1234) fd = at.to_full_dict() fd["startedEventId"].should.equal(1234) + +def test_activity_task_reset_heartbeat_clock(): + wfe = make_workflow_execution() + + with freeze_time("2015-01-01 12:00:00"): + task = ActivityTask( + activity_id="my-activity-123", + activity_type="foo", + input="optional", + scheduled_event_id=117, + workflow_execution=wfe, + ) + + task.last_heartbeat_timestamp.should.equal(1420110000.0) + + with freeze_time("2015-01-01 13:00:00"): + task.reset_heartbeat_clock() + + task.last_heartbeat_timestamp.should.equal(1420113600.0) diff --git a/tests/test_swf/responses/test_activity_tasks.py b/tests/test_swf/responses/test_activity_tasks.py index 3c7f82b5e..643c38f3f 100644 --- a/tests/test_swf/responses/test_activity_tasks.py +++ b/tests/test_swf/responses/test_activity_tasks.py @@ -171,3 +171,31 @@ def test_respond_activity_task_completed_with_wrong_token(): conn.respond_activity_task_failed.when.called_with( "not-a-correct-token" ).should.throw(SWFValidationException, "Invalid token") + + +# RecordActivityTaskHeartbeat endpoint +@mock_swf +def test_record_activity_task_heartbeat(): + conn = setup_workflow() + decision_token = conn.poll_for_decision_task("test-domain", "queue")["taskToken"] + conn.respond_decision_task_completed(decision_token, decisions=[ + SCHEDULE_ACTIVITY_TASK_DECISION + ]) + activity_token = conn.poll_for_activity_task("test-domain", "activity-task-list")["taskToken"] + + resp = conn.record_activity_task_heartbeat(activity_token, details="some progress details") + # TODO: check that "details" are reflected in ActivityTaskTimedOut event when a timeout occurs + resp.should.equal({"cancelRequested": False}) + +@mock_swf +def test_record_activity_task_heartbeat_with_wrong_token(): + conn = setup_workflow() + decision_token = conn.poll_for_decision_task("test-domain", "queue")["taskToken"] + conn.respond_decision_task_completed(decision_token, decisions=[ + SCHEDULE_ACTIVITY_TASK_DECISION + ]) + conn.poll_for_activity_task("test-domain", "activity-task-list")["taskToken"] + + conn.record_activity_task_heartbeat.when.called_with( + "bad-token", details="some progress details" + ).should.throw(SWFValidationException) diff --git a/tests/test_swf/test_utils.py b/tests/test_swf/test_utils.py index 6d11ba5fc..33bb8ada6 100644 --- a/tests/test_swf/test_utils.py +++ b/tests/test_swf/test_utils.py @@ -1,5 +1,11 @@ +from freezegun import freeze_time from sure import expect -from moto.swf.utils import decapitalize + +from moto.swf.utils import ( + decapitalize, + now_timestamp, +) + def test_decapitalize(): cases = { @@ -9,3 +15,7 @@ def test_decapitalize(): } for before, after in cases.iteritems(): decapitalize(before).should.equal(after) + +@freeze_time("2015-01-01 12:00:00") +def test_now_timestamp(): + now_timestamp().should.equal(1420110000.0) From 90c8797abdb33793dbaf62139e2cccc13a852d83 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Tue, 3 Nov 2015 00:28:13 +0100 Subject: [PATCH 61/94] Implement heartbeat timeout on SWF activity tasks --- moto/swf/models/__init__.py | 27 +++++++++++++++ moto/swf/models/activity_task.py | 19 ++++++++++- moto/swf/models/history_event.py | 9 +++++ moto/swf/models/workflow_execution.py | 21 ++++++++++++ tests/test_swf/models/test_activity_task.py | 26 +++++++++++++- .../test_swf/responses/test_activity_tasks.py | 33 +++++++++++------- tests/test_swf/responses/test_timeouts.py | 34 +++++++++++++++++++ tests/test_swf/utils.py | 23 +++++++++++++ 8 files changed, 178 insertions(+), 14 deletions(-) create mode 100644 tests/test_swf/responses/test_timeouts.py diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index 37c7827d7..13566ffe5 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -61,6 +61,11 @@ class SWFBackend(BaseBackend): if not isinstance(i, basestring): raise SWFSerializationException(parameter) + def _process_timeouts(self): + for domain in self.domains: + for wfe in domain.workflow_executions: + wfe._process_timeouts() + def list_domains(self, status, reverse_order=None): self._check_string(status) domains = [domain for domain in self.domains @@ -159,12 +164,16 @@ class SWFBackend(BaseBackend): self._check_string(domain_name) self._check_string(run_id) self._check_string(workflow_id) + # process timeouts on all objects + self._process_timeouts() domain = self._get_domain(domain_name) return domain.get_workflow_execution(workflow_id, run_id=run_id) def poll_for_decision_task(self, domain_name, task_list, identity=None): self._check_string(domain_name) self._check_string(task_list) + # process timeouts on all objects + self._process_timeouts() domain = self._get_domain(domain_name) # Real SWF cases: # - case 1: there's a decision task to return, return it @@ -196,6 +205,8 @@ class SWFBackend(BaseBackend): def count_pending_decision_tasks(self, domain_name, task_list): self._check_string(domain_name) self._check_string(task_list) + # process timeouts on all objects + self._process_timeouts() domain = self._get_domain(domain_name) count = 0 for wfe in domain.workflow_executions: @@ -208,6 +219,8 @@ class SWFBackend(BaseBackend): execution_context=None): self._check_string(task_token) self._check_none_or_string(execution_context) + # process timeouts on all objects + self._process_timeouts() # let's find decision task decision_task = None for domain in self.domains: @@ -259,6 +272,8 @@ class SWFBackend(BaseBackend): def poll_for_activity_task(self, domain_name, task_list, identity=None): self._check_string(domain_name) self._check_string(task_list) + # process timeouts on all objects + self._process_timeouts() domain = self._get_domain(domain_name) # Real SWF cases: # - case 1: there's an activity task to return, return it @@ -290,6 +305,8 @@ class SWFBackend(BaseBackend): def count_pending_activity_tasks(self, domain_name, task_list): self._check_string(domain_name) self._check_string(task_list) + # process timeouts on all objects + self._process_timeouts() domain = self._get_domain(domain_name) count = 0 for _task_list, tasks in domain.activity_task_lists.iteritems(): @@ -339,6 +356,8 @@ class SWFBackend(BaseBackend): def respond_activity_task_completed(self, task_token, result=None): self._check_string(task_token) self._check_none_or_string(result) + # process timeouts on all objects + self._process_timeouts() activity_task = self._find_activity_task_from_token(task_token) wfe = activity_task.workflow_execution wfe.complete_activity_task(activity_task.task_token, result=result) @@ -348,6 +367,8 @@ class SWFBackend(BaseBackend): # TODO: implement length limits on reason and details (common pb with client libs) self._check_none_or_string(reason) self._check_none_or_string(details) + # process timeouts on all objects + self._process_timeouts() activity_task = self._find_activity_task_from_token(task_token) wfe = activity_task.workflow_execution wfe.fail_activity_task(activity_task.task_token, reason=reason, details=details) @@ -360,6 +381,8 @@ class SWFBackend(BaseBackend): self._check_none_or_string(details) self._check_none_or_string(reason) self._check_none_or_string(run_id) + # process timeouts on all objects + self._process_timeouts() domain = self._get_domain(domain_name) wfe = domain.get_workflow_execution(workflow_id, run_id=run_id, raise_if_closed=True) wfe.terminate(child_policy=child_policy, details=details, reason=reason) @@ -367,8 +390,12 @@ class SWFBackend(BaseBackend): def record_activity_task_heartbeat(self, task_token, details=None): self._check_string(task_token) self._check_none_or_string(details) + # process timeouts on all objects + self._process_timeouts() activity_task = self._find_activity_task_from_token(task_token) activity_task.reset_heartbeat_clock() + if details: + activity_task.details = details swf_backends = {} diff --git a/moto/swf/models/activity_task.py b/moto/swf/models/activity_task.py index b099bc677..635c371ca 100644 --- a/moto/swf/models/activity_task.py +++ b/moto/swf/models/activity_task.py @@ -7,20 +7,27 @@ from ..utils import now_timestamp class ActivityTask(object): def __init__(self, activity_id, activity_type, scheduled_event_id, - workflow_execution, input=None): + workflow_execution, timeouts, input=None): self.activity_id = activity_id self.activity_type = activity_type + self.details = None self.input = input self.last_heartbeat_timestamp = now_timestamp() self.scheduled_event_id = scheduled_event_id self.started_event_id = None self.state = "SCHEDULED" self.task_token = str(uuid.uuid4()) + self.timeouts = timeouts + self.timeout_type = None self.workflow_execution = workflow_execution # this is *not* necessarily coherent with workflow execution history, # but that shouldn't be a problem for tests self.scheduled_at = datetime.now() + @property + def open(self): + return self.state in ["SCHEDULED", "STARTED"] + def to_full_dict(self): hsh = { "activityId": self.activity_id, @@ -45,3 +52,13 @@ class ActivityTask(object): def reset_heartbeat_clock(self): self.last_heartbeat_timestamp = now_timestamp() + + def has_timedout(self): + heartbeat_timeout_at = self.last_heartbeat_timestamp + \ + int(self.timeouts["heartbeatTimeout"]) + return heartbeat_timeout_at < now_timestamp() + + def process_timeouts(self): + if self.has_timedout(): + self.state = "TIMED_OUT" + self.timeout_type = "HEARTBEAT" diff --git a/moto/swf/models/history_event.py b/moto/swf/models/history_event.py index aa5d09498..37bf9b62b 100644 --- a/moto/swf/models/history_event.py +++ b/moto/swf/models/history_event.py @@ -143,6 +143,15 @@ class HistoryEvent(object): if self.reason: hsh["reason"] = self.reason return hsh + elif self.event_type == "ActivityTaskTimedOut": + hsh = { + "scheduledEventId": self.scheduled_event_id, + "startedEventId": self.started_event_id, + "timeoutType": self.timeout_type, + } + if self.details: + hsh["details"] = self.details + return hsh else: raise NotImplementedError( "HistoryEvent does not implement attributes for type '{}'".format(self.event_type) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 5674e892a..cb4cf89af 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -146,6 +146,26 @@ class WorkflowExecution(object): hsh["latestActivityTaskTimestamp"] = self.latest_activity_task_timestamp return hsh + def _process_timeouts(self): + self.should_schedule_decision_next = False + # TODO: process timeouts on workflow itself + # TODO: process timeouts on decision tasks + # activity tasks timeouts + for task in self.activity_tasks: + if task.open and task.has_timedout(): + self.should_schedule_decision_next = True + task.process_timeouts() + self._add_event( + "ActivityTaskTimedOut", + details=task.details, + scheduled_event_id=task.scheduled_event_id, + started_event_id=task.started_event_id, + timeout_type=task.timeout_type, + ) + # schedule decision task if needed + if self.should_schedule_decision_next: + self.schedule_decision_task() + def events(self, reverse_order=False): if reverse_order: return reversed(self._events) @@ -416,6 +436,7 @@ class WorkflowExecution(object): input=attributes.get("input"), scheduled_event_id=evt.event_id, workflow_execution=self, + timeouts=timeouts, ) self.domain.add_to_activity_task_list(task_list, task) self.open_counts["openActivityTasks"] += 1 diff --git a/tests/test_swf/models/test_activity_task.py b/tests/test_swf/models/test_activity_task.py index 5c47d091e..53e9c2cf8 100644 --- a/tests/test_swf/models/test_activity_task.py +++ b/tests/test_swf/models/test_activity_task.py @@ -6,7 +6,7 @@ from moto.swf.models import ( ActivityType, ) -from ..utils import make_workflow_execution +from ..utils import make_workflow_execution, ACTIVITY_TASK_TIMEOUTS def test_activity_task_creation(): @@ -17,6 +17,7 @@ def test_activity_task_creation(): input="optional", scheduled_event_id=117, workflow_execution=wfe, + timeouts=ACTIVITY_TASK_TIMEOUTS, ) task.workflow_execution.should.equal(wfe) task.state.should.equal("SCHEDULED") @@ -44,6 +45,7 @@ def test_activity_task_full_dict_representation(): activity_type=ActivityType("foo", "v1.0"), input="optional", scheduled_event_id=117, + timeouts=ACTIVITY_TASK_TIMEOUTS, workflow_execution=wfe, ) at.start(1234) @@ -69,6 +71,7 @@ def test_activity_task_reset_heartbeat_clock(): activity_type="foo", input="optional", scheduled_event_id=117, + timeouts=ACTIVITY_TASK_TIMEOUTS, workflow_execution=wfe, ) @@ -78,3 +81,24 @@ def test_activity_task_reset_heartbeat_clock(): task.reset_heartbeat_clock() task.last_heartbeat_timestamp.should.equal(1420113600.0) + +def test_activity_task_has_timedout(): + wfe = make_workflow_execution() + + with freeze_time("2015-01-01 12:00:00"): + task = ActivityTask( + activity_id="my-activity-123", + activity_type="foo", + input="optional", + scheduled_event_id=117, + timeouts=ACTIVITY_TASK_TIMEOUTS, + workflow_execution=wfe, + ) + task.has_timedout().should.equal(False) + + # activity task timeout is 300s == 5mins + with freeze_time("2015-01-01 12:06:00"): + task.has_timedout().should.equal(True) + task.process_timeouts() + task.state.should.equal("TIMED_OUT") + task.timeout_type.should.equal("HEARTBEAT") diff --git a/tests/test_swf/responses/test_activity_tasks.py b/tests/test_swf/responses/test_activity_tasks.py index 643c38f3f..6f84c663e 100644 --- a/tests/test_swf/responses/test_activity_tasks.py +++ b/tests/test_swf/responses/test_activity_tasks.py @@ -1,4 +1,5 @@ import boto +from freezegun import freeze_time from sure import expect from moto import mock_swf @@ -8,18 +9,9 @@ from moto.swf.exceptions import ( SWFUnknownResourceFault, ) -from ..utils import setup_workflow +from ..utils import setup_workflow, SCHEDULE_ACTIVITY_TASK_DECISION -SCHEDULE_ACTIVITY_TASK_DECISION = { - "decisionType": "ScheduleActivityTask", - "scheduleActivityTaskDecisionAttributes": { - "activityId": "my-activity-001", - "activityType": { "name": "test-activity", "version": "v1.1" }, - "taskList": { "name": "activity-task-list" }, - } -} - # PollForActivityTask endpoint @mock_swf def test_poll_for_activity_task_when_one(): @@ -183,8 +175,7 @@ def test_record_activity_task_heartbeat(): ]) activity_token = conn.poll_for_activity_task("test-domain", "activity-task-list")["taskToken"] - resp = conn.record_activity_task_heartbeat(activity_token, details="some progress details") - # TODO: check that "details" are reflected in ActivityTaskTimedOut event when a timeout occurs + resp = conn.record_activity_task_heartbeat(activity_token) resp.should.equal({"cancelRequested": False}) @mock_swf @@ -199,3 +190,21 @@ def test_record_activity_task_heartbeat_with_wrong_token(): conn.record_activity_task_heartbeat.when.called_with( "bad-token", details="some progress details" ).should.throw(SWFValidationException) + +@mock_swf +def test_record_activity_task_heartbeat_sets_details_in_case_of_timeout(): + conn = setup_workflow() + decision_token = conn.poll_for_decision_task("test-domain", "queue")["taskToken"] + conn.respond_decision_task_completed(decision_token, decisions=[ + SCHEDULE_ACTIVITY_TASK_DECISION + ]) + with freeze_time("2015-01-01 12:00:00"): + activity_token = conn.poll_for_activity_task("test-domain", "activity-task-list")["taskToken"] + conn.record_activity_task_heartbeat(activity_token, details="some progress details") + + with freeze_time("2015-01-01 12:05:30"): + # => Activity Task Heartbeat timeout reached!! + resp = conn.get_workflow_execution_history("test-domain", conn.run_id, "uid-abcd1234") + resp["events"][-2]["eventType"].should.equal("ActivityTaskTimedOut") + attrs = resp["events"][-2]["activityTaskTimedOutEventAttributes"] + attrs["details"].should.equal("some progress details") diff --git a/tests/test_swf/responses/test_timeouts.py b/tests/test_swf/responses/test_timeouts.py new file mode 100644 index 000000000..ca2377795 --- /dev/null +++ b/tests/test_swf/responses/test_timeouts.py @@ -0,0 +1,34 @@ +import boto +from freezegun import freeze_time +from sure import expect + +from moto import mock_swf + +from ..utils import setup_workflow, SCHEDULE_ACTIVITY_TASK_DECISION + + +# Activity Task Heartbeat timeout +# Default value in workflow helpers: 5 mins +@mock_swf +def test_activity_task_heartbeat_timeout(): + with freeze_time("2015-01-01 12:00:00"): + conn = setup_workflow() + decision_token = conn.poll_for_decision_task("test-domain", "queue")["taskToken"] + conn.respond_decision_task_completed(decision_token, decisions=[ + SCHEDULE_ACTIVITY_TASK_DECISION + ]) + conn.poll_for_activity_task("test-domain", "activity-task-list", identity="surprise") + + with freeze_time("2015-01-01 12:04:30"): + resp = conn.get_workflow_execution_history("test-domain", conn.run_id, "uid-abcd1234") + resp["events"][-1]["eventType"].should.equal("ActivityTaskStarted") + + with freeze_time("2015-01-01 12:05:30"): + # => Activity Task Heartbeat timeout reached!! + resp = conn.get_workflow_execution_history("test-domain", conn.run_id, "uid-abcd1234") + + resp["events"][-2]["eventType"].should.equal("ActivityTaskTimedOut") + attrs = resp["events"][-2]["activityTaskTimedOutEventAttributes"] + attrs["timeoutType"].should.equal("HEARTBEAT") + + resp["events"][-1]["eventType"].should.equal("DecisionTaskScheduled") diff --git a/tests/test_swf/utils.py b/tests/test_swf/utils.py index 6fcec9d46..e6d73fe9a 100644 --- a/tests/test_swf/utils.py +++ b/tests/test_swf/utils.py @@ -9,6 +9,29 @@ from moto.swf.models import ( ) +# Some useful constants +# Here are some activity timeouts we use in moto/swf tests ; they're extracted +# from semi-real world example, the goal is mostly to have predictible and +# intuitive behaviour in moto/swf own tests... +ACTIVITY_TASK_TIMEOUTS = { + "heartbeatTimeout": "300", # 5 mins + "scheduleToStartTimeout": "1800", # 30 mins + "startToCloseTimeout": "1800", # 30 mins + "scheduleToCloseTimeout": "2700", # 45 mins +} + +# Some useful decisions +SCHEDULE_ACTIVITY_TASK_DECISION = { + "decisionType": "ScheduleActivityTask", + "scheduleActivityTaskDecisionAttributes": { + "activityId": "my-activity-001", + "activityType": { "name": "test-activity", "version": "v1.1" }, + "taskList": { "name": "activity-task-list" }, + } +} +for key, value in ACTIVITY_TASK_TIMEOUTS.iteritems(): + SCHEDULE_ACTIVITY_TASK_DECISION["scheduleActivityTaskDecisionAttributes"][key] = value + # A test Domain def get_basic_domain(): return Domain("test-domain", "90") From e9732140e52802cf90cb82b969ffd8737a80e64f Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Tue, 3 Nov 2015 08:40:33 +0100 Subject: [PATCH 62/94] Fix python 2.6 compatibility for moto/swf --- moto/swf/exceptions.py | 16 ++++++++-------- moto/swf/models/__init__.py | 10 +++++----- moto/swf/models/domain.py | 6 +++--- moto/swf/models/generic_type.py | 4 ++-- moto/swf/models/history_event.py | 4 ++-- moto/swf/models/workflow_execution.py | 18 +++++++++--------- moto/swf/responses.py | 4 ++-- 7 files changed, 31 insertions(+), 31 deletions(-) diff --git a/moto/swf/exceptions.py b/moto/swf/exceptions.py index d79281100..abeb348b7 100644 --- a/moto/swf/exceptions.py +++ b/moto/swf/exceptions.py @@ -14,9 +14,9 @@ class SWFClientError(JSONResponseError): class SWFUnknownResourceFault(SWFClientError): def __init__(self, resource_type, resource_name=None): if resource_name: - message = "Unknown {}: {}".format(resource_type, resource_name) + message = "Unknown {0}: {1}".format(resource_type, resource_name) else: - message = "Unknown {}".format(resource_type) + message = "Unknown {0}".format(resource_type) super(SWFUnknownResourceFault, self).__init__( message, "com.amazonaws.swf.base.model#UnknownResourceFault") @@ -39,7 +39,7 @@ class SWFDomainDeprecatedFault(SWFClientError): class SWFSerializationException(JSONResponseError): def __init__(self, value): message = "class java.lang.Foo can not be converted to an String " - message += " (not a real SWF exception ; happened on: {})".format(value) + message += " (not a real SWF exception ; happened on: {0})".format(value) __type = "com.amazonaws.swf.base.model#SerializationException" super(SWFSerializationException, self).__init__( 400, "Bad Request", @@ -50,14 +50,14 @@ class SWFSerializationException(JSONResponseError): class SWFTypeAlreadyExistsFault(SWFClientError): def __init__(self, _type): super(SWFTypeAlreadyExistsFault, self).__init__( - "{}=[name={}, version={}]".format(_type.__class__.__name__, _type.name, _type.version), + "{0}=[name={1}, version={2}]".format(_type.__class__.__name__, _type.name, _type.version), "com.amazonaws.swf.base.model#TypeAlreadyExistsFault") class SWFTypeDeprecatedFault(SWFClientError): def __init__(self, _type): super(SWFTypeDeprecatedFault, self).__init__( - "{}=[name={}, version={}]".format(_type.__class__.__name__, _type.name, _type.version), + "{0}=[name={1}, version={2}]".format(_type.__class__.__name__, _type.name, _type.version), "com.amazonaws.swf.base.model#TypeDeprecatedFault") @@ -107,14 +107,14 @@ class SWFDecisionValidationException(SWFClientError): ) else: raise ValueError( - "Unhandled decision constraint type: {}".format(pb["type"]) + "Unhandled decision constraint type: {0}".format(pb["type"]) ) # prefix count = len(problems) if count < 2: - prefix = "{} validation error detected: " + prefix = "{0} validation error detected: " else: - prefix = "{} validation errors detected: " + prefix = "{0} validation errors detected: " super(SWFDecisionValidationException, self).__init__( prefix.format(count) + "; ".join(messages), "com.amazon.coral.validate#ValidationException" diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index 13566ffe5..3b9129c2a 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -117,7 +117,7 @@ class SWFBackend(BaseBackend): _type = domain.get_type(kind, name, version, ignore_empty=True) if _type: raise SWFTypeAlreadyExistsFault(_type) - _class = globals()["{}Type".format(kind.capitalize())] + _class = globals()["{0}Type".format(kind.capitalize())] _type = _class(name, version, **kwargs) domain.add_type(_type) @@ -245,7 +245,7 @@ class SWFBackend(BaseBackend): if wfe.execution_status != "OPEN": raise SWFUnknownResourceFault( "execution", - "WorkflowExecution=[workflowId={}, runId={}]".format( + "WorkflowExecution=[workflowId={0}, runId={1}]".format( wfe.workflow_id, wfe.run_id ) ) @@ -253,7 +253,7 @@ class SWFBackend(BaseBackend): if decision_task.state != "STARTED": if decision_task.state == "COMPLETED": raise SWFUnknownResourceFault( - "decision task, scheduledEventId = {}".format(decision_task.scheduled_event_id) + "decision task, scheduledEventId = {0}".format(decision_task.scheduled_event_id) ) else: raise ValueError( @@ -333,7 +333,7 @@ class SWFBackend(BaseBackend): if wfe.execution_status != "OPEN": raise SWFUnknownResourceFault( "execution", - "WorkflowExecution=[workflowId={}, runId={}]".format( + "WorkflowExecution=[workflowId={0}, runId={1}]".format( wfe.workflow_id, wfe.run_id ) ) @@ -341,7 +341,7 @@ class SWFBackend(BaseBackend): if activity_task.state != "STARTED": if activity_task.state == "COMPLETED": raise SWFUnknownResourceFault( - "activity, scheduledEventId = {}".format(activity_task.scheduled_event_id) + "activity, scheduledEventId = {0}".format(activity_task.scheduled_event_id) ) else: raise ValueError( diff --git a/moto/swf/models/domain.py b/moto/swf/models/domain.py index 0174661ce..066060b66 100644 --- a/moto/swf/models/domain.py +++ b/moto/swf/models/domain.py @@ -53,7 +53,7 @@ class Domain(object): if not ignore_empty: raise SWFUnknownResourceFault( "type", - "{}Type=[name={}, version={}]".format( + "{0}Type=[name={1}, version={2}]".format( kind.capitalize(), name, version ) ) @@ -91,10 +91,10 @@ class Domain(object): wfe = None if not wfe and raise_if_none: if run_id: - args = ["execution", "WorkflowExecution=[workflowId={}, runId={}]".format( + args = ["execution", "WorkflowExecution=[workflowId={0}, runId={1}]".format( workflow_id, run_id)] else: - args = ["execution, workflowId = {}".format(workflow_id)] + args = ["execution, workflowId = {0}".format(workflow_id)] raise SWFUnknownResourceFault(*args) # at last return workflow execution return wfe diff --git a/moto/swf/models/generic_type.py b/moto/swf/models/generic_type.py index 904296ab3..76fa98666 100644 --- a/moto/swf/models/generic_type.py +++ b/moto/swf/models/generic_type.py @@ -23,7 +23,7 @@ class GenericType(object): def __repr__(self): cls = self.__class__.__name__ attrs = "name: %(name)s, version: %(version)s, status: %(status)s" % self.__dict__ - return "{}({})".format(cls, attrs) + return "{0}({1})".format(cls, attrs) @property def kind(self): @@ -41,7 +41,7 @@ class GenericType(object): def to_medium_dict(self): hsh = { - "{}Type".format(self.kind): self.to_short_dict(), + "{0}Type".format(self.kind): self.to_short_dict(), "creationDate": 1420066800, "status": self.status, } diff --git a/moto/swf/models/history_event.py b/moto/swf/models/history_event.py index 37bf9b62b..176c92d7d 100644 --- a/moto/swf/models/history_event.py +++ b/moto/swf/models/history_event.py @@ -24,7 +24,7 @@ class HistoryEvent(object): } def _attributes_key(self): - key = "{}EventAttributes".format(self.event_type) + key = "{0}EventAttributes".format(self.event_type) return decapitalize(key) def event_attributes(self): @@ -154,5 +154,5 @@ class HistoryEvent(object): return hsh else: raise NotImplementedError( - "HistoryEvent does not implement attributes for type '{}'".format(self.event_type) + "HistoryEvent does not implement attributes for type '{0}'".format(self.event_type) ) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index cb4cf89af..8674d07fb 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -83,7 +83,7 @@ class WorkflowExecution(object): self.child_workflow_executions = [] def __repr__(self): - return "WorkflowExecution(run_id: {})".format(self.run_id) + return "WorkflowExecution(run_id: {0})".format(self.run_id) def _set_from_kwargs_or_workflow_type(self, kwargs, local_key, workflow_type_key=None): if workflow_type_key is None: @@ -219,7 +219,7 @@ class WorkflowExecution(object): if dt.task_token == task_token: return dt raise ValueError( - "No decision task with token: {}".format(task_token) + "No decision task with token: {0}".format(task_token) ) def start_decision_task(self, task_token, identity=None): @@ -260,7 +260,7 @@ class WorkflowExecution(object): if constraint["required"] and not value.get(key): problems.append({ "type": "null_value", - "where": "decisions.{}.member.{}.{}".format( + "where": "decisions.{0}.member.{1}.{2}".format( decision_id, kind, key ) }) @@ -297,7 +297,7 @@ class WorkflowExecution(object): attrs_to_check = filter(lambda x: x.endswith("DecisionAttributes"), dcs.keys()) if dcs["decisionType"] in self.KNOWN_DECISION_TYPES: decision_type = dcs["decisionType"] - decision_attr = "{}DecisionAttributes".format(decapitalize(decision_type)) + decision_attr = "{0}DecisionAttributes".format(decapitalize(decision_type)) attrs_to_check.append(decision_attr) for attr in attrs_to_check: problems += self._check_decision_attributes(attr, dcs.get(attr, {}), decision_number) @@ -306,7 +306,7 @@ class WorkflowExecution(object): problems.append({ "type": "bad_decision_type", "value": dcs["decisionType"], - "where": "decisions.{}.member.decisionType".format(decision_number), + "where": "decisions.{0}.member.decisionType".format(decision_number), "possible_values": ", ".join(self.KNOWN_DECISION_TYPES), }) @@ -322,7 +322,7 @@ class WorkflowExecution(object): # handle each decision separately, in order for decision in decisions: decision_type = decision["decisionType"] - attributes_key = "{}DecisionAttributes".format(decapitalize(decision_type)) + attributes_key = "{0}DecisionAttributes".format(decapitalize(decision_type)) attributes = decision.get(attributes_key, {}) if decision_type == "CompleteWorkflowExecution": self.complete(event_id, attributes.get("result")) @@ -341,7 +341,7 @@ class WorkflowExecution(object): # TODO: implement Decision type: SignalExternalWorkflowExecution # TODO: implement Decision type: StartChildWorkflowExecution # TODO: implement Decision type: StartTimer - raise NotImplementedError("Cannot handle decision: {}".format(decision_type)) + raise NotImplementedError("Cannot handle decision: {0}".format(decision_type)) # finally decrement counter if and only if everything went well self.open_counts["openDecisionTasks"] -= 1 @@ -419,7 +419,7 @@ class WorkflowExecution(object): if not timeouts[_type]: error_key = default_key.replace("default_task_", "default_") fail_schedule_activity_task(activity_type, - "{}_UNDEFINED".format(error_key.upper())) + "{0}_UNDEFINED".format(error_key.upper())) return # Only add event and increment counters now that nothing went wrong @@ -447,7 +447,7 @@ class WorkflowExecution(object): if task.task_token == task_token: return task raise ValueError( - "No activity task with token: {}".format(task_token) + "No activity task with token: {0}".format(task_token) ) def start_activity_task(self, task_token, identity=None): diff --git a/moto/swf/responses.py b/moto/swf/responses.py index 3c4d6ec88..75b533462 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -61,7 +61,7 @@ class SWFResponse(BaseResponse): def _describe_type(self, kind): domain = self._params["domain"] - _type_args = self._params["{}Type".format(kind)] + _type_args = self._params["{0}Type".format(kind)] name = _type_args["name"] version = _type_args["version"] _type = self.swf_backend.describe_type(kind, domain, name, version) @@ -70,7 +70,7 @@ class SWFResponse(BaseResponse): def _deprecate_type(self, kind): domain = self._params["domain"] - _type_args = self._params["{}Type".format(kind)] + _type_args = self._params["{0}Type".format(kind)] name = _type_args["name"] version = _type_args["version"] self.swf_backend.deprecate_type(kind, domain, name, version) From f4feec4727113f06efaaefcd267a45871ddccb90 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Tue, 3 Nov 2015 09:09:00 +0100 Subject: [PATCH 63/94] Fix timestamps in tests as Travis builds run on GMT time When launched manually, tests should be launched with the environment variable TZ=GMT. Maybe this could be useful to add that explicitly somewhere in the README or in the Makefile. --- tests/test_swf/models/test_activity_task.py | 4 ++-- tests/test_swf/models/test_history_event.py | 4 ++-- tests/test_swf/models/test_workflow_execution.py | 8 ++++---- tests/test_swf/responses/test_decision_tasks.py | 2 +- tests/test_swf/test_utils.py | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test_swf/models/test_activity_task.py b/tests/test_swf/models/test_activity_task.py index 53e9c2cf8..4057c947c 100644 --- a/tests/test_swf/models/test_activity_task.py +++ b/tests/test_swf/models/test_activity_task.py @@ -75,12 +75,12 @@ def test_activity_task_reset_heartbeat_clock(): workflow_execution=wfe, ) - task.last_heartbeat_timestamp.should.equal(1420110000.0) + task.last_heartbeat_timestamp.should.equal(1420113600.0) with freeze_time("2015-01-01 13:00:00"): task.reset_heartbeat_clock() - task.last_heartbeat_timestamp.should.equal(1420113600.0) + task.last_heartbeat_timestamp.should.equal(1420117200.0) def test_activity_task_has_timedout(): wfe = make_workflow_execution() diff --git a/tests/test_swf/models/test_history_event.py b/tests/test_swf/models/test_history_event.py index 89d73210c..8b6a7f711 100644 --- a/tests/test_swf/models/test_history_event.py +++ b/tests/test_swf/models/test_history_event.py @@ -9,7 +9,7 @@ def test_history_event_creation(): he = HistoryEvent(123, "DecisionTaskStarted", scheduled_event_id=2) he.event_id.should.equal(123) he.event_type.should.equal("DecisionTaskStarted") - he.event_timestamp.should.equal(1420110000.0) + he.event_timestamp.should.equal(1420113600.0) @freeze_time("2015-01-01 12:00:00") def test_history_event_to_dict_representation(): @@ -17,7 +17,7 @@ def test_history_event_to_dict_representation(): he.to_dict().should.equal({ "eventId": 123, "eventType": "DecisionTaskStarted", - "eventTimestamp": 1420110000.0, + "eventTimestamp": 1420113600.0, "decisionTaskStartedEventAttributes": { "scheduledEventId": 2 } diff --git a/tests/test_swf/models/test_workflow_execution.py b/tests/test_swf/models/test_workflow_execution.py index ced636969..5f14c51da 100644 --- a/tests/test_swf/models/test_workflow_execution.py +++ b/tests/test_swf/models/test_workflow_execution.py @@ -165,7 +165,7 @@ def test_workflow_execution_start(): wfe.events().should.equal([]) wfe.start() - wfe.start_timestamp.should.equal(1420110000.0) + wfe.start_timestamp.should.equal(1420113600.0) wfe.events().should.have.length_of(2) wfe.events()[0].event_type.should.equal("WorkflowExecutionStarted") wfe.events()[1].event_type.should.equal("DecisionTaskScheduled") @@ -177,7 +177,7 @@ def test_workflow_execution_complete(): wfe.execution_status.should.equal("CLOSED") wfe.close_status.should.equal("COMPLETED") - wfe.close_timestamp.should.equal(1420196400.0) + wfe.close_timestamp.should.equal(1420200000.0) wfe.events()[-1].event_type.should.equal("WorkflowExecutionCompleted") wfe.events()[-1].decision_task_completed_event_id.should.equal(123) wfe.events()[-1].result.should.equal("foo") @@ -189,7 +189,7 @@ def test_workflow_execution_fail(): wfe.execution_status.should.equal("CLOSED") wfe.close_status.should.equal("FAILED") - wfe.close_timestamp.should.equal(1420196400.0) + wfe.close_timestamp.should.equal(1420200000.0) wfe.events()[-1].event_type.should.equal("WorkflowExecutionFailed") wfe.events()[-1].decision_task_completed_event_id.should.equal(123) wfe.events()[-1].details.should.equal("some details") @@ -202,7 +202,7 @@ def test_workflow_execution_schedule_activity_task(): wfe.schedule_activity_task(123, VALID_ACTIVITY_TASK_ATTRIBUTES) - wfe.latest_activity_task_timestamp.should.equal(1420110000.0) + wfe.latest_activity_task_timestamp.should.equal(1420113600.0) wfe.open_counts["openActivityTasks"].should.equal(1) last_event = wfe.events()[-1] diff --git a/tests/test_swf/responses/test_decision_tasks.py b/tests/test_swf/responses/test_decision_tasks.py index 9510b31fd..7e9ecddb1 100644 --- a/tests/test_swf/responses/test_decision_tasks.py +++ b/tests/test_swf/responses/test_decision_tasks.py @@ -313,4 +313,4 @@ def test_respond_decision_task_completed_with_schedule_activity_task(): }) resp = conn.describe_workflow_execution("test-domain", conn.run_id, "uid-abcd1234") - resp["latestActivityTaskTimestamp"].should.equal(1420110000.0) + resp["latestActivityTaskTimestamp"].should.equal(1420113600.0) diff --git a/tests/test_swf/test_utils.py b/tests/test_swf/test_utils.py index 33bb8ada6..9f7da8857 100644 --- a/tests/test_swf/test_utils.py +++ b/tests/test_swf/test_utils.py @@ -18,4 +18,4 @@ def test_decapitalize(): @freeze_time("2015-01-01 12:00:00") def test_now_timestamp(): - now_timestamp().should.equal(1420110000.0) + now_timestamp().should.equal(1420113600.0) From 2cd3d5fb45ee4fb66d6dd1d72afc8a9a4e9cdd88 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Tue, 3 Nov 2015 09:17:56 +0100 Subject: [PATCH 64/94] Fix python 3.x compatibility regarding iterations on a dict Error on travis-ci was: AttributeError: 'dict' object has no attribute 'iteritems' And actually it's been removed in python 3.x in favor of dict.items() --- moto/swf/models/__init__.py | 10 +++++----- moto/swf/models/domain.py | 8 ++++---- moto/swf/models/generic_type.py | 2 +- moto/swf/models/history_event.py | 2 +- moto/swf/models/workflow_execution.py | 2 +- tests/test_swf/test_utils.py | 2 +- tests/test_swf/utils.py | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index 3b9129c2a..18db47e24 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -111,7 +111,7 @@ class SWFBackend(BaseBackend): self._check_string(domain_name) self._check_string(name) self._check_string(version) - for _, value in kwargs.iteritems(): + for _, value in kwargs.items(): self._check_none_or_string(value) domain = self._get_domain(domain_name) _type = domain.get_type(kind, name, version, ignore_empty=True) @@ -146,7 +146,7 @@ class SWFBackend(BaseBackend): self._check_string(workflow_name) self._check_string(workflow_version) self._check_none_or_list_of_strings(tag_list) - for _, value in kwargs.iteritems(): + for _, value in kwargs.items(): self._check_none_or_string(value) domain = self._get_domain(domain_name) @@ -190,7 +190,7 @@ class SWFBackend(BaseBackend): # # TODO: handle long polling (case 2) for decision tasks candidates = [] - for _task_list, tasks in domain.decision_task_lists.iteritems(): + for _task_list, tasks in domain.decision_task_lists.items(): if _task_list == task_list: candidates += filter(lambda t: t.state == "SCHEDULED", tasks) if any(candidates): @@ -290,7 +290,7 @@ class SWFBackend(BaseBackend): # # TODO: handle long polling (case 2) for activity tasks candidates = [] - for _task_list, tasks in domain.activity_task_lists.iteritems(): + for _task_list, tasks in domain.activity_task_lists.items(): if _task_list == task_list: candidates += filter(lambda t: t.state == "SCHEDULED", tasks) if any(candidates): @@ -309,7 +309,7 @@ class SWFBackend(BaseBackend): self._process_timeouts() domain = self._get_domain(domain_name) count = 0 - for _task_list, tasks in domain.activity_task_lists.iteritems(): + for _task_list, tasks in domain.activity_task_lists.items(): if _task_list == task_list: pending = [t for t in tasks if t.state in ["SCHEDULED", "STARTED"]] count += len(pending) diff --git a/moto/swf/models/domain.py b/moto/swf/models/domain.py index 066060b66..55bf1b8e8 100644 --- a/moto/swf/models/domain.py +++ b/moto/swf/models/domain.py @@ -63,8 +63,8 @@ class Domain(object): def find_types(self, kind, status): _all = [] - for _, family in self.types[kind].iteritems(): - for _, _type in family.iteritems(): + for _, family in self.types[kind].items(): + for _, _type in family.items(): if _type.status == status: _all.append(_type) return _all @@ -107,7 +107,7 @@ class Domain(object): @property def activity_tasks(self): _all = [] - for _, tasks in self.activity_task_lists.iteritems(): + for _, tasks in self.activity_task_lists.items(): _all += tasks return _all @@ -119,6 +119,6 @@ class Domain(object): @property def decision_tasks(self): _all = [] - for _, tasks in self.decision_task_lists.iteritems(): + for _, tasks in self.decision_task_lists.items(): _all += tasks return _all diff --git a/moto/swf/models/generic_type.py b/moto/swf/models/generic_type.py index 76fa98666..7c8389fbe 100644 --- a/moto/swf/models/generic_type.py +++ b/moto/swf/models/generic_type.py @@ -10,7 +10,7 @@ class GenericType(object): self.status = "REGISTERED" if "description" in kwargs: self.description = kwargs.pop("description") - for key, value in kwargs.iteritems(): + for key, value in kwargs.items(): self.__setattr__(key, value) # default values set to none for key in self._configuration_keys: diff --git a/moto/swf/models/history_event.py b/moto/swf/models/history_event.py index 176c92d7d..0b53f8659 100644 --- a/moto/swf/models/history_event.py +++ b/moto/swf/models/history_event.py @@ -10,7 +10,7 @@ class HistoryEvent(object): self.event_id = event_id self.event_type = event_type self.event_timestamp = now_timestamp() - for key, value in kwargs.iteritems(): + for key, value in kwargs.items(): self.__setattr__(key, value) # break soon if attributes are not valid self.event_attributes() diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 8674d07fb..14523b079 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -256,7 +256,7 @@ class WorkflowExecution(object): def _check_decision_attributes(self, kind, value, decision_id): problems = [] constraints = DECISIONS_FIELDS.get(kind, {}) - for key, constraint in constraints.iteritems(): + for key, constraint in constraints.items(): if constraint["required"] and not value.get(key): problems.append({ "type": "null_value", diff --git a/tests/test_swf/test_utils.py b/tests/test_swf/test_utils.py index 9f7da8857..f8ff08f22 100644 --- a/tests/test_swf/test_utils.py +++ b/tests/test_swf/test_utils.py @@ -13,7 +13,7 @@ def test_decapitalize(): "FooBar": "fooBar", "FOO BAR": "fOO BAR", } - for before, after in cases.iteritems(): + for before, after in cases.items(): decapitalize(before).should.equal(after) @freeze_time("2015-01-01 12:00:00") diff --git a/tests/test_swf/utils.py b/tests/test_swf/utils.py index e6d73fe9a..70cbc4c13 100644 --- a/tests/test_swf/utils.py +++ b/tests/test_swf/utils.py @@ -29,7 +29,7 @@ SCHEDULE_ACTIVITY_TASK_DECISION = { "taskList": { "name": "activity-task-list" }, } } -for key, value in ACTIVITY_TASK_TIMEOUTS.iteritems(): +for key, value in ACTIVITY_TASK_TIMEOUTS.items(): SCHEDULE_ACTIVITY_TASK_DECISION["scheduleActivityTaskDecisionAttributes"][key] = value # A test Domain From c0b4aadd923f635f09c2f221851f66167b0fb3ea Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Tue, 3 Nov 2015 09:19:03 +0100 Subject: [PATCH 65/94] Fix python 3.x compatibility regarding json loading Error on travis-ci was: TypeError: can't use a string pattern on a bytes-like object --- moto/swf/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/swf/responses.py b/moto/swf/responses.py index 75b533462..3f6e83330 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -48,7 +48,7 @@ class SWFResponse(BaseResponse): # SWF parameters are passed through a JSON body, so let's ease retrieval @property def _params(self): - return json.loads(self.body) + return json.loads(self.body.decode("utf-8")) def _list_types(self, kind): domain_name = self._params["domain"] From 3a5f679783f3b176017a13981779be2ce2fd19e9 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Tue, 3 Nov 2015 10:41:23 +0100 Subject: [PATCH 66/94] Fix python 3.3 compatibility in moto/swf regarging string types detection --- moto/swf/models/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index 18db47e24..8702bc374 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -1,4 +1,5 @@ from __future__ import unicode_literals +import six import boto.swf @@ -47,7 +48,7 @@ class SWFBackend(BaseBackend): self._check_string(parameter) def _check_string(self, parameter): - if not isinstance(parameter, basestring): + if not isinstance(parameter, six.string_types): raise SWFSerializationException(parameter) def _check_none_or_list_of_strings(self, parameter): @@ -58,7 +59,7 @@ class SWFBackend(BaseBackend): if not isinstance(parameter, list): raise SWFSerializationException(parameter) for i in parameter: - if not isinstance(i, basestring): + if not isinstance(i, six.string_types): raise SWFSerializationException(parameter) def _process_timeouts(self): From b386495520d1196d32e4ca59667b39f52b0699ef Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Tue, 3 Nov 2015 10:45:11 +0100 Subject: [PATCH 67/94] Use list comprehensions instead of filter() for easier moto/swf python 3.x compatibility --- moto/swf/models/__init__.py | 4 ++-- moto/swf/models/workflow_execution.py | 14 +++++--------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index 8702bc374..4d70fbd0e 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -193,7 +193,7 @@ class SWFBackend(BaseBackend): candidates = [] for _task_list, tasks in domain.decision_task_lists.items(): if _task_list == task_list: - candidates += filter(lambda t: t.state == "SCHEDULED", tasks) + candidates += [t for t in tasks if t.state == "SCHEDULED"] if any(candidates): # TODO: handle task priorities (but not supported by boto for now) task = min(candidates, key=lambda d: d.scheduled_at) @@ -293,7 +293,7 @@ class SWFBackend(BaseBackend): candidates = [] for _task_list, tasks in domain.activity_task_lists.items(): if _task_list == task_list: - candidates += filter(lambda t: t.state == "SCHEDULED", tasks) + candidates += [t for t in tasks if t.state == "SCHEDULED"] if any(candidates): # TODO: handle task priorities (but not supported by boto for now) task = min(candidates, key=lambda d: d.scheduled_at) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 14523b079..cdc356ab5 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -202,17 +202,13 @@ class WorkflowExecution(object): @property def decision_tasks(self): - return filter( - lambda t: t.workflow_execution == self, - self.domain.decision_tasks - ) + return [t for t in self.domain.decision_tasks + if t.workflow_execution == self] @property def activity_tasks(self): - return filter( - lambda t: t.workflow_execution == self, - self.domain.activity_tasks - ) + return [t for t in self.domain.activity_tasks + if t.workflow_execution == self] def _find_decision_task(self, task_token): for dt in self.decision_tasks: @@ -294,7 +290,7 @@ class WorkflowExecution(object): # check decision types mandatory attributes # NB: the real SWF service seems to check attributes even for attributes list # that are not in line with the decisionType, so we do the same - attrs_to_check = filter(lambda x: x.endswith("DecisionAttributes"), dcs.keys()) + attrs_to_check = [d for d in dcs.keys() if d.endswith("DecisionAttributes")] if dcs["decisionType"] in self.KNOWN_DECISION_TYPES: decision_type = dcs["decisionType"] decision_attr = "{0}DecisionAttributes".format(decapitalize(decision_type)) From e32fef50b614020823934874e73f18fe8e3ef6bd Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Tue, 3 Nov 2015 10:56:31 +0100 Subject: [PATCH 68/94] Fix random list ordering bugs on python 3.x in moto/swf tests --- tests/test_swf/models/test_domain.py | 4 ++-- tests/test_swf/models/test_generic_type.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_swf/models/test_domain.py b/tests/test_swf/models/test_domain.py index 0efa0029d..6430bc1ae 100644 --- a/tests/test_swf/models/test_domain.py +++ b/tests/test_swf/models/test_domain.py @@ -41,7 +41,7 @@ def test_domain_activity_tasks(): domain = Domain("my-domain", "60") domain.add_to_activity_task_list("foo", "bar") domain.add_to_activity_task_list("other", "baz") - domain.activity_tasks.should.equal(["bar", "baz"]) + sorted(domain.activity_tasks).should.equal(["bar", "baz"]) def test_domain_add_to_decision_task_list(): domain = Domain("my-domain", "60") @@ -54,7 +54,7 @@ def test_domain_decision_tasks(): domain = Domain("my-domain", "60") domain.add_to_decision_task_list("foo", "bar") domain.add_to_decision_task_list("other", "baz") - domain.decision_tasks.should.equal(["bar", "baz"]) + sorted(domain.decision_tasks).should.equal(["bar", "baz"]) def test_domain_get_workflow_execution(): domain = Domain("my-domain", "60") diff --git a/tests/test_swf/models/test_generic_type.py b/tests/test_swf/models/test_generic_type.py index 8937836e5..5d7f3d4d0 100644 --- a/tests/test_swf/models/test_generic_type.py +++ b/tests/test_swf/models/test_generic_type.py @@ -44,7 +44,8 @@ def test_type_full_dict_representation(): _type.to_full_dict()["configuration"]["justAnExampleTimeout"].should.equal("60") _type.non_whitelisted_property = "34" - _type.to_full_dict()["configuration"].keys().should.equal(["defaultTaskList", "justAnExampleTimeout"]) + keys = _type.to_full_dict()["configuration"].keys() + sorted(keys).should.equal(["defaultTaskList", "justAnExampleTimeout"]) def test_type_string_representation(): _type = FooType("test-foo", "v1.0") From 86973f2b87fa2b0293a4f1ff2dadb09a1a786244 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Wed, 4 Nov 2015 10:12:17 +0100 Subject: [PATCH 69/94] Implement start to close timeout on SWF decision tasks --- moto/swf/models/activity_task.py | 1 + moto/swf/models/decision_task.py | 19 +++++++++++++ moto/swf/models/history_event.py | 6 ++++ moto/swf/models/workflow_execution.py | 15 +++++++++- tests/test_swf/models/test_decision_task.py | 18 ++++++++++++ tests/test_swf/responses/test_timeouts.py | 31 +++++++++++++++++++++ 6 files changed, 89 insertions(+), 1 deletion(-) diff --git a/moto/swf/models/activity_task.py b/moto/swf/models/activity_task.py index 635c371ca..ccddb0ba6 100644 --- a/moto/swf/models/activity_task.py +++ b/moto/swf/models/activity_task.py @@ -54,6 +54,7 @@ class ActivityTask(object): self.last_heartbeat_timestamp = now_timestamp() def has_timedout(self): + # TODO: handle the "NONE" case heartbeat_timeout_at = self.last_heartbeat_timestamp + \ int(self.timeouts["heartbeatTimeout"]) return heartbeat_timeout_at < now_timestamp() diff --git a/moto/swf/models/decision_task.py b/moto/swf/models/decision_task.py index 967c94fa5..dfc8f5687 100644 --- a/moto/swf/models/decision_task.py +++ b/moto/swf/models/decision_task.py @@ -2,6 +2,8 @@ from __future__ import unicode_literals from datetime import datetime import uuid +from ..utils import now_timestamp + class DecisionTask(object): def __init__(self, workflow_execution, scheduled_event_id): @@ -11,10 +13,13 @@ class DecisionTask(object): self.scheduled_event_id = scheduled_event_id self.previous_started_event_id = 0 self.started_event_id = None + self.started_timestamp = None + self.start_to_close_timeout = self.workflow_execution.task_start_to_close_timeout self.state = "SCHEDULED" # this is *not* necessarily coherent with workflow execution history, # but that shouldn't be a problem for tests self.scheduled_at = datetime.now() + self.timeout_type = None def to_full_dict(self, reverse_order=False): events = self.workflow_execution.events(reverse_order=reverse_order) @@ -33,7 +38,21 @@ class DecisionTask(object): def start(self, started_event_id): self.state = "STARTED" + self.started_timestamp = now_timestamp() self.started_event_id = started_event_id def complete(self): self.state = "COMPLETED" + + def has_timedout(self): + if self.state != "STARTED": + return False + # TODO: handle the "NONE" case + start_to_close_timeout = self.started_timestamp + \ + int(self.start_to_close_timeout) + return start_to_close_timeout < now_timestamp() + + def process_timeouts(self): + if self.has_timedout(): + self.state = "TIMED_OUT" + self.timeout_type = "START_TO_CLOSE" diff --git a/moto/swf/models/history_event.py b/moto/swf/models/history_event.py index 0b53f8659..c4d1e61a9 100644 --- a/moto/swf/models/history_event.py +++ b/moto/swf/models/history_event.py @@ -152,6 +152,12 @@ class HistoryEvent(object): if self.details: hsh["details"] = self.details return hsh + elif self.event_type == "DecisionTaskTimedOut": + return { + "scheduledEventId": self.scheduled_event_id, + "startedEventId": self.started_event_id, + "timeoutType": self.timeout_type, + } else: raise NotImplementedError( "HistoryEvent does not implement attributes for type '{0}'".format(self.event_type) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index cdc356ab5..0de757551 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -148,8 +148,21 @@ class WorkflowExecution(object): def _process_timeouts(self): self.should_schedule_decision_next = False + # TODO: process timeouts on workflow itself - # TODO: process timeouts on decision tasks + + # decision tasks timeouts + for task in self.decision_tasks: + if task.state == "STARTED" and task.has_timedout(): + self.should_schedule_decision_next = True + task.process_timeouts() + self._add_event( + "DecisionTaskTimedOut", + scheduled_event_id=task.scheduled_event_id, + started_event_id=task.started_event_id, + timeout_type=task.timeout_type, + ) + # activity tasks timeouts for task in self.activity_tasks: if task.open and task.has_timedout(): diff --git a/tests/test_swf/models/test_decision_task.py b/tests/test_swf/models/test_decision_task.py index f1156a19e..64268b380 100644 --- a/tests/test_swf/models/test_decision_task.py +++ b/tests/test_swf/models/test_decision_task.py @@ -1,3 +1,4 @@ +from freezegun import freeze_time from sure import expect from moto.swf.models import DecisionTask @@ -29,3 +30,20 @@ def test_decision_task_full_dict_representation(): dt.start(1234) fd = dt.to_full_dict() fd["startedEventId"].should.equal(1234) + +def test_decision_task_has_timedout(): + wfe = make_workflow_execution() + wft = wfe.workflow_type + dt = DecisionTask(wfe, 123) + dt.has_timedout().should.equal(False) + + with freeze_time("2015-01-01 12:00:00"): + dt.start(1234) + dt.has_timedout().should.equal(False) + + # activity task timeout is 300s == 5mins + with freeze_time("2015-01-01 12:06:00"): + dt.has_timedout().should.equal(True) + + dt.complete() + dt.has_timedout().should.equal(False) diff --git a/tests/test_swf/responses/test_timeouts.py b/tests/test_swf/responses/test_timeouts.py index ca2377795..97593d2c4 100644 --- a/tests/test_swf/responses/test_timeouts.py +++ b/tests/test_swf/responses/test_timeouts.py @@ -32,3 +32,34 @@ def test_activity_task_heartbeat_timeout(): attrs["timeoutType"].should.equal("HEARTBEAT") resp["events"][-1]["eventType"].should.equal("DecisionTaskScheduled") + +# Decision Task Start to Close timeout +# Default value in workflow helpers: 5 mins +@mock_swf +def test_decision_task_start_to_close_timeout(): + pass + with freeze_time("2015-01-01 12:00:00"): + conn = setup_workflow() + conn.poll_for_decision_task("test-domain", "queue")["taskToken"] + + with freeze_time("2015-01-01 12:04:30"): + resp = conn.get_workflow_execution_history("test-domain", conn.run_id, "uid-abcd1234") + + event_types = [evt["eventType"] for evt in resp["events"]] + event_types.should.equal( + ["WorkflowExecutionStarted", "DecisionTaskScheduled", "DecisionTaskStarted"] + ) + + with freeze_time("2015-01-01 12:05:30"): + # => Decision Task Start to Close timeout reached!! + resp = conn.get_workflow_execution_history("test-domain", conn.run_id, "uid-abcd1234") + + event_types = [evt["eventType"] for evt in resp["events"]] + event_types.should.equal( + ["WorkflowExecutionStarted", "DecisionTaskScheduled", "DecisionTaskStarted", + "DecisionTaskTimedOut", "DecisionTaskScheduled"] + ) + attrs = resp["events"][-2]["decisionTaskTimedOutEventAttributes"] + attrs.should.equal({ + "scheduledEventId": 2, "startedEventId": 3, "timeoutType": "START_TO_CLOSE" + }) From f38d23e4837e706b05e6c292f37340cc700d5d49 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Wed, 4 Nov 2015 22:03:58 +0100 Subject: [PATCH 70/94] Implement start to close timeout on SWF workflow executions --- moto/swf/models/history_event.py | 5 ++++ moto/swf/models/workflow_execution.py | 25 +++++++++++++++- .../models/test_workflow_execution.py | 12 ++++++++ tests/test_swf/responses/test_timeouts.py | 29 +++++++++++++++++++ tests/test_swf/utils.py | 2 +- 5 files changed, 71 insertions(+), 2 deletions(-) diff --git a/moto/swf/models/history_event.py b/moto/swf/models/history_event.py index c4d1e61a9..6e4002345 100644 --- a/moto/swf/models/history_event.py +++ b/moto/swf/models/history_event.py @@ -158,6 +158,11 @@ class HistoryEvent(object): "startedEventId": self.started_event_id, "timeoutType": self.timeout_type, } + elif self.event_type == "WorkflowExecutionTimedOut": + return { + "childPolicy": self.child_policy, + "timeoutType": self.timeout_type, + } else: raise NotImplementedError( "HistoryEvent does not implement attributes for type '{0}'".format(self.event_type) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 0de757551..3645b645d 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -59,6 +59,7 @@ class WorkflowExecution(object): self.parent = None self.start_timestamp = None self.tag_list = [] # TODO + self.timeout_type = None self.workflow_type = workflow_type # args processing # NB: the order follows boto/SWF order of exceptions appearance (if no @@ -149,7 +150,15 @@ class WorkflowExecution(object): def _process_timeouts(self): self.should_schedule_decision_next = False - # TODO: process timeouts on workflow itself + # workflow execution timeout + if self.has_timedout(): + self.process_timeouts() + # TODO: process child policy on child workflows here or in process_timeouts() + self._add_event( + "WorkflowExecutionTimedOut", + child_policy=self.child_policy, + timeout_type=self.timeout_type, + ) # decision tasks timeouts for task in self.decision_tasks: @@ -512,3 +521,17 @@ class WorkflowExecution(object): self.execution_status = "CLOSED" self.close_status = "TERMINATED" self.close_cause = "OPERATOR_INITIATED" + + def has_timedout(self): + if self.execution_status != "OPEN" or not self.start_timestamp: + return False + # TODO: handle the "NONE" case + start_to_close_timeout = self.start_timestamp + \ + int(self.execution_start_to_close_timeout) + return start_to_close_timeout < now_timestamp() + + def process_timeouts(self): + if self.has_timedout(): + self.execution_status = "CLOSED" + self.close_status = "TIMED_OUT" + self.timeout_type = "START_TO_CLOSE" diff --git a/tests/test_swf/models/test_workflow_execution.py b/tests/test_swf/models/test_workflow_execution.py index 5f14c51da..bc3585f78 100644 --- a/tests/test_swf/models/test_workflow_execution.py +++ b/tests/test_swf/models/test_workflow_execution.py @@ -394,3 +394,15 @@ def test_terminate(): last_event.event_type.should.equal("WorkflowExecutionTerminated") # take default child_policy if not provided (as here) last_event.child_policy.should.equal("ABANDON") + +def test_has_timedout(): + wfe = make_workflow_execution() + wfe.has_timedout().should.equal(False) + + with freeze_time("2015-01-01 12:00:00"): + wfe.start() + wfe.has_timedout().should.equal(False) + + with freeze_time("2015-01-01 14:01"): + # 2 hours timeout reached + wfe.has_timedout().should.equal(True) diff --git a/tests/test_swf/responses/test_timeouts.py b/tests/test_swf/responses/test_timeouts.py index 97593d2c4..ecc1e6068 100644 --- a/tests/test_swf/responses/test_timeouts.py +++ b/tests/test_swf/responses/test_timeouts.py @@ -63,3 +63,32 @@ def test_decision_task_start_to_close_timeout(): attrs.should.equal({ "scheduledEventId": 2, "startedEventId": 3, "timeoutType": "START_TO_CLOSE" }) + +# Workflow Execution Start to Close timeout +# Default value in workflow helpers: 2 hours +@mock_swf +def test_workflow_execution_start_to_close_timeout(): + pass + with freeze_time("2015-01-01 12:00:00"): + conn = setup_workflow() + + with freeze_time("2015-01-01 13:59:30"): + resp = conn.get_workflow_execution_history("test-domain", conn.run_id, "uid-abcd1234") + + event_types = [evt["eventType"] for evt in resp["events"]] + event_types.should.equal( + ["WorkflowExecutionStarted", "DecisionTaskScheduled"] + ) + + with freeze_time("2015-01-01 14:00:30"): + # => Workflow Execution Start to Close timeout reached!! + resp = conn.get_workflow_execution_history("test-domain", conn.run_id, "uid-abcd1234") + + event_types = [evt["eventType"] for evt in resp["events"]] + event_types.should.equal( + ["WorkflowExecutionStarted", "DecisionTaskScheduled", "WorkflowExecutionTimedOut"] + ) + attrs = resp["events"][-1]["workflowExecutionTimedOutEventAttributes"] + attrs.should.equal({ + "childPolicy": "ABANDON", "timeoutType": "START_TO_CLOSE" + }) diff --git a/tests/test_swf/utils.py b/tests/test_swf/utils.py index 70cbc4c13..7e7ecc1fb 100644 --- a/tests/test_swf/utils.py +++ b/tests/test_swf/utils.py @@ -44,7 +44,7 @@ def _generic_workflow_type_attributes(): ], { "task_list": "queue", "default_child_policy": "ABANDON", - "default_execution_start_to_close_timeout": "300", + "default_execution_start_to_close_timeout": "7200", "default_task_start_to_close_timeout": "300", } From 9c3996ff58711b62d317e96af1cf183e1cfbe73d Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Wed, 4 Nov 2015 22:25:27 +0100 Subject: [PATCH 71/94] Add WorkflowExecution.open to clarify code in some places --- moto/swf/models/__init__.py | 4 ++-- moto/swf/models/domain.py | 2 +- moto/swf/models/workflow_execution.py | 6 +++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index 4d70fbd0e..88e362f4e 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -243,7 +243,7 @@ class SWFBackend(BaseBackend): raise SWFValidationException("Invalid token") # decision task found, but WorflowExecution is CLOSED wfe = decision_task.workflow_execution - if wfe.execution_status != "OPEN": + if not wfe.open: raise SWFUnknownResourceFault( "execution", "WorkflowExecution=[workflowId={0}, runId={1}]".format( @@ -331,7 +331,7 @@ class SWFBackend(BaseBackend): raise SWFValidationException("Invalid token") # activity task found, but WorflowExecution is CLOSED wfe = activity_task.workflow_execution - if wfe.execution_status != "OPEN": + if not wfe.open: raise SWFUnknownResourceFault( "execution", "WorkflowExecution=[workflowId={0}, runId={1}]".format( diff --git a/moto/swf/models/domain.py b/moto/swf/models/domain.py index 55bf1b8e8..824782c59 100644 --- a/moto/swf/models/domain.py +++ b/moto/swf/models/domain.py @@ -83,7 +83,7 @@ class Domain(object): if w.workflow_id == workflow_id and w.run_id == run_id] else: _all = [w for w in self.workflow_executions - if w.workflow_id == workflow_id and w.execution_status == "OPEN"] + if w.workflow_id == workflow_id and w.open] # reduce wfe = _all[0] if _all else None # raise if closed / none diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 3645b645d..aee678a1d 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -523,7 +523,7 @@ class WorkflowExecution(object): self.close_cause = "OPERATOR_INITIATED" def has_timedout(self): - if self.execution_status != "OPEN" or not self.start_timestamp: + if not self.open or not self.start_timestamp: return False # TODO: handle the "NONE" case start_to_close_timeout = self.start_timestamp + \ @@ -535,3 +535,7 @@ class WorkflowExecution(object): self.execution_status = "CLOSED" self.close_status = "TIMED_OUT" self.timeout_type = "START_TO_CLOSE" + + @property + def open(self): + return self.execution_status == "OPEN" From 65c95ab5bcf339af24f52ae0d03a7bcfa65553fd Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Wed, 4 Nov 2015 22:35:45 +0100 Subject: [PATCH 72/94] Ensure activity and decision tasks cannot timeout on a closed workflow --- moto/swf/models/activity_task.py | 2 ++ moto/swf/models/decision_task.py | 2 +- tests/test_swf/models/test_activity_task.py | 21 +++++++++++++++++++++ tests/test_swf/models/test_decision_task.py | 16 +++++++++++++++- tests/test_swf/models/test_domain.py | 10 +++++----- 5 files changed, 44 insertions(+), 7 deletions(-) diff --git a/moto/swf/models/activity_task.py b/moto/swf/models/activity_task.py index ccddb0ba6..91f6f0b21 100644 --- a/moto/swf/models/activity_task.py +++ b/moto/swf/models/activity_task.py @@ -54,6 +54,8 @@ class ActivityTask(object): self.last_heartbeat_timestamp = now_timestamp() def has_timedout(self): + if not self.workflow_execution.open: + return False # TODO: handle the "NONE" case heartbeat_timeout_at = self.last_heartbeat_timestamp + \ int(self.timeouts["heartbeatTimeout"]) diff --git a/moto/swf/models/decision_task.py b/moto/swf/models/decision_task.py index dfc8f5687..d024ae118 100644 --- a/moto/swf/models/decision_task.py +++ b/moto/swf/models/decision_task.py @@ -45,7 +45,7 @@ class DecisionTask(object): self.state = "COMPLETED" def has_timedout(self): - if self.state != "STARTED": + if self.state != "STARTED" or not self.workflow_execution.open: return False # TODO: handle the "NONE" case start_to_close_timeout = self.started_timestamp + \ diff --git a/tests/test_swf/models/test_activity_task.py b/tests/test_swf/models/test_activity_task.py index 4057c947c..f9f0e2ef7 100644 --- a/tests/test_swf/models/test_activity_task.py +++ b/tests/test_swf/models/test_activity_task.py @@ -102,3 +102,24 @@ def test_activity_task_has_timedout(): task.process_timeouts() task.state.should.equal("TIMED_OUT") task.timeout_type.should.equal("HEARTBEAT") + +def test_activity_task_cannot_timeout_on_closed_workflow_execution(): + with freeze_time("2015-01-01 12:00:00"): + wfe = make_workflow_execution() + wfe.start() + + with freeze_time("2015-01-01 13:58:00"): + task = ActivityTask( + activity_id="my-activity-123", + activity_type="foo", + input="optional", + scheduled_event_id=117, + timeouts=ACTIVITY_TASK_TIMEOUTS, + workflow_execution=wfe, + ) + + with freeze_time("2015-01-01 14:10:00"): + task.has_timedout().should.equal(True) + wfe.has_timedout().should.equal(True) + wfe.process_timeouts() + task.has_timedout().should.equal(False) diff --git a/tests/test_swf/models/test_decision_task.py b/tests/test_swf/models/test_decision_task.py index 64268b380..f0efb94c0 100644 --- a/tests/test_swf/models/test_decision_task.py +++ b/tests/test_swf/models/test_decision_task.py @@ -33,7 +33,6 @@ def test_decision_task_full_dict_representation(): def test_decision_task_has_timedout(): wfe = make_workflow_execution() - wft = wfe.workflow_type dt = DecisionTask(wfe, 123) dt.has_timedout().should.equal(False) @@ -47,3 +46,18 @@ def test_decision_task_has_timedout(): dt.complete() dt.has_timedout().should.equal(False) + +def test_decision_task_cannot_timeout_on_closed_workflow_execution(): + with freeze_time("2015-01-01 12:00:00"): + wfe = make_workflow_execution() + wfe.start() + + with freeze_time("2015-01-01 13:55:00"): + dt = DecisionTask(wfe, 123) + dt.start(1234) + + with freeze_time("2015-01-01 14:10:00"): + dt.has_timedout().should.equal(True) + wfe.has_timedout().should.equal(True) + wfe.process_timeouts() + dt.has_timedout().should.equal(False) diff --git a/tests/test_swf/models/test_domain.py b/tests/test_swf/models/test_domain.py index 6430bc1ae..515e633f9 100644 --- a/tests/test_swf/models/test_domain.py +++ b/tests/test_swf/models/test_domain.py @@ -8,7 +8,7 @@ from moto.swf.models import Domain # Fake WorkflowExecution for tests purposes WorkflowExecution = namedtuple( "WorkflowExecution", - ["workflow_id", "run_id", "execution_status"] + ["workflow_id", "run_id", "execution_status", "open"] ) @@ -59,10 +59,10 @@ def test_domain_decision_tasks(): def test_domain_get_workflow_execution(): domain = Domain("my-domain", "60") - wfe1 = WorkflowExecution(workflow_id="wf-id-1", run_id="run-id-1", execution_status="OPEN") - wfe2 = WorkflowExecution(workflow_id="wf-id-1", run_id="run-id-2", execution_status="CLOSED") - wfe3 = WorkflowExecution(workflow_id="wf-id-2", run_id="run-id-3", execution_status="OPEN") - wfe4 = WorkflowExecution(workflow_id="wf-id-3", run_id="run-id-4", execution_status="CLOSED") + wfe1 = WorkflowExecution(workflow_id="wf-id-1", run_id="run-id-1", execution_status="OPEN", open=True) + wfe2 = WorkflowExecution(workflow_id="wf-id-1", run_id="run-id-2", execution_status="CLOSED", open=False) + wfe3 = WorkflowExecution(workflow_id="wf-id-2", run_id="run-id-3", execution_status="OPEN", open=True) + wfe4 = WorkflowExecution(workflow_id="wf-id-3", run_id="run-id-4", execution_status="CLOSED", open=False) domain.workflow_executions = [wfe1, wfe2, wfe3, wfe4] # get workflow execution through workflow_id and run_id From 61bb55005213bd1803d9b03a42971f41f2fc0458 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Thu, 5 Nov 2015 01:12:51 +0100 Subject: [PATCH 73/94] Ensure activity and decision tasks cannot progress on a closed workflow This is a second barrier because I'm a little nervous about this and I don't want moto/swf to make any activity progress while in the real world service, it's strictly impossible once the execution is closed. Python doesn't seem to have any nice way of freezing an object so here we go with a manual boundary... --- moto/swf/exceptions.py | 5 +++++ moto/swf/models/activity_task.py | 15 +++++++++++++-- moto/swf/models/decision_task.py | 14 ++++++++++++-- tests/test_swf/models/test_activity_task.py | 19 +++++++++++++++++++ tests/test_swf/models/test_decision_task.py | 11 +++++++++++ 5 files changed, 60 insertions(+), 4 deletions(-) diff --git a/moto/swf/exceptions.py b/moto/swf/exceptions.py index abeb348b7..a2e12fd73 100644 --- a/moto/swf/exceptions.py +++ b/moto/swf/exceptions.py @@ -119,3 +119,8 @@ class SWFDecisionValidationException(SWFClientError): prefix.format(count) + "; ".join(messages), "com.amazon.coral.validate#ValidationException" ) + + +class SWFWorkflowExecutionClosedError(Exception): + def __str__(self): + return repr("Cannot change this object because the WorkflowExecution is closed") diff --git a/moto/swf/models/activity_task.py b/moto/swf/models/activity_task.py index 91f6f0b21..1f011cb8d 100644 --- a/moto/swf/models/activity_task.py +++ b/moto/swf/models/activity_task.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from datetime import datetime import uuid +from ..exceptions import SWFWorkflowExecutionClosedError from ..utils import now_timestamp @@ -24,6 +25,10 @@ class ActivityTask(object): # but that shouldn't be a problem for tests self.scheduled_at = datetime.now() + def _check_workflow_execution_open(self): + if not self.workflow_execution.open: + raise SWFWorkflowExecutionClosedError() + @property def open(self): return self.state in ["SCHEDULED", "STARTED"] @@ -45,9 +50,11 @@ class ActivityTask(object): self.started_event_id = started_event_id def complete(self): + self._check_workflow_execution_open() self.state = "COMPLETED" def fail(self): + self._check_workflow_execution_open() self.state = "FAILED" def reset_heartbeat_clock(self): @@ -63,5 +70,9 @@ class ActivityTask(object): def process_timeouts(self): if self.has_timedout(): - self.state = "TIMED_OUT" - self.timeout_type = "HEARTBEAT" + self.timeout() + + def timeout(self): + self._check_workflow_execution_open() + self.state = "TIMED_OUT" + self.timeout_type = "HEARTBEAT" diff --git a/moto/swf/models/decision_task.py b/moto/swf/models/decision_task.py index d024ae118..23822976c 100644 --- a/moto/swf/models/decision_task.py +++ b/moto/swf/models/decision_task.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from datetime import datetime import uuid +from ..exceptions import SWFWorkflowExecutionClosedError from ..utils import now_timestamp @@ -21,6 +22,10 @@ class DecisionTask(object): self.scheduled_at = datetime.now() self.timeout_type = None + def _check_workflow_execution_open(self): + if not self.workflow_execution.open: + raise SWFWorkflowExecutionClosedError() + def to_full_dict(self, reverse_order=False): events = self.workflow_execution.events(reverse_order=reverse_order) hsh = { @@ -42,6 +47,7 @@ class DecisionTask(object): self.started_event_id = started_event_id def complete(self): + self._check_workflow_execution_open() self.state = "COMPLETED" def has_timedout(self): @@ -54,5 +60,9 @@ class DecisionTask(object): def process_timeouts(self): if self.has_timedout(): - self.state = "TIMED_OUT" - self.timeout_type = "START_TO_CLOSE" + self.timeout() + + def timeout(self): + self._check_workflow_execution_open() + self.state = "TIMED_OUT" + self.timeout_type = "START_TO_CLOSE" diff --git a/tests/test_swf/models/test_activity_task.py b/tests/test_swf/models/test_activity_task.py index f9f0e2ef7..8b81cbdd5 100644 --- a/tests/test_swf/models/test_activity_task.py +++ b/tests/test_swf/models/test_activity_task.py @@ -1,6 +1,7 @@ from freezegun import freeze_time from sure import expect +from moto.swf.exceptions import SWFWorkflowExecutionClosedError from moto.swf.models import ( ActivityTask, ActivityType, @@ -123,3 +124,21 @@ def test_activity_task_cannot_timeout_on_closed_workflow_execution(): wfe.has_timedout().should.equal(True) wfe.process_timeouts() task.has_timedout().should.equal(False) + +def test_activity_task_cannot_change_state_on_closed_workflow_execution(): + wfe = make_workflow_execution() + wfe.start() + + task = ActivityTask( + activity_id="my-activity-123", + activity_type="foo", + input="optional", + scheduled_event_id=117, + timeouts=ACTIVITY_TASK_TIMEOUTS, + workflow_execution=wfe, + ) + wfe.complete(123) + + task.timeout.when.called_with().should.throw(SWFWorkflowExecutionClosedError) + task.complete.when.called_with().should.throw(SWFWorkflowExecutionClosedError) + task.fail.when.called_with().should.throw(SWFWorkflowExecutionClosedError) diff --git a/tests/test_swf/models/test_decision_task.py b/tests/test_swf/models/test_decision_task.py index f0efb94c0..ae2b59fdc 100644 --- a/tests/test_swf/models/test_decision_task.py +++ b/tests/test_swf/models/test_decision_task.py @@ -1,6 +1,7 @@ from freezegun import freeze_time from sure import expect +from moto.swf.exceptions import SWFWorkflowExecutionClosedError from moto.swf.models import DecisionTask from ..utils import make_workflow_execution @@ -61,3 +62,13 @@ def test_decision_task_cannot_timeout_on_closed_workflow_execution(): wfe.has_timedout().should.equal(True) wfe.process_timeouts() dt.has_timedout().should.equal(False) + +def test_decision_task_cannot_change_state_on_closed_workflow_execution(): + wfe = make_workflow_execution() + wfe.start() + task = DecisionTask(wfe, 123) + + wfe.complete(123) + + task.timeout.when.called_with().should.throw(SWFWorkflowExecutionClosedError) + task.complete.when.called_with().should.throw(SWFWorkflowExecutionClosedError) From d6185857909b99fea78ea92bc5803aca025497a8 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Thu, 5 Nov 2015 02:22:02 +0100 Subject: [PATCH 74/94] Refactor timeouts processing so it will be easier to compute them in order --- moto/swf/models/__init__.py | 1 + moto/swf/models/activity_task.py | 20 +++++++---- moto/swf/models/decision_task.py | 22 +++++++----- moto/swf/models/timeout.py | 12 +++++++ moto/swf/models/workflow_execution.py | 34 +++++++++++-------- tests/test_swf/models/test_activity_task.py | 15 ++++---- tests/test_swf/models/test_decision_task.py | 20 +++++------ tests/test_swf/models/test_timeout.py | 18 ++++++++++ .../models/test_workflow_execution.py | 11 +++--- 9 files changed, 101 insertions(+), 52 deletions(-) create mode 100644 moto/swf/models/timeout.py create mode 100644 tests/test_swf/models/test_timeout.py diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index 88e362f4e..f4fe246f8 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -20,6 +20,7 @@ from .decision_task import DecisionTask from .domain import Domain from .generic_type import GenericType from .history_event import HistoryEvent +from .timeout import Timeout from .workflow_type import WorkflowType from .workflow_execution import WorkflowExecution diff --git a/moto/swf/models/activity_task.py b/moto/swf/models/activity_task.py index 1f011cb8d..76d0eac70 100644 --- a/moto/swf/models/activity_task.py +++ b/moto/swf/models/activity_task.py @@ -5,6 +5,8 @@ import uuid from ..exceptions import SWFWorkflowExecutionClosedError from ..utils import now_timestamp +from .timeout import Timeout + class ActivityTask(object): def __init__(self, activity_id, activity_type, scheduled_event_id, @@ -60,19 +62,23 @@ class ActivityTask(object): def reset_heartbeat_clock(self): self.last_heartbeat_timestamp = now_timestamp() - def has_timedout(self): + def first_timeout(self): if not self.workflow_execution.open: - return False + return None # TODO: handle the "NONE" case heartbeat_timeout_at = self.last_heartbeat_timestamp + \ int(self.timeouts["heartbeatTimeout"]) - return heartbeat_timeout_at < now_timestamp() + _timeout = Timeout(self, heartbeat_timeout_at, "HEARTBEAT") + if _timeout.reached: + return _timeout + def process_timeouts(self): - if self.has_timedout(): - self.timeout() + _timeout = self.first_timeout() + if _timeout: + self.timeout(_timeout) - def timeout(self): + def timeout(self, _timeout): self._check_workflow_execution_open() self.state = "TIMED_OUT" - self.timeout_type = "HEARTBEAT" + self.timeout_type = _timeout.kind diff --git a/moto/swf/models/decision_task.py b/moto/swf/models/decision_task.py index 23822976c..fb7b9d080 100644 --- a/moto/swf/models/decision_task.py +++ b/moto/swf/models/decision_task.py @@ -5,6 +5,8 @@ import uuid from ..exceptions import SWFWorkflowExecutionClosedError from ..utils import now_timestamp +from .timeout import Timeout + class DecisionTask(object): def __init__(self, workflow_execution, scheduled_event_id): @@ -50,19 +52,21 @@ class DecisionTask(object): self._check_workflow_execution_open() self.state = "COMPLETED" - def has_timedout(self): + def first_timeout(self): if self.state != "STARTED" or not self.workflow_execution.open: - return False + return None # TODO: handle the "NONE" case - start_to_close_timeout = self.started_timestamp + \ - int(self.start_to_close_timeout) - return start_to_close_timeout < now_timestamp() + start_to_close_at = self.started_timestamp + int(self.start_to_close_timeout) + _timeout = Timeout(self, start_to_close_at, "START_TO_CLOSE") + if _timeout.reached: + return _timeout def process_timeouts(self): - if self.has_timedout(): - self.timeout() + _timeout = self.first_timeout() + if _timeout: + self.timeout(_timeout) - def timeout(self): + def timeout(self, _timeout): self._check_workflow_execution_open() self.state = "TIMED_OUT" - self.timeout_type = "START_TO_CLOSE" + self.timeout_type = _timeout.kind diff --git a/moto/swf/models/timeout.py b/moto/swf/models/timeout.py new file mode 100644 index 000000000..66cb3b84c --- /dev/null +++ b/moto/swf/models/timeout.py @@ -0,0 +1,12 @@ +from ..utils import now_timestamp + + +class Timeout(object): + def __init__(self, obj, timestamp, kind): + self.obj = obj + self.timestamp = timestamp + self.kind = kind + + @property + def reached(self): + return now_timestamp() >= self.timestamp diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index aee678a1d..2c1e74a02 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -18,6 +18,7 @@ from .activity_task import ActivityTask from .activity_type import ActivityType from .decision_task import DecisionTask from .history_event import HistoryEvent +from .timeout import Timeout # TODO: extract decision related logic into a Decision class @@ -151,8 +152,9 @@ class WorkflowExecution(object): self.should_schedule_decision_next = False # workflow execution timeout - if self.has_timedout(): - self.process_timeouts() + _timeout = self.first_timeout() + if _timeout: + self.execute_timeout(_timeout) # TODO: process child policy on child workflows here or in process_timeouts() self._add_event( "WorkflowExecutionTimedOut", @@ -162,7 +164,7 @@ class WorkflowExecution(object): # decision tasks timeouts for task in self.decision_tasks: - if task.state == "STARTED" and task.has_timedout(): + if task.state == "STARTED" and task.first_timeout(): self.should_schedule_decision_next = True task.process_timeouts() self._add_event( @@ -174,7 +176,7 @@ class WorkflowExecution(object): # activity tasks timeouts for task in self.activity_tasks: - if task.open and task.has_timedout(): + if task.open and task.first_timeout(): self.should_schedule_decision_next = True task.process_timeouts() self._add_event( @@ -522,19 +524,23 @@ class WorkflowExecution(object): self.close_status = "TERMINATED" self.close_cause = "OPERATOR_INITIATED" - def has_timedout(self): + def first_timeout(self): if not self.open or not self.start_timestamp: - return False - # TODO: handle the "NONE" case - start_to_close_timeout = self.start_timestamp + \ - int(self.execution_start_to_close_timeout) - return start_to_close_timeout < now_timestamp() + return None + start_to_close_at = self.start_timestamp + int(self.execution_start_to_close_timeout) + _timeout = Timeout(self, start_to_close_at, "START_TO_CLOSE") + if _timeout.reached: + return _timeout + + def execute_timeout(self, timeout): + self.execution_status = "CLOSED" + self.close_status = "TIMED_OUT" + self.timeout_type = timeout.kind def process_timeouts(self): - if self.has_timedout(): - self.execution_status = "CLOSED" - self.close_status = "TIMED_OUT" - self.timeout_type = "START_TO_CLOSE" + _timeout = self.first_timeout() + if _timeout: + self.execute_timeout(_timeout) @property def open(self): diff --git a/tests/test_swf/models/test_activity_task.py b/tests/test_swf/models/test_activity_task.py index 8b81cbdd5..ef4823cc7 100644 --- a/tests/test_swf/models/test_activity_task.py +++ b/tests/test_swf/models/test_activity_task.py @@ -5,6 +5,7 @@ from moto.swf.exceptions import SWFWorkflowExecutionClosedError from moto.swf.models import ( ActivityTask, ActivityType, + Timeout, ) from ..utils import make_workflow_execution, ACTIVITY_TASK_TIMEOUTS @@ -83,7 +84,7 @@ def test_activity_task_reset_heartbeat_clock(): task.last_heartbeat_timestamp.should.equal(1420117200.0) -def test_activity_task_has_timedout(): +def test_activity_task_first_timeout(): wfe = make_workflow_execution() with freeze_time("2015-01-01 12:00:00"): @@ -95,11 +96,11 @@ def test_activity_task_has_timedout(): timeouts=ACTIVITY_TASK_TIMEOUTS, workflow_execution=wfe, ) - task.has_timedout().should.equal(False) + task.first_timeout().should.be.none # activity task timeout is 300s == 5mins with freeze_time("2015-01-01 12:06:00"): - task.has_timedout().should.equal(True) + task.first_timeout().should.be.a(Timeout) task.process_timeouts() task.state.should.equal("TIMED_OUT") task.timeout_type.should.equal("HEARTBEAT") @@ -120,10 +121,10 @@ def test_activity_task_cannot_timeout_on_closed_workflow_execution(): ) with freeze_time("2015-01-01 14:10:00"): - task.has_timedout().should.equal(True) - wfe.has_timedout().should.equal(True) + task.first_timeout().should.be.a(Timeout) + wfe.first_timeout().should.be.a(Timeout) wfe.process_timeouts() - task.has_timedout().should.equal(False) + task.first_timeout().should.be.none def test_activity_task_cannot_change_state_on_closed_workflow_execution(): wfe = make_workflow_execution() @@ -139,6 +140,6 @@ def test_activity_task_cannot_change_state_on_closed_workflow_execution(): ) wfe.complete(123) - task.timeout.when.called_with().should.throw(SWFWorkflowExecutionClosedError) + task.timeout.when.called_with(Timeout(task, 0, "foo")).should.throw(SWFWorkflowExecutionClosedError) task.complete.when.called_with().should.throw(SWFWorkflowExecutionClosedError) task.fail.when.called_with().should.throw(SWFWorkflowExecutionClosedError) diff --git a/tests/test_swf/models/test_decision_task.py b/tests/test_swf/models/test_decision_task.py index ae2b59fdc..f85b83ebd 100644 --- a/tests/test_swf/models/test_decision_task.py +++ b/tests/test_swf/models/test_decision_task.py @@ -2,7 +2,7 @@ from freezegun import freeze_time from sure import expect from moto.swf.exceptions import SWFWorkflowExecutionClosedError -from moto.swf.models import DecisionTask +from moto.swf.models import DecisionTask, Timeout from ..utils import make_workflow_execution @@ -32,21 +32,21 @@ def test_decision_task_full_dict_representation(): fd = dt.to_full_dict() fd["startedEventId"].should.equal(1234) -def test_decision_task_has_timedout(): +def test_decision_task_first_timeout(): wfe = make_workflow_execution() dt = DecisionTask(wfe, 123) - dt.has_timedout().should.equal(False) + dt.first_timeout().should.be.none with freeze_time("2015-01-01 12:00:00"): dt.start(1234) - dt.has_timedout().should.equal(False) + dt.first_timeout().should.be.none # activity task timeout is 300s == 5mins with freeze_time("2015-01-01 12:06:00"): - dt.has_timedout().should.equal(True) + dt.first_timeout().should.be.a(Timeout) dt.complete() - dt.has_timedout().should.equal(False) + dt.first_timeout().should.be.none def test_decision_task_cannot_timeout_on_closed_workflow_execution(): with freeze_time("2015-01-01 12:00:00"): @@ -58,10 +58,10 @@ def test_decision_task_cannot_timeout_on_closed_workflow_execution(): dt.start(1234) with freeze_time("2015-01-01 14:10:00"): - dt.has_timedout().should.equal(True) - wfe.has_timedout().should.equal(True) + dt.first_timeout().should.be.a(Timeout) + wfe.first_timeout().should.be.a(Timeout) wfe.process_timeouts() - dt.has_timedout().should.equal(False) + dt.first_timeout().should.be.none def test_decision_task_cannot_change_state_on_closed_workflow_execution(): wfe = make_workflow_execution() @@ -70,5 +70,5 @@ def test_decision_task_cannot_change_state_on_closed_workflow_execution(): wfe.complete(123) - task.timeout.when.called_with().should.throw(SWFWorkflowExecutionClosedError) + task.timeout.when.called_with(Timeout(task, 0, "foo")).should.throw(SWFWorkflowExecutionClosedError) task.complete.when.called_with().should.throw(SWFWorkflowExecutionClosedError) diff --git a/tests/test_swf/models/test_timeout.py b/tests/test_swf/models/test_timeout.py new file mode 100644 index 000000000..6ba26b1d2 --- /dev/null +++ b/tests/test_swf/models/test_timeout.py @@ -0,0 +1,18 @@ +from freezegun import freeze_time +from sure import expect + +from moto.swf.models import Timeout, WorkflowExecution + +from ..utils import make_workflow_execution + +def test_timeout_creation(): + wfe = make_workflow_execution() + + # epoch 1420113600 == "2015-01-01 13:00:00" + timeout = Timeout(wfe, 1420117200, "START_TO_CLOSE") + + with freeze_time("2015-01-01 12:00:00"): + timeout.reached.should.be.falsy + + with freeze_time("2015-01-01 13:00:00"): + timeout.reached.should.be.truthy diff --git a/tests/test_swf/models/test_workflow_execution.py b/tests/test_swf/models/test_workflow_execution.py index bc3585f78..e533f925b 100644 --- a/tests/test_swf/models/test_workflow_execution.py +++ b/tests/test_swf/models/test_workflow_execution.py @@ -3,6 +3,7 @@ from freezegun import freeze_time from moto.swf.models import ( ActivityType, + Timeout, WorkflowType, WorkflowExecution, ) @@ -106,7 +107,7 @@ def test_workflow_execution_medium_dict_representation(): md["workflowType"].should.equal(wf_type.to_short_dict()) md["startTimestamp"].should.be.a('float') md["executionStatus"].should.equal("OPEN") - md["cancelRequested"].should.equal(False) + md["cancelRequested"].should.be.falsy md.should_not.contain("tagList") wfe.tag_list = ["foo", "bar", "baz"] @@ -395,14 +396,14 @@ def test_terminate(): # take default child_policy if not provided (as here) last_event.child_policy.should.equal("ABANDON") -def test_has_timedout(): +def test_first_timeout(): wfe = make_workflow_execution() - wfe.has_timedout().should.equal(False) + wfe.first_timeout().should.be.none with freeze_time("2015-01-01 12:00:00"): wfe.start() - wfe.has_timedout().should.equal(False) + wfe.first_timeout().should.be.none with freeze_time("2015-01-01 14:01"): # 2 hours timeout reached - wfe.has_timedout().should.equal(True) + wfe.first_timeout().should.be.a(Timeout) From 65c35bfa691cec8a59ad196269692d50b155a709 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Thu, 5 Nov 2015 02:40:16 +0100 Subject: [PATCH 75/94] Make timeout events appear at the right time in workflow history --- moto/swf/models/workflow_execution.py | 10 ++++++++-- tests/test_swf/responses/test_timeouts.py | 6 ++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 2c1e74a02..10ff48db1 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -159,16 +159,19 @@ class WorkflowExecution(object): self._add_event( "WorkflowExecutionTimedOut", child_policy=self.child_policy, + event_timestamp=_timeout.timestamp, timeout_type=self.timeout_type, ) # decision tasks timeouts for task in self.decision_tasks: - if task.state == "STARTED" and task.first_timeout(): + _timeout = task.first_timeout() + if task.state == "STARTED" and _timeout: self.should_schedule_decision_next = True task.process_timeouts() self._add_event( "DecisionTaskTimedOut", + event_timestamp=_timeout.timestamp, scheduled_event_id=task.scheduled_event_id, started_event_id=task.started_event_id, timeout_type=task.timeout_type, @@ -176,17 +179,20 @@ class WorkflowExecution(object): # activity tasks timeouts for task in self.activity_tasks: - if task.open and task.first_timeout(): + _timeout = task.first_timeout() + if task.open and _timeout: self.should_schedule_decision_next = True task.process_timeouts() self._add_event( "ActivityTaskTimedOut", details=task.details, + event_timestamp=_timeout.timestamp, scheduled_event_id=task.scheduled_event_id, started_event_id=task.started_event_id, timeout_type=task.timeout_type, ) # schedule decision task if needed + # TODO: make decision appear as if it has been scheduled immediately after the timeout if self.should_schedule_decision_next: self.schedule_decision_task() diff --git a/tests/test_swf/responses/test_timeouts.py b/tests/test_swf/responses/test_timeouts.py index ecc1e6068..237deaea8 100644 --- a/tests/test_swf/responses/test_timeouts.py +++ b/tests/test_swf/responses/test_timeouts.py @@ -30,6 +30,8 @@ def test_activity_task_heartbeat_timeout(): resp["events"][-2]["eventType"].should.equal("ActivityTaskTimedOut") attrs = resp["events"][-2]["activityTaskTimedOutEventAttributes"] attrs["timeoutType"].should.equal("HEARTBEAT") + # checks that event has been emitted at 12:05:00, not 12:05:30 + resp["events"][-2]["eventTimestamp"].should.equal(1420113900) resp["events"][-1]["eventType"].should.equal("DecisionTaskScheduled") @@ -63,6 +65,8 @@ def test_decision_task_start_to_close_timeout(): attrs.should.equal({ "scheduledEventId": 2, "startedEventId": 3, "timeoutType": "START_TO_CLOSE" }) + # checks that event has been emitted at 12:05:00, not 12:05:30 + resp["events"][-2]["eventTimestamp"].should.equal(1420113900) # Workflow Execution Start to Close timeout # Default value in workflow helpers: 2 hours @@ -92,3 +96,5 @@ def test_workflow_execution_start_to_close_timeout(): attrs.should.equal({ "childPolicy": "ABANDON", "timeoutType": "START_TO_CLOSE" }) + # checks that event has been emitted at 14:00:00, not 14:00:30 + resp["events"][-1]["eventTimestamp"].should.equal(1420120800) From d007dfe3ff43785ecd69f8751c759b62f5ee434f Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Thu, 5 Nov 2015 02:47:05 +0100 Subject: [PATCH 76/94] Remove process_timeouts() method in favor of timeout() and a helper for tests --- moto/swf/models/workflow_execution.py | 15 +++++---------- tests/test_swf/models/test_activity_task.py | 10 +++++++--- tests/test_swf/models/test_decision_task.py | 4 ++-- tests/test_swf/utils.py | 7 +++++++ 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 10ff48db1..0f619dca1 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -154,8 +154,8 @@ class WorkflowExecution(object): # workflow execution timeout _timeout = self.first_timeout() if _timeout: - self.execute_timeout(_timeout) - # TODO: process child policy on child workflows here or in process_timeouts() + self.timeout(_timeout) + # TODO: process child policy on child workflows here or in timeout() self._add_event( "WorkflowExecutionTimedOut", child_policy=self.child_policy, @@ -168,7 +168,7 @@ class WorkflowExecution(object): _timeout = task.first_timeout() if task.state == "STARTED" and _timeout: self.should_schedule_decision_next = True - task.process_timeouts() + task.timeout(_timeout) self._add_event( "DecisionTaskTimedOut", event_timestamp=_timeout.timestamp, @@ -182,7 +182,7 @@ class WorkflowExecution(object): _timeout = task.first_timeout() if task.open and _timeout: self.should_schedule_decision_next = True - task.process_timeouts() + task.timeout(_timeout) self._add_event( "ActivityTaskTimedOut", details=task.details, @@ -538,16 +538,11 @@ class WorkflowExecution(object): if _timeout.reached: return _timeout - def execute_timeout(self, timeout): + def timeout(self, timeout): self.execution_status = "CLOSED" self.close_status = "TIMED_OUT" self.timeout_type = timeout.kind - def process_timeouts(self): - _timeout = self.first_timeout() - if _timeout: - self.execute_timeout(_timeout) - @property def open(self): return self.execution_status == "OPEN" diff --git a/tests/test_swf/models/test_activity_task.py b/tests/test_swf/models/test_activity_task.py index ef4823cc7..4dbb3cc17 100644 --- a/tests/test_swf/models/test_activity_task.py +++ b/tests/test_swf/models/test_activity_task.py @@ -8,7 +8,11 @@ from moto.swf.models import ( Timeout, ) -from ..utils import make_workflow_execution, ACTIVITY_TASK_TIMEOUTS +from ..utils import ( + ACTIVITY_TASK_TIMEOUTS, + make_workflow_execution, + process_first_timeout, +) def test_activity_task_creation(): @@ -101,7 +105,7 @@ def test_activity_task_first_timeout(): # activity task timeout is 300s == 5mins with freeze_time("2015-01-01 12:06:00"): task.first_timeout().should.be.a(Timeout) - task.process_timeouts() + process_first_timeout(task) task.state.should.equal("TIMED_OUT") task.timeout_type.should.equal("HEARTBEAT") @@ -123,7 +127,7 @@ def test_activity_task_cannot_timeout_on_closed_workflow_execution(): with freeze_time("2015-01-01 14:10:00"): task.first_timeout().should.be.a(Timeout) wfe.first_timeout().should.be.a(Timeout) - wfe.process_timeouts() + process_first_timeout(wfe) task.first_timeout().should.be.none def test_activity_task_cannot_change_state_on_closed_workflow_execution(): diff --git a/tests/test_swf/models/test_decision_task.py b/tests/test_swf/models/test_decision_task.py index f85b83ebd..2c4439dd5 100644 --- a/tests/test_swf/models/test_decision_task.py +++ b/tests/test_swf/models/test_decision_task.py @@ -4,7 +4,7 @@ from sure import expect from moto.swf.exceptions import SWFWorkflowExecutionClosedError from moto.swf.models import DecisionTask, Timeout -from ..utils import make_workflow_execution +from ..utils import make_workflow_execution, process_first_timeout def test_decision_task_creation(): @@ -60,7 +60,7 @@ def test_decision_task_cannot_timeout_on_closed_workflow_execution(): with freeze_time("2015-01-01 14:10:00"): dt.first_timeout().should.be.a(Timeout) wfe.first_timeout().should.be.a(Timeout) - wfe.process_timeouts() + process_first_timeout(wfe) dt.first_timeout().should.be.none def test_decision_task_cannot_change_state_on_closed_workflow_execution(): diff --git a/tests/test_swf/utils.py b/tests/test_swf/utils.py index 7e7ecc1fb..d98294ea2 100644 --- a/tests/test_swf/utils.py +++ b/tests/test_swf/utils.py @@ -82,3 +82,10 @@ def setup_workflow(): wfe = conn.start_workflow_execution("test-domain", "uid-abcd1234", "test-workflow", "v1.0") conn.run_id = wfe["runId"] return conn + + +# A helper for processing the first timeout on a given object +def process_first_timeout(obj): + _timeout = obj.first_timeout() + if _timeout: + obj.timeout(_timeout) From 6027bf15c1dbc15cf27959021ab8099296bb44a3 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Thu, 5 Nov 2015 02:51:55 +0100 Subject: [PATCH 77/94] Move some timeout conditionals to concerned models --- moto/swf/models/activity_task.py | 2 +- moto/swf/models/decision_task.py | 6 +++++- moto/swf/models/workflow_execution.py | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/moto/swf/models/activity_task.py b/moto/swf/models/activity_task.py index 76d0eac70..eef8350e2 100644 --- a/moto/swf/models/activity_task.py +++ b/moto/swf/models/activity_task.py @@ -63,7 +63,7 @@ class ActivityTask(object): self.last_heartbeat_timestamp = now_timestamp() def first_timeout(self): - if not self.workflow_execution.open: + if not self.open or not self.workflow_execution.open: return None # TODO: handle the "NONE" case heartbeat_timeout_at = self.last_heartbeat_timestamp + \ diff --git a/moto/swf/models/decision_task.py b/moto/swf/models/decision_task.py index fb7b9d080..5474332a5 100644 --- a/moto/swf/models/decision_task.py +++ b/moto/swf/models/decision_task.py @@ -24,6 +24,10 @@ class DecisionTask(object): self.scheduled_at = datetime.now() self.timeout_type = None + @property + def started(self): + return self.state == "STARTED" + def _check_workflow_execution_open(self): if not self.workflow_execution.open: raise SWFWorkflowExecutionClosedError() @@ -53,7 +57,7 @@ class DecisionTask(object): self.state = "COMPLETED" def first_timeout(self): - if self.state != "STARTED" or not self.workflow_execution.open: + if not self.started or not self.workflow_execution.open: return None # TODO: handle the "NONE" case start_to_close_at = self.started_timestamp + int(self.start_to_close_timeout) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 0f619dca1..9a54f1729 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -166,7 +166,7 @@ class WorkflowExecution(object): # decision tasks timeouts for task in self.decision_tasks: _timeout = task.first_timeout() - if task.state == "STARTED" and _timeout: + if _timeout: self.should_schedule_decision_next = True task.timeout(_timeout) self._add_event( @@ -180,7 +180,7 @@ class WorkflowExecution(object): # activity tasks timeouts for task in self.activity_tasks: _timeout = task.first_timeout() - if task.open and _timeout: + if _timeout: self.should_schedule_decision_next = True task.timeout(_timeout) self._add_event( From 7f2cbb79b03987e0c35ff67431b6b2bb2c9d0ce8 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 9 Nov 2015 22:06:03 +0100 Subject: [PATCH 78/94] Refactor SWF workflow execution to ease next timeout change --- moto/swf/models/workflow_execution.py | 69 +++++++++++++++------------ 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 9a54f1729..0c8202bda 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -155,42 +155,19 @@ class WorkflowExecution(object): _timeout = self.first_timeout() if _timeout: self.timeout(_timeout) - # TODO: process child policy on child workflows here or in timeout() - self._add_event( - "WorkflowExecutionTimedOut", - child_policy=self.child_policy, - event_timestamp=_timeout.timestamp, - timeout_type=self.timeout_type, - ) # decision tasks timeouts for task in self.decision_tasks: _timeout = task.first_timeout() if _timeout: - self.should_schedule_decision_next = True - task.timeout(_timeout) - self._add_event( - "DecisionTaskTimedOut", - event_timestamp=_timeout.timestamp, - scheduled_event_id=task.scheduled_event_id, - started_event_id=task.started_event_id, - timeout_type=task.timeout_type, - ) + self.timeout_decision_task(_timeout) # activity tasks timeouts for task in self.activity_tasks: _timeout = task.first_timeout() if _timeout: - self.should_schedule_decision_next = True - task.timeout(_timeout) - self._add_event( - "ActivityTaskTimedOut", - details=task.details, - event_timestamp=_timeout.timestamp, - scheduled_event_id=task.scheduled_event_id, - started_event_id=task.started_event_id, - timeout_type=task.timeout_type, - ) + self.timeout_activity_task(_timeout) + # schedule decision task if needed # TODO: make decision appear as if it has been scheduled immediately after the timeout if self.should_schedule_decision_next: @@ -376,7 +353,7 @@ class WorkflowExecution(object): self.execution_status = "CLOSED" self.close_status = "COMPLETED" self.close_timestamp = now_timestamp() - evt = self._add_event( + self._add_event( "WorkflowExecutionCompleted", decision_task_completed_event_id=event_id, result=result, @@ -387,7 +364,7 @@ class WorkflowExecution(object): self.execution_status = "CLOSED" self.close_status = "FAILED" self.close_timestamp = now_timestamp() - evt = self._add_event( + self._add_event( "WorkflowExecutionFailed", decision_task_completed_event_id=event_id, details=details, @@ -487,7 +464,7 @@ class WorkflowExecution(object): def complete_activity_task(self, task_token, result=None): task = self._find_activity_task(task_token) - evt = self._add_event( + self._add_event( "ActivityTaskCompleted", scheduled_event_id=task.scheduled_event_id, started_event_id=task.started_event_id, @@ -500,7 +477,7 @@ class WorkflowExecution(object): def fail_activity_task(self, task_token, reason=None, details=None): task = self._find_activity_task(task_token) - evt = self._add_event( + self._add_event( "ActivityTaskFailed", scheduled_event_id=task.scheduled_event_id, started_event_id=task.started_event_id, @@ -539,9 +516,41 @@ class WorkflowExecution(object): return _timeout def timeout(self, timeout): + # TODO: process child policy on child workflows here or in the triggering function self.execution_status = "CLOSED" self.close_status = "TIMED_OUT" self.timeout_type = timeout.kind + self._add_event( + "WorkflowExecutionTimedOut", + child_policy=self.child_policy, + event_timestamp=timeout.timestamp, + timeout_type=self.timeout_type, + ) + + def timeout_decision_task(self, _timeout): + self.should_schedule_decision_next = True + task = _timeout.obj + task.timeout(_timeout) + self._add_event( + "DecisionTaskTimedOut", + event_timestamp=_timeout.timestamp, + scheduled_event_id=task.scheduled_event_id, + started_event_id=task.started_event_id, + timeout_type=task.timeout_type, + ) + + def timeout_activity_task(self, _timeout): + self.should_schedule_decision_next = True + task = _timeout.obj + task.timeout(_timeout) + self._add_event( + "ActivityTaskTimedOut", + details=task.details, + event_timestamp=_timeout.timestamp, + scheduled_event_id=task.scheduled_event_id, + started_event_id=task.started_event_id, + timeout_type=task.timeout_type, + ) @property def open(self): From 248975d4e6bcdc7845f1b97d529bee9dd0f8705c Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 9 Nov 2015 23:44:49 +0100 Subject: [PATCH 79/94] Improve SWF timeouts processing: now processed in order, one by one --- moto/swf/models/workflow_execution.py | 77 +++++++++++++++---- .../models/test_workflow_execution.py | 34 ++++++++ tests/test_swf/utils.py | 6 ++ 3 files changed, 100 insertions(+), 17 deletions(-) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 0c8202bda..4b55b8cbd 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -149,29 +149,65 @@ class WorkflowExecution(object): return hsh def _process_timeouts(self): - self.should_schedule_decision_next = False + """ + SWF timeouts can happen on different objects (workflow executions, + activity tasks, decision tasks) and should be processed in order. + + A specific timeout can change the workflow execution state and have an + impact on other timeouts: for instance, if the workflow execution + timeouts, subsequent timeouts on activity or decision tasks are + irrelevant ; if an activity task timeouts, other timeouts on this task + are irrelevant, and a new decision is fired, which could well timeout + before the end of the workflow. + + So the idea here is to find the earliest timeout that would have been + triggered, process it, then make the workflow state progress and repeat + the whole process. + """ + timeout_candidates = [] # workflow execution timeout - _timeout = self.first_timeout() - if _timeout: - self.timeout(_timeout) + timeout_candidates.append(self.first_timeout()) # decision tasks timeouts for task in self.decision_tasks: - _timeout = task.first_timeout() - if _timeout: - self.timeout_decision_task(_timeout) + timeout_candidates.append(task.first_timeout()) # activity tasks timeouts for task in self.activity_tasks: - _timeout = task.first_timeout() - if _timeout: - self.timeout_activity_task(_timeout) + timeout_candidates.append(task.first_timeout()) - # schedule decision task if needed - # TODO: make decision appear as if it has been scheduled immediately after the timeout - if self.should_schedule_decision_next: - self.schedule_decision_task() + # remove blank values (foo.first_timeout() is a Timeout or None) + timeout_candidates = filter(None, timeout_candidates) + + # now find the first timeout to process + first_timeout = None + if timeout_candidates: + first_timeout = min( + timeout_candidates, + key=lambda t: t.timestamp + ) + + if first_timeout: + should_schedule_decision_next = False + if isinstance(first_timeout.obj, WorkflowExecution): + self.timeout(first_timeout) + elif isinstance(first_timeout.obj, DecisionTask): + self.timeout_decision_task(first_timeout) + should_schedule_decision_next = True + elif isinstance(first_timeout.obj, ActivityTask): + self.timeout_activity_task(first_timeout) + should_schedule_decision_next = True + else: + raise NotImplementedError("Unhandled timeout object") + + # schedule decision task if needed + if should_schedule_decision_next: + self.schedule_decision_task() + + # the workflow execution progressed, let's see if another + # timeout should be processed + self._process_timeouts() def events(self, reverse_order=False): if reverse_order: @@ -196,7 +232,7 @@ class WorkflowExecution(object): ) self.schedule_decision_task() - def schedule_decision_task(self): + def _schedule_decision_task(self): evt = self._add_event( "DecisionTaskScheduled", workflow_execution=self, @@ -207,6 +243,15 @@ class WorkflowExecution(object): ) self.open_counts["openDecisionTasks"] += 1 + def schedule_decision_task(self): + self._schedule_decision_task() + + # Shortcut for tests: helps having auto-starting decision tasks when needed + def schedule_and_start_decision_task(self, identity=None): + self._schedule_decision_task() + decision_task = self.decision_tasks[-1] + self.start_decision_task(decision_task.task_token, identity=identity) + @property def decision_tasks(self): return [t for t in self.domain.decision_tasks @@ -528,7 +573,6 @@ class WorkflowExecution(object): ) def timeout_decision_task(self, _timeout): - self.should_schedule_decision_next = True task = _timeout.obj task.timeout(_timeout) self._add_event( @@ -540,7 +584,6 @@ class WorkflowExecution(object): ) def timeout_activity_task(self, _timeout): - self.should_schedule_decision_next = True task = _timeout.obj task.timeout(_timeout) self._add_event( diff --git a/tests/test_swf/models/test_workflow_execution.py b/tests/test_swf/models/test_workflow_execution.py index e533f925b..8a010eb42 100644 --- a/tests/test_swf/models/test_workflow_execution.py +++ b/tests/test_swf/models/test_workflow_execution.py @@ -12,6 +12,7 @@ from moto.swf.exceptions import ( ) from ..utils import ( + auto_start_decision_tasks, get_basic_domain, get_basic_workflow_type, make_workflow_execution, @@ -407,3 +408,36 @@ def test_first_timeout(): with freeze_time("2015-01-01 14:01"): # 2 hours timeout reached wfe.first_timeout().should.be.a(Timeout) + +# See moto/swf/models/workflow_execution.py "_process_timeouts()" for more details +def test_timeouts_are_processed_in_order_and_reevaluated(): + # Let's make a Workflow Execution with the following properties: + # - execution start to close timeout of 8 mins + # - (decision) task start to close timeout of 5 mins + # + # Now start the workflow execution, and look at the history 15 mins later: + # - a first decision task is fired just after workflow execution start + # - the first decision task should have timed out after 5 mins + # - that fires a new decision task (which we hack to start automatically) + # - then the workflow timeouts after 8 mins (shows gradual reevaluation) + # - but the last scheduled decision task should *not* timeout (workflow closed) + with freeze_time("2015-01-01 12:00:00"): + wfe = make_workflow_execution( + execution_start_to_close_timeout=8*60, + task_start_to_close_timeout=5*60, + ) + # decision will automatically start + wfe = auto_start_decision_tasks(wfe) + wfe.start() + event_idx = len(wfe.events()) + + with freeze_time("2015-01-01 12:08:00"): + wfe._process_timeouts() + + event_types = [e.event_type for e in wfe.events()[event_idx:]] + event_types.should.equal([ + "DecisionTaskTimedOut", + "DecisionTaskScheduled", + "DecisionTaskStarted", + "WorkflowExecutionTimedOut", + ]) diff --git a/tests/test_swf/utils.py b/tests/test_swf/utils.py index d98294ea2..352118340 100644 --- a/tests/test_swf/utils.py +++ b/tests/test_swf/utils.py @@ -66,6 +66,12 @@ def make_workflow_execution(**kwargs): return WorkflowExecution(domain, wft, "ab1234", **kwargs) +# Makes decision tasks start automatically on a given workflow +def auto_start_decision_tasks(wfe): + wfe.schedule_decision_task = wfe.schedule_and_start_decision_task + return wfe + + # Setup a complete example workflow and return the connection object @mock_swf def setup_workflow(): From de646cf7acec1d1ae2ea2a44bd48f044044bdc72 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Tue, 10 Nov 2015 00:22:02 +0100 Subject: [PATCH 80/94] Fix python 3 compatibility: filter() returns an iterator now --- moto/swf/models/workflow_execution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 4b55b8cbd..67601438b 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -178,7 +178,7 @@ class WorkflowExecution(object): timeout_candidates.append(task.first_timeout()) # remove blank values (foo.first_timeout() is a Timeout or None) - timeout_candidates = filter(None, timeout_candidates) + timeout_candidates = list(filter(None, timeout_candidates)) # now find the first timeout to process first_timeout = None From 5f0684fca5668d24fdb7b223b368de3c0607cb76 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Wed, 18 Nov 2015 22:00:57 +0100 Subject: [PATCH 81/94] Use dict.values() instead of dict.items() where possible (suggested in @spulec review) --- moto/swf/models/__init__.py | 4 ++-- moto/swf/models/domain.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index f4fe246f8..b9d7be0fd 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -113,7 +113,7 @@ class SWFBackend(BaseBackend): self._check_string(domain_name) self._check_string(name) self._check_string(version) - for _, value in kwargs.items(): + for value in kwargs.values(): self._check_none_or_string(value) domain = self._get_domain(domain_name) _type = domain.get_type(kind, name, version, ignore_empty=True) @@ -148,7 +148,7 @@ class SWFBackend(BaseBackend): self._check_string(workflow_name) self._check_string(workflow_version) self._check_none_or_list_of_strings(tag_list) - for _, value in kwargs.items(): + for value in kwargs.values(): self._check_none_or_string(value) domain = self._get_domain(domain_name) diff --git a/moto/swf/models/domain.py b/moto/swf/models/domain.py index 824782c59..98892a30b 100644 --- a/moto/swf/models/domain.py +++ b/moto/swf/models/domain.py @@ -63,8 +63,8 @@ class Domain(object): def find_types(self, kind, status): _all = [] - for _, family in self.types[kind].items(): - for _, _type in family.items(): + for family in self.types[kind].values(): + for _type in family.values(): if _type.status == status: _all.append(_type) return _all @@ -107,7 +107,7 @@ class Domain(object): @property def activity_tasks(self): _all = [] - for _, tasks in self.activity_task_lists.items(): + for tasks in self.activity_task_lists.values(): _all += tasks return _all @@ -119,6 +119,6 @@ class Domain(object): @property def decision_tasks(self): _all = [] - for _, tasks in self.decision_task_lists.items(): + for tasks in self.decision_task_lists.values(): _all += tasks return _all From 6b581edb55420ac9917a730a635f0ff3dbefd37b Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Wed, 18 Nov 2015 22:02:58 +0100 Subject: [PATCH 82/94] Use datetime.utcnow() instead of datetime.now() (suggested in @spulec review) --- moto/swf/models/activity_task.py | 2 +- moto/swf/models/decision_task.py | 2 +- moto/swf/utils.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/moto/swf/models/activity_task.py b/moto/swf/models/activity_task.py index eef8350e2..1a72d3449 100644 --- a/moto/swf/models/activity_task.py +++ b/moto/swf/models/activity_task.py @@ -25,7 +25,7 @@ class ActivityTask(object): self.workflow_execution = workflow_execution # this is *not* necessarily coherent with workflow execution history, # but that shouldn't be a problem for tests - self.scheduled_at = datetime.now() + self.scheduled_at = datetime.utcnow() def _check_workflow_execution_open(self): if not self.workflow_execution.open: diff --git a/moto/swf/models/decision_task.py b/moto/swf/models/decision_task.py index 5474332a5..b76888403 100644 --- a/moto/swf/models/decision_task.py +++ b/moto/swf/models/decision_task.py @@ -21,7 +21,7 @@ class DecisionTask(object): self.state = "SCHEDULED" # this is *not* necessarily coherent with workflow execution history, # but that shouldn't be a problem for tests - self.scheduled_at = datetime.now() + self.scheduled_at = datetime.utcnow() self.timeout_type = None @property diff --git a/moto/swf/utils.py b/moto/swf/utils.py index 02603bea9..a9c54ee3a 100644 --- a/moto/swf/utils.py +++ b/moto/swf/utils.py @@ -6,4 +6,4 @@ def decapitalize(key): return key[0].lower() + key[1:] def now_timestamp(): - return float(mktime(datetime.now().timetuple())) + return float(mktime(datetime.utcnow().timetuple())) From 26980f41a66b3afbe5ba67e8cb66354b6f4bdef0 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Thu, 19 Nov 2015 11:44:51 +0100 Subject: [PATCH 83/94] Replace globals() call with a static mapping (suggested in @spulec review) --- moto/swf/models/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index b9d7be0fd..29fda4a3a 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -25,6 +25,12 @@ from .workflow_type import WorkflowType from .workflow_execution import WorkflowExecution +KNOWN_SWF_TYPES = { + "activity": ActivityType, + "workflow": WorkflowType, +} + + class SWFBackend(BaseBackend): def __init__(self, region_name): self.region_name = region_name @@ -119,7 +125,7 @@ class SWFBackend(BaseBackend): _type = domain.get_type(kind, name, version, ignore_empty=True) if _type: raise SWFTypeAlreadyExistsFault(_type) - _class = globals()["{0}Type".format(kind.capitalize())] + _class = KNOWN_SWF_TYPES[kind] _type = _class(name, version, **kwargs) domain.add_type(_type) From 78ea7967ad55af1acc46d00a706b2855359f2319 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Thu, 19 Nov 2015 11:46:54 +0100 Subject: [PATCH 84/94] Remove overriden SWFResponse.call_action() thanks to 32dd72f Not necessary anymore: https://github.com/spulec/moto/commit/32dd72f6b7289620fb149bd876858d5293a118f0 --- moto/swf/responses.py | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/moto/swf/responses.py b/moto/swf/responses.py index 3f6e83330..0b8557a2e 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -14,37 +14,6 @@ class SWFResponse(BaseResponse): def swf_backend(self): return swf_backends[self.region] - # SWF actions are not dispatched via URLs but via a specific header called - # "x-amz-target", in the form of com.amazonaws.swf.service.model.SimpleWorkflowService. - # This is not supported directly in BaseResponse sor for now we override - # the call_action() method - # See: http://docs.aws.amazon.com/amazonswf/latest/developerguide/UsingJSON-swf.html - def call_action(self): - headers = self.response_headers - # Headers are case-insensitive. Probably a better way to do this. - match = self.headers.get('x-amz-target') or self.headers.get('X-Amz-Target') - if match: - # TODO: see if we can call "[-1]" in BaseResponse, which would - # allow to remove that - action = match.split(".")[-1] - - action = camelcase_to_underscores(action) - method_names = method_names_from_class(self.__class__) - if action in method_names: - method = getattr(self, action) - try: - response = method() - except HTTPException as http_error: - response = http_error.description, dict(status=http_error.code) - if isinstance(response, six.string_types): - return 200, headers, response - else: - body, new_headers = response - status = new_headers.get('status', 200) - headers.update(new_headers) - return status, headers, body - raise NotImplementedError("The {0} action has not been implemented".format(action)) - # SWF parameters are passed through a JSON body, so let's ease retrieval @property def _params(self): From e3fff8759b06dc80805d1f6fea6901466a68f479 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Thu, 19 Nov 2015 11:53:47 +0100 Subject: [PATCH 85/94] Add jbbarth in authors list --- AUTHORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.md b/AUTHORS.md index 77ea55cd2..be500fae8 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -41,3 +41,4 @@ Moto is written by Steve Pulec with contributions from: * [Zack Kourouma](https://github.com/zkourouma) * [Pior Bastida](https://github.com/pior) * [Dustin J. Mitchell](https://github.com/djmitche) +* [Jean-Baptiste Barth](https://github.com/jbbarth) From 45437368b2363f8ea47d93c9ebe63b5e56cd3594 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 23 Nov 2015 12:41:31 +0100 Subject: [PATCH 86/94] Move SWF type checks to response object (suggested in @spulec review) --- moto/swf/models/__init__.py | 74 ---------------------------- moto/swf/responses.py | 97 +++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 74 deletions(-) diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index 29fda4a3a..6b5c04cfb 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -9,7 +9,6 @@ from ..exceptions import ( SWFUnknownResourceFault, SWFDomainAlreadyExistsFault, SWFDomainDeprecatedFault, - SWFSerializationException, SWFTypeAlreadyExistsFault, SWFTypeDeprecatedFault, SWFValidationException, @@ -50,32 +49,12 @@ class SWFBackend(BaseBackend): return matching[0] return None - def _check_none_or_string(self, parameter): - if parameter is not None: - self._check_string(parameter) - - def _check_string(self, parameter): - if not isinstance(parameter, six.string_types): - raise SWFSerializationException(parameter) - - def _check_none_or_list_of_strings(self, parameter): - if parameter is not None: - self._check_list_of_strings(parameter) - - def _check_list_of_strings(self, parameter): - if not isinstance(parameter, list): - raise SWFSerializationException(parameter) - for i in parameter: - if not isinstance(i, six.string_types): - raise SWFSerializationException(parameter) - def _process_timeouts(self): for domain in self.domains: for wfe in domain.workflow_executions: wfe._process_timeouts() def list_domains(self, status, reverse_order=None): - self._check_string(status) domains = [domain for domain in self.domains if domain.status == status] domains = sorted(domains, key=lambda domain: domain.name) @@ -85,9 +64,6 @@ class SWFBackend(BaseBackend): def register_domain(self, name, workflow_execution_retention_period_in_days, description=None): - self._check_string(name) - self._check_string(workflow_execution_retention_period_in_days) - self._check_none_or_string(description) if self._get_domain(name, ignore_empty=True): raise SWFDomainAlreadyExistsFault(name) domain = Domain(name, workflow_execution_retention_period_in_days, @@ -95,19 +71,15 @@ class SWFBackend(BaseBackend): self.domains.append(domain) def deprecate_domain(self, name): - self._check_string(name) domain = self._get_domain(name) if domain.status == "DEPRECATED": raise SWFDomainDeprecatedFault(name) domain.status = "DEPRECATED" def describe_domain(self, name): - self._check_string(name) return self._get_domain(name) def list_types(self, kind, domain_name, status, reverse_order=None): - self._check_string(domain_name) - self._check_string(status) domain = self._get_domain(domain_name) _types = domain.find_types(kind, status) _types = sorted(_types, key=lambda domain: domain.name) @@ -116,11 +88,6 @@ class SWFBackend(BaseBackend): return _types def register_type(self, kind, domain_name, name, version, **kwargs): - self._check_string(domain_name) - self._check_string(name) - self._check_string(version) - for value in kwargs.values(): - self._check_none_or_string(value) domain = self._get_domain(domain_name) _type = domain.get_type(kind, name, version, ignore_empty=True) if _type: @@ -130,9 +97,6 @@ class SWFBackend(BaseBackend): domain.add_type(_type) def deprecate_type(self, kind, domain_name, name, version): - self._check_string(domain_name) - self._check_string(name) - self._check_string(version) domain = self._get_domain(domain_name) _type = domain.get_type(kind, name, version) if _type.status == "DEPRECATED": @@ -140,23 +104,12 @@ class SWFBackend(BaseBackend): _type.status = "DEPRECATED" def describe_type(self, kind, domain_name, name, version): - self._check_string(domain_name) - self._check_string(name) - self._check_string(version) domain = self._get_domain(domain_name) return domain.get_type(kind, name, version) def start_workflow_execution(self, domain_name, workflow_id, workflow_name, workflow_version, tag_list=None, **kwargs): - self._check_string(domain_name) - self._check_string(workflow_id) - self._check_string(workflow_name) - self._check_string(workflow_version) - self._check_none_or_list_of_strings(tag_list) - for value in kwargs.values(): - self._check_none_or_string(value) - domain = self._get_domain(domain_name) wf_type = domain.get_type("workflow", workflow_name, workflow_version) if wf_type.status == "DEPRECATED": @@ -169,17 +122,12 @@ class SWFBackend(BaseBackend): return wfe def describe_workflow_execution(self, domain_name, run_id, workflow_id): - self._check_string(domain_name) - self._check_string(run_id) - self._check_string(workflow_id) # process timeouts on all objects self._process_timeouts() domain = self._get_domain(domain_name) return domain.get_workflow_execution(workflow_id, run_id=run_id) def poll_for_decision_task(self, domain_name, task_list, identity=None): - self._check_string(domain_name) - self._check_string(task_list) # process timeouts on all objects self._process_timeouts() domain = self._get_domain(domain_name) @@ -211,8 +159,6 @@ class SWFBackend(BaseBackend): return None def count_pending_decision_tasks(self, domain_name, task_list): - self._check_string(domain_name) - self._check_string(task_list) # process timeouts on all objects self._process_timeouts() domain = self._get_domain(domain_name) @@ -225,8 +171,6 @@ class SWFBackend(BaseBackend): def respond_decision_task_completed(self, task_token, decisions=None, execution_context=None): - self._check_string(task_token) - self._check_none_or_string(execution_context) # process timeouts on all objects self._process_timeouts() # let's find decision task @@ -278,8 +222,6 @@ class SWFBackend(BaseBackend): execution_context=execution_context) def poll_for_activity_task(self, domain_name, task_list, identity=None): - self._check_string(domain_name) - self._check_string(task_list) # process timeouts on all objects self._process_timeouts() domain = self._get_domain(domain_name) @@ -311,8 +253,6 @@ class SWFBackend(BaseBackend): return None def count_pending_activity_tasks(self, domain_name, task_list): - self._check_string(domain_name) - self._check_string(task_list) # process timeouts on all objects self._process_timeouts() domain = self._get_domain(domain_name) @@ -362,8 +302,6 @@ class SWFBackend(BaseBackend): return activity_task def respond_activity_task_completed(self, task_token, result=None): - self._check_string(task_token) - self._check_none_or_string(result) # process timeouts on all objects self._process_timeouts() activity_task = self._find_activity_task_from_token(task_token) @@ -371,10 +309,6 @@ class SWFBackend(BaseBackend): wfe.complete_activity_task(activity_task.task_token, result=result) def respond_activity_task_failed(self, task_token, reason=None, details=None): - self._check_string(task_token) - # TODO: implement length limits on reason and details (common pb with client libs) - self._check_none_or_string(reason) - self._check_none_or_string(details) # process timeouts on all objects self._process_timeouts() activity_task = self._find_activity_task_from_token(task_token) @@ -383,12 +317,6 @@ class SWFBackend(BaseBackend): def terminate_workflow_execution(self, domain_name, workflow_id, child_policy=None, details=None, reason=None, run_id=None): - self._check_string(domain_name) - self._check_string(workflow_id) - self._check_none_or_string(child_policy) - self._check_none_or_string(details) - self._check_none_or_string(reason) - self._check_none_or_string(run_id) # process timeouts on all objects self._process_timeouts() domain = self._get_domain(domain_name) @@ -396,8 +324,6 @@ class SWFBackend(BaseBackend): wfe.terminate(child_policy=child_policy, details=details, reason=reason) def record_activity_task_heartbeat(self, task_token, details=None): - self._check_string(task_token) - self._check_none_or_string(details) # process timeouts on all objects self._process_timeouts() activity_task = self._find_activity_task_from_token(task_token) diff --git a/moto/swf/responses.py b/moto/swf/responses.py index 0b8557a2e..7f4188631 100644 --- a/moto/swf/responses.py +++ b/moto/swf/responses.py @@ -5,6 +5,7 @@ from moto.core.responses import BaseResponse from werkzeug.exceptions import HTTPException from moto.core.utils import camelcase_to_underscores, method_names_from_class +from .exceptions import SWFSerializationException from .models import swf_backends @@ -19,10 +20,31 @@ class SWFResponse(BaseResponse): def _params(self): return json.loads(self.body.decode("utf-8")) + def _check_none_or_string(self, parameter): + if parameter is not None: + self._check_string(parameter) + + def _check_string(self, parameter): + if not isinstance(parameter, six.string_types): + raise SWFSerializationException(parameter) + + def _check_none_or_list_of_strings(self, parameter): + if parameter is not None: + self._check_list_of_strings(parameter) + + def _check_list_of_strings(self, parameter): + if not isinstance(parameter, list): + raise SWFSerializationException(parameter) + for i in parameter: + if not isinstance(i, six.string_types): + raise SWFSerializationException(parameter) + def _list_types(self, kind): domain_name = self._params["domain"] status = self._params["registrationStatus"] reverse_order = self._params.get("reverseOrder", None) + self._check_string(domain_name) + self._check_string(status) types = self.swf_backend.list_types(kind, domain_name, status, reverse_order=reverse_order) return json.dumps({ "typeInfos": [_type.to_medium_dict() for _type in types] @@ -33,6 +55,9 @@ class SWFResponse(BaseResponse): _type_args = self._params["{0}Type".format(kind)] name = _type_args["name"] version = _type_args["version"] + self._check_string(domain) + self._check_string(name) + self._check_string(version) _type = self.swf_backend.describe_type(kind, domain, name, version) return json.dumps(_type.to_full_dict()) @@ -42,12 +67,16 @@ class SWFResponse(BaseResponse): _type_args = self._params["{0}Type".format(kind)] name = _type_args["name"] version = _type_args["version"] + self._check_string(domain) + self._check_string(name) + self._check_string(version) self.swf_backend.deprecate_type(kind, domain, name, version) return "" # TODO: implement pagination def list_domains(self): status = self._params["registrationStatus"] + self._check_string(status) reverse_order = self._params.get("reverseOrder", None) domains = self.swf_backend.list_domains(status, reverse_order=reverse_order) return json.dumps({ @@ -58,17 +87,22 @@ class SWFResponse(BaseResponse): name = self._params["name"] retention = self._params["workflowExecutionRetentionPeriodInDays"] description = self._params.get("description") + self._check_string(retention) + self._check_string(name) + self._check_none_or_string(description) domain = self.swf_backend.register_domain(name, retention, description=description) return "" def deprecate_domain(self): name = self._params["name"] + self._check_string(name) domain = self.swf_backend.deprecate_domain(name) return "" def describe_domain(self): name = self._params["name"] + self._check_string(name) domain = self.swf_backend.describe_domain(name) return json.dumps(domain.to_full_dict()) @@ -90,6 +124,17 @@ class SWFResponse(BaseResponse): default_task_schedule_to_start_timeout = self._params.get("defaultTaskScheduleToStartTimeout") default_task_start_to_close_timeout = self._params.get("defaultTaskStartToCloseTimeout") description = self._params.get("description") + + self._check_string(domain) + self._check_string(name) + self._check_string(version) + self._check_none_or_string(task_list) + self._check_none_or_string(default_task_heartbeat_timeout) + self._check_none_or_string(default_task_schedule_to_close_timeout) + self._check_none_or_string(default_task_schedule_to_start_timeout) + self._check_none_or_string(default_task_start_to_close_timeout) + self._check_none_or_string(description) + # TODO: add defaultTaskPriority when boto gets to support it activity_type = self.swf_backend.register_type( "activity", domain, name, version, task_list=task_list, @@ -123,6 +168,16 @@ class SWFResponse(BaseResponse): default_task_start_to_close_timeout = self._params.get("defaultTaskStartToCloseTimeout") default_execution_start_to_close_timeout = self._params.get("defaultExecutionStartToCloseTimeout") description = self._params.get("description") + + self._check_string(domain) + self._check_string(name) + self._check_string(version) + self._check_none_or_string(task_list) + self._check_none_or_string(default_child_policy) + self._check_none_or_string(default_task_start_to_close_timeout) + self._check_none_or_string(default_execution_start_to_close_timeout) + self._check_none_or_string(description) + # TODO: add defaultTaskPriority when boto gets to support it # TODO: add defaultLambdaRole when boto gets to support it workflow_type = self.swf_backend.register_type( @@ -157,6 +212,17 @@ class SWFResponse(BaseResponse): tag_list = self._params.get("tagList") task_start_to_close_timeout = self._params.get("taskStartToCloseTimeout") + self._check_string(domain) + self._check_string(workflow_id) + self._check_string(workflow_name) + self._check_string(workflow_version) + self._check_none_or_string(task_list) + self._check_none_or_string(child_policy) + self._check_none_or_string(execution_start_to_close_timeout) + self._check_none_or_string(input_) + self._check_none_or_list_of_strings(tag_list) + self._check_none_or_string(task_start_to_close_timeout) + wfe = self.swf_backend.start_workflow_execution( domain, workflow_id, workflow_name, workflow_version, task_list=task_list, child_policy=child_policy, @@ -175,6 +241,10 @@ class SWFResponse(BaseResponse): run_id = _workflow_execution["runId"] workflow_id = _workflow_execution["workflowId"] + self._check_string(domain_name) + self._check_string(run_id) + self._check_string(workflow_id) + wfe = self.swf_backend.describe_workflow_execution(domain_name, run_id, workflow_id) return json.dumps(wfe.to_full_dict()) @@ -195,6 +265,10 @@ class SWFResponse(BaseResponse): task_list = self._params["taskList"]["name"] identity = self._params.get("identity") reverse_order = self._params.get("reverseOrder", None) + + self._check_string(domain_name) + self._check_string(task_list) + decision = self.swf_backend.poll_for_decision_task( domain_name, task_list, identity=identity ) @@ -208,6 +282,8 @@ class SWFResponse(BaseResponse): def count_pending_decision_tasks(self): domain_name = self._params["domain"] task_list = self._params["taskList"]["name"] + self._check_string(domain_name) + self._check_string(task_list) count = self.swf_backend.count_pending_decision_tasks(domain_name, task_list) return json.dumps({"count": count, "truncated": False}) @@ -215,6 +291,8 @@ class SWFResponse(BaseResponse): task_token = self._params["taskToken"] execution_context = self._params.get("executionContext") decisions = self._params.get("decisions") + self._check_string(task_token) + self._check_none_or_string(execution_context) self.swf_backend.respond_decision_task_completed( task_token, decisions=decisions, execution_context=execution_context ) @@ -224,6 +302,9 @@ class SWFResponse(BaseResponse): domain_name = self._params["domain"] task_list = self._params["taskList"]["name"] identity = self._params.get("identity") + self._check_string(domain_name) + self._check_string(task_list) + self._check_none_or_string(identity) activity_task = self.swf_backend.poll_for_activity_task( domain_name, task_list, identity=identity ) @@ -237,12 +318,16 @@ class SWFResponse(BaseResponse): def count_pending_activity_tasks(self): domain_name = self._params["domain"] task_list = self._params["taskList"]["name"] + self._check_string(domain_name) + self._check_string(task_list) count = self.swf_backend.count_pending_activity_tasks(domain_name, task_list) return json.dumps({"count": count, "truncated": False}) def respond_activity_task_completed(self): task_token = self._params["taskToken"] result = self._params.get("result") + self._check_string(task_token) + self._check_none_or_string(result) self.swf_backend.respond_activity_task_completed( task_token, result=result ) @@ -252,6 +337,10 @@ class SWFResponse(BaseResponse): task_token = self._params["taskToken"] reason = self._params.get("reason") details = self._params.get("details") + self._check_string(task_token) + # TODO: implement length limits on reason and details (common pb with client libs) + self._check_none_or_string(reason) + self._check_none_or_string(details) self.swf_backend.respond_activity_task_failed( task_token, reason=reason, details=details ) @@ -264,6 +353,12 @@ class SWFResponse(BaseResponse): details = self._params.get("details") reason = self._params.get("reason") run_id = self._params.get("runId") + self._check_string(domain_name) + self._check_string(workflow_id) + self._check_none_or_string(child_policy) + self._check_none_or_string(details) + self._check_none_or_string(reason) + self._check_none_or_string(run_id) self.swf_backend.terminate_workflow_execution( domain_name, workflow_id, child_policy=child_policy, details=details, reason=reason, run_id=run_id @@ -273,6 +368,8 @@ class SWFResponse(BaseResponse): def record_activity_task_heartbeat(self): task_token = self._params["taskToken"] details = self._params.get("details") + self._check_string(task_token) + self._check_none_or_string(details) self.swf_backend.record_activity_task_heartbeat( task_token, details=details ) From a4dfdc82749873fefac1d0a5a411e178ed9f6da9 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 23 Nov 2015 14:04:14 +0100 Subject: [PATCH 87/94] Add basic tests for moto.core.utils.camelcase_to_underscores() --- tests/test_core/test_utils.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 tests/test_core/test_utils.py diff --git a/tests/test_core/test_utils.py b/tests/test_core/test_utils.py new file mode 100644 index 000000000..3e483819a --- /dev/null +++ b/tests/test_core/test_utils.py @@ -0,0 +1,14 @@ +from __future__ import unicode_literals +import sure + +from moto.core.utils import camelcase_to_underscores + + +def test_camelcase_to_underscores(): + cases = { + "theNewAttribute": "the_new_attribute", + "attri bute With Space": "attribute_with_space", + "FirstLetterCapital": "first_letter_capital", + } + for arg, expected in cases.items(): + camelcase_to_underscores(arg).should.equal(expected) From a06f8b15f546eb25f4afee3681cea912cab3ac01 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 23 Nov 2015 14:09:31 +0100 Subject: [PATCH 88/94] Add moto.core.utils.underscores_to_camelcase() --- moto/core/utils.py | 16 ++++++++++++++++ tests/test_core/test_utils.py | 10 +++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/moto/core/utils.py b/moto/core/utils.py index 98d6bdc23..81acdd6db 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -23,6 +23,22 @@ def camelcase_to_underscores(argument): return result +def underscores_to_camelcase(argument): + ''' Converts a camelcase param like the_new_attribute to the equivalent + camelcase version like theNewAttribute. Note that the first letter is + NOT capitalized by this function ''' + result = '' + previous_was_underscore = False + for char in argument: + if char != '_': + if previous_was_underscore: + result += char.upper() + else: + result += char + previous_was_underscore = char == '_' + return result + + def method_names_from_class(clazz): # On Python 2, methods are different from functions, and the `inspect` # predicates distinguish between them. On Python 3, methods are just diff --git a/tests/test_core/test_utils.py b/tests/test_core/test_utils.py index 3e483819a..6e27e6f49 100644 --- a/tests/test_core/test_utils.py +++ b/tests/test_core/test_utils.py @@ -1,7 +1,7 @@ from __future__ import unicode_literals import sure -from moto.core.utils import camelcase_to_underscores +from moto.core.utils import camelcase_to_underscores, underscores_to_camelcase def test_camelcase_to_underscores(): @@ -12,3 +12,11 @@ def test_camelcase_to_underscores(): } for arg, expected in cases.items(): camelcase_to_underscores(arg).should.equal(expected) + + +def test_underscores_to_camelcase(): + cases = { + "the_new_attribute": "theNewAttribute", + } + for arg, expected in cases.items(): + underscores_to_camelcase(arg).should.equal(expected) From 4b59c6b90730a080d10699cb25575f5e5f7316d1 Mon Sep 17 00:00:00 2001 From: earthmant Date: Mon, 23 Nov 2015 15:16:46 +0200 Subject: [PATCH 89/94] Support Associate Network ACL add the new_association_id property to NetworkACL object so that the template render for replace adds the ID and the associate_network_acl receives a response --- moto/ec2/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 446561d33..098e86328 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -2597,6 +2597,7 @@ class NetworkAclAssociation(object): subnet_id, network_acl_id): self.ec2_backend = ec2_backend self.id = new_association_id + self.new_association_id = new_association_id self.subnet_id = subnet_id self.network_acl_id = network_acl_id super(NetworkAclAssociation, self).__init__() From 566a90800e856dfb26ebd41dfe09ce0875f52b54 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Barth Date: Mon, 23 Nov 2015 14:51:58 +0100 Subject: [PATCH 90/94] Make SWF events formatting more generic (suggested in @spulec review) --- moto/swf/models/history_event.py | 191 ++++-------------- moto/swf/models/workflow_execution.py | 26 ++- .../models/test_workflow_execution.py | 44 ++-- 3 files changed, 87 insertions(+), 174 deletions(-) diff --git a/moto/swf/models/history_event.py b/moto/swf/models/history_event.py index 6e4002345..c602abee8 100644 --- a/moto/swf/models/history_event.py +++ b/moto/swf/models/history_event.py @@ -2,168 +2,65 @@ from __future__ import unicode_literals from datetime import datetime from time import mktime +from moto.core.utils import underscores_to_camelcase + from ..utils import decapitalize, now_timestamp +# We keep track of which history event types we support +# so that we'll be able to catch specific formatting +# for new events if needed. +SUPPORTED_HISTORY_EVENT_TYPES = ( + "WorkflowExecutionStarted", + "DecisionTaskScheduled", + "DecisionTaskStarted", + "DecisionTaskCompleted", + "WorkflowExecutionCompleted", + "WorkflowExecutionFailed", + "ActivityTaskScheduled", + "ScheduleActivityTaskFailed", + "ActivityTaskStarted", + "ActivityTaskCompleted", + "ActivityTaskFailed", + "WorkflowExecutionTerminated", + "ActivityTaskTimedOut", + "DecisionTaskTimedOut", + "WorkflowExecutionTimedOut", +) + class HistoryEvent(object): - def __init__(self, event_id, event_type, **kwargs): + def __init__(self, event_id, event_type, event_timestamp=None, **kwargs): + if event_type not in SUPPORTED_HISTORY_EVENT_TYPES: + raise NotImplementedError( + "HistoryEvent does not implement attributes for type '{0}'".format(event_type) + ) self.event_id = event_id self.event_type = event_type - self.event_timestamp = now_timestamp() + if event_timestamp: + self.event_timestamp = event_timestamp + else: + self.event_timestamp = now_timestamp() + # pre-populate a dict: {"camelCaseKey": value} + self.event_attributes = {} for key, value in kwargs.items(): - self.__setattr__(key, value) - # break soon if attributes are not valid - self.event_attributes() + if value: + camel_key = underscores_to_camelcase(key) + if key == "task_list": + value = { "name": value } + elif key == "workflow_type": + value = { "name": value.name, "version": value.version } + elif key == "activity_type": + value = value.to_short_dict() + self.event_attributes[camel_key] = value def to_dict(self): return { "eventId": self.event_id, "eventType": self.event_type, "eventTimestamp": self.event_timestamp, - self._attributes_key(): self.event_attributes() + self._attributes_key(): self.event_attributes } def _attributes_key(self): key = "{0}EventAttributes".format(self.event_type) return decapitalize(key) - - def event_attributes(self): - if self.event_type == "WorkflowExecutionStarted": - wfe = self.workflow_execution - hsh = { - "childPolicy": wfe.child_policy, - "executionStartToCloseTimeout": wfe.execution_start_to_close_timeout, - "parentInitiatedEventId": 0, - "taskList": { - "name": wfe.task_list - }, - "taskStartToCloseTimeout": wfe.task_start_to_close_timeout, - "workflowType": { - "name": wfe.workflow_type.name, - "version": wfe.workflow_type.version - } - } - return hsh - elif self.event_type == "DecisionTaskScheduled": - wfe = self.workflow_execution - return { - "startToCloseTimeout": wfe.task_start_to_close_timeout, - "taskList": {"name": wfe.task_list} - } - elif self.event_type == "DecisionTaskStarted": - hsh = { - "scheduledEventId": self.scheduled_event_id - } - if hasattr(self, "identity") and self.identity: - hsh["identity"] = self.identity - return hsh - elif self.event_type == "DecisionTaskCompleted": - hsh = { - "scheduledEventId": self.scheduled_event_id, - "startedEventId": self.started_event_id, - } - if hasattr(self, "execution_context") and self.execution_context: - hsh["executionContext"] = self.execution_context - return hsh - elif self.event_type == "WorkflowExecutionCompleted": - hsh = { - "decisionTaskCompletedEventId": self.decision_task_completed_event_id, - } - if hasattr(self, "result") and self.result: - hsh["result"] = self.result - return hsh - elif self.event_type == "WorkflowExecutionFailed": - hsh = { - "decisionTaskCompletedEventId": self.decision_task_completed_event_id, - } - if hasattr(self, "details") and self.details: - hsh["details"] = self.details - if hasattr(self, "reason") and self.reason: - hsh["reason"] = self.reason - return hsh - elif self.event_type == "ActivityTaskScheduled": - hsh = { - "activityId": self.attributes["activityId"], - "activityType": self.activity_type.to_short_dict(), - "decisionTaskCompletedEventId": self.decision_task_completed_event_id, - "taskList": { - "name": self.task_list, - }, - } - for attr in ["control", "heartbeatTimeout", "input", "scheduleToCloseTimeout", - "scheduleToStartTimeout", "startToCloseTimeout", "taskPriority"]: - if self.attributes.get(attr): - hsh[attr] = self.attributes[attr] - return hsh - elif self.event_type == "ScheduleActivityTaskFailed": - # TODO: implement other possible failure mode: OPEN_ACTIVITIES_LIMIT_EXCEEDED - # NB: some failure modes are not implemented and probably won't be implemented in the - # future, such as ACTIVITY_CREATION_RATE_EXCEEDED or OPERATION_NOT_PERMITTED - return { - "activityId": self.activity_id, - "activityType": self.activity_type.to_short_dict(), - "cause": self.cause, - "decisionTaskCompletedEventId": self.decision_task_completed_event_id, - } - elif self.event_type == "ActivityTaskStarted": - # TODO: merge it with DecisionTaskStarted - hsh = { - "scheduledEventId": self.scheduled_event_id - } - if hasattr(self, "identity") and self.identity: - hsh["identity"] = self.identity - return hsh - elif self.event_type == "ActivityTaskCompleted": - hsh = { - "scheduledEventId": self.scheduled_event_id, - "startedEventId": self.started_event_id, - } - if hasattr(self, "result") and self.result is not None: - hsh["result"] = self.result - return hsh - elif self.event_type == "ActivityTaskFailed": - # TODO: maybe merge it with ActivityTaskCompleted (different optional params tho) - hsh = { - "scheduledEventId": self.scheduled_event_id, - "startedEventId": self.started_event_id, - } - if hasattr(self, "reason") and self.reason is not None: - hsh["reason"] = self.reason - if hasattr(self, "details") and self.details is not None: - hsh["details"] = self.details - return hsh - elif self.event_type == "WorkflowExecutionTerminated": - hsh = { - "childPolicy": self.child_policy, - } - if self.cause: - hsh["cause"] = self.cause - if self.details: - hsh["details"] = self.details - if self.reason: - hsh["reason"] = self.reason - return hsh - elif self.event_type == "ActivityTaskTimedOut": - hsh = { - "scheduledEventId": self.scheduled_event_id, - "startedEventId": self.started_event_id, - "timeoutType": self.timeout_type, - } - if self.details: - hsh["details"] = self.details - return hsh - elif self.event_type == "DecisionTaskTimedOut": - return { - "scheduledEventId": self.scheduled_event_id, - "startedEventId": self.started_event_id, - "timeoutType": self.timeout_type, - } - elif self.event_type == "WorkflowExecutionTimedOut": - return { - "childPolicy": self.child_policy, - "timeoutType": self.timeout_type, - } - else: - raise NotImplementedError( - "HistoryEvent does not implement attributes for type '{0}'".format(self.event_type) - ) diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index 67601438b..aa08d8e9f 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -228,14 +228,21 @@ class WorkflowExecution(object): self.start_timestamp = now_timestamp() self._add_event( "WorkflowExecutionStarted", - workflow_execution=self, + child_policy=self.child_policy, + execution_start_to_close_timeout=self.execution_start_to_close_timeout, + # TODO: fix this hardcoded value + parent_initiated_event_id=0, + task_list=self.task_list, + task_start_to_close_timeout=self.task_start_to_close_timeout, + workflow_type=self.workflow_type, ) self.schedule_decision_task() def _schedule_decision_task(self): evt = self._add_event( "DecisionTaskScheduled", - workflow_execution=self, + start_to_close_timeout=self.task_start_to_close_timeout, + task_list=self.task_list, ) self.domain.add_to_decision_task_list( self.task_list, @@ -274,7 +281,6 @@ class WorkflowExecution(object): dt = self._find_decision_task(task_token) evt = self._add_event( "DecisionTaskStarted", - workflow_execution=self, scheduled_event_id=dt.scheduled_event_id, identity=identity ) @@ -419,6 +425,9 @@ class WorkflowExecution(object): def schedule_activity_task(self, event_id, attributes): # Helper function to avoid repeating ourselves in the next sections def fail_schedule_activity_task(_type, _cause): + # TODO: implement other possible failure mode: OPEN_ACTIVITIES_LIMIT_EXCEEDED + # NB: some failure modes are not implemented and probably won't be implemented in + # the future, such as ACTIVITY_CREATION_RATE_EXCEEDED or OPERATION_NOT_PERMITTED self._add_event( "ScheduleActivityTaskFailed", activity_id=attributes["activityId"], @@ -473,10 +482,17 @@ class WorkflowExecution(object): # Only add event and increment counters now that nothing went wrong evt = self._add_event( "ActivityTaskScheduled", - decision_task_completed_event_id=event_id, + activity_id=attributes["activityId"], activity_type=activity_type, - attributes=attributes, + control=attributes.get("control"), + decision_task_completed_event_id=event_id, + heartbeat_timeout=attributes.get("heartbeatTimeout"), + input=attributes.get("input"), + schedule_to_close_timeout=attributes.get("scheduleToCloseTimeout"), + schedule_to_start_timeout=attributes.get("scheduleToStartTimeout"), + start_to_close_timeout=attributes.get("startToCloseTimeout"), task_list=task_list, + task_priority=attributes.get("taskPriority"), ) task = ActivityTask( activity_id=attributes["activityId"], diff --git a/tests/test_swf/models/test_workflow_execution.py b/tests/test_swf/models/test_workflow_execution.py index 8a010eb42..0546c2f93 100644 --- a/tests/test_swf/models/test_workflow_execution.py +++ b/tests/test_swf/models/test_workflow_execution.py @@ -151,13 +151,13 @@ def test_workflow_execution_start_decision_task(): dt = wfe.decision_tasks[0] dt.state.should.equal("STARTED") wfe.events()[-1].event_type.should.equal("DecisionTaskStarted") - wfe.events()[-1].identity.should.equal("srv01") + wfe.events()[-1].event_attributes["identity"].should.equal("srv01") def test_workflow_execution_history_events_ids(): wfe = make_workflow_execution() - wfe._add_event("WorkflowExecutionStarted", workflow_execution=wfe) - wfe._add_event("DecisionTaskScheduled", workflow_execution=wfe) - wfe._add_event("DecisionTaskStarted", workflow_execution=wfe, scheduled_event_id=2) + wfe._add_event("WorkflowExecutionStarted") + wfe._add_event("DecisionTaskScheduled") + wfe._add_event("DecisionTaskStarted") ids = [evt.event_id for evt in wfe.events()] ids.should.equal([1, 2, 3]) @@ -181,8 +181,8 @@ def test_workflow_execution_complete(): wfe.close_status.should.equal("COMPLETED") wfe.close_timestamp.should.equal(1420200000.0) wfe.events()[-1].event_type.should.equal("WorkflowExecutionCompleted") - wfe.events()[-1].decision_task_completed_event_id.should.equal(123) - wfe.events()[-1].result.should.equal("foo") + wfe.events()[-1].event_attributes["decisionTaskCompletedEventId"].should.equal(123) + wfe.events()[-1].event_attributes["result"].should.equal("foo") @freeze_time("2015-01-02 12:00:00") def test_workflow_execution_fail(): @@ -193,9 +193,9 @@ def test_workflow_execution_fail(): wfe.close_status.should.equal("FAILED") wfe.close_timestamp.should.equal(1420200000.0) wfe.events()[-1].event_type.should.equal("WorkflowExecutionFailed") - wfe.events()[-1].decision_task_completed_event_id.should.equal(123) - wfe.events()[-1].details.should.equal("some details") - wfe.events()[-1].reason.should.equal("my rules") + wfe.events()[-1].event_attributes["decisionTaskCompletedEventId"].should.equal(123) + wfe.events()[-1].event_attributes["details"].should.equal("some details") + wfe.events()[-1].event_attributes["reason"].should.equal("my rules") @freeze_time("2015-01-01 12:00:00") def test_workflow_execution_schedule_activity_task(): @@ -209,8 +209,8 @@ def test_workflow_execution_schedule_activity_task(): wfe.open_counts["openActivityTasks"].should.equal(1) last_event = wfe.events()[-1] last_event.event_type.should.equal("ActivityTaskScheduled") - last_event.decision_task_completed_event_id.should.equal(123) - last_event.task_list.should.equal("task-list-name") + last_event.event_attributes["decisionTaskCompletedEventId"].should.equal(123) + last_event.event_attributes["taskList"]["name"].should.equal("task-list-name") wfe.activity_tasks.should.have.length_of(1) task = wfe.activity_tasks[0] @@ -235,7 +235,7 @@ def test_workflow_execution_schedule_activity_task_without_task_list_should_take wfe.open_counts["openActivityTasks"].should.equal(1) last_event = wfe.events()[-1] last_event.event_type.should.equal("ActivityTaskScheduled") - last_event.task_list.should.equal("foobar") + last_event.event_attributes["taskList"]["name"].should.equal("foobar") task = wfe.activity_tasks[0] wfe.domain.activity_task_lists["foobar"].should.contain(task) @@ -255,43 +255,43 @@ def test_workflow_execution_schedule_activity_task_should_fail_if_wrong_attribut wfe.schedule_activity_task(123, hsh) last_event = wfe.events()[-1] last_event.event_type.should.equal("ScheduleActivityTaskFailed") - last_event.cause.should.equal("ACTIVITY_TYPE_DOES_NOT_EXIST") + last_event.event_attributes["cause"].should.equal("ACTIVITY_TYPE_DOES_NOT_EXIST") hsh["activityType"]["name"] = "test-activity" wfe.schedule_activity_task(123, hsh) last_event = wfe.events()[-1] last_event.event_type.should.equal("ScheduleActivityTaskFailed") - last_event.cause.should.equal("ACTIVITY_TYPE_DEPRECATED") + last_event.event_attributes["cause"].should.equal("ACTIVITY_TYPE_DEPRECATED") hsh["activityType"]["version"] = "v1.2" wfe.schedule_activity_task(123, hsh) last_event = wfe.events()[-1] last_event.event_type.should.equal("ScheduleActivityTaskFailed") - last_event.cause.should.equal("DEFAULT_TASK_LIST_UNDEFINED") + last_event.event_attributes["cause"].should.equal("DEFAULT_TASK_LIST_UNDEFINED") hsh["taskList"] = { "name": "foobar" } wfe.schedule_activity_task(123, hsh) last_event = wfe.events()[-1] last_event.event_type.should.equal("ScheduleActivityTaskFailed") - last_event.cause.should.equal("DEFAULT_SCHEDULE_TO_START_TIMEOUT_UNDEFINED") + last_event.event_attributes["cause"].should.equal("DEFAULT_SCHEDULE_TO_START_TIMEOUT_UNDEFINED") hsh["scheduleToStartTimeout"] = "600" wfe.schedule_activity_task(123, hsh) last_event = wfe.events()[-1] last_event.event_type.should.equal("ScheduleActivityTaskFailed") - last_event.cause.should.equal("DEFAULT_SCHEDULE_TO_CLOSE_TIMEOUT_UNDEFINED") + last_event.event_attributes["cause"].should.equal("DEFAULT_SCHEDULE_TO_CLOSE_TIMEOUT_UNDEFINED") hsh["scheduleToCloseTimeout"] = "600" wfe.schedule_activity_task(123, hsh) last_event = wfe.events()[-1] last_event.event_type.should.equal("ScheduleActivityTaskFailed") - last_event.cause.should.equal("DEFAULT_START_TO_CLOSE_TIMEOUT_UNDEFINED") + last_event.event_attributes["cause"].should.equal("DEFAULT_START_TO_CLOSE_TIMEOUT_UNDEFINED") hsh["startToCloseTimeout"] = "600" wfe.schedule_activity_task(123, hsh) last_event = wfe.events()[-1] last_event.event_type.should.equal("ScheduleActivityTaskFailed") - last_event.cause.should.equal("DEFAULT_HEARTBEAT_TIMEOUT_UNDEFINED") + last_event.event_attributes["cause"].should.equal("DEFAULT_HEARTBEAT_TIMEOUT_UNDEFINED") wfe.open_counts["openActivityTasks"].should.equal(0) wfe.activity_tasks.should.have.length_of(0) @@ -351,7 +351,7 @@ def test_workflow_execution_schedule_activity_task_with_same_activity_id(): wfe.open_counts["openActivityTasks"].should.equal(1) last_event = wfe.events()[-1] last_event.event_type.should.equal("ScheduleActivityTaskFailed") - last_event.cause.should.equal("ACTIVITY_ID_ALREADY_IN_USE") + last_event.event_attributes["cause"].should.equal("ACTIVITY_ID_ALREADY_IN_USE") def test_workflow_execution_start_activity_task(): wfe = make_workflow_execution() @@ -361,7 +361,7 @@ def test_workflow_execution_start_activity_task(): task = wfe.activity_tasks[-1] task.state.should.equal("STARTED") wfe.events()[-1].event_type.should.equal("ActivityTaskStarted") - wfe.events()[-1].identity.should.equal("worker01") + wfe.events()[-1].event_attributes["identity"].should.equal("worker01") def test_complete_activity_task(): wfe = make_workflow_execution() @@ -395,7 +395,7 @@ def test_terminate(): last_event = wfe.events()[-1] last_event.event_type.should.equal("WorkflowExecutionTerminated") # take default child_policy if not provided (as here) - last_event.child_policy.should.equal("ABANDON") + last_event.event_attributes["childPolicy"].should.equal("ABANDON") def test_first_timeout(): wfe = make_workflow_execution() From 18fe3e41e9ee724ded8fa4e4021af320685e21ea Mon Sep 17 00:00:00 2001 From: earthmant Date: Mon, 23 Nov 2015 18:07:51 +0200 Subject: [PATCH 91/94] Support default ACL in a VPC a vpc usually has a default acl this makes sure that moto flags it and that the describe response has it in there --- moto/ec2/models.py | 6 ++++-- moto/ec2/responses/network_acls.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 098e86328..cf45546fe 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -2610,10 +2610,12 @@ class NetworkAcl(TaggedEC2Resource): self.vpc_id = vpc_id self.network_acl_entries = [] self.associations = {} - self.default = default + self.default = 'true' if default is True else 'false' def get_filter_value(self, filter_name): - if filter_name == "vpc-id": + if filter_name == "default": + return self.default + elif filter_name == "vpc-id": return self.vpc_id elif filter_name == "association.network-acl-id": return self.id diff --git a/moto/ec2/responses/network_acls.py b/moto/ec2/responses/network_acls.py index 684bf6821..ac90021a8 100644 --- a/moto/ec2/responses/network_acls.py +++ b/moto/ec2/responses/network_acls.py @@ -96,7 +96,7 @@ DESCRIBE_NETWORK_ACL_RESPONSE = """ {{ network_acl.id }} {{ network_acl.vpc_id }} - true + {{ network_acl.default }} {% for entry in network_acl.network_acl_entries %} From f1566cecf46332da2a0fcbbaec967c149b2c40d7 Mon Sep 17 00:00:00 2001 From: Andrew Garrett Date: Tue, 24 Nov 2015 23:44:55 +0000 Subject: [PATCH 92/94] Add support for KMS key rotation operations This adds support for the following KMS endpoints: * EnableKeyRotation * DisableKeyRotation * GetKeyRotationStatus Signed-off-by: Jesse Szwedko --- moto/kms/models.py | 11 ++++++ moto/kms/responses.py | 38 ++++++++++++++++++++ tests/test_kms/test_kms.py | 73 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 122 insertions(+) diff --git a/moto/kms/models.py b/moto/kms/models.py index a5adaede8..ec67759d2 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -15,6 +15,7 @@ class Key(object): self.enabled = True self.region = region self.account_id = "0123456789012" + self.key_rotation_status = False @property def arn(self): @@ -68,6 +69,16 @@ class KmsBackend(BaseBackend): def get_all_aliases(self): return self.key_to_aliases + def enable_key_rotation(self, key_id): + self.keys[key_id].key_rotation_status = True + + def disable_key_rotation(self, key_id): + self.keys[key_id].key_rotation_status = False + + def get_key_rotation_status(self, key_id): + return self.keys[key_id].key_rotation_status + + kms_backends = {} for region in boto.kms.regions(): kms_backends[region.name] = KmsBackend() diff --git a/moto/kms/responses.py b/moto/kms/responses.py index 0c4563f07..196a6b851 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -136,3 +136,41 @@ class KmsResponse(BaseResponse): 'Truncated': False, 'Aliases': response_aliases, }) + + def enable_key_rotation(self): + key_id = self.parameters.get('KeyId') + _assert_valid_key_id(key_id) + try: + self.kms_backend.enable_key_rotation(key_id) + except KeyError: + raise JSONResponseError(404, 'Not Found', body={ + 'message': "Key 'arn:aws:kms:{region}:012345678912:key/{key_id}' does not exist".format(region=self.region,key_id=key_id), + '__type': 'NotFoundException'}) + + return json.dumps(None) + + def disable_key_rotation(self): + key_id = self.parameters.get('KeyId') + _assert_valid_key_id(key_id) + try: + self.kms_backend.disable_key_rotation(key_id) + except KeyError: + raise JSONResponseError(404, 'Not Found', body={ + 'message': "Key 'arn:aws:kms:{region}:012345678912:key/{key_id}' does not exist".format(region=self.region,key_id=key_id), + '__type': 'NotFoundException'}) + return json.dumps(None) + + def get_key_rotation_status(self): + key_id = self.parameters.get('KeyId') + _assert_valid_key_id(key_id) + try: + rotation_enabled = self.kms_backend.get_key_rotation_status(key_id) + except KeyError: + raise JSONResponseError(404, 'Not Found', body={ + 'message': "Key 'arn:aws:kms:{region}:012345678912:key/{key_id}' does not exist".format(region=self.region,key_id=key_id), + '__type': 'NotFoundException'}) + return json.dumps({'KeyRotationEnabled': rotation_enabled}) + +def _assert_valid_key_id(key_id): + if not re.match(r'^[A-F0-9]{8}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{12}$', key_id, re.IGNORECASE): + raise JSONResponseError(404, 'Not Found', body={'message': ' Invalid keyId', '__type': 'NotFoundException'}) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index b68d9538f..5453ccb92 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -47,6 +47,70 @@ def test_list_keys(): keys['Keys'].should.have.length_of(2) +@mock_kms +def test_enable_key_rotation(): + conn = boto.kms.connect_to_region("us-west-2") + + key = conn.create_key(policy="my policy", description="my key", key_usage='ENCRYPT_DECRYPT') + key_id = key['KeyMetadata']['KeyId'] + + conn.enable_key_rotation(key_id) + + conn.get_key_rotation_status(key_id)['KeyRotationEnabled'].should.equal(True) + + +@mock_kms +def test_enable_key_rotation_with_missing_key(): + conn = boto.kms.connect_to_region("us-west-2") + conn.enable_key_rotation.when.called_with("not-a-key").should.throw(JSONResponseError) + + +@mock_kms +def test_disable_key_rotation(): + conn = boto.kms.connect_to_region("us-west-2") + + key = conn.create_key(policy="my policy", description="my key", key_usage='ENCRYPT_DECRYPT') + key_id = key['KeyMetadata']['KeyId'] + + conn.enable_key_rotation(key_id) + conn.get_key_rotation_status(key_id)['KeyRotationEnabled'].should.equal(True) + + conn.disable_key_rotation(key_id) + conn.get_key_rotation_status(key_id)['KeyRotationEnabled'].should.equal(False) + + +@mock_kms +def test_disable_key_rotation_with_missing_key(): + conn = boto.kms.connect_to_region("us-west-2") + conn.disable_key_rotation.when.called_with("not-a-key").should.throw(JSONResponseError) + + +@mock_kms +def test_get_key_rotation_status_with_missing_key(): + conn = boto.kms.connect_to_region("us-west-2") + conn.get_key_rotation_status.when.called_with("not-a-key").should.throw(JSONResponseError) + + +@mock_kms +def test_get_key_rotation_status(): + conn = boto.kms.connect_to_region("us-west-2") + + key = conn.create_key(policy="my policy", description="my key", key_usage='ENCRYPT_DECRYPT') + key_id = key['KeyMetadata']['KeyId'] + + conn.get_key_rotation_status(key_id)['KeyRotationEnabled'].should.equal(False) + + +@mock_kms +def test_create_key_defaults_key_rotation(): + conn = boto.kms.connect_to_region("us-west-2") + + key = conn.create_key(policy="my policy", description="my key", key_usage='ENCRYPT_DECRYPT') + key_id = key['KeyMetadata']['KeyId'] + + conn.get_key_rotation_status(key_id)['KeyRotationEnabled'].should.equal(False) + + @mock_kms def test__create_alias__returns_none_if_correct(): kms = boto.connect_kms() @@ -313,3 +377,12 @@ def test__list_aliases(): len([alias for alias in aliases if 'TargetKeyId' in alias and key_id == alias['TargetKeyId']]).should.equal(3) len(aliases).should.equal(7) + + +@mock_kms +def test__assert_valid_key_id(): + from moto.kms.responses import _assert_valid_key_id + import uuid + + _assert_valid_key_id.when.called_with("not-a-key").should.throw(JSONResponseError) + _assert_valid_key_id.when.called_with(str(uuid.uuid4())).should_not.throw(JSONResponseError) From 705ec314a3df4659b6ac07a5b6eddc3028713d89 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Fri, 27 Nov 2015 14:14:40 -0500 Subject: [PATCH 93/94] Cleanup different places using unix_time() --- moto/core/utils.py | 13 +++++++++++++ moto/dynamodb/models.py | 2 +- moto/dynamodb/utils.py | 6 ------ moto/dynamodb2/models.py | 12 ++++++------ moto/dynamodb2/utils.py | 6 ------ moto/sqs/models.py | 4 ++-- moto/sqs/utils.py | 12 ------------ moto/swf/models/activity_task.py | 11 +++++------ moto/swf/models/decision_task.py | 4 ++-- moto/swf/models/history_event.py | 13 ++++++------- moto/swf/models/timeout.py | 4 ++-- moto/swf/models/workflow_execution.py | 26 ++++++++++++-------------- moto/swf/utils.py | 6 ------ tests/test_core/test_utils.py | 11 +++++++++-- tests/test_swf/test_utils.py | 12 ++---------- 15 files changed, 60 insertions(+), 82 deletions(-) delete mode 100644 moto/dynamodb/utils.py delete mode 100644 moto/dynamodb2/utils.py diff --git a/moto/core/utils.py b/moto/core/utils.py index 81acdd6db..65f0f4576 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -1,4 +1,6 @@ from __future__ import unicode_literals + +import datetime import inspect import random import re @@ -103,3 +105,14 @@ def iso_8601_datetime_with_milliseconds(datetime): def rfc_1123_datetime(datetime): RFC1123 = '%a, %d %b %Y %H:%M:%S GMT' return datetime.strftime(RFC1123) + + +def unix_time(dt=None): + dt = dt or datetime.datetime.utcnow() + epoch = datetime.datetime.utcfromtimestamp(0) + delta = dt - epoch + return (delta.days * 86400) + (delta.seconds + (delta.microseconds / 1e6)) + + +def unix_time_millis(dt=None): + return unix_time(dt) * 1000.0 diff --git a/moto/dynamodb/models.py b/moto/dynamodb/models.py index 6039238e7..db595c28c 100644 --- a/moto/dynamodb/models.py +++ b/moto/dynamodb/models.py @@ -5,8 +5,8 @@ import json from moto.compat import OrderedDict from moto.core import BaseBackend +from moto.core.utils import unix_time from .comparisons import get_comparison_func -from .utils import unix_time class DynamoJsonEncoder(json.JSONEncoder): diff --git a/moto/dynamodb/utils.py b/moto/dynamodb/utils.py deleted file mode 100644 index 1adee245b..000000000 --- a/moto/dynamodb/utils.py +++ /dev/null @@ -1,6 +0,0 @@ -from __future__ import unicode_literals -import calendar - - -def unix_time(dt): - return calendar.timegm(dt.timetuple()) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 612a0c3d3..2b1a1b5ee 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -5,8 +5,8 @@ import json from moto.compat import OrderedDict from moto.core import BaseBackend +from moto.core.utils import unix_time from .comparisons import get_comparison_func -from .utils import unix_time class DynamoJsonEncoder(json.JSONEncoder): @@ -82,7 +82,7 @@ class Item(object): attributes = {} for attribute_key, attribute in self.attrs.items(): attributes[attribute_key] = { - attribute.type : attribute.value + attribute.type: attribute.value } return { @@ -204,7 +204,7 @@ class Table(object): keys.append(key['AttributeName']) return keys - def put_item(self, item_attrs, expected = None, overwrite = False): + def put_item(self, item_attrs, expected=None, overwrite=False): hash_value = DynamoType(item_attrs.get(self.hash_key_attr)) if self.has_range_key: range_value = DynamoType(item_attrs.get(self.range_key_attr)) @@ -228,13 +228,13 @@ class Table(object): if current is None: current_attr = {} - elif hasattr(current,'attrs'): + elif hasattr(current, 'attrs'): current_attr = current.attrs else: current_attr = current for key, val in expected.items(): - if 'Exists' in val and val['Exists'] == False: + if 'Exists' in val and val['Exists'] is False: if key in current_attr: raise ValueError("The conditional request failed") elif key not in current_attr: @@ -361,7 +361,7 @@ class DynamoDBBackend(BaseBackend): table.throughput = throughput return table - def put_item(self, table_name, item_attrs, expected = None, overwrite = False): + def put_item(self, table_name, item_attrs, expected=None, overwrite=False): table = self.tables.get(table_name) if not table: return None diff --git a/moto/dynamodb2/utils.py b/moto/dynamodb2/utils.py deleted file mode 100644 index 1adee245b..000000000 --- a/moto/dynamodb2/utils.py +++ /dev/null @@ -1,6 +0,0 @@ -from __future__ import unicode_literals -import calendar - - -def unix_time(dt): - return calendar.timegm(dt.timetuple()) diff --git a/moto/sqs/models.py b/moto/sqs/models.py index efb75dd9c..19268d519 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -8,8 +8,8 @@ from xml.sax.saxutils import escape import boto.sqs from moto.core import BaseBackend -from moto.core.utils import camelcase_to_underscores, get_random_message_id -from .utils import generate_receipt_handle, unix_time_millis +from moto.core.utils import camelcase_to_underscores, get_random_message_id, unix_time_millis +from .utils import generate_receipt_handle from .exceptions import ( ReceiptHandleIsInvalid, MessageNotInflight diff --git a/moto/sqs/utils.py b/moto/sqs/utils.py index 8dee7003f..a00ec1c79 100644 --- a/moto/sqs/utils.py +++ b/moto/sqs/utils.py @@ -1,5 +1,4 @@ from __future__ import unicode_literals -import datetime import random import string @@ -12,17 +11,6 @@ def generate_receipt_handle(): return ''.join(random.choice(string.ascii_lowercase) for x in range(length)) -def unix_time(dt=None): - dt = dt or datetime.datetime.utcnow() - epoch = datetime.datetime.utcfromtimestamp(0) - delta = dt - epoch - return (delta.days * 86400) + (delta.seconds + (delta.microseconds / 1e6)) - - -def unix_time_millis(dt=None): - return unix_time(dt) * 1000.0 - - def parse_message_attributes(querystring, base='', value_namespace='Value.'): message_attributes = {} index = 1 diff --git a/moto/swf/models/activity_task.py b/moto/swf/models/activity_task.py index 1a72d3449..eb361d258 100644 --- a/moto/swf/models/activity_task.py +++ b/moto/swf/models/activity_task.py @@ -2,8 +2,8 @@ from __future__ import unicode_literals from datetime import datetime import uuid +from moto.core.utils import unix_time from ..exceptions import SWFWorkflowExecutionClosedError -from ..utils import now_timestamp from .timeout import Timeout @@ -15,7 +15,7 @@ class ActivityTask(object): self.activity_type = activity_type self.details = None self.input = input - self.last_heartbeat_timestamp = now_timestamp() + self.last_heartbeat_timestamp = unix_time() self.scheduled_event_id = scheduled_event_id self.started_event_id = None self.state = "SCHEDULED" @@ -60,19 +60,18 @@ class ActivityTask(object): self.state = "FAILED" def reset_heartbeat_clock(self): - self.last_heartbeat_timestamp = now_timestamp() + self.last_heartbeat_timestamp = unix_time() def first_timeout(self): if not self.open or not self.workflow_execution.open: return None # TODO: handle the "NONE" case - heartbeat_timeout_at = self.last_heartbeat_timestamp + \ - int(self.timeouts["heartbeatTimeout"]) + heartbeat_timeout_at = (self.last_heartbeat_timestamp + + int(self.timeouts["heartbeatTimeout"])) _timeout = Timeout(self, heartbeat_timeout_at, "HEARTBEAT") if _timeout.reached: return _timeout - def process_timeouts(self): _timeout = self.first_timeout() if _timeout: diff --git a/moto/swf/models/decision_task.py b/moto/swf/models/decision_task.py index b76888403..bcd28f372 100644 --- a/moto/swf/models/decision_task.py +++ b/moto/swf/models/decision_task.py @@ -2,8 +2,8 @@ from __future__ import unicode_literals from datetime import datetime import uuid +from moto.core.utils import unix_time from ..exceptions import SWFWorkflowExecutionClosedError -from ..utils import now_timestamp from .timeout import Timeout @@ -49,7 +49,7 @@ class DecisionTask(object): def start(self, started_event_id): self.state = "STARTED" - self.started_timestamp = now_timestamp() + self.started_timestamp = unix_time() self.started_event_id = started_event_id def complete(self): diff --git a/moto/swf/models/history_event.py b/moto/swf/models/history_event.py index c602abee8..b181297f7 100644 --- a/moto/swf/models/history_event.py +++ b/moto/swf/models/history_event.py @@ -1,10 +1,8 @@ from __future__ import unicode_literals -from datetime import datetime -from time import mktime -from moto.core.utils import underscores_to_camelcase +from moto.core.utils import underscores_to_camelcase, unix_time -from ..utils import decapitalize, now_timestamp +from ..utils import decapitalize # We keep track of which history event types we support @@ -28,6 +26,7 @@ SUPPORTED_HISTORY_EVENT_TYPES = ( "WorkflowExecutionTimedOut", ) + class HistoryEvent(object): def __init__(self, event_id, event_type, event_timestamp=None, **kwargs): if event_type not in SUPPORTED_HISTORY_EVENT_TYPES: @@ -39,16 +38,16 @@ class HistoryEvent(object): if event_timestamp: self.event_timestamp = event_timestamp else: - self.event_timestamp = now_timestamp() + self.event_timestamp = unix_time() # pre-populate a dict: {"camelCaseKey": value} self.event_attributes = {} for key, value in kwargs.items(): if value: camel_key = underscores_to_camelcase(key) if key == "task_list": - value = { "name": value } + value = {"name": value} elif key == "workflow_type": - value = { "name": value.name, "version": value.version } + value = {"name": value.name, "version": value.version} elif key == "activity_type": value = value.to_short_dict() self.event_attributes[camel_key] = value diff --git a/moto/swf/models/timeout.py b/moto/swf/models/timeout.py index 66cb3b84c..cf0283760 100644 --- a/moto/swf/models/timeout.py +++ b/moto/swf/models/timeout.py @@ -1,4 +1,4 @@ -from ..utils import now_timestamp +from moto.core.utils import unix_time class Timeout(object): @@ -9,4 +9,4 @@ class Timeout(object): @property def reached(self): - return now_timestamp() >= self.timestamp + return unix_time() >= self.timestamp diff --git a/moto/swf/models/workflow_execution.py b/moto/swf/models/workflow_execution.py index aa08d8e9f..b241debce 100644 --- a/moto/swf/models/workflow_execution.py +++ b/moto/swf/models/workflow_execution.py @@ -1,9 +1,7 @@ from __future__ import unicode_literals -from datetime import datetime -from time import mktime import uuid -from moto.core.utils import camelcase_to_underscores +from moto.core.utils import camelcase_to_underscores, unix_time from ..constants import ( DECISIONS_FIELDS, @@ -13,7 +11,7 @@ from ..exceptions import ( SWFValidationException, SWFDecisionValidationException, ) -from ..utils import decapitalize, now_timestamp +from ..utils import decapitalize from .activity_task import ActivityTask from .activity_type import ActivityType from .decision_task import DecisionTask @@ -59,7 +57,7 @@ class WorkflowExecution(object): self.latest_execution_context = None self.parent = None self.start_timestamp = None - self.tag_list = [] # TODO + self.tag_list = [] # TODO self.timeout_type = None self.workflow_type = workflow_type # args processing @@ -89,7 +87,7 @@ class WorkflowExecution(object): def _set_from_kwargs_or_workflow_type(self, kwargs, local_key, workflow_type_key=None): if workflow_type_key is None: - workflow_type_key = "default_"+local_key + workflow_type_key = "default_" + local_key value = kwargs.get(local_key) if not value and hasattr(self.workflow_type, workflow_type_key): value = getattr(self.workflow_type, workflow_type_key) @@ -131,7 +129,7 @@ class WorkflowExecution(object): "taskList": {"name": self.task_list} } } - #configuration + # configuration for key in self._configuration_keys: attr = camelcase_to_underscores(key) if not hasattr(self, attr): @@ -139,9 +137,9 @@ class WorkflowExecution(object): if not getattr(self, attr): continue hsh["executionConfiguration"][key] = getattr(self, attr) - #counters + # counters hsh["openCounts"] = self.open_counts - #latest things + # latest things if self.latest_execution_context: hsh["latestExecutionContext"] = self.latest_execution_context if self.latest_activity_task_timestamp: @@ -225,7 +223,7 @@ class WorkflowExecution(object): return evt def start(self): - self.start_timestamp = now_timestamp() + self.start_timestamp = unix_time() self._add_event( "WorkflowExecutionStarted", child_policy=self.child_policy, @@ -403,7 +401,7 @@ class WorkflowExecution(object): def complete(self, event_id, result=None): self.execution_status = "CLOSED" self.close_status = "COMPLETED" - self.close_timestamp = now_timestamp() + self.close_timestamp = unix_time() self._add_event( "WorkflowExecutionCompleted", decision_task_completed_event_id=event_id, @@ -414,7 +412,7 @@ class WorkflowExecution(object): # TODO: implement lenght constraints on details/reason self.execution_status = "CLOSED" self.close_status = "FAILED" - self.close_timestamp = now_timestamp() + self.close_timestamp = unix_time() self._add_event( "WorkflowExecutionFailed", decision_task_completed_event_id=event_id, @@ -470,7 +468,7 @@ class WorkflowExecution(object): # find timeouts or default timeout, else fail timeouts = {} for _type in ["scheduleToStartTimeout", "scheduleToCloseTimeout", "startToCloseTimeout", "heartbeatTimeout"]: - default_key = "default_task_"+camelcase_to_underscores(_type) + default_key = "default_task_" + camelcase_to_underscores(_type) default_value = getattr(activity_type, default_key) timeouts[_type] = attributes.get(_type, default_value) if not timeouts[_type]: @@ -504,7 +502,7 @@ class WorkflowExecution(object): ) self.domain.add_to_activity_task_list(task_list, task) self.open_counts["openActivityTasks"] += 1 - self.latest_activity_task_timestamp = now_timestamp() + self.latest_activity_task_timestamp = unix_time() def _find_activity_task(self, task_token): for task in self.activity_tasks: diff --git a/moto/swf/utils.py b/moto/swf/utils.py index a9c54ee3a..de628ce50 100644 --- a/moto/swf/utils.py +++ b/moto/swf/utils.py @@ -1,9 +1,3 @@ -from datetime import datetime -from time import mktime - def decapitalize(key): return key[0].lower() + key[1:] - -def now_timestamp(): - return float(mktime(datetime.utcnow().timetuple())) diff --git a/tests/test_core/test_utils.py b/tests/test_core/test_utils.py index 6e27e6f49..76f0645af 100644 --- a/tests/test_core/test_utils.py +++ b/tests/test_core/test_utils.py @@ -1,7 +1,9 @@ from __future__ import unicode_literals -import sure -from moto.core.utils import camelcase_to_underscores, underscores_to_camelcase +import sure # noqa +from freezegun import freeze_time + +from moto.core.utils import camelcase_to_underscores, underscores_to_camelcase, unix_time def test_camelcase_to_underscores(): @@ -20,3 +22,8 @@ def test_underscores_to_camelcase(): } for arg, expected in cases.items(): underscores_to_camelcase(arg).should.equal(expected) + + +@freeze_time("2015-01-01 12:00:00") +def test_unix_time(): + unix_time().should.equal(1420113600.0) diff --git a/tests/test_swf/test_utils.py b/tests/test_swf/test_utils.py index f8ff08f22..ffa147037 100644 --- a/tests/test_swf/test_utils.py +++ b/tests/test_swf/test_utils.py @@ -1,10 +1,6 @@ -from freezegun import freeze_time -from sure import expect +import sure # noqa -from moto.swf.utils import ( - decapitalize, - now_timestamp, -) +from moto.swf.utils import decapitalize def test_decapitalize(): @@ -15,7 +11,3 @@ def test_decapitalize(): } for before, after in cases.items(): decapitalize(before).should.equal(after) - -@freeze_time("2015-01-01 12:00:00") -def test_now_timestamp(): - now_timestamp().should.equal(1420113600.0) From 704110d9c76b08db96f31cb7b63da45bc0396d4e Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Fri, 27 Nov 2015 14:46:50 -0500 Subject: [PATCH 94/94] 0.4.19 --- moto/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/__init__.py b/moto/__init__.py index 70ba9ca20..fbbca07be 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '0.4.18' +__version__ = '0.4.19' from .autoscaling import mock_autoscaling # flake8: noqa from .cloudformation import mock_cloudformation # flake8: noqa diff --git a/setup.py b/setup.py index a65f5e15e..f24155c15 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ extras_require = { setup( name='moto', - version='0.4.18', + version='0.4.19', description='A library that allows your python tests to easily' ' mock out the boto library', author='Steve Pulec',