Fix scaffold to support rest-json style API (#1291)

* append appropriate urls when scaffolding

* make dispatch for rest-api

* fix dispatch for rest-json

* fix moto/core/response to obtain path and body parameters

* small fixes

* remove unused import

* fix get_int_param

* fix scaffold

* fix formatting of scaffold

* fix misc

* escape service to handle service w/ hyphen like iot-data

* escape service w/ hyphen

* fix regexp to extract region from url

* escape service

* fix syntax

* skip loading body to json object when request body is None

moto/backends.py

@@ -37,6 +37,7 @@ from moto.sts import sts_backends
 from moto.xray import xray_backends
 from moto.batch import batch_backends
 
+
 BACKENDS = {
     'acm': acm_backends,
     'apigateway': apigateway_backends,
@@ -74,7 +75,7 @@ BACKENDS = {
     'sts': sts_backends,
     'route53': route53_backends,
     'lambda': lambda_backends,
-    'xray': xray_backends
+    'xray': xray_backends,
 }
 
 

moto/core/responses.py

@@ -17,6 +17,8 @@ from six.moves.urllib.parse import parse_qs, urlparse
 import xmltodict
 from pkg_resources import resource_filename
 from werkzeug.exceptions import HTTPException
+
+import boto3
 from moto.compat import OrderedDict
 from moto.core.utils import camelcase_to_underscores, method_names_from_class
 
@@ -103,7 +105,8 @@ class _TemplateEnvironmentMixin(object):
 class BaseResponse(_TemplateEnvironmentMixin):
 
     default_region = 'us-east-1'
-    region_regex = r'\.(.+?)\.amazonaws\.com'
+    # to extract region, use [^.]
+    region_regex = r'\.([^.]+?)\.amazonaws\.com'
     aws_service_spec = None
 
     @classmethod
@@ -151,12 +154,12 @@ class BaseResponse(_TemplateEnvironmentMixin):
             querystring.update(headers)
 
         querystring = _decode_dict(querystring)
-
         self.uri = full_url
         self.path = urlparse(full_url).path
         self.querystring = querystring
         self.method = request.method
         self.region = self.get_region_from_url(request, full_url)
+        self.uri_match = None
 
         self.headers = request.headers
         if 'host' not in self.headers:
@@ -178,6 +181,58 @@ class BaseResponse(_TemplateEnvironmentMixin):
         self.setup_class(request, full_url, headers)
         return self.call_action()
 
+    def uri_to_regexp(self, uri):
+        """converts uri w/ placeholder to regexp
+          '/cars/{carName}/drivers/{DriverName}'
+        -> '^/cars/.*/drivers/[^/]*$'
+
+          '/cars/{carName}/drivers/{DriverName}/drive'
+        -> '^/cars/.*/drivers/.*/drive$'
+
+        """
+        def _convert(elem, is_last):
+            if not re.match('^{.*}$', elem):
+                return elem
+            name = elem.replace('{', '').replace('}', '')
+            if is_last:
+                return '(?P<%s>[^/]*)' % name
+            return '(?P<%s>.*)' % name
+
+        elems = uri.split('/')
+        num_elems = len(elems)
+        regexp = '^{}$'.format('/'.join([_convert(elem, (i == num_elems - 1)) for i, elem in enumerate(elems)]))
+        return regexp
+
+    def _get_action_from_method_and_request_uri(self, method, request_uri):
+        """basically used for `rest-json` APIs
+        You can refer to example from link below
+        https://github.com/boto/botocore/blob/develop/botocore/data/iot/2015-05-28/service-2.json
+        """
+
+        # service response class should have 'SERVICE_NAME' class member,
+        # if you want to get action from method and url
+        if not hasattr(self, 'SERVICE_NAME'):
+            return None
+        service = self.SERVICE_NAME
+        conn = boto3.client(service)
+
+        # make cache if it does not exist yet
+        if not hasattr(self, 'method_urls'):
+            self.method_urls = defaultdict(lambda: defaultdict(str))
+            op_names = conn._service_model.operation_names
+            for op_name in op_names:
+                op_model = conn._service_model.operation_model(op_name)
+                _method = op_model.http['method']
+                uri_regexp = self.uri_to_regexp(op_model.http['requestUri'])
+                self.method_urls[_method][uri_regexp] = op_model.name
+        regexp_and_names = self.method_urls[method]
+        for regexp, name in regexp_and_names.items():
+            match = re.match(regexp, request_uri)
+            self.uri_match = match
+            if match:
+                return name
+        return None
+
     def _get_action(self):
         action = self.querystring.get('Action', [""])[0]
         if not action:  # Some services use a header for the action
@@ -186,7 +241,9 @@ class BaseResponse(_TemplateEnvironmentMixin):
                 'x-amz-target') or self.headers.get('X-Amz-Target')
             if match:
                 action = match.split(".")[-1]
-
+        # get action from method and uri
+        if not action:
+            return self._get_action_from_method_and_request_uri(self.method, self.path)
         return action
 
     def call_action(self):
@@ -221,6 +278,22 @@ class BaseResponse(_TemplateEnvironmentMixin):
         val = self.querystring.get(param_name)
         if val is not None:
             return val[0]
+
+        # try to get json body parameter
+        if self.body is not None:
+            try:
+                return json.loads(self.body)[param_name]
+            except ValueError:
+                pass
+            except KeyError:
+                pass
+        # try to get path parameter
+        if self.uri_match:
+            try:
+                return self.uri_match.group(param_name)
+            except IndexError:
+                # do nothing if param is not found
+                pass
         return if_none
 
     def _get_int_param(self, param_name, if_none=None):

scripts/scaffold.py

@@ -81,12 +81,14 @@ def select_service_and_operation():
         raise click.Abort()
     return service_name, operation_name
 
+def get_escaped_service(service):
+    return service.replace('-', '')
 
 def get_lib_dir(service):
-    return os.path.join('moto', service)
+    return os.path.join('moto', get_escaped_service(service))
 
 def get_test_dir(service):
-    return os.path.join('tests', 'test_{}'.format(service))
+    return os.path.join('tests', 'test_{}'.format(get_escaped_service(service)))
 
 
 def render_template(tmpl_dir, tmpl_filename, context, service, alt_filename=None):
@@ -117,7 +119,7 @@ def append_mock_to_init_py(service):
     filtered_lines = [_ for _ in lines if re.match('^from.*mock.*$', _)]
     last_import_line_index = lines.index(filtered_lines[-1])
 
-    new_line = 'from .{} import mock_{}  # flake8: noqa'.format(service, service)
+    new_line = 'from .{} import mock_{}  # flake8: noqa'.format(get_escaped_service(service), get_escaped_service(service))
     lines.insert(last_import_line_index + 1, new_line)
 
     body = '\n'.join(lines) + '\n'
@@ -135,7 +137,7 @@ def append_mock_import_to_backends_py(service):
     filtered_lines = [_ for _ in lines if re.match('^from.*backends.*$', _)]
     last_import_line_index = lines.index(filtered_lines[-1])
 
-    new_line = 'from moto.{} import {}_backends'.format(service, service)
+    new_line = 'from moto.{} import {}_backends'.format(get_escaped_service(service), get_escaped_service(service))
     lines.insert(last_import_line_index + 1, new_line)
 
     body = '\n'.join(lines) + '\n'
@@ -147,13 +149,12 @@ def append_mock_dict_to_backends_py(service):
     with open(path) as f:
         lines = [_.replace('\n', '') for _ in f.readlines()]
 
-        #     'xray': xray_backends
     if any(_ for _ in lines if re.match(".*'{}': {}_backends.*".format(service, service), _)):
         return
     filtered_lines = [_ for _ in lines if re.match(".*'.*':.*_backends.*", _)]
     last_elem_line_index = lines.index(filtered_lines[-1])
 
-    new_line = "    '{}': {}_backends,".format(service, service)
+    new_line = "    '{}': {}_backends,".format(service, get_escaped_service(service))
     prev_line = lines[last_elem_line_index]
     if not prev_line.endswith('{') and not prev_line.endswith(','):
         lines[last_elem_line_index] += ','
@@ -166,8 +167,8 @@ def append_mock_dict_to_backends_py(service):
 def initialize_service(service, operation, api_protocol):
     """create lib and test dirs if not exist
     """
-    lib_dir = os.path.join('moto', service)
+    lib_dir = get_lib_dir(service)
-    test_dir = os.path.join('tests', 'test_{}'.format(service))
+    test_dir = get_test_dir(service)
 
     print_progress('Initializing service', service, 'green')
 
@@ -178,7 +179,9 @@ def initialize_service(service, operation, api_protocol):
     tmpl_context = {
         'service': service,
         'service_class': service_class,
-        'endpoint_prefix': endpoint_prefix
+        'endpoint_prefix': endpoint_prefix,
+        'api_protocol': api_protocol,
+        'escaped_service': get_escaped_service(service)
     }
 
     # initialize service directory
@@ -202,7 +205,7 @@ def initialize_service(service, operation, api_protocol):
         os.makedirs(test_dir)
     tmpl_dir = os.path.join(TEMPLATE_DIR, 'test')
     for tmpl_filename in os.listdir(tmpl_dir):
-        alt_filename = 'test_{}.py'.format(service) if tmpl_filename == 'test_service.py.j2' else None
+        alt_filename = 'test_{}.py'.format(get_escaped_service(service)) if tmpl_filename == 'test_service.py.j2' else None
         render_template(
             tmpl_dir, tmpl_filename, tmpl_context, service, alt_filename
         )
@@ -212,9 +215,16 @@ def initialize_service(service, operation, api_protocol):
     append_mock_import_to_backends_py(service)
     append_mock_dict_to_backends_py(service)
 
+
 def to_upper_camel_case(s):
     return ''.join([_.title() for _ in s.split('_')])
 
+
+def to_lower_camel_case(s):
+    words = s.split('_')
+    return ''.join(words[:1] + [_.title() for _ in words[1:]])
+
+
 def to_snake_case(s):
     s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', s)
     return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
@@ -229,25 +239,28 @@ def get_function_in_responses(service, operation, protocol):
 
     aws_operation_name = to_upper_camel_case(operation)
     op_model = client._service_model.operation_model(aws_operation_name)
+    if not hasattr(op_model.output_shape, 'members'):
+        outputs = {}
+    else:
         outputs = op_model.output_shape.members
     inputs = op_model.input_shape.members
     input_names = [to_snake_case(_) for _ in inputs.keys() if _ not in INPUT_IGNORED_IN_BACKEND]
     output_names = [to_snake_case(_) for _ in outputs.keys() if _ not in OUTPUT_IGNORED_IN_BACKEND]
-    body = 'def {}(self):\n'.format(operation)
+    body = '\ndef {}(self):\n'.format(operation)
 
     for input_name, input_type in inputs.items():
         type_name = input_type.type_name
         if type_name == 'integer':
-            arg_line_tmpl = '    {} = _get_int_param("{}")\n'
+            arg_line_tmpl = '    {} = self._get_int_param("{}")\n'
         elif type_name == 'list':
             arg_line_tmpl = '    {} = self._get_list_prefix("{}.member")\n'
         else:
             arg_line_tmpl = '    {} = self._get_param("{}")\n'
         body += arg_line_tmpl.format(to_snake_case(input_name), input_name)
     if output_names:
-        body += '    {} = self.{}_backend.{}(\n'.format(','.join(output_names), service, operation)
+        body += '    {} = self.{}_backend.{}(\n'.format(', '.join(output_names), get_escaped_service(service), operation)
     else:
-        body += '    self.{}_backend.{}(\n'.format(service, operation)
+        body += '    self.{}_backend.{}(\n'.format(get_escaped_service(service), operation)
     for input_name in input_names:
         body += '        {}={},\n'.format(input_name, input_name)
 
@@ -257,9 +270,9 @@ def get_function_in_responses(service, operation, protocol):
         body += '    return template.render({})\n'.format(
             ', '.join(['{}={}'.format(_, _) for _ in output_names])
         )
-    elif protocol == 'json':
+    elif protocol in ['json', 'rest-json']:
-        body += '    # TODO: adjust reponse\n'
+        body += '    # TODO: adjust response\n'
-        body += '    return json.dumps({})\n'.format(','.join(['{}={}'.format(_, _) for _ in output_names]))
+        body += '    return json.dumps(dict({}))\n'.format(', '.join(['{}={}'.format(to_lower_camel_case(_), _) for _ in output_names]))
     return body
 
 
@@ -272,6 +285,9 @@ def get_function_in_models(service, operation):
     aws_operation_name = to_upper_camel_case(operation)
     op_model = client._service_model.operation_model(aws_operation_name)
     inputs = op_model.input_shape.members
+    if not hasattr(op_model.output_shape, 'members'):
+        outputs = {}
+    else:
         outputs = op_model.output_shape.members
     input_names = [to_snake_case(_) for _ in inputs.keys() if _ not in INPUT_IGNORED_IN_BACKEND]
     output_names = [to_snake_case(_) for _ in outputs.keys() if _ not in OUTPUT_IGNORED_IN_BACKEND]
@@ -280,7 +296,7 @@ def get_function_in_models(service, operation):
     else:
         body = 'def {}(self)\n'
     body += '    # implement here\n'
-    body += '    return {}\n'.format(', '.join(output_names))
+    body += '    return {}\n\n'.format(', '.join(output_names))
 
     return body
 
@@ -388,13 +404,13 @@ def insert_code_to_class(path, base_class, new_code):
         f.write(body)
 
 
-def insert_url(service, operation):
+def insert_url(service, operation, api_protocol):
     client = boto3.client(service)
     service_class = client.__class__.__name__
     aws_operation_name = to_upper_camel_case(operation)
     uri = client._service_model.operation_model(aws_operation_name).http['requestUri']
 
-    path = os.path.join(os.path.dirname(__file__), '..', 'moto', service, 'urls.py')
+    path = os.path.join(os.path.dirname(__file__), '..', 'moto', get_escaped_service(service), 'urls.py')
     with open(path) as f:
         lines = [_.replace('\n', '') for _ in f.readlines()]
 
@@ -413,27 +429,32 @@ def insert_url(service, operation):
     if not prev_line.endswith('{') and not prev_line.endswith(','):
         lines[last_elem_line_index] += ','
 
+    # generate url pattern
+    if api_protocol == 'rest-json':
+        new_line = "    '{0}/.*$': response.dispatch,"
+    else:
         new_line = "    '{0}%s$': %sResponse.dispatch," % (
             uri, service_class
         )
+    if new_line in lines:
+        return
     lines.insert(last_elem_line_index + 1, new_line)
 
     body = '\n'.join(lines) + '\n'
     with open(path, 'w') as f:
         f.write(body)
 
-
+def insert_codes(service, operation, api_protocol):
-def insert_query_codes(service, operation):
+    func_in_responses = get_function_in_responses(service, operation, api_protocol)
-    func_in_responses = get_function_in_responses(service, operation, 'query')
     func_in_models = get_function_in_models(service, operation)
-    template = get_response_query_template(service, operation)
-
     # edit responses.py
-    responses_path = 'moto/{}/responses.py'.format(service)
+    responses_path = 'moto/{}/responses.py'.format(get_escaped_service(service))
     print_progress('inserting code', responses_path, 'green')
     insert_code_to_class(responses_path, BaseResponse, func_in_responses)
 
     # insert template
+    if api_protocol == 'query':
+        template = get_response_query_template(service, operation)
         with open(responses_path) as f:
             lines = [_[:-1] for _ in f.readlines()]
         lines += template.splitlines()
@@ -441,53 +462,22 @@ def insert_query_codes(service, operation):
             f.write('\n'.join(lines))
 
     # edit models.py
-    models_path = 'moto/{}/models.py'.format(service)
+    models_path = 'moto/{}/models.py'.format(get_escaped_service(service))
     print_progress('inserting code', models_path, 'green')
     insert_code_to_class(models_path, BaseBackend, func_in_models)
 
     # edit urls.py
-    insert_url(service, operation)
+    insert_url(service, operation, api_protocol)
 
-def insert_json_codes(service, operation):
-    func_in_responses = get_function_in_responses(service, operation, 'json')
-    func_in_models = get_function_in_models(service, operation)
-
-    # edit responses.py
-    responses_path = 'moto/{}/responses.py'.format(service)
-    print_progress('inserting code', responses_path, 'green')
-    insert_code_to_class(responses_path, BaseResponse, func_in_responses)
-
-    # edit models.py
-    models_path = 'moto/{}/models.py'.format(service)
-    print_progress('inserting code', models_path, 'green')
-    insert_code_to_class(models_path, BaseBackend, func_in_models)
-
-    # edit urls.py
-    insert_url(service, operation)
-
-def insert_restjson_codes(service, operation):
-    func_in_models = get_function_in_models(service, operation)
-
-    print_progress('skipping inserting code to responses.py', "dont't know how to implement", 'yellow')
-    # edit models.py
-    models_path = 'moto/{}/models.py'.format(service)
-    print_progress('inserting code', models_path, 'green')
-    insert_code_to_class(models_path, BaseBackend, func_in_models)
-
-    # edit urls.py
-    insert_url(service, operation)
 
 @click.command()
 def main():
     service, operation = select_service_and_operation()
     api_protocol = boto3.client(service)._service_model.metadata['protocol']
     initialize_service(service, operation, api_protocol)
-    if api_protocol == 'query':
+
-        insert_query_codes(service, operation)
+    if api_protocol in ['query', 'json', 'rest-json']:
-    elif api_protocol == 'json':
+        insert_codes(service, operation, api_protocol)
-        insert_json_codes(service, operation)
-    elif api_protocol == 'rest-json':
-        insert_restjson_codes(service, operation)
     else:
         print_progress('skip inserting code', 'api protocol "{}" is not supported'.format(api_protocol), 'yellow')
 

scripts/template/lib/__init__.py.j2

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
-from .models import {{ service }}_backends
+from .models import {{ escaped_service }}_backends
 from ..core.models import base_decorator
 
-{{ service }}_backend = {{ service }}_backends['us-east-1']
+{{ escaped_service }}_backend = {{ escaped_service }}_backends['us-east-1']
-mock_{{ service }} = base_decorator({{ service }}_backends)
+mock_{{ escaped_service }} = base_decorator({{ escaped_service }}_backends)
 

scripts/template/lib/models.py.j2

@@ -17,4 +17,4 @@ class {{ service_class }}Backend(BaseBackend):
 
 
 available_regions = boto3.session.Session().get_available_regions("{{ service }}")
-{{ service }}_backends = {region: {{ service_class }}Backend(region) for region in available_regions}
+{{ escaped_service }}_backends = {region: {{ service_class }}Backend(region) for region in available_regions}

scripts/template/lib/responses.py.j2

@@ -1,12 +1,14 @@
 from __future__ import unicode_literals
 from moto.core.responses import BaseResponse
-from .models import {{ service }}_backends
+from .models import {{ escaped_service }}_backends
+import json
 
 
 class {{ service_class }}Response(BaseResponse):
+    SERVICE_NAME = '{{ service }}'
     @property
-    def {{ service }}_backend(self):
+    def {{ escaped_service }}_backend(self):
-        return {{ service }}_backends[self.region]
+        return {{ escaped_service }}_backends[self.region]
 
     # add methods from here
 

scripts/template/lib/urls.py.j2

@@ -5,5 +5,9 @@ url_bases = [
     "https?://{{ endpoint_prefix }}.(.+).amazonaws.com",
 ]
 
+{% if api_protocol == 'rest-json' %}
+response = {{ service_class }}Response()
+{% endif %}
+
 url_paths = {
 }

scripts/template/test/test_server.py.j2

@@ -3,14 +3,14 @@ from __future__ import unicode_literals
 import sure  # noqa
 
 import moto.server as server
-from moto import mock_{{ service }}
+from moto import mock_{{ escaped_service }}
 
 '''
 Test the different server responses
 '''
 
-@mock_{{ service }}
+@mock_{{ escaped_service }}
-def test_{{ service }}_list():
+def test_{{ escaped_service }}_list():
     backend = server.create_backend_app("{{ service }}")
     test_client = backend.test_client()
     # do test

scripts/template/test/test_service.py.j2

@@ -2,10 +2,10 @@ from __future__ import unicode_literals
 
 import boto3
 import sure  # noqa
-from moto import mock_{{ service }}
+from moto import mock_{{ escaped_service }}
 
 
-@mock_{{ service }}
+@mock_{{ escaped_service }}
 def test_list():
     # do test
     pass