From 56793a3b2a54879b9a7e1a555faf957e9fa6d7dc Mon Sep 17 00:00:00 2001 From: Toshiya Kawasaki Date: Wed, 25 Oct 2017 03:45:39 +0900 Subject: [PATCH] Fix scaffold to support rest-json style API (#1291) * append appropriate urls when scaffolding * make dispatch for rest-api * fix dispatch for rest-json * fix moto/core/response to obtain path and body parameters * small fixes * remove unused import * fix get_int_param * fix scaffold * fix formatting of scaffold * fix misc * escape service to handle service w/ hyphen like iot-data * escape service w/ hyphen * fix regexp to extract region from url * escape service * fix syntax * skip loading body to json object when request body is None --- moto/backends.py | 3 +- moto/core/responses.py | 79 ++++++++++++- scripts/scaffold.py | 136 +++++++++++------------ scripts/template/lib/__init__.py.j2 | 6 +- scripts/template/lib/models.py.j2 | 2 +- scripts/template/lib/responses.py.j2 | 8 +- scripts/template/lib/urls.py.j2 | 4 + scripts/template/test/test_server.py.j2 | 6 +- scripts/template/test/test_service.py.j2 | 4 +- 9 files changed, 159 insertions(+), 89 deletions(-) diff --git a/moto/backends.py b/moto/backends.py index d1ce0730e..49c1f9f0f 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -37,6 +37,7 @@ from moto.sts import sts_backends from moto.xray import xray_backends from moto.batch import batch_backends + BACKENDS = { 'acm': acm_backends, 'apigateway': apigateway_backends, @@ -74,7 +75,7 @@ BACKENDS = { 'sts': sts_backends, 'route53': route53_backends, 'lambda': lambda_backends, - 'xray': xray_backends + 'xray': xray_backends, } diff --git a/moto/core/responses.py b/moto/core/responses.py index 572a45229..b4d94c0ac 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -17,6 +17,8 @@ from six.moves.urllib.parse import parse_qs, urlparse import xmltodict from pkg_resources import resource_filename from werkzeug.exceptions import HTTPException + +import boto3 from moto.compat import OrderedDict from moto.core.utils import camelcase_to_underscores, method_names_from_class @@ -103,7 +105,8 @@ class _TemplateEnvironmentMixin(object): class BaseResponse(_TemplateEnvironmentMixin): default_region = 'us-east-1' - region_regex = r'\.(.+?)\.amazonaws\.com' + # to extract region, use [^.] + region_regex = r'\.([^.]+?)\.amazonaws\.com' aws_service_spec = None @classmethod @@ -151,12 +154,12 @@ class BaseResponse(_TemplateEnvironmentMixin): querystring.update(headers) querystring = _decode_dict(querystring) - self.uri = full_url self.path = urlparse(full_url).path self.querystring = querystring self.method = request.method self.region = self.get_region_from_url(request, full_url) + self.uri_match = None self.headers = request.headers if 'host' not in self.headers: @@ -178,6 +181,58 @@ class BaseResponse(_TemplateEnvironmentMixin): self.setup_class(request, full_url, headers) return self.call_action() + def uri_to_regexp(self, uri): + """converts uri w/ placeholder to regexp + '/cars/{carName}/drivers/{DriverName}' + -> '^/cars/.*/drivers/[^/]*$' + + '/cars/{carName}/drivers/{DriverName}/drive' + -> '^/cars/.*/drivers/.*/drive$' + + """ + def _convert(elem, is_last): + if not re.match('^{.*}$', elem): + return elem + name = elem.replace('{', '').replace('}', '') + if is_last: + return '(?P<%s>[^/]*)' % name + return '(?P<%s>.*)' % name + + elems = uri.split('/') + num_elems = len(elems) + regexp = '^{}$'.format('/'.join([_convert(elem, (i == num_elems - 1)) for i, elem in enumerate(elems)])) + return regexp + + def _get_action_from_method_and_request_uri(self, method, request_uri): + """basically used for `rest-json` APIs + You can refer to example from link below + https://github.com/boto/botocore/blob/develop/botocore/data/iot/2015-05-28/service-2.json + """ + + # service response class should have 'SERVICE_NAME' class member, + # if you want to get action from method and url + if not hasattr(self, 'SERVICE_NAME'): + return None + service = self.SERVICE_NAME + conn = boto3.client(service) + + # make cache if it does not exist yet + if not hasattr(self, 'method_urls'): + self.method_urls = defaultdict(lambda: defaultdict(str)) + op_names = conn._service_model.operation_names + for op_name in op_names: + op_model = conn._service_model.operation_model(op_name) + _method = op_model.http['method'] + uri_regexp = self.uri_to_regexp(op_model.http['requestUri']) + self.method_urls[_method][uri_regexp] = op_model.name + regexp_and_names = self.method_urls[method] + for regexp, name in regexp_and_names.items(): + match = re.match(regexp, request_uri) + self.uri_match = match + if match: + return name + return None + def _get_action(self): action = self.querystring.get('Action', [""])[0] if not action: # Some services use a header for the action @@ -186,7 +241,9 @@ class BaseResponse(_TemplateEnvironmentMixin): 'x-amz-target') or self.headers.get('X-Amz-Target') if match: action = match.split(".")[-1] - + # get action from method and uri + if not action: + return self._get_action_from_method_and_request_uri(self.method, self.path) return action def call_action(self): @@ -221,6 +278,22 @@ class BaseResponse(_TemplateEnvironmentMixin): val = self.querystring.get(param_name) if val is not None: return val[0] + + # try to get json body parameter + if self.body is not None: + try: + return json.loads(self.body)[param_name] + except ValueError: + pass + except KeyError: + pass + # try to get path parameter + if self.uri_match: + try: + return self.uri_match.group(param_name) + except IndexError: + # do nothing if param is not found + pass return if_none def _get_int_param(self, param_name, if_none=None): diff --git a/scripts/scaffold.py b/scripts/scaffold.py index b1c9f3a0f..6c83eeb50 100755 --- a/scripts/scaffold.py +++ b/scripts/scaffold.py @@ -81,12 +81,14 @@ def select_service_and_operation(): raise click.Abort() return service_name, operation_name +def get_escaped_service(service): + return service.replace('-', '') def get_lib_dir(service): - return os.path.join('moto', service) + return os.path.join('moto', get_escaped_service(service)) def get_test_dir(service): - return os.path.join('tests', 'test_{}'.format(service)) + return os.path.join('tests', 'test_{}'.format(get_escaped_service(service))) def render_template(tmpl_dir, tmpl_filename, context, service, alt_filename=None): @@ -117,7 +119,7 @@ def append_mock_to_init_py(service): filtered_lines = [_ for _ in lines if re.match('^from.*mock.*$', _)] last_import_line_index = lines.index(filtered_lines[-1]) - new_line = 'from .{} import mock_{} # flake8: noqa'.format(service, service) + new_line = 'from .{} import mock_{} # flake8: noqa'.format(get_escaped_service(service), get_escaped_service(service)) lines.insert(last_import_line_index + 1, new_line) body = '\n'.join(lines) + '\n' @@ -135,7 +137,7 @@ def append_mock_import_to_backends_py(service): filtered_lines = [_ for _ in lines if re.match('^from.*backends.*$', _)] last_import_line_index = lines.index(filtered_lines[-1]) - new_line = 'from moto.{} import {}_backends'.format(service, service) + new_line = 'from moto.{} import {}_backends'.format(get_escaped_service(service), get_escaped_service(service)) lines.insert(last_import_line_index + 1, new_line) body = '\n'.join(lines) + '\n' @@ -147,13 +149,12 @@ def append_mock_dict_to_backends_py(service): with open(path) as f: lines = [_.replace('\n', '') for _ in f.readlines()] - # 'xray': xray_backends if any(_ for _ in lines if re.match(".*'{}': {}_backends.*".format(service, service), _)): return filtered_lines = [_ for _ in lines if re.match(".*'.*':.*_backends.*", _)] last_elem_line_index = lines.index(filtered_lines[-1]) - new_line = " '{}': {}_backends,".format(service, service) + new_line = " '{}': {}_backends,".format(service, get_escaped_service(service)) prev_line = lines[last_elem_line_index] if not prev_line.endswith('{') and not prev_line.endswith(','): lines[last_elem_line_index] += ',' @@ -166,8 +167,8 @@ def append_mock_dict_to_backends_py(service): def initialize_service(service, operation, api_protocol): """create lib and test dirs if not exist """ - lib_dir = os.path.join('moto', service) - test_dir = os.path.join('tests', 'test_{}'.format(service)) + lib_dir = get_lib_dir(service) + test_dir = get_test_dir(service) print_progress('Initializing service', service, 'green') @@ -178,7 +179,9 @@ def initialize_service(service, operation, api_protocol): tmpl_context = { 'service': service, 'service_class': service_class, - 'endpoint_prefix': endpoint_prefix + 'endpoint_prefix': endpoint_prefix, + 'api_protocol': api_protocol, + 'escaped_service': get_escaped_service(service) } # initialize service directory @@ -202,7 +205,7 @@ def initialize_service(service, operation, api_protocol): os.makedirs(test_dir) tmpl_dir = os.path.join(TEMPLATE_DIR, 'test') for tmpl_filename in os.listdir(tmpl_dir): - alt_filename = 'test_{}.py'.format(service) if tmpl_filename == 'test_service.py.j2' else None + alt_filename = 'test_{}.py'.format(get_escaped_service(service)) if tmpl_filename == 'test_service.py.j2' else None render_template( tmpl_dir, tmpl_filename, tmpl_context, service, alt_filename ) @@ -212,9 +215,16 @@ def initialize_service(service, operation, api_protocol): append_mock_import_to_backends_py(service) append_mock_dict_to_backends_py(service) + def to_upper_camel_case(s): return ''.join([_.title() for _ in s.split('_')]) + +def to_lower_camel_case(s): + words = s.split('_') + return ''.join(words[:1] + [_.title() for _ in words[1:]]) + + def to_snake_case(s): s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', s) return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() @@ -229,25 +239,28 @@ def get_function_in_responses(service, operation, protocol): aws_operation_name = to_upper_camel_case(operation) op_model = client._service_model.operation_model(aws_operation_name) - outputs = op_model.output_shape.members + if not hasattr(op_model.output_shape, 'members'): + outputs = {} + else: + outputs = op_model.output_shape.members inputs = op_model.input_shape.members input_names = [to_snake_case(_) for _ in inputs.keys() if _ not in INPUT_IGNORED_IN_BACKEND] output_names = [to_snake_case(_) for _ in outputs.keys() if _ not in OUTPUT_IGNORED_IN_BACKEND] - body = 'def {}(self):\n'.format(operation) + body = '\ndef {}(self):\n'.format(operation) for input_name, input_type in inputs.items(): type_name = input_type.type_name if type_name == 'integer': - arg_line_tmpl = ' {} = _get_int_param("{}")\n' + arg_line_tmpl = ' {} = self._get_int_param("{}")\n' elif type_name == 'list': arg_line_tmpl = ' {} = self._get_list_prefix("{}.member")\n' else: arg_line_tmpl = ' {} = self._get_param("{}")\n' body += arg_line_tmpl.format(to_snake_case(input_name), input_name) if output_names: - body += ' {} = self.{}_backend.{}(\n'.format(','.join(output_names), service, operation) + body += ' {} = self.{}_backend.{}(\n'.format(', '.join(output_names), get_escaped_service(service), operation) else: - body += ' self.{}_backend.{}(\n'.format(service, operation) + body += ' self.{}_backend.{}(\n'.format(get_escaped_service(service), operation) for input_name in input_names: body += ' {}={},\n'.format(input_name, input_name) @@ -255,11 +268,11 @@ def get_function_in_responses(service, operation, protocol): if protocol == 'query': body += ' template = self.response_template({}_TEMPLATE)\n'.format(operation.upper()) body += ' return template.render({})\n'.format( - ','.join(['{}={}'.format(_, _) for _ in output_names]) + ', '.join(['{}={}'.format(_, _) for _ in output_names]) ) - elif protocol == 'json': - body += ' # TODO: adjust reponse\n' - body += ' return json.dumps({})\n'.format(','.join(['{}={}'.format(_, _) for _ in output_names])) + elif protocol in ['json', 'rest-json']: + body += ' # TODO: adjust response\n' + body += ' return json.dumps(dict({}))\n'.format(', '.join(['{}={}'.format(to_lower_camel_case(_), _) for _ in output_names])) return body @@ -272,7 +285,10 @@ def get_function_in_models(service, operation): aws_operation_name = to_upper_camel_case(operation) op_model = client._service_model.operation_model(aws_operation_name) inputs = op_model.input_shape.members - outputs = op_model.output_shape.members + if not hasattr(op_model.output_shape, 'members'): + outputs = {} + else: + outputs = op_model.output_shape.members input_names = [to_snake_case(_) for _ in inputs.keys() if _ not in INPUT_IGNORED_IN_BACKEND] output_names = [to_snake_case(_) for _ in outputs.keys() if _ not in OUTPUT_IGNORED_IN_BACKEND] if input_names: @@ -280,7 +296,7 @@ def get_function_in_models(service, operation): else: body = 'def {}(self)\n' body += ' # implement here\n' - body += ' return {}\n'.format(', '.join(output_names)) + body += ' return {}\n\n'.format(', '.join(output_names)) return body @@ -388,13 +404,13 @@ def insert_code_to_class(path, base_class, new_code): f.write(body) -def insert_url(service, operation): +def insert_url(service, operation, api_protocol): client = boto3.client(service) service_class = client.__class__.__name__ aws_operation_name = to_upper_camel_case(operation) uri = client._service_model.operation_model(aws_operation_name).http['requestUri'] - path = os.path.join(os.path.dirname(__file__), '..', 'moto', service, 'urls.py') + path = os.path.join(os.path.dirname(__file__), '..', 'moto', get_escaped_service(service), 'urls.py') with open(path) as f: lines = [_.replace('\n', '') for _ in f.readlines()] @@ -413,81 +429,55 @@ def insert_url(service, operation): if not prev_line.endswith('{') and not prev_line.endswith(','): lines[last_elem_line_index] += ',' - new_line = " '{0}%s$': %sResponse.dispatch," % ( - uri, service_class - ) + # generate url pattern + if api_protocol == 'rest-json': + new_line = " '{0}/.*$': response.dispatch," + else: + new_line = " '{0}%s$': %sResponse.dispatch," % ( + uri, service_class + ) + if new_line in lines: + return lines.insert(last_elem_line_index + 1, new_line) body = '\n'.join(lines) + '\n' with open(path, 'w') as f: f.write(body) - -def insert_query_codes(service, operation): - func_in_responses = get_function_in_responses(service, operation, 'query') +def insert_codes(service, operation, api_protocol): + func_in_responses = get_function_in_responses(service, operation, api_protocol) func_in_models = get_function_in_models(service, operation) - template = get_response_query_template(service, operation) - # edit responses.py - responses_path = 'moto/{}/responses.py'.format(service) + responses_path = 'moto/{}/responses.py'.format(get_escaped_service(service)) print_progress('inserting code', responses_path, 'green') insert_code_to_class(responses_path, BaseResponse, func_in_responses) # insert template - with open(responses_path) as f: - lines = [_[:-1] for _ in f.readlines()] - lines += template.splitlines() - with open(responses_path, 'w') as f: - f.write('\n'.join(lines)) + if api_protocol == 'query': + template = get_response_query_template(service, operation) + with open(responses_path) as f: + lines = [_[:-1] for _ in f.readlines()] + lines += template.splitlines() + with open(responses_path, 'w') as f: + f.write('\n'.join(lines)) # edit models.py - models_path = 'moto/{}/models.py'.format(service) + models_path = 'moto/{}/models.py'.format(get_escaped_service(service)) print_progress('inserting code', models_path, 'green') insert_code_to_class(models_path, BaseBackend, func_in_models) # edit urls.py - insert_url(service, operation) + insert_url(service, operation, api_protocol) -def insert_json_codes(service, operation): - func_in_responses = get_function_in_responses(service, operation, 'json') - func_in_models = get_function_in_models(service, operation) - - # edit responses.py - responses_path = 'moto/{}/responses.py'.format(service) - print_progress('inserting code', responses_path, 'green') - insert_code_to_class(responses_path, BaseResponse, func_in_responses) - - # edit models.py - models_path = 'moto/{}/models.py'.format(service) - print_progress('inserting code', models_path, 'green') - insert_code_to_class(models_path, BaseBackend, func_in_models) - - # edit urls.py - insert_url(service, operation) - -def insert_restjson_codes(service, operation): - func_in_models = get_function_in_models(service, operation) - - print_progress('skipping inserting code to responses.py', "dont't know how to implement", 'yellow') - # edit models.py - models_path = 'moto/{}/models.py'.format(service) - print_progress('inserting code', models_path, 'green') - insert_code_to_class(models_path, BaseBackend, func_in_models) - - # edit urls.py - insert_url(service, operation) @click.command() def main(): service, operation = select_service_and_operation() api_protocol = boto3.client(service)._service_model.metadata['protocol'] initialize_service(service, operation, api_protocol) - if api_protocol == 'query': - insert_query_codes(service, operation) - elif api_protocol == 'json': - insert_json_codes(service, operation) - elif api_protocol == 'rest-json': - insert_restjson_codes(service, operation) + + if api_protocol in ['query', 'json', 'rest-json']: + insert_codes(service, operation, api_protocol) else: print_progress('skip inserting code', 'api protocol "{}" is not supported'.format(api_protocol), 'yellow') diff --git a/scripts/template/lib/__init__.py.j2 b/scripts/template/lib/__init__.py.j2 index 8e5bf50c7..5aade5706 100644 --- a/scripts/template/lib/__init__.py.j2 +++ b/scripts/template/lib/__init__.py.j2 @@ -1,7 +1,7 @@ from __future__ import unicode_literals -from .models import {{ service }}_backends +from .models import {{ escaped_service }}_backends from ..core.models import base_decorator -{{ service }}_backend = {{ service }}_backends['us-east-1'] -mock_{{ service }} = base_decorator({{ service }}_backends) +{{ escaped_service }}_backend = {{ escaped_service }}_backends['us-east-1'] +mock_{{ escaped_service }} = base_decorator({{ escaped_service }}_backends) diff --git a/scripts/template/lib/models.py.j2 b/scripts/template/lib/models.py.j2 index 623321884..28fa4a4e1 100644 --- a/scripts/template/lib/models.py.j2 +++ b/scripts/template/lib/models.py.j2 @@ -17,4 +17,4 @@ class {{ service_class }}Backend(BaseBackend): available_regions = boto3.session.Session().get_available_regions("{{ service }}") -{{ service }}_backends = {region: {{ service_class }}Backend(region) for region in available_regions} +{{ escaped_service }}_backends = {region: {{ service_class }}Backend(region) for region in available_regions} diff --git a/scripts/template/lib/responses.py.j2 b/scripts/template/lib/responses.py.j2 index 85827e651..60d0048e3 100644 --- a/scripts/template/lib/responses.py.j2 +++ b/scripts/template/lib/responses.py.j2 @@ -1,12 +1,14 @@ from __future__ import unicode_literals from moto.core.responses import BaseResponse -from .models import {{ service }}_backends +from .models import {{ escaped_service }}_backends +import json class {{ service_class }}Response(BaseResponse): + SERVICE_NAME = '{{ service }}' @property - def {{ service }}_backend(self): - return {{ service }}_backends[self.region] + def {{ escaped_service }}_backend(self): + return {{ escaped_service }}_backends[self.region] # add methods from here diff --git a/scripts/template/lib/urls.py.j2 b/scripts/template/lib/urls.py.j2 index 53cc03c0e..47ae52f2d 100644 --- a/scripts/template/lib/urls.py.j2 +++ b/scripts/template/lib/urls.py.j2 @@ -5,5 +5,9 @@ url_bases = [ "https?://{{ endpoint_prefix }}.(.+).amazonaws.com", ] +{% if api_protocol == 'rest-json' %} +response = {{ service_class }}Response() +{% endif %} + url_paths = { } diff --git a/scripts/template/test/test_server.py.j2 b/scripts/template/test/test_server.py.j2 index f3963a743..c85dbf01c 100644 --- a/scripts/template/test/test_server.py.j2 +++ b/scripts/template/test/test_server.py.j2 @@ -3,14 +3,14 @@ from __future__ import unicode_literals import sure # noqa import moto.server as server -from moto import mock_{{ service }} +from moto import mock_{{ escaped_service }} ''' Test the different server responses ''' -@mock_{{ service }} -def test_{{ service }}_list(): +@mock_{{ escaped_service }} +def test_{{ escaped_service }}_list(): backend = server.create_backend_app("{{ service }}") test_client = backend.test_client() # do test diff --git a/scripts/template/test/test_service.py.j2 b/scripts/template/test/test_service.py.j2 index 076f92e27..799f6079f 100644 --- a/scripts/template/test/test_service.py.j2 +++ b/scripts/template/test/test_service.py.j2 @@ -2,10 +2,10 @@ from __future__ import unicode_literals import boto3 import sure # noqa -from moto import mock_{{ service }} +from moto import mock_{{ escaped_service }} -@mock_{{ service }} +@mock_{{ escaped_service }} def test_list(): # do test pass