Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix scaffold to support rest-json style API (#1291)

Browse Source 
* append appropriate urls when scaffolding

* make dispatch for rest-api

* fix dispatch for rest-json

* fix moto/core/response to obtain path and body parameters

* small fixes

* remove unused import

* fix get_int_param

* fix scaffold

* fix formatting of scaffold

* fix misc

* escape service to handle service w/ hyphen like iot-data

* escape service w/ hyphen

* fix regexp to extract region from url

* escape service

* fix syntax

* skip loading body to json object when request body is None
This commit is contained in:
Toshiya Kawasaki 2017-10-25 03:45:39 +09:00 committed by Jack Danger
parent 20ef9db885
commit 56793a3b2a
9 changed files with 159 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
moto/backends.py
View File

@ -37,6 +37,7 @@ from moto.sts import sts_backends
from moto.xray import xray_backends
from moto.batch import batch_backends
BACKENDS = {
'acm': acm_backends,
'apigateway': apigateway_backends,
@ -74,7 +75,7 @@ BACKENDS = {
'sts': sts_backends,
'route53': route53_backends,
'lambda': lambda_backends,
'xray': xray_backends
'xray': xray_backends,
}

79
moto/core/responses.py
View File

@ -17,6 +17,8 @@ from six.moves.urllib.parse import parse_qs, urlparse
import xmltodict
from pkg_resources import resource_filename
from werkzeug.exceptions import HTTPException
import boto3
from moto.compat import OrderedDict
from moto.core.utils import camelcase_to_underscores, method_names_from_class
@ -103,7 +105,8 @@ class _TemplateEnvironmentMixin(object):
class BaseResponse(_TemplateEnvironmentMixin):
default_region = 'us-east-1'
region_regex = r'\.(.+?)\.amazonaws\.com'
# to extract region, use [^.]
region_regex = r'\.([^.]+?)\.amazonaws\.com'
aws_service_spec = None
@classmethod
@ -151,12 +154,12 @@ class BaseResponse(_TemplateEnvironmentMixin):
querystring.update(headers)
querystring = _decode_dict(querystring)
self.uri = full_url
self.path = urlparse(full_url).path
self.querystring = querystring
self.method = request.method
self.region = self.get_region_from_url(request, full_url)
self.uri_match = None
self.headers = request.headers
if 'host' not in self.headers:
@ -178,6 +181,58 @@ class BaseResponse(_TemplateEnvironmentMixin):
self.setup_class(request, full_url, headers)
return self.call_action()
def uri_to_regexp(self, uri):
"""converts uri w/ placeholder to regexp
'/cars/{carName}/drivers/{DriverName}'
-> '^/cars/.*/drivers/[^/]*$'
'/cars/{carName}/drivers/{DriverName}/drive'
-> '^/cars/.*/drivers/.*/drive$'
"""
def _convert(elem, is_last):
if not re.match('^{.*}$', elem):
return elem
name = elem.replace('{', '').replace('}', '')
if is_last:
return '(?P<%s>[^/]*)' % name
return '(?P<%s>.*)' % name
elems = uri.split('/')
num_elems = len(elems)
regexp = '^{}$'.format('/'.join([_convert(elem, (i == num_elems - 1)) for i, elem in enumerate(elems)]))
return regexp
def _get_action_from_method_and_request_uri(self, method, request_uri):
"""basically used for `rest-json` APIs
You can refer to example from link below
https://github.com/boto/botocore/blob/develop/botocore/data/iot/2015-05-28/service-2.json
"""
# service response class should have 'SERVICE_NAME' class member,
# if you want to get action from method and url
if not hasattr(self, 'SERVICE_NAME'):
return None
service = self.SERVICE_NAME
conn = boto3.client(service)
# make cache if it does not exist yet
if not hasattr(self, 'method_urls'):
self.method_urls = defaultdict(lambda: defaultdict(str))
op_names = conn._service_model.operation_names
for op_name in op_names:
op_model = conn._service_model.operation_model(op_name)
_method = op_model.http['method']
uri_regexp = self.uri_to_regexp(op_model.http['requestUri'])
self.method_urls[_method][uri_regexp] = op_model.name
regexp_and_names = self.method_urls[method]
for regexp, name in regexp_and_names.items():
match = re.match(regexp, request_uri)
self.uri_match = match
if match:
return name
return None
def _get_action(self):
action = self.querystring.get('Action', [""])[0]
if not action: # Some services use a header for the action
@ -186,7 +241,9 @@ class BaseResponse(_TemplateEnvironmentMixin):
'x-amz-target') or self.headers.get('X-Amz-Target')
if match:
action = match.split(".")[-1]
# get action from method and uri
if not action:
return self._get_action_from_method_and_request_uri(self.method, self.path)
return action
def call_action(self):
@ -221,6 +278,22 @@ class BaseResponse(_TemplateEnvironmentMixin):
val = self.querystring.get(param_name)
if val is not None:
return val[0]
# try to get json body parameter
if self.body is not None:
try:
return json.loads(self.body)[param_name]
except ValueError:
pass
except KeyError:
pass
# try to get path parameter
if self.uri_match:
try:
return self.uri_match.group(param_name)
except IndexError:
# do nothing if param is not found
pass
return if_none
def _get_int_param(self, param_name, if_none=None):

136
scripts/scaffold.py
View File

@ -81,12 +81,14 @@ def select_service_and_operation():
raise click.Abort()
return service_name, operation_name
def get_escaped_service(service):
return service.replace('-', '')
def get_lib_dir(service):
return os.path.join('moto', service)
return os.path.join('moto', get_escaped_service(service))
def get_test_dir(service):
return os.path.join('tests', 'test_{}'.format(service))
return os.path.join('tests', 'test_{}'.format(get_escaped_service(service)))
def render_template(tmpl_dir, tmpl_filename, context, service, alt_filename=None):
@ -117,7 +119,7 @@ def append_mock_to_init_py(service):
filtered_lines = [_ for _ in lines if re.match('^from.*mock.*$', _)]
last_import_line_index = lines.index(filtered_lines[-1])
new_line = 'from .{} import mock_{} # flake8: noqa'.format(service, service)
new_line = 'from .{} import mock_{} # flake8: noqa'.format(get_escaped_service(service), get_escaped_service(service))
lines.insert(last_import_line_index + 1, new_line)
body = '\n'.join(lines) + '\n'
@ -135,7 +137,7 @@ def append_mock_import_to_backends_py(service):
filtered_lines = [_ for _ in lines if re.match('^from.*backends.*$', _)]
last_import_line_index = lines.index(filtered_lines[-1])
new_line = 'from moto.{} import {}_backends'.format(service, service)
new_line = 'from moto.{} import {}_backends'.format(get_escaped_service(service), get_escaped_service(service))
lines.insert(last_import_line_index + 1, new_line)
body = '\n'.join(lines) + '\n'
@ -147,13 +149,12 @@ def append_mock_dict_to_backends_py(service):
with open(path) as f:
lines = [_.replace('\n', '') for _ in f.readlines()]
# 'xray': xray_backends
if any(_ for _ in lines if re.match(".*'{}': {}_backends.*".format(service, service), _)):
return
filtered_lines = [_ for _ in lines if re.match(".*'.*':.*_backends.*", _)]
last_elem_line_index = lines.index(filtered_lines[-1])
new_line = " '{}': {}_backends,".format(service, service)
new_line = " '{}': {}_backends,".format(service, get_escaped_service(service))
prev_line = lines[last_elem_line_index]
if not prev_line.endswith('{') and not prev_line.endswith(','):
lines[last_elem_line_index] += ','
@ -166,8 +167,8 @@ def append_mock_dict_to_backends_py(service):
def initialize_service(service, operation, api_protocol):
"""create lib and test dirs if not exist
"""
lib_dir = os.path.join('moto', service)
test_dir = os.path.join('tests', 'test_{}'.format(service))
lib_dir = get_lib_dir(service)
test_dir = get_test_dir(service)
print_progress('Initializing service', service, 'green')
@ -178,7 +179,9 @@ def initialize_service(service, operation, api_protocol):
tmpl_context = {
'service': service,
'service_class': service_class,
'endpoint_prefix': endpoint_prefix
'endpoint_prefix': endpoint_prefix,
'api_protocol': api_protocol,
'escaped_service': get_escaped_service(service)
}
# initialize service directory
@ -202,7 +205,7 @@ def initialize_service(service, operation, api_protocol):
os.makedirs(test_dir)
tmpl_dir = os.path.join(TEMPLATE_DIR, 'test')
for tmpl_filename in os.listdir(tmpl_dir):
alt_filename = 'test_{}.py'.format(service) if tmpl_filename == 'test_service.py.j2' else None
alt_filename = 'test_{}.py'.format(get_escaped_service(service)) if tmpl_filename == 'test_service.py.j2' else None
render_template(
tmpl_dir, tmpl_filename, tmpl_context, service, alt_filename
)
@ -212,9 +215,16 @@ def initialize_service(service, operation, api_protocol):
append_mock_import_to_backends_py(service)
append_mock_dict_to_backends_py(service)
def to_upper_camel_case(s):
return ''.join([_.title() for _ in s.split('_')])
def to_lower_camel_case(s):
words = s.split('_')
return ''.join(words[:1] + [_.title() for _ in words[1:]])
def to_snake_case(s):
s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', s)
return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
@ -229,25 +239,28 @@ def get_function_in_responses(service, operation, protocol):
aws_operation_name = to_upper_camel_case(operation)
op_model = client._service_model.operation_model(aws_operation_name)
outputs = op_model.output_shape.members
if not hasattr(op_model.output_shape, 'members'):
outputs = {}
else:
outputs = op_model.output_shape.members
inputs = op_model.input_shape.members
input_names = [to_snake_case(_) for _ in inputs.keys() if _ not in INPUT_IGNORED_IN_BACKEND]
output_names = [to_snake_case(_) for _ in outputs.keys() if _ not in OUTPUT_IGNORED_IN_BACKEND]
body = 'def {}(self):\n'.format(operation)
body = '\ndef {}(self):\n'.format(operation)
for input_name, input_type in inputs.items():
type_name = input_type.type_name
if type_name == 'integer':
arg_line_tmpl = ' {} = _get_int_param("{}")\n'
arg_line_tmpl = ' {} = self._get_int_param("{}")\n'
elif type_name == 'list':
arg_line_tmpl = ' {} = self._get_list_prefix("{}.member")\n'
else:
arg_line_tmpl = ' {} = self._get_param("{}")\n'
body += arg_line_tmpl.format(to_snake_case(input_name), input_name)
if output_names:
body += ' {} = self.{}_backend.{}(\n'.format(','.join(output_names), service, operation)
body += ' {} = self.{}_backend.{}(\n'.format(', '.join(output_names), get_escaped_service(service), operation)
else:
body += ' self.{}_backend.{}(\n'.format(service, operation)
body += ' self.{}_backend.{}(\n'.format(get_escaped_service(service), operation)
for input_name in input_names:
body += ' {}={},\n'.format(input_name, input_name)
@ -255,11 +268,11 @@ def get_function_in_responses(service, operation, protocol):
if protocol == 'query':
body += ' template = self.response_template({}_TEMPLATE)\n'.format(operation.upper())
body += ' return template.render({})\n'.format(
','.join(['{}={}'.format(_, _) for _ in output_names])
', '.join(['{}={}'.format(_, _) for _ in output_names])
)
elif protocol == 'json':
body += ' # TODO: adjust reponse\n'
body += ' return json.dumps({})\n'.format(','.join(['{}={}'.format(_, _) for _ in output_names]))
elif protocol in ['json', 'rest-json']:
body += ' # TODO: adjust response\n'
body += ' return json.dumps(dict({}))\n'.format(', '.join(['{}={}'.format(to_lower_camel_case(_), _) for _ in output_names]))
return body
@ -272,7 +285,10 @@ def get_function_in_models(service, operation):
aws_operation_name = to_upper_camel_case(operation)
op_model = client._service_model.operation_model(aws_operation_name)
inputs = op_model.input_shape.members
outputs = op_model.output_shape.members
if not hasattr(op_model.output_shape, 'members'):
outputs = {}
else:
outputs = op_model.output_shape.members
input_names = [to_snake_case(_) for _ in inputs.keys() if _ not in INPUT_IGNORED_IN_BACKEND]
output_names = [to_snake_case(_) for _ in outputs.keys() if _ not in OUTPUT_IGNORED_IN_BACKEND]
if input_names:
@ -280,7 +296,7 @@ def get_function_in_models(service, operation):
else:
body = 'def {}(self)\n'
body += ' # implement here\n'
body += ' return {}\n'.format(', '.join(output_names))
body += ' return {}\n\n'.format(', '.join(output_names))
return body
@ -388,13 +404,13 @@ def insert_code_to_class(path, base_class, new_code):
f.write(body)
def insert_url(service, operation):
def insert_url(service, operation, api_protocol):
client = boto3.client(service)
service_class = client.__class__.__name__
aws_operation_name = to_upper_camel_case(operation)
uri = client._service_model.operation_model(aws_operation_name).http['requestUri']
path = os.path.join(os.path.dirname(__file__), '..', 'moto', service, 'urls.py')
path = os.path.join(os.path.dirname(__file__), '..', 'moto', get_escaped_service(service), 'urls.py')
with open(path) as f:
lines = [_.replace('\n', '') for _ in f.readlines()]
@ -413,81 +429,55 @@ def insert_url(service, operation):
if not prev_line.endswith('{') and not prev_line.endswith(','):
lines[last_elem_line_index] += ','
new_line = " '{0}%s$': %sResponse.dispatch," % (
uri, service_class
)
# generate url pattern
if api_protocol == 'rest-json':
new_line = " '{0}/.*$': response.dispatch,"
else:
new_line = " '{0}%s$': %sResponse.dispatch," % (
uri, service_class
)
if new_line in lines:
return
lines.insert(last_elem_line_index + 1, new_line)
body = '\n'.join(lines) + '\n'
with open(path, 'w') as f:
f.write(body)
def insert_query_codes(service, operation):
func_in_responses = get_function_in_responses(service, operation, 'query')
def insert_codes(service, operation, api_protocol):
func_in_responses = get_function_in_responses(service, operation, api_protocol)
func_in_models = get_function_in_models(service, operation)
template = get_response_query_template(service, operation)
# edit responses.py
responses_path = 'moto/{}/responses.py'.format(service)
responses_path = 'moto/{}/responses.py'.format(get_escaped_service(service))
print_progress('inserting code', responses_path, 'green')
insert_code_to_class(responses_path, BaseResponse, func_in_responses)
# insert template
with open(responses_path) as f:
lines = [_[:-1] for _ in f.readlines()]
lines += template.splitlines()
with open(responses_path, 'w') as f:
f.write('\n'.join(lines))
if api_protocol == 'query':
template = get_response_query_template(service, operation)
with open(responses_path) as f:
lines = [_[:-1] for _ in f.readlines()]
lines += template.splitlines()
with open(responses_path, 'w') as f:
f.write('\n'.join(lines))
# edit models.py
models_path = 'moto/{}/models.py'.format(service)
models_path = 'moto/{}/models.py'.format(get_escaped_service(service))
print_progress('inserting code', models_path, 'green')
insert_code_to_class(models_path, BaseBackend, func_in_models)
# edit urls.py
insert_url(service, operation)
insert_url(service, operation, api_protocol)
def insert_json_codes(service, operation):
func_in_responses = get_function_in_responses(service, operation, 'json')
func_in_models = get_function_in_models(service, operation)
# edit responses.py
responses_path = 'moto/{}/responses.py'.format(service)
print_progress('inserting code', responses_path, 'green')
insert_code_to_class(responses_path, BaseResponse, func_in_responses)
# edit models.py
models_path = 'moto/{}/models.py'.format(service)
print_progress('inserting code', models_path, 'green')
insert_code_to_class(models_path, BaseBackend, func_in_models)
# edit urls.py
insert_url(service, operation)
def insert_restjson_codes(service, operation):
func_in_models = get_function_in_models(service, operation)
print_progress('skipping inserting code to responses.py', "dont't know how to implement", 'yellow')
# edit models.py
models_path = 'moto/{}/models.py'.format(service)
print_progress('inserting code', models_path, 'green')
insert_code_to_class(models_path, BaseBackend, func_in_models)
# edit urls.py
insert_url(service, operation)
@click.command()
def main():
service, operation = select_service_and_operation()
api_protocol = boto3.client(service)._service_model.metadata['protocol']
initialize_service(service, operation, api_protocol)
if api_protocol == 'query':
insert_query_codes(service, operation)
elif api_protocol == 'json':
insert_json_codes(service, operation)
elif api_protocol == 'rest-json':
insert_restjson_codes(service, operation)
if api_protocol in ['query', 'json', 'rest-json']:
insert_codes(service, operation, api_protocol)
else:
print_progress('skip inserting code', 'api protocol "{}" is not supported'.format(api_protocol), 'yellow')

6
scripts/template/lib/__init__.py.j2
View File

@ -1,7 +1,7 @@
from __future__ import unicode_literals
from .models import {{ service }}_backends
from .models import {{ escaped_service }}_backends
from ..core.models import base_decorator
{{ service }}_backend = {{ service }}_backends['us-east-1']
mock_{{ service }} = base_decorator({{ service }}_backends)
{{ escaped_service }}_backend = {{ escaped_service }}_backends['us-east-1']
mock_{{ escaped_service }} = base_decorator({{ escaped_service }}_backends)

2
scripts/template/lib/models.py.j2
View File

@ -17,4 +17,4 @@ class {{ service_class }}Backend(BaseBackend):
available_regions = boto3.session.Session().get_available_regions("{{ service }}")
{{ service }}_backends = {region: {{ service_class }}Backend(region) for region in available_regions}
{{ escaped_service }}_backends = {region: {{ service_class }}Backend(region) for region in available_regions}

8
scripts/template/lib/responses.py.j2
View File

@ -1,12 +1,14 @@
from __future__ import unicode_literals
from moto.core.responses import BaseResponse
from .models import {{ service }}_backends
from .models import {{ escaped_service }}_backends
import json
class {{ service_class }}Response(BaseResponse):
SERVICE_NAME = '{{ service }}'
@property
def {{ service }}_backend(self):
return {{ service }}_backends[self.region]
def {{ escaped_service }}_backend(self):
return {{ escaped_service }}_backends[self.region]
# add methods from here

4
scripts/template/lib/urls.py.j2
View File

@ -5,5 +5,9 @@ url_bases = [
"https?://{{ endpoint_prefix }}.(.+).amazonaws.com",
]
{% if api_protocol == 'rest-json' %}
response = {{ service_class }}Response()
{% endif %}
url_paths = {
}

6
scripts/template/test/test_server.py.j2
View File

@ -3,14 +3,14 @@ from __future__ import unicode_literals
import sure # noqa
import moto.server as server
from moto import mock_{{ service }}
from moto import mock_{{ escaped_service }}
'''
Test the different server responses
'''
@mock_{{ service }}
def test_{{ service }}_list():
@mock_{{ escaped_service }}
def test_{{ escaped_service }}_list():
backend = server.create_backend_app("{{ service }}")
test_client = backend.test_client()
# do test

4
scripts/template/test/test_service.py.j2
View File

@ -2,10 +2,10 @@ from __future__ import unicode_literals
import boto3
import sure # noqa
from moto import mock_{{ service }}
from moto import mock_{{ escaped_service }}
@mock_{{ service }}
@mock_{{ escaped_service }}
def test_list():
# do test
pass