From 59968760436876a66327bd1949f02711920d52e8 Mon Sep 17 00:00:00 2001 From: Toshiya Kawasaki Date: Wed, 20 Sep 2017 02:10:10 +0900 Subject: [PATCH 01/12] select service and operation in scaffold --- requirements-dev.txt | 2 ++ scaffold.py | 86 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100755 scaffold.py diff --git a/requirements-dev.txt b/requirements-dev.txt index 28aaec601..602e6fbbe 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -9,3 +9,5 @@ flask boto3>=1.4.4 botocore>=1.5.77 six>=1.9 +prompt-toolkit==1.0.14 +click==6.7 diff --git a/scaffold.py b/scaffold.py new file mode 100755 index 000000000..e3889e62d --- /dev/null +++ b/scaffold.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +import os + +import click +from prompt_toolkit import ( + prompt +) +from prompt_toolkit.contrib.completers import WordCompleter +from prompt_toolkit.shortcuts import print_tokens + +from botocore import xform_name +from botocore.session import Session +import boto3 + +from implementation_coverage import ( + get_moto_implementation +) + + +def select_service_and_operation(): + service_names = Session().get_available_services() + service_completer = WordCompleter(service_names) + service_name = prompt('Select service: ', completer=service_completer) + if service_name not in service_names: + click.secho('{} is not valid service'.format(service_name), fg='red') + raise click.Abort() + moto_client = get_moto_implementation(service_name) + real_client = boto3.client(service_name, region_name='us-east-1') + implemented = [] + not_implemented = [] + + operation_names = [xform_name(op) for op in real_client.meta.service_model.operation_names] + for op in operation_names: + if moto_client and op in dir(moto_client): + implemented.append(op) + else: + not_implemented.append(op) + operation_completer = WordCompleter(operation_names) + + click.echo('==Current Implementation Status==') + for operation_name in operation_names: + check = 'X' if operation_name in implemented else ' ' + click.secho('[{}] {}'.format(check, operation_name)) + click.echo('=================================') + operation_name = prompt('Select Operation: ', completer=operation_completer) + + if operation_name not in operation_names: + click.secho('{} is not valid operation'.format(operation_name), fg='red') + raise click.Abort() + + if operation_name in implemented: + click.secho('{} is already implemented'.format(operation_name), fg='red') + raise click.Abort() + return service_name, operation_name + + +def create_dirs(service, operation): + """create lib and test dirs if not exist + """ + lib_dir = os.path.join('moto', service) + test_dir = os.path.join('test', 'test_{}'.format(service)) + if os.path.exists(lib_dir): + return + + click.secho('\tInitializing service\t', fg='green', nl=False) + click.secho(service) + + click.secho('\tcraeting\t', fg='green', nl=False) + click.echo(lib_dir) + os.mkdirs(lib_dir) + # do init lib dir + + if not os.path.exists(test_dir): + click.secho('\tcraeting\t', fg='green', nl=False) + click.echo(test_dir) + os.mkdirs(test_dir) + # do init test dir + + +@click.command() +def main(): + service, operation = select_service_and_operation() + create_dirs(service, operation) + +if __name__ == '__main__': + main() From 9cdc0d50703ccd8ff83a2a1b0ebcc5c5fb1e5d2a Mon Sep 17 00:00:00 2001 From: Toshiya Kawasaki Date: Wed, 20 Sep 2017 03:14:14 +0900 Subject: [PATCH 02/12] Create service and test directories when they don't exist --- scaffold.py | 83 ++++++++++++++++++++++++++------ template/lib/__init__.py.j2 | 7 +++ template/lib/exceptions.py.j2 | 4 ++ template/lib/models.py.j2 | 20 ++++++++ template/lib/responses.py.j2 | 15 ++++++ template/test/test_server.py.j2 | 16 ++++++ template/test/test_service.py.j2 | 11 +++++ 7 files changed, 140 insertions(+), 16 deletions(-) create mode 100644 template/lib/__init__.py.j2 create mode 100644 template/lib/exceptions.py.j2 create mode 100644 template/lib/models.py.j2 create mode 100644 template/lib/responses.py.j2 create mode 100644 template/test/test_server.py.j2 create mode 100644 template/test/test_service.py.j2 diff --git a/scaffold.py b/scaffold.py index e3889e62d..77cc997c5 100755 --- a/scaffold.py +++ b/scaffold.py @@ -2,6 +2,7 @@ import os import click +import jinja2 from prompt_toolkit import ( prompt ) @@ -15,6 +16,12 @@ import boto3 from implementation_coverage import ( get_moto_implementation ) +TEMPLATE_DIR = './template' + + +def print_progress(title, body, color): + click.secho('\t{}\t'.format(title), fg=color, nl=False) + click.echo(body) def select_service_and_operation(): @@ -54,33 +61,77 @@ def select_service_and_operation(): return service_name, operation_name -def create_dirs(service, operation): +def get_lib_dir(service): + return os.path.join('moto', service) + +def get_test_dir(service): + return os.path.join('tests', 'test_{}'.format(service)) + + +def render_teamplte(tmpl_dir, tmpl_filename, context, service, alt_filename=None): + is_test = True if 'test' in tmpl_dir else False + rendered = jinja2.Environment( + loader=jinja2.FileSystemLoader(tmpl_dir) + ).get_template(tmpl_filename).render(context) + + dirname = get_test_dir(service) if is_test else get_lib_dir(service) + filename = alt_filename or os.path.splitext(tmpl_filename)[0] + filepath = os.path.join(dirname, filename) + + if os.path.exists(filepath): + print_progress('skip creating', filepath, 'yellow') + else: + print_progress('creating', filepath, 'green') + with open(filepath, 'w') as f: + f.write(rendered) + + +def initialize_service(service, operation): """create lib and test dirs if not exist """ lib_dir = os.path.join('moto', service) - test_dir = os.path.join('test', 'test_{}'.format(service)) + test_dir = os.path.join('tests', 'test_{}'.format(service)) + + print_progress('Initializing service', service, 'green') + + service_class = boto3.client(service).__class__.__name__ + + tmpl_context = { + 'service': service, + 'service_class': service_class + } + + # initialize service directory if os.path.exists(lib_dir): - return + print_progress('skip creating', lib_dir, 'yellow') + else: + print_progress('creating', lib_dir, 'green') + os.makedirs(lib_dir) - click.secho('\tInitializing service\t', fg='green', nl=False) - click.secho(service) + tmpl_dir = os.path.join(TEMPLATE_DIR, 'lib') + for tmpl_filename in os.listdir(tmpl_dir): + render_teamplte( + tmpl_dir, tmpl_filename, tmpl_context, service + ) - click.secho('\tcraeting\t', fg='green', nl=False) - click.echo(lib_dir) - os.mkdirs(lib_dir) - # do init lib dir - - if not os.path.exists(test_dir): - click.secho('\tcraeting\t', fg='green', nl=False) - click.echo(test_dir) - os.mkdirs(test_dir) - # do init test dir + # initialize test directory + if os.path.exists(test_dir): + print_progress('skip creating', test_dir, 'yellow') + else: + print_progress('creating', test_dir, 'green') + os.makedirs(test_dir) + tmpl_dir = os.path.join(TEMPLATE_DIR, 'test') + for tmpl_filename in os.listdir(tmpl_dir): + alt_filename = 'test_{}.py'.format(service) if tmpl_filename == 'test_service.py.j2' else None + render_teamplte( + tmpl_dir, tmpl_filename, tmpl_context, service, alt_filename + ) @click.command() def main(): service, operation = select_service_and_operation() - create_dirs(service, operation) + initialize_service(service, operation) if __name__ == '__main__': main() diff --git a/template/lib/__init__.py.j2 b/template/lib/__init__.py.j2 new file mode 100644 index 000000000..8e5bf50c7 --- /dev/null +++ b/template/lib/__init__.py.j2 @@ -0,0 +1,7 @@ +from __future__ import unicode_literals +from .models import {{ service }}_backends +from ..core.models import base_decorator + +{{ service }}_backend = {{ service }}_backends['us-east-1'] +mock_{{ service }} = base_decorator({{ service }}_backends) + diff --git a/template/lib/exceptions.py.j2 b/template/lib/exceptions.py.j2 new file mode 100644 index 000000000..2e9a72b1a --- /dev/null +++ b/template/lib/exceptions.py.j2 @@ -0,0 +1,4 @@ +from __future__ import unicode_literals +from moto.core.exceptions import RESTError + + diff --git a/template/lib/models.py.j2 b/template/lib/models.py.j2 new file mode 100644 index 000000000..2a0097c1d --- /dev/null +++ b/template/lib/models.py.j2 @@ -0,0 +1,20 @@ +from __future__ import unicode_literals +import boto3 +from moto.core import BaseBackend, BaseModel + + +class {{ service_class }}Backend(BaseBackend): + def __init__(self, region_name=None): + super({{ service_class }}Backend, self).__init__() + self.region_name = region_name + + def reset(self): + region_name = self.region_name + self.__dict__ = {} + self.__init__(region_name) + + # add methods from here + + +available_regions = boto3.session.Session().get_available_regions("{{ service }}") +{{ service }}_backends = {region: {{ service_class }}Backend for region in available_regions} diff --git a/template/lib/responses.py.j2 b/template/lib/responses.py.j2 new file mode 100644 index 000000000..b27da5b9f --- /dev/null +++ b/template/lib/responses.py.j2 @@ -0,0 +1,15 @@ +from __future__ import unicode_literals +from moto.core.responses import BaseResponse +from .models import {{ service }}_backends + + +class {{ service_class }}Response(BaseResponse): + @property + def {{ service }}_backend(self): + return {{ service }}_backends[self.region] + + # add methods from here + + +# add teampltes from here + diff --git a/template/test/test_server.py.j2 b/template/test/test_server.py.j2 new file mode 100644 index 000000000..f3963a743 --- /dev/null +++ b/template/test/test_server.py.j2 @@ -0,0 +1,16 @@ +from __future__ import unicode_literals + +import sure # noqa + +import moto.server as server +from moto import mock_{{ service }} + +''' +Test the different server responses +''' + +@mock_{{ service }} +def test_{{ service }}_list(): + backend = server.create_backend_app("{{ service }}") + test_client = backend.test_client() + # do test diff --git a/template/test/test_service.py.j2 b/template/test/test_service.py.j2 new file mode 100644 index 000000000..076f92e27 --- /dev/null +++ b/template/test/test_service.py.j2 @@ -0,0 +1,11 @@ +from __future__ import unicode_literals + +import boto3 +import sure # noqa +from moto import mock_{{ service }} + + +@mock_{{ service }} +def test_list(): + # do test + pass From 719e7866ab80d8ac54810a6cf1a1f5e1094e6b01 Mon Sep 17 00:00:00 2001 From: Toshiya Kawasaki Date: Wed, 20 Sep 2017 04:36:11 +0900 Subject: [PATCH 03/12] auto generate teamplte --- requirements-dev.txt | 1 + scaffold.py | 148 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 148 insertions(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 602e6fbbe..7dda4026b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,3 +11,4 @@ botocore>=1.5.77 six>=1.9 prompt-toolkit==1.0.14 click==6.7 +inflection==0.3.1 diff --git a/scaffold.py b/scaffold.py index 77cc997c5..79133b9d3 100755 --- a/scaffold.py +++ b/scaffold.py @@ -1,5 +1,7 @@ #!/usr/bin/env python import os +import re +from lxml import etree import click import jinja2 @@ -16,8 +18,13 @@ import boto3 from implementation_coverage import ( get_moto_implementation ) +from inflection import singularize + TEMPLATE_DIR = './template' +INPUT_IGNORED_IN_BACKEND = ['Marker', 'PageSize'] +OUTPUT_IGNORED_IN_BACKEND = ['NextMarker'] + def print_progress(title, body, color): click.secho('\t{}\t'.format(title), fg=color, nl=False) @@ -127,11 +134,150 @@ def initialize_service(service, operation): tmpl_dir, tmpl_filename, tmpl_context, service, alt_filename ) +def to_upper_camel_case(s): + return ''.join([_.title() for _ in s.split('_')]) + +def to_snake_case(s): + s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', s) + return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() + + +def get_function_in_responses(service, operation): + """refers to definition of API in botocore, and autogenerates function + You can see example of elbv2 from link below. + https://github.com/boto/botocore/blob/develop/botocore/data/elbv2/2015-12-01/service-2.json + """ + client = boto3.client(service) + + aws_operation_name = to_upper_camel_case(operation) + op_model = client._service_model.operation_model(aws_operation_name) + outputs = op_model.output_shape.members + inputs = op_model.input_shape.members + input_names = [to_snake_case(_) for _ in inputs.keys() if _ not in INPUT_IGNORED_IN_BACKEND] + output_names = [to_snake_case(_) for _ in outputs.keys() if _ not in OUTPUT_IGNORED_IN_BACKEND] + body = 'def {}(self):\n'.format(operation) + + for input_name, input_type in inputs.items(): + type_name = input_type.type_name + if type_name == 'integer': + arg_line_tmpl = ' {} = _get_int_param("{}")\n' + elif type_name == 'list': + arg_line_tmpl = ' {} = self._get_list_prefix("{}.member")\n' + else: + arg_line_tmpl = ' {} = self._get_param("{}")\n' + body += arg_line_tmpl.format(to_snake_case(input_name), input_name) + if output_names: + body += ' {} = self.{}_backend.{}(\n'.format(','.join(output_names), service, operation) + else: + body += ' self.{}_backend.{}(\n'.format(service, operation) + for input_name in input_names: + body += ' {}={},\n'.format(input_name, input_name) + + body += ' )\n' + body += ' template = self.response_template({}_TEMPLATE)\n'.format(operation.upper()) + body += ' return template.render({})\n'.format( + ','.join(['{}={}'.format(_, _) for _ in output_names]) + ) + return body + + +def get_function_in_models(service, operation): + """refers to definition of API in botocore, and autogenerates function + You can see example of elbv2 from link below. + https://github.com/boto/botocore/blob/develop/botocore/data/elbv2/2015-12-01/service-2.json + """ + client = boto3.client(service) + aws_operation_name = to_upper_camel_case(operation) + op_model = client._service_model.operation_model(aws_operation_name) + inputs = op_model.input_shape.members + outputs = op_model.output_shape.members + input_names = [to_snake_case(_) for _ in inputs.keys() if _ not in INPUT_IGNORED_IN_BACKEND] + output_names = [to_snake_case(_) for _ in outputs.keys() if _ not in OUTPUT_IGNORED_IN_BACKEND] + if input_names: + body = 'def {}(self, {}):\n'.format(operation, ', '.join(input_names)) + else: + body = 'def {}(self)\n' + body += ' # implement here\n' + body += ' return {}\n'.format(', '.join(output_names)) + + return body + + +def _get_subtree(name, shape, replace_list, name_prefix=[]): + class_name = shape.__class__.__name__ + if class_name in ('StringShape', 'Shape'): + t = etree.Element(name) + if name_prefix: + t.text = '{{ %s.%s }}' % (name_prefix[-1], to_snake_case(name)) + else: + t.text = '{{ %s }}' % to_snake_case(name) + return t + elif class_name in ('ListShape', ): + replace_list.append((name, name_prefix)) + t = etree.Element(name) + t_member = etree.Element('member') + t.append(t_member) + for nested_name, nested_shape in shape.member.members.items(): + t_member.append(_get_subtree(nested_name, nested_shape, replace_list, name_prefix + [singularize(name.lower())])) + return t + raise ValueError('Not supported Shape') + + +def get_response_template(service, operation): + """refers to definition of API in botocore, and autogenerates template + You can see example of elbv2 from link below. + https://github.com/boto/botocore/blob/develop/botocore/data/elbv2/2015-12-01/service-2.json + """ + client = boto3.client(service) + aws_operation_name = to_upper_camel_case(operation) + op_model = client._service_model.operation_model(aws_operation_name) + result_wrapper = op_model.output_shape.serialization['resultWrapper'] + response_wrapper = result_wrapper.replace('Result', 'Response') + metadata = op_model.metadata + xml_namespace = metadata['xmlNamespace'] + + # build xml tree + t_root = etree.Element(response_wrapper, xmlns=xml_namespace) + + # build metadata + t_metadata = etree.Element('ResponseMetadata') + t_request_id = etree.Element('RequestId') + t_request_id.text = '1549581b-12b7-11e3-895e-1334aEXAMPLE' + t_metadata.append(t_request_id) + t_root.append(t_metadata) + + # build result + t_result = etree.Element(result_wrapper) + outputs = op_model.output_shape.members + replace_list = [] + for output_name, output_shape in outputs.items(): + t_result.append(_get_subtree(output_name, output_shape, replace_list)) + t_root.append(t_result) + body = etree.tostring(t_root, pretty_print=True).decode('utf-8') + for replace in replace_list: + name = replace[0] + prefix = replace[1] + singular_name = singularize(name) + + start_tag = '<%s>' % name + iter_name = '{}.{}'.format(prefix[-1], name.lower())if prefix else name.lower() + start_tag_to_replace = '<%s>\n{%% for %s in %s %%}' % (name, singular_name.lower(), iter_name) + # TODO: format indents + end_tag = '' % name + end_tag_to_replace = '{{ endfor }}\n' % name + + body = body.replace(start_tag, start_tag_to_replace) + body = body.replace(end_tag, end_tag_to_replace) + print(body) @click.command() def main(): service, operation = select_service_and_operation() initialize_service(service, operation) + if __name__ == '__main__': - main() +# print(get_function_in_responses('elbv2', 'describe_listeners')) +# print(get_function_in_models('elbv2', 'describe_listeners')) + get_response_template('elbv2', 'describe_listeners') +# main() From e330d7876ee89a08fc297d6fd5d83db6e891815e Mon Sep 17 00:00:00 2001 From: Toshiya Kawasaki Date: Thu, 21 Sep 2017 21:23:13 +0900 Subject: [PATCH 04/12] fix indent --- scaffold.py | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/scaffold.py b/scaffold.py index 79133b9d3..2c168da69 100755 --- a/scaffold.py +++ b/scaffold.py @@ -93,7 +93,7 @@ def render_teamplte(tmpl_dir, tmpl_filename, context, service, alt_filename=None f.write(rendered) -def initialize_service(service, operation): +def initialize_service(service, operation, api_protocol): """create lib and test dirs if not exist """ lib_dir = os.path.join('moto', service) @@ -142,7 +142,7 @@ def to_snake_case(s): return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() -def get_function_in_responses(service, operation): +def get_function_in_query_responses(service, operation): """refers to definition of API in botocore, and autogenerates function You can see example of elbv2 from link below. https://github.com/boto/botocore/blob/develop/botocore/data/elbv2/2015-12-01/service-2.json @@ -223,8 +223,10 @@ def _get_subtree(name, shape, replace_list, name_prefix=[]): raise ValueError('Not supported Shape') -def get_response_template(service, operation): +def get_response_query_template(service, operation): """refers to definition of API in botocore, and autogenerates template + Assume that response format is xml when protocol is query + You can see example of elbv2 from link below. https://github.com/boto/botocore/blob/develop/botocore/data/elbv2/2015-12-01/service-2.json """ @@ -254,6 +256,7 @@ def get_response_template(service, operation): t_result.append(_get_subtree(output_name, output_shape, replace_list)) t_root.append(t_result) body = etree.tostring(t_root, pretty_print=True).decode('utf-8') + body_lines = body.splitlines() for replace in replace_list: name = replace[0] prefix = replace[1] @@ -261,23 +264,39 @@ def get_response_template(service, operation): start_tag = '<%s>' % name iter_name = '{}.{}'.format(prefix[-1], name.lower())if prefix else name.lower() - start_tag_to_replace = '<%s>\n{%% for %s in %s %%}' % (name, singular_name.lower(), iter_name) - # TODO: format indents + loop_start = '{%% for %s in %s %%}' % (singular_name.lower(), iter_name) end_tag = '' % name - end_tag_to_replace = '{{ endfor }}\n' % name + loop_end = '{{ endfor }}' - body = body.replace(start_tag, start_tag_to_replace) - body = body.replace(end_tag, end_tag_to_replace) - print(body) + start_tag_indexes = [i for i, l in enumerate(body_lines) if start_tag in l] + if len(start_tag_indexes) != 1: + raise Exception('tag %s not found in response body' % start_tag) + start_tag_index = start_tag_indexes[0] + body_lines.insert(start_tag_index + 1, loop_start) + + end_tag_indexes = [i for i, l in enumerate(body_lines) if end_tag in l] + if len(end_tag_indexes) != 1: + raise Exception('tag %s not found in response body' % end_tag) + end_tag_index = end_tag_indexes[0] + body_lines.insert(end_tag_index, loop_end) + body = '\n'.join(body_lines) + return body @click.command() def main(): service, operation = select_service_and_operation() - initialize_service(service, operation) + api_protocol = boto3.client(service_name)._service_model.metadata['protocol'] + initialize_service(service, operation, api_protocol) + if api_protocol == 'query': + func_in_responses = get_function_in_responses(service, operation) + func_in_models = get_function_in_models(service, operation) + teamplte = get_response_xml_template(service, operation) + if __name__ == '__main__': # print(get_function_in_responses('elbv2', 'describe_listeners')) # print(get_function_in_models('elbv2', 'describe_listeners')) - get_response_template('elbv2', 'describe_listeners') + b = get_response_query_template('elbv2', 'describe_listeners') + print(b) # main() From 6c33888b0fd3795d27151ee13e6a74927197a874 Mon Sep 17 00:00:00 2001 From: Toshiya Kawasaki Date: Thu, 21 Sep 2017 21:54:14 +0900 Subject: [PATCH 05/12] insert functions and templates for query and json protocol --- requirements-dev.txt | 1 + scaffold.py | 110 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 88 insertions(+), 23 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 7dda4026b..6d84d7a86 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -12,3 +12,4 @@ six>=1.9 prompt-toolkit==1.0.14 click==6.7 inflection==0.3.1 +lxml==4.0.0 diff --git a/scaffold.py b/scaffold.py index 2c168da69..d2f06b127 100755 --- a/scaffold.py +++ b/scaffold.py @@ -1,6 +1,8 @@ #!/usr/bin/env python import os import re +import inspect +import importlib from lxml import etree import click @@ -15,6 +17,8 @@ from botocore import xform_name from botocore.session import Session import boto3 +from moto.core.responses import BaseResponse +from moto.core import BaseBackend from implementation_coverage import ( get_moto_implementation ) @@ -142,7 +146,7 @@ def to_snake_case(s): return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() -def get_function_in_query_responses(service, operation): +def get_function_in_responses(service, operation, protocol): """refers to definition of API in botocore, and autogenerates function You can see example of elbv2 from link below. https://github.com/boto/botocore/blob/develop/botocore/data/elbv2/2015-12-01/service-2.json @@ -174,10 +178,14 @@ def get_function_in_query_responses(service, operation): body += ' {}={},\n'.format(input_name, input_name) body += ' )\n' - body += ' template = self.response_template({}_TEMPLATE)\n'.format(operation.upper()) - body += ' return template.render({})\n'.format( - ','.join(['{}={}'.format(_, _) for _ in output_names]) - ) + if protocol == 'query': + body += ' template = self.response_template({}_TEMPLATE)\n'.format(operation.upper()) + body += ' return template.render({})\n'.format( + ','.join(['{}={}'.format(_, _) for _ in output_names]) + ) + elif protocol == 'json': + body += ' # TODO: adjust reponse\n' + body += ' return json.dumps({})\n'.format(','.join(['{}={}'.format(_, _) for _ in output_names])) return body @@ -255,8 +263,8 @@ def get_response_query_template(service, operation): for output_name, output_shape in outputs.items(): t_result.append(_get_subtree(output_name, output_shape, replace_list)) t_root.append(t_result) - body = etree.tostring(t_root, pretty_print=True).decode('utf-8') - body_lines = body.splitlines() + xml_body = etree.tostring(t_root, pretty_print=True).decode('utf-8') + xml_body_lines = xml_body.splitlines() for replace in replace_list: name = replace[0] prefix = replace[1] @@ -268,35 +276,91 @@ def get_response_query_template(service, operation): end_tag = '' % name loop_end = '{{ endfor }}' - start_tag_indexes = [i for i, l in enumerate(body_lines) if start_tag in l] + start_tag_indexes = [i for i, l in enumerate(xml_body_lines) if start_tag in l] if len(start_tag_indexes) != 1: raise Exception('tag %s not found in response body' % start_tag) start_tag_index = start_tag_indexes[0] - body_lines.insert(start_tag_index + 1, loop_start) + xml_body_lines.insert(start_tag_index + 1, loop_start) - end_tag_indexes = [i for i, l in enumerate(body_lines) if end_tag in l] + end_tag_indexes = [i for i, l in enumerate(xml_body_lines) if end_tag in l] if len(end_tag_indexes) != 1: raise Exception('tag %s not found in response body' % end_tag) end_tag_index = end_tag_indexes[0] - body_lines.insert(end_tag_index, loop_end) - body = '\n'.join(body_lines) + xml_body_lines.insert(end_tag_index, loop_end) + xml_body = '\n'.join(xml_body_lines) + body = '\n{}_TEMPLATE = """{}"""'.format(operation.upper(), xml_body) return body + +def insert_code_to_class(path, base_class, new_code): + with open(path) as f: + lines = [_.replace('\n', '') for _ in f.readlines()] + mod_path = os.path.splitext(path)[0].replace('/', '.') + mod = importlib.import_module(mod_path) + clsmembers = inspect.getmembers(mod, inspect.isclass) + _response_cls = [_[1] for _ in clsmembers if issubclass(_[1], base_class) and _[1] != base_class] + if len(_response_cls) != 1: + raise Exception('unknown error, number of clsmembers is not 1') + response_cls = _response_cls[0] + code_lines, line_no = inspect.getsourcelines(response_cls) + end_line_no = line_no + len(code_lines) + + func_lines = [' ' * 4 + _ for _ in new_code.splitlines()] + + lines = lines[:end_line_no] + func_lines + lines[end_line_no:] + + with open(path, 'w') as f: + f.write('\n'.join(lines)) + + +def insert_query_codes(service, operation): + func_in_responses = get_function_in_responses(service, operation, 'query') + func_in_models = get_function_in_models(service, operation) + template = get_response_query_template(service, operation) + + # edit responses.py + responses_path = 'moto/{}/responses.py'.format(service) + print_progress('inserting code', responses_path, 'green') + insert_code_to_class(responses_path, BaseResponse, func_in_responses) + + # insert template + with open(responses_path) as f: + lines = [_[:-1] for _ in f.readlines()] + lines += template.splitlines() + with open(responses_path, 'w') as f: + f.write('\n'.join(lines)) + + # edit models.py + models_path = 'moto/{}/models.py'.format(service) + print_progress('inserting code', models_path, 'green') + insert_code_to_class(models_path, BaseBackend, func_in_models) + +def insert_json_codes(service, operation): + func_in_responses = get_function_in_responses(service, operation, 'json') + func_in_models = get_function_in_models(service, operation) + + # edit responses.py + responses_path = 'moto/{}/responses.py'.format(service) + print_progress('inserting code', responses_path, 'green') + insert_code_to_class(responses_path, BaseResponse, func_in_responses) + + # edit models.py + models_path = 'moto/{}/models.py'.format(service) + print_progress('inserting code', models_path, 'green') + insert_code_to_class(models_path, BaseBackend, func_in_models) + @click.command() def main(): service, operation = select_service_and_operation() - api_protocol = boto3.client(service_name)._service_model.metadata['protocol'] + api_protocol = boto3.client(service)._service_model.metadata['protocol'] initialize_service(service, operation, api_protocol) if api_protocol == 'query': - func_in_responses = get_function_in_responses(service, operation) - func_in_models = get_function_in_models(service, operation) - teamplte = get_response_xml_template(service, operation) - - + insert_query_codes(service, operation) + elif api_protocol == 'json': + insert_json_codes(service, operation) + pass + else: + print_progress('skip inserting code', 'api protocol "{}" is not supported'.format(api_protocol), 'yellow') if __name__ == '__main__': -# print(get_function_in_responses('elbv2', 'describe_listeners')) -# print(get_function_in_models('elbv2', 'describe_listeners')) - b = get_response_query_template('elbv2', 'describe_listeners') - print(b) -# main() + main() From d0154b8e71266096c9140007f6ba767861b0fd4a Mon Sep 17 00:00:00 2001 From: Toshiya Kawasaki Date: Fri, 22 Sep 2017 00:01:01 +0900 Subject: [PATCH 06/12] insert functions and templates for query and rest-json protocol --- scaffold.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scaffold.py b/scaffold.py index d2f06b127..172c21bbb 100755 --- a/scaffold.py +++ b/scaffold.py @@ -349,6 +349,15 @@ def insert_json_codes(service, operation): print_progress('inserting code', models_path, 'green') insert_code_to_class(models_path, BaseBackend, func_in_models) +def insert_restjson_codes(service, operation): + func_in_models = get_function_in_models(service, operation) + + print_progress('skipping inserting code to responses.py', "dont't know how to implement", 'yellow') + # edit models.py + models_path = 'moto/{}/models.py'.format(service) + print_progress('inserting code', models_path, 'green') + insert_code_to_class(models_path, BaseBackend, func_in_models) + @click.command() def main(): service, operation = select_service_and_operation() @@ -358,7 +367,8 @@ def main(): insert_query_codes(service, operation) elif api_protocol == 'json': insert_json_codes(service, operation) - pass + elif api_protocol == 'rest-json': + insert_restjson_codes(service, operation) else: print_progress('skip inserting code', 'api protocol "{}" is not supported'.format(api_protocol), 'yellow') From 16f0868d420b31e3e00656854b1f19b4d9d15e47 Mon Sep 17 00:00:00 2001 From: Toshiya Kawasaki Date: Fri, 22 Sep 2017 00:03:52 +0900 Subject: [PATCH 07/12] rename script name --- scaffold.py => setup_new_function.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scaffold.py => setup_new_function.py (100%) diff --git a/scaffold.py b/setup_new_function.py similarity index 100% rename from scaffold.py rename to setup_new_function.py From 84fc4734fc9ffc0283869fc7e79b85ade0374a6b Mon Sep 17 00:00:00 2001 From: Toshiya Kawasaki Date: Fri, 22 Sep 2017 14:03:12 +0900 Subject: [PATCH 08/12] fix typo --- setup_new_function.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup_new_function.py b/setup_new_function.py index 172c21bbb..3927ace30 100755 --- a/setup_new_function.py +++ b/setup_new_function.py @@ -79,7 +79,7 @@ def get_test_dir(service): return os.path.join('tests', 'test_{}'.format(service)) -def render_teamplte(tmpl_dir, tmpl_filename, context, service, alt_filename=None): +def render_template(tmpl_dir, tmpl_filename, context, service, alt_filename=None): is_test = True if 'test' in tmpl_dir else False rendered = jinja2.Environment( loader=jinja2.FileSystemLoader(tmpl_dir) @@ -121,7 +121,7 @@ def initialize_service(service, operation, api_protocol): tmpl_dir = os.path.join(TEMPLATE_DIR, 'lib') for tmpl_filename in os.listdir(tmpl_dir): - render_teamplte( + render_template( tmpl_dir, tmpl_filename, tmpl_context, service ) @@ -134,7 +134,7 @@ def initialize_service(service, operation, api_protocol): tmpl_dir = os.path.join(TEMPLATE_DIR, 'test') for tmpl_filename in os.listdir(tmpl_dir): alt_filename = 'test_{}.py'.format(service) if tmpl_filename == 'test_service.py.j2' else None - render_teamplte( + render_template( tmpl_dir, tmpl_filename, tmpl_context, service, alt_filename ) From f8cdb50f461125cfbce22b2b5525dd618d0d9233 Mon Sep 17 00:00:00 2001 From: Toshiya Kawasaki Date: Fri, 22 Sep 2017 19:11:13 +0900 Subject: [PATCH 09/12] support python2 by using "u" string --- setup_new_function.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup_new_function.py b/setup_new_function.py index 3927ace30..0102d74b2 100755 --- a/setup_new_function.py +++ b/setup_new_function.py @@ -31,16 +31,16 @@ OUTPUT_IGNORED_IN_BACKEND = ['NextMarker'] def print_progress(title, body, color): - click.secho('\t{}\t'.format(title), fg=color, nl=False) + click.secho(u'\t{}\t'.format(title), fg=color, nl=False) click.echo(body) def select_service_and_operation(): service_names = Session().get_available_services() service_completer = WordCompleter(service_names) - service_name = prompt('Select service: ', completer=service_completer) + service_name = prompt(u'Select service: ', completer=service_completer) if service_name not in service_names: - click.secho('{} is not valid service'.format(service_name), fg='red') + click.secho(u'{} is not valid service'.format(service_name), fg='red') raise click.Abort() moto_client = get_moto_implementation(service_name) real_client = boto3.client(service_name, region_name='us-east-1') @@ -60,7 +60,7 @@ def select_service_and_operation(): check = 'X' if operation_name in implemented else ' ' click.secho('[{}] {}'.format(check, operation_name)) click.echo('=================================') - operation_name = prompt('Select Operation: ', completer=operation_completer) + operation_name = prompt(u'Select Operation: ', completer=operation_completer) if operation_name not in operation_names: click.secho('{} is not valid operation'.format(operation_name), fg='red') From 4cc4b36f1560e84e20d9e810b6ca713c1f59f24c Mon Sep 17 00:00:00 2001 From: Toshiya Kawasaki Date: Fri, 22 Sep 2017 19:23:10 +0900 Subject: [PATCH 10/12] move helper script to scripts dir and add one to Makefile --- Makefile | 4 ++++ .../implementation_coverage.py | 0 setup_new_function.py => scripts/scaffold.py | 2 +- {template => scripts/template}/lib/__init__.py.j2 | 0 {template => scripts/template}/lib/exceptions.py.j2 | 0 {template => scripts/template}/lib/models.py.j2 | 0 {template => scripts/template}/lib/responses.py.j2 | 0 {template => scripts/template}/test/test_server.py.j2 | 0 {template => scripts/template}/test/test_service.py.j2 | 0 9 files changed, 5 insertions(+), 1 deletion(-) rename implementation_coverage.py => scripts/implementation_coverage.py (100%) rename setup_new_function.py => scripts/scaffold.py (99%) rename {template => scripts/template}/lib/__init__.py.j2 (100%) rename {template => scripts/template}/lib/exceptions.py.j2 (100%) rename {template => scripts/template}/lib/models.py.j2 (100%) rename {template => scripts/template}/lib/responses.py.j2 (100%) rename {template => scripts/template}/test/test_server.py.j2 (100%) rename {template => scripts/template}/test/test_service.py.j2 (100%) diff --git a/Makefile b/Makefile index 3c5582c2d..38a73aa28 100644 --- a/Makefile +++ b/Makefile @@ -19,3 +19,7 @@ publish: python setup.py sdist bdist_wheel upload git tag `python setup.py --version` git push origin `python setup.py --version` + +scaffold: + @pip install -r requirements-dev.txt > /dev/null + @python scripts/scaffold.py diff --git a/implementation_coverage.py b/scripts/implementation_coverage.py similarity index 100% rename from implementation_coverage.py rename to scripts/implementation_coverage.py diff --git a/setup_new_function.py b/scripts/scaffold.py similarity index 99% rename from setup_new_function.py rename to scripts/scaffold.py index 0102d74b2..c38544abf 100755 --- a/setup_new_function.py +++ b/scripts/scaffold.py @@ -24,7 +24,7 @@ from implementation_coverage import ( ) from inflection import singularize -TEMPLATE_DIR = './template' +TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), './template') INPUT_IGNORED_IN_BACKEND = ['Marker', 'PageSize'] OUTPUT_IGNORED_IN_BACKEND = ['NextMarker'] diff --git a/template/lib/__init__.py.j2 b/scripts/template/lib/__init__.py.j2 similarity index 100% rename from template/lib/__init__.py.j2 rename to scripts/template/lib/__init__.py.j2 diff --git a/template/lib/exceptions.py.j2 b/scripts/template/lib/exceptions.py.j2 similarity index 100% rename from template/lib/exceptions.py.j2 rename to scripts/template/lib/exceptions.py.j2 diff --git a/template/lib/models.py.j2 b/scripts/template/lib/models.py.j2 similarity index 100% rename from template/lib/models.py.j2 rename to scripts/template/lib/models.py.j2 diff --git a/template/lib/responses.py.j2 b/scripts/template/lib/responses.py.j2 similarity index 100% rename from template/lib/responses.py.j2 rename to scripts/template/lib/responses.py.j2 diff --git a/template/test/test_server.py.j2 b/scripts/template/test/test_server.py.j2 similarity index 100% rename from template/test/test_server.py.j2 rename to scripts/template/test/test_server.py.j2 diff --git a/template/test/test_service.py.j2 b/scripts/template/test/test_service.py.j2 similarity index 100% rename from template/test/test_service.py.j2 rename to scripts/template/test/test_service.py.j2 From 316a638d9e7ab8b4dca6f9d1279fe5451556bb1a Mon Sep 17 00:00:00 2001 From: Toshiya Kawasaki Date: Sat, 23 Sep 2017 17:03:42 +0900 Subject: [PATCH 11/12] add description of scaffold.py --- scripts/scaffold.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/scaffold.py b/scripts/scaffold.py index c38544abf..94bdad1fc 100755 --- a/scripts/scaffold.py +++ b/scripts/scaffold.py @@ -1,4 +1,14 @@ #!/usr/bin/env python +"""This script generates template codes and response body for specified boto3's operation and apply to appropriate files. +You only have to select service and operation that you want to add. +This script looks at the botocore's definition file of specified service and operation, and auto-generates codes and reponses. +Basically, this script supports almost all services, as long as its protocol is `query`, `json` or `rest-json`. +Event if aws adds new services, this script will work as long as the protocol is known. + +TODO: + - This scripts don't generates functions in `responses.py` for `rest-json`, because I don't know the rule of it. want someone fix this. + - In some services's operations, this scripts might crash. Make new issue on github then. +""" import os import re import inspect From 0a4c2301c755243c329bcd055609f6c4b8676b85 Mon Sep 17 00:00:00 2001 From: Toshiya Kawasaki Date: Sat, 23 Sep 2017 17:14:04 +0900 Subject: [PATCH 12/12] fix bug that existing template breaks --- scripts/scaffold.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/scaffold.py b/scripts/scaffold.py index 94bdad1fc..bbc8de208 100755 --- a/scripts/scaffold.py +++ b/scripts/scaffold.py @@ -319,8 +319,9 @@ def insert_code_to_class(path, base_class, new_code): lines = lines[:end_line_no] + func_lines + lines[end_line_no:] + body = '\n'.join(lines) + '\n' with open(path, 'w') as f: - f.write('\n'.join(lines)) + f.write(body) def insert_query_codes(service, operation):