diff --git a/moto/autoscaling/models.py b/moto/autoscaling/models.py index 0fdd82ddb..ec46d1182 100644 --- a/moto/autoscaling/models.py +++ b/moto/autoscaling/models.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals from boto.ec2.blockdevicemapping import BlockDeviceType, BlockDeviceMapping +from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel from moto.ec2 import ec2_backends from moto.elb import elb_backends @@ -284,8 +285,8 @@ class FakeAutoScalingGroup(BaseModel): class AutoScalingBackend(BaseBackend): def __init__(self, ec2_backend, elb_backend): - self.autoscaling_groups = {} - self.launch_configurations = {} + self.autoscaling_groups = OrderedDict() + self.launch_configurations = OrderedDict() self.policies = {} self.ec2_backend = ec2_backend self.elb_backend = elb_backend diff --git a/moto/autoscaling/responses.py b/moto/autoscaling/responses.py index b1d160320..2c3bddd79 100644 --- a/moto/autoscaling/responses.py +++ b/moto/autoscaling/responses.py @@ -40,11 +40,22 @@ class AutoScalingResponse(BaseResponse): def describe_launch_configurations(self): names = self._get_multi_param('LaunchConfigurationNames.member') - launch_configurations = self.autoscaling_backend.describe_launch_configurations( - names) + all_launch_configurations = self.autoscaling_backend.describe_launch_configurations(names) + marker = self._get_param('NextToken') + all_names = [lc.name for lc in all_launch_configurations] + if marker: + start = all_names.index(marker) + 1 + else: + start = 0 + max_records = self._get_param('MaxRecords', 50) # the default is 100, but using 50 to make testing easier + launch_configurations_resp = all_launch_configurations[start:start + max_records] + next_token = None + if len(all_launch_configurations) > start + max_records: + next_token = launch_configurations_resp[-1].name + template = self.response_template( DESCRIBE_LAUNCH_CONFIGURATIONS_TEMPLATE) - return template.render(launch_configurations=launch_configurations) + return template.render(launch_configurations=launch_configurations_resp, next_token=next_token) def delete_launch_configuration(self): launch_configurations_name = self.querystring.get( @@ -78,9 +89,22 @@ class AutoScalingResponse(BaseResponse): def describe_auto_scaling_groups(self): names = self._get_multi_param("AutoScalingGroupNames.member") - groups = self.autoscaling_backend.describe_autoscaling_groups(names) + token = self._get_param("NextToken") + all_groups = self.autoscaling_backend.describe_autoscaling_groups(names) + all_names = [group.name for group in all_groups] + if token: + start = all_names.index(token) + 1 + else: + start = 0 + max_records = self._get_param("MaxRecords", 50) + if max_records > 100: + raise ValueError + groups = all_groups[start:start + max_records] + next_token = None + if max_records and len(all_groups) > start + max_records: + next_token = groups[-1].name template = self.response_template(DESCRIBE_AUTOSCALING_GROUPS_TEMPLATE) - return template.render(groups=groups) + return template.render(groups=groups, next_token=next_token) def update_auto_scaling_group(self): self.autoscaling_backend.update_autoscaling_group( @@ -239,6 +263,9 @@ DESCRIBE_LAUNCH_CONFIGURATIONS_TEMPLATE = """ {% endfor %} + {% if next_token %} + {{ next_token }} + {% endif %} d05a22f8-b690-11e2-bf8e-2113fEXAMPLE @@ -331,6 +358,9 @@ DESCRIBE_AUTOSCALING_GROUPS_TEMPLATE = """ {% endfor %} + {% if next_token %} + {{ next_token }} + {% endif %} 0f02a07d-b677-11e2-9eb0-dd50EXAMPLE diff --git a/moto/cloudformation/models.py b/moto/cloudformation/models.py index b58d1dcf0..892f85174 100644 --- a/moto/cloudformation/models.py +++ b/moto/cloudformation/models.py @@ -4,6 +4,7 @@ import json import uuid import boto.cloudformation +from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel from .parsing import ResourceMap, OutputMap @@ -121,7 +122,7 @@ class FakeEvent(BaseModel): class CloudFormationBackend(BaseBackend): def __init__(self): - self.stacks = {} + self.stacks = OrderedDict() self.deleted_stacks = {} def create_stack(self, name, template, parameters, region_name, notification_arns=None, tags=None, role_arn=None): @@ -152,7 +153,7 @@ class CloudFormationBackend(BaseBackend): return [stack] raise ValidationError(name_or_stack_id) else: - return stacks + return list(stacks) def list_stacks(self): return self.stacks.values() diff --git a/moto/cloudformation/responses.py b/moto/cloudformation/responses.py index f1e6d0415..60f647efa 100644 --- a/moto/cloudformation/responses.py +++ b/moto/cloudformation/responses.py @@ -72,10 +72,20 @@ class CloudFormationResponse(BaseResponse): stack_name_or_id = None if self._get_param('StackName'): stack_name_or_id = self.querystring.get('StackName')[0] + token = self._get_param('NextToken') stacks = self.cloudformation_backend.describe_stacks(stack_name_or_id) - + stack_ids = [stack.stack_id for stack in stacks] + if token: + start = stack_ids.index(token) + 1 + else: + start = 0 + max_results = 50 # using this to mske testing of paginated stacks more convenient than default 1 MB + stacks_resp = stacks[start:start + max_results] + next_token = None + if len(stacks) > (start + max_results): + next_token = stacks_resp[-1].stack_id template = self.response_template(DESCRIBE_STACKS_TEMPLATE) - return template.render(stacks=stacks) + return template.render(stacks=stacks_resp, next_token=next_token) def describe_stack_resource(self): stack_name = self._get_param('StackName') @@ -270,6 +280,9 @@ DESCRIBE_STACKS_TEMPLATE = """ {% endfor %} + {% if next_token %} + {{ next_token }} + {% endif %} """ diff --git a/moto/datapipeline/models.py b/moto/datapipeline/models.py index 20fc4b12b..bb8417a20 100644 --- a/moto/datapipeline/models.py +++ b/moto/datapipeline/models.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import datetime import boto.datapipeline +from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel from .utils import get_random_pipeline_id, remove_capitalization_of_dict_keys @@ -111,7 +112,7 @@ class Pipeline(BaseModel): class DataPipelineBackend(BaseBackend): def __init__(self): - self.pipelines = {} + self.pipelines = OrderedDict() def create_pipeline(self, name, unique_id, **kwargs): pipeline = Pipeline(name, unique_id, **kwargs) diff --git a/moto/datapipeline/responses.py b/moto/datapipeline/responses.py index e75367c49..e462e3981 100644 --- a/moto/datapipeline/responses.py +++ b/moto/datapipeline/responses.py @@ -31,12 +31,25 @@ class DataPipelineResponse(BaseResponse): }) def list_pipelines(self): - pipelines = self.datapipeline_backend.list_pipelines() + pipelines = list(self.datapipeline_backend.list_pipelines()) + pipeline_ids = [pipeline.pipeline_id for pipeline in pipelines] + max_pipelines = 50 + marker = self.parameters.get('marker') + if marker: + start = pipeline_ids.index(marker) + 1 + else: + start = 0 + pipelines_resp = pipelines[start:start + max_pipelines] + has_more_results = False + marker = None + if start + max_pipelines < len(pipeline_ids) - 1: + has_more_results = True + marker = pipelines_resp[-1].pipeline_id return json.dumps({ - "hasMoreResults": False, - "marker": None, + "hasMoreResults": has_more_results, + "marker": marker, "pipelineIdList": [ - pipeline.to_meta_json() for pipeline in pipelines + pipeline.to_meta_json() for pipeline in pipelines_resp ] }) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 2ee5da203..45be1818f 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -200,6 +200,11 @@ class Table(BaseModel): self.global_indexes = global_indexes if global_indexes else [] self.created_at = datetime.datetime.utcnow() self.items = defaultdict(dict) + self.table_arn = self._generate_arn(table_name) + self.tags = [] + + def _generate_arn(self, name): + return 'arn:aws:dynamodb:us-east-1:123456789011:table/' + name def describe(self, base_key='TableDescription'): results = { @@ -209,11 +214,12 @@ class Table(BaseModel): 'TableSizeBytes': 0, 'TableName': self.name, 'TableStatus': 'ACTIVE', + 'TableArn': self.table_arn, 'KeySchema': self.schema, 'ItemCount': len(self), 'CreationDateTime': unix_time(self.created_at), 'GlobalSecondaryIndexes': [index for index in self.global_indexes], - 'LocalSecondaryIndexes': [index for index in self.indexes] + 'LocalSecondaryIndexes': [index for index in self.indexes], } } return results @@ -505,6 +511,18 @@ class DynamoDBBackend(BaseBackend): def delete_table(self, name): return self.tables.pop(name, None) + def tag_resource(self, table_arn, tags): + for table in self.tables: + if self.tables[table].table_arn == table_arn: + self.tables[table].tags.extend(tags) + + def list_tags_of_resource(self, table_arn): + required_table = None + for table in self.tables: + if self.tables[table].table_arn == table_arn: + required_table = self.tables[table] + return required_table.tags + def update_table_throughput(self, name, throughput): table = self.tables[name] table.throughput = throughput diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 95d52ebdd..11d23a830 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -73,7 +73,7 @@ class DynamoHandler(BaseResponse): def list_tables(self): body = self.body - limit = body.get('Limit') + limit = body.get('Limit', 100) if body.get("ExclusiveStartTableName"): last = body.get("ExclusiveStartTableName") start = list(dynamodb_backend2.tables.keys()).index(last) + 1 @@ -124,6 +124,35 @@ class DynamoHandler(BaseResponse): er = 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException' return self.error(er) + def tag_resource(self): + tags = self.body['Tags'] + table_arn = self.body['ResourceArn'] + dynamodb_backend2.tag_resource(table_arn, tags) + return json.dumps({}) + + def list_tags_of_resource(self): + try: + table_arn = self.body['ResourceArn'] + all_tags = dynamodb_backend2.list_tags_of_resource(table_arn) + all_tag_keys = [tag['Key'] for tag in all_tags] + marker = self.body.get('NextToken') + if marker: + start = all_tag_keys.index(marker) + 1 + else: + start = 0 + max_items = 10 # there is no default, but using 10 to make testing easier + tags_resp = all_tags[start:start + max_items] + next_marker = None + if len(all_tags) > start + max_items: + next_marker = tags_resp[-1]['Key'] + if next_marker: + return json.dumps({'Tags': tags_resp, + 'NextToken': next_marker}) + return json.dumps({'Tags': tags_resp}) + except AttributeError: + er = 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException' + return self.error(er) + def update_table(self): name = self.body['TableName'] if 'GlobalSecondaryIndexUpdates' in self.body: diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 64cc02e09..8be13d867 100755 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -12,6 +12,7 @@ from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType from boto.ec2.spotinstancerequest import SpotInstanceRequest as BotoSpotRequest from boto.ec2.launchspecification import LaunchSpecification +from moto.compat import OrderedDict from moto.core import BaseBackend from moto.core.models import Model, BaseModel from moto.core.utils import iso_8601_datetime_with_milliseconds, camelcase_to_underscores @@ -618,7 +619,7 @@ class Instance(TaggedEC2Resource, BotoInstance): class InstanceBackend(object): def __init__(self): - self.reservations = {} + self.reservations = OrderedDict() super(InstanceBackend, self).__init__() def get_instance(self, instance_id): @@ -1049,12 +1050,22 @@ class AmiBackend(object): self.amis[ami_id] = ami return ami - def describe_images(self, ami_ids=(), filters=None): + def describe_images(self, ami_ids=(), filters=None, exec_users=None): + images = [] + if exec_users: + for ami_id in self.amis: + found = False + for user_id in exec_users: + if user_id in self.amis[ami_id].launch_permission_users: + found = True + if found: + images.append(self.amis[ami_id]) + if images == []: + return images if filters: - images = self.amis.values() + images = images or self.amis.values() return generic_filter(filters, images) else: - images = [] for ami_id in ami_ids: if ami_id in self.amis: images.append(self.amis[ami_id]) @@ -1766,6 +1777,9 @@ class Snapshot(TaggedEC2Resource): if filter_name == 'encrypted': return str(self.encrypted).lower() + if filter_name == 'status': + return self.status + filter_value = super(Snapshot, self).get_filter_value(filter_name) if filter_value is None: diff --git a/moto/ec2/responses/amis.py b/moto/ec2/responses/amis.py index ab5256976..74767aa6b 100755 --- a/moto/ec2/responses/amis.py +++ b/moto/ec2/responses/amis.py @@ -1,7 +1,7 @@ from __future__ import unicode_literals from moto.core.responses import BaseResponse from moto.ec2.utils import instance_ids_from_querystring, image_ids_from_querystring, \ - filters_from_querystring, sequence_from_querystring + filters_from_querystring, sequence_from_querystring, executable_users_from_querystring class AmisResponse(BaseResponse): @@ -43,8 +43,9 @@ class AmisResponse(BaseResponse): def describe_images(self): ami_ids = image_ids_from_querystring(self.querystring) filters = filters_from_querystring(self.querystring) + exec_users = executable_users_from_querystring(self.querystring) images = self.ec2_backend.describe_images( - ami_ids=ami_ids, filters=filters) + ami_ids=ami_ids, filters=filters, exec_users=exec_users) template = self.response_template(DESCRIBE_IMAGES_RESPONSE) return template.render(images=images) diff --git a/moto/ec2/responses/instances.py b/moto/ec2/responses/instances.py index 7902dc375..d964fc22b 100644 --- a/moto/ec2/responses/instances.py +++ b/moto/ec2/responses/instances.py @@ -11,6 +11,7 @@ class InstanceResponse(BaseResponse): def describe_instances(self): filter_dict = filters_from_querystring(self.querystring) instance_ids = instance_ids_from_querystring(self.querystring) + token = self._get_param("NextToken") if instance_ids: reservations = self.ec2_backend.get_reservations_by_instance_ids( instance_ids, filters=filter_dict) @@ -18,8 +19,18 @@ class InstanceResponse(BaseResponse): reservations = self.ec2_backend.all_reservations( make_copy=True, filters=filter_dict) + reservation_ids = [reservation.id for reservation in reservations] + if token: + start = reservation_ids.index(token) + 1 + else: + start = 0 + max_results = int(self._get_param('MaxResults', 100)) + reservations_resp = reservations[start:start + max_results] + next_token = None + if max_results and len(reservations) > (start + max_results): + next_token = reservations_resp[-1].id template = self.response_template(EC2_DESCRIBE_INSTANCES) - return template.render(reservations=reservations) + return template.render(reservations=reservations_resp, next_token=next_token) def run_instances(self): min_count = int(self.querystring.get('MinCount', ['1'])[0]) @@ -492,6 +503,9 @@ EC2_DESCRIBE_INSTANCES = """ start + page_size: + next_marker = load_balancers_resp[-1].name + template = self.response_template(DESCRIBE_LOAD_BALANCERS_TEMPLATE) - return template.render(load_balancers=load_balancers) + return template.render(load_balancers=load_balancers_resp, marker=next_marker) def delete_load_balancer_listeners(self): load_balancer_name = self._get_param('LoadBalancerName') @@ -493,6 +505,9 @@ DESCRIBE_LOAD_BALANCERS_TEMPLATE = """ start + page_size: + next_marker = instances_resp[-1].db_instance_identifier + template = self.response_template(DESCRIBE_DATABASES_TEMPLATE) - return template.render(databases=databases) + return template.render(databases=instances_resp, marker=next_marker) def modify_db_instance(self): db_instance_identifier = self._get_param('DBInstanceIdentifier') @@ -187,6 +199,9 @@ DESCRIBE_DATABASES_TEMPLATE = """=2.8", "boto>=2.36.0", + "boto3>=1.2.1", "cookies", "requests>=2.0", "xmltodict", diff --git a/tests/test_autoscaling/test_autoscaling.py b/tests/test_autoscaling/test_autoscaling.py index 9a6408999..8487ecb49 100644 --- a/tests/test_autoscaling/test_autoscaling.py +++ b/tests/test_autoscaling/test_autoscaling.py @@ -115,6 +115,30 @@ def test_create_autoscaling_groups_defaults(): list(group.tags).should.equal([]) +@mock_autoscaling +def test_list_many_autoscaling_groups(): + conn = boto3.client('autoscaling', region_name='us-east-1') + conn.create_launch_configuration(LaunchConfigurationName='TestLC') + + for i in range(51): + conn.create_auto_scaling_group(AutoScalingGroupName='TestGroup%d' % i, + MinSize=1, + MaxSize=2, + LaunchConfigurationName='TestLC') + + response = conn.describe_auto_scaling_groups() + groups = response["AutoScalingGroups"] + marker = response["NextToken"] + groups.should.have.length_of(50) + marker.should.equal(groups[-1]['AutoScalingGroupName']) + + response2 = conn.describe_auto_scaling_groups(NextToken=marker) + + groups.extend(response2["AutoScalingGroups"]) + groups.should.have.length_of(51) + assert 'NextToken' not in response2.keys() + + @mock_autoscaling_deprecated def test_autoscaling_group_describe_filter(): conn = boto.connect_autoscale() diff --git a/tests/test_autoscaling/test_launch_configurations.py b/tests/test_autoscaling/test_launch_configurations.py index 1c1486421..931fc8a7e 100644 --- a/tests/test_autoscaling/test_launch_configurations.py +++ b/tests/test_autoscaling/test_launch_configurations.py @@ -1,11 +1,13 @@ from __future__ import unicode_literals import boto +import boto3 from boto.ec2.autoscale.launchconfig import LaunchConfiguration from boto.ec2.blockdevicemapping import BlockDeviceType, BlockDeviceMapping import sure # noqa from moto import mock_autoscaling_deprecated +from moto import mock_autoscaling from tests.helpers import requires_boto_gte @@ -208,6 +210,25 @@ def test_launch_configuration_describe_filter(): conn.get_all_launch_configurations().should.have.length_of(3) +@mock_autoscaling +def test_launch_configuration_describe_paginated(): + conn = boto3.client('autoscaling', region_name='us-east-1') + for i in range(51): + conn.create_launch_configuration(LaunchConfigurationName='TestLC%d' % i) + + response = conn.describe_launch_configurations() + lcs = response["LaunchConfigurations"] + marker = response["NextToken"] + lcs.should.have.length_of(50) + marker.should.equal(lcs[-1]['LaunchConfigurationName']) + + response2 = conn.describe_launch_configurations(NextToken=marker) + + lcs.extend(response2["LaunchConfigurations"]) + lcs.should.have.length_of(51) + assert 'NextToken' not in response2.keys() + + @mock_autoscaling_deprecated def test_launch_configuration_delete(): conn = boto.connect_autoscale() diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py index 9a531010f..85815e9f8 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py @@ -144,6 +144,26 @@ def test_create_stack_from_s3_url(): 'TemplateBody'].should.equal(dummy_template) +@mock_cloudformation +def test_describe_stack_pagination(): + conn = boto3.client('cloudformation', region_name='us-east-1') + for i in range(100): + conn.create_stack( + StackName="test_stack", + TemplateBody=dummy_template_json, + ) + + resp = conn.describe_stacks() + stacks = resp['Stacks'] + stacks.should.have.length_of(50) + next_token = resp['NextToken'] + next_token.should_not.be.none + resp2 = conn.describe_stacks(NextToken=next_token) + stacks.extend(resp2['Stacks']) + stacks.should.have.length_of(100) + assert 'NextToken' not in resp2.keys() + + @mock_cloudformation def test_describe_stack_resources(): cf_conn = boto3.client('cloudformation', region_name='us-east-1') diff --git a/tests/test_datapipeline/test_datapipeline.py b/tests/test_datapipeline/test_datapipeline.py index 490e3bfa4..ce190c7e4 100644 --- a/tests/test_datapipeline/test_datapipeline.py +++ b/tests/test_datapipeline/test_datapipeline.py @@ -170,6 +170,19 @@ def test_listing_pipelines(): }) +@mock_datapipeline_deprecated +def test_listing_paginated_pipelines(): + conn = boto.datapipeline.connect_to_region("us-west-2") + for i in range(100): + conn.create_pipeline("mypipeline%d" % i, "some-unique-id%d" % i) + + response = conn.list_pipelines() + + response["hasMoreResults"].should.be(True) + response["marker"].should.equal(response["pipelineIdList"][-1]['id']) + response["pipelineIdList"].should.have.length_of(50) + + # testing a helper function def test_remove_capitalization_of_dict_keys(): result = remove_capitalization_of_dict_keys( diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 860333e50..7fec5c2bd 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -2,11 +2,13 @@ from __future__ import unicode_literals, print_function import six import boto +import boto3 import sure # noqa import requests from moto import mock_dynamodb2, mock_dynamodb2_deprecated from moto.dynamodb2 import dynamodb_backend2 from boto.exception import JSONResponseError +from botocore.exceptions import ClientError from tests.helpers import requires_boto_gte import tests.backport_assert_raises from nose.tools import assert_raises @@ -64,3 +66,86 @@ def test_describe_missing_table(): aws_secret_access_key="sk") with assert_raises(JSONResponseError): conn.describe_table('messages') + + +@requires_boto_gte("2.9") +@mock_dynamodb2 +def test_list_table_tags(): + name = 'TestTable' + conn = boto3.client('dynamodb', + region_name='us-west-2', + aws_access_key_id="ak", + aws_secret_access_key="sk") + conn.create_table(TableName=name, + KeySchema=[{'AttributeName':'id','KeyType':'HASH'}], + AttributeDefinitions=[{'AttributeName':'id','AttributeType':'S'}], + ProvisionedThroughput={'ReadCapacityUnits':5,'WriteCapacityUnits':5}) + table_description = conn.describe_table(TableName=name) + arn = table_description['Table']['TableArn'] + tags = [{'Key':'TestTag', 'Value': 'TestValue'}] + conn.tag_resource(ResourceArn=arn, + Tags=tags) + resp = conn.list_tags_of_resource(ResourceArn=arn) + assert resp["Tags"] == tags + + +@requires_boto_gte("2.9") +@mock_dynamodb2 +def test_list_table_tags_empty(): + name = 'TestTable' + conn = boto3.client('dynamodb', + region_name='us-west-2', + aws_access_key_id="ak", + aws_secret_access_key="sk") + conn.create_table(TableName=name, + KeySchema=[{'AttributeName':'id','KeyType':'HASH'}], + AttributeDefinitions=[{'AttributeName':'id','AttributeType':'S'}], + ProvisionedThroughput={'ReadCapacityUnits':5,'WriteCapacityUnits':5}) + table_description = conn.describe_table(TableName=name) + arn = table_description['Table']['TableArn'] + tags = [{'Key':'TestTag', 'Value': 'TestValue'}] + # conn.tag_resource(ResourceArn=arn, + # Tags=tags) + resp = conn.list_tags_of_resource(ResourceArn=arn) + assert resp["Tags"] == [] + + +@requires_boto_gte("2.9") +@mock_dynamodb2 +def test_list_table_tags_paginated(): + name = 'TestTable' + conn = boto3.client('dynamodb', + region_name='us-west-2', + aws_access_key_id="ak", + aws_secret_access_key="sk") + conn.create_table(TableName=name, + KeySchema=[{'AttributeName':'id','KeyType':'HASH'}], + AttributeDefinitions=[{'AttributeName':'id','AttributeType':'S'}], + ProvisionedThroughput={'ReadCapacityUnits':5,'WriteCapacityUnits':5}) + table_description = conn.describe_table(TableName=name) + arn = table_description['Table']['TableArn'] + for i in range(11): + tags = [{'Key':'TestTag%d' % i, 'Value': 'TestValue'}] + conn.tag_resource(ResourceArn=arn, + Tags=tags) + resp = conn.list_tags_of_resource(ResourceArn=arn) + assert len(resp["Tags"]) == 10 + assert 'NextToken' in resp.keys() + resp2 = conn.list_tags_of_resource(ResourceArn=arn, + NextToken=resp['NextToken']) + assert len(resp2["Tags"]) == 1 + assert 'NextToken' not in resp2.keys() + + +@requires_boto_gte("2.9") +@mock_dynamodb2 +def test_list_not_found_table_tags(): + conn = boto3.client('dynamodb', + region_name='us-west-2', + aws_access_key_id="ak", + aws_secret_access_key="sk") + arn = 'DymmyArn' + try: + conn.list_tags_of_resource(ResourceArn=arn) + except ClientError as exception: + assert exception.response['Error']['Code'] == "ResourceNotFoundException" diff --git a/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py b/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py index c740350ef..cf7c958d3 100644 --- a/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py +++ b/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py @@ -77,13 +77,14 @@ def test_create_table(): 'TableSizeBytes': 0, 'TableName': 'messages', 'TableStatus': 'ACTIVE', + 'TableArn': 'arn:aws:dynamodb:us-east-1:123456789011:table/messages', 'KeySchema': [ {'KeyType': 'HASH', 'AttributeName': 'forum_name'}, {'KeyType': 'RANGE', 'AttributeName': 'subject'} ], 'LocalSecondaryIndexes': [], 'ItemCount': 0, 'CreationDateTime': 1326499200.0, - 'GlobalSecondaryIndexes': [], + 'GlobalSecondaryIndexes': [] } } table.describe().should.equal(expected) @@ -109,6 +110,7 @@ def test_create_table_with_local_index(): 'TableSizeBytes': 0, 'TableName': 'messages', 'TableStatus': 'ACTIVE', + 'TableArn': 'arn:aws:dynamodb:us-east-1:123456789011:table/messages', 'KeySchema': [ {'KeyType': 'HASH', 'AttributeName': 'forum_name'}, {'KeyType': 'RANGE', 'AttributeName': 'subject'} @@ -125,7 +127,7 @@ def test_create_table_with_local_index(): ], 'ItemCount': 0, 'CreationDateTime': 1326499200.0, - 'GlobalSecondaryIndexes': [], + 'GlobalSecondaryIndexes': [] } } table.describe().should.equal(expected) diff --git a/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py b/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py index 36e1b6c61..e38194f36 100644 --- a/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py +++ b/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py @@ -44,6 +44,7 @@ def test_create_table(): 'TableSizeBytes': 0, 'TableName': 'messages', 'TableStatus': 'ACTIVE', + 'TableArn': 'arn:aws:dynamodb:us-east-1:123456789011:table/messages', 'KeySchema': [ {'KeyType': 'HASH', 'AttributeName': 'forum_name'} ], diff --git a/tests/test_ec2/test_amis.py b/tests/test_ec2/test_amis.py index c9570e1a6..ed251f527 100755 --- a/tests/test_ec2/test_amis.py +++ b/tests/test_ec2/test_amis.py @@ -4,17 +4,18 @@ import tests.backport_assert_raises # noqa from nose.tools import assert_raises import boto +import boto3 import boto.ec2 import boto3 from boto.exception import EC2ResponseError, EC2ResponseError import sure # noqa -from moto import mock_emr_deprecated, mock_ec2 +from moto import mock_ec2_deprecated, mock_ec2 from tests.helpers import requires_boto_gte -@mock_emr_deprecated +@mock_ec2_deprecated def test_ami_create_and_delete(): conn = boto.connect_ec2('the_key', 'the_secret') reservation = conn.run_instances('ami-1234abcd') @@ -75,7 +76,7 @@ def test_ami_create_and_delete(): @requires_boto_gte("2.14.0") -@mock_emr_deprecated +@mock_ec2_deprecated def test_ami_copy(): conn = boto.ec2.connect_to_region("us-west-1") reservation = conn.run_instances('ami-1234abcd') @@ -134,7 +135,7 @@ def test_ami_copy(): cm.exception.request_id.should_not.be.none -@mock_emr_deprecated +@mock_ec2_deprecated def test_ami_tagging(): conn = boto.connect_vpc('the_key', 'the_secret') reservation = conn.run_instances('ami-1234abcd') @@ -161,7 +162,7 @@ def test_ami_tagging(): image.tags["a key"].should.equal("some value") -@mock_emr_deprecated +@mock_ec2_deprecated def test_ami_create_from_missing_instance(): conn = boto.connect_ec2('the_key', 'the_secret') args = ["i-abcdefg", "test-ami", "this is a test ami"] @@ -173,7 +174,7 @@ def test_ami_create_from_missing_instance(): cm.exception.request_id.should_not.be.none -@mock_emr_deprecated +@mock_ec2_deprecated def test_ami_pulls_attributes_from_instance(): conn = boto.connect_ec2('the_key', 'the_secret') reservation = conn.run_instances('ami-1234abcd') @@ -185,7 +186,7 @@ def test_ami_pulls_attributes_from_instance(): image.kernel_id.should.equal('test-kernel') -@mock_emr_deprecated +@mock_ec2_deprecated def test_ami_filters(): conn = boto.connect_ec2('the_key', 'the_secret') @@ -242,7 +243,7 @@ def test_ami_filters(): set([ami.id for ami in amis_by_nonpublic]).should.equal(set([imageA.id])) -@mock_emr_deprecated +@mock_ec2_deprecated def test_ami_filtering_via_tag(): conn = boto.connect_vpc('the_key', 'the_secret') @@ -268,7 +269,7 @@ def test_ami_filtering_via_tag(): set([ami.id for ami in amis_by_tagB]).should.equal(set([imageB.id])) -@mock_emr_deprecated +@mock_ec2_deprecated def test_getting_missing_ami(): conn = boto.connect_ec2('the_key', 'the_secret') @@ -279,7 +280,7 @@ def test_getting_missing_ami(): cm.exception.request_id.should_not.be.none -@mock_emr_deprecated +@mock_ec2_deprecated def test_getting_malformed_ami(): conn = boto.connect_ec2('the_key', 'the_secret') @@ -290,7 +291,7 @@ def test_getting_malformed_ami(): cm.exception.request_id.should_not.be.none -@mock_emr_deprecated +@mock_ec2_deprecated def test_ami_attribute_group_permissions(): conn = boto.connect_ec2('the_key', 'the_secret') reservation = conn.run_instances('ami-1234abcd') @@ -350,7 +351,7 @@ def test_ami_attribute_group_permissions(): **REMOVE_GROUP_ARGS).should_not.throw(EC2ResponseError) -@mock_emr_deprecated +@mock_ec2_deprecated def test_ami_attribute_user_permissions(): conn = boto.connect_ec2('the_key', 'the_secret') reservation = conn.run_instances('ami-1234abcd') @@ -422,7 +423,107 @@ def test_ami_attribute_user_permissions(): **REMOVE_USERS_ARGS).should_not.throw(EC2ResponseError) -@mock_emr_deprecated +@mock_ec2_deprecated +def test_ami_describe_executable_users(): + conn = boto3.client('ec2', region_name='us-east-1') + ec2 = boto3.resource('ec2', 'us-east-1') + ec2.create_instances(ImageId='', + MinCount=1, + MaxCount=1) + response = conn.describe_instances(Filters=[{'Name': 'instance-state-name','Values': ['running']}]) + instance_id = response['Reservations'][0]['Instances'][0]['InstanceId'] + image_id = conn.create_image(InstanceId=instance_id, + Name='TestImage',)['ImageId'] + + + USER1 = '123456789011' + + ADD_USER_ARGS = {'ImageId': image_id, + 'Attribute': 'launchPermission', + 'OperationType': 'add', + 'UserIds': [USER1]} + + # Add users and get no images + conn.modify_image_attribute(**ADD_USER_ARGS) + + attributes = conn.describe_image_attribute(ImageId=image_id, + Attribute='LaunchPermissions', + DryRun=False) + attributes['LaunchPermissions'].should.have.length_of(1) + attributes['LaunchPermissions'][0]['UserId'].should.equal(USER1) + images = conn.describe_images(ExecutableUsers=[USER1])['Images'] + images.should.have.length_of(1) + images[0]['ImageId'].should.equal(image_id) + + +@mock_ec2_deprecated +def test_ami_describe_executable_users_negative(): + conn = boto3.client('ec2', region_name='us-east-1') + ec2 = boto3.resource('ec2', 'us-east-1') + ec2.create_instances(ImageId='', + MinCount=1, + MaxCount=1) + response = conn.describe_instances(Filters=[{'Name': 'instance-state-name','Values': ['running']}]) + instance_id = response['Reservations'][0]['Instances'][0]['InstanceId'] + image_id = conn.create_image(InstanceId=instance_id, + Name='TestImage')['ImageId'] + + + USER1 = '123456789011' + USER2 = '113355789012' + + ADD_USER_ARGS = {'ImageId': image_id, + 'Attribute': 'launchPermission', + 'OperationType': 'add', + 'UserIds': [USER1]} + + # Add users and get no images + conn.modify_image_attribute(**ADD_USER_ARGS) + + attributes = conn.describe_image_attribute(ImageId=image_id, + Attribute='LaunchPermissions', + DryRun=False) + attributes['LaunchPermissions'].should.have.length_of(1) + attributes['LaunchPermissions'][0]['UserId'].should.equal(USER1) + images = conn.describe_images(ExecutableUsers=[USER2])['Images'] + images.should.have.length_of(0) + + +@mock_ec2_deprecated +def test_ami_describe_executable_users_and_filter(): + conn = boto3.client('ec2', region_name='us-east-1') + ec2 = boto3.resource('ec2', 'us-east-1') + ec2.create_instances(ImageId='', + MinCount=1, + MaxCount=1) + response = conn.describe_instances(Filters=[{'Name': 'instance-state-name','Values': ['running']}]) + instance_id = response['Reservations'][0]['Instances'][0]['InstanceId'] + image_id = conn.create_image(InstanceId=instance_id, + Name='ImageToDelete',)['ImageId'] + + + USER1 = '123456789011' + + ADD_USER_ARGS = {'ImageId': image_id, + 'Attribute': 'launchPermission', + 'OperationType': 'add', + 'UserIds': [USER1]} + + # Add users and get no images + conn.modify_image_attribute(**ADD_USER_ARGS) + + attributes = conn.describe_image_attribute(ImageId=image_id, + Attribute='LaunchPermissions', + DryRun=False) + attributes['LaunchPermissions'].should.have.length_of(1) + attributes['LaunchPermissions'][0]['UserId'].should.equal(USER1) + images = conn.describe_images(ExecutableUsers=[USER1], + Filters=[{'Name': 'state', 'Values': ['available']}])['Images'] + images.should.have.length_of(1) + images[0]['ImageId'].should.equal(image_id) + + +@mock_ec2_deprecated def test_ami_attribute_user_and_group_permissions(): """ Boto supports adding/removing both users and groups at the same time. @@ -477,7 +578,7 @@ def test_ami_attribute_user_and_group_permissions(): image.is_public.should.equal(False) -@mock_emr_deprecated +@mock_ec2_deprecated def test_ami_attribute_error_cases(): conn = boto.connect_ec2('the_key', 'the_secret') reservation = conn.run_instances('ami-1234abcd') diff --git a/tests/test_ec2/test_elastic_block_store.py b/tests/test_ec2/test_elastic_block_store.py index 83c89d129..b238e68f9 100644 --- a/tests/test_ec2/test_elastic_block_store.py +++ b/tests/test_ec2/test_elastic_block_store.py @@ -336,6 +336,11 @@ def test_snapshot_filters(): set([snap.id for snap in snapshots_by_volume_id] ).should.equal(set([snapshot1.id, snapshot2.id])) + snapshots_by_status = conn.get_all_snapshots( + filters={'status': 'completed'}) + set([snap.id for snap in snapshots_by_status] + ).should.equal(set([snapshot1.id, snapshot2.id, snapshot3.id])) + snapshots_by_volume_size = conn.get_all_snapshots( filters={'volume-size': volume1.size}) set([snap.id for snap in snapshots_by_volume_size] diff --git a/tests/test_ec2/test_instances.py b/tests/test_ec2/test_instances.py index 49020555b..6687a7e6c 100644 --- a/tests/test_ec2/test_instances.py +++ b/tests/test_ec2/test_instances.py @@ -7,12 +7,13 @@ import base64 import datetime import boto +import boto3 from boto.ec2.instance import Reservation, InstanceAttribute from boto.exception import EC2ResponseError, EC2ResponseError from freezegun import freeze_time import sure # noqa -from moto import mock_ec2_deprecated +from moto import mock_ec2_deprecated, mock_ec2 from tests.helpers import requires_boto_gte @@ -157,6 +158,26 @@ def test_get_instances_by_id(): cm.exception.request_id.should_not.be.none +@mock_ec2 +def test_get_paginated_instances(): + image_id = 'ami-1234abcd' + client = boto3.client('ec2', region_name='us-east-1') + conn = boto3.resource('ec2', 'us-east-1') + for i in range(100): + conn.create_instances(ImageId=image_id, + MinCount=1, + MaxCount=1) + resp = client.describe_instances(MaxResults=50) + reservations = resp['Reservations'] + reservations.should.have.length_of(50) + next_token = resp['NextToken'] + next_token.should_not.be.none + resp2 = client.describe_instances(NextToken=next_token) + reservations.extend(resp2['Reservations']) + reservations.should.have.length_of(100) + assert 'NextToken' not in resp2.keys() + + @mock_ec2_deprecated def test_get_instances_filtering_by_state(): conn = boto.connect_ec2() @@ -337,6 +358,20 @@ def test_get_instances_filtering_by_architecture(): reservations[0].instances.should.have.length_of(1) +@mock_ec2 +def test_get_instances_filtering_by_image_id(): + image_id = 'ami-1234abcd' + client = boto3.client('ec2', region_name='us-east-1') + conn = boto3.resource('ec2', 'us-east-1') + conn.create_instances(ImageId=image_id, + MinCount=1, + MaxCount=1) + + reservations = client.describe_instances(Filters=[{'Name': 'image-id', + 'Values': [image_id]}])['Reservations'] + reservations[0]['Instances'].should.have.length_of(1) + + @mock_ec2_deprecated def test_get_instances_filtering_by_tag(): conn = boto.connect_ec2() diff --git a/tests/test_elb/test_elb.py b/tests/test_elb/test_elb.py index 4b5d59d6d..3c991d565 100644 --- a/tests/test_elb/test_elb.py +++ b/tests/test_elb/test_elb.py @@ -109,6 +109,27 @@ def test_create_and_delete_boto3_support(): 'LoadBalancerDescriptions']).should.have.length_of(0) +@mock_elb +def test_describe_paginated_balancers(): + client = boto3.client('elb', region_name='us-east-1') + + for i in range(51): + client.create_load_balancer( + LoadBalancerName='my-lb%d' % i, + Listeners=[ + {'Protocol': 'tcp', 'LoadBalancerPort': 80, 'InstancePort': 8080}], + AvailabilityZones=['us-east-1a', 'us-east-1b'] + ) + + resp = client.describe_load_balancers() + resp['LoadBalancerDescriptions'].should.have.length_of(50) + resp['NextMarker'].should.equal(resp['LoadBalancerDescriptions'][-1]['LoadBalancerName']) + resp2 = client.describe_load_balancers(Marker=resp['NextMarker']) + resp2['LoadBalancerDescriptions'].should.have.length_of(1) + assert 'NextToken' not in resp2.keys() + + + @mock_elb_deprecated def test_add_listener(): conn = boto.connect_elb() diff --git a/tests/test_emr/test_emr_boto3.py b/tests/test_emr/test_emr_boto3.py index b2877c7f5..830abdb85 100644 --- a/tests/test_emr/test_emr_boto3.py +++ b/tests/test_emr/test_emr_boto3.py @@ -127,6 +127,18 @@ def test_describe_cluster(): cl['VisibleToAllUsers'].should.equal(True) +@mock_emr +def test_describe_cluster_not_found(): + conn = boto3.client('emr', region_name='us-east-1') + raised = False + try: + cluster = conn.describe_cluster(ClusterId='DummyId') + except ClientError as e: + if e.response['Error']['Code'] == "ResourceNotFoundException": + raised = True + raised.should.equal(True) + + @mock_emr def test_describe_job_flows(): client = boto3.client('emr', region_name='us-east-1') diff --git a/tests/test_kinesis/test_kinesis.py b/tests/test_kinesis/test_kinesis.py index 5b2f9ccf3..26a87f35a 100644 --- a/tests/test_kinesis/test_kinesis.py +++ b/tests/test_kinesis/test_kinesis.py @@ -2,9 +2,10 @@ from __future__ import unicode_literals import boto.kinesis from boto.kinesis.exceptions import ResourceNotFoundException, InvalidArgumentException +import boto3 import sure # noqa -from moto import mock_kinesis_deprecated +from moto import mock_kinesis, mock_kinesis_deprecated @mock_kinesis_deprecated @@ -51,6 +52,25 @@ def test_list_and_delete_stream(): "not-a-stream").should.throw(ResourceNotFoundException) +@mock_kinesis +def test_list_many_streams(): + conn = boto3.client('kinesis', region_name="us-west-2") + + for i in range(11): + conn.create_stream(StreamName="stream%d" % i, ShardCount=1) + + resp = conn.list_streams() + stream_names = resp["StreamNames"] + has_more_streams = resp["HasMoreStreams"] + stream_names.should.have.length_of(10) + has_more_streams.should.be(True) + resp2 = conn.list_streams(ExclusiveStartStreamName=stream_names[-1]) + stream_names = resp2["StreamNames"] + has_more_streams = resp2["HasMoreStreams"] + stream_names.should.have.length_of(1) + has_more_streams.should.equal(False) + + @mock_kinesis_deprecated def test_basic_shard_iterator(): conn = boto.kinesis.connect_to_region("us-west-2") diff --git a/tests/test_rds/test_rds.py b/tests/test_rds/test_rds.py index 090147d11..0a474ee26 100644 --- a/tests/test_rds/test_rds.py +++ b/tests/test_rds/test_rds.py @@ -1,11 +1,12 @@ from __future__ import unicode_literals +import boto3 import boto.rds import boto.vpc from boto.exception import BotoServerError import sure # noqa -from moto import mock_ec2_deprecated, mock_rds_deprecated +from moto import mock_ec2_deprecated, mock_rds_deprecated, mock_rds from tests.helpers import disable_on_py3 @@ -45,6 +46,26 @@ def test_get_databases(): databases[0].id.should.equal("db-master-1") +@disable_on_py3() +@mock_rds +def test_get_databases_paginated(): + conn = boto3.client('rds', region_name="us-west-2") + + for i in range(51): + conn.create_db_instance(AllocatedStorage=5, + Port=5432, + DBInstanceIdentifier='rds%d' % i, + DBInstanceClass='db.t1.micro', + Engine='postgres') + + resp = conn.describe_db_instances() + resp["DBInstances"].should.have.length_of(50) + resp["Marker"].should.equal(resp["DBInstances"][-1]['DBInstanceIdentifier']) + + resp2 = conn.describe_db_instances(Marker=resp["Marker"]) + resp2["DBInstances"].should.have.length_of(1) + + @mock_rds_deprecated def test_describe_non_existant_database(): conn = boto.rds.connect_to_region("us-west-2") diff --git a/tests/test_rds2/test_rds2.py b/tests/test_rds2/test_rds2.py index 1e2e0abdf..915695ad8 100644 --- a/tests/test_rds2/test_rds2.py +++ b/tests/test_rds2/test_rds2.py @@ -65,6 +65,25 @@ def test_get_databases(): 'arn:aws:rds:us-west-2:1234567890:db:db-master-1') +@disable_on_py3() +@mock_rds2 +def test_get_databases_paginated(): + conn = boto3.client('rds', region_name="us-west-2") + + for i in range(51): + conn.create_db_instance(AllocatedStorage=5, + Port=5432, + DBInstanceIdentifier='rds%d' % i, + DBInstanceClass='db.t1.micro', + Engine='postgres') + + resp = conn.describe_db_instances() + resp["DBInstances"].should.have.length_of(50) + resp["Marker"].should.equal(resp["DBInstances"][-1]['DBInstanceIdentifier']) + + resp2 = conn.describe_db_instances(Marker=resp["Marker"]) + resp2["DBInstances"].should.have.length_of(1) + @disable_on_py3() @mock_rds2 def test_describe_non_existant_database():