From 2bb79824ce2a687bd602e077b371738cc6453e16 Mon Sep 17 00:00:00 2001 From: Rob Walker Date: Sun, 26 Jul 2015 09:37:20 +1000 Subject: [PATCH 01/13] Volume attachments to show in instance. Volumes and Snapshots to be searchable by their id Placement of instance to match region connection Times for creation and attachment to show based on api call --- moto/ec2/models.py | 25 +++++++++++++++++++++-- moto/ec2/responses/elastic_block_store.py | 22 ++++++++++++++------ moto/ec2/responses/instances.py | 19 +++++++++-------- 3 files changed, 50 insertions(+), 16 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 3f3fc4491..19d005867 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from collections import defaultdict import copy from datetime import datetime + import itertools import re @@ -97,6 +98,9 @@ from .utils import ( ) +def utc_date_and_time(): + return datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') + def validate_resource_ids(resource_ids): for resource_id in resource_ids: if not is_valid_resource_id(resource_id): @@ -309,14 +313,17 @@ class Instance(BotoInstance, TaggedEC2Resource): in_ec2_classic = not bool(self.subnet_id) self.key_name = kwargs.get("key_name") self.source_dest_check = "true" - self.launch_time = datetime.utcnow().isoformat() + self.launch_time = utc_date_and_time() associate_public_ip = kwargs.get("associate_public_ip", False) if in_ec2_classic: # If we are in EC2-Classic, autoassign a public IP associate_public_ip = True self.block_device_mapping = BlockDeviceMapping() - self.block_device_mapping['/dev/sda1'] = BlockDeviceType(volume_id=random_volume_id()) + # Default have an instance with root volume should you not wish to override with attach volume cmd. + # However this is a ghost volume and wont show up in get_all_volumes or snapshot-able. + self.block_device_mapping['/dev/sda1'] = BlockDeviceType(volume_id=random_volume_id(), status='attached', + attach_time=utc_date_and_time()) amis = self.ec2_backend.describe_images(filters={'image-id': image_id}) ami = amis[0] if amis else None @@ -343,6 +350,10 @@ class Instance(BotoInstance, TaggedEC2Resource): private_ip=kwargs.get("private_ip"), associate_public_ip=associate_public_ip) + @property + def get_block_device_mapping(self): + return self.block_device_mapping.items() + @property def private_ip(self): return self.nics[0].private_ip_address @@ -1349,6 +1360,7 @@ class SecurityGroupIngress(object): class VolumeAttachment(object): def __init__(self, volume, instance, device): self.volume = volume + self.attach_time = utc_date_and_time() self.instance = instance self.device = device @@ -1373,6 +1385,7 @@ class Volume(TaggedEC2Resource): self.id = volume_id self.size = size self.zone = zone + self.create_time = utc_date_and_time() self.attachment = None self.ec2_backend = ec2_backend @@ -1404,6 +1417,7 @@ class Snapshot(TaggedEC2Resource): self.id = snapshot_id self.volume = volume self.description = description + self.start_time = utc_date_and_time() self.create_volume_permission_groups = set() self.ec2_backend = ec2_backend @@ -1444,6 +1458,13 @@ class EBSBackend(object): return False volume.attachment = VolumeAttachment(volume, instance, device_path) + # Modify instance to capture mount of block device. + bdt = BlockDeviceType(volume_id=volume_id, status=volume.status, size=volume.size, + attach_time=utc_date_and_time()) + try: + instance.block_device_mapping[device_path] = bdt + except: + instance.block_device_mapping.setdefault(device_path, bdt) return volume.attachment def detach_volume(self, volume_id, instance_id, device_path): diff --git a/moto/ec2/responses/elastic_block_store.py b/moto/ec2/responses/elastic_block_store.py index 96586a9bb..abb371260 100644 --- a/moto/ec2/responses/elastic_block_store.py +++ b/moto/ec2/responses/elastic_block_store.py @@ -42,12 +42,22 @@ class ElasticBlockStore(BaseResponse): return DELETE_VOLUME_RESPONSE def describe_snapshots(self): + # querystring for multiple snapshotids results in SnapshotId.1, SnapshotId.2 etc + snapshot_ids = ','.join([','.join(s[1]) for s in self.querystring.items() if 'SnapshotId' in s[0]]) snapshots = self.ec2_backend.describe_snapshots() + # Describe snapshots to handle filter on snapshot_ids + snapshots = [s for s in snapshots if s.id in snapshot_ids] if snapshot_ids else snapshots + # snapshots = self.ec2_backend.describe_snapshots() template = self.response_template(DESCRIBE_SNAPSHOTS_RESPONSE) return template.render(snapshots=snapshots) def describe_volumes(self): + # querystring for multiple volumeids results in VolumeId.1, VolumeId.2 etc + volume_ids = ','.join([','.join(v[1]) for v in self.querystring.items() if 'VolumeId' in v[0]]) volumes = self.ec2_backend.describe_volumes() + # Describe volumes to handle filter on volume_ids + volumes = [v for v in volumes if v.id in volume_ids] if volume_ids else volumes + # volumes = self.ec2_backend.describe_volumes() template = self.response_template(DESCRIBE_VOLUMES_RESPONSE) return template.render(volumes=volumes) @@ -103,7 +113,7 @@ CREATE_VOLUME_RESPONSE = """ @@ -174,7 +184,7 @@ CREATE_SNAPSHOT_RESPONSE = """ebs /dev/sda1 + {% for device_name,deviceobject in instance.get_block_device_mapping %} - /dev/sda1 + {{ device_name }} - {{ instance.block_device_mapping['/dev/sda1'].volume_id }} - attached - 2015-01-01T00:00:00.000Z - true + {{ deviceobject.volume_id }} + {{ deviceobject.status }} + {{ deviceobject.attach_time }} + {{ deviceobject.delete_on_termination }} + {{deviceobject.size}} + {% endfor %} {{ instance.virtualization_type }} ABCDE1234567890123 @@ -547,7 +550,7 @@ EC2_INSTANCE_STATUS = """ {% for instance in instances %} {{ instance.id }} - us-east-1d + {{ instance.placement }} {{ instance.state_code }} {{ instance.state }} From e33777b60c27a58d1af19c80ecf06527ee633e16 Mon Sep 17 00:00:00 2001 From: Rob Walker Date: Sun, 26 Jul 2015 09:51:37 +1000 Subject: [PATCH 02/13] tidy up imports. --- moto/ec2/models.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 19d005867..59d936008 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -1,18 +1,17 @@ from __future__ import unicode_literals -from collections import defaultdict +import boto import copy -from datetime import datetime - import itertools import re +import six -import boto +from collections import defaultdict +from datetime import datetime from boto.ec2.instance import Instance as BotoInstance, Reservation from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType from boto.ec2.spotinstancerequest import SpotInstanceRequest as BotoSpotRequest from boto.ec2.launchspecification import LaunchSpecification -import six from moto.core import BaseBackend from moto.core.models import Model From 7b3f17e8f0b53f38ba5b3c8ec0c6ce3537ca289b Mon Sep 17 00:00:00 2001 From: Rob Walker Date: Sun, 26 Jul 2015 09:53:03 +1000 Subject: [PATCH 03/13] add modified tests for volume and snapshot changes. --- tests/test_ec2/test_elastic_block_store.py | 33 ++++++++++++++++++++++ tests/test_ec2/test_instances.py | 31 +++++++++++++++++++- 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/tests/test_ec2/test_elastic_block_store.py b/tests/test_ec2/test_elastic_block_store.py index 50fe6b7ad..5ab5fcdc5 100644 --- a/tests/test_ec2/test_elastic_block_store.py +++ b/tests/test_ec2/test_elastic_block_store.py @@ -33,6 +33,19 @@ def test_create_and_delete_volume(): cm.exception.status.should.equal(400) cm.exception.request_id.should_not.be.none +@mock_ec2 +def test_filter_volume_by_id(): + conn = boto.connect_ec2('the_key', 'the_secret') + volume1 = conn.create_volume(80, "us-east-1a") + volume2 = conn.create_volume(36, "us-east-1b") + volume3 = conn.create_volume(20, "us-east-1c") + vol1 = conn.get_all_volumes(volume_ids=volume3.id) + vol1.should.have.length_of(1) + vol1[0].size.should.equal(20) + vol1[0].zone.should.equal('us-east-1c') + vol2 = conn.get_all_volumes(volume_ids=[volume1.id, volume2.id]) + vol2.should.have.length_of(2) + @mock_ec2 def test_volume_attach_and_detach(): @@ -85,6 +98,7 @@ def test_create_snapshot(): snapshots = conn.get_all_snapshots() snapshots.should.have.length_of(1) snapshots[0].description.should.equal('a test snapshot') + snapshots[0].start_time.should_not.be.none # Create snapshot without description snapshot = volume.create_snapshot() @@ -100,6 +114,25 @@ def test_create_snapshot(): cm.exception.status.should.equal(400) cm.exception.request_id.should_not.be.none +@mock_ec2 +def test_filter_snapshot_by_id(): + conn = boto.connect_ec2('the_key', 'the_secret') + volume1 = conn.create_volume(36, "us-east-1a") + snap1 = volume1.create_snapshot('a test snapshot 1') + volume2 = conn.create_volume(42, 'us-east-1a') + snap2 = volume2.create_snapshot('a test snapshot 2') + volume3 = conn.create_volume(84, 'us-east-1a') + snap3 = volume3.create_snapshot('a test snapshot 3') + snapshots1 = conn.get_all_snapshots(snapshot_ids=snap2.id) + snapshots1.should.have.length_of(1) + snapshots1[0].volume_id.should.equal(volume2.id) + snapshots1[0].region.name.should.equal('us-east-1') + snapshots2 = conn.get_all_snapshots(snapshot_ids=[snap2.id, snap3.id]) + snapshots2.should.have.length_of(2) + for s in snapshots2: + s.start_time.should_not.be.none + s.volume_id.should.be.within([volume2.id, volume3.id]) + s.region.name.should.equal('us-east-1') @mock_ec2 def test_snapshot_attribute(): diff --git a/tests/test_ec2/test_instances.py b/tests/test_ec2/test_instances.py index 5ee732d5e..629fc67ee 100644 --- a/tests/test_ec2/test_instances.py +++ b/tests/test_ec2/test_instances.py @@ -53,7 +53,7 @@ def test_instance_launch_and_terminate(): instances.should.have.length_of(1) instances[0].id.should.equal(instance.id) instances[0].state.should.equal('running') - instances[0].launch_time.should.equal("2014-01-01T05:00:00") + instances[0].launch_time.should.equal("2014-01-01T05:00:00Z") instances[0].vpc_id.should.equal(None) root_device_name = instances[0].root_device_name @@ -66,6 +66,35 @@ def test_instance_launch_and_terminate(): instance = reservations[0].instances[0] instance.state.should.equal('terminated') +@freeze_time("2014-01-01 05:00:00") +@mock_ec2 +def test_instance_attach_volume(): + conn = boto.connect_ec2('the_key', 'the_secret') + reservation = conn.run_instances('ami-1234abcd') + instance = reservation.instances[0] + + vol1 = conn.create_volume(size=36, zone=conn.region.name) + vol1.attach(instance.id, "/dev/sda1") + vol1.update() + vol2 = conn.create_volume(size=65, zone=conn.region.name) + vol2.attach(instance.id, "/dev/sdb1") + vol2.update() + vol3 = conn.create_volume(size=130, zone=conn.region.name) + vol3.attach(instance.id, "/dev/sdc1") + vol3.update() + + reservations = conn.get_all_instances() + instance = reservations[0].instances[0] + + instance.block_device_mapping.should.have.length_of(3) + + for v in conn.get_all_volumes(volume_ids=[instance.block_device_mapping['/dev/sdc1'].volume_id]): + v.attach_data.instance_id.should.equal(instance.id) + v.attach_data.attach_time.should.equal(instance.launch_time) # can do due to freeze_time decorator. + v.create_time.should.equal(instance.launch_time) # can do due to freeze_time decorator. + v.region.name.should.equal(instance.region.name) + v.status.should.equal('in-use') + @mock_ec2 def test_get_instances_by_id(): From e4408152d196eafc8245e79b771d54b094ec212d Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sat, 1 Aug 2015 19:32:33 -0400 Subject: [PATCH 04/13] Add KeyConditionExpression to dynamo. --- moto/dynamodb2/comparisons.py | 32 ++++- moto/dynamodb2/responses.py | 76 ++++++++--- .../test_dynamodb_table_with_range_key.py | 128 ++++++++++++++++-- .../test_dynamodb_table_without_range_key.py | 67 ++++++--- 4 files changed, 246 insertions(+), 57 deletions(-) diff --git a/moto/dynamodb2/comparisons.py b/moto/dynamodb2/comparisons.py index 86f582179..808e120bc 100644 --- a/moto/dynamodb2/comparisons.py +++ b/moto/dynamodb2/comparisons.py @@ -1,12 +1,32 @@ from __future__ import unicode_literals # TODO add tests for all of these + +EQ_FUNCTION = lambda item_value, test_value: item_value == test_value +NE_FUNCTION = lambda item_value, test_value: item_value != test_value +LE_FUNCTION = lambda item_value, test_value: item_value <= test_value +LT_FUNCTION = lambda item_value, test_value: item_value < test_value +GE_FUNCTION = lambda item_value, test_value: item_value >= test_value +GT_FUNCTION = lambda item_value, test_value: item_value > test_value + COMPARISON_FUNCS = { - 'EQ': lambda item_value, test_value: item_value == test_value, - 'NE': lambda item_value, test_value: item_value != test_value, - 'LE': lambda item_value, test_value: item_value <= test_value, - 'LT': lambda item_value, test_value: item_value < test_value, - 'GE': lambda item_value, test_value: item_value >= test_value, - 'GT': lambda item_value, test_value: item_value > test_value, + 'EQ': EQ_FUNCTION, + '=': EQ_FUNCTION, + + 'NE': NE_FUNCTION, + '!=': NE_FUNCTION, + + 'LE': LE_FUNCTION, + '<=': LE_FUNCTION, + + 'LT': LT_FUNCTION, + '<': LT_FUNCTION, + + 'GE': GE_FUNCTION, + '>=': GE_FUNCTION, + + 'GT': GT_FUNCTION, + '>': GT_FUNCTION, + 'NULL': lambda item_value: item_value is None, 'NOT_NULL': lambda item_value: item_value is not None, 'CONTAINS': lambda item_value, test_value: test_value in item_value, diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 81b6a7f47..069d31229 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -228,28 +228,64 @@ class DynamoHandler(BaseResponse): def query(self): name = self.body['TableName'] - key_conditions = self.body['KeyConditions'] - hash_key_name, range_key_name = dynamodb_backend2.get_table_keys_name(name, key_conditions.keys()) - # hash_key_name, range_key_name = dynamodb_backend2.get_table_keys_name(name) - if hash_key_name is None: - er = "'com.amazonaws.dynamodb.v20120810#ResourceNotFoundException" - return self.error(er) - hash_key = key_conditions[hash_key_name]['AttributeValueList'][0] - if len(key_conditions) == 1: - range_comparison = None - range_values = [] - else: - if range_key_name is None: - er = "com.amazon.coral.validate#ValidationException" - return self.error(er) - else: - range_condition = key_conditions[range_key_name] - if range_condition: - range_comparison = range_condition['ComparisonOperator'] - range_values = range_condition['AttributeValueList'] + + # {u'KeyConditionExpression': u'#n0 = :v0', u'ExpressionAttributeValues': {u':v0': {u'S': u'johndoe'}}, u'ExpressionAttributeNames': {u'#n0': u'username'}} + key_condition_expression = self.body.get('KeyConditionExpression') + if key_condition_expression: + value_alias_map = self.body['ExpressionAttributeValues'] + + if " AND " in key_condition_expression: + expressions = key_condition_expression.split(" AND ", 1) + hash_key_expression = expressions[0] + # TODO implement more than one range expression and OR operators + range_key_expression = expressions[1].replace(")", "") + range_key_expression_components = range_key_expression.split() + range_comparison = range_key_expression_components[1] + if 'AND' in range_key_expression: + range_comparison = 'BETWEEN' + range_values = [ + value_alias_map[range_key_expression_components[2]], + value_alias_map[range_key_expression_components[4]], + ] + elif 'begins_with' in range_key_expression: + range_comparison = 'BEGINS_WITH' + range_values = [ + value_alias_map[range_key_expression_components[1]], + ] else: + range_values = [value_alias_map[range_key_expression_components[2]]] + else: + hash_key_expression = key_condition_expression + range_comparison = None + range_values = [] + + hash_key_value_alias = hash_key_expression.split("=")[1].strip() + hash_key = value_alias_map[hash_key_value_alias] + else: + # 'KeyConditions': {u'forum_name': {u'ComparisonOperator': u'EQ', u'AttributeValueList': [{u'S': u'the-key'}]}} + key_conditions = self.body.get('KeyConditions') + if key_conditions: + hash_key_name, range_key_name = dynamodb_backend2.get_table_keys_name(name, key_conditions.keys()) + if hash_key_name is None: + er = "'com.amazonaws.dynamodb.v20120810#ResourceNotFoundException" + return self.error(er) + hash_key = key_conditions[hash_key_name]['AttributeValueList'][0] + if len(key_conditions) == 1: range_comparison = None range_values = [] + else: + if range_key_name is None: + er = "com.amazon.coral.validate#ValidationException" + return self.error(er) + else: + range_condition = key_conditions[range_key_name] + if range_condition: + range_comparison = range_condition['ComparisonOperator'] + range_values = range_condition['AttributeValueList'] + else: + range_comparison = None + range_values = [] + items, last_page = dynamodb_backend2.query(name, hash_key, range_comparison, range_values) if items is None: er = 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException' @@ -260,7 +296,7 @@ class DynamoHandler(BaseResponse): items = items[:limit] reversed = self.body.get("ScanIndexForward") - if reversed is not False: + if reversed is False: items.reverse() result = { diff --git a/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py b/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py index e7ae60e7e..12e3aa15b 100644 --- a/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py +++ b/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py @@ -1,6 +1,8 @@ from __future__ import unicode_literals import boto +import boto3 +from boto3.dynamodb.conditions import Key import sure # noqa from freezegun import freeze_time from moto import mock_dynamodb2 @@ -253,31 +255,31 @@ def test_query(): table.count().should.equal(4) - results = table.query(forum_name__eq='the-key', subject__gt='1', consistent=True) + results = table.query_2(forum_name__eq='the-key', subject__gt='1', consistent=True) expected = ["123", "456", "789"] for index, item in enumerate(results): item["subject"].should.equal(expected[index]) - results = table.query(forum_name__eq="the-key", subject__gt='1', reverse=True) + results = table.query_2(forum_name__eq="the-key", subject__gt='1', reverse=True) for index, item in enumerate(results): item["subject"].should.equal(expected[len(expected) - 1 - index]) - results = table.query(forum_name__eq='the-key', subject__gt='1', consistent=True) + results = table.query_2(forum_name__eq='the-key', subject__gt='1', consistent=True) sum(1 for _ in results).should.equal(3) - results = table.query(forum_name__eq='the-key', subject__gt='234', consistent=True) + results = table.query_2(forum_name__eq='the-key', subject__gt='234', consistent=True) sum(1 for _ in results).should.equal(2) - results = table.query(forum_name__eq='the-key', subject__gt='9999') + results = table.query_2(forum_name__eq='the-key', subject__gt='9999') sum(1 for _ in results).should.equal(0) - results = table.query(forum_name__eq='the-key', subject__beginswith='12') + results = table.query_2(forum_name__eq='the-key', subject__beginswith='12') sum(1 for _ in results).should.equal(1) - results = table.query(forum_name__eq='the-key', subject__beginswith='7') + results = table.query_2(forum_name__eq='the-key', subject__beginswith='7') sum(1 for _ in results).should.equal(1) - results = table.query(forum_name__eq='the-key', subject__between=['567', '890']) + results = table.query_2(forum_name__eq='the-key', subject__between=['567', '890']) sum(1 for _ in results).should.equal(1) @@ -558,7 +560,6 @@ def test_lookup(): @mock_dynamodb2 def test_failed_overwrite(): - from decimal import Decimal table = Table.create('messages', schema=[ HashKey('id'), RangeKey('range'), @@ -567,19 +568,19 @@ def test_failed_overwrite(): 'write': 3, }) - data1 = {'id': '123', 'range': 'abc', 'data':'678'} + data1 = {'id': '123', 'range': 'abc', 'data': '678'} table.put_item(data=data1) - data2 = {'id': '123', 'range': 'abc', 'data':'345'} - table.put_item(data=data2, overwrite = True) + data2 = {'id': '123', 'range': 'abc', 'data': '345'} + table.put_item(data=data2, overwrite=True) - data3 = {'id': '123', 'range': 'abc', 'data':'812'} + data3 = {'id': '123', 'range': 'abc', 'data': '812'} table.put_item.when.called_with(data=data3).should.throw(ConditionalCheckFailedException) returned_item = table.lookup('123', 'abc') dict(returned_item).should.equal(data2) - data4 = {'id': '123', 'range': 'ghi', 'data':812} + data4 = {'id': '123', 'range': 'ghi', 'data': 812} table.put_item(data=data4) returned_item = table.lookup('123', 'ghi') @@ -593,7 +594,7 @@ def test_conflicting_writes(): RangeKey('range'), ]) - item_data = {'id': '123', 'range':'abc', 'data':'678'} + item_data = {'id': '123', 'range': 'abc', 'data': '678'} item1 = Item(table, item_data) item2 = Item(table, item_data) item1.save() @@ -603,3 +604,100 @@ def test_conflicting_writes(): item1.save() item2.save.when.called_with().should.throw(ConditionalCheckFailedException) + +""" +boto3 +""" + + +@mock_dynamodb2 +def test_boto3_conditions(): + dynamodb = boto3.resource('dynamodb', region_name='us-east-1') + + # Create the DynamoDB table. + table = dynamodb.create_table( + TableName='users', + KeySchema=[ + { + 'AttributeName': 'forum_name', + 'KeyType': 'HASH' + }, + { + 'AttributeName': 'subject', + 'KeyType': 'RANGE' + }, + ], + AttributeDefinitions=[ + { + 'AttributeName': 'forum_name', + 'AttributeType': 'S' + }, + { + 'AttributeName': 'subject', + 'AttributeType': 'S' + }, + ], + ProvisionedThroughput={ + 'ReadCapacityUnits': 5, + 'WriteCapacityUnits': 5 + } + ) + table = dynamodb.Table('users') + + table.put_item(Item={ + 'forum_name': 'the-key', + 'subject': '123' + }) + table.put_item(Item={ + 'forum_name': 'the-key', + 'subject': '456' + }) + table.put_item(Item={ + 'forum_name': 'the-key', + 'subject': '789' + }) + + # Test a query returning all items + results = table.query( + KeyConditionExpression=Key('forum_name').eq('the-key') & Key("subject").gt('1'), + ScanIndexForward=True, + ) + expected = ["123", "456", "789"] + for index, item in enumerate(results['Items']): + item["subject"].should.equal(expected[index]) + + # Return all items again, but in reverse + results = table.query( + KeyConditionExpression=Key('forum_name').eq('the-key') & Key("subject").gt('1'), + ScanIndexForward=False, + ) + for index, item in enumerate(reversed(results['Items'])): + item["subject"].should.equal(expected[index]) + + # Filter the subjects to only return some of the results + results = table.query( + KeyConditionExpression=Key('forum_name').eq('the-key') & Key("subject").gt('234'), + ConsistentRead=True, + ) + results['Count'].should.equal(2) + + # Filter to return no results + results = table.query( + KeyConditionExpression=Key('forum_name').eq('the-key') & Key("subject").gt('9999') + ) + results['Count'].should.equal(0) + + results = table.query( + KeyConditionExpression=Key('forum_name').eq('the-key') & Key("subject").begins_with('12') + ) + results['Count'].should.equal(1) + + results = table.query( + KeyConditionExpression=Key('forum_name').eq('the-key') & Key("subject").begins_with('7') + ) + results['Count'].should.equal(1) + + results = table.query( + KeyConditionExpression=Key('forum_name').eq('the-key') & Key("subject").between('567', '890') + ) + results['Count'].should.equal(1) diff --git a/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py b/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py index 1710bd650..808805b8d 100644 --- a/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py +++ b/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py @@ -1,6 +1,8 @@ from __future__ import unicode_literals import boto +import boto3 +from boto3.dynamodb.conditions import Key import sure # noqa from freezegun import freeze_time from boto.exception import JSONResponseError @@ -135,14 +137,6 @@ def test_item_put_without_table(): ).should.throw(JSONResponseError) -@requires_boto_gte("2.9") -@mock_dynamodb2 -def test_get_missing_item(): - table = create_table() - - table.get_item.when.called_with(test_hash=3241526475).should.throw(JSONResponseError) - - @requires_boto_gte("2.9") @mock_dynamodb2 def test_get_item_with_undeclared_table(): @@ -449,7 +443,6 @@ def test_update_item_set(): @mock_dynamodb2 def test_failed_overwrite(): - from decimal import Decimal table = Table.create('messages', schema=[ HashKey('id'), ], throughput={ @@ -457,19 +450,19 @@ def test_failed_overwrite(): 'write': 3, }) - data1 = {'id': '123', 'data':'678'} + data1 = {'id': '123', 'data': '678'} table.put_item(data=data1) - data2 = {'id': '123', 'data':'345'} - table.put_item(data=data2, overwrite = True) + data2 = {'id': '123', 'data': '345'} + table.put_item(data=data2, overwrite=True) - data3 = {'id': '123', 'data':'812'} + data3 = {'id': '123', 'data': '812'} table.put_item.when.called_with(data=data3).should.throw(ConditionalCheckFailedException) returned_item = table.lookup('123') dict(returned_item).should.equal(data2) - data4 = {'id': '124', 'data':812} + data4 = {'id': '124', 'data': 812} table.put_item(data=data4) returned_item = table.lookup('124') @@ -482,7 +475,7 @@ def test_conflicting_writes(): HashKey('id'), ]) - item_data = {'id': '123', 'data':'678'} + item_data = {'id': '123', 'data': '678'} item1 = Item(table, item_data) item2 = Item(table, item_data) item1.save() @@ -491,4 +484,46 @@ def test_conflicting_writes(): item2['data'] = '912' item1.save() - item2.save.when.called_with().should.throw(ConditionalCheckFailedException) \ No newline at end of file + item2.save.when.called_with().should.throw(ConditionalCheckFailedException) + + +""" +boto3 +""" + + +@mock_dynamodb2 +def test_boto3_conditions(): + dynamodb = boto3.resource('dynamodb', region_name='us-east-1') + + # Create the DynamoDB table. + table = dynamodb.create_table( + TableName='users', + KeySchema=[ + { + 'AttributeName': 'username', + 'KeyType': 'HASH' + }, + ], + AttributeDefinitions=[ + { + 'AttributeName': 'username', + 'AttributeType': 'S' + }, + ], + ProvisionedThroughput={ + 'ReadCapacityUnits': 5, + 'WriteCapacityUnits': 5 + } + ) + table = dynamodb.Table('users') + + table.put_item(Item={'username': 'johndoe'}) + table.put_item(Item={'username': 'janedoe'}) + + response = table.query( + KeyConditionExpression=Key('username').eq('johndoe') + ) + response['Count'].should.equal(1) + response['Items'].should.have.length_of(1) + response['Items'][0].should.equal({"username": "johndoe"}) From 491df739d6ac864b9aa236cbc376806ee8a116ed Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sat, 1 Aug 2015 19:35:09 -0400 Subject: [PATCH 05/13] Add boto3 to Travis. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index be2d97c86..3befdf927 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ matrix: env: BOTO_VERSION=2.36.0 install: - travis_retry pip install boto==$BOTO_VERSION + - travis_retry pip install boto3 - travis_retry pip install . - travis_retry pip install -r requirements-dev.txt - travis_retry pip install coveralls From ac4aef87a142dd04ba5f9f6ee42dab50cb77d79e Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sat, 1 Aug 2015 19:44:37 -0400 Subject: [PATCH 06/13] Unused exception. --- moto/ec2/models.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 59d936008..f800097d2 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -100,6 +100,7 @@ from .utils import ( def utc_date_and_time(): return datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') + def validate_resource_ids(resource_ids): for resource_id in resource_ids: if not is_valid_resource_id(resource_id): @@ -1460,10 +1461,7 @@ class EBSBackend(object): # Modify instance to capture mount of block device. bdt = BlockDeviceType(volume_id=volume_id, status=volume.status, size=volume.size, attach_time=utc_date_and_time()) - try: - instance.block_device_mapping[device_path] = bdt - except: - instance.block_device_mapping.setdefault(device_path, bdt) + instance.block_device_mapping[device_path] = bdt return volume.attachment def detach_volume(self, volume_id, instance_id, device_path): From 519726a70acd8bdbd724ecb437d96a08550e0f74 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 2 Aug 2015 09:45:40 -0400 Subject: [PATCH 07/13] Cleaning up SQS for boto3. Closes #385. --- moto/server.py | 11 +++++++-- moto/sqs/responses.py | 46 +++++++++++++++++++++----------------- moto/sqs/urls.py | 8 ++++--- tests/test_sqs/test_sqs.py | 15 +++++++++++++ 4 files changed, 54 insertions(+), 26 deletions(-) diff --git a/moto/server.py b/moto/server.py index af3ec55bd..4a7de1a70 100644 --- a/moto/server.py +++ b/moto/server.py @@ -79,10 +79,17 @@ def create_backend_app(service): else: endpoint = None - backend_app.route( + if endpoint in backend_app.view_functions: + # HACK: Sometimes we map the same view to multiple url_paths. Flask + # requries us to have different names. + endpoint += "2" + + backend_app.add_url_rule( url_path, endpoint=endpoint, - methods=HTTP_METHODS)(convert_flask_to_httpretty_response(handler)) + methods=HTTP_METHODS, + view_func=convert_flask_to_httpretty_response(handler), + ) return backend_app diff --git a/moto/sqs/responses.py b/moto/sqs/responses.py index 69b3f63df..d20372762 100644 --- a/moto/sqs/responses.py +++ b/moto/sqs/responses.py @@ -11,10 +11,11 @@ from .exceptions import ( ) MAXIMUM_VISIBILTY_TIMEOUT = 43200 +DEFAULT_RECEIVED_MESSAGES = 1 SQS_REGION_REGEX = r'://(.+?)\.queue\.amazonaws\.com' -class QueuesResponse(BaseResponse): +class SQSResponse(BaseResponse): region_regex = SQS_REGION_REGEX @@ -22,6 +23,14 @@ class QueuesResponse(BaseResponse): def sqs_backend(self): return sqs_backends[self.region] + def _get_queue_name(self): + try: + queue_name = self.querystring.get('QueueUrl')[0].split("/")[-1] + except TypeError: + # Fallback to reading from the URL + queue_name = self.path.split("/")[-1] + return queue_name + def create_queue(self): visibility_timeout = None if 'Attribute.1.Name' in self.querystring and self.querystring.get('Attribute.1.Name')[0] == 'VisibilityTimeout': @@ -47,17 +56,8 @@ class QueuesResponse(BaseResponse): template = self.response_template(LIST_QUEUES_RESPONSE) return template.render(queues=queues) - -class QueueResponse(BaseResponse): - - region_regex = SQS_REGION_REGEX - - @property - def sqs_backend(self): - return sqs_backends[self.region] - def change_message_visibility(self): - queue_name = self.path.split("/")[-1] + queue_name = self._get_queue_name() receipt_handle = self.querystring.get("ReceiptHandle")[0] visibility_timeout = int(self.querystring.get("VisibilityTimeout")[0]) @@ -79,20 +79,20 @@ class QueueResponse(BaseResponse): return template.render() def get_queue_attributes(self): - queue_name = self.path.split("/")[-1] + queue_name = self._get_queue_name() queue = self.sqs_backend.get_queue(queue_name) template = self.response_template(GET_QUEUE_ATTRIBUTES_RESPONSE) return template.render(queue=queue) def set_queue_attributes(self): - queue_name = self.path.split("/")[-1] + queue_name = self._get_queue_name() key = camelcase_to_underscores(self.querystring.get('Attribute.Name')[0]) value = self.querystring.get('Attribute.Value')[0] self.sqs_backend.set_queue_attribute(queue_name, key, value) return SET_QUEUE_ATTRIBUTE_RESPONSE def delete_queue(self): - queue_name = self.path.split("/")[-1] + queue_name = self._get_queue_name() queue = self.sqs_backend.delete_queue(queue_name) if not queue: return "A queue with name {0} does not exist".format(queue_name), dict(status=404) @@ -113,7 +113,8 @@ class QueueResponse(BaseResponse): except MessageAttributesInvalid as e: return e.description, dict(status=e.status_code) - queue_name = self.path.split("/")[-1] + queue_name = self._get_queue_name() + message = self.sqs_backend.send_message( queue_name, message, @@ -135,7 +136,7 @@ class QueueResponse(BaseResponse): 'SendMessageBatchRequestEntry.2.DelaySeconds': ['0'], """ - queue_name = self.path.split("/")[-1] + queue_name = self._get_queue_name() messages = [] for index in range(1, 11): @@ -164,7 +165,7 @@ class QueueResponse(BaseResponse): return template.render(messages=messages) def delete_message(self): - queue_name = self.path.split("/")[-1] + queue_name = self._get_queue_name() receipt_handle = self.querystring.get("ReceiptHandle")[0] self.sqs_backend.delete_message(queue_name, receipt_handle) template = self.response_template(DELETE_MESSAGE_RESPONSE) @@ -180,7 +181,7 @@ class QueueResponse(BaseResponse): 'DeleteMessageBatchRequestEntry.2.ReceiptHandle': ['zxcvfda...'], ... """ - queue_name = self.path.split("/")[-1] + queue_name = self._get_queue_name() message_ids = [] for index in range(1, 11): @@ -201,14 +202,17 @@ class QueueResponse(BaseResponse): return template.render(message_ids=message_ids) def purge_queue(self): - queue_name = self.path.split("/")[-1] + queue_name = self._get_queue_name() self.sqs_backend.purge_queue(queue_name) template = self.response_template(PURGE_QUEUE_RESPONSE) return template.render() def receive_message(self): - queue_name = self.path.split("/")[-1] - message_count = int(self.querystring.get("MaxNumberOfMessages")[0]) + queue_name = self._get_queue_name() + try: + message_count = int(self.querystring.get("MaxNumberOfMessages")[0]) + except TypeError: + message_count = DEFAULT_RECEIVED_MESSAGES messages = self.sqs_backend.receive_messages(queue_name, message_count) template = self.response_template(RECEIVE_MESSAGE_RESPONSE) output = template.render(messages=messages) diff --git a/moto/sqs/urls.py b/moto/sqs/urls.py index 527a867c2..0780615ab 100644 --- a/moto/sqs/urls.py +++ b/moto/sqs/urls.py @@ -1,11 +1,13 @@ from __future__ import unicode_literals -from .responses import QueueResponse, QueuesResponse +from .responses import SQSResponse url_bases = [ "https?://(.*?)(queue|sqs)(.*?).amazonaws.com" ] +dispatch = SQSResponse().dispatch + url_paths = { - '{0}/$': QueuesResponse.dispatch, - '{0}/(?P\d+)/(?P[a-zA-Z0-9\-_]+)': QueueResponse.dispatch, + '{0}/$': dispatch, + '{0}/(?P\d+)/(?P[a-zA-Z0-9\-_]+)': dispatch, } diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index bc6d36a4e..ba1e11e52 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals import boto +import boto3 from boto.exception import SQSError from boto.sqs.message import RawMessage, Message @@ -462,3 +463,17 @@ def test_delete_message_after_visibility_timeout(): m1_retrieved.delete() assert new_queue.count() == 0 + +""" +boto3 +""" + + +@mock_sqs +def test_boto3_message_send(): + sqs = boto3.resource('sqs', region_name='us-east-1') + queue = sqs.create_queue(QueueName="blah") + queue.send_message(MessageBody="derp") + + messages = queue.receive_messages() + messages.should.have.length_of(1) From ff8dd7530d324c684b53e8967e0b247f5f533fff Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 2 Aug 2015 09:54:23 -0400 Subject: [PATCH 08/13] Add boto3 s3 test. --- tests/test_s3/test_s3.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 8e1c35f5c..ca54988f7 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -8,6 +8,7 @@ from io import BytesIO import json import boto +import boto3 from boto.exception import S3CreateError, S3ResponseError from boto.s3.connection import S3Connection from boto.s3.key import Key @@ -869,3 +870,18 @@ def test_policy(): bucket = conn.get_bucket(bucket_name) bucket.get_policy().decode('utf-8').should.equal(policy) + + +""" +boto3 +""" + + +@mock_s3 +def test_boto3_bucket_create(): + s3 = boto3.resource('s3', region_name='us-east-1') + s3.create_bucket(Bucket="blah") + + s3.Object('blah', 'hello.txt').put(Body="some text") + + s3.Object('blah', 'hello.txt').get()['Body'].read().should.equal("some text") From dd092fa9dcc5b6446a245f8e4c3da1edbb2d7cc7 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 2 Aug 2015 09:59:16 -0400 Subject: [PATCH 09/13] Fix s3 py3 test for unicode. --- tests/test_s3/test_s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index ca54988f7..2b9fb7099 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -884,4 +884,4 @@ def test_boto3_bucket_create(): s3.Object('blah', 'hello.txt').put(Body="some text") - s3.Object('blah', 'hello.txt').get()['Body'].read().should.equal("some text") + s3.Object('blah', 'hello.txt').get()['Body'].read().decode("utf-8").should.equal("some text") From 8573333932ad4f4dcd6c80ed119c022cfdcdc81f Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 2 Aug 2015 10:04:13 -0400 Subject: [PATCH 10/13] 0.4.8 --- moto/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/__init__.py b/moto/__init__.py index 6aa89cad6..ee566ee68 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '0.4.7' +__version__ = '0.4.8' from .autoscaling import mock_autoscaling # flake8: noqa from .cloudformation import mock_cloudformation # flake8: noqa diff --git a/setup.py b/setup.py index f9371cc3e..50ab56eea 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ if sys.version_info < (2, 7): setup( name='moto', - version='0.4.7', + version='0.4.8', description='A library that allows your python tests to easily' ' mock out the boto library', author='Steve Pulec', From b2570d901edcbcaf850f430c7eda1bc45b2ca280 Mon Sep 17 00:00:00 2001 From: Jot Date: Sun, 2 Aug 2015 16:27:08 +0200 Subject: [PATCH 11/13] Enabled cloudformation in server mode with some tests --- moto/backends.py | 2 ++ moto/cloudformation/responses.py | 2 +- tests/test_cloudformation/test_server.py | 38 ++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/moto/backends.py b/moto/backends.py index de323a442..b45e0fa5e 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals from moto.autoscaling import autoscaling_backend from moto.cloudwatch import cloudwatch_backend +from moto.cloudformation import cloudformation_backend from moto.dynamodb import dynamodb_backend from moto.dynamodb2 import dynamodb_backend2 from moto.ec2 import ec2_backend @@ -20,6 +21,7 @@ from moto.route53 import route53_backend BACKENDS = { 'autoscaling': autoscaling_backend, + 'cloudformation': cloudformation_backend, 'cloudwatch': cloudwatch_backend, 'dynamodb': dynamodb_backend, 'dynamodb2': dynamodb_backend2, diff --git a/moto/cloudformation/responses.py b/moto/cloudformation/responses.py index fe0af6640..e0afa5fa3 100644 --- a/moto/cloudformation/responses.py +++ b/moto/cloudformation/responses.py @@ -161,7 +161,7 @@ LIST_STACKS_RESPONSE = """ {% for stack in stacks %} - {{ stack.id }} + {{ stack.stack_id }} {{ stack.status }} {{ stack.name }} 2011-05-23T15:47:44Z diff --git a/tests/test_cloudformation/test_server.py b/tests/test_cloudformation/test_server.py index baffc4882..d45c3dc4a 100644 --- a/tests/test_cloudformation/test_server.py +++ b/tests/test_cloudformation/test_server.py @@ -1 +1,39 @@ from __future__ import unicode_literals + +import json +import urllib +import re +import sure # noqa + +import moto.server as server + +''' +Test the different server responses +''' + +def test_cloudformation_server_get(): + backend = server.create_backend_app("cloudformation") + stack_name = 'test stack' + test_client = backend.test_client() + template_body = { + "Resources": {}, + } + res = test_client.get( + '/?{0}'.format( + urllib.urlencode({ + "Action": "CreateStack", + "StackName": stack_name, + "TemplateBody": json.dumps(template_body) + })), + headers={"Host":"cloudformation.us-east-1.amazonaws.com"} + ) + stack_id = json.loads(res.data)["CreateStackResponse"]["CreateStackResult"]["StackId"] + + res = test_client.get( + '/?Action=ListStacks', + headers={"Host":"cloudformation.us-east-1.amazonaws.com"} + ) + stacks = re.search("(.*)", res.data.decode('utf-8')) + + list_stack_id = stacks.groups()[0] + assert stack_id == list_stack_id From 4576f2873eaafb074f35ca80720934c9339570a4 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 2 Aug 2015 11:26:29 -0400 Subject: [PATCH 12/13] Allow passing of region name to s3bucket_path url for eu-central-1. cc ##348. --- moto/s3bucket_path/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/s3bucket_path/urls.py b/moto/s3bucket_path/urls.py index f37ab8cb7..c5dc86f2f 100644 --- a/moto/s3bucket_path/urls.py +++ b/moto/s3bucket_path/urls.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from .responses import S3BucketPathResponseInstance as ro url_bases = [ - "https?://s3.amazonaws.com" + "https?://s3(.*).amazonaws.com" ] From 71d2e1d7de172aaeb45e6990e0cc110b11fc2a6d Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 2 Aug 2015 11:37:10 -0400 Subject: [PATCH 13/13] Python3 fixes for cloudformation server. --- tests/test_cloudformation/test_server.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/test_cloudformation/test_server.py b/tests/test_cloudformation/test_server.py index d45c3dc4a..47a094630 100644 --- a/tests/test_cloudformation/test_server.py +++ b/tests/test_cloudformation/test_server.py @@ -1,9 +1,9 @@ from __future__ import unicode_literals import json -import urllib +from six.moves.urllib.parse import urlencode import re -import sure # noqa +import sure # noqa import moto.server as server @@ -11,6 +11,7 @@ import moto.server as server Test the different server responses ''' + def test_cloudformation_server_get(): backend = server.create_backend_app("cloudformation") stack_name = 'test stack' @@ -20,18 +21,19 @@ def test_cloudformation_server_get(): } res = test_client.get( '/?{0}'.format( - urllib.urlencode({ - "Action": "CreateStack", - "StackName": stack_name, - "TemplateBody": json.dumps(template_body) - })), - headers={"Host":"cloudformation.us-east-1.amazonaws.com"} + urlencode({ + "Action": "CreateStack", + "StackName": stack_name, + "TemplateBody": json.dumps(template_body) + }) + ), + headers={"Host": "cloudformation.us-east-1.amazonaws.com"} ) - stack_id = json.loads(res.data)["CreateStackResponse"]["CreateStackResult"]["StackId"] + stack_id = json.loads(res.data.decode("utf-8"))["CreateStackResponse"]["CreateStackResult"]["StackId"] res = test_client.get( '/?Action=ListStacks', - headers={"Host":"cloudformation.us-east-1.amazonaws.com"} + headers={"Host": "cloudformation.us-east-1.amazonaws.com"} ) stacks = re.search("(.*)", res.data.decode('utf-8'))