Merge branch 'master' into batch

CHANGELOG.md

@@ -3,6 +3,13 @@ Moto Changelog
 
 Latest
 ------
+1.1.22
+-----
+
+    * Lambda policies
+    * Dynamodb filter expressions
+    * EC2 Spot fleet improvements
+
 1.1.21
 -----
 

README.md

@@ -94,7 +94,8 @@ It gets even better! Moto isn't just for Python code and it isn't just for S3. L
 |------------------------------------------------------------------------------|
 | IAM                   | @mock_iam        | core endpoints done               |
 |------------------------------------------------------------------------------|
-| Lambda                | @mock_lambda     | basic endpoints done              |
+| Lambda                | @mock_lambda     | basic endpoints done, requires    |
+|                       |                  | docker                            |
 |------------------------------------------------------------------------------|
 | Logs                  | @mock_logs       | basic endpoints done              |
 |------------------------------------------------------------------------------|

moto/dynamodb2/comparisons.py

@@ -1,4 +1,6 @@
 from __future__ import unicode_literals
+import re
+import six
 # TODO add tests for all of these
 
 EQ_FUNCTION = lambda item_value, test_value: item_value == test_value  # flake8: noqa
@@ -39,3 +41,452 @@ COMPARISON_FUNCS = {
 
 def get_comparison_func(range_comparison):
     return COMPARISON_FUNCS.get(range_comparison)
+
+
+#
+def get_filter_expression(expr, names, values):
+    # Examples
+    # expr = 'Id > 5 AND attribute_exists(test) AND Id BETWEEN 5 AND 6 OR length < 6 AND contains(test, 1) AND 5 IN (4,5, 6) OR (Id < 5 AND 5 > Id)'
+    # expr = 'Id > 5 AND Subs < 7'
+
+    # Need to do some dodgyness for NOT i think.
+    if 'NOT' in expr:
+        raise NotImplementedError('NOT not supported yet')
+
+    if names is None:
+        names = {}
+    if values is None:
+        values = {}
+
+    # Do substitutions
+    for key, value in names.items():
+        expr = expr.replace(key, value)
+    for key, value in values.items():
+        if 'N' in value:
+            expr.replace(key, float(value['N']))
+        else:
+            expr = expr.replace(key, value['S'])
+
+    # Remove all spaces, tbf we could just skip them in the next step.
+    # The number of known options is really small so we can do a fair bit of cheating
+    #expr = list(re.sub('\s', '', expr))  # 'Id>5ANDattribute_exists(test)ORNOTlength<6'
+    expr = list(expr)
+
+    # DodgyTokenisation stage 1
+    def is_value(val):
+        return val not in ('<', '>', '=', '(', ')')
+
+    def contains_keyword(val):
+        for kw in ('BETWEEN', 'IN', 'AND', 'OR', 'NOT'):
+            if kw in val:
+                return kw
+        return None
+
+    def is_function(val):
+        return val in ('attribute_exists', 'attribute_not_exists', 'attribute_type', 'begins_with', 'contains', 'size')
+
+    # Does the main part of splitting between sections of characters
+    tokens = []
+    stack = ''
+    while len(expr) > 0:
+        current_char = expr.pop(0)
+
+        if current_char == ' ':
+            if len(stack) > 0:
+                tokens.append(stack)
+            stack = ''
+        elif current_char == ',':  # Split params ,
+            if len(stack) > 0:
+                tokens.append(stack)
+            stack = ''
+        elif is_value(current_char):
+            stack += current_char
+
+            kw = contains_keyword(stack)
+            if kw is not None:
+                # We have a kw in the stack, could be AND or something like 5AND
+                tmp = stack.replace(kw, '')
+                if len(tmp) > 0:
+                    tokens.append(tmp)
+                tokens.append(kw)
+                stack = ''
+        else:
+            if len(stack) > 0:
+                tokens.append(stack)
+            tokens.append(current_char)
+            stack = ''
+    if len(stack) > 0:
+        tokens.append(stack)
+
+    def is_op(val):
+        return val in ('<', '>', '=', '>=', '<=', '<>', 'BETWEEN', 'IN', 'AND', 'OR', 'NOT')
+
+    # DodgyTokenisation stage 2, it groups together some elements to make RPN'ing it later easier.
+    tokens2 = []
+    token_iterator = iter(tokens)
+    for token in token_iterator:
+        if token == '(':
+            tuple_list = []
+
+            next_token = six.next(token_iterator)
+            while next_token != ')':
+                try:
+                    next_token = int(next_token)
+                except ValueError:
+                    try:
+                        next_token = float(next_token)
+                    except ValueError:
+                        pass
+                tuple_list.append(next_token)
+                next_token = six.next(token_iterator)
+
+            # Sigh, we only want to group a tuple if it doesnt contain operators
+            if any([is_op(item) for item in tuple_list]):
+                tokens2.append('(')
+                tokens2.extend(tuple_list)
+                tokens2.append(')')
+            else:
+                tokens2.append(tuple(tuple_list))
+        elif token == 'BETWEEN':
+            field = tokens2.pop()
+            op1 = int(six.next(token_iterator))
+            and_op = six.next(token_iterator)
+            assert and_op == 'AND'
+            op2 = int(six.next(token_iterator))
+            tokens2.append(['between', field, op1, op2])
+
+        elif is_function(token):
+            function_list = [token]
+
+            lbracket = six.next(token_iterator)
+            assert lbracket == '('
+
+            next_token = six.next(token_iterator)
+            while next_token != ')':
+                function_list.append(next_token)
+                next_token = six.next(token_iterator)
+
+            tokens2.append(function_list)
+
+        else:
+            try:
+                token = int(token)
+            except ValueError:
+                try:
+                    token = float(token)
+                except ValueError:
+                    pass
+            tokens2.append(token)
+
+    # Start of the Shunting-Yard algorithm. <-- Proper beast algorithm!
+    def is_number(val):
+        return val not in ('<', '>', '=', '>=', '<=', '<>', 'BETWEEN', 'IN', 'AND', 'OR', 'NOT')
+
+    OPS = {'<': 5, '>': 5, '=': 5, '>=': 5, '<=': 5, '<>': 5, 'IN': 8, 'AND': 11, 'OR': 12, 'NOT': 10, 'BETWEEN': 9, '(': 100, ')': 100}
+
+    def shunting_yard(token_list):
+        output = []
+        op_stack = []
+
+        # Basically takes in an infix notation calculation, converts it to a reverse polish notation where there is no
+        # ambiguity on which order operators are applied.
+        while len(token_list) > 0:
+            token = token_list.pop(0)
+
+            if token == '(':
+                op_stack.append(token)
+            elif token == ')':
+                while len(op_stack) > 0 and op_stack[-1] != '(':
+                    output.append(op_stack.pop())
+                lbracket = op_stack.pop()
+                assert lbracket == '('
+
+            elif is_number(token):
+                output.append(token)
+            else:
+                # Must be operator kw
+                while len(op_stack) > 0 and OPS[op_stack[-1]] <= OPS[token]:
+                    output.append(op_stack.pop())
+                op_stack.append(token)
+        while len(op_stack) > 0:
+            output.append(op_stack.pop())
+
+        return output
+
+    output = shunting_yard(tokens2)
+
+    # Hacky function to convert dynamo functions (which are represented as lists) to their Class equivalent
+    def to_func(val):
+        if isinstance(val, list):
+            func_name = val.pop(0)
+            # Expand rest of the list to arguments
+            val = FUNC_CLASS[func_name](*val)
+
+        return val
+
+    # Simple reverse polish notation execution. Builts up a nested filter object.
+    # The filter object then takes a dynamo item and returns true/false
+    stack = []
+    for token in output:
+        if is_op(token):
+            op2 = stack.pop()
+            op1 = stack.pop()
+
+            op_cls = OP_CLASS[token]
+            stack.append(op_cls(op1, op2))
+        else:
+            stack.append(to_func(token))
+
+    result = stack.pop(0)
+    if len(stack) > 0:
+        raise ValueError('Malformed filter expression')
+
+    return result
+
+
+class Op(object):
+    """
+    Base class for a FilterExpression operator
+    """
+    OP = ''
+
+    def __init__(self, lhs, rhs):
+        self.lhs = lhs
+        self.rhs = rhs
+
+    def _lhs(self, item):
+        """
+        :type item: moto.dynamodb2.models.Item
+        """
+        lhs = self.lhs
+        if isinstance(self.lhs, (Op, Func)):
+            lhs = self.lhs.expr(item)
+        elif isinstance(self.lhs, six.string_types):
+            try:
+                lhs = item.attrs[self.lhs].cast_value
+            except Exception:
+                pass
+
+        return lhs
+
+    def _rhs(self, item):
+        rhs = self.rhs
+        if isinstance(self.rhs, (Op, Func)):
+            rhs = self.rhs.expr(item)
+        elif isinstance(self.rhs, six.string_types):
+            try:
+                rhs = item.attrs[self.rhs].cast_value
+            except Exception:
+                pass
+        return rhs
+
+    def expr(self, item):
+        return True
+
+    def __repr__(self):
+        return '({0} {1} {2})'.format(self.lhs, self.OP, self.rhs)
+
+
+class Func(object):
+    """
+    Base class for a FilterExpression function
+    """
+    FUNC = 'Unknown'
+
+    def expr(self, item):
+        return True
+
+    def __repr__(self):
+        return 'Func(...)'.format(self.FUNC)
+
+
+class OpAnd(Op):
+    OP = 'AND'
+
+    def expr(self, item):
+        lhs = self._lhs(item)
+        rhs = self._rhs(item)
+        return lhs and rhs
+
+
+class OpLessThan(Op):
+    OP = '<'
+
+    def expr(self, item):
+        lhs = self._lhs(item)
+        rhs = self._rhs(item)
+        return lhs < rhs
+
+
+class OpGreaterThan(Op):
+    OP = '>'
+
+    def expr(self, item):
+        lhs = self._lhs(item)
+        rhs = self._rhs(item)
+        return lhs > rhs
+
+
+class OpEqual(Op):
+    OP = '='
+
+    def expr(self, item):
+        lhs = self._lhs(item)
+        rhs = self._rhs(item)
+        return lhs == rhs
+
+
+class OpNotEqual(Op):
+    OP = '<>'
+
+    def expr(self, item):
+        lhs = self._lhs(item)
+        rhs = self._rhs(item)
+        return lhs == rhs
+
+
+class OpLessThanOrEqual(Op):
+    OP = '<='
+
+    def expr(self, item):
+        lhs = self._lhs(item)
+        rhs = self._rhs(item)
+        return lhs <= rhs
+
+
+class OpGreaterThanOrEqual(Op):
+    OP = '>='
+
+    def expr(self, item):
+        lhs = self._lhs(item)
+        rhs = self._rhs(item)
+        return lhs >= rhs
+
+
+class OpOr(Op):
+    OP = 'OR'
+
+    def expr(self, item):
+        lhs = self._lhs(item)
+        rhs = self._rhs(item)
+        return lhs or rhs
+
+
+class OpIn(Op):
+    OP = 'IN'
+
+    def expr(self, item):
+        lhs = self._lhs(item)
+        rhs = self._rhs(item)
+        return lhs in rhs
+
+
+class FuncAttrExists(Func):
+    FUNC = 'attribute_exists'
+
+    def __init__(self, attribute):
+        self.attr = attribute
+
+    def expr(self, item):
+        return self.attr in item.attrs
+
+
+class FuncAttrNotExists(Func):
+    FUNC = 'attribute_not_exists'
+
+    def __init__(self, attribute):
+        self.attr = attribute
+
+    def expr(self, item):
+        return self.attr not in item.attrs
+
+
+class FuncAttrType(Func):
+    FUNC = 'attribute_type'
+
+    def __init__(self, attribute, _type):
+        self.attr = attribute
+        self.type = _type
+
+    def expr(self, item):
+        return self.attr in item.attrs and item.attrs[self.attr].type == self.type
+
+
+class FuncBeginsWith(Func):
+    FUNC = 'begins_with'
+
+    def __init__(self, attribute, substr):
+        self.attr = attribute
+        self.substr = substr
+
+    def expr(self, item):
+        return self.attr in item.attrs and item.attrs[self.attr].type == 'S' and item.attrs[self.attr].value.startswith(self.substr)
+
+
+class FuncContains(Func):
+    FUNC = 'contains'
+
+    def __init__(self, attribute, operand):
+        self.attr = attribute
+        self.operand = operand
+
+    def expr(self, item):
+        if self.attr not in item.attrs:
+            return False
+
+        if item.attrs[self.attr].type in ('S', 'SS', 'NS', 'BS', 'L', 'M'):
+            return self.operand in item.attrs[self.attr].value
+        return False
+
+
+class FuncSize(Func):
+    FUNC = 'contains'
+
+    def __init__(self, attribute):
+        self.attr = attribute
+
+    def expr(self, item):
+        if self.attr not in item.attrs:
+            raise ValueError('Invalid attribute name {0}'.format(self.attr))
+
+        if item.attrs[self.attr].type in ('S', 'SS', 'NS', 'B', 'BS', 'L', 'M'):
+            return len(item.attrs[self.attr].value)
+        raise ValueError('Invalid filter expression')
+
+
+class FuncBetween(Func):
+    FUNC = 'between'
+
+    def __init__(self, attribute, start, end):
+        self.attr = attribute
+        self.start = start
+        self.end = end
+
+    def expr(self, item):
+        if self.attr not in item.attrs:
+            raise ValueError('Invalid attribute name {0}'.format(self.attr))
+
+        return self.start <= item.attrs[self.attr].cast_value <= self.end
+
+
+OP_CLASS = {
+    'AND': OpAnd,
+    'OR': OpOr,
+    'IN': OpIn,
+    '<': OpLessThan,
+    '>': OpGreaterThan,
+    '<=': OpLessThanOrEqual,
+    '>=': OpGreaterThanOrEqual,
+    '=': OpEqual,
+    '<>': OpNotEqual
+}
+
+FUNC_CLASS = {
+    'attribute_exists': FuncAttrExists,
+    'attribute_not_exists': FuncAttrNotExists,
+    'attribute_type': FuncAttrType,
+    'begins_with': FuncBeginsWith,
+    'contains': FuncContains,
+    'size': FuncSize,
+    'between': FuncBetween
+}

moto/dynamodb2/models.py

@@ -8,7 +8,7 @@ import re
 from moto.compat import OrderedDict
 from moto.core import BaseBackend, BaseModel
 from moto.core.utils import unix_time
-from .comparisons import get_comparison_func
+from .comparisons import get_comparison_func, get_filter_expression, Op
 
 
 class DynamoJsonEncoder(json.JSONEncoder):
@@ -508,15 +508,15 @@ class Table(BaseModel):
             else:
                 yield hash_set
 
-    def scan(self, filters, limit, exclusive_start_key):
+    def scan(self, filters, limit, exclusive_start_key, filter_expression=None):
         results = []
         scanned_count = 0
 
-        for result in self.all_items():
+        for item in self.all_items():
             scanned_count += 1
             passes_all_conditions = True
             for attribute_name, (comparison_operator, comparison_objs) in filters.items():
-                attribute = result.attrs.get(attribute_name)
+                attribute = item.attrs.get(attribute_name)
 
                 if attribute:
                     # Attribute found
@@ -532,8 +532,11 @@ class Table(BaseModel):
                     passes_all_conditions = False
                     break
 
+            if filter_expression is not None:
+                passes_all_conditions &= filter_expression.expr(item)
+
             if passes_all_conditions:
-                results.append(result)
+                results.append(item)
 
         results, last_evaluated_key = self._trim_results(results, limit,
                                                          exclusive_start_key)
@@ -698,7 +701,7 @@ class DynamoDBBackend(BaseBackend):
         return table.query(hash_key, range_comparison, range_values, limit,
                            exclusive_start_key, scan_index_forward, projection_expression, index_name, **filter_kwargs)
 
-    def scan(self, table_name, filters, limit, exclusive_start_key):
+    def scan(self, table_name, filters, limit, exclusive_start_key, filter_expression, expr_names, expr_values):
         table = self.tables.get(table_name)
         if not table:
             return None, None, None
@@ -708,7 +711,12 @@ class DynamoDBBackend(BaseBackend):
             dynamo_types = [DynamoType(value) for value in comparison_values]
             scan_filters[key] = (comparison_operator, dynamo_types)
 
-        return table.scan(scan_filters, limit, exclusive_start_key)
+        if filter_expression is not None:
+            filter_expression = get_filter_expression(filter_expression, expr_names, expr_values)
+        else:
+            filter_expression = Op(None, None)  # Will always eval to true
+
+        return table.scan(scan_filters, limit, exclusive_start_key, filter_expression)
 
     def update_item(self, table_name, key, update_expression, attribute_updates, expression_attribute_names,
                     expression_attribute_values, expected=None):

moto/dynamodb2/responses.py

@@ -432,13 +432,29 @@ class DynamoHandler(BaseResponse):
             comparison_values = scan_filter.get("AttributeValueList", [])
             filters[attribute_name] = (comparison_operator, comparison_values)
 
+        filter_expression = self.body.get('FilterExpression')
+        expression_attribute_values = self.body.get('ExpressionAttributeValues', {})
+        expression_attribute_names = self.body.get('ExpressionAttributeNames', {})
+
         exclusive_start_key = self.body.get('ExclusiveStartKey')
         limit = self.body.get("Limit")
 
-        items, scanned_count, last_evaluated_key = dynamodb_backend2.scan(name, filters,
+        try:
-                                                                          limit,
+            items, scanned_count, last_evaluated_key = dynamodb_backend2.scan(name, filters,
-                                                                          exclusive_start_key)
+                                                                              limit,
+                                                                              exclusive_start_key,
+                                                                              filter_expression,
+                                                                              expression_attribute_names,
+                                                                              expression_attribute_values)
+        except ValueError as err:
+            er = 'com.amazonaws.dynamodb.v20111205#ValidationError'
+            return self.error(er, 'Bad Filter Expression: {0}'.format(err))
+        except Exception as err:
+            er = 'com.amazonaws.dynamodb.v20111205#InternalFailure'
+            return self.error(er, 'Internal error. {0}'.format(err))
 
+        # Items should be a list, at least an empty one. Is None if table does not exist.
+        # Should really check this at the beginning
         if items is None:
             er = 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException'
             return self.error(er, 'Requested resource not found')

moto/ec2/models.py

@@ -109,6 +109,7 @@ from .utils import (
     random_vpn_connection_id,
     random_customer_gateway_id,
     is_tag_filter,
+    tag_filter_matches,
 )
 
 RESOURCES_DIR = os.path.join(os.path.dirname(__file__), 'resources')
@@ -374,6 +375,7 @@ class Instance(TaggedEC2Resource, BotoInstance):
         self.source_dest_check = "true"
         self.launch_time = utc_date_and_time()
         self.disable_api_termination = kwargs.get("disable_api_termination", False)
+        self._spot_fleet_id = kwargs.get("spot_fleet_id", None)
         associate_public_ip = kwargs.get("associate_public_ip", False)
         if in_ec2_classic:
             # If we are in EC2-Classic, autoassign a public IP
@@ -511,6 +513,14 @@ class Instance(TaggedEC2Resource, BotoInstance):
 
         self.teardown_defaults()
 
+        if self._spot_fleet_id:
+            spot_fleet = self.ec2_backend.get_spot_fleet_request(self._spot_fleet_id)
+            for spec in spot_fleet.launch_specs:
+                if spec.instance_type == self.instance_type and spec.subnet_id == self.subnet_id:
+                    break
+            spot_fleet.fulfilled_capacity -= spec.weighted_capacity
+            spot_fleet.spot_requests = [req for req in spot_fleet.spot_requests if req.instance != self]
+
         self._state.name = "terminated"
         self._state.code = 48
 
@@ -1300,7 +1310,7 @@ class SecurityGroup(TaggedEC2Resource):
         elif is_tag_filter(key):
             tag_value = self.get_filter_value(key)
             if isinstance(filter_value, list):
-                return any(v in tag_value for v in filter_value)
+                return tag_filter_matches(self, key, filter_value)
             return tag_value in filter_value
         else:
             attr_name = to_attr(key)
@@ -2623,7 +2633,7 @@ class SpotInstanceRequest(BotoSpotRequest, TaggedEC2Resource):
     def __init__(self, ec2_backend, spot_request_id, price, image_id, type,
                  valid_from, valid_until, launch_group, availability_zone_group,
                  key_name, security_groups, user_data, instance_type, placement,
-                 kernel_id, ramdisk_id, monitoring_enabled, subnet_id,
+                 kernel_id, ramdisk_id, monitoring_enabled, subnet_id, spot_fleet_id,
                  **kwargs):
         super(SpotInstanceRequest, self).__init__(**kwargs)
         ls = LaunchSpecification()
@@ -2646,6 +2656,7 @@ class SpotInstanceRequest(BotoSpotRequest, TaggedEC2Resource):
         ls.placement = placement
         ls.monitored = monitoring_enabled
         ls.subnet_id = subnet_id
+        self.spot_fleet_id = spot_fleet_id
 
         if security_groups:
             for group_name in security_groups:
@@ -2678,6 +2689,7 @@ class SpotInstanceRequest(BotoSpotRequest, TaggedEC2Resource):
             key_name=self.launch_specification.key_name,
             security_group_names=[],
             security_group_ids=self.launch_specification.groups,
+            spot_fleet_id=self.spot_fleet_id,
         )
         instance = reservation.instances[0]
         return instance
@@ -2693,7 +2705,7 @@ class SpotRequestBackend(object):
                                valid_until, launch_group, availability_zone_group,
                                key_name, security_groups, user_data,
                                instance_type, placement, kernel_id, ramdisk_id,
-                               monitoring_enabled, subnet_id):
+                               monitoring_enabled, subnet_id, spot_fleet_id=None):
         requests = []
         for _ in range(count):
             spot_request_id = random_spot_request_id()
@@ -2701,7 +2713,7 @@ class SpotRequestBackend(object):
                                           spot_request_id, price, image_id, type, valid_from, valid_until,
                                           launch_group, availability_zone_group, key_name, security_groups,
                                           user_data, instance_type, placement, kernel_id, ramdisk_id,
-                                          monitoring_enabled, subnet_id)
+                                          monitoring_enabled, subnet_id, spot_fleet_id)
             self.spot_instance_requests[spot_request_id] = request
             requests.append(request)
         return requests
@@ -2747,7 +2759,7 @@ class SpotFleetRequest(TaggedEC2Resource):
         self.iam_fleet_role = iam_fleet_role
         self.allocation_strategy = allocation_strategy
         self.state = "active"
-        self.fulfilled_capacity = self.target_capacity
+        self.fulfilled_capacity = 0.0
 
         self.launch_specs = []
         for spec in launch_specs:
@@ -2768,7 +2780,7 @@ class SpotFleetRequest(TaggedEC2Resource):
             )
 
         self.spot_requests = []
-        self.create_spot_requests()
+        self.create_spot_requests(self.target_capacity)
 
     @property
     def physical_resource_id(self):
@@ -2798,31 +2810,32 @@ class SpotFleetRequest(TaggedEC2Resource):
 
         return spot_fleet_request
 
-    def get_launch_spec_counts(self):
+    def get_launch_spec_counts(self, weight_to_add):
         weight_map = defaultdict(int)
 
+        weight_so_far = 0
         if self.allocation_strategy == 'diversified':
-            weight_so_far = 0
             launch_spec_index = 0
             while True:
                 launch_spec = self.launch_specs[
                     launch_spec_index % len(self.launch_specs)]
                 weight_map[launch_spec] += 1
                 weight_so_far += launch_spec.weighted_capacity
-                if weight_so_far >= self.target_capacity:
+                if weight_so_far >= weight_to_add:
                     break
                 launch_spec_index += 1
         else:  # lowestPrice
             cheapest_spec = sorted(
                 self.launch_specs, key=lambda spec: float(spec.spot_price))[0]
-            extra = 1 if self.target_capacity % cheapest_spec.weighted_capacity else 0
+            weight_so_far = weight_to_add + (weight_to_add % cheapest_spec.weighted_capacity)
             weight_map[cheapest_spec] = int(
-                self.target_capacity // cheapest_spec.weighted_capacity) + extra
+                weight_so_far // cheapest_spec.weighted_capacity)
 
-        return weight_map.items()
+        return weight_map, weight_so_far
 
-    def create_spot_requests(self):
+    def create_spot_requests(self, weight_to_add):
-        for launch_spec, count in self.get_launch_spec_counts():
+        weight_map, added_weight = self.get_launch_spec_counts(weight_to_add)
+        for launch_spec, count in weight_map.items():
             requests = self.ec2_backend.request_spot_instances(
                 price=launch_spec.spot_price,
                 image_id=launch_spec.image_id,
@@ -2841,12 +2854,28 @@ class SpotFleetRequest(TaggedEC2Resource):
                 ramdisk_id=None,
                 monitoring_enabled=launch_spec.monitoring,
                 subnet_id=launch_spec.subnet_id,
+                spot_fleet_id=self.id,
             )
             self.spot_requests.extend(requests)
+        self.fulfilled_capacity += added_weight
         return self.spot_requests
 
     def terminate_instances(self):
-        pass
+        instance_ids = []
+        new_fulfilled_capacity = self.fulfilled_capacity
+        for req in self.spot_requests:
+            instance = req.instance
+            for spec in self.launch_specs:
+                if spec.instance_type == instance.instance_type and spec.subnet_id == instance.subnet_id:
+                    break
+
+            if new_fulfilled_capacity - spec.weighted_capacity < self.target_capacity:
+                continue
+            new_fulfilled_capacity -= spec.weighted_capacity
+            instance_ids.append(instance.id)
+
+        self.spot_requests = [req for req in self.spot_requests if req.instance.id not in instance_ids]
+        self.ec2_backend.terminate_instances(instance_ids)
 
 
 class SpotFleetBackend(object):
@@ -2882,12 +2911,26 @@ class SpotFleetBackend(object):
     def cancel_spot_fleet_requests(self, spot_fleet_request_ids, terminate_instances):
         spot_requests = []
         for spot_fleet_request_id in spot_fleet_request_ids:
-            spot_fleet = self.spot_fleet_requests.pop(spot_fleet_request_id)
+            spot_fleet = self.spot_fleet_requests[spot_fleet_request_id]
             if terminate_instances:
+                spot_fleet.target_capacity = 0
                 spot_fleet.terminate_instances()
             spot_requests.append(spot_fleet)
+            del self.spot_fleet_requests[spot_fleet_request_id]
         return spot_requests
 
+    def modify_spot_fleet_request(self, spot_fleet_request_id, target_capacity, terminate_instances):
+        if target_capacity < 0:
+            raise ValueError('Cannot reduce spot fleet capacity below 0')
+        spot_fleet_request = self.spot_fleet_requests[spot_fleet_request_id]
+        delta = target_capacity - spot_fleet_request.fulfilled_capacity
+        spot_fleet_request.target_capacity = target_capacity
+        if delta > 0:
+            spot_fleet_request.create_spot_requests(delta)
+        elif delta < 0 and terminate_instances == 'Default':
+            spot_fleet_request.terminate_instances()
+        return True
+
 
 class ElasticAddress(object):
     def __init__(self, domain):

moto/ec2/responses/spot_fleets.py

@@ -29,6 +29,15 @@ class SpotFleets(BaseResponse):
         template = self.response_template(DESCRIBE_SPOT_FLEET_TEMPLATE)
         return template.render(requests=requests)
 
+    def modify_spot_fleet_request(self):
+        spot_fleet_request_id = self._get_param("SpotFleetRequestId")
+        target_capacity = self._get_int_param("TargetCapacity")
+        terminate_instances = self._get_param("ExcessCapacityTerminationPolicy", if_none="Default")
+        successful = self.ec2_backend.modify_spot_fleet_request(
+            spot_fleet_request_id, target_capacity, terminate_instances)
+        template = self.response_template(MODIFY_SPOT_FLEET_REQUEST_TEMPLATE)
+        return template.render(successful=successful)
+
     def request_spot_fleet(self):
         spot_config = self._get_dict_param("SpotFleetRequestConfig.")
         spot_price = spot_config['spot_price']
@@ -56,6 +65,11 @@ REQUEST_SPOT_FLEET_TEMPLATE = """<RequestSpotFleetResponse xmlns="http://ec2.ama
     <spotFleetRequestId>{{ request.id }}</spotFleetRequestId>
 </RequestSpotFleetResponse>"""
 
+MODIFY_SPOT_FLEET_REQUEST_TEMPLATE = """<ModifySpotFleetRequestResponse xmlns="http://ec2.amazonaws.com/doc/2016-09-15/">
+    <requestId>21681fea-9987-aef3-2121-example</requestId>
+    <return>{{ 'true' if successful else 'false' }}</return>
+</ModifySpotFleetRequestResponse>"""
+
 DESCRIBE_SPOT_FLEET_TEMPLATE = """<DescribeSpotFleetRequestsResponse xmlns="http://ec2.amazonaws.com/doc/2016-09-15/">
     <requestId>4d68a6cc-8f2e-4be1-b425-example</requestId>
     <spotFleetRequestConfigSet>

moto/ec2/utils.py

@@ -51,7 +51,7 @@ def random_ami_id():
 
 
 def random_instance_id():
-    return random_id(prefix=EC2_RESOURCE_TO_PREFIX['instance'])
+    return random_id(prefix=EC2_RESOURCE_TO_PREFIX['instance'], size=17)
 
 
 def random_reservation_id():

moto/iam/responses.py

@@ -1159,9 +1159,7 @@ CREATE_ACCESS_KEY_TEMPLATE = """<CreateAccessKeyResponse>
          <UserName>{{ key.user_name }}</UserName>
          <AccessKeyId>{{ key.access_key_id }}</AccessKeyId>
          <Status>{{ key.status }}</Status>
-         <SecretAccessKey>
+         <SecretAccessKey>{{ key.secret_access_key }}</SecretAccessKey>
-         {{ key.secret_access_key }}
-         </SecretAccessKey>
       </AccessKey>
    </CreateAccessKeyResult>
    <ResponseMetadata>

moto/packages/httpretty/core.py

@@ -103,6 +103,12 @@ try:  # pragma: no cover
 except ImportError:  # pragma: no cover
     ssl = None
 
+try:  # pragma: no cover
+    from requests.packages.urllib3.contrib.pyopenssl import inject_into_urllib3, extract_from_urllib3
+    pyopenssl_override = True
+except:
+    pyopenssl_override = False
+
 
 DEFAULT_HTTP_PORTS = frozenset([80])
 POTENTIAL_HTTP_PORTS = set(DEFAULT_HTTP_PORTS)
@@ -1013,6 +1019,9 @@ class httpretty(HttpBaseClass):
                 ssl.sslwrap_simple = old_sslwrap_simple
                 ssl.__dict__['sslwrap_simple'] = old_sslwrap_simple
 
+        if pyopenssl_override:
+            inject_into_urllib3()
+
     @classmethod
     def is_enabled(cls):
         return cls._is_enabled
@@ -1056,6 +1065,9 @@ class httpretty(HttpBaseClass):
                 ssl.sslwrap_simple = fake_wrap_socket
                 ssl.__dict__['sslwrap_simple'] = fake_wrap_socket
 
+        if pyopenssl_override:
+            extract_from_urllib3()
+
 
 def httprettified(test):
     "A decorator tests that use HTTPretty"

moto/sqs/exceptions.py

@@ -16,3 +16,8 @@ class MessageAttributesInvalid(Exception):
 
     def __init__(self, description):
         self.description = description
+
+
+class QueueDoesNotExist(Exception):
+    status_code = 404
+    description = "The specified queue does not exist for this wsdl version."

moto/sqs/models.py

@@ -12,7 +12,12 @@ import boto.sqs
 from moto.core import BaseBackend, BaseModel
 from moto.core.utils import camelcase_to_underscores, get_random_message_id, unix_time, unix_time_millis
 from .utils import generate_receipt_handle
-from .exceptions import ReceiptHandleIsInvalid, MessageNotInflight, MessageAttributesInvalid
+from .exceptions import (
+    MessageAttributesInvalid,
+    MessageNotInflight,
+    QueueDoesNotExist,
+    ReceiptHandleIsInvalid,
+)
 
 DEFAULT_ACCOUNT_ID = 123456789012
 DEFAULT_SENDER_ID = "AIDAIT2UOQQY3AUEKVGXU"
@@ -304,7 +309,10 @@ class SQSBackend(BaseBackend):
         return qs
 
     def get_queue(self, queue_name):
-        return self.queues.get(queue_name, None)
+        queue = self.queues.get(queue_name)
+        if queue is None:
+            raise QueueDoesNotExist()
+        return queue
 
     def delete_queue(self, queue_name):
         if queue_name in self.queues:

moto/sqs/responses.py

@@ -8,7 +8,8 @@ from .models import sqs_backends
 from .exceptions import (
     MessageAttributesInvalid,
     MessageNotInflight,
-    ReceiptHandleIsInvalid
+    QueueDoesNotExist,
+    ReceiptHandleIsInvalid,
 )
 
 MAXIMUM_VISIBILTY_TIMEOUT = 43200
@@ -76,7 +77,12 @@ class SQSResponse(BaseResponse):
     def get_queue_url(self):
         request_url = urlparse(self.uri)
         queue_name = self._get_param("QueueName")
-        queue = self.sqs_backend.get_queue(queue_name)
+
+        try:
+            queue = self.sqs_backend.get_queue(queue_name)
+        except QueueDoesNotExist as e:
+            return self._error('QueueDoesNotExist', e.description)
+
         if queue:
             template = self.response_template(GET_QUEUE_URL_RESPONSE)
             return template.render(queue=queue, request_url=request_url)
@@ -113,7 +119,11 @@ class SQSResponse(BaseResponse):
 
     def get_queue_attributes(self):
         queue_name = self._get_queue_name()
-        queue = self.sqs_backend.get_queue(queue_name)
+        try:
+            queue = self.sqs_backend.get_queue(queue_name)
+        except QueueDoesNotExist as e:
+            return self._error('QueueDoesNotExist', e.description)
+
         template = self.response_template(GET_QUEUE_ATTRIBUTES_RESPONSE)
         return template.render(queue=queue)
 
@@ -250,7 +260,11 @@ class SQSResponse(BaseResponse):
 
     def receive_message(self):
         queue_name = self._get_queue_name()
-        queue = self.sqs_backend.get_queue(queue_name)
+
+        try:
+            queue = self.sqs_backend.get_queue(queue_name)
+        except QueueDoesNotExist as e:
+            return self._error('QueueDoesNotExist', e.description)
 
         try:
             message_count = int(self.querystring.get("MaxNumberOfMessages")[0])

setup.py

@@ -36,7 +36,7 @@ else:
 
 setup(
     name='moto',
-    version='1.1.21',
+    version='1.1.22',
     description='A library that allows your python tests to easily'
                 ' mock out the boto library',
     author='Steve Pulec',

tests/test_dynamodb2/test_dynamodb.py

@@ -3,6 +3,7 @@ from __future__ import unicode_literals, print_function
 import six
 import boto
 import boto3
+from boto3.dynamodb.conditions import Attr
 import sure  # noqa
 import requests
 from moto import mock_dynamodb2, mock_dynamodb2_deprecated
@@ -12,6 +13,10 @@ from botocore.exceptions import ClientError
 from boto3.dynamodb.conditions import Key
 from tests.helpers import requires_boto_gte
 import tests.backport_assert_raises
+
+import moto.dynamodb2.comparisons
+import moto.dynamodb2.models
+
 from nose.tools import assert_raises
 try:
     import boto.dynamodb2
@@ -230,6 +235,7 @@ def test_scan_returns_consumed_capacity():
     assert 'CapacityUnits' in response['ConsumedCapacity']
     assert response['ConsumedCapacity']['TableName'] == name
 
+
 @requires_boto_gte("2.9")
 @mock_dynamodb2
 def test_query_returns_consumed_capacity():
@@ -280,6 +286,7 @@ def test_query_returns_consumed_capacity():
     assert 'CapacityUnits' in results['ConsumedCapacity']
     assert results['ConsumedCapacity']['CapacityUnits'] == 1
 
+
 @mock_dynamodb2
 def test_basic_projection_expressions():
     dynamodb = boto3.resource('dynamodb', region_name='us-east-1')
@@ -353,6 +360,7 @@ def test_basic_projection_expressions():
     assert 'body' in results['Items'][1]
     assert results['Items'][1]['body'] == 'yet another test message'
 
+
 @mock_dynamodb2
 def test_basic_projection_expressions_with_attr_expression_names():
     dynamodb = boto3.resource('dynamodb', region_name='us-east-1')
@@ -419,6 +427,7 @@ def test_basic_projection_expressions_with_attr_expression_names():
     assert 'attachment' in results['Items'][0]
     assert results['Items'][0]['attachment'] == 'something'
 
+
 @mock_dynamodb2
 def test_put_item_returns_consumed_capacity():
     dynamodb = boto3.resource('dynamodb', region_name='us-east-1')
@@ -461,6 +470,7 @@ def test_put_item_returns_consumed_capacity():
 
     assert 'ConsumedCapacity' in response
 
+
 @mock_dynamodb2
 def test_update_item_returns_consumed_capacity():
     dynamodb = boto3.resource('dynamodb', region_name='us-east-1')
@@ -514,6 +524,7 @@ def test_update_item_returns_consumed_capacity():
     assert 'CapacityUnits' in response['ConsumedCapacity']
     assert 'TableName' in response['ConsumedCapacity']
 
+
 @mock_dynamodb2
 def test_get_item_returns_consumed_capacity():
     dynamodb = boto3.resource('dynamodb', region_name='us-east-1')
@@ -562,3 +573,206 @@ def test_get_item_returns_consumed_capacity():
     assert 'ConsumedCapacity' in response
     assert 'CapacityUnits' in response['ConsumedCapacity']
     assert 'TableName' in response['ConsumedCapacity']
+
+
+def test_filter_expression():
+    # TODO NOT not yet supported
+    row1 = moto.dynamodb2.models.Item(None, None, None, None, {'Id': {'N': '8'}, 'Subs': {'N': '5'}, 'Desc': {'S': 'Some description'}, 'KV': {'SS': ['test1', 'test2']}})
+    row2 = moto.dynamodb2.models.Item(None, None, None, None, {'Id': {'N': '8'}, 'Subs': {'N': '10'}, 'Desc': {'S': 'A description'}, 'KV': {'SS': ['test3', 'test4']}})
+
+    # AND test
+    filter_expr = moto.dynamodb2.comparisons.get_filter_expression('Id > 5 AND Subs < 7', {}, {})
+    filter_expr.expr(row1).should.be(True)
+    filter_expr.expr(row2).should.be(False)
+
+    # OR test
+    filter_expr = moto.dynamodb2.comparisons.get_filter_expression('Id = 5 OR Id=8', {}, {})
+    filter_expr.expr(row1).should.be(True)
+
+    # BETWEEN test
+    filter_expr = moto.dynamodb2.comparisons.get_filter_expression('Id BETWEEN 5 AND 10', {}, {})
+    filter_expr.expr(row1).should.be(True)
+
+    # PAREN test
+    filter_expr = moto.dynamodb2.comparisons.get_filter_expression('Id = 8 AND (Subs = 8 OR Subs = 5)', {}, {})
+    filter_expr.expr(row1).should.be(True)
+
+    # IN test
+    filter_expr = moto.dynamodb2.comparisons.get_filter_expression('Id IN (7,8, 9)', {}, {})
+    filter_expr.expr(row1).should.be(True)
+
+    # attribute function tests
+    filter_expr = moto.dynamodb2.comparisons.get_filter_expression('attribute_exists(Id) AND attribute_not_exists(User)', {}, {})
+    filter_expr.expr(row1).should.be(True)
+
+    filter_expr = moto.dynamodb2.comparisons.get_filter_expression('attribute_type(Id, N)', {}, {})
+    filter_expr.expr(row1).should.be(True)
+
+    # beginswith function test
+    filter_expr = moto.dynamodb2.comparisons.get_filter_expression('begins_with(Desc, Some)', {}, {})
+    filter_expr.expr(row1).should.be(True)
+    filter_expr.expr(row2).should.be(False)
+
+    # contains function test
+    filter_expr = moto.dynamodb2.comparisons.get_filter_expression('contains(KV, test1)', {}, {})
+    filter_expr.expr(row1).should.be(True)
+    filter_expr.expr(row2).should.be(False)
+
+    # size function test
+    filter_expr = moto.dynamodb2.comparisons.get_filter_expression('size(Desc) > size(KV)', {}, {})
+    filter_expr.expr(row1).should.be(True)
+
+
+@mock_dynamodb2
+def test_scan_filter():
+    client = boto3.client('dynamodb', region_name='us-east-1')
+    dynamodb = boto3.resource('dynamodb', region_name='us-east-1')
+
+    # Create the DynamoDB table.
+    client.create_table(
+        TableName='test1',
+        AttributeDefinitions=[{'AttributeName': 'client', 'AttributeType': 'S'}, {'AttributeName': 'app', 'AttributeType': 'S'}],
+        KeySchema=[{'AttributeName': 'client', 'KeyType': 'HASH'}, {'AttributeName': 'app', 'KeyType': 'RANGE'}],
+        ProvisionedThroughput={'ReadCapacityUnits': 123, 'WriteCapacityUnits': 123}
+    )
+    client.put_item(
+        TableName='test1',
+        Item={
+            'client': {'S': 'client1'},
+            'app': {'S': 'app1'}
+        }
+    )
+
+    table = dynamodb.Table('test1')
+    response = table.scan(
+        FilterExpression=Attr('app').eq('app2')
+    )
+    assert response['Count'] == 0
+
+    response = table.scan(
+        FilterExpression=Attr('app').eq('app1')
+    )
+    assert response['Count'] == 1
+
+
+@mock_dynamodb2
+def test_bad_scan_filter():
+    client = boto3.client('dynamodb', region_name='us-east-1')
+    dynamodb = boto3.resource('dynamodb', region_name='us-east-1')
+
+    # Create the DynamoDB table.
+    client.create_table(
+        TableName='test1',
+        AttributeDefinitions=[{'AttributeName': 'client', 'AttributeType': 'S'}, {'AttributeName': 'app', 'AttributeType': 'S'}],
+        KeySchema=[{'AttributeName': 'client', 'KeyType': 'HASH'}, {'AttributeName': 'app', 'KeyType': 'RANGE'}],
+        ProvisionedThroughput={'ReadCapacityUnits': 123, 'WriteCapacityUnits': 123}
+    )
+    table = dynamodb.Table('test1')
+
+    # Bad expression
+    try:
+        table.scan(
+            FilterExpression='client test'
+        )
+    except ClientError as err:
+        err.response['Error']['Code'].should.equal('ValidationError')
+    else:
+        raise RuntimeError('Should of raised ResourceInUseException')
+
+
+
+@mock_dynamodb2
+def test_duplicate_create():
+    client = boto3.client('dynamodb', region_name='us-east-1')
+
+    # Create the DynamoDB table.
+    client.create_table(
+        TableName='test1',
+        AttributeDefinitions=[{'AttributeName': 'client', 'AttributeType': 'S'}, {'AttributeName': 'app', 'AttributeType': 'S'}],
+        KeySchema=[{'AttributeName': 'client', 'KeyType': 'HASH'}, {'AttributeName': 'app', 'KeyType': 'RANGE'}],
+        ProvisionedThroughput={'ReadCapacityUnits': 123, 'WriteCapacityUnits': 123}
+    )
+
+    try:
+        client.create_table(
+            TableName='test1',
+            AttributeDefinitions=[{'AttributeName': 'client', 'AttributeType': 'S'}, {'AttributeName': 'app', 'AttributeType': 'S'}],
+            KeySchema=[{'AttributeName': 'client', 'KeyType': 'HASH'}, {'AttributeName': 'app', 'KeyType': 'RANGE'}],
+            ProvisionedThroughput={'ReadCapacityUnits': 123, 'WriteCapacityUnits': 123}
+        )
+    except ClientError as err:
+        err.response['Error']['Code'].should.equal('ResourceInUseException')
+    else:
+        raise RuntimeError('Should of raised ResourceInUseException')
+
+
+@mock_dynamodb2
+def test_delete_table():
+    client = boto3.client('dynamodb', region_name='us-east-1')
+
+    # Create the DynamoDB table.
+    client.create_table(
+        TableName='test1',
+        AttributeDefinitions=[{'AttributeName': 'client', 'AttributeType': 'S'}, {'AttributeName': 'app', 'AttributeType': 'S'}],
+        KeySchema=[{'AttributeName': 'client', 'KeyType': 'HASH'}, {'AttributeName': 'app', 'KeyType': 'RANGE'}],
+        ProvisionedThroughput={'ReadCapacityUnits': 123, 'WriteCapacityUnits': 123}
+    )
+
+    client.delete_table(TableName='test1')
+
+    resp = client.list_tables()
+    len(resp['TableNames']).should.equal(0)
+
+    try:
+        client.delete_table(TableName='test1')
+    except ClientError as err:
+        err.response['Error']['Code'].should.equal('ResourceNotFoundException')
+    else:
+        raise RuntimeError('Should of raised ResourceNotFoundException')
+
+
+@mock_dynamodb2
+def test_delete_item():
+    client = boto3.client('dynamodb', region_name='us-east-1')
+    dynamodb = boto3.resource('dynamodb', region_name='us-east-1')
+
+    # Create the DynamoDB table.
+    client.create_table(
+        TableName='test1',
+        AttributeDefinitions=[{'AttributeName': 'client', 'AttributeType': 'S'}, {'AttributeName': 'app', 'AttributeType': 'S'}],
+        KeySchema=[{'AttributeName': 'client', 'KeyType': 'HASH'}, {'AttributeName': 'app', 'KeyType': 'RANGE'}],
+        ProvisionedThroughput={'ReadCapacityUnits': 123, 'WriteCapacityUnits': 123}
+    )
+    client.put_item(
+        TableName='test1',
+        Item={
+            'client': {'S': 'client1'},
+            'app': {'S': 'app1'}
+        }
+    )
+    client.put_item(
+        TableName='test1',
+        Item={
+            'client': {'S': 'client1'},
+            'app': {'S': 'app2'}
+        }
+    )
+
+    table = dynamodb.Table('test1')
+    response = table.scan()
+    assert response['Count'] == 2
+
+    # Test deletion and returning old value
+    response = table.delete_item(Key={'client': 'client1', 'app': 'app1'}, ReturnValues='ALL_OLD')
+    response['Attributes'].should.contain('client')
+    response['Attributes'].should.contain('app')
+
+    response = table.scan()
+    assert response['Count'] == 1
+
+    # Test deletion returning nothing
+    response = table.delete_item(Key={'client': 'client1', 'app': 'app2'})
+    len(response['Attributes']).should.equal(0)
+
+    response = table.scan()
+    assert response['Count'] == 0

tests/test_ec2/test_security_groups.py

@@ -613,6 +613,20 @@ def test_security_group_tagging_boto3():
     tag['Key'].should.equal("Test")
 
 
+@mock_ec2
+def test_security_group_wildcard_tag_filter_boto3():
+    conn = boto3.client('ec2', region_name='us-east-1')
+    sg = conn.create_security_group(GroupName="test-sg", Description="Test SG")
+    conn.create_tags(Resources=[sg['GroupId']], Tags=[
+                     {'Key': 'Test', 'Value': 'Tag'}])
+    describe = conn.describe_security_groups(
+        Filters=[{'Name': 'tag-value', 'Values': ['*']}])
+
+    tag = describe["SecurityGroups"][0]['Tags'][0]
+    tag['Value'].should.equal("Tag")
+    tag['Key'].should.equal("Test")
+
+
 @mock_ec2
 def test_authorize_and_revoke_in_bulk():
     ec2 = boto3.resource('ec2', region_name='us-west-1')

tests/test_ec2/test_spot_fleet.py

@@ -164,3 +164,155 @@ def test_cancel_spot_fleet_request():
     spot_fleet_requests = conn.describe_spot_fleet_requests(
         SpotFleetRequestIds=[spot_fleet_id])['SpotFleetRequestConfigs']
     len(spot_fleet_requests).should.equal(0)
+
+
+@mock_ec2
+def test_modify_spot_fleet_request_up():
+    conn = boto3.client("ec2", region_name='us-west-2')
+    subnet_id = get_subnet_id(conn)
+
+    spot_fleet_res = conn.request_spot_fleet(
+        SpotFleetRequestConfig=spot_config(subnet_id),
+    )
+    spot_fleet_id = spot_fleet_res['SpotFleetRequestId']
+
+    conn.modify_spot_fleet_request(
+        SpotFleetRequestId=spot_fleet_id, TargetCapacity=20)
+
+    instance_res = conn.describe_spot_fleet_instances(
+        SpotFleetRequestId=spot_fleet_id)
+    instances = instance_res['ActiveInstances']
+    len(instances).should.equal(10)
+
+    spot_fleet_config = conn.describe_spot_fleet_requests(
+        SpotFleetRequestIds=[spot_fleet_id])['SpotFleetRequestConfigs'][0]['SpotFleetRequestConfig']
+    spot_fleet_config['TargetCapacity'].should.equal(20)
+    spot_fleet_config['FulfilledCapacity'].should.equal(20.0)
+
+
+@mock_ec2
+def test_modify_spot_fleet_request_up_diversified():
+    conn = boto3.client("ec2", region_name='us-west-2')
+    subnet_id = get_subnet_id(conn)
+
+    spot_fleet_res = conn.request_spot_fleet(
+        SpotFleetRequestConfig=spot_config(
+            subnet_id, allocation_strategy='diversified'),
+    )
+    spot_fleet_id = spot_fleet_res['SpotFleetRequestId']
+
+    conn.modify_spot_fleet_request(
+        SpotFleetRequestId=spot_fleet_id, TargetCapacity=19)
+
+    instance_res = conn.describe_spot_fleet_instances(
+        SpotFleetRequestId=spot_fleet_id)
+    instances = instance_res['ActiveInstances']
+    len(instances).should.equal(7)
+
+    spot_fleet_config = conn.describe_spot_fleet_requests(
+        SpotFleetRequestIds=[spot_fleet_id])['SpotFleetRequestConfigs'][0]['SpotFleetRequestConfig']
+    spot_fleet_config['TargetCapacity'].should.equal(19)
+    spot_fleet_config['FulfilledCapacity'].should.equal(20.0)
+
+
+@mock_ec2
+def test_modify_spot_fleet_request_down_no_terminate():
+    conn = boto3.client("ec2", region_name='us-west-2')
+    subnet_id = get_subnet_id(conn)
+
+    spot_fleet_res = conn.request_spot_fleet(
+        SpotFleetRequestConfig=spot_config(subnet_id),
+    )
+    spot_fleet_id = spot_fleet_res['SpotFleetRequestId']
+
+    conn.modify_spot_fleet_request(
+        SpotFleetRequestId=spot_fleet_id, TargetCapacity=1, ExcessCapacityTerminationPolicy="noTermination")
+
+    instance_res = conn.describe_spot_fleet_instances(
+        SpotFleetRequestId=spot_fleet_id)
+    instances = instance_res['ActiveInstances']
+    len(instances).should.equal(3)
+
+    spot_fleet_config = conn.describe_spot_fleet_requests(
+        SpotFleetRequestIds=[spot_fleet_id])['SpotFleetRequestConfigs'][0]['SpotFleetRequestConfig']
+    spot_fleet_config['TargetCapacity'].should.equal(1)
+    spot_fleet_config['FulfilledCapacity'].should.equal(6.0)
+
+
+@mock_ec2
+def test_modify_spot_fleet_request_down_odd():
+    conn = boto3.client("ec2", region_name='us-west-2')
+    subnet_id = get_subnet_id(conn)
+
+    spot_fleet_res = conn.request_spot_fleet(
+        SpotFleetRequestConfig=spot_config(subnet_id),
+    )
+    spot_fleet_id = spot_fleet_res['SpotFleetRequestId']
+
+    conn.modify_spot_fleet_request(
+        SpotFleetRequestId=spot_fleet_id, TargetCapacity=7)
+    conn.modify_spot_fleet_request(
+        SpotFleetRequestId=spot_fleet_id, TargetCapacity=5)
+
+    instance_res = conn.describe_spot_fleet_instances(
+        SpotFleetRequestId=spot_fleet_id)
+    instances = instance_res['ActiveInstances']
+    len(instances).should.equal(3)
+
+    spot_fleet_config = conn.describe_spot_fleet_requests(
+        SpotFleetRequestIds=[spot_fleet_id])['SpotFleetRequestConfigs'][0]['SpotFleetRequestConfig']
+    spot_fleet_config['TargetCapacity'].should.equal(5)
+    spot_fleet_config['FulfilledCapacity'].should.equal(6.0)
+
+
+@mock_ec2
+def test_modify_spot_fleet_request_down():
+    conn = boto3.client("ec2", region_name='us-west-2')
+    subnet_id = get_subnet_id(conn)
+
+    spot_fleet_res = conn.request_spot_fleet(
+        SpotFleetRequestConfig=spot_config(subnet_id),
+    )
+    spot_fleet_id = spot_fleet_res['SpotFleetRequestId']
+
+    conn.modify_spot_fleet_request(
+        SpotFleetRequestId=spot_fleet_id, TargetCapacity=1)
+
+    instance_res = conn.describe_spot_fleet_instances(
+        SpotFleetRequestId=spot_fleet_id)
+    instances = instance_res['ActiveInstances']
+    len(instances).should.equal(1)
+
+    spot_fleet_config = conn.describe_spot_fleet_requests(
+        SpotFleetRequestIds=[spot_fleet_id])['SpotFleetRequestConfigs'][0]['SpotFleetRequestConfig']
+    spot_fleet_config['TargetCapacity'].should.equal(1)
+    spot_fleet_config['FulfilledCapacity'].should.equal(2.0)
+
+
+@mock_ec2
+def test_modify_spot_fleet_request_down_no_terminate_after_custom_terminate():
+    conn = boto3.client("ec2", region_name='us-west-2')
+    subnet_id = get_subnet_id(conn)
+
+    spot_fleet_res = conn.request_spot_fleet(
+        SpotFleetRequestConfig=spot_config(subnet_id),
+    )
+    spot_fleet_id = spot_fleet_res['SpotFleetRequestId']
+
+    instance_res = conn.describe_spot_fleet_instances(
+        SpotFleetRequestId=spot_fleet_id)
+    instances = instance_res['ActiveInstances']
+    conn.terminate_instances(InstanceIds=[i['InstanceId'] for i in instances[1:]])
+
+    conn.modify_spot_fleet_request(
+        SpotFleetRequestId=spot_fleet_id, TargetCapacity=1, ExcessCapacityTerminationPolicy="noTermination")
+
+    instance_res = conn.describe_spot_fleet_instances(
+        SpotFleetRequestId=spot_fleet_id)
+    instances = instance_res['ActiveInstances']
+    len(instances).should.equal(1)
+
+    spot_fleet_config = conn.describe_spot_fleet_requests(
+        SpotFleetRequestIds=[spot_fleet_id])['SpotFleetRequestConfigs'][0]['SpotFleetRequestConfig']
+    spot_fleet_config['TargetCapacity'].should.equal(1)
+    spot_fleet_config['FulfilledCapacity'].should.equal(2.0)

tests/test_sqs/test_sqs.py

@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import boto
 import boto3
 import botocore.exceptions
+from botocore.exceptions import ClientError
 from boto.exception import SQSError
 from boto.sqs.message import RawMessage, Message
 
@@ -33,6 +34,7 @@ def test_create_fifo_queue_fail():
     else:
         raise RuntimeError('Should of raised InvalidParameterValue Exception')
 
+
 @mock_sqs
 def test_create_fifo_queue():
     sqs = boto3.client('sqs', region_name='us-east-1')
@@ -49,10 +51,10 @@ def test_create_fifo_queue():
     response['Attributes']['FifoQueue'].should.equal('true')
 
 
-
 @mock_sqs
 def test_create_queue():
     sqs = boto3.resource('sqs', region_name='us-east-1')
+
     new_queue = sqs.create_queue(QueueName='test-queue')
     new_queue.should_not.be.none
     new_queue.should.have.property('url').should.contain('test-queue')
@@ -66,10 +68,19 @@ def test_create_queue():
 
 
 @mock_sqs
-def test_get_inexistent_queue():
+def test_get_nonexistent_queue():
     sqs = boto3.resource('sqs', region_name='us-east-1')
-    sqs.get_queue_by_name.when.called_with(
+    with assert_raises(ClientError) as err:
-        QueueName='nonexisting-queue').should.throw(botocore.exceptions.ClientError)
+        sqs.get_queue_by_name(QueueName='nonexisting-queue')
+    ex = err.exception
+    ex.operation_name.should.equal('GetQueueUrl')
+    ex.response['Error']['Code'].should.equal('QueueDoesNotExist')
+
+    with assert_raises(ClientError) as err:
+        sqs.Queue('http://whatever-incorrect-queue-address').load()
+    ex = err.exception
+    ex.operation_name.should.equal('GetQueueAttributes')
+    ex.response['Error']['Code'].should.equal('QueueDoesNotExist')
 
 
 @mock_sqs