Merge remote-tracking branch 'origin/master' into ImproveTemplatePerformance

Conflicts:
	moto/s3/responses.py

moto/ec2/models.py

@@ -1817,12 +1817,12 @@ class RouteTableBackend(object):
 
 class Route(object):
     def __init__(self, route_table, destination_cidr_block, local=False,
-                 internet_gateway=None, instance=None, interface=None, vpc_pcx=None):
+                 gateway=None, instance=None, interface=None, vpc_pcx=None):
         self.id = generate_route_id(route_table.id, destination_cidr_block)
         self.route_table = route_table
         self.destination_cidr_block = destination_cidr_block
         self.local = local
-        self.internet_gateway = internet_gateway
+        self.gateway = gateway
         self.instance = instance
         self.interface = interface
         self.vpc_pcx = vpc_pcx
@@ -1861,8 +1861,15 @@ class RouteBackend(object):
         if interface_id:
             self.raise_not_implemented_error("CreateRoute to NetworkInterfaceId")
 
+        gateway = None
+        if gateway_id:
+            if EC2_RESOURCE_TO_PREFIX['vpn-gateway'] in gateway_id:
+                gateway = self.get_vpn_gateway(gateway_id)
+            elif EC2_RESOURCE_TO_PREFIX['internet-gateway'] in gateway_id:
+                gateway = self.get_internet_gateway(gateway_id)
+
         route = Route(route_table, destination_cidr_block, local=local,
-                      internet_gateway=self.get_internet_gateway(gateway_id) if gateway_id else None,
+                      gateway=gateway,
                       instance=self.get_instance(instance_id) if instance_id else None,
                       interface=None,
                       vpc_pcx=self.get_vpc_peering_connection(vpc_peering_connection_id) if vpc_peering_connection_id else None)
@@ -1879,7 +1886,13 @@ class RouteBackend(object):
         if interface_id:
             self.raise_not_implemented_error("ReplaceRoute to NetworkInterfaceId")
 
-        route.internet_gateway = self.get_internet_gateway(gateway_id) if gateway_id else None
+        route.gateway = None
+        if gateway_id:
+            if EC2_RESOURCE_TO_PREFIX['vpn-gateway'] in gateway_id:
+                route.gateway = self.get_vpn_gateway(gateway_id)
+            elif EC2_RESOURCE_TO_PREFIX['internet-gateway'] in gateway_id:
+                route.gateway = self.get_internet_gateway(gateway_id)
+
         route.instance = self.get_instance(instance_id) if instance_id else None
         route.interface = None
         route.vpc_pcx = self.get_vpc_peering_connection(vpc_peering_connection_id) if vpc_peering_connection_id else None

moto/ec2/responses/route_tables.py

@@ -16,13 +16,13 @@ class RouteTables(BaseResponse):
         route_table_id = self.querystring.get('RouteTableId')[0]
         destination_cidr_block = self.querystring.get('DestinationCidrBlock')[0]
 
-        internet_gateway_id = optional_from_querystring('GatewayId', self.querystring)
+        gateway_id = optional_from_querystring('GatewayId', self.querystring)
         instance_id = optional_from_querystring('InstanceId', self.querystring)
         interface_id = optional_from_querystring('NetworkInterfaceId', self.querystring)
         pcx_id = optional_from_querystring('VpcPeeringConnectionId', self.querystring)
 
         self.ec2_backend.create_route(route_table_id, destination_cidr_block,
-            gateway_id=internet_gateway_id,
+            gateway_id=gateway_id,
             instance_id=instance_id,
             interface_id=interface_id,
             vpc_peering_connection_id=pcx_id)
@@ -66,13 +66,13 @@ class RouteTables(BaseResponse):
         route_table_id = self.querystring.get('RouteTableId')[0]
         destination_cidr_block = self.querystring.get('DestinationCidrBlock')[0]
 
-        internet_gateway_id = optional_from_querystring('GatewayId', self.querystring)
+        gateway_id = optional_from_querystring('GatewayId', self.querystring)
         instance_id = optional_from_querystring('InstanceId', self.querystring)
         interface_id = optional_from_querystring('NetworkInterfaceId', self.querystring)
         pcx_id = optional_from_querystring('VpcPeeringConnectionId', self.querystring)
 
         self.ec2_backend.replace_route(route_table_id, destination_cidr_block,
-            gateway_id=internet_gateway_id,
+            gateway_id=gateway_id,
             instance_id=instance_id,
             interface_id=interface_id,
             vpc_peering_connection_id=pcx_id)
@@ -151,8 +151,8 @@ DESCRIBE_ROUTE_TABLES_RESPONSE = """
                   <origin>CreateRouteTable</origin>
                   <state>active</state>
                 {% endif %}
-                {% if route.internet_gateway %}
+                {% if route.gateway %}
-                  <gatewayId>{{ route.internet_gateway.id }}</gatewayId>
+                  <gatewayId>{{ route.gateway.id }}</gatewayId>
                   <origin>CreateRoute</origin>
                   <state>active</state>
                 {% endif %}

moto/ec2/utils.py

@@ -324,16 +324,20 @@ filter_dict_attribute_mapping = {
     'state-reason-code': '_state_reason.code',
     'source-dest-check': 'source_dest_check',
     'vpc-id': 'vpc_id',
+    'group-id': 'security_groups',
+    'instance.group-id': 'security_groups'
 }
 
 
 def passes_filter_dict(instance, filter_dict):
     for filter_name, filter_values in filter_dict.items():
+          
         if filter_name in filter_dict_attribute_mapping:
             instance_attr = filter_dict_attribute_mapping[filter_name]
             instance_value = get_object_value(instance, instance_attr)
-            if instance_value not in filter_values:
+            if not instance_value_in_filter_values(instance_value, filter_values):
                 return False
+
         elif is_tag_filter(filter_name):
             if not tag_filter_matches(instance, filter_name, filter_values):
                 return False
@@ -343,6 +347,13 @@ def passes_filter_dict(instance, filter_dict):
                 filter_name)
     return True
 
+def instance_value_in_filter_values(instance_value, filter_values):
+    if isinstance(instance_value, list):
+        if not set(filter_values).intersection(set(instance_value)):
+            return False
+    elif instance_value not in filter_values: 
+            return False
+    return True
 
 def filter_reservations(reservations, filter_dict):
     result = []

moto/s3/models.py

@@ -160,12 +160,17 @@ class FakeMultipart(object):
 
 class FakeBucket(object):
 
-    def __init__(self, name):
+    def __init__(self, name, region_name):
         self.name = name
+        self.region_name = region_name
         self.keys = _VersionedKeyStore()
         self.multiparts = {}
         self.versioning_status = None
 
+    @property
+    def location(self):
+        return self.region_name
+
     @property
     def is_versioned(self):
         return self.versioning_status == 'Enabled'
@@ -184,10 +189,10 @@ class S3Backend(BaseBackend):
     def __init__(self):
         self.buckets = {}
 
-    def create_bucket(self, bucket_name):
+    def create_bucket(self, bucket_name, region_name):
         if bucket_name in self.buckets:
             raise BucketAlreadyExists()
-        new_bucket = FakeBucket(name=bucket_name)
+        new_bucket = FakeBucket(name=bucket_name, region_name=region_name)
         self.buckets[bucket_name] = new_bucket
         return new_bucket
 

moto/s3/responses.py

@@ -12,6 +12,9 @@ from .models import s3_backend
 from .utils import bucket_name_from_url, metadata_from_headers
 from xml.dom import minidom
 
+REGION_URL_REGEX = r'\.s3-(.+?)\.amazonaws\.com'
+DEFAULT_REGION_NAME = 'us-east-1'
+
 
 def parse_key_name(pth):
     return pth.lstrip("/")
@@ -45,6 +48,10 @@ class ResponseObject(_TemplateEnvironmentMixin):
         parsed_url = urlparse(full_url)
         querystring = parse_qs(parsed_url.query, keep_blank_values=True)
         method = request.method
+        region_name = DEFAULT_REGION_NAME
+        region_match = re.search(REGION_URL_REGEX, full_url)
+        if region_match:
+            region_name = region_match.groups()[0]
 
         bucket_name = self.bucket_name_from_url(full_url)
         if not bucket_name:
@@ -56,7 +63,7 @@ class ResponseObject(_TemplateEnvironmentMixin):
         elif method == 'GET':
             return self._bucket_response_get(bucket_name, querystring, headers)
         elif method == 'PUT':
-            return self._bucket_response_put(request, bucket_name, querystring, headers)
+            return self._bucket_response_put(request, region_name, bucket_name, querystring, headers)
         elif method == 'DELETE':
             return self._bucket_response_delete(bucket_name, headers)
         elif method == 'POST':
@@ -82,6 +89,10 @@ class ResponseObject(_TemplateEnvironmentMixin):
             return 200, headers, template.render(
                 bucket_name=bucket_name,
                 uploads=multiparts)
+        elif 'location' in querystring:
+            bucket = self.backend.get_bucket(bucket_name)
+            template = Template(S3_BUCKET_LOCATION)
+            return 200, headers, template.render(location=bucket.location)
         elif 'versioning' in querystring:
             versioning = self.backend.get_bucket_versioning(bucket_name)
             template = self.response_template(S3_BUCKET_GET_VERSIONING)
@@ -130,7 +141,7 @@ class ResponseObject(_TemplateEnvironmentMixin):
             result_folders=result_folders
         )
 
-    def _bucket_response_put(self, request, bucket_name, querystring, headers):
+    def _bucket_response_put(self, request, region_name, bucket_name, querystring, headers):
         if 'versioning' in querystring:
             ver = re.search('<Status>([A-Za-z]+)</Status>', request.body.decode('utf-8'))
             if ver:
@@ -141,9 +152,13 @@ class ResponseObject(_TemplateEnvironmentMixin):
                 return 404, headers, ""
         else:
             try:
-                new_bucket = self.backend.create_bucket(bucket_name)
+                new_bucket = self.backend.create_bucket(bucket_name, region_name)
             except BucketAlreadyExists:
-                return 409, headers, ""
+                if region_name == DEFAULT_REGION_NAME:
+                    # us-east-1 has different behavior
+                    new_bucket = self.backend.get_bucket(bucket_name)
+                else:
+                    return 409, headers, ""
             template = self.response_template(S3_BUCKET_CREATE_RESPONSE)
             return 200, headers, template.render(bucket=new_bucket)
 
@@ -459,6 +474,9 @@ S3_DELETE_BUCKET_WITH_ITEMS_ERROR = """<?xml version="1.0" encoding="UTF-8"?>
 <HostId>sdfgdsfgdsfgdfsdsfgdfs</HostId>
 </Error>"""
 
+S3_BUCKET_LOCATION = """<?xml version="1.0" encoding="UTF-8"?>
+<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">{{ location }}</LocationConstraint>"""
+
 S3_BUCKET_VERSIONING = """
 <?xml version="1.0" encoding="UTF-8"?>
 <VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">

moto/s3/utils.py

@@ -6,7 +6,7 @@ import six
 from six.moves.urllib.parse import urlparse, unquote
 import sys
 
-bucket_name_regex = re.compile("(.+).s3.amazonaws.com")
+bucket_name_regex = re.compile("(.+).s3(.*).amazonaws.com")
 
 
 def bucket_name_from_url(url):

tests/test_ec2/test_instances.py

@@ -600,3 +600,18 @@ def test_describe_instance_status_with_non_running_instances():
 
     status3 = next((s for s in all_status if s.id == instance3.id), None)
     status3.state_name.should.equal('running')
+
+@mock_ec2
+def test_get_instance_by_security_group():
+    conn = boto.connect_ec2('the_key', 'the_secret')
+    
+    conn.run_instances('ami-1234abcd')
+    instance = conn.get_only_instances()[0]
+
+    security_group = conn.create_security_group('test', 'test')
+    conn.modify_instance_attribute(instance.id, "groupSet", [security_group.id])
+    
+    security_group_instances = security_group.instances()
+
+    assert len(security_group_instances) == 1
+    assert security_group_instances[0].id == instance.id

tests/test_ec2/test_route_tables.py

@@ -419,6 +419,29 @@ def test_routes_vpc_peering_connection():
     new_route.destination_cidr_block.should.equal(ROUTE_CIDR)
 
 
+@requires_boto_gte("2.34.0")
+@mock_ec2
+def test_routes_vpn_gateway():
+
+    conn = boto.connect_vpc('the_key', 'the_secret')
+    vpc = conn.create_vpc("10.0.0.0/16")
+    main_route_table = conn.get_all_route_tables(filters={'association.main':'true','vpc-id':vpc.id})[0]
+    ROUTE_CIDR = "10.0.0.4/24"
+
+    vpn_gw = conn.create_vpn_gateway(type="ipsec.1")
+
+    conn.create_route(main_route_table.id, ROUTE_CIDR, gateway_id=vpn_gw.id)
+
+    main_route_table = conn.get_all_route_tables(main_route_table.id)[0]
+    new_routes = [route for route in main_route_table.routes if route.destination_cidr_block != vpc.cidr_block]
+    new_routes.should.have.length_of(1)
+
+    new_route = new_routes[0]
+    new_route.gateway_id.should.equal(vpn_gw.id)
+    new_route.instance_id.should.be.none
+    new_route.vpc_peering_connection_id.should.be.none
+
+
 @mock_ec2
 def test_network_acl_tagging():
 

tests/test_s3/test_s3.py

@@ -307,12 +307,29 @@ def test_bucket_with_dash():
 @mock_s3
 def test_create_existing_bucket():
     "Trying to create a bucket that already exists should raise an Error"
-    conn = boto.connect_s3('the_key', 'the_secret')
+    conn = boto.s3.connect_to_region("us-west-2")
     conn.create_bucket("foobar")
     with assert_raises(S3CreateError):
         conn.create_bucket('foobar')
 
 
+@mock_s3
+def test_create_existing_bucket_in_us_east_1():
+    "Trying to create a bucket that already exists in us-east-1 returns the bucket"
+
+    """"
+    http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+    Your previous request to create the named bucket succeeded and you already
+    own it. You get this error in all AWS regions except US Standard,
+    us-east-1. In us-east-1 region, you will get 200 OK, but it is no-op (if
+    bucket exists it Amazon S3 will not do anything).
+    """
+    conn = boto.s3.connect_to_region("us-east-1")
+    conn.create_bucket("foobar")
+    bucket = conn.create_bucket("foobar")
+    bucket.name.should.equal("foobar")
+
+
 @mock_s3
 def test_other_region():
     conn = S3Connection('key', 'secret', host='s3-website-ap-southeast-2.amazonaws.com')
@@ -668,3 +685,10 @@ def test_setting_content_encoding():
 
     key = bucket.get_key("keyname")
     key.content_encoding.should.equal("gzip")
+
+
+@mock_s3
+def test_bucket_location():
+    conn = boto.s3.connect_to_region("us-west-2")
+    bucket = conn.create_bucket('mybucket')
+    bucket.get_location().should.equal("us-west-2")