Cleanup S3 a bit and add more coverage

moto/s3/models.py

@@ -14,7 +14,7 @@ UPLOAD_PART_MIN_SIZE = 5242880
 
 
 class FakeKey(object):
-    def __init__(self, name, value, storage=None):
+    def __init__(self, name, value, storage="STANDARD"):
         self.name = name
         self.value = value
         self.last_modified = datetime.datetime.now()
@@ -22,7 +22,7 @@ class FakeKey(object):
         self._metadata = {}
         self._expiry = None
 
-    def copy(self, new_name = None):
+    def copy(self, new_name=None):
         r = copy.deepcopy(self)
         if new_name is not None:
             r.name = new_name
@@ -70,9 +70,8 @@ class FakeKey(object):
             'etag': self.etag,
             'last-modified': self.last_modified_RFC1123,
         }
-        if self._storage_class is not None:
+        if self._storage_class != 'STANDARD':
-            if self._storage_class != 'STANDARD':
+            r['x-amz-storage-class'] = self._storage_class
-                r['x-amz-storage-class'] = self._storage_class
         if self._expiry is not None:
             rhdr = 'ongoing-request="false", expiry-date="{0}"'
             r['x-amz-restore'] = rhdr.format(self.expiry_date)
@@ -84,10 +83,7 @@ class FakeKey(object):
 
     @property
     def storage_class(self):
-        if self._storage_class is not None:
+        return self._storage_class
-            return self._storage_class
-        else:
-            return 'STANDARD'
 
     @property
     def expiry_date(self):

moto/s3/responses.py

@@ -29,7 +29,6 @@ class ResponseObject(object):
         response = self._bucket_response(request, full_url, headers)
         if isinstance(response, basestring):
             return 200, headers, response
-
         else:
             status_code, headers, response_content = response
             return status_code, headers, response_content
@@ -45,82 +44,96 @@ class ResponseObject(object):
             return self.all_buckets()
 
         if method == 'HEAD':
-            bucket = self.backend.get_bucket(bucket_name)
+            return self._bucket_response_head(bucket_name, headers)
-            if bucket:
-                return 200, headers, ""
-            else:
-                return 404, headers, ""
-
         elif method == 'GET':
-            bucket = self.backend.get_bucket(bucket_name)
+            return self._bucket_response_get(bucket_name, querystring, headers)
-            if bucket:
-                prefix = querystring.get('prefix', [None])[0]
-                delimiter = querystring.get('delimiter', [None])[0]
-                result_keys, result_folders = self.backend.prefix_query(bucket, prefix, delimiter)
-                template = Template(S3_BUCKET_GET_RESPONSE)
-                return template.render(
-                    bucket=bucket,
-                    prefix=prefix,
-                    delimiter=delimiter,
-                    result_keys=result_keys,
-                    result_folders=result_folders
-                )
-            else:
-                return 404, headers, ""
         elif method == 'PUT':
-            try:
+            return self._bucket_response_put(bucket_name, headers)
-                new_bucket = self.backend.create_bucket(bucket_name)
-            except BucketAlreadyExists:
-                return 409, headers, ""
-            template = Template(S3_BUCKET_CREATE_RESPONSE)
-            return template.render(bucket=new_bucket)
         elif method == 'DELETE':
-            removed_bucket = self.backend.delete_bucket(bucket_name)
+            return self._bucket_response_delete(bucket_name, headers)
-            if removed_bucket is None:
-                # Non-existant bucket
-                template = Template(S3_DELETE_NON_EXISTING_BUCKET)
-                return 404, headers, template.render(bucket_name=bucket_name)
-            elif removed_bucket:
-                # Bucket exists
-                template = Template(S3_DELETE_BUCKET_SUCCESS)
-                return 204, headers, template.render(bucket=removed_bucket)
-            else:
-                # Tried to delete a bucket that still has keys
-                template = Template(S3_DELETE_BUCKET_WITH_ITEMS_ERROR)
-                return 409, headers, template.render(bucket=removed_bucket)
         elif method == 'POST':
-            #POST to bucket-url should create file from form
+            return self._bucket_response_post(request, bucket_name, headers)
-            if hasattr(request, 'form'):
-                #Not HTTPretty
-                form = request.form
-            else:
-                #HTTPretty, build new form object
-                form = {}
-                for kv in request.body.split('&'):
-                    k, v = kv.split('=')
-                    form[k] = v
-
-            key = form['key']
-            if 'file' in form:
-                f = form['file']
-            else:
-                f = request.files['file'].stream.read()
-
-            new_key = self.backend.set_key(bucket_name, key, f)
-
-            #Metadata
-            meta_regex = re.compile('^x-amz-meta-([a-zA-Z0-9\-_]+)$', flags=re.IGNORECASE)
-
-            for form_id in form:
-                result = meta_regex.match(form_id)
-                if result:
-                    meta_key = result.group(0).lower()
-                    metadata = form[form_id]
-                    new_key.set_metadata(meta_key, metadata)
-            return 200, headers, ""
         else:
             raise NotImplementedError("Method {0} has not been impelemented in the S3 backend yet".format(method))
 
+    def _bucket_response_head(self, bucket_name, headers):
+        bucket = self.backend.get_bucket(bucket_name)
+        if bucket:
+            return 200, headers, ""
+        else:
+            return 404, headers, ""
+
+    def _bucket_response_get(self, bucket_name, querystring, headers):
+        bucket = self.backend.get_bucket(bucket_name)
+        if bucket:
+            prefix = querystring.get('prefix', [None])[0]
+            delimiter = querystring.get('delimiter', [None])[0]
+            result_keys, result_folders = self.backend.prefix_query(bucket, prefix, delimiter)
+            template = Template(S3_BUCKET_GET_RESPONSE)
+            return template.render(
+                bucket=bucket,
+                prefix=prefix,
+                delimiter=delimiter,
+                result_keys=result_keys,
+                result_folders=result_folders
+            )
+        else:
+            return 404, headers, ""
+
+    def _bucket_response_put(self, bucket_name, headers):
+        try:
+            new_bucket = self.backend.create_bucket(bucket_name)
+        except BucketAlreadyExists:
+            return 409, headers, ""
+        template = Template(S3_BUCKET_CREATE_RESPONSE)
+        return template.render(bucket=new_bucket)
+
+    def _bucket_response_delete(self, bucket_name, headers):
+        removed_bucket = self.backend.delete_bucket(bucket_name)
+        if removed_bucket is None:
+            # Non-existant bucket
+            template = Template(S3_DELETE_NON_EXISTING_BUCKET)
+            return 404, headers, template.render(bucket_name=bucket_name)
+        elif removed_bucket:
+            # Bucket exists
+            template = Template(S3_DELETE_BUCKET_SUCCESS)
+            return 204, headers, template.render(bucket=removed_bucket)
+        else:
+            # Tried to delete a bucket that still has keys
+            template = Template(S3_DELETE_BUCKET_WITH_ITEMS_ERROR)
+            return 409, headers, template.render(bucket=removed_bucket)
+
+    def _bucket_response_post(self, request, bucket_name, headers):
+        #POST to bucket-url should create file from form
+        if hasattr(request, 'form'):
+            #Not HTTPretty
+            form = request.form
+        else:
+            #HTTPretty, build new form object
+            form = {}
+            for kv in request.body.split('&'):
+                k, v = kv.split('=')
+                form[k] = v
+
+        key = form['key']
+        if 'file' in form:
+            f = form['file']
+        else:
+            f = request.files['file'].stream.read()
+
+        new_key = self.backend.set_key(bucket_name, key, f)
+
+        #Metadata
+        meta_regex = re.compile('^x-amz-meta-([a-zA-Z0-9\-_]+)$', flags=re.IGNORECASE)
+
+        for form_id in form:
+            result = meta_regex.match(form_id)
+            if result:
+                meta_key = result.group(0).lower()
+                metadata = form[form_id]
+                new_key.set_metadata(meta_key, metadata)
+        return 200, headers, ""
+
     def key_response(self, request, full_url, headers):
         response = self._key_response(request, full_url, headers)
         if isinstance(response, basestring):
@@ -147,7 +160,6 @@ class ResponseObject(object):
         method = request.method
 
         key_name = self.parse_key_name(parsed_url.path)
-
         bucket_name = self.bucket_name_from_url(full_url)
 
         if hasattr(request, 'body'):
@@ -158,127 +170,142 @@ class ResponseObject(object):
             body = request.data
 
         if method == 'GET':
-            if 'uploadId' in query:
+            return self._key_response_get(bucket_name, query, key_name, headers)
-                upload_id = query['uploadId'][0]
+        elif method == 'PUT':
-                parts = self.backend.list_multipart(bucket_name, upload_id)
+            return self._key_response_put(request, body, bucket_name, query, key_name, headers)
-                template = Template(S3_MULTIPART_LIST_RESPONSE)
-                return 200, headers, template.render(
-                    bucket_name=bucket_name,
-                    key_name=key_name,
-                    upload_id=upload_id,
-                    count=len(parts),
-                    parts=parts
-                )
-            key = self.backend.get_key(bucket_name, key_name)
-            if key:
-                headers.update(key.metadata)
-                return 200, headers, key.value
-            else:
-                return 404, headers, ""
-        if method == 'PUT':
-            if 'uploadId' in query and 'partNumber' in query:
-                upload_id = query['uploadId'][0]
-                part_number = int(query['partNumber'][0])
-                if 'x-amz-copy-source' in request.headers:
-                    src = request.headers.get("x-amz-copy-source")
-                    src_bucket, src_key = src.split("/", 1)
-                    key = self.backend.copy_part(
-                        bucket_name, upload_id, part_number, src_bucket,
-                        src_key)
-                    template = Template(S3_MULTIPART_UPLOAD_RESPONSE)
-                    response = template.render(part=key)
-                else:
-                    key = self.backend.set_part(
-                        bucket_name, upload_id, part_number, body)
-                    response = ""
-                headers.update(key.response_dict)
-                return 200, headers, response
-
-            storage_class = request.headers.get('x-amz-storage-class', 'STANDARD')
-
-            if 'x-amz-copy-source' in request.headers:
-                # Copy key
-                src_bucket, src_key = request.headers.get("x-amz-copy-source").split("/", 1)
-                self.backend.copy_key(src_bucket, src_key, bucket_name, key_name,
-                                      storage=storage_class)
-                mdirective = request.headers.get('x-amz-metadata-directive')
-                if mdirective is not None and mdirective == 'REPLACE':
-                    new_key = self.backend.get_key(bucket_name, key_name)
-                    self._key_set_metadata(request, new_key, replace=True)
-                template = Template(S3_OBJECT_COPY_RESPONSE)
-                return template.render(key=src_key)
-            streaming_request = hasattr(request, 'streaming') and request.streaming
-            closing_connection = headers.get('connection') == 'close'
-            if closing_connection and streaming_request:
-                # Closing the connection of a streaming request. No more data
-                new_key = self.backend.get_key(bucket_name, key_name)
-            elif streaming_request:
-                # Streaming request, more data
-                new_key = self.backend.append_to_key(bucket_name, key_name, body)
-            else:
-                # Initial data
-                new_key = self.backend.set_key(bucket_name, key_name, body,
-                                               storage=storage_class)
-                request.streaming = True
-                self._key_set_metadata(request, new_key)
-
-            template = Template(S3_OBJECT_RESPONSE)
-            headers.update(new_key.response_dict)
-            return 200, headers, template.render(key=new_key)
         elif method == 'HEAD':
-            key = self.backend.get_key(bucket_name, key_name)
+            return self._key_response_head(bucket_name, key_name, headers)
-            if key:
-                headers.update(key.metadata)
-                headers.update(key.response_dict)
-                return 200, headers, ""
-            else:
-                return 404, headers, ""
         elif method == 'DELETE':
-            if 'uploadId' in query:
+            return self._key_response_delete(bucket_name, query, key_name, headers)
-                upload_id = query['uploadId'][0]
-                self.backend.cancel_multipart(bucket_name, upload_id)
-                return 204, headers, ""
-            removed_key = self.backend.delete_key(bucket_name, key_name)
-            template = Template(S3_DELETE_OBJECT_SUCCESS)
-            return 204, headers, template.render(bucket=removed_key)
         elif method == 'POST':
-            if body == '' and parsed_url.query == 'uploads':
+            return self._key_response_post(body, parsed_url, bucket_name, query, key_name, headers)
-                multipart = self.backend.initiate_multipart(bucket_name, key_name)
-                template = Template(S3_MULTIPART_INITIATE_RESPONSE)
-                response = template.render(
-                    bucket_name=bucket_name,
-                    key_name=key_name,
-                    upload_id=multipart.id,
-                )
-                return 200, headers, response
-
-            if 'uploadId' in query:
-                upload_id = query['uploadId'][0]
-                key = self.backend.complete_multipart(bucket_name, upload_id)
-
-                if key is not None:
-                    template = Template(S3_MULTIPART_COMPLETE_RESPONSE)
-                    return template.render(
-                        bucket_name=bucket_name,
-                        key_name=key.name,
-                        etag=key.etag,
-                    )
-                template = Template(S3_MULTIPART_COMPLETE_TOO_SMALL_ERROR)
-                return 400, headers, template.render()
-            elif parsed_url.query == 'restore':
-                es = minidom.parseString(body).getElementsByTagName('Days')
-                days = es[0].childNodes[0].wholeText
-                key = self.backend.get_key(bucket_name, key_name)
-                r = 202
-                if key.expiry_date is not None:
-                    r = 200
-                key.restore(int(days))
-                return r, headers, ""
-            else:
-                raise NotImplementedError("Method POST had only been implemented for multipart uploads and restore operations, so far")
         else:
             raise NotImplementedError("Method {0} has not been impelemented in the S3 backend yet".format(method))
 
+    def _key_response_get(self, bucket_name, query, key_name, headers):
+        if 'uploadId' in query:
+            upload_id = query['uploadId'][0]
+            parts = self.backend.list_multipart(bucket_name, upload_id)
+            template = Template(S3_MULTIPART_LIST_RESPONSE)
+            return 200, headers, template.render(
+                bucket_name=bucket_name,
+                key_name=key_name,
+                upload_id=upload_id,
+                count=len(parts),
+                parts=parts
+            )
+        key = self.backend.get_key(bucket_name, key_name)
+        if key:
+            headers.update(key.metadata)
+            return 200, headers, key.value
+        else:
+            return 404, headers, ""
+
+    def _key_response_put(self, request, body, bucket_name, query, key_name, headers):
+        if 'uploadId' in query and 'partNumber' in query:
+            upload_id = query['uploadId'][0]
+            part_number = int(query['partNumber'][0])
+            if 'x-amz-copy-source' in request.headers:
+                src = request.headers.get("x-amz-copy-source")
+                src_bucket, src_key = src.split("/", 1)
+                key = self.backend.copy_part(
+                    bucket_name, upload_id, part_number, src_bucket,
+                    src_key)
+                template = Template(S3_MULTIPART_UPLOAD_RESPONSE)
+                response = template.render(part=key)
+            else:
+                key = self.backend.set_part(
+                    bucket_name, upload_id, part_number, body)
+                response = ""
+            headers.update(key.response_dict)
+            return 200, headers, response
+
+        storage_class = request.headers.get('x-amz-storage-class', 'STANDARD')
+
+        if 'x-amz-copy-source' in request.headers:
+            # Copy key
+            src_bucket, src_key = request.headers.get("x-amz-copy-source").split("/", 1)
+            self.backend.copy_key(src_bucket, src_key, bucket_name, key_name,
+                                  storage=storage_class)
+            mdirective = request.headers.get('x-amz-metadata-directive')
+            if mdirective is not None and mdirective == 'REPLACE':
+                new_key = self.backend.get_key(bucket_name, key_name)
+                self._key_set_metadata(request, new_key, replace=True)
+            template = Template(S3_OBJECT_COPY_RESPONSE)
+            return template.render(key=src_key)
+        streaming_request = hasattr(request, 'streaming') and request.streaming
+        closing_connection = headers.get('connection') == 'close'
+        if closing_connection and streaming_request:
+            # Closing the connection of a streaming request. No more data
+            new_key = self.backend.get_key(bucket_name, key_name)
+        elif streaming_request:
+            # Streaming request, more data
+            new_key = self.backend.append_to_key(bucket_name, key_name, body)
+        else:
+            # Initial data
+            new_key = self.backend.set_key(bucket_name, key_name, body,
+                                           storage=storage_class)
+            request.streaming = True
+            self._key_set_metadata(request, new_key)
+
+        template = Template(S3_OBJECT_RESPONSE)
+        headers.update(new_key.response_dict)
+        return 200, headers, template.render(key=new_key)
+
+    def _key_response_head(self, bucket_name, key_name, headers):
+        key = self.backend.get_key(bucket_name, key_name)
+        if key:
+            headers.update(key.metadata)
+            headers.update(key.response_dict)
+            return 200, headers, ""
+        else:
+            return 404, headers, ""
+
+    def _key_response_delete(self, bucket_name, query, key_name, headers):
+        if 'uploadId' in query:
+            upload_id = query['uploadId'][0]
+            self.backend.cancel_multipart(bucket_name, upload_id)
+            return 204, headers, ""
+        removed_key = self.backend.delete_key(bucket_name, key_name)
+        template = Template(S3_DELETE_OBJECT_SUCCESS)
+        return 204, headers, template.render(bucket=removed_key)
+
+    def _key_response_post(self, body, parsed_url, bucket_name, query, key_name, headers):
+        if body == '' and parsed_url.query == 'uploads':
+            multipart = self.backend.initiate_multipart(bucket_name, key_name)
+            template = Template(S3_MULTIPART_INITIATE_RESPONSE)
+            response = template.render(
+                bucket_name=bucket_name,
+                key_name=key_name,
+                upload_id=multipart.id,
+            )
+            return 200, headers, response
+
+        if 'uploadId' in query:
+            upload_id = query['uploadId'][0]
+            key = self.backend.complete_multipart(bucket_name, upload_id)
+
+            if key is not None:
+                template = Template(S3_MULTIPART_COMPLETE_RESPONSE)
+                return template.render(
+                    bucket_name=bucket_name,
+                    key_name=key.name,
+                    etag=key.etag,
+                )
+            template = Template(S3_MULTIPART_COMPLETE_TOO_SMALL_ERROR)
+            return 400, headers, template.render()
+        elif parsed_url.query == 'restore':
+            es = minidom.parseString(body).getElementsByTagName('Days')
+            days = es[0].childNodes[0].wholeText
+            key = self.backend.get_key(bucket_name, key_name)
+            r = 202
+            if key.expiry_date is not None:
+                r = 200
+            key.restore(int(days))
+            return r, headers, ""
+        else:
+            raise NotImplementedError("Method POST had only been implemented for multipart uploads and restore operations, so far")
+
 S3ResponseInstance = ResponseObject(s3_backend, bucket_name_from_url, parse_key_name)
 
 S3_ALL_BUCKETS = """<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">

tests/test_s3/test_s3.py

@@ -66,6 +66,35 @@ def test_multipart_upload():
     bucket.get_key("the-key").get_contents_as_string().should.equal(part1 + part2)
 
 
+@mock_s3
+def test_multipart_upload_with_copy_key():
+    conn = boto.connect_s3('the_key', 'the_secret')
+    bucket = conn.create_bucket("foobar")
+    key = Key(bucket)
+    key.key = "original-key"
+    key.set_contents_from_string("key_value")
+
+    multipart = bucket.initiate_multipart_upload("the-key")
+    part1 = '0' * 5242880
+    multipart.upload_part_from_file(BytesIO(part1), 1)
+    multipart.copy_part_from_key("foobar", "original-key", 2)
+    multipart.complete_upload()
+    bucket.get_key("the-key").get_contents_as_string().should.equal(part1 + "key_value")
+
+
+@mock_s3
+def test_multipart_upload_cancel():
+    conn = boto.connect_s3('the_key', 'the_secret')
+    bucket = conn.create_bucket("foobar")
+
+    multipart = bucket.initiate_multipart_upload("the-key")
+    part1 = '0' * 5242880
+    multipart.upload_part_from_file(BytesIO(part1), 1)
+    multipart.cancel_upload()
+    # TODO we really need some sort of assertion here, but we don't currently
+    # have the ability to list mulipart uploads for a bucket.
+
+
 @mock_s3
 def test_missing_key():
     conn = boto.connect_s3('the_key', 'the_secret')

tests/test_s3bucket_path/test_bucket_path_server.py

@@ -30,6 +30,9 @@ def test_s3_server_bucket_create():
     res.status_code.should.equal(200)
     res.data.should.contain("ListBucketResult")
 
+    res = test_client.get('/missing-bucket', 'http://localhost:5000')
+    res.status_code.should.equal(404)
+
     res = test_client.put('/foobar/bar', 'http://localhost:5000', data='test value')
     res.status_code.should.equal(200)