utf 8 in key deletions V2 (#1321)

* supporting utf-8 in key deletions

* Fixed decoding of version body when regexing

* Fixed some more random errors

* Possible fix

* Fixed unused import

* Added UTF comment Py2

moto/core/models.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 from __future__ import absolute_import
 
@@ -176,16 +177,49 @@ class ServerModeMockAWS(BaseMockAWS):
             if 'endpoint_url' not in kwargs:
                 kwargs['endpoint_url'] = "http://localhost:5000"
             return real_boto3_resource(*args, **kwargs)
+
+        def fake_httplib_send_output(self, message_body=None, *args, **kwargs):
+            def _convert_to_bytes(mixed_buffer):
+                bytes_buffer = []
+                for chunk in mixed_buffer:
+                    if isinstance(chunk, six.text_type):
+                        bytes_buffer.append(chunk.encode('utf-8'))
+                    else:
+                        bytes_buffer.append(chunk)
+                msg = b"\r\n".join(bytes_buffer)
+                return msg
+
+            self._buffer.extend((b"", b""))
+            msg = _convert_to_bytes(self._buffer)
+            del self._buffer[:]
+            if isinstance(message_body, bytes):
+                msg += message_body
+                message_body = None
+            self.send(msg)
+            # if self._expect_header_set:
+            #     read, write, exc = select.select([self.sock], [], [self.sock], 1)
+            #     if read:
+            #         self._handle_expect_response(message_body)
+            #         return
+            if message_body is not None:
+                self.send(message_body)
+
         self._client_patcher = mock.patch('boto3.client', fake_boto3_client)
-        self._resource_patcher = mock.patch(
+        self._resource_patcher = mock.patch('boto3.resource', fake_boto3_resource)
-            'boto3.resource', fake_boto3_resource)
+        if six.PY2:
+            self._httplib_patcher = mock.patch('httplib.HTTPConnection._send_output', fake_httplib_send_output)
+
         self._client_patcher.start()
         self._resource_patcher.start()
+        if six.PY2:
+            self._httplib_patcher.start()
 
     def disable_patching(self):
         if self._client_patcher:
             self._client_patcher.stop()
             self._resource_patcher.stop()
+            if six.PY2:
+                self._httplib_patcher.stop()
 
 
 class Model(type):

moto/s3/responses.py

@@ -8,6 +8,7 @@ from six.moves.urllib.parse import parse_qs, urlparse
 
 import xmltodict
 
+from moto.packages.httpretty.core import HTTPrettyRequest
 from moto.core.responses import _TemplateEnvironmentMixin
 
 from moto.s3bucket_path.utils import bucket_name_from_url as bucketpath_bucket_name_from_url, parse_key_name as bucketpath_parse_key_name, is_delete_keys as bucketpath_is_delete_keys
@@ -113,7 +114,10 @@ class ResponseObject(_TemplateEnvironmentMixin):
             return 200, {}, response.encode("utf-8")
         else:
             status_code, headers, response_content = response
-            return status_code, headers, response_content.encode("utf-8")
+            if not isinstance(response_content, six.binary_type):
+                response_content = response_content.encode("utf-8")
+
+            return status_code, headers, response_content
 
     def _bucket_response(self, request, full_url, headers):
         parsed_url = urlparse(full_url)
@@ -139,6 +143,7 @@ class ResponseObject(_TemplateEnvironmentMixin):
             body = b''
         if isinstance(body, six.binary_type):
             body = body.decode('utf-8')
+        body = u'{0}'.format(body).encode('utf-8')
 
         if method == 'HEAD':
             return self._bucket_response_head(bucket_name, headers)
@@ -209,7 +214,7 @@ class ResponseObject(_TemplateEnvironmentMixin):
             if not website_configuration:
                 template = self.response_template(S3_NO_BUCKET_WEBSITE_CONFIG)
                 return 404, {}, template.render(bucket_name=bucket_name)
-            return website_configuration
+            return 200, {}, website_configuration
         elif 'acl' in querystring:
             bucket = self.backend.get_bucket(bucket_name)
             template = self.response_template(S3_OBJECT_ACL_RESPONSE)
@@ -355,7 +360,7 @@ class ResponseObject(_TemplateEnvironmentMixin):
         if not request.headers.get('Content-Length'):
             return 411, {}, "Content-Length required"
         if 'versioning' in querystring:
-            ver = re.search('<Status>([A-Za-z]+)</Status>', body)
+            ver = re.search('<Status>([A-Za-z]+)</Status>', body.decode())
             if ver:
                 self.backend.set_bucket_versioning(bucket_name, ver.group(1))
                 template = self.response_template(S3_BUCKET_VERSIONING)
@@ -444,7 +449,12 @@ class ResponseObject(_TemplateEnvironmentMixin):
     def _bucket_response_post(self, request, body, bucket_name, headers):
         if not request.headers.get('Content-Length'):
             return 411, {}, "Content-Length required"
-        path = request.path if hasattr(request, 'path') else request.path_url
+
+        if isinstance(request, HTTPrettyRequest):
+            path = request.path
+        else:
+            path = request.full_path if hasattr(request, 'full_path') else request.path_url
+
         if self.is_delete_keys(request, path, bucket_name):
             return self._bucket_response_delete_keys(request, body, bucket_name, headers)
 
@@ -454,6 +464,8 @@ class ResponseObject(_TemplateEnvironmentMixin):
             form = request.form
         else:
             # HTTPretty, build new form object
+            body = body.decode()
+
             form = {}
             for kv in body.split('&'):
                 k, v = kv.split('=')

tests/test_s3/test_s3.py

@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-
 from __future__ import unicode_literals
 
 import datetime
@@ -1865,7 +1864,7 @@ def test_boto3_list_object_versions():
 def test_boto3_delete_markers():
     s3 = boto3.client('s3', region_name='us-east-1')
     bucket_name = 'mybucket'
-    key = 'key-with-versions'
+    key = u'key-with-versions-and-unicode-ó'
     s3.create_bucket(Bucket=bucket_name)
     s3.put_bucket_versioning(
         Bucket=bucket_name,
@@ -1880,10 +1879,9 @@ def test_boto3_delete_markers():
             Key=key,
             Body=body
         )
-    s3.delete_object(
+
-        Bucket=bucket_name,
+    s3.delete_objects(Bucket=bucket_name, Delete={'Objects': [{'Key': key}]})
-        Key=key
+
-    )
     with assert_raises(ClientError) as e:
         s3.get_object(
             Bucket=bucket_name,
@@ -1905,12 +1903,18 @@ def test_boto3_delete_markers():
         Bucket=bucket_name
     )
     response['Versions'].should.have.length_of(2)
-    response['Versions'][-1]['IsLatest'].should.be.true
+
-    response['Versions'][0]['IsLatest'].should.be.false
+    # We've asserted there is only 2 records so one is newest, one is oldest
-    [(key_metadata['Key'], key_metadata['VersionId'])
+    latest = list(filter(lambda item: item['IsLatest'], response['Versions']))[0]
-     for key_metadata in response['Versions']].should.equal(
+    oldest = list(filter(lambda item: not item['IsLatest'], response['Versions']))[0]
-        [('key-with-versions', '0'), ('key-with-versions', '1')]
+
-    )
+    # Double check ordering of version ID's
+    latest['VersionId'].should.equal('1')
+    oldest['VersionId'].should.equal('0')
+
+    # Double check the name is still unicode
+    latest['Key'].should.equal('key-with-versions-and-unicode-ó')
+    oldest['Key'].should.equal('key-with-versions-and-unicode-ó')
 
 
 @mock_s3