utf 8 in key deletions V2 (#1321)
* supporting utf-8 in key deletions * Fixed decoding of version body when regexing * Fixed some more random errors * Possible fix * Fixed unused import * Added UTF comment Py2
This commit is contained in:
Terry Cain
Jack Danger
parent
d0a285536d
commit
d4745a575b
@ -1,3 +1,4 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from __future__ import unicode_literals
|
||||
from __future__ import absolute_import
|
||||
|
||||
@ -176,16 +177,49 @@ class ServerModeMockAWS(BaseMockAWS):
|
||||
if 'endpoint_url' not in kwargs:
|
||||
kwargs['endpoint_url'] = "http://localhost:5000"
|
||||
return real_boto3_resource(*args, **kwargs)
|
||||
|
||||
def fake_httplib_send_output(self, message_body=None, *args, **kwargs):
|
||||
def _convert_to_bytes(mixed_buffer):
|
||||
bytes_buffer = []
|
||||
for chunk in mixed_buffer:
|
||||
if isinstance(chunk, six.text_type):
|
||||
bytes_buffer.append(chunk.encode('utf-8'))
|
||||
else:
|
||||
bytes_buffer.append(chunk)
|
||||
msg = b"\r\n".join(bytes_buffer)
|
||||
return msg
|
||||
|
||||
self._buffer.extend((b"", b""))
|
||||
msg = _convert_to_bytes(self._buffer)
|
||||
del self._buffer[:]
|
||||
if isinstance(message_body, bytes):
|
||||
msg += message_body
|
||||
message_body = None
|
||||
self.send(msg)
|
||||
# if self._expect_header_set:
|
||||
# read, write, exc = select.select([self.sock], [], [self.sock], 1)
|
||||
# if read:
|
||||
# self._handle_expect_response(message_body)
|
||||
# return
|
||||
if message_body is not None:
|
||||
self.send(message_body)
|
||||
|
||||
self._client_patcher = mock.patch('boto3.client', fake_boto3_client)
|
||||
self._resource_patcher = mock.patch(
|
||||
'boto3.resource', fake_boto3_resource)
|
||||
self._resource_patcher = mock.patch('boto3.resource', fake_boto3_resource)
|
||||
if six.PY2:
|
||||
self._httplib_patcher = mock.patch('httplib.HTTPConnection._send_output', fake_httplib_send_output)
|
||||
|
||||
self._client_patcher.start()
|
||||
self._resource_patcher.start()
|
||||
if six.PY2:
|
||||
self._httplib_patcher.start()
|
||||
|
||||
def disable_patching(self):
|
||||
if self._client_patcher:
|
||||
self._client_patcher.stop()
|
||||
self._resource_patcher.stop()
|
||||
if six.PY2:
|
||||
self._httplib_patcher.stop()
|
||||
|
||||
|
||||
class Model(type):
|
||||
|
||||
@ -8,6 +8,7 @@ from six.moves.urllib.parse import parse_qs, urlparse
|
||||
|
||||
import xmltodict
|
||||
|
||||
from moto.packages.httpretty.core import HTTPrettyRequest
|
||||
from moto.core.responses import _TemplateEnvironmentMixin
|
||||
|
||||
from moto.s3bucket_path.utils import bucket_name_from_url as bucketpath_bucket_name_from_url, parse_key_name as bucketpath_parse_key_name, is_delete_keys as bucketpath_is_delete_keys
|
||||
@ -113,7 +114,10 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
||||
return 200, {}, response.encode("utf-8")
|
||||
else:
|
||||
status_code, headers, response_content = response
|
||||
return status_code, headers, response_content.encode("utf-8")
|
||||
if not isinstance(response_content, six.binary_type):
|
||||
response_content = response_content.encode("utf-8")
|
||||
|
||||
return status_code, headers, response_content
|
||||
|
||||
def _bucket_response(self, request, full_url, headers):
|
||||
parsed_url = urlparse(full_url)
|
||||
@ -139,6 +143,7 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
||||
body = b''
|
||||
if isinstance(body, six.binary_type):
|
||||
body = body.decode('utf-8')
|
||||
body = u'{0}'.format(body).encode('utf-8')
|
||||
|
||||
if method == 'HEAD':
|
||||
return self._bucket_response_head(bucket_name, headers)
|
||||
@ -209,7 +214,7 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
||||
if not website_configuration:
|
||||
template = self.response_template(S3_NO_BUCKET_WEBSITE_CONFIG)
|
||||
return 404, {}, template.render(bucket_name=bucket_name)
|
||||
return website_configuration
|
||||
return 200, {}, website_configuration
|
||||
elif 'acl' in querystring:
|
||||
bucket = self.backend.get_bucket(bucket_name)
|
||||
template = self.response_template(S3_OBJECT_ACL_RESPONSE)
|
||||
@ -355,7 +360,7 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
||||
if not request.headers.get('Content-Length'):
|
||||
return 411, {}, "Content-Length required"
|
||||
if 'versioning' in querystring:
|
||||
ver = re.search('<Status>([A-Za-z]+)</Status>', body)
|
||||
ver = re.search('<Status>([A-Za-z]+)</Status>', body.decode())
|
||||
if ver:
|
||||
self.backend.set_bucket_versioning(bucket_name, ver.group(1))
|
||||
template = self.response_template(S3_BUCKET_VERSIONING)
|
||||
@ -444,7 +449,12 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
||||
def _bucket_response_post(self, request, body, bucket_name, headers):
|
||||
if not request.headers.get('Content-Length'):
|
||||
return 411, {}, "Content-Length required"
|
||||
path = request.path if hasattr(request, 'path') else request.path_url
|
||||
|
||||
if isinstance(request, HTTPrettyRequest):
|
||||
path = request.path
|
||||
else:
|
||||
path = request.full_path if hasattr(request, 'full_path') else request.path_url
|
||||
|
||||
if self.is_delete_keys(request, path, bucket_name):
|
||||
return self._bucket_response_delete_keys(request, body, bucket_name, headers)
|
||||
|
||||
@ -454,6 +464,8 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
||||
form = request.form
|
||||
else:
|
||||
# HTTPretty, build new form object
|
||||
body = body.decode()
|
||||
|
||||
form = {}
|
||||
for kv in body.split('&'):
|
||||
k, v = kv.split('=')
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import datetime
|
||||
@ -1865,7 +1864,7 @@ def test_boto3_list_object_versions():
|
||||
def test_boto3_delete_markers():
|
||||
s3 = boto3.client('s3', region_name='us-east-1')
|
||||
bucket_name = 'mybucket'
|
||||
key = 'key-with-versions'
|
||||
key = u'key-with-versions-and-unicode-ó'
|
||||
s3.create_bucket(Bucket=bucket_name)
|
||||
s3.put_bucket_versioning(
|
||||
Bucket=bucket_name,
|
||||
@ -1880,10 +1879,9 @@ def test_boto3_delete_markers():
|
||||
Key=key,
|
||||
Body=body
|
||||
)
|
||||
s3.delete_object(
|
||||
Bucket=bucket_name,
|
||||
Key=key
|
||||
)
|
||||
|
||||
s3.delete_objects(Bucket=bucket_name, Delete={'Objects': [{'Key': key}]})
|
||||
|
||||
with assert_raises(ClientError) as e:
|
||||
s3.get_object(
|
||||
Bucket=bucket_name,
|
||||
@ -1905,12 +1903,18 @@ def test_boto3_delete_markers():
|
||||
Bucket=bucket_name
|
||||
)
|
||||
response['Versions'].should.have.length_of(2)
|
||||
response['Versions'][-1]['IsLatest'].should.be.true
|
||||
response['Versions'][0]['IsLatest'].should.be.false
|
||||
[(key_metadata['Key'], key_metadata['VersionId'])
|
||||
for key_metadata in response['Versions']].should.equal(
|
||||
[('key-with-versions', '0'), ('key-with-versions', '1')]
|
||||
)
|
||||
|
||||
# We've asserted there is only 2 records so one is newest, one is oldest
|
||||
latest = list(filter(lambda item: item['IsLatest'], response['Versions']))[0]
|
||||
oldest = list(filter(lambda item: not item['IsLatest'], response['Versions']))[0]
|
||||
|
||||
# Double check ordering of version ID's
|
||||
latest['VersionId'].should.equal('1')
|
||||
oldest['VersionId'].should.equal('0')
|
||||
|
||||
# Double check the name is still unicode
|
||||
latest['Key'].should.equal('key-with-versions-and-unicode-ó')
|
||||
oldest['Key'].should.equal('key-with-versions-and-unicode-ó')
|
||||
|
||||
|
||||
@mock_s3
|
||||
|
||||
Loading…
Reference in New Issue
Block a user