From 156ba56fdc94414e45ff83763d04fca91b111513 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 15 Apr 2019 19:57:42 -0500 Subject: [PATCH 1/5] set default status for s3 posts and add support for success_action_redirect. --- moto/s3/responses.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 22cd45c08..2f52e0d4a 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -776,8 +776,9 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): template = self.response_template(S3_DELETE_BUCKET_WITH_ITEMS_ERROR) return 409, {}, template.render(bucket=removed_bucket) - def _bucket_response_post(self, request, body, bucket_name): - if not request.headers.get("Content-Length"): + def _bucket_response_post(self, request, body, bucket_name, headers): + response_headers = {} + if not request.headers.get('Content-Length'): return 411, {}, "Content-Length required" path = self._get_path(request) @@ -810,13 +811,21 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): else: f = request.files["file"].stream.read() + if 'success_action_redirect' in form: + response_headers['Location'] = form['success_action_redirect'] + + if 'success_action_status' in form: + status_code = form['success_action_status'] + else: + status_code = 204 + new_key = self.backend.set_key(bucket_name, key, f) # Metadata metadata = metadata_from_headers(form) new_key.set_metadata(metadata) - return 200, {}, "" + return status_code, response_headers, "" @staticmethod def _get_path(request): From b3f6e5ab2fed73cfc9f66de92b16cfa52e3602bc Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 29 May 2019 15:22:29 -0500 Subject: [PATCH 2/5] add test --- moto/s3/responses.py | 2 ++ tests/test_s3/test_s3.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 2f52e0d4a..5526646a3 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -816,6 +816,8 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): if 'success_action_status' in form: status_code = form['success_action_status'] + elif 'success_action_redirect' in form: + status_code = 303 else: status_code = 204 diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 303ed523d..f7040e006 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -14,6 +14,7 @@ from io import BytesIO import mimetypes import zlib import pickle +import uuid import json import boto @@ -4428,3 +4429,34 @@ def test_s3_config_dict(): assert not logging_bucket["supplementaryConfiguration"].get( "BucketTaggingConfiguration" ) + + +@mock_s3 +def test_creating_presigned_post(): + bucket = 'presigned-test' + s3 = boto3.client('s3', region_name='us-east-1') + s3.create_bucket(Bucket=bucket) + success_url = 'http://localhost/completed' + fdata = b'test data\n' + file_uid = uuid.uuid4() + conditions = [ + {"Content-Type": 'text/plain'}, + {"x-amz-server-side-encryption": "AES256"}, + {'success_action_redirect': success_url}, + ] + conditions.append(["content-length-range", 1, 30]) + data = s3.generate_presigned_post( + Bucket=bucket, + Key='{file_uid}.txt'.format(file_uid=file_uid), + Fields={ + 'content-type': 'text/plain', + 'success_action_redirect': success_url, + 'x-amz-server-side-encryption': 'AES256' + }, + Conditions=conditions, + ExpiresIn=1000, + ) + resp = requests.post(data['url'], data=data['fields'], files={'file': fdata}, allow_redirects=False) + assert resp.headers['Location'] == url + assert resp.status_code == 303 + assert s3.get_object(Bucket=bucket, Key='{file_uuid}.txt'.format(file_uid=file_uid))['Body'].read() == fdata From 49b056563a2396727e17253d09e6924ce24ef09e Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 21 Apr 2020 19:51:48 -0500 Subject: [PATCH 3/5] process multipart form --- moto/s3/responses.py | 49 +++++++++++++++++++++++++++++----------- tests/test_s3/test_s3.py | 4 ++-- 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 5526646a3..92a82e4ff 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -7,7 +7,7 @@ import six from botocore.awsrequest import AWSPreparedRequest from moto.core.utils import str_to_rfc_1123_datetime, py2_strip_unicode_keys -from six.moves.urllib.parse import parse_qs, urlparse, unquote +from six.moves.urllib.parse import parse_qs, urlparse, unquote, parse_qsl import xmltodict @@ -143,6 +143,31 @@ def is_delete_keys(request, path, bucket_name): ) +def _process_multipart_formdata(request): + """ + When not using the live server, the request does not pass through flask, so it is not processed. + This will only be used in places where we end up with a requests PreparedRequest. + """ + form = {} + boundkey = request.headers['Content-Type'][len('multipart/form-data; boundary='):] + boundary = f'--{boundkey}' + data = request.body.decode().split(boundary) + fields = [field.split('\r\n\r\n') for field in data][1:-1] + for key, value in fields: + key, value = key.replace('\r\n', ''), value.replace('\r\n', '') + key = key.split('; ') + if len(key) == 2: + disposition, name = key + filename = None + else: + disposition, name, filename = key + name = name[len('name='):].strip('"') + if disposition.endswith('form-data'): + form[name] = value + import code; code.interact(local=locals()) + return form + + class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): def __init__(self, backend): super(ResponseObject, self).__init__() @@ -776,9 +801,9 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): template = self.response_template(S3_DELETE_BUCKET_WITH_ITEMS_ERROR) return 409, {}, template.render(bucket=removed_bucket) - def _bucket_response_post(self, request, body, bucket_name, headers): + def _bucket_response_post(self, request, body, bucket_name): response_headers = {} - if not request.headers.get('Content-Length'): + if not request.headers.get("Content-Length"): return 411, {}, "Content-Length required" path = self._get_path(request) @@ -796,14 +821,12 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): if hasattr(request, "form"): # Not HTTPretty form = request.form + elif request.headers.get('Content-Type').startswith('multipart/form-data'): + form = _process_multipart_formdata(request) else: # HTTPretty, build new form object body = body.decode() - - form = {} - for kv in body.split("&"): - k, v = kv.split("=") - form[k] = v + form = dict(parse_qsl(body)) key = form["key"] if "file" in form: @@ -811,12 +834,12 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): else: f = request.files["file"].stream.read() - if 'success_action_redirect' in form: - response_headers['Location'] = form['success_action_redirect'] + if "success_action_redirect" in form: + response_headers["Location"] = form["success_action_redirect"] - if 'success_action_status' in form: - status_code = form['success_action_status'] - elif 'success_action_redirect' in form: + if "success_action_status" in form: + status_code = form["success_action_status"] + elif "success_action_redirect" in form: status_code = 303 else: status_code = 204 diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index f7040e006..c226a7b3b 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -4457,6 +4457,6 @@ def test_creating_presigned_post(): ExpiresIn=1000, ) resp = requests.post(data['url'], data=data['fields'], files={'file': fdata}, allow_redirects=False) - assert resp.headers['Location'] == url + assert resp.headers['Location'] == success_url assert resp.status_code == 303 - assert s3.get_object(Bucket=bucket, Key='{file_uuid}.txt'.format(file_uid=file_uid))['Body'].read() == fdata + assert s3.get_object(Bucket=bucket, Key='{file_uid}.txt'.format(file_uid=file_uid))['Body'].read() == fdata From 4b0ba7320433b4b66488fc851c828f2ec1b56836 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 21 Apr 2020 20:13:53 -0500 Subject: [PATCH 4/5] use werkzeug hooray, thanks pallets discord! --- moto/s3/responses.py | 34 ++++++++-------------------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 92a82e4ff..965d15f57 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -5,6 +5,7 @@ import sys import six from botocore.awsrequest import AWSPreparedRequest +from werkzeug.wrappers import Request from moto.core.utils import str_to_rfc_1123_datetime, py2_strip_unicode_keys from six.moves.urllib.parse import parse_qs, urlparse, unquote, parse_qsl @@ -143,31 +144,6 @@ def is_delete_keys(request, path, bucket_name): ) -def _process_multipart_formdata(request): - """ - When not using the live server, the request does not pass through flask, so it is not processed. - This will only be used in places where we end up with a requests PreparedRequest. - """ - form = {} - boundkey = request.headers['Content-Type'][len('multipart/form-data; boundary='):] - boundary = f'--{boundkey}' - data = request.body.decode().split(boundary) - fields = [field.split('\r\n\r\n') for field in data][1:-1] - for key, value in fields: - key, value = key.replace('\r\n', ''), value.replace('\r\n', '') - key = key.split('; ') - if len(key) == 2: - disposition, name = key - filename = None - else: - disposition, name, filename = key - name = name[len('name='):].strip('"') - if disposition.endswith('form-data'): - form[name] = value - import code; code.interact(local=locals()) - return form - - class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): def __init__(self, backend): super(ResponseObject, self).__init__() @@ -822,7 +798,13 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): # Not HTTPretty form = request.form elif request.headers.get('Content-Type').startswith('multipart/form-data'): - form = _process_multipart_formdata(request) + request = Request.from_values( + input_stream=six.BytesIO(request.body), + content_length=request.headers['Content-Length'], + content_type=request.headers['Content-Type'], + method='POST', + ) + form = request.form else: # HTTPretty, build new form object body = body.decode() From 80b27a6b93d0c52d8f9f5349ec87efd036a66247 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 21 Apr 2020 21:43:32 -0500 Subject: [PATCH 5/5] blacken --- moto/s3/responses.py | 8 ++++---- tests/test_s3/test_s3.py | 33 ++++++++++++++++++++------------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 965d15f57..6ac139a14 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -797,12 +797,12 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): if hasattr(request, "form"): # Not HTTPretty form = request.form - elif request.headers.get('Content-Type').startswith('multipart/form-data'): + elif request.headers.get("Content-Type").startswith("multipart/form-data"): request = Request.from_values( input_stream=six.BytesIO(request.body), - content_length=request.headers['Content-Length'], - content_type=request.headers['Content-Type'], - method='POST', + content_length=request.headers["Content-Length"], + content_type=request.headers["Content-Type"], + method="POST", ) form = request.form else: diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index c226a7b3b..ffbd73966 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -4433,30 +4433,37 @@ def test_s3_config_dict(): @mock_s3 def test_creating_presigned_post(): - bucket = 'presigned-test' - s3 = boto3.client('s3', region_name='us-east-1') + bucket = "presigned-test" + s3 = boto3.client("s3", region_name="us-east-1") s3.create_bucket(Bucket=bucket) - success_url = 'http://localhost/completed' - fdata = b'test data\n' + success_url = "http://localhost/completed" + fdata = b"test data\n" file_uid = uuid.uuid4() conditions = [ - {"Content-Type": 'text/plain'}, + {"Content-Type": "text/plain"}, {"x-amz-server-side-encryption": "AES256"}, - {'success_action_redirect': success_url}, + {"success_action_redirect": success_url}, ] conditions.append(["content-length-range", 1, 30]) data = s3.generate_presigned_post( Bucket=bucket, - Key='{file_uid}.txt'.format(file_uid=file_uid), + Key="{file_uid}.txt".format(file_uid=file_uid), Fields={ - 'content-type': 'text/plain', - 'success_action_redirect': success_url, - 'x-amz-server-side-encryption': 'AES256' + "content-type": "text/plain", + "success_action_redirect": success_url, + "x-amz-server-side-encryption": "AES256", }, Conditions=conditions, ExpiresIn=1000, ) - resp = requests.post(data['url'], data=data['fields'], files={'file': fdata}, allow_redirects=False) - assert resp.headers['Location'] == success_url + resp = requests.post( + data["url"], data=data["fields"], files={"file": fdata}, allow_redirects=False + ) + assert resp.headers["Location"] == success_url assert resp.status_code == 303 - assert s3.get_object(Bucket=bucket, Key='{file_uid}.txt'.format(file_uid=file_uid))['Body'].read() == fdata + assert ( + s3.get_object(Bucket=bucket, Key="{file_uid}.txt".format(file_uid=file_uid))[ + "Body" + ].read() + == fdata + )