diff --git a/moto/core/responses.py b/moto/core/responses.py index bfdbac1e8..8e97b8f42 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -14,6 +14,7 @@ from moto.core.common_types import TYPE_RESPONSE, TYPE_IF_NONE from moto.core.exceptions import DryRunClientError from moto.core.utils import ( camelcase_to_underscores, + gzip_decompress, method_names_from_class, params_sort_function, ) @@ -232,6 +233,7 @@ class BaseResponse(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): def __init__(self, service_name: Optional[str] = None): super().__init__() self.service_name = service_name + self.allow_request_decompression = True @classmethod def dispatch(cls, *args: Any, **kwargs: Any) -> Any: # type: ignore[misc] @@ -262,6 +264,15 @@ class BaseResponse(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): querystring[key] = [value] raw_body = self.body + + # https://github.com/getmoto/moto/issues/6692 + # Content coming from SDK's can be GZipped for performance reasons + if ( + headers.get("Content-Encoding", "") == "gzip" + and self.allow_request_decompression + ): + self.body = gzip_decompress(self.body) + if isinstance(self.body, bytes) and not use_raw_body: self.body = self.body.decode("utf-8") diff --git a/moto/core/utils.py b/moto/core/utils.py index 813c4ad47..483ce1af7 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -3,6 +3,7 @@ import inspect import re import unicodedata from botocore.exceptions import ClientError +from gzip import decompress from typing import Any, Optional, List, Callable, Dict, Tuple from urllib.parse import urlparse, unquote from .common_types import TYPE_RESPONSE @@ -416,3 +417,7 @@ def _unquote_hex_characters(path: str) -> str: ) char_offset += (combo_end - combo_start) + len(character) - 1 return path + + +def gzip_decompress(body: bytes) -> bytes: + return decompress(body) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index e60027b21..701400b72 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -158,6 +158,11 @@ def parse_key_name(pth: str) -> str: class S3Response(BaseResponse): def __init__(self) -> None: super().__init__(service_name="s3") + # Whatever format requests come in, we should never touch them + # There are some nuances here - this decision should be method-specific, instead of service-specific + # E.G.: we don't want to touch put_object(), but we might have to decompress put_object_configuration() + # Taking the naive approach to never decompress anything from S3 for now + self.allow_request_decompression = False def get_safe_path_from_url(self, url: ParseResult) -> str: return self.get_safe_path(url.path) diff --git a/tests/test_core/test_responses.py b/tests/test_core/test_responses.py index 46ca6a45d..d33edb8ce 100644 --- a/tests/test_core/test_responses.py +++ b/tests/test_core/test_responses.py @@ -2,11 +2,13 @@ import datetime from unittest import SkipTest, mock from collections import OrderedDict +from gzip import compress as gzip_compress from botocore.awsrequest import AWSPreparedRequest from moto.core.responses import AWSServiceSpec, BaseResponse from moto.core.responses import flatten_json_request_body +from moto.s3.responses import S3Response from moto import settings from freezegun import freeze_time @@ -244,3 +246,38 @@ def test_response_metadata(): assert "date" in bc.response_headers if not settings.TEST_SERVER_MODE: assert bc.response_headers["date"] == "Sat, 20 May 2023 10:20:30 GMT" + + +def test_compression_gzip(): + body = '{"key": "%D0"}, "C": "#0 = :0"}' + request = AWSPreparedRequest( + "GET", + url="http://request", + headers={"Content-Encoding": "gzip"}, + body=_gzip_compress_body(body), + stream_output=False, + ) + response = BaseResponse() + response.setup_class(request, request.url, request.headers) + + assert body == response.body + + +def test_compression_gzip_in_s3(): + body = b"some random data" + request = AWSPreparedRequest( + "GET", + url="http://request", + headers={"Content-Encoding": "gzip"}, + body=body, + stream_output=False, + ) + response = S3Response() + response.setup_class(request, request.url, request.headers) + + assert body == response.body.encode("utf-8") + + +def _gzip_compress_body(body: str): + assert isinstance(body, str) + return gzip_compress(data=body.encode("utf-8"))