Core: Allow request decompression (#6726)

moto/core/responses.py

@@ -14,6 +14,7 @@ from moto.core.common_types import TYPE_RESPONSE, TYPE_IF_NONE
 from moto.core.exceptions import DryRunClientError
 from moto.core.utils import (
     camelcase_to_underscores,
+    gzip_decompress,
     method_names_from_class,
     params_sort_function,
 )
@@ -232,6 +233,7 @@ class BaseResponse(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
     def __init__(self, service_name: Optional[str] = None):
         super().__init__()
         self.service_name = service_name
+        self.allow_request_decompression = True
 
     @classmethod
     def dispatch(cls, *args: Any, **kwargs: Any) -> Any:  # type: ignore[misc]
@@ -262,6 +264,15 @@ class BaseResponse(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
                 querystring[key] = [value]
 
         raw_body = self.body
+
+        # https://github.com/getmoto/moto/issues/6692
+        # Content coming from SDK's can be GZipped for performance reasons
+        if (
+            headers.get("Content-Encoding", "") == "gzip"
+            and self.allow_request_decompression
+        ):
+            self.body = gzip_decompress(self.body)
+
         if isinstance(self.body, bytes) and not use_raw_body:
             self.body = self.body.decode("utf-8")
 

moto/core/utils.py

@@ -3,6 +3,7 @@ import inspect
 import re
 import unicodedata
 from botocore.exceptions import ClientError
+from gzip import decompress
 from typing import Any, Optional, List, Callable, Dict, Tuple
 from urllib.parse import urlparse, unquote
 from .common_types import TYPE_RESPONSE
@@ -416,3 +417,7 @@ def _unquote_hex_characters(path: str) -> str:
         )
         char_offset += (combo_end - combo_start) + len(character) - 1
     return path
+
+
+def gzip_decompress(body: bytes) -> bytes:
+    return decompress(body)

moto/s3/responses.py

@@ -158,6 +158,11 @@ def parse_key_name(pth: str) -> str:
 class S3Response(BaseResponse):
     def __init__(self) -> None:
         super().__init__(service_name="s3")
+        # Whatever format requests come in, we should never touch them
+        # There are some nuances here - this decision should be method-specific, instead of service-specific
+        # E.G.: we don't want to touch put_object(), but we might have to decompress put_object_configuration()
+        # Taking the naive approach to never decompress anything from S3 for now
+        self.allow_request_decompression = False
 
     def get_safe_path_from_url(self, url: ParseResult) -> str:
         return self.get_safe_path(url.path)

tests/test_core/test_responses.py

@@ -2,11 +2,13 @@ import datetime
 from unittest import SkipTest, mock
 
 from collections import OrderedDict
+from gzip import compress as gzip_compress
 
 from botocore.awsrequest import AWSPreparedRequest
 
 from moto.core.responses import AWSServiceSpec, BaseResponse
 from moto.core.responses import flatten_json_request_body
+from moto.s3.responses import S3Response
 from moto import settings
 from freezegun import freeze_time
 
@@ -244,3 +246,38 @@ def test_response_metadata():
     assert "date" in bc.response_headers
     if not settings.TEST_SERVER_MODE:
         assert bc.response_headers["date"] == "Sat, 20 May 2023 10:20:30 GMT"
+
+
+def test_compression_gzip():
+    body = '{"key": "%D0"}, "C": "#0 = :0"}'
+    request = AWSPreparedRequest(
+        "GET",
+        url="http://request",
+        headers={"Content-Encoding": "gzip"},
+        body=_gzip_compress_body(body),
+        stream_output=False,
+    )
+    response = BaseResponse()
+    response.setup_class(request, request.url, request.headers)
+
+    assert body == response.body
+
+
+def test_compression_gzip_in_s3():
+    body = b"some random data"
+    request = AWSPreparedRequest(
+        "GET",
+        url="http://request",
+        headers={"Content-Encoding": "gzip"},
+        body=body,
+        stream_output=False,
+    )
+    response = S3Response()
+    response.setup_class(request, request.url, request.headers)
+
+    assert body == response.body.encode("utf-8")
+
+
+def _gzip_compress_body(body: str):
+    assert isinstance(body, str)
+    return gzip_compress(data=body.encode("utf-8"))