S3 - key_name url encoding for listing (#5181)

moto/s3/models.py

@@ -14,6 +14,7 @@ import pytz
 import sys
 import time
 import uuid
+import urllib.parse
 
 from bisect import insort
 from importlib import reload
@@ -153,6 +154,10 @@ class FakeKey(BaseModel):
 
         self.s3_backend = s3_backend
 
+    @property
+    def safe_name(self):
+        return urllib.parse.quote(self.name, safe="")
+
     @property
     def version_id(self):
         return self._version_id

moto/s3/responses.py

@@ -590,6 +590,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
         result_keys, result_folders = self.backend.list_objects(
             bucket, prefix, delimiter
         )
+        encoding_type = querystring.get("encoding-type", [None])[0]
 
         if marker:
             result_keys = self._get_results_from_token(result_keys, marker)
@@ -611,6 +612,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
                 is_truncated=is_truncated,
                 next_marker=next_marker,
                 max_keys=max_keys,
+                encoding_type=encoding_type,
             ),
         )
 
@@ -642,6 +644,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
         fetch_owner = querystring.get("fetch-owner", [False])[0]
         max_keys = int(querystring.get("max-keys", [1000])[0])
         start_after = querystring.get("start-after", [None])[0]
+        encoding_type = querystring.get("encoding-type", [None])[0]
 
         if continuation_token or start_after:
             limit = continuation_token or start_after
@@ -666,6 +669,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
             is_truncated=is_truncated,
             next_continuation_token=next_continuation_token,
             start_after=None if continuation_token else start_after,
+            encoding_type=encoding_type,
         )
 
     @staticmethod
@@ -2051,13 +2055,16 @@ S3_BUCKET_GET_RESPONSE = """<?xml version="1.0" encoding="UTF-8"?>
   {% if delimiter %}
     <Delimiter>{{ delimiter }}</Delimiter>
   {% endif %}
+  {% if encoding_type %}
+    <EncodingType>{{ encoding_type }}</EncodingType>
+  {% endif %}
   <IsTruncated>{{ is_truncated }}</IsTruncated>
   {% if next_marker %}
     <NextMarker>{{ next_marker }}</NextMarker>
   {% endif %}
   {% for key in result_keys %}
     <Contents>
-      <Key>{{ key.name }}</Key>
+      <Key>{{ key.safe_name }}</Key>
       <LastModified>{{ key.last_modified_ISO8601 }}</LastModified>
       <ETag>{{ key.etag }}</ETag>
       <Size>{{ key.size }}</Size>
@@ -2087,6 +2094,9 @@ S3_BUCKET_GET_RESPONSE_V2 = """<?xml version="1.0" encoding="UTF-8"?>
   <KeyCount>{{ key_count }}</KeyCount>
 {% if delimiter %}
   <Delimiter>{{ delimiter }}</Delimiter>
+{% endif %}
+{% if encoding_type %}
+  <EncodingType>{{ encoding_type }}</EncodingType>
 {% endif %}
   <IsTruncated>{{ is_truncated }}</IsTruncated>
 {% if next_continuation_token %}
@@ -2097,7 +2107,7 @@ S3_BUCKET_GET_RESPONSE_V2 = """<?xml version="1.0" encoding="UTF-8"?>
 {% endif %}
   {% for key in result_keys %}
     <Contents>
-      <Key>{{ key.name }}</Key>
+      <Key>{{ key.safe_name }}</Key>
       <LastModified>{{ key.last_modified_ISO8601 }}</LastModified>
       <ETag>{{ key.etag }}</ETag>
       <Size>{{ key.size }}</Size>

tests/test_s3/test_s3.py

@@ -134,6 +134,24 @@ def test_empty_key():
     resp["Body"].read().should.equal(b"")
 
 
+@mock_s3
+def test_key_name_encoding_in_listing():
+    s3 = boto3.resource("s3", region_name=DEFAULT_REGION_NAME)
+    client = boto3.client("s3", region_name=DEFAULT_REGION_NAME)
+    s3.create_bucket(Bucket="foobar")
+
+    name = "6T7\x159\x12\r\x08.txt"
+
+    key = s3.Object("foobar", name)
+    key.put(Body=b"")
+
+    key_received = client.list_objects(Bucket="foobar")["Contents"][0]["Key"]
+    key_received.should.equal(name)
+
+    key_received = client.list_objects_v2(Bucket="foobar")["Contents"][0]["Key"]
+    key_received.should.equal(name)
+
+
 @mock_s3
 def test_empty_key_set_on_existing_key():
     s3 = boto3.resource("s3", region_name=DEFAULT_REGION_NAME)