From 01449532738c9b16934619dde5463a5c1e74de24 Mon Sep 17 00:00:00 2001
From: Bert Blommers <bblommers@users.noreply.github.com>
Date: Thu, 25 May 2023 16:37:45 +0000
Subject: [PATCH] S3 Select: Parse null-values (#6343)

---
 moto/dynamodb/parsing/partiql.py |  7 +++++--
 moto/s3/models.py                |  9 +++++++--
 setup.cfg                        | 12 ++++++------
 tests/test_s3/test_s3_select.py  | 11 +++++++++--
 4 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/moto/dynamodb/parsing/partiql.py b/moto/dynamodb/parsing/partiql.py
index 78a92c345..b148921f1 100644
--- a/moto/dynamodb/parsing/partiql.py
+++ b/moto/dynamodb/parsing/partiql.py
@@ -1,4 +1,7 @@
-from typing import Any, Dict, List
+from typing import Any, Dict, List, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from py_partiql_parser import QueryMetadata
 
 
 def query(
@@ -9,7 +12,7 @@ def query(
     return DynamoDBStatementParser(source_data).parse(statement, parameters)
 
 
-def get_query_metadata(statement: str) -> Any:
+def get_query_metadata(statement: str) -> "QueryMetadata":
     from py_partiql_parser import DynamoDBStatementParser
 
     return DynamoDBStatementParser.get_query_metadata(query=statement)
diff --git a/moto/s3/models.py b/moto/s3/models.py
index c7f4e9527..ae744e532 100644
--- a/moto/s3/models.py
+++ b/moto/s3/models.py
@@ -2486,9 +2486,14 @@ class S3Backend(BaseBackend, CloudWatchMetricProvider):
 
             use_headers = input_details["CSV"].get("FileHeaderInfo", "") == "USE"
             query_input = csv_to_json(query_input, use_headers)
+        query_result = parse_query(query_input, select_query)
+        from py_partiql_parser import SelectEncoder
+
         return [
-            json.dumps(x, indent=None, separators=(",", ":")).encode("utf-8")
-            for x in parse_query(query_input, select_query)
+            json.dumps(x, indent=None, separators=(",", ":"), cls=SelectEncoder).encode(
+                "utf-8"
+            )
+            for x in query_result
         ]
 
 
diff --git a/setup.cfg b/setup.cfg
index 6d37533a8..d807a85df 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -52,7 +52,7 @@ all =
     openapi-spec-validator>=0.2.8
     pyparsing>=3.0.7
     jsondiff>=1.1.2
-    py-partiql-parser==0.3.0
+    py-partiql-parser==0.3.1
     aws-xray-sdk!=0.96,>=0.93
     setuptools
 server =
@@ -66,7 +66,7 @@ server =
     openapi-spec-validator>=0.2.8
     pyparsing>=3.0.7
     jsondiff>=1.1.2
-    py-partiql-parser==0.3.0
+    py-partiql-parser==0.3.1
     aws-xray-sdk!=0.96,>=0.93
     setuptools
     flask!=2.2.0,!=2.2.1
@@ -100,7 +100,7 @@ cloudformation =
     openapi-spec-validator>=0.2.8
     pyparsing>=3.0.7
     jsondiff>=1.1.2
-    py-partiql-parser==0.3.0
+    py-partiql-parser==0.3.1
     aws-xray-sdk!=0.96,>=0.93
     setuptools
 cloudfront =
@@ -123,10 +123,10 @@ dms =
 ds = sshpubkeys>=3.1.0
 dynamodb =
     docker>=3.0.0
-    py-partiql-parser==0.3.0
+    py-partiql-parser==0.3.1
 dynamodbstreams =
     docker>=3.0.0
-    py-partiql-parser==0.3.0
+    py-partiql-parser==0.3.1
 ebs = sshpubkeys>=3.1.0
 ec2 = sshpubkeys>=3.1.0
 ec2instanceconnect =
@@ -183,7 +183,7 @@ route53 =
 route53resolver = sshpubkeys>=3.1.0
 s3 =
     PyYAML>=5.1
-    py-partiql-parser==0.3.0
+    py-partiql-parser==0.3.1
 s3control =
 sagemaker =
 sdb =
diff --git a/tests/test_s3/test_s3_select.py b/tests/test_s3/test_s3_select.py
index aea6cc9eb..8b9d77a3f 100644
--- a/tests/test_s3/test_s3_select.py
+++ b/tests/test_s3/test_s3_select.py
@@ -6,7 +6,7 @@ from unittest import TestCase
 from uuid import uuid4
 
 
-SIMPLE_JSON = {"a1": "b1", "a2": "b2"}
+SIMPLE_JSON = {"a1": "b1", "a2": "b2", "a3": None}
 SIMPLE_JSON2 = {"a1": "b2", "a3": "b3"}
 SIMPLE_LIST = [SIMPLE_JSON, SIMPLE_JSON2]
 SIMPLE_CSV = """a,b,c
@@ -47,7 +47,14 @@ class TestS3Select(TestCase):
             OutputSerialization={"JSON": {"RecordDelimiter": ","}},
         )
         result = list(x["Payload"])
-        result.should.contain({"Records": {"Payload": b'{"a1":"b1","a2":"b2"},'}})
+        result.should.contain(
+            {"Records": {"Payload": b'{"a1":"b1","a2":"b2","a3":null},'}}
+        )
+
+        # Verify result is valid JSON
+        json.loads(result[0]["Records"]["Payload"][0:-1].decode("utf-8"))
+
+        # Verify result contains metadata
         result.should.contain(
             {
                 "Stats": {