S3 Select: Parse null-values (#6343)

moto/dynamodb/parsing/partiql.py

@@ -1,4 +1,7 @@
-from typing import Any, Dict, List
+from typing import Any, Dict, List, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from py_partiql_parser import QueryMetadata
 
 
 def query(
@@ -9,7 +12,7 @@ def query(
     return DynamoDBStatementParser(source_data).parse(statement, parameters)
 
 
-def get_query_metadata(statement: str) -> Any:
+def get_query_metadata(statement: str) -> "QueryMetadata":
     from py_partiql_parser import DynamoDBStatementParser
 
     return DynamoDBStatementParser.get_query_metadata(query=statement)

moto/s3/models.py

@@ -2486,9 +2486,14 @@ class S3Backend(BaseBackend, CloudWatchMetricProvider):
 
             use_headers = input_details["CSV"].get("FileHeaderInfo", "") == "USE"
             query_input = csv_to_json(query_input, use_headers)
+        query_result = parse_query(query_input, select_query)
+        from py_partiql_parser import SelectEncoder
+
         return [
-            json.dumps(x, indent=None, separators=(",", ":")).encode("utf-8")
+            json.dumps(x, indent=None, separators=(",", ":"), cls=SelectEncoder).encode(
-            for x in parse_query(query_input, select_query)
+                "utf-8"
+            )
+            for x in query_result
         ]
 
 

setup.cfg

@@ -52,7 +52,7 @@ all =
     openapi-spec-validator>=0.2.8
     pyparsing>=3.0.7
     jsondiff>=1.1.2
-    py-partiql-parser==0.3.0
+    py-partiql-parser==0.3.1
     aws-xray-sdk!=0.96,>=0.93
     setuptools
 server =
@@ -66,7 +66,7 @@ server =
     openapi-spec-validator>=0.2.8
     pyparsing>=3.0.7
     jsondiff>=1.1.2
-    py-partiql-parser==0.3.0
+    py-partiql-parser==0.3.1
     aws-xray-sdk!=0.96,>=0.93
     setuptools
     flask!=2.2.0,!=2.2.1
@@ -100,7 +100,7 @@ cloudformation =
     openapi-spec-validator>=0.2.8
     pyparsing>=3.0.7
     jsondiff>=1.1.2
-    py-partiql-parser==0.3.0
+    py-partiql-parser==0.3.1
     aws-xray-sdk!=0.96,>=0.93
     setuptools
 cloudfront =
@@ -123,10 +123,10 @@ dms =
 ds = sshpubkeys>=3.1.0
 dynamodb =
     docker>=3.0.0
-    py-partiql-parser==0.3.0
+    py-partiql-parser==0.3.1
 dynamodbstreams =
     docker>=3.0.0
-    py-partiql-parser==0.3.0
+    py-partiql-parser==0.3.1
 ebs = sshpubkeys>=3.1.0
 ec2 = sshpubkeys>=3.1.0
 ec2instanceconnect =
@@ -183,7 +183,7 @@ route53 =
 route53resolver = sshpubkeys>=3.1.0
 s3 =
     PyYAML>=5.1
-    py-partiql-parser==0.3.0
+    py-partiql-parser==0.3.1
 s3control =
 sagemaker =
 sdb =

tests/test_s3/test_s3_select.py

@@ -6,7 +6,7 @@ from unittest import TestCase
 from uuid import uuid4
 
 
-SIMPLE_JSON = {"a1": "b1", "a2": "b2"}
+SIMPLE_JSON = {"a1": "b1", "a2": "b2", "a3": None}
 SIMPLE_JSON2 = {"a1": "b2", "a3": "b3"}
 SIMPLE_LIST = [SIMPLE_JSON, SIMPLE_JSON2]
 SIMPLE_CSV = """a,b,c
@@ -47,7 +47,14 @@ class TestS3Select(TestCase):
             OutputSerialization={"JSON": {"RecordDelimiter": ","}},
         )
         result = list(x["Payload"])
-        result.should.contain({"Records": {"Payload": b'{"a1":"b1","a2":"b2"},'}})
+        result.should.contain(
+            {"Records": {"Payload": b'{"a1":"b1","a2":"b2","a3":null},'}}
+        )
+
+        # Verify result is valid JSON
+        json.loads(result[0]["Records"]["Payload"][0:-1].decode("utf-8"))
+
+        # Verify result contains metadata
         result.should.contain(
             {
                 "Stats": {