S3: Optional support for CRC32C (#6534)

moto/s3/models.py

@@ -1523,12 +1523,20 @@ class S3Backend(BaseBackend, CloudWatchMetricProvider):
 
     Note that this only works if the environment variable is set **before** the mock is initialized.
 
+    ------------------------------------
+
     When using the MultiPart-API manually, the minimum part size is 5MB, just as with AWS. Use the following environment variable to lower this:
 
     .. sourcecode:: bash
 
         S3_UPLOAD_PART_MIN_SIZE=256
 
+    ------------------------------------
+
+    Install `moto[s3crc32c]` if you use the CRC32C algorithm, and absolutely need the correct value. Alternatively, you can install the `crc32c` dependency manually.
+
+    If this dependency is not installed, Moto will fall-back to the CRC32-computation when computing checksums.
+
     """
 
     def __init__(self, region_name: str, account_id: str):

moto/s3/utils.py

@@ -199,7 +199,15 @@ class _VersionedKeyStore(dict):  # type: ignore
 def compute_checksum(body: bytes, algorithm: str) -> bytes:
     if algorithm == "SHA1":
         hashed_body = _hash(hashlib.sha1, (body,))
-    elif algorithm == "CRC32" or algorithm == "CRC32C":
+    elif algorithm == "CRC32C":
+        try:
+            import crc32c
+
+            hashed_body = crc32c.crc32c(body).to_bytes(4, "big")
+        except:  # noqa: E722 Do not use bare except
+            # Optional library Can't be found - just revert to CRC32
+            hashed_body = binascii.crc32(body).to_bytes(4, "big")
+    elif algorithm == "CRC32":
         hashed_body = binascii.crc32(body).to_bytes(4, "big")
     else:
         hashed_body = _hash(hashlib.sha256, (body,))

setup.cfg

@@ -184,6 +184,10 @@ route53resolver = sshpubkeys>=3.1.0
 s3 =
     PyYAML>=5.1
     py-partiql-parser==0.3.3
+s3crc32c =
+    PyYAML>=5.1
+    py-partiql-parser==0.3.3
+    crc32c
 s3control =
 sagemaker =
 sdb =

tests/test_s3/test_s3_utils.py

@@ -139,7 +139,13 @@ def test_checksum_crc32():
 
 
 def test_checksum_crc32c():
-    compute_checksum(b"somedata", "CRC32C").should.equal(b"Uwy90A==")
+    try:
+        import crc32c  # noqa # pylint: disable=unused-import
+
+        compute_checksum(b"somedata", "CRC32C").should.equal(b"dB9qBQ==")
+    except:  # noqa: E722 Do not use bare except
+        # Optional library Can't be found - just revert to CRC32
+        compute_checksum(b"somedata", "CRC32C").should.equal(b"Uwy90A==")
 
 
 def test_cors_utils():