EBS: Some fixes to snapshot uploading (#6520)

* Two fixes to support Citrix AWS hypervisor plugin: ebs responses for putting a block and completing a snapshot should be 201 and 202 respectively rather than 200; If the body cannot be utf-8 decoded, try base64.Z

* Update responses.py

* Update responses.py

* Update responses.py

* Add EBS snapshot unit test

* EBS: put_snapshot_block() should support raw bytes

---------

Co-authored-by: Demetrios Tsillas <Demetrios.Tsillas@citrix.com>
Co-authored-by: Bert Blommers <info@bertblommers.nl>

.github/workflows/tests_sdk_dotnet.yml

@@ -28,8 +28,8 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-nuget
     - name: Install dependencies
-      run: cd other_langs/tests_dotnet && dotnet restore ExampleTestProject/
+      run: cd other_langs/tests_dotnet && dotnet restore ExampleTestProject/ && dotnet restore ebs
     - name: Run tests
       run: |
         mkdir ~/.aws && touch ~/.aws/credentials && echo -e "[default]\naws_access_key_id = test\naws_secret_access_key = test" > ~/.aws/credentials
-        cd other_langs/tests_dotnet && dotnet test ExampleTestProject/
+        cd other_langs/tests_dotnet && dotnet test ExampleTestProject/ && dotnet restore ebs

moto/ebs/responses.py

@@ -24,7 +24,7 @@ class EBSResponse(BaseResponse):
             return self.start_snapshot()
 
     def snapshot_block(self, request: Any, full_url: str, headers: Any) -> TYPE_RESPONSE:  # type: ignore[return]
-        self.setup_class(request, full_url, headers)
+        self.setup_class(request, full_url, headers, use_raw_body=True)
         if request.method == "PUT":
             return self.put_snapshot_block(full_url, headers)
         if request.method == "GET":
@@ -59,7 +59,7 @@ class EBSResponse(BaseResponse):
         self.setup_class(request, full_url, headers)
         snapshot_id = full_url.split("/")[-1]
         status = self.ebs_backend.complete_snapshot(snapshot_id=snapshot_id)
-        return 200, {}, json.dumps(status)
+        return 202, {}, json.dumps(status)
 
     def put_snapshot_block(self, full_url: str, headers: Any) -> TYPE_RESPONSE:
         """
@@ -82,7 +82,7 @@ class EBSResponse(BaseResponse):
             data_length=data_length,
         )
         return (
-            200,
+            201,
             {
                 "x-amz-Checksum": checksum,
                 "x-amz-Checksum-Algorithm": checksum_algorithm,

other_langs/tests_dotnet/ebs/ElasticBlockStore.csproj

@@ -0,0 +1,26 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+
+	  <IsPackable>true</IsPackable>
+	  <PackAsTool>true</PackAsTool>
+
+    <UserSecretsId>45a910a7-aba2-43ee-ad3c-ccad57f044d9</UserSecretsId>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="AWSSDK.EBS" Version="3.7.100.153" />
+    <PackageReference Include="FluentAssertions" Version="6.8.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="7.0.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
+    <PackageReference Include="xunit" Version="2.4.1" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+  </ItemGroup>
+
+</Project>

other_langs/tests_dotnet/ebs/UnitTest.cs

@@ -0,0 +1,56 @@
+using FluentAssertions;
+
+// To interact with Amazon EBS
+using Amazon.EBS;
+using Amazon.EBS.Model;
+
+using System.Security.Cryptography;
+
+
+public class EbsSnashotTest
+{
+    [Fact]
+    public async Task TestSnapshotCreation()
+    {
+
+        AmazonEBSConfig config = new AmazonEBSConfig()
+        {
+            ServiceURL = "http://localhost:5000",
+        };
+
+        AmazonEBSClient ebsClient = new AmazonEBSClient(config);
+
+        // https://docs.aws.amazon.com/ebs/latest/APIReference/API_StartSnapshot.html
+        var startSnapshotRequest = new StartSnapshotRequest { VolumeSize = 1 };
+        var startSnapshotResponse = await ebsClient.StartSnapshotAsync(startSnapshotRequest);
+        startSnapshotResponse.Status.Should().Be("pending");    
+
+        // https://docs.aws.amazon.com/ebs/latest/APIReference/API_PutSnapshotBlock.html
+        var blockData = new byte[] { 0x01, 0x02, 0x03, 0x04 };
+        Stream blockDataStream = new MemoryStream(blockData);
+        SHA256Managed sha256hasher = new SHA256Managed();
+        byte[] sha256Hash = sha256hasher.ComputeHash(blockDataStream);
+        var sha256checksum = Convert.ToBase64String(sha256Hash);
+        var putSnapshotBlockRequest = new PutSnapshotBlockRequest
+        {
+            BlockIndex = 0,
+            Checksum = sha256checksum,
+            ChecksumAlgorithm = "SHA256",
+            DataLength = blockData.Length,
+            SnapshotId = startSnapshotResponse.SnapshotId,
+            BlockData = blockDataStream
+        };
+        var putSnapshotBlockResponse = await ebsClient.PutSnapshotBlockAsync(putSnapshotBlockRequest);
+        putSnapshotBlockResponse.Checksum.Should().Be(sha256checksum);
+        putSnapshotBlockResponse.ChecksumAlgorithm.Should().Be("SHA256");
+
+        https://docs.aws.amazon.com/ebs/latest/APIReference/API_CompleteSnapshot.html
+        var completeSnapshotRequest = new CompleteSnapshotRequest
+        {
+            ChangedBlocksCount = 1,
+            SnapshotId = startSnapshotResponse.SnapshotId
+        };
+        var completeSnapshotResponse = await ebsClient.CompleteSnapshotAsync(completeSnapshotRequest);
+        completeSnapshotResponse.Status.Should().Be("completed");
+    }
+}

other_langs/tests_dotnet/ebs/Usings.cs

@@ -0,0 +1 @@
+global using Xunit;

tests/test_ebs/test_ebs.py

@@ -51,7 +51,7 @@ def test_complete_snapshot():
 
 @mock_ebs
 def test_put_snapshot_block():
-    data = b"data for this specific block"
+    data = b"data for this specific block\xbf"
     checksum = hashlib.sha256(data).hexdigest()
     client = boto3.client("ebs", region_name="eu-west-1")
     snapshot_id = client.start_snapshot(VolumeSize=720)["SnapshotId"]