Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Techdebt: Replace sure with regular assertions in Firehose (#6565)

Browse Source
This commit is contained in:
Bert Blommers 2023-07-27 22:25:43 +00:00 committed by GitHub
parent 689c93376a
commit a68a035038
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 166 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

257
tests/test_firehose/test_firehose_destination_types.py
View File

@ -1,6 +1,5 @@
"""Unit tests verifying various delivery stream destination content."""
import boto3
import sure # noqa # pylint: disable=unused-import
from moto import mock_firehose
from moto import settings
@ -118,43 +117,41 @@ def test_create_redshift_delivery_stream():
stream_description = response["DeliveryStreamDescription"]
# Sure and Freezegun don't play nicely together
_ = stream_description.pop("CreateTimestamp")
_ = stream_description.pop("LastUpdateTimestamp")
stream_description.pop("CreateTimestamp")
stream_description.pop("LastUpdateTimestamp")
stream_description.should.equal(
{
"DeliveryStreamName": stream_name,
"DeliveryStreamARN": stream_arn,
"DeliveryStreamStatus": "ACTIVE",
"DeliveryStreamType": "DirectPut",
"VersionId": "1",
"Destinations": [
{
"DestinationId": "destinationId-000000000001",
"RedshiftDestinationDescription": {
"RoleARN": f"arn:aws:iam::{ACCOUNT_ID}:role/firehose_delivery_role",
"ClusterJDBCURL": "jdbc:redshift://host.amazonaws.com:5439/database",
"CopyCommand": {
"DataTableName": "outputTable",
"CopyOptions": "CSV DELIMITER ',' NULL '\\0'",
},
"Username": "username",
"S3DestinationDescription": {
"RoleARN": f"arn:aws:iam::{ACCOUNT_ID}:role/firehose_delivery_role",
"BucketARN": "arn:aws:s3:::firehose-test",
"Prefix": "myFolder/",
"BufferingHints": {
"SizeInMBs": 123,
"IntervalInSeconds": 124,
},
"CompressionFormat": "UNCOMPRESSED",
},
assert stream_description == {
"DeliveryStreamName": stream_name,
"DeliveryStreamARN": stream_arn,
"DeliveryStreamStatus": "ACTIVE",
"DeliveryStreamType": "DirectPut",
"VersionId": "1",
"Destinations": [
{
"DestinationId": "destinationId-000000000001",
"RedshiftDestinationDescription": {
"RoleARN": f"arn:aws:iam::{ACCOUNT_ID}:role/firehose_delivery_role",
"ClusterJDBCURL": "jdbc:redshift://host.amazonaws.com:5439/database",
"CopyCommand": {
"DataTableName": "outputTable",
"CopyOptions": "CSV DELIMITER ',' NULL '\\0'",
},
}
],
"HasMoreDestinations": False,
}
)
"Username": "username",
"S3DestinationDescription": {
"RoleARN": f"arn:aws:iam::{ACCOUNT_ID}:role/firehose_delivery_role",
"BucketARN": "arn:aws:s3:::firehose-test",
"Prefix": "myFolder/",
"BufferingHints": {
"SizeInMBs": 123,
"IntervalInSeconds": 124,
},
"CompressionFormat": "UNCOMPRESSED",
},
},
}
],
"HasMoreDestinations": False,
}
@mock_firehose
@ -170,52 +167,48 @@ def test_create_extended_s3_delivery_stream():
stream_description = response["DeliveryStreamDescription"]
# Sure and Freezegun don't play nicely together
_ = stream_description.pop("CreateTimestamp")
_ = stream_description.pop("LastUpdateTimestamp")
stream_description.pop("CreateTimestamp")
stream_description.pop("LastUpdateTimestamp")
stream_description.should.equal(
{
"DeliveryStreamName": stream_name,
"DeliveryStreamARN": stream_arn,
"DeliveryStreamStatus": "ACTIVE",
"DeliveryStreamType": "DirectPut",
"VersionId": "1",
"Destinations": [
{
"DestinationId": "destinationId-000000000001",
"ExtendedS3DestinationDescription": {
"RoleARN": f"arn:aws:iam::{ACCOUNT_ID}:role/firehose_delivery_role",
"BucketARN": "arn:aws:s3:::firehose-test",
"Prefix": "myFolder/",
"CompressionFormat": "UNCOMPRESSED",
"DataFormatConversionConfiguration": {
"Enabled": True,
"InputFormatConfiguration": {
"Deserializer": {"HiveJsonSerDe": {}}
},
"OutputFormatConfiguration": {
"Serializer": {
"ParquetSerDe": {"Compression": "SNAPPY"}
}
},
"SchemaConfiguration": {
"DatabaseName": stream_name,
"RoleARN": f"arn:aws:iam::{ACCOUNT_ID}:role/firehose_delivery_role",
"TableName": "outputTable",
},
assert stream_description == {
"DeliveryStreamName": stream_name,
"DeliveryStreamARN": stream_arn,
"DeliveryStreamStatus": "ACTIVE",
"DeliveryStreamType": "DirectPut",
"VersionId": "1",
"Destinations": [
{
"DestinationId": "destinationId-000000000001",
"ExtendedS3DestinationDescription": {
"RoleARN": f"arn:aws:iam::{ACCOUNT_ID}:role/firehose_delivery_role",
"BucketARN": "arn:aws:s3:::firehose-test",
"Prefix": "myFolder/",
"CompressionFormat": "UNCOMPRESSED",
"DataFormatConversionConfiguration": {
"Enabled": True,
"InputFormatConfiguration": {
"Deserializer": {"HiveJsonSerDe": {}}
},
"OutputFormatConfiguration": {
"Serializer": {"ParquetSerDe": {"Compression": "SNAPPY"}}
},
"SchemaConfiguration": {
"DatabaseName": stream_name,
"RoleARN": f"arn:aws:iam::{ACCOUNT_ID}:role/firehose_delivery_role",
"TableName": "outputTable",
},
},
"S3DestinationDescription": {
"RoleARN": f"arn:aws:iam::{ACCOUNT_ID}:role/firehose_delivery_role",
"BucketARN": "arn:aws:s3:::firehose-test",
"Prefix": "myFolder/",
"CompressionFormat": "UNCOMPRESSED",
},
}
],
"HasMoreDestinations": False,
}
)
},
"S3DestinationDescription": {
"RoleARN": f"arn:aws:iam::{ACCOUNT_ID}:role/firehose_delivery_role",
"BucketARN": "arn:aws:s3:::firehose-test",
"Prefix": "myFolder/",
"CompressionFormat": "UNCOMPRESSED",
},
}
],
"HasMoreDestinations": False,
}
@mock_firehose
@ -234,40 +227,38 @@ def test_create_elasticsearch_delivery_stream():
_ = stream_description.pop("CreateTimestamp")
_ = stream_description.pop("LastUpdateTimestamp")
stream_description.should.equal(
{
"DeliveryStreamName": stream_name,
"DeliveryStreamARN": stream_arn,
"DeliveryStreamStatus": "ACTIVE",
"DeliveryStreamType": "DirectPut",
"VersionId": "1",
"Destinations": [
{
"DestinationId": "destinationId-000000000001",
"ElasticsearchDestinationDescription": {
assert stream_description == {
"DeliveryStreamName": stream_name,
"DeliveryStreamARN": stream_arn,
"DeliveryStreamStatus": "ACTIVE",
"DeliveryStreamType": "DirectPut",
"VersionId": "1",
"Destinations": [
{
"DestinationId": "destinationId-000000000001",
"ElasticsearchDestinationDescription": {
"RoleARN": f"arn:aws:iam::{ACCOUNT_ID}:role/firehose_delivery_role",
"DomainARN": "arn:aws:es:::domain/firehose-test",
"IndexName": "myIndex",
"TypeName": "UNCOMPRESSED",
"IndexRotationPeriod": "NoRotation",
"BufferingHints": {"IntervalInSeconds": 123, "SizeInMBs": 123},
"RetryOptions": {"DurationInSeconds": 123},
"S3DestinationDescription": {
"RoleARN": f"arn:aws:iam::{ACCOUNT_ID}:role/firehose_delivery_role",
"DomainARN": "arn:aws:es:::domain/firehose-test",
"IndexName": "myIndex",
"TypeName": "UNCOMPRESSED",
"IndexRotationPeriod": "NoRotation",
"BufferingHints": {"IntervalInSeconds": 123, "SizeInMBs": 123},
"RetryOptions": {"DurationInSeconds": 123},
"S3DestinationDescription": {
"RoleARN": f"arn:aws:iam::{ACCOUNT_ID}:role/firehose_delivery_role",
"BucketARN": "arn:aws:s3:::firehose-test",
"Prefix": "myFolder/",
"BufferingHints": {
"SizeInMBs": 123,
"IntervalInSeconds": 124,
},
"CompressionFormat": "UNCOMPRESSED",
"BucketARN": "arn:aws:s3:::firehose-test",
"Prefix": "myFolder/",
"BufferingHints": {
"SizeInMBs": 123,
"IntervalInSeconds": 124,
},
"CompressionFormat": "UNCOMPRESSED",
},
}
],
"HasMoreDestinations": False,
}
)
},
}
],
"HasMoreDestinations": False,
}
@mock_firehose
@ -292,28 +283,26 @@ def test_create_s3_delivery_stream():
stream_description = response["DeliveryStreamDescription"]
# Sure and Freezegun don't play nicely together
_ = stream_description.pop("CreateTimestamp")
_ = stream_description.pop("LastUpdateTimestamp")
stream_description.pop("CreateTimestamp")
stream_description.pop("LastUpdateTimestamp")
stream_description.should.equal(
{
"DeliveryStreamName": stream_name,
"DeliveryStreamARN": stream_arn,
"DeliveryStreamStatus": "ACTIVE",
"DeliveryStreamType": "DirectPut",
"VersionId": "1",
"Destinations": [
{
"DestinationId": "destinationId-000000000001",
"S3DestinationDescription": {
"RoleARN": f"arn:aws:iam::{ACCOUNT_ID}:role/firehose_delivery_role",
"BucketARN": "arn:aws:s3:::firehose-test",
"Prefix": "myFolder/",
"BufferingHints": {"SizeInMBs": 123, "IntervalInSeconds": 124},
"CompressionFormat": "UNCOMPRESSED",
},
}
],
"HasMoreDestinations": False,
}
)
assert stream_description == {
"DeliveryStreamName": stream_name,
"DeliveryStreamARN": stream_arn,
"DeliveryStreamStatus": "ACTIVE",
"DeliveryStreamType": "DirectPut",
"VersionId": "1",
"Destinations": [
{
"DestinationId": "destinationId-000000000001",
"S3DestinationDescription": {
"RoleARN": f"arn:aws:iam::{ACCOUNT_ID}:role/firehose_delivery_role",
"BucketARN": "arn:aws:s3:::firehose-test",
"Prefix": "myFolder/",
"BufferingHints": {"SizeInMBs": 123, "IntervalInSeconds": 124},
"CompressionFormat": "UNCOMPRESSED",
},
}
],
"HasMoreDestinations": False,
}

30
tests/test_firehose/test_firehose_encryption.py
View File

@ -19,7 +19,7 @@ def test_firehose_without_encryption():
resp = client.describe_delivery_stream(DeliveryStreamName=name)[
"DeliveryStreamDescription"
]
resp.shouldnt.have.key("DeliveryStreamEncryptionConfiguration")
assert "DeliveryStreamEncryptionConfiguration" not in resp
client.start_delivery_stream_encryption(
DeliveryStreamName=name,
@ -29,12 +29,10 @@ def test_firehose_without_encryption():
stream = client.describe_delivery_stream(DeliveryStreamName=name)[
"DeliveryStreamDescription"
]
stream.should.have.key("DeliveryStreamEncryptionConfiguration").equals(
{
"KeyType": "AWS_OWNED_CMK",
"Status": "ENABLED",
}
)
assert stream["DeliveryStreamEncryptionConfiguration"] == {
"KeyType": "AWS_OWNED_CMK",
"Status": "ENABLED",
}
@mock_firehose
@ -50,18 +48,16 @@ def test_firehose_with_encryption():
stream = client.describe_delivery_stream(DeliveryStreamName=name)[
"DeliveryStreamDescription"
]
stream.should.have.key("DeliveryStreamEncryptionConfiguration").equals(
{"KeyType": "AWS_OWNED_CMK"}
)
assert stream["DeliveryStreamEncryptionConfiguration"] == {
"KeyType": "AWS_OWNED_CMK"
}
client.stop_delivery_stream_encryption(DeliveryStreamName=name)
stream = client.describe_delivery_stream(DeliveryStreamName=name)[
"DeliveryStreamDescription"
]
stream.should.have.key("DeliveryStreamEncryptionConfiguration").should.have.key(
"Status"
).equals("DISABLED")
assert stream["DeliveryStreamEncryptionConfiguration"]["Status"] == "DISABLED"
@mock_firehose
@ -74,8 +70,8 @@ def test_start_encryption_on_unknown_stream():
DeliveryStreamEncryptionConfigurationInput={"KeyType": "AWS_OWNED_CMK"},
)
err = exc.value.response["Error"]
err["Code"].should.equal("ResourceNotFoundException")
err["Message"].should.equal("Firehose ? under account 123456789012 not found.")
assert err["Code"] == "ResourceNotFoundException"
assert err["Message"] == "Firehose ? under account 123456789012 not found."
@mock_firehose
@ -85,5 +81,5 @@ def test_stop_encryption_on_unknown_stream():
with pytest.raises(ClientError) as exc:
client.stop_delivery_stream_encryption(DeliveryStreamName="?")
err = exc.value.response["Error"]
err["Code"].should.equal("ResourceNotFoundException")
err["Message"].should.equal("Firehose ? under account 123456789012 not found.")
assert err["Code"] == "ResourceNotFoundException"
assert err["Message"] == "Firehose ? under account 123456789012 not found."

1
tests/test_firehose/test_firehose_put.py
View File

@ -1,6 +1,5 @@
"""Unit tests verifying put-related delivery stream APIs."""
import boto3
import sure # noqa pylint: disable=unused-import
from moto import mock_firehose
from moto import mock_s3

63
tests/test_firehose/test_http_destinations.py
View File

@ -1,5 +1,4 @@
import boto3
import sure # noqa # pylint: disable=unused-import
from moto import mock_firehose
from moto.core import DEFAULT_ACCOUNT_ID as ACCOUNT_ID
@ -22,40 +21,38 @@ def test_create_http_stream():
stream_description = response["DeliveryStreamDescription"]
# Sure and Freezegun don't play nicely together
_ = stream_description.pop("CreateTimestamp")
_ = stream_description.pop("LastUpdateTimestamp")
stream_description.pop("CreateTimestamp")
stream_description.pop("LastUpdateTimestamp")
stream_description.should.equal(
{
"DeliveryStreamName": stream_name,
"DeliveryStreamARN": stream_arn,
"DeliveryStreamStatus": "ACTIVE",
"DeliveryStreamType": "DirectPut",
"VersionId": "1",
"Destinations": [
{
"DestinationId": "destinationId-000000000001",
"HttpEndpointDestinationDescription": {
"EndpointConfiguration": {"Url": "google.com"},
"RetryOptions": {"DurationInSeconds": 100},
"BufferingHints": {"SizeInMBs": 123, "IntervalInSeconds": 124},
"CloudWatchLoggingOptions": {"Enabled": False},
"S3DestinationDescription": {
"RoleARN": f"arn:aws:iam::{ACCOUNT_ID}:role/firehose_delivery_role",
"BucketARN": "arn:aws:s3:::firehose-test",
"Prefix": "myFolder/",
"BufferingHints": {
"SizeInMBs": 123,
"IntervalInSeconds": 124,
},
"CompressionFormat": "UNCOMPRESSED",
assert stream_description == {
"DeliveryStreamName": stream_name,
"DeliveryStreamARN": stream_arn,
"DeliveryStreamStatus": "ACTIVE",
"DeliveryStreamType": "DirectPut",
"VersionId": "1",
"Destinations": [
{
"DestinationId": "destinationId-000000000001",
"HttpEndpointDestinationDescription": {
"EndpointConfiguration": {"Url": "google.com"},
"RetryOptions": {"DurationInSeconds": 100},
"BufferingHints": {"SizeInMBs": 123, "IntervalInSeconds": 124},
"CloudWatchLoggingOptions": {"Enabled": False},
"S3DestinationDescription": {
"RoleARN": f"arn:aws:iam::{ACCOUNT_ID}:role/firehose_delivery_role",
"BucketARN": "arn:aws:s3:::firehose-test",
"Prefix": "myFolder/",
"BufferingHints": {
"SizeInMBs": 123,
"IntervalInSeconds": 124,
},
"CompressionFormat": "UNCOMPRESSED",
},
}
],
"HasMoreDestinations": False,
}
)
},
}
],
"HasMoreDestinations": False,
}
@mock_firehose
@ -82,4 +79,4 @@ def test_update_s3_for_http_stream():
s3_desc = desc["Destinations"][0]["HttpEndpointDestinationDescription"][
"S3DestinationDescription"
]
s3_desc.should.have.key("ErrorOutputPrefix").equals("prefix2")
assert s3_desc["ErrorOutputPrefix"] == "prefix2"