348 lines
12 KiB
Python
348 lines
12 KiB
Python
import boto3
|
|
import pytest
|
|
from botocore.exceptions import ClientError
|
|
|
|
from moto import mock_aws
|
|
from moto.core import DEFAULT_ACCOUNT_ID as ACCOUNT_ID
|
|
|
|
from .test_kinesis import get_stream_arn
|
|
|
|
|
|
@mock_aws
|
|
def test_describe_stream_limit_parameter():
|
|
client = boto3.client("kinesis", region_name="us-west-2")
|
|
stream_name = "my_stream"
|
|
|
|
client.create_stream(StreamName=stream_name, ShardCount=5)
|
|
|
|
without_filter = client.describe_stream(StreamName=stream_name)["StreamDescription"]
|
|
assert len(without_filter["Shards"]) == 5
|
|
assert without_filter["HasMoreShards"] is False
|
|
|
|
with_filter = client.describe_stream(StreamName=stream_name, Limit=2)[
|
|
"StreamDescription"
|
|
]
|
|
assert len(with_filter["Shards"]) == 2
|
|
assert with_filter["HasMoreShards"] is True
|
|
|
|
with_filter = client.describe_stream(StreamName=stream_name, Limit=5)[
|
|
"StreamDescription"
|
|
]
|
|
assert len(with_filter["Shards"]) == 5
|
|
assert with_filter["HasMoreShards"] is False
|
|
|
|
with_filter = client.describe_stream(StreamName=stream_name, Limit=6)[
|
|
"StreamDescription"
|
|
]
|
|
assert len(with_filter["Shards"]) == 5
|
|
assert with_filter["HasMoreShards"] is False
|
|
|
|
|
|
@mock_aws
|
|
def test_list_shards():
|
|
conn = boto3.client("kinesis", region_name="us-west-2")
|
|
stream_name = "my_stream"
|
|
|
|
conn.create_stream(StreamName=stream_name, ShardCount=2)
|
|
|
|
# Create some data
|
|
for index in range(1, 100):
|
|
conn.put_record(
|
|
StreamName=stream_name, Data="data:" + str(index), PartitionKey=str(index)
|
|
)
|
|
|
|
shard_list = conn.list_shards(StreamName=stream_name)["Shards"]
|
|
assert len(shard_list) == 2
|
|
# Verify IDs
|
|
assert [s["ShardId"] for s in shard_list] == [
|
|
"shardId-000000000000",
|
|
"shardId-000000000001",
|
|
]
|
|
# Verify hash range
|
|
for shard in shard_list:
|
|
assert "HashKeyRange" in shard
|
|
assert "StartingHashKey" in shard["HashKeyRange"]
|
|
assert "EndingHashKey" in shard["HashKeyRange"]
|
|
assert shard_list[0]["HashKeyRange"]["EndingHashKey"] == str(
|
|
int(shard_list[1]["HashKeyRange"]["StartingHashKey"]) - 1
|
|
)
|
|
# Verify sequence numbers
|
|
for shard in shard_list:
|
|
assert "SequenceNumberRange" in shard
|
|
assert "StartingSequenceNumber" in shard["SequenceNumberRange"]
|
|
|
|
|
|
@mock_aws
|
|
def test_list_shards_paging():
|
|
client = boto3.client("kinesis", region_name="us-west-2")
|
|
stream_name = "my_stream"
|
|
client.create_stream(StreamName=stream_name, ShardCount=10)
|
|
|
|
# Get shard 1-10
|
|
shard_list = client.list_shards(StreamName=stream_name)
|
|
assert len(shard_list["Shards"]) == 10
|
|
assert "NextToken" not in shard_list
|
|
|
|
# Get shard 1-4
|
|
resp = client.list_shards(StreamName=stream_name, MaxResults=4)
|
|
assert len(resp["Shards"]) == 4
|
|
assert [s["ShardId"] for s in resp["Shards"]] == [
|
|
"shardId-000000000000",
|
|
"shardId-000000000001",
|
|
"shardId-000000000002",
|
|
"shardId-000000000003",
|
|
]
|
|
assert "NextToken" in resp
|
|
|
|
# Get shard 4-8
|
|
resp = client.list_shards(
|
|
StreamName=stream_name, MaxResults=4, NextToken=str(resp["NextToken"])
|
|
)
|
|
assert len(resp["Shards"]) == 4
|
|
assert [s["ShardId"] for s in resp["Shards"]] == [
|
|
"shardId-000000000004",
|
|
"shardId-000000000005",
|
|
"shardId-000000000006",
|
|
"shardId-000000000007",
|
|
]
|
|
assert "NextToken" in resp
|
|
|
|
# Get shard 8-10
|
|
resp = client.list_shards(
|
|
StreamName=stream_name, MaxResults=4, NextToken=str(resp["NextToken"])
|
|
)
|
|
assert len(resp["Shards"]) == 2
|
|
assert [s["ShardId"] for s in resp["Shards"]] == [
|
|
"shardId-000000000008",
|
|
"shardId-000000000009",
|
|
]
|
|
assert "NextToken" not in resp
|
|
|
|
|
|
@mock_aws
|
|
def test_create_shard():
|
|
client = boto3.client("kinesis", region_name="us-west-2")
|
|
client.create_stream(StreamName="my-stream", ShardCount=2)
|
|
|
|
resp = client.describe_stream(StreamName="my-stream")
|
|
desc = resp["StreamDescription"]
|
|
assert desc["StreamName"] == "my-stream"
|
|
assert (
|
|
desc["StreamARN"] == f"arn:aws:kinesis:us-west-2:{ACCOUNT_ID}:stream/my-stream"
|
|
)
|
|
assert len(desc["Shards"]) == 2
|
|
assert desc["StreamStatus"] == "ACTIVE"
|
|
assert desc["HasMoreShards"] is False
|
|
assert desc["RetentionPeriodHours"] == 24
|
|
assert "StreamCreationTimestamp" in desc
|
|
assert desc["EnhancedMonitoring"] == [{"ShardLevelMetrics": []}]
|
|
assert desc["EncryptionType"] == "NONE"
|
|
|
|
shards = desc["Shards"]
|
|
assert shards[0]["ShardId"] == "shardId-000000000000"
|
|
assert "HashKeyRange" in shards[0]
|
|
assert shards[0]["HashKeyRange"]["StartingHashKey"] == "0"
|
|
assert "EndingHashKey" in shards[0]["HashKeyRange"]
|
|
assert "SequenceNumberRange" in shards[0]
|
|
assert "StartingSequenceNumber" in shards[0]["SequenceNumberRange"]
|
|
assert "EndingSequenceNumber" not in shards[0]["SequenceNumberRange"]
|
|
|
|
|
|
@mock_aws
|
|
def test_split_shard_with_invalid_name():
|
|
client = boto3.client("kinesis", region_name="us-west-2")
|
|
client.create_stream(StreamName="my-stream", ShardCount=2)
|
|
|
|
with pytest.raises(ClientError) as exc:
|
|
client.split_shard(
|
|
StreamName="my-stream",
|
|
ShardToSplit="?",
|
|
NewStartingHashKey="170141183460469231731687303715884105728",
|
|
)
|
|
err = exc.value.response["Error"]
|
|
assert err["Code"] == "ValidationException"
|
|
assert (
|
|
err["Message"]
|
|
== "1 validation error detected: Value '?' at 'shardToSplit' failed to satisfy constraint: Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+"
|
|
)
|
|
|
|
|
|
@mock_aws
|
|
def test_split_shard_with_unknown_name():
|
|
client = boto3.client("kinesis", region_name="us-west-2")
|
|
client.create_stream(StreamName="my-stream", ShardCount=2)
|
|
|
|
with pytest.raises(ClientError) as exc:
|
|
client.split_shard(
|
|
StreamName="my-stream",
|
|
ShardToSplit="unknown",
|
|
NewStartingHashKey="170141183460469231731687303715884105728",
|
|
)
|
|
err = exc.value.response["Error"]
|
|
assert err["Code"] == "ResourceNotFoundException"
|
|
assert (
|
|
err["Message"]
|
|
== "Could not find shard unknown in stream my-stream under account 123456789012."
|
|
)
|
|
|
|
|
|
@mock_aws
|
|
def test_split_shard_invalid_hashkey():
|
|
client = boto3.client("kinesis", region_name="us-west-2")
|
|
client.create_stream(StreamName="my-stream", ShardCount=2)
|
|
|
|
with pytest.raises(ClientError) as exc:
|
|
client.split_shard(
|
|
StreamName="my-stream",
|
|
ShardToSplit="shardId-000000000001",
|
|
NewStartingHashKey="sth",
|
|
)
|
|
err = exc.value.response["Error"]
|
|
assert err["Code"] == "ValidationException"
|
|
assert (
|
|
err["Message"]
|
|
== "1 validation error detected: Value 'sth' at 'newStartingHashKey' failed to satisfy constraint: Member must satisfy regular expression pattern: 0|([1-9]\\d{0,38})"
|
|
)
|
|
|
|
|
|
@mock_aws
|
|
def test_split_shard_hashkey_out_of_bounds():
|
|
client = boto3.client("kinesis", region_name="us-west-2")
|
|
client.create_stream(StreamName="my-stream", ShardCount=2)
|
|
|
|
with pytest.raises(ClientError) as exc:
|
|
client.split_shard(
|
|
StreamName="my-stream",
|
|
ShardToSplit="shardId-000000000001",
|
|
NewStartingHashKey="170141183460469231731687303715884000000",
|
|
)
|
|
err = exc.value.response["Error"]
|
|
assert err["Code"] == "InvalidArgumentException"
|
|
assert (
|
|
err["Message"]
|
|
== f"NewStartingHashKey 170141183460469231731687303715884000000 used in SplitShard() on shard shardId-000000000001 in stream my-stream under account {ACCOUNT_ID} is not both greater than one plus the shard's StartingHashKey 170141183460469231731687303715884105728 and less than the shard's EndingHashKey 340282366920938463463374607431768211455."
|
|
)
|
|
|
|
|
|
@mock_aws
|
|
def test_split_shard():
|
|
client = boto3.client("kinesis", region_name="us-west-2")
|
|
stream_name = "my-stream"
|
|
client.create_stream(StreamName=stream_name, ShardCount=2)
|
|
|
|
for index in range(1, 100):
|
|
client.put_record(
|
|
StreamName=stream_name,
|
|
Data=f"data_{index}".encode("utf-8"),
|
|
PartitionKey=str(index),
|
|
)
|
|
|
|
original_shards = client.describe_stream(StreamName=stream_name)[
|
|
"StreamDescription"
|
|
]["Shards"]
|
|
|
|
client.split_shard(
|
|
StreamName=stream_name,
|
|
ShardToSplit="shardId-000000000001",
|
|
NewStartingHashKey="170141183460469231731687303715884105829",
|
|
)
|
|
|
|
resp = client.describe_stream(StreamName=stream_name)["StreamDescription"]
|
|
shards = resp["Shards"]
|
|
assert len(shards) == 4
|
|
assert shards[0]["ShardId"] == "shardId-000000000000"
|
|
assert "HashKeyRange" in shards[0]
|
|
assert "ParentShardId" not in shards[0]
|
|
|
|
assert shards[1]["ShardId"] == "shardId-000000000001"
|
|
assert "ParentShardId" not in shards[1]
|
|
assert "HashKeyRange" in shards[1]
|
|
assert (
|
|
shards[1]["HashKeyRange"]["StartingHashKey"]
|
|
== original_shards[1]["HashKeyRange"]["StartingHashKey"]
|
|
)
|
|
assert (
|
|
shards[1]["HashKeyRange"]["EndingHashKey"]
|
|
== original_shards[1]["HashKeyRange"]["EndingHashKey"]
|
|
)
|
|
assert "StartingSequenceNumber" in shards[1]["SequenceNumberRange"]
|
|
assert "EndingSequenceNumber" in shards[1]["SequenceNumberRange"]
|
|
|
|
assert shards[2]["ShardId"] == "shardId-000000000002"
|
|
assert shards[2]["ParentShardId"] == shards[1]["ShardId"]
|
|
assert "StartingSequenceNumber" in shards[2]["SequenceNumberRange"]
|
|
assert "EndingSequenceNumber" not in shards[2]["SequenceNumberRange"]
|
|
|
|
assert shards[3]["ShardId"] == "shardId-000000000003"
|
|
assert shards[3]["ParentShardId"] == shards[1]["ShardId"]
|
|
assert "StartingSequenceNumber" in shards[3]["SequenceNumberRange"]
|
|
assert "EndingSequenceNumber" not in shards[3]["SequenceNumberRange"]
|
|
|
|
|
|
@mock_aws
|
|
def test_split_shard_that_was_split_before():
|
|
client = boto3.client("kinesis", region_name="us-west-2")
|
|
client.create_stream(StreamName="my-stream", ShardCount=2)
|
|
stream_arn = get_stream_arn(client, "my-stream")
|
|
|
|
client.split_shard(
|
|
StreamARN=stream_arn,
|
|
ShardToSplit="shardId-000000000001",
|
|
NewStartingHashKey="170141183460469231731687303715884105829",
|
|
)
|
|
|
|
with pytest.raises(ClientError) as exc:
|
|
client.split_shard(
|
|
StreamName="my-stream",
|
|
ShardToSplit="shardId-000000000001",
|
|
NewStartingHashKey="170141183460469231731687303715884105829",
|
|
)
|
|
err = exc.value.response["Error"]
|
|
assert err["Code"] == "InvalidArgumentException"
|
|
assert (
|
|
err["Message"]
|
|
== f"Shard shardId-000000000001 in stream my-stream under account {ACCOUNT_ID} has already been merged or split, and thus is not eligible for merging or splitting."
|
|
)
|
|
|
|
|
|
@mock_aws
|
|
@pytest.mark.parametrize(
|
|
"initial,target,expected_total",
|
|
[(2, 4, 6), (4, 5, 15), (10, 13, 37), (4, 2, 6), (5, 3, 7), (10, 3, 17)],
|
|
)
|
|
def test_update_shard_count(initial, target, expected_total):
|
|
"""
|
|
Test that we update the shard count in a similar manner to AWS
|
|
Assert on: initial_shard_count, target_shard_count and total_shard_count
|
|
|
|
total_shard_count gives an idea of the number of splits/merges required to reach the target
|
|
|
|
These numbers have been verified against AWS
|
|
"""
|
|
client = boto3.client("kinesis", region_name="eu-west-1")
|
|
client.create_stream(StreamName="my-stream", ShardCount=initial)
|
|
|
|
resp = client.update_shard_count(
|
|
StreamName="my-stream", TargetShardCount=target, ScalingType="UNIFORM_SCALING"
|
|
)
|
|
|
|
assert resp["StreamName"] == "my-stream"
|
|
assert resp["CurrentShardCount"] == initial
|
|
assert resp["TargetShardCount"] == target
|
|
|
|
stream = client.describe_stream(StreamName="my-stream")["StreamDescription"]
|
|
assert stream["StreamStatus"] == "ACTIVE"
|
|
assert len(stream["Shards"]) == expected_total
|
|
|
|
active_shards = [
|
|
shard
|
|
for shard in stream["Shards"]
|
|
if "EndingSequenceNumber" not in shard["SequenceNumberRange"]
|
|
]
|
|
assert len(active_shards) == target
|
|
|
|
resp = client.describe_stream_summary(StreamName="my-stream")
|
|
stream = resp["StreamDescriptionSummary"]
|
|
|
|
assert stream["OpenShardCount"] == target
|