From cea9c8c9a38e69ee79787684ed60dd8f00f3934e Mon Sep 17 00:00:00 2001 From: gruebel Date: Tue, 26 Nov 2019 23:38:43 +0100 Subject: [PATCH] Fix order and nextToken handling in logs.get_log_events --- moto/logs/models.py | 81 +++++++++----- tests/test_logs/test_logs.py | 208 ++++++++++++++++++++++++++++------- 2 files changed, 225 insertions(+), 64 deletions(-) diff --git a/moto/logs/models.py b/moto/logs/models.py index 965b1e19a..d0639524e 100644 --- a/moto/logs/models.py +++ b/moto/logs/models.py @@ -1,7 +1,11 @@ from moto.core import BaseBackend import boto.logs from moto.core.utils import unix_time_millis -from .exceptions import ResourceNotFoundException, ResourceAlreadyExistsException +from .exceptions import ( + ResourceNotFoundException, + ResourceAlreadyExistsException, + InvalidParameterException, +) class LogEvent: @@ -118,41 +122,66 @@ class LogStream: return True - def get_paging_token_from_index(index, back=False): - if index is not None: - return "b/{:056d}".format(index) if back else "f/{:056d}".format(index) - return 0 - - def get_index_from_paging_token(token): + def get_index_and_direction_from_token(token): if token is not None: - return int(token[2:]) - return 0 + try: + return token[0], int(token[2:]) + except Exception: + raise InvalidParameterException( + "The specified nextToken is invalid." + ) + return None, 0 events = sorted( - filter(filter_func, self.events), - key=lambda event: event.timestamp, - reverse=start_from_head, + filter(filter_func, self.events), key=lambda event: event.timestamp, ) - next_index = get_index_from_paging_token(next_token) - back_index = next_index + + direction, index = get_index_and_direction_from_token(next_token) + limit_index = limit - 1 + final_index = len(events) - 1 + + if direction is None: + if start_from_head: + start_index = 0 + end_index = start_index + limit_index + else: + end_index = final_index + start_index = end_index - limit_index + elif direction == "f": + start_index = index + 1 + end_index = start_index + limit_index + elif direction == "b": + end_index = index - 1 + start_index = end_index - limit_index + else: + raise InvalidParameterException("The specified nextToken is invalid.") + + if start_index < 0: + start_index = 0 + elif start_index > final_index: + return ( + [], + "b/{:056d}".format(final_index), + "f/{:056d}".format(final_index), + ) + + if end_index > final_index: + end_index = final_index + elif end_index < 0: + return ( + [], + "b/{:056d}".format(0), + "f/{:056d}".format(0), + ) events_page = [ - event.to_response_dict() - for event in events[next_index : next_index + limit] + event.to_response_dict() for event in events[start_index : end_index + 1] ] - if next_index + limit < len(self.events): - next_index += limit - else: - next_index = len(self.events) - - back_index -= limit - if back_index <= 0: - back_index = 0 return ( events_page, - get_paging_token_from_index(back_index, True), - get_paging_token_from_index(next_index), + "b/{:056d}".format(start_index), + "f/{:056d}".format(end_index), ) def filter_log_events( diff --git a/tests/test_logs/test_logs.py b/tests/test_logs/test_logs.py index 57f457381..e8f60ff03 100644 --- a/tests/test_logs/test_logs.py +++ b/tests/test_logs/test_logs.py @@ -166,70 +166,202 @@ def test_delete_retention_policy(): @mock_logs def test_get_log_events(): - conn = boto3.client("logs", "us-west-2") + client = boto3.client("logs", "us-west-2") log_group_name = "test" log_stream_name = "stream" - conn.create_log_group(logGroupName=log_group_name) - conn.create_log_stream(logGroupName=log_group_name, logStreamName=log_stream_name) + client.create_log_group(logGroupName=log_group_name) + client.create_log_stream(logGroupName=log_group_name, logStreamName=log_stream_name) events = [{"timestamp": x, "message": str(x)} for x in range(20)] - conn.put_log_events( + client.put_log_events( logGroupName=log_group_name, logStreamName=log_stream_name, logEvents=events ) - resp = conn.get_log_events( + resp = client.get_log_events( logGroupName=log_group_name, logStreamName=log_stream_name, limit=10 ) resp["events"].should.have.length_of(10) - resp.should.have.key("nextForwardToken") - resp.should.have.key("nextBackwardToken") - resp["nextForwardToken"].should.equal( - "f/00000000000000000000000000000000000000000000000000000010" - ) - resp["nextBackwardToken"].should.equal( - "b/00000000000000000000000000000000000000000000000000000000" - ) - for i in range(10): - resp["events"][i]["timestamp"].should.equal(i) - resp["events"][i]["message"].should.equal(str(i)) - - next_token = resp["nextForwardToken"] - - resp = conn.get_log_events( - logGroupName=log_group_name, - logStreamName=log_stream_name, - nextToken=next_token, - limit=10, - ) - - resp["events"].should.have.length_of(10) - resp.should.have.key("nextForwardToken") - resp.should.have.key("nextBackwardToken") - resp["nextForwardToken"].should.equal( - "f/00000000000000000000000000000000000000000000000000000020" - ) - resp["nextBackwardToken"].should.equal( - "b/00000000000000000000000000000000000000000000000000000000" - ) for i in range(10): resp["events"][i]["timestamp"].should.equal(i + 10) resp["events"][i]["message"].should.equal(str(i + 10)) + resp["nextForwardToken"].should.equal( + "f/00000000000000000000000000000000000000000000000000000019" + ) + resp["nextBackwardToken"].should.equal( + "b/00000000000000000000000000000000000000000000000000000010" + ) - resp = conn.get_log_events( + resp = client.get_log_events( + logGroupName=log_group_name, + logStreamName=log_stream_name, + nextToken=resp["nextBackwardToken"], + limit=20, + ) + + resp["events"].should.have.length_of(10) + for i in range(10): + resp["events"][i]["timestamp"].should.equal(i) + resp["events"][i]["message"].should.equal(str(i)) + resp["nextForwardToken"].should.equal( + "f/00000000000000000000000000000000000000000000000000000009" + ) + resp["nextBackwardToken"].should.equal( + "b/00000000000000000000000000000000000000000000000000000000" + ) + + resp = client.get_log_events( logGroupName=log_group_name, logStreamName=log_stream_name, nextToken=resp["nextBackwardToken"], limit=10, ) + resp["events"].should.have.length_of(0) + resp["nextForwardToken"].should.equal( + "f/00000000000000000000000000000000000000000000000000000000" + ) + resp["nextBackwardToken"].should.equal( + "b/00000000000000000000000000000000000000000000000000000000" + ) + + resp = client.get_log_events( + logGroupName=log_group_name, + logStreamName=log_stream_name, + nextToken=resp["nextForwardToken"], + limit=1, + ) + + resp["events"].should.have.length_of(1) + resp["events"][0]["timestamp"].should.equal(1) + resp["events"][0]["message"].should.equal(str(1)) + resp["nextForwardToken"].should.equal( + "f/00000000000000000000000000000000000000000000000000000001" + ) + resp["nextBackwardToken"].should.equal( + "b/00000000000000000000000000000000000000000000000000000001" + ) + + +@mock_logs +def test_get_log_events_with_start_from_head(): + client = boto3.client("logs", "us-west-2") + log_group_name = "test" + log_stream_name = "stream" + client.create_log_group(logGroupName=log_group_name) + client.create_log_stream(logGroupName=log_group_name, logStreamName=log_stream_name) + + events = [{"timestamp": x, "message": str(x)} for x in range(20)] + + client.put_log_events( + logGroupName=log_group_name, logStreamName=log_stream_name, logEvents=events + ) + + resp = client.get_log_events( + logGroupName=log_group_name, + logStreamName=log_stream_name, + limit=10, + startFromHead=True, # this parameter is only relevant without the usage of nextToken + ) + resp["events"].should.have.length_of(10) - resp.should.have.key("nextForwardToken") - resp.should.have.key("nextBackwardToken") for i in range(10): resp["events"][i]["timestamp"].should.equal(i) resp["events"][i]["message"].should.equal(str(i)) + resp["nextForwardToken"].should.equal( + "f/00000000000000000000000000000000000000000000000000000009" + ) + resp["nextBackwardToken"].should.equal( + "b/00000000000000000000000000000000000000000000000000000000" + ) + + resp = client.get_log_events( + logGroupName=log_group_name, + logStreamName=log_stream_name, + nextToken=resp["nextForwardToken"], + limit=20, + ) + + resp["events"].should.have.length_of(10) + for i in range(10): + resp["events"][i]["timestamp"].should.equal(i + 10) + resp["events"][i]["message"].should.equal(str(i + 10)) + resp["nextForwardToken"].should.equal( + "f/00000000000000000000000000000000000000000000000000000019" + ) + resp["nextBackwardToken"].should.equal( + "b/00000000000000000000000000000000000000000000000000000010" + ) + + resp = client.get_log_events( + logGroupName=log_group_name, + logStreamName=log_stream_name, + nextToken=resp["nextForwardToken"], + limit=10, + ) + + resp["events"].should.have.length_of(0) + resp["nextForwardToken"].should.equal( + "f/00000000000000000000000000000000000000000000000000000019" + ) + resp["nextBackwardToken"].should.equal( + "b/00000000000000000000000000000000000000000000000000000019" + ) + + resp = client.get_log_events( + logGroupName=log_group_name, + logStreamName=log_stream_name, + nextToken=resp["nextBackwardToken"], + limit=1, + ) + + resp["events"].should.have.length_of(1) + resp["events"][0]["timestamp"].should.equal(18) + resp["events"][0]["message"].should.equal(str(18)) + resp["nextForwardToken"].should.equal( + "f/00000000000000000000000000000000000000000000000000000018" + ) + resp["nextBackwardToken"].should.equal( + "b/00000000000000000000000000000000000000000000000000000018" + ) + + +@mock_logs +def test_get_log_events_errors(): + client = boto3.client("logs", "us-west-2") + log_group_name = "test" + log_stream_name = "stream" + client.create_log_group(logGroupName=log_group_name) + client.create_log_stream(logGroupName=log_group_name, logStreamName=log_stream_name) + + with assert_raises(ClientError) as e: + client.get_log_events( + logGroupName=log_group_name, + logStreamName=log_stream_name, + nextToken="n/00000000000000000000000000000000000000000000000000000000", + ) + ex = e.exception + ex.operation_name.should.equal("GetLogEvents") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.equal("InvalidParameterException") + ex.response["Error"]["Message"].should.contain( + "The specified nextToken is invalid." + ) + + with assert_raises(ClientError) as e: + client.get_log_events( + logGroupName=log_group_name, + logStreamName=log_stream_name, + nextToken="not-existing-token", + ) + ex = e.exception + ex.operation_name.should.equal("GetLogEvents") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.equal("InvalidParameterException") + ex.response["Error"]["Message"].should.contain( + "The specified nextToken is invalid." + ) @mock_logs