Merge pull request #2604 from gruebel/fix-logs-get-log-events

Fix order and nextToken handling in logs.get_log_events

moto/logs/models.py

@@ -1,7 +1,11 @@
 from moto.core import BaseBackend
 import boto.logs
 from moto.core.utils import unix_time_millis
-from .exceptions import ResourceNotFoundException, ResourceAlreadyExistsException
+from .exceptions import (
+    ResourceNotFoundException,
+    ResourceAlreadyExistsException,
+    InvalidParameterException,
+)
 
 
 class LogEvent:
@@ -118,41 +122,66 @@ class LogStream:
 
             return True
 
-        def get_paging_token_from_index(index, back=False):
+        def get_index_and_direction_from_token(token):
-            if index is not None:
-                return "b/{:056d}".format(index) if back else "f/{:056d}".format(index)
-            return 0
-
-        def get_index_from_paging_token(token):
             if token is not None:
-                return int(token[2:])
+                try:
-            return 0
+                    return token[0], int(token[2:])
+                except Exception:
+                    raise InvalidParameterException(
+                        "The specified nextToken is invalid."
+                    )
+            return None, 0
 
         events = sorted(
-            filter(filter_func, self.events),
+            filter(filter_func, self.events), key=lambda event: event.timestamp,
-            key=lambda event: event.timestamp,
-            reverse=start_from_head,
         )
-        next_index = get_index_from_paging_token(next_token)
+
-        back_index = next_index
+        direction, index = get_index_and_direction_from_token(next_token)
+        limit_index = limit - 1
+        final_index = len(events) - 1
+
+        if direction is None:
+            if start_from_head:
+                start_index = 0
+                end_index = start_index + limit_index
+            else:
+                end_index = final_index
+                start_index = end_index - limit_index
+        elif direction == "f":
+            start_index = index + 1
+            end_index = start_index + limit_index
+        elif direction == "b":
+            end_index = index - 1
+            start_index = end_index - limit_index
+        else:
+            raise InvalidParameterException("The specified nextToken is invalid.")
+
+        if start_index < 0:
+            start_index = 0
+        elif start_index > final_index:
+            return (
+                [],
+                "b/{:056d}".format(final_index),
+                "f/{:056d}".format(final_index),
+            )
+
+        if end_index > final_index:
+            end_index = final_index
+        elif end_index < 0:
+            return (
+                [],
+                "b/{:056d}".format(0),
+                "f/{:056d}".format(0),
+            )
 
         events_page = [
-            event.to_response_dict()
+            event.to_response_dict() for event in events[start_index : end_index + 1]
-            for event in events[next_index : next_index + limit]
         ]
-        if next_index + limit < len(self.events):
-            next_index += limit
-        else:
-            next_index = len(self.events)
-
-        back_index -= limit
-        if back_index <= 0:
-            back_index = 0
 
         return (
             events_page,
-            get_paging_token_from_index(back_index, True),
+            "b/{:056d}".format(start_index),
-            get_paging_token_from_index(next_index),
+            "f/{:056d}".format(end_index),
         )
 
     def filter_log_events(

tests/test_logs/test_logs.py

@@ -166,70 +166,202 @@ def test_delete_retention_policy():
 
 @mock_logs
 def test_get_log_events():
-    conn = boto3.client("logs", "us-west-2")
+    client = boto3.client("logs", "us-west-2")
     log_group_name = "test"
     log_stream_name = "stream"
-    conn.create_log_group(logGroupName=log_group_name)
+    client.create_log_group(logGroupName=log_group_name)
-    conn.create_log_stream(logGroupName=log_group_name, logStreamName=log_stream_name)
+    client.create_log_stream(logGroupName=log_group_name, logStreamName=log_stream_name)
 
     events = [{"timestamp": x, "message": str(x)} for x in range(20)]
 
-    conn.put_log_events(
+    client.put_log_events(
         logGroupName=log_group_name, logStreamName=log_stream_name, logEvents=events
     )
 
-    resp = conn.get_log_events(
+    resp = client.get_log_events(
         logGroupName=log_group_name, logStreamName=log_stream_name, limit=10
     )
 
     resp["events"].should.have.length_of(10)
-    resp.should.have.key("nextForwardToken")
-    resp.should.have.key("nextBackwardToken")
-    resp["nextForwardToken"].should.equal(
-        "f/00000000000000000000000000000000000000000000000000000010"
-    )
-    resp["nextBackwardToken"].should.equal(
-        "b/00000000000000000000000000000000000000000000000000000000"
-    )
-    for i in range(10):
-        resp["events"][i]["timestamp"].should.equal(i)
-        resp["events"][i]["message"].should.equal(str(i))
-
-    next_token = resp["nextForwardToken"]
-
-    resp = conn.get_log_events(
-        logGroupName=log_group_name,
-        logStreamName=log_stream_name,
-        nextToken=next_token,
-        limit=10,
-    )
-
-    resp["events"].should.have.length_of(10)
-    resp.should.have.key("nextForwardToken")
-    resp.should.have.key("nextBackwardToken")
-    resp["nextForwardToken"].should.equal(
-        "f/00000000000000000000000000000000000000000000000000000020"
-    )
-    resp["nextBackwardToken"].should.equal(
-        "b/00000000000000000000000000000000000000000000000000000000"
-    )
     for i in range(10):
         resp["events"][i]["timestamp"].should.equal(i + 10)
         resp["events"][i]["message"].should.equal(str(i + 10))
+    resp["nextForwardToken"].should.equal(
+        "f/00000000000000000000000000000000000000000000000000000019"
+    )
+    resp["nextBackwardToken"].should.equal(
+        "b/00000000000000000000000000000000000000000000000000000010"
+    )
 
-    resp = conn.get_log_events(
+    resp = client.get_log_events(
+        logGroupName=log_group_name,
+        logStreamName=log_stream_name,
+        nextToken=resp["nextBackwardToken"],
+        limit=20,
+    )
+
+    resp["events"].should.have.length_of(10)
+    for i in range(10):
+        resp["events"][i]["timestamp"].should.equal(i)
+        resp["events"][i]["message"].should.equal(str(i))
+    resp["nextForwardToken"].should.equal(
+        "f/00000000000000000000000000000000000000000000000000000009"
+    )
+    resp["nextBackwardToken"].should.equal(
+        "b/00000000000000000000000000000000000000000000000000000000"
+    )
+
+    resp = client.get_log_events(
         logGroupName=log_group_name,
         logStreamName=log_stream_name,
         nextToken=resp["nextBackwardToken"],
         limit=10,
     )
 
+    resp["events"].should.have.length_of(0)
+    resp["nextForwardToken"].should.equal(
+        "f/00000000000000000000000000000000000000000000000000000000"
+    )
+    resp["nextBackwardToken"].should.equal(
+        "b/00000000000000000000000000000000000000000000000000000000"
+    )
+
+    resp = client.get_log_events(
+        logGroupName=log_group_name,
+        logStreamName=log_stream_name,
+        nextToken=resp["nextForwardToken"],
+        limit=1,
+    )
+
+    resp["events"].should.have.length_of(1)
+    resp["events"][0]["timestamp"].should.equal(1)
+    resp["events"][0]["message"].should.equal(str(1))
+    resp["nextForwardToken"].should.equal(
+        "f/00000000000000000000000000000000000000000000000000000001"
+    )
+    resp["nextBackwardToken"].should.equal(
+        "b/00000000000000000000000000000000000000000000000000000001"
+    )
+
+
+@mock_logs
+def test_get_log_events_with_start_from_head():
+    client = boto3.client("logs", "us-west-2")
+    log_group_name = "test"
+    log_stream_name = "stream"
+    client.create_log_group(logGroupName=log_group_name)
+    client.create_log_stream(logGroupName=log_group_name, logStreamName=log_stream_name)
+
+    events = [{"timestamp": x, "message": str(x)} for x in range(20)]
+
+    client.put_log_events(
+        logGroupName=log_group_name, logStreamName=log_stream_name, logEvents=events
+    )
+
+    resp = client.get_log_events(
+        logGroupName=log_group_name,
+        logStreamName=log_stream_name,
+        limit=10,
+        startFromHead=True,  # this parameter is only relevant without the usage of nextToken
+    )
+
     resp["events"].should.have.length_of(10)
-    resp.should.have.key("nextForwardToken")
-    resp.should.have.key("nextBackwardToken")
     for i in range(10):
         resp["events"][i]["timestamp"].should.equal(i)
         resp["events"][i]["message"].should.equal(str(i))
+    resp["nextForwardToken"].should.equal(
+        "f/00000000000000000000000000000000000000000000000000000009"
+    )
+    resp["nextBackwardToken"].should.equal(
+        "b/00000000000000000000000000000000000000000000000000000000"
+    )
+
+    resp = client.get_log_events(
+        logGroupName=log_group_name,
+        logStreamName=log_stream_name,
+        nextToken=resp["nextForwardToken"],
+        limit=20,
+    )
+
+    resp["events"].should.have.length_of(10)
+    for i in range(10):
+        resp["events"][i]["timestamp"].should.equal(i + 10)
+        resp["events"][i]["message"].should.equal(str(i + 10))
+    resp["nextForwardToken"].should.equal(
+        "f/00000000000000000000000000000000000000000000000000000019"
+    )
+    resp["nextBackwardToken"].should.equal(
+        "b/00000000000000000000000000000000000000000000000000000010"
+    )
+
+    resp = client.get_log_events(
+        logGroupName=log_group_name,
+        logStreamName=log_stream_name,
+        nextToken=resp["nextForwardToken"],
+        limit=10,
+    )
+
+    resp["events"].should.have.length_of(0)
+    resp["nextForwardToken"].should.equal(
+        "f/00000000000000000000000000000000000000000000000000000019"
+    )
+    resp["nextBackwardToken"].should.equal(
+        "b/00000000000000000000000000000000000000000000000000000019"
+    )
+
+    resp = client.get_log_events(
+        logGroupName=log_group_name,
+        logStreamName=log_stream_name,
+        nextToken=resp["nextBackwardToken"],
+        limit=1,
+    )
+
+    resp["events"].should.have.length_of(1)
+    resp["events"][0]["timestamp"].should.equal(18)
+    resp["events"][0]["message"].should.equal(str(18))
+    resp["nextForwardToken"].should.equal(
+        "f/00000000000000000000000000000000000000000000000000000018"
+    )
+    resp["nextBackwardToken"].should.equal(
+        "b/00000000000000000000000000000000000000000000000000000018"
+    )
+
+
+@mock_logs
+def test_get_log_events_errors():
+    client = boto3.client("logs", "us-west-2")
+    log_group_name = "test"
+    log_stream_name = "stream"
+    client.create_log_group(logGroupName=log_group_name)
+    client.create_log_stream(logGroupName=log_group_name, logStreamName=log_stream_name)
+
+    with assert_raises(ClientError) as e:
+        client.get_log_events(
+            logGroupName=log_group_name,
+            logStreamName=log_stream_name,
+            nextToken="n/00000000000000000000000000000000000000000000000000000000",
+        )
+    ex = e.exception
+    ex.operation_name.should.equal("GetLogEvents")
+    ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
+    ex.response["Error"]["Code"].should.equal("InvalidParameterException")
+    ex.response["Error"]["Message"].should.contain(
+        "The specified nextToken is invalid."
+    )
+
+    with assert_raises(ClientError) as e:
+        client.get_log_events(
+            logGroupName=log_group_name,
+            logStreamName=log_stream_name,
+            nextToken="not-existing-token",
+        )
+    ex = e.exception
+    ex.operation_name.should.equal("GetLogEvents")
+    ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
+    ex.response["Error"]["Code"].should.equal("InvalidParameterException")
+    ex.response["Error"]["Message"].should.contain(
+        "The specified nextToken is invalid."
+    )
 
 
 @mock_logs