Fix: nextToken value in logs:FilterLogEvents response (#3883)

* Fix: `nextToken` value in `logs:FilterLogEvents` response

Plagiarizing freely from @bpandola and his PR #3398, I have
modified the pagination for FilterLogEvents to more closely follow
the real AWS behaviour.

Fixes #3882

* Black reformatted my code.

* Remove timezone for python2.7 compatibility.

* Hopefully fix python2.7 compatibility for real.

* Additional test for a non-matching log group name in the nextToken.

moto/logs/models.py

@@ -366,13 +366,35 @@ class LogGroup:
         if interleaved:
             events = sorted(events, key=lambda event: event["timestamp"])
 
-        if next_token is None:
+        first_index = 0
-            next_token = 0
+        if next_token:
+            try:
+                group, stream, event_id = next_token.split("/")
+                if group != log_group_name:
+                    raise ValueError()
+                first_index = (
+                    next(
+                        index
+                        for (index, e) in enumerate(events)
+                        if e["logStreamName"] == stream and e["eventId"] == event_id
+                    )
+                    + 1
+                )
+            except (ValueError, StopIteration):
+                first_index = 0
+                # AWS returns an empty list if it receives an invalid token.
+                events = []
 
-        events_page = events[next_token : next_token + limit]
+        last_index = first_index + limit
-        next_token += limit
+        if last_index > len(events):
-        if next_token >= len(events):
+            last_index = len(events)
-            next_token = None
+        events_page = events[first_index:last_index]
+        next_token = None
+        if events_page and last_index < len(events):
+            last_event = events_page[-1]
+            next_token = "{}/{}/{}".format(
+                log_group_name, last_event["logStreamName"], last_event["eventId"]
+            )
 
         searched_streams = [
             {"logStreamName": stream.logStreamName, "searchedCompletely": True}

tests/test_logs/test_logs.py

@@ -1,12 +1,13 @@
-import boto3
 import os
-import sure  # noqa
+import time
+from unittest import SkipTest
+import boto3
 import six
 from botocore.exceptions import ClientError
+import pytest
+import sure  # noqa
 
 from moto import mock_logs, settings
-import pytest
-from unittest import SkipTest
 
 _logs_region = "us-east-1" if settings.TEST_SERVER_MODE else "us-west-2"
 
@@ -125,6 +126,65 @@ def test_filter_logs_raises_if_filter_pattern():
         )
 
 
+@mock_logs
+def test_filter_logs_paging():
+    conn = boto3.client("logs", "us-west-2")
+    log_group_name = "dummy"
+    log_stream_name = "stream"
+    conn.create_log_group(logGroupName=log_group_name)
+    conn.create_log_stream(logGroupName=log_group_name, logStreamName=log_stream_name)
+    timestamp = int(time.time())
+    messages = []
+    for i in range(25):
+        messages.append(
+            {"message": "Message number {}".format(i), "timestamp": timestamp}
+        )
+        timestamp += 100
+
+    conn.put_log_events(
+        logGroupName=log_group_name, logStreamName=log_stream_name, logEvents=messages
+    )
+    res = conn.filter_log_events(
+        logGroupName=log_group_name, logStreamNames=[log_stream_name], limit=20
+    )
+    events = res["events"]
+    events.should.have.length_of(20)
+    res["nextToken"].should.equal("dummy/stream/" + events[-1]["eventId"])
+
+    res = conn.filter_log_events(
+        logGroupName=log_group_name,
+        logStreamNames=[log_stream_name],
+        limit=20,
+        nextToken=res["nextToken"],
+    )
+    events += res["events"]
+    events.should.have.length_of(25)
+    res.should_not.have.key("nextToken")
+
+    for original_message, resulting_event in zip(messages, events):
+        resulting_event["eventId"].should.equal(str(resulting_event["eventId"]))
+        resulting_event["timestamp"].should.equal(original_message["timestamp"])
+        resulting_event["message"].should.equal(original_message["message"])
+
+    res = conn.filter_log_events(
+        logGroupName=log_group_name,
+        logStreamNames=[log_stream_name],
+        limit=20,
+        nextToken="invalid-token",
+    )
+    res["events"].should.have.length_of(0)
+    res.should_not.have.key("nextToken")
+
+    res = conn.filter_log_events(
+        logGroupName=log_group_name,
+        logStreamNames=[log_stream_name],
+        limit=20,
+        nextToken="wrong-group/stream/999",
+    )
+    res["events"].should.have.length_of(0)
+    res.should_not.have.key("nextToken")
+
+
 @mock_logs
 def test_put_retention_policy():
     conn = boto3.client("logs", "us-west-2")