Logs: create_export_task() now actually exports the data (#7082)

moto/logs/exceptions.py

@@ -11,7 +11,8 @@ class ResourceNotFoundException(LogsClientError):
     def __init__(self, msg: Optional[str] = None):
         self.code = 400
         super().__init__(
-            "ResourceNotFoundException", msg or "The specified log group does not exist"
+            "ResourceNotFoundException",
+            msg or "The specified log group does not exist.",
         )
 
 

moto/logs/models.py

@@ -1,5 +1,5 @@
-import json
 from datetime import datetime, timedelta
+from gzip import compress as gzip_compress
 from typing import Any, Dict, Iterable, List, Optional, Tuple
 
 from moto.core import BackendDict, BaseBackend, BaseModel, CloudFormationModel
@@ -13,7 +13,7 @@ from moto.logs.exceptions import (
 from moto.logs.logs_query import execute_query
 from moto.logs.metric_filters import MetricFilters
 from moto.moto_api._internal import mock_random
-from moto.s3.models import s3_backends
+from moto.s3.models import MissingBucket, s3_backends
 from moto.utilities.paginator import paginate
 from moto.utilities.tagging_service import TaggingService
 
@@ -1248,7 +1248,12 @@ class LogsBackend(BaseBackend):
         fromTime: int,
         to: int,
     ) -> str:
-        s3_backends[self.account_id]["global"].get_bucket(destination)
+        try:
+            s3_backends[self.account_id]["global"].get_bucket(destination)
+        except MissingBucket:
+            raise InvalidParameterException(
+                "The given bucket does not exist. Please make sure the bucket is valid."
+            )
         if logGroupName not in self.groups:
             raise ResourceNotFoundException()
         task_id = str(mock_random.uuid4())
@@ -1261,15 +1266,41 @@ class LogsBackend(BaseBackend):
             fromTime,
             to,
         )
-        if fromTime <= datetime.now().timestamp() <= to:
+
-            logs = self.filter_log_events(
+        s3_backends[self.account_id]["global"].put_object(
-                logGroupName, [], fromTime, to, None, None, "", False
+            bucket_name=destination,
-            )
+            key_name="aws-logs-write-test",
-            s3_backends[self.account_id]["global"].put_object(
+            value=b"Permission Check Successful",
-                destination, destinationPrefix, json.dumps(logs).encode("utf-8")
+        )
-            )
+
-            self.export_tasks[task_id].status["code"] = "completed"
+        if fromTime <= unix_time_millis(datetime.now()) <= to:
-            self.export_tasks[task_id].status["message"] = "Task is completed"
+            for stream_name in self.groups[logGroupName].streams.keys():
+                logs, _, _ = self.filter_log_events(
+                    log_group_name=logGroupName,
+                    log_stream_names=[stream_name],
+                    start_time=fromTime,
+                    end_time=to,
+                    limit=None,
+                    next_token=None,
+                    filter_pattern="",
+                    interleaved=False,
+                )
+                raw_logs = "\n".join(
+                    [
+                        f"{datetime.fromtimestamp(log['timestamp']/1000).strftime('%Y-%m-%dT%H:%M:%S.000Z')} {log['message']}"
+                        for log in logs
+                    ]
+                )
+                folder = str(mock_random.uuid4()) + "/" + stream_name.replace("/", "-")
+                key_name = f"{destinationPrefix}/{folder}/000000.gz"
+                s3_backends[self.account_id]["global"].put_object(
+                    bucket_name=destination,
+                    key_name=key_name,
+                    value=gzip_compress(raw_logs.encode("utf-8")),
+                )
+            self.export_tasks[task_id].status["code"] = "COMPLETED"
+            self.export_tasks[task_id].status["message"] = "Completed successfully"
+
         return task_id
 
     def describe_export_tasks(self, task_id: str) -> List[ExportTask]:

moto/logs/responses.py

@@ -438,7 +438,7 @@ class LogsResponse(BaseResponse):
             fromTime=self._get_int_param("from"),
             to=self._get_int_param("to"),
             destination=self._get_param("destination"),
-            destinationPrefix=self._get_param("destinationPrefix"),
+            destinationPrefix=self._get_param("destinationPrefix", "exportedlogs"),
             taskName=self._get_param("taskName"),
         )
         return json.dumps(dict(taskId=str(task_id)))

tests/test_logs/test_export_tasks.py

@@ -0,0 +1,269 @@
+import copy
+import json
+import os
+from datetime import datetime, timedelta
+from uuid import UUID, uuid4
+
+import boto3
+import pytest
+from botocore.exceptions import ClientError
+
+from moto import mock_logs, mock_s3, mock_sts, settings
+from moto.core.utils import unix_time_millis
+
+TEST_REGION = "us-east-1" if settings.TEST_SERVER_MODE else "us-west-2"
+allow_aws_request = (
+    os.environ.get("MOTO_TEST_ALLOW_AWS_REQUEST", "false").lower() == "true"
+)
+
+
+S3_POLICY = {
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Action": "s3:GetBucketAcl",
+            "Effect": "Allow",
+            "Resource": "arn:aws:s3:::{BUCKET_NAME}",
+            "Principal": {"Service": "logs.us-east-1.amazonaws.com"},
+            "Condition": {
+                "StringEquals": {"aws:SourceAccount": ["AccountId1"]},
+                "ArnLike": {"aws:SourceArn": ["..."]},
+            },
+        },
+        {
+            "Action": "s3:PutObject",
+            "Effect": "Allow",
+            "Resource": "arn:aws:s3:::{BUCKET_NAME}/*",
+            "Principal": {"Service": "logs.us-east-1.amazonaws.com"},
+            "Condition": {
+                "StringEquals": {
+                    "s3:x-amz-acl": "bucket-owner-full-control",
+                    "aws:SourceAccount": ["AccountId1"],
+                },
+                "ArnLike": {"aws:SourceArn": ["..."]},
+            },
+        },
+    ],
+}
+
+
+@pytest.fixture()
+def account_id():
+    if allow_aws_request:
+        identity = boto3.client("sts", region_name="us-east-1").get_caller_identity()
+        yield identity["Account"]
+    else:
+        with mock_sts():
+            identity = boto3.client(
+                "sts", region_name="us-east-1"
+            ).get_caller_identity()
+            yield identity["Account"]
+
+
+@pytest.fixture()
+def logs():
+    if allow_aws_request:
+        yield boto3.client("logs", region_name="us-east-1")
+    else:
+        with mock_logs():
+            yield boto3.client("logs", region_name="us-east-1")
+
+
+@pytest.fixture()
+def s3():
+    if allow_aws_request:
+        yield boto3.client("s3", region_name="us-east-1")
+    else:
+        with mock_s3():
+            yield boto3.client("s3", region_name="us-east-1")
+
+
+@pytest.fixture(scope="function")
+def log_group_name(logs):  # pylint: disable=redefined-outer-name
+    name = "/moto/logs_test/" + str(uuid4())[0:5]
+    logs.create_log_group(logGroupName=name)
+    yield name
+    logs.delete_log_group(logGroupName=name)
+
+
+@pytest.fixture()
+def bucket_name(s3, account_id):  # pylint: disable=redefined-outer-name
+    name = f"moto-logs-test-{str(uuid4())[0:6]}"
+    s3.create_bucket(Bucket=name)
+    policy = copy.copy(S3_POLICY)
+    policy["Statement"][0]["Resource"] = f"arn:aws:s3:::{name}"
+    policy["Statement"][0]["Condition"]["StringEquals"]["aws:SourceAccount"] = [
+        account_id
+    ]
+    policy["Statement"][0]["Condition"]["ArnLike"]["aws:SourceArn"] = [
+        f"arn:aws:logs:us-east-1:{account_id}:log-group:*"
+    ]
+    policy["Statement"][1]["Resource"] = f"arn:aws:s3:::{name}/*"
+    policy["Statement"][1]["Condition"]["StringEquals"]["aws:SourceAccount"] = [
+        account_id
+    ]
+    policy["Statement"][1]["Condition"]["ArnLike"]["aws:SourceArn"] = [
+        f"arn:aws:logs:us-east-1:{account_id}:log-group:*"
+    ]
+
+    s3.put_bucket_policy(Bucket=name, Policy=json.dumps(policy))
+    yield name
+
+    # Delete any files left behind
+    versions = s3.list_object_versions(Bucket=name).get("Versions", [])
+    for key in versions:
+        s3.delete_object(Bucket=name, Key=key["Key"], VersionId=key.get("VersionId"))
+    delete_markers = s3.list_object_versions(Bucket=name).get("DeleteMarkers", [])
+    for key in delete_markers:
+        s3.delete_object(Bucket=name, Key=key["Key"], VersionId=key.get("VersionId"))
+
+    # Delete bucket itself
+    s3.delete_bucket(Bucket=name)
+
+
+@pytest.mark.aws_verified
+def test_create_export_task_happy_path(
+    logs, s3, log_group_name, bucket_name
+):  # pylint: disable=redefined-outer-name
+    fromTime = 1611316574
+    to = 1642852574
+    resp = logs.create_export_task(
+        logGroupName=log_group_name,
+        fromTime=fromTime,
+        to=to,
+        destination=bucket_name,
+    )
+    # taskId resembles a valid UUID (i.e. a string of 32 hexadecimal digits)
+    assert UUID(resp["taskId"])
+    assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
+
+    # s3 bucket contains indication that permissions were successful
+    resp = s3.get_object(Bucket=bucket_name, Key="aws-logs-write-test")
+    assert resp["Body"].read() == b"Permission Check Successful"
+
+
+@pytest.mark.aws_verified
+def test_create_export_task_raises_ClientError_when_bucket_not_found(
+    logs, log_group_name  # pylint: disable=redefined-outer-name
+):
+    destination = "368a7022dea3dd621"
+    fromTime = 1611316574
+    to = 1642852574
+    with pytest.raises(ClientError) as exc:
+        logs.create_export_task(
+            logGroupName=log_group_name,
+            fromTime=fromTime,
+            to=to,
+            destination=destination,
+        )
+    err = exc.value.response["Error"]
+    assert err["Code"] == "InvalidParameterException"
+    assert (
+        err["Message"]
+        == "The given bucket does not exist. Please make sure the bucket is valid."
+    )
+
+
+@pytest.mark.aws_verified
+def test_create_export_raises_ResourceNotFoundException_log_group_not_found(
+    logs, bucket_name  # pylint: disable=redefined-outer-name
+):
+    with pytest.raises(logs.exceptions.ResourceNotFoundException) as exc:
+        logs.create_export_task(
+            logGroupName=f"/aws/nonexisting/{str(uuid4())[0:6]}",
+            fromTime=1611316574,
+            to=1642852574,
+            destination=bucket_name,
+        )
+    err = exc.value.response["Error"]
+    assert err["Code"] == "ResourceNotFoundException"
+    assert err["Message"] == "The specified log group does not exist."
+
+
+@pytest.mark.aws_verified
+def test_create_export_executes_export_task(
+    logs, s3, log_group_name, bucket_name
+):  # pylint: disable=redefined-outer-name
+    fromTime = int(unix_time_millis(datetime.now() - timedelta(days=1)))
+    to = int(unix_time_millis(datetime.now() + timedelta(days=1)))
+
+    logs.create_log_stream(logGroupName=log_group_name, logStreamName="/some/stream")
+
+    messages = [
+        {"timestamp": int(unix_time_millis()), "message": f"hello_{i}"}
+        for i in range(10)
+    ]
+    logs.put_log_events(
+        logGroupName=log_group_name, logStreamName="/some/stream", logEvents=messages
+    )
+
+    task_id = logs.create_export_task(
+        logGroupName=log_group_name,
+        fromTime=fromTime,
+        to=to,
+        destination=bucket_name,
+    )["taskId"]
+
+    task = logs.describe_export_tasks(taskId=task_id)["exportTasks"][0]
+    assert task["status"]["code"] in ["COMPLETED", "RUNNING"]
+    assert task["logGroupName"] == log_group_name
+    assert task["destination"] == bucket_name
+    assert task["destinationPrefix"] == "exportedlogs"
+    assert task["from"] == fromTime
+    assert task["to"] == to
+
+    objects = s3.list_objects(Bucket=bucket_name)["Contents"]
+    key_names = [o["Key"] for o in objects]
+    assert "aws-logs-write-test" in key_names
+
+
+def test_describe_export_tasks_happy_path(
+    logs, s3, log_group_name
+):  # pylint: disable=redefined-outer-name
+    destination = "mybucket"
+    fromTime = 1611316574
+    to = 1642852574
+    s3.create_bucket(Bucket=destination)
+    logs.create_export_task(
+        logGroupName=log_group_name,
+        fromTime=fromTime,
+        to=to,
+        destination=destination,
+    )
+    resp = logs.describe_export_tasks()
+    assert len(resp["exportTasks"]) == 1
+    assert resp["exportTasks"][0]["logGroupName"] == log_group_name
+    assert resp["exportTasks"][0]["destination"] == destination
+    assert resp["exportTasks"][0]["from"] == fromTime
+    assert resp["exportTasks"][0]["to"] == to
+    assert resp["exportTasks"][0]["status"]["code"] == "active"
+    assert resp["exportTasks"][0]["status"]["message"] == "Task is active"
+
+
+def test_describe_export_tasks_task_id(
+    logs, log_group_name, bucket_name
+):  # pylint: disable=redefined-outer-name
+    fromTime = 1611316574
+    to = 1642852574
+    resp = logs.create_export_task(
+        logGroupName=log_group_name,
+        fromTime=fromTime,
+        to=to,
+        destination=bucket_name,
+    )
+    taskId = resp["taskId"]
+    resp = logs.describe_export_tasks(taskId=taskId)
+    assert len(resp["exportTasks"]) == 1
+    assert resp["exportTasks"][0]["logGroupName"] == log_group_name
+    assert resp["exportTasks"][0]["destination"] == bucket_name
+    assert resp["exportTasks"][0]["from"] == fromTime
+    assert resp["exportTasks"][0]["to"] == to
+    assert resp["exportTasks"][0]["status"]["code"] == "active"
+    assert resp["exportTasks"][0]["status"]["message"] == "Task is active"
+
+
+def test_describe_export_tasks_raises_ResourceNotFoundException_task_id_not_found(
+    logs,
+):  # pylint: disable=redefined-outer-name
+    with pytest.raises(logs.exceptions.ResourceNotFoundException):
+        logs.describe_export_tasks(taskId="368a7022dea3dd621")

tests/test_logs/test_integration.py

@@ -480,7 +480,7 @@ def test_delete_subscription_filter_errors():
     assert ex.operation_name == "DeleteSubscriptionFilter"
     assert ex.response["ResponseMetadata"]["HTTPStatusCode"] == 400
     assert ex.response["Error"]["Code"] == "ResourceNotFoundException"
-    assert ex.response["Error"]["Message"] == "The specified log group does not exist"
+    assert ex.response["Error"]["Message"] == "The specified log group does not exist."
 
     # when
     with pytest.raises(ClientError) as e:
@@ -529,7 +529,7 @@ def test_put_subscription_filter_errors():
     assert ex.operation_name == "PutSubscriptionFilter"
     assert ex.response["ResponseMetadata"]["HTTPStatusCode"] == 400
     assert ex.response["Error"]["Code"] == "ResourceNotFoundException"
-    assert ex.response["Error"]["Message"] == "The specified log group does not exist"
+    assert ex.response["Error"]["Message"] == "The specified log group does not exist."
 
     # when
     with pytest.raises(ClientError) as e:

tests/test_logs/test_logs.py

@@ -1,13 +1,12 @@
 import json
-from datetime import datetime, timedelta
+from datetime import timedelta
-from uuid import UUID
 
 import boto3
 import pytest
 from botocore.exceptions import ClientError
 from freezegun import freeze_time
 
-from moto import mock_logs, mock_s3, settings
+from moto import mock_logs, settings
 from moto.core.utils import unix_time_millis, utcnow
 from moto.logs.models import MAX_RESOURCE_POLICIES_PER_REGION
 
@@ -1078,7 +1077,7 @@ def test_describe_subscription_filters_errors():
     assert "ResourceNotFoundException" in exc_value.response["Error"]["Code"]
     assert (
         exc_value.response["Error"]["Message"]
-        == "The specified log group does not exist"
+        == "The specified log group does not exist."
     )
 
 
@@ -1370,152 +1369,3 @@ def test_describe_log_streams_no_prefix():
     err = ex.value.response["Error"]
     assert err["Code"] == "InvalidParameterException"
     assert err["Message"] == "Cannot order by LastEventTime with a logStreamNamePrefix."
-
-
-@mock_s3
-@mock_logs
-def test_create_export_task_happy_path():
-    log_group_name = "/aws/codebuild/blah1"
-    destination = "mybucket"
-    fromTime = 1611316574
-    to = 1642852574
-    logs = boto3.client("logs", region_name="ap-southeast-1")
-    s3 = boto3.client("s3", "us-east-1")
-    logs.create_log_group(logGroupName=log_group_name)
-    s3.create_bucket(Bucket=destination)
-    resp = logs.create_export_task(
-        logGroupName=log_group_name,
-        fromTime=fromTime,
-        to=to,
-        destination=destination,
-    )
-    # taskId resembles a valid UUID (i.e. a string of 32 hexadecimal digits)
-    assert UUID(resp["taskId"])
-    assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
-
-
-@mock_logs
-def test_create_export_task_raises_ClientError_when_bucket_not_found():
-    log_group_name = "/aws/codebuild/blah1"
-    destination = "368a7022dea3dd621"
-    fromTime = 1611316574
-    to = 1642852574
-    logs = boto3.client("logs", region_name="ap-southeast-1")
-    logs.create_log_group(logGroupName=log_group_name)
-    with pytest.raises(ClientError):
-        logs.create_export_task(
-            logGroupName=log_group_name,
-            fromTime=fromTime,
-            to=to,
-            destination=destination,
-        )
-
-
-@mock_s3
-@mock_logs
-def test_create_export_raises_ResourceNotFoundException_log_group_not_found():
-    log_group_name = "/aws/codebuild/blah1"
-    destination = "mybucket"
-    fromTime = 1611316574
-    to = 1642852574
-    s3 = boto3.client("s3", region_name="us-east-1")
-    s3.create_bucket(Bucket=destination)
-    logs = boto3.client("logs", region_name="ap-southeast-1")
-    with pytest.raises(logs.exceptions.ResourceNotFoundException):
-        logs.create_export_task(
-            logGroupName=log_group_name,
-            fromTime=fromTime,
-            to=to,
-            destination=destination,
-        )
-
-
-@mock_s3
-@mock_logs
-def test_create_export_executes_export_task():
-    log_group_name = "/aws/codebuild/blah1"
-    destination = "mybucket"
-    fromTime = int(datetime.now().replace(hour=0, minute=0, second=0).timestamp())
-    to = int(datetime.now().replace(hour=23, minute=59, second=59).timestamp())
-    logs = boto3.client("logs", region_name="ap-southeast-1")
-    s3 = boto3.client("s3", "us-east-1")
-    logs.create_log_group(logGroupName=log_group_name)
-    s3.create_bucket(Bucket=destination)
-    resp = logs.create_export_task(
-        logGroupName=log_group_name,
-        fromTime=fromTime,
-        to=to,
-        destination=destination,
-    )
-    taskId = resp["taskId"]
-    resp = logs.describe_export_tasks(taskId=taskId)
-    assert len(resp["exportTasks"]) == 1
-    assert resp["exportTasks"][0]["logGroupName"] == log_group_name
-    assert resp["exportTasks"][0]["destination"] == destination
-    assert resp["exportTasks"][0]["from"] == fromTime
-    assert resp["exportTasks"][0]["to"] == to
-    assert resp["exportTasks"][0]["status"]["code"] == "completed"
-    assert resp["exportTasks"][0]["status"]["message"] == "Task is completed"
-
-
-@mock_s3
-@mock_logs
-def test_describe_export_tasks_happy_path():
-    log_group_name = "/aws/codebuild/blah1"
-    destination = "mybucket"
-    fromTime = 1611316574
-    to = 1642852574
-    logs = boto3.client("logs", region_name="ap-southeast-1")
-    s3 = boto3.client("s3", "us-east-1")
-    logs.create_log_group(logGroupName=log_group_name)
-    s3.create_bucket(Bucket=destination)
-    logs.create_export_task(
-        logGroupName=log_group_name,
-        fromTime=fromTime,
-        to=to,
-        destination=destination,
-    )
-    resp = logs.describe_export_tasks()
-    assert len(resp["exportTasks"]) == 1
-    assert resp["exportTasks"][0]["logGroupName"] == log_group_name
-    assert resp["exportTasks"][0]["destination"] == destination
-    assert resp["exportTasks"][0]["from"] == fromTime
-    assert resp["exportTasks"][0]["to"] == to
-    assert resp["exportTasks"][0]["status"]["code"] == "active"
-    assert resp["exportTasks"][0]["status"]["message"] == "Task is active"
-
-
-@mock_s3
-@mock_logs
-def test_describe_export_tasks_task_id():
-    log_group_name = "/aws/codebuild/blah1"
-    destination = "mybucket"
-    fromTime = 1611316574
-    to = 1642852574
-    logs = boto3.client("logs", region_name="ap-southeast-1")
-    s3 = boto3.client("s3", "us-east-1")
-    logs.create_log_group(logGroupName=log_group_name)
-    s3.create_bucket(Bucket=destination)
-    resp = logs.create_export_task(
-        logGroupName=log_group_name,
-        fromTime=fromTime,
-        to=to,
-        destination=destination,
-    )
-    taskId = resp["taskId"]
-    resp = logs.describe_export_tasks(taskId=taskId)
-    assert len(resp["exportTasks"]) == 1
-    assert resp["exportTasks"][0]["logGroupName"] == log_group_name
-    assert resp["exportTasks"][0]["destination"] == destination
-    assert resp["exportTasks"][0]["from"] == fromTime
-    assert resp["exportTasks"][0]["to"] == to
-    assert resp["exportTasks"][0]["status"]["code"] == "active"
-    assert resp["exportTasks"][0]["status"]["message"] == "Task is active"
-
-
-@mock_s3
-@mock_logs
-def test_describe_export_tasks_raises_ResourceNotFoundException_task_id_not_found():
-    logs = boto3.client("logs", region_name="ap-southeast-1")
-    with pytest.raises(logs.exceptions.ResourceNotFoundException):
-        logs.describe_export_tasks(taskId="368a7022dea3dd621")

tests/test_logs/test_logs_query/test_boto3.py

@@ -34,12 +34,9 @@ def test_start_query__unknown_log_group():
         )
 
     # then
-    exc_value = exc.value
+    err = exc.value.response["Error"]
-    assert "ResourceNotFoundException" in exc_value.response["Error"]["Code"]
+    assert err["Code"] == "ResourceNotFoundException"
-    assert (
+    assert err["Message"] == "The specified log group does not exist."
-        exc_value.response["Error"]["Message"]
-        == "The specified log group does not exist"
-    )
 
 
 @mock_logs