diff --git a/moto/logs/exceptions.py b/moto/logs/exceptions.py index a4118febb..4aea395c5 100644 --- a/moto/logs/exceptions.py +++ b/moto/logs/exceptions.py @@ -11,7 +11,8 @@ class ResourceNotFoundException(LogsClientError): def __init__(self, msg: Optional[str] = None): self.code = 400 super().__init__( - "ResourceNotFoundException", msg or "The specified log group does not exist" + "ResourceNotFoundException", + msg or "The specified log group does not exist.", ) diff --git a/moto/logs/models.py b/moto/logs/models.py index e7ac94a36..fbff77a9f 100644 --- a/moto/logs/models.py +++ b/moto/logs/models.py @@ -1,5 +1,5 @@ -import json from datetime import datetime, timedelta +from gzip import compress as gzip_compress from typing import Any, Dict, Iterable, List, Optional, Tuple from moto.core import BackendDict, BaseBackend, BaseModel, CloudFormationModel @@ -13,7 +13,7 @@ from moto.logs.exceptions import ( from moto.logs.logs_query import execute_query from moto.logs.metric_filters import MetricFilters from moto.moto_api._internal import mock_random -from moto.s3.models import s3_backends +from moto.s3.models import MissingBucket, s3_backends from moto.utilities.paginator import paginate from moto.utilities.tagging_service import TaggingService @@ -1248,7 +1248,12 @@ class LogsBackend(BaseBackend): fromTime: int, to: int, ) -> str: - s3_backends[self.account_id]["global"].get_bucket(destination) + try: + s3_backends[self.account_id]["global"].get_bucket(destination) + except MissingBucket: + raise InvalidParameterException( + "The given bucket does not exist. Please make sure the bucket is valid." + ) if logGroupName not in self.groups: raise ResourceNotFoundException() task_id = str(mock_random.uuid4()) @@ -1261,15 +1266,41 @@ class LogsBackend(BaseBackend): fromTime, to, ) - if fromTime <= datetime.now().timestamp() <= to: - logs = self.filter_log_events( - logGroupName, [], fromTime, to, None, None, "", False - ) - s3_backends[self.account_id]["global"].put_object( - destination, destinationPrefix, json.dumps(logs).encode("utf-8") - ) - self.export_tasks[task_id].status["code"] = "completed" - self.export_tasks[task_id].status["message"] = "Task is completed" + + s3_backends[self.account_id]["global"].put_object( + bucket_name=destination, + key_name="aws-logs-write-test", + value=b"Permission Check Successful", + ) + + if fromTime <= unix_time_millis(datetime.now()) <= to: + for stream_name in self.groups[logGroupName].streams.keys(): + logs, _, _ = self.filter_log_events( + log_group_name=logGroupName, + log_stream_names=[stream_name], + start_time=fromTime, + end_time=to, + limit=None, + next_token=None, + filter_pattern="", + interleaved=False, + ) + raw_logs = "\n".join( + [ + f"{datetime.fromtimestamp(log['timestamp']/1000).strftime('%Y-%m-%dT%H:%M:%S.000Z')} {log['message']}" + for log in logs + ] + ) + folder = str(mock_random.uuid4()) + "/" + stream_name.replace("/", "-") + key_name = f"{destinationPrefix}/{folder}/000000.gz" + s3_backends[self.account_id]["global"].put_object( + bucket_name=destination, + key_name=key_name, + value=gzip_compress(raw_logs.encode("utf-8")), + ) + self.export_tasks[task_id].status["code"] = "COMPLETED" + self.export_tasks[task_id].status["message"] = "Completed successfully" + return task_id def describe_export_tasks(self, task_id: str) -> List[ExportTask]: diff --git a/moto/logs/responses.py b/moto/logs/responses.py index 9b211018f..0a38fbe62 100644 --- a/moto/logs/responses.py +++ b/moto/logs/responses.py @@ -438,7 +438,7 @@ class LogsResponse(BaseResponse): fromTime=self._get_int_param("from"), to=self._get_int_param("to"), destination=self._get_param("destination"), - destinationPrefix=self._get_param("destinationPrefix"), + destinationPrefix=self._get_param("destinationPrefix", "exportedlogs"), taskName=self._get_param("taskName"), ) return json.dumps(dict(taskId=str(task_id))) diff --git a/tests/test_logs/test_export_tasks.py b/tests/test_logs/test_export_tasks.py new file mode 100644 index 000000000..7f685f734 --- /dev/null +++ b/tests/test_logs/test_export_tasks.py @@ -0,0 +1,269 @@ +import copy +import json +import os +from datetime import datetime, timedelta +from uuid import UUID, uuid4 + +import boto3 +import pytest +from botocore.exceptions import ClientError + +from moto import mock_logs, mock_s3, mock_sts, settings +from moto.core.utils import unix_time_millis + +TEST_REGION = "us-east-1" if settings.TEST_SERVER_MODE else "us-west-2" +allow_aws_request = ( + os.environ.get("MOTO_TEST_ALLOW_AWS_REQUEST", "false").lower() == "true" +) + + +S3_POLICY = { + "Version": "2012-10-17", + "Statement": [ + { + "Action": "s3:GetBucketAcl", + "Effect": "Allow", + "Resource": "arn:aws:s3:::{BUCKET_NAME}", + "Principal": {"Service": "logs.us-east-1.amazonaws.com"}, + "Condition": { + "StringEquals": {"aws:SourceAccount": ["AccountId1"]}, + "ArnLike": {"aws:SourceArn": ["..."]}, + }, + }, + { + "Action": "s3:PutObject", + "Effect": "Allow", + "Resource": "arn:aws:s3:::{BUCKET_NAME}/*", + "Principal": {"Service": "logs.us-east-1.amazonaws.com"}, + "Condition": { + "StringEquals": { + "s3:x-amz-acl": "bucket-owner-full-control", + "aws:SourceAccount": ["AccountId1"], + }, + "ArnLike": {"aws:SourceArn": ["..."]}, + }, + }, + ], +} + + +@pytest.fixture() +def account_id(): + if allow_aws_request: + identity = boto3.client("sts", region_name="us-east-1").get_caller_identity() + yield identity["Account"] + else: + with mock_sts(): + identity = boto3.client( + "sts", region_name="us-east-1" + ).get_caller_identity() + yield identity["Account"] + + +@pytest.fixture() +def logs(): + if allow_aws_request: + yield boto3.client("logs", region_name="us-east-1") + else: + with mock_logs(): + yield boto3.client("logs", region_name="us-east-1") + + +@pytest.fixture() +def s3(): + if allow_aws_request: + yield boto3.client("s3", region_name="us-east-1") + else: + with mock_s3(): + yield boto3.client("s3", region_name="us-east-1") + + +@pytest.fixture(scope="function") +def log_group_name(logs): # pylint: disable=redefined-outer-name + name = "/moto/logs_test/" + str(uuid4())[0:5] + logs.create_log_group(logGroupName=name) + yield name + logs.delete_log_group(logGroupName=name) + + +@pytest.fixture() +def bucket_name(s3, account_id): # pylint: disable=redefined-outer-name + name = f"moto-logs-test-{str(uuid4())[0:6]}" + s3.create_bucket(Bucket=name) + policy = copy.copy(S3_POLICY) + policy["Statement"][0]["Resource"] = f"arn:aws:s3:::{name}" + policy["Statement"][0]["Condition"]["StringEquals"]["aws:SourceAccount"] = [ + account_id + ] + policy["Statement"][0]["Condition"]["ArnLike"]["aws:SourceArn"] = [ + f"arn:aws:logs:us-east-1:{account_id}:log-group:*" + ] + policy["Statement"][1]["Resource"] = f"arn:aws:s3:::{name}/*" + policy["Statement"][1]["Condition"]["StringEquals"]["aws:SourceAccount"] = [ + account_id + ] + policy["Statement"][1]["Condition"]["ArnLike"]["aws:SourceArn"] = [ + f"arn:aws:logs:us-east-1:{account_id}:log-group:*" + ] + + s3.put_bucket_policy(Bucket=name, Policy=json.dumps(policy)) + yield name + + # Delete any files left behind + versions = s3.list_object_versions(Bucket=name).get("Versions", []) + for key in versions: + s3.delete_object(Bucket=name, Key=key["Key"], VersionId=key.get("VersionId")) + delete_markers = s3.list_object_versions(Bucket=name).get("DeleteMarkers", []) + for key in delete_markers: + s3.delete_object(Bucket=name, Key=key["Key"], VersionId=key.get("VersionId")) + + # Delete bucket itself + s3.delete_bucket(Bucket=name) + + +@pytest.mark.aws_verified +def test_create_export_task_happy_path( + logs, s3, log_group_name, bucket_name +): # pylint: disable=redefined-outer-name + fromTime = 1611316574 + to = 1642852574 + resp = logs.create_export_task( + logGroupName=log_group_name, + fromTime=fromTime, + to=to, + destination=bucket_name, + ) + # taskId resembles a valid UUID (i.e. a string of 32 hexadecimal digits) + assert UUID(resp["taskId"]) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # s3 bucket contains indication that permissions were successful + resp = s3.get_object(Bucket=bucket_name, Key="aws-logs-write-test") + assert resp["Body"].read() == b"Permission Check Successful" + + +@pytest.mark.aws_verified +def test_create_export_task_raises_ClientError_when_bucket_not_found( + logs, log_group_name # pylint: disable=redefined-outer-name +): + destination = "368a7022dea3dd621" + fromTime = 1611316574 + to = 1642852574 + with pytest.raises(ClientError) as exc: + logs.create_export_task( + logGroupName=log_group_name, + fromTime=fromTime, + to=to, + destination=destination, + ) + err = exc.value.response["Error"] + assert err["Code"] == "InvalidParameterException" + assert ( + err["Message"] + == "The given bucket does not exist. Please make sure the bucket is valid." + ) + + +@pytest.mark.aws_verified +def test_create_export_raises_ResourceNotFoundException_log_group_not_found( + logs, bucket_name # pylint: disable=redefined-outer-name +): + with pytest.raises(logs.exceptions.ResourceNotFoundException) as exc: + logs.create_export_task( + logGroupName=f"/aws/nonexisting/{str(uuid4())[0:6]}", + fromTime=1611316574, + to=1642852574, + destination=bucket_name, + ) + err = exc.value.response["Error"] + assert err["Code"] == "ResourceNotFoundException" + assert err["Message"] == "The specified log group does not exist." + + +@pytest.mark.aws_verified +def test_create_export_executes_export_task( + logs, s3, log_group_name, bucket_name +): # pylint: disable=redefined-outer-name + fromTime = int(unix_time_millis(datetime.now() - timedelta(days=1))) + to = int(unix_time_millis(datetime.now() + timedelta(days=1))) + + logs.create_log_stream(logGroupName=log_group_name, logStreamName="/some/stream") + + messages = [ + {"timestamp": int(unix_time_millis()), "message": f"hello_{i}"} + for i in range(10) + ] + logs.put_log_events( + logGroupName=log_group_name, logStreamName="/some/stream", logEvents=messages + ) + + task_id = logs.create_export_task( + logGroupName=log_group_name, + fromTime=fromTime, + to=to, + destination=bucket_name, + )["taskId"] + + task = logs.describe_export_tasks(taskId=task_id)["exportTasks"][0] + assert task["status"]["code"] in ["COMPLETED", "RUNNING"] + assert task["logGroupName"] == log_group_name + assert task["destination"] == bucket_name + assert task["destinationPrefix"] == "exportedlogs" + assert task["from"] == fromTime + assert task["to"] == to + + objects = s3.list_objects(Bucket=bucket_name)["Contents"] + key_names = [o["Key"] for o in objects] + assert "aws-logs-write-test" in key_names + + +def test_describe_export_tasks_happy_path( + logs, s3, log_group_name +): # pylint: disable=redefined-outer-name + destination = "mybucket" + fromTime = 1611316574 + to = 1642852574 + s3.create_bucket(Bucket=destination) + logs.create_export_task( + logGroupName=log_group_name, + fromTime=fromTime, + to=to, + destination=destination, + ) + resp = logs.describe_export_tasks() + assert len(resp["exportTasks"]) == 1 + assert resp["exportTasks"][0]["logGroupName"] == log_group_name + assert resp["exportTasks"][0]["destination"] == destination + assert resp["exportTasks"][0]["from"] == fromTime + assert resp["exportTasks"][0]["to"] == to + assert resp["exportTasks"][0]["status"]["code"] == "active" + assert resp["exportTasks"][0]["status"]["message"] == "Task is active" + + +def test_describe_export_tasks_task_id( + logs, log_group_name, bucket_name +): # pylint: disable=redefined-outer-name + fromTime = 1611316574 + to = 1642852574 + resp = logs.create_export_task( + logGroupName=log_group_name, + fromTime=fromTime, + to=to, + destination=bucket_name, + ) + taskId = resp["taskId"] + resp = logs.describe_export_tasks(taskId=taskId) + assert len(resp["exportTasks"]) == 1 + assert resp["exportTasks"][0]["logGroupName"] == log_group_name + assert resp["exportTasks"][0]["destination"] == bucket_name + assert resp["exportTasks"][0]["from"] == fromTime + assert resp["exportTasks"][0]["to"] == to + assert resp["exportTasks"][0]["status"]["code"] == "active" + assert resp["exportTasks"][0]["status"]["message"] == "Task is active" + + +def test_describe_export_tasks_raises_ResourceNotFoundException_task_id_not_found( + logs, +): # pylint: disable=redefined-outer-name + with pytest.raises(logs.exceptions.ResourceNotFoundException): + logs.describe_export_tasks(taskId="368a7022dea3dd621") diff --git a/tests/test_logs/test_integration.py b/tests/test_logs/test_integration.py index dbdbea54c..f8e6d64fa 100644 --- a/tests/test_logs/test_integration.py +++ b/tests/test_logs/test_integration.py @@ -480,7 +480,7 @@ def test_delete_subscription_filter_errors(): assert ex.operation_name == "DeleteSubscriptionFilter" assert ex.response["ResponseMetadata"]["HTTPStatusCode"] == 400 assert ex.response["Error"]["Code"] == "ResourceNotFoundException" - assert ex.response["Error"]["Message"] == "The specified log group does not exist" + assert ex.response["Error"]["Message"] == "The specified log group does not exist." # when with pytest.raises(ClientError) as e: @@ -529,7 +529,7 @@ def test_put_subscription_filter_errors(): assert ex.operation_name == "PutSubscriptionFilter" assert ex.response["ResponseMetadata"]["HTTPStatusCode"] == 400 assert ex.response["Error"]["Code"] == "ResourceNotFoundException" - assert ex.response["Error"]["Message"] == "The specified log group does not exist" + assert ex.response["Error"]["Message"] == "The specified log group does not exist." # when with pytest.raises(ClientError) as e: diff --git a/tests/test_logs/test_logs.py b/tests/test_logs/test_logs.py index 89c45c939..1e741717a 100644 --- a/tests/test_logs/test_logs.py +++ b/tests/test_logs/test_logs.py @@ -1,13 +1,12 @@ import json -from datetime import datetime, timedelta -from uuid import UUID +from datetime import timedelta import boto3 import pytest from botocore.exceptions import ClientError from freezegun import freeze_time -from moto import mock_logs, mock_s3, settings +from moto import mock_logs, settings from moto.core.utils import unix_time_millis, utcnow from moto.logs.models import MAX_RESOURCE_POLICIES_PER_REGION @@ -1078,7 +1077,7 @@ def test_describe_subscription_filters_errors(): assert "ResourceNotFoundException" in exc_value.response["Error"]["Code"] assert ( exc_value.response["Error"]["Message"] - == "The specified log group does not exist" + == "The specified log group does not exist." ) @@ -1370,152 +1369,3 @@ def test_describe_log_streams_no_prefix(): err = ex.value.response["Error"] assert err["Code"] == "InvalidParameterException" assert err["Message"] == "Cannot order by LastEventTime with a logStreamNamePrefix." - - -@mock_s3 -@mock_logs -def test_create_export_task_happy_path(): - log_group_name = "/aws/codebuild/blah1" - destination = "mybucket" - fromTime = 1611316574 - to = 1642852574 - logs = boto3.client("logs", region_name="ap-southeast-1") - s3 = boto3.client("s3", "us-east-1") - logs.create_log_group(logGroupName=log_group_name) - s3.create_bucket(Bucket=destination) - resp = logs.create_export_task( - logGroupName=log_group_name, - fromTime=fromTime, - to=to, - destination=destination, - ) - # taskId resembles a valid UUID (i.e. a string of 32 hexadecimal digits) - assert UUID(resp["taskId"]) - assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 - - -@mock_logs -def test_create_export_task_raises_ClientError_when_bucket_not_found(): - log_group_name = "/aws/codebuild/blah1" - destination = "368a7022dea3dd621" - fromTime = 1611316574 - to = 1642852574 - logs = boto3.client("logs", region_name="ap-southeast-1") - logs.create_log_group(logGroupName=log_group_name) - with pytest.raises(ClientError): - logs.create_export_task( - logGroupName=log_group_name, - fromTime=fromTime, - to=to, - destination=destination, - ) - - -@mock_s3 -@mock_logs -def test_create_export_raises_ResourceNotFoundException_log_group_not_found(): - log_group_name = "/aws/codebuild/blah1" - destination = "mybucket" - fromTime = 1611316574 - to = 1642852574 - s3 = boto3.client("s3", region_name="us-east-1") - s3.create_bucket(Bucket=destination) - logs = boto3.client("logs", region_name="ap-southeast-1") - with pytest.raises(logs.exceptions.ResourceNotFoundException): - logs.create_export_task( - logGroupName=log_group_name, - fromTime=fromTime, - to=to, - destination=destination, - ) - - -@mock_s3 -@mock_logs -def test_create_export_executes_export_task(): - log_group_name = "/aws/codebuild/blah1" - destination = "mybucket" - fromTime = int(datetime.now().replace(hour=0, minute=0, second=0).timestamp()) - to = int(datetime.now().replace(hour=23, minute=59, second=59).timestamp()) - logs = boto3.client("logs", region_name="ap-southeast-1") - s3 = boto3.client("s3", "us-east-1") - logs.create_log_group(logGroupName=log_group_name) - s3.create_bucket(Bucket=destination) - resp = logs.create_export_task( - logGroupName=log_group_name, - fromTime=fromTime, - to=to, - destination=destination, - ) - taskId = resp["taskId"] - resp = logs.describe_export_tasks(taskId=taskId) - assert len(resp["exportTasks"]) == 1 - assert resp["exportTasks"][0]["logGroupName"] == log_group_name - assert resp["exportTasks"][0]["destination"] == destination - assert resp["exportTasks"][0]["from"] == fromTime - assert resp["exportTasks"][0]["to"] == to - assert resp["exportTasks"][0]["status"]["code"] == "completed" - assert resp["exportTasks"][0]["status"]["message"] == "Task is completed" - - -@mock_s3 -@mock_logs -def test_describe_export_tasks_happy_path(): - log_group_name = "/aws/codebuild/blah1" - destination = "mybucket" - fromTime = 1611316574 - to = 1642852574 - logs = boto3.client("logs", region_name="ap-southeast-1") - s3 = boto3.client("s3", "us-east-1") - logs.create_log_group(logGroupName=log_group_name) - s3.create_bucket(Bucket=destination) - logs.create_export_task( - logGroupName=log_group_name, - fromTime=fromTime, - to=to, - destination=destination, - ) - resp = logs.describe_export_tasks() - assert len(resp["exportTasks"]) == 1 - assert resp["exportTasks"][0]["logGroupName"] == log_group_name - assert resp["exportTasks"][0]["destination"] == destination - assert resp["exportTasks"][0]["from"] == fromTime - assert resp["exportTasks"][0]["to"] == to - assert resp["exportTasks"][0]["status"]["code"] == "active" - assert resp["exportTasks"][0]["status"]["message"] == "Task is active" - - -@mock_s3 -@mock_logs -def test_describe_export_tasks_task_id(): - log_group_name = "/aws/codebuild/blah1" - destination = "mybucket" - fromTime = 1611316574 - to = 1642852574 - logs = boto3.client("logs", region_name="ap-southeast-1") - s3 = boto3.client("s3", "us-east-1") - logs.create_log_group(logGroupName=log_group_name) - s3.create_bucket(Bucket=destination) - resp = logs.create_export_task( - logGroupName=log_group_name, - fromTime=fromTime, - to=to, - destination=destination, - ) - taskId = resp["taskId"] - resp = logs.describe_export_tasks(taskId=taskId) - assert len(resp["exportTasks"]) == 1 - assert resp["exportTasks"][0]["logGroupName"] == log_group_name - assert resp["exportTasks"][0]["destination"] == destination - assert resp["exportTasks"][0]["from"] == fromTime - assert resp["exportTasks"][0]["to"] == to - assert resp["exportTasks"][0]["status"]["code"] == "active" - assert resp["exportTasks"][0]["status"]["message"] == "Task is active" - - -@mock_s3 -@mock_logs -def test_describe_export_tasks_raises_ResourceNotFoundException_task_id_not_found(): - logs = boto3.client("logs", region_name="ap-southeast-1") - with pytest.raises(logs.exceptions.ResourceNotFoundException): - logs.describe_export_tasks(taskId="368a7022dea3dd621") diff --git a/tests/test_logs/test_logs_query/test_boto3.py b/tests/test_logs/test_logs_query/test_boto3.py index 32e73cad3..2ce964ffa 100644 --- a/tests/test_logs/test_logs_query/test_boto3.py +++ b/tests/test_logs/test_logs_query/test_boto3.py @@ -34,12 +34,9 @@ def test_start_query__unknown_log_group(): ) # then - exc_value = exc.value - assert "ResourceNotFoundException" in exc_value.response["Error"]["Code"] - assert ( - exc_value.response["Error"]["Message"] - == "The specified log group does not exist" - ) + err = exc.value.response["Error"] + assert err["Code"] == "ResourceNotFoundException" + assert err["Message"] == "The specified log group does not exist." @mock_logs