Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Logs: create_export_task() now actually exports the data (#7082)

Browse Source
This commit is contained in:
Bert Blommers 2023-12-01 17:35:18 -01:00 committed by GitHub
parent 4506ff833d
commit ad316dd837
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 323 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
moto/logs/exceptions.py
View File

@ -11,7 +11,8 @@ class ResourceNotFoundException(LogsClientError):
def __init__(self, msg: Optional[str] = None):
self.code = 400
super().__init__(
"ResourceNotFoundException", msg or "The specified log group does not exist"
"ResourceNotFoundException",
msg or "The specified log group does not exist.",
)

55
moto/logs/models.py
View File

@ -1,5 +1,5 @@
import json
from datetime import datetime, timedelta
from gzip import compress as gzip_compress
from typing import Any, Dict, Iterable, List, Optional, Tuple
from moto.core import BackendDict, BaseBackend, BaseModel, CloudFormationModel
@ -13,7 +13,7 @@ from moto.logs.exceptions import (
from moto.logs.logs_query import execute_query
from moto.logs.metric_filters import MetricFilters
from moto.moto_api._internal import mock_random
from moto.s3.models import s3_backends
from moto.s3.models import MissingBucket, s3_backends
from moto.utilities.paginator import paginate
from moto.utilities.tagging_service import TaggingService
@ -1248,7 +1248,12 @@ class LogsBackend(BaseBackend):
fromTime: int,
to: int,
) -> str:
s3_backends[self.account_id]["global"].get_bucket(destination)
try:
s3_backends[self.account_id]["global"].get_bucket(destination)
except MissingBucket:
raise InvalidParameterException(
"The given bucket does not exist. Please make sure the bucket is valid."
)
if logGroupName not in self.groups:
raise ResourceNotFoundException()
task_id = str(mock_random.uuid4())
@ -1261,15 +1266,41 @@ class LogsBackend(BaseBackend):
fromTime,
to,
)
if fromTime <= datetime.now().timestamp() <= to:
logs = self.filter_log_events(
logGroupName, [], fromTime, to, None, None, "", False
)
s3_backends[self.account_id]["global"].put_object(
destination, destinationPrefix, json.dumps(logs).encode("utf-8")
)
self.export_tasks[task_id].status["code"] = "completed"
self.export_tasks[task_id].status["message"] = "Task is completed"
s3_backends[self.account_id]["global"].put_object(
bucket_name=destination,
key_name="aws-logs-write-test",
value=b"Permission Check Successful",
)
if fromTime <= unix_time_millis(datetime.now()) <= to:
for stream_name in self.groups[logGroupName].streams.keys():
logs, _, _ = self.filter_log_events(
log_group_name=logGroupName,
log_stream_names=[stream_name],
start_time=fromTime,
end_time=to,
limit=None,
next_token=None,
filter_pattern="",
interleaved=False,
)
raw_logs = "\n".join(
[
f"{datetime.fromtimestamp(log['timestamp']/1000).strftime('%Y-%m-%dT%H:%M:%S.000Z')} {log['message']}"
for log in logs
]
)
folder = str(mock_random.uuid4()) + "/" + stream_name.replace("/", "-")
key_name = f"{destinationPrefix}/{folder}/000000.gz"
s3_backends[self.account_id]["global"].put_object(
bucket_name=destination,
key_name=key_name,
value=gzip_compress(raw_logs.encode("utf-8")),
)
self.export_tasks[task_id].status["code"] = "COMPLETED"
self.export_tasks[task_id].status["message"] = "Completed successfully"
return task_id
def describe_export_tasks(self, task_id: str) -> List[ExportTask]:

2
moto/logs/responses.py
View File

@ -438,7 +438,7 @@ class LogsResponse(BaseResponse):
fromTime=self._get_int_param("from"),
to=self._get_int_param("to"),
destination=self._get_param("destination"),
destinationPrefix=self._get_param("destinationPrefix"),
destinationPrefix=self._get_param("destinationPrefix", "exportedlogs"),
taskName=self._get_param("taskName"),
)
return json.dumps(dict(taskId=str(task_id)))

269
tests/test_logs/test_export_tasks.py Normal file
View File

@ -0,0 +1,269 @@
import copy
import json
import os
from datetime import datetime, timedelta
from uuid import UUID, uuid4
import boto3
import pytest
from botocore.exceptions import ClientError
from moto import mock_logs, mock_s3, mock_sts, settings
from moto.core.utils import unix_time_millis
TEST_REGION = "us-east-1" if settings.TEST_SERVER_MODE else "us-west-2"
allow_aws_request = (
os.environ.get("MOTO_TEST_ALLOW_AWS_REQUEST", "false").lower() == "true"
)
S3_POLICY = {
"Version": "2012-10-17",
"Statement": [
{
"Action": "s3:GetBucketAcl",
"Effect": "Allow",
"Resource": "arn:aws:s3:::{BUCKET_NAME}",
"Principal": {"Service": "logs.us-east-1.amazonaws.com"},
"Condition": {
"StringEquals": {"aws:SourceAccount": ["AccountId1"]},
"ArnLike": {"aws:SourceArn": ["..."]},
},
},
{
"Action": "s3:PutObject",
"Effect": "Allow",
"Resource": "arn:aws:s3:::{BUCKET_NAME}/*",
"Principal": {"Service": "logs.us-east-1.amazonaws.com"},
"Condition": {
"StringEquals": {
"s3:x-amz-acl": "bucket-owner-full-control",
"aws:SourceAccount": ["AccountId1"],
},
"ArnLike": {"aws:SourceArn": ["..."]},
},
},
],
}
@pytest.fixture()
def account_id():
if allow_aws_request:
identity = boto3.client("sts", region_name="us-east-1").get_caller_identity()
yield identity["Account"]
else:
with mock_sts():
identity = boto3.client(
"sts", region_name="us-east-1"
).get_caller_identity()
yield identity["Account"]
@pytest.fixture()
def logs():
if allow_aws_request:
yield boto3.client("logs", region_name="us-east-1")
else:
with mock_logs():
yield boto3.client("logs", region_name="us-east-1")
@pytest.fixture()
def s3():
if allow_aws_request:
yield boto3.client("s3", region_name="us-east-1")
else:
with mock_s3():
yield boto3.client("s3", region_name="us-east-1")
@pytest.fixture(scope="function")
def log_group_name(logs): # pylint: disable=redefined-outer-name
name = "/moto/logs_test/" + str(uuid4())[0:5]
logs.create_log_group(logGroupName=name)
yield name
logs.delete_log_group(logGroupName=name)
@pytest.fixture()
def bucket_name(s3, account_id): # pylint: disable=redefined-outer-name
name = f"moto-logs-test-{str(uuid4())[0:6]}"
s3.create_bucket(Bucket=name)
policy = copy.copy(S3_POLICY)
policy["Statement"][0]["Resource"] = f"arn:aws:s3:::{name}"
policy["Statement"][0]["Condition"]["StringEquals"]["aws:SourceAccount"] = [
account_id
]
policy["Statement"][0]["Condition"]["ArnLike"]["aws:SourceArn"] = [
f"arn:aws:logs:us-east-1:{account_id}:log-group:*"
]
policy["Statement"][1]["Resource"] = f"arn:aws:s3:::{name}/*"
policy["Statement"][1]["Condition"]["StringEquals"]["aws:SourceAccount"] = [
account_id
]
policy["Statement"][1]["Condition"]["ArnLike"]["aws:SourceArn"] = [
f"arn:aws:logs:us-east-1:{account_id}:log-group:*"
]
s3.put_bucket_policy(Bucket=name, Policy=json.dumps(policy))
yield name
# Delete any files left behind
versions = s3.list_object_versions(Bucket=name).get("Versions", [])
for key in versions:
s3.delete_object(Bucket=name, Key=key["Key"], VersionId=key.get("VersionId"))
delete_markers = s3.list_object_versions(Bucket=name).get("DeleteMarkers", [])
for key in delete_markers:
s3.delete_object(Bucket=name, Key=key["Key"], VersionId=key.get("VersionId"))
# Delete bucket itself
s3.delete_bucket(Bucket=name)
@pytest.mark.aws_verified
def test_create_export_task_happy_path(
logs, s3, log_group_name, bucket_name
): # pylint: disable=redefined-outer-name
fromTime = 1611316574
to = 1642852574
resp = logs.create_export_task(
logGroupName=log_group_name,
fromTime=fromTime,
to=to,
destination=bucket_name,
)
# taskId resembles a valid UUID (i.e. a string of 32 hexadecimal digits)
assert UUID(resp["taskId"])
assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
# s3 bucket contains indication that permissions were successful
resp = s3.get_object(Bucket=bucket_name, Key="aws-logs-write-test")
assert resp["Body"].read() == b"Permission Check Successful"
@pytest.mark.aws_verified
def test_create_export_task_raises_ClientError_when_bucket_not_found(
logs, log_group_name # pylint: disable=redefined-outer-name
):
destination = "368a7022dea3dd621"
fromTime = 1611316574
to = 1642852574
with pytest.raises(ClientError) as exc:
logs.create_export_task(
logGroupName=log_group_name,
fromTime=fromTime,
to=to,
destination=destination,
)
err = exc.value.response["Error"]
assert err["Code"] == "InvalidParameterException"
assert (
err["Message"]
== "The given bucket does not exist. Please make sure the bucket is valid."
)
@pytest.mark.aws_verified
def test_create_export_raises_ResourceNotFoundException_log_group_not_found(
logs, bucket_name # pylint: disable=redefined-outer-name
):
with pytest.raises(logs.exceptions.ResourceNotFoundException) as exc:
logs.create_export_task(
logGroupName=f"/aws/nonexisting/{str(uuid4())[0:6]}",
fromTime=1611316574,
to=1642852574,
destination=bucket_name,
)
err = exc.value.response["Error"]
assert err["Code"] == "ResourceNotFoundException"
assert err["Message"] == "The specified log group does not exist."
@pytest.mark.aws_verified
def test_create_export_executes_export_task(
logs, s3, log_group_name, bucket_name
): # pylint: disable=redefined-outer-name
fromTime = int(unix_time_millis(datetime.now() - timedelta(days=1)))
to = int(unix_time_millis(datetime.now() + timedelta(days=1)))
logs.create_log_stream(logGroupName=log_group_name, logStreamName="/some/stream")
messages = [
{"timestamp": int(unix_time_millis()), "message": f"hello_{i}"}
for i in range(10)
]
logs.put_log_events(
logGroupName=log_group_name, logStreamName="/some/stream", logEvents=messages
)
task_id = logs.create_export_task(
logGroupName=log_group_name,
fromTime=fromTime,
to=to,
destination=bucket_name,
)["taskId"]
task = logs.describe_export_tasks(taskId=task_id)["exportTasks"][0]
assert task["status"]["code"] in ["COMPLETED", "RUNNING"]
assert task["logGroupName"] == log_group_name
assert task["destination"] == bucket_name
assert task["destinationPrefix"] == "exportedlogs"
assert task["from"] == fromTime
assert task["to"] == to
objects = s3.list_objects(Bucket=bucket_name)["Contents"]
key_names = [o["Key"] for o in objects]
assert "aws-logs-write-test" in key_names
def test_describe_export_tasks_happy_path(
logs, s3, log_group_name
): # pylint: disable=redefined-outer-name
destination = "mybucket"
fromTime = 1611316574
to = 1642852574
s3.create_bucket(Bucket=destination)
logs.create_export_task(
logGroupName=log_group_name,
fromTime=fromTime,
to=to,
destination=destination,
)
resp = logs.describe_export_tasks()
assert len(resp["exportTasks"]) == 1
assert resp["exportTasks"][0]["logGroupName"] == log_group_name
assert resp["exportTasks"][0]["destination"] == destination
assert resp["exportTasks"][0]["from"] == fromTime
assert resp["exportTasks"][0]["to"] == to
assert resp["exportTasks"][0]["status"]["code"] == "active"
assert resp["exportTasks"][0]["status"]["message"] == "Task is active"
def test_describe_export_tasks_task_id(
logs, log_group_name, bucket_name
): # pylint: disable=redefined-outer-name
fromTime = 1611316574
to = 1642852574
resp = logs.create_export_task(
logGroupName=log_group_name,
fromTime=fromTime,
to=to,
destination=bucket_name,
)
taskId = resp["taskId"]
resp = logs.describe_export_tasks(taskId=taskId)
assert len(resp["exportTasks"]) == 1
assert resp["exportTasks"][0]["logGroupName"] == log_group_name
assert resp["exportTasks"][0]["destination"] == bucket_name
assert resp["exportTasks"][0]["from"] == fromTime
assert resp["exportTasks"][0]["to"] == to
assert resp["exportTasks"][0]["status"]["code"] == "active"
assert resp["exportTasks"][0]["status"]["message"] == "Task is active"
def test_describe_export_tasks_raises_ResourceNotFoundException_task_id_not_found(
logs,
): # pylint: disable=redefined-outer-name
with pytest.raises(logs.exceptions.ResourceNotFoundException):
logs.describe_export_tasks(taskId="368a7022dea3dd621")

4
tests/test_logs/test_integration.py
View File

@ -480,7 +480,7 @@ def test_delete_subscription_filter_errors():
assert ex.operation_name == "DeleteSubscriptionFilter"
assert ex.response["ResponseMetadata"]["HTTPStatusCode"] == 400
assert ex.response["Error"]["Code"] == "ResourceNotFoundException"
assert ex.response["Error"]["Message"] == "The specified log group does not exist"
assert ex.response["Error"]["Message"] == "The specified log group does not exist."
# when
with pytest.raises(ClientError) as e:
@ -529,7 +529,7 @@ def test_put_subscription_filter_errors():
assert ex.operation_name == "PutSubscriptionFilter"
assert ex.response["ResponseMetadata"]["HTTPStatusCode"] == 400
assert ex.response["Error"]["Code"] == "ResourceNotFoundException"
assert ex.response["Error"]["Message"] == "The specified log group does not exist"
assert ex.response["Error"]["Message"] == "The specified log group does not exist."
# when
with pytest.raises(ClientError) as e:

156
tests/test_logs/test_logs.py
View File

@ -1,13 +1,12 @@
import json
from datetime import datetime, timedelta
from uuid import UUID
from datetime import timedelta
import boto3
import pytest
from botocore.exceptions import ClientError
from freezegun import freeze_time
from moto import mock_logs, mock_s3, settings
from moto import mock_logs, settings
from moto.core.utils import unix_time_millis, utcnow
from moto.logs.models import MAX_RESOURCE_POLICIES_PER_REGION
@ -1078,7 +1077,7 @@ def test_describe_subscription_filters_errors():
assert "ResourceNotFoundException" in exc_value.response["Error"]["Code"]
assert (
exc_value.response["Error"]["Message"]
== "The specified log group does not exist"
== "The specified log group does not exist."
)
@ -1370,152 +1369,3 @@ def test_describe_log_streams_no_prefix():
err = ex.value.response["Error"]
assert err["Code"] == "InvalidParameterException"
assert err["Message"] == "Cannot order by LastEventTime with a logStreamNamePrefix."
@mock_s3
@mock_logs
def test_create_export_task_happy_path():
log_group_name = "/aws/codebuild/blah1"
destination = "mybucket"
fromTime = 1611316574
to = 1642852574
logs = boto3.client("logs", region_name="ap-southeast-1")
s3 = boto3.client("s3", "us-east-1")
logs.create_log_group(logGroupName=log_group_name)
s3.create_bucket(Bucket=destination)
resp = logs.create_export_task(
logGroupName=log_group_name,
fromTime=fromTime,
to=to,
destination=destination,
)
# taskId resembles a valid UUID (i.e. a string of 32 hexadecimal digits)
assert UUID(resp["taskId"])
assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
@mock_logs
def test_create_export_task_raises_ClientError_when_bucket_not_found():
log_group_name = "/aws/codebuild/blah1"
destination = "368a7022dea3dd621"
fromTime = 1611316574
to = 1642852574
logs = boto3.client("logs", region_name="ap-southeast-1")
logs.create_log_group(logGroupName=log_group_name)
with pytest.raises(ClientError):
logs.create_export_task(
logGroupName=log_group_name,
fromTime=fromTime,
to=to,
destination=destination,
)
@mock_s3
@mock_logs
def test_create_export_raises_ResourceNotFoundException_log_group_not_found():
log_group_name = "/aws/codebuild/blah1"
destination = "mybucket"
fromTime = 1611316574
to = 1642852574
s3 = boto3.client("s3", region_name="us-east-1")
s3.create_bucket(Bucket=destination)
logs = boto3.client("logs", region_name="ap-southeast-1")
with pytest.raises(logs.exceptions.ResourceNotFoundException):
logs.create_export_task(
logGroupName=log_group_name,
fromTime=fromTime,
to=to,
destination=destination,
)
@mock_s3
@mock_logs
def test_create_export_executes_export_task():
log_group_name = "/aws/codebuild/blah1"
destination = "mybucket"
fromTime = int(datetime.now().replace(hour=0, minute=0, second=0).timestamp())
to = int(datetime.now().replace(hour=23, minute=59, second=59).timestamp())
logs = boto3.client("logs", region_name="ap-southeast-1")
s3 = boto3.client("s3", "us-east-1")
logs.create_log_group(logGroupName=log_group_name)
s3.create_bucket(Bucket=destination)
resp = logs.create_export_task(
logGroupName=log_group_name,
fromTime=fromTime,
to=to,
destination=destination,
)
taskId = resp["taskId"]
resp = logs.describe_export_tasks(taskId=taskId)
assert len(resp["exportTasks"]) == 1
assert resp["exportTasks"][0]["logGroupName"] == log_group_name
assert resp["exportTasks"][0]["destination"] == destination
assert resp["exportTasks"][0]["from"] == fromTime
assert resp["exportTasks"][0]["to"] == to
assert resp["exportTasks"][0]["status"]["code"] == "completed"
assert resp["exportTasks"][0]["status"]["message"] == "Task is completed"
@mock_s3
@mock_logs
def test_describe_export_tasks_happy_path():
log_group_name = "/aws/codebuild/blah1"
destination = "mybucket"
fromTime = 1611316574
to = 1642852574
logs = boto3.client("logs", region_name="ap-southeast-1")
s3 = boto3.client("s3", "us-east-1")
logs.create_log_group(logGroupName=log_group_name)
s3.create_bucket(Bucket=destination)
logs.create_export_task(
logGroupName=log_group_name,
fromTime=fromTime,
to=to,
destination=destination,
)
resp = logs.describe_export_tasks()
assert len(resp["exportTasks"]) == 1
assert resp["exportTasks"][0]["logGroupName"] == log_group_name
assert resp["exportTasks"][0]["destination"] == destination
assert resp["exportTasks"][0]["from"] == fromTime
assert resp["exportTasks"][0]["to"] == to
assert resp["exportTasks"][0]["status"]["code"] == "active"
assert resp["exportTasks"][0]["status"]["message"] == "Task is active"
@mock_s3
@mock_logs
def test_describe_export_tasks_task_id():
log_group_name = "/aws/codebuild/blah1"
destination = "mybucket"
fromTime = 1611316574
to = 1642852574
logs = boto3.client("logs", region_name="ap-southeast-1")
s3 = boto3.client("s3", "us-east-1")
logs.create_log_group(logGroupName=log_group_name)
s3.create_bucket(Bucket=destination)
resp = logs.create_export_task(
logGroupName=log_group_name,
fromTime=fromTime,
to=to,
destination=destination,
)
taskId = resp["taskId"]
resp = logs.describe_export_tasks(taskId=taskId)
assert len(resp["exportTasks"]) == 1
assert resp["exportTasks"][0]["logGroupName"] == log_group_name
assert resp["exportTasks"][0]["destination"] == destination
assert resp["exportTasks"][0]["from"] == fromTime
assert resp["exportTasks"][0]["to"] == to
assert resp["exportTasks"][0]["status"]["code"] == "active"
assert resp["exportTasks"][0]["status"]["message"] == "Task is active"
@mock_s3
@mock_logs
def test_describe_export_tasks_raises_ResourceNotFoundException_task_id_not_found():
logs = boto3.client("logs", region_name="ap-southeast-1")
with pytest.raises(logs.exceptions.ResourceNotFoundException):
logs.describe_export_tasks(taskId="368a7022dea3dd621")

9
tests/test_logs/test_logs_query/test_boto3.py
View File

@ -34,12 +34,9 @@ def test_start_query__unknown_log_group():
)
# then
exc_value = exc.value
assert "ResourceNotFoundException" in exc_value.response["Error"]["Code"]
assert (
exc_value.response["Error"]["Message"]
== "The specified log group does not exist"
)
err = exc.value.response["Error"]
assert err["Code"] == "ResourceNotFoundException"
assert err["Message"] == "The specified log group does not exist."
@mock_logs