1296 lines
45 KiB
Python
1296 lines
45 KiB
Python
import json
|
|
from datetime import timedelta, datetime
|
|
from typing import Any, Dict, Iterable, List, Tuple, Optional
|
|
from moto.core import BaseBackend, BackendDict, BaseModel
|
|
from moto.core import CloudFormationModel
|
|
from moto.core.utils import unix_time_millis, utcnow
|
|
from moto.logs.metric_filters import MetricFilters
|
|
from moto.logs.exceptions import (
|
|
ResourceNotFoundException,
|
|
ResourceAlreadyExistsException,
|
|
InvalidParameterException,
|
|
LimitExceededException,
|
|
)
|
|
from moto.logs.logs_query import execute_query
|
|
from moto.moto_api._internal import mock_random
|
|
from moto.s3.models import s3_backends
|
|
from moto.utilities.paginator import paginate
|
|
from moto.utilities.tagging_service import TaggingService
|
|
from .utils import PAGINATION_MODEL, EventMessageFilter
|
|
|
|
MAX_RESOURCE_POLICIES_PER_REGION = 10
|
|
|
|
|
|
class Destination(BaseModel):
|
|
def __init__(
|
|
self,
|
|
account_id: str,
|
|
region: str,
|
|
destination_name: str,
|
|
role_arn: str,
|
|
target_arn: str,
|
|
access_policy: Optional[str] = None,
|
|
):
|
|
self.access_policy = access_policy
|
|
self.arn = f"arn:aws:logs:{region}:{account_id}:destination:{destination_name}"
|
|
self.creation_time = int(unix_time_millis())
|
|
self.destination_name = destination_name
|
|
self.role_arn = role_arn
|
|
self.target_arn = target_arn
|
|
|
|
def to_dict(self) -> Dict[str, Any]:
|
|
return {
|
|
"accessPolicy": self.access_policy,
|
|
"arn": self.arn,
|
|
"creationTime": self.creation_time,
|
|
"destinationName": self.destination_name,
|
|
"roleArn": self.role_arn,
|
|
"targetArn": self.target_arn,
|
|
}
|
|
|
|
|
|
class LogQuery(BaseModel):
|
|
def __init__(
|
|
self,
|
|
query_id: str,
|
|
start_time: int,
|
|
end_time: int,
|
|
query: str,
|
|
log_groups: List["LogGroup"],
|
|
):
|
|
self.query_id = query_id
|
|
self.start_time = start_time
|
|
self.end_time = end_time
|
|
self.query = query
|
|
self.log_group_names = [lg.name for lg in log_groups]
|
|
self.create_time = unix_time_millis()
|
|
self.status = "Running"
|
|
self.results = execute_query(
|
|
log_groups=log_groups, query=query, start_time=start_time, end_time=end_time
|
|
)
|
|
self.status = "Complete"
|
|
|
|
def to_json(self, log_group_name: str) -> Dict[str, Any]:
|
|
return {
|
|
"queryId": self.query_id,
|
|
"queryString": self.query,
|
|
"status": self.status,
|
|
"createTime": self.create_time,
|
|
"logGroupName": log_group_name,
|
|
}
|
|
|
|
def to_result_json(self) -> Dict[str, Any]:
|
|
return {
|
|
"results": [
|
|
[{"field": key, "value": val} for key, val in result.items()]
|
|
for result in self.results
|
|
],
|
|
"status": self.status,
|
|
}
|
|
|
|
|
|
class LogEvent(BaseModel):
|
|
_event_id = 0
|
|
|
|
def __init__(self, ingestion_time: int, log_event: Dict[str, Any]):
|
|
self.ingestion_time = ingestion_time
|
|
self.timestamp = log_event["timestamp"]
|
|
self.message = log_event["message"]
|
|
self.event_id = self.__class__._event_id
|
|
self.__class__._event_id += 1
|
|
""
|
|
|
|
def to_filter_dict(self) -> Dict[str, Any]:
|
|
return {
|
|
"eventId": str(self.event_id),
|
|
"ingestionTime": self.ingestion_time,
|
|
# "logStreamName":
|
|
"message": self.message,
|
|
"timestamp": self.timestamp,
|
|
}
|
|
|
|
def to_response_dict(self) -> Dict[str, Any]:
|
|
return {
|
|
"ingestionTime": self.ingestion_time,
|
|
"message": self.message,
|
|
"timestamp": self.timestamp,
|
|
}
|
|
|
|
|
|
class LogStream(BaseModel):
|
|
_log_ids = 0
|
|
|
|
def __init__(self, log_group: "LogGroup", name: str):
|
|
self.account_id = log_group.account_id
|
|
self.region = log_group.region
|
|
self.log_group = log_group
|
|
self.arn = f"arn:aws:logs:{self.region}:{self.account_id}:log-group:{log_group.name}:log-stream:{name}"
|
|
self.creation_time = int(unix_time_millis())
|
|
self.first_event_timestamp = None
|
|
self.last_event_timestamp = None
|
|
self.last_ingestion_time: Optional[int] = None
|
|
self.log_stream_name = name
|
|
self.stored_bytes = 0
|
|
# I'm guessing this is token needed for sequenceToken by put_events
|
|
self.upload_sequence_token = 0
|
|
self.events: List[LogEvent] = []
|
|
|
|
self.__class__._log_ids += 1
|
|
|
|
def _update(self) -> None:
|
|
# events can be empty when stream is described soon after creation
|
|
self.first_event_timestamp = (
|
|
min([x.timestamp for x in self.events]) if self.events else None
|
|
)
|
|
self.last_event_timestamp = (
|
|
max([x.timestamp for x in self.events]) if self.events else None
|
|
)
|
|
|
|
def to_describe_dict(self) -> Dict[str, Any]:
|
|
# Compute start and end times
|
|
self._update()
|
|
|
|
res = {
|
|
"arn": self.arn,
|
|
"creationTime": self.creation_time,
|
|
"logStreamName": self.log_stream_name,
|
|
"storedBytes": self.stored_bytes,
|
|
}
|
|
if self.events:
|
|
rest = {
|
|
"firstEventTimestamp": self.first_event_timestamp,
|
|
"lastEventTimestamp": self.last_event_timestamp,
|
|
"lastIngestionTime": self.last_ingestion_time,
|
|
"uploadSequenceToken": str(self.upload_sequence_token),
|
|
}
|
|
res.update(rest)
|
|
return res
|
|
|
|
def put_log_events(self, log_events: List[Dict[str, Any]]) -> str:
|
|
# TODO: ensure sequence_token
|
|
# TODO: to be thread safe this would need a lock
|
|
self.last_ingestion_time = int(unix_time_millis())
|
|
# TODO: make this match AWS if possible
|
|
self.stored_bytes += sum(
|
|
[len(log_event["message"]) for log_event in log_events]
|
|
)
|
|
events = [
|
|
LogEvent(self.last_ingestion_time, log_event) for log_event in log_events
|
|
]
|
|
self.events += events
|
|
self.upload_sequence_token += 1
|
|
|
|
for subscription_filter in self.log_group.subscription_filters.values():
|
|
|
|
service = subscription_filter.destination_arn.split(":")[2]
|
|
formatted_log_events = [
|
|
{
|
|
"id": event.event_id,
|
|
"timestamp": event.timestamp,
|
|
"message": event.message,
|
|
}
|
|
for event in events
|
|
]
|
|
self._send_log_events(
|
|
service=service,
|
|
destination_arn=subscription_filter.destination_arn,
|
|
filter_name=subscription_filter.name,
|
|
log_events=formatted_log_events,
|
|
)
|
|
return f"{self.upload_sequence_token:056d}"
|
|
|
|
def _send_log_events(
|
|
self,
|
|
service: str,
|
|
destination_arn: str,
|
|
filter_name: str,
|
|
log_events: List[Dict[str, Any]],
|
|
) -> None:
|
|
|
|
if service == "lambda":
|
|
from moto.awslambda import lambda_backends # due to circular dependency
|
|
|
|
lambda_backends[self.account_id][self.region].send_log_event(
|
|
destination_arn,
|
|
filter_name,
|
|
self.log_group.name,
|
|
self.log_stream_name,
|
|
log_events,
|
|
)
|
|
elif service == "firehose":
|
|
from moto.firehose import firehose_backends
|
|
|
|
firehose_backends[self.account_id][self.region].send_log_event(
|
|
destination_arn,
|
|
filter_name,
|
|
self.log_group.name,
|
|
self.log_stream_name,
|
|
log_events,
|
|
)
|
|
elif service == "kinesis":
|
|
from moto.kinesis import kinesis_backends
|
|
|
|
kinesis = kinesis_backends[self.account_id][self.region]
|
|
kinesis.send_log_event(
|
|
destination_arn,
|
|
filter_name,
|
|
self.log_group.name,
|
|
self.log_stream_name,
|
|
log_events,
|
|
)
|
|
|
|
def get_log_events(
|
|
self,
|
|
start_time: str,
|
|
end_time: str,
|
|
limit: int,
|
|
next_token: Optional[str],
|
|
start_from_head: str,
|
|
) -> Tuple[List[Dict[str, Any]], Optional[str], Optional[str]]:
|
|
if limit is None:
|
|
limit = 10000
|
|
|
|
def filter_func(event: LogEvent) -> bool:
|
|
if start_time and event.timestamp < start_time:
|
|
return False
|
|
|
|
if end_time and event.timestamp > end_time:
|
|
return False
|
|
|
|
return True
|
|
|
|
def get_index_and_direction_from_token(
|
|
token: Optional[str],
|
|
) -> Tuple[Optional[str], int]:
|
|
if token is not None:
|
|
try:
|
|
return token[0], int(token[2:])
|
|
except Exception:
|
|
raise InvalidParameterException(
|
|
"The specified nextToken is invalid."
|
|
)
|
|
return None, 0
|
|
|
|
events = sorted(
|
|
filter(filter_func, self.events), key=lambda event: event.timestamp
|
|
)
|
|
|
|
direction, index = get_index_and_direction_from_token(next_token)
|
|
limit_index = limit - 1
|
|
final_index = len(events) - 1
|
|
|
|
if direction is None:
|
|
if start_from_head:
|
|
start_index = 0
|
|
end_index = start_index + limit_index
|
|
else:
|
|
end_index = final_index
|
|
start_index = end_index - limit_index
|
|
elif direction == "f":
|
|
start_index = index + 1
|
|
end_index = start_index + limit_index
|
|
elif direction == "b":
|
|
end_index = index - 1
|
|
start_index = end_index - limit_index
|
|
else:
|
|
raise InvalidParameterException("The specified nextToken is invalid.")
|
|
|
|
if start_index < 0:
|
|
start_index = 0
|
|
elif start_index > final_index:
|
|
return ([], f"b/{final_index:056d}", f"f/{final_index:056d}")
|
|
|
|
if end_index > final_index:
|
|
end_index = final_index
|
|
elif end_index < 0:
|
|
return ([], f"b/{0:056d}", f"f/{0:056d}")
|
|
|
|
events_page = [
|
|
event.to_response_dict() for event in events[start_index : end_index + 1]
|
|
]
|
|
|
|
return (events_page, f"b/{start_index:056d}", f"f/{end_index:056d}")
|
|
|
|
def filter_log_events(
|
|
self, start_time: int, end_time: int, filter_pattern: str
|
|
) -> List[Dict[str, Any]]:
|
|
def filter_func(event: LogEvent) -> bool:
|
|
if start_time and event.timestamp < start_time:
|
|
return False
|
|
|
|
if end_time and event.timestamp > end_time:
|
|
return False
|
|
|
|
if not EventMessageFilter(filter_pattern).matches(event.message):
|
|
return False
|
|
|
|
return True
|
|
|
|
events: List[Dict[str, Any]] = []
|
|
for event in sorted(
|
|
filter(filter_func, self.events), key=lambda x: x.timestamp
|
|
):
|
|
event_obj = event.to_filter_dict()
|
|
event_obj["logStreamName"] = self.log_stream_name
|
|
events.append(event_obj)
|
|
return events
|
|
|
|
|
|
class SubscriptionFilter(BaseModel):
|
|
def __init__(
|
|
self,
|
|
name: str,
|
|
log_group_name: str,
|
|
filter_pattern: str,
|
|
destination_arn: str,
|
|
role_arn: str,
|
|
):
|
|
self.name = name
|
|
self.log_group_name = log_group_name
|
|
self.filter_pattern = filter_pattern
|
|
self.destination_arn = destination_arn
|
|
self.role_arn = role_arn
|
|
self.creation_time = int(unix_time_millis())
|
|
|
|
def update(self, filter_pattern: str, destination_arn: str, role_arn: str) -> None:
|
|
self.filter_pattern = filter_pattern
|
|
self.destination_arn = destination_arn
|
|
self.role_arn = role_arn
|
|
|
|
def to_json(self) -> Dict[str, Any]:
|
|
return {
|
|
"filterName": self.name,
|
|
"logGroupName": self.log_group_name,
|
|
"filterPattern": self.filter_pattern,
|
|
"destinationArn": self.destination_arn,
|
|
"roleArn": self.role_arn,
|
|
"distribution": "ByLogStream",
|
|
"creationTime": self.creation_time,
|
|
}
|
|
|
|
|
|
class LogGroup(CloudFormationModel):
|
|
def __init__(
|
|
self,
|
|
account_id: str,
|
|
region: str,
|
|
name: str,
|
|
**kwargs: Any,
|
|
):
|
|
self.name = name
|
|
self.account_id = account_id
|
|
self.region = region
|
|
self.arn = f"arn:aws:logs:{region}:{account_id}:log-group:{name}"
|
|
self.creation_time = int(unix_time_millis())
|
|
self.streams: Dict[str, LogStream] = dict() # {name: LogStream}
|
|
# AWS defaults to Never Expire for log group retention
|
|
self.retention_in_days = kwargs.get("RetentionInDays")
|
|
self.subscription_filters: Dict[str, SubscriptionFilter] = {}
|
|
|
|
# The Amazon Resource Name (ARN) of the CMK to use when encrypting log data. It is optional.
|
|
# Docs:
|
|
# https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_CreateLogGroup.html
|
|
self.kms_key_id = kwargs.get("kmsKeyId")
|
|
|
|
@staticmethod
|
|
def cloudformation_name_type() -> str:
|
|
return "LogGroupName"
|
|
|
|
@staticmethod
|
|
def cloudformation_type() -> str:
|
|
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-logs-loggroup.html
|
|
return "AWS::Logs::LogGroup"
|
|
|
|
@classmethod
|
|
def create_from_cloudformation_json( # type: ignore[misc]
|
|
cls,
|
|
resource_name: str,
|
|
cloudformation_json: Any,
|
|
account_id: str,
|
|
region_name: str,
|
|
**kwargs: Any,
|
|
) -> "LogGroup":
|
|
properties = cloudformation_json["Properties"]
|
|
tags = properties.get("Tags", [])
|
|
tags = dict([tag.values() for tag in tags])
|
|
|
|
return logs_backends[account_id][region_name].create_log_group(
|
|
resource_name, tags, **properties
|
|
)
|
|
|
|
@classmethod
|
|
def has_cfn_attr(cls, attr: str) -> bool:
|
|
return attr in [
|
|
"Arn",
|
|
]
|
|
|
|
def get_cfn_attribute(self, attribute_name: str) -> str:
|
|
from moto.cloudformation.exceptions import UnformattedGetAttTemplateException
|
|
|
|
if attribute_name == "Arn":
|
|
return self.arn
|
|
raise UnformattedGetAttTemplateException()
|
|
|
|
@property
|
|
def physical_resource_id(self) -> str:
|
|
return self.name
|
|
|
|
def create_log_stream(self, log_stream_name: str) -> None:
|
|
if log_stream_name in self.streams:
|
|
raise ResourceAlreadyExistsException()
|
|
stream = LogStream(log_group=self, name=log_stream_name)
|
|
|
|
self.streams[log_stream_name] = stream
|
|
|
|
def delete_log_stream(self, log_stream_name: str) -> None:
|
|
if log_stream_name not in self.streams:
|
|
raise ResourceNotFoundException()
|
|
del self.streams[log_stream_name]
|
|
|
|
def describe_log_streams(
|
|
self,
|
|
descending: bool,
|
|
log_group_name: str,
|
|
log_stream_name_prefix: str,
|
|
order_by: str,
|
|
limit: int,
|
|
next_token: Optional[str] = None,
|
|
) -> Tuple[List[Dict[str, Any]], Optional[str]]:
|
|
# responses only log_stream_name, creation_time, arn, stored_bytes when no events are stored.
|
|
|
|
log_streams = [
|
|
(name, stream.to_describe_dict())
|
|
for name, stream in self.streams.items()
|
|
if name.startswith(log_stream_name_prefix)
|
|
]
|
|
|
|
def sorter(item: Any) -> Any:
|
|
return (
|
|
item[0]
|
|
if order_by == "LogStreamName"
|
|
else item[1].get("lastEventTimestamp", 0)
|
|
)
|
|
|
|
log_streams = sorted(log_streams, key=sorter, reverse=descending)
|
|
first_index = 0
|
|
if next_token:
|
|
try:
|
|
group, stream = next_token.split("@")
|
|
if group != log_group_name:
|
|
raise ValueError()
|
|
first_index = (
|
|
next(
|
|
index
|
|
for (index, e) in enumerate(log_streams)
|
|
if e[1]["logStreamName"] == stream
|
|
)
|
|
+ 1
|
|
)
|
|
except (ValueError, StopIteration):
|
|
first_index = 0
|
|
log_streams = []
|
|
|
|
last_index = first_index + limit
|
|
if last_index > len(log_streams):
|
|
last_index = len(log_streams)
|
|
log_streams_page = [x[1] for x in log_streams[first_index:last_index]]
|
|
new_token = None
|
|
if log_streams_page and last_index < len(log_streams):
|
|
new_token = f"{log_group_name}@{log_streams_page[-1]['logStreamName']}"
|
|
|
|
return log_streams_page, new_token
|
|
|
|
def put_log_events(
|
|
self,
|
|
log_stream_name: str,
|
|
log_events: List[Dict[str, Any]],
|
|
) -> str:
|
|
if log_stream_name not in self.streams:
|
|
raise ResourceNotFoundException("The specified log stream does not exist.")
|
|
stream = self.streams[log_stream_name]
|
|
return stream.put_log_events(log_events)
|
|
|
|
def get_log_events(
|
|
self,
|
|
log_stream_name: str,
|
|
start_time: str,
|
|
end_time: str,
|
|
limit: int,
|
|
next_token: Optional[str],
|
|
start_from_head: str,
|
|
) -> Tuple[List[Dict[str, Any]], Optional[str], Optional[str]]:
|
|
if log_stream_name not in self.streams:
|
|
raise ResourceNotFoundException()
|
|
stream = self.streams[log_stream_name]
|
|
return stream.get_log_events(
|
|
start_time,
|
|
end_time,
|
|
limit,
|
|
next_token,
|
|
start_from_head,
|
|
)
|
|
|
|
def filter_log_events(
|
|
self,
|
|
log_group_name: str,
|
|
log_stream_names: List[str],
|
|
start_time: int,
|
|
end_time: int,
|
|
limit: Optional[int],
|
|
next_token: Optional[str],
|
|
filter_pattern: str,
|
|
interleaved: bool,
|
|
) -> Tuple[List[Dict[str, Any]], Optional[str], List[Dict[str, Any]]]:
|
|
if not limit:
|
|
limit = 10000
|
|
streams = [
|
|
stream
|
|
for name, stream in self.streams.items()
|
|
if not log_stream_names or name in log_stream_names
|
|
]
|
|
|
|
events = []
|
|
for stream in streams:
|
|
events += stream.filter_log_events(start_time, end_time, filter_pattern)
|
|
|
|
if interleaved:
|
|
events = sorted(events, key=lambda event: event["timestamp"])
|
|
|
|
first_index = 0
|
|
if next_token:
|
|
try:
|
|
group, stream_name, event_id = next_token.split("@")
|
|
if group != log_group_name:
|
|
raise ValueError()
|
|
first_index = (
|
|
next(
|
|
index
|
|
for (index, e) in enumerate(events)
|
|
if e["logStreamName"] == stream_name
|
|
and e["eventId"] == event_id
|
|
)
|
|
+ 1
|
|
)
|
|
except (ValueError, StopIteration):
|
|
first_index = 0
|
|
# AWS returns an empty list if it receives an invalid token.
|
|
events = []
|
|
|
|
last_index = first_index + limit
|
|
if last_index > len(events):
|
|
last_index = len(events)
|
|
events_page = events[first_index:last_index]
|
|
next_token = None
|
|
if events_page and last_index < len(events):
|
|
last_event = events_page[-1]
|
|
next_token = f"{log_group_name}@{last_event['logStreamName']}@{last_event['eventId']}"
|
|
|
|
searched_streams = [
|
|
{"logStreamName": stream.log_stream_name, "searchedCompletely": True}
|
|
for stream in streams
|
|
]
|
|
return events_page, next_token, searched_streams
|
|
|
|
def to_describe_dict(self) -> Dict[str, Any]:
|
|
log_group = {
|
|
"arn": self.arn,
|
|
"creationTime": self.creation_time,
|
|
"logGroupName": self.name,
|
|
"metricFilterCount": 0,
|
|
"storedBytes": sum(s.stored_bytes for s in self.streams.values()),
|
|
}
|
|
# AWS only returns retentionInDays if a value is set for the log group (ie. not Never Expire)
|
|
if self.retention_in_days:
|
|
log_group["retentionInDays"] = self.retention_in_days
|
|
if self.kms_key_id:
|
|
log_group["kmsKeyId"] = self.kms_key_id
|
|
return log_group
|
|
|
|
def set_retention_policy(self, retention_in_days: Optional[str]) -> None:
|
|
self.retention_in_days = retention_in_days
|
|
|
|
def describe_subscription_filters(self) -> Iterable[SubscriptionFilter]:
|
|
return self.subscription_filters.values()
|
|
|
|
def put_subscription_filter(
|
|
self, filter_name: str, filter_pattern: str, destination_arn: str, role_arn: str
|
|
) -> None:
|
|
# only two subscription filters can be associated with a log group
|
|
if len(self.subscription_filters) == 2:
|
|
raise LimitExceededException()
|
|
|
|
# Update existing filter
|
|
if filter_name in self.subscription_filters:
|
|
self.subscription_filters[filter_name].update(
|
|
filter_pattern, destination_arn, role_arn
|
|
)
|
|
return
|
|
|
|
self.subscription_filters[filter_name] = SubscriptionFilter(
|
|
name=filter_name,
|
|
log_group_name=self.name,
|
|
filter_pattern=filter_pattern,
|
|
destination_arn=destination_arn,
|
|
role_arn=role_arn,
|
|
)
|
|
|
|
def delete_subscription_filter(self, filter_name: str) -> None:
|
|
if filter_name not in self.subscription_filters:
|
|
raise ResourceNotFoundException(
|
|
"The specified subscription filter does not exist."
|
|
)
|
|
|
|
self.subscription_filters.pop(filter_name)
|
|
|
|
|
|
class LogResourcePolicy(CloudFormationModel):
|
|
def __init__(self, policy_name: str, policy_document: str):
|
|
self.policy_name = policy_name
|
|
self.policy_document = policy_document
|
|
self.last_updated_time = int(unix_time_millis())
|
|
|
|
def update(self, policy_document: str) -> None:
|
|
self.policy_document = policy_document
|
|
self.last_updated_time = int(unix_time_millis())
|
|
|
|
def describe(self) -> Dict[str, Any]:
|
|
return {
|
|
"policyName": self.policy_name,
|
|
"policyDocument": self.policy_document,
|
|
"lastUpdatedTime": self.last_updated_time,
|
|
}
|
|
|
|
@property
|
|
def physical_resource_id(self) -> str:
|
|
return self.policy_name
|
|
|
|
@staticmethod
|
|
def cloudformation_name_type() -> str:
|
|
return "PolicyName"
|
|
|
|
@staticmethod
|
|
def cloudformation_type() -> str:
|
|
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-logs-resourcepolicy.html
|
|
return "AWS::Logs::ResourcePolicy"
|
|
|
|
@classmethod
|
|
def create_from_cloudformation_json( # type: ignore[misc]
|
|
cls,
|
|
resource_name: str,
|
|
cloudformation_json: Any,
|
|
account_id: str,
|
|
region_name: str,
|
|
**kwargs: Any,
|
|
) -> "LogResourcePolicy":
|
|
properties = cloudformation_json["Properties"]
|
|
policy_name = properties["PolicyName"]
|
|
policy_document = properties["PolicyDocument"]
|
|
return logs_backends[account_id][region_name].put_resource_policy(
|
|
policy_name, policy_document
|
|
)
|
|
|
|
@classmethod
|
|
def update_from_cloudformation_json( # type: ignore[misc]
|
|
cls,
|
|
original_resource: Any,
|
|
new_resource_name: str,
|
|
cloudformation_json: Any,
|
|
account_id: str,
|
|
region_name: str,
|
|
) -> "LogResourcePolicy":
|
|
properties = cloudformation_json["Properties"]
|
|
policy_name = properties["PolicyName"]
|
|
policy_document = properties["PolicyDocument"]
|
|
|
|
backend = logs_backends[account_id][region_name]
|
|
updated = backend.put_resource_policy(policy_name, policy_document)
|
|
# TODO: move `update by replacement logic` to cloudformation. this is required for implementing rollbacks
|
|
if original_resource.policy_name != policy_name:
|
|
backend.delete_resource_policy(original_resource.policy_name)
|
|
return updated
|
|
|
|
@classmethod
|
|
def delete_from_cloudformation_json( # type: ignore[misc]
|
|
cls,
|
|
resource_name: str,
|
|
cloudformation_json: Any,
|
|
account_id: str,
|
|
region_name: str,
|
|
) -> None:
|
|
logs_backends[account_id][region_name].delete_resource_policy(resource_name)
|
|
|
|
|
|
class ExportTask(BaseModel):
|
|
def __init__(
|
|
self,
|
|
task_id: str,
|
|
task_name: str,
|
|
log_group_name: str,
|
|
destination: str,
|
|
destination_prefix: str,
|
|
from_time: int,
|
|
to: int,
|
|
):
|
|
self.task_id = task_id
|
|
self.task_name = task_name
|
|
self.log_group_name = log_group_name
|
|
self.destination = destination
|
|
self.destination_prefix = destination_prefix
|
|
self.from_time = from_time
|
|
self.to = to
|
|
self.status = {"code": "active", "message": "Task is active"}
|
|
|
|
def to_json(self) -> Dict[str, Any]:
|
|
return {
|
|
"taskId": self.task_id,
|
|
"taskName": self.task_name,
|
|
"logGroupName": self.log_group_name,
|
|
"destination": self.destination,
|
|
"destinationPrefix": self.destination_prefix,
|
|
"from": self.from_time,
|
|
"to": self.to,
|
|
"status": self.status,
|
|
}
|
|
|
|
|
|
class LogsBackend(BaseBackend):
|
|
def __init__(self, region_name: str, account_id: str):
|
|
super().__init__(region_name, account_id)
|
|
self.groups: Dict[str, LogGroup] = dict()
|
|
self.filters = MetricFilters()
|
|
self.queries: Dict[str, LogQuery] = dict()
|
|
self.resource_policies: Dict[str, LogResourcePolicy] = dict()
|
|
self.destinations: Dict[str, Destination] = dict()
|
|
self.tagger = TaggingService()
|
|
self.export_tasks: Dict[str, ExportTask] = dict()
|
|
|
|
@staticmethod
|
|
def default_vpc_endpoint_service(
|
|
service_region: str, zones: List[str]
|
|
) -> List[Dict[str, str]]:
|
|
"""Default VPC endpoint service."""
|
|
return BaseBackend.default_vpc_endpoint_service_factory(
|
|
service_region, zones, "logs"
|
|
)
|
|
|
|
def create_log_group(
|
|
self, log_group_name: str, tags: Dict[str, str], **kwargs: Any
|
|
) -> LogGroup:
|
|
if log_group_name in self.groups:
|
|
raise ResourceAlreadyExistsException()
|
|
if len(log_group_name) > 512:
|
|
raise InvalidParameterException(
|
|
constraint="Member must have length less than or equal to 512",
|
|
parameter="logGroupName",
|
|
value=log_group_name,
|
|
)
|
|
self.groups[log_group_name] = LogGroup(
|
|
self.account_id, self.region_name, log_group_name, **kwargs
|
|
)
|
|
self.tag_resource(self.groups[log_group_name].arn, tags)
|
|
return self.groups[log_group_name]
|
|
|
|
def ensure_log_group(self, log_group_name: str) -> None:
|
|
if log_group_name in self.groups:
|
|
return
|
|
self.groups[log_group_name] = LogGroup(
|
|
self.account_id,
|
|
self.region_name,
|
|
log_group_name,
|
|
)
|
|
|
|
def delete_log_group(self, log_group_name: str) -> None:
|
|
if log_group_name not in self.groups:
|
|
raise ResourceNotFoundException()
|
|
del self.groups[log_group_name]
|
|
|
|
@paginate(pagination_model=PAGINATION_MODEL) # type: ignore[misc]
|
|
def describe_log_groups(
|
|
self, log_group_name_prefix: Optional[str] = None
|
|
) -> List[Dict[str, Any]]:
|
|
groups = [
|
|
group.to_describe_dict()
|
|
for name, group in self.groups.items()
|
|
if name.startswith(log_group_name_prefix or "")
|
|
]
|
|
groups = sorted(groups, key=lambda x: x["logGroupName"])
|
|
|
|
return groups
|
|
|
|
def get_destination(self, destination_name: str) -> Destination:
|
|
for destination in self.destinations:
|
|
if self.destinations[destination].destination_name == destination_name:
|
|
return self.destinations[destination]
|
|
raise ResourceNotFoundException()
|
|
|
|
def put_destination(
|
|
self,
|
|
destination_name: str,
|
|
role_arn: str,
|
|
target_arn: str,
|
|
tags: Dict[str, str],
|
|
) -> Destination:
|
|
for _, destination in self.destinations.items():
|
|
if destination.destination_name == destination_name:
|
|
if role_arn:
|
|
destination.role_arn = role_arn
|
|
if target_arn:
|
|
destination.target_arn = target_arn
|
|
return destination
|
|
destination = Destination(
|
|
self.account_id, self.region_name, destination_name, role_arn, target_arn
|
|
)
|
|
self.destinations[destination.arn] = destination
|
|
self.tag_resource(destination.arn, tags)
|
|
return destination
|
|
|
|
def delete_destination(self, destination_name: str) -> None:
|
|
destination = self.get_destination(destination_name)
|
|
self.destinations.pop(destination.arn)
|
|
return
|
|
|
|
def describe_destinations(
|
|
self, destination_name_prefix: str, limit: int, next_token: Optional[int] = None
|
|
) -> Tuple[List[Dict[str, Any]], Optional[int]]:
|
|
if limit > 50:
|
|
raise InvalidParameterException(
|
|
constraint="Member must have value less than or equal to 50",
|
|
parameter="limit",
|
|
value=limit,
|
|
)
|
|
|
|
result = []
|
|
for destination in self.destinations:
|
|
result.append(self.destinations[destination].to_dict())
|
|
if next_token:
|
|
result = result[: int(next_token)]
|
|
result = [
|
|
destination
|
|
for destination in result
|
|
if destination["destinationName"].startswith(destination_name_prefix)
|
|
]
|
|
return result, next_token
|
|
|
|
def put_destination_policy(self, destination_name: str, access_policy: str) -> None:
|
|
destination = self.get_destination(destination_name)
|
|
destination.access_policy = access_policy
|
|
return
|
|
|
|
def create_log_stream(self, log_group_name: str, log_stream_name: str) -> None:
|
|
if log_group_name not in self.groups:
|
|
raise ResourceNotFoundException()
|
|
log_group = self.groups[log_group_name]
|
|
log_group.create_log_stream(log_stream_name)
|
|
|
|
def ensure_log_stream(self, log_group_name: str, log_stream_name: str) -> None:
|
|
if log_group_name not in self.groups:
|
|
raise ResourceNotFoundException()
|
|
|
|
if log_stream_name in self.groups[log_group_name].streams:
|
|
return
|
|
|
|
self.create_log_stream(log_group_name, log_stream_name)
|
|
|
|
def delete_log_stream(self, log_group_name: str, log_stream_name: str) -> None:
|
|
if log_group_name not in self.groups:
|
|
raise ResourceNotFoundException()
|
|
log_group = self.groups[log_group_name]
|
|
log_group.delete_log_stream(log_stream_name)
|
|
|
|
def describe_log_streams(
|
|
self,
|
|
descending: bool,
|
|
limit: int,
|
|
log_group_name: str,
|
|
log_stream_name_prefix: str,
|
|
next_token: Optional[str],
|
|
order_by: str,
|
|
) -> Tuple[List[Dict[str, Any]], Optional[str]]:
|
|
if log_group_name not in self.groups:
|
|
raise ResourceNotFoundException()
|
|
if limit > 50:
|
|
raise InvalidParameterException(
|
|
constraint="Member must have value less than or equal to 50",
|
|
parameter="limit",
|
|
value=limit,
|
|
)
|
|
if order_by not in ["LogStreamName", "LastEventTime"]:
|
|
raise InvalidParameterException(
|
|
constraint="Member must satisfy enum value set: [LogStreamName, LastEventTime]",
|
|
parameter="orderBy",
|
|
value=order_by,
|
|
)
|
|
if order_by == "LastEventTime" and log_stream_name_prefix:
|
|
raise InvalidParameterException(
|
|
msg="Cannot order by LastEventTime with a logStreamNamePrefix."
|
|
)
|
|
log_group = self.groups[log_group_name]
|
|
return log_group.describe_log_streams(
|
|
descending=descending,
|
|
limit=limit,
|
|
log_group_name=log_group_name,
|
|
log_stream_name_prefix=log_stream_name_prefix,
|
|
next_token=next_token,
|
|
order_by=order_by,
|
|
)
|
|
|
|
def put_log_events(
|
|
self,
|
|
log_group_name: str,
|
|
log_stream_name: str,
|
|
log_events: List[Dict[str, Any]],
|
|
) -> Tuple[str, Dict[str, Any]]:
|
|
"""
|
|
The SequenceToken-parameter is not yet implemented
|
|
"""
|
|
if log_group_name not in self.groups:
|
|
raise ResourceNotFoundException()
|
|
log_group = self.groups[log_group_name]
|
|
|
|
# Only events from the last 14 days or 2 hours in the future are accepted
|
|
rejected_info = {}
|
|
allowed_events = []
|
|
last_timestamp = None
|
|
oldest = int(unix_time_millis(utcnow() - timedelta(days=14)))
|
|
newest = int(unix_time_millis(utcnow() + timedelta(hours=2)))
|
|
for idx, event in enumerate(log_events):
|
|
if last_timestamp and last_timestamp > event["timestamp"]:
|
|
raise InvalidParameterException(
|
|
"Log events in a single PutLogEvents request must be in chronological order."
|
|
)
|
|
if event["timestamp"] < oldest:
|
|
rejected_info["tooOldLogEventEndIndex"] = idx
|
|
elif event["timestamp"] > newest:
|
|
rejected_info["tooNewLogEventStartIndex"] = idx
|
|
else:
|
|
allowed_events.append(event)
|
|
last_timestamp = event["timestamp"]
|
|
|
|
token = log_group.put_log_events(log_stream_name, allowed_events)
|
|
return token, rejected_info
|
|
|
|
def get_log_events(
|
|
self,
|
|
log_group_name: str,
|
|
log_stream_name: str,
|
|
start_time: str,
|
|
end_time: str,
|
|
limit: int,
|
|
next_token: Optional[str],
|
|
start_from_head: str,
|
|
) -> Tuple[List[Dict[str, Any]], Optional[str], Optional[str]]:
|
|
if log_group_name not in self.groups:
|
|
raise ResourceNotFoundException()
|
|
if limit and limit > 10000:
|
|
raise InvalidParameterException(
|
|
constraint="Member must have value less than or equal to 10000",
|
|
parameter="limit",
|
|
value=limit,
|
|
)
|
|
log_group = self.groups[log_group_name]
|
|
return log_group.get_log_events(
|
|
log_stream_name, start_time, end_time, limit, next_token, start_from_head
|
|
)
|
|
|
|
def filter_log_events(
|
|
self,
|
|
log_group_name: str,
|
|
log_stream_names: List[str],
|
|
start_time: int,
|
|
end_time: int,
|
|
limit: Optional[int],
|
|
next_token: Optional[str],
|
|
filter_pattern: str,
|
|
interleaved: bool,
|
|
) -> Tuple[List[Dict[str, Any]], Optional[str], List[Dict[str, Any]]]:
|
|
"""
|
|
The following filter patterns are currently supported: Single Terms, Multiple Terms, Exact Phrases.
|
|
If the pattern is not supported, all events are returned.
|
|
"""
|
|
if log_group_name not in self.groups:
|
|
raise ResourceNotFoundException()
|
|
if limit and limit > 10000:
|
|
raise InvalidParameterException(
|
|
constraint="Member must have value less than or equal to 10000",
|
|
parameter="limit",
|
|
value=limit,
|
|
)
|
|
log_group = self.groups[log_group_name]
|
|
return log_group.filter_log_events(
|
|
log_group_name,
|
|
log_stream_names,
|
|
start_time,
|
|
end_time,
|
|
limit,
|
|
next_token,
|
|
filter_pattern,
|
|
interleaved,
|
|
)
|
|
|
|
def put_retention_policy(self, log_group_name: str, retention_in_days: str) -> None:
|
|
if log_group_name not in self.groups:
|
|
raise ResourceNotFoundException()
|
|
self.groups[log_group_name].set_retention_policy(retention_in_days)
|
|
|
|
def delete_retention_policy(self, log_group_name: str) -> None:
|
|
if log_group_name not in self.groups:
|
|
raise ResourceNotFoundException()
|
|
self.groups[log_group_name].set_retention_policy(None)
|
|
|
|
def describe_resource_policies(self) -> List[LogResourcePolicy]:
|
|
"""
|
|
Return list of resource policies.
|
|
|
|
The next_token and limit arguments are ignored. The maximum
|
|
number of resource policies per region is a small number (less
|
|
than 50), so pagination isn't needed.
|
|
"""
|
|
|
|
return list(self.resource_policies.values())
|
|
|
|
def put_resource_policy(
|
|
self, policy_name: str, policy_doc: str
|
|
) -> LogResourcePolicy:
|
|
"""
|
|
Creates/updates resource policy and return policy object
|
|
"""
|
|
if policy_name in self.resource_policies:
|
|
policy = self.resource_policies[policy_name]
|
|
policy.update(policy_doc)
|
|
return policy
|
|
if len(self.resource_policies) == MAX_RESOURCE_POLICIES_PER_REGION:
|
|
raise LimitExceededException()
|
|
policy = LogResourcePolicy(policy_name, policy_doc)
|
|
self.resource_policies[policy_name] = policy
|
|
return policy
|
|
|
|
def delete_resource_policy(self, policy_name: str) -> None:
|
|
"""
|
|
Remove resource policy with a policy name matching given name.
|
|
"""
|
|
if policy_name not in self.resource_policies:
|
|
raise ResourceNotFoundException(
|
|
msg=f"Policy with name [{policy_name}] does not exist"
|
|
)
|
|
del self.resource_policies[policy_name]
|
|
|
|
def list_tags_log_group(self, log_group_name: str) -> Dict[str, str]:
|
|
if log_group_name not in self.groups:
|
|
raise ResourceNotFoundException()
|
|
log_group = self.groups[log_group_name]
|
|
return self.list_tags_for_resource(log_group.arn)
|
|
|
|
def tag_log_group(self, log_group_name: str, tags: Dict[str, str]) -> None:
|
|
if log_group_name not in self.groups:
|
|
raise ResourceNotFoundException()
|
|
log_group = self.groups[log_group_name]
|
|
self.tag_resource(log_group.arn, tags)
|
|
|
|
def untag_log_group(self, log_group_name: str, tags: List[str]) -> None:
|
|
if log_group_name not in self.groups:
|
|
raise ResourceNotFoundException()
|
|
log_group = self.groups[log_group_name]
|
|
self.untag_resource(log_group.arn, tags)
|
|
|
|
def put_metric_filter(
|
|
self,
|
|
filter_name: str,
|
|
filter_pattern: str,
|
|
log_group_name: str,
|
|
metric_transformations: str,
|
|
) -> None:
|
|
self.filters.add_filter(
|
|
filter_name, filter_pattern, log_group_name, metric_transformations
|
|
)
|
|
|
|
def describe_metric_filters(
|
|
self,
|
|
prefix: Optional[str] = None,
|
|
log_group_name: Optional[str] = None,
|
|
metric_name: Optional[str] = None,
|
|
metric_namespace: Optional[str] = None,
|
|
) -> List[Dict[str, Any]]:
|
|
filters = self.filters.get_matching_filters(
|
|
prefix, log_group_name, metric_name, metric_namespace
|
|
)
|
|
return filters
|
|
|
|
def delete_metric_filter(
|
|
self, filter_name: Optional[str] = None, log_group_name: Optional[str] = None
|
|
) -> None:
|
|
self.filters.delete_filter(filter_name, log_group_name)
|
|
|
|
def describe_subscription_filters(
|
|
self, log_group_name: str
|
|
) -> Iterable[SubscriptionFilter]:
|
|
log_group = self.groups.get(log_group_name)
|
|
|
|
if not log_group:
|
|
raise ResourceNotFoundException()
|
|
|
|
return log_group.describe_subscription_filters()
|
|
|
|
def put_subscription_filter(
|
|
self,
|
|
log_group_name: str,
|
|
filter_name: str,
|
|
filter_pattern: str,
|
|
destination_arn: str,
|
|
role_arn: str,
|
|
) -> None:
|
|
log_group = self.groups.get(log_group_name)
|
|
|
|
if not log_group:
|
|
raise ResourceNotFoundException()
|
|
|
|
service = destination_arn.split(":")[2]
|
|
if service == "lambda":
|
|
from moto.awslambda import lambda_backends
|
|
|
|
try:
|
|
lambda_backends[self.account_id][self.region_name].get_function(
|
|
destination_arn
|
|
)
|
|
# no specific permission check implemented
|
|
except Exception:
|
|
raise InvalidParameterException(
|
|
"Could not execute the lambda function. Make sure you "
|
|
"have given CloudWatch Logs permission to execute your "
|
|
"function."
|
|
)
|
|
elif service == "firehose":
|
|
from moto.firehose import firehose_backends
|
|
|
|
firehose = firehose_backends[self.account_id][
|
|
self.region_name
|
|
].lookup_name_from_arn(destination_arn)
|
|
if not firehose:
|
|
raise InvalidParameterException(
|
|
"Could not deliver test message to specified Firehose "
|
|
"stream. Check if the given Firehose stream is in ACTIVE "
|
|
"state."
|
|
)
|
|
elif service == "kinesis":
|
|
from moto.kinesis import kinesis_backends
|
|
|
|
kinesis = kinesis_backends[self.account_id][self.region_name]
|
|
try:
|
|
kinesis.describe_stream(stream_arn=destination_arn, stream_name=None)
|
|
except Exception:
|
|
raise InvalidParameterException(
|
|
"Could not deliver test message to specified Kinesis stream. Verify the stream exists "
|
|
)
|
|
else:
|
|
# TODO: support Kinesis stream destinations
|
|
raise InvalidParameterException(
|
|
f"Service '{service}' has not implemented for "
|
|
f"put_subscription_filter()"
|
|
)
|
|
|
|
log_group.put_subscription_filter(
|
|
filter_name, filter_pattern, destination_arn, role_arn
|
|
)
|
|
|
|
def delete_subscription_filter(self, log_group_name: str, filter_name: str) -> None:
|
|
log_group = self.groups.get(log_group_name)
|
|
|
|
if not log_group:
|
|
raise ResourceNotFoundException()
|
|
|
|
log_group.delete_subscription_filter(filter_name)
|
|
|
|
def start_query(
|
|
self,
|
|
log_group_names: List[str],
|
|
start_time: int,
|
|
end_time: int,
|
|
query_string: str,
|
|
) -> str:
|
|
|
|
for log_group_name in log_group_names:
|
|
if log_group_name not in self.groups:
|
|
raise ResourceNotFoundException()
|
|
log_groups = [self.groups[name] for name in log_group_names]
|
|
|
|
query_id = str(mock_random.uuid1())
|
|
self.queries[query_id] = LogQuery(
|
|
query_id, start_time, end_time, query_string, log_groups
|
|
)
|
|
return query_id
|
|
|
|
def describe_queries(
|
|
self, log_stream_name: str, status: Optional[str]
|
|
) -> List[LogQuery]:
|
|
"""
|
|
Pagination is not yet implemented
|
|
"""
|
|
queries: List[LogQuery] = []
|
|
for query in self.queries.values():
|
|
if log_stream_name in query.log_group_names and (
|
|
not status or status == query.status
|
|
):
|
|
queries.append(query)
|
|
return queries
|
|
|
|
def get_query_results(self, query_id: str) -> LogQuery:
|
|
"""
|
|
Not all query commands are implemented yet. Please raise an issue if you encounter unexpected results.
|
|
"""
|
|
return self.queries[query_id]
|
|
|
|
def create_export_task(
|
|
self,
|
|
taskName: str,
|
|
logGroupName: str,
|
|
destination: str,
|
|
destinationPrefix: str,
|
|
fromTime: int,
|
|
to: int,
|
|
) -> str:
|
|
s3_backends[self.account_id]["global"].get_bucket(destination)
|
|
if logGroupName not in self.groups:
|
|
raise ResourceNotFoundException()
|
|
task_id = str(mock_random.uuid4())
|
|
self.export_tasks[task_id] = ExportTask(
|
|
task_id,
|
|
taskName,
|
|
logGroupName,
|
|
destination,
|
|
destinationPrefix,
|
|
fromTime,
|
|
to,
|
|
)
|
|
if fromTime <= datetime.now().timestamp() <= to:
|
|
logs = self.filter_log_events(
|
|
logGroupName, [], fromTime, to, None, None, "", False
|
|
)
|
|
s3_backends[self.account_id]["global"].put_object(
|
|
destination, destinationPrefix, json.dumps(logs)
|
|
)
|
|
self.export_tasks[task_id].status["code"] = "completed"
|
|
self.export_tasks[task_id].status["message"] = "Task is completed"
|
|
return task_id
|
|
|
|
def describe_export_tasks(self, task_id: str) -> List[ExportTask]:
|
|
"""
|
|
Pagination is not yet implemented
|
|
"""
|
|
if task_id:
|
|
if task_id not in self.export_tasks:
|
|
raise ResourceNotFoundException()
|
|
return [self.export_tasks[task_id]]
|
|
else:
|
|
return list(self.export_tasks.values())
|
|
|
|
def list_tags_for_resource(self, resource_arn: str) -> Dict[str, str]:
|
|
return self.tagger.get_tag_dict_for_resource(resource_arn)
|
|
|
|
def tag_resource(self, arn: str, tags: Dict[str, str]) -> None:
|
|
self.tagger.tag_resource(arn, TaggingService.convert_dict_to_tags_input(tags))
|
|
|
|
def untag_resource(self, arn: str, tag_keys: List[str]) -> None:
|
|
self.tagger.untag_resource_using_names(arn, tag_keys)
|
|
|
|
|
|
logs_backends = BackendDict(LogsBackend, "logs")
|