moto/moto/sns/models.py

853 lines
31 KiB
Python
Raw Normal View History

2014-05-11 22:56:44 -04:00
import datetime
import uuid
import json
2014-11-16 18:35:11 -05:00
import requests
import re
2014-05-11 22:56:44 -04:00
from boto3 import Session
from collections import OrderedDict
from moto.core import BaseBackend, BaseModel, CloudFormationModel
2019-10-31 08:44:26 -07:00
from moto.core.utils import (
iso_8601_datetime_with_milliseconds,
camelcase_to_underscores,
)
from moto.sqs import sqs_backends
lambda + SNS enhancements (#1048) * updates - support lambda messages from SNS - run lambda in docker container * decode output * populate timeout * simplify * whoops * skeletons of cloudwatchlogs * impl filter log streams * fix logging * PEP fixes * PEP fixes * fix reset * fix reset * add new endpoint * fix region name * add docker * try to fix tests * try to fix travis issue with boto * fix escaping in urls * fix environment variables * fix PEP * more pep * switch back to precise * another fix attempt * fix typo * fix lambda invoke * fix more unittests * work on getting this to work in new scheme * fix py2 * fix error * fix tests when running in server mode * more lambda fixes * try running with latest docker adapted from aiodocker * switch to docker python client * pep fixes * switch to docker volume * fix unittest * fix invoke from sns * fix zip2tar * add hack impl for get_function with zip * try fix * fix for py < 3.6 * add volume refcount * try to fix travis * docker test * fix yaml * try fix * update endpoints * fix * another attempt * try again * fix recursive import * refactor fix * revert changes with better fix * more reverts * wait for service to come up * add back detached mode * sleep and add another exception type * put this back for logging * put back with note * whoops :) * docker in docker! * fix invalid url * hopefully last fix! * fix lambda regions * fix protocol * travis!!!! * just run lambda test for now * use one print * fix escaping * another attempt * yet another * re-enable all tests * fixes * fix for py2 * revert change * fix for py2.7 * fix output ordering * remove this given there's a new unittest that covers it * changes based on review - add skeleton logs test file - switch to docker image that matches test env - fix mock_logs import * add readme entry
2017-09-27 16:04:58 -07:00
2017-03-16 22:28:30 -04:00
from .exceptions import (
2019-10-31 08:44:26 -07:00
SNSNotFoundError,
DuplicateSnsEndpointError,
SnsEndpointDisabled,
SNSInvalidParameter,
InvalidParameterValue,
InternalError,
ResourceNotFoundError,
TagLimitExceededError,
2017-03-16 22:28:30 -04:00
)
from .utils import make_arn_for_topic, make_arn_for_subscription, is_e164
2014-05-11 22:56:44 -04:00
from moto.core import ACCOUNT_ID as DEFAULT_ACCOUNT_ID
2019-12-16 21:25:20 -05:00
DEFAULT_PAGE_SIZE = 100
MAXIMUM_MESSAGE_LENGTH = 262144 # 256 KiB
MAXIMUM_SMS_MESSAGE_BYTES = 1600 # Amazon limit for a single publish SMS action
2014-05-11 22:56:44 -04:00
class Topic(CloudFormationModel):
2014-11-16 18:35:11 -05:00
def __init__(self, name, sns_backend):
2014-05-11 22:56:44 -04:00
self.name = name
2014-11-16 18:35:11 -05:00
self.sns_backend = sns_backend
2014-05-11 22:56:44 -04:00
self.account_id = DEFAULT_ACCOUNT_ID
self.display_name = ""
self.delivery_policy = ""
self.kms_master_key_id = ""
2017-02-27 20:53:57 -05:00
self.effective_delivery_policy = json.dumps(DEFAULT_EFFECTIVE_DELIVERY_POLICY)
2019-10-31 08:44:26 -07:00
self.arn = make_arn_for_topic(self.account_id, name, sns_backend.region_name)
2014-05-11 22:56:44 -04:00
self.subscriptions_pending = 0
self.subscriptions_confimed = 0
self.subscriptions_deleted = 0
2019-10-31 08:44:26 -07:00
self._policy_json = self._create_default_topic_policy(
sns_backend.region_name, self.account_id, name
)
2019-10-11 17:58:48 +02:00
self._tags = {}
self.fifo_topic = "false"
self.content_based_deduplication = "false"
2019-10-11 17:58:48 +02:00
def publish(self, message, subject=None, message_attributes=None):
2021-07-26 07:40:39 +01:00
message_id = str(uuid.uuid4())
subscriptions, _ = self.sns_backend.list_subscriptions(self.arn)
2014-05-11 22:56:44 -04:00
for subscription in subscriptions:
2019-10-31 08:44:26 -07:00
subscription.publish(
message,
message_id,
subject=subject,
message_attributes=message_attributes,
)
2014-05-11 22:56:44 -04:00
return message_id
def get_cfn_attribute(self, attribute_name):
from moto.cloudformation.exceptions import UnformattedGetAttTemplateException
2019-10-31 08:44:26 -07:00
if attribute_name == "TopicName":
return self.name
raise UnformattedGetAttTemplateException()
2015-01-17 19:48:08 -05:00
@property
def physical_resource_id(self):
return self.arn
@property
def policy(self):
return json.dumps(self._policy_json)
@policy.setter
def policy(self, policy):
self._policy_json = json.loads(policy)
@staticmethod
def cloudformation_name_type():
return "TopicName"
@staticmethod
def cloudformation_type():
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-sns-topic.html
return "AWS::SNS::Topic"
2015-01-17 19:48:08 -05:00
@classmethod
2019-10-31 08:44:26 -07:00
def create_from_cloudformation_json(
cls, resource_name, cloudformation_json, region_name
):
2015-01-17 19:48:08 -05:00
sns_backend = sns_backends[region_name]
2019-10-31 08:44:26 -07:00
properties = cloudformation_json["Properties"]
2015-01-17 19:48:08 -05:00
topic = sns_backend.create_topic(resource_name)
2015-01-17 19:48:08 -05:00
for subscription in properties.get("Subscription", []):
2019-10-31 08:44:26 -07:00
sns_backend.subscribe(
topic.arn, subscription["Endpoint"], subscription["Protocol"]
)
2015-01-17 19:48:08 -05:00
return topic
@classmethod
def update_from_cloudformation_json(
cls, original_resource, new_resource_name, cloudformation_json, region_name
):
cls.delete_from_cloudformation_json(
original_resource.name, cloudformation_json, region_name
)
return cls.create_from_cloudformation_json(
new_resource_name, cloudformation_json, region_name
)
@classmethod
def delete_from_cloudformation_json(
cls, resource_name, cloudformation_json, region_name
):
sns_backend = sns_backends[region_name]
properties = cloudformation_json["Properties"]
topic_name = properties.get(cls.cloudformation_name_type()) or resource_name
topic_arn = make_arn_for_topic(
DEFAULT_ACCOUNT_ID, topic_name, sns_backend.region_name
)
subscriptions, _ = sns_backend.list_subscriptions(topic_arn)
for subscription in subscriptions:
sns_backend.unsubscribe(subscription.arn)
sns_backend.delete_topic(topic_arn)
def _create_default_topic_policy(self, region_name, account_id, name):
return {
"Version": "2008-10-17",
"Id": "__default_policy_ID",
2019-10-31 08:44:26 -07:00
"Statement": [
{
"Effect": "Allow",
"Sid": "__default_statement_ID",
"Principal": {"AWS": "*"},
"Action": [
"SNS:GetTopicAttributes",
"SNS:SetTopicAttributes",
"SNS:AddPermission",
"SNS:RemovePermission",
"SNS:DeleteTopic",
"SNS:Subscribe",
"SNS:ListSubscriptionsByTopic",
"SNS:Publish",
"SNS:Receive",
],
"Resource": make_arn_for_topic(self.account_id, name, region_name),
"Condition": {"StringEquals": {"AWS:SourceOwner": str(account_id)}},
}
2019-10-31 08:44:26 -07:00
],
}
2014-05-11 22:56:44 -04:00
2017-03-11 23:41:12 -05:00
class Subscription(BaseModel):
2014-05-11 22:56:44 -04:00
def __init__(self, topic, endpoint, protocol):
self.topic = topic
self.endpoint = endpoint
self.protocol = protocol
self.arn = make_arn_for_subscription(self.topic.arn)
self.attributes = {}
self._filter_policy = None # filter policy as a dict, not json.
self.confirmed = False
2014-05-11 22:56:44 -04:00
2019-10-31 08:44:26 -07:00
def publish(self, message, message_id, subject=None, message_attributes=None):
if not self._matches_filter_policy(message_attributes):
return
2019-10-31 08:44:26 -07:00
if self.protocol == "sqs":
2014-05-11 22:56:44 -04:00
queue_name = self.endpoint.split(":")[-1]
region = self.endpoint.split(":")[3]
2019-10-31 08:44:26 -07:00
if self.attributes.get("RawMessageDelivery") != "true":
sqs_backends[region].send_message(
queue_name,
json.dumps(
self.get_post_data(
message,
message_id,
subject,
message_attributes=message_attributes,
),
sort_keys=True,
indent=2,
separators=(",", ": "),
2019-10-31 08:44:26 -07:00
),
)
else:
raw_message_attributes = {}
for key, value in message_attributes.items():
type = "string_value"
type_value = value["Value"]
if value["Type"].startswith("Binary"):
type = "binary_value"
elif value["Type"].startswith("Number"):
type_value = "{0:g}".format(value["Value"])
raw_message_attributes[key] = {
"data_type": value["Type"],
type: type_value,
}
sqs_backends[region].send_message(
queue_name, message, message_attributes=raw_message_attributes
)
2019-10-31 08:44:26 -07:00
elif self.protocol in ["http", "https"]:
2017-12-10 13:59:04 -08:00
post_data = self.get_post_data(message, message_id, subject)
2019-10-31 08:44:26 -07:00
requests.post(
self.endpoint,
json=post_data,
headers={"Content-Type": "text/plain; charset=UTF-8"},
)
elif self.protocol == "lambda":
lambda + SNS enhancements (#1048) * updates - support lambda messages from SNS - run lambda in docker container * decode output * populate timeout * simplify * whoops * skeletons of cloudwatchlogs * impl filter log streams * fix logging * PEP fixes * PEP fixes * fix reset * fix reset * add new endpoint * fix region name * add docker * try to fix tests * try to fix travis issue with boto * fix escaping in urls * fix environment variables * fix PEP * more pep * switch back to precise * another fix attempt * fix typo * fix lambda invoke * fix more unittests * work on getting this to work in new scheme * fix py2 * fix error * fix tests when running in server mode * more lambda fixes * try running with latest docker adapted from aiodocker * switch to docker python client * pep fixes * switch to docker volume * fix unittest * fix invoke from sns * fix zip2tar * add hack impl for get_function with zip * try fix * fix for py < 3.6 * add volume refcount * try to fix travis * docker test * fix yaml * try fix * update endpoints * fix * another attempt * try again * fix recursive import * refactor fix * revert changes with better fix * more reverts * wait for service to come up * add back detached mode * sleep and add another exception type * put this back for logging * put back with note * whoops :) * docker in docker! * fix invalid url * hopefully last fix! * fix lambda regions * fix protocol * travis!!!! * just run lambda test for now * use one print * fix escaping * another attempt * yet another * re-enable all tests * fixes * fix for py2 * revert change * fix for py2.7 * fix output ordering * remove this given there's a new unittest that covers it * changes based on review - add skeleton logs test file - switch to docker image that matches test env - fix mock_logs import * add readme entry
2017-09-27 16:04:58 -07:00
# TODO: support bad function name
# http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html
arr = self.endpoint.split(":")
region = arr[3]
qualifier = None
if len(arr) == 7:
2019-10-31 08:44:26 -07:00
assert arr[5] == "function"
function_name = arr[-1]
elif len(arr) == 8:
2019-10-31 08:44:26 -07:00
assert arr[5] == "function"
qualifier = arr[-1]
function_name = arr[-2]
else:
assert False
from moto.awslambda import lambda_backends
2019-10-31 08:44:26 -07:00
lambda_backends[region].send_sns_message(
function_name, message, subject=subject, qualifier=qualifier
)
2014-05-11 22:56:44 -04:00
def _matches_filter_policy(self, message_attributes):
# TODO: support Anything-but matching, prefix matching and
# numeric value matching.
if not self._filter_policy:
return True
if message_attributes is None:
message_attributes = {}
def _field_match(field, rules, message_attributes):
for rule in rules:
# TODO: boolean value matching is not supported, SNS behavior unknown
2021-07-26 07:40:39 +01:00
if isinstance(rule, str):
if field not in message_attributes:
return False
2019-10-31 08:44:26 -07:00
if message_attributes[field]["Value"] == rule:
return True
try:
2019-10-31 08:44:26 -07:00
json_data = json.loads(message_attributes[field]["Value"])
if rule in json_data:
return True
except (ValueError, TypeError):
pass
2021-07-26 07:40:39 +01:00
if isinstance(rule, (int, float)):
if field not in message_attributes:
return False
2019-10-31 08:44:26 -07:00
if message_attributes[field]["Type"] == "Number":
attribute_values = [message_attributes[field]["Value"]]
elif message_attributes[field]["Type"] == "String.Array":
try:
2019-10-31 08:44:26 -07:00
attribute_values = json.loads(
message_attributes[field]["Value"]
)
if not isinstance(attribute_values, list):
attribute_values = [attribute_values]
except (ValueError, TypeError):
return False
else:
return False
for attribute_values in attribute_values:
2019-11-16 12:31:45 -08:00
# Even the official documentation states a 5 digits of accuracy after the decimal point for numerics, in reality it is 6
# https://docs.aws.amazon.com/sns/latest/dg/sns-subscription-filter-policies.html#subscription-filter-policy-constraints
if int(attribute_values * 1000000) == int(rule * 1000000):
return True
if isinstance(rule, dict):
keyword = list(rule.keys())[0]
attributes = list(rule.values())[0]
2019-10-31 08:44:26 -07:00
if keyword == "exists":
if attributes and field in message_attributes:
return True
elif not attributes and field not in message_attributes:
return True
return False
2019-10-31 08:44:26 -07:00
return all(
_field_match(field, rules, message_attributes)
2021-07-26 07:40:39 +01:00
for field, rules in self._filter_policy.items()
2019-10-31 08:44:26 -07:00
)
2019-10-31 08:44:26 -07:00
def get_post_data(self, message, message_id, subject, message_attributes=None):
post_data = {
2014-05-11 22:56:44 -04:00
"Type": "Notification",
"MessageId": message_id,
"TopicArn": self.topic.arn,
"Subject": subject,
2014-05-11 22:56:44 -04:00
"Message": message,
2019-10-31 08:44:26 -07:00
"Timestamp": iso_8601_datetime_with_milliseconds(
datetime.datetime.utcnow()
),
2014-05-11 22:56:44 -04:00
"SignatureVersion": "1",
"Signature": "EXAMPLElDMXvB8r9R83tGoNn0ecwd5UjllzsvSvbItzfaMpN2nk5HVSw7XnOn/49IkxDKz8YrlH2qJXj2iZB0Zo2O71c4qQk1fMUDi3LGpij7RCW7AW9vYYsSqIKRnFS94ilu7NFhUzLiieYr4BKHpdTmdD6c0esKEYBpabxDSc=",
"SigningCertURL": "https://sns.us-east-1.amazonaws.com/SimpleNotificationService-f3ecfb7224c7233fe7bb5f59f96de52f.pem",
2019-12-16 21:25:20 -05:00
"UnsubscribeURL": "https://sns.us-east-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-east-1:{}:some-topic:2bcfbf39-05c3-41de-beaa-fcfcc21c8f55".format(
DEFAULT_ACCOUNT_ID
),
2014-05-11 22:56:44 -04:00
}
if message_attributes:
post_data["MessageAttributes"] = message_attributes
return post_data
2014-05-11 22:56:44 -04:00
2017-03-11 23:41:12 -05:00
class PlatformApplication(BaseModel):
2015-03-14 09:06:31 -04:00
def __init__(self, region, name, platform, attributes):
self.region = region
self.name = name
self.platform = platform
self.attributes = attributes
@property
def arn(self):
return "arn:aws:sns:{region}:{AccountId}:app/{platform}/{name}".format(
2019-12-16 21:25:20 -05:00
region=self.region,
platform=self.platform,
name=self.name,
AccountId=DEFAULT_ACCOUNT_ID,
2015-03-14 09:06:31 -04:00
)
2017-03-11 23:41:12 -05:00
class PlatformEndpoint(BaseModel):
2015-03-14 09:06:31 -04:00
def __init__(self, region, application, custom_user_data, token, attributes):
self.region = region
self.application = application
self.custom_user_data = custom_user_data
self.token = token
self.attributes = attributes
self.id = uuid.uuid4()
self.messages = OrderedDict()
self.__fixup_attributes()
def __fixup_attributes(self):
# When AWS returns the attributes dict, it always contains these two elements, so we need to
# automatically ensure they exist as well.
2019-10-31 08:44:26 -07:00
if "Token" not in self.attributes:
self.attributes["Token"] = self.token
if "Enabled" in self.attributes:
enabled = self.attributes["Enabled"]
self.attributes["Enabled"] = enabled.lower()
else:
self.attributes["Enabled"] = "true"
2015-03-14 09:06:31 -04:00
2017-03-16 22:28:30 -04:00
@property
def enabled(self):
2019-10-31 08:44:26 -07:00
return json.loads(self.attributes.get("Enabled", "true").lower())
2017-03-16 22:28:30 -04:00
2015-03-14 09:06:31 -04:00
@property
def arn(self):
return "arn:aws:sns:{region}:{AccountId}:endpoint/{platform}/{name}/{id}".format(
2015-03-14 09:06:31 -04:00
region=self.region,
AccountId=DEFAULT_ACCOUNT_ID,
2015-03-14 09:06:31 -04:00
platform=self.application.platform,
name=self.application.name,
id=self.id,
)
def publish(self, message):
2017-03-16 22:28:30 -04:00
if not self.enabled:
raise SnsEndpointDisabled("Endpoint %s disabled" % self.id)
2015-03-14 09:06:31 -04:00
# This is where we would actually send a message
2021-07-26 07:40:39 +01:00
message_id = str(uuid.uuid4())
self.messages[message_id] = message
2015-03-14 09:06:31 -04:00
return message_id
2014-05-11 22:56:44 -04:00
class SNSBackend(BaseBackend):
def __init__(self, region_name):
super(SNSBackend, self).__init__()
self.topics = OrderedDict()
2021-08-27 00:23:17 +09:00
self.subscriptions: OrderedDict[str, Subscription] = OrderedDict()
2015-03-14 09:06:31 -04:00
self.applications = {}
self.platform_endpoints = {}
self.region_name = region_name
self.sms_attributes = {}
self.sms_messages = OrderedDict()
2019-10-31 08:44:26 -07:00
self.opt_out_numbers = [
"+447420500600",
"+447420505401",
"+447632960543",
"+447632960028",
"+447700900149",
"+447700900550",
"+447700900545",
"+447700900907",
]
def reset(self):
region_name = self.region_name
self.__dict__ = {}
self.__init__(region_name)
2014-05-11 22:56:44 -04:00
@staticmethod
def default_vpc_endpoint_service(service_region, zones):
"""List of dicts representing default VPC endpoints for this service."""
return BaseBackend.default_vpc_endpoint_service_factory(
service_region, zones, "sns"
)
def update_sms_attributes(self, attrs):
self.sms_attributes.update(attrs)
2019-10-11 17:58:48 +02:00
def create_topic(self, name, attributes=None, tags=None):
if attributes is None:
attributes = {}
if (
attributes.get("FifoTopic")
and attributes.get("FifoTopic").lower() == "true"
):
fails_constraints = not re.match(r"^[a-zA-Z0-9_-]{1,256}\.fifo$", name)
msg = "Fifo Topic names must end with .fifo and must be made up of only uppercase and lowercase ASCII letters, numbers, underscores, and hyphens, and must be between 1 and 256 characters long."
else:
fails_constraints = not re.match(r"^[a-zA-Z0-9_-]{1,256}$", name)
msg = "Topic names must be made up of only uppercase and lowercase ASCII letters, numbers, underscores, and hyphens, and must be between 1 and 256 characters long."
if fails_constraints:
raise InvalidParameterValue(msg)
candidate_topic = Topic(name, self)
if attributes:
for attribute in attributes:
2019-10-31 08:44:26 -07:00
setattr(
candidate_topic,
camelcase_to_underscores(attribute),
attributes[attribute],
)
2019-10-11 17:58:48 +02:00
if tags:
candidate_topic._tags = tags
if candidate_topic.arn in self.topics:
return self.topics[candidate_topic.arn]
else:
self.topics[candidate_topic.arn] = candidate_topic
return candidate_topic
2014-05-11 22:56:44 -04:00
def _get_values_nexttoken(self, values_map, next_token=None):
if next_token is None or not next_token:
next_token = 0
next_token = int(next_token)
2019-10-31 08:44:26 -07:00
values = list(values_map.values())[next_token : next_token + DEFAULT_PAGE_SIZE]
if len(values) == DEFAULT_PAGE_SIZE:
next_token = next_token + DEFAULT_PAGE_SIZE
else:
next_token = None
return values, next_token
def _get_topic_subscriptions(self, topic):
return [sub for sub in self.subscriptions.values() if sub.topic == topic]
def list_topics(self, next_token=None):
return self._get_values_nexttoken(self.topics, next_token)
2014-05-11 22:56:44 -04:00
def delete_topic_subscriptions(self, topic):
Merge LocalStack changes into upstream moto (#4082) * fix OPTIONS requests on non-existing API GW integrations * add cloudformation models for API Gateway deployments * bump version * add backdoor to return CloudWatch metrics * Updating implementation coverage * Updating implementation coverage * add cloudformation models for API Gateway deployments * Updating implementation coverage * Updating implementation coverage * Implemented get-caller-identity returning real data depending on the access key used. * bump version * minor fixes * fix Number data_type for SQS message attribute * fix handling of encoding errors * bump version * make CF stack queryable before starting to initialize its resources * bump version * fix integration_method for API GW method integrations * fix undefined status in CF FakeStack * Fix apigateway issues with terraform v0.12.21 * resource_methods -> add handle for "DELETE" method * integrations -> fix issue that "httpMethod" wasn't included in body request (this value was set as the value from refer method resource) * bump version * Fix setting http method for API gateway integrations (#6) * bump version * remove duplicate methods * add storage class to S3 Key when completing multipart upload (#7) * fix SQS performance issues; bump version * add pagination to SecretsManager list-secrets (#9) * fix default parameter groups in RDS * fix adding S3 metadata headers with names containing dots (#13) * Updating implementation coverage * Updating implementation coverage * add cloudformation models for API Gateway deployments * Updating implementation coverage * Updating implementation coverage * Implemented get-caller-identity returning real data depending on the access key used. * make CF stack queryable before starting to initialize its resources * bump version * remove duplicate methods * fix adding S3 metadata headers with names containing dots (#13) * Update amis.json to support EKS AMI mocks (#15) * fix PascalCase for boolean value in ListMultipartUploads response (#17); fix _get_multi_param to parse nested list/dict query params * determine non-zero container exit code in Batch API * support filtering by dimensions in CW get_metric_statistics * fix storing attributes for ELBv2 Route entities; API GW refactorings for TF tests * add missing fields for API GW resources * fix error messages for Route53 (TF-compat) * various fixes for IAM resources (tf-compat) * minor fixes for API GW models (tf-compat) * minor fixes for API GW responses (tf-compat) * add s3 exception for bucket notification filter rule validation * change the way RESTErrors generate the response body and content-type header * fix lint errors and disable "black" syntax enforcement * remove return type hint in RESTError.get_body * add RESTError XML template for IAM exceptions * add support for API GW minimumCompressionSize * fix casing getting PrivateDnsEnabled API GW attribute * minor fixes for error responses * fix escaping special chars for IAM role descriptions (tf-compat) * minor fixes and tagging support for API GW and ELB v2 (tf-compat) * Merge branch 'master' into localstack * add "AlarmRule" attribute to enable support for composite CloudWatch metrics * fix recursive parsing of complex/nested query params * bump version * add API to delete S3 website configurations (#18) * use dict copy to allow parallelism and avoid concurrent modification exceptions in S3 * fix precondition check for etags in S3 (#19) * minor fix for user filtering in Cognito * fix API Gateway error response; avoid returning empty response templates (tf-compat) * support tags and tracingEnabled attribute for API GW stages * fix boolean value in S3 encryption response (#20) * fix connection arn structure * fix api destination arn structure * black format * release 2.0.3.37 * fix s3 exception tests see botocore/parsers.py:1002 where RequestId is removed from parsed * remove python 2 from build action * add test failure annotations in build action * fix events test arn comparisons * fix s3 encryption response test * return default value "0" if EC2 availableIpAddressCount is empty * fix extracting SecurityGroupIds for EC2 VPC endpoints * support deleting/updating API Gateway DomainNames * fix(events): Return empty string instead of null when no pattern is specified in EventPattern (tf-compat) (#22) * fix logic and revert CF changes to get tests running again (#21) * add support for EC2 customer gateway API (#25) * add support for EC2 Transit Gateway APIs (#24) * feat(logs): add `kmsKeyId` into `LogGroup` entity (#23) * minor change in ELBv2 logic to fix tests * feat(events): add APIs to describe and delete CloudWatch Events connections (#26) * add support for EC2 transit gateway route tables (#27) * pass transit gateway route table ID in Describe API, minor refactoring (#29) * add support for EC2 Transit Gateway Routes (#28) * fix region on ACM certificate import (#31) * add support for EC2 transit gateway attachments (#30) * add support for EC2 Transit Gateway VPN attachments (#32) * fix account ID for logs API * add support for DeleteOrganization API * feat(events): store raw filter representation for CloudWatch events patterns (tf-compat) (#36) * feat(events): add support to describe/update/delete CloudWatch API destinations (#35) * add Cognito UpdateIdentityPool, CW Logs PutResourcePolicy * feat(events): add support for tags in EventBus API (#38) * fix parameter validation for Batch compute environments (tf-compat) * revert merge conflicts in IMPLEMENTATION_COVERAGE.md * format code using black * restore original README; re-enable and fix CloudFormation tests * restore tests and old logic for CF stack parameters from SSM * parameterize RequestId/RequestID in response messages and revert related test changes * undo LocalStack-specific adaptations * minor fix * Update CodeCov config to reflect removal of Py2 * undo change related to CW metric filtering; add additional test for CW metric statistics with dimensions * Terraform - Extend whitelist of running tests Co-authored-by: acsbendi <acsbendi28@gmail.com> Co-authored-by: Phan Duong <duongpv@outlook.com> Co-authored-by: Thomas Rausch <thomas@thrau.at> Co-authored-by: Macwan Nevil <macnev2013@gmail.com> Co-authored-by: Dominik Schubert <dominik.schubert91@gmail.com> Co-authored-by: Gonzalo Saad <saad.gonzalo.ale@gmail.com> Co-authored-by: Mohit Alonja <monty16597@users.noreply.github.com> Co-authored-by: Miguel Gagliardo <migag9@gmail.com> Co-authored-by: Bert Blommers <info@bertblommers.nl>
2021-07-26 16:21:17 +02:00
for key, value in dict(self.subscriptions).items():
if value.topic == topic:
self.subscriptions.pop(key)
2014-05-11 22:56:44 -04:00
def delete_topic(self, arn):
try:
topic = self.get_topic(arn)
self.delete_topic_subscriptions(topic)
self.topics.pop(arn)
except KeyError:
raise SNSNotFoundError("Topic with arn {0} not found".format(arn))
2014-05-11 22:56:44 -04:00
def get_topic(self, arn):
2015-03-14 09:06:31 -04:00
try:
return self.topics[arn]
except KeyError:
2015-03-14 09:19:36 -04:00
raise SNSNotFoundError("Topic with arn {0} not found".format(arn))
2014-05-11 22:56:44 -04:00
def set_topic_attribute(self, topic_arn, attribute_name, attribute_value):
topic = self.get_topic(topic_arn)
setattr(topic, attribute_name, attribute_value)
def subscribe(self, topic_arn, endpoint, protocol):
if protocol == "sms":
if re.search(r"[./-]{2,}", endpoint) or re.search(
r"(^[./-]|[./-]$)", endpoint
):
raise SNSInvalidParameter("Invalid SMS endpoint: {}".format(endpoint))
reduced_endpoint = re.sub(r"[./-]", "", endpoint)
if not is_e164(reduced_endpoint):
raise SNSInvalidParameter("Invalid SMS endpoint: {}".format(endpoint))
2018-01-02 11:30:39 +11:00
# AWS doesn't create duplicates
old_subscription = self._find_subscription(topic_arn, endpoint, protocol)
if old_subscription:
return old_subscription
2014-05-11 22:56:44 -04:00
topic = self.get_topic(topic_arn)
subscription = Subscription(topic, endpoint, protocol)
attributes = {
2019-10-31 08:44:26 -07:00
"PendingConfirmation": "false",
"ConfirmationWasAuthenticated": "true",
2019-10-31 08:44:26 -07:00
"Endpoint": endpoint,
"TopicArn": topic_arn,
"Protocol": protocol,
"SubscriptionArn": subscription.arn,
"Owner": DEFAULT_ACCOUNT_ID,
"RawMessageDelivery": "false",
}
if protocol in ["http", "https"]:
attributes["EffectiveDeliveryPolicy"] = topic.effective_delivery_policy
subscription.attributes = attributes
2014-05-11 22:56:44 -04:00
self.subscriptions[subscription.arn] = subscription
return subscription
2018-01-02 11:30:39 +11:00
def _find_subscription(self, topic_arn, endpoint, protocol):
for subscription in self.subscriptions.values():
2019-10-31 08:44:26 -07:00
if (
subscription.topic.arn == topic_arn
and subscription.endpoint == endpoint
and subscription.protocol == protocol
):
2018-01-02 11:30:39 +11:00
return subscription
return None
2014-05-11 22:56:44 -04:00
def unsubscribe(self, subscription_arn):
self.subscriptions.pop(subscription_arn, None)
2014-05-11 22:56:44 -04:00
def list_subscriptions(self, topic_arn=None, next_token=None):
2014-05-11 22:56:44 -04:00
if topic_arn:
topic = self.get_topic(topic_arn)
2017-02-23 21:37:43 -05:00
filtered = OrderedDict(
2019-10-31 08:44:26 -07:00
[(sub.arn, sub) for sub in self._get_topic_subscriptions(topic)]
)
return self._get_values_nexttoken(filtered, next_token)
2014-05-11 22:56:44 -04:00
else:
return self._get_values_nexttoken(self.subscriptions, next_token)
2014-05-11 22:56:44 -04:00
def publish(
self,
message,
arn=None,
phone_number=None,
subject=None,
message_attributes=None,
):
if subject is not None and len(subject) > 100:
# Note that the AWS docs around length are wrong: https://github.com/spulec/moto/issues/1503
2019-10-31 08:44:26 -07:00
raise ValueError("Subject must be less than 100 characters")
2017-10-20 13:19:55 +01:00
if phone_number:
# This is only an approximation. In fact, we should try to use GSM-7 or UCS-2 encoding to count used bytes
if len(message) > MAXIMUM_SMS_MESSAGE_BYTES:
raise ValueError("SMS message must be less than 1600 bytes")
2021-07-26 07:40:39 +01:00
message_id = str(uuid.uuid4())
self.sms_messages[message_id] = (phone_number, message)
return message_id
if len(message) > MAXIMUM_MESSAGE_LENGTH:
2019-10-31 08:44:26 -07:00
raise InvalidParameterValue(
"An error occurred (InvalidParameter) when calling the Publish operation: Invalid parameter: Message too long"
)
2015-03-14 09:06:31 -04:00
try:
topic = self.get_topic(arn)
2019-10-31 08:44:26 -07:00
message_id = topic.publish(
message, subject=subject, message_attributes=message_attributes
)
2015-03-14 09:06:31 -04:00
except SNSNotFoundError:
endpoint = self.get_endpoint(arn)
message_id = endpoint.publish(message)
2014-05-11 22:56:44 -04:00
return message_id
2015-03-14 09:06:31 -04:00
def create_platform_application(self, region, name, platform, attributes):
application = PlatformApplication(region, name, platform, attributes)
self.applications[application.arn] = application
return application
def get_application(self, arn):
try:
return self.applications[arn]
except KeyError:
2019-10-31 08:44:26 -07:00
raise SNSNotFoundError("Application with arn {0} not found".format(arn))
2015-03-14 09:06:31 -04:00
def set_application_attributes(self, arn, attributes):
application = self.get_application(arn)
application.attributes.update(attributes)
return application
def list_platform_applications(self):
return self.applications.values()
def delete_platform_application(self, platform_arn):
self.applications.pop(platform_arn)
2019-10-31 08:44:26 -07:00
def create_platform_endpoint(
self, region, application, custom_user_data, token, attributes
):
for endpoint in self.platform_endpoints.values():
if token == endpoint.token:
Merge LocalStack changes into upstream moto (#4082) * fix OPTIONS requests on non-existing API GW integrations * add cloudformation models for API Gateway deployments * bump version * add backdoor to return CloudWatch metrics * Updating implementation coverage * Updating implementation coverage * add cloudformation models for API Gateway deployments * Updating implementation coverage * Updating implementation coverage * Implemented get-caller-identity returning real data depending on the access key used. * bump version * minor fixes * fix Number data_type for SQS message attribute * fix handling of encoding errors * bump version * make CF stack queryable before starting to initialize its resources * bump version * fix integration_method for API GW method integrations * fix undefined status in CF FakeStack * Fix apigateway issues with terraform v0.12.21 * resource_methods -> add handle for "DELETE" method * integrations -> fix issue that "httpMethod" wasn't included in body request (this value was set as the value from refer method resource) * bump version * Fix setting http method for API gateway integrations (#6) * bump version * remove duplicate methods * add storage class to S3 Key when completing multipart upload (#7) * fix SQS performance issues; bump version * add pagination to SecretsManager list-secrets (#9) * fix default parameter groups in RDS * fix adding S3 metadata headers with names containing dots (#13) * Updating implementation coverage * Updating implementation coverage * add cloudformation models for API Gateway deployments * Updating implementation coverage * Updating implementation coverage * Implemented get-caller-identity returning real data depending on the access key used. * make CF stack queryable before starting to initialize its resources * bump version * remove duplicate methods * fix adding S3 metadata headers with names containing dots (#13) * Update amis.json to support EKS AMI mocks (#15) * fix PascalCase for boolean value in ListMultipartUploads response (#17); fix _get_multi_param to parse nested list/dict query params * determine non-zero container exit code in Batch API * support filtering by dimensions in CW get_metric_statistics * fix storing attributes for ELBv2 Route entities; API GW refactorings for TF tests * add missing fields for API GW resources * fix error messages for Route53 (TF-compat) * various fixes for IAM resources (tf-compat) * minor fixes for API GW models (tf-compat) * minor fixes for API GW responses (tf-compat) * add s3 exception for bucket notification filter rule validation * change the way RESTErrors generate the response body and content-type header * fix lint errors and disable "black" syntax enforcement * remove return type hint in RESTError.get_body * add RESTError XML template for IAM exceptions * add support for API GW minimumCompressionSize * fix casing getting PrivateDnsEnabled API GW attribute * minor fixes for error responses * fix escaping special chars for IAM role descriptions (tf-compat) * minor fixes and tagging support for API GW and ELB v2 (tf-compat) * Merge branch 'master' into localstack * add "AlarmRule" attribute to enable support for composite CloudWatch metrics * fix recursive parsing of complex/nested query params * bump version * add API to delete S3 website configurations (#18) * use dict copy to allow parallelism and avoid concurrent modification exceptions in S3 * fix precondition check for etags in S3 (#19) * minor fix for user filtering in Cognito * fix API Gateway error response; avoid returning empty response templates (tf-compat) * support tags and tracingEnabled attribute for API GW stages * fix boolean value in S3 encryption response (#20) * fix connection arn structure * fix api destination arn structure * black format * release 2.0.3.37 * fix s3 exception tests see botocore/parsers.py:1002 where RequestId is removed from parsed * remove python 2 from build action * add test failure annotations in build action * fix events test arn comparisons * fix s3 encryption response test * return default value "0" if EC2 availableIpAddressCount is empty * fix extracting SecurityGroupIds for EC2 VPC endpoints * support deleting/updating API Gateway DomainNames * fix(events): Return empty string instead of null when no pattern is specified in EventPattern (tf-compat) (#22) * fix logic and revert CF changes to get tests running again (#21) * add support for EC2 customer gateway API (#25) * add support for EC2 Transit Gateway APIs (#24) * feat(logs): add `kmsKeyId` into `LogGroup` entity (#23) * minor change in ELBv2 logic to fix tests * feat(events): add APIs to describe and delete CloudWatch Events connections (#26) * add support for EC2 transit gateway route tables (#27) * pass transit gateway route table ID in Describe API, minor refactoring (#29) * add support for EC2 Transit Gateway Routes (#28) * fix region on ACM certificate import (#31) * add support for EC2 transit gateway attachments (#30) * add support for EC2 Transit Gateway VPN attachments (#32) * fix account ID for logs API * add support for DeleteOrganization API * feat(events): store raw filter representation for CloudWatch events patterns (tf-compat) (#36) * feat(events): add support to describe/update/delete CloudWatch API destinations (#35) * add Cognito UpdateIdentityPool, CW Logs PutResourcePolicy * feat(events): add support for tags in EventBus API (#38) * fix parameter validation for Batch compute environments (tf-compat) * revert merge conflicts in IMPLEMENTATION_COVERAGE.md * format code using black * restore original README; re-enable and fix CloudFormation tests * restore tests and old logic for CF stack parameters from SSM * parameterize RequestId/RequestID in response messages and revert related test changes * undo LocalStack-specific adaptations * minor fix * Update CodeCov config to reflect removal of Py2 * undo change related to CW metric filtering; add additional test for CW metric statistics with dimensions * Terraform - Extend whitelist of running tests Co-authored-by: acsbendi <acsbendi28@gmail.com> Co-authored-by: Phan Duong <duongpv@outlook.com> Co-authored-by: Thomas Rausch <thomas@thrau.at> Co-authored-by: Macwan Nevil <macnev2013@gmail.com> Co-authored-by: Dominik Schubert <dominik.schubert91@gmail.com> Co-authored-by: Gonzalo Saad <saad.gonzalo.ale@gmail.com> Co-authored-by: Mohit Alonja <monty16597@users.noreply.github.com> Co-authored-by: Miguel Gagliardo <migag9@gmail.com> Co-authored-by: Bert Blommers <info@bertblommers.nl>
2021-07-26 16:21:17 +02:00
if (
attributes.get("Enabled", "").lower()
== endpoint.attributes["Enabled"]
):
return endpoint
raise DuplicateSnsEndpointError(
"Duplicate endpoint token with different attributes: %s" % token
)
2017-02-23 21:37:43 -05:00
platform_endpoint = PlatformEndpoint(
2019-10-31 08:44:26 -07:00
region, application, custom_user_data, token, attributes
)
2015-03-14 09:06:31 -04:00
self.platform_endpoints[platform_endpoint.arn] = platform_endpoint
return platform_endpoint
def list_endpoints_by_platform_application(self, application_arn):
return [
2019-10-31 08:44:26 -07:00
endpoint
for endpoint in self.platform_endpoints.values()
2015-03-14 09:06:31 -04:00
if endpoint.application.arn == application_arn
]
def get_endpoint(self, arn):
try:
return self.platform_endpoints[arn]
except KeyError:
raise SNSNotFoundError("Endpoint does not exist")
2015-03-14 09:06:31 -04:00
def set_endpoint_attributes(self, arn, attributes):
endpoint = self.get_endpoint(arn)
if "Enabled" in attributes:
attributes["Enabled"] = attributes["Enabled"].lower()
2015-03-14 09:06:31 -04:00
endpoint.attributes.update(attributes)
return endpoint
def delete_endpoint(self, arn):
try:
del self.platform_endpoints[arn]
except KeyError:
2019-10-31 08:44:26 -07:00
raise SNSNotFoundError("Endpoint with arn {0} not found".format(arn))
def get_subscription_attributes(self, arn):
2021-08-27 00:23:17 +09:00
subscription = self.subscriptions.get(arn)
if not subscription:
raise SNSNotFoundError(
"Subscription does not exist", template="wrapped_single_error"
)
return subscription.attributes
def set_subscription_attributes(self, arn, name, value):
if name not in [
"RawMessageDelivery",
"DeliveryPolicy",
"FilterPolicy",
"RedrivePolicy",
]:
2019-10-31 08:44:26 -07:00
raise SNSInvalidParameter("AttributeName")
# TODO: should do validation
_subscription = [_ for _ in self.subscriptions.values() if _.arn == arn]
if not _subscription:
raise SNSNotFoundError("Subscription with arn {0} not found".format(arn))
subscription = _subscription[0]
subscription.attributes[name] = value
2019-10-31 08:44:26 -07:00
if name == "FilterPolicy":
filter_policy = json.loads(value)
self._validate_filter_policy(filter_policy)
subscription._filter_policy = filter_policy
def _validate_filter_policy(self, value):
# TODO: extend validation checks
combinations = 1
2021-07-26 07:40:39 +01:00
for rules in value.values():
combinations *= len(rules)
2019-11-16 12:31:45 -08:00
# Even the official documentation states the total combination of values must not exceed 100, in reality it is 150
# https://docs.aws.amazon.com/sns/latest/dg/sns-subscription-filter-policies.html#subscription-filter-policy-constraints
if combinations > 150:
2019-10-31 08:44:26 -07:00
raise SNSInvalidParameter(
"Invalid parameter: FilterPolicy: Filter policy is too complex"
)
2021-07-26 07:40:39 +01:00
for field, rules in value.items():
for rule in rules:
if rule is None:
continue
2021-07-26 07:40:39 +01:00
if isinstance(rule, str):
continue
if isinstance(rule, bool):
continue
2021-07-26 07:40:39 +01:00
if isinstance(rule, (int, float)):
if rule <= -1000000000 or rule >= 1000000000:
raise InternalError("Unknown")
continue
if isinstance(rule, dict):
keyword = list(rule.keys())[0]
attributes = list(rule.values())[0]
2019-10-31 08:44:26 -07:00
if keyword == "anything-but":
continue
2019-10-31 08:44:26 -07:00
elif keyword == "exists":
if not isinstance(attributes, bool):
2019-10-31 08:44:26 -07:00
raise SNSInvalidParameter(
"Invalid parameter: FilterPolicy: exists match pattern must be either true or false."
)
continue
2019-10-31 08:44:26 -07:00
elif keyword == "numeric":
continue
2019-10-31 08:44:26 -07:00
elif keyword == "prefix":
continue
else:
2019-10-31 08:44:26 -07:00
raise SNSInvalidParameter(
"Invalid parameter: FilterPolicy: Unrecognized match type {type}".format(
type=keyword
)
)
2019-10-31 08:44:26 -07:00
raise SNSInvalidParameter(
"Invalid parameter: FilterPolicy: Match value must be String, number, true, false, or null"
)
def add_permission(self, topic_arn, label, aws_account_ids, action_names):
if topic_arn not in self.topics:
2019-10-31 08:44:26 -07:00
raise SNSNotFoundError("Topic does not exist")
policy = self.topics[topic_arn]._policy_json
2019-10-31 08:44:26 -07:00
statement = next(
(
statement
for statement in policy["Statement"]
if statement["Sid"] == label
),
None,
)
if statement:
2019-10-31 08:44:26 -07:00
raise SNSInvalidParameter("Statement already exists")
if any(action_name not in VALID_POLICY_ACTIONS for action_name in action_names):
2019-10-31 08:44:26 -07:00
raise SNSInvalidParameter("Policy statement action out of service scope!")
2019-10-31 08:44:26 -07:00
principals = [
"arn:aws:iam::{}:root".format(account_id) for account_id in aws_account_ids
]
actions = ["SNS:{}".format(action_name) for action_name in action_names]
statement = {
2019-10-31 08:44:26 -07:00
"Sid": label,
"Effect": "Allow",
"Principal": {"AWS": principals[0] if len(principals) == 1 else principals},
"Action": actions[0] if len(actions) == 1 else actions,
"Resource": topic_arn,
}
2019-10-31 08:44:26 -07:00
self.topics[topic_arn]._policy_json["Statement"].append(statement)
def remove_permission(self, topic_arn, label):
if topic_arn not in self.topics:
2019-10-31 08:44:26 -07:00
raise SNSNotFoundError("Topic does not exist")
2019-10-31 08:44:26 -07:00
statements = self.topics[topic_arn]._policy_json["Statement"]
statements = [
statement for statement in statements if statement["Sid"] != label
]
2019-10-31 08:44:26 -07:00
self.topics[topic_arn]._policy_json["Statement"] = statements
2019-10-11 17:58:48 +02:00
def list_tags_for_resource(self, resource_arn):
2019-10-12 20:36:15 +02:00
if resource_arn not in self.topics:
raise ResourceNotFoundError
2019-10-11 17:58:48 +02:00
return self.topics[resource_arn]._tags
2019-10-12 20:36:15 +02:00
def tag_resource(self, resource_arn, tags):
if resource_arn not in self.topics:
raise ResourceNotFoundError
updated_tags = self.topics[resource_arn]._tags.copy()
updated_tags.update(tags)
if len(updated_tags) > 50:
raise TagLimitExceededError
self.topics[resource_arn]._tags = updated_tags
2019-10-12 21:10:51 +02:00
def untag_resource(self, resource_arn, tag_keys):
if resource_arn not in self.topics:
raise ResourceNotFoundError
for key in tag_keys:
self.topics[resource_arn]._tags.pop(key, None)
2015-03-14 09:06:31 -04:00
2014-11-16 18:35:11 -05:00
sns_backends = {}
2019-10-31 08:44:26 -07:00
for region in Session().get_available_regions("sns"):
sns_backends[region] = SNSBackend(region)
2019-12-26 17:12:22 +01:00
for region in Session().get_available_regions("sns", partition_name="aws-us-gov"):
sns_backends[region] = SNSBackend(region)
for region in Session().get_available_regions("sns", partition_name="aws-cn"):
sns_backends[region] = SNSBackend(region)
2014-05-11 22:56:44 -04:00
2017-02-27 20:53:57 -05:00
DEFAULT_EFFECTIVE_DELIVERY_POLICY = {
"defaultHealthyRetryPolicy": {
"numNoDelayRetries": 0,
"numMinDelayRetries": 0,
"minDelayTarget": 20,
"maxDelayTarget": 20,
"numMaxDelayRetries": 0,
"numRetries": 3,
"backoffFunction": "linear",
},
"sicklyRetryPolicy": None,
"throttlePolicy": None,
"guaranteed": False,
2017-02-27 20:53:57 -05:00
}
VALID_POLICY_ACTIONS = [
2019-10-31 08:44:26 -07:00
"GetTopicAttributes",
"SetTopicAttributes",
"AddPermission",
"RemovePermission",
"DeleteTopic",
"Subscribe",
"ListSubscriptionsByTopic",
"Publish",
"Receive",
]