moto/moto/core/models.py

912 lines
31 KiB
Python
Raw Normal View History

2013-02-18 21:09:40 +00:00
import functools
import inspect
import os
import random
2013-02-18 21:09:40 +00:00
import re
import string
import types
from abc import abstractmethod
from io import BytesIO
from collections import defaultdict
2021-08-05 16:59:25 +00:00
try:
from importlib.metadata import version
except ImportError:
from importlib_metadata import version
2020-06-27 14:11:41 +00:00
from botocore.config import Config
from botocore.handlers import BUILTIN_HANDLERS
from botocore.awsrequest import AWSResponse
from distutils.version import LooseVersion
2021-07-26 06:40:39 +00:00
from http.client import responses as http_responses
from urllib.parse import urlparse
from werkzeug.wrappers import Request
2013-02-18 21:09:40 +00:00
from moto import settings
import responses
2017-02-16 03:35:45 +00:00
from moto.packages.httpretty import HTTPretty
from unittest.mock import patch
2017-02-17 03:51:04 +00:00
from .utils import (
convert_httpretty_response,
convert_regex_to_flask_path,
convert_flask_to_responses_response,
)
2013-02-18 21:09:40 +00:00
2019-12-17 02:25:20 +00:00
ACCOUNT_ID = os.environ.get("MOTO_ACCOUNT_ID", "123456789012")
2021-07-26 06:40:39 +00:00
class BaseMockAWS:
nested_count = 0
mocks_active = False
def __init__(self, backends):
2020-01-17 01:08:06 +00:00
from moto.instance_metadata import instance_metadata_backend
from moto.core import moto_api_backend
self.backends = backends
self.backends_for_urls = {}
default_backends = {
2020-01-17 01:08:06 +00:00
"instance_metadata": instance_metadata_backend,
"moto_api": moto_api_backend,
}
self.backends_for_urls.update(self.backends)
self.backends_for_urls.update(default_backends)
self.FAKE_KEYS = {
2019-10-31 15:44:26 +00:00
"AWS_ACCESS_KEY_ID": "foobar_key",
"AWS_SECRET_ACCESS_KEY": "foobar_secret",
}
self.ORIG_KEYS = {}
self.default_session_mock = patch("boto3.DEFAULT_SESSION", None)
if self.__class__.nested_count == 0:
2017-02-16 03:35:45 +00:00
self.reset()
2013-02-28 03:25:15 +00:00
2015-06-27 23:01:01 +00:00
def __call__(self, func, reset=True):
if inspect.isclass(func):
return self.decorate_class(func)
2015-06-27 23:01:01 +00:00
return self.decorate_callable(func, reset)
2013-02-28 03:25:15 +00:00
def __enter__(self):
self.start()
2019-07-10 01:31:43 +00:00
return self
2013-02-28 03:25:15 +00:00
def __exit__(self, *args):
self.stop()
2015-04-03 03:40:40 +00:00
def start(self, reset=True):
if not self.__class__.mocks_active:
self.default_session_mock.start()
self.mock_env_variables()
self.__class__.mocks_active = True
self.__class__.nested_count += 1
2015-04-03 03:40:40 +00:00
if reset:
for backend in self.backends.values():
backend.reset()
2017-02-16 03:35:45 +00:00
self.enable_patching()
2013-02-28 03:25:15 +00:00
def stop(self):
self.__class__.nested_count -= 1
if self.__class__.nested_count < 0:
2019-10-31 15:44:26 +00:00
raise RuntimeError("Called stop() before start().")
2017-02-20 19:31:19 +00:00
if self.__class__.nested_count == 0:
if self.__class__.mocks_active:
try:
self.default_session_mock.stop()
except RuntimeError:
# We only need to check for this exception in Python 3.6 and 3.7
# https://bugs.python.org/issue36366
pass
self.unmock_env_variables()
self.__class__.mocks_active = False
2017-02-20 19:31:19 +00:00
self.disable_patching()
2013-02-28 03:25:15 +00:00
2015-06-27 23:01:01 +00:00
def decorate_callable(self, func, reset):
2013-02-28 03:25:15 +00:00
def wrapper(*args, **kwargs):
2015-06-27 23:01:01 +00:00
self.start(reset=reset)
try:
2013-02-28 03:25:15 +00:00
result = func(*args, **kwargs)
2015-06-27 23:01:01 +00:00
finally:
self.stop()
2013-02-28 03:25:15 +00:00
return result
2019-10-31 15:44:26 +00:00
2013-02-28 03:25:15 +00:00
functools.update_wrapper(wrapper, func)
wrapper.__wrapped__ = func
2013-02-28 03:25:15 +00:00
return wrapper
def decorate_class(self, klass):
for attr in dir(klass):
if attr.startswith("_"):
continue
attr_value = getattr(klass, attr)
if not hasattr(attr_value, "__call__"):
continue
2021-09-28 09:53:05 +00:00
if not hasattr(attr_value, "__name__"):
continue
# Check if this is a classmethod. If so, skip patching
if inspect.ismethod(attr_value) and attr_value.__self__ is klass:
continue
2018-09-05 09:39:09 +00:00
# Check if this is a staticmethod. If so, skip patching
for cls in inspect.getmro(klass):
if attr_value.__name__ not in cls.__dict__:
continue
bound_attr_value = cls.__dict__[attr_value.__name__]
if not isinstance(bound_attr_value, staticmethod):
break
else:
# It is a staticmethod, skip patching
continue
try:
2015-06-27 23:01:01 +00:00
setattr(klass, attr, self(attr_value, reset=False))
except TypeError:
# Sometimes we can't set this for built-in types
continue
return klass
def mock_env_variables(self):
# "Mock" the AWS credentials as they can't be mocked in Botocore currently
# self.env_variables_mocks = mock.patch.dict(os.environ, FAKE_KEYS)
# self.env_variables_mocks.start()
for k, v in self.FAKE_KEYS.items():
self.ORIG_KEYS[k] = os.environ.get(k, None)
os.environ[k] = v
def unmock_env_variables(self):
# This doesn't work in Python2 - for some reason, unmocking clears the entire os.environ dict
# Obviously bad user experience, and also breaks pytest - as it uses PYTEST_CURRENT_TEST as an env var
# self.env_variables_mocks.stop()
for k, v in self.ORIG_KEYS.items():
if v:
os.environ[k] = v
else:
del os.environ[k]
2013-02-28 03:25:15 +00:00
2017-02-16 03:35:45 +00:00
class HttprettyMockAWS(BaseMockAWS):
def reset(self):
HTTPretty.reset()
def enable_patching(self):
if not HTTPretty.is_enabled():
HTTPretty.enable()
for method in HTTPretty.METHODS:
for backend in self.backends_for_urls.values():
for key, value in backend.urls.items():
HTTPretty.register_uri(
method=method,
uri=re.compile(key),
body=convert_httpretty_response(value),
)
2017-02-16 03:35:45 +00:00
def disable_patching(self):
2017-02-20 19:31:19 +00:00
HTTPretty.disable()
HTTPretty.reset()
2017-02-16 03:35:45 +00:00
2019-10-31 15:44:26 +00:00
RESPONSES_METHODS = [
responses.GET,
responses.DELETE,
responses.HEAD,
responses.OPTIONS,
responses.PATCH,
responses.POST,
responses.PUT,
]
2017-02-16 03:35:45 +00:00
class CallbackResponse(responses.CallbackResponse):
2019-10-31 15:44:26 +00:00
"""
Need to subclass so we can change a couple things
2019-10-31 15:44:26 +00:00
"""
def get_response(self, request):
2019-10-31 15:44:26 +00:00
"""
Need to override this so we can pass decode_content=False
2019-10-31 15:44:26 +00:00
"""
if not isinstance(request, Request):
url = urlparse(request.url)
if request.body is None:
body = None
2021-07-26 06:40:39 +00:00
elif isinstance(request.body, str):
body = BytesIO(request.body.encode("UTF-8"))
elif hasattr(request.body, "read"):
2021-07-26 06:40:39 +00:00
body = BytesIO(request.body.read())
else:
2021-07-26 06:40:39 +00:00
body = BytesIO(request.body)
req = Request.from_values(
2020-04-22 15:02:25 +00:00
path="?".join([url.path, url.query]),
input_stream=body,
content_length=request.headers.get("Content-Length"),
content_type=request.headers.get("Content-Type"),
method=request.method,
2020-04-22 15:02:25 +00:00
base_url="{scheme}://{netloc}".format(
scheme=url.scheme, netloc=url.netloc
),
2021-07-26 06:40:39 +00:00
headers=[(k, v) for k, v in request.headers.items()],
)
request = req
headers = self.get_headers()
result = self.callback(request)
if isinstance(result, Exception):
raise result
status, r_headers, body = result
body = responses._handle_body(body)
headers.update(r_headers)
return responses.HTTPResponse(
status=status,
2021-07-26 06:40:39 +00:00
reason=http_responses.get(status),
body=body,
headers=headers,
preload_content=False,
# Need to not decode_content to mimic requests
decode_content=False,
)
def _url_matches(self, url, other, match_querystring=False):
2019-10-31 15:44:26 +00:00
"""
Need to override this so we can fix querystrings breaking regex matching
2019-10-31 15:44:26 +00:00
"""
if not match_querystring:
2019-10-31 15:44:26 +00:00
other = other.split("?", 1)[0]
if responses._is_string(url):
if responses._has_unicode(url):
url = responses._clean_unicode(url)
2021-07-26 06:40:39 +00:00
if not isinstance(other, str):
2019-10-31 15:44:26 +00:00
other = other.encode("ascii").decode("utf8")
return self._url_matches_strict(url, other)
elif isinstance(url, responses.Pattern) and url.match(other):
return True
else:
return False
2019-10-31 15:44:26 +00:00
botocore_mock = responses.RequestsMock(
assert_all_requests_are_fired=False,
target="botocore.vendored.requests.adapters.HTTPAdapter.send",
)
responses_mock = responses.RequestsMock(assert_all_requests_are_fired=False)
# Add passthrough to allow any other requests to work
2019-09-12 03:07:24 +00:00
# Since this uses .startswith, it applies to http and https requests.
responses_mock.add_passthru("http")
2017-02-20 19:31:19 +00:00
2017-02-24 02:37:43 +00:00
def _find_first_match_legacy(self, request):
matches = []
for i, match in enumerate(self._matches):
if match.matches(request):
matches.append(match)
# Look for implemented callbacks first
implemented_matches = [
m
for m in matches
if type(m) is not CallbackResponse or m.callback != not_implemented_callback
]
if implemented_matches:
return implemented_matches[0]
elif matches:
# We had matches, but all were of type not_implemented_callback
return matches[0]
return None
def _find_first_match(self, request):
matches = []
match_failed_reasons = []
for i, match in enumerate(self._matches):
match_result, reason = match.matches(request)
if match_result:
matches.append(match)
else:
match_failed_reasons.append(reason)
# Look for implemented callbacks first
implemented_matches = [
m
for m in matches
if type(m) is not CallbackResponse or m.callback != not_implemented_callback
]
if implemented_matches:
return implemented_matches[0], []
elif matches:
# We had matches, but all were of type not_implemented_callback
return matches[0], match_failed_reasons
return None, match_failed_reasons
# Modify behaviour of the matcher to only/always return the first match
# Default behaviour is to return subsequent matches for subsequent requests, which leads to https://github.com/spulec/moto/issues/2567
# - First request matches on the appropriate S3 URL
# - Same request, executed again, will be matched on the subsequent match, which happens to be the catch-all, not-yet-implemented, callback
# Fix: Always return the first match
2021-08-05 16:59:25 +00:00
RESPONSES_VERSION = version("responses")
if LooseVersion(RESPONSES_VERSION) < LooseVersion("0.12.1"):
responses_mock._find_match = types.MethodType(
_find_first_match_legacy, responses_mock
)
else:
responses_mock._find_match = types.MethodType(_find_first_match, responses_mock)
2019-10-31 15:44:26 +00:00
BOTOCORE_HTTP_METHODS = ["GET", "DELETE", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"]
class MockRawResponse(BytesIO):
def __init__(self, input):
2021-07-26 06:40:39 +00:00
if isinstance(input, str):
2019-10-31 15:44:26 +00:00
input = input.encode("utf-8")
super(MockRawResponse, self).__init__(input)
def stream(self, **kwargs):
contents = self.read()
while contents:
yield contents
contents = self.read()
2021-07-26 06:40:39 +00:00
class BotocoreStubber:
def __init__(self):
self.enabled = False
self.methods = defaultdict(list)
def reset(self):
self.methods.clear()
def register_response(self, method, pattern, response):
matchers = self.methods[method]
matchers.append((pattern, response))
def __call__(self, event_name, request, **kwargs):
if not self.enabled:
return None
response = None
response_callback = None
found_index = None
matchers = self.methods.get(request.method)
2019-10-31 15:44:26 +00:00
base_url = request.url.split("?", 1)[0]
for i, (pattern, callback) in enumerate(matchers):
if pattern.match(base_url):
if found_index is None:
found_index = i
response_callback = callback
else:
matchers.pop(found_index)
break
if response_callback is not None:
for header, value in request.headers.items():
2021-07-26 06:40:39 +00:00
if isinstance(value, bytes):
2019-10-31 15:44:26 +00:00
request.headers[header] = value.decode("utf-8")
status, headers, body = response_callback(
request, request.url, request.headers
)
body = MockRawResponse(body)
response = AWSResponse(request.url, status, headers, body)
return response
2018-10-01 16:45:12 +00:00
botocore_stubber = BotocoreStubber()
2019-10-31 15:44:26 +00:00
BUILTIN_HANDLERS.append(("before-send", botocore_stubber))
2018-10-01 16:45:12 +00:00
def not_implemented_callback(request):
status = 400
headers = {}
response = "The method is not implemented"
return status, headers, response
class BotocoreEventMockAWS(BaseMockAWS):
def reset(self):
botocore_stubber.reset()
2018-10-14 17:58:56 +00:00
responses_mock.reset()
def enable_patching(self):
botocore_stubber.enabled = True
for method in BOTOCORE_HTTP_METHODS:
for backend in self.backends_for_urls.values():
for key, value in backend.urls.items():
pattern = re.compile(key)
botocore_stubber.register_response(method, pattern, value)
2019-10-31 15:44:26 +00:00
if not hasattr(responses_mock, "_patcher") or not hasattr(
responses_mock._patcher, "target"
):
2018-10-14 17:58:56 +00:00
responses_mock.start()
for method in RESPONSES_METHODS:
# for backend in default_backends.values():
for backend in self.backends_for_urls.values():
for key, value in backend.urls.items():
responses_mock.add(
CallbackResponse(
method=method,
url=re.compile(key),
callback=convert_flask_to_responses_response(value),
match_querystring=False,
)
)
responses_mock.add(
CallbackResponse(
method=method,
url=re.compile(r"https?://.+\.amazonaws.com/.*"),
callback=not_implemented_callback,
match_querystring=False,
)
)
botocore_mock.add(
CallbackResponse(
method=method,
url=re.compile(r"https?://.+\.amazonaws.com/.*"),
callback=not_implemented_callback,
match_querystring=False,
)
)
2018-10-14 17:58:56 +00:00
def disable_patching(self):
botocore_stubber.enabled = False
self.reset()
2018-10-14 17:58:56 +00:00
try:
responses_mock.stop()
except RuntimeError:
pass
MockAWS = BotocoreEventMockAWS
2017-02-16 03:35:45 +00:00
2017-02-20 19:31:19 +00:00
2017-02-20 23:25:10 +00:00
class ServerModeMockAWS(BaseMockAWS):
def reset(self):
2021-10-05 17:11:07 +00:00
call_reset_api = os.environ.get("MOTO_CALL_RESET_API")
if not call_reset_api or call_reset_api.lower() != "false":
import requests
2019-10-31 15:44:26 +00:00
2021-10-05 17:11:07 +00:00
requests.post("http://localhost:5000/moto-api/reset")
2017-02-20 23:25:10 +00:00
def enable_patching(self):
if self.__class__.nested_count == 1:
# Just started
self.reset()
from boto3 import client as real_boto3_client, resource as real_boto3_resource
def fake_boto3_client(*args, **kwargs):
2020-06-27 18:46:26 +00:00
region = self._get_region(*args, **kwargs)
if region:
if "config" in kwargs:
kwargs["config"].__dict__["user_agent_extra"] += " region/" + region
else:
config = Config(user_agent_extra="region/" + region)
kwargs["config"] = config
2019-10-31 15:44:26 +00:00
if "endpoint_url" not in kwargs:
kwargs["endpoint_url"] = "http://localhost:5000"
2017-02-20 23:25:10 +00:00
return real_boto3_client(*args, **kwargs)
2017-02-24 02:37:43 +00:00
2017-02-20 23:25:10 +00:00
def fake_boto3_resource(*args, **kwargs):
2019-10-31 15:44:26 +00:00
if "endpoint_url" not in kwargs:
kwargs["endpoint_url"] = "http://localhost:5000"
2017-02-20 23:25:10 +00:00
return real_boto3_resource(*args, **kwargs)
self._client_patcher = patch("boto3.client", fake_boto3_client)
self._resource_patcher = patch("boto3.resource", fake_boto3_resource)
2017-02-20 23:25:10 +00:00
self._client_patcher.start()
self._resource_patcher.start()
2020-06-27 18:46:26 +00:00
def _get_region(self, *args, **kwargs):
if "region_name" in kwargs:
return kwargs["region_name"]
if type(args) == tuple and len(args) == 2:
2020-06-27 18:46:26 +00:00
service, region = args
return region
return None
2017-02-20 23:25:10 +00:00
def disable_patching(self):
if self._client_patcher:
self._client_patcher.stop()
self._resource_patcher.stop()
2017-02-24 02:37:43 +00:00
class Model(type):
def __new__(self, clsname, bases, namespace):
cls = super(Model, self).__new__(self, clsname, bases, namespace)
cls.__models__ = {}
2014-08-26 17:25:50 +00:00
for name, value in namespace.items():
model = getattr(value, "__returns_model__", False)
if model is not False:
cls.__models__[model] = name
for base in bases:
cls.__models__.update(getattr(base, "__models__", {}))
return cls
@staticmethod
def prop(model_name):
"""decorator to mark a class method as returning model values"""
2019-10-31 15:44:26 +00:00
def dec(f):
f.__returns_model__ = model_name
return f
2019-10-31 15:44:26 +00:00
return dec
2013-02-18 21:09:40 +00:00
2017-03-12 03:45:42 +00:00
model_data = defaultdict(dict)
2017-03-13 00:35:45 +00:00
2017-03-12 03:45:42 +00:00
class InstanceTrackerMeta(type):
def __new__(meta, name, bases, dct):
cls = super(InstanceTrackerMeta, meta).__new__(meta, name, bases, dct)
2019-10-31 15:44:26 +00:00
if name == "BaseModel":
2017-03-12 03:45:42 +00:00
return cls
service = cls.__module__.split(".")[1]
if name not in model_data[service]:
model_data[service][name] = cls
cls.instances = []
return cls
2017-03-13 00:35:45 +00:00
2021-07-26 06:40:39 +00:00
class BaseModel(metaclass=InstanceTrackerMeta):
2017-03-12 03:45:42 +00:00
def __new__(cls, *args, **kwargs):
instance = super(BaseModel, cls).__new__(cls)
2017-03-12 03:45:42 +00:00
cls.instances.append(instance)
return instance
# Parent class for every Model that can be instantiated by CloudFormation
# On subclasses, implement the two methods as @staticmethod to ensure correct behaviour of the CF parser
class CloudFormationModel(BaseModel):
@staticmethod
@abstractmethod
def cloudformation_name_type():
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-name.html
# This must be implemented as a staticmethod with no parameters
# Return None for resources that do not have a name property
pass
@staticmethod
@abstractmethod
def cloudformation_type():
# This must be implemented as a staticmethod with no parameters
# See for example https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-dynamodb-table.html
return "AWS::SERVICE::RESOURCE"
Merge LocalStack changes into upstream moto (#4082) * fix OPTIONS requests on non-existing API GW integrations * add cloudformation models for API Gateway deployments * bump version * add backdoor to return CloudWatch metrics * Updating implementation coverage * Updating implementation coverage * add cloudformation models for API Gateway deployments * Updating implementation coverage * Updating implementation coverage * Implemented get-caller-identity returning real data depending on the access key used. * bump version * minor fixes * fix Number data_type for SQS message attribute * fix handling of encoding errors * bump version * make CF stack queryable before starting to initialize its resources * bump version * fix integration_method for API GW method integrations * fix undefined status in CF FakeStack * Fix apigateway issues with terraform v0.12.21 * resource_methods -> add handle for "DELETE" method * integrations -> fix issue that "httpMethod" wasn't included in body request (this value was set as the value from refer method resource) * bump version * Fix setting http method for API gateway integrations (#6) * bump version * remove duplicate methods * add storage class to S3 Key when completing multipart upload (#7) * fix SQS performance issues; bump version * add pagination to SecretsManager list-secrets (#9) * fix default parameter groups in RDS * fix adding S3 metadata headers with names containing dots (#13) * Updating implementation coverage * Updating implementation coverage * add cloudformation models for API Gateway deployments * Updating implementation coverage * Updating implementation coverage * Implemented get-caller-identity returning real data depending on the access key used. * make CF stack queryable before starting to initialize its resources * bump version * remove duplicate methods * fix adding S3 metadata headers with names containing dots (#13) * Update amis.json to support EKS AMI mocks (#15) * fix PascalCase for boolean value in ListMultipartUploads response (#17); fix _get_multi_param to parse nested list/dict query params * determine non-zero container exit code in Batch API * support filtering by dimensions in CW get_metric_statistics * fix storing attributes for ELBv2 Route entities; API GW refactorings for TF tests * add missing fields for API GW resources * fix error messages for Route53 (TF-compat) * various fixes for IAM resources (tf-compat) * minor fixes for API GW models (tf-compat) * minor fixes for API GW responses (tf-compat) * add s3 exception for bucket notification filter rule validation * change the way RESTErrors generate the response body and content-type header * fix lint errors and disable "black" syntax enforcement * remove return type hint in RESTError.get_body * add RESTError XML template for IAM exceptions * add support for API GW minimumCompressionSize * fix casing getting PrivateDnsEnabled API GW attribute * minor fixes for error responses * fix escaping special chars for IAM role descriptions (tf-compat) * minor fixes and tagging support for API GW and ELB v2 (tf-compat) * Merge branch 'master' into localstack * add "AlarmRule" attribute to enable support for composite CloudWatch metrics * fix recursive parsing of complex/nested query params * bump version * add API to delete S3 website configurations (#18) * use dict copy to allow parallelism and avoid concurrent modification exceptions in S3 * fix precondition check for etags in S3 (#19) * minor fix for user filtering in Cognito * fix API Gateway error response; avoid returning empty response templates (tf-compat) * support tags and tracingEnabled attribute for API GW stages * fix boolean value in S3 encryption response (#20) * fix connection arn structure * fix api destination arn structure * black format * release 2.0.3.37 * fix s3 exception tests see botocore/parsers.py:1002 where RequestId is removed from parsed * remove python 2 from build action * add test failure annotations in build action * fix events test arn comparisons * fix s3 encryption response test * return default value "0" if EC2 availableIpAddressCount is empty * fix extracting SecurityGroupIds for EC2 VPC endpoints * support deleting/updating API Gateway DomainNames * fix(events): Return empty string instead of null when no pattern is specified in EventPattern (tf-compat) (#22) * fix logic and revert CF changes to get tests running again (#21) * add support for EC2 customer gateway API (#25) * add support for EC2 Transit Gateway APIs (#24) * feat(logs): add `kmsKeyId` into `LogGroup` entity (#23) * minor change in ELBv2 logic to fix tests * feat(events): add APIs to describe and delete CloudWatch Events connections (#26) * add support for EC2 transit gateway route tables (#27) * pass transit gateway route table ID in Describe API, minor refactoring (#29) * add support for EC2 Transit Gateway Routes (#28) * fix region on ACM certificate import (#31) * add support for EC2 transit gateway attachments (#30) * add support for EC2 Transit Gateway VPN attachments (#32) * fix account ID for logs API * add support for DeleteOrganization API * feat(events): store raw filter representation for CloudWatch events patterns (tf-compat) (#36) * feat(events): add support to describe/update/delete CloudWatch API destinations (#35) * add Cognito UpdateIdentityPool, CW Logs PutResourcePolicy * feat(events): add support for tags in EventBus API (#38) * fix parameter validation for Batch compute environments (tf-compat) * revert merge conflicts in IMPLEMENTATION_COVERAGE.md * format code using black * restore original README; re-enable and fix CloudFormation tests * restore tests and old logic for CF stack parameters from SSM * parameterize RequestId/RequestID in response messages and revert related test changes * undo LocalStack-specific adaptations * minor fix * Update CodeCov config to reflect removal of Py2 * undo change related to CW metric filtering; add additional test for CW metric statistics with dimensions * Terraform - Extend whitelist of running tests Co-authored-by: acsbendi <acsbendi28@gmail.com> Co-authored-by: Phan Duong <duongpv@outlook.com> Co-authored-by: Thomas Rausch <thomas@thrau.at> Co-authored-by: Macwan Nevil <macnev2013@gmail.com> Co-authored-by: Dominik Schubert <dominik.schubert91@gmail.com> Co-authored-by: Gonzalo Saad <saad.gonzalo.ale@gmail.com> Co-authored-by: Mohit Alonja <monty16597@users.noreply.github.com> Co-authored-by: Miguel Gagliardo <migag9@gmail.com> Co-authored-by: Bert Blommers <info@bertblommers.nl>
2021-07-26 14:21:17 +00:00
@classmethod
@abstractmethod
def create_from_cloudformation_json(
cls, resource_name, cloudformation_json, region_name
):
# This must be implemented as a classmethod with parameters:
# cls, resource_name, cloudformation_json, region_name
# Extract the resource parameters from the cloudformation json
# and return an instance of the resource class
pass
Merge LocalStack changes into upstream moto (#4082) * fix OPTIONS requests on non-existing API GW integrations * add cloudformation models for API Gateway deployments * bump version * add backdoor to return CloudWatch metrics * Updating implementation coverage * Updating implementation coverage * add cloudformation models for API Gateway deployments * Updating implementation coverage * Updating implementation coverage * Implemented get-caller-identity returning real data depending on the access key used. * bump version * minor fixes * fix Number data_type for SQS message attribute * fix handling of encoding errors * bump version * make CF stack queryable before starting to initialize its resources * bump version * fix integration_method for API GW method integrations * fix undefined status in CF FakeStack * Fix apigateway issues with terraform v0.12.21 * resource_methods -> add handle for "DELETE" method * integrations -> fix issue that "httpMethod" wasn't included in body request (this value was set as the value from refer method resource) * bump version * Fix setting http method for API gateway integrations (#6) * bump version * remove duplicate methods * add storage class to S3 Key when completing multipart upload (#7) * fix SQS performance issues; bump version * add pagination to SecretsManager list-secrets (#9) * fix default parameter groups in RDS * fix adding S3 metadata headers with names containing dots (#13) * Updating implementation coverage * Updating implementation coverage * add cloudformation models for API Gateway deployments * Updating implementation coverage * Updating implementation coverage * Implemented get-caller-identity returning real data depending on the access key used. * make CF stack queryable before starting to initialize its resources * bump version * remove duplicate methods * fix adding S3 metadata headers with names containing dots (#13) * Update amis.json to support EKS AMI mocks (#15) * fix PascalCase for boolean value in ListMultipartUploads response (#17); fix _get_multi_param to parse nested list/dict query params * determine non-zero container exit code in Batch API * support filtering by dimensions in CW get_metric_statistics * fix storing attributes for ELBv2 Route entities; API GW refactorings for TF tests * add missing fields for API GW resources * fix error messages for Route53 (TF-compat) * various fixes for IAM resources (tf-compat) * minor fixes for API GW models (tf-compat) * minor fixes for API GW responses (tf-compat) * add s3 exception for bucket notification filter rule validation * change the way RESTErrors generate the response body and content-type header * fix lint errors and disable "black" syntax enforcement * remove return type hint in RESTError.get_body * add RESTError XML template for IAM exceptions * add support for API GW minimumCompressionSize * fix casing getting PrivateDnsEnabled API GW attribute * minor fixes for error responses * fix escaping special chars for IAM role descriptions (tf-compat) * minor fixes and tagging support for API GW and ELB v2 (tf-compat) * Merge branch 'master' into localstack * add "AlarmRule" attribute to enable support for composite CloudWatch metrics * fix recursive parsing of complex/nested query params * bump version * add API to delete S3 website configurations (#18) * use dict copy to allow parallelism and avoid concurrent modification exceptions in S3 * fix precondition check for etags in S3 (#19) * minor fix for user filtering in Cognito * fix API Gateway error response; avoid returning empty response templates (tf-compat) * support tags and tracingEnabled attribute for API GW stages * fix boolean value in S3 encryption response (#20) * fix connection arn structure * fix api destination arn structure * black format * release 2.0.3.37 * fix s3 exception tests see botocore/parsers.py:1002 where RequestId is removed from parsed * remove python 2 from build action * add test failure annotations in build action * fix events test arn comparisons * fix s3 encryption response test * return default value "0" if EC2 availableIpAddressCount is empty * fix extracting SecurityGroupIds for EC2 VPC endpoints * support deleting/updating API Gateway DomainNames * fix(events): Return empty string instead of null when no pattern is specified in EventPattern (tf-compat) (#22) * fix logic and revert CF changes to get tests running again (#21) * add support for EC2 customer gateway API (#25) * add support for EC2 Transit Gateway APIs (#24) * feat(logs): add `kmsKeyId` into `LogGroup` entity (#23) * minor change in ELBv2 logic to fix tests * feat(events): add APIs to describe and delete CloudWatch Events connections (#26) * add support for EC2 transit gateway route tables (#27) * pass transit gateway route table ID in Describe API, minor refactoring (#29) * add support for EC2 Transit Gateway Routes (#28) * fix region on ACM certificate import (#31) * add support for EC2 transit gateway attachments (#30) * add support for EC2 Transit Gateway VPN attachments (#32) * fix account ID for logs API * add support for DeleteOrganization API * feat(events): store raw filter representation for CloudWatch events patterns (tf-compat) (#36) * feat(events): add support to describe/update/delete CloudWatch API destinations (#35) * add Cognito UpdateIdentityPool, CW Logs PutResourcePolicy * feat(events): add support for tags in EventBus API (#38) * fix parameter validation for Batch compute environments (tf-compat) * revert merge conflicts in IMPLEMENTATION_COVERAGE.md * format code using black * restore original README; re-enable and fix CloudFormation tests * restore tests and old logic for CF stack parameters from SSM * parameterize RequestId/RequestID in response messages and revert related test changes * undo LocalStack-specific adaptations * minor fix * Update CodeCov config to reflect removal of Py2 * undo change related to CW metric filtering; add additional test for CW metric statistics with dimensions * Terraform - Extend whitelist of running tests Co-authored-by: acsbendi <acsbendi28@gmail.com> Co-authored-by: Phan Duong <duongpv@outlook.com> Co-authored-by: Thomas Rausch <thomas@thrau.at> Co-authored-by: Macwan Nevil <macnev2013@gmail.com> Co-authored-by: Dominik Schubert <dominik.schubert91@gmail.com> Co-authored-by: Gonzalo Saad <saad.gonzalo.ale@gmail.com> Co-authored-by: Mohit Alonja <monty16597@users.noreply.github.com> Co-authored-by: Miguel Gagliardo <migag9@gmail.com> Co-authored-by: Bert Blommers <info@bertblommers.nl>
2021-07-26 14:21:17 +00:00
@classmethod
@abstractmethod
def update_from_cloudformation_json(
cls, original_resource, new_resource_name, cloudformation_json, region_name
):
# This must be implemented as a classmethod with parameters:
# cls, original_resource, new_resource_name, cloudformation_json, region_name
# Extract the resource parameters from the cloudformation json,
# delete the old resource and return the new one. Optionally inspect
# the change in parameters and no-op when nothing has changed.
pass
Merge LocalStack changes into upstream moto (#4082) * fix OPTIONS requests on non-existing API GW integrations * add cloudformation models for API Gateway deployments * bump version * add backdoor to return CloudWatch metrics * Updating implementation coverage * Updating implementation coverage * add cloudformation models for API Gateway deployments * Updating implementation coverage * Updating implementation coverage * Implemented get-caller-identity returning real data depending on the access key used. * bump version * minor fixes * fix Number data_type for SQS message attribute * fix handling of encoding errors * bump version * make CF stack queryable before starting to initialize its resources * bump version * fix integration_method for API GW method integrations * fix undefined status in CF FakeStack * Fix apigateway issues with terraform v0.12.21 * resource_methods -> add handle for "DELETE" method * integrations -> fix issue that "httpMethod" wasn't included in body request (this value was set as the value from refer method resource) * bump version * Fix setting http method for API gateway integrations (#6) * bump version * remove duplicate methods * add storage class to S3 Key when completing multipart upload (#7) * fix SQS performance issues; bump version * add pagination to SecretsManager list-secrets (#9) * fix default parameter groups in RDS * fix adding S3 metadata headers with names containing dots (#13) * Updating implementation coverage * Updating implementation coverage * add cloudformation models for API Gateway deployments * Updating implementation coverage * Updating implementation coverage * Implemented get-caller-identity returning real data depending on the access key used. * make CF stack queryable before starting to initialize its resources * bump version * remove duplicate methods * fix adding S3 metadata headers with names containing dots (#13) * Update amis.json to support EKS AMI mocks (#15) * fix PascalCase for boolean value in ListMultipartUploads response (#17); fix _get_multi_param to parse nested list/dict query params * determine non-zero container exit code in Batch API * support filtering by dimensions in CW get_metric_statistics * fix storing attributes for ELBv2 Route entities; API GW refactorings for TF tests * add missing fields for API GW resources * fix error messages for Route53 (TF-compat) * various fixes for IAM resources (tf-compat) * minor fixes for API GW models (tf-compat) * minor fixes for API GW responses (tf-compat) * add s3 exception for bucket notification filter rule validation * change the way RESTErrors generate the response body and content-type header * fix lint errors and disable "black" syntax enforcement * remove return type hint in RESTError.get_body * add RESTError XML template for IAM exceptions * add support for API GW minimumCompressionSize * fix casing getting PrivateDnsEnabled API GW attribute * minor fixes for error responses * fix escaping special chars for IAM role descriptions (tf-compat) * minor fixes and tagging support for API GW and ELB v2 (tf-compat) * Merge branch 'master' into localstack * add "AlarmRule" attribute to enable support for composite CloudWatch metrics * fix recursive parsing of complex/nested query params * bump version * add API to delete S3 website configurations (#18) * use dict copy to allow parallelism and avoid concurrent modification exceptions in S3 * fix precondition check for etags in S3 (#19) * minor fix for user filtering in Cognito * fix API Gateway error response; avoid returning empty response templates (tf-compat) * support tags and tracingEnabled attribute for API GW stages * fix boolean value in S3 encryption response (#20) * fix connection arn structure * fix api destination arn structure * black format * release 2.0.3.37 * fix s3 exception tests see botocore/parsers.py:1002 where RequestId is removed from parsed * remove python 2 from build action * add test failure annotations in build action * fix events test arn comparisons * fix s3 encryption response test * return default value "0" if EC2 availableIpAddressCount is empty * fix extracting SecurityGroupIds for EC2 VPC endpoints * support deleting/updating API Gateway DomainNames * fix(events): Return empty string instead of null when no pattern is specified in EventPattern (tf-compat) (#22) * fix logic and revert CF changes to get tests running again (#21) * add support for EC2 customer gateway API (#25) * add support for EC2 Transit Gateway APIs (#24) * feat(logs): add `kmsKeyId` into `LogGroup` entity (#23) * minor change in ELBv2 logic to fix tests * feat(events): add APIs to describe and delete CloudWatch Events connections (#26) * add support for EC2 transit gateway route tables (#27) * pass transit gateway route table ID in Describe API, minor refactoring (#29) * add support for EC2 Transit Gateway Routes (#28) * fix region on ACM certificate import (#31) * add support for EC2 transit gateway attachments (#30) * add support for EC2 Transit Gateway VPN attachments (#32) * fix account ID for logs API * add support for DeleteOrganization API * feat(events): store raw filter representation for CloudWatch events patterns (tf-compat) (#36) * feat(events): add support to describe/update/delete CloudWatch API destinations (#35) * add Cognito UpdateIdentityPool, CW Logs PutResourcePolicy * feat(events): add support for tags in EventBus API (#38) * fix parameter validation for Batch compute environments (tf-compat) * revert merge conflicts in IMPLEMENTATION_COVERAGE.md * format code using black * restore original README; re-enable and fix CloudFormation tests * restore tests and old logic for CF stack parameters from SSM * parameterize RequestId/RequestID in response messages and revert related test changes * undo LocalStack-specific adaptations * minor fix * Update CodeCov config to reflect removal of Py2 * undo change related to CW metric filtering; add additional test for CW metric statistics with dimensions * Terraform - Extend whitelist of running tests Co-authored-by: acsbendi <acsbendi28@gmail.com> Co-authored-by: Phan Duong <duongpv@outlook.com> Co-authored-by: Thomas Rausch <thomas@thrau.at> Co-authored-by: Macwan Nevil <macnev2013@gmail.com> Co-authored-by: Dominik Schubert <dominik.schubert91@gmail.com> Co-authored-by: Gonzalo Saad <saad.gonzalo.ale@gmail.com> Co-authored-by: Mohit Alonja <monty16597@users.noreply.github.com> Co-authored-by: Miguel Gagliardo <migag9@gmail.com> Co-authored-by: Bert Blommers <info@bertblommers.nl>
2021-07-26 14:21:17 +00:00
@classmethod
@abstractmethod
def delete_from_cloudformation_json(
cls, resource_name, cloudformation_json, region_name
):
# This must be implemented as a classmethod with parameters:
# cls, resource_name, cloudformation_json, region_name
# Extract the resource parameters from the cloudformation json
# and delete the resource. Do not include a return statement.
pass
2021-07-26 06:40:39 +00:00
class BaseBackend:
2019-07-11 02:16:11 +00:00
def _reset_model_refs(self):
# Remove all references to the models stored
2017-03-12 04:18:58 +00:00
for service, models in model_data.items():
for model_name, model in models.items():
model.instances = []
2019-07-11 02:16:11 +00:00
def reset(self):
self._reset_model_refs()
2013-02-26 04:48:17 +00:00
self.__dict__ = {}
2013-02-20 03:27:36 +00:00
self.__init__()
2013-02-18 21:09:40 +00:00
@property
2013-03-05 13:14:43 +00:00
def _url_module(self):
2013-02-18 21:09:40 +00:00
backend_module = self.__class__.__module__
backend_urls_module_name = backend_module.replace("models", "urls")
2019-10-31 15:44:26 +00:00
backend_urls_module = __import__(
backend_urls_module_name, fromlist=["url_bases", "url_paths"]
)
2013-03-05 13:14:43 +00:00
return backend_urls_module
@property
def urls(self):
"""
A dictionary of the urls to be mocked with this service and the handlers
that should be called in their place
"""
url_bases = self._url_module.url_bases
unformatted_paths = self._url_module.url_paths
urls = {}
for url_base in url_bases:
# The default URL_base will look like: http://service.[..].amazonaws.com/...
# This extension ensures support for the China regions
cn_url_base = re.sub(r"amazonaws\\?.com$", "amazonaws.com.cn", url_base)
2014-08-26 17:25:50 +00:00
for url_path, handler in unformatted_paths.items():
2013-03-05 13:14:43 +00:00
url = url_path.format(url_base)
urls[url] = handler
cn_url = url_path.format(cn_url_base)
urls[cn_url] = handler
2013-03-05 13:14:43 +00:00
2013-02-18 21:09:40 +00:00
return urls
2013-03-05 13:14:43 +00:00
@property
def url_paths(self):
"""
A dictionary of the paths of the urls to be mocked with this service and
the handlers that should be called in their place
"""
unformatted_paths = self._url_module.url_paths
paths = {}
2014-08-26 17:25:50 +00:00
for unformatted_path, handler in unformatted_paths.items():
2013-03-05 13:14:43 +00:00
path = unformatted_path.format("")
paths[path] = handler
return paths
@property
def url_bases(self):
"""
A list containing the url_bases extracted from urls.py
"""
return self._url_module.url_bases
2013-03-05 13:14:43 +00:00
@property
def flask_paths(self):
"""
The url paths that will be used for the flask server
"""
paths = {}
2014-08-26 17:25:50 +00:00
for url_path, handler in self.url_paths.items():
2013-03-05 13:14:43 +00:00
url_path = convert_regex_to_flask_path(url_path)
paths[url_path] = handler
return paths
@staticmethod
def default_vpc_endpoint_service(
service_region, zones,
): # pylint: disable=unused-argument
"""Invoke the factory method for any VPC endpoint(s) services."""
return None
@staticmethod
def vpce_random_number():
"""Return random number for a VPC endpoint service ID."""
return "".join([random.choice(string.hexdigits.lower()) for i in range(17)])
@staticmethod
def default_vpc_endpoint_service_factory(
service_region,
zones,
service="",
service_type="Interface",
private_dns_names=True,
special_service_name="",
policy_supported=True,
base_endpoint_dns_names=None,
): # pylint: disable=too-many-arguments
"""List of dicts representing default VPC endpoints for this service."""
if special_service_name:
service_name = f"com.amazonaws.{service_region}.{special_service_name}"
else:
service_name = f"com.amazonaws.{service_region}.{service}"
if not base_endpoint_dns_names:
base_endpoint_dns_names = [f"{service}.{service_region}.vpce.amazonaws.com"]
endpoint_service = {
"AcceptanceRequired": False,
"AvailabilityZones": zones,
"BaseEndpointDnsNames": base_endpoint_dns_names,
"ManagesVpcEndpoints": False,
"Owner": "amazon",
"ServiceId": f"vpce-svc-{BaseBackend.vpce_random_number()}",
"ServiceName": service_name,
"ServiceType": [{"ServiceType": service_type}],
"Tags": [],
"VpcEndpointPolicySupported": policy_supported,
}
# Don't know how private DNS names are different, so for now just
# one will be added.
if private_dns_names:
endpoint_service[
"PrivateDnsName"
] = f"{service}.{service_region}.amazonaws.com"
endpoint_service["PrivateDnsNameVerificationState"] = "verified"
endpoint_service["PrivateDnsNames"] = [
{"PrivateDnsName": f"{service}.{service_region}.amazonaws.com"}
]
return [endpoint_service]
2013-02-28 03:25:15 +00:00
def decorator(self, func=None):
if settings.TEST_SERVER_MODE:
2019-10-31 15:44:26 +00:00
mocked_backend = ServerModeMockAWS({"global": self})
else:
2019-10-31 15:44:26 +00:00
mocked_backend = MockAWS({"global": self})
2013-02-28 03:25:15 +00:00
if func:
return mocked_backend(func)
2013-02-28 03:25:15 +00:00
else:
return mocked_backend
2017-02-12 05:22:29 +00:00
2017-02-16 03:35:45 +00:00
def deprecated_decorator(self, func=None):
if func:
2019-10-31 15:44:26 +00:00
return HttprettyMockAWS({"global": self})(func)
2017-02-16 03:35:45 +00:00
else:
2019-10-31 15:44:26 +00:00
return HttprettyMockAWS({"global": self})
2017-02-16 03:35:45 +00:00
# def list_config_service_resources(self, resource_ids, resource_name, limit, next_token):
# """For AWS Config. This will list all of the resources of the given type and optional resource name and region"""
# raise NotImplementedError()
2021-07-26 06:40:39 +00:00
class ConfigQueryModel:
def __init__(self, backends):
"""Inits based on the resource type's backends (1 for each region if applicable)"""
self.backends = backends
2019-10-31 15:44:26 +00:00
def list_config_service_resources(
self,
resource_ids,
resource_name,
limit,
next_token,
backend_region=None,
resource_region=None,
2020-09-17 23:43:19 +00:00
aggregator=None,
2019-10-31 15:44:26 +00:00
):
"""For AWS Config. This will list all of the resources of the given type and optional resource name and region.
This supports both aggregated and non-aggregated listing. The following notes the difference:
- Non-Aggregated Listing -
This only lists resources within a region. The way that this is implemented in moto is based on the region
for the resource backend.
You must set the `backend_region` to the region that the API request arrived from. resource_region can be set to `None`.
- Aggregated Listing -
This lists resources from all potential regional backends. For non-global resource types, this should collect a full
list of resources from all the backends, and then be able to filter from the resource region. This is because an
aggregator can aggregate resources from multiple regions. In moto, aggregated regions will *assume full aggregation
from all resources in all regions for a given resource type*.
The `backend_region` should be set to `None` for these queries, and the `resource_region` should optionally be set to
the `Filters` region parameter to filter out resources that reside in a specific region.
For aggregated listings, pagination logic should be set such that the next page can properly span all the region backends.
As such, the proper way to implement is to first obtain a full list of results from all the region backends, and then filter
from there. It may be valuable to make this a concatenation of the region and resource name.
:param resource_ids: A list of resource IDs
:param resource_name: The individual name of a resource
:param limit: How many per page
:param next_token: The item that will page on
:param backend_region: The region for the backend to pull results from. Set to `None` if this is an aggregated query.
:param resource_region: The region for where the resources reside to pull results from. Set to `None` if this is a
non-aggregated query.
2020-09-17 23:43:19 +00:00
:param aggregator: If the query is an aggregated query, *AND* the resource has "non-standard" aggregation logic (mainly, IAM),
you'll need to pass aggregator used. In most cases, this should be omitted/set to `None`. See the
conditional logic under `if aggregator` in the moto/iam/config.py for the IAM example.
:return: This should return a list of Dicts that have the following fields:
[
{
'type': 'AWS::The AWS Config data type',
'name': 'The name of the resource',
'id': 'The ID of the resource',
'region': 'The region of the resource -- if global, then you may want to have the calling logic pass in the
aggregator region in for the resource region -- or just us-east-1 :P'
}
, ...
]
"""
raise NotImplementedError()
2019-10-31 15:44:26 +00:00
def get_config_resource(
self, resource_id, resource_name=None, backend_region=None, resource_region=None
):
"""For AWS Config. This will query the backend for the specific resource type configuration.
This supports both aggregated, and non-aggregated fetching -- for batched fetching -- the Config batching requests
will call this function N times to fetch the N objects needing to be fetched.
- Non-Aggregated Fetching -
This only fetches a resource config within a region. The way that this is implemented in moto is based on the region
for the resource backend.
You must set the `backend_region` to the region that the API request arrived from. `resource_region` should be set to `None`.
- Aggregated Fetching -
This fetches resources from all potential regional backends. For non-global resource types, this should collect a full
list of resources from all the backends, and then be able to filter from the resource region. This is because an
aggregator can aggregate resources from multiple regions. In moto, aggregated regions will *assume full aggregation
from all resources in all regions for a given resource type*.
...
:param resource_id:
:param resource_name:
:param backend_region:
:param resource_region:
:return:
"""
raise NotImplementedError()
2017-02-12 05:22:29 +00:00
2021-07-26 06:40:39 +00:00
class base_decorator:
2017-02-12 05:22:29 +00:00
mock_backend = MockAWS
def __init__(self, backends):
self.backends = backends
def __call__(self, func=None):
if self.mock_backend != HttprettyMockAWS and settings.TEST_SERVER_MODE:
mocked_backend = ServerModeMockAWS(self.backends)
else:
mocked_backend = self.mock_backend(self.backends)
2017-02-20 23:25:10 +00:00
2017-02-12 05:22:29 +00:00
if func:
return mocked_backend(func)
2017-02-12 05:22:29 +00:00
else:
return mocked_backend
2017-02-16 03:35:45 +00:00
class deprecated_base_decorator(base_decorator):
mock_backend = HttprettyMockAWS
2017-02-20 23:25:10 +00:00
class MotoAPIBackend(BaseBackend):
def reset(self):
2020-01-17 01:08:06 +00:00
import moto.backends as backends
2019-10-31 15:44:26 +00:00
for name, backends_ in backends.loaded_backends():
2017-02-20 23:25:10 +00:00
if name == "moto_api":
continue
2020-01-17 01:08:06 +00:00
for region_name, backend in backends_.items():
backend.reset()
2017-02-20 23:25:10 +00:00
self.__init__()
2017-02-24 02:37:43 +00:00
2017-02-20 23:25:10 +00:00
moto_api_backend = MotoAPIBackend()