From 37bdc12f4d7bd8c47ecffd8049b3855d27c28b4e Mon Sep 17 00:00:00 2001 From: acsbendi Date: Mon, 1 Jul 2019 18:58:31 +0200 Subject: [PATCH 01/36] Fixed linting errors. --- moto/iam/policy_validation.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/moto/iam/policy_validation.py b/moto/iam/policy_validation.py index 91ee79dab..0fcf0e5f2 100644 --- a/moto/iam/policy_validation.py +++ b/moto/iam/policy_validation.py @@ -88,7 +88,7 @@ VALID_RESOURCE_PATH_STARTING_VALUES = { class IAMPolicyDocumentValidator: - def __init__(self, policy_document: str): + def __init__(self, policy_document): self._policy_document: str = policy_document self._policy_json: dict = {} self._statements = [] @@ -308,7 +308,7 @@ class IAMPolicyDocumentValidator: for resource in sorted(statement[key], reverse=True): self._validate_resource_format(resource) if self._resource_error == "": - IAMPolicyDocumentValidator._legacy_parse_resource_like(statement, key) + IAMPolicyDocumentValidator._legacy_parse_resource_like(statement, key) def _validate_resource_format(self, resource): if resource != "*": @@ -327,12 +327,12 @@ class IAMPolicyDocumentValidator: arn3 = remaining_resource_parts[2] if len(remaining_resource_parts) > 2 else "*" arn4 = ":".join(remaining_resource_parts[3:]) if len(remaining_resource_parts) > 3 else "*" self._resource_error = 'Partition "{partition}" is not valid for resource "arn:{partition}:{arn1}:{arn2}:{arn3}:{arn4}".'.format( - partition=resource_partitions[0], - arn1=arn1, - arn2=arn2, - arn3=arn3, - arn4=arn4 - ) + partition=resource_partitions[0], + arn1=arn1, + arn2=arn2, + arn3=arn3, + arn4=arn4 + ) return if resource_partitions[1] != ":": From fbd07498549449b458a2ec2659773ad991b5ade6 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Tue, 2 Jul 2019 17:40:08 +0200 Subject: [PATCH 02/36] Implemented authentication for services except for S3. --- moto/core/authentication.py | 189 ++++++++++++++++++++++++++++++++++++ moto/core/exceptions.py | 31 ++++++ moto/core/responses.py | 50 +++++++--- moto/iam/responses.py | 14 +-- 4 files changed, 266 insertions(+), 18 deletions(-) create mode 100644 moto/core/authentication.py diff --git a/moto/core/authentication.py b/moto/core/authentication.py new file mode 100644 index 000000000..85b553d44 --- /dev/null +++ b/moto/core/authentication.py @@ -0,0 +1,189 @@ +import json +from enum import Enum + +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.credentials import Credentials +from moto.iam.models import ACCOUNT_ID, Policy + +from moto.iam import iam_backend + +from moto.core.exceptions import SignatureDoesNotMatchError, AccessDeniedError, InvalidClientTokenIdError + +ACCESS_KEY_STORE = { + "AKIAJDULPKHCC4KGTYVA": { + "owner": "avatao-user", + "secret_access_key": "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" + } +} + + +class IAMRequest: + + def __init__(self, method, path, data, headers): + print(f"Creating {IAMRequest.__name__} with method={method}, path={path}, data={data}, headers={headers}") + self._method = method + self._path = path + self._data = data + self._headers = headers + credential_scope = self._get_string_between('Credential=', ',', self._headers['Authorization']) + credential_data = credential_scope.split('/') + self._access_key = credential_data[0] + self._region = credential_data[2] + self._service = credential_data[3] + self._action = self._service + ":" + self._data["Action"][0] + + def check_signature(self): + original_signature = self._get_string_between('Signature=', ',', self._headers['Authorization']) + calculated_signature = self._calculate_signature() + if original_signature != calculated_signature: + raise SignatureDoesNotMatchError() + + def check_action_permitted(self): + iam_user_name = ACCESS_KEY_STORE[self._access_key]["owner"] + user_policies = self._collect_policies_for_iam_user(iam_user_name) + + permitted = False + for policy in user_policies: + iam_policy = IAMPolicy(policy) + permission_result = iam_policy.is_action_permitted(self._action) + if permission_result == PermissionResult.DENIED: + self._raise_access_denied(iam_user_name) + elif permission_result == PermissionResult.PERMITTED: + permitted = True + + if not permitted: + self._raise_access_denied(iam_user_name) + + def _raise_access_denied(self, iam_user_name): + raise AccessDeniedError( + account_id=ACCOUNT_ID, + iam_user_name=iam_user_name, + action=self._action + ) + + @staticmethod + def _collect_policies_for_iam_user(iam_user_name): + user_policies = [] + + inline_policy_names = iam_backend.list_user_policies(iam_user_name) + for inline_policy_name in inline_policy_names: + inline_policy = iam_backend.get_user_policy(iam_user_name, inline_policy_name) + user_policies.append(inline_policy) + + attached_policies, _ = iam_backend.list_attached_user_policies(iam_user_name) + user_policies += attached_policies + + user_groups = iam_backend.get_groups_for_user(iam_user_name) + for user_group in user_groups: + inline_group_policy_names = iam_backend.list_group_policies(user_group) + for inline_group_policy_name in inline_group_policy_names: + inline_user_group_policy = iam_backend.get_group_policy(user_group.name, inline_group_policy_name) + user_policies.append(inline_user_group_policy) + + attached_group_policies = iam_backend.list_attached_group_policies(user_group.name) + user_policies += attached_group_policies + + return user_policies + + def _create_auth(self): + if self._access_key not in ACCESS_KEY_STORE: + raise InvalidClientTokenIdError() + secret_key = ACCESS_KEY_STORE[self._access_key]["secret_access_key"] + + credentials = Credentials(self._access_key, secret_key) + return SigV4Auth(credentials, self._service, self._region) + + @staticmethod + def _create_headers_for_aws_request(signed_headers, original_headers): + headers = {} + for key, value in original_headers.items(): + if key.lower() in signed_headers: + headers[key] = value + return headers + + def _create_aws_request(self): + signed_headers = self._get_string_between('SignedHeaders=', ',', self._headers['Authorization']).split(';') + headers = self._create_headers_for_aws_request(signed_headers, self._headers) + request = AWSRequest(method=self._method, url=self._path, data=self._data, headers=headers) + request.context['timestamp'] = headers['X-Amz-Date'] + + return request + + def _calculate_signature(self): + auth = self._create_auth() + request = self._create_aws_request() + canonical_request = auth.canonical_request(request) + string_to_sign = auth.string_to_sign(request, canonical_request) + return auth.signature(string_to_sign, request) + + @staticmethod + def _get_string_between(first_separator, second_separator, string): + return string.partition(first_separator)[2].partition(second_separator)[0] + + +class IAMPolicy: + + def __init__(self, policy): + self._policy = policy + + def is_action_permitted(self, action): + if isinstance(self._policy, Policy): + default_version = next(policy_version for policy_version in self._policy.versions if policy_version.is_default) + policy_document = default_version.document + else: + policy_document = self._policy["policy_document"] + + policy_json = json.loads(policy_document) + + permitted = False + for policy_statement in policy_json["Statement"]: + iam_policy_statement = IAMPolicyStatement(policy_statement) + permission_result = iam_policy_statement.is_action_permitted(action) + if permission_result == PermissionResult.DENIED: + return permission_result + elif permission_result == PermissionResult.PERMITTED: + permitted = True + + if permitted: + return PermissionResult.PERMITTED + else: + return PermissionResult.NEUTRAL + + +class IAMPolicyStatement: + + def __init__(self, statement): + self._statement = statement + + def is_action_permitted(self, action): + is_action_concerned = False + + if "NotAction" in self._statement: + if not self._check_element_contains("NotAction", action): + is_action_concerned = True + else: # Action is present + if self._check_element_contains("Action", action): + is_action_concerned = True + + # TODO: check Resource/NotResource and Condition + + if is_action_concerned: + if self._statement["Effect"] == "Allow": + return PermissionResult.PERMITTED + else: # Deny + return PermissionResult.DENIED + else: + return PermissionResult.NEUTRAL + + def _check_element_contains(self, statement_element, value): + if isinstance(self._statement[statement_element], list): + return value in self._statement[statement_element] + else: # string + return value == self._statement[statement_element] + + +class PermissionResult(Enum): + PERMITTED = 1 + DENIED = 2 + NEUTRAL = 3 diff --git a/moto/core/exceptions.py b/moto/core/exceptions.py index 40202f7bd..ddcce91d5 100644 --- a/moto/core/exceptions.py +++ b/moto/core/exceptions.py @@ -65,3 +65,34 @@ class JsonRESTError(RESTError): def get_body(self, *args, **kwargs): return self.description + + +class SignatureDoesNotMatchError(RESTError): + code = 400 + + def __init__(self): + super(SignatureDoesNotMatchError, self).__init__( + 'SignatureDoesNotMatch', + "The request signature we calculated does not match the signature you provided. Check your AWS Secret Access Key and signing method. Consult the service documentation for details.") + + +class InvalidClientTokenIdError(RESTError): + code = 400 + + def __init__(self): + super(InvalidClientTokenIdError, self).__init__( + 'InvalidClientTokenId', + "The security token included in the request is invalid.") + + +class AccessDeniedError(RESTError): + code = 403 + + def __init__(self, account_id, iam_user_name, action): + super(AccessDeniedError, self).__init__( + 'AccessDenied', + "User: arn:aws:iam::{account_id}:user/{iam_user_name} is not authorized to perform: {operation}".format( + account_id=account_id, + iam_user_name=iam_user_name, + operation=action + )) diff --git a/moto/core/responses.py b/moto/core/responses.py index 9da36b865..4a3b3a1b9 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -8,6 +8,8 @@ import re import io import pytz + +from moto.core.authentication import IAMRequest from moto.core.exceptions import DryRunClientError from jinja2 import Environment, DictLoader, TemplateNotFound @@ -103,7 +105,22 @@ class _TemplateEnvironmentMixin(object): return self.environment.get_template(template_id) -class BaseResponse(_TemplateEnvironmentMixin): +class ActionAuthenticatorMixin(object): + + INITIALIZATION_STEP_COUNT = 5 + request_count = 0 + + def _authenticate_action(self): + iam_request = IAMRequest(method=self.method, path=self.path, data=self.querystring, headers=self.headers) + iam_request.check_signature() + + if ActionAuthenticatorMixin.request_count >= ActionAuthenticatorMixin.INITIALIZATION_STEP_COUNT: + iam_request.check_action_permitted() + else: + ActionAuthenticatorMixin.request_count += 1 + + +class BaseResponse(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): default_region = 'us-east-1' # to extract region, use [^.] @@ -273,6 +290,13 @@ class BaseResponse(_TemplateEnvironmentMixin): def call_action(self): headers = self.response_headers + + try: + self._authenticate_action() + except HTTPException as http_error: + response = http_error.description, dict(status=http_error.code) + return self._send_response(headers, response) + action = camelcase_to_underscores(self._get_action()) method_names = method_names_from_class(self.__class__) if action in method_names: @@ -285,16 +309,7 @@ class BaseResponse(_TemplateEnvironmentMixin): if isinstance(response, six.string_types): return 200, headers, response else: - if len(response) == 2: - body, new_headers = response - else: - status, new_headers, body = response - status = new_headers.get('status', 200) - headers.update(new_headers) - # Cast status to string - if "status" in headers: - headers['status'] = str(headers['status']) - return status, headers, body + return self._send_response(headers, response) if not action: return 404, headers, '' @@ -302,6 +317,19 @@ class BaseResponse(_TemplateEnvironmentMixin): raise NotImplementedError( "The {0} action has not been implemented".format(action)) + @staticmethod + def _send_response(headers, response): + if len(response) == 2: + body, new_headers = response + else: + status, new_headers, body = response + status = new_headers.get('status', 200) + headers.update(new_headers) + # Cast status to string + if "status" in headers: + headers['status'] = str(headers['status']) + return status, headers, body + def _get_param(self, param_name, if_none=None): val = self.querystring.get(param_name) if val is not None: diff --git a/moto/iam/responses.py b/moto/iam/responses.py index a0bcb0b56..62d696ba6 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -1,7 +1,9 @@ from __future__ import unicode_literals from moto.core.responses import BaseResponse -from .models import iam_backend, User +from .models import iam_backend + +AVATAO_USER_NAME = "avatao-user" class IamResponse(BaseResponse): @@ -425,11 +427,10 @@ class IamResponse(BaseResponse): def get_user(self): user_name = self._get_param('UserName') - if user_name: - user = iam_backend.get_user(user_name) - else: - user = User(name='default_user') - # If no user is specific, IAM returns the current user + if not user_name: + user_name = AVATAO_USER_NAME + # If no user is specified, IAM returns the current user + user = iam_backend.get_user(user_name) template = self.response_template(USER_TEMPLATE) return template.render(action='Get', user=user) @@ -457,7 +458,6 @@ class IamResponse(BaseResponse): def create_login_profile(self): user_name = self._get_param('UserName') password = self._get_param('Password') - password = self._get_param('Password') user = iam_backend.create_login_profile(user_name, password) template = self.response_template(CREATE_LOGIN_PROFILE_TEMPLATE) From 6061d5d5215e97d18a958f1d533236986b326735 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Tue, 2 Jul 2019 18:03:00 +0200 Subject: [PATCH 03/36] Introduced environment variable to delay the start of authorization. --- moto/core/responses.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/moto/core/responses.py b/moto/core/responses.py index 4a3b3a1b9..1cc1511e7 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import os from collections import defaultdict import datetime import json @@ -107,14 +108,14 @@ class _TemplateEnvironmentMixin(object): class ActionAuthenticatorMixin(object): - INITIALIZATION_STEP_COUNT = 5 + INITIAL_NO_AUTH_ACTION_COUNT = int(os.environ.get("INITIAL_NO_AUTH_ACTION_COUNT", 999999999)) request_count = 0 def _authenticate_action(self): iam_request = IAMRequest(method=self.method, path=self.path, data=self.querystring, headers=self.headers) iam_request.check_signature() - if ActionAuthenticatorMixin.request_count >= ActionAuthenticatorMixin.INITIALIZATION_STEP_COUNT: + if ActionAuthenticatorMixin.request_count >= ActionAuthenticatorMixin.INITIAL_NO_AUTH_ACTION_COUNT: iam_request.check_action_permitted() else: ActionAuthenticatorMixin.request_count += 1 From 7ec8f85438995fc67f4dbe276b4480d8fc5c40fd Mon Sep 17 00:00:00 2001 From: acsbendi Date: Tue, 2 Jul 2019 19:24:45 +0200 Subject: [PATCH 04/36] Implemented recognizing asterisks in Actions in policy statements. --- moto/core/authentication.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/moto/core/authentication.py b/moto/core/authentication.py index 85b553d44..048cd9a61 100644 --- a/moto/core/authentication.py +++ b/moto/core/authentication.py @@ -1,4 +1,5 @@ import json +import re from enum import Enum from botocore.auth import SigV4Auth @@ -160,10 +161,10 @@ class IAMPolicyStatement: is_action_concerned = False if "NotAction" in self._statement: - if not self._check_element_contains("NotAction", action): + if not self._check_element_matches("NotAction", action): is_action_concerned = True else: # Action is present - if self._check_element_contains("Action", action): + if self._check_element_matches("Action", action): is_action_concerned = True # TODO: check Resource/NotResource and Condition @@ -176,11 +177,20 @@ class IAMPolicyStatement: else: return PermissionResult.NEUTRAL - def _check_element_contains(self, statement_element, value): + def _check_element_matches(self, statement_element, value): if isinstance(self._statement[statement_element], list): - return value in self._statement[statement_element] + for statement_element_value in self._statement[statement_element]: + if self._match(statement_element_value, value): + return True + return False else: # string - return value == self._statement[statement_element] + return self._match(self._statement[statement_element], value) + + @staticmethod + def _match(pattern, string): + pattern = pattern.replace("*", ".*") + pattern = f"^{pattern}$" + return re.match(pattern, string) class PermissionResult(Enum): From 5dbec8aee524e12ec73f839b98d261aadcffae9d Mon Sep 17 00:00:00 2001 From: acsbendi Date: Thu, 4 Jul 2019 16:38:43 +0200 Subject: [PATCH 05/36] Implemented checking if S3 action is permitted. --- moto/core/authentication.py | 55 +++++++--- moto/core/responses.py | 16 ++- moto/s3/exceptions.py | 14 +++ moto/s3/responses.py | 204 ++++++++++++++++++++++++++++++++++-- moto/s3/urls.py | 9 -- 5 files changed, 260 insertions(+), 38 deletions(-) diff --git a/moto/core/authentication.py b/moto/core/authentication.py index 048cd9a61..e8eb50626 100644 --- a/moto/core/authentication.py +++ b/moto/core/authentication.py @@ -1,8 +1,9 @@ import json import re +from abc import ABC, abstractmethod from enum import Enum -from botocore.auth import SigV4Auth +from botocore.auth import SigV4Auth, S3SigV4Auth from botocore.awsrequest import AWSRequest from botocore.credentials import Credentials from moto.iam.models import ACCOUNT_ID, Policy @@ -10,6 +11,7 @@ from moto.iam.models import ACCOUNT_ID, Policy from moto.iam import iam_backend from moto.core.exceptions import SignatureDoesNotMatchError, AccessDeniedError, InvalidClientTokenIdError +from moto.s3.exceptions import BucketAccessDeniedError, S3AccessDeniedError ACCESS_KEY_STORE = { "AKIAJDULPKHCC4KGTYVA": { @@ -19,7 +21,7 @@ ACCESS_KEY_STORE = { } -class IAMRequest: +class IAMRequestBase(ABC): def __init__(self, method, path, data, headers): print(f"Creating {IAMRequest.__name__} with method={method}, path={path}, data={data}, headers={headers}") @@ -56,12 +58,9 @@ class IAMRequest: if not permitted: self._raise_access_denied(iam_user_name) + @abstractmethod def _raise_access_denied(self, iam_user_name): - raise AccessDeniedError( - account_id=ACCOUNT_ID, - iam_user_name=iam_user_name, - action=self._action - ) + pass @staticmethod def _collect_policies_for_iam_user(iam_user_name): @@ -87,13 +86,9 @@ class IAMRequest: return user_policies - def _create_auth(self): - if self._access_key not in ACCESS_KEY_STORE: - raise InvalidClientTokenIdError() - secret_key = ACCESS_KEY_STORE[self._access_key]["secret_access_key"] - - credentials = Credentials(self._access_key, secret_key) - return SigV4Auth(credentials, self._service, self._region) + @abstractmethod + def _create_auth(self, credentials): + pass @staticmethod def _create_headers_for_aws_request(signed_headers, original_headers): @@ -112,7 +107,12 @@ class IAMRequest: return request def _calculate_signature(self): - auth = self._create_auth() + if self._access_key not in ACCESS_KEY_STORE: + raise InvalidClientTokenIdError() + secret_key = ACCESS_KEY_STORE[self._access_key]["secret_access_key"] + + credentials = Credentials(self._access_key, secret_key) + auth = self._create_auth(credentials) request = self._create_aws_request() canonical_request = auth.canonical_request(request) string_to_sign = auth.string_to_sign(request, canonical_request) @@ -123,6 +123,31 @@ class IAMRequest: return string.partition(first_separator)[2].partition(second_separator)[0] +class IAMRequest(IAMRequestBase): + + def _create_auth(self, credentials): + return SigV4Auth(credentials, self._service, self._region) + + def _raise_access_denied(self, iam_user_name): + raise AccessDeniedError( + account_id=ACCOUNT_ID, + iam_user_name=iam_user_name, + action=self._action + ) + + +class S3IAMRequest(IAMRequestBase): + + def _create_auth(self, credentials): + return S3SigV4Auth(credentials, self._service, self._region) + + def _raise_access_denied(self, _): + if "BucketName" in self._data: + raise BucketAccessDeniedError(bucket=self._data["BucketName"]) + else: + raise S3AccessDeniedError() + + class IAMPolicy: def __init__(self, policy): diff --git a/moto/core/responses.py b/moto/core/responses.py index 1cc1511e7..5c59a26e7 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -10,7 +10,7 @@ import io import pytz -from moto.core.authentication import IAMRequest +from moto.core.authentication import IAMRequest, S3IAMRequest from moto.core.exceptions import DryRunClientError from jinja2 import Environment, DictLoader, TemplateNotFound @@ -111,8 +111,7 @@ class ActionAuthenticatorMixin(object): INITIAL_NO_AUTH_ACTION_COUNT = int(os.environ.get("INITIAL_NO_AUTH_ACTION_COUNT", 999999999)) request_count = 0 - def _authenticate_action(self): - iam_request = IAMRequest(method=self.method, path=self.path, data=self.querystring, headers=self.headers) + def _authenticate_action(self, iam_request): iam_request.check_signature() if ActionAuthenticatorMixin.request_count >= ActionAuthenticatorMixin.INITIAL_NO_AUTH_ACTION_COUNT: @@ -120,6 +119,14 @@ class ActionAuthenticatorMixin(object): else: ActionAuthenticatorMixin.request_count += 1 + def _authenticate_normal_action(self): + iam_request = IAMRequest(method=self.method, path=self.path, data=self.data, headers=self.headers) + self._authenticate_action(iam_request) + + def _authenticate_s3_action(self): + iam_request = S3IAMRequest(method=self.method, path=self.path, data=self.data, headers=self.headers) + self._authenticate_action(iam_request) + class BaseResponse(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): @@ -185,6 +192,7 @@ class BaseResponse(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): self.uri = full_url self.path = urlparse(full_url).path self.querystring = querystring + self.data = querystring self.method = request.method self.region = self.get_region_from_url(request, full_url) self.uri_match = None @@ -293,7 +301,7 @@ class BaseResponse(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): headers = self.response_headers try: - self._authenticate_action() + self._authenticate_normal_action() except HTTPException as http_error: response = http_error.description, dict(status=http_error.code) return self._send_response(headers, response) diff --git a/moto/s3/exceptions.py b/moto/s3/exceptions.py index 27c842111..6f4c9c996 100644 --- a/moto/s3/exceptions.py +++ b/moto/s3/exceptions.py @@ -199,3 +199,17 @@ class DuplicateTagKeys(S3ClientError): "InvalidTag", "Cannot provide multiple Tags with the same key", *args, **kwargs) + + +class S3AccessDeniedError(S3ClientError): + code = 403 + + def __init__(self, *args, **kwargs): + super(S3AccessDeniedError, self).__init__('AccessDenied', 'Access Denied', *args, **kwargs) + + +class BucketAccessDeniedError(BucketError): + code = 403 + + def __init__(self, *args, **kwargs): + super(BucketAccessDeniedError, self).__init__('AccessDenied', 'Access Denied', *args, **kwargs) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index e03666666..245f6db4e 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -3,13 +3,15 @@ from __future__ import unicode_literals import re import six +from werkzeug.exceptions import HTTPException + from moto.core.utils import str_to_rfc_1123_datetime from six.moves.urllib.parse import parse_qs, urlparse, unquote import xmltodict from moto.packages.httpretty.core import HTTPrettyRequest -from moto.core.responses import _TemplateEnvironmentMixin +from moto.core.responses import _TemplateEnvironmentMixin, ActionAuthenticatorMixin from moto.core.utils import path_url from moto.s3bucket_path.utils import bucket_name_from_url as bucketpath_bucket_name_from_url, \ @@ -25,6 +27,72 @@ from xml.dom import minidom DEFAULT_REGION_NAME = 'us-east-1' +ACTION_MAP = { + "BUCKET": { + "GET": { + "uploads": "ListBucketMultipartUploads", + "location": "GetBucketLocation", + "lifecycle": "GetLifecycleConfiguration", + "versioning": "GetBucketVersioning", + "policy": "GetBucketPolicy", + "website": "GetBucketWebsite", + "acl": "GetBucketAcl", + "tagging": "GetBucketTagging", + "logging": "GetBucketLogging", + "cors": "GetBucketCORS", + "notification": "GetBucketNotification", + "accelerate": "GetAccelerateConfiguration", + "versions": "ListBucketVersions", + "DEFAULT": "ListBucket" + }, + "PUT": { + "lifecycle": "PutLifecycleConfiguration", + "versioning": "PutBucketVersioning", + "policy": "PutBucketPolicy", + "website": "PutBucketWebsite", + "acl": "PutBucketAcl", + "tagging": "PutBucketTagging", + "logging": "PutBucketLogging", + "cors": "PutBucketCORS", + "notification": "PutBucketNotification", + "accelerate": "PutAccelerateConfiguration", + "DEFAULT": "CreateBucket" + }, + "DELETE": { + "lifecycle": "PutLifecycleConfiguration", + "policy": "DeleteBucketPolicy", + "tagging": "PutBucketTagging", + "cors": "PutBucketCORS", + "DEFAULT": "DeleteBucket" + } + }, + "KEY": { + "GET": { + "uploadId": "ListMultipartUploadParts", + "acl": "GetObjectAcl", + "tagging": "GetObjectTagging", + "versionId": "GetObjectVersion", + "DEFAULT": "GetObject" + }, + "PUT": { + "acl": "PutObjectAcl", + "tagging": "PutObjectTagging", + "DEFAULT": "PutObject" + }, + "DELETE": { + "uploadId": "AbortMultipartUpload", + "versionId": "DeleteObjectVersion", + "DEFAULT": " DeleteObject" + }, + "POST": { + "uploads": "PutObject", + "restore": "RestoreObject", + "uploadId": "PutObject" + } + } + +} + def parse_key_name(pth): return pth.lstrip("/") @@ -37,17 +105,27 @@ def is_delete_keys(request, path, bucket_name): ) -class ResponseObject(_TemplateEnvironmentMixin): +class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): def __init__(self, backend): super(ResponseObject, self).__init__() self.backend = backend + self.method = "" + self.path = "" + self.data = {} + self.headers = {} @property def should_autoescape(self): return True - def all_buckets(self): + def all_buckets(self, headers): + try: + self.data["Action"] = "ListAllMyBuckets" + self._authenticate_s3_action() + except HTTPException as http_error: + response = http_error.code, headers, http_error.description + return self._send_response(response) # No bucket specified. Listing all buckets all_buckets = self.backend.get_all_buckets() template = self.response_template(S3_ALL_BUCKETS) @@ -112,11 +190,18 @@ class ResponseObject(_TemplateEnvironmentMixin): return self.bucket_response(request, full_url, headers) def bucket_response(self, request, full_url, headers): + self.method = request.method + self.path = self._get_path(request) + self.headers = request.headers try: response = self._bucket_response(request, full_url, headers) except S3ClientError as s3error: response = s3error.code, {}, s3error.description + return self._send_response(response) + + @staticmethod + def _send_response(response): if isinstance(response, six.string_types): return 200, {}, response.encode("utf-8") else: @@ -124,18 +209,21 @@ class ResponseObject(_TemplateEnvironmentMixin): if not isinstance(response_content, six.binary_type): response_content = response_content.encode("utf-8") + print(f"response_content: {response_content}") + return status_code, headers, response_content def _bucket_response(self, request, full_url, headers): - parsed_url = urlparse(full_url) - querystring = parse_qs(parsed_url.query, keep_blank_values=True) + querystring = self._get_querystring(full_url) method = request.method region_name = parse_region_from_url(full_url) bucket_name = self.parse_bucket_name_from_url(request, full_url) if not bucket_name: # If no bucket specified, list all buckets - return self.all_buckets() + return self.all_buckets(headers) + + self.data["BucketName"] = bucket_name if hasattr(request, 'body'): # Boto @@ -163,6 +251,12 @@ class ResponseObject(_TemplateEnvironmentMixin): raise NotImplementedError( "Method {0} has not been impelemented in the S3 backend yet".format(method)) + @staticmethod + def _get_querystring(full_url): + parsed_url = urlparse(full_url) + querystring = parse_qs(parsed_url.query, keep_blank_values=True) + return querystring + def _bucket_response_head(self, bucket_name, headers): try: self.backend.get_bucket(bucket_name) @@ -175,6 +269,14 @@ class ResponseObject(_TemplateEnvironmentMixin): return 200, {}, "" def _bucket_response_get(self, bucket_name, querystring, headers): + self._set_action("BUCKET", "GET", querystring) + + try: + self._authenticate_s3_action() + except HTTPException as http_error: + response = http_error.code, headers, http_error.description + return self._send_response(response) + if 'uploads' in querystring: for unsup in ('delimiter', 'max-uploads'): if unsup in querystring: @@ -333,6 +435,15 @@ class ResponseObject(_TemplateEnvironmentMixin): max_keys=max_keys ) + def _set_action(self, action_resource_type, method, querystring): + action_set = False + for action_in_querystring, action in ACTION_MAP[action_resource_type][method].items(): + if action_in_querystring in querystring: + self.data["Action"] = action + action_set = True + if not action_set: + self.data["Action"] = ACTION_MAP[action_resource_type][method]["DEFAULT"] + def _handle_list_objects_v2(self, bucket_name, querystring): template = self.response_template(S3_BUCKET_GET_RESPONSE_V2) bucket = self.backend.get_bucket(bucket_name) @@ -396,6 +507,15 @@ class ResponseObject(_TemplateEnvironmentMixin): def _bucket_response_put(self, request, body, region_name, bucket_name, querystring, headers): if not request.headers.get('Content-Length'): return 411, {}, "Content-Length required" + + self._set_action("BUCKET", "PUT", querystring) + + try: + self._authenticate_s3_action() + except HTTPException as http_error: + response = http_error.code, headers, http_error.description + return self._send_response(response) + if 'versioning' in querystring: ver = re.search('([A-Za-z]+)', body.decode()) if ver: @@ -495,6 +615,14 @@ class ResponseObject(_TemplateEnvironmentMixin): return 200, {}, template.render(bucket=new_bucket) def _bucket_response_delete(self, body, bucket_name, querystring, headers): + self._set_action("BUCKET", "DELETE", querystring) + + try: + self._authenticate_s3_action() + except HTTPException as http_error: + response = http_error.code, headers, http_error.description + return self._send_response(response) + if 'policy' in querystring: self.backend.delete_bucket_policy(bucket_name, body) return 204, {}, "" @@ -525,14 +653,27 @@ class ResponseObject(_TemplateEnvironmentMixin): if not request.headers.get('Content-Length'): return 411, {}, "Content-Length required" - if isinstance(request, HTTPrettyRequest): - path = request.path - else: - path = request.full_path if hasattr(request, 'full_path') else path_url(request.url) + path = self._get_path(request) if self.is_delete_keys(request, path, bucket_name): + self.data["Action"] = "DeleteObject" + + try: + self._authenticate_s3_action() + except HTTPException as http_error: + response = http_error.code, headers, http_error.description + return self._send_response(response) + return self._bucket_response_delete_keys(request, body, bucket_name, headers) + self.data["Action"] = "PutObject" + + try: + self._authenticate_s3_action() + except HTTPException as http_error: + response = http_error.code, headers, http_error.description + return self._send_response(response) + # POST to bucket-url should create file from form if hasattr(request, 'form'): # Not HTTPretty @@ -560,6 +701,14 @@ class ResponseObject(_TemplateEnvironmentMixin): return 200, {}, "" + @staticmethod + def _get_path(request): + if isinstance(request, HTTPrettyRequest): + path = request.path + else: + path = request.full_path if hasattr(request, 'full_path') else path_url(request.url) + return path + def _bucket_response_delete_keys(self, request, body, bucket_name, headers): template = self.response_template(S3_DELETE_KEYS_RESPONSE) @@ -604,6 +753,9 @@ class ResponseObject(_TemplateEnvironmentMixin): return 206, response_headers, response_content[begin:end + 1] def key_response(self, request, full_url, headers): + self.method = request.method + self.path = self._get_path(request) + self.headers = request.headers response_headers = {} try: response = self._key_response(request, full_url, headers) @@ -671,6 +823,14 @@ class ResponseObject(_TemplateEnvironmentMixin): "Method {0} has not been implemented in the S3 backend yet".format(method)) def _key_response_get(self, bucket_name, query, key_name, headers): + self._set_action("KEY", "GET", query) + + try: + self._authenticate_s3_action() + except HTTPException as http_error: + response = http_error.code, headers, http_error.description + return self._send_response(response) + response_headers = {} if query.get('uploadId'): upload_id = query['uploadId'][0] @@ -700,6 +860,14 @@ class ResponseObject(_TemplateEnvironmentMixin): return 200, response_headers, key.value def _key_response_put(self, request, body, bucket_name, query, key_name, headers): + self._set_action("KEY", "PUT", query) + + try: + self._authenticate_s3_action() + except HTTPException as http_error: + response = http_error.code, headers, http_error.description + return self._send_response(response) + response_headers = {} if query.get('uploadId') and query.get('partNumber'): upload_id = query['uploadId'][0] @@ -1067,6 +1235,14 @@ class ResponseObject(_TemplateEnvironmentMixin): return config['Status'] def _key_response_delete(self, bucket_name, query, key_name, headers): + self._set_action("KEY", "DELETE", query) + + try: + self._authenticate_s3_action() + except HTTPException as http_error: + response = http_error.code, headers, http_error.description + return self._send_response(response) + if query.get('uploadId'): upload_id = query['uploadId'][0] self.backend.cancel_multipart(bucket_name, upload_id) @@ -1087,6 +1263,14 @@ class ResponseObject(_TemplateEnvironmentMixin): yield (pn, p.getElementsByTagName('ETag')[0].firstChild.wholeText) def _key_response_post(self, request, body, bucket_name, query, key_name, headers): + self._set_action("KEY", "POST", query) + + try: + self._authenticate_s3_action() + except HTTPException as http_error: + response = http_error.code, headers, http_error.description + return self._send_response(response) + if body == b'' and 'uploads' in query: metadata = metadata_from_headers(request.headers) multipart = self.backend.initiate_multipart( diff --git a/moto/s3/urls.py b/moto/s3/urls.py index 1d439a549..fa81568a4 100644 --- a/moto/s3/urls.py +++ b/moto/s3/urls.py @@ -7,15 +7,6 @@ url_bases = [ r"https?://(?P[a-zA-Z0-9\-_.]*)\.?s3(.*).amazonaws.com" ] - -def ambiguous_response1(*args, **kwargs): - return S3ResponseInstance.ambiguous_response(*args, **kwargs) - - -def ambiguous_response2(*args, **kwargs): - return S3ResponseInstance.ambiguous_response(*args, **kwargs) - - url_paths = { # subdomain bucket '{0}/$': S3ResponseInstance.bucket_response, From 86758182a7aa5736c1f9c6b29633143bfdb352b7 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Thu, 4 Jul 2019 16:42:11 +0200 Subject: [PATCH 06/36] Removed print. --- moto/s3/responses.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 245f6db4e..46d811f81 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -209,8 +209,6 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): if not isinstance(response_content, six.binary_type): response_content = response_content.encode("utf-8") - print(f"response_content: {response_content}") - return status_code, headers, response_content def _bucket_response(self, request, full_url, headers): From 8de3bdcf298f9d21e311cca0c46e75404133db85 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Thu, 4 Jul 2019 16:48:44 +0200 Subject: [PATCH 07/36] Fixed printing IAM request class' name. --- moto/core/authentication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/core/authentication.py b/moto/core/authentication.py index e8eb50626..96247970b 100644 --- a/moto/core/authentication.py +++ b/moto/core/authentication.py @@ -24,7 +24,7 @@ ACCESS_KEY_STORE = { class IAMRequestBase(ABC): def __init__(self, method, path, data, headers): - print(f"Creating {IAMRequest.__name__} with method={method}, path={path}, data={data}, headers={headers}") + print(f"Creating {self.__class__.__name__} with method={method}, path={path}, data={data}, headers={headers}") self._method = method self._path = path self._data = data From 9684e1b638b96e15ef4b2efc529e74c67b008f04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bendeg=C3=BAz=20=C3=81cs?= <30595431+acsbendi@users.noreply.github.com> Date: Thu, 4 Jul 2019 17:18:12 +0200 Subject: [PATCH 08/36] Abstract methods raise NotImplementedError --- moto/core/authentication.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/core/authentication.py b/moto/core/authentication.py index 96247970b..9ef687bad 100644 --- a/moto/core/authentication.py +++ b/moto/core/authentication.py @@ -60,7 +60,7 @@ class IAMRequestBase(ABC): @abstractmethod def _raise_access_denied(self, iam_user_name): - pass + raise NotImplementedError() @staticmethod def _collect_policies_for_iam_user(iam_user_name): @@ -88,7 +88,7 @@ class IAMRequestBase(ABC): @abstractmethod def _create_auth(self, credentials): - pass + raise NotImplementedError() @staticmethod def _create_headers_for_aws_request(signed_headers, original_headers): From 23957fe94091273ebee6d836d30be298d1d323e2 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Mon, 8 Jul 2019 19:57:14 +0200 Subject: [PATCH 09/36] Implemented finding credentials from already created IAM users and roles. --- moto/core/authentication.py | 191 +++++++++++++++++++++++++++--------- moto/core/exceptions.py | 26 ++++- moto/core/responses.py | 12 +-- moto/s3/exceptions.py | 32 ++++++ moto/sts/models.py | 4 + 5 files changed, 206 insertions(+), 59 deletions(-) diff --git a/moto/core/authentication.py b/moto/core/authentication.py index 9ef687bad..953eed186 100644 --- a/moto/core/authentication.py +++ b/moto/core/authentication.py @@ -6,19 +6,115 @@ from enum import Enum from botocore.auth import SigV4Auth, S3SigV4Auth from botocore.awsrequest import AWSRequest from botocore.credentials import Credentials + from moto.iam.models import ACCOUNT_ID, Policy - from moto.iam import iam_backend +from moto.core.exceptions import SignatureDoesNotMatchError, AccessDeniedError, InvalidClientTokenIdError, InvalidAccessKeyIdError, AuthFailureError +from moto.s3.exceptions import BucketAccessDeniedError, S3AccessDeniedError, BucketInvalidTokenError, S3InvalidTokenError, S3InvalidAccessKeyIdError, BucketInvalidAccessKeyIdError +from moto.sts import sts_backend -from moto.core.exceptions import SignatureDoesNotMatchError, AccessDeniedError, InvalidClientTokenIdError -from moto.s3.exceptions import BucketAccessDeniedError, S3AccessDeniedError -ACCESS_KEY_STORE = { - "AKIAJDULPKHCC4KGTYVA": { - "owner": "avatao-user", - "secret_access_key": "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" - } -} +def create_access_key(access_key_id, headers): + if access_key_id.startswith("AKIA") or "X-Amz-Security-Token" not in headers: + return IAMUserAccessKey(access_key_id, headers) + else: + return AssumedRoleAccessKey(access_key_id, headers) + + +class IAMUserAccessKey: + + def __init__(self, access_key_id, headers): + iam_users = iam_backend.list_users('/', None, None) + for iam_user in iam_users: + for access_key in iam_user.access_keys: + if access_key.access_key_id == access_key_id: + self._owner_user_name = iam_user.name + self._access_key_id = access_key_id + self._secret_access_key = access_key.secret_access_key + if "X-Amz-Security-Token" in headers: + raise CreateAccessKeyFailure(reason="InvalidToken") + return + raise CreateAccessKeyFailure(reason="InvalidId") + + @property + def arn(self): + return "arn:aws:iam::{account_id}:user/{iam_user_name}".format( + account_id=ACCOUNT_ID, + iam_user_name=self._owner_user_name + ) + + def create_credentials(self): + return Credentials(self._access_key_id, self._secret_access_key) + + def collect_policies(self): + user_policies = [] + + inline_policy_names = iam_backend.list_user_policies(self._owner_user_name) + for inline_policy_name in inline_policy_names: + inline_policy = iam_backend.get_user_policy(self._owner_user_name, inline_policy_name) + user_policies.append(inline_policy) + + attached_policies, _ = iam_backend.list_attached_user_policies(self._owner_user_name) + user_policies += attached_policies + + user_groups = iam_backend.get_groups_for_user(self._owner_user_name) + for user_group in user_groups: + inline_group_policy_names = iam_backend.list_group_policies(user_group) + for inline_group_policy_name in inline_group_policy_names: + inline_user_group_policy = iam_backend.get_group_policy(user_group.name, inline_group_policy_name) + user_policies.append(inline_user_group_policy) + + attached_group_policies = iam_backend.list_attached_group_policies(user_group.name) + user_policies += attached_group_policies + + return user_policies + + +class AssumedRoleAccessKey: + + def __init__(self, access_key_id, headers): + for assumed_role in sts_backend.assumed_roles: + if assumed_role.access_key_id == access_key_id: + self._access_key_id = access_key_id + self._secret_access_key = assumed_role.secret_access_key + self._session_token = assumed_role.session_token + self._owner_role_name = assumed_role.arn.split("/")[-1] + self._session_name = assumed_role.session_name + if headers["X-Amz-Security-Token"] != self._session_name: + raise CreateAccessKeyFailure(reason="InvalidToken") + return + raise CreateAccessKeyFailure(reason="InvalidId") + + @property + def arn(self): + return "arn:aws:sts::{account_id}:assumed-role/{role_name}/{session_name}".format( + account_id=ACCOUNT_ID, + role_name=self._owner_role_name, + session_name=self._session_name + ) + + def create_credentials(self): + return Credentials(self._access_key_id, self._secret_access_key, self._session_token) + + def collect_policies(self): + role_policies = [] + + inline_policy_names = iam_backend.list_role_policies(self._owner_role_name) + for inline_policy_name in inline_policy_names: + inline_policy = iam_backend.get_role_policy(self._owner_role_name, inline_policy_name) + role_policies.append(inline_policy) + + attached_policies, _ = iam_backend.list_attached_role_policies(self._owner_role_name) + role_policies += attached_policies + + return role_policies + + +class CreateAccessKeyFailure(Exception): + + def __init__(self, reason, *args): + super().__init__(*args) + self.reason = reason class IAMRequestBase(ABC): @@ -31,10 +127,13 @@ class IAMRequestBase(ABC): self._headers = headers credential_scope = self._get_string_between('Credential=', ',', self._headers['Authorization']) credential_data = credential_scope.split('/') - self._access_key = credential_data[0] self._region = credential_data[2] self._service = credential_data[3] self._action = self._service + ":" + self._data["Action"][0] + try: + self._access_key = create_access_key(access_key_id=credential_data[0], headers=headers) + except CreateAccessKeyFailure as e: + self._raise_invalid_access_key(e.reason) def check_signature(self): original_signature = self._get_string_between('Signature=', ',', self._headers['Authorization']) @@ -43,48 +142,30 @@ class IAMRequestBase(ABC): raise SignatureDoesNotMatchError() def check_action_permitted(self): - iam_user_name = ACCESS_KEY_STORE[self._access_key]["owner"] - user_policies = self._collect_policies_for_iam_user(iam_user_name) + self._check_action_permitted_for_iam_user() + + def _check_action_permitted_for_iam_user(self): + policies = self._access_key.collect_policies() permitted = False - for policy in user_policies: + for policy in policies: iam_policy = IAMPolicy(policy) permission_result = iam_policy.is_action_permitted(self._action) if permission_result == PermissionResult.DENIED: - self._raise_access_denied(iam_user_name) + self._raise_access_denied() elif permission_result == PermissionResult.PERMITTED: permitted = True if not permitted: - self._raise_access_denied(iam_user_name) + self._raise_access_denied() @abstractmethod - def _raise_access_denied(self, iam_user_name): + def _raise_access_denied(self): raise NotImplementedError() - @staticmethod - def _collect_policies_for_iam_user(iam_user_name): - user_policies = [] - - inline_policy_names = iam_backend.list_user_policies(iam_user_name) - for inline_policy_name in inline_policy_names: - inline_policy = iam_backend.get_user_policy(iam_user_name, inline_policy_name) - user_policies.append(inline_policy) - - attached_policies, _ = iam_backend.list_attached_user_policies(iam_user_name) - user_policies += attached_policies - - user_groups = iam_backend.get_groups_for_user(iam_user_name) - for user_group in user_groups: - inline_group_policy_names = iam_backend.list_group_policies(user_group) - for inline_group_policy_name in inline_group_policy_names: - inline_user_group_policy = iam_backend.get_group_policy(user_group.name, inline_group_policy_name) - user_policies.append(inline_user_group_policy) - - attached_group_policies = iam_backend.list_attached_group_policies(user_group.name) - user_policies += attached_group_policies - - return user_policies + @abstractmethod + def _raise_invalid_access_key(self, reason): + raise NotImplementedError() @abstractmethod def _create_auth(self, credentials): @@ -107,11 +188,7 @@ class IAMRequestBase(ABC): return request def _calculate_signature(self): - if self._access_key not in ACCESS_KEY_STORE: - raise InvalidClientTokenIdError() - secret_key = ACCESS_KEY_STORE[self._access_key]["secret_access_key"] - - credentials = Credentials(self._access_key, secret_key) + credentials = self._access_key.create_credentials() auth = self._create_auth(credentials) request = self._create_aws_request() canonical_request = auth.canonical_request(request) @@ -125,23 +202,41 @@ class IAMRequestBase(ABC): class IAMRequest(IAMRequestBase): + def _raise_invalid_access_key(self, _): + if self._service == "ec2": + raise AuthFailureError() + else: + raise InvalidClientTokenIdError() + def _create_auth(self, credentials): return SigV4Auth(credentials, self._service, self._region) - def _raise_access_denied(self, iam_user_name): + def _raise_access_denied(self): raise AccessDeniedError( - account_id=ACCOUNT_ID, - iam_user_name=iam_user_name, + user_arn=self._access_key.arn, action=self._action ) class S3IAMRequest(IAMRequestBase): + def _raise_invalid_access_key(self, reason): + + if reason == "InvalidToken": + if "BucketName" in self._data: + raise BucketInvalidTokenError(bucket=self._data["BucketName"]) + else: + raise S3InvalidTokenError() + else: + if "BucketName" in self._data: + raise BucketInvalidAccessKeyIdError(bucket=self._data["BucketName"]) + else: + raise S3InvalidAccessKeyIdError() + def _create_auth(self, credentials): return S3SigV4Auth(credentials, self._service, self._region) - def _raise_access_denied(self, _): + def _raise_access_denied(self): if "BucketName" in self._data: raise BucketAccessDeniedError(bucket=self._data["BucketName"]) else: diff --git a/moto/core/exceptions.py b/moto/core/exceptions.py index ddcce91d5..e18797fa4 100644 --- a/moto/core/exceptions.py +++ b/moto/core/exceptions.py @@ -88,11 +88,29 @@ class InvalidClientTokenIdError(RESTError): class AccessDeniedError(RESTError): code = 403 - def __init__(self, account_id, iam_user_name, action): + def __init__(self, user_arn, action): super(AccessDeniedError, self).__init__( 'AccessDenied', - "User: arn:aws:iam::{account_id}:user/{iam_user_name} is not authorized to perform: {operation}".format( - account_id=account_id, - iam_user_name=iam_user_name, + "User: {user_arn} is not authorized to perform: {operation}".format( + user_arn=user_arn, operation=action )) + + +class InvalidAccessKeyIdError(RESTError): + code = 400 + + def __init__(self): + super(InvalidAccessKeyIdError, self).__init__( + 'InvalidAccessKeyId', + "The AWS Access Key Id you provided does not exist in our records.") + + +class AuthFailureError(RESTError): + code = 400 + + def __init__(self): + super(AuthFailureError, self).__init__( + 'AuthFailure', + "AWS was not able to validate the provided access credentials") + diff --git a/moto/core/responses.py b/moto/core/responses.py index 5c59a26e7..4165732ab 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -111,21 +111,19 @@ class ActionAuthenticatorMixin(object): INITIAL_NO_AUTH_ACTION_COUNT = int(os.environ.get("INITIAL_NO_AUTH_ACTION_COUNT", 999999999)) request_count = 0 - def _authenticate_action(self, iam_request): - iam_request.check_signature() - + def _authenticate_action(self, iam_request_cls): if ActionAuthenticatorMixin.request_count >= ActionAuthenticatorMixin.INITIAL_NO_AUTH_ACTION_COUNT: + iam_request = iam_request_cls(method=self.method, path=self.path, data=self.data, headers=self.headers) + iam_request.check_signature() iam_request.check_action_permitted() else: ActionAuthenticatorMixin.request_count += 1 def _authenticate_normal_action(self): - iam_request = IAMRequest(method=self.method, path=self.path, data=self.data, headers=self.headers) - self._authenticate_action(iam_request) + self._authenticate_action(IAMRequest) def _authenticate_s3_action(self): - iam_request = S3IAMRequest(method=self.method, path=self.path, data=self.data, headers=self.headers) - self._authenticate_action(iam_request) + self._authenticate_action(S3IAMRequest) class BaseResponse(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): diff --git a/moto/s3/exceptions.py b/moto/s3/exceptions.py index 6f4c9c996..579d95148 100644 --- a/moto/s3/exceptions.py +++ b/moto/s3/exceptions.py @@ -213,3 +213,35 @@ class BucketAccessDeniedError(BucketError): def __init__(self, *args, **kwargs): super(BucketAccessDeniedError, self).__init__('AccessDenied', 'Access Denied', *args, **kwargs) + + +class S3InvalidTokenError(S3ClientError): + code = 400 + + def __init__(self, *args, **kwargs): + super(S3InvalidTokenError, self).__init__('InvalidToken', 'The provided token is malformed or otherwise invalid.', *args, **kwargs) + + +class BucketInvalidTokenError(BucketError): + code = 400 + + def __init__(self, *args, **kwargs): + super(BucketInvalidTokenError, self).__init__('InvalidToken', 'The provided token is malformed or otherwise invalid.', *args, **kwargs) + + +class S3InvalidAccessKeyIdError(S3ClientError): + code = 400 + + def __init__(self, *args, **kwargs): + super(S3InvalidAccessKeyIdError, self).__init__( + 'InvalidAccessKeyId', + "The AWS Access Key Id you provided does not exist in our records.", *args, **kwargs) + + +class BucketInvalidAccessKeyIdError(S3ClientError): + code = 400 + + def __init__(self, *args, **kwargs): + super(BucketInvalidAccessKeyIdError, self).__init__( + 'InvalidAccessKeyId', + "The AWS Access Key Id you provided does not exist in our records.", *args, **kwargs) diff --git a/moto/sts/models.py b/moto/sts/models.py index 983db5602..e437575bf 100644 --- a/moto/sts/models.py +++ b/moto/sts/models.py @@ -38,6 +38,9 @@ class AssumedRole(BaseModel): class STSBackend(BaseBackend): + def __init__(self): + self.assumed_roles = [] + def get_session_token(self, duration): token = Token(duration=duration) return token @@ -48,6 +51,7 @@ class STSBackend(BaseBackend): def assume_role(self, **kwargs): role = AssumedRole(**kwargs) + self.assumed_roles.append(role) return role From 1df4e8da2f590bd74831836c43b87d836607cb85 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Tue, 9 Jul 2019 19:44:23 +0200 Subject: [PATCH 10/36] Fixed bugs in processing policies belonging to assumed roles. --- moto/core/authentication.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/moto/core/authentication.py b/moto/core/authentication.py index 953eed186..88e1b6b42 100644 --- a/moto/core/authentication.py +++ b/moto/core/authentication.py @@ -6,6 +6,7 @@ from enum import Enum from botocore.auth import SigV4Auth, S3SigV4Auth from botocore.awsrequest import AWSRequest from botocore.credentials import Credentials +from six import string_types from moto.iam.models import ACCOUNT_ID, Policy from moto.iam import iam_backend @@ -80,7 +81,7 @@ class AssumedRoleAccessKey: self._session_token = assumed_role.session_token self._owner_role_name = assumed_role.arn.split("/")[-1] self._session_name = assumed_role.session_name - if headers["X-Amz-Security-Token"] != self._session_name: + if headers["X-Amz-Security-Token"] != self._session_token: raise CreateAccessKeyFailure(reason="InvalidToken") return raise CreateAccessKeyFailure(reason="InvalidId") @@ -101,7 +102,7 @@ class AssumedRoleAccessKey: inline_policy_names = iam_backend.list_role_policies(self._owner_role_name) for inline_policy_name in inline_policy_names: - inline_policy = iam_backend.get_role_policy(self._owner_role_name, inline_policy_name) + _, inline_policy = iam_backend.get_role_policy(self._owner_role_name, inline_policy_name) role_policies.append(inline_policy) attached_policies, _ = iam_backend.list_attached_role_policies(self._owner_role_name) @@ -252,6 +253,8 @@ class IAMPolicy: if isinstance(self._policy, Policy): default_version = next(policy_version for policy_version in self._policy.versions if policy_version.is_default) policy_document = default_version.document + elif isinstance(self._policy, string_types): + policy_document = self._policy else: policy_document = self._policy["policy_document"] From 947e26ce1b5873c5cfb1065d884bee4727fe4a2f Mon Sep 17 00:00:00 2001 From: acsbendi Date: Tue, 9 Jul 2019 19:46:04 +0200 Subject: [PATCH 11/36] Removed unused exception. --- moto/core/authentication.py | 2 +- moto/core/exceptions.py | 9 --------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/moto/core/authentication.py b/moto/core/authentication.py index 88e1b6b42..54f33b739 100644 --- a/moto/core/authentication.py +++ b/moto/core/authentication.py @@ -10,7 +10,7 @@ from six import string_types from moto.iam.models import ACCOUNT_ID, Policy from moto.iam import iam_backend -from moto.core.exceptions import SignatureDoesNotMatchError, AccessDeniedError, InvalidClientTokenIdError, InvalidAccessKeyIdError, AuthFailureError +from moto.core.exceptions import SignatureDoesNotMatchError, AccessDeniedError, InvalidClientTokenIdError, AuthFailureError from moto.s3.exceptions import BucketAccessDeniedError, S3AccessDeniedError, BucketInvalidTokenError, S3InvalidTokenError, S3InvalidAccessKeyIdError, BucketInvalidAccessKeyIdError from moto.sts import sts_backend diff --git a/moto/core/exceptions.py b/moto/core/exceptions.py index e18797fa4..58db5ed78 100644 --- a/moto/core/exceptions.py +++ b/moto/core/exceptions.py @@ -97,15 +97,6 @@ class AccessDeniedError(RESTError): )) -class InvalidAccessKeyIdError(RESTError): - code = 400 - - def __init__(self): - super(InvalidAccessKeyIdError, self).__init__( - 'InvalidAccessKeyId', - "The AWS Access Key Id you provided does not exist in our records.") - - class AuthFailureError(RESTError): code = 400 From bec0c5a273e14d66affbe1cebdefe54fc4f3574b Mon Sep 17 00:00:00 2001 From: acsbendi Date: Wed, 10 Jul 2019 20:42:23 +0200 Subject: [PATCH 12/36] Fixed S3 actions not handled properly. --- moto/core/authentication.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/moto/core/authentication.py b/moto/core/authentication.py index 54f33b739..523bd83f9 100644 --- a/moto/core/authentication.py +++ b/moto/core/authentication.py @@ -1,5 +1,6 @@ import json import re +import sys from abc import ABC, abstractmethod from enum import Enum @@ -121,7 +122,7 @@ class CreateAccessKeyFailure(Exception): class IAMRequestBase(ABC): def __init__(self, method, path, data, headers): - print(f"Creating {self.__class__.__name__} with method={method}, path={path}, data={data}, headers={headers}") + print(f"Creating {self.__class__.__name__} with method={method}, path={path}, data={data}, headers={headers}", file=sys.stderr) self._method = method self._path = path self._data = data @@ -130,7 +131,7 @@ class IAMRequestBase(ABC): credential_data = credential_scope.split('/') self._region = credential_data[2] self._service = credential_data[3] - self._action = self._service + ":" + self._data["Action"][0] + self._action = self._service + ":" + (self._data["Action"][0] if isinstance(self._data["Action"], list) else self._data["Action"]) try: self._access_key = create_access_key(access_key_id=credential_data[0], headers=headers) except CreateAccessKeyFailure as e: @@ -143,9 +144,6 @@ class IAMRequestBase(ABC): raise SignatureDoesNotMatchError() def check_action_permitted(self): - self._check_action_permitted_for_iam_user() - - def _check_action_permitted_for_iam_user(self): policies = self._access_key.collect_policies() permitted = False From 59f091bdea069a6c64c8a03ad8a54a230eb62640 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Thu, 11 Jul 2019 13:58:57 +0200 Subject: [PATCH 13/36] Default INITIAL_NO_AUTH_ACTION_COUNT should be infinity. --- moto/core/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/core/responses.py b/moto/core/responses.py index 4165732ab..7c2f1c737 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -108,7 +108,7 @@ class _TemplateEnvironmentMixin(object): class ActionAuthenticatorMixin(object): - INITIAL_NO_AUTH_ACTION_COUNT = int(os.environ.get("INITIAL_NO_AUTH_ACTION_COUNT", 999999999)) + INITIAL_NO_AUTH_ACTION_COUNT = float(os.environ.get("INITIAL_NO_AUTH_ACTION_COUNT", float("inf"))) request_count = 0 def _authenticate_action(self, iam_request_cls): From 9d992c9335e02bc231f17a4ffa12577a86d562bc Mon Sep 17 00:00:00 2001 From: acsbendi Date: Thu, 11 Jul 2019 14:22:42 +0200 Subject: [PATCH 14/36] Fixed error on single (non-list) Statements. --- moto/core/authentication.py | 38 +++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/moto/core/authentication.py b/moto/core/authentication.py index 523bd83f9..3e2d9e516 100644 --- a/moto/core/authentication.py +++ b/moto/core/authentication.py @@ -245,27 +245,29 @@ class S3IAMRequest(IAMRequestBase): class IAMPolicy: def __init__(self, policy): - self._policy = policy + if isinstance(policy, Policy): + default_version = next(policy_version for policy_version in policy.versions if policy_version.is_default) + policy_document = default_version.document + elif isinstance(policy, string_types): + policy_document = policy + else: + policy_document = policy["policy_document"] + + self._policy_json = json.loads(policy_document) def is_action_permitted(self, action): - if isinstance(self._policy, Policy): - default_version = next(policy_version for policy_version in self._policy.versions if policy_version.is_default) - policy_document = default_version.document - elif isinstance(self._policy, string_types): - policy_document = self._policy - else: - policy_document = self._policy["policy_document"] - - policy_json = json.loads(policy_document) - permitted = False - for policy_statement in policy_json["Statement"]: - iam_policy_statement = IAMPolicyStatement(policy_statement) - permission_result = iam_policy_statement.is_action_permitted(action) - if permission_result == PermissionResult.DENIED: - return permission_result - elif permission_result == PermissionResult.PERMITTED: - permitted = True + if isinstance(self._policy_json["Statement"], list): + for policy_statement in self._policy_json["Statement"]: + iam_policy_statement = IAMPolicyStatement(policy_statement) + permission_result = iam_policy_statement.is_action_permitted(action) + if permission_result == PermissionResult.DENIED: + return permission_result + elif permission_result == PermissionResult.PERMITTED: + permitted = True + else: # dict + iam_policy_statement = IAMPolicyStatement(self._policy_json["Statement"]) + return iam_policy_statement.is_action_permitted(action) if permitted: return PermissionResult.PERMITTED From de01adec57b70762a99dceeefbfedce653df61a3 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Sat, 13 Jul 2019 14:58:42 +0200 Subject: [PATCH 15/36] Fixed linting errors. --- moto/core/authentication.py | 4 +++- moto/core/exceptions.py | 1 - moto/s3/exceptions.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/moto/core/authentication.py b/moto/core/authentication.py index 3e2d9e516..324f8db2a 100644 --- a/moto/core/authentication.py +++ b/moto/core/authentication.py @@ -122,7 +122,9 @@ class CreateAccessKeyFailure(Exception): class IAMRequestBase(ABC): def __init__(self, method, path, data, headers): - print(f"Creating {self.__class__.__name__} with method={method}, path={path}, data={data}, headers={headers}", file=sys.stderr) + print("Creating {class_name} with method={method}, path={path}, data={data}, headers={headers}".format( + class_name=self.__class__.__name__, method=method, path=path, data=data, headers=headers), + file=sys.stderr) self._method = method self._path = path self._data = data diff --git a/moto/core/exceptions.py b/moto/core/exceptions.py index 58db5ed78..fcb1a0953 100644 --- a/moto/core/exceptions.py +++ b/moto/core/exceptions.py @@ -104,4 +104,3 @@ class AuthFailureError(RESTError): super(AuthFailureError, self).__init__( 'AuthFailure', "AWS was not able to validate the provided access credentials") - diff --git a/moto/s3/exceptions.py b/moto/s3/exceptions.py index 579d95148..c175d5066 100644 --- a/moto/s3/exceptions.py +++ b/moto/s3/exceptions.py @@ -219,7 +219,7 @@ class S3InvalidTokenError(S3ClientError): code = 400 def __init__(self, *args, **kwargs): - super(S3InvalidTokenError, self).__init__('InvalidToken', 'The provided token is malformed or otherwise invalid.', *args, **kwargs) + super(S3InvalidTokenError, self).__init__('InvalidToken', 'The provided token is malformed or otherwise invalid.', *args, **kwargs) class BucketInvalidTokenError(BucketError): From 7b096d690fc203baf3047cb23fece50f3b188756 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Sat, 13 Jul 2019 15:04:41 +0200 Subject: [PATCH 16/36] Replaced print with log.debug. --- moto/core/authentication.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/moto/core/authentication.py b/moto/core/authentication.py index 324f8db2a..dd9471628 100644 --- a/moto/core/authentication.py +++ b/moto/core/authentication.py @@ -1,6 +1,6 @@ import json +import logging import re -import sys from abc import ABC, abstractmethod from enum import Enum @@ -15,6 +15,8 @@ from moto.core.exceptions import SignatureDoesNotMatchError, AccessDeniedError, from moto.s3.exceptions import BucketAccessDeniedError, S3AccessDeniedError, BucketInvalidTokenError, S3InvalidTokenError, S3InvalidAccessKeyIdError, BucketInvalidAccessKeyIdError from moto.sts import sts_backend +log = logging.getLogger(__name__) + def create_access_key(access_key_id, headers): if access_key_id.startswith("AKIA") or "X-Amz-Security-Token" not in headers: @@ -122,9 +124,8 @@ class CreateAccessKeyFailure(Exception): class IAMRequestBase(ABC): def __init__(self, method, path, data, headers): - print("Creating {class_name} with method={method}, path={path}, data={data}, headers={headers}".format( - class_name=self.__class__.__name__, method=method, path=path, data=data, headers=headers), - file=sys.stderr) + log.debug("Creating {class_name} with method={method}, path={path}, data={data}, headers={headers}".format( + class_name=self.__class__.__name__, method=method, path=path, data=data, headers=headers)) self._method = method self._path = path self._data = data @@ -314,7 +315,7 @@ class IAMPolicyStatement: @staticmethod def _match(pattern, string): pattern = pattern.replace("*", ".*") - pattern = f"^{pattern}$" + pattern = "^{pattern}$".format(pattern=pattern) return re.match(pattern, string) From 7db2d0f38c1e578231009468d985ccdd79584ebf Mon Sep 17 00:00:00 2001 From: acsbendi Date: Sat, 13 Jul 2019 15:12:21 +0200 Subject: [PATCH 17/36] Use abc in a python2-compatible way. --- moto/core/authentication.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/moto/core/authentication.py b/moto/core/authentication.py index dd9471628..355b666e1 100644 --- a/moto/core/authentication.py +++ b/moto/core/authentication.py @@ -1,9 +1,10 @@ import json import logging import re -from abc import ABC, abstractmethod +from abc import abstractmethod, ABCMeta from enum import Enum +import six from botocore.auth import SigV4Auth, S3SigV4Auth from botocore.awsrequest import AWSRequest from botocore.credentials import Credentials @@ -25,7 +26,7 @@ def create_access_key(access_key_id, headers): return AssumedRoleAccessKey(access_key_id, headers) -class IAMUserAccessKey: +class IAMUserAccessKey(object): def __init__(self, access_key_id, headers): iam_users = iam_backend.list_users('/', None, None) @@ -74,7 +75,7 @@ class IAMUserAccessKey: return user_policies -class AssumedRoleAccessKey: +class AssumedRoleAccessKey(object): def __init__(self, access_key_id, headers): for assumed_role in sts_backend.assumed_roles: @@ -121,7 +122,8 @@ class CreateAccessKeyFailure(Exception): self.reason = reason -class IAMRequestBase(ABC): +@six.add_metaclass(ABCMeta) +class IAMRequestBase(object): def __init__(self, method, path, data, headers): log.debug("Creating {class_name} with method={method}, path={path}, data={data}, headers={headers}".format( @@ -245,7 +247,7 @@ class S3IAMRequest(IAMRequestBase): raise S3AccessDeniedError() -class IAMPolicy: +class IAMPolicy(object): def __init__(self, policy): if isinstance(policy, Policy): @@ -278,7 +280,7 @@ class IAMPolicy: return PermissionResult.NEUTRAL -class IAMPolicyStatement: +class IAMPolicyStatement(object): def __init__(self, statement): self._statement = statement From 1b8179992e5fcec5cad612b8211055f737962e39 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Sat, 13 Jul 2019 15:58:34 +0200 Subject: [PATCH 18/36] GetUser returns the IAM user who owns the access key in the request. --- moto/iam/models.py | 8 ++++++++ moto/iam/responses.py | 12 +++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/moto/iam/models.py b/moto/iam/models.py index f92568df4..bb19b8cad 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -1248,5 +1248,13 @@ class IAMBackend(BaseBackend): return saml_provider raise IAMNotFoundException("SamlProvider {0} not found".format(saml_provider_arn)) + def get_user_from_access_key_id(self, access_key_id): + for user_name, user in self.users.items(): + access_keys = self.get_all_access_keys(user_name) + for access_key in access_keys: + if access_key.access_key_id == access_key_id: + return user + return None + iam_backend = IAMBackend() diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 7671e8cb8..7ec6242f6 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -1,9 +1,8 @@ from __future__ import unicode_literals from moto.core.responses import BaseResponse -from .models import iam_backend -AVATAO_USER_NAME = "avatao-user" +from .models import iam_backend, User class IamResponse(BaseResponse): @@ -428,9 +427,12 @@ class IamResponse(BaseResponse): def get_user(self): user_name = self._get_param('UserName') if not user_name: - user_name = AVATAO_USER_NAME - # If no user is specified, IAM returns the current user - user = iam_backend.get_user(user_name) + access_key_id = self.get_current_user() + user = iam_backend.get_user_from_access_key_id(access_key_id) + if user is None: + user = User("default_user") + else: + user = iam_backend.get_user(user_name) template = self.response_template(USER_TEMPLATE) return template.render(action='Get', user=user) From 95799b99bc5a31f112b1b493a628a877ee652219 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Tue, 16 Jul 2019 16:27:50 +0200 Subject: [PATCH 19/36] Fixed incorrect authentication error handling in S3. --- moto/s3/responses.py | 109 ++++++++++++------------------------------- 1 file changed, 30 insertions(+), 79 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index a052e4cfb..e868b24be 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals import re import six -from werkzeug.exceptions import HTTPException from moto.core.utils import str_to_rfc_1123_datetime from six.moves.urllib.parse import parse_qs, urlparse, unquote @@ -119,13 +118,10 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): def should_autoescape(self): return True - def all_buckets(self, headers): - try: - self.data["Action"] = "ListAllMyBuckets" - self._authenticate_s3_action() - except HTTPException as http_error: - response = http_error.code, headers, http_error.description - return self._send_response(response) + def all_buckets(self): + self.data["Action"] = "ListAllMyBuckets" + self._authenticate_s3_action() + # No bucket specified. Listing all buckets all_buckets = self.backend.get_all_buckets() template = self.response_template(S3_ALL_BUCKETS) @@ -219,7 +215,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): bucket_name = self.parse_bucket_name_from_url(request, full_url) if not bucket_name: # If no bucket specified, list all buckets - return self.all_buckets(headers) + return self.all_buckets() self.data["BucketName"] = bucket_name @@ -236,15 +232,15 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): body = u'{0}'.format(body).encode('utf-8') if method == 'HEAD': - return self._bucket_response_head(bucket_name, headers) + return self._bucket_response_head(bucket_name) elif method == 'GET': - return self._bucket_response_get(bucket_name, querystring, headers) + return self._bucket_response_get(bucket_name, querystring) elif method == 'PUT': - return self._bucket_response_put(request, body, region_name, bucket_name, querystring, headers) + return self._bucket_response_put(request, body, region_name, bucket_name, querystring) elif method == 'DELETE': - return self._bucket_response_delete(body, bucket_name, querystring, headers) + return self._bucket_response_delete(body, bucket_name, querystring) elif method == 'POST': - return self._bucket_response_post(request, body, bucket_name, headers) + return self._bucket_response_post(request, body, bucket_name) else: raise NotImplementedError( "Method {0} has not been impelemented in the S3 backend yet".format(method)) @@ -255,7 +251,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): querystring = parse_qs(parsed_url.query, keep_blank_values=True) return querystring - def _bucket_response_head(self, bucket_name, headers): + def _bucket_response_head(self, bucket_name): try: self.backend.get_bucket(bucket_name) except MissingBucket: @@ -266,14 +262,9 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): return 404, {}, "" return 200, {}, "" - def _bucket_response_get(self, bucket_name, querystring, headers): + def _bucket_response_get(self, bucket_name, querystring): self._set_action("BUCKET", "GET", querystring) - - try: - self._authenticate_s3_action() - except HTTPException as http_error: - response = http_error.code, headers, http_error.description - return self._send_response(response) + self._authenticate_s3_action() if 'uploads' in querystring: for unsup in ('delimiter', 'max-uploads'): @@ -502,17 +493,12 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): next_continuation_token = None return result_keys, is_truncated, next_continuation_token - def _bucket_response_put(self, request, body, region_name, bucket_name, querystring, headers): + def _bucket_response_put(self, request, body, region_name, bucket_name, querystring): if not request.headers.get('Content-Length'): return 411, {}, "Content-Length required" self._set_action("BUCKET", "PUT", querystring) - - try: - self._authenticate_s3_action() - except HTTPException as http_error: - response = http_error.code, headers, http_error.description - return self._send_response(response) + self._authenticate_s3_action() if 'versioning' in querystring: ver = re.search('([A-Za-z]+)', body.decode()) @@ -612,14 +598,9 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): template = self.response_template(S3_BUCKET_CREATE_RESPONSE) return 200, {}, template.render(bucket=new_bucket) - def _bucket_response_delete(self, body, bucket_name, querystring, headers): + def _bucket_response_delete(self, body, bucket_name, querystring): self._set_action("BUCKET", "DELETE", querystring) - - try: - self._authenticate_s3_action() - except HTTPException as http_error: - response = http_error.code, headers, http_error.description - return self._send_response(response) + self._authenticate_s3_action() if 'policy' in querystring: self.backend.delete_bucket_policy(bucket_name, body) @@ -647,7 +628,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): S3_DELETE_BUCKET_WITH_ITEMS_ERROR) return 409, {}, template.render(bucket=removed_bucket) - def _bucket_response_post(self, request, body, bucket_name, headers): + def _bucket_response_post(self, request, body, bucket_name): if not request.headers.get('Content-Length'): return 411, {}, "Content-Length required" @@ -655,22 +636,12 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): if self.is_delete_keys(request, path, bucket_name): self.data["Action"] = "DeleteObject" + self._authenticate_s3_action() - try: - self._authenticate_s3_action() - except HTTPException as http_error: - response = http_error.code, headers, http_error.description - return self._send_response(response) - - return self._bucket_response_delete_keys(request, body, bucket_name, headers) + return self._bucket_response_delete_keys(request, body, bucket_name) self.data["Action"] = "PutObject" - - try: - self._authenticate_s3_action() - except HTTPException as http_error: - response = http_error.code, headers, http_error.description - return self._send_response(response) + self._authenticate_s3_action() # POST to bucket-url should create file from form if hasattr(request, 'form'): @@ -707,7 +678,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): path = request.full_path if hasattr(request, 'full_path') else path_url(request.url) return path - def _bucket_response_delete_keys(self, request, body, bucket_name, headers): + def _bucket_response_delete_keys(self, request, body, bucket_name): template = self.response_template(S3_DELETE_KEYS_RESPONSE) keys = minidom.parseString(body).getElementsByTagName('Key') @@ -813,21 +784,16 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): elif method == 'HEAD': return self._key_response_head(bucket_name, query, key_name, headers=request.headers) elif method == 'DELETE': - return self._key_response_delete(bucket_name, query, key_name, headers) + return self._key_response_delete(bucket_name, query, key_name) elif method == 'POST': - return self._key_response_post(request, body, bucket_name, query, key_name, headers) + return self._key_response_post(request, body, bucket_name, query, key_name) else: raise NotImplementedError( "Method {0} has not been implemented in the S3 backend yet".format(method)) def _key_response_get(self, bucket_name, query, key_name, headers): self._set_action("KEY", "GET", query) - - try: - self._authenticate_s3_action() - except HTTPException as http_error: - response = http_error.code, headers, http_error.description - return self._send_response(response) + self._authenticate_s3_action() response_headers = {} if query.get('uploadId'): @@ -864,12 +830,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): def _key_response_put(self, request, body, bucket_name, query, key_name, headers): self._set_action("KEY", "PUT", query) - - try: - self._authenticate_s3_action() - except HTTPException as http_error: - response = http_error.code, headers, http_error.description - return self._send_response(response) + self._authenticate_s3_action() response_headers = {} if query.get('uploadId') and query.get('partNumber'): @@ -1237,14 +1198,9 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): config = parsed_xml['AccelerateConfiguration'] return config['Status'] - def _key_response_delete(self, bucket_name, query, key_name, headers): + def _key_response_delete(self, bucket_name, query, key_name): self._set_action("KEY", "DELETE", query) - - try: - self._authenticate_s3_action() - except HTTPException as http_error: - response = http_error.code, headers, http_error.description - return self._send_response(response) + self._authenticate_s3_action() if query.get('uploadId'): upload_id = query['uploadId'][0] @@ -1265,14 +1221,9 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): raise InvalidPartOrder() yield (pn, p.getElementsByTagName('ETag')[0].firstChild.wholeText) - def _key_response_post(self, request, body, bucket_name, query, key_name, headers): + def _key_response_post(self, request, body, bucket_name, query, key_name): self._set_action("KEY", "POST", query) - - try: - self._authenticate_s3_action() - except HTTPException as http_error: - response = http_error.code, headers, http_error.description - return self._send_response(response) + self._authenticate_s3_action() if body == b'' and 'uploads' in query: metadata = metadata_from_headers(request.headers) From 3dd2e3a1b873dd66a64f48625552f72dce9f2e3f Mon Sep 17 00:00:00 2001 From: acsbendi Date: Wed, 24 Jul 2019 16:30:48 +0200 Subject: [PATCH 20/36] Moved INITIAL_NO_AUTH_ACTION_COUNT to settings. --- moto/core/responses.py | 6 ++---- moto/settings.py | 1 + 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/moto/core/responses.py b/moto/core/responses.py index 7c2f1c737..6cd9e24fb 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -1,6 +1,5 @@ from __future__ import unicode_literals -import os from collections import defaultdict import datetime import json @@ -25,7 +24,7 @@ from werkzeug.exceptions import HTTPException import boto3 from moto.compat import OrderedDict from moto.core.utils import camelcase_to_underscores, method_names_from_class - +from moto.settings import INITIAL_NO_AUTH_ACTION_COUNT log = logging.getLogger(__name__) @@ -108,11 +107,10 @@ class _TemplateEnvironmentMixin(object): class ActionAuthenticatorMixin(object): - INITIAL_NO_AUTH_ACTION_COUNT = float(os.environ.get("INITIAL_NO_AUTH_ACTION_COUNT", float("inf"))) request_count = 0 def _authenticate_action(self, iam_request_cls): - if ActionAuthenticatorMixin.request_count >= ActionAuthenticatorMixin.INITIAL_NO_AUTH_ACTION_COUNT: + if ActionAuthenticatorMixin.request_count >= INITIAL_NO_AUTH_ACTION_COUNT: iam_request = iam_request_cls(method=self.method, path=self.path, data=self.data, headers=self.headers) iam_request.check_signature() iam_request.check_action_permitted() diff --git a/moto/settings.py b/moto/settings.py index a5240f130..12402dc80 100644 --- a/moto/settings.py +++ b/moto/settings.py @@ -1,3 +1,4 @@ import os TEST_SERVER_MODE = os.environ.get('TEST_SERVER_MODE', '0').lower() == 'true' +INITIAL_NO_AUTH_ACTION_COUNT = float(os.environ.get('INITIAL_NO_AUTH_ACTION_COUNT', float('inf'))) From bbf003d335e4dcd3a19663d8a834f7c99aa0f21b Mon Sep 17 00:00:00 2001 From: acsbendi Date: Wed, 24 Jul 2019 17:21:33 +0200 Subject: [PATCH 21/36] Set correct HTTP codes for some auth-related errors. --- moto/core/exceptions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/moto/core/exceptions.py b/moto/core/exceptions.py index fcb1a0953..06cfd8895 100644 --- a/moto/core/exceptions.py +++ b/moto/core/exceptions.py @@ -68,7 +68,7 @@ class JsonRESTError(RESTError): class SignatureDoesNotMatchError(RESTError): - code = 400 + code = 403 def __init__(self): super(SignatureDoesNotMatchError, self).__init__( @@ -77,7 +77,7 @@ class SignatureDoesNotMatchError(RESTError): class InvalidClientTokenIdError(RESTError): - code = 400 + code = 403 def __init__(self): super(InvalidClientTokenIdError, self).__init__( @@ -98,7 +98,7 @@ class AccessDeniedError(RESTError): class AuthFailureError(RESTError): - code = 400 + code = 401 def __init__(self): super(AuthFailureError, self).__init__( From 15c872cffc32ed8180b08bc849f30affb1d2b1b6 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Wed, 24 Jul 2019 18:15:31 +0200 Subject: [PATCH 22/36] Created decorator for setting INITIAL_NO_AUTH_ACTION_COUNT. --- moto/core/__init__.py | 4 +++- moto/core/models.py | 16 ++++++++++++++++ moto/core/responses.py | 4 ++-- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/moto/core/__init__.py b/moto/core/__init__.py index 9e2c1e70f..9406a7ea0 100644 --- a/moto/core/__init__.py +++ b/moto/core/__init__.py @@ -1,4 +1,6 @@ from __future__ import unicode_literals -from .models import BaseModel, BaseBackend, moto_api_backend # flake8: noqa + +from .models import BaseModel, BaseBackend, moto_api_backend, set_initial_no_auth_action_count # flake8: noqa moto_api_backends = {"global": moto_api_backend} +set_initial_no_auth_action_count = set_initial_no_auth_action_count diff --git a/moto/core/models.py b/moto/core/models.py index 9fe1e96bd..68f799e3c 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -27,6 +27,22 @@ os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") +def set_initial_no_auth_action_count(initial_no_auth_action_count): + def decorator(function): + def wrapper(*args, **kwargs): + original_initial_no_auth_action_count = settings.INITIAL_NO_AUTH_ACTION_COUNT + settings.INITIAL_NO_AUTH_ACTION_COUNT = initial_no_auth_action_count + result = function(*args, **kwargs) + settings.INITIAL_NO_AUTH_ACTION_COUNT = original_initial_no_auth_action_count + return result + + functools.update_wrapper(wrapper, function) + wrapper.__wrapped__ = function + return wrapper + + return decorator + + class BaseMockAWS(object): nested_count = 0 diff --git a/moto/core/responses.py b/moto/core/responses.py index 6cd9e24fb..9f2f61fd6 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -24,7 +24,7 @@ from werkzeug.exceptions import HTTPException import boto3 from moto.compat import OrderedDict from moto.core.utils import camelcase_to_underscores, method_names_from_class -from moto.settings import INITIAL_NO_AUTH_ACTION_COUNT +from moto import settings log = logging.getLogger(__name__) @@ -110,7 +110,7 @@ class ActionAuthenticatorMixin(object): request_count = 0 def _authenticate_action(self, iam_request_cls): - if ActionAuthenticatorMixin.request_count >= INITIAL_NO_AUTH_ACTION_COUNT: + if ActionAuthenticatorMixin.request_count >= settings.INITIAL_NO_AUTH_ACTION_COUNT: iam_request = iam_request_cls(method=self.method, path=self.path, data=self.data, headers=self.headers) iam_request.check_signature() iam_request.check_action_permitted() From e22e8b5a673c717ef39674d1dd6cf49caa485606 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Wed, 24 Jul 2019 18:58:50 +0200 Subject: [PATCH 23/36] set_initial_no_auth_action_count should also set request_count to 0. --- moto/core/__init__.py | 5 +++-- moto/core/models.py | 16 ---------------- moto/core/responses.py | 20 ++++++++++++++++++++ 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/moto/core/__init__.py b/moto/core/__init__.py index 9406a7ea0..801e675df 100644 --- a/moto/core/__init__.py +++ b/moto/core/__init__.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals -from .models import BaseModel, BaseBackend, moto_api_backend, set_initial_no_auth_action_count # flake8: noqa +from .models import BaseModel, BaseBackend, moto_api_backend # flake8: noqa +from .responses import ActionAuthenticatorMixin moto_api_backends = {"global": moto_api_backend} -set_initial_no_auth_action_count = set_initial_no_auth_action_count +set_initial_no_auth_action_count = ActionAuthenticatorMixin.set_initial_no_auth_action_count diff --git a/moto/core/models.py b/moto/core/models.py index 68f799e3c..9fe1e96bd 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -27,22 +27,6 @@ os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") -def set_initial_no_auth_action_count(initial_no_auth_action_count): - def decorator(function): - def wrapper(*args, **kwargs): - original_initial_no_auth_action_count = settings.INITIAL_NO_AUTH_ACTION_COUNT - settings.INITIAL_NO_AUTH_ACTION_COUNT = initial_no_auth_action_count - result = function(*args, **kwargs) - settings.INITIAL_NO_AUTH_ACTION_COUNT = original_initial_no_auth_action_count - return result - - functools.update_wrapper(wrapper, function) - wrapper.__wrapped__ = function - return wrapper - - return decorator - - class BaseMockAWS(object): nested_count = 0 diff --git a/moto/core/responses.py b/moto/core/responses.py index 9f2f61fd6..5a6fdbf5b 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import functools from collections import defaultdict import datetime import json @@ -123,6 +124,25 @@ class ActionAuthenticatorMixin(object): def _authenticate_s3_action(self): self._authenticate_action(S3IAMRequest) + @staticmethod + def set_initial_no_auth_action_count(initial_no_auth_action_count): + def decorator(function): + def wrapper(*args, **kwargs): + original_initial_no_auth_action_count = settings.INITIAL_NO_AUTH_ACTION_COUNT + settings.INITIAL_NO_AUTH_ACTION_COUNT = initial_no_auth_action_count + ActionAuthenticatorMixin.request_count = 0 + try: + result = function(*args, **kwargs) + finally: + settings.INITIAL_NO_AUTH_ACTION_COUNT = original_initial_no_auth_action_count + return result + + functools.update_wrapper(wrapper, function) + wrapper.__wrapped__ = function + return wrapper + + return decorator + class BaseResponse(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): From d471eb69c0ca36ed06f27136caaffb9e346675f7 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Wed, 24 Jul 2019 19:47:39 +0200 Subject: [PATCH 24/36] For EC2 requests, AuthFailure should be raised instead of SignatureDoesNotMatch. --- moto/core/authentication.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/moto/core/authentication.py b/moto/core/authentication.py index 355b666e1..df7d1bf34 100644 --- a/moto/core/authentication.py +++ b/moto/core/authentication.py @@ -146,7 +146,7 @@ class IAMRequestBase(object): original_signature = self._get_string_between('Signature=', ',', self._headers['Authorization']) calculated_signature = self._calculate_signature() if original_signature != calculated_signature: - raise SignatureDoesNotMatchError() + self._raise_signature_does_not_match() def check_action_permitted(self): policies = self._access_key.collect_policies() @@ -163,6 +163,12 @@ class IAMRequestBase(object): if not permitted: self._raise_access_denied() + def _raise_signature_does_not_match(self): + if self._service == "ec2": + raise AuthFailureError() + else: + raise SignatureDoesNotMatchError() + @abstractmethod def _raise_access_denied(self): raise NotImplementedError() From d428acdb7c6f9e38d582725e1f44028ce74e4231 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Wed, 24 Jul 2019 21:01:11 +0200 Subject: [PATCH 25/36] Separate SignatureDoesNotMatchError for S3. --- moto/core/authentication.py | 30 ++++++++++++++++++++++++------ moto/s3/exceptions.py | 22 ++++++++++++++++++++-- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/moto/core/authentication.py b/moto/core/authentication.py index df7d1bf34..878a996e2 100644 --- a/moto/core/authentication.py +++ b/moto/core/authentication.py @@ -13,7 +13,16 @@ from six import string_types from moto.iam.models import ACCOUNT_ID, Policy from moto.iam import iam_backend from moto.core.exceptions import SignatureDoesNotMatchError, AccessDeniedError, InvalidClientTokenIdError, AuthFailureError -from moto.s3.exceptions import BucketAccessDeniedError, S3AccessDeniedError, BucketInvalidTokenError, S3InvalidTokenError, S3InvalidAccessKeyIdError, BucketInvalidAccessKeyIdError +from moto.s3.exceptions import ( + BucketAccessDeniedError, + S3AccessDeniedError, + BucketInvalidTokenError, + S3InvalidTokenError, + S3InvalidAccessKeyIdError, + BucketInvalidAccessKeyIdError, + BucketSignatureDoesNotMatchError, + S3SignatureDoesNotMatchError +) from moto.sts import sts_backend log = logging.getLogger(__name__) @@ -163,11 +172,9 @@ class IAMRequestBase(object): if not permitted: self._raise_access_denied() + @abstractmethod def _raise_signature_does_not_match(self): - if self._service == "ec2": - raise AuthFailureError() - else: - raise SignatureDoesNotMatchError() + raise NotImplementedError() @abstractmethod def _raise_access_denied(self): @@ -212,6 +219,12 @@ class IAMRequestBase(object): class IAMRequest(IAMRequestBase): + def _raise_signature_does_not_match(self): + if self._service == "ec2": + raise AuthFailureError() + else: + raise SignatureDoesNotMatchError() + def _raise_invalid_access_key(self, _): if self._service == "ec2": raise AuthFailureError() @@ -230,8 +243,13 @@ class IAMRequest(IAMRequestBase): class S3IAMRequest(IAMRequestBase): - def _raise_invalid_access_key(self, reason): + def _raise_signature_does_not_match(self): + if "BucketName" in self._data: + raise BucketSignatureDoesNotMatchError(bucket=self._data["BucketName"]) + else: + raise S3SignatureDoesNotMatchError() + def _raise_invalid_access_key(self, reason): if reason == "InvalidToken": if "BucketName" in self._data: raise BucketInvalidTokenError(bucket=self._data["BucketName"]) diff --git a/moto/s3/exceptions.py b/moto/s3/exceptions.py index c175d5066..f74fc21ae 100644 --- a/moto/s3/exceptions.py +++ b/moto/s3/exceptions.py @@ -230,7 +230,7 @@ class BucketInvalidTokenError(BucketError): class S3InvalidAccessKeyIdError(S3ClientError): - code = 400 + code = 403 def __init__(self, *args, **kwargs): super(S3InvalidAccessKeyIdError, self).__init__( @@ -239,9 +239,27 @@ class S3InvalidAccessKeyIdError(S3ClientError): class BucketInvalidAccessKeyIdError(S3ClientError): - code = 400 + code = 403 def __init__(self, *args, **kwargs): super(BucketInvalidAccessKeyIdError, self).__init__( 'InvalidAccessKeyId', "The AWS Access Key Id you provided does not exist in our records.", *args, **kwargs) + + +class S3SignatureDoesNotMatchError(S3ClientError): + code = 403 + + def __init__(self, *args, **kwargs): + super(S3SignatureDoesNotMatchError, self).__init__( + 'SignatureDoesNotMatch', + "The request signature we calculated does not match the signature you provided. Check your key and signing method.", *args, **kwargs) + + +class BucketSignatureDoesNotMatchError(S3ClientError): + code = 403 + + def __init__(self, *args, **kwargs): + super(BucketSignatureDoesNotMatchError, self).__init__( + 'SignatureDoesNotMatch', + "The request signature we calculated does not match the signature you provided. Check your key and signing method.", *args, **kwargs) From 45a380a807ca496e7aadbb335ea548d6f16c51da Mon Sep 17 00:00:00 2001 From: acsbendi Date: Wed, 24 Jul 2019 21:29:00 +0200 Subject: [PATCH 26/36] Fixed host not present in headers for S3 requests. --- moto/s3/responses.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index e868b24be..6dafa80a8 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -189,6 +189,8 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): self.method = request.method self.path = self._get_path(request) self.headers = request.headers + if 'host' not in self.headers: + self.headers['host'] = urlparse(full_url).netloc try: response = self._bucket_response(request, full_url, headers) except S3ClientError as s3error: From 3e1e27338093a9b6011252546c5fd46e654f1a74 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Wed, 24 Jul 2019 21:41:33 +0200 Subject: [PATCH 27/36] Fixed collecting policies from groups. --- moto/core/authentication.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/core/authentication.py b/moto/core/authentication.py index 878a996e2..2fec59808 100644 --- a/moto/core/authentication.py +++ b/moto/core/authentication.py @@ -73,12 +73,12 @@ class IAMUserAccessKey(object): user_groups = iam_backend.get_groups_for_user(self._owner_user_name) for user_group in user_groups: - inline_group_policy_names = iam_backend.list_group_policies(user_group) + inline_group_policy_names = iam_backend.list_group_policies(user_group.name) for inline_group_policy_name in inline_group_policy_names: inline_user_group_policy = iam_backend.get_group_policy(user_group.name, inline_group_policy_name) user_policies.append(inline_user_group_policy) - attached_group_policies = iam_backend.list_attached_group_policies(user_group.name) + attached_group_policies, _ = iam_backend.list_attached_group_policies(user_group.name) user_policies += attached_group_policies return user_policies From 290f8f9fd585d4def2dd099a11cb3cb97a138d86 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Fri, 26 Jul 2019 19:50:24 +0200 Subject: [PATCH 28/36] Fixed host header not included in S3 requests sometimes. --- moto/s3/responses.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 6dafa80a8..2617f139d 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -727,6 +727,8 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): self.method = request.method self.path = self._get_path(request) self.headers = request.headers + if 'host' not in self.headers: + self.headers['host'] = urlparse(full_url).netloc response_headers = {} try: response = self._key_response(request, full_url, headers) From cc843bb8c5e5dc5842c239fc4215031cb3c232f1 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Fri, 26 Jul 2019 20:40:15 +0200 Subject: [PATCH 29/36] Created tests for IAM auth. --- tests/test_core/test_auth.py | 555 +++++++++++++++++++++++++++++++++++ 1 file changed, 555 insertions(+) create mode 100644 tests/test_core/test_auth.py diff --git a/tests/test_core/test_auth.py b/tests/test_core/test_auth.py new file mode 100644 index 000000000..e3ce7df1d --- /dev/null +++ b/tests/test_core/test_auth.py @@ -0,0 +1,555 @@ +import json + +import boto3 +import sure # noqa +from botocore.exceptions import ClientError +# Ensure 'assert_raises' context manager support for Python 2.6 +import tests.backport_assert_raises +from nose.tools import assert_raises + +from moto import mock_iam, mock_ec2, mock_s3, mock_sts, mock_elbv2, mock_rds2 +from moto.core import set_initial_no_auth_action_count +from moto.iam.models import ACCOUNT_ID + + +@mock_iam +def create_user_with_access_key(user_name='test-user'): + client = boto3.client('iam', region_name='us-east-1') + client.create_user(UserName=user_name) + return client.create_access_key(UserName=user_name)['AccessKey'] + + +@mock_iam +def create_user_with_access_key_and_inline_policy(user_name, policy_document, policy_name='policy1'): + client = boto3.client('iam', region_name='us-east-1') + client.create_user(UserName=user_name) + client.put_user_policy(UserName=user_name, PolicyName=policy_name, PolicyDocument=json.dumps(policy_document)) + return client.create_access_key(UserName=user_name)['AccessKey'] + + +@mock_iam +def create_user_with_access_key_and_attached_policy(user_name, policy_document, policy_name='policy1'): + client = boto3.client('iam', region_name='us-east-1') + client.create_user(UserName=user_name) + policy_arn = client.create_policy( + PolicyName=policy_name, + PolicyDocument=json.dumps(policy_document) + )['Policy']['Arn'] + client.attach_user_policy(UserName=user_name, PolicyArn=policy_arn) + return client.create_access_key(UserName=user_name)['AccessKey'] + + +@mock_iam +def create_user_with_access_key_and_multiple_policies(user_name, inline_policy_document, + attached_policy_document, inline_policy_name='policy1', attached_policy_name='policy1'): + client = boto3.client('iam', region_name='us-east-1') + client.create_user(UserName=user_name) + policy_arn = client.create_policy( + PolicyName=attached_policy_name, + PolicyDocument=json.dumps(attached_policy_document) + )['Policy']['Arn'] + client.attach_user_policy(UserName=user_name, PolicyArn=policy_arn) + client.put_user_policy(UserName=user_name, PolicyName=inline_policy_name, PolicyDocument=json.dumps(inline_policy_document)) + return client.create_access_key(UserName=user_name)['AccessKey'] + + +def create_group_with_attached_policy_and_add_user(user_name, policy_document, + group_name='test-group', policy_name='policy1'): + client = boto3.client('iam', region_name='us-east-1') + client.create_group(GroupName=group_name) + policy_arn = client.create_policy( + PolicyName=policy_name, + PolicyDocument=json.dumps(policy_document) + )['Policy']['Arn'] + client.attach_group_policy(GroupName=group_name, PolicyArn=policy_arn) + client.add_user_to_group(GroupName=group_name, UserName=user_name) + + +def create_group_with_inline_policy_and_add_user(user_name, policy_document, + group_name='test-group', policy_name='policy1'): + client = boto3.client('iam', region_name='us-east-1') + client.create_group(GroupName=group_name) + client.put_group_policy( + GroupName=group_name, + PolicyName=policy_name, + PolicyDocument=json.dumps(policy_document) + ) + client.add_user_to_group(GroupName=group_name, UserName=user_name) + + +def create_group_with_multiple_policies_and_add_user(user_name, inline_policy_document, + attached_policy_document, group_name='test-group', + inline_policy_name='policy1', attached_policy_name='policy1'): + client = boto3.client('iam', region_name='us-east-1') + client.create_group(GroupName=group_name) + client.put_group_policy( + GroupName=group_name, + PolicyName=inline_policy_name, + PolicyDocument=json.dumps(inline_policy_document) + ) + policy_arn = client.create_policy( + PolicyName=attached_policy_name, + PolicyDocument=json.dumps(attached_policy_document) + )['Policy']['Arn'] + client.attach_group_policy(GroupName=group_name, PolicyArn=policy_arn) + client.add_user_to_group(GroupName=group_name, UserName=user_name) + + +@mock_iam +@mock_sts +def create_role_with_attached_policy_and_assume_it(role_name, trust_policy_document, + policy_document, session_name='session1', policy_name='policy1'): + iam_client = boto3.client('iam', region_name='us-east-1') + sts_client = boto3.client('sts', region_name='us-east-1') + role_arn = iam_client.create_role( + RoleName=role_name, + AssumeRolePolicyDocument=json.dumps(trust_policy_document) + )['Role']['Arn'] + policy_arn = iam_client.create_policy( + PolicyName=policy_name, + PolicyDocument=json.dumps(policy_document) + )['Policy']['Arn'] + iam_client.attach_role_policy(RoleName=role_name, PolicyArn=policy_arn) + return sts_client.assume_role(RoleArn=role_arn, RoleSessionName=session_name)['Credentials'] + + +@mock_iam +@mock_sts +def create_role_with_inline_policy_and_assume_it(role_name, trust_policy_document, + policy_document, session_name='session1', policy_name='policy1'): + iam_client = boto3.client('iam', region_name='us-east-1') + sts_client = boto3.client('sts', region_name='us-east-1') + role_arn = iam_client.create_role( + RoleName=role_name, + AssumeRolePolicyDocument=json.dumps(trust_policy_document) + )['Role']['Arn'] + iam_client.put_role_policy( + RoleName=role_name, + PolicyName=policy_name, + PolicyDocument=json.dumps(policy_document) + ) + return sts_client.assume_role(RoleArn=role_arn, RoleSessionName=session_name)['Credentials'] + + +@set_initial_no_auth_action_count(0) +@mock_iam +def test_invalid_client_token_id(): + client = boto3.client('iam', region_name='us-east-1', aws_access_key_id='invalid', aws_secret_access_key='invalid') + with assert_raises(ClientError) as ex: + client.get_user() + ex.exception.response['Error']['Code'].should.equal('InvalidClientTokenId') + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(403) + ex.exception.response['Error']['Message'].should.equal('The security token included in the request is invalid.') + + +@set_initial_no_auth_action_count(0) +@mock_ec2 +def test_auth_failure(): + client = boto3.client('ec2', region_name='us-east-1', aws_access_key_id='invalid', aws_secret_access_key='invalid') + with assert_raises(ClientError) as ex: + client.describe_instances() + ex.exception.response['Error']['Code'].should.equal('AuthFailure') + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(401) + ex.exception.response['Error']['Message'].should.equal('AWS was not able to validate the provided access credentials') + + +@set_initial_no_auth_action_count(2) +@mock_iam +def test_signature_does_not_match(): + access_key = create_user_with_access_key() + client = boto3.client('iam', region_name='us-east-1', + aws_access_key_id=access_key['AccessKeyId'], + aws_secret_access_key='invalid') + with assert_raises(ClientError) as ex: + client.get_user() + ex.exception.response['Error']['Code'].should.equal('SignatureDoesNotMatch') + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(403) + ex.exception.response['Error']['Message'].should.equal('The request signature we calculated does not match the signature you provided. Check your AWS Secret Access Key and signing method. Consult the service documentation for details.') + + +@set_initial_no_auth_action_count(2) +@mock_ec2 +def test_auth_failure_with_valid_access_key_id(): + access_key = create_user_with_access_key() + client = boto3.client('ec2', region_name='us-east-1', + aws_access_key_id=access_key['AccessKeyId'], + aws_secret_access_key='invalid') + with assert_raises(ClientError) as ex: + client.describe_instances() + ex.exception.response['Error']['Code'].should.equal('AuthFailure') + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(401) + ex.exception.response['Error']['Message'].should.equal('AWS was not able to validate the provided access credentials') + + +@set_initial_no_auth_action_count(2) +@mock_ec2 +def test_access_denied_with_no_policy(): + user_name = 'test-user' + access_key = create_user_with_access_key(user_name) + client = boto3.client('ec2', region_name='us-east-1', + aws_access_key_id=access_key['AccessKeyId'], + aws_secret_access_key=access_key['SecretAccessKey']) + with assert_raises(ClientError) as ex: + client.describe_instances() + ex.exception.response['Error']['Code'].should.equal('AccessDenied') + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(403) + ex.exception.response['Error']['Message'].should.equal( + 'User: arn:aws:iam::{account_id}:user/{user_name} is not authorized to perform: {operation}'.format( + account_id=ACCOUNT_ID, + user_name=user_name, + operation="ec2:DescribeInstances" + ) + ) + + +@set_initial_no_auth_action_count(3) +@mock_ec2 +def test_access_denied_with_not_allowing_policy(): + user_name = 'test-user' + inline_policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:Describe*" + ], + "Resource": "*" + } + ] + } + access_key = create_user_with_access_key_and_inline_policy(user_name, inline_policy_document) + client = boto3.client('ec2', region_name='us-east-1', + aws_access_key_id=access_key['AccessKeyId'], + aws_secret_access_key=access_key['SecretAccessKey']) + with assert_raises(ClientError) as ex: + client.run_instances(MaxCount=1, MinCount=1) + ex.exception.response['Error']['Code'].should.equal('AccessDenied') + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(403) + ex.exception.response['Error']['Message'].should.equal( + 'User: arn:aws:iam::{account_id}:user/{user_name} is not authorized to perform: {operation}'.format( + account_id=ACCOUNT_ID, + user_name=user_name, + operation="ec2:RunInstances" + ) + ) + + +@set_initial_no_auth_action_count(3) +@mock_ec2 +def test_access_denied_with_denying_policy(): + user_name = 'test-user' + inline_policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:*", + ], + "Resource": "*" + }, + { + "Effect": "Deny", + "Action": "ec2:CreateVpc", + "Resource": "*" + } + ] + } + access_key = create_user_with_access_key_and_inline_policy(user_name, inline_policy_document) + client = boto3.client('ec2', region_name='us-east-1', + aws_access_key_id=access_key['AccessKeyId'], + aws_secret_access_key=access_key['SecretAccessKey']) + with assert_raises(ClientError) as ex: + client.create_vpc(CidrBlock="10.0.0.0/16") + ex.exception.response['Error']['Code'].should.equal('AccessDenied') + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(403) + ex.exception.response['Error']['Message'].should.equal( + 'User: arn:aws:iam::{account_id}:user/{user_name} is not authorized to perform: {operation}'.format( + account_id=ACCOUNT_ID, + user_name=user_name, + operation="ec2:CreateVpc" + ) + ) + + +@set_initial_no_auth_action_count(3) +@mock_ec2 +def test_allowed_with_wildcard_action(): + user_name = 'test-user' + inline_policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "ec2:Describe*", + "Resource": "*" + } + ] + } + access_key = create_user_with_access_key_and_inline_policy(user_name, inline_policy_document) + client = boto3.client('ec2', region_name='us-east-1', + aws_access_key_id=access_key['AccessKeyId'], + aws_secret_access_key=access_key['SecretAccessKey']) + client.describe_tags()['Tags'].should.be.empty + + +@set_initial_no_auth_action_count(4) +@mock_iam +def test_allowed_with_explicit_action_in_attached_policy(): + user_name = 'test-user' + attached_policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "iam:ListGroups", + "Resource": "*" + } + ] + } + access_key = create_user_with_access_key_and_attached_policy(user_name, attached_policy_document) + client = boto3.client('iam', region_name='us-east-1', + aws_access_key_id=access_key['AccessKeyId'], + aws_secret_access_key=access_key['SecretAccessKey']) + client.list_groups()['Groups'].should.be.empty + + +@set_initial_no_auth_action_count(8) +@mock_s3 +@mock_iam +def test_s3_access_denied_with_denying_attached_group_policy(): + user_name = 'test-user' + attached_policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "s3:ListAllMyBuckets", + "Resource": "*" + } + ] + } + group_attached_policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Action": "s3:List*", + "Resource": "*" + } + ] + } + access_key = create_user_with_access_key_and_attached_policy(user_name, attached_policy_document) + create_group_with_attached_policy_and_add_user(user_name, group_attached_policy_document) + client = boto3.client('s3', region_name='us-east-1', + aws_access_key_id=access_key['AccessKeyId'], + aws_secret_access_key=access_key['SecretAccessKey']) + with assert_raises(ClientError) as ex: + client.list_buckets() + ex.exception.response['Error']['Code'].should.equal('AccessDenied') + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(403) + ex.exception.response['Error']['Message'].should.equal('Access Denied') + + +@set_initial_no_auth_action_count(6) +@mock_s3 +@mock_iam +def test_s3_access_denied_with_denying_inline_group_policy(): + user_name = 'test-user' + bucket_name = 'test-bucket' + inline_policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "*", + "Resource": "*" + } + ] + } + group_inline_policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Action": "s3:GetObject", + "Resource": "*" + } + ] + } + access_key = create_user_with_access_key_and_inline_policy(user_name, inline_policy_document) + create_group_with_inline_policy_and_add_user(user_name, group_inline_policy_document) + client = boto3.client('s3', region_name='us-east-1', + aws_access_key_id=access_key['AccessKeyId'], + aws_secret_access_key=access_key['SecretAccessKey']) + client.create_bucket(Bucket=bucket_name) + with assert_raises(ClientError) as ex: + client.get_object(Bucket=bucket_name, Key='sdfsdf') + ex.exception.response['Error']['Code'].should.equal('AccessDenied') + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(403) + ex.exception.response['Error']['Message'].should.equal('Access Denied') + + +@set_initial_no_auth_action_count(10) +@mock_iam +@mock_ec2 +def test_access_denied_with_many_irrelevant_policies(): + user_name = 'test-user' + inline_policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "ec2:Describe*", + "Resource": "*" + } + ] + } + attached_policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "s3:*", + "Resource": "*" + } + ] + } + group_inline_policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Action": "iam:List*", + "Resource": "*" + } + ] + } + group_attached_policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Action": "lambda:*", + "Resource": "*" + } + ] + } + access_key = create_user_with_access_key_and_multiple_policies(user_name, inline_policy_document, + attached_policy_document) + create_group_with_multiple_policies_and_add_user(user_name, group_inline_policy_document, + group_attached_policy_document) + client = boto3.client('ec2', region_name='us-east-1', + aws_access_key_id=access_key['AccessKeyId'], + aws_secret_access_key=access_key['SecretAccessKey']) + with assert_raises(ClientError) as ex: + client.create_key_pair(KeyName="TestKey") + ex.exception.response['Error']['Code'].should.equal('AccessDenied') + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(403) + ex.exception.response['Error']['Message'].should.equal( + 'User: arn:aws:iam::{account_id}:user/{user_name} is not authorized to perform: {operation}'.format( + account_id=ACCOUNT_ID, + user_name=user_name, + operation="ec2:CreateKeyPair" + ) + ) + + +@set_initial_no_auth_action_count(4) +@mock_iam +@mock_sts +@mock_ec2 +@mock_elbv2 +def test_allowed_with_temporary_credentials(): + role_name = 'test-role' + trust_policy_document = { + "Version": "2012-10-17", + "Statement": { + "Effect": "Allow", + "Principal": {"AWS": "arn:aws:iam::{account_id}:root".format(account_id=ACCOUNT_ID)}, + "Action": "sts:AssumeRole" + } + } + attached_policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:CreateLoadBalancer", + "ec2:DescribeSubnets" + ], + "Resource": "*" + } + ] + } + credentials = create_role_with_attached_policy_and_assume_it(role_name, trust_policy_document, attached_policy_document) + elbv2_client = boto3.client('elbv2', region_name='us-east-1', + aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken']) + ec2_client = boto3.client('ec2', region_name='us-east-1', + aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken']) + subnets = ec2_client.describe_subnets()['Subnets'] + len(subnets).should.be.greater_than(1) + elbv2_client.create_load_balancer( + Name='test-load-balancer', + Subnets=[ + subnets[0]['SubnetId'], + subnets[1]['SubnetId'] + ] + )['LoadBalancers'].should.have.length_of(1) + + +@set_initial_no_auth_action_count(3) +@mock_iam +@mock_sts +@mock_rds2 +def test_access_denied_with_temporary_credentials(): + role_name = 'test-role' + session_name = 'test-session' + trust_policy_document = { + "Version": "2012-10-17", + "Statement": { + "Effect": "Allow", + "Principal": {"AWS": "arn:aws:iam::{account_id}:root".format(account_id=ACCOUNT_ID)}, + "Action": "sts:AssumeRole" + } + } + attached_policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + 'rds:Describe*' + ], + "Resource": "*" + } + ] + } + credentials = create_role_with_inline_policy_and_assume_it(role_name, trust_policy_document, + attached_policy_document, session_name) + client = boto3.client('rds', region_name='us-east-1', + aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken']) + with assert_raises(ClientError) as ex: + client.create_db_instance( + DBInstanceIdentifier='test-db-instance', + DBInstanceClass='db.t3', + Engine='aurora-postgresql' + ) + ex.exception.response['Error']['Code'].should.equal('AccessDenied') + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(403) + ex.exception.response['Error']['Message'].should.equal( + 'User: arn:aws:sts::{account_id}:assumed-role/{role_name}/{session_name} is not authorized to perform: {operation}'.format( + account_id=ACCOUNT_ID, + role_name=role_name, + session_name=session_name, + operation="rds:CreateDBInstance" + ) + ) From 140f4110ac48df8135eeddffaca366e76bf33587 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Fri, 26 Jul 2019 20:41:40 +0200 Subject: [PATCH 30/36] set_initial_no_auth_action_count should restore request_count. --- moto/core/responses.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/moto/core/responses.py b/moto/core/responses.py index 5a6fdbf5b..fe3581800 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -129,12 +129,14 @@ class ActionAuthenticatorMixin(object): def decorator(function): def wrapper(*args, **kwargs): original_initial_no_auth_action_count = settings.INITIAL_NO_AUTH_ACTION_COUNT + original_request_count = ActionAuthenticatorMixin.request_count settings.INITIAL_NO_AUTH_ACTION_COUNT = initial_no_auth_action_count ActionAuthenticatorMixin.request_count = 0 try: result = function(*args, **kwargs) finally: settings.INITIAL_NO_AUTH_ACTION_COUNT = original_initial_no_auth_action_count + ActionAuthenticatorMixin.request_count = original_request_count return result functools.update_wrapper(wrapper, function) From f3f47d44ac296405310f9cc272cc32c1816f5d67 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Fri, 26 Jul 2019 21:05:04 +0200 Subject: [PATCH 31/36] Fixed error in python 2 and did some refactoring. --- .../{authentication.py => access_control.py} | 2 +- moto/core/responses.py | 14 ++++++------- moto/s3/responses.py | 20 +++++++++---------- 3 files changed, 18 insertions(+), 18 deletions(-) rename moto/core/{authentication.py => access_control.py} (99%) diff --git a/moto/core/authentication.py b/moto/core/access_control.py similarity index 99% rename from moto/core/authentication.py rename to moto/core/access_control.py index 2fec59808..0739fd167 100644 --- a/moto/core/authentication.py +++ b/moto/core/access_control.py @@ -127,7 +127,7 @@ class AssumedRoleAccessKey(object): class CreateAccessKeyFailure(Exception): def __init__(self, reason, *args): - super().__init__(*args) + super(CreateAccessKeyFailure, self).__init__(*args) self.reason = reason diff --git a/moto/core/responses.py b/moto/core/responses.py index fe3581800..2310dea2c 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -10,7 +10,7 @@ import io import pytz -from moto.core.authentication import IAMRequest, S3IAMRequest +from moto.core.access_control import IAMRequest, S3IAMRequest from moto.core.exceptions import DryRunClientError from jinja2 import Environment, DictLoader, TemplateNotFound @@ -110,7 +110,7 @@ class ActionAuthenticatorMixin(object): request_count = 0 - def _authenticate_action(self, iam_request_cls): + def _authenticate_and_authorize_action(self, iam_request_cls): if ActionAuthenticatorMixin.request_count >= settings.INITIAL_NO_AUTH_ACTION_COUNT: iam_request = iam_request_cls(method=self.method, path=self.path, data=self.data, headers=self.headers) iam_request.check_signature() @@ -118,11 +118,11 @@ class ActionAuthenticatorMixin(object): else: ActionAuthenticatorMixin.request_count += 1 - def _authenticate_normal_action(self): - self._authenticate_action(IAMRequest) + def _authenticate_and_authorize_normal_action(self): + self._authenticate_and_authorize_action(IAMRequest) - def _authenticate_s3_action(self): - self._authenticate_action(S3IAMRequest) + def _authenticate_and_authorize_s3_action(self): + self._authenticate_and_authorize_action(S3IAMRequest) @staticmethod def set_initial_no_auth_action_count(initial_no_auth_action_count): @@ -319,7 +319,7 @@ class BaseResponse(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): headers = self.response_headers try: - self._authenticate_normal_action() + self._authenticate_and_authorize_normal_action() except HTTPException as http_error: response = http_error.description, dict(status=http_error.code) return self._send_response(headers, response) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 2617f139d..b09ea966b 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -120,7 +120,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): def all_buckets(self): self.data["Action"] = "ListAllMyBuckets" - self._authenticate_s3_action() + self._authenticate_and_authorize_s3_action() # No bucket specified. Listing all buckets all_buckets = self.backend.get_all_buckets() @@ -266,7 +266,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): def _bucket_response_get(self, bucket_name, querystring): self._set_action("BUCKET", "GET", querystring) - self._authenticate_s3_action() + self._authenticate_and_authorize_s3_action() if 'uploads' in querystring: for unsup in ('delimiter', 'max-uploads'): @@ -500,7 +500,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): return 411, {}, "Content-Length required" self._set_action("BUCKET", "PUT", querystring) - self._authenticate_s3_action() + self._authenticate_and_authorize_s3_action() if 'versioning' in querystring: ver = re.search('([A-Za-z]+)', body.decode()) @@ -602,7 +602,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): def _bucket_response_delete(self, body, bucket_name, querystring): self._set_action("BUCKET", "DELETE", querystring) - self._authenticate_s3_action() + self._authenticate_and_authorize_s3_action() if 'policy' in querystring: self.backend.delete_bucket_policy(bucket_name, body) @@ -638,12 +638,12 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): if self.is_delete_keys(request, path, bucket_name): self.data["Action"] = "DeleteObject" - self._authenticate_s3_action() + self._authenticate_and_authorize_s3_action() return self._bucket_response_delete_keys(request, body, bucket_name) self.data["Action"] = "PutObject" - self._authenticate_s3_action() + self._authenticate_and_authorize_s3_action() # POST to bucket-url should create file from form if hasattr(request, 'form'): @@ -797,7 +797,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): def _key_response_get(self, bucket_name, query, key_name, headers): self._set_action("KEY", "GET", query) - self._authenticate_s3_action() + self._authenticate_and_authorize_s3_action() response_headers = {} if query.get('uploadId'): @@ -834,7 +834,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): def _key_response_put(self, request, body, bucket_name, query, key_name, headers): self._set_action("KEY", "PUT", query) - self._authenticate_s3_action() + self._authenticate_and_authorize_s3_action() response_headers = {} if query.get('uploadId') and query.get('partNumber'): @@ -1204,7 +1204,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): def _key_response_delete(self, bucket_name, query, key_name): self._set_action("KEY", "DELETE", query) - self._authenticate_s3_action() + self._authenticate_and_authorize_s3_action() if query.get('uploadId'): upload_id = query['uploadId'][0] @@ -1227,7 +1227,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): def _key_response_post(self, request, body, bucket_name, query, key_name): self._set_action("KEY", "POST", query) - self._authenticate_s3_action() + self._authenticate_and_authorize_s3_action() if body == b'' and 'uploads' in query: metadata = metadata_from_headers(request.headers) From de70d1787cf8b6f109199b9d7e71cdb890bf1665 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Fri, 26 Jul 2019 21:23:15 +0200 Subject: [PATCH 32/36] Collected TODOs in the header of the access_control file. --- moto/core/access_control.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/moto/core/access_control.py b/moto/core/access_control.py index 0739fd167..800b7550f 100644 --- a/moto/core/access_control.py +++ b/moto/core/access_control.py @@ -1,3 +1,17 @@ +""" +This implementation is NOT complete, there are many things to improve. +The following is a list of the most important missing features and inaccuracies. + +TODO add support for more principals, apart from IAM users and assumed IAM roles +TODO add support for the Resource and Condition parts of IAM policies +TODO add support and create tests for all services in moto (for example, API Gateway is probably not supported currently) +TODO implement service specific error messages (currently, EC2 and S3 are supported separately, everything else defaults to the errors IAM returns) +TODO include information about the action's resource in error messages (once the Resource element in IAM policies is supported) +TODO check all other actions that are performed by the action called by the user (for example, autoscaling:CreateAutoScalingGroup requires permission for iam:CreateServiceLinkedRole too - see https://docs.aws.amazon.com/autoscaling/ec2/userguide/control-access-using-iam.html) +TODO add support for resource-based policies + +""" + import json import logging import re @@ -319,8 +333,6 @@ class IAMPolicyStatement(object): if self._check_element_matches("Action", action): is_action_concerned = True - # TODO: check Resource/NotResource and Condition - if is_action_concerned: if self._statement["Effect"] == "Allow": return PermissionResult.PERMITTED From eb4a3ea90bc975f27e0d96d4c1df9bdbf0b082bb Mon Sep 17 00:00:00 2001 From: acsbendi Date: Fri, 26 Jul 2019 21:46:15 +0200 Subject: [PATCH 33/36] Added a section about IAM-like Access Control in README. --- README.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/README.md b/README.md index e4c88dec8..ff8595816 100644 --- a/README.md +++ b/README.md @@ -252,6 +252,47 @@ def test_my_model_save(): mock.stop() ``` +## IAM-like Access Control + +Moto also has the ability to authenticate and authorize actions, just like it's done by IAM in AWS. This functionality can be enabled by either setting the `INITIAL_NO_AUTH_ACTION_COUNT` environment variable or using the `set_initial_no_auth_action_count` decorator. Note that the current implementation is very basic, see [this file](https://github.com/spulec/moto/blob/master/moto/core/access_control.py) for more information. + +### `INITIAL_NO_AUTH_ACTION_COUNT` + +If this environment variable is set, moto will skip performing any authentication as many times as the variable's value, and only starts authenticating requests afterwards. If it is not set, it defaults to infinity, thus moto will never perform any authentication at all. + +### `set_initial_no_auth_action_count` + +This is a decorator that works similarly to the environment variable, but the settings are only valid in the function's scope. When the function returns, everything is restored. + +```python +@set_initial_no_auth_action_count(4) +@mock_ec2 +def test_describe_instances_allowed(): + policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "ec2:Describe*", + "Resource": "*" + } + ] + } + access_key = ... + # create access key for an IAM user/assumed role that has the policy above. + # this part should call __exactly__ 4 AWS actions, so that authentication and authorization starts exactly after this + + client = boto3.client('ec2', region_name='us-east-1', + aws_access_key_id=access_key['AccessKeyId'], + aws_secret_access_key=access_key['SecretAccessKey']) + + # if the IAM principal whose access key is used, does not have the permission to describe instances, this will fail + instances = client.describe_instances()['Reservations'][0]['Instances'] + assert len(instances) == 0 +``` + +See [the related test suite](https://github.com/spulec/moto/blob/master/tests/test_core/test_auth.py) for more examples. + ## Stand-alone Server Mode Moto also has a stand-alone server mode. This allows you to utilize From 1969338a8a9783bece7a29d06d145b0c9881b971 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Sat, 27 Jul 2019 00:12:28 +0200 Subject: [PATCH 34/36] Fixed set_initial_no_auth_action_count not working in server mode. --- moto/core/responses.py | 30 ++++++++++++++++++++++++------ moto/core/urls.py | 1 + 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/moto/core/responses.py b/moto/core/responses.py index 2310dea2c..682f02a76 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -7,6 +7,7 @@ import json import logging import re import io +import requests import pytz @@ -128,15 +129,23 @@ class ActionAuthenticatorMixin(object): def set_initial_no_auth_action_count(initial_no_auth_action_count): def decorator(function): def wrapper(*args, **kwargs): - original_initial_no_auth_action_count = settings.INITIAL_NO_AUTH_ACTION_COUNT - original_request_count = ActionAuthenticatorMixin.request_count - settings.INITIAL_NO_AUTH_ACTION_COUNT = initial_no_auth_action_count - ActionAuthenticatorMixin.request_count = 0 + if settings.TEST_SERVER_MODE: + response = requests.get("http://localhost:5000/moto-api/reset-auth") + original_initial_no_auth_action_count = response.json()['INITIAL_NO_AUTH_ACTION_COUNT'] + requests.post("http://localhost:5000/moto-api/reset-auth", data=str(initial_no_auth_action_count).encode()) + else: + original_initial_no_auth_action_count = settings.INITIAL_NO_AUTH_ACTION_COUNT + original_request_count = ActionAuthenticatorMixin.request_count + settings.INITIAL_NO_AUTH_ACTION_COUNT = initial_no_auth_action_count + ActionAuthenticatorMixin.request_count = 0 try: result = function(*args, **kwargs) finally: - settings.INITIAL_NO_AUTH_ACTION_COUNT = original_initial_no_auth_action_count - ActionAuthenticatorMixin.request_count = original_request_count + if settings.TEST_SERVER_MODE: + requests.post("http://localhost:5000/moto-api/reset-auth", data=str(original_initial_no_auth_action_count).encode()) + else: + ActionAuthenticatorMixin.request_count = original_request_count + settings.INITIAL_NO_AUTH_ACTION_COUNT = original_initial_no_auth_action_count return result functools.update_wrapper(wrapper, function) @@ -624,6 +633,15 @@ class MotoAPIResponse(BaseResponse): return 200, {}, json.dumps({"status": "ok"}) return 400, {}, json.dumps({"Error": "Need to POST to reset Moto"}) + def reset_auth_response(self, request, full_url, headers): + if request.method == "POST": + settings.INITIAL_NO_AUTH_ACTION_COUNT = float(request.data.decode()) + ActionAuthenticatorMixin.request_count = 0 + return 200, {}, json.dumps({"status": "ok"}) + elif request.method == "GET": + return 200, {}, json.dumps({"status": "ok", "INITIAL_NO_AUTH_ACTION_COUNT": str(settings.INITIAL_NO_AUTH_ACTION_COUNT)}) + return 400, {}, json.dumps({"Error": "Need to POST to reset Moto Auth"}) + def model_data(self, request, full_url, headers): from moto.core.models import model_data diff --git a/moto/core/urls.py b/moto/core/urls.py index 4d4906d77..46025221e 100644 --- a/moto/core/urls.py +++ b/moto/core/urls.py @@ -11,4 +11,5 @@ url_paths = { '{0}/moto-api/$': response_instance.dashboard, '{0}/moto-api/data.json': response_instance.model_data, '{0}/moto-api/reset': response_instance.reset_response, + '{0}/moto-api/reset-auth': response_instance.reset_auth_response, } From 62b25f9914d2229af10e186421fa0b3fb8a3c65f Mon Sep 17 00:00:00 2001 From: acsbendi Date: Sun, 28 Jul 2019 22:19:50 +0200 Subject: [PATCH 35/36] Added a few more tests to achieve better coverage. --- tests/test_core/test_auth.py | 130 +++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/tests/test_core/test_auth.py b/tests/test_core/test_auth.py index e3ce7df1d..3a1107eaa 100644 --- a/tests/test_core/test_auth.py +++ b/tests/test_core/test_auth.py @@ -553,3 +553,133 @@ def test_access_denied_with_temporary_credentials(): operation="rds:CreateDBInstance" ) ) + + +@set_initial_no_auth_action_count(3) +@mock_iam +def test_get_user_from_credentials(): + user_name = 'new-test-user' + inline_policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "iam:*", + "Resource": "*" + } + ] + } + access_key = create_user_with_access_key_and_inline_policy(user_name, inline_policy_document) + client = boto3.client('iam', region_name='us-east-1', + aws_access_key_id=access_key['AccessKeyId'], + aws_secret_access_key=access_key['SecretAccessKey']) + client.get_user()['User']['UserName'].should.equal(user_name) + + +@set_initial_no_auth_action_count(0) +@mock_s3 +def test_s3_invalid_access_key_id(): + client = boto3.client('s3', region_name='us-east-1', aws_access_key_id='invalid', aws_secret_access_key='invalid') + with assert_raises(ClientError) as ex: + client.list_buckets() + ex.exception.response['Error']['Code'].should.equal('InvalidAccessKeyId') + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(403) + ex.exception.response['Error']['Message'].should.equal('The AWS Access Key Id you provided does not exist in our records.') + + +@set_initial_no_auth_action_count(3) +@mock_s3 +@mock_iam +def test_s3_signature_does_not_match(): + bucket_name = 'test-bucket' + access_key = create_user_with_access_key() + client = boto3.client('s3', region_name='us-east-1', + aws_access_key_id=access_key['AccessKeyId'], + aws_secret_access_key='invalid') + client.create_bucket(Bucket=bucket_name) + with assert_raises(ClientError) as ex: + client.put_object(Bucket=bucket_name, Key="abc") + ex.exception.response['Error']['Code'].should.equal('SignatureDoesNotMatch') + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(403) + ex.exception.response['Error']['Message'].should.equal('The request signature we calculated does not match the signature you provided. Check your key and signing method.') + + +@set_initial_no_auth_action_count(7) +@mock_s3 +@mock_iam +def test_s3_access_denied_not_action(): + user_name = 'test-user' + bucket_name = 'test-bucket' + inline_policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "*", + "Resource": "*" + } + ] + } + group_inline_policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "NotAction": "iam:GetUser", + "Resource": "*" + } + ] + } + access_key = create_user_with_access_key_and_inline_policy(user_name, inline_policy_document) + create_group_with_inline_policy_and_add_user(user_name, group_inline_policy_document) + client = boto3.client('s3', region_name='us-east-1', + aws_access_key_id=access_key['AccessKeyId'], + aws_secret_access_key=access_key['SecretAccessKey']) + client.create_bucket(Bucket=bucket_name) + with assert_raises(ClientError) as ex: + client.delete_object(Bucket=bucket_name, Key='sdfsdf') + ex.exception.response['Error']['Code'].should.equal('AccessDenied') + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(403) + ex.exception.response['Error']['Message'].should.equal('Access Denied') + + +@set_initial_no_auth_action_count(4) +@mock_iam +@mock_sts +@mock_s3 +def test_s3_invalid_token_with_temporary_credentials(): + role_name = 'test-role' + session_name = 'test-session' + bucket_name = 'test-bucket-888' + trust_policy_document = { + "Version": "2012-10-17", + "Statement": { + "Effect": "Allow", + "Principal": {"AWS": "arn:aws:iam::{account_id}:root".format(account_id=ACCOUNT_ID)}, + "Action": "sts:AssumeRole" + } + } + attached_policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + '*' + ], + "Resource": "*" + } + ] + } + credentials = create_role_with_inline_policy_and_assume_it(role_name, trust_policy_document, + attached_policy_document, session_name) + client = boto3.client('s3', region_name='us-east-1', + aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token='invalid') + client.create_bucket(Bucket=bucket_name) + with assert_raises(ClientError) as ex: + client.list_bucket_metrics_configurations(Bucket=bucket_name) + ex.exception.response['Error']['Code'].should.equal('InvalidToken') + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(400) + ex.exception.response['Error']['Message'].should.equal('The provided token is malformed or otherwise invalid.') From 9edab5b423d29ab462284df16793f9dd78144ca9 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Sun, 28 Jul 2019 22:23:33 +0200 Subject: [PATCH 36/36] Simplified the reset-auth API. --- moto/core/responses.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/moto/core/responses.py b/moto/core/responses.py index 682f02a76..b60f10a20 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -130,9 +130,8 @@ class ActionAuthenticatorMixin(object): def decorator(function): def wrapper(*args, **kwargs): if settings.TEST_SERVER_MODE: - response = requests.get("http://localhost:5000/moto-api/reset-auth") - original_initial_no_auth_action_count = response.json()['INITIAL_NO_AUTH_ACTION_COUNT'] - requests.post("http://localhost:5000/moto-api/reset-auth", data=str(initial_no_auth_action_count).encode()) + response = requests.post("http://localhost:5000/moto-api/reset-auth", data=str(initial_no_auth_action_count).encode()) + original_initial_no_auth_action_count = response.json()['PREVIOUS_INITIAL_NO_AUTH_ACTION_COUNT'] else: original_initial_no_auth_action_count = settings.INITIAL_NO_AUTH_ACTION_COUNT original_request_count = ActionAuthenticatorMixin.request_count @@ -635,11 +634,10 @@ class MotoAPIResponse(BaseResponse): def reset_auth_response(self, request, full_url, headers): if request.method == "POST": + previous_initial_no_auth_action_count = settings.INITIAL_NO_AUTH_ACTION_COUNT settings.INITIAL_NO_AUTH_ACTION_COUNT = float(request.data.decode()) ActionAuthenticatorMixin.request_count = 0 - return 200, {}, json.dumps({"status": "ok"}) - elif request.method == "GET": - return 200, {}, json.dumps({"status": "ok", "INITIAL_NO_AUTH_ACTION_COUNT": str(settings.INITIAL_NO_AUTH_ACTION_COUNT)}) + return 200, {}, json.dumps({"status": "ok", "PREVIOUS_INITIAL_NO_AUTH_ACTION_COUNT": str(previous_initial_no_auth_action_count)}) return 400, {}, json.dumps({"Error": "Need to POST to reset Moto Auth"}) def model_data(self, request, full_url, headers):