From 6282e5124d726f806a54fdad9645fc236f9baf90 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 29 Jun 2022 20:12:56 +0000 Subject: [PATCH] CloudFront improvements (#5275) --- IMPLEMENTATION_COVERAGE.md | 6 +- docs/docs/services/cloudfront.rst | 7 +- moto/cloudfront/models.py | 126 ++++++++++--- moto/cloudfront/responses.py | 165 +++++++++++++----- moto/cloudfront/urls.py | 2 + tests/terraformtests/bin/run_go_test | 1 + ...h-Hardcode-endpoints-to-local-server.patch | 20 ++- .../etc/0006-CF-Reduce-wait-times.patch | 29 +++ .../terraform-tests.success.txt | 3 + .../cloudfront_test_scaffolding.py | 42 +++++ .../test_cloudfront_dist_tags.py | 21 +++ .../test_cloudfront_distributions.py | 138 ++++++++++++++- .../test_cloudfront_invalidation.py | 33 ++++ tests/test_cloudfront/test_server.py | 2 + 14 files changed, 504 insertions(+), 91 deletions(-) create mode 100644 tests/terraformtests/etc/0006-CF-Reduce-wait-times.patch create mode 100644 tests/test_cloudfront/test_cloudfront_dist_tags.py create mode 100644 tests/test_cloudfront/test_cloudfront_invalidation.py diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 22cf59c00..c921e2342 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -536,7 +536,7 @@ ## cloudfront
-5% implemented +7% implemented - [ ] associate_alias - [ ] create_cache_policy @@ -546,7 +546,7 @@ - [ ] create_field_level_encryption_config - [ ] create_field_level_encryption_profile - [ ] create_function -- [ ] create_invalidation +- [X] create_invalidation - [ ] create_key_group - [ ] create_monitoring_subscription - [ ] create_origin_request_policy @@ -613,7 +613,7 @@ - [ ] list_realtime_log_configs - [ ] list_response_headers_policies - [ ] list_streaming_distributions -- [ ] list_tags_for_resource +- [X] list_tags_for_resource - [ ] publish_function - [ ] tag_resource - [ ] test_function diff --git a/docs/docs/services/cloudfront.rst b/docs/docs/services/cloudfront.rst index 48c983208..4be6f846b 100644 --- a/docs/docs/services/cloudfront.rst +++ b/docs/docs/services/cloudfront.rst @@ -30,8 +30,7 @@ cloudfront - [ ] create_cloud_front_origin_access_identity - [X] create_distribution - This has been tested against an S3-distribution with the - simplest possible configuration. Please raise an issue if + Not all configuration options are supported yet. Please raise an issue if we're not persisting/returning the correct attributes for your use-case. @@ -40,7 +39,7 @@ cloudfront - [ ] create_field_level_encryption_config - [ ] create_field_level_encryption_profile - [ ] create_function -- [ ] create_invalidation +- [X] create_invalidation - [ ] create_key_group - [ ] create_monitoring_subscription - [ ] create_origin_request_policy @@ -116,7 +115,7 @@ cloudfront - [ ] list_realtime_log_configs - [ ] list_response_headers_policies - [ ] list_streaming_distributions -- [ ] list_tags_for_resource +- [X] list_tags_for_resource - [ ] publish_function - [ ] tag_resource - [ ] test_function diff --git a/moto/cloudfront/models.py b/moto/cloudfront/models.py index b947c0336..2215e9323 100644 --- a/moto/cloudfront/models.py +++ b/moto/cloudfront/models.py @@ -1,10 +1,12 @@ import random import string +from datetime import datetime from moto.core import get_account_id, BaseBackend, BaseModel -from moto.core.utils import BackendDict +from moto.core.utils import BackendDict, iso_8601_datetime_with_milliseconds from moto.moto_api import state_manager from moto.moto_api._internal.managed_state_model import ManagedState +from moto.utilities.tagging_service import TaggingService from uuid import uuid4 from .exceptions import ( @@ -39,9 +41,15 @@ class LambdaFunctionAssociation: class ForwardedValues: - def __init__(self): - self.query_string = "" - self.whitelisted_names = [] + def __init__(self, config): + self.query_string = config.get("QueryString", "false") + self.cookie_forward = config.get("Cookies", {}).get("Forward") or "none" + self.whitelisted_names = ( + config.get("Cookies", {}).get("WhitelistedNames", {}).get("Items") or {} + ) + self.whitelisted_names = self.whitelisted_names.get("Name") or [] + if isinstance(self.whitelisted_names, str): + self.whitelisted_names = [self.whitelisted_names] self.headers = [] self.query_string_cache_keys = [] @@ -54,22 +62,28 @@ class DefaultCacheBehaviour: self.trusted_key_groups_enabled = False self.trusted_key_groups = [] self.viewer_protocol_policy = config["ViewerProtocolPolicy"] - self.allowed_methods = ["HEAD", "GET"] - self.cached_methods = ["GET", "HEAD"] - self.smooth_streaming = True - self.compress = True + methods = config.get("AllowedMethods", {}) + self.allowed_methods = methods.get("Items", {}).get("Method", ["HEAD", "GET"]) + self.cached_methods = ( + methods.get("CachedMethods", {}) + .get("Items", {}) + .get("Method", ["GET", "HEAD"]) + ) + self.smooth_streaming = config.get("SmoothStreaming") or True + self.compress = config.get("Compress", "true").lower() == "true" self.lambda_function_associations = [] self.function_associations = [] self.field_level_encryption_id = "" - self.forwarded_values = ForwardedValues() - self.min_ttl = 0 - self.default_ttl = 0 - self.max_ttl = 0 + self.forwarded_values = ForwardedValues(config.get("ForwardedValues", {})) + self.min_ttl = config.get("MinTTL") or 0 + self.default_ttl = config.get("DefaultTTL") or 0 + self.max_ttl = config.get("MaxTTL") or 0 class Logging: def __init__(self): self.enabled = False + self.include_cookies = False class ViewerCertificate: @@ -79,6 +93,19 @@ class ViewerCertificate: self.certificate_source = "cloudfront" +class CustomOriginConfig: + def __init__(self, config): + self.http_port = config.get("HTTPPort") + self.https_port = config.get("HTTPSPort") + self.keep_alive = config.get("OriginKeepaliveTimeout") + self.protocol_policy = config.get("OriginProtocolPolicy") + self.read_timeout = config.get("OriginReadTimeout") + self.ssl_protocols = ( + config.get("OriginSslProtocols", {}).get("Items", {}).get("SslProtocol") + or [] + ) + + class Origin: def __init__(self, origin): self.id = origin["Id"] @@ -86,9 +113,9 @@ class Origin: self.custom_headers = [] self.s3_access_identity = "" self.custom_origin = None - self.origin_shield = None - self.connection_attempts = 3 - self.connection_timeout = 10 + self.origin_shield = origin.get("OriginShield") + self.connection_attempts = origin.get("ConnectionAttempts") or 3 + self.connection_timeout = origin.get("ConnectionTimeout") or 10 if "S3OriginConfig" not in origin and "CustomOriginConfig" not in origin: raise InvalidOriginServer @@ -99,22 +126,33 @@ class Origin: raise DomainNameNotAnS3Bucket self.s3_access_identity = origin["S3OriginConfig"]["OriginAccessIdentity"] + if "CustomOriginConfig" in origin: + self.custom_origin = CustomOriginConfig(origin["CustomOriginConfig"]) + + +class GeoRestrictions: + def __init__(self, config): + config = config.get("GeoRestriction") or {} + self._type = config.get("RestrictionType", "none") + self.restrictions = (config.get("Items") or {}).get("Location") or [] + class DistributionConfig: def __init__(self, config): self.config = config - self.aliases = config.get("Aliases", {}).get("Items", {}).get("CNAME", []) - self.comment = config.get("Comment", "") + self.aliases = ((config.get("Aliases") or {}).get("Items") or {}).get( + "CNAME" + ) or [] + self.comment = config.get("Comment") or "" self.default_cache_behavior = DefaultCacheBehaviour( config["DefaultCacheBehavior"] ) self.cache_behaviors = [] self.custom_error_responses = [] self.logging = Logging() - self.enabled = False + self.enabled = config.get("Enabled") or False self.viewer_certificate = ViewerCertificate() - self.geo_restriction_type = "none" - self.geo_restrictions = [] + self.geo_restriction = GeoRestrictions(config.get("Restrictions") or {}) self.caller_reference = config.get("CallerReference", str(uuid4())) self.origins = config["Origins"]["Items"]["Origin"] if not isinstance(self.origins, list): @@ -127,9 +165,10 @@ class DistributionConfig: raise OriginDoesNotExist self.origins = [Origin(o) for o in self.origins] - self.price_class = "PriceClass_All" - self.http_version = "http2" - self.is_ipv6_enabled = True + self.price_class = config.get("PriceClass", "PriceClass_All") + self.http_version = config.get("HttpVersion", "http2") + self.is_ipv6_enabled = config.get("IsIPV6Enabled", "true").lower() == "true" + self.default_root_object = config.get("DefaultRootObject") or "" class Distribution(BaseModel, ManagedState): @@ -161,29 +200,50 @@ class Distribution(BaseModel, ManagedState): self.in_progress_invalidation_batches = 0 self.has_active_trusted_key_groups = False self.domain_name = f"{Distribution.random_id(uppercase=False)}.cloudfront.net" + self.etag = Distribution.random_id() @property def location(self): return f"https://cloudfront.amazonaws.com/2020-05-31/distribution/{self.distribution_id}" + +class Invalidation(BaseModel): + @staticmethod + def random_id(uppercase=True): + ascii_set = string.ascii_uppercase if uppercase else string.ascii_lowercase + chars = list(range(10)) + list(ascii_set) + resource_id = random.choice(ascii_set) + "".join( + str(random.choice(chars)) for _ in range(12) + ) + return resource_id + + def __init__(self, distribution, paths, caller_ref): + self.invalidation_id = Invalidation.random_id() + self.create_time = iso_8601_datetime_with_milliseconds(datetime.now()) + self.distribution = distribution + self.status = "COMPLETED" + + self.paths = paths + self.caller_ref = caller_ref + @property - def etag(self): - return Distribution.random_id() + def location(self): + return self.distribution.location + f"/invalidation/{self.invalidation_id}" class CloudFrontBackend(BaseBackend): def __init__(self, region_name, account_id): super().__init__(region_name, account_id) self.distributions = dict() + self.tagger = TaggingService() state_manager.register_default_transition( "cloudfront::distribution", transition={"progression": "manual", "times": 1} ) - def create_distribution(self, distribution_config): + def create_distribution(self, distribution_config, tags): """ - This has been tested against an S3-distribution with the - simplest possible configuration. Please raise an issue if + Not all configuration options are supported yet. Please raise an issue if we're not persisting/returning the correct attributes for your use-case. """ @@ -193,6 +253,7 @@ class CloudFrontBackend(BaseBackend): if existing_dist: raise DistributionAlreadyExists(existing_dist.distribution_id) self.distributions[dist.distribution_id] = dist + self.tagger.tag_resource(dist.arn, tags) return dist, dist.location, dist.etag def get_distribution(self, distribution_id): @@ -248,6 +309,15 @@ class CloudFrontBackend(BaseBackend): dist.advance() return dist, dist.location, dist.etag + def create_invalidation(self, dist_id, paths, caller_ref): + dist, _ = self.get_distribution(dist_id) + invalidation = Invalidation(dist, paths, caller_ref) + + return invalidation + + def list_tags_for_resource(self, resource): + return self.tagger.list_tags_for_resource(resource) + cloudfront_backends = BackendDict( CloudFrontBackend, diff --git a/moto/cloudfront/responses.py b/moto/cloudfront/responses.py index 252d5b559..f5df5ff01 100644 --- a/moto/cloudfront/responses.py +++ b/moto/cloudfront/responses.py @@ -1,4 +1,5 @@ import xmltodict +from urllib.parse import unquote from moto.core.responses import BaseResponse from .models import cloudfront_backends @@ -22,11 +23,28 @@ class CloudFrontResponse(BaseResponse): if request.method == "GET": return self.list_distributions() + def invalidation(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + if request.method == "POST": + return self.create_invalidation() + + def tags(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + if request.method == "GET": + return self.list_tags_for_resource() + def create_distribution(self): params = self._get_xml_body() - distribution_config = params.get("DistributionConfig") + if "DistributionConfigWithTags" in params: + config = params.get("DistributionConfigWithTags") + tags = (config.get("Tags", {}).get("Items") or {}).get("Tag", []) + else: + config = params + tags = [] + distribution_config = config.get("DistributionConfig") distribution, location, e_tag = self.backend.create_distribution( - distribution_config=distribution_config + distribution_config=distribution_config, + tags=tags, ) template = self.response_template(CREATE_DISTRIBUTION_TEMPLATE) response = template.render(distribution=distribution, xmlns=XMLNS) @@ -69,6 +87,25 @@ class CloudFrontResponse(BaseResponse): headers = {"ETag": e_tag, "Location": location} return 200, headers, response + def create_invalidation(self): + dist_id = self.path.split("/")[-2] + params = self._get_xml_body()["InvalidationBatch"] + paths = ((params.get("Paths") or {}).get("Items") or {}).get("Path") or [] + caller_ref = params.get("CallerReference") + + invalidation = self.backend.create_invalidation(dist_id, paths, caller_ref) + template = self.response_template(CREATE_INVALIDATION_TEMPLATE) + response = template.render(invalidation=invalidation, xmlns=XMLNS) + + return 200, {"Location": invalidation.location}, response + + def list_tags_for_resource(self): + resource = unquote(self._get_param("Resource")) + tags = self.backend.list_tags_for_resource(resource=resource)["Tags"] + template = self.response_template(TAGS_TEMPLATE) + response = template.render(tags=tags, xmlns=XMLNS) + return 200, {}, response + DIST_META_TEMPLATE = """ {{ distribution.distribution_id }} @@ -90,7 +127,7 @@ DIST_CONFIG_TEMPLATE = """ {% endfor %} - {{ distribution.distribution_config.default_distribution_object }} + {{ distribution.distribution_config.default_root_object }} {{ distribution.distribution_config.origins|length }} @@ -108,32 +145,34 @@ DIST_CONFIG_TEMPLATE = """ {% endfor %} + {% if origin.s3_access_identity %} {{ origin.s3_access_identity }} + {% endif %} {% if origin.custom_origin %} {{ origin.custom_origin.http_port }} {{ origin.custom_origin.https_port }} - {{ OriginProtocolPolicy }} + {{ origin.custom_origin.protocol_policy }} - {{ origin.custom_origin.origin_ssl_protocols.quantity }} + {{ origin.custom_origin.ssl_protocols|length }} - {% for protocol in origin.custom_origin.origin_ssl_protocols %} - {{ protocol }} + {% for protocol in origin.custom_origin.ssl_protocols %} + {{ protocol }} {% endfor %} - {{ origin.custom_origin.origin_read_timeout }} - {{ origin.custom_origin.origin_keepalive_timeout }} + {{ origin.custom_origin.read_timeout }} + {{ origin.custom_origin.keep_alive }} {% endif %} {{ origin.connection_attempts }} {{ origin.connection_timeout }} {% if origin.origin_shield %} - {{ origin.origin_shield.enabled }} - {{ OriginShieldRegion }} + {{ origin.origin_shield.get("Enabled") }} + {{ origin.origin_shield.get("OriginShieldRegion") }} {% else %} @@ -175,7 +214,7 @@ DIST_CONFIG_TEMPLATE = """ {{ distribution.distribution_config.default_cache_behavior.target_origin_id }} - {{ distribution.distribution_config.default_cache_behavior.trusted_signers.enabled }} + {{ distribution.distribution_config.default_cache_behavior.trusted_signers_enabled }} {{ distribution.distribution_config.default_cache_behavior.trusted_signers|length }} {% for aws_account_number in distribution.distribution_config.default_cache_behavior.trusted_signers %} @@ -197,14 +236,14 @@ DIST_CONFIG_TEMPLATE = """ {{ distribution.distribution_config.default_cache_behavior.allowed_methods|length }} {% for method in distribution.distribution_config.default_cache_behavior.allowed_methods %} - {{ method }} + {{ method }} {% endfor %} {{ distribution.distribution_config.default_cache_behavior.cached_methods|length }} {% for method in distribution.distribution_config.default_cache_behavior.cached_methods %} - {{ method }} + {{ method }} {% endfor %} @@ -242,7 +281,7 @@ DIST_CONFIG_TEMPLATE = """ {{ distribution.distribution_config.default_cache_behavior.forwarded_values.query_string }} - {{ ItemSelection }} + {{ distribution.distribution_config.default_cache_behavior.forwarded_values.cookie_forward }} {{ distribution.distribution_config.default_cache_behavior.forwarded_values.whitelisted_names|length }} @@ -277,7 +316,7 @@ DIST_CONFIG_TEMPLATE = """ {{ distribution.distribution_config.cache_behaviors|length }} {% if distribution.distribution_config.cache_behaviors %} - {% for behaviour in distribution.distribution_config.cache_behaviors %} + {% for behaviour in distribution.distribution_config.cache_behaviors %} {{ behaviour.path_pattern }} {{ behaviour.target_origin_id }} @@ -290,33 +329,33 @@ DIST_CONFIG_TEMPLATE = """ - {{ cache_behavior_list.trusted_key_groups.enabled }} - {{ cache_behavior_list.trusted_key_groups.quantity }} + {{ behaviour.trusted_key_groups.enabled }} + {{ behaviour.trusted_key_groups.quantity }} - {% for trusted_key_group_id_list in cache_behavior_list.trusted_key_groups.TrustedKeyGroupIdList %} + {% for trusted_key_group_id_list in behaviour.trusted_key_groups.TrustedKeyGroupIdList %} {{ trusted_key_group_id_list.key_group }} {% endfor %} {{ ViewerProtocolPolicy }} - {{ cache_behavior_list.allowed_methods.quantity }} + {{ behaviour.allowed_methods.quantity }} - {% for methods_list in cache_behavior_list.allowed_methods.MethodsList %}{{ Method }}{% endfor %} + {% for methods_list in behaviour.allowed_methods.MethodsList %}{{ Method }}{% endfor %} - {{ cache_behavior_list.allowed_methods.cached_methods.quantity }} + {{ behaviour.allowed_methods.cached_methods.quantity }} - {% for methods_list in cache_behavior_list.allowed_methods.cached_methods.MethodsList %}{{ Method }}{% endfor %} + {% for methods_list in behaviour.allowed_methods.cached_methods.MethodsList %}{{ Method }}{% endfor %} - {{ cache_behavior_list.smooth_streaming }} - {{ cache_behavior_list.compress }} + {{ behaviour.smooth_streaming }} + {{ behaviour.compress }} - {{ cache_behavior_list.lambda_function_associations.quantity }} + {{ behaviour.lambda_function_associations.quantity }} - {% for lambda_function_association_list in cache_behavior_list.lambda_function_associations.LambdaFunctionAssociationList %} + {% for lambda_function_association_list in behaviour.lambda_function_associations.LambdaFunctionAssociationList %} {{ LambdaFunctionARN }} {{ EventType }} {{ lambda_function_association_list.include_body }} @@ -324,52 +363,52 @@ DIST_CONFIG_TEMPLATE = """ - {{ cache_behavior_list.function_associations.quantity }} + {{ behaviour.function_associations.quantity }} - {% for function_association_list in cache_behavior_list.function_associations.FunctionAssociationList %} + {% for function_association_list in behaviour.function_associations.FunctionAssociationList %} {{ FunctionARN }} {{ EventType }} {% endfor %} - {{ cache_behavior_list.field_level_encryption_id }} - {{ cache_behavior_list.realtime_log_config_arn }} - {{ cache_behavior_list.cache_policy_id }} - {{ cache_behavior_list.origin_request_policy_id }} - {{ cache_behavior_list.response_headers_policy_id }} + {{ behaviour.field_level_encryption_id }} + {{ behaviour.realtime_log_config_arn }} + {{ behaviour.cache_policy_id }} + {{ behaviour.origin_request_policy_id }} + {{ behaviour.response_headers_policy_id }} - {{ cache_behavior_list.forwarded_values.query_string }} + {{ behaviour.forwarded_values.query_string }} {{ ItemSelection }} - {{ cache_behavior_list.forwarded_values.cookies.whitelisted_names.quantity }} + {{ behaviour.forwarded_values.cookies.whitelisted_names.quantity }} - {% for cookie_name_list in cache_behavior_list.forwarded_values.cookies.whitelisted_names.CookieNameList %} + {% for cookie_name_list in behaviour.forwarded_values.cookies.whitelisted_names.CookieNameList %} {{ cookie_name_list.name }} {% endfor %} - {{ cache_behavior_list.forwarded_values.headers.quantity }} + {{ behaviour.forwarded_values.headers.quantity }} - {% for header_list in cache_behavior_list.forwarded_values.headers.HeaderList %} + {% for header_list in behaviour.forwarded_values.headers.HeaderList %} {{ header_list.name }} {% endfor %} - {{ cache_behavior_list.forwarded_values.query_string_cache_keys.quantity }} + {{ behaviour.forwarded_values.query_string_cache_keys.quantity }} - {% for query_string_cache_keys_list in cache_behavior_list.forwarded_values.query_string_cache_keys.QueryStringCacheKeysList %} + {% for query_string_cache_keys_list in behaviour.forwarded_values.query_string_cache_keys.QueryStringCacheKeysList %} {{ query_string_cache_keys_list.name }} {% endfor %} - {{ cache_behavior_list.min_ttl }} - {{ cache_behavior_list.default_ttl }} - {{ cache_behavior_list.max_ttl }} + {{ behaviour.min_ttl }} + {{ behaviour.default_ttl }} + {{ behaviour.max_ttl }} {% endfor %} {% endif %} @@ -407,11 +446,11 @@ DIST_CONFIG_TEMPLATE = """ - {{ distribution.distribution_config.geo_restriction_type }} - {{ distribution.distribution_config.geo_restrictions|length }} - {% if distribution.distribution_config.geo_restrictions %} + {{ distribution.distribution_config.geo_restriction._type }} + {{ distribution.distribution_config.geo_restriction.restrictions|length }} + {% if distribution.distribution_config.geo_restriction.restrictions %} - {% for location in distribution.distribution_config.geo_restrictions %} + {% for location in distribution.distribution_config.geo_restriction.restrictions %} {{ location }} {% endfor %} @@ -528,3 +567,33 @@ UPDATE_DISTRIBUTION_TEMPLATE = ( """ ) + +CREATE_INVALIDATION_TEMPLATE = """ + + {{ invalidation.invalidation_id }} + {{ invalidation.status }} + {{ invalidation.create_time }} + + {{ invalidation.caller_ref }} + + {{ invalidation.paths|length }} + + {% for path in invalidation.paths %}{{ path }}{% endfor %} + + + + +""" + +TAGS_TEMPLATE = """ + + + {% for tag in tags %} + + {{ tag["Key"] }} + {{ tag["Value"] }} + + {% endfor %} + + +""" diff --git a/moto/cloudfront/urls.py b/moto/cloudfront/urls.py index 9f165d39b..a65a5776c 100644 --- a/moto/cloudfront/urls.py +++ b/moto/cloudfront/urls.py @@ -11,4 +11,6 @@ url_paths = { "{0}/2020-05-31/distribution$": response.distributions, "{0}/2020-05-31/distribution/(?P[^/]+)$": response.individual_distribution, "{0}/2020-05-31/distribution/(?P[^/]+)/config$": response.update_distribution, + "{0}/2020-05-31/distribution/(?P[^/]+)/invalidation": response.invalidation, + "{0}/2020-05-31/tagging$": response.tags, } diff --git a/tests/terraformtests/bin/run_go_test b/tests/terraformtests/bin/run_go_test index 5b1e3b889..99654ed34 100755 --- a/tests/terraformtests/bin/run_go_test +++ b/tests/terraformtests/bin/run_go_test @@ -11,6 +11,7 @@ PATCH="etc/0001-Patch-Hardcode-endpoints-to-local-server.patch" (git apply $pwd/etc/0003-Patch-IAM-wait-times.patch > /dev/null 2>&1 && echo "Patched IAM") || echo "Not patching IAM - Directory was probably already patched." (git apply $pwd/etc/0004-DAX-Reduce-wait-times.patch > /dev/null 2>&1 && echo "Patched DAX") || echo "Not patching DAX - Directory was probably already patched." (git apply $pwd/etc/0005-Route53-Reduce-wait-times.patch > /dev/null 2>&1 && echo "Patched Route53") || echo "Not patching Route53 - Directory was probably already patched." +(git apply $pwd/etc/0006-CF-Reduce-wait-times.patch > /dev/null 2>&1 && echo "Patched CF") || echo "Not patching CF - Directory was probably already patched." ) ( diff --git a/tests/terraformtests/etc/0001-Patch-Hardcode-endpoints-to-local-server.patch b/tests/terraformtests/etc/0001-Patch-Hardcode-endpoints-to-local-server.patch index ca32abf4b..c1d8aebf9 100644 --- a/tests/terraformtests/etc/0001-Patch-Hardcode-endpoints-to-local-server.patch +++ b/tests/terraformtests/etc/0001-Patch-Hardcode-endpoints-to-local-server.patch @@ -1,12 +1,12 @@ -From 64093955e96cff42a797880b4a6921663af6040d Mon Sep 17 00:00:00 2001 +From 91f2d9c33f241cb9dfb3700eaa739a46a06a29bc Mon Sep 17 00:00:00 2001 From: Bert Blommers -Date: Sun, 19 Jun 2022 19:32:26 +0000 +Date: Wed, 29 Jun 2022 16:24:04 +0000 Subject: [PATCH] Patch: Hardcode endpoints --- internal/conns/config.go | 15 +++++++++++++++ - internal/provider/provider.go | 2 +- - 2 files changed, 16 insertions(+), 1 deletion(-) + internal/provider/provider.go | 4 ++-- + 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/internal/conns/config.go b/internal/conns/config.go index 7bfd3100fd..b59083068a 100644 @@ -37,10 +37,10 @@ index 7bfd3100fd..b59083068a 100644 AccessKey: c.AccessKey, APNInfo: StdUserAgentProducts(c.TerraformVersion), diff --git a/internal/provider/provider.go b/internal/provider/provider.go -index 7e6200d9ac..7005caccd3 100644 +index 7e6200d9ac..98d8c4fccc 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go -@@ -2082,7 +2082,7 @@ func providerConfigure(ctx context.Context, d *schema.ResourceData, terraformVer +@@ -2082,14 +2082,14 @@ func providerConfigure(ctx context.Context, d *schema.ResourceData, terraformVer CustomCABundle: d.Get("custom_ca_bundle").(string), EC2MetadataServiceEndpoint: d.Get("ec2_metadata_service_endpoint").(string), EC2MetadataServiceEndpointMode: d.Get("ec2_metadata_service_endpoint_mode").(string), @@ -49,6 +49,14 @@ index 7e6200d9ac..7005caccd3 100644 HTTPProxy: d.Get("http_proxy").(string), IgnoreTagsConfig: expandProviderIgnoreTags(d.Get("ignore_tags").([]interface{})), Insecure: d.Get("insecure").(bool), + MaxRetries: d.Get("max_retries").(int), + Profile: d.Get("profile").(string), + Region: d.Get("region").(string), +- S3UsePathStyle: d.Get("s3_use_path_style").(bool) || d.Get("s3_force_path_style").(bool), ++ S3UsePathStyle: true, + SecretKey: d.Get("secret_key").(string), + SkipCredsValidation: d.Get("skip_credentials_validation").(bool), + SkipGetEC2Platforms: d.Get("skip_get_ec2_platforms").(bool), -- 2.25.1 diff --git a/tests/terraformtests/etc/0006-CF-Reduce-wait-times.patch b/tests/terraformtests/etc/0006-CF-Reduce-wait-times.patch new file mode 100644 index 000000000..bb8354a2c --- /dev/null +++ b/tests/terraformtests/etc/0006-CF-Reduce-wait-times.patch @@ -0,0 +1,29 @@ +From bc72f0c3ec4a4d099d6d9c9ab8bb5c839957378f Mon Sep 17 00:00:00 2001 +From: Bert Blommers +Date: Wed, 29 Jun 2022 16:25:09 +0000 +Subject: [PATCH] Patch CF timings + +--- + internal/service/cloudfront/distribution.go | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/internal/service/cloudfront/distribution.go b/internal/service/cloudfront/distribution.go +index 3d34f2cf1c..17e7e17014 100644 +--- a/internal/service/cloudfront/distribution.go ++++ b/internal/service/cloudfront/distribution.go +@@ -1185,9 +1185,9 @@ func DistributionWaitUntilDeployed(id string, meta interface{}) error { + Pending: []string{"InProgress"}, + Target: []string{"Deployed"}, + Refresh: resourceWebDistributionStateRefreshFunc(id, meta), +- Timeout: 90 * time.Minute, +- MinTimeout: 15 * time.Second, +- Delay: 1 * time.Minute, ++ Timeout: 1 * time.Minute, ++ MinTimeout: 5 * time.Second, ++ Delay: 10 * time.Second, + } + + _, err := stateConf.WaitForState() +-- +2.25.1 + diff --git a/tests/terraformtests/terraform-tests.success.txt b/tests/terraformtests/terraform-tests.success.txt index 8f86036cf..30b0ba3ae 100644 --- a/tests/terraformtests/terraform-tests.success.txt +++ b/tests/terraformtests/terraform-tests.success.txt @@ -17,6 +17,9 @@ batch: - TestAccBatchJobDefinition ce: - TestAccCECostCategory +cloudfront: + - TestAccCloudFrontDistributionDataSource_basic + - TestAccCloudFrontDistribution_isIPV6Enabled cloudtrail: - TestAccCloudTrailServiceAccount cloudwatch: diff --git a/tests/test_cloudfront/cloudfront_test_scaffolding.py b/tests/test_cloudfront/cloudfront_test_scaffolding.py index 690d121d0..365586a2d 100644 --- a/tests/test_cloudfront/cloudfront_test_scaffolding.py +++ b/tests/test_cloudfront/cloudfront_test_scaffolding.py @@ -25,3 +25,45 @@ def example_distribution_config(ref): "Comment": "an optional comment that's not actually optional", "Enabled": False, } + + +def example_dist_config_with_tags(ref): + config = example_distribution_config(ref) + config["Tags"] = { + "Items": [{"Key": "k1", "Value": "v1"}, {"Key": "k2", "Value": "v2"}] + } + return config + + +def example_dist_custom_config(ref): + return { + "CallerReference": ref, + "Origins": { + "Quantity": 1, + "Items": [ + { + "Id": "origin1", + "DomainName": "asdf.s3.us-east-1.amazonaws.com", + "CustomOriginConfig": { + "HTTPPort": 80, + "HTTPSPort": 443, + "OriginKeepaliveTimeout": 5, + "OriginProtocolPolicy": "http-only", + "OriginReadTimeout": 30, + "OriginSslProtocols": { + "Quantity": 2, + "Items": ["TLSv1", "SSLv3"], + }, + }, + } + ], + }, + "DefaultCacheBehavior": { + "TargetOriginId": "origin1", + "ViewerProtocolPolicy": "allow-all", + "MinTTL": 10, + "ForwardedValues": {"QueryString": False, "Cookies": {"Forward": "none"}}, + }, + "Comment": "an optional comment that's not actually optional", + "Enabled": False, + } diff --git a/tests/test_cloudfront/test_cloudfront_dist_tags.py b/tests/test_cloudfront/test_cloudfront_dist_tags.py new file mode 100644 index 000000000..40b314f2e --- /dev/null +++ b/tests/test_cloudfront/test_cloudfront_dist_tags.py @@ -0,0 +1,21 @@ +import boto3 +from moto import mock_cloudfront +from . import cloudfront_test_scaffolding as scaffold +import sure # noqa # pylint: disable=unused-import + + +@mock_cloudfront +def test_create_distribution_with_tags(): + client = boto3.client("cloudfront", region_name="us-west-1") + config = scaffold.example_distribution_config("ref") + tags = {"Items": [{"Key": "k1", "Value": "v1"}, {"Key": "k2", "Value": "v2"}]} + config = {"DistributionConfig": config, "Tags": tags} + + resp = client.create_distribution_with_tags(DistributionConfigWithTags=config) + resp.should.have.key("Distribution") + + resp = client.list_tags_for_resource(Resource=resp["Distribution"]["ARN"]) + resp.should.have.key("Tags") + resp["Tags"].should.have.key("Items").length_of(2) + resp["Tags"]["Items"].should.contain({"Key": "k1", "Value": "v1"}) + resp["Tags"]["Items"].should.contain({"Key": "k2", "Value": "v2"}) diff --git a/tests/test_cloudfront/test_cloudfront_distributions.py b/tests/test_cloudfront/test_cloudfront_distributions.py index 5986a8a9f..e5b6cc587 100644 --- a/tests/test_cloudfront/test_cloudfront_distributions.py +++ b/tests/test_cloudfront/test_cloudfront_distributions.py @@ -127,11 +127,105 @@ def test_create_distribution_s3_minimum(): @mock_cloudfront -def test_create_distribution_with_additional_fields(): +def test_create_distribution_with_georestriction(): + client = boto3.client("cloudfront", region_name="us-west-1") + config = scaffold.example_distribution_config("ref") + config["Restrictions"] = { + "GeoRestriction": { + "RestrictionType": "whitelist", + "Quantity": 2, + "Items": ["GB", "US"], + } + } + + resp = client.create_distribution(DistributionConfig=config) + resp.should.have.key("Distribution") + + distribution = resp["Distribution"] + + distribution.should.have.key("DistributionConfig") + config = distribution["DistributionConfig"] + + config.should.have.key("Restrictions") + config["Restrictions"].should.have.key("GeoRestriction") + restriction = config["Restrictions"]["GeoRestriction"] + restriction.should.have.key("RestrictionType").equals("whitelist") + restriction.should.have.key("Quantity").equals(2) + restriction["Items"].should.contain("US") + restriction["Items"].should.contain("GB") + + +@mock_cloudfront +def test_create_distribution_with_allowed_methods(): + client = boto3.client("cloudfront", region_name="us-west-1") + config = scaffold.example_distribution_config("ref") + config["DefaultCacheBehavior"]["AllowedMethods"] = { + "Quantity": 3, + "Items": ["GET", "HEAD", "PUT"], + "CachedMethods": { + "Quantity": 7, + "Items": ["GET", "HEAD", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"], + }, + } + + dist = client.create_distribution(DistributionConfig=config)["Distribution"] + + dist.should.have.key("DistributionConfig") + cache = dist["DistributionConfig"]["DefaultCacheBehavior"] + + cache.should.have.key("AllowedMethods").equals( + { + "CachedMethods": { + "Items": ["GET", "HEAD", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"], + "Quantity": 7, + }, + "Items": ["GET", "HEAD", "PUT"], + "Quantity": 3, + } + ) + + +@mock_cloudfront +def test_create_distribution_with_origins(): + client = boto3.client("cloudfront", region_name="us-west-1") + config = scaffold.example_distribution_config("ref") + config["Origins"]["Items"][0]["ConnectionAttempts"] = 1 + config["Origins"]["Items"][0]["ConnectionTimeout"] = 2 + config["Origins"]["Items"][0]["OriginShield"] = { + "Enabled": True, + "OriginShieldRegion": "east", + } + + dist = client.create_distribution(DistributionConfig=config)["Distribution"] + + origin = dist["DistributionConfig"]["Origins"]["Items"][0] + origin.should.have.key("ConnectionAttempts").equals(1) + origin.should.have.key("ConnectionTimeout").equals(2) + origin.should.have.key("OriginShield").equals( + {"Enabled": True, "OriginShieldRegion": "east"} + ) + + +@mock_cloudfront +@pytest.mark.parametrize("compress", [True, False]) +@pytest.mark.parametrize("qs", [True, False]) +@pytest.mark.parametrize("smooth", [True, False]) +@pytest.mark.parametrize("ipv6", [True, False]) +def test_create_distribution_with_additional_fields(compress, qs, smooth, ipv6): client = boto3.client("cloudfront", region_name="us-west-1") config = scaffold.example_distribution_config("ref") + config["IsIPV6Enabled"] = ipv6 config["Aliases"] = {"Quantity": 2, "Items": ["alias1", "alias2"]} + config["DefaultCacheBehavior"]["ForwardedValues"]["Cookies"] = { + "Forward": "whitelist", + "WhitelistedNames": {"Quantity": 1, "Items": ["x-amz-header"]}, + } + config["DefaultCacheBehavior"]["ForwardedValues"]["QueryString"] = qs + config["DefaultCacheBehavior"]["Compress"] = compress + config["DefaultCacheBehavior"]["MinTTL"] = 10 + config["DefaultCacheBehavior"]["SmoothStreaming"] = smooth + config["PriceClass"] = "PriceClass_100" resp = client.create_distribution(DistributionConfig=config) distribution = resp["Distribution"] distribution.should.have.key("DistributionConfig") @@ -140,6 +234,21 @@ def test_create_distribution_with_additional_fields(): {"Items": ["alias1", "alias2"], "Quantity": 2} ) + config.should.have.key("PriceClass").equals("PriceClass_100") + config.should.have.key("IsIPV6Enabled").equals(ipv6) + + config["DefaultCacheBehavior"].should.have.key("Compress").equals(compress) + config["DefaultCacheBehavior"].should.have.key("MinTTL").equals(10) + config["DefaultCacheBehavior"].should.have.key("SmoothStreaming").equals(smooth) + + forwarded = config["DefaultCacheBehavior"]["ForwardedValues"] + forwarded.should.have.key("QueryString").equals(qs) + forwarded["Cookies"].should.have.key("Forward").equals("whitelist") + forwarded["Cookies"].should.have.key("WhitelistedNames") + forwarded["Cookies"]["WhitelistedNames"].should.have.key("Items").equals( + ["x-amz-header"] + ) + @mock_cloudfront def test_create_distribution_returns_etag(): @@ -179,7 +288,9 @@ def test_create_distribution_needs_unique_caller_reference(): dist2 = client.create_distribution(DistributionConfig=config) dist1_id.shouldnt.equal(dist2["Distribution"]["Id"]) - # TODO: Verify two exist, using the list_distributions method + resp = client.list_distributions()["DistributionList"] + resp.should.have.key("Quantity").equals(2) + resp.should.have.key("Items").length_of(2) @mock_cloudfront @@ -277,6 +388,29 @@ def test_create_distribution_with_invalid_s3_bucket(): ) +@mock_cloudfront +def test_create_distribution_custom_config(): + client = boto3.client("cloudfront", region_name="us-west-1") + config = scaffold.example_dist_custom_config("ref") + + dist = client.create_distribution(DistributionConfig=config)["Distribution"][ + "DistributionConfig" + ] + dist.should.have.key("Origins") + dist["Origins"].should.have.key("Items").length_of(1) + origin = dist["Origins"]["Items"][0] + + origin.should.have.key("CustomOriginConfig") + custom_config = origin["CustomOriginConfig"] + + custom_config.should.have.key("HTTPPort").equals(80) + custom_config.should.have.key("HTTPSPort").equals(443) + custom_config.should.have.key("OriginProtocolPolicy").equals("http-only") + custom_config.should.have.key("OriginSslProtocols").equals( + {"Items": ["TLSv1", "SSLv3"], "Quantity": 2} + ) + + @mock_cloudfront def test_list_distributions_without_any(): client = boto3.client("cloudfront", region_name="us-east-1") diff --git a/tests/test_cloudfront/test_cloudfront_invalidation.py b/tests/test_cloudfront/test_cloudfront_invalidation.py new file mode 100644 index 000000000..43f7c7583 --- /dev/null +++ b/tests/test_cloudfront/test_cloudfront_invalidation.py @@ -0,0 +1,33 @@ +import boto3 +from moto import mock_cloudfront +from . import cloudfront_test_scaffolding as scaffold +import sure # noqa # pylint: disable=unused-import + + +@mock_cloudfront +def test_create_invalidation(): + client = boto3.client("cloudfront", region_name="us-west-1") + config = scaffold.example_distribution_config("ref") + resp = client.create_distribution(DistributionConfig=config) + dist_id = resp["Distribution"]["Id"] + + resp = client.create_invalidation( + DistributionId=dist_id, + InvalidationBatch={ + "Paths": {"Quantity": 2, "Items": ["/path1", "/path2"]}, + "CallerReference": "ref2", + }, + ) + + resp.should.have.key("Location") + resp.should.have.key("Invalidation") + + resp["Invalidation"].should.have.key("Id") + resp["Invalidation"].should.have.key("Status").equals("COMPLETED") + resp["Invalidation"].should.have.key("CreateTime") + resp["Invalidation"].should.have.key("InvalidationBatch").equals( + { + "Paths": {"Quantity": 2, "Items": ["/path1", "/path2"]}, + "CallerReference": "ref2", + } + ) diff --git a/tests/test_cloudfront/test_server.py b/tests/test_cloudfront/test_server.py index 8e60ceadb..0cee4ea5b 100644 --- a/tests/test_cloudfront/test_server.py +++ b/tests/test_cloudfront/test_server.py @@ -1,9 +1,11 @@ import sure # noqa # pylint: disable=unused-import import xmltodict +from moto import mock_cloudfront import moto.server as server +@mock_cloudfront def test_cloudfront_list(): backend = server.create_backend_app("cloudfront") test_client = backend.test_client()