Prep release 5.0.3 (#7447)

.github/workflows/tests_real_aws.yml

@@ -42,4 +42,4 @@ jobs:
       env:
         MOTO_TEST_ALLOW_AWS_REQUEST: ${{ true }}
       run: |
-        pytest -sv tests/test_dynamodb/ tests/test_ec2/ tests/test_lakeformation/ tests/test_logs/ tests/test_ses/ tests/test_s3* tests/test_sns/ -m aws_verified
+        pytest -sv tests/test_dynamodb/ tests/test_ec2/ tests/test_iam/ tests/test_lakeformation/ tests/test_logs/ tests/test_sqs/ tests/test_ses/ tests/test_s3* tests/test_sns/ -m aws_verified

.github/workflows/tests_terraform_examples.yml

@@ -39,3 +39,31 @@ jobs:
         sleep 30
         terraform plan -detailed-exitcode
         terraform apply -destroy --auto-approve
+
+  test_create_only:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        service: ["ec2"]
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.8"
+    - name: Start MotoServer
+      run: |
+        pip install build
+        python -m build
+        docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:3.10-slim /moto/scripts/ci_moto_server.sh &
+        python scripts/ci_wait_for_server.py
+    - name: Run tests
+      run: |
+        mkdir ~/.aws && touch ~/.aws/credentials && echo -e "[default]\naws_access_key_id = test\naws_secret_access_key = test" > ~/.aws/credentials
+        cd other_langs/terraform/${{ matrix.service }}
+        terraform init
+        terraform apply --auto-approve

CHANGELOG.md

@@ -1,6 +1,54 @@
 Moto Changelog
 ==============
 
+5.0.3
+-----
+Docker Digest for 5.0.3: <autopopulateddigest>
+
+    General:
+        * New configuration options for:
+          - Passing URL's through the proxy
+          - Configuring DOcker-less services in ServerMode
+          See http://docs.getmoto.org/en/latest/docs/configuration/index.html
+
+    New Services:
+        * Route53Domains:
+            * delete_domain()
+            * list_domains()
+            * list_operations()
+            * register_domain()
+            * update_domain_nameservers()
+
+    New Methods:
+        * CostExplorer:
+            * get_cost_and_usage()
+
+        * ECR:
+            * get_registry_scanning_configuration()
+
+    Miscellaneous:
+        * ApiGateway: update_usage_plan() now supports adding apiStages
+        * Athena: get_query_execution() now returns exact OutputLocation file
+        * Autoscaling: describe_auto_scaling_groups() now supports the filters-argument
+        * CloudFront: create_distribution() now supports CustomHeaders
+        * CloudFront: update_distribution() now handles updates to DistributionConfig correctly
+        * CloudFormation - Now supports creation and deletion of AWS::EMR::Cluster
+        * CloudFormation - Now supports creation and deletion of AWS::EMR::SecurityConfiguration
+        * CloudFormation - Now supports creation and deletion of AWS::EFS::AccessPoint
+        * CloudFormation - Now supports creation and deletion of AWS::EFS::FileSystem
+        * CloudFormation - Now supports creation and deletion of AWS::EMR::InstanceGroupConfig
+        * CloudFormation - Now supports deletion of AWS::Logs::LogGroup
+        * CloudFormation: delete_stack() now handles resource dependencies better
+        * CloudWatch: put_metric_data() now supports large (compressed) requests
+        * CognitoIDP: admin_initiate_auth() and respond_to_auth_challenge() now support SMS_MFA
+        * DynamoDB: transact_write_items() now raises ValidationException when putting and deleting the same item
+        * EC2: authorize_security_group_egress/_ingress now support the TagSpecifications-argument
+        * EC2: describe_security_group_rules() now supports Tag-filters
+        * S3: EventBridge notifications are now supported for ObjectCreated:POST/COPY/MULTIPART_UPLOAD and ObjectDeleted
+        * SNS: subscribe() now adds support the `$or`, `equals-ignore-case` and `suffix` features in a FilterPolicy 
+        * SQS: send_message() should respect DelaySeconds of 0
+
+
 5.0.2
 -----
 Docker Digest for 5.0.2: _sha256:89cc6c764d714bf76e592a61f0c06fd142f672085e1dd3a53eb734aaeb4e14e2_

IMPLEMENTATION_COVERAGE.md

@@ -735,7 +735,7 @@
 
 ## ce
 <details>
-<summary>21% implemented</summary>
+<summary>20% implemented</summary>
 
 - [ ] create_anomaly_monitor
 - [ ] create_anomaly_subscription
@@ -747,6 +747,7 @@
 - [ ] get_anomalies
 - [ ] get_anomaly_monitors
 - [ ] get_anomaly_subscriptions
+- [ ] get_approximate_usage_records
 - [X] get_cost_and_usage
 - [ ] get_cost_and_usage_with_resources
 - [ ] get_cost_categories
@@ -2659,7 +2660,7 @@
 
 ## ecr
 <details>
-<summary>65% implemented</summary>
+<summary>67% implemented</summary>
 
 - [ ] batch_check_layer_availability
 - [X] batch_delete_image
@@ -3339,12 +3340,11 @@
 
 ## firehose
 <details>
-<summary>85% implemented</summary>
+<summary>100% implemented</summary>
 
 - [X] create_delivery_stream
 - [X] delete_delivery_stream
 - [X] describe_delivery_stream
-- [ ] get_kinesis_stream
 - [X] list_delivery_streams
 - [X] list_tags_for_delivery_stream
 - [X] put_record
@@ -3354,7 +3354,6 @@
 - [X] tag_delivery_stream
 - [X] untag_delivery_stream
 - [X] update_destination
-- [ ] verify_resources_exist_for_tagris
 </details>
 
 ## forecast
@@ -4922,6 +4921,7 @@
 - [ ] purchase_offering
 - [ ] reboot_input_device
 - [ ] reject_input_device_transfer
+- [ ] restart_channel_pipelines
 - [X] start_channel
 - [ ] start_input_device
 - [ ] start_input_device_maintenance_window
@@ -6393,6 +6393,46 @@
 - [ ] update_traffic_policy_instance
 </details>
 
+## route53domains
+<details>
+<summary>14% implemented</summary>
+
+- [ ] accept_domain_transfer_from_another_aws_account
+- [ ] associate_delegation_signer_to_domain
+- [ ] cancel_domain_transfer_to_another_aws_account
+- [ ] check_domain_availability
+- [ ] check_domain_transferability
+- [X] delete_domain
+- [ ] delete_tags_for_domain
+- [ ] disable_domain_auto_renew
+- [ ] disable_domain_transfer_lock
+- [ ] disassociate_delegation_signer_from_domain
+- [ ] enable_domain_auto_renew
+- [ ] enable_domain_transfer_lock
+- [ ] get_contact_reachability_status
+- [ ] get_domain_detail
+- [ ] get_domain_suggestions
+- [ ] get_operation_detail
+- [X] list_domains
+- [X] list_operations
+- [ ] list_prices
+- [ ] list_tags_for_domain
+- [ ] push_domain
+- [X] register_domain
+- [ ] reject_domain_transfer_from_another_aws_account
+- [ ] renew_domain
+- [ ] resend_contact_reachability_email
+- [ ] resend_operation_authorization
+- [ ] retrieve_domain_auth_code
+- [ ] transfer_domain
+- [ ] transfer_domain_to_another_aws_account
+- [ ] update_domain_contact
+- [ ] update_domain_contact_privacy
+- [X] update_domain_nameservers
+- [ ] update_tags_for_domain
+- [ ] view_billing
+</details>
+
 ## route53resolver
 <details>
 <summary>27% implemented</summary>
@@ -6469,7 +6509,7 @@
 
 ## s3
 <details>
-<summary>66% implemented</summary>
+<summary>67% implemented</summary>
 
 - [X] abort_multipart_upload
 - [X] complete_multipart_upload
@@ -6567,6 +6607,8 @@
 - [X] put_public_access_block
 - [ ] restore_object
 - [X] select_object_content
+- [X] upload_file
+- [X] upload_fileobj
 - [X] upload_part
 - [X] upload_part_copy
 - [ ] write_get_object_response
@@ -7945,6 +7987,7 @@
 - bedrock-runtime
 - billingconductor
 - braket
+- chatbot
 - chime
 - chime-sdk-identity
 - chime-sdk-media-pipelines
@@ -8122,7 +8165,6 @@
 - route53-recovery-cluster
 - route53-recovery-control-config
 - route53-recovery-readiness
-- route53domains
 - rum
 - s3outposts
 - sagemaker-a2i-runtime

docs/docs/services/ce.rst

@@ -38,6 +38,7 @@ ce
 - [ ] get_anomalies
 - [ ] get_anomaly_monitors
 - [ ] get_anomaly_subscriptions
+- [ ] get_approximate_usage_records
 - [X] get_cost_and_usage
   
         There is no validation yet on any of the input parameters.

docs/docs/services/cf.rst

@@ -113,7 +113,11 @@ Please let us know if you'd like support for a resource not yet listed here.
   +---------------------------------------+--------+--------+--------+----------------------------------------+
   |AWS::ECS::TaskDefinition               |    x   |    x   |        | - [ ] TaskDefinitionArn                |
   +---------------------------------------+--------+--------+--------+----------------------------------------+
-  |AWS::EFS::FileSystem                   |    x   |    x   |    x   | - [ ] FileSystemId                     |
+  |AWS::EFS::AccessPoint                  |    x   |        |        | - [ ] AccessPointId                    |
+  +---------------------------------------+--------+--------+--------+ - [ ] Arn                              |
+  |                                       |        |        |        |                                        |
+  +---------------------------------------+--------+--------+--------+----------------------------------------+
+  |AWS::EFS::FileSystem                   |    x   |        |    x   | - [ ] FileSystemId                     |
   +---------------------------------------+--------+--------+--------+ - [ ] Arn                              |
   |                                       |        |        |        |                                        |
   +---------------------------------------+--------+--------+--------+----------------------------------------+
@@ -121,7 +125,9 @@ Please let us know if you'd like support for a resource not yet listed here.
   +---------------------------------------+--------+--------+--------+ - [ ] Id                               |
   |                                       |        |        |        |                                        |
   +---------------------------------------+--------+--------+--------+----------------------------------------+
-  |AWS::EMR::Cluster                      |    x   |        |        | - [ ] MasterPublicDNS                  |
+  |AWS::EMR::Cluster                      |    x   |        |    x   | - [ ] MasterPublicDNS                  |
+  +---------------------------------------+--------+--------+--------+----------------------------------------+
+  |AWS::EMR::SecurityConfiguration        |    x   |        |    x   |                                        |
   +---------------------------------------+--------+--------+--------+----------------------------------------+
   |AWS::Events::Archive                   |    x   |    x   |        | - [x] Arn                              |
   +---------------------------------------+--------+--------+--------+----------------------------------------+
@@ -148,7 +154,7 @@ Please let us know if you'd like support for a resource not yet listed here.
   |AWS::IAM::Policy                       |        |        |        | - [ ] Id                               |
   +---------------------------------------+--------+--------+--------+----------------------------------------+
   |AWS::IAM::Role                         |    x   |        |    x   | - [x] Arn                              |
-  +---------------------------------------+--------+--------+--------+ - [ ] RoleId                           |
+  +---------------------------------------+--------+--------+--------+ - [x] RoleId                           |
   |                                       |        |        |        |                                        |
   +---------------------------------------+--------+--------+--------+----------------------------------------+
   |AWS::IAM::User                         |    x   |    x   |    x   | - [x] Arn                              |

docs/docs/services/firehose.rst

@@ -33,7 +33,6 @@ firehose
         are not currently processed/implemented.
         
 
-- [ ] get_kinesis_stream
 - [X] list_delivery_streams
   Return list of delivery streams in alphabetic order of names.
 
@@ -55,5 +54,4 @@ firehose
   Removes tags from specified delivery stream.
 
 - [X] update_destination
-- [ ] verify_resources_exist_for_tagris
 

docs/docs/services/medialive.rst

@@ -77,6 +77,7 @@ medialive
 - [ ] purchase_offering
 - [ ] reboot_input_device
 - [ ] reject_input_device_transfer
+- [ ] restart_channel_pipelines
 - [X] start_channel
 - [ ] start_input_device
 - [ ] start_input_device_maintenance_window

docs/docs/services/route53domains.rst

@@ -0,0 +1,55 @@
+.. _implementedservice_route53domains:
+
+.. |start-h3| raw:: html
+
+    <h3>
+
+.. |end-h3| raw:: html
+
+    </h3>
+
+==============
+route53domains
+==============
+
+.. autoclass:: moto.route53domains.models.Route53DomainsBackend
+
+|start-h3| Implemented features for this service |end-h3|
+
+- [ ] accept_domain_transfer_from_another_aws_account
+- [ ] associate_delegation_signer_to_domain
+- [ ] cancel_domain_transfer_to_another_aws_account
+- [ ] check_domain_availability
+- [ ] check_domain_transferability
+- [X] delete_domain
+- [ ] delete_tags_for_domain
+- [ ] disable_domain_auto_renew
+- [ ] disable_domain_transfer_lock
+- [ ] disassociate_delegation_signer_from_domain
+- [ ] enable_domain_auto_renew
+- [ ] enable_domain_transfer_lock
+- [ ] get_contact_reachability_status
+- [ ] get_domain_detail
+- [ ] get_domain_suggestions
+- [ ] get_operation_detail
+- [X] list_domains
+- [X] list_operations
+- [ ] list_prices
+- [ ] list_tags_for_domain
+- [ ] push_domain
+- [X] register_domain
+  Register a domain
+
+- [ ] reject_domain_transfer_from_another_aws_account
+- [ ] renew_domain
+- [ ] resend_contact_reachability_email
+- [ ] resend_operation_authorization
+- [ ] retrieve_domain_auth_code
+- [ ] transfer_domain
+- [ ] transfer_domain_to_another_aws_account
+- [ ] update_domain_contact
+- [ ] update_domain_contact_privacy
+- [X] update_domain_nameservers
+- [ ] update_tags_for_domain
+- [ ] view_billing
+

docs/docs/services/s3.rst

@@ -109,8 +109,11 @@ s3
          - EventBridge
 
         For the following events:
+         - 's3:ObjectCreated:CompleteMultipartUpload'
          - 's3:ObjectCreated:Copy'
+         - 's3:ObjectCreated:Post'
          - 's3:ObjectCreated:Put'
+         - 's3:ObjectDeleted'
         
 
 - [X] put_bucket_ownership_controls

moto/s3/models.py

@@ -2336,8 +2336,11 @@ class S3Backend(BaseBackend, CloudWatchMetricProvider):
          - EventBridge
 
         For the following events:
+         - 's3:ObjectCreated:CompleteMultipartUpload'
          - 's3:ObjectCreated:Copy'
+         - 's3:ObjectCreated:Post'
          - 's3:ObjectCreated:Put'
+         - 's3:ObjectDeleted'
         """
         bucket = self.get_bucket(bucket_name)
         bucket.set_notification_configuration(notification_config)
@@ -2851,6 +2854,16 @@ class S3Backend(BaseBackend, CloudWatchMetricProvider):
             for x in query_result
         ]
 
+    def upload_file(self) -> None:
+        # Listed for the implementation coverage
+        # Implementation part of responses.py
+        pass
+
+    def upload_fileobj(self) -> None:
+        # Listed for the implementation coverage
+        # Implementation part of responses.py
+        pass
+
 
 class S3BackendDict(BackendDict[S3Backend]):
     """

moto/sqs/models.py

@@ -850,15 +850,15 @@ class SQSBackend(BaseBackend):
     ) -> Message:
         queue = self.get_queue(queue_name)
 
+        self._validate_message(
+            queue, message_body, int(delay_seconds or 0), deduplication_id, group_id
+        )
+
         if delay_seconds is not None:
             delay_seconds = int(delay_seconds)
         else:
             delay_seconds = queue.delay_seconds  # type: ignore
 
-        self._validate_message(
-            queue, message_body, delay_seconds, deduplication_id, group_id
-        )
-
         message_id = str(random.uuid4())
         message = Message(message_id, message_body, system_attributes)
 

scripts/implementation_coverage.py

@@ -47,6 +47,9 @@ def calculate_extended_implementation_coverage():
         operation_names = [
             xform_name(op) for op in real_client.meta.service_model.operation_names
         ]
+        # Not part of the spec - but very much part of S3
+        if service_name == "s3":
+            operation_names.extend(["upload_file", "upload_fileobj"])
 
         for op in operation_names:
             if moto_client and op in dir(moto_client):
@@ -76,6 +79,9 @@ def calculate_implementation_coverage():
         operation_names = [
             xform_name(op) for op in real_client.meta.service_model.operation_names
         ]
+        # Not part of the spec - but very much part of S3
+        if service_name == "s3":
+            operation_names.extend(["upload_file", "upload_fileobj"])
         for op in operation_names:
             if moto_client and op in dir(moto_client):
                 implemented.append(op)

tests/test_iam/test_iam_cloudformation.py

@@ -1645,6 +1645,7 @@ Resources:
 """
 
 
+@pytest.mark.aws_verified
 @iam_aws_verified
 def test_delete_instance_profile_with_existing_role():
     region = "us-east-1"

tests/test_sqs/__init__.py

@@ -1 +1,28 @@
-# This file is intentionally left blank.
+import os
+from functools import wraps
+
+from moto import mock_aws
+
+
+def sqs_aws_verified(func):
+    """
+    Function that is verified to work against AWS.
+    Can be run against AWS at any time by setting:
+      MOTO_TEST_ALLOW_AWS_REQUEST=true
+
+    If this environment variable is not set, the function runs in a `mock_aws` context.
+    """
+
+    @wraps(func)
+    def pagination_wrapper(*args, **kwargs):
+        allow_aws_request = (
+            os.environ.get("MOTO_TEST_ALLOW_AWS_REQUEST", "false").lower() == "true"
+        )
+
+        if allow_aws_request:
+            return func(*args, **kwargs)
+        else:
+            with mock_aws():
+                return func(*args, **kwargs)
+
+    return pagination_wrapper

tests/test_sqs/test_sqs.py

@@ -23,6 +23,8 @@ from moto.sqs.models import (
 )
 from moto.utilities.distutils_version import LooseVersion
 
+from . import sqs_aws_verified
+
 TEST_POLICY = """
 {
   "Version":"2012-10-17",
@@ -3307,18 +3309,16 @@ def test_fifo_dedupe_error_no_message_dedupe_id_batch():
     )
 
 
-@mock_aws
+@sqs_aws_verified
+@pytest.mark.aws_verified
 @pytest.mark.parametrize(
     "queue_config", [{"FifoQueue": "true"}, {"FifoQueue": "true", "DelaySeconds": "10"}]
 )
 def test_send_message_delay_seconds_validation(queue_config):
     # Setup
     client = boto3.client("sqs", region_name=REGION)
-    q = "test.fifo"
+    q = f"moto_{str(uuid4())[0:6]}.fifo"
-    client.create_queue(
+    client.create_queue(QueueName=q, Attributes=queue_config)
-        QueueName=q,
-        Attributes=queue_config,
-    )
 
     # Execute
     with pytest.raises(ClientError) as err:
@@ -3330,16 +3330,6 @@ def test_send_message_delay_seconds_validation(queue_config):
             MessageDeduplicationId=str(uuid4()),
         )
 
-    # this should succeed regardless of DelaySeconds configuration on the q
-    # https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html#API_SendMessage_RequestSyntax
-    client.send_message(
-        QueueUrl=q,
-        MessageBody="test",
-        DelaySeconds=0,
-        MessageGroupId="test",
-        MessageDeduplicationId=str(uuid4()),
-    )
-
     # Verify
     ex = err.value
     assert ex.response["Error"]["Code"] == "InvalidParameterValue"
@@ -3348,5 +3338,22 @@ def test_send_message_delay_seconds_validation(queue_config):
         "The request include parameter that is not valid for this queue type."
     )
 
+    # this should succeed regardless of DelaySeconds configuration on the q
+    # https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html#API_SendMessage_RequestSyntax
+    client.send_message(
+        QueueUrl=q,
+        MessageBody="test",
+        DelaySeconds=0,
+        MessageGroupId="test",
+        MessageDeduplicationId=str(uuid4()),
+    )
+
+    client.send_message(
+        QueueUrl=q,
+        MessageBody="test",
+        MessageGroupId="test",
+        MessageDeduplicationId=str(uuid4()),
+    )
+
     # clean up for servertests
     client.delete_queue(QueueUrl=q)