diff --git a/.github/workflows/tests_real_aws.yml b/.github/workflows/tests_real_aws.yml new file mode 100644 index 000000000..b3fa3303a --- /dev/null +++ b/.github/workflows/tests_real_aws.yml @@ -0,0 +1,45 @@ +name: Execute tests against AWS +on: + schedule: + - cron: '00 6 * * 0' + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + permissions: + id-token: write + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Get pip cache dir + id: pip-cache + run: | + echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT + - name: pip cache + uses: actions/cache@v3 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: pip-3.11-${{ hashFiles('**/setup.cfg') }} + - name: Update pip + run: | + python -m pip install --upgrade pip + - name: Install project dependencies + run: | + pip install -r requirements-dev.txt + - name: Configure AWS + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-east-1 + role-to-assume: arn:aws:iam::682283128318:role/GithubActionsRole + - name: Test with pytest + env: + MOTO_TEST_ALLOW_AWS_REQUEST: ${{ true }} + run: | + pytest -sv tests/test_s3 -m aws_verified diff --git a/setup.cfg b/setup.cfg index 94d1f2796..8d4c9bcf0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -237,6 +237,7 @@ universal=1 markers = network: requires network connection requires_docker: requires running docker + aws_verified: Verified against AWS, and should be able to run against AWS [coverage:run] relative_files = True diff --git a/tests/test_s3/__init__.py b/tests/test_s3/__init__.py index e69de29bb..c65c7d84f 100644 --- a/tests/test_s3/__init__.py +++ b/tests/test_s3/__init__.py @@ -0,0 +1,69 @@ +import boto3 +import os +from functools import wraps +from moto import mock_s3 +from moto.s3.responses import DEFAULT_REGION_NAME +from uuid import uuid4 + + +def s3_aws_verified(func): + """ + Function that is verified to work against AWS. + Can be run against AWS at any time by setting: + MOTO_TEST_ALLOW_AWS_REQUEST=true + + If this environment variable is not set, the function runs in a `mock_s3` context. + + This decorator will: + - Create a bucket + - Run the test and pass the bucket_name as an argument + - Delete the objects and the bucket itself + """ + + @wraps(func) + def pagination_wrapper(): + client = boto3.client("s3", region_name=DEFAULT_REGION_NAME) + bucket_name = str(uuid4()) + + allow_aws_request = ( + os.environ.get("MOTO_TEST_ALLOW_AWS_REQUEST", "false").lower() == "true" + ) + + if allow_aws_request: + print(f"Test {func} will create {bucket_name}") + resp = create_bucket_and_test(bucket_name, client) + else: + with mock_s3(): + resp = create_bucket_and_test(bucket_name, client) + return resp + + def create_bucket_and_test(bucket_name, client): + client.create_bucket(Bucket=bucket_name) + client.put_bucket_tagging( + Bucket=bucket_name, + Tagging={"TagSet": [{"Key": "environment", "Value": "moto_tests"}]}, + ) + try: + resp = func(bucket_name) + finally: + ### CLEANUP ### + + versions = client.list_object_versions(Bucket=bucket_name).get( + "Versions", [] + ) + for key in versions: + client.delete_object( + Bucket=bucket_name, Key=key["Key"], VersionId=key.get("VersionId") + ) + delete_markers = client.list_object_versions(Bucket=bucket_name).get( + "DeleteMarkers", [] + ) + for key in delete_markers: + client.delete_object( + Bucket=bucket_name, Key=key["Key"], VersionId=key.get("VersionId") + ) + client.delete_bucket(Bucket=bucket_name) + + return resp + + return pagination_wrapper diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 578c938b5..7b270d0da 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -24,6 +24,8 @@ from moto.core.utils import utcnow from moto.s3.responses import DEFAULT_REGION_NAME import moto.s3.models as s3model +from . import s3_aws_verified + class MyModel: def __init__(self, name, value, metadata=None): @@ -1669,13 +1671,11 @@ def test_delete_versioned_bucket(): client.delete_bucket(Bucket="blah") -@mock_s3 -def test_delete_versioned_bucket_returns_metadata(): - # Verified against AWS - name = str(uuid4()) +@pytest.mark.aws_verified +@s3_aws_verified +def test_delete_versioned_bucket_returns_metadata(name=None): client = boto3.client("s3", region_name=DEFAULT_REGION_NAME) - client.create_bucket(Bucket=name) client.put_bucket_versioning( Bucket=name, VersioningConfiguration={"Status": "Enabled"} ) @@ -2033,11 +2033,11 @@ def test_get_bucket_cors(): assert len(resp["CORSRules"]) == 2 -@mock_s3 -def test_delete_bucket_cors(): +@pytest.mark.aws_verified +@s3_aws_verified +def test_delete_bucket_cors(bucket_name=None): s3_client = boto3.client("s3", region_name=DEFAULT_REGION_NAME) - bucket_name = "mybucket" - s3_client.create_bucket(Bucket=bucket_name) + s3_client.put_bucket_cors( Bucket=bucket_name, CORSConfiguration={