Move to pyproject.toml, instead of setup.py (#5821)

.github/workflows/build.yml

@@ -25,7 +25,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ${{ steps.pip-cache-dir.outputs.dir }}
-          key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}-4
+          key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}-4
       - name: Update pip
         if: ${{ steps.pip-cache.outputs.cache-hit != 'true' }}
         run: |
@@ -57,7 +57,7 @@ jobs:
       uses: actions/cache@v3
       with:
         path: ${{ steps.pip-cache.outputs.dir }}
-        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}-4
+        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
     # Update PIP
     - name: Update pip
       run: |
@@ -95,7 +95,7 @@ jobs:
       uses: actions/cache@v3
       with:
         path: ${{ steps.pip-cache.outputs.dir }}
-        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}-4
+        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
     - name: Update pip
       run: |
         python -m pip install --upgrade pip
@@ -133,7 +133,8 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Start MotoServer
       run: |
-        python setup.py sdist
+        pip install build
+        python -m build
         docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh &
         python scripts/ci_wait_for_server.py
     - name: Get pip cache dir
@@ -144,7 +145,7 @@ jobs:
       uses: actions/cache@v3
       with:
         path: ${{ steps.pip-cache.outputs.dir }}
-        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}-4
+        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
     - name: Update pip
       run: |
         python -m pip install --upgrade pip
@@ -196,9 +197,9 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Update & Build
         run: |
-          pip install wheel packaging
+          pip install build
           python update_version_from_git.py
-          python setup.py sdist bdist_wheel
+          python -m build
       - name: Publish to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:

.github/workflows/dockertests.yml

@@ -53,7 +53,8 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Start MotoServer on an unusual port
       run: |
-        python setup.py sdist
+        pip install --upgrade build
+        python -m build
         docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e MOTO_PORT=4555 -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 4555:4555 -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh &
         MOTO_PORT=4555 python scripts/ci_wait_for_server.py
     - name: Get pip cache dir
@@ -111,7 +112,8 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Start MotoServer on a custom Docker network bridge
       run: |
-        python setup.py sdist
+        pip install --upgrade build
+        python -m build
         docker network create -d bridge my-custom-network
         docker run --rm -t -e TEST_SERVER_MODE=true -e MOTO_DOCKER_NETWORK_NAME=my-custom-network -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 --network my-custom-network -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh &
         python scripts/ci_wait_for_server.py
@@ -168,7 +170,8 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Start MotoServer on an unusual port
       run: |
-        python setup.py sdist
+        pip install --upgrade build
+        python -m build
         docker run --rm -t -e MOTO_DOCKER_NETWORK_MODE=host -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e MOTO_PORT=4555 -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 4555:4555 -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh &
         MOTO_PORT=4555 python scripts/ci_wait_for_server.py
     - name: Get pip cache dir

.github/workflows/release.yml

@@ -27,7 +27,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install wheel setuptools packaging twine --upgrade
+        pip install wheel setuptools packaging twine build --upgrade
     - name: Verify Tag does not exist
       run: |
         ! git rev-parse ${{ env.VERSION }} || { echo "Ensure that no tag exists for ${{ env.VERSION }}" ; exit 1; }
@@ -37,7 +37,7 @@ jobs:
     - name: Set version number
       run: python update_version_from_git.py ${{ env.VERSION }}
     - name: Build
-      run: python setup.py sdist bdist_wheel
+      run: python -m build
     - name: Publish to PyPI
       uses: pypa/gh-action-pypi-publish@release/v1
       with:

.github/workflows/test_outdated_versions.yml

@@ -45,7 +45,7 @@ jobs:
 
     - name: Start MotoServer
       run: |
-        python setup.py sdist
+        python -m build
         docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh &
         python scripts/ci_wait_for_server.py
     - name: Test ServerMode/Coverage

.github/workflows/test_terraform.yml

@@ -31,8 +31,8 @@ jobs:
         python-version: "3.8"
     - name: Start MotoServer
       run: |
-        pip install PyYAML
+        pip install PyYAML build
-        python setup.py sdist
+        python -m build
         docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e MOTO_PORT=4566 -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 4566:4566 -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh &
         MOTO_PORT=4566 python scripts/ci_wait_for_server.py
     - name: Download Cache

MANIFEST.in

@@ -1,5 +1,5 @@
 include README.md LICENSE AUTHORS.md
-include requirements.txt requirements-dev.txt tox.ini
+include requirements.txt requirements-dev.txt
 include moto/config/resources/aws_managed_rules.json
 include moto/ec2/_models/*.py
 include moto/ec2/resources/instance_types.json

Makefile

@@ -15,7 +15,7 @@ else
 endif
 
 init:
-	@python setup.py develop
+	@pip install -e .
 	@pip install -r requirements-dev.txt
 
 lint:

docs/docs/getting_started.rst

@@ -21,7 +21,7 @@ If you want to install ``moto`` from source::
 
     git clone git://github.com/spulec/moto.git
     cd moto
-    python setup.py install
+    pip install '.[all]'
 
 
 Moto usage

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools >= 40.6.0", "wheel"]
+build-backend = "setuptools.build_meta"

requirements-dev.txt

@@ -10,4 +10,5 @@ inflection
 lxml
 mypy
 packaging
+build
 prompt_toolkit

setup.cfg

@@ -1,3 +1,214 @@
+[metadata]
+name = moto
+version = 4.1.0.dev
+long_description = file:README.md
+long_description_content_type = text/markdown
+author = Steve Pulec
+author_email = "spulec@gmail.com"
+url = https://github.com/spulec/moto
+license = Apache License 2.0
+test_suite = tests
+classifiers =
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
+    License :: OSI Approved :: Apache Software License
+    Topic :: Software Development :: Testing
+keywords = aws ec2 s3 boto3 mock
+project_urls =
+    Documentation = http://docs.getmoto.org/en/latest/
+    Issue tracker = https://github.com/spulec/moto/issues
+    Changelog = https://github.com/spulec/moto/blob/master/CHANGELOG.md
+
+[options]
+python_requires = >=3.7
+install_requires =
+    boto3>=1.9.201
+    botocore>=1.12.201
+    cryptography>=3.3.1
+    requests>=2.5
+    xmltodict
+    werkzeug>=0.5,!=2.2.0,!=2.2.1
+    python-dateutil<3.0.0,>=2.1
+    responses>=0.13.0
+    Jinja2>=2.10.1
+    importlib_metadata ; python_version < '3.8'
+package_dir =
+    moto = moto
+include_package_data = True
+
+[options.extras_require]
+all =
+    python-jose[cryptography]>=3.1.0,<4.0.0
+    ecdsa!=0.15
+    docker>=2.5.1
+    graphql-core
+    PyYAML>=5.1
+    cfn-lint>=0.40.0
+    sshpubkeys>=3.1.0
+    openapi-spec-validator>=0.2.8
+    pyparsing>=3.0.7
+    jsondiff>=1.1.2
+    aws-xray-sdk!=0.96,>=0.93
+    setuptools
+server =
+    python-jose[cryptography]>=3.1.0,<4.0.0
+    ecdsa!=0.15
+    docker>=2.5.1
+    graphql-core
+    PyYAML>=5.1
+    cfn-lint>=0.40.0
+    sshpubkeys>=3.1.0
+    openapi-spec-validator>=0.2.8
+    pyparsing>=3.0.7
+    jsondiff>=1.1.2
+    aws-xray-sdk!=0.96,>=0.93
+    setuptools
+    flask!=2.2.0,!=2.2.1
+    flask-cors
+acm =
+acmpca =
+amp =
+apigateway =
+    PyYAML>=5.1
+    python-jose[cryptography]>=3.1.0,<4.0.0
+    ecdsa!=0.15
+    openapi-spec-validator>=0.2.8
+apigatewayv2 = PyYAML>=5.1
+applicationautoscaling =
+appsync = graphql-core
+athena =
+autoscaling =
+awslambda = docker>=2.5.1
+batch = docker>=2.5.1
+batch_simple =
+budgets =
+ce =
+cloudformation =
+    python-jose[cryptography]>=3.1.0,<4.0.0
+    ecdsa!=0.15
+    docker>=2.5.1
+    graphql-core
+    PyYAML>=5.1
+    cfn-lint>=0.40.0
+    sshpubkeys>=3.1.0
+    openapi-spec-validator>=0.2.8
+    pyparsing>=3.0.7
+    jsondiff>=1.1.2
+    aws-xray-sdk!=0.96,>=0.93
+    setuptools
+cloudfront =
+cloudtrail =
+cloudwatch =
+codebuild =
+codecommit =
+codepipeline =
+cognitoidentity =
+cognitoidp =
+    python-jose[cryptography]>=3.1.0,<4.0.0
+    ecdsa!=0.15
+comprehend =
+config =
+databrew =
+datapipeline =
+datasync =
+dax =
+dms =
+ds = sshpubkeys>=3.1.0
+dynamodb = docker>=2.5.1
+dynamodbstreams = docker>=2.5.1
+ebs = sshpubkeys>=3.1.0
+ec2 = sshpubkeys>=3.1.0
+ec2instanceconnect =
+ecr =
+ecs =
+efs = sshpubkeys>=3.1.0
+eks =
+elasticache =
+elasticbeanstalk =
+elastictranscoder =
+elb =
+elbv2 =
+emr =
+emrcontainers =
+emrserverless =
+es =
+events =
+firehose =
+forecast =
+glacier =
+glue = pyparsing>=3.0.7
+greengrass =
+guardduty =
+iam =
+iot =
+iotdata = jsondiff>=1.1.2
+kinesis =
+kinesisvideo =
+kinesisvideoarchivedmedia =
+kms =
+logs =
+managedblockchain =
+mediaconnect =
+medialive =
+mediapackage =
+mediastore =
+mediastoredata =
+meteringmarketplace =
+mq =
+opsworks =
+organizations =
+personalize =
+pinpoint =
+polly =
+quicksight =
+ram =
+rds =
+redshift =
+redshiftdata =
+rekognition =
+resourcegroups =
+resourcegroupstaggingapi =
+route53 =
+route53resolver = sshpubkeys>=3.1.0
+s3 = PyYAML>=5.1
+s3control =
+sagemaker =
+sdb =
+secretsmanager =
+servicediscovery =
+servicequotas =
+ses =
+signer =
+sns =
+sqs =
+ssm =
+    PyYAML>=5.1
+ssoadmin =
+stepfunctions =
+sts =
+support =
+swf =
+textract =
+timestreamwrite =
+transcribe =
+wafv2 =
+# XRay module uses pkg_resources, but doesn't have an explicit dependency listed.  This is fixed in 2.9.0:
+# https://github.com/aws/aws-xray-sdk-python/issues/305
+xray =
+    aws-xray-sdk!=0.96,>=0.93
+    setuptools
+
+[options.entry_points]
+console_scripts =
+    moto_server = moto.server:main
+
+[bdist_wheel]
+universal=1
+
 [tool:pytest]
 markers =
     network: marks tests which require network connection

setup.py

@@ -1,160 +0,0 @@
-#!/usr/bin/env python
-from io import open
-import os
-import re
-from setuptools import setup, find_packages
-import moto.__init__ as service_list
-
-# Borrowed from pip at https://github.com/pypa/pip/blob/62c27dee45625e1b63d1e023b0656310f276e050/setup.py#L11-L15
-here = os.path.abspath(os.path.dirname(__file__))
-
-
-def read(*parts):
-    # intentionally *not* adding an encoding option to open, See:
-    #   https://github.com/pypa/virtualenv/issues/201#issuecomment-3145690
-    with open(os.path.join(here, *parts), "r") as fp:
-        return fp.read()
-
-
-def get_version():
-    version_file = read("moto", "__init__.py")
-    version_match = re.search(
-        r'^__version__ = [\'"]([^\'"]*)[\'"]', version_file, re.MULTILINE
-    )
-    if version_match:
-        return version_match.group(1)
-    raise RuntimeError("Unable to find version string.")
-
-
-install_requires = [
-    "boto3>=1.9.201",
-    "botocore>=1.12.201",
-    "cryptography>=3.3.1",
-    "requests>=2.5",
-    "xmltodict",
-    "werkzeug>=0.5,!=2.2.0,!=2.2.1",
-    "python-dateutil<3.0.0,>=2.1",
-    "responses>=0.13.0",
-    "MarkupSafe!=2.0.0a1",  # This is a Jinja2 dependency, 2.0.0a1 currently seems broken
-    "Jinja2>=2.10.1",
-    "importlib_metadata ; python_version < '3.8'",
-]
-
-_dep_PyYAML = "PyYAML>=5.1"
-_dep_python_jose = "python-jose[cryptography]>=3.1.0,<4.0.0"
-_dep_python_jose_ecdsa_pin = (
-    "ecdsa!=0.15"  # https://github.com/spulec/moto/pull/3263#discussion_r477404984
-)
-_dep_docker = "docker>=2.5.1"
-_dep_graphql = "graphql-core"
-_dep_jsondiff = "jsondiff>=1.1.2"
-_dep_aws_xray_sdk = "aws-xray-sdk!=0.96,>=0.93"
-_dep_idna = "idna<4,>=2.5"
-_dep_cfn_lint = "cfn-lint>=0.40.0"
-_dep_sshpubkeys = "sshpubkeys>=3.1.0"
-_dep_openapi = "openapi-spec-validator>=0.2.8"
-_dep_pyparsing = "pyparsing>=3.0.7"
-_setuptools = "setuptools"
-
-all_extra_deps = [
-    _dep_PyYAML,
-    _dep_python_jose,
-    _dep_python_jose_ecdsa_pin,
-    _dep_docker,
-    _dep_graphql,
-    _dep_jsondiff,
-    _dep_aws_xray_sdk,
-    _dep_idna,
-    _dep_cfn_lint,
-    _dep_sshpubkeys,
-    _dep_pyparsing,
-    _dep_openapi,
-    _setuptools,
-]
-all_server_deps = all_extra_deps + ["flask!=2.2.0,!=2.2.1", "flask-cors"]
-
-extras_per_service = {}
-for service_name in [
-    service[5:]
-    for service in dir(service_list)
-    if service.startswith("mock_") and not service == "mock_all"
-]:
-    extras_per_service[service_name] = []
-extras_per_service.update(
-    {
-        "apigateway": [_dep_PyYAML, _dep_python_jose, _dep_python_jose_ecdsa_pin, _dep_openapi],
-        "apigatewayv2": [_dep_PyYAML],
-        "appsync": [_dep_graphql],
-        "awslambda": [_dep_docker],
-        "batch": [_dep_docker],
-        "cognitoidp": [_dep_python_jose, _dep_python_jose_ecdsa_pin],
-        "ec2": [_dep_sshpubkeys],
-        "glue": [_dep_pyparsing],
-        "iotdata": [_dep_jsondiff],
-        "s3": [_dep_PyYAML],
-        "ses": [],
-        "sns": [],
-        "sqs": [],
-        "ssm": [_dep_PyYAML],
-        # XRay module uses pkg_resources, but doesn't have an explicit
-        # dependency listed.  This should be fixed in the next version:
-        # https://github.com/aws/aws-xray-sdk-python/issues/305
-        "xray": [_dep_aws_xray_sdk, _setuptools],
-    }
-)
-
-# When a Table has a Stream, we'll always need to import AWSLambda to search for a corresponding function to send the table data to
-extras_per_service["dynamodb"] = extras_per_service["awslambda"]
-extras_per_service["dynamodb2"] = extras_per_service["dynamodb"]
-extras_per_service["dynamodbstreams"] = extras_per_service["awslambda"]
-# EBS depends on EC2 to create snapshots
-extras_per_service["ebs"] = extras_per_service["ec2"]
-# EFS depends on EC2 to find subnets etc
-extras_per_service["efs"] = extras_per_service["ec2"]
-# DirectoryService needs EC2 to verify VPCs and subnets.
-extras_per_service["ds"] = extras_per_service["ec2"]
-extras_per_service["route53resolver"] = extras_per_service["ec2"]
-# CloudFormation imports everything, so install everything
-extras_per_service["cloudformation"] = all_extra_deps
-extras_require = {
-    "all": all_extra_deps,
-    "server": all_server_deps,
-}
-
-extras_require.update(extras_per_service)
-
-
-setup(
-    name="moto",
-    version=get_version(),
-    description="A library that allows your python tests to easily"
-    " mock out the boto library",
-    long_description=read("README.md"),
-    long_description_content_type="text/markdown",
-    author="Steve Pulec",
-    author_email="spulec@gmail.com",
-    url="https://github.com/spulec/moto",
-    entry_points={"console_scripts": ["moto_server = moto.server:main"]},
-    packages=find_packages(exclude=("tests", "tests.*")),
-    python_requires=">=3.7",
-    install_requires=install_requires,
-    extras_require=extras_require,
-    include_package_data=True,
-    license="Apache License 2.0",
-    test_suite="tests",
-    classifiers=[
-        "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
-        "Programming Language :: Python :: 3.11",
-        "License :: OSI Approved :: Apache Software License",
-        "Topic :: Software Development :: Testing",
-    ],
-    project_urls={
-        "Documentation": "http://docs.getmoto.org/en/latest/",
-        "Issue tracker": "https://github.com/spulec/moto/issues",
-        "Changelog": "https://github.com/spulec/moto/blob/master/CHANGELOG.md"
-    },
-)

update_version_from_git.py

@@ -29,9 +29,8 @@ import sys
 from packaging.version import Version
 
 
-def migrate_source_attribute(attr, to_this, target_file, regex):
+def migrate_source_attribute(attr, to_this, target_file):
     """Updates __magic__ attributes in the source file"""
-    change_this = re.compile(regex, re.S)
     new_file = []
     found = False
 
@@ -41,7 +40,7 @@ def migrate_source_attribute(attr, to_this, target_file, regex):
     for line in lines:
         if line.startswith(attr):
             found = True
-            line = re.sub(change_this, to_this, line)
+            line = to_this
         new_file.append(line)
 
     if found:
@@ -49,14 +48,19 @@ def migrate_source_attribute(attr, to_this, target_file, regex):
             fp.writelines(new_file)
 
 
-def migrate_version(target_file, new_version):
+def migrate_version(new_version):
-    """Updates __version__ in the source file"""
+    initpy = os.path.abspath("moto/__init__.py")
-    regex = r"['\"](.*)['\"]"
+    setupcfg = os.path.abspath("setup.cfg")
+    """Updates __version__ in the init file"""
     migrate_source_attribute(
         "__version__",
-        '"{new_version}"'.format(new_version=new_version),
+        to_this=f'__version__ = "{new_version}"\n',
-        target_file,
+        target_file=initpy,
-        regex,
+    )
+    migrate_source_attribute(
+        "version =",
+        to_this=f'version = {new_version}\n',
+        target_file=setupcfg,
     )
 
 
@@ -130,7 +134,6 @@ def release_version_correct():
     """
     if is_master_branch():
         # update for a pre release version.
-        initpy = os.path.abspath("moto/__init__.py")
 
         new_version = prerelease_version()
         print(
@@ -141,7 +144,7 @@ def release_version_correct():
         assert (
             len(new_version.split(".")) >= 4
         ), "moto/__init__.py version should be like 0.0.2.dev"
-        migrate_version(initpy, new_version)
+        migrate_version(new_version)
     else:
         assert False, "No non-master deployments yet"
 
@@ -155,8 +158,8 @@ if __name__ == "__main__":
             new_version = arg
         if new_version == "patch":
             new_version = increase_patch_version(get_version())
-        initpy = os.path.abspath("moto/__init__.py")
+
-        migrate_version(initpy, new_version)
+        migrate_version(new_version)
     else:
         print(
             "Invalid usage. Supply 0 or 1 arguments. "