moto/.github/workflows/build.yml

375 lines
12 KiB
YAML
Raw Normal View History

name: TestNDeploy
on: [push, pull_request]
2021-01-23 20:46:54 +00:00
jobs:
# Install and cache dependencies
terraformcache:
name: Caching Terraform dependencies
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [ 3.8 ]
steps:
- name: Checkout Terraform Tests repo
uses: actions/checkout@v2
with:
2021-08-12 15:55:53 +00:00
repository: localstack/localstack-terraform-test
ref: build
path: moto-terraform-tests
submodules: 'true'
- uses: actions/setup-go@v2
with:
go-version: '^1.16.0'
- run: go version
- name: cache
id: terraformcache
uses: actions/cache@v2
with:
path: '~/.cache'
2021-12-15 14:25:54 +00:00
key: 'terraformcache_download2'
- name: Download
2021-08-12 15:55:53 +00:00
env:
DOWNLOAD_TEST_BIN: 1
TEST_BIN_URL: "https://moto-terraform-test.s3.amazonaws.com/aws.test"
if: ${{ steps.terraformcache.outputs.cache-hit != 'true' }}
run: |
cd moto-terraform-tests
bin/install-aws-test
cd ..
cache:
name: Caching
2021-01-23 20:46:54 +00:00
runs-on: ubuntu-latest
strategy:
matrix:
2021-08-22 10:33:16 +00:00
python-version: [ 3.6, 3.7, 3.8, 3.9 ]
2021-01-23 20:46:54 +00:00
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Get pip cache dir
id: pip-cache-dir
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: pip cache
id: pip-cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache-dir.outputs.dir }}
key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}-4
- name: Update pip
2021-07-26 06:40:39 +00:00
if: ${{ steps.pip-cache.outputs.cache-hit != 'true' }}
run: |
python -m pip install --upgrade pip
- name: Install project dependencies
if: ${{ steps.pip-cache.outputs.cache-hit != 'true' }}
run: |
pip install -r requirements-dev.txt
lint:
name: Linting
runs-on: ubuntu-latest
needs: cache
strategy:
matrix:
python-version: [3.7]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
# Retrieve the previously cached dependencies
- name: Get pip cache dir
id: pip-cache
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: pip cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}-4
Merge LocalStack changes into upstream moto (#4082) * fix OPTIONS requests on non-existing API GW integrations * add cloudformation models for API Gateway deployments * bump version * add backdoor to return CloudWatch metrics * Updating implementation coverage * Updating implementation coverage * add cloudformation models for API Gateway deployments * Updating implementation coverage * Updating implementation coverage * Implemented get-caller-identity returning real data depending on the access key used. * bump version * minor fixes * fix Number data_type for SQS message attribute * fix handling of encoding errors * bump version * make CF stack queryable before starting to initialize its resources * bump version * fix integration_method for API GW method integrations * fix undefined status in CF FakeStack * Fix apigateway issues with terraform v0.12.21 * resource_methods -> add handle for "DELETE" method * integrations -> fix issue that "httpMethod" wasn't included in body request (this value was set as the value from refer method resource) * bump version * Fix setting http method for API gateway integrations (#6) * bump version * remove duplicate methods * add storage class to S3 Key when completing multipart upload (#7) * fix SQS performance issues; bump version * add pagination to SecretsManager list-secrets (#9) * fix default parameter groups in RDS * fix adding S3 metadata headers with names containing dots (#13) * Updating implementation coverage * Updating implementation coverage * add cloudformation models for API Gateway deployments * Updating implementation coverage * Updating implementation coverage * Implemented get-caller-identity returning real data depending on the access key used. * make CF stack queryable before starting to initialize its resources * bump version * remove duplicate methods * fix adding S3 metadata headers with names containing dots (#13) * Update amis.json to support EKS AMI mocks (#15) * fix PascalCase for boolean value in ListMultipartUploads response (#17); fix _get_multi_param to parse nested list/dict query params * determine non-zero container exit code in Batch API * support filtering by dimensions in CW get_metric_statistics * fix storing attributes for ELBv2 Route entities; API GW refactorings for TF tests * add missing fields for API GW resources * fix error messages for Route53 (TF-compat) * various fixes for IAM resources (tf-compat) * minor fixes for API GW models (tf-compat) * minor fixes for API GW responses (tf-compat) * add s3 exception for bucket notification filter rule validation * change the way RESTErrors generate the response body and content-type header * fix lint errors and disable "black" syntax enforcement * remove return type hint in RESTError.get_body * add RESTError XML template for IAM exceptions * add support for API GW minimumCompressionSize * fix casing getting PrivateDnsEnabled API GW attribute * minor fixes for error responses * fix escaping special chars for IAM role descriptions (tf-compat) * minor fixes and tagging support for API GW and ELB v2 (tf-compat) * Merge branch 'master' into localstack * add "AlarmRule" attribute to enable support for composite CloudWatch metrics * fix recursive parsing of complex/nested query params * bump version * add API to delete S3 website configurations (#18) * use dict copy to allow parallelism and avoid concurrent modification exceptions in S3 * fix precondition check for etags in S3 (#19) * minor fix for user filtering in Cognito * fix API Gateway error response; avoid returning empty response templates (tf-compat) * support tags and tracingEnabled attribute for API GW stages * fix boolean value in S3 encryption response (#20) * fix connection arn structure * fix api destination arn structure * black format * release 2.0.3.37 * fix s3 exception tests see botocore/parsers.py:1002 where RequestId is removed from parsed * remove python 2 from build action * add test failure annotations in build action * fix events test arn comparisons * fix s3 encryption response test * return default value "0" if EC2 availableIpAddressCount is empty * fix extracting SecurityGroupIds for EC2 VPC endpoints * support deleting/updating API Gateway DomainNames * fix(events): Return empty string instead of null when no pattern is specified in EventPattern (tf-compat) (#22) * fix logic and revert CF changes to get tests running again (#21) * add support for EC2 customer gateway API (#25) * add support for EC2 Transit Gateway APIs (#24) * feat(logs): add `kmsKeyId` into `LogGroup` entity (#23) * minor change in ELBv2 logic to fix tests * feat(events): add APIs to describe and delete CloudWatch Events connections (#26) * add support for EC2 transit gateway route tables (#27) * pass transit gateway route table ID in Describe API, minor refactoring (#29) * add support for EC2 Transit Gateway Routes (#28) * fix region on ACM certificate import (#31) * add support for EC2 transit gateway attachments (#30) * add support for EC2 Transit Gateway VPN attachments (#32) * fix account ID for logs API * add support for DeleteOrganization API * feat(events): store raw filter representation for CloudWatch events patterns (tf-compat) (#36) * feat(events): add support to describe/update/delete CloudWatch API destinations (#35) * add Cognito UpdateIdentityPool, CW Logs PutResourcePolicy * feat(events): add support for tags in EventBus API (#38) * fix parameter validation for Batch compute environments (tf-compat) * revert merge conflicts in IMPLEMENTATION_COVERAGE.md * format code using black * restore original README; re-enable and fix CloudFormation tests * restore tests and old logic for CF stack parameters from SSM * parameterize RequestId/RequestID in response messages and revert related test changes * undo LocalStack-specific adaptations * minor fix * Update CodeCov config to reflect removal of Py2 * undo change related to CW metric filtering; add additional test for CW metric statistics with dimensions * Terraform - Extend whitelist of running tests Co-authored-by: acsbendi <acsbendi28@gmail.com> Co-authored-by: Phan Duong <duongpv@outlook.com> Co-authored-by: Thomas Rausch <thomas@thrau.at> Co-authored-by: Macwan Nevil <macnev2013@gmail.com> Co-authored-by: Dominik Schubert <dominik.schubert91@gmail.com> Co-authored-by: Gonzalo Saad <saad.gonzalo.ale@gmail.com> Co-authored-by: Mohit Alonja <monty16597@users.noreply.github.com> Co-authored-by: Miguel Gagliardo <migag9@gmail.com> Co-authored-by: Bert Blommers <info@bertblommers.nl>
2021-07-26 14:21:17 +00:00
# Update PIP
- name: Update pip
run: |
python -m pip install --upgrade pip
# Still need to properly install the dependencies - it will only skip the download part
- name: Install project dependencies
run: |
pip install -r requirements-dev.txt
- name: Lint with flake8
run:
make lint
test:
name: Unit test
runs-on: ubuntu-latest
needs: lint
strategy:
fail-fast: false
matrix:
2021-08-22 10:33:16 +00:00
python-version: [3.6, 3.7, 3.8, 3.9]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Get pip cache dir
id: pip-cache
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: pip cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}-4
- name: Update pip
run: |
python -m pip install --upgrade pip
- name: Install project dependencies
run: |
pip install -r requirements-dev.txt
pip install pytest-cov
Merge LocalStack changes into upstream moto (#4082) * fix OPTIONS requests on non-existing API GW integrations * add cloudformation models for API Gateway deployments * bump version * add backdoor to return CloudWatch metrics * Updating implementation coverage * Updating implementation coverage * add cloudformation models for API Gateway deployments * Updating implementation coverage * Updating implementation coverage * Implemented get-caller-identity returning real data depending on the access key used. * bump version * minor fixes * fix Number data_type for SQS message attribute * fix handling of encoding errors * bump version * make CF stack queryable before starting to initialize its resources * bump version * fix integration_method for API GW method integrations * fix undefined status in CF FakeStack * Fix apigateway issues with terraform v0.12.21 * resource_methods -> add handle for "DELETE" method * integrations -> fix issue that "httpMethod" wasn't included in body request (this value was set as the value from refer method resource) * bump version * Fix setting http method for API gateway integrations (#6) * bump version * remove duplicate methods * add storage class to S3 Key when completing multipart upload (#7) * fix SQS performance issues; bump version * add pagination to SecretsManager list-secrets (#9) * fix default parameter groups in RDS * fix adding S3 metadata headers with names containing dots (#13) * Updating implementation coverage * Updating implementation coverage * add cloudformation models for API Gateway deployments * Updating implementation coverage * Updating implementation coverage * Implemented get-caller-identity returning real data depending on the access key used. * make CF stack queryable before starting to initialize its resources * bump version * remove duplicate methods * fix adding S3 metadata headers with names containing dots (#13) * Update amis.json to support EKS AMI mocks (#15) * fix PascalCase for boolean value in ListMultipartUploads response (#17); fix _get_multi_param to parse nested list/dict query params * determine non-zero container exit code in Batch API * support filtering by dimensions in CW get_metric_statistics * fix storing attributes for ELBv2 Route entities; API GW refactorings for TF tests * add missing fields for API GW resources * fix error messages for Route53 (TF-compat) * various fixes for IAM resources (tf-compat) * minor fixes for API GW models (tf-compat) * minor fixes for API GW responses (tf-compat) * add s3 exception for bucket notification filter rule validation * change the way RESTErrors generate the response body and content-type header * fix lint errors and disable "black" syntax enforcement * remove return type hint in RESTError.get_body * add RESTError XML template for IAM exceptions * add support for API GW minimumCompressionSize * fix casing getting PrivateDnsEnabled API GW attribute * minor fixes for error responses * fix escaping special chars for IAM role descriptions (tf-compat) * minor fixes and tagging support for API GW and ELB v2 (tf-compat) * Merge branch 'master' into localstack * add "AlarmRule" attribute to enable support for composite CloudWatch metrics * fix recursive parsing of complex/nested query params * bump version * add API to delete S3 website configurations (#18) * use dict copy to allow parallelism and avoid concurrent modification exceptions in S3 * fix precondition check for etags in S3 (#19) * minor fix for user filtering in Cognito * fix API Gateway error response; avoid returning empty response templates (tf-compat) * support tags and tracingEnabled attribute for API GW stages * fix boolean value in S3 encryption response (#20) * fix connection arn structure * fix api destination arn structure * black format * release 2.0.3.37 * fix s3 exception tests see botocore/parsers.py:1002 where RequestId is removed from parsed * remove python 2 from build action * add test failure annotations in build action * fix events test arn comparisons * fix s3 encryption response test * return default value "0" if EC2 availableIpAddressCount is empty * fix extracting SecurityGroupIds for EC2 VPC endpoints * support deleting/updating API Gateway DomainNames * fix(events): Return empty string instead of null when no pattern is specified in EventPattern (tf-compat) (#22) * fix logic and revert CF changes to get tests running again (#21) * add support for EC2 customer gateway API (#25) * add support for EC2 Transit Gateway APIs (#24) * feat(logs): add `kmsKeyId` into `LogGroup` entity (#23) * minor change in ELBv2 logic to fix tests * feat(events): add APIs to describe and delete CloudWatch Events connections (#26) * add support for EC2 transit gateway route tables (#27) * pass transit gateway route table ID in Describe API, minor refactoring (#29) * add support for EC2 Transit Gateway Routes (#28) * fix region on ACM certificate import (#31) * add support for EC2 transit gateway attachments (#30) * add support for EC2 Transit Gateway VPN attachments (#32) * fix account ID for logs API * add support for DeleteOrganization API * feat(events): store raw filter representation for CloudWatch events patterns (tf-compat) (#36) * feat(events): add support to describe/update/delete CloudWatch API destinations (#35) * add Cognito UpdateIdentityPool, CW Logs PutResourcePolicy * feat(events): add support for tags in EventBus API (#38) * fix parameter validation for Batch compute environments (tf-compat) * revert merge conflicts in IMPLEMENTATION_COVERAGE.md * format code using black * restore original README; re-enable and fix CloudFormation tests * restore tests and old logic for CF stack parameters from SSM * parameterize RequestId/RequestID in response messages and revert related test changes * undo LocalStack-specific adaptations * minor fix * Update CodeCov config to reflect removal of Py2 * undo change related to CW metric filtering; add additional test for CW metric statistics with dimensions * Terraform - Extend whitelist of running tests Co-authored-by: acsbendi <acsbendi28@gmail.com> Co-authored-by: Phan Duong <duongpv@outlook.com> Co-authored-by: Thomas Rausch <thomas@thrau.at> Co-authored-by: Macwan Nevil <macnev2013@gmail.com> Co-authored-by: Dominik Schubert <dominik.schubert91@gmail.com> Co-authored-by: Gonzalo Saad <saad.gonzalo.ale@gmail.com> Co-authored-by: Mohit Alonja <monty16597@users.noreply.github.com> Co-authored-by: Miguel Gagliardo <migag9@gmail.com> Co-authored-by: Bert Blommers <info@bertblommers.nl>
2021-07-26 14:21:17 +00:00
pip install pytest-github-actions-annotate-failures
# https://github.com/aws/aws-xray-sdk-python/issues/196
pip install "coverage<=4.5.4"
- name: Test with pytest
run: |
2021-10-05 17:11:07 +00:00
make test-only
- name: "Upload coverage to Codecov"
if: ${{ github.repository == 'spulec/moto'}}
uses: codecov/codecov-action@v1
with:
fail_ci_if_error: false
flags: unittests
testserver:
name: Unit tests in Server Mode
runs-on: ubuntu-latest
needs: lint
strategy:
matrix:
2021-08-22 10:33:16 +00:00
python-version: [3.6, 3.7, 3.8, 3.9]
steps:
- uses: actions/checkout@v2
2021-02-12 12:59:45 +00:00
with:
fetch-depth: 0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Start MotoServer
run: |
python setup.py sdist
2021-11-29 20:35:18 +00:00
docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh &
python scripts/ci_wait_for_server.py
- name: Get pip cache dir
id: pip-cache
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: pip cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}-4
- name: Update pip
run: |
python -m pip install --upgrade pip
- name: Install project dependencies
run: |
pip install -r requirements-dev.txt
pip install "coverage<=4.5.4"
- name: Test ServerMode/Coverage
env:
TEST_SERVER_MODE: ${{ true }}
run: |
2021-10-05 17:11:07 +00:00
make test-only
- name: "Upload coverage to Codecov"
if: ${{ github.repository == 'spulec/moto'}}
uses: codecov/codecov-action@v1
with:
fail_ci_if_error: false
flags: servertests
2021-10-05 17:11:07 +00:00
- name: "Stop MotoServer"
if: always()
run: |
mkdir serverlogs
pwd
ls -la
cp server_output.log serverlogs/server_output.log
docker stop motoserver
- name: Archive TF logs
if: always()
uses: actions/upload-artifact@v2
with:
name: motoserver-${{ matrix.python-version }}
path: |
serverlogs/*
terraform:
name: Terraform Tests
runs-on: ubuntu-latest
needs: terraformcache
strategy:
fail-fast: false
matrix:
python-version: [ 3.8 ]
part: ["aa", "ab", "ac"]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Checkout Terraform Tests repo
uses: actions/checkout@v2
with:
2021-08-12 15:55:53 +00:00
repository: localstack/localstack-terraform-test
ref: build
path: moto-terraform-tests
submodules: 'true'
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: cache
uses: actions/cache@v2
with:
path: '~/.cache'
2021-12-15 14:25:54 +00:00
key: 'terraformcache_download2'
- name: Start MotoServer
run: |
python setup.py sdist
2021-11-29 20:35:18 +00:00
docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e MOTO_PORT=4566 -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 4566:4566 -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh &
MOTO_PORT=4566 python scripts/ci_wait_for_server.py
# Poor man's parallelization
# Running them sequentially takes to much time
# And using the build in parallel-argument does not help with reducing runtime
# So we simply split the list of tests, and ask our CI for separate VM's to run them in parallel
- name: Get list of tests
run: |
2021-10-05 17:11:07 +00:00
cd moto-terraform-tests
bin/list-tests -i ../tests/terraform-tests.success.txt -e ../tests/terraform-tests.failures.txt > tftestlist.txt
split -n l/3 tftestlist.txt tf-split-
cd ..
- name: Run Terraform Tests
run: |
cd moto-terraform-tests
2021-10-05 17:11:07 +00:00
AWS_DEFAULT_REGION=us-east-1 AWS_ALTERNATE_REGION=eu-west-1 bin/run-tests -t -i tf-split-${{ matrix.part }} -e ../tests/terraform-tests.failures.txt
cd ..
- name: "Create report"
run: |
cd moto-terraform-tests
bin/create-report
bin/create-report-cli
cd ..
2021-08-12 15:55:53 +00:00
- name: Archive TF logs
2021-08-28 16:13:52 +00:00
if: always()
2021-08-12 15:55:53 +00:00
uses: actions/upload-artifact@v2
with:
2021-08-18 12:21:19 +00:00
name: buildfolder-${{ matrix.part }}
2021-08-12 15:55:53 +00:00
path: |
moto-terraform-tests/build/*
deploy:
name: Deploy
runs-on: ubuntu-latest
needs: [test, testserver, terraform ]
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'spulec/moto' }}
strategy:
matrix:
python-version: [3.8]
steps:
- uses: actions/checkout@v2
2021-03-03 09:57:08 +00:00
with:
fetch-depth: 0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Update & Build
run: |
pip install wheel packaging
python update_version_from_git.py
python setup.py sdist bdist_wheel
- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@master
with:
password: ${{ secrets.PYPI_API_TOKEN }}
- name: Build Docker release
2021-08-28 10:23:02 +00:00
run: |
docker build -t motoserver/moto . --tag moto:latest
- name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push
uses: docker/build-push-action@v2
with:
push: true
tags: motoserver/moto:latest
- name: Get version number
run: |
version=$(grep -oP '(?<=__version__ = ")[0-9.a-z]+(?=")' moto/__init__.py)
echo "moto_version=$version" >> $GITHUB_ENV
- uses: octokit/graphql-action@v2.x
name: Get PR info
id: get_pr
with:
query: |
query get_pr($owner:String!,$repo:String!,$commit:GitObjectID) {
repository(owner:$owner,name:$repo) {
object(oid:$commit) {
... on Commit {
associatedPullRequests(last: 1){
edges {
node {
baseRepository {
nameWithOwner
}
merged
number
}
}
}
}
}
}
}
owner: ${{ github.event.repository.owner.name }}
repo: ${{ github.event.repository.name }}
commit: "${{ github.sha }}"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Get PR number
run: |
nr="${{ fromJSON(steps.get_pr.outputs.data).repository.object.associatedPullRequests.edges[0].node.number }}"
repo="${{ fromJSON(steps.get_pr.outputs.data).repository.object.associatedPullRequests.edges[0].node.baseRepository.nameWithOwner }}"
if [ -z "$nr" ]
then
echo "PR nr not found in $msg"
echo "pr_found=false" >> $GITHUB_ENV
else
echo "PR NR: $nr"
echo "pr_nr=$nr" >> $GITHUB_ENV
echo "pr_repo=$repo" >> $GITHUB_ENV
echo "pr_found=true" >> $GITHUB_ENV
fi
- name: Leave PR comment with Moto version
uses: peter-evans/create-or-update-comment@v1
if: env.pr_found == 'true' && env.pr_repo == 'spulec/moto'
with:
issue-number: ${{ env.pr_nr }}
body: |
This is now part of moto >= ${{ env.moto_version }}