Admin: Introduce reusable CI workflows (#6004)

.github/workflows/build.yml

@@ -25,7 +25,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ${{ steps.pip-cache-dir.outputs.dir }}
-          key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}-4
+          key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
       - name: Update pip
         if: ${{ steps.pip-cache.outputs.cache-hit != 'true' }}
         run: |
@@ -71,151 +71,22 @@ jobs:
         make lint
 
   javatest:
-    name: Test behaviour for Java SDK
-    runs-on: ubuntu-latest
     needs: lint
-
+    uses: ./.github/workflows/tests_sdk_java.yml
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.8"
-      - name: Start MotoServer
-        run: |
-          pip install build
-          python -m build
-          docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh &
-          python scripts/ci_wait_for_server.py
-      - uses: actions/setup-java@v3
-        with:
-          distribution: 'temurin'
-          java-version: '17'
-          cache: 'maven'
-      - name: Build with Maven
-        run: |
-          mkdir ~/.aws && touch ~/.aws/credentials && echo -e "[default]\naws_access_key_id = test\naws_secret_access_key = test" > ~/.aws/credentials
-          cd other_langs/tests_java && mvn test
 
   test:
-    name: Unit test
+    needs: [lint]
-    runs-on: ubuntu-latest
+    uses: ./.github/workflows/tests_decoratormode.yml
-    needs: lint
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
-
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Get pip cache dir
-      id: pip-cache
-      run: |
-        echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
-    - name: pip cache
-      uses: actions/cache@v3
-      with:
-        path: ${{ steps.pip-cache.outputs.dir }}
-        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
-    - name: Update pip
-      run: |
-        python -m pip install --upgrade pip
-    - name: Install project dependencies
-      run: |
-        pip install -r requirements-dev.txt
-        pip install pytest-cov
-        pip install pytest-github-actions-annotate-failures
-    - name: Test with pytest
-      run: |
-        make test-only
-    - name: "Upload coverage to Codecov"
-      if: ${{ github.repository == 'getmoto/moto'}}
-      uses: codecov/codecov-action@v3
-      with:
-        fail_ci_if_error: false
-        flags: unittests
 
   testserver:
-    name: Unit tests in Server Mode
+    needs: [lint]
-    runs-on: ubuntu-latest
+    uses: ./.github/workflows/tests_servermode.yml
-    needs: lint
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
 
-    steps:
+  release:
-    - uses: actions/checkout@v3
+    name: Release
-      with:
-        fetch-depth: 0
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Start MotoServer
-      run: |
-        pip install build
-        python -m build
-        docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh &
-        python scripts/ci_wait_for_server.py
-    - name: Get pip cache dir
-      id: pip-cache
-      run: |
-        echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
-    - name: pip cache
-      uses: actions/cache@v3
-      with:
-        path: ${{ steps.pip-cache.outputs.dir }}
-        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
-    - name: Update pip
-      run: |
-        python -m pip install --upgrade pip
-    - name: Install project dependencies
-      run: |
-        pip install -r requirements-dev.txt
-    - name: Test ServerMode/Coverage
-      env:
-        TEST_SERVER_MODE: ${{ true }}
-      run: |
-        make test-only
-    - name: "Upload coverage to Codecov"
-      if: ${{ github.repository == 'getmoto/moto'}}
-      uses: codecov/codecov-action@v3
-      with:
-        fail_ci_if_error: false
-        flags: servertests
-    - name: "Stop MotoServer"
-      if: always()
-      run: |
-        mkdir serverlogs
-        pwd
-        ls -la
-        cp server_output.log serverlogs/server_output.log
-        docker stop motoserver
-    - name: Archive TF logs
-      if: always()
-      uses: actions/upload-artifact@v3
-      with:
-        name: motoserver-${{ matrix.python-version }}
-        path: |
-          serverlogs/*
-
-  deploy:
-    name: Deploy
     runs-on: ubuntu-latest
     needs: [javatest, test, testserver ]
     if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'getmoto/moto' }}
-    strategy:
-      matrix:
-        python-version: [3.8]
     steps:
       - uses: actions/checkout@v3
         with:

.github/workflows/dockertests.yml

@@ -25,7 +25,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ${{ steps.pip-cache-dir.outputs.dir }}
-          key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}
+          key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
       - name: Update pip
         if: ${{ steps.pip-cache.outputs.cache-hit != 'true' }}
         run: |
@@ -65,7 +65,7 @@ jobs:
       uses: actions/cache@v3
       with:
         path: ${{ steps.pip-cache.outputs.dir }}
-        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}
+        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
     - name: Update pip
       run: |
         python -m pip install --upgrade pip
@@ -125,7 +125,7 @@ jobs:
       uses: actions/cache@v3
       with:
         path: ${{ steps.pip-cache.outputs.dir }}
-        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}
+        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
     - name: Update pip
       run: |
         python -m pip install --upgrade pip
@@ -182,7 +182,7 @@ jobs:
       uses: actions/cache@v3
       with:
         path: ${{ steps.pip-cache.outputs.dir }}
-        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}
+        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
     - name: Update pip
       run: |
         python -m pip install --upgrade pip

.github/workflows/tests_decoratormode.yml

@@ -0,0 +1,45 @@
+name: Unit test
+on: [workflow_call]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Get pip cache dir
+      id: pip-cache
+      run: |
+        echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+    - name: pip cache
+      uses: actions/cache@v3
+      with:
+        path: ${{ steps.pip-cache.outputs.dir }}
+        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
+    - name: Update pip
+      run: |
+        python -m pip install --upgrade pip
+    - name: Install project dependencies
+      run: |
+        pip install -r requirements-dev.txt
+        pip install pytest-cov
+        pip install pytest-github-actions-annotate-failures
+    - name: Test with pytest
+      run: |
+        make test-only
+    - name: "Upload coverage to Codecov"
+      if: ${{ github.repository == 'getmoto/moto'}}
+      uses: codecov/codecov-action@v3
+      with:
+        fail_ci_if_error: false
+        flags: unittests

.github/workflows/tests_sdk_java.yml

@@ -0,0 +1,30 @@
+name: Java SDK test
+on: [workflow_call]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.8"
+    - name: Start MotoServer
+      run: |
+        pip install build
+        python -m build
+        docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh &
+        python scripts/ci_wait_for_server.py
+    - uses: actions/setup-java@v3
+      with:
+        distribution: 'temurin'
+        java-version: '17'
+        cache: 'maven'
+    - name: Build with Maven
+      run: |
+        mkdir ~/.aws && touch ~/.aws/credentials && echo -e "[default]\naws_access_key_id = test\naws_secret_access_key = test" > ~/.aws/credentials
+        cd other_langs/tests_java && mvn test

.github/workflows/tests_servermode.yml

@@ -0,0 +1,66 @@
+name: Unit tests in Server Mode
+on: [workflow_call]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Start MotoServer
+      run: |
+        pip install build
+        python -m build
+        docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh &
+        python scripts/ci_wait_for_server.py
+    - name: Get pip cache dir
+      id: pip-cache
+      run: |
+        echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+    - name: pip cache
+      uses: actions/cache@v3
+      with:
+        path: ${{ steps.pip-cache.outputs.dir }}
+        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
+    - name: Update pip
+      run: |
+        python -m pip install --upgrade pip
+    - name: Install project dependencies
+      run: |
+        pip install -r requirements-dev.txt
+    - name: Test ServerMode/Coverage
+      env:
+        TEST_SERVER_MODE: ${{ true }}
+      run: |
+        make test-only
+    - name: "Upload coverage to Codecov"
+      if: ${{ github.repository == 'getmoto/moto'}}
+      uses: codecov/codecov-action@v3
+      with:
+        fail_ci_if_error: false
+        flags: servertests
+    - name: "Stop MotoServer"
+      if: always()
+      run: |
+        mkdir serverlogs
+        pwd
+        ls -la
+        cp server_output.log serverlogs/server_output.log
+        docker stop motoserver
+    - name: Archive TF logs
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: motoserver-${{ matrix.python-version }}
+        path: |
+          serverlogs/*