diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index f425537c5..3710b8e4d 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -105,16 +105,10 @@ jobs:
         pip install -r requirements-dev.txt
         pip install pytest-cov
     - name: Test with pytest
-      if: ${{ matrix.python-version == 2.7 }}
-      run: |
-        make test-only
-    # Pytest-cov explicitly fails in Py2 for XRay tests
-    - name: Test with pytest/coverage
-      if: ${{ matrix.python-version != 2.7 }}
       run: |
         make test-coverage
     - name: "Upload coverage to Codecov"
-      if: ${{ matrix.python-version != 2.7 && github.repository == 'spulec/moto'}}
+      if: ${{ github.repository == 'spulec/moto'}}
       uses: codecov/codecov-action@v1
       with:
         fail_ci_if_error: true
@@ -158,20 +152,13 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements-dev.txt
-    - name: Test ServerMode
-      if: ${{ matrix.python-version == 2.7 }}
-      env:
-        TEST_SERVER_MODE: ${{ true }}
-      run: |
-        make test-only
     - name: Test ServerMode/Coverage
-      if: ${{ matrix.python-version != 2.7 }}
       env:
         TEST_SERVER_MODE: ${{ true }}
       run: |
         make test-coverage
     - name: "Upload coverage to Codecov"
-      if: ${{ matrix.python-version != 2.7 && github.repository == 'spulec/moto'}}
+      if: ${{ github.repository == 'spulec/moto'}}
       uses: codecov/codecov-action@v1
       with:
         fail_ci_if_error: true
diff --git a/codecov.yml b/codecov.yml
index a0f8353e4..4fe85e33f 100644
--- a/codecov.yml
+++ b/codecov.yml
@@ -1,7 +1,7 @@
 codecov:
   notify:
-    # Leave a GitHub comment after all Py3 builds have passed
-    after_n_builds: 6
+    # Leave a GitHub comment after all builds have passed
+    after_n_builds: 8
 coverage:
   status:
     project:
diff --git a/requirements-tests.txt b/requirements-tests.txt
index 847ce539e..02bdfdd2a 100644
--- a/requirements-tests.txt
+++ b/requirements-tests.txt
@@ -1,4 +1,7 @@
 pytest
-pytest-cov
+pytest-cov; python_version >= '3.6'
+# https://github.com/aws/aws-xray-sdk-python/issues/196
+# Downgrade pytest-cov to the last version that uses coverage < 5
+pytest-cov<=2.10.1; python_version < '3.6'
 sure==1.4.11
 freezegun