diff --git a/requirements.txt b/requirements.txt
index 4de489f8c..f5a476248 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,2 @@
 #  Please add requirements to setup.py
--e .
+-e .[all]
diff --git a/setup.py b/setup.py
index ffaa8b273..92ab7a5de 100755
--- a/setup.py
+++ b/setup.py
@@ -33,22 +33,13 @@ install_requires = [
     "boto>=2.36.0",
     "boto3>=1.9.201",
     "botocore>=1.12.201",
-    "cryptography>=2.3.0",
     "requests>=2.5",
     "xmltodict",
     "six>1.9",
     "werkzeug",
-    "PyYAML>=5.1",
     "pytz",
-    "ecdsa<0.15",
     "python-dateutil<3.0.0,>=2.1",
-    "python-jose[cryptography]>=3.1.0,<4.0.0",
-    "docker>=2.5.1",
-    "jsondiff>=1.1.2",
-    "aws-xray-sdk!=0.96,>=0.93",
     "responses>=0.9.0",
-    "idna<3,>=2.5",
-    "cfn-lint>=0.4.0",
     "MarkupSafe<2.0",  # This is a Jinja2 dependency, 2.0.0a1 currently seems broken
 ]
 
@@ -72,7 +63,6 @@ if PY2:
         "mock<=3.0.5",
         "more-itertools==5.0.0",
         "setuptools==44.0.0",
-        "sshpubkeys>=3.1.0,<4.0",
         "zipp==0.6.0",
     ]
 else:
@@ -81,14 +71,57 @@ else:
         "mock",
         "more-itertools",
         "setuptools",
-        "sshpubkeys>=3.1.0",
         "zipp",
     ]
 
+_dep_cryptography = "cryptography>=2.3.0"
+_dep_PyYAML = "PyYAML>=5.1"
+_dep_python_jose = "python-jose[cryptography]>=3.1.0,<4.0.0"
+_dep_python_jose_ecdsa_pin = "ecdsa<0.15"  # https://github.com/spulec/moto/pull/3263#discussion_r477404984
+_dep_docker = "docker>=2.5.1"
+_dep_jsondiff = "jsondiff>=1.1.2"
+_dep_aws_xray_sdk = "aws-xray-sdk!=0.96,>=0.93"
+_dep_idna = "idna<3,>=2.5"
+_dep_cfn_lint = "cfn-lint>=0.4.0"
+_dep_sshpubkeys_py2 = "sshpubkeys>=3.1.0,<4.0; python_version<'3'"
+_dep_sshpubkeys_py3 = "sshpubkeys>=3.1.0; python_version>'3'"
+
+all_extra_deps = [
+    _dep_cryptography,
+    _dep_PyYAML,
+    _dep_python_jose,
+    _dep_python_jose_ecdsa_pin,
+    _dep_docker,
+    _dep_jsondiff,
+    _dep_aws_xray_sdk,
+    _dep_idna,
+    _dep_cfn_lint,
+    _dep_sshpubkeys_py2,
+    _dep_sshpubkeys_py3,
+]
+
+# TODO: do we want to add ALL services here?
+# i.e. even those without extra dependencies.
+# Would be good for future-compatibility, I guess.
+extras_per_service = {
+    "ec2": [_dep_cryptography, _dep_sshpubkeys_py2, _dep_sshpubkeys_py3],
+    'acm': [_dep_cryptography],
+    'iam': [_dep_cryptography],
+    'cloudformation': [_dep_PyYAML, _dep_cfn_lint],
+    'cognitoidp': [_dep_python_jose, _dep_python_jose_ecdsa_pin],
+    'awslambda': [_dep_docker],
+    'batch': [_dep_docker],
+    'iotdata': [_dep_jsondiff],
+    'xray': [_dep_aws_xray_sdk],
+}
+
 extras_require = {
+    'all': all_extra_deps,
     'server': ['flask'],
 }
 
+extras_require.update(extras_per_service)
+
 # https://hynek.me/articles/conditional-python-dependencies/
 if int(setuptools.__version__.split(".", 1)[0]) < 18:
     if sys.version_info[0:2] < (3, 3):
diff --git a/travis_moto_server.sh b/travis_moto_server.sh
index 4be26073e..c764d1cd1 100755
--- a/travis_moto_server.sh
+++ b/travis_moto_server.sh
@@ -1,9 +1,8 @@
 #!/usr/bin/env bash
 set -e
-pip install flask
 # TravisCI on bionic dist uses old version of Docker Engine
 # which is incompatibile with newer docker-py
 # See https://github.com/docker/docker-py/issues/2639
 pip install "docker>=2.5.1,<=4.2.2"
-pip install /moto/dist/moto*.gz
-moto_server -H 0.0.0.0 -p 5000
\ No newline at end of file
+pip install $(ls /moto/dist/moto*.gz)[server,all]
+moto_server -H 0.0.0.0 -p 5000