diff --git a/moto/elbv2/exceptions.py b/moto/elbv2/exceptions.py
index 0947535eb..0bf9649d7 100644
--- a/moto/elbv2/exceptions.py
+++ b/moto/elbv2/exceptions.py
@@ -152,6 +152,13 @@ class InvalidDescribeRulesRequest(ELBClientError):
         )
 
 
+class ResourceInUseError(ELBClientError):
+
+    def __init__(self, msg="A specified resource is in use"):
+        super(ResourceInUseError, self).__init__(
+            "ResourceInUse", msg)
+
+
 class RuleNotFoundError(ELBClientError):
 
     def __init__(self):
diff --git a/moto/elbv2/models.py b/moto/elbv2/models.py
index 3c6afe7f5..8aa9ee9f0 100644
--- a/moto/elbv2/models.py
+++ b/moto/elbv2/models.py
@@ -21,6 +21,7 @@ from .exceptions import (
     InvalidActionTypeError,
     ActionTargetGroupNotFoundError,
     InvalidDescribeRulesRequest,
+    ResourceInUseError,
     RuleNotFoundError,
     DuplicatePriorityError,
     InvalidTargetGroupNameError,
@@ -426,10 +427,17 @@ class ELBv2Backend(BaseBackend):
         # however, boto3 does't raise error even if rule is not found
 
     def delete_target_group(self, target_group_arn):
-        target_group = self.target_groups.pop(target_group_arn, None)
+        if target_group_arn not in self.target_groups:
+            raise TargetGroupNotFoundError()
+
+        target_group = self.target_groups[target_group_arn]
         if target_group:
+            if self._any_listener_using(target_group_arn):
+                raise ResourceInUseError(
+                    "The target group '{}' is currently in use by a listener or a rule".format(
+                        target_group_arn))
+            del self.target_groups[target_group_arn]
             return target_group
-        raise TargetGroupNotFoundError()
 
     def delete_listener(self, listener_arn):
         for load_balancer in self.load_balancers.values():
@@ -539,6 +547,15 @@ class ELBv2Backend(BaseBackend):
             modified_rules.append(given_rule)
         return modified_rules
 
+    def _any_listener_using(self, target_group_arn):
+        for load_balancer in self.load_balancers.values():
+            for listener in load_balancer.listeners.values():
+                for rule in listener.rules:
+                    for action in rule.actions:
+                        if action.get('target_group_arn') == target_group_arn:
+                            return True
+        return False
+
 
 elbv2_backends = {}
 for region in ec2_backends.keys():
diff --git a/tests/test_elbv2/test_elbv2.py b/tests/test_elbv2/test_elbv2.py
index 21799ddcf..1a8494bd3 100644
--- a/tests/test_elbv2/test_elbv2.py
+++ b/tests/test_elbv2/test_elbv2.py
@@ -306,6 +306,13 @@ def test_create_target_group_and_listeners():
     response = conn.describe_listeners(ListenerArns=[http_listener_arn, https_listener_arn])
     response.get('Listeners').should.have.length_of(2)
 
+    # Try to delete the target group and it fails because there's a
+    # listener referencing it
+    with assert_raises(ClientError) as e:
+        conn.delete_target_group(TargetGroupArn=target_group.get('TargetGroupArn'))
+    e.exception.operation_name.should.equal('DeleteTargetGroup')
+    e.exception.args.should.equal(("An error occurred (ResourceInUse) when calling the DeleteTargetGroup operation: The target group 'arn:aws:elasticloadbalancing:us-east-1:1:targetgroup/a-target/50dc6c495c0c9188' is currently in use by a listener or a rule", ))  # NOQA
+
     # Delete one listener
     response = conn.describe_listeners(LoadBalancerArn=load_balancer_arn)
     response.get('Listeners').should.have.length_of(2)