diff --git a/moto/iam/models.py b/moto/iam/models.py
index 83562b7df..2d84a9681 100644
--- a/moto/iam/models.py
+++ b/moto/iam/models.py
@@ -1727,9 +1727,11 @@ class IAMBackend(BaseBackend):
         arns = dict((p.arn, p) for p in self.managed_policies.values())
         try:
             policy = arns[policy_arn]
-            policy.detach_from(self.get_role(role_name))
+            if policy.arn not in self.get_role(role_name).managed_policies.keys():
+                raise KeyError
         except KeyError:
             raise IAMNotFoundException("Policy {0} was not found.".format(policy_arn))
+        policy.detach_from(self.get_role(role_name))
 
     def attach_group_policy(self, policy_arn, group_name):
         arns = dict((p.arn, p) for p in self.managed_policies.values())
@@ -1745,6 +1747,8 @@ class IAMBackend(BaseBackend):
         arns = dict((p.arn, p) for p in self.managed_policies.values())
         try:
             policy = arns[policy_arn]
+            if policy.arn not in self.get_group(group_name).managed_policies.keys():
+                raise KeyError
         except KeyError:
             raise IAMNotFoundException("Policy {0} was not found.".format(policy_arn))
         policy.detach_from(self.get_group(group_name))
@@ -1761,6 +1765,8 @@ class IAMBackend(BaseBackend):
         arns = dict((p.arn, p) for p in self.managed_policies.values())
         try:
             policy = arns[policy_arn]
+            if policy.arn not in self.get_user(user_name).managed_policies.keys():
+                raise KeyError
         except KeyError:
             raise IAMNotFoundException("Policy {0} was not found.".format(policy_arn))
         policy.detach_from(self.get_user(user_name))
diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py
index 54547cdd7..ad83798ac 100644
--- a/tests/test_iam/test_iam.py
+++ b/tests/test_iam/test_iam.py
@@ -2084,6 +2084,81 @@ def test_attach_detach_user_policy():
     resp["AttachedPolicies"].should.have.length_of(0)
 
 
+@mock_iam()
+def test_only_detach_user_policy():
+    iam = boto3.resource("iam", region_name="us-east-1")
+    client = boto3.client("iam", region_name="us-east-1")
+
+    user = iam.create_user(UserName="test-user")
+
+    policy_name = "FreePolicy"
+    policy = iam.create_policy(
+        PolicyName=policy_name,
+        PolicyDocument=MOCK_POLICY,
+        Path="/mypolicy/",
+        Description="free floating policy",
+    )
+
+    resp = client.list_attached_user_policies(UserName=user.name)
+    resp["AttachedPolicies"].should.have.length_of(0)
+
+    with pytest.raises(ClientError) as exc:
+        client.detach_user_policy(UserName=user.name, PolicyArn=policy.arn)
+    err = exc.value.response["Error"]
+    err["Code"].should.equal("NoSuchEntity")
+    err["Message"].should.equal(f"Policy {policy.arn} was not found.")
+
+
+@mock_iam()
+def test_only_detach_group_policy():
+    iam = boto3.resource("iam", region_name="us-east-1")
+    client = boto3.client("iam", region_name="us-east-1")
+
+    group = iam.create_group(GroupName="test-group")
+
+    policy_name = "FreePolicy"
+    policy = iam.create_policy(
+        PolicyName=policy_name,
+        PolicyDocument=MOCK_POLICY,
+        Path="/mypolicy/",
+        Description="free floating policy",
+    )
+
+    resp = client.list_attached_group_policies(GroupName=group.name)
+    resp["AttachedPolicies"].should.have.length_of(0)
+
+    with pytest.raises(ClientError) as exc:
+        client.detach_group_policy(GroupName=group.name, PolicyArn=policy.arn)
+    err = exc.value.response["Error"]
+    err["Code"].should.equal("NoSuchEntity")
+    err["Message"].should.equal(f"Policy {policy.arn} was not found.")
+
+
+@mock_iam()
+def test_only_detach_role_policy():
+    iam = boto3.resource("iam", region_name="us-east-1")
+    client = boto3.client("iam", region_name="us-east-1")
+
+    role = iam.create_role(RoleName="test-role", AssumeRolePolicyDocument="{}")
+
+    policy_name = "FreePolicy"
+    policy = iam.create_policy(
+        PolicyName=policy_name,
+        PolicyDocument=MOCK_POLICY,
+        Path="/mypolicy/",
+        Description="free floating policy",
+    )
+
+    resp = client.list_attached_role_policies(RoleName=role.name)
+    resp["AttachedPolicies"].should.have.length_of(0)
+
+    with pytest.raises(ClientError) as exc:
+        client.detach_role_policy(RoleName=role.name, PolicyArn=policy.arn)
+    err = exc.value.response["Error"]
+    err["Code"].should.equal("NoSuchEntity")
+    err["Message"].should.equal(f"Policy {policy.arn} was not found.")
+
+
 @mock_iam
 def test_update_access_key():
     iam = boto3.resource("iam", region_name="us-east-1")
@@ -3215,7 +3290,7 @@ def test_get_account_summary():
             "ServerCertificatesQuota": 20,
             "MFADevices": 0,
             "UserPolicySizeQuota": 2048,
-            "PolicyVersionsInUse": 0,
+            "PolicyVersionsInUse": 1,
             "ServerCertificates": 0,
             "Roles": 0,
             "RolesQuota": 1000,
@@ -3287,7 +3362,7 @@ def test_get_account_summary():
             "ServerCertificatesQuota": 20,
             "MFADevices": 1,
             "UserPolicySizeQuota": 2048,
-            "PolicyVersionsInUse": 3,
+            "PolicyVersionsInUse": 4,
             "ServerCertificates": 1,
             "Roles": 1,
             "RolesQuota": 1000,