diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py
index 0e39a1da1..c9f3529a9 100644
--- a/moto/dynamodb2/responses.py
+++ b/moto/dynamodb2/responses.py
@@ -438,9 +438,12 @@ class DynamoHandler(BaseResponse):
                 all_indexes = (table.global_indexes or []) + (table.indexes or [])
                 indexes_by_name = dict((i["IndexName"], i) for i in all_indexes)
                 if index_name not in indexes_by_name:
-                    raise ValueError(
-                        "Invalid index: %s for table: %s. Available indexes are: %s"
-                        % (index_name, name, ", ".join(indexes_by_name.keys()))
+                    er = "com.amazonaws.dynamodb.v20120810#ResourceNotFoundException"
+                    return self.error(
+                        er,
+                        "Invalid index: {} for table: {}. Available indexes are: {}".format(
+                            index_name, name, ", ".join(indexes_by_name.keys())
+                        ),
                     )
 
                 index = indexes_by_name[index_name]["KeySchema"]
@@ -481,7 +484,9 @@ class DynamoHandler(BaseResponse):
                     ]
                 elif "begins_with" in range_key_expression:
                     range_comparison = "BEGINS_WITH"
-                    range_values = [value_alias_map[range_key_expression_components[1]]]
+                    range_values = [
+                        value_alias_map[range_key_expression_components[-1]]
+                    ]
                 else:
                     range_values = [value_alias_map[range_key_expression_components[2]]]
             else:
diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py
index 53194fbf4..1a8a70615 100644
--- a/tests/test_dynamodb2/test_dynamodb.py
+++ b/tests/test_dynamodb2/test_dynamodb.py
@@ -2630,6 +2630,44 @@ def test_scan_by_non_exists_index():
     )
 
 
+@mock_dynamodb2
+def test_query_by_non_exists_index():
+    dynamodb = boto3.client("dynamodb", region_name="us-east-1")
+
+    dynamodb.create_table(
+        TableName="test",
+        KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}],
+        AttributeDefinitions=[
+            {"AttributeName": "id", "AttributeType": "S"},
+            {"AttributeName": "gsi_col", "AttributeType": "S"},
+        ],
+        ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1},
+        GlobalSecondaryIndexes=[
+            {
+                "IndexName": "test_gsi",
+                "KeySchema": [{"AttributeName": "gsi_col", "KeyType": "HASH"}],
+                "Projection": {"ProjectionType": "ALL"},
+                "ProvisionedThroughput": {
+                    "ReadCapacityUnits": 1,
+                    "WriteCapacityUnits": 1,
+                },
+            }
+        ],
+    )
+
+    with assert_raises(ClientError) as ex:
+        dynamodb.query(
+            TableName="test",
+            IndexName="non_exists_index",
+            KeyConditionExpression="CarModel=M",
+        )
+
+    ex.exception.response["Error"]["Code"].should.equal("ResourceNotFoundException")
+    ex.exception.response["Error"]["Message"].should.equal(
+        "Invalid index: non_exists_index for table: test. Available indexes are: test_gsi"
+    )
+
+
 @mock_dynamodb2
 def test_batch_items_returns_all():
     dynamodb = _create_user_table()