Skip to content

Commit 943abd8

Browse files
ehsannasehsannas
authored
Serving OR Queries from the client-side index. (#3794)
* Serving OR Queries from the client-side index. * Address comments. * Add tests for index type of or queries. * Use limitToLast in some test cases.
1 parent 8e28b5b commit 943abd8

File tree

4 files changed

+259
-32
lines changed

4 files changed

+259
-32
lines changed

‎firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteIndexManager.java

Lines changed: 32 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -314,7 +314,9 @@ public IndexOffset getMinOffset(String collectionGroup) {
314314
@Override
315315
public IndexType getIndexType(Target target) {
316316
IndexType result = IndexType.FULL;
317-
for (Target subTarget : getSubTargets(target)) {
317+
List<Target> subTargets = getSubTargets(target);
318+
319+
for (Target subTarget : subTargets) {
318320
FieldIndex index = getFieldIndex(subTarget);
319321
if (index == null) {
320322
result = IndexType.NONE;
@@ -325,6 +327,17 @@ public IndexType getIndexType(Target target) {
325327
result = IndexType.PARTIAL;
326328
}
327329
}
330+
331+
// OR queries have more than one sub-target (one sub-target per DNF term). We currently consider
332+
// OR queries that have a `limit` to have a partial index. For such queries we perform sorting
333+
// and apply the limit in memory as a post-processing step.
334+
// TODO(orquery): If we have a FULL index *and* we have the index that can be used for sorting
335+
// all DNF branches on the same value, we can improve performance by performing a JOIN in SQL.
336+
// See b/235224019 for more information.
337+
if (target.hasLimit() && subTargets.size() > 1 && result == IndexType.FULL) {
338+
return IndexType.PARTIAL;
339+
}
340+
328341
return result;
329342
}
330343

@@ -509,9 +522,23 @@ public List<DocumentKey> getDocumentsMatchingTarget(Target target) {
509522
bindings.addAll(Arrays.asList(subQueryAndBindings).subList(1, subQueryAndBindings.length));
510523
}
511524

512-
String queryString =
513-
"SELECT DISTINCT document_key FROM (" + TextUtils.join(" UNION ", subQueries) + ")";
514-
525+
// We are constructing:
526+
// SELECT DISTINCT document_key FROM (
527+
// (SELECT ...) UNION (SELECT ...) UNION (SELECT ...)
528+
// ORDER BY ...
529+
// )
530+
// LIMIT ...
531+
//
532+
// Note: SQLite does not allow performing ORDER BY on each union clause. The ORDER BY must come
533+
// after the last union clause. Also note that LIMIT must be applied *after* the DISTINCT
534+
// operator has been performed. When dealing with multiple sub-targets, it's possible that the
535+
// same document_key appears multiple times.
536+
String unionSubTargets =
537+
TextUtils.join(" UNION ", subQueries)
538+
+ "ORDER BY directional_value, document_key "
539+
+ (target.getKeyOrder().equals(Direction.ASCENDING) ? "asc " : "desc ");
540+
541+
String queryString = "SELECT DISTINCT document_key FROM (" + unionSubTargets + ")";
515542
if (target.hasLimit()) {
516543
queryString = queryString + " LIMIT " + target.getLimit();
517544
}
@@ -557,11 +584,8 @@ private Object[] generateQueryAndBindings(
557584
statement.append("AND directional_value ").append(lowerBoundOp).append(" ? ");
558585
statement.append("AND directional_value ").append(upperBoundOp).append(" ? ");
559586

560-
// Create the UNION statement by repeating the above generated statement. We can then add
561-
// ordering and a limit clause.
587+
// Create the UNION statement by repeating the above generated statement.
562588
StringBuilder sql = repeatSequence(statement, statementCount, " UNION ");
563-
sql.append("ORDER BY directional_value, document_key ");
564-
sql.append(target.getKeyOrder().equals(Direction.ASCENDING) ? "asc " : "desc ");
565589

566590
if (notIn != null) {
567591
// Wrap the statement in a NOT-IN call.

‎firebase-firestore/src/test/java/com/google/firebase/firestore/local/QueryEngineTestCase.java

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -482,5 +482,45 @@ public void canPerformOrQueriesUsingFullCollectionScan() throws Exception {
482482
DocumentSet result5 =
483483
expectFullCollectionScan(() -> runQuery(query5, MISSING_LAST_LIMBO_FREE_SNAPSHOT));
484484
assertEquals(docSet(query5.comparator(), doc3), result5);
485+
486+
// Test with limits (implicit order by ASC): (a==1) || (b > 0) LIMIT 2
487+
Query query6 =
488+
query("coll").filter(orFilters(filter("a", "==", 1), filter("b", ">", 0))).limitToFirst(2);
489+
DocumentSet result6 =
490+
expectFullCollectionScan(() -> runQuery(query6, MISSING_LAST_LIMBO_FREE_SNAPSHOT));
491+
assertEquals(docSet(query6.comparator(), doc1, doc2), result6);
492+
493+
// Test with limits (implicit order by DESC): (a==1) || (b > 0) LIMIT_TO_LAST 2
494+
Query query7 =
495+
query("coll").filter(orFilters(filter("a", "==", 1), filter("b", ">", 0))).limitToLast(2);
496+
DocumentSet result7 =
497+
expectFullCollectionScan(() -> runQuery(query7, MISSING_LAST_LIMBO_FREE_SNAPSHOT));
498+
assertEquals(docSet(query7.comparator(), doc3, doc4), result7);
499+
500+
// Test with limits (explicit order by ASC): (a==2) || (b == 1) ORDER BY a LIMIT 1
501+
Query query8 =
502+
query("coll")
503+
.filter(orFilters(filter("a", "==", 2), filter("b", "==", 1)))
504+
.limitToFirst(1)
505+
.orderBy(orderBy("a", "asc"));
506+
DocumentSet result8 =
507+
expectFullCollectionScan(() -> runQuery(query8, MISSING_LAST_LIMBO_FREE_SNAPSHOT));
508+
assertEquals(docSet(query8.comparator(), doc5), result8);
509+
510+
// Test with limits (explicit order by DESC): (a==2) || (b == 1) ORDER BY a LIMIT_TO_LAST 1
511+
Query query9 =
512+
query("coll")
513+
.filter(orFilters(filter("a", "==", 2), filter("b", "==", 1)))
514+
.limitToLast(1)
515+
.orderBy(orderBy("a", "asc"));
516+
DocumentSet result9 =
517+
expectFullCollectionScan(() -> runQuery(query9, MISSING_LAST_LIMBO_FREE_SNAPSHOT));
518+
assertEquals(docSet(query9.comparator(), doc2), result9);
519+
520+
// Test with limits without orderBy (the __name__ ordering is the tie breaker).
521+
Query query10 =
522+
query("coll").filter(orFilters(filter("a", "==", 2), filter("b", "==", 1))).limitToFirst(1);
523+
DocumentSet result10 = expectFullCollectionScan(() -> runQuery(query10, SnapshotVersion.NONE));
524+
assertEquals(docSet(query10.comparator(), doc2), result10);
485525
}
486526
}

‎firebase-firestore/src/test/java/com/google/firebase/firestore/local/SQLiteIndexManagerTest.java

Lines changed: 84 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import static com.google.firebase.firestore.testutil.TestUtil.filter;
2626
import static com.google.firebase.firestore.testutil.TestUtil.key;
2727
import static com.google.firebase.firestore.testutil.TestUtil.map;
28+
import static com.google.firebase.firestore.testutil.TestUtil.orFilters;
2829
import static com.google.firebase.firestore.testutil.TestUtil.orderBy;
2930
import static com.google.firebase.firestore.testutil.TestUtil.path;
3031
import static com.google.firebase.firestore.testutil.TestUtil.query;
@@ -1007,90 +1008,149 @@ public void testPartialIndexAndFullIndex() throws Exception {
10071008
indexManager.addFieldIndex(fieldIndex("coll", "c", Kind.ASCENDING, "d", Kind.ASCENDING));
10081009

10091010
Query query1 = query("coll").filter(filter("a", "==", 1));
1010-
validateIsFullIndex(query1);
1011+
validateIndexType(query1, IndexManager.IndexType.FULL);
10111012

10121013
Query query2 = query("coll").filter(filter("b", "==", 1));
1013-
validateIsFullIndex(query2);
1014+
validateIndexType(query2, IndexManager.IndexType.FULL);
10141015

10151016
Query query3 = query("coll").filter(filter("a", "==", 1)).orderBy(orderBy("a"));
1016-
validateIsFullIndex(query3);
1017+
validateIndexType(query3, IndexManager.IndexType.FULL);
10171018

10181019
Query query4 = query("coll").filter(filter("b", "==", 1)).orderBy(orderBy("b"));
1019-
validateIsFullIndex(query4);
1020+
validateIndexType(query4, IndexManager.IndexType.FULL);
10201021

10211022
Query query5 = query("coll").filter(filter("a", "==", 1)).filter(filter("b", "==", 1));
1022-
validateIsPartialIndex(query5);
1023+
validateIndexType(query5, IndexManager.IndexType.PARTIAL);
10231024

10241025
Query query6 = query("coll").filter(filter("a", "==", 1)).orderBy(orderBy("b"));
1025-
validateIsPartialIndex(query6);
1026+
validateIndexType(query6, IndexManager.IndexType.PARTIAL);
10261027

10271028
Query query7 = query("coll").filter(filter("b", "==", 1)).orderBy(orderBy("a"));
1028-
validateIsPartialIndex(query7);
1029+
validateIndexType(query7, IndexManager.IndexType.PARTIAL);
10291030

10301031
Query query8 = query("coll").filter(filter("c", "==", 1)).filter(filter("d", "==", 1));
1031-
validateIsFullIndex(query8);
1032+
validateIndexType(query8, IndexManager.IndexType.FULL);
10321033

10331034
Query query9 =
10341035
query("coll")
10351036
.filter(filter("c", "==", 1))
10361037
.filter(filter("d", "==", 1))
10371038
.orderBy(orderBy("c"));
1038-
validateIsFullIndex(query9);
1039+
validateIndexType(query9, IndexManager.IndexType.FULL);
10391040

10401041
Query query10 =
10411042
query("coll")
10421043
.filter(filter("c", "==", 1))
10431044
.filter(filter("d", "==", 1))
10441045
.orderBy(orderBy("d"));
1045-
validateIsFullIndex(query10);
1046+
validateIndexType(query10, IndexManager.IndexType.FULL);
10461047

10471048
Query query11 =
10481049
query("coll")
10491050
.filter(filter("c", "==", 1))
10501051
.filter(filter("d", "==", 1))
10511052
.orderBy(orderBy("c"))
10521053
.orderBy(orderBy("d"));
1053-
validateIsFullIndex(query11);
1054+
validateIndexType(query11, IndexManager.IndexType.FULL);
10541055

10551056
Query query12 =
10561057
query("coll")
10571058
.filter(filter("c", "==", 1))
10581059
.filter(filter("d", "==", 1))
10591060
.orderBy(orderBy("d"))
10601061
.orderBy(orderBy("c"));
1061-
validateIsFullIndex(query12);
1062+
validateIndexType(query12, IndexManager.IndexType.FULL);
10621063

10631064
Query query13 =
10641065
query("coll")
10651066
.filter(filter("c", "==", 1))
10661067
.filter(filter("d", "==", 1))
10671068
.orderBy(orderBy("e"));
1068-
validateIsPartialIndex(query13);
1069+
validateIndexType(query13, IndexManager.IndexType.PARTIAL);
10691070

10701071
Query query14 = query("coll").filter(filter("c", "==", 1)).filter(filter("d", "<=", 1));
1071-
validateIsFullIndex(query14);
1072+
validateIndexType(query14, IndexManager.IndexType.FULL);
10721073

10731074
Query query15 =
10741075
query("coll")
10751076
.filter(filter("c", "==", 1))
10761077
.filter(filter("d", ">", 1))
10771078
.orderBy(orderBy("d"));
1078-
validateIsFullIndex(query15);
1079+
validateIndexType(query15, IndexManager.IndexType.FULL);
10791080
}
10801081

1081-
private void validateIsPartialIndex(Query query) {
1082-
validateIndex(query, false);
1083-
}
1082+
@Test
1083+
public void testIndexTypeForOrQueries() throws Exception {
1084+
indexManager.addFieldIndex(fieldIndex("coll", "a", Kind.ASCENDING));
1085+
indexManager.addFieldIndex(fieldIndex("coll", "a", Kind.DESCENDING));
1086+
indexManager.addFieldIndex(fieldIndex("coll", "b", Kind.ASCENDING));
1087+
indexManager.addFieldIndex(fieldIndex("coll", "b", Kind.ASCENDING, "a", Kind.ASCENDING));
1088+
1089+
// OR query without orderBy without limit which has missing sub-target indexes.
1090+
Query query1 = query("coll").filter(orFilters(filter("a", "==", 1), filter("c", "==", 1)));
1091+
validateIndexType(query1, IndexManager.IndexType.NONE);
1092+
1093+
// OR query with explicit orderBy without limit which has missing sub-target indexes.
1094+
Query query2 =
1095+
query("coll")
1096+
.filter(orFilters(filter("a", "==", 1), filter("c", "==", 1)))
1097+
.orderBy(orderBy("c"));
1098+
validateIndexType(query2, IndexManager.IndexType.NONE);
1099+
1100+
// OR query with implicit orderBy without limit which has missing sub-target indexes.
1101+
Query query3 = query("coll").filter(orFilters(filter("a", "==", 1), filter("c", ">", 1)));
1102+
validateIndexType(query3, IndexManager.IndexType.NONE);
1103+
1104+
// OR query with explicit orderBy with limit which has missing sub-target indexes.
1105+
Query query4 =
1106+
query("coll")
1107+
.filter(orFilters(filter("a", "==", 1), filter("c", "==", 1)))
1108+
.orderBy(orderBy("c"))
1109+
.limitToFirst(2);
1110+
validateIndexType(query4, IndexManager.IndexType.NONE);
10841111

1085-
private void validateIsFullIndex(Query query) {
1086-
validateIndex(query, true);
1112+
// OR query with implicit orderBy with limit which has missing sub-target indexes.
1113+
Query query5 =
1114+
query("coll").filter(orFilters(filter("a", "==", 1), filter("c", ">", 1))).limitToLast(2);
1115+
validateIndexType(query5, IndexManager.IndexType.NONE);
1116+
1117+
// OR query without orderBy without limit which has all sub-target indexes.
1118+
Query query6 = query("coll").filter(orFilters(filter("a", "==", 1), filter("b", "==", 1)));
1119+
validateIndexType(query6, IndexManager.IndexType.FULL);
1120+
1121+
// OR query with explicit orderBy without limit which has all sub-target indexes.
1122+
Query query7 =
1123+
query("coll")
1124+
.filter(orFilters(filter("a", "==", 1), filter("b", "==", 1)))
1125+
.orderBy(orderBy("a"));
1126+
validateIndexType(query7, IndexManager.IndexType.FULL);
1127+
1128+
// OR query with implicit orderBy without limit which has all sub-target indexes.
1129+
Query query8 = query("coll").filter(orFilters(filter("a", ">", 1), filter("b", "==", 1)));
1130+
validateIndexType(query8, IndexManager.IndexType.FULL);
1131+
1132+
// OR query without orderBy with limit which has all sub-target indexes.
1133+
Query query9 =
1134+
query("coll").filter(orFilters(filter("a", "==", 1), filter("b", "==", 1))).limitToFirst(2);
1135+
validateIndexType(query9, IndexManager.IndexType.PARTIAL);
1136+
1137+
// OR query with explicit orderBy with limit which has all sub-target indexes.
1138+
Query query10 =
1139+
query("coll")
1140+
.filter(orFilters(filter("a", "==", 1), filter("b", "==", 1)))
1141+
.orderBy(orderBy("a"))
1142+
.limitToFirst(2);
1143+
validateIndexType(query10, IndexManager.IndexType.PARTIAL);
1144+
1145+
// OR query with implicit orderBy with limit which has all sub-target indexes.
1146+
Query query11 =
1147+
query("coll").filter(orFilters(filter("a", ">", 1), filter("b", "==", 1))).limitToLast(2);
1148+
validateIndexType(query11, IndexManager.IndexType.PARTIAL);
10871149
}
10881150

1089-
private void validateIndex(Query query, boolean validateFullIndex) {
1151+
private void validateIndexType(Query query, IndexManager.IndexType expected) {
10901152
IndexManager.IndexType indexType = indexManager.getIndexType(query.toTarget());
1091-
assertEquals(
1092-
indexType,
1093-
validateFullIndex ? IndexManager.IndexType.FULL : IndexManager.IndexType.PARTIAL);
1153+
assertEquals(indexType, expected);
10941154
}
10951155

10961156
private void verifySequenceNumber(

0 commit comments

Comments
 (0)