Skip to content

Commit 22b0fc5

Browse files
cherylEnkiducherylEnkidu
committed
Auto Indexing heuristic experiment
1 parent c9990de commit 22b0fc5

File tree

4 files changed

+272
-0
lines changed

4 files changed

+272
-0
lines changed

‎firebase-firestore/src/main/java/com/google/firebase/firestore/local/QueryEngine.java

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
package com.google.firebase.firestore.local;
1616

17+
import static com.google.firebase.firestore.util.Assert.fail;
1718
import static com.google.firebase.firestore.util.Assert.hardAssert;
1819

1920
import com.google.firebase.database.collection.ImmutableSortedMap;
@@ -103,6 +104,22 @@ public ImmutableSortedMap<DocumentKey, Document> getDocumentsMatchingQuery(
103104
return result;
104105
}
105106

107+
public ImmutableSortedMap<DocumentKey, Document> getDocumentsMatchingQueryTest(
108+
Query query, boolean autoIndexing, QueryContext counter) {
109+
hardAssert(initialized, "initialize() not called");
110+
111+
ImmutableSortedMap<DocumentKey, Document> result;
112+
if (autoIndexing) {
113+
result = performQueryUsingIndex(query);
114+
if (result == null) {
115+
fail("createTargetIndices fails");
116+
}
117+
} else {
118+
result = executeFullCollectionScan(query, counter);
119+
}
120+
return result;
121+
}
122+
106123
private void CreateCacheIndices(Query query, QueryContext counter, int resultSize) {
107124
if (counter.fullScanCount > 2 * resultSize) {
108125
indexManager.createTargetIndices(query.toTarget());

‎firebase-firestore/src/main/java/com/google/firebase/firestore/model/TargetIndexMatcher.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,10 @@ public FieldIndex BuildTargetIndex() {
213213
}
214214

215215
for (OrderBy orderBy : orderBys) {
216+
if (orderBy.getField().isKeyField()) {
217+
continue;
218+
}
219+
216220
if (uniqueFields.contains(orderBy.getField())) {
217221
continue;
218222
}

‎firebase-firestore/src/test/java/com/google/firebase/firestore/local/AutoIndexingExperiment.java

Lines changed: 242 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,242 @@
1+
package com.google.firebase.firestore.local;
2+
3+
import static com.google.firebase.firestore.testutil.TestUtil.doc;
4+
import static com.google.firebase.firestore.testutil.TestUtil.docMap;
5+
import static com.google.firebase.firestore.testutil.TestUtil.filter;
6+
import static com.google.firebase.firestore.testutil.TestUtil.map;
7+
import static com.google.firebase.firestore.testutil.TestUtil.query;
8+
import static org.junit.Assert.assertEquals;
9+
10+
import com.google.android.gms.common.internal.Preconditions;
11+
import com.google.firebase.database.collection.ImmutableSortedMap;
12+
import com.google.firebase.database.collection.ImmutableSortedSet;
13+
import com.google.firebase.firestore.auth.User;
14+
import com.google.firebase.firestore.core.Query;
15+
import com.google.firebase.firestore.core.View;
16+
import com.google.firebase.firestore.model.Document;
17+
import com.google.firebase.firestore.model.DocumentKey;
18+
import com.google.firebase.firestore.model.DocumentSet;
19+
import com.google.firebase.firestore.model.FieldIndex;
20+
import com.google.firebase.firestore.model.MutableDocument;
21+
import java.util.ArrayList;
22+
import java.util.Arrays;
23+
import java.util.Collections;
24+
import java.util.List;
25+
import java.util.Map;
26+
import java.util.concurrent.Callable;
27+
import java.util.concurrent.TimeUnit;
28+
import javax.annotation.Nullable;
29+
import org.junit.Before;
30+
import org.junit.Test;
31+
import org.junit.runner.RunWith;
32+
import org.robolectric.RobolectricTestRunner;
33+
import org.robolectric.annotation.Config;
34+
35+
@RunWith(RobolectricTestRunner.class)
36+
@Config(manifest = Config.NONE)
37+
public class AutoIndexingExperiment {
38+
static List<Object> values =
39+
Arrays.asList(
40+
"Hello world",
41+
46239847,
42+
-1984092375,
43+
Arrays.asList(1, "foo", 3, 5, 8, 10, 11),
44+
Arrays.asList(1, "foo", 9, 5, 8),
45+
Double.NaN,
46+
map("nested", "random"));
47+
48+
private Persistence persistence;
49+
private RemoteDocumentCache remoteDocumentCache;
50+
private MutationQueue mutationQueue;
51+
private DocumentOverlayCache documentOverlayCache;
52+
53+
protected IndexManager indexManager;
54+
protected QueryEngine queryEngine;
55+
56+
private @Nullable Boolean expectFullCollectionScan;
57+
58+
@Before
59+
public void setUp() {
60+
expectFullCollectionScan = null;
61+
62+
persistence = PersistenceTestHelpers.createSQLitePersistence();
63+
64+
indexManager = persistence.getIndexManager(User.UNAUTHENTICATED);
65+
mutationQueue = persistence.getMutationQueue(User.UNAUTHENTICATED, indexManager);
66+
documentOverlayCache = persistence.getDocumentOverlayCache(User.UNAUTHENTICATED);
67+
remoteDocumentCache = persistence.getRemoteDocumentCache();
68+
queryEngine = new QueryEngine();
69+
70+
indexManager.start();
71+
mutationQueue.start();
72+
73+
remoteDocumentCache.setIndexManager(indexManager);
74+
75+
LocalDocumentsView localDocuments =
76+
new LocalDocumentsView(
77+
remoteDocumentCache, mutationQueue, documentOverlayCache, indexManager) {
78+
@Override
79+
public ImmutableSortedMap<DocumentKey, Document> getDocumentsMatchingQuery(
80+
Query query, FieldIndex.IndexOffset offset) {
81+
assertEquals(
82+
"Observed query execution mode did not match expectation",
83+
expectFullCollectionScan,
84+
FieldIndex.IndexOffset.NONE.equals(offset));
85+
return super.getDocumentsMatchingQuery(query, offset);
86+
}
87+
};
88+
queryEngine.initialize(localDocuments, indexManager, false);
89+
}
90+
91+
/** Adds the provided documents to the remote document cache. */
92+
protected void addDocument(MutableDocument... docs) {
93+
persistence.runTransaction(
94+
"addDocument",
95+
() -> {
96+
for (MutableDocument doc : docs) {
97+
remoteDocumentCache.add(doc, doc.getVersion());
98+
}
99+
});
100+
}
101+
102+
protected <T> T expectOptimizedCollectionScan(Callable<T> c) throws Exception {
103+
try {
104+
expectFullCollectionScan = false;
105+
return c.call();
106+
} finally {
107+
expectFullCollectionScan = null;
108+
}
109+
}
110+
111+
private <T> T expectFullCollectionScan(Callable<T> c) throws Exception {
112+
try {
113+
expectFullCollectionScan = true;
114+
return c.call();
115+
} finally {
116+
expectFullCollectionScan = null;
117+
}
118+
}
119+
120+
protected DocumentSet runQuery(Query query, boolean autoIndexing, QueryContext counter) {
121+
Preconditions.checkNotNull(
122+
expectFullCollectionScan,
123+
"Encountered runQuery() call not wrapped in expectOptimizedCollectionQuery()/expectFullCollectionQuery()");
124+
ImmutableSortedMap<DocumentKey, Document> docs =
125+
queryEngine.getDocumentsMatchingQueryTest(query, autoIndexing, counter);
126+
View view =
127+
new View(query, new ImmutableSortedSet<>(Collections.emptyList(), DocumentKey::compareTo));
128+
View.DocumentChanges viewDocChanges = view.computeDocChanges(docs);
129+
return view.applyChanges(viewDocChanges).getSnapshot().getDocuments();
130+
}
131+
132+
private void createTestingDocument(
133+
String basePath, int documentID, boolean isMatched, int numOfFields) {
134+
Map<String, Object> fields = map("match", isMatched);
135+
136+
// Randomly generate the rest of fields
137+
for (int i = 2; i <= numOfFields; i++) {
138+
int valueIndex = (int) (Math.random() * values.size()) % values.size();
139+
fields.put("field" + i, values.get(valueIndex));
140+
}
141+
142+
MutableDocument doc = doc(basePath + "/" + documentID, 1, fields);
143+
addDocument(doc);
144+
145+
indexManager.updateIndexEntries(docMap(doc));
146+
indexManager.updateCollectionGroup(basePath, FieldIndex.IndexOffset.fromDocument(doc));
147+
}
148+
149+
private void createTestingCollection(
150+
String basePath, int totalSetCount, int portion /*0 - 10*/, int numOfFields /* 1 - 30*/) {
151+
int documentCounter = 0;
152+
for (int i = 1; i <= totalSetCount; i++) {
153+
// Generate a random order list of 0 ... 9
154+
ArrayList<Integer> indexes = new ArrayList<>();
155+
for (int index = 0; index < 10; index++) {
156+
indexes.add(index);
157+
}
158+
Collections.shuffle(indexes);
159+
160+
for (int match = 0; match < portion; match++) {
161+
int currentID = documentCounter + indexes.get(match);
162+
createTestingDocument(basePath, currentID, true, numOfFields);
163+
}
164+
for (int unmatch = portion; unmatch < 10; unmatch++) {
165+
int currentID = documentCounter + indexes.get(unmatch);
166+
createTestingDocument(basePath, currentID, false, numOfFields);
167+
}
168+
documentCounter += 10;
169+
}
170+
}
171+
172+
@Test
173+
public void testCombinesIndexedWithNonIndexedResults() throws Exception {
174+
// Every set contains 10 documents
175+
final int numOfSet = 100;
176+
// could overflow. Currently it is safe when numOfSet set to 1000 and running on macbook M1
177+
long totalBeforeIndex = 0;
178+
long totalAfterIndex = 0;
179+
long totalDocumentCount = 0;
180+
long totalResultCount = 0;
181+
182+
// Temperate heuristic
183+
double without = 3.7;
184+
double with = 4.9;
185+
186+
for (int totalSetCount = 1; totalSetCount <= numOfSet; totalSetCount *= 10) {
187+
// portion stands for the percentage of documents matching query
188+
for (int portion = 0; portion <= 10; portion++) {
189+
for (int numOfFields = 1; numOfFields <= 31; numOfFields += 10) {
190+
String basePath = "documentCount" + totalSetCount;
191+
// Auto indexing
192+
Query query = query(basePath).filter(filter("match", "==", true));
193+
indexManager.createTargetIndices(query.toTarget());
194+
createTestingCollection(basePath, totalSetCount, portion, numOfFields);
195+
196+
QueryContext counterWithoutIndex = new QueryContext();
197+
long beforeAutoStart = System.nanoTime();
198+
DocumentSet results =
199+
expectFullCollectionScan(() -> runQuery(query, false, counterWithoutIndex));
200+
long beforeAutoEnd = System.nanoTime();
201+
long millisecondsBeforeAuto =
202+
TimeUnit.MILLISECONDS.convert(
203+
(beforeAutoEnd - beforeAutoStart), TimeUnit.NANOSECONDS);
204+
totalBeforeIndex += (beforeAutoEnd - beforeAutoStart);
205+
totalDocumentCount += counterWithoutIndex.fullScanCount;
206+
assertEquals(portion * totalSetCount, results.size());
207+
208+
QueryContext counterWithIndex = new QueryContext();
209+
long autoStart = System.nanoTime();
210+
results = expectOptimizedCollectionScan(() -> runQuery(query, true, counterWithIndex));
211+
long autoEnd = System.nanoTime();
212+
long millisecondsAfterAuto =
213+
TimeUnit.MILLISECONDS.convert((autoEnd - autoStart), TimeUnit.NANOSECONDS);
214+
totalAfterIndex += (autoEnd - autoStart);
215+
assertEquals(portion * totalSetCount, results.size());
216+
totalResultCount += results.size();
217+
if (millisecondsBeforeAuto > millisecondsAfterAuto) {
218+
System.out.println(
219+
"Auto Indexing saves time when total of documents inside collection is "
220+
+ totalSetCount * 10
221+
+ ". The matching percentage is "
222+
+ portion
223+
+ "0%. And each document contains "
224+
+ numOfFields
225+
+ " fields.\n"
226+
+ "Weight result for without auto indexing is "
227+
+ without * counterWithoutIndex.fullScanCount
228+
+ ". And weight result for auto indexing is "
229+
+ with * results.size());
230+
}
231+
}
232+
}
233+
}
234+
235+
System.out.println(
236+
"The time heuristic is "
237+
+ (totalBeforeIndex / totalDocumentCount)
238+
+ " before auto indexing");
239+
System.out.println(
240+
"The time heuristic is " + (totalAfterIndex / totalResultCount) + " after auto indexing");
241+
}
242+
}

‎firebase-firestore/src/testUtil/java/com/google/firebase/firestore/testutil/TestUtil.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,15 @@ public static <T extends Document> ImmutableSortedMap<DocumentKey, T> docMap(T..
238238
return map;
239239
}
240240

241+
public static <T extends Document> ImmutableSortedMap<DocumentKey, T> docMap(List<T> documents) {
242+
ImmutableSortedMap<DocumentKey, T> map =
243+
(ImmutableSortedMap<DocumentKey, T>) emptyDocumentMap();
244+
for (T maybeDocument : documents) {
245+
map = map.insert(maybeDocument.getKey(), maybeDocument);
246+
}
247+
return map;
248+
}
249+
241250
public static DocumentSet docSet(Comparator<Document> comparator, MutableDocument... documents) {
242251
DocumentSet set = DocumentSet.emptySet(comparator);
243252
for (MutableDocument document : documents) {

0 commit comments

Comments
 (0)