-
Notifications
You must be signed in to change notification settings - Fork 627
Initial base classes for index backfill #2950
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
6 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
50 changes: 50 additions & 0 deletions
50
firebase-firestore/src/main/java/com/google/firebase/firestore/index/IndexEntry.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| // Copyright 2021 Google LLC | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package com.google.firebase.firestore.index; | ||
|
|
||
| /** | ||
| * Represents an index entry saved by the SDK in the local storage. Temporary placeholder, since | ||
| * we'll probably serialize the indexValue right away rather than store it. | ||
| */ | ||
| // TODO(indexing) | ||
| public class IndexEntry { | ||
| private final int indexId; | ||
| private final byte[] indexValue; | ||
| private final String uid; | ||
| private final String documentId; | ||
|
|
||
| public IndexEntry(int indexId, byte[] indexValue, String uid, String documentId) { | ||
| this.indexId = indexId; | ||
| this.indexValue = indexValue; | ||
| this.uid = uid; | ||
| this.documentId = documentId; | ||
| } | ||
|
|
||
| public int getIndexId() { | ||
| return indexId; | ||
| } | ||
|
|
||
| public byte[] getIndexValue() { | ||
| return indexValue; | ||
| } | ||
|
|
||
| public String getUid() { | ||
| return uid; | ||
| } | ||
|
|
||
| public String getDocumentId() { | ||
| return documentId; | ||
| } | ||
| } | ||
152 changes: 152 additions & 0 deletions
152
firebase-firestore/src/main/java/com/google/firebase/firestore/local/IndexBackfiller.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,152 @@ | ||
| // Copyright 2021 Google LLC | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package com.google.firebase.firestore.local; | ||
|
|
||
| import androidx.annotation.Nullable; | ||
| import androidx.annotation.VisibleForTesting; | ||
| import com.google.firebase.firestore.index.IndexEntry; | ||
| import com.google.firebase.firestore.util.AsyncQueue; | ||
| import java.util.concurrent.TimeUnit; | ||
|
|
||
| /** Implements the steps for backfilling indexes. */ | ||
| public class IndexBackfiller { | ||
| /** How long we wait to try running index backfill after SDK initialization. */ | ||
| private static final long INITIAL_BACKFILL_DELAY_MS = TimeUnit.SECONDS.toMillis(15); | ||
| /** Minimum amount of time between backfill checks, after the first one. */ | ||
| private static final long REGULAR_BACKFILL_DELAY_MS = TimeUnit.MINUTES.toMillis(1); | ||
|
|
||
| private final SQLitePersistence persistence; | ||
|
|
||
| public IndexBackfiller(SQLitePersistence sqLitePersistence) { | ||
| this.persistence = sqLitePersistence; | ||
| } | ||
|
|
||
| public static class Results { | ||
| private final boolean hasRun; | ||
|
|
||
| private final int entriesAdded; | ||
| private final int entriesRemoved; | ||
|
|
||
| static IndexBackfiller.Results DidNotRun() { | ||
| return new IndexBackfiller.Results(/* hasRun= */ false, 0, 0); | ||
| } | ||
|
|
||
| Results(boolean hasRun, int entriesAdded, int entriesRemoved) { | ||
| this.hasRun = hasRun; | ||
| this.entriesAdded = entriesAdded; | ||
| this.entriesRemoved = entriesRemoved; | ||
| } | ||
|
|
||
| public boolean hasRun() { | ||
| return hasRun; | ||
| } | ||
|
|
||
| public int getEntriesAdded() { | ||
| return entriesAdded; | ||
| } | ||
|
|
||
| public int getEntriesRemoved() { | ||
| return entriesRemoved; | ||
| } | ||
| } | ||
|
|
||
| public class BackfillScheduler implements Scheduler { | ||
| private final AsyncQueue asyncQueue; | ||
| private final LocalStore localStore; | ||
| private boolean hasRun = false; | ||
| @Nullable private AsyncQueue.DelayedTask backfillTask; | ||
|
|
||
| public BackfillScheduler(AsyncQueue asyncQueue, LocalStore localStore) { | ||
| this.asyncQueue = asyncQueue; | ||
| this.localStore = localStore; | ||
| } | ||
|
|
||
| @Override | ||
| public void start() { | ||
| scheduleBackfill(); | ||
| } | ||
|
|
||
| @Override | ||
| public void stop() { | ||
| if (backfillTask != null) { | ||
| backfillTask.cancel(); | ||
| } | ||
| } | ||
|
|
||
| private void scheduleBackfill() { | ||
| long delay = hasRun ? REGULAR_BACKFILL_DELAY_MS : INITIAL_BACKFILL_DELAY_MS; | ||
| backfillTask = | ||
| asyncQueue.enqueueAfterDelay( | ||
| AsyncQueue.TimerId.INDEX_BACKFILL, | ||
| delay, | ||
| () -> { | ||
| localStore.backfillIndexes(IndexBackfiller.this); | ||
| hasRun = true; | ||
| scheduleBackfill(); | ||
| }); | ||
| } | ||
| } | ||
|
|
||
| public BackfillScheduler newScheduler(AsyncQueue asyncQueue, LocalStore localStore) { | ||
| return new BackfillScheduler(asyncQueue, localStore); | ||
| } | ||
|
|
||
| // TODO(indexing): Figure out which index entries to backfill. | ||
| public Results backfill() { | ||
| int numIndexesWritten = 0; | ||
| int numIndexesRemoved = 0; | ||
| return new Results(/* hasRun= */ true, numIndexesWritten, numIndexesRemoved); | ||
| } | ||
|
|
||
| @VisibleForTesting | ||
| void addIndexEntry(IndexEntry entry) { | ||
| persistence.execute( | ||
| "INSERT OR IGNORE INTO index_entries (" | ||
| + "index_id, " | ||
| + "index_value, " | ||
| + "uid, " | ||
| + "document_id) VALUES(?, ?, ?, ?)", | ||
| entry.getIndexId(), | ||
| entry.getIndexValue(), | ||
| entry.getUid(), | ||
| entry.getDocumentId()); | ||
| } | ||
|
|
||
| @VisibleForTesting | ||
| void removeIndexEntry(int indexId, String uid, String documentId) { | ||
| persistence.execute( | ||
| "DELETE FROM index_entries " | ||
| + "WHERE index_id = ? " | ||
| + "AND uid = ?" | ||
| + "AND document_id = ?", | ||
| indexId, | ||
| uid, | ||
| documentId); | ||
| ; | ||
| } | ||
|
|
||
| @Nullable | ||
| @VisibleForTesting | ||
| IndexEntry getIndexEntry(int indexId) { | ||
| return persistence | ||
| .query("SELECT index_value, uid, document_id FROM index_entries WHERE index_id = ?") | ||
| .binding(indexId) | ||
| .firstValue( | ||
| row -> | ||
| row == null | ||
| ? null | ||
| : new IndexEntry(indexId, row.getBlob(0), row.getString(1), row.getString(2))); | ||
| } | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Please add "TODO(indexing)".
Note that we already committed
FieldIndex. In #2963, FieldIndex also gets anindexId.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Done.