Skip to content

add update time to FieldIndex #2984

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Sep 20, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -311,8 +311,9 @@ public Index encodeFieldIndex(FieldIndex fieldIndex) {
return index.build();
}

public FieldIndex decodeFieldIndex(String collection_group, int indexId, Index index) {
FieldIndex fieldIndex = new FieldIndex(collection_group, indexId);
public FieldIndex decodeFieldIndex(
String collectionGroup, int indexId, Index index, int updateSeconds, int updateNanos) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please use SnapshotVersion for seconds+nanos.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done.

FieldIndex fieldIndex = new FieldIndex(collectionGroup, indexId);
for (Index.IndexField field : index.getFieldsList()) {
fieldIndex =
fieldIndex.withAddedField(
Expand All @@ -321,7 +322,8 @@ public FieldIndex decodeFieldIndex(String collection_group, int indexId, Index i
? FieldIndex.Segment.Kind.CONTAINS
: FieldIndex.Segment.Kind.ORDERED);
}

fieldIndex =
fieldIndex.withVersion(new SnapshotVersion(new Timestamp(updateSeconds, updateNanos)));
return fieldIndex;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -103,27 +103,36 @@ public void addFieldIndex(FieldIndex index) {
+ "index_id, "
+ "collection_group, "
+ "index_proto, "
+ "active) VALUES(?, ?, ?, ?)",
+ "active, "
+ "update_time_seconds, "
+ "update_time_nanos) VALUES(?, ?, ?, ?, ?, ?)",
currentMax + 1,
index.getCollectionId(),
index.getCollectionGroup(),
encodeFieldIndex(index),
true);
true,
index.getVersion().getTimestamp().getSeconds(),
index.getVersion().getTimestamp().getNanoseconds());
}

@Override
public void addIndexEntries(Document document) {
DocumentKey documentKey = document.getKey();
String collectionGroup = documentKey.getCollectionGroup();
db.query(
"SELECT index_id, index_proto FROM index_configuration WHERE collection_group = ? AND active = 1")
"SELECT index_id, index_proto, update_time_seconds, update_time_nanos "
+ "FROM index_configuration WHERE collection_group = ? AND active = 1")
.binding(collectionGroup)
.forEach(
row -> {
try {
int indexId = row.getInt(0);
FieldIndex fieldIndex =
serializer.decodeFieldIndex(
collectionGroup, row.getInt(0), Index.parseFrom(row.getBlob(1)));
collectionGroup,
row.getInt(0),
Index.parseFrom(row.getBlob(1)),
row.getInt(2),
row.getInt(3));

List<Value> values = extractFieldValue(document, fieldIndex);
if (values == null) return;
Expand Down Expand Up @@ -250,14 +259,18 @@ public Set<DocumentKey> getDocumentsMatchingTarget(Target target) {
List<FieldIndex> activeIndices = new ArrayList<>();

db.query(
"SELECT index_id, index_proto FROM index_configuration WHERE collection_group = ? AND active = 1")
"SELECT index_id, index_proto, update_time_seconds, update_time_nanos FROM index_configuration WHERE collection_group = ? AND active = 1")
.binding(collectionGroup)
.forEach(
row -> {
try {
FieldIndex fieldIndex =
serializer.decodeFieldIndex(
collectionGroup, row.getInt(0), Index.parseFrom(row.getBlob(1)));
collectionGroup,
row.getInt(0),
Index.parseFrom(row.getBlob(1)),
row.getInt(2),
row.getInt(3));
boolean matches = targetIndexMatcher.servedByIndex(fieldIndex);
if (matches) {
activeIndices.add(fieldIndex);
Expand Down Expand Up @@ -370,4 +383,28 @@ private boolean isMultiValueFilter(Target target, FieldPath fieldPath) {
private byte[] encodeFieldIndex(FieldIndex fieldIndex) {
return serializer.encodeFieldIndex(fieldIndex).toByteArray();
}

// TODO(indexing): Add support for fetching last N updated entries.
public List<FieldIndex> getFieldIndexes() {
List<FieldIndex> allIndexes = new ArrayList<>();
db.query(
"SELECT index_id, collection_group, index_proto, update_time_seconds, update_time_nanos FROM index_configuration "
+ "WHERE active = 1")
.forEach(
row -> {
try {
allIndexes.add(
serializer.decodeFieldIndex(
row.getString(1),
row.getInt(0),
Index.parseFrom(row.getBlob(2)),
row.getInt(3),
row.getInt(4)));
} catch (InvalidProtocolBufferException e) {
throw fail("Failed to decode index: " + e);
}
});

return allIndexes;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -56,29 +56,32 @@ public String toString() {
}
}

private final String collectionId;
private final String collectionGroup;
private final int indexId;
private final List<Segment> segments;
private final SnapshotVersion version;

public FieldIndex(String collectionId, int indexId) {
this.collectionId = collectionId;
public FieldIndex(String collectionGroup, int indexId) {
this.collectionGroup = collectionGroup;
this.segments = new ArrayList<>();
this.indexId = indexId;
this.version = SnapshotVersion.NONE;
}

public FieldIndex(String collectionId) {
this(collectionId, -1);
}

FieldIndex(String collectionId, int indexId, List<Segment> segments) {
this.collectionId = collectionId;
FieldIndex(String collectionGroup, int indexId, List<Segment> segments, SnapshotVersion version) {
this.collectionGroup = collectionGroup;
this.segments = segments;
this.indexId = indexId;
this.version = version;
}

/** The collection ID this index applies to. */
public String getCollectionId() {
return collectionId;
public String getCollectionGroup() {
return collectionGroup;
}

/**
Expand All @@ -97,6 +100,10 @@ public int segmentCount() {
return segments.size();
}

public SnapshotVersion getVersion() {
return version;
}

@NonNull
@Override
public Iterator<Segment> iterator() {
Expand All @@ -107,7 +114,12 @@ public Iterator<Segment> iterator() {
public FieldIndex withAddedField(FieldPath fieldPath, Segment.Kind kind) {
List<Segment> newSegments = new ArrayList<>(segments);
newSegments.add(new AutoValue_FieldIndex_Segment(fieldPath, kind));
return new FieldIndex(collectionId, indexId, newSegments);
return new FieldIndex(collectionGroup, indexId, newSegments, version);
}

/** Returns a new field index with the updated version. */
public FieldIndex withVersion(SnapshotVersion version) {
return new FieldIndex(collectionGroup, indexId, segments, version);
}

@Override
Expand All @@ -118,18 +130,22 @@ public boolean equals(Object o) {
FieldIndex fieldIndex = (FieldIndex) o;

if (!segments.equals(fieldIndex.segments)) return false;
return collectionId.equals(fieldIndex.collectionId);
if (!version.equals(fieldIndex.version)) return false;
return collectionGroup.equals(fieldIndex.collectionGroup);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

updateTime?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done.

}

@Override
public int hashCode() {
int result = collectionId.hashCode();
int result = collectionGroup.hashCode();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

updateTime?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done.

result = 31 * result + segments.hashCode();
result = 31 * result + version.hashCode();
return result;
}

@Override
public String toString() {
return String.format("FieldIndex{collectionId='%s', segments=%s}", collectionId, segments);
return String.format(
"FieldIndex{collectionGroup='%s', segments=%s, version=%s}",
collectionGroup, segments, version);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ public TargetIndexMatcher(Target target) {
* @throws AssertionError if the index is for a different collection
*/
public boolean servedByIndex(FieldIndex index) {
hardAssert(index.getCollectionId().equals(collectionId), "Collection IDs do not match");
hardAssert(index.getCollectionGroup().equals(collectionId), "Collection IDs do not match");
for (int i = 0; i < index.segmentCount(); ++i) {
if (!canUseSegment(index.getSegment(i))) {
return false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,15 @@
import static com.google.firebase.firestore.testutil.TestUtil.orderBy;
import static com.google.firebase.firestore.testutil.TestUtil.path;
import static com.google.firebase.firestore.testutil.TestUtil.query;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;

import com.google.firebase.Timestamp;
import com.google.firebase.firestore.core.Query;
import com.google.firebase.firestore.model.DocumentKey;
import com.google.firebase.firestore.model.FieldIndex;
import com.google.firebase.firestore.model.MutableDocument;
import com.google.firebase.firestore.model.SnapshotVersion;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -265,6 +268,19 @@ public void testCollectionGroup() {
verifyResults(query, "coll1/doc1", "coll2/doc2/coll1/doc1");
}

@Test
public void testUpdateTime() {
indexManager.addFieldIndex(
new FieldIndex("coll1")
.withAddedField(field("value"), FieldIndex.Segment.Kind.ORDERED)
.withVersion(new SnapshotVersion(new Timestamp(10, 20))));

List<FieldIndex> indexes = ((SQLiteIndexManager) indexManager).getFieldIndexes();
assertEquals(indexes.size(), 1);
FieldIndex index = indexes.get(0);
assertEquals(index.getVersion(), new SnapshotVersion(new Timestamp(10, 20)));
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This test seems better suited for our serialization tests.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The test here checks that the persistence queries in addFieldIndex and getFieldIndex are correct.


private void addDoc(String key, Map<String, Object> data) {
MutableDocument doc = doc(key, 1, data);
indexManager.addIndexEntries(doc);
Expand Down