Leveldb schema/key change for indexing (#9268)

Author: wu-hui

Firestore/core/src/local/leveldb_key.cc

@@ -50,6 +50,9 @@ const char* kCollectionParentsTable = "collection_parent";
 const char* kRemoteDocumentReadTimeTable = "remote_document_read_time";
 const char* kBundlesTable = "bundles";
 const char* kNamedQueriesTable = "named_queries";
+const char* kIndexConfigurationTable = "index_configuration";
+const char* kIndexStateTable = "index_state";
+const char* kIndexEntriesTable = "index_entries";
 
 /**
  * Labels for the components of keys. These serve to make keys self-describing.
@@ -115,6 +118,15 @@ enum ComponentLabel {
   /** A component containing the name of a named query. */
   QueryName = 18,
 
+  /** A component containing a index id. */
+  IndexId = 19,
+
+  /** A component containing an array index value. */
+  IndexArrayValue = 20,
+
+  /** A component containing a directional index value. */
+  IndexDirectionalValue = 21,
+
   /**
    * A path segment describes just a single segment in a resource path. Path
    * segments that occur sequentially in a key represent successive segments in
@@ -201,6 +213,18 @@ class Reader {
     return ReadLabeledString(ComponentLabel::QueryName);
   }
 
+  int32_t ReadIndexId() {
+    return ReadLabeledInt32(ComponentLabel::IndexId);
+  }
+
+  std::string ReadIndexArrayValue() {
+    return ReadLabeledString(ComponentLabel::IndexArrayValue);
+  }
+
+  std::string ReadIndexDirectionalValue() {
+    return ReadLabeledString(ComponentLabel::IndexDirectionalValue);
+  }
+
   /**
    * Reads a snapshot version, encoded as a component label and a pair of
    * seconds (int64) and nanoseconds (int32).
@@ -567,6 +591,21 @@ std::string Reader::Describe() {
       if (ok_) {
         absl::StrAppend(&description, " query_name=", query_name);
       }
+    } else if (label == ComponentLabel::IndexId) {
+      int32_t index_id = ReadIndexId();
+      if (ok_) {
+        absl::StrAppend(&description, " index_id=", index_id);
+      }
+    } else if (label == ComponentLabel::IndexArrayValue) {
+      std::string value = ReadIndexArrayValue();
+      if (ok_) {
+        absl::StrAppend(&description, " array_value=", std::move(value));
+      }
+    } else if (label == ComponentLabel::IndexDirectionalValue) {
+      std::string value = ReadIndexDirectionalValue();
+      if (ok_) {
+        absl::StrAppend(&description, " directional_value=", std::move(value));
+      }
     } else {
       absl::StrAppend(&description, " unknown label=", static_cast<int>(label));
       Fail();
@@ -650,6 +689,18 @@ class Writer {
     }
   }
 
+  void WriteIndexId(int32_t id) {
+    WriteLabeledInt32(ComponentLabel::IndexId, id);
+  }
+
+  void WriteIndexArrayValue(absl::string_view value) {
+    WriteLabeledString(ComponentLabel::IndexArrayValue, value);
+  }
+
+  void WriteIndexDirectionalValue(absl::string_view value) {
+    WriteLabeledString(ComponentLabel::IndexDirectionalValue, value);
+  }
+
  private:
   /** Writes a component label to the given key destination. */
   void WriteComponentLabel(ComponentLabel label) {
@@ -1094,6 +1145,87 @@ bool LevelDbNamedQueryKey::Decode(absl::string_view key) {
   return reader.ok();
 }
 
+std::string LevelDbIndexConfigurationKey::KeyPrefix() {
+  Writer writer;
+  writer.WriteTableName(kIndexConfigurationTable);
+  return writer.result();
+}
+
+std::string LevelDbIndexConfigurationKey::Key(int32_t id) {
+  Writer writer;
+  writer.WriteTableName(kIndexConfigurationTable);
+  writer.WriteIndexId(id);
+  writer.WriteTerminator();
+  return writer.result();
+}
+
+bool LevelDbIndexConfigurationKey::Decode(absl::string_view key) {
+  Reader reader{key};
+  reader.ReadTableNameMatching(kIndexConfigurationTable);
+  index_id_ = reader.ReadIndexId();
+  reader.ReadTerminator();
+  return reader.ok();
+}
+
+std::string LevelDbIndexStateKey::KeyPrefix() {
+  Writer writer;
+  writer.WriteTableName(kIndexStateTable);
+  return writer.result();
+}
+
+std::string LevelDbIndexStateKey::Key(int32_t index_id,
+                                      absl::string_view user_id) {
+  Writer writer;
+  writer.WriteTableName(kIndexStateTable);
+  writer.WriteIndexId(index_id);
+  writer.WriteUserId(user_id);
+  writer.WriteTerminator();
+  return writer.result();
+}
+
+bool LevelDbIndexStateKey::Decode(absl::string_view key) {
+  Reader reader{key};
+  reader.ReadTableNameMatching(kIndexStateTable);
+  index_id_ = reader.ReadIndexId();
+  user_id_ = reader.ReadUserId();
+  reader.ReadTerminator();
+  return reader.ok();
+}
+
+std::string LevelDbIndexEntryKey::KeyPrefix() {
+  Writer writer;
+  writer.WriteTableName(kIndexEntriesTable);
+  return writer.result();
+}
+
+std::string LevelDbIndexEntryKey::Key(int32_t index_id,
+                                      absl::string_view user_id,
+                                      absl::string_view array_value,
+                                      absl::string_view dicrectional_value,
+                                      absl::string_view document_key) {
+  Writer writer;
+  writer.WriteTableName(kIndexEntriesTable);
+  writer.WriteIndexId(index_id);
+  writer.WriteUserId(user_id);
+  writer.WriteIndexArrayValue(array_value);
+  writer.WriteIndexDirectionalValue(dicrectional_value);
+  writer.WriteDocumentId(document_key);
+  writer.WriteTerminator();
+  return writer.result();
+}
+
+bool LevelDbIndexEntryKey::Decode(absl::string_view key) {
+  Reader reader{key};
+  reader.ReadTableNameMatching(kIndexEntriesTable);
+  index_id_ = reader.ReadIndexId();
+  user_id_ = reader.ReadUserId();
+  array_value_ = reader.ReadIndexArrayValue();
+  directional_value_ = reader.ReadIndexDirectionalValue();
+  document_key_ = reader.ReadDocumentId();
+  reader.ReadTerminator();
+  return reader.ok();
+}
+
 }  // namespace local
 }  // namespace firestore
 }  // namespace firebase

Firestore/core/src/local/leveldb_key.h

@@ -97,6 +97,23 @@ namespace local {
 // named_queries:
 //   - table_name: string = "named_queries"
 //   - name: string
+//
+// index_configuration:
+//   - table_name: string = "index_configuration"
+//   - index_id: int32_t
+//
+// index_state:
+//   - table_name: string = "index_state"
+//   - index_id: int32_t
+//   - user_id: string
+//
+// index_entries:
+//   - table_name: string = "index_entries"
+//   - index_id: int32_t
+//   - user_id: string
+//   - array_value: string
+//   - directional_value: string
+//   - document_name: string
 
 /**
  * Parses the given key and returns a human readable description of its
@@ -713,6 +730,150 @@ class LevelDbNamedQueryKey {
   std::string name_;
 };
 
+/**
+ * A key in the index_configuration table, storing the index definition proto,
+ * and the collection (group) it applies to.
+ */
+class LevelDbIndexConfigurationKey {
+ public:
+  /**
+   * Creates a key prefix that points just before the first key of the table.
+   */
+  static std::string KeyPrefix();
+
+  /**
+   * Creates a key that points to the key for the given index id.
+   */
+  static std::string Key(int32_t id);
+
+  /**
+   * Decodes the given complete key, storing the decoded values in this
+   * instance.
+   *
+   * @return true if the key successfully decoded, false otherwise. If false is
+   * returned, this instance is in an undefined state until the next call to
+   * `Decode()`.
+   */
+  ABSL_MUST_USE_RESULT
+  bool Decode(absl::string_view key);
+
+  /** The index id for this entry. */
+  int32_t index_id() const {
+    return index_id_;
+  }
+
+ private:
+  int32_t index_id_;
+};
+
+/**
+ * A key in the index_state table, storing the per index per user state tracking
+ * backfilling state for each index.
+ */
+class LevelDbIndexStateKey {
+ public:
+  /**
+   * Creates a key prefix that points just before the first key of the table.
+   */
+  static std::string KeyPrefix();
+
+  /**
+   * Creates a key that points to the key for the given index id and user id.
+   */
+  static std::string Key(int32_t index_id, absl::string_view user_id);
+
+  /**
+   * Decodes the given complete key, storing the decoded values in this
+   * instance.
+   *
+   * @return true if the key successfully decoded, false otherwise. If false is
+   * returned, this instance is in an undefined state until the next call to
+   * `Decode()`.
+   */
+  ABSL_MUST_USE_RESULT
+  bool Decode(absl::string_view key);
+
+  /** The index id for this entry. */
+  int32_t index_id() const {
+    return index_id_;
+  }
+
+  /** The user id for this entry. */
+  const std::string& user_id() const {
+    return user_id_;
+  }
+
+ private:
+  int32_t index_id_;
+  std::string user_id_;
+};
+
+/**
+ * A key in the index_entries table, storing the the encoded entries for all
+ * fields used by a given index.
+ *
+ * Note: `array_value` is expected to be set for all queries.
+ */
+class LevelDbIndexEntryKey {
+ public:
+  /**
+   * Creates a key prefix that points just before the first key of the table.
+   */
+  static std::string KeyPrefix();
+
+  /**
+   * Creates a key that points to the key for the given index entry fields.
+   */
+  static std::string Key(int32_t index_id,
+                         absl::string_view user_id,
+                         absl::string_view array_value,
+                         absl::string_view directional_value,
+                         absl::string_view document_name);
+
+  /**
+   * Decodes the given complete key, storing the decoded values in this
+   * instance.
+   *
+   * @return true if the key successfully decoded, false otherwise. If false is
+   * returned, this instance is in an undefined state until the next call to
+   * `Decode()`.
+   */
+  ABSL_MUST_USE_RESULT
+  bool Decode(absl::string_view key);
+
+  /** The index id for this entry. */
+  int32_t index_id() const {
+    return index_id_;
+  }
+
+  /** The user id for this entry. */
+  const std::string& user_id() const {
+    return user_id_;
+  }
+
+  /** The encoded array index value for this entry. */
+  const std::string& array_value() const {
+    return array_value_;
+  }
+
+  /** The encoded directional index value for this entry. */
+  const std::string& directional_value() const {
+    return directional_value_;
+  }
+
+  /** The document key this entry points to. */
+  const std::string& document_key() const {
+    return document_key_;
+  }
+
+ private:
+  int32_t index_id_;
+  std::string user_id_;
+  std::string array_value_;
+  std::string directional_value_;
+  std::string document_key_;
+};
+
 }  // namespace local
 }  // namespace firestore
 }  // namespace firebase