Index deserialization from old pickle and fixes

Author: Dmitry Yashunin

hnswlib/hnswalg.h

@@ -768,7 +768,7 @@ namespace hnswlib {
          * @param internalId
          */
         void markDeletedInternal(tableint internalId) {
-            assert(internalId < max_elements_);
+            assert(internalId < cur_element_count);
             if (!isMarkedDeleted(internalId))
             {
                 unsigned char *ll_cur = ((unsigned char *)get_linklist0(internalId))+2;
@@ -800,7 +800,7 @@ namespace hnswlib {
          * @param internalId
          */
         void unmarkDeletedInternal(tableint internalId) {
-            assert(internalId < max_elements_);
+            assert(internalId < cur_element_count);
             if (isMarkedDeleted(internalId))
             {
                 unsigned char *ll_cur = ((unsigned char *)get_linklist0(internalId))+2;

python_bindings/bindings.cpp

@@ -70,8 +70,6 @@ inline void ParallelFor(size_t start, size_t end, size_t numThreads, Function fn
             std::rethrow_exception(lastException);
         }
     }
-
-
 }
 
 
@@ -108,7 +106,7 @@ class Index {
     default_ef=10;
   }
 
-  static const int ser_version = 1; // serialization version
+  static const int ser_version = 2; // serialization version
 
   std::string space_name;
   int dim;
@@ -477,7 +475,6 @@ class Index {
       assert_true(appr_alg->ef_construction_ == d["ef_construction"].cast<size_t>(), "Invalid value of ef_construction_ ");
 
       appr_alg->ef_ = d["ef"].cast<size_t>();
-      appr_alg->num_deleted_=d["num_deleted"].cast<size_t>();
 
       assert_true(appr_alg->size_links_per_element_ == d["size_links_per_element"].cast<size_t>(), "Invalid value of size_links_per_element_ ");
 
@@ -523,6 +520,26 @@ class Index {
 
           }
       }
+
+      // set num_deleted
+      if (d.contains("has_deletions"))
+      {
+        // create index from outdated pickle that doesn't contain num_deleted
+        appr_alg->num_deleted_ = 0;
+        std::cout << "Warning: creating index from outdated pickle" << std::endl;
+        bool has_deletions = d["has_deletions"].cast<bool>();
+        if (has_deletions)
+        {
+          for (size_t i = 0; i < appr_alg->cur_element_count; i++) {
+            if(appr_alg->isMarkedDeleted(i))
+              appr_alg->num_deleted_ += 1;
+            }
+        }
+      }
+      else
+      {
+        appr_alg->num_deleted_ = d["num_deleted"].cast<size_t>();
+      }
 }
 
     py::object knnQuery_return_numpy(py::object input, size_t k = 1, int num_threads = -1) {