#1997 index file missed after compact (#2108)

* #1997 index file missed after compact
Signed-off-by: Ngroot <yihua.mo@zilliz.com>

* changelog
Signed-off-by: Ngroot <yihua.mo@zilliz.com>

* typo
Signed-off-by: Ngroot <yihua.mo@zilliz.com>

CHANGELOG.md

@@ -7,8 +7,8 @@ Please mark all change in change log and use the issue from GitHub
 ## Bug
 -   \#1705 Limit the insert data batch size
 -   \#1929 Skip MySQL meta schema field width check
+-   \#1997 Index file missed after compact
 -   \#2073 Fix CheckDBConfigBackendUrl error message
-
 -   \#2076 CheckMetricConfigAddress error message
 
 ## Feature

core/src/db/DBImpl.cpp

@@ -986,18 +986,16 @@ DBImpl::CompactFile(const std::string& collection_id, const meta::SegmentSchema&
         return status;
     }
 
-    // Update collection files state
-    // if index type isn't IDMAP, set file type to TO_INDEX if file size exceed index_file_size
-    // else set file type to RAW, no need to build index
-    if (!utils::IsRawIndexType(compacted_file.engine_type_)) {
-        compacted_file.file_type_ = (segment_writer_ptr->Size() >= compacted_file.index_file_size_)
-                                        ? meta::SegmentSchema::TO_INDEX
-                                        : meta::SegmentSchema::RAW;
+    // Update compacted file state, if origin file is backup or to_index, set compected file to to_index
+    compacted_file.file_size_ = segment_writer_ptr->Size();
+    compacted_file.row_count_ = segment_writer_ptr->VectorCount();
+    if ((file.file_type_ == (int32_t)meta::SegmentSchema::BACKUP ||
+         file.file_type_ == (int32_t)meta::SegmentSchema::TO_INDEX) &&
+        (compacted_file.row_count_ > meta::BUILD_INDEX_THRESHOLD)) {
+        compacted_file.file_type_ = meta::SegmentSchema::TO_INDEX;
     } else {
         compacted_file.file_type_ = meta::SegmentSchema::RAW;
     }
-    compacted_file.file_size_ = segment_writer_ptr->Size();
-    compacted_file.row_count_ = segment_writer_ptr->VectorCount();
 
     if (compacted_file.row_count_ == 0) {
         LOG_ENGINE_DEBUG_ << "Compacted segment is empty. Mark it as TO_DELETE";

core/src/server/delivery/request/SearchByIDRequest.cpp

@@ -104,23 +104,7 @@ SearchByIDRequest::OnExecute() {
             return status;
         }
 
-        // step 6: check whether GPU search resource is enabled
-#ifdef MILVUS_GPU_VERSION
-        Config& config = Config::GetInstance();
-        bool gpu_enable;
-        config.GetGpuResourceConfigEnable(gpu_enable);
-        if (gpu_enable) {
-            std::vector<int64_t> search_resources;
-            config.GetGpuResourceConfigSearchResources(search_resources);
-            if (!search_resources.empty()) {
-                std::string err_msg = "SearchByID cannot be executed on GPU";
-                LOG_SERVER_ERROR_ << err_msg;
-                return Status(SERVER_UNSUPPORTED_ERROR, err_msg);
-            }
-        }
-#endif
-
-        // step 7: check collection's index type supports search by id
+        // step 6: check collection's index type supports search by id
         if (collection_schema.engine_type_ != (int32_t)engine::EngineType::FAISS_IDMAP &&
             collection_schema.engine_type_ != (int32_t)engine::EngineType::FAISS_BIN_IDMAP &&
             collection_schema.engine_type_ != (int32_t)engine::EngineType::FAISS_IVFFLAT &&
@@ -134,7 +118,7 @@ SearchByIDRequest::OnExecute() {
 
         rc.RecordSection("check validation");
 
-        // step 8: search vectors
+        // step 7: search vectors
         engine::ResultIds result_ids;
         engine::ResultDistances result_distances;
 
@@ -161,7 +145,7 @@ SearchByIDRequest::OnExecute() {
             return Status::OK();  // empty collection
         }
 
-        // step 9: construct result array
+        // step 8: construct result array
         milvus::server::ContextChild tracer(context_, "Constructing result");
         result_.row_num_ = id_array_.size();
         result_.distance_list_.swap(result_distances);

core/unittest/db/test_search_by_id.cpp

@@ -37,7 +37,7 @@ std::string
 GetCollectionName() {
     auto now = std::chrono::system_clock::now();
     auto micros = std::chrono::duration_cast<std::chrono::microseconds>(now.time_since_epoch()).count();
-    static std::string collection_name = std::to_string(micros);
+    std::string collection_name = std::to_string(micros);
     return collection_name;
 }
 
@@ -460,9 +460,7 @@ TEST_F(GetVectorByIdTest, WITH_DELETE_TEST) {
 }
 
 TEST_F(SearchByIdTest, BINARY_TEST) {
-    milvus::engine::meta::CollectionSchema collection_info;
-    collection_info.dimension_ = COLLECTION_DIM;
-    collection_info.collection_id_ = GetCollectionName();
+    milvus::engine::meta::CollectionSchema collection_info = BuildCollectionSchema();
     collection_info.engine_type_ = (int)milvus::engine::EngineType::FAISS_BIN_IDMAP;
     collection_info.metric_type_ = (int32_t)milvus::engine::MetricType::JACCARD;
     auto stat = db_->CreateCollection(collection_info);
@@ -474,7 +472,7 @@ TEST_F(SearchByIdTest, BINARY_TEST) {
     ASSERT_TRUE(stat.ok());
     ASSERT_EQ(collection_info_get.dimension_, COLLECTION_DIM);
 
-    int insert_loop = 10;
+    int insert_loop = 100;
     int64_t nb = 1000;
 
     for (int k = 0; k < insert_loop; ++k) {

sdk/examples/simple/src/ClientTest.cpp

@@ -298,10 +298,10 @@ ClientTest::Test() {
     DescribeCollection(collection_name);
 
     InsertEntities(collection_name, dim);
-    BuildSearchEntities(NQ, dim);
     Flush(collection_name);
     ShowCollectionInfo(collection_name);
 
+    BuildSearchEntities(NQ, dim);
     GetEntitiesByID(collection_name, search_id_array_);
 //    SearchEntities(collection_name, TOP_K, NPROBE);
     SearchEntitiesByID(collection_name, TOP_K, NPROBE);
@@ -309,11 +309,11 @@ ClientTest::Test() {
     CreateIndex(collection_name, INDEX_TYPE, NLIST);
     ShowCollectionInfo(collection_name);
 
-    PreloadCollection(collection_name);
-
     std::vector<int64_t> delete_ids = {search_id_array_[0], search_id_array_[1]};
     DeleteByIds(collection_name, delete_ids);
     CompactCollection(collection_name);
+
+    PreloadCollection(collection_name);
     SearchEntities(collection_name, TOP_K, NPROBE); // this line get two search error since we delete two entities
 
     DropIndex(collection_name);