separate rhnsw quantization data from index data and fix read compres… (#3485)

* separate rhnsw quantization data from index data and fix read compress data bug in passing
Signed-off-by: Ncmli <chengming.li@zilliz.com>

* fix unittest
Signed-off-by: Ncmli <chengming.li@zilliz.com>
Co-authored-by: Ncmli <chengming.li@zilliz.com>

core/src/codecs/VectorCompressFormat.cpp

@@ -55,6 +55,7 @@ VectorCompressFormat::Read(const storage::FSHandlerPtr& fs_ptr, const std::strin
         return Status(SERVER_UNEXPECTED_ERROR, "Invalid vector compress length: " + full_file_path);
     }
 
+    compress = std::make_shared<knowhere::Binary>();
     compress->data = std::shared_ptr<uint8_t[]>(new uint8_t[length]);
     compress->size = length;
 

core/src/db/Utils.cpp

@@ -180,7 +180,7 @@ RequireRawFile(const std::string& index_type) {
 
 bool
 RequireCompressFile(const std::string& index_type) {
-    return index_type == knowhere::IndexEnum::INDEX_RHNSWSQ;
+    return index_type == knowhere::IndexEnum::INDEX_RHNSWSQ || index_type == knowhere::IndexEnum::INDEX_RHNSWPQ;
 }
 
 }  // namespace utils

core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSWPQ.cpp

@@ -39,7 +39,7 @@ IndexRHNSWPQ::Serialize(const Config& config) {
     try {
         auto res_set = IndexRHNSW::Serialize(config);
         MemoryIOWriter writer;
-        writer.name = this->index_type() + "_Data";
+        writer.name = QUANTIZATION_DATA;
         auto real_idx = dynamic_cast<faiss::IndexRHNSWPQ*>(index_.get());
         if (real_idx == nullptr) {
             KNOWHERE_THROW_MSG("dynamic_cast<faiss::IndexRHNSWPQ*>(index_) failed during Serialize!");
@@ -59,7 +59,7 @@ IndexRHNSWPQ::Load(const BinarySet& index_binary) {
     try {
         IndexRHNSW::Load(index_binary);
         MemoryIOReader reader;
-        reader.name = this->index_type() + "_Data";
+        reader.name = QUANTIZATION_DATA;
         auto binary = index_binary.GetByName(reader.name);
 
         reader.total = static_cast<size_t>(binary->size);

core/src/index/knowhere/knowhere/index/vector_index/IndexRHNSWSQ.cpp

@@ -42,7 +42,7 @@ IndexRHNSWSQ::Serialize(const Config& config) {
     try {
         auto res_set = IndexRHNSW::Serialize(config);
         MemoryIOWriter writer;
-        writer.name = this->index_type() + "_Data";
+        writer.name = QUANTIZATION_DATA;
         auto real_idx = dynamic_cast<faiss::IndexRHNSWSQ*>(index_.get());
         if (real_idx == nullptr) {
             KNOWHERE_THROW_MSG("dynamic_cast<faiss::IndexRHNSWSQ*>(index_) failed during Serialize!");
@@ -62,7 +62,7 @@ IndexRHNSWSQ::Load(const BinarySet& index_binary) {
     try {
         IndexRHNSW::Load(index_binary);
         MemoryIOReader reader;
-        reader.name = this->index_type() + "_Data";
+        reader.name = QUANTIZATION_DATA;
         auto binary = index_binary.GetByName(reader.name);
 
         reader.total = static_cast<size_t>(binary->size);

core/src/index/unittest/test_rhnsw_pq.cpp

@@ -123,7 +123,7 @@ TEST_P(RHNSWPQTest, HNSW_serialize) {
         index_->Add(base_dataset, conf);
         auto binaryset = index_->Serialize(conf);
         auto bin_idx = binaryset.GetByName(index_->index_type() + "_Index");
-        auto bin_dat = binaryset.GetByName(index_->index_type() + "_Data");
+        auto bin_dat = binaryset.GetByName(QUANTIZATION_DATA);
 
         std::string filename_idx = "/tmp/RHNSWPQ_test_serialize_idx.bin";
         std::string filename_dat = "/tmp/RHNSWPQ_test_serialize_dat.bin";
@@ -137,7 +137,7 @@ TEST_P(RHNSWPQTest, HNSW_serialize) {
         std::shared_ptr<uint8_t[]> dat(load_dat);
         std::shared_ptr<uint8_t[]> idx(load_idx);
         binaryset.Append(new_idx->index_type() + "_Index", idx, bin_idx->size);
-        binaryset.Append(new_idx->index_type() + "_Data", dat, bin_dat->size);
+        binaryset.Append(QUANTIZATION_DATA, dat, bin_dat->size);
 
         new_idx->Load(binaryset);
         EXPECT_EQ(new_idx->Count(), nb);

core/src/index/unittest/test_rhnsw_sq8.cpp

@@ -124,7 +124,7 @@ TEST_P(RHNSWSQ8Test, HNSW_serialize) {
         index_->Add(base_dataset, conf);
         auto binaryset = index_->Serialize(conf);
         auto bin_idx = binaryset.GetByName(index_->index_type() + "_Index");
-        auto bin_dat = binaryset.GetByName(index_->index_type() + "_Data");
+        auto bin_dat = binaryset.GetByName(QUANTIZATION_DATA);
 
         std::string filename_idx = "/tmp/RHNSWSQ_test_serialize_idx.bin";
         std::string filename_dat = "/tmp/RHNSWSQ_test_serialize_dat.bin";
@@ -138,7 +138,7 @@ TEST_P(RHNSWSQ8Test, HNSW_serialize) {
         std::shared_ptr<uint8_t[]> dat(load_dat);
         std::shared_ptr<uint8_t[]> idx(load_idx);
         binaryset.Append(new_idx->index_type() + "_Index", idx, bin_idx->size);
-        binaryset.Append(new_idx->index_type() + "_Data", dat, bin_dat->size);
+        binaryset.Append(QUANTIZATION_DATA, dat, bin_dat->size);
 
         new_idx->Load(binaryset);
         EXPECT_EQ(new_idx->Count(), nb);