Merge pull request #711 from milvus-io/0.6.0

Merge from 0.6.0

CHANGELOG.md

@@ -52,6 +52,9 @@ Please mark all change in change log and use the ticket from JIRA.
 - \#658 - Milvus error out when building SQ8H index without GPU resources
 - \#668 - Update badge of README
 - \#670 - Random failure of unittest db_test::SEARCH_TEST
+- \#674 - Server down in stability test
+- \#696 - Metric_type changed from IP to L2
+- \#705 - Fix search SQ8H crash without GPU resource
 
 ## Feature
 - \#12 - Pure CPU version for Milvus

core/src/db/engine/ExecutionEngineImpl.cpp

@@ -400,10 +400,11 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) {
 Status
 ExecutionEngineImpl::CopyToIndexFileToGpu(uint64_t device_id) {
 #ifdef MILVUS_GPU_VERSION
+    // the ToIndexData is only a placeholder, cpu-copy-to-gpu action is performed in
     gpu_num_ = device_id;
     auto to_index_data = std::make_shared<ToIndexData>(PhysicalSize());
     cache::DataObjPtr obj = std::static_pointer_cast<cache::DataObj>(to_index_data);
-    milvus::cache::GpuCacheMgr::GetInstance(device_id)->InsertItem(location_, obj);
+    milvus::cache::GpuCacheMgr::GetInstance(device_id)->InsertItem(location_ + "_placeholder", obj);
 #endif
     return Status::OK();
 }
@@ -603,7 +604,7 @@ ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, int64_t npr
     }
 
     if (!status.ok()) {
-        ENGINE_LOG_ERROR << "Search error";
+        ENGINE_LOG_ERROR << "Search error:" << status.message();
     }
     return status;
 }
@@ -634,6 +635,7 @@ ExecutionEngineImpl::Init() {
     Status s = config.GetGpuResourceConfigBuildIndexResources(gpu_ids);
     if (!s.ok()) {
         gpu_num_ = knowhere::INVALID_VALUE;
+        return s;
     }
     for (auto id : gpu_ids) {
         if (gpu_num_ == id) {

core/src/db/meta/MySQLMetaImpl.cpp

@@ -1138,8 +1138,6 @@ MySQLMetaImpl::DropTableIndex(const std::string& table_id) {
             // set table index type to raw
             dropTableIndexQuery << "UPDATE " << META_TABLES
                                 << " SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE)
-                                << " ,nlist = " << std::to_string(DEFAULT_NLIST)
-                                << " ,metric_type = " << std::to_string(DEFAULT_METRIC_TYPE)
                                 << " WHERE table_id = " << mysqlpp::quote << table_id << ";";
 
             ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str();

core/src/db/meta/SqliteMetaImpl.cpp

@@ -733,9 +733,7 @@ SqliteMetaImpl::DropTableIndex(const std::string& table_id) {
         //set table index type to raw
         ConnectorPtr->update_all(
             set(
-                c(&TableSchema::engine_type_) = DEFAULT_ENGINE_TYPE,
-                c(&TableSchema::nlist_) = DEFAULT_NLIST,
-                c(&TableSchema::metric_type_) = DEFAULT_METRIC_TYPE),
+                c(&TableSchema::engine_type_) = DEFAULT_ENGINE_TYPE),
             where(
                 c(&TableSchema::table_id_) == table_id));
 

core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp

@@ -295,61 +295,5 @@ FaissIVFQuantizer::~FaissIVFQuantizer() {
     // else do nothing
 }
 
-#else
-
-QuantizerPtr
-IVFSQHybrid::LoadQuantizer(const Config& conf) {
-    return knowhere::QuantizerPtr();
-}
-
-void
-IVFSQHybrid::SetQuantizer(const QuantizerPtr& q) {
-}
-
-void
-IVFSQHybrid::UnsetQuantizer() {
-}
-
-VectorIndexPtr
-IVFSQHybrid::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) {
-    return nullptr;
-}
-
-std::pair<VectorIndexPtr, QuantizerPtr>
-IVFSQHybrid::CopyCpuToGpuWithQuantizer(const int64_t& device_id, const Config& config) {
-    KNOWHERE_THROW_MSG("Not yet implemented");
-}
-
-IndexModelPtr
-IVFSQHybrid::Train(const DatasetPtr& dataset, const Config& config) {
-    return GPUIVFSQ::Train(dataset, config);
-}
-
-VectorIndexPtr
-IVFSQHybrid::CopyGpuToCpu(const Config& config) {
-    return GPUIVFSQ::CopyGpuToCpu(config);
-}
-
-VectorIndexPtr
-IVFSQHybrid::CopyCpuToGpu(const int64_t& device_id, const Config& config) {
-    return IVF::CopyCpuToGpu(device_id, config);
-}
-
-void
-IVFSQHybrid::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels,
-                         const Config& cfg) {
-    GPUIVF::search_impl(n, data, k, distances, labels, cfg);
-}
-
-void
-IVFSQHybrid::LoadImpl(const BinarySet& index_binary) {
-    GPUIVF::LoadImpl(index_binary);
-}
-
-void
-IVFSQHybrid::set_index_model(IndexModelPtr model) {
-    GPUIVF::set_index_model(model);
-}
-
 #endif
 }  // namespace knowhere

core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h

@@ -36,7 +36,6 @@ struct FaissIVFQuantizer : public Quantizer {
     ~FaissIVFQuantizer() override;
 };
 using FaissIVFQuantizerPtr = std::shared_ptr<FaissIVFQuantizer>;
-#endif
 
 class IVFSQHybrid : public GPUIVFSQ {
  public:
@@ -93,5 +92,6 @@ class IVFSQHybrid : public GPUIVFSQ {
     int64_t gpu_mode = 0;  // 0,1,2
     int64_t quantizer_gpu_id_ = -1;
 };
+#endif
 
 }  // namespace knowhere

core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp

@@ -38,9 +38,11 @@ CopyGpuToCpu(const VectorIndexPtr& index, const Config& config) {
 
 VectorIndexPtr
 CopyCpuToGpu(const VectorIndexPtr& index, const int64_t& device_id, const Config& config) {
+#ifdef CUSTOMIZATION
     if (auto device_index = std::dynamic_pointer_cast<IVFSQHybrid>(index)) {
         return device_index->CopyCpuToGpu(device_id, config);
     }
+#endif
 
     if (auto device_index = std::dynamic_pointer_cast<GPUIndex>(index)) {
         return device_index->CopyGpuToGpu(device_id, config);

core/src/index/unittest/Helper.h

@@ -53,8 +53,10 @@ IndexFactory(const std::string& type) {
         return std::make_shared<knowhere::GPUIVFPQ>(DEVICEID);
     } else if (type == "GPUIVFSQ") {
         return std::make_shared<knowhere::GPUIVFSQ>(DEVICEID);
+#ifdef CUSTOMIZATION
     } else if (type == "IVFSQHybrid") {
         return std::make_shared<knowhere::IVFSQHybrid>(DEVICEID);
+#endif
 #endif
     }
 }

core/src/scheduler/task/BuildIndexTask.cpp

@@ -55,7 +55,7 @@ XBuildIndexTask::Load(milvus::scheduler::LoadType type, uint8_t device_id) {
                 type_str = "DISK2CPU";
             } else if (type == LoadType::CPU2GPU) {
                 stat = to_index_engine_->CopyToIndexFileToGpu(device_id);
-                type_str = "CPU2GPU";
+                type_str = "CPU2GPU:" + std::to_string(device_id);
             } else {
                 error_msg = "Wrong load type";
                 stat = Status(SERVER_UNEXPECTED_ERROR, error_msg);
@@ -86,7 +86,7 @@ XBuildIndexTask::Load(milvus::scheduler::LoadType type, uint8_t device_id) {
 
         size_t file_size = to_index_engine_->PhysicalSize();
 
-        std::string info = "Load file id:" + std::to_string(file_->id_) + " " + type_str +
+        std::string info = "Build index task load file id:" + std::to_string(file_->id_) + " " + type_str +
                            " file type:" + std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) +
                            " bytes from location: " + file_->location_ + " totally cost";
         double span = rc.ElapseFromBegin(info);
@@ -128,6 +128,7 @@ XBuildIndexTask::Execute() {
 
         // step 3: build index
         try {
+            ENGINE_LOG_DEBUG << "Begin build index for file:" + table_file.location_;
             index = to_index_engine_->BuildIndex(table_file.location_, (EngineType)table_file.engine_type_);
             if (index == nullptr) {
                 throw Exception(DB_ERROR, "index NULL");

core/src/scheduler/task/SearchTask.cpp

@@ -125,7 +125,7 @@ XSearchTask::Load(LoadType type, uint8_t device_id) {
                 hybrid = true;
             }
             stat = index_engine_->CopyToGpu(device_id, hybrid);
-            type_str = "CPU2GPU";
+            type_str = "CPU2GPU:" + std::to_string(device_id);
         } else if (type == LoadType::GPU2CPU) {
             stat = index_engine_->CopyToCpu();
             type_str = "GPU2CPU";
@@ -160,7 +160,7 @@ XSearchTask::Load(LoadType type, uint8_t device_id) {
 
     size_t file_size = index_engine_->PhysicalSize();
 
-    std::string info = "Load file id:" + std::to_string(file_->id_) +
+    std::string info = "Search task load file id:" + std::to_string(file_->id_) + " " + type_str +
                        " file type:" + std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) +
                        " bytes from location: " + file_->location_ + " totally cost";
     double span = rc.ElapseFromBegin(info);

core/src/wrapper/VecImpl.h

@@ -108,7 +108,7 @@ class ToIndexData : public cache::DataObj {
     }
 
  private:
-    int64_t size_;
+    int64_t size_ = 0;
 };
 
 }  // namespace engine

core/src/wrapper/VecIndex.cpp

@@ -24,6 +24,8 @@
 #include "knowhere/index/vector_index/IndexIVFSQ.h"
 #include "knowhere/index/vector_index/IndexNSG.h"
 #include "knowhere/index/vector_index/IndexSPTAG.h"
+#include "server/Config.h"
+#include "utils/Exception.h"
 #include "utils/Log.h"
 
 #ifdef MILVUS_GPU_VERSION
@@ -145,6 +147,10 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) {
             index = std::make_shared<knowhere::GPUIVF>(gpu_device);
             break;
         }
+        case IndexType::FAISS_IVFFLAT_MIX: {
+            index = std::make_shared<knowhere::GPUIVF>(gpu_device);
+            return std::make_shared<IVFMixIndex>(index, IndexType::FAISS_IVFFLAT_MIX);
+        }
         case IndexType::FAISS_IVFPQ_GPU: {
             index = std::make_shared<knowhere::GPUIVFPQ>(gpu_device);
             break;
@@ -161,16 +167,20 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) {
             index = std::make_shared<knowhere::GPUIVFSQ>(gpu_device);
             break;
         }
-        case IndexType::FAISS_IVFFLAT_MIX: {
-            index = std::make_shared<knowhere::GPUIVF>(gpu_device);
-            return std::make_shared<IVFMixIndex>(index, IndexType::FAISS_IVFFLAT_MIX);
-        }
+
+#endif
 #ifdef CUSTOMIZATION
         case IndexType::FAISS_IVFSQ8_HYBRID: {
-            index = std::make_shared<knowhere::IVFSQHybrid>(gpu_device);
-            return std::make_shared<IVFHybridIndex>(index, IndexType::FAISS_IVFSQ8_HYBRID);
+            server::Config& config = server::Config::GetInstance();
+            bool gpu_resource_enable = true;
+            config.GetGpuResourceConfigEnable(gpu_resource_enable);
+            if (gpu_resource_enable) {
+                index = std::make_shared<knowhere::IVFSQHybrid>(gpu_device);
+                return std::make_shared<IVFHybridIndex>(index, IndexType::FAISS_IVFSQ8_HYBRID);
+            } else {
+                throw Exception(DB_ERROR, "No GPU resources for IndexType::FAISS_IVFSQ8_HYBRID");
+            }
         }
-#endif
 #endif
         case IndexType::NSG_MIX: {  // TODO(linxj): bug.
             index = std::make_shared<knowhere::NSG>(gpu_device);

core/src/wrapper/gpu/GPUVecImpl.cpp

@@ -73,6 +73,7 @@ IVFMixIndex::Load(const knowhere::BinarySet& index_binary) {
     return Status::OK();
 }
 
+#ifdef CUSTOMIZATION
 knowhere::QuantizerPtr
 IVFHybridIndex::LoadQuantizer(const Config& conf) {
     // TODO(linxj): Hardcode here
@@ -158,6 +159,7 @@ IVFHybridIndex::CopyToGpuWithQuantizer(const int64_t& device_id, const Config& c
     }
     return std::make_pair(nullptr, nullptr);
 }
+#endif
 
 }  // namespace engine
 }  // namespace milvus

core/unittest/db/test_db.cpp

@@ -351,6 +351,19 @@ TEST_F(DBTest, SEARCH_TEST) {
         ASSERT_TRUE(stat.ok());
     }
 
+    index.engine_type_ = (int)milvus::engine::EngineType::FAISS_PQ;
+    db_->CreateIndex(TABLE_NAME, index);  // wait until build index finish
+
+    {
+        std::vector<std::string> tags;
+        milvus::engine::ResultIds result_ids;
+        milvus::engine::ResultDistances result_distances;
+        stat = db_->Query(TABLE_NAME, tags, k, nq, 10, xq.data(), result_ids, result_distances);
+        ASSERT_TRUE(stat.ok());
+        stat = db_->Query(TABLE_NAME, tags, k, 1100, 10, xq.data(), result_ids, result_distances);
+        ASSERT_TRUE(stat.ok());
+    }
+
 #ifdef CUSTOMIZATION
     // test FAISS_IVFSQ8H optimizer
     index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8H;
@@ -375,7 +388,13 @@ TEST_F(DBTest, SEARCH_TEST) {
 
     {  // search by specify index file
         milvus::engine::meta::DatesT dates;
-        std::vector<std::string> file_ids = {"1", "2", "3", "4", "5", "6"};
+        std::vector<std::string> file_ids;
+        // sometimes this case run fast to merge file and build index, old file will be deleted immediately,
+        // so the QueryByFileID cannot get files to search
+        // input 100 files ids to avoid random failure of this case
+        for (int i = 0; i < 100; i++) {
+            file_ids.push_back(std::to_string(i));
+        }
         result_ids.clear();
         result_dists.clear();
         stat = db_->QueryByFileID(TABLE_NAME, file_ids, k, nq, 10, xq.data(), dates, result_ids, result_dists);

install.md

@@ -103,7 +103,7 @@ $ docker run -it -p 19530:19530 -d milvusdb/milvus-cpu-build-env:v0.6.0-ubuntu18
 Start a GPU container:
 
 ```shell
-$ docker run — runtime=nvidia -it -p 19530:19530 -d milvusdb/milvus-gpu-build-env:v0.6.0-ubuntu18.04
+$ docker run --runtime=nvidia -it -p 19530:19530 -d milvusdb/milvus-gpu-build-env:v0.6.0-ubuntu18.04
 ```
 To enter the container: