From 058cdf03bbc1b89d41f4a8874ba53c102f726617 Mon Sep 17 00:00:00 2001 From: Xiaohai Xu Date: Sat, 1 Aug 2020 09:46:53 +0800 Subject: [PATCH] Add ivf index nprobe logging (#3070) * add nprobe logging Signed-off-by: sahuang * add all cases Signed-off-by: sahuang --- core/src/db/DBImpl.cpp | 2 ++ .../index/knowhere/knowhere/common/Log.cpp | 30 +++++++++++++++++++ core/src/index/knowhere/knowhere/common/Log.h | 18 +++++++++++ .../knowhere/index/vector_index/IndexIVF.cpp | 1 + .../index/vector_index/helpers/FaissIO.cpp | 6 ++++ .../index/vector_index/helpers/FaissIO.h | 4 +++ core/src/index/thirdparty/faiss/IndexIVF.cpp | 17 +++++++++++ .../index/thirdparty/faiss/utils/utils.cpp | 12 ++++++++ core/src/index/thirdparty/faiss/utils/utils.h | 13 ++++++++ 9 files changed, 103 insertions(+) diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 75218767..44959002 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -35,6 +35,7 @@ #include "db/merge/MergeManagerFactory.h" #include "engine/EngineFactory.h" #include "index/knowhere/knowhere/index/vector_index/helpers/BuilderSuspend.h" +#include "index/knowhere/knowhere/index/vector_index/helpers/FaissIO.h" #include "index/thirdparty/faiss/utils/distances.h" #include "insert/MemManagerFactory.h" #include "meta/MetaConsts.h" @@ -95,6 +96,7 @@ DBImpl::DBImpl(const DBOptions& options) SetIdentity("DBImpl"); AddCacheInsertDataListener(); AddUseBlasThresholdListener(); + knowhere::enable_faiss_logging(); Start(); } diff --git a/core/src/index/knowhere/knowhere/common/Log.cpp b/core/src/index/knowhere/knowhere/common/Log.cpp index 75578d10..9f112c55 100644 --- a/core/src/index/knowhere/knowhere/common/Log.cpp +++ b/core/src/index/knowhere/knowhere/common/Log.cpp @@ -51,5 +51,35 @@ GetThreadName() { return thread_name; } +void +log_trace_(const std::string& s) { + LOG_KNOWHERE_TRACE_ << s; +} + +void +log_debug_(const std::string& s) { + LOG_KNOWHERE_DEBUG_ << s; +} + +void +log_info_(const std::string& s) { + LOG_KNOWHERE_INFO_ << s; +} + +void +log_warning_(const std::string& s) { + LOG_KNOWHERE_WARNING_ << s; +} + +void +log_error_(const std::string& s) { + LOG_KNOWHERE_ERROR_ << s; +} + +void +log_fatal_(const std::string& s) { + LOG_KNOWHERE_FATAL_ << s; +} + } // namespace knowhere } // namespace milvus diff --git a/core/src/index/knowhere/knowhere/common/Log.h b/core/src/index/knowhere/knowhere/common/Log.h index 428f0687..db1677e3 100644 --- a/core/src/index/knowhere/knowhere/common/Log.h +++ b/core/src/index/knowhere/knowhere/common/Log.h @@ -27,6 +27,24 @@ SetThreadName(const std::string& name); std::string GetThreadName(); +void +log_trace_(const std::string&); + +void +log_debug_(const std::string&); + +void +log_info_(const std::string&); + +void +log_warning_(const std::string&); + +void +log_error_(const std::string&); + +void +log_fatal_(const std::string&); + /* * Please use LOG_MODULE_LEVEL_C macro in member function of class * and LOG_MODULE_LEVEL_ macro in other functions. diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp index 357046b5..b2266850 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -35,6 +35,7 @@ #include "knowhere/common/Log.h" #include "knowhere/index/vector_index/IndexIVF.h" #include "knowhere/index/vector_index/adapter/VectorAdapter.h" +#include "knowhere/index/vector_index/helpers/FaissIO.h" #include "knowhere/index/vector_index/helpers/IndexParameter.h" #ifdef MILVUS_GPU_VERSION #include "knowhere/index/vector_index/gpu/IndexGPUIVF.h" diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp index 645d4f90..b64db7cb 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp @@ -11,6 +11,7 @@ #include +#include "knowhere/common/Log.h" #include "knowhere/index/vector_index/helpers/FaissIO.h" namespace milvus { @@ -60,5 +61,10 @@ MemoryIOReader::operator()(void* ptr, size_t size, size_t nitems) { return nitems; } +void +enable_faiss_logging() { + faiss::LOG_DEBUG_ = &log_debug_; +} + } // namespace knowhere } // namespace milvus diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissIO.h b/core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissIO.h index 08a98e3d..e777c674 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissIO.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissIO.h @@ -12,6 +12,7 @@ #pragma once #include +#include namespace milvus { namespace knowhere { @@ -46,5 +47,8 @@ struct MemoryIOReader : public faiss::IOReader { } }; +void +enable_faiss_logging(); + } // namespace knowhere } // namespace milvus diff --git a/core/src/index/thirdparty/faiss/IndexIVF.cpp b/core/src/index/thirdparty/faiss/IndexIVF.cpp index fa6f050c..73423743 100644 --- a/core/src/index/thirdparty/faiss/IndexIVF.cpp +++ b/core/src/index/thirdparty/faiss/IndexIVF.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -314,6 +315,22 @@ void IndexIVF::search (idx_t n, const float *x, idx_t k, search_preassigned (n, x, k, idx.get(), coarse_dis.get(), distances, labels, false, nullptr, bitset); indexIVF_stats.search_time += getmillisecs() - t0; + + // string + if (LOG_DEBUG_) { + auto ids = idx.get(); + for (size_t i = 0; i < n; i++) { + std::stringstream ss; + ss << "Query #" << i << ", nprobe list: "; + for (size_t j = 0; j < nprobe; j++) { + if (j != 0) { + ss << ","; + } + ss << ids[i * nprobe + j]; + } + (*LOG_DEBUG_)(ss.str()); + } + } } #if 0 diff --git a/core/src/index/thirdparty/faiss/utils/utils.cpp b/core/src/index/thirdparty/faiss/utils/utils.cpp index 78248626..20b2e295 100644 --- a/core/src/index/thirdparty/faiss/utils/utils.cpp +++ b/core/src/index/thirdparty/faiss/utils/utils.cpp @@ -708,4 +708,16 @@ int64_t get_L3_Size() { return l3_size; } +void (*LOG_TRACE_)(const std::string&); + +void (*LOG_DEBUG_)(const std::string&); + +void (*LOG_INFO_)(const std::string&); + +void (*LOG_WARNING_)(const std::string&); + +void (*LOG_FATAL_)(const std::string&); + +void (*LOG_ERROR_)(const std::string&); + } // namespace faiss diff --git a/core/src/index/thirdparty/faiss/utils/utils.h b/core/src/index/thirdparty/faiss/utils/utils.h index 3a75570a..9be65b10 100644 --- a/core/src/index/thirdparty/faiss/utils/utils.h +++ b/core/src/index/thirdparty/faiss/utils/utils.h @@ -17,6 +17,7 @@ #define FAISS_utils_h #include +#include #include @@ -163,6 +164,18 @@ bool check_openmp(); /** get the size of L3 cache */ int64_t get_L3_Size(); +extern void (*LOG_TRACE_)(const std::string&); + +extern void (*LOG_DEBUG_)(const std::string&); + +extern void (*LOG_INFO_)(const std::string&); + +extern void (*LOG_WARNING_)(const std::string&); + +extern void (*LOG_FATAL_)(const std::string&); + +extern void (*LOG_ERROR_)(const std::string&); + } // namspace faiss -- GitLab