diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 752187675a4e16f58eb55553b9383af2190203dc..4495900266d1d56ba90e3a4fa43608a49ece86aa 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -35,6 +35,7 @@ #include "db/merge/MergeManagerFactory.h" #include "engine/EngineFactory.h" #include "index/knowhere/knowhere/index/vector_index/helpers/BuilderSuspend.h" +#include "index/knowhere/knowhere/index/vector_index/helpers/FaissIO.h" #include "index/thirdparty/faiss/utils/distances.h" #include "insert/MemManagerFactory.h" #include "meta/MetaConsts.h" @@ -95,6 +96,7 @@ DBImpl::DBImpl(const DBOptions& options) SetIdentity("DBImpl"); AddCacheInsertDataListener(); AddUseBlasThresholdListener(); + knowhere::enable_faiss_logging(); Start(); } diff --git a/core/src/index/knowhere/knowhere/common/Log.cpp b/core/src/index/knowhere/knowhere/common/Log.cpp index 75578d105bdb4b27063b85359d15d8525b33663c..9f112c55cde8df2ab20e01d1b88e26d31041f192 100644 --- a/core/src/index/knowhere/knowhere/common/Log.cpp +++ b/core/src/index/knowhere/knowhere/common/Log.cpp @@ -51,5 +51,35 @@ GetThreadName() { return thread_name; } +void +log_trace_(const std::string& s) { + LOG_KNOWHERE_TRACE_ << s; +} + +void +log_debug_(const std::string& s) { + LOG_KNOWHERE_DEBUG_ << s; +} + +void +log_info_(const std::string& s) { + LOG_KNOWHERE_INFO_ << s; +} + +void +log_warning_(const std::string& s) { + LOG_KNOWHERE_WARNING_ << s; +} + +void +log_error_(const std::string& s) { + LOG_KNOWHERE_ERROR_ << s; +} + +void +log_fatal_(const std::string& s) { + LOG_KNOWHERE_FATAL_ << s; +} + } // namespace knowhere } // namespace milvus diff --git a/core/src/index/knowhere/knowhere/common/Log.h b/core/src/index/knowhere/knowhere/common/Log.h index 428f06874f62ae1ae4b1589643c4143b3e700321..db1677e3209edb192a5662610657a36af404cf7b 100644 --- a/core/src/index/knowhere/knowhere/common/Log.h +++ b/core/src/index/knowhere/knowhere/common/Log.h @@ -27,6 +27,24 @@ SetThreadName(const std::string& name); std::string GetThreadName(); +void +log_trace_(const std::string&); + +void +log_debug_(const std::string&); + +void +log_info_(const std::string&); + +void +log_warning_(const std::string&); + +void +log_error_(const std::string&); + +void +log_fatal_(const std::string&); + /* * Please use LOG_MODULE_LEVEL_C macro in member function of class * and LOG_MODULE_LEVEL_ macro in other functions. diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp index 357046b53855b072340b6d969db152f737010d7b..b226685010ffc69afa0c0dc8439fbf2eedee1bcb 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -35,6 +35,7 @@ #include "knowhere/common/Log.h" #include "knowhere/index/vector_index/IndexIVF.h" #include "knowhere/index/vector_index/adapter/VectorAdapter.h" +#include "knowhere/index/vector_index/helpers/FaissIO.h" #include "knowhere/index/vector_index/helpers/IndexParameter.h" #ifdef MILVUS_GPU_VERSION #include "knowhere/index/vector_index/gpu/IndexGPUIVF.h" diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp index 645d4f9064cc16abd3677199e5eb32d59fffc263..b64db7cbf998dff1d5fa76492cc28c0d4a6aad47 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp @@ -11,6 +11,7 @@ #include +#include "knowhere/common/Log.h" #include "knowhere/index/vector_index/helpers/FaissIO.h" namespace milvus { @@ -60,5 +61,10 @@ MemoryIOReader::operator()(void* ptr, size_t size, size_t nitems) { return nitems; } +void +enable_faiss_logging() { + faiss::LOG_DEBUG_ = &log_debug_; +} + } // namespace knowhere } // namespace milvus diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissIO.h b/core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissIO.h index 08a98e3d0591133d7ab5ba908569a98fa96be0a6..e777c6746eff98c68a1a50cd7c53cf1eb93fdbae 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissIO.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissIO.h @@ -12,6 +12,7 @@ #pragma once #include +#include namespace milvus { namespace knowhere { @@ -46,5 +47,8 @@ struct MemoryIOReader : public faiss::IOReader { } }; +void +enable_faiss_logging(); + } // namespace knowhere } // namespace milvus diff --git a/core/src/index/thirdparty/faiss/IndexIVF.cpp b/core/src/index/thirdparty/faiss/IndexIVF.cpp index fa6f050ca6eea4aebdb8f380dfcd24a387d986e7..73423743276507289d47e92bb432d901b71934e5 100644 --- a/core/src/index/thirdparty/faiss/IndexIVF.cpp +++ b/core/src/index/thirdparty/faiss/IndexIVF.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -314,6 +315,22 @@ void IndexIVF::search (idx_t n, const float *x, idx_t k, search_preassigned (n, x, k, idx.get(), coarse_dis.get(), distances, labels, false, nullptr, bitset); indexIVF_stats.search_time += getmillisecs() - t0; + + // string + if (LOG_DEBUG_) { + auto ids = idx.get(); + for (size_t i = 0; i < n; i++) { + std::stringstream ss; + ss << "Query #" << i << ", nprobe list: "; + for (size_t j = 0; j < nprobe; j++) { + if (j != 0) { + ss << ","; + } + ss << ids[i * nprobe + j]; + } + (*LOG_DEBUG_)(ss.str()); + } + } } #if 0 diff --git a/core/src/index/thirdparty/faiss/utils/utils.cpp b/core/src/index/thirdparty/faiss/utils/utils.cpp index 78248626d44ed3cf955d746ac6600955ea2c6911..20b2e29553020f44fef14e6e425556a15f538fc4 100644 --- a/core/src/index/thirdparty/faiss/utils/utils.cpp +++ b/core/src/index/thirdparty/faiss/utils/utils.cpp @@ -708,4 +708,16 @@ int64_t get_L3_Size() { return l3_size; } +void (*LOG_TRACE_)(const std::string&); + +void (*LOG_DEBUG_)(const std::string&); + +void (*LOG_INFO_)(const std::string&); + +void (*LOG_WARNING_)(const std::string&); + +void (*LOG_FATAL_)(const std::string&); + +void (*LOG_ERROR_)(const std::string&); + } // namespace faiss diff --git a/core/src/index/thirdparty/faiss/utils/utils.h b/core/src/index/thirdparty/faiss/utils/utils.h index 3a75570a23f96f3657af1af13f6c8145fb14656e..9be65b10a096e8dc330c27e2853a4a7d906a03bb 100644 --- a/core/src/index/thirdparty/faiss/utils/utils.h +++ b/core/src/index/thirdparty/faiss/utils/utils.h @@ -17,6 +17,7 @@ #define FAISS_utils_h #include +#include #include @@ -163,6 +164,18 @@ bool check_openmp(); /** get the size of L3 cache */ int64_t get_L3_Size(); +extern void (*LOG_TRACE_)(const std::string&); + +extern void (*LOG_DEBUG_)(const std::string&); + +extern void (*LOG_INFO_)(const std::string&); + +extern void (*LOG_WARNING_)(const std::string&); + +extern void (*LOG_FATAL_)(const std::string&); + +extern void (*LOG_ERROR_)(const std::string&); + } // namspace faiss