diff --git a/core/conf/milvus.template b/core/conf/milvus.template index 2ccab5e20052b3bb04dd198fd2befd5bb9b5e41f..c8eedb46c27e270565134442141762ff3df156f2 100644 --- a/core/conf/milvus.template +++ b/core/conf/milvus.template @@ -76,26 +76,10 @@ storage: # | files in advance before implementing data changes. WAL | | | # | ensures the atomicity and durability for Milvus operations.| | | #----------------------+------------------------------------------------------------+------------+-----------------+ -# recovery_error_ignore| Whether to ignore logs with errors that happens during WAL | Boolean | false | -# | recovery. If true, when Milvus restarts for recovery and | | | -# | there are errors in WAL log files, log files with errors | | | -# | are ignored. If false, Milvus does not restart when there | | | -# | are errors in WAL log files. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# buffer_size | Sum total of the read buffer and the write buffer in Bytes.| String | 256MB | -# | buffer_size must be in range [64MB, 4096MB]. | | | -# | If the value you specified is out of range, Milvus | | | -# | automatically uses the boundary value closest to the | | | -# | specified value. It is recommended you set buffer_size to | | | -# | a value greater than the inserted data size of a single | | | -# | insert operation for better performance. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ # path | Location of WAL log files. | String | | #----------------------+------------------------------------------------------------+------------+-----------------+ wal: enable: true - recovery_error_ignore: false - buffer_size: 256MB path: @MILVUS_DB_PATH@/wal #----------------------+------------------------------------------------------------+------------+-----------------+ diff --git a/core/src/codecs/VectorIndexFormat.cpp b/core/src/codecs/VectorIndexFormat.cpp index 04ab2f0303c33b2c04926bb7c288ee751098e913..8b6e541b83a87949180b8e847808a0b81cffeec3 100644 --- a/core/src/codecs/VectorIndexFormat.cpp +++ b/core/src/codecs/VectorIndexFormat.cpp @@ -160,8 +160,8 @@ VectorIndexFormat::ConstructIndex(const std::string& index_name, knowhere::Binar } if (compress_data != nullptr) { - LOG_ENGINE_DEBUG_ << "load index with " << SQ8_DATA << " " << compress_data->size; - index_data.Append(SQ8_DATA, compress_data); + LOG_ENGINE_DEBUG_ << "load index with " << QUANTIZATION_DATA << " " << compress_data->size; + index_data.Append(QUANTIZATION_DATA, compress_data); length += compress_data->size; } @@ -217,7 +217,7 @@ VectorIndexFormat::WriteCompress(const storage::FSHandlerPtr& fs_ptr, const std: auto binaryset = index->Serialize(knowhere::Config()); - auto sq8_data = binaryset.Erase(SQ8_DATA); + auto sq8_data = binaryset.Erase(QUANTIZATION_DATA); if (sq8_data != nullptr) { auto& ss_codec = codec::Codec::instance(); ss_codec.GetVectorCompressFormat()->Write(fs_ptr, file_path, sq8_data); diff --git a/core/src/db/Constants.h b/core/src/db/Constants.h index 5fb4bf5047cab6223809766d47715c34b87c9578..13fe8c6a61b0f814affbdb0851ee4b233e711351 100644 --- a/core/src/db/Constants.h +++ b/core/src/db/Constants.h @@ -21,16 +21,16 @@ constexpr int64_t MB = 1LL << 20; constexpr int64_t GB = 1LL << 30; constexpr int64_t TB = 1LL << 40; -constexpr int64_t MAX_MEM_SEGMENT_SIZE = 128 * MB; +constexpr int64_t MAX_MEM_SEGMENT_SIZE = 128 * MB; // max data size of one segment in insert buffer -constexpr int64_t MAX_NAME_LENGTH = 255; -constexpr int64_t MAX_DIMENSION = 32768; -constexpr int32_t MAX_SEGMENT_ROW_COUNT = 4 * 1024 * 1024; -constexpr int64_t DEFAULT_SEGMENT_ROW_COUNT = 100000; // default row count per segment when creating collection -constexpr int64_t MAX_INSERT_DATA_SIZE = 256 * MB; -constexpr int64_t MAX_WAL_FILE_SIZE = 256 * MB; +constexpr int64_t MAX_NAME_LENGTH = 255; // max string length for collection/partition/field name +constexpr int64_t MAX_DIMENSION = 32768; // max dimension of vector field +constexpr int32_t MAX_SEGMENT_ROW_COUNT = 4 * 1024 * 1024; // max row count of one segment +constexpr int64_t DEFAULT_SEGMENT_ROW_COUNT = 512 * 1024; // default row count per segment when creating collection +constexpr int64_t MAX_INSERT_DATA_SIZE = 256 * MB; // max data size in one insert action +constexpr int64_t MAX_WAL_FILE_SIZE = 256 * MB; // max file size of wal file -constexpr int64_t BUILD_INEDX_RETRY_TIMES = 3; +constexpr int64_t BUILD_INEDX_RETRY_TIMES = 3; // retry times if build index failed } // namespace engine } // namespace milvus diff --git a/core/src/db/SnapshotUtils.cpp b/core/src/db/SnapshotUtils.cpp index 988ffeeaea4ad4afbb22f45955bcabc39e972a4f..e1e1bbd391cafb9dff44c0bf413367a34afe0141 100644 --- a/core/src/db/SnapshotUtils.cpp +++ b/core/src/db/SnapshotUtils.cpp @@ -61,10 +61,10 @@ SetSnapshotIndex(const std::string& collection_name, const std::string& field_na json[engine::PARAM_INDEX_EXTRA_PARAMS] = index_info.extra_params_; index_element->SetParams(json); - if (index_info.index_name_ == knowhere::IndexEnum::INDEX_RHNSWSQ) { + if (utils::RequireCompressFile(index_info.index_type_)) { auto compress_element = std::make_shared(ss->GetCollectionId(), field->GetID(), ELEMENT_INDEX_COMPRESS, - milvus::engine::FieldElementType::FET_COMPRESS_SQ8); + milvus::engine::FieldElementType::FET_COMPRESS); ss_context.new_field_elements.push_back(compress_element); } } @@ -135,7 +135,7 @@ DeleteSnapshotIndex(const std::string& collection_name, const std::string& field std::vector elements = ss->GetFieldElementsByField(name); for (auto& element : elements) { if (element->GetFEtype() == engine::FieldElementType::FET_INDEX || - element->GetFEtype() == engine::FieldElementType::FET_COMPRESS_SQ8) { + element->GetFEtype() == engine::FieldElementType::FET_COMPRESS) { snapshot::OperationContext context; context.stale_field_elements.push_back(element); auto op = std::make_shared(context, ss); diff --git a/core/src/db/Types.h b/core/src/db/Types.h index 5eb132ae6ab8d9f563857671c965de08b26390f2..2dc2a983e380c253a70423a59a3cbd3aebf4fa2c 100644 --- a/core/src/db/Types.h +++ b/core/src/db/Types.h @@ -85,7 +85,7 @@ enum class FieldElementType { FET_BLOOM_FILTER = 2, FET_DELETED_DOCS = 3, FET_INDEX = 4, - FET_COMPRESS_SQ8 = 5, + FET_COMPRESS = 5, }; /////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/core/src/db/Utils.cpp b/core/src/db/Utils.cpp index 04db5f29ce308a9dd591556893d0553169118e98..f90721d23253a4ff4b3b54b454bd2da41bbff78b 100644 --- a/core/src/db/Utils.cpp +++ b/core/src/db/Utils.cpp @@ -25,8 +25,11 @@ #include "db/Types.h" #ifdef MILVUS_GPU_VERSION + #include "cache/GpuCacheMgr.h" + #endif + #include "config/ServerConfig.h" //#include "storage/s3/S3ClientWrapper.h" #include "knowhere/index/vector_index/helpers/IndexParameter.h" @@ -169,6 +172,17 @@ GetSizeOfChunk(const engine::DataChunkPtr& chunk) { return total_size; } +bool +RequireRawFile(const std::string& index_type) { + return index_type == knowhere::IndexEnum::INDEX_FAISS_IVFFLAT || index_type == knowhere::IndexEnum::INDEX_NSG || + index_type == knowhere::IndexEnum::INDEX_HNSW; +} + +bool +RequireCompressFile(const std::string& index_type) { + return index_type == knowhere::IndexEnum::INDEX_RHNSWSQ; +} + } // namespace utils } // namespace engine } // namespace milvus diff --git a/core/src/db/Utils.h b/core/src/db/Utils.h index 395b756309ac4233f60a1c1760a0f627c6b485ee..10735f62d05f7cab183646a730cd64fd72d99106 100644 --- a/core/src/db/Utils.h +++ b/core/src/db/Utils.h @@ -58,6 +58,12 @@ GetIDFromChunk(const engine::DataChunkPtr& chunk, engine::IDNumbers& ids); int64_t GetSizeOfChunk(const engine::DataChunkPtr& chunk); +bool +RequireRawFile(const std::string& index_type); + +bool +RequireCompressFile(const std::string& index_type); + } // namespace utils } // namespace engine } // namespace milvus diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 578bb07e6c30baa2a32b28f382a3b99d976bb752..10b1661172feb10d6820d9764165354cdad4cdb3 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -727,9 +727,8 @@ ExecutionEngineImpl::CreateSnapshotIndexFile(AddSegmentFileOperation& operation, } // create snapshot compress file - std::string index_name = index_element->GetName(); - if (index_name == knowhere::IndexEnum::INDEX_RHNSWSQ) { - auto compress_visitor = field_visitor->GetElementVisitor(engine::FieldElementType::FET_COMPRESS_SQ8); + if (utils::RequireCompressFile(index_info.index_type_)) { + auto compress_visitor = field_visitor->GetElementVisitor(engine::FieldElementType::FET_COMPRESS); if (compress_visitor == nullptr) { return Status(DB_ERROR, "Could not build index: compress element not exist"); // something wrong in CreateIndex diff --git a/core/src/index/knowhere/knowhere/index/vector_index/VecIndex.h b/core/src/index/knowhere/knowhere/index/vector_index/VecIndex.h index d5e74f90ce69498ce119cbe2f0311a5eea26ea00..8f48831f6fc515ee7defff3acdf3eb5334b62f96 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/VecIndex.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/VecIndex.h @@ -27,7 +27,7 @@ namespace knowhere { #define INDEX_DATA "INDEX_DATA" #define RAW_DATA "RAW_DATA" -#define SQ8_DATA "SQ8_DATA" +#define QUANTIZATION_DATA "QUANTIZATION_DATA" class VecIndex : public Index { public: diff --git a/core/src/segment/SegmentReader.cpp b/core/src/segment/SegmentReader.cpp index 0c3a9c078a55a6db3b86bd25b39defe951e10924..b24c92602f1b7abd4794497a53ed2c9c67ddd26c 100644 --- a/core/src/segment/SegmentReader.cpp +++ b/core/src/segment/SegmentReader.cpp @@ -25,11 +25,11 @@ #include "codecs/Codec.h" #include "db/SnapshotUtils.h" #include "db/Types.h" +#include "db/Utils.h" #include "db/snapshot/ResourceHelper.h" #include "knowhere/index/vector_index/VecIndex.h" #include "knowhere/index/vector_index/VecIndexFactory.h" #include "knowhere/index/vector_index/adapter/VectorAdapter.h" -#include "knowhere/index/vector_index/helpers/IndexParameter.h" #include "storage/disk/DiskIOReader.h" #include "storage/disk/DiskIOWriter.h" #include "storage/disk/DiskOperation.h" @@ -342,8 +342,7 @@ SegmentReader::LoadVectorIndex(const std::string& field_name, knowhere::VecIndex // for some kinds index(IVF), read raw file auto index_type = index_visitor->GetElement()->GetTypeName(); - if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFFLAT || index_type == knowhere::IndexEnum::INDEX_NSG || - index_type == knowhere::IndexEnum::INDEX_HNSW) { + if (engine::utils::RequireRawFile(index_type)) { engine::BinaryDataPtr fixed_data; auto status = segment_ptr_->GetFixedFieldData(field_name, fixed_data); if (status.ok()) { @@ -355,9 +354,9 @@ SegmentReader::LoadVectorIndex(const std::string& field_name, knowhere::VecIndex } } - // for some kinds index(SQ8), read compress file - if (index_type == knowhere::IndexEnum::INDEX_RHNSWSQ) { - if (auto visitor = field_visitor->GetElementVisitor(engine::FieldElementType::FET_COMPRESS_SQ8)) { + // for some kinds index(RHNSWSQ), read compress file + if (engine::utils::RequireCompressFile(index_type)) { + if (auto visitor = field_visitor->GetElementVisitor(engine::FieldElementType::FET_COMPRESS)) { auto file_path = engine::snapshot::GetResPath(dir_collections_, visitor->GetFile()); ss_codec.GetVectorIndexFormat()->ReadCompress(fs_ptr_, file_path, compress_data); diff --git a/core/src/segment/SegmentWriter.cpp b/core/src/segment/SegmentWriter.cpp index 96f0a26cc1735f1525e050a2b25a068298ff4836..60c8848240b45019f3ac7cc10bde6e231585c800 100644 --- a/core/src/segment/SegmentWriter.cpp +++ b/core/src/segment/SegmentWriter.cpp @@ -401,7 +401,7 @@ SegmentWriter::WriteVectorIndex(const std::string& field_name) { // serialize compress file { - auto element_visitor = field->GetElementVisitor(engine::FieldElementType::FET_COMPRESS_SQ8); + auto element_visitor = field->GetElementVisitor(engine::FieldElementType::FET_COMPRESS); if (element_visitor && element_visitor->GetFile()) { auto segment_file = element_visitor->GetFile(); std::string file_path = diff --git a/core/src/server/delivery/request/DescribeIndexReq.cpp b/core/src/server/delivery/request/DescribeIndexReq.cpp index 522a814086232891a58ea59eb4ca710cb77a1961..429591a8be262559199242dc3e4a70b0671f4a84 100644 --- a/core/src/server/delivery/request/DescribeIndexReq.cpp +++ b/core/src/server/delivery/request/DescribeIndexReq.cpp @@ -64,7 +64,7 @@ DescribeIndexReq::OnExecute() { } } - json_params_[engine::PARAM_INDEX_TYPE] = index.index_name_; + json_params_[engine::PARAM_INDEX_TYPE] = index.index_type_; json_params_[engine::PARAM_INDEX_METRIC_TYPE] = index.metric_name_; json_params_[engine::PARAM_INDEX_EXTRA_PARAMS] = index.extra_params_; } catch (std::exception& ex) { diff --git a/sdk/examples/simple/src/ClientTest.cpp b/sdk/examples/simple/src/ClientTest.cpp index 0436816de2b27178e8fc3cfa28ebcffe21927e12..f9b67f51069b42aa4d9776890de17fc417b7637e 100644 --- a/sdk/examples/simple/src/ClientTest.cpp +++ b/sdk/examples/simple/src/ClientTest.cpp @@ -32,7 +32,7 @@ constexpr int64_t NQ = 5; constexpr int64_t TOP_K = 10; constexpr int64_t NPROBE = 32; constexpr int64_t SEARCH_TARGET = BATCH_ENTITY_COUNT / 2; // change this value, result is different -constexpr int64_t ADD_ENTITY_LOOP = 1; +constexpr int64_t ADD_ENTITY_LOOP = 10; constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFFLAT; constexpr int32_t NLIST = 16384; const char* PARTITION_TAG = "part"; diff --git a/tests/milvus_python_test/collection/test_create_collection.py b/tests/milvus_python_test/collection/test_create_collection.py index 429873411b2514759de41aa038638f2f2946f117..ab26131155ca43bf709818de02e49f16f3ff8ceb 100644 --- a/tests/milvus_python_test/collection/test_create_collection.py +++ b/tests/milvus_python_test/collection/test_create_collection.py @@ -13,7 +13,7 @@ from utils import * nb = 1 dim = 128 collection_id = "create_collection" -default_segment_row_count = 100000 +default_segment_row_count = 512 * 1024 drop_collection_interval_time = 3 segment_row_count = 5000 default_fields = gen_default_fields()