diff --git a/core/src/db/CMakeLists.txt b/core/src/db/CMakeLists.txt index 57acd0c66da2a66f4fa687e1bd2a495efe48bc72..7408bbe245e9fcc4bcb0c18b33178eeb5908eca6 100644 --- a/core/src/db/CMakeLists.txt +++ b/core/src/db/CMakeLists.txt @@ -122,6 +122,7 @@ endif () # **************************** Link Libraries with milvus engine **************************** target_link_libraries( milvus_engine PUBLIC knowhere + server segment cache codecs diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index fcd6c54f0d75b144fc927f78f528c19470fb79b5..f89137e998f448cc27e78b32e05dfb6b8db3922b 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -35,6 +35,7 @@ #include "scheduler/job/SearchJob.h" #include "segment/SegmentReader.h" #include "segment/SegmentWriter.h" +#include "server/ValidationUtil.h" #include "utils/Exception.h" #include "utils/StringHelpFunctions.h" #include "utils/TimeRecorder.h" @@ -312,6 +313,7 @@ DBImpl::HasPartition(const std::string& collection_name, const std::string& part CHECK_INITIALIZED; snapshot::ScopedSnapshotT ss; + STATUS_CHECK(server::ValidatePartitionTags({partition_tag})); STATUS_CHECK(snapshot::Snapshots::GetInstance().GetSnapshot(ss, collection_name)); auto partition_tags = std::move(ss->GetPartitionNames()); diff --git a/core/src/server/ValidationUtil.cpp b/core/src/server/ValidationUtil.cpp index 0870f444329e1824fb9616dfb37cae1c0f6af16b..6d3b91047d2a0bb79f44b056a7b9f4c5471a5380 100644 --- a/core/src/server/ValidationUtil.cpp +++ b/core/src/server/ValidationUtil.cpp @@ -116,7 +116,7 @@ ValidateCollectionName(const std::string& collection_name) { int64_t table_name_size = collection_name.size(); for (int64_t i = 1; i < table_name_size; ++i) { char name_char = collection_name[i]; - if (name_char != '_' && std::isalnum(name_char) == 0) { + if (name_char != '_' && name_char != '$' && std::isalnum(name_char) == 0) { std::string msg = invalid_msg + "Collection name can only contain numbers, letters, and underscores."; LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_COLLECTION_NAME, msg); @@ -424,6 +424,40 @@ ValidateSearchTopk(int64_t top_k) { Status ValidatePartitionTags(const std::vector& partition_tags) { for (const std::string& tag : partition_tags) { + // Partition nametag shouldn't be empty. + if (tag.empty()) { + std::string msg = "Partition tag should not be empty."; + LOG_SERVER_ERROR_ << msg; + return Status(SERVER_INVALID_PARTITION_TAG, msg); + } + + std::string invalid_msg = "Invalid partition tag: " + tag + ". "; + // Partition tag size shouldn't exceed 255. + if (tag.size() > engine::MAX_NAME_LENGTH) { + std::string msg = invalid_msg + "The length of a partition tag must be less than 255 characters."; + LOG_SERVER_ERROR_ << msg; + return Status(SERVER_INVALID_PARTITION_TAG, msg); + } + + // Partition tag first character should be underscore or character. + char first_char = tag[0]; + if (first_char != '_' && std::isalnum(first_char) == 0) { + std::string msg = invalid_msg + "The first character of a partition tag must be an underscore or letter."; + LOG_SERVER_ERROR_ << msg; + return Status(SERVER_INVALID_PARTITION_TAG, msg); + } + + int64_t tag_size = tag.size(); + for (int64_t i = 1; i < tag_size; ++i) { + char name_char = tag[i]; + if (name_char != '_' && name_char != '$' && std::isalnum(name_char) == 0) { + std::string msg = invalid_msg + "Partition tag can only contain numbers, letters, and underscores."; + LOG_SERVER_ERROR_ << msg; + return Status(SERVER_INVALID_PARTITION_TAG, msg); + } + } + +#if 0 // trim side-blank of tag, only compare valid characters // for example: " ab cd " is treated as "ab cd" std::string valid_tag = tag; @@ -441,18 +475,7 @@ ValidatePartitionTags(const std::vector& partition_tags) { LOG_SERVER_ERROR_ << msg; return Status(SERVER_INVALID_PARTITION_TAG, msg); } - } - - return Status::OK(); -} - -Status -ValidateInsertDataSize(const engine::DataChunkPtr& data) { - int64_t chunk_size = engine::utils::GetSizeOfChunk(data); - if (chunk_size > engine::MAX_INSERT_DATA_SIZE) { - std::string msg = "The amount of data inserted each time cannot exceed " + - std::to_string(engine::MAX_INSERT_DATA_SIZE / engine::MB) + " MB"; - return Status(SERVER_INVALID_ROWRECORD_ARRAY, msg); +#endif } return Status::OK(); diff --git a/tests/milvus_python_test/collection/test_collection_stats.py b/tests/milvus_python_test/collection/test_collection_stats.py index faa8295d1629dac548173a105de363bdce74f1e2..b6e14d56bf57a1f0d3e7182b5f8506eeebff7c1a 100644 --- a/tests/milvus_python_test/collection/test_collection_stats.py +++ b/tests/milvus_python_test/collection/test_collection_stats.py @@ -11,7 +11,7 @@ segment_row_count = 5000 nprobe = 1 top_k = 1 epsilon = 0.0001 -tag = "1970-01-01" +tag = "1970_01_01" nb = 6000 nlist = 1024 collection_id = "collection_stats" diff --git a/tests/milvus_python_test/entity/test_delete.py b/tests/milvus_python_test/entity/test_delete.py index 2b0bf382ad21a8b5d73e5f64944011b4850df535..13bb9b5faaa73ebd184f59009ca8aead93f944fc 100644 --- a/tests/milvus_python_test/entity/test_delete.py +++ b/tests/milvus_python_test/entity/test_delete.py @@ -13,7 +13,7 @@ dim = 128 segment_row_count = 5000 collection_id = "test_delete" DELETE_TIMEOUT = 60 -tag = "1970-01-01" +tag = "1970_01_01" nb = 6000 field_name = default_float_vec_field_name entity = gen_entities(1) diff --git a/tests/milvus_python_test/entity/test_get_entity_by_id.py b/tests/milvus_python_test/entity/test_get_entity_by_id.py index 753e53a172a4e0f0be773436995c702d45a79a26..3239117fb5cbea37001dea7e742641add48b82b5 100644 --- a/tests/milvus_python_test/entity/test_get_entity_by_id.py +++ b/tests/milvus_python_test/entity/test_get_entity_by_id.py @@ -14,7 +14,7 @@ dim = 128 segment_row_count = 5000 collection_id = "test_get" DELETE_TIMEOUT = 60 -tag = "1970-01-01" +tag = "1970_01_01" nb = 6000 entity = gen_entities(1) binary_entity = gen_binary_entities(1) diff --git a/tests/milvus_python_test/entity/test_insert.py b/tests/milvus_python_test/entity/test_insert.py index 682bc3c2ab6056a05ba1ceac222a3fb8e40251b6..6dd9189b6caae50bddca02943017326bb402b49b 100644 --- a/tests/milvus_python_test/entity/test_insert.py +++ b/tests/milvus_python_test/entity/test_insert.py @@ -12,7 +12,7 @@ dim = 128 segment_row_count = 5000 collection_id = "test_insert" ADD_TIMEOUT = 60 -tag = "1970-01-01" +tag = "1970_01_01" insert_interval_time = 1.5 nb = 6000 field_name = default_float_vec_field_name diff --git a/tests/milvus_python_test/entity/test_list_id_in_segment.py b/tests/milvus_python_test/entity/test_list_id_in_segment.py index 37778747e75f8723c6dc141ccd80f075cf53aa10..86902a72556d22ef7d2160dc5f2cc1efe6a54ce4 100644 --- a/tests/milvus_python_test/entity/test_list_id_in_segment.py +++ b/tests/milvus_python_test/entity/test_list_id_in_segment.py @@ -10,7 +10,7 @@ from utils import * dim = 128 segment_row_count = 100000 nb = 6000 -tag = "1970-01-01" +tag = "1970_01_01" field_name = default_float_vec_field_name binary_field_name = default_binary_vec_field_name collection_id = "list_id_in_segment" diff --git a/tests/milvus_python_test/entity/test_search.py b/tests/milvus_python_test/entity/test_search.py index 6f868d3906ba7f2108d25273d1521d4294c687ac..0a05215aa4f6fb92244a933a8e3a3ae1ea14eb17 100644 --- a/tests/milvus_python_test/entity/test_search.py +++ b/tests/milvus_python_test/entity/test_search.py @@ -14,7 +14,7 @@ dim = 128 segment_row_count = 5000 top_k_limit = 2048 collection_id = "search" -tag = "1970-01-01" +tag = "1970_01_01" insert_interval_time = 1.5 nb = 6000 top_k = 10 diff --git a/tests/milvus_python_test/stability/test_mysql.py b/tests/milvus_python_test/stability/test_mysql.py index b4c87ae126e490ae9d38d588af08b88d33522b9b..2c6a3f693e6af4948ea7e9be907ef48a160aab43 100644 --- a/tests/milvus_python_test/stability/test_mysql.py +++ b/tests/milvus_python_test/stability/test_mysql.py @@ -12,7 +12,7 @@ dim = 128 index_file_size = 10 collection_id = "mysql_failure" nprobe = 1 -tag = "1970-01-01" +tag = "1970_01_01" class TestMysql: diff --git a/tests/milvus_python_test/stability/test_restart.py b/tests/milvus_python_test/stability/test_restart.py index d002dc40b9a3ec3f016603404544ba47083e7592..9d274a0f6a5f2dbf2ce78932cf2d5184cee5d44d 100644 --- a/tests/milvus_python_test/stability/test_restart.py +++ b/tests/milvus_python_test/stability/test_restart.py @@ -13,7 +13,7 @@ dim = 128 index_file_size = 10 collection_id = "test_partition_restart" nprobe = 1 -tag = "1970-01-01" +tag = "1970_01_01" class TestRestartBase: diff --git a/tests/milvus_python_test/test_compact.py b/tests/milvus_python_test/test_compact.py index 5bb43c16e76927c04525e0f7c39d80b07804b816..5895601078b58b98518c244a0218b164fb2eb82c 100644 --- a/tests/milvus_python_test/test_compact.py +++ b/tests/milvus_python_test/test_compact.py @@ -11,7 +11,7 @@ index_file_size = 10 COMPACT_TIMEOUT = 180 nprobe = 1 top_k = 1 -tag = "1970-01-01" +tag = "1970_01_01" nb = 6000 nq = 2 segment_row_count = 5000 diff --git a/tests/milvus_python_test/test_config.py b/tests/milvus_python_test/test_config.py index 0dcc9b31113185d68a398a2ec1ac890c7bb78827..a0453e9caff78b1831a7c7f960f4f9c9d759fcdd 100644 --- a/tests/milvus_python_test/test_config.py +++ b/tests/milvus_python_test/test_config.py @@ -14,7 +14,7 @@ index_file_size = 10 CONFIG_TIMEOUT = 80 nprobe = 1 top_k = 1 -tag = "1970-01-01" +tag = "1970_01_01" nb = 6000 diff --git a/tests/milvus_python_test/test_flush.py b/tests/milvus_python_test/test_flush.py index a64682b64928ebc7cc025eadf8d7c9364364c448..35e15d17aa8e55b8c725228ad9d7d14fa57454f6 100644 --- a/tests/milvus_python_test/test_flush.py +++ b/tests/milvus_python_test/test_flush.py @@ -12,7 +12,7 @@ index_file_size = 10 collection_id = "test_flush" DELETE_TIMEOUT = 60 nprobe = 1 -tag = "1970-01-01" +tag = "1970_01_01" top_k = 1 nb = 6000 tag = "partition_tag" diff --git a/tests/milvus_python_test/test_index.py b/tests/milvus_python_test/test_index.py index 1c57ef7a505e8d401cfad2442c27637fb9412afc..8c33a359e75bc29c72fb19b8c0302c8635f4d16e 100644 --- a/tests/milvus_python_test/test_index.py +++ b/tests/milvus_python_test/test_index.py @@ -14,7 +14,7 @@ index_file_size = 10 BUILD_TIMEOUT = 300 nprobe = 1 top_k = 5 -tag = "1970-01-01" +tag = "1970_01_01" NLIST = 4046 INVALID_NLIST = 100000000 field_name = "float_vector" diff --git a/tests/milvus_python_test/test_partition.py b/tests/milvus_python_test/test_partition.py index 3bf685b93b78d887c320a9c2e588558d7ff7a253..81770147743d141aa98bf2b51a226022236b37b2 100644 --- a/tests/milvus_python_test/test_partition.py +++ b/tests/milvus_python_test/test_partition.py @@ -12,7 +12,7 @@ dim = 128 segment_row_count = 5000 collection_id = "partition" nprobe = 1 -tag = "1970-01-01" +tag = "1970_01_01" TIMEOUT = 120 nb = 6000 tag = "partition_tag" diff --git a/tests/milvus_python_test/test_wal.py b/tests/milvus_python_test/test_wal.py index 286b8128ecf97e6405564d57709150c9b02a3bf9..2d0086134aa3b1d7df13f46e4f58ad3cac2a21f4 100644 --- a/tests/milvus_python_test/test_wal.py +++ b/tests/milvus_python_test/test_wal.py @@ -10,7 +10,7 @@ dim = 128 collection_id = "test_wal" segment_row_count = 5000 WAL_TIMEOUT = 60 -tag = "1970-01-01" +tag = "1970_01_01" insert_interval_time = 1.5 nb = 6000 field_name = "float_vector"